COMPMID-3776: Indirect GEMM
Signed-off-by: Georgios Pinitas <georgios.pinitas@arm.com>
Change-Id: I51a1b0f098bc3a8c408c50c92221e4df3061e12c
Reviewed-on: https://review.mlplatform.org/c/ml/ComputeLibrary/+/4343
Tested-by: Arm Jenkins <bsgcomp@arm.com>
Reviewed-by: Sang-Hoon Park <sang-hoon.park@arm.com>
Reviewed-by: Michele Di Giorgio <michele.digiorgio@arm.com>
Comments-Addressed: Arm Jenkins <bsgcomp@arm.com>
diff --git a/src/core/NEON/NEKernels.h b/src/core/NEON/NEKernels.h
index 6756293..79c4bce 100644
--- a/src/core/NEON/NEKernels.h
+++ b/src/core/NEON/NEKernels.h
@@ -72,7 +72,6 @@
#include "src/core/NEON/kernels/NEFlattenLayerKernel.h"
#include "src/core/NEON/kernels/NEFloorKernel.h"
#include "src/core/NEON/kernels/NEFuseBatchNormalizationKernel.h"
-#include "src/core/NEON/kernels/NEGEMMAssemblyBaseKernel.h"
#include "src/core/NEON/kernels/NEGEMMInterleave4x4Kernel.h"
#include "src/core/NEON/kernels/NEGEMMLowpMatrixMultiplyKernel.h"
#include "src/core/NEON/kernels/NEGEMMLowpOffsetContributionKernel.h"
diff --git a/src/core/NEON/kernels/NEGEMMAssemblyBaseKernel.h b/src/core/NEON/kernels/NEGEMMAssemblyBaseKernel.h
deleted file mode 100644
index 775a2c0..0000000
--- a/src/core/NEON/kernels/NEGEMMAssemblyBaseKernel.h
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Copyright (c) 2017-2020 Arm Limited.
- *
- * SPDX-License-Identifier: MIT
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to
- * deal in the Software without restriction, including without limitation the
- * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
- * sell copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-#ifndef ARM_COMPUTE_NEGEMMASSEMBLYBASE_H
-#define ARM_COMPUTE_NEGEMMASSEMBLYBASE_H
-
-#include "src/core/NEON/INEKernel.h"
-
-namespace arm_compute
-{
-class ITensor;
-
-/** Base class for GEMM NEON kernels implemented in Assembly. */
-class NEGEMMAssemblyBaseKernel : public INEKernel
-{
-public:
- const char *name() const override
- {
- return "NEGEMMAssemblyBaseKernel";
- }
- /** Constructor */
- NEGEMMAssemblyBaseKernel()
- : _input0(nullptr), _input1(nullptr), _output(nullptr), _workspace(nullptr), _alpha(1.f), _beta(0.f), _is_transposed_0(false), _is_transposed_1(false)
- {
- }
-
- /** Prevent instances of this class from being copied (As this class contains pointers) */
- NEGEMMAssemblyBaseKernel(const NEGEMMAssemblyBaseKernel &) = delete;
- /** Prevent instances of this class from being copied (As this class contains pointers) */
- NEGEMMAssemblyBaseKernel &operator=(const NEGEMMAssemblyBaseKernel &) = delete;
- /** Allow instances of this class to be moved */
- NEGEMMAssemblyBaseKernel(NEGEMMAssemblyBaseKernel &&) = default;
- /** Allow instances of this class to be moved */
- NEGEMMAssemblyBaseKernel &operator=(NEGEMMAssemblyBaseKernel &&) = default;
-
- virtual ~NEGEMMAssemblyBaseKernel() = default;
-
- /** Initialise the kernel's input and output.
- *
- * The computed function is C = a * AxB + b * C.
- *
- * @param[in] input0 Input tensor containing the Matrix A. Data types supported: F32
- * @param[in] input1 Input tensor containing the Matrix B. Data types supported: same as @p input0
- * @param[in,out] output Output tensor to store the result of matrix multiplication. If @p beta is not zero the values are multiplied by @p beta before the result is accumulated. Otherwise the values are overwritten by the result. Data types supported: same as @p input0.
- * @param[out] workspace Space for intermediate results.
- * @param[in] alpha Weight of the matrix product
- * @param[in] beta Weight of the accumulation.
- * @param[in] is_transposed_0 (Optional)True if @p input0 is transposed else false. (Defaults to false)
- * @param[in] is_transposed_1 (Optional)True if @p input1 is transposed else false. (Defaults to false)
- */
- void configure(const ITensor *input0, const ITensor *input1, ITensor *output, ITensor *workspace, float alpha = 1.f, float beta = 0.f, bool is_transposed_0 = false, bool is_transposed_1 = false)
- {
- internal_configure(input0, input1, output, workspace, alpha, beta, is_transposed_0, is_transposed_1);
- }
-
-protected:
- virtual void internal_configure(const ITensor *input0, const ITensor *input1, ITensor *output, ITensor *workspace, float alpha, float beta, bool _is_transposed_0, bool _is_transposed_1) = 0;
-
- const ITensor *_input0;
- const ITensor *_input1;
- ITensor *_output;
- ITensor *_workspace;
- float _alpha;
- float _beta;
- bool _is_transposed_0;
- bool _is_transposed_1;
-};
-} // namespace arm_compute
-#endif /*ARM_COMPUTE_NEGEMMASSEMBLYBASE_H*/
diff --git a/src/core/NEON/kernels/arm_gemm/convolver.hpp b/src/core/NEON/kernels/arm_gemm/convolver.hpp
new file mode 100644
index 0000000..1cd9595
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/convolver.hpp
@@ -0,0 +1,182 @@
+/*
+ * Copyright (c) 2020 Arm Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#pragma once
+
+#include "convolution_parameters.hpp"
+
+#include <algorithm>
+#include <cstddef>
+#include <tuple>
+#include <vector>
+
+namespace arm_gemm {
+
+// Class to assist with convolution calculations.
+//
+// This is framed as a hierarchy of objects:
+//
+// - Top level object which depends only on convolution parameters. This sets up std::vectors for the padding and
+// kernel offset arrays. From this you can request:
+//
+// - Mid level object (e.g. instantiated at start of 'ConvolutionInterleave'). This holds specifics about the
+// input tensor, and the desired column range. Calculations specific to this can be done once when this is set
+// up. From this you can request:
+//
+// - Low level object (instantiated for each range of rows). This contains methods to actually populate a row
+// pointer array.
+
+
+template<typename T>
+class convolver {
+private:
+ const ConvolutionParameters m_params;
+
+ // Vector of padding data
+ const std::vector<T> m_pad_row;
+
+ // X/Y offsets for each kernel position
+ std::vector<int> m_kernel_y;
+ std::vector<int> m_kernel_x;
+
+ class column_handler {
+ private:
+ const convolver<T> &m_parent;
+
+ // Base/stride of input image
+ const T * const m_input_base;
+ const size_t m_input_stride;
+
+ // Starting kernel point and channel offset within that point
+ const unsigned int m_start_pos;
+ const unsigned int m_start_offset;
+
+ // Total length to process, rounded length of each input channel block.
+ const unsigned int m_length;
+ const unsigned int m_rounded_stringlen;
+
+ class row_handler {
+ private:
+ const convolver<T> &m_convolver;
+ const column_handler &m_parent;
+
+ // These variables track progress through the current block of rows
+ unsigned int m_start_output_y=0;
+ unsigned int m_start_output_x=0;
+
+ unsigned int m_length_remaining=0;
+ unsigned int m_current_pos=0;
+
+ unsigned int m_active_height=0;
+
+ public:
+ row_handler(const column_handler &parent, unsigned int start_row, unsigned int active_height) :
+ m_convolver(parent.m_parent),
+ m_parent(parent),
+ m_start_output_y(start_row / m_convolver.m_params.output_width),
+ m_start_output_x(start_row % m_convolver.m_params.output_width),
+ m_length_remaining(m_parent.m_length),
+ m_current_pos(m_parent.m_start_pos),
+ m_active_height(active_height) { }
+
+ bool finished() const {
+ return (m_length_remaining == 0);
+ }
+
+ std::tuple<unsigned int, unsigned int> next_block(const T ** const row_ptr) {
+ if (finished()) {
+ return { 0, 0 };
+ }
+
+ // "in_width" in the amount of data that will be read in (copied)
+ // "out_width" is the total amount of data that will be produced (including padding)
+ unsigned int offset = (m_current_pos == m_parent.m_start_pos) ? m_parent.m_start_offset : 0;
+ unsigned int in_width = std::min(m_length_remaining, static_cast<unsigned int>(m_convolver.m_params.input_channels) - offset);
+ unsigned int out_width = std::min(m_length_remaining, m_parent.m_rounded_stringlen - offset);
+
+ unsigned int output_y = m_start_output_y;
+ unsigned int output_x = m_start_output_x;
+
+ for (unsigned int row=0; row<m_active_height; row++) {
+ int input_y = (output_y * m_convolver.m_params.output_stride_h) + m_convolver.m_kernel_y[m_current_pos];
+ int input_x = (output_x * m_convolver.m_params.output_stride_w) + m_convolver.m_kernel_x[m_current_pos];
+
+ // Out-of-bounds points will read the padding data,
+ // otherwise find the correct address in the input image.
+ if (input_y < 0 || input_y >= m_convolver.m_params.input_height || input_x < 0 || input_x >= m_convolver.m_params.input_width) {
+ row_ptr[row] = m_convolver.m_pad_row.data();
+ } else {
+ row_ptr[row] = m_parent.m_input_base + ((input_y * m_convolver.m_params.input_width) + input_x) * m_parent.m_input_stride;
+ }
+
+ output_x++;
+ if (output_x == m_convolver.m_params.output_width) {
+ output_y++;
+ output_x=0;
+ }
+ }
+
+ m_current_pos++;
+ m_length_remaining-=out_width;
+
+ return { in_width, offset };
+ }
+ }; // end of "row handler" class
+
+ public:
+ column_handler(const convolver<T> &parent, const T *input_base, size_t input_stride,
+ unsigned int k_start, unsigned int k_end, unsigned int rounded_stringlen)
+ : m_parent(parent), m_input_base(input_base), m_input_stride(input_stride),
+ m_start_pos(k_start / rounded_stringlen),
+ m_start_offset(k_start % rounded_stringlen),
+ m_length(k_end - k_start),
+ m_rounded_stringlen(rounded_stringlen) { }
+
+ row_handler process_rows(unsigned int start_row, unsigned int active_height) const {
+ return row_handler(*this, start_row, active_height);
+ }
+ }; // end of "column handler" class
+
+public:
+ convolver(ConvolutionParameters params) :
+ m_params (params), m_pad_row(params.input_channels, static_cast<T>(params.padding_value)),
+ m_kernel_y(params.kernel_width * params.kernel_height, 0),
+ m_kernel_x(params.kernel_width * params.kernel_height, 0) {
+
+ // Kernel points are addressed across, then down (assumed weight layout is WHIO)
+ for (unsigned int ky=0; ky<params.kernel_height; ky++) {
+ for (unsigned int kx=0; kx<params.kernel_width; kx++) {
+ unsigned int n = (ky * params.kernel_width) + kx;
+ m_kernel_y[n] = ky - params.padding_top;
+ m_kernel_x[n] = kx - params.padding_left;
+ }
+ }
+ }
+
+ column_handler process_columns(const T *input_base, size_t input_stride,
+ unsigned int k_start, unsigned int k_end, unsigned int rounded_stringlen) const {
+ return column_handler(*this, input_base, input_stride, k_start, k_end, rounded_stringlen);
+ }
+};
+
+} // namespace arm_gemm
diff --git a/src/core/NEON/kernels/arm_gemm/gemm_bf16.cpp b/src/core/NEON/kernels/arm_gemm/gemm_bf16.cpp
index f3b6652..96b9734 100644
--- a/src/core/NEON/kernels/arm_gemm/gemm_bf16.cpp
+++ b/src/core/NEON/kernels/arm_gemm/gemm_bf16.cpp
@@ -25,93 +25,78 @@
#include "bfloat.hpp"
#include "gemm_common.hpp"
#include "gemm_hybrid.hpp"
+#include "gemm_hybrid_indirect.hpp"
#include "gemm_implementation.hpp"
#include "gemm_interleaved.hpp"
#include "gemv_batched.hpp"
#include "gemv_pretransposed.hpp"
-#include "kernels/a64_interleaved_bf16fp32_dot_12x8.hpp"
-#include "kernels/a64_interleaved_bf16fp32_mmla_12x8.hpp"
-#include "kernels/a64_sgemm_12x8.hpp"
+#include "kernels/a64_hybrid_bf16fp32_dot_6x16.hpp"
+#include "kernels/a64_interleaved_bf16fp32_dot_8x12.hpp"
+#include "kernels/a64_interleaved_bf16fp32_mmla_8x12.hpp"
+#include "kernels/a64_sgemm_8x12.hpp"
#include "kernels/a32_sgemm_8x6.hpp"
-#include "kernels/sve_interleaved_bf16fp32_dot_3VLx8.hpp"
-#include "kernels/sve_interleaved_bf16fp32_mmla_3VLx8.hpp"
-#include "kernels/sve_hybrid_bf16fp32_dot_4VLx4.hpp"
-#include "kernels/sve_hybrid_bf16fp32_mmla_4VLx4.hpp"
-#include "kernels/sve_hybrid_bf16fp32_mmla_6VLx2.hpp"
-#include "kernels/sve_hybrid_bf16fp32_mmla_8VLx2.hpp"
+#include "kernels/sve_interleaved_bf16fp32_dot_8x3VL.hpp"
+#include "kernels/sve_interleaved_bf16fp32_mmla_8x3VL.hpp"
+#include "kernels/sve_hybrid_bf16fp32_dot_6x4VL.hpp"
namespace arm_gemm {
static const GemmImplementation<bfloat16, float> gemm_bf16_methods[] =
{
#ifdef V8P6_BF
-# ifdef __ARM_FEATURE_SVE
-{
- GemmMethod::GEMM_HYBRID,
- "hybrid_bf16fp32_mmla_6VLx2",
- [](const GemmArgs &args) { return (args._Ksize>=8); },
- [](const GemmArgs &args) { return ((args._Msize <= 4) && (args._Nsize <= hybrid_bf16fp32_mmla_6VLx2::out_width())); },
- [](const GemmArgs &args) { return new GemmHybrid<hybrid_bf16fp32_mmla_6VLx2, bfloat16, float>(args); }
-},
-{
- GemmMethod::GEMM_HYBRID,
- "hybrid_bf16fp32_mmla_8VLx2",
- [](const GemmArgs &args) { return (args._Ksize>=8); },
- [](const GemmArgs &args) { return (args._Msize <= 4); },
- [](const GemmArgs &args) { return new GemmHybrid<hybrid_bf16fp32_mmla_8VLx2, bfloat16, float>(args); }
-},
-{
- GemmMethod::GEMM_HYBRID,
- "hybrid_bf16fp32_mmla_4VLx4",
- [](const GemmArgs &args) { return (args._Ksize>=8); },
- [](const GemmArgs &args) { return ((args._Ksize <= 128) && (args._Nsize <= 128)); },
- [](const GemmArgs &args) { return new GemmHybrid<hybrid_bf16fp32_mmla_4VLx4, bfloat16, float>(args); }
-},
-{
- GemmMethod::GEMM_HYBRID,
- "hybrid_bf16fp32_dot_4VLx4",
- [](const GemmArgs &args) { return (args._Ksize>=8); },
- [](const GemmArgs &args) { return ((args._Ksize <= 128) && (args._Nsize <= 128)); },
- [](const GemmArgs &args) { return new GemmHybrid<hybrid_bf16fp32_dot_4VLx4, bfloat16, float>(args); }
-},
+#ifdef __ARM_FEATURE_SVE
{ // gemm_bf16_interleaved
GemmMethod::GEMM_INTERLEAVED,
- "interleaved_bf16fp32_mmla_3VLx8",
+ "sve_interleaved_bf16fp32_mmla_8x3VL",
[](const GemmArgs &args) { return (args._Ksize>4); },
nullptr,
- [](const GemmArgs &args) { return new GemmInterleaved<interleaved_bf16fp32_mmla_3VLx8, bfloat16, float>(args); }
+ [](const GemmArgs &args) { return new GemmInterleaved<cls_sve_interleaved_bf16fp32_mmla_8x3VL, bfloat16, float>(args); }
+},
+{
+ GemmMethod::GEMM_HYBRID,
+ "sve_hybrid_bf16fp32_dot_6x4VL",
+ nullptr,
+ [](const GemmArgs &args) { return ((args._Ksize <= 128) && (args._Nsize <= 128)); },
+ [](const GemmArgs &args) { return new GemmHybridIndirect<cls_sve_hybrid_bf16fp32_dot_6x4VL, bfloat16, float>(args); }
},
{ // gemm_bf16_interleaved
GemmMethod::GEMM_INTERLEAVED,
- "interleaved_bf16fp32_dot_3VLx8",
+ "sve_interleaved_bf16fp32_dot_8x3VL",
[](const GemmArgs &args) { return (args._Ksize>2); },
nullptr,
- [](const GemmArgs &args) { return new GemmInterleaved<interleaved_bf16fp32_dot_3VLx8, bfloat16, float>(args); }
+ [](const GemmArgs &args) { return new GemmInterleaved<cls_sve_interleaved_bf16fp32_dot_8x3VL, bfloat16, float>(args); }
},
# endif // SVE
{ // gemm_bf16_interleaved
GemmMethod::GEMM_INTERLEAVED,
- "interleaved_bf16fp32_mmla_12x8",
+ "a64_interleaved_bf16fp32_mmla_8x12",
[](const GemmArgs &args) { return (args._Ksize>4); },
nullptr,
- [](const GemmArgs &args) { return new GemmInterleaved<interleaved_bf16fp32_mmla_12x8, bfloat16, float>(args); }
+ [](const GemmArgs &args) { return new GemmInterleaved<cls_a64_interleaved_bf16fp32_mmla_8x12, bfloat16, float>(args); }
+},
+{
+ GemmMethod::GEMM_HYBRID,
+ "a64_hybrid_bf16fp32_dot_6x16",
+ nullptr,
+ nullptr,
+ [](const GemmArgs &args) { return new GemmHybridIndirect<cls_a64_hybrid_bf16fp32_dot_6x16, bfloat16, float>(args); }
},
{ // gemm_bf16_interleaved
GemmMethod::GEMM_INTERLEAVED,
- "interleaved_bf16fp32_dot_12x8",
+ "a64_interleaved_bf16fp32_dot_8x12",
[](const GemmArgs &args) { return (args._Ksize>2); },
nullptr,
- [](const GemmArgs &args) { return new GemmInterleaved<interleaved_bf16fp32_dot_12x8, bfloat16, float>(args); }
+ [](const GemmArgs &args) { return new GemmInterleaved<cls_a64_interleaved_bf16fp32_dot_8x12, bfloat16, float>(args); }
},
#endif // V8P6_BF
#ifdef __aarch64__
{
GemmMethod::GEMM_INTERLEAVED,
- "sgemm_12x8",
+ "a64_sgemm_8x12",
nullptr,
nullptr,
- [](const GemmArgs &args) { return new GemmInterleaved<sgemm_12x8, bfloat16, float>(args); }
+ [](const GemmArgs &args) { return new GemmInterleaved<cls_a64_sgemm_8x12, bfloat16, float>(args); }
},
#elif defined(__arm__)
{
diff --git a/src/core/NEON/kernels/arm_gemm/gemm_fp16.cpp b/src/core/NEON/kernels/arm_gemm/gemm_fp16.cpp
index 9101221..de2e4f2 100644
--- a/src/core/NEON/kernels/arm_gemm/gemm_fp16.cpp
+++ b/src/core/NEON/kernels/arm_gemm/gemm_fp16.cpp
@@ -29,15 +29,17 @@
#include "gemm_common.hpp"
#include "gemm_hybrid.hpp"
+#include "gemm_hybrid_indirect.hpp"
#include "gemm_implementation.hpp"
#include "gemm_interleaved.hpp"
#include "gemm_interleaved_pretransposed_2d.hpp"
#include "kernels/a32_sgemm_8x6.hpp"
-#include "kernels/a64_hgemm_24x8.hpp"
-#include "kernels/a64_sgemm_12x8.hpp"
-#include "kernels/sve_hybrid_fp16_mla_4VLx4.hpp"
-#include "kernels/sve_interleaved_fp16_mla_3VLx8.hpp"
+#include "kernels/a64_hgemm_8x24.hpp"
+#include "kernels/a64_hybrid_fp16_mla_6x32.hpp"
+#include "kernels/a64_sgemm_8x12.hpp"
+#include "kernels/sve_hybrid_fp16_mla_6x4VL.hpp"
+#include "kernels/sve_interleaved_fp16_mla_8x3VL.hpp"
namespace arm_gemm {
@@ -45,61 +47,51 @@
#if defined(__ARM_FEATURE_SVE)
{
GemmMethod::GEMM_HYBRID,
- "hybrid_fp16_mla_4VLx4",
- [](const GemmArgs &args) { return (args._Ksize >= 8); },
+ "sve_hybrid_fp16_mla_6x4VL",
+ nullptr,
[](const GemmArgs &args) { return ((args._Ksize <= 256) && (args._Nsize <= 256)) || ((args._nmulti > 1) && ((args._Msize / args._maxthreads) < 8)); },
- [](const GemmArgs &args) { return new GemmHybrid<hybrid_fp16_mla_4VLx4, __fp16, __fp16>(args); }
+ [](const GemmArgs &args) { return new GemmHybridIndirect<cls_a64_hybrid_fp16_mla_6x32, __fp16, __fp16>(args); }
},
{
GemmMethod::GEMM_INTERLEAVED,
- "interleaved_fp16_mla_3VLx8",
+ "sve_interleaved_fp16_mla_8x3VL",
[](const GemmArgs &args) { return (args._Ksize > 4); },
nullptr,
- [](const GemmArgs &args) { return new GemmInterleaved<interleaved_fp16_mla_3VLx8, __fp16, __fp16>(args); }
+ [](const GemmArgs &args) { return new GemmInterleaved<cls_sve_interleaved_fp16_mla_8x3VL, __fp16, __fp16>(args); }
},
#endif
#if defined(__aarch64__) && (defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) || defined(FP16_KERNELS))
{
- GemmMethod::GEMM_INTERLEAVED_2D,
- "hgemm_24x8_2d",
+ GemmMethod::GEMM_HYBRID,
+ "a64_hybrid_fp16_mla_6x32",
#ifndef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
[](const GemmArgs &args) { return args._ci->has_fp16(); },
#else
nullptr,
#endif
- [](const GemmArgs &args) { return args._maxthreads >= 8; },
- [](const GemmArgs &args) { return new GemmInterleavedPretransposed2d<hgemm_24x8, __fp16, __fp16>(args); }
+ [](const GemmArgs &args) { return ((args._Ksize <= 256) && (args._Nsize <= 256)) || ((args._nmulti > 1) && ((args._Msize / args._maxthreads) < 8)); },
+ [](const GemmArgs &args) { return new GemmHybridIndirect<cls_a64_hybrid_fp16_mla_6x32, __fp16, __fp16>(args); }
},
{
GemmMethod::GEMM_INTERLEAVED,
- "hgemm_24x8_1d",
+ "a64_hgemm_8x24",
#ifndef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
[](const GemmArgs &args) { return args._ci->has_fp16(); },
#else
nullptr,
#endif
nullptr,
- [](const GemmArgs &args) { return new GemmInterleaved<hgemm_24x8, __fp16, __fp16>(args); }
+ [](const GemmArgs &args) { return new GemmInterleaved<cls_a64_hgemm_8x24, __fp16, __fp16>(args); }
},
-
#endif // aarch64 && FP16
#ifdef __aarch64__
-//Pretranpose, 2D split
-{
- GemmMethod::GEMM_INTERLEAVED_2D,
- "sgemm_12x8_2d",
- nullptr,
- [](const GemmArgs &args) { return args._maxthreads >= 8; },
- [](const GemmArgs &args) { return new GemmInterleavedPretransposed2d<sgemm_12x8, __fp16, __fp16>(args); }
-},
-//Tranpose, 1D split, with blockmanager
{
GemmMethod::GEMM_INTERLEAVED,
- "sgemm_12x8_1d",
+ "a64_sgemm_8x12",
nullptr,
nullptr,
- [](const GemmArgs &args) { return new GemmInterleaved<sgemm_12x8, __fp16, __fp16>(args); }
+ [](const GemmArgs &args) { return new GemmInterleaved<cls_a64_sgemm_8x12, __fp16, __fp16>(args); }
},
#elif defined(__arm__)
{
diff --git a/src/core/NEON/kernels/arm_gemm/gemm_fp32.cpp b/src/core/NEON/kernels/arm_gemm/gemm_fp32.cpp
index ddb438f..e9e335f 100644
--- a/src/core/NEON/kernels/arm_gemm/gemm_fp32.cpp
+++ b/src/core/NEON/kernels/arm_gemm/gemm_fp32.cpp
@@ -24,6 +24,7 @@
#include "arm_gemm.hpp"
#include "gemm_common.hpp"
#include "gemm_hybrid.hpp"
+#include "gemm_hybrid_indirect.hpp"
#include "gemm_implementation.hpp"
#include "gemm_interleaved.hpp"
#include "gemm_interleaved_pretransposed_2d.hpp"
@@ -31,127 +32,130 @@
#include "gemv_pretransposed.hpp"
#include "kernels/a32_sgemm_8x6.hpp"
-#include "kernels/a64_hybrid_fp32_mla_16x4.hpp"
-#include "kernels/a64_hybrid_fp32_mla_4x8.hpp"
-#include "kernels/a64_smallK_hybrid_fp32_mla_4x6.hpp"
-#include "kernels/a64_smallK_hybrid_fp32_mla_4x8.hpp"
-#include "kernels/a64_sgemm_12x8.hpp"
-#include "kernels/a64_sgemv_pretransposed.hpp"
+#include "kernels/a64_gemv_fp32_mla_32.hpp"
+#include "kernels/a64_hybrid_fp32_mla_6x16.hpp"
+#include "kernels/a64_hybrid_fp32_mla_8x4.hpp"
+#include "kernels/a64_sgemm_8x12.hpp"
+#include "kernels/a64_smallK_hybrid_fp32_mla_6x4.hpp"
+#include "kernels/a64_smallK_hybrid_fp32_mla_8x4.hpp"
-#include "kernels/sve_hybrid_fp32_mla_4VLx4.hpp"
-#include "kernels/sve_hybrid_fp32_mmla_4VLx4.hpp"
-#include "kernels/sve_interleaved_fp32_mla_3VLx8.hpp"
-#include "kernels/sve_interleaved_fp32_mmla_3VLx8.hpp"
-#include "kernels/sve_smallK_hybrid_fp32_mla_1VLx8.hpp"
+#include "kernels/sve_gemv_fp32_mla_8VL.hpp"
+#include "kernels/sve_hybrid_fp32_mla_6x4VL.hpp"
+#include "kernels/sve_hybrid_fp32_mla_8x1VL.hpp"
+#include "kernels/sve_interleaved_fp32_mla_8x3VL.hpp"
+#include "kernels/sve_interleaved_fp32_mmla_8x3VL.hpp"
+#include "kernels/sve_smallK_hybrid_fp32_mla_8x1VL.hpp"
namespace arm_gemm {
static const GemmImplementation<float, float> gemm_fp32_methods[] =
{
+// GEMV cases - starting with 'gemv_batched' wrapper to turn batched GEMV into GEMM.
{
GemmMethod::GEMV_BATCHED,
"gemv_batched",
- [](const GemmArgs &args) { return (args._Msize==1) && (args._nbatches>1); },
+ [](const GemmArgs &args) { return args._Msize==1 && args._nbatches>1 && !args._indirect_input; },
nullptr,
[](const GemmArgs &args) { return new GemvBatched<float, float>(args); }
},
#ifdef __aarch64__
-{
- GemmMethod::GEMV_PRETRANSPOSED,
- "sgemv_pretransposed",
- [](const GemmArgs &args) { return (args._Msize==1 && args._nbatches==1); },
- nullptr,
- [](const GemmArgs &args) { return new GemvPretransposed<sgemv_pretransposed, float, float>(args); }
-},
-#if defined(__ARM_FEATURE_SVE) && defined(MMLA_FP32)
+#ifdef __ARM_FEATURE_SVE
{
GemmMethod::GEMM_HYBRID,
- "hybrid_fp32_mmla_4VLx4",
- [](const GemmArgs &args) { return (args._Ksize >= 4); },
- [](const GemmArgs &args) { return ((args._Ksize <= 256) && (args._Nsize <= 256)) || ((args._nmulti > 1) && ((args._Msize / args._maxthreads) < 8)); },
- [](const GemmArgs &args) { return new GemmHybrid<hybrid_fp32_mmla_4VLx4, float, float>(args); }
+ "sve_gemv_fp32_mla_8VL",
+ [](const GemmArgs &args) { return args._Msize==1 && args._nbatches==1 && !args._indirect_input; },
+ nullptr,
+ [](const GemmArgs &args) { return new GemvPretransposed<cls_sve_gemv_fp32_mla_8VL, float, float>(args); }
},
+#endif
+{
+ GemmMethod::GEMM_HYBRID,
+ "a64_gemv_fp32_mla_32",
+ [](const GemmArgs &args) { return args._Msize==1 && args._nbatches==1 && !args._indirect_input; },
+ nullptr,
+ [](const GemmArgs &args) { return new GemvPretransposed<cls_a64_gemv_fp32_mla_32, float, float>(args); }
+},
+
+// MMLA next due to higher throughput (SVE only)
+#if defined(__ARM_FEATURE_SVE) && defined(MMLA_FP32)
{
GemmMethod::GEMM_INTERLEAVED,
- "interleaved_fp32_mmla_3VLx8",
+ "sve_interleaved_fp32_mmla_8x3VL",
[](const GemmArgs &args) { return (args._Ksize>4); },
nullptr,
- [](const GemmArgs &args) { return new GemmInterleaved<interleaved_fp32_mmla_3VLx8, float, float>(args); }
+ [](const GemmArgs &args) { return new GemmInterleaved<cls_sve_interleaved_fp32_mmla_8x3VL, float, float>(args); }
},
#endif // __ARM_FEATURE_SVE && MMLA_FP32
#ifdef __ARM_FEATURE_SVE
-// SVE smallk / hybrid methods
+// SVE smallk / hybrid methods
{
GemmMethod::GEMM_HYBRID,
- "smallK_hybrid_fp32_mla_1VLx8",
- [](const GemmArgs &args) { return (args._Ksize <= 24); },
+ "sve_smallK_hybrid_fp32_mla_8x1VL",
+ [](const GemmArgs &args) { return args._Ksize <= 24 && !args._indirect_input; },
nullptr,
- [](const GemmArgs &args) { return new GemmHybrid<smallK_hybrid_fp32_mla_1VLx8, float, float>(args); }
+ [](const GemmArgs &args) { return new GemmHybrid<cls_sve_smallK_hybrid_fp32_mla_8x1VL, float, float>(args); }
},
{
GemmMethod::GEMM_HYBRID,
- "hybrid_fp32_mla_4VLx4",
- [](const GemmArgs &args) { return (args._Ksize >= 4); },
+ "sve_hybrid_fp32_mla_8x1VL",
+ nullptr,
+ [](const GemmArgs &args) { return (args._Nsize < 12); },
+ [](const GemmArgs &args) { return new GemmHybridIndirect<cls_sve_hybrid_fp32_mla_8x1VL, float, float>(args); }
+},
+{
+ GemmMethod::GEMM_HYBRID,
+ "sve_hybrid_fp32_mla_6x4VL",
+ nullptr,
[](const GemmArgs &args) { return ((args._Ksize <= 256) && (args._Nsize <= 256)) || ((args._nmulti > 1) && ((args._Msize / args._maxthreads) < 8)); },
- [](const GemmArgs &args) { return new GemmHybrid<hybrid_fp32_mla_4VLx4, float, float>(args); }
+ [](const GemmArgs &args) { return new GemmHybridIndirect<cls_sve_hybrid_fp32_mla_6x4VL, float, float>(args); }
},
#endif // __ARM_FEATURE_SVE
// NEON hybrid methods
{
GemmMethod::GEMM_HYBRID,
- "smallK_hybrid_fp32_mla_4x8",
- [](const GemmArgs &args) { return (args._Ksize <= 8) && (args._Nsize % 4)==0; },
+ "a64_smallK_hybrid_fp32_mla_8x4",
+ [](const GemmArgs &args) { return args._Ksize <= 8 && (args._Nsize % 4)==0 && !args._indirect_input; },
nullptr,
- [](const GemmArgs &args) { return new GemmHybrid<smallK_hybrid_fp32_mla_4x8, float, float>(args); }
+ [](const GemmArgs &args) { return new GemmHybrid<cls_a64_smallK_hybrid_fp32_mla_8x4, float, float>(args); }
},
{
GemmMethod::GEMM_HYBRID,
- "smallK_hybrid_fp32_mla_4x6",
- [](const GemmArgs &args) { return (args._Ksize > 8) && (args._Ksize <= 16) && (args._Nsize % 4)==0; },
+ "a64_smallK_hybrid_fp32_mla_6x4",
+ [](const GemmArgs &args) { return (args._Ksize > 8 && args._Ksize <= 16) && (args._Nsize % 4)==0 && !args._indirect_input; },
nullptr,
- [](const GemmArgs &args) { return new GemmHybrid<smallK_hybrid_fp32_mla_4x6, float, float>(args); }
+ [](const GemmArgs &args) { return new GemmHybrid<cls_a64_smallK_hybrid_fp32_mla_6x4, float, float>(args); }
},
{
GemmMethod::GEMM_HYBRID,
- "hybrid_fp32_mla_4x8_normal",
- [](const GemmArgs &args) { return (args._Ksize >= 4); },
+ "a64_hybrid_fp32_mla_8x4",
+ nullptr,
[](const GemmArgs &args) { return (args._Nsize < 12); },
- [](const GemmArgs &args) { return new GemmHybrid<hybrid_fp32_mla_4x8, float, float>(args); }
+ [](const GemmArgs &args) { return new GemmHybridIndirect<cls_a64_hybrid_fp32_mla_8x4, float, float>(args); }
},
GemmImplementation<float, float>::with_estimate(
GemmMethod::GEMM_HYBRID,
- "hybrid_fp32_mla_16x4",
- [](const GemmArgs &args) { return (args._Ksize >= 4); },
- [](const GemmArgs &args) { return GemmHybrid<hybrid_fp32_mla_16x4, float, float>::estimate_cycles(args, hybrid_fp32_mla_16x4::get_performance_parameters(args._ci)); },
- [](const GemmArgs &args) { return new GemmHybrid<hybrid_fp32_mla_16x4, float, float>(args); }
+ "a64_hybrid_fp32_mla_6x16",
+ nullptr,
+ [](const GemmArgs &args) { return GemmHybridIndirect<cls_a64_hybrid_fp32_mla_6x16, float, float>::estimate_cycles(args, cls_a64_hybrid_fp32_mla_6x16::get_performance_parameters(args._ci)); },
+ [](const GemmArgs &args) { return new GemmHybridIndirect<cls_a64_hybrid_fp32_mla_6x16, float, float>(args); }
),
-
#ifdef __ARM_FEATURE_SVE
{
GemmMethod::GEMM_INTERLEAVED,
- "interleaved_fp32_mla_3VLx8",
+ "sve_interleaved_fp32_mla_8x3VL",
[](const GemmArgs &args) { return (args._Ksize>4); },
nullptr,
- [](const GemmArgs &args) { return new GemmInterleaved<interleaved_fp32_mla_3VLx8, float, float>(args); }
+ [](const GemmArgs &args) { return new GemmInterleaved<cls_sve_interleaved_fp32_mla_8x3VL, float, float>(args); }
},
#endif // __ARM_FEATURE_SVE
-// Pretranposed, 2D split
-GemmImplementation<float, float>::with_estimate(
- GemmMethod::GEMM_INTERLEAVED_2D,
- "sgemm_12x8_2d",
- nullptr,
- [](const GemmArgs &args) { return GemmInterleavedPretransposed2d<sgemm_12x8, float, float>::estimate_cycles(args, sgemm_12x8::get_performance_parameters(args._ci)); },
- [](const GemmArgs &args) { return new GemmInterleavedPretransposed2d<sgemm_12x8, float, float>(args); }
-),
-// 1D split (with pretransposed or not)
GemmImplementation<float, float>::with_estimate(
GemmMethod::GEMM_INTERLEAVED,
- "sgemm_12x8_1d",
+ "a64_sgemm_8x12",
nullptr,
- [](const GemmArgs &args) { return GemmInterleaved<sgemm_12x8, float, float>::estimate_cycles(args, sgemm_12x8::get_performance_parameters(args._ci)); },
- [](const GemmArgs &args) { return new GemmInterleaved<sgemm_12x8, float, float>(args); }
+ [](const GemmArgs &args) { return GemmInterleaved<cls_a64_sgemm_8x12, float, float>::estimate_cycles(args, cls_a64_sgemm_8x12::get_performance_parameters(args._ci)); },
+ [](const GemmArgs &args) { return new GemmInterleaved<cls_a64_sgemm_8x12, float, float>(args); }
),
#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_gemm/gemm_hybrid.hpp b/src/core/NEON/kernels/arm_gemm/gemm_hybrid.hpp
index 7a983ed..d702cff 100644
--- a/src/core/NEON/kernels/arm_gemm/gemm_hybrid.hpp
+++ b/src/core/NEON/kernels/arm_gemm/gemm_hybrid.hpp
@@ -77,51 +77,43 @@
return args._cfg->inner_block_size;
}
- const unsigned int L1_size = args._ci->get_L1_cache_size();
+ // Target block size (512 for FP32, scaling for other types). Don't block until size reaches 1.5X this.
+ unsigned int target_block_size = 2048 / sizeof(To);
- // k_block: Find out how much of the larger array can be loaded into half the cache.
- // This should account for associative caches.
- unsigned int k_block = (L1_size / 2) / (sizeof(Toi) * (std::max(strategy::out_width(), strategy::out_height())));
+ if (args._Ksize >= ((3 * target_block_size) / 2)) {
+ unsigned int target_blocks = iceildiv(args._Ksize, target_block_size);
- // Needs to be (at least a single) multiple of the K unroll level.
- k_block /= strategy::k_unroll();
- k_block = std::max(k_block, 1U) * strategy::k_unroll();
+ unsigned int block_size = iceildiv(args._Ksize, target_blocks);
- // Now tune to presented problem size; this is how many blocks we need.
- unsigned int numk_blocks = iceildiv(args._Ksize, k_block);
+ block_size = roundup(block_size, strategy::k_unroll());
- // So divide the space equally into that many blocks.
- k_block = iceildiv(args._Ksize, numk_blocks);
+ return block_size;
+ }
- // And round UP to the K unroll level required.
- k_block = roundup(k_block, strategy::k_unroll());
-
- return k_block;
+ return args._Ksize;
}
+ // New N blocking strategy: if it's narrow, or much taller than it is wide, do the full width. Otherwise do a
+ // single block.
static unsigned int compute_n_block(const GemmArgs &args) {
if (args._cfg && args._cfg->outer_block_size) {
return args._cfg->outer_block_size;
}
- const unsigned int k_block = compute_k_block(args);
- const unsigned int L2_size = args._ci->get_L2_cache_size();
+ if (args._Nsize <= 64) {
+ return args._Nsize;
+ }
- // n_block: Work out how many rows (of length k_block) will fit in the L2
- // Don't allocate more than 90% of the L2 to allow for overheads, and subtract off the L1 contents.
- unsigned int n_block = (((L2_size * 9) / 10) - (k_block * sizeof(Toi) * (strategy::out_width() + strategy::out_height()))) /
- (sizeof(Toi) * k_block);
+ if ((args._Msize / args._Nsize) > 155) {
+ return args._Nsize;
+ }
- // Needs to be (at least a single) multiple of the kernel output width.
- n_block /= strategy::out_width();
- n_block = std::max(n_block, 1U) * strategy::out_width();
+ // Go slightly wider if thread count and depth are small.
+ if ((args._Ksize <= 128) && (args._maxthreads <= 16)) {
+ return strategy::out_width() * 3;
+ }
- // And tune to the presented problem size.
- unsigned int numblocks = iceildiv(args._Nsize, n_block);
- n_block = iceildiv(args._Nsize, numblocks);
- n_block = roundup(n_block, strategy::out_width());
-
- return n_block;
+ return strategy::out_width();
}
public:
diff --git a/src/core/NEON/kernels/arm_gemm/gemm_hybrid_indirect.hpp b/src/core/NEON/kernels/arm_gemm/gemm_hybrid_indirect.hpp
new file mode 100644
index 0000000..eede1a4
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/gemm_hybrid_indirect.hpp
@@ -0,0 +1,621 @@
+/*
+ * Copyright (c) 2017-2020 Arm Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#pragma once
+
+#include <alloca.h>
+
+#include <algorithm>
+#include <cassert>
+
+#include "arm_gemm.hpp"
+#include "bias_adder.hpp"
+#include "convolver.hpp"
+#include "ndrange.hpp"
+#include "performance_parameters.hpp"
+#include "transform.hpp"
+#include "utils.hpp"
+
+#ifdef CYCLE_PROFILING
+#include "profiler.hpp"
+#endif
+
+#ifndef UNUSED
+#define __I_DEFINED_UNUSED
+#define UNUSED(x) ((void)(x))
+#endif
+
+namespace arm_gemm {
+
+namespace {
+
+// We need to invoke the kernel differently for quantizing and non-quantizing cases, so here is a shim class to do
+// that.
+
+template<typename OutputStage, bool SeparateQuantize = false>
+class run_hybrid_kernel {
+public:
+ template<typename strategy, typename To, typename Tr>
+ static void run (
+#ifdef CYCLE_PROFILING
+ profiler &prof,
+#endif
+ const strategy &strat, unsigned int num_strings, const unsigned int *string_ptr, IndirectInputArg<To> A_arg, unsigned int M, unsigned int N,
+ unsigned int kern_k, const To *b_ptr, IndirectOutputArg<Tr> output_arg, const Tr *bias_ptr, Activation act, bool accumulate,
+ const OutputStage &os, const int32_t *col_bias, unsigned int n_0 );
+};
+
+template<>
+template<typename strategy, typename To, typename Tr>
+void run_hybrid_kernel<Nothing, false>::run(
+#ifdef CYCLE_PROFILING
+ profiler &prof,
+#endif
+ const strategy &strat, unsigned int num_strings, const unsigned int *string_ptr, IndirectInputArg<To> A_arg, unsigned int M, unsigned int N,
+ unsigned int kern_k, const To *b_ptr, IndirectOutputArg<Tr> output_arg, const Tr *bias_ptr, Activation act, bool accumulate,
+ const Nothing &, const int32_t *, unsigned int) {
+#ifdef CYCLE_PROFILING
+ auto p = prof.ScopedProfiler(PROFILE_KERNEL, (unsigned long)M * kern_k * roundup(N, strategy::out_width()));
+#endif
+ UNUSED(kern_k);
+
+ strat.kernel(num_strings, string_ptr, A_arg, M, N, b_ptr, output_arg, bias_ptr, act, accumulate);
+}
+
+template<>
+template<typename strategy, typename To, typename Tr>
+void run_hybrid_kernel<Requantize32, false>::run(
+#ifdef CYCLE_PROFILING
+ profiler &prof,
+#endif
+ const strategy &strat, unsigned int num_strings, const unsigned int *string_ptr, IndirectInputArg<To> A_arg, unsigned int M, unsigned int N,
+ unsigned int kern_k, const To *b_ptr, IndirectOutputArg<Tr> output_arg, const Tr *, Activation, bool,
+ const Requantize32 &os, const int32_t *col_bias, unsigned int n_0 ) {
+#ifdef CYCLE_PROFILING
+ auto p = prof.ScopedProfiler(PROFILE_KERNEL, (unsigned long)M * kern_k * roundup(N, strategy::out_width()));
+#endif
+ UNUSED(kern_k);
+
+ strat.kernel(num_strings, string_ptr, A_arg, M, N, b_ptr, output_arg, &os, col_bias + n_0, n_0);
+}
+
+template<>
+template<typename strategy, typename To, typename Tr>
+void run_hybrid_kernel<Requantize32, true>::run(
+#ifdef CYCLE_PROFILING
+ profiler &prof,
+#endif
+ const strategy &strat, unsigned int num_strings, const unsigned int *string_ptr, IndirectInputArg<To> A_arg, unsigned int M, unsigned int N,
+ unsigned int kern_k, const To *b_ptr, IndirectOutputArg<Tr> output_arg, const Tr *, Activation, bool,
+ const Requantize32 &os, const int32_t *col_bias, unsigned int n_0 ) {
+ UNUSED(kern_k);
+ // On this route we will only process one kernel height at a time and will make sure this happens in the driver loop.
+ assert(M <= strategy::out_height());
+ // We don't yet support indirect output (as the quantizer can't do it).
+ assert(output_arg.is_indirect == false);
+
+ // We need a row sum buffer and intermediate output buffer.
+ // These go on the stack as they are not too large, using an automatic array and alloca() respectively.
+ int32_t row_sums[strategy::out_height()];
+ typename strategy::result_type *result_buffer;
+
+ unsigned int output_width = roundup(N, strategy::out_width());
+
+ result_buffer = reinterpret_cast<typename strategy::result_type *>(alloca(output_width * strategy::out_height() * sizeof(typename strategy::result_type)));
+
+ {
+#ifdef CYCLE_PROFILING
+ auto p = prof.ScopedProfiler(PROFILE_KERNEL, (unsigned long)M * kern_k * roundup(N, strategy::out_width()));
+#endif
+ // Perform the GEMM, into the output buffer.
+ strat.kernel(num_strings, string_ptr, A_arg, M, N, b_ptr, IndirectOutputArg<typename strategy::result_type>(result_buffer, output_width), nullptr, Activation(), false);
+ }
+
+ if (os.b_offset != 0) {
+#ifdef CYCLE_PROFILING
+ auto p = prof.ScopedProfiler(PROFILE_ROWSUMS, (unsigned long)M * kern_k);
+#endif
+ row_sums_indirect(num_strings, string_ptr, A_arg, M, row_sums, &os);
+ } else {
+ memset(row_sums, 0, sizeof(int32_t) * strategy::out_height());
+ }
+
+ {
+#ifdef CYCLE_PROFILING
+ auto p = prof.ScopedProfiler(PROFILE_QUANTIZE, (unsigned long)M * N);
+#endif
+ // Quantize
+ requantize_block_32(os, N, M, result_buffer, output_width, output_arg.direct.base, output_arg.direct.stride, row_sums, col_bias + n_0, n_0);
+ }
+}
+
+} // anonymous namespace
+
+// Implementation of the GemmCommon abstract class.
+template<typename strategy, typename To, typename Tr, typename OutputStage = Nothing, bool SeparateQuantize = false>
+class GemmHybridIndirect : public GemmCommon<To, Tr> {
+ typedef typename strategy::operand_type Toi;
+ typedef typename strategy::result_type Tri;
+
+ GemmArgs _args;
+ OutputStage _os = {};
+
+ /* Quantized support (in addition to 'output stage' above) */
+ int32_t *_col_bias = nullptr;
+
+ const unsigned int _Ktotal;
+ const unsigned int _rounded_Ksize;
+
+ /* Blocking info */
+ const unsigned int _k_block;
+ const unsigned int _n_block;
+ const unsigned int _Mround;
+
+ /* Pretransposed buffer. */
+ const Toi *_B_transposed=nullptr;
+
+ /* Indirect parameters. _indirect_buf doubles as a flag to indicate that "indirect" transform should be used. */
+ const To * const * const * _indirect_buf = nullptr;
+
+ /* Convolver - only set up for convolution problems, so also doubles as a flag. */
+ std::unique_ptr<convolver<To>> _convolver = nullptr;
+
+ // Array of pointers to output rows
+// Tr * const * _output_ptrs;
+
+ const NDRange<4> _window_range;
+
+ unsigned int get_col_sum_size() const {
+ if (std::is_same<OutputStage, Requantize32>::value) {
+ return _args._Nsize * _args._nmulti * sizeof(int32_t);
+ } else {
+ return 0;
+ }
+ }
+
+ static unsigned int get_ktotal(const GemmArgs &args) {
+ return args._Ksections * roundup(args._Ksize, strategy::k_unroll());
+ }
+
+ static unsigned int compute_k_block(const GemmArgs &args) {
+ // Some kernels don't support accumulate mode - these can't do K blocking at all.
+ if (!strategy::supports_accumulate() || std::is_same<OutputStage, Requantize32>::value) {
+ return get_ktotal(args);
+ }
+
+ if (args._cfg && args._cfg->inner_block_size) {
+ return args._cfg->inner_block_size;
+ }
+
+ // Experimental data suggests an optimal block size of 512 for FP32 (scaling accordingly for other
+ // datatypes); but don't divide into blocks until we hit 1.5X this size.
+ unsigned int target_block_size = 2048 / sizeof(To);
+ auto ktotal = get_ktotal(args);
+
+ if (ktotal > ((target_block_size*3)/2)) {
+ unsigned int target_blocks = iceildiv(ktotal, target_block_size);
+
+ unsigned int block_size = iceildiv(ktotal, target_blocks);
+
+ block_size = roundup(block_size, strategy::k_unroll());
+
+ return block_size;
+ }
+
+ return ktotal;
+ }
+
+ // New N blocking strategy: if it's narrow, or much taller than it is wide, do the full width. Otherwise do a
+ // single block.
+ static unsigned int compute_n_block(const GemmArgs &args, const OutputStage os = {}) {
+ if (args._cfg && args._cfg->outer_block_size) {
+ return args._cfg->outer_block_size;
+ }
+
+ if (args._Nsize <= 64) {
+ return args._Nsize;
+ }
+
+ if ((args._Msize / args._Nsize) > 155) {
+ return args._Nsize;
+ }
+
+ // "Asymmetric" quantizing GEMMs require a different approach - the tall skinny blocks we would otherwise
+ // use imply a great deal of repeated work performing the row sums. If row sums are involved, work out how
+ // much "column" parallelism is going to be required and set the block size accordingly.
+ if (std::is_same<OutputStage, Requantize32>::value) {
+ const Requantize32 *qp = reinterpret_cast<const Requantize32 *>(&os);
+
+ // Row sums only needed if b_offset isn't 0
+ if (qp->b_offset != 0) {
+ // We can already parallelize across batches, multis and rows (in units of 'out_height')
+ int multi_row_parallelism = args._nmulti * args._nbatches * iceildiv(args._Msize, strategy::out_height());
+
+ // If this isn't enough, we will need to split up the columns too.
+ if (multi_row_parallelism < args._maxthreads) {
+ unsigned int columns_needed = iceildiv(args._maxthreads, multi_row_parallelism);
+
+ unsigned int n_block = iceildiv(args._Nsize, columns_needed);
+
+ return roundup(n_block, strategy::out_width());
+ }
+
+ // Multi/Batch/Row parallelism is enough - don't split up the columns.
+ return args._Nsize;
+ }
+ }
+
+ if (args._Ksize <= 128 && args._maxthreads <= 16) {
+ return strategy::out_width() * 3;
+ }
+
+ return strategy::out_width();
+ }
+
+public:
+ GemmHybridIndirect(GemmHybridIndirect &) = delete;
+ GemmHybridIndirect & operator= (GemmHybridIndirect &) = delete;
+
+ /* Constructor */
+ GemmHybridIndirect(const GemmArgs &args, const OutputStage &os)
+ : _args(args), _os(os), _Ktotal(get_ktotal(args)),
+ _rounded_Ksize(roundup(args._Ksize, strategy::k_unroll())),
+ _k_block(compute_k_block(args)), _n_block(compute_n_block(args, os)),
+ _Mround(roundup(args._Msize, strategy::out_height())),
+ _window_range(iceildiv(args._Msize, strategy::out_height()), args._nbatches,
+ iceildiv(args._Nsize, _n_block), args._nmulti)
+ {
+ // We take a copy of the arguments (not a pointer or reference), but there is no lifetime requirement on the
+ // GemmConfig. Clear out the pointer to avoid accidents.
+ _args._cfg = nullptr;
+ }
+
+ /* Constructor without OutputStage */
+ GemmHybridIndirect(const GemmArgs &args)
+ : _args(args), _Ktotal(get_ktotal(args)),
+ _rounded_Ksize(roundup(args._Ksize, strategy::k_unroll())),
+ _k_block(compute_k_block(args)), _n_block(compute_n_block(args)),
+ _Mround(roundup(args._Msize, strategy::out_height())),
+ _window_range(iceildiv(args._Msize, strategy::out_height()), args._nbatches,
+ iceildiv(args._Nsize, _n_block), args._nmulti)
+ {
+ // We take a copy of the arguments (not a pointer or reference), but there is no lifetime requirement on the
+ // GemmConfig. Clear out the pointer to avoid accidents.
+ _args._cfg = nullptr;
+ }
+
+ // Interface implementation - Compulsory functions
+ ndrange_t get_window_size() const override {
+ return { _window_range.total_size() };
+ }
+
+ // This kernel can always be dynamically scheduled.
+ bool supports_dynamic_scheduling() const override {
+ return true;
+ }
+
+ // Execute
+ void execute(const ndcoord_t &work_range, const ndcoord_t &, int) override {
+#ifdef CYCLE_PROFILING
+ profiler prof;
+#endif
+ strategy strat(_args._ci);
+
+ std::vector<const To *> in_row_ptrs;
+ std::vector<const To * const *> in_row_strings;
+ std::vector<unsigned int> string_lengths;
+
+ // In convolution mode, we need input pointers.
+ if (_convolver) {
+ in_row_ptrs = std::vector<const To *>(strategy::out_height() * _args._Ksections, nullptr);
+ in_row_strings = std::vector<const To * const *>(_args._Ksections, nullptr);
+
+ for (unsigned int i=0; i<_args._Ksections; i++) {
+ in_row_strings[i] = &(in_row_ptrs[i * strategy::out_height()]);
+ }
+ }
+
+ // In any indirect mode, we need the string lengths.
+ if (_args._indirect_input) {
+ string_lengths = std::vector<unsigned int>(_args._Ksections, 0);
+ }
+
+ /* Make sure we've been set up correctly. */
+ assert(_B_transposed);
+ static_assert(std::is_same<To, Toi>::value, "gemm_native: Operand types must be the same.");
+// static_assert(std::is_same<Tr, Tri>::value, "gemm_native: Result types must be the same.");
+
+ /* For now, each work item implies all the K for a given output
+ * pixel (so we don't need to synchronize access to the output
+ * array). So separate the loop over K blocks here. */
+ for (unsigned int k0=0; k0<_Ktotal; k0+=_k_block) {
+ unsigned int kmax = std::min(k0 + _k_block, _Ktotal);
+ unsigned int kern_k = roundup(kmax-k0, strategy::k_unroll());
+
+ const bool first_pass = (k0 == 0);
+ const bool last_pass = (kmax == _Ktotal);
+
+ unsigned int first_section = (k0 / _rounded_Ksize);
+ unsigned int first_offset = (k0 % _rounded_Ksize);
+ unsigned int kleft = kern_k;
+ unsigned int sections=0;
+ unsigned int offset = first_offset;
+
+ if (_args._indirect_input) {
+ while (kleft) {
+ // When chopping into sections: the amount that goes into 'string_lengths' is the amount to be
+ // processed (excluding padding). But the amount we subtract from 'kleft' takes account of any
+ // padding applied.
+ string_lengths[sections] = std::min(kleft, _args._Ksize - offset);
+ kleft -= std::min(kleft, _rounded_Ksize - offset);
+ sections++;
+ offset=0;
+ }
+ }
+
+ auto p = _window_range.iterator(work_range.get_position(0), work_range.get_position_end(0));
+
+ if (p.done()) {
+ return;
+ }
+
+ // Process rows either 'out_height' rows at a time, or do all valid rows at once with a single kernel call.
+ // The separate quantizer path only handles one block of rows at a time (as it has to store sums and intermediate results).
+ // THe convolution path only generates the pointers for one block of rows at a time.
+ const bool process_all_rows = (!SeparateQuantize && !_convolver);
+
+ do {
+ const unsigned int m_start = p.dim(0) * strategy::out_height();
+ const unsigned int m_end = process_all_rows ? std::min(p.dim0_max() * strategy::out_height(), _args._Msize) : std::min(m_start + strategy::out_height(), _args._Msize);
+// const unsigned int m_end = std::min(m_start + strategy::out_height(), _args._Msize);
+ const unsigned int batch = p.dim(1);
+ const unsigned int n0 = p.dim(2) * _n_block;
+ const unsigned int nmax = std::min(n0 + _n_block, _args._Nsize);
+ const unsigned int multi = p.dim(3);
+
+ const Toi *b_panel = _B_transposed +
+ (multi * roundup(_args._Nsize, strategy::out_width()) * _Ktotal) +
+ (k0 * roundup(_args._Nsize, strategy::out_width())) +
+ (n0 * kern_k);
+
+ IndirectOutputArg<Tr> out_arg(this->_Cptr + (multi * this->_C_multi_stride) + (batch * this->_C_batch_stride) + (m_start * this->_ldc) + n0, this->_ldc);
+
+#ifdef CYCLE_PROFILING
+ auto p = prof.ScopedProfiler(PROFILE_KERNEL, (unsigned long)(m_end - m_start) * kern_k * roundup(nmax-n0, strategy::out_width()));
+#endif
+ if (_indirect_buf) {
+ run_hybrid_kernel<OutputStage, SeparateQuantize>::run(
+#ifdef CYCLE_PROFILING
+ prof,
+#endif
+ strat, sections, string_lengths.data(),
+ IndirectInputArg<To>(_indirect_buf + (multi * _args._nbatches * _args._Ksections) + (batch * _args._Ksections) + first_section, m_start, first_offset),
+ (m_end - m_start), (nmax - n0), kern_k, b_panel, out_arg,
+ (this->_bias && first_pass) ? this->_bias + (multi * this->_bias_multi_stride) + n0 : nullptr,
+ last_pass ? _args._act : Activation(),
+ !first_pass,
+ // Quantization parameters
+ _os, _col_bias+(multi * _args._Nsize), n0);
+ } else if (_convolver) {
+ auto conv_cols = _convolver->process_columns(this->_Aptr + (multi * this->_A_multi_stride) + (batch * this->_A_batch_stride), this->_lda, k0, kmax, _rounded_Ksize);
+
+ unsigned int pos=0;
+ auto conv_rows = conv_cols.process_rows(m_start, m_end - m_start);
+
+ while (!conv_rows.finished()) {
+ unsigned int width, conv_offset;
+
+ assert(pos < sections);
+
+ std::tie(width, conv_offset) = conv_rows.next_block(&(in_row_ptrs[pos * strategy::out_height()]));
+
+ if (pos==0) {
+ assert(conv_offset == first_offset);
+ }
+ assert(width == string_lengths[pos]);
+ pos++;
+ }
+ assert(pos == sections);
+
+ run_hybrid_kernel<OutputStage, SeparateQuantize>::run(
+#ifdef CYCLE_PROFILING
+ prof,
+#endif
+ strat, sections, string_lengths.data(),
+ IndirectInputArg<To>(in_row_strings.data(), 0, first_offset),
+ (m_end - m_start), (nmax - n0), kern_k, b_panel, out_arg,
+ (this->_bias && first_pass) ? this->_bias + (multi * this->_bias_multi_stride) + n0 : nullptr,
+ last_pass ? _args._act : Activation(),
+ !first_pass,
+ // Quantization parameters
+ _os, _col_bias+(multi * _args._Nsize), n0);
+ } else {
+ // Length to process. This needs to exclude padding, but 'kmax' potentially includes it.
+ const unsigned int len = (std::min(_args._Ksize, kmax) - k0);
+
+ run_hybrid_kernel<OutputStage, SeparateQuantize>::run(
+#ifdef CYCLE_PROFILING
+ prof,
+#endif
+ strat, 1, &len,
+ IndirectInputArg<To>(this->_Aptr + (multi * this->_A_multi_stride) + (batch * this->_A_batch_stride) + m_start * this->_lda + k0, this->_lda),
+ (m_end - m_start), (nmax - n0), kern_k, b_panel, out_arg,
+ (this->_bias && first_pass) ? this->_bias + (multi * this->_bias_multi_stride) + n0 : nullptr,
+ last_pass ? _args._act : Activation(),
+ !first_pass,
+ // Quantization parameters
+ _os, _col_bias+(multi * _args._Nsize), n0);
+ }
+ } while (process_all_rows ? p.next_dim1() : p.next_dim0());
+ }
+ }
+
+ // Interface implementation - pretransposed
+ bool B_is_pretransposed() const override {
+ return true;
+ }
+
+ bool B_pretranspose_required() const override {
+ return (_B_transposed==nullptr);
+ }
+
+ size_t get_B_pretransposed_array_size() const override {
+ // Start with actual pretransposed buffer...
+ size_t size = roundup(_args._Nsize, strategy::out_width()) * _Ktotal * _args._nmulti * sizeof(Toi);
+
+ // Space for result row pointers (not strictly needed any more but retained for indirect output testing)
+ size += _args._Msize * _args._nbatches * _args._nmulti * sizeof(const Tr *);
+
+ if (std::is_same<OutputStage, Requantize32>::value) {
+ size += get_col_sum_size();
+ }
+
+ return size;
+ }
+
+ void pretranspose_B_array(void *in_buffer, const To *B, const int ldb, const int B_multi_stride) override {
+ if (std::is_same<OutputStage, Requantize32>::value) {
+ _col_bias = reinterpret_cast<int32_t *>(in_buffer);
+
+ Requantize32 *qp_ptr = reinterpret_cast<Requantize32 *>(&_os);
+
+ for (unsigned int i=0; i<_args._nmulti; i++) {
+ // The input is assumed not to have any padding between sections, so straightforward Ksize * Ksections computation gets the total size.
+ compute_col_sums(*qp_ptr, _args._Nsize, _args._Ksize * _args._Ksections, B + (i * B_multi_stride), ldb, _col_bias + (i * _args._Nsize), _args._Ksize * _args._Ksections, i, 0);
+ }
+ }
+
+ // Put the transposed data after the column sums - in non-transposing cases get_col_sum_size() == 0
+ uintptr_t buffer_int = reinterpret_cast<uintptr_t>(in_buffer);
+ Toi *buffer = reinterpret_cast<Toi *>(buffer_int + get_col_sum_size());
+ _B_transposed = buffer;
+
+ strategy strat(_args._ci);
+
+ for (unsigned int multi=0; multi<_args._nmulti; multi++) {
+ for (unsigned int k0=0; k0<_Ktotal; k0+=_k_block) {
+ const unsigned int kmax=std::min(k0 + _k_block, _Ktotal);
+
+ /* Figure out the size of each block. */
+ unsigned int k_size = kmax - k0;
+
+ // We need to insert padding at the end of each K section.
+ // The computation needed is a little delicate - the coordinates from the block walker are expressed in
+ // terms of the full, padded, _Ktotal.
+ // But we need to transform each section with reference to the original, unpadded, input, letting the
+ // transform pad each section as needed.
+
+ // This is needed for computations below.
+ const unsigned int rounded_section_size = roundup(_args._Ksize, strategy::k_unroll());
+
+ // The expected output format is also an entire <out_width> columns interleaved, then the next set of
+ // columns, and so on. This means, as we are breaking it up vertically, we have to do it one column at
+ // a time.
+ for (unsigned int x0=0; x0 < _args._Nsize; x0 += strategy::out_width() ){
+ unsigned int xmax = std::min(x0 + strategy::out_width(), _args._Nsize);
+
+ // Track where we are and how much work is left.
+ unsigned int kpos = k0;
+ unsigned int kleft = k_size;
+
+ while (kleft) {
+ // Which section are we in? Based on the rounded-up section size.
+ unsigned int k_section_base = kpos / rounded_section_size;
+ // How far into the section are we?
+ unsigned int k_offset = kpos - (k_section_base * rounded_section_size);
+
+ // We will either copy the rest of this section, or to the end of the requested length.
+ unsigned int k_length = std::min(_args._Ksize - k_offset, kleft);
+
+ strat.transforms.PrepareB(buffer, B + (multi * B_multi_stride), ldb,
+ x0, xmax,
+ (k_section_base * _args._Ksize) + k_offset, // K starting point - compute row to read based on our section and the true section length.
+ (k_section_base * _args._Ksize) + k_offset + k_length); // K end point - starting point plus length computed above.
+
+ // We need to modify our position based on the ROUNDED version of what we just did.
+ unsigned int padded_length = roundup(k_length, strategy::k_unroll());
+
+ buffer += strategy::out_width() * padded_length;
+
+ kpos += padded_length;
+ kleft -= padded_length;
+ }
+ }
+ }
+ }
+ }
+
+ void set_pretransposed_B_data(void *in_buffer) override {
+ // Put the transposed data after the column sums - in non-transposing cases get_col_sum_size() == 0
+ uintptr_t buffer_int = reinterpret_cast<uintptr_t>(in_buffer);
+ _B_transposed = reinterpret_cast<Toi *>(buffer_int + get_col_sum_size());
+ _col_bias = reinterpret_cast<int32_t *>(in_buffer);
+ }
+
+ // Estimate cycles for given problem given provided parameters
+ static uint64_t estimate_cycles(const GemmArgs &args, const PerformanceParameters ¶ms) {
+ // Note: Current hybrid kernels don't actually round up height (they
+ // have paths for each possible height). Might need to make this
+ // configurable in future.
+ uint64_t total_macs = static_cast<uint64_t>(args._nbatches) * args._nmulti * args._Msize * roundup(args._Nsize, strategy::out_width()) * roundup(args._Ksize, strategy::k_unroll());
+
+ float mac_cycles = static_cast<float>(total_macs) / params.kernel_macs_cycle;
+
+ // TODO: A bit of a kludge here: current hybrid kernels incur extra
+ // overhead where the width is not a multiple of kernel width. It's
+ // most noticable where the overall width is quite low, so add 15%
+ // penalty for such widths.
+ if ((args._Nsize < strategy::out_width()) || (args._Nsize > strategy::out_width() && args._Nsize < 2*strategy::out_width())) {
+ mac_cycles *= 1.15f;
+ }
+
+ uint64_t total_cycles = mac_cycles;
+
+ return total_cycles;
+ }
+
+ void set_quantized_bias(const int32_t *bias, size_t bias_multi_stride) override {
+ if (std::is_same<OutputStage, Requantize32>::value) {
+ Requantize32 *qp = reinterpret_cast<Requantize32 *>(&_os);
+
+ qp->bias = bias;
+ qp->bias_multi_stride = bias_multi_stride;
+ }
+ }
+
+ void set_indirect_parameters(size_t string_len, const To * const * const *ptr) override {
+ assert(string_len == _args._Ksize);
+ _indirect_buf = ptr;
+ }
+
+ void set_convolution_parameters(ConvolutionParameters parms) override {
+ assert(parms.input_channels == _args._Ksize);
+ _convolver = std::unique_ptr<convolver<To>>(new convolver<To>(parms));
+ }
+};
+
+} // namespace arm_gemm
+
+#ifdef __I_DEFINED_UNUSED
+#undef UNUSED
+#endif
diff --git a/src/core/NEON/kernels/arm_gemm/gemm_hybrid_quantized.hpp b/src/core/NEON/kernels/arm_gemm/gemm_hybrid_quantized.hpp
index 915227f..7a5fa87 100644
--- a/src/core/NEON/kernels/arm_gemm/gemm_hybrid_quantized.hpp
+++ b/src/core/NEON/kernels/arm_gemm/gemm_hybrid_quantized.hpp
@@ -118,18 +118,27 @@
// n_block: Work out how many rows (of length k_block) will fit in the L2
// Don't allocate more than 90% of the L2 to allow for overheads, and subtract off the L1 contents.
- unsigned int n_block = (((L2_size * 9) / 10) - (k_block * sizeof(Toi) * (strategy::out_width() + strategy::out_height()))) /
- (sizeof(Toi) * k_block);
+ const unsigned int scaled_l2_size = (L2_size * 9) / 10;
+ const unsigned int k_block_area = k_block * sizeof(Toi) * (strategy::out_width() + strategy::out_height());
+
+ // .. if the L1 contents is bigger than the L2, just return a minimal size block.
+ if (k_block_area > scaled_l2_size) {
+ return strategy::out_width();
+ }
+
+ unsigned int n_block = (scaled_l2_size - k_block_area) / (sizeof(Toi) * k_block);
// Needs to be (at least a single) multiple of the kernel output width.
n_block /= strategy::out_width();
- n_block = std::max(n_block, 1U) * strategy::out_width();
+ n_block = std::max(n_block, 1u) * strategy::out_width();
// And tune to the presented problem size.
unsigned int numblocks = iceildiv(args._Nsize, n_block);
n_block = iceildiv(args._Nsize, numblocks);
n_block = roundup(n_block, strategy::out_width());
+ assert(n_block > 0);
+
return n_block;
}
diff --git a/src/core/NEON/kernels/arm_gemm/gemm_hybrid_quantized_inline.hpp b/src/core/NEON/kernels/arm_gemm/gemm_hybrid_quantized_inline.hpp
new file mode 100644
index 0000000..7376b5f
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/gemm_hybrid_quantized_inline.hpp
@@ -0,0 +1,265 @@
+/*
+ * Copyright (c) 2017-2019 Arm Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#pragma once
+
+#include <assert.h>
+
+#include <algorithm>
+
+#include "arm_gemm.hpp"
+#include "ndrange.hpp"
+#include "utils.hpp"
+
+#include "mergeresults.hpp"
+#include "transform.hpp"
+
+#ifdef CYCLE_PROFILING
+#include "profiler.hpp"
+#endif
+
+namespace arm_gemm {
+
+// Implementation of the GemmCommon abstract class.
+template<typename strategy, typename To, typename Tr>
+class GemmHybridQuantizedInline : public GemmCommon<To, Tr> {
+ typedef typename strategy::operand_type Toi;
+ typedef typename strategy::result_type Tri;
+
+ /* const properties set by constructor */
+ const CPUInfo * const _ci;
+
+ const unsigned int _Msize;
+ const unsigned int _Nsize;
+ const unsigned int _Ksize;
+
+ const unsigned int _nbatches;
+ const unsigned int _nmulti;
+
+ /* Blocking info */
+ const unsigned int _k_block;
+ const unsigned int _n_block;
+ const unsigned int _Mround;
+
+ /* Pretransposed buffer. */
+ const Toi *_B_transposed=nullptr;
+
+ const NDRange<4> _window_range;
+
+ Requantize32 _qp;
+ int32_t *col_bias = nullptr;
+
+ void *working_space = nullptr;
+
+ unsigned int _nthreads;
+
+ unsigned int get_col_sum_size() const {
+ return _Nsize * _nmulti * sizeof(int32_t);
+ }
+
+ static unsigned int compute_k_block(const GemmArgs &args) {
+ // We don't support K blocks as we only temporarily store 32 bit results.
+ return args._Ksize;
+
+ if (args._cfg && args._cfg->inner_block_size) {
+ return args._cfg->inner_block_size;
+ }
+
+ const unsigned int L1_size = args._ci->get_L1_cache_size();
+
+ // k_block: Find out how much of the larger array can be loaded into half the cache.
+ // This should account for associative caches.
+ unsigned int k_block = (L1_size / 2) / (sizeof(Toi) * (std::max(strategy::out_width(), strategy::out_height())));
+
+ // Needs to be (at least a single) multiple of the K unroll level.
+ k_block /= strategy::k_unroll();
+ k_block = std::max(k_block, 1U) * strategy::k_unroll();
+
+ // Now tune to presented problem size; this is how many blocks we need.
+ unsigned int numk_blocks = iceildiv(args._Ksize, k_block);
+
+ // So divide the space equally into that many blocks.
+ k_block = iceildiv(args._Ksize, numk_blocks);
+
+ // And round UP to the K unroll level required.
+ k_block = roundup(k_block, strategy::k_unroll());
+
+ return k_block;
+ }
+
+ static unsigned int compute_n_block(const GemmArgs &args) {
+ if (args._cfg && args._cfg->outer_block_size) {
+ return args._cfg->outer_block_size;
+ }
+
+ const unsigned int k_block = compute_k_block(args);
+ const unsigned int L2_size = args._ci->get_L2_cache_size();
+
+ // n_block: Work out how many rows (of length k_block) will fit in the L2
+ // Don't allocate more than 90% of the L2 to allow for overheads, and subtract off the L1 contents.
+ unsigned int n_block = (((L2_size * 9) / 10) - (k_block * sizeof(Toi) * (strategy::out_width() + strategy::out_height()))) /
+ (sizeof(Toi) * k_block);
+
+ // Needs to be (at least a single) multiple of the kernel output width.
+ n_block /= strategy::out_width();
+ n_block = std::max(n_block, 1U) * strategy::out_width();
+
+ // And tune to the presented problem size.
+ unsigned int numblocks = iceildiv(args._Nsize, n_block);
+ n_block = iceildiv(args._Nsize, numblocks);
+ n_block = roundup(n_block, strategy::out_width());
+
+ return n_block;
+ }
+
+public:
+ GemmHybridQuantizedInline(GemmHybridQuantizedInline &) = delete;
+ GemmHybridQuantizedInline & operator= (GemmHybridQuantizedInline &) = delete;
+
+ /* Constructor */
+ GemmHybridQuantizedInline(const GemmArgs &args, const Requantize32 &qp)
+ : _ci(args._ci), _Msize(args._Msize), _Nsize(args._Nsize), _Ksize(args._Ksize),
+ _nbatches(args._nbatches), _nmulti(args._nmulti),
+ _k_block(compute_k_block(args)), _n_block(compute_n_block(args)),
+ _Mround(roundup(args._Msize, strategy::out_height())),
+ _window_range(iceildiv(args._Msize, strategy::out_height()), _nbatches, iceildiv(_Nsize, _n_block), _nmulti),
+ _qp (qp), _nthreads(args._maxthreads) { }
+
+ // Interface implementation - Compulsory functions
+ ndrange_t get_window_size() const override {
+ return { _window_range.total_size() };
+ }
+
+ // This kernel can always be dynamically scheduled.
+ bool supports_dynamic_scheduling() const override {
+ return true;
+ }
+
+ // Execute
+ void execute(const ndcoord_t &work_range, const ndcoord_t &, int) override {
+#ifdef CYCLE_PROFILING
+ profiler prof;
+#endif
+ strategy strat(_ci);
+
+ /* Make sure we've been set up correctly. */
+ assert(_B_transposed);
+ static_assert(std::is_same<To, Toi>::value, "gemm_native: Operand types must be the same.");
+
+ /* For now, each work item implies all the K for a given output
+ * pixel (so we don't need to synchronize access to the output
+ * array). So separate the loop over K blocks here. */
+ for (unsigned int k0=0; k0<_Ksize; k0+=_k_block) {
+ unsigned int kmax = std::min(k0 + _k_block, _Ksize);
+ unsigned int kern_k = roundup(kmax-k0, strategy::k_unroll());
+
+ auto p = _window_range.iterator(work_range.get_position(0), work_range.get_position_end(0));
+
+ if (p.done()) {
+ return;
+ }
+
+ do {
+ const unsigned int m_start = p.dim(0) * strategy::out_height();
+ const unsigned int m_end = std::min(p.dim0_max() * strategy::out_height(), _Msize);
+ const unsigned int batch = p.dim(1);
+ const unsigned int n0 = p.dim(2) * _n_block;
+ const unsigned int nmax = std::min(n0 + _n_block, _Nsize);
+ const unsigned int multi = p.dim(3);
+
+ const Toi *b_panel = _B_transposed +
+ (multi * roundup(_Nsize, strategy::out_width()) * roundup(_Ksize, strategy::k_unroll())) +
+ (k0 * roundup(_Nsize, strategy::out_width())) +
+ (n0 * kern_k);
+
+ {
+#ifdef CYCLE_PROFILING
+ auto p = prof.ScopedProfiler(PROFILE_KERNEL, (m_end - m_start) * kern_k * roundup(nmax-n0, strategy::out_width()));
+#endif
+ strat.kernel(this->_Aptr + (multi * this->_A_multi_stride) + (batch * this->_A_batch_stride) + (m_start * this->_lda) + k0, this->_lda,
+ b_panel,
+ this->_Cptr + (multi * this->_C_multi_stride) + (batch * this->_C_batch_stride) + (m_start * this->_ldc) + n0, this->_ldc,
+ (m_end - m_start), (nmax - n0), kmax - k0,
+ col_bias + (multi * _Nsize) + n0, _qp);
+ }
+ } while (p.next_dim1());
+ }
+ }
+
+ // Interface implementation - pretransposed
+ bool B_is_pretransposed() const override {
+ return true;
+ }
+
+ bool B_pretranspose_required() const override {
+ return (_B_transposed==nullptr);
+ }
+
+ size_t get_B_pretransposed_array_size() const override {
+ return get_col_sum_size() + (roundup(_Nsize, strategy::out_width()) * roundup(_Ksize, strategy::k_unroll()) * _nmulti * sizeof(Toi));
+ }
+
+ void pretranspose_B_array(void *in_buffer, const To *B, const int ldb, const int B_multi_stride) override {
+ col_bias = reinterpret_cast<int32_t *>(in_buffer);
+
+ for (unsigned int i=0; i<_nmulti; i++) {
+ compute_col_sums(_qp, _Nsize, _Ksize, B + (i * B_multi_stride), ldb, col_bias + (i * _Nsize), _Ksize, i, 0);
+ }
+
+ uintptr_t buffer_int = reinterpret_cast<uintptr_t>(in_buffer);
+ Toi *buffer = reinterpret_cast<Toi *>(buffer_int + get_col_sum_size());
+ _B_transposed = buffer;
+ strategy strat(_ci);
+
+ for (unsigned int multi=0; multi<_nmulti; multi++) {
+ for (unsigned int k0=0; k0<_Ksize; k0+=_k_block) {
+ const unsigned int kmax = std::min(k0 + _k_block, _Ksize);
+ const unsigned int k_size = roundup(kmax-k0, strategy::k_unroll());
+
+ for (unsigned int x0=0; x0<_Nsize; x0+=_n_block) {
+ const unsigned int xmax = std::min(x0+_n_block, _Nsize);
+
+ const unsigned int size = roundup(xmax-x0, strategy::out_width()) * k_size;
+
+ strat.transforms.PrepareB( buffer, B + (multi * B_multi_stride), ldb,
+ x0, xmax, k0, kmax);
+
+ buffer += size;
+ }
+ }
+ }
+ }
+
+ void set_pretransposed_B_data(void *in_buffer) override {
+ uintptr_t buffer_int = reinterpret_cast<uintptr_t>(in_buffer);
+ _B_transposed = reinterpret_cast<Toi *>(buffer_int + get_col_sum_size());
+ col_bias = reinterpret_cast<int32_t *>(in_buffer);
+ }
+
+ void set_quantized_bias(const int32_t *bias, size_t bias_multi_stride) override {
+ _qp.bias = bias;
+ _qp.bias_multi_stride = bias_multi_stride;
+ }
+};
+
+} // namespace arm_gemm
diff --git a/src/core/NEON/kernels/arm_gemm/gemm_implementation.hpp b/src/core/NEON/kernels/arm_gemm/gemm_implementation.hpp
index 261e7d2..f6a0fc5 100644
--- a/src/core/NEON/kernels/arm_gemm/gemm_implementation.hpp
+++ b/src/core/NEON/kernels/arm_gemm/gemm_implementation.hpp
@@ -37,9 +37,9 @@
struct GemmImplementation {
const GemmMethod method;
const char * name;
- std::function<bool(const GemmArgs &, const OutputStage &)> is_supported;
- std::function<uint64_t(const GemmArgs &, const OutputStage &)> cycle_estimate;
- std::function<GemmCommon<Top, Tret> *(const GemmArgs &, const OutputStage &)> instantiate;
+ std::function<bool(const GemmArgs &, const OutputStage &)> is_supported = {};
+ std::function<uint64_t(const GemmArgs &, const OutputStage &)> cycle_estimate = {};
+ std::function<GemmCommon<Top, Tret> *(const GemmArgs &, const OutputStage &)> instantiate = {};
bool do_is_supported(const GemmArgs &args, const OutputStage &os) const {
if (is_supported != nullptr) {
@@ -57,13 +57,13 @@
}
}
- GemmImplementation(const GemmImplementation &) = default;
- GemmImplementation &operator= (const GemmImplementation &) = default;
-
GemmCommon<Top, Tret> *do_instantiate(const GemmArgs &args, const OutputStage &os) const {
return instantiate(args, os);
}
+ GemmImplementation(const GemmImplementation &) = default;
+ GemmImplementation & operator= (const GemmImplementation &) = default;
+
GemmImplementation(GemmMethod m, const char *n,
std::function<bool(const GemmArgs &, const OutputStage &)> is_supported, std::function<bool(const GemmArgs &, const OutputStage &)> is_recommended,
std::function<GemmCommon<Top, Tret> *(const GemmArgs &, const OutputStage &)> instantiate) :
@@ -79,9 +79,9 @@
struct GemmImplementation<Top, Tret, Nothing> {
const GemmMethod method;
const char * name;
- std::function<bool(const GemmArgs &)> is_supported;
- std::function<uint64_t(const GemmArgs &)> cycle_estimate;
- std::function<GemmCommon<Top, Tret> *(const GemmArgs &)> instantiate;
+ std::function<bool(const GemmArgs &)> is_supported = {};
+ std::function<uint64_t(const GemmArgs &)> cycle_estimate = {};
+ std::function<GemmCommon<Top, Tret> *(const GemmArgs &)> instantiate = {};
bool do_is_supported(const GemmArgs &args, const Nothing &) const {
if (is_supported != nullptr) {
@@ -103,7 +103,6 @@
return instantiate(args);
}
-
static GemmImplementation with_estimate(GemmMethod m, const char *n,
std::function<bool(const GemmArgs &)> is_supported, std::function<uint64_t(const GemmArgs &)> cycle_estimate,
std::function<GemmCommon<Top, Tret> *(const GemmArgs &)> instantiate) {
@@ -116,7 +115,10 @@
return impl;
}
- GemmImplementation(GemmMethod m, const char * n) : method(m), name(n), is_supported(nullptr), cycle_estimate(nullptr), instantiate(nullptr) {}
+ GemmImplementation(const GemmImplementation &) = default;
+ GemmImplementation & operator= (const GemmImplementation &) = default;
+
+ GemmImplementation(GemmMethod m, const char * n) : method(m), name(n) {}
GemmImplementation(GemmMethod m, const char *n,
std::function<bool(const GemmArgs &)> is_supported, std::function<bool(const GemmArgs &)> is_recommended,
@@ -124,9 +126,6 @@
method(m), name(n), is_supported(is_supported),
cycle_estimate( [is_recommended](const GemmArgs &args) -> uint64_t { return (is_recommended == nullptr) ? 0 : (is_recommended(args) ? 0 : UINT64_MAX); } ),
instantiate(instantiate) { }
-
- GemmImplementation(const GemmImplementation &) = default;
- GemmImplementation &operator=(const GemmImplementation &) = default;
};
/* "Master" function implemented for each valid combination of types.
@@ -211,6 +210,7 @@
for (const GemmImplementation<Top, Tret, OutputStage> *i = gemms; i->method != GemmMethod::DEFAULT; i++) {
/* Check that this implementation supports the presented problem. */
+
if (!i->do_is_supported(args, os)) {
continue;
}
diff --git a/src/core/NEON/kernels/arm_gemm/gemm_int16.cpp b/src/core/NEON/kernels/arm_gemm/gemm_int16.cpp
index da68233..a3a6195 100644
--- a/src/core/NEON/kernels/arm_gemm/gemm_int16.cpp
+++ b/src/core/NEON/kernels/arm_gemm/gemm_int16.cpp
@@ -28,17 +28,17 @@
#include "gemm_implementation.hpp"
#include "gemm_interleaved.hpp"
-#include "kernels/a64_gemm_s16_12x8.hpp"
+#include "kernels/a64_gemm_s16_8x12.hpp"
namespace arm_gemm {
static const GemmImplementation<int16_t, int32_t> gemm_s16_methods[] = {
{
GemmMethod::GEMM_INTERLEAVED,
- "gemm_s16_12x8",
+ "a64_gemm_s16_8x12",
nullptr,
nullptr,
- [](const GemmArgs &args) { return new GemmInterleaved<gemm_s16_12x8, int16_t, int32_t>(args); }
+ [](const GemmArgs &args) { return new GemmInterleaved<cls_a64_gemm_s16_8x12, int16_t, int32_t>(args); }
},
{
GemmMethod::DEFAULT,
diff --git a/src/core/NEON/kernels/arm_gemm/gemm_int8.cpp b/src/core/NEON/kernels/arm_gemm/gemm_int8.cpp
index 147caee..31f2250 100644
--- a/src/core/NEON/kernels/arm_gemm/gemm_int8.cpp
+++ b/src/core/NEON/kernels/arm_gemm/gemm_int8.cpp
@@ -26,21 +26,22 @@
#include "arm_gemm.hpp"
#include "gemm_common.hpp"
#include "gemm_hybrid.hpp"
+#include "gemm_hybrid_indirect.hpp"
#include "gemm_implementation.hpp"
#include "gemm_interleaved.hpp"
-#include "gemm_interleaved_pretransposed_2d.hpp"
-#include "kernels/a64_gemm_s16_12x8.hpp"
-#include "kernels/a64_gemm_s8_12x8.hpp"
+#include "kernels/a64_gemm_s16_8x12.hpp"
+#include "kernels/a64_gemm_s8_8x12.hpp"
#include "kernels/a64_gemm_s8_4x4.hpp"
-#include "kernels/a64_hybrid_s8s32_dot_16x4.hpp"
-#include "kernels/a64_interleaved_s8s32_mmla_12x8.hpp"
-#include "kernels/a64_smallK_hybrid_s8s32_dot_4x6.hpp"
-#include "kernels/a64_smallK_hybrid_s8s32_dot_4x8.hpp"
-#include "kernels/sve_hybrid_s8s32_dot_4VLx4.hpp"
-#include "kernels/sve_interleaved_s8s32_dot_3VLx8.hpp"
-#include "kernels/sve_interleaved_s8s32_mmla_3VLx8.hpp"
-#include "kernels/sve_smallK_hybrid_s8s32_dot_1VLx8.hpp"
+#include "kernels/a64_hybrid_s8s32_dot_6x16.hpp"
+#include "kernels/a64_interleaved_s8s32_mmla_8x12.hpp"
+#include "kernels/a64_smallK_hybrid_s8s32_dot_6x4.hpp"
+#include "kernels/a64_smallK_hybrid_s8s32_dot_8x4.hpp"
+
+#include "kernels/sve_hybrid_s8s32_dot_6x4VL.hpp"
+#include "kernels/sve_interleaved_s8s32_dot_8x3VL.hpp"
+#include "kernels/sve_interleaved_s8s32_mmla_8x3VL.hpp"
+#include "kernels/sve_smallK_hybrid_s8s32_dot_8x1VL.hpp"
namespace arm_gemm {
@@ -49,106 +50,84 @@
#ifdef MMLA_INT8
{
GemmMethod::GEMM_INTERLEAVED,
- "interleaved_s8s32_mmla_3VLx8",
+ "sve_interleaved_s8s32_mmla_8x3VL",
[](const GemmArgs &args) { return (args._Ksize>8); },
nullptr,
- [](const GemmArgs &args) { return new GemmInterleaved<interleaved_s8s32_mmla_3VLx8, int8_t, int32_t>(args); }
+ [](const GemmArgs &args) { return new GemmInterleaved<cls_sve_interleaved_s8s32_mmla_8x3VL, int8_t, int32_t>(args); }
},
#endif
{
GemmMethod::GEMM_HYBRID,
- "smallK_hybrid_s8s32_dot_1VLx8",
- [](const GemmArgs &args) { return args._Ksize<=64; },
+ "sve_smallK_hybrid_s8s32_dot_8x1VL",
+ [](const GemmArgs &args) { return args._Ksize<=64 && !args._indirect_input; },
nullptr,
- [](const GemmArgs &args) { return new GemmHybrid<smallK_hybrid_s8s32_dot_1VLx8, int8_t, int32_t>(args); }
+ [](const GemmArgs &args) { return new GemmHybrid<cls_sve_smallK_hybrid_s8s32_dot_8x1VL, int8_t, int32_t>(args); }
},
{
GemmMethod::GEMM_HYBRID,
- "hybrid_s8s32_dot_4VLx4",
+ "sve_hybrid_s8s32_dot_6x4VL",
[](const GemmArgs &args) { return args._Ksize>=16; },
[](const GemmArgs &args) { return ((args._Ksize <= 128) && (args._Nsize <= 128)) || ((args._nmulti > 1) && ((args._Msize / args._maxthreads) < 8)); },
- [](const GemmArgs &args) { return new GemmHybrid<hybrid_s8s32_dot_4VLx4, int8_t, int32_t>(args); }
+ [](const GemmArgs &args) { return new GemmHybridIndirect<cls_sve_hybrid_s8s32_dot_6x4VL, int8_t, int32_t>(args); }
},
{
GemmMethod::GEMM_INTERLEAVED,
- "interleaved_s8s32_dot_3VLx8",
+ "sve_interleaved_s8s32_dot_8x3VL",
[](const GemmArgs &args) { return (args._Ksize>4); },
nullptr,
- [](const GemmArgs &args) { return new GemmInterleaved<interleaved_s8s32_dot_3VLx8, int8_t, int32_t>(args); }
+ [](const GemmArgs &args) { return new GemmInterleaved<cls_sve_interleaved_s8s32_dot_8x3VL, int8_t, int32_t>(args); }
},
-#endif
+#endif // SVE
#ifdef MMLA_INT8
{
GemmMethod::GEMM_INTERLEAVED,
- "interleaved_s8s32_mmla_12x8",
+ "a64_interleaved_s8s32_mmla_8x12",
[](const GemmArgs &args) { return (args._Ksize>8); },
nullptr,
- [](const GemmArgs &args) { return new GemmInterleaved<interleaved_s8s32_mmla_12x8, int8_t, int32_t>(args); }
+ [](const GemmArgs &args) { return new GemmInterleaved<cls_a64_interleaved_s8s32_mmla_8x12, int8_t, int32_t>(args); }
},
#endif
{
GemmMethod::GEMM_HYBRID,
- "smallK_hybrid_s8s32_dot_4x8",
- [](const GemmArgs &args) { return args._ci->has_dotprod() && (args._Nsize % 4 == 0) && (args._Ksize<=32); },
+ "a64_smallK_hybrid_s8s32_dot_8x4",
+ [](const GemmArgs &args) { return args._ci->has_dotprod() && (args._Nsize % 4 == 0) && (args._Ksize<=32) && !args._indirect_input; },
nullptr,
- [](const GemmArgs &args) { return new GemmHybrid<smallK_hybrid_s8s32_dot_4x8, int8_t, int32_t>(args); }
+ [](const GemmArgs &args) { return new GemmHybrid<cls_a64_smallK_hybrid_s8s32_dot_8x4, int8_t, int32_t>(args); }
},
{
GemmMethod::GEMM_HYBRID,
- "smallK_hybrid_s8s32_dot_4x6",
- [](const GemmArgs &args) { return args._ci->has_dotprod() && (args._Nsize % 4 == 0) && (args._Ksize>32) && (args._Ksize<=64); },
+ "a64_smallK_hybrid_s8s32_dot_6x4",
+ [](const GemmArgs &args) { return args._ci->has_dotprod() && (args._Nsize % 4 == 0) && (args._Ksize>32) && (args._Ksize<=64) && !args._indirect_input; },
nullptr,
- [](const GemmArgs &args) { return new GemmHybrid<smallK_hybrid_s8s32_dot_4x6, int8_t, int32_t>(args); }
+ [](const GemmArgs &args) { return new GemmHybrid<cls_a64_smallK_hybrid_s8s32_dot_6x4, int8_t, int32_t>(args); }
+},
+{
+ GemmMethod::GEMM_INTERLEAVED,
+ "a64_gemm_s16_8x12",
+ nullptr,
+ [](const GemmArgs &args) { return args._ci->get_cpu_model() == CPUModel::A53 && args._Ksize>4; },
+ [](const GemmArgs &args) { return new GemmInterleaved<cls_a64_gemm_s16_8x12, int8_t, int32_t>(args); },
},
{
GemmMethod::GEMM_HYBRID,
- "hybrid_s8s32_dot_16x4",
- [](const GemmArgs &args) { return args._ci->has_dotprod() && args._Ksize>=16; },
+ "a64_hybrid_s8s32_dot_6x16",
+ [](const GemmArgs &args) { return args._ci->has_dotprod(); },
[](const GemmArgs &args) { return args._Nsize<=256 && args._Ksize>128; },
- [](const GemmArgs &args) { return new GemmHybrid<hybrid_s8s32_dot_16x4, int8_t, int32_t>(args); }
-},
-{
- GemmMethod::GEMM_INTERLEAVED_2D,
- "gemm_s8_12x8_2d",
- [](const GemmArgs &args) { return args._ci->has_dotprod(); },
- [](const GemmArgs &args) { return (args._maxthreads >= 8) && (args._Msize >= 8) && (args._Nsize >= 8); },
- [](const GemmArgs &args) { return new GemmInterleavedPretransposed2d<gemm_s8_12x8, int8_t, int32_t>(args); }
+ [](const GemmArgs &args) { return new GemmHybridIndirect<cls_a64_hybrid_s8s32_dot_6x16, int8_t, int32_t>(args); }
},
{
GemmMethod::GEMM_INTERLEAVED,
- "gemm_s8_12x8_1d",
+ "a64_gemm_s8_8x12",
[](const GemmArgs &args) { return args._ci->has_dotprod(); },
nullptr,
- [](const GemmArgs &args) { return new GemmInterleaved<gemm_s8_12x8, int8_t, int32_t>(args); }
-},
-{
- GemmMethod::GEMM_INTERLEAVED_2D,
- "gemm_s16_12x8_2d",
- nullptr,
- [](const GemmArgs &args) { return args._ci->get_cpu_model() == CPUModel::A53 && args._Msize > 4 && (args._Msize / args._maxthreads) < 8; },
- [](const GemmArgs &args) { return new GemmInterleavedPretransposed2d<gemm_s16_12x8, int8_t, int32_t>(args); },
+ [](const GemmArgs &args) { return new GemmInterleaved<cls_a64_gemm_s8_8x12, int8_t, int32_t>(args); }
},
{
GemmMethod::GEMM_INTERLEAVED,
- "gemm_s16_12x8_1d",
- nullptr,
- [](const GemmArgs &args) { return args._ci->get_cpu_model() == CPUModel::A53 && args._Msize > 4; },
- [](const GemmArgs &args) { return new GemmInterleaved<gemm_s16_12x8, int8_t, int32_t>(args); },
-},
-{
- GemmMethod::GEMM_INTERLEAVED_2D,
- "gemm_s8_4x4_2d",
- nullptr,
- [](const GemmArgs &args) { return ((args._maxthreads >= 8) && (args._Msize >= 8) && (args._Nsize >= 8)) ||
- ((args._Msize / args._maxthreads) < 4); },
- [](const GemmArgs &args) { return new GemmInterleavedPretransposed2d<gemm_s8_4x4, int8_t, int32_t>(args); }
-},
-{
- GemmMethod::GEMM_INTERLEAVED,
- "gemm_s8_4x4_1d",
+ "a64_gemm_s8_4x4",
nullptr,
nullptr,
- [](const GemmArgs &args) { return new GemmInterleaved<gemm_s8_4x4, int8_t, int32_t>(args); }
+ [](const GemmArgs &args) { return new GemmInterleaved<cls_a64_gemm_s8_4x4, int8_t, int32_t>(args); }
},
{
GemmMethod::DEFAULT,
diff --git a/src/core/NEON/kernels/arm_gemm/gemm_interleaved.hpp b/src/core/NEON/kernels/arm_gemm/gemm_interleaved.hpp
index c4dceef..92c1086 100644
--- a/src/core/NEON/kernels/arm_gemm/gemm_interleaved.hpp
+++ b/src/core/NEON/kernels/arm_gemm/gemm_interleaved.hpp
@@ -27,11 +27,12 @@
#include <cassert>
#include "arm_gemm.hpp"
-#include "utils.hpp"
-
+#include "convolver.hpp"
#include "mergeresults.hpp"
#include "performance_parameters.hpp"
+#include "quantized.hpp"
#include "transform.hpp"
+#include "utils.hpp"
#ifdef CYCLE_PROFILING
#include "profiler.hpp"
@@ -46,12 +47,212 @@
//
// This implementation interleaves the source matrices in blocks - good for
// larger matrices.
+
namespace arm_gemm {
-template<typename strategy, typename To, typename Tr>
+namespace {
+
+// Some kernels output to a linear buffer and require a separate merge step.
+// Others output directly to the matrix result. This helper class calls the
+// appropriate functions, using templating to avoid calling non-existent
+// functions.
+template<bool MergeStep, typename OutputStage>
+class kernel_and_merge {
+public:
+ template<typename strategy, typename To, typename Tr, typename Tri, typename Tab>
+ static void run (
+#ifdef CYCLE_PROFILING
+ profiler &prof,
+#endif
+ strategy &strat, const To *a_ptr, const To *b_panel, Tri *c_panel,
+ Tr *c_ptr, int ldc, int kern_k, unsigned int m_0,
+ unsigned int m_max, unsigned int n_0, unsigned int n_max, const Tr *biasptr,
+ const Activation &act, bool accumulate, const OutputStage &os, const int32_t *col_bias,
+ Tab *acc_buff);
+};
+
+// Run a kernel and call the separate merge step
+template<>
+template<typename strategy, typename To, typename Tr, typename Tri, typename Tab>
+void kernel_and_merge<true, Nothing>::run(
+#ifdef CYCLE_PROFILING
+ profiler &prof,
+#endif
+ strategy &strat, const To *a_ptr, const To *b_panel, Tri *c_panel,
+ Tr *c_ptr, int ldc, int kern_k, unsigned int m_0,
+ unsigned int m_max, unsigned int n_0, unsigned int n_max, const Tr *biasptr,
+ const Activation &act, bool accumulate, const Nothing &, const int32_t *, Tab *)
+{
+ const int bblocks = iceildiv(n_max - n_0, strategy::out_width());
+
+ {
+#ifdef CYCLE_PROFILING
+ auto p=prof.ScopedProfiler(PROFILE_KERNEL, (strategy::out_height() * bblocks * strategy::out_width() * kern_k));
+#endif
+
+ strat.kernel(a_ptr, b_panel, c_panel, 1, bblocks, kern_k);
+ }
+
+ {
+#ifdef CYCLE_PROFILING
+ auto p=prof.ScopedProfiler(PROFILE_MERGE, (strategy::out_height() * bblocks * strategy::out_width() * sizeof(Tr)));
+#endif
+ strat.transforms.Merge(c_ptr, c_panel, ldc, m_0, m_max, n_0, n_max, biasptr, act, accumulate);
+ }
+}
+
+// Run a kernel with integrated merge
+template<>
+template<typename strategy, typename To, typename Tr, typename Tri, typename Tab>
+void kernel_and_merge<false, Nothing>::run(
+#ifdef CYCLE_PROFILING
+ profiler &prof,
+#endif
+ strategy &strat, const To *a_ptr, const To *b_panel, Tri *,
+ Tr *c_ptr, int ldc, int kern_k, unsigned int m_0, unsigned int m_max,
+ unsigned int n_0, unsigned int n_max, const Tr *biasptr,
+ const Activation &act, bool accumulate, const Nothing &, const int32_t *,
+ Tab *acc_buff)
+{
+#ifdef CYCLE_PROFILING
+ auto p=prof.ScopedProfiler(PROFILE_KERNEL, (m_max - m_0) * (n_max - n_0) * kern_k);
+#endif
+
+ // We need to offset the C pointer, but as it might be NULL (requesting output to accumulation buffer) we need
+ // to be careful not to offset a null pointer.
+ Tri *offset_c_ptr;
+
+ if (c_ptr == nullptr) {
+ offset_c_ptr = nullptr;
+ } else {
+ offset_c_ptr = c_ptr + m_0 * ldc + n_0;
+ }
+
+ strat.kernel(// A and B pointers are just the packed panels.
+ a_ptr, b_panel,
+ // Provide relevant part of output array and row stride.
+ offset_c_ptr, ldc,
+ // M, N, K sizes
+ m_max-m_0, n_max - n_0, kern_k,
+ // Bias, activation, accumulation. Need to offset the bias as needed.
+ biasptr ? biasptr + n_0 : nullptr, act, accumulate,
+ // Accumulation buffer.
+ acc_buff );
+}
+
+// Run a kernel with integrated merge, quantizing
+template<>
+template<typename strategy, typename To, typename Tr, typename Tri, typename Tab>
+void kernel_and_merge<false, Requantize32>::run(
+#ifdef CYCLE_PROFILING
+ profiler &prof,
+#endif
+ strategy &strat, const To *a_ptr, const To *b_panel, Tri *,
+ Tr *c_ptr, int ldc, int kern_k, unsigned int m_0, unsigned int m_max,
+ unsigned int n_0, unsigned int n_max, const Tr *,
+ const Activation &, bool accumulate, const Requantize32 &qp, const int32_t *col_bias,
+ Tab *acc_buff)
+{
+#ifdef CYCLE_PROFILING
+ auto p=prof.ScopedProfiler(PROFILE_KERNEL, (m_max - m_0) * (n_max - n_0) * kern_k);
+#endif
+
+ strat.kernel(// A and B pointers are just the packed panels.
+ a_ptr, b_panel,
+ // Provide relevant part of output array and row stride.
+ c_ptr + m_0 * ldc + n_0, ldc,
+ // M, N, K sizes
+ m_max-m_0, n_max - n_0, kern_k,
+ // Bias, activation, accumulation. Need to offset the bias as needed.
+ col_bias + n_0, qp, n_0, accumulate, acc_buff);
+}
+
+// Run a kernel and call the separate quantize step
+template<>
+template<typename strategy, typename To, typename Tr, typename Tri, typename Tab>
+void kernel_and_merge<true, Requantize32>::run(
+#ifdef CYCLE_PROFILING
+ profiler &prof,
+#endif
+ strategy &strat, const To *a_ptr, const To *b_panel, Tri *c_panel,
+ Tr *c_ptr, int ldc, int kern_k, unsigned int m_0,
+ unsigned int m_max, unsigned int n_0, unsigned int n_max, const Tr *,
+ const Activation &, bool, const Requantize32 &qp, const int32_t *col_bias,
+ Tab *)
+{
+ const int bblocks = iceildiv(n_max - n_0, strategy::out_width());
+
+ {
+#ifdef CYCLE_PROFILING
+ auto p=prof.ScopedProfiler(PROFILE_KERNEL, (strategy::out_height() * bblocks * strategy::out_width() * kern_k));
+#endif
+
+ strat.kernel(a_ptr, b_panel, c_panel, 1, bblocks, kern_k);
+ }
+
+ {
+#ifdef CYCLE_PROFILING
+ auto p=prof.ScopedProfiler(PROFILE_QUANTIZE, (strategy::out_height() * bblocks * strategy::out_width() * sizeof(Tr)));
+#endif
+ // The interleaved kernel outputs in blocks - each block is a
+ // row-major matrix of size out_width * out_height. The merge
+ // kernels are designed to deal with this but the requantizer is
+ // not, so we need to requantize one block at a time.
+ for (int i=0; i<bblocks; i++) {
+ unsigned int n_start = n_0 + (strategy::out_width() * i);
+ unsigned int n_end = std::min(n_start + strategy::out_width(), n_max);
+
+ // The row bias is interleaved with the transposed A data, get a pointer to it here.
+ const int32_t *row_bias = reinterpret_cast<const int32_t *>(a_ptr + strategy::out_height() * kern_k);
+
+ requantize_block_32(qp, (n_end - n_start), (m_max-m_0),
+ c_panel + (i * strategy::out_width() * strategy::out_height()), strategy::out_width(),
+ c_ptr + m_0 * ldc + n_start, ldc,
+ row_bias, col_bias + n_start, n_start);
+ }
+ }
+}
+
+// Integer GEMMs can be used in two contexts - "normal" where the full 32-bit output is required, or in
+// "requantizing" context where the output will be requantized.
+//
+// These require different input transforms, as if we are requantizing we want to sum the rows of the A input, and
+// if we are not we don't.
+//
+// This helper class allows the appropriate transforms to be found, without requiring kernels that don't support
+// quantization to define useless "quantized" transforms.
+template<typename strategy, bool quantized>
+class transform_type {
+public:
+ typedef decltype(strategy::transforms) type;
+};
+
+template<typename strategy>
+class transform_type<strategy, true> {
+public:
+ typedef decltype(strategy::transforms_quantized) type;
+};
+
+// We need a similar trick here to figure out what type the accumulator buffer should be.
+template<typename strategy, typename OutputStage>
+class accumulate_buffer_type {
+public:
+ typedef typename strategy::result_type type;
+};
+
+template<typename strategy>
+class accumulate_buffer_type<strategy, Requantize32> {
+public:
+ typedef int32_t type;
+};
+
+} // anonymous namespace
+
+template<typename strategy, typename To, typename Tr, typename OutputStage=Nothing, bool MergeStep=true, bool ForceThreadColumns=false>
class GemmInterleaved : public GemmCommon<To, Tr> {
typedef typename strategy::operand_type Toi;
typedef typename strategy::result_type Tri;
+ typedef typename accumulate_buffer_type<strategy, OutputStage>::type Tab;
/* const properties set by constructor */
const CPUInfo * const _ci;
@@ -59,10 +260,15 @@
const unsigned int _Msize;
const unsigned int _Nsize;
const unsigned int _Ksize;
+ const unsigned int _Ksections;
+ const unsigned int _Ktotal;
+ const unsigned int _rounded_Ksize;
const unsigned int _nbatches;
const unsigned int _nmulti;
+ const bool _thread_columns;
+
const Activation _act;
const int _maxthreads;
@@ -77,30 +283,59 @@
const Toi *_B_transposed=nullptr;
void *_working_space=nullptr;
+ Tab *_accumulation_buffer=nullptr;
+
+ /* Output stage */
+ OutputStage _os;
+
+ /* Quantized support (in addition to 'output stage' above */
+ int32_t *col_bias = nullptr;
+
+ /* Indirect parameters. _indirect_buf doubles as a flag to indicate that "indirect" transform should be used. */
+ const To * const * const * _indirect_buf = nullptr;
+
+ /* Convolver - only set up for convolution problems, so also doubles as a flag. */
+ std::unique_ptr<convolver<To>> _convolver = nullptr;
+
+ unsigned int get_col_sum_size() const {
+ if (std::is_same<OutputStage, Requantize32>::value) {
+ return _Nsize * _nmulti * sizeof(int32_t);
+ } else {
+ return 0;
+ }
+ }
+
/* We will need to walk through the blocks of B in a few contexts, so
* factor that out. */
class blockwalker {
private:
/* Size loops, etc. based on our parent's configuration */
- const GemmInterleaved<strategy, To, Tr> &_parent;
+ const GemmInterleaved<strategy, To, Tr, OutputStage, MergeStep, ForceThreadColumns> &_parent;
/* K, X and multi parameters for current iteration. */
unsigned int _k0=0, _x0=0, _multi=0;
+ /* Range of X to iterate over - used in "ForceThreadColumns" cases */
+ unsigned int _x_start=0;
+ unsigned int _x_end=_parent._Nsize;
+
unsigned int _index=0;
bool _done=false;
bool _newkblock=true;
bool _newmulti=true;
public:
- blockwalker(const GemmInterleaved<strategy, To, Tr> &parent) : _parent(parent) { }
+ blockwalker(const GemmInterleaved<strategy, To, Tr, OutputStage, MergeStep, ForceThreadColumns> &parent) : _parent(parent) { }
+
+ blockwalker(const GemmInterleaved<strategy, To, Tr, OutputStage, MergeStep, ForceThreadColumns> &parent,
+ unsigned int x_start, unsigned int x_end) : _parent(parent), _x0 (_x_start), _x_start(x_start), _x_end(x_end) { }
unsigned int xmax() {
- return std::min(_x0 + _parent._x_block, _parent._Nsize);
+ return std::min(_x0 + _parent._x_block, _x_end);
}
unsigned int kmax() {
- return std::min(_k0 + _parent._k_block, _parent._Ksize);
+ return std::min(_k0 + _parent._k_block, _parent._Ktotal);
}
/* Advance to the next block, return false at the end. */
@@ -111,10 +346,10 @@
_newkblock=false;
_x0 += _parent._x_block;
- if (_x0 >= _parent._Nsize) {
- _x0=0;
+ if (_x0 >= _x_end) {
+ _x0=_x_start;
_k0 += _parent._k_block;
- if (_k0 >= _parent._Ksize) {
+ if (_k0 >= _parent._Ktotal) {
_k0=0;
_multi++;
if (_multi >= _parent._nmulti) {
@@ -138,14 +373,125 @@
bool newkblock(void) { return _newkblock; }
};
- // A working size: One of these needed, regardless of thread count. Divided according to window.
- size_t get_a_working_size() const {
- return ROUND_UP(sizeof(Toi) * _k_block * _Mround * _nbatches);
+ // "k block" has two distinct uses: figuring out which iterations of K
+ // to actually process, but also various size/pointer computations. The
+ // latter needs to take account of the extra space needed for the row
+ // sums, if appropriate.
+ unsigned int get_total_k_depth() const {
+ unsigned int k_depth = _k_block;
+
+ if (std::is_same<OutputStage, Requantize32>::value) {
+ k_depth += sizeof(int32_t) / sizeof(Toi);
+ }
+
+ return k_depth;
}
- // C working size: One needed per thread.
+ // A working size.
+ size_t get_a_working_size() const {
+ if (_thread_columns) {
+ // For 2D threading: allocate a buffer of one block of rows per thread
+ return ROUND_UP(sizeof(Toi) * get_total_k_depth() * strategy::out_height() * _maxthreads);
+ } else {
+ // For 1D threaded: one of these needed, regardless of thread count. Divided according to window.
+ return ROUND_UP(sizeof(Toi) * get_total_k_depth() * _Mround * _nbatches);
+ }
+ }
+
+ // C working size: One needed per thread. Not needed if there is no merge step.
size_t get_c_working_size() const {
- return ROUND_UP(sizeof(Tri) * _x_block * strategy::out_height());
+ if (MergeStep) {
+ return ROUND_UP(sizeof(Tri) * _x_block * strategy::out_height());
+ } else {
+ return 0;
+ }
+ }
+
+ // Accumulation buffer size
+ size_t get_accumulation_buffer_size() const {
+ // We only support an accumulation buffer for non-merge cases.
+ if (MergeStep) {
+ return 0;
+ }
+
+ // Check if we are actually blocking
+ if (_k_block == _Ktotal) {
+ return 0;
+ }
+
+ // We are no-merge, non-quantized with active blocking: accumulation buffer needed.
+ size_t size_per_buffer = sizeof(Tab) * strategy::out_height() * strategy::out_width();
+ size_t num_buffers = iceildiv(_Msize, strategy::out_height()) * iceildiv(_Nsize, strategy::out_width()) * _nbatches * _nmulti;
+
+ return num_buffers * size_per_buffer;
+ }
+
+ // Get pointer into accumulation buffer
+ Tab *get_accumulation_buffer(unsigned int M, unsigned int N, unsigned int batch, unsigned int multi) const {
+ // Don't do anything if there's no buffer.
+ if (_accumulation_buffer == nullptr) {
+ return nullptr;
+ }
+
+ // Here we are indexing an appropriately sized pointer, so no sizeof() needed to convert to bytes.
+ size_t size_per_buffer = strategy::out_height() * strategy::out_width();
+
+ size_t buffer_rows = iceildiv(_Msize, strategy::out_height());
+ size_t buffer_cols = iceildiv(_Nsize, strategy::out_width());
+ size_t buffers_per_batch = (buffer_rows * buffer_cols);
+ size_t buffers_per_multi = buffers_per_batch * _nbatches;
+
+ // M/N must reference the top-left corner of a block.
+ size_t row = M / strategy::out_height();
+ assert(M % strategy::out_height() == 0);
+ size_t col = N / strategy::out_width();
+ assert(N % strategy::out_width() == 0);
+
+ size_t buffer_index = multi * buffers_per_multi + batch * buffers_per_batch + row * buffer_cols + col;
+
+ return _accumulation_buffer + (buffer_index * size_per_buffer);
+ }
+
+ int32_t row_sum_multiplier() const {
+ if (std::is_same<OutputStage, Requantize32>::value) {
+ const Requantize32 *qp = reinterpret_cast<const Requantize32 *>(&_os);
+
+ return -qp->b_offset;
+ }
+
+ return 0;
+ }
+
+ // Heuristics to decide whether to use the 'thread columns' regime
+ static bool is_thread_columns(const GemmArgs &args) {
+ // For now, there is a templace parameter to force it.
+ if (ForceThreadColumns) {
+ return true;
+ }
+
+ // Never do this for single threaded cases.
+ if (args._maxthreads == 1) {
+ return false;
+ }
+
+ // How many blocks of work are available for threading on M?
+ int m_blocks = iceildiv(args._Msize, strategy::out_height()) * args._nbatches;
+
+ // If we just can't share the work across threads with the row threading regime.
+ if (args._maxthreads > m_blocks) {
+ return true;
+ }
+
+ // If the row threading regime is too wasteful (20% threshold)
+ if (((roundup(m_blocks, args._maxthreads) * 100) / m_blocks) > 120) {
+ return true;
+ }
+
+ return false;
+ }
+
+ static unsigned int get_ktotal(const GemmArgs &args) {
+ return args._Ksections * roundup(args._Ksize, strategy::k_unroll());
}
static unsigned int get_k_block_size(const GemmArgs &args) {
@@ -153,6 +499,11 @@
return args._cfg->inner_block_size;
}
+ // K blocking not supported if we are requantizing.
+ if (std::is_same<OutputStage, Requantize32>::value) {
+ return get_ktotal(args);
+ }
+
const unsigned int L1_size = args._ci->get_L1_cache_size();
unsigned int k_block;
@@ -165,57 +516,83 @@
k_block = std::max(k_block, 1U) * strategy::k_unroll();
// Now tune to presented problem size; this is how many blocks we need.
- unsigned int num_k_blocks = iceildiv(args._Ksize, k_block);
+ unsigned int num_k_blocks = iceildiv(get_ktotal(args), k_block);
// So divide the space equally into that many blocks.
- k_block = iceildiv(args._Ksize, num_k_blocks);
+ k_block = iceildiv(get_ktotal(args), num_k_blocks);
// And round UP to the K unroll level required.
k_block = roundup(k_block, strategy::k_unroll());
+ assert(k_block > 0);
+
return k_block;
}
+ static unsigned int get_x_block_size(const GemmArgs &args) {
+ if (is_thread_columns(args)) {
+ // In 2D mode, override X block, because we will process width first.
+ return roundup(args._Nsize, strategy::out_width());
+ }
+
+ if (args._cfg && args._cfg->outer_block_size) {
+ return roundup(args._cfg->outer_block_size, strategy::out_width());
+ }
+
+ unsigned int x_block;
+ const unsigned int L2_size = args._ci->get_L2_cache_size();
+ const unsigned int k_block = get_k_block_size(args);
+
+ // x_block: Work out how many rows (of length k_block) will fit in the L2
+ // Don't allocate more than 90% of the L2 to allow for overheads, and subtract off the L1 contents.
+ const unsigned int scaled_l2_size = (L2_size * 9) / 10;
+ const unsigned int k_block_area = k_block * sizeof(Toi) * (strategy::out_width() + strategy::out_height());
+
+ // .. if the L1 contents is bigger than the L2, just return a minimal size block.
+ if (k_block_area > scaled_l2_size) {
+ return strategy::out_width();
+ }
+
+ x_block = (scaled_l2_size - k_block_area) / (sizeof(Toi) * k_block);
+
+ // Needs to be (at least a single) multiple of the kernel output width.
+ x_block /= strategy::out_width();
+ x_block = std::max(x_block, 1u) * strategy::out_width();
+
+ // And tune to the presented problem size.
+ unsigned int num_x_blocks = iceildiv(args._Nsize, x_block);
+ x_block = iceildiv(args._Nsize, num_x_blocks);
+
+ x_block = roundup(x_block, strategy::out_width());
+
+ assert(x_block > 0);
+
+ return x_block;
+ }
+
public:
GemmInterleaved(GemmInterleaved &) = delete;
GemmInterleaved & operator= (GemmInterleaved &) = delete;
/* Constructor */
+ GemmInterleaved(const GemmArgs &args, const OutputStage &os)
+ : _ci(args._ci), _Msize(args._Msize), _Nsize(args._Nsize), _Ksize(args._Ksize),
+ _Ksections(args._Ksections), _Ktotal(get_ktotal(args)),
+ _rounded_Ksize(roundup(_Ksize, strategy::k_unroll())),
+ _nbatches(args._nbatches), _nmulti(args._nmulti), _thread_columns(is_thread_columns(args)),
+ _act(args._act), _maxthreads(args._maxthreads), _nthreads(args._maxthreads),
+ _k_block(get_k_block_size(args)), _x_block(get_x_block_size(args)), _Mround(roundup(args._Msize, strategy::out_height())),
+ _os(os) { }
+
+ /* Constructor without OutputStage */
GemmInterleaved(const GemmArgs &args)
: _ci(args._ci), _Msize(args._Msize), _Nsize(args._Nsize), _Ksize(args._Ksize),
- _nbatches(args._nbatches), _nmulti(args._nmulti),
+ _Ksections(args._Ksections), _Ktotal(get_ktotal(args)),
+ _rounded_Ksize(roundup(_Ksize, strategy::k_unroll())),
+ _nbatches(args._nbatches), _nmulti(args._nmulti), _thread_columns(is_thread_columns(args)),
_act(args._act), _maxthreads(args._maxthreads), _nthreads(args._maxthreads),
- _k_block(get_k_block_size(args)) {
- const unsigned int L2_size = _ci->get_L2_cache_size();
-
- assert(_maxthreads > 0);
-
- // Work out blocking parameters, or override from provided GemmConfig
- // TODO: Move outer block into a static function too.
- if (args._cfg && args._cfg->outer_block_size) {
- _x_block = args._cfg->outer_block_size;
- } else {
- // x_block: Work out how many rows (of length k_block) will fit in the L2
- // Don't allocate more than 90% of the L2 to allow for overheads, and subtract off the L1 contents.
- _x_block = (((L2_size * 9) / 10) - (_k_block * sizeof(Toi) * (strategy::out_width() + strategy::out_height()))) /
- (sizeof(Toi) * _k_block);
-
- // Needs to be (at least a single) multiple of the kernel output width.
- _x_block /= strategy::out_width();
- _x_block = std::max(_x_block, 1U) * strategy::out_width();
-
- // And tune to the presented problem size.
- unsigned int num_x_blocks = iceildiv(_Nsize, _x_block);
- _x_block = iceildiv(_Nsize, num_x_blocks);
-
- _x_block = iceildiv(_x_block, strategy::out_width());
- _x_block *= strategy::out_width();
- }
-
- // Work out the rounded size of M - needed for some buffers.
- _Mround = iceildiv(_Msize, strategy::out_height());
- _Mround *= strategy::out_height();
- }
+ _k_block(get_k_block_size(args)), _x_block(get_x_block_size(args)), _Mround(roundup(args._Msize, strategy::out_height())),
+ _os() { }
// Interface implementation - Compulsory functions
@@ -224,8 +601,14 @@
// not multi for now (as this would cause problems with the buffer
// manager).
ndrange_t get_window_size() const override {
- // _Mround is a multiple of out_height by definition.
- return { (_Mround / strategy::out_height()) * _nbatches };
+ unsigned int row_blocks = (_Mround / strategy::out_height()) * _nbatches;
+
+ if (_thread_columns) {
+ return { row_blocks, iceildiv(_Nsize, strategy::out_width()) };
+ } else {
+ // _Mround is a multiple of out_height by definition.
+ return { row_blocks };
+ }
}
// set_nthreads: pass on to buffer manager to avoid it waiting for non-existant threads.
@@ -235,117 +618,262 @@
// Execute
void execute(const ndcoord_t &work_range, const ndcoord_t &, int threadid) override {
- const auto start = work_range.get_position(0);
- const auto end = work_range.get_position_end(0);
#ifdef CYCLE_PROFILING
profiler prof;
#endif
- strategy strat(_ci);
-
- blockwalker current(*this);
-
- /* Translate 'start' and 'end' into a position within the batches and rows. */
- const unsigned int window_per_batch = _Mround / strategy::out_height();
- unsigned int batch_0 = start / window_per_batch;
- unsigned int batch_end = end / window_per_batch;
-
- /* Compute the M values to operate on */
- unsigned int m_0 = (start - (batch_0 * window_per_batch)) * strategy::out_height();
- unsigned int m_max = (end - (batch_end * window_per_batch)) * strategy::out_height();
/* Make sure we've been set up correctly. */
assert(_B_transposed);
assert(_working_space);
int8_t *working_space_bytes = reinterpret_cast<int8_t *>(_working_space);
- // Private buffers. Treat working_space as an array of C buffers
- // (one per thread) first, followed by the (window-divided) A
- // buffer.
- // Set a_panel to the base of the A buffers - compute offsets into it based on M/batches later.
- Toi * const a_panel = reinterpret_cast<Toi *>(working_space_bytes + (_maxthreads * get_c_working_size()));
- Tri * const c_panel = reinterpret_cast<Tri *>(working_space_bytes + (threadid * get_c_working_size()));
+ /* Align if needed */
+ intptr_t working_space_v = reinterpret_cast<intptr_t>(_working_space);
+ if (working_space_v & 0x3f) {
+ intptr_t alignment_offset = 0x40 - (working_space_v & 0x3f);
+ working_space_bytes += alignment_offset;
+ }
- const Toi *b_panel;
- b_panel = _B_transposed;
+ strategy strat(_ci);
- //printf("Starting GEMM loop, x_block=%d, k_block=%d\n", _x_block, _k_block);
+ const auto start = work_range.get_position(0);
+ const auto end = work_range.get_position_end(0);
- // newkblock() is always true on the first iteration, so this will be set properly on the first loop.
- int kern_k = 0;
+ /* Translate 'start' and 'end' into a position within the batches and rows. */
+ const unsigned int window_per_batch = _Mround / strategy::out_height();
+ unsigned int batch_0 = start / window_per_batch;
+ unsigned int batch_end = end / window_per_batch;
- for (;!current.done();current.advance()) {
- if (current.newkblock()) {
+ // In ThreadColumns mode, process work one horizontal strip at a time.
+ // Transpose the block of needed rows at the start, then do all the work on that block.
+ if (_thread_columns) {
+ const auto start_x = work_range.get_position(1) * strategy::out_width();
+ const auto end_x = std::min(work_range.get_position_end(1) * strategy::out_width(), _Nsize);
+
+ Tri * const c_panel = reinterpret_cast<Tri *>(working_space_bytes + (threadid * get_c_working_size()));
+ Toi * const a_panel = reinterpret_cast<Toi *>(working_space_bytes + (_maxthreads * get_c_working_size()) +
+ (threadid * sizeof(Toi) * get_total_k_depth() * strategy::out_height()));
+
+ for (unsigned int multi=0; multi<_nmulti; multi++) {
+ for (unsigned int k0=0; k0<_Ktotal; k0+=_k_block) {
+ unsigned int kmax=std::min(k0+_k_block, _Ktotal);
+
+ unsigned int rounded_width = roundup(_Nsize, strategy::out_width());
+
+ const bool first_pass = (k0==0);
+ const bool last_pass = (kmax==_Ktotal);
+
+ // Figure out how many "K" the kernel will actually process.
+ unsigned int kern_k = roundup(kmax - k0, strategy::k_unroll());
+
+ const Toi *b_ptr = _B_transposed + (rounded_width * _Ktotal * multi) + (k0 * rounded_width) + (start_x * kern_k);
+
+ unsigned int batch = batch_0;
+ unsigned int start_row = (start - (batch_0 * window_per_batch)) * strategy::out_height();
+
+ for (unsigned int p=start; p<end; p++) {
+ unsigned int end_row = std::min(start_row + strategy::out_height(), _Msize);
+
+ // Set up transposed 'A' block
+ {
#ifdef CYCLE_PROFILING
- auto p=prof.ScopedProfiler(PROFILE_PREPA, (end - start) * strategy::out_height() * (current.kmax()-current.k0()) * sizeof(Toi));
+ auto p=prof.ScopedProfiler(PROFILE_PREPA, strategy::out_height() * (kmax-k0) * sizeof(Toi));
#endif
+ // See comment above on transform_type<> class: this extracts either 'transforms' or
+ // 'transforms_quantized' as appropriate.
+ typename transform_type<strategy, MergeStep && std::is_same<OutputStage, Requantize32>::value>::type transforms;
+
+ if (_indirect_buf != nullptr) {
+ transforms.PrepareA_indirect(a_panel,
+ _indirect_buf + (multi * _nbatches * _Ksections) + (batch * _Ksections), _Ksize,
+ _rounded_Ksize, start_row, end_row, k0, kmax, row_sum_multiplier());
+ } else if (_convolver) {
+ transforms.PrepareA_convolution(a_panel,
+ this->_Aptr + (batch * this->_A_batch_stride) + (multi * this->_A_multi_stride),
+ this->_lda, *_convolver, _rounded_Ksize, start_row, end_row, k0, kmax, row_sum_multiplier());
+ } else {
+ transforms.PrepareA(a_panel,
+ this->_Aptr + (batch * this->_A_batch_stride) + (multi * this->_A_multi_stride),
+ this->_lda, start_row, end_row, k0, std::min(kmax, _Ksize), row_sum_multiplier());
+ }
+ }
+
+ // Perform the kernel and merge step, either separately or together as required.
+ kernel_and_merge<MergeStep, OutputStage>::run(
+ #ifdef CYCLE_PROFILING
+ prof,
+ #endif
+ // Strategy and panel pointers
+ strat, a_panel, b_ptr, c_panel,
+ // Result buffer pointers
+ this->_Cptr + (batch * this->_C_batch_stride) + (multi * this->_C_multi_stride), this->_ldc,
+ // K size, and M/N ranges
+ kern_k, start_row, end_row, start_x, end_x,
+ // Only do bias on the first pass
+ ((first_pass && this->_bias) ? this->_bias + (multi * this->_bias_multi_stride) : nullptr),
+ // Only do activation on the last pass, and accumulation on any non-first pass.
+ (last_pass ? _act : Activation()), !first_pass,
+ // Pass in quantization parameters for requantizing kernels (others will ignore)
+ _os, col_bias + (multi * _Nsize),
+ // Accumulation buffer (not yet implemented on this path)
+ static_cast<Tab *>(nullptr));
+
+ /* Increment to the next block */
+ start_row += strategy::out_height();
+ if (start_row >= _Msize) {
+ start_row = 0;
+ batch++;
+ }
+ }
+ }
+ }
+ } else {
+ blockwalker current(*this);
+
+ /* Compute the M values to operate on */
+ unsigned int m_0 = (start - (batch_0 * window_per_batch)) * strategy::out_height();
+ unsigned int m_max = (end - (batch_end * window_per_batch)) * strategy::out_height();
+
+ // Private buffers. Treat working_space as an array of C buffers
+ // (one per thread) first, followed by the (window-divided) A
+ // buffer.
+ // Set a_panel to the base of the A buffers - compute offsets into it based on M/batches later.
+ Toi * const a_panel = reinterpret_cast<Toi *>(working_space_bytes + (_maxthreads * get_c_working_size()));
+ Tri * const c_panel = reinterpret_cast<Tri *>(working_space_bytes + (threadid * get_c_working_size()));
+
+ const Toi *b_panel;
+ b_panel = _B_transposed;
+
+ // newkblock() is always true on the first iteration, so these will be set properly on the first loop.
+
+ // kern_k tracks the accumulation depth for the CURRENT K block a_panel_stride similarly tracks the total
+ // stride of the A panel (i.e. with 4 added for cases with embedded row sums)
+
+ // These are distinct from k_block and get_total_k_depth() which are based on the target K block size, and
+ // used for addressing inside a_panel.
+
+ // In cases where K blocking is in use and the blocks are not all the same size, the (smaller) final block
+ // won't use all the memory allocated.
+ unsigned int kern_k = 0;
+ unsigned int a_panel_stride = 0;
+
+ for (;!current.done();current.advance()) {
+ if (current.newkblock()) {
+#ifdef CYCLE_PROFILING
+ auto p=prof.ScopedProfiler(PROFILE_PREPA, (end - start) * strategy::out_height() * (current.kmax()-current.k0()) * sizeof(Toi));
+#endif
+ // See comment above on transform_type<> class: this extracts either 'transforms' or
+ // 'transforms_quantized' as appropriate.
+ typename transform_type<strategy, MergeStep && std::is_same<OutputStage, Requantize32>::value>::type transforms;
+
+ for (unsigned int batch = batch_0; batch <= batch_end; batch++) {
+ unsigned int first_m = (batch == batch_0) ? m_0 : 0;
+ unsigned int last_m = (batch == batch_end) ? m_max : _Msize;
+
+ if (first_m >= last_m)
+ continue;
+
+ if (_indirect_buf != nullptr) {
+ transforms.PrepareA_indirect(a_panel + ((batch * _Mround + first_m) * get_total_k_depth()),
+ _indirect_buf + (current.multi() * _nbatches * _Ksections) + (batch * _Ksections), _Ksize,
+ _rounded_Ksize, first_m, last_m, current.k0(), current.kmax(), row_sum_multiplier());
+ } else if (_convolver) {
+ transforms.PrepareA_convolution(a_panel + ((batch * _Mround + first_m) * get_total_k_depth()),
+ this->_Aptr + (batch * this->_A_batch_stride) + (current.multi() * this->_A_multi_stride),
+ this->_lda, *_convolver, _rounded_Ksize, first_m, last_m, current.k0(), current.kmax(), row_sum_multiplier());
+ } else {
+ transforms.PrepareA(a_panel + ((batch * _Mround + first_m) * get_total_k_depth()),
+ this->_Aptr + (batch * this->_A_batch_stride) + (current.multi() * this->_A_multi_stride),
+ this->_lda, first_m, last_m, current.k0(), std::min(_Ksize, current.kmax()), row_sum_multiplier());
+ }
+ }
+
+ // Figure out how many "K" the kernel will actually process.
+ kern_k = roundup(current.kmax() - current.k0(), strategy::k_unroll());
+
+ // Requantizing GEMMs have the row sums built in to the
+ // transposed data, so the stride between rows is 4 bytes
+ // larger than the (rounded) K value.
+
+ if(std::is_same<OutputStage, Requantize32>::value) {
+ a_panel_stride = kern_k + (sizeof(int32_t) / sizeof(Toi));
+ } else {
+ a_panel_stride = kern_k;
+ }
+ }
+
+ /* Do the actual work. */
for (unsigned int batch = batch_0; batch <= batch_end; batch++) {
unsigned int first_m = (batch == batch_0) ? m_0 : 0;
unsigned int last_m = (batch == batch_end) ? m_max : _Msize;
+ const Toi *a_ptr = a_panel + (batch * _Mround + first_m) * get_total_k_depth();
+
if (first_m >= last_m)
continue;
- strat.transforms.PrepareA(a_panel + ((batch * _Mround + first_m) * _k_block),
- this->_Aptr + (batch * this->_A_batch_stride) + (current.multi() * this->_A_multi_stride),
- this->_lda, first_m, last_m, current.k0(), current.kmax());
- }
+ // For the merge case we need to do this out_height() rows
+ // at a time, as that is the size of our intermediate
+ // buffer. If we are not doing that, we can do all the
+ // relevant rows in one go.
+ unsigned int m_step = MergeStep ? strategy::out_height() : (last_m - first_m);
- // Figure out how many "K" the kernel will actually process.
- kern_k = iceildiv(current.kmax() - current.k0(), strategy::k_unroll());
- kern_k *= strat.k_unroll();
- }
-
- int bblocks = iceildiv(current.xmax() - current.x0(), strategy::out_width());
-
- /* Do the actual work. */
- for (unsigned int batch = batch_0; batch <= batch_end; batch++) {
- unsigned int first_m = (batch == batch_0) ? m_0 : 0;
- unsigned int last_m = (batch == batch_end) ? m_max : _Msize;
-
- const Toi *a_ptr = a_panel + (batch * _Mround + first_m) * _k_block;
-
- if (first_m >= last_m)
- continue;
-
- for (unsigned int y=first_m; y<last_m; y+=strategy::out_height()) {
- unsigned int ymax = std::min(_Msize, y + strategy::out_height());
-
- {
-#ifdef CYCLE_PROFILING
- auto p=prof.ScopedProfiler(PROFILE_KERNEL, (strategy::out_height() * bblocks * strategy::out_width() * kern_k));
-#endif
-
- strat.kernel(a_ptr, b_panel, c_panel, 1, bblocks, kern_k);
-
- a_ptr += (strategy::out_height() * kern_k);
+ // But in the case where we have an accumulation buffer, we can't do that after all, unless
+ // there is no N blocking.
+ if (_accumulation_buffer && ((current.x0() != 0) || (current.xmax() < _Nsize))) {
+ m_step = strategy::out_height();
}
- {
-#ifdef CYCLE_PROFILING
- auto p=prof.ScopedProfiler(PROFILE_MERGE, (strategy::out_height() * bblocks * strategy::out_width() * sizeof(Tr)));
-#endif
- /* Only activate on last pass, only add bias on first pass, ask for accumulation on any non-first pass */
- const bool first_pass = current.k0()==0;
- const bool last_pass = current.kmax()==_Ksize;
+ for (unsigned int y=first_m; y<last_m; y+=m_step) {
+ unsigned int ymax = std::min(_Msize, y + m_step);
- strat.transforms.Merge(this->_Cptr + (batch * this->_C_batch_stride) + (current.multi() * this->_C_multi_stride),
- c_panel, this->_ldc, y, ymax, current.x0(), current.xmax(),
- ((first_pass && this->_bias) ? this->_bias + (current.multi() * this->_bias_multi_stride) : nullptr),
- (last_pass ? _act : Activation()), !first_pass);
+ const bool first_pass = (current.k0() == 0);
+ const bool last_pass = (current.kmax() == _Ktotal);
+
+ // Pointer to appropriate part of result array.
+ Tr *result_ptr = this->_Cptr + (batch * this->_C_batch_stride) + (current.multi() * this->_C_multi_stride);
+
+ // If we are using an accumulation buffer, we don't pass the result buffer to ask the kernel
+ // to write things into the accumulation buffer instead, except on the last pass.
+ if (_accumulation_buffer && !last_pass) {
+ result_ptr = nullptr;
+ }
+
+ // Perform the kernel and merge step, either separately or together as required.
+ kernel_and_merge<MergeStep, OutputStage>::run(
+ #ifdef CYCLE_PROFILING
+ prof,
+ #endif
+ // Strategy and panel pointers
+ strat, a_ptr, b_panel, c_panel,
+ // Result buffer pointers
+ result_ptr, this->_ldc,
+ // K size, and M/N ranges
+ kern_k, y, ymax, current.x0(), current.xmax(),
+ // Only do bias on the first pass
+ ((first_pass && this->_bias) ? this->_bias + (current.multi() * this->_bias_multi_stride) : nullptr),
+ // Only do activation on the last pass, and accumulation on any non-first pass.
+ (last_pass ? _act : Activation()), !first_pass,
+ // Pass in quantization parameters for requantizing kernels (others will ignore)
+ _os, col_bias + (current.multi() * _Nsize),
+ // Accumulation buffer
+ get_accumulation_buffer(y, current.x0(), batch, current.multi()) );
+
+ a_ptr += (strategy::out_height() * a_panel_stride);
}
}
- }
- b_panel += (bblocks * strat.out_width() * kern_k);
+ b_panel += (roundup(current.xmax() - current.x0(), strategy::out_width()) * kern_k);
+ }
}
}
// Interface implementation - working space
size_t get_working_size() const override {
- // In all cases, we need one A buffer plus a C buffer per thread.
- size_t size = get_a_working_size() + (get_c_working_size() * _maxthreads);
+ // In all cases, we need one A buffer plus a C buffer per thread, plus an accumulation buffer.
+ size_t size = get_a_working_size() + (get_c_working_size() * _maxthreads) + get_accumulation_buffer_size();
- size += 64; // Add on a cache line extra for alignment.
+ size += 128; // Add on two cache lines extra for alignment.
return size;
}
@@ -362,9 +890,22 @@
}
working_space_bytes += diff;
+ working_space_int += diff;
// Pretransposed case: just set internal pointer to parameter value.
_working_space = reinterpret_cast<void *>(working_space_bytes);
+
+ // Set up accumulation buffer
+ if (get_accumulation_buffer_size() > 0) {
+ intptr_t acc_buff_int = working_space_int + get_a_working_size() + (get_c_working_size() * _maxthreads);
+ // Make sure the accumulation buffer is aligned (needed if the other blocks are not a multiple of cache line length)
+ if (acc_buff_int & 0x3F) {
+ acc_buff_int += (0x40 - (acc_buff_int & 0x3F));
+ }
+ _accumulation_buffer = reinterpret_cast<Tab *>(acc_buff_int);
+ } else {
+ _accumulation_buffer = nullptr;
+ }
}
// Interface implementation - pretransposed
@@ -376,56 +917,105 @@
return (_B_transposed==nullptr);
}
- // TODO: this could almost certainly be considerably simpler.
size_t get_B_pretransposed_array_size() const override {
- size_t total=0;
- blockwalker current(*this);
+ unsigned int x_size = roundup(_Nsize, strategy::out_width());
- do {
- /* Figure out the size of each block. */
- unsigned int x_size = (current.xmax() - current.x0());
- unsigned int k_size = (current.kmax() - current.k0());
-
- /* Round sizes up as needed. */
- x_size = iceildiv(x_size, strategy::out_width());
- x_size *= strategy::out_width();
-
- k_size = iceildiv(k_size, strategy::k_unroll());
- k_size *= strategy::k_unroll();
-
- total += x_size * k_size * sizeof(Toi);
- } while (current.advance());
-
- return total;
+ return (x_size * _Ktotal * _nmulti * sizeof(Toi)) + get_col_sum_size();
}
void pretranspose_B_array(void *in_buffer, const To *B, const int ldb, const int B_multi_stride) override {
- blockwalker current(*this);
- Toi *buffer = reinterpret_cast<Toi *>(in_buffer);
+ if (std::is_same<OutputStage, Requantize32>::value) {
+ col_bias = reinterpret_cast<int32_t *>(in_buffer);
+
+ Requantize32 *qp_ptr = reinterpret_cast<Requantize32 *>(&_os);
+
+ for (unsigned int i=0; i<_nmulti; i++) {
+ // The input is assumed not to have any padding between sections, so straightforward Ksize * Ksections computation gets the total size.
+ compute_col_sums(*qp_ptr, _Nsize, _Ksize * _Ksections, B + (i * B_multi_stride), ldb, col_bias + (i * _Nsize), _Ksize * _Ksections, i, 0);
+ }
+ }
+
+ // Put the transposed data after the column sums - in non-transposing cases get_col_sum_size() == 0
+ uintptr_t buffer_int = reinterpret_cast<uintptr_t>(in_buffer);
+ Toi *buffer = reinterpret_cast<Toi *>(buffer_int + get_col_sum_size());
_B_transposed = buffer;
+
+ blockwalker current(*this);
strategy strat(_ci);
do {
/* Figure out the size of each block. */
- unsigned int x_size = (current.xmax() - current.x0());
unsigned int k_size = (current.kmax() - current.k0());
- /* Round sizes up as needed. */
- x_size = iceildiv(x_size, strategy::out_width());
- x_size *= strategy::out_width();
+ // We need to insert padding at the end of each K section.
+ // The computation needed is a little delicate - the coordinates from the block walker are expressed in
+ // terms of the full, padded, _Ktotal.
+ // But we need to transform each section with reference to the original, unpadded, input, letting the
+ // transform pad each section as needed.
- k_size = iceildiv(k_size, strategy::k_unroll());
- k_size *= strategy::k_unroll();
+ // This is needed for computations below.
+ const unsigned int rounded_section_size = roundup(_Ksize, strategy::k_unroll());
- strat.transforms.PrepareB(buffer, B + (current.multi() * B_multi_stride), ldb,
- current.x0(), current.xmax(), current.k0(), current.kmax());
+ // The expected output format is also an entire <out_width> columns interleaved, then the next set of
+ // columns, and so on. This means, as we are breaking it up vertically, we have to do it one column at
+ // a time.
+ for (unsigned int x0=current.x0(); x0 < current.xmax(); x0 += strategy::out_width() ){
+ unsigned int xmax = std::min(x0 + strategy::out_width(), current.xmax());
- buffer += (x_size * k_size);
+ // Track where we are and how much work is left.
+ unsigned int kpos = current.k0();
+ unsigned int kleft = k_size;
+
+ while (kleft) {
+ // Which section are we in? Based on the rounded-up section size.
+ unsigned int k_section_base = kpos / rounded_section_size;
+ // How far into the section are we?
+ unsigned int k_offset = kpos - (k_section_base * rounded_section_size);
+
+ // We will either copy the rest of this section, or to the end of the requested length.
+ unsigned int k_length = std::min(_Ksize - k_offset, kleft);
+
+ strat.transforms.PrepareB(buffer, B + (current.multi() * B_multi_stride), ldb,
+ x0, xmax,
+ (k_section_base * _Ksize) + k_offset, // K starting point - compute row to read based on our section and the true section length.
+ (k_section_base * _Ksize) + k_offset + k_length); // K end point - starting point plus length computed above.
+
+ // We need to modify our position based on the ROUNDED version of what we just did.
+ unsigned int padded_length = roundup(k_length, strategy::k_unroll());
+
+ buffer += strategy::out_width() * padded_length;
+
+ kpos += padded_length;
+ kleft -= padded_length;
+ }
+ }
} while (current.advance());
}
void set_pretransposed_B_data(void *in_buffer) override {
- _B_transposed = reinterpret_cast<Toi *>(in_buffer);
+ // Put the transposed data after the column sums - in non-transposing cases get_col_sum_size() == 0
+ uintptr_t buffer_int = reinterpret_cast<uintptr_t>(in_buffer);
+ _B_transposed = reinterpret_cast<Toi *>(buffer_int + get_col_sum_size());
+ col_bias = reinterpret_cast<int32_t *>(in_buffer);
+ }
+
+ void set_quantized_bias(const int32_t *bias, size_t bias_multi_stride) override {
+ if (std::is_same<OutputStage, Requantize32>::value) {
+ Requantize32 *qp = reinterpret_cast<Requantize32 *>(&_os);
+
+ qp->bias = bias;
+ qp->bias_multi_stride = bias_multi_stride;
+ }
+ }
+
+ void set_indirect_parameters(size_t string_len, const To * const * const *ptr) override {
+ assert(string_len == _Ksize);
+ _indirect_buf = ptr;
+ }
+
+ void set_convolution_parameters(ConvolutionParameters parms) override {
+ assert(parms.input_channels == _Ksize);
+ _convolver = std::unique_ptr<convolver<To>>(new convolver<To>(parms));
}
// Estimate cycles for given problem given provided parameters
@@ -454,4 +1044,14 @@
}
};
+// Aliases for the variations
+template<typename strategy, typename To, typename Tr, typename OutputStage=Nothing>
+using GemmInterleavedNoMerge = GemmInterleaved<strategy, To, Tr, OutputStage, false>;
+
+template<typename strategy, typename To, typename Tr>
+using GemmInterleavedPretransposedNoMergeQuantizedInline = GemmInterleaved<strategy, To, Tr, Requantize32, false>;
+
+template<typename strategy, typename To, typename Tr>
+using GemmInterleavedQuantized = GemmInterleaved<strategy, To, Tr, Requantize32>;
+
} // namespace arm_gemm
diff --git a/src/core/NEON/kernels/arm_gemm/gemm_interleaved_pretransposed_2d.hpp b/src/core/NEON/kernels/arm_gemm/gemm_interleaved_pretransposed_2d.hpp
index bdccd05..b71f390 100644
--- a/src/core/NEON/kernels/arm_gemm/gemm_interleaved_pretransposed_2d.hpp
+++ b/src/core/NEON/kernels/arm_gemm/gemm_interleaved_pretransposed_2d.hpp
@@ -250,7 +250,8 @@
first_m,
last_m,
current.k0(),
- current.kmax());
+ current.kmax(),
+ 0);
}
}
diff --git a/src/core/NEON/kernels/arm_gemm/gemm_qint8.cpp b/src/core/NEON/kernels/arm_gemm/gemm_qint8.cpp
index 04cac60..05c5116 100644
--- a/src/core/NEON/kernels/arm_gemm/gemm_qint8.cpp
+++ b/src/core/NEON/kernels/arm_gemm/gemm_qint8.cpp
@@ -25,68 +25,151 @@
#include "arm_gemm.hpp"
-#include "kernels/a64_hybrid_s8s32_dot_16x4.hpp"
-#include "kernels/a64_smallK_hybrid_s8s32_dot_4x6.hpp"
-#include "kernels/a64_smallK_hybrid_s8s32_dot_4x8.hpp"
-#include "kernels/sve_hybrid_s8s32_dot_4VLx4.hpp"
-#include "kernels/sve_smallK_hybrid_s8s32_dot_1VLx8.hpp"
+#include "kernels/a64_gemm_s16_8x12.hpp"
+#include "kernels/a64_gemm_s8_4x4.hpp"
+#include "kernels/a64_gemm_s8_8x12.hpp"
+#include "kernels/a64_hybrid_s8qa_dot_4x16.hpp"
+#include "kernels/a64_hybrid_s8qs_dot_6x16.hpp"
+#include "kernels/a64_hybrid_s8s32_dot_6x16.hpp"
+#include "kernels/a64_interleaved_s8s32_mmla_8x12.hpp"
+#include "kernels/a64_smallK_hybrid_s8s32_dot_6x4.hpp"
+#include "kernels/a64_smallK_hybrid_s8s32_dot_8x4.hpp"
+#include "kernels/sve_hybrid_s8s32_dot_6x4VL.hpp"
+#include "kernels/sve_hybrid_s8qa_dot_4x4VL.hpp"
+#include "kernels/sve_hybrid_s8qs_dot_6x4VL.hpp"
+#include "kernels/sve_interleaved_s8s32_dot_8x3VL.hpp"
+#include "kernels/sve_interleaved_s8s32_mmla_8x3VL.hpp"
+#include "kernels/sve_smallK_hybrid_s8s32_dot_8x1VL.hpp"
+
+#include "gemm_hybrid_indirect.hpp"
#include "gemm_hybrid_quantized.hpp"
+#include "gemm_hybrid_quantized_inline.hpp"
+#include "gemm_interleaved.hpp"
#include "quantize_wrapper.hpp"
+#include "utils.hpp"
namespace arm_gemm {
static const GemmImplementation<int8_t, int8_t, Requantize32> gemm_qint8_methods[] =
{
#ifdef __ARM_FEATURE_SVE
+#ifdef MMLA_INT8
{
- GemmMethod::GEMM_HYBRID_QUANTIZED,
- "smallK_hybrid_s8s32_dot_1VLx8",
- [](const GemmArgs &args, const Requantize32 &) { return args._Ksize<=64; },
+ GemmMethod::GEMM_INTERLEAVED,
+ "sve_interleaved_s8s32_mmla_8x3VL",
+ [](const GemmArgs &args, const Requantize32 &) { return (args._Ksize>8); },
nullptr,
- [](const GemmArgs &args, const Requantize32 &qp) { return new GemmHybridQuantized<smallK_hybrid_s8s32_dot_1VLx8, int8_t, int8_t>(args, qp); }
-},
-{
- GemmMethod::GEMM_HYBRID_QUANTIZED,
- "hybrid_s8s32_dot_4VLx4",
- [](const GemmArgs &args, const Requantize32 &) { return args._Ksize>=16; },
- [](const GemmArgs &args, const Requantize32 &) { return ((args._Ksize <= 128) && (args._Nsize <= 128)) || ((args._nmulti > 1) && ((args._Msize / args._maxthreads) < 8)); },
- [](const GemmArgs &args, const Requantize32 &qp) { return new GemmHybridQuantized<hybrid_s8s32_dot_4VLx4, int8_t, int8_t>(args, qp); }
+ [](const GemmArgs &args, const Requantize32 &qp) { return new GemmInterleavedQuantized<cls_sve_interleaved_s8s32_mmla_8x3VL, int8_t, int8_t>(args, qp); }
},
#endif
{
GemmMethod::GEMM_HYBRID_QUANTIZED,
- "smallK_hybrid_s8s32_dot_4x8",
- [](const GemmArgs &args, const Requantize32 &) { return args._ci->has_dotprod() && (args._Nsize % 4 == 0) && (args._Ksize<=32); },
+ "sve_smallK_hybrid_s8s32_dot_8x1VL",
+ [](const GemmArgs &args, const Requantize32 &) { return args._Ksize<=64 && !args._indirect_input; },
nullptr,
- [](const GemmArgs &args, const Requantize32 &qp) { return new GemmHybridQuantized<smallK_hybrid_s8s32_dot_4x8, int8_t, int8_t>(args, qp); }
+ [](const GemmArgs &args, const Requantize32 &qp) { return new GemmHybridQuantized<cls_sve_smallK_hybrid_s8s32_dot_8x1VL, int8_t, int8_t>(args, qp); }
+},
+#ifdef SVE2
+{
+ GemmMethod::GEMM_HYBRID,
+ "sve_hybrid_s8qs_dot_6x4VL",
+ [](const GemmArgs &args, const Requantize32 &qp) { return quant_hybrid_symmetric(qp); },
+ nullptr,
+ [](const GemmArgs &args, const Requantize32 &qp) { return new GemmHybridIndirect<cls_sve_hybrid_s8qs_dot_6x4VL, int8_t, int8_t, Requantize32>(args, qp); }
+},
+{
+ GemmMethod::GEMM_HYBRID,
+ "sve_hybrid_s8qa_dot_4x4VL",
+ [](const GemmArgs &args, const Requantize32 &qp) { return quant_hybrid_asymmetric(qp); },
+ nullptr,
+ [](const GemmArgs &args, const Requantize32 &qp) { return new GemmHybridIndirect<cls_sve_hybrid_s8qa_dot_4x4VL, int8_t, int8_t, Requantize32>(args, qp); }
+},
+#endif
+{
+ GemmMethod::GEMM_HYBRID,
+ "sve_hybrid_s8s32_dot_6x4VL",
+ nullptr,
+ nullptr,
+ [](const GemmArgs &args, const Requantize32 &qp) { return new GemmHybridIndirect<cls_sve_hybrid_s8s32_dot_6x4VL, int8_t, int8_t, Requantize32, true>(args, qp); }
+},
+{
+ GemmMethod::GEMM_INTERLEAVED,
+ "sve_interleaved_s8s32_dot_8x3VL",
+ [](const GemmArgs &args, const Requantize32 &) { return (args._Ksize>4); },
+ nullptr,
+ [](const GemmArgs &args, const Requantize32 &qp) { return new GemmInterleavedQuantized<cls_sve_interleaved_s8s32_dot_8x3VL, int8_t, int8_t>(args, qp); }
+},
+#endif // SVE
+#ifdef MMLA_INT8
+{
+ GemmMethod::GEMM_INTERLEAVED,
+ "a64_interleaved_s8s32_mmla_8x12",
+ [](const GemmArgs &args, const Requantize32 &) { return (args._Ksize>8); },
+ nullptr,
+ [](const GemmArgs &args, const Requantize32 &qp) { return new GemmInterleavedQuantized<cls_a64_interleaved_s8s32_mmla_8x12, int8_t, int8_t>(args, qp); }
+},
+#endif
+{
+ GemmMethod::GEMM_HYBRID_QUANTIZED,
+ "a64_smallK_hybrid_s8s32_dot_8x4",
+ [](const GemmArgs &args, const Requantize32 &) { return args._ci->has_dotprod() && (args._Nsize % 4 == 0) && (args._Ksize<=32) && !args._indirect_input; },
+ nullptr,
+ [](const GemmArgs &args, const Requantize32 &qp) { return new GemmHybridQuantized<cls_a64_smallK_hybrid_s8s32_dot_8x4, int8_t, int8_t>(args, qp); }
},
{
GemmMethod::GEMM_HYBRID_QUANTIZED,
- "smallK_hybrid_s8s32_dot_4x6",
- [](const GemmArgs &args, const Requantize32 &) { return args._ci->has_dotprod() && (args._Nsize % 4 == 0) && (args._Ksize>32) && (args._Ksize<=64); },
+ "a64_smallK_hybrid_s8s32_dot_6x4",
+ [](const GemmArgs &args, const Requantize32 &) { return args._ci->has_dotprod() && (args._Nsize % 4 == 0) && (args._Ksize>32) && (args._Ksize<=64) && !args._indirect_input; },
nullptr,
- [](const GemmArgs &args, const Requantize32 &qp) { return new GemmHybridQuantized<smallK_hybrid_s8s32_dot_4x6, int8_t, int8_t>(args, qp); }
+ [](const GemmArgs &args, const Requantize32 &qp) { return new GemmHybridQuantized<cls_a64_smallK_hybrid_s8s32_dot_6x4, int8_t, int8_t>(args, qp); }
},
{
- GemmMethod::GEMM_HYBRID_QUANTIZED,
- "hybrid_s8s32_dot_16x4",
- [](const GemmArgs &args, const Requantize32 &) { return args._ci->has_dotprod() && args._Ksize>=16; },
- [](const GemmArgs &args, const Requantize32 &) { return args._Nsize<=256 && args._Ksize>128; },
- [](const GemmArgs &args, const Requantize32 &qp) { return new GemmHybridQuantized<hybrid_s8s32_dot_16x4, int8_t, int8_t>(args, qp); }
-},
-/** QUANTIZE_WRAPPER_2D enables 2D parallelisation hint for IScheduler in NEGEMMAssemblyDispatch */
-{
- GemmMethod::QUANTIZE_WRAPPER_2D,
- "quantized_wrapper_2d",
+ GemmMethod::GEMM_INTERLEAVED,
+ "a64_gemm_s16_8x12",
nullptr,
- [](const GemmArgs &args, const Requantize32 &) { return (args._maxthreads >= 8) && (args._Msize >= 8) && (args._Nsize >= 8);},
- [](const GemmArgs &args, const Requantize32 &qp) { return new QuantizeWrapper<int8_t, int8_t, int32_t>(args, qp); }
+ [](const GemmArgs &args, const Requantize32 &) { return args._ci->get_cpu_model() == CPUModel::A53; },
+ [](const GemmArgs &args, const Requantize32 &qp) { return new GemmInterleavedQuantized<cls_a64_gemm_s16_8x12, int8_t, int8_t>(args, qp); }
+},
+{
+ GemmMethod::GEMM_HYBRID,
+ "a64_hybrid_s8qs_dot_6x16",
+ [](const GemmArgs &args, const Requantize32 &qp) { return args._ci->has_dotprod() && quant_hybrid_symmetric(qp); },
+ nullptr,
+ [](const GemmArgs &args, const Requantize32 &qp) { return new GemmHybridIndirect<cls_a64_hybrid_s8qs_dot_6x16, int8_t, int8_t, Requantize32>(args, qp); }
+},
+{
+ GemmMethod::GEMM_HYBRID,
+ "a64_hybrid_s8qa_dot_4x16",
+ [](const GemmArgs &args, const Requantize32 &qp) { return args._ci->has_dotprod() && quant_hybrid_asymmetric(qp); },
+ nullptr,
+ [](const GemmArgs &args, const Requantize32 &qp) { return new GemmHybridIndirect<cls_a64_hybrid_s8qa_dot_4x16, int8_t, int8_t, Requantize32>(args, qp); }
+},
+{
+ GemmMethod::GEMM_HYBRID,
+ "a64_hybrid_s8s32_dot_6x16",
+ [](const GemmArgs &args, const Requantize32 &) { return args._ci->has_dotprod(); },
+ nullptr,
+ [](const GemmArgs &args, const Requantize32 &qp) { return new GemmHybridIndirect<cls_a64_hybrid_s8s32_dot_6x16, int8_t, int8_t, Requantize32, true>(args, qp); }
+},
+{
+ GemmMethod::GEMM_INTERLEAVED,
+ "a64_gemm_s8_8x12",
+ [](const GemmArgs &args, const Requantize32 &) { return args._ci->has_dotprod(); },
+ nullptr,
+ [](const GemmArgs &args, const Requantize32 &qp) { return new GemmInterleavedQuantized<cls_a64_gemm_s8_8x12, int8_t, int8_t>(args, qp); }
+},
+{
+ GemmMethod::GEMM_INTERLEAVED,
+ "a64_gemm_s8_4x4",
+ nullptr,
+ nullptr,
+ [](const GemmArgs &args, const Requantize32 &qp) { return new GemmInterleavedQuantized<cls_a64_gemm_s8_4x4, int8_t, int8_t>(args, qp); }
},
{
GemmMethod::QUANTIZE_WRAPPER,
"quantized_wrapper",
- nullptr,
+ [](const GemmArgs &args, const Requantize32 &) { return !args._indirect_input; },
nullptr,
[](const GemmArgs &args, const Requantize32 &qp) { return new QuantizeWrapper<int8_t, int8_t, int32_t>(args, qp); }
},
diff --git a/src/core/NEON/kernels/arm_gemm/gemm_quint8.cpp b/src/core/NEON/kernels/arm_gemm/gemm_quint8.cpp
index 0125f9c..7342fda 100644
--- a/src/core/NEON/kernels/arm_gemm/gemm_quint8.cpp
+++ b/src/core/NEON/kernels/arm_gemm/gemm_quint8.cpp
@@ -25,13 +25,25 @@
#include "arm_gemm.hpp"
-#include "kernels/a64_hybrid_u8u32_dot_16x4.hpp"
-#include "kernels/a64_smallK_hybrid_u8u32_dot_4x6.hpp"
-#include "kernels/a64_smallK_hybrid_u8u32_dot_4x8.hpp"
-#include "kernels/sve_hybrid_u8u32_dot_4VLx4.hpp"
-#include "kernels/sve_smallK_hybrid_u8u32_dot_1VLx8.hpp"
+#include "kernels/a64_gemm_u16_8x12.hpp"
+#include "kernels/a64_gemm_u8_4x4.hpp"
+#include "kernels/a64_gemm_u8_8x12.hpp"
+#include "kernels/a64_hybrid_u8qa_dot_4x16.hpp"
+#include "kernels/a64_hybrid_u8u32_dot_6x16.hpp"
+#include "kernels/a64_interleaved_u8u32_mmla_8x12.hpp"
+#include "kernels/a64_smallK_hybrid_u8u32_dot_6x4.hpp"
+#include "kernels/a64_smallK_hybrid_u8u32_dot_8x4.hpp"
+#include "kernels/sve_hybrid_u8u32_dot_6x4VL.hpp"
+#include "kernels/sve_hybrid_u8qa_dot_4x4VL.hpp"
+#include "kernels/sve_interleaved_u8u32_dot_8x3VL.hpp"
+#include "kernels/sve_interleaved_u8u32_mmla_8x3VL.hpp"
+#include "kernels/sve_smallK_hybrid_u8u32_dot_8x1VL.hpp"
+
+#include "gemm_hybrid_indirect.hpp"
#include "gemm_hybrid_quantized.hpp"
+#include "gemm_hybrid_quantized_inline.hpp"
+#include "gemm_interleaved.hpp"
#include "quantize_wrapper.hpp"
namespace arm_gemm {
@@ -39,54 +51,108 @@
static const GemmImplementation<uint8_t, uint8_t, Requantize32> gemm_quint8_methods[] =
{
#ifdef __ARM_FEATURE_SVE
+#ifdef MMLA_INT8
{
- GemmMethod::GEMM_HYBRID_QUANTIZED,
- "smallK_hybrid_u8u32_dot_1VLx8",
- [](const GemmArgs &args, const Requantize32 &) { return args._Ksize<=64; },
+ GemmMethod::GEMM_INTERLEAVED,
+ "sve_interleaved_u8u32_mmla_8x3VL",
+ [](const GemmArgs &args, const Requantize32 &) { return (args._Ksize>8); },
nullptr,
- [](const GemmArgs &args, const Requantize32 &qp) { return new GemmHybridQuantized<smallK_hybrid_u8u32_dot_1VLx8, uint8_t, uint8_t>(args, qp); }
-},
-{
- GemmMethod::GEMM_HYBRID_QUANTIZED,
- "hybrid_u8u32_dot_4VLx4",
- [](const GemmArgs &args, const Requantize32 &) { return args._Ksize>=16; },
- [](const GemmArgs &args, const Requantize32 &) { return ((args._Ksize <= 128) && (args._Nsize <= 128)) || ((args._nmulti > 1) && ((args._Msize / args._maxthreads) < 8)); },
- [](const GemmArgs &args, const Requantize32 &qp) { return new GemmHybridQuantized<hybrid_u8u32_dot_4VLx4, uint8_t, uint8_t>(args, qp); }
+ [](const GemmArgs &args, const Requantize32 &qp) { return new GemmInterleavedQuantized<cls_sve_interleaved_u8u32_mmla_8x3VL, uint8_t, uint8_t>(args, qp); }
},
#endif
{
GemmMethod::GEMM_HYBRID_QUANTIZED,
- "smallK_hybrid_u8u32_dot_4x8",
- [](const GemmArgs &args, const Requantize32 &) { return args._ci->has_dotprod() && (args._Nsize % 4 == 0) && (args._Ksize<=32); },
+ "sve_smallK_hybrid_u8u32_dot_8x1VL",
+ [](const GemmArgs &args, const Requantize32 &) { return args._Ksize<=64 && !args._indirect_input; },
nullptr,
- [](const GemmArgs &args, const Requantize32 &qp) { return new GemmHybridQuantized<smallK_hybrid_u8u32_dot_4x8, uint8_t, uint8_t>(args, qp); }
+ [](const GemmArgs &args, const Requantize32 &qp) { return new GemmHybridQuantized<cls_sve_smallK_hybrid_u8u32_dot_8x1VL, uint8_t, uint8_t>(args, qp); }
+},
+#ifdef SVE2 // Requantizing kernels include some SVE2 only instructions (SQRDMULH, SRSHL)
+{
+ GemmMethod::GEMM_HYBRID,
+ "sve_hybrid_u8qa_dot_4x4VL",
+ [](const GemmArgs &args, const Requantize32 &qp) { return quant_hybrid_asymmetric(qp); },
+ nullptr,
+ [](const GemmArgs &args, const Requantize32 &qp) { return new GemmHybridIndirect<cls_sve_hybrid_u8qa_dot_4x4VL, uint8_t, uint8_t, Requantize32>(args, qp); }
+},
+#endif
+{
+ GemmMethod::GEMM_HYBRID,
+ "sve_hybrid_u8u32_dot_6x4VL",
+ nullptr,
+ nullptr,
+ [](const GemmArgs &args, const Requantize32 &qp) { return new GemmHybridIndirect<cls_sve_hybrid_u8u32_dot_6x4VL, uint8_t, uint8_t, Requantize32, true>(args, qp); }
+},
+{
+ GemmMethod::GEMM_INTERLEAVED,
+ "sve_interleaved_u8u32_dot_8x3VL",
+ [](const GemmArgs &args, const Requantize32 &) { return (args._Ksize>4); },
+ nullptr,
+ [](const GemmArgs &args, const Requantize32 &qp) { return new GemmInterleavedQuantized<cls_sve_interleaved_u8u32_dot_8x3VL, uint8_t, uint8_t>(args, qp); }
+},
+#endif
+#ifdef MMLA_INT8
+{
+ GemmMethod::GEMM_INTERLEAVED,
+ "a64_interleaved_u8u32_mmla_8x12",
+ [](const GemmArgs &args, const Requantize32 &) { return (args._Ksize>8); },
+ nullptr,
+ [](const GemmArgs &args, const Requantize32 &qp) { return new GemmInterleavedQuantized<cls_a64_interleaved_u8u32_mmla_8x12, uint8_t, uint8_t>(args, qp); }
+},
+#endif
+{
+ GemmMethod::GEMM_HYBRID_QUANTIZED,
+ "a64_smallK_hybrid_u8u32_dot_8x4",
+ [](const GemmArgs &args, const Requantize32 &) { return args._ci->has_dotprod() && (args._Nsize % 4 == 0) && (args._Ksize<=32) && !args._indirect_input; },
+ nullptr,
+ [](const GemmArgs &args, const Requantize32 &qp) { return new GemmHybridQuantized<cls_a64_smallK_hybrid_u8u32_dot_8x4, uint8_t, uint8_t>(args, qp); }
},
{
GemmMethod::GEMM_HYBRID_QUANTIZED,
- "smallK_hybrid_u8u32_dot_4x6",
- [](const GemmArgs &args, const Requantize32 &) { return args._ci->has_dotprod() && (args._Nsize % 4 == 0) && (args._Ksize>32) && (args._Ksize<=64); },
+ "a64_smallK_hybrid_u8u32_dot_6x4",
+ [](const GemmArgs &args, const Requantize32 &) { return args._ci->has_dotprod() && (args._Nsize % 4 == 0) && (args._Ksize>32) && (args._Ksize<=64) && !args._indirect_input; },
nullptr,
- [](const GemmArgs &args, const Requantize32 &qp) { return new GemmHybridQuantized<smallK_hybrid_u8u32_dot_4x6, uint8_t, uint8_t>(args, qp); }
+ [](const GemmArgs &args, const Requantize32 &qp) { return new GemmHybridQuantized<cls_a64_smallK_hybrid_u8u32_dot_6x4, uint8_t, uint8_t>(args, qp); }
},
{
- GemmMethod::GEMM_HYBRID_QUANTIZED,
- "hybrid_u8u32_dot_16x4",
- [](const GemmArgs &args, const Requantize32 &) { return args._ci->has_dotprod() && args._Ksize>=16; },
- [](const GemmArgs &args, const Requantize32 &) { return ((args._Nsize<=256) && (args._Ksize>128)) || (args._maxthreads >= 8); },
- [](const GemmArgs &args, const Requantize32 &qp) { return new GemmHybridQuantized<hybrid_u8u32_dot_16x4, uint8_t, uint8_t>(args, qp); }
-},
-/** QUANTIZE_WRAPPER_2D enables 2D parallelisation hint for IScheduler in NEGEMMAssemblyDispatch */
-{
- GemmMethod::QUANTIZE_WRAPPER_2D,
- "quantized_wrapper_2d",
+ GemmMethod::GEMM_INTERLEAVED,
+ "a64_gemm_u16_8x12",
nullptr,
- [](const GemmArgs &args, const Requantize32 &) { return (args._maxthreads >= 8) && (args._Msize >= 8) && (args._Nsize >= 8);},
- [](const GemmArgs &args, const Requantize32 &qp) { return new QuantizeWrapper<uint8_t, uint8_t, uint32_t>(args, qp); }
+ [](const GemmArgs &args, const Requantize32 &) { return args._ci->get_cpu_model() == CPUModel::A53; },
+ [](const GemmArgs &args, const Requantize32 &qp) { return new GemmInterleavedQuantized<cls_a64_gemm_u16_8x12, uint8_t, uint8_t>(args, qp); },
+},
+{
+ GemmMethod::GEMM_HYBRID,
+ "a64_hybrid_u8qa_dot_4x16",
+ [](const GemmArgs &args, const Requantize32 &qp) { return args._ci->has_dotprod() && quant_hybrid_asymmetric(qp); },
+ [](const GemmArgs &args, const Requantize32 &) { return args._Nsize<=256 && args._Ksize>128; },
+ [](const GemmArgs &args, const Requantize32 &qp) { return new GemmHybridIndirect<cls_a64_hybrid_u8qa_dot_4x16, uint8_t, uint8_t, Requantize32>(args, qp); }
+},
+{
+ GemmMethod::GEMM_HYBRID,
+ "a64_hybrid_u8u32_dot_6x16",
+ [](const GemmArgs &args, const Requantize32 &) { return args._ci->has_dotprod(); },
+ [](const GemmArgs &args, const Requantize32 &) { return args._Nsize<=256 && args._Ksize>128; },
+ [](const GemmArgs &args, const Requantize32 &qp) { return new GemmHybridIndirect<cls_a64_hybrid_u8u32_dot_6x16, uint8_t, uint8_t, Requantize32, true>(args, qp); }
+},
+{
+ GemmMethod::GEMM_INTERLEAVED,
+ "a64_gemm_u8_8x12",
+ [](const GemmArgs &args, const Requantize32 &) { return args._ci->has_dotprod(); },
+ nullptr,
+ [](const GemmArgs &args, const Requantize32 &qp) { return new GemmInterleavedQuantized<cls_a64_gemm_u8_8x12, uint8_t, uint8_t>(args, qp); }
+},
+{
+ GemmMethod::GEMM_INTERLEAVED,
+ "a64_gemm_u8_4x4",
+ nullptr,
+ nullptr,
+ [](const GemmArgs &args, const Requantize32 &qp) { return new GemmInterleavedQuantized<cls_a64_gemm_u8_4x4, uint8_t, uint8_t>(args, qp); }
},
{
GemmMethod::QUANTIZE_WRAPPER,
"quantized_wrapper",
- nullptr,
+ [](const GemmArgs &args, const Requantize32 &) { return !args._indirect_input; },
nullptr,
[](const GemmArgs &args, const Requantize32 &qp) { return new QuantizeWrapper<uint8_t, uint8_t, uint32_t>(args, qp); }
},
diff --git a/src/core/NEON/kernels/arm_gemm/gemm_uint16.cpp b/src/core/NEON/kernels/arm_gemm/gemm_uint16.cpp
index 5e06443..10a35e7 100644
--- a/src/core/NEON/kernels/arm_gemm/gemm_uint16.cpp
+++ b/src/core/NEON/kernels/arm_gemm/gemm_uint16.cpp
@@ -28,17 +28,17 @@
#include "gemm_implementation.hpp"
#include "gemm_interleaved.hpp"
-#include "kernels/a64_gemm_u16_12x8.hpp"
+#include "kernels/a64_gemm_u16_8x12.hpp"
namespace arm_gemm {
static const GemmImplementation<uint16_t, uint32_t> gemm_u16_methods[] = {
{
GemmMethod::GEMM_INTERLEAVED,
- "gemm_u16_12x8",
+ "a64_gemm_u16_8x12",
nullptr,
nullptr,
- [](const GemmArgs &args) { return new GemmInterleaved<gemm_u16_12x8, uint16_t, uint32_t>(args); }
+ [](const GemmArgs &args) { return new GemmInterleaved<cls_a64_gemm_u16_8x12, uint16_t, uint32_t>(args); }
},
{
GemmMethod::DEFAULT,
diff --git a/src/core/NEON/kernels/arm_gemm/gemm_uint8.cpp b/src/core/NEON/kernels/arm_gemm/gemm_uint8.cpp
index 06e68cb..c300b8c 100644
--- a/src/core/NEON/kernels/arm_gemm/gemm_uint8.cpp
+++ b/src/core/NEON/kernels/arm_gemm/gemm_uint8.cpp
@@ -29,18 +29,20 @@
#include "gemm_interleaved.hpp"
#include "gemm_interleaved_pretransposed_2d.hpp"
#include "gemm_hybrid.hpp"
+#include "gemm_hybrid_indirect.hpp"
-#include "kernels/a64_gemm_u16_12x8.hpp"
-#include "kernels/a64_gemm_u8_12x8.hpp"
+#include "kernels/a64_gemm_u16_8x12.hpp"
#include "kernels/a64_gemm_u8_4x4.hpp"
-#include "kernels/a64_hybrid_u8u32_dot_16x4.hpp"
-#include "kernels/a64_interleaved_u8u32_mmla_12x8.hpp"
-#include "kernels/a64_smallK_hybrid_u8u32_dot_4x6.hpp"
-#include "kernels/a64_smallK_hybrid_u8u32_dot_4x8.hpp"
-#include "kernels/sve_hybrid_u8u32_dot_4VLx4.hpp"
-#include "kernels/sve_interleaved_u8u32_dot_3VLx8.hpp"
-#include "kernels/sve_interleaved_u8u32_mmla_3VLx8.hpp"
-#include "kernels/sve_smallK_hybrid_u8u32_dot_1VLx8.hpp"
+#include "kernels/a64_gemm_u8_8x12.hpp"
+#include "kernels/a64_hybrid_u8u32_dot_6x16.hpp"
+#include "kernels/a64_interleaved_u8u32_mmla_8x12.hpp"
+#include "kernels/a64_smallK_hybrid_u8u32_dot_6x4.hpp"
+#include "kernels/a64_smallK_hybrid_u8u32_dot_8x4.hpp"
+
+#include "kernels/sve_hybrid_u8u32_dot_6x4VL.hpp"
+#include "kernels/sve_interleaved_u8u32_dot_8x3VL.hpp"
+#include "kernels/sve_interleaved_u8u32_mmla_8x3VL.hpp"
+#include "kernels/sve_smallK_hybrid_u8u32_dot_8x1VL.hpp"
namespace arm_gemm {
@@ -49,106 +51,84 @@
#ifdef MMLA_INT8
{
GemmMethod::GEMM_INTERLEAVED,
- "interleaved_u8u32_mmla_3VLx8",
+ "sve_interleaved_u8u32_mmla_8x3VL",
[](const GemmArgs &args) { return (args._Ksize>8); },
nullptr,
- [](const GemmArgs &args) { return new GemmInterleaved<interleaved_u8u32_mmla_3VLx8, uint8_t, uint32_t>(args); }
+ [](const GemmArgs &args) { return new GemmInterleaved<cls_sve_interleaved_u8u32_mmla_8x3VL, uint8_t, uint32_t>(args); }
},
#endif
{
GemmMethod::GEMM_HYBRID,
- "smallK_hybrid_u8u32_dot_1VLx8",
- [](const GemmArgs &args) { return args._Ksize<=64; },
+ "smallK_hybrid_u8u32_dot_8x1VL",
+ [](const GemmArgs &args) { return args._Ksize<=64 && !args._indirect_input; },
nullptr,
- [](const GemmArgs &args) { return new GemmHybrid<smallK_hybrid_u8u32_dot_1VLx8, uint8_t, uint32_t>(args); }
+ [](const GemmArgs &args) { return new GemmHybrid<cls_sve_smallK_hybrid_u8u32_dot_8x1VL, uint8_t, uint32_t>(args); }
},
{
GemmMethod::GEMM_HYBRID,
- "hybrid_u8u32_dot_4VLx4",
- [](const GemmArgs &args) { return args._Ksize>=16; },
+ "sve_hybrid_u8u32_dot_6x4VL",
+ nullptr,
[](const GemmArgs &args) { return ((args._Ksize <= 128) && (args._Nsize <= 128)) || ((args._nmulti > 1) && ((args._Msize / args._maxthreads) < 8)); },
- [](const GemmArgs &args) { return new GemmHybrid<hybrid_u8u32_dot_4VLx4, uint8_t, uint32_t>(args); }
+ [](const GemmArgs &args) { return new GemmHybridIndirect<cls_sve_hybrid_u8u32_dot_6x4VL, uint8_t, uint32_t>(args); }
},
{
GemmMethod::GEMM_INTERLEAVED,
- "interleaved_u8u32_dot_3VLx8",
+ "sve_interleaved_u8u32_dot_8x3VL",
[](const GemmArgs &args) { return (args._Ksize>4); },
nullptr,
- [](const GemmArgs &args) { return new GemmInterleaved<interleaved_u8u32_dot_3VLx8, uint8_t, uint32_t>(args); }
+ [](const GemmArgs &args) { return new GemmInterleaved<cls_sve_interleaved_u8u32_dot_8x3VL, uint8_t, uint32_t>(args); }
},
#endif
#ifdef MMLA_INT8
{
GemmMethod::GEMM_INTERLEAVED,
- "interleaved_u8u32_mmla_12x8",
+ "a64_interleaved_u8u32_mmla_8x12",
[](const GemmArgs &args) { return (args._Ksize>8); },
nullptr,
- [](const GemmArgs &args) { return new GemmInterleaved<interleaved_u8u32_mmla_12x8, uint8_t, uint32_t>(args); }
+ [](const GemmArgs &args) { return new GemmInterleaved<cls_a64_interleaved_u8u32_mmla_8x12, uint8_t, uint32_t>(args); }
},
#endif
{
GemmMethod::GEMM_HYBRID,
- "smallK_hybrid_u8u32_dot_4x8",
- [](const GemmArgs &args) { return args._ci->has_dotprod() && (args._Nsize % 4 == 0) && (args._Ksize<=32); },
+ "a64_smallK_hybrid_u8u32_dot_8x4",
+ [](const GemmArgs &args) { return args._ci->has_dotprod() && (args._Nsize % 4 == 0) && (args._Ksize<=32) && !args._indirect_input; },
nullptr,
- [](const GemmArgs &args) { return new GemmHybrid<smallK_hybrid_u8u32_dot_4x8, uint8_t, uint32_t>(args); }
+ [](const GemmArgs &args) { return new GemmHybrid<cls_a64_smallK_hybrid_u8u32_dot_8x4, uint8_t, uint32_t>(args); }
},
{
GemmMethod::GEMM_HYBRID,
- "smallK_hybrid_u8u32_dot_4x6",
- [](const GemmArgs &args) { return args._ci->has_dotprod() && (args._Nsize % 4 == 0) && (args._Ksize>32) && (args._Ksize<=64); },
+ "a64_smallK_hybrid_u8u32_dot_6x4",
+ [](const GemmArgs &args) { return args._ci->has_dotprod() && (args._Nsize % 4 == 0) && (args._Ksize>32) && (args._Ksize<=64) && !args._indirect_input; },
nullptr,
- [](const GemmArgs &args) { return new GemmHybrid<smallK_hybrid_u8u32_dot_4x6, uint8_t, uint32_t>(args); }
+ [](const GemmArgs &args) { return new GemmHybrid<cls_a64_smallK_hybrid_u8u32_dot_6x4, uint8_t, uint32_t>(args); }
+},
+{
+ GemmMethod::GEMM_INTERLEAVED,
+ "a64_gemm_u16_8x12",
+ nullptr,
+ [](const GemmArgs &args) { return args._ci->get_cpu_model() == CPUModel::A53; },
+ [](const GemmArgs &args) { return new GemmInterleaved<cls_a64_gemm_u16_8x12, uint8_t, uint32_t>(args); },
},
{
GemmMethod::GEMM_HYBRID,
- "hybrid_u8u32_dot_16x4",
- [](const GemmArgs &args) { return args._ci->has_dotprod() && args._Ksize>=16; },
+ "a64_hybrid_u8u32_dot_6x16",
+ [](const GemmArgs &args) { return args._ci->has_dotprod(); },
[](const GemmArgs &args) { return args._Nsize<=256 && args._Ksize>128; },
- [](const GemmArgs &args) { return new GemmHybrid<hybrid_u8u32_dot_16x4, uint8_t, uint32_t>(args); }
-},
-{
- GemmMethod::GEMM_INTERLEAVED_2D,
- "gemm_u8_12x8_2d",
- [](const GemmArgs &args) { return args._ci->has_dotprod(); },
- [](const GemmArgs &args) { return (args._maxthreads >= 8) && (args._Msize >= 8) && (args._Nsize >= 8) ; },
- [](const GemmArgs &args) { return new GemmInterleavedPretransposed2d<gemm_u8_12x8, uint8_t, uint32_t>(args); }
+ [](const GemmArgs &args) { return new GemmHybridIndirect<cls_a64_hybrid_u8u32_dot_6x16, uint8_t, uint32_t>(args); }
},
{
GemmMethod::GEMM_INTERLEAVED,
- "gemm_u8_12x8_1d",
+ "a64_gemm_u8_8x12",
[](const GemmArgs &args) { return args._ci->has_dotprod(); },
nullptr,
- [](const GemmArgs &args) { return new GemmInterleaved<gemm_u8_12x8, uint8_t, uint32_t>(args); }
-},
-{
- GemmMethod::GEMM_INTERLEAVED_2D,
- "gemm_u16_12x8_2d",
- nullptr,
- [](const GemmArgs &args) { return args._ci->get_cpu_model() == CPUModel::A53 && args._Msize > 4 && (args._Msize / args._maxthreads) < 8; },
- [](const GemmArgs &args) { return new GemmInterleavedPretransposed2d<gemm_u16_12x8, uint8_t, uint32_t>(args); },
+ [](const GemmArgs &args) { return new GemmInterleaved<cls_a64_gemm_u8_8x12, uint8_t, uint32_t>(args); }
},
{
GemmMethod::GEMM_INTERLEAVED,
- "gemm_u16_12x8_1d",
- nullptr,
- [](const GemmArgs &args) { return args._ci->get_cpu_model() == CPUModel::A53 && args._Msize > 4; },
- [](const GemmArgs &args) { return new GemmInterleaved<gemm_u16_12x8, uint8_t, uint32_t>(args); },
-},
-{
- GemmMethod::GEMM_INTERLEAVED_2D,
- "gemm_u8_4x4_2d",
- nullptr,
- [](const GemmArgs &args) { return ((args._maxthreads >= 8) && (args._Msize >= 8) && (args._Nsize >= 8)) ||
- ((args._Msize / args._maxthreads) < 4); },
- [](const GemmArgs &args) { return new GemmInterleavedPretransposed2d<gemm_u8_4x4, uint8_t, uint32_t>(args); }
-},
-{
- GemmMethod::GEMM_INTERLEAVED,
- "gemm_u8_4x4_1d",
+ "a64_gemm_u8_4x4",
nullptr,
nullptr,
- [](const GemmArgs &args) { return new GemmInterleaved<gemm_u8_4x4, uint8_t, uint32_t>(args); }
+ [](const GemmArgs &args) { return new GemmInterleaved<cls_a64_gemm_u8_4x4, uint8_t, uint32_t>(args); }
},
{
GemmMethod::DEFAULT,
diff --git a/src/core/NEON/kernels/arm_gemm/gemv_pretransposed.hpp b/src/core/NEON/kernels/arm_gemm/gemv_pretransposed.hpp
index 47909cd..9de44fc 100644
--- a/src/core/NEON/kernels/arm_gemm/gemv_pretransposed.hpp
+++ b/src/core/NEON/kernels/arm_gemm/gemv_pretransposed.hpp
@@ -46,46 +46,39 @@
typedef typename strategy::operand_type Toi;
typedef typename strategy::result_type Tri;
- const unsigned int _Nsize;
- const unsigned int _Ksize;
-
- const unsigned int _nmultis;
-
- const Activation _act;
-
- const CPUInfo * const _ci;
+ const GemmArgs _args;
const unsigned int _buffer_per_multi;
- unsigned int m_block=0;
+ unsigned int k_block=0;
unsigned int n_block=0;
- const Toi *_A_pretransposed = nullptr;
+ const Toi *_B_pretransposed = nullptr;
public:
GemvPretransposed(GemvPretransposed &) = delete;
GemvPretransposed & operator= (GemvPretransposed &) = delete;
GemvPretransposed(const GemmArgs &args)
- : _Nsize(args._Nsize), _Ksize(args._Ksize), _nmultis(args._nmulti), _act(args._act), _ci(args._ci),
- _buffer_per_multi(_Ksize * iceildiv(_Nsize, strategy::A_interleave()) * strategy::A_interleave()) {
+ : _args(args),
+ _buffer_per_multi(args._Ksize * roundup(args._Nsize, strategy::out_width())) {
/* For now don't do any blocking. TODO: figure out if we should. */
- if (args._cfg && args._cfg->inner_block_size) {
- m_block = args._cfg->inner_block_size;
+ if (strategy::supports_accumulate() && args._cfg && args._cfg->inner_block_size) {
+ k_block = args._cfg->inner_block_size;
} else {
- m_block = _Ksize;
+ k_block = args._Ksize;
}
if (args._cfg && args._cfg->outer_block_size) {
n_block = args._cfg->outer_block_size;
} else {
- n_block = _Nsize;
+ n_block = args._Nsize;
}
}
// Window is number of out_width blocks, times number of multis.
ndrange_t get_window_size() const override {
- return { iceildiv(_Nsize, strategy::out_width()) * _nmultis };
+ return { iceildiv(_args._Nsize, strategy::out_width()) * _args._nmulti };
}
// Actually execute the GEMV.
@@ -93,13 +86,13 @@
#ifdef CYCLE_PROFILING
profiler prof;
#endif
- strategy strat(_ci);
+ strategy strat(_args._ci);
const auto start = work_range.get_position(0);
const auto end = work_range.get_position_end(0);
/* Break the window values down into multis of interest... */
- const unsigned int window_per_multi = iceildiv(_Nsize, strategy::out_width());
+ const unsigned int window_per_multi = iceildiv(_args._Nsize, strategy::out_width());
const unsigned int multi_0 = start / window_per_multi;
const unsigned int multi_end = end / window_per_multi;
@@ -111,36 +104,25 @@
for (unsigned int multi=multi_0; multi<=multi_end; multi++) {
const unsigned int n_start = (multi==multi_0) ? n_0 : 0;
- const unsigned int n_end = (multi==multi_end) ? n_max : _Nsize;
+ const unsigned int n_end = (multi==multi_end) ? n_max : _args._Nsize;
if (n_end <= n_start)
continue;
- for (unsigned int m0=0; m0<_Ksize; m0+=m_block) {
- unsigned int mmax = std::min(m0 + m_block, _Ksize);
+ for (unsigned int k0=0; k0<_args._Ksize; k0+=k_block) {
+ unsigned int kmax = std::min(k0 + k_block, _args._Ksize);
for (unsigned int n=n_start; n<n_end; n+=n_block) {
unsigned int nmax = std::min(n + n_block, n_end);
#ifdef CYCLE_PROFILING
- auto p = prof.ScopedProfiler(PROFILE_KERNEL, (mmax-m0) * (nmax-n));
+ auto p = prof.ScopedProfiler(PROFILE_KERNEL, (kmax-k0) * (nmax-n));
#endif
- /* This assumes that the underlying call was a GEMM with M=1; for the N=1 case we would have to pick up this->_Bptr below instead */
- strat.kernel(_A_pretransposed + (multi * _buffer_per_multi) + (n * _Ksize) + (m0 * strategy::A_interleave()),
- (_Ksize * strategy::A_interleave()),
- this->_Aptr + (multi * this->_A_multi_stride) + m0,
+ strat.kernel(this->_Aptr + (multi * this->_A_multi_stride) + k0,
+ _B_pretransposed + (multi * _buffer_per_multi) + (n * roundup(_args._Ksize, strategy::k_unroll())) + (k0 * strategy::out_width()),
this->_Cptr + (multi * this->_C_multi_stride) + n,
- static_cast<Tr>(0), (mmax-m0), (nmax-n));
-
- // Handle activation separately for now
- if (this->_bias) {
- activator<true>(this->_Cptr + (multi * this->_C_multi_stride) + n, 0,
- this->_bias + (multi * this->_bias_multi_stride) + n,
- _act, 1, (nmax-n));
- } else {
- activator<false>(this->_Cptr + (multi * this->_C_multi_stride) + n, 0,
- static_cast<const Tr *>(nullptr),
- _act, 1, (nmax-n));
- }
+ (nmax - n), (kmax-k0),
+ this->_bias ? this->_bias + (multi * this->_bias_multi_stride) + n : nullptr,
+ _args._act, (k0 != 0));
}
}
}
@@ -152,33 +134,27 @@
}
bool B_pretranspose_required() const override {
- /* Transpose is required if _A_pretransposed is still nullptr */
- return (_A_pretransposed == nullptr);
+ /* Transpose is required if _B_pretransposed is still nullptr */
+ return (_B_pretransposed == nullptr);
}
size_t get_B_pretransposed_array_size() const override {
- return _buffer_per_multi * _nmultis * sizeof(To);
+ return _buffer_per_multi * _args._nmulti * sizeof(To);
}
void pretranspose_B_array(void *buffer, const To *B, const int ldb, const int B_multi_stride) override {
- Toi *A_buffer = reinterpret_cast<Toi *>(buffer);
+ Toi *B_buffer = reinterpret_cast<Toi *>(buffer);
+ strategy strat(_args._ci);
- for (unsigned int multi=0; multi<_nmultis; multi++) {
- /* Reverse sense here as we are dealing with B rather than A. So if
- * strategy::A_transpose is false and _trB is false, we still
- * transpose. */
- if (strategy::A_transpose()) {
- Transform<strategy::A_interleave(), strategy::A_block(), false>(A_buffer + (multi * _buffer_per_multi), B + (multi * B_multi_stride), ldb, 0, _Nsize, 0, _Ksize);
- } else {
- Transform<strategy::A_interleave(), strategy::A_block(), true>(A_buffer + (multi * _buffer_per_multi), B + (multi * B_multi_stride), ldb, 0, _Nsize, 0, _Ksize);
- }
+ for (unsigned int multi=0; multi<_args._nmulti; multi++) {
+ strat.transforms.PrepareB(B_buffer + (multi * _buffer_per_multi), B + (multi * B_multi_stride), ldb, 0, _args._Nsize, 0, _args._Ksize);
}
- _A_pretransposed = A_buffer;
+ _B_pretransposed = B_buffer;
}
void set_pretransposed_B_data(void *buffer) override {
- _A_pretransposed = reinterpret_cast<Toi *>(buffer);
+ _B_pretransposed = reinterpret_cast<Toi *>(buffer);
}
};
diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a32_interleave6_block1_fp32_fp32.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a32_interleave6_block1_fp32_fp32.hpp
new file mode 100644
index 0000000..807511f
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a32_interleave6_block1_fp32_fp32.hpp
@@ -0,0 +1,151 @@
+/*
+ * Copyright (c) 2017-2018 Arm Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#pragma once
+
+#ifdef __arm__
+
+#include <arm_neon.h>
+
+#include "../asmlib.hpp"
+
+template<>
+void interleave_block<6, 1, VLType::None, false>(
+ float * &outptr, const float * const * in, size_t width, size_t height,
+ size_t row_offset, bool
+)
+{
+ const float *inptr0 = in[0] + row_offset;
+ const float *inptr1 = in[1] + row_offset;
+ const float *inptr2 = in[2] + row_offset;
+ const float *inptr3 = in[3] + row_offset;
+ const float *inptr4 = in[4] + row_offset;
+ const float *inptr5 = in[5] + row_offset;
+
+ // Cope with ragged cases by aliasing the first row (which is always valid).
+ // The nonsense output produced will be suppressed later anyway.
+ switch (height) {
+ case 1:
+ inptr1 = inptr0;
+ // fall through
+ case 2:
+ inptr2 = inptr0;
+ // fall through
+ case 3:
+ inptr3 = inptr0;
+ // fall through
+ case 4:
+ inptr4 = inptr0;
+ // fall through
+ case 5:
+ inptr5 = inptr0;
+ // fall through
+ default:
+ case 6:
+ break;
+ }
+
+ //prefetch_2x(inptr0);
+ //prefetch_2x(inptr1);
+ //prefetch_2x(inptr2);
+ //prefetch_2x(inptr3);
+ //prefetch_2x(inptr4);
+ //prefetch_2x(inptr5);
+
+ for (;width>7;width-=8) {
+ __asm __volatile (
+ // Load up 8 elements (2 vectors) from each of 8 sources.
+ "VLD1.32 {d0-d3}, [%[inptr0]]!\n" // q0=A0A1A2A3
+ "VLD1.32 {d4-d7}, [%[inptr1]]!\n" // q2=B0B1B2B3
+ "VLD1.32 {d8-d11}, [%[inptr2]]!\n" // q4=C0C1C2C3
+ "VZIP.32 q0, q4\n" // q0=A0C0A1C1, q4 = A2C2A3C3
+ "VLD1.32 {d12-d15}, [%[inptr3]]!\n" // q6=D0D1D2D3
+ "VZIP.32 q2, q6\n" // q2=B0D0B1D1, q6 = B2D2B3D3
+ "VLD1.32 {d16-d19}, [%[inptr4]]!\n"
+ "VLD1.32 {d20-d23}, [%[inptr5]]!\n"
+ "VZIP.32 q8, q10\n" // q8=E0F0E1F1, q10 = E2F2E3F3
+ ASM_PREFETCH("[%[inptr0], #128]")
+ "VZIP.32 q0, q2\n" // q0 = A0B0C0D0, q2 = A1B1C1D1
+
+ // Store first elements
+ "VST1.32 {d0-d1}, [%[outptr]]!\n"
+ "VST1.32 {d16}, [%[outptr]]!\n"
+
+ "VZIP.32 q4, q6\n" // q4 = A2B2C2D2, q6 = A3B3C3D3
+
+ // Store second elements
+ "VST1.32 {d4-d5}, [%[outptr]]!\n"
+ "VZIP.32 q1, q5\n"
+ ASM_PREFETCH("[%[inptr1], #128]")
+ "VST1.32 {d17}, [%[outptr]]!\n"
+ "VZIP.32 q3, q7\n"
+
+ // Store third elements
+ "VZIP.32 q9, q11\n"
+ "VST1.32 {d8-d9}, [%[outptr]]!\n"
+ "VZIP.32 q1, q3\n"
+ ASM_PREFETCH("[%[inptr2], #128]")
+ "VST1.32 {d20}, [%[outptr]]!\n"
+
+ // Store fourth elements
+ "VZIP.32 q5, q7\n"
+ "VST1.32 {d12-d13}, [%[outptr]]!\n"
+ ASM_PREFETCH("[%[inptr3], #128]")
+ "VST1.32 {d21}, [%[outptr]]!\n"
+
+ // Fifth
+ "VST1.32 {d2-d3}, [%[outptr]]!\n"
+ ASM_PREFETCH("[%[inptr4], #128]")
+ "VST1.32 {d18}, [%[outptr]]!\n"
+
+ // Sixth
+ "VST1.32 {d6-d7}, [%[outptr]]!\n"
+ ASM_PREFETCH("[%[inptr5], #128]")
+ "VST1.32 {d19}, [%[outptr]]!\n"
+
+ // Seventh
+ "VST1.32 {d10-d11}, [%[outptr]]!\n"
+ "VST1.32 {d22}, [%[outptr]]!\n"
+
+ // Eighth
+ "VST1.32 {d14-d15}, [%[outptr]]!\n"
+ "VST1.32 {d23}, [%[outptr]]!\n"
+
+ : [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3),
+ [inptr4] "+r" (inptr4), [inptr5] "+r" (inptr5), [outptr] "+r" (outptr)
+ :
+ : "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q8", "q9", "q10", "q11", "q12", "memory"
+ );
+ }
+
+ for (;width>0;width--) {
+ *outptr++ = *inptr0++;
+ *outptr++ = *inptr1++;
+ *outptr++ = *inptr2++;
+ *outptr++ = *inptr3++;
+ *outptr++ = *inptr4++;
+ *outptr++ = *inptr5++;
+ }
+}
+
+#endif // __arm__
diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave4_block16_s8_s8.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave4_block16_s8_s8.hpp
new file mode 100644
index 0000000..8054c2b
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave4_block16_s8_s8.hpp
@@ -0,0 +1,193 @@
+/*
+ * Copyright (c) 2019-2020 Arm Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+#ifdef __aarch64__
+
+template<>
+void interleave_block<4, 16, VLType::None, false>(
+ int8_t * &out_ptr, const int8_t * const * in, size_t width, size_t height,
+ size_t row_offset, bool
+)
+{
+ __asm__ __volatile__(
+ "ldr x22, [%x[in], #0x0]\n"
+ "cmp %x[height], #0x4\n"
+ "ldr x21, [%x[in], #0x8]\n"
+ "add x22, x22, %x[row_offset]\n"
+ "ldr x20, [%x[in], #0x10]\n"
+ "ldr x19, [%x[in], #0x18]\n"
+ "add x21, x21, %x[row_offset]\n"
+ "add x20, x20, %x[row_offset]\n"
+ "add x19, x19, %x[row_offset]\n"
+ "beq 1f\n"
+ "mov x19, x22\n"
+ "cmp %x[height], #0x2\n"
+ "csel x21, x21, x22, GE\n"
+ "csel x20, x20, x22, GT\n"
+ "1:" // no_pointer_adj
+ "prfm pldl1keep, [x22, #0x0]\n"
+ "cmp %x[width], #0x10\n"
+ "prfm pldl1keep, [x21, #0x0]\n"
+ "prfm pldl1keep, [x20, #0x0]\n"
+ "prfm pldl1keep, [x19, #0x0]\n"
+ "prfm pldl1keep, [x22, #0x40]\n"
+ "prfm pldl1keep, [x21, #0x40]\n"
+ "prfm pldl1keep, [x20, #0x40]\n"
+ "prfm pldl1keep, [x19, #0x40]\n"
+ "blt 3f\n"
+ "2:" // Main loop head
+ "ldr q19, [x22], #0x10\n"
+ "prfm pldl1keep, [x22, #0x70]\n"
+ "ldr q18, [x21], #0x10\n"
+ "ldr q17, [x20], #0x10\n"
+ "prfm pldl1keep, [x21, #0x70]\n"
+ "ldr q16, [x19], #0x10\n"
+ "prfm pldl1keep, [x20, #0x70]\n"
+ "str q19, [%x[out_ptr], #0x0]\n"
+ "str q18, [%x[out_ptr], #0x10]\n"
+ "prfm pldl1keep, [x19, #0x70]\n"
+ "str q17, [%x[out_ptr], #0x20]\n"
+ "str q16, [%x[out_ptr], #0x30]\n"
+ "subs %x[width], %x[width], #0x10\n"
+ "cmp %x[width], #0x10\n"
+ "add %x[out_ptr], %x[out_ptr], #0x40\n"
+ "bge 2b\n"
+ "3:" // Main loop skip
+ "cbz %x[width], 12f\n"
+ "tbz %x[width], #3, 7f\n"
+ "ldr d19, [x22], #0x8\n"
+ "ldr d18, [x21], #0x8\n"
+ "ldr d17, [x20], #0x8\n"
+ "ldr d16, [x19], #0x8\n"
+ "tbz %x[width], #2, 5f\n"
+ "ld1 { v19.s }[2], [x22], #0x4\n"
+ "ld1 { v18.s }[2], [x21], #0x4\n"
+ "ld1 { v17.s }[2], [x20], #0x4\n"
+ "ld1 { v16.s }[2], [x19], #0x4\n"
+ "tbz %x[width], #1, 4f\n"
+ "ld1 { v19.h }[6], [x22], #0x2\n"
+ "ld1 { v18.h }[6], [x21], #0x2\n"
+ "ld1 { v17.h }[6], [x20], #0x2\n"
+ "ld1 { v16.h }[6], [x19], #0x2\n"
+ "tbz %x[width], #0, 11f\n"
+ "ld1 { v19.b }[14], [x22]\n"
+ "ld1 { v18.b }[14], [x21]\n"
+ "ld1 { v17.b }[14], [x20]\n"
+ "ld1 { v16.b }[14], [x19]\n"
+ "b 11f\n"
+ "4:" // odd_loads_1_12
+ "tbz %x[width], #0, 11f\n"
+ "ld1 { v19.b }[12], [x22]\n"
+ "ld1 { v18.b }[12], [x21]\n"
+ "ld1 { v17.b }[12], [x20]\n"
+ "ld1 { v16.b }[12], [x19]\n"
+ "b 11f\n"
+ "5:" // odd_loads_2_8
+ "tbz %x[width], #1, 6f\n"
+ "ld1 { v19.h }[4], [x22], #0x2\n"
+ "ld1 { v18.h }[4], [x21], #0x2\n"
+ "ld1 { v17.h }[4], [x20], #0x2\n"
+ "ld1 { v16.h }[4], [x19], #0x2\n"
+ "tbz %x[width], #0, 11f\n"
+ "ld1 { v19.b }[10], [x22]\n"
+ "ld1 { v18.b }[10], [x21]\n"
+ "ld1 { v17.b }[10], [x20]\n"
+ "ld1 { v16.b }[10], [x19]\n"
+ "b 11f\n"
+ "6:" // odd_loads_1_8
+ "tbz %x[width], #0, 11f\n"
+ "ld1 { v19.b }[8], [x22]\n"
+ "ld1 { v18.b }[8], [x21]\n"
+ "ld1 { v17.b }[8], [x20]\n"
+ "ld1 { v16.b }[8], [x19]\n"
+ "b 11f\n"
+ "7:" // odd_loads_4_0
+ "tbz %x[width], #2, 9f\n"
+ "ldr s19, [x22], #0x4\n"
+ "ldr s18, [x21], #0x4\n"
+ "ldr s17, [x20], #0x4\n"
+ "ldr s16, [x19], #0x4\n"
+ "tbz %x[width], #1, 8f\n"
+ "ld1 { v19.h }[2], [x22], #0x2\n"
+ "ld1 { v18.h }[2], [x21], #0x2\n"
+ "ld1 { v17.h }[2], [x20], #0x2\n"
+ "ld1 { v16.h }[2], [x19], #0x2\n"
+ "tbz %x[width], #0, 11f\n"
+ "ld1 { v19.b }[6], [x22]\n"
+ "ld1 { v18.b }[6], [x21]\n"
+ "ld1 { v17.b }[6], [x20]\n"
+ "ld1 { v16.b }[6], [x19]\n"
+ "b 11f\n"
+ "8:" // odd_loads_1_4
+ "tbz %x[width], #0, 11f\n"
+ "ld1 { v19.b }[4], [x22]\n"
+ "ld1 { v18.b }[4], [x21]\n"
+ "ld1 { v17.b }[4], [x20]\n"
+ "ld1 { v16.b }[4], [x19]\n"
+ "b 11f\n"
+ "9:" // odd_loads_2_0
+ "tbz %x[width], #1, 10f\n"
+ "ldr h19, [x22], #0x2\n"
+ "ldr h18, [x21], #0x2\n"
+ "ldr h17, [x20], #0x2\n"
+ "ldr h16, [x19], #0x2\n"
+ "tbz %x[width], #0, 11f\n"
+ "ld1 { v19.b }[2], [x22]\n"
+ "ld1 { v18.b }[2], [x21]\n"
+ "ld1 { v17.b }[2], [x20]\n"
+ "ld1 { v16.b }[2], [x19]\n"
+ "b 11f\n"
+ "10:" // odd_loads_1_0
+ "ldr b19, [x22, #0x0]\n"
+ "ldr b18, [x21, #0x0]\n"
+ "ldr b17, [x20, #0x0]\n"
+ "ldr b16, [x19, #0x0]\n"
+ "11:" // Odd load end
+ "str q19, [%x[out_ptr], #0x0]\n"
+ "str q18, [%x[out_ptr], #0x10]\n"
+ "str q17, [%x[out_ptr], #0x20]\n"
+ "str q16, [%x[out_ptr], #0x30]\n"
+ "add %x[out_ptr], %x[out_ptr], #0x40\n"
+ "12:" // Odds skip
+
+ : [out_ptr] "+r" (out_ptr), [width] "+r" (width)
+ : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset)
+ : "cc", "memory", "v16", "v17", "v18", "v19", "x19", "x20", "x21", "x22"
+ );
+}
+
+template<>
+void interleave_block<4, 16, VLType::None, false>(
+ uint8_t * &out_ptr, const uint8_t * const * in, size_t width, size_t height,
+ size_t row_offset, bool
+)
+{
+ int8_t * &out_cast = reinterpret_cast<int8_t * &>(out_ptr);
+ const int8_t * const * in_cast = reinterpret_cast<const int8_t * const *>(in);
+
+ interleave_block<4, 16, VLType::None, false>(out_cast, in_cast, width, height, row_offset, false);
+}
+
+
+#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave4_block16_s8_s8_summing.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave4_block16_s8_s8_summing.hpp
new file mode 100644
index 0000000..1650916
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave4_block16_s8_s8_summing.hpp
@@ -0,0 +1,225 @@
+/*
+ * Copyright (c) 2019-2020 Arm Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+#ifdef __aarch64__
+
+template<>
+void interleave_block<4, 16, VLType::None, true>(
+ int8_t * &out_ptr, const int8_t * const * in, size_t width, size_t height,
+ size_t row_offset, bool first
+)
+{
+ __asm__ __volatile__(
+ "movi v28.8h, #0x0\n"
+ "ldr x23, [%x[in], #0x0]\n"
+ "mov x22, #0x0\n"
+ "movi v27.8h, #0x0\n"
+ "ldr x21, [%x[in], #0x8]\n"
+ "cmp %x[height], #0x4\n"
+ "movi v26.8h, #0x0\n"
+ "ldr x20, [%x[in], #0x10]\n"
+ "add x23, x23, %x[row_offset]\n"
+ "movi v25.8h, #0x0\n"
+ "ldr x19, [%x[in], #0x18]\n"
+ "movi v24.4s, #0x0\n"
+ "add x21, x21, %x[row_offset]\n"
+ "movi v23.4s, #0x0\n"
+ "add x20, x20, %x[row_offset]\n"
+ "movi v22.4s, #0x0\n"
+ "add x19, x19, %x[row_offset]\n"
+ "movi v21.4s, #0x0\n"
+ "beq 1f\n"
+ "mov x19, x23\n"
+ "cmp %x[height], #0x2\n"
+ "csel x21, x21, x23, GE\n"
+ "csel x20, x20, x23, GT\n"
+ "1:" // no_pointer_adj
+ "movi v20.4s, #0x0\n"
+ "prfm pldl1keep, [x23, #0x0]\n"
+ "prfm pldl1keep, [x21, #0x0]\n"
+ "prfm pldl1keep, [x20, #0x0]\n"
+ "prfm pldl1keep, [x19, #0x0]\n"
+ "prfm pldl1keep, [x23, #0x40]\n"
+ "prfm pldl1keep, [x21, #0x40]\n"
+ "prfm pldl1keep, [x20, #0x40]\n"
+ "prfm pldl1keep, [x19, #0x40]\n"
+ "cbnz %w[first], 2f\n"
+ "sub %x[out_ptr], %x[out_ptr], #0x10\n"
+ "ld1 { v20.4s }, [%x[out_ptr]]\n"
+ "2:" // first_pass
+ "cmp %x[width], #0x10\n"
+ "blt 5f\n"
+ "3:" // Main loop head
+ "cmp x22, #0x7e\n"
+ "ble 4f\n"
+ "sadalp v24.4s, v28.8h\n"
+ "movi v28.8h, #0x0\n"
+ "sadalp v23.4s, v27.8h\n"
+ "movi v27.8h, #0x0\n"
+ "sadalp v22.4s, v26.8h\n"
+ "movi v26.8h, #0x0\n"
+ "sadalp v21.4s, v25.8h\n"
+ "movi v25.8h, #0x0\n"
+ "mov x22, #0x0\n"
+ "4:" // no_accumulate_16
+ "ldr q19, [x23], #0x10\n"
+ "prfm pldl1keep, [x23, #0x70]\n"
+ "ldr q18, [x21], #0x10\n"
+ "ldr q17, [x20], #0x10\n"
+ "prfm pldl1keep, [x21, #0x70]\n"
+ "ldr q16, [x19], #0x10\n"
+ "prfm pldl1keep, [x20, #0x70]\n"
+ "str q19, [%x[out_ptr], #0x0]\n"
+ "sadalp v28.8h, v19.16b\n"
+ "prfm pldl1keep, [x19, #0x70]\n"
+ "str q18, [%x[out_ptr], #0x10]\n"
+ "sadalp v27.8h, v18.16b\n"
+ "str q17, [%x[out_ptr], #0x20]\n"
+ "sadalp v26.8h, v17.16b\n"
+ "str q16, [%x[out_ptr], #0x30]\n"
+ "sadalp v25.8h, v16.16b\n"
+ "add x22, x22, #0x1\n"
+ "subs %x[width], %x[width], #0x10\n"
+ "cmp %x[width], #0x10\n"
+ "add %x[out_ptr], %x[out_ptr], #0x40\n"
+ "bge 3b\n"
+ "5:" // Main loop skip
+ "cbz %x[width], 14f\n"
+ "tbz %x[width], #3, 9f\n"
+ "ldr d19, [x23], #0x8\n"
+ "ldr d18, [x21], #0x8\n"
+ "ldr d17, [x20], #0x8\n"
+ "ldr d16, [x19], #0x8\n"
+ "tbz %x[width], #2, 7f\n"
+ "ld1 { v19.s }[2], [x23], #0x4\n"
+ "ld1 { v18.s }[2], [x21], #0x4\n"
+ "ld1 { v17.s }[2], [x20], #0x4\n"
+ "ld1 { v16.s }[2], [x19], #0x4\n"
+ "tbz %x[width], #1, 6f\n"
+ "ld1 { v19.h }[6], [x23], #0x2\n"
+ "ld1 { v18.h }[6], [x21], #0x2\n"
+ "ld1 { v17.h }[6], [x20], #0x2\n"
+ "ld1 { v16.h }[6], [x19], #0x2\n"
+ "tbz %x[width], #0, 13f\n"
+ "ld1 { v19.b }[14], [x23]\n"
+ "ld1 { v18.b }[14], [x21]\n"
+ "ld1 { v17.b }[14], [x20]\n"
+ "ld1 { v16.b }[14], [x19]\n"
+ "b 13f\n"
+ "6:" // odd_loads_1_12
+ "tbz %x[width], #0, 13f\n"
+ "ld1 { v19.b }[12], [x23]\n"
+ "ld1 { v18.b }[12], [x21]\n"
+ "ld1 { v17.b }[12], [x20]\n"
+ "ld1 { v16.b }[12], [x19]\n"
+ "b 13f\n"
+ "7:" // odd_loads_2_8
+ "tbz %x[width], #1, 8f\n"
+ "ld1 { v19.h }[4], [x23], #0x2\n"
+ "ld1 { v18.h }[4], [x21], #0x2\n"
+ "ld1 { v17.h }[4], [x20], #0x2\n"
+ "ld1 { v16.h }[4], [x19], #0x2\n"
+ "tbz %x[width], #0, 13f\n"
+ "ld1 { v19.b }[10], [x23]\n"
+ "ld1 { v18.b }[10], [x21]\n"
+ "ld1 { v17.b }[10], [x20]\n"
+ "ld1 { v16.b }[10], [x19]\n"
+ "b 13f\n"
+ "8:" // odd_loads_1_8
+ "tbz %x[width], #0, 13f\n"
+ "ld1 { v19.b }[8], [x23]\n"
+ "ld1 { v18.b }[8], [x21]\n"
+ "ld1 { v17.b }[8], [x20]\n"
+ "ld1 { v16.b }[8], [x19]\n"
+ "b 13f\n"
+ "9:" // odd_loads_4_0
+ "tbz %x[width], #2, 11f\n"
+ "ldr s19, [x23], #0x4\n"
+ "ldr s18, [x21], #0x4\n"
+ "ldr s17, [x20], #0x4\n"
+ "ldr s16, [x19], #0x4\n"
+ "tbz %x[width], #1, 10f\n"
+ "ld1 { v19.h }[2], [x23], #0x2\n"
+ "ld1 { v18.h }[2], [x21], #0x2\n"
+ "ld1 { v17.h }[2], [x20], #0x2\n"
+ "ld1 { v16.h }[2], [x19], #0x2\n"
+ "tbz %x[width], #0, 13f\n"
+ "ld1 { v19.b }[6], [x23]\n"
+ "ld1 { v18.b }[6], [x21]\n"
+ "ld1 { v17.b }[6], [x20]\n"
+ "ld1 { v16.b }[6], [x19]\n"
+ "b 13f\n"
+ "10:" // odd_loads_1_4
+ "tbz %x[width], #0, 13f\n"
+ "ld1 { v19.b }[4], [x23]\n"
+ "ld1 { v18.b }[4], [x21]\n"
+ "ld1 { v17.b }[4], [x20]\n"
+ "ld1 { v16.b }[4], [x19]\n"
+ "b 13f\n"
+ "11:" // odd_loads_2_0
+ "tbz %x[width], #1, 12f\n"
+ "ldr h19, [x23], #0x2\n"
+ "ldr h18, [x21], #0x2\n"
+ "ldr h17, [x20], #0x2\n"
+ "ldr h16, [x19], #0x2\n"
+ "tbz %x[width], #0, 13f\n"
+ "ld1 { v19.b }[2], [x23]\n"
+ "ld1 { v18.b }[2], [x21]\n"
+ "ld1 { v17.b }[2], [x20]\n"
+ "ld1 { v16.b }[2], [x19]\n"
+ "b 13f\n"
+ "12:" // odd_loads_1_0
+ "ldr b19, [x23, #0x0]\n"
+ "ldr b18, [x21, #0x0]\n"
+ "ldr b17, [x20, #0x0]\n"
+ "ldr b16, [x19, #0x0]\n"
+ "13:" // Odd load end
+ "str q19, [%x[out_ptr], #0x0]\n"
+ "sadalp v28.8h, v19.16b\n"
+ "str q18, [%x[out_ptr], #0x10]\n"
+ "sadalp v27.8h, v18.16b\n"
+ "str q17, [%x[out_ptr], #0x20]\n"
+ "sadalp v26.8h, v17.16b\n"
+ "str q16, [%x[out_ptr], #0x30]\n"
+ "sadalp v25.8h, v16.16b\n"
+ "add %x[out_ptr], %x[out_ptr], #0x40\n"
+ "14:" // Odds skip
+ "sadalp v24.4s, v28.8h\n"
+ "sadalp v23.4s, v27.8h\n"
+ "addp v24.4s, v24.4s, v23.4s\n"
+ "sadalp v22.4s, v26.8h\n"
+ "sadalp v21.4s, v25.8h\n"
+ "addp v23.4s, v22.4s, v21.4s\n"
+ "addp v24.4s, v24.4s, v23.4s\n"
+ "add v24.4s, v24.4s, v20.4s\n"
+ "str q24, [%x[out_ptr], #0x0]\n"
+ "add %x[out_ptr], %x[out_ptr], #0x10\n"
+ : [out_ptr] "+r" (out_ptr), [width] "+r" (width)
+ : [first] "r" (first), [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset)
+ : "cc", "memory", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "x19", "x20", "x21", "x22", "x23"
+ );
+}
+
+
+#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave4_block16_u8_u8_summing.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave4_block16_u8_u8_summing.hpp
new file mode 100644
index 0000000..af3efb2
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave4_block16_u8_u8_summing.hpp
@@ -0,0 +1,225 @@
+/*
+ * Copyright (c) 2019-2020 Arm Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+#ifdef __aarch64__
+
+template<>
+void interleave_block<4, 16, VLType::None, true>(
+ uint8_t * &out_ptr, const uint8_t * const * in, size_t width, size_t height,
+ size_t row_offset, bool first
+)
+{
+ __asm__ __volatile__(
+ "movi v28.8h, #0x0\n"
+ "ldr x23, [%x[in], #0x0]\n"
+ "mov x22, #0x0\n"
+ "movi v27.8h, #0x0\n"
+ "ldr x21, [%x[in], #0x8]\n"
+ "cmp %x[height], #0x4\n"
+ "movi v26.8h, #0x0\n"
+ "ldr x20, [%x[in], #0x10]\n"
+ "add x23, x23, %x[row_offset]\n"
+ "movi v25.8h, #0x0\n"
+ "ldr x19, [%x[in], #0x18]\n"
+ "movi v24.4s, #0x0\n"
+ "add x21, x21, %x[row_offset]\n"
+ "movi v23.4s, #0x0\n"
+ "add x20, x20, %x[row_offset]\n"
+ "movi v22.4s, #0x0\n"
+ "add x19, x19, %x[row_offset]\n"
+ "movi v21.4s, #0x0\n"
+ "beq 1f\n"
+ "mov x19, x23\n"
+ "cmp %x[height], #0x2\n"
+ "csel x21, x21, x23, GE\n"
+ "csel x20, x20, x23, GT\n"
+ "1:" // no_pointer_adj
+ "movi v20.4s, #0x0\n"
+ "prfm pldl1keep, [x23, #0x0]\n"
+ "prfm pldl1keep, [x21, #0x0]\n"
+ "prfm pldl1keep, [x20, #0x0]\n"
+ "prfm pldl1keep, [x19, #0x0]\n"
+ "prfm pldl1keep, [x23, #0x40]\n"
+ "prfm pldl1keep, [x21, #0x40]\n"
+ "prfm pldl1keep, [x20, #0x40]\n"
+ "prfm pldl1keep, [x19, #0x40]\n"
+ "cbnz %w[first], 2f\n"
+ "sub %x[out_ptr], %x[out_ptr], #0x10\n"
+ "ld1 { v20.4s }, [%x[out_ptr]]\n"
+ "2:" // first_pass
+ "cmp %x[width], #0x10\n"
+ "blt 5f\n"
+ "3:" // Main loop head
+ "cmp x22, #0x7e\n"
+ "ble 4f\n"
+ "uadalp v24.4s, v28.8h\n"
+ "movi v28.8h, #0x0\n"
+ "uadalp v23.4s, v27.8h\n"
+ "movi v27.8h, #0x0\n"
+ "uadalp v22.4s, v26.8h\n"
+ "movi v26.8h, #0x0\n"
+ "uadalp v21.4s, v25.8h\n"
+ "movi v25.8h, #0x0\n"
+ "mov x22, #0x0\n"
+ "4:" // no_accumulate_16
+ "ldr q19, [x23], #0x10\n"
+ "prfm pldl1keep, [x23, #0x70]\n"
+ "ldr q18, [x21], #0x10\n"
+ "ldr q17, [x20], #0x10\n"
+ "prfm pldl1keep, [x21, #0x70]\n"
+ "ldr q16, [x19], #0x10\n"
+ "prfm pldl1keep, [x20, #0x70]\n"
+ "str q19, [%x[out_ptr], #0x0]\n"
+ "uadalp v28.8h, v19.16b\n"
+ "prfm pldl1keep, [x19, #0x70]\n"
+ "str q18, [%x[out_ptr], #0x10]\n"
+ "uadalp v27.8h, v18.16b\n"
+ "str q17, [%x[out_ptr], #0x20]\n"
+ "uadalp v26.8h, v17.16b\n"
+ "str q16, [%x[out_ptr], #0x30]\n"
+ "uadalp v25.8h, v16.16b\n"
+ "add x22, x22, #0x1\n"
+ "subs %x[width], %x[width], #0x10\n"
+ "cmp %x[width], #0x10\n"
+ "add %x[out_ptr], %x[out_ptr], #0x40\n"
+ "bge 3b\n"
+ "5:" // Main loop skip
+ "cbz %x[width], 14f\n"
+ "tbz %x[width], #3, 9f\n"
+ "ldr d19, [x23], #0x8\n"
+ "ldr d18, [x21], #0x8\n"
+ "ldr d17, [x20], #0x8\n"
+ "ldr d16, [x19], #0x8\n"
+ "tbz %x[width], #2, 7f\n"
+ "ld1 { v19.s }[2], [x23], #0x4\n"
+ "ld1 { v18.s }[2], [x21], #0x4\n"
+ "ld1 { v17.s }[2], [x20], #0x4\n"
+ "ld1 { v16.s }[2], [x19], #0x4\n"
+ "tbz %x[width], #1, 6f\n"
+ "ld1 { v19.h }[6], [x23], #0x2\n"
+ "ld1 { v18.h }[6], [x21], #0x2\n"
+ "ld1 { v17.h }[6], [x20], #0x2\n"
+ "ld1 { v16.h }[6], [x19], #0x2\n"
+ "tbz %x[width], #0, 13f\n"
+ "ld1 { v19.b }[14], [x23]\n"
+ "ld1 { v18.b }[14], [x21]\n"
+ "ld1 { v17.b }[14], [x20]\n"
+ "ld1 { v16.b }[14], [x19]\n"
+ "b 13f\n"
+ "6:" // odd_loads_1_12
+ "tbz %x[width], #0, 13f\n"
+ "ld1 { v19.b }[12], [x23]\n"
+ "ld1 { v18.b }[12], [x21]\n"
+ "ld1 { v17.b }[12], [x20]\n"
+ "ld1 { v16.b }[12], [x19]\n"
+ "b 13f\n"
+ "7:" // odd_loads_2_8
+ "tbz %x[width], #1, 8f\n"
+ "ld1 { v19.h }[4], [x23], #0x2\n"
+ "ld1 { v18.h }[4], [x21], #0x2\n"
+ "ld1 { v17.h }[4], [x20], #0x2\n"
+ "ld1 { v16.h }[4], [x19], #0x2\n"
+ "tbz %x[width], #0, 13f\n"
+ "ld1 { v19.b }[10], [x23]\n"
+ "ld1 { v18.b }[10], [x21]\n"
+ "ld1 { v17.b }[10], [x20]\n"
+ "ld1 { v16.b }[10], [x19]\n"
+ "b 13f\n"
+ "8:" // odd_loads_1_8
+ "tbz %x[width], #0, 13f\n"
+ "ld1 { v19.b }[8], [x23]\n"
+ "ld1 { v18.b }[8], [x21]\n"
+ "ld1 { v17.b }[8], [x20]\n"
+ "ld1 { v16.b }[8], [x19]\n"
+ "b 13f\n"
+ "9:" // odd_loads_4_0
+ "tbz %x[width], #2, 11f\n"
+ "ldr s19, [x23], #0x4\n"
+ "ldr s18, [x21], #0x4\n"
+ "ldr s17, [x20], #0x4\n"
+ "ldr s16, [x19], #0x4\n"
+ "tbz %x[width], #1, 10f\n"
+ "ld1 { v19.h }[2], [x23], #0x2\n"
+ "ld1 { v18.h }[2], [x21], #0x2\n"
+ "ld1 { v17.h }[2], [x20], #0x2\n"
+ "ld1 { v16.h }[2], [x19], #0x2\n"
+ "tbz %x[width], #0, 13f\n"
+ "ld1 { v19.b }[6], [x23]\n"
+ "ld1 { v18.b }[6], [x21]\n"
+ "ld1 { v17.b }[6], [x20]\n"
+ "ld1 { v16.b }[6], [x19]\n"
+ "b 13f\n"
+ "10:" // odd_loads_1_4
+ "tbz %x[width], #0, 13f\n"
+ "ld1 { v19.b }[4], [x23]\n"
+ "ld1 { v18.b }[4], [x21]\n"
+ "ld1 { v17.b }[4], [x20]\n"
+ "ld1 { v16.b }[4], [x19]\n"
+ "b 13f\n"
+ "11:" // odd_loads_2_0
+ "tbz %x[width], #1, 12f\n"
+ "ldr h19, [x23], #0x2\n"
+ "ldr h18, [x21], #0x2\n"
+ "ldr h17, [x20], #0x2\n"
+ "ldr h16, [x19], #0x2\n"
+ "tbz %x[width], #0, 13f\n"
+ "ld1 { v19.b }[2], [x23]\n"
+ "ld1 { v18.b }[2], [x21]\n"
+ "ld1 { v17.b }[2], [x20]\n"
+ "ld1 { v16.b }[2], [x19]\n"
+ "b 13f\n"
+ "12:" // odd_loads_1_0
+ "ldr b19, [x23, #0x0]\n"
+ "ldr b18, [x21, #0x0]\n"
+ "ldr b17, [x20, #0x0]\n"
+ "ldr b16, [x19, #0x0]\n"
+ "13:" // Odd load end
+ "str q19, [%x[out_ptr], #0x0]\n"
+ "uadalp v28.8h, v19.16b\n"
+ "str q18, [%x[out_ptr], #0x10]\n"
+ "uadalp v27.8h, v18.16b\n"
+ "str q17, [%x[out_ptr], #0x20]\n"
+ "uadalp v26.8h, v17.16b\n"
+ "str q16, [%x[out_ptr], #0x30]\n"
+ "uadalp v25.8h, v16.16b\n"
+ "add %x[out_ptr], %x[out_ptr], #0x40\n"
+ "14:" // Odds skip
+ "uadalp v24.4s, v28.8h\n"
+ "uadalp v23.4s, v27.8h\n"
+ "addp v24.4s, v24.4s, v23.4s\n"
+ "uadalp v22.4s, v26.8h\n"
+ "uadalp v21.4s, v25.8h\n"
+ "addp v23.4s, v22.4s, v21.4s\n"
+ "addp v24.4s, v24.4s, v23.4s\n"
+ "add v24.4s, v24.4s, v20.4s\n"
+ "str q24, [%x[out_ptr], #0x0]\n"
+ "add %x[out_ptr], %x[out_ptr], #0x10\n"
+ : [out_ptr] "+r" (out_ptr), [width] "+r" (width)
+ : [first] "r" (first), [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset)
+ : "cc", "memory", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "x19", "x20", "x21", "x22", "x23"
+ );
+}
+
+
+#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_bf16_fp32.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_bf16_fp32.hpp
new file mode 100644
index 0000000..34d25f2
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_bf16_fp32.hpp
@@ -0,0 +1,213 @@
+/*
+ * Copyright (c) 2019-2020 Arm Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+#ifdef __aarch64__
+
+template<>
+void interleave_block<8, 1, VLType::None, false>(
+ float * &out_ptr, const bfloat16 * const * in, size_t width, size_t height,
+ size_t row_offset, bool
+)
+{
+ __asm__ __volatile__(
+ "movi v29.8h, #0x0\n"
+ "ldr x27, [%x[in], #0x0]\n"
+ "cmp %x[height], #0x8\n"
+ "ldr x26, [%x[in], #0x8]\n"
+ "add x27, x27, %x[row_offset], LSL #1\n"
+ "ldr x25, [%x[in], #0x10]\n"
+ "ldr x24, [%x[in], #0x18]\n"
+ "add x26, x26, %x[row_offset], LSL #1\n"
+ "ldr x23, [%x[in], #0x20]\n"
+ "add x25, x25, %x[row_offset], LSL #1\n"
+ "ldr x22, [%x[in], #0x28]\n"
+ "ldr x21, [%x[in], #0x30]\n"
+ "add x24, x24, %x[row_offset], LSL #1\n"
+ "ldr x20, [%x[in], #0x38]\n"
+ "add x23, x23, %x[row_offset], LSL #1\n"
+ "add x22, x22, %x[row_offset], LSL #1\n"
+ "add x21, x21, %x[row_offset], LSL #1\n"
+ "add x20, x20, %x[row_offset], LSL #1\n"
+ "beq 1f\n"
+ "mov x20, x27\n"
+ "cmp %x[height], #0x2\n"
+ "csel x26, x26, x27, GE\n"
+ "csel x25, x25, x27, GT\n"
+ "cmp %x[height], #0x4\n"
+ "csel x24, x24, x27, GE\n"
+ "csel x23, x23, x27, GT\n"
+ "cmp %x[height], #0x6\n"
+ "csel x22, x22, x27, GE\n"
+ "csel x21, x21, x27, GT\n"
+ "1:" // no_pointer_adj
+ "prfm pldl1keep, [x27, #0x0]\n"
+ "cmp %x[width], #0x4\n"
+ "prfm pldl1keep, [x26, #0x0]\n"
+ "prfm pldl1keep, [x25, #0x0]\n"
+ "prfm pldl1keep, [x24, #0x0]\n"
+ "prfm pldl1keep, [x23, #0x0]\n"
+ "prfm pldl1keep, [x22, #0x0]\n"
+ "prfm pldl1keep, [x21, #0x0]\n"
+ "prfm pldl1keep, [x20, #0x0]\n"
+ "prfm pldl1keep, [x27, #0x40]\n"
+ "prfm pldl1keep, [x26, #0x40]\n"
+ "prfm pldl1keep, [x25, #0x40]\n"
+ "prfm pldl1keep, [x24, #0x40]\n"
+ "prfm pldl1keep, [x23, #0x40]\n"
+ "prfm pldl1keep, [x22, #0x40]\n"
+ "prfm pldl1keep, [x21, #0x40]\n"
+ "prfm pldl1keep, [x20, #0x40]\n"
+ "blt 3f\n"
+ "2:" // Main loop head
+ "ldr d28, [x27], #0x8\n"
+ "zip1 v28.8h, v29.8h, v28.8h\n"
+ "prfm pldl1keep, [x27, #0x70]\n"
+ "ldr d27, [x26], #0x8\n"
+ "zip1 v27.8h, v29.8h, v27.8h\n"
+ "prfm pldl1keep, [x26, #0x70]\n"
+ "ldr d26, [x25], #0x8\n"
+ "zip1 v26.8h, v29.8h, v26.8h\n"
+ "prfm pldl1keep, [x25, #0x70]\n"
+ "ldr d25, [x24], #0x8\n"
+ "zip1 v20.4s, v28.4s, v26.4s\n"
+ "prfm pldl1keep, [x24, #0x70]\n"
+ "zip1 v25.8h, v29.8h, v25.8h\n"
+ "ldr d24, [x23], #0x8\n"
+ "zip1 v19.4s, v27.4s, v25.4s\n"
+ "prfm pldl1keep, [x23, #0x70]\n"
+ "zip1 v24.8h, v29.8h, v24.8h\n"
+ "ldr d23, [x22], #0x8\n"
+ "zip1 v16.4s, v20.4s, v19.4s\n"
+ "prfm pldl1keep, [x22, #0x70]\n"
+ "zip1 v23.8h, v29.8h, v23.8h\n"
+ "ldr d22, [x21], #0x8\n"
+ "zip2 v19.4s, v20.4s, v19.4s\n"
+ "prfm pldl1keep, [x21, #0x70]\n"
+ "zip1 v22.8h, v29.8h, v22.8h\n"
+ "ldr d21, [x20], #0x8\n"
+ "zip1 v18.4s, v24.4s, v22.4s\n"
+ "prfm pldl1keep, [x20, #0x70]\n"
+ "zip1 v21.8h, v29.8h, v21.8h\n"
+ "str q16, [%x[out_ptr], #0x0]\n"
+ "zip1 v17.4s, v23.4s, v21.4s\n"
+ "subs %x[width], %x[width], #0x4\n"
+ "zip2 v20.4s, v28.4s, v26.4s\n"
+ "cmp %x[width], #0x4\n"
+ "zip1 v16.4s, v18.4s, v17.4s\n"
+ "str q16, [%x[out_ptr], #0x10]\n"
+ "zip2 v16.4s, v18.4s, v17.4s\n"
+ "str q19, [%x[out_ptr], #0x20]\n"
+ "zip2 v19.4s, v27.4s, v25.4s\n"
+ "str q16, [%x[out_ptr], #0x30]\n"
+ "zip1 v16.4s, v20.4s, v19.4s\n"
+ "str q16, [%x[out_ptr], #0x40]\n"
+ "zip2 v18.4s, v24.4s, v22.4s\n"
+ "zip2 v17.4s, v23.4s, v21.4s\n"
+ "zip1 v16.4s, v18.4s, v17.4s\n"
+ "str q16, [%x[out_ptr], #0x50]\n"
+ "zip2 v16.4s, v20.4s, v19.4s\n"
+ "str q16, [%x[out_ptr], #0x60]\n"
+ "zip2 v16.4s, v18.4s, v17.4s\n"
+ "str q16, [%x[out_ptr], #0x70]\n"
+ "add %x[out_ptr], %x[out_ptr], #0x80\n"
+ "bge 2b\n"
+ "3:" // Main loop skip
+ "cbz %x[width], 6f\n"
+ "tbz %x[width], #1, 4f\n"
+ "ldr s28, [x27], #0x4\n"
+ "ldr s27, [x26], #0x4\n"
+ "ldr s26, [x25], #0x4\n"
+ "ldr s25, [x24], #0x4\n"
+ "ldr s24, [x23], #0x4\n"
+ "ldr s23, [x22], #0x4\n"
+ "ldr s22, [x21], #0x4\n"
+ "ldr s21, [x20], #0x4\n"
+ "mov x19, #0x2\n"
+ "tbz %x[width], #0, 5f\n"
+ "ld1 { v28.h }[2], [x27]\n"
+ "ld1 { v27.h }[2], [x26]\n"
+ "ld1 { v26.h }[2], [x25]\n"
+ "ld1 { v25.h }[2], [x24]\n"
+ "ld1 { v24.h }[2], [x23]\n"
+ "ld1 { v23.h }[2], [x22]\n"
+ "ld1 { v22.h }[2], [x21]\n"
+ "ld1 { v21.h }[2], [x20]\n"
+ "mov x19, #0x3\n"
+ "b 5f\n"
+ "4:" // odd_loads_1_0
+ "ldr h28, [x27, #0x0]\n"
+ "ldr h27, [x26, #0x0]\n"
+ "ldr h26, [x25, #0x0]\n"
+ "ldr h25, [x24, #0x0]\n"
+ "ldr h24, [x23, #0x0]\n"
+ "ldr h23, [x22, #0x0]\n"
+ "ldr h22, [x21, #0x0]\n"
+ "ldr h21, [x20, #0x0]\n"
+ "mov x19, #0x1\n"
+ "5:" // Odd load end
+ "zip1 v28.8h, v29.8h, v28.8h\n"
+ "subs x19, x19, #0x1\n"
+ "zip1 v27.8h, v29.8h, v27.8h\n"
+ "zip1 v26.8h, v29.8h, v26.8h\n"
+ "zip1 v25.8h, v29.8h, v25.8h\n"
+ "zip1 v24.8h, v29.8h, v24.8h\n"
+ "zip1 v23.8h, v29.8h, v23.8h\n"
+ "zip1 v22.8h, v29.8h, v22.8h\n"
+ "zip1 v21.8h, v29.8h, v21.8h\n"
+ "zip1 v20.4s, v28.4s, v26.4s\n"
+ "zip1 v19.4s, v27.4s, v25.4s\n"
+ "zip1 v16.4s, v20.4s, v19.4s\n"
+ "str q16, [%x[out_ptr], #0x0]\n"
+ "zip1 v18.4s, v24.4s, v22.4s\n"
+ "zip1 v17.4s, v23.4s, v21.4s\n"
+ "zip1 v16.4s, v18.4s, v17.4s\n"
+ "str q16, [%x[out_ptr], #0x10]\n"
+ "add %x[out_ptr], %x[out_ptr], #0x20\n"
+ "beq 6f\n"
+ "zip2 v19.4s, v20.4s, v19.4s\n"
+ "zip2 v16.4s, v18.4s, v17.4s\n"
+ "str q19, [%x[out_ptr], #0x0]\n"
+ "str q16, [%x[out_ptr], #0x10]\n"
+ "subs x19, x19, #0x1\n"
+ "add %x[out_ptr], %x[out_ptr], #0x20\n"
+ "beq 6f\n"
+ "zip2 v20.4s, v28.4s, v26.4s\n"
+ "zip2 v19.4s, v27.4s, v25.4s\n"
+ "zip1 v16.4s, v20.4s, v19.4s\n"
+ "str q16, [%x[out_ptr], #0x0]\n"
+ "zip2 v18.4s, v24.4s, v22.4s\n"
+ "zip2 v17.4s, v23.4s, v21.4s\n"
+ "zip1 v16.4s, v18.4s, v17.4s\n"
+ "str q16, [%x[out_ptr], #0x10]\n"
+ "add %x[out_ptr], %x[out_ptr], #0x20\n"
+ "6:" // Odds skip
+
+ : [out_ptr] "+r" (out_ptr), [width] "+r" (width)
+ : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset)
+ : "cc", "memory", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27"
+ );
+}
+
+
+#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_fp16_fp16.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_fp16_fp16.hpp
new file mode 100644
index 0000000..d547957
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_fp16_fp16.hpp
@@ -0,0 +1,270 @@
+/*
+ * Copyright (c) 2019-2020 Arm Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+#ifdef __aarch64__
+
+template<>
+void interleave_block<8, 1, VLType::None, false>(
+ __fp16 * &out_ptr, const __fp16 * const * in, size_t width, size_t height,
+ size_t row_offset, bool
+)
+{
+ __asm__ __volatile__(
+ "ldr x27, [%x[in], #0x0]\n"
+ "cmp %x[height], #0x8\n"
+ "ldr x26, [%x[in], #0x8]\n"
+ "add x27, x27, %x[row_offset], LSL #1\n"
+ "ldr x25, [%x[in], #0x10]\n"
+ "ldr x24, [%x[in], #0x18]\n"
+ "add x26, x26, %x[row_offset], LSL #1\n"
+ "ldr x23, [%x[in], #0x20]\n"
+ "add x25, x25, %x[row_offset], LSL #1\n"
+ "ldr x22, [%x[in], #0x28]\n"
+ "ldr x21, [%x[in], #0x30]\n"
+ "add x24, x24, %x[row_offset], LSL #1\n"
+ "ldr x20, [%x[in], #0x38]\n"
+ "add x23, x23, %x[row_offset], LSL #1\n"
+ "add x22, x22, %x[row_offset], LSL #1\n"
+ "add x21, x21, %x[row_offset], LSL #1\n"
+ "add x20, x20, %x[row_offset], LSL #1\n"
+ "beq 1f\n"
+ "mov x20, x27\n"
+ "cmp %x[height], #0x2\n"
+ "csel x26, x26, x27, GE\n"
+ "csel x25, x25, x27, GT\n"
+ "cmp %x[height], #0x4\n"
+ "csel x24, x24, x27, GE\n"
+ "csel x23, x23, x27, GT\n"
+ "cmp %x[height], #0x6\n"
+ "csel x22, x22, x27, GE\n"
+ "csel x21, x21, x27, GT\n"
+ "1:" // no_pointer_adj
+ "prfm pldl1keep, [x27, #0x0]\n"
+ "cmp %x[width], #0x8\n"
+ "prfm pldl1keep, [x26, #0x0]\n"
+ "prfm pldl1keep, [x25, #0x0]\n"
+ "prfm pldl1keep, [x24, #0x0]\n"
+ "prfm pldl1keep, [x23, #0x0]\n"
+ "prfm pldl1keep, [x22, #0x0]\n"
+ "prfm pldl1keep, [x21, #0x0]\n"
+ "prfm pldl1keep, [x20, #0x0]\n"
+ "prfm pldl1keep, [x27, #0x40]\n"
+ "prfm pldl1keep, [x26, #0x40]\n"
+ "prfm pldl1keep, [x25, #0x40]\n"
+ "prfm pldl1keep, [x24, #0x40]\n"
+ "prfm pldl1keep, [x23, #0x40]\n"
+ "prfm pldl1keep, [x22, #0x40]\n"
+ "prfm pldl1keep, [x21, #0x40]\n"
+ "prfm pldl1keep, [x20, #0x40]\n"
+ "blt 3f\n"
+ "2:" // Main loop head
+ "ldr q30, [x27], #0x10\n"
+ "prfm pldl1keep, [x27, #0x70]\n"
+ "ldr q29, [x26], #0x10\n"
+ "ldr q28, [x25], #0x10\n"
+ "prfm pldl1keep, [x26, #0x70]\n"
+ "ldr q27, [x24], #0x10\n"
+ "prfm pldl1keep, [x25, #0x70]\n"
+ "ldr q24, [x23], #0x10\n"
+ "zip1 v26.8h, v30.8h, v24.8h\n"
+ "prfm pldl1keep, [x24, #0x70]\n"
+ "ldr q25, [x22], #0x10\n"
+ "zip2 v24.8h, v30.8h, v24.8h\n"
+ "prfm pldl1keep, [x23, #0x70]\n"
+ "ldr q23, [x21], #0x10\n"
+ "zip1 v21.8h, v29.8h, v25.8h\n"
+ "prfm pldl1keep, [x22, #0x70]\n"
+ "ldr q22, [x20], #0x10\n"
+ "zip1 v18.8h, v28.8h, v23.8h\n"
+ "prfm pldl1keep, [x21, #0x70]\n"
+ "subs %x[width], %x[width], #0x8\n"
+ "zip1 v20.8h, v26.8h, v18.8h\n"
+ "prfm pldl1keep, [x20, #0x70]\n"
+ "zip1 v19.8h, v27.8h, v22.8h\n"
+ "cmp %x[width], #0x8\n"
+ "zip1 v17.8h, v21.8h, v19.8h\n"
+ "zip2 v18.8h, v26.8h, v18.8h\n"
+ "zip1 v16.8h, v20.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x0]\n"
+ "zip2 v16.8h, v20.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x10]\n"
+ "zip2 v17.8h, v21.8h, v19.8h\n"
+ "zip1 v16.8h, v18.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x20]\n"
+ "zip2 v16.8h, v18.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x30]\n"
+ "zip2 v21.8h, v28.8h, v23.8h\n"
+ "zip1 v18.8h, v24.8h, v21.8h\n"
+ "zip2 v20.8h, v29.8h, v25.8h\n"
+ "zip2 v19.8h, v27.8h, v22.8h\n"
+ "zip1 v17.8h, v20.8h, v19.8h\n"
+ "zip1 v16.8h, v18.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x40]\n"
+ "zip2 v16.8h, v18.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x50]\n"
+ "zip2 v18.8h, v24.8h, v21.8h\n"
+ "zip2 v17.8h, v20.8h, v19.8h\n"
+ "zip1 v16.8h, v18.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x60]\n"
+ "zip2 v16.8h, v18.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x70]\n"
+ "add %x[out_ptr], %x[out_ptr], #0x80\n"
+ "bge 2b\n"
+ "3:" // Main loop skip
+ "cbz %x[width], 8f\n"
+ "tbz %x[width], #2, 5f\n"
+ "ldr d30, [x27], #0x8\n"
+ "ldr d29, [x26], #0x8\n"
+ "ldr d28, [x25], #0x8\n"
+ "ldr d27, [x24], #0x8\n"
+ "ldr d24, [x23], #0x8\n"
+ "ldr d25, [x22], #0x8\n"
+ "ldr d23, [x21], #0x8\n"
+ "ldr d22, [x20], #0x8\n"
+ "tbz %x[width], #1, 4f\n"
+ "ld1 { v30.s }[2], [x27], #0x4\n"
+ "ld1 { v29.s }[2], [x26], #0x4\n"
+ "ld1 { v28.s }[2], [x25], #0x4\n"
+ "ld1 { v27.s }[2], [x24], #0x4\n"
+ "ld1 { v24.s }[2], [x23], #0x4\n"
+ "ld1 { v25.s }[2], [x22], #0x4\n"
+ "ld1 { v23.s }[2], [x21], #0x4\n"
+ "ld1 { v22.s }[2], [x20], #0x4\n"
+ "mov x19, #0x6\n"
+ "tbz %x[width], #0, 7f\n"
+ "ld1 { v30.h }[6], [x27]\n"
+ "ld1 { v29.h }[6], [x26]\n"
+ "ld1 { v28.h }[6], [x25]\n"
+ "ld1 { v27.h }[6], [x24]\n"
+ "ld1 { v24.h }[6], [x23]\n"
+ "ld1 { v25.h }[6], [x22]\n"
+ "ld1 { v23.h }[6], [x21]\n"
+ "ld1 { v22.h }[6], [x20]\n"
+ "mov x19, #0x7\n"
+ "b 7f\n"
+ "4:" // odd_loads_1_4
+ "mov x19, #0x4\n"
+ "tbz %x[width], #0, 7f\n"
+ "ld1 { v30.h }[4], [x27]\n"
+ "ld1 { v29.h }[4], [x26]\n"
+ "ld1 { v28.h }[4], [x25]\n"
+ "ld1 { v27.h }[4], [x24]\n"
+ "ld1 { v24.h }[4], [x23]\n"
+ "ld1 { v25.h }[4], [x22]\n"
+ "ld1 { v23.h }[4], [x21]\n"
+ "ld1 { v22.h }[4], [x20]\n"
+ "mov x19, #0x5\n"
+ "b 7f\n"
+ "5:" // odd_loads_2_0
+ "tbz %x[width], #1, 6f\n"
+ "ldr s30, [x27], #0x4\n"
+ "ldr s29, [x26], #0x4\n"
+ "ldr s28, [x25], #0x4\n"
+ "ldr s27, [x24], #0x4\n"
+ "ldr s24, [x23], #0x4\n"
+ "ldr s25, [x22], #0x4\n"
+ "ldr s23, [x21], #0x4\n"
+ "ldr s22, [x20], #0x4\n"
+ "mov x19, #0x2\n"
+ "tbz %x[width], #0, 7f\n"
+ "ld1 { v30.h }[2], [x27]\n"
+ "ld1 { v29.h }[2], [x26]\n"
+ "ld1 { v28.h }[2], [x25]\n"
+ "ld1 { v27.h }[2], [x24]\n"
+ "ld1 { v24.h }[2], [x23]\n"
+ "ld1 { v25.h }[2], [x22]\n"
+ "ld1 { v23.h }[2], [x21]\n"
+ "ld1 { v22.h }[2], [x20]\n"
+ "mov x19, #0x3\n"
+ "b 7f\n"
+ "6:" // odd_loads_1_0
+ "ldr h30, [x27, #0x0]\n"
+ "ldr h29, [x26, #0x0]\n"
+ "ldr h28, [x25, #0x0]\n"
+ "ldr h27, [x24, #0x0]\n"
+ "ldr h24, [x23, #0x0]\n"
+ "ldr h25, [x22, #0x0]\n"
+ "ldr h23, [x21, #0x0]\n"
+ "ldr h22, [x20, #0x0]\n"
+ "mov x19, #0x1\n"
+ "7:" // Odd load end
+ "zip1 v26.8h, v30.8h, v24.8h\n"
+ "subs x19, x19, #0x1\n"
+ "zip1 v18.8h, v28.8h, v23.8h\n"
+ "zip1 v20.8h, v26.8h, v18.8h\n"
+ "zip1 v21.8h, v29.8h, v25.8h\n"
+ "zip1 v19.8h, v27.8h, v22.8h\n"
+ "zip1 v17.8h, v21.8h, v19.8h\n"
+ "zip1 v16.8h, v20.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x0]\n"
+ "add %x[out_ptr], %x[out_ptr], #0x10\n"
+ "beq 8f\n"
+ "zip2 v16.8h, v20.8h, v17.8h\n"
+ "subs x19, x19, #0x1\n"
+ "str q16, [%x[out_ptr], #0x0]\n"
+ "add %x[out_ptr], %x[out_ptr], #0x10\n"
+ "beq 8f\n"
+ "zip2 v18.8h, v26.8h, v18.8h\n"
+ "zip2 v17.8h, v21.8h, v19.8h\n"
+ "subs x19, x19, #0x1\n"
+ "zip1 v16.8h, v18.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x0]\n"
+ "add %x[out_ptr], %x[out_ptr], #0x10\n"
+ "beq 8f\n"
+ "zip2 v16.8h, v18.8h, v17.8h\n"
+ "subs x19, x19, #0x1\n"
+ "str q16, [%x[out_ptr], #0x0]\n"
+ "add %x[out_ptr], %x[out_ptr], #0x10\n"
+ "beq 8f\n"
+ "zip2 v24.8h, v30.8h, v24.8h\n"
+ "zip2 v21.8h, v28.8h, v23.8h\n"
+ "subs x19, x19, #0x1\n"
+ "zip1 v18.8h, v24.8h, v21.8h\n"
+ "zip2 v20.8h, v29.8h, v25.8h\n"
+ "zip2 v19.8h, v27.8h, v22.8h\n"
+ "zip1 v17.8h, v20.8h, v19.8h\n"
+ "zip1 v16.8h, v18.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x0]\n"
+ "add %x[out_ptr], %x[out_ptr], #0x10\n"
+ "beq 8f\n"
+ "zip2 v16.8h, v18.8h, v17.8h\n"
+ "subs x19, x19, #0x1\n"
+ "str q16, [%x[out_ptr], #0x0]\n"
+ "add %x[out_ptr], %x[out_ptr], #0x10\n"
+ "beq 8f\n"
+ "zip2 v18.8h, v24.8h, v21.8h\n"
+ "zip2 v17.8h, v20.8h, v19.8h\n"
+ "zip1 v16.8h, v18.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x0]\n"
+ "add %x[out_ptr], %x[out_ptr], #0x10\n"
+ "8:" // Odds skip
+
+ : [out_ptr] "+r" (out_ptr), [width] "+r" (width)
+ : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset)
+ : "cc", "memory", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27"
+ );
+}
+
+
+#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_fp16_fp32.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_fp16_fp32.hpp
new file mode 100644
index 0000000..b45e622
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_fp16_fp32.hpp
@@ -0,0 +1,212 @@
+/*
+ * Copyright (c) 2019-2020 Arm Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+#ifdef __aarch64__
+
+template<>
+void interleave_block<8, 1, VLType::None, false>(
+ float * &out_ptr, const __fp16 * const * in, size_t width, size_t height,
+ size_t row_offset, bool
+)
+{
+ __asm__ __volatile__(
+ "ldr x27, [%x[in], #0x0]\n"
+ "cmp %x[height], #0x8\n"
+ "ldr x26, [%x[in], #0x8]\n"
+ "add x27, x27, %x[row_offset], LSL #1\n"
+ "ldr x25, [%x[in], #0x10]\n"
+ "ldr x24, [%x[in], #0x18]\n"
+ "add x26, x26, %x[row_offset], LSL #1\n"
+ "ldr x23, [%x[in], #0x20]\n"
+ "add x25, x25, %x[row_offset], LSL #1\n"
+ "ldr x22, [%x[in], #0x28]\n"
+ "ldr x21, [%x[in], #0x30]\n"
+ "add x24, x24, %x[row_offset], LSL #1\n"
+ "ldr x20, [%x[in], #0x38]\n"
+ "add x23, x23, %x[row_offset], LSL #1\n"
+ "add x22, x22, %x[row_offset], LSL #1\n"
+ "add x21, x21, %x[row_offset], LSL #1\n"
+ "add x20, x20, %x[row_offset], LSL #1\n"
+ "beq 1f\n"
+ "mov x20, x27\n"
+ "cmp %x[height], #0x2\n"
+ "csel x26, x26, x27, GE\n"
+ "csel x25, x25, x27, GT\n"
+ "cmp %x[height], #0x4\n"
+ "csel x24, x24, x27, GE\n"
+ "csel x23, x23, x27, GT\n"
+ "cmp %x[height], #0x6\n"
+ "csel x22, x22, x27, GE\n"
+ "csel x21, x21, x27, GT\n"
+ "1:" // no_pointer_adj
+ "prfm pldl1keep, [x27, #0x0]\n"
+ "cmp %x[width], #0x4\n"
+ "prfm pldl1keep, [x26, #0x0]\n"
+ "prfm pldl1keep, [x25, #0x0]\n"
+ "prfm pldl1keep, [x24, #0x0]\n"
+ "prfm pldl1keep, [x23, #0x0]\n"
+ "prfm pldl1keep, [x22, #0x0]\n"
+ "prfm pldl1keep, [x21, #0x0]\n"
+ "prfm pldl1keep, [x20, #0x0]\n"
+ "prfm pldl1keep, [x27, #0x40]\n"
+ "prfm pldl1keep, [x26, #0x40]\n"
+ "prfm pldl1keep, [x25, #0x40]\n"
+ "prfm pldl1keep, [x24, #0x40]\n"
+ "prfm pldl1keep, [x23, #0x40]\n"
+ "prfm pldl1keep, [x22, #0x40]\n"
+ "prfm pldl1keep, [x21, #0x40]\n"
+ "prfm pldl1keep, [x20, #0x40]\n"
+ "blt 3f\n"
+ "2:" // Main loop head
+ "ldr d29, [x27], #0x8\n"
+ "prfm pldl1keep, [x27, #0x70]\n"
+ "ldr d28, [x26], #0x8\n"
+ "ldr d27, [x25], #0x8\n"
+ "prfm pldl1keep, [x26, #0x70]\n"
+ "ldr d26, [x24], #0x8\n"
+ "prfm pldl1keep, [x25, #0x70]\n"
+ "ldr d25, [x23], #0x8\n"
+ "ldr d24, [x22], #0x8\n"
+ "prfm pldl1keep, [x24, #0x70]\n"
+ "ldr d23, [x21], #0x8\n"
+ "ldr d22, [x20], #0x8\n"
+ "prfm pldl1keep, [x23, #0x70]\n"
+ "prfm pldl1keep, [x22, #0x70]\n"
+ "fcvtl v29.4s, v29.4h\n"
+ "fcvtl v28.4s, v28.4h\n"
+ "prfm pldl1keep, [x21, #0x70]\n"
+ "fcvtl v27.4s, v27.4h\n"
+ "zip1 v20.4s, v29.4s, v27.4s\n"
+ "prfm pldl1keep, [x20, #0x70]\n"
+ "fcvtl v26.4s, v26.4h\n"
+ "zip2 v18.4s, v29.4s, v27.4s\n"
+ "fcvtl v25.4s, v25.4h\n"
+ "fcvtl v24.4s, v24.4h\n"
+ "zip1 v19.4s, v28.4s, v26.4s\n"
+ "fcvtl v23.4s, v23.4h\n"
+ "zip2 v17.4s, v28.4s, v26.4s\n"
+ "fcvtl v22.4s, v22.4h\n"
+ "zip1 v16.4s, v20.4s, v19.4s\n"
+ "str q16, [%x[out_ptr], #0x0]\n"
+ "zip2 v21.4s, v20.4s, v19.4s\n"
+ "subs %x[width], %x[width], #0x4\n"
+ "zip1 v20.4s, v18.4s, v17.4s\n"
+ "cmp %x[width], #0x4\n"
+ "zip2 v19.4s, v18.4s, v17.4s\n"
+ "zip1 v18.4s, v25.4s, v23.4s\n"
+ "zip1 v17.4s, v24.4s, v22.4s\n"
+ "zip1 v16.4s, v18.4s, v17.4s\n"
+ "str q16, [%x[out_ptr], #0x10]\n"
+ "zip2 v16.4s, v18.4s, v17.4s\n"
+ "str q21, [%x[out_ptr], #0x20]\n"
+ "zip2 v18.4s, v25.4s, v23.4s\n"
+ "str q16, [%x[out_ptr], #0x30]\n"
+ "zip2 v17.4s, v24.4s, v22.4s\n"
+ "str q20, [%x[out_ptr], #0x40]\n"
+ "zip1 v16.4s, v18.4s, v17.4s\n"
+ "str q16, [%x[out_ptr], #0x50]\n"
+ "zip2 v16.4s, v18.4s, v17.4s\n"
+ "str q19, [%x[out_ptr], #0x60]\n"
+ "str q16, [%x[out_ptr], #0x70]\n"
+ "add %x[out_ptr], %x[out_ptr], #0x80\n"
+ "bge 2b\n"
+ "3:" // Main loop skip
+ "cbz %x[width], 6f\n"
+ "tbz %x[width], #1, 4f\n"
+ "ldr s29, [x27], #0x4\n"
+ "ldr s28, [x26], #0x4\n"
+ "ldr s27, [x25], #0x4\n"
+ "ldr s26, [x24], #0x4\n"
+ "ldr s25, [x23], #0x4\n"
+ "ldr s24, [x22], #0x4\n"
+ "ldr s23, [x21], #0x4\n"
+ "ldr s22, [x20], #0x4\n"
+ "mov x19, #0x2\n"
+ "tbz %x[width], #0, 5f\n"
+ "ld1 { v29.h }[2], [x27]\n"
+ "ld1 { v28.h }[2], [x26]\n"
+ "ld1 { v27.h }[2], [x25]\n"
+ "ld1 { v26.h }[2], [x24]\n"
+ "ld1 { v25.h }[2], [x23]\n"
+ "ld1 { v24.h }[2], [x22]\n"
+ "ld1 { v23.h }[2], [x21]\n"
+ "ld1 { v22.h }[2], [x20]\n"
+ "mov x19, #0x3\n"
+ "b 5f\n"
+ "4:" // odd_loads_1_0
+ "ldr h29, [x27, #0x0]\n"
+ "ldr h28, [x26, #0x0]\n"
+ "ldr h27, [x25, #0x0]\n"
+ "ldr h26, [x24, #0x0]\n"
+ "ldr h25, [x23, #0x0]\n"
+ "ldr h24, [x22, #0x0]\n"
+ "ldr h23, [x21, #0x0]\n"
+ "ldr h22, [x20, #0x0]\n"
+ "mov x19, #0x1\n"
+ "5:" // Odd load end
+ "fcvtl v29.4s, v29.4h\n"
+ "fcvtl v28.4s, v28.4h\n"
+ "fcvtl v27.4s, v27.4h\n"
+ "zip1 v20.4s, v29.4s, v27.4s\n"
+ "fcvtl v26.4s, v26.4h\n"
+ "fcvtl v25.4s, v25.4h\n"
+ "zip1 v19.4s, v28.4s, v26.4s\n"
+ "fcvtl v24.4s, v24.4h\n"
+ "fcvtl v23.4s, v23.4h\n"
+ "zip1 v16.4s, v20.4s, v19.4s\n"
+ "fcvtl v22.4s, v22.4h\n"
+ "zip1 v18.4s, v25.4s, v23.4s\n"
+ "str q16, [%x[out_ptr], #0x0]\n"
+ "subs x19, x19, #0x1\n"
+ "zip1 v17.4s, v24.4s, v22.4s\n"
+ "zip1 v16.4s, v18.4s, v17.4s\n"
+ "str q16, [%x[out_ptr], #0x10]\n"
+ "add %x[out_ptr], %x[out_ptr], #0x20\n"
+ "beq 6f\n"
+ "zip2 v21.4s, v20.4s, v19.4s\n"
+ "zip2 v16.4s, v18.4s, v17.4s\n"
+ "str q21, [%x[out_ptr], #0x0]\n"
+ "str q16, [%x[out_ptr], #0x10]\n"
+ "subs x19, x19, #0x1\n"
+ "add %x[out_ptr], %x[out_ptr], #0x20\n"
+ "beq 6f\n"
+ "zip2 v18.4s, v29.4s, v27.4s\n"
+ "zip2 v17.4s, v28.4s, v26.4s\n"
+ "zip1 v20.4s, v18.4s, v17.4s\n"
+ "str q20, [%x[out_ptr], #0x0]\n"
+ "zip2 v18.4s, v25.4s, v23.4s\n"
+ "zip2 v17.4s, v24.4s, v22.4s\n"
+ "zip1 v16.4s, v18.4s, v17.4s\n"
+ "str q16, [%x[out_ptr], #0x10]\n"
+ "add %x[out_ptr], %x[out_ptr], #0x20\n"
+ "6:" // Odds skip
+
+ : [out_ptr] "+r" (out_ptr), [width] "+r" (width)
+ : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset)
+ : "cc", "memory", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27"
+ );
+}
+
+
+#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_fp32_fp32.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_fp32_fp32.hpp
new file mode 100644
index 0000000..3f38859
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_fp32_fp32.hpp
@@ -0,0 +1,196 @@
+/*
+ * Copyright (c) 2019-2020 Arm Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+#ifdef __aarch64__
+
+template<>
+void interleave_block<8, 1, VLType::None, false>(
+ float * &out_ptr, const float * const * in, size_t width, size_t height,
+ size_t row_offset, bool
+)
+{
+ __asm__ __volatile__(
+ "ldr x27, [%x[in], #0x0]\n"
+ "cmp %x[height], #0x8\n"
+ "ldr x26, [%x[in], #0x8]\n"
+ "add x27, x27, %x[row_offset], LSL #2\n"
+ "ldr x25, [%x[in], #0x10]\n"
+ "ldr x24, [%x[in], #0x18]\n"
+ "add x26, x26, %x[row_offset], LSL #2\n"
+ "ldr x23, [%x[in], #0x20]\n"
+ "add x25, x25, %x[row_offset], LSL #2\n"
+ "ldr x22, [%x[in], #0x28]\n"
+ "ldr x21, [%x[in], #0x30]\n"
+ "add x24, x24, %x[row_offset], LSL #2\n"
+ "ldr x20, [%x[in], #0x38]\n"
+ "add x23, x23, %x[row_offset], LSL #2\n"
+ "add x22, x22, %x[row_offset], LSL #2\n"
+ "add x21, x21, %x[row_offset], LSL #2\n"
+ "add x20, x20, %x[row_offset], LSL #2\n"
+ "beq 1f\n"
+ "mov x20, x27\n"
+ "cmp %x[height], #0x2\n"
+ "csel x26, x26, x27, GE\n"
+ "csel x25, x25, x27, GT\n"
+ "cmp %x[height], #0x4\n"
+ "csel x24, x24, x27, GE\n"
+ "csel x23, x23, x27, GT\n"
+ "cmp %x[height], #0x6\n"
+ "csel x22, x22, x27, GE\n"
+ "csel x21, x21, x27, GT\n"
+ "1:" // no_pointer_adj
+ "prfm pldl1keep, [x27, #0x0]\n"
+ "cmp %x[width], #0x4\n"
+ "prfm pldl1keep, [x26, #0x0]\n"
+ "prfm pldl1keep, [x25, #0x0]\n"
+ "prfm pldl1keep, [x24, #0x0]\n"
+ "prfm pldl1keep, [x23, #0x0]\n"
+ "prfm pldl1keep, [x22, #0x0]\n"
+ "prfm pldl1keep, [x21, #0x0]\n"
+ "prfm pldl1keep, [x20, #0x0]\n"
+ "prfm pldl1keep, [x27, #0x40]\n"
+ "prfm pldl1keep, [x26, #0x40]\n"
+ "prfm pldl1keep, [x25, #0x40]\n"
+ "prfm pldl1keep, [x24, #0x40]\n"
+ "prfm pldl1keep, [x23, #0x40]\n"
+ "prfm pldl1keep, [x22, #0x40]\n"
+ "prfm pldl1keep, [x21, #0x40]\n"
+ "prfm pldl1keep, [x20, #0x40]\n"
+ "blt 3f\n"
+ "2:" // Main loop head
+ "ldr q28, [x27], #0x10\n"
+ "prfm pldl1keep, [x27, #0x70]\n"
+ "ldr q27, [x26], #0x10\n"
+ "ldr q26, [x25], #0x10\n"
+ "zip1 v23.4s, v28.4s, v26.4s\n"
+ "prfm pldl1keep, [x26, #0x70]\n"
+ "ldr q22, [x24], #0x10\n"
+ "zip2 v26.4s, v28.4s, v26.4s\n"
+ "prfm pldl1keep, [x25, #0x70]\n"
+ "ldr q25, [x23], #0x10\n"
+ "zip1 v20.4s, v27.4s, v22.4s\n"
+ "prfm pldl1keep, [x24, #0x70]\n"
+ "ldr q24, [x22], #0x10\n"
+ "zip1 v16.4s, v23.4s, v20.4s\n"
+ "prfm pldl1keep, [x23, #0x70]\n"
+ "ldr q19, [x21], #0x10\n"
+ "zip2 v23.4s, v23.4s, v20.4s\n"
+ "prfm pldl1keep, [x22, #0x70]\n"
+ "zip2 v22.4s, v27.4s, v22.4s\n"
+ "ldr q21, [x20], #0x10\n"
+ "zip1 v18.4s, v25.4s, v19.4s\n"
+ "prfm pldl1keep, [x21, #0x70]\n"
+ "str q16, [%x[out_ptr], #0x0]\n"
+ "zip1 v20.4s, v26.4s, v22.4s\n"
+ "prfm pldl1keep, [x20, #0x70]\n"
+ "zip1 v16.4s, v24.4s, v21.4s\n"
+ "subs %x[width], %x[width], #0x4\n"
+ "zip1 v17.4s, v18.4s, v16.4s\n"
+ "cmp %x[width], #0x4\n"
+ "zip2 v16.4s, v18.4s, v16.4s\n"
+ "str q17, [%x[out_ptr], #0x10]\n"
+ "zip2 v19.4s, v25.4s, v19.4s\n"
+ "str q23, [%x[out_ptr], #0x20]\n"
+ "zip2 v18.4s, v24.4s, v21.4s\n"
+ "str q16, [%x[out_ptr], #0x30]\n"
+ "zip1 v16.4s, v19.4s, v18.4s\n"
+ "str q20, [%x[out_ptr], #0x40]\n"
+ "zip2 v17.4s, v26.4s, v22.4s\n"
+ "str q16, [%x[out_ptr], #0x50]\n"
+ "zip2 v16.4s, v19.4s, v18.4s\n"
+ "str q17, [%x[out_ptr], #0x60]\n"
+ "str q16, [%x[out_ptr], #0x70]\n"
+ "add %x[out_ptr], %x[out_ptr], #0x80\n"
+ "bge 2b\n"
+ "3:" // Main loop skip
+ "cbz %x[width], 6f\n"
+ "tbz %x[width], #1, 4f\n"
+ "ldr d28, [x27], #0x8\n"
+ "ldr d27, [x26], #0x8\n"
+ "ldr d26, [x25], #0x8\n"
+ "ldr d22, [x24], #0x8\n"
+ "ldr d25, [x23], #0x8\n"
+ "ldr d24, [x22], #0x8\n"
+ "ldr d19, [x21], #0x8\n"
+ "ldr d21, [x20], #0x8\n"
+ "mov x19, #0x2\n"
+ "tbz %x[width], #0, 5f\n"
+ "ld1 { v28.s }[2], [x27]\n"
+ "ld1 { v27.s }[2], [x26]\n"
+ "ld1 { v26.s }[2], [x25]\n"
+ "ld1 { v22.s }[2], [x24]\n"
+ "ld1 { v25.s }[2], [x23]\n"
+ "ld1 { v24.s }[2], [x22]\n"
+ "ld1 { v19.s }[2], [x21]\n"
+ "ld1 { v21.s }[2], [x20]\n"
+ "mov x19, #0x3\n"
+ "b 5f\n"
+ "4:" // odd_loads_1_0
+ "ldr s28, [x27, #0x0]\n"
+ "ldr s27, [x26, #0x0]\n"
+ "ldr s26, [x25, #0x0]\n"
+ "ldr s22, [x24, #0x0]\n"
+ "ldr s25, [x23, #0x0]\n"
+ "ldr s24, [x22, #0x0]\n"
+ "ldr s19, [x21, #0x0]\n"
+ "ldr s21, [x20, #0x0]\n"
+ "mov x19, #0x1\n"
+ "5:" // Odd load end
+ "zip1 v23.4s, v28.4s, v26.4s\n"
+ "subs x19, x19, #0x1\n"
+ "zip1 v20.4s, v27.4s, v22.4s\n"
+ "zip1 v16.4s, v23.4s, v20.4s\n"
+ "str q16, [%x[out_ptr], #0x0]\n"
+ "zip1 v18.4s, v25.4s, v19.4s\n"
+ "zip1 v16.4s, v24.4s, v21.4s\n"
+ "zip1 v17.4s, v18.4s, v16.4s\n"
+ "str q17, [%x[out_ptr], #0x10]\n"
+ "add %x[out_ptr], %x[out_ptr], #0x20\n"
+ "beq 6f\n"
+ "zip2 v23.4s, v23.4s, v20.4s\n"
+ "zip2 v16.4s, v18.4s, v16.4s\n"
+ "str q23, [%x[out_ptr], #0x0]\n"
+ "str q16, [%x[out_ptr], #0x10]\n"
+ "subs x19, x19, #0x1\n"
+ "add %x[out_ptr], %x[out_ptr], #0x20\n"
+ "beq 6f\n"
+ "zip2 v26.4s, v28.4s, v26.4s\n"
+ "zip2 v22.4s, v27.4s, v22.4s\n"
+ "zip1 v20.4s, v26.4s, v22.4s\n"
+ "str q20, [%x[out_ptr], #0x0]\n"
+ "zip2 v19.4s, v25.4s, v19.4s\n"
+ "zip2 v18.4s, v24.4s, v21.4s\n"
+ "zip1 v16.4s, v19.4s, v18.4s\n"
+ "str q16, [%x[out_ptr], #0x10]\n"
+ "add %x[out_ptr], %x[out_ptr], #0x20\n"
+ "6:" // Odds skip
+
+ : [out_ptr] "+r" (out_ptr), [width] "+r" (width)
+ : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset)
+ : "cc", "memory", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27"
+ );
+}
+
+
+#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_s16_s16.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_s16_s16.hpp
new file mode 100644
index 0000000..03f552a
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_s16_s16.hpp
@@ -0,0 +1,282 @@
+/*
+ * Copyright (c) 2019-2020 Arm Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+#ifdef __aarch64__
+
+template<>
+void interleave_block<8, 1, VLType::None, false>(
+ int16_t * &out_ptr, const int16_t * const * in, size_t width, size_t height,
+ size_t row_offset, bool
+)
+{
+ __asm__ __volatile__(
+ "ldr x27, [%x[in], #0x0]\n"
+ "cmp %x[height], #0x8\n"
+ "ldr x26, [%x[in], #0x8]\n"
+ "add x27, x27, %x[row_offset], LSL #1\n"
+ "ldr x25, [%x[in], #0x10]\n"
+ "ldr x24, [%x[in], #0x18]\n"
+ "add x26, x26, %x[row_offset], LSL #1\n"
+ "ldr x23, [%x[in], #0x20]\n"
+ "add x25, x25, %x[row_offset], LSL #1\n"
+ "ldr x22, [%x[in], #0x28]\n"
+ "ldr x21, [%x[in], #0x30]\n"
+ "add x24, x24, %x[row_offset], LSL #1\n"
+ "ldr x20, [%x[in], #0x38]\n"
+ "add x23, x23, %x[row_offset], LSL #1\n"
+ "add x22, x22, %x[row_offset], LSL #1\n"
+ "add x21, x21, %x[row_offset], LSL #1\n"
+ "add x20, x20, %x[row_offset], LSL #1\n"
+ "beq 1f\n"
+ "mov x20, x27\n"
+ "cmp %x[height], #0x2\n"
+ "csel x26, x26, x27, GE\n"
+ "csel x25, x25, x27, GT\n"
+ "cmp %x[height], #0x4\n"
+ "csel x24, x24, x27, GE\n"
+ "csel x23, x23, x27, GT\n"
+ "cmp %x[height], #0x6\n"
+ "csel x22, x22, x27, GE\n"
+ "csel x21, x21, x27, GT\n"
+ "1:" // no_pointer_adj
+ "prfm pldl1keep, [x27, #0x0]\n"
+ "cmp %x[width], #0x8\n"
+ "prfm pldl1keep, [x26, #0x0]\n"
+ "prfm pldl1keep, [x25, #0x0]\n"
+ "prfm pldl1keep, [x24, #0x0]\n"
+ "prfm pldl1keep, [x23, #0x0]\n"
+ "prfm pldl1keep, [x22, #0x0]\n"
+ "prfm pldl1keep, [x21, #0x0]\n"
+ "prfm pldl1keep, [x20, #0x0]\n"
+ "prfm pldl1keep, [x27, #0x40]\n"
+ "prfm pldl1keep, [x26, #0x40]\n"
+ "prfm pldl1keep, [x25, #0x40]\n"
+ "prfm pldl1keep, [x24, #0x40]\n"
+ "prfm pldl1keep, [x23, #0x40]\n"
+ "prfm pldl1keep, [x22, #0x40]\n"
+ "prfm pldl1keep, [x21, #0x40]\n"
+ "prfm pldl1keep, [x20, #0x40]\n"
+ "blt 3f\n"
+ "2:" // Main loop head
+ "ldr q30, [x27], #0x10\n"
+ "prfm pldl1keep, [x27, #0x70]\n"
+ "ldr q29, [x26], #0x10\n"
+ "ldr q28, [x25], #0x10\n"
+ "prfm pldl1keep, [x26, #0x70]\n"
+ "ldr q27, [x24], #0x10\n"
+ "prfm pldl1keep, [x25, #0x70]\n"
+ "ldr q24, [x23], #0x10\n"
+ "zip1 v26.8h, v30.8h, v24.8h\n"
+ "prfm pldl1keep, [x24, #0x70]\n"
+ "ldr q25, [x22], #0x10\n"
+ "zip2 v24.8h, v30.8h, v24.8h\n"
+ "prfm pldl1keep, [x23, #0x70]\n"
+ "ldr q23, [x21], #0x10\n"
+ "zip1 v21.8h, v29.8h, v25.8h\n"
+ "prfm pldl1keep, [x22, #0x70]\n"
+ "ldr q22, [x20], #0x10\n"
+ "zip1 v18.8h, v28.8h, v23.8h\n"
+ "prfm pldl1keep, [x21, #0x70]\n"
+ "subs %x[width], %x[width], #0x8\n"
+ "zip1 v20.8h, v26.8h, v18.8h\n"
+ "prfm pldl1keep, [x20, #0x70]\n"
+ "zip1 v19.8h, v27.8h, v22.8h\n"
+ "cmp %x[width], #0x8\n"
+ "zip1 v17.8h, v21.8h, v19.8h\n"
+ "zip2 v18.8h, v26.8h, v18.8h\n"
+ "zip1 v16.8h, v20.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x0]\n"
+ "zip2 v16.8h, v20.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x10]\n"
+ "zip2 v17.8h, v21.8h, v19.8h\n"
+ "zip1 v16.8h, v18.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x20]\n"
+ "zip2 v16.8h, v18.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x30]\n"
+ "zip2 v21.8h, v28.8h, v23.8h\n"
+ "zip1 v18.8h, v24.8h, v21.8h\n"
+ "zip2 v20.8h, v29.8h, v25.8h\n"
+ "zip2 v19.8h, v27.8h, v22.8h\n"
+ "zip1 v17.8h, v20.8h, v19.8h\n"
+ "zip1 v16.8h, v18.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x40]\n"
+ "zip2 v16.8h, v18.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x50]\n"
+ "zip2 v18.8h, v24.8h, v21.8h\n"
+ "zip2 v17.8h, v20.8h, v19.8h\n"
+ "zip1 v16.8h, v18.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x60]\n"
+ "zip2 v16.8h, v18.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x70]\n"
+ "add %x[out_ptr], %x[out_ptr], #0x80\n"
+ "bge 2b\n"
+ "3:" // Main loop skip
+ "cbz %x[width], 8f\n"
+ "tbz %x[width], #2, 5f\n"
+ "ldr d30, [x27], #0x8\n"
+ "ldr d29, [x26], #0x8\n"
+ "ldr d28, [x25], #0x8\n"
+ "ldr d27, [x24], #0x8\n"
+ "ldr d24, [x23], #0x8\n"
+ "ldr d25, [x22], #0x8\n"
+ "ldr d23, [x21], #0x8\n"
+ "ldr d22, [x20], #0x8\n"
+ "tbz %x[width], #1, 4f\n"
+ "ld1 { v30.s }[2], [x27], #0x4\n"
+ "ld1 { v29.s }[2], [x26], #0x4\n"
+ "ld1 { v28.s }[2], [x25], #0x4\n"
+ "ld1 { v27.s }[2], [x24], #0x4\n"
+ "ld1 { v24.s }[2], [x23], #0x4\n"
+ "ld1 { v25.s }[2], [x22], #0x4\n"
+ "ld1 { v23.s }[2], [x21], #0x4\n"
+ "ld1 { v22.s }[2], [x20], #0x4\n"
+ "mov x19, #0x6\n"
+ "tbz %x[width], #0, 7f\n"
+ "ld1 { v30.h }[6], [x27]\n"
+ "ld1 { v29.h }[6], [x26]\n"
+ "ld1 { v28.h }[6], [x25]\n"
+ "ld1 { v27.h }[6], [x24]\n"
+ "ld1 { v24.h }[6], [x23]\n"
+ "ld1 { v25.h }[6], [x22]\n"
+ "ld1 { v23.h }[6], [x21]\n"
+ "ld1 { v22.h }[6], [x20]\n"
+ "mov x19, #0x7\n"
+ "b 7f\n"
+ "4:" // odd_loads_1_4
+ "mov x19, #0x4\n"
+ "tbz %x[width], #0, 7f\n"
+ "ld1 { v30.h }[4], [x27]\n"
+ "ld1 { v29.h }[4], [x26]\n"
+ "ld1 { v28.h }[4], [x25]\n"
+ "ld1 { v27.h }[4], [x24]\n"
+ "ld1 { v24.h }[4], [x23]\n"
+ "ld1 { v25.h }[4], [x22]\n"
+ "ld1 { v23.h }[4], [x21]\n"
+ "ld1 { v22.h }[4], [x20]\n"
+ "mov x19, #0x5\n"
+ "b 7f\n"
+ "5:" // odd_loads_2_0
+ "tbz %x[width], #1, 6f\n"
+ "ldr s30, [x27], #0x4\n"
+ "ldr s29, [x26], #0x4\n"
+ "ldr s28, [x25], #0x4\n"
+ "ldr s27, [x24], #0x4\n"
+ "ldr s24, [x23], #0x4\n"
+ "ldr s25, [x22], #0x4\n"
+ "ldr s23, [x21], #0x4\n"
+ "ldr s22, [x20], #0x4\n"
+ "mov x19, #0x2\n"
+ "tbz %x[width], #0, 7f\n"
+ "ld1 { v30.h }[2], [x27]\n"
+ "ld1 { v29.h }[2], [x26]\n"
+ "ld1 { v28.h }[2], [x25]\n"
+ "ld1 { v27.h }[2], [x24]\n"
+ "ld1 { v24.h }[2], [x23]\n"
+ "ld1 { v25.h }[2], [x22]\n"
+ "ld1 { v23.h }[2], [x21]\n"
+ "ld1 { v22.h }[2], [x20]\n"
+ "mov x19, #0x3\n"
+ "b 7f\n"
+ "6:" // odd_loads_1_0
+ "ldr h30, [x27, #0x0]\n"
+ "ldr h29, [x26, #0x0]\n"
+ "ldr h28, [x25, #0x0]\n"
+ "ldr h27, [x24, #0x0]\n"
+ "ldr h24, [x23, #0x0]\n"
+ "ldr h25, [x22, #0x0]\n"
+ "ldr h23, [x21, #0x0]\n"
+ "ldr h22, [x20, #0x0]\n"
+ "mov x19, #0x1\n"
+ "7:" // Odd load end
+ "zip1 v26.8h, v30.8h, v24.8h\n"
+ "subs x19, x19, #0x1\n"
+ "zip1 v18.8h, v28.8h, v23.8h\n"
+ "zip1 v20.8h, v26.8h, v18.8h\n"
+ "zip1 v21.8h, v29.8h, v25.8h\n"
+ "zip1 v19.8h, v27.8h, v22.8h\n"
+ "zip1 v17.8h, v21.8h, v19.8h\n"
+ "zip1 v16.8h, v20.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x0]\n"
+ "add %x[out_ptr], %x[out_ptr], #0x10\n"
+ "beq 8f\n"
+ "zip2 v16.8h, v20.8h, v17.8h\n"
+ "subs x19, x19, #0x1\n"
+ "str q16, [%x[out_ptr], #0x0]\n"
+ "add %x[out_ptr], %x[out_ptr], #0x10\n"
+ "beq 8f\n"
+ "zip2 v18.8h, v26.8h, v18.8h\n"
+ "zip2 v17.8h, v21.8h, v19.8h\n"
+ "subs x19, x19, #0x1\n"
+ "zip1 v16.8h, v18.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x0]\n"
+ "add %x[out_ptr], %x[out_ptr], #0x10\n"
+ "beq 8f\n"
+ "zip2 v16.8h, v18.8h, v17.8h\n"
+ "subs x19, x19, #0x1\n"
+ "str q16, [%x[out_ptr], #0x0]\n"
+ "add %x[out_ptr], %x[out_ptr], #0x10\n"
+ "beq 8f\n"
+ "zip2 v24.8h, v30.8h, v24.8h\n"
+ "zip2 v21.8h, v28.8h, v23.8h\n"
+ "subs x19, x19, #0x1\n"
+ "zip1 v18.8h, v24.8h, v21.8h\n"
+ "zip2 v20.8h, v29.8h, v25.8h\n"
+ "zip2 v19.8h, v27.8h, v22.8h\n"
+ "zip1 v17.8h, v20.8h, v19.8h\n"
+ "zip1 v16.8h, v18.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x0]\n"
+ "add %x[out_ptr], %x[out_ptr], #0x10\n"
+ "beq 8f\n"
+ "zip2 v16.8h, v18.8h, v17.8h\n"
+ "subs x19, x19, #0x1\n"
+ "str q16, [%x[out_ptr], #0x0]\n"
+ "add %x[out_ptr], %x[out_ptr], #0x10\n"
+ "beq 8f\n"
+ "zip2 v18.8h, v24.8h, v21.8h\n"
+ "zip2 v17.8h, v20.8h, v19.8h\n"
+ "zip1 v16.8h, v18.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x0]\n"
+ "add %x[out_ptr], %x[out_ptr], #0x10\n"
+ "8:" // Odds skip
+
+ : [out_ptr] "+r" (out_ptr), [width] "+r" (width)
+ : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset)
+ : "cc", "memory", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27"
+ );
+}
+
+template<>
+void interleave_block<8, 1, VLType::None, false>(
+ uint16_t * &out_ptr, const uint16_t * const * in, size_t width, size_t height,
+ size_t row_offset, bool
+)
+{
+ int16_t * &out_cast = reinterpret_cast<int16_t * &>(out_ptr);
+ const int16_t * const * in_cast = reinterpret_cast<const int16_t * const *>(in);
+
+ interleave_block<8, 1, VLType::None, false>(out_cast, in_cast, width, height, row_offset, false);
+}
+
+
+#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_s16_s16_summing.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_s16_s16_summing.hpp
new file mode 100644
index 0000000..35c7719
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_s16_s16_summing.hpp
@@ -0,0 +1,306 @@
+/*
+ * Copyright (c) 2019-2020 Arm Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+#ifdef __aarch64__
+
+template<>
+void interleave_block<8, 1, VLType::None, true>(
+ int16_t * &out_ptr, const int16_t * const * in, size_t width, size_t height,
+ size_t row_offset, bool first
+)
+{
+ __asm__ __volatile__(
+ "movi v1.8h, #0x0\n"
+ "ldr x27, [%x[in], #0x0]\n"
+ "mov x19, #0x0\n"
+ "movi v0.4s, #0x0\n"
+ "ldr x26, [%x[in], #0x8]\n"
+ "cmp %x[height], #0x8\n"
+ "movi v31.4s, #0x0\n"
+ "ldr x25, [%x[in], #0x10]\n"
+ "add x27, x27, %x[row_offset], LSL #1\n"
+ "ldr x24, [%x[in], #0x18]\n"
+ "ldr x23, [%x[in], #0x20]\n"
+ "add x26, x26, %x[row_offset], LSL #1\n"
+ "ldr x22, [%x[in], #0x28]\n"
+ "add x25, x25, %x[row_offset], LSL #1\n"
+ "ldr x21, [%x[in], #0x30]\n"
+ "add x24, x24, %x[row_offset], LSL #1\n"
+ "ldr x20, [%x[in], #0x38]\n"
+ "add x23, x23, %x[row_offset], LSL #1\n"
+ "add x22, x22, %x[row_offset], LSL #1\n"
+ "add x21, x21, %x[row_offset], LSL #1\n"
+ "add x20, x20, %x[row_offset], LSL #1\n"
+ "beq 1f\n"
+ "mov x20, x27\n"
+ "cmp %x[height], #0x2\n"
+ "csel x26, x26, x27, GE\n"
+ "csel x25, x25, x27, GT\n"
+ "cmp %x[height], #0x4\n"
+ "csel x24, x24, x27, GE\n"
+ "csel x23, x23, x27, GT\n"
+ "cmp %x[height], #0x6\n"
+ "csel x22, x22, x27, GE\n"
+ "csel x21, x21, x27, GT\n"
+ "1:" // no_pointer_adj
+ "prfm pldl1keep, [x27, #0x0]\n"
+ "prfm pldl1keep, [x26, #0x0]\n"
+ "prfm pldl1keep, [x25, #0x0]\n"
+ "prfm pldl1keep, [x24, #0x0]\n"
+ "prfm pldl1keep, [x23, #0x0]\n"
+ "prfm pldl1keep, [x22, #0x0]\n"
+ "prfm pldl1keep, [x21, #0x0]\n"
+ "prfm pldl1keep, [x20, #0x0]\n"
+ "prfm pldl1keep, [x27, #0x40]\n"
+ "prfm pldl1keep, [x26, #0x40]\n"
+ "prfm pldl1keep, [x25, #0x40]\n"
+ "prfm pldl1keep, [x24, #0x40]\n"
+ "prfm pldl1keep, [x23, #0x40]\n"
+ "prfm pldl1keep, [x22, #0x40]\n"
+ "prfm pldl1keep, [x21, #0x40]\n"
+ "prfm pldl1keep, [x20, #0x40]\n"
+ "cbnz %w[first], 2f\n"
+ "sub %x[out_ptr], %x[out_ptr], #0x20\n"
+ "ld1 { v0.4s }, [%x[out_ptr]]\n"
+ "ldr q31, [%x[out_ptr], #0x10]\n"
+ "2:" // first_pass
+ "cmp %x[width], #0x8\n"
+ "blt 5f\n"
+ "3:" // Main loop head
+ "cmp x19, #0xe\n"
+ "ble 4f\n"
+ "saddw v0.4s, v0.4s, v1.4h\n"
+ "saddw2 v31.4s, v31.4s, v1.8h\n"
+ "mov x19, #0x0\n"
+ "movi v1.8h, #0x0\n"
+ "4:" // no_accumulate_16
+ "ldr q30, [x27], #0x10\n"
+ "prfm pldl1keep, [x27, #0x70]\n"
+ "ldr q29, [x26], #0x10\n"
+ "ldr q28, [x25], #0x10\n"
+ "prfm pldl1keep, [x26, #0x70]\n"
+ "ldr q27, [x24], #0x10\n"
+ "prfm pldl1keep, [x25, #0x70]\n"
+ "ldr q24, [x23], #0x10\n"
+ "zip1 v26.8h, v30.8h, v24.8h\n"
+ "prfm pldl1keep, [x24, #0x70]\n"
+ "ldr q25, [x22], #0x10\n"
+ "zip2 v24.8h, v30.8h, v24.8h\n"
+ "prfm pldl1keep, [x23, #0x70]\n"
+ "ldr q23, [x21], #0x10\n"
+ "zip1 v21.8h, v29.8h, v25.8h\n"
+ "prfm pldl1keep, [x22, #0x70]\n"
+ "ldr q22, [x20], #0x10\n"
+ "zip1 v18.8h, v28.8h, v23.8h\n"
+ "prfm pldl1keep, [x21, #0x70]\n"
+ "add x19, x19, #0x1\n"
+ "zip1 v20.8h, v26.8h, v18.8h\n"
+ "prfm pldl1keep, [x20, #0x70]\n"
+ "zip1 v19.8h, v27.8h, v22.8h\n"
+ "subs %x[width], %x[width], #0x8\n"
+ "zip1 v17.8h, v21.8h, v19.8h\n"
+ "cmp %x[width], #0x8\n"
+ "zip2 v18.8h, v26.8h, v18.8h\n"
+ "zip1 v16.8h, v20.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x0]\n"
+ "add v1.8h, v1.8h, v16.8h\n"
+ "zip2 v17.8h, v20.8h, v17.8h\n"
+ "str q17, [%x[out_ptr], #0x10]\n"
+ "zip2 v16.8h, v21.8h, v19.8h\n"
+ "add v1.8h, v1.8h, v17.8h\n"
+ "zip1 v17.8h, v18.8h, v16.8h\n"
+ "str q17, [%x[out_ptr], #0x20]\n"
+ "zip2 v16.8h, v18.8h, v16.8h\n"
+ "str q16, [%x[out_ptr], #0x30]\n"
+ "add v1.8h, v1.8h, v17.8h\n"
+ "zip2 v21.8h, v28.8h, v23.8h\n"
+ "zip1 v18.8h, v24.8h, v21.8h\n"
+ "add v1.8h, v1.8h, v16.8h\n"
+ "zip2 v20.8h, v29.8h, v25.8h\n"
+ "zip2 v19.8h, v27.8h, v22.8h\n"
+ "zip1 v17.8h, v20.8h, v19.8h\n"
+ "zip1 v16.8h, v18.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x40]\n"
+ "add v1.8h, v1.8h, v16.8h\n"
+ "zip2 v16.8h, v18.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x50]\n"
+ "zip2 v18.8h, v24.8h, v21.8h\n"
+ "add v1.8h, v1.8h, v16.8h\n"
+ "zip2 v17.8h, v20.8h, v19.8h\n"
+ "zip1 v16.8h, v18.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x60]\n"
+ "add v1.8h, v1.8h, v16.8h\n"
+ "zip2 v16.8h, v18.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x70]\n"
+ "add %x[out_ptr], %x[out_ptr], #0x80\n"
+ "add v1.8h, v1.8h, v16.8h\n"
+ "bge 3b\n"
+ "5:" // Main loop skip
+ "cbz %x[width], 10f\n"
+ "tbz %x[width], #2, 7f\n"
+ "ldr d30, [x27], #0x8\n"
+ "ldr d29, [x26], #0x8\n"
+ "ldr d28, [x25], #0x8\n"
+ "ldr d27, [x24], #0x8\n"
+ "ldr d24, [x23], #0x8\n"
+ "ldr d25, [x22], #0x8\n"
+ "ldr d23, [x21], #0x8\n"
+ "ldr d22, [x20], #0x8\n"
+ "tbz %x[width], #1, 6f\n"
+ "ld1 { v30.s }[2], [x27], #0x4\n"
+ "ld1 { v29.s }[2], [x26], #0x4\n"
+ "ld1 { v28.s }[2], [x25], #0x4\n"
+ "ld1 { v27.s }[2], [x24], #0x4\n"
+ "ld1 { v24.s }[2], [x23], #0x4\n"
+ "ld1 { v25.s }[2], [x22], #0x4\n"
+ "ld1 { v23.s }[2], [x21], #0x4\n"
+ "ld1 { v22.s }[2], [x20], #0x4\n"
+ "mov x19, #0x6\n"
+ "tbz %x[width], #0, 9f\n"
+ "ld1 { v30.h }[6], [x27]\n"
+ "ld1 { v29.h }[6], [x26]\n"
+ "ld1 { v28.h }[6], [x25]\n"
+ "ld1 { v27.h }[6], [x24]\n"
+ "ld1 { v24.h }[6], [x23]\n"
+ "ld1 { v25.h }[6], [x22]\n"
+ "ld1 { v23.h }[6], [x21]\n"
+ "ld1 { v22.h }[6], [x20]\n"
+ "mov x19, #0x7\n"
+ "b 9f\n"
+ "6:" // odd_loads_1_4
+ "mov x19, #0x4\n"
+ "tbz %x[width], #0, 9f\n"
+ "ld1 { v30.h }[4], [x27]\n"
+ "ld1 { v29.h }[4], [x26]\n"
+ "ld1 { v28.h }[4], [x25]\n"
+ "ld1 { v27.h }[4], [x24]\n"
+ "ld1 { v24.h }[4], [x23]\n"
+ "ld1 { v25.h }[4], [x22]\n"
+ "ld1 { v23.h }[4], [x21]\n"
+ "ld1 { v22.h }[4], [x20]\n"
+ "mov x19, #0x5\n"
+ "b 9f\n"
+ "7:" // odd_loads_2_0
+ "tbz %x[width], #1, 8f\n"
+ "ldr s30, [x27], #0x4\n"
+ "ldr s29, [x26], #0x4\n"
+ "ldr s28, [x25], #0x4\n"
+ "ldr s27, [x24], #0x4\n"
+ "ldr s24, [x23], #0x4\n"
+ "ldr s25, [x22], #0x4\n"
+ "ldr s23, [x21], #0x4\n"
+ "ldr s22, [x20], #0x4\n"
+ "mov x19, #0x2\n"
+ "tbz %x[width], #0, 9f\n"
+ "ld1 { v30.h }[2], [x27]\n"
+ "ld1 { v29.h }[2], [x26]\n"
+ "ld1 { v28.h }[2], [x25]\n"
+ "ld1 { v27.h }[2], [x24]\n"
+ "ld1 { v24.h }[2], [x23]\n"
+ "ld1 { v25.h }[2], [x22]\n"
+ "ld1 { v23.h }[2], [x21]\n"
+ "ld1 { v22.h }[2], [x20]\n"
+ "mov x19, #0x3\n"
+ "b 9f\n"
+ "8:" // odd_loads_1_0
+ "ldr h30, [x27, #0x0]\n"
+ "ldr h29, [x26, #0x0]\n"
+ "ldr h28, [x25, #0x0]\n"
+ "ldr h27, [x24, #0x0]\n"
+ "ldr h24, [x23, #0x0]\n"
+ "ldr h25, [x22, #0x0]\n"
+ "ldr h23, [x21, #0x0]\n"
+ "ldr h22, [x20, #0x0]\n"
+ "mov x19, #0x1\n"
+ "9:" // Odd load end
+ "zip1 v26.8h, v30.8h, v24.8h\n"
+ "subs x19, x19, #0x1\n"
+ "zip1 v18.8h, v28.8h, v23.8h\n"
+ "zip1 v20.8h, v26.8h, v18.8h\n"
+ "zip1 v21.8h, v29.8h, v25.8h\n"
+ "zip1 v19.8h, v27.8h, v22.8h\n"
+ "zip1 v17.8h, v21.8h, v19.8h\n"
+ "zip1 v16.8h, v20.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x0]\n"
+ "add v1.8h, v1.8h, v16.8h\n"
+ "add %x[out_ptr], %x[out_ptr], #0x10\n"
+ "beq 10f\n"
+ "zip2 v17.8h, v20.8h, v17.8h\n"
+ "subs x19, x19, #0x1\n"
+ "add v1.8h, v1.8h, v17.8h\n"
+ "str q17, [%x[out_ptr], #0x0]\n"
+ "add %x[out_ptr], %x[out_ptr], #0x10\n"
+ "beq 10f\n"
+ "zip2 v18.8h, v26.8h, v18.8h\n"
+ "zip2 v16.8h, v21.8h, v19.8h\n"
+ "subs x19, x19, #0x1\n"
+ "zip1 v17.8h, v18.8h, v16.8h\n"
+ "str q17, [%x[out_ptr], #0x0]\n"
+ "add v1.8h, v1.8h, v17.8h\n"
+ "add %x[out_ptr], %x[out_ptr], #0x10\n"
+ "beq 10f\n"
+ "zip2 v16.8h, v18.8h, v16.8h\n"
+ "subs x19, x19, #0x1\n"
+ "add v1.8h, v1.8h, v16.8h\n"
+ "str q16, [%x[out_ptr], #0x0]\n"
+ "add %x[out_ptr], %x[out_ptr], #0x10\n"
+ "beq 10f\n"
+ "zip2 v24.8h, v30.8h, v24.8h\n"
+ "zip2 v21.8h, v28.8h, v23.8h\n"
+ "subs x19, x19, #0x1\n"
+ "zip1 v18.8h, v24.8h, v21.8h\n"
+ "zip2 v20.8h, v29.8h, v25.8h\n"
+ "zip2 v19.8h, v27.8h, v22.8h\n"
+ "zip1 v17.8h, v20.8h, v19.8h\n"
+ "zip1 v16.8h, v18.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x0]\n"
+ "add v1.8h, v1.8h, v16.8h\n"
+ "add %x[out_ptr], %x[out_ptr], #0x10\n"
+ "beq 10f\n"
+ "zip2 v16.8h, v18.8h, v17.8h\n"
+ "subs x19, x19, #0x1\n"
+ "add v1.8h, v1.8h, v16.8h\n"
+ "str q16, [%x[out_ptr], #0x0]\n"
+ "add %x[out_ptr], %x[out_ptr], #0x10\n"
+ "beq 10f\n"
+ "zip2 v18.8h, v24.8h, v21.8h\n"
+ "zip2 v17.8h, v20.8h, v19.8h\n"
+ "zip1 v16.8h, v18.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x0]\n"
+ "add v1.8h, v1.8h, v16.8h\n"
+ "add %x[out_ptr], %x[out_ptr], #0x10\n"
+ "10:" // Odds skip
+ "saddw v0.4s, v0.4s, v1.4h\n"
+ "str q0, [%x[out_ptr], #0x0]\n"
+ "saddw2 v31.4s, v31.4s, v1.8h\n"
+ "str q31, [%x[out_ptr], #0x10]\n"
+ "add %x[out_ptr], %x[out_ptr], #0x20\n"
+ : [out_ptr] "+r" (out_ptr), [width] "+r" (width)
+ : [first] "r" (first), [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset)
+ : "cc", "memory", "v0", "v1", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27"
+ );
+}
+
+
+#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_s8_s16.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_s8_s16.hpp
new file mode 100644
index 0000000..582836f
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_s8_s16.hpp
@@ -0,0 +1,286 @@
+/*
+ * Copyright (c) 2019-2020 Arm Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+#ifdef __aarch64__
+
+template<>
+void interleave_block<8, 1, VLType::None, false>(
+ int16_t * &out_ptr, const int8_t * const * in, size_t width, size_t height,
+ size_t row_offset, bool
+)
+{
+ __asm__ __volatile__(
+ "ldr x27, [%x[in], #0x0]\n"
+ "cmp %x[height], #0x8\n"
+ "ldr x26, [%x[in], #0x8]\n"
+ "add x27, x27, %x[row_offset]\n"
+ "ldr x25, [%x[in], #0x10]\n"
+ "ldr x24, [%x[in], #0x18]\n"
+ "add x26, x26, %x[row_offset]\n"
+ "ldr x23, [%x[in], #0x20]\n"
+ "add x25, x25, %x[row_offset]\n"
+ "ldr x22, [%x[in], #0x28]\n"
+ "ldr x21, [%x[in], #0x30]\n"
+ "add x24, x24, %x[row_offset]\n"
+ "ldr x20, [%x[in], #0x38]\n"
+ "add x23, x23, %x[row_offset]\n"
+ "add x22, x22, %x[row_offset]\n"
+ "add x21, x21, %x[row_offset]\n"
+ "add x20, x20, %x[row_offset]\n"
+ "beq 1f\n"
+ "mov x20, x27\n"
+ "cmp %x[height], #0x2\n"
+ "csel x26, x26, x27, GE\n"
+ "csel x25, x25, x27, GT\n"
+ "cmp %x[height], #0x4\n"
+ "csel x24, x24, x27, GE\n"
+ "csel x23, x23, x27, GT\n"
+ "cmp %x[height], #0x6\n"
+ "csel x22, x22, x27, GE\n"
+ "csel x21, x21, x27, GT\n"
+ "1:" // no_pointer_adj
+ "prfm pldl1keep, [x27, #0x0]\n"
+ "cmp %x[width], #0x8\n"
+ "prfm pldl1keep, [x26, #0x0]\n"
+ "prfm pldl1keep, [x25, #0x0]\n"
+ "prfm pldl1keep, [x24, #0x0]\n"
+ "prfm pldl1keep, [x23, #0x0]\n"
+ "prfm pldl1keep, [x22, #0x0]\n"
+ "prfm pldl1keep, [x21, #0x0]\n"
+ "prfm pldl1keep, [x20, #0x0]\n"
+ "prfm pldl1keep, [x27, #0x40]\n"
+ "prfm pldl1keep, [x26, #0x40]\n"
+ "prfm pldl1keep, [x25, #0x40]\n"
+ "prfm pldl1keep, [x24, #0x40]\n"
+ "prfm pldl1keep, [x23, #0x40]\n"
+ "prfm pldl1keep, [x22, #0x40]\n"
+ "prfm pldl1keep, [x21, #0x40]\n"
+ "prfm pldl1keep, [x20, #0x40]\n"
+ "blt 3f\n"
+ "2:" // Main loop head
+ "ldr d30, [x27], #0x8\n"
+ "prfm pldl1keep, [x27, #0x70]\n"
+ "ldr d29, [x26], #0x8\n"
+ "ldr d28, [x25], #0x8\n"
+ "prfm pldl1keep, [x26, #0x70]\n"
+ "ldr d27, [x24], #0x8\n"
+ "prfm pldl1keep, [x25, #0x70]\n"
+ "ldr d23, [x23], #0x8\n"
+ "ldr d21, [x22], #0x8\n"
+ "prfm pldl1keep, [x24, #0x70]\n"
+ "ldr d26, [x21], #0x8\n"
+ "ldr d25, [x20], #0x8\n"
+ "prfm pldl1keep, [x23, #0x70]\n"
+ "prfm pldl1keep, [x22, #0x70]\n"
+ "sshll v30.8h, v30.8b, #0x0\n"
+ "sshll v29.8h, v29.8b, #0x0\n"
+ "prfm pldl1keep, [x21, #0x70]\n"
+ "sshll v28.8h, v28.8b, #0x0\n"
+ "prfm pldl1keep, [x20, #0x70]\n"
+ "sshll v27.8h, v27.8b, #0x0\n"
+ "sshll v23.8h, v23.8b, #0x0\n"
+ "zip1 v24.8h, v30.8h, v23.8h\n"
+ "sshll v21.8h, v21.8b, #0x0\n"
+ "zip2 v23.8h, v30.8h, v23.8h\n"
+ "sshll v26.8h, v26.8b, #0x0\n"
+ "sshll v25.8h, v25.8b, #0x0\n"
+ "zip1 v22.8h, v29.8h, v21.8h\n"
+ "subs %x[width], %x[width], #0x8\n"
+ "zip2 v21.8h, v29.8h, v21.8h\n"
+ "cmp %x[width], #0x8\n"
+ "zip1 v20.8h, v28.8h, v26.8h\n"
+ "zip1 v18.8h, v24.8h, v20.8h\n"
+ "zip1 v19.8h, v27.8h, v25.8h\n"
+ "zip1 v17.8h, v22.8h, v19.8h\n"
+ "zip1 v16.8h, v18.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x0]\n"
+ "zip2 v16.8h, v18.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x10]\n"
+ "zip2 v18.8h, v24.8h, v20.8h\n"
+ "zip2 v17.8h, v22.8h, v19.8h\n"
+ "zip1 v16.8h, v18.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x20]\n"
+ "zip2 v16.8h, v18.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x30]\n"
+ "zip2 v20.8h, v28.8h, v26.8h\n"
+ "zip1 v18.8h, v23.8h, v20.8h\n"
+ "zip2 v19.8h, v27.8h, v25.8h\n"
+ "zip1 v17.8h, v21.8h, v19.8h\n"
+ "zip1 v16.8h, v18.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x40]\n"
+ "zip2 v16.8h, v18.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x50]\n"
+ "zip2 v18.8h, v23.8h, v20.8h\n"
+ "zip2 v17.8h, v21.8h, v19.8h\n"
+ "zip1 v16.8h, v18.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x60]\n"
+ "zip2 v16.8h, v18.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x70]\n"
+ "add %x[out_ptr], %x[out_ptr], #0x80\n"
+ "bge 2b\n"
+ "3:" // Main loop skip
+ "cbz %x[width], 8f\n"
+ "tbz %x[width], #2, 5f\n"
+ "ldr s30, [x27], #0x4\n"
+ "ldr s29, [x26], #0x4\n"
+ "ldr s28, [x25], #0x4\n"
+ "ldr s27, [x24], #0x4\n"
+ "ldr s23, [x23], #0x4\n"
+ "ldr s21, [x22], #0x4\n"
+ "ldr s26, [x21], #0x4\n"
+ "ldr s25, [x20], #0x4\n"
+ "tbz %x[width], #1, 4f\n"
+ "ld1 { v30.h }[2], [x27], #0x2\n"
+ "ld1 { v29.h }[2], [x26], #0x2\n"
+ "ld1 { v28.h }[2], [x25], #0x2\n"
+ "ld1 { v27.h }[2], [x24], #0x2\n"
+ "ld1 { v23.h }[2], [x23], #0x2\n"
+ "ld1 { v21.h }[2], [x22], #0x2\n"
+ "ld1 { v26.h }[2], [x21], #0x2\n"
+ "ld1 { v25.h }[2], [x20], #0x2\n"
+ "mov x19, #0x6\n"
+ "tbz %x[width], #0, 7f\n"
+ "ld1 { v30.b }[6], [x27]\n"
+ "ld1 { v29.b }[6], [x26]\n"
+ "ld1 { v28.b }[6], [x25]\n"
+ "ld1 { v27.b }[6], [x24]\n"
+ "ld1 { v23.b }[6], [x23]\n"
+ "ld1 { v21.b }[6], [x22]\n"
+ "ld1 { v26.b }[6], [x21]\n"
+ "ld1 { v25.b }[6], [x20]\n"
+ "mov x19, #0x7\n"
+ "b 7f\n"
+ "4:" // odd_loads_1_4
+ "mov x19, #0x4\n"
+ "tbz %x[width], #0, 7f\n"
+ "ld1 { v30.b }[4], [x27]\n"
+ "ld1 { v29.b }[4], [x26]\n"
+ "ld1 { v28.b }[4], [x25]\n"
+ "ld1 { v27.b }[4], [x24]\n"
+ "ld1 { v23.b }[4], [x23]\n"
+ "ld1 { v21.b }[4], [x22]\n"
+ "ld1 { v26.b }[4], [x21]\n"
+ "ld1 { v25.b }[4], [x20]\n"
+ "mov x19, #0x5\n"
+ "b 7f\n"
+ "5:" // odd_loads_2_0
+ "tbz %x[width], #1, 6f\n"
+ "ldr h30, [x27], #0x2\n"
+ "ldr h29, [x26], #0x2\n"
+ "ldr h28, [x25], #0x2\n"
+ "ldr h27, [x24], #0x2\n"
+ "ldr h23, [x23], #0x2\n"
+ "ldr h21, [x22], #0x2\n"
+ "ldr h26, [x21], #0x2\n"
+ "ldr h25, [x20], #0x2\n"
+ "mov x19, #0x2\n"
+ "tbz %x[width], #0, 7f\n"
+ "ld1 { v30.b }[2], [x27]\n"
+ "ld1 { v29.b }[2], [x26]\n"
+ "ld1 { v28.b }[2], [x25]\n"
+ "ld1 { v27.b }[2], [x24]\n"
+ "ld1 { v23.b }[2], [x23]\n"
+ "ld1 { v21.b }[2], [x22]\n"
+ "ld1 { v26.b }[2], [x21]\n"
+ "ld1 { v25.b }[2], [x20]\n"
+ "mov x19, #0x3\n"
+ "b 7f\n"
+ "6:" // odd_loads_1_0
+ "ldr b30, [x27, #0x0]\n"
+ "ldr b29, [x26, #0x0]\n"
+ "ldr b28, [x25, #0x0]\n"
+ "ldr b27, [x24, #0x0]\n"
+ "ldr b23, [x23, #0x0]\n"
+ "ldr b21, [x22, #0x0]\n"
+ "ldr b26, [x21, #0x0]\n"
+ "ldr b25, [x20, #0x0]\n"
+ "mov x19, #0x1\n"
+ "7:" // Odd load end
+ "sshll v30.8h, v30.8b, #0x0\n"
+ "sshll v29.8h, v29.8b, #0x0\n"
+ "sshll v28.8h, v28.8b, #0x0\n"
+ "sshll v27.8h, v27.8b, #0x0\n"
+ "sshll v23.8h, v23.8b, #0x0\n"
+ "zip1 v24.8h, v30.8h, v23.8h\n"
+ "sshll v21.8h, v21.8b, #0x0\n"
+ "sshll v26.8h, v26.8b, #0x0\n"
+ "zip1 v20.8h, v28.8h, v26.8h\n"
+ "sshll v25.8h, v25.8b, #0x0\n"
+ "zip1 v22.8h, v29.8h, v21.8h\n"
+ "subs x19, x19, #0x1\n"
+ "zip1 v18.8h, v24.8h, v20.8h\n"
+ "zip1 v19.8h, v27.8h, v25.8h\n"
+ "zip1 v17.8h, v22.8h, v19.8h\n"
+ "zip1 v16.8h, v18.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x0]\n"
+ "add %x[out_ptr], %x[out_ptr], #0x10\n"
+ "beq 8f\n"
+ "zip2 v16.8h, v18.8h, v17.8h\n"
+ "subs x19, x19, #0x1\n"
+ "str q16, [%x[out_ptr], #0x0]\n"
+ "add %x[out_ptr], %x[out_ptr], #0x10\n"
+ "beq 8f\n"
+ "zip2 v18.8h, v24.8h, v20.8h\n"
+ "zip2 v17.8h, v22.8h, v19.8h\n"
+ "subs x19, x19, #0x1\n"
+ "zip1 v16.8h, v18.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x0]\n"
+ "add %x[out_ptr], %x[out_ptr], #0x10\n"
+ "beq 8f\n"
+ "zip2 v16.8h, v18.8h, v17.8h\n"
+ "subs x19, x19, #0x1\n"
+ "str q16, [%x[out_ptr], #0x0]\n"
+ "add %x[out_ptr], %x[out_ptr], #0x10\n"
+ "beq 8f\n"
+ "zip2 v23.8h, v30.8h, v23.8h\n"
+ "zip2 v20.8h, v28.8h, v26.8h\n"
+ "subs x19, x19, #0x1\n"
+ "zip1 v18.8h, v23.8h, v20.8h\n"
+ "zip2 v21.8h, v29.8h, v21.8h\n"
+ "zip2 v19.8h, v27.8h, v25.8h\n"
+ "zip1 v17.8h, v21.8h, v19.8h\n"
+ "zip1 v16.8h, v18.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x0]\n"
+ "add %x[out_ptr], %x[out_ptr], #0x10\n"
+ "beq 8f\n"
+ "zip2 v16.8h, v18.8h, v17.8h\n"
+ "subs x19, x19, #0x1\n"
+ "str q16, [%x[out_ptr], #0x0]\n"
+ "add %x[out_ptr], %x[out_ptr], #0x10\n"
+ "beq 8f\n"
+ "zip2 v18.8h, v23.8h, v20.8h\n"
+ "zip2 v17.8h, v21.8h, v19.8h\n"
+ "zip1 v16.8h, v18.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x0]\n"
+ "add %x[out_ptr], %x[out_ptr], #0x10\n"
+ "8:" // Odds skip
+
+ : [out_ptr] "+r" (out_ptr), [width] "+r" (width)
+ : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset)
+ : "cc", "memory", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27"
+ );
+}
+
+
+#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_s8_s16_summing.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_s8_s16_summing.hpp
new file mode 100644
index 0000000..35dc3dc
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_s8_s16_summing.hpp
@@ -0,0 +1,322 @@
+/*
+ * Copyright (c) 2019-2020 Arm Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+#ifdef __aarch64__
+
+template<>
+void interleave_block<8, 1, VLType::None, true>(
+ int16_t * &out_ptr, const int8_t * const * in, size_t width, size_t height,
+ size_t row_offset, bool first
+)
+{
+ __asm__ __volatile__(
+ "movi v1.8h, #0x0\n"
+ "ldr x27, [%x[in], #0x0]\n"
+ "mov x19, #0x0\n"
+ "movi v0.4s, #0x0\n"
+ "ldr x26, [%x[in], #0x8]\n"
+ "cmp %x[height], #0x8\n"
+ "movi v31.4s, #0x0\n"
+ "ldr x25, [%x[in], #0x10]\n"
+ "add x27, x27, %x[row_offset]\n"
+ "ldr x24, [%x[in], #0x18]\n"
+ "ldr x23, [%x[in], #0x20]\n"
+ "add x26, x26, %x[row_offset]\n"
+ "ldr x22, [%x[in], #0x28]\n"
+ "add x25, x25, %x[row_offset]\n"
+ "ldr x21, [%x[in], #0x30]\n"
+ "add x24, x24, %x[row_offset]\n"
+ "ldr x20, [%x[in], #0x38]\n"
+ "add x23, x23, %x[row_offset]\n"
+ "add x22, x22, %x[row_offset]\n"
+ "add x21, x21, %x[row_offset]\n"
+ "add x20, x20, %x[row_offset]\n"
+ "beq 1f\n"
+ "mov x20, x27\n"
+ "cmp %x[height], #0x2\n"
+ "csel x26, x26, x27, GE\n"
+ "csel x25, x25, x27, GT\n"
+ "cmp %x[height], #0x4\n"
+ "csel x24, x24, x27, GE\n"
+ "csel x23, x23, x27, GT\n"
+ "cmp %x[height], #0x6\n"
+ "csel x22, x22, x27, GE\n"
+ "csel x21, x21, x27, GT\n"
+ "1:" // no_pointer_adj
+ "prfm pldl1keep, [x27, #0x0]\n"
+ "prfm pldl1keep, [x26, #0x0]\n"
+ "prfm pldl1keep, [x25, #0x0]\n"
+ "prfm pldl1keep, [x24, #0x0]\n"
+ "prfm pldl1keep, [x23, #0x0]\n"
+ "prfm pldl1keep, [x22, #0x0]\n"
+ "prfm pldl1keep, [x21, #0x0]\n"
+ "prfm pldl1keep, [x20, #0x0]\n"
+ "prfm pldl1keep, [x27, #0x40]\n"
+ "prfm pldl1keep, [x26, #0x40]\n"
+ "prfm pldl1keep, [x25, #0x40]\n"
+ "prfm pldl1keep, [x24, #0x40]\n"
+ "prfm pldl1keep, [x23, #0x40]\n"
+ "prfm pldl1keep, [x22, #0x40]\n"
+ "prfm pldl1keep, [x21, #0x40]\n"
+ "prfm pldl1keep, [x20, #0x40]\n"
+ "cbnz %w[first], 2f\n"
+ "sub %x[out_ptr], %x[out_ptr], #0x20\n"
+ "ld1 { v0.4s }, [%x[out_ptr]]\n"
+ "ldr q31, [%x[out_ptr], #0x10]\n"
+ "2:" // first_pass
+ "cmp %x[width], #0x8\n"
+ "blt 5f\n"
+ "3:" // Main loop head
+ "cmp x19, #0xe\n"
+ "ble 4f\n"
+ "saddw v0.4s, v0.4s, v1.4h\n"
+ "saddw2 v31.4s, v31.4s, v1.8h\n"
+ "mov x19, #0x0\n"
+ "movi v1.8h, #0x0\n"
+ "4:" // no_accumulate_16
+ "ldr d30, [x27], #0x8\n"
+ "prfm pldl1keep, [x27, #0x70]\n"
+ "ldr d29, [x26], #0x8\n"
+ "ldr d28, [x25], #0x8\n"
+ "prfm pldl1keep, [x26, #0x70]\n"
+ "ldr d27, [x24], #0x8\n"
+ "prfm pldl1keep, [x25, #0x70]\n"
+ "ldr d23, [x23], #0x8\n"
+ "ldr d21, [x22], #0x8\n"
+ "prfm pldl1keep, [x24, #0x70]\n"
+ "ldr d26, [x21], #0x8\n"
+ "ldr d25, [x20], #0x8\n"
+ "prfm pldl1keep, [x23, #0x70]\n"
+ "prfm pldl1keep, [x22, #0x70]\n"
+ "sshll v30.8h, v30.8b, #0x0\n"
+ "sshll v29.8h, v29.8b, #0x0\n"
+ "prfm pldl1keep, [x21, #0x70]\n"
+ "sshll v28.8h, v28.8b, #0x0\n"
+ "prfm pldl1keep, [x20, #0x70]\n"
+ "sshll v27.8h, v27.8b, #0x0\n"
+ "sshll v23.8h, v23.8b, #0x0\n"
+ "zip1 v24.8h, v30.8h, v23.8h\n"
+ "sshll v21.8h, v21.8b, #0x0\n"
+ "zip2 v23.8h, v30.8h, v23.8h\n"
+ "sshll v26.8h, v26.8b, #0x0\n"
+ "sshll v25.8h, v25.8b, #0x0\n"
+ "zip1 v22.8h, v29.8h, v21.8h\n"
+ "add x19, x19, #0x1\n"
+ "zip2 v21.8h, v29.8h, v21.8h\n"
+ "subs %x[width], %x[width], #0x8\n"
+ "zip1 v20.8h, v28.8h, v26.8h\n"
+ "cmp %x[width], #0x8\n"
+ "zip1 v18.8h, v24.8h, v20.8h\n"
+ "zip1 v19.8h, v27.8h, v25.8h\n"
+ "zip1 v17.8h, v22.8h, v19.8h\n"
+ "zip1 v16.8h, v18.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x0]\n"
+ "add v1.8h, v1.8h, v16.8h\n"
+ "zip2 v16.8h, v18.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x10]\n"
+ "zip2 v18.8h, v24.8h, v20.8h\n"
+ "add v1.8h, v1.8h, v16.8h\n"
+ "zip2 v17.8h, v22.8h, v19.8h\n"
+ "zip1 v16.8h, v18.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x20]\n"
+ "add v1.8h, v1.8h, v16.8h\n"
+ "zip2 v16.8h, v18.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x30]\n"
+ "zip2 v20.8h, v28.8h, v26.8h\n"
+ "add v1.8h, v1.8h, v16.8h\n"
+ "zip1 v18.8h, v23.8h, v20.8h\n"
+ "zip2 v19.8h, v27.8h, v25.8h\n"
+ "zip1 v17.8h, v21.8h, v19.8h\n"
+ "zip1 v16.8h, v18.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x40]\n"
+ "add v1.8h, v1.8h, v16.8h\n"
+ "zip2 v16.8h, v18.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x50]\n"
+ "zip2 v18.8h, v23.8h, v20.8h\n"
+ "add v1.8h, v1.8h, v16.8h\n"
+ "zip2 v17.8h, v21.8h, v19.8h\n"
+ "zip1 v16.8h, v18.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x60]\n"
+ "add v1.8h, v1.8h, v16.8h\n"
+ "zip2 v16.8h, v18.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x70]\n"
+ "add %x[out_ptr], %x[out_ptr], #0x80\n"
+ "add v1.8h, v1.8h, v16.8h\n"
+ "bge 3b\n"
+ "5:" // Main loop skip
+ "cbz %x[width], 10f\n"
+ "tbz %x[width], #2, 7f\n"
+ "ldr s30, [x27], #0x4\n"
+ "ldr s29, [x26], #0x4\n"
+ "ldr s28, [x25], #0x4\n"
+ "ldr s27, [x24], #0x4\n"
+ "ldr s23, [x23], #0x4\n"
+ "ldr s21, [x22], #0x4\n"
+ "ldr s26, [x21], #0x4\n"
+ "ldr s25, [x20], #0x4\n"
+ "tbz %x[width], #1, 6f\n"
+ "ld1 { v30.h }[2], [x27], #0x2\n"
+ "ld1 { v29.h }[2], [x26], #0x2\n"
+ "ld1 { v28.h }[2], [x25], #0x2\n"
+ "ld1 { v27.h }[2], [x24], #0x2\n"
+ "ld1 { v23.h }[2], [x23], #0x2\n"
+ "ld1 { v21.h }[2], [x22], #0x2\n"
+ "ld1 { v26.h }[2], [x21], #0x2\n"
+ "ld1 { v25.h }[2], [x20], #0x2\n"
+ "mov x19, #0x6\n"
+ "tbz %x[width], #0, 9f\n"
+ "ld1 { v30.b }[6], [x27]\n"
+ "ld1 { v29.b }[6], [x26]\n"
+ "ld1 { v28.b }[6], [x25]\n"
+ "ld1 { v27.b }[6], [x24]\n"
+ "ld1 { v23.b }[6], [x23]\n"
+ "ld1 { v21.b }[6], [x22]\n"
+ "ld1 { v26.b }[6], [x21]\n"
+ "ld1 { v25.b }[6], [x20]\n"
+ "mov x19, #0x7\n"
+ "b 9f\n"
+ "6:" // odd_loads_1_4
+ "mov x19, #0x4\n"
+ "tbz %x[width], #0, 9f\n"
+ "ld1 { v30.b }[4], [x27]\n"
+ "ld1 { v29.b }[4], [x26]\n"
+ "ld1 { v28.b }[4], [x25]\n"
+ "ld1 { v27.b }[4], [x24]\n"
+ "ld1 { v23.b }[4], [x23]\n"
+ "ld1 { v21.b }[4], [x22]\n"
+ "ld1 { v26.b }[4], [x21]\n"
+ "ld1 { v25.b }[4], [x20]\n"
+ "mov x19, #0x5\n"
+ "b 9f\n"
+ "7:" // odd_loads_2_0
+ "tbz %x[width], #1, 8f\n"
+ "ldr h30, [x27], #0x2\n"
+ "ldr h29, [x26], #0x2\n"
+ "ldr h28, [x25], #0x2\n"
+ "ldr h27, [x24], #0x2\n"
+ "ldr h23, [x23], #0x2\n"
+ "ldr h21, [x22], #0x2\n"
+ "ldr h26, [x21], #0x2\n"
+ "ldr h25, [x20], #0x2\n"
+ "mov x19, #0x2\n"
+ "tbz %x[width], #0, 9f\n"
+ "ld1 { v30.b }[2], [x27]\n"
+ "ld1 { v29.b }[2], [x26]\n"
+ "ld1 { v28.b }[2], [x25]\n"
+ "ld1 { v27.b }[2], [x24]\n"
+ "ld1 { v23.b }[2], [x23]\n"
+ "ld1 { v21.b }[2], [x22]\n"
+ "ld1 { v26.b }[2], [x21]\n"
+ "ld1 { v25.b }[2], [x20]\n"
+ "mov x19, #0x3\n"
+ "b 9f\n"
+ "8:" // odd_loads_1_0
+ "ldr b30, [x27, #0x0]\n"
+ "ldr b29, [x26, #0x0]\n"
+ "ldr b28, [x25, #0x0]\n"
+ "ldr b27, [x24, #0x0]\n"
+ "ldr b23, [x23, #0x0]\n"
+ "ldr b21, [x22, #0x0]\n"
+ "ldr b26, [x21, #0x0]\n"
+ "ldr b25, [x20, #0x0]\n"
+ "mov x19, #0x1\n"
+ "9:" // Odd load end
+ "sshll v30.8h, v30.8b, #0x0\n"
+ "sshll v29.8h, v29.8b, #0x0\n"
+ "sshll v28.8h, v28.8b, #0x0\n"
+ "sshll v27.8h, v27.8b, #0x0\n"
+ "sshll v23.8h, v23.8b, #0x0\n"
+ "zip1 v24.8h, v30.8h, v23.8h\n"
+ "sshll v21.8h, v21.8b, #0x0\n"
+ "sshll v26.8h, v26.8b, #0x0\n"
+ "zip1 v20.8h, v28.8h, v26.8h\n"
+ "sshll v25.8h, v25.8b, #0x0\n"
+ "zip1 v22.8h, v29.8h, v21.8h\n"
+ "subs x19, x19, #0x1\n"
+ "zip1 v18.8h, v24.8h, v20.8h\n"
+ "zip1 v19.8h, v27.8h, v25.8h\n"
+ "zip1 v17.8h, v22.8h, v19.8h\n"
+ "zip1 v16.8h, v18.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x0]\n"
+ "add v1.8h, v1.8h, v16.8h\n"
+ "add %x[out_ptr], %x[out_ptr], #0x10\n"
+ "beq 10f\n"
+ "zip2 v16.8h, v18.8h, v17.8h\n"
+ "subs x19, x19, #0x1\n"
+ "add v1.8h, v1.8h, v16.8h\n"
+ "str q16, [%x[out_ptr], #0x0]\n"
+ "add %x[out_ptr], %x[out_ptr], #0x10\n"
+ "beq 10f\n"
+ "zip2 v18.8h, v24.8h, v20.8h\n"
+ "zip2 v17.8h, v22.8h, v19.8h\n"
+ "subs x19, x19, #0x1\n"
+ "zip1 v16.8h, v18.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x0]\n"
+ "add v1.8h, v1.8h, v16.8h\n"
+ "add %x[out_ptr], %x[out_ptr], #0x10\n"
+ "beq 10f\n"
+ "zip2 v16.8h, v18.8h, v17.8h\n"
+ "subs x19, x19, #0x1\n"
+ "add v1.8h, v1.8h, v16.8h\n"
+ "str q16, [%x[out_ptr], #0x0]\n"
+ "add %x[out_ptr], %x[out_ptr], #0x10\n"
+ "beq 10f\n"
+ "zip2 v23.8h, v30.8h, v23.8h\n"
+ "zip2 v20.8h, v28.8h, v26.8h\n"
+ "subs x19, x19, #0x1\n"
+ "zip1 v18.8h, v23.8h, v20.8h\n"
+ "zip2 v21.8h, v29.8h, v21.8h\n"
+ "zip2 v19.8h, v27.8h, v25.8h\n"
+ "zip1 v17.8h, v21.8h, v19.8h\n"
+ "zip1 v16.8h, v18.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x0]\n"
+ "add v1.8h, v1.8h, v16.8h\n"
+ "add %x[out_ptr], %x[out_ptr], #0x10\n"
+ "beq 10f\n"
+ "zip2 v16.8h, v18.8h, v17.8h\n"
+ "subs x19, x19, #0x1\n"
+ "add v1.8h, v1.8h, v16.8h\n"
+ "str q16, [%x[out_ptr], #0x0]\n"
+ "add %x[out_ptr], %x[out_ptr], #0x10\n"
+ "beq 10f\n"
+ "zip2 v18.8h, v23.8h, v20.8h\n"
+ "zip2 v17.8h, v21.8h, v19.8h\n"
+ "zip1 v16.8h, v18.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x0]\n"
+ "add v1.8h, v1.8h, v16.8h\n"
+ "add %x[out_ptr], %x[out_ptr], #0x10\n"
+ "10:" // Odds skip
+ "saddw v0.4s, v0.4s, v1.4h\n"
+ "str q0, [%x[out_ptr], #0x0]\n"
+ "saddw2 v31.4s, v31.4s, v1.8h\n"
+ "str q31, [%x[out_ptr], #0x10]\n"
+ "add %x[out_ptr], %x[out_ptr], #0x20\n"
+ : [out_ptr] "+r" (out_ptr), [width] "+r" (width)
+ : [first] "r" (first), [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset)
+ : "cc", "memory", "v0", "v1", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27"
+ );
+}
+
+
+#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_u16_u16_summing.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_u16_u16_summing.hpp
new file mode 100644
index 0000000..bfa8989
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_u16_u16_summing.hpp
@@ -0,0 +1,306 @@
+/*
+ * Copyright (c) 2019-2020 Arm Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+#ifdef __aarch64__
+
+template<>
+void interleave_block<8, 1, VLType::None, true>(
+ uint16_t * &out_ptr, const uint16_t * const * in, size_t width, size_t height,
+ size_t row_offset, bool first
+)
+{
+ __asm__ __volatile__(
+ "movi v1.8h, #0x0\n"
+ "ldr x27, [%x[in], #0x0]\n"
+ "mov x19, #0x0\n"
+ "movi v0.4s, #0x0\n"
+ "ldr x26, [%x[in], #0x8]\n"
+ "cmp %x[height], #0x8\n"
+ "movi v31.4s, #0x0\n"
+ "ldr x25, [%x[in], #0x10]\n"
+ "add x27, x27, %x[row_offset], LSL #1\n"
+ "ldr x24, [%x[in], #0x18]\n"
+ "ldr x23, [%x[in], #0x20]\n"
+ "add x26, x26, %x[row_offset], LSL #1\n"
+ "ldr x22, [%x[in], #0x28]\n"
+ "add x25, x25, %x[row_offset], LSL #1\n"
+ "ldr x21, [%x[in], #0x30]\n"
+ "add x24, x24, %x[row_offset], LSL #1\n"
+ "ldr x20, [%x[in], #0x38]\n"
+ "add x23, x23, %x[row_offset], LSL #1\n"
+ "add x22, x22, %x[row_offset], LSL #1\n"
+ "add x21, x21, %x[row_offset], LSL #1\n"
+ "add x20, x20, %x[row_offset], LSL #1\n"
+ "beq 1f\n"
+ "mov x20, x27\n"
+ "cmp %x[height], #0x2\n"
+ "csel x26, x26, x27, GE\n"
+ "csel x25, x25, x27, GT\n"
+ "cmp %x[height], #0x4\n"
+ "csel x24, x24, x27, GE\n"
+ "csel x23, x23, x27, GT\n"
+ "cmp %x[height], #0x6\n"
+ "csel x22, x22, x27, GE\n"
+ "csel x21, x21, x27, GT\n"
+ "1:" // no_pointer_adj
+ "prfm pldl1keep, [x27, #0x0]\n"
+ "prfm pldl1keep, [x26, #0x0]\n"
+ "prfm pldl1keep, [x25, #0x0]\n"
+ "prfm pldl1keep, [x24, #0x0]\n"
+ "prfm pldl1keep, [x23, #0x0]\n"
+ "prfm pldl1keep, [x22, #0x0]\n"
+ "prfm pldl1keep, [x21, #0x0]\n"
+ "prfm pldl1keep, [x20, #0x0]\n"
+ "prfm pldl1keep, [x27, #0x40]\n"
+ "prfm pldl1keep, [x26, #0x40]\n"
+ "prfm pldl1keep, [x25, #0x40]\n"
+ "prfm pldl1keep, [x24, #0x40]\n"
+ "prfm pldl1keep, [x23, #0x40]\n"
+ "prfm pldl1keep, [x22, #0x40]\n"
+ "prfm pldl1keep, [x21, #0x40]\n"
+ "prfm pldl1keep, [x20, #0x40]\n"
+ "cbnz %w[first], 2f\n"
+ "sub %x[out_ptr], %x[out_ptr], #0x20\n"
+ "ld1 { v0.4s }, [%x[out_ptr]]\n"
+ "ldr q31, [%x[out_ptr], #0x10]\n"
+ "2:" // first_pass
+ "cmp %x[width], #0x8\n"
+ "blt 5f\n"
+ "3:" // Main loop head
+ "cmp x19, #0xe\n"
+ "ble 4f\n"
+ "uaddw v0.4s, v0.4s, v1.4h\n"
+ "uaddw2 v31.4s, v31.4s, v1.8h\n"
+ "mov x19, #0x0\n"
+ "movi v1.8h, #0x0\n"
+ "4:" // no_accumulate_16
+ "ldr q30, [x27], #0x10\n"
+ "prfm pldl1keep, [x27, #0x70]\n"
+ "ldr q29, [x26], #0x10\n"
+ "ldr q28, [x25], #0x10\n"
+ "prfm pldl1keep, [x26, #0x70]\n"
+ "ldr q27, [x24], #0x10\n"
+ "prfm pldl1keep, [x25, #0x70]\n"
+ "ldr q24, [x23], #0x10\n"
+ "zip1 v26.8h, v30.8h, v24.8h\n"
+ "prfm pldl1keep, [x24, #0x70]\n"
+ "ldr q25, [x22], #0x10\n"
+ "zip2 v24.8h, v30.8h, v24.8h\n"
+ "prfm pldl1keep, [x23, #0x70]\n"
+ "ldr q23, [x21], #0x10\n"
+ "zip1 v21.8h, v29.8h, v25.8h\n"
+ "prfm pldl1keep, [x22, #0x70]\n"
+ "ldr q22, [x20], #0x10\n"
+ "zip1 v18.8h, v28.8h, v23.8h\n"
+ "prfm pldl1keep, [x21, #0x70]\n"
+ "add x19, x19, #0x1\n"
+ "zip1 v20.8h, v26.8h, v18.8h\n"
+ "prfm pldl1keep, [x20, #0x70]\n"
+ "zip1 v19.8h, v27.8h, v22.8h\n"
+ "subs %x[width], %x[width], #0x8\n"
+ "zip1 v17.8h, v21.8h, v19.8h\n"
+ "cmp %x[width], #0x8\n"
+ "zip2 v18.8h, v26.8h, v18.8h\n"
+ "zip1 v16.8h, v20.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x0]\n"
+ "add v1.8h, v1.8h, v16.8h\n"
+ "zip2 v17.8h, v20.8h, v17.8h\n"
+ "str q17, [%x[out_ptr], #0x10]\n"
+ "zip2 v16.8h, v21.8h, v19.8h\n"
+ "add v1.8h, v1.8h, v17.8h\n"
+ "zip1 v17.8h, v18.8h, v16.8h\n"
+ "str q17, [%x[out_ptr], #0x20]\n"
+ "zip2 v16.8h, v18.8h, v16.8h\n"
+ "str q16, [%x[out_ptr], #0x30]\n"
+ "add v1.8h, v1.8h, v17.8h\n"
+ "zip2 v21.8h, v28.8h, v23.8h\n"
+ "zip1 v18.8h, v24.8h, v21.8h\n"
+ "add v1.8h, v1.8h, v16.8h\n"
+ "zip2 v20.8h, v29.8h, v25.8h\n"
+ "zip2 v19.8h, v27.8h, v22.8h\n"
+ "zip1 v17.8h, v20.8h, v19.8h\n"
+ "zip1 v16.8h, v18.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x40]\n"
+ "add v1.8h, v1.8h, v16.8h\n"
+ "zip2 v16.8h, v18.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x50]\n"
+ "zip2 v18.8h, v24.8h, v21.8h\n"
+ "add v1.8h, v1.8h, v16.8h\n"
+ "zip2 v17.8h, v20.8h, v19.8h\n"
+ "zip1 v16.8h, v18.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x60]\n"
+ "add v1.8h, v1.8h, v16.8h\n"
+ "zip2 v16.8h, v18.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x70]\n"
+ "add %x[out_ptr], %x[out_ptr], #0x80\n"
+ "add v1.8h, v1.8h, v16.8h\n"
+ "bge 3b\n"
+ "5:" // Main loop skip
+ "cbz %x[width], 10f\n"
+ "tbz %x[width], #2, 7f\n"
+ "ldr d30, [x27], #0x8\n"
+ "ldr d29, [x26], #0x8\n"
+ "ldr d28, [x25], #0x8\n"
+ "ldr d27, [x24], #0x8\n"
+ "ldr d24, [x23], #0x8\n"
+ "ldr d25, [x22], #0x8\n"
+ "ldr d23, [x21], #0x8\n"
+ "ldr d22, [x20], #0x8\n"
+ "tbz %x[width], #1, 6f\n"
+ "ld1 { v30.s }[2], [x27], #0x4\n"
+ "ld1 { v29.s }[2], [x26], #0x4\n"
+ "ld1 { v28.s }[2], [x25], #0x4\n"
+ "ld1 { v27.s }[2], [x24], #0x4\n"
+ "ld1 { v24.s }[2], [x23], #0x4\n"
+ "ld1 { v25.s }[2], [x22], #0x4\n"
+ "ld1 { v23.s }[2], [x21], #0x4\n"
+ "ld1 { v22.s }[2], [x20], #0x4\n"
+ "mov x19, #0x6\n"
+ "tbz %x[width], #0, 9f\n"
+ "ld1 { v30.h }[6], [x27]\n"
+ "ld1 { v29.h }[6], [x26]\n"
+ "ld1 { v28.h }[6], [x25]\n"
+ "ld1 { v27.h }[6], [x24]\n"
+ "ld1 { v24.h }[6], [x23]\n"
+ "ld1 { v25.h }[6], [x22]\n"
+ "ld1 { v23.h }[6], [x21]\n"
+ "ld1 { v22.h }[6], [x20]\n"
+ "mov x19, #0x7\n"
+ "b 9f\n"
+ "6:" // odd_loads_1_4
+ "mov x19, #0x4\n"
+ "tbz %x[width], #0, 9f\n"
+ "ld1 { v30.h }[4], [x27]\n"
+ "ld1 { v29.h }[4], [x26]\n"
+ "ld1 { v28.h }[4], [x25]\n"
+ "ld1 { v27.h }[4], [x24]\n"
+ "ld1 { v24.h }[4], [x23]\n"
+ "ld1 { v25.h }[4], [x22]\n"
+ "ld1 { v23.h }[4], [x21]\n"
+ "ld1 { v22.h }[4], [x20]\n"
+ "mov x19, #0x5\n"
+ "b 9f\n"
+ "7:" // odd_loads_2_0
+ "tbz %x[width], #1, 8f\n"
+ "ldr s30, [x27], #0x4\n"
+ "ldr s29, [x26], #0x4\n"
+ "ldr s28, [x25], #0x4\n"
+ "ldr s27, [x24], #0x4\n"
+ "ldr s24, [x23], #0x4\n"
+ "ldr s25, [x22], #0x4\n"
+ "ldr s23, [x21], #0x4\n"
+ "ldr s22, [x20], #0x4\n"
+ "mov x19, #0x2\n"
+ "tbz %x[width], #0, 9f\n"
+ "ld1 { v30.h }[2], [x27]\n"
+ "ld1 { v29.h }[2], [x26]\n"
+ "ld1 { v28.h }[2], [x25]\n"
+ "ld1 { v27.h }[2], [x24]\n"
+ "ld1 { v24.h }[2], [x23]\n"
+ "ld1 { v25.h }[2], [x22]\n"
+ "ld1 { v23.h }[2], [x21]\n"
+ "ld1 { v22.h }[2], [x20]\n"
+ "mov x19, #0x3\n"
+ "b 9f\n"
+ "8:" // odd_loads_1_0
+ "ldr h30, [x27, #0x0]\n"
+ "ldr h29, [x26, #0x0]\n"
+ "ldr h28, [x25, #0x0]\n"
+ "ldr h27, [x24, #0x0]\n"
+ "ldr h24, [x23, #0x0]\n"
+ "ldr h25, [x22, #0x0]\n"
+ "ldr h23, [x21, #0x0]\n"
+ "ldr h22, [x20, #0x0]\n"
+ "mov x19, #0x1\n"
+ "9:" // Odd load end
+ "zip1 v26.8h, v30.8h, v24.8h\n"
+ "subs x19, x19, #0x1\n"
+ "zip1 v18.8h, v28.8h, v23.8h\n"
+ "zip1 v20.8h, v26.8h, v18.8h\n"
+ "zip1 v21.8h, v29.8h, v25.8h\n"
+ "zip1 v19.8h, v27.8h, v22.8h\n"
+ "zip1 v17.8h, v21.8h, v19.8h\n"
+ "zip1 v16.8h, v20.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x0]\n"
+ "add v1.8h, v1.8h, v16.8h\n"
+ "add %x[out_ptr], %x[out_ptr], #0x10\n"
+ "beq 10f\n"
+ "zip2 v17.8h, v20.8h, v17.8h\n"
+ "subs x19, x19, #0x1\n"
+ "add v1.8h, v1.8h, v17.8h\n"
+ "str q17, [%x[out_ptr], #0x0]\n"
+ "add %x[out_ptr], %x[out_ptr], #0x10\n"
+ "beq 10f\n"
+ "zip2 v18.8h, v26.8h, v18.8h\n"
+ "zip2 v16.8h, v21.8h, v19.8h\n"
+ "subs x19, x19, #0x1\n"
+ "zip1 v17.8h, v18.8h, v16.8h\n"
+ "str q17, [%x[out_ptr], #0x0]\n"
+ "add v1.8h, v1.8h, v17.8h\n"
+ "add %x[out_ptr], %x[out_ptr], #0x10\n"
+ "beq 10f\n"
+ "zip2 v16.8h, v18.8h, v16.8h\n"
+ "subs x19, x19, #0x1\n"
+ "add v1.8h, v1.8h, v16.8h\n"
+ "str q16, [%x[out_ptr], #0x0]\n"
+ "add %x[out_ptr], %x[out_ptr], #0x10\n"
+ "beq 10f\n"
+ "zip2 v24.8h, v30.8h, v24.8h\n"
+ "zip2 v21.8h, v28.8h, v23.8h\n"
+ "subs x19, x19, #0x1\n"
+ "zip1 v18.8h, v24.8h, v21.8h\n"
+ "zip2 v20.8h, v29.8h, v25.8h\n"
+ "zip2 v19.8h, v27.8h, v22.8h\n"
+ "zip1 v17.8h, v20.8h, v19.8h\n"
+ "zip1 v16.8h, v18.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x0]\n"
+ "add v1.8h, v1.8h, v16.8h\n"
+ "add %x[out_ptr], %x[out_ptr], #0x10\n"
+ "beq 10f\n"
+ "zip2 v16.8h, v18.8h, v17.8h\n"
+ "subs x19, x19, #0x1\n"
+ "add v1.8h, v1.8h, v16.8h\n"
+ "str q16, [%x[out_ptr], #0x0]\n"
+ "add %x[out_ptr], %x[out_ptr], #0x10\n"
+ "beq 10f\n"
+ "zip2 v18.8h, v24.8h, v21.8h\n"
+ "zip2 v17.8h, v20.8h, v19.8h\n"
+ "zip1 v16.8h, v18.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x0]\n"
+ "add v1.8h, v1.8h, v16.8h\n"
+ "add %x[out_ptr], %x[out_ptr], #0x10\n"
+ "10:" // Odds skip
+ "uaddw v0.4s, v0.4s, v1.4h\n"
+ "str q0, [%x[out_ptr], #0x0]\n"
+ "uaddw2 v31.4s, v31.4s, v1.8h\n"
+ "str q31, [%x[out_ptr], #0x10]\n"
+ "add %x[out_ptr], %x[out_ptr], #0x20\n"
+ : [out_ptr] "+r" (out_ptr), [width] "+r" (width)
+ : [first] "r" (first), [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset)
+ : "cc", "memory", "v0", "v1", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27"
+ );
+}
+
+
+#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_u8_u16.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_u8_u16.hpp
new file mode 100644
index 0000000..86b90f1
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_u8_u16.hpp
@@ -0,0 +1,286 @@
+/*
+ * Copyright (c) 2019-2020 Arm Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+#ifdef __aarch64__
+
+template<>
+void interleave_block<8, 1, VLType::None, false>(
+ uint16_t * &out_ptr, const uint8_t * const * in, size_t width, size_t height,
+ size_t row_offset, bool
+)
+{
+ __asm__ __volatile__(
+ "ldr x27, [%x[in], #0x0]\n"
+ "cmp %x[height], #0x8\n"
+ "ldr x26, [%x[in], #0x8]\n"
+ "add x27, x27, %x[row_offset]\n"
+ "ldr x25, [%x[in], #0x10]\n"
+ "ldr x24, [%x[in], #0x18]\n"
+ "add x26, x26, %x[row_offset]\n"
+ "ldr x23, [%x[in], #0x20]\n"
+ "add x25, x25, %x[row_offset]\n"
+ "ldr x22, [%x[in], #0x28]\n"
+ "ldr x21, [%x[in], #0x30]\n"
+ "add x24, x24, %x[row_offset]\n"
+ "ldr x20, [%x[in], #0x38]\n"
+ "add x23, x23, %x[row_offset]\n"
+ "add x22, x22, %x[row_offset]\n"
+ "add x21, x21, %x[row_offset]\n"
+ "add x20, x20, %x[row_offset]\n"
+ "beq 1f\n"
+ "mov x20, x27\n"
+ "cmp %x[height], #0x2\n"
+ "csel x26, x26, x27, GE\n"
+ "csel x25, x25, x27, GT\n"
+ "cmp %x[height], #0x4\n"
+ "csel x24, x24, x27, GE\n"
+ "csel x23, x23, x27, GT\n"
+ "cmp %x[height], #0x6\n"
+ "csel x22, x22, x27, GE\n"
+ "csel x21, x21, x27, GT\n"
+ "1:" // no_pointer_adj
+ "prfm pldl1keep, [x27, #0x0]\n"
+ "cmp %x[width], #0x8\n"
+ "prfm pldl1keep, [x26, #0x0]\n"
+ "prfm pldl1keep, [x25, #0x0]\n"
+ "prfm pldl1keep, [x24, #0x0]\n"
+ "prfm pldl1keep, [x23, #0x0]\n"
+ "prfm pldl1keep, [x22, #0x0]\n"
+ "prfm pldl1keep, [x21, #0x0]\n"
+ "prfm pldl1keep, [x20, #0x0]\n"
+ "prfm pldl1keep, [x27, #0x40]\n"
+ "prfm pldl1keep, [x26, #0x40]\n"
+ "prfm pldl1keep, [x25, #0x40]\n"
+ "prfm pldl1keep, [x24, #0x40]\n"
+ "prfm pldl1keep, [x23, #0x40]\n"
+ "prfm pldl1keep, [x22, #0x40]\n"
+ "prfm pldl1keep, [x21, #0x40]\n"
+ "prfm pldl1keep, [x20, #0x40]\n"
+ "blt 3f\n"
+ "2:" // Main loop head
+ "ldr d30, [x27], #0x8\n"
+ "prfm pldl1keep, [x27, #0x70]\n"
+ "ldr d29, [x26], #0x8\n"
+ "ldr d28, [x25], #0x8\n"
+ "prfm pldl1keep, [x26, #0x70]\n"
+ "ldr d27, [x24], #0x8\n"
+ "prfm pldl1keep, [x25, #0x70]\n"
+ "ldr d23, [x23], #0x8\n"
+ "ldr d21, [x22], #0x8\n"
+ "prfm pldl1keep, [x24, #0x70]\n"
+ "ldr d26, [x21], #0x8\n"
+ "ldr d25, [x20], #0x8\n"
+ "prfm pldl1keep, [x23, #0x70]\n"
+ "prfm pldl1keep, [x22, #0x70]\n"
+ "ushll v30.8h, v30.8b, #0x0\n"
+ "ushll v29.8h, v29.8b, #0x0\n"
+ "prfm pldl1keep, [x21, #0x70]\n"
+ "ushll v28.8h, v28.8b, #0x0\n"
+ "prfm pldl1keep, [x20, #0x70]\n"
+ "ushll v27.8h, v27.8b, #0x0\n"
+ "ushll v23.8h, v23.8b, #0x0\n"
+ "zip1 v24.8h, v30.8h, v23.8h\n"
+ "ushll v21.8h, v21.8b, #0x0\n"
+ "zip2 v23.8h, v30.8h, v23.8h\n"
+ "ushll v26.8h, v26.8b, #0x0\n"
+ "ushll v25.8h, v25.8b, #0x0\n"
+ "zip1 v22.8h, v29.8h, v21.8h\n"
+ "subs %x[width], %x[width], #0x8\n"
+ "zip2 v21.8h, v29.8h, v21.8h\n"
+ "cmp %x[width], #0x8\n"
+ "zip1 v20.8h, v28.8h, v26.8h\n"
+ "zip1 v18.8h, v24.8h, v20.8h\n"
+ "zip1 v19.8h, v27.8h, v25.8h\n"
+ "zip1 v17.8h, v22.8h, v19.8h\n"
+ "zip1 v16.8h, v18.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x0]\n"
+ "zip2 v16.8h, v18.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x10]\n"
+ "zip2 v18.8h, v24.8h, v20.8h\n"
+ "zip2 v17.8h, v22.8h, v19.8h\n"
+ "zip1 v16.8h, v18.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x20]\n"
+ "zip2 v16.8h, v18.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x30]\n"
+ "zip2 v20.8h, v28.8h, v26.8h\n"
+ "zip1 v18.8h, v23.8h, v20.8h\n"
+ "zip2 v19.8h, v27.8h, v25.8h\n"
+ "zip1 v17.8h, v21.8h, v19.8h\n"
+ "zip1 v16.8h, v18.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x40]\n"
+ "zip2 v16.8h, v18.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x50]\n"
+ "zip2 v18.8h, v23.8h, v20.8h\n"
+ "zip2 v17.8h, v21.8h, v19.8h\n"
+ "zip1 v16.8h, v18.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x60]\n"
+ "zip2 v16.8h, v18.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x70]\n"
+ "add %x[out_ptr], %x[out_ptr], #0x80\n"
+ "bge 2b\n"
+ "3:" // Main loop skip
+ "cbz %x[width], 8f\n"
+ "tbz %x[width], #2, 5f\n"
+ "ldr s30, [x27], #0x4\n"
+ "ldr s29, [x26], #0x4\n"
+ "ldr s28, [x25], #0x4\n"
+ "ldr s27, [x24], #0x4\n"
+ "ldr s23, [x23], #0x4\n"
+ "ldr s21, [x22], #0x4\n"
+ "ldr s26, [x21], #0x4\n"
+ "ldr s25, [x20], #0x4\n"
+ "tbz %x[width], #1, 4f\n"
+ "ld1 { v30.h }[2], [x27], #0x2\n"
+ "ld1 { v29.h }[2], [x26], #0x2\n"
+ "ld1 { v28.h }[2], [x25], #0x2\n"
+ "ld1 { v27.h }[2], [x24], #0x2\n"
+ "ld1 { v23.h }[2], [x23], #0x2\n"
+ "ld1 { v21.h }[2], [x22], #0x2\n"
+ "ld1 { v26.h }[2], [x21], #0x2\n"
+ "ld1 { v25.h }[2], [x20], #0x2\n"
+ "mov x19, #0x6\n"
+ "tbz %x[width], #0, 7f\n"
+ "ld1 { v30.b }[6], [x27]\n"
+ "ld1 { v29.b }[6], [x26]\n"
+ "ld1 { v28.b }[6], [x25]\n"
+ "ld1 { v27.b }[6], [x24]\n"
+ "ld1 { v23.b }[6], [x23]\n"
+ "ld1 { v21.b }[6], [x22]\n"
+ "ld1 { v26.b }[6], [x21]\n"
+ "ld1 { v25.b }[6], [x20]\n"
+ "mov x19, #0x7\n"
+ "b 7f\n"
+ "4:" // odd_loads_1_4
+ "mov x19, #0x4\n"
+ "tbz %x[width], #0, 7f\n"
+ "ld1 { v30.b }[4], [x27]\n"
+ "ld1 { v29.b }[4], [x26]\n"
+ "ld1 { v28.b }[4], [x25]\n"
+ "ld1 { v27.b }[4], [x24]\n"
+ "ld1 { v23.b }[4], [x23]\n"
+ "ld1 { v21.b }[4], [x22]\n"
+ "ld1 { v26.b }[4], [x21]\n"
+ "ld1 { v25.b }[4], [x20]\n"
+ "mov x19, #0x5\n"
+ "b 7f\n"
+ "5:" // odd_loads_2_0
+ "tbz %x[width], #1, 6f\n"
+ "ldr h30, [x27], #0x2\n"
+ "ldr h29, [x26], #0x2\n"
+ "ldr h28, [x25], #0x2\n"
+ "ldr h27, [x24], #0x2\n"
+ "ldr h23, [x23], #0x2\n"
+ "ldr h21, [x22], #0x2\n"
+ "ldr h26, [x21], #0x2\n"
+ "ldr h25, [x20], #0x2\n"
+ "mov x19, #0x2\n"
+ "tbz %x[width], #0, 7f\n"
+ "ld1 { v30.b }[2], [x27]\n"
+ "ld1 { v29.b }[2], [x26]\n"
+ "ld1 { v28.b }[2], [x25]\n"
+ "ld1 { v27.b }[2], [x24]\n"
+ "ld1 { v23.b }[2], [x23]\n"
+ "ld1 { v21.b }[2], [x22]\n"
+ "ld1 { v26.b }[2], [x21]\n"
+ "ld1 { v25.b }[2], [x20]\n"
+ "mov x19, #0x3\n"
+ "b 7f\n"
+ "6:" // odd_loads_1_0
+ "ldr b30, [x27, #0x0]\n"
+ "ldr b29, [x26, #0x0]\n"
+ "ldr b28, [x25, #0x0]\n"
+ "ldr b27, [x24, #0x0]\n"
+ "ldr b23, [x23, #0x0]\n"
+ "ldr b21, [x22, #0x0]\n"
+ "ldr b26, [x21, #0x0]\n"
+ "ldr b25, [x20, #0x0]\n"
+ "mov x19, #0x1\n"
+ "7:" // Odd load end
+ "ushll v30.8h, v30.8b, #0x0\n"
+ "ushll v29.8h, v29.8b, #0x0\n"
+ "ushll v28.8h, v28.8b, #0x0\n"
+ "ushll v27.8h, v27.8b, #0x0\n"
+ "ushll v23.8h, v23.8b, #0x0\n"
+ "zip1 v24.8h, v30.8h, v23.8h\n"
+ "ushll v21.8h, v21.8b, #0x0\n"
+ "ushll v26.8h, v26.8b, #0x0\n"
+ "zip1 v20.8h, v28.8h, v26.8h\n"
+ "ushll v25.8h, v25.8b, #0x0\n"
+ "zip1 v22.8h, v29.8h, v21.8h\n"
+ "subs x19, x19, #0x1\n"
+ "zip1 v18.8h, v24.8h, v20.8h\n"
+ "zip1 v19.8h, v27.8h, v25.8h\n"
+ "zip1 v17.8h, v22.8h, v19.8h\n"
+ "zip1 v16.8h, v18.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x0]\n"
+ "add %x[out_ptr], %x[out_ptr], #0x10\n"
+ "beq 8f\n"
+ "zip2 v16.8h, v18.8h, v17.8h\n"
+ "subs x19, x19, #0x1\n"
+ "str q16, [%x[out_ptr], #0x0]\n"
+ "add %x[out_ptr], %x[out_ptr], #0x10\n"
+ "beq 8f\n"
+ "zip2 v18.8h, v24.8h, v20.8h\n"
+ "zip2 v17.8h, v22.8h, v19.8h\n"
+ "subs x19, x19, #0x1\n"
+ "zip1 v16.8h, v18.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x0]\n"
+ "add %x[out_ptr], %x[out_ptr], #0x10\n"
+ "beq 8f\n"
+ "zip2 v16.8h, v18.8h, v17.8h\n"
+ "subs x19, x19, #0x1\n"
+ "str q16, [%x[out_ptr], #0x0]\n"
+ "add %x[out_ptr], %x[out_ptr], #0x10\n"
+ "beq 8f\n"
+ "zip2 v23.8h, v30.8h, v23.8h\n"
+ "zip2 v20.8h, v28.8h, v26.8h\n"
+ "subs x19, x19, #0x1\n"
+ "zip1 v18.8h, v23.8h, v20.8h\n"
+ "zip2 v21.8h, v29.8h, v21.8h\n"
+ "zip2 v19.8h, v27.8h, v25.8h\n"
+ "zip1 v17.8h, v21.8h, v19.8h\n"
+ "zip1 v16.8h, v18.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x0]\n"
+ "add %x[out_ptr], %x[out_ptr], #0x10\n"
+ "beq 8f\n"
+ "zip2 v16.8h, v18.8h, v17.8h\n"
+ "subs x19, x19, #0x1\n"
+ "str q16, [%x[out_ptr], #0x0]\n"
+ "add %x[out_ptr], %x[out_ptr], #0x10\n"
+ "beq 8f\n"
+ "zip2 v18.8h, v23.8h, v20.8h\n"
+ "zip2 v17.8h, v21.8h, v19.8h\n"
+ "zip1 v16.8h, v18.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x0]\n"
+ "add %x[out_ptr], %x[out_ptr], #0x10\n"
+ "8:" // Odds skip
+
+ : [out_ptr] "+r" (out_ptr), [width] "+r" (width)
+ : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset)
+ : "cc", "memory", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27"
+ );
+}
+
+
+#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_u8_u16_summing.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_u8_u16_summing.hpp
new file mode 100644
index 0000000..cefb70c
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_u8_u16_summing.hpp
@@ -0,0 +1,322 @@
+/*
+ * Copyright (c) 2019-2020 Arm Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+#ifdef __aarch64__
+
+template<>
+void interleave_block<8, 1, VLType::None, true>(
+ uint16_t * &out_ptr, const uint8_t * const * in, size_t width, size_t height,
+ size_t row_offset, bool first
+)
+{
+ __asm__ __volatile__(
+ "movi v1.8h, #0x0\n"
+ "ldr x27, [%x[in], #0x0]\n"
+ "mov x19, #0x0\n"
+ "movi v0.4s, #0x0\n"
+ "ldr x26, [%x[in], #0x8]\n"
+ "cmp %x[height], #0x8\n"
+ "movi v31.4s, #0x0\n"
+ "ldr x25, [%x[in], #0x10]\n"
+ "add x27, x27, %x[row_offset]\n"
+ "ldr x24, [%x[in], #0x18]\n"
+ "ldr x23, [%x[in], #0x20]\n"
+ "add x26, x26, %x[row_offset]\n"
+ "ldr x22, [%x[in], #0x28]\n"
+ "add x25, x25, %x[row_offset]\n"
+ "ldr x21, [%x[in], #0x30]\n"
+ "add x24, x24, %x[row_offset]\n"
+ "ldr x20, [%x[in], #0x38]\n"
+ "add x23, x23, %x[row_offset]\n"
+ "add x22, x22, %x[row_offset]\n"
+ "add x21, x21, %x[row_offset]\n"
+ "add x20, x20, %x[row_offset]\n"
+ "beq 1f\n"
+ "mov x20, x27\n"
+ "cmp %x[height], #0x2\n"
+ "csel x26, x26, x27, GE\n"
+ "csel x25, x25, x27, GT\n"
+ "cmp %x[height], #0x4\n"
+ "csel x24, x24, x27, GE\n"
+ "csel x23, x23, x27, GT\n"
+ "cmp %x[height], #0x6\n"
+ "csel x22, x22, x27, GE\n"
+ "csel x21, x21, x27, GT\n"
+ "1:" // no_pointer_adj
+ "prfm pldl1keep, [x27, #0x0]\n"
+ "prfm pldl1keep, [x26, #0x0]\n"
+ "prfm pldl1keep, [x25, #0x0]\n"
+ "prfm pldl1keep, [x24, #0x0]\n"
+ "prfm pldl1keep, [x23, #0x0]\n"
+ "prfm pldl1keep, [x22, #0x0]\n"
+ "prfm pldl1keep, [x21, #0x0]\n"
+ "prfm pldl1keep, [x20, #0x0]\n"
+ "prfm pldl1keep, [x27, #0x40]\n"
+ "prfm pldl1keep, [x26, #0x40]\n"
+ "prfm pldl1keep, [x25, #0x40]\n"
+ "prfm pldl1keep, [x24, #0x40]\n"
+ "prfm pldl1keep, [x23, #0x40]\n"
+ "prfm pldl1keep, [x22, #0x40]\n"
+ "prfm pldl1keep, [x21, #0x40]\n"
+ "prfm pldl1keep, [x20, #0x40]\n"
+ "cbnz %w[first], 2f\n"
+ "sub %x[out_ptr], %x[out_ptr], #0x20\n"
+ "ld1 { v0.4s }, [%x[out_ptr]]\n"
+ "ldr q31, [%x[out_ptr], #0x10]\n"
+ "2:" // first_pass
+ "cmp %x[width], #0x8\n"
+ "blt 5f\n"
+ "3:" // Main loop head
+ "cmp x19, #0xe\n"
+ "ble 4f\n"
+ "uaddw v0.4s, v0.4s, v1.4h\n"
+ "uaddw2 v31.4s, v31.4s, v1.8h\n"
+ "mov x19, #0x0\n"
+ "movi v1.8h, #0x0\n"
+ "4:" // no_accumulate_16
+ "ldr d30, [x27], #0x8\n"
+ "prfm pldl1keep, [x27, #0x70]\n"
+ "ldr d29, [x26], #0x8\n"
+ "ldr d28, [x25], #0x8\n"
+ "prfm pldl1keep, [x26, #0x70]\n"
+ "ldr d27, [x24], #0x8\n"
+ "prfm pldl1keep, [x25, #0x70]\n"
+ "ldr d23, [x23], #0x8\n"
+ "ldr d21, [x22], #0x8\n"
+ "prfm pldl1keep, [x24, #0x70]\n"
+ "ldr d26, [x21], #0x8\n"
+ "ldr d25, [x20], #0x8\n"
+ "prfm pldl1keep, [x23, #0x70]\n"
+ "prfm pldl1keep, [x22, #0x70]\n"
+ "ushll v30.8h, v30.8b, #0x0\n"
+ "ushll v29.8h, v29.8b, #0x0\n"
+ "prfm pldl1keep, [x21, #0x70]\n"
+ "ushll v28.8h, v28.8b, #0x0\n"
+ "prfm pldl1keep, [x20, #0x70]\n"
+ "ushll v27.8h, v27.8b, #0x0\n"
+ "ushll v23.8h, v23.8b, #0x0\n"
+ "zip1 v24.8h, v30.8h, v23.8h\n"
+ "ushll v21.8h, v21.8b, #0x0\n"
+ "zip2 v23.8h, v30.8h, v23.8h\n"
+ "ushll v26.8h, v26.8b, #0x0\n"
+ "ushll v25.8h, v25.8b, #0x0\n"
+ "zip1 v22.8h, v29.8h, v21.8h\n"
+ "add x19, x19, #0x1\n"
+ "zip2 v21.8h, v29.8h, v21.8h\n"
+ "subs %x[width], %x[width], #0x8\n"
+ "zip1 v20.8h, v28.8h, v26.8h\n"
+ "cmp %x[width], #0x8\n"
+ "zip1 v18.8h, v24.8h, v20.8h\n"
+ "zip1 v19.8h, v27.8h, v25.8h\n"
+ "zip1 v17.8h, v22.8h, v19.8h\n"
+ "zip1 v16.8h, v18.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x0]\n"
+ "add v1.8h, v1.8h, v16.8h\n"
+ "zip2 v16.8h, v18.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x10]\n"
+ "zip2 v18.8h, v24.8h, v20.8h\n"
+ "add v1.8h, v1.8h, v16.8h\n"
+ "zip2 v17.8h, v22.8h, v19.8h\n"
+ "zip1 v16.8h, v18.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x20]\n"
+ "add v1.8h, v1.8h, v16.8h\n"
+ "zip2 v16.8h, v18.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x30]\n"
+ "zip2 v20.8h, v28.8h, v26.8h\n"
+ "add v1.8h, v1.8h, v16.8h\n"
+ "zip1 v18.8h, v23.8h, v20.8h\n"
+ "zip2 v19.8h, v27.8h, v25.8h\n"
+ "zip1 v17.8h, v21.8h, v19.8h\n"
+ "zip1 v16.8h, v18.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x40]\n"
+ "add v1.8h, v1.8h, v16.8h\n"
+ "zip2 v16.8h, v18.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x50]\n"
+ "zip2 v18.8h, v23.8h, v20.8h\n"
+ "add v1.8h, v1.8h, v16.8h\n"
+ "zip2 v17.8h, v21.8h, v19.8h\n"
+ "zip1 v16.8h, v18.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x60]\n"
+ "add v1.8h, v1.8h, v16.8h\n"
+ "zip2 v16.8h, v18.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x70]\n"
+ "add %x[out_ptr], %x[out_ptr], #0x80\n"
+ "add v1.8h, v1.8h, v16.8h\n"
+ "bge 3b\n"
+ "5:" // Main loop skip
+ "cbz %x[width], 10f\n"
+ "tbz %x[width], #2, 7f\n"
+ "ldr s30, [x27], #0x4\n"
+ "ldr s29, [x26], #0x4\n"
+ "ldr s28, [x25], #0x4\n"
+ "ldr s27, [x24], #0x4\n"
+ "ldr s23, [x23], #0x4\n"
+ "ldr s21, [x22], #0x4\n"
+ "ldr s26, [x21], #0x4\n"
+ "ldr s25, [x20], #0x4\n"
+ "tbz %x[width], #1, 6f\n"
+ "ld1 { v30.h }[2], [x27], #0x2\n"
+ "ld1 { v29.h }[2], [x26], #0x2\n"
+ "ld1 { v28.h }[2], [x25], #0x2\n"
+ "ld1 { v27.h }[2], [x24], #0x2\n"
+ "ld1 { v23.h }[2], [x23], #0x2\n"
+ "ld1 { v21.h }[2], [x22], #0x2\n"
+ "ld1 { v26.h }[2], [x21], #0x2\n"
+ "ld1 { v25.h }[2], [x20], #0x2\n"
+ "mov x19, #0x6\n"
+ "tbz %x[width], #0, 9f\n"
+ "ld1 { v30.b }[6], [x27]\n"
+ "ld1 { v29.b }[6], [x26]\n"
+ "ld1 { v28.b }[6], [x25]\n"
+ "ld1 { v27.b }[6], [x24]\n"
+ "ld1 { v23.b }[6], [x23]\n"
+ "ld1 { v21.b }[6], [x22]\n"
+ "ld1 { v26.b }[6], [x21]\n"
+ "ld1 { v25.b }[6], [x20]\n"
+ "mov x19, #0x7\n"
+ "b 9f\n"
+ "6:" // odd_loads_1_4
+ "mov x19, #0x4\n"
+ "tbz %x[width], #0, 9f\n"
+ "ld1 { v30.b }[4], [x27]\n"
+ "ld1 { v29.b }[4], [x26]\n"
+ "ld1 { v28.b }[4], [x25]\n"
+ "ld1 { v27.b }[4], [x24]\n"
+ "ld1 { v23.b }[4], [x23]\n"
+ "ld1 { v21.b }[4], [x22]\n"
+ "ld1 { v26.b }[4], [x21]\n"
+ "ld1 { v25.b }[4], [x20]\n"
+ "mov x19, #0x5\n"
+ "b 9f\n"
+ "7:" // odd_loads_2_0
+ "tbz %x[width], #1, 8f\n"
+ "ldr h30, [x27], #0x2\n"
+ "ldr h29, [x26], #0x2\n"
+ "ldr h28, [x25], #0x2\n"
+ "ldr h27, [x24], #0x2\n"
+ "ldr h23, [x23], #0x2\n"
+ "ldr h21, [x22], #0x2\n"
+ "ldr h26, [x21], #0x2\n"
+ "ldr h25, [x20], #0x2\n"
+ "mov x19, #0x2\n"
+ "tbz %x[width], #0, 9f\n"
+ "ld1 { v30.b }[2], [x27]\n"
+ "ld1 { v29.b }[2], [x26]\n"
+ "ld1 { v28.b }[2], [x25]\n"
+ "ld1 { v27.b }[2], [x24]\n"
+ "ld1 { v23.b }[2], [x23]\n"
+ "ld1 { v21.b }[2], [x22]\n"
+ "ld1 { v26.b }[2], [x21]\n"
+ "ld1 { v25.b }[2], [x20]\n"
+ "mov x19, #0x3\n"
+ "b 9f\n"
+ "8:" // odd_loads_1_0
+ "ldr b30, [x27, #0x0]\n"
+ "ldr b29, [x26, #0x0]\n"
+ "ldr b28, [x25, #0x0]\n"
+ "ldr b27, [x24, #0x0]\n"
+ "ldr b23, [x23, #0x0]\n"
+ "ldr b21, [x22, #0x0]\n"
+ "ldr b26, [x21, #0x0]\n"
+ "ldr b25, [x20, #0x0]\n"
+ "mov x19, #0x1\n"
+ "9:" // Odd load end
+ "ushll v30.8h, v30.8b, #0x0\n"
+ "ushll v29.8h, v29.8b, #0x0\n"
+ "ushll v28.8h, v28.8b, #0x0\n"
+ "ushll v27.8h, v27.8b, #0x0\n"
+ "ushll v23.8h, v23.8b, #0x0\n"
+ "zip1 v24.8h, v30.8h, v23.8h\n"
+ "ushll v21.8h, v21.8b, #0x0\n"
+ "ushll v26.8h, v26.8b, #0x0\n"
+ "zip1 v20.8h, v28.8h, v26.8h\n"
+ "ushll v25.8h, v25.8b, #0x0\n"
+ "zip1 v22.8h, v29.8h, v21.8h\n"
+ "subs x19, x19, #0x1\n"
+ "zip1 v18.8h, v24.8h, v20.8h\n"
+ "zip1 v19.8h, v27.8h, v25.8h\n"
+ "zip1 v17.8h, v22.8h, v19.8h\n"
+ "zip1 v16.8h, v18.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x0]\n"
+ "add v1.8h, v1.8h, v16.8h\n"
+ "add %x[out_ptr], %x[out_ptr], #0x10\n"
+ "beq 10f\n"
+ "zip2 v16.8h, v18.8h, v17.8h\n"
+ "subs x19, x19, #0x1\n"
+ "add v1.8h, v1.8h, v16.8h\n"
+ "str q16, [%x[out_ptr], #0x0]\n"
+ "add %x[out_ptr], %x[out_ptr], #0x10\n"
+ "beq 10f\n"
+ "zip2 v18.8h, v24.8h, v20.8h\n"
+ "zip2 v17.8h, v22.8h, v19.8h\n"
+ "subs x19, x19, #0x1\n"
+ "zip1 v16.8h, v18.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x0]\n"
+ "add v1.8h, v1.8h, v16.8h\n"
+ "add %x[out_ptr], %x[out_ptr], #0x10\n"
+ "beq 10f\n"
+ "zip2 v16.8h, v18.8h, v17.8h\n"
+ "subs x19, x19, #0x1\n"
+ "add v1.8h, v1.8h, v16.8h\n"
+ "str q16, [%x[out_ptr], #0x0]\n"
+ "add %x[out_ptr], %x[out_ptr], #0x10\n"
+ "beq 10f\n"
+ "zip2 v23.8h, v30.8h, v23.8h\n"
+ "zip2 v20.8h, v28.8h, v26.8h\n"
+ "subs x19, x19, #0x1\n"
+ "zip1 v18.8h, v23.8h, v20.8h\n"
+ "zip2 v21.8h, v29.8h, v21.8h\n"
+ "zip2 v19.8h, v27.8h, v25.8h\n"
+ "zip1 v17.8h, v21.8h, v19.8h\n"
+ "zip1 v16.8h, v18.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x0]\n"
+ "add v1.8h, v1.8h, v16.8h\n"
+ "add %x[out_ptr], %x[out_ptr], #0x10\n"
+ "beq 10f\n"
+ "zip2 v16.8h, v18.8h, v17.8h\n"
+ "subs x19, x19, #0x1\n"
+ "add v1.8h, v1.8h, v16.8h\n"
+ "str q16, [%x[out_ptr], #0x0]\n"
+ "add %x[out_ptr], %x[out_ptr], #0x10\n"
+ "beq 10f\n"
+ "zip2 v18.8h, v23.8h, v20.8h\n"
+ "zip2 v17.8h, v21.8h, v19.8h\n"
+ "zip1 v16.8h, v18.8h, v17.8h\n"
+ "str q16, [%x[out_ptr], #0x0]\n"
+ "add v1.8h, v1.8h, v16.8h\n"
+ "add %x[out_ptr], %x[out_ptr], #0x10\n"
+ "10:" // Odds skip
+ "uaddw v0.4s, v0.4s, v1.4h\n"
+ "str q0, [%x[out_ptr], #0x0]\n"
+ "uaddw2 v31.4s, v31.4s, v1.8h\n"
+ "str q31, [%x[out_ptr], #0x10]\n"
+ "add %x[out_ptr], %x[out_ptr], #0x20\n"
+ : [out_ptr] "+r" (out_ptr), [width] "+r" (width)
+ : [first] "r" (first), [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset)
+ : "cc", "memory", "v0", "v1", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27"
+ );
+}
+
+
+#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block2_bf16_bf16.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block2_bf16_bf16.hpp
new file mode 100644
index 0000000..5377edc
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block2_bf16_bf16.hpp
@@ -0,0 +1,247 @@
+/*
+ * Copyright (c) 2019-2020 Arm Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+#ifdef __aarch64__
+
+template<>
+void interleave_block<8, 2, VLType::None, false>(
+ bfloat16 * &out_ptr, const bfloat16 * const * in, size_t width, size_t height,
+ size_t row_offset, bool
+)
+{
+ __asm__ __volatile__(
+ "ldr x27, [%x[in], #0x0]\n"
+ "cmp %x[height], #0x8\n"
+ "ldr x26, [%x[in], #0x8]\n"
+ "add x27, x27, %x[row_offset], LSL #1\n"
+ "ldr x25, [%x[in], #0x10]\n"
+ "ldr x24, [%x[in], #0x18]\n"
+ "add x26, x26, %x[row_offset], LSL #1\n"
+ "ldr x23, [%x[in], #0x20]\n"
+ "add x25, x25, %x[row_offset], LSL #1\n"
+ "ldr x22, [%x[in], #0x28]\n"
+ "ldr x21, [%x[in], #0x30]\n"
+ "add x24, x24, %x[row_offset], LSL #1\n"
+ "ldr x20, [%x[in], #0x38]\n"
+ "add x23, x23, %x[row_offset], LSL #1\n"
+ "add x22, x22, %x[row_offset], LSL #1\n"
+ "add x21, x21, %x[row_offset], LSL #1\n"
+ "add x20, x20, %x[row_offset], LSL #1\n"
+ "beq 1f\n"
+ "mov x20, x27\n"
+ "cmp %x[height], #0x2\n"
+ "csel x26, x26, x27, GE\n"
+ "csel x25, x25, x27, GT\n"
+ "cmp %x[height], #0x4\n"
+ "csel x24, x24, x27, GE\n"
+ "csel x23, x23, x27, GT\n"
+ "cmp %x[height], #0x6\n"
+ "csel x22, x22, x27, GE\n"
+ "csel x21, x21, x27, GT\n"
+ "1:" // no_pointer_adj
+ "prfm pldl1keep, [x27, #0x0]\n"
+ "cmp %x[width], #0x8\n"
+ "prfm pldl1keep, [x26, #0x0]\n"
+ "prfm pldl1keep, [x25, #0x0]\n"
+ "prfm pldl1keep, [x24, #0x0]\n"
+ "prfm pldl1keep, [x23, #0x0]\n"
+ "prfm pldl1keep, [x22, #0x0]\n"
+ "prfm pldl1keep, [x21, #0x0]\n"
+ "prfm pldl1keep, [x20, #0x0]\n"
+ "prfm pldl1keep, [x27, #0x40]\n"
+ "prfm pldl1keep, [x26, #0x40]\n"
+ "prfm pldl1keep, [x25, #0x40]\n"
+ "prfm pldl1keep, [x24, #0x40]\n"
+ "prfm pldl1keep, [x23, #0x40]\n"
+ "prfm pldl1keep, [x22, #0x40]\n"
+ "prfm pldl1keep, [x21, #0x40]\n"
+ "prfm pldl1keep, [x20, #0x40]\n"
+ "blt 3f\n"
+ "2:" // Main loop head
+ "ldr q28, [x27], #0x10\n"
+ "prfm pldl1keep, [x27, #0x70]\n"
+ "ldr q27, [x26], #0x10\n"
+ "ldr q26, [x25], #0x10\n"
+ "zip1 v23.4s, v28.4s, v26.4s\n"
+ "prfm pldl1keep, [x26, #0x70]\n"
+ "ldr q22, [x24], #0x10\n"
+ "zip2 v26.4s, v28.4s, v26.4s\n"
+ "prfm pldl1keep, [x25, #0x70]\n"
+ "ldr q25, [x23], #0x10\n"
+ "zip1 v20.4s, v27.4s, v22.4s\n"
+ "prfm pldl1keep, [x24, #0x70]\n"
+ "ldr q24, [x22], #0x10\n"
+ "zip1 v16.4s, v23.4s, v20.4s\n"
+ "prfm pldl1keep, [x23, #0x70]\n"
+ "ldr q19, [x21], #0x10\n"
+ "zip2 v23.4s, v23.4s, v20.4s\n"
+ "prfm pldl1keep, [x22, #0x70]\n"
+ "zip2 v22.4s, v27.4s, v22.4s\n"
+ "ldr q21, [x20], #0x10\n"
+ "zip1 v18.4s, v25.4s, v19.4s\n"
+ "prfm pldl1keep, [x21, #0x70]\n"
+ "str q16, [%x[out_ptr], #0x0]\n"
+ "zip1 v20.4s, v26.4s, v22.4s\n"
+ "prfm pldl1keep, [x20, #0x70]\n"
+ "zip1 v16.4s, v24.4s, v21.4s\n"
+ "subs %x[width], %x[width], #0x8\n"
+ "zip1 v17.4s, v18.4s, v16.4s\n"
+ "cmp %x[width], #0x8\n"
+ "zip2 v16.4s, v18.4s, v16.4s\n"
+ "str q17, [%x[out_ptr], #0x10]\n"
+ "zip2 v19.4s, v25.4s, v19.4s\n"
+ "str q23, [%x[out_ptr], #0x20]\n"
+ "zip2 v18.4s, v24.4s, v21.4s\n"
+ "str q16, [%x[out_ptr], #0x30]\n"
+ "zip1 v16.4s, v19.4s, v18.4s\n"
+ "str q20, [%x[out_ptr], #0x40]\n"
+ "zip2 v17.4s, v26.4s, v22.4s\n"
+ "str q16, [%x[out_ptr], #0x50]\n"
+ "zip2 v16.4s, v19.4s, v18.4s\n"
+ "str q17, [%x[out_ptr], #0x60]\n"
+ "str q16, [%x[out_ptr], #0x70]\n"
+ "add %x[out_ptr], %x[out_ptr], #0x80\n"
+ "bge 2b\n"
+ "3:" // Main loop skip
+ "cbz %x[width], 8f\n"
+ "tbz %x[width], #2, 5f\n"
+ "ldr d28, [x27], #0x8\n"
+ "ldr d27, [x26], #0x8\n"
+ "ldr d26, [x25], #0x8\n"
+ "ldr d22, [x24], #0x8\n"
+ "ldr d25, [x23], #0x8\n"
+ "ldr d24, [x22], #0x8\n"
+ "ldr d19, [x21], #0x8\n"
+ "ldr d21, [x20], #0x8\n"
+ "tbz %x[width], #1, 4f\n"
+ "ld1 { v28.s }[2], [x27], #0x4\n"
+ "ld1 { v27.s }[2], [x26], #0x4\n"
+ "ld1 { v26.s }[2], [x25], #0x4\n"
+ "ld1 { v22.s }[2], [x24], #0x4\n"
+ "ld1 { v25.s }[2], [x23], #0x4\n"
+ "ld1 { v24.s }[2], [x22], #0x4\n"
+ "ld1 { v19.s }[2], [x21], #0x4\n"
+ "ld1 { v21.s }[2], [x20], #0x4\n"
+ "mov x19, #0x3\n"
+ "tbz %x[width], #0, 7f\n"
+ "ld1 { v28.h }[6], [x27]\n"
+ "ld1 { v27.h }[6], [x26]\n"
+ "ld1 { v26.h }[6], [x25]\n"
+ "ld1 { v22.h }[6], [x24]\n"
+ "ld1 { v25.h }[6], [x23]\n"
+ "ld1 { v24.h }[6], [x22]\n"
+ "ld1 { v19.h }[6], [x21]\n"
+ "ld1 { v21.h }[6], [x20]\n"
+ "mov x19, #0x4\n"
+ "b 7f\n"
+ "4:" // odd_loads_1_4
+ "mov x19, #0x2\n"
+ "tbz %x[width], #0, 7f\n"
+ "ld1 { v28.h }[4], [x27]\n"
+ "ld1 { v27.h }[4], [x26]\n"
+ "ld1 { v26.h }[4], [x25]\n"
+ "ld1 { v22.h }[4], [x24]\n"
+ "ld1 { v25.h }[4], [x23]\n"
+ "ld1 { v24.h }[4], [x22]\n"
+ "ld1 { v19.h }[4], [x21]\n"
+ "ld1 { v21.h }[4], [x20]\n"
+ "mov x19, #0x3\n"
+ "b 7f\n"
+ "5:" // odd_loads_2_0
+ "tbz %x[width], #1, 6f\n"
+ "ldr s28, [x27], #0x4\n"
+ "ldr s27, [x26], #0x4\n"
+ "ldr s26, [x25], #0x4\n"
+ "ldr s22, [x24], #0x4\n"
+ "ldr s25, [x23], #0x4\n"
+ "ldr s24, [x22], #0x4\n"
+ "ldr s19, [x21], #0x4\n"
+ "ldr s21, [x20], #0x4\n"
+ "mov x19, #0x1\n"
+ "tbz %x[width], #0, 7f\n"
+ "ld1 { v28.h }[2], [x27]\n"
+ "ld1 { v27.h }[2], [x26]\n"
+ "ld1 { v26.h }[2], [x25]\n"
+ "ld1 { v22.h }[2], [x24]\n"
+ "ld1 { v25.h }[2], [x23]\n"
+ "ld1 { v24.h }[2], [x22]\n"
+ "ld1 { v19.h }[2], [x21]\n"
+ "ld1 { v21.h }[2], [x20]\n"
+ "mov x19, #0x2\n"
+ "b 7f\n"
+ "6:" // odd_loads_1_0
+ "ldr h28, [x27, #0x0]\n"
+ "ldr h27, [x26, #0x0]\n"
+ "ldr h26, [x25, #0x0]\n"
+ "ldr h22, [x24, #0x0]\n"
+ "ldr h25, [x23, #0x0]\n"
+ "ldr h24, [x22, #0x0]\n"
+ "ldr h19, [x21, #0x0]\n"
+ "ldr h21, [x20, #0x0]\n"
+ "mov x19, #0x1\n"
+ "7:" // Odd load end
+ "zip1 v23.4s, v28.4s, v26.4s\n"
+ "subs x19, x19, #0x1\n"
+ "zip1 v20.4s, v27.4s, v22.4s\n"
+ "zip1 v16.4s, v23.4s, v20.4s\n"
+ "str q16, [%x[out_ptr], #0x0]\n"
+ "zip1 v18.4s, v25.4s, v19.4s\n"
+ "zip1 v16.4s, v24.4s, v21.4s\n"
+ "zip1 v17.4s, v18.4s, v16.4s\n"
+ "str q17, [%x[out_ptr], #0x10]\n"
+ "add %x[out_ptr], %x[out_ptr], #0x20\n"
+ "beq 8f\n"
+ "zip2 v23.4s, v23.4s, v20.4s\n"
+ "zip2 v16.4s, v18.4s, v16.4s\n"
+ "str q23, [%x[out_ptr], #0x0]\n"
+ "str q16, [%x[out_ptr], #0x10]\n"
+ "subs x19, x19, #0x1\n"
+ "add %x[out_ptr], %x[out_ptr], #0x20\n"
+ "beq 8f\n"
+ "zip2 v26.4s, v28.4s, v26.4s\n"
+ "zip2 v22.4s, v27.4s, v22.4s\n"
+ "subs x19, x19, #0x1\n"
+ "zip1 v20.4s, v26.4s, v22.4s\n"
+ "str q20, [%x[out_ptr], #0x0]\n"
+ "zip2 v19.4s, v25.4s, v19.4s\n"
+ "zip2 v18.4s, v24.4s, v21.4s\n"
+ "zip1 v16.4s, v19.4s, v18.4s\n"
+ "str q16, [%x[out_ptr], #0x10]\n"
+ "add %x[out_ptr], %x[out_ptr], #0x20\n"
+ "beq 8f\n"
+ "zip2 v17.4s, v26.4s, v22.4s\n"
+ "zip2 v16.4s, v19.4s, v18.4s\n"
+ "str q17, [%x[out_ptr], #0x0]\n"
+ "str q16, [%x[out_ptr], #0x10]\n"
+ "add %x[out_ptr], %x[out_ptr], #0x20\n"
+ "8:" // Odds skip
+
+ : [out_ptr] "+r" (out_ptr), [width] "+r" (width)
+ : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset)
+ : "cc", "memory", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27"
+ );
+}
+
+
+#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block2_fp32_fp32.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block2_fp32_fp32.hpp
new file mode 100644
index 0000000..3aea6a8
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block2_fp32_fp32.hpp
@@ -0,0 +1,181 @@
+/*
+ * Copyright (c) 2019-2020 Arm Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+#ifdef __aarch64__
+
+template<>
+void interleave_block<8, 2, VLType::None, false>(
+ float * &out_ptr, const float * const * in, size_t width, size_t height,
+ size_t row_offset, bool
+)
+{
+ __asm__ __volatile__(
+ "ldr x27, [%x[in], #0x0]\n"
+ "cmp %x[height], #0x8\n"
+ "ldr x26, [%x[in], #0x8]\n"
+ "add x27, x27, %x[row_offset], LSL #2\n"
+ "ldr x25, [%x[in], #0x10]\n"
+ "ldr x24, [%x[in], #0x18]\n"
+ "add x26, x26, %x[row_offset], LSL #2\n"
+ "ldr x23, [%x[in], #0x20]\n"
+ "add x25, x25, %x[row_offset], LSL #2\n"
+ "ldr x22, [%x[in], #0x28]\n"
+ "ldr x21, [%x[in], #0x30]\n"
+ "add x24, x24, %x[row_offset], LSL #2\n"
+ "ldr x20, [%x[in], #0x38]\n"
+ "add x23, x23, %x[row_offset], LSL #2\n"
+ "add x22, x22, %x[row_offset], LSL #2\n"
+ "add x21, x21, %x[row_offset], LSL #2\n"
+ "add x20, x20, %x[row_offset], LSL #2\n"
+ "beq 1f\n"
+ "mov x20, x27\n"
+ "cmp %x[height], #0x2\n"
+ "csel x26, x26, x27, GE\n"
+ "csel x25, x25, x27, GT\n"
+ "cmp %x[height], #0x4\n"
+ "csel x24, x24, x27, GE\n"
+ "csel x23, x23, x27, GT\n"
+ "cmp %x[height], #0x6\n"
+ "csel x22, x22, x27, GE\n"
+ "csel x21, x21, x27, GT\n"
+ "1:" // no_pointer_adj
+ "prfm pldl1keep, [x27, #0x0]\n"
+ "cmp %x[width], #0x4\n"
+ "prfm pldl1keep, [x26, #0x0]\n"
+ "prfm pldl1keep, [x25, #0x0]\n"
+ "prfm pldl1keep, [x24, #0x0]\n"
+ "prfm pldl1keep, [x23, #0x0]\n"
+ "prfm pldl1keep, [x22, #0x0]\n"
+ "prfm pldl1keep, [x21, #0x0]\n"
+ "prfm pldl1keep, [x20, #0x0]\n"
+ "prfm pldl1keep, [x27, #0x40]\n"
+ "prfm pldl1keep, [x26, #0x40]\n"
+ "prfm pldl1keep, [x25, #0x40]\n"
+ "prfm pldl1keep, [x24, #0x40]\n"
+ "prfm pldl1keep, [x23, #0x40]\n"
+ "prfm pldl1keep, [x22, #0x40]\n"
+ "prfm pldl1keep, [x21, #0x40]\n"
+ "prfm pldl1keep, [x20, #0x40]\n"
+ "blt 3f\n"
+ "2:" // Main loop head
+ "ldr q27, [x27], #0x10\n"
+ "prfm pldl1keep, [x27, #0x70]\n"
+ "ldr q24, [x26], #0x10\n"
+ "zip1 v26.2d, v27.2d, v24.2d\n"
+ "prfm pldl1keep, [x26, #0x70]\n"
+ "ldr q25, [x25], #0x10\n"
+ "zip2 v24.2d, v27.2d, v24.2d\n"
+ "prfm pldl1keep, [x25, #0x70]\n"
+ "ldr q21, [x24], #0x10\n"
+ "zip1 v23.2d, v25.2d, v21.2d\n"
+ "prfm pldl1keep, [x24, #0x70]\n"
+ "ldr q22, [x23], #0x10\n"
+ "zip2 v21.2d, v25.2d, v21.2d\n"
+ "prfm pldl1keep, [x23, #0x70]\n"
+ "ldr q18, [x22], #0x10\n"
+ "zip1 v20.2d, v22.2d, v18.2d\n"
+ "prfm pldl1keep, [x22, #0x70]\n"
+ "ldr q19, [x21], #0x10\n"
+ "zip2 v18.2d, v22.2d, v18.2d\n"
+ "prfm pldl1keep, [x21, #0x70]\n"
+ "ldr q16, [x20], #0x10\n"
+ "zip1 v17.2d, v19.2d, v16.2d\n"
+ "prfm pldl1keep, [x20, #0x70]\n"
+ "str q26, [%x[out_ptr], #0x0]\n"
+ "zip2 v16.2d, v19.2d, v16.2d\n"
+ "str q23, [%x[out_ptr], #0x10]\n"
+ "str q20, [%x[out_ptr], #0x20]\n"
+ "str q17, [%x[out_ptr], #0x30]\n"
+ "str q24, [%x[out_ptr], #0x40]\n"
+ "str q21, [%x[out_ptr], #0x50]\n"
+ "str q18, [%x[out_ptr], #0x60]\n"
+ "str q16, [%x[out_ptr], #0x70]\n"
+ "subs %x[width], %x[width], #0x4\n"
+ "cmp %x[width], #0x4\n"
+ "add %x[out_ptr], %x[out_ptr], #0x80\n"
+ "bge 2b\n"
+ "3:" // Main loop skip
+ "cbz %x[width], 6f\n"
+ "tbz %x[width], #1, 4f\n"
+ "ldr d27, [x27], #0x8\n"
+ "ldr d24, [x26], #0x8\n"
+ "ldr d25, [x25], #0x8\n"
+ "ldr d21, [x24], #0x8\n"
+ "ldr d22, [x23], #0x8\n"
+ "ldr d18, [x22], #0x8\n"
+ "ldr d19, [x21], #0x8\n"
+ "ldr d16, [x20], #0x8\n"
+ "mov x19, #0x1\n"
+ "tbz %x[width], #0, 5f\n"
+ "ld1 { v27.s }[2], [x27]\n"
+ "ld1 { v24.s }[2], [x26]\n"
+ "ld1 { v25.s }[2], [x25]\n"
+ "ld1 { v21.s }[2], [x24]\n"
+ "ld1 { v22.s }[2], [x23]\n"
+ "ld1 { v18.s }[2], [x22]\n"
+ "ld1 { v19.s }[2], [x21]\n"
+ "ld1 { v16.s }[2], [x20]\n"
+ "mov x19, #0x2\n"
+ "b 5f\n"
+ "4:" // odd_loads_1_0
+ "ldr s27, [x27, #0x0]\n"
+ "ldr s24, [x26, #0x0]\n"
+ "ldr s25, [x25, #0x0]\n"
+ "ldr s21, [x24, #0x0]\n"
+ "ldr s22, [x23, #0x0]\n"
+ "ldr s18, [x22, #0x0]\n"
+ "ldr s19, [x21, #0x0]\n"
+ "ldr s16, [x20, #0x0]\n"
+ "mov x19, #0x1\n"
+ "5:" // Odd load end
+ "zip1 v26.2d, v27.2d, v24.2d\n"
+ "subs x19, x19, #0x1\n"
+ "zip1 v23.2d, v25.2d, v21.2d\n"
+ "str q26, [%x[out_ptr], #0x0]\n"
+ "zip1 v20.2d, v22.2d, v18.2d\n"
+ "str q23, [%x[out_ptr], #0x10]\n"
+ "zip1 v17.2d, v19.2d, v16.2d\n"
+ "str q20, [%x[out_ptr], #0x20]\n"
+ "str q17, [%x[out_ptr], #0x30]\n"
+ "add %x[out_ptr], %x[out_ptr], #0x40\n"
+ "beq 6f\n"
+ "zip2 v24.2d, v27.2d, v24.2d\n"
+ "zip2 v21.2d, v25.2d, v21.2d\n"
+ "str q24, [%x[out_ptr], #0x0]\n"
+ "zip2 v18.2d, v22.2d, v18.2d\n"
+ "str q21, [%x[out_ptr], #0x10]\n"
+ "zip2 v16.2d, v19.2d, v16.2d\n"
+ "str q18, [%x[out_ptr], #0x20]\n"
+ "str q16, [%x[out_ptr], #0x30]\n"
+ "add %x[out_ptr], %x[out_ptr], #0x40\n"
+ "6:" // Odds skip
+
+ : [out_ptr] "+r" (out_ptr), [width] "+r" (width)
+ : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset)
+ : "cc", "memory", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27"
+ );
+}
+
+
+#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block4_bf16_bf16.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block4_bf16_bf16.hpp
new file mode 100644
index 0000000..4780b77
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block4_bf16_bf16.hpp
@@ -0,0 +1,223 @@
+/*
+ * Copyright (c) 2019-2020 Arm Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+#ifdef __aarch64__
+
+template<>
+void interleave_block<8, 4, VLType::None, false>(
+ bfloat16 * &out_ptr, const bfloat16 * const * in, size_t width, size_t height,
+ size_t row_offset, bool
+)
+{
+ __asm__ __volatile__(
+ "ldr x27, [%x[in], #0x0]\n"
+ "cmp %x[height], #0x8\n"
+ "ldr x26, [%x[in], #0x8]\n"
+ "add x27, x27, %x[row_offset], LSL #1\n"
+ "ldr x25, [%x[in], #0x10]\n"
+ "ldr x24, [%x[in], #0x18]\n"
+ "add x26, x26, %x[row_offset], LSL #1\n"
+ "ldr x23, [%x[in], #0x20]\n"
+ "add x25, x25, %x[row_offset], LSL #1\n"
+ "ldr x22, [%x[in], #0x28]\n"
+ "ldr x21, [%x[in], #0x30]\n"
+ "add x24, x24, %x[row_offset], LSL #1\n"
+ "ldr x20, [%x[in], #0x38]\n"
+ "add x23, x23, %x[row_offset], LSL #1\n"
+ "add x22, x22, %x[row_offset], LSL #1\n"
+ "add x21, x21, %x[row_offset], LSL #1\n"
+ "add x20, x20, %x[row_offset], LSL #1\n"
+ "beq 1f\n"
+ "mov x20, x27\n"
+ "cmp %x[height], #0x2\n"
+ "csel x26, x26, x27, GE\n"
+ "csel x25, x25, x27, GT\n"
+ "cmp %x[height], #0x4\n"
+ "csel x24, x24, x27, GE\n"
+ "csel x23, x23, x27, GT\n"
+ "cmp %x[height], #0x6\n"
+ "csel x22, x22, x27, GE\n"
+ "csel x21, x21, x27, GT\n"
+ "1:" // no_pointer_adj
+ "prfm pldl1keep, [x27, #0x0]\n"
+ "cmp %x[width], #0x8\n"
+ "prfm pldl1keep, [x26, #0x0]\n"
+ "prfm pldl1keep, [x25, #0x0]\n"
+ "prfm pldl1keep, [x24, #0x0]\n"
+ "prfm pldl1keep, [x23, #0x0]\n"
+ "prfm pldl1keep, [x22, #0x0]\n"
+ "prfm pldl1keep, [x21, #0x0]\n"
+ "prfm pldl1keep, [x20, #0x0]\n"
+ "prfm pldl1keep, [x27, #0x40]\n"
+ "prfm pldl1keep, [x26, #0x40]\n"
+ "prfm pldl1keep, [x25, #0x40]\n"
+ "prfm pldl1keep, [x24, #0x40]\n"
+ "prfm pldl1keep, [x23, #0x40]\n"
+ "prfm pldl1keep, [x22, #0x40]\n"
+ "prfm pldl1keep, [x21, #0x40]\n"
+ "prfm pldl1keep, [x20, #0x40]\n"
+ "blt 3f\n"
+ "2:" // Main loop head
+ "ldr q27, [x27], #0x10\n"
+ "prfm pldl1keep, [x27, #0x70]\n"
+ "ldr q24, [x26], #0x10\n"
+ "zip1 v26.2d, v27.2d, v24.2d\n"
+ "prfm pldl1keep, [x26, #0x70]\n"
+ "ldr q25, [x25], #0x10\n"
+ "zip2 v24.2d, v27.2d, v24.2d\n"
+ "prfm pldl1keep, [x25, #0x70]\n"
+ "ldr q21, [x24], #0x10\n"
+ "zip1 v23.2d, v25.2d, v21.2d\n"
+ "prfm pldl1keep, [x24, #0x70]\n"
+ "ldr q22, [x23], #0x10\n"
+ "zip2 v21.2d, v25.2d, v21.2d\n"
+ "prfm pldl1keep, [x23, #0x70]\n"
+ "ldr q18, [x22], #0x10\n"
+ "zip1 v20.2d, v22.2d, v18.2d\n"
+ "prfm pldl1keep, [x22, #0x70]\n"
+ "ldr q19, [x21], #0x10\n"
+ "zip2 v18.2d, v22.2d, v18.2d\n"
+ "prfm pldl1keep, [x21, #0x70]\n"
+ "ldr q16, [x20], #0x10\n"
+ "zip1 v17.2d, v19.2d, v16.2d\n"
+ "prfm pldl1keep, [x20, #0x70]\n"
+ "str q26, [%x[out_ptr], #0x0]\n"
+ "zip2 v16.2d, v19.2d, v16.2d\n"
+ "str q23, [%x[out_ptr], #0x10]\n"
+ "str q20, [%x[out_ptr], #0x20]\n"
+ "str q17, [%x[out_ptr], #0x30]\n"
+ "str q24, [%x[out_ptr], #0x40]\n"
+ "str q21, [%x[out_ptr], #0x50]\n"
+ "str q18, [%x[out_ptr], #0x60]\n"
+ "str q16, [%x[out_ptr], #0x70]\n"
+ "subs %x[width], %x[width], #0x8\n"
+ "cmp %x[width], #0x8\n"
+ "add %x[out_ptr], %x[out_ptr], #0x80\n"
+ "bge 2b\n"
+ "3:" // Main loop skip
+ "cbz %x[width], 8f\n"
+ "tbz %x[width], #2, 5f\n"
+ "ldr d27, [x27], #0x8\n"
+ "ldr d24, [x26], #0x8\n"
+ "ldr d25, [x25], #0x8\n"
+ "ldr d21, [x24], #0x8\n"
+ "ldr d22, [x23], #0x8\n"
+ "ldr d18, [x22], #0x8\n"
+ "ldr d19, [x21], #0x8\n"
+ "ldr d16, [x20], #0x8\n"
+ "tbz %x[width], #1, 4f\n"
+ "ld1 { v27.s }[2], [x27], #0x4\n"
+ "ld1 { v24.s }[2], [x26], #0x4\n"
+ "ld1 { v25.s }[2], [x25], #0x4\n"
+ "ld1 { v21.s }[2], [x24], #0x4\n"
+ "ld1 { v22.s }[2], [x23], #0x4\n"
+ "ld1 { v18.s }[2], [x22], #0x4\n"
+ "ld1 { v19.s }[2], [x21], #0x4\n"
+ "ld1 { v16.s }[2], [x20], #0x4\n"
+ "mov x19, #0x2\n"
+ "tbz %x[width], #0, 7f\n"
+ "ld1 { v27.h }[6], [x27]\n"
+ "ld1 { v24.h }[6], [x26]\n"
+ "ld1 { v25.h }[6], [x25]\n"
+ "ld1 { v21.h }[6], [x24]\n"
+ "ld1 { v22.h }[6], [x23]\n"
+ "ld1 { v18.h }[6], [x22]\n"
+ "ld1 { v19.h }[6], [x21]\n"
+ "ld1 { v16.h }[6], [x20]\n"
+ "b 7f\n"
+ "4:" // odd_loads_1_4
+ "mov x19, #0x1\n"
+ "tbz %x[width], #0, 7f\n"
+ "ld1 { v27.h }[4], [x27]\n"
+ "ld1 { v24.h }[4], [x26]\n"
+ "ld1 { v25.h }[4], [x25]\n"
+ "ld1 { v21.h }[4], [x24]\n"
+ "ld1 { v22.h }[4], [x23]\n"
+ "ld1 { v18.h }[4], [x22]\n"
+ "ld1 { v19.h }[4], [x21]\n"
+ "ld1 { v16.h }[4], [x20]\n"
+ "mov x19, #0x2\n"
+ "b 7f\n"
+ "5:" // odd_loads_2_0
+ "tbz %x[width], #1, 6f\n"
+ "ldr s27, [x27], #0x4\n"
+ "ldr s24, [x26], #0x4\n"
+ "ldr s25, [x25], #0x4\n"
+ "ldr s21, [x24], #0x4\n"
+ "ldr s22, [x23], #0x4\n"
+ "ldr s18, [x22], #0x4\n"
+ "ldr s19, [x21], #0x4\n"
+ "ldr s16, [x20], #0x4\n"
+ "mov x19, #0x1\n"
+ "tbz %x[width], #0, 7f\n"
+ "ld1 { v27.h }[2], [x27]\n"
+ "ld1 { v24.h }[2], [x26]\n"
+ "ld1 { v25.h }[2], [x25]\n"
+ "ld1 { v21.h }[2], [x24]\n"
+ "ld1 { v22.h }[2], [x23]\n"
+ "ld1 { v18.h }[2], [x22]\n"
+ "ld1 { v19.h }[2], [x21]\n"
+ "ld1 { v16.h }[2], [x20]\n"
+ "b 7f\n"
+ "6:" // odd_loads_1_0
+ "ldr h27, [x27, #0x0]\n"
+ "ldr h24, [x26, #0x0]\n"
+ "ldr h25, [x25, #0x0]\n"
+ "ldr h21, [x24, #0x0]\n"
+ "ldr h22, [x23, #0x0]\n"
+ "ldr h18, [x22, #0x0]\n"
+ "ldr h19, [x21, #0x0]\n"
+ "ldr h16, [x20, #0x0]\n"
+ "mov x19, #0x1\n"
+ "7:" // Odd load end
+ "zip1 v26.2d, v27.2d, v24.2d\n"
+ "subs x19, x19, #0x1\n"
+ "zip1 v23.2d, v25.2d, v21.2d\n"
+ "str q26, [%x[out_ptr], #0x0]\n"
+ "zip1 v20.2d, v22.2d, v18.2d\n"
+ "str q23, [%x[out_ptr], #0x10]\n"
+ "zip1 v17.2d, v19.2d, v16.2d\n"
+ "str q20, [%x[out_ptr], #0x20]\n"
+ "str q17, [%x[out_ptr], #0x30]\n"
+ "add %x[out_ptr], %x[out_ptr], #0x40\n"
+ "beq 8f\n"
+ "zip2 v24.2d, v27.2d, v24.2d\n"
+ "zip2 v21.2d, v25.2d, v21.2d\n"
+ "str q24, [%x[out_ptr], #0x0]\n"
+ "zip2 v18.2d, v22.2d, v18.2d\n"
+ "str q21, [%x[out_ptr], #0x10]\n"
+ "zip2 v16.2d, v19.2d, v16.2d\n"
+ "str q18, [%x[out_ptr], #0x20]\n"
+ "str q16, [%x[out_ptr], #0x30]\n"
+ "add %x[out_ptr], %x[out_ptr], #0x40\n"
+ "8:" // Odds skip
+
+ : [out_ptr] "+r" (out_ptr), [width] "+r" (width)
+ : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset)
+ : "cc", "memory", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27"
+ );
+}
+
+
+#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block4_s8_s8.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block4_s8_s8.hpp
new file mode 100644
index 0000000..a9034f5
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block4_s8_s8.hpp
@@ -0,0 +1,343 @@
+/*
+ * Copyright (c) 2019-2020 Arm Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+#ifdef __aarch64__
+
+template<>
+void interleave_block<8, 4, VLType::None, false>(
+ int8_t * &out_ptr, const int8_t * const * in, size_t width, size_t height,
+ size_t row_offset, bool
+)
+{
+ __asm__ __volatile__(
+ "ldr x27, [%x[in], #0x0]\n"
+ "cmp %x[height], #0x8\n"
+ "ldr x26, [%x[in], #0x8]\n"
+ "add x27, x27, %x[row_offset]\n"
+ "ldr x25, [%x[in], #0x10]\n"
+ "ldr x24, [%x[in], #0x18]\n"
+ "add x26, x26, %x[row_offset]\n"
+ "ldr x23, [%x[in], #0x20]\n"
+ "add x25, x25, %x[row_offset]\n"
+ "ldr x22, [%x[in], #0x28]\n"
+ "ldr x21, [%x[in], #0x30]\n"
+ "add x24, x24, %x[row_offset]\n"
+ "ldr x20, [%x[in], #0x38]\n"
+ "add x23, x23, %x[row_offset]\n"
+ "add x22, x22, %x[row_offset]\n"
+ "add x21, x21, %x[row_offset]\n"
+ "add x20, x20, %x[row_offset]\n"
+ "beq 1f\n"
+ "mov x20, x27\n"
+ "cmp %x[height], #0x2\n"
+ "csel x26, x26, x27, GE\n"
+ "csel x25, x25, x27, GT\n"
+ "cmp %x[height], #0x4\n"
+ "csel x24, x24, x27, GE\n"
+ "csel x23, x23, x27, GT\n"
+ "cmp %x[height], #0x6\n"
+ "csel x22, x22, x27, GE\n"
+ "csel x21, x21, x27, GT\n"
+ "1:" // no_pointer_adj
+ "prfm pldl1keep, [x27, #0x0]\n"
+ "cmp %x[width], #0x10\n"
+ "prfm pldl1keep, [x26, #0x0]\n"
+ "prfm pldl1keep, [x25, #0x0]\n"
+ "prfm pldl1keep, [x24, #0x0]\n"
+ "prfm pldl1keep, [x23, #0x0]\n"
+ "prfm pldl1keep, [x22, #0x0]\n"
+ "prfm pldl1keep, [x21, #0x0]\n"
+ "prfm pldl1keep, [x20, #0x0]\n"
+ "prfm pldl1keep, [x27, #0x40]\n"
+ "prfm pldl1keep, [x26, #0x40]\n"
+ "prfm pldl1keep, [x25, #0x40]\n"
+ "prfm pldl1keep, [x24, #0x40]\n"
+ "prfm pldl1keep, [x23, #0x40]\n"
+ "prfm pldl1keep, [x22, #0x40]\n"
+ "prfm pldl1keep, [x21, #0x40]\n"
+ "prfm pldl1keep, [x20, #0x40]\n"
+ "blt 3f\n"
+ "2:" // Main loop head
+ "ldr q28, [x27], #0x10\n"
+ "prfm pldl1keep, [x27, #0x70]\n"
+ "ldr q27, [x26], #0x10\n"
+ "ldr q26, [x25], #0x10\n"
+ "zip1 v23.4s, v28.4s, v26.4s\n"
+ "prfm pldl1keep, [x26, #0x70]\n"
+ "ldr q22, [x24], #0x10\n"
+ "zip2 v26.4s, v28.4s, v26.4s\n"
+ "prfm pldl1keep, [x25, #0x70]\n"
+ "ldr q25, [x23], #0x10\n"
+ "zip1 v20.4s, v27.4s, v22.4s\n"
+ "prfm pldl1keep, [x24, #0x70]\n"
+ "ldr q24, [x22], #0x10\n"
+ "zip1 v16.4s, v23.4s, v20.4s\n"
+ "prfm pldl1keep, [x23, #0x70]\n"
+ "ldr q19, [x21], #0x10\n"
+ "zip2 v23.4s, v23.4s, v20.4s\n"
+ "prfm pldl1keep, [x22, #0x70]\n"
+ "zip2 v22.4s, v27.4s, v22.4s\n"
+ "ldr q21, [x20], #0x10\n"
+ "zip1 v18.4s, v25.4s, v19.4s\n"
+ "prfm pldl1keep, [x21, #0x70]\n"
+ "str q16, [%x[out_ptr], #0x0]\n"
+ "zip1 v20.4s, v26.4s, v22.4s\n"
+ "prfm pldl1keep, [x20, #0x70]\n"
+ "zip1 v16.4s, v24.4s, v21.4s\n"
+ "subs %x[width], %x[width], #0x10\n"
+ "zip1 v17.4s, v18.4s, v16.4s\n"
+ "cmp %x[width], #0x10\n"
+ "zip2 v16.4s, v18.4s, v16.4s\n"
+ "str q17, [%x[out_ptr], #0x10]\n"
+ "zip2 v19.4s, v25.4s, v19.4s\n"
+ "str q23, [%x[out_ptr], #0x20]\n"
+ "zip2 v18.4s, v24.4s, v21.4s\n"
+ "str q16, [%x[out_ptr], #0x30]\n"
+ "zip1 v16.4s, v19.4s, v18.4s\n"
+ "str q20, [%x[out_ptr], #0x40]\n"
+ "zip2 v17.4s, v26.4s, v22.4s\n"
+ "str q16, [%x[out_ptr], #0x50]\n"
+ "zip2 v16.4s, v19.4s, v18.4s\n"
+ "str q17, [%x[out_ptr], #0x60]\n"
+ "str q16, [%x[out_ptr], #0x70]\n"
+ "add %x[out_ptr], %x[out_ptr], #0x80\n"
+ "bge 2b\n"
+ "3:" // Main loop skip
+ "cbz %x[width], 12f\n"
+ "tbz %x[width], #3, 7f\n"
+ "ldr d28, [x27], #0x8\n"
+ "ldr d27, [x26], #0x8\n"
+ "ldr d26, [x25], #0x8\n"
+ "ldr d22, [x24], #0x8\n"
+ "ldr d25, [x23], #0x8\n"
+ "ldr d24, [x22], #0x8\n"
+ "ldr d19, [x21], #0x8\n"
+ "ldr d21, [x20], #0x8\n"
+ "tbz %x[width], #2, 5f\n"
+ "ld1 { v28.s }[2], [x27], #0x4\n"
+ "ld1 { v27.s }[2], [x26], #0x4\n"
+ "ld1 { v26.s }[2], [x25], #0x4\n"
+ "ld1 { v22.s }[2], [x24], #0x4\n"
+ "ld1 { v25.s }[2], [x23], #0x4\n"
+ "ld1 { v24.s }[2], [x22], #0x4\n"
+ "ld1 { v19.s }[2], [x21], #0x4\n"
+ "ld1 { v21.s }[2], [x20], #0x4\n"
+ "tbz %x[width], #1, 4f\n"
+ "ld1 { v28.h }[6], [x27], #0x2\n"
+ "ld1 { v27.h }[6], [x26], #0x2\n"
+ "ld1 { v26.h }[6], [x25], #0x2\n"
+ "ld1 { v22.h }[6], [x24], #0x2\n"
+ "ld1 { v25.h }[6], [x23], #0x2\n"
+ "ld1 { v24.h }[6], [x22], #0x2\n"
+ "ld1 { v19.h }[6], [x21], #0x2\n"
+ "ld1 { v21.h }[6], [x20], #0x2\n"
+ "mov x19, #0x4\n"
+ "tbz %x[width], #0, 11f\n"
+ "ld1 { v28.b }[14], [x27]\n"
+ "ld1 { v27.b }[14], [x26]\n"
+ "ld1 { v26.b }[14], [x25]\n"
+ "ld1 { v22.b }[14], [x24]\n"
+ "ld1 { v25.b }[14], [x23]\n"
+ "ld1 { v24.b }[14], [x22]\n"
+ "ld1 { v19.b }[14], [x21]\n"
+ "ld1 { v21.b }[14], [x20]\n"
+ "b 11f\n"
+ "4:" // odd_loads_1_12
+ "mov x19, #0x3\n"
+ "tbz %x[width], #0, 11f\n"
+ "ld1 { v28.b }[12], [x27]\n"
+ "ld1 { v27.b }[12], [x26]\n"
+ "ld1 { v26.b }[12], [x25]\n"
+ "ld1 { v22.b }[12], [x24]\n"
+ "ld1 { v25.b }[12], [x23]\n"
+ "ld1 { v24.b }[12], [x22]\n"
+ "ld1 { v19.b }[12], [x21]\n"
+ "ld1 { v21.b }[12], [x20]\n"
+ "mov x19, #0x4\n"
+ "b 11f\n"
+ "5:" // odd_loads_2_8
+ "tbz %x[width], #1, 6f\n"
+ "ld1 { v28.h }[4], [x27], #0x2\n"
+ "ld1 { v27.h }[4], [x26], #0x2\n"
+ "ld1 { v26.h }[4], [x25], #0x2\n"
+ "ld1 { v22.h }[4], [x24], #0x2\n"
+ "ld1 { v25.h }[4], [x23], #0x2\n"
+ "ld1 { v24.h }[4], [x22], #0x2\n"
+ "ld1 { v19.h }[4], [x21], #0x2\n"
+ "ld1 { v21.h }[4], [x20], #0x2\n"
+ "mov x19, #0x3\n"
+ "tbz %x[width], #0, 11f\n"
+ "ld1 { v28.b }[10], [x27]\n"
+ "ld1 { v27.b }[10], [x26]\n"
+ "ld1 { v26.b }[10], [x25]\n"
+ "ld1 { v22.b }[10], [x24]\n"
+ "ld1 { v25.b }[10], [x23]\n"
+ "ld1 { v24.b }[10], [x22]\n"
+ "ld1 { v19.b }[10], [x21]\n"
+ "ld1 { v21.b }[10], [x20]\n"
+ "b 11f\n"
+ "6:" // odd_loads_1_8
+ "mov x19, #0x2\n"
+ "tbz %x[width], #0, 11f\n"
+ "ld1 { v28.b }[8], [x27]\n"
+ "ld1 { v27.b }[8], [x26]\n"
+ "ld1 { v26.b }[8], [x25]\n"
+ "ld1 { v22.b }[8], [x24]\n"
+ "ld1 { v25.b }[8], [x23]\n"
+ "ld1 { v24.b }[8], [x22]\n"
+ "ld1 { v19.b }[8], [x21]\n"
+ "ld1 { v21.b }[8], [x20]\n"
+ "mov x19, #0x3\n"
+ "b 11f\n"
+ "7:" // odd_loads_4_0
+ "tbz %x[width], #2, 9f\n"
+ "ldr s28, [x27], #0x4\n"
+ "ldr s27, [x26], #0x4\n"
+ "ldr s26, [x25], #0x4\n"
+ "ldr s22, [x24], #0x4\n"
+ "ldr s25, [x23], #0x4\n"
+ "ldr s24, [x22], #0x4\n"
+ "ldr s19, [x21], #0x4\n"
+ "ldr s21, [x20], #0x4\n"
+ "tbz %x[width], #1, 8f\n"
+ "ld1 { v28.h }[2], [x27], #0x2\n"
+ "ld1 { v27.h }[2], [x26], #0x2\n"
+ "ld1 { v26.h }[2], [x25], #0x2\n"
+ "ld1 { v22.h }[2], [x24], #0x2\n"
+ "ld1 { v25.h }[2], [x23], #0x2\n"
+ "ld1 { v24.h }[2], [x22], #0x2\n"
+ "ld1 { v19.h }[2], [x21], #0x2\n"
+ "ld1 { v21.h }[2], [x20], #0x2\n"
+ "mov x19, #0x2\n"
+ "tbz %x[width], #0, 11f\n"
+ "ld1 { v28.b }[6], [x27]\n"
+ "ld1 { v27.b }[6], [x26]\n"
+ "ld1 { v26.b }[6], [x25]\n"
+ "ld1 { v22.b }[6], [x24]\n"
+ "ld1 { v25.b }[6], [x23]\n"
+ "ld1 { v24.b }[6], [x22]\n"
+ "ld1 { v19.b }[6], [x21]\n"
+ "ld1 { v21.b }[6], [x20]\n"
+ "b 11f\n"
+ "8:" // odd_loads_1_4
+ "mov x19, #0x1\n"
+ "tbz %x[width], #0, 11f\n"
+ "ld1 { v28.b }[4], [x27]\n"
+ "ld1 { v27.b }[4], [x26]\n"
+ "ld1 { v26.b }[4], [x25]\n"
+ "ld1 { v22.b }[4], [x24]\n"
+ "ld1 { v25.b }[4], [x23]\n"
+ "ld1 { v24.b }[4], [x22]\n"
+ "ld1 { v19.b }[4], [x21]\n"
+ "ld1 { v21.b }[4], [x20]\n"
+ "mov x19, #0x2\n"
+ "b 11f\n"
+ "9:" // odd_loads_2_0
+ "tbz %x[width], #1, 10f\n"
+ "ldr h28, [x27], #0x2\n"
+ "ldr h27, [x26], #0x2\n"
+ "ldr h26, [x25], #0x2\n"
+ "ldr h22, [x24], #0x2\n"
+ "ldr h25, [x23], #0x2\n"
+ "ldr h24, [x22], #0x2\n"
+ "ldr h19, [x21], #0x2\n"
+ "ldr h21, [x20], #0x2\n"
+ "mov x19, #0x1\n"
+ "tbz %x[width], #0, 11f\n"
+ "ld1 { v28.b }[2], [x27]\n"
+ "ld1 { v27.b }[2], [x26]\n"
+ "ld1 { v26.b }[2], [x25]\n"
+ "ld1 { v22.b }[2], [x24]\n"
+ "ld1 { v25.b }[2], [x23]\n"
+ "ld1 { v24.b }[2], [x22]\n"
+ "ld1 { v19.b }[2], [x21]\n"
+ "ld1 { v21.b }[2], [x20]\n"
+ "b 11f\n"
+ "10:" // odd_loads_1_0
+ "ldr b28, [x27, #0x0]\n"
+ "ldr b27, [x26, #0x0]\n"
+ "ldr b26, [x25, #0x0]\n"
+ "ldr b22, [x24, #0x0]\n"
+ "ldr b25, [x23, #0x0]\n"
+ "ldr b24, [x22, #0x0]\n"
+ "ldr b19, [x21, #0x0]\n"
+ "ldr b21, [x20, #0x0]\n"
+ "mov x19, #0x1\n"
+ "11:" // Odd load end
+ "zip1 v23.4s, v28.4s, v26.4s\n"
+ "subs x19, x19, #0x1\n"
+ "zip1 v20.4s, v27.4s, v22.4s\n"
+ "zip1 v16.4s, v23.4s, v20.4s\n"
+ "str q16, [%x[out_ptr], #0x0]\n"
+ "zip1 v18.4s, v25.4s, v19.4s\n"
+ "zip1 v16.4s, v24.4s, v21.4s\n"
+ "zip1 v17.4s, v18.4s, v16.4s\n"
+ "str q17, [%x[out_ptr], #0x10]\n"
+ "add %x[out_ptr], %x[out_ptr], #0x20\n"
+ "beq 12f\n"
+ "zip2 v23.4s, v23.4s, v20.4s\n"
+ "zip2 v16.4s, v18.4s, v16.4s\n"
+ "str q23, [%x[out_ptr], #0x0]\n"
+ "str q16, [%x[out_ptr], #0x10]\n"
+ "subs x19, x19, #0x1\n"
+ "add %x[out_ptr], %x[out_ptr], #0x20\n"
+ "beq 12f\n"
+ "zip2 v26.4s, v28.4s, v26.4s\n"
+ "zip2 v22.4s, v27.4s, v22.4s\n"
+ "subs x19, x19, #0x1\n"
+ "zip1 v20.4s, v26.4s, v22.4s\n"
+ "str q20, [%x[out_ptr], #0x0]\n"
+ "zip2 v19.4s, v25.4s, v19.4s\n"
+ "zip2 v18.4s, v24.4s, v21.4s\n"
+ "zip1 v16.4s, v19.4s, v18.4s\n"
+ "str q16, [%x[out_ptr], #0x10]\n"
+ "add %x[out_ptr], %x[out_ptr], #0x20\n"
+ "beq 12f\n"
+ "zip2 v17.4s, v26.4s, v22.4s\n"
+ "zip2 v16.4s, v19.4s, v18.4s\n"
+ "str q17, [%x[out_ptr], #0x0]\n"
+ "str q16, [%x[out_ptr], #0x10]\n"
+ "add %x[out_ptr], %x[out_ptr], #0x20\n"
+ "12:" // Odds skip
+
+ : [out_ptr] "+r" (out_ptr), [width] "+r" (width)
+ : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset)
+ : "cc", "memory", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27"
+ );
+}
+
+template<>
+void interleave_block<8, 4, VLType::None, false>(
+ uint8_t * &out_ptr, const uint8_t * const * in, size_t width, size_t height,
+ size_t row_offset, bool
+)
+{
+ int8_t * &out_cast = reinterpret_cast<int8_t * &>(out_ptr);
+ const int8_t * const * in_cast = reinterpret_cast<const int8_t * const *>(in);
+
+ interleave_block<8, 4, VLType::None, false>(out_cast, in_cast, width, height, row_offset, false);
+}
+
+
+#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block4_s8_s8_summing.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block4_s8_s8_summing.hpp
new file mode 100644
index 0000000..2831cb7
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block4_s8_s8_summing.hpp
@@ -0,0 +1,370 @@
+/*
+ * Copyright (c) 2019-2020 Arm Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+#ifdef __aarch64__
+
+template<>
+void interleave_block<8, 4, VLType::None, true>(
+ int8_t * &out_ptr, const int8_t * const * in, size_t width, size_t height,
+ size_t row_offset, bool first
+)
+{
+ __asm__ __volatile__(
+ "movi v1.8h, #0x0\n"
+ "ldr x27, [%x[in], #0x0]\n"
+ "mov x19, #0x0\n"
+ "movi v0.8h, #0x0\n"
+ "ldr x26, [%x[in], #0x8]\n"
+ "cmp %x[height], #0x8\n"
+ "movi v31.4s, #0x0\n"
+ "ldr x25, [%x[in], #0x10]\n"
+ "add x27, x27, %x[row_offset]\n"
+ "movi v30.4s, #0x0\n"
+ "ldr x24, [%x[in], #0x18]\n"
+ "ldr x23, [%x[in], #0x20]\n"
+ "add x26, x26, %x[row_offset]\n"
+ "ldr x22, [%x[in], #0x28]\n"
+ "add x25, x25, %x[row_offset]\n"
+ "ldr x21, [%x[in], #0x30]\n"
+ "add x24, x24, %x[row_offset]\n"
+ "ldr x20, [%x[in], #0x38]\n"
+ "add x23, x23, %x[row_offset]\n"
+ "add x22, x22, %x[row_offset]\n"
+ "add x21, x21, %x[row_offset]\n"
+ "add x20, x20, %x[row_offset]\n"
+ "beq 1f\n"
+ "mov x20, x27\n"
+ "cmp %x[height], #0x2\n"
+ "csel x26, x26, x27, GE\n"
+ "csel x25, x25, x27, GT\n"
+ "cmp %x[height], #0x4\n"
+ "csel x24, x24, x27, GE\n"
+ "csel x23, x23, x27, GT\n"
+ "cmp %x[height], #0x6\n"
+ "csel x22, x22, x27, GE\n"
+ "csel x21, x21, x27, GT\n"
+ "1:" // no_pointer_adj
+ "prfm pldl1keep, [x27, #0x0]\n"
+ "prfm pldl1keep, [x26, #0x0]\n"
+ "prfm pldl1keep, [x25, #0x0]\n"
+ "prfm pldl1keep, [x24, #0x0]\n"
+ "prfm pldl1keep, [x23, #0x0]\n"
+ "prfm pldl1keep, [x22, #0x0]\n"
+ "prfm pldl1keep, [x21, #0x0]\n"
+ "prfm pldl1keep, [x20, #0x0]\n"
+ "prfm pldl1keep, [x27, #0x40]\n"
+ "prfm pldl1keep, [x26, #0x40]\n"
+ "prfm pldl1keep, [x25, #0x40]\n"
+ "prfm pldl1keep, [x24, #0x40]\n"
+ "prfm pldl1keep, [x23, #0x40]\n"
+ "prfm pldl1keep, [x22, #0x40]\n"
+ "prfm pldl1keep, [x21, #0x40]\n"
+ "prfm pldl1keep, [x20, #0x40]\n"
+ "cbnz %w[first], 2f\n"
+ "sub %x[out_ptr], %x[out_ptr], #0x20\n"
+ "ld1 { v31.4s }, [%x[out_ptr]]\n"
+ "ldr q30, [%x[out_ptr], #0x10]\n"
+ "2:" // first_pass
+ "cmp %x[width], #0x10\n"
+ "blt 5f\n"
+ "3:" // Main loop head
+ "cmp x19, #0x1e\n"
+ "ble 4f\n"
+ "sadalp v31.4s, v1.8h\n"
+ "movi v1.8h, #0x0\n"
+ "sadalp v30.4s, v0.8h\n"
+ "movi v0.8h, #0x0\n"
+ "mov x19, #0x0\n"
+ "4:" // no_accumulate_16
+ "ldr q29, [x27], #0x10\n"
+ "prfm pldl1keep, [x27, #0x70]\n"
+ "ldr q28, [x26], #0x10\n"
+ "ldr q27, [x25], #0x10\n"
+ "zip1 v23.4s, v29.4s, v27.4s\n"
+ "prfm pldl1keep, [x26, #0x70]\n"
+ "ldr q21, [x24], #0x10\n"
+ "zip2 v27.4s, v29.4s, v27.4s\n"
+ "prfm pldl1keep, [x25, #0x70]\n"
+ "ldr q26, [x23], #0x10\n"
+ "zip1 v20.4s, v28.4s, v21.4s\n"
+ "prfm pldl1keep, [x24, #0x70]\n"
+ "ldr q25, [x22], #0x10\n"
+ "zip1 v16.4s, v23.4s, v20.4s\n"
+ "prfm pldl1keep, [x23, #0x70]\n"
+ "ldr q19, [x21], #0x10\n"
+ "zip2 v24.4s, v23.4s, v20.4s\n"
+ "prfm pldl1keep, [x22, #0x70]\n"
+ "zip2 v23.4s, v28.4s, v21.4s\n"
+ "ldr q22, [x20], #0x10\n"
+ "zip1 v18.4s, v26.4s, v19.4s\n"
+ "prfm pldl1keep, [x21, #0x70]\n"
+ "str q16, [%x[out_ptr], #0x0]\n"
+ "zip1 v21.4s, v27.4s, v23.4s\n"
+ "prfm pldl1keep, [x20, #0x70]\n"
+ "zip1 v17.4s, v25.4s, v22.4s\n"
+ "sadalp v1.8h, v16.16b\n"
+ "zip1 v16.4s, v18.4s, v17.4s\n"
+ "add x19, x19, #0x1\n"
+ "zip2 v20.4s, v18.4s, v17.4s\n"
+ "str q16, [%x[out_ptr], #0x10]\n"
+ "zip2 v19.4s, v26.4s, v19.4s\n"
+ "sadalp v0.8h, v16.16b\n"
+ "zip2 v16.4s, v25.4s, v22.4s\n"
+ "str q24, [%x[out_ptr], #0x20]\n"
+ "zip1 v18.4s, v19.4s, v16.4s\n"
+ "sadalp v1.8h, v24.16b\n"
+ "zip2 v17.4s, v27.4s, v23.4s\n"
+ "str q20, [%x[out_ptr], #0x30]\n"
+ "zip2 v16.4s, v19.4s, v16.4s\n"
+ "str q21, [%x[out_ptr], #0x40]\n"
+ "str q18, [%x[out_ptr], #0x50]\n"
+ "sadalp v0.8h, v20.16b\n"
+ "str q17, [%x[out_ptr], #0x60]\n"
+ "sadalp v1.8h, v21.16b\n"
+ "str q16, [%x[out_ptr], #0x70]\n"
+ "subs %x[width], %x[width], #0x10\n"
+ "sadalp v0.8h, v18.16b\n"
+ "cmp %x[width], #0x10\n"
+ "sadalp v1.8h, v17.16b\n"
+ "add %x[out_ptr], %x[out_ptr], #0x80\n"
+ "sadalp v0.8h, v16.16b\n"
+ "bge 3b\n"
+ "5:" // Main loop skip
+ "cbz %x[width], 14f\n"
+ "tbz %x[width], #3, 9f\n"
+ "ldr d29, [x27], #0x8\n"
+ "ldr d28, [x26], #0x8\n"
+ "ldr d27, [x25], #0x8\n"
+ "ldr d21, [x24], #0x8\n"
+ "ldr d26, [x23], #0x8\n"
+ "ldr d25, [x22], #0x8\n"
+ "ldr d19, [x21], #0x8\n"
+ "ldr d22, [x20], #0x8\n"
+ "tbz %x[width], #2, 7f\n"
+ "ld1 { v29.s }[2], [x27], #0x4\n"
+ "ld1 { v28.s }[2], [x26], #0x4\n"
+ "ld1 { v27.s }[2], [x25], #0x4\n"
+ "ld1 { v21.s }[2], [x24], #0x4\n"
+ "ld1 { v26.s }[2], [x23], #0x4\n"
+ "ld1 { v25.s }[2], [x22], #0x4\n"
+ "ld1 { v19.s }[2], [x21], #0x4\n"
+ "ld1 { v22.s }[2], [x20], #0x4\n"
+ "tbz %x[width], #1, 6f\n"
+ "ld1 { v29.h }[6], [x27], #0x2\n"
+ "ld1 { v28.h }[6], [x26], #0x2\n"
+ "ld1 { v27.h }[6], [x25], #0x2\n"
+ "ld1 { v21.h }[6], [x24], #0x2\n"
+ "ld1 { v26.h }[6], [x23], #0x2\n"
+ "ld1 { v25.h }[6], [x22], #0x2\n"
+ "ld1 { v19.h }[6], [x21], #0x2\n"
+ "ld1 { v22.h }[6], [x20], #0x2\n"
+ "mov x19, #0x4\n"
+ "tbz %x[width], #0, 13f\n"
+ "ld1 { v29.b }[14], [x27]\n"
+ "ld1 { v28.b }[14], [x26]\n"
+ "ld1 { v27.b }[14], [x25]\n"
+ "ld1 { v21.b }[14], [x24]\n"
+ "ld1 { v26.b }[14], [x23]\n"
+ "ld1 { v25.b }[14], [x22]\n"
+ "ld1 { v19.b }[14], [x21]\n"
+ "ld1 { v22.b }[14], [x20]\n"
+ "b 13f\n"
+ "6:" // odd_loads_1_12
+ "mov x19, #0x3\n"
+ "tbz %x[width], #0, 13f\n"
+ "ld1 { v29.b }[12], [x27]\n"
+ "ld1 { v28.b }[12], [x26]\n"
+ "ld1 { v27.b }[12], [x25]\n"
+ "ld1 { v21.b }[12], [x24]\n"
+ "ld1 { v26.b }[12], [x23]\n"
+ "ld1 { v25.b }[12], [x22]\n"
+ "ld1 { v19.b }[12], [x21]\n"
+ "ld1 { v22.b }[12], [x20]\n"
+ "mov x19, #0x4\n"
+ "b 13f\n"
+ "7:" // odd_loads_2_8
+ "tbz %x[width], #1, 8f\n"
+ "ld1 { v29.h }[4], [x27], #0x2\n"
+ "ld1 { v28.h }[4], [x26], #0x2\n"
+ "ld1 { v27.h }[4], [x25], #0x2\n"
+ "ld1 { v21.h }[4], [x24], #0x2\n"
+ "ld1 { v26.h }[4], [x23], #0x2\n"
+ "ld1 { v25.h }[4], [x22], #0x2\n"
+ "ld1 { v19.h }[4], [x21], #0x2\n"
+ "ld1 { v22.h }[4], [x20], #0x2\n"
+ "mov x19, #0x3\n"
+ "tbz %x[width], #0, 13f\n"
+ "ld1 { v29.b }[10], [x27]\n"
+ "ld1 { v28.b }[10], [x26]\n"
+ "ld1 { v27.b }[10], [x25]\n"
+ "ld1 { v21.b }[10], [x24]\n"
+ "ld1 { v26.b }[10], [x23]\n"
+ "ld1 { v25.b }[10], [x22]\n"
+ "ld1 { v19.b }[10], [x21]\n"
+ "ld1 { v22.b }[10], [x20]\n"
+ "b 13f\n"
+ "8:" // odd_loads_1_8
+ "mov x19, #0x2\n"
+ "tbz %x[width], #0, 13f\n"
+ "ld1 { v29.b }[8], [x27]\n"
+ "ld1 { v28.b }[8], [x26]\n"
+ "ld1 { v27.b }[8], [x25]\n"
+ "ld1 { v21.b }[8], [x24]\n"
+ "ld1 { v26.b }[8], [x23]\n"
+ "ld1 { v25.b }[8], [x22]\n"
+ "ld1 { v19.b }[8], [x21]\n"
+ "ld1 { v22.b }[8], [x20]\n"
+ "mov x19, #0x3\n"
+ "b 13f\n"
+ "9:" // odd_loads_4_0
+ "tbz %x[width], #2, 11f\n"
+ "ldr s29, [x27], #0x4\n"
+ "ldr s28, [x26], #0x4\n"
+ "ldr s27, [x25], #0x4\n"
+ "ldr s21, [x24], #0x4\n"
+ "ldr s26, [x23], #0x4\n"
+ "ldr s25, [x22], #0x4\n"
+ "ldr s19, [x21], #0x4\n"
+ "ldr s22, [x20], #0x4\n"
+ "tbz %x[width], #1, 10f\n"
+ "ld1 { v29.h }[2], [x27], #0x2\n"
+ "ld1 { v28.h }[2], [x26], #0x2\n"
+ "ld1 { v27.h }[2], [x25], #0x2\n"
+ "ld1 { v21.h }[2], [x24], #0x2\n"
+ "ld1 { v26.h }[2], [x23], #0x2\n"
+ "ld1 { v25.h }[2], [x22], #0x2\n"
+ "ld1 { v19.h }[2], [x21], #0x2\n"
+ "ld1 { v22.h }[2], [x20], #0x2\n"
+ "mov x19, #0x2\n"
+ "tbz %x[width], #0, 13f\n"
+ "ld1 { v29.b }[6], [x27]\n"
+ "ld1 { v28.b }[6], [x26]\n"
+ "ld1 { v27.b }[6], [x25]\n"
+ "ld1 { v21.b }[6], [x24]\n"
+ "ld1 { v26.b }[6], [x23]\n"
+ "ld1 { v25.b }[6], [x22]\n"
+ "ld1 { v19.b }[6], [x21]\n"
+ "ld1 { v22.b }[6], [x20]\n"
+ "b 13f\n"
+ "10:" // odd_loads_1_4
+ "mov x19, #0x1\n"
+ "tbz %x[width], #0, 13f\n"
+ "ld1 { v29.b }[4], [x27]\n"
+ "ld1 { v28.b }[4], [x26]\n"
+ "ld1 { v27.b }[4], [x25]\n"
+ "ld1 { v21.b }[4], [x24]\n"
+ "ld1 { v26.b }[4], [x23]\n"
+ "ld1 { v25.b }[4], [x22]\n"
+ "ld1 { v19.b }[4], [x21]\n"
+ "ld1 { v22.b }[4], [x20]\n"
+ "mov x19, #0x2\n"
+ "b 13f\n"
+ "11:" // odd_loads_2_0
+ "tbz %x[width], #1, 12f\n"
+ "ldr h29, [x27], #0x2\n"
+ "ldr h28, [x26], #0x2\n"
+ "ldr h27, [x25], #0x2\n"
+ "ldr h21, [x24], #0x2\n"
+ "ldr h26, [x23], #0x2\n"
+ "ldr h25, [x22], #0x2\n"
+ "ldr h19, [x21], #0x2\n"
+ "ldr h22, [x20], #0x2\n"
+ "mov x19, #0x1\n"
+ "tbz %x[width], #0, 13f\n"
+ "ld1 { v29.b }[2], [x27]\n"
+ "ld1 { v28.b }[2], [x26]\n"
+ "ld1 { v27.b }[2], [x25]\n"
+ "ld1 { v21.b }[2], [x24]\n"
+ "ld1 { v26.b }[2], [x23]\n"
+ "ld1 { v25.b }[2], [x22]\n"
+ "ld1 { v19.b }[2], [x21]\n"
+ "ld1 { v22.b }[2], [x20]\n"
+ "b 13f\n"
+ "12:" // odd_loads_1_0
+ "ldr b29, [x27, #0x0]\n"
+ "ldr b28, [x26, #0x0]\n"
+ "ldr b27, [x25, #0x0]\n"
+ "ldr b21, [x24, #0x0]\n"
+ "ldr b26, [x23, #0x0]\n"
+ "ldr b25, [x22, #0x0]\n"
+ "ldr b19, [x21, #0x0]\n"
+ "ldr b22, [x20, #0x0]\n"
+ "mov x19, #0x1\n"
+ "13:" // Odd load end
+ "zip1 v23.4s, v29.4s, v27.4s\n"
+ "subs x19, x19, #0x1\n"
+ "zip1 v20.4s, v28.4s, v21.4s\n"
+ "zip1 v16.4s, v23.4s, v20.4s\n"
+ "str q16, [%x[out_ptr], #0x0]\n"
+ "zip1 v18.4s, v26.4s, v19.4s\n"
+ "sadalp v1.8h, v16.16b\n"
+ "zip1 v17.4s, v25.4s, v22.4s\n"
+ "zip1 v16.4s, v18.4s, v17.4s\n"
+ "str q16, [%x[out_ptr], #0x10]\n"
+ "sadalp v0.8h, v16.16b\n"
+ "add %x[out_ptr], %x[out_ptr], #0x20\n"
+ "beq 14f\n"
+ "zip2 v24.4s, v23.4s, v20.4s\n"
+ "zip2 v20.4s, v18.4s, v17.4s\n"
+ "str q24, [%x[out_ptr], #0x0]\n"
+ "sadalp v1.8h, v24.16b\n"
+ "str q20, [%x[out_ptr], #0x10]\n"
+ "sadalp v0.8h, v20.16b\n"
+ "subs x19, x19, #0x1\n"
+ "add %x[out_ptr], %x[out_ptr], #0x20\n"
+ "beq 14f\n"
+ "zip2 v27.4s, v29.4s, v27.4s\n"
+ "zip2 v23.4s, v28.4s, v21.4s\n"
+ "subs x19, x19, #0x1\n"
+ "zip1 v21.4s, v27.4s, v23.4s\n"
+ "str q21, [%x[out_ptr], #0x0]\n"
+ "zip2 v19.4s, v26.4s, v19.4s\n"
+ "sadalp v1.8h, v21.16b\n"
+ "zip2 v16.4s, v25.4s, v22.4s\n"
+ "zip1 v18.4s, v19.4s, v16.4s\n"
+ "str q18, [%x[out_ptr], #0x10]\n"
+ "sadalp v0.8h, v18.16b\n"
+ "add %x[out_ptr], %x[out_ptr], #0x20\n"
+ "beq 14f\n"
+ "zip2 v17.4s, v27.4s, v23.4s\n"
+ "zip2 v16.4s, v19.4s, v16.4s\n"
+ "str q17, [%x[out_ptr], #0x0]\n"
+ "sadalp v1.8h, v17.16b\n"
+ "str q16, [%x[out_ptr], #0x10]\n"
+ "sadalp v0.8h, v16.16b\n"
+ "add %x[out_ptr], %x[out_ptr], #0x20\n"
+ "14:" // Odds skip
+ "sadalp v31.4s, v1.8h\n"
+ "sadalp v30.4s, v0.8h\n"
+ "str q31, [%x[out_ptr], #0x0]\n"
+ "str q30, [%x[out_ptr], #0x10]\n"
+ "add %x[out_ptr], %x[out_ptr], #0x20\n"
+ : [out_ptr] "+r" (out_ptr), [width] "+r" (width)
+ : [first] "r" (first), [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset)
+ : "cc", "memory", "v0", "v1", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27"
+ );
+}
+
+
+#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block4_u8_u8_summing.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block4_u8_u8_summing.hpp
new file mode 100644
index 0000000..7c7857b
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block4_u8_u8_summing.hpp
@@ -0,0 +1,370 @@
+/*
+ * Copyright (c) 2019-2020 Arm Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+#ifdef __aarch64__
+
+template<>
+void interleave_block<8, 4, VLType::None, true>(
+ uint8_t * &out_ptr, const uint8_t * const * in, size_t width, size_t height,
+ size_t row_offset, bool first
+)
+{
+ __asm__ __volatile__(
+ "movi v1.8h, #0x0\n"
+ "ldr x27, [%x[in], #0x0]\n"
+ "mov x19, #0x0\n"
+ "movi v0.8h, #0x0\n"
+ "ldr x26, [%x[in], #0x8]\n"
+ "cmp %x[height], #0x8\n"
+ "movi v31.4s, #0x0\n"
+ "ldr x25, [%x[in], #0x10]\n"
+ "add x27, x27, %x[row_offset]\n"
+ "movi v30.4s, #0x0\n"
+ "ldr x24, [%x[in], #0x18]\n"
+ "ldr x23, [%x[in], #0x20]\n"
+ "add x26, x26, %x[row_offset]\n"
+ "ldr x22, [%x[in], #0x28]\n"
+ "add x25, x25, %x[row_offset]\n"
+ "ldr x21, [%x[in], #0x30]\n"
+ "add x24, x24, %x[row_offset]\n"
+ "ldr x20, [%x[in], #0x38]\n"
+ "add x23, x23, %x[row_offset]\n"
+ "add x22, x22, %x[row_offset]\n"
+ "add x21, x21, %x[row_offset]\n"
+ "add x20, x20, %x[row_offset]\n"
+ "beq 1f\n"
+ "mov x20, x27\n"
+ "cmp %x[height], #0x2\n"
+ "csel x26, x26, x27, GE\n"
+ "csel x25, x25, x27, GT\n"
+ "cmp %x[height], #0x4\n"
+ "csel x24, x24, x27, GE\n"
+ "csel x23, x23, x27, GT\n"
+ "cmp %x[height], #0x6\n"
+ "csel x22, x22, x27, GE\n"
+ "csel x21, x21, x27, GT\n"
+ "1:" // no_pointer_adj
+ "prfm pldl1keep, [x27, #0x0]\n"
+ "prfm pldl1keep, [x26, #0x0]\n"
+ "prfm pldl1keep, [x25, #0x0]\n"
+ "prfm pldl1keep, [x24, #0x0]\n"
+ "prfm pldl1keep, [x23, #0x0]\n"
+ "prfm pldl1keep, [x22, #0x0]\n"
+ "prfm pldl1keep, [x21, #0x0]\n"
+ "prfm pldl1keep, [x20, #0x0]\n"
+ "prfm pldl1keep, [x27, #0x40]\n"
+ "prfm pldl1keep, [x26, #0x40]\n"
+ "prfm pldl1keep, [x25, #0x40]\n"
+ "prfm pldl1keep, [x24, #0x40]\n"
+ "prfm pldl1keep, [x23, #0x40]\n"
+ "prfm pldl1keep, [x22, #0x40]\n"
+ "prfm pldl1keep, [x21, #0x40]\n"
+ "prfm pldl1keep, [x20, #0x40]\n"
+ "cbnz %w[first], 2f\n"
+ "sub %x[out_ptr], %x[out_ptr], #0x20\n"
+ "ld1 { v31.4s }, [%x[out_ptr]]\n"
+ "ldr q30, [%x[out_ptr], #0x10]\n"
+ "2:" // first_pass
+ "cmp %x[width], #0x10\n"
+ "blt 5f\n"
+ "3:" // Main loop head
+ "cmp x19, #0x1e\n"
+ "ble 4f\n"
+ "uadalp v31.4s, v1.8h\n"
+ "movi v1.8h, #0x0\n"
+ "uadalp v30.4s, v0.8h\n"
+ "movi v0.8h, #0x0\n"
+ "mov x19, #0x0\n"
+ "4:" // no_accumulate_16
+ "ldr q29, [x27], #0x10\n"
+ "prfm pldl1keep, [x27, #0x70]\n"
+ "ldr q28, [x26], #0x10\n"
+ "ldr q27, [x25], #0x10\n"
+ "zip1 v23.4s, v29.4s, v27.4s\n"
+ "prfm pldl1keep, [x26, #0x70]\n"
+ "ldr q21, [x24], #0x10\n"
+ "zip2 v27.4s, v29.4s, v27.4s\n"
+ "prfm pldl1keep, [x25, #0x70]\n"
+ "ldr q26, [x23], #0x10\n"
+ "zip1 v20.4s, v28.4s, v21.4s\n"
+ "prfm pldl1keep, [x24, #0x70]\n"
+ "ldr q25, [x22], #0x10\n"
+ "zip1 v16.4s, v23.4s, v20.4s\n"
+ "prfm pldl1keep, [x23, #0x70]\n"
+ "ldr q19, [x21], #0x10\n"
+ "zip2 v24.4s, v23.4s, v20.4s\n"
+ "prfm pldl1keep, [x22, #0x70]\n"
+ "zip2 v23.4s, v28.4s, v21.4s\n"
+ "ldr q22, [x20], #0x10\n"
+ "zip1 v18.4s, v26.4s, v19.4s\n"
+ "prfm pldl1keep, [x21, #0x70]\n"
+ "str q16, [%x[out_ptr], #0x0]\n"
+ "zip1 v21.4s, v27.4s, v23.4s\n"
+ "prfm pldl1keep, [x20, #0x70]\n"
+ "zip1 v17.4s, v25.4s, v22.4s\n"
+ "uadalp v1.8h, v16.16b\n"
+ "zip1 v16.4s, v18.4s, v17.4s\n"
+ "add x19, x19, #0x1\n"
+ "zip2 v20.4s, v18.4s, v17.4s\n"
+ "str q16, [%x[out_ptr], #0x10]\n"
+ "zip2 v19.4s, v26.4s, v19.4s\n"
+ "uadalp v0.8h, v16.16b\n"
+ "zip2 v16.4s, v25.4s, v22.4s\n"
+ "str q24, [%x[out_ptr], #0x20]\n"
+ "zip1 v18.4s, v19.4s, v16.4s\n"
+ "uadalp v1.8h, v24.16b\n"
+ "zip2 v17.4s, v27.4s, v23.4s\n"
+ "str q20, [%x[out_ptr], #0x30]\n"
+ "zip2 v16.4s, v19.4s, v16.4s\n"
+ "str q21, [%x[out_ptr], #0x40]\n"
+ "str q18, [%x[out_ptr], #0x50]\n"
+ "uadalp v0.8h, v20.16b\n"
+ "str q17, [%x[out_ptr], #0x60]\n"
+ "uadalp v1.8h, v21.16b\n"
+ "str q16, [%x[out_ptr], #0x70]\n"
+ "subs %x[width], %x[width], #0x10\n"
+ "uadalp v0.8h, v18.16b\n"
+ "cmp %x[width], #0x10\n"
+ "uadalp v1.8h, v17.16b\n"
+ "add %x[out_ptr], %x[out_ptr], #0x80\n"
+ "uadalp v0.8h, v16.16b\n"
+ "bge 3b\n"
+ "5:" // Main loop skip
+ "cbz %x[width], 14f\n"
+ "tbz %x[width], #3, 9f\n"
+ "ldr d29, [x27], #0x8\n"
+ "ldr d28, [x26], #0x8\n"
+ "ldr d27, [x25], #0x8\n"
+ "ldr d21, [x24], #0x8\n"
+ "ldr d26, [x23], #0x8\n"
+ "ldr d25, [x22], #0x8\n"
+ "ldr d19, [x21], #0x8\n"
+ "ldr d22, [x20], #0x8\n"
+ "tbz %x[width], #2, 7f\n"
+ "ld1 { v29.s }[2], [x27], #0x4\n"
+ "ld1 { v28.s }[2], [x26], #0x4\n"
+ "ld1 { v27.s }[2], [x25], #0x4\n"
+ "ld1 { v21.s }[2], [x24], #0x4\n"
+ "ld1 { v26.s }[2], [x23], #0x4\n"
+ "ld1 { v25.s }[2], [x22], #0x4\n"
+ "ld1 { v19.s }[2], [x21], #0x4\n"
+ "ld1 { v22.s }[2], [x20], #0x4\n"
+ "tbz %x[width], #1, 6f\n"
+ "ld1 { v29.h }[6], [x27], #0x2\n"
+ "ld1 { v28.h }[6], [x26], #0x2\n"
+ "ld1 { v27.h }[6], [x25], #0x2\n"
+ "ld1 { v21.h }[6], [x24], #0x2\n"
+ "ld1 { v26.h }[6], [x23], #0x2\n"
+ "ld1 { v25.h }[6], [x22], #0x2\n"
+ "ld1 { v19.h }[6], [x21], #0x2\n"
+ "ld1 { v22.h }[6], [x20], #0x2\n"
+ "mov x19, #0x4\n"
+ "tbz %x[width], #0, 13f\n"
+ "ld1 { v29.b }[14], [x27]\n"
+ "ld1 { v28.b }[14], [x26]\n"
+ "ld1 { v27.b }[14], [x25]\n"
+ "ld1 { v21.b }[14], [x24]\n"
+ "ld1 { v26.b }[14], [x23]\n"
+ "ld1 { v25.b }[14], [x22]\n"
+ "ld1 { v19.b }[14], [x21]\n"
+ "ld1 { v22.b }[14], [x20]\n"
+ "b 13f\n"
+ "6:" // odd_loads_1_12
+ "mov x19, #0x3\n"
+ "tbz %x[width], #0, 13f\n"
+ "ld1 { v29.b }[12], [x27]\n"
+ "ld1 { v28.b }[12], [x26]\n"
+ "ld1 { v27.b }[12], [x25]\n"
+ "ld1 { v21.b }[12], [x24]\n"
+ "ld1 { v26.b }[12], [x23]\n"
+ "ld1 { v25.b }[12], [x22]\n"
+ "ld1 { v19.b }[12], [x21]\n"
+ "ld1 { v22.b }[12], [x20]\n"
+ "mov x19, #0x4\n"
+ "b 13f\n"
+ "7:" // odd_loads_2_8
+ "tbz %x[width], #1, 8f\n"
+ "ld1 { v29.h }[4], [x27], #0x2\n"
+ "ld1 { v28.h }[4], [x26], #0x2\n"
+ "ld1 { v27.h }[4], [x25], #0x2\n"
+ "ld1 { v21.h }[4], [x24], #0x2\n"
+ "ld1 { v26.h }[4], [x23], #0x2\n"
+ "ld1 { v25.h }[4], [x22], #0x2\n"
+ "ld1 { v19.h }[4], [x21], #0x2\n"
+ "ld1 { v22.h }[4], [x20], #0x2\n"
+ "mov x19, #0x3\n"
+ "tbz %x[width], #0, 13f\n"
+ "ld1 { v29.b }[10], [x27]\n"
+ "ld1 { v28.b }[10], [x26]\n"
+ "ld1 { v27.b }[10], [x25]\n"
+ "ld1 { v21.b }[10], [x24]\n"
+ "ld1 { v26.b }[10], [x23]\n"
+ "ld1 { v25.b }[10], [x22]\n"
+ "ld1 { v19.b }[10], [x21]\n"
+ "ld1 { v22.b }[10], [x20]\n"
+ "b 13f\n"
+ "8:" // odd_loads_1_8
+ "mov x19, #0x2\n"
+ "tbz %x[width], #0, 13f\n"
+ "ld1 { v29.b }[8], [x27]\n"
+ "ld1 { v28.b }[8], [x26]\n"
+ "ld1 { v27.b }[8], [x25]\n"
+ "ld1 { v21.b }[8], [x24]\n"
+ "ld1 { v26.b }[8], [x23]\n"
+ "ld1 { v25.b }[8], [x22]\n"
+ "ld1 { v19.b }[8], [x21]\n"
+ "ld1 { v22.b }[8], [x20]\n"
+ "mov x19, #0x3\n"
+ "b 13f\n"
+ "9:" // odd_loads_4_0
+ "tbz %x[width], #2, 11f\n"
+ "ldr s29, [x27], #0x4\n"
+ "ldr s28, [x26], #0x4\n"
+ "ldr s27, [x25], #0x4\n"
+ "ldr s21, [x24], #0x4\n"
+ "ldr s26, [x23], #0x4\n"
+ "ldr s25, [x22], #0x4\n"
+ "ldr s19, [x21], #0x4\n"
+ "ldr s22, [x20], #0x4\n"
+ "tbz %x[width], #1, 10f\n"
+ "ld1 { v29.h }[2], [x27], #0x2\n"
+ "ld1 { v28.h }[2], [x26], #0x2\n"
+ "ld1 { v27.h }[2], [x25], #0x2\n"
+ "ld1 { v21.h }[2], [x24], #0x2\n"
+ "ld1 { v26.h }[2], [x23], #0x2\n"
+ "ld1 { v25.h }[2], [x22], #0x2\n"
+ "ld1 { v19.h }[2], [x21], #0x2\n"
+ "ld1 { v22.h }[2], [x20], #0x2\n"
+ "mov x19, #0x2\n"
+ "tbz %x[width], #0, 13f\n"
+ "ld1 { v29.b }[6], [x27]\n"
+ "ld1 { v28.b }[6], [x26]\n"
+ "ld1 { v27.b }[6], [x25]\n"
+ "ld1 { v21.b }[6], [x24]\n"
+ "ld1 { v26.b }[6], [x23]\n"
+ "ld1 { v25.b }[6], [x22]\n"
+ "ld1 { v19.b }[6], [x21]\n"
+ "ld1 { v22.b }[6], [x20]\n"
+ "b 13f\n"
+ "10:" // odd_loads_1_4
+ "mov x19, #0x1\n"
+ "tbz %x[width], #0, 13f\n"
+ "ld1 { v29.b }[4], [x27]\n"
+ "ld1 { v28.b }[4], [x26]\n"
+ "ld1 { v27.b }[4], [x25]\n"
+ "ld1 { v21.b }[4], [x24]\n"
+ "ld1 { v26.b }[4], [x23]\n"
+ "ld1 { v25.b }[4], [x22]\n"
+ "ld1 { v19.b }[4], [x21]\n"
+ "ld1 { v22.b }[4], [x20]\n"
+ "mov x19, #0x2\n"
+ "b 13f\n"
+ "11:" // odd_loads_2_0
+ "tbz %x[width], #1, 12f\n"
+ "ldr h29, [x27], #0x2\n"
+ "ldr h28, [x26], #0x2\n"
+ "ldr h27, [x25], #0x2\n"
+ "ldr h21, [x24], #0x2\n"
+ "ldr h26, [x23], #0x2\n"
+ "ldr h25, [x22], #0x2\n"
+ "ldr h19, [x21], #0x2\n"
+ "ldr h22, [x20], #0x2\n"
+ "mov x19, #0x1\n"
+ "tbz %x[width], #0, 13f\n"
+ "ld1 { v29.b }[2], [x27]\n"
+ "ld1 { v28.b }[2], [x26]\n"
+ "ld1 { v27.b }[2], [x25]\n"
+ "ld1 { v21.b }[2], [x24]\n"
+ "ld1 { v26.b }[2], [x23]\n"
+ "ld1 { v25.b }[2], [x22]\n"
+ "ld1 { v19.b }[2], [x21]\n"
+ "ld1 { v22.b }[2], [x20]\n"
+ "b 13f\n"
+ "12:" // odd_loads_1_0
+ "ldr b29, [x27, #0x0]\n"
+ "ldr b28, [x26, #0x0]\n"
+ "ldr b27, [x25, #0x0]\n"
+ "ldr b21, [x24, #0x0]\n"
+ "ldr b26, [x23, #0x0]\n"
+ "ldr b25, [x22, #0x0]\n"
+ "ldr b19, [x21, #0x0]\n"
+ "ldr b22, [x20, #0x0]\n"
+ "mov x19, #0x1\n"
+ "13:" // Odd load end
+ "zip1 v23.4s, v29.4s, v27.4s\n"
+ "subs x19, x19, #0x1\n"
+ "zip1 v20.4s, v28.4s, v21.4s\n"
+ "zip1 v16.4s, v23.4s, v20.4s\n"
+ "str q16, [%x[out_ptr], #0x0]\n"
+ "zip1 v18.4s, v26.4s, v19.4s\n"
+ "uadalp v1.8h, v16.16b\n"
+ "zip1 v17.4s, v25.4s, v22.4s\n"
+ "zip1 v16.4s, v18.4s, v17.4s\n"
+ "str q16, [%x[out_ptr], #0x10]\n"
+ "uadalp v0.8h, v16.16b\n"
+ "add %x[out_ptr], %x[out_ptr], #0x20\n"
+ "beq 14f\n"
+ "zip2 v24.4s, v23.4s, v20.4s\n"
+ "zip2 v20.4s, v18.4s, v17.4s\n"
+ "str q24, [%x[out_ptr], #0x0]\n"
+ "uadalp v1.8h, v24.16b\n"
+ "str q20, [%x[out_ptr], #0x10]\n"
+ "uadalp v0.8h, v20.16b\n"
+ "subs x19, x19, #0x1\n"
+ "add %x[out_ptr], %x[out_ptr], #0x20\n"
+ "beq 14f\n"
+ "zip2 v27.4s, v29.4s, v27.4s\n"
+ "zip2 v23.4s, v28.4s, v21.4s\n"
+ "subs x19, x19, #0x1\n"
+ "zip1 v21.4s, v27.4s, v23.4s\n"
+ "str q21, [%x[out_ptr], #0x0]\n"
+ "zip2 v19.4s, v26.4s, v19.4s\n"
+ "uadalp v1.8h, v21.16b\n"
+ "zip2 v16.4s, v25.4s, v22.4s\n"
+ "zip1 v18.4s, v19.4s, v16.4s\n"
+ "str q18, [%x[out_ptr], #0x10]\n"
+ "uadalp v0.8h, v18.16b\n"
+ "add %x[out_ptr], %x[out_ptr], #0x20\n"
+ "beq 14f\n"
+ "zip2 v17.4s, v27.4s, v23.4s\n"
+ "zip2 v16.4s, v19.4s, v16.4s\n"
+ "str q17, [%x[out_ptr], #0x0]\n"
+ "uadalp v1.8h, v17.16b\n"
+ "str q16, [%x[out_ptr], #0x10]\n"
+ "uadalp v0.8h, v16.16b\n"
+ "add %x[out_ptr], %x[out_ptr], #0x20\n"
+ "14:" // Odds skip
+ "uadalp v31.4s, v1.8h\n"
+ "uadalp v30.4s, v0.8h\n"
+ "str q31, [%x[out_ptr], #0x0]\n"
+ "str q30, [%x[out_ptr], #0x10]\n"
+ "add %x[out_ptr], %x[out_ptr], #0x20\n"
+ : [out_ptr] "+r" (out_ptr), [width] "+r" (width)
+ : [first] "r" (first), [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset)
+ : "cc", "memory", "v0", "v1", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27"
+ );
+}
+
+
+#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block8_s8_s8.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block8_s8_s8.hpp
new file mode 100644
index 0000000..704a4c9
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block8_s8_s8.hpp
@@ -0,0 +1,319 @@
+/*
+ * Copyright (c) 2019-2020 Arm Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+#ifdef __aarch64__
+
+template<>
+void interleave_block<8, 8, VLType::None, false>(
+ int8_t * &out_ptr, const int8_t * const * in, size_t width, size_t height,
+ size_t row_offset, bool
+)
+{
+ __asm__ __volatile__(
+ "ldr x27, [%x[in], #0x0]\n"
+ "cmp %x[height], #0x8\n"
+ "ldr x26, [%x[in], #0x8]\n"
+ "add x27, x27, %x[row_offset]\n"
+ "ldr x25, [%x[in], #0x10]\n"
+ "ldr x24, [%x[in], #0x18]\n"
+ "add x26, x26, %x[row_offset]\n"
+ "ldr x23, [%x[in], #0x20]\n"
+ "add x25, x25, %x[row_offset]\n"
+ "ldr x22, [%x[in], #0x28]\n"
+ "ldr x21, [%x[in], #0x30]\n"
+ "add x24, x24, %x[row_offset]\n"
+ "ldr x20, [%x[in], #0x38]\n"
+ "add x23, x23, %x[row_offset]\n"
+ "add x22, x22, %x[row_offset]\n"
+ "add x21, x21, %x[row_offset]\n"
+ "add x20, x20, %x[row_offset]\n"
+ "beq 1f\n"
+ "mov x20, x27\n"
+ "cmp %x[height], #0x2\n"
+ "csel x26, x26, x27, GE\n"
+ "csel x25, x25, x27, GT\n"
+ "cmp %x[height], #0x4\n"
+ "csel x24, x24, x27, GE\n"
+ "csel x23, x23, x27, GT\n"
+ "cmp %x[height], #0x6\n"
+ "csel x22, x22, x27, GE\n"
+ "csel x21, x21, x27, GT\n"
+ "1:" // no_pointer_adj
+ "prfm pldl1keep, [x27, #0x0]\n"
+ "cmp %x[width], #0x10\n"
+ "prfm pldl1keep, [x26, #0x0]\n"
+ "prfm pldl1keep, [x25, #0x0]\n"
+ "prfm pldl1keep, [x24, #0x0]\n"
+ "prfm pldl1keep, [x23, #0x0]\n"
+ "prfm pldl1keep, [x22, #0x0]\n"
+ "prfm pldl1keep, [x21, #0x0]\n"
+ "prfm pldl1keep, [x20, #0x0]\n"
+ "prfm pldl1keep, [x27, #0x40]\n"
+ "prfm pldl1keep, [x26, #0x40]\n"
+ "prfm pldl1keep, [x25, #0x40]\n"
+ "prfm pldl1keep, [x24, #0x40]\n"
+ "prfm pldl1keep, [x23, #0x40]\n"
+ "prfm pldl1keep, [x22, #0x40]\n"
+ "prfm pldl1keep, [x21, #0x40]\n"
+ "prfm pldl1keep, [x20, #0x40]\n"
+ "blt 3f\n"
+ "2:" // Main loop head
+ "ldr q27, [x27], #0x10\n"
+ "prfm pldl1keep, [x27, #0x70]\n"
+ "ldr q24, [x26], #0x10\n"
+ "zip1 v26.2d, v27.2d, v24.2d\n"
+ "prfm pldl1keep, [x26, #0x70]\n"
+ "ldr q25, [x25], #0x10\n"
+ "zip2 v24.2d, v27.2d, v24.2d\n"
+ "prfm pldl1keep, [x25, #0x70]\n"
+ "ldr q21, [x24], #0x10\n"
+ "zip1 v23.2d, v25.2d, v21.2d\n"
+ "prfm pldl1keep, [x24, #0x70]\n"
+ "ldr q22, [x23], #0x10\n"
+ "zip2 v21.2d, v25.2d, v21.2d\n"
+ "prfm pldl1keep, [x23, #0x70]\n"
+ "ldr q18, [x22], #0x10\n"
+ "zip1 v20.2d, v22.2d, v18.2d\n"
+ "prfm pldl1keep, [x22, #0x70]\n"
+ "ldr q19, [x21], #0x10\n"
+ "zip2 v18.2d, v22.2d, v18.2d\n"
+ "prfm pldl1keep, [x21, #0x70]\n"
+ "ldr q16, [x20], #0x10\n"
+ "zip1 v17.2d, v19.2d, v16.2d\n"
+ "prfm pldl1keep, [x20, #0x70]\n"
+ "str q26, [%x[out_ptr], #0x0]\n"
+ "zip2 v16.2d, v19.2d, v16.2d\n"
+ "str q23, [%x[out_ptr], #0x10]\n"
+ "str q20, [%x[out_ptr], #0x20]\n"
+ "str q17, [%x[out_ptr], #0x30]\n"
+ "str q24, [%x[out_ptr], #0x40]\n"
+ "str q21, [%x[out_ptr], #0x50]\n"
+ "str q18, [%x[out_ptr], #0x60]\n"
+ "str q16, [%x[out_ptr], #0x70]\n"
+ "subs %x[width], %x[width], #0x10\n"
+ "cmp %x[width], #0x10\n"
+ "add %x[out_ptr], %x[out_ptr], #0x80\n"
+ "bge 2b\n"
+ "3:" // Main loop skip
+ "cbz %x[width], 12f\n"
+ "tbz %x[width], #3, 7f\n"
+ "ldr d27, [x27], #0x8\n"
+ "ldr d24, [x26], #0x8\n"
+ "ldr d25, [x25], #0x8\n"
+ "ldr d21, [x24], #0x8\n"
+ "ldr d22, [x23], #0x8\n"
+ "ldr d18, [x22], #0x8\n"
+ "ldr d19, [x21], #0x8\n"
+ "ldr d16, [x20], #0x8\n"
+ "tbz %x[width], #2, 5f\n"
+ "ld1 { v27.s }[2], [x27], #0x4\n"
+ "ld1 { v24.s }[2], [x26], #0x4\n"
+ "ld1 { v25.s }[2], [x25], #0x4\n"
+ "ld1 { v21.s }[2], [x24], #0x4\n"
+ "ld1 { v22.s }[2], [x23], #0x4\n"
+ "ld1 { v18.s }[2], [x22], #0x4\n"
+ "ld1 { v19.s }[2], [x21], #0x4\n"
+ "ld1 { v16.s }[2], [x20], #0x4\n"
+ "tbz %x[width], #1, 4f\n"
+ "ld1 { v27.h }[6], [x27], #0x2\n"
+ "ld1 { v24.h }[6], [x26], #0x2\n"
+ "ld1 { v25.h }[6], [x25], #0x2\n"
+ "ld1 { v21.h }[6], [x24], #0x2\n"
+ "ld1 { v22.h }[6], [x23], #0x2\n"
+ "ld1 { v18.h }[6], [x22], #0x2\n"
+ "ld1 { v19.h }[6], [x21], #0x2\n"
+ "ld1 { v16.h }[6], [x20], #0x2\n"
+ "mov x19, #0x2\n"
+ "tbz %x[width], #0, 11f\n"
+ "ld1 { v27.b }[14], [x27]\n"
+ "ld1 { v24.b }[14], [x26]\n"
+ "ld1 { v25.b }[14], [x25]\n"
+ "ld1 { v21.b }[14], [x24]\n"
+ "ld1 { v22.b }[14], [x23]\n"
+ "ld1 { v18.b }[14], [x22]\n"
+ "ld1 { v19.b }[14], [x21]\n"
+ "ld1 { v16.b }[14], [x20]\n"
+ "b 11f\n"
+ "4:" // odd_loads_1_12
+ "mov x19, #0x2\n"
+ "tbz %x[width], #0, 11f\n"
+ "ld1 { v27.b }[12], [x27]\n"
+ "ld1 { v24.b }[12], [x26]\n"
+ "ld1 { v25.b }[12], [x25]\n"
+ "ld1 { v21.b }[12], [x24]\n"
+ "ld1 { v22.b }[12], [x23]\n"
+ "ld1 { v18.b }[12], [x22]\n"
+ "ld1 { v19.b }[12], [x21]\n"
+ "ld1 { v16.b }[12], [x20]\n"
+ "b 11f\n"
+ "5:" // odd_loads_2_8
+ "tbz %x[width], #1, 6f\n"
+ "ld1 { v27.h }[4], [x27], #0x2\n"
+ "ld1 { v24.h }[4], [x26], #0x2\n"
+ "ld1 { v25.h }[4], [x25], #0x2\n"
+ "ld1 { v21.h }[4], [x24], #0x2\n"
+ "ld1 { v22.h }[4], [x23], #0x2\n"
+ "ld1 { v18.h }[4], [x22], #0x2\n"
+ "ld1 { v19.h }[4], [x21], #0x2\n"
+ "ld1 { v16.h }[4], [x20], #0x2\n"
+ "mov x19, #0x2\n"
+ "tbz %x[width], #0, 11f\n"
+ "ld1 { v27.b }[10], [x27]\n"
+ "ld1 { v24.b }[10], [x26]\n"
+ "ld1 { v25.b }[10], [x25]\n"
+ "ld1 { v21.b }[10], [x24]\n"
+ "ld1 { v22.b }[10], [x23]\n"
+ "ld1 { v18.b }[10], [x22]\n"
+ "ld1 { v19.b }[10], [x21]\n"
+ "ld1 { v16.b }[10], [x20]\n"
+ "b 11f\n"
+ "6:" // odd_loads_1_8
+ "mov x19, #0x1\n"
+ "tbz %x[width], #0, 11f\n"
+ "ld1 { v27.b }[8], [x27]\n"
+ "ld1 { v24.b }[8], [x26]\n"
+ "ld1 { v25.b }[8], [x25]\n"
+ "ld1 { v21.b }[8], [x24]\n"
+ "ld1 { v22.b }[8], [x23]\n"
+ "ld1 { v18.b }[8], [x22]\n"
+ "ld1 { v19.b }[8], [x21]\n"
+ "ld1 { v16.b }[8], [x20]\n"
+ "mov x19, #0x2\n"
+ "b 11f\n"
+ "7:" // odd_loads_4_0
+ "tbz %x[width], #2, 9f\n"
+ "ldr s27, [x27], #0x4\n"
+ "ldr s24, [x26], #0x4\n"
+ "ldr s25, [x25], #0x4\n"
+ "ldr s21, [x24], #0x4\n"
+ "ldr s22, [x23], #0x4\n"
+ "ldr s18, [x22], #0x4\n"
+ "ldr s19, [x21], #0x4\n"
+ "ldr s16, [x20], #0x4\n"
+ "tbz %x[width], #1, 8f\n"
+ "ld1 { v27.h }[2], [x27], #0x2\n"
+ "ld1 { v24.h }[2], [x26], #0x2\n"
+ "ld1 { v25.h }[2], [x25], #0x2\n"
+ "ld1 { v21.h }[2], [x24], #0x2\n"
+ "ld1 { v22.h }[2], [x23], #0x2\n"
+ "ld1 { v18.h }[2], [x22], #0x2\n"
+ "ld1 { v19.h }[2], [x21], #0x2\n"
+ "ld1 { v16.h }[2], [x20], #0x2\n"
+ "mov x19, #0x1\n"
+ "tbz %x[width], #0, 11f\n"
+ "ld1 { v27.b }[6], [x27]\n"
+ "ld1 { v24.b }[6], [x26]\n"
+ "ld1 { v25.b }[6], [x25]\n"
+ "ld1 { v21.b }[6], [x24]\n"
+ "ld1 { v22.b }[6], [x23]\n"
+ "ld1 { v18.b }[6], [x22]\n"
+ "ld1 { v19.b }[6], [x21]\n"
+ "ld1 { v16.b }[6], [x20]\n"
+ "b 11f\n"
+ "8:" // odd_loads_1_4
+ "mov x19, #0x1\n"
+ "tbz %x[width], #0, 11f\n"
+ "ld1 { v27.b }[4], [x27]\n"
+ "ld1 { v24.b }[4], [x26]\n"
+ "ld1 { v25.b }[4], [x25]\n"
+ "ld1 { v21.b }[4], [x24]\n"
+ "ld1 { v22.b }[4], [x23]\n"
+ "ld1 { v18.b }[4], [x22]\n"
+ "ld1 { v19.b }[4], [x21]\n"
+ "ld1 { v16.b }[4], [x20]\n"
+ "b 11f\n"
+ "9:" // odd_loads_2_0
+ "tbz %x[width], #1, 10f\n"
+ "ldr h27, [x27], #0x2\n"
+ "ldr h24, [x26], #0x2\n"
+ "ldr h25, [x25], #0x2\n"
+ "ldr h21, [x24], #0x2\n"
+ "ldr h22, [x23], #0x2\n"
+ "ldr h18, [x22], #0x2\n"
+ "ldr h19, [x21], #0x2\n"
+ "ldr h16, [x20], #0x2\n"
+ "mov x19, #0x1\n"
+ "tbz %x[width], #0, 11f\n"
+ "ld1 { v27.b }[2], [x27]\n"
+ "ld1 { v24.b }[2], [x26]\n"
+ "ld1 { v25.b }[2], [x25]\n"
+ "ld1 { v21.b }[2], [x24]\n"
+ "ld1 { v22.b }[2], [x23]\n"
+ "ld1 { v18.b }[2], [x22]\n"
+ "ld1 { v19.b }[2], [x21]\n"
+ "ld1 { v16.b }[2], [x20]\n"
+ "b 11f\n"
+ "10:" // odd_loads_1_0
+ "ldr b27, [x27, #0x0]\n"
+ "ldr b24, [x26, #0x0]\n"
+ "ldr b25, [x25, #0x0]\n"
+ "ldr b21, [x24, #0x0]\n"
+ "ldr b22, [x23, #0x0]\n"
+ "ldr b18, [x22, #0x0]\n"
+ "ldr b19, [x21, #0x0]\n"
+ "ldr b16, [x20, #0x0]\n"
+ "mov x19, #0x1\n"
+ "11:" // Odd load end
+ "zip1 v26.2d, v27.2d, v24.2d\n"
+ "subs x19, x19, #0x1\n"
+ "zip1 v23.2d, v25.2d, v21.2d\n"
+ "str q26, [%x[out_ptr], #0x0]\n"
+ "zip1 v20.2d, v22.2d, v18.2d\n"
+ "str q23, [%x[out_ptr], #0x10]\n"
+ "zip1 v17.2d, v19.2d, v16.2d\n"
+ "str q20, [%x[out_ptr], #0x20]\n"
+ "str q17, [%x[out_ptr], #0x30]\n"
+ "add %x[out_ptr], %x[out_ptr], #0x40\n"
+ "beq 12f\n"
+ "zip2 v24.2d, v27.2d, v24.2d\n"
+ "zip2 v21.2d, v25.2d, v21.2d\n"
+ "str q24, [%x[out_ptr], #0x0]\n"
+ "zip2 v18.2d, v22.2d, v18.2d\n"
+ "str q21, [%x[out_ptr], #0x10]\n"
+ "zip2 v16.2d, v19.2d, v16.2d\n"
+ "str q18, [%x[out_ptr], #0x20]\n"
+ "str q16, [%x[out_ptr], #0x30]\n"
+ "add %x[out_ptr], %x[out_ptr], #0x40\n"
+ "12:" // Odds skip
+
+ : [out_ptr] "+r" (out_ptr), [width] "+r" (width)
+ : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset)
+ : "cc", "memory", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27"
+ );
+}
+
+template<>
+void interleave_block<8, 8, VLType::None, false>(
+ uint8_t * &out_ptr, const uint8_t * const * in, size_t width, size_t height,
+ size_t row_offset, bool
+)
+{
+ int8_t * &out_cast = reinterpret_cast<int8_t * &>(out_ptr);
+ const int8_t * const * in_cast = reinterpret_cast<const int8_t * const *>(in);
+
+ interleave_block<8, 8, VLType::None, false>(out_cast, in_cast, width, height, row_offset, false);
+}
+
+
+#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block8_s8_s8_summing.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block8_s8_s8_summing.hpp
new file mode 100644
index 0000000..2317ece
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block8_s8_s8_summing.hpp
@@ -0,0 +1,362 @@
+/*
+ * Copyright (c) 2019-2020 Arm Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+#ifdef __aarch64__
+
+template<>
+void interleave_block<8, 8, VLType::None, true>(
+ int8_t * &out_ptr, const int8_t * const * in, size_t width, size_t height,
+ size_t row_offset, bool first
+)
+{
+ __asm__ __volatile__(
+ "movi v5.8h, #0x0\n"
+ "ldr x27, [%x[in], #0x0]\n"
+ "mov x19, #0x0\n"
+ "movi v4.8h, #0x0\n"
+ "ldr x26, [%x[in], #0x8]\n"
+ "cmp %x[height], #0x8\n"
+ "movi v3.8h, #0x0\n"
+ "ldr x25, [%x[in], #0x10]\n"
+ "add x27, x27, %x[row_offset]\n"
+ "movi v2.8h, #0x0\n"
+ "ldr x24, [%x[in], #0x18]\n"
+ "movi v1.4s, #0x0\n"
+ "ldr x23, [%x[in], #0x20]\n"
+ "add x26, x26, %x[row_offset]\n"
+ "movi v0.4s, #0x0\n"
+ "ldr x22, [%x[in], #0x28]\n"
+ "add x25, x25, %x[row_offset]\n"
+ "movi v31.4s, #0x0\n"
+ "ldr x21, [%x[in], #0x30]\n"
+ "add x24, x24, %x[row_offset]\n"
+ "movi v30.4s, #0x0\n"
+ "ldr x20, [%x[in], #0x38]\n"
+ "add x23, x23, %x[row_offset]\n"
+ "add x22, x22, %x[row_offset]\n"
+ "add x21, x21, %x[row_offset]\n"
+ "add x20, x20, %x[row_offset]\n"
+ "beq 1f\n"
+ "mov x20, x27\n"
+ "cmp %x[height], #0x2\n"
+ "csel x26, x26, x27, GE\n"
+ "csel x25, x25, x27, GT\n"
+ "cmp %x[height], #0x4\n"
+ "csel x24, x24, x27, GE\n"
+ "csel x23, x23, x27, GT\n"
+ "cmp %x[height], #0x6\n"
+ "csel x22, x22, x27, GE\n"
+ "csel x21, x21, x27, GT\n"
+ "1:" // no_pointer_adj
+ "movi v29.4s, #0x0\n"
+ "prfm pldl1keep, [x27, #0x0]\n"
+ "movi v28.4s, #0x0\n"
+ "prfm pldl1keep, [x26, #0x0]\n"
+ "prfm pldl1keep, [x25, #0x0]\n"
+ "prfm pldl1keep, [x24, #0x0]\n"
+ "prfm pldl1keep, [x23, #0x0]\n"
+ "prfm pldl1keep, [x22, #0x0]\n"
+ "prfm pldl1keep, [x21, #0x0]\n"
+ "prfm pldl1keep, [x20, #0x0]\n"
+ "prfm pldl1keep, [x27, #0x40]\n"
+ "prfm pldl1keep, [x26, #0x40]\n"
+ "prfm pldl1keep, [x25, #0x40]\n"
+ "prfm pldl1keep, [x24, #0x40]\n"
+ "prfm pldl1keep, [x23, #0x40]\n"
+ "prfm pldl1keep, [x22, #0x40]\n"
+ "prfm pldl1keep, [x21, #0x40]\n"
+ "prfm pldl1keep, [x20, #0x40]\n"
+ "cbnz %w[first], 2f\n"
+ "sub %x[out_ptr], %x[out_ptr], #0x20\n"
+ "ld1 { v29.4s }, [%x[out_ptr]]\n"
+ "ldr q28, [%x[out_ptr], #0x10]\n"
+ "2:" // first_pass
+ "cmp %x[width], #0x10\n"
+ "blt 5f\n"
+ "3:" // Main loop head
+ "cmp x19, #0x3e\n"
+ "ble 4f\n"
+ "sadalp v1.4s, v5.8h\n"
+ "movi v5.8h, #0x0\n"
+ "sadalp v0.4s, v4.8h\n"
+ "movi v4.8h, #0x0\n"
+ "sadalp v31.4s, v3.8h\n"
+ "movi v3.8h, #0x0\n"
+ "sadalp v30.4s, v2.8h\n"
+ "movi v2.8h, #0x0\n"
+ "mov x19, #0x0\n"
+ "4:" // no_accumulate_16
+ "ldr q27, [x27], #0x10\n"
+ "prfm pldl1keep, [x27, #0x70]\n"
+ "ldr q24, [x26], #0x10\n"
+ "zip1 v26.2d, v27.2d, v24.2d\n"
+ "prfm pldl1keep, [x26, #0x70]\n"
+ "ldr q25, [x25], #0x10\n"
+ "zip2 v24.2d, v27.2d, v24.2d\n"
+ "prfm pldl1keep, [x25, #0x70]\n"
+ "ldr q21, [x24], #0x10\n"
+ "zip1 v23.2d, v25.2d, v21.2d\n"
+ "prfm pldl1keep, [x24, #0x70]\n"
+ "ldr q22, [x23], #0x10\n"
+ "zip2 v21.2d, v25.2d, v21.2d\n"
+ "prfm pldl1keep, [x23, #0x70]\n"
+ "ldr q18, [x22], #0x10\n"
+ "zip1 v20.2d, v22.2d, v18.2d\n"
+ "prfm pldl1keep, [x22, #0x70]\n"
+ "ldr q19, [x21], #0x10\n"
+ "zip2 v18.2d, v22.2d, v18.2d\n"
+ "prfm pldl1keep, [x21, #0x70]\n"
+ "ldr q16, [x20], #0x10\n"
+ "zip1 v17.2d, v19.2d, v16.2d\n"
+ "prfm pldl1keep, [x20, #0x70]\n"
+ "str q26, [%x[out_ptr], #0x0]\n"
+ "zip2 v16.2d, v19.2d, v16.2d\n"
+ "sadalp v5.8h, v26.16b\n"
+ "str q23, [%x[out_ptr], #0x10]\n"
+ "sadalp v4.8h, v23.16b\n"
+ "str q20, [%x[out_ptr], #0x20]\n"
+ "sadalp v3.8h, v20.16b\n"
+ "str q17, [%x[out_ptr], #0x30]\n"
+ "sadalp v2.8h, v17.16b\n"
+ "str q24, [%x[out_ptr], #0x40]\n"
+ "sadalp v5.8h, v24.16b\n"
+ "str q21, [%x[out_ptr], #0x50]\n"
+ "sadalp v4.8h, v21.16b\n"
+ "str q18, [%x[out_ptr], #0x60]\n"
+ "sadalp v3.8h, v18.16b\n"
+ "str q16, [%x[out_ptr], #0x70]\n"
+ "sadalp v2.8h, v16.16b\n"
+ "add x19, x19, #0x1\n"
+ "subs %x[width], %x[width], #0x10\n"
+ "cmp %x[width], #0x10\n"
+ "add %x[out_ptr], %x[out_ptr], #0x80\n"
+ "bge 3b\n"
+ "5:" // Main loop skip
+ "cbz %x[width], 14f\n"
+ "tbz %x[width], #3, 9f\n"
+ "ldr d27, [x27], #0x8\n"
+ "ldr d24, [x26], #0x8\n"
+ "ldr d25, [x25], #0x8\n"
+ "ldr d21, [x24], #0x8\n"
+ "ldr d22, [x23], #0x8\n"
+ "ldr d18, [x22], #0x8\n"
+ "ldr d19, [x21], #0x8\n"
+ "ldr d16, [x20], #0x8\n"
+ "tbz %x[width], #2, 7f\n"
+ "ld1 { v27.s }[2], [x27], #0x4\n"
+ "ld1 { v24.s }[2], [x26], #0x4\n"
+ "ld1 { v25.s }[2], [x25], #0x4\n"
+ "ld1 { v21.s }[2], [x24], #0x4\n"
+ "ld1 { v22.s }[2], [x23], #0x4\n"
+ "ld1 { v18.s }[2], [x22], #0x4\n"
+ "ld1 { v19.s }[2], [x21], #0x4\n"
+ "ld1 { v16.s }[2], [x20], #0x4\n"
+ "tbz %x[width], #1, 6f\n"
+ "ld1 { v27.h }[6], [x27], #0x2\n"
+ "ld1 { v24.h }[6], [x26], #0x2\n"
+ "ld1 { v25.h }[6], [x25], #0x2\n"
+ "ld1 { v21.h }[6], [x24], #0x2\n"
+ "ld1 { v22.h }[6], [x23], #0x2\n"
+ "ld1 { v18.h }[6], [x22], #0x2\n"
+ "ld1 { v19.h }[6], [x21], #0x2\n"
+ "ld1 { v16.h }[6], [x20], #0x2\n"
+ "mov x19, #0x2\n"
+ "tbz %x[width], #0, 13f\n"
+ "ld1 { v27.b }[14], [x27]\n"
+ "ld1 { v24.b }[14], [x26]\n"
+ "ld1 { v25.b }[14], [x25]\n"
+ "ld1 { v21.b }[14], [x24]\n"
+ "ld1 { v22.b }[14], [x23]\n"
+ "ld1 { v18.b }[14], [x22]\n"
+ "ld1 { v19.b }[14], [x21]\n"
+ "ld1 { v16.b }[14], [x20]\n"
+ "b 13f\n"
+ "6:" // odd_loads_1_12
+ "mov x19, #0x2\n"
+ "tbz %x[width], #0, 13f\n"
+ "ld1 { v27.b }[12], [x27]\n"
+ "ld1 { v24.b }[12], [x26]\n"
+ "ld1 { v25.b }[12], [x25]\n"
+ "ld1 { v21.b }[12], [x24]\n"
+ "ld1 { v22.b }[12], [x23]\n"
+ "ld1 { v18.b }[12], [x22]\n"
+ "ld1 { v19.b }[12], [x21]\n"
+ "ld1 { v16.b }[12], [x20]\n"
+ "b 13f\n"
+ "7:" // odd_loads_2_8
+ "tbz %x[width], #1, 8f\n"
+ "ld1 { v27.h }[4], [x27], #0x2\n"
+ "ld1 { v24.h }[4], [x26], #0x2\n"
+ "ld1 { v25.h }[4], [x25], #0x2\n"
+ "ld1 { v21.h }[4], [x24], #0x2\n"
+ "ld1 { v22.h }[4], [x23], #0x2\n"
+ "ld1 { v18.h }[4], [x22], #0x2\n"
+ "ld1 { v19.h }[4], [x21], #0x2\n"
+ "ld1 { v16.h }[4], [x20], #0x2\n"
+ "mov x19, #0x2\n"
+ "tbz %x[width], #0, 13f\n"
+ "ld1 { v27.b }[10], [x27]\n"
+ "ld1 { v24.b }[10], [x26]\n"
+ "ld1 { v25.b }[10], [x25]\n"
+ "ld1 { v21.b }[10], [x24]\n"
+ "ld1 { v22.b }[10], [x23]\n"
+ "ld1 { v18.b }[10], [x22]\n"
+ "ld1 { v19.b }[10], [x21]\n"
+ "ld1 { v16.b }[10], [x20]\n"
+ "b 13f\n"
+ "8:" // odd_loads_1_8
+ "mov x19, #0x1\n"
+ "tbz %x[width], #0, 13f\n"
+ "ld1 { v27.b }[8], [x27]\n"
+ "ld1 { v24.b }[8], [x26]\n"
+ "ld1 { v25.b }[8], [x25]\n"
+ "ld1 { v21.b }[8], [x24]\n"
+ "ld1 { v22.b }[8], [x23]\n"
+ "ld1 { v18.b }[8], [x22]\n"
+ "ld1 { v19.b }[8], [x21]\n"
+ "ld1 { v16.b }[8], [x20]\n"
+ "mov x19, #0x2\n"
+ "b 13f\n"
+ "9:" // odd_loads_4_0
+ "tbz %x[width], #2, 11f\n"
+ "ldr s27, [x27], #0x4\n"
+ "ldr s24, [x26], #0x4\n"
+ "ldr s25, [x25], #0x4\n"
+ "ldr s21, [x24], #0x4\n"
+ "ldr s22, [x23], #0x4\n"
+ "ldr s18, [x22], #0x4\n"
+ "ldr s19, [x21], #0x4\n"
+ "ldr s16, [x20], #0x4\n"
+ "tbz %x[width], #1, 10f\n"
+ "ld1 { v27.h }[2], [x27], #0x2\n"
+ "ld1 { v24.h }[2], [x26], #0x2\n"
+ "ld1 { v25.h }[2], [x25], #0x2\n"
+ "ld1 { v21.h }[2], [x24], #0x2\n"
+ "ld1 { v22.h }[2], [x23], #0x2\n"
+ "ld1 { v18.h }[2], [x22], #0x2\n"
+ "ld1 { v19.h }[2], [x21], #0x2\n"
+ "ld1 { v16.h }[2], [x20], #0x2\n"
+ "mov x19, #0x1\n"
+ "tbz %x[width], #0, 13f\n"
+ "ld1 { v27.b }[6], [x27]\n"
+ "ld1 { v24.b }[6], [x26]\n"
+ "ld1 { v25.b }[6], [x25]\n"
+ "ld1 { v21.b }[6], [x24]\n"
+ "ld1 { v22.b }[6], [x23]\n"
+ "ld1 { v18.b }[6], [x22]\n"
+ "ld1 { v19.b }[6], [x21]\n"
+ "ld1 { v16.b }[6], [x20]\n"
+ "b 13f\n"
+ "10:" // odd_loads_1_4
+ "mov x19, #0x1\n"
+ "tbz %x[width], #0, 13f\n"
+ "ld1 { v27.b }[4], [x27]\n"
+ "ld1 { v24.b }[4], [x26]\n"
+ "ld1 { v25.b }[4], [x25]\n"
+ "ld1 { v21.b }[4], [x24]\n"
+ "ld1 { v22.b }[4], [x23]\n"
+ "ld1 { v18.b }[4], [x22]\n"
+ "ld1 { v19.b }[4], [x21]\n"
+ "ld1 { v16.b }[4], [x20]\n"
+ "b 13f\n"
+ "11:" // odd_loads_2_0
+ "tbz %x[width], #1, 12f\n"
+ "ldr h27, [x27], #0x2\n"
+ "ldr h24, [x26], #0x2\n"
+ "ldr h25, [x25], #0x2\n"
+ "ldr h21, [x24], #0x2\n"
+ "ldr h22, [x23], #0x2\n"
+ "ldr h18, [x22], #0x2\n"
+ "ldr h19, [x21], #0x2\n"
+ "ldr h16, [x20], #0x2\n"
+ "mov x19, #0x1\n"
+ "tbz %x[width], #0, 13f\n"
+ "ld1 { v27.b }[2], [x27]\n"
+ "ld1 { v24.b }[2], [x26]\n"
+ "ld1 { v25.b }[2], [x25]\n"
+ "ld1 { v21.b }[2], [x24]\n"
+ "ld1 { v22.b }[2], [x23]\n"
+ "ld1 { v18.b }[2], [x22]\n"
+ "ld1 { v19.b }[2], [x21]\n"
+ "ld1 { v16.b }[2], [x20]\n"
+ "b 13f\n"
+ "12:" // odd_loads_1_0
+ "ldr b27, [x27, #0x0]\n"
+ "ldr b24, [x26, #0x0]\n"
+ "ldr b25, [x25, #0x0]\n"
+ "ldr b21, [x24, #0x0]\n"
+ "ldr b22, [x23, #0x0]\n"
+ "ldr b18, [x22, #0x0]\n"
+ "ldr b19, [x21, #0x0]\n"
+ "ldr b16, [x20, #0x0]\n"
+ "mov x19, #0x1\n"
+ "13:" // Odd load end
+ "zip1 v26.2d, v27.2d, v24.2d\n"
+ "subs x19, x19, #0x1\n"
+ "zip1 v23.2d, v25.2d, v21.2d\n"
+ "str q26, [%x[out_ptr], #0x0]\n"
+ "zip1 v20.2d, v22.2d, v18.2d\n"
+ "sadalp v5.8h, v26.16b\n"
+ "zip1 v17.2d, v19.2d, v16.2d\n"
+ "str q23, [%x[out_ptr], #0x10]\n"
+ "sadalp v4.8h, v23.16b\n"
+ "str q20, [%x[out_ptr], #0x20]\n"
+ "sadalp v3.8h, v20.16b\n"
+ "str q17, [%x[out_ptr], #0x30]\n"
+ "sadalp v2.8h, v17.16b\n"
+ "add %x[out_ptr], %x[out_ptr], #0x40\n"
+ "beq 14f\n"
+ "zip2 v24.2d, v27.2d, v24.2d\n"
+ "zip2 v21.2d, v25.2d, v21.2d\n"
+ "str q24, [%x[out_ptr], #0x0]\n"
+ "zip2 v18.2d, v22.2d, v18.2d\n"
+ "sadalp v5.8h, v24.16b\n"
+ "zip2 v16.2d, v19.2d, v16.2d\n"
+ "str q21, [%x[out_ptr], #0x10]\n"
+ "sadalp v4.8h, v21.16b\n"
+ "str q18, [%x[out_ptr], #0x20]\n"
+ "sadalp v3.8h, v18.16b\n"
+ "str q16, [%x[out_ptr], #0x30]\n"
+ "sadalp v2.8h, v16.16b\n"
+ "add %x[out_ptr], %x[out_ptr], #0x40\n"
+ "14:" // Odds skip
+ "sadalp v1.4s, v5.8h\n"
+ "sadalp v0.4s, v4.8h\n"
+ "addp v1.4s, v1.4s, v0.4s\n"
+ "sadalp v31.4s, v3.8h\n"
+ "sadalp v30.4s, v2.8h\n"
+ "add v1.4s, v1.4s, v29.4s\n"
+ "str q1, [%x[out_ptr], #0x0]\n"
+ "addp v0.4s, v31.4s, v30.4s\n"
+ "add v0.4s, v0.4s, v28.4s\n"
+ "str q0, [%x[out_ptr], #0x10]\n"
+ "add %x[out_ptr], %x[out_ptr], #0x20\n"
+ : [out_ptr] "+r" (out_ptr), [width] "+r" (width)
+ : [first] "r" (first), [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset)
+ : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27"
+ );
+}
+
+
+#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block8_u8_u8_summing.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block8_u8_u8_summing.hpp
new file mode 100644
index 0000000..07164d6
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block8_u8_u8_summing.hpp
@@ -0,0 +1,362 @@
+/*
+ * Copyright (c) 2019-2020 Arm Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+#ifdef __aarch64__
+
+template<>
+void interleave_block<8, 8, VLType::None, true>(
+ uint8_t * &out_ptr, const uint8_t * const * in, size_t width, size_t height,
+ size_t row_offset, bool first
+)
+{
+ __asm__ __volatile__(
+ "movi v5.8h, #0x0\n"
+ "ldr x27, [%x[in], #0x0]\n"
+ "mov x19, #0x0\n"
+ "movi v4.8h, #0x0\n"
+ "ldr x26, [%x[in], #0x8]\n"
+ "cmp %x[height], #0x8\n"
+ "movi v3.8h, #0x0\n"
+ "ldr x25, [%x[in], #0x10]\n"
+ "add x27, x27, %x[row_offset]\n"
+ "movi v2.8h, #0x0\n"
+ "ldr x24, [%x[in], #0x18]\n"
+ "movi v1.4s, #0x0\n"
+ "ldr x23, [%x[in], #0x20]\n"
+ "add x26, x26, %x[row_offset]\n"
+ "movi v0.4s, #0x0\n"
+ "ldr x22, [%x[in], #0x28]\n"
+ "add x25, x25, %x[row_offset]\n"
+ "movi v31.4s, #0x0\n"
+ "ldr x21, [%x[in], #0x30]\n"
+ "add x24, x24, %x[row_offset]\n"
+ "movi v30.4s, #0x0\n"
+ "ldr x20, [%x[in], #0x38]\n"
+ "add x23, x23, %x[row_offset]\n"
+ "add x22, x22, %x[row_offset]\n"
+ "add x21, x21, %x[row_offset]\n"
+ "add x20, x20, %x[row_offset]\n"
+ "beq 1f\n"
+ "mov x20, x27\n"
+ "cmp %x[height], #0x2\n"
+ "csel x26, x26, x27, GE\n"
+ "csel x25, x25, x27, GT\n"
+ "cmp %x[height], #0x4\n"
+ "csel x24, x24, x27, GE\n"
+ "csel x23, x23, x27, GT\n"
+ "cmp %x[height], #0x6\n"
+ "csel x22, x22, x27, GE\n"
+ "csel x21, x21, x27, GT\n"
+ "1:" // no_pointer_adj
+ "movi v29.4s, #0x0\n"
+ "prfm pldl1keep, [x27, #0x0]\n"
+ "movi v28.4s, #0x0\n"
+ "prfm pldl1keep, [x26, #0x0]\n"
+ "prfm pldl1keep, [x25, #0x0]\n"
+ "prfm pldl1keep, [x24, #0x0]\n"
+ "prfm pldl1keep, [x23, #0x0]\n"
+ "prfm pldl1keep, [x22, #0x0]\n"
+ "prfm pldl1keep, [x21, #0x0]\n"
+ "prfm pldl1keep, [x20, #0x0]\n"
+ "prfm pldl1keep, [x27, #0x40]\n"
+ "prfm pldl1keep, [x26, #0x40]\n"
+ "prfm pldl1keep, [x25, #0x40]\n"
+ "prfm pldl1keep, [x24, #0x40]\n"
+ "prfm pldl1keep, [x23, #0x40]\n"
+ "prfm pldl1keep, [x22, #0x40]\n"
+ "prfm pldl1keep, [x21, #0x40]\n"
+ "prfm pldl1keep, [x20, #0x40]\n"
+ "cbnz %w[first], 2f\n"
+ "sub %x[out_ptr], %x[out_ptr], #0x20\n"
+ "ld1 { v29.4s }, [%x[out_ptr]]\n"
+ "ldr q28, [%x[out_ptr], #0x10]\n"
+ "2:" // first_pass
+ "cmp %x[width], #0x10\n"
+ "blt 5f\n"
+ "3:" // Main loop head
+ "cmp x19, #0x3e\n"
+ "ble 4f\n"
+ "uadalp v1.4s, v5.8h\n"
+ "movi v5.8h, #0x0\n"
+ "uadalp v0.4s, v4.8h\n"
+ "movi v4.8h, #0x0\n"
+ "uadalp v31.4s, v3.8h\n"
+ "movi v3.8h, #0x0\n"
+ "uadalp v30.4s, v2.8h\n"
+ "movi v2.8h, #0x0\n"
+ "mov x19, #0x0\n"
+ "4:" // no_accumulate_16
+ "ldr q27, [x27], #0x10\n"
+ "prfm pldl1keep, [x27, #0x70]\n"
+ "ldr q24, [x26], #0x10\n"
+ "zip1 v26.2d, v27.2d, v24.2d\n"
+ "prfm pldl1keep, [x26, #0x70]\n"
+ "ldr q25, [x25], #0x10\n"
+ "zip2 v24.2d, v27.2d, v24.2d\n"
+ "prfm pldl1keep, [x25, #0x70]\n"
+ "ldr q21, [x24], #0x10\n"
+ "zip1 v23.2d, v25.2d, v21.2d\n"
+ "prfm pldl1keep, [x24, #0x70]\n"
+ "ldr q22, [x23], #0x10\n"
+ "zip2 v21.2d, v25.2d, v21.2d\n"
+ "prfm pldl1keep, [x23, #0x70]\n"
+ "ldr q18, [x22], #0x10\n"
+ "zip1 v20.2d, v22.2d, v18.2d\n"
+ "prfm pldl1keep, [x22, #0x70]\n"
+ "ldr q19, [x21], #0x10\n"
+ "zip2 v18.2d, v22.2d, v18.2d\n"
+ "prfm pldl1keep, [x21, #0x70]\n"
+ "ldr q16, [x20], #0x10\n"
+ "zip1 v17.2d, v19.2d, v16.2d\n"
+ "prfm pldl1keep, [x20, #0x70]\n"
+ "str q26, [%x[out_ptr], #0x0]\n"
+ "zip2 v16.2d, v19.2d, v16.2d\n"
+ "uadalp v5.8h, v26.16b\n"
+ "str q23, [%x[out_ptr], #0x10]\n"
+ "uadalp v4.8h, v23.16b\n"
+ "str q20, [%x[out_ptr], #0x20]\n"
+ "uadalp v3.8h, v20.16b\n"
+ "str q17, [%x[out_ptr], #0x30]\n"
+ "uadalp v2.8h, v17.16b\n"
+ "str q24, [%x[out_ptr], #0x40]\n"
+ "uadalp v5.8h, v24.16b\n"
+ "str q21, [%x[out_ptr], #0x50]\n"
+ "uadalp v4.8h, v21.16b\n"
+ "str q18, [%x[out_ptr], #0x60]\n"
+ "uadalp v3.8h, v18.16b\n"
+ "str q16, [%x[out_ptr], #0x70]\n"
+ "uadalp v2.8h, v16.16b\n"
+ "add x19, x19, #0x1\n"
+ "subs %x[width], %x[width], #0x10\n"
+ "cmp %x[width], #0x10\n"
+ "add %x[out_ptr], %x[out_ptr], #0x80\n"
+ "bge 3b\n"
+ "5:" // Main loop skip
+ "cbz %x[width], 14f\n"
+ "tbz %x[width], #3, 9f\n"
+ "ldr d27, [x27], #0x8\n"
+ "ldr d24, [x26], #0x8\n"
+ "ldr d25, [x25], #0x8\n"
+ "ldr d21, [x24], #0x8\n"
+ "ldr d22, [x23], #0x8\n"
+ "ldr d18, [x22], #0x8\n"
+ "ldr d19, [x21], #0x8\n"
+ "ldr d16, [x20], #0x8\n"
+ "tbz %x[width], #2, 7f\n"
+ "ld1 { v27.s }[2], [x27], #0x4\n"
+ "ld1 { v24.s }[2], [x26], #0x4\n"
+ "ld1 { v25.s }[2], [x25], #0x4\n"
+ "ld1 { v21.s }[2], [x24], #0x4\n"
+ "ld1 { v22.s }[2], [x23], #0x4\n"
+ "ld1 { v18.s }[2], [x22], #0x4\n"
+ "ld1 { v19.s }[2], [x21], #0x4\n"
+ "ld1 { v16.s }[2], [x20], #0x4\n"
+ "tbz %x[width], #1, 6f\n"
+ "ld1 { v27.h }[6], [x27], #0x2\n"
+ "ld1 { v24.h }[6], [x26], #0x2\n"
+ "ld1 { v25.h }[6], [x25], #0x2\n"
+ "ld1 { v21.h }[6], [x24], #0x2\n"
+ "ld1 { v22.h }[6], [x23], #0x2\n"
+ "ld1 { v18.h }[6], [x22], #0x2\n"
+ "ld1 { v19.h }[6], [x21], #0x2\n"
+ "ld1 { v16.h }[6], [x20], #0x2\n"
+ "mov x19, #0x2\n"
+ "tbz %x[width], #0, 13f\n"
+ "ld1 { v27.b }[14], [x27]\n"
+ "ld1 { v24.b }[14], [x26]\n"
+ "ld1 { v25.b }[14], [x25]\n"
+ "ld1 { v21.b }[14], [x24]\n"
+ "ld1 { v22.b }[14], [x23]\n"
+ "ld1 { v18.b }[14], [x22]\n"
+ "ld1 { v19.b }[14], [x21]\n"
+ "ld1 { v16.b }[14], [x20]\n"
+ "b 13f\n"
+ "6:" // odd_loads_1_12
+ "mov x19, #0x2\n"
+ "tbz %x[width], #0, 13f\n"
+ "ld1 { v27.b }[12], [x27]\n"
+ "ld1 { v24.b }[12], [x26]\n"
+ "ld1 { v25.b }[12], [x25]\n"
+ "ld1 { v21.b }[12], [x24]\n"
+ "ld1 { v22.b }[12], [x23]\n"
+ "ld1 { v18.b }[12], [x22]\n"
+ "ld1 { v19.b }[12], [x21]\n"
+ "ld1 { v16.b }[12], [x20]\n"
+ "b 13f\n"
+ "7:" // odd_loads_2_8
+ "tbz %x[width], #1, 8f\n"
+ "ld1 { v27.h }[4], [x27], #0x2\n"
+ "ld1 { v24.h }[4], [x26], #0x2\n"
+ "ld1 { v25.h }[4], [x25], #0x2\n"
+ "ld1 { v21.h }[4], [x24], #0x2\n"
+ "ld1 { v22.h }[4], [x23], #0x2\n"
+ "ld1 { v18.h }[4], [x22], #0x2\n"
+ "ld1 { v19.h }[4], [x21], #0x2\n"
+ "ld1 { v16.h }[4], [x20], #0x2\n"
+ "mov x19, #0x2\n"
+ "tbz %x[width], #0, 13f\n"
+ "ld1 { v27.b }[10], [x27]\n"
+ "ld1 { v24.b }[10], [x26]\n"
+ "ld1 { v25.b }[10], [x25]\n"
+ "ld1 { v21.b }[10], [x24]\n"
+ "ld1 { v22.b }[10], [x23]\n"
+ "ld1 { v18.b }[10], [x22]\n"
+ "ld1 { v19.b }[10], [x21]\n"
+ "ld1 { v16.b }[10], [x20]\n"
+ "b 13f\n"
+ "8:" // odd_loads_1_8
+ "mov x19, #0x1\n"
+ "tbz %x[width], #0, 13f\n"
+ "ld1 { v27.b }[8], [x27]\n"
+ "ld1 { v24.b }[8], [x26]\n"
+ "ld1 { v25.b }[8], [x25]\n"
+ "ld1 { v21.b }[8], [x24]\n"
+ "ld1 { v22.b }[8], [x23]\n"
+ "ld1 { v18.b }[8], [x22]\n"
+ "ld1 { v19.b }[8], [x21]\n"
+ "ld1 { v16.b }[8], [x20]\n"
+ "mov x19, #0x2\n"
+ "b 13f\n"
+ "9:" // odd_loads_4_0
+ "tbz %x[width], #2, 11f\n"
+ "ldr s27, [x27], #0x4\n"
+ "ldr s24, [x26], #0x4\n"
+ "ldr s25, [x25], #0x4\n"
+ "ldr s21, [x24], #0x4\n"
+ "ldr s22, [x23], #0x4\n"
+ "ldr s18, [x22], #0x4\n"
+ "ldr s19, [x21], #0x4\n"
+ "ldr s16, [x20], #0x4\n"
+ "tbz %x[width], #1, 10f\n"
+ "ld1 { v27.h }[2], [x27], #0x2\n"
+ "ld1 { v24.h }[2], [x26], #0x2\n"
+ "ld1 { v25.h }[2], [x25], #0x2\n"
+ "ld1 { v21.h }[2], [x24], #0x2\n"
+ "ld1 { v22.h }[2], [x23], #0x2\n"
+ "ld1 { v18.h }[2], [x22], #0x2\n"
+ "ld1 { v19.h }[2], [x21], #0x2\n"
+ "ld1 { v16.h }[2], [x20], #0x2\n"
+ "mov x19, #0x1\n"
+ "tbz %x[width], #0, 13f\n"
+ "ld1 { v27.b }[6], [x27]\n"
+ "ld1 { v24.b }[6], [x26]\n"
+ "ld1 { v25.b }[6], [x25]\n"
+ "ld1 { v21.b }[6], [x24]\n"
+ "ld1 { v22.b }[6], [x23]\n"
+ "ld1 { v18.b }[6], [x22]\n"
+ "ld1 { v19.b }[6], [x21]\n"
+ "ld1 { v16.b }[6], [x20]\n"
+ "b 13f\n"
+ "10:" // odd_loads_1_4
+ "mov x19, #0x1\n"
+ "tbz %x[width], #0, 13f\n"
+ "ld1 { v27.b }[4], [x27]\n"
+ "ld1 { v24.b }[4], [x26]\n"
+ "ld1 { v25.b }[4], [x25]\n"
+ "ld1 { v21.b }[4], [x24]\n"
+ "ld1 { v22.b }[4], [x23]\n"
+ "ld1 { v18.b }[4], [x22]\n"
+ "ld1 { v19.b }[4], [x21]\n"
+ "ld1 { v16.b }[4], [x20]\n"
+ "b 13f\n"
+ "11:" // odd_loads_2_0
+ "tbz %x[width], #1, 12f\n"
+ "ldr h27, [x27], #0x2\n"
+ "ldr h24, [x26], #0x2\n"
+ "ldr h25, [x25], #0x2\n"
+ "ldr h21, [x24], #0x2\n"
+ "ldr h22, [x23], #0x2\n"
+ "ldr h18, [x22], #0x2\n"
+ "ldr h19, [x21], #0x2\n"
+ "ldr h16, [x20], #0x2\n"
+ "mov x19, #0x1\n"
+ "tbz %x[width], #0, 13f\n"
+ "ld1 { v27.b }[2], [x27]\n"
+ "ld1 { v24.b }[2], [x26]\n"
+ "ld1 { v25.b }[2], [x25]\n"
+ "ld1 { v21.b }[2], [x24]\n"
+ "ld1 { v22.b }[2], [x23]\n"
+ "ld1 { v18.b }[2], [x22]\n"
+ "ld1 { v19.b }[2], [x21]\n"
+ "ld1 { v16.b }[2], [x20]\n"
+ "b 13f\n"
+ "12:" // odd_loads_1_0
+ "ldr b27, [x27, #0x0]\n"
+ "ldr b24, [x26, #0x0]\n"
+ "ldr b25, [x25, #0x0]\n"
+ "ldr b21, [x24, #0x0]\n"
+ "ldr b22, [x23, #0x0]\n"
+ "ldr b18, [x22, #0x0]\n"
+ "ldr b19, [x21, #0x0]\n"
+ "ldr b16, [x20, #0x0]\n"
+ "mov x19, #0x1\n"
+ "13:" // Odd load end
+ "zip1 v26.2d, v27.2d, v24.2d\n"
+ "subs x19, x19, #0x1\n"
+ "zip1 v23.2d, v25.2d, v21.2d\n"
+ "str q26, [%x[out_ptr], #0x0]\n"
+ "zip1 v20.2d, v22.2d, v18.2d\n"
+ "uadalp v5.8h, v26.16b\n"
+ "zip1 v17.2d, v19.2d, v16.2d\n"
+ "str q23, [%x[out_ptr], #0x10]\n"
+ "uadalp v4.8h, v23.16b\n"
+ "str q20, [%x[out_ptr], #0x20]\n"
+ "uadalp v3.8h, v20.16b\n"
+ "str q17, [%x[out_ptr], #0x30]\n"
+ "uadalp v2.8h, v17.16b\n"
+ "add %x[out_ptr], %x[out_ptr], #0x40\n"
+ "beq 14f\n"
+ "zip2 v24.2d, v27.2d, v24.2d\n"
+ "zip2 v21.2d, v25.2d, v21.2d\n"
+ "str q24, [%x[out_ptr], #0x0]\n"
+ "zip2 v18.2d, v22.2d, v18.2d\n"
+ "uadalp v5.8h, v24.16b\n"
+ "zip2 v16.2d, v19.2d, v16.2d\n"
+ "str q21, [%x[out_ptr], #0x10]\n"
+ "uadalp v4.8h, v21.16b\n"
+ "str q18, [%x[out_ptr], #0x20]\n"
+ "uadalp v3.8h, v18.16b\n"
+ "str q16, [%x[out_ptr], #0x30]\n"
+ "uadalp v2.8h, v16.16b\n"
+ "add %x[out_ptr], %x[out_ptr], #0x40\n"
+ "14:" // Odds skip
+ "uadalp v1.4s, v5.8h\n"
+ "uadalp v0.4s, v4.8h\n"
+ "addp v1.4s, v1.4s, v0.4s\n"
+ "uadalp v31.4s, v3.8h\n"
+ "uadalp v30.4s, v2.8h\n"
+ "add v1.4s, v1.4s, v29.4s\n"
+ "str q1, [%x[out_ptr], #0x0]\n"
+ "addp v0.4s, v31.4s, v30.4s\n"
+ "add v0.4s, v0.4s, v28.4s\n"
+ "str q0, [%x[out_ptr], #0x10]\n"
+ "add %x[out_ptr], %x[out_ptr], #0x20\n"
+ : [out_ptr] "+r" (out_ptr), [width] "+r" (width)
+ : [first] "r" (first), [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset)
+ : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27"
+ );
+}
+
+
+#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/list.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/list.hpp
new file mode 100644
index 0000000..52b49c0
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/list.hpp
@@ -0,0 +1,48 @@
+/*
+ * Copyright (c) 2020 Arm Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "a32_interleave6_block1_fp32_fp32.hpp"
+#include "a64_interleave4_block16_s8_s8.hpp"
+#include "a64_interleave4_block16_s8_s8_summing.hpp"
+#include "a64_interleave4_block16_u8_u8_summing.hpp"
+#include "a64_interleave8_block1_bf16_fp32.hpp"
+#include "a64_interleave8_block1_fp16_fp16.hpp"
+#include "a64_interleave8_block1_fp16_fp32.hpp"
+#include "a64_interleave8_block1_fp32_fp32.hpp"
+#include "a64_interleave8_block1_s16_s16.hpp"
+#include "a64_interleave8_block1_s16_s16_summing.hpp"
+#include "a64_interleave8_block1_s8_s16.hpp"
+#include "a64_interleave8_block1_s8_s16_summing.hpp"
+#include "a64_interleave8_block1_u16_u16_summing.hpp"
+#include "a64_interleave8_block1_u8_u16.hpp"
+#include "a64_interleave8_block1_u8_u16_summing.hpp"
+#include "a64_interleave8_block2_bf16_bf16.hpp"
+#include "a64_interleave8_block2_fp32_fp32.hpp"
+#include "a64_interleave8_block4_bf16_bf16.hpp"
+#include "a64_interleave8_block4_s8_s8.hpp"
+#include "a64_interleave8_block4_s8_s8_summing.hpp"
+#include "a64_interleave8_block4_u8_u8_summing.hpp"
+#include "a64_interleave8_block8_s8_s8.hpp"
+#include "a64_interleave8_block8_s8_s8_summing.hpp"
+#include "a64_interleave8_block8_u8_u8_summing.hpp"
diff --git a/src/core/NEON/kernels/arm_gemm/interleave_indirect.cpp b/src/core/NEON/kernels/arm_gemm/interleave_indirect.cpp
new file mode 100644
index 0000000..2b3e170
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/interleave_indirect.cpp
@@ -0,0 +1,409 @@
+/*
+ * Copyright (c) 2020 Arm Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "asmlib.hpp"
+#include "convolution_parameters.hpp"
+#include "convolver.hpp"
+#include "interleave_indirect.hpp"
+#include "bfloat.hpp"
+
+#include <alloca.h>
+
+#include <algorithm>
+#include <cstddef>
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <tuple>
+#include <type_traits>
+#include <vector>
+
+#include <arm_neon.h>
+
+#include "utils.hpp"
+
+namespace arm_gemm {
+
+/*
+ * Core function that does heavy lifting - interleave 'int_by' rows of width 'width' together.
+ *
+ * 'height' indicates the actual number of rows to interleave, so if it's less than int_by then the remaining
+ * entries are padded (note that this is "GEMM" padding rather than convolution padding, so there is no need to pad
+ * with a particular value.
+ *
+ * Note that it is not expected for this templated version to ever be used - all cases that matter should be
+ * explicitly specialized with an optimized implementation.
+ */
+template<unsigned int height_vectors, unsigned int block, VLType vlt, bool integrate_sums, typename TIn, typename TOut>
+void interleave_block( TOut * &out, const TIn * const *in, size_t width, size_t height, size_t row_offset, bool first) {
+ const unsigned int int_by = height_vectors * (vlt == VLType::SVE ? get_vector_length<TOut>() / block : 1);
+
+ std::vector<int32_t> the_sums;
+
+ if (integrate_sums) {
+ the_sums = std::vector<int32_t>(int_by, 0);
+
+ if (!first) {
+ // In 'integrate sums' mode, we dump the sums at the end on each pass.
+
+ // On the last pass this is correct, but on other passes it is not -
+ // so on the subsequent pass we need to take the output written by
+ // the previous pass as starting point for the sums, and then
+ // overwrite them with new interleaved data.
+ int32_t *out_int32 = reinterpret_cast<int32_t *>(out);
+
+ // Rewind pointer to where we wrote out the sums last time.
+ out_int32 -= int_by;
+
+ // Restore the running sums.
+ memcpy(the_sums.data(), out_int32, int_by * sizeof(int32_t));
+
+ // Update the "real" pointer so that the next output will clobber the old sums.
+ out = reinterpret_cast<TOut *>(out_int32);
+ }
+ }
+
+ for (unsigned int pos=0; pos<width; pos+=block) {
+ for (unsigned int row=0; row<int_by; row++) {
+ // Row out of range - pad 'block' entries.
+ if (row >= height) {
+ for (unsigned int col=0; col<block; col++) {
+ *out++ = 0;
+ }
+ continue;
+ }
+
+ for (unsigned int col=0; col<block; col++) {
+ // Column out of range - pad a single entry
+ if (pos + col >= width) {
+ *out++ = 0;
+ continue;
+ }
+
+ if (integrate_sums) {
+ the_sums[row] += in[row][row_offset + pos + col];
+ }
+
+ *out++ = in[row][row_offset + pos + col];
+ }
+ }
+ }
+
+ if (integrate_sums) {
+ int32_t *out_int32 = reinterpret_cast<int32_t *>(out);
+
+ memcpy(out_int32, the_sums.data(), int_by * sizeof(int32_t));
+
+ out = reinterpret_cast<TOut *>(out_int32 + int_by);
+ }
+}
+
+template<unsigned int height_vectors, unsigned int block, VLType vlt, typename TOut>
+inline void FixupRowSums(TOut * &out, const int32_t row_sum_multiplier) {
+ const unsigned int height = height_vectors * (vlt == VLType::SVE ? get_vector_length<TOut>() / block : 1);
+
+ // If we are integrating row sums, we need to do some fix up, depending on whether the multiplier is non-zero or not.
+ if (row_sum_multiplier) {
+ // Non-zero: interleave_block<>() will have done the sums, so 'out' will point to the start of the
+ // next block (post sums).
+ // We need to go back and apply the multiplier to the computed sums. We don't need to change 'out'.
+ int32_t *out_int32 = reinterpret_cast<int32_t *>(out);
+
+ out_int32 -= height;
+ for (unsigned int i=0; i<height; i++) {
+ out_int32[i] *= row_sum_multiplier;
+ }
+ } else {
+ // Zero: interleave_block<>() will *not* have done the sums, so 'out' will point to the start of the
+ // sum block. We need to insert the (zero) sums, and advance 'out'.
+ int32_t *out_int32 = reinterpret_cast<int32_t *>(out);
+
+ for (unsigned int i=0; i<height; i++) {
+ out_int32[i] = 0;
+ }
+
+ out_int32 += height;
+
+ out = reinterpret_cast<TOut *>(out_int32);
+ }
+}
+
+template<unsigned int height_vectors, unsigned int block, VLType vlt, typename TIn, typename TOut>
+void IndirectInterleave(TOut *out, const TIn * const * const *ptr, unsigned int stringlen,
+ unsigned int rounded_stringlen, const unsigned int y0, const unsigned int ymax,
+ const unsigned int k0, const unsigned int kmax, bool integrate_sums,
+ const int32_t row_sum_multiplier) {
+ const unsigned int height = height_vectors * (vlt == VLType::SVE ? get_vector_length<TOut>() / block : 1);
+
+ // 'interleave_block' implementations are entitled to read a pointer for each row they handle from the input
+ // pointer array, even for out of range rows (although they must not subsequently dereference those pointers for
+ // out of range rows). This allows interleave_block to use techniques like row predication, or loading all
+ // pointers and conditionally overriding the out of range ones.
+
+ // This is problematic in the "pure" indirect case when we get to the last rows, where it can lead to out of
+ // range reads. Avoid this with a local buffer to use in last-rows cases. Use alloca as a std::vector can be
+ // expensive in highly threaded scenarios.
+ const TIn **row_ptrs = reinterpret_cast<const TIn **>(alloca(height * sizeof(const TIn *)));
+
+ // Figure out the starting position based on k0 (with rounded length)
+ unsigned int start_string = k0 / rounded_stringlen;
+ unsigned int start_stringpos = k0 % rounded_stringlen;
+
+ // Process blocks of 'height' height...
+ for (unsigned int ybase = y0; ybase < ymax; ybase+=height) {
+ // Height to process
+ unsigned int active_height = std::min(ymax - ybase, height);
+
+ // Track our progress through the various strings
+ unsigned int k_left = (kmax - k0);
+ unsigned int string = start_string;
+ unsigned int stringpos = start_stringpos;
+
+ bool first = true;
+
+ // Prepare to call 'interleave_block' above for each string encompassed by K range
+ while (k_left > 0) {
+ // Width to process - and the width we will generate (with padding)
+ unsigned int in_width = std::min(k_left, stringlen - stringpos);
+ unsigned int out_width = std::min(k_left, rounded_stringlen - stringpos);
+
+ const TIn * const *row_base = ptr[string] + ybase;
+
+ // If not all rows are valid, copy the ones that are into local array (see above comment).
+ if (active_height < height) {
+ for (unsigned int i=0; i<active_height; i++) {
+ row_ptrs[i] = ptr[string][ybase + i];
+ }
+
+ row_base = row_ptrs;
+ }
+
+ // 'integrate_sums' is a function parameter rather than a template parameter to prevent duplicating too
+ // much code. However, integrated sums make no sense for non-integral types and won't ever be
+ // requested. So put a type trait check here to avoid generating pointless code.
+ if (std::is_integral<TOut>::value && integrate_sums && row_sum_multiplier) {
+ interleave_block<height_vectors, block, vlt, true>(out, row_base, in_width, active_height, stringpos, first);
+ } else {
+ interleave_block<height_vectors, block, vlt, false>(out, row_base, in_width, active_height, stringpos, first);
+ }
+
+ k_left -= out_width;
+ string++;
+ stringpos=0;
+ first=false;
+ }
+
+ if (std::is_integral<TOut>::value && integrate_sums) {
+ FixupRowSums<height_vectors, block, vlt>(out, row_sum_multiplier);
+ }
+ }
+}
+
+template<unsigned int height_vectors, unsigned int block, VLType vlt, typename TIn, typename TOut>
+void ConvolutionInterleave(TOut *out, const TIn *in, size_t in_stride, const convolver<TIn> &conv, const unsigned int rounded_stringlen,
+ const unsigned int y0, const unsigned int ymax, const unsigned int k0, const unsigned int kmax, bool integrate_sums, const int32_t row_sum_multiplier) {
+ const unsigned int height = height_vectors * (vlt == VLType::SVE ? get_vector_length<TOut>() / block : 1);
+
+ auto conv_cols = conv.process_columns(in, in_stride, k0, kmax, rounded_stringlen);
+
+ // Use alloca here as a std::vector can be expensive in highly threaded scenarios.
+ const TIn **row_ptrs = reinterpret_cast<const TIn **>(alloca(height * sizeof(const TIn *)));
+
+ for (unsigned int ybase = y0; ybase < ymax; ybase += height) {
+ // How many of the rows are active - the rest will get padded in interleave_block.
+ unsigned int active_height = std::min(ymax - ybase, height);
+ bool first = true;
+
+ auto conv_rows = conv_cols.process_rows(ybase, active_height);
+
+ while (!conv_rows.finished()) {
+ unsigned int width, offset;
+
+ // Get next set of parameters
+ std::tie(width, offset) = conv_rows.next_block(row_ptrs);
+
+ // Perform the interleave
+ if (std::is_integral<TOut>::value && integrate_sums && row_sum_multiplier) {
+ interleave_block<height_vectors, block, vlt, true>(out, row_ptrs, width, active_height, offset, first);
+ } else {
+ interleave_block<height_vectors, block, vlt, false>(out, row_ptrs, width, active_height, offset, first);
+ }
+
+ first=false;
+ }
+
+ if (std::is_integral<TOut>::value && integrate_sums) {
+ FixupRowSums<height_vectors, block, vlt>(out, row_sum_multiplier);
+ }
+ }
+}
+
+template<unsigned int height_vectors, unsigned int block, VLType vlt, typename TIn, typename TOut>
+void Interleave(TOut *out, const TIn *in, size_t in_stride, const unsigned int y0, const unsigned int ymax, const unsigned int k0, const unsigned int kmax, bool integrate_sums, const int32_t row_sum_multiplier) {
+ const unsigned int height = height_vectors * (vlt == VLType::SVE ? get_vector_length<TOut>() / block : 1);
+
+ // Use alloca here as a std::vector can be expensive in highly threaded scenarios.
+ const TIn **row_ptrs = reinterpret_cast<const TIn **>(alloca(height * sizeof(const TIn *)));
+
+ const unsigned int width=kmax-k0;
+
+ for (unsigned int y=y0; y<ymax; y+=height) {
+ for (unsigned int r=0; r<height; r++) {
+ row_ptrs[r] = in + ((y + r) * in_stride);
+ }
+
+ if (std::is_integral<TOut>::value && integrate_sums && row_sum_multiplier) {
+ interleave_block<height_vectors, block, vlt, true>(out, row_ptrs, width, std::min(height, ymax-y), k0, true);
+ } else {
+ interleave_block<height_vectors, block, vlt, false>(out, row_ptrs, width, std::min(height, ymax-y), k0, true);
+ }
+
+ if (std::is_integral<TOut>::value && integrate_sums) {
+ FixupRowSums<height_vectors, block, vlt>(out, row_sum_multiplier);
+ }
+ }
+}
+
+#include "indirect-interleaves/list.hpp"
+
+/**** Instantiate needed implementations ****/
+
+/* AArch32 */
+#ifdef __arm__
+/* FP32 */
+/* NEON implementation (height 6) */
+template void IndirectInterleave<6, 1, VLType::None>(float *, const float * const * const *, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, bool, int32_t);
+template void ConvolutionInterleave<6, 1, VLType::None>(float *, const float *, size_t, const convolver<float> &, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, bool, int32_t);
+template void Interleave<6, 1, VLType::None>(float *, const float *, size_t, unsigned int, unsigned int, unsigned int, unsigned int, bool, int32_t);
+
+/* FP16 */
+#if __ARM_FP16_ARGS
+/* NEON implementation using FP32 kernel (height 6) */
+template void IndirectInterleave<6, 1, VLType::None>(float *, const __fp16 * const * const *, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, bool, int32_t);
+template void ConvolutionInterleave<6, 1, VLType::None>(float *, const __fp16 *, size_t, const convolver<__fp16> &, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, bool, int32_t);
+template void Interleave<6, 1, VLType::None>(float *, const __fp16 *, size_t, unsigned int, unsigned int, unsigned int, unsigned int, bool, int32_t);
+#endif /* __ARM_FP16_ARGS */
+
+/* BF16 */
+/* NEON implementation using FP32 kernel */
+template void IndirectInterleave<6, 1, VLType::None>(float *, const bfloat16 * const * const *, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, bool, int32_t);
+template void ConvolutionInterleave<6, 1, VLType::None>(float *, const bfloat16 *, size_t, const convolver<bfloat16> &, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, bool, int32_t);
+template void Interleave<6, 1, VLType::None>(float *, const bfloat16 *, size_t, unsigned int, unsigned int, unsigned int, unsigned int, bool, int32_t);
+#endif
+
+/* AArch64 */
+#ifdef __aarch64__
+/* FP64 */
+/* NEON/SVE implementation (height 8) */
+template void IndirectInterleave<8, 1, VLType::None>(double *, const double * const * const *, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, bool, int32_t);
+template void ConvolutionInterleave<8, 1, VLType::None>(double *, const double *, size_t, const convolver<double> &, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, bool, int32_t);
+template void Interleave<8, 1, VLType::None>(double *, const double *, size_t, unsigned int, unsigned int, unsigned int, unsigned int, bool, int32_t);
+
+/* FP32 */
+/* NEON/SVE implementation (height 8) */
+template void IndirectInterleave<8, 1, VLType::None>(float *, const float * const * const *, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, bool, int32_t);
+template void ConvolutionInterleave<8, 1, VLType::None>(float *, const float *, size_t, const convolver<float> &, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, bool, int32_t);
+template void Interleave<8, 1, VLType::None>(float *, const float *, size_t, unsigned int, unsigned int, unsigned int, unsigned int, bool, int32_t);
+
+/* FMMLA */
+template void IndirectInterleave<8, 2, VLType::None>(float *, const float * const * const *, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, bool, int32_t);
+template void ConvolutionInterleave<8, 2, VLType::None>(float *, const float *, size_t, const convolver<float> &, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, bool, int32_t);
+template void Interleave<8, 2, VLType::None>(float *, const float *, size_t, unsigned int, unsigned int, unsigned int, unsigned int, bool, int32_t);
+
+/* FP16 */
+template void IndirectInterleave<8, 1, VLType::None>(__fp16 *, const __fp16 * const * const *, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, bool, int32_t);
+template void ConvolutionInterleave<8, 1, VLType::None>(__fp16 *, const __fp16 *, size_t, const convolver<__fp16> &, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, bool, int32_t);
+template void Interleave<8, 1, VLType::None>(__fp16 *, const __fp16 *, size_t, unsigned int, unsigned int, unsigned int, unsigned int, bool, int32_t);
+
+template void IndirectInterleave<8, 1, VLType::None>(float *, const __fp16 * const * const *, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, bool, int32_t);
+template void ConvolutionInterleave<8, 1, VLType::None>(float *, const __fp16 *, size_t, const convolver<__fp16> &, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, bool, int32_t);
+template void Interleave<8, 1, VLType::None>(float *, const __fp16 *, size_t, unsigned int, unsigned int, unsigned int, unsigned int, bool, int32_t);
+
+/* BF16 */
+/* NEON/SVE BFDOT */
+template void IndirectInterleave<8, 2, VLType::None>(bfloat16 *, const bfloat16 * const * const *, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, bool, int32_t);
+template void ConvolutionInterleave<8, 2, VLType::None>(bfloat16 *, const bfloat16 *, size_t, const convolver<bfloat16> &, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, bool, int32_t);
+template void Interleave<8, 2, VLType::None>(bfloat16 *, const bfloat16 *, size_t, unsigned int, unsigned int, unsigned int, unsigned int, bool, int32_t);
+
+template void IndirectInterleave<8, 4, VLType::None>(bfloat16 *, const bfloat16 * const * const *, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, bool, int32_t);
+template void ConvolutionInterleave<8, 4, VLType::None>(bfloat16 *, const bfloat16 *, size_t, const convolver<bfloat16> &, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, bool, int32_t);
+template void Interleave<8, 4, VLType::None>(bfloat16 *, const bfloat16 *, size_t, unsigned int, unsigned int, unsigned int, unsigned int, bool, int32_t);
+
+/* NEON/SVE using FP32 kernel */
+template void IndirectInterleave<8, 1, VLType::None>(float *, const bfloat16 * const * const *, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, bool, int32_t);
+template void ConvolutionInterleave<8, 1, VLType::None>(float *, const bfloat16 *, size_t, const convolver<bfloat16> &, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, bool, int32_t);
+template void Interleave<8, 1, VLType::None>(float *, const bfloat16 *, size_t, unsigned int, unsigned int, unsigned int, unsigned int, bool, int32_t);
+
+/* INT16 */
+template void IndirectInterleave<8, 1, VLType::None>(int16_t *, const int16_t * const * const *, unsigned int, unsigned int, unsigned int y0, unsigned int ymax, unsigned int k0, unsigned int kmax, bool, int32_t);
+template void ConvolutionInterleave<8, 1, VLType::None>(int16_t *, const int16_t *, size_t, const convolver<int16_t> &, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, bool, int32_t);
+template void Interleave<8, 1, VLType::None>(int16_t *, const int16_t *, size_t, unsigned int, unsigned int, unsigned int, unsigned int, bool, int32_t);
+
+template void IndirectInterleave<8, 1, VLType::None>(uint16_t *, const uint16_t * const * const *, unsigned int, unsigned int, unsigned int y0, unsigned int ymax, unsigned int k0, unsigned int kmax, bool, int32_t);
+template void ConvolutionInterleave<8, 1, VLType::None>(uint16_t *, const uint16_t *, size_t, const convolver<uint16_t> &, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, bool, int32_t);
+template void Interleave<8, 1, VLType::None>(uint16_t *, const uint16_t *, size_t, unsigned int, unsigned int, unsigned int, unsigned int, bool, int32_t);
+
+/* INT8 */
+/* NEON SMLA/SMLAL (height 4, block 16) */
+template void IndirectInterleave<4, 16, VLType::None>(int8_t *, const int8_t * const * const *, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, bool, int32_t);
+template void ConvolutionInterleave<4, 16, VLType::None>(int8_t *, const int8_t *, size_t, const convolver<int8_t> &, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, bool, int32_t);
+template void Interleave<4, 16, VLType::None>(int8_t *, const int8_t *, size_t, unsigned int, unsigned int, unsigned int, unsigned int, bool, int32_t);
+
+/* NEON SDOT (height 8, block 4) */
+template void IndirectInterleave<8, 4, VLType::None>(int8_t *, const int8_t * const * const *, unsigned int, unsigned int, unsigned int y0, unsigned int ymax, unsigned int k0, unsigned int kmax, bool, int32_t);
+template void ConvolutionInterleave<8, 4, VLType::None>(int8_t *, const int8_t *, size_t, const convolver<int8_t> &, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, bool, int32_t);
+template void Interleave<8, 4, VLType::None>(int8_t *, const int8_t *, size_t, unsigned int, unsigned int, unsigned int, unsigned int, bool, int32_t);
+
+/* MMLA SMMLA (height 8, block 8) */
+template void IndirectInterleave<8, 8, VLType::None>(int8_t *, const int8_t * const * const *, unsigned int, unsigned int, unsigned int y0, unsigned int ymax, unsigned int k0, unsigned int kmax, bool, int32_t);
+template void ConvolutionInterleave<8, 8, VLType::None>(int8_t *, const int8_t *, size_t, const convolver<int8_t> &, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, bool, int32_t);
+template void Interleave<8, 8, VLType::None>(int8_t *, const int8_t *, size_t, unsigned int, unsigned int, unsigned int, unsigned int, bool, int32_t);
+
+/* NEON SDOT (height 8, block 1) */
+template void IndirectInterleave<8, 1, VLType::None>(int16_t *, const int8_t * const * const *, unsigned int, unsigned int, unsigned int y0, unsigned int ymax, unsigned int k0, unsigned int kmax, bool, int32_t);
+template void ConvolutionInterleave<8, 1, VLType::None>(int16_t *, const int8_t *, size_t, const convolver<int8_t> &, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, bool, int32_t);
+template void Interleave<8, 1, VLType::None>(int16_t *, const int8_t *, size_t, unsigned int, unsigned int, unsigned int, unsigned int, bool, int32_t);
+
+/* NEON SMLA/SMLAL (height 4, block 16) */
+template void IndirectInterleave<4, 16, VLType::None>(uint8_t *, const uint8_t * const * const *, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, bool, int32_t);
+template void ConvolutionInterleave<4, 16, VLType::None>(uint8_t *, const uint8_t *, size_t, const convolver<uint8_t> &, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, bool, int32_t);
+template void Interleave<4, 16, VLType::None>(uint8_t *, const uint8_t *, size_t, unsigned int, unsigned int, unsigned int, unsigned int, bool, int32_t);
+
+/* NEON SDOT (height 8, block 4) */
+template void IndirectInterleave<8, 4, VLType::None>(uint8_t *, const uint8_t * const * const *, unsigned int, unsigned int, unsigned int y0, unsigned int ymax, unsigned int k0, unsigned int kmax, bool, int32_t);
+template void ConvolutionInterleave<8, 4, VLType::None>(uint8_t *, const uint8_t *, size_t, const convolver<uint8_t> &, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, bool, int32_t);
+template void Interleave<8, 4, VLType::None>(uint8_t *, const uint8_t *, size_t, unsigned int, unsigned int, unsigned int, unsigned int, bool, int32_t);
+
+/* MMLA SMMLA (height 8, block 8) */
+template void IndirectInterleave<8, 8, VLType::None>(uint8_t *, const uint8_t * const * const *, unsigned int, unsigned int, unsigned int y0, unsigned int ymax, unsigned int k0, unsigned int kmax, bool, int32_t);
+template void ConvolutionInterleave<8, 8, VLType::None>(uint8_t *, const uint8_t *, size_t, const convolver<uint8_t> &, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, bool, int32_t);
+template void Interleave<8, 8, VLType::None>(uint8_t *, const uint8_t *, size_t, unsigned int, unsigned int, unsigned int, unsigned int, bool, int32_t);
+
+/* NEON 16-bit (height 8, block 1) */
+template void IndirectInterleave<8, 1, VLType::None>(uint16_t *, const uint8_t * const * const *, unsigned int, unsigned int, unsigned int y0, unsigned int ymax, unsigned int k0, unsigned int kmax, bool, int32_t);
+template void ConvolutionInterleave<8, 1, VLType::None>(uint16_t *, const uint8_t *, size_t, const convolver<uint8_t> &, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, bool, int32_t);
+template void Interleave<8, 1, VLType::None>(uint16_t *, const uint8_t *, size_t, unsigned int, unsigned int, unsigned int, unsigned int, bool, int32_t);
+#endif // __aarch64__
+
+} // namespace arm_gemm
diff --git a/src/core/NEON/kernels/arm_gemm/interleave_indirect.hpp b/src/core/NEON/kernels/arm_gemm/interleave_indirect.hpp
new file mode 100644
index 0000000..660577f
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/interleave_indirect.hpp
@@ -0,0 +1,43 @@
+/*
+ * Copyright (c) 2020 Arm Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <cstddef>
+#include <cstdint>
+
+#include "convolution_parameters.hpp"
+#include "convolver.hpp"
+#include "utils.hpp"
+
+namespace arm_gemm {
+
+template<unsigned int height_vectors, unsigned int block, VLType vlt, typename TIn, typename TOut>
+void IndirectInterleave(TOut *out, const TIn * const * const *ptr, unsigned int stringlen, unsigned int rounded_stringlen, unsigned int y0, unsigned int ymax, unsigned int k0, unsigned int kmax, bool, int32_t);
+
+template<unsigned int height_vectors, unsigned int block, VLType vlt, typename TIn, typename TOut>
+void ConvolutionInterleave(TOut *out, const TIn *in, size_t in_stride, const convolver<TIn> &conv, const unsigned int rounded_stringlen, const unsigned int y0, const unsigned int ymax, const unsigned int k0, const unsigned int kmax, bool, int32_t);
+
+template<unsigned int height_vectors, unsigned int block, VLType vlt, typename TIn, typename TOut>
+void Interleave(TOut *out, const TIn *in, size_t in_stride, const unsigned int y0, const unsigned int ymax, const unsigned int k0, const unsigned int kmax, bool, int32_t);
+
+} // namespace arm_gemm
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_s16_12x8.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_s16_8x12.hpp
similarity index 87%
rename from src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_s16_12x8.hpp
rename to src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_s16_8x12.hpp
index 0f0e5a7..8bf8d84 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_s16_12x8.hpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_s16_8x12.hpp
@@ -30,9 +30,9 @@
namespace arm_gemm {
// Actual kernel implementations
-void a64_gemm_s16_asimd_12x8(const int16_t *, const int16_t *, int32_t *, int, int, int);
+void a64_gemm_s16_asimd_8x12(const int16_t *, const int16_t *, int32_t *, int, int, int);
-// 12x8 SGEMM "strategy" class.
+// 8x12 SGEMM "strategy" class.
//
// This describes the characteristics of a family of kernels, in terms of
// the required interleave properties and the output block size.
@@ -40,7 +40,7 @@
// All kernels in the family must share these characteristics. The actual
// kernel to be used can be chosen at runtime, based on the CPU_type
// structure.
-class gemm_s16_12x8 {
+class cls_a64_gemm_s16_8x12 {
public:
typedef int16_t operand_type;
typedef int32_t result_type;
@@ -62,10 +62,11 @@
// Use the standard fixed size transforms.
StdTransformsFixed<operand_type, result_type, 8, 12> transforms = {};
+ StdTransformsFixed<operand_type, result_type, 8, 12, 1, true> transforms_quantized = {};
- kern_type kernel = a64_gemm_s16_asimd_12x8;
+ kern_type kernel = a64_gemm_s16_asimd_8x12;
- gemm_s16_12x8(const CPUInfo *) { }
+ cls_a64_gemm_s16_8x12(const CPUInfo *) { }
};
} // namespace arm_gemm
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_s16_12x8/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_s16_8x12/generic.cpp
similarity index 99%
rename from src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_s16_12x8/generic.cpp
rename to src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_s16_8x12/generic.cpp
index 7052f83..a77938f 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_s16_12x8/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_s16_8x12/generic.cpp
@@ -29,7 +29,7 @@
namespace arm_gemm {
-void a64_gemm_s16_asimd_12x8(const int16_t *Apanel, const int16_t *Bpanel, int32_t *Cpanel, int ablocks, int bblocks, int K)
+void a64_gemm_s16_asimd_8x12(const int16_t *Apanel, const int16_t *Bpanel, int32_t *Cpanel, int ablocks, int bblocks, int K)
{
const int16_t *a_ptr = Apanel;
int32_t *c_ptr = Cpanel;
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_s8_4x4.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_s8_4x4.hpp
index 256acc4..b68a5f5 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_s8_4x4.hpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_s8_4x4.hpp
@@ -34,7 +34,7 @@
#include "arm_gemm.hpp"
-class gemm_s8_4x4 {
+class cls_a64_gemm_s8_4x4 {
public:
typedef int8_t operand_type;
typedef int32_t result_type;
@@ -56,10 +56,11 @@
// Use the standard fixed size transforms.
StdTransformsFixed<operand_type, result_type, 4, 4, 16> transforms = {};
+ StdTransformsFixed<operand_type, result_type, 4, 4, 16, true> transforms_quantized = {};
kern_type kernel=a64_gemm_s8_4x4;
- gemm_s8_4x4(const CPUInfo *) { }
+ cls_a64_gemm_s8_4x4(const CPUInfo *) { }
};
} // namespace arm_gemm
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_s8_12x8.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_s8_8x12.hpp
similarity index 80%
rename from src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_s8_12x8.hpp
rename to src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_s8_8x12.hpp
index 0e294bf..eee817e 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_s8_12x8.hpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_s8_8x12.hpp
@@ -32,11 +32,11 @@
namespace arm_gemm {
// Load the actual kernel
-void a64_gemm_s8_12x8(const int8_t *, const int8_t *, int32_t *, int, int, int);
-void a64_gemm_s8_12x8_a55r1(const int8_t *, const int8_t *, int32_t *, int, int, int);
-void a64_gemm_s8_12x8_x1(const int8_t *, const int8_t *, int32_t *, int, int, int);
+void a64_gemm_s8_8x12(const int8_t *, const int8_t *, int32_t *, int, int, int);
+void a64_gemm_s8_8x12_a55r1(const int8_t *, const int8_t *, int32_t *, int, int, int);
+void a64_gemm_s8_8x12_x1(const int8_t *, const int8_t *, int32_t *, int, int, int);
-class gemm_s8_12x8 {
+class cls_a64_gemm_s8_8x12 {
public:
typedef int8_t operand_type;
typedef int32_t result_type;
@@ -58,16 +58,17 @@
// Use the standard fixed size transforms.
StdTransformsFixed<operand_type, result_type, 8, 12, 4> transforms = {};
+ StdTransformsFixed<operand_type, result_type, 8, 12, 4, true> transforms_quantized = {};
- kern_type kernel = a64_gemm_s8_12x8;
+ kern_type kernel = a64_gemm_s8_8x12;
- gemm_s8_12x8(const CPUInfo *ci) {
+ cls_a64_gemm_s8_8x12(const CPUInfo *ci) {
auto mod = ci->get_cpu_model();
if (mod == CPUModel::A55r1) {
- kernel = a64_gemm_s8_12x8_a55r1;
+ kernel = a64_gemm_s8_8x12_a55r1;
} else if (mod == CPUModel::X1) {
- kernel = a64_gemm_s8_12x8_x1;
+ kernel = a64_gemm_s8_8x12_x1;
}
}
};
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_s8_12x8/a55r1.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_s8_8x12/a55r1.cpp
similarity index 99%
rename from src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_s8_12x8/a55r1.cpp
rename to src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_s8_8x12/a55r1.cpp
index ddd8124..bb5226e 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_s8_12x8/a55r1.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_s8_8x12/a55r1.cpp
@@ -29,7 +29,7 @@
namespace arm_gemm {
-void a64_gemm_s8_12x8_a55r1(const int8_t *Apanel, const int8_t *Bpanel, int32_t *Cpanel, const int ablocks, const int bblocks, const int K) {
+void a64_gemm_s8_8x12_a55r1(const int8_t *Apanel, const int8_t *Bpanel, int32_t *Cpanel, const int ablocks, const int bblocks, const int K) {
const int8_t *a_ptr = Apanel;
int32_t *c_ptr = Cpanel;
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_s8_12x8/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_s8_8x12/generic.cpp
similarity index 99%
rename from src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_s8_12x8/generic.cpp
rename to src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_s8_8x12/generic.cpp
index a7abaed..7bf36a5 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_s8_12x8/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_s8_8x12/generic.cpp
@@ -29,7 +29,7 @@
namespace arm_gemm {
-void a64_gemm_s8_12x8(const int8_t *Apanel, const int8_t *Bpanel, int32_t *Cpanel, int ablocks, int bblocks, int K) {
+void a64_gemm_s8_8x12(const int8_t *Apanel, const int8_t *Bpanel, int32_t *Cpanel, int ablocks, int bblocks, int K) {
const int8_t *a_ptr = Apanel;
int32_t *c_ptr = Cpanel;
// We divide K by 4 because the sdot instruction processes 4 elements at a time.
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_s8_12x8/x1.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_s8_8x12/x1.cpp
similarity index 99%
rename from src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_s8_12x8/x1.cpp
rename to src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_s8_8x12/x1.cpp
index 446fcf8..afd2427 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_s8_12x8/x1.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_s8_8x12/x1.cpp
@@ -29,7 +29,7 @@
namespace arm_gemm {
-void a64_gemm_s8_12x8_x1(const int8_t *Apanel, const int8_t *Bpanel, int32_t *Cpanel, int ablocks, int bblocks, int K) {
+void a64_gemm_s8_8x12_x1(const int8_t *Apanel, const int8_t *Bpanel, int32_t *Cpanel, int ablocks, int bblocks, int K) {
const int8_t *a_ptr = Apanel;
int32_t *c_ptr = Cpanel;
// We divide K by 4 because the sdot instruction processes 4 elements at a time.
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_u16_12x8.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_u16_8x12.hpp
similarity index 78%
rename from src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_u16_12x8.hpp
rename to src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_u16_8x12.hpp
index b862040..e49ebbd 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_u16_12x8.hpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_u16_8x12.hpp
@@ -30,17 +30,9 @@
namespace arm_gemm {
// Actual kernel implementations
-void a64_gemm_u16_asimd_12x8(const uint16_t *, const uint16_t *, uint32_t *, int, int, int);
+void a64_gemm_u16_asimd_8x12(const uint16_t *, const uint16_t *, uint32_t *, int, int, int);
-// 12x8 SGEMM "strategy" class.
-//
-// This describes the characteristics of a family of kernels, in terms of
-// the required interleave properties and the output block size.
-//
-// All kernels in the family must share these characteristics. The actual
-// kernel to be used can be chosen at runtime, based on the CPU_type
-// structure.
-class gemm_u16_12x8 {
+class cls_a64_gemm_u16_8x12 {
public:
typedef uint16_t operand_type;
typedef uint32_t result_type;
@@ -62,10 +54,11 @@
// Use the standard fixed size transforms.
StdTransformsFixed<operand_type, result_type, 8, 12> transforms = {};
+ StdTransformsFixed<operand_type, result_type, 8, 12, 1, true> transforms_quantized = {};
- kern_type kernel = a64_gemm_u16_asimd_12x8;
+ kern_type kernel = a64_gemm_u16_asimd_8x12;
- gemm_u16_12x8(const CPUInfo *) { }
+ cls_a64_gemm_u16_8x12(const CPUInfo *) { }
};
} // namespace arm_gemm
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_u16_12x8/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_u16_8x12/generic.cpp
similarity index 99%
rename from src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_u16_12x8/generic.cpp
rename to src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_u16_8x12/generic.cpp
index 66f0b7c..98da783 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_u16_12x8/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_u16_8x12/generic.cpp
@@ -29,7 +29,7 @@
namespace arm_gemm {
-void a64_gemm_u16_asimd_12x8(const uint16_t *Apanel, const uint16_t *Bpanel, uint32_t *Cpanel, int ablocks, int bblocks, int K)
+void a64_gemm_u16_asimd_8x12(const uint16_t *Apanel, const uint16_t *Bpanel, uint32_t *Cpanel, int ablocks, int bblocks, int K)
{
const uint16_t *a_ptr = Apanel;
uint32_t *c_ptr = Cpanel;
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_u8_4x4.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_u8_4x4.hpp
index 134007b..854b675 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_u8_4x4.hpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_u8_4x4.hpp
@@ -32,7 +32,7 @@
// Kernel definition
void a64_gemm_u8_4x4(const uint8_t *Apanel, const uint8_t *Bpanel, uint32_t *Cpanel, int ablocks, int bblocks, int K);
-class gemm_u8_4x4 {
+class cls_a64_gemm_u8_4x4 {
public:
typedef uint8_t operand_type;
typedef uint32_t result_type;
@@ -64,10 +64,11 @@
// Use the standard fixed size transforms.
StdTransformsFixed<operand_type, result_type, 4, 4, 16> transforms = {};
+ StdTransformsFixed<operand_type, result_type, 4, 4, 16, true> transforms_quantized = {};
kern_type kernel = a64_gemm_u8_4x4;
- gemm_u8_4x4(const CPUInfo *) { }
+ cls_a64_gemm_u8_4x4(const CPUInfo *) { }
};
} // namespace arm_gemm
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_u8_12x8.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_u8_8x12.hpp
similarity index 82%
rename from src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_u8_12x8.hpp
rename to src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_u8_8x12.hpp
index c0990ec..256ba2e 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_u8_12x8.hpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_u8_8x12.hpp
@@ -30,11 +30,11 @@
namespace arm_gemm {
// Load the actual kernel
-void a64_gemm_u8_12x8(const uint8_t *, const uint8_t *, uint32_t *, int, int, int);
-void a64_gemm_u8_12x8_a55r1(const uint8_t *, const uint8_t *, uint32_t *, int, int, int);
-void a64_gemm_u8_12x8_x1(const uint8_t *, const uint8_t *, uint32_t *, int, int, int);
+void a64_gemm_u8_8x12(const uint8_t *, const uint8_t *, uint32_t *, int, int, int);
+void a64_gemm_u8_8x12_a55r1(const uint8_t *, const uint8_t *, uint32_t *, int, int, int);
+void a64_gemm_u8_8x12_x1(const uint8_t *, const uint8_t *, uint32_t *, int, int, int);
-class gemm_u8_12x8 {
+class cls_a64_gemm_u8_8x12 {
public:
typedef uint8_t operand_type;
typedef uint32_t result_type;
@@ -66,16 +66,17 @@
// Use the standard fixed sized transforms.
StdTransformsFixed<operand_type, result_type, 8, 12, 4> transforms = {};
+ StdTransformsFixed<operand_type, result_type, 8, 12, 4, true> transforms_quantized = {};
- kern_type kernel = a64_gemm_u8_12x8;
+ kern_type kernel = a64_gemm_u8_8x12;
- gemm_u8_12x8(const CPUInfo *ci) {
+ cls_a64_gemm_u8_8x12(const CPUInfo *ci) {
auto mod = ci->get_cpu_model();
if (mod == CPUModel::A55r1) {
- kernel = a64_gemm_u8_12x8_a55r1;
+ kernel = a64_gemm_u8_8x12_a55r1;
} else if (mod == CPUModel::X1) {
- kernel = a64_gemm_u8_12x8_x1;
+ kernel = a64_gemm_u8_8x12_x1;
}
}
};
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_u8_12x8/a55r1.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_u8_8x12/a55r1.cpp
similarity index 99%
rename from src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_u8_12x8/a55r1.cpp
rename to src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_u8_8x12/a55r1.cpp
index c9a8a82..63869c9 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_u8_12x8/a55r1.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_u8_8x12/a55r1.cpp
@@ -29,7 +29,7 @@
namespace arm_gemm {
-void a64_gemm_u8_12x8_a55r1(const uint8_t *Apanel, const uint8_t *Bpanel, uint32_t *Cpanel, const int ablocks, const int bblocks, const int K) {
+void a64_gemm_u8_8x12_a55r1(const uint8_t *Apanel, const uint8_t *Bpanel, uint32_t *Cpanel, const int ablocks, const int bblocks, const int K) {
const uint8_t *a_ptr = Apanel;
uint32_t *c_ptr = Cpanel;
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_u8_12x8/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_u8_8x12/generic.cpp
similarity index 99%
rename from src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_u8_12x8/generic.cpp
rename to src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_u8_8x12/generic.cpp
index 821e742..ff60cbc 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_u8_12x8/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_u8_8x12/generic.cpp
@@ -29,7 +29,7 @@
namespace arm_gemm {
-void a64_gemm_u8_12x8(const uint8_t *Apanel, const uint8_t *Bpanel, uint32_t *Cpanel, int ablocks, int bblocks, int K) {
+void a64_gemm_u8_8x12(const uint8_t *Apanel, const uint8_t *Bpanel, uint32_t *Cpanel, int ablocks, int bblocks, int K) {
const uint8_t *a_ptr = Apanel;
uint32_t *c_ptr = Cpanel;
// We divide K by 4 because the udot instruction processes 4 elements at a time.
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_u8_12x8/x1.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_u8_8x12/x1.cpp
similarity index 99%
rename from src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_u8_12x8/x1.cpp
rename to src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_u8_8x12/x1.cpp
index 7fac673..1c1196b 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_u8_12x8/x1.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_u8_8x12/x1.cpp
@@ -29,7 +29,7 @@
namespace arm_gemm {
-void a64_gemm_u8_12x8_x1(const uint8_t *Apanel, const uint8_t *Bpanel, uint32_t *Cpanel, int ablocks, int bblocks, int K) {
+void a64_gemm_u8_8x12_x1(const uint8_t *Apanel, const uint8_t *Bpanel, uint32_t *Cpanel, int ablocks, int bblocks, int K) {
const uint8_t *a_ptr = Apanel;
uint32_t *c_ptr = Cpanel;
// We divide K by 4 because the udot instruction processes 4 elements at a time.
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_4x8.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_gemv_fp32_mla_32.hpp
similarity index 75%
rename from src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_4x8.hpp
rename to src/core/NEON/kernels/arm_gemm/kernels/a64_gemv_fp32_mla_32.hpp
index b60401b..b531725 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_4x8.hpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_gemv_fp32_mla_32.hpp
@@ -25,32 +25,26 @@
#ifdef __aarch64__
-
+#include "../performance_parameters.hpp"
#include "../std_transforms_fixed.hpp"
namespace arm_gemm
{
// Actual kernel implementations
-void a64_hybrid_fp32_mla_4x8(const float *, int, const float *, float *, int, int, int, int, const float *, Activation, bool);
+void a64_gemv_fp32_mla_32(const float *, const float *, float *, size_t, size_t, const float *, Activation, bool);
-class hybrid_fp32_mla_4x8
+class cls_a64_gemv_fp32_mla_32
{
public:
typedef float operand_type;
typedef float result_type;
- typedef void (*kern_type)(const float *, int, const float *, float *, int, int, int, int, const float *, Activation, bool);
-
- /* Kernel blocking parameters */
- static constexpr unsigned int out_height()
- {
- return 8;
- }
+ typedef void (*kern_type)(const float *, const float *, float *, size_t, size_t, const float *, Activation, bool);
static unsigned int out_width()
{
- return 4;
+ return 32;
}
static constexpr unsigned int k_unroll()
@@ -73,14 +67,13 @@
return true;
}
- StdTransformsFixed<operand_type, result_type, 8, 4, 1> transforms = {};
+ StdTransformsFixed<operand_type, result_type, 1, 32, 1> transforms = {};
// Default to the generic kernel
- kern_type kernel=a64_hybrid_fp32_mla_4x8;
+ kern_type kernel=a64_gemv_fp32_mla_32;
- hybrid_fp32_mla_4x8(const CPUInfo *)
+ cls_a64_gemv_fp32_mla_32(const CPUInfo *)
{
-
}
};
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_gemv_fp32_mla_32/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_gemv_fp32_mla_32/generic.cpp
new file mode 100644
index 0000000..a2af8d6
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_gemv_fp32_mla_32/generic.cpp
@@ -0,0 +1,1546 @@
+/*
+ * Copyright (c) 2019-2020 Arm Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+#ifdef __aarch64__
+
+#include "arm_gemm.hpp"
+#include "../../utils.hpp"
+
+#include <cassert>
+
+namespace arm_gemm {
+
+void a64_gemv_fp32_mla_32 (
+ const float *A_ptr, const float *B_ptr, float *output_ptr,
+ size_t N, size_t K,
+ const float *bias, Activation act, bool
+)
+{
+ struct KernelArgs {
+ float maxval = static_cast<float>(std::numeric_limits<float>::infinity());
+ float minval = - static_cast<float>(std::numeric_limits<float>::infinity());
+ const float *B_ptr = {};
+ size_t output_offset = {};
+ unsigned int input_initial_col = {};
+ } ka;
+
+ unsigned long flags=0;
+ ka.B_ptr = B_ptr;
+ switch(act.type) {
+ default:
+ case Activation::Type::None:
+ break;
+ case Activation::Type::BoundedReLU:
+ ka.maxval = static_cast<float>(act.param1);
+ /* fall through */
+ case Activation::Type::ReLU:
+ ka.minval = 0;
+ flags |= 0x2;
+ break;
+ }
+ __asm__ __volatile__(
+ "add x22, %x[N], #0x3\n"
+ "mov x21, %x[bias]\n"
+ "lsr x22, x22, #0x2\n"
+ "1:" // Column loop
+ "cmp x22, #0x8\n"
+ "bge 85f\n"
+ "cmp x22, #0x6\n"
+ "bgt 73f\n"
+ "beq 61f\n"
+ "cmp x22, #0x4\n"
+ "bgt 49f\n"
+ "beq 37f\n"
+ "cmp x22, #0x2\n"
+ "bgt 25f\n"
+ "beq 13f\n"
+ "mov x20, %x[K]\n"
+ "mov x19, %x[A_ptr]\n"
+ "cbz x21, 2f\n"
+ "ldr q24, [x21, #0x0]\n"
+ "add x21, x21, #0x10\n"
+ "b 3f\n"
+ "2:" // Width 1: no bias
+ "movi v24.16b, #0x0\n"
+ "3:" // Width 1: setup done
+ "cmp x20, #0x4\n"
+ "blt 6f\n"
+ "cmp x20, #0x8\n"
+ "blt 5f\n"
+ "4:" // Width 1: Multiply loop: Main loop head
+ "ldr q0, [x19, #0x0]\n"
+ "ldr q1, [%x[B_ptr], #0x0]\n"
+ "fmla v24.4s, v1.4s, v0.s[0]\n"
+ "add %x[B_ptr], %x[B_ptr], #0x80\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "ldr q2, [%x[B_ptr], #0x0]\n"
+ "fmla v24.4s, v2.4s, v0.s[1]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "add %x[B_ptr], %x[B_ptr], #0x80\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "ldr q3, [%x[B_ptr], #0x0]\n"
+ "fmla v24.4s, v3.4s, v0.s[2]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "add %x[B_ptr], %x[B_ptr], #0x80\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "ldr q4, [%x[B_ptr], #0x0]\n"
+ "fmla v24.4s, v4.4s, v0.s[3]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "add %x[B_ptr], %x[B_ptr], #0x80\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "add x19, x19, #0x10\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "sub x20, x20, #0x4\n"
+ "prfm pldl1keep, [x19, #0x80]\n"
+ "cmp x20, #0x8\n"
+ "bge 4b\n"
+ "5:" // Width 1: Multiply loop: Single iteration only
+ "sub x20, x20, #0x4\n"
+ "ldr q0, [x19, #0x0]\n"
+ "ldr q5, [%x[B_ptr], #0x0]\n"
+ "fmla v24.4s, v5.4s, v0.s[0]\n"
+ "add %x[B_ptr], %x[B_ptr], #0x80\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "ldr q6, [%x[B_ptr], #0x0]\n"
+ "fmla v24.4s, v6.4s, v0.s[1]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "add %x[B_ptr], %x[B_ptr], #0x80\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "ldr q7, [%x[B_ptr], #0x0]\n"
+ "fmla v24.4s, v7.4s, v0.s[2]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "add %x[B_ptr], %x[B_ptr], #0x80\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "ldr q8, [%x[B_ptr], #0x0]\n"
+ "fmla v24.4s, v8.4s, v0.s[3]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "add %x[B_ptr], %x[B_ptr], #0x80\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "add x19, x19, #0x10\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "prfm pldl1keep, [x19, #0x80]\n"
+ "6:" // Width 1: Multiply loop: Main loop skip
+ "cbz x20, 8f\n"
+ "7:" // Width 1: Multiply loop: Odd block loop
+ "ldr s0, [x19], #0x4\n"
+ "ldr q9, [%x[B_ptr], #0x0]\n"
+ "fmla v24.4s, v9.4s, v0.s[0]\n"
+ "add %x[B_ptr], %x[B_ptr], #0x80\n"
+ "sub x20, x20, #0x1\n"
+ "cbnz x20, 7b\n"
+ "8:" // Width 1: Multiply loop: No odd multiplies
+ "prfm pstl1keep, [%x[output_ptr], #0x0]\n"
+ "tbz %x[flags], #1, 9f\n"
+ "add x19, %x[args_ptr], %[offset_min]\n"
+ "ld1r { v17.4s }, [x19]\n"
+ "add x19, %x[args_ptr], %[offset_max]\n"
+ "ld1r { v16.4s }, [x19]\n"
+ "fmin v24.4s, v24.4s, v16.4s\n"
+ "fmax v24.4s, v24.4s, v17.4s\n"
+ "9:" // Width 1: No activation
+ "cmp %x[N], #0x4\n"
+ "blt 10f\n"
+ "str q24, [%x[output_ptr], #0x0]\n"
+ "add %x[output_ptr], %x[output_ptr], #0x10\n"
+ "b 12f\n"
+ "10:" // Width 1: Partial writeback
+ "tbz %x[N], #1, 11f\n"
+ "str d24, [%x[output_ptr]], #0x8\n"
+ "tbz %x[N], #0, 12f\n"
+ "st1 { v24.s }[2], [%x[output_ptr]]\n"
+ "b 12f\n"
+ "11:" // Width 1: Partial direct writeback: partial_1_0
+ "str s24, [%x[output_ptr], #0x0]\n"
+ "12:" // Width 1: Writeback done
+ "b 97f\n"
+ "13:" // Width 2
+ "mov x20, %x[K]\n"
+ "mov x19, %x[A_ptr]\n"
+ "cbz x21, 14f\n"
+ "ldr q24, [x21, #0x0]\n"
+ "ldr q25, [x21, #0x10]\n"
+ "add x21, x21, #0x20\n"
+ "b 15f\n"
+ "14:" // Width 2: no bias
+ "movi v24.16b, #0x0\n"
+ "movi v25.16b, #0x0\n"
+ "15:" // Width 2: setup done
+ "cmp x20, #0x4\n"
+ "blt 18f\n"
+ "cmp x20, #0x8\n"
+ "blt 17f\n"
+ "16:" // Width 2: Multiply loop: Main loop head
+ "ldr q0, [x19, #0x0]\n"
+ "ldr q1, [%x[B_ptr], #0x0]\n"
+ "fmla v24.4s, v1.4s, v0.s[0]\n"
+ "ldr q2, [%x[B_ptr], #0x10]\n"
+ "add %x[B_ptr], %x[B_ptr], #0x80\n"
+ "fmla v25.4s, v2.4s, v0.s[0]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "ldr q3, [%x[B_ptr], #0x0]\n"
+ "fmla v24.4s, v3.4s, v0.s[1]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "ldr q4, [%x[B_ptr], #0x10]\n"
+ "fmla v25.4s, v4.4s, v0.s[1]\n"
+ "add %x[B_ptr], %x[B_ptr], #0x80\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "ldr q5, [%x[B_ptr], #0x0]\n"
+ "fmla v24.4s, v5.4s, v0.s[2]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "ldr q6, [%x[B_ptr], #0x10]\n"
+ "fmla v25.4s, v6.4s, v0.s[2]\n"
+ "add %x[B_ptr], %x[B_ptr], #0x80\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "ldr q7, [%x[B_ptr], #0x0]\n"
+ "fmla v24.4s, v7.4s, v0.s[3]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "ldr q8, [%x[B_ptr], #0x10]\n"
+ "fmla v25.4s, v8.4s, v0.s[3]\n"
+ "add %x[B_ptr], %x[B_ptr], #0x80\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "add x19, x19, #0x10\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "sub x20, x20, #0x4\n"
+ "prfm pldl1keep, [x19, #0x80]\n"
+ "cmp x20, #0x8\n"
+ "bge 16b\n"
+ "17:" // Width 2: Multiply loop: Single iteration only
+ "sub x20, x20, #0x4\n"
+ "ldr q0, [x19, #0x0]\n"
+ "ldr q9, [%x[B_ptr], #0x0]\n"
+ "fmla v24.4s, v9.4s, v0.s[0]\n"
+ "ldr q10, [%x[B_ptr], #0x10]\n"
+ "add %x[B_ptr], %x[B_ptr], #0x80\n"
+ "fmla v25.4s, v10.4s, v0.s[0]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "ldr q11, [%x[B_ptr], #0x0]\n"
+ "fmla v24.4s, v11.4s, v0.s[1]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "ldr q12, [%x[B_ptr], #0x10]\n"
+ "fmla v25.4s, v12.4s, v0.s[1]\n"
+ "add %x[B_ptr], %x[B_ptr], #0x80\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "ldr q13, [%x[B_ptr], #0x0]\n"
+ "fmla v24.4s, v13.4s, v0.s[2]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "ldr q14, [%x[B_ptr], #0x10]\n"
+ "fmla v25.4s, v14.4s, v0.s[2]\n"
+ "add %x[B_ptr], %x[B_ptr], #0x80\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "ldr q15, [%x[B_ptr], #0x0]\n"
+ "fmla v24.4s, v15.4s, v0.s[3]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "ldr q16, [%x[B_ptr], #0x10]\n"
+ "fmla v25.4s, v16.4s, v0.s[3]\n"
+ "add %x[B_ptr], %x[B_ptr], #0x80\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "add x19, x19, #0x10\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "prfm pldl1keep, [x19, #0x80]\n"
+ "18:" // Width 2: Multiply loop: Main loop skip
+ "cbz x20, 20f\n"
+ "19:" // Width 2: Multiply loop: Odd block loop
+ "ldr s0, [x19], #0x4\n"
+ "ldr q17, [%x[B_ptr], #0x0]\n"
+ "fmla v24.4s, v17.4s, v0.s[0]\n"
+ "ldr q18, [%x[B_ptr], #0x10]\n"
+ "add %x[B_ptr], %x[B_ptr], #0x80\n"
+ "fmla v25.4s, v18.4s, v0.s[0]\n"
+ "sub x20, x20, #0x1\n"
+ "cbnz x20, 19b\n"
+ "20:" // Width 2: Multiply loop: No odd multiplies
+ "prfm pstl1keep, [%x[output_ptr], #0x0]\n"
+ "tbz %x[flags], #1, 21f\n"
+ "add x19, %x[args_ptr], %[offset_min]\n"
+ "ld1r { v17.4s }, [x19]\n"
+ "add x19, %x[args_ptr], %[offset_max]\n"
+ "ld1r { v16.4s }, [x19]\n"
+ "fmin v24.4s, v24.4s, v16.4s\n"
+ "fmin v25.4s, v25.4s, v16.4s\n"
+ "fmax v24.4s, v24.4s, v17.4s\n"
+ "fmax v25.4s, v25.4s, v17.4s\n"
+ "21:" // Width 2: No activation
+ "str q24, [%x[output_ptr], #0x0]\n"
+ "cmp %x[N], #0x8\n"
+ "add %x[output_ptr], %x[output_ptr], #0x10\n"
+ "blt 22f\n"
+ "str q25, [%x[output_ptr], #0x0]\n"
+ "add %x[output_ptr], %x[output_ptr], #0x10\n"
+ "b 24f\n"
+ "22:" // Width 2: Partial writeback
+ "tbz %x[N], #1, 23f\n"
+ "str d25, [%x[output_ptr]], #0x8\n"
+ "tbz %x[N], #0, 24f\n"
+ "st1 { v25.s }[2], [%x[output_ptr]]\n"
+ "b 24f\n"
+ "23:" // Width 2: Partial direct writeback: partial_1_4
+ "tbz %x[N], #0, 24f\n"
+ "str s25, [%x[output_ptr], #0x0]\n"
+ "24:" // Width 2: Writeback done
+ "b 97f\n"
+ "25:" // Width 3
+ "mov x20, %x[K]\n"
+ "mov x19, %x[A_ptr]\n"
+ "cbz x21, 26f\n"
+ "ldr q24, [x21, #0x0]\n"
+ "ldr q25, [x21, #0x10]\n"
+ "ldr q26, [x21, #0x20]\n"
+ "add x21, x21, #0x30\n"
+ "b 27f\n"
+ "26:" // Width 3: no bias
+ "movi v24.16b, #0x0\n"
+ "movi v25.16b, #0x0\n"
+ "movi v26.16b, #0x0\n"
+ "27:" // Width 3: setup done
+ "cmp x20, #0x4\n"
+ "blt 30f\n"
+ "cmp x20, #0x8\n"
+ "blt 29f\n"
+ "28:" // Width 3: Multiply loop: Main loop head
+ "ldr q0, [x19, #0x0]\n"
+ "ldr q1, [%x[B_ptr], #0x0]\n"
+ "fmla v24.4s, v1.4s, v0.s[0]\n"
+ "ldr q2, [%x[B_ptr], #0x10]\n"
+ "ldr q3, [%x[B_ptr], #0x20]\n"
+ "fmla v25.4s, v2.4s, v0.s[0]\n"
+ "add %x[B_ptr], %x[B_ptr], #0x80\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "fmla v26.4s, v3.4s, v0.s[0]\n"
+ "ldr q4, [%x[B_ptr], #0x0]\n"
+ "fmla v24.4s, v4.4s, v0.s[1]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "ldr q5, [%x[B_ptr], #0x10]\n"
+ "fmla v25.4s, v5.4s, v0.s[1]\n"
+ "ldr q6, [%x[B_ptr], #0x20]\n"
+ "add %x[B_ptr], %x[B_ptr], #0x80\n"
+ "fmla v26.4s, v6.4s, v0.s[1]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "ldr q7, [%x[B_ptr], #0x0]\n"
+ "fmla v24.4s, v7.4s, v0.s[2]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "ldr q8, [%x[B_ptr], #0x10]\n"
+ "fmla v25.4s, v8.4s, v0.s[2]\n"
+ "ldr q9, [%x[B_ptr], #0x20]\n"
+ "add %x[B_ptr], %x[B_ptr], #0x80\n"
+ "fmla v26.4s, v9.4s, v0.s[2]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "ldr q10, [%x[B_ptr], #0x0]\n"
+ "fmla v24.4s, v10.4s, v0.s[3]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "ldr q11, [%x[B_ptr], #0x10]\n"
+ "fmla v25.4s, v11.4s, v0.s[3]\n"
+ "ldr q12, [%x[B_ptr], #0x20]\n"
+ "add %x[B_ptr], %x[B_ptr], #0x80\n"
+ "fmla v26.4s, v12.4s, v0.s[3]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "add x19, x19, #0x10\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "sub x20, x20, #0x4\n"
+ "prfm pldl1keep, [x19, #0x80]\n"
+ "cmp x20, #0x8\n"
+ "bge 28b\n"
+ "29:" // Width 3: Multiply loop: Single iteration only
+ "sub x20, x20, #0x4\n"
+ "ldr q0, [x19, #0x0]\n"
+ "ldr q13, [%x[B_ptr], #0x0]\n"
+ "fmla v24.4s, v13.4s, v0.s[0]\n"
+ "ldr q14, [%x[B_ptr], #0x10]\n"
+ "ldr q15, [%x[B_ptr], #0x20]\n"
+ "fmla v25.4s, v14.4s, v0.s[0]\n"
+ "add %x[B_ptr], %x[B_ptr], #0x80\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "fmla v26.4s, v15.4s, v0.s[0]\n"
+ "ldr q16, [%x[B_ptr], #0x0]\n"
+ "fmla v24.4s, v16.4s, v0.s[1]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "ldr q17, [%x[B_ptr], #0x10]\n"
+ "fmla v25.4s, v17.4s, v0.s[1]\n"
+ "ldr q18, [%x[B_ptr], #0x20]\n"
+ "add %x[B_ptr], %x[B_ptr], #0x80\n"
+ "fmla v26.4s, v18.4s, v0.s[1]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "ldr q19, [%x[B_ptr], #0x0]\n"
+ "fmla v24.4s, v19.4s, v0.s[2]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "ldr q20, [%x[B_ptr], #0x10]\n"
+ "fmla v25.4s, v20.4s, v0.s[2]\n"
+ "ldr q21, [%x[B_ptr], #0x20]\n"
+ "add %x[B_ptr], %x[B_ptr], #0x80\n"
+ "fmla v26.4s, v21.4s, v0.s[2]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "ldr q22, [%x[B_ptr], #0x0]\n"
+ "fmla v24.4s, v22.4s, v0.s[3]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "ldr q23, [%x[B_ptr], #0x10]\n"
+ "fmla v25.4s, v23.4s, v0.s[3]\n"
+ "ldr q1, [%x[B_ptr], #0x20]\n"
+ "add %x[B_ptr], %x[B_ptr], #0x80\n"
+ "fmla v26.4s, v1.4s, v0.s[3]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "add x19, x19, #0x10\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "prfm pldl1keep, [x19, #0x80]\n"
+ "30:" // Width 3: Multiply loop: Main loop skip
+ "cbz x20, 32f\n"
+ "31:" // Width 3: Multiply loop: Odd block loop
+ "ldr s0, [x19], #0x4\n"
+ "ldr q2, [%x[B_ptr], #0x0]\n"
+ "fmla v24.4s, v2.4s, v0.s[0]\n"
+ "ldr q3, [%x[B_ptr], #0x10]\n"
+ "ldr q4, [%x[B_ptr], #0x20]\n"
+ "fmla v25.4s, v3.4s, v0.s[0]\n"
+ "add %x[B_ptr], %x[B_ptr], #0x80\n"
+ "fmla v26.4s, v4.4s, v0.s[0]\n"
+ "sub x20, x20, #0x1\n"
+ "cbnz x20, 31b\n"
+ "32:" // Width 3: Multiply loop: No odd multiplies
+ "prfm pstl1keep, [%x[output_ptr], #0x0]\n"
+ "tbz %x[flags], #1, 33f\n"
+ "add x19, %x[args_ptr], %[offset_min]\n"
+ "ld1r { v17.4s }, [x19]\n"
+ "add x19, %x[args_ptr], %[offset_max]\n"
+ "ld1r { v16.4s }, [x19]\n"
+ "fmin v24.4s, v24.4s, v16.4s\n"
+ "fmin v25.4s, v25.4s, v16.4s\n"
+ "fmin v26.4s, v26.4s, v16.4s\n"
+ "fmax v24.4s, v24.4s, v17.4s\n"
+ "fmax v25.4s, v25.4s, v17.4s\n"
+ "fmax v26.4s, v26.4s, v17.4s\n"
+ "33:" // Width 3: No activation
+ "str q24, [%x[output_ptr], #0x0]\n"
+ "str q25, [%x[output_ptr], #0x10]\n"
+ "cmp %x[N], #0xc\n"
+ "add %x[output_ptr], %x[output_ptr], #0x20\n"
+ "blt 34f\n"
+ "str q26, [%x[output_ptr], #0x0]\n"
+ "add %x[output_ptr], %x[output_ptr], #0x10\n"
+ "b 36f\n"
+ "34:" // Width 3: Partial writeback
+ "tbz %x[N], #1, 35f\n"
+ "str d26, [%x[output_ptr]], #0x8\n"
+ "tbz %x[N], #0, 36f\n"
+ "st1 { v26.s }[2], [%x[output_ptr]]\n"
+ "b 36f\n"
+ "35:" // Width 3: Partial direct writeback: partial_1_8
+ "tbz %x[N], #0, 36f\n"
+ "str s26, [%x[output_ptr], #0x0]\n"
+ "36:" // Width 3: Writeback done
+ "b 97f\n"
+ "37:" // Width 4
+ "mov x20, %x[K]\n"
+ "mov x19, %x[A_ptr]\n"
+ "cbz x21, 38f\n"
+ "ldr q24, [x21, #0x0]\n"
+ "ldr q25, [x21, #0x10]\n"
+ "ldr q26, [x21, #0x20]\n"
+ "ldr q27, [x21, #0x30]\n"
+ "add x21, x21, #0x40\n"
+ "b 39f\n"
+ "38:" // Width 4: no bias
+ "movi v24.16b, #0x0\n"
+ "movi v25.16b, #0x0\n"
+ "movi v26.16b, #0x0\n"
+ "movi v27.16b, #0x0\n"
+ "39:" // Width 4: setup done
+ "cmp x20, #0x4\n"
+ "blt 42f\n"
+ "cmp x20, #0x8\n"
+ "blt 41f\n"
+ "40:" // Width 4: Multiply loop: Main loop head
+ "ldr q0, [x19, #0x0]\n"
+ "ldr q1, [%x[B_ptr], #0x0]\n"
+ "fmla v24.4s, v1.4s, v0.s[0]\n"
+ "ldr q2, [%x[B_ptr], #0x10]\n"
+ "ldr q3, [%x[B_ptr], #0x20]\n"
+ "fmla v25.4s, v2.4s, v0.s[0]\n"
+ "ldr q4, [%x[B_ptr], #0x30]\n"
+ "fmla v26.4s, v3.4s, v0.s[0]\n"
+ "add %x[B_ptr], %x[B_ptr], #0x80\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "fmla v27.4s, v4.4s, v0.s[0]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "ldr q5, [%x[B_ptr], #0x0]\n"
+ "fmla v24.4s, v5.4s, v0.s[1]\n"
+ "ldr q6, [%x[B_ptr], #0x10]\n"
+ "ldr q7, [%x[B_ptr], #0x20]\n"
+ "fmla v25.4s, v6.4s, v0.s[1]\n"
+ "ldr q8, [%x[B_ptr], #0x30]\n"
+ "fmla v26.4s, v7.4s, v0.s[1]\n"
+ "add %x[B_ptr], %x[B_ptr], #0x80\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "fmla v27.4s, v8.4s, v0.s[1]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "ldr q9, [%x[B_ptr], #0x0]\n"
+ "fmla v24.4s, v9.4s, v0.s[2]\n"
+ "ldr q10, [%x[B_ptr], #0x10]\n"
+ "ldr q11, [%x[B_ptr], #0x20]\n"
+ "fmla v25.4s, v10.4s, v0.s[2]\n"
+ "ldr q12, [%x[B_ptr], #0x30]\n"
+ "fmla v26.4s, v11.4s, v0.s[2]\n"
+ "add %x[B_ptr], %x[B_ptr], #0x80\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "fmla v27.4s, v12.4s, v0.s[2]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "ldr q13, [%x[B_ptr], #0x0]\n"
+ "fmla v24.4s, v13.4s, v0.s[3]\n"
+ "ldr q14, [%x[B_ptr], #0x10]\n"
+ "ldr q15, [%x[B_ptr], #0x20]\n"
+ "fmla v25.4s, v14.4s, v0.s[3]\n"
+ "ldr q16, [%x[B_ptr], #0x30]\n"
+ "fmla v26.4s, v15.4s, v0.s[3]\n"
+ "add %x[B_ptr], %x[B_ptr], #0x80\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "fmla v27.4s, v16.4s, v0.s[3]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "add x19, x19, #0x10\n"
+ "prfm pldl1keep, [x19, #0x80]\n"
+ "sub x20, x20, #0x4\n"
+ "cmp x20, #0x8\n"
+ "bge 40b\n"
+ "41:" // Width 4: Multiply loop: Single iteration only
+ "sub x20, x20, #0x4\n"
+ "ldr q0, [x19, #0x0]\n"
+ "ldr q17, [%x[B_ptr], #0x0]\n"
+ "fmla v24.4s, v17.4s, v0.s[0]\n"
+ "ldr q18, [%x[B_ptr], #0x10]\n"
+ "ldr q19, [%x[B_ptr], #0x20]\n"
+ "fmla v25.4s, v18.4s, v0.s[0]\n"
+ "ldr q20, [%x[B_ptr], #0x30]\n"
+ "fmla v26.4s, v19.4s, v0.s[0]\n"
+ "add %x[B_ptr], %x[B_ptr], #0x80\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "fmla v27.4s, v20.4s, v0.s[0]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "ldr q21, [%x[B_ptr], #0x0]\n"
+ "fmla v24.4s, v21.4s, v0.s[1]\n"
+ "ldr q22, [%x[B_ptr], #0x10]\n"
+ "ldr q23, [%x[B_ptr], #0x20]\n"
+ "fmla v25.4s, v22.4s, v0.s[1]\n"
+ "ldr q1, [%x[B_ptr], #0x30]\n"
+ "fmla v26.4s, v23.4s, v0.s[1]\n"
+ "add %x[B_ptr], %x[B_ptr], #0x80\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "fmla v27.4s, v1.4s, v0.s[1]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "ldr q2, [%x[B_ptr], #0x0]\n"
+ "fmla v24.4s, v2.4s, v0.s[2]\n"
+ "ldr q3, [%x[B_ptr], #0x10]\n"
+ "ldr q4, [%x[B_ptr], #0x20]\n"
+ "fmla v25.4s, v3.4s, v0.s[2]\n"
+ "ldr q5, [%x[B_ptr], #0x30]\n"
+ "fmla v26.4s, v4.4s, v0.s[2]\n"
+ "add %x[B_ptr], %x[B_ptr], #0x80\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "fmla v27.4s, v5.4s, v0.s[2]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "ldr q6, [%x[B_ptr], #0x0]\n"
+ "fmla v24.4s, v6.4s, v0.s[3]\n"
+ "ldr q7, [%x[B_ptr], #0x10]\n"
+ "ldr q8, [%x[B_ptr], #0x20]\n"
+ "fmla v25.4s, v7.4s, v0.s[3]\n"
+ "ldr q9, [%x[B_ptr], #0x30]\n"
+ "fmla v26.4s, v8.4s, v0.s[3]\n"
+ "add %x[B_ptr], %x[B_ptr], #0x80\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "fmla v27.4s, v9.4s, v0.s[3]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "add x19, x19, #0x10\n"
+ "prfm pldl1keep, [x19, #0x80]\n"
+ "42:" // Width 4: Multiply loop: Main loop skip
+ "cbz x20, 44f\n"
+ "43:" // Width 4: Multiply loop: Odd block loop
+ "ldr s0, [x19], #0x4\n"
+ "ldr q10, [%x[B_ptr], #0x0]\n"
+ "fmla v24.4s, v10.4s, v0.s[0]\n"
+ "ldr q11, [%x[B_ptr], #0x10]\n"
+ "ldr q12, [%x[B_ptr], #0x20]\n"
+ "fmla v25.4s, v11.4s, v0.s[0]\n"
+ "ldr q13, [%x[B_ptr], #0x30]\n"
+ "fmla v26.4s, v12.4s, v0.s[0]\n"
+ "add %x[B_ptr], %x[B_ptr], #0x80\n"
+ "sub x20, x20, #0x1\n"
+ "fmla v27.4s, v13.4s, v0.s[0]\n"
+ "cbnz x20, 43b\n"
+ "44:" // Width 4: Multiply loop: No odd multiplies
+ "prfm pstl1keep, [%x[output_ptr], #0x0]\n"
+ "tbz %x[flags], #1, 45f\n"
+ "add x19, %x[args_ptr], %[offset_min]\n"
+ "ld1r { v17.4s }, [x19]\n"
+ "add x19, %x[args_ptr], %[offset_max]\n"
+ "ld1r { v16.4s }, [x19]\n"
+ "fmin v24.4s, v24.4s, v16.4s\n"
+ "fmin v25.4s, v25.4s, v16.4s\n"
+ "fmin v26.4s, v26.4s, v16.4s\n"
+ "fmin v27.4s, v27.4s, v16.4s\n"
+ "fmax v24.4s, v24.4s, v17.4s\n"
+ "fmax v25.4s, v25.4s, v17.4s\n"
+ "fmax v26.4s, v26.4s, v17.4s\n"
+ "fmax v27.4s, v27.4s, v17.4s\n"
+ "45:" // Width 4: No activation
+ "str q24, [%x[output_ptr], #0x0]\n"
+ "str q25, [%x[output_ptr], #0x10]\n"
+ "str q26, [%x[output_ptr], #0x20]\n"
+ "cmp %x[N], #0x10\n"
+ "add %x[output_ptr], %x[output_ptr], #0x30\n"
+ "blt 46f\n"
+ "str q27, [%x[output_ptr], #0x0]\n"
+ "add %x[output_ptr], %x[output_ptr], #0x10\n"
+ "b 48f\n"
+ "46:" // Width 4: Partial writeback
+ "tbz %x[N], #1, 47f\n"
+ "str d27, [%x[output_ptr]], #0x8\n"
+ "tbz %x[N], #0, 48f\n"
+ "st1 { v27.s }[2], [%x[output_ptr]]\n"
+ "b 48f\n"
+ "47:" // Width 4: Partial direct writeback: partial_1_12
+ "tbz %x[N], #0, 48f\n"
+ "str s27, [%x[output_ptr], #0x0]\n"
+ "48:" // Width 4: Writeback done
+ "b 97f\n"
+ "49:" // Width 5
+ "mov x20, %x[K]\n"
+ "mov x19, %x[A_ptr]\n"
+ "cbz x21, 50f\n"
+ "ldr q24, [x21, #0x0]\n"
+ "ldr q25, [x21, #0x10]\n"
+ "ldr q26, [x21, #0x20]\n"
+ "ldr q27, [x21, #0x30]\n"
+ "ldr q28, [x21, #0x40]\n"
+ "add x21, x21, #0x50\n"
+ "b 51f\n"
+ "50:" // Width 5: no bias
+ "movi v24.16b, #0x0\n"
+ "movi v25.16b, #0x0\n"
+ "movi v26.16b, #0x0\n"
+ "movi v27.16b, #0x0\n"
+ "movi v28.16b, #0x0\n"
+ "51:" // Width 5: setup done
+ "cmp x20, #0x4\n"
+ "blt 54f\n"
+ "cmp x20, #0x8\n"
+ "blt 53f\n"
+ "52:" // Width 5: Multiply loop: Main loop head
+ "ldr q0, [x19, #0x0]\n"
+ "ldr q1, [%x[B_ptr], #0x0]\n"
+ "fmla v24.4s, v1.4s, v0.s[0]\n"
+ "ldr q2, [%x[B_ptr], #0x10]\n"
+ "ldr q3, [%x[B_ptr], #0x20]\n"
+ "fmla v25.4s, v2.4s, v0.s[0]\n"
+ "ldr q4, [%x[B_ptr], #0x30]\n"
+ "fmla v26.4s, v3.4s, v0.s[0]\n"
+ "ldr q5, [%x[B_ptr], #0x40]\n"
+ "add %x[B_ptr], %x[B_ptr], #0x80\n"
+ "fmla v27.4s, v4.4s, v0.s[0]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "ldr q6, [%x[B_ptr], #0x0]\n"
+ "fmla v28.4s, v5.4s, v0.s[0]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "ldr q7, [%x[B_ptr], #0x10]\n"
+ "fmla v24.4s, v6.4s, v0.s[1]\n"
+ "ldr q8, [%x[B_ptr], #0x20]\n"
+ "ldr q9, [%x[B_ptr], #0x30]\n"
+ "fmla v25.4s, v7.4s, v0.s[1]\n"
+ "ldr q10, [%x[B_ptr], #0x40]\n"
+ "fmla v26.4s, v8.4s, v0.s[1]\n"
+ "add %x[B_ptr], %x[B_ptr], #0x80\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "fmla v27.4s, v9.4s, v0.s[1]\n"
+ "ldr q11, [%x[B_ptr], #0x0]\n"
+ "fmla v28.4s, v10.4s, v0.s[1]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "ldr q12, [%x[B_ptr], #0x10]\n"
+ "fmla v24.4s, v11.4s, v0.s[2]\n"
+ "ldr q13, [%x[B_ptr], #0x20]\n"
+ "ldr q14, [%x[B_ptr], #0x30]\n"
+ "fmla v25.4s, v12.4s, v0.s[2]\n"
+ "ldr q15, [%x[B_ptr], #0x40]\n"
+ "fmla v26.4s, v13.4s, v0.s[2]\n"
+ "add %x[B_ptr], %x[B_ptr], #0x80\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "fmla v27.4s, v14.4s, v0.s[2]\n"
+ "ldr q16, [%x[B_ptr], #0x0]\n"
+ "fmla v28.4s, v15.4s, v0.s[2]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "ldr q17, [%x[B_ptr], #0x10]\n"
+ "fmla v24.4s, v16.4s, v0.s[3]\n"
+ "ldr q18, [%x[B_ptr], #0x20]\n"
+ "ldr q19, [%x[B_ptr], #0x30]\n"
+ "fmla v25.4s, v17.4s, v0.s[3]\n"
+ "ldr q20, [%x[B_ptr], #0x40]\n"
+ "fmla v26.4s, v18.4s, v0.s[3]\n"
+ "add %x[B_ptr], %x[B_ptr], #0x80\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "fmla v27.4s, v19.4s, v0.s[3]\n"
+ "add x19, x19, #0x10\n"
+ "fmla v28.4s, v20.4s, v0.s[3]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "sub x20, x20, #0x4\n"
+ "prfm pldl1keep, [x19, #0x80]\n"
+ "cmp x20, #0x8\n"
+ "bge 52b\n"
+ "53:" // Width 5: Multiply loop: Single iteration only
+ "sub x20, x20, #0x4\n"
+ "ldr q0, [x19, #0x0]\n"
+ "ldr q21, [%x[B_ptr], #0x0]\n"
+ "fmla v24.4s, v21.4s, v0.s[0]\n"
+ "ldr q22, [%x[B_ptr], #0x10]\n"
+ "ldr q23, [%x[B_ptr], #0x20]\n"
+ "fmla v25.4s, v22.4s, v0.s[0]\n"
+ "ldr q1, [%x[B_ptr], #0x30]\n"
+ "fmla v26.4s, v23.4s, v0.s[0]\n"
+ "ldr q2, [%x[B_ptr], #0x40]\n"
+ "add %x[B_ptr], %x[B_ptr], #0x80\n"
+ "fmla v27.4s, v1.4s, v0.s[0]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "ldr q3, [%x[B_ptr], #0x0]\n"
+ "fmla v28.4s, v2.4s, v0.s[0]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "ldr q4, [%x[B_ptr], #0x10]\n"
+ "fmla v24.4s, v3.4s, v0.s[1]\n"
+ "ldr q5, [%x[B_ptr], #0x20]\n"
+ "ldr q6, [%x[B_ptr], #0x30]\n"
+ "fmla v25.4s, v4.4s, v0.s[1]\n"
+ "ldr q7, [%x[B_ptr], #0x40]\n"
+ "fmla v26.4s, v5.4s, v0.s[1]\n"
+ "add %x[B_ptr], %x[B_ptr], #0x80\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "fmla v27.4s, v6.4s, v0.s[1]\n"
+ "ldr q8, [%x[B_ptr], #0x0]\n"
+ "fmla v28.4s, v7.4s, v0.s[1]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "ldr q9, [%x[B_ptr], #0x10]\n"
+ "fmla v24.4s, v8.4s, v0.s[2]\n"
+ "ldr q10, [%x[B_ptr], #0x20]\n"
+ "ldr q11, [%x[B_ptr], #0x30]\n"
+ "fmla v25.4s, v9.4s, v0.s[2]\n"
+ "ldr q12, [%x[B_ptr], #0x40]\n"
+ "fmla v26.4s, v10.4s, v0.s[2]\n"
+ "add %x[B_ptr], %x[B_ptr], #0x80\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "fmla v27.4s, v11.4s, v0.s[2]\n"
+ "ldr q13, [%x[B_ptr], #0x0]\n"
+ "fmla v28.4s, v12.4s, v0.s[2]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "ldr q14, [%x[B_ptr], #0x10]\n"
+ "fmla v24.4s, v13.4s, v0.s[3]\n"
+ "ldr q15, [%x[B_ptr], #0x20]\n"
+ "ldr q16, [%x[B_ptr], #0x30]\n"
+ "fmla v25.4s, v14.4s, v0.s[3]\n"
+ "ldr q17, [%x[B_ptr], #0x40]\n"
+ "fmla v26.4s, v15.4s, v0.s[3]\n"
+ "add %x[B_ptr], %x[B_ptr], #0x80\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "fmla v27.4s, v16.4s, v0.s[3]\n"
+ "add x19, x19, #0x10\n"
+ "fmla v28.4s, v17.4s, v0.s[3]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "prfm pldl1keep, [x19, #0x80]\n"
+ "54:" // Width 5: Multiply loop: Main loop skip
+ "cbz x20, 56f\n"
+ "55:" // Width 5: Multiply loop: Odd block loop
+ "ldr s0, [x19], #0x4\n"
+ "ldr q18, [%x[B_ptr], #0x0]\n"
+ "fmla v24.4s, v18.4s, v0.s[0]\n"
+ "ldr q19, [%x[B_ptr], #0x10]\n"
+ "ldr q20, [%x[B_ptr], #0x20]\n"
+ "fmla v25.4s, v19.4s, v0.s[0]\n"
+ "ldr q21, [%x[B_ptr], #0x30]\n"
+ "fmla v26.4s, v20.4s, v0.s[0]\n"
+ "ldr q22, [%x[B_ptr], #0x40]\n"
+ "add %x[B_ptr], %x[B_ptr], #0x80\n"
+ "fmla v27.4s, v21.4s, v0.s[0]\n"
+ "sub x20, x20, #0x1\n"
+ "fmla v28.4s, v22.4s, v0.s[0]\n"
+ "cbnz x20, 55b\n"
+ "56:" // Width 5: Multiply loop: No odd multiplies
+ "prfm pstl1keep, [%x[output_ptr], #0x0]\n"
+ "tbz %x[flags], #1, 57f\n"
+ "add x19, %x[args_ptr], %[offset_min]\n"
+ "ld1r { v17.4s }, [x19]\n"
+ "add x19, %x[args_ptr], %[offset_max]\n"
+ "ld1r { v16.4s }, [x19]\n"
+ "fmin v24.4s, v24.4s, v16.4s\n"
+ "fmin v25.4s, v25.4s, v16.4s\n"
+ "fmin v26.4s, v26.4s, v16.4s\n"
+ "fmin v27.4s, v27.4s, v16.4s\n"
+ "fmax v24.4s, v24.4s, v17.4s\n"
+ "fmax v25.4s, v25.4s, v17.4s\n"
+ "fmax v26.4s, v26.4s, v17.4s\n"
+ "fmax v27.4s, v27.4s, v17.4s\n"
+ "fmin v28.4s, v28.4s, v16.4s\n"
+ "fmax v28.4s, v28.4s, v17.4s\n"
+ "57:" // Width 5: No activation
+ "str q24, [%x[output_ptr], #0x0]\n"
+ "str q25, [%x[output_ptr], #0x10]\n"
+ "str q26, [%x[output_ptr], #0x20]\n"
+ "str q27, [%x[output_ptr], #0x30]\n"
+ "cmp %x[N], #0x14\n"
+ "add %x[output_ptr], %x[output_ptr], #0x40\n"
+ "blt 58f\n"
+ "str q28, [%x[output_ptr], #0x0]\n"
+ "add %x[output_ptr], %x[output_ptr], #0x10\n"
+ "b 60f\n"
+ "58:" // Width 5: Partial writeback
+ "tbz %x[N], #1, 59f\n"
+ "str d28, [%x[output_ptr]], #0x8\n"
+ "tbz %x[N], #0, 60f\n"
+ "st1 { v28.s }[2], [%x[output_ptr]]\n"
+ "b 60f\n"
+ "59:" // Width 5: Partial direct writeback: partial_1_16
+ "tbz %x[N], #0, 60f\n"
+ "str s28, [%x[output_ptr], #0x0]\n"
+ "60:" // Width 5: Writeback done
+ "b 97f\n"
+ "61:" // Width 6
+ "mov x20, %x[K]\n"
+ "mov x19, %x[A_ptr]\n"
+ "cbz x21, 62f\n"
+ "ldr q24, [x21, #0x0]\n"
+ "ldr q25, [x21, #0x10]\n"
+ "ldr q26, [x21, #0x20]\n"
+ "ldr q27, [x21, #0x30]\n"
+ "ldr q28, [x21, #0x40]\n"
+ "ldr q29, [x21, #0x50]\n"
+ "add x21, x21, #0x60\n"
+ "b 63f\n"
+ "62:" // Width 6: no bias
+ "movi v24.16b, #0x0\n"
+ "movi v25.16b, #0x0\n"
+ "movi v26.16b, #0x0\n"
+ "movi v27.16b, #0x0\n"
+ "movi v28.16b, #0x0\n"
+ "movi v29.16b, #0x0\n"
+ "63:" // Width 6: setup done
+ "cmp x20, #0x4\n"
+ "blt 66f\n"
+ "cmp x20, #0x8\n"
+ "blt 65f\n"
+ "64:" // Width 6: Multiply loop: Main loop head
+ "ldr q0, [x19, #0x0]\n"
+ "ldr q1, [%x[B_ptr], #0x0]\n"
+ "fmla v24.4s, v1.4s, v0.s[0]\n"
+ "ldr q2, [%x[B_ptr], #0x10]\n"
+ "ldr q3, [%x[B_ptr], #0x20]\n"
+ "fmla v25.4s, v2.4s, v0.s[0]\n"
+ "ldr q4, [%x[B_ptr], #0x30]\n"
+ "fmla v26.4s, v3.4s, v0.s[0]\n"
+ "ldr q5, [%x[B_ptr], #0x40]\n"
+ "ldr q6, [%x[B_ptr], #0x50]\n"
+ "fmla v27.4s, v4.4s, v0.s[0]\n"
+ "add %x[B_ptr], %x[B_ptr], #0x80\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "fmla v28.4s, v5.4s, v0.s[0]\n"
+ "ldr q7, [%x[B_ptr], #0x0]\n"
+ "fmla v29.4s, v6.4s, v0.s[0]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "ldr q8, [%x[B_ptr], #0x10]\n"
+ "fmla v24.4s, v7.4s, v0.s[1]\n"
+ "ldr q9, [%x[B_ptr], #0x20]\n"
+ "ldr q10, [%x[B_ptr], #0x30]\n"
+ "fmla v25.4s, v8.4s, v0.s[1]\n"
+ "ldr q11, [%x[B_ptr], #0x40]\n"
+ "fmla v26.4s, v9.4s, v0.s[1]\n"
+ "ldr q12, [%x[B_ptr], #0x50]\n"
+ "fmla v27.4s, v10.4s, v0.s[1]\n"
+ "add %x[B_ptr], %x[B_ptr], #0x80\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "fmla v28.4s, v11.4s, v0.s[1]\n"
+ "ldr q13, [%x[B_ptr], #0x0]\n"
+ "fmla v29.4s, v12.4s, v0.s[1]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "ldr q14, [%x[B_ptr], #0x10]\n"
+ "fmla v24.4s, v13.4s, v0.s[2]\n"
+ "ldr q15, [%x[B_ptr], #0x20]\n"
+ "ldr q16, [%x[B_ptr], #0x30]\n"
+ "fmla v25.4s, v14.4s, v0.s[2]\n"
+ "ldr q17, [%x[B_ptr], #0x40]\n"
+ "ldr q18, [%x[B_ptr], #0x50]\n"
+ "fmla v26.4s, v15.4s, v0.s[2]\n"
+ "add %x[B_ptr], %x[B_ptr], #0x80\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "fmla v27.4s, v16.4s, v0.s[2]\n"
+ "ldr q19, [%x[B_ptr], #0x0]\n"
+ "fmla v28.4s, v17.4s, v0.s[2]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "ldr q20, [%x[B_ptr], #0x10]\n"
+ "fmla v29.4s, v18.4s, v0.s[2]\n"
+ "ldr q21, [%x[B_ptr], #0x20]\n"
+ "ldr q22, [%x[B_ptr], #0x30]\n"
+ "fmla v24.4s, v19.4s, v0.s[3]\n"
+ "ldr q23, [%x[B_ptr], #0x40]\n"
+ "ldr q1, [%x[B_ptr], #0x50]\n"
+ "fmla v25.4s, v20.4s, v0.s[3]\n"
+ "add %x[B_ptr], %x[B_ptr], #0x80\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "fmla v26.4s, v21.4s, v0.s[3]\n"
+ "add x19, x19, #0x10\n"
+ "fmla v27.4s, v22.4s, v0.s[3]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "sub x20, x20, #0x4\n"
+ "fmla v28.4s, v23.4s, v0.s[3]\n"
+ "prfm pldl1keep, [x19, #0x80]\n"
+ "cmp x20, #0x8\n"
+ "fmla v29.4s, v1.4s, v0.s[3]\n"
+ "bge 64b\n"
+ "65:" // Width 6: Multiply loop: Single iteration only
+ "sub x20, x20, #0x4\n"
+ "ldr q0, [x19, #0x0]\n"
+ "ldr q2, [%x[B_ptr], #0x0]\n"
+ "fmla v24.4s, v2.4s, v0.s[0]\n"
+ "ldr q3, [%x[B_ptr], #0x10]\n"
+ "ldr q4, [%x[B_ptr], #0x20]\n"
+ "fmla v25.4s, v3.4s, v0.s[0]\n"
+ "ldr q5, [%x[B_ptr], #0x30]\n"
+ "fmla v26.4s, v4.4s, v0.s[0]\n"
+ "ldr q6, [%x[B_ptr], #0x40]\n"
+ "ldr q7, [%x[B_ptr], #0x50]\n"
+ "fmla v27.4s, v5.4s, v0.s[0]\n"
+ "add %x[B_ptr], %x[B_ptr], #0x80\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "fmla v28.4s, v6.4s, v0.s[0]\n"
+ "ldr q8, [%x[B_ptr], #0x0]\n"
+ "fmla v29.4s, v7.4s, v0.s[0]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "ldr q9, [%x[B_ptr], #0x10]\n"
+ "fmla v24.4s, v8.4s, v0.s[1]\n"
+ "ldr q10, [%x[B_ptr], #0x20]\n"
+ "ldr q11, [%x[B_ptr], #0x30]\n"
+ "fmla v25.4s, v9.4s, v0.s[1]\n"
+ "ldr q12, [%x[B_ptr], #0x40]\n"
+ "fmla v26.4s, v10.4s, v0.s[1]\n"
+ "ldr q13, [%x[B_ptr], #0x50]\n"
+ "fmla v27.4s, v11.4s, v0.s[1]\n"
+ "add %x[B_ptr], %x[B_ptr], #0x80\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "fmla v28.4s, v12.4s, v0.s[1]\n"
+ "ldr q14, [%x[B_ptr], #0x0]\n"
+ "fmla v29.4s, v13.4s, v0.s[1]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "ldr q15, [%x[B_ptr], #0x10]\n"
+ "fmla v24.4s, v14.4s, v0.s[2]\n"
+ "ldr q16, [%x[B_ptr], #0x20]\n"
+ "ldr q17, [%x[B_ptr], #0x30]\n"
+ "fmla v25.4s, v15.4s, v0.s[2]\n"
+ "ldr q18, [%x[B_ptr], #0x40]\n"
+ "ldr q19, [%x[B_ptr], #0x50]\n"
+ "fmla v26.4s, v16.4s, v0.s[2]\n"
+ "add %x[B_ptr], %x[B_ptr], #0x80\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "fmla v27.4s, v17.4s, v0.s[2]\n"
+ "ldr q20, [%x[B_ptr], #0x0]\n"
+ "fmla v28.4s, v18.4s, v0.s[2]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "ldr q21, [%x[B_ptr], #0x10]\n"
+ "fmla v29.4s, v19.4s, v0.s[2]\n"
+ "ldr q22, [%x[B_ptr], #0x20]\n"
+ "ldr q23, [%x[B_ptr], #0x30]\n"
+ "fmla v24.4s, v20.4s, v0.s[3]\n"
+ "ldr q1, [%x[B_ptr], #0x40]\n"
+ "ldr q2, [%x[B_ptr], #0x50]\n"
+ "fmla v25.4s, v21.4s, v0.s[3]\n"
+ "add %x[B_ptr], %x[B_ptr], #0x80\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "fmla v26.4s, v22.4s, v0.s[3]\n"
+ "add x19, x19, #0x10\n"
+ "fmla v27.4s, v23.4s, v0.s[3]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "prfm pldl1keep, [x19, #0x80]\n"
+ "fmla v28.4s, v1.4s, v0.s[3]\n"
+ "fmla v29.4s, v2.4s, v0.s[3]\n"
+ "66:" // Width 6: Multiply loop: Main loop skip
+ "cbz x20, 68f\n"
+ "67:" // Width 6: Multiply loop: Odd block loop
+ "ldr s0, [x19], #0x4\n"
+ "ldr q3, [%x[B_ptr], #0x0]\n"
+ "fmla v24.4s, v3.4s, v0.s[0]\n"
+ "ldr q4, [%x[B_ptr], #0x10]\n"
+ "ldr q5, [%x[B_ptr], #0x20]\n"
+ "fmla v25.4s, v4.4s, v0.s[0]\n"
+ "ldr q6, [%x[B_ptr], #0x30]\n"
+ "fmla v26.4s, v5.4s, v0.s[0]\n"
+ "ldr q7, [%x[B_ptr], #0x40]\n"
+ "ldr q8, [%x[B_ptr], #0x50]\n"
+ "fmla v27.4s, v6.4s, v0.s[0]\n"
+ "add %x[B_ptr], %x[B_ptr], #0x80\n"
+ "sub x20, x20, #0x1\n"
+ "fmla v28.4s, v7.4s, v0.s[0]\n"
+ "fmla v29.4s, v8.4s, v0.s[0]\n"
+ "cbnz x20, 67b\n"
+ "68:" // Width 6: Multiply loop: No odd multiplies
+ "prfm pstl1keep, [%x[output_ptr], #0x0]\n"
+ "tbz %x[flags], #1, 69f\n"
+ "add x19, %x[args_ptr], %[offset_min]\n"
+ "ld1r { v17.4s }, [x19]\n"
+ "add x19, %x[args_ptr], %[offset_max]\n"
+ "ld1r { v16.4s }, [x19]\n"
+ "fmin v24.4s, v24.4s, v16.4s\n"
+ "fmin v25.4s, v25.4s, v16.4s\n"
+ "fmin v26.4s, v26.4s, v16.4s\n"
+ "fmin v27.4s, v27.4s, v16.4s\n"
+ "fmax v24.4s, v24.4s, v17.4s\n"
+ "fmax v25.4s, v25.4s, v17.4s\n"
+ "fmax v26.4s, v26.4s, v17.4s\n"
+ "fmax v27.4s, v27.4s, v17.4s\n"
+ "fmin v28.4s, v28.4s, v16.4s\n"
+ "fmin v29.4s, v29.4s, v16.4s\n"
+ "fmax v28.4s, v28.4s, v17.4s\n"
+ "fmax v29.4s, v29.4s, v17.4s\n"
+ "69:" // Width 6: No activation
+ "str q24, [%x[output_ptr], #0x0]\n"
+ "str q25, [%x[output_ptr], #0x10]\n"
+ "str q26, [%x[output_ptr], #0x20]\n"
+ "str q27, [%x[output_ptr], #0x30]\n"
+ "str q28, [%x[output_ptr], #0x40]\n"
+ "cmp %x[N], #0x18\n"
+ "add %x[output_ptr], %x[output_ptr], #0x50\n"
+ "blt 70f\n"
+ "str q29, [%x[output_ptr], #0x0]\n"
+ "add %x[output_ptr], %x[output_ptr], #0x10\n"
+ "b 72f\n"
+ "70:" // Width 6: Partial writeback
+ "tbz %x[N], #1, 71f\n"
+ "str d29, [%x[output_ptr]], #0x8\n"
+ "tbz %x[N], #0, 72f\n"
+ "st1 { v29.s }[2], [%x[output_ptr]]\n"
+ "b 72f\n"
+ "71:" // Width 6: Partial direct writeback: partial_1_20
+ "tbz %x[N], #0, 72f\n"
+ "str s29, [%x[output_ptr], #0x0]\n"
+ "72:" // Width 6: Writeback done
+ "b 97f\n"
+ "73:" // Width 7
+ "mov x20, %x[K]\n"
+ "mov x19, %x[A_ptr]\n"
+ "cbz x21, 74f\n"
+ "ldr q24, [x21, #0x0]\n"
+ "ldr q25, [x21, #0x10]\n"
+ "ldr q26, [x21, #0x20]\n"
+ "ldr q27, [x21, #0x30]\n"
+ "ldr q28, [x21, #0x40]\n"
+ "ldr q29, [x21, #0x50]\n"
+ "ldr q30, [x21, #0x60]\n"
+ "add x21, x21, #0x70\n"
+ "b 75f\n"
+ "74:" // Width 7: no bias
+ "movi v24.16b, #0x0\n"
+ "movi v25.16b, #0x0\n"
+ "movi v26.16b, #0x0\n"
+ "movi v27.16b, #0x0\n"
+ "movi v28.16b, #0x0\n"
+ "movi v29.16b, #0x0\n"
+ "movi v30.16b, #0x0\n"
+ "75:" // Width 7: setup done
+ "cmp x20, #0x4\n"
+ "blt 78f\n"
+ "cmp x20, #0x8\n"
+ "blt 77f\n"
+ "76:" // Width 7: Multiply loop: Main loop head
+ "ldr q0, [x19, #0x0]\n"
+ "ldr q1, [%x[B_ptr], #0x0]\n"
+ "fmla v24.4s, v1.4s, v0.s[0]\n"
+ "ldr q2, [%x[B_ptr], #0x10]\n"
+ "ldr q3, [%x[B_ptr], #0x20]\n"
+ "fmla v25.4s, v2.4s, v0.s[0]\n"
+ "ldr q4, [%x[B_ptr], #0x30]\n"
+ "fmla v26.4s, v3.4s, v0.s[0]\n"
+ "ldr q5, [%x[B_ptr], #0x40]\n"
+ "ldr q6, [%x[B_ptr], #0x50]\n"
+ "fmla v27.4s, v4.4s, v0.s[0]\n"
+ "ldr q7, [%x[B_ptr], #0x60]\n"
+ "add %x[B_ptr], %x[B_ptr], #0x80\n"
+ "fmla v28.4s, v5.4s, v0.s[0]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "fmla v29.4s, v6.4s, v0.s[0]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "ldr q8, [%x[B_ptr], #0x0]\n"
+ "fmla v30.4s, v7.4s, v0.s[0]\n"
+ "ldr q9, [%x[B_ptr], #0x10]\n"
+ "ldr q10, [%x[B_ptr], #0x20]\n"
+ "fmla v24.4s, v8.4s, v0.s[1]\n"
+ "ldr q11, [%x[B_ptr], #0x30]\n"
+ "ldr q12, [%x[B_ptr], #0x40]\n"
+ "fmla v25.4s, v9.4s, v0.s[1]\n"
+ "ldr q13, [%x[B_ptr], #0x50]\n"
+ "fmla v26.4s, v10.4s, v0.s[1]\n"
+ "ldr q14, [%x[B_ptr], #0x60]\n"
+ "add %x[B_ptr], %x[B_ptr], #0x80\n"
+ "fmla v27.4s, v11.4s, v0.s[1]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "fmla v28.4s, v12.4s, v0.s[1]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "ldr q15, [%x[B_ptr], #0x0]\n"
+ "fmla v29.4s, v13.4s, v0.s[1]\n"
+ "ldr q16, [%x[B_ptr], #0x10]\n"
+ "ldr q17, [%x[B_ptr], #0x20]\n"
+ "fmla v30.4s, v14.4s, v0.s[1]\n"
+ "ldr q18, [%x[B_ptr], #0x30]\n"
+ "fmla v24.4s, v15.4s, v0.s[2]\n"
+ "ldr q19, [%x[B_ptr], #0x40]\n"
+ "ldr q20, [%x[B_ptr], #0x50]\n"
+ "fmla v25.4s, v16.4s, v0.s[2]\n"
+ "ldr q21, [%x[B_ptr], #0x60]\n"
+ "fmla v26.4s, v17.4s, v0.s[2]\n"
+ "add %x[B_ptr], %x[B_ptr], #0x80\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "fmla v27.4s, v18.4s, v0.s[2]\n"
+ "ldr q22, [%x[B_ptr], #0x0]\n"
+ "fmla v28.4s, v19.4s, v0.s[2]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "ldr q23, [%x[B_ptr], #0x10]\n"
+ "fmla v29.4s, v20.4s, v0.s[2]\n"
+ "ldr q1, [%x[B_ptr], #0x20]\n"
+ "ldr q2, [%x[B_ptr], #0x30]\n"
+ "fmla v30.4s, v21.4s, v0.s[2]\n"
+ "ldr q3, [%x[B_ptr], #0x40]\n"
+ "fmla v24.4s, v22.4s, v0.s[3]\n"
+ "ldr q4, [%x[B_ptr], #0x50]\n"
+ "ldr q5, [%x[B_ptr], #0x60]\n"
+ "fmla v25.4s, v23.4s, v0.s[3]\n"
+ "add %x[B_ptr], %x[B_ptr], #0x80\n"
+ "fmla v26.4s, v1.4s, v0.s[3]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "fmla v27.4s, v2.4s, v0.s[3]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "add x19, x19, #0x10\n"
+ "fmla v28.4s, v3.4s, v0.s[3]\n"
+ "prfm pldl1keep, [x19, #0x80]\n"
+ "sub x20, x20, #0x4\n"
+ "fmla v29.4s, v4.4s, v0.s[3]\n"
+ "cmp x20, #0x8\n"
+ "fmla v30.4s, v5.4s, v0.s[3]\n"
+ "bge 76b\n"
+ "77:" // Width 7: Multiply loop: Single iteration only
+ "sub x20, x20, #0x4\n"
+ "ldr q0, [x19, #0x0]\n"
+ "ldr q6, [%x[B_ptr], #0x0]\n"
+ "fmla v24.4s, v6.4s, v0.s[0]\n"
+ "ldr q7, [%x[B_ptr], #0x10]\n"
+ "ldr q8, [%x[B_ptr], #0x20]\n"
+ "fmla v25.4s, v7.4s, v0.s[0]\n"
+ "ldr q9, [%x[B_ptr], #0x30]\n"
+ "fmla v26.4s, v8.4s, v0.s[0]\n"
+ "ldr q10, [%x[B_ptr], #0x40]\n"
+ "ldr q11, [%x[B_ptr], #0x50]\n"
+ "fmla v27.4s, v9.4s, v0.s[0]\n"
+ "ldr q12, [%x[B_ptr], #0x60]\n"
+ "add %x[B_ptr], %x[B_ptr], #0x80\n"
+ "fmla v28.4s, v10.4s, v0.s[0]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "fmla v29.4s, v11.4s, v0.s[0]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "ldr q13, [%x[B_ptr], #0x0]\n"
+ "fmla v30.4s, v12.4s, v0.s[0]\n"
+ "ldr q14, [%x[B_ptr], #0x10]\n"
+ "ldr q15, [%x[B_ptr], #0x20]\n"
+ "fmla v24.4s, v13.4s, v0.s[1]\n"
+ "ldr q16, [%x[B_ptr], #0x30]\n"
+ "ldr q17, [%x[B_ptr], #0x40]\n"
+ "fmla v25.4s, v14.4s, v0.s[1]\n"
+ "ldr q18, [%x[B_ptr], #0x50]\n"
+ "fmla v26.4s, v15.4s, v0.s[1]\n"
+ "ldr q19, [%x[B_ptr], #0x60]\n"
+ "add %x[B_ptr], %x[B_ptr], #0x80\n"
+ "fmla v27.4s, v16.4s, v0.s[1]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "fmla v28.4s, v17.4s, v0.s[1]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "ldr q20, [%x[B_ptr], #0x0]\n"
+ "fmla v29.4s, v18.4s, v0.s[1]\n"
+ "ldr q21, [%x[B_ptr], #0x10]\n"
+ "ldr q22, [%x[B_ptr], #0x20]\n"
+ "fmla v30.4s, v19.4s, v0.s[1]\n"
+ "ldr q23, [%x[B_ptr], #0x30]\n"
+ "fmla v24.4s, v20.4s, v0.s[2]\n"
+ "ldr q1, [%x[B_ptr], #0x40]\n"
+ "ldr q2, [%x[B_ptr], #0x50]\n"
+ "fmla v25.4s, v21.4s, v0.s[2]\n"
+ "ldr q3, [%x[B_ptr], #0x60]\n"
+ "fmla v26.4s, v22.4s, v0.s[2]\n"
+ "add %x[B_ptr], %x[B_ptr], #0x80\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "fmla v27.4s, v23.4s, v0.s[2]\n"
+ "ldr q4, [%x[B_ptr], #0x0]\n"
+ "fmla v28.4s, v1.4s, v0.s[2]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "ldr q5, [%x[B_ptr], #0x10]\n"
+ "fmla v29.4s, v2.4s, v0.s[2]\n"
+ "ldr q6, [%x[B_ptr], #0x20]\n"
+ "ldr q7, [%x[B_ptr], #0x30]\n"
+ "fmla v30.4s, v3.4s, v0.s[2]\n"
+ "ldr q8, [%x[B_ptr], #0x40]\n"
+ "fmla v24.4s, v4.4s, v0.s[3]\n"
+ "ldr q9, [%x[B_ptr], #0x50]\n"
+ "ldr q10, [%x[B_ptr], #0x60]\n"
+ "fmla v25.4s, v5.4s, v0.s[3]\n"
+ "add %x[B_ptr], %x[B_ptr], #0x80\n"
+ "fmla v26.4s, v6.4s, v0.s[3]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "fmla v27.4s, v7.4s, v0.s[3]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "add x19, x19, #0x10\n"
+ "fmla v28.4s, v8.4s, v0.s[3]\n"
+ "prfm pldl1keep, [x19, #0x80]\n"
+ "fmla v29.4s, v9.4s, v0.s[3]\n"
+ "fmla v30.4s, v10.4s, v0.s[3]\n"
+ "78:" // Width 7: Multiply loop: Main loop skip
+ "cbz x20, 80f\n"
+ "79:" // Width 7: Multiply loop: Odd block loop
+ "ldr s0, [x19], #0x4\n"
+ "ldr q11, [%x[B_ptr], #0x0]\n"
+ "fmla v24.4s, v11.4s, v0.s[0]\n"
+ "ldr q12, [%x[B_ptr], #0x10]\n"
+ "ldr q13, [%x[B_ptr], #0x20]\n"
+ "fmla v25.4s, v12.4s, v0.s[0]\n"
+ "ldr q14, [%x[B_ptr], #0x30]\n"
+ "fmla v26.4s, v13.4s, v0.s[0]\n"
+ "ldr q15, [%x[B_ptr], #0x40]\n"
+ "ldr q16, [%x[B_ptr], #0x50]\n"
+ "fmla v27.4s, v14.4s, v0.s[0]\n"
+ "ldr q17, [%x[B_ptr], #0x60]\n"
+ "add %x[B_ptr], %x[B_ptr], #0x80\n"
+ "fmla v28.4s, v15.4s, v0.s[0]\n"
+ "fmla v29.4s, v16.4s, v0.s[0]\n"
+ "sub x20, x20, #0x1\n"
+ "fmla v30.4s, v17.4s, v0.s[0]\n"
+ "cbnz x20, 79b\n"
+ "80:" // Width 7: Multiply loop: No odd multiplies
+ "prfm pstl1keep, [%x[output_ptr], #0x0]\n"
+ "tbz %x[flags], #1, 81f\n"
+ "add x19, %x[args_ptr], %[offset_min]\n"
+ "ld1r { v17.4s }, [x19]\n"
+ "add x19, %x[args_ptr], %[offset_max]\n"
+ "ld1r { v16.4s }, [x19]\n"
+ "fmin v24.4s, v24.4s, v16.4s\n"
+ "fmin v25.4s, v25.4s, v16.4s\n"
+ "fmin v26.4s, v26.4s, v16.4s\n"
+ "fmin v27.4s, v27.4s, v16.4s\n"
+ "fmax v24.4s, v24.4s, v17.4s\n"
+ "fmax v25.4s, v25.4s, v17.4s\n"
+ "fmax v26.4s, v26.4s, v17.4s\n"
+ "fmax v27.4s, v27.4s, v17.4s\n"
+ "fmin v28.4s, v28.4s, v16.4s\n"
+ "fmin v29.4s, v29.4s, v16.4s\n"
+ "fmin v30.4s, v30.4s, v16.4s\n"
+ "fmax v28.4s, v28.4s, v17.4s\n"
+ "fmax v29.4s, v29.4s, v17.4s\n"
+ "fmax v30.4s, v30.4s, v17.4s\n"
+ "81:" // Width 7: No activation
+ "str q24, [%x[output_ptr], #0x0]\n"
+ "str q25, [%x[output_ptr], #0x10]\n"
+ "str q26, [%x[output_ptr], #0x20]\n"
+ "str q27, [%x[output_ptr], #0x30]\n"
+ "str q28, [%x[output_ptr], #0x40]\n"
+ "str q29, [%x[output_ptr], #0x50]\n"
+ "cmp %x[N], #0x1c\n"
+ "add %x[output_ptr], %x[output_ptr], #0x60\n"
+ "blt 82f\n"
+ "str q30, [%x[output_ptr], #0x0]\n"
+ "add %x[output_ptr], %x[output_ptr], #0x10\n"
+ "b 84f\n"
+ "82:" // Width 7: Partial writeback
+ "tbz %x[N], #1, 83f\n"
+ "str d30, [%x[output_ptr]], #0x8\n"
+ "tbz %x[N], #0, 84f\n"
+ "st1 { v30.s }[2], [%x[output_ptr]]\n"
+ "b 84f\n"
+ "83:" // Width 7: Partial direct writeback: partial_1_24
+ "tbz %x[N], #0, 84f\n"
+ "str s30, [%x[output_ptr], #0x0]\n"
+ "84:" // Width 7: Writeback done
+ "b 97f\n"
+ "85:" // Width 8
+ "mov x20, %x[K]\n"
+ "mov x19, %x[A_ptr]\n"
+ "cbz x21, 86f\n"
+ "ldr q24, [x21, #0x0]\n"
+ "ldr q25, [x21, #0x10]\n"
+ "ldr q26, [x21, #0x20]\n"
+ "ldr q27, [x21, #0x30]\n"
+ "ldr q28, [x21, #0x40]\n"
+ "ldr q29, [x21, #0x50]\n"
+ "ldr q30, [x21, #0x60]\n"
+ "ldr q31, [x21, #0x70]\n"
+ "add x21, x21, #0x80\n"
+ "b 87f\n"
+ "86:" // Width 8: no bias
+ "movi v24.16b, #0x0\n"
+ "movi v25.16b, #0x0\n"
+ "movi v26.16b, #0x0\n"
+ "movi v27.16b, #0x0\n"
+ "movi v28.16b, #0x0\n"
+ "movi v29.16b, #0x0\n"
+ "movi v30.16b, #0x0\n"
+ "movi v31.16b, #0x0\n"
+ "87:" // Width 8: setup done
+ "cmp x20, #0x4\n"
+ "blt 90f\n"
+ "cmp x20, #0x8\n"
+ "blt 89f\n"
+ "88:" // Width 8: Multiply loop: Main loop head
+ "ldr q0, [x19, #0x0]\n"
+ "ldr q1, [%x[B_ptr], #0x0]\n"
+ "fmla v24.4s, v1.4s, v0.s[0]\n"
+ "ldr q2, [%x[B_ptr], #0x10]\n"
+ "ldr q3, [%x[B_ptr], #0x20]\n"
+ "fmla v25.4s, v2.4s, v0.s[0]\n"
+ "ldr q4, [%x[B_ptr], #0x30]\n"
+ "fmla v26.4s, v3.4s, v0.s[0]\n"
+ "ldr q5, [%x[B_ptr], #0x40]\n"
+ "ldr q6, [%x[B_ptr], #0x50]\n"
+ "fmla v27.4s, v4.4s, v0.s[0]\n"
+ "ldr q7, [%x[B_ptr], #0x60]\n"
+ "ldr q8, [%x[B_ptr], #0x70]\n"
+ "fmla v28.4s, v5.4s, v0.s[0]\n"
+ "fmla v29.4s, v6.4s, v0.s[0]\n"
+ "add %x[B_ptr], %x[B_ptr], #0x80\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "fmla v30.4s, v7.4s, v0.s[0]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "ldr q9, [%x[B_ptr], #0x0]\n"
+ "fmla v31.4s, v8.4s, v0.s[0]\n"
+ "ldr q10, [%x[B_ptr], #0x10]\n"
+ "ldr q11, [%x[B_ptr], #0x20]\n"
+ "fmla v24.4s, v9.4s, v0.s[1]\n"
+ "ldr q12, [%x[B_ptr], #0x30]\n"
+ "ldr q13, [%x[B_ptr], #0x40]\n"
+ "fmla v25.4s, v10.4s, v0.s[1]\n"
+ "fmla v26.4s, v11.4s, v0.s[1]\n"
+ "ldr q14, [%x[B_ptr], #0x50]\n"
+ "ldr q15, [%x[B_ptr], #0x60]\n"
+ "fmla v27.4s, v12.4s, v0.s[1]\n"
+ "ldr q16, [%x[B_ptr], #0x70]\n"
+ "fmla v28.4s, v13.4s, v0.s[1]\n"
+ "add %x[B_ptr], %x[B_ptr], #0x80\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "fmla v29.4s, v14.4s, v0.s[1]\n"
+ "ldr q17, [%x[B_ptr], #0x0]\n"
+ "fmla v30.4s, v15.4s, v0.s[1]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "ldr q18, [%x[B_ptr], #0x10]\n"
+ "fmla v31.4s, v16.4s, v0.s[1]\n"
+ "ldr q19, [%x[B_ptr], #0x20]\n"
+ "ldr q20, [%x[B_ptr], #0x30]\n"
+ "fmla v24.4s, v17.4s, v0.s[2]\n"
+ "ldr q21, [%x[B_ptr], #0x40]\n"
+ "ldr q22, [%x[B_ptr], #0x50]\n"
+ "fmla v25.4s, v18.4s, v0.s[2]\n"
+ "ldr q23, [%x[B_ptr], #0x60]\n"
+ "fmla v26.4s, v19.4s, v0.s[2]\n"
+ "ldr q1, [%x[B_ptr], #0x70]\n"
+ "fmla v27.4s, v20.4s, v0.s[2]\n"
+ "add %x[B_ptr], %x[B_ptr], #0x80\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "fmla v28.4s, v21.4s, v0.s[2]\n"
+ "ldr q2, [%x[B_ptr], #0x0]\n"
+ "fmla v29.4s, v22.4s, v0.s[2]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "ldr q3, [%x[B_ptr], #0x10]\n"
+ "fmla v30.4s, v23.4s, v0.s[2]\n"
+ "ldr q4, [%x[B_ptr], #0x20]\n"
+ "ldr q5, [%x[B_ptr], #0x30]\n"
+ "fmla v31.4s, v1.4s, v0.s[2]\n"
+ "ldr q6, [%x[B_ptr], #0x40]\n"
+ "fmla v24.4s, v2.4s, v0.s[3]\n"
+ "ldr q7, [%x[B_ptr], #0x50]\n"
+ "ldr q8, [%x[B_ptr], #0x60]\n"
+ "fmla v25.4s, v3.4s, v0.s[3]\n"
+ "ldr q9, [%x[B_ptr], #0x70]\n"
+ "fmla v26.4s, v4.4s, v0.s[3]\n"
+ "fmla v27.4s, v5.4s, v0.s[3]\n"
+ "add %x[B_ptr], %x[B_ptr], #0x80\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "fmla v28.4s, v6.4s, v0.s[3]\n"
+ "add x19, x19, #0x10\n"
+ "fmla v29.4s, v7.4s, v0.s[3]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "sub x20, x20, #0x4\n"
+ "fmla v30.4s, v8.4s, v0.s[3]\n"
+ "prfm pldl1keep, [x19, #0x80]\n"
+ "cmp x20, #0x8\n"
+ "fmla v31.4s, v9.4s, v0.s[3]\n"
+ "bge 88b\n"
+ "89:" // Width 8: Multiply loop: Single iteration only
+ "sub x20, x20, #0x4\n"
+ "ldr q0, [x19, #0x0]\n"
+ "ldr q10, [%x[B_ptr], #0x0]\n"
+ "fmla v24.4s, v10.4s, v0.s[0]\n"
+ "ldr q11, [%x[B_ptr], #0x10]\n"
+ "ldr q12, [%x[B_ptr], #0x20]\n"
+ "fmla v25.4s, v11.4s, v0.s[0]\n"
+ "ldr q13, [%x[B_ptr], #0x30]\n"
+ "fmla v26.4s, v12.4s, v0.s[0]\n"
+ "ldr q14, [%x[B_ptr], #0x40]\n"
+ "ldr q15, [%x[B_ptr], #0x50]\n"
+ "fmla v27.4s, v13.4s, v0.s[0]\n"
+ "ldr q16, [%x[B_ptr], #0x60]\n"
+ "ldr q17, [%x[B_ptr], #0x70]\n"
+ "fmla v28.4s, v14.4s, v0.s[0]\n"
+ "fmla v29.4s, v15.4s, v0.s[0]\n"
+ "add %x[B_ptr], %x[B_ptr], #0x80\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "fmla v30.4s, v16.4s, v0.s[0]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "ldr q18, [%x[B_ptr], #0x0]\n"
+ "fmla v31.4s, v17.4s, v0.s[0]\n"
+ "ldr q19, [%x[B_ptr], #0x10]\n"
+ "ldr q20, [%x[B_ptr], #0x20]\n"
+ "fmla v24.4s, v18.4s, v0.s[1]\n"
+ "ldr q21, [%x[B_ptr], #0x30]\n"
+ "ldr q22, [%x[B_ptr], #0x40]\n"
+ "fmla v25.4s, v19.4s, v0.s[1]\n"
+ "fmla v26.4s, v20.4s, v0.s[1]\n"
+ "ldr q23, [%x[B_ptr], #0x50]\n"
+ "ldr q1, [%x[B_ptr], #0x60]\n"
+ "fmla v27.4s, v21.4s, v0.s[1]\n"
+ "ldr q2, [%x[B_ptr], #0x70]\n"
+ "fmla v28.4s, v22.4s, v0.s[1]\n"
+ "add %x[B_ptr], %x[B_ptr], #0x80\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "fmla v29.4s, v23.4s, v0.s[1]\n"
+ "ldr q3, [%x[B_ptr], #0x0]\n"
+ "fmla v30.4s, v1.4s, v0.s[1]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "ldr q4, [%x[B_ptr], #0x10]\n"
+ "fmla v31.4s, v2.4s, v0.s[1]\n"
+ "ldr q5, [%x[B_ptr], #0x20]\n"
+ "ldr q6, [%x[B_ptr], #0x30]\n"
+ "fmla v24.4s, v3.4s, v0.s[2]\n"
+ "ldr q7, [%x[B_ptr], #0x40]\n"
+ "ldr q8, [%x[B_ptr], #0x50]\n"
+ "fmla v25.4s, v4.4s, v0.s[2]\n"
+ "ldr q9, [%x[B_ptr], #0x60]\n"
+ "fmla v26.4s, v5.4s, v0.s[2]\n"
+ "ldr q10, [%x[B_ptr], #0x70]\n"
+ "fmla v27.4s, v6.4s, v0.s[2]\n"
+ "add %x[B_ptr], %x[B_ptr], #0x80\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "fmla v28.4s, v7.4s, v0.s[2]\n"
+ "ldr q11, [%x[B_ptr], #0x0]\n"
+ "fmla v29.4s, v8.4s, v0.s[2]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "ldr q12, [%x[B_ptr], #0x10]\n"
+ "fmla v30.4s, v9.4s, v0.s[2]\n"
+ "ldr q13, [%x[B_ptr], #0x20]\n"
+ "ldr q14, [%x[B_ptr], #0x30]\n"
+ "fmla v31.4s, v10.4s, v0.s[2]\n"
+ "ldr q15, [%x[B_ptr], #0x40]\n"
+ "fmla v24.4s, v11.4s, v0.s[3]\n"
+ "ldr q16, [%x[B_ptr], #0x50]\n"
+ "ldr q17, [%x[B_ptr], #0x60]\n"
+ "fmla v25.4s, v12.4s, v0.s[3]\n"
+ "ldr q18, [%x[B_ptr], #0x70]\n"
+ "fmla v26.4s, v13.4s, v0.s[3]\n"
+ "fmla v27.4s, v14.4s, v0.s[3]\n"
+ "add %x[B_ptr], %x[B_ptr], #0x80\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "fmla v28.4s, v15.4s, v0.s[3]\n"
+ "add x19, x19, #0x10\n"
+ "fmla v29.4s, v16.4s, v0.s[3]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "fmla v30.4s, v17.4s, v0.s[3]\n"
+ "prfm pldl1keep, [x19, #0x80]\n"
+ "fmla v31.4s, v18.4s, v0.s[3]\n"
+ "90:" // Width 8: Multiply loop: Main loop skip
+ "cbz x20, 92f\n"
+ "91:" // Width 8: Multiply loop: Odd block loop
+ "ldr s0, [x19], #0x4\n"
+ "ldr q19, [%x[B_ptr], #0x0]\n"
+ "fmla v24.4s, v19.4s, v0.s[0]\n"
+ "ldr q20, [%x[B_ptr], #0x10]\n"
+ "ldr q21, [%x[B_ptr], #0x20]\n"
+ "fmla v25.4s, v20.4s, v0.s[0]\n"
+ "ldr q22, [%x[B_ptr], #0x30]\n"
+ "fmla v26.4s, v21.4s, v0.s[0]\n"
+ "ldr q23, [%x[B_ptr], #0x40]\n"
+ "ldr q1, [%x[B_ptr], #0x50]\n"
+ "fmla v27.4s, v22.4s, v0.s[0]\n"
+ "ldr q2, [%x[B_ptr], #0x60]\n"
+ "ldr q3, [%x[B_ptr], #0x70]\n"
+ "fmla v28.4s, v23.4s, v0.s[0]\n"
+ "fmla v29.4s, v1.4s, v0.s[0]\n"
+ "add %x[B_ptr], %x[B_ptr], #0x80\n"
+ "sub x20, x20, #0x1\n"
+ "fmla v30.4s, v2.4s, v0.s[0]\n"
+ "fmla v31.4s, v3.4s, v0.s[0]\n"
+ "cbnz x20, 91b\n"
+ "92:" // Width 8: Multiply loop: No odd multiplies
+ "prfm pstl1keep, [%x[output_ptr], #0x0]\n"
+ "tbz %x[flags], #1, 93f\n"
+ "add x19, %x[args_ptr], %[offset_min]\n"
+ "ld1r { v17.4s }, [x19]\n"
+ "add x19, %x[args_ptr], %[offset_max]\n"
+ "ld1r { v16.4s }, [x19]\n"
+ "fmin v24.4s, v24.4s, v16.4s\n"
+ "fmin v25.4s, v25.4s, v16.4s\n"
+ "fmin v26.4s, v26.4s, v16.4s\n"
+ "fmin v27.4s, v27.4s, v16.4s\n"
+ "fmax v24.4s, v24.4s, v17.4s\n"
+ "fmax v25.4s, v25.4s, v17.4s\n"
+ "fmax v26.4s, v26.4s, v17.4s\n"
+ "fmax v27.4s, v27.4s, v17.4s\n"
+ "fmin v28.4s, v28.4s, v16.4s\n"
+ "fmin v29.4s, v29.4s, v16.4s\n"
+ "fmin v30.4s, v30.4s, v16.4s\n"
+ "fmax v28.4s, v28.4s, v17.4s\n"
+ "fmax v29.4s, v29.4s, v17.4s\n"
+ "fmax v30.4s, v30.4s, v17.4s\n"
+ "fmin v31.4s, v31.4s, v16.4s\n"
+ "fmax v31.4s, v31.4s, v17.4s\n"
+ "93:" // Width 8: No activation
+ "str q24, [%x[output_ptr], #0x0]\n"
+ "str q25, [%x[output_ptr], #0x10]\n"
+ "str q26, [%x[output_ptr], #0x20]\n"
+ "str q27, [%x[output_ptr], #0x30]\n"
+ "str q28, [%x[output_ptr], #0x40]\n"
+ "str q29, [%x[output_ptr], #0x50]\n"
+ "str q30, [%x[output_ptr], #0x60]\n"
+ "cmp %x[N], #0x20\n"
+ "add %x[output_ptr], %x[output_ptr], #0x70\n"
+ "blt 94f\n"
+ "str q31, [%x[output_ptr], #0x0]\n"
+ "add %x[output_ptr], %x[output_ptr], #0x10\n"
+ "b 96f\n"
+ "94:" // Width 8: Partial writeback
+ "tbz %x[N], #1, 95f\n"
+ "str d31, [%x[output_ptr]], #0x8\n"
+ "tbz %x[N], #0, 96f\n"
+ "st1 { v31.s }[2], [%x[output_ptr]]\n"
+ "b 96f\n"
+ "95:" // Width 8: Partial direct writeback: partial_1_28
+ "tbz %x[N], #0, 96f\n"
+ "str s31, [%x[output_ptr], #0x0]\n"
+ "96:" // Width 8: Writeback done
+ "subs x22, x22, #0x8\n"
+ "sub %x[N], %x[N], #0x20\n"
+ "bgt 1b\n"
+ "97:" // Exit
+
+ : [B_ptr] "+r" (B_ptr), [N] "+r" (N), [output_ptr] "+r" (output_ptr)
+ : [A_ptr] "r" (A_ptr), [K] "r" (K), [args_ptr] "r" (&ka), [bias] "r" (bias), [flags] "r" (flags), [offset_max] "I" (offsetof(KernelArgs, maxval)), [offset_min] "I" (offsetof(KernelArgs, minval))
+ : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x19", "x20", "x21", "x22"
+ );
+}
+
+} // namespace arm_gemm
+
+#endif
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hgemm_24x8.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hgemm_8x24.hpp
similarity index 83%
rename from src/core/NEON/kernels/arm_gemm/kernels/a64_hgemm_24x8.hpp
rename to src/core/NEON/kernels/arm_gemm/kernels/a64_hgemm_8x24.hpp
index 79cae60..73fb5b7 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hgemm_24x8.hpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hgemm_8x24.hpp
@@ -30,15 +30,15 @@
namespace arm_gemm {
// Actual kernel implementations
-void a64_hgemm_asimd_24x8(const __fp16 *, const __fp16 *, __fp16 *, int, int, int);
-void a64_hgemm_asimd_24x8_a55r1(const __fp16 *, const __fp16 *, __fp16 *, int, int, int);
-void a64_hgemm_asimd_24x8_x1(const __fp16 *, const __fp16 *, __fp16 *, int, int, int);
+void a64_hgemm_asimd_8x24(const __fp16 *, const __fp16 *, __fp16 *, int, int, int);
+void a64_hgemm_asimd_8x24_a55r1(const __fp16 *, const __fp16 *, __fp16 *, int, int, int);
+void a64_hgemm_asimd_8x24_x1(const __fp16 *, const __fp16 *, __fp16 *, int, int, int);
-// 24x8 HGEMM "strategy" class. Describes the kernel properties.
+// 8x24 HGEMM "strategy" class. Describes the kernel properties.
//
// The generic "gemm_opt" function will instantiate one of these (allowing
// the constructor to pick a kernel implementation).
-class hgemm_24x8 {
+class cls_a64_hgemm_8x24 {
public:
typedef __fp16 operand_type;
typedef __fp16 result_type;
@@ -62,15 +62,15 @@
StdTransformsFixed<operand_type, result_type, 8, 24> transforms = {};
// Default to the generic kernel
- kern_type kernel = a64_hgemm_asimd_24x8;
+ kern_type kernel = a64_hgemm_asimd_8x24;
- hgemm_24x8(const CPUInfo *ci) {
+ cls_a64_hgemm_8x24(const CPUInfo *ci) {
auto model = ci->get_cpu_model();
if (model == CPUModel::A55r1) {
- kernel = a64_hgemm_asimd_24x8_a55r1;
+ kernel = a64_hgemm_asimd_8x24_a55r1;
} else if (model == CPUModel::X1) {
- kernel = a64_hgemm_asimd_24x8_x1;
+ kernel = a64_hgemm_asimd_8x24_x1;
}
}
};
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hgemm_24x8/a55r1.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hgemm_8x24/a55r1.cpp
similarity index 99%
rename from src/core/NEON/kernels/arm_gemm/kernels/a64_hgemm_24x8/a55r1.cpp
rename to src/core/NEON/kernels/arm_gemm/kernels/a64_hgemm_8x24/a55r1.cpp
index 829ae30..29cdd33 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hgemm_24x8/a55r1.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hgemm_8x24/a55r1.cpp
@@ -41,7 +41,7 @@
namespace arm_gemm {
-void a64_hgemm_asimd_24x8_a55r1(const __fp16 *Apanel, const __fp16 *Bpanel, __fp16 *Cpanel, int ablocks, int bblocks, int K) {
+void a64_hgemm_asimd_8x24_a55r1(const __fp16 *Apanel, const __fp16 *Bpanel, __fp16 *Cpanel, int ablocks, int bblocks, int K) {
const __fp16 *a_ptr = Apanel;
__fp16 *c_ptr = Cpanel;
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hgemm_24x8/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hgemm_8x24/generic.cpp
similarity index 98%
rename from src/core/NEON/kernels/arm_gemm/kernels/a64_hgemm_24x8/generic.cpp
rename to src/core/NEON/kernels/arm_gemm/kernels/a64_hgemm_8x24/generic.cpp
index 657fade..c9c48dd 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hgemm_24x8/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hgemm_8x24/generic.cpp
@@ -34,14 +34,14 @@
// Assume that "Apanel" points to a chunk of A blocks (each size 8xK) in read-order.
// Assume that "Bpanel" points to a chunk of B blocks (each size 24xK) in read-order.
// Assume that "Cpanel" points to a chunk of C output blocks (each size
-// 24x8), the chunks being arranged in a row major fashion.
+// 8x24), the chunks being arranged in a row major fashion.
//
// Note that the intent of this is that either ablocks or bblocks will be 1
// - this construction allows the output loop to proceed in either order.
namespace arm_gemm {
-void a64_hgemm_asimd_24x8(const __fp16 *Apanel, const __fp16 *Bpanel, __fp16 *Cpanel, int ablocks, int bblocks, int K) {
+void a64_hgemm_asimd_8x24(const __fp16 *Apanel, const __fp16 *Bpanel, __fp16 *Cpanel, int ablocks, int bblocks, int K) {
const __fp16 *a_ptr = Apanel;
__fp16 *c_ptr = Cpanel;
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hgemm_24x8/x1.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hgemm_8x24/x1.cpp
similarity index 98%
rename from src/core/NEON/kernels/arm_gemm/kernels/a64_hgemm_24x8/x1.cpp
rename to src/core/NEON/kernels/arm_gemm/kernels/a64_hgemm_8x24/x1.cpp
index 3bb8334..a6d2405 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hgemm_24x8/x1.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hgemm_8x24/x1.cpp
@@ -34,14 +34,14 @@
// Assume that "Apanel" points to a chunk of A blocks (each size 8xK) in read-order.
// Assume that "Bpanel" points to a chunk of B blocks (each size 24xK) in read-order.
// Assume that "Cpanel" points to a chunk of C output blocks (each size
-// 24x8), the chunks being arranged in a row major fashion.
+// 8x24), the chunks being arranged in a row major fashion.
//
// Note that the intent of this is that either ablocks or bblocks will be 1
// - this construction allows the output loop to proceed in either order.
namespace arm_gemm {
-void a64_hgemm_asimd_24x8_x1(const __fp16 *Apanel, const __fp16 *Bpanel, __fp16 *Cpanel, int ablocks, int bblocks, int K) {
+void a64_hgemm_asimd_8x24_x1(const __fp16 *Apanel, const __fp16 *Bpanel, __fp16 *Cpanel, int ablocks, int bblocks, int K) {
const __fp16 *a_ptr = Apanel;
__fp16 *c_ptr = Cpanel;
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_4x6.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_bf16fp32_dot_6x16.hpp
similarity index 63%
copy from src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_4x6.hpp
copy to src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_bf16fp32_dot_6x16.hpp
index 6f31efe..a76c994 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_4x6.hpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_bf16fp32_dot_6x16.hpp
@@ -10,34 +10,44 @@
* sell copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
*/
#pragma once
-
#ifdef __aarch64__
+#include "../std_transforms_fixed.hpp"
+#include "../bfloat.hpp"
+
+#define ARGLIST \
+ unsigned int, const unsigned int *, \
+ IndirectInputArg<bfloat16>, \
+ size_t, size_t, \
+ const bfloat16 *, \
+ IndirectOutputArg<float>, \
+ const float *, Activation, bool
+
namespace arm_gemm
{
// Actual kernel implementations
-void a64_smallK_hybrid_fp32_mla_4x6(const float *, int, const float *, float *, int, int, int, int, const float *, Activation, bool);
+void a64_hybrid_bf16fp32_dot_6x16( ARGLIST );
-class smallK_hybrid_fp32_mla_4x6
+class cls_a64_hybrid_bf16fp32_dot_6x16
{
public:
- typedef float operand_type;
+ typedef bfloat16 operand_type;
typedef float result_type;
- typedef void (*kern_type)(const float *, int, const float *, float *, int, int, int, int, const float *, Activation, bool);
+ typedef void (*kern_type)( ARGLIST );
/* Kernel blocking parameters */
static constexpr unsigned int out_height()
@@ -47,40 +57,30 @@
static unsigned int out_width()
{
- return 4;
+ return 16;
}
static constexpr unsigned int k_unroll()
{
- return 1;
+ return 2;
}
static constexpr bool supports_accumulate()
{
- return false;
- }
-
- static constexpr bool supports_bias()
- {
return true;
}
- static constexpr bool supports_activation()
- {
- return true;
- }
-
- StdTransformsFixed<operand_type, result_type, 6, 4, 1> transforms = {};
+ StdTransformsFixed<operand_type, result_type, 6, 16, 2> transforms = {};
// Default to the generic kernel
- kern_type kernel=a64_smallK_hybrid_fp32_mla_4x6;
+ kern_type kernel=a64_hybrid_bf16fp32_dot_6x16;
- smallK_hybrid_fp32_mla_4x6(const CPUInfo *)
+ cls_a64_hybrid_bf16fp32_dot_6x16(const CPUInfo *)
{
-
}
};
} // namespace arm_gemm
+#undef ARGLIST
#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_bf16fp32_dot_6x16/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_bf16fp32_dot_6x16/generic.cpp
new file mode 100644
index 0000000..be680ed
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_bf16fp32_dot_6x16/generic.cpp
@@ -0,0 +1,3668 @@
+/*
+ * Copyright (c) 2019-2020 Arm Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+#ifdef __aarch64__
+
+#include "arm_gemm.hpp"
+#include "../../utils.hpp"
+#include "../../bfloat.hpp"
+
+#include <cassert>
+
+namespace arm_gemm {
+
+void a64_hybrid_bf16fp32_dot_6x16 (
+ unsigned int num_strings, const unsigned int *string_lengths, IndirectInputArg<bfloat16> A_arg,
+ size_t M, size_t N, const bfloat16 *B_ptr, IndirectOutputArg<float> output_arg,
+ const float *bias, Activation act, bool accumulate
+)
+{
+ struct KernelArgs {
+ float maxval = static_cast<float>(std::numeric_limits<float>::infinity());
+ float minval = - static_cast<float>(std::numeric_limits<float>::infinity());
+ unsigned int num_strings = {};
+ const unsigned int *string_lengths = {};
+ size_t N = {};
+ const bfloat16 *B_ptr = {};
+ size_t output_offset = {};
+ size_t input_initial_col = {};
+ size_t input_offset = {};
+ } ka;
+
+ unsigned long flags=0;
+ void *output_ptr;
+ void *input_ptr;
+
+ if (output_arg.is_indirect) {
+ output_ptr=(void *)(output_arg.indirect.ptr);
+ ka.output_offset=output_arg.indirect.offset;
+ flags |= 0x4;
+ } else {
+ output_ptr=(void *)(output_arg.direct.base);
+ ka.output_offset=output_arg.direct.stride;
+ }
+
+ if (A_arg.is_indirect) {
+ input_ptr=(void *)(A_arg.indirect.ptr);
+ ka.input_offset=A_arg.indirect.start_row;
+ ka.input_initial_col=A_arg.indirect.start_col;
+ flags |= 0x8;
+ } else {
+ assert(num_strings==1);
+ input_ptr=(void *)(A_arg.direct.base);
+ ka.input_offset=A_arg.direct.stride;
+ }
+ if (accumulate) {
+ flags |= 0x1;
+ }
+ ka.num_strings = num_strings;
+ ka.string_lengths = string_lengths;
+ ka.N = N;
+ ka.B_ptr = B_ptr;
+ switch(act.type) {
+ default:
+ case Activation::Type::None:
+ break;
+ case Activation::Type::BoundedReLU:
+ ka.maxval = static_cast<float>(act.param1);
+ /* fall through */
+ case Activation::Type::ReLU:
+ ka.minval = 0;
+ flags |= 0x2;
+ break;
+ }
+ __asm__ __volatile__(
+
+ "1:" // Row loop
+ "cmp %x[M], #0x6\n"
+ "bge 186f\n"
+ "cmp %x[M], #0x4\n"
+ "bgt 149f\n"
+ "beq 112f\n"
+ "cmp %x[M], #0x2\n"
+ "bgt 75f\n"
+ "beq 38f\n"
+ "ldr x16, [%x[args_ptr], %[offsetof_N]]\n"
+ "ldr x15, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "mov x14, %x[bias]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 2f\n"
+ "ldr x13, [%x[output_ptr], #0x0]\n"
+ "add x13, x13, x19, LSL #2\n"
+ "b 3f\n"
+ "2:" // Height 1: setup direct output
+ "mov x13, %x[output_ptr]\n"
+ "3:" // Height 1: Column loop
+ "cbz x14, 4f\n"
+ "ldr q8, [x14, #0x0]\n"
+ "ldr q9, [x14, #0x10]\n"
+ "ldr q10, [x14, #0x20]\n"
+ "ldr q11, [x14, #0x30]\n"
+ "add x14, x14, #0x40\n"
+ "b 15f\n"
+ "4:" // Height 1: no bias
+ "tbz %x[flags], #0, 14f\n"
+ "cmp x16, #0x10\n"
+ "bge 13f\n"
+ "tbz x16, #3, 8f\n"
+ "ld1 { v8.4s }, [x13], #0x10\n"
+ "ld1 { v9.4s }, [x13], #0x10\n"
+ "tbz x16, #2, 6f\n"
+ "ld1 { v10.4s }, [x13], #0x10\n"
+ "tbz x16, #1, 5f\n"
+ "mov x19, #0x38\n"
+ "ldr d11, [x13], #0x8\n"
+ "tbz x16, #0, 12f\n"
+ "ld1 { v11.s }[2], [x13]\n"
+ "b 12f\n"
+ "5:" // Height 1: Partial accumulate: partial_1_12
+ "mov x19, #0x30\n"
+ "tbz x16, #0, 12f\n"
+ "ldr s11, [x13, #0x0]\n"
+ "b 12f\n"
+ "6:" // Height 1: Partial accumulate: partial_2_8
+ "tbz x16, #1, 7f\n"
+ "ldr d10, [x13], #0x8\n"
+ "mov x19, #0x28\n"
+ "tbz x16, #0, 12f\n"
+ "ld1 { v10.s }[2], [x13]\n"
+ "b 12f\n"
+ "7:" // Height 1: Partial accumulate: partial_1_8
+ "mov x19, #0x20\n"
+ "tbz x16, #0, 12f\n"
+ "ldr s10, [x13, #0x0]\n"
+ "b 12f\n"
+ "8:" // Height 1: Partial accumulate: partial_4_0
+ "tbz x16, #2, 10f\n"
+ "ld1 { v8.4s }, [x13], #0x10\n"
+ "tbz x16, #1, 9f\n"
+ "mov x19, #0x18\n"
+ "ldr d9, [x13], #0x8\n"
+ "tbz x16, #0, 12f\n"
+ "ld1 { v9.s }[2], [x13]\n"
+ "b 12f\n"
+ "9:" // Height 1: Partial accumulate: partial_1_4
+ "mov x19, #0x10\n"
+ "tbz x16, #0, 12f\n"
+ "ldr s9, [x13, #0x0]\n"
+ "b 12f\n"
+ "10:" // Height 1: Partial accumulate: partial_2_0
+ "tbz x16, #1, 11f\n"
+ "ldr d8, [x13], #0x8\n"
+ "mov x19, #0x8\n"
+ "tbz x16, #0, 12f\n"
+ "ld1 { v8.s }[2], [x13]\n"
+ "b 12f\n"
+ "11:" // Height 1: Partial accumulate: partial_1_0
+ "mov x19, #0x0\n"
+ "ldr s8, [x13, #0x0]\n"
+ "12:" // Height 1: Partial accumulate: Done
+ "sub x13, x13, x19\n"
+ "b 15f\n"
+ "13:" // Height 1: full accumulate
+ "ldr q8, [x13, #0x0]\n"
+ "ldr q9, [x13, #0x10]\n"
+ "ldr q10, [x13, #0x20]\n"
+ "ldr q11, [x13, #0x30]\n"
+ "b 15f\n"
+ "14:" // Height 1: no accumulate
+ "movi v8.16b, #0x0\n"
+ "movi v9.16b, #0x0\n"
+ "movi v10.16b, #0x0\n"
+ "movi v11.16b, #0x0\n"
+ "15:" // Height 1: setup done
+ "mov x12, #0x0\n"
+ "16:" // Height 1: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w11, [x20, x12, LSL #0x2]\n"
+ "tbz %x[flags], #3, 17f\n"
+ "ldr x20, [%x[input_ptr], x12, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x10, [x20, #0x0]\n"
+ "cbnz x12, 18f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x10, x10, x19, LSL #1\n"
+ "b 18f\n"
+ "17:" // Height 1: setup direct input
+ "mov x10, %x[input_ptr]\n"
+ "18:" // Height 1: input setup done
+ "cmp x11, #0x8\n"
+ "blt 21f\n"
+ "cmp x11, #0x10\n"
+ "blt 20f\n"
+ "19:" // Height 1: Multiply loop: Main loop head
+ "ldr q0, [x10, #0x0]\n"
+ "ldr q6, [x15, #0x0]\n"
+ ".inst 0x4f40f0c8 // bfdot v8.4s, v6.8h, v0.h[0]\n"
+ "ldr q7, [x15, #0x10]\n"
+ "ldr q6, [x15, #0x20]\n"
+ ".inst 0x4f40f0e9 // bfdot v9.4s, v7.8h, v0.h[0]\n"
+ "ldr q7, [x15, #0x30]\n"
+ ".inst 0x4f40f0ca // bfdot v10.4s, v6.8h, v0.h[0]\n"
+ "ldr q6, [x15, #0x40]\n"
+ "add x10, x10, #0x10\n"
+ ".inst 0x4f40f0eb // bfdot v11.4s, v7.8h, v0.h[0]\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "ldr q7, [x15, #0x50]\n"
+ ".inst 0x4f60f0c8 // bfdot v8.4s, v6.8h, v0.h[1]\n"
+ "ldr q6, [x15, #0x60]\n"
+ "sub x11, x11, #0x8\n"
+ ".inst 0x4f60f0e9 // bfdot v9.4s, v7.8h, v0.h[1]\n"
+ "ldr q7, [x15, #0x70]\n"
+ "cmp x11, #0x10\n"
+ ".inst 0x4f60f0ca // bfdot v10.4s, v6.8h, v0.h[1]\n"
+ "ldr q6, [x15, #0x80]\n"
+ ".inst 0x4f60f0eb // bfdot v11.4s, v7.8h, v0.h[1]\n"
+ "ldr q7, [x15, #0x90]\n"
+ ".inst 0x4f40f8c8 // bfdot v8.4s, v6.8h, v0.h[2]\n"
+ "ldr q6, [x15, #0xa0]\n"
+ ".inst 0x4f40f8e9 // bfdot v9.4s, v7.8h, v0.h[2]\n"
+ "ldr q7, [x15, #0xb0]\n"
+ ".inst 0x4f40f8ca // bfdot v10.4s, v6.8h, v0.h[2]\n"
+ "ldr q6, [x15, #0xc0]\n"
+ ".inst 0x4f40f8eb // bfdot v11.4s, v7.8h, v0.h[2]\n"
+ "ldr q7, [x15, #0xd0]\n"
+ ".inst 0x4f60f8c8 // bfdot v8.4s, v6.8h, v0.h[3]\n"
+ "ldr q6, [x15, #0xe0]\n"
+ ".inst 0x4f60f8e9 // bfdot v9.4s, v7.8h, v0.h[3]\n"
+ "ldr q7, [x15, #0xf0]\n"
+ "add x15, x15, #0x100\n"
+ ".inst 0x4f60f8ca // bfdot v10.4s, v6.8h, v0.h[3]\n"
+ ".inst 0x4f60f8eb // bfdot v11.4s, v7.8h, v0.h[3]\n"
+ "bge 19b\n"
+ "20:" // Height 1: Multiply loop: Single iteration only
+ "sub x11, x11, #0x8\n"
+ "ldr q0, [x10, #0x0]\n"
+ "ldr q6, [x15, #0x0]\n"
+ ".inst 0x4f40f0c8 // bfdot v8.4s, v6.8h, v0.h[0]\n"
+ "ldr q7, [x15, #0x10]\n"
+ "ldr q6, [x15, #0x20]\n"
+ ".inst 0x4f40f0e9 // bfdot v9.4s, v7.8h, v0.h[0]\n"
+ "ldr q7, [x15, #0x30]\n"
+ ".inst 0x4f40f0ca // bfdot v10.4s, v6.8h, v0.h[0]\n"
+ "ldr q6, [x15, #0x40]\n"
+ "add x10, x10, #0x10\n"
+ ".inst 0x4f40f0eb // bfdot v11.4s, v7.8h, v0.h[0]\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "ldr q7, [x15, #0x50]\n"
+ ".inst 0x4f60f0c8 // bfdot v8.4s, v6.8h, v0.h[1]\n"
+ "ldr q6, [x15, #0x60]\n"
+ ".inst 0x4f60f0e9 // bfdot v9.4s, v7.8h, v0.h[1]\n"
+ "ldr q7, [x15, #0x70]\n"
+ ".inst 0x4f60f0ca // bfdot v10.4s, v6.8h, v0.h[1]\n"
+ "ldr q6, [x15, #0x80]\n"
+ ".inst 0x4f60f0eb // bfdot v11.4s, v7.8h, v0.h[1]\n"
+ "ldr q7, [x15, #0x90]\n"
+ ".inst 0x4f40f8c8 // bfdot v8.4s, v6.8h, v0.h[2]\n"
+ "ldr q6, [x15, #0xa0]\n"
+ ".inst 0x4f40f8e9 // bfdot v9.4s, v7.8h, v0.h[2]\n"
+ "ldr q7, [x15, #0xb0]\n"
+ ".inst 0x4f40f8ca // bfdot v10.4s, v6.8h, v0.h[2]\n"
+ "ldr q6, [x15, #0xc0]\n"
+ ".inst 0x4f40f8eb // bfdot v11.4s, v7.8h, v0.h[2]\n"
+ "ldr q7, [x15, #0xd0]\n"
+ ".inst 0x4f60f8c8 // bfdot v8.4s, v6.8h, v0.h[3]\n"
+ "ldr q6, [x15, #0xe0]\n"
+ ".inst 0x4f60f8e9 // bfdot v9.4s, v7.8h, v0.h[3]\n"
+ "ldr q7, [x15, #0xf0]\n"
+ "add x15, x15, #0x100\n"
+ ".inst 0x4f60f8ca // bfdot v10.4s, v6.8h, v0.h[3]\n"
+ ".inst 0x4f60f8eb // bfdot v11.4s, v7.8h, v0.h[3]\n"
+ "21:" // Height 1: Multiply loop: Main loop skip
+ "cbz x11, 26f\n"
+ "cmp x11, #0x2\n"
+ "blt 23f\n"
+ "22:" // Height 1: Multiply loop: Odd block loop
+ "ldr s0, [x10], #0x4\n"
+ "ldr q6, [x15, #0x0]\n"
+ ".inst 0x4f40f0c8 // bfdot v8.4s, v6.8h, v0.h[0]\n"
+ "ldr q7, [x15, #0x10]\n"
+ "ldr q6, [x15, #0x20]\n"
+ ".inst 0x4f40f0e9 // bfdot v9.4s, v7.8h, v0.h[0]\n"
+ "ldr q7, [x15, #0x30]\n"
+ ".inst 0x4f40f0ca // bfdot v10.4s, v6.8h, v0.h[0]\n"
+ "sub x11, x11, #0x2\n"
+ "add x15, x15, #0x40\n"
+ ".inst 0x4f40f0eb // bfdot v11.4s, v7.8h, v0.h[0]\n"
+ "cmp x11, #0x2\n"
+ "bge 22b\n"
+ "cbz x11, 26f\n"
+ "23:" // Height 1: Multiply loop: Skip odd blocks
+ "tbz x11, #1, 24f\n"
+ "ldr s0, [x10], #0x4\n"
+ "tbz x11, #0, 25f\n"
+ "ld1 { v0.h }[2], [x10]\n"
+ "b 25f\n"
+ "24:" // Height 1: Multiply loop: Ragged operand read: partial_1_0
+ "ldr h0, [x10, #0x0]\n"
+ "25:" // Height 1: Multiply loop: Ragged operand read: Done
+ "ldr q6, [x15, #0x0]\n"
+ ".inst 0x4f40f0c8 // bfdot v8.4s, v6.8h, v0.h[0]\n"
+ "ldr q7, [x15, #0x10]\n"
+ "ldr q6, [x15, #0x20]\n"
+ ".inst 0x4f40f0e9 // bfdot v9.4s, v7.8h, v0.h[0]\n"
+ "ldr q7, [x15, #0x30]\n"
+ ".inst 0x4f40f0ca // bfdot v10.4s, v6.8h, v0.h[0]\n"
+ "add x15, x15, #0x40\n"
+ ".inst 0x4f40f0eb // bfdot v11.4s, v7.8h, v0.h[0]\n"
+ "26:" // Height 1: Multiply loop: No odd multiplies
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "add x12, x12, #0x1\n"
+ "cmp x12, x19\n"
+ "bne 16b\n"
+ "prfm pstl1keep, [x13, #0x0]\n"
+ "tbz %x[flags], #1, 27f\n"
+ "add x19, %x[args_ptr], %[offset_min]\n"
+ "ld1r { v1.4s }, [x19]\n"
+ "add x19, %x[args_ptr], %[offset_max]\n"
+ "ld1r { v0.4s }, [x19]\n"
+ "fmin v8.4s, v8.4s, v0.4s\n"
+ "fmin v9.4s, v9.4s, v0.4s\n"
+ "fmin v10.4s, v10.4s, v0.4s\n"
+ "fmin v11.4s, v11.4s, v0.4s\n"
+ "fmax v8.4s, v8.4s, v1.4s\n"
+ "fmax v9.4s, v9.4s, v1.4s\n"
+ "fmax v10.4s, v10.4s, v1.4s\n"
+ "fmax v11.4s, v11.4s, v1.4s\n"
+ "27:" // Height 1: No activation
+ "cmp x16, #0x10\n"
+ "bge 36f\n"
+ "tbz x16, #3, 31f\n"
+ "st1 { v8.4s }, [x13], #0x10\n"
+ "st1 { v9.4s }, [x13], #0x10\n"
+ "tbz x16, #2, 29f\n"
+ "st1 { v10.4s }, [x13], #0x10\n"
+ "tbz x16, #1, 28f\n"
+ "str d11, [x13], #0x8\n"
+ "tbz x16, #0, 35f\n"
+ "st1 { v11.s }[2], [x13]\n"
+ "b 35f\n"
+ "28:" // Height 1: Partial direct writeback: partial_1_12
+ "tbz x16, #0, 35f\n"
+ "str s11, [x13, #0x0]\n"
+ "b 35f\n"
+ "29:" // Height 1: Partial direct writeback: partial_2_8
+ "tbz x16, #1, 30f\n"
+ "str d10, [x13], #0x8\n"
+ "tbz x16, #0, 35f\n"
+ "st1 { v10.s }[2], [x13]\n"
+ "b 35f\n"
+ "30:" // Height 1: Partial direct writeback: partial_1_8
+ "tbz x16, #0, 35f\n"
+ "str s10, [x13, #0x0]\n"
+ "b 35f\n"
+ "31:" // Height 1: Partial direct writeback: partial_4_0
+ "tbz x16, #2, 33f\n"
+ "st1 { v8.4s }, [x13], #0x10\n"
+ "tbz x16, #1, 32f\n"
+ "str d9, [x13], #0x8\n"
+ "tbz x16, #0, 35f\n"
+ "st1 { v9.s }[2], [x13]\n"
+ "b 35f\n"
+ "32:" // Height 1: Partial direct writeback: partial_1_4
+ "tbz x16, #0, 35f\n"
+ "str s9, [x13, #0x0]\n"
+ "b 35f\n"
+ "33:" // Height 1: Partial direct writeback: partial_2_0
+ "tbz x16, #1, 34f\n"
+ "str d8, [x13], #0x8\n"
+ "tbz x16, #0, 35f\n"
+ "st1 { v8.s }[2], [x13]\n"
+ "b 35f\n"
+ "34:" // Height 1: Partial direct writeback: partial_1_0
+ "str s8, [x13, #0x0]\n"
+ "35:" // Height 1: Partial direct writeback: Done
+ "b 37f\n"
+ "36:" // Height 1: Full writeback
+ "str q8, [x13, #0x0]\n"
+ "str q9, [x13, #0x10]\n"
+ "str q10, [x13, #0x20]\n"
+ "str q11, [x13, #0x30]\n"
+ "add x13, x13, #0x40\n"
+ "37:" // Height 1: Writeback done
+ "subs x16, x16, #0x10\n"
+ "bgt 3b\n"
+ "b 224f\n"
+ "38:" // Height 2
+ "ldr x16, [%x[args_ptr], %[offsetof_N]]\n"
+ "mov x14, %x[bias]\n"
+ "ldr x15, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 39f\n"
+ "ldr x13, [%x[output_ptr], #0x0]\n"
+ "add x13, x13, x19, LSL #2\n"
+ "ldr x9, [%x[output_ptr], #0x8]\n"
+ "add x9, x9, x19, LSL #2\n"
+ "b 40f\n"
+ "39:" // Height 2: setup direct output
+ "mov x13, %x[output_ptr]\n"
+ "add x9, x13, x19, LSL #2\n"
+ "40:" // Height 2: Column loop
+ "cbz x14, 41f\n"
+ "ldr q8, [x14, #0x0]\n"
+ "mov v12.16b, v8.16b\n"
+ "ldr q9, [x14, #0x10]\n"
+ "ldr q10, [x14, #0x20]\n"
+ "mov v13.16b, v9.16b\n"
+ "ldr q11, [x14, #0x30]\n"
+ "mov v14.16b, v10.16b\n"
+ "add x14, x14, #0x40\n"
+ "mov v15.16b, v11.16b\n"
+ "b 52f\n"
+ "41:" // Height 2: no bias
+ "tbz %x[flags], #0, 51f\n"
+ "cmp x16, #0x10\n"
+ "bge 50f\n"
+ "tbz x16, #3, 45f\n"
+ "ld1 { v8.4s }, [x13], #0x10\n"
+ "ld1 { v12.4s }, [x9], #0x10\n"
+ "ld1 { v9.4s }, [x13], #0x10\n"
+ "ld1 { v13.4s }, [x9], #0x10\n"
+ "tbz x16, #2, 43f\n"
+ "ld1 { v10.4s }, [x13], #0x10\n"
+ "ld1 { v14.4s }, [x9], #0x10\n"
+ "tbz x16, #1, 42f\n"
+ "mov x19, #0x38\n"
+ "ldr d11, [x13], #0x8\n"
+ "ldr d15, [x9], #0x8\n"
+ "tbz x16, #0, 49f\n"
+ "ld1 { v11.s }[2], [x13]\n"
+ "ld1 { v15.s }[2], [x9]\n"
+ "b 49f\n"
+ "42:" // Height 2: Partial accumulate: partial_1_12
+ "mov x19, #0x30\n"
+ "tbz x16, #0, 49f\n"
+ "ldr s11, [x13, #0x0]\n"
+ "ldr s15, [x9, #0x0]\n"
+ "b 49f\n"
+ "43:" // Height 2: Partial accumulate: partial_2_8
+ "tbz x16, #1, 44f\n"
+ "ldr d10, [x13], #0x8\n"
+ "ldr d14, [x9], #0x8\n"
+ "mov x19, #0x28\n"
+ "tbz x16, #0, 49f\n"
+ "ld1 { v10.s }[2], [x13]\n"
+ "ld1 { v14.s }[2], [x9]\n"
+ "b 49f\n"
+ "44:" // Height 2: Partial accumulate: partial_1_8
+ "mov x19, #0x20\n"
+ "tbz x16, #0, 49f\n"
+ "ldr s10, [x13, #0x0]\n"
+ "ldr s14, [x9, #0x0]\n"
+ "b 49f\n"
+ "45:" // Height 2: Partial accumulate: partial_4_0
+ "tbz x16, #2, 47f\n"
+ "ld1 { v8.4s }, [x13], #0x10\n"
+ "ld1 { v12.4s }, [x9], #0x10\n"
+ "tbz x16, #1, 46f\n"
+ "mov x19, #0x18\n"
+ "ldr d9, [x13], #0x8\n"
+ "ldr d13, [x9], #0x8\n"
+ "tbz x16, #0, 49f\n"
+ "ld1 { v9.s }[2], [x13]\n"
+ "ld1 { v13.s }[2], [x9]\n"
+ "b 49f\n"
+ "46:" // Height 2: Partial accumulate: partial_1_4
+ "mov x19, #0x10\n"
+ "tbz x16, #0, 49f\n"
+ "ldr s9, [x13, #0x0]\n"
+ "ldr s13, [x9, #0x0]\n"
+ "b 49f\n"
+ "47:" // Height 2: Partial accumulate: partial_2_0
+ "tbz x16, #1, 48f\n"
+ "ldr d8, [x13], #0x8\n"
+ "ldr d12, [x9], #0x8\n"
+ "mov x19, #0x8\n"
+ "tbz x16, #0, 49f\n"
+ "ld1 { v8.s }[2], [x13]\n"
+ "ld1 { v12.s }[2], [x9]\n"
+ "b 49f\n"
+ "48:" // Height 2: Partial accumulate: partial_1_0
+ "mov x19, #0x0\n"
+ "ldr s8, [x13, #0x0]\n"
+ "ldr s12, [x9, #0x0]\n"
+ "49:" // Height 2: Partial accumulate: Done
+ "sub x13, x13, x19\n"
+ "sub x9, x9, x19\n"
+ "b 52f\n"
+ "50:" // Height 2: full accumulate
+ "ldr q8, [x13, #0x0]\n"
+ "ldr q9, [x13, #0x10]\n"
+ "ldr q10, [x13, #0x20]\n"
+ "ldr q11, [x13, #0x30]\n"
+ "ldr q12, [x9, #0x0]\n"
+ "ldr q13, [x9, #0x10]\n"
+ "ldr q14, [x9, #0x20]\n"
+ "ldr q15, [x9, #0x30]\n"
+ "b 52f\n"
+ "51:" // Height 2: no accumulate
+ "movi v8.16b, #0x0\n"
+ "movi v9.16b, #0x0\n"
+ "movi v10.16b, #0x0\n"
+ "movi v11.16b, #0x0\n"
+ "movi v12.16b, #0x0\n"
+ "movi v13.16b, #0x0\n"
+ "movi v14.16b, #0x0\n"
+ "movi v15.16b, #0x0\n"
+ "52:" // Height 2: setup done
+ "mov x12, #0x0\n"
+ "53:" // Height 2: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w11, [x20, x12, LSL #0x2]\n"
+ "tbz %x[flags], #3, 54f\n"
+ "ldr x20, [%x[input_ptr], x12, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x10, [x20, #0x0]\n"
+ "ldr x28, [x20, #0x8]\n"
+ "cbnz x12, 55f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x10, x10, x19, LSL #1\n"
+ "add x28, x28, x19, LSL #1\n"
+ "b 55f\n"
+ "54:" // Height 2: setup direct input
+ "mov x10, %x[input_ptr]\n"
+ "add x28, x10, x19, LSL #1\n"
+ "55:" // Height 2: input setup done
+ "cmp x11, #0x8\n"
+ "blt 58f\n"
+ "cmp x11, #0x10\n"
+ "blt 57f\n"
+ "56:" // Height 2: Multiply loop: Main loop head
+ "ldr q0, [x10, #0x0]\n"
+ "ldr q1, [x28, #0x0]\n"
+ "ldr q6, [x15, #0x0]\n"
+ ".inst 0x4f40f0c8 // bfdot v8.4s, v6.8h, v0.h[0]\n"
+ "ldr q7, [x15, #0x10]\n"
+ ".inst 0x4f41f0cc // bfdot v12.4s, v6.8h, v1.h[0]\n"
+ "ldr q6, [x15, #0x20]\n"
+ "add x10, x10, #0x10\n"
+ ".inst 0x4f40f0e9 // bfdot v9.4s, v7.8h, v0.h[0]\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x28, x28, #0x10\n"
+ ".inst 0x4f41f0ed // bfdot v13.4s, v7.8h, v1.h[0]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "ldr q7, [x15, #0x30]\n"
+ ".inst 0x4f40f0ca // bfdot v10.4s, v6.8h, v0.h[0]\n"
+ "sub x11, x11, #0x8\n"
+ ".inst 0x4f41f0ce // bfdot v14.4s, v6.8h, v1.h[0]\n"
+ "ldr q6, [x15, #0x40]\n"
+ "cmp x11, #0x10\n"
+ ".inst 0x4f40f0eb // bfdot v11.4s, v7.8h, v0.h[0]\n"
+ ".inst 0x4f41f0ef // bfdot v15.4s, v7.8h, v1.h[0]\n"
+ "ldr q7, [x15, #0x50]\n"
+ ".inst 0x4f60f0c8 // bfdot v8.4s, v6.8h, v0.h[1]\n"
+ ".inst 0x4f61f0cc // bfdot v12.4s, v6.8h, v1.h[1]\n"
+ "ldr q6, [x15, #0x60]\n"
+ ".inst 0x4f60f0e9 // bfdot v9.4s, v7.8h, v0.h[1]\n"
+ ".inst 0x4f61f0ed // bfdot v13.4s, v7.8h, v1.h[1]\n"
+ "ldr q7, [x15, #0x70]\n"
+ ".inst 0x4f60f0ca // bfdot v10.4s, v6.8h, v0.h[1]\n"
+ ".inst 0x4f61f0ce // bfdot v14.4s, v6.8h, v1.h[1]\n"
+ "ldr q6, [x15, #0x80]\n"
+ ".inst 0x4f60f0eb // bfdot v11.4s, v7.8h, v0.h[1]\n"
+ ".inst 0x4f61f0ef // bfdot v15.4s, v7.8h, v1.h[1]\n"
+ "ldr q7, [x15, #0x90]\n"
+ ".inst 0x4f40f8c8 // bfdot v8.4s, v6.8h, v0.h[2]\n"
+ ".inst 0x4f41f8cc // bfdot v12.4s, v6.8h, v1.h[2]\n"
+ "ldr q6, [x15, #0xa0]\n"
+ ".inst 0x4f40f8e9 // bfdot v9.4s, v7.8h, v0.h[2]\n"
+ ".inst 0x4f41f8ed // bfdot v13.4s, v7.8h, v1.h[2]\n"
+ "ldr q7, [x15, #0xb0]\n"
+ ".inst 0x4f40f8ca // bfdot v10.4s, v6.8h, v0.h[2]\n"
+ ".inst 0x4f41f8ce // bfdot v14.4s, v6.8h, v1.h[2]\n"
+ "ldr q6, [x15, #0xc0]\n"
+ ".inst 0x4f40f8eb // bfdot v11.4s, v7.8h, v0.h[2]\n"
+ ".inst 0x4f41f8ef // bfdot v15.4s, v7.8h, v1.h[2]\n"
+ "ldr q7, [x15, #0xd0]\n"
+ ".inst 0x4f60f8c8 // bfdot v8.4s, v6.8h, v0.h[3]\n"
+ ".inst 0x4f61f8cc // bfdot v12.4s, v6.8h, v1.h[3]\n"
+ "ldr q6, [x15, #0xe0]\n"
+ ".inst 0x4f60f8e9 // bfdot v9.4s, v7.8h, v0.h[3]\n"
+ ".inst 0x4f61f8ed // bfdot v13.4s, v7.8h, v1.h[3]\n"
+ "ldr q7, [x15, #0xf0]\n"
+ "add x15, x15, #0x100\n"
+ ".inst 0x4f60f8ca // bfdot v10.4s, v6.8h, v0.h[3]\n"
+ ".inst 0x4f61f8ce // bfdot v14.4s, v6.8h, v1.h[3]\n"
+ ".inst 0x4f60f8eb // bfdot v11.4s, v7.8h, v0.h[3]\n"
+ ".inst 0x4f61f8ef // bfdot v15.4s, v7.8h, v1.h[3]\n"
+ "bge 56b\n"
+ "57:" // Height 2: Multiply loop: Single iteration only
+ "sub x11, x11, #0x8\n"
+ "ldr q0, [x10, #0x0]\n"
+ "ldr q1, [x28, #0x0]\n"
+ "ldr q6, [x15, #0x0]\n"
+ ".inst 0x4f40f0c8 // bfdot v8.4s, v6.8h, v0.h[0]\n"
+ "ldr q7, [x15, #0x10]\n"
+ ".inst 0x4f41f0cc // bfdot v12.4s, v6.8h, v1.h[0]\n"
+ "ldr q6, [x15, #0x20]\n"
+ "add x10, x10, #0x10\n"
+ ".inst 0x4f40f0e9 // bfdot v9.4s, v7.8h, v0.h[0]\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x28, x28, #0x10\n"
+ ".inst 0x4f41f0ed // bfdot v13.4s, v7.8h, v1.h[0]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "ldr q7, [x15, #0x30]\n"
+ ".inst 0x4f40f0ca // bfdot v10.4s, v6.8h, v0.h[0]\n"
+ ".inst 0x4f41f0ce // bfdot v14.4s, v6.8h, v1.h[0]\n"
+ "ldr q6, [x15, #0x40]\n"
+ ".inst 0x4f40f0eb // bfdot v11.4s, v7.8h, v0.h[0]\n"
+ ".inst 0x4f41f0ef // bfdot v15.4s, v7.8h, v1.h[0]\n"
+ "ldr q7, [x15, #0x50]\n"
+ ".inst 0x4f60f0c8 // bfdot v8.4s, v6.8h, v0.h[1]\n"
+ ".inst 0x4f61f0cc // bfdot v12.4s, v6.8h, v1.h[1]\n"
+ "ldr q6, [x15, #0x60]\n"
+ ".inst 0x4f60f0e9 // bfdot v9.4s, v7.8h, v0.h[1]\n"
+ ".inst 0x4f61f0ed // bfdot v13.4s, v7.8h, v1.h[1]\n"
+ "ldr q7, [x15, #0x70]\n"
+ ".inst 0x4f60f0ca // bfdot v10.4s, v6.8h, v0.h[1]\n"
+ ".inst 0x4f61f0ce // bfdot v14.4s, v6.8h, v1.h[1]\n"
+ "ldr q6, [x15, #0x80]\n"
+ ".inst 0x4f60f0eb // bfdot v11.4s, v7.8h, v0.h[1]\n"
+ ".inst 0x4f61f0ef // bfdot v15.4s, v7.8h, v1.h[1]\n"
+ "ldr q7, [x15, #0x90]\n"
+ ".inst 0x4f40f8c8 // bfdot v8.4s, v6.8h, v0.h[2]\n"
+ ".inst 0x4f41f8cc // bfdot v12.4s, v6.8h, v1.h[2]\n"
+ "ldr q6, [x15, #0xa0]\n"
+ ".inst 0x4f40f8e9 // bfdot v9.4s, v7.8h, v0.h[2]\n"
+ ".inst 0x4f41f8ed // bfdot v13.4s, v7.8h, v1.h[2]\n"
+ "ldr q7, [x15, #0xb0]\n"
+ ".inst 0x4f40f8ca // bfdot v10.4s, v6.8h, v0.h[2]\n"
+ ".inst 0x4f41f8ce // bfdot v14.4s, v6.8h, v1.h[2]\n"
+ "ldr q6, [x15, #0xc0]\n"
+ ".inst 0x4f40f8eb // bfdot v11.4s, v7.8h, v0.h[2]\n"
+ ".inst 0x4f41f8ef // bfdot v15.4s, v7.8h, v1.h[2]\n"
+ "ldr q7, [x15, #0xd0]\n"
+ ".inst 0x4f60f8c8 // bfdot v8.4s, v6.8h, v0.h[3]\n"
+ ".inst 0x4f61f8cc // bfdot v12.4s, v6.8h, v1.h[3]\n"
+ "ldr q6, [x15, #0xe0]\n"
+ ".inst 0x4f60f8e9 // bfdot v9.4s, v7.8h, v0.h[3]\n"
+ ".inst 0x4f61f8ed // bfdot v13.4s, v7.8h, v1.h[3]\n"
+ "ldr q7, [x15, #0xf0]\n"
+ "add x15, x15, #0x100\n"
+ ".inst 0x4f60f8ca // bfdot v10.4s, v6.8h, v0.h[3]\n"
+ ".inst 0x4f61f8ce // bfdot v14.4s, v6.8h, v1.h[3]\n"
+ ".inst 0x4f60f8eb // bfdot v11.4s, v7.8h, v0.h[3]\n"
+ ".inst 0x4f61f8ef // bfdot v15.4s, v7.8h, v1.h[3]\n"
+ "58:" // Height 2: Multiply loop: Main loop skip
+ "cbz x11, 63f\n"
+ "cmp x11, #0x2\n"
+ "blt 60f\n"
+ "59:" // Height 2: Multiply loop: Odd block loop
+ "ldr s0, [x10], #0x4\n"
+ "ldr s1, [x28], #0x4\n"
+ "ldr q6, [x15, #0x0]\n"
+ ".inst 0x4f40f0c8 // bfdot v8.4s, v6.8h, v0.h[0]\n"
+ "ldr q7, [x15, #0x10]\n"
+ ".inst 0x4f41f0cc // bfdot v12.4s, v6.8h, v1.h[0]\n"
+ "ldr q6, [x15, #0x20]\n"
+ "sub x11, x11, #0x2\n"
+ ".inst 0x4f40f0e9 // bfdot v9.4s, v7.8h, v0.h[0]\n"
+ "cmp x11, #0x2\n"
+ ".inst 0x4f41f0ed // bfdot v13.4s, v7.8h, v1.h[0]\n"
+ "ldr q7, [x15, #0x30]\n"
+ ".inst 0x4f40f0ca // bfdot v10.4s, v6.8h, v0.h[0]\n"
+ "add x15, x15, #0x40\n"
+ ".inst 0x4f41f0ce // bfdot v14.4s, v6.8h, v1.h[0]\n"
+ ".inst 0x4f40f0eb // bfdot v11.4s, v7.8h, v0.h[0]\n"
+ ".inst 0x4f41f0ef // bfdot v15.4s, v7.8h, v1.h[0]\n"
+ "bge 59b\n"
+ "cbz x11, 63f\n"
+ "60:" // Height 2: Multiply loop: Skip odd blocks
+ "tbz x11, #1, 61f\n"
+ "ldr s0, [x10], #0x4\n"
+ "ldr s1, [x28], #0x4\n"
+ "tbz x11, #0, 62f\n"
+ "ld1 { v0.h }[2], [x10]\n"
+ "ld1 { v1.h }[2], [x28]\n"
+ "b 62f\n"
+ "61:" // Height 2: Multiply loop: Ragged operand read: partial_1_0
+ "ldr h0, [x10, #0x0]\n"
+ "ldr h1, [x28, #0x0]\n"
+ "62:" // Height 2: Multiply loop: Ragged operand read: Done
+ "ldr q6, [x15, #0x0]\n"
+ ".inst 0x4f40f0c8 // bfdot v8.4s, v6.8h, v0.h[0]\n"
+ "ldr q7, [x15, #0x10]\n"
+ ".inst 0x4f41f0cc // bfdot v12.4s, v6.8h, v1.h[0]\n"
+ "ldr q6, [x15, #0x20]\n"
+ ".inst 0x4f40f0e9 // bfdot v9.4s, v7.8h, v0.h[0]\n"
+ ".inst 0x4f41f0ed // bfdot v13.4s, v7.8h, v1.h[0]\n"
+ "ldr q7, [x15, #0x30]\n"
+ ".inst 0x4f40f0ca // bfdot v10.4s, v6.8h, v0.h[0]\n"
+ "add x15, x15, #0x40\n"
+ ".inst 0x4f41f0ce // bfdot v14.4s, v6.8h, v1.h[0]\n"
+ ".inst 0x4f40f0eb // bfdot v11.4s, v7.8h, v0.h[0]\n"
+ ".inst 0x4f41f0ef // bfdot v15.4s, v7.8h, v1.h[0]\n"
+ "63:" // Height 2: Multiply loop: No odd multiplies
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "add x12, x12, #0x1\n"
+ "cmp x12, x19\n"
+ "bne 53b\n"
+ "prfm pstl1keep, [x13, #0x0]\n"
+ "prfm pstl1keep, [x9, #0x0]\n"
+ "tbz %x[flags], #1, 64f\n"
+ "add x19, %x[args_ptr], %[offset_min]\n"
+ "ld1r { v1.4s }, [x19]\n"
+ "add x19, %x[args_ptr], %[offset_max]\n"
+ "ld1r { v0.4s }, [x19]\n"
+ "fmin v8.4s, v8.4s, v0.4s\n"
+ "fmin v9.4s, v9.4s, v0.4s\n"
+ "fmin v10.4s, v10.4s, v0.4s\n"
+ "fmin v11.4s, v11.4s, v0.4s\n"
+ "fmax v8.4s, v8.4s, v1.4s\n"
+ "fmax v9.4s, v9.4s, v1.4s\n"
+ "fmax v10.4s, v10.4s, v1.4s\n"
+ "fmax v11.4s, v11.4s, v1.4s\n"
+ "fmin v12.4s, v12.4s, v0.4s\n"
+ "fmin v13.4s, v13.4s, v0.4s\n"
+ "fmin v14.4s, v14.4s, v0.4s\n"
+ "fmax v12.4s, v12.4s, v1.4s\n"
+ "fmax v13.4s, v13.4s, v1.4s\n"
+ "fmax v14.4s, v14.4s, v1.4s\n"
+ "fmin v15.4s, v15.4s, v0.4s\n"
+ "fmax v15.4s, v15.4s, v1.4s\n"
+ "64:" // Height 2: No activation
+ "cmp x16, #0x10\n"
+ "bge 73f\n"
+ "tbz x16, #3, 68f\n"
+ "st1 { v8.4s }, [x13], #0x10\n"
+ "st1 { v9.4s }, [x13], #0x10\n"
+ "st1 { v12.4s }, [x9], #0x10\n"
+ "st1 { v13.4s }, [x9], #0x10\n"
+ "tbz x16, #2, 66f\n"
+ "st1 { v10.4s }, [x13], #0x10\n"
+ "st1 { v14.4s }, [x9], #0x10\n"
+ "tbz x16, #1, 65f\n"
+ "str d11, [x13], #0x8\n"
+ "str d15, [x9], #0x8\n"
+ "tbz x16, #0, 72f\n"
+ "st1 { v11.s }[2], [x13]\n"
+ "st1 { v15.s }[2], [x9]\n"
+ "b 72f\n"
+ "65:" // Height 2: Partial direct writeback: partial_1_12
+ "tbz x16, #0, 72f\n"
+ "str s11, [x13, #0x0]\n"
+ "str s15, [x9, #0x0]\n"
+ "b 72f\n"
+ "66:" // Height 2: Partial direct writeback: partial_2_8
+ "tbz x16, #1, 67f\n"
+ "str d10, [x13], #0x8\n"
+ "str d14, [x9], #0x8\n"
+ "tbz x16, #0, 72f\n"
+ "st1 { v10.s }[2], [x13]\n"
+ "st1 { v14.s }[2], [x9]\n"
+ "b 72f\n"
+ "67:" // Height 2: Partial direct writeback: partial_1_8
+ "tbz x16, #0, 72f\n"
+ "str s10, [x13, #0x0]\n"
+ "str s14, [x9, #0x0]\n"
+ "b 72f\n"
+ "68:" // Height 2: Partial direct writeback: partial_4_0
+ "tbz x16, #2, 70f\n"
+ "st1 { v8.4s }, [x13], #0x10\n"
+ "st1 { v12.4s }, [x9], #0x10\n"
+ "tbz x16, #1, 69f\n"
+ "str d9, [x13], #0x8\n"
+ "str d13, [x9], #0x8\n"
+ "tbz x16, #0, 72f\n"
+ "st1 { v9.s }[2], [x13]\n"
+ "st1 { v13.s }[2], [x9]\n"
+ "b 72f\n"
+ "69:" // Height 2: Partial direct writeback: partial_1_4
+ "tbz x16, #0, 72f\n"
+ "str s9, [x13, #0x0]\n"
+ "str s13, [x9, #0x0]\n"
+ "b 72f\n"
+ "70:" // Height 2: Partial direct writeback: partial_2_0
+ "tbz x16, #1, 71f\n"
+ "str d8, [x13], #0x8\n"
+ "str d12, [x9], #0x8\n"
+ "tbz x16, #0, 72f\n"
+ "st1 { v8.s }[2], [x13]\n"
+ "st1 { v12.s }[2], [x9]\n"
+ "b 72f\n"
+ "71:" // Height 2: Partial direct writeback: partial_1_0
+ "str s8, [x13, #0x0]\n"
+ "str s12, [x9, #0x0]\n"
+ "72:" // Height 2: Partial direct writeback: Done
+ "b 74f\n"
+ "73:" // Height 2: Full writeback
+ "str q8, [x13, #0x0]\n"
+ "str q9, [x13, #0x10]\n"
+ "str q10, [x13, #0x20]\n"
+ "str q11, [x13, #0x30]\n"
+ "str q12, [x9, #0x0]\n"
+ "str q13, [x9, #0x10]\n"
+ "str q14, [x9, #0x20]\n"
+ "str q15, [x9, #0x30]\n"
+ "add x13, x13, #0x40\n"
+ "add x9, x9, #0x40\n"
+ "74:" // Height 2: Writeback done
+ "subs x16, x16, #0x10\n"
+ "bgt 40b\n"
+ "b 224f\n"
+ "75:" // Height 3
+ "ldr x16, [%x[args_ptr], %[offsetof_N]]\n"
+ "mov x14, %x[bias]\n"
+ "ldr x15, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 76f\n"
+ "ldr x13, [%x[output_ptr], #0x0]\n"
+ "add x13, x13, x19, LSL #2\n"
+ "ldr x9, [%x[output_ptr], #0x8]\n"
+ "ldr x27, [%x[output_ptr], #0x10]\n"
+ "add x9, x9, x19, LSL #2\n"
+ "add x27, x27, x19, LSL #2\n"
+ "b 77f\n"
+ "76:" // Height 3: setup direct output
+ "mov x13, %x[output_ptr]\n"
+ "add x9, x13, x19, LSL #2\n"
+ "add x27, x9, x19, LSL #2\n"
+ "77:" // Height 3: Column loop
+ "cbz x14, 78f\n"
+ "ldr q8, [x14, #0x0]\n"
+ "mov v12.16b, v8.16b\n"
+ "ldr q9, [x14, #0x10]\n"
+ "mov v16.16b, v8.16b\n"
+ "ldr q10, [x14, #0x20]\n"
+ "ldr q11, [x14, #0x30]\n"
+ "mov v13.16b, v9.16b\n"
+ "add x14, x14, #0x40\n"
+ "mov v17.16b, v9.16b\n"
+ "mov v14.16b, v10.16b\n"
+ "mov v15.16b, v11.16b\n"
+ "mov v18.16b, v10.16b\n"
+ "mov v19.16b, v11.16b\n"
+ "b 89f\n"
+ "78:" // Height 3: no bias
+ "tbz %x[flags], #0, 88f\n"
+ "cmp x16, #0x10\n"
+ "bge 87f\n"
+ "tbz x16, #3, 82f\n"
+ "ld1 { v8.4s }, [x13], #0x10\n"
+ "ld1 { v12.4s }, [x9], #0x10\n"
+ "ld1 { v16.4s }, [x27], #0x10\n"
+ "ld1 { v9.4s }, [x13], #0x10\n"
+ "ld1 { v13.4s }, [x9], #0x10\n"
+ "ld1 { v17.4s }, [x27], #0x10\n"
+ "tbz x16, #2, 80f\n"
+ "ld1 { v10.4s }, [x13], #0x10\n"
+ "ld1 { v14.4s }, [x9], #0x10\n"
+ "ld1 { v18.4s }, [x27], #0x10\n"
+ "tbz x16, #1, 79f\n"
+ "mov x19, #0x38\n"
+ "ldr d11, [x13], #0x8\n"
+ "ldr d15, [x9], #0x8\n"
+ "ldr d19, [x27], #0x8\n"
+ "tbz x16, #0, 86f\n"
+ "ld1 { v11.s }[2], [x13]\n"
+ "ld1 { v15.s }[2], [x9]\n"
+ "ld1 { v19.s }[2], [x27]\n"
+ "b 86f\n"
+ "79:" // Height 3: Partial accumulate: partial_1_12
+ "mov x19, #0x30\n"
+ "tbz x16, #0, 86f\n"
+ "ldr s11, [x13, #0x0]\n"
+ "ldr s15, [x9, #0x0]\n"
+ "ldr s19, [x27, #0x0]\n"
+ "b 86f\n"
+ "80:" // Height 3: Partial accumulate: partial_2_8
+ "tbz x16, #1, 81f\n"
+ "ldr d10, [x13], #0x8\n"
+ "ldr d14, [x9], #0x8\n"
+ "ldr d18, [x27], #0x8\n"
+ "mov x19, #0x28\n"
+ "tbz x16, #0, 86f\n"
+ "ld1 { v10.s }[2], [x13]\n"
+ "ld1 { v14.s }[2], [x9]\n"
+ "ld1 { v18.s }[2], [x27]\n"
+ "b 86f\n"
+ "81:" // Height 3: Partial accumulate: partial_1_8
+ "mov x19, #0x20\n"
+ "tbz x16, #0, 86f\n"
+ "ldr s10, [x13, #0x0]\n"
+ "ldr s14, [x9, #0x0]\n"
+ "ldr s18, [x27, #0x0]\n"
+ "b 86f\n"
+ "82:" // Height 3: Partial accumulate: partial_4_0
+ "tbz x16, #2, 84f\n"
+ "ld1 { v8.4s }, [x13], #0x10\n"
+ "ld1 { v12.4s }, [x9], #0x10\n"
+ "ld1 { v16.4s }, [x27], #0x10\n"
+ "tbz x16, #1, 83f\n"
+ "mov x19, #0x18\n"
+ "ldr d9, [x13], #0x8\n"
+ "ldr d13, [x9], #0x8\n"
+ "ldr d17, [x27], #0x8\n"
+ "tbz x16, #0, 86f\n"
+ "ld1 { v9.s }[2], [x13]\n"
+ "ld1 { v13.s }[2], [x9]\n"
+ "ld1 { v17.s }[2], [x27]\n"
+ "b 86f\n"
+ "83:" // Height 3: Partial accumulate: partial_1_4
+ "mov x19, #0x10\n"
+ "tbz x16, #0, 86f\n"
+ "ldr s9, [x13, #0x0]\n"
+ "ldr s13, [x9, #0x0]\n"
+ "ldr s17, [x27, #0x0]\n"
+ "b 86f\n"
+ "84:" // Height 3: Partial accumulate: partial_2_0
+ "tbz x16, #1, 85f\n"
+ "ldr d8, [x13], #0x8\n"
+ "ldr d12, [x9], #0x8\n"
+ "ldr d16, [x27], #0x8\n"
+ "mov x19, #0x8\n"
+ "tbz x16, #0, 86f\n"
+ "ld1 { v8.s }[2], [x13]\n"
+ "ld1 { v12.s }[2], [x9]\n"
+ "ld1 { v16.s }[2], [x27]\n"
+ "b 86f\n"
+ "85:" // Height 3: Partial accumulate: partial_1_0
+ "mov x19, #0x0\n"
+ "ldr s8, [x13, #0x0]\n"
+ "ldr s12, [x9, #0x0]\n"
+ "ldr s16, [x27, #0x0]\n"
+ "86:" // Height 3: Partial accumulate: Done
+ "sub x13, x13, x19\n"
+ "sub x9, x9, x19\n"
+ "sub x27, x27, x19\n"
+ "b 89f\n"
+ "87:" // Height 3: full accumulate
+ "ldr q8, [x13, #0x0]\n"
+ "ldr q9, [x13, #0x10]\n"
+ "ldr q10, [x13, #0x20]\n"
+ "ldr q11, [x13, #0x30]\n"
+ "ldr q12, [x9, #0x0]\n"
+ "ldr q13, [x9, #0x10]\n"
+ "ldr q14, [x9, #0x20]\n"
+ "ldr q15, [x9, #0x30]\n"
+ "ldr q16, [x27, #0x0]\n"
+ "ldr q17, [x27, #0x10]\n"
+ "ldr q18, [x27, #0x20]\n"
+ "ldr q19, [x27, #0x30]\n"
+ "b 89f\n"
+ "88:" // Height 3: no accumulate
+ "movi v8.16b, #0x0\n"
+ "movi v9.16b, #0x0\n"
+ "movi v10.16b, #0x0\n"
+ "movi v11.16b, #0x0\n"
+ "movi v12.16b, #0x0\n"
+ "movi v13.16b, #0x0\n"
+ "movi v14.16b, #0x0\n"
+ "movi v15.16b, #0x0\n"
+ "movi v16.16b, #0x0\n"
+ "movi v17.16b, #0x0\n"
+ "movi v18.16b, #0x0\n"
+ "movi v19.16b, #0x0\n"
+ "89:" // Height 3: setup done
+ "mov x12, #0x0\n"
+ "90:" // Height 3: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w11, [x20, x12, LSL #0x2]\n"
+ "tbz %x[flags], #3, 91f\n"
+ "ldr x20, [%x[input_ptr], x12, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x10, [x20, #0x0]\n"
+ "ldr x28, [x20, #0x8]\n"
+ "ldr x26, [x20, #0x10]\n"
+ "cbnz x12, 92f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x10, x10, x19, LSL #1\n"
+ "add x28, x28, x19, LSL #1\n"
+ "add x26, x26, x19, LSL #1\n"
+ "b 92f\n"
+ "91:" // Height 3: setup direct input
+ "mov x10, %x[input_ptr]\n"
+ "add x28, x10, x19, LSL #1\n"
+ "add x26, x28, x19, LSL #1\n"
+ "92:" // Height 3: input setup done
+ "cmp x11, #0x8\n"
+ "blt 95f\n"
+ "cmp x11, #0x10\n"
+ "blt 94f\n"
+ "93:" // Height 3: Multiply loop: Main loop head
+ "ldr q0, [x10, #0x0]\n"
+ "ldr q1, [x28, #0x0]\n"
+ "ldr q2, [x26, #0x0]\n"
+ "ldr q6, [x15, #0x0]\n"
+ ".inst 0x4f40f0c8 // bfdot v8.4s, v6.8h, v0.h[0]\n"
+ "ldr q7, [x15, #0x10]\n"
+ ".inst 0x4f41f0cc // bfdot v12.4s, v6.8h, v1.h[0]\n"
+ "add x10, x10, #0x10\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ ".inst 0x4f42f0d0 // bfdot v16.4s, v6.8h, v2.h[0]\n"
+ "ldr q6, [x15, #0x20]\n"
+ ".inst 0x4f40f0e9 // bfdot v9.4s, v7.8h, v0.h[0]\n"
+ "add x28, x28, #0x10\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ ".inst 0x4f41f0ed // bfdot v13.4s, v7.8h, v1.h[0]\n"
+ "add x26, x26, #0x10\n"
+ ".inst 0x4f42f0f1 // bfdot v17.4s, v7.8h, v2.h[0]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "ldr q7, [x15, #0x30]\n"
+ ".inst 0x4f40f0ca // bfdot v10.4s, v6.8h, v0.h[0]\n"
+ "sub x11, x11, #0x8\n"
+ ".inst 0x4f41f0ce // bfdot v14.4s, v6.8h, v1.h[0]\n"
+ "cmp x11, #0x10\n"
+ ".inst 0x4f42f0d2 // bfdot v18.4s, v6.8h, v2.h[0]\n"
+ "ldr q6, [x15, #0x40]\n"
+ ".inst 0x4f40f0eb // bfdot v11.4s, v7.8h, v0.h[0]\n"
+ ".inst 0x4f41f0ef // bfdot v15.4s, v7.8h, v1.h[0]\n"
+ ".inst 0x4f42f0f3 // bfdot v19.4s, v7.8h, v2.h[0]\n"
+ "ldr q7, [x15, #0x50]\n"
+ ".inst 0x4f60f0c8 // bfdot v8.4s, v6.8h, v0.h[1]\n"
+ ".inst 0x4f61f0cc // bfdot v12.4s, v6.8h, v1.h[1]\n"
+ ".inst 0x4f62f0d0 // bfdot v16.4s, v6.8h, v2.h[1]\n"
+ "ldr q6, [x15, #0x60]\n"
+ ".inst 0x4f60f0e9 // bfdot v9.4s, v7.8h, v0.h[1]\n"
+ ".inst 0x4f61f0ed // bfdot v13.4s, v7.8h, v1.h[1]\n"
+ ".inst 0x4f62f0f1 // bfdot v17.4s, v7.8h, v2.h[1]\n"
+ "ldr q7, [x15, #0x70]\n"
+ ".inst 0x4f60f0ca // bfdot v10.4s, v6.8h, v0.h[1]\n"
+ ".inst 0x4f61f0ce // bfdot v14.4s, v6.8h, v1.h[1]\n"
+ ".inst 0x4f62f0d2 // bfdot v18.4s, v6.8h, v2.h[1]\n"
+ "ldr q6, [x15, #0x80]\n"
+ ".inst 0x4f60f0eb // bfdot v11.4s, v7.8h, v0.h[1]\n"
+ ".inst 0x4f61f0ef // bfdot v15.4s, v7.8h, v1.h[1]\n"
+ ".inst 0x4f62f0f3 // bfdot v19.4s, v7.8h, v2.h[1]\n"
+ "ldr q7, [x15, #0x90]\n"
+ ".inst 0x4f40f8c8 // bfdot v8.4s, v6.8h, v0.h[2]\n"
+ ".inst 0x4f41f8cc // bfdot v12.4s, v6.8h, v1.h[2]\n"
+ ".inst 0x4f42f8d0 // bfdot v16.4s, v6.8h, v2.h[2]\n"
+ "ldr q6, [x15, #0xa0]\n"
+ ".inst 0x4f40f8e9 // bfdot v9.4s, v7.8h, v0.h[2]\n"
+ ".inst 0x4f41f8ed // bfdot v13.4s, v7.8h, v1.h[2]\n"
+ ".inst 0x4f42f8f1 // bfdot v17.4s, v7.8h, v2.h[2]\n"
+ "ldr q7, [x15, #0xb0]\n"
+ ".inst 0x4f40f8ca // bfdot v10.4s, v6.8h, v0.h[2]\n"
+ ".inst 0x4f41f8ce // bfdot v14.4s, v6.8h, v1.h[2]\n"
+ ".inst 0x4f42f8d2 // bfdot v18.4s, v6.8h, v2.h[2]\n"
+ "ldr q6, [x15, #0xc0]\n"
+ ".inst 0x4f40f8eb // bfdot v11.4s, v7.8h, v0.h[2]\n"
+ ".inst 0x4f41f8ef // bfdot v15.4s, v7.8h, v1.h[2]\n"
+ ".inst 0x4f42f8f3 // bfdot v19.4s, v7.8h, v2.h[2]\n"
+ "ldr q7, [x15, #0xd0]\n"
+ ".inst 0x4f60f8c8 // bfdot v8.4s, v6.8h, v0.h[3]\n"
+ ".inst 0x4f61f8cc // bfdot v12.4s, v6.8h, v1.h[3]\n"
+ ".inst 0x4f62f8d0 // bfdot v16.4s, v6.8h, v2.h[3]\n"
+ "ldr q6, [x15, #0xe0]\n"
+ ".inst 0x4f60f8e9 // bfdot v9.4s, v7.8h, v0.h[3]\n"
+ ".inst 0x4f61f8ed // bfdot v13.4s, v7.8h, v1.h[3]\n"
+ ".inst 0x4f62f8f1 // bfdot v17.4s, v7.8h, v2.h[3]\n"
+ "ldr q7, [x15, #0xf0]\n"
+ "add x15, x15, #0x100\n"
+ ".inst 0x4f60f8ca // bfdot v10.4s, v6.8h, v0.h[3]\n"
+ ".inst 0x4f61f8ce // bfdot v14.4s, v6.8h, v1.h[3]\n"
+ ".inst 0x4f62f8d2 // bfdot v18.4s, v6.8h, v2.h[3]\n"
+ ".inst 0x4f60f8eb // bfdot v11.4s, v7.8h, v0.h[3]\n"
+ ".inst 0x4f61f8ef // bfdot v15.4s, v7.8h, v1.h[3]\n"
+ ".inst 0x4f62f8f3 // bfdot v19.4s, v7.8h, v2.h[3]\n"
+ "bge 93b\n"
+ "94:" // Height 3: Multiply loop: Single iteration only
+ "sub x11, x11, #0x8\n"
+ "ldr q0, [x10, #0x0]\n"
+ "ldr q1, [x28, #0x0]\n"
+ "ldr q2, [x26, #0x0]\n"
+ "ldr q6, [x15, #0x0]\n"
+ ".inst 0x4f40f0c8 // bfdot v8.4s, v6.8h, v0.h[0]\n"
+ "ldr q7, [x15, #0x10]\n"
+ ".inst 0x4f41f0cc // bfdot v12.4s, v6.8h, v1.h[0]\n"
+ "add x10, x10, #0x10\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ ".inst 0x4f42f0d0 // bfdot v16.4s, v6.8h, v2.h[0]\n"
+ "ldr q6, [x15, #0x20]\n"
+ ".inst 0x4f40f0e9 // bfdot v9.4s, v7.8h, v0.h[0]\n"
+ "add x28, x28, #0x10\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ ".inst 0x4f41f0ed // bfdot v13.4s, v7.8h, v1.h[0]\n"
+ "add x26, x26, #0x10\n"
+ ".inst 0x4f42f0f1 // bfdot v17.4s, v7.8h, v2.h[0]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "ldr q7, [x15, #0x30]\n"
+ ".inst 0x4f40f0ca // bfdot v10.4s, v6.8h, v0.h[0]\n"
+ ".inst 0x4f41f0ce // bfdot v14.4s, v6.8h, v1.h[0]\n"
+ ".inst 0x4f42f0d2 // bfdot v18.4s, v6.8h, v2.h[0]\n"
+ "ldr q6, [x15, #0x40]\n"
+ ".inst 0x4f40f0eb // bfdot v11.4s, v7.8h, v0.h[0]\n"
+ ".inst 0x4f41f0ef // bfdot v15.4s, v7.8h, v1.h[0]\n"
+ ".inst 0x4f42f0f3 // bfdot v19.4s, v7.8h, v2.h[0]\n"
+ "ldr q7, [x15, #0x50]\n"
+ ".inst 0x4f60f0c8 // bfdot v8.4s, v6.8h, v0.h[1]\n"
+ ".inst 0x4f61f0cc // bfdot v12.4s, v6.8h, v1.h[1]\n"
+ ".inst 0x4f62f0d0 // bfdot v16.4s, v6.8h, v2.h[1]\n"
+ "ldr q6, [x15, #0x60]\n"
+ ".inst 0x4f60f0e9 // bfdot v9.4s, v7.8h, v0.h[1]\n"
+ ".inst 0x4f61f0ed // bfdot v13.4s, v7.8h, v1.h[1]\n"
+ ".inst 0x4f62f0f1 // bfdot v17.4s, v7.8h, v2.h[1]\n"
+ "ldr q7, [x15, #0x70]\n"
+ ".inst 0x4f60f0ca // bfdot v10.4s, v6.8h, v0.h[1]\n"
+ ".inst 0x4f61f0ce // bfdot v14.4s, v6.8h, v1.h[1]\n"
+ ".inst 0x4f62f0d2 // bfdot v18.4s, v6.8h, v2.h[1]\n"
+ "ldr q6, [x15, #0x80]\n"
+ ".inst 0x4f60f0eb // bfdot v11.4s, v7.8h, v0.h[1]\n"
+ ".inst 0x4f61f0ef // bfdot v15.4s, v7.8h, v1.h[1]\n"
+ ".inst 0x4f62f0f3 // bfdot v19.4s, v7.8h, v2.h[1]\n"
+ "ldr q7, [x15, #0x90]\n"
+ ".inst 0x4f40f8c8 // bfdot v8.4s, v6.8h, v0.h[2]\n"
+ ".inst 0x4f41f8cc // bfdot v12.4s, v6.8h, v1.h[2]\n"
+ ".inst 0x4f42f8d0 // bfdot v16.4s, v6.8h, v2.h[2]\n"
+ "ldr q6, [x15, #0xa0]\n"
+ ".inst 0x4f40f8e9 // bfdot v9.4s, v7.8h, v0.h[2]\n"
+ ".inst 0x4f41f8ed // bfdot v13.4s, v7.8h, v1.h[2]\n"
+ ".inst 0x4f42f8f1 // bfdot v17.4s, v7.8h, v2.h[2]\n"
+ "ldr q7, [x15, #0xb0]\n"
+ ".inst 0x4f40f8ca // bfdot v10.4s, v6.8h, v0.h[2]\n"
+ ".inst 0x4f41f8ce // bfdot v14.4s, v6.8h, v1.h[2]\n"
+ ".inst 0x4f42f8d2 // bfdot v18.4s, v6.8h, v2.h[2]\n"
+ "ldr q6, [x15, #0xc0]\n"
+ ".inst 0x4f40f8eb // bfdot v11.4s, v7.8h, v0.h[2]\n"
+ ".inst 0x4f41f8ef // bfdot v15.4s, v7.8h, v1.h[2]\n"
+ ".inst 0x4f42f8f3 // bfdot v19.4s, v7.8h, v2.h[2]\n"
+ "ldr q7, [x15, #0xd0]\n"
+ ".inst 0x4f60f8c8 // bfdot v8.4s, v6.8h, v0.h[3]\n"
+ ".inst 0x4f61f8cc // bfdot v12.4s, v6.8h, v1.h[3]\n"
+ ".inst 0x4f62f8d0 // bfdot v16.4s, v6.8h, v2.h[3]\n"
+ "ldr q6, [x15, #0xe0]\n"
+ ".inst 0x4f60f8e9 // bfdot v9.4s, v7.8h, v0.h[3]\n"
+ ".inst 0x4f61f8ed // bfdot v13.4s, v7.8h, v1.h[3]\n"
+ ".inst 0x4f62f8f1 // bfdot v17.4s, v7.8h, v2.h[3]\n"
+ "ldr q7, [x15, #0xf0]\n"
+ "add x15, x15, #0x100\n"
+ ".inst 0x4f60f8ca // bfdot v10.4s, v6.8h, v0.h[3]\n"
+ ".inst 0x4f61f8ce // bfdot v14.4s, v6.8h, v1.h[3]\n"
+ ".inst 0x4f62f8d2 // bfdot v18.4s, v6.8h, v2.h[3]\n"
+ ".inst 0x4f60f8eb // bfdot v11.4s, v7.8h, v0.h[3]\n"
+ ".inst 0x4f61f8ef // bfdot v15.4s, v7.8h, v1.h[3]\n"
+ ".inst 0x4f62f8f3 // bfdot v19.4s, v7.8h, v2.h[3]\n"
+ "95:" // Height 3: Multiply loop: Main loop skip
+ "cbz x11, 100f\n"
+ "cmp x11, #0x2\n"
+ "blt 97f\n"
+ "96:" // Height 3: Multiply loop: Odd block loop
+ "ldr s0, [x10], #0x4\n"
+ "ldr s1, [x28], #0x4\n"
+ "ldr s2, [x26], #0x4\n"
+ "ldr q6, [x15, #0x0]\n"
+ ".inst 0x4f40f0c8 // bfdot v8.4s, v6.8h, v0.h[0]\n"
+ "ldr q7, [x15, #0x10]\n"
+ ".inst 0x4f41f0cc // bfdot v12.4s, v6.8h, v1.h[0]\n"
+ "sub x11, x11, #0x2\n"
+ ".inst 0x4f42f0d0 // bfdot v16.4s, v6.8h, v2.h[0]\n"
+ "ldr q6, [x15, #0x20]\n"
+ "cmp x11, #0x2\n"
+ ".inst 0x4f40f0e9 // bfdot v9.4s, v7.8h, v0.h[0]\n"
+ ".inst 0x4f41f0ed // bfdot v13.4s, v7.8h, v1.h[0]\n"
+ ".inst 0x4f42f0f1 // bfdot v17.4s, v7.8h, v2.h[0]\n"
+ "ldr q7, [x15, #0x30]\n"
+ ".inst 0x4f40f0ca // bfdot v10.4s, v6.8h, v0.h[0]\n"
+ "add x15, x15, #0x40\n"
+ ".inst 0x4f41f0ce // bfdot v14.4s, v6.8h, v1.h[0]\n"
+ ".inst 0x4f42f0d2 // bfdot v18.4s, v6.8h, v2.h[0]\n"
+ ".inst 0x4f40f0eb // bfdot v11.4s, v7.8h, v0.h[0]\n"
+ ".inst 0x4f41f0ef // bfdot v15.4s, v7.8h, v1.h[0]\n"
+ ".inst 0x4f42f0f3 // bfdot v19.4s, v7.8h, v2.h[0]\n"
+ "bge 96b\n"
+ "cbz x11, 100f\n"
+ "97:" // Height 3: Multiply loop: Skip odd blocks
+ "tbz x11, #1, 98f\n"
+ "ldr s0, [x10], #0x4\n"
+ "ldr s1, [x28], #0x4\n"
+ "ldr s2, [x26], #0x4\n"
+ "tbz x11, #0, 99f\n"
+ "ld1 { v0.h }[2], [x10]\n"
+ "ld1 { v1.h }[2], [x28]\n"
+ "ld1 { v2.h }[2], [x26]\n"
+ "b 99f\n"
+ "98:" // Height 3: Multiply loop: Ragged operand read: partial_1_0
+ "ldr h0, [x10, #0x0]\n"
+ "ldr h1, [x28, #0x0]\n"
+ "ldr h2, [x26, #0x0]\n"
+ "99:" // Height 3: Multiply loop: Ragged operand read: Done
+ "ldr q6, [x15, #0x0]\n"
+ ".inst 0x4f40f0c8 // bfdot v8.4s, v6.8h, v0.h[0]\n"
+ "ldr q7, [x15, #0x10]\n"
+ ".inst 0x4f41f0cc // bfdot v12.4s, v6.8h, v1.h[0]\n"
+ ".inst 0x4f42f0d0 // bfdot v16.4s, v6.8h, v2.h[0]\n"
+ "ldr q6, [x15, #0x20]\n"
+ ".inst 0x4f40f0e9 // bfdot v9.4s, v7.8h, v0.h[0]\n"
+ ".inst 0x4f41f0ed // bfdot v13.4s, v7.8h, v1.h[0]\n"
+ ".inst 0x4f42f0f1 // bfdot v17.4s, v7.8h, v2.h[0]\n"
+ "ldr q7, [x15, #0x30]\n"
+ ".inst 0x4f40f0ca // bfdot v10.4s, v6.8h, v0.h[0]\n"
+ "add x15, x15, #0x40\n"
+ ".inst 0x4f41f0ce // bfdot v14.4s, v6.8h, v1.h[0]\n"
+ ".inst 0x4f42f0d2 // bfdot v18.4s, v6.8h, v2.h[0]\n"
+ ".inst 0x4f40f0eb // bfdot v11.4s, v7.8h, v0.h[0]\n"
+ ".inst 0x4f41f0ef // bfdot v15.4s, v7.8h, v1.h[0]\n"
+ ".inst 0x4f42f0f3 // bfdot v19.4s, v7.8h, v2.h[0]\n"
+ "100:" // Height 3: Multiply loop: No odd multiplies
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "add x12, x12, #0x1\n"
+ "cmp x12, x19\n"
+ "bne 90b\n"
+ "prfm pstl1keep, [x13, #0x0]\n"
+ "prfm pstl1keep, [x9, #0x0]\n"
+ "prfm pstl1keep, [x27, #0x0]\n"
+ "tbz %x[flags], #1, 101f\n"
+ "add x19, %x[args_ptr], %[offset_min]\n"
+ "ld1r { v1.4s }, [x19]\n"
+ "add x19, %x[args_ptr], %[offset_max]\n"
+ "ld1r { v0.4s }, [x19]\n"
+ "fmin v8.4s, v8.4s, v0.4s\n"
+ "fmin v9.4s, v9.4s, v0.4s\n"
+ "fmin v10.4s, v10.4s, v0.4s\n"
+ "fmin v11.4s, v11.4s, v0.4s\n"
+ "fmax v8.4s, v8.4s, v1.4s\n"
+ "fmax v9.4s, v9.4s, v1.4s\n"
+ "fmax v10.4s, v10.4s, v1.4s\n"
+ "fmax v11.4s, v11.4s, v1.4s\n"
+ "fmin v12.4s, v12.4s, v0.4s\n"
+ "fmin v13.4s, v13.4s, v0.4s\n"
+ "fmin v14.4s, v14.4s, v0.4s\n"
+ "fmax v12.4s, v12.4s, v1.4s\n"
+ "fmax v13.4s, v13.4s, v1.4s\n"
+ "fmax v14.4s, v14.4s, v1.4s\n"
+ "fmin v15.4s, v15.4s, v0.4s\n"
+ "fmin v16.4s, v16.4s, v0.4s\n"
+ "fmin v17.4s, v17.4s, v0.4s\n"
+ "fmax v15.4s, v15.4s, v1.4s\n"
+ "fmax v16.4s, v16.4s, v1.4s\n"
+ "fmax v17.4s, v17.4s, v1.4s\n"
+ "fmin v18.4s, v18.4s, v0.4s\n"
+ "fmin v19.4s, v19.4s, v0.4s\n"
+ "fmax v18.4s, v18.4s, v1.4s\n"
+ "fmax v19.4s, v19.4s, v1.4s\n"
+ "101:" // Height 3: No activation
+ "cmp x16, #0x10\n"
+ "bge 110f\n"
+ "tbz x16, #3, 105f\n"
+ "st1 { v8.4s }, [x13], #0x10\n"
+ "st1 { v9.4s }, [x13], #0x10\n"
+ "st1 { v12.4s }, [x9], #0x10\n"
+ "st1 { v13.4s }, [x9], #0x10\n"
+ "st1 { v16.4s }, [x27], #0x10\n"
+ "st1 { v17.4s }, [x27], #0x10\n"
+ "tbz x16, #2, 103f\n"
+ "st1 { v10.4s }, [x13], #0x10\n"
+ "st1 { v14.4s }, [x9], #0x10\n"
+ "st1 { v18.4s }, [x27], #0x10\n"
+ "tbz x16, #1, 102f\n"
+ "str d11, [x13], #0x8\n"
+ "str d15, [x9], #0x8\n"
+ "str d19, [x27], #0x8\n"
+ "tbz x16, #0, 109f\n"
+ "st1 { v11.s }[2], [x13]\n"
+ "st1 { v15.s }[2], [x9]\n"
+ "st1 { v19.s }[2], [x27]\n"
+ "b 109f\n"
+ "102:" // Height 3: Partial direct writeback: partial_1_12
+ "tbz x16, #0, 109f\n"
+ "str s11, [x13, #0x0]\n"
+ "str s15, [x9, #0x0]\n"
+ "str s19, [x27, #0x0]\n"
+ "b 109f\n"
+ "103:" // Height 3: Partial direct writeback: partial_2_8
+ "tbz x16, #1, 104f\n"
+ "str d10, [x13], #0x8\n"
+ "str d14, [x9], #0x8\n"
+ "str d18, [x27], #0x8\n"
+ "tbz x16, #0, 109f\n"
+ "st1 { v10.s }[2], [x13]\n"
+ "st1 { v14.s }[2], [x9]\n"
+ "st1 { v18.s }[2], [x27]\n"
+ "b 109f\n"
+ "104:" // Height 3: Partial direct writeback: partial_1_8
+ "tbz x16, #0, 109f\n"
+ "str s10, [x13, #0x0]\n"
+ "str s14, [x9, #0x0]\n"
+ "str s18, [x27, #0x0]\n"
+ "b 109f\n"
+ "105:" // Height 3: Partial direct writeback: partial_4_0
+ "tbz x16, #2, 107f\n"
+ "st1 { v8.4s }, [x13], #0x10\n"
+ "st1 { v12.4s }, [x9], #0x10\n"
+ "st1 { v16.4s }, [x27], #0x10\n"
+ "tbz x16, #1, 106f\n"
+ "str d9, [x13], #0x8\n"
+ "str d13, [x9], #0x8\n"
+ "str d17, [x27], #0x8\n"
+ "tbz x16, #0, 109f\n"
+ "st1 { v9.s }[2], [x13]\n"
+ "st1 { v13.s }[2], [x9]\n"
+ "st1 { v17.s }[2], [x27]\n"
+ "b 109f\n"
+ "106:" // Height 3: Partial direct writeback: partial_1_4
+ "tbz x16, #0, 109f\n"
+ "str s9, [x13, #0x0]\n"
+ "str s13, [x9, #0x0]\n"
+ "str s17, [x27, #0x0]\n"
+ "b 109f\n"
+ "107:" // Height 3: Partial direct writeback: partial_2_0
+ "tbz x16, #1, 108f\n"
+ "str d8, [x13], #0x8\n"
+ "str d12, [x9], #0x8\n"
+ "str d16, [x27], #0x8\n"
+ "tbz x16, #0, 109f\n"
+ "st1 { v8.s }[2], [x13]\n"
+ "st1 { v12.s }[2], [x9]\n"
+ "st1 { v16.s }[2], [x27]\n"
+ "b 109f\n"
+ "108:" // Height 3: Partial direct writeback: partial_1_0
+ "str s8, [x13, #0x0]\n"
+ "str s12, [x9, #0x0]\n"
+ "str s16, [x27, #0x0]\n"
+ "109:" // Height 3: Partial direct writeback: Done
+ "b 111f\n"
+ "110:" // Height 3: Full writeback
+ "str q8, [x13, #0x0]\n"
+ "str q9, [x13, #0x10]\n"
+ "str q10, [x13, #0x20]\n"
+ "str q11, [x13, #0x30]\n"
+ "str q12, [x9, #0x0]\n"
+ "str q13, [x9, #0x10]\n"
+ "str q14, [x9, #0x20]\n"
+ "str q15, [x9, #0x30]\n"
+ "str q16, [x27, #0x0]\n"
+ "str q17, [x27, #0x10]\n"
+ "str q18, [x27, #0x20]\n"
+ "str q19, [x27, #0x30]\n"
+ "add x13, x13, #0x40\n"
+ "add x9, x9, #0x40\n"
+ "add x27, x27, #0x40\n"
+ "111:" // Height 3: Writeback done
+ "subs x16, x16, #0x10\n"
+ "bgt 77b\n"
+ "b 224f\n"
+ "112:" // Height 4
+ "ldr x16, [%x[args_ptr], %[offsetof_N]]\n"
+ "mov x14, %x[bias]\n"
+ "ldr x15, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 113f\n"
+ "ldr x13, [%x[output_ptr], #0x0]\n"
+ "add x13, x13, x19, LSL #2\n"
+ "ldr x9, [%x[output_ptr], #0x8]\n"
+ "ldr x27, [%x[output_ptr], #0x10]\n"
+ "add x9, x9, x19, LSL #2\n"
+ "ldr x25, [%x[output_ptr], #0x18]\n"
+ "add x27, x27, x19, LSL #2\n"
+ "add x25, x25, x19, LSL #2\n"
+ "b 114f\n"
+ "113:" // Height 4: setup direct output
+ "mov x13, %x[output_ptr]\n"
+ "add x9, x13, x19, LSL #2\n"
+ "add x27, x9, x19, LSL #2\n"
+ "add x25, x27, x19, LSL #2\n"
+ "114:" // Height 4: Column loop
+ "cbz x14, 115f\n"
+ "ldr q8, [x14, #0x0]\n"
+ "mov v12.16b, v8.16b\n"
+ "ldr q9, [x14, #0x10]\n"
+ "mov v16.16b, v8.16b\n"
+ "ldr q10, [x14, #0x20]\n"
+ "mov v20.16b, v8.16b\n"
+ "ldr q11, [x14, #0x30]\n"
+ "add x14, x14, #0x40\n"
+ "mov v13.16b, v9.16b\n"
+ "mov v17.16b, v9.16b\n"
+ "mov v14.16b, v10.16b\n"
+ "mov v15.16b, v11.16b\n"
+ "mov v18.16b, v10.16b\n"
+ "mov v19.16b, v11.16b\n"
+ "mov v21.16b, v9.16b\n"
+ "mov v22.16b, v10.16b\n"
+ "mov v23.16b, v11.16b\n"
+ "b 126f\n"
+ "115:" // Height 4: no bias
+ "tbz %x[flags], #0, 125f\n"
+ "cmp x16, #0x10\n"
+ "bge 124f\n"
+ "tbz x16, #3, 119f\n"
+ "ld1 { v8.4s }, [x13], #0x10\n"
+ "ld1 { v12.4s }, [x9], #0x10\n"
+ "ld1 { v16.4s }, [x27], #0x10\n"
+ "ld1 { v20.4s }, [x25], #0x10\n"
+ "ld1 { v9.4s }, [x13], #0x10\n"
+ "ld1 { v13.4s }, [x9], #0x10\n"
+ "ld1 { v17.4s }, [x27], #0x10\n"
+ "ld1 { v21.4s }, [x25], #0x10\n"
+ "tbz x16, #2, 117f\n"
+ "ld1 { v10.4s }, [x13], #0x10\n"
+ "ld1 { v14.4s }, [x9], #0x10\n"
+ "ld1 { v18.4s }, [x27], #0x10\n"
+ "ld1 { v22.4s }, [x25], #0x10\n"
+ "tbz x16, #1, 116f\n"
+ "mov x19, #0x38\n"
+ "ldr d11, [x13], #0x8\n"
+ "ldr d15, [x9], #0x8\n"
+ "ldr d19, [x27], #0x8\n"
+ "ldr d23, [x25], #0x8\n"
+ "tbz x16, #0, 123f\n"
+ "ld1 { v11.s }[2], [x13]\n"
+ "ld1 { v15.s }[2], [x9]\n"
+ "ld1 { v19.s }[2], [x27]\n"
+ "ld1 { v23.s }[2], [x25]\n"
+ "b 123f\n"
+ "116:" // Height 4: Partial accumulate: partial_1_12
+ "mov x19, #0x30\n"
+ "tbz x16, #0, 123f\n"
+ "ldr s11, [x13, #0x0]\n"
+ "ldr s15, [x9, #0x0]\n"
+ "ldr s19, [x27, #0x0]\n"
+ "ldr s23, [x25, #0x0]\n"
+ "b 123f\n"
+ "117:" // Height 4: Partial accumulate: partial_2_8
+ "tbz x16, #1, 118f\n"
+ "ldr d10, [x13], #0x8\n"
+ "ldr d14, [x9], #0x8\n"
+ "ldr d18, [x27], #0x8\n"
+ "ldr d22, [x25], #0x8\n"
+ "mov x19, #0x28\n"
+ "tbz x16, #0, 123f\n"
+ "ld1 { v10.s }[2], [x13]\n"
+ "ld1 { v14.s }[2], [x9]\n"
+ "ld1 { v18.s }[2], [x27]\n"
+ "ld1 { v22.s }[2], [x25]\n"
+ "b 123f\n"
+ "118:" // Height 4: Partial accumulate: partial_1_8
+ "mov x19, #0x20\n"
+ "tbz x16, #0, 123f\n"
+ "ldr s10, [x13, #0x0]\n"
+ "ldr s14, [x9, #0x0]\n"
+ "ldr s18, [x27, #0x0]\n"
+ "ldr s22, [x25, #0x0]\n"
+ "b 123f\n"
+ "119:" // Height 4: Partial accumulate: partial_4_0
+ "tbz x16, #2, 121f\n"
+ "ld1 { v8.4s }, [x13], #0x10\n"
+ "ld1 { v12.4s }, [x9], #0x10\n"
+ "ld1 { v16.4s }, [x27], #0x10\n"
+ "ld1 { v20.4s }, [x25], #0x10\n"
+ "tbz x16, #1, 120f\n"
+ "mov x19, #0x18\n"
+ "ldr d9, [x13], #0x8\n"
+ "ldr d13, [x9], #0x8\n"
+ "ldr d17, [x27], #0x8\n"
+ "ldr d21, [x25], #0x8\n"
+ "tbz x16, #0, 123f\n"
+ "ld1 { v9.s }[2], [x13]\n"
+ "ld1 { v13.s }[2], [x9]\n"
+ "ld1 { v17.s }[2], [x27]\n"
+ "ld1 { v21.s }[2], [x25]\n"
+ "b 123f\n"
+ "120:" // Height 4: Partial accumulate: partial_1_4
+ "mov x19, #0x10\n"
+ "tbz x16, #0, 123f\n"
+ "ldr s9, [x13, #0x0]\n"
+ "ldr s13, [x9, #0x0]\n"
+ "ldr s17, [x27, #0x0]\n"
+ "ldr s21, [x25, #0x0]\n"
+ "b 123f\n"
+ "121:" // Height 4: Partial accumulate: partial_2_0
+ "tbz x16, #1, 122f\n"
+ "ldr d8, [x13], #0x8\n"
+ "ldr d12, [x9], #0x8\n"
+ "ldr d16, [x27], #0x8\n"
+ "ldr d20, [x25], #0x8\n"
+ "mov x19, #0x8\n"
+ "tbz x16, #0, 123f\n"
+ "ld1 { v8.s }[2], [x13]\n"
+ "ld1 { v12.s }[2], [x9]\n"
+ "ld1 { v16.s }[2], [x27]\n"
+ "ld1 { v20.s }[2], [x25]\n"
+ "b 123f\n"
+ "122:" // Height 4: Partial accumulate: partial_1_0
+ "mov x19, #0x0\n"
+ "ldr s8, [x13, #0x0]\n"
+ "ldr s12, [x9, #0x0]\n"
+ "ldr s16, [x27, #0x0]\n"
+ "ldr s20, [x25, #0x0]\n"
+ "123:" // Height 4: Partial accumulate: Done
+ "sub x13, x13, x19\n"
+ "sub x9, x9, x19\n"
+ "sub x27, x27, x19\n"
+ "sub x25, x25, x19\n"
+ "b 126f\n"
+ "124:" // Height 4: full accumulate
+ "ldr q8, [x13, #0x0]\n"
+ "ldr q9, [x13, #0x10]\n"
+ "ldr q10, [x13, #0x20]\n"
+ "ldr q11, [x13, #0x30]\n"
+ "ldr q12, [x9, #0x0]\n"
+ "ldr q13, [x9, #0x10]\n"
+ "ldr q14, [x9, #0x20]\n"
+ "ldr q15, [x9, #0x30]\n"
+ "ldr q16, [x27, #0x0]\n"
+ "ldr q17, [x27, #0x10]\n"
+ "ldr q18, [x27, #0x20]\n"
+ "ldr q19, [x27, #0x30]\n"
+ "ldr q20, [x25, #0x0]\n"
+ "ldr q21, [x25, #0x10]\n"
+ "ldr q22, [x25, #0x20]\n"
+ "ldr q23, [x25, #0x30]\n"
+ "b 126f\n"
+ "125:" // Height 4: no accumulate
+ "movi v8.16b, #0x0\n"
+ "movi v9.16b, #0x0\n"
+ "movi v10.16b, #0x0\n"
+ "movi v11.16b, #0x0\n"
+ "movi v12.16b, #0x0\n"
+ "movi v13.16b, #0x0\n"
+ "movi v14.16b, #0x0\n"
+ "movi v15.16b, #0x0\n"
+ "movi v16.16b, #0x0\n"
+ "movi v17.16b, #0x0\n"
+ "movi v18.16b, #0x0\n"
+ "movi v19.16b, #0x0\n"
+ "movi v20.16b, #0x0\n"
+ "movi v21.16b, #0x0\n"
+ "movi v22.16b, #0x0\n"
+ "movi v23.16b, #0x0\n"
+ "126:" // Height 4: setup done
+ "mov x12, #0x0\n"
+ "127:" // Height 4: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w11, [x20, x12, LSL #0x2]\n"
+ "tbz %x[flags], #3, 128f\n"
+ "ldr x20, [%x[input_ptr], x12, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x10, [x20, #0x0]\n"
+ "ldr x28, [x20, #0x8]\n"
+ "ldr x26, [x20, #0x10]\n"
+ "ldr x24, [x20, #0x18]\n"
+ "cbnz x12, 129f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x10, x10, x19, LSL #1\n"
+ "add x28, x28, x19, LSL #1\n"
+ "add x26, x26, x19, LSL #1\n"
+ "add x24, x24, x19, LSL #1\n"
+ "b 129f\n"
+ "128:" // Height 4: setup direct input
+ "mov x10, %x[input_ptr]\n"
+ "add x28, x10, x19, LSL #1\n"
+ "add x26, x28, x19, LSL #1\n"
+ "add x24, x26, x19, LSL #1\n"
+ "129:" // Height 4: input setup done
+ "cmp x11, #0x8\n"
+ "blt 132f\n"
+ "cmp x11, #0x10\n"
+ "blt 131f\n"
+ "130:" // Height 4: Multiply loop: Main loop head
+ "ldr q0, [x10, #0x0]\n"
+ "ldr q1, [x28, #0x0]\n"
+ "ldr q2, [x26, #0x0]\n"
+ "ldr q3, [x24, #0x0]\n"
+ "ldr q6, [x15, #0x0]\n"
+ ".inst 0x4f40f0c8 // bfdot v8.4s, v6.8h, v0.h[0]\n"
+ "ldr q7, [x15, #0x10]\n"
+ ".inst 0x4f41f0cc // bfdot v12.4s, v6.8h, v1.h[0]\n"
+ "add x10, x10, #0x10\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ ".inst 0x4f42f0d0 // bfdot v16.4s, v6.8h, v2.h[0]\n"
+ "add x28, x28, #0x10\n"
+ ".inst 0x4f43f0d4 // bfdot v20.4s, v6.8h, v3.h[0]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "ldr q6, [x15, #0x20]\n"
+ ".inst 0x4f40f0e9 // bfdot v9.4s, v7.8h, v0.h[0]\n"
+ "add x26, x26, #0x10\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ ".inst 0x4f41f0ed // bfdot v13.4s, v7.8h, v1.h[0]\n"
+ "add x24, x24, #0x10\n"
+ ".inst 0x4f42f0f1 // bfdot v17.4s, v7.8h, v2.h[0]\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "sub x11, x11, #0x8\n"
+ ".inst 0x4f43f0f5 // bfdot v21.4s, v7.8h, v3.h[0]\n"
+ "ldr q7, [x15, #0x30]\n"
+ ".inst 0x4f40f0ca // bfdot v10.4s, v6.8h, v0.h[0]\n"
+ "cmp x11, #0x10\n"
+ ".inst 0x4f41f0ce // bfdot v14.4s, v6.8h, v1.h[0]\n"
+ ".inst 0x4f42f0d2 // bfdot v18.4s, v6.8h, v2.h[0]\n"
+ ".inst 0x4f43f0d6 // bfdot v22.4s, v6.8h, v3.h[0]\n"
+ "ldr q6, [x15, #0x40]\n"
+ ".inst 0x4f40f0eb // bfdot v11.4s, v7.8h, v0.h[0]\n"
+ ".inst 0x4f41f0ef // bfdot v15.4s, v7.8h, v1.h[0]\n"
+ ".inst 0x4f42f0f3 // bfdot v19.4s, v7.8h, v2.h[0]\n"
+ ".inst 0x4f43f0f7 // bfdot v23.4s, v7.8h, v3.h[0]\n"
+ "ldr q7, [x15, #0x50]\n"
+ ".inst 0x4f60f0c8 // bfdot v8.4s, v6.8h, v0.h[1]\n"
+ ".inst 0x4f61f0cc // bfdot v12.4s, v6.8h, v1.h[1]\n"
+ ".inst 0x4f62f0d0 // bfdot v16.4s, v6.8h, v2.h[1]\n"
+ ".inst 0x4f63f0d4 // bfdot v20.4s, v6.8h, v3.h[1]\n"
+ "ldr q6, [x15, #0x60]\n"
+ ".inst 0x4f60f0e9 // bfdot v9.4s, v7.8h, v0.h[1]\n"
+ ".inst 0x4f61f0ed // bfdot v13.4s, v7.8h, v1.h[1]\n"
+ ".inst 0x4f62f0f1 // bfdot v17.4s, v7.8h, v2.h[1]\n"
+ ".inst 0x4f63f0f5 // bfdot v21.4s, v7.8h, v3.h[1]\n"
+ "ldr q7, [x15, #0x70]\n"
+ ".inst 0x4f60f0ca // bfdot v10.4s, v6.8h, v0.h[1]\n"
+ ".inst 0x4f61f0ce // bfdot v14.4s, v6.8h, v1.h[1]\n"
+ ".inst 0x4f62f0d2 // bfdot v18.4s, v6.8h, v2.h[1]\n"
+ ".inst 0x4f63f0d6 // bfdot v22.4s, v6.8h, v3.h[1]\n"
+ "ldr q6, [x15, #0x80]\n"
+ ".inst 0x4f60f0eb // bfdot v11.4s, v7.8h, v0.h[1]\n"
+ ".inst 0x4f61f0ef // bfdot v15.4s, v7.8h, v1.h[1]\n"
+ ".inst 0x4f62f0f3 // bfdot v19.4s, v7.8h, v2.h[1]\n"
+ ".inst 0x4f63f0f7 // bfdot v23.4s, v7.8h, v3.h[1]\n"
+ "ldr q7, [x15, #0x90]\n"
+ ".inst 0x4f40f8c8 // bfdot v8.4s, v6.8h, v0.h[2]\n"
+ ".inst 0x4f41f8cc // bfdot v12.4s, v6.8h, v1.h[2]\n"
+ ".inst 0x4f42f8d0 // bfdot v16.4s, v6.8h, v2.h[2]\n"
+ ".inst 0x4f43f8d4 // bfdot v20.4s, v6.8h, v3.h[2]\n"
+ "ldr q6, [x15, #0xa0]\n"
+ ".inst 0x4f40f8e9 // bfdot v9.4s, v7.8h, v0.h[2]\n"
+ ".inst 0x4f41f8ed // bfdot v13.4s, v7.8h, v1.h[2]\n"
+ ".inst 0x4f42f8f1 // bfdot v17.4s, v7.8h, v2.h[2]\n"
+ ".inst 0x4f43f8f5 // bfdot v21.4s, v7.8h, v3.h[2]\n"
+ "ldr q7, [x15, #0xb0]\n"
+ ".inst 0x4f40f8ca // bfdot v10.4s, v6.8h, v0.h[2]\n"
+ ".inst 0x4f41f8ce // bfdot v14.4s, v6.8h, v1.h[2]\n"
+ ".inst 0x4f42f8d2 // bfdot v18.4s, v6.8h, v2.h[2]\n"
+ ".inst 0x4f43f8d6 // bfdot v22.4s, v6.8h, v3.h[2]\n"
+ "ldr q6, [x15, #0xc0]\n"
+ ".inst 0x4f40f8eb // bfdot v11.4s, v7.8h, v0.h[2]\n"
+ ".inst 0x4f41f8ef // bfdot v15.4s, v7.8h, v1.h[2]\n"
+ ".inst 0x4f42f8f3 // bfdot v19.4s, v7.8h, v2.h[2]\n"
+ ".inst 0x4f43f8f7 // bfdot v23.4s, v7.8h, v3.h[2]\n"
+ "ldr q7, [x15, #0xd0]\n"
+ ".inst 0x4f60f8c8 // bfdot v8.4s, v6.8h, v0.h[3]\n"
+ ".inst 0x4f61f8cc // bfdot v12.4s, v6.8h, v1.h[3]\n"
+ ".inst 0x4f62f8d0 // bfdot v16.4s, v6.8h, v2.h[3]\n"
+ ".inst 0x4f63f8d4 // bfdot v20.4s, v6.8h, v3.h[3]\n"
+ "ldr q6, [x15, #0xe0]\n"
+ ".inst 0x4f60f8e9 // bfdot v9.4s, v7.8h, v0.h[3]\n"
+ ".inst 0x4f61f8ed // bfdot v13.4s, v7.8h, v1.h[3]\n"
+ ".inst 0x4f62f8f1 // bfdot v17.4s, v7.8h, v2.h[3]\n"
+ ".inst 0x4f63f8f5 // bfdot v21.4s, v7.8h, v3.h[3]\n"
+ "ldr q7, [x15, #0xf0]\n"
+ "add x15, x15, #0x100\n"
+ ".inst 0x4f60f8ca // bfdot v10.4s, v6.8h, v0.h[3]\n"
+ ".inst 0x4f61f8ce // bfdot v14.4s, v6.8h, v1.h[3]\n"
+ ".inst 0x4f62f8d2 // bfdot v18.4s, v6.8h, v2.h[3]\n"
+ ".inst 0x4f63f8d6 // bfdot v22.4s, v6.8h, v3.h[3]\n"
+ ".inst 0x4f60f8eb // bfdot v11.4s, v7.8h, v0.h[3]\n"
+ ".inst 0x4f61f8ef // bfdot v15.4s, v7.8h, v1.h[3]\n"
+ ".inst 0x4f62f8f3 // bfdot v19.4s, v7.8h, v2.h[3]\n"
+ ".inst 0x4f63f8f7 // bfdot v23.4s, v7.8h, v3.h[3]\n"
+ "bge 130b\n"
+ "131:" // Height 4: Multiply loop: Single iteration only
+ "sub x11, x11, #0x8\n"
+ "ldr q0, [x10, #0x0]\n"
+ "ldr q1, [x28, #0x0]\n"
+ "ldr q2, [x26, #0x0]\n"
+ "ldr q3, [x24, #0x0]\n"
+ "ldr q6, [x15, #0x0]\n"
+ ".inst 0x4f40f0c8 // bfdot v8.4s, v6.8h, v0.h[0]\n"
+ "ldr q7, [x15, #0x10]\n"
+ ".inst 0x4f41f0cc // bfdot v12.4s, v6.8h, v1.h[0]\n"
+ "add x10, x10, #0x10\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ ".inst 0x4f42f0d0 // bfdot v16.4s, v6.8h, v2.h[0]\n"
+ "add x28, x28, #0x10\n"
+ ".inst 0x4f43f0d4 // bfdot v20.4s, v6.8h, v3.h[0]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "ldr q6, [x15, #0x20]\n"
+ ".inst 0x4f40f0e9 // bfdot v9.4s, v7.8h, v0.h[0]\n"
+ "add x26, x26, #0x10\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ ".inst 0x4f41f0ed // bfdot v13.4s, v7.8h, v1.h[0]\n"
+ "add x24, x24, #0x10\n"
+ ".inst 0x4f42f0f1 // bfdot v17.4s, v7.8h, v2.h[0]\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ ".inst 0x4f43f0f5 // bfdot v21.4s, v7.8h, v3.h[0]\n"
+ "ldr q7, [x15, #0x30]\n"
+ ".inst 0x4f40f0ca // bfdot v10.4s, v6.8h, v0.h[0]\n"
+ ".inst 0x4f41f0ce // bfdot v14.4s, v6.8h, v1.h[0]\n"
+ ".inst 0x4f42f0d2 // bfdot v18.4s, v6.8h, v2.h[0]\n"
+ ".inst 0x4f43f0d6 // bfdot v22.4s, v6.8h, v3.h[0]\n"
+ "ldr q6, [x15, #0x40]\n"
+ ".inst 0x4f40f0eb // bfdot v11.4s, v7.8h, v0.h[0]\n"
+ ".inst 0x4f41f0ef // bfdot v15.4s, v7.8h, v1.h[0]\n"
+ ".inst 0x4f42f0f3 // bfdot v19.4s, v7.8h, v2.h[0]\n"
+ ".inst 0x4f43f0f7 // bfdot v23.4s, v7.8h, v3.h[0]\n"
+ "ldr q7, [x15, #0x50]\n"
+ ".inst 0x4f60f0c8 // bfdot v8.4s, v6.8h, v0.h[1]\n"
+ ".inst 0x4f61f0cc // bfdot v12.4s, v6.8h, v1.h[1]\n"
+ ".inst 0x4f62f0d0 // bfdot v16.4s, v6.8h, v2.h[1]\n"
+ ".inst 0x4f63f0d4 // bfdot v20.4s, v6.8h, v3.h[1]\n"
+ "ldr q6, [x15, #0x60]\n"
+ ".inst 0x4f60f0e9 // bfdot v9.4s, v7.8h, v0.h[1]\n"
+ ".inst 0x4f61f0ed // bfdot v13.4s, v7.8h, v1.h[1]\n"
+ ".inst 0x4f62f0f1 // bfdot v17.4s, v7.8h, v2.h[1]\n"
+ ".inst 0x4f63f0f5 // bfdot v21.4s, v7.8h, v3.h[1]\n"
+ "ldr q7, [x15, #0x70]\n"
+ ".inst 0x4f60f0ca // bfdot v10.4s, v6.8h, v0.h[1]\n"
+ ".inst 0x4f61f0ce // bfdot v14.4s, v6.8h, v1.h[1]\n"
+ ".inst 0x4f62f0d2 // bfdot v18.4s, v6.8h, v2.h[1]\n"
+ ".inst 0x4f63f0d6 // bfdot v22.4s, v6.8h, v3.h[1]\n"
+ "ldr q6, [x15, #0x80]\n"
+ ".inst 0x4f60f0eb // bfdot v11.4s, v7.8h, v0.h[1]\n"
+ ".inst 0x4f61f0ef // bfdot v15.4s, v7.8h, v1.h[1]\n"
+ ".inst 0x4f62f0f3 // bfdot v19.4s, v7.8h, v2.h[1]\n"
+ ".inst 0x4f63f0f7 // bfdot v23.4s, v7.8h, v3.h[1]\n"
+ "ldr q7, [x15, #0x90]\n"
+ ".inst 0x4f40f8c8 // bfdot v8.4s, v6.8h, v0.h[2]\n"
+ ".inst 0x4f41f8cc // bfdot v12.4s, v6.8h, v1.h[2]\n"
+ ".inst 0x4f42f8d0 // bfdot v16.4s, v6.8h, v2.h[2]\n"
+ ".inst 0x4f43f8d4 // bfdot v20.4s, v6.8h, v3.h[2]\n"
+ "ldr q6, [x15, #0xa0]\n"
+ ".inst 0x4f40f8e9 // bfdot v9.4s, v7.8h, v0.h[2]\n"
+ ".inst 0x4f41f8ed // bfdot v13.4s, v7.8h, v1.h[2]\n"
+ ".inst 0x4f42f8f1 // bfdot v17.4s, v7.8h, v2.h[2]\n"
+ ".inst 0x4f43f8f5 // bfdot v21.4s, v7.8h, v3.h[2]\n"
+ "ldr q7, [x15, #0xb0]\n"
+ ".inst 0x4f40f8ca // bfdot v10.4s, v6.8h, v0.h[2]\n"
+ ".inst 0x4f41f8ce // bfdot v14.4s, v6.8h, v1.h[2]\n"
+ ".inst 0x4f42f8d2 // bfdot v18.4s, v6.8h, v2.h[2]\n"
+ ".inst 0x4f43f8d6 // bfdot v22.4s, v6.8h, v3.h[2]\n"
+ "ldr q6, [x15, #0xc0]\n"
+ ".inst 0x4f40f8eb // bfdot v11.4s, v7.8h, v0.h[2]\n"
+ ".inst 0x4f41f8ef // bfdot v15.4s, v7.8h, v1.h[2]\n"
+ ".inst 0x4f42f8f3 // bfdot v19.4s, v7.8h, v2.h[2]\n"
+ ".inst 0x4f43f8f7 // bfdot v23.4s, v7.8h, v3.h[2]\n"
+ "ldr q7, [x15, #0xd0]\n"
+ ".inst 0x4f60f8c8 // bfdot v8.4s, v6.8h, v0.h[3]\n"
+ ".inst 0x4f61f8cc // bfdot v12.4s, v6.8h, v1.h[3]\n"
+ ".inst 0x4f62f8d0 // bfdot v16.4s, v6.8h, v2.h[3]\n"
+ ".inst 0x4f63f8d4 // bfdot v20.4s, v6.8h, v3.h[3]\n"
+ "ldr q6, [x15, #0xe0]\n"
+ ".inst 0x4f60f8e9 // bfdot v9.4s, v7.8h, v0.h[3]\n"
+ ".inst 0x4f61f8ed // bfdot v13.4s, v7.8h, v1.h[3]\n"
+ ".inst 0x4f62f8f1 // bfdot v17.4s, v7.8h, v2.h[3]\n"
+ ".inst 0x4f63f8f5 // bfdot v21.4s, v7.8h, v3.h[3]\n"
+ "ldr q7, [x15, #0xf0]\n"
+ "add x15, x15, #0x100\n"
+ ".inst 0x4f60f8ca // bfdot v10.4s, v6.8h, v0.h[3]\n"
+ ".inst 0x4f61f8ce // bfdot v14.4s, v6.8h, v1.h[3]\n"
+ ".inst 0x4f62f8d2 // bfdot v18.4s, v6.8h, v2.h[3]\n"
+ ".inst 0x4f63f8d6 // bfdot v22.4s, v6.8h, v3.h[3]\n"
+ ".inst 0x4f60f8eb // bfdot v11.4s, v7.8h, v0.h[3]\n"
+ ".inst 0x4f61f8ef // bfdot v15.4s, v7.8h, v1.h[3]\n"
+ ".inst 0x4f62f8f3 // bfdot v19.4s, v7.8h, v2.h[3]\n"
+ ".inst 0x4f63f8f7 // bfdot v23.4s, v7.8h, v3.h[3]\n"
+ "132:" // Height 4: Multiply loop: Main loop skip
+ "cbz x11, 137f\n"
+ "cmp x11, #0x2\n"
+ "blt 134f\n"
+ "133:" // Height 4: Multiply loop: Odd block loop
+ "ldr s0, [x10], #0x4\n"
+ "ldr s1, [x28], #0x4\n"
+ "ldr s2, [x26], #0x4\n"
+ "ldr s3, [x24], #0x4\n"
+ "ldr q6, [x15, #0x0]\n"
+ ".inst 0x4f40f0c8 // bfdot v8.4s, v6.8h, v0.h[0]\n"
+ "ldr q7, [x15, #0x10]\n"
+ ".inst 0x4f41f0cc // bfdot v12.4s, v6.8h, v1.h[0]\n"
+ "sub x11, x11, #0x2\n"
+ ".inst 0x4f42f0d0 // bfdot v16.4s, v6.8h, v2.h[0]\n"
+ "cmp x11, #0x2\n"
+ ".inst 0x4f43f0d4 // bfdot v20.4s, v6.8h, v3.h[0]\n"
+ "ldr q6, [x15, #0x20]\n"
+ ".inst 0x4f40f0e9 // bfdot v9.4s, v7.8h, v0.h[0]\n"
+ ".inst 0x4f41f0ed // bfdot v13.4s, v7.8h, v1.h[0]\n"
+ ".inst 0x4f42f0f1 // bfdot v17.4s, v7.8h, v2.h[0]\n"
+ ".inst 0x4f43f0f5 // bfdot v21.4s, v7.8h, v3.h[0]\n"
+ "ldr q7, [x15, #0x30]\n"
+ ".inst 0x4f40f0ca // bfdot v10.4s, v6.8h, v0.h[0]\n"
+ "add x15, x15, #0x40\n"
+ ".inst 0x4f41f0ce // bfdot v14.4s, v6.8h, v1.h[0]\n"
+ ".inst 0x4f42f0d2 // bfdot v18.4s, v6.8h, v2.h[0]\n"
+ ".inst 0x4f43f0d6 // bfdot v22.4s, v6.8h, v3.h[0]\n"
+ ".inst 0x4f40f0eb // bfdot v11.4s, v7.8h, v0.h[0]\n"
+ ".inst 0x4f41f0ef // bfdot v15.4s, v7.8h, v1.h[0]\n"
+ ".inst 0x4f42f0f3 // bfdot v19.4s, v7.8h, v2.h[0]\n"
+ ".inst 0x4f43f0f7 // bfdot v23.4s, v7.8h, v3.h[0]\n"
+ "bge 133b\n"
+ "cbz x11, 137f\n"
+ "134:" // Height 4: Multiply loop: Skip odd blocks
+ "tbz x11, #1, 135f\n"
+ "ldr s0, [x10], #0x4\n"
+ "ldr s1, [x28], #0x4\n"
+ "ldr s2, [x26], #0x4\n"
+ "ldr s3, [x24], #0x4\n"
+ "tbz x11, #0, 136f\n"
+ "ld1 { v0.h }[2], [x10]\n"
+ "ld1 { v1.h }[2], [x28]\n"
+ "ld1 { v2.h }[2], [x26]\n"
+ "ld1 { v3.h }[2], [x24]\n"
+ "b 136f\n"
+ "135:" // Height 4: Multiply loop: Ragged operand read: partial_1_0
+ "ldr h0, [x10, #0x0]\n"
+ "ldr h1, [x28, #0x0]\n"
+ "ldr h2, [x26, #0x0]\n"
+ "ldr h3, [x24, #0x0]\n"
+ "136:" // Height 4: Multiply loop: Ragged operand read: Done
+ "ldr q6, [x15, #0x0]\n"
+ ".inst 0x4f40f0c8 // bfdot v8.4s, v6.8h, v0.h[0]\n"
+ "ldr q7, [x15, #0x10]\n"
+ ".inst 0x4f41f0cc // bfdot v12.4s, v6.8h, v1.h[0]\n"
+ ".inst 0x4f42f0d0 // bfdot v16.4s, v6.8h, v2.h[0]\n"
+ ".inst 0x4f43f0d4 // bfdot v20.4s, v6.8h, v3.h[0]\n"
+ "ldr q6, [x15, #0x20]\n"
+ ".inst 0x4f40f0e9 // bfdot v9.4s, v7.8h, v0.h[0]\n"
+ ".inst 0x4f41f0ed // bfdot v13.4s, v7.8h, v1.h[0]\n"
+ ".inst 0x4f42f0f1 // bfdot v17.4s, v7.8h, v2.h[0]\n"
+ ".inst 0x4f43f0f5 // bfdot v21.4s, v7.8h, v3.h[0]\n"
+ "ldr q7, [x15, #0x30]\n"
+ ".inst 0x4f40f0ca // bfdot v10.4s, v6.8h, v0.h[0]\n"
+ "add x15, x15, #0x40\n"
+ ".inst 0x4f41f0ce // bfdot v14.4s, v6.8h, v1.h[0]\n"
+ ".inst 0x4f42f0d2 // bfdot v18.4s, v6.8h, v2.h[0]\n"
+ ".inst 0x4f43f0d6 // bfdot v22.4s, v6.8h, v3.h[0]\n"
+ ".inst 0x4f40f0eb // bfdot v11.4s, v7.8h, v0.h[0]\n"
+ ".inst 0x4f41f0ef // bfdot v15.4s, v7.8h, v1.h[0]\n"
+ ".inst 0x4f42f0f3 // bfdot v19.4s, v7.8h, v2.h[0]\n"
+ ".inst 0x4f43f0f7 // bfdot v23.4s, v7.8h, v3.h[0]\n"
+ "137:" // Height 4: Multiply loop: No odd multiplies
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "add x12, x12, #0x1\n"
+ "cmp x12, x19\n"
+ "bne 127b\n"
+ "prfm pstl1keep, [x13, #0x0]\n"
+ "prfm pstl1keep, [x9, #0x0]\n"
+ "prfm pstl1keep, [x27, #0x0]\n"
+ "prfm pstl1keep, [x25, #0x0]\n"
+ "tbz %x[flags], #1, 138f\n"
+ "add x19, %x[args_ptr], %[offset_min]\n"
+ "ld1r { v1.4s }, [x19]\n"
+ "add x19, %x[args_ptr], %[offset_max]\n"
+ "ld1r { v0.4s }, [x19]\n"
+ "fmin v8.4s, v8.4s, v0.4s\n"
+ "fmin v9.4s, v9.4s, v0.4s\n"
+ "fmin v10.4s, v10.4s, v0.4s\n"
+ "fmin v11.4s, v11.4s, v0.4s\n"
+ "fmax v8.4s, v8.4s, v1.4s\n"
+ "fmax v9.4s, v9.4s, v1.4s\n"
+ "fmax v10.4s, v10.4s, v1.4s\n"
+ "fmax v11.4s, v11.4s, v1.4s\n"
+ "fmin v12.4s, v12.4s, v0.4s\n"
+ "fmin v13.4s, v13.4s, v0.4s\n"
+ "fmin v14.4s, v14.4s, v0.4s\n"
+ "fmax v12.4s, v12.4s, v1.4s\n"
+ "fmax v13.4s, v13.4s, v1.4s\n"
+ "fmax v14.4s, v14.4s, v1.4s\n"
+ "fmin v15.4s, v15.4s, v0.4s\n"
+ "fmin v16.4s, v16.4s, v0.4s\n"
+ "fmin v17.4s, v17.4s, v0.4s\n"
+ "fmax v15.4s, v15.4s, v1.4s\n"
+ "fmax v16.4s, v16.4s, v1.4s\n"
+ "fmax v17.4s, v17.4s, v1.4s\n"
+ "fmin v18.4s, v18.4s, v0.4s\n"
+ "fmin v19.4s, v19.4s, v0.4s\n"
+ "fmin v20.4s, v20.4s, v0.4s\n"
+ "fmax v18.4s, v18.4s, v1.4s\n"
+ "fmax v19.4s, v19.4s, v1.4s\n"
+ "fmax v20.4s, v20.4s, v1.4s\n"
+ "fmin v21.4s, v21.4s, v0.4s\n"
+ "fmin v22.4s, v22.4s, v0.4s\n"
+ "fmin v23.4s, v23.4s, v0.4s\n"
+ "fmax v21.4s, v21.4s, v1.4s\n"
+ "fmax v22.4s, v22.4s, v1.4s\n"
+ "fmax v23.4s, v23.4s, v1.4s\n"
+ "138:" // Height 4: No activation
+ "cmp x16, #0x10\n"
+ "bge 147f\n"
+ "tbz x16, #3, 142f\n"
+ "st1 { v8.4s }, [x13], #0x10\n"
+ "st1 { v9.4s }, [x13], #0x10\n"
+ "st1 { v12.4s }, [x9], #0x10\n"
+ "st1 { v13.4s }, [x9], #0x10\n"
+ "st1 { v16.4s }, [x27], #0x10\n"
+ "st1 { v17.4s }, [x27], #0x10\n"
+ "st1 { v20.4s }, [x25], #0x10\n"
+ "st1 { v21.4s }, [x25], #0x10\n"
+ "tbz x16, #2, 140f\n"
+ "st1 { v10.4s }, [x13], #0x10\n"
+ "st1 { v14.4s }, [x9], #0x10\n"
+ "st1 { v18.4s }, [x27], #0x10\n"
+ "st1 { v22.4s }, [x25], #0x10\n"
+ "tbz x16, #1, 139f\n"
+ "str d11, [x13], #0x8\n"
+ "str d15, [x9], #0x8\n"
+ "str d19, [x27], #0x8\n"
+ "str d23, [x25], #0x8\n"
+ "tbz x16, #0, 146f\n"
+ "st1 { v11.s }[2], [x13]\n"
+ "st1 { v15.s }[2], [x9]\n"
+ "st1 { v19.s }[2], [x27]\n"
+ "st1 { v23.s }[2], [x25]\n"
+ "b 146f\n"
+ "139:" // Height 4: Partial direct writeback: partial_1_12
+ "tbz x16, #0, 146f\n"
+ "str s11, [x13, #0x0]\n"
+ "str s15, [x9, #0x0]\n"
+ "str s19, [x27, #0x0]\n"
+ "str s23, [x25, #0x0]\n"
+ "b 146f\n"
+ "140:" // Height 4: Partial direct writeback: partial_2_8
+ "tbz x16, #1, 141f\n"
+ "str d10, [x13], #0x8\n"
+ "str d14, [x9], #0x8\n"
+ "str d18, [x27], #0x8\n"
+ "str d22, [x25], #0x8\n"
+ "tbz x16, #0, 146f\n"
+ "st1 { v10.s }[2], [x13]\n"
+ "st1 { v14.s }[2], [x9]\n"
+ "st1 { v18.s }[2], [x27]\n"
+ "st1 { v22.s }[2], [x25]\n"
+ "b 146f\n"
+ "141:" // Height 4: Partial direct writeback: partial_1_8
+ "tbz x16, #0, 146f\n"
+ "str s10, [x13, #0x0]\n"
+ "str s14, [x9, #0x0]\n"
+ "str s18, [x27, #0x0]\n"
+ "str s22, [x25, #0x0]\n"
+ "b 146f\n"
+ "142:" // Height 4: Partial direct writeback: partial_4_0
+ "tbz x16, #2, 144f\n"
+ "st1 { v8.4s }, [x13], #0x10\n"
+ "st1 { v12.4s }, [x9], #0x10\n"
+ "st1 { v16.4s }, [x27], #0x10\n"
+ "st1 { v20.4s }, [x25], #0x10\n"
+ "tbz x16, #1, 143f\n"
+ "str d9, [x13], #0x8\n"
+ "str d13, [x9], #0x8\n"
+ "str d17, [x27], #0x8\n"
+ "str d21, [x25], #0x8\n"
+ "tbz x16, #0, 146f\n"
+ "st1 { v9.s }[2], [x13]\n"
+ "st1 { v13.s }[2], [x9]\n"
+ "st1 { v17.s }[2], [x27]\n"
+ "st1 { v21.s }[2], [x25]\n"
+ "b 146f\n"
+ "143:" // Height 4: Partial direct writeback: partial_1_4
+ "tbz x16, #0, 146f\n"
+ "str s9, [x13, #0x0]\n"
+ "str s13, [x9, #0x0]\n"
+ "str s17, [x27, #0x0]\n"
+ "str s21, [x25, #0x0]\n"
+ "b 146f\n"
+ "144:" // Height 4: Partial direct writeback: partial_2_0
+ "tbz x16, #1, 145f\n"
+ "str d8, [x13], #0x8\n"
+ "str d12, [x9], #0x8\n"
+ "str d16, [x27], #0x8\n"
+ "str d20, [x25], #0x8\n"
+ "tbz x16, #0, 146f\n"
+ "st1 { v8.s }[2], [x13]\n"
+ "st1 { v12.s }[2], [x9]\n"
+ "st1 { v16.s }[2], [x27]\n"
+ "st1 { v20.s }[2], [x25]\n"
+ "b 146f\n"
+ "145:" // Height 4: Partial direct writeback: partial_1_0
+ "str s8, [x13, #0x0]\n"
+ "str s12, [x9, #0x0]\n"
+ "str s16, [x27, #0x0]\n"
+ "str s20, [x25, #0x0]\n"
+ "146:" // Height 4: Partial direct writeback: Done
+ "b 148f\n"
+ "147:" // Height 4: Full writeback
+ "str q8, [x13, #0x0]\n"
+ "str q9, [x13, #0x10]\n"
+ "str q10, [x13, #0x20]\n"
+ "str q11, [x13, #0x30]\n"
+ "str q12, [x9, #0x0]\n"
+ "str q13, [x9, #0x10]\n"
+ "str q14, [x9, #0x20]\n"
+ "str q15, [x9, #0x30]\n"
+ "str q16, [x27, #0x0]\n"
+ "str q17, [x27, #0x10]\n"
+ "str q18, [x27, #0x20]\n"
+ "str q19, [x27, #0x30]\n"
+ "str q20, [x25, #0x0]\n"
+ "str q21, [x25, #0x10]\n"
+ "str q22, [x25, #0x20]\n"
+ "str q23, [x25, #0x30]\n"
+ "add x13, x13, #0x40\n"
+ "add x9, x9, #0x40\n"
+ "add x27, x27, #0x40\n"
+ "add x25, x25, #0x40\n"
+ "148:" // Height 4: Writeback done
+ "subs x16, x16, #0x10\n"
+ "bgt 114b\n"
+ "b 224f\n"
+ "149:" // Height 5
+ "ldr x16, [%x[args_ptr], %[offsetof_N]]\n"
+ "mov x14, %x[bias]\n"
+ "ldr x15, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 150f\n"
+ "ldr x13, [%x[output_ptr], #0x0]\n"
+ "add x13, x13, x19, LSL #2\n"
+ "ldr x9, [%x[output_ptr], #0x8]\n"
+ "ldr x27, [%x[output_ptr], #0x10]\n"
+ "add x9, x9, x19, LSL #2\n"
+ "ldr x25, [%x[output_ptr], #0x18]\n"
+ "ldr x23, [%x[output_ptr], #0x20]\n"
+ "add x27, x27, x19, LSL #2\n"
+ "add x25, x25, x19, LSL #2\n"
+ "add x23, x23, x19, LSL #2\n"
+ "b 151f\n"
+ "150:" // Height 5: setup direct output
+ "mov x13, %x[output_ptr]\n"
+ "add x9, x13, x19, LSL #2\n"
+ "add x27, x9, x19, LSL #2\n"
+ "add x25, x27, x19, LSL #2\n"
+ "add x23, x25, x19, LSL #2\n"
+ "151:" // Height 5: Column loop
+ "cbz x14, 152f\n"
+ "ldr q8, [x14, #0x0]\n"
+ "mov v12.16b, v8.16b\n"
+ "ldr q9, [x14, #0x10]\n"
+ "mov v16.16b, v8.16b\n"
+ "ldr q10, [x14, #0x20]\n"
+ "mov v20.16b, v8.16b\n"
+ "ldr q11, [x14, #0x30]\n"
+ "mov v24.16b, v8.16b\n"
+ "add x14, x14, #0x40\n"
+ "mov v13.16b, v9.16b\n"
+ "mov v17.16b, v9.16b\n"
+ "mov v14.16b, v10.16b\n"
+ "mov v15.16b, v11.16b\n"
+ "mov v18.16b, v10.16b\n"
+ "mov v19.16b, v11.16b\n"
+ "mov v21.16b, v9.16b\n"
+ "mov v22.16b, v10.16b\n"
+ "mov v23.16b, v11.16b\n"
+ "mov v25.16b, v9.16b\n"
+ "mov v26.16b, v10.16b\n"
+ "mov v27.16b, v11.16b\n"
+ "b 163f\n"
+ "152:" // Height 5: no bias
+ "tbz %x[flags], #0, 162f\n"
+ "cmp x16, #0x10\n"
+ "bge 161f\n"
+ "tbz x16, #3, 156f\n"
+ "ld1 { v8.4s }, [x13], #0x10\n"
+ "ld1 { v12.4s }, [x9], #0x10\n"
+ "ld1 { v16.4s }, [x27], #0x10\n"
+ "ld1 { v20.4s }, [x25], #0x10\n"
+ "ld1 { v24.4s }, [x23], #0x10\n"
+ "ld1 { v9.4s }, [x13], #0x10\n"
+ "ld1 { v13.4s }, [x9], #0x10\n"
+ "ld1 { v17.4s }, [x27], #0x10\n"
+ "ld1 { v21.4s }, [x25], #0x10\n"
+ "ld1 { v25.4s }, [x23], #0x10\n"
+ "tbz x16, #2, 154f\n"
+ "ld1 { v10.4s }, [x13], #0x10\n"
+ "ld1 { v14.4s }, [x9], #0x10\n"
+ "ld1 { v18.4s }, [x27], #0x10\n"
+ "ld1 { v22.4s }, [x25], #0x10\n"
+ "ld1 { v26.4s }, [x23], #0x10\n"
+ "tbz x16, #1, 153f\n"
+ "mov x19, #0x38\n"
+ "ldr d11, [x13], #0x8\n"
+ "ldr d15, [x9], #0x8\n"
+ "ldr d19, [x27], #0x8\n"
+ "ldr d23, [x25], #0x8\n"
+ "ldr d27, [x23], #0x8\n"
+ "tbz x16, #0, 160f\n"
+ "ld1 { v11.s }[2], [x13]\n"
+ "ld1 { v15.s }[2], [x9]\n"
+ "ld1 { v19.s }[2], [x27]\n"
+ "ld1 { v23.s }[2], [x25]\n"
+ "ld1 { v27.s }[2], [x23]\n"
+ "b 160f\n"
+ "153:" // Height 5: Partial accumulate: partial_1_12
+ "mov x19, #0x30\n"
+ "tbz x16, #0, 160f\n"
+ "ldr s11, [x13, #0x0]\n"
+ "ldr s15, [x9, #0x0]\n"
+ "ldr s19, [x27, #0x0]\n"
+ "ldr s23, [x25, #0x0]\n"
+ "ldr s27, [x23, #0x0]\n"
+ "b 160f\n"
+ "154:" // Height 5: Partial accumulate: partial_2_8
+ "tbz x16, #1, 155f\n"
+ "ldr d10, [x13], #0x8\n"
+ "ldr d14, [x9], #0x8\n"
+ "ldr d18, [x27], #0x8\n"
+ "ldr d22, [x25], #0x8\n"
+ "ldr d26, [x23], #0x8\n"
+ "mov x19, #0x28\n"
+ "tbz x16, #0, 160f\n"
+ "ld1 { v10.s }[2], [x13]\n"
+ "ld1 { v14.s }[2], [x9]\n"
+ "ld1 { v18.s }[2], [x27]\n"
+ "ld1 { v22.s }[2], [x25]\n"
+ "ld1 { v26.s }[2], [x23]\n"
+ "b 160f\n"
+ "155:" // Height 5: Partial accumulate: partial_1_8
+ "mov x19, #0x20\n"
+ "tbz x16, #0, 160f\n"
+ "ldr s10, [x13, #0x0]\n"
+ "ldr s14, [x9, #0x0]\n"
+ "ldr s18, [x27, #0x0]\n"
+ "ldr s22, [x25, #0x0]\n"
+ "ldr s26, [x23, #0x0]\n"
+ "b 160f\n"
+ "156:" // Height 5: Partial accumulate: partial_4_0
+ "tbz x16, #2, 158f\n"
+ "ld1 { v8.4s }, [x13], #0x10\n"
+ "ld1 { v12.4s }, [x9], #0x10\n"
+ "ld1 { v16.4s }, [x27], #0x10\n"
+ "ld1 { v20.4s }, [x25], #0x10\n"
+ "ld1 { v24.4s }, [x23], #0x10\n"
+ "tbz x16, #1, 157f\n"
+ "mov x19, #0x18\n"
+ "ldr d9, [x13], #0x8\n"
+ "ldr d13, [x9], #0x8\n"
+ "ldr d17, [x27], #0x8\n"
+ "ldr d21, [x25], #0x8\n"
+ "ldr d25, [x23], #0x8\n"
+ "tbz x16, #0, 160f\n"
+ "ld1 { v9.s }[2], [x13]\n"
+ "ld1 { v13.s }[2], [x9]\n"
+ "ld1 { v17.s }[2], [x27]\n"
+ "ld1 { v21.s }[2], [x25]\n"
+ "ld1 { v25.s }[2], [x23]\n"
+ "b 160f\n"
+ "157:" // Height 5: Partial accumulate: partial_1_4
+ "mov x19, #0x10\n"
+ "tbz x16, #0, 160f\n"
+ "ldr s9, [x13, #0x0]\n"
+ "ldr s13, [x9, #0x0]\n"
+ "ldr s17, [x27, #0x0]\n"
+ "ldr s21, [x25, #0x0]\n"
+ "ldr s25, [x23, #0x0]\n"
+ "b 160f\n"
+ "158:" // Height 5: Partial accumulate: partial_2_0
+ "tbz x16, #1, 159f\n"
+ "ldr d8, [x13], #0x8\n"
+ "ldr d12, [x9], #0x8\n"
+ "ldr d16, [x27], #0x8\n"
+ "ldr d20, [x25], #0x8\n"
+ "ldr d24, [x23], #0x8\n"
+ "mov x19, #0x8\n"
+ "tbz x16, #0, 160f\n"
+ "ld1 { v8.s }[2], [x13]\n"
+ "ld1 { v12.s }[2], [x9]\n"
+ "ld1 { v16.s }[2], [x27]\n"
+ "ld1 { v20.s }[2], [x25]\n"
+ "ld1 { v24.s }[2], [x23]\n"
+ "b 160f\n"
+ "159:" // Height 5: Partial accumulate: partial_1_0
+ "mov x19, #0x0\n"
+ "ldr s8, [x13, #0x0]\n"
+ "ldr s12, [x9, #0x0]\n"
+ "ldr s16, [x27, #0x0]\n"
+ "ldr s20, [x25, #0x0]\n"
+ "ldr s24, [x23, #0x0]\n"
+ "160:" // Height 5: Partial accumulate: Done
+ "sub x13, x13, x19\n"
+ "sub x9, x9, x19\n"
+ "sub x27, x27, x19\n"
+ "sub x25, x25, x19\n"
+ "sub x23, x23, x19\n"
+ "b 163f\n"
+ "161:" // Height 5: full accumulate
+ "ldr q8, [x13, #0x0]\n"
+ "ldr q9, [x13, #0x10]\n"
+ "ldr q10, [x13, #0x20]\n"
+ "ldr q11, [x13, #0x30]\n"
+ "ldr q12, [x9, #0x0]\n"
+ "ldr q13, [x9, #0x10]\n"
+ "ldr q14, [x9, #0x20]\n"
+ "ldr q15, [x9, #0x30]\n"
+ "ldr q16, [x27, #0x0]\n"
+ "ldr q17, [x27, #0x10]\n"
+ "ldr q18, [x27, #0x20]\n"
+ "ldr q19, [x27, #0x30]\n"
+ "ldr q20, [x25, #0x0]\n"
+ "ldr q21, [x25, #0x10]\n"
+ "ldr q22, [x25, #0x20]\n"
+ "ldr q23, [x25, #0x30]\n"
+ "ldr q24, [x23, #0x0]\n"
+ "ldr q25, [x23, #0x10]\n"
+ "ldr q26, [x23, #0x20]\n"
+ "ldr q27, [x23, #0x30]\n"
+ "b 163f\n"
+ "162:" // Height 5: no accumulate
+ "movi v8.16b, #0x0\n"
+ "movi v9.16b, #0x0\n"
+ "movi v10.16b, #0x0\n"
+ "movi v11.16b, #0x0\n"
+ "movi v12.16b, #0x0\n"
+ "movi v13.16b, #0x0\n"
+ "movi v14.16b, #0x0\n"
+ "movi v15.16b, #0x0\n"
+ "movi v16.16b, #0x0\n"
+ "movi v17.16b, #0x0\n"
+ "movi v18.16b, #0x0\n"
+ "movi v19.16b, #0x0\n"
+ "movi v20.16b, #0x0\n"
+ "movi v21.16b, #0x0\n"
+ "movi v22.16b, #0x0\n"
+ "movi v23.16b, #0x0\n"
+ "movi v24.16b, #0x0\n"
+ "movi v25.16b, #0x0\n"
+ "movi v26.16b, #0x0\n"
+ "movi v27.16b, #0x0\n"
+ "163:" // Height 5: setup done
+ "mov x12, #0x0\n"
+ "164:" // Height 5: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w11, [x20, x12, LSL #0x2]\n"
+ "tbz %x[flags], #3, 165f\n"
+ "ldr x20, [%x[input_ptr], x12, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x10, [x20, #0x0]\n"
+ "ldr x28, [x20, #0x8]\n"
+ "ldr x26, [x20, #0x10]\n"
+ "ldr x24, [x20, #0x18]\n"
+ "ldr x22, [x20, #0x20]\n"
+ "cbnz x12, 166f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x10, x10, x19, LSL #1\n"
+ "add x28, x28, x19, LSL #1\n"
+ "add x26, x26, x19, LSL #1\n"
+ "add x24, x24, x19, LSL #1\n"
+ "add x22, x22, x19, LSL #1\n"
+ "b 166f\n"
+ "165:" // Height 5: setup direct input
+ "mov x10, %x[input_ptr]\n"
+ "add x28, x10, x19, LSL #1\n"
+ "add x26, x28, x19, LSL #1\n"
+ "add x24, x26, x19, LSL #1\n"
+ "add x22, x24, x19, LSL #1\n"
+ "166:" // Height 5: input setup done
+ "cmp x11, #0x8\n"
+ "blt 169f\n"
+ "cmp x11, #0x10\n"
+ "blt 168f\n"
+ "167:" // Height 5: Multiply loop: Main loop head
+ "ldr q0, [x10, #0x0]\n"
+ "ldr q1, [x28, #0x0]\n"
+ "ldr q2, [x26, #0x0]\n"
+ "ldr q3, [x24, #0x0]\n"
+ "ldr q4, [x22, #0x0]\n"
+ "ldr q6, [x15, #0x0]\n"
+ ".inst 0x4f40f0c8 // bfdot v8.4s, v6.8h, v0.h[0]\n"
+ "ldr q7, [x15, #0x10]\n"
+ ".inst 0x4f41f0cc // bfdot v12.4s, v6.8h, v1.h[0]\n"
+ "add x10, x10, #0x10\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ ".inst 0x4f42f0d0 // bfdot v16.4s, v6.8h, v2.h[0]\n"
+ "add x28, x28, #0x10\n"
+ ".inst 0x4f43f0d4 // bfdot v20.4s, v6.8h, v3.h[0]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "add x26, x26, #0x10\n"
+ ".inst 0x4f44f0d8 // bfdot v24.4s, v6.8h, v4.h[0]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "ldr q6, [x15, #0x20]\n"
+ ".inst 0x4f40f0e9 // bfdot v9.4s, v7.8h, v0.h[0]\n"
+ "add x24, x24, #0x10\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ ".inst 0x4f41f0ed // bfdot v13.4s, v7.8h, v1.h[0]\n"
+ "add x22, x22, #0x10\n"
+ ".inst 0x4f42f0f1 // bfdot v17.4s, v7.8h, v2.h[0]\n"
+ "prfm pldl1keep, [x22, #0x80]\n"
+ "sub x11, x11, #0x8\n"
+ ".inst 0x4f43f0f5 // bfdot v21.4s, v7.8h, v3.h[0]\n"
+ "cmp x11, #0x10\n"
+ ".inst 0x4f44f0f9 // bfdot v25.4s, v7.8h, v4.h[0]\n"
+ "ldr q7, [x15, #0x30]\n"
+ ".inst 0x4f40f0ca // bfdot v10.4s, v6.8h, v0.h[0]\n"
+ ".inst 0x4f41f0ce // bfdot v14.4s, v6.8h, v1.h[0]\n"
+ ".inst 0x4f42f0d2 // bfdot v18.4s, v6.8h, v2.h[0]\n"
+ ".inst 0x4f43f0d6 // bfdot v22.4s, v6.8h, v3.h[0]\n"
+ ".inst 0x4f44f0da // bfdot v26.4s, v6.8h, v4.h[0]\n"
+ "ldr q6, [x15, #0x40]\n"
+ ".inst 0x4f40f0eb // bfdot v11.4s, v7.8h, v0.h[0]\n"
+ ".inst 0x4f41f0ef // bfdot v15.4s, v7.8h, v1.h[0]\n"
+ ".inst 0x4f42f0f3 // bfdot v19.4s, v7.8h, v2.h[0]\n"
+ ".inst 0x4f43f0f7 // bfdot v23.4s, v7.8h, v3.h[0]\n"
+ ".inst 0x4f44f0fb // bfdot v27.4s, v7.8h, v4.h[0]\n"
+ "ldr q7, [x15, #0x50]\n"
+ ".inst 0x4f60f0c8 // bfdot v8.4s, v6.8h, v0.h[1]\n"
+ ".inst 0x4f61f0cc // bfdot v12.4s, v6.8h, v1.h[1]\n"
+ ".inst 0x4f62f0d0 // bfdot v16.4s, v6.8h, v2.h[1]\n"
+ ".inst 0x4f63f0d4 // bfdot v20.4s, v6.8h, v3.h[1]\n"
+ ".inst 0x4f64f0d8 // bfdot v24.4s, v6.8h, v4.h[1]\n"
+ "ldr q6, [x15, #0x60]\n"
+ ".inst 0x4f60f0e9 // bfdot v9.4s, v7.8h, v0.h[1]\n"
+ ".inst 0x4f61f0ed // bfdot v13.4s, v7.8h, v1.h[1]\n"
+ ".inst 0x4f62f0f1 // bfdot v17.4s, v7.8h, v2.h[1]\n"
+ ".inst 0x4f63f0f5 // bfdot v21.4s, v7.8h, v3.h[1]\n"
+ ".inst 0x4f64f0f9 // bfdot v25.4s, v7.8h, v4.h[1]\n"
+ "ldr q7, [x15, #0x70]\n"
+ ".inst 0x4f60f0ca // bfdot v10.4s, v6.8h, v0.h[1]\n"
+ ".inst 0x4f61f0ce // bfdot v14.4s, v6.8h, v1.h[1]\n"
+ ".inst 0x4f62f0d2 // bfdot v18.4s, v6.8h, v2.h[1]\n"
+ ".inst 0x4f63f0d6 // bfdot v22.4s, v6.8h, v3.h[1]\n"
+ ".inst 0x4f64f0da // bfdot v26.4s, v6.8h, v4.h[1]\n"
+ "ldr q6, [x15, #0x80]\n"
+ ".inst 0x4f60f0eb // bfdot v11.4s, v7.8h, v0.h[1]\n"
+ ".inst 0x4f61f0ef // bfdot v15.4s, v7.8h, v1.h[1]\n"
+ ".inst 0x4f62f0f3 // bfdot v19.4s, v7.8h, v2.h[1]\n"
+ ".inst 0x4f63f0f7 // bfdot v23.4s, v7.8h, v3.h[1]\n"
+ ".inst 0x4f64f0fb // bfdot v27.4s, v7.8h, v4.h[1]\n"
+ "ldr q7, [x15, #0x90]\n"
+ ".inst 0x4f40f8c8 // bfdot v8.4s, v6.8h, v0.h[2]\n"
+ ".inst 0x4f41f8cc // bfdot v12.4s, v6.8h, v1.h[2]\n"
+ ".inst 0x4f42f8d0 // bfdot v16.4s, v6.8h, v2.h[2]\n"
+ ".inst 0x4f43f8d4 // bfdot v20.4s, v6.8h, v3.h[2]\n"
+ ".inst 0x4f44f8d8 // bfdot v24.4s, v6.8h, v4.h[2]\n"
+ "ldr q6, [x15, #0xa0]\n"
+ ".inst 0x4f40f8e9 // bfdot v9.4s, v7.8h, v0.h[2]\n"
+ ".inst 0x4f41f8ed // bfdot v13.4s, v7.8h, v1.h[2]\n"
+ ".inst 0x4f42f8f1 // bfdot v17.4s, v7.8h, v2.h[2]\n"
+ ".inst 0x4f43f8f5 // bfdot v21.4s, v7.8h, v3.h[2]\n"
+ ".inst 0x4f44f8f9 // bfdot v25.4s, v7.8h, v4.h[2]\n"
+ "ldr q7, [x15, #0xb0]\n"
+ ".inst 0x4f40f8ca // bfdot v10.4s, v6.8h, v0.h[2]\n"
+ ".inst 0x4f41f8ce // bfdot v14.4s, v6.8h, v1.h[2]\n"
+ ".inst 0x4f42f8d2 // bfdot v18.4s, v6.8h, v2.h[2]\n"
+ ".inst 0x4f43f8d6 // bfdot v22.4s, v6.8h, v3.h[2]\n"
+ ".inst 0x4f44f8da // bfdot v26.4s, v6.8h, v4.h[2]\n"
+ "ldr q6, [x15, #0xc0]\n"
+ ".inst 0x4f40f8eb // bfdot v11.4s, v7.8h, v0.h[2]\n"
+ ".inst 0x4f41f8ef // bfdot v15.4s, v7.8h, v1.h[2]\n"
+ ".inst 0x4f42f8f3 // bfdot v19.4s, v7.8h, v2.h[2]\n"
+ ".inst 0x4f43f8f7 // bfdot v23.4s, v7.8h, v3.h[2]\n"
+ ".inst 0x4f44f8fb // bfdot v27.4s, v7.8h, v4.h[2]\n"
+ "ldr q7, [x15, #0xd0]\n"
+ ".inst 0x4f60f8c8 // bfdot v8.4s, v6.8h, v0.h[3]\n"
+ ".inst 0x4f61f8cc // bfdot v12.4s, v6.8h, v1.h[3]\n"
+ ".inst 0x4f62f8d0 // bfdot v16.4s, v6.8h, v2.h[3]\n"
+ ".inst 0x4f63f8d4 // bfdot v20.4s, v6.8h, v3.h[3]\n"
+ ".inst 0x4f64f8d8 // bfdot v24.4s, v6.8h, v4.h[3]\n"
+ "ldr q6, [x15, #0xe0]\n"
+ ".inst 0x4f60f8e9 // bfdot v9.4s, v7.8h, v0.h[3]\n"
+ ".inst 0x4f61f8ed // bfdot v13.4s, v7.8h, v1.h[3]\n"
+ ".inst 0x4f62f8f1 // bfdot v17.4s, v7.8h, v2.h[3]\n"
+ ".inst 0x4f63f8f5 // bfdot v21.4s, v7.8h, v3.h[3]\n"
+ ".inst 0x4f64f8f9 // bfdot v25.4s, v7.8h, v4.h[3]\n"
+ "ldr q7, [x15, #0xf0]\n"
+ ".inst 0x4f60f8ca // bfdot v10.4s, v6.8h, v0.h[3]\n"
+ "add x15, x15, #0x100\n"
+ ".inst 0x4f61f8ce // bfdot v14.4s, v6.8h, v1.h[3]\n"
+ ".inst 0x4f62f8d2 // bfdot v18.4s, v6.8h, v2.h[3]\n"
+ ".inst 0x4f63f8d6 // bfdot v22.4s, v6.8h, v3.h[3]\n"
+ ".inst 0x4f64f8da // bfdot v26.4s, v6.8h, v4.h[3]\n"
+ ".inst 0x4f60f8eb // bfdot v11.4s, v7.8h, v0.h[3]\n"
+ ".inst 0x4f61f8ef // bfdot v15.4s, v7.8h, v1.h[3]\n"
+ ".inst 0x4f62f8f3 // bfdot v19.4s, v7.8h, v2.h[3]\n"
+ ".inst 0x4f63f8f7 // bfdot v23.4s, v7.8h, v3.h[3]\n"
+ ".inst 0x4f64f8fb // bfdot v27.4s, v7.8h, v4.h[3]\n"
+ "bge 167b\n"
+ "168:" // Height 5: Multiply loop: Single iteration only
+ "sub x11, x11, #0x8\n"
+ "ldr q0, [x10, #0x0]\n"
+ "ldr q1, [x28, #0x0]\n"
+ "ldr q2, [x26, #0x0]\n"
+ "ldr q3, [x24, #0x0]\n"
+ "ldr q4, [x22, #0x0]\n"
+ "ldr q6, [x15, #0x0]\n"
+ ".inst 0x4f40f0c8 // bfdot v8.4s, v6.8h, v0.h[0]\n"
+ "ldr q7, [x15, #0x10]\n"
+ ".inst 0x4f41f0cc // bfdot v12.4s, v6.8h, v1.h[0]\n"
+ "add x10, x10, #0x10\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ ".inst 0x4f42f0d0 // bfdot v16.4s, v6.8h, v2.h[0]\n"
+ "add x28, x28, #0x10\n"
+ ".inst 0x4f43f0d4 // bfdot v20.4s, v6.8h, v3.h[0]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "add x26, x26, #0x10\n"
+ ".inst 0x4f44f0d8 // bfdot v24.4s, v6.8h, v4.h[0]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "ldr q6, [x15, #0x20]\n"
+ ".inst 0x4f40f0e9 // bfdot v9.4s, v7.8h, v0.h[0]\n"
+ "add x24, x24, #0x10\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ ".inst 0x4f41f0ed // bfdot v13.4s, v7.8h, v1.h[0]\n"
+ "add x22, x22, #0x10\n"
+ ".inst 0x4f42f0f1 // bfdot v17.4s, v7.8h, v2.h[0]\n"
+ "prfm pldl1keep, [x22, #0x80]\n"
+ ".inst 0x4f43f0f5 // bfdot v21.4s, v7.8h, v3.h[0]\n"
+ ".inst 0x4f44f0f9 // bfdot v25.4s, v7.8h, v4.h[0]\n"
+ "ldr q7, [x15, #0x30]\n"
+ ".inst 0x4f40f0ca // bfdot v10.4s, v6.8h, v0.h[0]\n"
+ ".inst 0x4f41f0ce // bfdot v14.4s, v6.8h, v1.h[0]\n"
+ ".inst 0x4f42f0d2 // bfdot v18.4s, v6.8h, v2.h[0]\n"
+ ".inst 0x4f43f0d6 // bfdot v22.4s, v6.8h, v3.h[0]\n"
+ ".inst 0x4f44f0da // bfdot v26.4s, v6.8h, v4.h[0]\n"
+ "ldr q6, [x15, #0x40]\n"
+ ".inst 0x4f40f0eb // bfdot v11.4s, v7.8h, v0.h[0]\n"
+ ".inst 0x4f41f0ef // bfdot v15.4s, v7.8h, v1.h[0]\n"
+ ".inst 0x4f42f0f3 // bfdot v19.4s, v7.8h, v2.h[0]\n"
+ ".inst 0x4f43f0f7 // bfdot v23.4s, v7.8h, v3.h[0]\n"
+ ".inst 0x4f44f0fb // bfdot v27.4s, v7.8h, v4.h[0]\n"
+ "ldr q7, [x15, #0x50]\n"
+ ".inst 0x4f60f0c8 // bfdot v8.4s, v6.8h, v0.h[1]\n"
+ ".inst 0x4f61f0cc // bfdot v12.4s, v6.8h, v1.h[1]\n"
+ ".inst 0x4f62f0d0 // bfdot v16.4s, v6.8h, v2.h[1]\n"
+ ".inst 0x4f63f0d4 // bfdot v20.4s, v6.8h, v3.h[1]\n"
+ ".inst 0x4f64f0d8 // bfdot v24.4s, v6.8h, v4.h[1]\n"
+ "ldr q6, [x15, #0x60]\n"
+ ".inst 0x4f60f0e9 // bfdot v9.4s, v7.8h, v0.h[1]\n"
+ ".inst 0x4f61f0ed // bfdot v13.4s, v7.8h, v1.h[1]\n"
+ ".inst 0x4f62f0f1 // bfdot v17.4s, v7.8h, v2.h[1]\n"
+ ".inst 0x4f63f0f5 // bfdot v21.4s, v7.8h, v3.h[1]\n"
+ ".inst 0x4f64f0f9 // bfdot v25.4s, v7.8h, v4.h[1]\n"
+ "ldr q7, [x15, #0x70]\n"
+ ".inst 0x4f60f0ca // bfdot v10.4s, v6.8h, v0.h[1]\n"
+ ".inst 0x4f61f0ce // bfdot v14.4s, v6.8h, v1.h[1]\n"
+ ".inst 0x4f62f0d2 // bfdot v18.4s, v6.8h, v2.h[1]\n"
+ ".inst 0x4f63f0d6 // bfdot v22.4s, v6.8h, v3.h[1]\n"
+ ".inst 0x4f64f0da // bfdot v26.4s, v6.8h, v4.h[1]\n"
+ "ldr q6, [x15, #0x80]\n"
+ ".inst 0x4f60f0eb // bfdot v11.4s, v7.8h, v0.h[1]\n"
+ ".inst 0x4f61f0ef // bfdot v15.4s, v7.8h, v1.h[1]\n"
+ ".inst 0x4f62f0f3 // bfdot v19.4s, v7.8h, v2.h[1]\n"
+ ".inst 0x4f63f0f7 // bfdot v23.4s, v7.8h, v3.h[1]\n"
+ ".inst 0x4f64f0fb // bfdot v27.4s, v7.8h, v4.h[1]\n"
+ "ldr q7, [x15, #0x90]\n"
+ ".inst 0x4f40f8c8 // bfdot v8.4s, v6.8h, v0.h[2]\n"
+ ".inst 0x4f41f8cc // bfdot v12.4s, v6.8h, v1.h[2]\n"
+ ".inst 0x4f42f8d0 // bfdot v16.4s, v6.8h, v2.h[2]\n"
+ ".inst 0x4f43f8d4 // bfdot v20.4s, v6.8h, v3.h[2]\n"
+ ".inst 0x4f44f8d8 // bfdot v24.4s, v6.8h, v4.h[2]\n"
+ "ldr q6, [x15, #0xa0]\n"
+ ".inst 0x4f40f8e9 // bfdot v9.4s, v7.8h, v0.h[2]\n"
+ ".inst 0x4f41f8ed // bfdot v13.4s, v7.8h, v1.h[2]\n"
+ ".inst 0x4f42f8f1 // bfdot v17.4s, v7.8h, v2.h[2]\n"
+ ".inst 0x4f43f8f5 // bfdot v21.4s, v7.8h, v3.h[2]\n"
+ ".inst 0x4f44f8f9 // bfdot v25.4s, v7.8h, v4.h[2]\n"
+ "ldr q7, [x15, #0xb0]\n"
+ ".inst 0x4f40f8ca // bfdot v10.4s, v6.8h, v0.h[2]\n"
+ ".inst 0x4f41f8ce // bfdot v14.4s, v6.8h, v1.h[2]\n"
+ ".inst 0x4f42f8d2 // bfdot v18.4s, v6.8h, v2.h[2]\n"
+ ".inst 0x4f43f8d6 // bfdot v22.4s, v6.8h, v3.h[2]\n"
+ ".inst 0x4f44f8da // bfdot v26.4s, v6.8h, v4.h[2]\n"
+ "ldr q6, [x15, #0xc0]\n"
+ ".inst 0x4f40f8eb // bfdot v11.4s, v7.8h, v0.h[2]\n"
+ ".inst 0x4f41f8ef // bfdot v15.4s, v7.8h, v1.h[2]\n"
+ ".inst 0x4f42f8f3 // bfdot v19.4s, v7.8h, v2.h[2]\n"
+ ".inst 0x4f43f8f7 // bfdot v23.4s, v7.8h, v3.h[2]\n"
+ ".inst 0x4f44f8fb // bfdot v27.4s, v7.8h, v4.h[2]\n"
+ "ldr q7, [x15, #0xd0]\n"
+ ".inst 0x4f60f8c8 // bfdot v8.4s, v6.8h, v0.h[3]\n"
+ ".inst 0x4f61f8cc // bfdot v12.4s, v6.8h, v1.h[3]\n"
+ ".inst 0x4f62f8d0 // bfdot v16.4s, v6.8h, v2.h[3]\n"
+ ".inst 0x4f63f8d4 // bfdot v20.4s, v6.8h, v3.h[3]\n"
+ ".inst 0x4f64f8d8 // bfdot v24.4s, v6.8h, v4.h[3]\n"
+ "ldr q6, [x15, #0xe0]\n"
+ ".inst 0x4f60f8e9 // bfdot v9.4s, v7.8h, v0.h[3]\n"
+ ".inst 0x4f61f8ed // bfdot v13.4s, v7.8h, v1.h[3]\n"
+ ".inst 0x4f62f8f1 // bfdot v17.4s, v7.8h, v2.h[3]\n"
+ ".inst 0x4f63f8f5 // bfdot v21.4s, v7.8h, v3.h[3]\n"
+ ".inst 0x4f64f8f9 // bfdot v25.4s, v7.8h, v4.h[3]\n"
+ "ldr q7, [x15, #0xf0]\n"
+ ".inst 0x4f60f8ca // bfdot v10.4s, v6.8h, v0.h[3]\n"
+ "add x15, x15, #0x100\n"
+ ".inst 0x4f61f8ce // bfdot v14.4s, v6.8h, v1.h[3]\n"
+ ".inst 0x4f62f8d2 // bfdot v18.4s, v6.8h, v2.h[3]\n"
+ ".inst 0x4f63f8d6 // bfdot v22.4s, v6.8h, v3.h[3]\n"
+ ".inst 0x4f64f8da // bfdot v26.4s, v6.8h, v4.h[3]\n"
+ ".inst 0x4f60f8eb // bfdot v11.4s, v7.8h, v0.h[3]\n"
+ ".inst 0x4f61f8ef // bfdot v15.4s, v7.8h, v1.h[3]\n"
+ ".inst 0x4f62f8f3 // bfdot v19.4s, v7.8h, v2.h[3]\n"
+ ".inst 0x4f63f8f7 // bfdot v23.4s, v7.8h, v3.h[3]\n"
+ ".inst 0x4f64f8fb // bfdot v27.4s, v7.8h, v4.h[3]\n"
+ "169:" // Height 5: Multiply loop: Main loop skip
+ "cbz x11, 174f\n"
+ "cmp x11, #0x2\n"
+ "blt 171f\n"
+ "170:" // Height 5: Multiply loop: Odd block loop
+ "ldr s0, [x10], #0x4\n"
+ "ldr s1, [x28], #0x4\n"
+ "ldr s2, [x26], #0x4\n"
+ "ldr s3, [x24], #0x4\n"
+ "ldr s4, [x22], #0x4\n"
+ "ldr q6, [x15, #0x0]\n"
+ ".inst 0x4f40f0c8 // bfdot v8.4s, v6.8h, v0.h[0]\n"
+ "ldr q7, [x15, #0x10]\n"
+ ".inst 0x4f41f0cc // bfdot v12.4s, v6.8h, v1.h[0]\n"
+ "sub x11, x11, #0x2\n"
+ ".inst 0x4f42f0d0 // bfdot v16.4s, v6.8h, v2.h[0]\n"
+ "cmp x11, #0x2\n"
+ ".inst 0x4f43f0d4 // bfdot v20.4s, v6.8h, v3.h[0]\n"
+ ".inst 0x4f44f0d8 // bfdot v24.4s, v6.8h, v4.h[0]\n"
+ "ldr q6, [x15, #0x20]\n"
+ ".inst 0x4f40f0e9 // bfdot v9.4s, v7.8h, v0.h[0]\n"
+ ".inst 0x4f41f0ed // bfdot v13.4s, v7.8h, v1.h[0]\n"
+ ".inst 0x4f42f0f1 // bfdot v17.4s, v7.8h, v2.h[0]\n"
+ ".inst 0x4f43f0f5 // bfdot v21.4s, v7.8h, v3.h[0]\n"
+ ".inst 0x4f44f0f9 // bfdot v25.4s, v7.8h, v4.h[0]\n"
+ "ldr q7, [x15, #0x30]\n"
+ ".inst 0x4f40f0ca // bfdot v10.4s, v6.8h, v0.h[0]\n"
+ "add x15, x15, #0x40\n"
+ ".inst 0x4f41f0ce // bfdot v14.4s, v6.8h, v1.h[0]\n"
+ ".inst 0x4f42f0d2 // bfdot v18.4s, v6.8h, v2.h[0]\n"
+ ".inst 0x4f43f0d6 // bfdot v22.4s, v6.8h, v3.h[0]\n"
+ ".inst 0x4f44f0da // bfdot v26.4s, v6.8h, v4.h[0]\n"
+ ".inst 0x4f40f0eb // bfdot v11.4s, v7.8h, v0.h[0]\n"
+ ".inst 0x4f41f0ef // bfdot v15.4s, v7.8h, v1.h[0]\n"
+ ".inst 0x4f42f0f3 // bfdot v19.4s, v7.8h, v2.h[0]\n"
+ ".inst 0x4f43f0f7 // bfdot v23.4s, v7.8h, v3.h[0]\n"
+ ".inst 0x4f44f0fb // bfdot v27.4s, v7.8h, v4.h[0]\n"
+ "bge 170b\n"
+ "cbz x11, 174f\n"
+ "171:" // Height 5: Multiply loop: Skip odd blocks
+ "tbz x11, #1, 172f\n"
+ "ldr s0, [x10], #0x4\n"
+ "ldr s1, [x28], #0x4\n"
+ "ldr s2, [x26], #0x4\n"
+ "ldr s3, [x24], #0x4\n"
+ "ldr s4, [x22], #0x4\n"
+ "tbz x11, #0, 173f\n"
+ "ld1 { v0.h }[2], [x10]\n"
+ "ld1 { v1.h }[2], [x28]\n"
+ "ld1 { v2.h }[2], [x26]\n"
+ "ld1 { v3.h }[2], [x24]\n"
+ "ld1 { v4.h }[2], [x22]\n"
+ "b 173f\n"
+ "172:" // Height 5: Multiply loop: Ragged operand read: partial_1_0
+ "ldr h0, [x10, #0x0]\n"
+ "ldr h1, [x28, #0x0]\n"
+ "ldr h2, [x26, #0x0]\n"
+ "ldr h3, [x24, #0x0]\n"
+ "ldr h4, [x22, #0x0]\n"
+ "173:" // Height 5: Multiply loop: Ragged operand read: Done
+ "ldr q6, [x15, #0x0]\n"
+ ".inst 0x4f40f0c8 // bfdot v8.4s, v6.8h, v0.h[0]\n"
+ "ldr q7, [x15, #0x10]\n"
+ ".inst 0x4f41f0cc // bfdot v12.4s, v6.8h, v1.h[0]\n"
+ ".inst 0x4f42f0d0 // bfdot v16.4s, v6.8h, v2.h[0]\n"
+ ".inst 0x4f43f0d4 // bfdot v20.4s, v6.8h, v3.h[0]\n"
+ ".inst 0x4f44f0d8 // bfdot v24.4s, v6.8h, v4.h[0]\n"
+ "ldr q6, [x15, #0x20]\n"
+ ".inst 0x4f40f0e9 // bfdot v9.4s, v7.8h, v0.h[0]\n"
+ ".inst 0x4f41f0ed // bfdot v13.4s, v7.8h, v1.h[0]\n"
+ ".inst 0x4f42f0f1 // bfdot v17.4s, v7.8h, v2.h[0]\n"
+ ".inst 0x4f43f0f5 // bfdot v21.4s, v7.8h, v3.h[0]\n"
+ ".inst 0x4f44f0f9 // bfdot v25.4s, v7.8h, v4.h[0]\n"
+ "ldr q7, [x15, #0x30]\n"
+ ".inst 0x4f40f0ca // bfdot v10.4s, v6.8h, v0.h[0]\n"
+ "add x15, x15, #0x40\n"
+ ".inst 0x4f41f0ce // bfdot v14.4s, v6.8h, v1.h[0]\n"
+ ".inst 0x4f42f0d2 // bfdot v18.4s, v6.8h, v2.h[0]\n"
+ ".inst 0x4f43f0d6 // bfdot v22.4s, v6.8h, v3.h[0]\n"
+ ".inst 0x4f44f0da // bfdot v26.4s, v6.8h, v4.h[0]\n"
+ ".inst 0x4f40f0eb // bfdot v11.4s, v7.8h, v0.h[0]\n"
+ ".inst 0x4f41f0ef // bfdot v15.4s, v7.8h, v1.h[0]\n"
+ ".inst 0x4f42f0f3 // bfdot v19.4s, v7.8h, v2.h[0]\n"
+ ".inst 0x4f43f0f7 // bfdot v23.4s, v7.8h, v3.h[0]\n"
+ ".inst 0x4f44f0fb // bfdot v27.4s, v7.8h, v4.h[0]\n"
+ "174:" // Height 5: Multiply loop: No odd multiplies
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "add x12, x12, #0x1\n"
+ "cmp x12, x19\n"
+ "bne 164b\n"
+ "prfm pstl1keep, [x13, #0x0]\n"
+ "prfm pstl1keep, [x9, #0x0]\n"
+ "prfm pstl1keep, [x27, #0x0]\n"
+ "prfm pstl1keep, [x25, #0x0]\n"
+ "prfm pstl1keep, [x23, #0x0]\n"
+ "tbz %x[flags], #1, 175f\n"
+ "add x19, %x[args_ptr], %[offset_min]\n"
+ "ld1r { v1.4s }, [x19]\n"
+ "add x19, %x[args_ptr], %[offset_max]\n"
+ "ld1r { v0.4s }, [x19]\n"
+ "fmin v8.4s, v8.4s, v0.4s\n"
+ "fmin v9.4s, v9.4s, v0.4s\n"
+ "fmin v10.4s, v10.4s, v0.4s\n"
+ "fmin v11.4s, v11.4s, v0.4s\n"
+ "fmax v8.4s, v8.4s, v1.4s\n"
+ "fmax v9.4s, v9.4s, v1.4s\n"
+ "fmax v10.4s, v10.4s, v1.4s\n"
+ "fmax v11.4s, v11.4s, v1.4s\n"
+ "fmin v12.4s, v12.4s, v0.4s\n"
+ "fmin v13.4s, v13.4s, v0.4s\n"
+ "fmin v14.4s, v14.4s, v0.4s\n"
+ "fmax v12.4s, v12.4s, v1.4s\n"
+ "fmax v13.4s, v13.4s, v1.4s\n"
+ "fmax v14.4s, v14.4s, v1.4s\n"
+ "fmin v15.4s, v15.4s, v0.4s\n"
+ "fmin v16.4s, v16.4s, v0.4s\n"
+ "fmin v17.4s, v17.4s, v0.4s\n"
+ "fmax v15.4s, v15.4s, v1.4s\n"
+ "fmax v16.4s, v16.4s, v1.4s\n"
+ "fmax v17.4s, v17.4s, v1.4s\n"
+ "fmin v18.4s, v18.4s, v0.4s\n"
+ "fmin v19.4s, v19.4s, v0.4s\n"
+ "fmin v20.4s, v20.4s, v0.4s\n"
+ "fmax v18.4s, v18.4s, v1.4s\n"
+ "fmax v19.4s, v19.4s, v1.4s\n"
+ "fmax v20.4s, v20.4s, v1.4s\n"
+ "fmin v21.4s, v21.4s, v0.4s\n"
+ "fmin v22.4s, v22.4s, v0.4s\n"
+ "fmin v23.4s, v23.4s, v0.4s\n"
+ "fmax v21.4s, v21.4s, v1.4s\n"
+ "fmax v22.4s, v22.4s, v1.4s\n"
+ "fmax v23.4s, v23.4s, v1.4s\n"
+ "fmin v24.4s, v24.4s, v0.4s\n"
+ "fmin v25.4s, v25.4s, v0.4s\n"
+ "fmin v26.4s, v26.4s, v0.4s\n"
+ "fmax v24.4s, v24.4s, v1.4s\n"
+ "fmax v25.4s, v25.4s, v1.4s\n"
+ "fmax v26.4s, v26.4s, v1.4s\n"
+ "fmin v27.4s, v27.4s, v0.4s\n"
+ "fmax v27.4s, v27.4s, v1.4s\n"
+ "175:" // Height 5: No activation
+ "cmp x16, #0x10\n"
+ "bge 184f\n"
+ "tbz x16, #3, 179f\n"
+ "st1 { v8.4s }, [x13], #0x10\n"
+ "st1 { v9.4s }, [x13], #0x10\n"
+ "st1 { v12.4s }, [x9], #0x10\n"
+ "st1 { v13.4s }, [x9], #0x10\n"
+ "st1 { v16.4s }, [x27], #0x10\n"
+ "st1 { v17.4s }, [x27], #0x10\n"
+ "st1 { v20.4s }, [x25], #0x10\n"
+ "st1 { v21.4s }, [x25], #0x10\n"
+ "st1 { v24.4s }, [x23], #0x10\n"
+ "st1 { v25.4s }, [x23], #0x10\n"
+ "tbz x16, #2, 177f\n"
+ "st1 { v10.4s }, [x13], #0x10\n"
+ "st1 { v14.4s }, [x9], #0x10\n"
+ "st1 { v18.4s }, [x27], #0x10\n"
+ "st1 { v22.4s }, [x25], #0x10\n"
+ "st1 { v26.4s }, [x23], #0x10\n"
+ "tbz x16, #1, 176f\n"
+ "str d11, [x13], #0x8\n"
+ "str d15, [x9], #0x8\n"
+ "str d19, [x27], #0x8\n"
+ "str d23, [x25], #0x8\n"
+ "str d27, [x23], #0x8\n"
+ "tbz x16, #0, 183f\n"
+ "st1 { v11.s }[2], [x13]\n"
+ "st1 { v15.s }[2], [x9]\n"
+ "st1 { v19.s }[2], [x27]\n"
+ "st1 { v23.s }[2], [x25]\n"
+ "st1 { v27.s }[2], [x23]\n"
+ "b 183f\n"
+ "176:" // Height 5: Partial direct writeback: partial_1_12
+ "tbz x16, #0, 183f\n"
+ "str s11, [x13, #0x0]\n"
+ "str s15, [x9, #0x0]\n"
+ "str s19, [x27, #0x0]\n"
+ "str s23, [x25, #0x0]\n"
+ "str s27, [x23, #0x0]\n"
+ "b 183f\n"
+ "177:" // Height 5: Partial direct writeback: partial_2_8
+ "tbz x16, #1, 178f\n"
+ "str d10, [x13], #0x8\n"
+ "str d14, [x9], #0x8\n"
+ "str d18, [x27], #0x8\n"
+ "str d22, [x25], #0x8\n"
+ "str d26, [x23], #0x8\n"
+ "tbz x16, #0, 183f\n"
+ "st1 { v10.s }[2], [x13]\n"
+ "st1 { v14.s }[2], [x9]\n"
+ "st1 { v18.s }[2], [x27]\n"
+ "st1 { v22.s }[2], [x25]\n"
+ "st1 { v26.s }[2], [x23]\n"
+ "b 183f\n"
+ "178:" // Height 5: Partial direct writeback: partial_1_8
+ "tbz x16, #0, 183f\n"
+ "str s10, [x13, #0x0]\n"
+ "str s14, [x9, #0x0]\n"
+ "str s18, [x27, #0x0]\n"
+ "str s22, [x25, #0x0]\n"
+ "str s26, [x23, #0x0]\n"
+ "b 183f\n"
+ "179:" // Height 5: Partial direct writeback: partial_4_0
+ "tbz x16, #2, 181f\n"
+ "st1 { v8.4s }, [x13], #0x10\n"
+ "st1 { v12.4s }, [x9], #0x10\n"
+ "st1 { v16.4s }, [x27], #0x10\n"
+ "st1 { v20.4s }, [x25], #0x10\n"
+ "st1 { v24.4s }, [x23], #0x10\n"
+ "tbz x16, #1, 180f\n"
+ "str d9, [x13], #0x8\n"
+ "str d13, [x9], #0x8\n"
+ "str d17, [x27], #0x8\n"
+ "str d21, [x25], #0x8\n"
+ "str d25, [x23], #0x8\n"
+ "tbz x16, #0, 183f\n"
+ "st1 { v9.s }[2], [x13]\n"
+ "st1 { v13.s }[2], [x9]\n"
+ "st1 { v17.s }[2], [x27]\n"
+ "st1 { v21.s }[2], [x25]\n"
+ "st1 { v25.s }[2], [x23]\n"
+ "b 183f\n"
+ "180:" // Height 5: Partial direct writeback: partial_1_4
+ "tbz x16, #0, 183f\n"
+ "str s9, [x13, #0x0]\n"
+ "str s13, [x9, #0x0]\n"
+ "str s17, [x27, #0x0]\n"
+ "str s21, [x25, #0x0]\n"
+ "str s25, [x23, #0x0]\n"
+ "b 183f\n"
+ "181:" // Height 5: Partial direct writeback: partial_2_0
+ "tbz x16, #1, 182f\n"
+ "str d8, [x13], #0x8\n"
+ "str d12, [x9], #0x8\n"
+ "str d16, [x27], #0x8\n"
+ "str d20, [x25], #0x8\n"
+ "str d24, [x23], #0x8\n"
+ "tbz x16, #0, 183f\n"
+ "st1 { v8.s }[2], [x13]\n"
+ "st1 { v12.s }[2], [x9]\n"
+ "st1 { v16.s }[2], [x27]\n"
+ "st1 { v20.s }[2], [x25]\n"
+ "st1 { v24.s }[2], [x23]\n"
+ "b 183f\n"
+ "182:" // Height 5: Partial direct writeback: partial_1_0
+ "str s8, [x13, #0x0]\n"
+ "str s12, [x9, #0x0]\n"
+ "str s16, [x27, #0x0]\n"
+ "str s20, [x25, #0x0]\n"
+ "str s24, [x23, #0x0]\n"
+ "183:" // Height 5: Partial direct writeback: Done
+ "b 185f\n"
+ "184:" // Height 5: Full writeback
+ "str q8, [x13, #0x0]\n"
+ "str q9, [x13, #0x10]\n"
+ "str q10, [x13, #0x20]\n"
+ "str q11, [x13, #0x30]\n"
+ "str q12, [x9, #0x0]\n"
+ "str q13, [x9, #0x10]\n"
+ "str q14, [x9, #0x20]\n"
+ "str q15, [x9, #0x30]\n"
+ "str q16, [x27, #0x0]\n"
+ "str q17, [x27, #0x10]\n"
+ "str q18, [x27, #0x20]\n"
+ "str q19, [x27, #0x30]\n"
+ "str q20, [x25, #0x0]\n"
+ "str q21, [x25, #0x10]\n"
+ "str q22, [x25, #0x20]\n"
+ "str q23, [x25, #0x30]\n"
+ "str q24, [x23, #0x0]\n"
+ "str q25, [x23, #0x10]\n"
+ "str q26, [x23, #0x20]\n"
+ "str q27, [x23, #0x30]\n"
+ "add x13, x13, #0x40\n"
+ "add x9, x9, #0x40\n"
+ "add x27, x27, #0x40\n"
+ "add x25, x25, #0x40\n"
+ "add x23, x23, #0x40\n"
+ "185:" // Height 5: Writeback done
+ "subs x16, x16, #0x10\n"
+ "bgt 151b\n"
+ "b 224f\n"
+ "186:" // Height 6
+ "ldr x16, [%x[args_ptr], %[offsetof_N]]\n"
+ "mov x14, %x[bias]\n"
+ "ldr x15, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 187f\n"
+ "ldr x13, [%x[output_ptr], #0x0]\n"
+ "add x13, x13, x19, LSL #2\n"
+ "ldr x9, [%x[output_ptr], #0x8]\n"
+ "ldr x27, [%x[output_ptr], #0x10]\n"
+ "add x9, x9, x19, LSL #2\n"
+ "ldr x25, [%x[output_ptr], #0x18]\n"
+ "ldr x23, [%x[output_ptr], #0x20]\n"
+ "add x27, x27, x19, LSL #2\n"
+ "ldr x21, [%x[output_ptr], #0x28]\n"
+ "add %x[output_ptr], %x[output_ptr], #0x30\n"
+ "add x25, x25, x19, LSL #2\n"
+ "add x23, x23, x19, LSL #2\n"
+ "add x21, x21, x19, LSL #2\n"
+ "b 188f\n"
+ "187:" // Height 6: setup direct output
+ "mov x13, %x[output_ptr]\n"
+ "add x9, x13, x19, LSL #2\n"
+ "add x27, x9, x19, LSL #2\n"
+ "add x25, x27, x19, LSL #2\n"
+ "add x23, x25, x19, LSL #2\n"
+ "add x21, x23, x19, LSL #2\n"
+ "add %x[output_ptr], x21, x19, LSL #2\n"
+ "188:" // Height 6: Column loop
+ "cbz x14, 189f\n"
+ "ldr q8, [x14, #0x0]\n"
+ "mov v12.16b, v8.16b\n"
+ "ldr q9, [x14, #0x10]\n"
+ "mov v16.16b, v8.16b\n"
+ "ldr q10, [x14, #0x20]\n"
+ "mov v20.16b, v8.16b\n"
+ "ldr q11, [x14, #0x30]\n"
+ "mov v24.16b, v8.16b\n"
+ "add x14, x14, #0x40\n"
+ "mov v28.16b, v8.16b\n"
+ "mov v13.16b, v9.16b\n"
+ "mov v17.16b, v9.16b\n"
+ "mov v14.16b, v10.16b\n"
+ "mov v15.16b, v11.16b\n"
+ "mov v18.16b, v10.16b\n"
+ "mov v19.16b, v11.16b\n"
+ "mov v21.16b, v9.16b\n"
+ "mov v22.16b, v10.16b\n"
+ "mov v23.16b, v11.16b\n"
+ "mov v25.16b, v9.16b\n"
+ "mov v26.16b, v10.16b\n"
+ "mov v27.16b, v11.16b\n"
+ "mov v29.16b, v9.16b\n"
+ "mov v30.16b, v10.16b\n"
+ "mov v31.16b, v11.16b\n"
+ "b 200f\n"
+ "189:" // Height 6: no bias
+ "tbz %x[flags], #0, 199f\n"
+ "cmp x16, #0x10\n"
+ "bge 198f\n"
+ "tbz x16, #3, 193f\n"
+ "ld1 { v8.4s }, [x13], #0x10\n"
+ "ld1 { v12.4s }, [x9], #0x10\n"
+ "ld1 { v16.4s }, [x27], #0x10\n"
+ "ld1 { v20.4s }, [x25], #0x10\n"
+ "ld1 { v24.4s }, [x23], #0x10\n"
+ "ld1 { v28.4s }, [x21], #0x10\n"
+ "ld1 { v9.4s }, [x13], #0x10\n"
+ "ld1 { v13.4s }, [x9], #0x10\n"
+ "ld1 { v17.4s }, [x27], #0x10\n"
+ "ld1 { v21.4s }, [x25], #0x10\n"
+ "ld1 { v25.4s }, [x23], #0x10\n"
+ "ld1 { v29.4s }, [x21], #0x10\n"
+ "tbz x16, #2, 191f\n"
+ "ld1 { v10.4s }, [x13], #0x10\n"
+ "ld1 { v14.4s }, [x9], #0x10\n"
+ "ld1 { v18.4s }, [x27], #0x10\n"
+ "ld1 { v22.4s }, [x25], #0x10\n"
+ "ld1 { v26.4s }, [x23], #0x10\n"
+ "ld1 { v30.4s }, [x21], #0x10\n"
+ "tbz x16, #1, 190f\n"
+ "mov x19, #0x38\n"
+ "ldr d11, [x13], #0x8\n"
+ "ldr d15, [x9], #0x8\n"
+ "ldr d19, [x27], #0x8\n"
+ "ldr d23, [x25], #0x8\n"
+ "ldr d27, [x23], #0x8\n"
+ "ldr d31, [x21], #0x8\n"
+ "tbz x16, #0, 197f\n"
+ "ld1 { v11.s }[2], [x13]\n"
+ "ld1 { v15.s }[2], [x9]\n"
+ "ld1 { v19.s }[2], [x27]\n"
+ "ld1 { v23.s }[2], [x25]\n"
+ "ld1 { v27.s }[2], [x23]\n"
+ "ld1 { v31.s }[2], [x21]\n"
+ "b 197f\n"
+ "190:" // Height 6: Partial accumulate: partial_1_12
+ "mov x19, #0x30\n"
+ "tbz x16, #0, 197f\n"
+ "ldr s11, [x13, #0x0]\n"
+ "ldr s15, [x9, #0x0]\n"
+ "ldr s19, [x27, #0x0]\n"
+ "ldr s23, [x25, #0x0]\n"
+ "ldr s27, [x23, #0x0]\n"
+ "ldr s31, [x21, #0x0]\n"
+ "b 197f\n"
+ "191:" // Height 6: Partial accumulate: partial_2_8
+ "tbz x16, #1, 192f\n"
+ "ldr d10, [x13], #0x8\n"
+ "ldr d14, [x9], #0x8\n"
+ "ldr d18, [x27], #0x8\n"
+ "ldr d22, [x25], #0x8\n"
+ "ldr d26, [x23], #0x8\n"
+ "ldr d30, [x21], #0x8\n"
+ "mov x19, #0x28\n"
+ "tbz x16, #0, 197f\n"
+ "ld1 { v10.s }[2], [x13]\n"
+ "ld1 { v14.s }[2], [x9]\n"
+ "ld1 { v18.s }[2], [x27]\n"
+ "ld1 { v22.s }[2], [x25]\n"
+ "ld1 { v26.s }[2], [x23]\n"
+ "ld1 { v30.s }[2], [x21]\n"
+ "b 197f\n"
+ "192:" // Height 6: Partial accumulate: partial_1_8
+ "mov x19, #0x20\n"
+ "tbz x16, #0, 197f\n"
+ "ldr s10, [x13, #0x0]\n"
+ "ldr s14, [x9, #0x0]\n"
+ "ldr s18, [x27, #0x0]\n"
+ "ldr s22, [x25, #0x0]\n"
+ "ldr s26, [x23, #0x0]\n"
+ "ldr s30, [x21, #0x0]\n"
+ "b 197f\n"
+ "193:" // Height 6: Partial accumulate: partial_4_0
+ "tbz x16, #2, 195f\n"
+ "ld1 { v8.4s }, [x13], #0x10\n"
+ "ld1 { v12.4s }, [x9], #0x10\n"
+ "ld1 { v16.4s }, [x27], #0x10\n"
+ "ld1 { v20.4s }, [x25], #0x10\n"
+ "ld1 { v24.4s }, [x23], #0x10\n"
+ "ld1 { v28.4s }, [x21], #0x10\n"
+ "tbz x16, #1, 194f\n"
+ "mov x19, #0x18\n"
+ "ldr d9, [x13], #0x8\n"
+ "ldr d13, [x9], #0x8\n"
+ "ldr d17, [x27], #0x8\n"
+ "ldr d21, [x25], #0x8\n"
+ "ldr d25, [x23], #0x8\n"
+ "ldr d29, [x21], #0x8\n"
+ "tbz x16, #0, 197f\n"
+ "ld1 { v9.s }[2], [x13]\n"
+ "ld1 { v13.s }[2], [x9]\n"
+ "ld1 { v17.s }[2], [x27]\n"
+ "ld1 { v21.s }[2], [x25]\n"
+ "ld1 { v25.s }[2], [x23]\n"
+ "ld1 { v29.s }[2], [x21]\n"
+ "b 197f\n"
+ "194:" // Height 6: Partial accumulate: partial_1_4
+ "mov x19, #0x10\n"
+ "tbz x16, #0, 197f\n"
+ "ldr s9, [x13, #0x0]\n"
+ "ldr s13, [x9, #0x0]\n"
+ "ldr s17, [x27, #0x0]\n"
+ "ldr s21, [x25, #0x0]\n"
+ "ldr s25, [x23, #0x0]\n"
+ "ldr s29, [x21, #0x0]\n"
+ "b 197f\n"
+ "195:" // Height 6: Partial accumulate: partial_2_0
+ "tbz x16, #1, 196f\n"
+ "ldr d8, [x13], #0x8\n"
+ "ldr d12, [x9], #0x8\n"
+ "ldr d16, [x27], #0x8\n"
+ "ldr d20, [x25], #0x8\n"
+ "ldr d24, [x23], #0x8\n"
+ "ldr d28, [x21], #0x8\n"
+ "mov x19, #0x8\n"
+ "tbz x16, #0, 197f\n"
+ "ld1 { v8.s }[2], [x13]\n"
+ "ld1 { v12.s }[2], [x9]\n"
+ "ld1 { v16.s }[2], [x27]\n"
+ "ld1 { v20.s }[2], [x25]\n"
+ "ld1 { v24.s }[2], [x23]\n"
+ "ld1 { v28.s }[2], [x21]\n"
+ "b 197f\n"
+ "196:" // Height 6: Partial accumulate: partial_1_0
+ "mov x19, #0x0\n"
+ "ldr s8, [x13, #0x0]\n"
+ "ldr s12, [x9, #0x0]\n"
+ "ldr s16, [x27, #0x0]\n"
+ "ldr s20, [x25, #0x0]\n"
+ "ldr s24, [x23, #0x0]\n"
+ "ldr s28, [x21, #0x0]\n"
+ "197:" // Height 6: Partial accumulate: Done
+ "sub x13, x13, x19\n"
+ "sub x9, x9, x19\n"
+ "sub x27, x27, x19\n"
+ "sub x25, x25, x19\n"
+ "sub x23, x23, x19\n"
+ "sub x21, x21, x19\n"
+ "b 200f\n"
+ "198:" // Height 6: full accumulate
+ "ldr q8, [x13, #0x0]\n"
+ "ldr q9, [x13, #0x10]\n"
+ "ldr q10, [x13, #0x20]\n"
+ "ldr q11, [x13, #0x30]\n"
+ "ldr q12, [x9, #0x0]\n"
+ "ldr q13, [x9, #0x10]\n"
+ "ldr q14, [x9, #0x20]\n"
+ "ldr q15, [x9, #0x30]\n"
+ "ldr q16, [x27, #0x0]\n"
+ "ldr q17, [x27, #0x10]\n"
+ "ldr q18, [x27, #0x20]\n"
+ "ldr q19, [x27, #0x30]\n"
+ "ldr q20, [x25, #0x0]\n"
+ "ldr q21, [x25, #0x10]\n"
+ "ldr q22, [x25, #0x20]\n"
+ "ldr q23, [x25, #0x30]\n"
+ "ldr q24, [x23, #0x0]\n"
+ "ldr q25, [x23, #0x10]\n"
+ "ldr q26, [x23, #0x20]\n"
+ "ldr q27, [x23, #0x30]\n"
+ "ldr q28, [x21, #0x0]\n"
+ "ldr q29, [x21, #0x10]\n"
+ "ldr q30, [x21, #0x20]\n"
+ "ldr q31, [x21, #0x30]\n"
+ "b 200f\n"
+ "199:" // Height 6: no accumulate
+ "movi v8.16b, #0x0\n"
+ "movi v9.16b, #0x0\n"
+ "movi v10.16b, #0x0\n"
+ "movi v11.16b, #0x0\n"
+ "movi v12.16b, #0x0\n"
+ "movi v13.16b, #0x0\n"
+ "movi v14.16b, #0x0\n"
+ "movi v15.16b, #0x0\n"
+ "movi v16.16b, #0x0\n"
+ "movi v17.16b, #0x0\n"
+ "movi v18.16b, #0x0\n"
+ "movi v19.16b, #0x0\n"
+ "movi v20.16b, #0x0\n"
+ "movi v21.16b, #0x0\n"
+ "movi v22.16b, #0x0\n"
+ "movi v23.16b, #0x0\n"
+ "movi v24.16b, #0x0\n"
+ "movi v25.16b, #0x0\n"
+ "movi v26.16b, #0x0\n"
+ "movi v27.16b, #0x0\n"
+ "movi v28.16b, #0x0\n"
+ "movi v29.16b, #0x0\n"
+ "movi v30.16b, #0x0\n"
+ "movi v31.16b, #0x0\n"
+ "200:" // Height 6: setup done
+ "mov x12, #0x0\n"
+ "201:" // Height 6: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w11, [x20, x12, LSL #0x2]\n"
+ "tbz %x[flags], #3, 202f\n"
+ "ldr x20, [%x[input_ptr], x12, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x10, [x20, #0x0]\n"
+ "ldr x28, [x20, #0x8]\n"
+ "ldr x26, [x20, #0x10]\n"
+ "ldr x24, [x20, #0x18]\n"
+ "ldr x22, [x20, #0x20]\n"
+ "ldr x20, [x20, #0x28]\n"
+ "cbnz x12, 203f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x10, x10, x19, LSL #1\n"
+ "add x28, x28, x19, LSL #1\n"
+ "add x26, x26, x19, LSL #1\n"
+ "add x24, x24, x19, LSL #1\n"
+ "add x22, x22, x19, LSL #1\n"
+ "add x20, x20, x19, LSL #1\n"
+ "b 203f\n"
+ "202:" // Height 6: setup direct input
+ "mov x10, %x[input_ptr]\n"
+ "add x28, x10, x19, LSL #1\n"
+ "add x26, x28, x19, LSL #1\n"
+ "add x24, x26, x19, LSL #1\n"
+ "add x22, x24, x19, LSL #1\n"
+ "add x20, x22, x19, LSL #1\n"
+ "203:" // Height 6: input setup done
+ "cmp x11, #0x8\n"
+ "blt 206f\n"
+ "cmp x11, #0x10\n"
+ "blt 205f\n"
+ "204:" // Height 6: Multiply loop: Main loop head
+ "ldr q0, [x10, #0x0]\n"
+ "ldr q1, [x28, #0x0]\n"
+ "ldr q2, [x26, #0x0]\n"
+ "ldr q3, [x24, #0x0]\n"
+ "ldr q4, [x22, #0x0]\n"
+ "ldr q5, [x20, #0x0]\n"
+ "ldr q6, [x15, #0x0]\n"
+ ".inst 0x4f40f0c8 // bfdot v8.4s, v6.8h, v0.h[0]\n"
+ "ldr q7, [x15, #0x10]\n"
+ ".inst 0x4f41f0cc // bfdot v12.4s, v6.8h, v1.h[0]\n"
+ "add x10, x10, #0x10\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ ".inst 0x4f42f0d0 // bfdot v16.4s, v6.8h, v2.h[0]\n"
+ "add x28, x28, #0x10\n"
+ ".inst 0x4f43f0d4 // bfdot v20.4s, v6.8h, v3.h[0]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "add x26, x26, #0x10\n"
+ ".inst 0x4f44f0d8 // bfdot v24.4s, v6.8h, v4.h[0]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "add x24, x24, #0x10\n"
+ ".inst 0x4f45f0dc // bfdot v28.4s, v6.8h, v5.h[0]\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "ldr q6, [x15, #0x20]\n"
+ ".inst 0x4f40f0e9 // bfdot v9.4s, v7.8h, v0.h[0]\n"
+ "add x22, x22, #0x10\n"
+ "prfm pldl1keep, [x22, #0x80]\n"
+ ".inst 0x4f41f0ed // bfdot v13.4s, v7.8h, v1.h[0]\n"
+ "add x20, x20, #0x10\n"
+ ".inst 0x4f42f0f1 // bfdot v17.4s, v7.8h, v2.h[0]\n"
+ "prfm pldl1keep, [x20, #0x80]\n"
+ "sub x11, x11, #0x8\n"
+ ".inst 0x4f43f0f5 // bfdot v21.4s, v7.8h, v3.h[0]\n"
+ "cmp x11, #0x10\n"
+ ".inst 0x4f44f0f9 // bfdot v25.4s, v7.8h, v4.h[0]\n"
+ ".inst 0x4f45f0fd // bfdot v29.4s, v7.8h, v5.h[0]\n"
+ "ldr q7, [x15, #0x30]\n"
+ ".inst 0x4f40f0ca // bfdot v10.4s, v6.8h, v0.h[0]\n"
+ ".inst 0x4f41f0ce // bfdot v14.4s, v6.8h, v1.h[0]\n"
+ ".inst 0x4f42f0d2 // bfdot v18.4s, v6.8h, v2.h[0]\n"
+ ".inst 0x4f43f0d6 // bfdot v22.4s, v6.8h, v3.h[0]\n"
+ ".inst 0x4f44f0da // bfdot v26.4s, v6.8h, v4.h[0]\n"
+ ".inst 0x4f45f0de // bfdot v30.4s, v6.8h, v5.h[0]\n"
+ "ldr q6, [x15, #0x40]\n"
+ ".inst 0x4f40f0eb // bfdot v11.4s, v7.8h, v0.h[0]\n"
+ ".inst 0x4f41f0ef // bfdot v15.4s, v7.8h, v1.h[0]\n"
+ ".inst 0x4f42f0f3 // bfdot v19.4s, v7.8h, v2.h[0]\n"
+ ".inst 0x4f43f0f7 // bfdot v23.4s, v7.8h, v3.h[0]\n"
+ ".inst 0x4f44f0fb // bfdot v27.4s, v7.8h, v4.h[0]\n"
+ ".inst 0x4f45f0ff // bfdot v31.4s, v7.8h, v5.h[0]\n"
+ "ldr q7, [x15, #0x50]\n"
+ ".inst 0x4f60f0c8 // bfdot v8.4s, v6.8h, v0.h[1]\n"
+ ".inst 0x4f61f0cc // bfdot v12.4s, v6.8h, v1.h[1]\n"
+ ".inst 0x4f62f0d0 // bfdot v16.4s, v6.8h, v2.h[1]\n"
+ ".inst 0x4f63f0d4 // bfdot v20.4s, v6.8h, v3.h[1]\n"
+ ".inst 0x4f64f0d8 // bfdot v24.4s, v6.8h, v4.h[1]\n"
+ ".inst 0x4f65f0dc // bfdot v28.4s, v6.8h, v5.h[1]\n"
+ "ldr q6, [x15, #0x60]\n"
+ ".inst 0x4f60f0e9 // bfdot v9.4s, v7.8h, v0.h[1]\n"
+ ".inst 0x4f61f0ed // bfdot v13.4s, v7.8h, v1.h[1]\n"
+ ".inst 0x4f62f0f1 // bfdot v17.4s, v7.8h, v2.h[1]\n"
+ ".inst 0x4f63f0f5 // bfdot v21.4s, v7.8h, v3.h[1]\n"
+ ".inst 0x4f64f0f9 // bfdot v25.4s, v7.8h, v4.h[1]\n"
+ ".inst 0x4f65f0fd // bfdot v29.4s, v7.8h, v5.h[1]\n"
+ "ldr q7, [x15, #0x70]\n"
+ ".inst 0x4f60f0ca // bfdot v10.4s, v6.8h, v0.h[1]\n"
+ ".inst 0x4f61f0ce // bfdot v14.4s, v6.8h, v1.h[1]\n"
+ ".inst 0x4f62f0d2 // bfdot v18.4s, v6.8h, v2.h[1]\n"
+ ".inst 0x4f63f0d6 // bfdot v22.4s, v6.8h, v3.h[1]\n"
+ ".inst 0x4f64f0da // bfdot v26.4s, v6.8h, v4.h[1]\n"
+ ".inst 0x4f65f0de // bfdot v30.4s, v6.8h, v5.h[1]\n"
+ "ldr q6, [x15, #0x80]\n"
+ ".inst 0x4f60f0eb // bfdot v11.4s, v7.8h, v0.h[1]\n"
+ ".inst 0x4f61f0ef // bfdot v15.4s, v7.8h, v1.h[1]\n"
+ ".inst 0x4f62f0f3 // bfdot v19.4s, v7.8h, v2.h[1]\n"
+ ".inst 0x4f63f0f7 // bfdot v23.4s, v7.8h, v3.h[1]\n"
+ ".inst 0x4f64f0fb // bfdot v27.4s, v7.8h, v4.h[1]\n"
+ ".inst 0x4f65f0ff // bfdot v31.4s, v7.8h, v5.h[1]\n"
+ "ldr q7, [x15, #0x90]\n"
+ ".inst 0x4f40f8c8 // bfdot v8.4s, v6.8h, v0.h[2]\n"
+ ".inst 0x4f41f8cc // bfdot v12.4s, v6.8h, v1.h[2]\n"
+ ".inst 0x4f42f8d0 // bfdot v16.4s, v6.8h, v2.h[2]\n"
+ ".inst 0x4f43f8d4 // bfdot v20.4s, v6.8h, v3.h[2]\n"
+ ".inst 0x4f44f8d8 // bfdot v24.4s, v6.8h, v4.h[2]\n"
+ ".inst 0x4f45f8dc // bfdot v28.4s, v6.8h, v5.h[2]\n"
+ "ldr q6, [x15, #0xa0]\n"
+ ".inst 0x4f40f8e9 // bfdot v9.4s, v7.8h, v0.h[2]\n"
+ ".inst 0x4f41f8ed // bfdot v13.4s, v7.8h, v1.h[2]\n"
+ ".inst 0x4f42f8f1 // bfdot v17.4s, v7.8h, v2.h[2]\n"
+ ".inst 0x4f43f8f5 // bfdot v21.4s, v7.8h, v3.h[2]\n"
+ ".inst 0x4f44f8f9 // bfdot v25.4s, v7.8h, v4.h[2]\n"
+ ".inst 0x4f45f8fd // bfdot v29.4s, v7.8h, v5.h[2]\n"
+ "ldr q7, [x15, #0xb0]\n"
+ ".inst 0x4f40f8ca // bfdot v10.4s, v6.8h, v0.h[2]\n"
+ ".inst 0x4f41f8ce // bfdot v14.4s, v6.8h, v1.h[2]\n"
+ ".inst 0x4f42f8d2 // bfdot v18.4s, v6.8h, v2.h[2]\n"
+ ".inst 0x4f43f8d6 // bfdot v22.4s, v6.8h, v3.h[2]\n"
+ ".inst 0x4f44f8da // bfdot v26.4s, v6.8h, v4.h[2]\n"
+ ".inst 0x4f45f8de // bfdot v30.4s, v6.8h, v5.h[2]\n"
+ "ldr q6, [x15, #0xc0]\n"
+ ".inst 0x4f40f8eb // bfdot v11.4s, v7.8h, v0.h[2]\n"
+ ".inst 0x4f41f8ef // bfdot v15.4s, v7.8h, v1.h[2]\n"
+ ".inst 0x4f42f8f3 // bfdot v19.4s, v7.8h, v2.h[2]\n"
+ ".inst 0x4f43f8f7 // bfdot v23.4s, v7.8h, v3.h[2]\n"
+ ".inst 0x4f44f8fb // bfdot v27.4s, v7.8h, v4.h[2]\n"
+ ".inst 0x4f45f8ff // bfdot v31.4s, v7.8h, v5.h[2]\n"
+ "ldr q7, [x15, #0xd0]\n"
+ ".inst 0x4f60f8c8 // bfdot v8.4s, v6.8h, v0.h[3]\n"
+ ".inst 0x4f61f8cc // bfdot v12.4s, v6.8h, v1.h[3]\n"
+ ".inst 0x4f62f8d0 // bfdot v16.4s, v6.8h, v2.h[3]\n"
+ ".inst 0x4f63f8d4 // bfdot v20.4s, v6.8h, v3.h[3]\n"
+ ".inst 0x4f64f8d8 // bfdot v24.4s, v6.8h, v4.h[3]\n"
+ ".inst 0x4f65f8dc // bfdot v28.4s, v6.8h, v5.h[3]\n"
+ "ldr q6, [x15, #0xe0]\n"
+ ".inst 0x4f60f8e9 // bfdot v9.4s, v7.8h, v0.h[3]\n"
+ ".inst 0x4f61f8ed // bfdot v13.4s, v7.8h, v1.h[3]\n"
+ ".inst 0x4f62f8f1 // bfdot v17.4s, v7.8h, v2.h[3]\n"
+ ".inst 0x4f63f8f5 // bfdot v21.4s, v7.8h, v3.h[3]\n"
+ ".inst 0x4f64f8f9 // bfdot v25.4s, v7.8h, v4.h[3]\n"
+ ".inst 0x4f65f8fd // bfdot v29.4s, v7.8h, v5.h[3]\n"
+ "ldr q7, [x15, #0xf0]\n"
+ ".inst 0x4f60f8ca // bfdot v10.4s, v6.8h, v0.h[3]\n"
+ "add x15, x15, #0x100\n"
+ ".inst 0x4f61f8ce // bfdot v14.4s, v6.8h, v1.h[3]\n"
+ ".inst 0x4f62f8d2 // bfdot v18.4s, v6.8h, v2.h[3]\n"
+ ".inst 0x4f63f8d6 // bfdot v22.4s, v6.8h, v3.h[3]\n"
+ ".inst 0x4f64f8da // bfdot v26.4s, v6.8h, v4.h[3]\n"
+ ".inst 0x4f65f8de // bfdot v30.4s, v6.8h, v5.h[3]\n"
+ ".inst 0x4f60f8eb // bfdot v11.4s, v7.8h, v0.h[3]\n"
+ ".inst 0x4f61f8ef // bfdot v15.4s, v7.8h, v1.h[3]\n"
+ ".inst 0x4f62f8f3 // bfdot v19.4s, v7.8h, v2.h[3]\n"
+ ".inst 0x4f63f8f7 // bfdot v23.4s, v7.8h, v3.h[3]\n"
+ ".inst 0x4f64f8fb // bfdot v27.4s, v7.8h, v4.h[3]\n"
+ ".inst 0x4f65f8ff // bfdot v31.4s, v7.8h, v5.h[3]\n"
+ "bge 204b\n"
+ "205:" // Height 6: Multiply loop: Single iteration only
+ "sub x11, x11, #0x8\n"
+ "ldr q0, [x10, #0x0]\n"
+ "ldr q1, [x28, #0x0]\n"
+ "ldr q2, [x26, #0x0]\n"
+ "ldr q3, [x24, #0x0]\n"
+ "ldr q4, [x22, #0x0]\n"
+ "ldr q5, [x20, #0x0]\n"
+ "ldr q6, [x15, #0x0]\n"
+ ".inst 0x4f40f0c8 // bfdot v8.4s, v6.8h, v0.h[0]\n"
+ "ldr q7, [x15, #0x10]\n"
+ ".inst 0x4f41f0cc // bfdot v12.4s, v6.8h, v1.h[0]\n"
+ "add x10, x10, #0x10\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ ".inst 0x4f42f0d0 // bfdot v16.4s, v6.8h, v2.h[0]\n"
+ "add x28, x28, #0x10\n"
+ ".inst 0x4f43f0d4 // bfdot v20.4s, v6.8h, v3.h[0]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "add x26, x26, #0x10\n"
+ ".inst 0x4f44f0d8 // bfdot v24.4s, v6.8h, v4.h[0]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "add x24, x24, #0x10\n"
+ ".inst 0x4f45f0dc // bfdot v28.4s, v6.8h, v5.h[0]\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "ldr q6, [x15, #0x20]\n"
+ ".inst 0x4f40f0e9 // bfdot v9.4s, v7.8h, v0.h[0]\n"
+ "add x22, x22, #0x10\n"
+ "prfm pldl1keep, [x22, #0x80]\n"
+ ".inst 0x4f41f0ed // bfdot v13.4s, v7.8h, v1.h[0]\n"
+ "add x20, x20, #0x10\n"
+ ".inst 0x4f42f0f1 // bfdot v17.4s, v7.8h, v2.h[0]\n"
+ "prfm pldl1keep, [x20, #0x80]\n"
+ ".inst 0x4f43f0f5 // bfdot v21.4s, v7.8h, v3.h[0]\n"
+ ".inst 0x4f44f0f9 // bfdot v25.4s, v7.8h, v4.h[0]\n"
+ ".inst 0x4f45f0fd // bfdot v29.4s, v7.8h, v5.h[0]\n"
+ "ldr q7, [x15, #0x30]\n"
+ ".inst 0x4f40f0ca // bfdot v10.4s, v6.8h, v0.h[0]\n"
+ ".inst 0x4f41f0ce // bfdot v14.4s, v6.8h, v1.h[0]\n"
+ ".inst 0x4f42f0d2 // bfdot v18.4s, v6.8h, v2.h[0]\n"
+ ".inst 0x4f43f0d6 // bfdot v22.4s, v6.8h, v3.h[0]\n"
+ ".inst 0x4f44f0da // bfdot v26.4s, v6.8h, v4.h[0]\n"
+ ".inst 0x4f45f0de // bfdot v30.4s, v6.8h, v5.h[0]\n"
+ "ldr q6, [x15, #0x40]\n"
+ ".inst 0x4f40f0eb // bfdot v11.4s, v7.8h, v0.h[0]\n"
+ ".inst 0x4f41f0ef // bfdot v15.4s, v7.8h, v1.h[0]\n"
+ ".inst 0x4f42f0f3 // bfdot v19.4s, v7.8h, v2.h[0]\n"
+ ".inst 0x4f43f0f7 // bfdot v23.4s, v7.8h, v3.h[0]\n"
+ ".inst 0x4f44f0fb // bfdot v27.4s, v7.8h, v4.h[0]\n"
+ ".inst 0x4f45f0ff // bfdot v31.4s, v7.8h, v5.h[0]\n"
+ "ldr q7, [x15, #0x50]\n"
+ ".inst 0x4f60f0c8 // bfdot v8.4s, v6.8h, v0.h[1]\n"
+ ".inst 0x4f61f0cc // bfdot v12.4s, v6.8h, v1.h[1]\n"
+ ".inst 0x4f62f0d0 // bfdot v16.4s, v6.8h, v2.h[1]\n"
+ ".inst 0x4f63f0d4 // bfdot v20.4s, v6.8h, v3.h[1]\n"
+ ".inst 0x4f64f0d8 // bfdot v24.4s, v6.8h, v4.h[1]\n"
+ ".inst 0x4f65f0dc // bfdot v28.4s, v6.8h, v5.h[1]\n"
+ "ldr q6, [x15, #0x60]\n"
+ ".inst 0x4f60f0e9 // bfdot v9.4s, v7.8h, v0.h[1]\n"
+ ".inst 0x4f61f0ed // bfdot v13.4s, v7.8h, v1.h[1]\n"
+ ".inst 0x4f62f0f1 // bfdot v17.4s, v7.8h, v2.h[1]\n"
+ ".inst 0x4f63f0f5 // bfdot v21.4s, v7.8h, v3.h[1]\n"
+ ".inst 0x4f64f0f9 // bfdot v25.4s, v7.8h, v4.h[1]\n"
+ ".inst 0x4f65f0fd // bfdot v29.4s, v7.8h, v5.h[1]\n"
+ "ldr q7, [x15, #0x70]\n"
+ ".inst 0x4f60f0ca // bfdot v10.4s, v6.8h, v0.h[1]\n"
+ ".inst 0x4f61f0ce // bfdot v14.4s, v6.8h, v1.h[1]\n"
+ ".inst 0x4f62f0d2 // bfdot v18.4s, v6.8h, v2.h[1]\n"
+ ".inst 0x4f63f0d6 // bfdot v22.4s, v6.8h, v3.h[1]\n"
+ ".inst 0x4f64f0da // bfdot v26.4s, v6.8h, v4.h[1]\n"
+ ".inst 0x4f65f0de // bfdot v30.4s, v6.8h, v5.h[1]\n"
+ "ldr q6, [x15, #0x80]\n"
+ ".inst 0x4f60f0eb // bfdot v11.4s, v7.8h, v0.h[1]\n"
+ ".inst 0x4f61f0ef // bfdot v15.4s, v7.8h, v1.h[1]\n"
+ ".inst 0x4f62f0f3 // bfdot v19.4s, v7.8h, v2.h[1]\n"
+ ".inst 0x4f63f0f7 // bfdot v23.4s, v7.8h, v3.h[1]\n"
+ ".inst 0x4f64f0fb // bfdot v27.4s, v7.8h, v4.h[1]\n"
+ ".inst 0x4f65f0ff // bfdot v31.4s, v7.8h, v5.h[1]\n"
+ "ldr q7, [x15, #0x90]\n"
+ ".inst 0x4f40f8c8 // bfdot v8.4s, v6.8h, v0.h[2]\n"
+ ".inst 0x4f41f8cc // bfdot v12.4s, v6.8h, v1.h[2]\n"
+ ".inst 0x4f42f8d0 // bfdot v16.4s, v6.8h, v2.h[2]\n"
+ ".inst 0x4f43f8d4 // bfdot v20.4s, v6.8h, v3.h[2]\n"
+ ".inst 0x4f44f8d8 // bfdot v24.4s, v6.8h, v4.h[2]\n"
+ ".inst 0x4f45f8dc // bfdot v28.4s, v6.8h, v5.h[2]\n"
+ "ldr q6, [x15, #0xa0]\n"
+ ".inst 0x4f40f8e9 // bfdot v9.4s, v7.8h, v0.h[2]\n"
+ ".inst 0x4f41f8ed // bfdot v13.4s, v7.8h, v1.h[2]\n"
+ ".inst 0x4f42f8f1 // bfdot v17.4s, v7.8h, v2.h[2]\n"
+ ".inst 0x4f43f8f5 // bfdot v21.4s, v7.8h, v3.h[2]\n"
+ ".inst 0x4f44f8f9 // bfdot v25.4s, v7.8h, v4.h[2]\n"
+ ".inst 0x4f45f8fd // bfdot v29.4s, v7.8h, v5.h[2]\n"
+ "ldr q7, [x15, #0xb0]\n"
+ ".inst 0x4f40f8ca // bfdot v10.4s, v6.8h, v0.h[2]\n"
+ ".inst 0x4f41f8ce // bfdot v14.4s, v6.8h, v1.h[2]\n"
+ ".inst 0x4f42f8d2 // bfdot v18.4s, v6.8h, v2.h[2]\n"
+ ".inst 0x4f43f8d6 // bfdot v22.4s, v6.8h, v3.h[2]\n"
+ ".inst 0x4f44f8da // bfdot v26.4s, v6.8h, v4.h[2]\n"
+ ".inst 0x4f45f8de // bfdot v30.4s, v6.8h, v5.h[2]\n"
+ "ldr q6, [x15, #0xc0]\n"
+ ".inst 0x4f40f8eb // bfdot v11.4s, v7.8h, v0.h[2]\n"
+ ".inst 0x4f41f8ef // bfdot v15.4s, v7.8h, v1.h[2]\n"
+ ".inst 0x4f42f8f3 // bfdot v19.4s, v7.8h, v2.h[2]\n"
+ ".inst 0x4f43f8f7 // bfdot v23.4s, v7.8h, v3.h[2]\n"
+ ".inst 0x4f44f8fb // bfdot v27.4s, v7.8h, v4.h[2]\n"
+ ".inst 0x4f45f8ff // bfdot v31.4s, v7.8h, v5.h[2]\n"
+ "ldr q7, [x15, #0xd0]\n"
+ ".inst 0x4f60f8c8 // bfdot v8.4s, v6.8h, v0.h[3]\n"
+ ".inst 0x4f61f8cc // bfdot v12.4s, v6.8h, v1.h[3]\n"
+ ".inst 0x4f62f8d0 // bfdot v16.4s, v6.8h, v2.h[3]\n"
+ ".inst 0x4f63f8d4 // bfdot v20.4s, v6.8h, v3.h[3]\n"
+ ".inst 0x4f64f8d8 // bfdot v24.4s, v6.8h, v4.h[3]\n"
+ ".inst 0x4f65f8dc // bfdot v28.4s, v6.8h, v5.h[3]\n"
+ "ldr q6, [x15, #0xe0]\n"
+ ".inst 0x4f60f8e9 // bfdot v9.4s, v7.8h, v0.h[3]\n"
+ ".inst 0x4f61f8ed // bfdot v13.4s, v7.8h, v1.h[3]\n"
+ ".inst 0x4f62f8f1 // bfdot v17.4s, v7.8h, v2.h[3]\n"
+ ".inst 0x4f63f8f5 // bfdot v21.4s, v7.8h, v3.h[3]\n"
+ ".inst 0x4f64f8f9 // bfdot v25.4s, v7.8h, v4.h[3]\n"
+ ".inst 0x4f65f8fd // bfdot v29.4s, v7.8h, v5.h[3]\n"
+ "ldr q7, [x15, #0xf0]\n"
+ ".inst 0x4f60f8ca // bfdot v10.4s, v6.8h, v0.h[3]\n"
+ "add x15, x15, #0x100\n"
+ ".inst 0x4f61f8ce // bfdot v14.4s, v6.8h, v1.h[3]\n"
+ ".inst 0x4f62f8d2 // bfdot v18.4s, v6.8h, v2.h[3]\n"
+ ".inst 0x4f63f8d6 // bfdot v22.4s, v6.8h, v3.h[3]\n"
+ ".inst 0x4f64f8da // bfdot v26.4s, v6.8h, v4.h[3]\n"
+ ".inst 0x4f65f8de // bfdot v30.4s, v6.8h, v5.h[3]\n"
+ ".inst 0x4f60f8eb // bfdot v11.4s, v7.8h, v0.h[3]\n"
+ ".inst 0x4f61f8ef // bfdot v15.4s, v7.8h, v1.h[3]\n"
+ ".inst 0x4f62f8f3 // bfdot v19.4s, v7.8h, v2.h[3]\n"
+ ".inst 0x4f63f8f7 // bfdot v23.4s, v7.8h, v3.h[3]\n"
+ ".inst 0x4f64f8fb // bfdot v27.4s, v7.8h, v4.h[3]\n"
+ ".inst 0x4f65f8ff // bfdot v31.4s, v7.8h, v5.h[3]\n"
+ "206:" // Height 6: Multiply loop: Main loop skip
+ "cbz x11, 211f\n"
+ "cmp x11, #0x2\n"
+ "blt 208f\n"
+ "207:" // Height 6: Multiply loop: Odd block loop
+ "ldr s0, [x10], #0x4\n"
+ "ldr s1, [x28], #0x4\n"
+ "ldr s2, [x26], #0x4\n"
+ "ldr s3, [x24], #0x4\n"
+ "ldr s4, [x22], #0x4\n"
+ "ldr s5, [x20], #0x4\n"
+ "ldr q6, [x15, #0x0]\n"
+ ".inst 0x4f40f0c8 // bfdot v8.4s, v6.8h, v0.h[0]\n"
+ "ldr q7, [x15, #0x10]\n"
+ ".inst 0x4f41f0cc // bfdot v12.4s, v6.8h, v1.h[0]\n"
+ "sub x11, x11, #0x2\n"
+ ".inst 0x4f42f0d0 // bfdot v16.4s, v6.8h, v2.h[0]\n"
+ "cmp x11, #0x2\n"
+ ".inst 0x4f43f0d4 // bfdot v20.4s, v6.8h, v3.h[0]\n"
+ ".inst 0x4f44f0d8 // bfdot v24.4s, v6.8h, v4.h[0]\n"
+ ".inst 0x4f45f0dc // bfdot v28.4s, v6.8h, v5.h[0]\n"
+ "ldr q6, [x15, #0x20]\n"
+ ".inst 0x4f40f0e9 // bfdot v9.4s, v7.8h, v0.h[0]\n"
+ ".inst 0x4f41f0ed // bfdot v13.4s, v7.8h, v1.h[0]\n"
+ ".inst 0x4f42f0f1 // bfdot v17.4s, v7.8h, v2.h[0]\n"
+ ".inst 0x4f43f0f5 // bfdot v21.4s, v7.8h, v3.h[0]\n"
+ ".inst 0x4f44f0f9 // bfdot v25.4s, v7.8h, v4.h[0]\n"
+ ".inst 0x4f45f0fd // bfdot v29.4s, v7.8h, v5.h[0]\n"
+ "ldr q7, [x15, #0x30]\n"
+ ".inst 0x4f40f0ca // bfdot v10.4s, v6.8h, v0.h[0]\n"
+ "add x15, x15, #0x40\n"
+ ".inst 0x4f41f0ce // bfdot v14.4s, v6.8h, v1.h[0]\n"
+ ".inst 0x4f42f0d2 // bfdot v18.4s, v6.8h, v2.h[0]\n"
+ ".inst 0x4f43f0d6 // bfdot v22.4s, v6.8h, v3.h[0]\n"
+ ".inst 0x4f44f0da // bfdot v26.4s, v6.8h, v4.h[0]\n"
+ ".inst 0x4f45f0de // bfdot v30.4s, v6.8h, v5.h[0]\n"
+ ".inst 0x4f40f0eb // bfdot v11.4s, v7.8h, v0.h[0]\n"
+ ".inst 0x4f41f0ef // bfdot v15.4s, v7.8h, v1.h[0]\n"
+ ".inst 0x4f42f0f3 // bfdot v19.4s, v7.8h, v2.h[0]\n"
+ ".inst 0x4f43f0f7 // bfdot v23.4s, v7.8h, v3.h[0]\n"
+ ".inst 0x4f44f0fb // bfdot v27.4s, v7.8h, v4.h[0]\n"
+ ".inst 0x4f45f0ff // bfdot v31.4s, v7.8h, v5.h[0]\n"
+ "bge 207b\n"
+ "cbz x11, 211f\n"
+ "208:" // Height 6: Multiply loop: Skip odd blocks
+ "tbz x11, #1, 209f\n"
+ "ldr s0, [x10], #0x4\n"
+ "ldr s1, [x28], #0x4\n"
+ "ldr s2, [x26], #0x4\n"
+ "ldr s3, [x24], #0x4\n"
+ "ldr s4, [x22], #0x4\n"
+ "ldr s5, [x20], #0x4\n"
+ "tbz x11, #0, 210f\n"
+ "ld1 { v0.h }[2], [x10]\n"
+ "ld1 { v1.h }[2], [x28]\n"
+ "ld1 { v2.h }[2], [x26]\n"
+ "ld1 { v3.h }[2], [x24]\n"
+ "ld1 { v4.h }[2], [x22]\n"
+ "ld1 { v5.h }[2], [x20]\n"
+ "b 210f\n"
+ "209:" // Height 6: Multiply loop: Ragged operand read: partial_1_0
+ "ldr h0, [x10, #0x0]\n"
+ "ldr h1, [x28, #0x0]\n"
+ "ldr h2, [x26, #0x0]\n"
+ "ldr h3, [x24, #0x0]\n"
+ "ldr h4, [x22, #0x0]\n"
+ "ldr h5, [x20, #0x0]\n"
+ "210:" // Height 6: Multiply loop: Ragged operand read: Done
+ "ldr q6, [x15, #0x0]\n"
+ ".inst 0x4f40f0c8 // bfdot v8.4s, v6.8h, v0.h[0]\n"
+ "ldr q7, [x15, #0x10]\n"
+ ".inst 0x4f41f0cc // bfdot v12.4s, v6.8h, v1.h[0]\n"
+ ".inst 0x4f42f0d0 // bfdot v16.4s, v6.8h, v2.h[0]\n"
+ ".inst 0x4f43f0d4 // bfdot v20.4s, v6.8h, v3.h[0]\n"
+ ".inst 0x4f44f0d8 // bfdot v24.4s, v6.8h, v4.h[0]\n"
+ ".inst 0x4f45f0dc // bfdot v28.4s, v6.8h, v5.h[0]\n"
+ "ldr q6, [x15, #0x20]\n"
+ ".inst 0x4f40f0e9 // bfdot v9.4s, v7.8h, v0.h[0]\n"
+ ".inst 0x4f41f0ed // bfdot v13.4s, v7.8h, v1.h[0]\n"
+ ".inst 0x4f42f0f1 // bfdot v17.4s, v7.8h, v2.h[0]\n"
+ ".inst 0x4f43f0f5 // bfdot v21.4s, v7.8h, v3.h[0]\n"
+ ".inst 0x4f44f0f9 // bfdot v25.4s, v7.8h, v4.h[0]\n"
+ ".inst 0x4f45f0fd // bfdot v29.4s, v7.8h, v5.h[0]\n"
+ "ldr q7, [x15, #0x30]\n"
+ ".inst 0x4f40f0ca // bfdot v10.4s, v6.8h, v0.h[0]\n"
+ "add x15, x15, #0x40\n"
+ ".inst 0x4f41f0ce // bfdot v14.4s, v6.8h, v1.h[0]\n"
+ ".inst 0x4f42f0d2 // bfdot v18.4s, v6.8h, v2.h[0]\n"
+ ".inst 0x4f43f0d6 // bfdot v22.4s, v6.8h, v3.h[0]\n"
+ ".inst 0x4f44f0da // bfdot v26.4s, v6.8h, v4.h[0]\n"
+ ".inst 0x4f45f0de // bfdot v30.4s, v6.8h, v5.h[0]\n"
+ ".inst 0x4f40f0eb // bfdot v11.4s, v7.8h, v0.h[0]\n"
+ ".inst 0x4f41f0ef // bfdot v15.4s, v7.8h, v1.h[0]\n"
+ ".inst 0x4f42f0f3 // bfdot v19.4s, v7.8h, v2.h[0]\n"
+ ".inst 0x4f43f0f7 // bfdot v23.4s, v7.8h, v3.h[0]\n"
+ ".inst 0x4f44f0fb // bfdot v27.4s, v7.8h, v4.h[0]\n"
+ ".inst 0x4f45f0ff // bfdot v31.4s, v7.8h, v5.h[0]\n"
+ "211:" // Height 6: Multiply loop: No odd multiplies
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "add x12, x12, #0x1\n"
+ "cmp x12, x19\n"
+ "bne 201b\n"
+ "prfm pstl1keep, [x13, #0x0]\n"
+ "prfm pstl1keep, [x9, #0x0]\n"
+ "prfm pstl1keep, [x27, #0x0]\n"
+ "prfm pstl1keep, [x25, #0x0]\n"
+ "prfm pstl1keep, [x23, #0x0]\n"
+ "prfm pstl1keep, [x21, #0x0]\n"
+ "tbz %x[flags], #1, 212f\n"
+ "add x19, %x[args_ptr], %[offset_min]\n"
+ "ld1r { v1.4s }, [x19]\n"
+ "add x19, %x[args_ptr], %[offset_max]\n"
+ "ld1r { v0.4s }, [x19]\n"
+ "fmin v8.4s, v8.4s, v0.4s\n"
+ "fmin v9.4s, v9.4s, v0.4s\n"
+ "fmin v10.4s, v10.4s, v0.4s\n"
+ "fmin v11.4s, v11.4s, v0.4s\n"
+ "fmax v8.4s, v8.4s, v1.4s\n"
+ "fmax v9.4s, v9.4s, v1.4s\n"
+ "fmax v10.4s, v10.4s, v1.4s\n"
+ "fmax v11.4s, v11.4s, v1.4s\n"
+ "fmin v12.4s, v12.4s, v0.4s\n"
+ "fmin v13.4s, v13.4s, v0.4s\n"
+ "fmin v14.4s, v14.4s, v0.4s\n"
+ "fmax v12.4s, v12.4s, v1.4s\n"
+ "fmax v13.4s, v13.4s, v1.4s\n"
+ "fmax v14.4s, v14.4s, v1.4s\n"
+ "fmin v15.4s, v15.4s, v0.4s\n"
+ "fmin v16.4s, v16.4s, v0.4s\n"
+ "fmin v17.4s, v17.4s, v0.4s\n"
+ "fmax v15.4s, v15.4s, v1.4s\n"
+ "fmax v16.4s, v16.4s, v1.4s\n"
+ "fmax v17.4s, v17.4s, v1.4s\n"
+ "fmin v18.4s, v18.4s, v0.4s\n"
+ "fmin v19.4s, v19.4s, v0.4s\n"
+ "fmin v20.4s, v20.4s, v0.4s\n"
+ "fmax v18.4s, v18.4s, v1.4s\n"
+ "fmax v19.4s, v19.4s, v1.4s\n"
+ "fmax v20.4s, v20.4s, v1.4s\n"
+ "fmin v21.4s, v21.4s, v0.4s\n"
+ "fmin v22.4s, v22.4s, v0.4s\n"
+ "fmin v23.4s, v23.4s, v0.4s\n"
+ "fmax v21.4s, v21.4s, v1.4s\n"
+ "fmax v22.4s, v22.4s, v1.4s\n"
+ "fmax v23.4s, v23.4s, v1.4s\n"
+ "fmin v24.4s, v24.4s, v0.4s\n"
+ "fmin v25.4s, v25.4s, v0.4s\n"
+ "fmin v26.4s, v26.4s, v0.4s\n"
+ "fmax v24.4s, v24.4s, v1.4s\n"
+ "fmax v25.4s, v25.4s, v1.4s\n"
+ "fmax v26.4s, v26.4s, v1.4s\n"
+ "fmin v27.4s, v27.4s, v0.4s\n"
+ "fmin v28.4s, v28.4s, v0.4s\n"
+ "fmin v29.4s, v29.4s, v0.4s\n"
+ "fmax v27.4s, v27.4s, v1.4s\n"
+ "fmax v28.4s, v28.4s, v1.4s\n"
+ "fmax v29.4s, v29.4s, v1.4s\n"
+ "fmin v30.4s, v30.4s, v0.4s\n"
+ "fmin v31.4s, v31.4s, v0.4s\n"
+ "fmax v30.4s, v30.4s, v1.4s\n"
+ "fmax v31.4s, v31.4s, v1.4s\n"
+ "212:" // Height 6: No activation
+ "cmp x16, #0x10\n"
+ "bge 221f\n"
+ "tbz x16, #3, 216f\n"
+ "st1 { v8.4s }, [x13], #0x10\n"
+ "st1 { v9.4s }, [x13], #0x10\n"
+ "st1 { v12.4s }, [x9], #0x10\n"
+ "st1 { v13.4s }, [x9], #0x10\n"
+ "st1 { v16.4s }, [x27], #0x10\n"
+ "st1 { v17.4s }, [x27], #0x10\n"
+ "st1 { v20.4s }, [x25], #0x10\n"
+ "st1 { v21.4s }, [x25], #0x10\n"
+ "st1 { v24.4s }, [x23], #0x10\n"
+ "st1 { v25.4s }, [x23], #0x10\n"
+ "st1 { v28.4s }, [x21], #0x10\n"
+ "st1 { v29.4s }, [x21], #0x10\n"
+ "tbz x16, #2, 214f\n"
+ "st1 { v10.4s }, [x13], #0x10\n"
+ "st1 { v14.4s }, [x9], #0x10\n"
+ "st1 { v18.4s }, [x27], #0x10\n"
+ "st1 { v22.4s }, [x25], #0x10\n"
+ "st1 { v26.4s }, [x23], #0x10\n"
+ "st1 { v30.4s }, [x21], #0x10\n"
+ "tbz x16, #1, 213f\n"
+ "str d11, [x13], #0x8\n"
+ "str d15, [x9], #0x8\n"
+ "str d19, [x27], #0x8\n"
+ "str d23, [x25], #0x8\n"
+ "str d27, [x23], #0x8\n"
+ "str d31, [x21], #0x8\n"
+ "tbz x16, #0, 220f\n"
+ "st1 { v11.s }[2], [x13]\n"
+ "st1 { v15.s }[2], [x9]\n"
+ "st1 { v19.s }[2], [x27]\n"
+ "st1 { v23.s }[2], [x25]\n"
+ "st1 { v27.s }[2], [x23]\n"
+ "st1 { v31.s }[2], [x21]\n"
+ "b 220f\n"
+ "213:" // Height 6: Partial direct writeback: partial_1_12
+ "tbz x16, #0, 220f\n"
+ "str s11, [x13, #0x0]\n"
+ "str s15, [x9, #0x0]\n"
+ "str s19, [x27, #0x0]\n"
+ "str s23, [x25, #0x0]\n"
+ "str s27, [x23, #0x0]\n"
+ "str s31, [x21, #0x0]\n"
+ "b 220f\n"
+ "214:" // Height 6: Partial direct writeback: partial_2_8
+ "tbz x16, #1, 215f\n"
+ "str d10, [x13], #0x8\n"
+ "str d14, [x9], #0x8\n"
+ "str d18, [x27], #0x8\n"
+ "str d22, [x25], #0x8\n"
+ "str d26, [x23], #0x8\n"
+ "str d30, [x21], #0x8\n"
+ "tbz x16, #0, 220f\n"
+ "st1 { v10.s }[2], [x13]\n"
+ "st1 { v14.s }[2], [x9]\n"
+ "st1 { v18.s }[2], [x27]\n"
+ "st1 { v22.s }[2], [x25]\n"
+ "st1 { v26.s }[2], [x23]\n"
+ "st1 { v30.s }[2], [x21]\n"
+ "b 220f\n"
+ "215:" // Height 6: Partial direct writeback: partial_1_8
+ "tbz x16, #0, 220f\n"
+ "str s10, [x13, #0x0]\n"
+ "str s14, [x9, #0x0]\n"
+ "str s18, [x27, #0x0]\n"
+ "str s22, [x25, #0x0]\n"
+ "str s26, [x23, #0x0]\n"
+ "str s30, [x21, #0x0]\n"
+ "b 220f\n"
+ "216:" // Height 6: Partial direct writeback: partial_4_0
+ "tbz x16, #2, 218f\n"
+ "st1 { v8.4s }, [x13], #0x10\n"
+ "st1 { v12.4s }, [x9], #0x10\n"
+ "st1 { v16.4s }, [x27], #0x10\n"
+ "st1 { v20.4s }, [x25], #0x10\n"
+ "st1 { v24.4s }, [x23], #0x10\n"
+ "st1 { v28.4s }, [x21], #0x10\n"
+ "tbz x16, #1, 217f\n"
+ "str d9, [x13], #0x8\n"
+ "str d13, [x9], #0x8\n"
+ "str d17, [x27], #0x8\n"
+ "str d21, [x25], #0x8\n"
+ "str d25, [x23], #0x8\n"
+ "str d29, [x21], #0x8\n"
+ "tbz x16, #0, 220f\n"
+ "st1 { v9.s }[2], [x13]\n"
+ "st1 { v13.s }[2], [x9]\n"
+ "st1 { v17.s }[2], [x27]\n"
+ "st1 { v21.s }[2], [x25]\n"
+ "st1 { v25.s }[2], [x23]\n"
+ "st1 { v29.s }[2], [x21]\n"
+ "b 220f\n"
+ "217:" // Height 6: Partial direct writeback: partial_1_4
+ "tbz x16, #0, 220f\n"
+ "str s9, [x13, #0x0]\n"
+ "str s13, [x9, #0x0]\n"
+ "str s17, [x27, #0x0]\n"
+ "str s21, [x25, #0x0]\n"
+ "str s25, [x23, #0x0]\n"
+ "str s29, [x21, #0x0]\n"
+ "b 220f\n"
+ "218:" // Height 6: Partial direct writeback: partial_2_0
+ "tbz x16, #1, 219f\n"
+ "str d8, [x13], #0x8\n"
+ "str d12, [x9], #0x8\n"
+ "str d16, [x27], #0x8\n"
+ "str d20, [x25], #0x8\n"
+ "str d24, [x23], #0x8\n"
+ "str d28, [x21], #0x8\n"
+ "tbz x16, #0, 220f\n"
+ "st1 { v8.s }[2], [x13]\n"
+ "st1 { v12.s }[2], [x9]\n"
+ "st1 { v16.s }[2], [x27]\n"
+ "st1 { v20.s }[2], [x25]\n"
+ "st1 { v24.s }[2], [x23]\n"
+ "st1 { v28.s }[2], [x21]\n"
+ "b 220f\n"
+ "219:" // Height 6: Partial direct writeback: partial_1_0
+ "str s8, [x13, #0x0]\n"
+ "str s12, [x9, #0x0]\n"
+ "str s16, [x27, #0x0]\n"
+ "str s20, [x25, #0x0]\n"
+ "str s24, [x23, #0x0]\n"
+ "str s28, [x21, #0x0]\n"
+ "220:" // Height 6: Partial direct writeback: Done
+ "b 222f\n"
+ "221:" // Height 6: Full writeback
+ "str q8, [x13, #0x0]\n"
+ "str q9, [x13, #0x10]\n"
+ "str q10, [x13, #0x20]\n"
+ "str q11, [x13, #0x30]\n"
+ "str q12, [x9, #0x0]\n"
+ "str q13, [x9, #0x10]\n"
+ "str q14, [x9, #0x20]\n"
+ "str q15, [x9, #0x30]\n"
+ "str q16, [x27, #0x0]\n"
+ "str q17, [x27, #0x10]\n"
+ "str q18, [x27, #0x20]\n"
+ "str q19, [x27, #0x30]\n"
+ "str q20, [x25, #0x0]\n"
+ "str q21, [x25, #0x10]\n"
+ "str q22, [x25, #0x20]\n"
+ "str q23, [x25, #0x30]\n"
+ "str q24, [x23, #0x0]\n"
+ "str q25, [x23, #0x10]\n"
+ "str q26, [x23, #0x20]\n"
+ "str q27, [x23, #0x30]\n"
+ "str q28, [x21, #0x0]\n"
+ "str q29, [x21, #0x10]\n"
+ "str q30, [x21, #0x20]\n"
+ "str q31, [x21, #0x30]\n"
+ "add x13, x13, #0x40\n"
+ "add x9, x9, #0x40\n"
+ "add x27, x27, #0x40\n"
+ "add x25, x25, #0x40\n"
+ "add x23, x23, #0x40\n"
+ "add x21, x21, #0x40\n"
+ "222:" // Height 6: Writeback done
+ "subs x16, x16, #0x10\n"
+ "bgt 188b\n"
+ "subs %x[M], %x[M], #0x6\n"
+ "beq 224f\n"
+ "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "tbz %x[flags], #3, 223f\n"
+ "add x20, x20, #0x6\n"
+ "str x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "b 1b\n"
+ "223:" // Update direct input
+ "mov x19, #0xc\n"
+ "madd %x[input_ptr], x19, x20, %x[input_ptr]\n"
+ "b 1b\n"
+ "224:" // Exit
+
+ : [M] "+r" (M), [input_ptr] "+r" (input_ptr), [output_ptr] "+r" (output_ptr)
+ : [args_ptr] "r" (&ka), [bias] "r" (bias), [flags] "r" (flags), [offset_max] "I" (offsetof(KernelArgs, maxval)), [offset_min] "I" (offsetof(KernelArgs, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths))
+ : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
+ );
+}
+
+} // namespace arm_gemm
+#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_4x8.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp16_mla_6x32.hpp
similarity index 61%
copy from src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_4x8.hpp
copy to src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp16_mla_6x32.hpp
index e9a0948..46de985 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_4x8.hpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp16_mla_6x32.hpp
@@ -10,44 +10,53 @@
* sell copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
*/
#pragma once
+#if defined(__aarch64__) && (defined(FP16_KERNELS) || defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC))
-#ifdef __aarch64__
+#include "../std_transforms_fixed.hpp"
+
+#define ARGLIST \
+ unsigned int, const unsigned int *, \
+ IndirectInputArg<__fp16>, \
+ size_t, size_t, \
+ const __fp16 *, \
+ IndirectOutputArg<__fp16>, \
+ const __fp16 *, Activation, bool
namespace arm_gemm
{
// Actual kernel implementations
-void a64_smallK_hybrid_fp32_mla_4x8(const float *, int, const float *, float *, int, int, int, int, const float *, Activation, bool);
+void a64_hybrid_fp16_mla_6x32( ARGLIST );
-class smallK_hybrid_fp32_mla_4x8
+class cls_a64_hybrid_fp16_mla_6x32
{
public:
- typedef float operand_type;
- typedef float result_type;
+ typedef __fp16 operand_type;
+ typedef __fp16 result_type;
- typedef void (*kern_type)(const float *, int, const float *, float *, int, int, int, int, const float *, Activation, bool);
+ typedef void (*kern_type)( ARGLIST );
/* Kernel blocking parameters */
static constexpr unsigned int out_height()
{
- return 8;
+ return 6;
}
static unsigned int out_width()
{
- return 4;
+ return 32;
}
static constexpr unsigned int k_unroll()
@@ -57,30 +66,20 @@
static constexpr bool supports_accumulate()
{
- return false;
- }
-
- static constexpr bool supports_bias()
- {
return true;
}
- static constexpr bool supports_activation()
- {
- return true;
- }
-
- StdTransformsFixed<operand_type, result_type, 8, 4, 1> transforms = {};
+ StdTransformsFixed<operand_type, result_type, 6, 32, 1> transforms = {};
// Default to the generic kernel
- kern_type kernel=a64_smallK_hybrid_fp32_mla_4x8;
+ kern_type kernel=a64_hybrid_fp16_mla_6x32;
- smallK_hybrid_fp32_mla_4x8(const CPUInfo *)
+ cls_a64_hybrid_fp16_mla_6x32(const CPUInfo *)
{
-
}
};
} // namespace arm_gemm
+#undef ARGLIST
#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp16_mla_6x32/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp16_mla_6x32/generic.cpp
new file mode 100644
index 0000000..ff6cbec
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp16_mla_6x32/generic.cpp
@@ -0,0 +1,5400 @@
+/*
+ * Copyright (c) 2019-2020 Arm Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+#if defined(__aarch64__) && (defined(FP16_KERNELS) || defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC))
+
+#include "arm_gemm.hpp"
+#include "../../utils.hpp"
+
+#include <cassert>
+
+namespace arm_gemm {
+
+void a64_hybrid_fp16_mla_6x32 (
+ unsigned int num_strings, const unsigned int *string_lengths, IndirectInputArg<__fp16> A_arg,
+ size_t M, size_t N, const __fp16 *B_ptr, IndirectOutputArg<__fp16> output_arg,
+ const __fp16 *bias, Activation act, bool accumulate
+)
+{
+ struct KernelArgs {
+ __fp16 maxval = static_cast<__fp16>(std::numeric_limits<float>::infinity());
+ __fp16 minval = - static_cast<__fp16>(std::numeric_limits<float>::infinity());
+ unsigned int num_strings = {};
+ const unsigned int *string_lengths = {};
+ size_t N = {};
+ const __fp16 *B_ptr = {};
+ size_t output_offset = {};
+ size_t input_initial_col = {};
+ size_t input_offset = {};
+ } ka;
+
+ unsigned long flags=0;
+ void *output_ptr;
+ void *input_ptr;
+
+ if (output_arg.is_indirect) {
+ output_ptr=(void *)(output_arg.indirect.ptr);
+ ka.output_offset=output_arg.indirect.offset;
+ flags |= 0x4;
+ } else {
+ output_ptr=(void *)(output_arg.direct.base);
+ ka.output_offset=output_arg.direct.stride;
+ }
+
+ if (A_arg.is_indirect) {
+ input_ptr=(void *)(A_arg.indirect.ptr);
+ ka.input_offset=A_arg.indirect.start_row;
+ ka.input_initial_col=A_arg.indirect.start_col;
+ flags |= 0x8;
+ } else {
+ assert(num_strings==1);
+ input_ptr=(void *)(A_arg.direct.base);
+ ka.input_offset=A_arg.direct.stride;
+ }
+ if (accumulate) {
+ flags |= 0x1;
+ }
+ ka.num_strings = num_strings;
+ ka.string_lengths = string_lengths;
+ ka.N = N;
+ ka.B_ptr = B_ptr;
+ switch(act.type) {
+ default:
+ case Activation::Type::None:
+ break;
+ case Activation::Type::BoundedReLU:
+ ka.maxval = static_cast<__fp16>(act.param1);
+ /* fall through */
+ case Activation::Type::ReLU:
+ ka.minval = 0;
+ flags |= 0x2;
+ break;
+ }
+ __asm__ __volatile__(
+#ifndef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
+ ".arch armv8.2-a+fp16\n"
+#endif
+ "1:" // Row loop
+ "cmp %x[M], #0x6\n"
+ "bge 251f\n"
+ "cmp %x[M], #0x4\n"
+ "bgt 201f\n"
+ "beq 151f\n"
+ "cmp %x[M], #0x2\n"
+ "bgt 101f\n"
+ "beq 51f\n"
+ "ldr x16, [%x[args_ptr], %[offsetof_N]]\n"
+ "ldr x15, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "mov x14, %x[bias]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 2f\n"
+ "ldr x13, [%x[output_ptr], #0x0]\n"
+ "add x13, x13, x19, LSL #1\n"
+ "b 3f\n"
+ "2:" // Height 1: setup direct output
+ "mov x13, %x[output_ptr]\n"
+ "3:" // Height 1: Column loop
+ "cbz x14, 4f\n"
+ "ldr q8, [x14, #0x0]\n"
+ "ldr q9, [x14, #0x10]\n"
+ "ldr q10, [x14, #0x20]\n"
+ "ldr q11, [x14, #0x30]\n"
+ "add x14, x14, #0x40\n"
+ "b 23f\n"
+ "4:" // Height 1: no bias
+ "tbz %x[flags], #0, 22f\n"
+ "cmp x16, #0x20\n"
+ "bge 21f\n"
+ "tbz x16, #4, 12f\n"
+ "ld1 { v8.8h }, [x13], #0x10\n"
+ "ld1 { v9.8h }, [x13], #0x10\n"
+ "tbz x16, #3, 8f\n"
+ "ld1 { v10.8h }, [x13], #0x10\n"
+ "tbz x16, #2, 6f\n"
+ "ldr d11, [x13], #0x8\n"
+ "tbz x16, #1, 5f\n"
+ "mov x19, #0x3c\n"
+ "ld1 { v11.s }[2], [x13], #0x4\n"
+ "tbz x16, #0, 20f\n"
+ "ld1 { v11.h }[6], [x13]\n"
+ "b 20f\n"
+ "5:" // Height 1: Partial accumulate: partial_1_28
+ "mov x19, #0x38\n"
+ "tbz x16, #0, 20f\n"
+ "ld1 { v11.h }[4], [x13]\n"
+ "b 20f\n"
+ "6:" // Height 1: Partial accumulate: partial_2_24
+ "tbz x16, #1, 7f\n"
+ "ldr s11, [x13], #0x4\n"
+ "mov x19, #0x34\n"
+ "tbz x16, #0, 20f\n"
+ "ld1 { v11.h }[2], [x13]\n"
+ "b 20f\n"
+ "7:" // Height 1: Partial accumulate: partial_1_24
+ "mov x19, #0x30\n"
+ "tbz x16, #0, 20f\n"
+ "ldr h11, [x13, #0x0]\n"
+ "b 20f\n"
+ "8:" // Height 1: Partial accumulate: partial_4_16
+ "tbz x16, #2, 10f\n"
+ "ldr d10, [x13], #0x8\n"
+ "tbz x16, #1, 9f\n"
+ "mov x19, #0x2c\n"
+ "ld1 { v10.s }[2], [x13], #0x4\n"
+ "tbz x16, #0, 20f\n"
+ "ld1 { v10.h }[6], [x13]\n"
+ "b 20f\n"
+ "9:" // Height 1: Partial accumulate: partial_1_20
+ "mov x19, #0x28\n"
+ "tbz x16, #0, 20f\n"
+ "ld1 { v10.h }[4], [x13]\n"
+ "b 20f\n"
+ "10:" // Height 1: Partial accumulate: partial_2_16
+ "tbz x16, #1, 11f\n"
+ "ldr s10, [x13], #0x4\n"
+ "mov x19, #0x24\n"
+ "tbz x16, #0, 20f\n"
+ "ld1 { v10.h }[2], [x13]\n"
+ "b 20f\n"
+ "11:" // Height 1: Partial accumulate: partial_1_16
+ "mov x19, #0x20\n"
+ "tbz x16, #0, 20f\n"
+ "ldr h10, [x13, #0x0]\n"
+ "b 20f\n"
+ "12:" // Height 1: Partial accumulate: partial_8_0
+ "tbz x16, #3, 16f\n"
+ "ld1 { v8.8h }, [x13], #0x10\n"
+ "tbz x16, #2, 14f\n"
+ "ldr d9, [x13], #0x8\n"
+ "tbz x16, #1, 13f\n"
+ "mov x19, #0x1c\n"
+ "ld1 { v9.s }[2], [x13], #0x4\n"
+ "tbz x16, #0, 20f\n"
+ "ld1 { v9.h }[6], [x13]\n"
+ "b 20f\n"
+ "13:" // Height 1: Partial accumulate: partial_1_12
+ "mov x19, #0x18\n"
+ "tbz x16, #0, 20f\n"
+ "ld1 { v9.h }[4], [x13]\n"
+ "b 20f\n"
+ "14:" // Height 1: Partial accumulate: partial_2_8
+ "tbz x16, #1, 15f\n"
+ "ldr s9, [x13], #0x4\n"
+ "mov x19, #0x14\n"
+ "tbz x16, #0, 20f\n"
+ "ld1 { v9.h }[2], [x13]\n"
+ "b 20f\n"
+ "15:" // Height 1: Partial accumulate: partial_1_8
+ "mov x19, #0x10\n"
+ "tbz x16, #0, 20f\n"
+ "ldr h9, [x13, #0x0]\n"
+ "b 20f\n"
+ "16:" // Height 1: Partial accumulate: partial_4_0
+ "tbz x16, #2, 18f\n"
+ "ldr d8, [x13], #0x8\n"
+ "tbz x16, #1, 17f\n"
+ "mov x19, #0xc\n"
+ "ld1 { v8.s }[2], [x13], #0x4\n"
+ "tbz x16, #0, 20f\n"
+ "ld1 { v8.h }[6], [x13]\n"
+ "b 20f\n"
+ "17:" // Height 1: Partial accumulate: partial_1_4
+ "mov x19, #0x8\n"
+ "tbz x16, #0, 20f\n"
+ "ld1 { v8.h }[4], [x13]\n"
+ "b 20f\n"
+ "18:" // Height 1: Partial accumulate: partial_2_0
+ "tbz x16, #1, 19f\n"
+ "ldr s8, [x13], #0x4\n"
+ "mov x19, #0x4\n"
+ "tbz x16, #0, 20f\n"
+ "ld1 { v8.h }[2], [x13]\n"
+ "b 20f\n"
+ "19:" // Height 1: Partial accumulate: partial_1_0
+ "mov x19, #0x0\n"
+ "ldr h8, [x13, #0x0]\n"
+ "20:" // Height 1: Partial accumulate: Done
+ "sub x13, x13, x19\n"
+ "b 23f\n"
+ "21:" // Height 1: full accumulate
+ "ldr q8, [x13, #0x0]\n"
+ "ldr q9, [x13, #0x10]\n"
+ "ldr q10, [x13, #0x20]\n"
+ "ldr q11, [x13, #0x30]\n"
+ "b 23f\n"
+ "22:" // Height 1: no accumulate
+ "movi v8.16b, #0x0\n"
+ "movi v9.16b, #0x0\n"
+ "movi v10.16b, #0x0\n"
+ "movi v11.16b, #0x0\n"
+ "23:" // Height 1: setup done
+ "mov x12, #0x0\n"
+ "24:" // Height 1: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w11, [x20, x12, LSL #0x2]\n"
+ "tbz %x[flags], #3, 25f\n"
+ "ldr x20, [%x[input_ptr], x12, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x10, [x20, #0x0]\n"
+ "cbnz x12, 26f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x10, x10, x19, LSL #1\n"
+ "b 26f\n"
+ "25:" // Height 1: setup direct input
+ "mov x10, %x[input_ptr]\n"
+ "26:" // Height 1: input setup done
+ "cmp x11, #0x8\n"
+ "blt 29f\n"
+ "cmp x11, #0x10\n"
+ "blt 28f\n"
+ "27:" // Height 1: Multiply loop: Main loop head
+ "ldr q0, [x10, #0x0]\n"
+ "ldr q6, [x15, #0x0]\n"
+ "fmla v8.8h, v6.8h, v0.h[0]\n"
+ "ldr q7, [x15, #0x10]\n"
+ "ldr q6, [x15, #0x20]\n"
+ "fmla v9.8h, v7.8h, v0.h[0]\n"
+ "ldr q7, [x15, #0x30]\n"
+ "fmla v10.8h, v6.8h, v0.h[0]\n"
+ "ldr q6, [x15, #0x40]\n"
+ "add x10, x10, #0x10\n"
+ "fmla v11.8h, v7.8h, v0.h[0]\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "ldr q7, [x15, #0x50]\n"
+ "fmla v8.8h, v6.8h, v0.h[1]\n"
+ "ldr q6, [x15, #0x60]\n"
+ "sub x11, x11, #0x8\n"
+ "fmla v9.8h, v7.8h, v0.h[1]\n"
+ "ldr q7, [x15, #0x70]\n"
+ "cmp x11, #0x10\n"
+ "fmla v10.8h, v6.8h, v0.h[1]\n"
+ "ldr q6, [x15, #0x80]\n"
+ "fmla v11.8h, v7.8h, v0.h[1]\n"
+ "ldr q7, [x15, #0x90]\n"
+ "fmla v8.8h, v6.8h, v0.h[2]\n"
+ "ldr q6, [x15, #0xa0]\n"
+ "fmla v9.8h, v7.8h, v0.h[2]\n"
+ "ldr q7, [x15, #0xb0]\n"
+ "fmla v10.8h, v6.8h, v0.h[2]\n"
+ "ldr q6, [x15, #0xc0]\n"
+ "fmla v11.8h, v7.8h, v0.h[2]\n"
+ "ldr q7, [x15, #0xd0]\n"
+ "fmla v8.8h, v6.8h, v0.h[3]\n"
+ "ldr q6, [x15, #0xe0]\n"
+ "fmla v9.8h, v7.8h, v0.h[3]\n"
+ "ldr q7, [x15, #0xf0]\n"
+ "fmla v10.8h, v6.8h, v0.h[3]\n"
+ "ldr q6, [x15, #0x100]\n"
+ "fmla v11.8h, v7.8h, v0.h[3]\n"
+ "ldr q7, [x15, #0x110]\n"
+ "fmla v8.8h, v6.8h, v0.h[4]\n"
+ "ldr q6, [x15, #0x120]\n"
+ "fmla v9.8h, v7.8h, v0.h[4]\n"
+ "ldr q7, [x15, #0x130]\n"
+ "fmla v10.8h, v6.8h, v0.h[4]\n"
+ "ldr q6, [x15, #0x140]\n"
+ "fmla v11.8h, v7.8h, v0.h[4]\n"
+ "ldr q7, [x15, #0x150]\n"
+ "fmla v8.8h, v6.8h, v0.h[5]\n"
+ "ldr q6, [x15, #0x160]\n"
+ "fmla v9.8h, v7.8h, v0.h[5]\n"
+ "ldr q7, [x15, #0x170]\n"
+ "fmla v10.8h, v6.8h, v0.h[5]\n"
+ "ldr q6, [x15, #0x180]\n"
+ "fmla v11.8h, v7.8h, v0.h[5]\n"
+ "ldr q7, [x15, #0x190]\n"
+ "fmla v8.8h, v6.8h, v0.h[6]\n"
+ "ldr q6, [x15, #0x1a0]\n"
+ "fmla v9.8h, v7.8h, v0.h[6]\n"
+ "ldr q7, [x15, #0x1b0]\n"
+ "fmla v10.8h, v6.8h, v0.h[6]\n"
+ "ldr q6, [x15, #0x1c0]\n"
+ "fmla v11.8h, v7.8h, v0.h[6]\n"
+ "ldr q7, [x15, #0x1d0]\n"
+ "fmla v8.8h, v6.8h, v0.h[7]\n"
+ "ldr q6, [x15, #0x1e0]\n"
+ "fmla v9.8h, v7.8h, v0.h[7]\n"
+ "ldr q7, [x15, #0x1f0]\n"
+ "add x15, x15, #0x200\n"
+ "fmla v10.8h, v6.8h, v0.h[7]\n"
+ "fmla v11.8h, v7.8h, v0.h[7]\n"
+ "bge 27b\n"
+ "28:" // Height 1: Multiply loop: Single iteration only
+ "sub x11, x11, #0x8\n"
+ "ldr q0, [x10, #0x0]\n"
+ "ldr q6, [x15, #0x0]\n"
+ "fmla v8.8h, v6.8h, v0.h[0]\n"
+ "ldr q7, [x15, #0x10]\n"
+ "ldr q6, [x15, #0x20]\n"
+ "fmla v9.8h, v7.8h, v0.h[0]\n"
+ "ldr q7, [x15, #0x30]\n"
+ "fmla v10.8h, v6.8h, v0.h[0]\n"
+ "ldr q6, [x15, #0x40]\n"
+ "add x10, x10, #0x10\n"
+ "fmla v11.8h, v7.8h, v0.h[0]\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "ldr q7, [x15, #0x50]\n"
+ "fmla v8.8h, v6.8h, v0.h[1]\n"
+ "ldr q6, [x15, #0x60]\n"
+ "fmla v9.8h, v7.8h, v0.h[1]\n"
+ "ldr q7, [x15, #0x70]\n"
+ "fmla v10.8h, v6.8h, v0.h[1]\n"
+ "ldr q6, [x15, #0x80]\n"
+ "fmla v11.8h, v7.8h, v0.h[1]\n"
+ "ldr q7, [x15, #0x90]\n"
+ "fmla v8.8h, v6.8h, v0.h[2]\n"
+ "ldr q6, [x15, #0xa0]\n"
+ "fmla v9.8h, v7.8h, v0.h[2]\n"
+ "ldr q7, [x15, #0xb0]\n"
+ "fmla v10.8h, v6.8h, v0.h[2]\n"
+ "ldr q6, [x15, #0xc0]\n"
+ "fmla v11.8h, v7.8h, v0.h[2]\n"
+ "ldr q7, [x15, #0xd0]\n"
+ "fmla v8.8h, v6.8h, v0.h[3]\n"
+ "ldr q6, [x15, #0xe0]\n"
+ "fmla v9.8h, v7.8h, v0.h[3]\n"
+ "ldr q7, [x15, #0xf0]\n"
+ "fmla v10.8h, v6.8h, v0.h[3]\n"
+ "ldr q6, [x15, #0x100]\n"
+ "fmla v11.8h, v7.8h, v0.h[3]\n"
+ "ldr q7, [x15, #0x110]\n"
+ "fmla v8.8h, v6.8h, v0.h[4]\n"
+ "ldr q6, [x15, #0x120]\n"
+ "fmla v9.8h, v7.8h, v0.h[4]\n"
+ "ldr q7, [x15, #0x130]\n"
+ "fmla v10.8h, v6.8h, v0.h[4]\n"
+ "ldr q6, [x15, #0x140]\n"
+ "fmla v11.8h, v7.8h, v0.h[4]\n"
+ "ldr q7, [x15, #0x150]\n"
+ "fmla v8.8h, v6.8h, v0.h[5]\n"
+ "ldr q6, [x15, #0x160]\n"
+ "fmla v9.8h, v7.8h, v0.h[5]\n"
+ "ldr q7, [x15, #0x170]\n"
+ "fmla v10.8h, v6.8h, v0.h[5]\n"
+ "ldr q6, [x15, #0x180]\n"
+ "fmla v11.8h, v7.8h, v0.h[5]\n"
+ "ldr q7, [x15, #0x190]\n"
+ "fmla v8.8h, v6.8h, v0.h[6]\n"
+ "ldr q6, [x15, #0x1a0]\n"
+ "fmla v9.8h, v7.8h, v0.h[6]\n"
+ "ldr q7, [x15, #0x1b0]\n"
+ "fmla v10.8h, v6.8h, v0.h[6]\n"
+ "ldr q6, [x15, #0x1c0]\n"
+ "fmla v11.8h, v7.8h, v0.h[6]\n"
+ "ldr q7, [x15, #0x1d0]\n"
+ "fmla v8.8h, v6.8h, v0.h[7]\n"
+ "ldr q6, [x15, #0x1e0]\n"
+ "fmla v9.8h, v7.8h, v0.h[7]\n"
+ "ldr q7, [x15, #0x1f0]\n"
+ "add x15, x15, #0x200\n"
+ "fmla v10.8h, v6.8h, v0.h[7]\n"
+ "fmla v11.8h, v7.8h, v0.h[7]\n"
+ "29:" // Height 1: Multiply loop: Main loop skip
+ "cbz x11, 31f\n"
+ "30:" // Height 1: Multiply loop: Odd block loop
+ "ldr h0, [x10], #0x2\n"
+ "ldr q6, [x15, #0x0]\n"
+ "fmla v8.8h, v6.8h, v0.h[0]\n"
+ "ldr q7, [x15, #0x10]\n"
+ "ldr q6, [x15, #0x20]\n"
+ "fmla v9.8h, v7.8h, v0.h[0]\n"
+ "ldr q7, [x15, #0x30]\n"
+ "fmla v10.8h, v6.8h, v0.h[0]\n"
+ "sub x11, x11, #0x1\n"
+ "add x15, x15, #0x40\n"
+ "fmla v11.8h, v7.8h, v0.h[0]\n"
+ "cbnz x11, 30b\n"
+ "31:" // Height 1: Multiply loop: No odd multiplies
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "add x12, x12, #0x1\n"
+ "cmp x12, x19\n"
+ "bne 24b\n"
+ "prfm pstl1keep, [x13, #0x0]\n"
+ "tbz %x[flags], #1, 32f\n"
+ "add x19, %x[args_ptr], %[offset_min]\n"
+ "ld1r { v1.8h }, [x19]\n"
+ "add x19, %x[args_ptr], %[offset_max]\n"
+ "ld1r { v0.8h }, [x19]\n"
+ "fmin v8.8h, v8.8h, v0.8h\n"
+ "fmin v9.8h, v9.8h, v0.8h\n"
+ "fmin v10.8h, v10.8h, v0.8h\n"
+ "fmin v11.8h, v11.8h, v0.8h\n"
+ "fmax v8.8h, v8.8h, v1.8h\n"
+ "fmax v9.8h, v9.8h, v1.8h\n"
+ "fmax v10.8h, v10.8h, v1.8h\n"
+ "fmax v11.8h, v11.8h, v1.8h\n"
+ "32:" // Height 1: No activation
+ "cmp x16, #0x20\n"
+ "bge 49f\n"
+ "tbz x16, #4, 40f\n"
+ "st1 { v8.8h }, [x13], #0x10\n"
+ "st1 { v9.8h }, [x13], #0x10\n"
+ "tbz x16, #3, 36f\n"
+ "st1 { v10.8h }, [x13], #0x10\n"
+ "tbz x16, #2, 34f\n"
+ "str d11, [x13], #0x8\n"
+ "tbz x16, #1, 33f\n"
+ "st1 { v11.s }[2], [x13], #0x4\n"
+ "tbz x16, #0, 48f\n"
+ "st1 { v11.h }[6], [x13]\n"
+ "b 48f\n"
+ "33:" // Height 1: Partial direct writeback: partial_1_28
+ "tbz x16, #0, 48f\n"
+ "st1 { v11.h }[4], [x13]\n"
+ "b 48f\n"
+ "34:" // Height 1: Partial direct writeback: partial_2_24
+ "tbz x16, #1, 35f\n"
+ "str s11, [x13], #0x4\n"
+ "tbz x16, #0, 48f\n"
+ "st1 { v11.h }[2], [x13]\n"
+ "b 48f\n"
+ "35:" // Height 1: Partial direct writeback: partial_1_24
+ "tbz x16, #0, 48f\n"
+ "str h11, [x13, #0x0]\n"
+ "b 48f\n"
+ "36:" // Height 1: Partial direct writeback: partial_4_16
+ "tbz x16, #2, 38f\n"
+ "str d10, [x13], #0x8\n"
+ "tbz x16, #1, 37f\n"
+ "st1 { v10.s }[2], [x13], #0x4\n"
+ "tbz x16, #0, 48f\n"
+ "st1 { v10.h }[6], [x13]\n"
+ "b 48f\n"
+ "37:" // Height 1: Partial direct writeback: partial_1_20
+ "tbz x16, #0, 48f\n"
+ "st1 { v10.h }[4], [x13]\n"
+ "b 48f\n"
+ "38:" // Height 1: Partial direct writeback: partial_2_16
+ "tbz x16, #1, 39f\n"
+ "str s10, [x13], #0x4\n"
+ "tbz x16, #0, 48f\n"
+ "st1 { v10.h }[2], [x13]\n"
+ "b 48f\n"
+ "39:" // Height 1: Partial direct writeback: partial_1_16
+ "tbz x16, #0, 48f\n"
+ "str h10, [x13, #0x0]\n"
+ "b 48f\n"
+ "40:" // Height 1: Partial direct writeback: partial_8_0
+ "tbz x16, #3, 44f\n"
+ "st1 { v8.8h }, [x13], #0x10\n"
+ "tbz x16, #2, 42f\n"
+ "str d9, [x13], #0x8\n"
+ "tbz x16, #1, 41f\n"
+ "st1 { v9.s }[2], [x13], #0x4\n"
+ "tbz x16, #0, 48f\n"
+ "st1 { v9.h }[6], [x13]\n"
+ "b 48f\n"
+ "41:" // Height 1: Partial direct writeback: partial_1_12
+ "tbz x16, #0, 48f\n"
+ "st1 { v9.h }[4], [x13]\n"
+ "b 48f\n"
+ "42:" // Height 1: Partial direct writeback: partial_2_8
+ "tbz x16, #1, 43f\n"
+ "str s9, [x13], #0x4\n"
+ "tbz x16, #0, 48f\n"
+ "st1 { v9.h }[2], [x13]\n"
+ "b 48f\n"
+ "43:" // Height 1: Partial direct writeback: partial_1_8
+ "tbz x16, #0, 48f\n"
+ "str h9, [x13, #0x0]\n"
+ "b 48f\n"
+ "44:" // Height 1: Partial direct writeback: partial_4_0
+ "tbz x16, #2, 46f\n"
+ "str d8, [x13], #0x8\n"
+ "tbz x16, #1, 45f\n"
+ "st1 { v8.s }[2], [x13], #0x4\n"
+ "tbz x16, #0, 48f\n"
+ "st1 { v8.h }[6], [x13]\n"
+ "b 48f\n"
+ "45:" // Height 1: Partial direct writeback: partial_1_4
+ "tbz x16, #0, 48f\n"
+ "st1 { v8.h }[4], [x13]\n"
+ "b 48f\n"
+ "46:" // Height 1: Partial direct writeback: partial_2_0
+ "tbz x16, #1, 47f\n"
+ "str s8, [x13], #0x4\n"
+ "tbz x16, #0, 48f\n"
+ "st1 { v8.h }[2], [x13]\n"
+ "b 48f\n"
+ "47:" // Height 1: Partial direct writeback: partial_1_0
+ "str h8, [x13, #0x0]\n"
+ "48:" // Height 1: Partial direct writeback: Done
+ "b 50f\n"
+ "49:" // Height 1: Full writeback
+ "str q8, [x13, #0x0]\n"
+ "str q9, [x13, #0x10]\n"
+ "str q10, [x13, #0x20]\n"
+ "str q11, [x13, #0x30]\n"
+ "add x13, x13, #0x40\n"
+ "50:" // Height 1: Writeback done
+ "subs x16, x16, #0x20\n"
+ "bgt 3b\n"
+ "b 302f\n"
+ "51:" // Height 2
+ "ldr x16, [%x[args_ptr], %[offsetof_N]]\n"
+ "mov x14, %x[bias]\n"
+ "ldr x15, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 52f\n"
+ "ldr x13, [%x[output_ptr], #0x0]\n"
+ "add x13, x13, x19, LSL #1\n"
+ "ldr x9, [%x[output_ptr], #0x8]\n"
+ "add x9, x9, x19, LSL #1\n"
+ "b 53f\n"
+ "52:" // Height 2: setup direct output
+ "mov x13, %x[output_ptr]\n"
+ "add x9, x13, x19, LSL #1\n"
+ "53:" // Height 2: Column loop
+ "cbz x14, 54f\n"
+ "ldr q8, [x14, #0x0]\n"
+ "mov v12.16b, v8.16b\n"
+ "ldr q9, [x14, #0x10]\n"
+ "ldr q10, [x14, #0x20]\n"
+ "mov v13.16b, v9.16b\n"
+ "ldr q11, [x14, #0x30]\n"
+ "mov v14.16b, v10.16b\n"
+ "add x14, x14, #0x40\n"
+ "mov v15.16b, v11.16b\n"
+ "b 73f\n"
+ "54:" // Height 2: no bias
+ "tbz %x[flags], #0, 72f\n"
+ "cmp x16, #0x20\n"
+ "bge 71f\n"
+ "tbz x16, #4, 62f\n"
+ "ld1 { v8.8h }, [x13], #0x10\n"
+ "ld1 { v12.8h }, [x9], #0x10\n"
+ "ld1 { v9.8h }, [x13], #0x10\n"
+ "ld1 { v13.8h }, [x9], #0x10\n"
+ "tbz x16, #3, 58f\n"
+ "ld1 { v10.8h }, [x13], #0x10\n"
+ "ld1 { v14.8h }, [x9], #0x10\n"
+ "tbz x16, #2, 56f\n"
+ "ldr d11, [x13], #0x8\n"
+ "ldr d15, [x9], #0x8\n"
+ "tbz x16, #1, 55f\n"
+ "mov x19, #0x3c\n"
+ "ld1 { v11.s }[2], [x13], #0x4\n"
+ "ld1 { v15.s }[2], [x9], #0x4\n"
+ "tbz x16, #0, 70f\n"
+ "ld1 { v11.h }[6], [x13]\n"
+ "ld1 { v15.h }[6], [x9]\n"
+ "b 70f\n"
+ "55:" // Height 2: Partial accumulate: partial_1_28
+ "mov x19, #0x38\n"
+ "tbz x16, #0, 70f\n"
+ "ld1 { v11.h }[4], [x13]\n"
+ "ld1 { v15.h }[4], [x9]\n"
+ "b 70f\n"
+ "56:" // Height 2: Partial accumulate: partial_2_24
+ "tbz x16, #1, 57f\n"
+ "ldr s11, [x13], #0x4\n"
+ "ldr s15, [x9], #0x4\n"
+ "mov x19, #0x34\n"
+ "tbz x16, #0, 70f\n"
+ "ld1 { v11.h }[2], [x13]\n"
+ "ld1 { v15.h }[2], [x9]\n"
+ "b 70f\n"
+ "57:" // Height 2: Partial accumulate: partial_1_24
+ "mov x19, #0x30\n"
+ "tbz x16, #0, 70f\n"
+ "ldr h11, [x13, #0x0]\n"
+ "ldr h15, [x9, #0x0]\n"
+ "b 70f\n"
+ "58:" // Height 2: Partial accumulate: partial_4_16
+ "tbz x16, #2, 60f\n"
+ "ldr d10, [x13], #0x8\n"
+ "ldr d14, [x9], #0x8\n"
+ "tbz x16, #1, 59f\n"
+ "mov x19, #0x2c\n"
+ "ld1 { v10.s }[2], [x13], #0x4\n"
+ "ld1 { v14.s }[2], [x9], #0x4\n"
+ "tbz x16, #0, 70f\n"
+ "ld1 { v10.h }[6], [x13]\n"
+ "ld1 { v14.h }[6], [x9]\n"
+ "b 70f\n"
+ "59:" // Height 2: Partial accumulate: partial_1_20
+ "mov x19, #0x28\n"
+ "tbz x16, #0, 70f\n"
+ "ld1 { v10.h }[4], [x13]\n"
+ "ld1 { v14.h }[4], [x9]\n"
+ "b 70f\n"
+ "60:" // Height 2: Partial accumulate: partial_2_16
+ "tbz x16, #1, 61f\n"
+ "ldr s10, [x13], #0x4\n"
+ "ldr s14, [x9], #0x4\n"
+ "mov x19, #0x24\n"
+ "tbz x16, #0, 70f\n"
+ "ld1 { v10.h }[2], [x13]\n"
+ "ld1 { v14.h }[2], [x9]\n"
+ "b 70f\n"
+ "61:" // Height 2: Partial accumulate: partial_1_16
+ "mov x19, #0x20\n"
+ "tbz x16, #0, 70f\n"
+ "ldr h10, [x13, #0x0]\n"
+ "ldr h14, [x9, #0x0]\n"
+ "b 70f\n"
+ "62:" // Height 2: Partial accumulate: partial_8_0
+ "tbz x16, #3, 66f\n"
+ "ld1 { v8.8h }, [x13], #0x10\n"
+ "ld1 { v12.8h }, [x9], #0x10\n"
+ "tbz x16, #2, 64f\n"
+ "ldr d9, [x13], #0x8\n"
+ "ldr d13, [x9], #0x8\n"
+ "tbz x16, #1, 63f\n"
+ "mov x19, #0x1c\n"
+ "ld1 { v9.s }[2], [x13], #0x4\n"
+ "ld1 { v13.s }[2], [x9], #0x4\n"
+ "tbz x16, #0, 70f\n"
+ "ld1 { v9.h }[6], [x13]\n"
+ "ld1 { v13.h }[6], [x9]\n"
+ "b 70f\n"
+ "63:" // Height 2: Partial accumulate: partial_1_12
+ "mov x19, #0x18\n"
+ "tbz x16, #0, 70f\n"
+ "ld1 { v9.h }[4], [x13]\n"
+ "ld1 { v13.h }[4], [x9]\n"
+ "b 70f\n"
+ "64:" // Height 2: Partial accumulate: partial_2_8
+ "tbz x16, #1, 65f\n"
+ "ldr s9, [x13], #0x4\n"
+ "ldr s13, [x9], #0x4\n"
+ "mov x19, #0x14\n"
+ "tbz x16, #0, 70f\n"
+ "ld1 { v9.h }[2], [x13]\n"
+ "ld1 { v13.h }[2], [x9]\n"
+ "b 70f\n"
+ "65:" // Height 2: Partial accumulate: partial_1_8
+ "mov x19, #0x10\n"
+ "tbz x16, #0, 70f\n"
+ "ldr h9, [x13, #0x0]\n"
+ "ldr h13, [x9, #0x0]\n"
+ "b 70f\n"
+ "66:" // Height 2: Partial accumulate: partial_4_0
+ "tbz x16, #2, 68f\n"
+ "ldr d8, [x13], #0x8\n"
+ "ldr d12, [x9], #0x8\n"
+ "tbz x16, #1, 67f\n"
+ "mov x19, #0xc\n"
+ "ld1 { v8.s }[2], [x13], #0x4\n"
+ "ld1 { v12.s }[2], [x9], #0x4\n"
+ "tbz x16, #0, 70f\n"
+ "ld1 { v8.h }[6], [x13]\n"
+ "ld1 { v12.h }[6], [x9]\n"
+ "b 70f\n"
+ "67:" // Height 2: Partial accumulate: partial_1_4
+ "mov x19, #0x8\n"
+ "tbz x16, #0, 70f\n"
+ "ld1 { v8.h }[4], [x13]\n"
+ "ld1 { v12.h }[4], [x9]\n"
+ "b 70f\n"
+ "68:" // Height 2: Partial accumulate: partial_2_0
+ "tbz x16, #1, 69f\n"
+ "ldr s8, [x13], #0x4\n"
+ "ldr s12, [x9], #0x4\n"
+ "mov x19, #0x4\n"
+ "tbz x16, #0, 70f\n"
+ "ld1 { v8.h }[2], [x13]\n"
+ "ld1 { v12.h }[2], [x9]\n"
+ "b 70f\n"
+ "69:" // Height 2: Partial accumulate: partial_1_0
+ "mov x19, #0x0\n"
+ "ldr h8, [x13, #0x0]\n"
+ "ldr h12, [x9, #0x0]\n"
+ "70:" // Height 2: Partial accumulate: Done
+ "sub x13, x13, x19\n"
+ "sub x9, x9, x19\n"
+ "b 73f\n"
+ "71:" // Height 2: full accumulate
+ "ldr q8, [x13, #0x0]\n"
+ "ldr q9, [x13, #0x10]\n"
+ "ldr q10, [x13, #0x20]\n"
+ "ldr q11, [x13, #0x30]\n"
+ "ldr q12, [x9, #0x0]\n"
+ "ldr q13, [x9, #0x10]\n"
+ "ldr q14, [x9, #0x20]\n"
+ "ldr q15, [x9, #0x30]\n"
+ "b 73f\n"
+ "72:" // Height 2: no accumulate
+ "movi v8.16b, #0x0\n"
+ "movi v9.16b, #0x0\n"
+ "movi v10.16b, #0x0\n"
+ "movi v11.16b, #0x0\n"
+ "movi v12.16b, #0x0\n"
+ "movi v13.16b, #0x0\n"
+ "movi v14.16b, #0x0\n"
+ "movi v15.16b, #0x0\n"
+ "73:" // Height 2: setup done
+ "mov x12, #0x0\n"
+ "74:" // Height 2: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w11, [x20, x12, LSL #0x2]\n"
+ "tbz %x[flags], #3, 75f\n"
+ "ldr x20, [%x[input_ptr], x12, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x10, [x20, #0x0]\n"
+ "ldr x28, [x20, #0x8]\n"
+ "cbnz x12, 76f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x10, x10, x19, LSL #1\n"
+ "add x28, x28, x19, LSL #1\n"
+ "b 76f\n"
+ "75:" // Height 2: setup direct input
+ "mov x10, %x[input_ptr]\n"
+ "add x28, x10, x19, LSL #1\n"
+ "76:" // Height 2: input setup done
+ "cmp x11, #0x8\n"
+ "blt 79f\n"
+ "cmp x11, #0x10\n"
+ "blt 78f\n"
+ "77:" // Height 2: Multiply loop: Main loop head
+ "ldr q0, [x10, #0x0]\n"
+ "ldr q1, [x28, #0x0]\n"
+ "ldr q6, [x15, #0x0]\n"
+ "fmla v8.8h, v6.8h, v0.h[0]\n"
+ "ldr q7, [x15, #0x10]\n"
+ "fmla v12.8h, v6.8h, v1.h[0]\n"
+ "ldr q6, [x15, #0x20]\n"
+ "add x10, x10, #0x10\n"
+ "fmla v9.8h, v7.8h, v0.h[0]\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x28, x28, #0x10\n"
+ "fmla v13.8h, v7.8h, v1.h[0]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "ldr q7, [x15, #0x30]\n"
+ "fmla v10.8h, v6.8h, v0.h[0]\n"
+ "sub x11, x11, #0x8\n"
+ "fmla v14.8h, v6.8h, v1.h[0]\n"
+ "ldr q6, [x15, #0x40]\n"
+ "cmp x11, #0x10\n"
+ "fmla v11.8h, v7.8h, v0.h[0]\n"
+ "fmla v15.8h, v7.8h, v1.h[0]\n"
+ "ldr q7, [x15, #0x50]\n"
+ "fmla v8.8h, v6.8h, v0.h[1]\n"
+ "fmla v12.8h, v6.8h, v1.h[1]\n"
+ "ldr q6, [x15, #0x60]\n"
+ "fmla v9.8h, v7.8h, v0.h[1]\n"
+ "fmla v13.8h, v7.8h, v1.h[1]\n"
+ "ldr q7, [x15, #0x70]\n"
+ "fmla v10.8h, v6.8h, v0.h[1]\n"
+ "fmla v14.8h, v6.8h, v1.h[1]\n"
+ "ldr q6, [x15, #0x80]\n"
+ "fmla v11.8h, v7.8h, v0.h[1]\n"
+ "fmla v15.8h, v7.8h, v1.h[1]\n"
+ "ldr q7, [x15, #0x90]\n"
+ "fmla v8.8h, v6.8h, v0.h[2]\n"
+ "fmla v12.8h, v6.8h, v1.h[2]\n"
+ "ldr q6, [x15, #0xa0]\n"
+ "fmla v9.8h, v7.8h, v0.h[2]\n"
+ "fmla v13.8h, v7.8h, v1.h[2]\n"
+ "ldr q7, [x15, #0xb0]\n"
+ "fmla v10.8h, v6.8h, v0.h[2]\n"
+ "fmla v14.8h, v6.8h, v1.h[2]\n"
+ "ldr q6, [x15, #0xc0]\n"
+ "fmla v11.8h, v7.8h, v0.h[2]\n"
+ "fmla v15.8h, v7.8h, v1.h[2]\n"
+ "ldr q7, [x15, #0xd0]\n"
+ "fmla v8.8h, v6.8h, v0.h[3]\n"
+ "fmla v12.8h, v6.8h, v1.h[3]\n"
+ "ldr q6, [x15, #0xe0]\n"
+ "fmla v9.8h, v7.8h, v0.h[3]\n"
+ "fmla v13.8h, v7.8h, v1.h[3]\n"
+ "ldr q7, [x15, #0xf0]\n"
+ "fmla v10.8h, v6.8h, v0.h[3]\n"
+ "fmla v14.8h, v6.8h, v1.h[3]\n"
+ "ldr q6, [x15, #0x100]\n"
+ "fmla v11.8h, v7.8h, v0.h[3]\n"
+ "fmla v15.8h, v7.8h, v1.h[3]\n"
+ "ldr q7, [x15, #0x110]\n"
+ "fmla v8.8h, v6.8h, v0.h[4]\n"
+ "fmla v12.8h, v6.8h, v1.h[4]\n"
+ "ldr q6, [x15, #0x120]\n"
+ "fmla v9.8h, v7.8h, v0.h[4]\n"
+ "fmla v13.8h, v7.8h, v1.h[4]\n"
+ "ldr q7, [x15, #0x130]\n"
+ "fmla v10.8h, v6.8h, v0.h[4]\n"
+ "fmla v14.8h, v6.8h, v1.h[4]\n"
+ "ldr q6, [x15, #0x140]\n"
+ "fmla v11.8h, v7.8h, v0.h[4]\n"
+ "fmla v15.8h, v7.8h, v1.h[4]\n"
+ "ldr q7, [x15, #0x150]\n"
+ "fmla v8.8h, v6.8h, v0.h[5]\n"
+ "fmla v12.8h, v6.8h, v1.h[5]\n"
+ "ldr q6, [x15, #0x160]\n"
+ "fmla v9.8h, v7.8h, v0.h[5]\n"
+ "fmla v13.8h, v7.8h, v1.h[5]\n"
+ "ldr q7, [x15, #0x170]\n"
+ "fmla v10.8h, v6.8h, v0.h[5]\n"
+ "fmla v14.8h, v6.8h, v1.h[5]\n"
+ "ldr q6, [x15, #0x180]\n"
+ "fmla v11.8h, v7.8h, v0.h[5]\n"
+ "fmla v15.8h, v7.8h, v1.h[5]\n"
+ "ldr q7, [x15, #0x190]\n"
+ "fmla v8.8h, v6.8h, v0.h[6]\n"
+ "fmla v12.8h, v6.8h, v1.h[6]\n"
+ "ldr q6, [x15, #0x1a0]\n"
+ "fmla v9.8h, v7.8h, v0.h[6]\n"
+ "fmla v13.8h, v7.8h, v1.h[6]\n"
+ "ldr q7, [x15, #0x1b0]\n"
+ "fmla v10.8h, v6.8h, v0.h[6]\n"
+ "fmla v14.8h, v6.8h, v1.h[6]\n"
+ "ldr q6, [x15, #0x1c0]\n"
+ "fmla v11.8h, v7.8h, v0.h[6]\n"
+ "fmla v15.8h, v7.8h, v1.h[6]\n"
+ "ldr q7, [x15, #0x1d0]\n"
+ "fmla v8.8h, v6.8h, v0.h[7]\n"
+ "fmla v12.8h, v6.8h, v1.h[7]\n"
+ "ldr q6, [x15, #0x1e0]\n"
+ "fmla v9.8h, v7.8h, v0.h[7]\n"
+ "fmla v13.8h, v7.8h, v1.h[7]\n"
+ "ldr q7, [x15, #0x1f0]\n"
+ "add x15, x15, #0x200\n"
+ "fmla v10.8h, v6.8h, v0.h[7]\n"
+ "fmla v14.8h, v6.8h, v1.h[7]\n"
+ "fmla v11.8h, v7.8h, v0.h[7]\n"
+ "fmla v15.8h, v7.8h, v1.h[7]\n"
+ "bge 77b\n"
+ "78:" // Height 2: Multiply loop: Single iteration only
+ "sub x11, x11, #0x8\n"
+ "ldr q0, [x10, #0x0]\n"
+ "ldr q1, [x28, #0x0]\n"
+ "ldr q6, [x15, #0x0]\n"
+ "fmla v8.8h, v6.8h, v0.h[0]\n"
+ "ldr q7, [x15, #0x10]\n"
+ "fmla v12.8h, v6.8h, v1.h[0]\n"
+ "ldr q6, [x15, #0x20]\n"
+ "add x10, x10, #0x10\n"
+ "fmla v9.8h, v7.8h, v0.h[0]\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x28, x28, #0x10\n"
+ "fmla v13.8h, v7.8h, v1.h[0]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "ldr q7, [x15, #0x30]\n"
+ "fmla v10.8h, v6.8h, v0.h[0]\n"
+ "fmla v14.8h, v6.8h, v1.h[0]\n"
+ "ldr q6, [x15, #0x40]\n"
+ "fmla v11.8h, v7.8h, v0.h[0]\n"
+ "fmla v15.8h, v7.8h, v1.h[0]\n"
+ "ldr q7, [x15, #0x50]\n"
+ "fmla v8.8h, v6.8h, v0.h[1]\n"
+ "fmla v12.8h, v6.8h, v1.h[1]\n"
+ "ldr q6, [x15, #0x60]\n"
+ "fmla v9.8h, v7.8h, v0.h[1]\n"
+ "fmla v13.8h, v7.8h, v1.h[1]\n"
+ "ldr q7, [x15, #0x70]\n"
+ "fmla v10.8h, v6.8h, v0.h[1]\n"
+ "fmla v14.8h, v6.8h, v1.h[1]\n"
+ "ldr q6, [x15, #0x80]\n"
+ "fmla v11.8h, v7.8h, v0.h[1]\n"
+ "fmla v15.8h, v7.8h, v1.h[1]\n"
+ "ldr q7, [x15, #0x90]\n"
+ "fmla v8.8h, v6.8h, v0.h[2]\n"
+ "fmla v12.8h, v6.8h, v1.h[2]\n"
+ "ldr q6, [x15, #0xa0]\n"
+ "fmla v9.8h, v7.8h, v0.h[2]\n"
+ "fmla v13.8h, v7.8h, v1.h[2]\n"
+ "ldr q7, [x15, #0xb0]\n"
+ "fmla v10.8h, v6.8h, v0.h[2]\n"
+ "fmla v14.8h, v6.8h, v1.h[2]\n"
+ "ldr q6, [x15, #0xc0]\n"
+ "fmla v11.8h, v7.8h, v0.h[2]\n"
+ "fmla v15.8h, v7.8h, v1.h[2]\n"
+ "ldr q7, [x15, #0xd0]\n"
+ "fmla v8.8h, v6.8h, v0.h[3]\n"
+ "fmla v12.8h, v6.8h, v1.h[3]\n"
+ "ldr q6, [x15, #0xe0]\n"
+ "fmla v9.8h, v7.8h, v0.h[3]\n"
+ "fmla v13.8h, v7.8h, v1.h[3]\n"
+ "ldr q7, [x15, #0xf0]\n"
+ "fmla v10.8h, v6.8h, v0.h[3]\n"
+ "fmla v14.8h, v6.8h, v1.h[3]\n"
+ "ldr q6, [x15, #0x100]\n"
+ "fmla v11.8h, v7.8h, v0.h[3]\n"
+ "fmla v15.8h, v7.8h, v1.h[3]\n"
+ "ldr q7, [x15, #0x110]\n"
+ "fmla v8.8h, v6.8h, v0.h[4]\n"
+ "fmla v12.8h, v6.8h, v1.h[4]\n"
+ "ldr q6, [x15, #0x120]\n"
+ "fmla v9.8h, v7.8h, v0.h[4]\n"
+ "fmla v13.8h, v7.8h, v1.h[4]\n"
+ "ldr q7, [x15, #0x130]\n"
+ "fmla v10.8h, v6.8h, v0.h[4]\n"
+ "fmla v14.8h, v6.8h, v1.h[4]\n"
+ "ldr q6, [x15, #0x140]\n"
+ "fmla v11.8h, v7.8h, v0.h[4]\n"
+ "fmla v15.8h, v7.8h, v1.h[4]\n"
+ "ldr q7, [x15, #0x150]\n"
+ "fmla v8.8h, v6.8h, v0.h[5]\n"
+ "fmla v12.8h, v6.8h, v1.h[5]\n"
+ "ldr q6, [x15, #0x160]\n"
+ "fmla v9.8h, v7.8h, v0.h[5]\n"
+ "fmla v13.8h, v7.8h, v1.h[5]\n"
+ "ldr q7, [x15, #0x170]\n"
+ "fmla v10.8h, v6.8h, v0.h[5]\n"
+ "fmla v14.8h, v6.8h, v1.h[5]\n"
+ "ldr q6, [x15, #0x180]\n"
+ "fmla v11.8h, v7.8h, v0.h[5]\n"
+ "fmla v15.8h, v7.8h, v1.h[5]\n"
+ "ldr q7, [x15, #0x190]\n"
+ "fmla v8.8h, v6.8h, v0.h[6]\n"
+ "fmla v12.8h, v6.8h, v1.h[6]\n"
+ "ldr q6, [x15, #0x1a0]\n"
+ "fmla v9.8h, v7.8h, v0.h[6]\n"
+ "fmla v13.8h, v7.8h, v1.h[6]\n"
+ "ldr q7, [x15, #0x1b0]\n"
+ "fmla v10.8h, v6.8h, v0.h[6]\n"
+ "fmla v14.8h, v6.8h, v1.h[6]\n"
+ "ldr q6, [x15, #0x1c0]\n"
+ "fmla v11.8h, v7.8h, v0.h[6]\n"
+ "fmla v15.8h, v7.8h, v1.h[6]\n"
+ "ldr q7, [x15, #0x1d0]\n"
+ "fmla v8.8h, v6.8h, v0.h[7]\n"
+ "fmla v12.8h, v6.8h, v1.h[7]\n"
+ "ldr q6, [x15, #0x1e0]\n"
+ "fmla v9.8h, v7.8h, v0.h[7]\n"
+ "fmla v13.8h, v7.8h, v1.h[7]\n"
+ "ldr q7, [x15, #0x1f0]\n"
+ "add x15, x15, #0x200\n"
+ "fmla v10.8h, v6.8h, v0.h[7]\n"
+ "fmla v14.8h, v6.8h, v1.h[7]\n"
+ "fmla v11.8h, v7.8h, v0.h[7]\n"
+ "fmla v15.8h, v7.8h, v1.h[7]\n"
+ "79:" // Height 2: Multiply loop: Main loop skip
+ "cbz x11, 81f\n"
+ "80:" // Height 2: Multiply loop: Odd block loop
+ "ldr h0, [x10], #0x2\n"
+ "ldr h1, [x28], #0x2\n"
+ "ldr q6, [x15, #0x0]\n"
+ "fmla v8.8h, v6.8h, v0.h[0]\n"
+ "ldr q7, [x15, #0x10]\n"
+ "fmla v12.8h, v6.8h, v1.h[0]\n"
+ "ldr q6, [x15, #0x20]\n"
+ "sub x11, x11, #0x1\n"
+ "fmla v9.8h, v7.8h, v0.h[0]\n"
+ "fmla v13.8h, v7.8h, v1.h[0]\n"
+ "ldr q7, [x15, #0x30]\n"
+ "fmla v10.8h, v6.8h, v0.h[0]\n"
+ "add x15, x15, #0x40\n"
+ "fmla v14.8h, v6.8h, v1.h[0]\n"
+ "fmla v11.8h, v7.8h, v0.h[0]\n"
+ "fmla v15.8h, v7.8h, v1.h[0]\n"
+ "cbnz x11, 80b\n"
+ "81:" // Height 2: Multiply loop: No odd multiplies
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "add x12, x12, #0x1\n"
+ "cmp x12, x19\n"
+ "bne 74b\n"
+ "prfm pstl1keep, [x13, #0x0]\n"
+ "prfm pstl1keep, [x9, #0x0]\n"
+ "tbz %x[flags], #1, 82f\n"
+ "add x19, %x[args_ptr], %[offset_min]\n"
+ "ld1r { v1.8h }, [x19]\n"
+ "add x19, %x[args_ptr], %[offset_max]\n"
+ "ld1r { v0.8h }, [x19]\n"
+ "fmin v8.8h, v8.8h, v0.8h\n"
+ "fmin v9.8h, v9.8h, v0.8h\n"
+ "fmin v10.8h, v10.8h, v0.8h\n"
+ "fmin v11.8h, v11.8h, v0.8h\n"
+ "fmax v8.8h, v8.8h, v1.8h\n"
+ "fmax v9.8h, v9.8h, v1.8h\n"
+ "fmax v10.8h, v10.8h, v1.8h\n"
+ "fmax v11.8h, v11.8h, v1.8h\n"
+ "fmin v12.8h, v12.8h, v0.8h\n"
+ "fmin v13.8h, v13.8h, v0.8h\n"
+ "fmin v14.8h, v14.8h, v0.8h\n"
+ "fmax v12.8h, v12.8h, v1.8h\n"
+ "fmax v13.8h, v13.8h, v1.8h\n"
+ "fmax v14.8h, v14.8h, v1.8h\n"
+ "fmin v15.8h, v15.8h, v0.8h\n"
+ "fmax v15.8h, v15.8h, v1.8h\n"
+ "82:" // Height 2: No activation
+ "cmp x16, #0x20\n"
+ "bge 99f\n"
+ "tbz x16, #4, 90f\n"
+ "st1 { v8.8h }, [x13], #0x10\n"
+ "st1 { v9.8h }, [x13], #0x10\n"
+ "st1 { v12.8h }, [x9], #0x10\n"
+ "st1 { v13.8h }, [x9], #0x10\n"
+ "tbz x16, #3, 86f\n"
+ "st1 { v10.8h }, [x13], #0x10\n"
+ "st1 { v14.8h }, [x9], #0x10\n"
+ "tbz x16, #2, 84f\n"
+ "str d11, [x13], #0x8\n"
+ "str d15, [x9], #0x8\n"
+ "tbz x16, #1, 83f\n"
+ "st1 { v11.s }[2], [x13], #0x4\n"
+ "st1 { v15.s }[2], [x9], #0x4\n"
+ "tbz x16, #0, 98f\n"
+ "st1 { v11.h }[6], [x13]\n"
+ "st1 { v15.h }[6], [x9]\n"
+ "b 98f\n"
+ "83:" // Height 2: Partial direct writeback: partial_1_28
+ "tbz x16, #0, 98f\n"
+ "st1 { v11.h }[4], [x13]\n"
+ "st1 { v15.h }[4], [x9]\n"
+ "b 98f\n"
+ "84:" // Height 2: Partial direct writeback: partial_2_24
+ "tbz x16, #1, 85f\n"
+ "str s11, [x13], #0x4\n"
+ "str s15, [x9], #0x4\n"
+ "tbz x16, #0, 98f\n"
+ "st1 { v11.h }[2], [x13]\n"
+ "st1 { v15.h }[2], [x9]\n"
+ "b 98f\n"
+ "85:" // Height 2: Partial direct writeback: partial_1_24
+ "tbz x16, #0, 98f\n"
+ "str h11, [x13, #0x0]\n"
+ "str h15, [x9, #0x0]\n"
+ "b 98f\n"
+ "86:" // Height 2: Partial direct writeback: partial_4_16
+ "tbz x16, #2, 88f\n"
+ "str d10, [x13], #0x8\n"
+ "str d14, [x9], #0x8\n"
+ "tbz x16, #1, 87f\n"
+ "st1 { v10.s }[2], [x13], #0x4\n"
+ "st1 { v14.s }[2], [x9], #0x4\n"
+ "tbz x16, #0, 98f\n"
+ "st1 { v10.h }[6], [x13]\n"
+ "st1 { v14.h }[6], [x9]\n"
+ "b 98f\n"
+ "87:" // Height 2: Partial direct writeback: partial_1_20
+ "tbz x16, #0, 98f\n"
+ "st1 { v10.h }[4], [x13]\n"
+ "st1 { v14.h }[4], [x9]\n"
+ "b 98f\n"
+ "88:" // Height 2: Partial direct writeback: partial_2_16
+ "tbz x16, #1, 89f\n"
+ "str s10, [x13], #0x4\n"
+ "str s14, [x9], #0x4\n"
+ "tbz x16, #0, 98f\n"
+ "st1 { v10.h }[2], [x13]\n"
+ "st1 { v14.h }[2], [x9]\n"
+ "b 98f\n"
+ "89:" // Height 2: Partial direct writeback: partial_1_16
+ "tbz x16, #0, 98f\n"
+ "str h10, [x13, #0x0]\n"
+ "str h14, [x9, #0x0]\n"
+ "b 98f\n"
+ "90:" // Height 2: Partial direct writeback: partial_8_0
+ "tbz x16, #3, 94f\n"
+ "st1 { v8.8h }, [x13], #0x10\n"
+ "st1 { v12.8h }, [x9], #0x10\n"
+ "tbz x16, #2, 92f\n"
+ "str d9, [x13], #0x8\n"
+ "str d13, [x9], #0x8\n"
+ "tbz x16, #1, 91f\n"
+ "st1 { v9.s }[2], [x13], #0x4\n"
+ "st1 { v13.s }[2], [x9], #0x4\n"
+ "tbz x16, #0, 98f\n"
+ "st1 { v9.h }[6], [x13]\n"
+ "st1 { v13.h }[6], [x9]\n"
+ "b 98f\n"
+ "91:" // Height 2: Partial direct writeback: partial_1_12
+ "tbz x16, #0, 98f\n"
+ "st1 { v9.h }[4], [x13]\n"
+ "st1 { v13.h }[4], [x9]\n"
+ "b 98f\n"
+ "92:" // Height 2: Partial direct writeback: partial_2_8
+ "tbz x16, #1, 93f\n"
+ "str s9, [x13], #0x4\n"
+ "str s13, [x9], #0x4\n"
+ "tbz x16, #0, 98f\n"
+ "st1 { v9.h }[2], [x13]\n"
+ "st1 { v13.h }[2], [x9]\n"
+ "b 98f\n"
+ "93:" // Height 2: Partial direct writeback: partial_1_8
+ "tbz x16, #0, 98f\n"
+ "str h9, [x13, #0x0]\n"
+ "str h13, [x9, #0x0]\n"
+ "b 98f\n"
+ "94:" // Height 2: Partial direct writeback: partial_4_0
+ "tbz x16, #2, 96f\n"
+ "str d8, [x13], #0x8\n"
+ "str d12, [x9], #0x8\n"
+ "tbz x16, #1, 95f\n"
+ "st1 { v8.s }[2], [x13], #0x4\n"
+ "st1 { v12.s }[2], [x9], #0x4\n"
+ "tbz x16, #0, 98f\n"
+ "st1 { v8.h }[6], [x13]\n"
+ "st1 { v12.h }[6], [x9]\n"
+ "b 98f\n"
+ "95:" // Height 2: Partial direct writeback: partial_1_4
+ "tbz x16, #0, 98f\n"
+ "st1 { v8.h }[4], [x13]\n"
+ "st1 { v12.h }[4], [x9]\n"
+ "b 98f\n"
+ "96:" // Height 2: Partial direct writeback: partial_2_0
+ "tbz x16, #1, 97f\n"
+ "str s8, [x13], #0x4\n"
+ "str s12, [x9], #0x4\n"
+ "tbz x16, #0, 98f\n"
+ "st1 { v8.h }[2], [x13]\n"
+ "st1 { v12.h }[2], [x9]\n"
+ "b 98f\n"
+ "97:" // Height 2: Partial direct writeback: partial_1_0
+ "str h8, [x13, #0x0]\n"
+ "str h12, [x9, #0x0]\n"
+ "98:" // Height 2: Partial direct writeback: Done
+ "b 100f\n"
+ "99:" // Height 2: Full writeback
+ "str q8, [x13, #0x0]\n"
+ "str q9, [x13, #0x10]\n"
+ "str q10, [x13, #0x20]\n"
+ "str q11, [x13, #0x30]\n"
+ "str q12, [x9, #0x0]\n"
+ "str q13, [x9, #0x10]\n"
+ "str q14, [x9, #0x20]\n"
+ "str q15, [x9, #0x30]\n"
+ "add x13, x13, #0x40\n"
+ "add x9, x9, #0x40\n"
+ "100:" // Height 2: Writeback done
+ "subs x16, x16, #0x20\n"
+ "bgt 53b\n"
+ "b 302f\n"
+ "101:" // Height 3
+ "ldr x16, [%x[args_ptr], %[offsetof_N]]\n"
+ "mov x14, %x[bias]\n"
+ "ldr x15, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 102f\n"
+ "ldr x13, [%x[output_ptr], #0x0]\n"
+ "add x13, x13, x19, LSL #1\n"
+ "ldr x9, [%x[output_ptr], #0x8]\n"
+ "ldr x27, [%x[output_ptr], #0x10]\n"
+ "add x9, x9, x19, LSL #1\n"
+ "add x27, x27, x19, LSL #1\n"
+ "b 103f\n"
+ "102:" // Height 3: setup direct output
+ "mov x13, %x[output_ptr]\n"
+ "add x9, x13, x19, LSL #1\n"
+ "add x27, x9, x19, LSL #1\n"
+ "103:" // Height 3: Column loop
+ "cbz x14, 104f\n"
+ "ldr q8, [x14, #0x0]\n"
+ "mov v12.16b, v8.16b\n"
+ "ldr q9, [x14, #0x10]\n"
+ "mov v16.16b, v8.16b\n"
+ "ldr q10, [x14, #0x20]\n"
+ "ldr q11, [x14, #0x30]\n"
+ "mov v13.16b, v9.16b\n"
+ "add x14, x14, #0x40\n"
+ "mov v17.16b, v9.16b\n"
+ "mov v14.16b, v10.16b\n"
+ "mov v15.16b, v11.16b\n"
+ "mov v18.16b, v10.16b\n"
+ "mov v19.16b, v11.16b\n"
+ "b 123f\n"
+ "104:" // Height 3: no bias
+ "tbz %x[flags], #0, 122f\n"
+ "cmp x16, #0x20\n"
+ "bge 121f\n"
+ "tbz x16, #4, 112f\n"
+ "ld1 { v8.8h }, [x13], #0x10\n"
+ "ld1 { v12.8h }, [x9], #0x10\n"
+ "ld1 { v16.8h }, [x27], #0x10\n"
+ "ld1 { v9.8h }, [x13], #0x10\n"
+ "ld1 { v13.8h }, [x9], #0x10\n"
+ "ld1 { v17.8h }, [x27], #0x10\n"
+ "tbz x16, #3, 108f\n"
+ "ld1 { v10.8h }, [x13], #0x10\n"
+ "ld1 { v14.8h }, [x9], #0x10\n"
+ "ld1 { v18.8h }, [x27], #0x10\n"
+ "tbz x16, #2, 106f\n"
+ "ldr d11, [x13], #0x8\n"
+ "ldr d15, [x9], #0x8\n"
+ "ldr d19, [x27], #0x8\n"
+ "tbz x16, #1, 105f\n"
+ "mov x19, #0x3c\n"
+ "ld1 { v11.s }[2], [x13], #0x4\n"
+ "ld1 { v15.s }[2], [x9], #0x4\n"
+ "ld1 { v19.s }[2], [x27], #0x4\n"
+ "tbz x16, #0, 120f\n"
+ "ld1 { v11.h }[6], [x13]\n"
+ "ld1 { v15.h }[6], [x9]\n"
+ "ld1 { v19.h }[6], [x27]\n"
+ "b 120f\n"
+ "105:" // Height 3: Partial accumulate: partial_1_28
+ "mov x19, #0x38\n"
+ "tbz x16, #0, 120f\n"
+ "ld1 { v11.h }[4], [x13]\n"
+ "ld1 { v15.h }[4], [x9]\n"
+ "ld1 { v19.h }[4], [x27]\n"
+ "b 120f\n"
+ "106:" // Height 3: Partial accumulate: partial_2_24
+ "tbz x16, #1, 107f\n"
+ "ldr s11, [x13], #0x4\n"
+ "ldr s15, [x9], #0x4\n"
+ "ldr s19, [x27], #0x4\n"
+ "mov x19, #0x34\n"
+ "tbz x16, #0, 120f\n"
+ "ld1 { v11.h }[2], [x13]\n"
+ "ld1 { v15.h }[2], [x9]\n"
+ "ld1 { v19.h }[2], [x27]\n"
+ "b 120f\n"
+ "107:" // Height 3: Partial accumulate: partial_1_24
+ "mov x19, #0x30\n"
+ "tbz x16, #0, 120f\n"
+ "ldr h11, [x13, #0x0]\n"
+ "ldr h15, [x9, #0x0]\n"
+ "ldr h19, [x27, #0x0]\n"
+ "b 120f\n"
+ "108:" // Height 3: Partial accumulate: partial_4_16
+ "tbz x16, #2, 110f\n"
+ "ldr d10, [x13], #0x8\n"
+ "ldr d14, [x9], #0x8\n"
+ "ldr d18, [x27], #0x8\n"
+ "tbz x16, #1, 109f\n"
+ "mov x19, #0x2c\n"
+ "ld1 { v10.s }[2], [x13], #0x4\n"
+ "ld1 { v14.s }[2], [x9], #0x4\n"
+ "ld1 { v18.s }[2], [x27], #0x4\n"
+ "tbz x16, #0, 120f\n"
+ "ld1 { v10.h }[6], [x13]\n"
+ "ld1 { v14.h }[6], [x9]\n"
+ "ld1 { v18.h }[6], [x27]\n"
+ "b 120f\n"
+ "109:" // Height 3: Partial accumulate: partial_1_20
+ "mov x19, #0x28\n"
+ "tbz x16, #0, 120f\n"
+ "ld1 { v10.h }[4], [x13]\n"
+ "ld1 { v14.h }[4], [x9]\n"
+ "ld1 { v18.h }[4], [x27]\n"
+ "b 120f\n"
+ "110:" // Height 3: Partial accumulate: partial_2_16
+ "tbz x16, #1, 111f\n"
+ "ldr s10, [x13], #0x4\n"
+ "ldr s14, [x9], #0x4\n"
+ "ldr s18, [x27], #0x4\n"
+ "mov x19, #0x24\n"
+ "tbz x16, #0, 120f\n"
+ "ld1 { v10.h }[2], [x13]\n"
+ "ld1 { v14.h }[2], [x9]\n"
+ "ld1 { v18.h }[2], [x27]\n"
+ "b 120f\n"
+ "111:" // Height 3: Partial accumulate: partial_1_16
+ "mov x19, #0x20\n"
+ "tbz x16, #0, 120f\n"
+ "ldr h10, [x13, #0x0]\n"
+ "ldr h14, [x9, #0x0]\n"
+ "ldr h18, [x27, #0x0]\n"
+ "b 120f\n"
+ "112:" // Height 3: Partial accumulate: partial_8_0
+ "tbz x16, #3, 116f\n"
+ "ld1 { v8.8h }, [x13], #0x10\n"
+ "ld1 { v12.8h }, [x9], #0x10\n"
+ "ld1 { v16.8h }, [x27], #0x10\n"
+ "tbz x16, #2, 114f\n"
+ "ldr d9, [x13], #0x8\n"
+ "ldr d13, [x9], #0x8\n"
+ "ldr d17, [x27], #0x8\n"
+ "tbz x16, #1, 113f\n"
+ "mov x19, #0x1c\n"
+ "ld1 { v9.s }[2], [x13], #0x4\n"
+ "ld1 { v13.s }[2], [x9], #0x4\n"
+ "ld1 { v17.s }[2], [x27], #0x4\n"
+ "tbz x16, #0, 120f\n"
+ "ld1 { v9.h }[6], [x13]\n"
+ "ld1 { v13.h }[6], [x9]\n"
+ "ld1 { v17.h }[6], [x27]\n"
+ "b 120f\n"
+ "113:" // Height 3: Partial accumulate: partial_1_12
+ "mov x19, #0x18\n"
+ "tbz x16, #0, 120f\n"
+ "ld1 { v9.h }[4], [x13]\n"
+ "ld1 { v13.h }[4], [x9]\n"
+ "ld1 { v17.h }[4], [x27]\n"
+ "b 120f\n"
+ "114:" // Height 3: Partial accumulate: partial_2_8
+ "tbz x16, #1, 115f\n"
+ "ldr s9, [x13], #0x4\n"
+ "ldr s13, [x9], #0x4\n"
+ "ldr s17, [x27], #0x4\n"
+ "mov x19, #0x14\n"
+ "tbz x16, #0, 120f\n"
+ "ld1 { v9.h }[2], [x13]\n"
+ "ld1 { v13.h }[2], [x9]\n"
+ "ld1 { v17.h }[2], [x27]\n"
+ "b 120f\n"
+ "115:" // Height 3: Partial accumulate: partial_1_8
+ "mov x19, #0x10\n"
+ "tbz x16, #0, 120f\n"
+ "ldr h9, [x13, #0x0]\n"
+ "ldr h13, [x9, #0x0]\n"
+ "ldr h17, [x27, #0x0]\n"
+ "b 120f\n"
+ "116:" // Height 3: Partial accumulate: partial_4_0
+ "tbz x16, #2, 118f\n"
+ "ldr d8, [x13], #0x8\n"
+ "ldr d12, [x9], #0x8\n"
+ "ldr d16, [x27], #0x8\n"
+ "tbz x16, #1, 117f\n"
+ "mov x19, #0xc\n"
+ "ld1 { v8.s }[2], [x13], #0x4\n"
+ "ld1 { v12.s }[2], [x9], #0x4\n"
+ "ld1 { v16.s }[2], [x27], #0x4\n"
+ "tbz x16, #0, 120f\n"
+ "ld1 { v8.h }[6], [x13]\n"
+ "ld1 { v12.h }[6], [x9]\n"
+ "ld1 { v16.h }[6], [x27]\n"
+ "b 120f\n"
+ "117:" // Height 3: Partial accumulate: partial_1_4
+ "mov x19, #0x8\n"
+ "tbz x16, #0, 120f\n"
+ "ld1 { v8.h }[4], [x13]\n"
+ "ld1 { v12.h }[4], [x9]\n"
+ "ld1 { v16.h }[4], [x27]\n"
+ "b 120f\n"
+ "118:" // Height 3: Partial accumulate: partial_2_0
+ "tbz x16, #1, 119f\n"
+ "ldr s8, [x13], #0x4\n"
+ "ldr s12, [x9], #0x4\n"
+ "ldr s16, [x27], #0x4\n"
+ "mov x19, #0x4\n"
+ "tbz x16, #0, 120f\n"
+ "ld1 { v8.h }[2], [x13]\n"
+ "ld1 { v12.h }[2], [x9]\n"
+ "ld1 { v16.h }[2], [x27]\n"
+ "b 120f\n"
+ "119:" // Height 3: Partial accumulate: partial_1_0
+ "mov x19, #0x0\n"
+ "ldr h8, [x13, #0x0]\n"
+ "ldr h12, [x9, #0x0]\n"
+ "ldr h16, [x27, #0x0]\n"
+ "120:" // Height 3: Partial accumulate: Done
+ "sub x13, x13, x19\n"
+ "sub x9, x9, x19\n"
+ "sub x27, x27, x19\n"
+ "b 123f\n"
+ "121:" // Height 3: full accumulate
+ "ldr q8, [x13, #0x0]\n"
+ "ldr q9, [x13, #0x10]\n"
+ "ldr q10, [x13, #0x20]\n"
+ "ldr q11, [x13, #0x30]\n"
+ "ldr q12, [x9, #0x0]\n"
+ "ldr q13, [x9, #0x10]\n"
+ "ldr q14, [x9, #0x20]\n"
+ "ldr q15, [x9, #0x30]\n"
+ "ldr q16, [x27, #0x0]\n"
+ "ldr q17, [x27, #0x10]\n"
+ "ldr q18, [x27, #0x20]\n"
+ "ldr q19, [x27, #0x30]\n"
+ "b 123f\n"
+ "122:" // Height 3: no accumulate
+ "movi v8.16b, #0x0\n"
+ "movi v9.16b, #0x0\n"
+ "movi v10.16b, #0x0\n"
+ "movi v11.16b, #0x0\n"
+ "movi v12.16b, #0x0\n"
+ "movi v13.16b, #0x0\n"
+ "movi v14.16b, #0x0\n"
+ "movi v15.16b, #0x0\n"
+ "movi v16.16b, #0x0\n"
+ "movi v17.16b, #0x0\n"
+ "movi v18.16b, #0x0\n"
+ "movi v19.16b, #0x0\n"
+ "123:" // Height 3: setup done
+ "mov x12, #0x0\n"
+ "124:" // Height 3: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w11, [x20, x12, LSL #0x2]\n"
+ "tbz %x[flags], #3, 125f\n"
+ "ldr x20, [%x[input_ptr], x12, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x10, [x20, #0x0]\n"
+ "ldr x28, [x20, #0x8]\n"
+ "ldr x26, [x20, #0x10]\n"
+ "cbnz x12, 126f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x10, x10, x19, LSL #1\n"
+ "add x28, x28, x19, LSL #1\n"
+ "add x26, x26, x19, LSL #1\n"
+ "b 126f\n"
+ "125:" // Height 3: setup direct input
+ "mov x10, %x[input_ptr]\n"
+ "add x28, x10, x19, LSL #1\n"
+ "add x26, x28, x19, LSL #1\n"
+ "126:" // Height 3: input setup done
+ "cmp x11, #0x8\n"
+ "blt 129f\n"
+ "cmp x11, #0x10\n"
+ "blt 128f\n"
+ "127:" // Height 3: Multiply loop: Main loop head
+ "ldr q0, [x10, #0x0]\n"
+ "ldr q1, [x28, #0x0]\n"
+ "ldr q2, [x26, #0x0]\n"
+ "ldr q6, [x15, #0x0]\n"
+ "fmla v8.8h, v6.8h, v0.h[0]\n"
+ "ldr q7, [x15, #0x10]\n"
+ "fmla v12.8h, v6.8h, v1.h[0]\n"
+ "add x10, x10, #0x10\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "fmla v16.8h, v6.8h, v2.h[0]\n"
+ "ldr q6, [x15, #0x20]\n"
+ "fmla v9.8h, v7.8h, v0.h[0]\n"
+ "add x28, x28, #0x10\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "fmla v13.8h, v7.8h, v1.h[0]\n"
+ "add x26, x26, #0x10\n"
+ "fmla v17.8h, v7.8h, v2.h[0]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "ldr q7, [x15, #0x30]\n"
+ "fmla v10.8h, v6.8h, v0.h[0]\n"
+ "sub x11, x11, #0x8\n"
+ "fmla v14.8h, v6.8h, v1.h[0]\n"
+ "cmp x11, #0x10\n"
+ "fmla v18.8h, v6.8h, v2.h[0]\n"
+ "ldr q6, [x15, #0x40]\n"
+ "fmla v11.8h, v7.8h, v0.h[0]\n"
+ "fmla v15.8h, v7.8h, v1.h[0]\n"
+ "fmla v19.8h, v7.8h, v2.h[0]\n"
+ "ldr q7, [x15, #0x50]\n"
+ "fmla v8.8h, v6.8h, v0.h[1]\n"
+ "fmla v12.8h, v6.8h, v1.h[1]\n"
+ "fmla v16.8h, v6.8h, v2.h[1]\n"
+ "ldr q6, [x15, #0x60]\n"
+ "fmla v9.8h, v7.8h, v0.h[1]\n"
+ "fmla v13.8h, v7.8h, v1.h[1]\n"
+ "fmla v17.8h, v7.8h, v2.h[1]\n"
+ "ldr q7, [x15, #0x70]\n"
+ "fmla v10.8h, v6.8h, v0.h[1]\n"
+ "fmla v14.8h, v6.8h, v1.h[1]\n"
+ "fmla v18.8h, v6.8h, v2.h[1]\n"
+ "ldr q6, [x15, #0x80]\n"
+ "fmla v11.8h, v7.8h, v0.h[1]\n"
+ "fmla v15.8h, v7.8h, v1.h[1]\n"
+ "fmla v19.8h, v7.8h, v2.h[1]\n"
+ "ldr q7, [x15, #0x90]\n"
+ "fmla v8.8h, v6.8h, v0.h[2]\n"
+ "fmla v12.8h, v6.8h, v1.h[2]\n"
+ "fmla v16.8h, v6.8h, v2.h[2]\n"
+ "ldr q6, [x15, #0xa0]\n"
+ "fmla v9.8h, v7.8h, v0.h[2]\n"
+ "fmla v13.8h, v7.8h, v1.h[2]\n"
+ "fmla v17.8h, v7.8h, v2.h[2]\n"
+ "ldr q7, [x15, #0xb0]\n"
+ "fmla v10.8h, v6.8h, v0.h[2]\n"
+ "fmla v14.8h, v6.8h, v1.h[2]\n"
+ "fmla v18.8h, v6.8h, v2.h[2]\n"
+ "ldr q6, [x15, #0xc0]\n"
+ "fmla v11.8h, v7.8h, v0.h[2]\n"
+ "fmla v15.8h, v7.8h, v1.h[2]\n"
+ "fmla v19.8h, v7.8h, v2.h[2]\n"
+ "ldr q7, [x15, #0xd0]\n"
+ "fmla v8.8h, v6.8h, v0.h[3]\n"
+ "fmla v12.8h, v6.8h, v1.h[3]\n"
+ "fmla v16.8h, v6.8h, v2.h[3]\n"
+ "ldr q6, [x15, #0xe0]\n"
+ "fmla v9.8h, v7.8h, v0.h[3]\n"
+ "fmla v13.8h, v7.8h, v1.h[3]\n"
+ "fmla v17.8h, v7.8h, v2.h[3]\n"
+ "ldr q7, [x15, #0xf0]\n"
+ "fmla v10.8h, v6.8h, v0.h[3]\n"
+ "fmla v14.8h, v6.8h, v1.h[3]\n"
+ "fmla v18.8h, v6.8h, v2.h[3]\n"
+ "ldr q6, [x15, #0x100]\n"
+ "fmla v11.8h, v7.8h, v0.h[3]\n"
+ "fmla v15.8h, v7.8h, v1.h[3]\n"
+ "fmla v19.8h, v7.8h, v2.h[3]\n"
+ "ldr q7, [x15, #0x110]\n"
+ "fmla v8.8h, v6.8h, v0.h[4]\n"
+ "fmla v12.8h, v6.8h, v1.h[4]\n"
+ "fmla v16.8h, v6.8h, v2.h[4]\n"
+ "ldr q6, [x15, #0x120]\n"
+ "fmla v9.8h, v7.8h, v0.h[4]\n"
+ "fmla v13.8h, v7.8h, v1.h[4]\n"
+ "fmla v17.8h, v7.8h, v2.h[4]\n"
+ "ldr q7, [x15, #0x130]\n"
+ "fmla v10.8h, v6.8h, v0.h[4]\n"
+ "fmla v14.8h, v6.8h, v1.h[4]\n"
+ "fmla v18.8h, v6.8h, v2.h[4]\n"
+ "ldr q6, [x15, #0x140]\n"
+ "fmla v11.8h, v7.8h, v0.h[4]\n"
+ "fmla v15.8h, v7.8h, v1.h[4]\n"
+ "fmla v19.8h, v7.8h, v2.h[4]\n"
+ "ldr q7, [x15, #0x150]\n"
+ "fmla v8.8h, v6.8h, v0.h[5]\n"
+ "fmla v12.8h, v6.8h, v1.h[5]\n"
+ "fmla v16.8h, v6.8h, v2.h[5]\n"
+ "ldr q6, [x15, #0x160]\n"
+ "fmla v9.8h, v7.8h, v0.h[5]\n"
+ "fmla v13.8h, v7.8h, v1.h[5]\n"
+ "fmla v17.8h, v7.8h, v2.h[5]\n"
+ "ldr q7, [x15, #0x170]\n"
+ "fmla v10.8h, v6.8h, v0.h[5]\n"
+ "fmla v14.8h, v6.8h, v1.h[5]\n"
+ "fmla v18.8h, v6.8h, v2.h[5]\n"
+ "ldr q6, [x15, #0x180]\n"
+ "fmla v11.8h, v7.8h, v0.h[5]\n"
+ "fmla v15.8h, v7.8h, v1.h[5]\n"
+ "fmla v19.8h, v7.8h, v2.h[5]\n"
+ "ldr q7, [x15, #0x190]\n"
+ "fmla v8.8h, v6.8h, v0.h[6]\n"
+ "fmla v12.8h, v6.8h, v1.h[6]\n"
+ "fmla v16.8h, v6.8h, v2.h[6]\n"
+ "ldr q6, [x15, #0x1a0]\n"
+ "fmla v9.8h, v7.8h, v0.h[6]\n"
+ "fmla v13.8h, v7.8h, v1.h[6]\n"
+ "fmla v17.8h, v7.8h, v2.h[6]\n"
+ "ldr q7, [x15, #0x1b0]\n"
+ "fmla v10.8h, v6.8h, v0.h[6]\n"
+ "fmla v14.8h, v6.8h, v1.h[6]\n"
+ "fmla v18.8h, v6.8h, v2.h[6]\n"
+ "ldr q6, [x15, #0x1c0]\n"
+ "fmla v11.8h, v7.8h, v0.h[6]\n"
+ "fmla v15.8h, v7.8h, v1.h[6]\n"
+ "fmla v19.8h, v7.8h, v2.h[6]\n"
+ "ldr q7, [x15, #0x1d0]\n"
+ "fmla v8.8h, v6.8h, v0.h[7]\n"
+ "fmla v12.8h, v6.8h, v1.h[7]\n"
+ "fmla v16.8h, v6.8h, v2.h[7]\n"
+ "ldr q6, [x15, #0x1e0]\n"
+ "fmla v9.8h, v7.8h, v0.h[7]\n"
+ "fmla v13.8h, v7.8h, v1.h[7]\n"
+ "fmla v17.8h, v7.8h, v2.h[7]\n"
+ "ldr q7, [x15, #0x1f0]\n"
+ "add x15, x15, #0x200\n"
+ "fmla v10.8h, v6.8h, v0.h[7]\n"
+ "fmla v14.8h, v6.8h, v1.h[7]\n"
+ "fmla v18.8h, v6.8h, v2.h[7]\n"
+ "fmla v11.8h, v7.8h, v0.h[7]\n"
+ "fmla v15.8h, v7.8h, v1.h[7]\n"
+ "fmla v19.8h, v7.8h, v2.h[7]\n"
+ "bge 127b\n"
+ "128:" // Height 3: Multiply loop: Single iteration only
+ "sub x11, x11, #0x8\n"
+ "ldr q0, [x10, #0x0]\n"
+ "ldr q1, [x28, #0x0]\n"
+ "ldr q2, [x26, #0x0]\n"
+ "ldr q6, [x15, #0x0]\n"
+ "fmla v8.8h, v6.8h, v0.h[0]\n"
+ "ldr q7, [x15, #0x10]\n"
+ "fmla v12.8h, v6.8h, v1.h[0]\n"
+ "add x10, x10, #0x10\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "fmla v16.8h, v6.8h, v2.h[0]\n"
+ "ldr q6, [x15, #0x20]\n"
+ "fmla v9.8h, v7.8h, v0.h[0]\n"
+ "add x28, x28, #0x10\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "fmla v13.8h, v7.8h, v1.h[0]\n"
+ "add x26, x26, #0x10\n"
+ "fmla v17.8h, v7.8h, v2.h[0]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "ldr q7, [x15, #0x30]\n"
+ "fmla v10.8h, v6.8h, v0.h[0]\n"
+ "fmla v14.8h, v6.8h, v1.h[0]\n"
+ "fmla v18.8h, v6.8h, v2.h[0]\n"
+ "ldr q6, [x15, #0x40]\n"
+ "fmla v11.8h, v7.8h, v0.h[0]\n"
+ "fmla v15.8h, v7.8h, v1.h[0]\n"
+ "fmla v19.8h, v7.8h, v2.h[0]\n"
+ "ldr q7, [x15, #0x50]\n"
+ "fmla v8.8h, v6.8h, v0.h[1]\n"
+ "fmla v12.8h, v6.8h, v1.h[1]\n"
+ "fmla v16.8h, v6.8h, v2.h[1]\n"
+ "ldr q6, [x15, #0x60]\n"
+ "fmla v9.8h, v7.8h, v0.h[1]\n"
+ "fmla v13.8h, v7.8h, v1.h[1]\n"
+ "fmla v17.8h, v7.8h, v2.h[1]\n"
+ "ldr q7, [x15, #0x70]\n"
+ "fmla v10.8h, v6.8h, v0.h[1]\n"
+ "fmla v14.8h, v6.8h, v1.h[1]\n"
+ "fmla v18.8h, v6.8h, v2.h[1]\n"
+ "ldr q6, [x15, #0x80]\n"
+ "fmla v11.8h, v7.8h, v0.h[1]\n"
+ "fmla v15.8h, v7.8h, v1.h[1]\n"
+ "fmla v19.8h, v7.8h, v2.h[1]\n"
+ "ldr q7, [x15, #0x90]\n"
+ "fmla v8.8h, v6.8h, v0.h[2]\n"
+ "fmla v12.8h, v6.8h, v1.h[2]\n"
+ "fmla v16.8h, v6.8h, v2.h[2]\n"
+ "ldr q6, [x15, #0xa0]\n"
+ "fmla v9.8h, v7.8h, v0.h[2]\n"
+ "fmla v13.8h, v7.8h, v1.h[2]\n"
+ "fmla v17.8h, v7.8h, v2.h[2]\n"
+ "ldr q7, [x15, #0xb0]\n"
+ "fmla v10.8h, v6.8h, v0.h[2]\n"
+ "fmla v14.8h, v6.8h, v1.h[2]\n"
+ "fmla v18.8h, v6.8h, v2.h[2]\n"
+ "ldr q6, [x15, #0xc0]\n"
+ "fmla v11.8h, v7.8h, v0.h[2]\n"
+ "fmla v15.8h, v7.8h, v1.h[2]\n"
+ "fmla v19.8h, v7.8h, v2.h[2]\n"
+ "ldr q7, [x15, #0xd0]\n"
+ "fmla v8.8h, v6.8h, v0.h[3]\n"
+ "fmla v12.8h, v6.8h, v1.h[3]\n"
+ "fmla v16.8h, v6.8h, v2.h[3]\n"
+ "ldr q6, [x15, #0xe0]\n"
+ "fmla v9.8h, v7.8h, v0.h[3]\n"
+ "fmla v13.8h, v7.8h, v1.h[3]\n"
+ "fmla v17.8h, v7.8h, v2.h[3]\n"
+ "ldr q7, [x15, #0xf0]\n"
+ "fmla v10.8h, v6.8h, v0.h[3]\n"
+ "fmla v14.8h, v6.8h, v1.h[3]\n"
+ "fmla v18.8h, v6.8h, v2.h[3]\n"
+ "ldr q6, [x15, #0x100]\n"
+ "fmla v11.8h, v7.8h, v0.h[3]\n"
+ "fmla v15.8h, v7.8h, v1.h[3]\n"
+ "fmla v19.8h, v7.8h, v2.h[3]\n"
+ "ldr q7, [x15, #0x110]\n"
+ "fmla v8.8h, v6.8h, v0.h[4]\n"
+ "fmla v12.8h, v6.8h, v1.h[4]\n"
+ "fmla v16.8h, v6.8h, v2.h[4]\n"
+ "ldr q6, [x15, #0x120]\n"
+ "fmla v9.8h, v7.8h, v0.h[4]\n"
+ "fmla v13.8h, v7.8h, v1.h[4]\n"
+ "fmla v17.8h, v7.8h, v2.h[4]\n"
+ "ldr q7, [x15, #0x130]\n"
+ "fmla v10.8h, v6.8h, v0.h[4]\n"
+ "fmla v14.8h, v6.8h, v1.h[4]\n"
+ "fmla v18.8h, v6.8h, v2.h[4]\n"
+ "ldr q6, [x15, #0x140]\n"
+ "fmla v11.8h, v7.8h, v0.h[4]\n"
+ "fmla v15.8h, v7.8h, v1.h[4]\n"
+ "fmla v19.8h, v7.8h, v2.h[4]\n"
+ "ldr q7, [x15, #0x150]\n"
+ "fmla v8.8h, v6.8h, v0.h[5]\n"
+ "fmla v12.8h, v6.8h, v1.h[5]\n"
+ "fmla v16.8h, v6.8h, v2.h[5]\n"
+ "ldr q6, [x15, #0x160]\n"
+ "fmla v9.8h, v7.8h, v0.h[5]\n"
+ "fmla v13.8h, v7.8h, v1.h[5]\n"
+ "fmla v17.8h, v7.8h, v2.h[5]\n"
+ "ldr q7, [x15, #0x170]\n"
+ "fmla v10.8h, v6.8h, v0.h[5]\n"
+ "fmla v14.8h, v6.8h, v1.h[5]\n"
+ "fmla v18.8h, v6.8h, v2.h[5]\n"
+ "ldr q6, [x15, #0x180]\n"
+ "fmla v11.8h, v7.8h, v0.h[5]\n"
+ "fmla v15.8h, v7.8h, v1.h[5]\n"
+ "fmla v19.8h, v7.8h, v2.h[5]\n"
+ "ldr q7, [x15, #0x190]\n"
+ "fmla v8.8h, v6.8h, v0.h[6]\n"
+ "fmla v12.8h, v6.8h, v1.h[6]\n"
+ "fmla v16.8h, v6.8h, v2.h[6]\n"
+ "ldr q6, [x15, #0x1a0]\n"
+ "fmla v9.8h, v7.8h, v0.h[6]\n"
+ "fmla v13.8h, v7.8h, v1.h[6]\n"
+ "fmla v17.8h, v7.8h, v2.h[6]\n"
+ "ldr q7, [x15, #0x1b0]\n"
+ "fmla v10.8h, v6.8h, v0.h[6]\n"
+ "fmla v14.8h, v6.8h, v1.h[6]\n"
+ "fmla v18.8h, v6.8h, v2.h[6]\n"
+ "ldr q6, [x15, #0x1c0]\n"
+ "fmla v11.8h, v7.8h, v0.h[6]\n"
+ "fmla v15.8h, v7.8h, v1.h[6]\n"
+ "fmla v19.8h, v7.8h, v2.h[6]\n"
+ "ldr q7, [x15, #0x1d0]\n"
+ "fmla v8.8h, v6.8h, v0.h[7]\n"
+ "fmla v12.8h, v6.8h, v1.h[7]\n"
+ "fmla v16.8h, v6.8h, v2.h[7]\n"
+ "ldr q6, [x15, #0x1e0]\n"
+ "fmla v9.8h, v7.8h, v0.h[7]\n"
+ "fmla v13.8h, v7.8h, v1.h[7]\n"
+ "fmla v17.8h, v7.8h, v2.h[7]\n"
+ "ldr q7, [x15, #0x1f0]\n"
+ "add x15, x15, #0x200\n"
+ "fmla v10.8h, v6.8h, v0.h[7]\n"
+ "fmla v14.8h, v6.8h, v1.h[7]\n"
+ "fmla v18.8h, v6.8h, v2.h[7]\n"
+ "fmla v11.8h, v7.8h, v0.h[7]\n"
+ "fmla v15.8h, v7.8h, v1.h[7]\n"
+ "fmla v19.8h, v7.8h, v2.h[7]\n"
+ "129:" // Height 3: Multiply loop: Main loop skip
+ "cbz x11, 131f\n"
+ "130:" // Height 3: Multiply loop: Odd block loop
+ "ldr h0, [x10], #0x2\n"
+ "ldr h1, [x28], #0x2\n"
+ "ldr h2, [x26], #0x2\n"
+ "ldr q6, [x15, #0x0]\n"
+ "fmla v8.8h, v6.8h, v0.h[0]\n"
+ "ldr q7, [x15, #0x10]\n"
+ "fmla v12.8h, v6.8h, v1.h[0]\n"
+ "sub x11, x11, #0x1\n"
+ "fmla v16.8h, v6.8h, v2.h[0]\n"
+ "ldr q6, [x15, #0x20]\n"
+ "fmla v9.8h, v7.8h, v0.h[0]\n"
+ "fmla v13.8h, v7.8h, v1.h[0]\n"
+ "fmla v17.8h, v7.8h, v2.h[0]\n"
+ "ldr q7, [x15, #0x30]\n"
+ "fmla v10.8h, v6.8h, v0.h[0]\n"
+ "add x15, x15, #0x40\n"
+ "fmla v14.8h, v6.8h, v1.h[0]\n"
+ "fmla v18.8h, v6.8h, v2.h[0]\n"
+ "fmla v11.8h, v7.8h, v0.h[0]\n"
+ "fmla v15.8h, v7.8h, v1.h[0]\n"
+ "fmla v19.8h, v7.8h, v2.h[0]\n"
+ "cbnz x11, 130b\n"
+ "131:" // Height 3: Multiply loop: No odd multiplies
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "add x12, x12, #0x1\n"
+ "cmp x12, x19\n"
+ "bne 124b\n"
+ "prfm pstl1keep, [x13, #0x0]\n"
+ "prfm pstl1keep, [x9, #0x0]\n"
+ "prfm pstl1keep, [x27, #0x0]\n"
+ "tbz %x[flags], #1, 132f\n"
+ "add x19, %x[args_ptr], %[offset_min]\n"
+ "ld1r { v1.8h }, [x19]\n"
+ "add x19, %x[args_ptr], %[offset_max]\n"
+ "ld1r { v0.8h }, [x19]\n"
+ "fmin v8.8h, v8.8h, v0.8h\n"
+ "fmin v9.8h, v9.8h, v0.8h\n"
+ "fmin v10.8h, v10.8h, v0.8h\n"
+ "fmin v11.8h, v11.8h, v0.8h\n"
+ "fmax v8.8h, v8.8h, v1.8h\n"
+ "fmax v9.8h, v9.8h, v1.8h\n"
+ "fmax v10.8h, v10.8h, v1.8h\n"
+ "fmax v11.8h, v11.8h, v1.8h\n"
+ "fmin v12.8h, v12.8h, v0.8h\n"
+ "fmin v13.8h, v13.8h, v0.8h\n"
+ "fmin v14.8h, v14.8h, v0.8h\n"
+ "fmax v12.8h, v12.8h, v1.8h\n"
+ "fmax v13.8h, v13.8h, v1.8h\n"
+ "fmax v14.8h, v14.8h, v1.8h\n"
+ "fmin v15.8h, v15.8h, v0.8h\n"
+ "fmin v16.8h, v16.8h, v0.8h\n"
+ "fmin v17.8h, v17.8h, v0.8h\n"
+ "fmax v15.8h, v15.8h, v1.8h\n"
+ "fmax v16.8h, v16.8h, v1.8h\n"
+ "fmax v17.8h, v17.8h, v1.8h\n"
+ "fmin v18.8h, v18.8h, v0.8h\n"
+ "fmin v19.8h, v19.8h, v0.8h\n"
+ "fmax v18.8h, v18.8h, v1.8h\n"
+ "fmax v19.8h, v19.8h, v1.8h\n"
+ "132:" // Height 3: No activation
+ "cmp x16, #0x20\n"
+ "bge 149f\n"
+ "tbz x16, #4, 140f\n"
+ "st1 { v8.8h }, [x13], #0x10\n"
+ "st1 { v9.8h }, [x13], #0x10\n"
+ "st1 { v12.8h }, [x9], #0x10\n"
+ "st1 { v13.8h }, [x9], #0x10\n"
+ "st1 { v16.8h }, [x27], #0x10\n"
+ "st1 { v17.8h }, [x27], #0x10\n"
+ "tbz x16, #3, 136f\n"
+ "st1 { v10.8h }, [x13], #0x10\n"
+ "st1 { v14.8h }, [x9], #0x10\n"
+ "st1 { v18.8h }, [x27], #0x10\n"
+ "tbz x16, #2, 134f\n"
+ "str d11, [x13], #0x8\n"
+ "str d15, [x9], #0x8\n"
+ "str d19, [x27], #0x8\n"
+ "tbz x16, #1, 133f\n"
+ "st1 { v11.s }[2], [x13], #0x4\n"
+ "st1 { v15.s }[2], [x9], #0x4\n"
+ "st1 { v19.s }[2], [x27], #0x4\n"
+ "tbz x16, #0, 148f\n"
+ "st1 { v11.h }[6], [x13]\n"
+ "st1 { v15.h }[6], [x9]\n"
+ "st1 { v19.h }[6], [x27]\n"
+ "b 148f\n"
+ "133:" // Height 3: Partial direct writeback: partial_1_28
+ "tbz x16, #0, 148f\n"
+ "st1 { v11.h }[4], [x13]\n"
+ "st1 { v15.h }[4], [x9]\n"
+ "st1 { v19.h }[4], [x27]\n"
+ "b 148f\n"
+ "134:" // Height 3: Partial direct writeback: partial_2_24
+ "tbz x16, #1, 135f\n"
+ "str s11, [x13], #0x4\n"
+ "str s15, [x9], #0x4\n"
+ "str s19, [x27], #0x4\n"
+ "tbz x16, #0, 148f\n"
+ "st1 { v11.h }[2], [x13]\n"
+ "st1 { v15.h }[2], [x9]\n"
+ "st1 { v19.h }[2], [x27]\n"
+ "b 148f\n"
+ "135:" // Height 3: Partial direct writeback: partial_1_24
+ "tbz x16, #0, 148f\n"
+ "str h11, [x13, #0x0]\n"
+ "str h15, [x9, #0x0]\n"
+ "str h19, [x27, #0x0]\n"
+ "b 148f\n"
+ "136:" // Height 3: Partial direct writeback: partial_4_16
+ "tbz x16, #2, 138f\n"
+ "str d10, [x13], #0x8\n"
+ "str d14, [x9], #0x8\n"
+ "str d18, [x27], #0x8\n"
+ "tbz x16, #1, 137f\n"
+ "st1 { v10.s }[2], [x13], #0x4\n"
+ "st1 { v14.s }[2], [x9], #0x4\n"
+ "st1 { v18.s }[2], [x27], #0x4\n"
+ "tbz x16, #0, 148f\n"
+ "st1 { v10.h }[6], [x13]\n"
+ "st1 { v14.h }[6], [x9]\n"
+ "st1 { v18.h }[6], [x27]\n"
+ "b 148f\n"
+ "137:" // Height 3: Partial direct writeback: partial_1_20
+ "tbz x16, #0, 148f\n"
+ "st1 { v10.h }[4], [x13]\n"
+ "st1 { v14.h }[4], [x9]\n"
+ "st1 { v18.h }[4], [x27]\n"
+ "b 148f\n"
+ "138:" // Height 3: Partial direct writeback: partial_2_16
+ "tbz x16, #1, 139f\n"
+ "str s10, [x13], #0x4\n"
+ "str s14, [x9], #0x4\n"
+ "str s18, [x27], #0x4\n"
+ "tbz x16, #0, 148f\n"
+ "st1 { v10.h }[2], [x13]\n"
+ "st1 { v14.h }[2], [x9]\n"
+ "st1 { v18.h }[2], [x27]\n"
+ "b 148f\n"
+ "139:" // Height 3: Partial direct writeback: partial_1_16
+ "tbz x16, #0, 148f\n"
+ "str h10, [x13, #0x0]\n"
+ "str h14, [x9, #0x0]\n"
+ "str h18, [x27, #0x0]\n"
+ "b 148f\n"
+ "140:" // Height 3: Partial direct writeback: partial_8_0
+ "tbz x16, #3, 144f\n"
+ "st1 { v8.8h }, [x13], #0x10\n"
+ "st1 { v12.8h }, [x9], #0x10\n"
+ "st1 { v16.8h }, [x27], #0x10\n"
+ "tbz x16, #2, 142f\n"
+ "str d9, [x13], #0x8\n"
+ "str d13, [x9], #0x8\n"
+ "str d17, [x27], #0x8\n"
+ "tbz x16, #1, 141f\n"
+ "st1 { v9.s }[2], [x13], #0x4\n"
+ "st1 { v13.s }[2], [x9], #0x4\n"
+ "st1 { v17.s }[2], [x27], #0x4\n"
+ "tbz x16, #0, 148f\n"
+ "st1 { v9.h }[6], [x13]\n"
+ "st1 { v13.h }[6], [x9]\n"
+ "st1 { v17.h }[6], [x27]\n"
+ "b 148f\n"
+ "141:" // Height 3: Partial direct writeback: partial_1_12
+ "tbz x16, #0, 148f\n"
+ "st1 { v9.h }[4], [x13]\n"
+ "st1 { v13.h }[4], [x9]\n"
+ "st1 { v17.h }[4], [x27]\n"
+ "b 148f\n"
+ "142:" // Height 3: Partial direct writeback: partial_2_8
+ "tbz x16, #1, 143f\n"
+ "str s9, [x13], #0x4\n"
+ "str s13, [x9], #0x4\n"
+ "str s17, [x27], #0x4\n"
+ "tbz x16, #0, 148f\n"
+ "st1 { v9.h }[2], [x13]\n"
+ "st1 { v13.h }[2], [x9]\n"
+ "st1 { v17.h }[2], [x27]\n"
+ "b 148f\n"
+ "143:" // Height 3: Partial direct writeback: partial_1_8
+ "tbz x16, #0, 148f\n"
+ "str h9, [x13, #0x0]\n"
+ "str h13, [x9, #0x0]\n"
+ "str h17, [x27, #0x0]\n"
+ "b 148f\n"
+ "144:" // Height 3: Partial direct writeback: partial_4_0
+ "tbz x16, #2, 146f\n"
+ "str d8, [x13], #0x8\n"
+ "str d12, [x9], #0x8\n"
+ "str d16, [x27], #0x8\n"
+ "tbz x16, #1, 145f\n"
+ "st1 { v8.s }[2], [x13], #0x4\n"
+ "st1 { v12.s }[2], [x9], #0x4\n"
+ "st1 { v16.s }[2], [x27], #0x4\n"
+ "tbz x16, #0, 148f\n"
+ "st1 { v8.h }[6], [x13]\n"
+ "st1 { v12.h }[6], [x9]\n"
+ "st1 { v16.h }[6], [x27]\n"
+ "b 148f\n"
+ "145:" // Height 3: Partial direct writeback: partial_1_4
+ "tbz x16, #0, 148f\n"
+ "st1 { v8.h }[4], [x13]\n"
+ "st1 { v12.h }[4], [x9]\n"
+ "st1 { v16.h }[4], [x27]\n"
+ "b 148f\n"
+ "146:" // Height 3: Partial direct writeback: partial_2_0
+ "tbz x16, #1, 147f\n"
+ "str s8, [x13], #0x4\n"
+ "str s12, [x9], #0x4\n"
+ "str s16, [x27], #0x4\n"
+ "tbz x16, #0, 148f\n"
+ "st1 { v8.h }[2], [x13]\n"
+ "st1 { v12.h }[2], [x9]\n"
+ "st1 { v16.h }[2], [x27]\n"
+ "b 148f\n"
+ "147:" // Height 3: Partial direct writeback: partial_1_0
+ "str h8, [x13, #0x0]\n"
+ "str h12, [x9, #0x0]\n"
+ "str h16, [x27, #0x0]\n"
+ "148:" // Height 3: Partial direct writeback: Done
+ "b 150f\n"
+ "149:" // Height 3: Full writeback
+ "str q8, [x13, #0x0]\n"
+ "str q9, [x13, #0x10]\n"
+ "str q10, [x13, #0x20]\n"
+ "str q11, [x13, #0x30]\n"
+ "str q12, [x9, #0x0]\n"
+ "str q13, [x9, #0x10]\n"
+ "str q14, [x9, #0x20]\n"
+ "str q15, [x9, #0x30]\n"
+ "str q16, [x27, #0x0]\n"
+ "str q17, [x27, #0x10]\n"
+ "str q18, [x27, #0x20]\n"
+ "str q19, [x27, #0x30]\n"
+ "add x13, x13, #0x40\n"
+ "add x9, x9, #0x40\n"
+ "add x27, x27, #0x40\n"
+ "150:" // Height 3: Writeback done
+ "subs x16, x16, #0x20\n"
+ "bgt 103b\n"
+ "b 302f\n"
+ "151:" // Height 4
+ "ldr x16, [%x[args_ptr], %[offsetof_N]]\n"
+ "mov x14, %x[bias]\n"
+ "ldr x15, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 152f\n"
+ "ldr x13, [%x[output_ptr], #0x0]\n"
+ "add x13, x13, x19, LSL #1\n"
+ "ldr x9, [%x[output_ptr], #0x8]\n"
+ "ldr x27, [%x[output_ptr], #0x10]\n"
+ "add x9, x9, x19, LSL #1\n"
+ "ldr x25, [%x[output_ptr], #0x18]\n"
+ "add x27, x27, x19, LSL #1\n"
+ "add x25, x25, x19, LSL #1\n"
+ "b 153f\n"
+ "152:" // Height 4: setup direct output
+ "mov x13, %x[output_ptr]\n"
+ "add x9, x13, x19, LSL #1\n"
+ "add x27, x9, x19, LSL #1\n"
+ "add x25, x27, x19, LSL #1\n"
+ "153:" // Height 4: Column loop
+ "cbz x14, 154f\n"
+ "ldr q8, [x14, #0x0]\n"
+ "mov v12.16b, v8.16b\n"
+ "ldr q9, [x14, #0x10]\n"
+ "mov v16.16b, v8.16b\n"
+ "ldr q10, [x14, #0x20]\n"
+ "mov v20.16b, v8.16b\n"
+ "ldr q11, [x14, #0x30]\n"
+ "add x14, x14, #0x40\n"
+ "mov v13.16b, v9.16b\n"
+ "mov v17.16b, v9.16b\n"
+ "mov v14.16b, v10.16b\n"
+ "mov v15.16b, v11.16b\n"
+ "mov v18.16b, v10.16b\n"
+ "mov v19.16b, v11.16b\n"
+ "mov v21.16b, v9.16b\n"
+ "mov v22.16b, v10.16b\n"
+ "mov v23.16b, v11.16b\n"
+ "b 173f\n"
+ "154:" // Height 4: no bias
+ "tbz %x[flags], #0, 172f\n"
+ "cmp x16, #0x20\n"
+ "bge 171f\n"
+ "tbz x16, #4, 162f\n"
+ "ld1 { v8.8h }, [x13], #0x10\n"
+ "ld1 { v12.8h }, [x9], #0x10\n"
+ "ld1 { v16.8h }, [x27], #0x10\n"
+ "ld1 { v20.8h }, [x25], #0x10\n"
+ "ld1 { v9.8h }, [x13], #0x10\n"
+ "ld1 { v13.8h }, [x9], #0x10\n"
+ "ld1 { v17.8h }, [x27], #0x10\n"
+ "ld1 { v21.8h }, [x25], #0x10\n"
+ "tbz x16, #3, 158f\n"
+ "ld1 { v10.8h }, [x13], #0x10\n"
+ "ld1 { v14.8h }, [x9], #0x10\n"
+ "ld1 { v18.8h }, [x27], #0x10\n"
+ "ld1 { v22.8h }, [x25], #0x10\n"
+ "tbz x16, #2, 156f\n"
+ "ldr d11, [x13], #0x8\n"
+ "ldr d15, [x9], #0x8\n"
+ "ldr d19, [x27], #0x8\n"
+ "ldr d23, [x25], #0x8\n"
+ "tbz x16, #1, 155f\n"
+ "mov x19, #0x3c\n"
+ "ld1 { v11.s }[2], [x13], #0x4\n"
+ "ld1 { v15.s }[2], [x9], #0x4\n"
+ "ld1 { v19.s }[2], [x27], #0x4\n"
+ "ld1 { v23.s }[2], [x25], #0x4\n"
+ "tbz x16, #0, 170f\n"
+ "ld1 { v11.h }[6], [x13]\n"
+ "ld1 { v15.h }[6], [x9]\n"
+ "ld1 { v19.h }[6], [x27]\n"
+ "ld1 { v23.h }[6], [x25]\n"
+ "b 170f\n"
+ "155:" // Height 4: Partial accumulate: partial_1_28
+ "mov x19, #0x38\n"
+ "tbz x16, #0, 170f\n"
+ "ld1 { v11.h }[4], [x13]\n"
+ "ld1 { v15.h }[4], [x9]\n"
+ "ld1 { v19.h }[4], [x27]\n"
+ "ld1 { v23.h }[4], [x25]\n"
+ "b 170f\n"
+ "156:" // Height 4: Partial accumulate: partial_2_24
+ "tbz x16, #1, 157f\n"
+ "ldr s11, [x13], #0x4\n"
+ "ldr s15, [x9], #0x4\n"
+ "ldr s19, [x27], #0x4\n"
+ "ldr s23, [x25], #0x4\n"
+ "mov x19, #0x34\n"
+ "tbz x16, #0, 170f\n"
+ "ld1 { v11.h }[2], [x13]\n"
+ "ld1 { v15.h }[2], [x9]\n"
+ "ld1 { v19.h }[2], [x27]\n"
+ "ld1 { v23.h }[2], [x25]\n"
+ "b 170f\n"
+ "157:" // Height 4: Partial accumulate: partial_1_24
+ "mov x19, #0x30\n"
+ "tbz x16, #0, 170f\n"
+ "ldr h11, [x13, #0x0]\n"
+ "ldr h15, [x9, #0x0]\n"
+ "ldr h19, [x27, #0x0]\n"
+ "ldr h23, [x25, #0x0]\n"
+ "b 170f\n"
+ "158:" // Height 4: Partial accumulate: partial_4_16
+ "tbz x16, #2, 160f\n"
+ "ldr d10, [x13], #0x8\n"
+ "ldr d14, [x9], #0x8\n"
+ "ldr d18, [x27], #0x8\n"
+ "ldr d22, [x25], #0x8\n"
+ "tbz x16, #1, 159f\n"
+ "ld1 { v10.s }[2], [x13], #0x4\n"
+ "ld1 { v14.s }[2], [x9], #0x4\n"
+ "ld1 { v18.s }[2], [x27], #0x4\n"
+ "ld1 { v22.s }[2], [x25], #0x4\n"
+ "mov x19, #0x2c\n"
+ "tbz x16, #0, 170f\n"
+ "ld1 { v10.h }[6], [x13]\n"
+ "ld1 { v14.h }[6], [x9]\n"
+ "ld1 { v18.h }[6], [x27]\n"
+ "ld1 { v22.h }[6], [x25]\n"
+ "b 170f\n"
+ "159:" // Height 4: Partial accumulate: partial_1_20
+ "mov x19, #0x28\n"
+ "tbz x16, #0, 170f\n"
+ "ld1 { v10.h }[4], [x13]\n"
+ "ld1 { v14.h }[4], [x9]\n"
+ "ld1 { v18.h }[4], [x27]\n"
+ "ld1 { v22.h }[4], [x25]\n"
+ "b 170f\n"
+ "160:" // Height 4: Partial accumulate: partial_2_16
+ "tbz x16, #1, 161f\n"
+ "ldr s10, [x13], #0x4\n"
+ "ldr s14, [x9], #0x4\n"
+ "ldr s18, [x27], #0x4\n"
+ "ldr s22, [x25], #0x4\n"
+ "mov x19, #0x24\n"
+ "tbz x16, #0, 170f\n"
+ "ld1 { v10.h }[2], [x13]\n"
+ "ld1 { v14.h }[2], [x9]\n"
+ "ld1 { v18.h }[2], [x27]\n"
+ "ld1 { v22.h }[2], [x25]\n"
+ "b 170f\n"
+ "161:" // Height 4: Partial accumulate: partial_1_16
+ "mov x19, #0x20\n"
+ "tbz x16, #0, 170f\n"
+ "ldr h10, [x13, #0x0]\n"
+ "ldr h14, [x9, #0x0]\n"
+ "ldr h18, [x27, #0x0]\n"
+ "ldr h22, [x25, #0x0]\n"
+ "b 170f\n"
+ "162:" // Height 4: Partial accumulate: partial_8_0
+ "tbz x16, #3, 166f\n"
+ "ld1 { v8.8h }, [x13], #0x10\n"
+ "ld1 { v12.8h }, [x9], #0x10\n"
+ "ld1 { v16.8h }, [x27], #0x10\n"
+ "ld1 { v20.8h }, [x25], #0x10\n"
+ "tbz x16, #2, 164f\n"
+ "ldr d9, [x13], #0x8\n"
+ "ldr d13, [x9], #0x8\n"
+ "ldr d17, [x27], #0x8\n"
+ "ldr d21, [x25], #0x8\n"
+ "tbz x16, #1, 163f\n"
+ "mov x19, #0x1c\n"
+ "ld1 { v9.s }[2], [x13], #0x4\n"
+ "ld1 { v13.s }[2], [x9], #0x4\n"
+ "ld1 { v17.s }[2], [x27], #0x4\n"
+ "ld1 { v21.s }[2], [x25], #0x4\n"
+ "tbz x16, #0, 170f\n"
+ "ld1 { v9.h }[6], [x13]\n"
+ "ld1 { v13.h }[6], [x9]\n"
+ "ld1 { v17.h }[6], [x27]\n"
+ "ld1 { v21.h }[6], [x25]\n"
+ "b 170f\n"
+ "163:" // Height 4: Partial accumulate: partial_1_12
+ "mov x19, #0x18\n"
+ "tbz x16, #0, 170f\n"
+ "ld1 { v9.h }[4], [x13]\n"
+ "ld1 { v13.h }[4], [x9]\n"
+ "ld1 { v17.h }[4], [x27]\n"
+ "ld1 { v21.h }[4], [x25]\n"
+ "b 170f\n"
+ "164:" // Height 4: Partial accumulate: partial_2_8
+ "tbz x16, #1, 165f\n"
+ "ldr s9, [x13], #0x4\n"
+ "ldr s13, [x9], #0x4\n"
+ "ldr s17, [x27], #0x4\n"
+ "ldr s21, [x25], #0x4\n"
+ "mov x19, #0x14\n"
+ "tbz x16, #0, 170f\n"
+ "ld1 { v9.h }[2], [x13]\n"
+ "ld1 { v13.h }[2], [x9]\n"
+ "ld1 { v17.h }[2], [x27]\n"
+ "ld1 { v21.h }[2], [x25]\n"
+ "b 170f\n"
+ "165:" // Height 4: Partial accumulate: partial_1_8
+ "mov x19, #0x10\n"
+ "tbz x16, #0, 170f\n"
+ "ldr h9, [x13, #0x0]\n"
+ "ldr h13, [x9, #0x0]\n"
+ "ldr h17, [x27, #0x0]\n"
+ "ldr h21, [x25, #0x0]\n"
+ "b 170f\n"
+ "166:" // Height 4: Partial accumulate: partial_4_0
+ "tbz x16, #2, 168f\n"
+ "ldr d8, [x13], #0x8\n"
+ "ldr d12, [x9], #0x8\n"
+ "ldr d16, [x27], #0x8\n"
+ "ldr d20, [x25], #0x8\n"
+ "tbz x16, #1, 167f\n"
+ "ld1 { v8.s }[2], [x13], #0x4\n"
+ "ld1 { v12.s }[2], [x9], #0x4\n"
+ "ld1 { v16.s }[2], [x27], #0x4\n"
+ "ld1 { v20.s }[2], [x25], #0x4\n"
+ "mov x19, #0xc\n"
+ "tbz x16, #0, 170f\n"
+ "ld1 { v8.h }[6], [x13]\n"
+ "ld1 { v12.h }[6], [x9]\n"
+ "ld1 { v16.h }[6], [x27]\n"
+ "ld1 { v20.h }[6], [x25]\n"
+ "b 170f\n"
+ "167:" // Height 4: Partial accumulate: partial_1_4
+ "mov x19, #0x8\n"
+ "tbz x16, #0, 170f\n"
+ "ld1 { v8.h }[4], [x13]\n"
+ "ld1 { v12.h }[4], [x9]\n"
+ "ld1 { v16.h }[4], [x27]\n"
+ "ld1 { v20.h }[4], [x25]\n"
+ "b 170f\n"
+ "168:" // Height 4: Partial accumulate: partial_2_0
+ "tbz x16, #1, 169f\n"
+ "ldr s8, [x13], #0x4\n"
+ "ldr s12, [x9], #0x4\n"
+ "ldr s16, [x27], #0x4\n"
+ "ldr s20, [x25], #0x4\n"
+ "mov x19, #0x4\n"
+ "tbz x16, #0, 170f\n"
+ "ld1 { v8.h }[2], [x13]\n"
+ "ld1 { v12.h }[2], [x9]\n"
+ "ld1 { v16.h }[2], [x27]\n"
+ "ld1 { v20.h }[2], [x25]\n"
+ "b 170f\n"
+ "169:" // Height 4: Partial accumulate: partial_1_0
+ "mov x19, #0x0\n"
+ "ldr h8, [x13, #0x0]\n"
+ "ldr h12, [x9, #0x0]\n"
+ "ldr h16, [x27, #0x0]\n"
+ "ldr h20, [x25, #0x0]\n"
+ "170:" // Height 4: Partial accumulate: Done
+ "sub x13, x13, x19\n"
+ "sub x9, x9, x19\n"
+ "sub x27, x27, x19\n"
+ "sub x25, x25, x19\n"
+ "b 173f\n"
+ "171:" // Height 4: full accumulate
+ "ldr q8, [x13, #0x0]\n"
+ "ldr q9, [x13, #0x10]\n"
+ "ldr q10, [x13, #0x20]\n"
+ "ldr q11, [x13, #0x30]\n"
+ "ldr q12, [x9, #0x0]\n"
+ "ldr q13, [x9, #0x10]\n"
+ "ldr q14, [x9, #0x20]\n"
+ "ldr q15, [x9, #0x30]\n"
+ "ldr q16, [x27, #0x0]\n"
+ "ldr q17, [x27, #0x10]\n"
+ "ldr q18, [x27, #0x20]\n"
+ "ldr q19, [x27, #0x30]\n"
+ "ldr q20, [x25, #0x0]\n"
+ "ldr q21, [x25, #0x10]\n"
+ "ldr q22, [x25, #0x20]\n"
+ "ldr q23, [x25, #0x30]\n"
+ "b 173f\n"
+ "172:" // Height 4: no accumulate
+ "movi v8.16b, #0x0\n"
+ "movi v9.16b, #0x0\n"
+ "movi v10.16b, #0x0\n"
+ "movi v11.16b, #0x0\n"
+ "movi v12.16b, #0x0\n"
+ "movi v13.16b, #0x0\n"
+ "movi v14.16b, #0x0\n"
+ "movi v15.16b, #0x0\n"
+ "movi v16.16b, #0x0\n"
+ "movi v17.16b, #0x0\n"
+ "movi v18.16b, #0x0\n"
+ "movi v19.16b, #0x0\n"
+ "movi v20.16b, #0x0\n"
+ "movi v21.16b, #0x0\n"
+ "movi v22.16b, #0x0\n"
+ "movi v23.16b, #0x0\n"
+ "173:" // Height 4: setup done
+ "mov x12, #0x0\n"
+ "174:" // Height 4: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w11, [x20, x12, LSL #0x2]\n"
+ "tbz %x[flags], #3, 175f\n"
+ "ldr x20, [%x[input_ptr], x12, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x10, [x20, #0x0]\n"
+ "ldr x28, [x20, #0x8]\n"
+ "ldr x26, [x20, #0x10]\n"
+ "ldr x24, [x20, #0x18]\n"
+ "cbnz x12, 176f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x10, x10, x19, LSL #1\n"
+ "add x28, x28, x19, LSL #1\n"
+ "add x26, x26, x19, LSL #1\n"
+ "add x24, x24, x19, LSL #1\n"
+ "b 176f\n"
+ "175:" // Height 4: setup direct input
+ "mov x10, %x[input_ptr]\n"
+ "add x28, x10, x19, LSL #1\n"
+ "add x26, x28, x19, LSL #1\n"
+ "add x24, x26, x19, LSL #1\n"
+ "176:" // Height 4: input setup done
+ "cmp x11, #0x8\n"
+ "blt 179f\n"
+ "cmp x11, #0x10\n"
+ "blt 178f\n"
+ "177:" // Height 4: Multiply loop: Main loop head
+ "ldr q0, [x10, #0x0]\n"
+ "ldr q1, [x28, #0x0]\n"
+ "ldr q2, [x26, #0x0]\n"
+ "ldr q3, [x24, #0x0]\n"
+ "ldr q6, [x15, #0x0]\n"
+ "fmla v8.8h, v6.8h, v0.h[0]\n"
+ "ldr q7, [x15, #0x10]\n"
+ "fmla v12.8h, v6.8h, v1.h[0]\n"
+ "add x10, x10, #0x10\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "fmla v16.8h, v6.8h, v2.h[0]\n"
+ "add x28, x28, #0x10\n"
+ "fmla v20.8h, v6.8h, v3.h[0]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "ldr q6, [x15, #0x20]\n"
+ "fmla v9.8h, v7.8h, v0.h[0]\n"
+ "add x26, x26, #0x10\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "fmla v13.8h, v7.8h, v1.h[0]\n"
+ "add x24, x24, #0x10\n"
+ "fmla v17.8h, v7.8h, v2.h[0]\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "sub x11, x11, #0x8\n"
+ "fmla v21.8h, v7.8h, v3.h[0]\n"
+ "ldr q7, [x15, #0x30]\n"
+ "fmla v10.8h, v6.8h, v0.h[0]\n"
+ "cmp x11, #0x10\n"
+ "fmla v14.8h, v6.8h, v1.h[0]\n"
+ "fmla v18.8h, v6.8h, v2.h[0]\n"
+ "fmla v22.8h, v6.8h, v3.h[0]\n"
+ "ldr q6, [x15, #0x40]\n"
+ "fmla v11.8h, v7.8h, v0.h[0]\n"
+ "fmla v15.8h, v7.8h, v1.h[0]\n"
+ "fmla v19.8h, v7.8h, v2.h[0]\n"
+ "fmla v23.8h, v7.8h, v3.h[0]\n"
+ "ldr q7, [x15, #0x50]\n"
+ "fmla v8.8h, v6.8h, v0.h[1]\n"
+ "fmla v12.8h, v6.8h, v1.h[1]\n"
+ "fmla v16.8h, v6.8h, v2.h[1]\n"
+ "fmla v20.8h, v6.8h, v3.h[1]\n"
+ "ldr q6, [x15, #0x60]\n"
+ "fmla v9.8h, v7.8h, v0.h[1]\n"
+ "fmla v13.8h, v7.8h, v1.h[1]\n"
+ "fmla v17.8h, v7.8h, v2.h[1]\n"
+ "fmla v21.8h, v7.8h, v3.h[1]\n"
+ "ldr q7, [x15, #0x70]\n"
+ "fmla v10.8h, v6.8h, v0.h[1]\n"
+ "fmla v14.8h, v6.8h, v1.h[1]\n"
+ "fmla v18.8h, v6.8h, v2.h[1]\n"
+ "fmla v22.8h, v6.8h, v3.h[1]\n"
+ "ldr q6, [x15, #0x80]\n"
+ "fmla v11.8h, v7.8h, v0.h[1]\n"
+ "fmla v15.8h, v7.8h, v1.h[1]\n"
+ "fmla v19.8h, v7.8h, v2.h[1]\n"
+ "fmla v23.8h, v7.8h, v3.h[1]\n"
+ "ldr q7, [x15, #0x90]\n"
+ "fmla v8.8h, v6.8h, v0.h[2]\n"
+ "fmla v12.8h, v6.8h, v1.h[2]\n"
+ "fmla v16.8h, v6.8h, v2.h[2]\n"
+ "fmla v20.8h, v6.8h, v3.h[2]\n"
+ "ldr q6, [x15, #0xa0]\n"
+ "fmla v9.8h, v7.8h, v0.h[2]\n"
+ "fmla v13.8h, v7.8h, v1.h[2]\n"
+ "fmla v17.8h, v7.8h, v2.h[2]\n"
+ "fmla v21.8h, v7.8h, v3.h[2]\n"
+ "ldr q7, [x15, #0xb0]\n"
+ "fmla v10.8h, v6.8h, v0.h[2]\n"
+ "fmla v14.8h, v6.8h, v1.h[2]\n"
+ "fmla v18.8h, v6.8h, v2.h[2]\n"
+ "fmla v22.8h, v6.8h, v3.h[2]\n"
+ "ldr q6, [x15, #0xc0]\n"
+ "fmla v11.8h, v7.8h, v0.h[2]\n"
+ "fmla v15.8h, v7.8h, v1.h[2]\n"
+ "fmla v19.8h, v7.8h, v2.h[2]\n"
+ "fmla v23.8h, v7.8h, v3.h[2]\n"
+ "ldr q7, [x15, #0xd0]\n"
+ "fmla v8.8h, v6.8h, v0.h[3]\n"
+ "fmla v12.8h, v6.8h, v1.h[3]\n"
+ "fmla v16.8h, v6.8h, v2.h[3]\n"
+ "fmla v20.8h, v6.8h, v3.h[3]\n"
+ "ldr q6, [x15, #0xe0]\n"
+ "fmla v9.8h, v7.8h, v0.h[3]\n"
+ "fmla v13.8h, v7.8h, v1.h[3]\n"
+ "fmla v17.8h, v7.8h, v2.h[3]\n"
+ "fmla v21.8h, v7.8h, v3.h[3]\n"
+ "ldr q7, [x15, #0xf0]\n"
+ "fmla v10.8h, v6.8h, v0.h[3]\n"
+ "fmla v14.8h, v6.8h, v1.h[3]\n"
+ "fmla v18.8h, v6.8h, v2.h[3]\n"
+ "fmla v22.8h, v6.8h, v3.h[3]\n"
+ "ldr q6, [x15, #0x100]\n"
+ "fmla v11.8h, v7.8h, v0.h[3]\n"
+ "fmla v15.8h, v7.8h, v1.h[3]\n"
+ "fmla v19.8h, v7.8h, v2.h[3]\n"
+ "fmla v23.8h, v7.8h, v3.h[3]\n"
+ "ldr q7, [x15, #0x110]\n"
+ "fmla v8.8h, v6.8h, v0.h[4]\n"
+ "fmla v12.8h, v6.8h, v1.h[4]\n"
+ "fmla v16.8h, v6.8h, v2.h[4]\n"
+ "fmla v20.8h, v6.8h, v3.h[4]\n"
+ "ldr q6, [x15, #0x120]\n"
+ "fmla v9.8h, v7.8h, v0.h[4]\n"
+ "fmla v13.8h, v7.8h, v1.h[4]\n"
+ "fmla v17.8h, v7.8h, v2.h[4]\n"
+ "fmla v21.8h, v7.8h, v3.h[4]\n"
+ "ldr q7, [x15, #0x130]\n"
+ "fmla v10.8h, v6.8h, v0.h[4]\n"
+ "fmla v14.8h, v6.8h, v1.h[4]\n"
+ "fmla v18.8h, v6.8h, v2.h[4]\n"
+ "fmla v22.8h, v6.8h, v3.h[4]\n"
+ "ldr q6, [x15, #0x140]\n"
+ "fmla v11.8h, v7.8h, v0.h[4]\n"
+ "fmla v15.8h, v7.8h, v1.h[4]\n"
+ "fmla v19.8h, v7.8h, v2.h[4]\n"
+ "fmla v23.8h, v7.8h, v3.h[4]\n"
+ "ldr q7, [x15, #0x150]\n"
+ "fmla v8.8h, v6.8h, v0.h[5]\n"
+ "fmla v12.8h, v6.8h, v1.h[5]\n"
+ "fmla v16.8h, v6.8h, v2.h[5]\n"
+ "fmla v20.8h, v6.8h, v3.h[5]\n"
+ "ldr q6, [x15, #0x160]\n"
+ "fmla v9.8h, v7.8h, v0.h[5]\n"
+ "fmla v13.8h, v7.8h, v1.h[5]\n"
+ "fmla v17.8h, v7.8h, v2.h[5]\n"
+ "fmla v21.8h, v7.8h, v3.h[5]\n"
+ "ldr q7, [x15, #0x170]\n"
+ "fmla v10.8h, v6.8h, v0.h[5]\n"
+ "fmla v14.8h, v6.8h, v1.h[5]\n"
+ "fmla v18.8h, v6.8h, v2.h[5]\n"
+ "fmla v22.8h, v6.8h, v3.h[5]\n"
+ "ldr q6, [x15, #0x180]\n"
+ "fmla v11.8h, v7.8h, v0.h[5]\n"
+ "fmla v15.8h, v7.8h, v1.h[5]\n"
+ "fmla v19.8h, v7.8h, v2.h[5]\n"
+ "fmla v23.8h, v7.8h, v3.h[5]\n"
+ "ldr q7, [x15, #0x190]\n"
+ "fmla v8.8h, v6.8h, v0.h[6]\n"
+ "fmla v12.8h, v6.8h, v1.h[6]\n"
+ "fmla v16.8h, v6.8h, v2.h[6]\n"
+ "fmla v20.8h, v6.8h, v3.h[6]\n"
+ "ldr q6, [x15, #0x1a0]\n"
+ "fmla v9.8h, v7.8h, v0.h[6]\n"
+ "fmla v13.8h, v7.8h, v1.h[6]\n"
+ "fmla v17.8h, v7.8h, v2.h[6]\n"
+ "fmla v21.8h, v7.8h, v3.h[6]\n"
+ "ldr q7, [x15, #0x1b0]\n"
+ "fmla v10.8h, v6.8h, v0.h[6]\n"
+ "fmla v14.8h, v6.8h, v1.h[6]\n"
+ "fmla v18.8h, v6.8h, v2.h[6]\n"
+ "fmla v22.8h, v6.8h, v3.h[6]\n"
+ "ldr q6, [x15, #0x1c0]\n"
+ "fmla v11.8h, v7.8h, v0.h[6]\n"
+ "fmla v15.8h, v7.8h, v1.h[6]\n"
+ "fmla v19.8h, v7.8h, v2.h[6]\n"
+ "fmla v23.8h, v7.8h, v3.h[6]\n"
+ "ldr q7, [x15, #0x1d0]\n"
+ "fmla v8.8h, v6.8h, v0.h[7]\n"
+ "fmla v12.8h, v6.8h, v1.h[7]\n"
+ "fmla v16.8h, v6.8h, v2.h[7]\n"
+ "fmla v20.8h, v6.8h, v3.h[7]\n"
+ "ldr q6, [x15, #0x1e0]\n"
+ "fmla v9.8h, v7.8h, v0.h[7]\n"
+ "fmla v13.8h, v7.8h, v1.h[7]\n"
+ "fmla v17.8h, v7.8h, v2.h[7]\n"
+ "fmla v21.8h, v7.8h, v3.h[7]\n"
+ "ldr q7, [x15, #0x1f0]\n"
+ "add x15, x15, #0x200\n"
+ "fmla v10.8h, v6.8h, v0.h[7]\n"
+ "fmla v14.8h, v6.8h, v1.h[7]\n"
+ "fmla v18.8h, v6.8h, v2.h[7]\n"
+ "fmla v22.8h, v6.8h, v3.h[7]\n"
+ "fmla v11.8h, v7.8h, v0.h[7]\n"
+ "fmla v15.8h, v7.8h, v1.h[7]\n"
+ "fmla v19.8h, v7.8h, v2.h[7]\n"
+ "fmla v23.8h, v7.8h, v3.h[7]\n"
+ "bge 177b\n"
+ "178:" // Height 4: Multiply loop: Single iteration only
+ "sub x11, x11, #0x8\n"
+ "ldr q0, [x10, #0x0]\n"
+ "ldr q1, [x28, #0x0]\n"
+ "ldr q2, [x26, #0x0]\n"
+ "ldr q3, [x24, #0x0]\n"
+ "ldr q6, [x15, #0x0]\n"
+ "fmla v8.8h, v6.8h, v0.h[0]\n"
+ "ldr q7, [x15, #0x10]\n"
+ "fmla v12.8h, v6.8h, v1.h[0]\n"
+ "add x10, x10, #0x10\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "fmla v16.8h, v6.8h, v2.h[0]\n"
+ "add x28, x28, #0x10\n"
+ "fmla v20.8h, v6.8h, v3.h[0]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "ldr q6, [x15, #0x20]\n"
+ "fmla v9.8h, v7.8h, v0.h[0]\n"
+ "add x26, x26, #0x10\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "fmla v13.8h, v7.8h, v1.h[0]\n"
+ "add x24, x24, #0x10\n"
+ "fmla v17.8h, v7.8h, v2.h[0]\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "fmla v21.8h, v7.8h, v3.h[0]\n"
+ "ldr q7, [x15, #0x30]\n"
+ "fmla v10.8h, v6.8h, v0.h[0]\n"
+ "fmla v14.8h, v6.8h, v1.h[0]\n"
+ "fmla v18.8h, v6.8h, v2.h[0]\n"
+ "fmla v22.8h, v6.8h, v3.h[0]\n"
+ "ldr q6, [x15, #0x40]\n"
+ "fmla v11.8h, v7.8h, v0.h[0]\n"
+ "fmla v15.8h, v7.8h, v1.h[0]\n"
+ "fmla v19.8h, v7.8h, v2.h[0]\n"
+ "fmla v23.8h, v7.8h, v3.h[0]\n"
+ "ldr q7, [x15, #0x50]\n"
+ "fmla v8.8h, v6.8h, v0.h[1]\n"
+ "fmla v12.8h, v6.8h, v1.h[1]\n"
+ "fmla v16.8h, v6.8h, v2.h[1]\n"
+ "fmla v20.8h, v6.8h, v3.h[1]\n"
+ "ldr q6, [x15, #0x60]\n"
+ "fmla v9.8h, v7.8h, v0.h[1]\n"
+ "fmla v13.8h, v7.8h, v1.h[1]\n"
+ "fmla v17.8h, v7.8h, v2.h[1]\n"
+ "fmla v21.8h, v7.8h, v3.h[1]\n"
+ "ldr q7, [x15, #0x70]\n"
+ "fmla v10.8h, v6.8h, v0.h[1]\n"
+ "fmla v14.8h, v6.8h, v1.h[1]\n"
+ "fmla v18.8h, v6.8h, v2.h[1]\n"
+ "fmla v22.8h, v6.8h, v3.h[1]\n"
+ "ldr q6, [x15, #0x80]\n"
+ "fmla v11.8h, v7.8h, v0.h[1]\n"
+ "fmla v15.8h, v7.8h, v1.h[1]\n"
+ "fmla v19.8h, v7.8h, v2.h[1]\n"
+ "fmla v23.8h, v7.8h, v3.h[1]\n"
+ "ldr q7, [x15, #0x90]\n"
+ "fmla v8.8h, v6.8h, v0.h[2]\n"
+ "fmla v12.8h, v6.8h, v1.h[2]\n"
+ "fmla v16.8h, v6.8h, v2.h[2]\n"
+ "fmla v20.8h, v6.8h, v3.h[2]\n"
+ "ldr q6, [x15, #0xa0]\n"
+ "fmla v9.8h, v7.8h, v0.h[2]\n"
+ "fmla v13.8h, v7.8h, v1.h[2]\n"
+ "fmla v17.8h, v7.8h, v2.h[2]\n"
+ "fmla v21.8h, v7.8h, v3.h[2]\n"
+ "ldr q7, [x15, #0xb0]\n"
+ "fmla v10.8h, v6.8h, v0.h[2]\n"
+ "fmla v14.8h, v6.8h, v1.h[2]\n"
+ "fmla v18.8h, v6.8h, v2.h[2]\n"
+ "fmla v22.8h, v6.8h, v3.h[2]\n"
+ "ldr q6, [x15, #0xc0]\n"
+ "fmla v11.8h, v7.8h, v0.h[2]\n"
+ "fmla v15.8h, v7.8h, v1.h[2]\n"
+ "fmla v19.8h, v7.8h, v2.h[2]\n"
+ "fmla v23.8h, v7.8h, v3.h[2]\n"
+ "ldr q7, [x15, #0xd0]\n"
+ "fmla v8.8h, v6.8h, v0.h[3]\n"
+ "fmla v12.8h, v6.8h, v1.h[3]\n"
+ "fmla v16.8h, v6.8h, v2.h[3]\n"
+ "fmla v20.8h, v6.8h, v3.h[3]\n"
+ "ldr q6, [x15, #0xe0]\n"
+ "fmla v9.8h, v7.8h, v0.h[3]\n"
+ "fmla v13.8h, v7.8h, v1.h[3]\n"
+ "fmla v17.8h, v7.8h, v2.h[3]\n"
+ "fmla v21.8h, v7.8h, v3.h[3]\n"
+ "ldr q7, [x15, #0xf0]\n"
+ "fmla v10.8h, v6.8h, v0.h[3]\n"
+ "fmla v14.8h, v6.8h, v1.h[3]\n"
+ "fmla v18.8h, v6.8h, v2.h[3]\n"
+ "fmla v22.8h, v6.8h, v3.h[3]\n"
+ "ldr q6, [x15, #0x100]\n"
+ "fmla v11.8h, v7.8h, v0.h[3]\n"
+ "fmla v15.8h, v7.8h, v1.h[3]\n"
+ "fmla v19.8h, v7.8h, v2.h[3]\n"
+ "fmla v23.8h, v7.8h, v3.h[3]\n"
+ "ldr q7, [x15, #0x110]\n"
+ "fmla v8.8h, v6.8h, v0.h[4]\n"
+ "fmla v12.8h, v6.8h, v1.h[4]\n"
+ "fmla v16.8h, v6.8h, v2.h[4]\n"
+ "fmla v20.8h, v6.8h, v3.h[4]\n"
+ "ldr q6, [x15, #0x120]\n"
+ "fmla v9.8h, v7.8h, v0.h[4]\n"
+ "fmla v13.8h, v7.8h, v1.h[4]\n"
+ "fmla v17.8h, v7.8h, v2.h[4]\n"
+ "fmla v21.8h, v7.8h, v3.h[4]\n"
+ "ldr q7, [x15, #0x130]\n"
+ "fmla v10.8h, v6.8h, v0.h[4]\n"
+ "fmla v14.8h, v6.8h, v1.h[4]\n"
+ "fmla v18.8h, v6.8h, v2.h[4]\n"
+ "fmla v22.8h, v6.8h, v3.h[4]\n"
+ "ldr q6, [x15, #0x140]\n"
+ "fmla v11.8h, v7.8h, v0.h[4]\n"
+ "fmla v15.8h, v7.8h, v1.h[4]\n"
+ "fmla v19.8h, v7.8h, v2.h[4]\n"
+ "fmla v23.8h, v7.8h, v3.h[4]\n"
+ "ldr q7, [x15, #0x150]\n"
+ "fmla v8.8h, v6.8h, v0.h[5]\n"
+ "fmla v12.8h, v6.8h, v1.h[5]\n"
+ "fmla v16.8h, v6.8h, v2.h[5]\n"
+ "fmla v20.8h, v6.8h, v3.h[5]\n"
+ "ldr q6, [x15, #0x160]\n"
+ "fmla v9.8h, v7.8h, v0.h[5]\n"
+ "fmla v13.8h, v7.8h, v1.h[5]\n"
+ "fmla v17.8h, v7.8h, v2.h[5]\n"
+ "fmla v21.8h, v7.8h, v3.h[5]\n"
+ "ldr q7, [x15, #0x170]\n"
+ "fmla v10.8h, v6.8h, v0.h[5]\n"
+ "fmla v14.8h, v6.8h, v1.h[5]\n"
+ "fmla v18.8h, v6.8h, v2.h[5]\n"
+ "fmla v22.8h, v6.8h, v3.h[5]\n"
+ "ldr q6, [x15, #0x180]\n"
+ "fmla v11.8h, v7.8h, v0.h[5]\n"
+ "fmla v15.8h, v7.8h, v1.h[5]\n"
+ "fmla v19.8h, v7.8h, v2.h[5]\n"
+ "fmla v23.8h, v7.8h, v3.h[5]\n"
+ "ldr q7, [x15, #0x190]\n"
+ "fmla v8.8h, v6.8h, v0.h[6]\n"
+ "fmla v12.8h, v6.8h, v1.h[6]\n"
+ "fmla v16.8h, v6.8h, v2.h[6]\n"
+ "fmla v20.8h, v6.8h, v3.h[6]\n"
+ "ldr q6, [x15, #0x1a0]\n"
+ "fmla v9.8h, v7.8h, v0.h[6]\n"
+ "fmla v13.8h, v7.8h, v1.h[6]\n"
+ "fmla v17.8h, v7.8h, v2.h[6]\n"
+ "fmla v21.8h, v7.8h, v3.h[6]\n"
+ "ldr q7, [x15, #0x1b0]\n"
+ "fmla v10.8h, v6.8h, v0.h[6]\n"
+ "fmla v14.8h, v6.8h, v1.h[6]\n"
+ "fmla v18.8h, v6.8h, v2.h[6]\n"
+ "fmla v22.8h, v6.8h, v3.h[6]\n"
+ "ldr q6, [x15, #0x1c0]\n"
+ "fmla v11.8h, v7.8h, v0.h[6]\n"
+ "fmla v15.8h, v7.8h, v1.h[6]\n"
+ "fmla v19.8h, v7.8h, v2.h[6]\n"
+ "fmla v23.8h, v7.8h, v3.h[6]\n"
+ "ldr q7, [x15, #0x1d0]\n"
+ "fmla v8.8h, v6.8h, v0.h[7]\n"
+ "fmla v12.8h, v6.8h, v1.h[7]\n"
+ "fmla v16.8h, v6.8h, v2.h[7]\n"
+ "fmla v20.8h, v6.8h, v3.h[7]\n"
+ "ldr q6, [x15, #0x1e0]\n"
+ "fmla v9.8h, v7.8h, v0.h[7]\n"
+ "fmla v13.8h, v7.8h, v1.h[7]\n"
+ "fmla v17.8h, v7.8h, v2.h[7]\n"
+ "fmla v21.8h, v7.8h, v3.h[7]\n"
+ "ldr q7, [x15, #0x1f0]\n"
+ "add x15, x15, #0x200\n"
+ "fmla v10.8h, v6.8h, v0.h[7]\n"
+ "fmla v14.8h, v6.8h, v1.h[7]\n"
+ "fmla v18.8h, v6.8h, v2.h[7]\n"
+ "fmla v22.8h, v6.8h, v3.h[7]\n"
+ "fmla v11.8h, v7.8h, v0.h[7]\n"
+ "fmla v15.8h, v7.8h, v1.h[7]\n"
+ "fmla v19.8h, v7.8h, v2.h[7]\n"
+ "fmla v23.8h, v7.8h, v3.h[7]\n"
+ "179:" // Height 4: Multiply loop: Main loop skip
+ "cbz x11, 181f\n"
+ "180:" // Height 4: Multiply loop: Odd block loop
+ "ldr h0, [x10], #0x2\n"
+ "ldr h1, [x28], #0x2\n"
+ "ldr h2, [x26], #0x2\n"
+ "ldr h3, [x24], #0x2\n"
+ "ldr q6, [x15, #0x0]\n"
+ "fmla v8.8h, v6.8h, v0.h[0]\n"
+ "ldr q7, [x15, #0x10]\n"
+ "fmla v12.8h, v6.8h, v1.h[0]\n"
+ "sub x11, x11, #0x1\n"
+ "fmla v16.8h, v6.8h, v2.h[0]\n"
+ "fmla v20.8h, v6.8h, v3.h[0]\n"
+ "ldr q6, [x15, #0x20]\n"
+ "fmla v9.8h, v7.8h, v0.h[0]\n"
+ "fmla v13.8h, v7.8h, v1.h[0]\n"
+ "fmla v17.8h, v7.8h, v2.h[0]\n"
+ "fmla v21.8h, v7.8h, v3.h[0]\n"
+ "ldr q7, [x15, #0x30]\n"
+ "fmla v10.8h, v6.8h, v0.h[0]\n"
+ "add x15, x15, #0x40\n"
+ "fmla v14.8h, v6.8h, v1.h[0]\n"
+ "fmla v18.8h, v6.8h, v2.h[0]\n"
+ "fmla v22.8h, v6.8h, v3.h[0]\n"
+ "fmla v11.8h, v7.8h, v0.h[0]\n"
+ "fmla v15.8h, v7.8h, v1.h[0]\n"
+ "fmla v19.8h, v7.8h, v2.h[0]\n"
+ "fmla v23.8h, v7.8h, v3.h[0]\n"
+ "cbnz x11, 180b\n"
+ "181:" // Height 4: Multiply loop: No odd multiplies
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "add x12, x12, #0x1\n"
+ "cmp x12, x19\n"
+ "bne 174b\n"
+ "prfm pstl1keep, [x13, #0x0]\n"
+ "prfm pstl1keep, [x9, #0x0]\n"
+ "prfm pstl1keep, [x27, #0x0]\n"
+ "prfm pstl1keep, [x25, #0x0]\n"
+ "tbz %x[flags], #1, 182f\n"
+ "add x19, %x[args_ptr], %[offset_min]\n"
+ "ld1r { v1.8h }, [x19]\n"
+ "add x19, %x[args_ptr], %[offset_max]\n"
+ "ld1r { v0.8h }, [x19]\n"
+ "fmin v8.8h, v8.8h, v0.8h\n"
+ "fmin v9.8h, v9.8h, v0.8h\n"
+ "fmin v10.8h, v10.8h, v0.8h\n"
+ "fmin v11.8h, v11.8h, v0.8h\n"
+ "fmax v8.8h, v8.8h, v1.8h\n"
+ "fmax v9.8h, v9.8h, v1.8h\n"
+ "fmax v10.8h, v10.8h, v1.8h\n"
+ "fmax v11.8h, v11.8h, v1.8h\n"
+ "fmin v12.8h, v12.8h, v0.8h\n"
+ "fmin v13.8h, v13.8h, v0.8h\n"
+ "fmin v14.8h, v14.8h, v0.8h\n"
+ "fmax v12.8h, v12.8h, v1.8h\n"
+ "fmax v13.8h, v13.8h, v1.8h\n"
+ "fmax v14.8h, v14.8h, v1.8h\n"
+ "fmin v15.8h, v15.8h, v0.8h\n"
+ "fmin v16.8h, v16.8h, v0.8h\n"
+ "fmin v17.8h, v17.8h, v0.8h\n"
+ "fmax v15.8h, v15.8h, v1.8h\n"
+ "fmax v16.8h, v16.8h, v1.8h\n"
+ "fmax v17.8h, v17.8h, v1.8h\n"
+ "fmin v18.8h, v18.8h, v0.8h\n"
+ "fmin v19.8h, v19.8h, v0.8h\n"
+ "fmin v20.8h, v20.8h, v0.8h\n"
+ "fmax v18.8h, v18.8h, v1.8h\n"
+ "fmax v19.8h, v19.8h, v1.8h\n"
+ "fmax v20.8h, v20.8h, v1.8h\n"
+ "fmin v21.8h, v21.8h, v0.8h\n"
+ "fmin v22.8h, v22.8h, v0.8h\n"
+ "fmin v23.8h, v23.8h, v0.8h\n"
+ "fmax v21.8h, v21.8h, v1.8h\n"
+ "fmax v22.8h, v22.8h, v1.8h\n"
+ "fmax v23.8h, v23.8h, v1.8h\n"
+ "182:" // Height 4: No activation
+ "cmp x16, #0x20\n"
+ "bge 199f\n"
+ "tbz x16, #4, 190f\n"
+ "st1 { v8.8h }, [x13], #0x10\n"
+ "st1 { v9.8h }, [x13], #0x10\n"
+ "st1 { v12.8h }, [x9], #0x10\n"
+ "st1 { v13.8h }, [x9], #0x10\n"
+ "st1 { v16.8h }, [x27], #0x10\n"
+ "st1 { v17.8h }, [x27], #0x10\n"
+ "st1 { v20.8h }, [x25], #0x10\n"
+ "st1 { v21.8h }, [x25], #0x10\n"
+ "tbz x16, #3, 186f\n"
+ "st1 { v10.8h }, [x13], #0x10\n"
+ "st1 { v14.8h }, [x9], #0x10\n"
+ "st1 { v18.8h }, [x27], #0x10\n"
+ "st1 { v22.8h }, [x25], #0x10\n"
+ "tbz x16, #2, 184f\n"
+ "str d11, [x13], #0x8\n"
+ "str d15, [x9], #0x8\n"
+ "str d19, [x27], #0x8\n"
+ "str d23, [x25], #0x8\n"
+ "tbz x16, #1, 183f\n"
+ "st1 { v11.s }[2], [x13], #0x4\n"
+ "st1 { v15.s }[2], [x9], #0x4\n"
+ "st1 { v19.s }[2], [x27], #0x4\n"
+ "st1 { v23.s }[2], [x25], #0x4\n"
+ "tbz x16, #0, 198f\n"
+ "st1 { v11.h }[6], [x13]\n"
+ "st1 { v15.h }[6], [x9]\n"
+ "st1 { v19.h }[6], [x27]\n"
+ "st1 { v23.h }[6], [x25]\n"
+ "b 198f\n"
+ "183:" // Height 4: Partial direct writeback: partial_1_28
+ "tbz x16, #0, 198f\n"
+ "st1 { v11.h }[4], [x13]\n"
+ "st1 { v15.h }[4], [x9]\n"
+ "st1 { v19.h }[4], [x27]\n"
+ "st1 { v23.h }[4], [x25]\n"
+ "b 198f\n"
+ "184:" // Height 4: Partial direct writeback: partial_2_24
+ "tbz x16, #1, 185f\n"
+ "str s11, [x13], #0x4\n"
+ "str s15, [x9], #0x4\n"
+ "str s19, [x27], #0x4\n"
+ "str s23, [x25], #0x4\n"
+ "tbz x16, #0, 198f\n"
+ "st1 { v11.h }[2], [x13]\n"
+ "st1 { v15.h }[2], [x9]\n"
+ "st1 { v19.h }[2], [x27]\n"
+ "st1 { v23.h }[2], [x25]\n"
+ "b 198f\n"
+ "185:" // Height 4: Partial direct writeback: partial_1_24
+ "tbz x16, #0, 198f\n"
+ "str h11, [x13, #0x0]\n"
+ "str h15, [x9, #0x0]\n"
+ "str h19, [x27, #0x0]\n"
+ "str h23, [x25, #0x0]\n"
+ "b 198f\n"
+ "186:" // Height 4: Partial direct writeback: partial_4_16
+ "tbz x16, #2, 188f\n"
+ "str d10, [x13], #0x8\n"
+ "str d14, [x9], #0x8\n"
+ "str d18, [x27], #0x8\n"
+ "str d22, [x25], #0x8\n"
+ "tbz x16, #1, 187f\n"
+ "st1 { v10.s }[2], [x13], #0x4\n"
+ "st1 { v14.s }[2], [x9], #0x4\n"
+ "st1 { v18.s }[2], [x27], #0x4\n"
+ "st1 { v22.s }[2], [x25], #0x4\n"
+ "tbz x16, #0, 198f\n"
+ "st1 { v10.h }[6], [x13]\n"
+ "st1 { v14.h }[6], [x9]\n"
+ "st1 { v18.h }[6], [x27]\n"
+ "st1 { v22.h }[6], [x25]\n"
+ "b 198f\n"
+ "187:" // Height 4: Partial direct writeback: partial_1_20
+ "tbz x16, #0, 198f\n"
+ "st1 { v10.h }[4], [x13]\n"
+ "st1 { v14.h }[4], [x9]\n"
+ "st1 { v18.h }[4], [x27]\n"
+ "st1 { v22.h }[4], [x25]\n"
+ "b 198f\n"
+ "188:" // Height 4: Partial direct writeback: partial_2_16
+ "tbz x16, #1, 189f\n"
+ "str s10, [x13], #0x4\n"
+ "str s14, [x9], #0x4\n"
+ "str s18, [x27], #0x4\n"
+ "str s22, [x25], #0x4\n"
+ "tbz x16, #0, 198f\n"
+ "st1 { v10.h }[2], [x13]\n"
+ "st1 { v14.h }[2], [x9]\n"
+ "st1 { v18.h }[2], [x27]\n"
+ "st1 { v22.h }[2], [x25]\n"
+ "b 198f\n"
+ "189:" // Height 4: Partial direct writeback: partial_1_16
+ "tbz x16, #0, 198f\n"
+ "str h10, [x13, #0x0]\n"
+ "str h14, [x9, #0x0]\n"
+ "str h18, [x27, #0x0]\n"
+ "str h22, [x25, #0x0]\n"
+ "b 198f\n"
+ "190:" // Height 4: Partial direct writeback: partial_8_0
+ "tbz x16, #3, 194f\n"
+ "st1 { v8.8h }, [x13], #0x10\n"
+ "st1 { v12.8h }, [x9], #0x10\n"
+ "st1 { v16.8h }, [x27], #0x10\n"
+ "st1 { v20.8h }, [x25], #0x10\n"
+ "tbz x16, #2, 192f\n"
+ "str d9, [x13], #0x8\n"
+ "str d13, [x9], #0x8\n"
+ "str d17, [x27], #0x8\n"
+ "str d21, [x25], #0x8\n"
+ "tbz x16, #1, 191f\n"
+ "st1 { v9.s }[2], [x13], #0x4\n"
+ "st1 { v13.s }[2], [x9], #0x4\n"
+ "st1 { v17.s }[2], [x27], #0x4\n"
+ "st1 { v21.s }[2], [x25], #0x4\n"
+ "tbz x16, #0, 198f\n"
+ "st1 { v9.h }[6], [x13]\n"
+ "st1 { v13.h }[6], [x9]\n"
+ "st1 { v17.h }[6], [x27]\n"
+ "st1 { v21.h }[6], [x25]\n"
+ "b 198f\n"
+ "191:" // Height 4: Partial direct writeback: partial_1_12
+ "tbz x16, #0, 198f\n"
+ "st1 { v9.h }[4], [x13]\n"
+ "st1 { v13.h }[4], [x9]\n"
+ "st1 { v17.h }[4], [x27]\n"
+ "st1 { v21.h }[4], [x25]\n"
+ "b 198f\n"
+ "192:" // Height 4: Partial direct writeback: partial_2_8
+ "tbz x16, #1, 193f\n"
+ "str s9, [x13], #0x4\n"
+ "str s13, [x9], #0x4\n"
+ "str s17, [x27], #0x4\n"
+ "str s21, [x25], #0x4\n"
+ "tbz x16, #0, 198f\n"
+ "st1 { v9.h }[2], [x13]\n"
+ "st1 { v13.h }[2], [x9]\n"
+ "st1 { v17.h }[2], [x27]\n"
+ "st1 { v21.h }[2], [x25]\n"
+ "b 198f\n"
+ "193:" // Height 4: Partial direct writeback: partial_1_8
+ "tbz x16, #0, 198f\n"
+ "str h9, [x13, #0x0]\n"
+ "str h13, [x9, #0x0]\n"
+ "str h17, [x27, #0x0]\n"
+ "str h21, [x25, #0x0]\n"
+ "b 198f\n"
+ "194:" // Height 4: Partial direct writeback: partial_4_0
+ "tbz x16, #2, 196f\n"
+ "str d8, [x13], #0x8\n"
+ "str d12, [x9], #0x8\n"
+ "str d16, [x27], #0x8\n"
+ "str d20, [x25], #0x8\n"
+ "tbz x16, #1, 195f\n"
+ "st1 { v8.s }[2], [x13], #0x4\n"
+ "st1 { v12.s }[2], [x9], #0x4\n"
+ "st1 { v16.s }[2], [x27], #0x4\n"
+ "st1 { v20.s }[2], [x25], #0x4\n"
+ "tbz x16, #0, 198f\n"
+ "st1 { v8.h }[6], [x13]\n"
+ "st1 { v12.h }[6], [x9]\n"
+ "st1 { v16.h }[6], [x27]\n"
+ "st1 { v20.h }[6], [x25]\n"
+ "b 198f\n"
+ "195:" // Height 4: Partial direct writeback: partial_1_4
+ "tbz x16, #0, 198f\n"
+ "st1 { v8.h }[4], [x13]\n"
+ "st1 { v12.h }[4], [x9]\n"
+ "st1 { v16.h }[4], [x27]\n"
+ "st1 { v20.h }[4], [x25]\n"
+ "b 198f\n"
+ "196:" // Height 4: Partial direct writeback: partial_2_0
+ "tbz x16, #1, 197f\n"
+ "str s8, [x13], #0x4\n"
+ "str s12, [x9], #0x4\n"
+ "str s16, [x27], #0x4\n"
+ "str s20, [x25], #0x4\n"
+ "tbz x16, #0, 198f\n"
+ "st1 { v8.h }[2], [x13]\n"
+ "st1 { v12.h }[2], [x9]\n"
+ "st1 { v16.h }[2], [x27]\n"
+ "st1 { v20.h }[2], [x25]\n"
+ "b 198f\n"
+ "197:" // Height 4: Partial direct writeback: partial_1_0
+ "str h8, [x13, #0x0]\n"
+ "str h12, [x9, #0x0]\n"
+ "str h16, [x27, #0x0]\n"
+ "str h20, [x25, #0x0]\n"
+ "198:" // Height 4: Partial direct writeback: Done
+ "b 200f\n"
+ "199:" // Height 4: Full writeback
+ "str q8, [x13, #0x0]\n"
+ "str q9, [x13, #0x10]\n"
+ "str q10, [x13, #0x20]\n"
+ "str q11, [x13, #0x30]\n"
+ "str q12, [x9, #0x0]\n"
+ "str q13, [x9, #0x10]\n"
+ "str q14, [x9, #0x20]\n"
+ "str q15, [x9, #0x30]\n"
+ "str q16, [x27, #0x0]\n"
+ "str q17, [x27, #0x10]\n"
+ "str q18, [x27, #0x20]\n"
+ "str q19, [x27, #0x30]\n"
+ "str q20, [x25, #0x0]\n"
+ "str q21, [x25, #0x10]\n"
+ "str q22, [x25, #0x20]\n"
+ "str q23, [x25, #0x30]\n"
+ "add x13, x13, #0x40\n"
+ "add x9, x9, #0x40\n"
+ "add x27, x27, #0x40\n"
+ "add x25, x25, #0x40\n"
+ "200:" // Height 4: Writeback done
+ "subs x16, x16, #0x20\n"
+ "bgt 153b\n"
+ "b 302f\n"
+ "201:" // Height 5
+ "ldr x16, [%x[args_ptr], %[offsetof_N]]\n"
+ "mov x14, %x[bias]\n"
+ "ldr x15, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 202f\n"
+ "ldr x13, [%x[output_ptr], #0x0]\n"
+ "add x13, x13, x19, LSL #1\n"
+ "ldr x9, [%x[output_ptr], #0x8]\n"
+ "ldr x27, [%x[output_ptr], #0x10]\n"
+ "add x9, x9, x19, LSL #1\n"
+ "ldr x25, [%x[output_ptr], #0x18]\n"
+ "ldr x23, [%x[output_ptr], #0x20]\n"
+ "add x27, x27, x19, LSL #1\n"
+ "add x25, x25, x19, LSL #1\n"
+ "add x23, x23, x19, LSL #1\n"
+ "b 203f\n"
+ "202:" // Height 5: setup direct output
+ "mov x13, %x[output_ptr]\n"
+ "add x9, x13, x19, LSL #1\n"
+ "add x27, x9, x19, LSL #1\n"
+ "add x25, x27, x19, LSL #1\n"
+ "add x23, x25, x19, LSL #1\n"
+ "203:" // Height 5: Column loop
+ "cbz x14, 204f\n"
+ "ldr q8, [x14, #0x0]\n"
+ "mov v12.16b, v8.16b\n"
+ "ldr q9, [x14, #0x10]\n"
+ "mov v16.16b, v8.16b\n"
+ "ldr q10, [x14, #0x20]\n"
+ "mov v20.16b, v8.16b\n"
+ "ldr q11, [x14, #0x30]\n"
+ "mov v24.16b, v8.16b\n"
+ "add x14, x14, #0x40\n"
+ "mov v13.16b, v9.16b\n"
+ "mov v17.16b, v9.16b\n"
+ "mov v14.16b, v10.16b\n"
+ "mov v15.16b, v11.16b\n"
+ "mov v18.16b, v10.16b\n"
+ "mov v19.16b, v11.16b\n"
+ "mov v21.16b, v9.16b\n"
+ "mov v22.16b, v10.16b\n"
+ "mov v23.16b, v11.16b\n"
+ "mov v25.16b, v9.16b\n"
+ "mov v26.16b, v10.16b\n"
+ "mov v27.16b, v11.16b\n"
+ "b 223f\n"
+ "204:" // Height 5: no bias
+ "tbz %x[flags], #0, 222f\n"
+ "cmp x16, #0x20\n"
+ "bge 221f\n"
+ "tbz x16, #4, 212f\n"
+ "ld1 { v8.8h }, [x13], #0x10\n"
+ "ld1 { v12.8h }, [x9], #0x10\n"
+ "ld1 { v16.8h }, [x27], #0x10\n"
+ "ld1 { v20.8h }, [x25], #0x10\n"
+ "ld1 { v24.8h }, [x23], #0x10\n"
+ "ld1 { v9.8h }, [x13], #0x10\n"
+ "ld1 { v13.8h }, [x9], #0x10\n"
+ "ld1 { v17.8h }, [x27], #0x10\n"
+ "ld1 { v21.8h }, [x25], #0x10\n"
+ "ld1 { v25.8h }, [x23], #0x10\n"
+ "tbz x16, #3, 208f\n"
+ "ld1 { v10.8h }, [x13], #0x10\n"
+ "ld1 { v14.8h }, [x9], #0x10\n"
+ "ld1 { v18.8h }, [x27], #0x10\n"
+ "ld1 { v22.8h }, [x25], #0x10\n"
+ "ld1 { v26.8h }, [x23], #0x10\n"
+ "tbz x16, #2, 206f\n"
+ "ldr d11, [x13], #0x8\n"
+ "ldr d15, [x9], #0x8\n"
+ "ldr d19, [x27], #0x8\n"
+ "ldr d23, [x25], #0x8\n"
+ "ldr d27, [x23], #0x8\n"
+ "tbz x16, #1, 205f\n"
+ "ld1 { v11.s }[2], [x13], #0x4\n"
+ "ld1 { v15.s }[2], [x9], #0x4\n"
+ "ld1 { v19.s }[2], [x27], #0x4\n"
+ "ld1 { v23.s }[2], [x25], #0x4\n"
+ "ld1 { v27.s }[2], [x23], #0x4\n"
+ "mov x19, #0x3c\n"
+ "tbz x16, #0, 220f\n"
+ "ld1 { v11.h }[6], [x13]\n"
+ "ld1 { v15.h }[6], [x9]\n"
+ "ld1 { v19.h }[6], [x27]\n"
+ "ld1 { v23.h }[6], [x25]\n"
+ "ld1 { v27.h }[6], [x23]\n"
+ "b 220f\n"
+ "205:" // Height 5: Partial accumulate: partial_1_28
+ "mov x19, #0x38\n"
+ "tbz x16, #0, 220f\n"
+ "ld1 { v11.h }[4], [x13]\n"
+ "ld1 { v15.h }[4], [x9]\n"
+ "ld1 { v19.h }[4], [x27]\n"
+ "ld1 { v23.h }[4], [x25]\n"
+ "ld1 { v27.h }[4], [x23]\n"
+ "b 220f\n"
+ "206:" // Height 5: Partial accumulate: partial_2_24
+ "tbz x16, #1, 207f\n"
+ "ldr s11, [x13], #0x4\n"
+ "ldr s15, [x9], #0x4\n"
+ "ldr s19, [x27], #0x4\n"
+ "ldr s23, [x25], #0x4\n"
+ "ldr s27, [x23], #0x4\n"
+ "mov x19, #0x34\n"
+ "tbz x16, #0, 220f\n"
+ "ld1 { v11.h }[2], [x13]\n"
+ "ld1 { v15.h }[2], [x9]\n"
+ "ld1 { v19.h }[2], [x27]\n"
+ "ld1 { v23.h }[2], [x25]\n"
+ "ld1 { v27.h }[2], [x23]\n"
+ "b 220f\n"
+ "207:" // Height 5: Partial accumulate: partial_1_24
+ "mov x19, #0x30\n"
+ "tbz x16, #0, 220f\n"
+ "ldr h11, [x13, #0x0]\n"
+ "ldr h15, [x9, #0x0]\n"
+ "ldr h19, [x27, #0x0]\n"
+ "ldr h23, [x25, #0x0]\n"
+ "ldr h27, [x23, #0x0]\n"
+ "b 220f\n"
+ "208:" // Height 5: Partial accumulate: partial_4_16
+ "tbz x16, #2, 210f\n"
+ "ldr d10, [x13], #0x8\n"
+ "ldr d14, [x9], #0x8\n"
+ "ldr d18, [x27], #0x8\n"
+ "ldr d22, [x25], #0x8\n"
+ "ldr d26, [x23], #0x8\n"
+ "tbz x16, #1, 209f\n"
+ "ld1 { v10.s }[2], [x13], #0x4\n"
+ "ld1 { v14.s }[2], [x9], #0x4\n"
+ "ld1 { v18.s }[2], [x27], #0x4\n"
+ "ld1 { v22.s }[2], [x25], #0x4\n"
+ "ld1 { v26.s }[2], [x23], #0x4\n"
+ "mov x19, #0x2c\n"
+ "tbz x16, #0, 220f\n"
+ "ld1 { v10.h }[6], [x13]\n"
+ "ld1 { v14.h }[6], [x9]\n"
+ "ld1 { v18.h }[6], [x27]\n"
+ "ld1 { v22.h }[6], [x25]\n"
+ "ld1 { v26.h }[6], [x23]\n"
+ "b 220f\n"
+ "209:" // Height 5: Partial accumulate: partial_1_20
+ "mov x19, #0x28\n"
+ "tbz x16, #0, 220f\n"
+ "ld1 { v10.h }[4], [x13]\n"
+ "ld1 { v14.h }[4], [x9]\n"
+ "ld1 { v18.h }[4], [x27]\n"
+ "ld1 { v22.h }[4], [x25]\n"
+ "ld1 { v26.h }[4], [x23]\n"
+ "b 220f\n"
+ "210:" // Height 5: Partial accumulate: partial_2_16
+ "tbz x16, #1, 211f\n"
+ "ldr s10, [x13], #0x4\n"
+ "ldr s14, [x9], #0x4\n"
+ "ldr s18, [x27], #0x4\n"
+ "ldr s22, [x25], #0x4\n"
+ "ldr s26, [x23], #0x4\n"
+ "mov x19, #0x24\n"
+ "tbz x16, #0, 220f\n"
+ "ld1 { v10.h }[2], [x13]\n"
+ "ld1 { v14.h }[2], [x9]\n"
+ "ld1 { v18.h }[2], [x27]\n"
+ "ld1 { v22.h }[2], [x25]\n"
+ "ld1 { v26.h }[2], [x23]\n"
+ "b 220f\n"
+ "211:" // Height 5: Partial accumulate: partial_1_16
+ "mov x19, #0x20\n"
+ "tbz x16, #0, 220f\n"
+ "ldr h10, [x13, #0x0]\n"
+ "ldr h14, [x9, #0x0]\n"
+ "ldr h18, [x27, #0x0]\n"
+ "ldr h22, [x25, #0x0]\n"
+ "ldr h26, [x23, #0x0]\n"
+ "b 220f\n"
+ "212:" // Height 5: Partial accumulate: partial_8_0
+ "tbz x16, #3, 216f\n"
+ "ld1 { v8.8h }, [x13], #0x10\n"
+ "ld1 { v12.8h }, [x9], #0x10\n"
+ "ld1 { v16.8h }, [x27], #0x10\n"
+ "ld1 { v20.8h }, [x25], #0x10\n"
+ "ld1 { v24.8h }, [x23], #0x10\n"
+ "tbz x16, #2, 214f\n"
+ "ldr d9, [x13], #0x8\n"
+ "ldr d13, [x9], #0x8\n"
+ "ldr d17, [x27], #0x8\n"
+ "ldr d21, [x25], #0x8\n"
+ "ldr d25, [x23], #0x8\n"
+ "tbz x16, #1, 213f\n"
+ "ld1 { v9.s }[2], [x13], #0x4\n"
+ "ld1 { v13.s }[2], [x9], #0x4\n"
+ "ld1 { v17.s }[2], [x27], #0x4\n"
+ "ld1 { v21.s }[2], [x25], #0x4\n"
+ "ld1 { v25.s }[2], [x23], #0x4\n"
+ "mov x19, #0x1c\n"
+ "tbz x16, #0, 220f\n"
+ "ld1 { v9.h }[6], [x13]\n"
+ "ld1 { v13.h }[6], [x9]\n"
+ "ld1 { v17.h }[6], [x27]\n"
+ "ld1 { v21.h }[6], [x25]\n"
+ "ld1 { v25.h }[6], [x23]\n"
+ "b 220f\n"
+ "213:" // Height 5: Partial accumulate: partial_1_12
+ "mov x19, #0x18\n"
+ "tbz x16, #0, 220f\n"
+ "ld1 { v9.h }[4], [x13]\n"
+ "ld1 { v13.h }[4], [x9]\n"
+ "ld1 { v17.h }[4], [x27]\n"
+ "ld1 { v21.h }[4], [x25]\n"
+ "ld1 { v25.h }[4], [x23]\n"
+ "b 220f\n"
+ "214:" // Height 5: Partial accumulate: partial_2_8
+ "tbz x16, #1, 215f\n"
+ "ldr s9, [x13], #0x4\n"
+ "ldr s13, [x9], #0x4\n"
+ "ldr s17, [x27], #0x4\n"
+ "ldr s21, [x25], #0x4\n"
+ "ldr s25, [x23], #0x4\n"
+ "mov x19, #0x14\n"
+ "tbz x16, #0, 220f\n"
+ "ld1 { v9.h }[2], [x13]\n"
+ "ld1 { v13.h }[2], [x9]\n"
+ "ld1 { v17.h }[2], [x27]\n"
+ "ld1 { v21.h }[2], [x25]\n"
+ "ld1 { v25.h }[2], [x23]\n"
+ "b 220f\n"
+ "215:" // Height 5: Partial accumulate: partial_1_8
+ "mov x19, #0x10\n"
+ "tbz x16, #0, 220f\n"
+ "ldr h9, [x13, #0x0]\n"
+ "ldr h13, [x9, #0x0]\n"
+ "ldr h17, [x27, #0x0]\n"
+ "ldr h21, [x25, #0x0]\n"
+ "ldr h25, [x23, #0x0]\n"
+ "b 220f\n"
+ "216:" // Height 5: Partial accumulate: partial_4_0
+ "tbz x16, #2, 218f\n"
+ "ldr d8, [x13], #0x8\n"
+ "ldr d12, [x9], #0x8\n"
+ "ldr d16, [x27], #0x8\n"
+ "ldr d20, [x25], #0x8\n"
+ "ldr d24, [x23], #0x8\n"
+ "tbz x16, #1, 217f\n"
+ "ld1 { v8.s }[2], [x13], #0x4\n"
+ "ld1 { v12.s }[2], [x9], #0x4\n"
+ "ld1 { v16.s }[2], [x27], #0x4\n"
+ "ld1 { v20.s }[2], [x25], #0x4\n"
+ "ld1 { v24.s }[2], [x23], #0x4\n"
+ "mov x19, #0xc\n"
+ "tbz x16, #0, 220f\n"
+ "ld1 { v8.h }[6], [x13]\n"
+ "ld1 { v12.h }[6], [x9]\n"
+ "ld1 { v16.h }[6], [x27]\n"
+ "ld1 { v20.h }[6], [x25]\n"
+ "ld1 { v24.h }[6], [x23]\n"
+ "b 220f\n"
+ "217:" // Height 5: Partial accumulate: partial_1_4
+ "mov x19, #0x8\n"
+ "tbz x16, #0, 220f\n"
+ "ld1 { v8.h }[4], [x13]\n"
+ "ld1 { v12.h }[4], [x9]\n"
+ "ld1 { v16.h }[4], [x27]\n"
+ "ld1 { v20.h }[4], [x25]\n"
+ "ld1 { v24.h }[4], [x23]\n"
+ "b 220f\n"
+ "218:" // Height 5: Partial accumulate: partial_2_0
+ "tbz x16, #1, 219f\n"
+ "ldr s8, [x13], #0x4\n"
+ "ldr s12, [x9], #0x4\n"
+ "ldr s16, [x27], #0x4\n"
+ "ldr s20, [x25], #0x4\n"
+ "ldr s24, [x23], #0x4\n"
+ "mov x19, #0x4\n"
+ "tbz x16, #0, 220f\n"
+ "ld1 { v8.h }[2], [x13]\n"
+ "ld1 { v12.h }[2], [x9]\n"
+ "ld1 { v16.h }[2], [x27]\n"
+ "ld1 { v20.h }[2], [x25]\n"
+ "ld1 { v24.h }[2], [x23]\n"
+ "b 220f\n"
+ "219:" // Height 5: Partial accumulate: partial_1_0
+ "mov x19, #0x0\n"
+ "ldr h8, [x13, #0x0]\n"
+ "ldr h12, [x9, #0x0]\n"
+ "ldr h16, [x27, #0x0]\n"
+ "ldr h20, [x25, #0x0]\n"
+ "ldr h24, [x23, #0x0]\n"
+ "220:" // Height 5: Partial accumulate: Done
+ "sub x13, x13, x19\n"
+ "sub x9, x9, x19\n"
+ "sub x27, x27, x19\n"
+ "sub x25, x25, x19\n"
+ "sub x23, x23, x19\n"
+ "b 223f\n"
+ "221:" // Height 5: full accumulate
+ "ldr q8, [x13, #0x0]\n"
+ "ldr q9, [x13, #0x10]\n"
+ "ldr q10, [x13, #0x20]\n"
+ "ldr q11, [x13, #0x30]\n"
+ "ldr q12, [x9, #0x0]\n"
+ "ldr q13, [x9, #0x10]\n"
+ "ldr q14, [x9, #0x20]\n"
+ "ldr q15, [x9, #0x30]\n"
+ "ldr q16, [x27, #0x0]\n"
+ "ldr q17, [x27, #0x10]\n"
+ "ldr q18, [x27, #0x20]\n"
+ "ldr q19, [x27, #0x30]\n"
+ "ldr q20, [x25, #0x0]\n"
+ "ldr q21, [x25, #0x10]\n"
+ "ldr q22, [x25, #0x20]\n"
+ "ldr q23, [x25, #0x30]\n"
+ "ldr q24, [x23, #0x0]\n"
+ "ldr q25, [x23, #0x10]\n"
+ "ldr q26, [x23, #0x20]\n"
+ "ldr q27, [x23, #0x30]\n"
+ "b 223f\n"
+ "222:" // Height 5: no accumulate
+ "movi v8.16b, #0x0\n"
+ "movi v9.16b, #0x0\n"
+ "movi v10.16b, #0x0\n"
+ "movi v11.16b, #0x0\n"
+ "movi v12.16b, #0x0\n"
+ "movi v13.16b, #0x0\n"
+ "movi v14.16b, #0x0\n"
+ "movi v15.16b, #0x0\n"
+ "movi v16.16b, #0x0\n"
+ "movi v17.16b, #0x0\n"
+ "movi v18.16b, #0x0\n"
+ "movi v19.16b, #0x0\n"
+ "movi v20.16b, #0x0\n"
+ "movi v21.16b, #0x0\n"
+ "movi v22.16b, #0x0\n"
+ "movi v23.16b, #0x0\n"
+ "movi v24.16b, #0x0\n"
+ "movi v25.16b, #0x0\n"
+ "movi v26.16b, #0x0\n"
+ "movi v27.16b, #0x0\n"
+ "223:" // Height 5: setup done
+ "mov x12, #0x0\n"
+ "224:" // Height 5: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w11, [x20, x12, LSL #0x2]\n"
+ "tbz %x[flags], #3, 225f\n"
+ "ldr x20, [%x[input_ptr], x12, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x10, [x20, #0x0]\n"
+ "ldr x28, [x20, #0x8]\n"
+ "ldr x26, [x20, #0x10]\n"
+ "ldr x24, [x20, #0x18]\n"
+ "ldr x22, [x20, #0x20]\n"
+ "cbnz x12, 226f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x10, x10, x19, LSL #1\n"
+ "add x28, x28, x19, LSL #1\n"
+ "add x26, x26, x19, LSL #1\n"
+ "add x24, x24, x19, LSL #1\n"
+ "add x22, x22, x19, LSL #1\n"
+ "b 226f\n"
+ "225:" // Height 5: setup direct input
+ "mov x10, %x[input_ptr]\n"
+ "add x28, x10, x19, LSL #1\n"
+ "add x26, x28, x19, LSL #1\n"
+ "add x24, x26, x19, LSL #1\n"
+ "add x22, x24, x19, LSL #1\n"
+ "226:" // Height 5: input setup done
+ "cmp x11, #0x8\n"
+ "blt 229f\n"
+ "cmp x11, #0x10\n"
+ "blt 228f\n"
+ "227:" // Height 5: Multiply loop: Main loop head
+ "ldr q0, [x10, #0x0]\n"
+ "ldr q1, [x28, #0x0]\n"
+ "ldr q2, [x26, #0x0]\n"
+ "ldr q3, [x24, #0x0]\n"
+ "ldr q4, [x22, #0x0]\n"
+ "ldr q6, [x15, #0x0]\n"
+ "fmla v8.8h, v6.8h, v0.h[0]\n"
+ "ldr q7, [x15, #0x10]\n"
+ "fmla v12.8h, v6.8h, v1.h[0]\n"
+ "add x10, x10, #0x10\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "fmla v16.8h, v6.8h, v2.h[0]\n"
+ "add x28, x28, #0x10\n"
+ "fmla v20.8h, v6.8h, v3.h[0]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "add x26, x26, #0x10\n"
+ "fmla v24.8h, v6.8h, v4.h[0]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "ldr q6, [x15, #0x20]\n"
+ "fmla v9.8h, v7.8h, v0.h[0]\n"
+ "add x24, x24, #0x10\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "fmla v13.8h, v7.8h, v1.h[0]\n"
+ "add x22, x22, #0x10\n"
+ "fmla v17.8h, v7.8h, v2.h[0]\n"
+ "prfm pldl1keep, [x22, #0x80]\n"
+ "sub x11, x11, #0x8\n"
+ "fmla v21.8h, v7.8h, v3.h[0]\n"
+ "cmp x11, #0x10\n"
+ "fmla v25.8h, v7.8h, v4.h[0]\n"
+ "ldr q7, [x15, #0x30]\n"
+ "fmla v10.8h, v6.8h, v0.h[0]\n"
+ "fmla v14.8h, v6.8h, v1.h[0]\n"
+ "fmla v18.8h, v6.8h, v2.h[0]\n"
+ "fmla v22.8h, v6.8h, v3.h[0]\n"
+ "fmla v26.8h, v6.8h, v4.h[0]\n"
+ "ldr q6, [x15, #0x40]\n"
+ "fmla v11.8h, v7.8h, v0.h[0]\n"
+ "fmla v15.8h, v7.8h, v1.h[0]\n"
+ "fmla v19.8h, v7.8h, v2.h[0]\n"
+ "fmla v23.8h, v7.8h, v3.h[0]\n"
+ "fmla v27.8h, v7.8h, v4.h[0]\n"
+ "ldr q7, [x15, #0x50]\n"
+ "fmla v8.8h, v6.8h, v0.h[1]\n"
+ "fmla v12.8h, v6.8h, v1.h[1]\n"
+ "fmla v16.8h, v6.8h, v2.h[1]\n"
+ "fmla v20.8h, v6.8h, v3.h[1]\n"
+ "fmla v24.8h, v6.8h, v4.h[1]\n"
+ "ldr q6, [x15, #0x60]\n"
+ "fmla v9.8h, v7.8h, v0.h[1]\n"
+ "fmla v13.8h, v7.8h, v1.h[1]\n"
+ "fmla v17.8h, v7.8h, v2.h[1]\n"
+ "fmla v21.8h, v7.8h, v3.h[1]\n"
+ "fmla v25.8h, v7.8h, v4.h[1]\n"
+ "ldr q7, [x15, #0x70]\n"
+ "fmla v10.8h, v6.8h, v0.h[1]\n"
+ "fmla v14.8h, v6.8h, v1.h[1]\n"
+ "fmla v18.8h, v6.8h, v2.h[1]\n"
+ "fmla v22.8h, v6.8h, v3.h[1]\n"
+ "fmla v26.8h, v6.8h, v4.h[1]\n"
+ "ldr q6, [x15, #0x80]\n"
+ "fmla v11.8h, v7.8h, v0.h[1]\n"
+ "fmla v15.8h, v7.8h, v1.h[1]\n"
+ "fmla v19.8h, v7.8h, v2.h[1]\n"
+ "fmla v23.8h, v7.8h, v3.h[1]\n"
+ "fmla v27.8h, v7.8h, v4.h[1]\n"
+ "ldr q7, [x15, #0x90]\n"
+ "fmla v8.8h, v6.8h, v0.h[2]\n"
+ "fmla v12.8h, v6.8h, v1.h[2]\n"
+ "fmla v16.8h, v6.8h, v2.h[2]\n"
+ "fmla v20.8h, v6.8h, v3.h[2]\n"
+ "fmla v24.8h, v6.8h, v4.h[2]\n"
+ "ldr q6, [x15, #0xa0]\n"
+ "fmla v9.8h, v7.8h, v0.h[2]\n"
+ "fmla v13.8h, v7.8h, v1.h[2]\n"
+ "fmla v17.8h, v7.8h, v2.h[2]\n"
+ "fmla v21.8h, v7.8h, v3.h[2]\n"
+ "fmla v25.8h, v7.8h, v4.h[2]\n"
+ "ldr q7, [x15, #0xb0]\n"
+ "fmla v10.8h, v6.8h, v0.h[2]\n"
+ "fmla v14.8h, v6.8h, v1.h[2]\n"
+ "fmla v18.8h, v6.8h, v2.h[2]\n"
+ "fmla v22.8h, v6.8h, v3.h[2]\n"
+ "fmla v26.8h, v6.8h, v4.h[2]\n"
+ "ldr q6, [x15, #0xc0]\n"
+ "fmla v11.8h, v7.8h, v0.h[2]\n"
+ "fmla v15.8h, v7.8h, v1.h[2]\n"
+ "fmla v19.8h, v7.8h, v2.h[2]\n"
+ "fmla v23.8h, v7.8h, v3.h[2]\n"
+ "fmla v27.8h, v7.8h, v4.h[2]\n"
+ "ldr q7, [x15, #0xd0]\n"
+ "fmla v8.8h, v6.8h, v0.h[3]\n"
+ "fmla v12.8h, v6.8h, v1.h[3]\n"
+ "fmla v16.8h, v6.8h, v2.h[3]\n"
+ "fmla v20.8h, v6.8h, v3.h[3]\n"
+ "fmla v24.8h, v6.8h, v4.h[3]\n"
+ "ldr q6, [x15, #0xe0]\n"
+ "fmla v9.8h, v7.8h, v0.h[3]\n"
+ "fmla v13.8h, v7.8h, v1.h[3]\n"
+ "fmla v17.8h, v7.8h, v2.h[3]\n"
+ "fmla v21.8h, v7.8h, v3.h[3]\n"
+ "fmla v25.8h, v7.8h, v4.h[3]\n"
+ "ldr q7, [x15, #0xf0]\n"
+ "fmla v10.8h, v6.8h, v0.h[3]\n"
+ "fmla v14.8h, v6.8h, v1.h[3]\n"
+ "fmla v18.8h, v6.8h, v2.h[3]\n"
+ "fmla v22.8h, v6.8h, v3.h[3]\n"
+ "fmla v26.8h, v6.8h, v4.h[3]\n"
+ "ldr q6, [x15, #0x100]\n"
+ "fmla v11.8h, v7.8h, v0.h[3]\n"
+ "fmla v15.8h, v7.8h, v1.h[3]\n"
+ "fmla v19.8h, v7.8h, v2.h[3]\n"
+ "fmla v23.8h, v7.8h, v3.h[3]\n"
+ "fmla v27.8h, v7.8h, v4.h[3]\n"
+ "ldr q7, [x15, #0x110]\n"
+ "fmla v8.8h, v6.8h, v0.h[4]\n"
+ "fmla v12.8h, v6.8h, v1.h[4]\n"
+ "fmla v16.8h, v6.8h, v2.h[4]\n"
+ "fmla v20.8h, v6.8h, v3.h[4]\n"
+ "fmla v24.8h, v6.8h, v4.h[4]\n"
+ "ldr q6, [x15, #0x120]\n"
+ "fmla v9.8h, v7.8h, v0.h[4]\n"
+ "fmla v13.8h, v7.8h, v1.h[4]\n"
+ "fmla v17.8h, v7.8h, v2.h[4]\n"
+ "fmla v21.8h, v7.8h, v3.h[4]\n"
+ "fmla v25.8h, v7.8h, v4.h[4]\n"
+ "ldr q7, [x15, #0x130]\n"
+ "fmla v10.8h, v6.8h, v0.h[4]\n"
+ "fmla v14.8h, v6.8h, v1.h[4]\n"
+ "fmla v18.8h, v6.8h, v2.h[4]\n"
+ "fmla v22.8h, v6.8h, v3.h[4]\n"
+ "fmla v26.8h, v6.8h, v4.h[4]\n"
+ "ldr q6, [x15, #0x140]\n"
+ "fmla v11.8h, v7.8h, v0.h[4]\n"
+ "fmla v15.8h, v7.8h, v1.h[4]\n"
+ "fmla v19.8h, v7.8h, v2.h[4]\n"
+ "fmla v23.8h, v7.8h, v3.h[4]\n"
+ "fmla v27.8h, v7.8h, v4.h[4]\n"
+ "ldr q7, [x15, #0x150]\n"
+ "fmla v8.8h, v6.8h, v0.h[5]\n"
+ "fmla v12.8h, v6.8h, v1.h[5]\n"
+ "fmla v16.8h, v6.8h, v2.h[5]\n"
+ "fmla v20.8h, v6.8h, v3.h[5]\n"
+ "fmla v24.8h, v6.8h, v4.h[5]\n"
+ "ldr q6, [x15, #0x160]\n"
+ "fmla v9.8h, v7.8h, v0.h[5]\n"
+ "fmla v13.8h, v7.8h, v1.h[5]\n"
+ "fmla v17.8h, v7.8h, v2.h[5]\n"
+ "fmla v21.8h, v7.8h, v3.h[5]\n"
+ "fmla v25.8h, v7.8h, v4.h[5]\n"
+ "ldr q7, [x15, #0x170]\n"
+ "fmla v10.8h, v6.8h, v0.h[5]\n"
+ "fmla v14.8h, v6.8h, v1.h[5]\n"
+ "fmla v18.8h, v6.8h, v2.h[5]\n"
+ "fmla v22.8h, v6.8h, v3.h[5]\n"
+ "fmla v26.8h, v6.8h, v4.h[5]\n"
+ "ldr q6, [x15, #0x180]\n"
+ "fmla v11.8h, v7.8h, v0.h[5]\n"
+ "fmla v15.8h, v7.8h, v1.h[5]\n"
+ "fmla v19.8h, v7.8h, v2.h[5]\n"
+ "fmla v23.8h, v7.8h, v3.h[5]\n"
+ "fmla v27.8h, v7.8h, v4.h[5]\n"
+ "ldr q7, [x15, #0x190]\n"
+ "fmla v8.8h, v6.8h, v0.h[6]\n"
+ "fmla v12.8h, v6.8h, v1.h[6]\n"
+ "fmla v16.8h, v6.8h, v2.h[6]\n"
+ "fmla v20.8h, v6.8h, v3.h[6]\n"
+ "fmla v24.8h, v6.8h, v4.h[6]\n"
+ "ldr q6, [x15, #0x1a0]\n"
+ "fmla v9.8h, v7.8h, v0.h[6]\n"
+ "fmla v13.8h, v7.8h, v1.h[6]\n"
+ "fmla v17.8h, v7.8h, v2.h[6]\n"
+ "fmla v21.8h, v7.8h, v3.h[6]\n"
+ "fmla v25.8h, v7.8h, v4.h[6]\n"
+ "ldr q7, [x15, #0x1b0]\n"
+ "fmla v10.8h, v6.8h, v0.h[6]\n"
+ "fmla v14.8h, v6.8h, v1.h[6]\n"
+ "fmla v18.8h, v6.8h, v2.h[6]\n"
+ "fmla v22.8h, v6.8h, v3.h[6]\n"
+ "fmla v26.8h, v6.8h, v4.h[6]\n"
+ "ldr q6, [x15, #0x1c0]\n"
+ "fmla v11.8h, v7.8h, v0.h[6]\n"
+ "fmla v15.8h, v7.8h, v1.h[6]\n"
+ "fmla v19.8h, v7.8h, v2.h[6]\n"
+ "fmla v23.8h, v7.8h, v3.h[6]\n"
+ "fmla v27.8h, v7.8h, v4.h[6]\n"
+ "ldr q7, [x15, #0x1d0]\n"
+ "fmla v8.8h, v6.8h, v0.h[7]\n"
+ "fmla v12.8h, v6.8h, v1.h[7]\n"
+ "fmla v16.8h, v6.8h, v2.h[7]\n"
+ "fmla v20.8h, v6.8h, v3.h[7]\n"
+ "fmla v24.8h, v6.8h, v4.h[7]\n"
+ "ldr q6, [x15, #0x1e0]\n"
+ "fmla v9.8h, v7.8h, v0.h[7]\n"
+ "fmla v13.8h, v7.8h, v1.h[7]\n"
+ "fmla v17.8h, v7.8h, v2.h[7]\n"
+ "fmla v21.8h, v7.8h, v3.h[7]\n"
+ "fmla v25.8h, v7.8h, v4.h[7]\n"
+ "ldr q7, [x15, #0x1f0]\n"
+ "fmla v10.8h, v6.8h, v0.h[7]\n"
+ "add x15, x15, #0x200\n"
+ "fmla v14.8h, v6.8h, v1.h[7]\n"
+ "fmla v18.8h, v6.8h, v2.h[7]\n"
+ "fmla v22.8h, v6.8h, v3.h[7]\n"
+ "fmla v26.8h, v6.8h, v4.h[7]\n"
+ "fmla v11.8h, v7.8h, v0.h[7]\n"
+ "fmla v15.8h, v7.8h, v1.h[7]\n"
+ "fmla v19.8h, v7.8h, v2.h[7]\n"
+ "fmla v23.8h, v7.8h, v3.h[7]\n"
+ "fmla v27.8h, v7.8h, v4.h[7]\n"
+ "bge 227b\n"
+ "228:" // Height 5: Multiply loop: Single iteration only
+ "sub x11, x11, #0x8\n"
+ "ldr q0, [x10, #0x0]\n"
+ "ldr q1, [x28, #0x0]\n"
+ "ldr q2, [x26, #0x0]\n"
+ "ldr q3, [x24, #0x0]\n"
+ "ldr q4, [x22, #0x0]\n"
+ "ldr q6, [x15, #0x0]\n"
+ "fmla v8.8h, v6.8h, v0.h[0]\n"
+ "ldr q7, [x15, #0x10]\n"
+ "fmla v12.8h, v6.8h, v1.h[0]\n"
+ "add x10, x10, #0x10\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "fmla v16.8h, v6.8h, v2.h[0]\n"
+ "add x28, x28, #0x10\n"
+ "fmla v20.8h, v6.8h, v3.h[0]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "add x26, x26, #0x10\n"
+ "fmla v24.8h, v6.8h, v4.h[0]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "ldr q6, [x15, #0x20]\n"
+ "fmla v9.8h, v7.8h, v0.h[0]\n"
+ "add x24, x24, #0x10\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "fmla v13.8h, v7.8h, v1.h[0]\n"
+ "add x22, x22, #0x10\n"
+ "fmla v17.8h, v7.8h, v2.h[0]\n"
+ "prfm pldl1keep, [x22, #0x80]\n"
+ "fmla v21.8h, v7.8h, v3.h[0]\n"
+ "fmla v25.8h, v7.8h, v4.h[0]\n"
+ "ldr q7, [x15, #0x30]\n"
+ "fmla v10.8h, v6.8h, v0.h[0]\n"
+ "fmla v14.8h, v6.8h, v1.h[0]\n"
+ "fmla v18.8h, v6.8h, v2.h[0]\n"
+ "fmla v22.8h, v6.8h, v3.h[0]\n"
+ "fmla v26.8h, v6.8h, v4.h[0]\n"
+ "ldr q6, [x15, #0x40]\n"
+ "fmla v11.8h, v7.8h, v0.h[0]\n"
+ "fmla v15.8h, v7.8h, v1.h[0]\n"
+ "fmla v19.8h, v7.8h, v2.h[0]\n"
+ "fmla v23.8h, v7.8h, v3.h[0]\n"
+ "fmla v27.8h, v7.8h, v4.h[0]\n"
+ "ldr q7, [x15, #0x50]\n"
+ "fmla v8.8h, v6.8h, v0.h[1]\n"
+ "fmla v12.8h, v6.8h, v1.h[1]\n"
+ "fmla v16.8h, v6.8h, v2.h[1]\n"
+ "fmla v20.8h, v6.8h, v3.h[1]\n"
+ "fmla v24.8h, v6.8h, v4.h[1]\n"
+ "ldr q6, [x15, #0x60]\n"
+ "fmla v9.8h, v7.8h, v0.h[1]\n"
+ "fmla v13.8h, v7.8h, v1.h[1]\n"
+ "fmla v17.8h, v7.8h, v2.h[1]\n"
+ "fmla v21.8h, v7.8h, v3.h[1]\n"
+ "fmla v25.8h, v7.8h, v4.h[1]\n"
+ "ldr q7, [x15, #0x70]\n"
+ "fmla v10.8h, v6.8h, v0.h[1]\n"
+ "fmla v14.8h, v6.8h, v1.h[1]\n"
+ "fmla v18.8h, v6.8h, v2.h[1]\n"
+ "fmla v22.8h, v6.8h, v3.h[1]\n"
+ "fmla v26.8h, v6.8h, v4.h[1]\n"
+ "ldr q6, [x15, #0x80]\n"
+ "fmla v11.8h, v7.8h, v0.h[1]\n"
+ "fmla v15.8h, v7.8h, v1.h[1]\n"
+ "fmla v19.8h, v7.8h, v2.h[1]\n"
+ "fmla v23.8h, v7.8h, v3.h[1]\n"
+ "fmla v27.8h, v7.8h, v4.h[1]\n"
+ "ldr q7, [x15, #0x90]\n"
+ "fmla v8.8h, v6.8h, v0.h[2]\n"
+ "fmla v12.8h, v6.8h, v1.h[2]\n"
+ "fmla v16.8h, v6.8h, v2.h[2]\n"
+ "fmla v20.8h, v6.8h, v3.h[2]\n"
+ "fmla v24.8h, v6.8h, v4.h[2]\n"
+ "ldr q6, [x15, #0xa0]\n"
+ "fmla v9.8h, v7.8h, v0.h[2]\n"
+ "fmla v13.8h, v7.8h, v1.h[2]\n"
+ "fmla v17.8h, v7.8h, v2.h[2]\n"
+ "fmla v21.8h, v7.8h, v3.h[2]\n"
+ "fmla v25.8h, v7.8h, v4.h[2]\n"
+ "ldr q7, [x15, #0xb0]\n"
+ "fmla v10.8h, v6.8h, v0.h[2]\n"
+ "fmla v14.8h, v6.8h, v1.h[2]\n"
+ "fmla v18.8h, v6.8h, v2.h[2]\n"
+ "fmla v22.8h, v6.8h, v3.h[2]\n"
+ "fmla v26.8h, v6.8h, v4.h[2]\n"
+ "ldr q6, [x15, #0xc0]\n"
+ "fmla v11.8h, v7.8h, v0.h[2]\n"
+ "fmla v15.8h, v7.8h, v1.h[2]\n"
+ "fmla v19.8h, v7.8h, v2.h[2]\n"
+ "fmla v23.8h, v7.8h, v3.h[2]\n"
+ "fmla v27.8h, v7.8h, v4.h[2]\n"
+ "ldr q7, [x15, #0xd0]\n"
+ "fmla v8.8h, v6.8h, v0.h[3]\n"
+ "fmla v12.8h, v6.8h, v1.h[3]\n"
+ "fmla v16.8h, v6.8h, v2.h[3]\n"
+ "fmla v20.8h, v6.8h, v3.h[3]\n"
+ "fmla v24.8h, v6.8h, v4.h[3]\n"
+ "ldr q6, [x15, #0xe0]\n"
+ "fmla v9.8h, v7.8h, v0.h[3]\n"
+ "fmla v13.8h, v7.8h, v1.h[3]\n"
+ "fmla v17.8h, v7.8h, v2.h[3]\n"
+ "fmla v21.8h, v7.8h, v3.h[3]\n"
+ "fmla v25.8h, v7.8h, v4.h[3]\n"
+ "ldr q7, [x15, #0xf0]\n"
+ "fmla v10.8h, v6.8h, v0.h[3]\n"
+ "fmla v14.8h, v6.8h, v1.h[3]\n"
+ "fmla v18.8h, v6.8h, v2.h[3]\n"
+ "fmla v22.8h, v6.8h, v3.h[3]\n"
+ "fmla v26.8h, v6.8h, v4.h[3]\n"
+ "ldr q6, [x15, #0x100]\n"
+ "fmla v11.8h, v7.8h, v0.h[3]\n"
+ "fmla v15.8h, v7.8h, v1.h[3]\n"
+ "fmla v19.8h, v7.8h, v2.h[3]\n"
+ "fmla v23.8h, v7.8h, v3.h[3]\n"
+ "fmla v27.8h, v7.8h, v4.h[3]\n"
+ "ldr q7, [x15, #0x110]\n"
+ "fmla v8.8h, v6.8h, v0.h[4]\n"
+ "fmla v12.8h, v6.8h, v1.h[4]\n"
+ "fmla v16.8h, v6.8h, v2.h[4]\n"
+ "fmla v20.8h, v6.8h, v3.h[4]\n"
+ "fmla v24.8h, v6.8h, v4.h[4]\n"
+ "ldr q6, [x15, #0x120]\n"
+ "fmla v9.8h, v7.8h, v0.h[4]\n"
+ "fmla v13.8h, v7.8h, v1.h[4]\n"
+ "fmla v17.8h, v7.8h, v2.h[4]\n"
+ "fmla v21.8h, v7.8h, v3.h[4]\n"
+ "fmla v25.8h, v7.8h, v4.h[4]\n"
+ "ldr q7, [x15, #0x130]\n"
+ "fmla v10.8h, v6.8h, v0.h[4]\n"
+ "fmla v14.8h, v6.8h, v1.h[4]\n"
+ "fmla v18.8h, v6.8h, v2.h[4]\n"
+ "fmla v22.8h, v6.8h, v3.h[4]\n"
+ "fmla v26.8h, v6.8h, v4.h[4]\n"
+ "ldr q6, [x15, #0x140]\n"
+ "fmla v11.8h, v7.8h, v0.h[4]\n"
+ "fmla v15.8h, v7.8h, v1.h[4]\n"
+ "fmla v19.8h, v7.8h, v2.h[4]\n"
+ "fmla v23.8h, v7.8h, v3.h[4]\n"
+ "fmla v27.8h, v7.8h, v4.h[4]\n"
+ "ldr q7, [x15, #0x150]\n"
+ "fmla v8.8h, v6.8h, v0.h[5]\n"
+ "fmla v12.8h, v6.8h, v1.h[5]\n"
+ "fmla v16.8h, v6.8h, v2.h[5]\n"
+ "fmla v20.8h, v6.8h, v3.h[5]\n"
+ "fmla v24.8h, v6.8h, v4.h[5]\n"
+ "ldr q6, [x15, #0x160]\n"
+ "fmla v9.8h, v7.8h, v0.h[5]\n"
+ "fmla v13.8h, v7.8h, v1.h[5]\n"
+ "fmla v17.8h, v7.8h, v2.h[5]\n"
+ "fmla v21.8h, v7.8h, v3.h[5]\n"
+ "fmla v25.8h, v7.8h, v4.h[5]\n"
+ "ldr q7, [x15, #0x170]\n"
+ "fmla v10.8h, v6.8h, v0.h[5]\n"
+ "fmla v14.8h, v6.8h, v1.h[5]\n"
+ "fmla v18.8h, v6.8h, v2.h[5]\n"
+ "fmla v22.8h, v6.8h, v3.h[5]\n"
+ "fmla v26.8h, v6.8h, v4.h[5]\n"
+ "ldr q6, [x15, #0x180]\n"
+ "fmla v11.8h, v7.8h, v0.h[5]\n"
+ "fmla v15.8h, v7.8h, v1.h[5]\n"
+ "fmla v19.8h, v7.8h, v2.h[5]\n"
+ "fmla v23.8h, v7.8h, v3.h[5]\n"
+ "fmla v27.8h, v7.8h, v4.h[5]\n"
+ "ldr q7, [x15, #0x190]\n"
+ "fmla v8.8h, v6.8h, v0.h[6]\n"
+ "fmla v12.8h, v6.8h, v1.h[6]\n"
+ "fmla v16.8h, v6.8h, v2.h[6]\n"
+ "fmla v20.8h, v6.8h, v3.h[6]\n"
+ "fmla v24.8h, v6.8h, v4.h[6]\n"
+ "ldr q6, [x15, #0x1a0]\n"
+ "fmla v9.8h, v7.8h, v0.h[6]\n"
+ "fmla v13.8h, v7.8h, v1.h[6]\n"
+ "fmla v17.8h, v7.8h, v2.h[6]\n"
+ "fmla v21.8h, v7.8h, v3.h[6]\n"
+ "fmla v25.8h, v7.8h, v4.h[6]\n"
+ "ldr q7, [x15, #0x1b0]\n"
+ "fmla v10.8h, v6.8h, v0.h[6]\n"
+ "fmla v14.8h, v6.8h, v1.h[6]\n"
+ "fmla v18.8h, v6.8h, v2.h[6]\n"
+ "fmla v22.8h, v6.8h, v3.h[6]\n"
+ "fmla v26.8h, v6.8h, v4.h[6]\n"
+ "ldr q6, [x15, #0x1c0]\n"
+ "fmla v11.8h, v7.8h, v0.h[6]\n"
+ "fmla v15.8h, v7.8h, v1.h[6]\n"
+ "fmla v19.8h, v7.8h, v2.h[6]\n"
+ "fmla v23.8h, v7.8h, v3.h[6]\n"
+ "fmla v27.8h, v7.8h, v4.h[6]\n"
+ "ldr q7, [x15, #0x1d0]\n"
+ "fmla v8.8h, v6.8h, v0.h[7]\n"
+ "fmla v12.8h, v6.8h, v1.h[7]\n"
+ "fmla v16.8h, v6.8h, v2.h[7]\n"
+ "fmla v20.8h, v6.8h, v3.h[7]\n"
+ "fmla v24.8h, v6.8h, v4.h[7]\n"
+ "ldr q6, [x15, #0x1e0]\n"
+ "fmla v9.8h, v7.8h, v0.h[7]\n"
+ "fmla v13.8h, v7.8h, v1.h[7]\n"
+ "fmla v17.8h, v7.8h, v2.h[7]\n"
+ "fmla v21.8h, v7.8h, v3.h[7]\n"
+ "fmla v25.8h, v7.8h, v4.h[7]\n"
+ "ldr q7, [x15, #0x1f0]\n"
+ "fmla v10.8h, v6.8h, v0.h[7]\n"
+ "add x15, x15, #0x200\n"
+ "fmla v14.8h, v6.8h, v1.h[7]\n"
+ "fmla v18.8h, v6.8h, v2.h[7]\n"
+ "fmla v22.8h, v6.8h, v3.h[7]\n"
+ "fmla v26.8h, v6.8h, v4.h[7]\n"
+ "fmla v11.8h, v7.8h, v0.h[7]\n"
+ "fmla v15.8h, v7.8h, v1.h[7]\n"
+ "fmla v19.8h, v7.8h, v2.h[7]\n"
+ "fmla v23.8h, v7.8h, v3.h[7]\n"
+ "fmla v27.8h, v7.8h, v4.h[7]\n"
+ "229:" // Height 5: Multiply loop: Main loop skip
+ "cbz x11, 231f\n"
+ "230:" // Height 5: Multiply loop: Odd block loop
+ "ldr h0, [x10], #0x2\n"
+ "ldr h1, [x28], #0x2\n"
+ "ldr h2, [x26], #0x2\n"
+ "ldr h3, [x24], #0x2\n"
+ "ldr h4, [x22], #0x2\n"
+ "ldr q6, [x15, #0x0]\n"
+ "fmla v8.8h, v6.8h, v0.h[0]\n"
+ "ldr q7, [x15, #0x10]\n"
+ "fmla v12.8h, v6.8h, v1.h[0]\n"
+ "sub x11, x11, #0x1\n"
+ "fmla v16.8h, v6.8h, v2.h[0]\n"
+ "fmla v20.8h, v6.8h, v3.h[0]\n"
+ "fmla v24.8h, v6.8h, v4.h[0]\n"
+ "ldr q6, [x15, #0x20]\n"
+ "fmla v9.8h, v7.8h, v0.h[0]\n"
+ "fmla v13.8h, v7.8h, v1.h[0]\n"
+ "fmla v17.8h, v7.8h, v2.h[0]\n"
+ "fmla v21.8h, v7.8h, v3.h[0]\n"
+ "fmla v25.8h, v7.8h, v4.h[0]\n"
+ "ldr q7, [x15, #0x30]\n"
+ "fmla v10.8h, v6.8h, v0.h[0]\n"
+ "add x15, x15, #0x40\n"
+ "fmla v14.8h, v6.8h, v1.h[0]\n"
+ "fmla v18.8h, v6.8h, v2.h[0]\n"
+ "fmla v22.8h, v6.8h, v3.h[0]\n"
+ "fmla v26.8h, v6.8h, v4.h[0]\n"
+ "fmla v11.8h, v7.8h, v0.h[0]\n"
+ "fmla v15.8h, v7.8h, v1.h[0]\n"
+ "fmla v19.8h, v7.8h, v2.h[0]\n"
+ "fmla v23.8h, v7.8h, v3.h[0]\n"
+ "fmla v27.8h, v7.8h, v4.h[0]\n"
+ "cbnz x11, 230b\n"
+ "231:" // Height 5: Multiply loop: No odd multiplies
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "add x12, x12, #0x1\n"
+ "cmp x12, x19\n"
+ "bne 224b\n"
+ "prfm pstl1keep, [x13, #0x0]\n"
+ "prfm pstl1keep, [x9, #0x0]\n"
+ "prfm pstl1keep, [x27, #0x0]\n"
+ "prfm pstl1keep, [x25, #0x0]\n"
+ "prfm pstl1keep, [x23, #0x0]\n"
+ "tbz %x[flags], #1, 232f\n"
+ "add x19, %x[args_ptr], %[offset_min]\n"
+ "ld1r { v1.8h }, [x19]\n"
+ "add x19, %x[args_ptr], %[offset_max]\n"
+ "ld1r { v0.8h }, [x19]\n"
+ "fmin v8.8h, v8.8h, v0.8h\n"
+ "fmin v9.8h, v9.8h, v0.8h\n"
+ "fmin v10.8h, v10.8h, v0.8h\n"
+ "fmin v11.8h, v11.8h, v0.8h\n"
+ "fmax v8.8h, v8.8h, v1.8h\n"
+ "fmax v9.8h, v9.8h, v1.8h\n"
+ "fmax v10.8h, v10.8h, v1.8h\n"
+ "fmax v11.8h, v11.8h, v1.8h\n"
+ "fmin v12.8h, v12.8h, v0.8h\n"
+ "fmin v13.8h, v13.8h, v0.8h\n"
+ "fmin v14.8h, v14.8h, v0.8h\n"
+ "fmax v12.8h, v12.8h, v1.8h\n"
+ "fmax v13.8h, v13.8h, v1.8h\n"
+ "fmax v14.8h, v14.8h, v1.8h\n"
+ "fmin v15.8h, v15.8h, v0.8h\n"
+ "fmin v16.8h, v16.8h, v0.8h\n"
+ "fmin v17.8h, v17.8h, v0.8h\n"
+ "fmax v15.8h, v15.8h, v1.8h\n"
+ "fmax v16.8h, v16.8h, v1.8h\n"
+ "fmax v17.8h, v17.8h, v1.8h\n"
+ "fmin v18.8h, v18.8h, v0.8h\n"
+ "fmin v19.8h, v19.8h, v0.8h\n"
+ "fmin v20.8h, v20.8h, v0.8h\n"
+ "fmax v18.8h, v18.8h, v1.8h\n"
+ "fmax v19.8h, v19.8h, v1.8h\n"
+ "fmax v20.8h, v20.8h, v1.8h\n"
+ "fmin v21.8h, v21.8h, v0.8h\n"
+ "fmin v22.8h, v22.8h, v0.8h\n"
+ "fmin v23.8h, v23.8h, v0.8h\n"
+ "fmax v21.8h, v21.8h, v1.8h\n"
+ "fmax v22.8h, v22.8h, v1.8h\n"
+ "fmax v23.8h, v23.8h, v1.8h\n"
+ "fmin v24.8h, v24.8h, v0.8h\n"
+ "fmin v25.8h, v25.8h, v0.8h\n"
+ "fmin v26.8h, v26.8h, v0.8h\n"
+ "fmax v24.8h, v24.8h, v1.8h\n"
+ "fmax v25.8h, v25.8h, v1.8h\n"
+ "fmax v26.8h, v26.8h, v1.8h\n"
+ "fmin v27.8h, v27.8h, v0.8h\n"
+ "fmax v27.8h, v27.8h, v1.8h\n"
+ "232:" // Height 5: No activation
+ "cmp x16, #0x20\n"
+ "bge 249f\n"
+ "tbz x16, #4, 240f\n"
+ "st1 { v8.8h }, [x13], #0x10\n"
+ "st1 { v9.8h }, [x13], #0x10\n"
+ "st1 { v12.8h }, [x9], #0x10\n"
+ "st1 { v13.8h }, [x9], #0x10\n"
+ "st1 { v16.8h }, [x27], #0x10\n"
+ "st1 { v17.8h }, [x27], #0x10\n"
+ "st1 { v20.8h }, [x25], #0x10\n"
+ "st1 { v21.8h }, [x25], #0x10\n"
+ "st1 { v24.8h }, [x23], #0x10\n"
+ "st1 { v25.8h }, [x23], #0x10\n"
+ "tbz x16, #3, 236f\n"
+ "st1 { v10.8h }, [x13], #0x10\n"
+ "st1 { v14.8h }, [x9], #0x10\n"
+ "st1 { v18.8h }, [x27], #0x10\n"
+ "st1 { v22.8h }, [x25], #0x10\n"
+ "st1 { v26.8h }, [x23], #0x10\n"
+ "tbz x16, #2, 234f\n"
+ "str d11, [x13], #0x8\n"
+ "str d15, [x9], #0x8\n"
+ "str d19, [x27], #0x8\n"
+ "str d23, [x25], #0x8\n"
+ "str d27, [x23], #0x8\n"
+ "tbz x16, #1, 233f\n"
+ "st1 { v11.s }[2], [x13], #0x4\n"
+ "st1 { v15.s }[2], [x9], #0x4\n"
+ "st1 { v19.s }[2], [x27], #0x4\n"
+ "st1 { v23.s }[2], [x25], #0x4\n"
+ "st1 { v27.s }[2], [x23], #0x4\n"
+ "tbz x16, #0, 248f\n"
+ "st1 { v11.h }[6], [x13]\n"
+ "st1 { v15.h }[6], [x9]\n"
+ "st1 { v19.h }[6], [x27]\n"
+ "st1 { v23.h }[6], [x25]\n"
+ "st1 { v27.h }[6], [x23]\n"
+ "b 248f\n"
+ "233:" // Height 5: Partial direct writeback: partial_1_28
+ "tbz x16, #0, 248f\n"
+ "st1 { v11.h }[4], [x13]\n"
+ "st1 { v15.h }[4], [x9]\n"
+ "st1 { v19.h }[4], [x27]\n"
+ "st1 { v23.h }[4], [x25]\n"
+ "st1 { v27.h }[4], [x23]\n"
+ "b 248f\n"
+ "234:" // Height 5: Partial direct writeback: partial_2_24
+ "tbz x16, #1, 235f\n"
+ "str s11, [x13], #0x4\n"
+ "str s15, [x9], #0x4\n"
+ "str s19, [x27], #0x4\n"
+ "str s23, [x25], #0x4\n"
+ "str s27, [x23], #0x4\n"
+ "tbz x16, #0, 248f\n"
+ "st1 { v11.h }[2], [x13]\n"
+ "st1 { v15.h }[2], [x9]\n"
+ "st1 { v19.h }[2], [x27]\n"
+ "st1 { v23.h }[2], [x25]\n"
+ "st1 { v27.h }[2], [x23]\n"
+ "b 248f\n"
+ "235:" // Height 5: Partial direct writeback: partial_1_24
+ "tbz x16, #0, 248f\n"
+ "str h11, [x13, #0x0]\n"
+ "str h15, [x9, #0x0]\n"
+ "str h19, [x27, #0x0]\n"
+ "str h23, [x25, #0x0]\n"
+ "str h27, [x23, #0x0]\n"
+ "b 248f\n"
+ "236:" // Height 5: Partial direct writeback: partial_4_16
+ "tbz x16, #2, 238f\n"
+ "str d10, [x13], #0x8\n"
+ "str d14, [x9], #0x8\n"
+ "str d18, [x27], #0x8\n"
+ "str d22, [x25], #0x8\n"
+ "str d26, [x23], #0x8\n"
+ "tbz x16, #1, 237f\n"
+ "st1 { v10.s }[2], [x13], #0x4\n"
+ "st1 { v14.s }[2], [x9], #0x4\n"
+ "st1 { v18.s }[2], [x27], #0x4\n"
+ "st1 { v22.s }[2], [x25], #0x4\n"
+ "st1 { v26.s }[2], [x23], #0x4\n"
+ "tbz x16, #0, 248f\n"
+ "st1 { v10.h }[6], [x13]\n"
+ "st1 { v14.h }[6], [x9]\n"
+ "st1 { v18.h }[6], [x27]\n"
+ "st1 { v22.h }[6], [x25]\n"
+ "st1 { v26.h }[6], [x23]\n"
+ "b 248f\n"
+ "237:" // Height 5: Partial direct writeback: partial_1_20
+ "tbz x16, #0, 248f\n"
+ "st1 { v10.h }[4], [x13]\n"
+ "st1 { v14.h }[4], [x9]\n"
+ "st1 { v18.h }[4], [x27]\n"
+ "st1 { v22.h }[4], [x25]\n"
+ "st1 { v26.h }[4], [x23]\n"
+ "b 248f\n"
+ "238:" // Height 5: Partial direct writeback: partial_2_16
+ "tbz x16, #1, 239f\n"
+ "str s10, [x13], #0x4\n"
+ "str s14, [x9], #0x4\n"
+ "str s18, [x27], #0x4\n"
+ "str s22, [x25], #0x4\n"
+ "str s26, [x23], #0x4\n"
+ "tbz x16, #0, 248f\n"
+ "st1 { v10.h }[2], [x13]\n"
+ "st1 { v14.h }[2], [x9]\n"
+ "st1 { v18.h }[2], [x27]\n"
+ "st1 { v22.h }[2], [x25]\n"
+ "st1 { v26.h }[2], [x23]\n"
+ "b 248f\n"
+ "239:" // Height 5: Partial direct writeback: partial_1_16
+ "tbz x16, #0, 248f\n"
+ "str h10, [x13, #0x0]\n"
+ "str h14, [x9, #0x0]\n"
+ "str h18, [x27, #0x0]\n"
+ "str h22, [x25, #0x0]\n"
+ "str h26, [x23, #0x0]\n"
+ "b 248f\n"
+ "240:" // Height 5: Partial direct writeback: partial_8_0
+ "tbz x16, #3, 244f\n"
+ "st1 { v8.8h }, [x13], #0x10\n"
+ "st1 { v12.8h }, [x9], #0x10\n"
+ "st1 { v16.8h }, [x27], #0x10\n"
+ "st1 { v20.8h }, [x25], #0x10\n"
+ "st1 { v24.8h }, [x23], #0x10\n"
+ "tbz x16, #2, 242f\n"
+ "str d9, [x13], #0x8\n"
+ "str d13, [x9], #0x8\n"
+ "str d17, [x27], #0x8\n"
+ "str d21, [x25], #0x8\n"
+ "str d25, [x23], #0x8\n"
+ "tbz x16, #1, 241f\n"
+ "st1 { v9.s }[2], [x13], #0x4\n"
+ "st1 { v13.s }[2], [x9], #0x4\n"
+ "st1 { v17.s }[2], [x27], #0x4\n"
+ "st1 { v21.s }[2], [x25], #0x4\n"
+ "st1 { v25.s }[2], [x23], #0x4\n"
+ "tbz x16, #0, 248f\n"
+ "st1 { v9.h }[6], [x13]\n"
+ "st1 { v13.h }[6], [x9]\n"
+ "st1 { v17.h }[6], [x27]\n"
+ "st1 { v21.h }[6], [x25]\n"
+ "st1 { v25.h }[6], [x23]\n"
+ "b 248f\n"
+ "241:" // Height 5: Partial direct writeback: partial_1_12
+ "tbz x16, #0, 248f\n"
+ "st1 { v9.h }[4], [x13]\n"
+ "st1 { v13.h }[4], [x9]\n"
+ "st1 { v17.h }[4], [x27]\n"
+ "st1 { v21.h }[4], [x25]\n"
+ "st1 { v25.h }[4], [x23]\n"
+ "b 248f\n"
+ "242:" // Height 5: Partial direct writeback: partial_2_8
+ "tbz x16, #1, 243f\n"
+ "str s9, [x13], #0x4\n"
+ "str s13, [x9], #0x4\n"
+ "str s17, [x27], #0x4\n"
+ "str s21, [x25], #0x4\n"
+ "str s25, [x23], #0x4\n"
+ "tbz x16, #0, 248f\n"
+ "st1 { v9.h }[2], [x13]\n"
+ "st1 { v13.h }[2], [x9]\n"
+ "st1 { v17.h }[2], [x27]\n"
+ "st1 { v21.h }[2], [x25]\n"
+ "st1 { v25.h }[2], [x23]\n"
+ "b 248f\n"
+ "243:" // Height 5: Partial direct writeback: partial_1_8
+ "tbz x16, #0, 248f\n"
+ "str h9, [x13, #0x0]\n"
+ "str h13, [x9, #0x0]\n"
+ "str h17, [x27, #0x0]\n"
+ "str h21, [x25, #0x0]\n"
+ "str h25, [x23, #0x0]\n"
+ "b 248f\n"
+ "244:" // Height 5: Partial direct writeback: partial_4_0
+ "tbz x16, #2, 246f\n"
+ "str d8, [x13], #0x8\n"
+ "str d12, [x9], #0x8\n"
+ "str d16, [x27], #0x8\n"
+ "str d20, [x25], #0x8\n"
+ "str d24, [x23], #0x8\n"
+ "tbz x16, #1, 245f\n"
+ "st1 { v8.s }[2], [x13], #0x4\n"
+ "st1 { v12.s }[2], [x9], #0x4\n"
+ "st1 { v16.s }[2], [x27], #0x4\n"
+ "st1 { v20.s }[2], [x25], #0x4\n"
+ "st1 { v24.s }[2], [x23], #0x4\n"
+ "tbz x16, #0, 248f\n"
+ "st1 { v8.h }[6], [x13]\n"
+ "st1 { v12.h }[6], [x9]\n"
+ "st1 { v16.h }[6], [x27]\n"
+ "st1 { v20.h }[6], [x25]\n"
+ "st1 { v24.h }[6], [x23]\n"
+ "b 248f\n"
+ "245:" // Height 5: Partial direct writeback: partial_1_4
+ "tbz x16, #0, 248f\n"
+ "st1 { v8.h }[4], [x13]\n"
+ "st1 { v12.h }[4], [x9]\n"
+ "st1 { v16.h }[4], [x27]\n"
+ "st1 { v20.h }[4], [x25]\n"
+ "st1 { v24.h }[4], [x23]\n"
+ "b 248f\n"
+ "246:" // Height 5: Partial direct writeback: partial_2_0
+ "tbz x16, #1, 247f\n"
+ "str s8, [x13], #0x4\n"
+ "str s12, [x9], #0x4\n"
+ "str s16, [x27], #0x4\n"
+ "str s20, [x25], #0x4\n"
+ "str s24, [x23], #0x4\n"
+ "tbz x16, #0, 248f\n"
+ "st1 { v8.h }[2], [x13]\n"
+ "st1 { v12.h }[2], [x9]\n"
+ "st1 { v16.h }[2], [x27]\n"
+ "st1 { v20.h }[2], [x25]\n"
+ "st1 { v24.h }[2], [x23]\n"
+ "b 248f\n"
+ "247:" // Height 5: Partial direct writeback: partial_1_0
+ "str h8, [x13, #0x0]\n"
+ "str h12, [x9, #0x0]\n"
+ "str h16, [x27, #0x0]\n"
+ "str h20, [x25, #0x0]\n"
+ "str h24, [x23, #0x0]\n"
+ "248:" // Height 5: Partial direct writeback: Done
+ "b 250f\n"
+ "249:" // Height 5: Full writeback
+ "str q8, [x13, #0x0]\n"
+ "str q9, [x13, #0x10]\n"
+ "str q10, [x13, #0x20]\n"
+ "str q11, [x13, #0x30]\n"
+ "str q12, [x9, #0x0]\n"
+ "str q13, [x9, #0x10]\n"
+ "str q14, [x9, #0x20]\n"
+ "str q15, [x9, #0x30]\n"
+ "str q16, [x27, #0x0]\n"
+ "str q17, [x27, #0x10]\n"
+ "str q18, [x27, #0x20]\n"
+ "str q19, [x27, #0x30]\n"
+ "str q20, [x25, #0x0]\n"
+ "str q21, [x25, #0x10]\n"
+ "str q22, [x25, #0x20]\n"
+ "str q23, [x25, #0x30]\n"
+ "str q24, [x23, #0x0]\n"
+ "str q25, [x23, #0x10]\n"
+ "str q26, [x23, #0x20]\n"
+ "str q27, [x23, #0x30]\n"
+ "add x13, x13, #0x40\n"
+ "add x9, x9, #0x40\n"
+ "add x27, x27, #0x40\n"
+ "add x25, x25, #0x40\n"
+ "add x23, x23, #0x40\n"
+ "250:" // Height 5: Writeback done
+ "subs x16, x16, #0x20\n"
+ "bgt 203b\n"
+ "b 302f\n"
+ "251:" // Height 6
+ "ldr x16, [%x[args_ptr], %[offsetof_N]]\n"
+ "mov x14, %x[bias]\n"
+ "ldr x15, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 252f\n"
+ "ldr x13, [%x[output_ptr], #0x0]\n"
+ "add x13, x13, x19, LSL #1\n"
+ "ldr x9, [%x[output_ptr], #0x8]\n"
+ "ldr x27, [%x[output_ptr], #0x10]\n"
+ "add x9, x9, x19, LSL #1\n"
+ "ldr x25, [%x[output_ptr], #0x18]\n"
+ "ldr x23, [%x[output_ptr], #0x20]\n"
+ "add x27, x27, x19, LSL #1\n"
+ "ldr x21, [%x[output_ptr], #0x28]\n"
+ "add %x[output_ptr], %x[output_ptr], #0x30\n"
+ "add x25, x25, x19, LSL #1\n"
+ "add x23, x23, x19, LSL #1\n"
+ "add x21, x21, x19, LSL #1\n"
+ "b 253f\n"
+ "252:" // Height 6: setup direct output
+ "mov x13, %x[output_ptr]\n"
+ "add x9, x13, x19, LSL #1\n"
+ "add x27, x9, x19, LSL #1\n"
+ "add x25, x27, x19, LSL #1\n"
+ "add x23, x25, x19, LSL #1\n"
+ "add x21, x23, x19, LSL #1\n"
+ "add %x[output_ptr], x21, x19, LSL #1\n"
+ "253:" // Height 6: Column loop
+ "cbz x14, 254f\n"
+ "ldr q8, [x14, #0x0]\n"
+ "mov v12.16b, v8.16b\n"
+ "ldr q9, [x14, #0x10]\n"
+ "mov v16.16b, v8.16b\n"
+ "ldr q10, [x14, #0x20]\n"
+ "mov v20.16b, v8.16b\n"
+ "ldr q11, [x14, #0x30]\n"
+ "mov v24.16b, v8.16b\n"
+ "add x14, x14, #0x40\n"
+ "mov v28.16b, v8.16b\n"
+ "mov v13.16b, v9.16b\n"
+ "mov v17.16b, v9.16b\n"
+ "mov v14.16b, v10.16b\n"
+ "mov v15.16b, v11.16b\n"
+ "mov v18.16b, v10.16b\n"
+ "mov v19.16b, v11.16b\n"
+ "mov v21.16b, v9.16b\n"
+ "mov v22.16b, v10.16b\n"
+ "mov v23.16b, v11.16b\n"
+ "mov v25.16b, v9.16b\n"
+ "mov v26.16b, v10.16b\n"
+ "mov v27.16b, v11.16b\n"
+ "mov v29.16b, v9.16b\n"
+ "mov v30.16b, v10.16b\n"
+ "mov v31.16b, v11.16b\n"
+ "b 273f\n"
+ "254:" // Height 6: no bias
+ "tbz %x[flags], #0, 272f\n"
+ "cmp x16, #0x20\n"
+ "bge 271f\n"
+ "tbz x16, #4, 262f\n"
+ "ld1 { v8.8h }, [x13], #0x10\n"
+ "ld1 { v12.8h }, [x9], #0x10\n"
+ "ld1 { v16.8h }, [x27], #0x10\n"
+ "ld1 { v20.8h }, [x25], #0x10\n"
+ "ld1 { v24.8h }, [x23], #0x10\n"
+ "ld1 { v28.8h }, [x21], #0x10\n"
+ "ld1 { v9.8h }, [x13], #0x10\n"
+ "ld1 { v13.8h }, [x9], #0x10\n"
+ "ld1 { v17.8h }, [x27], #0x10\n"
+ "ld1 { v21.8h }, [x25], #0x10\n"
+ "ld1 { v25.8h }, [x23], #0x10\n"
+ "ld1 { v29.8h }, [x21], #0x10\n"
+ "tbz x16, #3, 258f\n"
+ "ld1 { v10.8h }, [x13], #0x10\n"
+ "ld1 { v14.8h }, [x9], #0x10\n"
+ "ld1 { v18.8h }, [x27], #0x10\n"
+ "ld1 { v22.8h }, [x25], #0x10\n"
+ "ld1 { v26.8h }, [x23], #0x10\n"
+ "ld1 { v30.8h }, [x21], #0x10\n"
+ "tbz x16, #2, 256f\n"
+ "ldr d11, [x13], #0x8\n"
+ "ldr d15, [x9], #0x8\n"
+ "ldr d19, [x27], #0x8\n"
+ "ldr d23, [x25], #0x8\n"
+ "ldr d27, [x23], #0x8\n"
+ "ldr d31, [x21], #0x8\n"
+ "tbz x16, #1, 255f\n"
+ "ld1 { v11.s }[2], [x13], #0x4\n"
+ "ld1 { v15.s }[2], [x9], #0x4\n"
+ "ld1 { v19.s }[2], [x27], #0x4\n"
+ "ld1 { v23.s }[2], [x25], #0x4\n"
+ "ld1 { v27.s }[2], [x23], #0x4\n"
+ "ld1 { v31.s }[2], [x21], #0x4\n"
+ "mov x19, #0x3c\n"
+ "tbz x16, #0, 270f\n"
+ "ld1 { v11.h }[6], [x13]\n"
+ "ld1 { v15.h }[6], [x9]\n"
+ "ld1 { v19.h }[6], [x27]\n"
+ "ld1 { v23.h }[6], [x25]\n"
+ "ld1 { v27.h }[6], [x23]\n"
+ "ld1 { v31.h }[6], [x21]\n"
+ "b 270f\n"
+ "255:" // Height 6: Partial accumulate: partial_1_28
+ "mov x19, #0x38\n"
+ "tbz x16, #0, 270f\n"
+ "ld1 { v11.h }[4], [x13]\n"
+ "ld1 { v15.h }[4], [x9]\n"
+ "ld1 { v19.h }[4], [x27]\n"
+ "ld1 { v23.h }[4], [x25]\n"
+ "ld1 { v27.h }[4], [x23]\n"
+ "ld1 { v31.h }[4], [x21]\n"
+ "b 270f\n"
+ "256:" // Height 6: Partial accumulate: partial_2_24
+ "tbz x16, #1, 257f\n"
+ "ldr s11, [x13], #0x4\n"
+ "ldr s15, [x9], #0x4\n"
+ "ldr s19, [x27], #0x4\n"
+ "ldr s23, [x25], #0x4\n"
+ "ldr s27, [x23], #0x4\n"
+ "ldr s31, [x21], #0x4\n"
+ "mov x19, #0x34\n"
+ "tbz x16, #0, 270f\n"
+ "ld1 { v11.h }[2], [x13]\n"
+ "ld1 { v15.h }[2], [x9]\n"
+ "ld1 { v19.h }[2], [x27]\n"
+ "ld1 { v23.h }[2], [x25]\n"
+ "ld1 { v27.h }[2], [x23]\n"
+ "ld1 { v31.h }[2], [x21]\n"
+ "b 270f\n"
+ "257:" // Height 6: Partial accumulate: partial_1_24
+ "mov x19, #0x30\n"
+ "tbz x16, #0, 270f\n"
+ "ldr h11, [x13, #0x0]\n"
+ "ldr h15, [x9, #0x0]\n"
+ "ldr h19, [x27, #0x0]\n"
+ "ldr h23, [x25, #0x0]\n"
+ "ldr h27, [x23, #0x0]\n"
+ "ldr h31, [x21, #0x0]\n"
+ "b 270f\n"
+ "258:" // Height 6: Partial accumulate: partial_4_16
+ "tbz x16, #2, 260f\n"
+ "ldr d10, [x13], #0x8\n"
+ "ldr d14, [x9], #0x8\n"
+ "ldr d18, [x27], #0x8\n"
+ "ldr d22, [x25], #0x8\n"
+ "ldr d26, [x23], #0x8\n"
+ "ldr d30, [x21], #0x8\n"
+ "tbz x16, #1, 259f\n"
+ "ld1 { v10.s }[2], [x13], #0x4\n"
+ "ld1 { v14.s }[2], [x9], #0x4\n"
+ "ld1 { v18.s }[2], [x27], #0x4\n"
+ "ld1 { v22.s }[2], [x25], #0x4\n"
+ "ld1 { v26.s }[2], [x23], #0x4\n"
+ "ld1 { v30.s }[2], [x21], #0x4\n"
+ "mov x19, #0x2c\n"
+ "tbz x16, #0, 270f\n"
+ "ld1 { v10.h }[6], [x13]\n"
+ "ld1 { v14.h }[6], [x9]\n"
+ "ld1 { v18.h }[6], [x27]\n"
+ "ld1 { v22.h }[6], [x25]\n"
+ "ld1 { v26.h }[6], [x23]\n"
+ "ld1 { v30.h }[6], [x21]\n"
+ "b 270f\n"
+ "259:" // Height 6: Partial accumulate: partial_1_20
+ "mov x19, #0x28\n"
+ "tbz x16, #0, 270f\n"
+ "ld1 { v10.h }[4], [x13]\n"
+ "ld1 { v14.h }[4], [x9]\n"
+ "ld1 { v18.h }[4], [x27]\n"
+ "ld1 { v22.h }[4], [x25]\n"
+ "ld1 { v26.h }[4], [x23]\n"
+ "ld1 { v30.h }[4], [x21]\n"
+ "b 270f\n"
+ "260:" // Height 6: Partial accumulate: partial_2_16
+ "tbz x16, #1, 261f\n"
+ "ldr s10, [x13], #0x4\n"
+ "ldr s14, [x9], #0x4\n"
+ "ldr s18, [x27], #0x4\n"
+ "ldr s22, [x25], #0x4\n"
+ "ldr s26, [x23], #0x4\n"
+ "ldr s30, [x21], #0x4\n"
+ "mov x19, #0x24\n"
+ "tbz x16, #0, 270f\n"
+ "ld1 { v10.h }[2], [x13]\n"
+ "ld1 { v14.h }[2], [x9]\n"
+ "ld1 { v18.h }[2], [x27]\n"
+ "ld1 { v22.h }[2], [x25]\n"
+ "ld1 { v26.h }[2], [x23]\n"
+ "ld1 { v30.h }[2], [x21]\n"
+ "b 270f\n"
+ "261:" // Height 6: Partial accumulate: partial_1_16
+ "mov x19, #0x20\n"
+ "tbz x16, #0, 270f\n"
+ "ldr h10, [x13, #0x0]\n"
+ "ldr h14, [x9, #0x0]\n"
+ "ldr h18, [x27, #0x0]\n"
+ "ldr h22, [x25, #0x0]\n"
+ "ldr h26, [x23, #0x0]\n"
+ "ldr h30, [x21, #0x0]\n"
+ "b 270f\n"
+ "262:" // Height 6: Partial accumulate: partial_8_0
+ "tbz x16, #3, 266f\n"
+ "ld1 { v8.8h }, [x13], #0x10\n"
+ "ld1 { v12.8h }, [x9], #0x10\n"
+ "ld1 { v16.8h }, [x27], #0x10\n"
+ "ld1 { v20.8h }, [x25], #0x10\n"
+ "ld1 { v24.8h }, [x23], #0x10\n"
+ "ld1 { v28.8h }, [x21], #0x10\n"
+ "tbz x16, #2, 264f\n"
+ "ldr d9, [x13], #0x8\n"
+ "ldr d13, [x9], #0x8\n"
+ "ldr d17, [x27], #0x8\n"
+ "ldr d21, [x25], #0x8\n"
+ "ldr d25, [x23], #0x8\n"
+ "ldr d29, [x21], #0x8\n"
+ "tbz x16, #1, 263f\n"
+ "ld1 { v9.s }[2], [x13], #0x4\n"
+ "ld1 { v13.s }[2], [x9], #0x4\n"
+ "ld1 { v17.s }[2], [x27], #0x4\n"
+ "ld1 { v21.s }[2], [x25], #0x4\n"
+ "ld1 { v25.s }[2], [x23], #0x4\n"
+ "ld1 { v29.s }[2], [x21], #0x4\n"
+ "mov x19, #0x1c\n"
+ "tbz x16, #0, 270f\n"
+ "ld1 { v9.h }[6], [x13]\n"
+ "ld1 { v13.h }[6], [x9]\n"
+ "ld1 { v17.h }[6], [x27]\n"
+ "ld1 { v21.h }[6], [x25]\n"
+ "ld1 { v25.h }[6], [x23]\n"
+ "ld1 { v29.h }[6], [x21]\n"
+ "b 270f\n"
+ "263:" // Height 6: Partial accumulate: partial_1_12
+ "mov x19, #0x18\n"
+ "tbz x16, #0, 270f\n"
+ "ld1 { v9.h }[4], [x13]\n"
+ "ld1 { v13.h }[4], [x9]\n"
+ "ld1 { v17.h }[4], [x27]\n"
+ "ld1 { v21.h }[4], [x25]\n"
+ "ld1 { v25.h }[4], [x23]\n"
+ "ld1 { v29.h }[4], [x21]\n"
+ "b 270f\n"
+ "264:" // Height 6: Partial accumulate: partial_2_8
+ "tbz x16, #1, 265f\n"
+ "ldr s9, [x13], #0x4\n"
+ "ldr s13, [x9], #0x4\n"
+ "ldr s17, [x27], #0x4\n"
+ "ldr s21, [x25], #0x4\n"
+ "ldr s25, [x23], #0x4\n"
+ "ldr s29, [x21], #0x4\n"
+ "mov x19, #0x14\n"
+ "tbz x16, #0, 270f\n"
+ "ld1 { v9.h }[2], [x13]\n"
+ "ld1 { v13.h }[2], [x9]\n"
+ "ld1 { v17.h }[2], [x27]\n"
+ "ld1 { v21.h }[2], [x25]\n"
+ "ld1 { v25.h }[2], [x23]\n"
+ "ld1 { v29.h }[2], [x21]\n"
+ "b 270f\n"
+ "265:" // Height 6: Partial accumulate: partial_1_8
+ "mov x19, #0x10\n"
+ "tbz x16, #0, 270f\n"
+ "ldr h9, [x13, #0x0]\n"
+ "ldr h13, [x9, #0x0]\n"
+ "ldr h17, [x27, #0x0]\n"
+ "ldr h21, [x25, #0x0]\n"
+ "ldr h25, [x23, #0x0]\n"
+ "ldr h29, [x21, #0x0]\n"
+ "b 270f\n"
+ "266:" // Height 6: Partial accumulate: partial_4_0
+ "tbz x16, #2, 268f\n"
+ "ldr d8, [x13], #0x8\n"
+ "ldr d12, [x9], #0x8\n"
+ "ldr d16, [x27], #0x8\n"
+ "ldr d20, [x25], #0x8\n"
+ "ldr d24, [x23], #0x8\n"
+ "ldr d28, [x21], #0x8\n"
+ "tbz x16, #1, 267f\n"
+ "ld1 { v8.s }[2], [x13], #0x4\n"
+ "ld1 { v12.s }[2], [x9], #0x4\n"
+ "ld1 { v16.s }[2], [x27], #0x4\n"
+ "ld1 { v20.s }[2], [x25], #0x4\n"
+ "ld1 { v24.s }[2], [x23], #0x4\n"
+ "ld1 { v28.s }[2], [x21], #0x4\n"
+ "mov x19, #0xc\n"
+ "tbz x16, #0, 270f\n"
+ "ld1 { v8.h }[6], [x13]\n"
+ "ld1 { v12.h }[6], [x9]\n"
+ "ld1 { v16.h }[6], [x27]\n"
+ "ld1 { v20.h }[6], [x25]\n"
+ "ld1 { v24.h }[6], [x23]\n"
+ "ld1 { v28.h }[6], [x21]\n"
+ "b 270f\n"
+ "267:" // Height 6: Partial accumulate: partial_1_4
+ "mov x19, #0x8\n"
+ "tbz x16, #0, 270f\n"
+ "ld1 { v8.h }[4], [x13]\n"
+ "ld1 { v12.h }[4], [x9]\n"
+ "ld1 { v16.h }[4], [x27]\n"
+ "ld1 { v20.h }[4], [x25]\n"
+ "ld1 { v24.h }[4], [x23]\n"
+ "ld1 { v28.h }[4], [x21]\n"
+ "b 270f\n"
+ "268:" // Height 6: Partial accumulate: partial_2_0
+ "tbz x16, #1, 269f\n"
+ "ldr s8, [x13], #0x4\n"
+ "ldr s12, [x9], #0x4\n"
+ "ldr s16, [x27], #0x4\n"
+ "ldr s20, [x25], #0x4\n"
+ "ldr s24, [x23], #0x4\n"
+ "ldr s28, [x21], #0x4\n"
+ "mov x19, #0x4\n"
+ "tbz x16, #0, 270f\n"
+ "ld1 { v8.h }[2], [x13]\n"
+ "ld1 { v12.h }[2], [x9]\n"
+ "ld1 { v16.h }[2], [x27]\n"
+ "ld1 { v20.h }[2], [x25]\n"
+ "ld1 { v24.h }[2], [x23]\n"
+ "ld1 { v28.h }[2], [x21]\n"
+ "b 270f\n"
+ "269:" // Height 6: Partial accumulate: partial_1_0
+ "mov x19, #0x0\n"
+ "ldr h8, [x13, #0x0]\n"
+ "ldr h12, [x9, #0x0]\n"
+ "ldr h16, [x27, #0x0]\n"
+ "ldr h20, [x25, #0x0]\n"
+ "ldr h24, [x23, #0x0]\n"
+ "ldr h28, [x21, #0x0]\n"
+ "270:" // Height 6: Partial accumulate: Done
+ "sub x13, x13, x19\n"
+ "sub x9, x9, x19\n"
+ "sub x27, x27, x19\n"
+ "sub x25, x25, x19\n"
+ "sub x23, x23, x19\n"
+ "sub x21, x21, x19\n"
+ "b 273f\n"
+ "271:" // Height 6: full accumulate
+ "ldr q8, [x13, #0x0]\n"
+ "ldr q9, [x13, #0x10]\n"
+ "ldr q10, [x13, #0x20]\n"
+ "ldr q11, [x13, #0x30]\n"
+ "ldr q12, [x9, #0x0]\n"
+ "ldr q13, [x9, #0x10]\n"
+ "ldr q14, [x9, #0x20]\n"
+ "ldr q15, [x9, #0x30]\n"
+ "ldr q16, [x27, #0x0]\n"
+ "ldr q17, [x27, #0x10]\n"
+ "ldr q18, [x27, #0x20]\n"
+ "ldr q19, [x27, #0x30]\n"
+ "ldr q20, [x25, #0x0]\n"
+ "ldr q21, [x25, #0x10]\n"
+ "ldr q22, [x25, #0x20]\n"
+ "ldr q23, [x25, #0x30]\n"
+ "ldr q24, [x23, #0x0]\n"
+ "ldr q25, [x23, #0x10]\n"
+ "ldr q26, [x23, #0x20]\n"
+ "ldr q27, [x23, #0x30]\n"
+ "ldr q28, [x21, #0x0]\n"
+ "ldr q29, [x21, #0x10]\n"
+ "ldr q30, [x21, #0x20]\n"
+ "ldr q31, [x21, #0x30]\n"
+ "b 273f\n"
+ "272:" // Height 6: no accumulate
+ "movi v8.16b, #0x0\n"
+ "movi v9.16b, #0x0\n"
+ "movi v10.16b, #0x0\n"
+ "movi v11.16b, #0x0\n"
+ "movi v12.16b, #0x0\n"
+ "movi v13.16b, #0x0\n"
+ "movi v14.16b, #0x0\n"
+ "movi v15.16b, #0x0\n"
+ "movi v16.16b, #0x0\n"
+ "movi v17.16b, #0x0\n"
+ "movi v18.16b, #0x0\n"
+ "movi v19.16b, #0x0\n"
+ "movi v20.16b, #0x0\n"
+ "movi v21.16b, #0x0\n"
+ "movi v22.16b, #0x0\n"
+ "movi v23.16b, #0x0\n"
+ "movi v24.16b, #0x0\n"
+ "movi v25.16b, #0x0\n"
+ "movi v26.16b, #0x0\n"
+ "movi v27.16b, #0x0\n"
+ "movi v28.16b, #0x0\n"
+ "movi v29.16b, #0x0\n"
+ "movi v30.16b, #0x0\n"
+ "movi v31.16b, #0x0\n"
+ "273:" // Height 6: setup done
+ "mov x12, #0x0\n"
+ "274:" // Height 6: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w11, [x20, x12, LSL #0x2]\n"
+ "tbz %x[flags], #3, 275f\n"
+ "ldr x20, [%x[input_ptr], x12, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x10, [x20, #0x0]\n"
+ "ldr x28, [x20, #0x8]\n"
+ "ldr x26, [x20, #0x10]\n"
+ "ldr x24, [x20, #0x18]\n"
+ "ldr x22, [x20, #0x20]\n"
+ "ldr x20, [x20, #0x28]\n"
+ "cbnz x12, 276f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x10, x10, x19, LSL #1\n"
+ "add x28, x28, x19, LSL #1\n"
+ "add x26, x26, x19, LSL #1\n"
+ "add x24, x24, x19, LSL #1\n"
+ "add x22, x22, x19, LSL #1\n"
+ "add x20, x20, x19, LSL #1\n"
+ "b 276f\n"
+ "275:" // Height 6: setup direct input
+ "mov x10, %x[input_ptr]\n"
+ "add x28, x10, x19, LSL #1\n"
+ "add x26, x28, x19, LSL #1\n"
+ "add x24, x26, x19, LSL #1\n"
+ "add x22, x24, x19, LSL #1\n"
+ "add x20, x22, x19, LSL #1\n"
+ "276:" // Height 6: input setup done
+ "cmp x11, #0x8\n"
+ "blt 279f\n"
+ "cmp x11, #0x10\n"
+ "blt 278f\n"
+ "277:" // Height 6: Multiply loop: Main loop head
+ "ldr q0, [x10, #0x0]\n"
+ "ldr q1, [x28, #0x0]\n"
+ "ldr q2, [x26, #0x0]\n"
+ "ldr q3, [x24, #0x0]\n"
+ "ldr q4, [x22, #0x0]\n"
+ "ldr q5, [x20, #0x0]\n"
+ "ldr q6, [x15, #0x0]\n"
+ "fmla v8.8h, v6.8h, v0.h[0]\n"
+ "ldr q7, [x15, #0x10]\n"
+ "fmla v12.8h, v6.8h, v1.h[0]\n"
+ "add x10, x10, #0x10\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "fmla v16.8h, v6.8h, v2.h[0]\n"
+ "add x28, x28, #0x10\n"
+ "fmla v20.8h, v6.8h, v3.h[0]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "add x26, x26, #0x10\n"
+ "fmla v24.8h, v6.8h, v4.h[0]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "add x24, x24, #0x10\n"
+ "fmla v28.8h, v6.8h, v5.h[0]\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "ldr q6, [x15, #0x20]\n"
+ "fmla v9.8h, v7.8h, v0.h[0]\n"
+ "add x22, x22, #0x10\n"
+ "prfm pldl1keep, [x22, #0x80]\n"
+ "fmla v13.8h, v7.8h, v1.h[0]\n"
+ "add x20, x20, #0x10\n"
+ "fmla v17.8h, v7.8h, v2.h[0]\n"
+ "prfm pldl1keep, [x20, #0x80]\n"
+ "sub x11, x11, #0x8\n"
+ "fmla v21.8h, v7.8h, v3.h[0]\n"
+ "cmp x11, #0x10\n"
+ "fmla v25.8h, v7.8h, v4.h[0]\n"
+ "fmla v29.8h, v7.8h, v5.h[0]\n"
+ "ldr q7, [x15, #0x30]\n"
+ "fmla v10.8h, v6.8h, v0.h[0]\n"
+ "fmla v14.8h, v6.8h, v1.h[0]\n"
+ "fmla v18.8h, v6.8h, v2.h[0]\n"
+ "fmla v22.8h, v6.8h, v3.h[0]\n"
+ "fmla v26.8h, v6.8h, v4.h[0]\n"
+ "fmla v30.8h, v6.8h, v5.h[0]\n"
+ "ldr q6, [x15, #0x40]\n"
+ "fmla v11.8h, v7.8h, v0.h[0]\n"
+ "fmla v15.8h, v7.8h, v1.h[0]\n"
+ "fmla v19.8h, v7.8h, v2.h[0]\n"
+ "fmla v23.8h, v7.8h, v3.h[0]\n"
+ "fmla v27.8h, v7.8h, v4.h[0]\n"
+ "fmla v31.8h, v7.8h, v5.h[0]\n"
+ "ldr q7, [x15, #0x50]\n"
+ "fmla v8.8h, v6.8h, v0.h[1]\n"
+ "fmla v12.8h, v6.8h, v1.h[1]\n"
+ "fmla v16.8h, v6.8h, v2.h[1]\n"
+ "fmla v20.8h, v6.8h, v3.h[1]\n"
+ "fmla v24.8h, v6.8h, v4.h[1]\n"
+ "fmla v28.8h, v6.8h, v5.h[1]\n"
+ "ldr q6, [x15, #0x60]\n"
+ "fmla v9.8h, v7.8h, v0.h[1]\n"
+ "fmla v13.8h, v7.8h, v1.h[1]\n"
+ "fmla v17.8h, v7.8h, v2.h[1]\n"
+ "fmla v21.8h, v7.8h, v3.h[1]\n"
+ "fmla v25.8h, v7.8h, v4.h[1]\n"
+ "fmla v29.8h, v7.8h, v5.h[1]\n"
+ "ldr q7, [x15, #0x70]\n"
+ "fmla v10.8h, v6.8h, v0.h[1]\n"
+ "fmla v14.8h, v6.8h, v1.h[1]\n"
+ "fmla v18.8h, v6.8h, v2.h[1]\n"
+ "fmla v22.8h, v6.8h, v3.h[1]\n"
+ "fmla v26.8h, v6.8h, v4.h[1]\n"
+ "fmla v30.8h, v6.8h, v5.h[1]\n"
+ "ldr q6, [x15, #0x80]\n"
+ "fmla v11.8h, v7.8h, v0.h[1]\n"
+ "fmla v15.8h, v7.8h, v1.h[1]\n"
+ "fmla v19.8h, v7.8h, v2.h[1]\n"
+ "fmla v23.8h, v7.8h, v3.h[1]\n"
+ "fmla v27.8h, v7.8h, v4.h[1]\n"
+ "fmla v31.8h, v7.8h, v5.h[1]\n"
+ "ldr q7, [x15, #0x90]\n"
+ "fmla v8.8h, v6.8h, v0.h[2]\n"
+ "fmla v12.8h, v6.8h, v1.h[2]\n"
+ "fmla v16.8h, v6.8h, v2.h[2]\n"
+ "fmla v20.8h, v6.8h, v3.h[2]\n"
+ "fmla v24.8h, v6.8h, v4.h[2]\n"
+ "fmla v28.8h, v6.8h, v5.h[2]\n"
+ "ldr q6, [x15, #0xa0]\n"
+ "fmla v9.8h, v7.8h, v0.h[2]\n"
+ "fmla v13.8h, v7.8h, v1.h[2]\n"
+ "fmla v17.8h, v7.8h, v2.h[2]\n"
+ "fmla v21.8h, v7.8h, v3.h[2]\n"
+ "fmla v25.8h, v7.8h, v4.h[2]\n"
+ "fmla v29.8h, v7.8h, v5.h[2]\n"
+ "ldr q7, [x15, #0xb0]\n"
+ "fmla v10.8h, v6.8h, v0.h[2]\n"
+ "fmla v14.8h, v6.8h, v1.h[2]\n"
+ "fmla v18.8h, v6.8h, v2.h[2]\n"
+ "fmla v22.8h, v6.8h, v3.h[2]\n"
+ "fmla v26.8h, v6.8h, v4.h[2]\n"
+ "fmla v30.8h, v6.8h, v5.h[2]\n"
+ "ldr q6, [x15, #0xc0]\n"
+ "fmla v11.8h, v7.8h, v0.h[2]\n"
+ "fmla v15.8h, v7.8h, v1.h[2]\n"
+ "fmla v19.8h, v7.8h, v2.h[2]\n"
+ "fmla v23.8h, v7.8h, v3.h[2]\n"
+ "fmla v27.8h, v7.8h, v4.h[2]\n"
+ "fmla v31.8h, v7.8h, v5.h[2]\n"
+ "ldr q7, [x15, #0xd0]\n"
+ "fmla v8.8h, v6.8h, v0.h[3]\n"
+ "fmla v12.8h, v6.8h, v1.h[3]\n"
+ "fmla v16.8h, v6.8h, v2.h[3]\n"
+ "fmla v20.8h, v6.8h, v3.h[3]\n"
+ "fmla v24.8h, v6.8h, v4.h[3]\n"
+ "fmla v28.8h, v6.8h, v5.h[3]\n"
+ "ldr q6, [x15, #0xe0]\n"
+ "fmla v9.8h, v7.8h, v0.h[3]\n"
+ "fmla v13.8h, v7.8h, v1.h[3]\n"
+ "fmla v17.8h, v7.8h, v2.h[3]\n"
+ "fmla v21.8h, v7.8h, v3.h[3]\n"
+ "fmla v25.8h, v7.8h, v4.h[3]\n"
+ "fmla v29.8h, v7.8h, v5.h[3]\n"
+ "ldr q7, [x15, #0xf0]\n"
+ "fmla v10.8h, v6.8h, v0.h[3]\n"
+ "fmla v14.8h, v6.8h, v1.h[3]\n"
+ "fmla v18.8h, v6.8h, v2.h[3]\n"
+ "fmla v22.8h, v6.8h, v3.h[3]\n"
+ "fmla v26.8h, v6.8h, v4.h[3]\n"
+ "fmla v30.8h, v6.8h, v5.h[3]\n"
+ "ldr q6, [x15, #0x100]\n"
+ "fmla v11.8h, v7.8h, v0.h[3]\n"
+ "fmla v15.8h, v7.8h, v1.h[3]\n"
+ "fmla v19.8h, v7.8h, v2.h[3]\n"
+ "fmla v23.8h, v7.8h, v3.h[3]\n"
+ "fmla v27.8h, v7.8h, v4.h[3]\n"
+ "fmla v31.8h, v7.8h, v5.h[3]\n"
+ "ldr q7, [x15, #0x110]\n"
+ "fmla v8.8h, v6.8h, v0.h[4]\n"
+ "fmla v12.8h, v6.8h, v1.h[4]\n"
+ "fmla v16.8h, v6.8h, v2.h[4]\n"
+ "fmla v20.8h, v6.8h, v3.h[4]\n"
+ "fmla v24.8h, v6.8h, v4.h[4]\n"
+ "fmla v28.8h, v6.8h, v5.h[4]\n"
+ "ldr q6, [x15, #0x120]\n"
+ "fmla v9.8h, v7.8h, v0.h[4]\n"
+ "fmla v13.8h, v7.8h, v1.h[4]\n"
+ "fmla v17.8h, v7.8h, v2.h[4]\n"
+ "fmla v21.8h, v7.8h, v3.h[4]\n"
+ "fmla v25.8h, v7.8h, v4.h[4]\n"
+ "fmla v29.8h, v7.8h, v5.h[4]\n"
+ "ldr q7, [x15, #0x130]\n"
+ "fmla v10.8h, v6.8h, v0.h[4]\n"
+ "fmla v14.8h, v6.8h, v1.h[4]\n"
+ "fmla v18.8h, v6.8h, v2.h[4]\n"
+ "fmla v22.8h, v6.8h, v3.h[4]\n"
+ "fmla v26.8h, v6.8h, v4.h[4]\n"
+ "fmla v30.8h, v6.8h, v5.h[4]\n"
+ "ldr q6, [x15, #0x140]\n"
+ "fmla v11.8h, v7.8h, v0.h[4]\n"
+ "fmla v15.8h, v7.8h, v1.h[4]\n"
+ "fmla v19.8h, v7.8h, v2.h[4]\n"
+ "fmla v23.8h, v7.8h, v3.h[4]\n"
+ "fmla v27.8h, v7.8h, v4.h[4]\n"
+ "fmla v31.8h, v7.8h, v5.h[4]\n"
+ "ldr q7, [x15, #0x150]\n"
+ "fmla v8.8h, v6.8h, v0.h[5]\n"
+ "fmla v12.8h, v6.8h, v1.h[5]\n"
+ "fmla v16.8h, v6.8h, v2.h[5]\n"
+ "fmla v20.8h, v6.8h, v3.h[5]\n"
+ "fmla v24.8h, v6.8h, v4.h[5]\n"
+ "fmla v28.8h, v6.8h, v5.h[5]\n"
+ "ldr q6, [x15, #0x160]\n"
+ "fmla v9.8h, v7.8h, v0.h[5]\n"
+ "fmla v13.8h, v7.8h, v1.h[5]\n"
+ "fmla v17.8h, v7.8h, v2.h[5]\n"
+ "fmla v21.8h, v7.8h, v3.h[5]\n"
+ "fmla v25.8h, v7.8h, v4.h[5]\n"
+ "fmla v29.8h, v7.8h, v5.h[5]\n"
+ "ldr q7, [x15, #0x170]\n"
+ "fmla v10.8h, v6.8h, v0.h[5]\n"
+ "fmla v14.8h, v6.8h, v1.h[5]\n"
+ "fmla v18.8h, v6.8h, v2.h[5]\n"
+ "fmla v22.8h, v6.8h, v3.h[5]\n"
+ "fmla v26.8h, v6.8h, v4.h[5]\n"
+ "fmla v30.8h, v6.8h, v5.h[5]\n"
+ "ldr q6, [x15, #0x180]\n"
+ "fmla v11.8h, v7.8h, v0.h[5]\n"
+ "fmla v15.8h, v7.8h, v1.h[5]\n"
+ "fmla v19.8h, v7.8h, v2.h[5]\n"
+ "fmla v23.8h, v7.8h, v3.h[5]\n"
+ "fmla v27.8h, v7.8h, v4.h[5]\n"
+ "fmla v31.8h, v7.8h, v5.h[5]\n"
+ "ldr q7, [x15, #0x190]\n"
+ "fmla v8.8h, v6.8h, v0.h[6]\n"
+ "fmla v12.8h, v6.8h, v1.h[6]\n"
+ "fmla v16.8h, v6.8h, v2.h[6]\n"
+ "fmla v20.8h, v6.8h, v3.h[6]\n"
+ "fmla v24.8h, v6.8h, v4.h[6]\n"
+ "fmla v28.8h, v6.8h, v5.h[6]\n"
+ "ldr q6, [x15, #0x1a0]\n"
+ "fmla v9.8h, v7.8h, v0.h[6]\n"
+ "fmla v13.8h, v7.8h, v1.h[6]\n"
+ "fmla v17.8h, v7.8h, v2.h[6]\n"
+ "fmla v21.8h, v7.8h, v3.h[6]\n"
+ "fmla v25.8h, v7.8h, v4.h[6]\n"
+ "fmla v29.8h, v7.8h, v5.h[6]\n"
+ "ldr q7, [x15, #0x1b0]\n"
+ "fmla v10.8h, v6.8h, v0.h[6]\n"
+ "fmla v14.8h, v6.8h, v1.h[6]\n"
+ "fmla v18.8h, v6.8h, v2.h[6]\n"
+ "fmla v22.8h, v6.8h, v3.h[6]\n"
+ "fmla v26.8h, v6.8h, v4.h[6]\n"
+ "fmla v30.8h, v6.8h, v5.h[6]\n"
+ "ldr q6, [x15, #0x1c0]\n"
+ "fmla v11.8h, v7.8h, v0.h[6]\n"
+ "fmla v15.8h, v7.8h, v1.h[6]\n"
+ "fmla v19.8h, v7.8h, v2.h[6]\n"
+ "fmla v23.8h, v7.8h, v3.h[6]\n"
+ "fmla v27.8h, v7.8h, v4.h[6]\n"
+ "fmla v31.8h, v7.8h, v5.h[6]\n"
+ "ldr q7, [x15, #0x1d0]\n"
+ "fmla v8.8h, v6.8h, v0.h[7]\n"
+ "fmla v12.8h, v6.8h, v1.h[7]\n"
+ "fmla v16.8h, v6.8h, v2.h[7]\n"
+ "fmla v20.8h, v6.8h, v3.h[7]\n"
+ "fmla v24.8h, v6.8h, v4.h[7]\n"
+ "fmla v28.8h, v6.8h, v5.h[7]\n"
+ "ldr q6, [x15, #0x1e0]\n"
+ "fmla v9.8h, v7.8h, v0.h[7]\n"
+ "fmla v13.8h, v7.8h, v1.h[7]\n"
+ "fmla v17.8h, v7.8h, v2.h[7]\n"
+ "fmla v21.8h, v7.8h, v3.h[7]\n"
+ "fmla v25.8h, v7.8h, v4.h[7]\n"
+ "fmla v29.8h, v7.8h, v5.h[7]\n"
+ "ldr q7, [x15, #0x1f0]\n"
+ "fmla v10.8h, v6.8h, v0.h[7]\n"
+ "add x15, x15, #0x200\n"
+ "fmla v14.8h, v6.8h, v1.h[7]\n"
+ "fmla v18.8h, v6.8h, v2.h[7]\n"
+ "fmla v22.8h, v6.8h, v3.h[7]\n"
+ "fmla v26.8h, v6.8h, v4.h[7]\n"
+ "fmla v30.8h, v6.8h, v5.h[7]\n"
+ "fmla v11.8h, v7.8h, v0.h[7]\n"
+ "fmla v15.8h, v7.8h, v1.h[7]\n"
+ "fmla v19.8h, v7.8h, v2.h[7]\n"
+ "fmla v23.8h, v7.8h, v3.h[7]\n"
+ "fmla v27.8h, v7.8h, v4.h[7]\n"
+ "fmla v31.8h, v7.8h, v5.h[7]\n"
+ "bge 277b\n"
+ "278:" // Height 6: Multiply loop: Single iteration only
+ "sub x11, x11, #0x8\n"
+ "ldr q0, [x10, #0x0]\n"
+ "ldr q1, [x28, #0x0]\n"
+ "ldr q2, [x26, #0x0]\n"
+ "ldr q3, [x24, #0x0]\n"
+ "ldr q4, [x22, #0x0]\n"
+ "ldr q5, [x20, #0x0]\n"
+ "ldr q6, [x15, #0x0]\n"
+ "fmla v8.8h, v6.8h, v0.h[0]\n"
+ "ldr q7, [x15, #0x10]\n"
+ "fmla v12.8h, v6.8h, v1.h[0]\n"
+ "add x10, x10, #0x10\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "fmla v16.8h, v6.8h, v2.h[0]\n"
+ "add x28, x28, #0x10\n"
+ "fmla v20.8h, v6.8h, v3.h[0]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "add x26, x26, #0x10\n"
+ "fmla v24.8h, v6.8h, v4.h[0]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "add x24, x24, #0x10\n"
+ "fmla v28.8h, v6.8h, v5.h[0]\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "ldr q6, [x15, #0x20]\n"
+ "fmla v9.8h, v7.8h, v0.h[0]\n"
+ "add x22, x22, #0x10\n"
+ "prfm pldl1keep, [x22, #0x80]\n"
+ "fmla v13.8h, v7.8h, v1.h[0]\n"
+ "add x20, x20, #0x10\n"
+ "fmla v17.8h, v7.8h, v2.h[0]\n"
+ "prfm pldl1keep, [x20, #0x80]\n"
+ "fmla v21.8h, v7.8h, v3.h[0]\n"
+ "fmla v25.8h, v7.8h, v4.h[0]\n"
+ "fmla v29.8h, v7.8h, v5.h[0]\n"
+ "ldr q7, [x15, #0x30]\n"
+ "fmla v10.8h, v6.8h, v0.h[0]\n"
+ "fmla v14.8h, v6.8h, v1.h[0]\n"
+ "fmla v18.8h, v6.8h, v2.h[0]\n"
+ "fmla v22.8h, v6.8h, v3.h[0]\n"
+ "fmla v26.8h, v6.8h, v4.h[0]\n"
+ "fmla v30.8h, v6.8h, v5.h[0]\n"
+ "ldr q6, [x15, #0x40]\n"
+ "fmla v11.8h, v7.8h, v0.h[0]\n"
+ "fmla v15.8h, v7.8h, v1.h[0]\n"
+ "fmla v19.8h, v7.8h, v2.h[0]\n"
+ "fmla v23.8h, v7.8h, v3.h[0]\n"
+ "fmla v27.8h, v7.8h, v4.h[0]\n"
+ "fmla v31.8h, v7.8h, v5.h[0]\n"
+ "ldr q7, [x15, #0x50]\n"
+ "fmla v8.8h, v6.8h, v0.h[1]\n"
+ "fmla v12.8h, v6.8h, v1.h[1]\n"
+ "fmla v16.8h, v6.8h, v2.h[1]\n"
+ "fmla v20.8h, v6.8h, v3.h[1]\n"
+ "fmla v24.8h, v6.8h, v4.h[1]\n"
+ "fmla v28.8h, v6.8h, v5.h[1]\n"
+ "ldr q6, [x15, #0x60]\n"
+ "fmla v9.8h, v7.8h, v0.h[1]\n"
+ "fmla v13.8h, v7.8h, v1.h[1]\n"
+ "fmla v17.8h, v7.8h, v2.h[1]\n"
+ "fmla v21.8h, v7.8h, v3.h[1]\n"
+ "fmla v25.8h, v7.8h, v4.h[1]\n"
+ "fmla v29.8h, v7.8h, v5.h[1]\n"
+ "ldr q7, [x15, #0x70]\n"
+ "fmla v10.8h, v6.8h, v0.h[1]\n"
+ "fmla v14.8h, v6.8h, v1.h[1]\n"
+ "fmla v18.8h, v6.8h, v2.h[1]\n"
+ "fmla v22.8h, v6.8h, v3.h[1]\n"
+ "fmla v26.8h, v6.8h, v4.h[1]\n"
+ "fmla v30.8h, v6.8h, v5.h[1]\n"
+ "ldr q6, [x15, #0x80]\n"
+ "fmla v11.8h, v7.8h, v0.h[1]\n"
+ "fmla v15.8h, v7.8h, v1.h[1]\n"
+ "fmla v19.8h, v7.8h, v2.h[1]\n"
+ "fmla v23.8h, v7.8h, v3.h[1]\n"
+ "fmla v27.8h, v7.8h, v4.h[1]\n"
+ "fmla v31.8h, v7.8h, v5.h[1]\n"
+ "ldr q7, [x15, #0x90]\n"
+ "fmla v8.8h, v6.8h, v0.h[2]\n"
+ "fmla v12.8h, v6.8h, v1.h[2]\n"
+ "fmla v16.8h, v6.8h, v2.h[2]\n"
+ "fmla v20.8h, v6.8h, v3.h[2]\n"
+ "fmla v24.8h, v6.8h, v4.h[2]\n"
+ "fmla v28.8h, v6.8h, v5.h[2]\n"
+ "ldr q6, [x15, #0xa0]\n"
+ "fmla v9.8h, v7.8h, v0.h[2]\n"
+ "fmla v13.8h, v7.8h, v1.h[2]\n"
+ "fmla v17.8h, v7.8h, v2.h[2]\n"
+ "fmla v21.8h, v7.8h, v3.h[2]\n"
+ "fmla v25.8h, v7.8h, v4.h[2]\n"
+ "fmla v29.8h, v7.8h, v5.h[2]\n"
+ "ldr q7, [x15, #0xb0]\n"
+ "fmla v10.8h, v6.8h, v0.h[2]\n"
+ "fmla v14.8h, v6.8h, v1.h[2]\n"
+ "fmla v18.8h, v6.8h, v2.h[2]\n"
+ "fmla v22.8h, v6.8h, v3.h[2]\n"
+ "fmla v26.8h, v6.8h, v4.h[2]\n"
+ "fmla v30.8h, v6.8h, v5.h[2]\n"
+ "ldr q6, [x15, #0xc0]\n"
+ "fmla v11.8h, v7.8h, v0.h[2]\n"
+ "fmla v15.8h, v7.8h, v1.h[2]\n"
+ "fmla v19.8h, v7.8h, v2.h[2]\n"
+ "fmla v23.8h, v7.8h, v3.h[2]\n"
+ "fmla v27.8h, v7.8h, v4.h[2]\n"
+ "fmla v31.8h, v7.8h, v5.h[2]\n"
+ "ldr q7, [x15, #0xd0]\n"
+ "fmla v8.8h, v6.8h, v0.h[3]\n"
+ "fmla v12.8h, v6.8h, v1.h[3]\n"
+ "fmla v16.8h, v6.8h, v2.h[3]\n"
+ "fmla v20.8h, v6.8h, v3.h[3]\n"
+ "fmla v24.8h, v6.8h, v4.h[3]\n"
+ "fmla v28.8h, v6.8h, v5.h[3]\n"
+ "ldr q6, [x15, #0xe0]\n"
+ "fmla v9.8h, v7.8h, v0.h[3]\n"
+ "fmla v13.8h, v7.8h, v1.h[3]\n"
+ "fmla v17.8h, v7.8h, v2.h[3]\n"
+ "fmla v21.8h, v7.8h, v3.h[3]\n"
+ "fmla v25.8h, v7.8h, v4.h[3]\n"
+ "fmla v29.8h, v7.8h, v5.h[3]\n"
+ "ldr q7, [x15, #0xf0]\n"
+ "fmla v10.8h, v6.8h, v0.h[3]\n"
+ "fmla v14.8h, v6.8h, v1.h[3]\n"
+ "fmla v18.8h, v6.8h, v2.h[3]\n"
+ "fmla v22.8h, v6.8h, v3.h[3]\n"
+ "fmla v26.8h, v6.8h, v4.h[3]\n"
+ "fmla v30.8h, v6.8h, v5.h[3]\n"
+ "ldr q6, [x15, #0x100]\n"
+ "fmla v11.8h, v7.8h, v0.h[3]\n"
+ "fmla v15.8h, v7.8h, v1.h[3]\n"
+ "fmla v19.8h, v7.8h, v2.h[3]\n"
+ "fmla v23.8h, v7.8h, v3.h[3]\n"
+ "fmla v27.8h, v7.8h, v4.h[3]\n"
+ "fmla v31.8h, v7.8h, v5.h[3]\n"
+ "ldr q7, [x15, #0x110]\n"
+ "fmla v8.8h, v6.8h, v0.h[4]\n"
+ "fmla v12.8h, v6.8h, v1.h[4]\n"
+ "fmla v16.8h, v6.8h, v2.h[4]\n"
+ "fmla v20.8h, v6.8h, v3.h[4]\n"
+ "fmla v24.8h, v6.8h, v4.h[4]\n"
+ "fmla v28.8h, v6.8h, v5.h[4]\n"
+ "ldr q6, [x15, #0x120]\n"
+ "fmla v9.8h, v7.8h, v0.h[4]\n"
+ "fmla v13.8h, v7.8h, v1.h[4]\n"
+ "fmla v17.8h, v7.8h, v2.h[4]\n"
+ "fmla v21.8h, v7.8h, v3.h[4]\n"
+ "fmla v25.8h, v7.8h, v4.h[4]\n"
+ "fmla v29.8h, v7.8h, v5.h[4]\n"
+ "ldr q7, [x15, #0x130]\n"
+ "fmla v10.8h, v6.8h, v0.h[4]\n"
+ "fmla v14.8h, v6.8h, v1.h[4]\n"
+ "fmla v18.8h, v6.8h, v2.h[4]\n"
+ "fmla v22.8h, v6.8h, v3.h[4]\n"
+ "fmla v26.8h, v6.8h, v4.h[4]\n"
+ "fmla v30.8h, v6.8h, v5.h[4]\n"
+ "ldr q6, [x15, #0x140]\n"
+ "fmla v11.8h, v7.8h, v0.h[4]\n"
+ "fmla v15.8h, v7.8h, v1.h[4]\n"
+ "fmla v19.8h, v7.8h, v2.h[4]\n"
+ "fmla v23.8h, v7.8h, v3.h[4]\n"
+ "fmla v27.8h, v7.8h, v4.h[4]\n"
+ "fmla v31.8h, v7.8h, v5.h[4]\n"
+ "ldr q7, [x15, #0x150]\n"
+ "fmla v8.8h, v6.8h, v0.h[5]\n"
+ "fmla v12.8h, v6.8h, v1.h[5]\n"
+ "fmla v16.8h, v6.8h, v2.h[5]\n"
+ "fmla v20.8h, v6.8h, v3.h[5]\n"
+ "fmla v24.8h, v6.8h, v4.h[5]\n"
+ "fmla v28.8h, v6.8h, v5.h[5]\n"
+ "ldr q6, [x15, #0x160]\n"
+ "fmla v9.8h, v7.8h, v0.h[5]\n"
+ "fmla v13.8h, v7.8h, v1.h[5]\n"
+ "fmla v17.8h, v7.8h, v2.h[5]\n"
+ "fmla v21.8h, v7.8h, v3.h[5]\n"
+ "fmla v25.8h, v7.8h, v4.h[5]\n"
+ "fmla v29.8h, v7.8h, v5.h[5]\n"
+ "ldr q7, [x15, #0x170]\n"
+ "fmla v10.8h, v6.8h, v0.h[5]\n"
+ "fmla v14.8h, v6.8h, v1.h[5]\n"
+ "fmla v18.8h, v6.8h, v2.h[5]\n"
+ "fmla v22.8h, v6.8h, v3.h[5]\n"
+ "fmla v26.8h, v6.8h, v4.h[5]\n"
+ "fmla v30.8h, v6.8h, v5.h[5]\n"
+ "ldr q6, [x15, #0x180]\n"
+ "fmla v11.8h, v7.8h, v0.h[5]\n"
+ "fmla v15.8h, v7.8h, v1.h[5]\n"
+ "fmla v19.8h, v7.8h, v2.h[5]\n"
+ "fmla v23.8h, v7.8h, v3.h[5]\n"
+ "fmla v27.8h, v7.8h, v4.h[5]\n"
+ "fmla v31.8h, v7.8h, v5.h[5]\n"
+ "ldr q7, [x15, #0x190]\n"
+ "fmla v8.8h, v6.8h, v0.h[6]\n"
+ "fmla v12.8h, v6.8h, v1.h[6]\n"
+ "fmla v16.8h, v6.8h, v2.h[6]\n"
+ "fmla v20.8h, v6.8h, v3.h[6]\n"
+ "fmla v24.8h, v6.8h, v4.h[6]\n"
+ "fmla v28.8h, v6.8h, v5.h[6]\n"
+ "ldr q6, [x15, #0x1a0]\n"
+ "fmla v9.8h, v7.8h, v0.h[6]\n"
+ "fmla v13.8h, v7.8h, v1.h[6]\n"
+ "fmla v17.8h, v7.8h, v2.h[6]\n"
+ "fmla v21.8h, v7.8h, v3.h[6]\n"
+ "fmla v25.8h, v7.8h, v4.h[6]\n"
+ "fmla v29.8h, v7.8h, v5.h[6]\n"
+ "ldr q7, [x15, #0x1b0]\n"
+ "fmla v10.8h, v6.8h, v0.h[6]\n"
+ "fmla v14.8h, v6.8h, v1.h[6]\n"
+ "fmla v18.8h, v6.8h, v2.h[6]\n"
+ "fmla v22.8h, v6.8h, v3.h[6]\n"
+ "fmla v26.8h, v6.8h, v4.h[6]\n"
+ "fmla v30.8h, v6.8h, v5.h[6]\n"
+ "ldr q6, [x15, #0x1c0]\n"
+ "fmla v11.8h, v7.8h, v0.h[6]\n"
+ "fmla v15.8h, v7.8h, v1.h[6]\n"
+ "fmla v19.8h, v7.8h, v2.h[6]\n"
+ "fmla v23.8h, v7.8h, v3.h[6]\n"
+ "fmla v27.8h, v7.8h, v4.h[6]\n"
+ "fmla v31.8h, v7.8h, v5.h[6]\n"
+ "ldr q7, [x15, #0x1d0]\n"
+ "fmla v8.8h, v6.8h, v0.h[7]\n"
+ "fmla v12.8h, v6.8h, v1.h[7]\n"
+ "fmla v16.8h, v6.8h, v2.h[7]\n"
+ "fmla v20.8h, v6.8h, v3.h[7]\n"
+ "fmla v24.8h, v6.8h, v4.h[7]\n"
+ "fmla v28.8h, v6.8h, v5.h[7]\n"
+ "ldr q6, [x15, #0x1e0]\n"
+ "fmla v9.8h, v7.8h, v0.h[7]\n"
+ "fmla v13.8h, v7.8h, v1.h[7]\n"
+ "fmla v17.8h, v7.8h, v2.h[7]\n"
+ "fmla v21.8h, v7.8h, v3.h[7]\n"
+ "fmla v25.8h, v7.8h, v4.h[7]\n"
+ "fmla v29.8h, v7.8h, v5.h[7]\n"
+ "ldr q7, [x15, #0x1f0]\n"
+ "fmla v10.8h, v6.8h, v0.h[7]\n"
+ "add x15, x15, #0x200\n"
+ "fmla v14.8h, v6.8h, v1.h[7]\n"
+ "fmla v18.8h, v6.8h, v2.h[7]\n"
+ "fmla v22.8h, v6.8h, v3.h[7]\n"
+ "fmla v26.8h, v6.8h, v4.h[7]\n"
+ "fmla v30.8h, v6.8h, v5.h[7]\n"
+ "fmla v11.8h, v7.8h, v0.h[7]\n"
+ "fmla v15.8h, v7.8h, v1.h[7]\n"
+ "fmla v19.8h, v7.8h, v2.h[7]\n"
+ "fmla v23.8h, v7.8h, v3.h[7]\n"
+ "fmla v27.8h, v7.8h, v4.h[7]\n"
+ "fmla v31.8h, v7.8h, v5.h[7]\n"
+ "279:" // Height 6: Multiply loop: Main loop skip
+ "cbz x11, 281f\n"
+ "280:" // Height 6: Multiply loop: Odd block loop
+ "ldr h0, [x10], #0x2\n"
+ "ldr h1, [x28], #0x2\n"
+ "ldr h2, [x26], #0x2\n"
+ "ldr h3, [x24], #0x2\n"
+ "ldr h4, [x22], #0x2\n"
+ "ldr h5, [x20], #0x2\n"
+ "ldr q6, [x15, #0x0]\n"
+ "fmla v8.8h, v6.8h, v0.h[0]\n"
+ "ldr q7, [x15, #0x10]\n"
+ "fmla v12.8h, v6.8h, v1.h[0]\n"
+ "sub x11, x11, #0x1\n"
+ "fmla v16.8h, v6.8h, v2.h[0]\n"
+ "fmla v20.8h, v6.8h, v3.h[0]\n"
+ "fmla v24.8h, v6.8h, v4.h[0]\n"
+ "fmla v28.8h, v6.8h, v5.h[0]\n"
+ "ldr q6, [x15, #0x20]\n"
+ "fmla v9.8h, v7.8h, v0.h[0]\n"
+ "fmla v13.8h, v7.8h, v1.h[0]\n"
+ "fmla v17.8h, v7.8h, v2.h[0]\n"
+ "fmla v21.8h, v7.8h, v3.h[0]\n"
+ "fmla v25.8h, v7.8h, v4.h[0]\n"
+ "fmla v29.8h, v7.8h, v5.h[0]\n"
+ "ldr q7, [x15, #0x30]\n"
+ "fmla v10.8h, v6.8h, v0.h[0]\n"
+ "add x15, x15, #0x40\n"
+ "fmla v14.8h, v6.8h, v1.h[0]\n"
+ "fmla v18.8h, v6.8h, v2.h[0]\n"
+ "fmla v22.8h, v6.8h, v3.h[0]\n"
+ "fmla v26.8h, v6.8h, v4.h[0]\n"
+ "fmla v30.8h, v6.8h, v5.h[0]\n"
+ "fmla v11.8h, v7.8h, v0.h[0]\n"
+ "fmla v15.8h, v7.8h, v1.h[0]\n"
+ "fmla v19.8h, v7.8h, v2.h[0]\n"
+ "fmla v23.8h, v7.8h, v3.h[0]\n"
+ "fmla v27.8h, v7.8h, v4.h[0]\n"
+ "fmla v31.8h, v7.8h, v5.h[0]\n"
+ "cbnz x11, 280b\n"
+ "281:" // Height 6: Multiply loop: No odd multiplies
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "add x12, x12, #0x1\n"
+ "cmp x12, x19\n"
+ "bne 274b\n"
+ "prfm pstl1keep, [x13, #0x0]\n"
+ "prfm pstl1keep, [x9, #0x0]\n"
+ "prfm pstl1keep, [x27, #0x0]\n"
+ "prfm pstl1keep, [x25, #0x0]\n"
+ "prfm pstl1keep, [x23, #0x0]\n"
+ "prfm pstl1keep, [x21, #0x0]\n"
+ "tbz %x[flags], #1, 282f\n"
+ "add x19, %x[args_ptr], %[offset_min]\n"
+ "ld1r { v1.8h }, [x19]\n"
+ "add x19, %x[args_ptr], %[offset_max]\n"
+ "ld1r { v0.8h }, [x19]\n"
+ "fmin v8.8h, v8.8h, v0.8h\n"
+ "fmin v9.8h, v9.8h, v0.8h\n"
+ "fmin v10.8h, v10.8h, v0.8h\n"
+ "fmin v11.8h, v11.8h, v0.8h\n"
+ "fmax v8.8h, v8.8h, v1.8h\n"
+ "fmax v9.8h, v9.8h, v1.8h\n"
+ "fmax v10.8h, v10.8h, v1.8h\n"
+ "fmax v11.8h, v11.8h, v1.8h\n"
+ "fmin v12.8h, v12.8h, v0.8h\n"
+ "fmin v13.8h, v13.8h, v0.8h\n"
+ "fmin v14.8h, v14.8h, v0.8h\n"
+ "fmax v12.8h, v12.8h, v1.8h\n"
+ "fmax v13.8h, v13.8h, v1.8h\n"
+ "fmax v14.8h, v14.8h, v1.8h\n"
+ "fmin v15.8h, v15.8h, v0.8h\n"
+ "fmin v16.8h, v16.8h, v0.8h\n"
+ "fmin v17.8h, v17.8h, v0.8h\n"
+ "fmax v15.8h, v15.8h, v1.8h\n"
+ "fmax v16.8h, v16.8h, v1.8h\n"
+ "fmax v17.8h, v17.8h, v1.8h\n"
+ "fmin v18.8h, v18.8h, v0.8h\n"
+ "fmin v19.8h, v19.8h, v0.8h\n"
+ "fmin v20.8h, v20.8h, v0.8h\n"
+ "fmax v18.8h, v18.8h, v1.8h\n"
+ "fmax v19.8h, v19.8h, v1.8h\n"
+ "fmax v20.8h, v20.8h, v1.8h\n"
+ "fmin v21.8h, v21.8h, v0.8h\n"
+ "fmin v22.8h, v22.8h, v0.8h\n"
+ "fmin v23.8h, v23.8h, v0.8h\n"
+ "fmax v21.8h, v21.8h, v1.8h\n"
+ "fmax v22.8h, v22.8h, v1.8h\n"
+ "fmax v23.8h, v23.8h, v1.8h\n"
+ "fmin v24.8h, v24.8h, v0.8h\n"
+ "fmin v25.8h, v25.8h, v0.8h\n"
+ "fmin v26.8h, v26.8h, v0.8h\n"
+ "fmax v24.8h, v24.8h, v1.8h\n"
+ "fmax v25.8h, v25.8h, v1.8h\n"
+ "fmax v26.8h, v26.8h, v1.8h\n"
+ "fmin v27.8h, v27.8h, v0.8h\n"
+ "fmin v28.8h, v28.8h, v0.8h\n"
+ "fmin v29.8h, v29.8h, v0.8h\n"
+ "fmax v27.8h, v27.8h, v1.8h\n"
+ "fmax v28.8h, v28.8h, v1.8h\n"
+ "fmax v29.8h, v29.8h, v1.8h\n"
+ "fmin v30.8h, v30.8h, v0.8h\n"
+ "fmin v31.8h, v31.8h, v0.8h\n"
+ "fmax v30.8h, v30.8h, v1.8h\n"
+ "fmax v31.8h, v31.8h, v1.8h\n"
+ "282:" // Height 6: No activation
+ "cmp x16, #0x20\n"
+ "bge 299f\n"
+ "tbz x16, #4, 290f\n"
+ "st1 { v8.8h }, [x13], #0x10\n"
+ "st1 { v9.8h }, [x13], #0x10\n"
+ "st1 { v12.8h }, [x9], #0x10\n"
+ "st1 { v13.8h }, [x9], #0x10\n"
+ "st1 { v16.8h }, [x27], #0x10\n"
+ "st1 { v17.8h }, [x27], #0x10\n"
+ "st1 { v20.8h }, [x25], #0x10\n"
+ "st1 { v21.8h }, [x25], #0x10\n"
+ "st1 { v24.8h }, [x23], #0x10\n"
+ "st1 { v25.8h }, [x23], #0x10\n"
+ "st1 { v28.8h }, [x21], #0x10\n"
+ "st1 { v29.8h }, [x21], #0x10\n"
+ "tbz x16, #3, 286f\n"
+ "st1 { v10.8h }, [x13], #0x10\n"
+ "st1 { v14.8h }, [x9], #0x10\n"
+ "st1 { v18.8h }, [x27], #0x10\n"
+ "st1 { v22.8h }, [x25], #0x10\n"
+ "st1 { v26.8h }, [x23], #0x10\n"
+ "st1 { v30.8h }, [x21], #0x10\n"
+ "tbz x16, #2, 284f\n"
+ "str d11, [x13], #0x8\n"
+ "str d15, [x9], #0x8\n"
+ "str d19, [x27], #0x8\n"
+ "str d23, [x25], #0x8\n"
+ "str d27, [x23], #0x8\n"
+ "str d31, [x21], #0x8\n"
+ "tbz x16, #1, 283f\n"
+ "st1 { v11.s }[2], [x13], #0x4\n"
+ "st1 { v15.s }[2], [x9], #0x4\n"
+ "st1 { v19.s }[2], [x27], #0x4\n"
+ "st1 { v23.s }[2], [x25], #0x4\n"
+ "st1 { v27.s }[2], [x23], #0x4\n"
+ "st1 { v31.s }[2], [x21], #0x4\n"
+ "tbz x16, #0, 298f\n"
+ "st1 { v11.h }[6], [x13]\n"
+ "st1 { v15.h }[6], [x9]\n"
+ "st1 { v19.h }[6], [x27]\n"
+ "st1 { v23.h }[6], [x25]\n"
+ "st1 { v27.h }[6], [x23]\n"
+ "st1 { v31.h }[6], [x21]\n"
+ "b 298f\n"
+ "283:" // Height 6: Partial direct writeback: partial_1_28
+ "tbz x16, #0, 298f\n"
+ "st1 { v11.h }[4], [x13]\n"
+ "st1 { v15.h }[4], [x9]\n"
+ "st1 { v19.h }[4], [x27]\n"
+ "st1 { v23.h }[4], [x25]\n"
+ "st1 { v27.h }[4], [x23]\n"
+ "st1 { v31.h }[4], [x21]\n"
+ "b 298f\n"
+ "284:" // Height 6: Partial direct writeback: partial_2_24
+ "tbz x16, #1, 285f\n"
+ "str s11, [x13], #0x4\n"
+ "str s15, [x9], #0x4\n"
+ "str s19, [x27], #0x4\n"
+ "str s23, [x25], #0x4\n"
+ "str s27, [x23], #0x4\n"
+ "str s31, [x21], #0x4\n"
+ "tbz x16, #0, 298f\n"
+ "st1 { v11.h }[2], [x13]\n"
+ "st1 { v15.h }[2], [x9]\n"
+ "st1 { v19.h }[2], [x27]\n"
+ "st1 { v23.h }[2], [x25]\n"
+ "st1 { v27.h }[2], [x23]\n"
+ "st1 { v31.h }[2], [x21]\n"
+ "b 298f\n"
+ "285:" // Height 6: Partial direct writeback: partial_1_24
+ "tbz x16, #0, 298f\n"
+ "str h11, [x13, #0x0]\n"
+ "str h15, [x9, #0x0]\n"
+ "str h19, [x27, #0x0]\n"
+ "str h23, [x25, #0x0]\n"
+ "str h27, [x23, #0x0]\n"
+ "str h31, [x21, #0x0]\n"
+ "b 298f\n"
+ "286:" // Height 6: Partial direct writeback: partial_4_16
+ "tbz x16, #2, 288f\n"
+ "str d10, [x13], #0x8\n"
+ "str d14, [x9], #0x8\n"
+ "str d18, [x27], #0x8\n"
+ "str d22, [x25], #0x8\n"
+ "str d26, [x23], #0x8\n"
+ "str d30, [x21], #0x8\n"
+ "tbz x16, #1, 287f\n"
+ "st1 { v10.s }[2], [x13], #0x4\n"
+ "st1 { v14.s }[2], [x9], #0x4\n"
+ "st1 { v18.s }[2], [x27], #0x4\n"
+ "st1 { v22.s }[2], [x25], #0x4\n"
+ "st1 { v26.s }[2], [x23], #0x4\n"
+ "st1 { v30.s }[2], [x21], #0x4\n"
+ "tbz x16, #0, 298f\n"
+ "st1 { v10.h }[6], [x13]\n"
+ "st1 { v14.h }[6], [x9]\n"
+ "st1 { v18.h }[6], [x27]\n"
+ "st1 { v22.h }[6], [x25]\n"
+ "st1 { v26.h }[6], [x23]\n"
+ "st1 { v30.h }[6], [x21]\n"
+ "b 298f\n"
+ "287:" // Height 6: Partial direct writeback: partial_1_20
+ "tbz x16, #0, 298f\n"
+ "st1 { v10.h }[4], [x13]\n"
+ "st1 { v14.h }[4], [x9]\n"
+ "st1 { v18.h }[4], [x27]\n"
+ "st1 { v22.h }[4], [x25]\n"
+ "st1 { v26.h }[4], [x23]\n"
+ "st1 { v30.h }[4], [x21]\n"
+ "b 298f\n"
+ "288:" // Height 6: Partial direct writeback: partial_2_16
+ "tbz x16, #1, 289f\n"
+ "str s10, [x13], #0x4\n"
+ "str s14, [x9], #0x4\n"
+ "str s18, [x27], #0x4\n"
+ "str s22, [x25], #0x4\n"
+ "str s26, [x23], #0x4\n"
+ "str s30, [x21], #0x4\n"
+ "tbz x16, #0, 298f\n"
+ "st1 { v10.h }[2], [x13]\n"
+ "st1 { v14.h }[2], [x9]\n"
+ "st1 { v18.h }[2], [x27]\n"
+ "st1 { v22.h }[2], [x25]\n"
+ "st1 { v26.h }[2], [x23]\n"
+ "st1 { v30.h }[2], [x21]\n"
+ "b 298f\n"
+ "289:" // Height 6: Partial direct writeback: partial_1_16
+ "tbz x16, #0, 298f\n"
+ "str h10, [x13, #0x0]\n"
+ "str h14, [x9, #0x0]\n"
+ "str h18, [x27, #0x0]\n"
+ "str h22, [x25, #0x0]\n"
+ "str h26, [x23, #0x0]\n"
+ "str h30, [x21, #0x0]\n"
+ "b 298f\n"
+ "290:" // Height 6: Partial direct writeback: partial_8_0
+ "tbz x16, #3, 294f\n"
+ "st1 { v8.8h }, [x13], #0x10\n"
+ "st1 { v12.8h }, [x9], #0x10\n"
+ "st1 { v16.8h }, [x27], #0x10\n"
+ "st1 { v20.8h }, [x25], #0x10\n"
+ "st1 { v24.8h }, [x23], #0x10\n"
+ "st1 { v28.8h }, [x21], #0x10\n"
+ "tbz x16, #2, 292f\n"
+ "str d9, [x13], #0x8\n"
+ "str d13, [x9], #0x8\n"
+ "str d17, [x27], #0x8\n"
+ "str d21, [x25], #0x8\n"
+ "str d25, [x23], #0x8\n"
+ "str d29, [x21], #0x8\n"
+ "tbz x16, #1, 291f\n"
+ "st1 { v9.s }[2], [x13], #0x4\n"
+ "st1 { v13.s }[2], [x9], #0x4\n"
+ "st1 { v17.s }[2], [x27], #0x4\n"
+ "st1 { v21.s }[2], [x25], #0x4\n"
+ "st1 { v25.s }[2], [x23], #0x4\n"
+ "st1 { v29.s }[2], [x21], #0x4\n"
+ "tbz x16, #0, 298f\n"
+ "st1 { v9.h }[6], [x13]\n"
+ "st1 { v13.h }[6], [x9]\n"
+ "st1 { v17.h }[6], [x27]\n"
+ "st1 { v21.h }[6], [x25]\n"
+ "st1 { v25.h }[6], [x23]\n"
+ "st1 { v29.h }[6], [x21]\n"
+ "b 298f\n"
+ "291:" // Height 6: Partial direct writeback: partial_1_12
+ "tbz x16, #0, 298f\n"
+ "st1 { v9.h }[4], [x13]\n"
+ "st1 { v13.h }[4], [x9]\n"
+ "st1 { v17.h }[4], [x27]\n"
+ "st1 { v21.h }[4], [x25]\n"
+ "st1 { v25.h }[4], [x23]\n"
+ "st1 { v29.h }[4], [x21]\n"
+ "b 298f\n"
+ "292:" // Height 6: Partial direct writeback: partial_2_8
+ "tbz x16, #1, 293f\n"
+ "str s9, [x13], #0x4\n"
+ "str s13, [x9], #0x4\n"
+ "str s17, [x27], #0x4\n"
+ "str s21, [x25], #0x4\n"
+ "str s25, [x23], #0x4\n"
+ "str s29, [x21], #0x4\n"
+ "tbz x16, #0, 298f\n"
+ "st1 { v9.h }[2], [x13]\n"
+ "st1 { v13.h }[2], [x9]\n"
+ "st1 { v17.h }[2], [x27]\n"
+ "st1 { v21.h }[2], [x25]\n"
+ "st1 { v25.h }[2], [x23]\n"
+ "st1 { v29.h }[2], [x21]\n"
+ "b 298f\n"
+ "293:" // Height 6: Partial direct writeback: partial_1_8
+ "tbz x16, #0, 298f\n"
+ "str h9, [x13, #0x0]\n"
+ "str h13, [x9, #0x0]\n"
+ "str h17, [x27, #0x0]\n"
+ "str h21, [x25, #0x0]\n"
+ "str h25, [x23, #0x0]\n"
+ "str h29, [x21, #0x0]\n"
+ "b 298f\n"
+ "294:" // Height 6: Partial direct writeback: partial_4_0
+ "tbz x16, #2, 296f\n"
+ "str d8, [x13], #0x8\n"
+ "str d12, [x9], #0x8\n"
+ "str d16, [x27], #0x8\n"
+ "str d20, [x25], #0x8\n"
+ "str d24, [x23], #0x8\n"
+ "str d28, [x21], #0x8\n"
+ "tbz x16, #1, 295f\n"
+ "st1 { v8.s }[2], [x13], #0x4\n"
+ "st1 { v12.s }[2], [x9], #0x4\n"
+ "st1 { v16.s }[2], [x27], #0x4\n"
+ "st1 { v20.s }[2], [x25], #0x4\n"
+ "st1 { v24.s }[2], [x23], #0x4\n"
+ "st1 { v28.s }[2], [x21], #0x4\n"
+ "tbz x16, #0, 298f\n"
+ "st1 { v8.h }[6], [x13]\n"
+ "st1 { v12.h }[6], [x9]\n"
+ "st1 { v16.h }[6], [x27]\n"
+ "st1 { v20.h }[6], [x25]\n"
+ "st1 { v24.h }[6], [x23]\n"
+ "st1 { v28.h }[6], [x21]\n"
+ "b 298f\n"
+ "295:" // Height 6: Partial direct writeback: partial_1_4
+ "tbz x16, #0, 298f\n"
+ "st1 { v8.h }[4], [x13]\n"
+ "st1 { v12.h }[4], [x9]\n"
+ "st1 { v16.h }[4], [x27]\n"
+ "st1 { v20.h }[4], [x25]\n"
+ "st1 { v24.h }[4], [x23]\n"
+ "st1 { v28.h }[4], [x21]\n"
+ "b 298f\n"
+ "296:" // Height 6: Partial direct writeback: partial_2_0
+ "tbz x16, #1, 297f\n"
+ "str s8, [x13], #0x4\n"
+ "str s12, [x9], #0x4\n"
+ "str s16, [x27], #0x4\n"
+ "str s20, [x25], #0x4\n"
+ "str s24, [x23], #0x4\n"
+ "str s28, [x21], #0x4\n"
+ "tbz x16, #0, 298f\n"
+ "st1 { v8.h }[2], [x13]\n"
+ "st1 { v12.h }[2], [x9]\n"
+ "st1 { v16.h }[2], [x27]\n"
+ "st1 { v20.h }[2], [x25]\n"
+ "st1 { v24.h }[2], [x23]\n"
+ "st1 { v28.h }[2], [x21]\n"
+ "b 298f\n"
+ "297:" // Height 6: Partial direct writeback: partial_1_0
+ "str h8, [x13, #0x0]\n"
+ "str h12, [x9, #0x0]\n"
+ "str h16, [x27, #0x0]\n"
+ "str h20, [x25, #0x0]\n"
+ "str h24, [x23, #0x0]\n"
+ "str h28, [x21, #0x0]\n"
+ "298:" // Height 6: Partial direct writeback: Done
+ "b 300f\n"
+ "299:" // Height 6: Full writeback
+ "str q8, [x13, #0x0]\n"
+ "str q9, [x13, #0x10]\n"
+ "str q10, [x13, #0x20]\n"
+ "str q11, [x13, #0x30]\n"
+ "str q12, [x9, #0x0]\n"
+ "str q13, [x9, #0x10]\n"
+ "str q14, [x9, #0x20]\n"
+ "str q15, [x9, #0x30]\n"
+ "str q16, [x27, #0x0]\n"
+ "str q17, [x27, #0x10]\n"
+ "str q18, [x27, #0x20]\n"
+ "str q19, [x27, #0x30]\n"
+ "str q20, [x25, #0x0]\n"
+ "str q21, [x25, #0x10]\n"
+ "str q22, [x25, #0x20]\n"
+ "str q23, [x25, #0x30]\n"
+ "str q24, [x23, #0x0]\n"
+ "str q25, [x23, #0x10]\n"
+ "str q26, [x23, #0x20]\n"
+ "str q27, [x23, #0x30]\n"
+ "str q28, [x21, #0x0]\n"
+ "str q29, [x21, #0x10]\n"
+ "str q30, [x21, #0x20]\n"
+ "str q31, [x21, #0x30]\n"
+ "add x13, x13, #0x40\n"
+ "add x9, x9, #0x40\n"
+ "add x27, x27, #0x40\n"
+ "add x25, x25, #0x40\n"
+ "add x23, x23, #0x40\n"
+ "add x21, x21, #0x40\n"
+ "300:" // Height 6: Writeback done
+ "subs x16, x16, #0x20\n"
+ "bgt 253b\n"
+ "subs %x[M], %x[M], #0x6\n"
+ "beq 302f\n"
+ "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "tbz %x[flags], #3, 301f\n"
+ "add x20, x20, #0x6\n"
+ "str x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "b 1b\n"
+ "301:" // Update direct input
+ "mov x19, #0xc\n"
+ "madd %x[input_ptr], x19, x20, %x[input_ptr]\n"
+ "b 1b\n"
+ "302:" // Exit
+
+ : [M] "+r" (M), [input_ptr] "+r" (input_ptr), [output_ptr] "+r" (output_ptr)
+ : [args_ptr] "r" (&ka), [bias] "r" (bias), [flags] "r" (flags), [offset_max] "I" (offsetof(KernelArgs, maxval)), [offset_min] "I" (offsetof(KernelArgs, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths))
+ : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
+ );
+}
+
+} // namespace arm_gemm
+#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_16x4.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_16x4.hpp
deleted file mode 100644
index 4147ab6..0000000
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_16x4.hpp
+++ /dev/null
@@ -1,111 +0,0 @@
-/*
- * Copyright (c) 2018-2020 Arm Limited.
- *
- * SPDX-License-Identifier: MIT
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to
- * deal in the Software without restriction, including without limitation the
- * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
- * sell copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-#pragma once
-
-#ifdef __aarch64__
-
-#include "../performance_parameters.hpp"
-#include "../std_transforms_fixed.hpp"
-
-namespace arm_gemm
-{
-
-// Actual kernel implementations
-void a64_hybrid_fp32_mla_16x4(const float *, int, const float *, float *, int, int, int, int, const float *, Activation, bool);
-void a64_hybrid_fp32_mla_16x4_a55(const float *, int, const float *, float *, int, int, int, int, const float *, Activation, bool);
-void a64_hybrid_fp32_mla_16x4_x1(const float *, int, const float *, float *, int, int, int, int, const float *, Activation, bool);
-
-class hybrid_fp32_mla_16x4
-{
-public:
- typedef float operand_type;
- typedef float result_type;
-
- typedef void (*kern_type)(const float *, int, const float *, float *, int, int, int, int, const float *, Activation, bool);
-
- /* Kernel blocking parameters */
- static constexpr unsigned int out_height()
- {
- return 4;
- }
-
- static unsigned int out_width()
- {
- return 16;
- }
-
- static constexpr unsigned int k_unroll()
- {
- return 1;
- }
-
- static constexpr bool supports_accumulate()
- {
- return true;
- }
-
- static constexpr bool supports_bias()
- {
- return true;
- }
-
- static constexpr bool supports_activation()
- {
- return true;
- }
-
- static PerformanceParameters get_performance_parameters(const CPUInfo *ci) {
- switch (ci->get_cpu_model()) {
- case CPUModel::A55r1:
- return { 2.866 };
-
- case CPUModel::A53:
- return { 1.419 };
-
- case CPUModel::A73:
- return { 2.551 };
-
- default:
- return { 6.25 };
- }
- }
-
- StdTransformsFixed<operand_type, result_type, 4, 16, 1> transforms = {};
-
- // Default to the generic kernel
- kern_type kernel=a64_hybrid_fp32_mla_16x4;
-
- hybrid_fp32_mla_16x4(const CPUInfo *ci)
- {
- if (ci->get_cpu_model() == CPUModel::A55r1) {
- kernel = a64_hybrid_fp32_mla_16x4_a55;
- } else if (ci->get_cpu_model() == CPUModel::X1) {
- kernel = a64_hybrid_fp32_mla_16x4_x1;
- }
- }
-};
-
-} // namespace arm_gemm
-
-#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_16x4/a55.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_16x4/a55.cpp
deleted file mode 100644
index 94fcd10..0000000
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_16x4/a55.cpp
+++ /dev/null
@@ -1,2427 +0,0 @@
-/*
- * Copyright (c) 2018-2020 Arm Limited.
- *
- * SPDX-License-Identifier: MIT
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to
- * deal in the Software without restriction, including without limitation the
- * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
- * sell copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-#ifdef __aarch64__
-
-#include <algorithm>
-
-#include "arm_gemm.hpp"
-
-#include "../../asmlib.hpp"
-#include "../../utils.hpp"
-
-namespace arm_gemm {
-
-void a64_hybrid_fp32_mla_16x4_a55(const float *A, int lda, const float *B, float *C, int ldc, int M, int N, int K, const float *bias, Activation act, bool accumulate) {
- const int K_stride = K;
- const long loops_count = ((K + 4) / 8) - 1;
- K -= loops_count * 8;
- const long regs_count = (K / 4) - 1;
- K -= (regs_count + 1) * 4;
- const long blocks_count = K / 1;
- float nullbias[16];
- if (!accumulate && !bias) {
- memset(nullbias, 0, (16 * sizeof(float)));
- }
- float minval = - static_cast<float>(std::numeric_limits<float>::infinity());
- float maxval = static_cast<float>(std::numeric_limits<float>::infinity());
- const float * const minptr = &minval;
- const float * const maxptr = &maxval;
-
- switch(act.type)
- {
- default:
- case Activation::Type::None:
- break;
- case Activation::Type::BoundedReLU:
- maxval = static_cast<float>(act.param1);
- /* fall through */
- case Activation::Type::ReLU:
- minval = 0.0f;
- break;
- }
-
- int rows_to_compute;
-
- for (int y=0; y<M; y+=rows_to_compute) {
- const float * const a_ptr0_base = A + (y * lda);
- const unsigned long ldab = lda * sizeof(float);
-
- float *c_ptr0 = C + (y * ldc);
-
- rows_to_compute = M-y;
- if (rows_to_compute > 4) {
- if (rows_to_compute % 4) {
- rows_to_compute = 4 - 1;
- } else {
- rows_to_compute = 4;
- }
- }
-
- for (int x0=0; x0<N; x0+=16ul) {
- const long width = std::min((unsigned long)N-x0, 16ul);
- long loops = loops_count;
- long regs = regs_count;
- long blocks = blocks_count;
- const float *a_ptr0 = a_ptr0_base;
- const float *b_ptr0 = B + (K_stride * x0);
- const bool use_result_buffer = (width < 16);
- float result_buffer[64];
- const unsigned long ldcb = (use_result_buffer ? 16 : ldc) * sizeof(float);
- float *c_ptr_real = c_ptr0;
- if (use_result_buffer && accumulate) {
- for(int cy=0; cy<std::min(M-y, 4); cy++) {
- for(unsigned int cx=0; cx<width; cx++) {
- result_buffer[cy * 16 + cx] = c_ptr_real[cy * ldc + cx];
- }
- }
- }
- if (use_result_buffer) {
- c_ptr0 = result_buffer;
- }
- const float *biasptr = bias ? bias+x0 : nullbias;
-
- switch(rows_to_compute) {
- case 1:
- __asm __volatile (
- "temploadreg0 .req X0\n"
- "temploadreg1 .req X1\n"
- "temploadreg2 .req X2\n"
- "temploadreg3 .req X3\n"
- "cbnz %[accumulate], 1f\n"
- "ldr q16, [%[biasptr]]\n"
- "ldr q17, [%[biasptr], #0x10]\n"
- "ldr q18, [%[biasptr], #0x20]\n"
- "ldr q19, [%[biasptr], #0x30]\n"
- "ldr q0, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ldr q8, [%[b_ptr0]]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "ldr q16, [%[c_ptr0]]\n"
- "ldr q17, [%[c_ptr0], #0x10]\n"
- "ldr q18, [%[c_ptr0], #0x20]\n"
- "ldr q19, [%[c_ptr0], #0x30]\n"
- "ldr q0, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ldr q8, [%[b_ptr0]]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "cbz %[loops], 2f\n"
- "3:\n"
- "fmla v16.4s, v8.4s, v0.s[0]\n"
- "ins v14.d[1], temploadreg2\n"
- "fmla v17.4s, v9.4s, v0.s[0]\n"
- "ldr d15, [%[b_ptr0], #-0x10]\n"
- "fmla v18.4s, v10.4s, v0.s[0]\n"
- "ldr temploadreg3, [%[b_ptr0], #-0x8]\n"
- "fmla v19.4s, v11.4s, v0.s[0]\n"
- "ldr d4, [%[a_ptr0]]\n"
- "fmla v16.4s, v12.4s, v0.s[1]\n"
- "ldr temploadreg0, [%[a_ptr0], #0x8]\n"
- "fmla v17.4s, v13.4s, v0.s[1]\n"
- "ldr d8, [%[b_ptr0]]\n"
- "fmla v18.4s, v14.4s, v0.s[1]\n"
- "ldr d9, [%[b_ptr0], #0x10]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ins v4.d[1], temploadreg0\n"
- "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
- "ldr d10, [%[b_ptr0], #0x20]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- "ldr d11, [%[b_ptr0], #0x30]\n"
- "ins v15.d[1], temploadreg3\n"
- "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- "ldr d12, [%[b_ptr0], #0x40]\n"
- "ins v8.d[1], temploadreg0\n"
- "fmla v19.4s, v15.4s, v0.s[1]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- "ldr d13, [%[b_ptr0], #0x50]\n"
- "ins v9.d[1], temploadreg1\n"
- "fmla v16.4s, v8.4s, v0.s[2]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- "ins v10.d[1], temploadreg2\n"
- "fmla v17.4s, v9.4s, v0.s[2]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- "ldr d15, [%[b_ptr0], #0x70]\n"
- "ins v11.d[1], temploadreg3\n"
- "fmla v18.4s, v10.4s, v0.s[2]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- "ins v12.d[1], temploadreg0\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- "fmla v19.4s, v11.4s, v0.s[2]\n"
- "ldr d8, [%[b_ptr0], #-0x80]\n"
- "ldr temploadreg0, [%[b_ptr0], #-0x78]\n"
- "fmla v16.4s, v12.4s, v0.s[3]\n"
- "ldr d9, [%[b_ptr0], #-0x70]\n"
- "ins v13.d[1], temploadreg1\n"
- "ldr temploadreg1, [%[b_ptr0], #-0x68]\n"
- "ldr d10, [%[b_ptr0], #-0x60]\n"
- "ins v14.d[1], temploadreg2\n"
- "fmla v17.4s, v13.4s, v0.s[3]\n"
- "ldr temploadreg2, [%[b_ptr0], #-0x58]\n"
- "ldr d11, [%[b_ptr0], #-0x50]\n"
- "ins v15.d[1], temploadreg3\n"
- "fmla v18.4s, v14.4s, v0.s[3]\n"
- "ldr temploadreg3, [%[b_ptr0], #-0x48]\n"
- "ldr d12, [%[b_ptr0], #-0x40]\n"
- "ins v8.d[1], temploadreg0\n"
- "fmla v19.4s, v15.4s, v0.s[3]\n"
- "ldr temploadreg0, [%[b_ptr0], #-0x38]\n"
- "ldr d13, [%[b_ptr0], #-0x30]\n"
- "ins v9.d[1], temploadreg1\n"
- "fmla v16.4s, v8.4s, v4.s[0]\n"
- "ldr temploadreg1, [%[b_ptr0], #-0x28]\n"
- "ldr d14, [%[b_ptr0], #-0x20]\n"
- "ins v10.d[1], temploadreg2\n"
- "fmla v17.4s, v9.4s, v4.s[0]\n"
- "ldr temploadreg2, [%[b_ptr0], #-0x18]\n"
- "ldr d15, [%[b_ptr0], #-0x10]\n"
- "ins v11.d[1], temploadreg3\n"
- "fmla v18.4s, v10.4s, v4.s[0]\n"
- "ldr temploadreg3, [%[b_ptr0], #-0x8]\n"
- "ldr d0, [%[a_ptr0], #-0x10]\n"
- "ins v12.d[1], temploadreg0\n"
- "fmla v19.4s, v11.4s, v4.s[0]\n"
- "ldr temploadreg0, [%[a_ptr0], #-0x8]\n"
- "ldr d8, [%[b_ptr0]]\n"
- "ldr d9, [%[b_ptr0], #0x10]\n"
- "fmla v16.4s, v12.4s, v4.s[1]\n"
- "ins v0.d[1], temploadreg0\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- "ins v13.d[1], temploadreg1\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- "ldr d10, [%[b_ptr0], #0x20]\n"
- "ins v14.d[1], temploadreg2\n"
- "fmla v17.4s, v13.4s, v4.s[1]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- "ldr d11, [%[b_ptr0], #0x30]\n"
- "ins v15.d[1], temploadreg3\n"
- "fmla v18.4s, v14.4s, v4.s[1]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- "ldr d12, [%[b_ptr0], #0x40]\n"
- "ins v8.d[1], temploadreg0\n"
- "fmla v19.4s, v15.4s, v4.s[1]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- "ldr d13, [%[b_ptr0], #0x50]\n"
- "ins v9.d[1], temploadreg1\n"
- "fmla v16.4s, v8.4s, v4.s[2]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- "ins v10.d[1], temploadreg2\n"
- "fmla v17.4s, v9.4s, v4.s[2]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- "ldr d15, [%[b_ptr0], #0x70]\n"
- "ins v11.d[1], temploadreg3\n"
- "fmla v18.4s, v10.4s, v4.s[2]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- "ins v12.d[1], temploadreg0\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- "fmla v19.4s, v11.4s, v4.s[2]\n"
- "ldr d8, [%[b_ptr0], #-0x80]\n"
- "ldr temploadreg0, [%[b_ptr0], #-0x78]\n"
- "fmla v16.4s, v12.4s, v4.s[3]\n"
- "ldr d9, [%[b_ptr0], #-0x70]\n"
- "ins v13.d[1], temploadreg1\n"
- "ldr temploadreg1, [%[b_ptr0], #-0x68]\n"
- "ldr d10, [%[b_ptr0], #-0x60]\n"
- "ins v14.d[1], temploadreg2\n"
- "fmla v17.4s, v13.4s, v4.s[3]\n"
- "ldr temploadreg2, [%[b_ptr0], #-0x58]\n"
- "ldr d11, [%[b_ptr0], #-0x50]\n"
- "ins v15.d[1], temploadreg3\n"
- "fmla v18.4s, v14.4s, v4.s[3]\n"
- "ldr temploadreg3, [%[b_ptr0], #-0x48]\n"
- "ldr d12, [%[b_ptr0], #-0x40]\n"
- "ins v8.d[1], temploadreg0\n"
- "fmla v19.4s, v15.4s, v4.s[3]\n"
- "ldr temploadreg0, [%[b_ptr0], #-0x38]\n"
- "ldr d13, [%[b_ptr0], #-0x30]\n"
- "ins v9.d[1], temploadreg1\n"
- "ldr temploadreg1, [%[b_ptr0], #-0x28]\n"
- "ldr d14, [%[b_ptr0], #-0x20]\n"
- "ins v10.d[1], temploadreg2\n"
- "ldr temploadreg2, [%[b_ptr0], #-0x18]\n"
- "ins v11.d[1], temploadreg3\n"
- "ins v12.d[1], temploadreg0\n"
- "ins v13.d[1], temploadreg1\n"
- "b.ne 3b\n"
- "2:\n"
- "ins v14.d[1], temploadreg2\n"
- "prfm PSTL1KEEP, [%[c_ptr0]]\n"
- "ldr d15, [%[b_ptr0], #-0x10]\n"
- "ldr temploadreg3, [%[b_ptr0], #-0x8]\n"
- "ins v15.d[1], temploadreg3\n"
- "cbz %[regs], 4f\n"
- "fmla v16.4s, v8.4s, v0.s[0]\n"
- "ldr d4, [%[a_ptr0]]\n"
- "fmla v17.4s, v9.4s, v0.s[0]\n"
- "ldr temploadreg0, [%[a_ptr0], #0x8]\n"
- "fmla v18.4s, v10.4s, v0.s[0]\n"
- "ldr d8, [%[b_ptr0]]\n"
- "fmla v19.4s, v11.4s, v0.s[0]\n"
- "ldr d9, [%[b_ptr0], #0x10]\n"
- "fmla v16.4s, v12.4s, v0.s[1]\n"
- "ins v4.d[1], temploadreg0\n"
- "fmla v17.4s, v13.4s, v0.s[1]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- "fmla v18.4s, v14.4s, v0.s[1]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- "fmla v19.4s, v15.4s, v0.s[1]\n"
- "ldr d10, [%[b_ptr0], #0x20]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ldr d11, [%[b_ptr0], #0x30]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- "ldr d12, [%[b_ptr0], #0x40]\n"
- "ins v8.d[1], temploadreg0\n"
- "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- "ldr d13, [%[b_ptr0], #0x50]\n"
- "ins v9.d[1], temploadreg1\n"
- "fmla v16.4s, v8.4s, v0.s[2]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- "ins v10.d[1], temploadreg2\n"
- "fmla v17.4s, v9.4s, v0.s[2]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- "ldr d15, [%[b_ptr0], #0x70]\n"
- "ins v11.d[1], temploadreg3\n"
- "fmla v18.4s, v10.4s, v0.s[2]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- "ins v12.d[1], temploadreg0\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- "fmla v19.4s, v11.4s, v0.s[2]\n"
- "ldr d8, [%[b_ptr0], #-0x80]\n"
- "ldr temploadreg0, [%[b_ptr0], #-0x78]\n"
- "fmla v16.4s, v12.4s, v0.s[3]\n"
- "ldr d9, [%[b_ptr0], #-0x70]\n"
- "ins v13.d[1], temploadreg1\n"
- "ldr temploadreg1, [%[b_ptr0], #-0x68]\n"
- "ldr d10, [%[b_ptr0], #-0x60]\n"
- "ins v14.d[1], temploadreg2\n"
- "fmla v17.4s, v13.4s, v0.s[3]\n"
- "ldr temploadreg2, [%[b_ptr0], #-0x58]\n"
- "ldr d11, [%[b_ptr0], #-0x50]\n"
- "ins v15.d[1], temploadreg3\n"
- "fmla v18.4s, v14.4s, v0.s[3]\n"
- "ldr temploadreg3, [%[b_ptr0], #-0x48]\n"
- "ldr d12, [%[b_ptr0], #-0x40]\n"
- "ins v8.d[1], temploadreg0\n"
- "fmla v19.4s, v15.4s, v0.s[3]\n"
- "ldr temploadreg0, [%[b_ptr0], #-0x38]\n"
- "ldr d13, [%[b_ptr0], #-0x30]\n"
- "ins v9.d[1], temploadreg1\n"
- "fmla v16.4s, v8.4s, v4.s[0]\n"
- "ldr temploadreg1, [%[b_ptr0], #-0x28]\n"
- "ldr d14, [%[b_ptr0], #-0x20]\n"
- "ins v10.d[1], temploadreg2\n"
- "fmla v17.4s, v9.4s, v4.s[0]\n"
- "ldr temploadreg2, [%[b_ptr0], #-0x18]\n"
- "ldr d15, [%[b_ptr0], #-0x10]\n"
- "ins v11.d[1], temploadreg3\n"
- "fmla v18.4s, v10.4s, v4.s[0]\n"
- "ldr temploadreg3, [%[b_ptr0], #-0x8]\n"
- "ldr d8, [%[b_ptr0]]\n"
- "ins v12.d[1], temploadreg0\n"
- "fmla v19.4s, v11.4s, v4.s[0]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- "ldr d9, [%[b_ptr0], #0x10]\n"
- "ins v13.d[1], temploadreg1\n"
- "fmla v16.4s, v12.4s, v4.s[1]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- "ldr d10, [%[b_ptr0], #0x20]\n"
- "ins v14.d[1], temploadreg2\n"
- "fmla v17.4s, v13.4s, v4.s[1]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- "ldr d11, [%[b_ptr0], #0x30]\n"
- "ins v15.d[1], temploadreg3\n"
- "fmla v18.4s, v14.4s, v4.s[1]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- "ldr d12, [%[b_ptr0], #0x40]\n"
- "ins v8.d[1], temploadreg0\n"
- "fmla v19.4s, v15.4s, v4.s[1]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- "ldr d13, [%[b_ptr0], #0x50]\n"
- "ins v9.d[1], temploadreg1\n"
- "fmla v16.4s, v8.4s, v4.s[2]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- "ins v10.d[1], temploadreg2\n"
- "fmla v17.4s, v9.4s, v4.s[2]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- "ldr d15, [%[b_ptr0], #0x70]\n"
- "ins v11.d[1], temploadreg3\n"
- "fmla v18.4s, v10.4s, v4.s[2]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- "ins v12.d[1], temploadreg0\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "fmla v19.4s, v11.4s, v4.s[2]\n"
- "ins v13.d[1], temploadreg1\n"
- "ins v14.d[1], temploadreg2\n"
- "fmla v16.4s, v12.4s, v4.s[3]\n"
- "ins v15.d[1], temploadreg3\n"
- "fmla v17.4s, v13.4s, v4.s[3]\n"
- "fmla v18.4s, v14.4s, v4.s[3]\n"
- "fmla v19.4s, v15.4s, v4.s[3]\n"
- "b 5f\n"
- "4:\n"
- "fmla v16.4s, v8.4s, v0.s[0]\n"
- "ldr d8, [%[b_ptr0]]\n"
- "fmla v17.4s, v9.4s, v0.s[0]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- "fmla v18.4s, v10.4s, v0.s[0]\n"
- "ldr d9, [%[b_ptr0], #0x10]\n"
- "fmla v19.4s, v11.4s, v0.s[0]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- "fmla v16.4s, v12.4s, v0.s[1]\n"
- "ldr d10, [%[b_ptr0], #0x20]\n"
- "fmla v17.4s, v13.4s, v0.s[1]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- "fmla v18.4s, v14.4s, v0.s[1]\n"
- "ldr d11, [%[b_ptr0], #0x30]\n"
- "fmla v19.4s, v15.4s, v0.s[1]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- "ldr d12, [%[b_ptr0], #0x40]\n"
- "ins v8.d[1], temploadreg0\n"
- "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- "ldr d13, [%[b_ptr0], #0x50]\n"
- "ins v9.d[1], temploadreg1\n"
- "fmla v16.4s, v8.4s, v0.s[2]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- "ins v10.d[1], temploadreg2\n"
- "fmla v17.4s, v9.4s, v0.s[2]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- "ldr d15, [%[b_ptr0], #0x70]\n"
- "ins v11.d[1], temploadreg3\n"
- "fmla v18.4s, v10.4s, v0.s[2]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- "ins v12.d[1], temploadreg0\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "fmla v19.4s, v11.4s, v0.s[2]\n"
- "ins v13.d[1], temploadreg1\n"
- "ins v14.d[1], temploadreg2\n"
- "fmla v16.4s, v12.4s, v0.s[3]\n"
- "ins v15.d[1], temploadreg3\n"
- "fmla v17.4s, v13.4s, v0.s[3]\n"
- "fmla v18.4s, v14.4s, v0.s[3]\n"
- "fmla v19.4s, v15.4s, v0.s[3]\n"
- "5:\n"
- "cbz %[blocks], 6f\n"
- "7:\n"
- "ldr q8, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "ldr s0, [%[a_ptr0]]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "add %[a_ptr0], %[a_ptr0], #0x4\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "add %[b_ptr0], %[b_ptr0], #0x40\n"
- "fmla v16.4s, v8.4s, v0.s[0]\n"
- "fmla v17.4s, v9.4s, v0.s[0]\n"
- "fmla v18.4s, v10.4s, v0.s[0]\n"
- "fmla v19.4s, v11.4s, v0.s[0]\n"
- "b.ne 7b\n"
- "6:\n"
- "ld1r {v14.4s}, [%[minptr]]\n"
- "ld1r {v15.4s}, [%[maxptr]]\n"
- "fmax v16.4s, v16.4s, v14.4s\n"
- "fmax v17.4s, v17.4s, v14.4s\n"
- "fmax v18.4s, v18.4s, v14.4s\n"
- "fmax v19.4s, v19.4s, v14.4s\n"
- "fmin v16.4s, v16.4s, v15.4s\n"
- "fmin v17.4s, v17.4s, v15.4s\n"
- "fmin v18.4s, v18.4s, v15.4s\n"
- "fmin v19.4s, v19.4s, v15.4s\n"
- "str q16, [%[c_ptr0]]\n"
- "str q17, [%[c_ptr0], #0x10]\n"
- "str q18, [%[c_ptr0], #0x20]\n"
- "str q19, [%[c_ptr0], #0x30]\n"
- "add %[c_ptr0], %[c_ptr0], #0x40\n"
- ".unreq temploadreg0\n"
- ".unreq temploadreg1\n"
- ".unreq temploadreg2\n"
- ".unreq temploadreg3\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks)
- : [width] "r" (width), [accumulate] "r" (static_cast<uint64_t>(accumulate)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr)
- : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "cc", "memory"
- );
- break;
- case 2:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "c_ptr1 .req X1\n"
- "temploadreg0 .req X2\n"
- "temploadreg1 .req X3\n"
- "temploadreg2 .req X4\n"
- "temploadreg3 .req X5\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "cbnz %[accumulate], 1f\n"
- "ldr q16, [%[biasptr]]\n"
- "ldr q17, [%[biasptr], #0x10]\n"
- "ldr q18, [%[biasptr], #0x20]\n"
- "ldr q19, [%[biasptr], #0x30]\n"
- "mov v20.16b, v16.16b\n"
- "ldr q0, [%[a_ptr0]]\n"
- "mov v21.16b, v17.16b\n"
- "ldr q1, [a_ptr1]\n"
- "mov v22.16b, v18.16b\n"
- "ldr q8, [%[b_ptr0]]\n"
- "mov v23.16b, v19.16b\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "ldr q16, [%[c_ptr0]]\n"
- "ldr q17, [%[c_ptr0], #0x10]\n"
- "ldr q18, [%[c_ptr0], #0x20]\n"
- "ldr q19, [%[c_ptr0], #0x30]\n"
- "ldr q20, [c_ptr1]\n"
- "ldr q21, [c_ptr1, #0x10]\n"
- "ldr q22, [c_ptr1, #0x20]\n"
- "ldr q23, [c_ptr1, #0x30]\n"
- "ldr q0, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ldr q1, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "ldr q8, [%[b_ptr0]]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "cbz %[loops], 2f\n"
- "3:\n"
- "fmla v16.4s, v8.4s, v0.s[0]\n"
- "ins v14.d[1], temploadreg2\n"
- "fmla v20.4s, v8.4s, v1.s[0]\n"
- "ldr d15, [%[b_ptr0], #-0x10]\n"
- "fmla v17.4s, v9.4s, v0.s[0]\n"
- "ldr temploadreg3, [%[b_ptr0], #-0x8]\n"
- "fmla v21.4s, v9.4s, v1.s[0]\n"
- "ldr d4, [%[a_ptr0]]\n"
- "fmla v18.4s, v10.4s, v0.s[0]\n"
- "ldr temploadreg0, [%[a_ptr0], #0x8]\n"
- "fmla v22.4s, v10.4s, v1.s[0]\n"
- "ldr d5, [a_ptr1]\n"
- "fmla v19.4s, v11.4s, v0.s[0]\n"
- "ldr temploadreg1, [a_ptr1, #0x8]\n"
- "fmla v23.4s, v11.4s, v1.s[0]\n"
- "ldr d8, [%[b_ptr0]]\n"
- "fmla v16.4s, v12.4s, v0.s[1]\n"
- "ins v4.d[1], temploadreg0\n"
- "fmla v20.4s, v12.4s, v1.s[1]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- "fmla v17.4s, v13.4s, v0.s[1]\n"
- "ldr d9, [%[b_ptr0], #0x10]\n"
- "fmla v21.4s, v13.4s, v1.s[1]\n"
- "ins v5.d[1], temploadreg1\n"
- "fmla v18.4s, v14.4s, v0.s[1]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- "fmla v22.4s, v14.4s, v1.s[1]\n"
- "ldr d10, [%[b_ptr0], #0x20]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ldr d11, [%[b_ptr0], #0x30]\n"
- "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n"
- "ins v15.d[1], temploadreg3\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
- "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- "add a_ptr1, a_ptr1, #0x20\n"
- "fmla v19.4s, v15.4s, v0.s[1]\n"
- "ldr d12, [%[b_ptr0], #0x40]\n"
- "fmla v23.4s, v15.4s, v1.s[1]\n"
- "ins v8.d[1], temploadreg0\n"
- "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- "prfm PLDL1KEEP, [a_ptr1, #0x40]\n"
- "ldr d13, [%[b_ptr0], #0x50]\n"
- "fmla v16.4s, v8.4s, v0.s[2]\n"
- "ins v9.d[1], temploadreg1\n"
- "fmla v20.4s, v8.4s, v1.s[2]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- "ins v10.d[1], temploadreg2\n"
- "fmla v17.4s, v9.4s, v0.s[2]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- "fmla v21.4s, v9.4s, v1.s[2]\n"
- "ldr d15, [%[b_ptr0], #0x70]\n"
- "ins v11.d[1], temploadreg3\n"
- "fmla v18.4s, v10.4s, v0.s[2]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- "fmla v22.4s, v10.4s, v1.s[2]\n"
- "ins v12.d[1], temploadreg0\n"
- "ins v13.d[1], temploadreg1\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- "fmla v19.4s, v11.4s, v0.s[2]\n"
- "ldr d8, [%[b_ptr0], #-0x80]\n"
- "fmla v23.4s, v11.4s, v1.s[2]\n"
- "ldr temploadreg0, [%[b_ptr0], #-0x78]\n"
- "fmla v16.4s, v12.4s, v0.s[3]\n"
- "ldr d9, [%[b_ptr0], #-0x70]\n"
- "fmla v20.4s, v12.4s, v1.s[3]\n"
- "ldr temploadreg1, [%[b_ptr0], #-0x68]\n"
- "fmla v17.4s, v13.4s, v0.s[3]\n"
- "ldr d10, [%[b_ptr0], #-0x60]\n"
- "fmla v21.4s, v13.4s, v1.s[3]\n"
- "ins v14.d[1], temploadreg2\n"
- "ldr temploadreg2, [%[b_ptr0], #-0x58]\n"
- "ldr d11, [%[b_ptr0], #-0x50]\n"
- "ins v15.d[1], temploadreg3\n"
- "fmla v18.4s, v14.4s, v0.s[3]\n"
- "ldr temploadreg3, [%[b_ptr0], #-0x48]\n"
- "fmla v22.4s, v14.4s, v1.s[3]\n"
- "ldr d12, [%[b_ptr0], #-0x40]\n"
- "ins v8.d[1], temploadreg0\n"
- "fmla v19.4s, v15.4s, v0.s[3]\n"
- "ldr temploadreg0, [%[b_ptr0], #-0x38]\n"
- "fmla v23.4s, v15.4s, v1.s[3]\n"
- "ldr d13, [%[b_ptr0], #-0x30]\n"
- "ins v9.d[1], temploadreg1\n"
- "fmla v16.4s, v8.4s, v4.s[0]\n"
- "ldr temploadreg1, [%[b_ptr0], #-0x28]\n"
- "fmla v20.4s, v8.4s, v5.s[0]\n"
- "ldr d14, [%[b_ptr0], #-0x20]\n"
- "ins v10.d[1], temploadreg2\n"
- "fmla v17.4s, v9.4s, v4.s[0]\n"
- "ldr temploadreg2, [%[b_ptr0], #-0x18]\n"
- "fmla v21.4s, v9.4s, v5.s[0]\n"
- "ldr d15, [%[b_ptr0], #-0x10]\n"
- "ins v11.d[1], temploadreg3\n"
- "fmla v18.4s, v10.4s, v4.s[0]\n"
- "ldr temploadreg3, [%[b_ptr0], #-0x8]\n"
- "fmla v22.4s, v10.4s, v5.s[0]\n"
- "ldr d0, [%[a_ptr0], #-0x10]\n"
- "ins v12.d[1], temploadreg0\n"
- "fmla v19.4s, v11.4s, v4.s[0]\n"
- "ldr temploadreg0, [%[a_ptr0], #-0x8]\n"
- "fmla v23.4s, v11.4s, v5.s[0]\n"
- "ldr d1, [a_ptr1, #-0x10]\n"
- "ins v13.d[1], temploadreg1\n"
- "fmla v16.4s, v12.4s, v4.s[1]\n"
- "ldr temploadreg1, [a_ptr1, #-0x8]\n"
- "fmla v20.4s, v12.4s, v5.s[1]\n"
- "ldr d8, [%[b_ptr0]]\n"
- "ins v0.d[1], temploadreg0\n"
- "fmla v17.4s, v13.4s, v4.s[1]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- "fmla v21.4s, v13.4s, v5.s[1]\n"
- "ldr d9, [%[b_ptr0], #0x10]\n"
- "ins v1.d[1], temploadreg1\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- "ldr d10, [%[b_ptr0], #0x20]\n"
- "ins v14.d[1], temploadreg2\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- "ldr d11, [%[b_ptr0], #0x30]\n"
- "ins v15.d[1], temploadreg3\n"
- "fmla v18.4s, v14.4s, v4.s[1]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- "fmla v22.4s, v14.4s, v5.s[1]\n"
- "ldr d12, [%[b_ptr0], #0x40]\n"
- "ins v8.d[1], temploadreg0\n"
- "fmla v19.4s, v15.4s, v4.s[1]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- "fmla v23.4s, v15.4s, v5.s[1]\n"
- "ldr d13, [%[b_ptr0], #0x50]\n"
- "ins v9.d[1], temploadreg1\n"
- "fmla v16.4s, v8.4s, v4.s[2]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- "fmla v20.4s, v8.4s, v5.s[2]\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- "ins v10.d[1], temploadreg2\n"
- "fmla v17.4s, v9.4s, v4.s[2]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- "fmla v21.4s, v9.4s, v5.s[2]\n"
- "ldr d15, [%[b_ptr0], #0x70]\n"
- "ins v11.d[1], temploadreg3\n"
- "fmla v18.4s, v10.4s, v4.s[2]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- "fmla v22.4s, v10.4s, v5.s[2]\n"
- "ins v12.d[1], temploadreg0\n"
- "ins v13.d[1], temploadreg1\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- "fmla v19.4s, v11.4s, v4.s[2]\n"
- "ldr d8, [%[b_ptr0], #-0x80]\n"
- "fmla v23.4s, v11.4s, v5.s[2]\n"
- "ldr temploadreg0, [%[b_ptr0], #-0x78]\n"
- "fmla v16.4s, v12.4s, v4.s[3]\n"
- "ldr d9, [%[b_ptr0], #-0x70]\n"
- "fmla v20.4s, v12.4s, v5.s[3]\n"
- "ldr temploadreg1, [%[b_ptr0], #-0x68]\n"
- "fmla v17.4s, v13.4s, v4.s[3]\n"
- "ldr d10, [%[b_ptr0], #-0x60]\n"
- "fmla v21.4s, v13.4s, v5.s[3]\n"
- "ins v14.d[1], temploadreg2\n"
- "ldr temploadreg2, [%[b_ptr0], #-0x58]\n"
- "ldr d11, [%[b_ptr0], #-0x50]\n"
- "ins v15.d[1], temploadreg3\n"
- "fmla v18.4s, v14.4s, v4.s[3]\n"
- "ldr temploadreg3, [%[b_ptr0], #-0x48]\n"
- "fmla v22.4s, v14.4s, v5.s[3]\n"
- "ldr d12, [%[b_ptr0], #-0x40]\n"
- "ins v8.d[1], temploadreg0\n"
- "fmla v19.4s, v15.4s, v4.s[3]\n"
- "ldr temploadreg0, [%[b_ptr0], #-0x38]\n"
- "fmla v23.4s, v15.4s, v5.s[3]\n"
- "ldr d13, [%[b_ptr0], #-0x30]\n"
- "ins v9.d[1], temploadreg1\n"
- "ldr temploadreg1, [%[b_ptr0], #-0x28]\n"
- "ldr d14, [%[b_ptr0], #-0x20]\n"
- "ins v10.d[1], temploadreg2\n"
- "ldr temploadreg2, [%[b_ptr0], #-0x18]\n"
- "ins v11.d[1], temploadreg3\n"
- "ins v12.d[1], temploadreg0\n"
- "ins v13.d[1], temploadreg1\n"
- "b.ne 3b\n"
- "2:\n"
- "ins v14.d[1], temploadreg2\n"
- "prfm PSTL1KEEP, [%[c_ptr0]]\n"
- "ldr d15, [%[b_ptr0], #-0x10]\n"
- "prfm PSTL1KEEP, [c_ptr1]\n"
- "ldr temploadreg3, [%[b_ptr0], #-0x8]\n"
- "ins v15.d[1], temploadreg3\n"
- "cbz %[regs], 4f\n"
- "fmla v16.4s, v8.4s, v0.s[0]\n"
- "ldr d4, [%[a_ptr0]]\n"
- "fmla v20.4s, v8.4s, v1.s[0]\n"
- "ldr temploadreg0, [%[a_ptr0], #0x8]\n"
- "fmla v17.4s, v9.4s, v0.s[0]\n"
- "ldr d5, [a_ptr1]\n"
- "fmla v21.4s, v9.4s, v1.s[0]\n"
- "ldr temploadreg1, [a_ptr1, #0x8]\n"
- "fmla v18.4s, v10.4s, v0.s[0]\n"
- "ldr d8, [%[b_ptr0]]\n"
- "fmla v22.4s, v10.4s, v1.s[0]\n"
- "ins v4.d[1], temploadreg0\n"
- "fmla v19.4s, v11.4s, v0.s[0]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- "fmla v23.4s, v11.4s, v1.s[0]\n"
- "ldr d9, [%[b_ptr0], #0x10]\n"
- "fmla v16.4s, v12.4s, v0.s[1]\n"
- "ins v5.d[1], temploadreg1\n"
- "fmla v20.4s, v12.4s, v1.s[1]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- "fmla v17.4s, v13.4s, v0.s[1]\n"
- "ldr d10, [%[b_ptr0], #0x20]\n"
- "fmla v21.4s, v13.4s, v1.s[1]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- "fmla v18.4s, v14.4s, v0.s[1]\n"
- "ldr d11, [%[b_ptr0], #0x30]\n"
- "fmla v22.4s, v14.4s, v1.s[1]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- "fmla v19.4s, v15.4s, v0.s[1]\n"
- "ldr d12, [%[b_ptr0], #0x40]\n"
- "fmla v23.4s, v15.4s, v1.s[1]\n"
- "ins v8.d[1], temploadreg0\n"
- "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ldr d13, [%[b_ptr0], #0x50]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "fmla v16.4s, v8.4s, v0.s[2]\n"
- "ins v9.d[1], temploadreg1\n"
- "fmla v20.4s, v8.4s, v1.s[2]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- "ins v10.d[1], temploadreg2\n"
- "fmla v17.4s, v9.4s, v0.s[2]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- "fmla v21.4s, v9.4s, v1.s[2]\n"
- "ldr d15, [%[b_ptr0], #0x70]\n"
- "ins v11.d[1], temploadreg3\n"
- "fmla v18.4s, v10.4s, v0.s[2]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- "fmla v22.4s, v10.4s, v1.s[2]\n"
- "ins v12.d[1], temploadreg0\n"
- "ins v13.d[1], temploadreg1\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- "fmla v19.4s, v11.4s, v0.s[2]\n"
- "ldr d8, [%[b_ptr0], #-0x80]\n"
- "fmla v23.4s, v11.4s, v1.s[2]\n"
- "ldr temploadreg0, [%[b_ptr0], #-0x78]\n"
- "fmla v16.4s, v12.4s, v0.s[3]\n"
- "ldr d9, [%[b_ptr0], #-0x70]\n"
- "fmla v20.4s, v12.4s, v1.s[3]\n"
- "ldr temploadreg1, [%[b_ptr0], #-0x68]\n"
- "fmla v17.4s, v13.4s, v0.s[3]\n"
- "ldr d10, [%[b_ptr0], #-0x60]\n"
- "fmla v21.4s, v13.4s, v1.s[3]\n"
- "ins v14.d[1], temploadreg2\n"
- "ldr temploadreg2, [%[b_ptr0], #-0x58]\n"
- "ldr d11, [%[b_ptr0], #-0x50]\n"
- "ins v15.d[1], temploadreg3\n"
- "fmla v18.4s, v14.4s, v0.s[3]\n"
- "ldr temploadreg3, [%[b_ptr0], #-0x48]\n"
- "fmla v22.4s, v14.4s, v1.s[3]\n"
- "ldr d12, [%[b_ptr0], #-0x40]\n"
- "ins v8.d[1], temploadreg0\n"
- "fmla v19.4s, v15.4s, v0.s[3]\n"
- "ldr temploadreg0, [%[b_ptr0], #-0x38]\n"
- "fmla v23.4s, v15.4s, v1.s[3]\n"
- "ldr d13, [%[b_ptr0], #-0x30]\n"
- "ins v9.d[1], temploadreg1\n"
- "fmla v16.4s, v8.4s, v4.s[0]\n"
- "ldr temploadreg1, [%[b_ptr0], #-0x28]\n"
- "fmla v20.4s, v8.4s, v5.s[0]\n"
- "ldr d14, [%[b_ptr0], #-0x20]\n"
- "ins v10.d[1], temploadreg2\n"
- "fmla v17.4s, v9.4s, v4.s[0]\n"
- "ldr temploadreg2, [%[b_ptr0], #-0x18]\n"
- "fmla v21.4s, v9.4s, v5.s[0]\n"
- "ldr d15, [%[b_ptr0], #-0x10]\n"
- "ins v11.d[1], temploadreg3\n"
- "fmla v18.4s, v10.4s, v4.s[0]\n"
- "ldr temploadreg3, [%[b_ptr0], #-0x8]\n"
- "fmla v22.4s, v10.4s, v5.s[0]\n"
- "ldr d8, [%[b_ptr0]]\n"
- "ins v12.d[1], temploadreg0\n"
- "fmla v19.4s, v11.4s, v4.s[0]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- "fmla v23.4s, v11.4s, v5.s[0]\n"
- "ldr d9, [%[b_ptr0], #0x10]\n"
- "ins v13.d[1], temploadreg1\n"
- "fmla v16.4s, v12.4s, v4.s[1]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- "fmla v20.4s, v12.4s, v5.s[1]\n"
- "ldr d10, [%[b_ptr0], #0x20]\n"
- "ins v14.d[1], temploadreg2\n"
- "fmla v17.4s, v13.4s, v4.s[1]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- "fmla v21.4s, v13.4s, v5.s[1]\n"
- "ldr d11, [%[b_ptr0], #0x30]\n"
- "ins v15.d[1], temploadreg3\n"
- "fmla v18.4s, v14.4s, v4.s[1]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- "fmla v22.4s, v14.4s, v5.s[1]\n"
- "ldr d12, [%[b_ptr0], #0x40]\n"
- "ins v8.d[1], temploadreg0\n"
- "fmla v19.4s, v15.4s, v4.s[1]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- "fmla v23.4s, v15.4s, v5.s[1]\n"
- "ldr d13, [%[b_ptr0], #0x50]\n"
- "ins v9.d[1], temploadreg1\n"
- "fmla v16.4s, v8.4s, v4.s[2]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- "fmla v20.4s, v8.4s, v5.s[2]\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- "ins v10.d[1], temploadreg2\n"
- "fmla v17.4s, v9.4s, v4.s[2]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- "fmla v21.4s, v9.4s, v5.s[2]\n"
- "ldr d15, [%[b_ptr0], #0x70]\n"
- "ins v11.d[1], temploadreg3\n"
- "fmla v18.4s, v10.4s, v4.s[2]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- "fmla v22.4s, v10.4s, v5.s[2]\n"
- "ins v12.d[1], temploadreg0\n"
- "ins v13.d[1], temploadreg1\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "fmla v19.4s, v11.4s, v4.s[2]\n"
- "ins v14.d[1], temploadreg2\n"
- "fmla v23.4s, v11.4s, v5.s[2]\n"
- "ins v15.d[1], temploadreg3\n"
- "fmla v16.4s, v12.4s, v4.s[3]\n"
- "fmla v20.4s, v12.4s, v5.s[3]\n"
- "fmla v17.4s, v13.4s, v4.s[3]\n"
- "fmla v21.4s, v13.4s, v5.s[3]\n"
- "fmla v18.4s, v14.4s, v4.s[3]\n"
- "fmla v22.4s, v14.4s, v5.s[3]\n"
- "fmla v19.4s, v15.4s, v4.s[3]\n"
- "fmla v23.4s, v15.4s, v5.s[3]\n"
- "b 5f\n"
- "4:\n"
- "fmla v16.4s, v8.4s, v0.s[0]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- "fmla v20.4s, v8.4s, v1.s[0]\n"
- "ldr d8, [%[b_ptr0]]\n"
- "fmla v17.4s, v9.4s, v0.s[0]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- "fmla v21.4s, v9.4s, v1.s[0]\n"
- "ldr d9, [%[b_ptr0], #0x10]\n"
- "fmla v18.4s, v10.4s, v0.s[0]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- "fmla v22.4s, v10.4s, v1.s[0]\n"
- "ldr d10, [%[b_ptr0], #0x20]\n"
- "fmla v19.4s, v11.4s, v0.s[0]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- "fmla v23.4s, v11.4s, v1.s[0]\n"
- "ldr d11, [%[b_ptr0], #0x30]\n"
- "fmla v16.4s, v12.4s, v0.s[1]\n"
- "ins v8.d[1], temploadreg0\n"
- "fmla v20.4s, v12.4s, v1.s[1]\n"
- "ldr d12, [%[b_ptr0], #0x40]\n"
- "fmla v17.4s, v13.4s, v0.s[1]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- "fmla v21.4s, v13.4s, v1.s[1]\n"
- "ldr d13, [%[b_ptr0], #0x50]\n"
- "fmla v18.4s, v14.4s, v0.s[1]\n"
- "ins v9.d[1], temploadreg1\n"
- "fmla v22.4s, v14.4s, v1.s[1]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- "fmla v19.4s, v15.4s, v0.s[1]\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- "fmla v23.4s, v15.4s, v1.s[1]\n"
- "ins v10.d[1], temploadreg2\n"
- "fmla v16.4s, v8.4s, v0.s[2]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- "fmla v20.4s, v8.4s, v1.s[2]\n"
- "ldr d15, [%[b_ptr0], #0x70]\n"
- "fmla v17.4s, v9.4s, v0.s[2]\n"
- "ins v11.d[1], temploadreg3\n"
- "fmla v21.4s, v9.4s, v1.s[2]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- "fmla v18.4s, v10.4s, v0.s[2]\n"
- "ins v12.d[1], temploadreg0\n"
- "fmla v22.4s, v10.4s, v1.s[2]\n"
- "ins v13.d[1], temploadreg1\n"
- "fmla v19.4s, v11.4s, v0.s[2]\n"
- "ins v14.d[1], temploadreg2\n"
- "fmla v23.4s, v11.4s, v1.s[2]\n"
- "ins v15.d[1], temploadreg3\n"
- "fmla v16.4s, v12.4s, v0.s[3]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "fmla v20.4s, v12.4s, v1.s[3]\n"
- "fmla v17.4s, v13.4s, v0.s[3]\n"
- "fmla v21.4s, v13.4s, v1.s[3]\n"
- "fmla v18.4s, v14.4s, v0.s[3]\n"
- "fmla v22.4s, v14.4s, v1.s[3]\n"
- "fmla v19.4s, v15.4s, v0.s[3]\n"
- "fmla v23.4s, v15.4s, v1.s[3]\n"
- "5:\n"
- "cbz %[blocks], 6f\n"
- "7:\n"
- "ldr q8, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "ldr s0, [%[a_ptr0]]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "add %[a_ptr0], %[a_ptr0], #0x4\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "add %[b_ptr0], %[b_ptr0], #0x40\n"
- "fmla v16.4s, v8.4s, v0.s[0]\n"
- "ldr s1, [a_ptr1]\n"
- "fmla v17.4s, v9.4s, v0.s[0]\n"
- "add a_ptr1, a_ptr1, #0x4\n"
- "fmla v18.4s, v10.4s, v0.s[0]\n"
- "fmla v20.4s, v8.4s, v1.s[0]\n"
- "fmla v21.4s, v9.4s, v1.s[0]\n"
- "fmla v22.4s, v10.4s, v1.s[0]\n"
- "fmla v19.4s, v11.4s, v0.s[0]\n"
- "fmla v23.4s, v11.4s, v1.s[0]\n"
- "b.ne 7b\n"
- "6:\n"
- "ld1r {v14.4s}, [%[minptr]]\n"
- "ld1r {v15.4s}, [%[maxptr]]\n"
- "fmax v16.4s, v16.4s, v14.4s\n"
- "fmax v17.4s, v17.4s, v14.4s\n"
- "fmax v18.4s, v18.4s, v14.4s\n"
- "fmax v19.4s, v19.4s, v14.4s\n"
- "fmin v16.4s, v16.4s, v15.4s\n"
- "fmin v17.4s, v17.4s, v15.4s\n"
- "fmin v18.4s, v18.4s, v15.4s\n"
- "fmin v19.4s, v19.4s, v15.4s\n"
- "str q16, [%[c_ptr0]]\n"
- "fmax v20.4s, v20.4s, v14.4s\n"
- "fmax v21.4s, v21.4s, v14.4s\n"
- "fmax v22.4s, v22.4s, v14.4s\n"
- "str q17, [%[c_ptr0], #0x10]\n"
- "fmax v23.4s, v23.4s, v14.4s\n"
- "fmin v20.4s, v20.4s, v15.4s\n"
- "fmin v21.4s, v21.4s, v15.4s\n"
- "str q18, [%[c_ptr0], #0x20]\n"
- "fmin v22.4s, v22.4s, v15.4s\n"
- "fmin v23.4s, v23.4s, v15.4s\n"
- "str q19, [%[c_ptr0], #0x30]\n"
- "add %[c_ptr0], %[c_ptr0], #0x40\n"
- "str q20, [c_ptr1]\n"
- "str q21, [c_ptr1, #0x10]\n"
- "str q22, [c_ptr1, #0x20]\n"
- "str q23, [c_ptr1, #0x30]\n"
- ".unreq a_ptr1\n"
- ".unreq c_ptr1\n"
- ".unreq temploadreg0\n"
- ".unreq temploadreg1\n"
- ".unreq temploadreg2\n"
- ".unreq temploadreg3\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks)
- : [width] "r" (width), [accumulate] "r" (static_cast<uint64_t>(accumulate)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr)
- : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "x4", "x5", "cc", "memory"
- );
- break;
- case 3:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "c_ptr1 .req X2\n"
- "c_ptr2 .req X3\n"
- "temploadreg0 .req X4\n"
- "temploadreg1 .req X5\n"
- "temploadreg2 .req X6\n"
- "temploadreg3 .req X7\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "cbnz %[accumulate], 1f\n"
- "ldr q16, [%[biasptr]]\n"
- "ldr q17, [%[biasptr], #0x10]\n"
- "ldr q18, [%[biasptr], #0x20]\n"
- "ldr q19, [%[biasptr], #0x30]\n"
- "mov v20.16b, v16.16b\n"
- "ldr q0, [%[a_ptr0]]\n"
- "mov v21.16b, v17.16b\n"
- "ldr q1, [a_ptr1]\n"
- "mov v22.16b, v18.16b\n"
- "ldr q2, [a_ptr2]\n"
- "mov v23.16b, v19.16b\n"
- "ldr q8, [%[b_ptr0]]\n"
- "mov v24.16b, v16.16b\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "mov v25.16b, v17.16b\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "mov v26.16b, v18.16b\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "mov v27.16b, v19.16b\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "ldr q16, [%[c_ptr0]]\n"
- "ldr q17, [%[c_ptr0], #0x10]\n"
- "ldr q18, [%[c_ptr0], #0x20]\n"
- "ldr q19, [%[c_ptr0], #0x30]\n"
- "ldr q20, [c_ptr1]\n"
- "ldr q21, [c_ptr1, #0x10]\n"
- "ldr q22, [c_ptr1, #0x20]\n"
- "ldr q23, [c_ptr1, #0x30]\n"
- "ldr q24, [c_ptr2]\n"
- "ldr q25, [c_ptr2, #0x10]\n"
- "ldr q26, [c_ptr2, #0x20]\n"
- "ldr q27, [c_ptr2, #0x30]\n"
- "ldr q0, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ldr q1, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "ldr q2, [a_ptr2]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "ldr q8, [%[b_ptr0]]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "cbz %[loops], 2f\n"
- "3:\n"
- "fmla v16.4s, v8.4s, v0.s[0]\n"
- "ins v14.d[1], temploadreg2\n"
- "fmla v20.4s, v8.4s, v1.s[0]\n"
- "ldr d15, [%[b_ptr0], #-0x10]\n"
- "fmla v24.4s, v8.4s, v2.s[0]\n"
- "ldr temploadreg3, [%[b_ptr0], #-0x8]\n"
- "fmla v17.4s, v9.4s, v0.s[0]\n"
- "ldr d4, [%[a_ptr0]]\n"
- "fmla v21.4s, v9.4s, v1.s[0]\n"
- "ldr temploadreg0, [%[a_ptr0], #0x8]\n"
- "fmla v25.4s, v9.4s, v2.s[0]\n"
- "ldr d5, [a_ptr1]\n"
- "fmla v18.4s, v10.4s, v0.s[0]\n"
- "ldr temploadreg1, [a_ptr1, #0x8]\n"
- "fmla v22.4s, v10.4s, v1.s[0]\n"
- "ldr d6, [a_ptr2]\n"
- "fmla v26.4s, v10.4s, v2.s[0]\n"
- "ldr temploadreg2, [a_ptr2, #0x8]\n"
- "fmla v19.4s, v11.4s, v0.s[0]\n"
- "ldr d8, [%[b_ptr0]]\n"
- "fmla v23.4s, v11.4s, v1.s[0]\n"
- "ins v4.d[1], temploadreg0\n"
- "fmla v27.4s, v11.4s, v2.s[0]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- "fmla v16.4s, v12.4s, v0.s[1]\n"
- "ldr d9, [%[b_ptr0], #0x10]\n"
- "fmla v20.4s, v12.4s, v1.s[1]\n"
- "ins v5.d[1], temploadreg1\n"
- "fmla v24.4s, v12.4s, v2.s[1]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- "fmla v17.4s, v13.4s, v0.s[1]\n"
- "ldr d10, [%[b_ptr0], #0x20]\n"
- "fmla v21.4s, v13.4s, v1.s[1]\n"
- "ins v6.d[1], temploadreg2\n"
- "fmla v25.4s, v13.4s, v2.s[1]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- "fmla v18.4s, v14.4s, v0.s[1]\n"
- "ldr d11, [%[b_ptr0], #0x30]\n"
- "fmla v22.4s, v14.4s, v1.s[1]\n"
- "ins v15.d[1], temploadreg3\n"
- "fmla v26.4s, v14.4s, v2.s[1]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- "ldr d12, [%[b_ptr0], #0x40]\n"
- "subs %[loops], %[loops], #0x1\n"
- "fmla v19.4s, v15.4s, v0.s[1]\n"
- "ins v8.d[1], temploadreg0\n"
- "fmla v23.4s, v15.4s, v1.s[1]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- "fmla v27.4s, v15.4s, v2.s[1]\n"
- "ldr d13, [%[b_ptr0], #0x50]\n"
- "ins v9.d[1], temploadreg1\n"
- "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n"
- "fmla v16.4s, v8.4s, v0.s[2]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- "fmla v20.4s, v8.4s, v1.s[2]\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- "fmla v24.4s, v8.4s, v2.s[2]\n"
- "ins v10.d[1], temploadreg2\n"
- "fmla v17.4s, v9.4s, v0.s[2]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- "fmla v21.4s, v9.4s, v1.s[2]\n"
- "ldr d15, [%[b_ptr0], #0x70]\n"
- "fmla v25.4s, v9.4s, v2.s[2]\n"
- "ins v11.d[1], temploadreg3\n"
- "fmla v18.4s, v10.4s, v0.s[2]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- "fmla v22.4s, v10.4s, v1.s[2]\n"
- "ins v12.d[1], temploadreg0\n"
- "fmla v26.4s, v10.4s, v2.s[2]\n"
- "ins v13.d[1], temploadreg1\n"
- "fmla v19.4s, v11.4s, v0.s[2]\n"
- "ins v14.d[1], temploadreg2\n"
- "fmla v23.4s, v11.4s, v1.s[2]\n"
- "ins v15.d[1], temploadreg3\n"
- "fmla v27.4s, v11.4s, v2.s[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- "fmla v16.4s, v12.4s, v0.s[3]\n"
- "ldr d8, [%[b_ptr0], #-0x80]\n"
- "fmla v20.4s, v12.4s, v1.s[3]\n"
- "ldr temploadreg0, [%[b_ptr0], #-0x78]\n"
- "fmla v24.4s, v12.4s, v2.s[3]\n"
- "ldr d9, [%[b_ptr0], #-0x70]\n"
- "fmla v17.4s, v13.4s, v0.s[3]\n"
- "ldr temploadreg1, [%[b_ptr0], #-0x68]\n"
- "fmla v21.4s, v13.4s, v1.s[3]\n"
- "ldr d10, [%[b_ptr0], #-0x60]\n"
- "fmla v25.4s, v13.4s, v2.s[3]\n"
- "ldr temploadreg2, [%[b_ptr0], #-0x58]\n"
- "fmla v18.4s, v14.4s, v0.s[3]\n"
- "ldr d11, [%[b_ptr0], #-0x50]\n"
- "fmla v22.4s, v14.4s, v1.s[3]\n"
- "ldr temploadreg3, [%[b_ptr0], #-0x48]\n"
- "fmla v26.4s, v14.4s, v2.s[3]\n"
- "ldr d12, [%[b_ptr0], #-0x40]\n"
- "fmla v19.4s, v15.4s, v0.s[3]\n"
- "ins v8.d[1], temploadreg0\n"
- "fmla v23.4s, v15.4s, v1.s[3]\n"
- "ldr temploadreg0, [%[b_ptr0], #-0x38]\n"
- "fmla v27.4s, v15.4s, v2.s[3]\n"
- "ldr d13, [%[b_ptr0], #-0x30]\n"
- "ins v9.d[1], temploadreg1\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
- "fmla v16.4s, v8.4s, v4.s[0]\n"
- "ldr temploadreg1, [%[b_ptr0], #-0x28]\n"
- "fmla v20.4s, v8.4s, v5.s[0]\n"
- "ldr d14, [%[b_ptr0], #-0x20]\n"
- "fmla v24.4s, v8.4s, v6.s[0]\n"
- "ins v10.d[1], temploadreg2\n"
- "fmla v17.4s, v9.4s, v4.s[0]\n"
- "ldr temploadreg2, [%[b_ptr0], #-0x18]\n"
- "fmla v21.4s, v9.4s, v5.s[0]\n"
- "ldr d15, [%[b_ptr0], #-0x10]\n"
- "fmla v25.4s, v9.4s, v6.s[0]\n"
- "ins v11.d[1], temploadreg3\n"
- "fmla v18.4s, v10.4s, v4.s[0]\n"
- "ldr temploadreg3, [%[b_ptr0], #-0x8]\n"
- "fmla v22.4s, v10.4s, v5.s[0]\n"
- "ldr d0, [%[a_ptr0], #-0x10]\n"
- "fmla v26.4s, v10.4s, v6.s[0]\n"
- "ins v12.d[1], temploadreg0\n"
- "fmla v19.4s, v11.4s, v4.s[0]\n"
- "ldr temploadreg0, [%[a_ptr0], #-0x8]\n"
- "fmla v23.4s, v11.4s, v5.s[0]\n"
- "ins v13.d[1], temploadreg1\n"
- "fmla v27.4s, v11.4s, v6.s[0]\n"
- "ins v14.d[1], temploadreg2\n"
- "fmla v16.4s, v12.4s, v4.s[1]\n"
- "ldr d8, [%[b_ptr0]]\n"
- "fmla v20.4s, v12.4s, v5.s[1]\n"
- "ins v0.d[1], temploadreg0\n"
- "fmla v24.4s, v12.4s, v6.s[1]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- "fmla v17.4s, v13.4s, v4.s[1]\n"
- "ldr d9, [%[b_ptr0], #0x10]\n"
- "fmla v21.4s, v13.4s, v5.s[1]\n"
- "ldr d10, [%[b_ptr0], #0x20]\n"
- "fmla v25.4s, v13.4s, v6.s[1]\n"
- "ldr d11, [%[b_ptr0], #0x30]\n"
- "fmla v18.4s, v14.4s, v4.s[1]\n"
- "ins v15.d[1], temploadreg3\n"
- "fmla v22.4s, v14.4s, v5.s[1]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- "fmla v26.4s, v14.4s, v6.s[1]\n"
- "ldr d12, [%[b_ptr0], #0x40]\n"
- "ins v8.d[1], temploadreg0\n"
- "add a_ptr1, a_ptr1, #0x20\n"
- "fmla v19.4s, v15.4s, v4.s[1]\n"
- "ldr d1, [a_ptr1, #-0x10]\n"
- "fmla v23.4s, v15.4s, v5.s[1]\n"
- "ldr temploadreg1, [a_ptr1, #-0x8]\n"
- "fmla v27.4s, v15.4s, v6.s[1]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- "fmla v16.4s, v8.4s, v4.s[2]\n"
- "ldr d13, [%[b_ptr0], #0x50]\n"
- "fmla v20.4s, v8.4s, v5.s[2]\n"
- "ins v1.d[1], temploadreg1\n"
- "fmla v24.4s, v8.4s, v6.s[2]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- "add a_ptr2, a_ptr2, #0x20\n"
- "ldr d15, [%[b_ptr0], #0x70]\n"
- "prfm PLDL1KEEP, [a_ptr1, #0x40]\n"
- "ldr d2, [a_ptr2, #-0x10]\n"
- "prfm PLDL1KEEP, [a_ptr2, #0x40]\n"
- "ldr temploadreg2, [a_ptr2, #-0x8]\n"
- "ins v9.d[1], temploadreg1\n"
- "ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- "ins v11.d[1], temploadreg3\n"
- "ins v2.d[1], temploadreg2\n"
- "fmla v17.4s, v9.4s, v4.s[2]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- "fmla v21.4s, v9.4s, v5.s[2]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- "fmla v25.4s, v9.4s, v6.s[2]\n"
- "ins v12.d[1], temploadreg0\n"
- "fmla v19.4s, v11.4s, v4.s[2]\n"
- "ins v10.d[1], temploadreg2\n"
- "fmla v23.4s, v11.4s, v5.s[2]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- "fmla v27.4s, v11.4s, v6.s[2]\n"
- "ins v13.d[1], temploadreg1\n"
- "fmla v16.4s, v12.4s, v4.s[3]\n"
- "ins v15.d[1], temploadreg3\n"
- "fmla v18.4s, v10.4s, v4.s[2]\n"
- "ins v14.d[1], temploadreg2\n"
- "fmla v22.4s, v10.4s, v5.s[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- "fmla v26.4s, v10.4s, v6.s[2]\n"
- "ldr d8, [%[b_ptr0], #-0x80]\n"
- "fmla v20.4s, v12.4s, v5.s[3]\n"
- "ldr temploadreg0, [%[b_ptr0], #-0x78]\n"
- "fmla v24.4s, v12.4s, v6.s[3]\n"
- "ldr d9, [%[b_ptr0], #-0x70]\n"
- "fmla v17.4s, v13.4s, v4.s[3]\n"
- "ldr temploadreg1, [%[b_ptr0], #-0x68]\n"
- "fmla v21.4s, v13.4s, v5.s[3]\n"
- "ldr d10, [%[b_ptr0], #-0x60]\n"
- "fmla v25.4s, v13.4s, v6.s[3]\n"
- "ldr temploadreg2, [%[b_ptr0], #-0x58]\n"
- "fmla v18.4s, v14.4s, v4.s[3]\n"
- "ldr d11, [%[b_ptr0], #-0x50]\n"
- "fmla v22.4s, v14.4s, v5.s[3]\n"
- "ldr temploadreg3, [%[b_ptr0], #-0x48]\n"
- "fmla v26.4s, v14.4s, v6.s[3]\n"
- "ldr d12, [%[b_ptr0], #-0x40]\n"
- "fmla v19.4s, v15.4s, v4.s[3]\n"
- "ins v8.d[1], temploadreg0\n"
- "fmla v23.4s, v15.4s, v5.s[3]\n"
- "ldr temploadreg0, [%[b_ptr0], #-0x38]\n"
- "fmla v27.4s, v15.4s, v6.s[3]\n"
- "ldr d13, [%[b_ptr0], #-0x30]\n"
- "ins v9.d[1], temploadreg1\n"
- "ldr temploadreg1, [%[b_ptr0], #-0x28]\n"
- "ldr d14, [%[b_ptr0], #-0x20]\n"
- "ins v10.d[1], temploadreg2\n"
- "ldr temploadreg2, [%[b_ptr0], #-0x18]\n"
- "ins v11.d[1], temploadreg3\n"
- "ins v12.d[1], temploadreg0\n"
- "ins v13.d[1], temploadreg1\n"
- "b.ne 3b\n"
- "2:\n"
- "ins v14.d[1], temploadreg2\n"
- "prfm PSTL1KEEP, [%[c_ptr0]]\n"
- "ldr d15, [%[b_ptr0], #-0x10]\n"
- "prfm PSTL1KEEP, [c_ptr1]\n"
- "ldr temploadreg3, [%[b_ptr0], #-0x8]\n"
- "prfm PSTL1KEEP, [c_ptr2]\n"
- "ins v15.d[1], temploadreg3\n"
- "cbz %[regs], 4f\n"
- "fmla v16.4s, v8.4s, v0.s[0]\n"
- "ldr d4, [%[a_ptr0]]\n"
- "fmla v20.4s, v8.4s, v1.s[0]\n"
- "ldr temploadreg0, [%[a_ptr0], #0x8]\n"
- "fmla v24.4s, v8.4s, v2.s[0]\n"
- "ldr d5, [a_ptr1]\n"
- "fmla v17.4s, v9.4s, v0.s[0]\n"
- "ldr temploadreg1, [a_ptr1, #0x8]\n"
- "fmla v21.4s, v9.4s, v1.s[0]\n"
- "ldr d6, [a_ptr2]\n"
- "fmla v25.4s, v9.4s, v2.s[0]\n"
- "ldr temploadreg2, [a_ptr2, #0x8]\n"
- "fmla v18.4s, v10.4s, v0.s[0]\n"
- "ldr d8, [%[b_ptr0]]\n"
- "fmla v22.4s, v10.4s, v1.s[0]\n"
- "ins v4.d[1], temploadreg0\n"
- "fmla v26.4s, v10.4s, v2.s[0]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- "fmla v19.4s, v11.4s, v0.s[0]\n"
- "ldr d9, [%[b_ptr0], #0x10]\n"
- "fmla v23.4s, v11.4s, v1.s[0]\n"
- "ins v5.d[1], temploadreg1\n"
- "fmla v27.4s, v11.4s, v2.s[0]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- "fmla v16.4s, v12.4s, v0.s[1]\n"
- "ldr d10, [%[b_ptr0], #0x20]\n"
- "fmla v20.4s, v12.4s, v1.s[1]\n"
- "ins v6.d[1], temploadreg2\n"
- "fmla v24.4s, v12.4s, v2.s[1]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- "fmla v17.4s, v13.4s, v0.s[1]\n"
- "ldr d11, [%[b_ptr0], #0x30]\n"
- "fmla v21.4s, v13.4s, v1.s[1]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- "fmla v25.4s, v13.4s, v2.s[1]\n"
- "ldr d12, [%[b_ptr0], #0x40]\n"
- "fmla v18.4s, v14.4s, v0.s[1]\n"
- "ins v8.d[1], temploadreg0\n"
- "fmla v22.4s, v14.4s, v1.s[1]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- "fmla v26.4s, v14.4s, v2.s[1]\n"
- "ldr d13, [%[b_ptr0], #0x50]\n"
- "fmla v19.4s, v15.4s, v0.s[1]\n"
- "ins v9.d[1], temploadreg1\n"
- "fmla v23.4s, v15.4s, v1.s[1]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- "fmla v27.4s, v15.4s, v2.s[1]\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- "fmla v16.4s, v8.4s, v0.s[2]\n"
- "ins v10.d[1], temploadreg2\n"
- "fmla v20.4s, v8.4s, v1.s[2]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- "fmla v24.4s, v8.4s, v2.s[2]\n"
- "ldr d15, [%[b_ptr0], #0x70]\n"
- "fmla v17.4s, v9.4s, v0.s[2]\n"
- "ins v11.d[1], temploadreg3\n"
- "fmla v21.4s, v9.4s, v1.s[2]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- "fmla v25.4s, v9.4s, v2.s[2]\n"
- "ins v12.d[1], temploadreg0\n"
- "fmla v18.4s, v10.4s, v0.s[2]\n"
- "ins v13.d[1], temploadreg1\n"
- "fmla v22.4s, v10.4s, v1.s[2]\n"
- "ins v14.d[1], temploadreg2\n"
- "fmla v26.4s, v10.4s, v2.s[2]\n"
- "ins v15.d[1], temploadreg3\n"
- "fmla v19.4s, v11.4s, v0.s[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- "fmla v23.4s, v11.4s, v1.s[2]\n"
- "ldr d8, [%[b_ptr0], #-0x80]\n"
- "fmla v27.4s, v11.4s, v2.s[2]\n"
- "ldr temploadreg0, [%[b_ptr0], #-0x78]\n"
- "fmla v16.4s, v12.4s, v0.s[3]\n"
- "ldr d9, [%[b_ptr0], #-0x70]\n"
- "fmla v20.4s, v12.4s, v1.s[3]\n"
- "ldr temploadreg1, [%[b_ptr0], #-0x68]\n"
- "fmla v24.4s, v12.4s, v2.s[3]\n"
- "ldr d10, [%[b_ptr0], #-0x60]\n"
- "fmla v17.4s, v13.4s, v0.s[3]\n"
- "ldr temploadreg2, [%[b_ptr0], #-0x58]\n"
- "fmla v21.4s, v13.4s, v1.s[3]\n"
- "ldr d11, [%[b_ptr0], #-0x50]\n"
- "fmla v25.4s, v13.4s, v2.s[3]\n"
- "ldr temploadreg3, [%[b_ptr0], #-0x48]\n"
- "fmla v18.4s, v14.4s, v0.s[3]\n"
- "ldr d12, [%[b_ptr0], #-0x40]\n"
- "fmla v22.4s, v14.4s, v1.s[3]\n"
- "ins v8.d[1], temploadreg0\n"
- "fmla v26.4s, v14.4s, v2.s[3]\n"
- "ldr temploadreg0, [%[b_ptr0], #-0x38]\n"
- "fmla v19.4s, v15.4s, v0.s[3]\n"
- "ldr d13, [%[b_ptr0], #-0x30]\n"
- "fmla v23.4s, v15.4s, v1.s[3]\n"
- "ins v9.d[1], temploadreg1\n"
- "fmla v27.4s, v15.4s, v2.s[3]\n"
- "ldr temploadreg1, [%[b_ptr0], #-0x28]\n"
- "fmla v16.4s, v8.4s, v4.s[0]\n"
- "ldr d14, [%[b_ptr0], #-0x20]\n"
- "fmla v20.4s, v8.4s, v5.s[0]\n"
- "ins v10.d[1], temploadreg2\n"
- "fmla v24.4s, v8.4s, v6.s[0]\n"
- "ldr temploadreg2, [%[b_ptr0], #-0x18]\n"
- "fmla v17.4s, v9.4s, v4.s[0]\n"
- "ldr d15, [%[b_ptr0], #-0x10]\n"
- "fmla v21.4s, v9.4s, v5.s[0]\n"
- "ins v11.d[1], temploadreg3\n"
- "fmla v25.4s, v9.4s, v6.s[0]\n"
- "ldr temploadreg3, [%[b_ptr0], #-0x8]\n"
- "fmla v18.4s, v10.4s, v4.s[0]\n"
- "ldr d8, [%[b_ptr0]]\n"
- "fmla v22.4s, v10.4s, v5.s[0]\n"
- "ins v12.d[1], temploadreg0\n"
- "fmla v26.4s, v10.4s, v6.s[0]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- "fmla v19.4s, v11.4s, v4.s[0]\n"
- "ldr d9, [%[b_ptr0], #0x10]\n"
- "fmla v23.4s, v11.4s, v5.s[0]\n"
- "ins v13.d[1], temploadreg1\n"
- "fmla v27.4s, v11.4s, v6.s[0]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- "fmla v16.4s, v12.4s, v4.s[1]\n"
- "ldr d10, [%[b_ptr0], #0x20]\n"
- "fmla v20.4s, v12.4s, v5.s[1]\n"
- "ins v14.d[1], temploadreg2\n"
- "fmla v24.4s, v12.4s, v6.s[1]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- "fmla v17.4s, v13.4s, v4.s[1]\n"
- "ldr d11, [%[b_ptr0], #0x30]\n"
- "fmla v21.4s, v13.4s, v5.s[1]\n"
- "ins v15.d[1], temploadreg3\n"
- "fmla v25.4s, v13.4s, v6.s[1]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- "fmla v18.4s, v14.4s, v4.s[1]\n"
- "ldr d12, [%[b_ptr0], #0x40]\n"
- "fmla v22.4s, v14.4s, v5.s[1]\n"
- "ins v8.d[1], temploadreg0\n"
- "fmla v26.4s, v14.4s, v6.s[1]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- "fmla v19.4s, v15.4s, v4.s[1]\n"
- "ldr d13, [%[b_ptr0], #0x50]\n"
- "fmla v23.4s, v15.4s, v5.s[1]\n"
- "ins v9.d[1], temploadreg1\n"
- "fmla v27.4s, v15.4s, v6.s[1]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- "fmla v16.4s, v8.4s, v4.s[2]\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- "fmla v20.4s, v8.4s, v5.s[2]\n"
- "ins v10.d[1], temploadreg2\n"
- "fmla v24.4s, v8.4s, v6.s[2]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- "fmla v17.4s, v9.4s, v4.s[2]\n"
- "ldr d15, [%[b_ptr0], #0x70]\n"
- "fmla v21.4s, v9.4s, v5.s[2]\n"
- "ins v11.d[1], temploadreg3\n"
- "fmla v25.4s, v9.4s, v6.s[2]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- "fmla v18.4s, v10.4s, v4.s[2]\n"
- "ins v12.d[1], temploadreg0\n"
- "fmla v22.4s, v10.4s, v5.s[2]\n"
- "ins v13.d[1], temploadreg1\n"
- "fmla v26.4s, v10.4s, v6.s[2]\n"
- "ins v14.d[1], temploadreg2\n"
- "fmla v19.4s, v11.4s, v4.s[2]\n"
- "ins v15.d[1], temploadreg3\n"
- "fmla v23.4s, v11.4s, v5.s[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "fmla v27.4s, v11.4s, v6.s[2]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "fmla v16.4s, v12.4s, v4.s[3]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "fmla v20.4s, v12.4s, v5.s[3]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "fmla v24.4s, v12.4s, v6.s[3]\n"
- "fmla v17.4s, v13.4s, v4.s[3]\n"
- "fmla v21.4s, v13.4s, v5.s[3]\n"
- "fmla v25.4s, v13.4s, v6.s[3]\n"
- "fmla v18.4s, v14.4s, v4.s[3]\n"
- "fmla v22.4s, v14.4s, v5.s[3]\n"
- "fmla v26.4s, v14.4s, v6.s[3]\n"
- "fmla v19.4s, v15.4s, v4.s[3]\n"
- "fmla v23.4s, v15.4s, v5.s[3]\n"
- "fmla v27.4s, v15.4s, v6.s[3]\n"
- "b 5f\n"
- "4:\n"
- "fmla v16.4s, v8.4s, v0.s[0]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- "fmla v20.4s, v8.4s, v1.s[0]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- "fmla v24.4s, v8.4s, v2.s[0]\n"
- "ldr d8, [%[b_ptr0]]\n"
- "fmla v17.4s, v9.4s, v0.s[0]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- "fmla v21.4s, v9.4s, v1.s[0]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- "fmla v25.4s, v9.4s, v2.s[0]\n"
- "ldr d9, [%[b_ptr0], #0x10]\n"
- "fmla v18.4s, v10.4s, v0.s[0]\n"
- "ins v8.d[1], temploadreg0\n"
- "fmla v22.4s, v10.4s, v1.s[0]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- "fmla v26.4s, v10.4s, v2.s[0]\n"
- "ldr d10, [%[b_ptr0], #0x20]\n"
- "fmla v19.4s, v11.4s, v0.s[0]\n"
- "ins v9.d[1], temploadreg1\n"
- "fmla v23.4s, v11.4s, v1.s[0]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- "fmla v27.4s, v11.4s, v2.s[0]\n"
- "ldr d11, [%[b_ptr0], #0x30]\n"
- "fmla v16.4s, v12.4s, v0.s[1]\n"
- "ins v10.d[1], temploadreg2\n"
- "fmla v20.4s, v12.4s, v1.s[1]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- "fmla v24.4s, v12.4s, v2.s[1]\n"
- "ldr d12, [%[b_ptr0], #0x40]\n"
- "fmla v17.4s, v13.4s, v0.s[1]\n"
- "ins v11.d[1], temploadreg3\n"
- "fmla v21.4s, v13.4s, v1.s[1]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- "fmla v25.4s, v13.4s, v2.s[1]\n"
- "ldr d13, [%[b_ptr0], #0x50]\n"
- "fmla v18.4s, v14.4s, v0.s[1]\n"
- "ins v12.d[1], temploadreg0\n"
- "fmla v22.4s, v14.4s, v1.s[1]\n"
- "fmla v26.4s, v14.4s, v2.s[1]\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- "fmla v19.4s, v15.4s, v0.s[1]\n"
- "ins v13.d[1], temploadreg1\n"
- "fmla v23.4s, v15.4s, v1.s[1]\n"
- "fmla v27.4s, v15.4s, v2.s[1]\n"
- "ldr d15, [%[b_ptr0], #0x70]\n"
- "fmla v16.4s, v8.4s, v0.s[2]\n"
- "ins v14.d[1], temploadreg2\n"
- "fmla v20.4s, v8.4s, v1.s[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "fmla v24.4s, v8.4s, v2.s[2]\n"
- "ins v15.d[1], temploadreg3\n"
- "fmla v17.4s, v9.4s, v0.s[2]\n"
- "fmla v21.4s, v9.4s, v1.s[2]\n"
- "fmla v25.4s, v9.4s, v2.s[2]\n"
- "fmla v18.4s, v10.4s, v0.s[2]\n"
- "fmla v22.4s, v10.4s, v1.s[2]\n"
- "fmla v26.4s, v10.4s, v2.s[2]\n"
- "fmla v19.4s, v11.4s, v0.s[2]\n"
- "fmla v23.4s, v11.4s, v1.s[2]\n"
- "fmla v27.4s, v11.4s, v2.s[2]\n"
- "fmla v16.4s, v12.4s, v0.s[3]\n"
- "fmla v20.4s, v12.4s, v1.s[3]\n"
- "fmla v24.4s, v12.4s, v2.s[3]\n"
- "fmla v17.4s, v13.4s, v0.s[3]\n"
- "fmla v21.4s, v13.4s, v1.s[3]\n"
- "fmla v25.4s, v13.4s, v2.s[3]\n"
- "fmla v18.4s, v14.4s, v0.s[3]\n"
- "fmla v22.4s, v14.4s, v1.s[3]\n"
- "fmla v26.4s, v14.4s, v2.s[3]\n"
- "fmla v19.4s, v15.4s, v0.s[3]\n"
- "fmla v23.4s, v15.4s, v1.s[3]\n"
- "fmla v27.4s, v15.4s, v2.s[3]\n"
- "5:\n"
- "cbz %[blocks], 6f\n"
- "7:\n"
- "ldr q8, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "ldr s0, [%[a_ptr0]]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "add %[a_ptr0], %[a_ptr0], #0x4\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "add %[b_ptr0], %[b_ptr0], #0x40\n"
- "fmla v16.4s, v8.4s, v0.s[0]\n"
- "ldr s1, [a_ptr1]\n"
- "fmla v17.4s, v9.4s, v0.s[0]\n"
- "add a_ptr1, a_ptr1, #0x4\n"
- "fmla v18.4s, v10.4s, v0.s[0]\n"
- "ldr s2, [a_ptr2]\n"
- "fmla v20.4s, v8.4s, v1.s[0]\n"
- "add a_ptr2, a_ptr2, #0x4\n"
- "fmla v21.4s, v9.4s, v1.s[0]\n"
- "fmla v24.4s, v8.4s, v2.s[0]\n"
- "fmla v25.4s, v9.4s, v2.s[0]\n"
- "fmla v22.4s, v10.4s, v1.s[0]\n"
- "fmla v26.4s, v10.4s, v2.s[0]\n"
- "fmla v19.4s, v11.4s, v0.s[0]\n"
- "fmla v23.4s, v11.4s, v1.s[0]\n"
- "fmla v27.4s, v11.4s, v2.s[0]\n"
- "b.ne 7b\n"
- "6:\n"
- "ld1r {v14.4s}, [%[minptr]]\n"
- "ld1r {v15.4s}, [%[maxptr]]\n"
- "fmax v16.4s, v16.4s, v14.4s\n"
- "fmax v17.4s, v17.4s, v14.4s\n"
- "fmax v18.4s, v18.4s, v14.4s\n"
- "fmax v19.4s, v19.4s, v14.4s\n"
- "fmin v16.4s, v16.4s, v15.4s\n"
- "fmin v17.4s, v17.4s, v15.4s\n"
- "fmin v18.4s, v18.4s, v15.4s\n"
- "fmin v19.4s, v19.4s, v15.4s\n"
- "str q16, [%[c_ptr0]]\n"
- "fmax v20.4s, v20.4s, v14.4s\n"
- "fmax v21.4s, v21.4s, v14.4s\n"
- "fmax v22.4s, v22.4s, v14.4s\n"
- "str q17, [%[c_ptr0], #0x10]\n"
- "fmax v23.4s, v23.4s, v14.4s\n"
- "fmin v20.4s, v20.4s, v15.4s\n"
- "fmin v21.4s, v21.4s, v15.4s\n"
- "str q18, [%[c_ptr0], #0x20]\n"
- "fmin v22.4s, v22.4s, v15.4s\n"
- "fmin v23.4s, v23.4s, v15.4s\n"
- "fmax v24.4s, v24.4s, v14.4s\n"
- "str q19, [%[c_ptr0], #0x30]\n"
- "fmax v25.4s, v25.4s, v14.4s\n"
- "add %[c_ptr0], %[c_ptr0], #0x40\n"
- "fmax v26.4s, v26.4s, v14.4s\n"
- "str q20, [c_ptr1]\n"
- "fmin v24.4s, v24.4s, v15.4s\n"
- "fmin v25.4s, v25.4s, v15.4s\n"
- "fmax v27.4s, v27.4s, v14.4s\n"
- "str q21, [c_ptr1, #0x10]\n"
- "fmin v26.4s, v26.4s, v15.4s\n"
- "fmin v27.4s, v27.4s, v15.4s\n"
- "str q22, [c_ptr1, #0x20]\n"
- "str q23, [c_ptr1, #0x30]\n"
- "str q24, [c_ptr2]\n"
- "str q25, [c_ptr2, #0x10]\n"
- "str q26, [c_ptr2, #0x20]\n"
- "str q27, [c_ptr2, #0x30]\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- ".unreq temploadreg0\n"
- ".unreq temploadreg1\n"
- ".unreq temploadreg2\n"
- ".unreq temploadreg3\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks)
- : [width] "r" (width), [accumulate] "r" (static_cast<uint64_t>(accumulate)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr)
- : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "cc", "memory"
- );
- break;
- default:
- case 4:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "a_ptr3 .req X2\n"
- "c_ptr1 .req X3\n"
- "c_ptr2 .req X4\n"
- "c_ptr3 .req X5\n"
- "temploadreg0 .req X6\n"
- "temploadreg1 .req X7\n"
- "temploadreg2 .req X8\n"
- "temploadreg3 .req X9\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "add a_ptr3, a_ptr2, %[lda]\n"
- "add c_ptr3, c_ptr2, %[ldc]\n"
- "cbnz %[accumulate], 1f\n"
- "ldr q16, [%[biasptr]]\n"
- "ldr q17, [%[biasptr], #0x10]\n"
- "ldr q18, [%[biasptr], #0x20]\n"
- "ldr q19, [%[biasptr], #0x30]\n"
- "mov v20.16b, v16.16b\n"
- "ldr q0, [%[a_ptr0]]\n"
- "mov v21.16b, v17.16b\n"
- "ldr q1, [a_ptr1]\n"
- "mov v22.16b, v18.16b\n"
- "ldr q2, [a_ptr2]\n"
- "mov v23.16b, v19.16b\n"
- "ldr q3, [a_ptr3]\n"
- "mov v24.16b, v16.16b\n"
- "ldr q8, [%[b_ptr0]]\n"
- "mov v25.16b, v17.16b\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "mov v26.16b, v18.16b\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "mov v27.16b, v19.16b\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "mov v28.16b, v16.16b\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- "mov v29.16b, v17.16b\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- "mov v30.16b, v18.16b\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- "mov v31.16b, v19.16b\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "ins v14.d[1], temploadreg2\n"
- "add a_ptr3, a_ptr3, #0x10\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "ldr q16, [%[c_ptr0]]\n"
- "ldr q17, [%[c_ptr0], #0x10]\n"
- "ldr q18, [%[c_ptr0], #0x20]\n"
- "ldr q19, [%[c_ptr0], #0x30]\n"
- "ldr q20, [c_ptr1]\n"
- "ldr q21, [c_ptr1, #0x10]\n"
- "ldr q22, [c_ptr1, #0x20]\n"
- "ldr q23, [c_ptr1, #0x30]\n"
- "ldr q24, [c_ptr2]\n"
- "ldr q25, [c_ptr2, #0x10]\n"
- "ldr q26, [c_ptr2, #0x20]\n"
- "ldr q27, [c_ptr2, #0x30]\n"
- "ldr q28, [c_ptr3]\n"
- "ldr q29, [c_ptr3, #0x10]\n"
- "ldr q30, [c_ptr3, #0x20]\n"
- "ldr q31, [c_ptr3, #0x30]\n"
- "ldr q0, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ldr q1, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "ldr q2, [a_ptr2]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "ldr q3, [a_ptr3]\n"
- "add a_ptr3, a_ptr3, #0x10\n"
- "ldr q8, [%[b_ptr0]]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "ins v14.d[1], temploadreg2\n"
- "cbz %[loops], 2f\n"
- "3:\n"
- "fmla v16.4s, v8.4s, v0.s[0]\n"
- "ldr d15, [%[b_ptr0], #-0x10]\n"
- "fmla v20.4s, v8.4s, v1.s[0]\n"
- "ldr temploadreg3, [%[b_ptr0], #-0x8]\n"
- "fmla v24.4s, v8.4s, v2.s[0]\n"
- "ldr d4, [%[a_ptr0]]\n"
- "fmla v28.4s, v8.4s, v3.s[0]\n"
- "ldr temploadreg0, [%[a_ptr0], #0x8]\n"
- "fmla v17.4s, v9.4s, v0.s[0]\n"
- "ldr d5, [a_ptr1]\n"
- "fmla v21.4s, v9.4s, v1.s[0]\n"
- "ldr temploadreg1, [a_ptr1, #0x8]\n"
- "fmla v25.4s, v9.4s, v2.s[0]\n"
- "ldr d6, [a_ptr2]\n"
- "fmla v29.4s, v9.4s, v3.s[0]\n"
- "ldr temploadreg2, [a_ptr2, #0x8]\n"
- "fmla v18.4s, v10.4s, v0.s[0]\n"
- "ldr d7, [a_ptr3]\n"
- "fmla v22.4s, v10.4s, v1.s[0]\n"
- "ins v15.d[1], temploadreg3\n"
- "fmla v26.4s, v10.4s, v2.s[0]\n"
- "ldr temploadreg3, [a_ptr3, #0x8]\n"
- "fmla v30.4s, v10.4s, v3.s[0]\n"
- "ldr d8, [%[b_ptr0]]\n"
- "fmla v19.4s, v11.4s, v0.s[0]\n"
- "ins v4.d[1], temploadreg0\n"
- "fmla v23.4s, v11.4s, v1.s[0]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- "fmla v27.4s, v11.4s, v2.s[0]\n"
- "ldr d9, [%[b_ptr0], #0x10]\n"
- "fmla v31.4s, v11.4s, v3.s[0]\n"
- "ins v5.d[1], temploadreg1\n"
- "fmla v16.4s, v12.4s, v0.s[1]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- "fmla v20.4s, v12.4s, v1.s[1]\n"
- "ldr d10, [%[b_ptr0], #0x20]\n"
- "fmla v24.4s, v12.4s, v2.s[1]\n"
- "ins v6.d[1], temploadreg2\n"
- "fmla v28.4s, v12.4s, v3.s[1]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- "fmla v17.4s, v13.4s, v0.s[1]\n"
- "ldr d11, [%[b_ptr0], #0x30]\n"
- "fmla v21.4s, v13.4s, v1.s[1]\n"
- "ins v7.d[1], temploadreg3\n"
- "fmla v25.4s, v13.4s, v2.s[1]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- "fmla v29.4s, v13.4s, v3.s[1]\n"
- "ldr d12, [%[b_ptr0], #0x40]\n"
- "fmla v18.4s, v14.4s, v0.s[1]\n"
- "ins v8.d[1], temploadreg0\n"
- "fmla v22.4s, v14.4s, v1.s[1]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- "fmla v26.4s, v14.4s, v2.s[1]\n"
- "ldr d13, [%[b_ptr0], #0x50]\n"
- "fmla v30.4s, v14.4s, v3.s[1]\n"
- "ins v9.d[1], temploadreg1\n"
- "fmla v19.4s, v15.4s, v0.s[1]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- "fmla v23.4s, v15.4s, v1.s[1]\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- "fmla v27.4s, v15.4s, v2.s[1]\n"
- "ins v10.d[1], temploadreg2\n"
- "fmla v31.4s, v15.4s, v3.s[1]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- "fmla v16.4s, v8.4s, v0.s[2]\n"
- "ldr d15, [%[b_ptr0], #0x70]\n"
- "fmla v20.4s, v8.4s, v1.s[2]\n"
- "ins v11.d[1], temploadreg3\n"
- "fmla v24.4s, v8.4s, v2.s[2]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- "fmla v28.4s, v8.4s, v3.s[2]\n"
- "ins v12.d[1], temploadreg0\n"
- "fmla v17.4s, v9.4s, v0.s[2]\n"
- "ins v13.d[1], temploadreg1\n"
- "fmla v21.4s, v9.4s, v1.s[2]\n"
- "ins v14.d[1], temploadreg2\n"
- "fmla v25.4s, v9.4s, v2.s[2]\n"
- "ins v15.d[1], temploadreg3\n"
- "fmla v29.4s, v9.4s, v3.s[2]\n"
- "subs %[loops], %[loops], #0x1\n"
- "fmla v18.4s, v10.4s, v0.s[2]\n"
- "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n"
- "fmla v22.4s, v10.4s, v1.s[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- "fmla v26.4s, v10.4s, v2.s[2]\n"
- "ldr d8, [%[b_ptr0], #-0x80]\n"
- "fmla v30.4s, v10.4s, v3.s[2]\n"
- "ldr temploadreg0, [%[b_ptr0], #-0x78]\n"
- "fmla v19.4s, v11.4s, v0.s[2]\n"
- "ldr d9, [%[b_ptr0], #-0x70]\n"
- "fmla v23.4s, v11.4s, v1.s[2]\n"
- "ldr temploadreg1, [%[b_ptr0], #-0x68]\n"
- "fmla v27.4s, v11.4s, v2.s[2]\n"
- "ldr d10, [%[b_ptr0], #-0x60]\n"
- "fmla v31.4s, v11.4s, v3.s[2]\n"
- "ldr temploadreg2, [%[b_ptr0], #-0x58]\n"
- "fmla v16.4s, v12.4s, v0.s[3]\n"
- "ldr d11, [%[b_ptr0], #-0x50]\n"
- "fmla v20.4s, v12.4s, v1.s[3]\n"
- "ldr temploadreg3, [%[b_ptr0], #-0x48]\n"
- "fmla v24.4s, v12.4s, v2.s[3]\n"
- "ins v8.d[1], temploadreg0\n"
- "fmla v28.4s, v12.4s, v3.s[3]\n"
- "ldr d12, [%[b_ptr0], #-0x40]\n"
- "fmla v17.4s, v13.4s, v0.s[3]\n"
- "ldr temploadreg0, [%[b_ptr0], #-0x38]\n"
- "fmla v21.4s, v13.4s, v1.s[3]\n"
- "ins v9.d[1], temploadreg1\n"
- "fmla v25.4s, v13.4s, v2.s[3]\n"
- "ldr temploadreg1, [%[b_ptr0], #-0x28]\n"
- "fmla v29.4s, v13.4s, v3.s[3]\n"
- "ldr d13, [%[b_ptr0], #-0x30]\n"
- "fmla v18.4s, v14.4s, v0.s[3]\n"
- "ins v10.d[1], temploadreg2\n"
- "fmla v22.4s, v14.4s, v1.s[3]\n"
- "ldr temploadreg2, [%[b_ptr0], #-0x18]\n"
- "fmla v26.4s, v14.4s, v2.s[3]\n"
- "ins v11.d[1], temploadreg3\n"
- "fmla v30.4s, v14.4s, v3.s[3]\n"
- "ldr d14, [%[b_ptr0], #-0x20]\n"
- "fmla v19.4s, v15.4s, v0.s[3]\n"
- "ldr temploadreg3, [%[b_ptr0], #-0x8]\n"
- "fmla v23.4s, v15.4s, v1.s[3]\n"
- "ins v12.d[1], temploadreg0\n"
- "fmla v27.4s, v15.4s, v2.s[3]\n"
- "ins v13.d[1], temploadreg1\n"
- "fmla v31.4s, v15.4s, v3.s[3]\n"
- "ldr d15, [%[b_ptr0], #-0x10]\n"
- "fmla v16.4s, v8.4s, v4.s[0]\n"
- "ins v14.d[1], temploadreg2\n"
- "fmla v20.4s, v8.4s, v5.s[0]\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
- "fmla v24.4s, v8.4s, v6.s[0]\n"
- "ldr d0, [%[a_ptr0], #-0x10]\n"
- "fmla v28.4s, v8.4s, v7.s[0]\n"
- "ldr temploadreg0, [%[a_ptr0], #-0x8]\n"
- "fmla v17.4s, v9.4s, v4.s[0]\n"
- "ins v15.d[1], temploadreg3\n"
- "fmla v21.4s, v9.4s, v5.s[0]\n"
- "ldr d8, [%[b_ptr0]]\n"
- "fmla v25.4s, v9.4s, v6.s[0]\n"
- "ins v0.d[1], temploadreg0\n"
- "fmla v29.4s, v9.4s, v7.s[0]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- "fmla v18.4s, v10.4s, v4.s[0]\n"
- "ldr d9, [%[b_ptr0], #0x10]\n"
- "fmla v22.4s, v10.4s, v5.s[0]\n"
- "add a_ptr1, a_ptr1, #0x20\n"
- "fmla v26.4s, v10.4s, v6.s[0]\n"
- "ldr d1, [a_ptr1, #-0x10]\n"
- "fmla v30.4s, v10.4s, v7.s[0]\n"
- "ldr temploadreg1, [a_ptr1, #-0x8]\n"
- "fmla v19.4s, v11.4s, v4.s[0]\n"
- "ldr d10, [%[b_ptr0], #0x20]\n"
- "fmla v23.4s, v11.4s, v5.s[0]\n"
- "ins v8.d[1], temploadreg0\n"
- "fmla v27.4s, v11.4s, v6.s[0]\n"
- "ins v1.d[1], temploadreg1\n"
- "fmla v31.4s, v11.4s, v7.s[0]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- "fmla v16.4s, v12.4s, v4.s[1]\n"
- "ldr d11, [%[b_ptr0], #0x30]\n"
- "fmla v20.4s, v12.4s, v5.s[1]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- "fmla v24.4s, v12.4s, v6.s[1]\n"
- "ins v9.d[1], temploadreg1\n"
- "fmla v28.4s, v12.4s, v7.s[1]\n"
- "ldr d12, [%[b_ptr0], #0x40]\n"
- "fmla v17.4s, v13.4s, v4.s[1]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- "fmla v21.4s, v13.4s, v5.s[1]\n"
- "add a_ptr2, a_ptr2, #0x20\n"
- "fmla v25.4s, v13.4s, v6.s[1]\n"
- "ldr d2, [a_ptr2, #-0x10]\n"
- "fmla v29.4s, v13.4s, v7.s[1]\n"
- "ldr temploadreg2, [a_ptr2, #-0x8]\n"
- "fmla v18.4s, v14.4s, v4.s[1]\n"
- "ldr d13, [%[b_ptr0], #0x50]\n"
- "fmla v22.4s, v14.4s, v5.s[1]\n"
- "ins v12.d[1], temploadreg0\n"
- "fmla v26.4s, v14.4s, v6.s[1]\n"
- "ins v2.d[1], temploadreg2\n"
- "fmla v30.4s, v14.4s, v7.s[1]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- "fmla v19.4s, v15.4s, v4.s[1]\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- "fmla v23.4s, v15.4s, v5.s[1]\n"
- "ins v13.d[1], temploadreg1\n"
- "fmla v27.4s, v15.4s, v6.s[1]\n"
- "ins v10.d[1], temploadreg2\n"
- "fmla v31.4s, v15.4s, v7.s[1]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- "fmla v16.4s, v8.4s, v4.s[2]\n"
- "ldr d15, [%[b_ptr0], #0x70]\n"
- "fmla v20.4s, v8.4s, v5.s[2]\n"
- "add a_ptr3, a_ptr3, #0x20\n"
- "fmla v24.4s, v8.4s, v6.s[2]\n"
- "ldr d3, [a_ptr3, #-0x10]\n"
- "fmla v28.4s, v8.4s, v7.s[2]\n"
- "ldr temploadreg3, [a_ptr3, #-0x8]\n"
- "fmla v17.4s, v9.4s, v4.s[2]\n"
- "ins v14.d[1], temploadreg2\n"
- "fmla v21.4s, v9.4s, v5.s[2]\n"
- "prfm PLDL1KEEP, [a_ptr1, #0x40]\n"
- "fmla v25.4s, v9.4s, v6.s[2]\n"
- "ins v3.d[1], temploadreg3\n"
- "fmla v29.4s, v9.4s, v7.s[2]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- "fmla v18.4s, v10.4s, v4.s[2]\n"
- "prfm PLDL1KEEP, [a_ptr2, #0x40]\n"
- "fmla v22.4s, v10.4s, v5.s[2]\n"
- "ins v11.d[1], temploadreg3\n"
- "fmla v26.4s, v10.4s, v6.s[2]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- "fmla v30.4s, v10.4s, v7.s[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- "fmla v19.4s, v11.4s, v4.s[2]\n"
- "ldr d8, [%[b_ptr0], #-0x80]\n"
- "fmla v23.4s, v11.4s, v5.s[2]\n"
- "ldr temploadreg0, [%[b_ptr0], #-0x78]\n"
- "fmla v27.4s, v11.4s, v6.s[2]\n"
- "ldr d9, [%[b_ptr0], #-0x70]\n"
- "fmla v31.4s, v11.4s, v7.s[2]\n"
- "ldr temploadreg1, [%[b_ptr0], #-0x68]\n"
- "fmla v16.4s, v12.4s, v4.s[3]\n"
- "ldr d10, [%[b_ptr0], #-0x60]\n"
- "fmla v20.4s, v12.4s, v5.s[3]\n"
- "ldr temploadreg2, [%[b_ptr0], #-0x58]\n"
- "fmla v24.4s, v12.4s, v6.s[3]\n"
- "ldr d11, [%[b_ptr0], #-0x50]\n"
- "fmla v28.4s, v12.4s, v7.s[3]\n"
- "ins v15.d[1], temploadreg3\n"
- "fmla v17.4s, v13.4s, v4.s[3]\n"
- "ldr temploadreg3, [%[b_ptr0], #-0x48]\n"
- "fmla v21.4s, v13.4s, v5.s[3]\n"
- "ldr d12, [%[b_ptr0], #-0x40]\n"
- "fmla v25.4s, v13.4s, v6.s[3]\n"
- "ins v8.d[1], temploadreg0\n"
- "fmla v29.4s, v13.4s, v7.s[3]\n"
- "ldr temploadreg0, [%[b_ptr0], #-0x38]\n"
- "fmla v18.4s, v14.4s, v4.s[3]\n"
- "ldr d13, [%[b_ptr0], #-0x30]\n"
- "fmla v22.4s, v14.4s, v5.s[3]\n"
- "ins v9.d[1], temploadreg1\n"
- "fmla v26.4s, v14.4s, v6.s[3]\n"
- "ldr temploadreg1, [%[b_ptr0], #-0x28]\n"
- "fmla v30.4s, v14.4s, v7.s[3]\n"
- "ldr d14, [%[b_ptr0], #-0x20]\n"
- "fmla v19.4s, v15.4s, v4.s[3]\n"
- "ins v10.d[1], temploadreg2\n"
- "fmla v23.4s, v15.4s, v5.s[3]\n"
- "ldr temploadreg2, [%[b_ptr0], #-0x18]\n"
- "fmla v27.4s, v15.4s, v6.s[3]\n"
- "ins v11.d[1], temploadreg3\n"
- "fmla v31.4s, v15.4s, v7.s[3]\n"
- "ins v12.d[1], temploadreg0\n"
- "ins v13.d[1], temploadreg1\n"
- "prfm PLDL1KEEP, [a_ptr3, #0x40]\n"
- "ins v14.d[1], temploadreg2\n"
- "b.ne 3b\n"
- "2:\n"
- "ldr d15, [%[b_ptr0], #-0x10]\n"
- "prfm PSTL1KEEP, [%[c_ptr0]]\n"
- "ldr temploadreg3, [%[b_ptr0], #-0x8]\n"
- "prfm PSTL1KEEP, [c_ptr1]\n"
- "prfm PSTL1KEEP, [c_ptr2]\n"
- "prfm PSTL1KEEP, [c_ptr3]\n"
- "ins v15.d[1], temploadreg3\n"
- "cbz %[regs], 4f\n"
- "fmla v16.4s, v8.4s, v0.s[0]\n"
- "ldr d4, [%[a_ptr0]]\n"
- "fmla v20.4s, v8.4s, v1.s[0]\n"
- "ldr temploadreg0, [%[a_ptr0], #0x8]\n"
- "fmla v24.4s, v8.4s, v2.s[0]\n"
- "ldr d5, [a_ptr1]\n"
- "fmla v28.4s, v8.4s, v3.s[0]\n"
- "ldr temploadreg1, [a_ptr1, #0x8]\n"
- "fmla v17.4s, v9.4s, v0.s[0]\n"
- "ldr d6, [a_ptr2]\n"
- "fmla v21.4s, v9.4s, v1.s[0]\n"
- "ldr temploadreg2, [a_ptr2, #0x8]\n"
- "fmla v25.4s, v9.4s, v2.s[0]\n"
- "ldr d7, [a_ptr3]\n"
- "fmla v29.4s, v9.4s, v3.s[0]\n"
- "ldr temploadreg3, [a_ptr3, #0x8]\n"
- "fmla v18.4s, v10.4s, v0.s[0]\n"
- "ldr d8, [%[b_ptr0]]\n"
- "fmla v22.4s, v10.4s, v1.s[0]\n"
- "ins v4.d[1], temploadreg0\n"
- "fmla v26.4s, v10.4s, v2.s[0]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- "fmla v30.4s, v10.4s, v3.s[0]\n"
- "ldr d9, [%[b_ptr0], #0x10]\n"
- "fmla v19.4s, v11.4s, v0.s[0]\n"
- "ins v5.d[1], temploadreg1\n"
- "fmla v23.4s, v11.4s, v1.s[0]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- "fmla v27.4s, v11.4s, v2.s[0]\n"
- "ldr d10, [%[b_ptr0], #0x20]\n"
- "fmla v31.4s, v11.4s, v3.s[0]\n"
- "ins v6.d[1], temploadreg2\n"
- "fmla v16.4s, v12.4s, v0.s[1]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- "fmla v20.4s, v12.4s, v1.s[1]\n"
- "ldr d11, [%[b_ptr0], #0x30]\n"
- "fmla v24.4s, v12.4s, v2.s[1]\n"
- "ins v7.d[1], temploadreg3\n"
- "fmla v28.4s, v12.4s, v3.s[1]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- "fmla v17.4s, v13.4s, v0.s[1]\n"
- "ldr d12, [%[b_ptr0], #0x40]\n"
- "fmla v21.4s, v13.4s, v1.s[1]\n"
- "ins v8.d[1], temploadreg0\n"
- "fmla v25.4s, v13.4s, v2.s[1]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- "fmla v29.4s, v13.4s, v3.s[1]\n"
- "ldr d13, [%[b_ptr0], #0x50]\n"
- "fmla v18.4s, v14.4s, v0.s[1]\n"
- "ins v9.d[1], temploadreg1\n"
- "fmla v22.4s, v14.4s, v1.s[1]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- "fmla v26.4s, v14.4s, v2.s[1]\n"
- "ins v10.d[1], temploadreg2\n"
- "fmla v30.4s, v14.4s, v3.s[1]\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- "fmla v19.4s, v15.4s, v0.s[1]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- "fmla v23.4s, v15.4s, v1.s[1]\n"
- "ins v11.d[1], temploadreg3\n"
- "fmla v27.4s, v15.4s, v2.s[1]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- "fmla v31.4s, v15.4s, v3.s[1]\n"
- "ldr d15, [%[b_ptr0], #0x70]\n"
- "fmla v16.4s, v8.4s, v0.s[2]\n"
- "ins v12.d[1], temploadreg0\n"
- "fmla v20.4s, v8.4s, v1.s[2]\n"
- "ins v13.d[1], temploadreg1\n"
- "fmla v24.4s, v8.4s, v2.s[2]\n"
- "ins v14.d[1], temploadreg2\n"
- "fmla v28.4s, v8.4s, v3.s[2]\n"
- "ins v15.d[1], temploadreg3\n"
- "fmla v17.4s, v9.4s, v0.s[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- "fmla v21.4s, v9.4s, v1.s[2]\n"
- "ldr d8, [%[b_ptr0], #-0x80]\n"
- "fmla v25.4s, v9.4s, v2.s[2]\n"
- "ldr temploadreg0, [%[b_ptr0], #-0x78]\n"
- "fmla v29.4s, v9.4s, v3.s[2]\n"
- "ldr d9, [%[b_ptr0], #-0x70]\n"
- "fmla v18.4s, v10.4s, v0.s[2]\n"
- "ldr temploadreg1, [%[b_ptr0], #-0x68]\n"
- "fmla v22.4s, v10.4s, v1.s[2]\n"
- "ldr temploadreg2, [%[b_ptr0], #-0x58]\n"
- "fmla v26.4s, v10.4s, v2.s[2]\n"
- "ldr temploadreg3, [%[b_ptr0], #-0x48]\n"
- "fmla v30.4s, v10.4s, v3.s[2]\n"
- "ldr d10, [%[b_ptr0], #-0x60]\n"
- "fmla v19.4s, v11.4s, v0.s[2]\n"
- "ins v8.d[1], temploadreg0\n"
- "fmla v23.4s, v11.4s, v1.s[2]\n"
- "ldr temploadreg0, [%[b_ptr0], #-0x38]\n"
- "fmla v27.4s, v11.4s, v2.s[2]\n"
- "ins v9.d[1], temploadreg1\n"
- "fmla v31.4s, v11.4s, v3.s[2]\n"
- "ldr d11, [%[b_ptr0], #-0x50]\n"
- "fmla v16.4s, v12.4s, v0.s[3]\n"
- "ldr temploadreg1, [%[b_ptr0], #-0x28]\n"
- "fmla v20.4s, v12.4s, v1.s[3]\n"
- "ins v10.d[1], temploadreg2\n"
- "fmla v24.4s, v12.4s, v2.s[3]\n"
- "ldr temploadreg2, [%[b_ptr0], #-0x18]\n"
- "fmla v28.4s, v12.4s, v3.s[3]\n"
- "ldr d12, [%[b_ptr0], #-0x40]\n"
- "fmla v17.4s, v13.4s, v0.s[3]\n"
- "ins v11.d[1], temploadreg3\n"
- "fmla v21.4s, v13.4s, v1.s[3]\n"
- "ldr temploadreg3, [%[b_ptr0], #-0x8]\n"
- "fmla v25.4s, v13.4s, v2.s[3]\n"
- "ins v12.d[1], temploadreg0\n"
- "fmla v29.4s, v13.4s, v3.s[3]\n"
- "ldr d13, [%[b_ptr0], #-0x30]\n"
- "fmla v18.4s, v14.4s, v0.s[3]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- "fmla v22.4s, v14.4s, v1.s[3]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "fmla v26.4s, v14.4s, v2.s[3]\n"
- "ins v13.d[1], temploadreg1\n"
- "fmla v30.4s, v14.4s, v3.s[3]\n"
- "ldr d14, [%[b_ptr0], #-0x20]\n"
- "fmla v19.4s, v15.4s, v0.s[3]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- "fmla v23.4s, v15.4s, v1.s[3]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "fmla v27.4s, v15.4s, v2.s[3]\n"
- "ins v14.d[1], temploadreg2\n"
- "fmla v31.4s, v15.4s, v3.s[3]\n"
- "ldr d15, [%[b_ptr0], #-0x10]\n"
- "fmla v16.4s, v8.4s, v4.s[0]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- "fmla v20.4s, v8.4s, v5.s[0]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "fmla v24.4s, v8.4s, v6.s[0]\n"
- "ins v15.d[1], temploadreg3\n"
- "fmla v28.4s, v8.4s, v7.s[0]\n"
- "ldr d8, [%[b_ptr0]]\n"
- "fmla v17.4s, v9.4s, v4.s[0]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- "fmla v21.4s, v9.4s, v5.s[0]\n"
- "add a_ptr3, a_ptr3, #0x10\n"
- "fmla v25.4s, v9.4s, v6.s[0]\n"
- "ins v8.d[1], temploadreg0\n"
- "fmla v29.4s, v9.4s, v7.s[0]\n"
- "ldr d9, [%[b_ptr0], #0x10]\n"
- "fmla v18.4s, v10.4s, v4.s[0]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- "fmla v22.4s, v10.4s, v5.s[0]\n"
- "fmla v26.4s, v10.4s, v6.s[0]\n"
- "ins v9.d[1], temploadreg1\n"
- "fmla v30.4s, v10.4s, v7.s[0]\n"
- "ldr d10, [%[b_ptr0], #0x20]\n"
- "fmla v19.4s, v11.4s, v4.s[0]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- "fmla v23.4s, v11.4s, v5.s[0]\n"
- "fmla v27.4s, v11.4s, v6.s[0]\n"
- "ins v10.d[1], temploadreg2\n"
- "fmla v31.4s, v11.4s, v7.s[0]\n"
- "ldr d11, [%[b_ptr0], #0x30]\n"
- "fmla v16.4s, v12.4s, v4.s[1]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- "fmla v20.4s, v12.4s, v5.s[1]\n"
- "fmla v24.4s, v12.4s, v6.s[1]\n"
- "ins v11.d[1], temploadreg3\n"
- "fmla v28.4s, v12.4s, v7.s[1]\n"
- "ldr d12, [%[b_ptr0], #0x40]\n"
- "fmla v17.4s, v13.4s, v4.s[1]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- "fmla v21.4s, v13.4s, v5.s[1]\n"
- "fmla v25.4s, v13.4s, v6.s[1]\n"
- "ins v12.d[1], temploadreg0\n"
- "fmla v29.4s, v13.4s, v7.s[1]\n"
- "ldr d13, [%[b_ptr0], #0x50]\n"
- "fmla v18.4s, v14.4s, v4.s[1]\n"
- "fmla v22.4s, v14.4s, v5.s[1]\n"
- "fmla v26.4s, v14.4s, v6.s[1]\n"
- "ins v13.d[1], temploadreg1\n"
- "fmla v30.4s, v14.4s, v7.s[1]\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- "fmla v19.4s, v15.4s, v4.s[1]\n"
- "fmla v23.4s, v15.4s, v5.s[1]\n"
- "fmla v27.4s, v15.4s, v6.s[1]\n"
- "ins v14.d[1], temploadreg2\n"
- "fmla v31.4s, v15.4s, v7.s[1]\n"
- "ldr d15, [%[b_ptr0], #0x70]\n"
- "fmla v16.4s, v8.4s, v4.s[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "fmla v20.4s, v8.4s, v5.s[2]\n"
- "ins v15.d[1], temploadreg3\n"
- "fmla v24.4s, v8.4s, v6.s[2]\n"
- "fmla v28.4s, v8.4s, v7.s[2]\n"
- "fmla v17.4s, v9.4s, v4.s[2]\n"
- "fmla v21.4s, v9.4s, v5.s[2]\n"
- "fmla v25.4s, v9.4s, v6.s[2]\n"
- "fmla v29.4s, v9.4s, v7.s[2]\n"
- "fmla v18.4s, v10.4s, v4.s[2]\n"
- "fmla v22.4s, v10.4s, v5.s[2]\n"
- "fmla v26.4s, v10.4s, v6.s[2]\n"
- "fmla v30.4s, v10.4s, v7.s[2]\n"
- "fmla v19.4s, v11.4s, v4.s[2]\n"
- "fmla v23.4s, v11.4s, v5.s[2]\n"
- "fmla v27.4s, v11.4s, v6.s[2]\n"
- "fmla v31.4s, v11.4s, v7.s[2]\n"
- "fmla v16.4s, v12.4s, v4.s[3]\n"
- "fmla v20.4s, v12.4s, v5.s[3]\n"
- "fmla v24.4s, v12.4s, v6.s[3]\n"
- "fmla v28.4s, v12.4s, v7.s[3]\n"
- "fmla v17.4s, v13.4s, v4.s[3]\n"
- "fmla v21.4s, v13.4s, v5.s[3]\n"
- "fmla v25.4s, v13.4s, v6.s[3]\n"
- "fmla v29.4s, v13.4s, v7.s[3]\n"
- "fmla v18.4s, v14.4s, v4.s[3]\n"
- "fmla v22.4s, v14.4s, v5.s[3]\n"
- "fmla v26.4s, v14.4s, v6.s[3]\n"
- "fmla v30.4s, v14.4s, v7.s[3]\n"
- "fmla v19.4s, v15.4s, v4.s[3]\n"
- "fmla v23.4s, v15.4s, v5.s[3]\n"
- "fmla v27.4s, v15.4s, v6.s[3]\n"
- "fmla v31.4s, v15.4s, v7.s[3]\n"
- "b 5f\n"
- "4:\n"
- "fmla v16.4s, v8.4s, v0.s[0]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- "fmla v20.4s, v8.4s, v1.s[0]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- "fmla v24.4s, v8.4s, v2.s[0]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- "fmla v28.4s, v8.4s, v3.s[0]\n"
- "ldr d8, [%[b_ptr0]]\n"
- "fmla v17.4s, v9.4s, v0.s[0]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- "fmla v21.4s, v9.4s, v1.s[0]\n"
- "fmla v25.4s, v9.4s, v2.s[0]\n"
- "ins v8.d[1], temploadreg0\n"
- "fmla v29.4s, v9.4s, v3.s[0]\n"
- "ldr d9, [%[b_ptr0], #0x10]\n"
- "fmla v18.4s, v10.4s, v0.s[0]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- "fmla v22.4s, v10.4s, v1.s[0]\n"
- "fmla v26.4s, v10.4s, v2.s[0]\n"
- "ins v9.d[1], temploadreg1\n"
- "fmla v30.4s, v10.4s, v3.s[0]\n"
- "ldr d10, [%[b_ptr0], #0x20]\n"
- "fmla v19.4s, v11.4s, v0.s[0]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- "fmla v23.4s, v11.4s, v1.s[0]\n"
- "fmla v27.4s, v11.4s, v2.s[0]\n"
- "ins v10.d[1], temploadreg2\n"
- "fmla v31.4s, v11.4s, v3.s[0]\n"
- "ldr d11, [%[b_ptr0], #0x30]\n"
- "fmla v16.4s, v12.4s, v0.s[1]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- "fmla v20.4s, v12.4s, v1.s[1]\n"
- "fmla v24.4s, v12.4s, v2.s[1]\n"
- "ins v11.d[1], temploadreg3\n"
- "fmla v28.4s, v12.4s, v3.s[1]\n"
- "ldr d12, [%[b_ptr0], #0x40]\n"
- "fmla v17.4s, v13.4s, v0.s[1]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- "fmla v21.4s, v13.4s, v1.s[1]\n"
- "fmla v25.4s, v13.4s, v2.s[1]\n"
- "ins v12.d[1], temploadreg0\n"
- "fmla v29.4s, v13.4s, v3.s[1]\n"
- "ldr d13, [%[b_ptr0], #0x50]\n"
- "fmla v18.4s, v14.4s, v0.s[1]\n"
- "fmla v22.4s, v14.4s, v1.s[1]\n"
- "fmla v26.4s, v14.4s, v2.s[1]\n"
- "ins v13.d[1], temploadreg1\n"
- "fmla v30.4s, v14.4s, v3.s[1]\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- "fmla v19.4s, v15.4s, v0.s[1]\n"
- "fmla v23.4s, v15.4s, v1.s[1]\n"
- "fmla v27.4s, v15.4s, v2.s[1]\n"
- "ins v14.d[1], temploadreg2\n"
- "fmla v31.4s, v15.4s, v3.s[1]\n"
- "ldr d15, [%[b_ptr0], #0x70]\n"
- "fmla v16.4s, v8.4s, v0.s[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "fmla v20.4s, v8.4s, v1.s[2]\n"
- "ins v15.d[1], temploadreg3\n"
- "fmla v24.4s, v8.4s, v2.s[2]\n"
- "fmla v28.4s, v8.4s, v3.s[2]\n"
- "fmla v17.4s, v9.4s, v0.s[2]\n"
- "fmla v21.4s, v9.4s, v1.s[2]\n"
- "fmla v25.4s, v9.4s, v2.s[2]\n"
- "fmla v29.4s, v9.4s, v3.s[2]\n"
- "fmla v18.4s, v10.4s, v0.s[2]\n"
- "fmla v22.4s, v10.4s, v1.s[2]\n"
- "fmla v26.4s, v10.4s, v2.s[2]\n"
- "fmla v30.4s, v10.4s, v3.s[2]\n"
- "fmla v19.4s, v11.4s, v0.s[2]\n"
- "fmla v23.4s, v11.4s, v1.s[2]\n"
- "fmla v27.4s, v11.4s, v2.s[2]\n"
- "fmla v31.4s, v11.4s, v3.s[2]\n"
- "fmla v16.4s, v12.4s, v0.s[3]\n"
- "fmla v20.4s, v12.4s, v1.s[3]\n"
- "fmla v24.4s, v12.4s, v2.s[3]\n"
- "fmla v28.4s, v12.4s, v3.s[3]\n"
- "fmla v17.4s, v13.4s, v0.s[3]\n"
- "fmla v21.4s, v13.4s, v1.s[3]\n"
- "fmla v25.4s, v13.4s, v2.s[3]\n"
- "fmla v29.4s, v13.4s, v3.s[3]\n"
- "fmla v18.4s, v14.4s, v0.s[3]\n"
- "fmla v22.4s, v14.4s, v1.s[3]\n"
- "fmla v26.4s, v14.4s, v2.s[3]\n"
- "fmla v30.4s, v14.4s, v3.s[3]\n"
- "fmla v19.4s, v15.4s, v0.s[3]\n"
- "fmla v23.4s, v15.4s, v1.s[3]\n"
- "fmla v27.4s, v15.4s, v2.s[3]\n"
- "fmla v31.4s, v15.4s, v3.s[3]\n"
- "5:\n"
- "cbz %[blocks], 6f\n"
- "7:\n"
- "ldr q8, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "ldr s0, [%[a_ptr0]]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "add %[a_ptr0], %[a_ptr0], #0x4\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "add %[b_ptr0], %[b_ptr0], #0x40\n"
- "fmla v16.4s, v8.4s, v0.s[0]\n"
- "ldr s1, [a_ptr1]\n"
- "fmla v17.4s, v9.4s, v0.s[0]\n"
- "add a_ptr1, a_ptr1, #0x4\n"
- "fmla v18.4s, v10.4s, v0.s[0]\n"
- "ldr s2, [a_ptr2]\n"
- "fmla v20.4s, v8.4s, v1.s[0]\n"
- "add a_ptr2, a_ptr2, #0x4\n"
- "fmla v21.4s, v9.4s, v1.s[0]\n"
- "ldr s3, [a_ptr3]\n"
- "fmla v24.4s, v8.4s, v2.s[0]\n"
- "add a_ptr3, a_ptr3, #0x4\n"
- "fmla v25.4s, v9.4s, v2.s[0]\n"
- "fmla v28.4s, v8.4s, v3.s[0]\n"
- "fmla v29.4s, v9.4s, v3.s[0]\n"
- "fmla v22.4s, v10.4s, v1.s[0]\n"
- "fmla v26.4s, v10.4s, v2.s[0]\n"
- "fmla v30.4s, v10.4s, v3.s[0]\n"
- "fmla v19.4s, v11.4s, v0.s[0]\n"
- "fmla v23.4s, v11.4s, v1.s[0]\n"
- "fmla v27.4s, v11.4s, v2.s[0]\n"
- "fmla v31.4s, v11.4s, v3.s[0]\n"
- "b.ne 7b\n"
- "6:\n"
- "ld1r {v14.4s}, [%[minptr]]\n"
- "ld1r {v15.4s}, [%[maxptr]]\n"
- "fmax v16.4s, v16.4s, v14.4s\n"
- "fmax v17.4s, v17.4s, v14.4s\n"
- "fmax v18.4s, v18.4s, v14.4s\n"
- "fmax v19.4s, v19.4s, v14.4s\n"
- "fmin v16.4s, v16.4s, v15.4s\n"
- "fmin v17.4s, v17.4s, v15.4s\n"
- "fmin v18.4s, v18.4s, v15.4s\n"
- "fmin v19.4s, v19.4s, v15.4s\n"
- "str q16, [%[c_ptr0]]\n"
- "fmax v20.4s, v20.4s, v14.4s\n"
- "fmax v21.4s, v21.4s, v14.4s\n"
- "fmax v22.4s, v22.4s, v14.4s\n"
- "str q17, [%[c_ptr0], #0x10]\n"
- "fmax v23.4s, v23.4s, v14.4s\n"
- "fmin v20.4s, v20.4s, v15.4s\n"
- "fmin v21.4s, v21.4s, v15.4s\n"
- "str q18, [%[c_ptr0], #0x20]\n"
- "fmin v22.4s, v22.4s, v15.4s\n"
- "fmin v23.4s, v23.4s, v15.4s\n"
- "fmax v24.4s, v24.4s, v14.4s\n"
- "str q19, [%[c_ptr0], #0x30]\n"
- "fmax v25.4s, v25.4s, v14.4s\n"
- "add %[c_ptr0], %[c_ptr0], #0x40\n"
- "fmax v26.4s, v26.4s, v14.4s\n"
- "str q20, [c_ptr1]\n"
- "fmin v24.4s, v24.4s, v15.4s\n"
- "fmin v25.4s, v25.4s, v15.4s\n"
- "fmax v27.4s, v27.4s, v14.4s\n"
- "str q21, [c_ptr1, #0x10]\n"
- "fmin v26.4s, v26.4s, v15.4s\n"
- "fmax v28.4s, v28.4s, v14.4s\n"
- "fmax v29.4s, v29.4s, v14.4s\n"
- "str q22, [c_ptr1, #0x20]\n"
- "fmin v27.4s, v27.4s, v15.4s\n"
- "fmax v30.4s, v30.4s, v14.4s\n"
- "fmin v28.4s, v28.4s, v15.4s\n"
- "str q23, [c_ptr1, #0x30]\n"
- "fmin v29.4s, v29.4s, v15.4s\n"
- "fmax v31.4s, v31.4s, v14.4s\n"
- "fmin v30.4s, v30.4s, v15.4s\n"
- "str q24, [c_ptr2]\n"
- "fmin v31.4s, v31.4s, v15.4s\n"
- "str q25, [c_ptr2, #0x10]\n"
- "str q26, [c_ptr2, #0x20]\n"
- "str q27, [c_ptr2, #0x30]\n"
- "str q28, [c_ptr3]\n"
- "str q29, [c_ptr3, #0x10]\n"
- "str q30, [c_ptr3, #0x20]\n"
- "str q31, [c_ptr3, #0x30]\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq a_ptr3\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- ".unreq c_ptr3\n"
- ".unreq temploadreg0\n"
- ".unreq temploadreg1\n"
- ".unreq temploadreg2\n"
- ".unreq temploadreg3\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks)
- : [width] "r" (width), [accumulate] "r" (static_cast<uint64_t>(accumulate)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr)
- : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "cc", "memory"
- );
- break;
- }
- if (use_result_buffer) {
- for(int cy=0; cy<std::min(M-y, 4); cy++) {
- for(unsigned int cx=0; cx<width; cx++) {
- c_ptr_real[cy * ldc + cx] = result_buffer[cy * 16 + cx];
- }
- }
- }
- }
- }
-}
-
-} // namespace arm_gemm
-
-#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_16x4/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_16x4/generic.cpp
deleted file mode 100644
index 016bef4..0000000
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_16x4/generic.cpp
+++ /dev/null
@@ -1,1802 +0,0 @@
-/*
- * Copyright (c) 2018-2020 Arm Limited.
- *
- * SPDX-License-Identifier: MIT
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to
- * deal in the Software without restriction, including without limitation the
- * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
- * sell copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-#ifdef __aarch64__
-
-#include <algorithm>
-
-#include "arm_gemm.hpp"
-
-#include "../../asmlib.hpp"
-#include "../../utils.hpp"
-
-namespace arm_gemm {
-
-void a64_hybrid_fp32_mla_16x4(const float *A, int lda, const float *B, float *C, int ldc, int M, int N, int K, const float *bias, Activation act, bool accumulate) {
- const int K_stride = K;
- const long loops_count = ((K + 4) / 8) - 1;
- K -= loops_count * 8;
- const long regs_count = (K / 4) - 1;
- K -= (regs_count + 1) * 4;
- const long blocks_count = K / 1;
- float nullbias[16];
- if (!accumulate && !bias) {
- memset(nullbias, 0, (16 * sizeof(float)));
- }
- float minval = - static_cast<float>(std::numeric_limits<float>::infinity());
- float maxval = static_cast<float>(std::numeric_limits<float>::infinity());
- const float * const minptr = &minval;
- const float * const maxptr = &maxval;
-
- switch(act.type)
- {
- default:
- case Activation::Type::None:
- break;
- case Activation::Type::BoundedReLU:
- maxval = static_cast<float>(act.param1);
- /* fall through */
- case Activation::Type::ReLU:
- minval = 0.0f;
- break;
- }
-
- int rows_to_compute;
-
- for (int y=0; y<M; y+=rows_to_compute) {
- const float * const a_ptr0_base = A + (y * lda);
- const unsigned long ldab = lda * sizeof(float);
-
- float *c_ptr0 = C + (y * ldc);
-
- rows_to_compute = M-y;
- if (rows_to_compute > 4) {
- if (rows_to_compute % 4) {
- rows_to_compute = 4 - 1;
- } else {
- rows_to_compute = 4;
- }
- }
-
- for (int x0=0; x0<N; x0+=16ul) {
- const long width = std::min((unsigned long)N-x0, 16ul);
- long loops = loops_count;
- long regs = regs_count;
- long blocks = blocks_count;
- const float *a_ptr0 = a_ptr0_base;
- const float *b_ptr0 = B + (K_stride * x0);
- const bool use_result_buffer = (width < 16);
- float result_buffer[64];
- const unsigned long ldcb = (use_result_buffer ? 16 : ldc) * sizeof(float);
- float *c_ptr_real = c_ptr0;
- if (use_result_buffer && accumulate) {
- for(int cy=0; cy<std::min(M-y, 4); cy++) {
- for(unsigned int cx=0; cx<width; cx++) {
- result_buffer[cy * 16 + cx] = c_ptr_real[cy * ldc + cx];
- }
- }
- }
- if (use_result_buffer) {
- c_ptr0 = result_buffer;
- }
- const float *biasptr = bias ? bias+x0 : nullbias;
-
- switch(rows_to_compute) {
- case 1:
- __asm __volatile (
- "cbnz %[accumulate], 1f\n"
- "ldr q16, [%[biasptr]]\n"
- "ldr q17, [%[biasptr], #0x10]\n"
- "ldr q18, [%[biasptr], #0x20]\n"
- "ldr q19, [%[biasptr], #0x30]\n"
- "ldr q0, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ldr q8, [%[b_ptr0]]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "ldr q16, [%[c_ptr0]]\n"
- "ldr q17, [%[c_ptr0], #0x10]\n"
- "ldr q18, [%[c_ptr0], #0x20]\n"
- "ldr q19, [%[c_ptr0], #0x30]\n"
- "ldr q0, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ldr q8, [%[b_ptr0]]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "cbz %[loops], 2f\n"
- "3:\n"
- "fmla v16.4s, v8.4s, v0.s[0]\n"
- "ldr q15, [%[b_ptr0], #-0x10]\n"
- "fmla v17.4s, v9.4s, v0.s[0]\n"
- "ldr q4, [%[a_ptr0]]\n"
- "fmla v18.4s, v10.4s, v0.s[0]\n"
- "ldr q8, [%[b_ptr0]]\n"
- "fmla v19.4s, v11.4s, v0.s[0]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "fmla v16.4s, v12.4s, v0.s[1]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "fmla v17.4s, v13.4s, v0.s[1]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "fmla v18.4s, v14.4s, v0.s[1]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- "fmla v19.4s, v15.4s, v0.s[1]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- "fmla v16.4s, v8.4s, v0.s[2]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- "fmla v17.4s, v9.4s, v0.s[2]\n"
- "ldr q15, [%[b_ptr0], #0x70]\n"
- "fmla v18.4s, v10.4s, v0.s[2]\n"
- "subs %[loops], %[loops], #0x1\n"
- "fmla v19.4s, v11.4s, v0.s[2]\n"
- "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n"
- "fmla v16.4s, v12.4s, v0.s[3]\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- "fmla v17.4s, v13.4s, v0.s[3]\n"
- "ldr q8, [%[b_ptr0], #-0x80]\n"
- "fmla v18.4s, v14.4s, v0.s[3]\n"
- "ldr q9, [%[b_ptr0], #-0x70]\n"
- "fmla v19.4s, v15.4s, v0.s[3]\n"
- "ldr q10, [%[b_ptr0], #-0x60]\n"
- "ldr q11, [%[b_ptr0], #-0x50]\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
- "fmla v16.4s, v8.4s, v4.s[0]\n"
- "ldr q12, [%[b_ptr0], #-0x40]\n"
- "fmla v17.4s, v9.4s, v4.s[0]\n"
- "ldr q13, [%[b_ptr0], #-0x30]\n"
- "fmla v18.4s, v10.4s, v4.s[0]\n"
- "ldr q14, [%[b_ptr0], #-0x20]\n"
- "fmla v19.4s, v11.4s, v4.s[0]\n"
- "ldr q15, [%[b_ptr0], #-0x10]\n"
- "fmla v16.4s, v12.4s, v4.s[1]\n"
- "ldr q0, [%[a_ptr0], #-0x10]\n"
- "fmla v17.4s, v13.4s, v4.s[1]\n"
- "ldr q8, [%[b_ptr0]]\n"
- "fmla v18.4s, v14.4s, v4.s[1]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "fmla v19.4s, v15.4s, v4.s[1]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "fmla v16.4s, v8.4s, v4.s[2]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- "fmla v17.4s, v9.4s, v4.s[2]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- "fmla v18.4s, v10.4s, v4.s[2]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- "fmla v19.4s, v11.4s, v4.s[2]\n"
- "ldr q15, [%[b_ptr0], #0x70]\n"
- "fmla v16.4s, v12.4s, v4.s[3]\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- "fmla v17.4s, v13.4s, v4.s[3]\n"
- "ldr q8, [%[b_ptr0], #-0x80]\n"
- "fmla v18.4s, v14.4s, v4.s[3]\n"
- "ldr q9, [%[b_ptr0], #-0x70]\n"
- "fmla v19.4s, v15.4s, v4.s[3]\n"
- "ldr q10, [%[b_ptr0], #-0x60]\n"
- "ldr q11, [%[b_ptr0], #-0x50]\n"
- "ldr q12, [%[b_ptr0], #-0x40]\n"
- "ldr q13, [%[b_ptr0], #-0x30]\n"
- "ldr q14, [%[b_ptr0], #-0x20]\n"
- "b.ne 3b\n"
- "2:\n"
- "ldr q15, [%[b_ptr0], #-0x10]\n"
- "prfm PSTL1KEEP, [%[c_ptr0]]\n"
- "cbz %[regs], 4f\n"
- "fmla v16.4s, v8.4s, v0.s[0]\n"
- "ldr q4, [%[a_ptr0]]\n"
- "fmla v17.4s, v9.4s, v0.s[0]\n"
- "ldr q8, [%[b_ptr0]]\n"
- "fmla v18.4s, v10.4s, v0.s[0]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "fmla v19.4s, v11.4s, v0.s[0]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "fmla v16.4s, v12.4s, v0.s[1]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "fmla v17.4s, v13.4s, v0.s[1]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- "fmla v18.4s, v14.4s, v0.s[1]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- "fmla v19.4s, v15.4s, v0.s[1]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- "fmla v16.4s, v8.4s, v0.s[2]\n"
- "ldr q15, [%[b_ptr0], #0x70]\n"
- "fmla v17.4s, v9.4s, v0.s[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- "fmla v18.4s, v10.4s, v0.s[2]\n"
- "ldr q8, [%[b_ptr0], #-0x80]\n"
- "fmla v19.4s, v11.4s, v0.s[2]\n"
- "ldr q9, [%[b_ptr0], #-0x70]\n"
- "fmla v16.4s, v12.4s, v0.s[3]\n"
- "ldr q10, [%[b_ptr0], #-0x60]\n"
- "fmla v17.4s, v13.4s, v0.s[3]\n"
- "ldr q11, [%[b_ptr0], #-0x50]\n"
- "fmla v18.4s, v14.4s, v0.s[3]\n"
- "ldr q12, [%[b_ptr0], #-0x40]\n"
- "fmla v19.4s, v15.4s, v0.s[3]\n"
- "ldr q13, [%[b_ptr0], #-0x30]\n"
- "fmla v16.4s, v8.4s, v4.s[0]\n"
- "ldr q14, [%[b_ptr0], #-0x20]\n"
- "fmla v17.4s, v9.4s, v4.s[0]\n"
- "ldr q15, [%[b_ptr0], #-0x10]\n"
- "fmla v18.4s, v10.4s, v4.s[0]\n"
- "ldr q8, [%[b_ptr0]]\n"
- "fmla v19.4s, v11.4s, v4.s[0]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "fmla v16.4s, v12.4s, v4.s[1]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "fmla v17.4s, v13.4s, v4.s[1]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "fmla v18.4s, v14.4s, v4.s[1]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- "fmla v19.4s, v15.4s, v4.s[1]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- "fmla v16.4s, v8.4s, v4.s[2]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- "fmla v17.4s, v9.4s, v4.s[2]\n"
- "ldr q15, [%[b_ptr0], #0x70]\n"
- "fmla v18.4s, v10.4s, v4.s[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "fmla v19.4s, v11.4s, v4.s[2]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "fmla v16.4s, v12.4s, v4.s[3]\n"
- "fmla v17.4s, v13.4s, v4.s[3]\n"
- "fmla v18.4s, v14.4s, v4.s[3]\n"
- "fmla v19.4s, v15.4s, v4.s[3]\n"
- "b 5f\n"
- "4:\n"
- "fmla v16.4s, v8.4s, v0.s[0]\n"
- "ldr q8, [%[b_ptr0]]\n"
- "fmla v17.4s, v9.4s, v0.s[0]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "fmla v18.4s, v10.4s, v0.s[0]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "fmla v19.4s, v11.4s, v0.s[0]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "fmla v16.4s, v12.4s, v0.s[1]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- "fmla v17.4s, v13.4s, v0.s[1]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- "fmla v18.4s, v14.4s, v0.s[1]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- "fmla v19.4s, v15.4s, v0.s[1]\n"
- "ldr q15, [%[b_ptr0], #0x70]\n"
- "fmla v16.4s, v8.4s, v0.s[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "fmla v17.4s, v9.4s, v0.s[2]\n"
- "fmla v18.4s, v10.4s, v0.s[2]\n"
- "fmla v19.4s, v11.4s, v0.s[2]\n"
- "fmla v16.4s, v12.4s, v0.s[3]\n"
- "fmla v17.4s, v13.4s, v0.s[3]\n"
- "fmla v18.4s, v14.4s, v0.s[3]\n"
- "fmla v19.4s, v15.4s, v0.s[3]\n"
- "5:\n"
- "cbz %[blocks], 6f\n"
- "7:\n"
- "ldr q8, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "ldr s0, [%[a_ptr0]]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "add %[a_ptr0], %[a_ptr0], #0x4\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "add %[b_ptr0], %[b_ptr0], #0x40\n"
- "fmla v16.4s, v8.4s, v0.s[0]\n"
- "fmla v17.4s, v9.4s, v0.s[0]\n"
- "fmla v18.4s, v10.4s, v0.s[0]\n"
- "fmla v19.4s, v11.4s, v0.s[0]\n"
- "b.ne 7b\n"
- "6:\n"
- "ld1r {v14.4s}, [%[minptr]]\n"
- "ld1r {v15.4s}, [%[maxptr]]\n"
- "fmax v16.4s, v16.4s, v14.4s\n"
- "fmax v17.4s, v17.4s, v14.4s\n"
- "fmax v18.4s, v18.4s, v14.4s\n"
- "fmax v19.4s, v19.4s, v14.4s\n"
- "fmin v16.4s, v16.4s, v15.4s\n"
- "fmin v17.4s, v17.4s, v15.4s\n"
- "fmin v18.4s, v18.4s, v15.4s\n"
- "fmin v19.4s, v19.4s, v15.4s\n"
- "str q16, [%[c_ptr0]]\n"
- "str q17, [%[c_ptr0], #0x10]\n"
- "str q18, [%[c_ptr0], #0x20]\n"
- "str q19, [%[c_ptr0], #0x30]\n"
- "add %[c_ptr0], %[c_ptr0], #0x40\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks)
- : [width] "r" (width), [accumulate] "r" (static_cast<uint64_t>(accumulate)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr)
- : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "cc", "memory"
- );
- break;
- case 2:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "c_ptr1 .req X1\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "cbnz %[accumulate], 1f\n"
- "ldr q16, [%[biasptr]]\n"
- "ldr q17, [%[biasptr], #0x10]\n"
- "ldr q18, [%[biasptr], #0x20]\n"
- "ldr q19, [%[biasptr], #0x30]\n"
- "mov v20.16b, v16.16b\n"
- "ldr q0, [%[a_ptr0]]\n"
- "mov v21.16b, v17.16b\n"
- "ldr q1, [a_ptr1]\n"
- "mov v22.16b, v18.16b\n"
- "ldr q8, [%[b_ptr0]]\n"
- "mov v23.16b, v19.16b\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "ldr q16, [%[c_ptr0]]\n"
- "ldr q17, [%[c_ptr0], #0x10]\n"
- "ldr q18, [%[c_ptr0], #0x20]\n"
- "ldr q19, [%[c_ptr0], #0x30]\n"
- "ldr q20, [c_ptr1]\n"
- "ldr q21, [c_ptr1, #0x10]\n"
- "ldr q22, [c_ptr1, #0x20]\n"
- "ldr q23, [c_ptr1, #0x30]\n"
- "ldr q0, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ldr q1, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "ldr q8, [%[b_ptr0]]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "cbz %[loops], 2f\n"
- "3:\n"
- "fmla v16.4s, v8.4s, v0.s[0]\n"
- "ldr q15, [%[b_ptr0], #-0x10]\n"
- "fmla v20.4s, v8.4s, v1.s[0]\n"
- "ldr q4, [%[a_ptr0]]\n"
- "fmla v17.4s, v9.4s, v0.s[0]\n"
- "ldr q5, [a_ptr1]\n"
- "fmla v21.4s, v9.4s, v1.s[0]\n"
- "ldr q8, [%[b_ptr0]]\n"
- "fmla v18.4s, v10.4s, v0.s[0]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "fmla v22.4s, v10.4s, v1.s[0]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "fmla v19.4s, v11.4s, v0.s[0]\n"
- "subs %[loops], %[loops], #0x1\n"
- "fmla v23.4s, v11.4s, v1.s[0]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "fmla v16.4s, v12.4s, v0.s[1]\n"
- "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n"
- "fmla v20.4s, v12.4s, v1.s[1]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- "fmla v17.4s, v13.4s, v0.s[1]\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
- "fmla v21.4s, v13.4s, v1.s[1]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- "fmla v18.4s, v14.4s, v0.s[1]\n"
- "add a_ptr1, a_ptr1, #0x20\n"
- "fmla v22.4s, v14.4s, v1.s[1]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- "fmla v19.4s, v15.4s, v0.s[1]\n"
- "prfm PLDL1KEEP, [a_ptr1, #0x40]\n"
- "fmla v23.4s, v15.4s, v1.s[1]\n"
- "ldr q15, [%[b_ptr0], #0x70]\n"
- "fmla v16.4s, v8.4s, v0.s[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- "fmla v20.4s, v8.4s, v1.s[2]\n"
- "ldr q8, [%[b_ptr0], #-0x80]\n"
- "fmla v17.4s, v9.4s, v0.s[2]\n"
- "fmla v21.4s, v9.4s, v1.s[2]\n"
- "ldr q9, [%[b_ptr0], #-0x70]\n"
- "fmla v18.4s, v10.4s, v0.s[2]\n"
- "fmla v22.4s, v10.4s, v1.s[2]\n"
- "ldr q10, [%[b_ptr0], #-0x60]\n"
- "fmla v19.4s, v11.4s, v0.s[2]\n"
- "fmla v23.4s, v11.4s, v1.s[2]\n"
- "ldr q11, [%[b_ptr0], #-0x50]\n"
- "fmla v16.4s, v12.4s, v0.s[3]\n"
- "fmla v20.4s, v12.4s, v1.s[3]\n"
- "ldr q12, [%[b_ptr0], #-0x40]\n"
- "fmla v17.4s, v13.4s, v0.s[3]\n"
- "fmla v21.4s, v13.4s, v1.s[3]\n"
- "ldr q13, [%[b_ptr0], #-0x30]\n"
- "fmla v18.4s, v14.4s, v0.s[3]\n"
- "fmla v22.4s, v14.4s, v1.s[3]\n"
- "ldr q14, [%[b_ptr0], #-0x20]\n"
- "fmla v19.4s, v15.4s, v0.s[3]\n"
- "ldr q0, [%[a_ptr0], #-0x10]\n"
- "fmla v23.4s, v15.4s, v1.s[3]\n"
- "ldr q15, [%[b_ptr0], #-0x10]\n"
- "fmla v16.4s, v8.4s, v4.s[0]\n"
- "ldr q1, [a_ptr1, #-0x10]\n"
- "fmla v20.4s, v8.4s, v5.s[0]\n"
- "ldr q8, [%[b_ptr0]]\n"
- "fmla v17.4s, v9.4s, v4.s[0]\n"
- "fmla v21.4s, v9.4s, v5.s[0]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "fmla v18.4s, v10.4s, v4.s[0]\n"
- "fmla v22.4s, v10.4s, v5.s[0]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "fmla v19.4s, v11.4s, v4.s[0]\n"
- "fmla v23.4s, v11.4s, v5.s[0]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "fmla v16.4s, v12.4s, v4.s[1]\n"
- "fmla v20.4s, v12.4s, v5.s[1]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- "fmla v17.4s, v13.4s, v4.s[1]\n"
- "fmla v21.4s, v13.4s, v5.s[1]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- "fmla v18.4s, v14.4s, v4.s[1]\n"
- "fmla v22.4s, v14.4s, v5.s[1]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- "fmla v19.4s, v15.4s, v4.s[1]\n"
- "fmla v23.4s, v15.4s, v5.s[1]\n"
- "ldr q15, [%[b_ptr0], #0x70]\n"
- "fmla v16.4s, v8.4s, v4.s[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- "fmla v20.4s, v8.4s, v5.s[2]\n"
- "ldr q8, [%[b_ptr0], #-0x80]\n"
- "fmla v17.4s, v9.4s, v4.s[2]\n"
- "fmla v21.4s, v9.4s, v5.s[2]\n"
- "ldr q9, [%[b_ptr0], #-0x70]\n"
- "fmla v18.4s, v10.4s, v4.s[2]\n"
- "fmla v22.4s, v10.4s, v5.s[2]\n"
- "ldr q10, [%[b_ptr0], #-0x60]\n"
- "fmla v19.4s, v11.4s, v4.s[2]\n"
- "fmla v23.4s, v11.4s, v5.s[2]\n"
- "ldr q11, [%[b_ptr0], #-0x50]\n"
- "fmla v16.4s, v12.4s, v4.s[3]\n"
- "fmla v20.4s, v12.4s, v5.s[3]\n"
- "ldr q12, [%[b_ptr0], #-0x40]\n"
- "fmla v17.4s, v13.4s, v4.s[3]\n"
- "fmla v21.4s, v13.4s, v5.s[3]\n"
- "ldr q13, [%[b_ptr0], #-0x30]\n"
- "fmla v18.4s, v14.4s, v4.s[3]\n"
- "fmla v22.4s, v14.4s, v5.s[3]\n"
- "ldr q14, [%[b_ptr0], #-0x20]\n"
- "fmla v19.4s, v15.4s, v4.s[3]\n"
- "fmla v23.4s, v15.4s, v5.s[3]\n"
- "b.ne 3b\n"
- "2:\n"
- "ldr q15, [%[b_ptr0], #-0x10]\n"
- "prfm PSTL1KEEP, [%[c_ptr0]]\n"
- "prfm PSTL1KEEP, [c_ptr1]\n"
- "cbz %[regs], 4f\n"
- "fmla v16.4s, v8.4s, v0.s[0]\n"
- "ldr q4, [%[a_ptr0]]\n"
- "fmla v20.4s, v8.4s, v1.s[0]\n"
- "ldr q5, [a_ptr1]\n"
- "fmla v17.4s, v9.4s, v0.s[0]\n"
- "ldr q8, [%[b_ptr0]]\n"
- "fmla v21.4s, v9.4s, v1.s[0]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "fmla v18.4s, v10.4s, v0.s[0]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "fmla v22.4s, v10.4s, v1.s[0]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "fmla v19.4s, v11.4s, v0.s[0]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "fmla v23.4s, v11.4s, v1.s[0]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "fmla v16.4s, v12.4s, v0.s[1]\n"
- "fmla v20.4s, v12.4s, v1.s[1]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- "fmla v17.4s, v13.4s, v0.s[1]\n"
- "fmla v21.4s, v13.4s, v1.s[1]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- "fmla v18.4s, v14.4s, v0.s[1]\n"
- "fmla v22.4s, v14.4s, v1.s[1]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- "fmla v19.4s, v15.4s, v0.s[1]\n"
- "fmla v23.4s, v15.4s, v1.s[1]\n"
- "ldr q15, [%[b_ptr0], #0x70]\n"
- "fmla v16.4s, v8.4s, v0.s[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- "fmla v20.4s, v8.4s, v1.s[2]\n"
- "ldr q8, [%[b_ptr0], #-0x80]\n"
- "fmla v17.4s, v9.4s, v0.s[2]\n"
- "fmla v21.4s, v9.4s, v1.s[2]\n"
- "ldr q9, [%[b_ptr0], #-0x70]\n"
- "fmla v18.4s, v10.4s, v0.s[2]\n"
- "fmla v22.4s, v10.4s, v1.s[2]\n"
- "ldr q10, [%[b_ptr0], #-0x60]\n"
- "fmla v19.4s, v11.4s, v0.s[2]\n"
- "fmla v23.4s, v11.4s, v1.s[2]\n"
- "ldr q11, [%[b_ptr0], #-0x50]\n"
- "fmla v16.4s, v12.4s, v0.s[3]\n"
- "fmla v20.4s, v12.4s, v1.s[3]\n"
- "ldr q12, [%[b_ptr0], #-0x40]\n"
- "fmla v17.4s, v13.4s, v0.s[3]\n"
- "fmla v21.4s, v13.4s, v1.s[3]\n"
- "ldr q13, [%[b_ptr0], #-0x30]\n"
- "fmla v18.4s, v14.4s, v0.s[3]\n"
- "fmla v22.4s, v14.4s, v1.s[3]\n"
- "ldr q14, [%[b_ptr0], #-0x20]\n"
- "fmla v19.4s, v15.4s, v0.s[3]\n"
- "fmla v23.4s, v15.4s, v1.s[3]\n"
- "ldr q15, [%[b_ptr0], #-0x10]\n"
- "fmla v16.4s, v8.4s, v4.s[0]\n"
- "fmla v20.4s, v8.4s, v5.s[0]\n"
- "ldr q8, [%[b_ptr0]]\n"
- "fmla v17.4s, v9.4s, v4.s[0]\n"
- "fmla v21.4s, v9.4s, v5.s[0]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "fmla v18.4s, v10.4s, v4.s[0]\n"
- "fmla v22.4s, v10.4s, v5.s[0]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "fmla v19.4s, v11.4s, v4.s[0]\n"
- "fmla v23.4s, v11.4s, v5.s[0]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "fmla v16.4s, v12.4s, v4.s[1]\n"
- "fmla v20.4s, v12.4s, v5.s[1]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- "fmla v17.4s, v13.4s, v4.s[1]\n"
- "fmla v21.4s, v13.4s, v5.s[1]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- "fmla v18.4s, v14.4s, v4.s[1]\n"
- "fmla v22.4s, v14.4s, v5.s[1]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- "fmla v19.4s, v15.4s, v4.s[1]\n"
- "fmla v23.4s, v15.4s, v5.s[1]\n"
- "ldr q15, [%[b_ptr0], #0x70]\n"
- "fmla v16.4s, v8.4s, v4.s[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "fmla v20.4s, v8.4s, v5.s[2]\n"
- "fmla v17.4s, v9.4s, v4.s[2]\n"
- "fmla v21.4s, v9.4s, v5.s[2]\n"
- "fmla v18.4s, v10.4s, v4.s[2]\n"
- "fmla v22.4s, v10.4s, v5.s[2]\n"
- "fmla v19.4s, v11.4s, v4.s[2]\n"
- "fmla v23.4s, v11.4s, v5.s[2]\n"
- "fmla v16.4s, v12.4s, v4.s[3]\n"
- "fmla v20.4s, v12.4s, v5.s[3]\n"
- "fmla v17.4s, v13.4s, v4.s[3]\n"
- "fmla v21.4s, v13.4s, v5.s[3]\n"
- "fmla v18.4s, v14.4s, v4.s[3]\n"
- "fmla v22.4s, v14.4s, v5.s[3]\n"
- "fmla v19.4s, v15.4s, v4.s[3]\n"
- "fmla v23.4s, v15.4s, v5.s[3]\n"
- "b 5f\n"
- "4:\n"
- "fmla v16.4s, v8.4s, v0.s[0]\n"
- "fmla v20.4s, v8.4s, v1.s[0]\n"
- "ldr q8, [%[b_ptr0]]\n"
- "fmla v17.4s, v9.4s, v0.s[0]\n"
- "fmla v21.4s, v9.4s, v1.s[0]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "fmla v18.4s, v10.4s, v0.s[0]\n"
- "fmla v22.4s, v10.4s, v1.s[0]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "fmla v19.4s, v11.4s, v0.s[0]\n"
- "fmla v23.4s, v11.4s, v1.s[0]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "fmla v16.4s, v12.4s, v0.s[1]\n"
- "fmla v20.4s, v12.4s, v1.s[1]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- "fmla v17.4s, v13.4s, v0.s[1]\n"
- "fmla v21.4s, v13.4s, v1.s[1]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- "fmla v18.4s, v14.4s, v0.s[1]\n"
- "fmla v22.4s, v14.4s, v1.s[1]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- "fmla v19.4s, v15.4s, v0.s[1]\n"
- "fmla v23.4s, v15.4s, v1.s[1]\n"
- "ldr q15, [%[b_ptr0], #0x70]\n"
- "fmla v16.4s, v8.4s, v0.s[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "fmla v20.4s, v8.4s, v1.s[2]\n"
- "fmla v17.4s, v9.4s, v0.s[2]\n"
- "fmla v21.4s, v9.4s, v1.s[2]\n"
- "fmla v18.4s, v10.4s, v0.s[2]\n"
- "fmla v22.4s, v10.4s, v1.s[2]\n"
- "fmla v19.4s, v11.4s, v0.s[2]\n"
- "fmla v23.4s, v11.4s, v1.s[2]\n"
- "fmla v16.4s, v12.4s, v0.s[3]\n"
- "fmla v20.4s, v12.4s, v1.s[3]\n"
- "fmla v17.4s, v13.4s, v0.s[3]\n"
- "fmla v21.4s, v13.4s, v1.s[3]\n"
- "fmla v18.4s, v14.4s, v0.s[3]\n"
- "fmla v22.4s, v14.4s, v1.s[3]\n"
- "fmla v19.4s, v15.4s, v0.s[3]\n"
- "fmla v23.4s, v15.4s, v1.s[3]\n"
- "5:\n"
- "cbz %[blocks], 6f\n"
- "7:\n"
- "ldr q8, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "ldr s0, [%[a_ptr0]]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "add %[a_ptr0], %[a_ptr0], #0x4\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "add %[b_ptr0], %[b_ptr0], #0x40\n"
- "fmla v16.4s, v8.4s, v0.s[0]\n"
- "ldr s1, [a_ptr1]\n"
- "fmla v17.4s, v9.4s, v0.s[0]\n"
- "add a_ptr1, a_ptr1, #0x4\n"
- "fmla v18.4s, v10.4s, v0.s[0]\n"
- "fmla v20.4s, v8.4s, v1.s[0]\n"
- "fmla v21.4s, v9.4s, v1.s[0]\n"
- "fmla v22.4s, v10.4s, v1.s[0]\n"
- "fmla v19.4s, v11.4s, v0.s[0]\n"
- "fmla v23.4s, v11.4s, v1.s[0]\n"
- "b.ne 7b\n"
- "6:\n"
- "ld1r {v14.4s}, [%[minptr]]\n"
- "ld1r {v15.4s}, [%[maxptr]]\n"
- "fmax v16.4s, v16.4s, v14.4s\n"
- "fmax v17.4s, v17.4s, v14.4s\n"
- "fmax v18.4s, v18.4s, v14.4s\n"
- "fmax v19.4s, v19.4s, v14.4s\n"
- "fmin v16.4s, v16.4s, v15.4s\n"
- "fmin v17.4s, v17.4s, v15.4s\n"
- "fmin v18.4s, v18.4s, v15.4s\n"
- "fmin v19.4s, v19.4s, v15.4s\n"
- "str q16, [%[c_ptr0]]\n"
- "fmax v20.4s, v20.4s, v14.4s\n"
- "fmax v21.4s, v21.4s, v14.4s\n"
- "fmax v22.4s, v22.4s, v14.4s\n"
- "str q17, [%[c_ptr0], #0x10]\n"
- "fmax v23.4s, v23.4s, v14.4s\n"
- "fmin v20.4s, v20.4s, v15.4s\n"
- "fmin v21.4s, v21.4s, v15.4s\n"
- "str q18, [%[c_ptr0], #0x20]\n"
- "fmin v22.4s, v22.4s, v15.4s\n"
- "fmin v23.4s, v23.4s, v15.4s\n"
- "str q19, [%[c_ptr0], #0x30]\n"
- "add %[c_ptr0], %[c_ptr0], #0x40\n"
- "str q20, [c_ptr1]\n"
- "str q21, [c_ptr1, #0x10]\n"
- "str q22, [c_ptr1, #0x20]\n"
- "str q23, [c_ptr1, #0x30]\n"
- ".unreq a_ptr1\n"
- ".unreq c_ptr1\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks)
- : [width] "r" (width), [accumulate] "r" (static_cast<uint64_t>(accumulate)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr)
- : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "cc", "memory"
- );
- break;
- case 3:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "c_ptr1 .req X2\n"
- "c_ptr2 .req X3\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "cbnz %[accumulate], 1f\n"
- "ldr q16, [%[biasptr]]\n"
- "ldr q17, [%[biasptr], #0x10]\n"
- "ldr q18, [%[biasptr], #0x20]\n"
- "ldr q19, [%[biasptr], #0x30]\n"
- "mov v20.16b, v16.16b\n"
- "ldr q0, [%[a_ptr0]]\n"
- "mov v21.16b, v17.16b\n"
- "ldr q1, [a_ptr1]\n"
- "mov v22.16b, v18.16b\n"
- "ldr q2, [a_ptr2]\n"
- "mov v23.16b, v19.16b\n"
- "ldr q8, [%[b_ptr0]]\n"
- "mov v24.16b, v16.16b\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "mov v25.16b, v17.16b\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "mov v26.16b, v18.16b\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "mov v27.16b, v19.16b\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "ldr q16, [%[c_ptr0]]\n"
- "ldr q17, [%[c_ptr0], #0x10]\n"
- "ldr q18, [%[c_ptr0], #0x20]\n"
- "ldr q19, [%[c_ptr0], #0x30]\n"
- "ldr q20, [c_ptr1]\n"
- "ldr q21, [c_ptr1, #0x10]\n"
- "ldr q22, [c_ptr1, #0x20]\n"
- "ldr q23, [c_ptr1, #0x30]\n"
- "ldr q24, [c_ptr2]\n"
- "ldr q25, [c_ptr2, #0x10]\n"
- "ldr q26, [c_ptr2, #0x20]\n"
- "ldr q27, [c_ptr2, #0x30]\n"
- "ldr q0, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ldr q1, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "ldr q2, [a_ptr2]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "ldr q8, [%[b_ptr0]]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "cbz %[loops], 2f\n"
- "3:\n"
- "fmla v16.4s, v8.4s, v0.s[0]\n"
- "ldr q15, [%[b_ptr0], #-0x10]\n"
- "fmla v20.4s, v8.4s, v1.s[0]\n"
- "ldr q4, [%[a_ptr0]]\n"
- "fmla v24.4s, v8.4s, v2.s[0]\n"
- "ldr q5, [a_ptr1]\n"
- "fmla v17.4s, v9.4s, v0.s[0]\n"
- "ldr q6, [a_ptr2]\n"
- "fmla v21.4s, v9.4s, v1.s[0]\n"
- "ldr q8, [%[b_ptr0]]\n"
- "fmla v25.4s, v9.4s, v2.s[0]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "fmla v18.4s, v10.4s, v0.s[0]\n"
- "subs %[loops], %[loops], #0x1\n"
- "fmla v22.4s, v10.4s, v1.s[0]\n"
- "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n"
- "fmla v26.4s, v10.4s, v2.s[0]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "fmla v19.4s, v11.4s, v0.s[0]\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
- "fmla v23.4s, v11.4s, v1.s[0]\n"
- "add a_ptr1, a_ptr1, #0x20\n"
- "fmla v27.4s, v11.4s, v2.s[0]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "fmla v16.4s, v12.4s, v0.s[1]\n"
- "add a_ptr2, a_ptr2, #0x20\n"
- "fmla v20.4s, v12.4s, v1.s[1]\n"
- "prfm PLDL1KEEP, [a_ptr1, #0x40]\n"
- "fmla v24.4s, v12.4s, v2.s[1]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- "fmla v17.4s, v13.4s, v0.s[1]\n"
- "prfm PLDL1KEEP, [a_ptr2, #0x40]\n"
- "fmla v21.4s, v13.4s, v1.s[1]\n"
- "fmla v25.4s, v13.4s, v2.s[1]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- "fmla v18.4s, v14.4s, v0.s[1]\n"
- "fmla v22.4s, v14.4s, v1.s[1]\n"
- "fmla v26.4s, v14.4s, v2.s[1]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- "fmla v19.4s, v15.4s, v0.s[1]\n"
- "fmla v23.4s, v15.4s, v1.s[1]\n"
- "fmla v27.4s, v15.4s, v2.s[1]\n"
- "ldr q15, [%[b_ptr0], #0x70]\n"
- "fmla v16.4s, v8.4s, v0.s[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- "fmla v20.4s, v8.4s, v1.s[2]\n"
- "fmla v24.4s, v8.4s, v2.s[2]\n"
- "ldr q8, [%[b_ptr0], #-0x80]\n"
- "fmla v17.4s, v9.4s, v0.s[2]\n"
- "fmla v21.4s, v9.4s, v1.s[2]\n"
- "fmla v25.4s, v9.4s, v2.s[2]\n"
- "ldr q9, [%[b_ptr0], #-0x70]\n"
- "fmla v18.4s, v10.4s, v0.s[2]\n"
- "fmla v22.4s, v10.4s, v1.s[2]\n"
- "fmla v26.4s, v10.4s, v2.s[2]\n"
- "ldr q10, [%[b_ptr0], #-0x60]\n"
- "fmla v19.4s, v11.4s, v0.s[2]\n"
- "fmla v23.4s, v11.4s, v1.s[2]\n"
- "fmla v27.4s, v11.4s, v2.s[2]\n"
- "ldr q11, [%[b_ptr0], #-0x50]\n"
- "fmla v16.4s, v12.4s, v0.s[3]\n"
- "fmla v20.4s, v12.4s, v1.s[3]\n"
- "fmla v24.4s, v12.4s, v2.s[3]\n"
- "ldr q12, [%[b_ptr0], #-0x40]\n"
- "fmla v17.4s, v13.4s, v0.s[3]\n"
- "fmla v21.4s, v13.4s, v1.s[3]\n"
- "fmla v25.4s, v13.4s, v2.s[3]\n"
- "ldr q13, [%[b_ptr0], #-0x30]\n"
- "fmla v18.4s, v14.4s, v0.s[3]\n"
- "fmla v22.4s, v14.4s, v1.s[3]\n"
- "fmla v26.4s, v14.4s, v2.s[3]\n"
- "ldr q14, [%[b_ptr0], #-0x20]\n"
- "fmla v19.4s, v15.4s, v0.s[3]\n"
- "ldr q0, [%[a_ptr0], #-0x10]\n"
- "fmla v23.4s, v15.4s, v1.s[3]\n"
- "ldr q1, [a_ptr1, #-0x10]\n"
- "fmla v27.4s, v15.4s, v2.s[3]\n"
- "ldr q15, [%[b_ptr0], #-0x10]\n"
- "fmla v16.4s, v8.4s, v4.s[0]\n"
- "ldr q2, [a_ptr2, #-0x10]\n"
- "fmla v20.4s, v8.4s, v5.s[0]\n"
- "fmla v24.4s, v8.4s, v6.s[0]\n"
- "ldr q8, [%[b_ptr0]]\n"
- "fmla v17.4s, v9.4s, v4.s[0]\n"
- "fmla v21.4s, v9.4s, v5.s[0]\n"
- "fmla v25.4s, v9.4s, v6.s[0]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "fmla v18.4s, v10.4s, v4.s[0]\n"
- "fmla v22.4s, v10.4s, v5.s[0]\n"
- "fmla v26.4s, v10.4s, v6.s[0]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "fmla v19.4s, v11.4s, v4.s[0]\n"
- "fmla v23.4s, v11.4s, v5.s[0]\n"
- "fmla v27.4s, v11.4s, v6.s[0]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "fmla v16.4s, v12.4s, v4.s[1]\n"
- "fmla v20.4s, v12.4s, v5.s[1]\n"
- "fmla v24.4s, v12.4s, v6.s[1]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- "fmla v17.4s, v13.4s, v4.s[1]\n"
- "fmla v21.4s, v13.4s, v5.s[1]\n"
- "fmla v25.4s, v13.4s, v6.s[1]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- "fmla v18.4s, v14.4s, v4.s[1]\n"
- "fmla v22.4s, v14.4s, v5.s[1]\n"
- "fmla v26.4s, v14.4s, v6.s[1]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- "fmla v19.4s, v15.4s, v4.s[1]\n"
- "fmla v23.4s, v15.4s, v5.s[1]\n"
- "fmla v27.4s, v15.4s, v6.s[1]\n"
- "ldr q15, [%[b_ptr0], #0x70]\n"
- "fmla v16.4s, v8.4s, v4.s[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- "fmla v20.4s, v8.4s, v5.s[2]\n"
- "fmla v24.4s, v8.4s, v6.s[2]\n"
- "ldr q8, [%[b_ptr0], #-0x80]\n"
- "fmla v17.4s, v9.4s, v4.s[2]\n"
- "fmla v21.4s, v9.4s, v5.s[2]\n"
- "fmla v25.4s, v9.4s, v6.s[2]\n"
- "ldr q9, [%[b_ptr0], #-0x70]\n"
- "fmla v18.4s, v10.4s, v4.s[2]\n"
- "fmla v22.4s, v10.4s, v5.s[2]\n"
- "fmla v26.4s, v10.4s, v6.s[2]\n"
- "ldr q10, [%[b_ptr0], #-0x60]\n"
- "fmla v19.4s, v11.4s, v4.s[2]\n"
- "fmla v23.4s, v11.4s, v5.s[2]\n"
- "fmla v27.4s, v11.4s, v6.s[2]\n"
- "ldr q11, [%[b_ptr0], #-0x50]\n"
- "fmla v16.4s, v12.4s, v4.s[3]\n"
- "fmla v20.4s, v12.4s, v5.s[3]\n"
- "fmla v24.4s, v12.4s, v6.s[3]\n"
- "ldr q12, [%[b_ptr0], #-0x40]\n"
- "fmla v17.4s, v13.4s, v4.s[3]\n"
- "fmla v21.4s, v13.4s, v5.s[3]\n"
- "fmla v25.4s, v13.4s, v6.s[3]\n"
- "ldr q13, [%[b_ptr0], #-0x30]\n"
- "fmla v18.4s, v14.4s, v4.s[3]\n"
- "fmla v22.4s, v14.4s, v5.s[3]\n"
- "fmla v26.4s, v14.4s, v6.s[3]\n"
- "ldr q14, [%[b_ptr0], #-0x20]\n"
- "fmla v19.4s, v15.4s, v4.s[3]\n"
- "fmla v23.4s, v15.4s, v5.s[3]\n"
- "fmla v27.4s, v15.4s, v6.s[3]\n"
- "b.ne 3b\n"
- "2:\n"
- "ldr q15, [%[b_ptr0], #-0x10]\n"
- "prfm PSTL1KEEP, [%[c_ptr0]]\n"
- "prfm PSTL1KEEP, [c_ptr1]\n"
- "prfm PSTL1KEEP, [c_ptr2]\n"
- "cbz %[regs], 4f\n"
- "fmla v16.4s, v8.4s, v0.s[0]\n"
- "ldr q4, [%[a_ptr0]]\n"
- "fmla v20.4s, v8.4s, v1.s[0]\n"
- "ldr q5, [a_ptr1]\n"
- "fmla v24.4s, v8.4s, v2.s[0]\n"
- "ldr q6, [a_ptr2]\n"
- "fmla v17.4s, v9.4s, v0.s[0]\n"
- "ldr q8, [%[b_ptr0]]\n"
- "fmla v21.4s, v9.4s, v1.s[0]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "fmla v25.4s, v9.4s, v2.s[0]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "fmla v18.4s, v10.4s, v0.s[0]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "fmla v22.4s, v10.4s, v1.s[0]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "fmla v26.4s, v10.4s, v2.s[0]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "fmla v19.4s, v11.4s, v0.s[0]\n"
- "fmla v23.4s, v11.4s, v1.s[0]\n"
- "fmla v27.4s, v11.4s, v2.s[0]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "fmla v16.4s, v12.4s, v0.s[1]\n"
- "fmla v20.4s, v12.4s, v1.s[1]\n"
- "fmla v24.4s, v12.4s, v2.s[1]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- "fmla v17.4s, v13.4s, v0.s[1]\n"
- "fmla v21.4s, v13.4s, v1.s[1]\n"
- "fmla v25.4s, v13.4s, v2.s[1]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- "fmla v18.4s, v14.4s, v0.s[1]\n"
- "fmla v22.4s, v14.4s, v1.s[1]\n"
- "fmla v26.4s, v14.4s, v2.s[1]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- "fmla v19.4s, v15.4s, v0.s[1]\n"
- "fmla v23.4s, v15.4s, v1.s[1]\n"
- "fmla v27.4s, v15.4s, v2.s[1]\n"
- "ldr q15, [%[b_ptr0], #0x70]\n"
- "fmla v16.4s, v8.4s, v0.s[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- "fmla v20.4s, v8.4s, v1.s[2]\n"
- "fmla v24.4s, v8.4s, v2.s[2]\n"
- "ldr q8, [%[b_ptr0], #-0x80]\n"
- "fmla v17.4s, v9.4s, v0.s[2]\n"
- "fmla v21.4s, v9.4s, v1.s[2]\n"
- "fmla v25.4s, v9.4s, v2.s[2]\n"
- "ldr q9, [%[b_ptr0], #-0x70]\n"
- "fmla v18.4s, v10.4s, v0.s[2]\n"
- "fmla v22.4s, v10.4s, v1.s[2]\n"
- "fmla v26.4s, v10.4s, v2.s[2]\n"
- "ldr q10, [%[b_ptr0], #-0x60]\n"
- "fmla v19.4s, v11.4s, v0.s[2]\n"
- "fmla v23.4s, v11.4s, v1.s[2]\n"
- "fmla v27.4s, v11.4s, v2.s[2]\n"
- "ldr q11, [%[b_ptr0], #-0x50]\n"
- "fmla v16.4s, v12.4s, v0.s[3]\n"
- "fmla v20.4s, v12.4s, v1.s[3]\n"
- "fmla v24.4s, v12.4s, v2.s[3]\n"
- "ldr q12, [%[b_ptr0], #-0x40]\n"
- "fmla v17.4s, v13.4s, v0.s[3]\n"
- "fmla v21.4s, v13.4s, v1.s[3]\n"
- "fmla v25.4s, v13.4s, v2.s[3]\n"
- "ldr q13, [%[b_ptr0], #-0x30]\n"
- "fmla v18.4s, v14.4s, v0.s[3]\n"
- "fmla v22.4s, v14.4s, v1.s[3]\n"
- "fmla v26.4s, v14.4s, v2.s[3]\n"
- "ldr q14, [%[b_ptr0], #-0x20]\n"
- "fmla v19.4s, v15.4s, v0.s[3]\n"
- "fmla v23.4s, v15.4s, v1.s[3]\n"
- "fmla v27.4s, v15.4s, v2.s[3]\n"
- "ldr q15, [%[b_ptr0], #-0x10]\n"
- "fmla v16.4s, v8.4s, v4.s[0]\n"
- "fmla v20.4s, v8.4s, v5.s[0]\n"
- "fmla v24.4s, v8.4s, v6.s[0]\n"
- "ldr q8, [%[b_ptr0]]\n"
- "fmla v17.4s, v9.4s, v4.s[0]\n"
- "fmla v21.4s, v9.4s, v5.s[0]\n"
- "fmla v25.4s, v9.4s, v6.s[0]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "fmla v18.4s, v10.4s, v4.s[0]\n"
- "fmla v22.4s, v10.4s, v5.s[0]\n"
- "fmla v26.4s, v10.4s, v6.s[0]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "fmla v19.4s, v11.4s, v4.s[0]\n"
- "fmla v23.4s, v11.4s, v5.s[0]\n"
- "fmla v27.4s, v11.4s, v6.s[0]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "fmla v16.4s, v12.4s, v4.s[1]\n"
- "fmla v20.4s, v12.4s, v5.s[1]\n"
- "fmla v24.4s, v12.4s, v6.s[1]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- "fmla v17.4s, v13.4s, v4.s[1]\n"
- "fmla v21.4s, v13.4s, v5.s[1]\n"
- "fmla v25.4s, v13.4s, v6.s[1]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- "fmla v18.4s, v14.4s, v4.s[1]\n"
- "fmla v22.4s, v14.4s, v5.s[1]\n"
- "fmla v26.4s, v14.4s, v6.s[1]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- "fmla v19.4s, v15.4s, v4.s[1]\n"
- "fmla v23.4s, v15.4s, v5.s[1]\n"
- "fmla v27.4s, v15.4s, v6.s[1]\n"
- "ldr q15, [%[b_ptr0], #0x70]\n"
- "fmla v16.4s, v8.4s, v4.s[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "fmla v20.4s, v8.4s, v5.s[2]\n"
- "fmla v24.4s, v8.4s, v6.s[2]\n"
- "fmla v17.4s, v9.4s, v4.s[2]\n"
- "fmla v21.4s, v9.4s, v5.s[2]\n"
- "fmla v25.4s, v9.4s, v6.s[2]\n"
- "fmla v18.4s, v10.4s, v4.s[2]\n"
- "fmla v22.4s, v10.4s, v5.s[2]\n"
- "fmla v26.4s, v10.4s, v6.s[2]\n"
- "fmla v19.4s, v11.4s, v4.s[2]\n"
- "fmla v23.4s, v11.4s, v5.s[2]\n"
- "fmla v27.4s, v11.4s, v6.s[2]\n"
- "fmla v16.4s, v12.4s, v4.s[3]\n"
- "fmla v20.4s, v12.4s, v5.s[3]\n"
- "fmla v24.4s, v12.4s, v6.s[3]\n"
- "fmla v17.4s, v13.4s, v4.s[3]\n"
- "fmla v21.4s, v13.4s, v5.s[3]\n"
- "fmla v25.4s, v13.4s, v6.s[3]\n"
- "fmla v18.4s, v14.4s, v4.s[3]\n"
- "fmla v22.4s, v14.4s, v5.s[3]\n"
- "fmla v26.4s, v14.4s, v6.s[3]\n"
- "fmla v19.4s, v15.4s, v4.s[3]\n"
- "fmla v23.4s, v15.4s, v5.s[3]\n"
- "fmla v27.4s, v15.4s, v6.s[3]\n"
- "b 5f\n"
- "4:\n"
- "fmla v16.4s, v8.4s, v0.s[0]\n"
- "fmla v20.4s, v8.4s, v1.s[0]\n"
- "fmla v24.4s, v8.4s, v2.s[0]\n"
- "ldr q8, [%[b_ptr0]]\n"
- "fmla v17.4s, v9.4s, v0.s[0]\n"
- "fmla v21.4s, v9.4s, v1.s[0]\n"
- "fmla v25.4s, v9.4s, v2.s[0]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "fmla v18.4s, v10.4s, v0.s[0]\n"
- "fmla v22.4s, v10.4s, v1.s[0]\n"
- "fmla v26.4s, v10.4s, v2.s[0]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "fmla v19.4s, v11.4s, v0.s[0]\n"
- "fmla v23.4s, v11.4s, v1.s[0]\n"
- "fmla v27.4s, v11.4s, v2.s[0]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "fmla v16.4s, v12.4s, v0.s[1]\n"
- "fmla v20.4s, v12.4s, v1.s[1]\n"
- "fmla v24.4s, v12.4s, v2.s[1]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- "fmla v17.4s, v13.4s, v0.s[1]\n"
- "fmla v21.4s, v13.4s, v1.s[1]\n"
- "fmla v25.4s, v13.4s, v2.s[1]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- "fmla v18.4s, v14.4s, v0.s[1]\n"
- "fmla v22.4s, v14.4s, v1.s[1]\n"
- "fmla v26.4s, v14.4s, v2.s[1]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- "fmla v19.4s, v15.4s, v0.s[1]\n"
- "fmla v23.4s, v15.4s, v1.s[1]\n"
- "fmla v27.4s, v15.4s, v2.s[1]\n"
- "ldr q15, [%[b_ptr0], #0x70]\n"
- "fmla v16.4s, v8.4s, v0.s[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "fmla v20.4s, v8.4s, v1.s[2]\n"
- "fmla v24.4s, v8.4s, v2.s[2]\n"
- "fmla v17.4s, v9.4s, v0.s[2]\n"
- "fmla v21.4s, v9.4s, v1.s[2]\n"
- "fmla v25.4s, v9.4s, v2.s[2]\n"
- "fmla v18.4s, v10.4s, v0.s[2]\n"
- "fmla v22.4s, v10.4s, v1.s[2]\n"
- "fmla v26.4s, v10.4s, v2.s[2]\n"
- "fmla v19.4s, v11.4s, v0.s[2]\n"
- "fmla v23.4s, v11.4s, v1.s[2]\n"
- "fmla v27.4s, v11.4s, v2.s[2]\n"
- "fmla v16.4s, v12.4s, v0.s[3]\n"
- "fmla v20.4s, v12.4s, v1.s[3]\n"
- "fmla v24.4s, v12.4s, v2.s[3]\n"
- "fmla v17.4s, v13.4s, v0.s[3]\n"
- "fmla v21.4s, v13.4s, v1.s[3]\n"
- "fmla v25.4s, v13.4s, v2.s[3]\n"
- "fmla v18.4s, v14.4s, v0.s[3]\n"
- "fmla v22.4s, v14.4s, v1.s[3]\n"
- "fmla v26.4s, v14.4s, v2.s[3]\n"
- "fmla v19.4s, v15.4s, v0.s[3]\n"
- "fmla v23.4s, v15.4s, v1.s[3]\n"
- "fmla v27.4s, v15.4s, v2.s[3]\n"
- "5:\n"
- "cbz %[blocks], 6f\n"
- "7:\n"
- "ldr q8, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "ldr s0, [%[a_ptr0]]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "add %[a_ptr0], %[a_ptr0], #0x4\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "add %[b_ptr0], %[b_ptr0], #0x40\n"
- "fmla v16.4s, v8.4s, v0.s[0]\n"
- "ldr s1, [a_ptr1]\n"
- "fmla v17.4s, v9.4s, v0.s[0]\n"
- "add a_ptr1, a_ptr1, #0x4\n"
- "fmla v18.4s, v10.4s, v0.s[0]\n"
- "ldr s2, [a_ptr2]\n"
- "fmla v20.4s, v8.4s, v1.s[0]\n"
- "add a_ptr2, a_ptr2, #0x4\n"
- "fmla v21.4s, v9.4s, v1.s[0]\n"
- "fmla v24.4s, v8.4s, v2.s[0]\n"
- "fmla v25.4s, v9.4s, v2.s[0]\n"
- "fmla v22.4s, v10.4s, v1.s[0]\n"
- "fmla v26.4s, v10.4s, v2.s[0]\n"
- "fmla v19.4s, v11.4s, v0.s[0]\n"
- "fmla v23.4s, v11.4s, v1.s[0]\n"
- "fmla v27.4s, v11.4s, v2.s[0]\n"
- "b.ne 7b\n"
- "6:\n"
- "ld1r {v14.4s}, [%[minptr]]\n"
- "ld1r {v15.4s}, [%[maxptr]]\n"
- "fmax v16.4s, v16.4s, v14.4s\n"
- "fmax v17.4s, v17.4s, v14.4s\n"
- "fmax v18.4s, v18.4s, v14.4s\n"
- "fmax v19.4s, v19.4s, v14.4s\n"
- "fmin v16.4s, v16.4s, v15.4s\n"
- "fmin v17.4s, v17.4s, v15.4s\n"
- "fmin v18.4s, v18.4s, v15.4s\n"
- "fmin v19.4s, v19.4s, v15.4s\n"
- "str q16, [%[c_ptr0]]\n"
- "fmax v20.4s, v20.4s, v14.4s\n"
- "fmax v21.4s, v21.4s, v14.4s\n"
- "fmax v22.4s, v22.4s, v14.4s\n"
- "str q17, [%[c_ptr0], #0x10]\n"
- "fmax v23.4s, v23.4s, v14.4s\n"
- "fmin v20.4s, v20.4s, v15.4s\n"
- "fmin v21.4s, v21.4s, v15.4s\n"
- "str q18, [%[c_ptr0], #0x20]\n"
- "fmin v22.4s, v22.4s, v15.4s\n"
- "fmin v23.4s, v23.4s, v15.4s\n"
- "fmax v24.4s, v24.4s, v14.4s\n"
- "str q19, [%[c_ptr0], #0x30]\n"
- "fmax v25.4s, v25.4s, v14.4s\n"
- "add %[c_ptr0], %[c_ptr0], #0x40\n"
- "fmax v26.4s, v26.4s, v14.4s\n"
- "str q20, [c_ptr1]\n"
- "fmin v24.4s, v24.4s, v15.4s\n"
- "fmin v25.4s, v25.4s, v15.4s\n"
- "fmax v27.4s, v27.4s, v14.4s\n"
- "str q21, [c_ptr1, #0x10]\n"
- "fmin v26.4s, v26.4s, v15.4s\n"
- "fmin v27.4s, v27.4s, v15.4s\n"
- "str q22, [c_ptr1, #0x20]\n"
- "str q23, [c_ptr1, #0x30]\n"
- "str q24, [c_ptr2]\n"
- "str q25, [c_ptr2, #0x10]\n"
- "str q26, [c_ptr2, #0x20]\n"
- "str q27, [c_ptr2, #0x30]\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks)
- : [width] "r" (width), [accumulate] "r" (static_cast<uint64_t>(accumulate)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr)
- : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "cc", "memory"
- );
- break;
- default:
- case 4:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "a_ptr3 .req X2\n"
- "c_ptr1 .req X3\n"
- "c_ptr2 .req X4\n"
- "c_ptr3 .req X5\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "add a_ptr3, a_ptr2, %[lda]\n"
- "add c_ptr3, c_ptr2, %[ldc]\n"
- "cbnz %[accumulate], 1f\n"
- "ldr q16, [%[biasptr]]\n"
- "ldr q17, [%[biasptr], #0x10]\n"
- "ldr q18, [%[biasptr], #0x20]\n"
- "ldr q19, [%[biasptr], #0x30]\n"
- "mov v20.16b, v16.16b\n"
- "ldr q0, [%[a_ptr0]]\n"
- "mov v21.16b, v17.16b\n"
- "ldr q1, [a_ptr1]\n"
- "mov v22.16b, v18.16b\n"
- "ldr q2, [a_ptr2]\n"
- "mov v23.16b, v19.16b\n"
- "ldr q3, [a_ptr3]\n"
- "mov v24.16b, v16.16b\n"
- "ldr q8, [%[b_ptr0]]\n"
- "mov v25.16b, v17.16b\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "mov v26.16b, v18.16b\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "mov v27.16b, v19.16b\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "mov v28.16b, v16.16b\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- "mov v29.16b, v17.16b\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- "mov v30.16b, v18.16b\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- "mov v31.16b, v19.16b\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "add a_ptr3, a_ptr3, #0x10\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "ldr q16, [%[c_ptr0]]\n"
- "ldr q17, [%[c_ptr0], #0x10]\n"
- "ldr q18, [%[c_ptr0], #0x20]\n"
- "ldr q19, [%[c_ptr0], #0x30]\n"
- "ldr q20, [c_ptr1]\n"
- "ldr q21, [c_ptr1, #0x10]\n"
- "ldr q22, [c_ptr1, #0x20]\n"
- "ldr q23, [c_ptr1, #0x30]\n"
- "ldr q24, [c_ptr2]\n"
- "ldr q25, [c_ptr2, #0x10]\n"
- "ldr q26, [c_ptr2, #0x20]\n"
- "ldr q27, [c_ptr2, #0x30]\n"
- "ldr q28, [c_ptr3]\n"
- "ldr q29, [c_ptr3, #0x10]\n"
- "ldr q30, [c_ptr3, #0x20]\n"
- "ldr q31, [c_ptr3, #0x30]\n"
- "ldr q0, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ldr q1, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "ldr q2, [a_ptr2]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "ldr q3, [a_ptr3]\n"
- "add a_ptr3, a_ptr3, #0x10\n"
- "ldr q8, [%[b_ptr0]]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "cbz %[loops], 2f\n"
- "3:\n"
- "fmla v16.4s, v8.4s, v0.s[0]\n"
- "ldr q15, [%[b_ptr0], #-0x10]\n"
- "fmla v20.4s, v8.4s, v1.s[0]\n"
- "ldr q4, [%[a_ptr0]]\n"
- "fmla v24.4s, v8.4s, v2.s[0]\n"
- "ldr q5, [a_ptr1]\n"
- "fmla v28.4s, v8.4s, v3.s[0]\n"
- "ldr q6, [a_ptr2]\n"
- "fmla v17.4s, v9.4s, v0.s[0]\n"
- "ldr q7, [a_ptr3]\n"
- "fmla v21.4s, v9.4s, v1.s[0]\n"
- "ldr q8, [%[b_ptr0]]\n"
- "fmla v25.4s, v9.4s, v2.s[0]\n"
- "subs %[loops], %[loops], #0x1\n"
- "fmla v29.4s, v9.4s, v3.s[0]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "fmla v18.4s, v10.4s, v0.s[0]\n"
- "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n"
- "fmla v22.4s, v10.4s, v1.s[0]\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
- "fmla v26.4s, v10.4s, v2.s[0]\n"
- "add a_ptr1, a_ptr1, #0x20\n"
- "fmla v30.4s, v10.4s, v3.s[0]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "fmla v19.4s, v11.4s, v0.s[0]\n"
- "add a_ptr2, a_ptr2, #0x20\n"
- "fmla v23.4s, v11.4s, v1.s[0]\n"
- "add a_ptr3, a_ptr3, #0x20\n"
- "fmla v27.4s, v11.4s, v2.s[0]\n"
- "prfm PLDL1KEEP, [a_ptr1, #0x40]\n"
- "fmla v31.4s, v11.4s, v3.s[0]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "fmla v16.4s, v12.4s, v0.s[1]\n"
- "prfm PLDL1KEEP, [a_ptr2, #0x40]\n"
- "fmla v20.4s, v12.4s, v1.s[1]\n"
- "prfm PLDL1KEEP, [a_ptr3, #0x40]\n"
- "fmla v24.4s, v12.4s, v2.s[1]\n"
- "fmla v28.4s, v12.4s, v3.s[1]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- "fmla v17.4s, v13.4s, v0.s[1]\n"
- "fmla v21.4s, v13.4s, v1.s[1]\n"
- "fmla v25.4s, v13.4s, v2.s[1]\n"
- "fmla v29.4s, v13.4s, v3.s[1]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- "fmla v18.4s, v14.4s, v0.s[1]\n"
- "fmla v22.4s, v14.4s, v1.s[1]\n"
- "fmla v26.4s, v14.4s, v2.s[1]\n"
- "fmla v30.4s, v14.4s, v3.s[1]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- "fmla v19.4s, v15.4s, v0.s[1]\n"
- "fmla v23.4s, v15.4s, v1.s[1]\n"
- "fmla v27.4s, v15.4s, v2.s[1]\n"
- "fmla v31.4s, v15.4s, v3.s[1]\n"
- "ldr q15, [%[b_ptr0], #0x70]\n"
- "fmla v16.4s, v8.4s, v0.s[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- "fmla v20.4s, v8.4s, v1.s[2]\n"
- "fmla v24.4s, v8.4s, v2.s[2]\n"
- "fmla v28.4s, v8.4s, v3.s[2]\n"
- "ldr q8, [%[b_ptr0], #-0x80]\n"
- "fmla v17.4s, v9.4s, v0.s[2]\n"
- "fmla v21.4s, v9.4s, v1.s[2]\n"
- "fmla v25.4s, v9.4s, v2.s[2]\n"
- "fmla v29.4s, v9.4s, v3.s[2]\n"
- "ldr q9, [%[b_ptr0], #-0x70]\n"
- "fmla v18.4s, v10.4s, v0.s[2]\n"
- "fmla v22.4s, v10.4s, v1.s[2]\n"
- "fmla v26.4s, v10.4s, v2.s[2]\n"
- "fmla v30.4s, v10.4s, v3.s[2]\n"
- "ldr q10, [%[b_ptr0], #-0x60]\n"
- "fmla v19.4s, v11.4s, v0.s[2]\n"
- "fmla v23.4s, v11.4s, v1.s[2]\n"
- "fmla v27.4s, v11.4s, v2.s[2]\n"
- "fmla v31.4s, v11.4s, v3.s[2]\n"
- "ldr q11, [%[b_ptr0], #-0x50]\n"
- "fmla v16.4s, v12.4s, v0.s[3]\n"
- "fmla v20.4s, v12.4s, v1.s[3]\n"
- "fmla v24.4s, v12.4s, v2.s[3]\n"
- "fmla v28.4s, v12.4s, v3.s[3]\n"
- "ldr q12, [%[b_ptr0], #-0x40]\n"
- "fmla v17.4s, v13.4s, v0.s[3]\n"
- "fmla v21.4s, v13.4s, v1.s[3]\n"
- "fmla v25.4s, v13.4s, v2.s[3]\n"
- "fmla v29.4s, v13.4s, v3.s[3]\n"
- "ldr q13, [%[b_ptr0], #-0x30]\n"
- "fmla v18.4s, v14.4s, v0.s[3]\n"
- "fmla v22.4s, v14.4s, v1.s[3]\n"
- "fmla v26.4s, v14.4s, v2.s[3]\n"
- "fmla v30.4s, v14.4s, v3.s[3]\n"
- "ldr q14, [%[b_ptr0], #-0x20]\n"
- "fmla v19.4s, v15.4s, v0.s[3]\n"
- "ldr q0, [%[a_ptr0], #-0x10]\n"
- "fmla v23.4s, v15.4s, v1.s[3]\n"
- "ldr q1, [a_ptr1, #-0x10]\n"
- "fmla v27.4s, v15.4s, v2.s[3]\n"
- "ldr q2, [a_ptr2, #-0x10]\n"
- "fmla v31.4s, v15.4s, v3.s[3]\n"
- "ldr q15, [%[b_ptr0], #-0x10]\n"
- "fmla v16.4s, v8.4s, v4.s[0]\n"
- "ldr q3, [a_ptr3, #-0x10]\n"
- "fmla v20.4s, v8.4s, v5.s[0]\n"
- "fmla v24.4s, v8.4s, v6.s[0]\n"
- "fmla v28.4s, v8.4s, v7.s[0]\n"
- "ldr q8, [%[b_ptr0]]\n"
- "fmla v17.4s, v9.4s, v4.s[0]\n"
- "fmla v21.4s, v9.4s, v5.s[0]\n"
- "fmla v25.4s, v9.4s, v6.s[0]\n"
- "fmla v29.4s, v9.4s, v7.s[0]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "fmla v18.4s, v10.4s, v4.s[0]\n"
- "fmla v22.4s, v10.4s, v5.s[0]\n"
- "fmla v26.4s, v10.4s, v6.s[0]\n"
- "fmla v30.4s, v10.4s, v7.s[0]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "fmla v19.4s, v11.4s, v4.s[0]\n"
- "fmla v23.4s, v11.4s, v5.s[0]\n"
- "fmla v27.4s, v11.4s, v6.s[0]\n"
- "fmla v31.4s, v11.4s, v7.s[0]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "fmla v16.4s, v12.4s, v4.s[1]\n"
- "fmla v20.4s, v12.4s, v5.s[1]\n"
- "fmla v24.4s, v12.4s, v6.s[1]\n"
- "fmla v28.4s, v12.4s, v7.s[1]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- "fmla v17.4s, v13.4s, v4.s[1]\n"
- "fmla v21.4s, v13.4s, v5.s[1]\n"
- "fmla v25.4s, v13.4s, v6.s[1]\n"
- "fmla v29.4s, v13.4s, v7.s[1]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- "fmla v18.4s, v14.4s, v4.s[1]\n"
- "fmla v22.4s, v14.4s, v5.s[1]\n"
- "fmla v26.4s, v14.4s, v6.s[1]\n"
- "fmla v30.4s, v14.4s, v7.s[1]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- "fmla v19.4s, v15.4s, v4.s[1]\n"
- "fmla v23.4s, v15.4s, v5.s[1]\n"
- "fmla v27.4s, v15.4s, v6.s[1]\n"
- "fmla v31.4s, v15.4s, v7.s[1]\n"
- "ldr q15, [%[b_ptr0], #0x70]\n"
- "fmla v16.4s, v8.4s, v4.s[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- "fmla v20.4s, v8.4s, v5.s[2]\n"
- "fmla v24.4s, v8.4s, v6.s[2]\n"
- "fmla v28.4s, v8.4s, v7.s[2]\n"
- "ldr q8, [%[b_ptr0], #-0x80]\n"
- "fmla v17.4s, v9.4s, v4.s[2]\n"
- "fmla v21.4s, v9.4s, v5.s[2]\n"
- "fmla v25.4s, v9.4s, v6.s[2]\n"
- "fmla v29.4s, v9.4s, v7.s[2]\n"
- "ldr q9, [%[b_ptr0], #-0x70]\n"
- "fmla v18.4s, v10.4s, v4.s[2]\n"
- "fmla v22.4s, v10.4s, v5.s[2]\n"
- "fmla v26.4s, v10.4s, v6.s[2]\n"
- "fmla v30.4s, v10.4s, v7.s[2]\n"
- "ldr q10, [%[b_ptr0], #-0x60]\n"
- "fmla v19.4s, v11.4s, v4.s[2]\n"
- "fmla v23.4s, v11.4s, v5.s[2]\n"
- "fmla v27.4s, v11.4s, v6.s[2]\n"
- "fmla v31.4s, v11.4s, v7.s[2]\n"
- "ldr q11, [%[b_ptr0], #-0x50]\n"
- "fmla v16.4s, v12.4s, v4.s[3]\n"
- "fmla v20.4s, v12.4s, v5.s[3]\n"
- "fmla v24.4s, v12.4s, v6.s[3]\n"
- "fmla v28.4s, v12.4s, v7.s[3]\n"
- "ldr q12, [%[b_ptr0], #-0x40]\n"
- "fmla v17.4s, v13.4s, v4.s[3]\n"
- "fmla v21.4s, v13.4s, v5.s[3]\n"
- "fmla v25.4s, v13.4s, v6.s[3]\n"
- "fmla v29.4s, v13.4s, v7.s[3]\n"
- "ldr q13, [%[b_ptr0], #-0x30]\n"
- "fmla v18.4s, v14.4s, v4.s[3]\n"
- "fmla v22.4s, v14.4s, v5.s[3]\n"
- "fmla v26.4s, v14.4s, v6.s[3]\n"
- "fmla v30.4s, v14.4s, v7.s[3]\n"
- "ldr q14, [%[b_ptr0], #-0x20]\n"
- "fmla v19.4s, v15.4s, v4.s[3]\n"
- "fmla v23.4s, v15.4s, v5.s[3]\n"
- "fmla v27.4s, v15.4s, v6.s[3]\n"
- "fmla v31.4s, v15.4s, v7.s[3]\n"
- "b.ne 3b\n"
- "2:\n"
- "ldr q15, [%[b_ptr0], #-0x10]\n"
- "prfm PSTL1KEEP, [%[c_ptr0]]\n"
- "prfm PSTL1KEEP, [c_ptr1]\n"
- "prfm PSTL1KEEP, [c_ptr2]\n"
- "prfm PSTL1KEEP, [c_ptr3]\n"
- "cbz %[regs], 4f\n"
- "fmla v16.4s, v8.4s, v0.s[0]\n"
- "ldr q4, [%[a_ptr0]]\n"
- "fmla v20.4s, v8.4s, v1.s[0]\n"
- "ldr q5, [a_ptr1]\n"
- "fmla v24.4s, v8.4s, v2.s[0]\n"
- "ldr q6, [a_ptr2]\n"
- "fmla v28.4s, v8.4s, v3.s[0]\n"
- "ldr q7, [a_ptr3]\n"
- "fmla v17.4s, v9.4s, v0.s[0]\n"
- "ldr q8, [%[b_ptr0]]\n"
- "fmla v21.4s, v9.4s, v1.s[0]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "fmla v25.4s, v9.4s, v2.s[0]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "fmla v29.4s, v9.4s, v3.s[0]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "fmla v18.4s, v10.4s, v0.s[0]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "fmla v22.4s, v10.4s, v1.s[0]\n"
- "add a_ptr3, a_ptr3, #0x10\n"
- "fmla v26.4s, v10.4s, v2.s[0]\n"
- "fmla v30.4s, v10.4s, v3.s[0]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "fmla v19.4s, v11.4s, v0.s[0]\n"
- "fmla v23.4s, v11.4s, v1.s[0]\n"
- "fmla v27.4s, v11.4s, v2.s[0]\n"
- "fmla v31.4s, v11.4s, v3.s[0]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "fmla v16.4s, v12.4s, v0.s[1]\n"
- "fmla v20.4s, v12.4s, v1.s[1]\n"
- "fmla v24.4s, v12.4s, v2.s[1]\n"
- "fmla v28.4s, v12.4s, v3.s[1]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- "fmla v17.4s, v13.4s, v0.s[1]\n"
- "fmla v21.4s, v13.4s, v1.s[1]\n"
- "fmla v25.4s, v13.4s, v2.s[1]\n"
- "fmla v29.4s, v13.4s, v3.s[1]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- "fmla v18.4s, v14.4s, v0.s[1]\n"
- "fmla v22.4s, v14.4s, v1.s[1]\n"
- "fmla v26.4s, v14.4s, v2.s[1]\n"
- "fmla v30.4s, v14.4s, v3.s[1]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- "fmla v19.4s, v15.4s, v0.s[1]\n"
- "fmla v23.4s, v15.4s, v1.s[1]\n"
- "fmla v27.4s, v15.4s, v2.s[1]\n"
- "fmla v31.4s, v15.4s, v3.s[1]\n"
- "ldr q15, [%[b_ptr0], #0x70]\n"
- "fmla v16.4s, v8.4s, v0.s[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- "fmla v20.4s, v8.4s, v1.s[2]\n"
- "fmla v24.4s, v8.4s, v2.s[2]\n"
- "fmla v28.4s, v8.4s, v3.s[2]\n"
- "ldr q8, [%[b_ptr0], #-0x80]\n"
- "fmla v17.4s, v9.4s, v0.s[2]\n"
- "fmla v21.4s, v9.4s, v1.s[2]\n"
- "fmla v25.4s, v9.4s, v2.s[2]\n"
- "fmla v29.4s, v9.4s, v3.s[2]\n"
- "ldr q9, [%[b_ptr0], #-0x70]\n"
- "fmla v18.4s, v10.4s, v0.s[2]\n"
- "fmla v22.4s, v10.4s, v1.s[2]\n"
- "fmla v26.4s, v10.4s, v2.s[2]\n"
- "fmla v30.4s, v10.4s, v3.s[2]\n"
- "ldr q10, [%[b_ptr0], #-0x60]\n"
- "fmla v19.4s, v11.4s, v0.s[2]\n"
- "fmla v23.4s, v11.4s, v1.s[2]\n"
- "fmla v27.4s, v11.4s, v2.s[2]\n"
- "fmla v31.4s, v11.4s, v3.s[2]\n"
- "ldr q11, [%[b_ptr0], #-0x50]\n"
- "fmla v16.4s, v12.4s, v0.s[3]\n"
- "fmla v20.4s, v12.4s, v1.s[3]\n"
- "fmla v24.4s, v12.4s, v2.s[3]\n"
- "fmla v28.4s, v12.4s, v3.s[3]\n"
- "ldr q12, [%[b_ptr0], #-0x40]\n"
- "fmla v17.4s, v13.4s, v0.s[3]\n"
- "fmla v21.4s, v13.4s, v1.s[3]\n"
- "fmla v25.4s, v13.4s, v2.s[3]\n"
- "fmla v29.4s, v13.4s, v3.s[3]\n"
- "ldr q13, [%[b_ptr0], #-0x30]\n"
- "fmla v18.4s, v14.4s, v0.s[3]\n"
- "fmla v22.4s, v14.4s, v1.s[3]\n"
- "fmla v26.4s, v14.4s, v2.s[3]\n"
- "fmla v30.4s, v14.4s, v3.s[3]\n"
- "ldr q14, [%[b_ptr0], #-0x20]\n"
- "fmla v19.4s, v15.4s, v0.s[3]\n"
- "fmla v23.4s, v15.4s, v1.s[3]\n"
- "fmla v27.4s, v15.4s, v2.s[3]\n"
- "fmla v31.4s, v15.4s, v3.s[3]\n"
- "ldr q15, [%[b_ptr0], #-0x10]\n"
- "fmla v16.4s, v8.4s, v4.s[0]\n"
- "fmla v20.4s, v8.4s, v5.s[0]\n"
- "fmla v24.4s, v8.4s, v6.s[0]\n"
- "fmla v28.4s, v8.4s, v7.s[0]\n"
- "ldr q8, [%[b_ptr0]]\n"
- "fmla v17.4s, v9.4s, v4.s[0]\n"
- "fmla v21.4s, v9.4s, v5.s[0]\n"
- "fmla v25.4s, v9.4s, v6.s[0]\n"
- "fmla v29.4s, v9.4s, v7.s[0]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "fmla v18.4s, v10.4s, v4.s[0]\n"
- "fmla v22.4s, v10.4s, v5.s[0]\n"
- "fmla v26.4s, v10.4s, v6.s[0]\n"
- "fmla v30.4s, v10.4s, v7.s[0]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "fmla v19.4s, v11.4s, v4.s[0]\n"
- "fmla v23.4s, v11.4s, v5.s[0]\n"
- "fmla v27.4s, v11.4s, v6.s[0]\n"
- "fmla v31.4s, v11.4s, v7.s[0]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "fmla v16.4s, v12.4s, v4.s[1]\n"
- "fmla v20.4s, v12.4s, v5.s[1]\n"
- "fmla v24.4s, v12.4s, v6.s[1]\n"
- "fmla v28.4s, v12.4s, v7.s[1]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- "fmla v17.4s, v13.4s, v4.s[1]\n"
- "fmla v21.4s, v13.4s, v5.s[1]\n"
- "fmla v25.4s, v13.4s, v6.s[1]\n"
- "fmla v29.4s, v13.4s, v7.s[1]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- "fmla v18.4s, v14.4s, v4.s[1]\n"
- "fmla v22.4s, v14.4s, v5.s[1]\n"
- "fmla v26.4s, v14.4s, v6.s[1]\n"
- "fmla v30.4s, v14.4s, v7.s[1]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- "fmla v19.4s, v15.4s, v4.s[1]\n"
- "fmla v23.4s, v15.4s, v5.s[1]\n"
- "fmla v27.4s, v15.4s, v6.s[1]\n"
- "fmla v31.4s, v15.4s, v7.s[1]\n"
- "ldr q15, [%[b_ptr0], #0x70]\n"
- "fmla v16.4s, v8.4s, v4.s[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "fmla v20.4s, v8.4s, v5.s[2]\n"
- "fmla v24.4s, v8.4s, v6.s[2]\n"
- "fmla v28.4s, v8.4s, v7.s[2]\n"
- "fmla v17.4s, v9.4s, v4.s[2]\n"
- "fmla v21.4s, v9.4s, v5.s[2]\n"
- "fmla v25.4s, v9.4s, v6.s[2]\n"
- "fmla v29.4s, v9.4s, v7.s[2]\n"
- "fmla v18.4s, v10.4s, v4.s[2]\n"
- "fmla v22.4s, v10.4s, v5.s[2]\n"
- "fmla v26.4s, v10.4s, v6.s[2]\n"
- "fmla v30.4s, v10.4s, v7.s[2]\n"
- "fmla v19.4s, v11.4s, v4.s[2]\n"
- "fmla v23.4s, v11.4s, v5.s[2]\n"
- "fmla v27.4s, v11.4s, v6.s[2]\n"
- "fmla v31.4s, v11.4s, v7.s[2]\n"
- "fmla v16.4s, v12.4s, v4.s[3]\n"
- "fmla v20.4s, v12.4s, v5.s[3]\n"
- "fmla v24.4s, v12.4s, v6.s[3]\n"
- "fmla v28.4s, v12.4s, v7.s[3]\n"
- "fmla v17.4s, v13.4s, v4.s[3]\n"
- "fmla v21.4s, v13.4s, v5.s[3]\n"
- "fmla v25.4s, v13.4s, v6.s[3]\n"
- "fmla v29.4s, v13.4s, v7.s[3]\n"
- "fmla v18.4s, v14.4s, v4.s[3]\n"
- "fmla v22.4s, v14.4s, v5.s[3]\n"
- "fmla v26.4s, v14.4s, v6.s[3]\n"
- "fmla v30.4s, v14.4s, v7.s[3]\n"
- "fmla v19.4s, v15.4s, v4.s[3]\n"
- "fmla v23.4s, v15.4s, v5.s[3]\n"
- "fmla v27.4s, v15.4s, v6.s[3]\n"
- "fmla v31.4s, v15.4s, v7.s[3]\n"
- "b 5f\n"
- "4:\n"
- "fmla v16.4s, v8.4s, v0.s[0]\n"
- "fmla v20.4s, v8.4s, v1.s[0]\n"
- "fmla v24.4s, v8.4s, v2.s[0]\n"
- "fmla v28.4s, v8.4s, v3.s[0]\n"
- "ldr q8, [%[b_ptr0]]\n"
- "fmla v17.4s, v9.4s, v0.s[0]\n"
- "fmla v21.4s, v9.4s, v1.s[0]\n"
- "fmla v25.4s, v9.4s, v2.s[0]\n"
- "fmla v29.4s, v9.4s, v3.s[0]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "fmla v18.4s, v10.4s, v0.s[0]\n"
- "fmla v22.4s, v10.4s, v1.s[0]\n"
- "fmla v26.4s, v10.4s, v2.s[0]\n"
- "fmla v30.4s, v10.4s, v3.s[0]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "fmla v19.4s, v11.4s, v0.s[0]\n"
- "fmla v23.4s, v11.4s, v1.s[0]\n"
- "fmla v27.4s, v11.4s, v2.s[0]\n"
- "fmla v31.4s, v11.4s, v3.s[0]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "fmla v16.4s, v12.4s, v0.s[1]\n"
- "fmla v20.4s, v12.4s, v1.s[1]\n"
- "fmla v24.4s, v12.4s, v2.s[1]\n"
- "fmla v28.4s, v12.4s, v3.s[1]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- "fmla v17.4s, v13.4s, v0.s[1]\n"
- "fmla v21.4s, v13.4s, v1.s[1]\n"
- "fmla v25.4s, v13.4s, v2.s[1]\n"
- "fmla v29.4s, v13.4s, v3.s[1]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- "fmla v18.4s, v14.4s, v0.s[1]\n"
- "fmla v22.4s, v14.4s, v1.s[1]\n"
- "fmla v26.4s, v14.4s, v2.s[1]\n"
- "fmla v30.4s, v14.4s, v3.s[1]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- "fmla v19.4s, v15.4s, v0.s[1]\n"
- "fmla v23.4s, v15.4s, v1.s[1]\n"
- "fmla v27.4s, v15.4s, v2.s[1]\n"
- "fmla v31.4s, v15.4s, v3.s[1]\n"
- "ldr q15, [%[b_ptr0], #0x70]\n"
- "fmla v16.4s, v8.4s, v0.s[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "fmla v20.4s, v8.4s, v1.s[2]\n"
- "fmla v24.4s, v8.4s, v2.s[2]\n"
- "fmla v28.4s, v8.4s, v3.s[2]\n"
- "fmla v17.4s, v9.4s, v0.s[2]\n"
- "fmla v21.4s, v9.4s, v1.s[2]\n"
- "fmla v25.4s, v9.4s, v2.s[2]\n"
- "fmla v29.4s, v9.4s, v3.s[2]\n"
- "fmla v18.4s, v10.4s, v0.s[2]\n"
- "fmla v22.4s, v10.4s, v1.s[2]\n"
- "fmla v26.4s, v10.4s, v2.s[2]\n"
- "fmla v30.4s, v10.4s, v3.s[2]\n"
- "fmla v19.4s, v11.4s, v0.s[2]\n"
- "fmla v23.4s, v11.4s, v1.s[2]\n"
- "fmla v27.4s, v11.4s, v2.s[2]\n"
- "fmla v31.4s, v11.4s, v3.s[2]\n"
- "fmla v16.4s, v12.4s, v0.s[3]\n"
- "fmla v20.4s, v12.4s, v1.s[3]\n"
- "fmla v24.4s, v12.4s, v2.s[3]\n"
- "fmla v28.4s, v12.4s, v3.s[3]\n"
- "fmla v17.4s, v13.4s, v0.s[3]\n"
- "fmla v21.4s, v13.4s, v1.s[3]\n"
- "fmla v25.4s, v13.4s, v2.s[3]\n"
- "fmla v29.4s, v13.4s, v3.s[3]\n"
- "fmla v18.4s, v14.4s, v0.s[3]\n"
- "fmla v22.4s, v14.4s, v1.s[3]\n"
- "fmla v26.4s, v14.4s, v2.s[3]\n"
- "fmla v30.4s, v14.4s, v3.s[3]\n"
- "fmla v19.4s, v15.4s, v0.s[3]\n"
- "fmla v23.4s, v15.4s, v1.s[3]\n"
- "fmla v27.4s, v15.4s, v2.s[3]\n"
- "fmla v31.4s, v15.4s, v3.s[3]\n"
- "5:\n"
- "cbz %[blocks], 6f\n"
- "7:\n"
- "ldr q8, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "ldr s0, [%[a_ptr0]]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "add %[a_ptr0], %[a_ptr0], #0x4\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "add %[b_ptr0], %[b_ptr0], #0x40\n"
- "fmla v16.4s, v8.4s, v0.s[0]\n"
- "ldr s1, [a_ptr1]\n"
- "fmla v17.4s, v9.4s, v0.s[0]\n"
- "add a_ptr1, a_ptr1, #0x4\n"
- "fmla v18.4s, v10.4s, v0.s[0]\n"
- "ldr s2, [a_ptr2]\n"
- "fmla v20.4s, v8.4s, v1.s[0]\n"
- "add a_ptr2, a_ptr2, #0x4\n"
- "fmla v21.4s, v9.4s, v1.s[0]\n"
- "ldr s3, [a_ptr3]\n"
- "fmla v24.4s, v8.4s, v2.s[0]\n"
- "add a_ptr3, a_ptr3, #0x4\n"
- "fmla v25.4s, v9.4s, v2.s[0]\n"
- "fmla v28.4s, v8.4s, v3.s[0]\n"
- "fmla v29.4s, v9.4s, v3.s[0]\n"
- "fmla v22.4s, v10.4s, v1.s[0]\n"
- "fmla v26.4s, v10.4s, v2.s[0]\n"
- "fmla v30.4s, v10.4s, v3.s[0]\n"
- "fmla v19.4s, v11.4s, v0.s[0]\n"
- "fmla v23.4s, v11.4s, v1.s[0]\n"
- "fmla v27.4s, v11.4s, v2.s[0]\n"
- "fmla v31.4s, v11.4s, v3.s[0]\n"
- "b.ne 7b\n"
- "6:\n"
- "ld1r {v14.4s}, [%[minptr]]\n"
- "ld1r {v15.4s}, [%[maxptr]]\n"
- "fmax v16.4s, v16.4s, v14.4s\n"
- "fmax v17.4s, v17.4s, v14.4s\n"
- "fmax v18.4s, v18.4s, v14.4s\n"
- "fmax v19.4s, v19.4s, v14.4s\n"
- "fmin v16.4s, v16.4s, v15.4s\n"
- "fmin v17.4s, v17.4s, v15.4s\n"
- "fmin v18.4s, v18.4s, v15.4s\n"
- "fmin v19.4s, v19.4s, v15.4s\n"
- "str q16, [%[c_ptr0]]\n"
- "fmax v20.4s, v20.4s, v14.4s\n"
- "fmax v21.4s, v21.4s, v14.4s\n"
- "fmax v22.4s, v22.4s, v14.4s\n"
- "str q17, [%[c_ptr0], #0x10]\n"
- "fmax v23.4s, v23.4s, v14.4s\n"
- "fmin v20.4s, v20.4s, v15.4s\n"
- "fmin v21.4s, v21.4s, v15.4s\n"
- "str q18, [%[c_ptr0], #0x20]\n"
- "fmin v22.4s, v22.4s, v15.4s\n"
- "fmin v23.4s, v23.4s, v15.4s\n"
- "fmax v24.4s, v24.4s, v14.4s\n"
- "str q19, [%[c_ptr0], #0x30]\n"
- "fmax v25.4s, v25.4s, v14.4s\n"
- "add %[c_ptr0], %[c_ptr0], #0x40\n"
- "fmax v26.4s, v26.4s, v14.4s\n"
- "str q20, [c_ptr1]\n"
- "fmin v24.4s, v24.4s, v15.4s\n"
- "fmin v25.4s, v25.4s, v15.4s\n"
- "fmax v27.4s, v27.4s, v14.4s\n"
- "str q21, [c_ptr1, #0x10]\n"
- "fmin v26.4s, v26.4s, v15.4s\n"
- "fmax v28.4s, v28.4s, v14.4s\n"
- "fmax v29.4s, v29.4s, v14.4s\n"
- "str q22, [c_ptr1, #0x20]\n"
- "fmin v27.4s, v27.4s, v15.4s\n"
- "fmax v30.4s, v30.4s, v14.4s\n"
- "fmin v28.4s, v28.4s, v15.4s\n"
- "str q23, [c_ptr1, #0x30]\n"
- "fmin v29.4s, v29.4s, v15.4s\n"
- "fmax v31.4s, v31.4s, v14.4s\n"
- "fmin v30.4s, v30.4s, v15.4s\n"
- "str q24, [c_ptr2]\n"
- "fmin v31.4s, v31.4s, v15.4s\n"
- "str q25, [c_ptr2, #0x10]\n"
- "str q26, [c_ptr2, #0x20]\n"
- "str q27, [c_ptr2, #0x30]\n"
- "str q28, [c_ptr3]\n"
- "str q29, [c_ptr3, #0x10]\n"
- "str q30, [c_ptr3, #0x20]\n"
- "str q31, [c_ptr3, #0x30]\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq a_ptr3\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- ".unreq c_ptr3\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks)
- : [width] "r" (width), [accumulate] "r" (static_cast<uint64_t>(accumulate)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr)
- : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "x4", "x5", "cc", "memory"
- );
- break;
- }
- if (use_result_buffer) {
- for(int cy=0; cy<std::min(M-y, 4); cy++) {
- for(unsigned int cx=0; cx<width; cx++) {
- c_ptr_real[cy * ldc + cx] = result_buffer[cy * 16 + cx];
- }
- }
- }
- }
- }
-}
-
-} // namespace arm_gemm
-
-#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_16x4/x1.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_16x4/x1.cpp
deleted file mode 100644
index 3f1df76..0000000
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_16x4/x1.cpp
+++ /dev/null
@@ -1,1810 +0,0 @@
-/*
- * Copyright (c) 2018-2020 Arm Limited.
- *
- * SPDX-License-Identifier: MIT
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to
- * deal in the Software without restriction, including without limitation the
- * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
- * sell copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-#ifdef __aarch64__
-
-#include <algorithm>
-
-#include "arm_gemm.hpp"
-
-#include "../../asmlib.hpp"
-#include "../../utils.hpp"
-
-namespace arm_gemm {
-
-void a64_hybrid_fp32_mla_16x4_x1(const float *A, int lda, const float *B, float *C, int ldc, int M, int N, int K, const float *bias, Activation act, bool accumulate) {
- const int K_stride = K;
- const long loops_count = ((K + 4) / 8) - 1;
- K -= loops_count * 8;
- const long regs_count = (K / 4) - 1;
- K -= (regs_count + 1) * 4;
- const long blocks_count = K / 1;
- float nullbias[16];
- if (!accumulate && !bias) {
- memset(nullbias, 0, (16 * sizeof(float)));
- }
- float minval = - static_cast<float>(std::numeric_limits<float>::infinity());
- float maxval = static_cast<float>(std::numeric_limits<float>::infinity());
- const float * const minptr = &minval;
- const float * const maxptr = &maxval;
-
- switch(act.type)
- {
- default:
- case Activation::Type::None:
- break;
- case Activation::Type::BoundedReLU:
- maxval = static_cast<float>(act.param1);
- /* fall through */
- case Activation::Type::ReLU:
- minval = 0.0f;
- break;
- }
-
- int rows_to_compute;
-
- for (int y=0; y<M; y+=rows_to_compute) {
- const float * const a_ptr0_base = A + (y * lda);
- const unsigned long ldab = lda * sizeof(float);
-
- float *c_ptr0 = C + (y * ldc);
-
- rows_to_compute = M-y;
- if (rows_to_compute > 4) {
- if (rows_to_compute % 4) {
- rows_to_compute = 4 - 1;
- } else {
- rows_to_compute = 4;
- }
- }
-
- for (int x0=0; x0<N; x0+=16ul) {
- const long width = std::min((unsigned long)N-x0, 16ul);
- long loops = loops_count;
- long regs = regs_count;
- long blocks = blocks_count;
- const float *a_ptr0 = a_ptr0_base;
- const float *b_ptr0 = B + (K_stride * x0);
- const bool use_result_buffer = (width < 16);
- float result_buffer[64];
- const unsigned long ldcb = (use_result_buffer ? 16 : ldc) * sizeof(float);
- float *c_ptr_real = c_ptr0;
- if (use_result_buffer && accumulate) {
- for(int cy=0; cy<std::min(M-y, 4); cy++) {
- for(unsigned int cx=0; cx<width; cx++) {
- result_buffer[cy * 16 + cx] = c_ptr_real[cy * ldc + cx];
- }
- }
- }
- if (use_result_buffer) {
- c_ptr0 = result_buffer;
- }
- const float *biasptr = bias ? bias+x0 : nullbias;
-
- switch(rows_to_compute) {
- case 1:
- __asm __volatile (
- "cbnz %[accumulate], 1f\n"
- "ldr q16, [%[biasptr]]\n"
- "ldr q17, [%[biasptr], #0x10]\n"
- "ldr q18, [%[biasptr], #0x20]\n"
- "ldr q19, [%[biasptr], #0x30]\n"
- "ldr q0, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ldr q8, [%[b_ptr0]]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "add %[b_ptr0], %[b_ptr0], #0x40\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "ldr q16, [%[c_ptr0]]\n"
- "ldr q17, [%[c_ptr0], #0x10]\n"
- "ldr q18, [%[c_ptr0], #0x20]\n"
- "ldr q19, [%[c_ptr0], #0x30]\n"
- "ldr q0, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ldr q8, [%[b_ptr0]]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "add %[b_ptr0], %[b_ptr0], #0x40\n"
- "cbz %[loops], 2f\n"
- "3:\n"
- "fmla v16.4s, v8.4s, v0.s[0]\n"
- "ldr q11, [%[b_ptr0], #-0x10]\n"
- "fmla v17.4s, v9.4s, v0.s[0]\n"
- "ldr q4, [%[a_ptr0]]\n"
- "fmla v18.4s, v10.4s, v0.s[0]\n"
- "ldr q8, [%[b_ptr0]]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "subs %[loops], %[loops], #0x1\n"
- "fmla v19.4s, v11.4s, v0.s[0]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "fmla v16.4s, v8.4s, v0.s[1]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "fmla v17.4s, v9.4s, v0.s[1]\n"
- "ldr q8, [%[b_ptr0], #0x40]\n"
- "ldr q9, [%[b_ptr0], #0x50]\n"
- "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n"
- "fmla v18.4s, v10.4s, v0.s[1]\n"
- "ldr q10, [%[b_ptr0], #0x60]\n"
- "fmla v19.4s, v11.4s, v0.s[1]\n"
- "ldr q11, [%[b_ptr0], #0x70]\n"
- "fmla v16.4s, v8.4s, v0.s[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- "fmla v17.4s, v9.4s, v0.s[2]\n"
- "ldr q8, [%[b_ptr0], #-0x80]\n"
- "fmla v18.4s, v10.4s, v0.s[2]\n"
- "ldr q9, [%[b_ptr0], #-0x70]\n"
- "fmla v19.4s, v11.4s, v0.s[2]\n"
- "ldr q10, [%[b_ptr0], #-0x60]\n"
- "ldr q11, [%[b_ptr0], #-0x50]\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
- "fmla v16.4s, v8.4s, v0.s[3]\n"
- "ldr q8, [%[b_ptr0], #-0x40]\n"
- "fmla v17.4s, v9.4s, v0.s[3]\n"
- "ldr q9, [%[b_ptr0], #-0x30]\n"
- "fmla v18.4s, v10.4s, v0.s[3]\n"
- "ldr q10, [%[b_ptr0], #-0x20]\n"
- "fmla v19.4s, v11.4s, v0.s[3]\n"
- "ldr q11, [%[b_ptr0], #-0x10]\n"
- "fmla v16.4s, v8.4s, v4.s[0]\n"
- "ldr q0, [%[a_ptr0], #-0x10]\n"
- "fmla v17.4s, v9.4s, v4.s[0]\n"
- "ldr q8, [%[b_ptr0]]\n"
- "fmla v18.4s, v10.4s, v4.s[0]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "fmla v19.4s, v11.4s, v4.s[0]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "fmla v16.4s, v8.4s, v4.s[1]\n"
- "ldr q8, [%[b_ptr0], #0x40]\n"
- "fmla v17.4s, v9.4s, v4.s[1]\n"
- "ldr q9, [%[b_ptr0], #0x50]\n"
- "fmla v18.4s, v10.4s, v4.s[1]\n"
- "ldr q10, [%[b_ptr0], #0x60]\n"
- "fmla v19.4s, v11.4s, v4.s[1]\n"
- "ldr q11, [%[b_ptr0], #0x70]\n"
- "fmla v16.4s, v8.4s, v4.s[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- "fmla v17.4s, v9.4s, v4.s[2]\n"
- "ldr q8, [%[b_ptr0], #-0x80]\n"
- "fmla v18.4s, v10.4s, v4.s[2]\n"
- "ldr q9, [%[b_ptr0], #-0x70]\n"
- "fmla v19.4s, v11.4s, v4.s[2]\n"
- "ldr q10, [%[b_ptr0], #-0x60]\n"
- "ldr q11, [%[b_ptr0], #-0x50]\n"
- "fmla v16.4s, v8.4s, v4.s[3]\n"
- "ldr q8, [%[b_ptr0], #-0x40]\n"
- "fmla v17.4s, v9.4s, v4.s[3]\n"
- "ldr q9, [%[b_ptr0], #-0x30]\n"
- "fmla v18.4s, v10.4s, v4.s[3]\n"
- "ldr q10, [%[b_ptr0], #-0x20]\n"
- "fmla v19.4s, v11.4s, v4.s[3]\n"
- "b.ne 3b\n"
- "2:\n"
- "ldr q11, [%[b_ptr0], #-0x10]\n"
- "prfm PSTL1KEEP, [%[c_ptr0]]\n"
- "cbz %[regs], 4f\n"
- "fmla v16.4s, v8.4s, v0.s[0]\n"
- "ldr q4, [%[a_ptr0]]\n"
- "fmla v17.4s, v9.4s, v0.s[0]\n"
- "ldr q8, [%[b_ptr0]]\n"
- "fmla v18.4s, v10.4s, v0.s[0]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "fmla v19.4s, v11.4s, v0.s[0]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "fmla v16.4s, v8.4s, v0.s[1]\n"
- "ldr q8, [%[b_ptr0], #0x40]\n"
- "fmla v17.4s, v9.4s, v0.s[1]\n"
- "ldr q9, [%[b_ptr0], #0x50]\n"
- "fmla v18.4s, v10.4s, v0.s[1]\n"
- "ldr q10, [%[b_ptr0], #0x60]\n"
- "fmla v19.4s, v11.4s, v0.s[1]\n"
- "ldr q11, [%[b_ptr0], #0x70]\n"
- "fmla v16.4s, v8.4s, v0.s[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- "fmla v17.4s, v9.4s, v0.s[2]\n"
- "ldr q8, [%[b_ptr0], #-0x80]\n"
- "fmla v18.4s, v10.4s, v0.s[2]\n"
- "ldr q9, [%[b_ptr0], #-0x70]\n"
- "fmla v19.4s, v11.4s, v0.s[2]\n"
- "ldr q10, [%[b_ptr0], #-0x60]\n"
- "ldr q11, [%[b_ptr0], #-0x50]\n"
- "fmla v16.4s, v8.4s, v0.s[3]\n"
- "ldr q8, [%[b_ptr0], #-0x40]\n"
- "fmla v17.4s, v9.4s, v0.s[3]\n"
- "ldr q9, [%[b_ptr0], #-0x30]\n"
- "fmla v18.4s, v10.4s, v0.s[3]\n"
- "ldr q10, [%[b_ptr0], #-0x20]\n"
- "fmla v19.4s, v11.4s, v0.s[3]\n"
- "ldr q11, [%[b_ptr0], #-0x10]\n"
- "fmla v16.4s, v8.4s, v4.s[0]\n"
- "ldr q8, [%[b_ptr0]]\n"
- "fmla v17.4s, v9.4s, v4.s[0]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "fmla v18.4s, v10.4s, v4.s[0]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "fmla v19.4s, v11.4s, v4.s[0]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "fmla v16.4s, v8.4s, v4.s[1]\n"
- "ldr q8, [%[b_ptr0], #0x40]\n"
- "fmla v17.4s, v9.4s, v4.s[1]\n"
- "ldr q9, [%[b_ptr0], #0x50]\n"
- "fmla v18.4s, v10.4s, v4.s[1]\n"
- "ldr q10, [%[b_ptr0], #0x60]\n"
- "fmla v19.4s, v11.4s, v4.s[1]\n"
- "ldr q11, [%[b_ptr0], #0x70]\n"
- "fmla v16.4s, v8.4s, v4.s[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- "fmla v17.4s, v9.4s, v4.s[2]\n"
- "ldr q8, [%[b_ptr0], #-0x80]\n"
- "fmla v18.4s, v10.4s, v4.s[2]\n"
- "ldr q9, [%[b_ptr0], #-0x70]\n"
- "fmla v19.4s, v11.4s, v4.s[2]\n"
- "ldr q10, [%[b_ptr0], #-0x60]\n"
- "ldr q11, [%[b_ptr0], #-0x50]\n"
- "add %[b_ptr0], %[b_ptr0], #-0x40\n"
- "fmla v16.4s, v8.4s, v4.s[3]\n"
- "fmla v17.4s, v9.4s, v4.s[3]\n"
- "fmla v18.4s, v10.4s, v4.s[3]\n"
- "fmla v19.4s, v11.4s, v4.s[3]\n"
- "b 5f\n"
- "4:\n"
- "fmla v16.4s, v8.4s, v0.s[0]\n"
- "ldr q8, [%[b_ptr0]]\n"
- "fmla v17.4s, v9.4s, v0.s[0]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "fmla v18.4s, v10.4s, v0.s[0]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "fmla v19.4s, v11.4s, v0.s[0]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "fmla v16.4s, v8.4s, v0.s[1]\n"
- "ldr q8, [%[b_ptr0], #0x40]\n"
- "fmla v17.4s, v9.4s, v0.s[1]\n"
- "ldr q9, [%[b_ptr0], #0x50]\n"
- "fmla v18.4s, v10.4s, v0.s[1]\n"
- "ldr q10, [%[b_ptr0], #0x60]\n"
- "fmla v19.4s, v11.4s, v0.s[1]\n"
- "ldr q11, [%[b_ptr0], #0x70]\n"
- "fmla v16.4s, v8.4s, v0.s[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- "fmla v17.4s, v9.4s, v0.s[2]\n"
- "ldr q8, [%[b_ptr0], #-0x80]\n"
- "fmla v18.4s, v10.4s, v0.s[2]\n"
- "ldr q9, [%[b_ptr0], #-0x70]\n"
- "fmla v19.4s, v11.4s, v0.s[2]\n"
- "ldr q10, [%[b_ptr0], #-0x60]\n"
- "ldr q11, [%[b_ptr0], #-0x50]\n"
- "add %[b_ptr0], %[b_ptr0], #-0x40\n"
- "fmla v16.4s, v8.4s, v0.s[3]\n"
- "fmla v17.4s, v9.4s, v0.s[3]\n"
- "fmla v18.4s, v10.4s, v0.s[3]\n"
- "fmla v19.4s, v11.4s, v0.s[3]\n"
- "5:\n"
- "cbz %[blocks], 6f\n"
- "7:\n"
- "ldr q8, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "ldr s0, [%[a_ptr0]]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "add %[a_ptr0], %[a_ptr0], #0x4\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "add %[b_ptr0], %[b_ptr0], #0x40\n"
- "fmla v16.4s, v8.4s, v0.s[0]\n"
- "fmla v17.4s, v9.4s, v0.s[0]\n"
- "fmla v18.4s, v10.4s, v0.s[0]\n"
- "fmla v19.4s, v11.4s, v0.s[0]\n"
- "b.ne 7b\n"
- "6:\n"
- "ld1r {v14.4s}, [%[minptr]]\n"
- "ld1r {v15.4s}, [%[maxptr]]\n"
- "fmax v16.4s, v16.4s, v14.4s\n"
- "fmax v17.4s, v17.4s, v14.4s\n"
- "fmax v18.4s, v18.4s, v14.4s\n"
- "fmax v19.4s, v19.4s, v14.4s\n"
- "fmin v16.4s, v16.4s, v15.4s\n"
- "fmin v17.4s, v17.4s, v15.4s\n"
- "fmin v18.4s, v18.4s, v15.4s\n"
- "fmin v19.4s, v19.4s, v15.4s\n"
- "str q16, [%[c_ptr0]]\n"
- "str q17, [%[c_ptr0], #0x10]\n"
- "str q18, [%[c_ptr0], #0x20]\n"
- "str q19, [%[c_ptr0], #0x30]\n"
- "add %[c_ptr0], %[c_ptr0], #0x40\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks)
- : [width] "r" (width), [accumulate] "r" (static_cast<uint64_t>(accumulate)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr)
- : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "cc", "memory"
- );
- break;
- case 2:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "c_ptr1 .req X1\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "cbnz %[accumulate], 1f\n"
- "ldr q16, [%[biasptr]]\n"
- "ldr q17, [%[biasptr], #0x10]\n"
- "ldr q18, [%[biasptr], #0x20]\n"
- "ldr q19, [%[biasptr], #0x30]\n"
- "mov v20.16b, v16.16b\n"
- "ldr q0, [%[a_ptr0]]\n"
- "mov v21.16b, v17.16b\n"
- "ldr q1, [a_ptr1]\n"
- "mov v22.16b, v18.16b\n"
- "ldr q8, [%[b_ptr0]]\n"
- "mov v23.16b, v19.16b\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "add %[b_ptr0], %[b_ptr0], #0x40\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "ldr q16, [%[c_ptr0]]\n"
- "ldr q17, [%[c_ptr0], #0x10]\n"
- "ldr q18, [%[c_ptr0], #0x20]\n"
- "ldr q19, [%[c_ptr0], #0x30]\n"
- "ldr q20, [c_ptr1]\n"
- "ldr q21, [c_ptr1, #0x10]\n"
- "ldr q22, [c_ptr1, #0x20]\n"
- "ldr q23, [c_ptr1, #0x30]\n"
- "ldr q0, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ldr q1, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "ldr q8, [%[b_ptr0]]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "add %[b_ptr0], %[b_ptr0], #0x40\n"
- "cbz %[loops], 2f\n"
- "3:\n"
- "fmla v16.4s, v8.4s, v0.s[0]\n"
- "ldr q11, [%[b_ptr0], #-0x10]\n"
- "fmla v20.4s, v8.4s, v1.s[0]\n"
- "ldr q4, [%[a_ptr0]]\n"
- "fmla v17.4s, v9.4s, v0.s[0]\n"
- "ldr q5, [a_ptr1]\n"
- "fmla v21.4s, v9.4s, v1.s[0]\n"
- "ldr q8, [%[b_ptr0]]\n"
- "fmla v18.4s, v10.4s, v0.s[0]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "fmla v22.4s, v10.4s, v1.s[0]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "fmla v19.4s, v11.4s, v0.s[0]\n"
- "subs %[loops], %[loops], #0x1\n"
- "fmla v23.4s, v11.4s, v1.s[0]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "fmla v16.4s, v8.4s, v0.s[1]\n"
- "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n"
- "fmla v20.4s, v8.4s, v1.s[1]\n"
- "ldr q8, [%[b_ptr0], #0x40]\n"
- "fmla v17.4s, v9.4s, v0.s[1]\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
- "fmla v21.4s, v9.4s, v1.s[1]\n"
- "ldr q9, [%[b_ptr0], #0x50]\n"
- "fmla v18.4s, v10.4s, v0.s[1]\n"
- "add a_ptr1, a_ptr1, #0x20\n"
- "fmla v22.4s, v10.4s, v1.s[1]\n"
- "ldr q10, [%[b_ptr0], #0x60]\n"
- "fmla v19.4s, v11.4s, v0.s[1]\n"
- "prfm PLDL1KEEP, [a_ptr1, #0x40]\n"
- "fmla v23.4s, v11.4s, v1.s[1]\n"
- "ldr q11, [%[b_ptr0], #0x70]\n"
- "fmla v16.4s, v8.4s, v0.s[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- "fmla v20.4s, v8.4s, v1.s[2]\n"
- "ldr q8, [%[b_ptr0], #-0x80]\n"
- "fmla v17.4s, v9.4s, v0.s[2]\n"
- "fmla v21.4s, v9.4s, v1.s[2]\n"
- "ldr q9, [%[b_ptr0], #-0x70]\n"
- "fmla v18.4s, v10.4s, v0.s[2]\n"
- "fmla v22.4s, v10.4s, v1.s[2]\n"
- "ldr q10, [%[b_ptr0], #-0x60]\n"
- "fmla v19.4s, v11.4s, v0.s[2]\n"
- "fmla v23.4s, v11.4s, v1.s[2]\n"
- "ldr q11, [%[b_ptr0], #-0x50]\n"
- "fmla v16.4s, v8.4s, v0.s[3]\n"
- "fmla v20.4s, v8.4s, v1.s[3]\n"
- "ldr q8, [%[b_ptr0], #-0x40]\n"
- "fmla v17.4s, v9.4s, v0.s[3]\n"
- "fmla v21.4s, v9.4s, v1.s[3]\n"
- "ldr q9, [%[b_ptr0], #-0x30]\n"
- "fmla v18.4s, v10.4s, v0.s[3]\n"
- "fmla v22.4s, v10.4s, v1.s[3]\n"
- "ldr q10, [%[b_ptr0], #-0x20]\n"
- "fmla v19.4s, v11.4s, v0.s[3]\n"
- "ldr q0, [%[a_ptr0], #-0x10]\n"
- "fmla v23.4s, v11.4s, v1.s[3]\n"
- "ldr q11, [%[b_ptr0], #-0x10]\n"
- "fmla v16.4s, v8.4s, v4.s[0]\n"
- "ldr q1, [a_ptr1, #-0x10]\n"
- "fmla v20.4s, v8.4s, v5.s[0]\n"
- "ldr q8, [%[b_ptr0]]\n"
- "fmla v17.4s, v9.4s, v4.s[0]\n"
- "fmla v21.4s, v9.4s, v5.s[0]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "fmla v18.4s, v10.4s, v4.s[0]\n"
- "fmla v22.4s, v10.4s, v5.s[0]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "fmla v19.4s, v11.4s, v4.s[0]\n"
- "fmla v23.4s, v11.4s, v5.s[0]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "fmla v16.4s, v8.4s, v4.s[1]\n"
- "fmla v20.4s, v8.4s, v5.s[1]\n"
- "ldr q8, [%[b_ptr0], #0x40]\n"
- "fmla v17.4s, v9.4s, v4.s[1]\n"
- "fmla v21.4s, v9.4s, v5.s[1]\n"
- "ldr q9, [%[b_ptr0], #0x50]\n"
- "fmla v18.4s, v10.4s, v4.s[1]\n"
- "fmla v22.4s, v10.4s, v5.s[1]\n"
- "ldr q10, [%[b_ptr0], #0x60]\n"
- "fmla v19.4s, v11.4s, v4.s[1]\n"
- "fmla v23.4s, v11.4s, v5.s[1]\n"
- "ldr q11, [%[b_ptr0], #0x70]\n"
- "fmla v16.4s, v8.4s, v4.s[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- "fmla v20.4s, v8.4s, v5.s[2]\n"
- "ldr q8, [%[b_ptr0], #-0x80]\n"
- "fmla v17.4s, v9.4s, v4.s[2]\n"
- "fmla v21.4s, v9.4s, v5.s[2]\n"
- "ldr q9, [%[b_ptr0], #-0x70]\n"
- "fmla v18.4s, v10.4s, v4.s[2]\n"
- "fmla v22.4s, v10.4s, v5.s[2]\n"
- "ldr q10, [%[b_ptr0], #-0x60]\n"
- "fmla v19.4s, v11.4s, v4.s[2]\n"
- "fmla v23.4s, v11.4s, v5.s[2]\n"
- "ldr q11, [%[b_ptr0], #-0x50]\n"
- "fmla v16.4s, v8.4s, v4.s[3]\n"
- "fmla v20.4s, v8.4s, v5.s[3]\n"
- "ldr q8, [%[b_ptr0], #-0x40]\n"
- "fmla v17.4s, v9.4s, v4.s[3]\n"
- "fmla v21.4s, v9.4s, v5.s[3]\n"
- "ldr q9, [%[b_ptr0], #-0x30]\n"
- "fmla v18.4s, v10.4s, v4.s[3]\n"
- "fmla v22.4s, v10.4s, v5.s[3]\n"
- "ldr q10, [%[b_ptr0], #-0x20]\n"
- "fmla v19.4s, v11.4s, v4.s[3]\n"
- "fmla v23.4s, v11.4s, v5.s[3]\n"
- "b.ne 3b\n"
- "2:\n"
- "ldr q11, [%[b_ptr0], #-0x10]\n"
- "prfm PSTL1KEEP, [%[c_ptr0]]\n"
- "prfm PSTL1KEEP, [c_ptr1]\n"
- "cbz %[regs], 4f\n"
- "fmla v16.4s, v8.4s, v0.s[0]\n"
- "ldr q4, [%[a_ptr0]]\n"
- "fmla v20.4s, v8.4s, v1.s[0]\n"
- "ldr q5, [a_ptr1]\n"
- "fmla v17.4s, v9.4s, v0.s[0]\n"
- "ldr q8, [%[b_ptr0]]\n"
- "fmla v21.4s, v9.4s, v1.s[0]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "fmla v18.4s, v10.4s, v0.s[0]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "fmla v22.4s, v10.4s, v1.s[0]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "fmla v19.4s, v11.4s, v0.s[0]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "fmla v23.4s, v11.4s, v1.s[0]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "fmla v16.4s, v8.4s, v0.s[1]\n"
- "fmla v20.4s, v8.4s, v1.s[1]\n"
- "ldr q8, [%[b_ptr0], #0x40]\n"
- "fmla v17.4s, v9.4s, v0.s[1]\n"
- "fmla v21.4s, v9.4s, v1.s[1]\n"
- "ldr q9, [%[b_ptr0], #0x50]\n"
- "fmla v18.4s, v10.4s, v0.s[1]\n"
- "fmla v22.4s, v10.4s, v1.s[1]\n"
- "ldr q10, [%[b_ptr0], #0x60]\n"
- "fmla v19.4s, v11.4s, v0.s[1]\n"
- "fmla v23.4s, v11.4s, v1.s[1]\n"
- "ldr q11, [%[b_ptr0], #0x70]\n"
- "fmla v16.4s, v8.4s, v0.s[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- "fmla v20.4s, v8.4s, v1.s[2]\n"
- "ldr q8, [%[b_ptr0], #-0x80]\n"
- "fmla v17.4s, v9.4s, v0.s[2]\n"
- "fmla v21.4s, v9.4s, v1.s[2]\n"
- "ldr q9, [%[b_ptr0], #-0x70]\n"
- "fmla v18.4s, v10.4s, v0.s[2]\n"
- "fmla v22.4s, v10.4s, v1.s[2]\n"
- "ldr q10, [%[b_ptr0], #-0x60]\n"
- "fmla v19.4s, v11.4s, v0.s[2]\n"
- "fmla v23.4s, v11.4s, v1.s[2]\n"
- "ldr q11, [%[b_ptr0], #-0x50]\n"
- "fmla v16.4s, v8.4s, v0.s[3]\n"
- "fmla v20.4s, v8.4s, v1.s[3]\n"
- "ldr q8, [%[b_ptr0], #-0x40]\n"
- "fmla v17.4s, v9.4s, v0.s[3]\n"
- "fmla v21.4s, v9.4s, v1.s[3]\n"
- "ldr q9, [%[b_ptr0], #-0x30]\n"
- "fmla v18.4s, v10.4s, v0.s[3]\n"
- "fmla v22.4s, v10.4s, v1.s[3]\n"
- "ldr q10, [%[b_ptr0], #-0x20]\n"
- "fmla v19.4s, v11.4s, v0.s[3]\n"
- "fmla v23.4s, v11.4s, v1.s[3]\n"
- "ldr q11, [%[b_ptr0], #-0x10]\n"
- "fmla v16.4s, v8.4s, v4.s[0]\n"
- "fmla v20.4s, v8.4s, v5.s[0]\n"
- "ldr q8, [%[b_ptr0]]\n"
- "fmla v17.4s, v9.4s, v4.s[0]\n"
- "fmla v21.4s, v9.4s, v5.s[0]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "fmla v18.4s, v10.4s, v4.s[0]\n"
- "fmla v22.4s, v10.4s, v5.s[0]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "fmla v19.4s, v11.4s, v4.s[0]\n"
- "fmla v23.4s, v11.4s, v5.s[0]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "fmla v16.4s, v8.4s, v4.s[1]\n"
- "fmla v20.4s, v8.4s, v5.s[1]\n"
- "ldr q8, [%[b_ptr0], #0x40]\n"
- "fmla v17.4s, v9.4s, v4.s[1]\n"
- "fmla v21.4s, v9.4s, v5.s[1]\n"
- "ldr q9, [%[b_ptr0], #0x50]\n"
- "fmla v18.4s, v10.4s, v4.s[1]\n"
- "fmla v22.4s, v10.4s, v5.s[1]\n"
- "ldr q10, [%[b_ptr0], #0x60]\n"
- "fmla v19.4s, v11.4s, v4.s[1]\n"
- "fmla v23.4s, v11.4s, v5.s[1]\n"
- "ldr q11, [%[b_ptr0], #0x70]\n"
- "fmla v16.4s, v8.4s, v4.s[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- "fmla v20.4s, v8.4s, v5.s[2]\n"
- "ldr q8, [%[b_ptr0], #-0x80]\n"
- "fmla v17.4s, v9.4s, v4.s[2]\n"
- "fmla v21.4s, v9.4s, v5.s[2]\n"
- "ldr q9, [%[b_ptr0], #-0x70]\n"
- "fmla v18.4s, v10.4s, v4.s[2]\n"
- "fmla v22.4s, v10.4s, v5.s[2]\n"
- "ldr q10, [%[b_ptr0], #-0x60]\n"
- "fmla v19.4s, v11.4s, v4.s[2]\n"
- "fmla v23.4s, v11.4s, v5.s[2]\n"
- "ldr q11, [%[b_ptr0], #-0x50]\n"
- "fmla v16.4s, v8.4s, v4.s[3]\n"
- "add %[b_ptr0], %[b_ptr0], #-0x40\n"
- "fmla v20.4s, v8.4s, v5.s[3]\n"
- "fmla v17.4s, v9.4s, v4.s[3]\n"
- "fmla v21.4s, v9.4s, v5.s[3]\n"
- "fmla v18.4s, v10.4s, v4.s[3]\n"
- "fmla v22.4s, v10.4s, v5.s[3]\n"
- "fmla v19.4s, v11.4s, v4.s[3]\n"
- "fmla v23.4s, v11.4s, v5.s[3]\n"
- "b 5f\n"
- "4:\n"
- "fmla v16.4s, v8.4s, v0.s[0]\n"
- "fmla v20.4s, v8.4s, v1.s[0]\n"
- "ldr q8, [%[b_ptr0]]\n"
- "fmla v17.4s, v9.4s, v0.s[0]\n"
- "fmla v21.4s, v9.4s, v1.s[0]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "fmla v18.4s, v10.4s, v0.s[0]\n"
- "fmla v22.4s, v10.4s, v1.s[0]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "fmla v19.4s, v11.4s, v0.s[0]\n"
- "fmla v23.4s, v11.4s, v1.s[0]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "fmla v16.4s, v8.4s, v0.s[1]\n"
- "fmla v20.4s, v8.4s, v1.s[1]\n"
- "ldr q8, [%[b_ptr0], #0x40]\n"
- "fmla v17.4s, v9.4s, v0.s[1]\n"
- "fmla v21.4s, v9.4s, v1.s[1]\n"
- "ldr q9, [%[b_ptr0], #0x50]\n"
- "fmla v18.4s, v10.4s, v0.s[1]\n"
- "fmla v22.4s, v10.4s, v1.s[1]\n"
- "ldr q10, [%[b_ptr0], #0x60]\n"
- "fmla v19.4s, v11.4s, v0.s[1]\n"
- "fmla v23.4s, v11.4s, v1.s[1]\n"
- "ldr q11, [%[b_ptr0], #0x70]\n"
- "fmla v16.4s, v8.4s, v0.s[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- "fmla v20.4s, v8.4s, v1.s[2]\n"
- "ldr q8, [%[b_ptr0], #-0x80]\n"
- "fmla v17.4s, v9.4s, v0.s[2]\n"
- "fmla v21.4s, v9.4s, v1.s[2]\n"
- "ldr q9, [%[b_ptr0], #-0x70]\n"
- "fmla v18.4s, v10.4s, v0.s[2]\n"
- "fmla v22.4s, v10.4s, v1.s[2]\n"
- "ldr q10, [%[b_ptr0], #-0x60]\n"
- "fmla v19.4s, v11.4s, v0.s[2]\n"
- "fmla v23.4s, v11.4s, v1.s[2]\n"
- "ldr q11, [%[b_ptr0], #-0x50]\n"
- "fmla v16.4s, v8.4s, v0.s[3]\n"
- "add %[b_ptr0], %[b_ptr0], #-0x40\n"
- "fmla v20.4s, v8.4s, v1.s[3]\n"
- "fmla v17.4s, v9.4s, v0.s[3]\n"
- "fmla v21.4s, v9.4s, v1.s[3]\n"
- "fmla v18.4s, v10.4s, v0.s[3]\n"
- "fmla v22.4s, v10.4s, v1.s[3]\n"
- "fmla v19.4s, v11.4s, v0.s[3]\n"
- "fmla v23.4s, v11.4s, v1.s[3]\n"
- "5:\n"
- "cbz %[blocks], 6f\n"
- "7:\n"
- "ldr q8, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "ldr s0, [%[a_ptr0]]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "add %[a_ptr0], %[a_ptr0], #0x4\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "add %[b_ptr0], %[b_ptr0], #0x40\n"
- "fmla v16.4s, v8.4s, v0.s[0]\n"
- "ldr s1, [a_ptr1]\n"
- "fmla v17.4s, v9.4s, v0.s[0]\n"
- "add a_ptr1, a_ptr1, #0x4\n"
- "fmla v18.4s, v10.4s, v0.s[0]\n"
- "fmla v20.4s, v8.4s, v1.s[0]\n"
- "fmla v21.4s, v9.4s, v1.s[0]\n"
- "fmla v22.4s, v10.4s, v1.s[0]\n"
- "fmla v19.4s, v11.4s, v0.s[0]\n"
- "fmla v23.4s, v11.4s, v1.s[0]\n"
- "b.ne 7b\n"
- "6:\n"
- "ld1r {v14.4s}, [%[minptr]]\n"
- "ld1r {v15.4s}, [%[maxptr]]\n"
- "fmax v16.4s, v16.4s, v14.4s\n"
- "fmax v17.4s, v17.4s, v14.4s\n"
- "fmax v18.4s, v18.4s, v14.4s\n"
- "fmax v19.4s, v19.4s, v14.4s\n"
- "fmin v16.4s, v16.4s, v15.4s\n"
- "fmin v17.4s, v17.4s, v15.4s\n"
- "fmin v18.4s, v18.4s, v15.4s\n"
- "fmin v19.4s, v19.4s, v15.4s\n"
- "str q16, [%[c_ptr0]]\n"
- "fmax v20.4s, v20.4s, v14.4s\n"
- "fmax v21.4s, v21.4s, v14.4s\n"
- "fmax v22.4s, v22.4s, v14.4s\n"
- "str q17, [%[c_ptr0], #0x10]\n"
- "fmax v23.4s, v23.4s, v14.4s\n"
- "fmin v20.4s, v20.4s, v15.4s\n"
- "fmin v21.4s, v21.4s, v15.4s\n"
- "str q18, [%[c_ptr0], #0x20]\n"
- "fmin v22.4s, v22.4s, v15.4s\n"
- "fmin v23.4s, v23.4s, v15.4s\n"
- "str q19, [%[c_ptr0], #0x30]\n"
- "add %[c_ptr0], %[c_ptr0], #0x40\n"
- "str q20, [c_ptr1]\n"
- "str q21, [c_ptr1, #0x10]\n"
- "str q22, [c_ptr1, #0x20]\n"
- "str q23, [c_ptr1, #0x30]\n"
- ".unreq a_ptr1\n"
- ".unreq c_ptr1\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks)
- : [width] "r" (width), [accumulate] "r" (static_cast<uint64_t>(accumulate)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr)
- : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "cc", "memory"
- );
- break;
- case 3:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "c_ptr1 .req X2\n"
- "c_ptr2 .req X3\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "cbnz %[accumulate], 1f\n"
- "ldr q16, [%[biasptr]]\n"
- "ldr q17, [%[biasptr], #0x10]\n"
- "ldr q18, [%[biasptr], #0x20]\n"
- "ldr q19, [%[biasptr], #0x30]\n"
- "mov v20.16b, v16.16b\n"
- "ldr q0, [%[a_ptr0]]\n"
- "mov v21.16b, v17.16b\n"
- "ldr q1, [a_ptr1]\n"
- "mov v22.16b, v18.16b\n"
- "ldr q2, [a_ptr2]\n"
- "mov v23.16b, v19.16b\n"
- "ldr q8, [%[b_ptr0]]\n"
- "mov v24.16b, v16.16b\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "mov v25.16b, v17.16b\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "mov v26.16b, v18.16b\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "mov v27.16b, v19.16b\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "add %[b_ptr0], %[b_ptr0], #0x40\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "ldr q16, [%[c_ptr0]]\n"
- "ldr q17, [%[c_ptr0], #0x10]\n"
- "ldr q18, [%[c_ptr0], #0x20]\n"
- "ldr q19, [%[c_ptr0], #0x30]\n"
- "ldr q20, [c_ptr1]\n"
- "ldr q21, [c_ptr1, #0x10]\n"
- "ldr q22, [c_ptr1, #0x20]\n"
- "ldr q23, [c_ptr1, #0x30]\n"
- "ldr q24, [c_ptr2]\n"
- "ldr q25, [c_ptr2, #0x10]\n"
- "ldr q26, [c_ptr2, #0x20]\n"
- "ldr q27, [c_ptr2, #0x30]\n"
- "ldr q0, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ldr q1, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "ldr q2, [a_ptr2]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "ldr q8, [%[b_ptr0]]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "add %[b_ptr0], %[b_ptr0], #0x40\n"
- "cbz %[loops], 2f\n"
- "3:\n"
- "fmla v16.4s, v8.4s, v0.s[0]\n"
- "ldr q11, [%[b_ptr0], #-0x10]\n"
- "fmla v20.4s, v8.4s, v1.s[0]\n"
- "ldr q4, [%[a_ptr0]]\n"
- "fmla v24.4s, v8.4s, v2.s[0]\n"
- "ldr q5, [a_ptr1]\n"
- "fmla v17.4s, v9.4s, v0.s[0]\n"
- "ldr q6, [a_ptr2]\n"
- "fmla v21.4s, v9.4s, v1.s[0]\n"
- "ldr q8, [%[b_ptr0]]\n"
- "fmla v25.4s, v9.4s, v2.s[0]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "fmla v18.4s, v10.4s, v0.s[0]\n"
- "subs %[loops], %[loops], #0x1\n"
- "fmla v22.4s, v10.4s, v1.s[0]\n"
- "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n"
- "fmla v26.4s, v10.4s, v2.s[0]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "fmla v19.4s, v11.4s, v0.s[0]\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
- "fmla v23.4s, v11.4s, v1.s[0]\n"
- "add a_ptr1, a_ptr1, #0x20\n"
- "fmla v27.4s, v11.4s, v2.s[0]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "fmla v16.4s, v8.4s, v0.s[1]\n"
- "add a_ptr2, a_ptr2, #0x20\n"
- "fmla v20.4s, v8.4s, v1.s[1]\n"
- "prfm PLDL1KEEP, [a_ptr1, #0x40]\n"
- "fmla v24.4s, v8.4s, v2.s[1]\n"
- "ldr q8, [%[b_ptr0], #0x40]\n"
- "fmla v17.4s, v9.4s, v0.s[1]\n"
- "prfm PLDL1KEEP, [a_ptr2, #0x40]\n"
- "fmla v21.4s, v9.4s, v1.s[1]\n"
- "fmla v25.4s, v9.4s, v2.s[1]\n"
- "ldr q9, [%[b_ptr0], #0x50]\n"
- "fmla v18.4s, v10.4s, v0.s[1]\n"
- "fmla v22.4s, v10.4s, v1.s[1]\n"
- "fmla v26.4s, v10.4s, v2.s[1]\n"
- "ldr q10, [%[b_ptr0], #0x60]\n"
- "fmla v19.4s, v11.4s, v0.s[1]\n"
- "fmla v23.4s, v11.4s, v1.s[1]\n"
- "fmla v27.4s, v11.4s, v2.s[1]\n"
- "ldr q11, [%[b_ptr0], #0x70]\n"
- "fmla v16.4s, v8.4s, v0.s[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- "fmla v20.4s, v8.4s, v1.s[2]\n"
- "fmla v24.4s, v8.4s, v2.s[2]\n"
- "ldr q8, [%[b_ptr0], #-0x80]\n"
- "fmla v17.4s, v9.4s, v0.s[2]\n"
- "fmla v21.4s, v9.4s, v1.s[2]\n"
- "fmla v25.4s, v9.4s, v2.s[2]\n"
- "ldr q9, [%[b_ptr0], #-0x70]\n"
- "fmla v18.4s, v10.4s, v0.s[2]\n"
- "fmla v22.4s, v10.4s, v1.s[2]\n"
- "fmla v26.4s, v10.4s, v2.s[2]\n"
- "ldr q10, [%[b_ptr0], #-0x60]\n"
- "fmla v19.4s, v11.4s, v0.s[2]\n"
- "fmla v23.4s, v11.4s, v1.s[2]\n"
- "fmla v27.4s, v11.4s, v2.s[2]\n"
- "ldr q11, [%[b_ptr0], #-0x50]\n"
- "fmla v16.4s, v8.4s, v0.s[3]\n"
- "fmla v20.4s, v8.4s, v1.s[3]\n"
- "fmla v24.4s, v8.4s, v2.s[3]\n"
- "ldr q8, [%[b_ptr0], #-0x40]\n"
- "fmla v17.4s, v9.4s, v0.s[3]\n"
- "fmla v21.4s, v9.4s, v1.s[3]\n"
- "fmla v25.4s, v9.4s, v2.s[3]\n"
- "ldr q9, [%[b_ptr0], #-0x30]\n"
- "fmla v18.4s, v10.4s, v0.s[3]\n"
- "fmla v22.4s, v10.4s, v1.s[3]\n"
- "fmla v26.4s, v10.4s, v2.s[3]\n"
- "ldr q10, [%[b_ptr0], #-0x20]\n"
- "fmla v19.4s, v11.4s, v0.s[3]\n"
- "ldr q0, [%[a_ptr0], #-0x10]\n"
- "fmla v23.4s, v11.4s, v1.s[3]\n"
- "ldr q1, [a_ptr1, #-0x10]\n"
- "fmla v27.4s, v11.4s, v2.s[3]\n"
- "ldr q11, [%[b_ptr0], #-0x10]\n"
- "fmla v16.4s, v8.4s, v4.s[0]\n"
- "ldr q2, [a_ptr2, #-0x10]\n"
- "fmla v20.4s, v8.4s, v5.s[0]\n"
- "fmla v24.4s, v8.4s, v6.s[0]\n"
- "ldr q8, [%[b_ptr0]]\n"
- "fmla v17.4s, v9.4s, v4.s[0]\n"
- "fmla v21.4s, v9.4s, v5.s[0]\n"
- "fmla v25.4s, v9.4s, v6.s[0]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "fmla v18.4s, v10.4s, v4.s[0]\n"
- "fmla v22.4s, v10.4s, v5.s[0]\n"
- "fmla v26.4s, v10.4s, v6.s[0]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "fmla v19.4s, v11.4s, v4.s[0]\n"
- "fmla v23.4s, v11.4s, v5.s[0]\n"
- "fmla v27.4s, v11.4s, v6.s[0]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "fmla v16.4s, v8.4s, v4.s[1]\n"
- "fmla v20.4s, v8.4s, v5.s[1]\n"
- "fmla v24.4s, v8.4s, v6.s[1]\n"
- "ldr q8, [%[b_ptr0], #0x40]\n"
- "fmla v17.4s, v9.4s, v4.s[1]\n"
- "fmla v21.4s, v9.4s, v5.s[1]\n"
- "fmla v25.4s, v9.4s, v6.s[1]\n"
- "ldr q9, [%[b_ptr0], #0x50]\n"
- "fmla v18.4s, v10.4s, v4.s[1]\n"
- "fmla v22.4s, v10.4s, v5.s[1]\n"
- "fmla v26.4s, v10.4s, v6.s[1]\n"
- "ldr q10, [%[b_ptr0], #0x60]\n"
- "fmla v19.4s, v11.4s, v4.s[1]\n"
- "fmla v23.4s, v11.4s, v5.s[1]\n"
- "fmla v27.4s, v11.4s, v6.s[1]\n"
- "ldr q11, [%[b_ptr0], #0x70]\n"
- "fmla v16.4s, v8.4s, v4.s[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- "fmla v20.4s, v8.4s, v5.s[2]\n"
- "fmla v24.4s, v8.4s, v6.s[2]\n"
- "ldr q8, [%[b_ptr0], #-0x80]\n"
- "fmla v17.4s, v9.4s, v4.s[2]\n"
- "fmla v21.4s, v9.4s, v5.s[2]\n"
- "fmla v25.4s, v9.4s, v6.s[2]\n"
- "ldr q9, [%[b_ptr0], #-0x70]\n"
- "fmla v18.4s, v10.4s, v4.s[2]\n"
- "fmla v22.4s, v10.4s, v5.s[2]\n"
- "fmla v26.4s, v10.4s, v6.s[2]\n"
- "ldr q10, [%[b_ptr0], #-0x60]\n"
- "fmla v19.4s, v11.4s, v4.s[2]\n"
- "fmla v23.4s, v11.4s, v5.s[2]\n"
- "fmla v27.4s, v11.4s, v6.s[2]\n"
- "ldr q11, [%[b_ptr0], #-0x50]\n"
- "fmla v16.4s, v8.4s, v4.s[3]\n"
- "fmla v20.4s, v8.4s, v5.s[3]\n"
- "fmla v24.4s, v8.4s, v6.s[3]\n"
- "ldr q8, [%[b_ptr0], #-0x40]\n"
- "fmla v17.4s, v9.4s, v4.s[3]\n"
- "fmla v21.4s, v9.4s, v5.s[3]\n"
- "fmla v25.4s, v9.4s, v6.s[3]\n"
- "ldr q9, [%[b_ptr0], #-0x30]\n"
- "fmla v18.4s, v10.4s, v4.s[3]\n"
- "fmla v22.4s, v10.4s, v5.s[3]\n"
- "fmla v26.4s, v10.4s, v6.s[3]\n"
- "ldr q10, [%[b_ptr0], #-0x20]\n"
- "fmla v19.4s, v11.4s, v4.s[3]\n"
- "fmla v23.4s, v11.4s, v5.s[3]\n"
- "fmla v27.4s, v11.4s, v6.s[3]\n"
- "b.ne 3b\n"
- "2:\n"
- "ldr q11, [%[b_ptr0], #-0x10]\n"
- "prfm PSTL1KEEP, [%[c_ptr0]]\n"
- "prfm PSTL1KEEP, [c_ptr1]\n"
- "prfm PSTL1KEEP, [c_ptr2]\n"
- "cbz %[regs], 4f\n"
- "fmla v16.4s, v8.4s, v0.s[0]\n"
- "ldr q4, [%[a_ptr0]]\n"
- "fmla v20.4s, v8.4s, v1.s[0]\n"
- "ldr q5, [a_ptr1]\n"
- "fmla v24.4s, v8.4s, v2.s[0]\n"
- "ldr q6, [a_ptr2]\n"
- "fmla v17.4s, v9.4s, v0.s[0]\n"
- "ldr q8, [%[b_ptr0]]\n"
- "fmla v21.4s, v9.4s, v1.s[0]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "fmla v25.4s, v9.4s, v2.s[0]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "fmla v18.4s, v10.4s, v0.s[0]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "fmla v22.4s, v10.4s, v1.s[0]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "fmla v26.4s, v10.4s, v2.s[0]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "fmla v19.4s, v11.4s, v0.s[0]\n"
- "fmla v23.4s, v11.4s, v1.s[0]\n"
- "fmla v27.4s, v11.4s, v2.s[0]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "fmla v16.4s, v8.4s, v0.s[1]\n"
- "fmla v20.4s, v8.4s, v1.s[1]\n"
- "fmla v24.4s, v8.4s, v2.s[1]\n"
- "ldr q8, [%[b_ptr0], #0x40]\n"
- "fmla v17.4s, v9.4s, v0.s[1]\n"
- "fmla v21.4s, v9.4s, v1.s[1]\n"
- "fmla v25.4s, v9.4s, v2.s[1]\n"
- "ldr q9, [%[b_ptr0], #0x50]\n"
- "fmla v18.4s, v10.4s, v0.s[1]\n"
- "fmla v22.4s, v10.4s, v1.s[1]\n"
- "fmla v26.4s, v10.4s, v2.s[1]\n"
- "ldr q10, [%[b_ptr0], #0x60]\n"
- "fmla v19.4s, v11.4s, v0.s[1]\n"
- "fmla v23.4s, v11.4s, v1.s[1]\n"
- "fmla v27.4s, v11.4s, v2.s[1]\n"
- "ldr q11, [%[b_ptr0], #0x70]\n"
- "fmla v16.4s, v8.4s, v0.s[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- "fmla v20.4s, v8.4s, v1.s[2]\n"
- "fmla v24.4s, v8.4s, v2.s[2]\n"
- "ldr q8, [%[b_ptr0], #-0x80]\n"
- "fmla v17.4s, v9.4s, v0.s[2]\n"
- "fmla v21.4s, v9.4s, v1.s[2]\n"
- "fmla v25.4s, v9.4s, v2.s[2]\n"
- "ldr q9, [%[b_ptr0], #-0x70]\n"
- "fmla v18.4s, v10.4s, v0.s[2]\n"
- "fmla v22.4s, v10.4s, v1.s[2]\n"
- "fmla v26.4s, v10.4s, v2.s[2]\n"
- "ldr q10, [%[b_ptr0], #-0x60]\n"
- "fmla v19.4s, v11.4s, v0.s[2]\n"
- "fmla v23.4s, v11.4s, v1.s[2]\n"
- "fmla v27.4s, v11.4s, v2.s[2]\n"
- "ldr q11, [%[b_ptr0], #-0x50]\n"
- "fmla v16.4s, v8.4s, v0.s[3]\n"
- "fmla v20.4s, v8.4s, v1.s[3]\n"
- "fmla v24.4s, v8.4s, v2.s[3]\n"
- "ldr q8, [%[b_ptr0], #-0x40]\n"
- "fmla v17.4s, v9.4s, v0.s[3]\n"
- "fmla v21.4s, v9.4s, v1.s[3]\n"
- "fmla v25.4s, v9.4s, v2.s[3]\n"
- "ldr q9, [%[b_ptr0], #-0x30]\n"
- "fmla v18.4s, v10.4s, v0.s[3]\n"
- "fmla v22.4s, v10.4s, v1.s[3]\n"
- "fmla v26.4s, v10.4s, v2.s[3]\n"
- "ldr q10, [%[b_ptr0], #-0x20]\n"
- "fmla v19.4s, v11.4s, v0.s[3]\n"
- "fmla v23.4s, v11.4s, v1.s[3]\n"
- "fmla v27.4s, v11.4s, v2.s[3]\n"
- "ldr q11, [%[b_ptr0], #-0x10]\n"
- "fmla v16.4s, v8.4s, v4.s[0]\n"
- "fmla v20.4s, v8.4s, v5.s[0]\n"
- "fmla v24.4s, v8.4s, v6.s[0]\n"
- "ldr q8, [%[b_ptr0]]\n"
- "fmla v17.4s, v9.4s, v4.s[0]\n"
- "fmla v21.4s, v9.4s, v5.s[0]\n"
- "fmla v25.4s, v9.4s, v6.s[0]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "fmla v18.4s, v10.4s, v4.s[0]\n"
- "fmla v22.4s, v10.4s, v5.s[0]\n"
- "fmla v26.4s, v10.4s, v6.s[0]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "fmla v19.4s, v11.4s, v4.s[0]\n"
- "fmla v23.4s, v11.4s, v5.s[0]\n"
- "fmla v27.4s, v11.4s, v6.s[0]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "fmla v16.4s, v8.4s, v4.s[1]\n"
- "fmla v20.4s, v8.4s, v5.s[1]\n"
- "fmla v24.4s, v8.4s, v6.s[1]\n"
- "ldr q8, [%[b_ptr0], #0x40]\n"
- "fmla v17.4s, v9.4s, v4.s[1]\n"
- "fmla v21.4s, v9.4s, v5.s[1]\n"
- "fmla v25.4s, v9.4s, v6.s[1]\n"
- "ldr q9, [%[b_ptr0], #0x50]\n"
- "fmla v18.4s, v10.4s, v4.s[1]\n"
- "fmla v22.4s, v10.4s, v5.s[1]\n"
- "fmla v26.4s, v10.4s, v6.s[1]\n"
- "ldr q10, [%[b_ptr0], #0x60]\n"
- "fmla v19.4s, v11.4s, v4.s[1]\n"
- "fmla v23.4s, v11.4s, v5.s[1]\n"
- "fmla v27.4s, v11.4s, v6.s[1]\n"
- "ldr q11, [%[b_ptr0], #0x70]\n"
- "fmla v16.4s, v8.4s, v4.s[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- "fmla v20.4s, v8.4s, v5.s[2]\n"
- "fmla v24.4s, v8.4s, v6.s[2]\n"
- "ldr q8, [%[b_ptr0], #-0x80]\n"
- "fmla v17.4s, v9.4s, v4.s[2]\n"
- "fmla v21.4s, v9.4s, v5.s[2]\n"
- "fmla v25.4s, v9.4s, v6.s[2]\n"
- "ldr q9, [%[b_ptr0], #-0x70]\n"
- "fmla v18.4s, v10.4s, v4.s[2]\n"
- "fmla v22.4s, v10.4s, v5.s[2]\n"
- "fmla v26.4s, v10.4s, v6.s[2]\n"
- "ldr q10, [%[b_ptr0], #-0x60]\n"
- "fmla v19.4s, v11.4s, v4.s[2]\n"
- "fmla v23.4s, v11.4s, v5.s[2]\n"
- "fmla v27.4s, v11.4s, v6.s[2]\n"
- "ldr q11, [%[b_ptr0], #-0x50]\n"
- "fmla v16.4s, v8.4s, v4.s[3]\n"
- "add %[b_ptr0], %[b_ptr0], #-0x40\n"
- "fmla v20.4s, v8.4s, v5.s[3]\n"
- "fmla v24.4s, v8.4s, v6.s[3]\n"
- "fmla v17.4s, v9.4s, v4.s[3]\n"
- "fmla v21.4s, v9.4s, v5.s[3]\n"
- "fmla v25.4s, v9.4s, v6.s[3]\n"
- "fmla v18.4s, v10.4s, v4.s[3]\n"
- "fmla v22.4s, v10.4s, v5.s[3]\n"
- "fmla v26.4s, v10.4s, v6.s[3]\n"
- "fmla v19.4s, v11.4s, v4.s[3]\n"
- "fmla v23.4s, v11.4s, v5.s[3]\n"
- "fmla v27.4s, v11.4s, v6.s[3]\n"
- "b 5f\n"
- "4:\n"
- "fmla v16.4s, v8.4s, v0.s[0]\n"
- "fmla v20.4s, v8.4s, v1.s[0]\n"
- "fmla v24.4s, v8.4s, v2.s[0]\n"
- "ldr q8, [%[b_ptr0]]\n"
- "fmla v17.4s, v9.4s, v0.s[0]\n"
- "fmla v21.4s, v9.4s, v1.s[0]\n"
- "fmla v25.4s, v9.4s, v2.s[0]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "fmla v18.4s, v10.4s, v0.s[0]\n"
- "fmla v22.4s, v10.4s, v1.s[0]\n"
- "fmla v26.4s, v10.4s, v2.s[0]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "fmla v19.4s, v11.4s, v0.s[0]\n"
- "fmla v23.4s, v11.4s, v1.s[0]\n"
- "fmla v27.4s, v11.4s, v2.s[0]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "fmla v16.4s, v8.4s, v0.s[1]\n"
- "fmla v20.4s, v8.4s, v1.s[1]\n"
- "fmla v24.4s, v8.4s, v2.s[1]\n"
- "ldr q8, [%[b_ptr0], #0x40]\n"
- "fmla v17.4s, v9.4s, v0.s[1]\n"
- "fmla v21.4s, v9.4s, v1.s[1]\n"
- "fmla v25.4s, v9.4s, v2.s[1]\n"
- "ldr q9, [%[b_ptr0], #0x50]\n"
- "fmla v18.4s, v10.4s, v0.s[1]\n"
- "fmla v22.4s, v10.4s, v1.s[1]\n"
- "fmla v26.4s, v10.4s, v2.s[1]\n"
- "ldr q10, [%[b_ptr0], #0x60]\n"
- "fmla v19.4s, v11.4s, v0.s[1]\n"
- "fmla v23.4s, v11.4s, v1.s[1]\n"
- "fmla v27.4s, v11.4s, v2.s[1]\n"
- "ldr q11, [%[b_ptr0], #0x70]\n"
- "fmla v16.4s, v8.4s, v0.s[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- "fmla v20.4s, v8.4s, v1.s[2]\n"
- "fmla v24.4s, v8.4s, v2.s[2]\n"
- "ldr q8, [%[b_ptr0], #-0x80]\n"
- "fmla v17.4s, v9.4s, v0.s[2]\n"
- "fmla v21.4s, v9.4s, v1.s[2]\n"
- "fmla v25.4s, v9.4s, v2.s[2]\n"
- "ldr q9, [%[b_ptr0], #-0x70]\n"
- "fmla v18.4s, v10.4s, v0.s[2]\n"
- "fmla v22.4s, v10.4s, v1.s[2]\n"
- "fmla v26.4s, v10.4s, v2.s[2]\n"
- "ldr q10, [%[b_ptr0], #-0x60]\n"
- "fmla v19.4s, v11.4s, v0.s[2]\n"
- "fmla v23.4s, v11.4s, v1.s[2]\n"
- "fmla v27.4s, v11.4s, v2.s[2]\n"
- "ldr q11, [%[b_ptr0], #-0x50]\n"
- "fmla v16.4s, v8.4s, v0.s[3]\n"
- "add %[b_ptr0], %[b_ptr0], #-0x40\n"
- "fmla v20.4s, v8.4s, v1.s[3]\n"
- "fmla v24.4s, v8.4s, v2.s[3]\n"
- "fmla v17.4s, v9.4s, v0.s[3]\n"
- "fmla v21.4s, v9.4s, v1.s[3]\n"
- "fmla v25.4s, v9.4s, v2.s[3]\n"
- "fmla v18.4s, v10.4s, v0.s[3]\n"
- "fmla v22.4s, v10.4s, v1.s[3]\n"
- "fmla v26.4s, v10.4s, v2.s[3]\n"
- "fmla v19.4s, v11.4s, v0.s[3]\n"
- "fmla v23.4s, v11.4s, v1.s[3]\n"
- "fmla v27.4s, v11.4s, v2.s[3]\n"
- "5:\n"
- "cbz %[blocks], 6f\n"
- "7:\n"
- "ldr q8, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "ldr s0, [%[a_ptr0]]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "add %[a_ptr0], %[a_ptr0], #0x4\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "add %[b_ptr0], %[b_ptr0], #0x40\n"
- "fmla v16.4s, v8.4s, v0.s[0]\n"
- "ldr s1, [a_ptr1]\n"
- "fmla v17.4s, v9.4s, v0.s[0]\n"
- "add a_ptr1, a_ptr1, #0x4\n"
- "fmla v18.4s, v10.4s, v0.s[0]\n"
- "ldr s2, [a_ptr2]\n"
- "fmla v20.4s, v8.4s, v1.s[0]\n"
- "add a_ptr2, a_ptr2, #0x4\n"
- "fmla v21.4s, v9.4s, v1.s[0]\n"
- "fmla v24.4s, v8.4s, v2.s[0]\n"
- "fmla v25.4s, v9.4s, v2.s[0]\n"
- "fmla v22.4s, v10.4s, v1.s[0]\n"
- "fmla v26.4s, v10.4s, v2.s[0]\n"
- "fmla v19.4s, v11.4s, v0.s[0]\n"
- "fmla v23.4s, v11.4s, v1.s[0]\n"
- "fmla v27.4s, v11.4s, v2.s[0]\n"
- "b.ne 7b\n"
- "6:\n"
- "ld1r {v14.4s}, [%[minptr]]\n"
- "ld1r {v15.4s}, [%[maxptr]]\n"
- "fmax v16.4s, v16.4s, v14.4s\n"
- "fmax v17.4s, v17.4s, v14.4s\n"
- "fmax v18.4s, v18.4s, v14.4s\n"
- "fmax v19.4s, v19.4s, v14.4s\n"
- "fmin v16.4s, v16.4s, v15.4s\n"
- "fmin v17.4s, v17.4s, v15.4s\n"
- "fmin v18.4s, v18.4s, v15.4s\n"
- "fmin v19.4s, v19.4s, v15.4s\n"
- "str q16, [%[c_ptr0]]\n"
- "fmax v20.4s, v20.4s, v14.4s\n"
- "fmax v21.4s, v21.4s, v14.4s\n"
- "fmax v22.4s, v22.4s, v14.4s\n"
- "str q17, [%[c_ptr0], #0x10]\n"
- "fmax v23.4s, v23.4s, v14.4s\n"
- "fmin v20.4s, v20.4s, v15.4s\n"
- "fmin v21.4s, v21.4s, v15.4s\n"
- "str q18, [%[c_ptr0], #0x20]\n"
- "fmin v22.4s, v22.4s, v15.4s\n"
- "fmin v23.4s, v23.4s, v15.4s\n"
- "fmax v24.4s, v24.4s, v14.4s\n"
- "str q19, [%[c_ptr0], #0x30]\n"
- "fmax v25.4s, v25.4s, v14.4s\n"
- "add %[c_ptr0], %[c_ptr0], #0x40\n"
- "fmax v26.4s, v26.4s, v14.4s\n"
- "str q20, [c_ptr1]\n"
- "fmin v24.4s, v24.4s, v15.4s\n"
- "fmin v25.4s, v25.4s, v15.4s\n"
- "fmax v27.4s, v27.4s, v14.4s\n"
- "str q21, [c_ptr1, #0x10]\n"
- "fmin v26.4s, v26.4s, v15.4s\n"
- "fmin v27.4s, v27.4s, v15.4s\n"
- "str q22, [c_ptr1, #0x20]\n"
- "str q23, [c_ptr1, #0x30]\n"
- "str q24, [c_ptr2]\n"
- "str q25, [c_ptr2, #0x10]\n"
- "str q26, [c_ptr2, #0x20]\n"
- "str q27, [c_ptr2, #0x30]\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks)
- : [width] "r" (width), [accumulate] "r" (static_cast<uint64_t>(accumulate)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr)
- : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "cc", "memory"
- );
- break;
- default:
- case 4:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "a_ptr3 .req X2\n"
- "c_ptr1 .req X3\n"
- "c_ptr2 .req X4\n"
- "c_ptr3 .req X5\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "add a_ptr3, a_ptr2, %[lda]\n"
- "add c_ptr3, c_ptr2, %[ldc]\n"
- "cbnz %[accumulate], 1f\n"
- "ldr q16, [%[biasptr]]\n"
- "ldr q17, [%[biasptr], #0x10]\n"
- "ldr q18, [%[biasptr], #0x20]\n"
- "ldr q19, [%[biasptr], #0x30]\n"
- "mov v20.16b, v16.16b\n"
- "ldr q0, [%[a_ptr0]]\n"
- "mov v21.16b, v17.16b\n"
- "ldr q1, [a_ptr1]\n"
- "mov v22.16b, v18.16b\n"
- "ldr q2, [a_ptr2]\n"
- "mov v23.16b, v19.16b\n"
- "ldr q3, [a_ptr3]\n"
- "mov v24.16b, v16.16b\n"
- "ldr q8, [%[b_ptr0]]\n"
- "mov v25.16b, v17.16b\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "mov v26.16b, v18.16b\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "mov v27.16b, v19.16b\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "mov v28.16b, v16.16b\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "mov v29.16b, v17.16b\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "mov v30.16b, v18.16b\n"
- "add a_ptr3, a_ptr3, #0x10\n"
- "mov v31.16b, v19.16b\n"
- "add %[b_ptr0], %[b_ptr0], #0x40\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "ldr q16, [%[c_ptr0]]\n"
- "ldr q17, [%[c_ptr0], #0x10]\n"
- "ldr q18, [%[c_ptr0], #0x20]\n"
- "ldr q19, [%[c_ptr0], #0x30]\n"
- "ldr q20, [c_ptr1]\n"
- "ldr q21, [c_ptr1, #0x10]\n"
- "ldr q22, [c_ptr1, #0x20]\n"
- "ldr q23, [c_ptr1, #0x30]\n"
- "ldr q24, [c_ptr2]\n"
- "ldr q25, [c_ptr2, #0x10]\n"
- "ldr q26, [c_ptr2, #0x20]\n"
- "ldr q27, [c_ptr2, #0x30]\n"
- "ldr q28, [c_ptr3]\n"
- "ldr q29, [c_ptr3, #0x10]\n"
- "ldr q30, [c_ptr3, #0x20]\n"
- "ldr q31, [c_ptr3, #0x30]\n"
- "ldr q0, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ldr q1, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "ldr q2, [a_ptr2]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "ldr q3, [a_ptr3]\n"
- "add a_ptr3, a_ptr3, #0x10\n"
- "ldr q8, [%[b_ptr0]]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "add %[b_ptr0], %[b_ptr0], #0x40\n"
- "cbz %[loops], 2f\n"
- "3:\n"
- "fmla v16.4s, v8.4s, v0.s[0]\n"
- "ldr q11, [%[b_ptr0], #-0x10]\n"
- "fmla v20.4s, v8.4s, v1.s[0]\n"
- "ldr q4, [%[a_ptr0]]\n"
- "fmla v24.4s, v8.4s, v2.s[0]\n"
- "ldr q5, [a_ptr1]\n"
- "fmla v28.4s, v8.4s, v3.s[0]\n"
- "ldr q6, [a_ptr2]\n"
- "fmla v17.4s, v9.4s, v0.s[0]\n"
- "ldr q7, [a_ptr3]\n"
- "fmla v21.4s, v9.4s, v1.s[0]\n"
- "ldr q8, [%[b_ptr0]]\n"
- "fmla v25.4s, v9.4s, v2.s[0]\n"
- "subs %[loops], %[loops], #0x1\n"
- "fmla v29.4s, v9.4s, v3.s[0]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "fmla v18.4s, v10.4s, v0.s[0]\n"
- "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n"
- "fmla v22.4s, v10.4s, v1.s[0]\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
- "fmla v26.4s, v10.4s, v2.s[0]\n"
- "add a_ptr1, a_ptr1, #0x20\n"
- "fmla v30.4s, v10.4s, v3.s[0]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "fmla v19.4s, v11.4s, v0.s[0]\n"
- "add a_ptr2, a_ptr2, #0x20\n"
- "fmla v23.4s, v11.4s, v1.s[0]\n"
- "add a_ptr3, a_ptr3, #0x20\n"
- "fmla v27.4s, v11.4s, v2.s[0]\n"
- "prfm PLDL1KEEP, [a_ptr1, #0x40]\n"
- "fmla v31.4s, v11.4s, v3.s[0]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "fmla v16.4s, v8.4s, v0.s[1]\n"
- "prfm PLDL1KEEP, [a_ptr2, #0x40]\n"
- "fmla v20.4s, v8.4s, v1.s[1]\n"
- "prfm PLDL1KEEP, [a_ptr3, #0x40]\n"
- "fmla v24.4s, v8.4s, v2.s[1]\n"
- "fmla v28.4s, v8.4s, v3.s[1]\n"
- "ldr q8, [%[b_ptr0], #0x40]\n"
- "fmla v17.4s, v9.4s, v0.s[1]\n"
- "fmla v21.4s, v9.4s, v1.s[1]\n"
- "fmla v25.4s, v9.4s, v2.s[1]\n"
- "fmla v29.4s, v9.4s, v3.s[1]\n"
- "ldr q9, [%[b_ptr0], #0x50]\n"
- "fmla v18.4s, v10.4s, v0.s[1]\n"
- "fmla v22.4s, v10.4s, v1.s[1]\n"
- "fmla v26.4s, v10.4s, v2.s[1]\n"
- "fmla v30.4s, v10.4s, v3.s[1]\n"
- "ldr q10, [%[b_ptr0], #0x60]\n"
- "fmla v19.4s, v11.4s, v0.s[1]\n"
- "fmla v23.4s, v11.4s, v1.s[1]\n"
- "fmla v27.4s, v11.4s, v2.s[1]\n"
- "fmla v31.4s, v11.4s, v3.s[1]\n"
- "ldr q11, [%[b_ptr0], #0x70]\n"
- "fmla v16.4s, v8.4s, v0.s[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- "fmla v20.4s, v8.4s, v1.s[2]\n"
- "fmla v24.4s, v8.4s, v2.s[2]\n"
- "fmla v28.4s, v8.4s, v3.s[2]\n"
- "ldr q8, [%[b_ptr0], #-0x80]\n"
- "fmla v17.4s, v9.4s, v0.s[2]\n"
- "fmla v21.4s, v9.4s, v1.s[2]\n"
- "fmla v25.4s, v9.4s, v2.s[2]\n"
- "fmla v29.4s, v9.4s, v3.s[2]\n"
- "ldr q9, [%[b_ptr0], #-0x70]\n"
- "fmla v18.4s, v10.4s, v0.s[2]\n"
- "fmla v22.4s, v10.4s, v1.s[2]\n"
- "fmla v26.4s, v10.4s, v2.s[2]\n"
- "fmla v30.4s, v10.4s, v3.s[2]\n"
- "ldr q10, [%[b_ptr0], #-0x60]\n"
- "fmla v19.4s, v11.4s, v0.s[2]\n"
- "fmla v23.4s, v11.4s, v1.s[2]\n"
- "fmla v27.4s, v11.4s, v2.s[2]\n"
- "fmla v31.4s, v11.4s, v3.s[2]\n"
- "ldr q11, [%[b_ptr0], #-0x50]\n"
- "fmla v16.4s, v8.4s, v0.s[3]\n"
- "fmla v20.4s, v8.4s, v1.s[3]\n"
- "fmla v24.4s, v8.4s, v2.s[3]\n"
- "fmla v28.4s, v8.4s, v3.s[3]\n"
- "ldr q8, [%[b_ptr0], #-0x40]\n"
- "fmla v17.4s, v9.4s, v0.s[3]\n"
- "fmla v21.4s, v9.4s, v1.s[3]\n"
- "fmla v25.4s, v9.4s, v2.s[3]\n"
- "fmla v29.4s, v9.4s, v3.s[3]\n"
- "ldr q9, [%[b_ptr0], #-0x30]\n"
- "fmla v18.4s, v10.4s, v0.s[3]\n"
- "fmla v22.4s, v10.4s, v1.s[3]\n"
- "fmla v26.4s, v10.4s, v2.s[3]\n"
- "fmla v30.4s, v10.4s, v3.s[3]\n"
- "ldr q10, [%[b_ptr0], #-0x20]\n"
- "fmla v19.4s, v11.4s, v0.s[3]\n"
- "ldr q0, [%[a_ptr0], #-0x10]\n"
- "fmla v23.4s, v11.4s, v1.s[3]\n"
- "ldr q1, [a_ptr1, #-0x10]\n"
- "fmla v27.4s, v11.4s, v2.s[3]\n"
- "ldr q2, [a_ptr2, #-0x10]\n"
- "fmla v31.4s, v11.4s, v3.s[3]\n"
- "ldr q11, [%[b_ptr0], #-0x10]\n"
- "fmla v16.4s, v8.4s, v4.s[0]\n"
- "ldr q3, [a_ptr3, #-0x10]\n"
- "fmla v20.4s, v8.4s, v5.s[0]\n"
- "fmla v24.4s, v8.4s, v6.s[0]\n"
- "fmla v28.4s, v8.4s, v7.s[0]\n"
- "ldr q8, [%[b_ptr0]]\n"
- "fmla v17.4s, v9.4s, v4.s[0]\n"
- "fmla v21.4s, v9.4s, v5.s[0]\n"
- "fmla v25.4s, v9.4s, v6.s[0]\n"
- "fmla v29.4s, v9.4s, v7.s[0]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "fmla v18.4s, v10.4s, v4.s[0]\n"
- "fmla v22.4s, v10.4s, v5.s[0]\n"
- "fmla v26.4s, v10.4s, v6.s[0]\n"
- "fmla v30.4s, v10.4s, v7.s[0]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "fmla v19.4s, v11.4s, v4.s[0]\n"
- "fmla v23.4s, v11.4s, v5.s[0]\n"
- "fmla v27.4s, v11.4s, v6.s[0]\n"
- "fmla v31.4s, v11.4s, v7.s[0]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "fmla v16.4s, v8.4s, v4.s[1]\n"
- "fmla v20.4s, v8.4s, v5.s[1]\n"
- "fmla v24.4s, v8.4s, v6.s[1]\n"
- "fmla v28.4s, v8.4s, v7.s[1]\n"
- "ldr q8, [%[b_ptr0], #0x40]\n"
- "fmla v17.4s, v9.4s, v4.s[1]\n"
- "fmla v21.4s, v9.4s, v5.s[1]\n"
- "fmla v25.4s, v9.4s, v6.s[1]\n"
- "fmla v29.4s, v9.4s, v7.s[1]\n"
- "ldr q9, [%[b_ptr0], #0x50]\n"
- "fmla v18.4s, v10.4s, v4.s[1]\n"
- "fmla v22.4s, v10.4s, v5.s[1]\n"
- "fmla v26.4s, v10.4s, v6.s[1]\n"
- "fmla v30.4s, v10.4s, v7.s[1]\n"
- "ldr q10, [%[b_ptr0], #0x60]\n"
- "fmla v19.4s, v11.4s, v4.s[1]\n"
- "fmla v23.4s, v11.4s, v5.s[1]\n"
- "fmla v27.4s, v11.4s, v6.s[1]\n"
- "fmla v31.4s, v11.4s, v7.s[1]\n"
- "ldr q11, [%[b_ptr0], #0x70]\n"
- "fmla v16.4s, v8.4s, v4.s[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- "fmla v20.4s, v8.4s, v5.s[2]\n"
- "fmla v24.4s, v8.4s, v6.s[2]\n"
- "fmla v28.4s, v8.4s, v7.s[2]\n"
- "ldr q8, [%[b_ptr0], #-0x80]\n"
- "fmla v17.4s, v9.4s, v4.s[2]\n"
- "fmla v21.4s, v9.4s, v5.s[2]\n"
- "fmla v25.4s, v9.4s, v6.s[2]\n"
- "fmla v29.4s, v9.4s, v7.s[2]\n"
- "ldr q9, [%[b_ptr0], #-0x70]\n"
- "fmla v18.4s, v10.4s, v4.s[2]\n"
- "fmla v22.4s, v10.4s, v5.s[2]\n"
- "fmla v26.4s, v10.4s, v6.s[2]\n"
- "fmla v30.4s, v10.4s, v7.s[2]\n"
- "ldr q10, [%[b_ptr0], #-0x60]\n"
- "fmla v19.4s, v11.4s, v4.s[2]\n"
- "fmla v23.4s, v11.4s, v5.s[2]\n"
- "fmla v27.4s, v11.4s, v6.s[2]\n"
- "fmla v31.4s, v11.4s, v7.s[2]\n"
- "ldr q11, [%[b_ptr0], #-0x50]\n"
- "fmla v16.4s, v8.4s, v4.s[3]\n"
- "fmla v20.4s, v8.4s, v5.s[3]\n"
- "fmla v24.4s, v8.4s, v6.s[3]\n"
- "fmla v28.4s, v8.4s, v7.s[3]\n"
- "ldr q8, [%[b_ptr0], #-0x40]\n"
- "fmla v17.4s, v9.4s, v4.s[3]\n"
- "fmla v21.4s, v9.4s, v5.s[3]\n"
- "fmla v25.4s, v9.4s, v6.s[3]\n"
- "fmla v29.4s, v9.4s, v7.s[3]\n"
- "ldr q9, [%[b_ptr0], #-0x30]\n"
- "fmla v18.4s, v10.4s, v4.s[3]\n"
- "fmla v22.4s, v10.4s, v5.s[3]\n"
- "fmla v26.4s, v10.4s, v6.s[3]\n"
- "fmla v30.4s, v10.4s, v7.s[3]\n"
- "ldr q10, [%[b_ptr0], #-0x20]\n"
- "fmla v19.4s, v11.4s, v4.s[3]\n"
- "fmla v23.4s, v11.4s, v5.s[3]\n"
- "fmla v27.4s, v11.4s, v6.s[3]\n"
- "fmla v31.4s, v11.4s, v7.s[3]\n"
- "b.ne 3b\n"
- "2:\n"
- "ldr q11, [%[b_ptr0], #-0x10]\n"
- "prfm PSTL1KEEP, [%[c_ptr0]]\n"
- "prfm PSTL1KEEP, [c_ptr1]\n"
- "prfm PSTL1KEEP, [c_ptr2]\n"
- "prfm PSTL1KEEP, [c_ptr3]\n"
- "cbz %[regs], 4f\n"
- "fmla v16.4s, v8.4s, v0.s[0]\n"
- "ldr q4, [%[a_ptr0]]\n"
- "fmla v20.4s, v8.4s, v1.s[0]\n"
- "ldr q5, [a_ptr1]\n"
- "fmla v24.4s, v8.4s, v2.s[0]\n"
- "ldr q6, [a_ptr2]\n"
- "fmla v28.4s, v8.4s, v3.s[0]\n"
- "ldr q7, [a_ptr3]\n"
- "fmla v17.4s, v9.4s, v0.s[0]\n"
- "ldr q8, [%[b_ptr0]]\n"
- "fmla v21.4s, v9.4s, v1.s[0]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "fmla v25.4s, v9.4s, v2.s[0]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "fmla v29.4s, v9.4s, v3.s[0]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "fmla v18.4s, v10.4s, v0.s[0]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "fmla v22.4s, v10.4s, v1.s[0]\n"
- "add a_ptr3, a_ptr3, #0x10\n"
- "fmla v26.4s, v10.4s, v2.s[0]\n"
- "fmla v30.4s, v10.4s, v3.s[0]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "fmla v19.4s, v11.4s, v0.s[0]\n"
- "fmla v23.4s, v11.4s, v1.s[0]\n"
- "fmla v27.4s, v11.4s, v2.s[0]\n"
- "fmla v31.4s, v11.4s, v3.s[0]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "fmla v16.4s, v8.4s, v0.s[1]\n"
- "fmla v20.4s, v8.4s, v1.s[1]\n"
- "fmla v24.4s, v8.4s, v2.s[1]\n"
- "fmla v28.4s, v8.4s, v3.s[1]\n"
- "ldr q8, [%[b_ptr0], #0x40]\n"
- "fmla v17.4s, v9.4s, v0.s[1]\n"
- "fmla v21.4s, v9.4s, v1.s[1]\n"
- "fmla v25.4s, v9.4s, v2.s[1]\n"
- "fmla v29.4s, v9.4s, v3.s[1]\n"
- "ldr q9, [%[b_ptr0], #0x50]\n"
- "fmla v18.4s, v10.4s, v0.s[1]\n"
- "fmla v22.4s, v10.4s, v1.s[1]\n"
- "fmla v26.4s, v10.4s, v2.s[1]\n"
- "fmla v30.4s, v10.4s, v3.s[1]\n"
- "ldr q10, [%[b_ptr0], #0x60]\n"
- "fmla v19.4s, v11.4s, v0.s[1]\n"
- "fmla v23.4s, v11.4s, v1.s[1]\n"
- "fmla v27.4s, v11.4s, v2.s[1]\n"
- "fmla v31.4s, v11.4s, v3.s[1]\n"
- "ldr q11, [%[b_ptr0], #0x70]\n"
- "fmla v16.4s, v8.4s, v0.s[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- "fmla v20.4s, v8.4s, v1.s[2]\n"
- "fmla v24.4s, v8.4s, v2.s[2]\n"
- "fmla v28.4s, v8.4s, v3.s[2]\n"
- "ldr q8, [%[b_ptr0], #-0x80]\n"
- "fmla v17.4s, v9.4s, v0.s[2]\n"
- "fmla v21.4s, v9.4s, v1.s[2]\n"
- "fmla v25.4s, v9.4s, v2.s[2]\n"
- "fmla v29.4s, v9.4s, v3.s[2]\n"
- "ldr q9, [%[b_ptr0], #-0x70]\n"
- "fmla v18.4s, v10.4s, v0.s[2]\n"
- "fmla v22.4s, v10.4s, v1.s[2]\n"
- "fmla v26.4s, v10.4s, v2.s[2]\n"
- "fmla v30.4s, v10.4s, v3.s[2]\n"
- "ldr q10, [%[b_ptr0], #-0x60]\n"
- "fmla v19.4s, v11.4s, v0.s[2]\n"
- "fmla v23.4s, v11.4s, v1.s[2]\n"
- "fmla v27.4s, v11.4s, v2.s[2]\n"
- "fmla v31.4s, v11.4s, v3.s[2]\n"
- "ldr q11, [%[b_ptr0], #-0x50]\n"
- "fmla v16.4s, v8.4s, v0.s[3]\n"
- "fmla v20.4s, v8.4s, v1.s[3]\n"
- "fmla v24.4s, v8.4s, v2.s[3]\n"
- "fmla v28.4s, v8.4s, v3.s[3]\n"
- "ldr q8, [%[b_ptr0], #-0x40]\n"
- "fmla v17.4s, v9.4s, v0.s[3]\n"
- "fmla v21.4s, v9.4s, v1.s[3]\n"
- "fmla v25.4s, v9.4s, v2.s[3]\n"
- "fmla v29.4s, v9.4s, v3.s[3]\n"
- "ldr q9, [%[b_ptr0], #-0x30]\n"
- "fmla v18.4s, v10.4s, v0.s[3]\n"
- "fmla v22.4s, v10.4s, v1.s[3]\n"
- "fmla v26.4s, v10.4s, v2.s[3]\n"
- "fmla v30.4s, v10.4s, v3.s[3]\n"
- "ldr q10, [%[b_ptr0], #-0x20]\n"
- "fmla v19.4s, v11.4s, v0.s[3]\n"
- "fmla v23.4s, v11.4s, v1.s[3]\n"
- "fmla v27.4s, v11.4s, v2.s[3]\n"
- "fmla v31.4s, v11.4s, v3.s[3]\n"
- "ldr q11, [%[b_ptr0], #-0x10]\n"
- "fmla v16.4s, v8.4s, v4.s[0]\n"
- "fmla v20.4s, v8.4s, v5.s[0]\n"
- "fmla v24.4s, v8.4s, v6.s[0]\n"
- "fmla v28.4s, v8.4s, v7.s[0]\n"
- "ldr q8, [%[b_ptr0]]\n"
- "fmla v17.4s, v9.4s, v4.s[0]\n"
- "fmla v21.4s, v9.4s, v5.s[0]\n"
- "fmla v25.4s, v9.4s, v6.s[0]\n"
- "fmla v29.4s, v9.4s, v7.s[0]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "fmla v18.4s, v10.4s, v4.s[0]\n"
- "fmla v22.4s, v10.4s, v5.s[0]\n"
- "fmla v26.4s, v10.4s, v6.s[0]\n"
- "fmla v30.4s, v10.4s, v7.s[0]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "fmla v19.4s, v11.4s, v4.s[0]\n"
- "fmla v23.4s, v11.4s, v5.s[0]\n"
- "fmla v27.4s, v11.4s, v6.s[0]\n"
- "fmla v31.4s, v11.4s, v7.s[0]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "fmla v16.4s, v8.4s, v4.s[1]\n"
- "fmla v20.4s, v8.4s, v5.s[1]\n"
- "fmla v24.4s, v8.4s, v6.s[1]\n"
- "fmla v28.4s, v8.4s, v7.s[1]\n"
- "ldr q8, [%[b_ptr0], #0x40]\n"
- "fmla v17.4s, v9.4s, v4.s[1]\n"
- "fmla v21.4s, v9.4s, v5.s[1]\n"
- "fmla v25.4s, v9.4s, v6.s[1]\n"
- "fmla v29.4s, v9.4s, v7.s[1]\n"
- "ldr q9, [%[b_ptr0], #0x50]\n"
- "fmla v18.4s, v10.4s, v4.s[1]\n"
- "fmla v22.4s, v10.4s, v5.s[1]\n"
- "fmla v26.4s, v10.4s, v6.s[1]\n"
- "fmla v30.4s, v10.4s, v7.s[1]\n"
- "ldr q10, [%[b_ptr0], #0x60]\n"
- "fmla v19.4s, v11.4s, v4.s[1]\n"
- "fmla v23.4s, v11.4s, v5.s[1]\n"
- "fmla v27.4s, v11.4s, v6.s[1]\n"
- "fmla v31.4s, v11.4s, v7.s[1]\n"
- "ldr q11, [%[b_ptr0], #0x70]\n"
- "fmla v16.4s, v8.4s, v4.s[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- "fmla v20.4s, v8.4s, v5.s[2]\n"
- "fmla v24.4s, v8.4s, v6.s[2]\n"
- "fmla v28.4s, v8.4s, v7.s[2]\n"
- "ldr q8, [%[b_ptr0], #-0x80]\n"
- "fmla v17.4s, v9.4s, v4.s[2]\n"
- "fmla v21.4s, v9.4s, v5.s[2]\n"
- "fmla v25.4s, v9.4s, v6.s[2]\n"
- "fmla v29.4s, v9.4s, v7.s[2]\n"
- "ldr q9, [%[b_ptr0], #-0x70]\n"
- "fmla v18.4s, v10.4s, v4.s[2]\n"
- "fmla v22.4s, v10.4s, v5.s[2]\n"
- "fmla v26.4s, v10.4s, v6.s[2]\n"
- "fmla v30.4s, v10.4s, v7.s[2]\n"
- "ldr q10, [%[b_ptr0], #-0x60]\n"
- "fmla v19.4s, v11.4s, v4.s[2]\n"
- "fmla v23.4s, v11.4s, v5.s[2]\n"
- "fmla v27.4s, v11.4s, v6.s[2]\n"
- "fmla v31.4s, v11.4s, v7.s[2]\n"
- "ldr q11, [%[b_ptr0], #-0x50]\n"
- "fmla v16.4s, v8.4s, v4.s[3]\n"
- "add %[b_ptr0], %[b_ptr0], #-0x40\n"
- "fmla v20.4s, v8.4s, v5.s[3]\n"
- "fmla v24.4s, v8.4s, v6.s[3]\n"
- "fmla v28.4s, v8.4s, v7.s[3]\n"
- "fmla v17.4s, v9.4s, v4.s[3]\n"
- "fmla v21.4s, v9.4s, v5.s[3]\n"
- "fmla v25.4s, v9.4s, v6.s[3]\n"
- "fmla v29.4s, v9.4s, v7.s[3]\n"
- "fmla v18.4s, v10.4s, v4.s[3]\n"
- "fmla v22.4s, v10.4s, v5.s[3]\n"
- "fmla v26.4s, v10.4s, v6.s[3]\n"
- "fmla v30.4s, v10.4s, v7.s[3]\n"
- "fmla v19.4s, v11.4s, v4.s[3]\n"
- "fmla v23.4s, v11.4s, v5.s[3]\n"
- "fmla v27.4s, v11.4s, v6.s[3]\n"
- "fmla v31.4s, v11.4s, v7.s[3]\n"
- "b 5f\n"
- "4:\n"
- "fmla v16.4s, v8.4s, v0.s[0]\n"
- "fmla v20.4s, v8.4s, v1.s[0]\n"
- "fmla v24.4s, v8.4s, v2.s[0]\n"
- "fmla v28.4s, v8.4s, v3.s[0]\n"
- "ldr q8, [%[b_ptr0]]\n"
- "fmla v17.4s, v9.4s, v0.s[0]\n"
- "fmla v21.4s, v9.4s, v1.s[0]\n"
- "fmla v25.4s, v9.4s, v2.s[0]\n"
- "fmla v29.4s, v9.4s, v3.s[0]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "fmla v18.4s, v10.4s, v0.s[0]\n"
- "fmla v22.4s, v10.4s, v1.s[0]\n"
- "fmla v26.4s, v10.4s, v2.s[0]\n"
- "fmla v30.4s, v10.4s, v3.s[0]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "fmla v19.4s, v11.4s, v0.s[0]\n"
- "fmla v23.4s, v11.4s, v1.s[0]\n"
- "fmla v27.4s, v11.4s, v2.s[0]\n"
- "fmla v31.4s, v11.4s, v3.s[0]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "fmla v16.4s, v8.4s, v0.s[1]\n"
- "fmla v20.4s, v8.4s, v1.s[1]\n"
- "fmla v24.4s, v8.4s, v2.s[1]\n"
- "fmla v28.4s, v8.4s, v3.s[1]\n"
- "ldr q8, [%[b_ptr0], #0x40]\n"
- "fmla v17.4s, v9.4s, v0.s[1]\n"
- "fmla v21.4s, v9.4s, v1.s[1]\n"
- "fmla v25.4s, v9.4s, v2.s[1]\n"
- "fmla v29.4s, v9.4s, v3.s[1]\n"
- "ldr q9, [%[b_ptr0], #0x50]\n"
- "fmla v18.4s, v10.4s, v0.s[1]\n"
- "fmla v22.4s, v10.4s, v1.s[1]\n"
- "fmla v26.4s, v10.4s, v2.s[1]\n"
- "fmla v30.4s, v10.4s, v3.s[1]\n"
- "ldr q10, [%[b_ptr0], #0x60]\n"
- "fmla v19.4s, v11.4s, v0.s[1]\n"
- "fmla v23.4s, v11.4s, v1.s[1]\n"
- "fmla v27.4s, v11.4s, v2.s[1]\n"
- "fmla v31.4s, v11.4s, v3.s[1]\n"
- "ldr q11, [%[b_ptr0], #0x70]\n"
- "fmla v16.4s, v8.4s, v0.s[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- "fmla v20.4s, v8.4s, v1.s[2]\n"
- "fmla v24.4s, v8.4s, v2.s[2]\n"
- "fmla v28.4s, v8.4s, v3.s[2]\n"
- "ldr q8, [%[b_ptr0], #-0x80]\n"
- "fmla v17.4s, v9.4s, v0.s[2]\n"
- "fmla v21.4s, v9.4s, v1.s[2]\n"
- "fmla v25.4s, v9.4s, v2.s[2]\n"
- "fmla v29.4s, v9.4s, v3.s[2]\n"
- "ldr q9, [%[b_ptr0], #-0x70]\n"
- "fmla v18.4s, v10.4s, v0.s[2]\n"
- "fmla v22.4s, v10.4s, v1.s[2]\n"
- "fmla v26.4s, v10.4s, v2.s[2]\n"
- "fmla v30.4s, v10.4s, v3.s[2]\n"
- "ldr q10, [%[b_ptr0], #-0x60]\n"
- "fmla v19.4s, v11.4s, v0.s[2]\n"
- "fmla v23.4s, v11.4s, v1.s[2]\n"
- "fmla v27.4s, v11.4s, v2.s[2]\n"
- "fmla v31.4s, v11.4s, v3.s[2]\n"
- "ldr q11, [%[b_ptr0], #-0x50]\n"
- "fmla v16.4s, v8.4s, v0.s[3]\n"
- "add %[b_ptr0], %[b_ptr0], #-0x40\n"
- "fmla v20.4s, v8.4s, v1.s[3]\n"
- "fmla v24.4s, v8.4s, v2.s[3]\n"
- "fmla v28.4s, v8.4s, v3.s[3]\n"
- "fmla v17.4s, v9.4s, v0.s[3]\n"
- "fmla v21.4s, v9.4s, v1.s[3]\n"
- "fmla v25.4s, v9.4s, v2.s[3]\n"
- "fmla v29.4s, v9.4s, v3.s[3]\n"
- "fmla v18.4s, v10.4s, v0.s[3]\n"
- "fmla v22.4s, v10.4s, v1.s[3]\n"
- "fmla v26.4s, v10.4s, v2.s[3]\n"
- "fmla v30.4s, v10.4s, v3.s[3]\n"
- "fmla v19.4s, v11.4s, v0.s[3]\n"
- "fmla v23.4s, v11.4s, v1.s[3]\n"
- "fmla v27.4s, v11.4s, v2.s[3]\n"
- "fmla v31.4s, v11.4s, v3.s[3]\n"
- "5:\n"
- "cbz %[blocks], 6f\n"
- "7:\n"
- "ldr q8, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "ldr s0, [%[a_ptr0]]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "add %[a_ptr0], %[a_ptr0], #0x4\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "add %[b_ptr0], %[b_ptr0], #0x40\n"
- "fmla v16.4s, v8.4s, v0.s[0]\n"
- "ldr s1, [a_ptr1]\n"
- "fmla v17.4s, v9.4s, v0.s[0]\n"
- "add a_ptr1, a_ptr1, #0x4\n"
- "fmla v18.4s, v10.4s, v0.s[0]\n"
- "ldr s2, [a_ptr2]\n"
- "fmla v20.4s, v8.4s, v1.s[0]\n"
- "add a_ptr2, a_ptr2, #0x4\n"
- "fmla v21.4s, v9.4s, v1.s[0]\n"
- "ldr s3, [a_ptr3]\n"
- "fmla v24.4s, v8.4s, v2.s[0]\n"
- "add a_ptr3, a_ptr3, #0x4\n"
- "fmla v25.4s, v9.4s, v2.s[0]\n"
- "fmla v28.4s, v8.4s, v3.s[0]\n"
- "fmla v29.4s, v9.4s, v3.s[0]\n"
- "fmla v22.4s, v10.4s, v1.s[0]\n"
- "fmla v26.4s, v10.4s, v2.s[0]\n"
- "fmla v30.4s, v10.4s, v3.s[0]\n"
- "fmla v19.4s, v11.4s, v0.s[0]\n"
- "fmla v23.4s, v11.4s, v1.s[0]\n"
- "fmla v27.4s, v11.4s, v2.s[0]\n"
- "fmla v31.4s, v11.4s, v3.s[0]\n"
- "b.ne 7b\n"
- "6:\n"
- "ld1r {v14.4s}, [%[minptr]]\n"
- "ld1r {v15.4s}, [%[maxptr]]\n"
- "fmax v16.4s, v16.4s, v14.4s\n"
- "fmax v17.4s, v17.4s, v14.4s\n"
- "fmax v18.4s, v18.4s, v14.4s\n"
- "fmax v19.4s, v19.4s, v14.4s\n"
- "fmin v16.4s, v16.4s, v15.4s\n"
- "fmin v17.4s, v17.4s, v15.4s\n"
- "fmin v18.4s, v18.4s, v15.4s\n"
- "fmin v19.4s, v19.4s, v15.4s\n"
- "str q16, [%[c_ptr0]]\n"
- "fmax v20.4s, v20.4s, v14.4s\n"
- "fmax v21.4s, v21.4s, v14.4s\n"
- "fmax v22.4s, v22.4s, v14.4s\n"
- "str q17, [%[c_ptr0], #0x10]\n"
- "fmax v23.4s, v23.4s, v14.4s\n"
- "fmin v20.4s, v20.4s, v15.4s\n"
- "fmin v21.4s, v21.4s, v15.4s\n"
- "str q18, [%[c_ptr0], #0x20]\n"
- "fmin v22.4s, v22.4s, v15.4s\n"
- "fmin v23.4s, v23.4s, v15.4s\n"
- "fmax v24.4s, v24.4s, v14.4s\n"
- "str q19, [%[c_ptr0], #0x30]\n"
- "fmax v25.4s, v25.4s, v14.4s\n"
- "add %[c_ptr0], %[c_ptr0], #0x40\n"
- "fmax v26.4s, v26.4s, v14.4s\n"
- "str q20, [c_ptr1]\n"
- "fmin v24.4s, v24.4s, v15.4s\n"
- "fmin v25.4s, v25.4s, v15.4s\n"
- "fmax v27.4s, v27.4s, v14.4s\n"
- "str q21, [c_ptr1, #0x10]\n"
- "fmin v26.4s, v26.4s, v15.4s\n"
- "fmax v28.4s, v28.4s, v14.4s\n"
- "fmax v29.4s, v29.4s, v14.4s\n"
- "str q22, [c_ptr1, #0x20]\n"
- "fmin v27.4s, v27.4s, v15.4s\n"
- "fmax v30.4s, v30.4s, v14.4s\n"
- "fmin v28.4s, v28.4s, v15.4s\n"
- "str q23, [c_ptr1, #0x30]\n"
- "fmin v29.4s, v29.4s, v15.4s\n"
- "fmax v31.4s, v31.4s, v14.4s\n"
- "fmin v30.4s, v30.4s, v15.4s\n"
- "str q24, [c_ptr2]\n"
- "fmin v31.4s, v31.4s, v15.4s\n"
- "str q25, [c_ptr2, #0x10]\n"
- "str q26, [c_ptr2, #0x20]\n"
- "str q27, [c_ptr2, #0x30]\n"
- "str q28, [c_ptr3]\n"
- "str q29, [c_ptr3, #0x10]\n"
- "str q30, [c_ptr3, #0x20]\n"
- "str q31, [c_ptr3, #0x30]\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq a_ptr3\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- ".unreq c_ptr3\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks)
- : [width] "r" (width), [accumulate] "r" (static_cast<uint64_t>(accumulate)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr)
- : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "x4", "x5", "cc", "memory"
- );
- break;
- }
- if (use_result_buffer) {
- for(int cy=0; cy<std::min(M-y, 4); cy++) {
- for(unsigned int cx=0; cx<width; cx++) {
- c_ptr_real[cy * ldc + cx] = result_buffer[cy * 16 + cx];
- }
- }
- }
- }
- }
-}
-
-} // namespace arm_gemm
-
-#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_4x8/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_4x8/generic.cpp
deleted file mode 100644
index 7442d25..0000000
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_4x8/generic.cpp
+++ /dev/null
@@ -1,1934 +0,0 @@
-/*
- * Copyright (c) 2018-2020 Arm Limited.
- *
- * SPDX-License-Identifier: MIT
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to
- * deal in the Software without restriction, including without limitation the
- * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
- * sell copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-#ifdef __aarch64__
-
-#include <algorithm>
-
-#include "arm_gemm.hpp"
-
-#include "../../asmlib.hpp"
-#include "../../utils.hpp"
-
-namespace arm_gemm {
-
-void a64_hybrid_fp32_mla_4x8(const float *A, int lda, const float *B, float *C, int ldc, int M, int N, int K, const float *bias, Activation act, bool accumulate) {
- const int K_stride = K;
- const long loops_count = ((K + 4) / 8) - 1;
- K -= loops_count * 8;
- const long regs_count = (K / 4) - 1;
- K -= (regs_count + 1) * 4;
- const long blocks_count = K / 1;
- float nullbias[4];
- if (!accumulate && !bias) {
- memset(nullbias, 0, (4 * sizeof(float)));
- }
- float minval = - static_cast<float>(std::numeric_limits<float>::infinity());
- float maxval = static_cast<float>(std::numeric_limits<float>::infinity());
- const float * const minptr = &minval;
- const float * const maxptr = &maxval;
-
- switch(act.type)
- {
- default:
- case Activation::Type::None:
- break;
- case Activation::Type::BoundedReLU:
- maxval = static_cast<float>(act.param1);
- /* fall through */
- case Activation::Type::ReLU:
- minval = 0.0f;
- break;
- }
-
- int rows_to_compute;
-
- for (int y=0; y<M; y+=rows_to_compute) {
- const float * const a_ptr0_base = A + (y * lda);
- const unsigned long ldab = lda * sizeof(float);
-
- float *c_ptr0 = C + (y * ldc);
-
- rows_to_compute = M-y;
- if (rows_to_compute > 8) {
- if (rows_to_compute % 8) {
- rows_to_compute = 8 - 1;
- } else {
- rows_to_compute = 8;
- }
- }
-
- for (int x0=0; x0<N; x0+=4ul) {
- const long width = std::min((unsigned long)N-x0, 4ul);
- long loops = loops_count;
- long regs = regs_count;
- long blocks = blocks_count;
- const float *a_ptr0 = a_ptr0_base;
- const float *b_ptr0 = B + (K_stride * x0);
- const bool use_result_buffer = (width < 4);
- float result_buffer[32];
- const unsigned long ldcb = (use_result_buffer ? 4 : ldc) * sizeof(float);
- float *c_ptr_real = c_ptr0;
- if (use_result_buffer && accumulate) {
- for(int cy=0; cy<std::min(M-y, 8); cy++) {
- for(unsigned int cx=0; cx<width; cx++) {
- result_buffer[cy * 4 + cx] = c_ptr_real[cy * ldc + cx];
- }
- }
- }
- if (use_result_buffer) {
- c_ptr0 = result_buffer;
- }
- const float *biasptr = bias ? bias+x0 : nullbias;
-
- switch(rows_to_compute) {
- case 1:
- __asm __volatile (
- "ldr q24, [%[biasptr]]\n"
- "ldr q0, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ldr q16, [%[b_ptr0]]\n"
- "ldr q17, [%[b_ptr0], #0x10]\n"
- "ldr q18, [%[b_ptr0], #0x20]\n"
- "add %[b_ptr0], %[b_ptr0], #0x40\n"
- "cbz %[loops], 1f\n"
- "2:\n"
- "fmla v24.4s, v16.4s, v0.s[0]\n"
- "ldr q19, [%[b_ptr0], #-0x10]\n"
- "ldr q8, [%[a_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ldr q16, [%[b_ptr0]]\n"
- "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n"
- "fmla v24.4s, v17.4s, v0.s[1]\n"
- "ldr q17, [%[b_ptr0], #0x10]\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
- "fmla v24.4s, v18.4s, v0.s[2]\n"
- "ldr q18, [%[b_ptr0], #0x20]\n"
- "fmla v24.4s, v19.4s, v0.s[3]\n"
- "ldr q19, [%[b_ptr0], #0x30]\n"
- "ldr q0, [%[a_ptr0], #-0x10]\n"
- "fmla v24.4s, v16.4s, v8.s[0]\n"
- "ldr q16, [%[b_ptr0], #0x40]\n"
- "fmla v24.4s, v17.4s, v8.s[1]\n"
- "ldr q17, [%[b_ptr0], #0x50]\n"
- "fmla v24.4s, v18.4s, v8.s[2]\n"
- "ldr q18, [%[b_ptr0], #0x60]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "fmla v24.4s, v19.4s, v8.s[3]\n"
- "b.ne 2b\n"
- "1:\n"
- "ldr q19, [%[b_ptr0], #-0x10]\n"
- "prfm PSTL1KEEP, [%[c_ptr0]]\n"
- "cbz %[regs], 3f\n"
- "fmla v24.4s, v16.4s, v0.s[0]\n"
- "ldr q8, [%[a_ptr0]]\n"
- "ldr q16, [%[b_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "fmla v24.4s, v17.4s, v0.s[1]\n"
- "ldr q17, [%[b_ptr0], #0x10]\n"
- "fmla v24.4s, v18.4s, v0.s[2]\n"
- "ldr q18, [%[b_ptr0], #0x20]\n"
- "fmla v24.4s, v19.4s, v0.s[3]\n"
- "ldr q19, [%[b_ptr0], #0x30]\n"
- "add %[b_ptr0], %[b_ptr0], #0x40\n"
- "fmla v24.4s, v16.4s, v8.s[0]\n"
- "fmla v24.4s, v17.4s, v8.s[1]\n"
- "fmla v24.4s, v18.4s, v8.s[2]\n"
- "fmla v24.4s, v19.4s, v8.s[3]\n"
- "b 4f\n"
- "3:\n"
- "fmla v24.4s, v16.4s, v0.s[0]\n"
- "fmla v24.4s, v17.4s, v0.s[1]\n"
- "fmla v24.4s, v18.4s, v0.s[2]\n"
- "fmla v24.4s, v19.4s, v0.s[3]\n"
- "4:\n"
- "cbz %[blocks], 5f\n"
- "6:\n"
- "ldr q16, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "add %[b_ptr0], %[b_ptr0], #0x10\n"
- "ldr s0, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x4\n"
- "fmla v24.4s, v16.4s, v0.s[0]\n"
- "b.ne 6b\n"
- "5:\n"
- "ld1r {v22.4s}, [%[minptr]]\n"
- "ld1r {v23.4s}, [%[maxptr]]\n"
- "fmax v24.4s, v24.4s, v22.4s\n"
- "fmin v24.4s, v24.4s, v23.4s\n"
- "str q24, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], #0x10\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks)
- : [width] "r" (width), [accumulate] "r" (static_cast<uint64_t>(accumulate)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr)
- : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "cc", "memory"
- );
- break;
- case 2:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "c_ptr1 .req X1\n"
- "ldr q24, [%[biasptr]]\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "ldr q0, [%[a_ptr0]]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "mov v25.16b, v24.16b\n"
- "ldr q1, [a_ptr1]\n"
- "ldr q16, [%[b_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ldr q17, [%[b_ptr0], #0x10]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "ldr q18, [%[b_ptr0], #0x20]\n"
- "add %[b_ptr0], %[b_ptr0], #0x40\n"
- "cbz %[loops], 1f\n"
- "2:\n"
- "fmla v24.4s, v16.4s, v0.s[0]\n"
- "ldr q19, [%[b_ptr0], #-0x10]\n"
- "fmla v25.4s, v16.4s, v1.s[0]\n"
- "ldr q8, [%[a_ptr0]]\n"
- "ldr q9, [a_ptr1]\n"
- "subs %[loops], %[loops], #0x1\n"
- "fmla v24.4s, v17.4s, v0.s[1]\n"
- "ldr q16, [%[b_ptr0]]\n"
- "fmla v25.4s, v17.4s, v1.s[1]\n"
- "ldr q17, [%[b_ptr0], #0x10]\n"
- "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
- "fmla v24.4s, v18.4s, v0.s[2]\n"
- "add a_ptr1, a_ptr1, #0x20\n"
- "fmla v25.4s, v18.4s, v1.s[2]\n"
- "ldr q18, [%[b_ptr0], #0x20]\n"
- "prfm PLDL1KEEP, [a_ptr1, #0x40]\n"
- "fmla v24.4s, v19.4s, v0.s[3]\n"
- "ldr q0, [%[a_ptr0], #-0x10]\n"
- "fmla v25.4s, v19.4s, v1.s[3]\n"
- "ldr q19, [%[b_ptr0], #0x30]\n"
- "ldr q1, [a_ptr1, #-0x10]\n"
- "fmla v24.4s, v16.4s, v8.s[0]\n"
- "fmla v25.4s, v16.4s, v9.s[0]\n"
- "ldr q16, [%[b_ptr0], #0x40]\n"
- "fmla v24.4s, v17.4s, v8.s[1]\n"
- "fmla v25.4s, v17.4s, v9.s[1]\n"
- "ldr q17, [%[b_ptr0], #0x50]\n"
- "fmla v24.4s, v18.4s, v8.s[2]\n"
- "fmla v25.4s, v18.4s, v9.s[2]\n"
- "ldr q18, [%[b_ptr0], #0x60]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "fmla v24.4s, v19.4s, v8.s[3]\n"
- "fmla v25.4s, v19.4s, v9.s[3]\n"
- "b.ne 2b\n"
- "1:\n"
- "ldr q19, [%[b_ptr0], #-0x10]\n"
- "prfm PSTL1KEEP, [%[c_ptr0]]\n"
- "prfm PSTL1KEEP, [c_ptr1]\n"
- "cbz %[regs], 3f\n"
- "fmla v24.4s, v16.4s, v0.s[0]\n"
- "ldr q8, [%[a_ptr0]]\n"
- "fmla v25.4s, v16.4s, v1.s[0]\n"
- "ldr q9, [a_ptr1]\n"
- "ldr q16, [%[b_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "fmla v24.4s, v17.4s, v0.s[1]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "fmla v25.4s, v17.4s, v1.s[1]\n"
- "ldr q17, [%[b_ptr0], #0x10]\n"
- "fmla v24.4s, v18.4s, v0.s[2]\n"
- "fmla v25.4s, v18.4s, v1.s[2]\n"
- "ldr q18, [%[b_ptr0], #0x20]\n"
- "fmla v24.4s, v19.4s, v0.s[3]\n"
- "fmla v25.4s, v19.4s, v1.s[3]\n"
- "ldr q19, [%[b_ptr0], #0x30]\n"
- "add %[b_ptr0], %[b_ptr0], #0x40\n"
- "fmla v24.4s, v16.4s, v8.s[0]\n"
- "fmla v25.4s, v16.4s, v9.s[0]\n"
- "fmla v24.4s, v17.4s, v8.s[1]\n"
- "fmla v25.4s, v17.4s, v9.s[1]\n"
- "fmla v24.4s, v18.4s, v8.s[2]\n"
- "fmla v25.4s, v18.4s, v9.s[2]\n"
- "fmla v24.4s, v19.4s, v8.s[3]\n"
- "fmla v25.4s, v19.4s, v9.s[3]\n"
- "b 4f\n"
- "3:\n"
- "fmla v24.4s, v16.4s, v0.s[0]\n"
- "fmla v25.4s, v16.4s, v1.s[0]\n"
- "fmla v24.4s, v17.4s, v0.s[1]\n"
- "fmla v25.4s, v17.4s, v1.s[1]\n"
- "fmla v24.4s, v18.4s, v0.s[2]\n"
- "fmla v25.4s, v18.4s, v1.s[2]\n"
- "fmla v24.4s, v19.4s, v0.s[3]\n"
- "fmla v25.4s, v19.4s, v1.s[3]\n"
- "4:\n"
- "cbz %[blocks], 5f\n"
- "6:\n"
- "ldr q16, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "add %[b_ptr0], %[b_ptr0], #0x10\n"
- "ldr s0, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x4\n"
- "ldr s1, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, #0x4\n"
- "fmla v24.4s, v16.4s, v0.s[0]\n"
- "fmla v25.4s, v16.4s, v1.s[0]\n"
- "b.ne 6b\n"
- "5:\n"
- "ld1r {v22.4s}, [%[minptr]]\n"
- "ld1r {v23.4s}, [%[maxptr]]\n"
- "fmax v24.4s, v24.4s, v22.4s\n"
- "fmax v25.4s, v25.4s, v22.4s\n"
- "fmin v24.4s, v24.4s, v23.4s\n"
- "fmin v25.4s, v25.4s, v23.4s\n"
- "str q24, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], #0x10\n"
- "str q25, [c_ptr1]\n"
- ".unreq a_ptr1\n"
- ".unreq c_ptr1\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks)
- : [width] "r" (width), [accumulate] "r" (static_cast<uint64_t>(accumulate)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr)
- : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "cc", "memory"
- );
- break;
- case 3:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "c_ptr1 .req X2\n"
- "c_ptr2 .req X3\n"
- "ldr q24, [%[biasptr]]\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "ldr q0, [%[a_ptr0]]\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "mov v25.16b, v24.16b\n"
- "ldr q1, [a_ptr1]\n"
- "mov v26.16b, v24.16b\n"
- "ldr q2, [a_ptr2]\n"
- "ldr q16, [%[b_ptr0]]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "ldr q17, [%[b_ptr0], #0x10]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "ldr q18, [%[b_ptr0], #0x20]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "add %[b_ptr0], %[b_ptr0], #0x40\n"
- "cbz %[loops], 1f\n"
- "2:\n"
- "fmla v24.4s, v16.4s, v0.s[0]\n"
- "ldr q19, [%[b_ptr0], #-0x10]\n"
- "fmla v25.4s, v16.4s, v1.s[0]\n"
- "ldr q8, [%[a_ptr0]]\n"
- "fmla v26.4s, v16.4s, v2.s[0]\n"
- "ldr q9, [a_ptr1]\n"
- "ldr q10, [a_ptr2]\n"
- "subs %[loops], %[loops], #0x1\n"
- "fmla v24.4s, v17.4s, v0.s[1]\n"
- "ldr q16, [%[b_ptr0]]\n"
- "fmla v25.4s, v17.4s, v1.s[1]\n"
- "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n"
- "fmla v26.4s, v17.4s, v2.s[1]\n"
- "ldr q17, [%[b_ptr0], #0x10]\n"
- "fmla v24.4s, v18.4s, v0.s[2]\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
- "fmla v25.4s, v18.4s, v1.s[2]\n"
- "add a_ptr1, a_ptr1, #0x20\n"
- "fmla v26.4s, v18.4s, v2.s[2]\n"
- "ldr q18, [%[b_ptr0], #0x20]\n"
- "fmla v24.4s, v19.4s, v0.s[3]\n"
- "ldr q0, [%[a_ptr0], #-0x10]\n"
- "fmla v25.4s, v19.4s, v1.s[3]\n"
- "ldr q1, [a_ptr1, #-0x10]\n"
- "add a_ptr2, a_ptr2, #0x20\n"
- "fmla v26.4s, v19.4s, v2.s[3]\n"
- "ldr q19, [%[b_ptr0], #0x30]\n"
- "fmla v24.4s, v16.4s, v8.s[0]\n"
- "ldr q2, [a_ptr2, #-0x10]\n"
- "fmla v25.4s, v16.4s, v9.s[0]\n"
- "prfm PLDL1KEEP, [a_ptr1, #0x40]\n"
- "fmla v26.4s, v16.4s, v10.s[0]\n"
- "ldr q16, [%[b_ptr0], #0x40]\n"
- "fmla v24.4s, v17.4s, v8.s[1]\n"
- "prfm PLDL1KEEP, [a_ptr2, #0x40]\n"
- "fmla v25.4s, v17.4s, v9.s[1]\n"
- "fmla v26.4s, v17.4s, v10.s[1]\n"
- "ldr q17, [%[b_ptr0], #0x50]\n"
- "fmla v24.4s, v18.4s, v8.s[2]\n"
- "fmla v25.4s, v18.4s, v9.s[2]\n"
- "fmla v26.4s, v18.4s, v10.s[2]\n"
- "ldr q18, [%[b_ptr0], #0x60]\n"
- "fmla v24.4s, v19.4s, v8.s[3]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "fmla v25.4s, v19.4s, v9.s[3]\n"
- "fmla v26.4s, v19.4s, v10.s[3]\n"
- "b.ne 2b\n"
- "1:\n"
- "ldr q19, [%[b_ptr0], #-0x10]\n"
- "prfm PSTL1KEEP, [%[c_ptr0]]\n"
- "prfm PSTL1KEEP, [c_ptr1]\n"
- "prfm PSTL1KEEP, [c_ptr2]\n"
- "cbz %[regs], 3f\n"
- "fmla v24.4s, v16.4s, v0.s[0]\n"
- "ldr q8, [%[a_ptr0]]\n"
- "fmla v25.4s, v16.4s, v1.s[0]\n"
- "ldr q9, [a_ptr1]\n"
- "fmla v26.4s, v16.4s, v2.s[0]\n"
- "ldr q10, [a_ptr2]\n"
- "ldr q16, [%[b_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "fmla v24.4s, v17.4s, v0.s[1]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "fmla v25.4s, v17.4s, v1.s[1]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "fmla v26.4s, v17.4s, v2.s[1]\n"
- "ldr q17, [%[b_ptr0], #0x10]\n"
- "fmla v24.4s, v18.4s, v0.s[2]\n"
- "fmla v25.4s, v18.4s, v1.s[2]\n"
- "fmla v26.4s, v18.4s, v2.s[2]\n"
- "ldr q18, [%[b_ptr0], #0x20]\n"
- "fmla v24.4s, v19.4s, v0.s[3]\n"
- "fmla v25.4s, v19.4s, v1.s[3]\n"
- "fmla v26.4s, v19.4s, v2.s[3]\n"
- "ldr q19, [%[b_ptr0], #0x30]\n"
- "fmla v24.4s, v16.4s, v8.s[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x40\n"
- "fmla v25.4s, v16.4s, v9.s[0]\n"
- "fmla v26.4s, v16.4s, v10.s[0]\n"
- "fmla v24.4s, v17.4s, v8.s[1]\n"
- "fmla v25.4s, v17.4s, v9.s[1]\n"
- "fmla v26.4s, v17.4s, v10.s[1]\n"
- "fmla v24.4s, v18.4s, v8.s[2]\n"
- "fmla v25.4s, v18.4s, v9.s[2]\n"
- "fmla v26.4s, v18.4s, v10.s[2]\n"
- "fmla v24.4s, v19.4s, v8.s[3]\n"
- "fmla v25.4s, v19.4s, v9.s[3]\n"
- "fmla v26.4s, v19.4s, v10.s[3]\n"
- "b 4f\n"
- "3:\n"
- "fmla v24.4s, v16.4s, v0.s[0]\n"
- "fmla v25.4s, v16.4s, v1.s[0]\n"
- "fmla v26.4s, v16.4s, v2.s[0]\n"
- "fmla v24.4s, v17.4s, v0.s[1]\n"
- "fmla v25.4s, v17.4s, v1.s[1]\n"
- "fmla v26.4s, v17.4s, v2.s[1]\n"
- "fmla v24.4s, v18.4s, v0.s[2]\n"
- "fmla v25.4s, v18.4s, v1.s[2]\n"
- "fmla v26.4s, v18.4s, v2.s[2]\n"
- "fmla v24.4s, v19.4s, v0.s[3]\n"
- "fmla v25.4s, v19.4s, v1.s[3]\n"
- "fmla v26.4s, v19.4s, v2.s[3]\n"
- "4:\n"
- "cbz %[blocks], 5f\n"
- "6:\n"
- "ldr q16, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "add %[b_ptr0], %[b_ptr0], #0x10\n"
- "ldr s0, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x4\n"
- "ldr s1, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, #0x4\n"
- "fmla v24.4s, v16.4s, v0.s[0]\n"
- "ldr s2, [a_ptr2]\n"
- "fmla v25.4s, v16.4s, v1.s[0]\n"
- "add a_ptr2, a_ptr2, #0x4\n"
- "fmla v26.4s, v16.4s, v2.s[0]\n"
- "b.ne 6b\n"
- "5:\n"
- "ld1r {v22.4s}, [%[minptr]]\n"
- "ld1r {v23.4s}, [%[maxptr]]\n"
- "fmax v24.4s, v24.4s, v22.4s\n"
- "fmax v25.4s, v25.4s, v22.4s\n"
- "fmax v26.4s, v26.4s, v22.4s\n"
- "fmin v24.4s, v24.4s, v23.4s\n"
- "fmin v25.4s, v25.4s, v23.4s\n"
- "fmin v26.4s, v26.4s, v23.4s\n"
- "str q24, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], #0x10\n"
- "str q25, [c_ptr1]\n"
- "str q26, [c_ptr2]\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks)
- : [width] "r" (width), [accumulate] "r" (static_cast<uint64_t>(accumulate)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr)
- : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "cc", "memory"
- );
- break;
- case 4:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "a_ptr3 .req X2\n"
- "c_ptr1 .req X3\n"
- "c_ptr2 .req X4\n"
- "c_ptr3 .req X5\n"
- "ldr q24, [%[biasptr]]\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "ldr q0, [%[a_ptr0]]\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "mov v25.16b, v24.16b\n"
- "ldr q1, [a_ptr1]\n"
- "mov v26.16b, v24.16b\n"
- "ldr q2, [a_ptr2]\n"
- "mov v27.16b, v24.16b\n"
- "ldr q16, [%[b_ptr0]]\n"
- "ldr q17, [%[b_ptr0], #0x10]\n"
- "add a_ptr3, a_ptr2, %[lda]\n"
- "ldr q18, [%[b_ptr0], #0x20]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "ldr q3, [a_ptr3]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "add c_ptr3, c_ptr2, %[ldc]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "add a_ptr3, a_ptr3, #0x10\n"
- "add %[b_ptr0], %[b_ptr0], #0x40\n"
- "cbz %[loops], 1f\n"
- "2:\n"
- "fmla v24.4s, v16.4s, v0.s[0]\n"
- "ldr q19, [%[b_ptr0], #-0x10]\n"
- "fmla v25.4s, v16.4s, v1.s[0]\n"
- "ldr q8, [%[a_ptr0]]\n"
- "fmla v26.4s, v16.4s, v2.s[0]\n"
- "ldr q9, [a_ptr1]\n"
- "fmla v27.4s, v16.4s, v3.s[0]\n"
- "ldr q10, [a_ptr2]\n"
- "fmla v24.4s, v17.4s, v0.s[1]\n"
- "ldr q11, [a_ptr3]\n"
- "fmla v25.4s, v17.4s, v1.s[1]\n"
- "ldr q16, [%[b_ptr0]]\n"
- "fmla v26.4s, v17.4s, v2.s[1]\n"
- "subs %[loops], %[loops], #0x1\n"
- "fmla v27.4s, v17.4s, v3.s[1]\n"
- "ldr q17, [%[b_ptr0], #0x10]\n"
- "fmla v24.4s, v18.4s, v0.s[2]\n"
- "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n"
- "fmla v25.4s, v18.4s, v1.s[2]\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
- "fmla v26.4s, v18.4s, v2.s[2]\n"
- "add a_ptr1, a_ptr1, #0x20\n"
- "fmla v27.4s, v18.4s, v3.s[2]\n"
- "ldr q18, [%[b_ptr0], #0x20]\n"
- "fmla v24.4s, v19.4s, v0.s[3]\n"
- "ldr q0, [%[a_ptr0], #-0x10]\n"
- "fmla v25.4s, v19.4s, v1.s[3]\n"
- "ldr q1, [a_ptr1, #-0x10]\n"
- "fmla v26.4s, v19.4s, v2.s[3]\n"
- "add a_ptr2, a_ptr2, #0x20\n"
- "fmla v27.4s, v19.4s, v3.s[3]\n"
- "ldr q19, [%[b_ptr0], #0x30]\n"
- "fmla v24.4s, v16.4s, v8.s[0]\n"
- "ldr q2, [a_ptr2, #-0x10]\n"
- "fmla v25.4s, v16.4s, v9.s[0]\n"
- "add a_ptr3, a_ptr3, #0x20\n"
- "fmla v26.4s, v16.4s, v10.s[0]\n"
- "ldr q3, [a_ptr3, #-0x10]\n"
- "fmla v27.4s, v16.4s, v11.s[0]\n"
- "ldr q16, [%[b_ptr0], #0x40]\n"
- "fmla v24.4s, v17.4s, v8.s[1]\n"
- "prfm PLDL1KEEP, [a_ptr1, #0x40]\n"
- "fmla v25.4s, v17.4s, v9.s[1]\n"
- "prfm PLDL1KEEP, [a_ptr2, #0x40]\n"
- "fmla v26.4s, v17.4s, v10.s[1]\n"
- "prfm PLDL1KEEP, [a_ptr3, #0x40]\n"
- "fmla v27.4s, v17.4s, v11.s[1]\n"
- "ldr q17, [%[b_ptr0], #0x50]\n"
- "fmla v24.4s, v18.4s, v8.s[2]\n"
- "fmla v25.4s, v18.4s, v9.s[2]\n"
- "fmla v26.4s, v18.4s, v10.s[2]\n"
- "fmla v27.4s, v18.4s, v11.s[2]\n"
- "ldr q18, [%[b_ptr0], #0x60]\n"
- "fmla v24.4s, v19.4s, v8.s[3]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "fmla v25.4s, v19.4s, v9.s[3]\n"
- "fmla v26.4s, v19.4s, v10.s[3]\n"
- "fmla v27.4s, v19.4s, v11.s[3]\n"
- "b.ne 2b\n"
- "1:\n"
- "ldr q19, [%[b_ptr0], #-0x10]\n"
- "prfm PSTL1KEEP, [%[c_ptr0]]\n"
- "prfm PSTL1KEEP, [c_ptr1]\n"
- "prfm PSTL1KEEP, [c_ptr2]\n"
- "prfm PSTL1KEEP, [c_ptr3]\n"
- "cbz %[regs], 3f\n"
- "fmla v24.4s, v16.4s, v0.s[0]\n"
- "ldr q8, [%[a_ptr0]]\n"
- "fmla v25.4s, v16.4s, v1.s[0]\n"
- "ldr q9, [a_ptr1]\n"
- "fmla v26.4s, v16.4s, v2.s[0]\n"
- "ldr q10, [a_ptr2]\n"
- "fmla v27.4s, v16.4s, v3.s[0]\n"
- "ldr q11, [a_ptr3]\n"
- "fmla v24.4s, v17.4s, v0.s[1]\n"
- "ldr q16, [%[b_ptr0]]\n"
- "fmla v25.4s, v17.4s, v1.s[1]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "fmla v26.4s, v17.4s, v2.s[1]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "fmla v27.4s, v17.4s, v3.s[1]\n"
- "ldr q17, [%[b_ptr0], #0x10]\n"
- "fmla v24.4s, v18.4s, v0.s[2]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "fmla v25.4s, v18.4s, v1.s[2]\n"
- "add a_ptr3, a_ptr3, #0x10\n"
- "fmla v26.4s, v18.4s, v2.s[2]\n"
- "fmla v27.4s, v18.4s, v3.s[2]\n"
- "ldr q18, [%[b_ptr0], #0x20]\n"
- "fmla v24.4s, v19.4s, v0.s[3]\n"
- "fmla v25.4s, v19.4s, v1.s[3]\n"
- "fmla v26.4s, v19.4s, v2.s[3]\n"
- "fmla v27.4s, v19.4s, v3.s[3]\n"
- "ldr q19, [%[b_ptr0], #0x30]\n"
- "fmla v24.4s, v16.4s, v8.s[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x40\n"
- "fmla v25.4s, v16.4s, v9.s[0]\n"
- "fmla v26.4s, v16.4s, v10.s[0]\n"
- "fmla v27.4s, v16.4s, v11.s[0]\n"
- "fmla v24.4s, v17.4s, v8.s[1]\n"
- "fmla v25.4s, v17.4s, v9.s[1]\n"
- "fmla v26.4s, v17.4s, v10.s[1]\n"
- "fmla v27.4s, v17.4s, v11.s[1]\n"
- "fmla v24.4s, v18.4s, v8.s[2]\n"
- "fmla v25.4s, v18.4s, v9.s[2]\n"
- "fmla v26.4s, v18.4s, v10.s[2]\n"
- "fmla v27.4s, v18.4s, v11.s[2]\n"
- "fmla v24.4s, v19.4s, v8.s[3]\n"
- "fmla v25.4s, v19.4s, v9.s[3]\n"
- "fmla v26.4s, v19.4s, v10.s[3]\n"
- "fmla v27.4s, v19.4s, v11.s[3]\n"
- "b 4f\n"
- "3:\n"
- "fmla v24.4s, v16.4s, v0.s[0]\n"
- "fmla v25.4s, v16.4s, v1.s[0]\n"
- "fmla v26.4s, v16.4s, v2.s[0]\n"
- "fmla v27.4s, v16.4s, v3.s[0]\n"
- "fmla v24.4s, v17.4s, v0.s[1]\n"
- "fmla v25.4s, v17.4s, v1.s[1]\n"
- "fmla v26.4s, v17.4s, v2.s[1]\n"
- "fmla v27.4s, v17.4s, v3.s[1]\n"
- "fmla v24.4s, v18.4s, v0.s[2]\n"
- "fmla v25.4s, v18.4s, v1.s[2]\n"
- "fmla v26.4s, v18.4s, v2.s[2]\n"
- "fmla v27.4s, v18.4s, v3.s[2]\n"
- "fmla v24.4s, v19.4s, v0.s[3]\n"
- "fmla v25.4s, v19.4s, v1.s[3]\n"
- "fmla v26.4s, v19.4s, v2.s[3]\n"
- "fmla v27.4s, v19.4s, v3.s[3]\n"
- "4:\n"
- "cbz %[blocks], 5f\n"
- "6:\n"
- "ldr q16, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "add %[b_ptr0], %[b_ptr0], #0x10\n"
- "ldr s0, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x4\n"
- "ldr s1, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, #0x4\n"
- "fmla v24.4s, v16.4s, v0.s[0]\n"
- "ldr s2, [a_ptr2]\n"
- "fmla v25.4s, v16.4s, v1.s[0]\n"
- "add a_ptr2, a_ptr2, #0x4\n"
- "ldr s3, [a_ptr3]\n"
- "fmla v26.4s, v16.4s, v2.s[0]\n"
- "add a_ptr3, a_ptr3, #0x4\n"
- "fmla v27.4s, v16.4s, v3.s[0]\n"
- "b.ne 6b\n"
- "5:\n"
- "ld1r {v22.4s}, [%[minptr]]\n"
- "ld1r {v23.4s}, [%[maxptr]]\n"
- "fmax v24.4s, v24.4s, v22.4s\n"
- "fmax v25.4s, v25.4s, v22.4s\n"
- "fmax v26.4s, v26.4s, v22.4s\n"
- "fmax v27.4s, v27.4s, v22.4s\n"
- "fmin v24.4s, v24.4s, v23.4s\n"
- "fmin v25.4s, v25.4s, v23.4s\n"
- "fmin v26.4s, v26.4s, v23.4s\n"
- "fmin v27.4s, v27.4s, v23.4s\n"
- "str q24, [%[c_ptr0]]\n"
- "add %[c_ptr0], %[c_ptr0], #0x10\n"
- "str q25, [c_ptr1]\n"
- "str q26, [c_ptr2]\n"
- "str q27, [c_ptr3]\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq a_ptr3\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- ".unreq c_ptr3\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks)
- : [width] "r" (width), [accumulate] "r" (static_cast<uint64_t>(accumulate)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr)
- : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "x4", "x5", "cc", "memory"
- );
- break;
- case 5:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "a_ptr3 .req X2\n"
- "a_ptr4 .req X3\n"
- "c_ptr1 .req X4\n"
- "c_ptr2 .req X5\n"
- "c_ptr3 .req X6\n"
- "c_ptr4 .req X7\n"
- "ldr q24, [%[biasptr]]\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "ldr q0, [%[a_ptr0]]\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "mov v25.16b, v24.16b\n"
- "ldr q1, [a_ptr1]\n"
- "mov v26.16b, v24.16b\n"
- "ldr q2, [a_ptr2]\n"
- "mov v27.16b, v24.16b\n"
- "ldr q16, [%[b_ptr0]]\n"
- "mov v28.16b, v24.16b\n"
- "ldr q17, [%[b_ptr0], #0x10]\n"
- "ldr q18, [%[b_ptr0], #0x20]\n"
- "add a_ptr3, a_ptr2, %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "ldr q3, [a_ptr3]\n"
- "add a_ptr4, a_ptr3, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "ldr q4, [a_ptr4]\n"
- "add c_ptr3, c_ptr2, %[ldc]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "add c_ptr4, c_ptr3, %[ldc]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "add a_ptr3, a_ptr3, #0x10\n"
- "add a_ptr4, a_ptr4, #0x10\n"
- "add %[b_ptr0], %[b_ptr0], #0x40\n"
- "cbz %[loops], 1f\n"
- "2:\n"
- "fmla v24.4s, v16.4s, v0.s[0]\n"
- "ldr q19, [%[b_ptr0], #-0x10]\n"
- "fmla v25.4s, v16.4s, v1.s[0]\n"
- "ldr q8, [%[a_ptr0]]\n"
- "fmla v26.4s, v16.4s, v2.s[0]\n"
- "ldr q9, [a_ptr1]\n"
- "fmla v27.4s, v16.4s, v3.s[0]\n"
- "ldr q10, [a_ptr2]\n"
- "fmla v28.4s, v16.4s, v4.s[0]\n"
- "ldr q11, [a_ptr3]\n"
- "fmla v24.4s, v17.4s, v0.s[1]\n"
- "ldr q12, [a_ptr4]\n"
- "fmla v25.4s, v17.4s, v1.s[1]\n"
- "ldr q16, [%[b_ptr0]]\n"
- "fmla v26.4s, v17.4s, v2.s[1]\n"
- "subs %[loops], %[loops], #0x1\n"
- "fmla v27.4s, v17.4s, v3.s[1]\n"
- "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n"
- "fmla v28.4s, v17.4s, v4.s[1]\n"
- "ldr q17, [%[b_ptr0], #0x10]\n"
- "fmla v24.4s, v18.4s, v0.s[2]\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
- "fmla v25.4s, v18.4s, v1.s[2]\n"
- "add a_ptr1, a_ptr1, #0x20\n"
- "fmla v26.4s, v18.4s, v2.s[2]\n"
- "add a_ptr2, a_ptr2, #0x20\n"
- "fmla v27.4s, v18.4s, v3.s[2]\n"
- "add a_ptr3, a_ptr3, #0x20\n"
- "fmla v28.4s, v18.4s, v4.s[2]\n"
- "ldr q18, [%[b_ptr0], #0x20]\n"
- "fmla v24.4s, v19.4s, v0.s[3]\n"
- "ldr q0, [%[a_ptr0], #-0x10]\n"
- "fmla v25.4s, v19.4s, v1.s[3]\n"
- "ldr q1, [a_ptr1, #-0x10]\n"
- "fmla v26.4s, v19.4s, v2.s[3]\n"
- "ldr q2, [a_ptr2, #-0x10]\n"
- "fmla v27.4s, v19.4s, v3.s[3]\n"
- "ldr q3, [a_ptr3, #-0x10]\n"
- "fmla v28.4s, v19.4s, v4.s[3]\n"
- "ldr q19, [%[b_ptr0], #0x30]\n"
- "fmla v24.4s, v16.4s, v8.s[0]\n"
- "add a_ptr4, a_ptr4, #0x20\n"
- "fmla v25.4s, v16.4s, v9.s[0]\n"
- "ldr q4, [a_ptr4, #-0x10]\n"
- "fmla v26.4s, v16.4s, v10.s[0]\n"
- "prfm PLDL1KEEP, [a_ptr1, #0x40]\n"
- "fmla v27.4s, v16.4s, v11.s[0]\n"
- "prfm PLDL1KEEP, [a_ptr2, #0x40]\n"
- "fmla v28.4s, v16.4s, v12.s[0]\n"
- "ldr q16, [%[b_ptr0], #0x40]\n"
- "fmla v24.4s, v17.4s, v8.s[1]\n"
- "prfm PLDL1KEEP, [a_ptr3, #0x40]\n"
- "fmla v25.4s, v17.4s, v9.s[1]\n"
- "fmla v26.4s, v17.4s, v10.s[1]\n"
- "fmla v27.4s, v17.4s, v11.s[1]\n"
- "fmla v28.4s, v17.4s, v12.s[1]\n"
- "ldr q17, [%[b_ptr0], #0x50]\n"
- "fmla v24.4s, v18.4s, v8.s[2]\n"
- "fmla v25.4s, v18.4s, v9.s[2]\n"
- "fmla v26.4s, v18.4s, v10.s[2]\n"
- "fmla v27.4s, v18.4s, v11.s[2]\n"
- "fmla v28.4s, v18.4s, v12.s[2]\n"
- "ldr q18, [%[b_ptr0], #0x60]\n"
- "fmla v24.4s, v19.4s, v8.s[3]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "fmla v25.4s, v19.4s, v9.s[3]\n"
- "fmla v26.4s, v19.4s, v10.s[3]\n"
- "fmla v27.4s, v19.4s, v11.s[3]\n"
- "fmla v28.4s, v19.4s, v12.s[3]\n"
- "b.ne 2b\n"
- "1:\n"
- "ldr q19, [%[b_ptr0], #-0x10]\n"
- "prfm PSTL1KEEP, [%[c_ptr0]]\n"
- "prfm PSTL1KEEP, [c_ptr1]\n"
- "prfm PSTL1KEEP, [c_ptr2]\n"
- "prfm PSTL1KEEP, [c_ptr3]\n"
- "prfm PSTL1KEEP, [c_ptr4]\n"
- "cbz %[regs], 3f\n"
- "fmla v24.4s, v16.4s, v0.s[0]\n"
- "ldr q8, [%[a_ptr0]]\n"
- "fmla v25.4s, v16.4s, v1.s[0]\n"
- "ldr q9, [a_ptr1]\n"
- "fmla v26.4s, v16.4s, v2.s[0]\n"
- "ldr q10, [a_ptr2]\n"
- "fmla v27.4s, v16.4s, v3.s[0]\n"
- "ldr q11, [a_ptr3]\n"
- "fmla v28.4s, v16.4s, v4.s[0]\n"
- "ldr q12, [a_ptr4]\n"
- "fmla v24.4s, v17.4s, v0.s[1]\n"
- "ldr q16, [%[b_ptr0]]\n"
- "fmla v25.4s, v17.4s, v1.s[1]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "fmla v26.4s, v17.4s, v2.s[1]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "fmla v27.4s, v17.4s, v3.s[1]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "fmla v28.4s, v17.4s, v4.s[1]\n"
- "ldr q17, [%[b_ptr0], #0x10]\n"
- "fmla v24.4s, v18.4s, v0.s[2]\n"
- "add a_ptr3, a_ptr3, #0x10\n"
- "fmla v25.4s, v18.4s, v1.s[2]\n"
- "add a_ptr4, a_ptr4, #0x10\n"
- "fmla v26.4s, v18.4s, v2.s[2]\n"
- "fmla v27.4s, v18.4s, v3.s[2]\n"
- "fmla v28.4s, v18.4s, v4.s[2]\n"
- "ldr q18, [%[b_ptr0], #0x20]\n"
- "fmla v24.4s, v19.4s, v0.s[3]\n"
- "fmla v25.4s, v19.4s, v1.s[3]\n"
- "fmla v26.4s, v19.4s, v2.s[3]\n"
- "fmla v27.4s, v19.4s, v3.s[3]\n"
- "fmla v28.4s, v19.4s, v4.s[3]\n"
- "ldr q19, [%[b_ptr0], #0x30]\n"
- "fmla v24.4s, v16.4s, v8.s[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x40\n"
- "fmla v25.4s, v16.4s, v9.s[0]\n"
- "fmla v26.4s, v16.4s, v10.s[0]\n"
- "fmla v27.4s, v16.4s, v11.s[0]\n"
- "fmla v28.4s, v16.4s, v12.s[0]\n"
- "fmla v24.4s, v17.4s, v8.s[1]\n"
- "fmla v25.4s, v17.4s, v9.s[1]\n"
- "fmla v26.4s, v17.4s, v10.s[1]\n"
- "fmla v27.4s, v17.4s, v11.s[1]\n"
- "fmla v28.4s, v17.4s, v12.s[1]\n"
- "fmla v24.4s, v18.4s, v8.s[2]\n"
- "fmla v25.4s, v18.4s, v9.s[2]\n"
- "fmla v26.4s, v18.4s, v10.s[2]\n"
- "fmla v27.4s, v18.4s, v11.s[2]\n"
- "fmla v28.4s, v18.4s, v12.s[2]\n"
- "fmla v24.4s, v19.4s, v8.s[3]\n"
- "fmla v25.4s, v19.4s, v9.s[3]\n"
- "fmla v26.4s, v19.4s, v10.s[3]\n"
- "fmla v27.4s, v19.4s, v11.s[3]\n"
- "fmla v28.4s, v19.4s, v12.s[3]\n"
- "b 4f\n"
- "3:\n"
- "fmla v24.4s, v16.4s, v0.s[0]\n"
- "fmla v25.4s, v16.4s, v1.s[0]\n"
- "fmla v26.4s, v16.4s, v2.s[0]\n"
- "fmla v27.4s, v16.4s, v3.s[0]\n"
- "fmla v28.4s, v16.4s, v4.s[0]\n"
- "fmla v24.4s, v17.4s, v0.s[1]\n"
- "fmla v25.4s, v17.4s, v1.s[1]\n"
- "fmla v26.4s, v17.4s, v2.s[1]\n"
- "fmla v27.4s, v17.4s, v3.s[1]\n"
- "fmla v28.4s, v17.4s, v4.s[1]\n"
- "fmla v24.4s, v18.4s, v0.s[2]\n"
- "fmla v25.4s, v18.4s, v1.s[2]\n"
- "fmla v26.4s, v18.4s, v2.s[2]\n"
- "fmla v27.4s, v18.4s, v3.s[2]\n"
- "fmla v28.4s, v18.4s, v4.s[2]\n"
- "fmla v24.4s, v19.4s, v0.s[3]\n"
- "fmla v25.4s, v19.4s, v1.s[3]\n"
- "fmla v26.4s, v19.4s, v2.s[3]\n"
- "fmla v27.4s, v19.4s, v3.s[3]\n"
- "fmla v28.4s, v19.4s, v4.s[3]\n"
- "4:\n"
- "cbz %[blocks], 5f\n"
- "6:\n"
- "ldr q16, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "add %[b_ptr0], %[b_ptr0], #0x10\n"
- "ldr s0, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x4\n"
- "ldr s1, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, #0x4\n"
- "fmla v24.4s, v16.4s, v0.s[0]\n"
- "ldr s2, [a_ptr2]\n"
- "fmla v25.4s, v16.4s, v1.s[0]\n"
- "add a_ptr2, a_ptr2, #0x4\n"
- "ldr s3, [a_ptr3]\n"
- "fmla v26.4s, v16.4s, v2.s[0]\n"
- "add a_ptr3, a_ptr3, #0x4\n"
- "ldr s4, [a_ptr4]\n"
- "fmla v27.4s, v16.4s, v3.s[0]\n"
- "add a_ptr4, a_ptr4, #0x4\n"
- "fmla v28.4s, v16.4s, v4.s[0]\n"
- "b.ne 6b\n"
- "5:\n"
- "ld1r {v22.4s}, [%[minptr]]\n"
- "ld1r {v23.4s}, [%[maxptr]]\n"
- "fmax v24.4s, v24.4s, v22.4s\n"
- "fmax v25.4s, v25.4s, v22.4s\n"
- "fmax v26.4s, v26.4s, v22.4s\n"
- "fmax v27.4s, v27.4s, v22.4s\n"
- "fmin v24.4s, v24.4s, v23.4s\n"
- "fmin v25.4s, v25.4s, v23.4s\n"
- "fmin v26.4s, v26.4s, v23.4s\n"
- "fmin v27.4s, v27.4s, v23.4s\n"
- "str q24, [%[c_ptr0]]\n"
- "fmax v28.4s, v28.4s, v22.4s\n"
- "add %[c_ptr0], %[c_ptr0], #0x10\n"
- "str q25, [c_ptr1]\n"
- "fmin v28.4s, v28.4s, v23.4s\n"
- "str q26, [c_ptr2]\n"
- "str q27, [c_ptr3]\n"
- "str q28, [c_ptr4]\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq a_ptr3\n"
- ".unreq a_ptr4\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- ".unreq c_ptr3\n"
- ".unreq c_ptr4\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks)
- : [width] "r" (width), [accumulate] "r" (static_cast<uint64_t>(accumulate)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr)
- : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "cc", "memory"
- );
- break;
- case 6:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "a_ptr3 .req X2\n"
- "a_ptr4 .req X3\n"
- "a_ptr5 .req X4\n"
- "c_ptr1 .req X5\n"
- "c_ptr2 .req X6\n"
- "c_ptr3 .req X7\n"
- "c_ptr4 .req X8\n"
- "c_ptr5 .req X9\n"
- "ldr q24, [%[biasptr]]\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "ldr q0, [%[a_ptr0]]\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "mov v25.16b, v24.16b\n"
- "ldr q1, [a_ptr1]\n"
- "mov v26.16b, v24.16b\n"
- "ldr q2, [a_ptr2]\n"
- "mov v27.16b, v24.16b\n"
- "ldr q16, [%[b_ptr0]]\n"
- "mov v28.16b, v24.16b\n"
- "ldr q17, [%[b_ptr0], #0x10]\n"
- "mov v29.16b, v24.16b\n"
- "ldr q18, [%[b_ptr0], #0x20]\n"
- "add a_ptr3, a_ptr2, %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "ldr q3, [a_ptr3]\n"
- "add a_ptr4, a_ptr3, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "ldr q4, [a_ptr4]\n"
- "add a_ptr5, a_ptr4, %[lda]\n"
- "add c_ptr3, c_ptr2, %[ldc]\n"
- "ldr q5, [a_ptr5]\n"
- "add c_ptr4, c_ptr3, %[ldc]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "add c_ptr5, c_ptr4, %[ldc]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "add a_ptr3, a_ptr3, #0x10\n"
- "add a_ptr4, a_ptr4, #0x10\n"
- "add a_ptr5, a_ptr5, #0x10\n"
- "add %[b_ptr0], %[b_ptr0], #0x40\n"
- "cbz %[loops], 1f\n"
- "2:\n"
- "fmla v24.4s, v16.4s, v0.s[0]\n"
- "ldr q19, [%[b_ptr0], #-0x10]\n"
- "fmla v25.4s, v16.4s, v1.s[0]\n"
- "ldr q8, [%[a_ptr0]]\n"
- "fmla v26.4s, v16.4s, v2.s[0]\n"
- "ldr q9, [a_ptr1]\n"
- "fmla v27.4s, v16.4s, v3.s[0]\n"
- "ldr q10, [a_ptr2]\n"
- "fmla v28.4s, v16.4s, v4.s[0]\n"
- "ldr q11, [a_ptr3]\n"
- "fmla v29.4s, v16.4s, v5.s[0]\n"
- "ldr q12, [a_ptr4]\n"
- "fmla v24.4s, v17.4s, v0.s[1]\n"
- "ldr q13, [a_ptr5]\n"
- "fmla v25.4s, v17.4s, v1.s[1]\n"
- "ldr q16, [%[b_ptr0]]\n"
- "fmla v26.4s, v17.4s, v2.s[1]\n"
- "subs %[loops], %[loops], #0x1\n"
- "fmla v27.4s, v17.4s, v3.s[1]\n"
- "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n"
- "fmla v28.4s, v17.4s, v4.s[1]\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
- "fmla v29.4s, v17.4s, v5.s[1]\n"
- "ldr q17, [%[b_ptr0], #0x10]\n"
- "fmla v24.4s, v18.4s, v0.s[2]\n"
- "add a_ptr1, a_ptr1, #0x20\n"
- "fmla v25.4s, v18.4s, v1.s[2]\n"
- "add a_ptr2, a_ptr2, #0x20\n"
- "fmla v26.4s, v18.4s, v2.s[2]\n"
- "add a_ptr3, a_ptr3, #0x20\n"
- "fmla v27.4s, v18.4s, v3.s[2]\n"
- "add a_ptr4, a_ptr4, #0x20\n"
- "fmla v28.4s, v18.4s, v4.s[2]\n"
- "add a_ptr5, a_ptr5, #0x20\n"
- "fmla v29.4s, v18.4s, v5.s[2]\n"
- "ldr q18, [%[b_ptr0], #0x20]\n"
- "fmla v24.4s, v19.4s, v0.s[3]\n"
- "ldr q0, [%[a_ptr0], #-0x10]\n"
- "fmla v25.4s, v19.4s, v1.s[3]\n"
- "ldr q1, [a_ptr1, #-0x10]\n"
- "fmla v26.4s, v19.4s, v2.s[3]\n"
- "ldr q2, [a_ptr2, #-0x10]\n"
- "fmla v27.4s, v19.4s, v3.s[3]\n"
- "ldr q3, [a_ptr3, #-0x10]\n"
- "fmla v28.4s, v19.4s, v4.s[3]\n"
- "ldr q4, [a_ptr4, #-0x10]\n"
- "fmla v29.4s, v19.4s, v5.s[3]\n"
- "ldr q19, [%[b_ptr0], #0x30]\n"
- "fmla v24.4s, v16.4s, v8.s[0]\n"
- "ldr q5, [a_ptr5, #-0x10]\n"
- "fmla v25.4s, v16.4s, v9.s[0]\n"
- "prfm PLDL1KEEP, [a_ptr1, #0x40]\n"
- "fmla v26.4s, v16.4s, v10.s[0]\n"
- "prfm PLDL1KEEP, [a_ptr2, #0x40]\n"
- "fmla v27.4s, v16.4s, v11.s[0]\n"
- "prfm PLDL1KEEP, [a_ptr3, #0x40]\n"
- "fmla v28.4s, v16.4s, v12.s[0]\n"
- "fmla v29.4s, v16.4s, v13.s[0]\n"
- "ldr q16, [%[b_ptr0], #0x40]\n"
- "fmla v24.4s, v17.4s, v8.s[1]\n"
- "fmla v25.4s, v17.4s, v9.s[1]\n"
- "fmla v26.4s, v17.4s, v10.s[1]\n"
- "fmla v27.4s, v17.4s, v11.s[1]\n"
- "fmla v28.4s, v17.4s, v12.s[1]\n"
- "fmla v29.4s, v17.4s, v13.s[1]\n"
- "ldr q17, [%[b_ptr0], #0x50]\n"
- "fmla v24.4s, v18.4s, v8.s[2]\n"
- "fmla v25.4s, v18.4s, v9.s[2]\n"
- "fmla v26.4s, v18.4s, v10.s[2]\n"
- "fmla v27.4s, v18.4s, v11.s[2]\n"
- "fmla v28.4s, v18.4s, v12.s[2]\n"
- "fmla v29.4s, v18.4s, v13.s[2]\n"
- "ldr q18, [%[b_ptr0], #0x60]\n"
- "fmla v24.4s, v19.4s, v8.s[3]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "fmla v25.4s, v19.4s, v9.s[3]\n"
- "fmla v26.4s, v19.4s, v10.s[3]\n"
- "fmla v27.4s, v19.4s, v11.s[3]\n"
- "fmla v28.4s, v19.4s, v12.s[3]\n"
- "fmla v29.4s, v19.4s, v13.s[3]\n"
- "b.ne 2b\n"
- "1:\n"
- "ldr q19, [%[b_ptr0], #-0x10]\n"
- "prfm PSTL1KEEP, [%[c_ptr0]]\n"
- "prfm PSTL1KEEP, [c_ptr1]\n"
- "prfm PSTL1KEEP, [c_ptr2]\n"
- "prfm PSTL1KEEP, [c_ptr3]\n"
- "prfm PSTL1KEEP, [c_ptr4]\n"
- "prfm PSTL1KEEP, [c_ptr5]\n"
- "cbz %[regs], 3f\n"
- "fmla v24.4s, v16.4s, v0.s[0]\n"
- "ldr q8, [%[a_ptr0]]\n"
- "fmla v25.4s, v16.4s, v1.s[0]\n"
- "ldr q9, [a_ptr1]\n"
- "fmla v26.4s, v16.4s, v2.s[0]\n"
- "ldr q10, [a_ptr2]\n"
- "fmla v27.4s, v16.4s, v3.s[0]\n"
- "ldr q11, [a_ptr3]\n"
- "fmla v28.4s, v16.4s, v4.s[0]\n"
- "ldr q12, [a_ptr4]\n"
- "fmla v29.4s, v16.4s, v5.s[0]\n"
- "ldr q13, [a_ptr5]\n"
- "fmla v24.4s, v17.4s, v0.s[1]\n"
- "ldr q16, [%[b_ptr0]]\n"
- "fmla v25.4s, v17.4s, v1.s[1]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "fmla v26.4s, v17.4s, v2.s[1]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "fmla v27.4s, v17.4s, v3.s[1]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "fmla v28.4s, v17.4s, v4.s[1]\n"
- "add a_ptr3, a_ptr3, #0x10\n"
- "fmla v29.4s, v17.4s, v5.s[1]\n"
- "ldr q17, [%[b_ptr0], #0x10]\n"
- "fmla v24.4s, v18.4s, v0.s[2]\n"
- "add a_ptr4, a_ptr4, #0x10\n"
- "fmla v25.4s, v18.4s, v1.s[2]\n"
- "add a_ptr5, a_ptr5, #0x10\n"
- "fmla v26.4s, v18.4s, v2.s[2]\n"
- "fmla v27.4s, v18.4s, v3.s[2]\n"
- "fmla v28.4s, v18.4s, v4.s[2]\n"
- "fmla v29.4s, v18.4s, v5.s[2]\n"
- "ldr q18, [%[b_ptr0], #0x20]\n"
- "fmla v24.4s, v19.4s, v0.s[3]\n"
- "fmla v25.4s, v19.4s, v1.s[3]\n"
- "fmla v26.4s, v19.4s, v2.s[3]\n"
- "fmla v27.4s, v19.4s, v3.s[3]\n"
- "fmla v28.4s, v19.4s, v4.s[3]\n"
- "fmla v29.4s, v19.4s, v5.s[3]\n"
- "ldr q19, [%[b_ptr0], #0x30]\n"
- "fmla v24.4s, v16.4s, v8.s[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x40\n"
- "fmla v25.4s, v16.4s, v9.s[0]\n"
- "fmla v26.4s, v16.4s, v10.s[0]\n"
- "fmla v27.4s, v16.4s, v11.s[0]\n"
- "fmla v28.4s, v16.4s, v12.s[0]\n"
- "fmla v29.4s, v16.4s, v13.s[0]\n"
- "fmla v24.4s, v17.4s, v8.s[1]\n"
- "fmla v25.4s, v17.4s, v9.s[1]\n"
- "fmla v26.4s, v17.4s, v10.s[1]\n"
- "fmla v27.4s, v17.4s, v11.s[1]\n"
- "fmla v28.4s, v17.4s, v12.s[1]\n"
- "fmla v29.4s, v17.4s, v13.s[1]\n"
- "fmla v24.4s, v18.4s, v8.s[2]\n"
- "fmla v25.4s, v18.4s, v9.s[2]\n"
- "fmla v26.4s, v18.4s, v10.s[2]\n"
- "fmla v27.4s, v18.4s, v11.s[2]\n"
- "fmla v28.4s, v18.4s, v12.s[2]\n"
- "fmla v29.4s, v18.4s, v13.s[2]\n"
- "fmla v24.4s, v19.4s, v8.s[3]\n"
- "fmla v25.4s, v19.4s, v9.s[3]\n"
- "fmla v26.4s, v19.4s, v10.s[3]\n"
- "fmla v27.4s, v19.4s, v11.s[3]\n"
- "fmla v28.4s, v19.4s, v12.s[3]\n"
- "fmla v29.4s, v19.4s, v13.s[3]\n"
- "b 4f\n"
- "3:\n"
- "fmla v24.4s, v16.4s, v0.s[0]\n"
- "fmla v25.4s, v16.4s, v1.s[0]\n"
- "fmla v26.4s, v16.4s, v2.s[0]\n"
- "fmla v27.4s, v16.4s, v3.s[0]\n"
- "fmla v28.4s, v16.4s, v4.s[0]\n"
- "fmla v29.4s, v16.4s, v5.s[0]\n"
- "fmla v24.4s, v17.4s, v0.s[1]\n"
- "fmla v25.4s, v17.4s, v1.s[1]\n"
- "fmla v26.4s, v17.4s, v2.s[1]\n"
- "fmla v27.4s, v17.4s, v3.s[1]\n"
- "fmla v28.4s, v17.4s, v4.s[1]\n"
- "fmla v29.4s, v17.4s, v5.s[1]\n"
- "fmla v24.4s, v18.4s, v0.s[2]\n"
- "fmla v25.4s, v18.4s, v1.s[2]\n"
- "fmla v26.4s, v18.4s, v2.s[2]\n"
- "fmla v27.4s, v18.4s, v3.s[2]\n"
- "fmla v28.4s, v18.4s, v4.s[2]\n"
- "fmla v29.4s, v18.4s, v5.s[2]\n"
- "fmla v24.4s, v19.4s, v0.s[3]\n"
- "fmla v25.4s, v19.4s, v1.s[3]\n"
- "fmla v26.4s, v19.4s, v2.s[3]\n"
- "fmla v27.4s, v19.4s, v3.s[3]\n"
- "fmla v28.4s, v19.4s, v4.s[3]\n"
- "fmla v29.4s, v19.4s, v5.s[3]\n"
- "4:\n"
- "cbz %[blocks], 5f\n"
- "6:\n"
- "ldr q16, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "add %[b_ptr0], %[b_ptr0], #0x10\n"
- "ldr s0, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x4\n"
- "ldr s1, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, #0x4\n"
- "fmla v24.4s, v16.4s, v0.s[0]\n"
- "ldr s2, [a_ptr2]\n"
- "fmla v25.4s, v16.4s, v1.s[0]\n"
- "add a_ptr2, a_ptr2, #0x4\n"
- "ldr s3, [a_ptr3]\n"
- "fmla v26.4s, v16.4s, v2.s[0]\n"
- "add a_ptr3, a_ptr3, #0x4\n"
- "ldr s4, [a_ptr4]\n"
- "fmla v27.4s, v16.4s, v3.s[0]\n"
- "add a_ptr4, a_ptr4, #0x4\n"
- "ldr s5, [a_ptr5]\n"
- "fmla v28.4s, v16.4s, v4.s[0]\n"
- "add a_ptr5, a_ptr5, #0x4\n"
- "fmla v29.4s, v16.4s, v5.s[0]\n"
- "b.ne 6b\n"
- "5:\n"
- "ld1r {v22.4s}, [%[minptr]]\n"
- "ld1r {v23.4s}, [%[maxptr]]\n"
- "fmax v24.4s, v24.4s, v22.4s\n"
- "fmax v25.4s, v25.4s, v22.4s\n"
- "fmax v26.4s, v26.4s, v22.4s\n"
- "fmax v27.4s, v27.4s, v22.4s\n"
- "fmin v24.4s, v24.4s, v23.4s\n"
- "fmin v25.4s, v25.4s, v23.4s\n"
- "fmin v26.4s, v26.4s, v23.4s\n"
- "fmin v27.4s, v27.4s, v23.4s\n"
- "str q24, [%[c_ptr0]]\n"
- "fmax v28.4s, v28.4s, v22.4s\n"
- "add %[c_ptr0], %[c_ptr0], #0x10\n"
- "fmax v29.4s, v29.4s, v22.4s\n"
- "str q25, [c_ptr1]\n"
- "fmin v28.4s, v28.4s, v23.4s\n"
- "fmin v29.4s, v29.4s, v23.4s\n"
- "str q26, [c_ptr2]\n"
- "str q27, [c_ptr3]\n"
- "str q28, [c_ptr4]\n"
- "str q29, [c_ptr5]\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq a_ptr3\n"
- ".unreq a_ptr4\n"
- ".unreq a_ptr5\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- ".unreq c_ptr3\n"
- ".unreq c_ptr4\n"
- ".unreq c_ptr5\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks)
- : [width] "r" (width), [accumulate] "r" (static_cast<uint64_t>(accumulate)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr)
- : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "cc", "memory"
- );
- break;
- case 7:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "a_ptr3 .req X2\n"
- "a_ptr4 .req X3\n"
- "a_ptr5 .req X4\n"
- "a_ptr6 .req X5\n"
- "c_ptr1 .req X6\n"
- "c_ptr2 .req X7\n"
- "c_ptr3 .req X8\n"
- "c_ptr4 .req X9\n"
- "c_ptr5 .req X10\n"
- "c_ptr6 .req X11\n"
- "ldr q24, [%[biasptr]]\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "ldr q0, [%[a_ptr0]]\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "mov v25.16b, v24.16b\n"
- "ldr q1, [a_ptr1]\n"
- "mov v26.16b, v24.16b\n"
- "ldr q2, [a_ptr2]\n"
- "mov v27.16b, v24.16b\n"
- "ldr q16, [%[b_ptr0]]\n"
- "mov v28.16b, v24.16b\n"
- "ldr q17, [%[b_ptr0], #0x10]\n"
- "mov v29.16b, v24.16b\n"
- "ldr q18, [%[b_ptr0], #0x20]\n"
- "mov v30.16b, v24.16b\n"
- "add a_ptr3, a_ptr2, %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "ldr q3, [a_ptr3]\n"
- "add a_ptr4, a_ptr3, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "ldr q4, [a_ptr4]\n"
- "add a_ptr5, a_ptr4, %[lda]\n"
- "add c_ptr3, c_ptr2, %[ldc]\n"
- "ldr q5, [a_ptr5]\n"
- "add a_ptr6, a_ptr5, %[lda]\n"
- "add c_ptr4, c_ptr3, %[ldc]\n"
- "ldr q6, [a_ptr6]\n"
- "add c_ptr5, c_ptr4, %[ldc]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "add c_ptr6, c_ptr5, %[ldc]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "add a_ptr3, a_ptr3, #0x10\n"
- "add a_ptr4, a_ptr4, #0x10\n"
- "add a_ptr5, a_ptr5, #0x10\n"
- "add a_ptr6, a_ptr6, #0x10\n"
- "add %[b_ptr0], %[b_ptr0], #0x40\n"
- "cbz %[loops], 1f\n"
- "2:\n"
- "fmla v24.4s, v16.4s, v0.s[0]\n"
- "ldr q19, [%[b_ptr0], #-0x10]\n"
- "fmla v25.4s, v16.4s, v1.s[0]\n"
- "ldr q8, [%[a_ptr0]]\n"
- "fmla v26.4s, v16.4s, v2.s[0]\n"
- "ldr q9, [a_ptr1]\n"
- "fmla v27.4s, v16.4s, v3.s[0]\n"
- "ldr q10, [a_ptr2]\n"
- "fmla v28.4s, v16.4s, v4.s[0]\n"
- "ldr q11, [a_ptr3]\n"
- "fmla v29.4s, v16.4s, v5.s[0]\n"
- "ldr q12, [a_ptr4]\n"
- "fmla v30.4s, v16.4s, v6.s[0]\n"
- "ldr q13, [a_ptr5]\n"
- "fmla v24.4s, v17.4s, v0.s[1]\n"
- "ldr q14, [a_ptr6]\n"
- "fmla v25.4s, v17.4s, v1.s[1]\n"
- "ldr q16, [%[b_ptr0]]\n"
- "fmla v26.4s, v17.4s, v2.s[1]\n"
- "subs %[loops], %[loops], #0x1\n"
- "fmla v27.4s, v17.4s, v3.s[1]\n"
- "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n"
- "fmla v28.4s, v17.4s, v4.s[1]\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
- "fmla v29.4s, v17.4s, v5.s[1]\n"
- "add a_ptr1, a_ptr1, #0x20\n"
- "fmla v30.4s, v17.4s, v6.s[1]\n"
- "ldr q17, [%[b_ptr0], #0x10]\n"
- "fmla v24.4s, v18.4s, v0.s[2]\n"
- "add a_ptr2, a_ptr2, #0x20\n"
- "fmla v25.4s, v18.4s, v1.s[2]\n"
- "add a_ptr3, a_ptr3, #0x20\n"
- "fmla v26.4s, v18.4s, v2.s[2]\n"
- "add a_ptr4, a_ptr4, #0x20\n"
- "fmla v27.4s, v18.4s, v3.s[2]\n"
- "add a_ptr5, a_ptr5, #0x20\n"
- "fmla v28.4s, v18.4s, v4.s[2]\n"
- "add a_ptr6, a_ptr6, #0x20\n"
- "fmla v29.4s, v18.4s, v5.s[2]\n"
- "prfm PLDL1KEEP, [a_ptr1, #0x40]\n"
- "fmla v30.4s, v18.4s, v6.s[2]\n"
- "ldr q18, [%[b_ptr0], #0x20]\n"
- "fmla v24.4s, v19.4s, v0.s[3]\n"
- "ldr q0, [%[a_ptr0], #-0x10]\n"
- "fmla v25.4s, v19.4s, v1.s[3]\n"
- "ldr q1, [a_ptr1, #-0x10]\n"
- "fmla v26.4s, v19.4s, v2.s[3]\n"
- "ldr q2, [a_ptr2, #-0x10]\n"
- "fmla v27.4s, v19.4s, v3.s[3]\n"
- "ldr q3, [a_ptr3, #-0x10]\n"
- "fmla v28.4s, v19.4s, v4.s[3]\n"
- "ldr q4, [a_ptr4, #-0x10]\n"
- "fmla v29.4s, v19.4s, v5.s[3]\n"
- "ldr q5, [a_ptr5, #-0x10]\n"
- "fmla v30.4s, v19.4s, v6.s[3]\n"
- "ldr q19, [%[b_ptr0], #0x30]\n"
- "fmla v24.4s, v16.4s, v8.s[0]\n"
- "ldr q6, [a_ptr6, #-0x10]\n"
- "fmla v25.4s, v16.4s, v9.s[0]\n"
- "prfm PLDL1KEEP, [a_ptr2, #0x40]\n"
- "fmla v26.4s, v16.4s, v10.s[0]\n"
- "prfm PLDL1KEEP, [a_ptr3, #0x40]\n"
- "fmla v27.4s, v16.4s, v11.s[0]\n"
- "fmla v28.4s, v16.4s, v12.s[0]\n"
- "fmla v29.4s, v16.4s, v13.s[0]\n"
- "fmla v30.4s, v16.4s, v14.s[0]\n"
- "ldr q16, [%[b_ptr0], #0x40]\n"
- "fmla v24.4s, v17.4s, v8.s[1]\n"
- "fmla v25.4s, v17.4s, v9.s[1]\n"
- "fmla v26.4s, v17.4s, v10.s[1]\n"
- "fmla v27.4s, v17.4s, v11.s[1]\n"
- "fmla v28.4s, v17.4s, v12.s[1]\n"
- "fmla v29.4s, v17.4s, v13.s[1]\n"
- "fmla v30.4s, v17.4s, v14.s[1]\n"
- "ldr q17, [%[b_ptr0], #0x50]\n"
- "fmla v24.4s, v18.4s, v8.s[2]\n"
- "fmla v25.4s, v18.4s, v9.s[2]\n"
- "fmla v26.4s, v18.4s, v10.s[2]\n"
- "fmla v27.4s, v18.4s, v11.s[2]\n"
- "fmla v28.4s, v18.4s, v12.s[2]\n"
- "fmla v29.4s, v18.4s, v13.s[2]\n"
- "fmla v30.4s, v18.4s, v14.s[2]\n"
- "ldr q18, [%[b_ptr0], #0x60]\n"
- "fmla v24.4s, v19.4s, v8.s[3]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "fmla v25.4s, v19.4s, v9.s[3]\n"
- "fmla v26.4s, v19.4s, v10.s[3]\n"
- "fmla v27.4s, v19.4s, v11.s[3]\n"
- "fmla v28.4s, v19.4s, v12.s[3]\n"
- "fmla v29.4s, v19.4s, v13.s[3]\n"
- "fmla v30.4s, v19.4s, v14.s[3]\n"
- "b.ne 2b\n"
- "1:\n"
- "ldr q19, [%[b_ptr0], #-0x10]\n"
- "prfm PSTL1KEEP, [%[c_ptr0]]\n"
- "prfm PSTL1KEEP, [c_ptr1]\n"
- "prfm PSTL1KEEP, [c_ptr2]\n"
- "prfm PSTL1KEEP, [c_ptr3]\n"
- "prfm PSTL1KEEP, [c_ptr4]\n"
- "prfm PSTL1KEEP, [c_ptr5]\n"
- "prfm PSTL1KEEP, [c_ptr6]\n"
- "cbz %[regs], 3f\n"
- "fmla v24.4s, v16.4s, v0.s[0]\n"
- "ldr q8, [%[a_ptr0]]\n"
- "fmla v25.4s, v16.4s, v1.s[0]\n"
- "ldr q9, [a_ptr1]\n"
- "fmla v26.4s, v16.4s, v2.s[0]\n"
- "ldr q10, [a_ptr2]\n"
- "fmla v27.4s, v16.4s, v3.s[0]\n"
- "ldr q11, [a_ptr3]\n"
- "fmla v28.4s, v16.4s, v4.s[0]\n"
- "ldr q12, [a_ptr4]\n"
- "fmla v29.4s, v16.4s, v5.s[0]\n"
- "ldr q13, [a_ptr5]\n"
- "fmla v30.4s, v16.4s, v6.s[0]\n"
- "ldr q14, [a_ptr6]\n"
- "fmla v24.4s, v17.4s, v0.s[1]\n"
- "ldr q16, [%[b_ptr0]]\n"
- "fmla v25.4s, v17.4s, v1.s[1]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "fmla v26.4s, v17.4s, v2.s[1]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "fmla v27.4s, v17.4s, v3.s[1]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "fmla v28.4s, v17.4s, v4.s[1]\n"
- "add a_ptr3, a_ptr3, #0x10\n"
- "fmla v29.4s, v17.4s, v5.s[1]\n"
- "add a_ptr4, a_ptr4, #0x10\n"
- "fmla v30.4s, v17.4s, v6.s[1]\n"
- "ldr q17, [%[b_ptr0], #0x10]\n"
- "fmla v24.4s, v18.4s, v0.s[2]\n"
- "add a_ptr5, a_ptr5, #0x10\n"
- "fmla v25.4s, v18.4s, v1.s[2]\n"
- "add a_ptr6, a_ptr6, #0x10\n"
- "fmla v26.4s, v18.4s, v2.s[2]\n"
- "fmla v27.4s, v18.4s, v3.s[2]\n"
- "fmla v28.4s, v18.4s, v4.s[2]\n"
- "fmla v29.4s, v18.4s, v5.s[2]\n"
- "fmla v30.4s, v18.4s, v6.s[2]\n"
- "ldr q18, [%[b_ptr0], #0x20]\n"
- "fmla v24.4s, v19.4s, v0.s[3]\n"
- "fmla v25.4s, v19.4s, v1.s[3]\n"
- "fmla v26.4s, v19.4s, v2.s[3]\n"
- "fmla v27.4s, v19.4s, v3.s[3]\n"
- "fmla v28.4s, v19.4s, v4.s[3]\n"
- "fmla v29.4s, v19.4s, v5.s[3]\n"
- "fmla v30.4s, v19.4s, v6.s[3]\n"
- "ldr q19, [%[b_ptr0], #0x30]\n"
- "fmla v24.4s, v16.4s, v8.s[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x40\n"
- "fmla v25.4s, v16.4s, v9.s[0]\n"
- "fmla v26.4s, v16.4s, v10.s[0]\n"
- "fmla v27.4s, v16.4s, v11.s[0]\n"
- "fmla v28.4s, v16.4s, v12.s[0]\n"
- "fmla v29.4s, v16.4s, v13.s[0]\n"
- "fmla v30.4s, v16.4s, v14.s[0]\n"
- "fmla v24.4s, v17.4s, v8.s[1]\n"
- "fmla v25.4s, v17.4s, v9.s[1]\n"
- "fmla v26.4s, v17.4s, v10.s[1]\n"
- "fmla v27.4s, v17.4s, v11.s[1]\n"
- "fmla v28.4s, v17.4s, v12.s[1]\n"
- "fmla v29.4s, v17.4s, v13.s[1]\n"
- "fmla v30.4s, v17.4s, v14.s[1]\n"
- "fmla v24.4s, v18.4s, v8.s[2]\n"
- "fmla v25.4s, v18.4s, v9.s[2]\n"
- "fmla v26.4s, v18.4s, v10.s[2]\n"
- "fmla v27.4s, v18.4s, v11.s[2]\n"
- "fmla v28.4s, v18.4s, v12.s[2]\n"
- "fmla v29.4s, v18.4s, v13.s[2]\n"
- "fmla v30.4s, v18.4s, v14.s[2]\n"
- "fmla v24.4s, v19.4s, v8.s[3]\n"
- "fmla v25.4s, v19.4s, v9.s[3]\n"
- "fmla v26.4s, v19.4s, v10.s[3]\n"
- "fmla v27.4s, v19.4s, v11.s[3]\n"
- "fmla v28.4s, v19.4s, v12.s[3]\n"
- "fmla v29.4s, v19.4s, v13.s[3]\n"
- "fmla v30.4s, v19.4s, v14.s[3]\n"
- "b 4f\n"
- "3:\n"
- "fmla v24.4s, v16.4s, v0.s[0]\n"
- "fmla v25.4s, v16.4s, v1.s[0]\n"
- "fmla v26.4s, v16.4s, v2.s[0]\n"
- "fmla v27.4s, v16.4s, v3.s[0]\n"
- "fmla v28.4s, v16.4s, v4.s[0]\n"
- "fmla v29.4s, v16.4s, v5.s[0]\n"
- "fmla v30.4s, v16.4s, v6.s[0]\n"
- "fmla v24.4s, v17.4s, v0.s[1]\n"
- "fmla v25.4s, v17.4s, v1.s[1]\n"
- "fmla v26.4s, v17.4s, v2.s[1]\n"
- "fmla v27.4s, v17.4s, v3.s[1]\n"
- "fmla v28.4s, v17.4s, v4.s[1]\n"
- "fmla v29.4s, v17.4s, v5.s[1]\n"
- "fmla v30.4s, v17.4s, v6.s[1]\n"
- "fmla v24.4s, v18.4s, v0.s[2]\n"
- "fmla v25.4s, v18.4s, v1.s[2]\n"
- "fmla v26.4s, v18.4s, v2.s[2]\n"
- "fmla v27.4s, v18.4s, v3.s[2]\n"
- "fmla v28.4s, v18.4s, v4.s[2]\n"
- "fmla v29.4s, v18.4s, v5.s[2]\n"
- "fmla v30.4s, v18.4s, v6.s[2]\n"
- "fmla v24.4s, v19.4s, v0.s[3]\n"
- "fmla v25.4s, v19.4s, v1.s[3]\n"
- "fmla v26.4s, v19.4s, v2.s[3]\n"
- "fmla v27.4s, v19.4s, v3.s[3]\n"
- "fmla v28.4s, v19.4s, v4.s[3]\n"
- "fmla v29.4s, v19.4s, v5.s[3]\n"
- "fmla v30.4s, v19.4s, v6.s[3]\n"
- "4:\n"
- "cbz %[blocks], 5f\n"
- "6:\n"
- "ldr q16, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "add %[b_ptr0], %[b_ptr0], #0x10\n"
- "ldr s0, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x4\n"
- "ldr s1, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, #0x4\n"
- "fmla v24.4s, v16.4s, v0.s[0]\n"
- "ldr s2, [a_ptr2]\n"
- "fmla v25.4s, v16.4s, v1.s[0]\n"
- "add a_ptr2, a_ptr2, #0x4\n"
- "ldr s3, [a_ptr3]\n"
- "fmla v26.4s, v16.4s, v2.s[0]\n"
- "add a_ptr3, a_ptr3, #0x4\n"
- "ldr s4, [a_ptr4]\n"
- "fmla v27.4s, v16.4s, v3.s[0]\n"
- "add a_ptr4, a_ptr4, #0x4\n"
- "ldr s5, [a_ptr5]\n"
- "fmla v28.4s, v16.4s, v4.s[0]\n"
- "add a_ptr5, a_ptr5, #0x4\n"
- "ldr s6, [a_ptr6]\n"
- "fmla v29.4s, v16.4s, v5.s[0]\n"
- "add a_ptr6, a_ptr6, #0x4\n"
- "fmla v30.4s, v16.4s, v6.s[0]\n"
- "b.ne 6b\n"
- "5:\n"
- "ld1r {v22.4s}, [%[minptr]]\n"
- "ld1r {v23.4s}, [%[maxptr]]\n"
- "fmax v24.4s, v24.4s, v22.4s\n"
- "fmax v25.4s, v25.4s, v22.4s\n"
- "fmax v26.4s, v26.4s, v22.4s\n"
- "fmax v27.4s, v27.4s, v22.4s\n"
- "fmin v24.4s, v24.4s, v23.4s\n"
- "fmin v25.4s, v25.4s, v23.4s\n"
- "fmin v26.4s, v26.4s, v23.4s\n"
- "fmin v27.4s, v27.4s, v23.4s\n"
- "str q24, [%[c_ptr0]]\n"
- "fmax v28.4s, v28.4s, v22.4s\n"
- "add %[c_ptr0], %[c_ptr0], #0x10\n"
- "fmax v29.4s, v29.4s, v22.4s\n"
- "str q25, [c_ptr1]\n"
- "fmax v30.4s, v30.4s, v22.4s\n"
- "fmin v28.4s, v28.4s, v23.4s\n"
- "fmin v29.4s, v29.4s, v23.4s\n"
- "str q26, [c_ptr2]\n"
- "fmin v30.4s, v30.4s, v23.4s\n"
- "str q27, [c_ptr3]\n"
- "str q28, [c_ptr4]\n"
- "str q29, [c_ptr5]\n"
- "str q30, [c_ptr6]\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq a_ptr3\n"
- ".unreq a_ptr4\n"
- ".unreq a_ptr5\n"
- ".unreq a_ptr6\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- ".unreq c_ptr3\n"
- ".unreq c_ptr4\n"
- ".unreq c_ptr5\n"
- ".unreq c_ptr6\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks)
- : [width] "r" (width), [accumulate] "r" (static_cast<uint64_t>(accumulate)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr)
- : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "cc", "memory"
- );
- break;
- default:
- case 8:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "a_ptr3 .req X2\n"
- "a_ptr4 .req X3\n"
- "a_ptr5 .req X4\n"
- "a_ptr6 .req X5\n"
- "a_ptr7 .req X6\n"
- "c_ptr1 .req X7\n"
- "c_ptr2 .req X8\n"
- "c_ptr3 .req X9\n"
- "c_ptr4 .req X10\n"
- "c_ptr5 .req X11\n"
- "c_ptr6 .req X12\n"
- "c_ptr7 .req X13\n"
- "ldr q24, [%[biasptr]]\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "ldr q0, [%[a_ptr0]]\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "mov v25.16b, v24.16b\n"
- "ldr q1, [a_ptr1]\n"
- "mov v26.16b, v24.16b\n"
- "ldr q2, [a_ptr2]\n"
- "mov v27.16b, v24.16b\n"
- "ldr q16, [%[b_ptr0]]\n"
- "mov v28.16b, v24.16b\n"
- "ldr q17, [%[b_ptr0], #0x10]\n"
- "mov v29.16b, v24.16b\n"
- "ldr q18, [%[b_ptr0], #0x20]\n"
- "mov v30.16b, v24.16b\n"
- "add a_ptr3, a_ptr2, %[lda]\n"
- "mov v31.16b, v24.16b\n"
- "ldr q3, [a_ptr3]\n"
- "add a_ptr4, a_ptr3, %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "ldr q4, [a_ptr4]\n"
- "add a_ptr5, a_ptr4, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "ldr q5, [a_ptr5]\n"
- "add a_ptr6, a_ptr5, %[lda]\n"
- "add c_ptr3, c_ptr2, %[ldc]\n"
- "ldr q6, [a_ptr6]\n"
- "add a_ptr7, a_ptr6, %[lda]\n"
- "add c_ptr4, c_ptr3, %[ldc]\n"
- "ldr q7, [a_ptr7]\n"
- "add c_ptr5, c_ptr4, %[ldc]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "add c_ptr6, c_ptr5, %[ldc]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "add c_ptr7, c_ptr6, %[ldc]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "add a_ptr3, a_ptr3, #0x10\n"
- "add a_ptr4, a_ptr4, #0x10\n"
- "add a_ptr5, a_ptr5, #0x10\n"
- "add a_ptr6, a_ptr6, #0x10\n"
- "add a_ptr7, a_ptr7, #0x10\n"
- "add %[b_ptr0], %[b_ptr0], #0x40\n"
- "cbz %[loops], 1f\n"
- "2:\n"
- "fmla v24.4s, v16.4s, v0.s[0]\n"
- "ldr q19, [%[b_ptr0], #-0x10]\n"
- "fmla v25.4s, v16.4s, v1.s[0]\n"
- "ldr q8, [%[a_ptr0]]\n"
- "fmla v26.4s, v16.4s, v2.s[0]\n"
- "ldr q9, [a_ptr1]\n"
- "fmla v27.4s, v16.4s, v3.s[0]\n"
- "ldr q10, [a_ptr2]\n"
- "fmla v28.4s, v16.4s, v4.s[0]\n"
- "ldr q11, [a_ptr3]\n"
- "fmla v29.4s, v16.4s, v5.s[0]\n"
- "ldr q12, [a_ptr4]\n"
- "fmla v30.4s, v16.4s, v6.s[0]\n"
- "ldr q13, [a_ptr5]\n"
- "fmla v31.4s, v16.4s, v7.s[0]\n"
- "ldr q14, [a_ptr6]\n"
- "fmla v24.4s, v17.4s, v0.s[1]\n"
- "ldr q15, [a_ptr7]\n"
- "fmla v25.4s, v17.4s, v1.s[1]\n"
- "ldr q16, [%[b_ptr0]]\n"
- "fmla v26.4s, v17.4s, v2.s[1]\n"
- "subs %[loops], %[loops], #0x1\n"
- "fmla v27.4s, v17.4s, v3.s[1]\n"
- "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n"
- "fmla v28.4s, v17.4s, v4.s[1]\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
- "fmla v29.4s, v17.4s, v5.s[1]\n"
- "add a_ptr1, a_ptr1, #0x20\n"
- "fmla v30.4s, v17.4s, v6.s[1]\n"
- "add a_ptr2, a_ptr2, #0x20\n"
- "fmla v31.4s, v17.4s, v7.s[1]\n"
- "ldr q17, [%[b_ptr0], #0x10]\n"
- "fmla v24.4s, v18.4s, v0.s[2]\n"
- "add a_ptr3, a_ptr3, #0x20\n"
- "fmla v25.4s, v18.4s, v1.s[2]\n"
- "add a_ptr4, a_ptr4, #0x20\n"
- "fmla v26.4s, v18.4s, v2.s[2]\n"
- "add a_ptr5, a_ptr5, #0x20\n"
- "fmla v27.4s, v18.4s, v3.s[2]\n"
- "add a_ptr6, a_ptr6, #0x20\n"
- "fmla v28.4s, v18.4s, v4.s[2]\n"
- "add a_ptr7, a_ptr7, #0x20\n"
- "fmla v29.4s, v18.4s, v5.s[2]\n"
- "prfm PLDL1KEEP, [a_ptr1, #0x40]\n"
- "fmla v30.4s, v18.4s, v6.s[2]\n"
- "prfm PLDL1KEEP, [a_ptr2, #0x40]\n"
- "fmla v31.4s, v18.4s, v7.s[2]\n"
- "ldr q18, [%[b_ptr0], #0x20]\n"
- "fmla v24.4s, v19.4s, v0.s[3]\n"
- "ldr q0, [%[a_ptr0], #-0x10]\n"
- "fmla v25.4s, v19.4s, v1.s[3]\n"
- "ldr q1, [a_ptr1, #-0x10]\n"
- "fmla v26.4s, v19.4s, v2.s[3]\n"
- "ldr q2, [a_ptr2, #-0x10]\n"
- "fmla v27.4s, v19.4s, v3.s[3]\n"
- "ldr q3, [a_ptr3, #-0x10]\n"
- "fmla v28.4s, v19.4s, v4.s[3]\n"
- "ldr q4, [a_ptr4, #-0x10]\n"
- "fmla v29.4s, v19.4s, v5.s[3]\n"
- "ldr q5, [a_ptr5, #-0x10]\n"
- "fmla v30.4s, v19.4s, v6.s[3]\n"
- "ldr q6, [a_ptr6, #-0x10]\n"
- "fmla v31.4s, v19.4s, v7.s[3]\n"
- "ldr q19, [%[b_ptr0], #0x30]\n"
- "fmla v24.4s, v16.4s, v8.s[0]\n"
- "ldr q7, [a_ptr7, #-0x10]\n"
- "fmla v25.4s, v16.4s, v9.s[0]\n"
- "prfm PLDL1KEEP, [a_ptr3, #0x40]\n"
- "fmla v26.4s, v16.4s, v10.s[0]\n"
- "fmla v27.4s, v16.4s, v11.s[0]\n"
- "fmla v28.4s, v16.4s, v12.s[0]\n"
- "fmla v29.4s, v16.4s, v13.s[0]\n"
- "fmla v30.4s, v16.4s, v14.s[0]\n"
- "fmla v31.4s, v16.4s, v15.s[0]\n"
- "ldr q16, [%[b_ptr0], #0x40]\n"
- "fmla v24.4s, v17.4s, v8.s[1]\n"
- "fmla v25.4s, v17.4s, v9.s[1]\n"
- "fmla v26.4s, v17.4s, v10.s[1]\n"
- "fmla v27.4s, v17.4s, v11.s[1]\n"
- "fmla v28.4s, v17.4s, v12.s[1]\n"
- "fmla v29.4s, v17.4s, v13.s[1]\n"
- "fmla v30.4s, v17.4s, v14.s[1]\n"
- "fmla v31.4s, v17.4s, v15.s[1]\n"
- "ldr q17, [%[b_ptr0], #0x50]\n"
- "fmla v24.4s, v18.4s, v8.s[2]\n"
- "fmla v25.4s, v18.4s, v9.s[2]\n"
- "fmla v26.4s, v18.4s, v10.s[2]\n"
- "fmla v27.4s, v18.4s, v11.s[2]\n"
- "fmla v28.4s, v18.4s, v12.s[2]\n"
- "fmla v29.4s, v18.4s, v13.s[2]\n"
- "fmla v30.4s, v18.4s, v14.s[2]\n"
- "fmla v31.4s, v18.4s, v15.s[2]\n"
- "ldr q18, [%[b_ptr0], #0x60]\n"
- "fmla v24.4s, v19.4s, v8.s[3]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "fmla v25.4s, v19.4s, v9.s[3]\n"
- "fmla v26.4s, v19.4s, v10.s[3]\n"
- "fmla v27.4s, v19.4s, v11.s[3]\n"
- "fmla v28.4s, v19.4s, v12.s[3]\n"
- "fmla v29.4s, v19.4s, v13.s[3]\n"
- "fmla v30.4s, v19.4s, v14.s[3]\n"
- "fmla v31.4s, v19.4s, v15.s[3]\n"
- "b.ne 2b\n"
- "1:\n"
- "ldr q19, [%[b_ptr0], #-0x10]\n"
- "prfm PSTL1KEEP, [%[c_ptr0]]\n"
- "prfm PSTL1KEEP, [c_ptr1]\n"
- "prfm PSTL1KEEP, [c_ptr2]\n"
- "prfm PSTL1KEEP, [c_ptr3]\n"
- "prfm PSTL1KEEP, [c_ptr4]\n"
- "prfm PSTL1KEEP, [c_ptr5]\n"
- "prfm PSTL1KEEP, [c_ptr6]\n"
- "prfm PSTL1KEEP, [c_ptr7]\n"
- "cbz %[regs], 3f\n"
- "fmla v24.4s, v16.4s, v0.s[0]\n"
- "ldr q8, [%[a_ptr0]]\n"
- "fmla v25.4s, v16.4s, v1.s[0]\n"
- "ldr q9, [a_ptr1]\n"
- "fmla v26.4s, v16.4s, v2.s[0]\n"
- "ldr q10, [a_ptr2]\n"
- "fmla v27.4s, v16.4s, v3.s[0]\n"
- "ldr q11, [a_ptr3]\n"
- "fmla v28.4s, v16.4s, v4.s[0]\n"
- "ldr q12, [a_ptr4]\n"
- "fmla v29.4s, v16.4s, v5.s[0]\n"
- "ldr q13, [a_ptr5]\n"
- "fmla v30.4s, v16.4s, v6.s[0]\n"
- "ldr q14, [a_ptr6]\n"
- "fmla v31.4s, v16.4s, v7.s[0]\n"
- "ldr q15, [a_ptr7]\n"
- "fmla v24.4s, v17.4s, v0.s[1]\n"
- "ldr q16, [%[b_ptr0]]\n"
- "fmla v25.4s, v17.4s, v1.s[1]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "fmla v26.4s, v17.4s, v2.s[1]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "fmla v27.4s, v17.4s, v3.s[1]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "fmla v28.4s, v17.4s, v4.s[1]\n"
- "add a_ptr3, a_ptr3, #0x10\n"
- "fmla v29.4s, v17.4s, v5.s[1]\n"
- "add a_ptr4, a_ptr4, #0x10\n"
- "fmla v30.4s, v17.4s, v6.s[1]\n"
- "add a_ptr5, a_ptr5, #0x10\n"
- "fmla v31.4s, v17.4s, v7.s[1]\n"
- "ldr q17, [%[b_ptr0], #0x10]\n"
- "fmla v24.4s, v18.4s, v0.s[2]\n"
- "add a_ptr6, a_ptr6, #0x10\n"
- "fmla v25.4s, v18.4s, v1.s[2]\n"
- "add a_ptr7, a_ptr7, #0x10\n"
- "fmla v26.4s, v18.4s, v2.s[2]\n"
- "fmla v27.4s, v18.4s, v3.s[2]\n"
- "fmla v28.4s, v18.4s, v4.s[2]\n"
- "fmla v29.4s, v18.4s, v5.s[2]\n"
- "fmla v30.4s, v18.4s, v6.s[2]\n"
- "fmla v31.4s, v18.4s, v7.s[2]\n"
- "ldr q18, [%[b_ptr0], #0x20]\n"
- "fmla v24.4s, v19.4s, v0.s[3]\n"
- "fmla v25.4s, v19.4s, v1.s[3]\n"
- "fmla v26.4s, v19.4s, v2.s[3]\n"
- "fmla v27.4s, v19.4s, v3.s[3]\n"
- "fmla v28.4s, v19.4s, v4.s[3]\n"
- "fmla v29.4s, v19.4s, v5.s[3]\n"
- "fmla v30.4s, v19.4s, v6.s[3]\n"
- "fmla v31.4s, v19.4s, v7.s[3]\n"
- "ldr q19, [%[b_ptr0], #0x30]\n"
- "fmla v24.4s, v16.4s, v8.s[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x40\n"
- "fmla v25.4s, v16.4s, v9.s[0]\n"
- "fmla v26.4s, v16.4s, v10.s[0]\n"
- "fmla v27.4s, v16.4s, v11.s[0]\n"
- "fmla v28.4s, v16.4s, v12.s[0]\n"
- "fmla v29.4s, v16.4s, v13.s[0]\n"
- "fmla v30.4s, v16.4s, v14.s[0]\n"
- "fmla v31.4s, v16.4s, v15.s[0]\n"
- "fmla v24.4s, v17.4s, v8.s[1]\n"
- "fmla v25.4s, v17.4s, v9.s[1]\n"
- "fmla v26.4s, v17.4s, v10.s[1]\n"
- "fmla v27.4s, v17.4s, v11.s[1]\n"
- "fmla v28.4s, v17.4s, v12.s[1]\n"
- "fmla v29.4s, v17.4s, v13.s[1]\n"
- "fmla v30.4s, v17.4s, v14.s[1]\n"
- "fmla v31.4s, v17.4s, v15.s[1]\n"
- "fmla v24.4s, v18.4s, v8.s[2]\n"
- "fmla v25.4s, v18.4s, v9.s[2]\n"
- "fmla v26.4s, v18.4s, v10.s[2]\n"
- "fmla v27.4s, v18.4s, v11.s[2]\n"
- "fmla v28.4s, v18.4s, v12.s[2]\n"
- "fmla v29.4s, v18.4s, v13.s[2]\n"
- "fmla v30.4s, v18.4s, v14.s[2]\n"
- "fmla v31.4s, v18.4s, v15.s[2]\n"
- "fmla v24.4s, v19.4s, v8.s[3]\n"
- "fmla v25.4s, v19.4s, v9.s[3]\n"
- "fmla v26.4s, v19.4s, v10.s[3]\n"
- "fmla v27.4s, v19.4s, v11.s[3]\n"
- "fmla v28.4s, v19.4s, v12.s[3]\n"
- "fmla v29.4s, v19.4s, v13.s[3]\n"
- "fmla v30.4s, v19.4s, v14.s[3]\n"
- "fmla v31.4s, v19.4s, v15.s[3]\n"
- "b 4f\n"
- "3:\n"
- "fmla v24.4s, v16.4s, v0.s[0]\n"
- "fmla v25.4s, v16.4s, v1.s[0]\n"
- "fmla v26.4s, v16.4s, v2.s[0]\n"
- "fmla v27.4s, v16.4s, v3.s[0]\n"
- "fmla v28.4s, v16.4s, v4.s[0]\n"
- "fmla v29.4s, v16.4s, v5.s[0]\n"
- "fmla v30.4s, v16.4s, v6.s[0]\n"
- "fmla v31.4s, v16.4s, v7.s[0]\n"
- "fmla v24.4s, v17.4s, v0.s[1]\n"
- "fmla v25.4s, v17.4s, v1.s[1]\n"
- "fmla v26.4s, v17.4s, v2.s[1]\n"
- "fmla v27.4s, v17.4s, v3.s[1]\n"
- "fmla v28.4s, v17.4s, v4.s[1]\n"
- "fmla v29.4s, v17.4s, v5.s[1]\n"
- "fmla v30.4s, v17.4s, v6.s[1]\n"
- "fmla v31.4s, v17.4s, v7.s[1]\n"
- "fmla v24.4s, v18.4s, v0.s[2]\n"
- "fmla v25.4s, v18.4s, v1.s[2]\n"
- "fmla v26.4s, v18.4s, v2.s[2]\n"
- "fmla v27.4s, v18.4s, v3.s[2]\n"
- "fmla v28.4s, v18.4s, v4.s[2]\n"
- "fmla v29.4s, v18.4s, v5.s[2]\n"
- "fmla v30.4s, v18.4s, v6.s[2]\n"
- "fmla v31.4s, v18.4s, v7.s[2]\n"
- "fmla v24.4s, v19.4s, v0.s[3]\n"
- "fmla v25.4s, v19.4s, v1.s[3]\n"
- "fmla v26.4s, v19.4s, v2.s[3]\n"
- "fmla v27.4s, v19.4s, v3.s[3]\n"
- "fmla v28.4s, v19.4s, v4.s[3]\n"
- "fmla v29.4s, v19.4s, v5.s[3]\n"
- "fmla v30.4s, v19.4s, v6.s[3]\n"
- "fmla v31.4s, v19.4s, v7.s[3]\n"
- "4:\n"
- "cbz %[blocks], 5f\n"
- "6:\n"
- "ldr q16, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "add %[b_ptr0], %[b_ptr0], #0x10\n"
- "ldr s0, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x4\n"
- "ldr s1, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, #0x4\n"
- "fmla v24.4s, v16.4s, v0.s[0]\n"
- "ldr s2, [a_ptr2]\n"
- "fmla v25.4s, v16.4s, v1.s[0]\n"
- "add a_ptr2, a_ptr2, #0x4\n"
- "ldr s3, [a_ptr3]\n"
- "fmla v26.4s, v16.4s, v2.s[0]\n"
- "add a_ptr3, a_ptr3, #0x4\n"
- "ldr s4, [a_ptr4]\n"
- "fmla v27.4s, v16.4s, v3.s[0]\n"
- "add a_ptr4, a_ptr4, #0x4\n"
- "ldr s5, [a_ptr5]\n"
- "fmla v28.4s, v16.4s, v4.s[0]\n"
- "add a_ptr5, a_ptr5, #0x4\n"
- "ldr s6, [a_ptr6]\n"
- "fmla v29.4s, v16.4s, v5.s[0]\n"
- "add a_ptr6, a_ptr6, #0x4\n"
- "ldr s7, [a_ptr7]\n"
- "fmla v30.4s, v16.4s, v6.s[0]\n"
- "add a_ptr7, a_ptr7, #0x4\n"
- "fmla v31.4s, v16.4s, v7.s[0]\n"
- "b.ne 6b\n"
- "5:\n"
- "ld1r {v22.4s}, [%[minptr]]\n"
- "ld1r {v23.4s}, [%[maxptr]]\n"
- "fmax v24.4s, v24.4s, v22.4s\n"
- "fmax v25.4s, v25.4s, v22.4s\n"
- "fmax v26.4s, v26.4s, v22.4s\n"
- "fmax v27.4s, v27.4s, v22.4s\n"
- "fmin v24.4s, v24.4s, v23.4s\n"
- "fmin v25.4s, v25.4s, v23.4s\n"
- "fmin v26.4s, v26.4s, v23.4s\n"
- "fmin v27.4s, v27.4s, v23.4s\n"
- "str q24, [%[c_ptr0]]\n"
- "fmax v28.4s, v28.4s, v22.4s\n"
- "add %[c_ptr0], %[c_ptr0], #0x10\n"
- "fmax v29.4s, v29.4s, v22.4s\n"
- "str q25, [c_ptr1]\n"
- "fmax v30.4s, v30.4s, v22.4s\n"
- "fmin v28.4s, v28.4s, v23.4s\n"
- "fmax v31.4s, v31.4s, v22.4s\n"
- "str q26, [c_ptr2]\n"
- "fmin v29.4s, v29.4s, v23.4s\n"
- "fmin v30.4s, v30.4s, v23.4s\n"
- "fmin v31.4s, v31.4s, v23.4s\n"
- "str q27, [c_ptr3]\n"
- "str q28, [c_ptr4]\n"
- "str q29, [c_ptr5]\n"
- "str q30, [c_ptr6]\n"
- "str q31, [c_ptr7]\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq a_ptr3\n"
- ".unreq a_ptr4\n"
- ".unreq a_ptr5\n"
- ".unreq a_ptr6\n"
- ".unreq a_ptr7\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- ".unreq c_ptr3\n"
- ".unreq c_ptr4\n"
- ".unreq c_ptr5\n"
- ".unreq c_ptr6\n"
- ".unreq c_ptr7\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks)
- : [width] "r" (width), [accumulate] "r" (static_cast<uint64_t>(accumulate)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr)
- : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "cc", "memory"
- );
- break;
- }
- if (use_result_buffer) {
- for(int cy=0; cy<std::min(M-y, 8); cy++) {
- for(unsigned int cx=0; cx<width; cx++) {
- c_ptr_real[cy * ldc + cx] = result_buffer[cy * 4 + cx];
- }
- }
- }
- }
- }
-}
-
-} // namespace arm_gemm
-
-#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_6x16.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_6x16.hpp
new file mode 100644
index 0000000..e0c61e4
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_6x16.hpp
@@ -0,0 +1,102 @@
+/*
+ * Copyright (c) 2019-2020 Arm Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+#pragma once
+#ifdef __aarch64__
+
+#include "../performance_parameters.hpp"
+#include "../std_transforms_fixed.hpp"
+
+#define ARGLIST \
+ unsigned int, const unsigned int *, \
+ IndirectInputArg<float>, \
+ size_t, size_t, \
+ const float *, \
+ IndirectOutputArg<float>, \
+ const float *, Activation, bool
+
+namespace arm_gemm
+{
+
+// Actual kernel implementations
+void a64_hybrid_fp32_mla_6x16( ARGLIST );
+
+class cls_a64_hybrid_fp32_mla_6x16
+{
+public:
+ typedef float operand_type;
+ typedef float result_type;
+
+ typedef void (*kern_type)( ARGLIST );
+
+ /* Kernel blocking parameters */
+ static constexpr unsigned int out_height()
+ {
+ return 6;
+ }
+
+ static unsigned int out_width()
+ {
+ return 16;
+ }
+
+ static constexpr unsigned int k_unroll()
+ {
+ return 1;
+ }
+
+ static constexpr bool supports_accumulate()
+ {
+ return true;
+ }
+
+ static PerformanceParameters get_performance_parameters(const CPUInfo *ci) {
+ switch (ci->get_cpu_model()) {
+ case CPUModel::A55r1:
+ return { 2.00 };
+
+ case CPUModel::A53:
+ return { 1.43 };
+
+ case CPUModel::A73:
+ return { 2.56 };
+
+ default:
+ return { 6.26 };
+ }
+ }
+
+ StdTransformsFixed<operand_type, result_type, 6, 16, 1> transforms = {};
+
+ // Default to the generic kernel
+ kern_type kernel=a64_hybrid_fp32_mla_6x16;
+
+ cls_a64_hybrid_fp32_mla_6x16(const CPUInfo *)
+ {
+ }
+};
+
+} // namespace arm_gemm
+
+#undef ARGLIST
+#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_6x16/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_6x16/generic.cpp
new file mode 100644
index 0000000..884e898
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_6x16/generic.cpp
@@ -0,0 +1,3430 @@
+/*
+ * Copyright (c) 2019-2020 Arm Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+#ifdef __aarch64__
+
+#include "arm_gemm.hpp"
+#include "../../utils.hpp"
+
+#include <cassert>
+
+namespace arm_gemm {
+
+void a64_hybrid_fp32_mla_6x16 (
+ unsigned int num_strings, const unsigned int *string_lengths, IndirectInputArg<float> A_arg,
+ size_t M, size_t N, const float *B_ptr, IndirectOutputArg<float> output_arg,
+ const float *bias, Activation act, bool accumulate
+)
+{
+ struct KernelArgs {
+ float maxval = static_cast<float>(std::numeric_limits<float>::infinity());
+ float minval = - static_cast<float>(std::numeric_limits<float>::infinity());
+ unsigned int num_strings = {};
+ const unsigned int *string_lengths = {};
+ size_t N = {};
+ const float *B_ptr = {};
+ size_t output_offset = {};
+ size_t input_initial_col = {};
+ size_t input_offset = {};
+ } ka;
+
+ unsigned long flags=0;
+ void *output_ptr;
+ void *input_ptr;
+
+ if (output_arg.is_indirect) {
+ output_ptr=(void *)(output_arg.indirect.ptr);
+ ka.output_offset=output_arg.indirect.offset;
+ flags |= 0x4;
+ } else {
+ output_ptr=(void *)(output_arg.direct.base);
+ ka.output_offset=output_arg.direct.stride;
+ }
+
+ if (A_arg.is_indirect) {
+ input_ptr=(void *)(A_arg.indirect.ptr);
+ ka.input_offset=A_arg.indirect.start_row;
+ ka.input_initial_col=A_arg.indirect.start_col;
+ flags |= 0x8;
+ } else {
+ assert(num_strings==1);
+ input_ptr=(void *)(A_arg.direct.base);
+ ka.input_offset=A_arg.direct.stride;
+ }
+ if (accumulate) {
+ flags |= 0x1;
+ }
+ ka.num_strings = num_strings;
+ ka.string_lengths = string_lengths;
+ ka.N = N;
+ ka.B_ptr = B_ptr;
+ switch(act.type) {
+ default:
+ case Activation::Type::None:
+ break;
+ case Activation::Type::BoundedReLU:
+ ka.maxval = static_cast<float>(act.param1);
+ /* fall through */
+ case Activation::Type::ReLU:
+ ka.minval = 0;
+ flags |= 0x2;
+ break;
+ }
+ __asm__ __volatile__(
+
+ "1:" // Row loop
+ "cmp %x[M], #0x6\n"
+ "bge 171f\n"
+ "cmp %x[M], #0x4\n"
+ "bgt 137f\n"
+ "beq 103f\n"
+ "cmp %x[M], #0x2\n"
+ "bgt 69f\n"
+ "beq 35f\n"
+ "ldr x16, [%x[args_ptr], %[offsetof_N]]\n"
+ "ldr x15, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "mov x14, %x[bias]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 2f\n"
+ "ldr x13, [%x[output_ptr], #0x0]\n"
+ "add x13, x13, x19, LSL #2\n"
+ "b 3f\n"
+ "2:" // Height 1: setup direct output
+ "mov x13, %x[output_ptr]\n"
+ "3:" // Height 1: Column loop
+ "cbz x14, 4f\n"
+ "ldr q8, [x14, #0x0]\n"
+ "ldr q9, [x14, #0x10]\n"
+ "ldr q10, [x14, #0x20]\n"
+ "ldr q11, [x14, #0x30]\n"
+ "add x14, x14, #0x40\n"
+ "b 15f\n"
+ "4:" // Height 1: no bias
+ "tbz %x[flags], #0, 14f\n"
+ "cmp x16, #0x10\n"
+ "bge 13f\n"
+ "tbz x16, #3, 8f\n"
+ "ld1 { v8.4s }, [x13], #0x10\n"
+ "ld1 { v9.4s }, [x13], #0x10\n"
+ "tbz x16, #2, 6f\n"
+ "ld1 { v10.4s }, [x13], #0x10\n"
+ "tbz x16, #1, 5f\n"
+ "mov x19, #0x38\n"
+ "ldr d11, [x13], #0x8\n"
+ "tbz x16, #0, 12f\n"
+ "ld1 { v11.s }[2], [x13]\n"
+ "b 12f\n"
+ "5:" // Height 1: Partial accumulate: partial_1_12
+ "mov x19, #0x30\n"
+ "tbz x16, #0, 12f\n"
+ "ldr s11, [x13, #0x0]\n"
+ "b 12f\n"
+ "6:" // Height 1: Partial accumulate: partial_2_8
+ "tbz x16, #1, 7f\n"
+ "ldr d10, [x13], #0x8\n"
+ "mov x19, #0x28\n"
+ "tbz x16, #0, 12f\n"
+ "ld1 { v10.s }[2], [x13]\n"
+ "b 12f\n"
+ "7:" // Height 1: Partial accumulate: partial_1_8
+ "mov x19, #0x20\n"
+ "tbz x16, #0, 12f\n"
+ "ldr s10, [x13, #0x0]\n"
+ "b 12f\n"
+ "8:" // Height 1: Partial accumulate: partial_4_0
+ "tbz x16, #2, 10f\n"
+ "ld1 { v8.4s }, [x13], #0x10\n"
+ "tbz x16, #1, 9f\n"
+ "mov x19, #0x18\n"
+ "ldr d9, [x13], #0x8\n"
+ "tbz x16, #0, 12f\n"
+ "ld1 { v9.s }[2], [x13]\n"
+ "b 12f\n"
+ "9:" // Height 1: Partial accumulate: partial_1_4
+ "mov x19, #0x10\n"
+ "tbz x16, #0, 12f\n"
+ "ldr s9, [x13, #0x0]\n"
+ "b 12f\n"
+ "10:" // Height 1: Partial accumulate: partial_2_0
+ "tbz x16, #1, 11f\n"
+ "ldr d8, [x13], #0x8\n"
+ "mov x19, #0x8\n"
+ "tbz x16, #0, 12f\n"
+ "ld1 { v8.s }[2], [x13]\n"
+ "b 12f\n"
+ "11:" // Height 1: Partial accumulate: partial_1_0
+ "mov x19, #0x0\n"
+ "ldr s8, [x13, #0x0]\n"
+ "12:" // Height 1: Partial accumulate: Done
+ "sub x13, x13, x19\n"
+ "b 15f\n"
+ "13:" // Height 1: full accumulate
+ "ldr q8, [x13, #0x0]\n"
+ "ldr q9, [x13, #0x10]\n"
+ "ldr q10, [x13, #0x20]\n"
+ "ldr q11, [x13, #0x30]\n"
+ "b 15f\n"
+ "14:" // Height 1: no accumulate
+ "movi v8.16b, #0x0\n"
+ "movi v9.16b, #0x0\n"
+ "movi v10.16b, #0x0\n"
+ "movi v11.16b, #0x0\n"
+ "15:" // Height 1: setup done
+ "mov x12, #0x0\n"
+ "16:" // Height 1: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w11, [x20, x12, LSL #0x2]\n"
+ "tbz %x[flags], #3, 17f\n"
+ "ldr x20, [%x[input_ptr], x12, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x10, [x20, #0x0]\n"
+ "cbnz x12, 18f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x10, x10, x19, LSL #2\n"
+ "b 18f\n"
+ "17:" // Height 1: setup direct input
+ "mov x10, %x[input_ptr]\n"
+ "18:" // Height 1: input setup done
+ "cmp x11, #0x4\n"
+ "blt 21f\n"
+ "cmp x11, #0x8\n"
+ "blt 20f\n"
+ "19:" // Height 1: Multiply loop: Main loop head
+ "ldr q0, [x10, #0x0]\n"
+ "ldr q6, [x15, #0x0]\n"
+ "fmla v8.4s, v6.4s, v0.s[0]\n"
+ "ldr q7, [x15, #0x10]\n"
+ "ldr q6, [x15, #0x20]\n"
+ "fmla v9.4s, v7.4s, v0.s[0]\n"
+ "ldr q7, [x15, #0x30]\n"
+ "fmla v10.4s, v6.4s, v0.s[0]\n"
+ "ldr q6, [x15, #0x40]\n"
+ "add x10, x10, #0x10\n"
+ "fmla v11.4s, v7.4s, v0.s[0]\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "ldr q7, [x15, #0x50]\n"
+ "fmla v8.4s, v6.4s, v0.s[1]\n"
+ "ldr q6, [x15, #0x60]\n"
+ "sub x11, x11, #0x4\n"
+ "fmla v9.4s, v7.4s, v0.s[1]\n"
+ "ldr q7, [x15, #0x70]\n"
+ "cmp x11, #0x8\n"
+ "fmla v10.4s, v6.4s, v0.s[1]\n"
+ "ldr q6, [x15, #0x80]\n"
+ "fmla v11.4s, v7.4s, v0.s[1]\n"
+ "ldr q7, [x15, #0x90]\n"
+ "fmla v8.4s, v6.4s, v0.s[2]\n"
+ "ldr q6, [x15, #0xa0]\n"
+ "fmla v9.4s, v7.4s, v0.s[2]\n"
+ "ldr q7, [x15, #0xb0]\n"
+ "fmla v10.4s, v6.4s, v0.s[2]\n"
+ "ldr q6, [x15, #0xc0]\n"
+ "fmla v11.4s, v7.4s, v0.s[2]\n"
+ "ldr q7, [x15, #0xd0]\n"
+ "fmla v8.4s, v6.4s, v0.s[3]\n"
+ "ldr q6, [x15, #0xe0]\n"
+ "fmla v9.4s, v7.4s, v0.s[3]\n"
+ "ldr q7, [x15, #0xf0]\n"
+ "add x15, x15, #0x100\n"
+ "fmla v10.4s, v6.4s, v0.s[3]\n"
+ "fmla v11.4s, v7.4s, v0.s[3]\n"
+ "bge 19b\n"
+ "20:" // Height 1: Multiply loop: Single iteration only
+ "sub x11, x11, #0x4\n"
+ "ldr q0, [x10, #0x0]\n"
+ "ldr q6, [x15, #0x0]\n"
+ "fmla v8.4s, v6.4s, v0.s[0]\n"
+ "ldr q7, [x15, #0x10]\n"
+ "ldr q6, [x15, #0x20]\n"
+ "fmla v9.4s, v7.4s, v0.s[0]\n"
+ "ldr q7, [x15, #0x30]\n"
+ "fmla v10.4s, v6.4s, v0.s[0]\n"
+ "ldr q6, [x15, #0x40]\n"
+ "add x10, x10, #0x10\n"
+ "fmla v11.4s, v7.4s, v0.s[0]\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "ldr q7, [x15, #0x50]\n"
+ "fmla v8.4s, v6.4s, v0.s[1]\n"
+ "ldr q6, [x15, #0x60]\n"
+ "fmla v9.4s, v7.4s, v0.s[1]\n"
+ "ldr q7, [x15, #0x70]\n"
+ "fmla v10.4s, v6.4s, v0.s[1]\n"
+ "ldr q6, [x15, #0x80]\n"
+ "fmla v11.4s, v7.4s, v0.s[1]\n"
+ "ldr q7, [x15, #0x90]\n"
+ "fmla v8.4s, v6.4s, v0.s[2]\n"
+ "ldr q6, [x15, #0xa0]\n"
+ "fmla v9.4s, v7.4s, v0.s[2]\n"
+ "ldr q7, [x15, #0xb0]\n"
+ "fmla v10.4s, v6.4s, v0.s[2]\n"
+ "ldr q6, [x15, #0xc0]\n"
+ "fmla v11.4s, v7.4s, v0.s[2]\n"
+ "ldr q7, [x15, #0xd0]\n"
+ "fmla v8.4s, v6.4s, v0.s[3]\n"
+ "ldr q6, [x15, #0xe0]\n"
+ "fmla v9.4s, v7.4s, v0.s[3]\n"
+ "ldr q7, [x15, #0xf0]\n"
+ "add x15, x15, #0x100\n"
+ "fmla v10.4s, v6.4s, v0.s[3]\n"
+ "fmla v11.4s, v7.4s, v0.s[3]\n"
+ "21:" // Height 1: Multiply loop: Main loop skip
+ "cbz x11, 23f\n"
+ "22:" // Height 1: Multiply loop: Odd block loop
+ "ldr s0, [x10], #0x4\n"
+ "ldr q6, [x15, #0x0]\n"
+ "fmla v8.4s, v6.4s, v0.s[0]\n"
+ "ldr q7, [x15, #0x10]\n"
+ "ldr q6, [x15, #0x20]\n"
+ "fmla v9.4s, v7.4s, v0.s[0]\n"
+ "ldr q7, [x15, #0x30]\n"
+ "fmla v10.4s, v6.4s, v0.s[0]\n"
+ "sub x11, x11, #0x1\n"
+ "add x15, x15, #0x40\n"
+ "fmla v11.4s, v7.4s, v0.s[0]\n"
+ "cbnz x11, 22b\n"
+ "23:" // Height 1: Multiply loop: No odd multiplies
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "add x12, x12, #0x1\n"
+ "cmp x12, x19\n"
+ "bne 16b\n"
+ "prfm pstl1keep, [x13, #0x0]\n"
+ "tbz %x[flags], #1, 24f\n"
+ "add x19, %x[args_ptr], %[offset_min]\n"
+ "ld1r { v1.4s }, [x19]\n"
+ "add x19, %x[args_ptr], %[offset_max]\n"
+ "ld1r { v0.4s }, [x19]\n"
+ "fmin v8.4s, v8.4s, v0.4s\n"
+ "fmin v9.4s, v9.4s, v0.4s\n"
+ "fmin v10.4s, v10.4s, v0.4s\n"
+ "fmin v11.4s, v11.4s, v0.4s\n"
+ "fmax v8.4s, v8.4s, v1.4s\n"
+ "fmax v9.4s, v9.4s, v1.4s\n"
+ "fmax v10.4s, v10.4s, v1.4s\n"
+ "fmax v11.4s, v11.4s, v1.4s\n"
+ "24:" // Height 1: No activation
+ "cmp x16, #0x10\n"
+ "bge 33f\n"
+ "tbz x16, #3, 28f\n"
+ "st1 { v8.4s }, [x13], #0x10\n"
+ "st1 { v9.4s }, [x13], #0x10\n"
+ "tbz x16, #2, 26f\n"
+ "st1 { v10.4s }, [x13], #0x10\n"
+ "tbz x16, #1, 25f\n"
+ "str d11, [x13], #0x8\n"
+ "tbz x16, #0, 32f\n"
+ "st1 { v11.s }[2], [x13]\n"
+ "b 32f\n"
+ "25:" // Height 1: Partial direct writeback: partial_1_12
+ "tbz x16, #0, 32f\n"
+ "str s11, [x13, #0x0]\n"
+ "b 32f\n"
+ "26:" // Height 1: Partial direct writeback: partial_2_8
+ "tbz x16, #1, 27f\n"
+ "str d10, [x13], #0x8\n"
+ "tbz x16, #0, 32f\n"
+ "st1 { v10.s }[2], [x13]\n"
+ "b 32f\n"
+ "27:" // Height 1: Partial direct writeback: partial_1_8
+ "tbz x16, #0, 32f\n"
+ "str s10, [x13, #0x0]\n"
+ "b 32f\n"
+ "28:" // Height 1: Partial direct writeback: partial_4_0
+ "tbz x16, #2, 30f\n"
+ "st1 { v8.4s }, [x13], #0x10\n"
+ "tbz x16, #1, 29f\n"
+ "str d9, [x13], #0x8\n"
+ "tbz x16, #0, 32f\n"
+ "st1 { v9.s }[2], [x13]\n"
+ "b 32f\n"
+ "29:" // Height 1: Partial direct writeback: partial_1_4
+ "tbz x16, #0, 32f\n"
+ "str s9, [x13, #0x0]\n"
+ "b 32f\n"
+ "30:" // Height 1: Partial direct writeback: partial_2_0
+ "tbz x16, #1, 31f\n"
+ "str d8, [x13], #0x8\n"
+ "tbz x16, #0, 32f\n"
+ "st1 { v8.s }[2], [x13]\n"
+ "b 32f\n"
+ "31:" // Height 1: Partial direct writeback: partial_1_0
+ "str s8, [x13, #0x0]\n"
+ "32:" // Height 1: Partial direct writeback: Done
+ "b 34f\n"
+ "33:" // Height 1: Full writeback
+ "str q8, [x13, #0x0]\n"
+ "str q9, [x13, #0x10]\n"
+ "str q10, [x13, #0x20]\n"
+ "str q11, [x13, #0x30]\n"
+ "add x13, x13, #0x40\n"
+ "34:" // Height 1: Writeback done
+ "subs x16, x16, #0x10\n"
+ "bgt 3b\n"
+ "b 206f\n"
+ "35:" // Height 2
+ "ldr x16, [%x[args_ptr], %[offsetof_N]]\n"
+ "mov x14, %x[bias]\n"
+ "ldr x15, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 36f\n"
+ "ldr x13, [%x[output_ptr], #0x0]\n"
+ "add x13, x13, x19, LSL #2\n"
+ "ldr x9, [%x[output_ptr], #0x8]\n"
+ "add x9, x9, x19, LSL #2\n"
+ "b 37f\n"
+ "36:" // Height 2: setup direct output
+ "mov x13, %x[output_ptr]\n"
+ "add x9, x13, x19, LSL #2\n"
+ "37:" // Height 2: Column loop
+ "cbz x14, 38f\n"
+ "ldr q8, [x14, #0x0]\n"
+ "mov v12.16b, v8.16b\n"
+ "ldr q9, [x14, #0x10]\n"
+ "ldr q10, [x14, #0x20]\n"
+ "mov v13.16b, v9.16b\n"
+ "ldr q11, [x14, #0x30]\n"
+ "mov v14.16b, v10.16b\n"
+ "add x14, x14, #0x40\n"
+ "mov v15.16b, v11.16b\n"
+ "b 49f\n"
+ "38:" // Height 2: no bias
+ "tbz %x[flags], #0, 48f\n"
+ "cmp x16, #0x10\n"
+ "bge 47f\n"
+ "tbz x16, #3, 42f\n"
+ "ld1 { v8.4s }, [x13], #0x10\n"
+ "ld1 { v12.4s }, [x9], #0x10\n"
+ "ld1 { v9.4s }, [x13], #0x10\n"
+ "ld1 { v13.4s }, [x9], #0x10\n"
+ "tbz x16, #2, 40f\n"
+ "ld1 { v10.4s }, [x13], #0x10\n"
+ "ld1 { v14.4s }, [x9], #0x10\n"
+ "tbz x16, #1, 39f\n"
+ "mov x19, #0x38\n"
+ "ldr d11, [x13], #0x8\n"
+ "ldr d15, [x9], #0x8\n"
+ "tbz x16, #0, 46f\n"
+ "ld1 { v11.s }[2], [x13]\n"
+ "ld1 { v15.s }[2], [x9]\n"
+ "b 46f\n"
+ "39:" // Height 2: Partial accumulate: partial_1_12
+ "mov x19, #0x30\n"
+ "tbz x16, #0, 46f\n"
+ "ldr s11, [x13, #0x0]\n"
+ "ldr s15, [x9, #0x0]\n"
+ "b 46f\n"
+ "40:" // Height 2: Partial accumulate: partial_2_8
+ "tbz x16, #1, 41f\n"
+ "ldr d10, [x13], #0x8\n"
+ "ldr d14, [x9], #0x8\n"
+ "mov x19, #0x28\n"
+ "tbz x16, #0, 46f\n"
+ "ld1 { v10.s }[2], [x13]\n"
+ "ld1 { v14.s }[2], [x9]\n"
+ "b 46f\n"
+ "41:" // Height 2: Partial accumulate: partial_1_8
+ "mov x19, #0x20\n"
+ "tbz x16, #0, 46f\n"
+ "ldr s10, [x13, #0x0]\n"
+ "ldr s14, [x9, #0x0]\n"
+ "b 46f\n"
+ "42:" // Height 2: Partial accumulate: partial_4_0
+ "tbz x16, #2, 44f\n"
+ "ld1 { v8.4s }, [x13], #0x10\n"
+ "ld1 { v12.4s }, [x9], #0x10\n"
+ "tbz x16, #1, 43f\n"
+ "mov x19, #0x18\n"
+ "ldr d9, [x13], #0x8\n"
+ "ldr d13, [x9], #0x8\n"
+ "tbz x16, #0, 46f\n"
+ "ld1 { v9.s }[2], [x13]\n"
+ "ld1 { v13.s }[2], [x9]\n"
+ "b 46f\n"
+ "43:" // Height 2: Partial accumulate: partial_1_4
+ "mov x19, #0x10\n"
+ "tbz x16, #0, 46f\n"
+ "ldr s9, [x13, #0x0]\n"
+ "ldr s13, [x9, #0x0]\n"
+ "b 46f\n"
+ "44:" // Height 2: Partial accumulate: partial_2_0
+ "tbz x16, #1, 45f\n"
+ "ldr d8, [x13], #0x8\n"
+ "ldr d12, [x9], #0x8\n"
+ "mov x19, #0x8\n"
+ "tbz x16, #0, 46f\n"
+ "ld1 { v8.s }[2], [x13]\n"
+ "ld1 { v12.s }[2], [x9]\n"
+ "b 46f\n"
+ "45:" // Height 2: Partial accumulate: partial_1_0
+ "mov x19, #0x0\n"
+ "ldr s8, [x13, #0x0]\n"
+ "ldr s12, [x9, #0x0]\n"
+ "46:" // Height 2: Partial accumulate: Done
+ "sub x13, x13, x19\n"
+ "sub x9, x9, x19\n"
+ "b 49f\n"
+ "47:" // Height 2: full accumulate
+ "ldr q8, [x13, #0x0]\n"
+ "ldr q9, [x13, #0x10]\n"
+ "ldr q10, [x13, #0x20]\n"
+ "ldr q11, [x13, #0x30]\n"
+ "ldr q12, [x9, #0x0]\n"
+ "ldr q13, [x9, #0x10]\n"
+ "ldr q14, [x9, #0x20]\n"
+ "ldr q15, [x9, #0x30]\n"
+ "b 49f\n"
+ "48:" // Height 2: no accumulate
+ "movi v8.16b, #0x0\n"
+ "movi v9.16b, #0x0\n"
+ "movi v10.16b, #0x0\n"
+ "movi v11.16b, #0x0\n"
+ "movi v12.16b, #0x0\n"
+ "movi v13.16b, #0x0\n"
+ "movi v14.16b, #0x0\n"
+ "movi v15.16b, #0x0\n"
+ "49:" // Height 2: setup done
+ "mov x12, #0x0\n"
+ "50:" // Height 2: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w11, [x20, x12, LSL #0x2]\n"
+ "tbz %x[flags], #3, 51f\n"
+ "ldr x20, [%x[input_ptr], x12, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x10, [x20, #0x0]\n"
+ "ldr x28, [x20, #0x8]\n"
+ "cbnz x12, 52f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x10, x10, x19, LSL #2\n"
+ "add x28, x28, x19, LSL #2\n"
+ "b 52f\n"
+ "51:" // Height 2: setup direct input
+ "mov x10, %x[input_ptr]\n"
+ "add x28, x10, x19, LSL #2\n"
+ "52:" // Height 2: input setup done
+ "cmp x11, #0x4\n"
+ "blt 55f\n"
+ "cmp x11, #0x8\n"
+ "blt 54f\n"
+ "53:" // Height 2: Multiply loop: Main loop head
+ "ldr q0, [x10, #0x0]\n"
+ "ldr q1, [x28, #0x0]\n"
+ "ldr q6, [x15, #0x0]\n"
+ "fmla v8.4s, v6.4s, v0.s[0]\n"
+ "ldr q7, [x15, #0x10]\n"
+ "fmla v12.4s, v6.4s, v1.s[0]\n"
+ "ldr q6, [x15, #0x20]\n"
+ "add x10, x10, #0x10\n"
+ "fmla v9.4s, v7.4s, v0.s[0]\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x28, x28, #0x10\n"
+ "fmla v13.4s, v7.4s, v1.s[0]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "ldr q7, [x15, #0x30]\n"
+ "fmla v10.4s, v6.4s, v0.s[0]\n"
+ "sub x11, x11, #0x4\n"
+ "fmla v14.4s, v6.4s, v1.s[0]\n"
+ "ldr q6, [x15, #0x40]\n"
+ "cmp x11, #0x8\n"
+ "fmla v11.4s, v7.4s, v0.s[0]\n"
+ "fmla v15.4s, v7.4s, v1.s[0]\n"
+ "ldr q7, [x15, #0x50]\n"
+ "fmla v8.4s, v6.4s, v0.s[1]\n"
+ "fmla v12.4s, v6.4s, v1.s[1]\n"
+ "ldr q6, [x15, #0x60]\n"
+ "fmla v9.4s, v7.4s, v0.s[1]\n"
+ "fmla v13.4s, v7.4s, v1.s[1]\n"
+ "ldr q7, [x15, #0x70]\n"
+ "fmla v10.4s, v6.4s, v0.s[1]\n"
+ "fmla v14.4s, v6.4s, v1.s[1]\n"
+ "ldr q6, [x15, #0x80]\n"
+ "fmla v11.4s, v7.4s, v0.s[1]\n"
+ "fmla v15.4s, v7.4s, v1.s[1]\n"
+ "ldr q7, [x15, #0x90]\n"
+ "fmla v8.4s, v6.4s, v0.s[2]\n"
+ "fmla v12.4s, v6.4s, v1.s[2]\n"
+ "ldr q6, [x15, #0xa0]\n"
+ "fmla v9.4s, v7.4s, v0.s[2]\n"
+ "fmla v13.4s, v7.4s, v1.s[2]\n"
+ "ldr q7, [x15, #0xb0]\n"
+ "fmla v10.4s, v6.4s, v0.s[2]\n"
+ "fmla v14.4s, v6.4s, v1.s[2]\n"
+ "ldr q6, [x15, #0xc0]\n"
+ "fmla v11.4s, v7.4s, v0.s[2]\n"
+ "fmla v15.4s, v7.4s, v1.s[2]\n"
+ "ldr q7, [x15, #0xd0]\n"
+ "fmla v8.4s, v6.4s, v0.s[3]\n"
+ "fmla v12.4s, v6.4s, v1.s[3]\n"
+ "ldr q6, [x15, #0xe0]\n"
+ "fmla v9.4s, v7.4s, v0.s[3]\n"
+ "fmla v13.4s, v7.4s, v1.s[3]\n"
+ "ldr q7, [x15, #0xf0]\n"
+ "add x15, x15, #0x100\n"
+ "fmla v10.4s, v6.4s, v0.s[3]\n"
+ "fmla v14.4s, v6.4s, v1.s[3]\n"
+ "fmla v11.4s, v7.4s, v0.s[3]\n"
+ "fmla v15.4s, v7.4s, v1.s[3]\n"
+ "bge 53b\n"
+ "54:" // Height 2: Multiply loop: Single iteration only
+ "sub x11, x11, #0x4\n"
+ "ldr q0, [x10, #0x0]\n"
+ "ldr q1, [x28, #0x0]\n"
+ "ldr q6, [x15, #0x0]\n"
+ "fmla v8.4s, v6.4s, v0.s[0]\n"
+ "ldr q7, [x15, #0x10]\n"
+ "fmla v12.4s, v6.4s, v1.s[0]\n"
+ "ldr q6, [x15, #0x20]\n"
+ "add x10, x10, #0x10\n"
+ "fmla v9.4s, v7.4s, v0.s[0]\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x28, x28, #0x10\n"
+ "fmla v13.4s, v7.4s, v1.s[0]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "ldr q7, [x15, #0x30]\n"
+ "fmla v10.4s, v6.4s, v0.s[0]\n"
+ "fmla v14.4s, v6.4s, v1.s[0]\n"
+ "ldr q6, [x15, #0x40]\n"
+ "fmla v11.4s, v7.4s, v0.s[0]\n"
+ "fmla v15.4s, v7.4s, v1.s[0]\n"
+ "ldr q7, [x15, #0x50]\n"
+ "fmla v8.4s, v6.4s, v0.s[1]\n"
+ "fmla v12.4s, v6.4s, v1.s[1]\n"
+ "ldr q6, [x15, #0x60]\n"
+ "fmla v9.4s, v7.4s, v0.s[1]\n"
+ "fmla v13.4s, v7.4s, v1.s[1]\n"
+ "ldr q7, [x15, #0x70]\n"
+ "fmla v10.4s, v6.4s, v0.s[1]\n"
+ "fmla v14.4s, v6.4s, v1.s[1]\n"
+ "ldr q6, [x15, #0x80]\n"
+ "fmla v11.4s, v7.4s, v0.s[1]\n"
+ "fmla v15.4s, v7.4s, v1.s[1]\n"
+ "ldr q7, [x15, #0x90]\n"
+ "fmla v8.4s, v6.4s, v0.s[2]\n"
+ "fmla v12.4s, v6.4s, v1.s[2]\n"
+ "ldr q6, [x15, #0xa0]\n"
+ "fmla v9.4s, v7.4s, v0.s[2]\n"
+ "fmla v13.4s, v7.4s, v1.s[2]\n"
+ "ldr q7, [x15, #0xb0]\n"
+ "fmla v10.4s, v6.4s, v0.s[2]\n"
+ "fmla v14.4s, v6.4s, v1.s[2]\n"
+ "ldr q6, [x15, #0xc0]\n"
+ "fmla v11.4s, v7.4s, v0.s[2]\n"
+ "fmla v15.4s, v7.4s, v1.s[2]\n"
+ "ldr q7, [x15, #0xd0]\n"
+ "fmla v8.4s, v6.4s, v0.s[3]\n"
+ "fmla v12.4s, v6.4s, v1.s[3]\n"
+ "ldr q6, [x15, #0xe0]\n"
+ "fmla v9.4s, v7.4s, v0.s[3]\n"
+ "fmla v13.4s, v7.4s, v1.s[3]\n"
+ "ldr q7, [x15, #0xf0]\n"
+ "add x15, x15, #0x100\n"
+ "fmla v10.4s, v6.4s, v0.s[3]\n"
+ "fmla v14.4s, v6.4s, v1.s[3]\n"
+ "fmla v11.4s, v7.4s, v0.s[3]\n"
+ "fmla v15.4s, v7.4s, v1.s[3]\n"
+ "55:" // Height 2: Multiply loop: Main loop skip
+ "cbz x11, 57f\n"
+ "56:" // Height 2: Multiply loop: Odd block loop
+ "ldr s0, [x10], #0x4\n"
+ "ldr s1, [x28], #0x4\n"
+ "ldr q6, [x15, #0x0]\n"
+ "fmla v8.4s, v6.4s, v0.s[0]\n"
+ "ldr q7, [x15, #0x10]\n"
+ "fmla v12.4s, v6.4s, v1.s[0]\n"
+ "ldr q6, [x15, #0x20]\n"
+ "sub x11, x11, #0x1\n"
+ "fmla v9.4s, v7.4s, v0.s[0]\n"
+ "fmla v13.4s, v7.4s, v1.s[0]\n"
+ "ldr q7, [x15, #0x30]\n"
+ "fmla v10.4s, v6.4s, v0.s[0]\n"
+ "add x15, x15, #0x40\n"
+ "fmla v14.4s, v6.4s, v1.s[0]\n"
+ "fmla v11.4s, v7.4s, v0.s[0]\n"
+ "fmla v15.4s, v7.4s, v1.s[0]\n"
+ "cbnz x11, 56b\n"
+ "57:" // Height 2: Multiply loop: No odd multiplies
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "add x12, x12, #0x1\n"
+ "cmp x12, x19\n"
+ "bne 50b\n"
+ "prfm pstl1keep, [x13, #0x0]\n"
+ "prfm pstl1keep, [x9, #0x0]\n"
+ "tbz %x[flags], #1, 58f\n"
+ "add x19, %x[args_ptr], %[offset_min]\n"
+ "ld1r { v1.4s }, [x19]\n"
+ "add x19, %x[args_ptr], %[offset_max]\n"
+ "ld1r { v0.4s }, [x19]\n"
+ "fmin v8.4s, v8.4s, v0.4s\n"
+ "fmin v9.4s, v9.4s, v0.4s\n"
+ "fmin v10.4s, v10.4s, v0.4s\n"
+ "fmin v11.4s, v11.4s, v0.4s\n"
+ "fmax v8.4s, v8.4s, v1.4s\n"
+ "fmax v9.4s, v9.4s, v1.4s\n"
+ "fmax v10.4s, v10.4s, v1.4s\n"
+ "fmax v11.4s, v11.4s, v1.4s\n"
+ "fmin v12.4s, v12.4s, v0.4s\n"
+ "fmin v13.4s, v13.4s, v0.4s\n"
+ "fmin v14.4s, v14.4s, v0.4s\n"
+ "fmax v12.4s, v12.4s, v1.4s\n"
+ "fmax v13.4s, v13.4s, v1.4s\n"
+ "fmax v14.4s, v14.4s, v1.4s\n"
+ "fmin v15.4s, v15.4s, v0.4s\n"
+ "fmax v15.4s, v15.4s, v1.4s\n"
+ "58:" // Height 2: No activation
+ "cmp x16, #0x10\n"
+ "bge 67f\n"
+ "tbz x16, #3, 62f\n"
+ "st1 { v8.4s }, [x13], #0x10\n"
+ "st1 { v9.4s }, [x13], #0x10\n"
+ "st1 { v12.4s }, [x9], #0x10\n"
+ "st1 { v13.4s }, [x9], #0x10\n"
+ "tbz x16, #2, 60f\n"
+ "st1 { v10.4s }, [x13], #0x10\n"
+ "st1 { v14.4s }, [x9], #0x10\n"
+ "tbz x16, #1, 59f\n"
+ "str d11, [x13], #0x8\n"
+ "str d15, [x9], #0x8\n"
+ "tbz x16, #0, 66f\n"
+ "st1 { v11.s }[2], [x13]\n"
+ "st1 { v15.s }[2], [x9]\n"
+ "b 66f\n"
+ "59:" // Height 2: Partial direct writeback: partial_1_12
+ "tbz x16, #0, 66f\n"
+ "str s11, [x13, #0x0]\n"
+ "str s15, [x9, #0x0]\n"
+ "b 66f\n"
+ "60:" // Height 2: Partial direct writeback: partial_2_8
+ "tbz x16, #1, 61f\n"
+ "str d10, [x13], #0x8\n"
+ "str d14, [x9], #0x8\n"
+ "tbz x16, #0, 66f\n"
+ "st1 { v10.s }[2], [x13]\n"
+ "st1 { v14.s }[2], [x9]\n"
+ "b 66f\n"
+ "61:" // Height 2: Partial direct writeback: partial_1_8
+ "tbz x16, #0, 66f\n"
+ "str s10, [x13, #0x0]\n"
+ "str s14, [x9, #0x0]\n"
+ "b 66f\n"
+ "62:" // Height 2: Partial direct writeback: partial_4_0
+ "tbz x16, #2, 64f\n"
+ "st1 { v8.4s }, [x13], #0x10\n"
+ "st1 { v12.4s }, [x9], #0x10\n"
+ "tbz x16, #1, 63f\n"
+ "str d9, [x13], #0x8\n"
+ "str d13, [x9], #0x8\n"
+ "tbz x16, #0, 66f\n"
+ "st1 { v9.s }[2], [x13]\n"
+ "st1 { v13.s }[2], [x9]\n"
+ "b 66f\n"
+ "63:" // Height 2: Partial direct writeback: partial_1_4
+ "tbz x16, #0, 66f\n"
+ "str s9, [x13, #0x0]\n"
+ "str s13, [x9, #0x0]\n"
+ "b 66f\n"
+ "64:" // Height 2: Partial direct writeback: partial_2_0
+ "tbz x16, #1, 65f\n"
+ "str d8, [x13], #0x8\n"
+ "str d12, [x9], #0x8\n"
+ "tbz x16, #0, 66f\n"
+ "st1 { v8.s }[2], [x13]\n"
+ "st1 { v12.s }[2], [x9]\n"
+ "b 66f\n"
+ "65:" // Height 2: Partial direct writeback: partial_1_0
+ "str s8, [x13, #0x0]\n"
+ "str s12, [x9, #0x0]\n"
+ "66:" // Height 2: Partial direct writeback: Done
+ "b 68f\n"
+ "67:" // Height 2: Full writeback
+ "str q8, [x13, #0x0]\n"
+ "str q9, [x13, #0x10]\n"
+ "str q10, [x13, #0x20]\n"
+ "str q11, [x13, #0x30]\n"
+ "str q12, [x9, #0x0]\n"
+ "str q13, [x9, #0x10]\n"
+ "str q14, [x9, #0x20]\n"
+ "str q15, [x9, #0x30]\n"
+ "add x13, x13, #0x40\n"
+ "add x9, x9, #0x40\n"
+ "68:" // Height 2: Writeback done
+ "subs x16, x16, #0x10\n"
+ "bgt 37b\n"
+ "b 206f\n"
+ "69:" // Height 3
+ "ldr x16, [%x[args_ptr], %[offsetof_N]]\n"
+ "mov x14, %x[bias]\n"
+ "ldr x15, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 70f\n"
+ "ldr x13, [%x[output_ptr], #0x0]\n"
+ "add x13, x13, x19, LSL #2\n"
+ "ldr x9, [%x[output_ptr], #0x8]\n"
+ "ldr x27, [%x[output_ptr], #0x10]\n"
+ "add x9, x9, x19, LSL #2\n"
+ "add x27, x27, x19, LSL #2\n"
+ "b 71f\n"
+ "70:" // Height 3: setup direct output
+ "mov x13, %x[output_ptr]\n"
+ "add x9, x13, x19, LSL #2\n"
+ "add x27, x9, x19, LSL #2\n"
+ "71:" // Height 3: Column loop
+ "cbz x14, 72f\n"
+ "ldr q8, [x14, #0x0]\n"
+ "mov v12.16b, v8.16b\n"
+ "ldr q9, [x14, #0x10]\n"
+ "mov v16.16b, v8.16b\n"
+ "ldr q10, [x14, #0x20]\n"
+ "ldr q11, [x14, #0x30]\n"
+ "mov v13.16b, v9.16b\n"
+ "add x14, x14, #0x40\n"
+ "mov v17.16b, v9.16b\n"
+ "mov v14.16b, v10.16b\n"
+ "mov v15.16b, v11.16b\n"
+ "mov v18.16b, v10.16b\n"
+ "mov v19.16b, v11.16b\n"
+ "b 83f\n"
+ "72:" // Height 3: no bias
+ "tbz %x[flags], #0, 82f\n"
+ "cmp x16, #0x10\n"
+ "bge 81f\n"
+ "tbz x16, #3, 76f\n"
+ "ld1 { v8.4s }, [x13], #0x10\n"
+ "ld1 { v12.4s }, [x9], #0x10\n"
+ "ld1 { v16.4s }, [x27], #0x10\n"
+ "ld1 { v9.4s }, [x13], #0x10\n"
+ "ld1 { v13.4s }, [x9], #0x10\n"
+ "ld1 { v17.4s }, [x27], #0x10\n"
+ "tbz x16, #2, 74f\n"
+ "ld1 { v10.4s }, [x13], #0x10\n"
+ "ld1 { v14.4s }, [x9], #0x10\n"
+ "ld1 { v18.4s }, [x27], #0x10\n"
+ "tbz x16, #1, 73f\n"
+ "mov x19, #0x38\n"
+ "ldr d11, [x13], #0x8\n"
+ "ldr d15, [x9], #0x8\n"
+ "ldr d19, [x27], #0x8\n"
+ "tbz x16, #0, 80f\n"
+ "ld1 { v11.s }[2], [x13]\n"
+ "ld1 { v15.s }[2], [x9]\n"
+ "ld1 { v19.s }[2], [x27]\n"
+ "b 80f\n"
+ "73:" // Height 3: Partial accumulate: partial_1_12
+ "mov x19, #0x30\n"
+ "tbz x16, #0, 80f\n"
+ "ldr s11, [x13, #0x0]\n"
+ "ldr s15, [x9, #0x0]\n"
+ "ldr s19, [x27, #0x0]\n"
+ "b 80f\n"
+ "74:" // Height 3: Partial accumulate: partial_2_8
+ "tbz x16, #1, 75f\n"
+ "ldr d10, [x13], #0x8\n"
+ "ldr d14, [x9], #0x8\n"
+ "ldr d18, [x27], #0x8\n"
+ "mov x19, #0x28\n"
+ "tbz x16, #0, 80f\n"
+ "ld1 { v10.s }[2], [x13]\n"
+ "ld1 { v14.s }[2], [x9]\n"
+ "ld1 { v18.s }[2], [x27]\n"
+ "b 80f\n"
+ "75:" // Height 3: Partial accumulate: partial_1_8
+ "mov x19, #0x20\n"
+ "tbz x16, #0, 80f\n"
+ "ldr s10, [x13, #0x0]\n"
+ "ldr s14, [x9, #0x0]\n"
+ "ldr s18, [x27, #0x0]\n"
+ "b 80f\n"
+ "76:" // Height 3: Partial accumulate: partial_4_0
+ "tbz x16, #2, 78f\n"
+ "ld1 { v8.4s }, [x13], #0x10\n"
+ "ld1 { v12.4s }, [x9], #0x10\n"
+ "ld1 { v16.4s }, [x27], #0x10\n"
+ "tbz x16, #1, 77f\n"
+ "mov x19, #0x18\n"
+ "ldr d9, [x13], #0x8\n"
+ "ldr d13, [x9], #0x8\n"
+ "ldr d17, [x27], #0x8\n"
+ "tbz x16, #0, 80f\n"
+ "ld1 { v9.s }[2], [x13]\n"
+ "ld1 { v13.s }[2], [x9]\n"
+ "ld1 { v17.s }[2], [x27]\n"
+ "b 80f\n"
+ "77:" // Height 3: Partial accumulate: partial_1_4
+ "mov x19, #0x10\n"
+ "tbz x16, #0, 80f\n"
+ "ldr s9, [x13, #0x0]\n"
+ "ldr s13, [x9, #0x0]\n"
+ "ldr s17, [x27, #0x0]\n"
+ "b 80f\n"
+ "78:" // Height 3: Partial accumulate: partial_2_0
+ "tbz x16, #1, 79f\n"
+ "ldr d8, [x13], #0x8\n"
+ "ldr d12, [x9], #0x8\n"
+ "ldr d16, [x27], #0x8\n"
+ "mov x19, #0x8\n"
+ "tbz x16, #0, 80f\n"
+ "ld1 { v8.s }[2], [x13]\n"
+ "ld1 { v12.s }[2], [x9]\n"
+ "ld1 { v16.s }[2], [x27]\n"
+ "b 80f\n"
+ "79:" // Height 3: Partial accumulate: partial_1_0
+ "mov x19, #0x0\n"
+ "ldr s8, [x13, #0x0]\n"
+ "ldr s12, [x9, #0x0]\n"
+ "ldr s16, [x27, #0x0]\n"
+ "80:" // Height 3: Partial accumulate: Done
+ "sub x13, x13, x19\n"
+ "sub x9, x9, x19\n"
+ "sub x27, x27, x19\n"
+ "b 83f\n"
+ "81:" // Height 3: full accumulate
+ "ldr q8, [x13, #0x0]\n"
+ "ldr q9, [x13, #0x10]\n"
+ "ldr q10, [x13, #0x20]\n"
+ "ldr q11, [x13, #0x30]\n"
+ "ldr q12, [x9, #0x0]\n"
+ "ldr q13, [x9, #0x10]\n"
+ "ldr q14, [x9, #0x20]\n"
+ "ldr q15, [x9, #0x30]\n"
+ "ldr q16, [x27, #0x0]\n"
+ "ldr q17, [x27, #0x10]\n"
+ "ldr q18, [x27, #0x20]\n"
+ "ldr q19, [x27, #0x30]\n"
+ "b 83f\n"
+ "82:" // Height 3: no accumulate
+ "movi v8.16b, #0x0\n"
+ "movi v9.16b, #0x0\n"
+ "movi v10.16b, #0x0\n"
+ "movi v11.16b, #0x0\n"
+ "movi v12.16b, #0x0\n"
+ "movi v13.16b, #0x0\n"
+ "movi v14.16b, #0x0\n"
+ "movi v15.16b, #0x0\n"
+ "movi v16.16b, #0x0\n"
+ "movi v17.16b, #0x0\n"
+ "movi v18.16b, #0x0\n"
+ "movi v19.16b, #0x0\n"
+ "83:" // Height 3: setup done
+ "mov x12, #0x0\n"
+ "84:" // Height 3: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w11, [x20, x12, LSL #0x2]\n"
+ "tbz %x[flags], #3, 85f\n"
+ "ldr x20, [%x[input_ptr], x12, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x10, [x20, #0x0]\n"
+ "ldr x28, [x20, #0x8]\n"
+ "ldr x26, [x20, #0x10]\n"
+ "cbnz x12, 86f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x10, x10, x19, LSL #2\n"
+ "add x28, x28, x19, LSL #2\n"
+ "add x26, x26, x19, LSL #2\n"
+ "b 86f\n"
+ "85:" // Height 3: setup direct input
+ "mov x10, %x[input_ptr]\n"
+ "add x28, x10, x19, LSL #2\n"
+ "add x26, x28, x19, LSL #2\n"
+ "86:" // Height 3: input setup done
+ "cmp x11, #0x4\n"
+ "blt 89f\n"
+ "cmp x11, #0x8\n"
+ "blt 88f\n"
+ "87:" // Height 3: Multiply loop: Main loop head
+ "ldr q0, [x10, #0x0]\n"
+ "ldr q1, [x28, #0x0]\n"
+ "ldr q2, [x26, #0x0]\n"
+ "ldr q6, [x15, #0x0]\n"
+ "fmla v8.4s, v6.4s, v0.s[0]\n"
+ "ldr q7, [x15, #0x10]\n"
+ "fmla v12.4s, v6.4s, v1.s[0]\n"
+ "add x10, x10, #0x10\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "fmla v16.4s, v6.4s, v2.s[0]\n"
+ "ldr q6, [x15, #0x20]\n"
+ "fmla v9.4s, v7.4s, v0.s[0]\n"
+ "add x28, x28, #0x10\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "fmla v13.4s, v7.4s, v1.s[0]\n"
+ "add x26, x26, #0x10\n"
+ "fmla v17.4s, v7.4s, v2.s[0]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "ldr q7, [x15, #0x30]\n"
+ "fmla v10.4s, v6.4s, v0.s[0]\n"
+ "sub x11, x11, #0x4\n"
+ "fmla v14.4s, v6.4s, v1.s[0]\n"
+ "cmp x11, #0x8\n"
+ "fmla v18.4s, v6.4s, v2.s[0]\n"
+ "ldr q6, [x15, #0x40]\n"
+ "fmla v11.4s, v7.4s, v0.s[0]\n"
+ "fmla v15.4s, v7.4s, v1.s[0]\n"
+ "fmla v19.4s, v7.4s, v2.s[0]\n"
+ "ldr q7, [x15, #0x50]\n"
+ "fmla v8.4s, v6.4s, v0.s[1]\n"
+ "fmla v12.4s, v6.4s, v1.s[1]\n"
+ "fmla v16.4s, v6.4s, v2.s[1]\n"
+ "ldr q6, [x15, #0x60]\n"
+ "fmla v9.4s, v7.4s, v0.s[1]\n"
+ "fmla v13.4s, v7.4s, v1.s[1]\n"
+ "fmla v17.4s, v7.4s, v2.s[1]\n"
+ "ldr q7, [x15, #0x70]\n"
+ "fmla v10.4s, v6.4s, v0.s[1]\n"
+ "fmla v14.4s, v6.4s, v1.s[1]\n"
+ "fmla v18.4s, v6.4s, v2.s[1]\n"
+ "ldr q6, [x15, #0x80]\n"
+ "fmla v11.4s, v7.4s, v0.s[1]\n"
+ "fmla v15.4s, v7.4s, v1.s[1]\n"
+ "fmla v19.4s, v7.4s, v2.s[1]\n"
+ "ldr q7, [x15, #0x90]\n"
+ "fmla v8.4s, v6.4s, v0.s[2]\n"
+ "fmla v12.4s, v6.4s, v1.s[2]\n"
+ "fmla v16.4s, v6.4s, v2.s[2]\n"
+ "ldr q6, [x15, #0xa0]\n"
+ "fmla v9.4s, v7.4s, v0.s[2]\n"
+ "fmla v13.4s, v7.4s, v1.s[2]\n"
+ "fmla v17.4s, v7.4s, v2.s[2]\n"
+ "ldr q7, [x15, #0xb0]\n"
+ "fmla v10.4s, v6.4s, v0.s[2]\n"
+ "fmla v14.4s, v6.4s, v1.s[2]\n"
+ "fmla v18.4s, v6.4s, v2.s[2]\n"
+ "ldr q6, [x15, #0xc0]\n"
+ "fmla v11.4s, v7.4s, v0.s[2]\n"
+ "fmla v15.4s, v7.4s, v1.s[2]\n"
+ "fmla v19.4s, v7.4s, v2.s[2]\n"
+ "ldr q7, [x15, #0xd0]\n"
+ "fmla v8.4s, v6.4s, v0.s[3]\n"
+ "fmla v12.4s, v6.4s, v1.s[3]\n"
+ "fmla v16.4s, v6.4s, v2.s[3]\n"
+ "ldr q6, [x15, #0xe0]\n"
+ "fmla v9.4s, v7.4s, v0.s[3]\n"
+ "fmla v13.4s, v7.4s, v1.s[3]\n"
+ "fmla v17.4s, v7.4s, v2.s[3]\n"
+ "ldr q7, [x15, #0xf0]\n"
+ "add x15, x15, #0x100\n"
+ "fmla v10.4s, v6.4s, v0.s[3]\n"
+ "fmla v14.4s, v6.4s, v1.s[3]\n"
+ "fmla v18.4s, v6.4s, v2.s[3]\n"
+ "fmla v11.4s, v7.4s, v0.s[3]\n"
+ "fmla v15.4s, v7.4s, v1.s[3]\n"
+ "fmla v19.4s, v7.4s, v2.s[3]\n"
+ "bge 87b\n"
+ "88:" // Height 3: Multiply loop: Single iteration only
+ "sub x11, x11, #0x4\n"
+ "ldr q0, [x10, #0x0]\n"
+ "ldr q1, [x28, #0x0]\n"
+ "ldr q2, [x26, #0x0]\n"
+ "ldr q6, [x15, #0x0]\n"
+ "fmla v8.4s, v6.4s, v0.s[0]\n"
+ "ldr q7, [x15, #0x10]\n"
+ "fmla v12.4s, v6.4s, v1.s[0]\n"
+ "add x10, x10, #0x10\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "fmla v16.4s, v6.4s, v2.s[0]\n"
+ "ldr q6, [x15, #0x20]\n"
+ "fmla v9.4s, v7.4s, v0.s[0]\n"
+ "add x28, x28, #0x10\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "fmla v13.4s, v7.4s, v1.s[0]\n"
+ "add x26, x26, #0x10\n"
+ "fmla v17.4s, v7.4s, v2.s[0]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "ldr q7, [x15, #0x30]\n"
+ "fmla v10.4s, v6.4s, v0.s[0]\n"
+ "fmla v14.4s, v6.4s, v1.s[0]\n"
+ "fmla v18.4s, v6.4s, v2.s[0]\n"
+ "ldr q6, [x15, #0x40]\n"
+ "fmla v11.4s, v7.4s, v0.s[0]\n"
+ "fmla v15.4s, v7.4s, v1.s[0]\n"
+ "fmla v19.4s, v7.4s, v2.s[0]\n"
+ "ldr q7, [x15, #0x50]\n"
+ "fmla v8.4s, v6.4s, v0.s[1]\n"
+ "fmla v12.4s, v6.4s, v1.s[1]\n"
+ "fmla v16.4s, v6.4s, v2.s[1]\n"
+ "ldr q6, [x15, #0x60]\n"
+ "fmla v9.4s, v7.4s, v0.s[1]\n"
+ "fmla v13.4s, v7.4s, v1.s[1]\n"
+ "fmla v17.4s, v7.4s, v2.s[1]\n"
+ "ldr q7, [x15, #0x70]\n"
+ "fmla v10.4s, v6.4s, v0.s[1]\n"
+ "fmla v14.4s, v6.4s, v1.s[1]\n"
+ "fmla v18.4s, v6.4s, v2.s[1]\n"
+ "ldr q6, [x15, #0x80]\n"
+ "fmla v11.4s, v7.4s, v0.s[1]\n"
+ "fmla v15.4s, v7.4s, v1.s[1]\n"
+ "fmla v19.4s, v7.4s, v2.s[1]\n"
+ "ldr q7, [x15, #0x90]\n"
+ "fmla v8.4s, v6.4s, v0.s[2]\n"
+ "fmla v12.4s, v6.4s, v1.s[2]\n"
+ "fmla v16.4s, v6.4s, v2.s[2]\n"
+ "ldr q6, [x15, #0xa0]\n"
+ "fmla v9.4s, v7.4s, v0.s[2]\n"
+ "fmla v13.4s, v7.4s, v1.s[2]\n"
+ "fmla v17.4s, v7.4s, v2.s[2]\n"
+ "ldr q7, [x15, #0xb0]\n"
+ "fmla v10.4s, v6.4s, v0.s[2]\n"
+ "fmla v14.4s, v6.4s, v1.s[2]\n"
+ "fmla v18.4s, v6.4s, v2.s[2]\n"
+ "ldr q6, [x15, #0xc0]\n"
+ "fmla v11.4s, v7.4s, v0.s[2]\n"
+ "fmla v15.4s, v7.4s, v1.s[2]\n"
+ "fmla v19.4s, v7.4s, v2.s[2]\n"
+ "ldr q7, [x15, #0xd0]\n"
+ "fmla v8.4s, v6.4s, v0.s[3]\n"
+ "fmla v12.4s, v6.4s, v1.s[3]\n"
+ "fmla v16.4s, v6.4s, v2.s[3]\n"
+ "ldr q6, [x15, #0xe0]\n"
+ "fmla v9.4s, v7.4s, v0.s[3]\n"
+ "fmla v13.4s, v7.4s, v1.s[3]\n"
+ "fmla v17.4s, v7.4s, v2.s[3]\n"
+ "ldr q7, [x15, #0xf0]\n"
+ "add x15, x15, #0x100\n"
+ "fmla v10.4s, v6.4s, v0.s[3]\n"
+ "fmla v14.4s, v6.4s, v1.s[3]\n"
+ "fmla v18.4s, v6.4s, v2.s[3]\n"
+ "fmla v11.4s, v7.4s, v0.s[3]\n"
+ "fmla v15.4s, v7.4s, v1.s[3]\n"
+ "fmla v19.4s, v7.4s, v2.s[3]\n"
+ "89:" // Height 3: Multiply loop: Main loop skip
+ "cbz x11, 91f\n"
+ "90:" // Height 3: Multiply loop: Odd block loop
+ "ldr s0, [x10], #0x4\n"
+ "ldr s1, [x28], #0x4\n"
+ "ldr s2, [x26], #0x4\n"
+ "ldr q6, [x15, #0x0]\n"
+ "fmla v8.4s, v6.4s, v0.s[0]\n"
+ "ldr q7, [x15, #0x10]\n"
+ "fmla v12.4s, v6.4s, v1.s[0]\n"
+ "sub x11, x11, #0x1\n"
+ "fmla v16.4s, v6.4s, v2.s[0]\n"
+ "ldr q6, [x15, #0x20]\n"
+ "fmla v9.4s, v7.4s, v0.s[0]\n"
+ "fmla v13.4s, v7.4s, v1.s[0]\n"
+ "fmla v17.4s, v7.4s, v2.s[0]\n"
+ "ldr q7, [x15, #0x30]\n"
+ "fmla v10.4s, v6.4s, v0.s[0]\n"
+ "add x15, x15, #0x40\n"
+ "fmla v14.4s, v6.4s, v1.s[0]\n"
+ "fmla v18.4s, v6.4s, v2.s[0]\n"
+ "fmla v11.4s, v7.4s, v0.s[0]\n"
+ "fmla v15.4s, v7.4s, v1.s[0]\n"
+ "fmla v19.4s, v7.4s, v2.s[0]\n"
+ "cbnz x11, 90b\n"
+ "91:" // Height 3: Multiply loop: No odd multiplies
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "add x12, x12, #0x1\n"
+ "cmp x12, x19\n"
+ "bne 84b\n"
+ "prfm pstl1keep, [x13, #0x0]\n"
+ "prfm pstl1keep, [x9, #0x0]\n"
+ "prfm pstl1keep, [x27, #0x0]\n"
+ "tbz %x[flags], #1, 92f\n"
+ "add x19, %x[args_ptr], %[offset_min]\n"
+ "ld1r { v1.4s }, [x19]\n"
+ "add x19, %x[args_ptr], %[offset_max]\n"
+ "ld1r { v0.4s }, [x19]\n"
+ "fmin v8.4s, v8.4s, v0.4s\n"
+ "fmin v9.4s, v9.4s, v0.4s\n"
+ "fmin v10.4s, v10.4s, v0.4s\n"
+ "fmin v11.4s, v11.4s, v0.4s\n"
+ "fmax v8.4s, v8.4s, v1.4s\n"
+ "fmax v9.4s, v9.4s, v1.4s\n"
+ "fmax v10.4s, v10.4s, v1.4s\n"
+ "fmax v11.4s, v11.4s, v1.4s\n"
+ "fmin v12.4s, v12.4s, v0.4s\n"
+ "fmin v13.4s, v13.4s, v0.4s\n"
+ "fmin v14.4s, v14.4s, v0.4s\n"
+ "fmax v12.4s, v12.4s, v1.4s\n"
+ "fmax v13.4s, v13.4s, v1.4s\n"
+ "fmax v14.4s, v14.4s, v1.4s\n"
+ "fmin v15.4s, v15.4s, v0.4s\n"
+ "fmin v16.4s, v16.4s, v0.4s\n"
+ "fmin v17.4s, v17.4s, v0.4s\n"
+ "fmax v15.4s, v15.4s, v1.4s\n"
+ "fmax v16.4s, v16.4s, v1.4s\n"
+ "fmax v17.4s, v17.4s, v1.4s\n"
+ "fmin v18.4s, v18.4s, v0.4s\n"
+ "fmin v19.4s, v19.4s, v0.4s\n"
+ "fmax v18.4s, v18.4s, v1.4s\n"
+ "fmax v19.4s, v19.4s, v1.4s\n"
+ "92:" // Height 3: No activation
+ "cmp x16, #0x10\n"
+ "bge 101f\n"
+ "tbz x16, #3, 96f\n"
+ "st1 { v8.4s }, [x13], #0x10\n"
+ "st1 { v9.4s }, [x13], #0x10\n"
+ "st1 { v12.4s }, [x9], #0x10\n"
+ "st1 { v13.4s }, [x9], #0x10\n"
+ "st1 { v16.4s }, [x27], #0x10\n"
+ "st1 { v17.4s }, [x27], #0x10\n"
+ "tbz x16, #2, 94f\n"
+ "st1 { v10.4s }, [x13], #0x10\n"
+ "st1 { v14.4s }, [x9], #0x10\n"
+ "st1 { v18.4s }, [x27], #0x10\n"
+ "tbz x16, #1, 93f\n"
+ "str d11, [x13], #0x8\n"
+ "str d15, [x9], #0x8\n"
+ "str d19, [x27], #0x8\n"
+ "tbz x16, #0, 100f\n"
+ "st1 { v11.s }[2], [x13]\n"
+ "st1 { v15.s }[2], [x9]\n"
+ "st1 { v19.s }[2], [x27]\n"
+ "b 100f\n"
+ "93:" // Height 3: Partial direct writeback: partial_1_12
+ "tbz x16, #0, 100f\n"
+ "str s11, [x13, #0x0]\n"
+ "str s15, [x9, #0x0]\n"
+ "str s19, [x27, #0x0]\n"
+ "b 100f\n"
+ "94:" // Height 3: Partial direct writeback: partial_2_8
+ "tbz x16, #1, 95f\n"
+ "str d10, [x13], #0x8\n"
+ "str d14, [x9], #0x8\n"
+ "str d18, [x27], #0x8\n"
+ "tbz x16, #0, 100f\n"
+ "st1 { v10.s }[2], [x13]\n"
+ "st1 { v14.s }[2], [x9]\n"
+ "st1 { v18.s }[2], [x27]\n"
+ "b 100f\n"
+ "95:" // Height 3: Partial direct writeback: partial_1_8
+ "tbz x16, #0, 100f\n"
+ "str s10, [x13, #0x0]\n"
+ "str s14, [x9, #0x0]\n"
+ "str s18, [x27, #0x0]\n"
+ "b 100f\n"
+ "96:" // Height 3: Partial direct writeback: partial_4_0
+ "tbz x16, #2, 98f\n"
+ "st1 { v8.4s }, [x13], #0x10\n"
+ "st1 { v12.4s }, [x9], #0x10\n"
+ "st1 { v16.4s }, [x27], #0x10\n"
+ "tbz x16, #1, 97f\n"
+ "str d9, [x13], #0x8\n"
+ "str d13, [x9], #0x8\n"
+ "str d17, [x27], #0x8\n"
+ "tbz x16, #0, 100f\n"
+ "st1 { v9.s }[2], [x13]\n"
+ "st1 { v13.s }[2], [x9]\n"
+ "st1 { v17.s }[2], [x27]\n"
+ "b 100f\n"
+ "97:" // Height 3: Partial direct writeback: partial_1_4
+ "tbz x16, #0, 100f\n"
+ "str s9, [x13, #0x0]\n"
+ "str s13, [x9, #0x0]\n"
+ "str s17, [x27, #0x0]\n"
+ "b 100f\n"
+ "98:" // Height 3: Partial direct writeback: partial_2_0
+ "tbz x16, #1, 99f\n"
+ "str d8, [x13], #0x8\n"
+ "str d12, [x9], #0x8\n"
+ "str d16, [x27], #0x8\n"
+ "tbz x16, #0, 100f\n"
+ "st1 { v8.s }[2], [x13]\n"
+ "st1 { v12.s }[2], [x9]\n"
+ "st1 { v16.s }[2], [x27]\n"
+ "b 100f\n"
+ "99:" // Height 3: Partial direct writeback: partial_1_0
+ "str s8, [x13, #0x0]\n"
+ "str s12, [x9, #0x0]\n"
+ "str s16, [x27, #0x0]\n"
+ "100:" // Height 3: Partial direct writeback: Done
+ "b 102f\n"
+ "101:" // Height 3: Full writeback
+ "str q8, [x13, #0x0]\n"
+ "str q9, [x13, #0x10]\n"
+ "str q10, [x13, #0x20]\n"
+ "str q11, [x13, #0x30]\n"
+ "str q12, [x9, #0x0]\n"
+ "str q13, [x9, #0x10]\n"
+ "str q14, [x9, #0x20]\n"
+ "str q15, [x9, #0x30]\n"
+ "str q16, [x27, #0x0]\n"
+ "str q17, [x27, #0x10]\n"
+ "str q18, [x27, #0x20]\n"
+ "str q19, [x27, #0x30]\n"
+ "add x13, x13, #0x40\n"
+ "add x9, x9, #0x40\n"
+ "add x27, x27, #0x40\n"
+ "102:" // Height 3: Writeback done
+ "subs x16, x16, #0x10\n"
+ "bgt 71b\n"
+ "b 206f\n"
+ "103:" // Height 4
+ "ldr x16, [%x[args_ptr], %[offsetof_N]]\n"
+ "mov x14, %x[bias]\n"
+ "ldr x15, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 104f\n"
+ "ldr x13, [%x[output_ptr], #0x0]\n"
+ "add x13, x13, x19, LSL #2\n"
+ "ldr x9, [%x[output_ptr], #0x8]\n"
+ "ldr x27, [%x[output_ptr], #0x10]\n"
+ "add x9, x9, x19, LSL #2\n"
+ "ldr x25, [%x[output_ptr], #0x18]\n"
+ "add x27, x27, x19, LSL #2\n"
+ "add x25, x25, x19, LSL #2\n"
+ "b 105f\n"
+ "104:" // Height 4: setup direct output
+ "mov x13, %x[output_ptr]\n"
+ "add x9, x13, x19, LSL #2\n"
+ "add x27, x9, x19, LSL #2\n"
+ "add x25, x27, x19, LSL #2\n"
+ "105:" // Height 4: Column loop
+ "cbz x14, 106f\n"
+ "ldr q8, [x14, #0x0]\n"
+ "mov v12.16b, v8.16b\n"
+ "ldr q9, [x14, #0x10]\n"
+ "mov v16.16b, v8.16b\n"
+ "ldr q10, [x14, #0x20]\n"
+ "mov v20.16b, v8.16b\n"
+ "ldr q11, [x14, #0x30]\n"
+ "add x14, x14, #0x40\n"
+ "mov v13.16b, v9.16b\n"
+ "mov v17.16b, v9.16b\n"
+ "mov v14.16b, v10.16b\n"
+ "mov v15.16b, v11.16b\n"
+ "mov v18.16b, v10.16b\n"
+ "mov v19.16b, v11.16b\n"
+ "mov v21.16b, v9.16b\n"
+ "mov v22.16b, v10.16b\n"
+ "mov v23.16b, v11.16b\n"
+ "b 117f\n"
+ "106:" // Height 4: no bias
+ "tbz %x[flags], #0, 116f\n"
+ "cmp x16, #0x10\n"
+ "bge 115f\n"
+ "tbz x16, #3, 110f\n"
+ "ld1 { v8.4s }, [x13], #0x10\n"
+ "ld1 { v12.4s }, [x9], #0x10\n"
+ "ld1 { v16.4s }, [x27], #0x10\n"
+ "ld1 { v20.4s }, [x25], #0x10\n"
+ "ld1 { v9.4s }, [x13], #0x10\n"
+ "ld1 { v13.4s }, [x9], #0x10\n"
+ "ld1 { v17.4s }, [x27], #0x10\n"
+ "ld1 { v21.4s }, [x25], #0x10\n"
+ "tbz x16, #2, 108f\n"
+ "ld1 { v10.4s }, [x13], #0x10\n"
+ "ld1 { v14.4s }, [x9], #0x10\n"
+ "ld1 { v18.4s }, [x27], #0x10\n"
+ "ld1 { v22.4s }, [x25], #0x10\n"
+ "tbz x16, #1, 107f\n"
+ "mov x19, #0x38\n"
+ "ldr d11, [x13], #0x8\n"
+ "ldr d15, [x9], #0x8\n"
+ "ldr d19, [x27], #0x8\n"
+ "ldr d23, [x25], #0x8\n"
+ "tbz x16, #0, 114f\n"
+ "ld1 { v11.s }[2], [x13]\n"
+ "ld1 { v15.s }[2], [x9]\n"
+ "ld1 { v19.s }[2], [x27]\n"
+ "ld1 { v23.s }[2], [x25]\n"
+ "b 114f\n"
+ "107:" // Height 4: Partial accumulate: partial_1_12
+ "mov x19, #0x30\n"
+ "tbz x16, #0, 114f\n"
+ "ldr s11, [x13, #0x0]\n"
+ "ldr s15, [x9, #0x0]\n"
+ "ldr s19, [x27, #0x0]\n"
+ "ldr s23, [x25, #0x0]\n"
+ "b 114f\n"
+ "108:" // Height 4: Partial accumulate: partial_2_8
+ "tbz x16, #1, 109f\n"
+ "ldr d10, [x13], #0x8\n"
+ "ldr d14, [x9], #0x8\n"
+ "ldr d18, [x27], #0x8\n"
+ "ldr d22, [x25], #0x8\n"
+ "mov x19, #0x28\n"
+ "tbz x16, #0, 114f\n"
+ "ld1 { v10.s }[2], [x13]\n"
+ "ld1 { v14.s }[2], [x9]\n"
+ "ld1 { v18.s }[2], [x27]\n"
+ "ld1 { v22.s }[2], [x25]\n"
+ "b 114f\n"
+ "109:" // Height 4: Partial accumulate: partial_1_8
+ "mov x19, #0x20\n"
+ "tbz x16, #0, 114f\n"
+ "ldr s10, [x13, #0x0]\n"
+ "ldr s14, [x9, #0x0]\n"
+ "ldr s18, [x27, #0x0]\n"
+ "ldr s22, [x25, #0x0]\n"
+ "b 114f\n"
+ "110:" // Height 4: Partial accumulate: partial_4_0
+ "tbz x16, #2, 112f\n"
+ "ld1 { v8.4s }, [x13], #0x10\n"
+ "ld1 { v12.4s }, [x9], #0x10\n"
+ "ld1 { v16.4s }, [x27], #0x10\n"
+ "ld1 { v20.4s }, [x25], #0x10\n"
+ "tbz x16, #1, 111f\n"
+ "mov x19, #0x18\n"
+ "ldr d9, [x13], #0x8\n"
+ "ldr d13, [x9], #0x8\n"
+ "ldr d17, [x27], #0x8\n"
+ "ldr d21, [x25], #0x8\n"
+ "tbz x16, #0, 114f\n"
+ "ld1 { v9.s }[2], [x13]\n"
+ "ld1 { v13.s }[2], [x9]\n"
+ "ld1 { v17.s }[2], [x27]\n"
+ "ld1 { v21.s }[2], [x25]\n"
+ "b 114f\n"
+ "111:" // Height 4: Partial accumulate: partial_1_4
+ "mov x19, #0x10\n"
+ "tbz x16, #0, 114f\n"
+ "ldr s9, [x13, #0x0]\n"
+ "ldr s13, [x9, #0x0]\n"
+ "ldr s17, [x27, #0x0]\n"
+ "ldr s21, [x25, #0x0]\n"
+ "b 114f\n"
+ "112:" // Height 4: Partial accumulate: partial_2_0
+ "tbz x16, #1, 113f\n"
+ "ldr d8, [x13], #0x8\n"
+ "ldr d12, [x9], #0x8\n"
+ "ldr d16, [x27], #0x8\n"
+ "ldr d20, [x25], #0x8\n"
+ "mov x19, #0x8\n"
+ "tbz x16, #0, 114f\n"
+ "ld1 { v8.s }[2], [x13]\n"
+ "ld1 { v12.s }[2], [x9]\n"
+ "ld1 { v16.s }[2], [x27]\n"
+ "ld1 { v20.s }[2], [x25]\n"
+ "b 114f\n"
+ "113:" // Height 4: Partial accumulate: partial_1_0
+ "mov x19, #0x0\n"
+ "ldr s8, [x13, #0x0]\n"
+ "ldr s12, [x9, #0x0]\n"
+ "ldr s16, [x27, #0x0]\n"
+ "ldr s20, [x25, #0x0]\n"
+ "114:" // Height 4: Partial accumulate: Done
+ "sub x13, x13, x19\n"
+ "sub x9, x9, x19\n"
+ "sub x27, x27, x19\n"
+ "sub x25, x25, x19\n"
+ "b 117f\n"
+ "115:" // Height 4: full accumulate
+ "ldr q8, [x13, #0x0]\n"
+ "ldr q9, [x13, #0x10]\n"
+ "ldr q10, [x13, #0x20]\n"
+ "ldr q11, [x13, #0x30]\n"
+ "ldr q12, [x9, #0x0]\n"
+ "ldr q13, [x9, #0x10]\n"
+ "ldr q14, [x9, #0x20]\n"
+ "ldr q15, [x9, #0x30]\n"
+ "ldr q16, [x27, #0x0]\n"
+ "ldr q17, [x27, #0x10]\n"
+ "ldr q18, [x27, #0x20]\n"
+ "ldr q19, [x27, #0x30]\n"
+ "ldr q20, [x25, #0x0]\n"
+ "ldr q21, [x25, #0x10]\n"
+ "ldr q22, [x25, #0x20]\n"
+ "ldr q23, [x25, #0x30]\n"
+ "b 117f\n"
+ "116:" // Height 4: no accumulate
+ "movi v8.16b, #0x0\n"
+ "movi v9.16b, #0x0\n"
+ "movi v10.16b, #0x0\n"
+ "movi v11.16b, #0x0\n"
+ "movi v12.16b, #0x0\n"
+ "movi v13.16b, #0x0\n"
+ "movi v14.16b, #0x0\n"
+ "movi v15.16b, #0x0\n"
+ "movi v16.16b, #0x0\n"
+ "movi v17.16b, #0x0\n"
+ "movi v18.16b, #0x0\n"
+ "movi v19.16b, #0x0\n"
+ "movi v20.16b, #0x0\n"
+ "movi v21.16b, #0x0\n"
+ "movi v22.16b, #0x0\n"
+ "movi v23.16b, #0x0\n"
+ "117:" // Height 4: setup done
+ "mov x12, #0x0\n"
+ "118:" // Height 4: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w11, [x20, x12, LSL #0x2]\n"
+ "tbz %x[flags], #3, 119f\n"
+ "ldr x20, [%x[input_ptr], x12, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x10, [x20, #0x0]\n"
+ "ldr x28, [x20, #0x8]\n"
+ "ldr x26, [x20, #0x10]\n"
+ "ldr x24, [x20, #0x18]\n"
+ "cbnz x12, 120f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x10, x10, x19, LSL #2\n"
+ "add x28, x28, x19, LSL #2\n"
+ "add x26, x26, x19, LSL #2\n"
+ "add x24, x24, x19, LSL #2\n"
+ "b 120f\n"
+ "119:" // Height 4: setup direct input
+ "mov x10, %x[input_ptr]\n"
+ "add x28, x10, x19, LSL #2\n"
+ "add x26, x28, x19, LSL #2\n"
+ "add x24, x26, x19, LSL #2\n"
+ "120:" // Height 4: input setup done
+ "cmp x11, #0x4\n"
+ "blt 123f\n"
+ "cmp x11, #0x8\n"
+ "blt 122f\n"
+ "121:" // Height 4: Multiply loop: Main loop head
+ "ldr q0, [x10, #0x0]\n"
+ "ldr q1, [x28, #0x0]\n"
+ "ldr q2, [x26, #0x0]\n"
+ "ldr q3, [x24, #0x0]\n"
+ "ldr q6, [x15, #0x0]\n"
+ "fmla v8.4s, v6.4s, v0.s[0]\n"
+ "ldr q7, [x15, #0x10]\n"
+ "fmla v12.4s, v6.4s, v1.s[0]\n"
+ "add x10, x10, #0x10\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "fmla v16.4s, v6.4s, v2.s[0]\n"
+ "add x28, x28, #0x10\n"
+ "fmla v20.4s, v6.4s, v3.s[0]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "ldr q6, [x15, #0x20]\n"
+ "fmla v9.4s, v7.4s, v0.s[0]\n"
+ "add x26, x26, #0x10\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "fmla v13.4s, v7.4s, v1.s[0]\n"
+ "add x24, x24, #0x10\n"
+ "fmla v17.4s, v7.4s, v2.s[0]\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "sub x11, x11, #0x4\n"
+ "fmla v21.4s, v7.4s, v3.s[0]\n"
+ "ldr q7, [x15, #0x30]\n"
+ "fmla v10.4s, v6.4s, v0.s[0]\n"
+ "cmp x11, #0x8\n"
+ "fmla v14.4s, v6.4s, v1.s[0]\n"
+ "fmla v18.4s, v6.4s, v2.s[0]\n"
+ "fmla v22.4s, v6.4s, v3.s[0]\n"
+ "ldr q6, [x15, #0x40]\n"
+ "fmla v11.4s, v7.4s, v0.s[0]\n"
+ "fmla v15.4s, v7.4s, v1.s[0]\n"
+ "fmla v19.4s, v7.4s, v2.s[0]\n"
+ "fmla v23.4s, v7.4s, v3.s[0]\n"
+ "ldr q7, [x15, #0x50]\n"
+ "fmla v8.4s, v6.4s, v0.s[1]\n"
+ "fmla v12.4s, v6.4s, v1.s[1]\n"
+ "fmla v16.4s, v6.4s, v2.s[1]\n"
+ "fmla v20.4s, v6.4s, v3.s[1]\n"
+ "ldr q6, [x15, #0x60]\n"
+ "fmla v9.4s, v7.4s, v0.s[1]\n"
+ "fmla v13.4s, v7.4s, v1.s[1]\n"
+ "fmla v17.4s, v7.4s, v2.s[1]\n"
+ "fmla v21.4s, v7.4s, v3.s[1]\n"
+ "ldr q7, [x15, #0x70]\n"
+ "fmla v10.4s, v6.4s, v0.s[1]\n"
+ "fmla v14.4s, v6.4s, v1.s[1]\n"
+ "fmla v18.4s, v6.4s, v2.s[1]\n"
+ "fmla v22.4s, v6.4s, v3.s[1]\n"
+ "ldr q6, [x15, #0x80]\n"
+ "fmla v11.4s, v7.4s, v0.s[1]\n"
+ "fmla v15.4s, v7.4s, v1.s[1]\n"
+ "fmla v19.4s, v7.4s, v2.s[1]\n"
+ "fmla v23.4s, v7.4s, v3.s[1]\n"
+ "ldr q7, [x15, #0x90]\n"
+ "fmla v8.4s, v6.4s, v0.s[2]\n"
+ "fmla v12.4s, v6.4s, v1.s[2]\n"
+ "fmla v16.4s, v6.4s, v2.s[2]\n"
+ "fmla v20.4s, v6.4s, v3.s[2]\n"
+ "ldr q6, [x15, #0xa0]\n"
+ "fmla v9.4s, v7.4s, v0.s[2]\n"
+ "fmla v13.4s, v7.4s, v1.s[2]\n"
+ "fmla v17.4s, v7.4s, v2.s[2]\n"
+ "fmla v21.4s, v7.4s, v3.s[2]\n"
+ "ldr q7, [x15, #0xb0]\n"
+ "fmla v10.4s, v6.4s, v0.s[2]\n"
+ "fmla v14.4s, v6.4s, v1.s[2]\n"
+ "fmla v18.4s, v6.4s, v2.s[2]\n"
+ "fmla v22.4s, v6.4s, v3.s[2]\n"
+ "ldr q6, [x15, #0xc0]\n"
+ "fmla v11.4s, v7.4s, v0.s[2]\n"
+ "fmla v15.4s, v7.4s, v1.s[2]\n"
+ "fmla v19.4s, v7.4s, v2.s[2]\n"
+ "fmla v23.4s, v7.4s, v3.s[2]\n"
+ "ldr q7, [x15, #0xd0]\n"
+ "fmla v8.4s, v6.4s, v0.s[3]\n"
+ "fmla v12.4s, v6.4s, v1.s[3]\n"
+ "fmla v16.4s, v6.4s, v2.s[3]\n"
+ "fmla v20.4s, v6.4s, v3.s[3]\n"
+ "ldr q6, [x15, #0xe0]\n"
+ "fmla v9.4s, v7.4s, v0.s[3]\n"
+ "fmla v13.4s, v7.4s, v1.s[3]\n"
+ "fmla v17.4s, v7.4s, v2.s[3]\n"
+ "fmla v21.4s, v7.4s, v3.s[3]\n"
+ "ldr q7, [x15, #0xf0]\n"
+ "add x15, x15, #0x100\n"
+ "fmla v10.4s, v6.4s, v0.s[3]\n"
+ "fmla v14.4s, v6.4s, v1.s[3]\n"
+ "fmla v18.4s, v6.4s, v2.s[3]\n"
+ "fmla v22.4s, v6.4s, v3.s[3]\n"
+ "fmla v11.4s, v7.4s, v0.s[3]\n"
+ "fmla v15.4s, v7.4s, v1.s[3]\n"
+ "fmla v19.4s, v7.4s, v2.s[3]\n"
+ "fmla v23.4s, v7.4s, v3.s[3]\n"
+ "bge 121b\n"
+ "122:" // Height 4: Multiply loop: Single iteration only
+ "sub x11, x11, #0x4\n"
+ "ldr q0, [x10, #0x0]\n"
+ "ldr q1, [x28, #0x0]\n"
+ "ldr q2, [x26, #0x0]\n"
+ "ldr q3, [x24, #0x0]\n"
+ "ldr q6, [x15, #0x0]\n"
+ "fmla v8.4s, v6.4s, v0.s[0]\n"
+ "ldr q7, [x15, #0x10]\n"
+ "fmla v12.4s, v6.4s, v1.s[0]\n"
+ "add x10, x10, #0x10\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "fmla v16.4s, v6.4s, v2.s[0]\n"
+ "add x28, x28, #0x10\n"
+ "fmla v20.4s, v6.4s, v3.s[0]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "ldr q6, [x15, #0x20]\n"
+ "fmla v9.4s, v7.4s, v0.s[0]\n"
+ "add x26, x26, #0x10\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "fmla v13.4s, v7.4s, v1.s[0]\n"
+ "add x24, x24, #0x10\n"
+ "fmla v17.4s, v7.4s, v2.s[0]\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "fmla v21.4s, v7.4s, v3.s[0]\n"
+ "ldr q7, [x15, #0x30]\n"
+ "fmla v10.4s, v6.4s, v0.s[0]\n"
+ "fmla v14.4s, v6.4s, v1.s[0]\n"
+ "fmla v18.4s, v6.4s, v2.s[0]\n"
+ "fmla v22.4s, v6.4s, v3.s[0]\n"
+ "ldr q6, [x15, #0x40]\n"
+ "fmla v11.4s, v7.4s, v0.s[0]\n"
+ "fmla v15.4s, v7.4s, v1.s[0]\n"
+ "fmla v19.4s, v7.4s, v2.s[0]\n"
+ "fmla v23.4s, v7.4s, v3.s[0]\n"
+ "ldr q7, [x15, #0x50]\n"
+ "fmla v8.4s, v6.4s, v0.s[1]\n"
+ "fmla v12.4s, v6.4s, v1.s[1]\n"
+ "fmla v16.4s, v6.4s, v2.s[1]\n"
+ "fmla v20.4s, v6.4s, v3.s[1]\n"
+ "ldr q6, [x15, #0x60]\n"
+ "fmla v9.4s, v7.4s, v0.s[1]\n"
+ "fmla v13.4s, v7.4s, v1.s[1]\n"
+ "fmla v17.4s, v7.4s, v2.s[1]\n"
+ "fmla v21.4s, v7.4s, v3.s[1]\n"
+ "ldr q7, [x15, #0x70]\n"
+ "fmla v10.4s, v6.4s, v0.s[1]\n"
+ "fmla v14.4s, v6.4s, v1.s[1]\n"
+ "fmla v18.4s, v6.4s, v2.s[1]\n"
+ "fmla v22.4s, v6.4s, v3.s[1]\n"
+ "ldr q6, [x15, #0x80]\n"
+ "fmla v11.4s, v7.4s, v0.s[1]\n"
+ "fmla v15.4s, v7.4s, v1.s[1]\n"
+ "fmla v19.4s, v7.4s, v2.s[1]\n"
+ "fmla v23.4s, v7.4s, v3.s[1]\n"
+ "ldr q7, [x15, #0x90]\n"
+ "fmla v8.4s, v6.4s, v0.s[2]\n"
+ "fmla v12.4s, v6.4s, v1.s[2]\n"
+ "fmla v16.4s, v6.4s, v2.s[2]\n"
+ "fmla v20.4s, v6.4s, v3.s[2]\n"
+ "ldr q6, [x15, #0xa0]\n"
+ "fmla v9.4s, v7.4s, v0.s[2]\n"
+ "fmla v13.4s, v7.4s, v1.s[2]\n"
+ "fmla v17.4s, v7.4s, v2.s[2]\n"
+ "fmla v21.4s, v7.4s, v3.s[2]\n"
+ "ldr q7, [x15, #0xb0]\n"
+ "fmla v10.4s, v6.4s, v0.s[2]\n"
+ "fmla v14.4s, v6.4s, v1.s[2]\n"
+ "fmla v18.4s, v6.4s, v2.s[2]\n"
+ "fmla v22.4s, v6.4s, v3.s[2]\n"
+ "ldr q6, [x15, #0xc0]\n"
+ "fmla v11.4s, v7.4s, v0.s[2]\n"
+ "fmla v15.4s, v7.4s, v1.s[2]\n"
+ "fmla v19.4s, v7.4s, v2.s[2]\n"
+ "fmla v23.4s, v7.4s, v3.s[2]\n"
+ "ldr q7, [x15, #0xd0]\n"
+ "fmla v8.4s, v6.4s, v0.s[3]\n"
+ "fmla v12.4s, v6.4s, v1.s[3]\n"
+ "fmla v16.4s, v6.4s, v2.s[3]\n"
+ "fmla v20.4s, v6.4s, v3.s[3]\n"
+ "ldr q6, [x15, #0xe0]\n"
+ "fmla v9.4s, v7.4s, v0.s[3]\n"
+ "fmla v13.4s, v7.4s, v1.s[3]\n"
+ "fmla v17.4s, v7.4s, v2.s[3]\n"
+ "fmla v21.4s, v7.4s, v3.s[3]\n"
+ "ldr q7, [x15, #0xf0]\n"
+ "add x15, x15, #0x100\n"
+ "fmla v10.4s, v6.4s, v0.s[3]\n"
+ "fmla v14.4s, v6.4s, v1.s[3]\n"
+ "fmla v18.4s, v6.4s, v2.s[3]\n"
+ "fmla v22.4s, v6.4s, v3.s[3]\n"
+ "fmla v11.4s, v7.4s, v0.s[3]\n"
+ "fmla v15.4s, v7.4s, v1.s[3]\n"
+ "fmla v19.4s, v7.4s, v2.s[3]\n"
+ "fmla v23.4s, v7.4s, v3.s[3]\n"
+ "123:" // Height 4: Multiply loop: Main loop skip
+ "cbz x11, 125f\n"
+ "124:" // Height 4: Multiply loop: Odd block loop
+ "ldr s0, [x10], #0x4\n"
+ "ldr s1, [x28], #0x4\n"
+ "ldr s2, [x26], #0x4\n"
+ "ldr s3, [x24], #0x4\n"
+ "ldr q6, [x15, #0x0]\n"
+ "fmla v8.4s, v6.4s, v0.s[0]\n"
+ "ldr q7, [x15, #0x10]\n"
+ "fmla v12.4s, v6.4s, v1.s[0]\n"
+ "sub x11, x11, #0x1\n"
+ "fmla v16.4s, v6.4s, v2.s[0]\n"
+ "fmla v20.4s, v6.4s, v3.s[0]\n"
+ "ldr q6, [x15, #0x20]\n"
+ "fmla v9.4s, v7.4s, v0.s[0]\n"
+ "fmla v13.4s, v7.4s, v1.s[0]\n"
+ "fmla v17.4s, v7.4s, v2.s[0]\n"
+ "fmla v21.4s, v7.4s, v3.s[0]\n"
+ "ldr q7, [x15, #0x30]\n"
+ "fmla v10.4s, v6.4s, v0.s[0]\n"
+ "add x15, x15, #0x40\n"
+ "fmla v14.4s, v6.4s, v1.s[0]\n"
+ "fmla v18.4s, v6.4s, v2.s[0]\n"
+ "fmla v22.4s, v6.4s, v3.s[0]\n"
+ "fmla v11.4s, v7.4s, v0.s[0]\n"
+ "fmla v15.4s, v7.4s, v1.s[0]\n"
+ "fmla v19.4s, v7.4s, v2.s[0]\n"
+ "fmla v23.4s, v7.4s, v3.s[0]\n"
+ "cbnz x11, 124b\n"
+ "125:" // Height 4: Multiply loop: No odd multiplies
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "add x12, x12, #0x1\n"
+ "cmp x12, x19\n"
+ "bne 118b\n"
+ "prfm pstl1keep, [x13, #0x0]\n"
+ "prfm pstl1keep, [x9, #0x0]\n"
+ "prfm pstl1keep, [x27, #0x0]\n"
+ "prfm pstl1keep, [x25, #0x0]\n"
+ "tbz %x[flags], #1, 126f\n"
+ "add x19, %x[args_ptr], %[offset_min]\n"
+ "ld1r { v1.4s }, [x19]\n"
+ "add x19, %x[args_ptr], %[offset_max]\n"
+ "ld1r { v0.4s }, [x19]\n"
+ "fmin v8.4s, v8.4s, v0.4s\n"
+ "fmin v9.4s, v9.4s, v0.4s\n"
+ "fmin v10.4s, v10.4s, v0.4s\n"
+ "fmin v11.4s, v11.4s, v0.4s\n"
+ "fmax v8.4s, v8.4s, v1.4s\n"
+ "fmax v9.4s, v9.4s, v1.4s\n"
+ "fmax v10.4s, v10.4s, v1.4s\n"
+ "fmax v11.4s, v11.4s, v1.4s\n"
+ "fmin v12.4s, v12.4s, v0.4s\n"
+ "fmin v13.4s, v13.4s, v0.4s\n"
+ "fmin v14.4s, v14.4s, v0.4s\n"
+ "fmax v12.4s, v12.4s, v1.4s\n"
+ "fmax v13.4s, v13.4s, v1.4s\n"
+ "fmax v14.4s, v14.4s, v1.4s\n"
+ "fmin v15.4s, v15.4s, v0.4s\n"
+ "fmin v16.4s, v16.4s, v0.4s\n"
+ "fmin v17.4s, v17.4s, v0.4s\n"
+ "fmax v15.4s, v15.4s, v1.4s\n"
+ "fmax v16.4s, v16.4s, v1.4s\n"
+ "fmax v17.4s, v17.4s, v1.4s\n"
+ "fmin v18.4s, v18.4s, v0.4s\n"
+ "fmin v19.4s, v19.4s, v0.4s\n"
+ "fmin v20.4s, v20.4s, v0.4s\n"
+ "fmax v18.4s, v18.4s, v1.4s\n"
+ "fmax v19.4s, v19.4s, v1.4s\n"
+ "fmax v20.4s, v20.4s, v1.4s\n"
+ "fmin v21.4s, v21.4s, v0.4s\n"
+ "fmin v22.4s, v22.4s, v0.4s\n"
+ "fmin v23.4s, v23.4s, v0.4s\n"
+ "fmax v21.4s, v21.4s, v1.4s\n"
+ "fmax v22.4s, v22.4s, v1.4s\n"
+ "fmax v23.4s, v23.4s, v1.4s\n"
+ "126:" // Height 4: No activation
+ "cmp x16, #0x10\n"
+ "bge 135f\n"
+ "tbz x16, #3, 130f\n"
+ "st1 { v8.4s }, [x13], #0x10\n"
+ "st1 { v9.4s }, [x13], #0x10\n"
+ "st1 { v12.4s }, [x9], #0x10\n"
+ "st1 { v13.4s }, [x9], #0x10\n"
+ "st1 { v16.4s }, [x27], #0x10\n"
+ "st1 { v17.4s }, [x27], #0x10\n"
+ "st1 { v20.4s }, [x25], #0x10\n"
+ "st1 { v21.4s }, [x25], #0x10\n"
+ "tbz x16, #2, 128f\n"
+ "st1 { v10.4s }, [x13], #0x10\n"
+ "st1 { v14.4s }, [x9], #0x10\n"
+ "st1 { v18.4s }, [x27], #0x10\n"
+ "st1 { v22.4s }, [x25], #0x10\n"
+ "tbz x16, #1, 127f\n"
+ "str d11, [x13], #0x8\n"
+ "str d15, [x9], #0x8\n"
+ "str d19, [x27], #0x8\n"
+ "str d23, [x25], #0x8\n"
+ "tbz x16, #0, 134f\n"
+ "st1 { v11.s }[2], [x13]\n"
+ "st1 { v15.s }[2], [x9]\n"
+ "st1 { v19.s }[2], [x27]\n"
+ "st1 { v23.s }[2], [x25]\n"
+ "b 134f\n"
+ "127:" // Height 4: Partial direct writeback: partial_1_12
+ "tbz x16, #0, 134f\n"
+ "str s11, [x13, #0x0]\n"
+ "str s15, [x9, #0x0]\n"
+ "str s19, [x27, #0x0]\n"
+ "str s23, [x25, #0x0]\n"
+ "b 134f\n"
+ "128:" // Height 4: Partial direct writeback: partial_2_8
+ "tbz x16, #1, 129f\n"
+ "str d10, [x13], #0x8\n"
+ "str d14, [x9], #0x8\n"
+ "str d18, [x27], #0x8\n"
+ "str d22, [x25], #0x8\n"
+ "tbz x16, #0, 134f\n"
+ "st1 { v10.s }[2], [x13]\n"
+ "st1 { v14.s }[2], [x9]\n"
+ "st1 { v18.s }[2], [x27]\n"
+ "st1 { v22.s }[2], [x25]\n"
+ "b 134f\n"
+ "129:" // Height 4: Partial direct writeback: partial_1_8
+ "tbz x16, #0, 134f\n"
+ "str s10, [x13, #0x0]\n"
+ "str s14, [x9, #0x0]\n"
+ "str s18, [x27, #0x0]\n"
+ "str s22, [x25, #0x0]\n"
+ "b 134f\n"
+ "130:" // Height 4: Partial direct writeback: partial_4_0
+ "tbz x16, #2, 132f\n"
+ "st1 { v8.4s }, [x13], #0x10\n"
+ "st1 { v12.4s }, [x9], #0x10\n"
+ "st1 { v16.4s }, [x27], #0x10\n"
+ "st1 { v20.4s }, [x25], #0x10\n"
+ "tbz x16, #1, 131f\n"
+ "str d9, [x13], #0x8\n"
+ "str d13, [x9], #0x8\n"
+ "str d17, [x27], #0x8\n"
+ "str d21, [x25], #0x8\n"
+ "tbz x16, #0, 134f\n"
+ "st1 { v9.s }[2], [x13]\n"
+ "st1 { v13.s }[2], [x9]\n"
+ "st1 { v17.s }[2], [x27]\n"
+ "st1 { v21.s }[2], [x25]\n"
+ "b 134f\n"
+ "131:" // Height 4: Partial direct writeback: partial_1_4
+ "tbz x16, #0, 134f\n"
+ "str s9, [x13, #0x0]\n"
+ "str s13, [x9, #0x0]\n"
+ "str s17, [x27, #0x0]\n"
+ "str s21, [x25, #0x0]\n"
+ "b 134f\n"
+ "132:" // Height 4: Partial direct writeback: partial_2_0
+ "tbz x16, #1, 133f\n"
+ "str d8, [x13], #0x8\n"
+ "str d12, [x9], #0x8\n"
+ "str d16, [x27], #0x8\n"
+ "str d20, [x25], #0x8\n"
+ "tbz x16, #0, 134f\n"
+ "st1 { v8.s }[2], [x13]\n"
+ "st1 { v12.s }[2], [x9]\n"
+ "st1 { v16.s }[2], [x27]\n"
+ "st1 { v20.s }[2], [x25]\n"
+ "b 134f\n"
+ "133:" // Height 4: Partial direct writeback: partial_1_0
+ "str s8, [x13, #0x0]\n"
+ "str s12, [x9, #0x0]\n"
+ "str s16, [x27, #0x0]\n"
+ "str s20, [x25, #0x0]\n"
+ "134:" // Height 4: Partial direct writeback: Done
+ "b 136f\n"
+ "135:" // Height 4: Full writeback
+ "str q8, [x13, #0x0]\n"
+ "str q9, [x13, #0x10]\n"
+ "str q10, [x13, #0x20]\n"
+ "str q11, [x13, #0x30]\n"
+ "str q12, [x9, #0x0]\n"
+ "str q13, [x9, #0x10]\n"
+ "str q14, [x9, #0x20]\n"
+ "str q15, [x9, #0x30]\n"
+ "str q16, [x27, #0x0]\n"
+ "str q17, [x27, #0x10]\n"
+ "str q18, [x27, #0x20]\n"
+ "str q19, [x27, #0x30]\n"
+ "str q20, [x25, #0x0]\n"
+ "str q21, [x25, #0x10]\n"
+ "str q22, [x25, #0x20]\n"
+ "str q23, [x25, #0x30]\n"
+ "add x13, x13, #0x40\n"
+ "add x9, x9, #0x40\n"
+ "add x27, x27, #0x40\n"
+ "add x25, x25, #0x40\n"
+ "136:" // Height 4: Writeback done
+ "subs x16, x16, #0x10\n"
+ "bgt 105b\n"
+ "b 206f\n"
+ "137:" // Height 5
+ "ldr x16, [%x[args_ptr], %[offsetof_N]]\n"
+ "mov x14, %x[bias]\n"
+ "ldr x15, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 138f\n"
+ "ldr x13, [%x[output_ptr], #0x0]\n"
+ "add x13, x13, x19, LSL #2\n"
+ "ldr x9, [%x[output_ptr], #0x8]\n"
+ "ldr x27, [%x[output_ptr], #0x10]\n"
+ "add x9, x9, x19, LSL #2\n"
+ "ldr x25, [%x[output_ptr], #0x18]\n"
+ "ldr x23, [%x[output_ptr], #0x20]\n"
+ "add x27, x27, x19, LSL #2\n"
+ "add x25, x25, x19, LSL #2\n"
+ "add x23, x23, x19, LSL #2\n"
+ "b 139f\n"
+ "138:" // Height 5: setup direct output
+ "mov x13, %x[output_ptr]\n"
+ "add x9, x13, x19, LSL #2\n"
+ "add x27, x9, x19, LSL #2\n"
+ "add x25, x27, x19, LSL #2\n"
+ "add x23, x25, x19, LSL #2\n"
+ "139:" // Height 5: Column loop
+ "cbz x14, 140f\n"
+ "ldr q8, [x14, #0x0]\n"
+ "mov v12.16b, v8.16b\n"
+ "ldr q9, [x14, #0x10]\n"
+ "mov v16.16b, v8.16b\n"
+ "ldr q10, [x14, #0x20]\n"
+ "mov v20.16b, v8.16b\n"
+ "ldr q11, [x14, #0x30]\n"
+ "mov v24.16b, v8.16b\n"
+ "add x14, x14, #0x40\n"
+ "mov v13.16b, v9.16b\n"
+ "mov v17.16b, v9.16b\n"
+ "mov v14.16b, v10.16b\n"
+ "mov v15.16b, v11.16b\n"
+ "mov v18.16b, v10.16b\n"
+ "mov v19.16b, v11.16b\n"
+ "mov v21.16b, v9.16b\n"
+ "mov v22.16b, v10.16b\n"
+ "mov v23.16b, v11.16b\n"
+ "mov v25.16b, v9.16b\n"
+ "mov v26.16b, v10.16b\n"
+ "mov v27.16b, v11.16b\n"
+ "b 151f\n"
+ "140:" // Height 5: no bias
+ "tbz %x[flags], #0, 150f\n"
+ "cmp x16, #0x10\n"
+ "bge 149f\n"
+ "tbz x16, #3, 144f\n"
+ "ld1 { v8.4s }, [x13], #0x10\n"
+ "ld1 { v12.4s }, [x9], #0x10\n"
+ "ld1 { v16.4s }, [x27], #0x10\n"
+ "ld1 { v20.4s }, [x25], #0x10\n"
+ "ld1 { v24.4s }, [x23], #0x10\n"
+ "ld1 { v9.4s }, [x13], #0x10\n"
+ "ld1 { v13.4s }, [x9], #0x10\n"
+ "ld1 { v17.4s }, [x27], #0x10\n"
+ "ld1 { v21.4s }, [x25], #0x10\n"
+ "ld1 { v25.4s }, [x23], #0x10\n"
+ "tbz x16, #2, 142f\n"
+ "ld1 { v10.4s }, [x13], #0x10\n"
+ "ld1 { v14.4s }, [x9], #0x10\n"
+ "ld1 { v18.4s }, [x27], #0x10\n"
+ "ld1 { v22.4s }, [x25], #0x10\n"
+ "ld1 { v26.4s }, [x23], #0x10\n"
+ "tbz x16, #1, 141f\n"
+ "mov x19, #0x38\n"
+ "ldr d11, [x13], #0x8\n"
+ "ldr d15, [x9], #0x8\n"
+ "ldr d19, [x27], #0x8\n"
+ "ldr d23, [x25], #0x8\n"
+ "ldr d27, [x23], #0x8\n"
+ "tbz x16, #0, 148f\n"
+ "ld1 { v11.s }[2], [x13]\n"
+ "ld1 { v15.s }[2], [x9]\n"
+ "ld1 { v19.s }[2], [x27]\n"
+ "ld1 { v23.s }[2], [x25]\n"
+ "ld1 { v27.s }[2], [x23]\n"
+ "b 148f\n"
+ "141:" // Height 5: Partial accumulate: partial_1_12
+ "mov x19, #0x30\n"
+ "tbz x16, #0, 148f\n"
+ "ldr s11, [x13, #0x0]\n"
+ "ldr s15, [x9, #0x0]\n"
+ "ldr s19, [x27, #0x0]\n"
+ "ldr s23, [x25, #0x0]\n"
+ "ldr s27, [x23, #0x0]\n"
+ "b 148f\n"
+ "142:" // Height 5: Partial accumulate: partial_2_8
+ "tbz x16, #1, 143f\n"
+ "ldr d10, [x13], #0x8\n"
+ "ldr d14, [x9], #0x8\n"
+ "ldr d18, [x27], #0x8\n"
+ "ldr d22, [x25], #0x8\n"
+ "ldr d26, [x23], #0x8\n"
+ "mov x19, #0x28\n"
+ "tbz x16, #0, 148f\n"
+ "ld1 { v10.s }[2], [x13]\n"
+ "ld1 { v14.s }[2], [x9]\n"
+ "ld1 { v18.s }[2], [x27]\n"
+ "ld1 { v22.s }[2], [x25]\n"
+ "ld1 { v26.s }[2], [x23]\n"
+ "b 148f\n"
+ "143:" // Height 5: Partial accumulate: partial_1_8
+ "mov x19, #0x20\n"
+ "tbz x16, #0, 148f\n"
+ "ldr s10, [x13, #0x0]\n"
+ "ldr s14, [x9, #0x0]\n"
+ "ldr s18, [x27, #0x0]\n"
+ "ldr s22, [x25, #0x0]\n"
+ "ldr s26, [x23, #0x0]\n"
+ "b 148f\n"
+ "144:" // Height 5: Partial accumulate: partial_4_0
+ "tbz x16, #2, 146f\n"
+ "ld1 { v8.4s }, [x13], #0x10\n"
+ "ld1 { v12.4s }, [x9], #0x10\n"
+ "ld1 { v16.4s }, [x27], #0x10\n"
+ "ld1 { v20.4s }, [x25], #0x10\n"
+ "ld1 { v24.4s }, [x23], #0x10\n"
+ "tbz x16, #1, 145f\n"
+ "mov x19, #0x18\n"
+ "ldr d9, [x13], #0x8\n"
+ "ldr d13, [x9], #0x8\n"
+ "ldr d17, [x27], #0x8\n"
+ "ldr d21, [x25], #0x8\n"
+ "ldr d25, [x23], #0x8\n"
+ "tbz x16, #0, 148f\n"
+ "ld1 { v9.s }[2], [x13]\n"
+ "ld1 { v13.s }[2], [x9]\n"
+ "ld1 { v17.s }[2], [x27]\n"
+ "ld1 { v21.s }[2], [x25]\n"
+ "ld1 { v25.s }[2], [x23]\n"
+ "b 148f\n"
+ "145:" // Height 5: Partial accumulate: partial_1_4
+ "mov x19, #0x10\n"
+ "tbz x16, #0, 148f\n"
+ "ldr s9, [x13, #0x0]\n"
+ "ldr s13, [x9, #0x0]\n"
+ "ldr s17, [x27, #0x0]\n"
+ "ldr s21, [x25, #0x0]\n"
+ "ldr s25, [x23, #0x0]\n"
+ "b 148f\n"
+ "146:" // Height 5: Partial accumulate: partial_2_0
+ "tbz x16, #1, 147f\n"
+ "ldr d8, [x13], #0x8\n"
+ "ldr d12, [x9], #0x8\n"
+ "ldr d16, [x27], #0x8\n"
+ "ldr d20, [x25], #0x8\n"
+ "ldr d24, [x23], #0x8\n"
+ "mov x19, #0x8\n"
+ "tbz x16, #0, 148f\n"
+ "ld1 { v8.s }[2], [x13]\n"
+ "ld1 { v12.s }[2], [x9]\n"
+ "ld1 { v16.s }[2], [x27]\n"
+ "ld1 { v20.s }[2], [x25]\n"
+ "ld1 { v24.s }[2], [x23]\n"
+ "b 148f\n"
+ "147:" // Height 5: Partial accumulate: partial_1_0
+ "mov x19, #0x0\n"
+ "ldr s8, [x13, #0x0]\n"
+ "ldr s12, [x9, #0x0]\n"
+ "ldr s16, [x27, #0x0]\n"
+ "ldr s20, [x25, #0x0]\n"
+ "ldr s24, [x23, #0x0]\n"
+ "148:" // Height 5: Partial accumulate: Done
+ "sub x13, x13, x19\n"
+ "sub x9, x9, x19\n"
+ "sub x27, x27, x19\n"
+ "sub x25, x25, x19\n"
+ "sub x23, x23, x19\n"
+ "b 151f\n"
+ "149:" // Height 5: full accumulate
+ "ldr q8, [x13, #0x0]\n"
+ "ldr q9, [x13, #0x10]\n"
+ "ldr q10, [x13, #0x20]\n"
+ "ldr q11, [x13, #0x30]\n"
+ "ldr q12, [x9, #0x0]\n"
+ "ldr q13, [x9, #0x10]\n"
+ "ldr q14, [x9, #0x20]\n"
+ "ldr q15, [x9, #0x30]\n"
+ "ldr q16, [x27, #0x0]\n"
+ "ldr q17, [x27, #0x10]\n"
+ "ldr q18, [x27, #0x20]\n"
+ "ldr q19, [x27, #0x30]\n"
+ "ldr q20, [x25, #0x0]\n"
+ "ldr q21, [x25, #0x10]\n"
+ "ldr q22, [x25, #0x20]\n"
+ "ldr q23, [x25, #0x30]\n"
+ "ldr q24, [x23, #0x0]\n"
+ "ldr q25, [x23, #0x10]\n"
+ "ldr q26, [x23, #0x20]\n"
+ "ldr q27, [x23, #0x30]\n"
+ "b 151f\n"
+ "150:" // Height 5: no accumulate
+ "movi v8.16b, #0x0\n"
+ "movi v9.16b, #0x0\n"
+ "movi v10.16b, #0x0\n"
+ "movi v11.16b, #0x0\n"
+ "movi v12.16b, #0x0\n"
+ "movi v13.16b, #0x0\n"
+ "movi v14.16b, #0x0\n"
+ "movi v15.16b, #0x0\n"
+ "movi v16.16b, #0x0\n"
+ "movi v17.16b, #0x0\n"
+ "movi v18.16b, #0x0\n"
+ "movi v19.16b, #0x0\n"
+ "movi v20.16b, #0x0\n"
+ "movi v21.16b, #0x0\n"
+ "movi v22.16b, #0x0\n"
+ "movi v23.16b, #0x0\n"
+ "movi v24.16b, #0x0\n"
+ "movi v25.16b, #0x0\n"
+ "movi v26.16b, #0x0\n"
+ "movi v27.16b, #0x0\n"
+ "151:" // Height 5: setup done
+ "mov x12, #0x0\n"
+ "152:" // Height 5: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w11, [x20, x12, LSL #0x2]\n"
+ "tbz %x[flags], #3, 153f\n"
+ "ldr x20, [%x[input_ptr], x12, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x10, [x20, #0x0]\n"
+ "ldr x28, [x20, #0x8]\n"
+ "ldr x26, [x20, #0x10]\n"
+ "ldr x24, [x20, #0x18]\n"
+ "ldr x22, [x20, #0x20]\n"
+ "cbnz x12, 154f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x10, x10, x19, LSL #2\n"
+ "add x28, x28, x19, LSL #2\n"
+ "add x26, x26, x19, LSL #2\n"
+ "add x24, x24, x19, LSL #2\n"
+ "add x22, x22, x19, LSL #2\n"
+ "b 154f\n"
+ "153:" // Height 5: setup direct input
+ "mov x10, %x[input_ptr]\n"
+ "add x28, x10, x19, LSL #2\n"
+ "add x26, x28, x19, LSL #2\n"
+ "add x24, x26, x19, LSL #2\n"
+ "add x22, x24, x19, LSL #2\n"
+ "154:" // Height 5: input setup done
+ "cmp x11, #0x4\n"
+ "blt 157f\n"
+ "cmp x11, #0x8\n"
+ "blt 156f\n"
+ "155:" // Height 5: Multiply loop: Main loop head
+ "ldr q0, [x10, #0x0]\n"
+ "ldr q1, [x28, #0x0]\n"
+ "ldr q2, [x26, #0x0]\n"
+ "ldr q3, [x24, #0x0]\n"
+ "ldr q4, [x22, #0x0]\n"
+ "ldr q6, [x15, #0x0]\n"
+ "fmla v8.4s, v6.4s, v0.s[0]\n"
+ "ldr q7, [x15, #0x10]\n"
+ "fmla v12.4s, v6.4s, v1.s[0]\n"
+ "add x10, x10, #0x10\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "fmla v16.4s, v6.4s, v2.s[0]\n"
+ "add x28, x28, #0x10\n"
+ "fmla v20.4s, v6.4s, v3.s[0]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "add x26, x26, #0x10\n"
+ "fmla v24.4s, v6.4s, v4.s[0]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "ldr q6, [x15, #0x20]\n"
+ "fmla v9.4s, v7.4s, v0.s[0]\n"
+ "add x24, x24, #0x10\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "fmla v13.4s, v7.4s, v1.s[0]\n"
+ "add x22, x22, #0x10\n"
+ "fmla v17.4s, v7.4s, v2.s[0]\n"
+ "prfm pldl1keep, [x22, #0x80]\n"
+ "sub x11, x11, #0x4\n"
+ "fmla v21.4s, v7.4s, v3.s[0]\n"
+ "cmp x11, #0x8\n"
+ "fmla v25.4s, v7.4s, v4.s[0]\n"
+ "ldr q7, [x15, #0x30]\n"
+ "fmla v10.4s, v6.4s, v0.s[0]\n"
+ "fmla v14.4s, v6.4s, v1.s[0]\n"
+ "fmla v18.4s, v6.4s, v2.s[0]\n"
+ "fmla v22.4s, v6.4s, v3.s[0]\n"
+ "fmla v26.4s, v6.4s, v4.s[0]\n"
+ "ldr q6, [x15, #0x40]\n"
+ "fmla v11.4s, v7.4s, v0.s[0]\n"
+ "fmla v15.4s, v7.4s, v1.s[0]\n"
+ "fmla v19.4s, v7.4s, v2.s[0]\n"
+ "fmla v23.4s, v7.4s, v3.s[0]\n"
+ "fmla v27.4s, v7.4s, v4.s[0]\n"
+ "ldr q7, [x15, #0x50]\n"
+ "fmla v8.4s, v6.4s, v0.s[1]\n"
+ "fmla v12.4s, v6.4s, v1.s[1]\n"
+ "fmla v16.4s, v6.4s, v2.s[1]\n"
+ "fmla v20.4s, v6.4s, v3.s[1]\n"
+ "fmla v24.4s, v6.4s, v4.s[1]\n"
+ "ldr q6, [x15, #0x60]\n"
+ "fmla v9.4s, v7.4s, v0.s[1]\n"
+ "fmla v13.4s, v7.4s, v1.s[1]\n"
+ "fmla v17.4s, v7.4s, v2.s[1]\n"
+ "fmla v21.4s, v7.4s, v3.s[1]\n"
+ "fmla v25.4s, v7.4s, v4.s[1]\n"
+ "ldr q7, [x15, #0x70]\n"
+ "fmla v10.4s, v6.4s, v0.s[1]\n"
+ "fmla v14.4s, v6.4s, v1.s[1]\n"
+ "fmla v18.4s, v6.4s, v2.s[1]\n"
+ "fmla v22.4s, v6.4s, v3.s[1]\n"
+ "fmla v26.4s, v6.4s, v4.s[1]\n"
+ "ldr q6, [x15, #0x80]\n"
+ "fmla v11.4s, v7.4s, v0.s[1]\n"
+ "fmla v15.4s, v7.4s, v1.s[1]\n"
+ "fmla v19.4s, v7.4s, v2.s[1]\n"
+ "fmla v23.4s, v7.4s, v3.s[1]\n"
+ "fmla v27.4s, v7.4s, v4.s[1]\n"
+ "ldr q7, [x15, #0x90]\n"
+ "fmla v8.4s, v6.4s, v0.s[2]\n"
+ "fmla v12.4s, v6.4s, v1.s[2]\n"
+ "fmla v16.4s, v6.4s, v2.s[2]\n"
+ "fmla v20.4s, v6.4s, v3.s[2]\n"
+ "fmla v24.4s, v6.4s, v4.s[2]\n"
+ "ldr q6, [x15, #0xa0]\n"
+ "fmla v9.4s, v7.4s, v0.s[2]\n"
+ "fmla v13.4s, v7.4s, v1.s[2]\n"
+ "fmla v17.4s, v7.4s, v2.s[2]\n"
+ "fmla v21.4s, v7.4s, v3.s[2]\n"
+ "fmla v25.4s, v7.4s, v4.s[2]\n"
+ "ldr q7, [x15, #0xb0]\n"
+ "fmla v10.4s, v6.4s, v0.s[2]\n"
+ "fmla v14.4s, v6.4s, v1.s[2]\n"
+ "fmla v18.4s, v6.4s, v2.s[2]\n"
+ "fmla v22.4s, v6.4s, v3.s[2]\n"
+ "fmla v26.4s, v6.4s, v4.s[2]\n"
+ "ldr q6, [x15, #0xc0]\n"
+ "fmla v11.4s, v7.4s, v0.s[2]\n"
+ "fmla v15.4s, v7.4s, v1.s[2]\n"
+ "fmla v19.4s, v7.4s, v2.s[2]\n"
+ "fmla v23.4s, v7.4s, v3.s[2]\n"
+ "fmla v27.4s, v7.4s, v4.s[2]\n"
+ "ldr q7, [x15, #0xd0]\n"
+ "fmla v8.4s, v6.4s, v0.s[3]\n"
+ "fmla v12.4s, v6.4s, v1.s[3]\n"
+ "fmla v16.4s, v6.4s, v2.s[3]\n"
+ "fmla v20.4s, v6.4s, v3.s[3]\n"
+ "fmla v24.4s, v6.4s, v4.s[3]\n"
+ "ldr q6, [x15, #0xe0]\n"
+ "fmla v9.4s, v7.4s, v0.s[3]\n"
+ "fmla v13.4s, v7.4s, v1.s[3]\n"
+ "fmla v17.4s, v7.4s, v2.s[3]\n"
+ "fmla v21.4s, v7.4s, v3.s[3]\n"
+ "fmla v25.4s, v7.4s, v4.s[3]\n"
+ "ldr q7, [x15, #0xf0]\n"
+ "fmla v10.4s, v6.4s, v0.s[3]\n"
+ "add x15, x15, #0x100\n"
+ "fmla v14.4s, v6.4s, v1.s[3]\n"
+ "fmla v18.4s, v6.4s, v2.s[3]\n"
+ "fmla v22.4s, v6.4s, v3.s[3]\n"
+ "fmla v26.4s, v6.4s, v4.s[3]\n"
+ "fmla v11.4s, v7.4s, v0.s[3]\n"
+ "fmla v15.4s, v7.4s, v1.s[3]\n"
+ "fmla v19.4s, v7.4s, v2.s[3]\n"
+ "fmla v23.4s, v7.4s, v3.s[3]\n"
+ "fmla v27.4s, v7.4s, v4.s[3]\n"
+ "bge 155b\n"
+ "156:" // Height 5: Multiply loop: Single iteration only
+ "sub x11, x11, #0x4\n"
+ "ldr q0, [x10, #0x0]\n"
+ "ldr q1, [x28, #0x0]\n"
+ "ldr q2, [x26, #0x0]\n"
+ "ldr q3, [x24, #0x0]\n"
+ "ldr q4, [x22, #0x0]\n"
+ "ldr q6, [x15, #0x0]\n"
+ "fmla v8.4s, v6.4s, v0.s[0]\n"
+ "ldr q7, [x15, #0x10]\n"
+ "fmla v12.4s, v6.4s, v1.s[0]\n"
+ "add x10, x10, #0x10\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "fmla v16.4s, v6.4s, v2.s[0]\n"
+ "add x28, x28, #0x10\n"
+ "fmla v20.4s, v6.4s, v3.s[0]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "add x26, x26, #0x10\n"
+ "fmla v24.4s, v6.4s, v4.s[0]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "ldr q6, [x15, #0x20]\n"
+ "fmla v9.4s, v7.4s, v0.s[0]\n"
+ "add x24, x24, #0x10\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "fmla v13.4s, v7.4s, v1.s[0]\n"
+ "add x22, x22, #0x10\n"
+ "fmla v17.4s, v7.4s, v2.s[0]\n"
+ "prfm pldl1keep, [x22, #0x80]\n"
+ "fmla v21.4s, v7.4s, v3.s[0]\n"
+ "fmla v25.4s, v7.4s, v4.s[0]\n"
+ "ldr q7, [x15, #0x30]\n"
+ "fmla v10.4s, v6.4s, v0.s[0]\n"
+ "fmla v14.4s, v6.4s, v1.s[0]\n"
+ "fmla v18.4s, v6.4s, v2.s[0]\n"
+ "fmla v22.4s, v6.4s, v3.s[0]\n"
+ "fmla v26.4s, v6.4s, v4.s[0]\n"
+ "ldr q6, [x15, #0x40]\n"
+ "fmla v11.4s, v7.4s, v0.s[0]\n"
+ "fmla v15.4s, v7.4s, v1.s[0]\n"
+ "fmla v19.4s, v7.4s, v2.s[0]\n"
+ "fmla v23.4s, v7.4s, v3.s[0]\n"
+ "fmla v27.4s, v7.4s, v4.s[0]\n"
+ "ldr q7, [x15, #0x50]\n"
+ "fmla v8.4s, v6.4s, v0.s[1]\n"
+ "fmla v12.4s, v6.4s, v1.s[1]\n"
+ "fmla v16.4s, v6.4s, v2.s[1]\n"
+ "fmla v20.4s, v6.4s, v3.s[1]\n"
+ "fmla v24.4s, v6.4s, v4.s[1]\n"
+ "ldr q6, [x15, #0x60]\n"
+ "fmla v9.4s, v7.4s, v0.s[1]\n"
+ "fmla v13.4s, v7.4s, v1.s[1]\n"
+ "fmla v17.4s, v7.4s, v2.s[1]\n"
+ "fmla v21.4s, v7.4s, v3.s[1]\n"
+ "fmla v25.4s, v7.4s, v4.s[1]\n"
+ "ldr q7, [x15, #0x70]\n"
+ "fmla v10.4s, v6.4s, v0.s[1]\n"
+ "fmla v14.4s, v6.4s, v1.s[1]\n"
+ "fmla v18.4s, v6.4s, v2.s[1]\n"
+ "fmla v22.4s, v6.4s, v3.s[1]\n"
+ "fmla v26.4s, v6.4s, v4.s[1]\n"
+ "ldr q6, [x15, #0x80]\n"
+ "fmla v11.4s, v7.4s, v0.s[1]\n"
+ "fmla v15.4s, v7.4s, v1.s[1]\n"
+ "fmla v19.4s, v7.4s, v2.s[1]\n"
+ "fmla v23.4s, v7.4s, v3.s[1]\n"
+ "fmla v27.4s, v7.4s, v4.s[1]\n"
+ "ldr q7, [x15, #0x90]\n"
+ "fmla v8.4s, v6.4s, v0.s[2]\n"
+ "fmla v12.4s, v6.4s, v1.s[2]\n"
+ "fmla v16.4s, v6.4s, v2.s[2]\n"
+ "fmla v20.4s, v6.4s, v3.s[2]\n"
+ "fmla v24.4s, v6.4s, v4.s[2]\n"
+ "ldr q6, [x15, #0xa0]\n"
+ "fmla v9.4s, v7.4s, v0.s[2]\n"
+ "fmla v13.4s, v7.4s, v1.s[2]\n"
+ "fmla v17.4s, v7.4s, v2.s[2]\n"
+ "fmla v21.4s, v7.4s, v3.s[2]\n"
+ "fmla v25.4s, v7.4s, v4.s[2]\n"
+ "ldr q7, [x15, #0xb0]\n"
+ "fmla v10.4s, v6.4s, v0.s[2]\n"
+ "fmla v14.4s, v6.4s, v1.s[2]\n"
+ "fmla v18.4s, v6.4s, v2.s[2]\n"
+ "fmla v22.4s, v6.4s, v3.s[2]\n"
+ "fmla v26.4s, v6.4s, v4.s[2]\n"
+ "ldr q6, [x15, #0xc0]\n"
+ "fmla v11.4s, v7.4s, v0.s[2]\n"
+ "fmla v15.4s, v7.4s, v1.s[2]\n"
+ "fmla v19.4s, v7.4s, v2.s[2]\n"
+ "fmla v23.4s, v7.4s, v3.s[2]\n"
+ "fmla v27.4s, v7.4s, v4.s[2]\n"
+ "ldr q7, [x15, #0xd0]\n"
+ "fmla v8.4s, v6.4s, v0.s[3]\n"
+ "fmla v12.4s, v6.4s, v1.s[3]\n"
+ "fmla v16.4s, v6.4s, v2.s[3]\n"
+ "fmla v20.4s, v6.4s, v3.s[3]\n"
+ "fmla v24.4s, v6.4s, v4.s[3]\n"
+ "ldr q6, [x15, #0xe0]\n"
+ "fmla v9.4s, v7.4s, v0.s[3]\n"
+ "fmla v13.4s, v7.4s, v1.s[3]\n"
+ "fmla v17.4s, v7.4s, v2.s[3]\n"
+ "fmla v21.4s, v7.4s, v3.s[3]\n"
+ "fmla v25.4s, v7.4s, v4.s[3]\n"
+ "ldr q7, [x15, #0xf0]\n"
+ "fmla v10.4s, v6.4s, v0.s[3]\n"
+ "add x15, x15, #0x100\n"
+ "fmla v14.4s, v6.4s, v1.s[3]\n"
+ "fmla v18.4s, v6.4s, v2.s[3]\n"
+ "fmla v22.4s, v6.4s, v3.s[3]\n"
+ "fmla v26.4s, v6.4s, v4.s[3]\n"
+ "fmla v11.4s, v7.4s, v0.s[3]\n"
+ "fmla v15.4s, v7.4s, v1.s[3]\n"
+ "fmla v19.4s, v7.4s, v2.s[3]\n"
+ "fmla v23.4s, v7.4s, v3.s[3]\n"
+ "fmla v27.4s, v7.4s, v4.s[3]\n"
+ "157:" // Height 5: Multiply loop: Main loop skip
+ "cbz x11, 159f\n"
+ "158:" // Height 5: Multiply loop: Odd block loop
+ "ldr s0, [x10], #0x4\n"
+ "ldr s1, [x28], #0x4\n"
+ "ldr s2, [x26], #0x4\n"
+ "ldr s3, [x24], #0x4\n"
+ "ldr s4, [x22], #0x4\n"
+ "ldr q6, [x15, #0x0]\n"
+ "fmla v8.4s, v6.4s, v0.s[0]\n"
+ "ldr q7, [x15, #0x10]\n"
+ "fmla v12.4s, v6.4s, v1.s[0]\n"
+ "sub x11, x11, #0x1\n"
+ "fmla v16.4s, v6.4s, v2.s[0]\n"
+ "fmla v20.4s, v6.4s, v3.s[0]\n"
+ "fmla v24.4s, v6.4s, v4.s[0]\n"
+ "ldr q6, [x15, #0x20]\n"
+ "fmla v9.4s, v7.4s, v0.s[0]\n"
+ "fmla v13.4s, v7.4s, v1.s[0]\n"
+ "fmla v17.4s, v7.4s, v2.s[0]\n"
+ "fmla v21.4s, v7.4s, v3.s[0]\n"
+ "fmla v25.4s, v7.4s, v4.s[0]\n"
+ "ldr q7, [x15, #0x30]\n"
+ "fmla v10.4s, v6.4s, v0.s[0]\n"
+ "add x15, x15, #0x40\n"
+ "fmla v14.4s, v6.4s, v1.s[0]\n"
+ "fmla v18.4s, v6.4s, v2.s[0]\n"
+ "fmla v22.4s, v6.4s, v3.s[0]\n"
+ "fmla v26.4s, v6.4s, v4.s[0]\n"
+ "fmla v11.4s, v7.4s, v0.s[0]\n"
+ "fmla v15.4s, v7.4s, v1.s[0]\n"
+ "fmla v19.4s, v7.4s, v2.s[0]\n"
+ "fmla v23.4s, v7.4s, v3.s[0]\n"
+ "fmla v27.4s, v7.4s, v4.s[0]\n"
+ "cbnz x11, 158b\n"
+ "159:" // Height 5: Multiply loop: No odd multiplies
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "add x12, x12, #0x1\n"
+ "cmp x12, x19\n"
+ "bne 152b\n"
+ "prfm pstl1keep, [x13, #0x0]\n"
+ "prfm pstl1keep, [x9, #0x0]\n"
+ "prfm pstl1keep, [x27, #0x0]\n"
+ "prfm pstl1keep, [x25, #0x0]\n"
+ "prfm pstl1keep, [x23, #0x0]\n"
+ "tbz %x[flags], #1, 160f\n"
+ "add x19, %x[args_ptr], %[offset_min]\n"
+ "ld1r { v1.4s }, [x19]\n"
+ "add x19, %x[args_ptr], %[offset_max]\n"
+ "ld1r { v0.4s }, [x19]\n"
+ "fmin v8.4s, v8.4s, v0.4s\n"
+ "fmin v9.4s, v9.4s, v0.4s\n"
+ "fmin v10.4s, v10.4s, v0.4s\n"
+ "fmin v11.4s, v11.4s, v0.4s\n"
+ "fmax v8.4s, v8.4s, v1.4s\n"
+ "fmax v9.4s, v9.4s, v1.4s\n"
+ "fmax v10.4s, v10.4s, v1.4s\n"
+ "fmax v11.4s, v11.4s, v1.4s\n"
+ "fmin v12.4s, v12.4s, v0.4s\n"
+ "fmin v13.4s, v13.4s, v0.4s\n"
+ "fmin v14.4s, v14.4s, v0.4s\n"
+ "fmax v12.4s, v12.4s, v1.4s\n"
+ "fmax v13.4s, v13.4s, v1.4s\n"
+ "fmax v14.4s, v14.4s, v1.4s\n"
+ "fmin v15.4s, v15.4s, v0.4s\n"
+ "fmin v16.4s, v16.4s, v0.4s\n"
+ "fmin v17.4s, v17.4s, v0.4s\n"
+ "fmax v15.4s, v15.4s, v1.4s\n"
+ "fmax v16.4s, v16.4s, v1.4s\n"
+ "fmax v17.4s, v17.4s, v1.4s\n"
+ "fmin v18.4s, v18.4s, v0.4s\n"
+ "fmin v19.4s, v19.4s, v0.4s\n"
+ "fmin v20.4s, v20.4s, v0.4s\n"
+ "fmax v18.4s, v18.4s, v1.4s\n"
+ "fmax v19.4s, v19.4s, v1.4s\n"
+ "fmax v20.4s, v20.4s, v1.4s\n"
+ "fmin v21.4s, v21.4s, v0.4s\n"
+ "fmin v22.4s, v22.4s, v0.4s\n"
+ "fmin v23.4s, v23.4s, v0.4s\n"
+ "fmax v21.4s, v21.4s, v1.4s\n"
+ "fmax v22.4s, v22.4s, v1.4s\n"
+ "fmax v23.4s, v23.4s, v1.4s\n"
+ "fmin v24.4s, v24.4s, v0.4s\n"
+ "fmin v25.4s, v25.4s, v0.4s\n"
+ "fmin v26.4s, v26.4s, v0.4s\n"
+ "fmax v24.4s, v24.4s, v1.4s\n"
+ "fmax v25.4s, v25.4s, v1.4s\n"
+ "fmax v26.4s, v26.4s, v1.4s\n"
+ "fmin v27.4s, v27.4s, v0.4s\n"
+ "fmax v27.4s, v27.4s, v1.4s\n"
+ "160:" // Height 5: No activation
+ "cmp x16, #0x10\n"
+ "bge 169f\n"
+ "tbz x16, #3, 164f\n"
+ "st1 { v8.4s }, [x13], #0x10\n"
+ "st1 { v9.4s }, [x13], #0x10\n"
+ "st1 { v12.4s }, [x9], #0x10\n"
+ "st1 { v13.4s }, [x9], #0x10\n"
+ "st1 { v16.4s }, [x27], #0x10\n"
+ "st1 { v17.4s }, [x27], #0x10\n"
+ "st1 { v20.4s }, [x25], #0x10\n"
+ "st1 { v21.4s }, [x25], #0x10\n"
+ "st1 { v24.4s }, [x23], #0x10\n"
+ "st1 { v25.4s }, [x23], #0x10\n"
+ "tbz x16, #2, 162f\n"
+ "st1 { v10.4s }, [x13], #0x10\n"
+ "st1 { v14.4s }, [x9], #0x10\n"
+ "st1 { v18.4s }, [x27], #0x10\n"
+ "st1 { v22.4s }, [x25], #0x10\n"
+ "st1 { v26.4s }, [x23], #0x10\n"
+ "tbz x16, #1, 161f\n"
+ "str d11, [x13], #0x8\n"
+ "str d15, [x9], #0x8\n"
+ "str d19, [x27], #0x8\n"
+ "str d23, [x25], #0x8\n"
+ "str d27, [x23], #0x8\n"
+ "tbz x16, #0, 168f\n"
+ "st1 { v11.s }[2], [x13]\n"
+ "st1 { v15.s }[2], [x9]\n"
+ "st1 { v19.s }[2], [x27]\n"
+ "st1 { v23.s }[2], [x25]\n"
+ "st1 { v27.s }[2], [x23]\n"
+ "b 168f\n"
+ "161:" // Height 5: Partial direct writeback: partial_1_12
+ "tbz x16, #0, 168f\n"
+ "str s11, [x13, #0x0]\n"
+ "str s15, [x9, #0x0]\n"
+ "str s19, [x27, #0x0]\n"
+ "str s23, [x25, #0x0]\n"
+ "str s27, [x23, #0x0]\n"
+ "b 168f\n"
+ "162:" // Height 5: Partial direct writeback: partial_2_8
+ "tbz x16, #1, 163f\n"
+ "str d10, [x13], #0x8\n"
+ "str d14, [x9], #0x8\n"
+ "str d18, [x27], #0x8\n"
+ "str d22, [x25], #0x8\n"
+ "str d26, [x23], #0x8\n"
+ "tbz x16, #0, 168f\n"
+ "st1 { v10.s }[2], [x13]\n"
+ "st1 { v14.s }[2], [x9]\n"
+ "st1 { v18.s }[2], [x27]\n"
+ "st1 { v22.s }[2], [x25]\n"
+ "st1 { v26.s }[2], [x23]\n"
+ "b 168f\n"
+ "163:" // Height 5: Partial direct writeback: partial_1_8
+ "tbz x16, #0, 168f\n"
+ "str s10, [x13, #0x0]\n"
+ "str s14, [x9, #0x0]\n"
+ "str s18, [x27, #0x0]\n"
+ "str s22, [x25, #0x0]\n"
+ "str s26, [x23, #0x0]\n"
+ "b 168f\n"
+ "164:" // Height 5: Partial direct writeback: partial_4_0
+ "tbz x16, #2, 166f\n"
+ "st1 { v8.4s }, [x13], #0x10\n"
+ "st1 { v12.4s }, [x9], #0x10\n"
+ "st1 { v16.4s }, [x27], #0x10\n"
+ "st1 { v20.4s }, [x25], #0x10\n"
+ "st1 { v24.4s }, [x23], #0x10\n"
+ "tbz x16, #1, 165f\n"
+ "str d9, [x13], #0x8\n"
+ "str d13, [x9], #0x8\n"
+ "str d17, [x27], #0x8\n"
+ "str d21, [x25], #0x8\n"
+ "str d25, [x23], #0x8\n"
+ "tbz x16, #0, 168f\n"
+ "st1 { v9.s }[2], [x13]\n"
+ "st1 { v13.s }[2], [x9]\n"
+ "st1 { v17.s }[2], [x27]\n"
+ "st1 { v21.s }[2], [x25]\n"
+ "st1 { v25.s }[2], [x23]\n"
+ "b 168f\n"
+ "165:" // Height 5: Partial direct writeback: partial_1_4
+ "tbz x16, #0, 168f\n"
+ "str s9, [x13, #0x0]\n"
+ "str s13, [x9, #0x0]\n"
+ "str s17, [x27, #0x0]\n"
+ "str s21, [x25, #0x0]\n"
+ "str s25, [x23, #0x0]\n"
+ "b 168f\n"
+ "166:" // Height 5: Partial direct writeback: partial_2_0
+ "tbz x16, #1, 167f\n"
+ "str d8, [x13], #0x8\n"
+ "str d12, [x9], #0x8\n"
+ "str d16, [x27], #0x8\n"
+ "str d20, [x25], #0x8\n"
+ "str d24, [x23], #0x8\n"
+ "tbz x16, #0, 168f\n"
+ "st1 { v8.s }[2], [x13]\n"
+ "st1 { v12.s }[2], [x9]\n"
+ "st1 { v16.s }[2], [x27]\n"
+ "st1 { v20.s }[2], [x25]\n"
+ "st1 { v24.s }[2], [x23]\n"
+ "b 168f\n"
+ "167:" // Height 5: Partial direct writeback: partial_1_0
+ "str s8, [x13, #0x0]\n"
+ "str s12, [x9, #0x0]\n"
+ "str s16, [x27, #0x0]\n"
+ "str s20, [x25, #0x0]\n"
+ "str s24, [x23, #0x0]\n"
+ "168:" // Height 5: Partial direct writeback: Done
+ "b 170f\n"
+ "169:" // Height 5: Full writeback
+ "str q8, [x13, #0x0]\n"
+ "str q9, [x13, #0x10]\n"
+ "str q10, [x13, #0x20]\n"
+ "str q11, [x13, #0x30]\n"
+ "str q12, [x9, #0x0]\n"
+ "str q13, [x9, #0x10]\n"
+ "str q14, [x9, #0x20]\n"
+ "str q15, [x9, #0x30]\n"
+ "str q16, [x27, #0x0]\n"
+ "str q17, [x27, #0x10]\n"
+ "str q18, [x27, #0x20]\n"
+ "str q19, [x27, #0x30]\n"
+ "str q20, [x25, #0x0]\n"
+ "str q21, [x25, #0x10]\n"
+ "str q22, [x25, #0x20]\n"
+ "str q23, [x25, #0x30]\n"
+ "str q24, [x23, #0x0]\n"
+ "str q25, [x23, #0x10]\n"
+ "str q26, [x23, #0x20]\n"
+ "str q27, [x23, #0x30]\n"
+ "add x13, x13, #0x40\n"
+ "add x9, x9, #0x40\n"
+ "add x27, x27, #0x40\n"
+ "add x25, x25, #0x40\n"
+ "add x23, x23, #0x40\n"
+ "170:" // Height 5: Writeback done
+ "subs x16, x16, #0x10\n"
+ "bgt 139b\n"
+ "b 206f\n"
+ "171:" // Height 6
+ "ldr x16, [%x[args_ptr], %[offsetof_N]]\n"
+ "mov x14, %x[bias]\n"
+ "ldr x15, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 172f\n"
+ "ldr x13, [%x[output_ptr], #0x0]\n"
+ "add x13, x13, x19, LSL #2\n"
+ "ldr x9, [%x[output_ptr], #0x8]\n"
+ "ldr x27, [%x[output_ptr], #0x10]\n"
+ "add x9, x9, x19, LSL #2\n"
+ "ldr x25, [%x[output_ptr], #0x18]\n"
+ "ldr x23, [%x[output_ptr], #0x20]\n"
+ "add x27, x27, x19, LSL #2\n"
+ "ldr x21, [%x[output_ptr], #0x28]\n"
+ "add %x[output_ptr], %x[output_ptr], #0x30\n"
+ "add x25, x25, x19, LSL #2\n"
+ "add x23, x23, x19, LSL #2\n"
+ "add x21, x21, x19, LSL #2\n"
+ "b 173f\n"
+ "172:" // Height 6: setup direct output
+ "mov x13, %x[output_ptr]\n"
+ "add x9, x13, x19, LSL #2\n"
+ "add x27, x9, x19, LSL #2\n"
+ "add x25, x27, x19, LSL #2\n"
+ "add x23, x25, x19, LSL #2\n"
+ "add x21, x23, x19, LSL #2\n"
+ "add %x[output_ptr], x21, x19, LSL #2\n"
+ "173:" // Height 6: Column loop
+ "cbz x14, 174f\n"
+ "ldr q8, [x14, #0x0]\n"
+ "mov v12.16b, v8.16b\n"
+ "ldr q9, [x14, #0x10]\n"
+ "mov v16.16b, v8.16b\n"
+ "ldr q10, [x14, #0x20]\n"
+ "mov v20.16b, v8.16b\n"
+ "ldr q11, [x14, #0x30]\n"
+ "mov v24.16b, v8.16b\n"
+ "add x14, x14, #0x40\n"
+ "mov v28.16b, v8.16b\n"
+ "mov v13.16b, v9.16b\n"
+ "mov v17.16b, v9.16b\n"
+ "mov v14.16b, v10.16b\n"
+ "mov v15.16b, v11.16b\n"
+ "mov v18.16b, v10.16b\n"
+ "mov v19.16b, v11.16b\n"
+ "mov v21.16b, v9.16b\n"
+ "mov v22.16b, v10.16b\n"
+ "mov v23.16b, v11.16b\n"
+ "mov v25.16b, v9.16b\n"
+ "mov v26.16b, v10.16b\n"
+ "mov v27.16b, v11.16b\n"
+ "mov v29.16b, v9.16b\n"
+ "mov v30.16b, v10.16b\n"
+ "mov v31.16b, v11.16b\n"
+ "b 185f\n"
+ "174:" // Height 6: no bias
+ "tbz %x[flags], #0, 184f\n"
+ "cmp x16, #0x10\n"
+ "bge 183f\n"
+ "tbz x16, #3, 178f\n"
+ "ld1 { v8.4s }, [x13], #0x10\n"
+ "ld1 { v12.4s }, [x9], #0x10\n"
+ "ld1 { v16.4s }, [x27], #0x10\n"
+ "ld1 { v20.4s }, [x25], #0x10\n"
+ "ld1 { v24.4s }, [x23], #0x10\n"
+ "ld1 { v28.4s }, [x21], #0x10\n"
+ "ld1 { v9.4s }, [x13], #0x10\n"
+ "ld1 { v13.4s }, [x9], #0x10\n"
+ "ld1 { v17.4s }, [x27], #0x10\n"
+ "ld1 { v21.4s }, [x25], #0x10\n"
+ "ld1 { v25.4s }, [x23], #0x10\n"
+ "ld1 { v29.4s }, [x21], #0x10\n"
+ "tbz x16, #2, 176f\n"
+ "ld1 { v10.4s }, [x13], #0x10\n"
+ "ld1 { v14.4s }, [x9], #0x10\n"
+ "ld1 { v18.4s }, [x27], #0x10\n"
+ "ld1 { v22.4s }, [x25], #0x10\n"
+ "ld1 { v26.4s }, [x23], #0x10\n"
+ "ld1 { v30.4s }, [x21], #0x10\n"
+ "tbz x16, #1, 175f\n"
+ "mov x19, #0x38\n"
+ "ldr d11, [x13], #0x8\n"
+ "ldr d15, [x9], #0x8\n"
+ "ldr d19, [x27], #0x8\n"
+ "ldr d23, [x25], #0x8\n"
+ "ldr d27, [x23], #0x8\n"
+ "ldr d31, [x21], #0x8\n"
+ "tbz x16, #0, 182f\n"
+ "ld1 { v11.s }[2], [x13]\n"
+ "ld1 { v15.s }[2], [x9]\n"
+ "ld1 { v19.s }[2], [x27]\n"
+ "ld1 { v23.s }[2], [x25]\n"
+ "ld1 { v27.s }[2], [x23]\n"
+ "ld1 { v31.s }[2], [x21]\n"
+ "b 182f\n"
+ "175:" // Height 6: Partial accumulate: partial_1_12
+ "mov x19, #0x30\n"
+ "tbz x16, #0, 182f\n"
+ "ldr s11, [x13, #0x0]\n"
+ "ldr s15, [x9, #0x0]\n"
+ "ldr s19, [x27, #0x0]\n"
+ "ldr s23, [x25, #0x0]\n"
+ "ldr s27, [x23, #0x0]\n"
+ "ldr s31, [x21, #0x0]\n"
+ "b 182f\n"
+ "176:" // Height 6: Partial accumulate: partial_2_8
+ "tbz x16, #1, 177f\n"
+ "ldr d10, [x13], #0x8\n"
+ "ldr d14, [x9], #0x8\n"
+ "ldr d18, [x27], #0x8\n"
+ "ldr d22, [x25], #0x8\n"
+ "ldr d26, [x23], #0x8\n"
+ "ldr d30, [x21], #0x8\n"
+ "mov x19, #0x28\n"
+ "tbz x16, #0, 182f\n"
+ "ld1 { v10.s }[2], [x13]\n"
+ "ld1 { v14.s }[2], [x9]\n"
+ "ld1 { v18.s }[2], [x27]\n"
+ "ld1 { v22.s }[2], [x25]\n"
+ "ld1 { v26.s }[2], [x23]\n"
+ "ld1 { v30.s }[2], [x21]\n"
+ "b 182f\n"
+ "177:" // Height 6: Partial accumulate: partial_1_8
+ "mov x19, #0x20\n"
+ "tbz x16, #0, 182f\n"
+ "ldr s10, [x13, #0x0]\n"
+ "ldr s14, [x9, #0x0]\n"
+ "ldr s18, [x27, #0x0]\n"
+ "ldr s22, [x25, #0x0]\n"
+ "ldr s26, [x23, #0x0]\n"
+ "ldr s30, [x21, #0x0]\n"
+ "b 182f\n"
+ "178:" // Height 6: Partial accumulate: partial_4_0
+ "tbz x16, #2, 180f\n"
+ "ld1 { v8.4s }, [x13], #0x10\n"
+ "ld1 { v12.4s }, [x9], #0x10\n"
+ "ld1 { v16.4s }, [x27], #0x10\n"
+ "ld1 { v20.4s }, [x25], #0x10\n"
+ "ld1 { v24.4s }, [x23], #0x10\n"
+ "ld1 { v28.4s }, [x21], #0x10\n"
+ "tbz x16, #1, 179f\n"
+ "mov x19, #0x18\n"
+ "ldr d9, [x13], #0x8\n"
+ "ldr d13, [x9], #0x8\n"
+ "ldr d17, [x27], #0x8\n"
+ "ldr d21, [x25], #0x8\n"
+ "ldr d25, [x23], #0x8\n"
+ "ldr d29, [x21], #0x8\n"
+ "tbz x16, #0, 182f\n"
+ "ld1 { v9.s }[2], [x13]\n"
+ "ld1 { v13.s }[2], [x9]\n"
+ "ld1 { v17.s }[2], [x27]\n"
+ "ld1 { v21.s }[2], [x25]\n"
+ "ld1 { v25.s }[2], [x23]\n"
+ "ld1 { v29.s }[2], [x21]\n"
+ "b 182f\n"
+ "179:" // Height 6: Partial accumulate: partial_1_4
+ "mov x19, #0x10\n"
+ "tbz x16, #0, 182f\n"
+ "ldr s9, [x13, #0x0]\n"
+ "ldr s13, [x9, #0x0]\n"
+ "ldr s17, [x27, #0x0]\n"
+ "ldr s21, [x25, #0x0]\n"
+ "ldr s25, [x23, #0x0]\n"
+ "ldr s29, [x21, #0x0]\n"
+ "b 182f\n"
+ "180:" // Height 6: Partial accumulate: partial_2_0
+ "tbz x16, #1, 181f\n"
+ "ldr d8, [x13], #0x8\n"
+ "ldr d12, [x9], #0x8\n"
+ "ldr d16, [x27], #0x8\n"
+ "ldr d20, [x25], #0x8\n"
+ "ldr d24, [x23], #0x8\n"
+ "ldr d28, [x21], #0x8\n"
+ "mov x19, #0x8\n"
+ "tbz x16, #0, 182f\n"
+ "ld1 { v8.s }[2], [x13]\n"
+ "ld1 { v12.s }[2], [x9]\n"
+ "ld1 { v16.s }[2], [x27]\n"
+ "ld1 { v20.s }[2], [x25]\n"
+ "ld1 { v24.s }[2], [x23]\n"
+ "ld1 { v28.s }[2], [x21]\n"
+ "b 182f\n"
+ "181:" // Height 6: Partial accumulate: partial_1_0
+ "mov x19, #0x0\n"
+ "ldr s8, [x13, #0x0]\n"
+ "ldr s12, [x9, #0x0]\n"
+ "ldr s16, [x27, #0x0]\n"
+ "ldr s20, [x25, #0x0]\n"
+ "ldr s24, [x23, #0x0]\n"
+ "ldr s28, [x21, #0x0]\n"
+ "182:" // Height 6: Partial accumulate: Done
+ "sub x13, x13, x19\n"
+ "sub x9, x9, x19\n"
+ "sub x27, x27, x19\n"
+ "sub x25, x25, x19\n"
+ "sub x23, x23, x19\n"
+ "sub x21, x21, x19\n"
+ "b 185f\n"
+ "183:" // Height 6: full accumulate
+ "ldr q8, [x13, #0x0]\n"
+ "ldr q9, [x13, #0x10]\n"
+ "ldr q10, [x13, #0x20]\n"
+ "ldr q11, [x13, #0x30]\n"
+ "ldr q12, [x9, #0x0]\n"
+ "ldr q13, [x9, #0x10]\n"
+ "ldr q14, [x9, #0x20]\n"
+ "ldr q15, [x9, #0x30]\n"
+ "ldr q16, [x27, #0x0]\n"
+ "ldr q17, [x27, #0x10]\n"
+ "ldr q18, [x27, #0x20]\n"
+ "ldr q19, [x27, #0x30]\n"
+ "ldr q20, [x25, #0x0]\n"
+ "ldr q21, [x25, #0x10]\n"
+ "ldr q22, [x25, #0x20]\n"
+ "ldr q23, [x25, #0x30]\n"
+ "ldr q24, [x23, #0x0]\n"
+ "ldr q25, [x23, #0x10]\n"
+ "ldr q26, [x23, #0x20]\n"
+ "ldr q27, [x23, #0x30]\n"
+ "ldr q28, [x21, #0x0]\n"
+ "ldr q29, [x21, #0x10]\n"
+ "ldr q30, [x21, #0x20]\n"
+ "ldr q31, [x21, #0x30]\n"
+ "b 185f\n"
+ "184:" // Height 6: no accumulate
+ "movi v8.16b, #0x0\n"
+ "movi v9.16b, #0x0\n"
+ "movi v10.16b, #0x0\n"
+ "movi v11.16b, #0x0\n"
+ "movi v12.16b, #0x0\n"
+ "movi v13.16b, #0x0\n"
+ "movi v14.16b, #0x0\n"
+ "movi v15.16b, #0x0\n"
+ "movi v16.16b, #0x0\n"
+ "movi v17.16b, #0x0\n"
+ "movi v18.16b, #0x0\n"
+ "movi v19.16b, #0x0\n"
+ "movi v20.16b, #0x0\n"
+ "movi v21.16b, #0x0\n"
+ "movi v22.16b, #0x0\n"
+ "movi v23.16b, #0x0\n"
+ "movi v24.16b, #0x0\n"
+ "movi v25.16b, #0x0\n"
+ "movi v26.16b, #0x0\n"
+ "movi v27.16b, #0x0\n"
+ "movi v28.16b, #0x0\n"
+ "movi v29.16b, #0x0\n"
+ "movi v30.16b, #0x0\n"
+ "movi v31.16b, #0x0\n"
+ "185:" // Height 6: setup done
+ "mov x12, #0x0\n"
+ "186:" // Height 6: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w11, [x20, x12, LSL #0x2]\n"
+ "tbz %x[flags], #3, 187f\n"
+ "ldr x20, [%x[input_ptr], x12, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x10, [x20, #0x0]\n"
+ "ldr x28, [x20, #0x8]\n"
+ "ldr x26, [x20, #0x10]\n"
+ "ldr x24, [x20, #0x18]\n"
+ "ldr x22, [x20, #0x20]\n"
+ "ldr x20, [x20, #0x28]\n"
+ "cbnz x12, 188f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x10, x10, x19, LSL #2\n"
+ "add x28, x28, x19, LSL #2\n"
+ "add x26, x26, x19, LSL #2\n"
+ "add x24, x24, x19, LSL #2\n"
+ "add x22, x22, x19, LSL #2\n"
+ "add x20, x20, x19, LSL #2\n"
+ "b 188f\n"
+ "187:" // Height 6: setup direct input
+ "mov x10, %x[input_ptr]\n"
+ "add x28, x10, x19, LSL #2\n"
+ "add x26, x28, x19, LSL #2\n"
+ "add x24, x26, x19, LSL #2\n"
+ "add x22, x24, x19, LSL #2\n"
+ "add x20, x22, x19, LSL #2\n"
+ "188:" // Height 6: input setup done
+ "cmp x11, #0x4\n"
+ "blt 191f\n"
+ "cmp x11, #0x8\n"
+ "blt 190f\n"
+ "189:" // Height 6: Multiply loop: Main loop head
+ "ldr q0, [x10, #0x0]\n"
+ "ldr q1, [x28, #0x0]\n"
+ "ldr q2, [x26, #0x0]\n"
+ "ldr q3, [x24, #0x0]\n"
+ "ldr q4, [x22, #0x0]\n"
+ "ldr q5, [x20, #0x0]\n"
+ "ldr q6, [x15, #0x0]\n"
+ "fmla v8.4s, v6.4s, v0.s[0]\n"
+ "ldr q7, [x15, #0x10]\n"
+ "fmla v12.4s, v6.4s, v1.s[0]\n"
+ "add x10, x10, #0x10\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "fmla v16.4s, v6.4s, v2.s[0]\n"
+ "add x28, x28, #0x10\n"
+ "fmla v20.4s, v6.4s, v3.s[0]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "add x26, x26, #0x10\n"
+ "fmla v24.4s, v6.4s, v4.s[0]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "add x24, x24, #0x10\n"
+ "fmla v28.4s, v6.4s, v5.s[0]\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "ldr q6, [x15, #0x20]\n"
+ "fmla v9.4s, v7.4s, v0.s[0]\n"
+ "add x22, x22, #0x10\n"
+ "prfm pldl1keep, [x22, #0x80]\n"
+ "fmla v13.4s, v7.4s, v1.s[0]\n"
+ "add x20, x20, #0x10\n"
+ "fmla v17.4s, v7.4s, v2.s[0]\n"
+ "prfm pldl1keep, [x20, #0x80]\n"
+ "sub x11, x11, #0x4\n"
+ "fmla v21.4s, v7.4s, v3.s[0]\n"
+ "cmp x11, #0x8\n"
+ "fmla v25.4s, v7.4s, v4.s[0]\n"
+ "fmla v29.4s, v7.4s, v5.s[0]\n"
+ "ldr q7, [x15, #0x30]\n"
+ "fmla v10.4s, v6.4s, v0.s[0]\n"
+ "fmla v14.4s, v6.4s, v1.s[0]\n"
+ "fmla v18.4s, v6.4s, v2.s[0]\n"
+ "fmla v22.4s, v6.4s, v3.s[0]\n"
+ "fmla v26.4s, v6.4s, v4.s[0]\n"
+ "fmla v30.4s, v6.4s, v5.s[0]\n"
+ "ldr q6, [x15, #0x40]\n"
+ "fmla v11.4s, v7.4s, v0.s[0]\n"
+ "fmla v15.4s, v7.4s, v1.s[0]\n"
+ "fmla v19.4s, v7.4s, v2.s[0]\n"
+ "fmla v23.4s, v7.4s, v3.s[0]\n"
+ "fmla v27.4s, v7.4s, v4.s[0]\n"
+ "fmla v31.4s, v7.4s, v5.s[0]\n"
+ "ldr q7, [x15, #0x50]\n"
+ "fmla v8.4s, v6.4s, v0.s[1]\n"
+ "fmla v12.4s, v6.4s, v1.s[1]\n"
+ "fmla v16.4s, v6.4s, v2.s[1]\n"
+ "fmla v20.4s, v6.4s, v3.s[1]\n"
+ "fmla v24.4s, v6.4s, v4.s[1]\n"
+ "fmla v28.4s, v6.4s, v5.s[1]\n"
+ "ldr q6, [x15, #0x60]\n"
+ "fmla v9.4s, v7.4s, v0.s[1]\n"
+ "fmla v13.4s, v7.4s, v1.s[1]\n"
+ "fmla v17.4s, v7.4s, v2.s[1]\n"
+ "fmla v21.4s, v7.4s, v3.s[1]\n"
+ "fmla v25.4s, v7.4s, v4.s[1]\n"
+ "fmla v29.4s, v7.4s, v5.s[1]\n"
+ "ldr q7, [x15, #0x70]\n"
+ "fmla v10.4s, v6.4s, v0.s[1]\n"
+ "fmla v14.4s, v6.4s, v1.s[1]\n"
+ "fmla v18.4s, v6.4s, v2.s[1]\n"
+ "fmla v22.4s, v6.4s, v3.s[1]\n"
+ "fmla v26.4s, v6.4s, v4.s[1]\n"
+ "fmla v30.4s, v6.4s, v5.s[1]\n"
+ "ldr q6, [x15, #0x80]\n"
+ "fmla v11.4s, v7.4s, v0.s[1]\n"
+ "fmla v15.4s, v7.4s, v1.s[1]\n"
+ "fmla v19.4s, v7.4s, v2.s[1]\n"
+ "fmla v23.4s, v7.4s, v3.s[1]\n"
+ "fmla v27.4s, v7.4s, v4.s[1]\n"
+ "fmla v31.4s, v7.4s, v5.s[1]\n"
+ "ldr q7, [x15, #0x90]\n"
+ "fmla v8.4s, v6.4s, v0.s[2]\n"
+ "fmla v12.4s, v6.4s, v1.s[2]\n"
+ "fmla v16.4s, v6.4s, v2.s[2]\n"
+ "fmla v20.4s, v6.4s, v3.s[2]\n"
+ "fmla v24.4s, v6.4s, v4.s[2]\n"
+ "fmla v28.4s, v6.4s, v5.s[2]\n"
+ "ldr q6, [x15, #0xa0]\n"
+ "fmla v9.4s, v7.4s, v0.s[2]\n"
+ "fmla v13.4s, v7.4s, v1.s[2]\n"
+ "fmla v17.4s, v7.4s, v2.s[2]\n"
+ "fmla v21.4s, v7.4s, v3.s[2]\n"
+ "fmla v25.4s, v7.4s, v4.s[2]\n"
+ "fmla v29.4s, v7.4s, v5.s[2]\n"
+ "ldr q7, [x15, #0xb0]\n"
+ "fmla v10.4s, v6.4s, v0.s[2]\n"
+ "fmla v14.4s, v6.4s, v1.s[2]\n"
+ "fmla v18.4s, v6.4s, v2.s[2]\n"
+ "fmla v22.4s, v6.4s, v3.s[2]\n"
+ "fmla v26.4s, v6.4s, v4.s[2]\n"
+ "fmla v30.4s, v6.4s, v5.s[2]\n"
+ "ldr q6, [x15, #0xc0]\n"
+ "fmla v11.4s, v7.4s, v0.s[2]\n"
+ "fmla v15.4s, v7.4s, v1.s[2]\n"
+ "fmla v19.4s, v7.4s, v2.s[2]\n"
+ "fmla v23.4s, v7.4s, v3.s[2]\n"
+ "fmla v27.4s, v7.4s, v4.s[2]\n"
+ "fmla v31.4s, v7.4s, v5.s[2]\n"
+ "ldr q7, [x15, #0xd0]\n"
+ "fmla v8.4s, v6.4s, v0.s[3]\n"
+ "fmla v12.4s, v6.4s, v1.s[3]\n"
+ "fmla v16.4s, v6.4s, v2.s[3]\n"
+ "fmla v20.4s, v6.4s, v3.s[3]\n"
+ "fmla v24.4s, v6.4s, v4.s[3]\n"
+ "fmla v28.4s, v6.4s, v5.s[3]\n"
+ "ldr q6, [x15, #0xe0]\n"
+ "fmla v9.4s, v7.4s, v0.s[3]\n"
+ "fmla v13.4s, v7.4s, v1.s[3]\n"
+ "fmla v17.4s, v7.4s, v2.s[3]\n"
+ "fmla v21.4s, v7.4s, v3.s[3]\n"
+ "fmla v25.4s, v7.4s, v4.s[3]\n"
+ "fmla v29.4s, v7.4s, v5.s[3]\n"
+ "ldr q7, [x15, #0xf0]\n"
+ "fmla v10.4s, v6.4s, v0.s[3]\n"
+ "add x15, x15, #0x100\n"
+ "fmla v14.4s, v6.4s, v1.s[3]\n"
+ "fmla v18.4s, v6.4s, v2.s[3]\n"
+ "fmla v22.4s, v6.4s, v3.s[3]\n"
+ "fmla v26.4s, v6.4s, v4.s[3]\n"
+ "fmla v30.4s, v6.4s, v5.s[3]\n"
+ "fmla v11.4s, v7.4s, v0.s[3]\n"
+ "fmla v15.4s, v7.4s, v1.s[3]\n"
+ "fmla v19.4s, v7.4s, v2.s[3]\n"
+ "fmla v23.4s, v7.4s, v3.s[3]\n"
+ "fmla v27.4s, v7.4s, v4.s[3]\n"
+ "fmla v31.4s, v7.4s, v5.s[3]\n"
+ "bge 189b\n"
+ "190:" // Height 6: Multiply loop: Single iteration only
+ "sub x11, x11, #0x4\n"
+ "ldr q0, [x10, #0x0]\n"
+ "ldr q1, [x28, #0x0]\n"
+ "ldr q2, [x26, #0x0]\n"
+ "ldr q3, [x24, #0x0]\n"
+ "ldr q4, [x22, #0x0]\n"
+ "ldr q5, [x20, #0x0]\n"
+ "ldr q6, [x15, #0x0]\n"
+ "fmla v8.4s, v6.4s, v0.s[0]\n"
+ "ldr q7, [x15, #0x10]\n"
+ "fmla v12.4s, v6.4s, v1.s[0]\n"
+ "add x10, x10, #0x10\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "fmla v16.4s, v6.4s, v2.s[0]\n"
+ "add x28, x28, #0x10\n"
+ "fmla v20.4s, v6.4s, v3.s[0]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "add x26, x26, #0x10\n"
+ "fmla v24.4s, v6.4s, v4.s[0]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "add x24, x24, #0x10\n"
+ "fmla v28.4s, v6.4s, v5.s[0]\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "ldr q6, [x15, #0x20]\n"
+ "fmla v9.4s, v7.4s, v0.s[0]\n"
+ "add x22, x22, #0x10\n"
+ "prfm pldl1keep, [x22, #0x80]\n"
+ "fmla v13.4s, v7.4s, v1.s[0]\n"
+ "add x20, x20, #0x10\n"
+ "fmla v17.4s, v7.4s, v2.s[0]\n"
+ "prfm pldl1keep, [x20, #0x80]\n"
+ "fmla v21.4s, v7.4s, v3.s[0]\n"
+ "fmla v25.4s, v7.4s, v4.s[0]\n"
+ "fmla v29.4s, v7.4s, v5.s[0]\n"
+ "ldr q7, [x15, #0x30]\n"
+ "fmla v10.4s, v6.4s, v0.s[0]\n"
+ "fmla v14.4s, v6.4s, v1.s[0]\n"
+ "fmla v18.4s, v6.4s, v2.s[0]\n"
+ "fmla v22.4s, v6.4s, v3.s[0]\n"
+ "fmla v26.4s, v6.4s, v4.s[0]\n"
+ "fmla v30.4s, v6.4s, v5.s[0]\n"
+ "ldr q6, [x15, #0x40]\n"
+ "fmla v11.4s, v7.4s, v0.s[0]\n"
+ "fmla v15.4s, v7.4s, v1.s[0]\n"
+ "fmla v19.4s, v7.4s, v2.s[0]\n"
+ "fmla v23.4s, v7.4s, v3.s[0]\n"
+ "fmla v27.4s, v7.4s, v4.s[0]\n"
+ "fmla v31.4s, v7.4s, v5.s[0]\n"
+ "ldr q7, [x15, #0x50]\n"
+ "fmla v8.4s, v6.4s, v0.s[1]\n"
+ "fmla v12.4s, v6.4s, v1.s[1]\n"
+ "fmla v16.4s, v6.4s, v2.s[1]\n"
+ "fmla v20.4s, v6.4s, v3.s[1]\n"
+ "fmla v24.4s, v6.4s, v4.s[1]\n"
+ "fmla v28.4s, v6.4s, v5.s[1]\n"
+ "ldr q6, [x15, #0x60]\n"
+ "fmla v9.4s, v7.4s, v0.s[1]\n"
+ "fmla v13.4s, v7.4s, v1.s[1]\n"
+ "fmla v17.4s, v7.4s, v2.s[1]\n"
+ "fmla v21.4s, v7.4s, v3.s[1]\n"
+ "fmla v25.4s, v7.4s, v4.s[1]\n"
+ "fmla v29.4s, v7.4s, v5.s[1]\n"
+ "ldr q7, [x15, #0x70]\n"
+ "fmla v10.4s, v6.4s, v0.s[1]\n"
+ "fmla v14.4s, v6.4s, v1.s[1]\n"
+ "fmla v18.4s, v6.4s, v2.s[1]\n"
+ "fmla v22.4s, v6.4s, v3.s[1]\n"
+ "fmla v26.4s, v6.4s, v4.s[1]\n"
+ "fmla v30.4s, v6.4s, v5.s[1]\n"
+ "ldr q6, [x15, #0x80]\n"
+ "fmla v11.4s, v7.4s, v0.s[1]\n"
+ "fmla v15.4s, v7.4s, v1.s[1]\n"
+ "fmla v19.4s, v7.4s, v2.s[1]\n"
+ "fmla v23.4s, v7.4s, v3.s[1]\n"
+ "fmla v27.4s, v7.4s, v4.s[1]\n"
+ "fmla v31.4s, v7.4s, v5.s[1]\n"
+ "ldr q7, [x15, #0x90]\n"
+ "fmla v8.4s, v6.4s, v0.s[2]\n"
+ "fmla v12.4s, v6.4s, v1.s[2]\n"
+ "fmla v16.4s, v6.4s, v2.s[2]\n"
+ "fmla v20.4s, v6.4s, v3.s[2]\n"
+ "fmla v24.4s, v6.4s, v4.s[2]\n"
+ "fmla v28.4s, v6.4s, v5.s[2]\n"
+ "ldr q6, [x15, #0xa0]\n"
+ "fmla v9.4s, v7.4s, v0.s[2]\n"
+ "fmla v13.4s, v7.4s, v1.s[2]\n"
+ "fmla v17.4s, v7.4s, v2.s[2]\n"
+ "fmla v21.4s, v7.4s, v3.s[2]\n"
+ "fmla v25.4s, v7.4s, v4.s[2]\n"
+ "fmla v29.4s, v7.4s, v5.s[2]\n"
+ "ldr q7, [x15, #0xb0]\n"
+ "fmla v10.4s, v6.4s, v0.s[2]\n"
+ "fmla v14.4s, v6.4s, v1.s[2]\n"
+ "fmla v18.4s, v6.4s, v2.s[2]\n"
+ "fmla v22.4s, v6.4s, v3.s[2]\n"
+ "fmla v26.4s, v6.4s, v4.s[2]\n"
+ "fmla v30.4s, v6.4s, v5.s[2]\n"
+ "ldr q6, [x15, #0xc0]\n"
+ "fmla v11.4s, v7.4s, v0.s[2]\n"
+ "fmla v15.4s, v7.4s, v1.s[2]\n"
+ "fmla v19.4s, v7.4s, v2.s[2]\n"
+ "fmla v23.4s, v7.4s, v3.s[2]\n"
+ "fmla v27.4s, v7.4s, v4.s[2]\n"
+ "fmla v31.4s, v7.4s, v5.s[2]\n"
+ "ldr q7, [x15, #0xd0]\n"
+ "fmla v8.4s, v6.4s, v0.s[3]\n"
+ "fmla v12.4s, v6.4s, v1.s[3]\n"
+ "fmla v16.4s, v6.4s, v2.s[3]\n"
+ "fmla v20.4s, v6.4s, v3.s[3]\n"
+ "fmla v24.4s, v6.4s, v4.s[3]\n"
+ "fmla v28.4s, v6.4s, v5.s[3]\n"
+ "ldr q6, [x15, #0xe0]\n"
+ "fmla v9.4s, v7.4s, v0.s[3]\n"
+ "fmla v13.4s, v7.4s, v1.s[3]\n"
+ "fmla v17.4s, v7.4s, v2.s[3]\n"
+ "fmla v21.4s, v7.4s, v3.s[3]\n"
+ "fmla v25.4s, v7.4s, v4.s[3]\n"
+ "fmla v29.4s, v7.4s, v5.s[3]\n"
+ "ldr q7, [x15, #0xf0]\n"
+ "fmla v10.4s, v6.4s, v0.s[3]\n"
+ "add x15, x15, #0x100\n"
+ "fmla v14.4s, v6.4s, v1.s[3]\n"
+ "fmla v18.4s, v6.4s, v2.s[3]\n"
+ "fmla v22.4s, v6.4s, v3.s[3]\n"
+ "fmla v26.4s, v6.4s, v4.s[3]\n"
+ "fmla v30.4s, v6.4s, v5.s[3]\n"
+ "fmla v11.4s, v7.4s, v0.s[3]\n"
+ "fmla v15.4s, v7.4s, v1.s[3]\n"
+ "fmla v19.4s, v7.4s, v2.s[3]\n"
+ "fmla v23.4s, v7.4s, v3.s[3]\n"
+ "fmla v27.4s, v7.4s, v4.s[3]\n"
+ "fmla v31.4s, v7.4s, v5.s[3]\n"
+ "191:" // Height 6: Multiply loop: Main loop skip
+ "cbz x11, 193f\n"
+ "192:" // Height 6: Multiply loop: Odd block loop
+ "ldr s0, [x10], #0x4\n"
+ "ldr s1, [x28], #0x4\n"
+ "ldr s2, [x26], #0x4\n"
+ "ldr s3, [x24], #0x4\n"
+ "ldr s4, [x22], #0x4\n"
+ "ldr s5, [x20], #0x4\n"
+ "ldr q6, [x15, #0x0]\n"
+ "fmla v8.4s, v6.4s, v0.s[0]\n"
+ "ldr q7, [x15, #0x10]\n"
+ "fmla v12.4s, v6.4s, v1.s[0]\n"
+ "sub x11, x11, #0x1\n"
+ "fmla v16.4s, v6.4s, v2.s[0]\n"
+ "fmla v20.4s, v6.4s, v3.s[0]\n"
+ "fmla v24.4s, v6.4s, v4.s[0]\n"
+ "fmla v28.4s, v6.4s, v5.s[0]\n"
+ "ldr q6, [x15, #0x20]\n"
+ "fmla v9.4s, v7.4s, v0.s[0]\n"
+ "fmla v13.4s, v7.4s, v1.s[0]\n"
+ "fmla v17.4s, v7.4s, v2.s[0]\n"
+ "fmla v21.4s, v7.4s, v3.s[0]\n"
+ "fmla v25.4s, v7.4s, v4.s[0]\n"
+ "fmla v29.4s, v7.4s, v5.s[0]\n"
+ "ldr q7, [x15, #0x30]\n"
+ "fmla v10.4s, v6.4s, v0.s[0]\n"
+ "add x15, x15, #0x40\n"
+ "fmla v14.4s, v6.4s, v1.s[0]\n"
+ "fmla v18.4s, v6.4s, v2.s[0]\n"
+ "fmla v22.4s, v6.4s, v3.s[0]\n"
+ "fmla v26.4s, v6.4s, v4.s[0]\n"
+ "fmla v30.4s, v6.4s, v5.s[0]\n"
+ "fmla v11.4s, v7.4s, v0.s[0]\n"
+ "fmla v15.4s, v7.4s, v1.s[0]\n"
+ "fmla v19.4s, v7.4s, v2.s[0]\n"
+ "fmla v23.4s, v7.4s, v3.s[0]\n"
+ "fmla v27.4s, v7.4s, v4.s[0]\n"
+ "fmla v31.4s, v7.4s, v5.s[0]\n"
+ "cbnz x11, 192b\n"
+ "193:" // Height 6: Multiply loop: No odd multiplies
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "add x12, x12, #0x1\n"
+ "cmp x12, x19\n"
+ "bne 186b\n"
+ "prfm pstl1keep, [x13, #0x0]\n"
+ "prfm pstl1keep, [x9, #0x0]\n"
+ "prfm pstl1keep, [x27, #0x0]\n"
+ "prfm pstl1keep, [x25, #0x0]\n"
+ "prfm pstl1keep, [x23, #0x0]\n"
+ "prfm pstl1keep, [x21, #0x0]\n"
+ "tbz %x[flags], #1, 194f\n"
+ "add x19, %x[args_ptr], %[offset_min]\n"
+ "ld1r { v1.4s }, [x19]\n"
+ "add x19, %x[args_ptr], %[offset_max]\n"
+ "ld1r { v0.4s }, [x19]\n"
+ "fmin v8.4s, v8.4s, v0.4s\n"
+ "fmin v9.4s, v9.4s, v0.4s\n"
+ "fmin v10.4s, v10.4s, v0.4s\n"
+ "fmin v11.4s, v11.4s, v0.4s\n"
+ "fmax v8.4s, v8.4s, v1.4s\n"
+ "fmax v9.4s, v9.4s, v1.4s\n"
+ "fmax v10.4s, v10.4s, v1.4s\n"
+ "fmax v11.4s, v11.4s, v1.4s\n"
+ "fmin v12.4s, v12.4s, v0.4s\n"
+ "fmin v13.4s, v13.4s, v0.4s\n"
+ "fmin v14.4s, v14.4s, v0.4s\n"
+ "fmax v12.4s, v12.4s, v1.4s\n"
+ "fmax v13.4s, v13.4s, v1.4s\n"
+ "fmax v14.4s, v14.4s, v1.4s\n"
+ "fmin v15.4s, v15.4s, v0.4s\n"
+ "fmin v16.4s, v16.4s, v0.4s\n"
+ "fmin v17.4s, v17.4s, v0.4s\n"
+ "fmax v15.4s, v15.4s, v1.4s\n"
+ "fmax v16.4s, v16.4s, v1.4s\n"
+ "fmax v17.4s, v17.4s, v1.4s\n"
+ "fmin v18.4s, v18.4s, v0.4s\n"
+ "fmin v19.4s, v19.4s, v0.4s\n"
+ "fmin v20.4s, v20.4s, v0.4s\n"
+ "fmax v18.4s, v18.4s, v1.4s\n"
+ "fmax v19.4s, v19.4s, v1.4s\n"
+ "fmax v20.4s, v20.4s, v1.4s\n"
+ "fmin v21.4s, v21.4s, v0.4s\n"
+ "fmin v22.4s, v22.4s, v0.4s\n"
+ "fmin v23.4s, v23.4s, v0.4s\n"
+ "fmax v21.4s, v21.4s, v1.4s\n"
+ "fmax v22.4s, v22.4s, v1.4s\n"
+ "fmax v23.4s, v23.4s, v1.4s\n"
+ "fmin v24.4s, v24.4s, v0.4s\n"
+ "fmin v25.4s, v25.4s, v0.4s\n"
+ "fmin v26.4s, v26.4s, v0.4s\n"
+ "fmax v24.4s, v24.4s, v1.4s\n"
+ "fmax v25.4s, v25.4s, v1.4s\n"
+ "fmax v26.4s, v26.4s, v1.4s\n"
+ "fmin v27.4s, v27.4s, v0.4s\n"
+ "fmin v28.4s, v28.4s, v0.4s\n"
+ "fmin v29.4s, v29.4s, v0.4s\n"
+ "fmax v27.4s, v27.4s, v1.4s\n"
+ "fmax v28.4s, v28.4s, v1.4s\n"
+ "fmax v29.4s, v29.4s, v1.4s\n"
+ "fmin v30.4s, v30.4s, v0.4s\n"
+ "fmin v31.4s, v31.4s, v0.4s\n"
+ "fmax v30.4s, v30.4s, v1.4s\n"
+ "fmax v31.4s, v31.4s, v1.4s\n"
+ "194:" // Height 6: No activation
+ "cmp x16, #0x10\n"
+ "bge 203f\n"
+ "tbz x16, #3, 198f\n"
+ "st1 { v8.4s }, [x13], #0x10\n"
+ "st1 { v9.4s }, [x13], #0x10\n"
+ "st1 { v12.4s }, [x9], #0x10\n"
+ "st1 { v13.4s }, [x9], #0x10\n"
+ "st1 { v16.4s }, [x27], #0x10\n"
+ "st1 { v17.4s }, [x27], #0x10\n"
+ "st1 { v20.4s }, [x25], #0x10\n"
+ "st1 { v21.4s }, [x25], #0x10\n"
+ "st1 { v24.4s }, [x23], #0x10\n"
+ "st1 { v25.4s }, [x23], #0x10\n"
+ "st1 { v28.4s }, [x21], #0x10\n"
+ "st1 { v29.4s }, [x21], #0x10\n"
+ "tbz x16, #2, 196f\n"
+ "st1 { v10.4s }, [x13], #0x10\n"
+ "st1 { v14.4s }, [x9], #0x10\n"
+ "st1 { v18.4s }, [x27], #0x10\n"
+ "st1 { v22.4s }, [x25], #0x10\n"
+ "st1 { v26.4s }, [x23], #0x10\n"
+ "st1 { v30.4s }, [x21], #0x10\n"
+ "tbz x16, #1, 195f\n"
+ "str d11, [x13], #0x8\n"
+ "str d15, [x9], #0x8\n"
+ "str d19, [x27], #0x8\n"
+ "str d23, [x25], #0x8\n"
+ "str d27, [x23], #0x8\n"
+ "str d31, [x21], #0x8\n"
+ "tbz x16, #0, 202f\n"
+ "st1 { v11.s }[2], [x13]\n"
+ "st1 { v15.s }[2], [x9]\n"
+ "st1 { v19.s }[2], [x27]\n"
+ "st1 { v23.s }[2], [x25]\n"
+ "st1 { v27.s }[2], [x23]\n"
+ "st1 { v31.s }[2], [x21]\n"
+ "b 202f\n"
+ "195:" // Height 6: Partial direct writeback: partial_1_12
+ "tbz x16, #0, 202f\n"
+ "str s11, [x13, #0x0]\n"
+ "str s15, [x9, #0x0]\n"
+ "str s19, [x27, #0x0]\n"
+ "str s23, [x25, #0x0]\n"
+ "str s27, [x23, #0x0]\n"
+ "str s31, [x21, #0x0]\n"
+ "b 202f\n"
+ "196:" // Height 6: Partial direct writeback: partial_2_8
+ "tbz x16, #1, 197f\n"
+ "str d10, [x13], #0x8\n"
+ "str d14, [x9], #0x8\n"
+ "str d18, [x27], #0x8\n"
+ "str d22, [x25], #0x8\n"
+ "str d26, [x23], #0x8\n"
+ "str d30, [x21], #0x8\n"
+ "tbz x16, #0, 202f\n"
+ "st1 { v10.s }[2], [x13]\n"
+ "st1 { v14.s }[2], [x9]\n"
+ "st1 { v18.s }[2], [x27]\n"
+ "st1 { v22.s }[2], [x25]\n"
+ "st1 { v26.s }[2], [x23]\n"
+ "st1 { v30.s }[2], [x21]\n"
+ "b 202f\n"
+ "197:" // Height 6: Partial direct writeback: partial_1_8
+ "tbz x16, #0, 202f\n"
+ "str s10, [x13, #0x0]\n"
+ "str s14, [x9, #0x0]\n"
+ "str s18, [x27, #0x0]\n"
+ "str s22, [x25, #0x0]\n"
+ "str s26, [x23, #0x0]\n"
+ "str s30, [x21, #0x0]\n"
+ "b 202f\n"
+ "198:" // Height 6: Partial direct writeback: partial_4_0
+ "tbz x16, #2, 200f\n"
+ "st1 { v8.4s }, [x13], #0x10\n"
+ "st1 { v12.4s }, [x9], #0x10\n"
+ "st1 { v16.4s }, [x27], #0x10\n"
+ "st1 { v20.4s }, [x25], #0x10\n"
+ "st1 { v24.4s }, [x23], #0x10\n"
+ "st1 { v28.4s }, [x21], #0x10\n"
+ "tbz x16, #1, 199f\n"
+ "str d9, [x13], #0x8\n"
+ "str d13, [x9], #0x8\n"
+ "str d17, [x27], #0x8\n"
+ "str d21, [x25], #0x8\n"
+ "str d25, [x23], #0x8\n"
+ "str d29, [x21], #0x8\n"
+ "tbz x16, #0, 202f\n"
+ "st1 { v9.s }[2], [x13]\n"
+ "st1 { v13.s }[2], [x9]\n"
+ "st1 { v17.s }[2], [x27]\n"
+ "st1 { v21.s }[2], [x25]\n"
+ "st1 { v25.s }[2], [x23]\n"
+ "st1 { v29.s }[2], [x21]\n"
+ "b 202f\n"
+ "199:" // Height 6: Partial direct writeback: partial_1_4
+ "tbz x16, #0, 202f\n"
+ "str s9, [x13, #0x0]\n"
+ "str s13, [x9, #0x0]\n"
+ "str s17, [x27, #0x0]\n"
+ "str s21, [x25, #0x0]\n"
+ "str s25, [x23, #0x0]\n"
+ "str s29, [x21, #0x0]\n"
+ "b 202f\n"
+ "200:" // Height 6: Partial direct writeback: partial_2_0
+ "tbz x16, #1, 201f\n"
+ "str d8, [x13], #0x8\n"
+ "str d12, [x9], #0x8\n"
+ "str d16, [x27], #0x8\n"
+ "str d20, [x25], #0x8\n"
+ "str d24, [x23], #0x8\n"
+ "str d28, [x21], #0x8\n"
+ "tbz x16, #0, 202f\n"
+ "st1 { v8.s }[2], [x13]\n"
+ "st1 { v12.s }[2], [x9]\n"
+ "st1 { v16.s }[2], [x27]\n"
+ "st1 { v20.s }[2], [x25]\n"
+ "st1 { v24.s }[2], [x23]\n"
+ "st1 { v28.s }[2], [x21]\n"
+ "b 202f\n"
+ "201:" // Height 6: Partial direct writeback: partial_1_0
+ "str s8, [x13, #0x0]\n"
+ "str s12, [x9, #0x0]\n"
+ "str s16, [x27, #0x0]\n"
+ "str s20, [x25, #0x0]\n"
+ "str s24, [x23, #0x0]\n"
+ "str s28, [x21, #0x0]\n"
+ "202:" // Height 6: Partial direct writeback: Done
+ "b 204f\n"
+ "203:" // Height 6: Full writeback
+ "str q8, [x13, #0x0]\n"
+ "str q9, [x13, #0x10]\n"
+ "str q10, [x13, #0x20]\n"
+ "str q11, [x13, #0x30]\n"
+ "str q12, [x9, #0x0]\n"
+ "str q13, [x9, #0x10]\n"
+ "str q14, [x9, #0x20]\n"
+ "str q15, [x9, #0x30]\n"
+ "str q16, [x27, #0x0]\n"
+ "str q17, [x27, #0x10]\n"
+ "str q18, [x27, #0x20]\n"
+ "str q19, [x27, #0x30]\n"
+ "str q20, [x25, #0x0]\n"
+ "str q21, [x25, #0x10]\n"
+ "str q22, [x25, #0x20]\n"
+ "str q23, [x25, #0x30]\n"
+ "str q24, [x23, #0x0]\n"
+ "str q25, [x23, #0x10]\n"
+ "str q26, [x23, #0x20]\n"
+ "str q27, [x23, #0x30]\n"
+ "str q28, [x21, #0x0]\n"
+ "str q29, [x21, #0x10]\n"
+ "str q30, [x21, #0x20]\n"
+ "str q31, [x21, #0x30]\n"
+ "add x13, x13, #0x40\n"
+ "add x9, x9, #0x40\n"
+ "add x27, x27, #0x40\n"
+ "add x25, x25, #0x40\n"
+ "add x23, x23, #0x40\n"
+ "add x21, x21, #0x40\n"
+ "204:" // Height 6: Writeback done
+ "subs x16, x16, #0x10\n"
+ "bgt 173b\n"
+ "subs %x[M], %x[M], #0x6\n"
+ "beq 206f\n"
+ "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "tbz %x[flags], #3, 205f\n"
+ "add x20, x20, #0x6\n"
+ "str x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "b 1b\n"
+ "205:" // Update direct input
+ "mov x19, #0x18\n"
+ "madd %x[input_ptr], x19, x20, %x[input_ptr]\n"
+ "b 1b\n"
+ "206:" // Exit
+
+ : [M] "+r" (M), [input_ptr] "+r" (input_ptr), [output_ptr] "+r" (output_ptr)
+ : [args_ptr] "r" (&ka), [bias] "r" (bias), [flags] "r" (flags), [offset_max] "I" (offsetof(KernelArgs, maxval)), [offset_min] "I" (offsetof(KernelArgs, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths))
+ : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
+ );
+}
+
+} // namespace arm_gemm
+#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_4x8.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_8x4.hpp
similarity index 69%
copy from src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_4x8.hpp
copy to src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_8x4.hpp
index e9a0948..043d064 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_4x8.hpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_8x4.hpp
@@ -10,34 +10,43 @@
* sell copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
*/
#pragma once
-
#ifdef __aarch64__
+#include "../std_transforms_fixed.hpp"
+
+#define ARGLIST \
+ unsigned int, const unsigned int *, \
+ IndirectInputArg<float>, \
+ size_t, size_t, \
+ const float *, \
+ IndirectOutputArg<float>, \
+ const float *, Activation, bool
+
namespace arm_gemm
{
// Actual kernel implementations
-void a64_smallK_hybrid_fp32_mla_4x8(const float *, int, const float *, float *, int, int, int, int, const float *, Activation, bool);
+void a64_hybrid_fp32_mla_8x4( ARGLIST );
-class smallK_hybrid_fp32_mla_4x8
+class cls_a64_hybrid_fp32_mla_8x4
{
public:
typedef float operand_type;
typedef float result_type;
- typedef void (*kern_type)(const float *, int, const float *, float *, int, int, int, int, const float *, Activation, bool);
+ typedef void (*kern_type)( ARGLIST );
/* Kernel blocking parameters */
static constexpr unsigned int out_height()
@@ -57,30 +66,20 @@
static constexpr bool supports_accumulate()
{
- return false;
- }
-
- static constexpr bool supports_bias()
- {
- return true;
- }
-
- static constexpr bool supports_activation()
- {
return true;
}
StdTransformsFixed<operand_type, result_type, 8, 4, 1> transforms = {};
// Default to the generic kernel
- kern_type kernel=a64_smallK_hybrid_fp32_mla_4x8;
+ kern_type kernel=a64_hybrid_fp32_mla_8x4;
- smallK_hybrid_fp32_mla_4x8(const CPUInfo *)
+ cls_a64_hybrid_fp32_mla_8x4(const CPUInfo *)
{
-
}
};
} // namespace arm_gemm
+#undef ARGLIST
#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_8x4/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_8x4/generic.cpp
new file mode 100644
index 0000000..3ab6cad
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_8x4/generic.cpp
@@ -0,0 +1,2195 @@
+/*
+ * Copyright (c) 2019-2020 Arm Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+#ifdef __aarch64__
+
+#include "arm_gemm.hpp"
+#include "../../utils.hpp"
+
+#include <cassert>
+
+namespace arm_gemm {
+
+void a64_hybrid_fp32_mla_8x4 (
+ unsigned int num_strings, const unsigned int *string_lengths, IndirectInputArg<float> A_arg,
+ size_t M, size_t N, const float *B_ptr, IndirectOutputArg<float> output_arg,
+ const float *bias, Activation act, bool accumulate
+)
+{
+ struct KernelArgs {
+ float maxval = static_cast<float>(std::numeric_limits<float>::infinity());
+ float minval = - static_cast<float>(std::numeric_limits<float>::infinity());
+ unsigned int num_strings = {};
+ const unsigned int *string_lengths = {};
+ size_t N = {};
+ const float *B_ptr = {};
+ size_t output_offset = {};
+ size_t input_initial_col = {};
+ size_t input_offset = {};
+ } ka;
+
+ unsigned long flags=0;
+ void *output_ptr;
+ void *input_ptr;
+
+ if (output_arg.is_indirect) {
+ output_ptr=(void *)(output_arg.indirect.ptr);
+ ka.output_offset=output_arg.indirect.offset;
+ flags |= 0x4;
+ } else {
+ output_ptr=(void *)(output_arg.direct.base);
+ ka.output_offset=output_arg.direct.stride;
+ }
+
+ if (A_arg.is_indirect) {
+ input_ptr=(void *)(A_arg.indirect.ptr);
+ ka.input_offset=A_arg.indirect.start_row;
+ ka.input_initial_col=A_arg.indirect.start_col;
+ flags |= 0x8;
+ } else {
+ assert(num_strings==1);
+ input_ptr=(void *)(A_arg.direct.base);
+ ka.input_offset=A_arg.direct.stride;
+ }
+ if (accumulate) {
+ flags |= 0x1;
+ }
+ ka.num_strings = num_strings;
+ ka.string_lengths = string_lengths;
+ ka.N = N;
+ ka.B_ptr = B_ptr;
+ switch(act.type) {
+ default:
+ case Activation::Type::None:
+ break;
+ case Activation::Type::BoundedReLU:
+ ka.maxval = static_cast<float>(act.param1);
+ /* fall through */
+ case Activation::Type::ReLU:
+ ka.minval = 0;
+ flags |= 0x2;
+ break;
+ }
+ __asm__ __volatile__(
+
+ "1:" // Row loop
+ "cmp %x[M], #0x8\n"
+ "bge 155f\n"
+ "cmp %x[M], #0x6\n"
+ "bgt 133f\n"
+ "beq 111f\n"
+ "cmp %x[M], #0x4\n"
+ "bgt 89f\n"
+ "beq 67f\n"
+ "cmp %x[M], #0x2\n"
+ "bgt 45f\n"
+ "beq 23f\n"
+ "ldr x6, [%x[args_ptr], %[offsetof_N]]\n"
+ "ldr x7, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "mov x8, %x[bias]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 2f\n"
+ "ldr x17, [%x[output_ptr], #0x0]\n"
+ "add x17, x17, x19, LSL #2\n"
+ "b 3f\n"
+ "2:" // Height 1: setup direct output
+ "mov x17, %x[output_ptr]\n"
+ "3:" // Height 1: Column loop
+ "cbz x8, 4f\n"
+ "ldr q24, [x8, #0x0]\n"
+ "add x8, x8, #0x10\n"
+ "b 9f\n"
+ "4:" // Height 1: no bias
+ "tbz %x[flags], #0, 8f\n"
+ "cmp x6, #0x4\n"
+ "bge 7f\n"
+ "tbz x6, #1, 5f\n"
+ "ldr d24, [x17], #0x8\n"
+ "mov x19, #0x8\n"
+ "tbz x6, #0, 6f\n"
+ "ld1 { v24.s }[2], [x17]\n"
+ "b 6f\n"
+ "5:" // Height 1: Partial accumulate: partial_1_0
+ "mov x19, #0x0\n"
+ "ldr s24, [x17, #0x0]\n"
+ "6:" // Height 1: Partial accumulate: Done
+ "sub x17, x17, x19\n"
+ "b 9f\n"
+ "7:" // Height 1: full accumulate
+ "ldr q24, [x17, #0x0]\n"
+ "b 9f\n"
+ "8:" // Height 1: no accumulate
+ "movi v24.16b, #0x0\n"
+ "9:" // Height 1: setup done
+ "mov x16, #0x0\n"
+ "10:" // Height 1: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w15, [x20, x16, LSL #0x2]\n"
+ "tbz %x[flags], #3, 11f\n"
+ "ldr x20, [%x[input_ptr], x16, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x14, [x20, #0x0]\n"
+ "cbnz x16, 12f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x14, x14, x19, LSL #2\n"
+ "b 12f\n"
+ "11:" // Height 1: setup direct input
+ "mov x14, %x[input_ptr]\n"
+ "12:" // Height 1: input setup done
+ "cmp x15, #0x4\n"
+ "blt 15f\n"
+ "cmp x15, #0x8\n"
+ "blt 14f\n"
+ "13:" // Height 1: Multiply loop: Main loop head
+ "ldr q0, [x14, #0x0]\n"
+ "ldr q8, [x7, #0x0]\n"
+ "fmla v24.4s, v8.4s, v0.s[0]\n"
+ "ldr q9, [x7, #0x10]\n"
+ "ldr q10, [x7, #0x20]\n"
+ "fmla v24.4s, v9.4s, v0.s[1]\n"
+ "ldr q11, [x7, #0x30]\n"
+ "add x14, x14, #0x10\n"
+ "fmla v24.4s, v10.4s, v0.s[2]\n"
+ "prfm pldl1keep, [x14, #0x80]\n"
+ "sub x15, x15, #0x4\n"
+ "fmla v24.4s, v11.4s, v0.s[3]\n"
+ "cmp x15, #0x8\n"
+ "add x7, x7, #0x40\n"
+ "bge 13b\n"
+ "14:" // Height 1: Multiply loop: Single iteration only
+ "sub x15, x15, #0x4\n"
+ "ldr q0, [x14, #0x0]\n"
+ "ldr q12, [x7, #0x0]\n"
+ "fmla v24.4s, v12.4s, v0.s[0]\n"
+ "ldr q13, [x7, #0x10]\n"
+ "ldr q14, [x7, #0x20]\n"
+ "fmla v24.4s, v13.4s, v0.s[1]\n"
+ "ldr q15, [x7, #0x30]\n"
+ "add x14, x14, #0x10\n"
+ "fmla v24.4s, v14.4s, v0.s[2]\n"
+ "prfm pldl1keep, [x14, #0x80]\n"
+ "add x7, x7, #0x40\n"
+ "fmla v24.4s, v15.4s, v0.s[3]\n"
+ "15:" // Height 1: Multiply loop: Main loop skip
+ "cbz x15, 17f\n"
+ "16:" // Height 1: Multiply loop: Odd block loop
+ "ldr s0, [x14], #0x4\n"
+ "ldr q16, [x7, #0x0]\n"
+ "fmla v24.4s, v16.4s, v0.s[0]\n"
+ "sub x15, x15, #0x1\n"
+ "add x7, x7, #0x10\n"
+ "cbnz x15, 16b\n"
+ "17:" // Height 1: Multiply loop: No odd multiplies
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "add x16, x16, #0x1\n"
+ "cmp x16, x19\n"
+ "bne 10b\n"
+ "prfm pstl1keep, [x17, #0x0]\n"
+ "tbz %x[flags], #1, 18f\n"
+ "add x19, %x[args_ptr], %[offset_min]\n"
+ "ld1r { v17.4s }, [x19]\n"
+ "add x19, %x[args_ptr], %[offset_max]\n"
+ "ld1r { v16.4s }, [x19]\n"
+ "fmin v24.4s, v24.4s, v16.4s\n"
+ "fmax v24.4s, v24.4s, v17.4s\n"
+ "18:" // Height 1: No activation
+ "cmp x6, #0x4\n"
+ "bge 21f\n"
+ "tbz x6, #1, 19f\n"
+ "str d24, [x17], #0x8\n"
+ "tbz x6, #0, 20f\n"
+ "st1 { v24.s }[2], [x17]\n"
+ "b 20f\n"
+ "19:" // Height 1: Partial direct writeback: partial_1_0
+ "str s24, [x17, #0x0]\n"
+ "20:" // Height 1: Partial direct writeback: Done
+ "b 22f\n"
+ "21:" // Height 1: Full writeback
+ "str q24, [x17, #0x0]\n"
+ "add x17, x17, #0x10\n"
+ "22:" // Height 1: Writeback done
+ "subs x6, x6, #0x4\n"
+ "bgt 3b\n"
+ "b 178f\n"
+ "23:" // Height 2
+ "ldr x6, [%x[args_ptr], %[offsetof_N]]\n"
+ "mov x8, %x[bias]\n"
+ "ldr x7, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 24f\n"
+ "ldr x17, [%x[output_ptr], #0x0]\n"
+ "add x17, x17, x19, LSL #2\n"
+ "ldr x13, [%x[output_ptr], #0x8]\n"
+ "add x13, x13, x19, LSL #2\n"
+ "b 25f\n"
+ "24:" // Height 2: setup direct output
+ "mov x17, %x[output_ptr]\n"
+ "add x13, x17, x19, LSL #2\n"
+ "25:" // Height 2: Column loop
+ "cbz x8, 26f\n"
+ "ldr q24, [x8, #0x0]\n"
+ "mov v25.16b, v24.16b\n"
+ "add x8, x8, #0x10\n"
+ "b 31f\n"
+ "26:" // Height 2: no bias
+ "tbz %x[flags], #0, 30f\n"
+ "cmp x6, #0x4\n"
+ "bge 29f\n"
+ "tbz x6, #1, 27f\n"
+ "ldr d24, [x17], #0x8\n"
+ "ldr d25, [x13], #0x8\n"
+ "mov x19, #0x8\n"
+ "tbz x6, #0, 28f\n"
+ "ld1 { v24.s }[2], [x17]\n"
+ "ld1 { v25.s }[2], [x13]\n"
+ "b 28f\n"
+ "27:" // Height 2: Partial accumulate: partial_1_0
+ "mov x19, #0x0\n"
+ "ldr s24, [x17, #0x0]\n"
+ "ldr s25, [x13, #0x0]\n"
+ "28:" // Height 2: Partial accumulate: Done
+ "sub x17, x17, x19\n"
+ "sub x13, x13, x19\n"
+ "b 31f\n"
+ "29:" // Height 2: full accumulate
+ "ldr q24, [x17, #0x0]\n"
+ "ldr q25, [x13, #0x0]\n"
+ "b 31f\n"
+ "30:" // Height 2: no accumulate
+ "movi v24.16b, #0x0\n"
+ "movi v25.16b, #0x0\n"
+ "31:" // Height 2: setup done
+ "mov x16, #0x0\n"
+ "32:" // Height 2: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w15, [x20, x16, LSL #0x2]\n"
+ "tbz %x[flags], #3, 33f\n"
+ "ldr x20, [%x[input_ptr], x16, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x14, [x20, #0x0]\n"
+ "ldr x12, [x20, #0x8]\n"
+ "cbnz x16, 34f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x14, x14, x19, LSL #2\n"
+ "add x12, x12, x19, LSL #2\n"
+ "b 34f\n"
+ "33:" // Height 2: setup direct input
+ "mov x14, %x[input_ptr]\n"
+ "add x12, x14, x19, LSL #2\n"
+ "34:" // Height 2: input setup done
+ "cmp x15, #0x4\n"
+ "blt 37f\n"
+ "cmp x15, #0x8\n"
+ "blt 36f\n"
+ "35:" // Height 2: Multiply loop: Main loop head
+ "ldr q0, [x14, #0x0]\n"
+ "ldr q1, [x12, #0x0]\n"
+ "ldr q8, [x7, #0x0]\n"
+ "fmla v24.4s, v8.4s, v0.s[0]\n"
+ "ldr q9, [x7, #0x10]\n"
+ "fmla v25.4s, v8.4s, v1.s[0]\n"
+ "ldr q10, [x7, #0x20]\n"
+ "ldr q11, [x7, #0x30]\n"
+ "fmla v24.4s, v9.4s, v0.s[1]\n"
+ "add x14, x14, #0x10\n"
+ "prfm pldl1keep, [x14, #0x80]\n"
+ "fmla v25.4s, v9.4s, v1.s[1]\n"
+ "add x12, x12, #0x10\n"
+ "fmla v24.4s, v10.4s, v0.s[2]\n"
+ "prfm pldl1keep, [x12, #0x80]\n"
+ "sub x15, x15, #0x4\n"
+ "fmla v25.4s, v10.4s, v1.s[2]\n"
+ "cmp x15, #0x8\n"
+ "add x7, x7, #0x40\n"
+ "fmla v24.4s, v11.4s, v0.s[3]\n"
+ "fmla v25.4s, v11.4s, v1.s[3]\n"
+ "bge 35b\n"
+ "36:" // Height 2: Multiply loop: Single iteration only
+ "sub x15, x15, #0x4\n"
+ "ldr q0, [x14, #0x0]\n"
+ "ldr q1, [x12, #0x0]\n"
+ "ldr q12, [x7, #0x0]\n"
+ "fmla v24.4s, v12.4s, v0.s[0]\n"
+ "ldr q13, [x7, #0x10]\n"
+ "fmla v25.4s, v12.4s, v1.s[0]\n"
+ "ldr q14, [x7, #0x20]\n"
+ "ldr q15, [x7, #0x30]\n"
+ "fmla v24.4s, v13.4s, v0.s[1]\n"
+ "add x14, x14, #0x10\n"
+ "prfm pldl1keep, [x14, #0x80]\n"
+ "fmla v25.4s, v13.4s, v1.s[1]\n"
+ "add x12, x12, #0x10\n"
+ "fmla v24.4s, v14.4s, v0.s[2]\n"
+ "prfm pldl1keep, [x12, #0x80]\n"
+ "add x7, x7, #0x40\n"
+ "fmla v25.4s, v14.4s, v1.s[2]\n"
+ "fmla v24.4s, v15.4s, v0.s[3]\n"
+ "fmla v25.4s, v15.4s, v1.s[3]\n"
+ "37:" // Height 2: Multiply loop: Main loop skip
+ "cbz x15, 39f\n"
+ "38:" // Height 2: Multiply loop: Odd block loop
+ "ldr s0, [x14], #0x4\n"
+ "ldr s1, [x12], #0x4\n"
+ "ldr q16, [x7, #0x0]\n"
+ "fmla v24.4s, v16.4s, v0.s[0]\n"
+ "sub x15, x15, #0x1\n"
+ "fmla v25.4s, v16.4s, v1.s[0]\n"
+ "add x7, x7, #0x10\n"
+ "cbnz x15, 38b\n"
+ "39:" // Height 2: Multiply loop: No odd multiplies
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "add x16, x16, #0x1\n"
+ "cmp x16, x19\n"
+ "bne 32b\n"
+ "prfm pstl1keep, [x17, #0x0]\n"
+ "prfm pstl1keep, [x13, #0x0]\n"
+ "tbz %x[flags], #1, 40f\n"
+ "add x19, %x[args_ptr], %[offset_min]\n"
+ "ld1r { v17.4s }, [x19]\n"
+ "add x19, %x[args_ptr], %[offset_max]\n"
+ "ld1r { v16.4s }, [x19]\n"
+ "fmin v24.4s, v24.4s, v16.4s\n"
+ "fmin v25.4s, v25.4s, v16.4s\n"
+ "fmax v24.4s, v24.4s, v17.4s\n"
+ "fmax v25.4s, v25.4s, v17.4s\n"
+ "40:" // Height 2: No activation
+ "cmp x6, #0x4\n"
+ "bge 43f\n"
+ "tbz x6, #1, 41f\n"
+ "str d24, [x17], #0x8\n"
+ "str d25, [x13], #0x8\n"
+ "tbz x6, #0, 42f\n"
+ "st1 { v24.s }[2], [x17]\n"
+ "st1 { v25.s }[2], [x13]\n"
+ "b 42f\n"
+ "41:" // Height 2: Partial direct writeback: partial_1_0
+ "str s24, [x17, #0x0]\n"
+ "str s25, [x13, #0x0]\n"
+ "42:" // Height 2: Partial direct writeback: Done
+ "b 44f\n"
+ "43:" // Height 2: Full writeback
+ "str q24, [x17, #0x0]\n"
+ "str q25, [x13, #0x0]\n"
+ "add x17, x17, #0x10\n"
+ "add x13, x13, #0x10\n"
+ "44:" // Height 2: Writeback done
+ "subs x6, x6, #0x4\n"
+ "bgt 25b\n"
+ "b 178f\n"
+ "45:" // Height 3
+ "ldr x6, [%x[args_ptr], %[offsetof_N]]\n"
+ "mov x8, %x[bias]\n"
+ "ldr x7, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 46f\n"
+ "ldr x17, [%x[output_ptr], #0x0]\n"
+ "add x17, x17, x19, LSL #2\n"
+ "ldr x13, [%x[output_ptr], #0x8]\n"
+ "ldr x11, [%x[output_ptr], #0x10]\n"
+ "add x13, x13, x19, LSL #2\n"
+ "add x11, x11, x19, LSL #2\n"
+ "b 47f\n"
+ "46:" // Height 3: setup direct output
+ "mov x17, %x[output_ptr]\n"
+ "add x13, x17, x19, LSL #2\n"
+ "add x11, x13, x19, LSL #2\n"
+ "47:" // Height 3: Column loop
+ "cbz x8, 48f\n"
+ "ldr q24, [x8, #0x0]\n"
+ "mov v25.16b, v24.16b\n"
+ "add x8, x8, #0x10\n"
+ "mov v26.16b, v24.16b\n"
+ "b 53f\n"
+ "48:" // Height 3: no bias
+ "tbz %x[flags], #0, 52f\n"
+ "cmp x6, #0x4\n"
+ "bge 51f\n"
+ "tbz x6, #1, 49f\n"
+ "ldr d24, [x17], #0x8\n"
+ "ldr d25, [x13], #0x8\n"
+ "ldr d26, [x11], #0x8\n"
+ "mov x19, #0x8\n"
+ "tbz x6, #0, 50f\n"
+ "ld1 { v24.s }[2], [x17]\n"
+ "ld1 { v25.s }[2], [x13]\n"
+ "ld1 { v26.s }[2], [x11]\n"
+ "b 50f\n"
+ "49:" // Height 3: Partial accumulate: partial_1_0
+ "mov x19, #0x0\n"
+ "ldr s24, [x17, #0x0]\n"
+ "ldr s25, [x13, #0x0]\n"
+ "ldr s26, [x11, #0x0]\n"
+ "50:" // Height 3: Partial accumulate: Done
+ "sub x17, x17, x19\n"
+ "sub x13, x13, x19\n"
+ "sub x11, x11, x19\n"
+ "b 53f\n"
+ "51:" // Height 3: full accumulate
+ "ldr q24, [x17, #0x0]\n"
+ "ldr q25, [x13, #0x0]\n"
+ "ldr q26, [x11, #0x0]\n"
+ "b 53f\n"
+ "52:" // Height 3: no accumulate
+ "movi v24.16b, #0x0\n"
+ "movi v25.16b, #0x0\n"
+ "movi v26.16b, #0x0\n"
+ "53:" // Height 3: setup done
+ "mov x16, #0x0\n"
+ "54:" // Height 3: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w15, [x20, x16, LSL #0x2]\n"
+ "tbz %x[flags], #3, 55f\n"
+ "ldr x20, [%x[input_ptr], x16, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x14, [x20, #0x0]\n"
+ "ldr x12, [x20, #0x8]\n"
+ "ldr x10, [x20, #0x10]\n"
+ "cbnz x16, 56f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x14, x14, x19, LSL #2\n"
+ "add x12, x12, x19, LSL #2\n"
+ "add x10, x10, x19, LSL #2\n"
+ "b 56f\n"
+ "55:" // Height 3: setup direct input
+ "mov x14, %x[input_ptr]\n"
+ "add x12, x14, x19, LSL #2\n"
+ "add x10, x12, x19, LSL #2\n"
+ "56:" // Height 3: input setup done
+ "cmp x15, #0x4\n"
+ "blt 59f\n"
+ "cmp x15, #0x8\n"
+ "blt 58f\n"
+ "57:" // Height 3: Multiply loop: Main loop head
+ "ldr q0, [x14, #0x0]\n"
+ "ldr q1, [x12, #0x0]\n"
+ "ldr q2, [x10, #0x0]\n"
+ "ldr q8, [x7, #0x0]\n"
+ "fmla v24.4s, v8.4s, v0.s[0]\n"
+ "ldr q9, [x7, #0x10]\n"
+ "fmla v25.4s, v8.4s, v1.s[0]\n"
+ "ldr q10, [x7, #0x20]\n"
+ "fmla v26.4s, v8.4s, v2.s[0]\n"
+ "ldr q11, [x7, #0x30]\n"
+ "add x14, x14, #0x10\n"
+ "fmla v24.4s, v9.4s, v0.s[1]\n"
+ "prfm pldl1keep, [x14, #0x80]\n"
+ "add x12, x12, #0x10\n"
+ "fmla v25.4s, v9.4s, v1.s[1]\n"
+ "prfm pldl1keep, [x12, #0x80]\n"
+ "add x10, x10, #0x10\n"
+ "fmla v26.4s, v9.4s, v2.s[1]\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "sub x15, x15, #0x4\n"
+ "fmla v24.4s, v10.4s, v0.s[2]\n"
+ "cmp x15, #0x8\n"
+ "add x7, x7, #0x40\n"
+ "fmla v25.4s, v10.4s, v1.s[2]\n"
+ "fmla v26.4s, v10.4s, v2.s[2]\n"
+ "fmla v24.4s, v11.4s, v0.s[3]\n"
+ "fmla v25.4s, v11.4s, v1.s[3]\n"
+ "fmla v26.4s, v11.4s, v2.s[3]\n"
+ "bge 57b\n"
+ "58:" // Height 3: Multiply loop: Single iteration only
+ "sub x15, x15, #0x4\n"
+ "ldr q0, [x14, #0x0]\n"
+ "ldr q1, [x12, #0x0]\n"
+ "ldr q2, [x10, #0x0]\n"
+ "ldr q12, [x7, #0x0]\n"
+ "fmla v24.4s, v12.4s, v0.s[0]\n"
+ "ldr q13, [x7, #0x10]\n"
+ "fmla v25.4s, v12.4s, v1.s[0]\n"
+ "ldr q14, [x7, #0x20]\n"
+ "fmla v26.4s, v12.4s, v2.s[0]\n"
+ "ldr q15, [x7, #0x30]\n"
+ "add x14, x14, #0x10\n"
+ "fmla v24.4s, v13.4s, v0.s[1]\n"
+ "prfm pldl1keep, [x14, #0x80]\n"
+ "add x12, x12, #0x10\n"
+ "fmla v25.4s, v13.4s, v1.s[1]\n"
+ "prfm pldl1keep, [x12, #0x80]\n"
+ "add x10, x10, #0x10\n"
+ "fmla v26.4s, v13.4s, v2.s[1]\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x7, x7, #0x40\n"
+ "fmla v24.4s, v14.4s, v0.s[2]\n"
+ "fmla v25.4s, v14.4s, v1.s[2]\n"
+ "fmla v26.4s, v14.4s, v2.s[2]\n"
+ "fmla v24.4s, v15.4s, v0.s[3]\n"
+ "fmla v25.4s, v15.4s, v1.s[3]\n"
+ "fmla v26.4s, v15.4s, v2.s[3]\n"
+ "59:" // Height 3: Multiply loop: Main loop skip
+ "cbz x15, 61f\n"
+ "60:" // Height 3: Multiply loop: Odd block loop
+ "ldr s0, [x14], #0x4\n"
+ "ldr s1, [x12], #0x4\n"
+ "ldr s2, [x10], #0x4\n"
+ "ldr q16, [x7, #0x0]\n"
+ "fmla v24.4s, v16.4s, v0.s[0]\n"
+ "sub x15, x15, #0x1\n"
+ "fmla v25.4s, v16.4s, v1.s[0]\n"
+ "add x7, x7, #0x10\n"
+ "fmla v26.4s, v16.4s, v2.s[0]\n"
+ "cbnz x15, 60b\n"
+ "61:" // Height 3: Multiply loop: No odd multiplies
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "add x16, x16, #0x1\n"
+ "cmp x16, x19\n"
+ "bne 54b\n"
+ "prfm pstl1keep, [x17, #0x0]\n"
+ "prfm pstl1keep, [x13, #0x0]\n"
+ "prfm pstl1keep, [x11, #0x0]\n"
+ "tbz %x[flags], #1, 62f\n"
+ "add x19, %x[args_ptr], %[offset_min]\n"
+ "ld1r { v17.4s }, [x19]\n"
+ "add x19, %x[args_ptr], %[offset_max]\n"
+ "ld1r { v16.4s }, [x19]\n"
+ "fmin v24.4s, v24.4s, v16.4s\n"
+ "fmin v25.4s, v25.4s, v16.4s\n"
+ "fmin v26.4s, v26.4s, v16.4s\n"
+ "fmax v24.4s, v24.4s, v17.4s\n"
+ "fmax v25.4s, v25.4s, v17.4s\n"
+ "fmax v26.4s, v26.4s, v17.4s\n"
+ "62:" // Height 3: No activation
+ "cmp x6, #0x4\n"
+ "bge 65f\n"
+ "tbz x6, #1, 63f\n"
+ "str d24, [x17], #0x8\n"
+ "str d25, [x13], #0x8\n"
+ "str d26, [x11], #0x8\n"
+ "tbz x6, #0, 64f\n"
+ "st1 { v24.s }[2], [x17]\n"
+ "st1 { v25.s }[2], [x13]\n"
+ "st1 { v26.s }[2], [x11]\n"
+ "b 64f\n"
+ "63:" // Height 3: Partial direct writeback: partial_1_0
+ "str s24, [x17, #0x0]\n"
+ "str s25, [x13, #0x0]\n"
+ "str s26, [x11, #0x0]\n"
+ "64:" // Height 3: Partial direct writeback: Done
+ "b 66f\n"
+ "65:" // Height 3: Full writeback
+ "str q24, [x17, #0x0]\n"
+ "str q25, [x13, #0x0]\n"
+ "str q26, [x11, #0x0]\n"
+ "add x17, x17, #0x10\n"
+ "add x13, x13, #0x10\n"
+ "add x11, x11, #0x10\n"
+ "66:" // Height 3: Writeback done
+ "subs x6, x6, #0x4\n"
+ "bgt 47b\n"
+ "b 178f\n"
+ "67:" // Height 4
+ "ldr x6, [%x[args_ptr], %[offsetof_N]]\n"
+ "mov x8, %x[bias]\n"
+ "ldr x7, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 68f\n"
+ "ldr x17, [%x[output_ptr], #0x0]\n"
+ "add x17, x17, x19, LSL #2\n"
+ "ldr x13, [%x[output_ptr], #0x8]\n"
+ "ldr x11, [%x[output_ptr], #0x10]\n"
+ "add x13, x13, x19, LSL #2\n"
+ "ldr x9, [%x[output_ptr], #0x18]\n"
+ "add x11, x11, x19, LSL #2\n"
+ "add x9, x9, x19, LSL #2\n"
+ "b 69f\n"
+ "68:" // Height 4: setup direct output
+ "mov x17, %x[output_ptr]\n"
+ "add x13, x17, x19, LSL #2\n"
+ "add x11, x13, x19, LSL #2\n"
+ "add x9, x11, x19, LSL #2\n"
+ "69:" // Height 4: Column loop
+ "cbz x8, 70f\n"
+ "ldr q24, [x8, #0x0]\n"
+ "mov v25.16b, v24.16b\n"
+ "add x8, x8, #0x10\n"
+ "mov v26.16b, v24.16b\n"
+ "mov v27.16b, v24.16b\n"
+ "b 75f\n"
+ "70:" // Height 4: no bias
+ "tbz %x[flags], #0, 74f\n"
+ "cmp x6, #0x4\n"
+ "bge 73f\n"
+ "tbz x6, #1, 71f\n"
+ "ldr d24, [x17], #0x8\n"
+ "ldr d25, [x13], #0x8\n"
+ "ldr d26, [x11], #0x8\n"
+ "ldr d27, [x9], #0x8\n"
+ "mov x19, #0x8\n"
+ "tbz x6, #0, 72f\n"
+ "ld1 { v24.s }[2], [x17]\n"
+ "ld1 { v25.s }[2], [x13]\n"
+ "ld1 { v26.s }[2], [x11]\n"
+ "ld1 { v27.s }[2], [x9]\n"
+ "b 72f\n"
+ "71:" // Height 4: Partial accumulate: partial_1_0
+ "mov x19, #0x0\n"
+ "ldr s24, [x17, #0x0]\n"
+ "ldr s25, [x13, #0x0]\n"
+ "ldr s26, [x11, #0x0]\n"
+ "ldr s27, [x9, #0x0]\n"
+ "72:" // Height 4: Partial accumulate: Done
+ "sub x17, x17, x19\n"
+ "sub x13, x13, x19\n"
+ "sub x11, x11, x19\n"
+ "sub x9, x9, x19\n"
+ "b 75f\n"
+ "73:" // Height 4: full accumulate
+ "ldr q24, [x17, #0x0]\n"
+ "ldr q25, [x13, #0x0]\n"
+ "ldr q26, [x11, #0x0]\n"
+ "ldr q27, [x9, #0x0]\n"
+ "b 75f\n"
+ "74:" // Height 4: no accumulate
+ "movi v24.16b, #0x0\n"
+ "movi v25.16b, #0x0\n"
+ "movi v26.16b, #0x0\n"
+ "movi v27.16b, #0x0\n"
+ "75:" // Height 4: setup done
+ "mov x16, #0x0\n"
+ "76:" // Height 4: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w15, [x20, x16, LSL #0x2]\n"
+ "tbz %x[flags], #3, 77f\n"
+ "ldr x20, [%x[input_ptr], x16, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x14, [x20, #0x0]\n"
+ "ldr x12, [x20, #0x8]\n"
+ "ldr x10, [x20, #0x10]\n"
+ "ldr x28, [x20, #0x18]\n"
+ "cbnz x16, 78f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x14, x14, x19, LSL #2\n"
+ "add x12, x12, x19, LSL #2\n"
+ "add x10, x10, x19, LSL #2\n"
+ "add x28, x28, x19, LSL #2\n"
+ "b 78f\n"
+ "77:" // Height 4: setup direct input
+ "mov x14, %x[input_ptr]\n"
+ "add x12, x14, x19, LSL #2\n"
+ "add x10, x12, x19, LSL #2\n"
+ "add x28, x10, x19, LSL #2\n"
+ "78:" // Height 4: input setup done
+ "cmp x15, #0x4\n"
+ "blt 81f\n"
+ "cmp x15, #0x8\n"
+ "blt 80f\n"
+ "79:" // Height 4: Multiply loop: Main loop head
+ "ldr q0, [x14, #0x0]\n"
+ "ldr q1, [x12, #0x0]\n"
+ "ldr q2, [x10, #0x0]\n"
+ "ldr q3, [x28, #0x0]\n"
+ "ldr q8, [x7, #0x0]\n"
+ "fmla v24.4s, v8.4s, v0.s[0]\n"
+ "ldr q9, [x7, #0x10]\n"
+ "fmla v25.4s, v8.4s, v1.s[0]\n"
+ "ldr q10, [x7, #0x20]\n"
+ "fmla v26.4s, v8.4s, v2.s[0]\n"
+ "ldr q11, [x7, #0x30]\n"
+ "fmla v27.4s, v8.4s, v3.s[0]\n"
+ "add x14, x14, #0x10\n"
+ "prfm pldl1keep, [x14, #0x80]\n"
+ "fmla v24.4s, v9.4s, v0.s[1]\n"
+ "add x12, x12, #0x10\n"
+ "prfm pldl1keep, [x12, #0x80]\n"
+ "fmla v25.4s, v9.4s, v1.s[1]\n"
+ "add x10, x10, #0x10\n"
+ "fmla v26.4s, v9.4s, v2.s[1]\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x28, x28, #0x10\n"
+ "fmla v27.4s, v9.4s, v3.s[1]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "sub x15, x15, #0x4\n"
+ "fmla v24.4s, v10.4s, v0.s[2]\n"
+ "cmp x15, #0x8\n"
+ "add x7, x7, #0x40\n"
+ "fmla v25.4s, v10.4s, v1.s[2]\n"
+ "fmla v26.4s, v10.4s, v2.s[2]\n"
+ "fmla v27.4s, v10.4s, v3.s[2]\n"
+ "fmla v24.4s, v11.4s, v0.s[3]\n"
+ "fmla v25.4s, v11.4s, v1.s[3]\n"
+ "fmla v26.4s, v11.4s, v2.s[3]\n"
+ "fmla v27.4s, v11.4s, v3.s[3]\n"
+ "bge 79b\n"
+ "80:" // Height 4: Multiply loop: Single iteration only
+ "sub x15, x15, #0x4\n"
+ "ldr q0, [x14, #0x0]\n"
+ "ldr q1, [x12, #0x0]\n"
+ "ldr q2, [x10, #0x0]\n"
+ "ldr q3, [x28, #0x0]\n"
+ "ldr q12, [x7, #0x0]\n"
+ "fmla v24.4s, v12.4s, v0.s[0]\n"
+ "ldr q13, [x7, #0x10]\n"
+ "fmla v25.4s, v12.4s, v1.s[0]\n"
+ "ldr q14, [x7, #0x20]\n"
+ "fmla v26.4s, v12.4s, v2.s[0]\n"
+ "ldr q15, [x7, #0x30]\n"
+ "fmla v27.4s, v12.4s, v3.s[0]\n"
+ "add x14, x14, #0x10\n"
+ "prfm pldl1keep, [x14, #0x80]\n"
+ "fmla v24.4s, v13.4s, v0.s[1]\n"
+ "add x12, x12, #0x10\n"
+ "prfm pldl1keep, [x12, #0x80]\n"
+ "fmla v25.4s, v13.4s, v1.s[1]\n"
+ "add x10, x10, #0x10\n"
+ "fmla v26.4s, v13.4s, v2.s[1]\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x28, x28, #0x10\n"
+ "fmla v27.4s, v13.4s, v3.s[1]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "add x7, x7, #0x40\n"
+ "fmla v24.4s, v14.4s, v0.s[2]\n"
+ "fmla v25.4s, v14.4s, v1.s[2]\n"
+ "fmla v26.4s, v14.4s, v2.s[2]\n"
+ "fmla v27.4s, v14.4s, v3.s[2]\n"
+ "fmla v24.4s, v15.4s, v0.s[3]\n"
+ "fmla v25.4s, v15.4s, v1.s[3]\n"
+ "fmla v26.4s, v15.4s, v2.s[3]\n"
+ "fmla v27.4s, v15.4s, v3.s[3]\n"
+ "81:" // Height 4: Multiply loop: Main loop skip
+ "cbz x15, 83f\n"
+ "82:" // Height 4: Multiply loop: Odd block loop
+ "ldr s0, [x14], #0x4\n"
+ "ldr s1, [x12], #0x4\n"
+ "ldr s2, [x10], #0x4\n"
+ "ldr s3, [x28], #0x4\n"
+ "ldr q16, [x7, #0x0]\n"
+ "fmla v24.4s, v16.4s, v0.s[0]\n"
+ "sub x15, x15, #0x1\n"
+ "fmla v25.4s, v16.4s, v1.s[0]\n"
+ "add x7, x7, #0x10\n"
+ "fmla v26.4s, v16.4s, v2.s[0]\n"
+ "fmla v27.4s, v16.4s, v3.s[0]\n"
+ "cbnz x15, 82b\n"
+ "83:" // Height 4: Multiply loop: No odd multiplies
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "add x16, x16, #0x1\n"
+ "cmp x16, x19\n"
+ "bne 76b\n"
+ "prfm pstl1keep, [x17, #0x0]\n"
+ "prfm pstl1keep, [x13, #0x0]\n"
+ "prfm pstl1keep, [x11, #0x0]\n"
+ "prfm pstl1keep, [x9, #0x0]\n"
+ "tbz %x[flags], #1, 84f\n"
+ "add x19, %x[args_ptr], %[offset_min]\n"
+ "ld1r { v17.4s }, [x19]\n"
+ "add x19, %x[args_ptr], %[offset_max]\n"
+ "ld1r { v16.4s }, [x19]\n"
+ "fmin v24.4s, v24.4s, v16.4s\n"
+ "fmin v25.4s, v25.4s, v16.4s\n"
+ "fmin v26.4s, v26.4s, v16.4s\n"
+ "fmin v27.4s, v27.4s, v16.4s\n"
+ "fmax v24.4s, v24.4s, v17.4s\n"
+ "fmax v25.4s, v25.4s, v17.4s\n"
+ "fmax v26.4s, v26.4s, v17.4s\n"
+ "fmax v27.4s, v27.4s, v17.4s\n"
+ "84:" // Height 4: No activation
+ "cmp x6, #0x4\n"
+ "bge 87f\n"
+ "tbz x6, #1, 85f\n"
+ "str d24, [x17], #0x8\n"
+ "str d25, [x13], #0x8\n"
+ "str d26, [x11], #0x8\n"
+ "str d27, [x9], #0x8\n"
+ "tbz x6, #0, 86f\n"
+ "st1 { v24.s }[2], [x17]\n"
+ "st1 { v25.s }[2], [x13]\n"
+ "st1 { v26.s }[2], [x11]\n"
+ "st1 { v27.s }[2], [x9]\n"
+ "b 86f\n"
+ "85:" // Height 4: Partial direct writeback: partial_1_0
+ "str s24, [x17, #0x0]\n"
+ "str s25, [x13, #0x0]\n"
+ "str s26, [x11, #0x0]\n"
+ "str s27, [x9, #0x0]\n"
+ "86:" // Height 4: Partial direct writeback: Done
+ "b 88f\n"
+ "87:" // Height 4: Full writeback
+ "str q24, [x17, #0x0]\n"
+ "str q25, [x13, #0x0]\n"
+ "str q26, [x11, #0x0]\n"
+ "str q27, [x9, #0x0]\n"
+ "add x17, x17, #0x10\n"
+ "add x13, x13, #0x10\n"
+ "add x11, x11, #0x10\n"
+ "add x9, x9, #0x10\n"
+ "88:" // Height 4: Writeback done
+ "subs x6, x6, #0x4\n"
+ "bgt 69b\n"
+ "b 178f\n"
+ "89:" // Height 5
+ "ldr x6, [%x[args_ptr], %[offsetof_N]]\n"
+ "mov x8, %x[bias]\n"
+ "ldr x7, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 90f\n"
+ "ldr x17, [%x[output_ptr], #0x0]\n"
+ "add x17, x17, x19, LSL #2\n"
+ "ldr x13, [%x[output_ptr], #0x8]\n"
+ "ldr x11, [%x[output_ptr], #0x10]\n"
+ "add x13, x13, x19, LSL #2\n"
+ "ldr x9, [%x[output_ptr], #0x18]\n"
+ "ldr x27, [%x[output_ptr], #0x20]\n"
+ "add x11, x11, x19, LSL #2\n"
+ "add x9, x9, x19, LSL #2\n"
+ "add x27, x27, x19, LSL #2\n"
+ "b 91f\n"
+ "90:" // Height 5: setup direct output
+ "mov x17, %x[output_ptr]\n"
+ "add x13, x17, x19, LSL #2\n"
+ "add x11, x13, x19, LSL #2\n"
+ "add x9, x11, x19, LSL #2\n"
+ "add x27, x9, x19, LSL #2\n"
+ "91:" // Height 5: Column loop
+ "cbz x8, 92f\n"
+ "ldr q24, [x8, #0x0]\n"
+ "mov v25.16b, v24.16b\n"
+ "add x8, x8, #0x10\n"
+ "mov v26.16b, v24.16b\n"
+ "mov v27.16b, v24.16b\n"
+ "mov v28.16b, v24.16b\n"
+ "b 97f\n"
+ "92:" // Height 5: no bias
+ "tbz %x[flags], #0, 96f\n"
+ "cmp x6, #0x4\n"
+ "bge 95f\n"
+ "tbz x6, #1, 93f\n"
+ "ldr d24, [x17], #0x8\n"
+ "ldr d25, [x13], #0x8\n"
+ "ldr d26, [x11], #0x8\n"
+ "ldr d27, [x9], #0x8\n"
+ "ldr d28, [x27], #0x8\n"
+ "mov x19, #0x8\n"
+ "tbz x6, #0, 94f\n"
+ "ld1 { v24.s }[2], [x17]\n"
+ "ld1 { v25.s }[2], [x13]\n"
+ "ld1 { v26.s }[2], [x11]\n"
+ "ld1 { v27.s }[2], [x9]\n"
+ "ld1 { v28.s }[2], [x27]\n"
+ "b 94f\n"
+ "93:" // Height 5: Partial accumulate: partial_1_0
+ "mov x19, #0x0\n"
+ "ldr s24, [x17, #0x0]\n"
+ "ldr s25, [x13, #0x0]\n"
+ "ldr s26, [x11, #0x0]\n"
+ "ldr s27, [x9, #0x0]\n"
+ "ldr s28, [x27, #0x0]\n"
+ "94:" // Height 5: Partial accumulate: Done
+ "sub x17, x17, x19\n"
+ "sub x13, x13, x19\n"
+ "sub x11, x11, x19\n"
+ "sub x9, x9, x19\n"
+ "sub x27, x27, x19\n"
+ "b 97f\n"
+ "95:" // Height 5: full accumulate
+ "ldr q24, [x17, #0x0]\n"
+ "ldr q25, [x13, #0x0]\n"
+ "ldr q26, [x11, #0x0]\n"
+ "ldr q27, [x9, #0x0]\n"
+ "ldr q28, [x27, #0x0]\n"
+ "b 97f\n"
+ "96:" // Height 5: no accumulate
+ "movi v24.16b, #0x0\n"
+ "movi v25.16b, #0x0\n"
+ "movi v26.16b, #0x0\n"
+ "movi v27.16b, #0x0\n"
+ "movi v28.16b, #0x0\n"
+ "97:" // Height 5: setup done
+ "mov x16, #0x0\n"
+ "98:" // Height 5: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w15, [x20, x16, LSL #0x2]\n"
+ "tbz %x[flags], #3, 99f\n"
+ "ldr x20, [%x[input_ptr], x16, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x14, [x20, #0x0]\n"
+ "ldr x12, [x20, #0x8]\n"
+ "ldr x10, [x20, #0x10]\n"
+ "ldr x28, [x20, #0x18]\n"
+ "ldr x26, [x20, #0x20]\n"
+ "cbnz x16, 100f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x14, x14, x19, LSL #2\n"
+ "add x12, x12, x19, LSL #2\n"
+ "add x10, x10, x19, LSL #2\n"
+ "add x28, x28, x19, LSL #2\n"
+ "add x26, x26, x19, LSL #2\n"
+ "b 100f\n"
+ "99:" // Height 5: setup direct input
+ "mov x14, %x[input_ptr]\n"
+ "add x12, x14, x19, LSL #2\n"
+ "add x10, x12, x19, LSL #2\n"
+ "add x28, x10, x19, LSL #2\n"
+ "add x26, x28, x19, LSL #2\n"
+ "100:" // Height 5: input setup done
+ "cmp x15, #0x4\n"
+ "blt 103f\n"
+ "cmp x15, #0x8\n"
+ "blt 102f\n"
+ "101:" // Height 5: Multiply loop: Main loop head
+ "ldr q0, [x14, #0x0]\n"
+ "ldr q1, [x12, #0x0]\n"
+ "ldr q2, [x10, #0x0]\n"
+ "ldr q3, [x28, #0x0]\n"
+ "ldr q4, [x26, #0x0]\n"
+ "ldr q8, [x7, #0x0]\n"
+ "fmla v24.4s, v8.4s, v0.s[0]\n"
+ "ldr q9, [x7, #0x10]\n"
+ "fmla v25.4s, v8.4s, v1.s[0]\n"
+ "ldr q10, [x7, #0x20]\n"
+ "fmla v26.4s, v8.4s, v2.s[0]\n"
+ "ldr q11, [x7, #0x30]\n"
+ "fmla v27.4s, v8.4s, v3.s[0]\n"
+ "add x14, x14, #0x10\n"
+ "prfm pldl1keep, [x14, #0x80]\n"
+ "fmla v28.4s, v8.4s, v4.s[0]\n"
+ "add x12, x12, #0x10\n"
+ "fmla v24.4s, v9.4s, v0.s[1]\n"
+ "prfm pldl1keep, [x12, #0x80]\n"
+ "add x10, x10, #0x10\n"
+ "fmla v25.4s, v9.4s, v1.s[1]\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x28, x28, #0x10\n"
+ "fmla v26.4s, v9.4s, v2.s[1]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "add x26, x26, #0x10\n"
+ "fmla v27.4s, v9.4s, v3.s[1]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "sub x15, x15, #0x4\n"
+ "fmla v28.4s, v9.4s, v4.s[1]\n"
+ "cmp x15, #0x8\n"
+ "fmla v24.4s, v10.4s, v0.s[2]\n"
+ "add x7, x7, #0x40\n"
+ "fmla v25.4s, v10.4s, v1.s[2]\n"
+ "fmla v26.4s, v10.4s, v2.s[2]\n"
+ "fmla v27.4s, v10.4s, v3.s[2]\n"
+ "fmla v28.4s, v10.4s, v4.s[2]\n"
+ "fmla v24.4s, v11.4s, v0.s[3]\n"
+ "fmla v25.4s, v11.4s, v1.s[3]\n"
+ "fmla v26.4s, v11.4s, v2.s[3]\n"
+ "fmla v27.4s, v11.4s, v3.s[3]\n"
+ "fmla v28.4s, v11.4s, v4.s[3]\n"
+ "bge 101b\n"
+ "102:" // Height 5: Multiply loop: Single iteration only
+ "sub x15, x15, #0x4\n"
+ "ldr q0, [x14, #0x0]\n"
+ "ldr q1, [x12, #0x0]\n"
+ "ldr q2, [x10, #0x0]\n"
+ "ldr q3, [x28, #0x0]\n"
+ "ldr q4, [x26, #0x0]\n"
+ "ldr q12, [x7, #0x0]\n"
+ "fmla v24.4s, v12.4s, v0.s[0]\n"
+ "ldr q13, [x7, #0x10]\n"
+ "fmla v25.4s, v12.4s, v1.s[0]\n"
+ "ldr q14, [x7, #0x20]\n"
+ "fmla v26.4s, v12.4s, v2.s[0]\n"
+ "ldr q15, [x7, #0x30]\n"
+ "fmla v27.4s, v12.4s, v3.s[0]\n"
+ "add x14, x14, #0x10\n"
+ "prfm pldl1keep, [x14, #0x80]\n"
+ "fmla v28.4s, v12.4s, v4.s[0]\n"
+ "add x12, x12, #0x10\n"
+ "fmla v24.4s, v13.4s, v0.s[1]\n"
+ "prfm pldl1keep, [x12, #0x80]\n"
+ "add x10, x10, #0x10\n"
+ "fmla v25.4s, v13.4s, v1.s[1]\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x28, x28, #0x10\n"
+ "fmla v26.4s, v13.4s, v2.s[1]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "add x26, x26, #0x10\n"
+ "fmla v27.4s, v13.4s, v3.s[1]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "add x7, x7, #0x40\n"
+ "fmla v28.4s, v13.4s, v4.s[1]\n"
+ "fmla v24.4s, v14.4s, v0.s[2]\n"
+ "fmla v25.4s, v14.4s, v1.s[2]\n"
+ "fmla v26.4s, v14.4s, v2.s[2]\n"
+ "fmla v27.4s, v14.4s, v3.s[2]\n"
+ "fmla v28.4s, v14.4s, v4.s[2]\n"
+ "fmla v24.4s, v15.4s, v0.s[3]\n"
+ "fmla v25.4s, v15.4s, v1.s[3]\n"
+ "fmla v26.4s, v15.4s, v2.s[3]\n"
+ "fmla v27.4s, v15.4s, v3.s[3]\n"
+ "fmla v28.4s, v15.4s, v4.s[3]\n"
+ "103:" // Height 5: Multiply loop: Main loop skip
+ "cbz x15, 105f\n"
+ "104:" // Height 5: Multiply loop: Odd block loop
+ "ldr s0, [x14], #0x4\n"
+ "ldr s1, [x12], #0x4\n"
+ "ldr s2, [x10], #0x4\n"
+ "ldr s3, [x28], #0x4\n"
+ "ldr s4, [x26], #0x4\n"
+ "ldr q16, [x7, #0x0]\n"
+ "fmla v24.4s, v16.4s, v0.s[0]\n"
+ "sub x15, x15, #0x1\n"
+ "fmla v25.4s, v16.4s, v1.s[0]\n"
+ "add x7, x7, #0x10\n"
+ "fmla v26.4s, v16.4s, v2.s[0]\n"
+ "fmla v27.4s, v16.4s, v3.s[0]\n"
+ "fmla v28.4s, v16.4s, v4.s[0]\n"
+ "cbnz x15, 104b\n"
+ "105:" // Height 5: Multiply loop: No odd multiplies
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "add x16, x16, #0x1\n"
+ "cmp x16, x19\n"
+ "bne 98b\n"
+ "prfm pstl1keep, [x17, #0x0]\n"
+ "prfm pstl1keep, [x13, #0x0]\n"
+ "prfm pstl1keep, [x11, #0x0]\n"
+ "prfm pstl1keep, [x9, #0x0]\n"
+ "prfm pstl1keep, [x27, #0x0]\n"
+ "tbz %x[flags], #1, 106f\n"
+ "add x19, %x[args_ptr], %[offset_min]\n"
+ "ld1r { v17.4s }, [x19]\n"
+ "add x19, %x[args_ptr], %[offset_max]\n"
+ "ld1r { v16.4s }, [x19]\n"
+ "fmin v24.4s, v24.4s, v16.4s\n"
+ "fmin v25.4s, v25.4s, v16.4s\n"
+ "fmin v26.4s, v26.4s, v16.4s\n"
+ "fmin v27.4s, v27.4s, v16.4s\n"
+ "fmax v24.4s, v24.4s, v17.4s\n"
+ "fmax v25.4s, v25.4s, v17.4s\n"
+ "fmax v26.4s, v26.4s, v17.4s\n"
+ "fmax v27.4s, v27.4s, v17.4s\n"
+ "fmin v28.4s, v28.4s, v16.4s\n"
+ "fmax v28.4s, v28.4s, v17.4s\n"
+ "106:" // Height 5: No activation
+ "cmp x6, #0x4\n"
+ "bge 109f\n"
+ "tbz x6, #1, 107f\n"
+ "str d24, [x17], #0x8\n"
+ "str d25, [x13], #0x8\n"
+ "str d26, [x11], #0x8\n"
+ "str d27, [x9], #0x8\n"
+ "str d28, [x27], #0x8\n"
+ "tbz x6, #0, 108f\n"
+ "st1 { v24.s }[2], [x17]\n"
+ "st1 { v25.s }[2], [x13]\n"
+ "st1 { v26.s }[2], [x11]\n"
+ "st1 { v27.s }[2], [x9]\n"
+ "st1 { v28.s }[2], [x27]\n"
+ "b 108f\n"
+ "107:" // Height 5: Partial direct writeback: partial_1_0
+ "str s24, [x17, #0x0]\n"
+ "str s25, [x13, #0x0]\n"
+ "str s26, [x11, #0x0]\n"
+ "str s27, [x9, #0x0]\n"
+ "str s28, [x27, #0x0]\n"
+ "108:" // Height 5: Partial direct writeback: Done
+ "b 110f\n"
+ "109:" // Height 5: Full writeback
+ "str q24, [x17, #0x0]\n"
+ "str q25, [x13, #0x0]\n"
+ "str q26, [x11, #0x0]\n"
+ "str q27, [x9, #0x0]\n"
+ "str q28, [x27, #0x0]\n"
+ "add x17, x17, #0x10\n"
+ "add x13, x13, #0x10\n"
+ "add x11, x11, #0x10\n"
+ "add x9, x9, #0x10\n"
+ "add x27, x27, #0x10\n"
+ "110:" // Height 5: Writeback done
+ "subs x6, x6, #0x4\n"
+ "bgt 91b\n"
+ "b 178f\n"
+ "111:" // Height 6
+ "ldr x6, [%x[args_ptr], %[offsetof_N]]\n"
+ "mov x8, %x[bias]\n"
+ "ldr x7, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 112f\n"
+ "ldr x17, [%x[output_ptr], #0x0]\n"
+ "add x17, x17, x19, LSL #2\n"
+ "ldr x13, [%x[output_ptr], #0x8]\n"
+ "ldr x11, [%x[output_ptr], #0x10]\n"
+ "add x13, x13, x19, LSL #2\n"
+ "ldr x9, [%x[output_ptr], #0x18]\n"
+ "ldr x27, [%x[output_ptr], #0x20]\n"
+ "add x11, x11, x19, LSL #2\n"
+ "ldr x25, [%x[output_ptr], #0x28]\n"
+ "add x9, x9, x19, LSL #2\n"
+ "add x27, x27, x19, LSL #2\n"
+ "add x25, x25, x19, LSL #2\n"
+ "b 113f\n"
+ "112:" // Height 6: setup direct output
+ "mov x17, %x[output_ptr]\n"
+ "add x13, x17, x19, LSL #2\n"
+ "add x11, x13, x19, LSL #2\n"
+ "add x9, x11, x19, LSL #2\n"
+ "add x27, x9, x19, LSL #2\n"
+ "add x25, x27, x19, LSL #2\n"
+ "113:" // Height 6: Column loop
+ "cbz x8, 114f\n"
+ "ldr q24, [x8, #0x0]\n"
+ "mov v25.16b, v24.16b\n"
+ "add x8, x8, #0x10\n"
+ "mov v26.16b, v24.16b\n"
+ "mov v27.16b, v24.16b\n"
+ "mov v28.16b, v24.16b\n"
+ "mov v29.16b, v24.16b\n"
+ "b 119f\n"
+ "114:" // Height 6: no bias
+ "tbz %x[flags], #0, 118f\n"
+ "cmp x6, #0x4\n"
+ "bge 117f\n"
+ "tbz x6, #1, 115f\n"
+ "ldr d24, [x17], #0x8\n"
+ "ldr d25, [x13], #0x8\n"
+ "ldr d26, [x11], #0x8\n"
+ "ldr d27, [x9], #0x8\n"
+ "ldr d28, [x27], #0x8\n"
+ "ldr d29, [x25], #0x8\n"
+ "mov x19, #0x8\n"
+ "tbz x6, #0, 116f\n"
+ "ld1 { v24.s }[2], [x17]\n"
+ "ld1 { v25.s }[2], [x13]\n"
+ "ld1 { v26.s }[2], [x11]\n"
+ "ld1 { v27.s }[2], [x9]\n"
+ "ld1 { v28.s }[2], [x27]\n"
+ "ld1 { v29.s }[2], [x25]\n"
+ "b 116f\n"
+ "115:" // Height 6: Partial accumulate: partial_1_0
+ "mov x19, #0x0\n"
+ "ldr s24, [x17, #0x0]\n"
+ "ldr s25, [x13, #0x0]\n"
+ "ldr s26, [x11, #0x0]\n"
+ "ldr s27, [x9, #0x0]\n"
+ "ldr s28, [x27, #0x0]\n"
+ "ldr s29, [x25, #0x0]\n"
+ "116:" // Height 6: Partial accumulate: Done
+ "sub x17, x17, x19\n"
+ "sub x13, x13, x19\n"
+ "sub x11, x11, x19\n"
+ "sub x9, x9, x19\n"
+ "sub x27, x27, x19\n"
+ "sub x25, x25, x19\n"
+ "b 119f\n"
+ "117:" // Height 6: full accumulate
+ "ldr q24, [x17, #0x0]\n"
+ "ldr q25, [x13, #0x0]\n"
+ "ldr q26, [x11, #0x0]\n"
+ "ldr q27, [x9, #0x0]\n"
+ "ldr q28, [x27, #0x0]\n"
+ "ldr q29, [x25, #0x0]\n"
+ "b 119f\n"
+ "118:" // Height 6: no accumulate
+ "movi v24.16b, #0x0\n"
+ "movi v25.16b, #0x0\n"
+ "movi v26.16b, #0x0\n"
+ "movi v27.16b, #0x0\n"
+ "movi v28.16b, #0x0\n"
+ "movi v29.16b, #0x0\n"
+ "119:" // Height 6: setup done
+ "mov x16, #0x0\n"
+ "120:" // Height 6: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w15, [x20, x16, LSL #0x2]\n"
+ "tbz %x[flags], #3, 121f\n"
+ "ldr x20, [%x[input_ptr], x16, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x14, [x20, #0x0]\n"
+ "ldr x12, [x20, #0x8]\n"
+ "ldr x10, [x20, #0x10]\n"
+ "ldr x28, [x20, #0x18]\n"
+ "ldr x26, [x20, #0x20]\n"
+ "ldr x24, [x20, #0x28]\n"
+ "cbnz x16, 122f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x14, x14, x19, LSL #2\n"
+ "add x12, x12, x19, LSL #2\n"
+ "add x10, x10, x19, LSL #2\n"
+ "add x28, x28, x19, LSL #2\n"
+ "add x26, x26, x19, LSL #2\n"
+ "add x24, x24, x19, LSL #2\n"
+ "b 122f\n"
+ "121:" // Height 6: setup direct input
+ "mov x14, %x[input_ptr]\n"
+ "add x12, x14, x19, LSL #2\n"
+ "add x10, x12, x19, LSL #2\n"
+ "add x28, x10, x19, LSL #2\n"
+ "add x26, x28, x19, LSL #2\n"
+ "add x24, x26, x19, LSL #2\n"
+ "122:" // Height 6: input setup done
+ "cmp x15, #0x4\n"
+ "blt 125f\n"
+ "cmp x15, #0x8\n"
+ "blt 124f\n"
+ "123:" // Height 6: Multiply loop: Main loop head
+ "ldr q0, [x14, #0x0]\n"
+ "ldr q1, [x12, #0x0]\n"
+ "ldr q2, [x10, #0x0]\n"
+ "ldr q3, [x28, #0x0]\n"
+ "ldr q4, [x26, #0x0]\n"
+ "ldr q5, [x24, #0x0]\n"
+ "ldr q8, [x7, #0x0]\n"
+ "fmla v24.4s, v8.4s, v0.s[0]\n"
+ "ldr q9, [x7, #0x10]\n"
+ "fmla v25.4s, v8.4s, v1.s[0]\n"
+ "ldr q10, [x7, #0x20]\n"
+ "fmla v26.4s, v8.4s, v2.s[0]\n"
+ "ldr q11, [x7, #0x30]\n"
+ "fmla v27.4s, v8.4s, v3.s[0]\n"
+ "add x14, x14, #0x10\n"
+ "prfm pldl1keep, [x14, #0x80]\n"
+ "fmla v28.4s, v8.4s, v4.s[0]\n"
+ "add x12, x12, #0x10\n"
+ "fmla v29.4s, v8.4s, v5.s[0]\n"
+ "prfm pldl1keep, [x12, #0x80]\n"
+ "add x10, x10, #0x10\n"
+ "fmla v24.4s, v9.4s, v0.s[1]\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x28, x28, #0x10\n"
+ "fmla v25.4s, v9.4s, v1.s[1]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "add x26, x26, #0x10\n"
+ "fmla v26.4s, v9.4s, v2.s[1]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "add x24, x24, #0x10\n"
+ "fmla v27.4s, v9.4s, v3.s[1]\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "sub x15, x15, #0x4\n"
+ "fmla v28.4s, v9.4s, v4.s[1]\n"
+ "cmp x15, #0x8\n"
+ "fmla v29.4s, v9.4s, v5.s[1]\n"
+ "add x7, x7, #0x40\n"
+ "fmla v24.4s, v10.4s, v0.s[2]\n"
+ "fmla v25.4s, v10.4s, v1.s[2]\n"
+ "fmla v26.4s, v10.4s, v2.s[2]\n"
+ "fmla v27.4s, v10.4s, v3.s[2]\n"
+ "fmla v28.4s, v10.4s, v4.s[2]\n"
+ "fmla v29.4s, v10.4s, v5.s[2]\n"
+ "fmla v24.4s, v11.4s, v0.s[3]\n"
+ "fmla v25.4s, v11.4s, v1.s[3]\n"
+ "fmla v26.4s, v11.4s, v2.s[3]\n"
+ "fmla v27.4s, v11.4s, v3.s[3]\n"
+ "fmla v28.4s, v11.4s, v4.s[3]\n"
+ "fmla v29.4s, v11.4s, v5.s[3]\n"
+ "bge 123b\n"
+ "124:" // Height 6: Multiply loop: Single iteration only
+ "sub x15, x15, #0x4\n"
+ "ldr q0, [x14, #0x0]\n"
+ "ldr q1, [x12, #0x0]\n"
+ "ldr q2, [x10, #0x0]\n"
+ "ldr q3, [x28, #0x0]\n"
+ "ldr q4, [x26, #0x0]\n"
+ "ldr q5, [x24, #0x0]\n"
+ "ldr q12, [x7, #0x0]\n"
+ "fmla v24.4s, v12.4s, v0.s[0]\n"
+ "ldr q13, [x7, #0x10]\n"
+ "fmla v25.4s, v12.4s, v1.s[0]\n"
+ "ldr q14, [x7, #0x20]\n"
+ "fmla v26.4s, v12.4s, v2.s[0]\n"
+ "ldr q15, [x7, #0x30]\n"
+ "fmla v27.4s, v12.4s, v3.s[0]\n"
+ "add x14, x14, #0x10\n"
+ "prfm pldl1keep, [x14, #0x80]\n"
+ "fmla v28.4s, v12.4s, v4.s[0]\n"
+ "add x12, x12, #0x10\n"
+ "fmla v29.4s, v12.4s, v5.s[0]\n"
+ "prfm pldl1keep, [x12, #0x80]\n"
+ "add x10, x10, #0x10\n"
+ "fmla v24.4s, v13.4s, v0.s[1]\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x28, x28, #0x10\n"
+ "fmla v25.4s, v13.4s, v1.s[1]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "add x26, x26, #0x10\n"
+ "fmla v26.4s, v13.4s, v2.s[1]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "add x24, x24, #0x10\n"
+ "fmla v27.4s, v13.4s, v3.s[1]\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "add x7, x7, #0x40\n"
+ "fmla v28.4s, v13.4s, v4.s[1]\n"
+ "fmla v29.4s, v13.4s, v5.s[1]\n"
+ "fmla v24.4s, v14.4s, v0.s[2]\n"
+ "fmla v25.4s, v14.4s, v1.s[2]\n"
+ "fmla v26.4s, v14.4s, v2.s[2]\n"
+ "fmla v27.4s, v14.4s, v3.s[2]\n"
+ "fmla v28.4s, v14.4s, v4.s[2]\n"
+ "fmla v29.4s, v14.4s, v5.s[2]\n"
+ "fmla v24.4s, v15.4s, v0.s[3]\n"
+ "fmla v25.4s, v15.4s, v1.s[3]\n"
+ "fmla v26.4s, v15.4s, v2.s[3]\n"
+ "fmla v27.4s, v15.4s, v3.s[3]\n"
+ "fmla v28.4s, v15.4s, v4.s[3]\n"
+ "fmla v29.4s, v15.4s, v5.s[3]\n"
+ "125:" // Height 6: Multiply loop: Main loop skip
+ "cbz x15, 127f\n"
+ "126:" // Height 6: Multiply loop: Odd block loop
+ "ldr s0, [x14], #0x4\n"
+ "ldr s1, [x12], #0x4\n"
+ "ldr s2, [x10], #0x4\n"
+ "ldr s3, [x28], #0x4\n"
+ "ldr s4, [x26], #0x4\n"
+ "ldr s5, [x24], #0x4\n"
+ "ldr q16, [x7, #0x0]\n"
+ "fmla v24.4s, v16.4s, v0.s[0]\n"
+ "sub x15, x15, #0x1\n"
+ "fmla v25.4s, v16.4s, v1.s[0]\n"
+ "add x7, x7, #0x10\n"
+ "fmla v26.4s, v16.4s, v2.s[0]\n"
+ "fmla v27.4s, v16.4s, v3.s[0]\n"
+ "fmla v28.4s, v16.4s, v4.s[0]\n"
+ "fmla v29.4s, v16.4s, v5.s[0]\n"
+ "cbnz x15, 126b\n"
+ "127:" // Height 6: Multiply loop: No odd multiplies
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "add x16, x16, #0x1\n"
+ "cmp x16, x19\n"
+ "bne 120b\n"
+ "prfm pstl1keep, [x17, #0x0]\n"
+ "prfm pstl1keep, [x13, #0x0]\n"
+ "prfm pstl1keep, [x11, #0x0]\n"
+ "prfm pstl1keep, [x9, #0x0]\n"
+ "prfm pstl1keep, [x27, #0x0]\n"
+ "prfm pstl1keep, [x25, #0x0]\n"
+ "tbz %x[flags], #1, 128f\n"
+ "add x19, %x[args_ptr], %[offset_min]\n"
+ "ld1r { v17.4s }, [x19]\n"
+ "add x19, %x[args_ptr], %[offset_max]\n"
+ "ld1r { v16.4s }, [x19]\n"
+ "fmin v24.4s, v24.4s, v16.4s\n"
+ "fmin v25.4s, v25.4s, v16.4s\n"
+ "fmin v26.4s, v26.4s, v16.4s\n"
+ "fmin v27.4s, v27.4s, v16.4s\n"
+ "fmax v24.4s, v24.4s, v17.4s\n"
+ "fmax v25.4s, v25.4s, v17.4s\n"
+ "fmax v26.4s, v26.4s, v17.4s\n"
+ "fmax v27.4s, v27.4s, v17.4s\n"
+ "fmin v28.4s, v28.4s, v16.4s\n"
+ "fmin v29.4s, v29.4s, v16.4s\n"
+ "fmax v28.4s, v28.4s, v17.4s\n"
+ "fmax v29.4s, v29.4s, v17.4s\n"
+ "128:" // Height 6: No activation
+ "cmp x6, #0x4\n"
+ "bge 131f\n"
+ "tbz x6, #1, 129f\n"
+ "str d24, [x17], #0x8\n"
+ "str d25, [x13], #0x8\n"
+ "str d26, [x11], #0x8\n"
+ "str d27, [x9], #0x8\n"
+ "str d28, [x27], #0x8\n"
+ "str d29, [x25], #0x8\n"
+ "tbz x6, #0, 130f\n"
+ "st1 { v24.s }[2], [x17]\n"
+ "st1 { v25.s }[2], [x13]\n"
+ "st1 { v26.s }[2], [x11]\n"
+ "st1 { v27.s }[2], [x9]\n"
+ "st1 { v28.s }[2], [x27]\n"
+ "st1 { v29.s }[2], [x25]\n"
+ "b 130f\n"
+ "129:" // Height 6: Partial direct writeback: partial_1_0
+ "str s24, [x17, #0x0]\n"
+ "str s25, [x13, #0x0]\n"
+ "str s26, [x11, #0x0]\n"
+ "str s27, [x9, #0x0]\n"
+ "str s28, [x27, #0x0]\n"
+ "str s29, [x25, #0x0]\n"
+ "130:" // Height 6: Partial direct writeback: Done
+ "b 132f\n"
+ "131:" // Height 6: Full writeback
+ "str q24, [x17, #0x0]\n"
+ "str q25, [x13, #0x0]\n"
+ "str q26, [x11, #0x0]\n"
+ "str q27, [x9, #0x0]\n"
+ "str q28, [x27, #0x0]\n"
+ "str q29, [x25, #0x0]\n"
+ "add x17, x17, #0x10\n"
+ "add x13, x13, #0x10\n"
+ "add x11, x11, #0x10\n"
+ "add x9, x9, #0x10\n"
+ "add x27, x27, #0x10\n"
+ "add x25, x25, #0x10\n"
+ "132:" // Height 6: Writeback done
+ "subs x6, x6, #0x4\n"
+ "bgt 113b\n"
+ "b 178f\n"
+ "133:" // Height 7
+ "ldr x6, [%x[args_ptr], %[offsetof_N]]\n"
+ "mov x8, %x[bias]\n"
+ "ldr x7, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 134f\n"
+ "ldr x17, [%x[output_ptr], #0x0]\n"
+ "add x17, x17, x19, LSL #2\n"
+ "ldr x13, [%x[output_ptr], #0x8]\n"
+ "ldr x11, [%x[output_ptr], #0x10]\n"
+ "add x13, x13, x19, LSL #2\n"
+ "ldr x9, [%x[output_ptr], #0x18]\n"
+ "ldr x27, [%x[output_ptr], #0x20]\n"
+ "add x11, x11, x19, LSL #2\n"
+ "ldr x25, [%x[output_ptr], #0x28]\n"
+ "add x9, x9, x19, LSL #2\n"
+ "ldr x23, [%x[output_ptr], #0x30]\n"
+ "add x27, x27, x19, LSL #2\n"
+ "add x25, x25, x19, LSL #2\n"
+ "add x23, x23, x19, LSL #2\n"
+ "b 135f\n"
+ "134:" // Height 7: setup direct output
+ "mov x17, %x[output_ptr]\n"
+ "add x13, x17, x19, LSL #2\n"
+ "add x11, x13, x19, LSL #2\n"
+ "add x9, x11, x19, LSL #2\n"
+ "add x27, x9, x19, LSL #2\n"
+ "add x25, x27, x19, LSL #2\n"
+ "add x23, x25, x19, LSL #2\n"
+ "135:" // Height 7: Column loop
+ "cbz x8, 136f\n"
+ "ldr q24, [x8, #0x0]\n"
+ "mov v25.16b, v24.16b\n"
+ "add x8, x8, #0x10\n"
+ "mov v26.16b, v24.16b\n"
+ "mov v27.16b, v24.16b\n"
+ "mov v28.16b, v24.16b\n"
+ "mov v29.16b, v24.16b\n"
+ "mov v30.16b, v24.16b\n"
+ "b 141f\n"
+ "136:" // Height 7: no bias
+ "tbz %x[flags], #0, 140f\n"
+ "cmp x6, #0x4\n"
+ "bge 139f\n"
+ "tbz x6, #1, 137f\n"
+ "ldr d24, [x17], #0x8\n"
+ "ldr d25, [x13], #0x8\n"
+ "ldr d26, [x11], #0x8\n"
+ "ldr d27, [x9], #0x8\n"
+ "ldr d28, [x27], #0x8\n"
+ "ldr d29, [x25], #0x8\n"
+ "ldr d30, [x23], #0x8\n"
+ "mov x19, #0x8\n"
+ "tbz x6, #0, 138f\n"
+ "ld1 { v24.s }[2], [x17]\n"
+ "ld1 { v25.s }[2], [x13]\n"
+ "ld1 { v26.s }[2], [x11]\n"
+ "ld1 { v27.s }[2], [x9]\n"
+ "ld1 { v28.s }[2], [x27]\n"
+ "ld1 { v29.s }[2], [x25]\n"
+ "ld1 { v30.s }[2], [x23]\n"
+ "b 138f\n"
+ "137:" // Height 7: Partial accumulate: partial_1_0
+ "mov x19, #0x0\n"
+ "ldr s24, [x17, #0x0]\n"
+ "ldr s25, [x13, #0x0]\n"
+ "ldr s26, [x11, #0x0]\n"
+ "ldr s27, [x9, #0x0]\n"
+ "ldr s28, [x27, #0x0]\n"
+ "ldr s29, [x25, #0x0]\n"
+ "ldr s30, [x23, #0x0]\n"
+ "138:" // Height 7: Partial accumulate: Done
+ "sub x17, x17, x19\n"
+ "sub x13, x13, x19\n"
+ "sub x11, x11, x19\n"
+ "sub x9, x9, x19\n"
+ "sub x27, x27, x19\n"
+ "sub x25, x25, x19\n"
+ "sub x23, x23, x19\n"
+ "b 141f\n"
+ "139:" // Height 7: full accumulate
+ "ldr q24, [x17, #0x0]\n"
+ "ldr q25, [x13, #0x0]\n"
+ "ldr q26, [x11, #0x0]\n"
+ "ldr q27, [x9, #0x0]\n"
+ "ldr q28, [x27, #0x0]\n"
+ "ldr q29, [x25, #0x0]\n"
+ "ldr q30, [x23, #0x0]\n"
+ "b 141f\n"
+ "140:" // Height 7: no accumulate
+ "movi v24.16b, #0x0\n"
+ "movi v25.16b, #0x0\n"
+ "movi v26.16b, #0x0\n"
+ "movi v27.16b, #0x0\n"
+ "movi v28.16b, #0x0\n"
+ "movi v29.16b, #0x0\n"
+ "movi v30.16b, #0x0\n"
+ "141:" // Height 7: setup done
+ "mov x16, #0x0\n"
+ "142:" // Height 7: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w15, [x20, x16, LSL #0x2]\n"
+ "tbz %x[flags], #3, 143f\n"
+ "ldr x20, [%x[input_ptr], x16, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x14, [x20, #0x0]\n"
+ "ldr x12, [x20, #0x8]\n"
+ "ldr x10, [x20, #0x10]\n"
+ "ldr x28, [x20, #0x18]\n"
+ "ldr x26, [x20, #0x20]\n"
+ "ldr x24, [x20, #0x28]\n"
+ "ldr x22, [x20, #0x30]\n"
+ "cbnz x16, 144f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x14, x14, x19, LSL #2\n"
+ "add x12, x12, x19, LSL #2\n"
+ "add x10, x10, x19, LSL #2\n"
+ "add x28, x28, x19, LSL #2\n"
+ "add x26, x26, x19, LSL #2\n"
+ "add x24, x24, x19, LSL #2\n"
+ "add x22, x22, x19, LSL #2\n"
+ "b 144f\n"
+ "143:" // Height 7: setup direct input
+ "mov x14, %x[input_ptr]\n"
+ "add x12, x14, x19, LSL #2\n"
+ "add x10, x12, x19, LSL #2\n"
+ "add x28, x10, x19, LSL #2\n"
+ "add x26, x28, x19, LSL #2\n"
+ "add x24, x26, x19, LSL #2\n"
+ "add x22, x24, x19, LSL #2\n"
+ "144:" // Height 7: input setup done
+ "cmp x15, #0x4\n"
+ "blt 147f\n"
+ "cmp x15, #0x8\n"
+ "blt 146f\n"
+ "145:" // Height 7: Multiply loop: Main loop head
+ "ldr q0, [x14, #0x0]\n"
+ "ldr q1, [x12, #0x0]\n"
+ "ldr q2, [x10, #0x0]\n"
+ "ldr q3, [x28, #0x0]\n"
+ "ldr q4, [x26, #0x0]\n"
+ "ldr q5, [x24, #0x0]\n"
+ "ldr q6, [x22, #0x0]\n"
+ "ldr q8, [x7, #0x0]\n"
+ "fmla v24.4s, v8.4s, v0.s[0]\n"
+ "ldr q9, [x7, #0x10]\n"
+ "fmla v25.4s, v8.4s, v1.s[0]\n"
+ "ldr q10, [x7, #0x20]\n"
+ "fmla v26.4s, v8.4s, v2.s[0]\n"
+ "ldr q11, [x7, #0x30]\n"
+ "fmla v27.4s, v8.4s, v3.s[0]\n"
+ "add x14, x14, #0x10\n"
+ "prfm pldl1keep, [x14, #0x80]\n"
+ "fmla v28.4s, v8.4s, v4.s[0]\n"
+ "add x12, x12, #0x10\n"
+ "fmla v29.4s, v8.4s, v5.s[0]\n"
+ "prfm pldl1keep, [x12, #0x80]\n"
+ "add x10, x10, #0x10\n"
+ "fmla v30.4s, v8.4s, v6.s[0]\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x28, x28, #0x10\n"
+ "fmla v24.4s, v9.4s, v0.s[1]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "add x26, x26, #0x10\n"
+ "fmla v25.4s, v9.4s, v1.s[1]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "add x24, x24, #0x10\n"
+ "fmla v26.4s, v9.4s, v2.s[1]\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "add x22, x22, #0x10\n"
+ "fmla v27.4s, v9.4s, v3.s[1]\n"
+ "prfm pldl1keep, [x22, #0x80]\n"
+ "sub x15, x15, #0x4\n"
+ "fmla v28.4s, v9.4s, v4.s[1]\n"
+ "cmp x15, #0x8\n"
+ "fmla v29.4s, v9.4s, v5.s[1]\n"
+ "add x7, x7, #0x40\n"
+ "fmla v30.4s, v9.4s, v6.s[1]\n"
+ "fmla v24.4s, v10.4s, v0.s[2]\n"
+ "fmla v25.4s, v10.4s, v1.s[2]\n"
+ "fmla v26.4s, v10.4s, v2.s[2]\n"
+ "fmla v27.4s, v10.4s, v3.s[2]\n"
+ "fmla v28.4s, v10.4s, v4.s[2]\n"
+ "fmla v29.4s, v10.4s, v5.s[2]\n"
+ "fmla v30.4s, v10.4s, v6.s[2]\n"
+ "fmla v24.4s, v11.4s, v0.s[3]\n"
+ "fmla v25.4s, v11.4s, v1.s[3]\n"
+ "fmla v26.4s, v11.4s, v2.s[3]\n"
+ "fmla v27.4s, v11.4s, v3.s[3]\n"
+ "fmla v28.4s, v11.4s, v4.s[3]\n"
+ "fmla v29.4s, v11.4s, v5.s[3]\n"
+ "fmla v30.4s, v11.4s, v6.s[3]\n"
+ "bge 145b\n"
+ "146:" // Height 7: Multiply loop: Single iteration only
+ "sub x15, x15, #0x4\n"
+ "ldr q0, [x14, #0x0]\n"
+ "ldr q1, [x12, #0x0]\n"
+ "ldr q2, [x10, #0x0]\n"
+ "ldr q3, [x28, #0x0]\n"
+ "ldr q4, [x26, #0x0]\n"
+ "ldr q5, [x24, #0x0]\n"
+ "ldr q6, [x22, #0x0]\n"
+ "ldr q12, [x7, #0x0]\n"
+ "fmla v24.4s, v12.4s, v0.s[0]\n"
+ "ldr q13, [x7, #0x10]\n"
+ "fmla v25.4s, v12.4s, v1.s[0]\n"
+ "ldr q14, [x7, #0x20]\n"
+ "fmla v26.4s, v12.4s, v2.s[0]\n"
+ "ldr q15, [x7, #0x30]\n"
+ "fmla v27.4s, v12.4s, v3.s[0]\n"
+ "add x14, x14, #0x10\n"
+ "prfm pldl1keep, [x14, #0x80]\n"
+ "fmla v28.4s, v12.4s, v4.s[0]\n"
+ "add x12, x12, #0x10\n"
+ "fmla v29.4s, v12.4s, v5.s[0]\n"
+ "prfm pldl1keep, [x12, #0x80]\n"
+ "add x10, x10, #0x10\n"
+ "fmla v30.4s, v12.4s, v6.s[0]\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x28, x28, #0x10\n"
+ "fmla v24.4s, v13.4s, v0.s[1]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "add x26, x26, #0x10\n"
+ "fmla v25.4s, v13.4s, v1.s[1]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "add x24, x24, #0x10\n"
+ "fmla v26.4s, v13.4s, v2.s[1]\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "add x22, x22, #0x10\n"
+ "fmla v27.4s, v13.4s, v3.s[1]\n"
+ "prfm pldl1keep, [x22, #0x80]\n"
+ "add x7, x7, #0x40\n"
+ "fmla v28.4s, v13.4s, v4.s[1]\n"
+ "fmla v29.4s, v13.4s, v5.s[1]\n"
+ "fmla v30.4s, v13.4s, v6.s[1]\n"
+ "fmla v24.4s, v14.4s, v0.s[2]\n"
+ "fmla v25.4s, v14.4s, v1.s[2]\n"
+ "fmla v26.4s, v14.4s, v2.s[2]\n"
+ "fmla v27.4s, v14.4s, v3.s[2]\n"
+ "fmla v28.4s, v14.4s, v4.s[2]\n"
+ "fmla v29.4s, v14.4s, v5.s[2]\n"
+ "fmla v30.4s, v14.4s, v6.s[2]\n"
+ "fmla v24.4s, v15.4s, v0.s[3]\n"
+ "fmla v25.4s, v15.4s, v1.s[3]\n"
+ "fmla v26.4s, v15.4s, v2.s[3]\n"
+ "fmla v27.4s, v15.4s, v3.s[3]\n"
+ "fmla v28.4s, v15.4s, v4.s[3]\n"
+ "fmla v29.4s, v15.4s, v5.s[3]\n"
+ "fmla v30.4s, v15.4s, v6.s[3]\n"
+ "147:" // Height 7: Multiply loop: Main loop skip
+ "cbz x15, 149f\n"
+ "148:" // Height 7: Multiply loop: Odd block loop
+ "ldr s0, [x14], #0x4\n"
+ "ldr s1, [x12], #0x4\n"
+ "ldr s2, [x10], #0x4\n"
+ "ldr s3, [x28], #0x4\n"
+ "ldr s4, [x26], #0x4\n"
+ "ldr s5, [x24], #0x4\n"
+ "ldr s6, [x22], #0x4\n"
+ "ldr q16, [x7, #0x0]\n"
+ "fmla v24.4s, v16.4s, v0.s[0]\n"
+ "sub x15, x15, #0x1\n"
+ "fmla v25.4s, v16.4s, v1.s[0]\n"
+ "add x7, x7, #0x10\n"
+ "fmla v26.4s, v16.4s, v2.s[0]\n"
+ "fmla v27.4s, v16.4s, v3.s[0]\n"
+ "fmla v28.4s, v16.4s, v4.s[0]\n"
+ "fmla v29.4s, v16.4s, v5.s[0]\n"
+ "fmla v30.4s, v16.4s, v6.s[0]\n"
+ "cbnz x15, 148b\n"
+ "149:" // Height 7: Multiply loop: No odd multiplies
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "add x16, x16, #0x1\n"
+ "cmp x16, x19\n"
+ "bne 142b\n"
+ "prfm pstl1keep, [x17, #0x0]\n"
+ "prfm pstl1keep, [x13, #0x0]\n"
+ "prfm pstl1keep, [x11, #0x0]\n"
+ "prfm pstl1keep, [x9, #0x0]\n"
+ "prfm pstl1keep, [x27, #0x0]\n"
+ "prfm pstl1keep, [x25, #0x0]\n"
+ "prfm pstl1keep, [x23, #0x0]\n"
+ "tbz %x[flags], #1, 150f\n"
+ "add x19, %x[args_ptr], %[offset_min]\n"
+ "ld1r { v17.4s }, [x19]\n"
+ "add x19, %x[args_ptr], %[offset_max]\n"
+ "ld1r { v16.4s }, [x19]\n"
+ "fmin v24.4s, v24.4s, v16.4s\n"
+ "fmin v25.4s, v25.4s, v16.4s\n"
+ "fmin v26.4s, v26.4s, v16.4s\n"
+ "fmin v27.4s, v27.4s, v16.4s\n"
+ "fmax v24.4s, v24.4s, v17.4s\n"
+ "fmax v25.4s, v25.4s, v17.4s\n"
+ "fmax v26.4s, v26.4s, v17.4s\n"
+ "fmax v27.4s, v27.4s, v17.4s\n"
+ "fmin v28.4s, v28.4s, v16.4s\n"
+ "fmin v29.4s, v29.4s, v16.4s\n"
+ "fmin v30.4s, v30.4s, v16.4s\n"
+ "fmax v28.4s, v28.4s, v17.4s\n"
+ "fmax v29.4s, v29.4s, v17.4s\n"
+ "fmax v30.4s, v30.4s, v17.4s\n"
+ "150:" // Height 7: No activation
+ "cmp x6, #0x4\n"
+ "bge 153f\n"
+ "tbz x6, #1, 151f\n"
+ "str d24, [x17], #0x8\n"
+ "str d25, [x13], #0x8\n"
+ "str d26, [x11], #0x8\n"
+ "str d27, [x9], #0x8\n"
+ "str d28, [x27], #0x8\n"
+ "str d29, [x25], #0x8\n"
+ "str d30, [x23], #0x8\n"
+ "tbz x6, #0, 152f\n"
+ "st1 { v24.s }[2], [x17]\n"
+ "st1 { v25.s }[2], [x13]\n"
+ "st1 { v26.s }[2], [x11]\n"
+ "st1 { v27.s }[2], [x9]\n"
+ "st1 { v28.s }[2], [x27]\n"
+ "st1 { v29.s }[2], [x25]\n"
+ "st1 { v30.s }[2], [x23]\n"
+ "b 152f\n"
+ "151:" // Height 7: Partial direct writeback: partial_1_0
+ "str s24, [x17, #0x0]\n"
+ "str s25, [x13, #0x0]\n"
+ "str s26, [x11, #0x0]\n"
+ "str s27, [x9, #0x0]\n"
+ "str s28, [x27, #0x0]\n"
+ "str s29, [x25, #0x0]\n"
+ "str s30, [x23, #0x0]\n"
+ "152:" // Height 7: Partial direct writeback: Done
+ "b 154f\n"
+ "153:" // Height 7: Full writeback
+ "str q24, [x17, #0x0]\n"
+ "str q25, [x13, #0x0]\n"
+ "str q26, [x11, #0x0]\n"
+ "str q27, [x9, #0x0]\n"
+ "str q28, [x27, #0x0]\n"
+ "str q29, [x25, #0x0]\n"
+ "str q30, [x23, #0x0]\n"
+ "add x17, x17, #0x10\n"
+ "add x13, x13, #0x10\n"
+ "add x11, x11, #0x10\n"
+ "add x9, x9, #0x10\n"
+ "add x27, x27, #0x10\n"
+ "add x25, x25, #0x10\n"
+ "add x23, x23, #0x10\n"
+ "154:" // Height 7: Writeback done
+ "subs x6, x6, #0x4\n"
+ "bgt 135b\n"
+ "b 178f\n"
+ "155:" // Height 8
+ "ldr x6, [%x[args_ptr], %[offsetof_N]]\n"
+ "mov x8, %x[bias]\n"
+ "ldr x7, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 156f\n"
+ "ldr x17, [%x[output_ptr], #0x0]\n"
+ "add x17, x17, x19, LSL #2\n"
+ "ldr x13, [%x[output_ptr], #0x8]\n"
+ "ldr x11, [%x[output_ptr], #0x10]\n"
+ "add x13, x13, x19, LSL #2\n"
+ "ldr x9, [%x[output_ptr], #0x18]\n"
+ "ldr x27, [%x[output_ptr], #0x20]\n"
+ "add x11, x11, x19, LSL #2\n"
+ "ldr x25, [%x[output_ptr], #0x28]\n"
+ "add x9, x9, x19, LSL #2\n"
+ "ldr x23, [%x[output_ptr], #0x30]\n"
+ "ldr x21, [%x[output_ptr], #0x38]\n"
+ "add x27, x27, x19, LSL #2\n"
+ "add x25, x25, x19, LSL #2\n"
+ "add %x[output_ptr], %x[output_ptr], #0x40\n"
+ "add x23, x23, x19, LSL #2\n"
+ "add x21, x21, x19, LSL #2\n"
+ "b 157f\n"
+ "156:" // Height 8: setup direct output
+ "mov x17, %x[output_ptr]\n"
+ "add x13, x17, x19, LSL #2\n"
+ "add x11, x13, x19, LSL #2\n"
+ "add x9, x11, x19, LSL #2\n"
+ "add x27, x9, x19, LSL #2\n"
+ "add x25, x27, x19, LSL #2\n"
+ "add x23, x25, x19, LSL #2\n"
+ "add x21, x23, x19, LSL #2\n"
+ "add %x[output_ptr], x21, x19, LSL #2\n"
+ "157:" // Height 8: Column loop
+ "cbz x8, 158f\n"
+ "ldr q24, [x8, #0x0]\n"
+ "mov v25.16b, v24.16b\n"
+ "add x8, x8, #0x10\n"
+ "mov v26.16b, v24.16b\n"
+ "mov v27.16b, v24.16b\n"
+ "mov v28.16b, v24.16b\n"
+ "mov v29.16b, v24.16b\n"
+ "mov v30.16b, v24.16b\n"
+ "mov v31.16b, v24.16b\n"
+ "b 163f\n"
+ "158:" // Height 8: no bias
+ "tbz %x[flags], #0, 162f\n"
+ "cmp x6, #0x4\n"
+ "bge 161f\n"
+ "tbz x6, #1, 159f\n"
+ "ldr d24, [x17], #0x8\n"
+ "ldr d25, [x13], #0x8\n"
+ "ldr d26, [x11], #0x8\n"
+ "ldr d27, [x9], #0x8\n"
+ "ldr d28, [x27], #0x8\n"
+ "ldr d29, [x25], #0x8\n"
+ "ldr d30, [x23], #0x8\n"
+ "ldr d31, [x21], #0x8\n"
+ "mov x19, #0x8\n"
+ "tbz x6, #0, 160f\n"
+ "ld1 { v24.s }[2], [x17]\n"
+ "ld1 { v25.s }[2], [x13]\n"
+ "ld1 { v26.s }[2], [x11]\n"
+ "ld1 { v27.s }[2], [x9]\n"
+ "ld1 { v28.s }[2], [x27]\n"
+ "ld1 { v29.s }[2], [x25]\n"
+ "ld1 { v30.s }[2], [x23]\n"
+ "ld1 { v31.s }[2], [x21]\n"
+ "b 160f\n"
+ "159:" // Height 8: Partial accumulate: partial_1_0
+ "mov x19, #0x0\n"
+ "ldr s24, [x17, #0x0]\n"
+ "ldr s25, [x13, #0x0]\n"
+ "ldr s26, [x11, #0x0]\n"
+ "ldr s27, [x9, #0x0]\n"
+ "ldr s28, [x27, #0x0]\n"
+ "ldr s29, [x25, #0x0]\n"
+ "ldr s30, [x23, #0x0]\n"
+ "ldr s31, [x21, #0x0]\n"
+ "160:" // Height 8: Partial accumulate: Done
+ "sub x17, x17, x19\n"
+ "sub x13, x13, x19\n"
+ "sub x11, x11, x19\n"
+ "sub x9, x9, x19\n"
+ "sub x27, x27, x19\n"
+ "sub x25, x25, x19\n"
+ "sub x23, x23, x19\n"
+ "sub x21, x21, x19\n"
+ "b 163f\n"
+ "161:" // Height 8: full accumulate
+ "ldr q24, [x17, #0x0]\n"
+ "ldr q25, [x13, #0x0]\n"
+ "ldr q26, [x11, #0x0]\n"
+ "ldr q27, [x9, #0x0]\n"
+ "ldr q28, [x27, #0x0]\n"
+ "ldr q29, [x25, #0x0]\n"
+ "ldr q30, [x23, #0x0]\n"
+ "ldr q31, [x21, #0x0]\n"
+ "b 163f\n"
+ "162:" // Height 8: no accumulate
+ "movi v24.16b, #0x0\n"
+ "movi v25.16b, #0x0\n"
+ "movi v26.16b, #0x0\n"
+ "movi v27.16b, #0x0\n"
+ "movi v28.16b, #0x0\n"
+ "movi v29.16b, #0x0\n"
+ "movi v30.16b, #0x0\n"
+ "movi v31.16b, #0x0\n"
+ "163:" // Height 8: setup done
+ "mov x16, #0x0\n"
+ "164:" // Height 8: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w15, [x20, x16, LSL #0x2]\n"
+ "tbz %x[flags], #3, 165f\n"
+ "ldr x20, [%x[input_ptr], x16, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x14, [x20, #0x0]\n"
+ "ldr x12, [x20, #0x8]\n"
+ "ldr x10, [x20, #0x10]\n"
+ "ldr x28, [x20, #0x18]\n"
+ "ldr x26, [x20, #0x20]\n"
+ "ldr x24, [x20, #0x28]\n"
+ "ldr x22, [x20, #0x30]\n"
+ "ldr x20, [x20, #0x38]\n"
+ "cbnz x16, 166f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x14, x14, x19, LSL #2\n"
+ "add x12, x12, x19, LSL #2\n"
+ "add x10, x10, x19, LSL #2\n"
+ "add x28, x28, x19, LSL #2\n"
+ "add x26, x26, x19, LSL #2\n"
+ "add x24, x24, x19, LSL #2\n"
+ "add x22, x22, x19, LSL #2\n"
+ "add x20, x20, x19, LSL #2\n"
+ "b 166f\n"
+ "165:" // Height 8: setup direct input
+ "mov x14, %x[input_ptr]\n"
+ "add x12, x14, x19, LSL #2\n"
+ "add x10, x12, x19, LSL #2\n"
+ "add x28, x10, x19, LSL #2\n"
+ "add x26, x28, x19, LSL #2\n"
+ "add x24, x26, x19, LSL #2\n"
+ "add x22, x24, x19, LSL #2\n"
+ "add x20, x22, x19, LSL #2\n"
+ "166:" // Height 8: input setup done
+ "cmp x15, #0x4\n"
+ "blt 169f\n"
+ "cmp x15, #0x8\n"
+ "blt 168f\n"
+ "167:" // Height 8: Multiply loop: Main loop head
+ "ldr q0, [x14, #0x0]\n"
+ "ldr q1, [x12, #0x0]\n"
+ "ldr q2, [x10, #0x0]\n"
+ "ldr q3, [x28, #0x0]\n"
+ "ldr q4, [x26, #0x0]\n"
+ "ldr q5, [x24, #0x0]\n"
+ "ldr q6, [x22, #0x0]\n"
+ "ldr q7, [x20, #0x0]\n"
+ "ldr q8, [x7, #0x0]\n"
+ "fmla v24.4s, v8.4s, v0.s[0]\n"
+ "ldr q9, [x7, #0x10]\n"
+ "fmla v25.4s, v8.4s, v1.s[0]\n"
+ "ldr q10, [x7, #0x20]\n"
+ "fmla v26.4s, v8.4s, v2.s[0]\n"
+ "ldr q11, [x7, #0x30]\n"
+ "fmla v27.4s, v8.4s, v3.s[0]\n"
+ "add x14, x14, #0x10\n"
+ "prfm pldl1keep, [x14, #0x80]\n"
+ "fmla v28.4s, v8.4s, v4.s[0]\n"
+ "add x12, x12, #0x10\n"
+ "fmla v29.4s, v8.4s, v5.s[0]\n"
+ "prfm pldl1keep, [x12, #0x80]\n"
+ "add x10, x10, #0x10\n"
+ "fmla v30.4s, v8.4s, v6.s[0]\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x28, x28, #0x10\n"
+ "fmla v31.4s, v8.4s, v7.s[0]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "add x26, x26, #0x10\n"
+ "fmla v24.4s, v9.4s, v0.s[1]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "add x24, x24, #0x10\n"
+ "fmla v25.4s, v9.4s, v1.s[1]\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "add x22, x22, #0x10\n"
+ "fmla v26.4s, v9.4s, v2.s[1]\n"
+ "prfm pldl1keep, [x22, #0x80]\n"
+ "add x20, x20, #0x10\n"
+ "fmla v27.4s, v9.4s, v3.s[1]\n"
+ "prfm pldl1keep, [x20, #0x80]\n"
+ "sub x15, x15, #0x4\n"
+ "fmla v28.4s, v9.4s, v4.s[1]\n"
+ "cmp x15, #0x8\n"
+ "fmla v29.4s, v9.4s, v5.s[1]\n"
+ "add x7, x7, #0x40\n"
+ "fmla v30.4s, v9.4s, v6.s[1]\n"
+ "fmla v31.4s, v9.4s, v7.s[1]\n"
+ "fmla v24.4s, v10.4s, v0.s[2]\n"
+ "fmla v25.4s, v10.4s, v1.s[2]\n"
+ "fmla v26.4s, v10.4s, v2.s[2]\n"
+ "fmla v27.4s, v10.4s, v3.s[2]\n"
+ "fmla v28.4s, v10.4s, v4.s[2]\n"
+ "fmla v29.4s, v10.4s, v5.s[2]\n"
+ "fmla v30.4s, v10.4s, v6.s[2]\n"
+ "fmla v31.4s, v10.4s, v7.s[2]\n"
+ "fmla v24.4s, v11.4s, v0.s[3]\n"
+ "fmla v25.4s, v11.4s, v1.s[3]\n"
+ "fmla v26.4s, v11.4s, v2.s[3]\n"
+ "fmla v27.4s, v11.4s, v3.s[3]\n"
+ "fmla v28.4s, v11.4s, v4.s[3]\n"
+ "fmla v29.4s, v11.4s, v5.s[3]\n"
+ "fmla v30.4s, v11.4s, v6.s[3]\n"
+ "fmla v31.4s, v11.4s, v7.s[3]\n"
+ "bge 167b\n"
+ "168:" // Height 8: Multiply loop: Single iteration only
+ "sub x15, x15, #0x4\n"
+ "ldr q0, [x14, #0x0]\n"
+ "ldr q1, [x12, #0x0]\n"
+ "ldr q2, [x10, #0x0]\n"
+ "ldr q3, [x28, #0x0]\n"
+ "ldr q4, [x26, #0x0]\n"
+ "ldr q5, [x24, #0x0]\n"
+ "ldr q6, [x22, #0x0]\n"
+ "ldr q7, [x20, #0x0]\n"
+ "ldr q12, [x7, #0x0]\n"
+ "fmla v24.4s, v12.4s, v0.s[0]\n"
+ "ldr q13, [x7, #0x10]\n"
+ "fmla v25.4s, v12.4s, v1.s[0]\n"
+ "ldr q14, [x7, #0x20]\n"
+ "fmla v26.4s, v12.4s, v2.s[0]\n"
+ "ldr q15, [x7, #0x30]\n"
+ "fmla v27.4s, v12.4s, v3.s[0]\n"
+ "add x14, x14, #0x10\n"
+ "prfm pldl1keep, [x14, #0x80]\n"
+ "fmla v28.4s, v12.4s, v4.s[0]\n"
+ "add x12, x12, #0x10\n"
+ "fmla v29.4s, v12.4s, v5.s[0]\n"
+ "prfm pldl1keep, [x12, #0x80]\n"
+ "add x10, x10, #0x10\n"
+ "fmla v30.4s, v12.4s, v6.s[0]\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x28, x28, #0x10\n"
+ "fmla v31.4s, v12.4s, v7.s[0]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "add x26, x26, #0x10\n"
+ "fmla v24.4s, v13.4s, v0.s[1]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "add x24, x24, #0x10\n"
+ "fmla v25.4s, v13.4s, v1.s[1]\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "add x22, x22, #0x10\n"
+ "fmla v26.4s, v13.4s, v2.s[1]\n"
+ "prfm pldl1keep, [x22, #0x80]\n"
+ "add x20, x20, #0x10\n"
+ "fmla v27.4s, v13.4s, v3.s[1]\n"
+ "prfm pldl1keep, [x20, #0x80]\n"
+ "add x7, x7, #0x40\n"
+ "fmla v28.4s, v13.4s, v4.s[1]\n"
+ "fmla v29.4s, v13.4s, v5.s[1]\n"
+ "fmla v30.4s, v13.4s, v6.s[1]\n"
+ "fmla v31.4s, v13.4s, v7.s[1]\n"
+ "fmla v24.4s, v14.4s, v0.s[2]\n"
+ "fmla v25.4s, v14.4s, v1.s[2]\n"
+ "fmla v26.4s, v14.4s, v2.s[2]\n"
+ "fmla v27.4s, v14.4s, v3.s[2]\n"
+ "fmla v28.4s, v14.4s, v4.s[2]\n"
+ "fmla v29.4s, v14.4s, v5.s[2]\n"
+ "fmla v30.4s, v14.4s, v6.s[2]\n"
+ "fmla v31.4s, v14.4s, v7.s[2]\n"
+ "fmla v24.4s, v15.4s, v0.s[3]\n"
+ "fmla v25.4s, v15.4s, v1.s[3]\n"
+ "fmla v26.4s, v15.4s, v2.s[3]\n"
+ "fmla v27.4s, v15.4s, v3.s[3]\n"
+ "fmla v28.4s, v15.4s, v4.s[3]\n"
+ "fmla v29.4s, v15.4s, v5.s[3]\n"
+ "fmla v30.4s, v15.4s, v6.s[3]\n"
+ "fmla v31.4s, v15.4s, v7.s[3]\n"
+ "169:" // Height 8: Multiply loop: Main loop skip
+ "cbz x15, 171f\n"
+ "170:" // Height 8: Multiply loop: Odd block loop
+ "ldr s0, [x14], #0x4\n"
+ "ldr s1, [x12], #0x4\n"
+ "ldr s2, [x10], #0x4\n"
+ "ldr s3, [x28], #0x4\n"
+ "ldr s4, [x26], #0x4\n"
+ "ldr s5, [x24], #0x4\n"
+ "ldr s6, [x22], #0x4\n"
+ "ldr s7, [x20], #0x4\n"
+ "ldr q16, [x7, #0x0]\n"
+ "fmla v24.4s, v16.4s, v0.s[0]\n"
+ "sub x15, x15, #0x1\n"
+ "fmla v25.4s, v16.4s, v1.s[0]\n"
+ "add x7, x7, #0x10\n"
+ "fmla v26.4s, v16.4s, v2.s[0]\n"
+ "fmla v27.4s, v16.4s, v3.s[0]\n"
+ "fmla v28.4s, v16.4s, v4.s[0]\n"
+ "fmla v29.4s, v16.4s, v5.s[0]\n"
+ "fmla v30.4s, v16.4s, v6.s[0]\n"
+ "fmla v31.4s, v16.4s, v7.s[0]\n"
+ "cbnz x15, 170b\n"
+ "171:" // Height 8: Multiply loop: No odd multiplies
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "add x16, x16, #0x1\n"
+ "cmp x16, x19\n"
+ "bne 164b\n"
+ "prfm pstl1keep, [x17, #0x0]\n"
+ "prfm pstl1keep, [x13, #0x0]\n"
+ "prfm pstl1keep, [x11, #0x0]\n"
+ "prfm pstl1keep, [x9, #0x0]\n"
+ "prfm pstl1keep, [x27, #0x0]\n"
+ "prfm pstl1keep, [x25, #0x0]\n"
+ "prfm pstl1keep, [x23, #0x0]\n"
+ "prfm pstl1keep, [x21, #0x0]\n"
+ "tbz %x[flags], #1, 172f\n"
+ "add x19, %x[args_ptr], %[offset_min]\n"
+ "ld1r { v17.4s }, [x19]\n"
+ "add x19, %x[args_ptr], %[offset_max]\n"
+ "ld1r { v16.4s }, [x19]\n"
+ "fmin v24.4s, v24.4s, v16.4s\n"
+ "fmin v25.4s, v25.4s, v16.4s\n"
+ "fmin v26.4s, v26.4s, v16.4s\n"
+ "fmin v27.4s, v27.4s, v16.4s\n"
+ "fmax v24.4s, v24.4s, v17.4s\n"
+ "fmax v25.4s, v25.4s, v17.4s\n"
+ "fmax v26.4s, v26.4s, v17.4s\n"
+ "fmax v27.4s, v27.4s, v17.4s\n"
+ "fmin v28.4s, v28.4s, v16.4s\n"
+ "fmin v29.4s, v29.4s, v16.4s\n"
+ "fmin v30.4s, v30.4s, v16.4s\n"
+ "fmax v28.4s, v28.4s, v17.4s\n"
+ "fmax v29.4s, v29.4s, v17.4s\n"
+ "fmax v30.4s, v30.4s, v17.4s\n"
+ "fmin v31.4s, v31.4s, v16.4s\n"
+ "fmax v31.4s, v31.4s, v17.4s\n"
+ "172:" // Height 8: No activation
+ "cmp x6, #0x4\n"
+ "bge 175f\n"
+ "tbz x6, #1, 173f\n"
+ "str d24, [x17], #0x8\n"
+ "str d25, [x13], #0x8\n"
+ "str d26, [x11], #0x8\n"
+ "str d27, [x9], #0x8\n"
+ "str d28, [x27], #0x8\n"
+ "str d29, [x25], #0x8\n"
+ "str d30, [x23], #0x8\n"
+ "str d31, [x21], #0x8\n"
+ "tbz x6, #0, 174f\n"
+ "st1 { v24.s }[2], [x17]\n"
+ "st1 { v25.s }[2], [x13]\n"
+ "st1 { v26.s }[2], [x11]\n"
+ "st1 { v27.s }[2], [x9]\n"
+ "st1 { v28.s }[2], [x27]\n"
+ "st1 { v29.s }[2], [x25]\n"
+ "st1 { v30.s }[2], [x23]\n"
+ "st1 { v31.s }[2], [x21]\n"
+ "b 174f\n"
+ "173:" // Height 8: Partial direct writeback: partial_1_0
+ "str s24, [x17, #0x0]\n"
+ "str s25, [x13, #0x0]\n"
+ "str s26, [x11, #0x0]\n"
+ "str s27, [x9, #0x0]\n"
+ "str s28, [x27, #0x0]\n"
+ "str s29, [x25, #0x0]\n"
+ "str s30, [x23, #0x0]\n"
+ "str s31, [x21, #0x0]\n"
+ "174:" // Height 8: Partial direct writeback: Done
+ "b 176f\n"
+ "175:" // Height 8: Full writeback
+ "str q24, [x17, #0x0]\n"
+ "str q25, [x13, #0x0]\n"
+ "str q26, [x11, #0x0]\n"
+ "str q27, [x9, #0x0]\n"
+ "str q28, [x27, #0x0]\n"
+ "str q29, [x25, #0x0]\n"
+ "str q30, [x23, #0x0]\n"
+ "str q31, [x21, #0x0]\n"
+ "add x17, x17, #0x10\n"
+ "add x13, x13, #0x10\n"
+ "add x11, x11, #0x10\n"
+ "add x9, x9, #0x10\n"
+ "add x27, x27, #0x10\n"
+ "add x25, x25, #0x10\n"
+ "add x23, x23, #0x10\n"
+ "add x21, x21, #0x10\n"
+ "176:" // Height 8: Writeback done
+ "subs x6, x6, #0x4\n"
+ "bgt 157b\n"
+ "subs %x[M], %x[M], #0x8\n"
+ "beq 178f\n"
+ "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "tbz %x[flags], #3, 177f\n"
+ "add x20, x20, #0x8\n"
+ "str x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "b 1b\n"
+ "177:" // Update direct input
+ "mov x19, #0x20\n"
+ "madd %x[input_ptr], x19, x20, %x[input_ptr]\n"
+ "b 1b\n"
+ "178:" // Exit
+
+ : [M] "+r" (M), [input_ptr] "+r" (input_ptr), [output_ptr] "+r" (output_ptr)
+ : [args_ptr] "r" (&ka), [bias] "r" (bias), [flags] "r" (flags), [offset_max] "I" (offsetof(KernelArgs, maxval)), [offset_min] "I" (offsetof(KernelArgs, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths))
+ : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
+ );
+}
+
+} // namespace arm_gemm
+#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_4x6.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qa_dot_4x16.hpp
similarity index 62%
copy from src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_4x6.hpp
copy to src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qa_dot_4x16.hpp
index 6f31efe..4bb7a1e 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_4x6.hpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qa_dot_4x16.hpp
@@ -10,49 +10,58 @@
* sell copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
*/
#pragma once
-
#ifdef __aarch64__
+#include "../std_transforms_fixed.hpp"
+
+#define ARGLIST \
+ unsigned int, const unsigned int *, \
+ IndirectInputArg<int8_t>, \
+ size_t, size_t, \
+ const int8_t *, \
+ IndirectOutputArg<int8_t>, \
+ const Requantize32 *, const int32_t *, unsigned int
+
namespace arm_gemm
{
// Actual kernel implementations
-void a64_smallK_hybrid_fp32_mla_4x6(const float *, int, const float *, float *, int, int, int, int, const float *, Activation, bool);
+void a64_hybrid_s8qa_dot_4x16( ARGLIST );
-class smallK_hybrid_fp32_mla_4x6
+class cls_a64_hybrid_s8qa_dot_4x16
{
public:
- typedef float operand_type;
- typedef float result_type;
+ typedef int8_t operand_type;
+ typedef int8_t result_type;
- typedef void (*kern_type)(const float *, int, const float *, float *, int, int, int, int, const float *, Activation, bool);
+ typedef void (*kern_type)( ARGLIST );
/* Kernel blocking parameters */
static constexpr unsigned int out_height()
{
- return 6;
+ return 4;
}
static unsigned int out_width()
{
- return 4;
+ return 16;
}
static constexpr unsigned int k_unroll()
{
- return 1;
+ return 4;
}
static constexpr bool supports_accumulate()
@@ -60,27 +69,17 @@
return false;
}
- static constexpr bool supports_bias()
- {
- return true;
- }
-
- static constexpr bool supports_activation()
- {
- return true;
- }
-
- StdTransformsFixed<operand_type, result_type, 6, 4, 1> transforms = {};
+ StdTransformsFixed<operand_type, result_type, 4, 16, 4> transforms = {};
// Default to the generic kernel
- kern_type kernel=a64_smallK_hybrid_fp32_mla_4x6;
+ kern_type kernel=a64_hybrid_s8qa_dot_4x16;
- smallK_hybrid_fp32_mla_4x6(const CPUInfo *)
+ cls_a64_hybrid_s8qa_dot_4x16(const CPUInfo *)
{
-
}
};
} // namespace arm_gemm
+#undef ARGLIST
#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qa_dot_4x16/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qa_dot_4x16/generic.cpp
new file mode 100644
index 0000000..3fb365b
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qa_dot_4x16/generic.cpp
@@ -0,0 +1,2072 @@
+/*
+ * Copyright (c) 2019-2020 Arm Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+#ifdef __aarch64__
+
+#include "arm_gemm.hpp"
+#include "../../utils.hpp"
+
+#include <cassert>
+#include <cstdint>
+
+namespace arm_gemm {
+
+void a64_hybrid_s8qa_dot_4x16 (
+ unsigned int num_strings, const unsigned int *string_lengths, IndirectInputArg<int8_t> A_arg,
+ size_t M, size_t N, const int8_t *B_ptr, IndirectOutputArg<int8_t> output_arg,
+ const Requantize32 *qp, const int32_t *col_bias, unsigned int
+)
+{
+ struct KernelArgs {
+ unsigned int num_strings = {};
+ const unsigned int *string_lengths = {};
+ size_t N = {};
+ const int8_t *B_ptr = {};
+ size_t output_offset = {};
+ size_t input_initial_col = {};
+ size_t input_offset = {};
+ } ka;
+
+ unsigned long flags=0;
+ void *output_ptr;
+ void *input_ptr;
+
+ if (output_arg.is_indirect) {
+ output_ptr=(void *)(output_arg.indirect.ptr);
+ ka.output_offset=output_arg.indirect.offset;
+ flags |= 0x4;
+ } else {
+ output_ptr=(void *)(output_arg.direct.base);
+ ka.output_offset=output_arg.direct.stride;
+ }
+
+ if (A_arg.is_indirect) {
+ input_ptr=(void *)(A_arg.indirect.ptr);
+ ka.input_offset=A_arg.indirect.start_row;
+ ka.input_initial_col=A_arg.indirect.start_col;
+ flags |= 0x8;
+ } else {
+ assert(num_strings==1);
+ input_ptr=(void *)(A_arg.direct.base);
+ ka.input_offset=A_arg.direct.stride;
+ }
+ ka.num_strings = num_strings;
+ ka.string_lengths = string_lengths;
+ ka.N = N;
+ ka.B_ptr = B_ptr;
+ if (qp->c_offset > qp->minval) {
+ flags |= 0x20;
+ }
+ __asm__ __volatile__(
+
+ "1:" // Row loop
+ "cmp %x[M], #0x4\n"
+ "bge 94f\n"
+ "cmp %x[M], #0x2\n"
+ "bgt 63f\n"
+ "beq 32f\n"
+ "movi v11.4s, #0x0\n"
+ "ldr x12, [%x[args_ptr], %[offsetof_N]]\n"
+ "movi v12.4s, #0x0\n"
+ "ldr x11, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "mov x10, %x[col_bias]\n"
+ "movi v13.4s, #0x0\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "bic %x[flags], %x[flags], #0x80000000\n"
+ "movi v14.4s, #0x0\n"
+ "movi v15.16b, #0x1\n"
+ "tbz %x[flags], #2, 2f\n"
+ "ldr x9, [%x[output_ptr], #0x0]\n"
+ "add x9, x9, x19\n"
+ "b 3f\n"
+ "2:" // Height 1: setup direct output
+ "mov x9, %x[output_ptr]\n"
+ "3:" // Height 1: Column loop
+ "movi v16.4s, #0x0\n"
+ "movi v17.4s, #0x0\n"
+ "movi v18.4s, #0x0\n"
+ "movi v19.4s, #0x0\n"
+ "4:" // Height 1: setup done
+ "mov x28, #0x0\n"
+ "5:" // Height 1: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w27, [x20, x28, LSL #0x2]\n"
+ "tbz %x[flags], #3, 6f\n"
+ "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x26, [x20, #0x0]\n"
+ "cbnz x28, 7f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x26, x26, x19\n"
+ "b 7f\n"
+ "6:" // Height 1: setup direct input
+ "mov x26, %x[input_ptr]\n"
+ "7:" // Height 1: input setup done
+ "cmp x27, #0x10\n"
+ "blt 12f\n"
+ "cmp x27, #0x20\n"
+ "blt 10f\n"
+ "8:" // Height 1: Multiply loop: Main loop head
+ "ldr q0, [x26, #0x0]\n"
+ "ldr q4, [x11, #0x0]\n"
+ ".inst 0x4f80e090 // sdot v16.4s, v4.16b, v0.4b[0]\n"
+ "ldr q5, [x11, #0x10]\n"
+ "ldr q6, [x11, #0x20]\n"
+ ".inst 0x4f80e0b1 // sdot v17.4s, v5.16b, v0.4b[0]\n"
+ "ldr q7, [x11, #0x30]\n"
+ ".inst 0x4f80e0d2 // sdot v18.4s, v6.16b, v0.4b[0]\n"
+ "ldr q8, [x11, #0x40]\n"
+ "ldr q9, [x11, #0x50]\n"
+ ".inst 0x4f80e0f3 // sdot v19.4s, v7.16b, v0.4b[0]\n"
+ "ldr q10, [x11, #0x60]\n"
+ "ldr q4, [x11, #0x70]\n"
+ ".inst 0x4fa0e110 // sdot v16.4s, v8.16b, v0.4b[1]\n"
+ ".inst 0x4fa0e131 // sdot v17.4s, v9.16b, v0.4b[1]\n"
+ "ldr q5, [x11, #0x80]\n"
+ "ldr q6, [x11, #0x90]\n"
+ ".inst 0x4fa0e152 // sdot v18.4s, v10.16b, v0.4b[1]\n"
+ "ldr q7, [x11, #0xa0]\n"
+ ".inst 0x4fa0e093 // sdot v19.4s, v4.16b, v0.4b[1]\n"
+ "ldr q8, [x11, #0xb0]\n"
+ "ldr q9, [x11, #0xc0]\n"
+ ".inst 0x4f80e8b0 // sdot v16.4s, v5.16b, v0.4b[2]\n"
+ ".inst 0x4f80e8d1 // sdot v17.4s, v6.16b, v0.4b[2]\n"
+ "ldr q10, [x11, #0xd0]\n"
+ "ldr q4, [x11, #0xe0]\n"
+ ".inst 0x4f80e8f2 // sdot v18.4s, v7.16b, v0.4b[2]\n"
+ "ldr q5, [x11, #0xf0]\n"
+ "add x26, x26, #0x10\n"
+ ".inst 0x4f80e913 // sdot v19.4s, v8.16b, v0.4b[2]\n"
+ ".inst 0x4fa0e930 // sdot v16.4s, v9.16b, v0.4b[3]\n"
+ "add x11, x11, #0x100\n"
+ ".inst 0x4fa0e951 // sdot v17.4s, v10.16b, v0.4b[3]\n"
+ ".inst 0x4fa0e892 // sdot v18.4s, v4.16b, v0.4b[3]\n"
+ ".inst 0x4fa0e8b3 // sdot v19.4s, v5.16b, v0.4b[3]\n"
+ "tbnz %x[flags], #31, 9f\n"
+ ".inst 0x4e8f940b // sdot v11.4s, v0.16b, v15.16b\n"
+ "9:" // Height 1: Multiply loop: unique 1: skip row sum
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "sub x27, x27, #0x10\n"
+ "cmp x27, #0x20\n"
+ "bge 8b\n"
+ "10:" // Height 1: Multiply loop: Single iteration only
+ "sub x27, x27, #0x10\n"
+ "ldr q0, [x26, #0x0]\n"
+ "ldr q6, [x11, #0x0]\n"
+ ".inst 0x4f80e0d0 // sdot v16.4s, v6.16b, v0.4b[0]\n"
+ "ldr q7, [x11, #0x10]\n"
+ "ldr q8, [x11, #0x20]\n"
+ ".inst 0x4f80e0f1 // sdot v17.4s, v7.16b, v0.4b[0]\n"
+ "ldr q9, [x11, #0x30]\n"
+ ".inst 0x4f80e112 // sdot v18.4s, v8.16b, v0.4b[0]\n"
+ "ldr q10, [x11, #0x40]\n"
+ "ldr q4, [x11, #0x50]\n"
+ ".inst 0x4f80e133 // sdot v19.4s, v9.16b, v0.4b[0]\n"
+ "ldr q5, [x11, #0x60]\n"
+ "ldr q6, [x11, #0x70]\n"
+ ".inst 0x4fa0e150 // sdot v16.4s, v10.16b, v0.4b[1]\n"
+ ".inst 0x4fa0e091 // sdot v17.4s, v4.16b, v0.4b[1]\n"
+ "ldr q7, [x11, #0x80]\n"
+ "ldr q8, [x11, #0x90]\n"
+ ".inst 0x4fa0e0b2 // sdot v18.4s, v5.16b, v0.4b[1]\n"
+ "ldr q9, [x11, #0xa0]\n"
+ ".inst 0x4fa0e0d3 // sdot v19.4s, v6.16b, v0.4b[1]\n"
+ "ldr q10, [x11, #0xb0]\n"
+ "ldr q4, [x11, #0xc0]\n"
+ ".inst 0x4f80e8f0 // sdot v16.4s, v7.16b, v0.4b[2]\n"
+ ".inst 0x4f80e911 // sdot v17.4s, v8.16b, v0.4b[2]\n"
+ "ldr q5, [x11, #0xd0]\n"
+ "ldr q6, [x11, #0xe0]\n"
+ ".inst 0x4f80e932 // sdot v18.4s, v9.16b, v0.4b[2]\n"
+ "ldr q7, [x11, #0xf0]\n"
+ "add x26, x26, #0x10\n"
+ ".inst 0x4f80e953 // sdot v19.4s, v10.16b, v0.4b[2]\n"
+ ".inst 0x4fa0e890 // sdot v16.4s, v4.16b, v0.4b[3]\n"
+ "add x11, x11, #0x100\n"
+ ".inst 0x4fa0e8b1 // sdot v17.4s, v5.16b, v0.4b[3]\n"
+ ".inst 0x4fa0e8d2 // sdot v18.4s, v6.16b, v0.4b[3]\n"
+ ".inst 0x4fa0e8f3 // sdot v19.4s, v7.16b, v0.4b[3]\n"
+ "tbnz %x[flags], #31, 11f\n"
+ ".inst 0x4e8f940b // sdot v11.4s, v0.16b, v15.16b\n"
+ "11:" // Height 1: Multiply loop: unique 2: skip row sum
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "12:" // Height 1: Multiply loop: Main loop skip
+ "cbz x27, 19f\n"
+ "cmp x27, #0x4\n"
+ "blt 15f\n"
+ "13:" // Height 1: Multiply loop: Odd block loop
+ "ldr s0, [x26], #0x4\n"
+ "tbnz %x[flags], #31, 14f\n"
+ ".inst 0x4e8f940b // sdot v11.4s, v0.16b, v15.16b\n"
+ "14:" // Height 1: Multiply loop: unique 3: skip row sum
+ "ldr q8, [x11, #0x0]\n"
+ ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
+ "ldr q9, [x11, #0x10]\n"
+ "ldr q10, [x11, #0x20]\n"
+ ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
+ "ldr q4, [x11, #0x30]\n"
+ ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
+ "sub x27, x27, #0x4\n"
+ "add x11, x11, #0x40\n"
+ ".inst 0x4f80e093 // sdot v19.4s, v4.16b, v0.4b[0]\n"
+ "cmp x27, #0x4\n"
+ "bge 13b\n"
+ "cbz x27, 19f\n"
+ "15:" // Height 1: Multiply loop: Skip odd blocks
+ "tbz x27, #1, 16f\n"
+ "ldr h0, [x26], #0x2\n"
+ "tbz x27, #0, 17f\n"
+ "ld1 { v0.b }[2], [x26]\n"
+ "b 17f\n"
+ "16:" // Height 1: Multiply loop: Ragged operand read: partial_1_0
+ "ldr b0, [x26, #0x0]\n"
+ "17:" // Height 1: Multiply loop: Ragged operand read: Done
+ "tbnz %x[flags], #31, 18f\n"
+ ".inst 0x4e8f940b // sdot v11.4s, v0.16b, v15.16b\n"
+ "18:" // Height 1: Multiply loop: unique 4: skip row sum
+ "ldr q5, [x11, #0x0]\n"
+ ".inst 0x4f80e0b0 // sdot v16.4s, v5.16b, v0.4b[0]\n"
+ "ldr q6, [x11, #0x10]\n"
+ "ldr q7, [x11, #0x20]\n"
+ ".inst 0x4f80e0d1 // sdot v17.4s, v6.16b, v0.4b[0]\n"
+ "ldr q8, [x11, #0x30]\n"
+ ".inst 0x4f80e0f2 // sdot v18.4s, v7.16b, v0.4b[0]\n"
+ "add x11, x11, #0x40\n"
+ ".inst 0x4f80e113 // sdot v19.4s, v8.16b, v0.4b[0]\n"
+ "19:" // Height 1: Multiply loop: No odd multiplies
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "add x28, x28, #0x1\n"
+ "cmp x28, x19\n"
+ "bne 5b\n"
+ "prfm pstl1keep, [x9, #0x0]\n"
+ "tbnz %x[flags], #31, 20f\n"
+ "addp v11.4s, v11.4s, v11.4s\n"
+ "add x19, %x[qp], %[b_offset]\n"
+ "addp v11.4s, v11.4s, v11.4s\n"
+ "ld1r { v1.4s }, [x19]\n"
+ "neg v1.4s, v1.4s\n"
+ "mul v11.4s, v11.4s, v1.4s\n"
+ "20:" // Height 1: skip row sum fixup
+ "add v16.4s, v16.4s, v11.4s\n"
+ "orr %x[flags], %x[flags], #0x80000000\n"
+ "add v17.4s, v17.4s, v11.4s\n"
+ "ldr q0, [x10, #0x0]\n"
+ "add v18.4s, v18.4s, v11.4s\n"
+ "ldr q1, [x10, #0x10]\n"
+ "add v19.4s, v19.4s, v11.4s\n"
+ "ldr q2, [x10, #0x20]\n"
+ "ldr q3, [x10, #0x30]\n"
+ "add v16.4s, v16.4s, v0.4s\n"
+ "add x20, %x[qp], %[per_layer_right_shift]\n"
+ "ld1r { v0.4s }, [x20]\n"
+ "add v17.4s, v17.4s, v1.4s\n"
+ "add x19, %x[qp], %[per_layer_mul]\n"
+ "add v18.4s, v18.4s, v2.4s\n"
+ "ld1r { v4.4s }, [x19]\n"
+ "add x10, x10, #0x40\n"
+ "add v19.4s, v19.4s, v3.4s\n"
+ "sqrdmulh v16.4s, v16.4s, v4.4s\n"
+ "sqrdmulh v17.4s, v17.4s, v4.4s\n"
+ "sqrdmulh v18.4s, v18.4s, v4.4s\n"
+ "sqrdmulh v19.4s, v19.4s, v4.4s\n"
+ "tbz %x[flags], #5, 21f\n"
+ "and v4.16b, v16.16b, v0.16b\n"
+ "sshr v4.4s, v4.4s, #0x1f\n"
+ "and v5.16b, v17.16b, v0.16b\n"
+ "and v6.16b, v18.16b, v0.16b\n"
+ "sshr v5.4s, v5.4s, #0x1f\n"
+ "and v7.16b, v19.16b, v0.16b\n"
+ "sshr v6.4s, v6.4s, #0x1f\n"
+ "sqadd v16.4s, v16.4s, v4.4s\n"
+ "sshr v7.4s, v7.4s, #0x1f\n"
+ "sqadd v17.4s, v17.4s, v5.4s\n"
+ "sqadd v18.4s, v18.4s, v6.4s\n"
+ "sqadd v19.4s, v19.4s, v7.4s\n"
+ "21:" // Height 1: no shift correction
+ "srshl v16.4s, v16.4s, v0.4s\n"
+ "add x19, %x[qp], %[c_offset]\n"
+ "ld1r { v4.4s }, [x19]\n"
+ "srshl v17.4s, v17.4s, v0.4s\n"
+ "add x19, %x[qp], %[minval]\n"
+ "srshl v18.4s, v18.4s, v0.4s\n"
+ "ld1r { v5.4s }, [x19]\n"
+ "add x19, %x[qp], %[maxval]\n"
+ "srshl v19.4s, v19.4s, v0.4s\n"
+ "ld1r { v6.4s }, [x19]\n"
+ "cmp x12, #0x10\n"
+ "add v16.4s, v16.4s, v4.4s\n"
+ "add v17.4s, v17.4s, v4.4s\n"
+ "add v18.4s, v18.4s, v4.4s\n"
+ "add v19.4s, v19.4s, v4.4s\n"
+ "smin v16.4s, v16.4s, v6.4s\n"
+ "smin v17.4s, v17.4s, v6.4s\n"
+ "smin v18.4s, v18.4s, v6.4s\n"
+ "smax v16.4s, v16.4s, v5.4s\n"
+ "smax v17.4s, v17.4s, v5.4s\n"
+ "smax v18.4s, v18.4s, v5.4s\n"
+ "smin v19.4s, v19.4s, v6.4s\n"
+ "uzp1 v16.8h, v16.8h, v17.8h\n"
+ "smax v19.4s, v19.4s, v5.4s\n"
+ "uzp1 v17.8h, v18.8h, v19.8h\n"
+ "uzp1 v16.16b, v16.16b, v17.16b\n"
+ "bge 30f\n"
+ "tbz x12, #3, 25f\n"
+ "str d16, [x9], #0x8\n"
+ "tbz x12, #2, 23f\n"
+ "st1 { v16.s }[2], [x9], #0x4\n"
+ "tbz x12, #1, 22f\n"
+ "st1 { v16.h }[6], [x9], #0x2\n"
+ "tbz x12, #0, 29f\n"
+ "st1 { v16.b }[14], [x9]\n"
+ "b 29f\n"
+ "22:" // Height 1: Partial direct writeback: partial_1_12
+ "tbz x12, #0, 29f\n"
+ "st1 { v16.b }[12], [x9]\n"
+ "b 29f\n"
+ "23:" // Height 1: Partial direct writeback: partial_2_8
+ "tbz x12, #1, 24f\n"
+ "st1 { v16.h }[4], [x9], #0x2\n"
+ "tbz x12, #0, 29f\n"
+ "st1 { v16.b }[10], [x9]\n"
+ "b 29f\n"
+ "24:" // Height 1: Partial direct writeback: partial_1_8
+ "tbz x12, #0, 29f\n"
+ "st1 { v16.b }[8], [x9]\n"
+ "b 29f\n"
+ "25:" // Height 1: Partial direct writeback: partial_4_0
+ "tbz x12, #2, 27f\n"
+ "str s16, [x9], #0x4\n"
+ "tbz x12, #1, 26f\n"
+ "st1 { v16.h }[2], [x9], #0x2\n"
+ "tbz x12, #0, 29f\n"
+ "st1 { v16.b }[6], [x9]\n"
+ "b 29f\n"
+ "26:" // Height 1: Partial direct writeback: partial_1_4
+ "tbz x12, #0, 29f\n"
+ "st1 { v16.b }[4], [x9]\n"
+ "b 29f\n"
+ "27:" // Height 1: Partial direct writeback: partial_2_0
+ "tbz x12, #1, 28f\n"
+ "str h16, [x9], #0x2\n"
+ "tbz x12, #0, 29f\n"
+ "st1 { v16.b }[2], [x9]\n"
+ "b 29f\n"
+ "28:" // Height 1: Partial direct writeback: partial_1_0
+ "str b16, [x9, #0x0]\n"
+ "29:" // Height 1: Partial direct writeback: Done
+ "b 31f\n"
+ "30:" // Height 1: Full writeback
+ "str q16, [x9, #0x0]\n"
+ "add x9, x9, #0x10\n"
+ "31:" // Height 1: Writeback done
+ "subs x12, x12, #0x10\n"
+ "bgt 3b\n"
+ "b 126f\n"
+ "32:" // Height 2
+ "movi v11.4s, #0x0\n"
+ "ldr x12, [%x[args_ptr], %[offsetof_N]]\n"
+ "mov x10, %x[col_bias]\n"
+ "movi v12.4s, #0x0\n"
+ "ldr x11, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "bic %x[flags], %x[flags], #0x80000000\n"
+ "movi v13.4s, #0x0\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "movi v14.4s, #0x0\n"
+ "movi v15.16b, #0x1\n"
+ "tbz %x[flags], #2, 33f\n"
+ "ldr x9, [%x[output_ptr], #0x0]\n"
+ "ldr x25, [%x[output_ptr], #0x8]\n"
+ "add x9, x9, x19\n"
+ "add x25, x25, x19\n"
+ "b 34f\n"
+ "33:" // Height 2: setup direct output
+ "mov x9, %x[output_ptr]\n"
+ "add x25, x9, x19\n"
+ "34:" // Height 2: Column loop
+ "movi v16.4s, #0x0\n"
+ "movi v17.4s, #0x0\n"
+ "movi v18.4s, #0x0\n"
+ "movi v19.4s, #0x0\n"
+ "movi v20.4s, #0x0\n"
+ "movi v21.4s, #0x0\n"
+ "movi v22.4s, #0x0\n"
+ "movi v23.4s, #0x0\n"
+ "35:" // Height 2: setup done
+ "mov x28, #0x0\n"
+ "36:" // Height 2: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w27, [x20, x28, LSL #0x2]\n"
+ "tbz %x[flags], #3, 37f\n"
+ "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x26, [x20, #0x0]\n"
+ "ldr x24, [x20, #0x8]\n"
+ "cbnz x28, 38f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x26, x26, x19\n"
+ "add x24, x24, x19\n"
+ "b 38f\n"
+ "37:" // Height 2: setup direct input
+ "mov x26, %x[input_ptr]\n"
+ "add x24, x26, x19\n"
+ "38:" // Height 2: input setup done
+ "cmp x27, #0x10\n"
+ "blt 43f\n"
+ "cmp x27, #0x20\n"
+ "blt 41f\n"
+ "39:" // Height 2: Multiply loop: Main loop head
+ "ldr q0, [x26, #0x0]\n"
+ "ldr q1, [x24, #0x0]\n"
+ "ldr q4, [x11, #0x0]\n"
+ ".inst 0x4f80e090 // sdot v16.4s, v4.16b, v0.4b[0]\n"
+ "ldr q5, [x11, #0x10]\n"
+ ".inst 0x4f81e094 // sdot v20.4s, v4.16b, v1.4b[0]\n"
+ "ldr q6, [x11, #0x20]\n"
+ "ldr q7, [x11, #0x30]\n"
+ ".inst 0x4f80e0b1 // sdot v17.4s, v5.16b, v0.4b[0]\n"
+ "ldr q8, [x11, #0x40]\n"
+ ".inst 0x4f81e0b5 // sdot v21.4s, v5.16b, v1.4b[0]\n"
+ "ldr q9, [x11, #0x50]\n"
+ ".inst 0x4f80e0d2 // sdot v18.4s, v6.16b, v0.4b[0]\n"
+ "ldr q10, [x11, #0x60]\n"
+ ".inst 0x4f81e0d6 // sdot v22.4s, v6.16b, v1.4b[0]\n"
+ "ldr q4, [x11, #0x70]\n"
+ ".inst 0x4f80e0f3 // sdot v19.4s, v7.16b, v0.4b[0]\n"
+ "ldr q5, [x11, #0x80]\n"
+ ".inst 0x4f81e0f7 // sdot v23.4s, v7.16b, v1.4b[0]\n"
+ "ldr q6, [x11, #0x90]\n"
+ ".inst 0x4fa0e110 // sdot v16.4s, v8.16b, v0.4b[1]\n"
+ "ldr q7, [x11, #0xa0]\n"
+ ".inst 0x4fa1e114 // sdot v20.4s, v8.16b, v1.4b[1]\n"
+ "ldr q8, [x11, #0xb0]\n"
+ ".inst 0x4fa0e131 // sdot v17.4s, v9.16b, v0.4b[1]\n"
+ "add x26, x26, #0x10\n"
+ ".inst 0x4fa1e135 // sdot v21.4s, v9.16b, v1.4b[1]\n"
+ "ldr q9, [x11, #0xc0]\n"
+ ".inst 0x4fa0e152 // sdot v18.4s, v10.16b, v0.4b[1]\n"
+ "add x24, x24, #0x10\n"
+ ".inst 0x4fa1e156 // sdot v22.4s, v10.16b, v1.4b[1]\n"
+ "ldr q10, [x11, #0xd0]\n"
+ ".inst 0x4fa0e093 // sdot v19.4s, v4.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e097 // sdot v23.4s, v4.16b, v1.4b[1]\n"
+ "ldr q4, [x11, #0xe0]\n"
+ ".inst 0x4f80e8b0 // sdot v16.4s, v5.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8b4 // sdot v20.4s, v5.16b, v1.4b[2]\n"
+ "ldr q5, [x11, #0xf0]\n"
+ ".inst 0x4f80e8d1 // sdot v17.4s, v6.16b, v0.4b[2]\n"
+ "add x11, x11, #0x100\n"
+ ".inst 0x4f81e8d5 // sdot v21.4s, v6.16b, v1.4b[2]\n"
+ ".inst 0x4f80e8f2 // sdot v18.4s, v7.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8f6 // sdot v22.4s, v7.16b, v1.4b[2]\n"
+ ".inst 0x4f80e913 // sdot v19.4s, v8.16b, v0.4b[2]\n"
+ ".inst 0x4f81e917 // sdot v23.4s, v8.16b, v1.4b[2]\n"
+ ".inst 0x4fa0e930 // sdot v16.4s, v9.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e934 // sdot v20.4s, v9.16b, v1.4b[3]\n"
+ ".inst 0x4fa0e951 // sdot v17.4s, v10.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e955 // sdot v21.4s, v10.16b, v1.4b[3]\n"
+ ".inst 0x4fa0e892 // sdot v18.4s, v4.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e896 // sdot v22.4s, v4.16b, v1.4b[3]\n"
+ ".inst 0x4fa0e8b3 // sdot v19.4s, v5.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e8b7 // sdot v23.4s, v5.16b, v1.4b[3]\n"
+ "tbnz %x[flags], #31, 40f\n"
+ ".inst 0x4e8f940b // sdot v11.4s, v0.16b, v15.16b\n"
+ ".inst 0x4e8f942c // sdot v12.4s, v1.16b, v15.16b\n"
+ "40:" // Height 2: Multiply loop: unique 5: skip row sum
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "sub x27, x27, #0x10\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "cmp x27, #0x20\n"
+ "bge 39b\n"
+ "41:" // Height 2: Multiply loop: Single iteration only
+ "sub x27, x27, #0x10\n"
+ "ldr q0, [x26, #0x0]\n"
+ "ldr q1, [x24, #0x0]\n"
+ "ldr q6, [x11, #0x0]\n"
+ ".inst 0x4f80e0d0 // sdot v16.4s, v6.16b, v0.4b[0]\n"
+ "ldr q7, [x11, #0x10]\n"
+ ".inst 0x4f81e0d4 // sdot v20.4s, v6.16b, v1.4b[0]\n"
+ "ldr q8, [x11, #0x20]\n"
+ "ldr q9, [x11, #0x30]\n"
+ ".inst 0x4f80e0f1 // sdot v17.4s, v7.16b, v0.4b[0]\n"
+ "ldr q10, [x11, #0x40]\n"
+ ".inst 0x4f81e0f5 // sdot v21.4s, v7.16b, v1.4b[0]\n"
+ "ldr q4, [x11, #0x50]\n"
+ ".inst 0x4f80e112 // sdot v18.4s, v8.16b, v0.4b[0]\n"
+ "ldr q5, [x11, #0x60]\n"
+ ".inst 0x4f81e116 // sdot v22.4s, v8.16b, v1.4b[0]\n"
+ "ldr q6, [x11, #0x70]\n"
+ ".inst 0x4f80e133 // sdot v19.4s, v9.16b, v0.4b[0]\n"
+ "ldr q7, [x11, #0x80]\n"
+ ".inst 0x4f81e137 // sdot v23.4s, v9.16b, v1.4b[0]\n"
+ "ldr q8, [x11, #0x90]\n"
+ ".inst 0x4fa0e150 // sdot v16.4s, v10.16b, v0.4b[1]\n"
+ "ldr q9, [x11, #0xa0]\n"
+ ".inst 0x4fa1e154 // sdot v20.4s, v10.16b, v1.4b[1]\n"
+ "ldr q10, [x11, #0xb0]\n"
+ ".inst 0x4fa0e091 // sdot v17.4s, v4.16b, v0.4b[1]\n"
+ "add x26, x26, #0x10\n"
+ ".inst 0x4fa1e095 // sdot v21.4s, v4.16b, v1.4b[1]\n"
+ "ldr q4, [x11, #0xc0]\n"
+ ".inst 0x4fa0e0b2 // sdot v18.4s, v5.16b, v0.4b[1]\n"
+ "add x24, x24, #0x10\n"
+ ".inst 0x4fa1e0b6 // sdot v22.4s, v5.16b, v1.4b[1]\n"
+ "ldr q5, [x11, #0xd0]\n"
+ ".inst 0x4fa0e0d3 // sdot v19.4s, v6.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e0d7 // sdot v23.4s, v6.16b, v1.4b[1]\n"
+ "ldr q6, [x11, #0xe0]\n"
+ ".inst 0x4f80e8f0 // sdot v16.4s, v7.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8f4 // sdot v20.4s, v7.16b, v1.4b[2]\n"
+ "ldr q7, [x11, #0xf0]\n"
+ ".inst 0x4f80e911 // sdot v17.4s, v8.16b, v0.4b[2]\n"
+ "add x11, x11, #0x100\n"
+ ".inst 0x4f81e915 // sdot v21.4s, v8.16b, v1.4b[2]\n"
+ ".inst 0x4f80e932 // sdot v18.4s, v9.16b, v0.4b[2]\n"
+ ".inst 0x4f81e936 // sdot v22.4s, v9.16b, v1.4b[2]\n"
+ ".inst 0x4f80e953 // sdot v19.4s, v10.16b, v0.4b[2]\n"
+ ".inst 0x4f81e957 // sdot v23.4s, v10.16b, v1.4b[2]\n"
+ ".inst 0x4fa0e890 // sdot v16.4s, v4.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e894 // sdot v20.4s, v4.16b, v1.4b[3]\n"
+ ".inst 0x4fa0e8b1 // sdot v17.4s, v5.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e8b5 // sdot v21.4s, v5.16b, v1.4b[3]\n"
+ ".inst 0x4fa0e8d2 // sdot v18.4s, v6.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e8d6 // sdot v22.4s, v6.16b, v1.4b[3]\n"
+ ".inst 0x4fa0e8f3 // sdot v19.4s, v7.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e8f7 // sdot v23.4s, v7.16b, v1.4b[3]\n"
+ "tbnz %x[flags], #31, 42f\n"
+ ".inst 0x4e8f940b // sdot v11.4s, v0.16b, v15.16b\n"
+ ".inst 0x4e8f942c // sdot v12.4s, v1.16b, v15.16b\n"
+ "42:" // Height 2: Multiply loop: unique 6: skip row sum
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "43:" // Height 2: Multiply loop: Main loop skip
+ "cbz x27, 50f\n"
+ "cmp x27, #0x4\n"
+ "blt 46f\n"
+ "44:" // Height 2: Multiply loop: Odd block loop
+ "ldr s0, [x26], #0x4\n"
+ "ldr s1, [x24], #0x4\n"
+ "tbnz %x[flags], #31, 45f\n"
+ ".inst 0x4e8f940b // sdot v11.4s, v0.16b, v15.16b\n"
+ ".inst 0x4e8f942c // sdot v12.4s, v1.16b, v15.16b\n"
+ "45:" // Height 2: Multiply loop: unique 7: skip row sum
+ "ldr q8, [x11, #0x0]\n"
+ ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
+ "ldr q9, [x11, #0x10]\n"
+ ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
+ "ldr q10, [x11, #0x20]\n"
+ "ldr q4, [x11, #0x30]\n"
+ ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
+ "sub x27, x27, #0x4\n"
+ ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
+ "cmp x27, #0x4\n"
+ ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
+ "add x11, x11, #0x40\n"
+ ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
+ ".inst 0x4f80e093 // sdot v19.4s, v4.16b, v0.4b[0]\n"
+ ".inst 0x4f81e097 // sdot v23.4s, v4.16b, v1.4b[0]\n"
+ "bge 44b\n"
+ "cbz x27, 50f\n"
+ "46:" // Height 2: Multiply loop: Skip odd blocks
+ "tbz x27, #1, 47f\n"
+ "ldr h0, [x26], #0x2\n"
+ "ldr h1, [x24], #0x2\n"
+ "tbz x27, #0, 48f\n"
+ "ld1 { v0.b }[2], [x26]\n"
+ "ld1 { v1.b }[2], [x24]\n"
+ "b 48f\n"
+ "47:" // Height 2: Multiply loop: Ragged operand read: partial_1_0
+ "ldr b0, [x26, #0x0]\n"
+ "ldr b1, [x24, #0x0]\n"
+ "48:" // Height 2: Multiply loop: Ragged operand read: Done
+ "tbnz %x[flags], #31, 49f\n"
+ ".inst 0x4e8f940b // sdot v11.4s, v0.16b, v15.16b\n"
+ ".inst 0x4e8f942c // sdot v12.4s, v1.16b, v15.16b\n"
+ "49:" // Height 2: Multiply loop: unique 8: skip row sum
+ "ldr q5, [x11, #0x0]\n"
+ ".inst 0x4f80e0b0 // sdot v16.4s, v5.16b, v0.4b[0]\n"
+ "ldr q6, [x11, #0x10]\n"
+ ".inst 0x4f81e0b4 // sdot v20.4s, v5.16b, v1.4b[0]\n"
+ "ldr q7, [x11, #0x20]\n"
+ "ldr q8, [x11, #0x30]\n"
+ ".inst 0x4f80e0d1 // sdot v17.4s, v6.16b, v0.4b[0]\n"
+ "add x11, x11, #0x40\n"
+ ".inst 0x4f81e0d5 // sdot v21.4s, v6.16b, v1.4b[0]\n"
+ ".inst 0x4f80e0f2 // sdot v18.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x4f81e0f6 // sdot v22.4s, v7.16b, v1.4b[0]\n"
+ ".inst 0x4f80e113 // sdot v19.4s, v8.16b, v0.4b[0]\n"
+ ".inst 0x4f81e117 // sdot v23.4s, v8.16b, v1.4b[0]\n"
+ "50:" // Height 2: Multiply loop: No odd multiplies
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "add x28, x28, #0x1\n"
+ "cmp x28, x19\n"
+ "bne 36b\n"
+ "prfm pstl1keep, [x9, #0x0]\n"
+ "prfm pstl1keep, [x25, #0x0]\n"
+ "tbnz %x[flags], #31, 51f\n"
+ "addp v11.4s, v11.4s, v11.4s\n"
+ "add x19, %x[qp], %[b_offset]\n"
+ "ld1r { v2.4s }, [x19]\n"
+ "addp v12.4s, v12.4s, v12.4s\n"
+ "addp v11.4s, v11.4s, v11.4s\n"
+ "addp v12.4s, v12.4s, v12.4s\n"
+ "neg v2.4s, v2.4s\n"
+ "mul v11.4s, v11.4s, v2.4s\n"
+ "mul v12.4s, v12.4s, v2.4s\n"
+ "51:" // Height 2: skip row sum fixup
+ "add v16.4s, v16.4s, v11.4s\n"
+ "orr %x[flags], %x[flags], #0x80000000\n"
+ "add v17.4s, v17.4s, v11.4s\n"
+ "ldr q0, [x10, #0x0]\n"
+ "add v18.4s, v18.4s, v11.4s\n"
+ "ldr q1, [x10, #0x10]\n"
+ "add v19.4s, v19.4s, v11.4s\n"
+ "ldr q2, [x10, #0x20]\n"
+ "add v20.4s, v20.4s, v12.4s\n"
+ "ldr q3, [x10, #0x30]\n"
+ "add v21.4s, v21.4s, v12.4s\n"
+ "add x20, %x[qp], %[per_layer_right_shift]\n"
+ "add v22.4s, v22.4s, v12.4s\n"
+ "add x19, %x[qp], %[per_layer_mul]\n"
+ "ld1r { v4.4s }, [x19]\n"
+ "add v23.4s, v23.4s, v12.4s\n"
+ "add x10, x10, #0x40\n"
+ "add v16.4s, v16.4s, v0.4s\n"
+ "add v17.4s, v17.4s, v1.4s\n"
+ "add v18.4s, v18.4s, v2.4s\n"
+ "add v19.4s, v19.4s, v3.4s\n"
+ "add v20.4s, v20.4s, v0.4s\n"
+ "ld1r { v0.4s }, [x20]\n"
+ "add v21.4s, v21.4s, v1.4s\n"
+ "add v22.4s, v22.4s, v2.4s\n"
+ "add v23.4s, v23.4s, v3.4s\n"
+ "sqrdmulh v16.4s, v16.4s, v4.4s\n"
+ "sqrdmulh v17.4s, v17.4s, v4.4s\n"
+ "sqrdmulh v18.4s, v18.4s, v4.4s\n"
+ "sqrdmulh v19.4s, v19.4s, v4.4s\n"
+ "sqrdmulh v20.4s, v20.4s, v4.4s\n"
+ "sqrdmulh v21.4s, v21.4s, v4.4s\n"
+ "sqrdmulh v22.4s, v22.4s, v4.4s\n"
+ "sqrdmulh v23.4s, v23.4s, v4.4s\n"
+ "tbz %x[flags], #5, 52f\n"
+ "and v4.16b, v16.16b, v0.16b\n"
+ "sshr v4.4s, v4.4s, #0x1f\n"
+ "and v5.16b, v17.16b, v0.16b\n"
+ "and v6.16b, v18.16b, v0.16b\n"
+ "sshr v5.4s, v5.4s, #0x1f\n"
+ "and v7.16b, v19.16b, v0.16b\n"
+ "and v8.16b, v20.16b, v0.16b\n"
+ "sshr v6.4s, v6.4s, #0x1f\n"
+ "and v9.16b, v21.16b, v0.16b\n"
+ "sshr v7.4s, v7.4s, #0x1f\n"
+ "sqadd v16.4s, v16.4s, v4.4s\n"
+ "and v10.16b, v22.16b, v0.16b\n"
+ "sshr v8.4s, v8.4s, #0x1f\n"
+ "and v4.16b, v23.16b, v0.16b\n"
+ "sshr v9.4s, v9.4s, #0x1f\n"
+ "sqadd v17.4s, v17.4s, v5.4s\n"
+ "sshr v10.4s, v10.4s, #0x1f\n"
+ "sqadd v18.4s, v18.4s, v6.4s\n"
+ "sshr v4.4s, v4.4s, #0x1f\n"
+ "sqadd v19.4s, v19.4s, v7.4s\n"
+ "sqadd v20.4s, v20.4s, v8.4s\n"
+ "sqadd v21.4s, v21.4s, v9.4s\n"
+ "sqadd v22.4s, v22.4s, v10.4s\n"
+ "sqadd v23.4s, v23.4s, v4.4s\n"
+ "52:" // Height 2: no shift correction
+ "srshl v16.4s, v16.4s, v0.4s\n"
+ "add x19, %x[qp], %[c_offset]\n"
+ "ld1r { v4.4s }, [x19]\n"
+ "srshl v17.4s, v17.4s, v0.4s\n"
+ "add x19, %x[qp], %[minval]\n"
+ "srshl v18.4s, v18.4s, v0.4s\n"
+ "ld1r { v5.4s }, [x19]\n"
+ "add x19, %x[qp], %[maxval]\n"
+ "srshl v19.4s, v19.4s, v0.4s\n"
+ "ld1r { v6.4s }, [x19]\n"
+ "cmp x12, #0x10\n"
+ "srshl v20.4s, v20.4s, v0.4s\n"
+ "srshl v21.4s, v21.4s, v0.4s\n"
+ "srshl v22.4s, v22.4s, v0.4s\n"
+ "srshl v23.4s, v23.4s, v0.4s\n"
+ "add v16.4s, v16.4s, v4.4s\n"
+ "add v17.4s, v17.4s, v4.4s\n"
+ "add v18.4s, v18.4s, v4.4s\n"
+ "smin v16.4s, v16.4s, v6.4s\n"
+ "smin v17.4s, v17.4s, v6.4s\n"
+ "smin v18.4s, v18.4s, v6.4s\n"
+ "smax v16.4s, v16.4s, v5.4s\n"
+ "smax v17.4s, v17.4s, v5.4s\n"
+ "smax v18.4s, v18.4s, v5.4s\n"
+ "add v19.4s, v19.4s, v4.4s\n"
+ "add v20.4s, v20.4s, v4.4s\n"
+ "add v21.4s, v21.4s, v4.4s\n"
+ "smin v19.4s, v19.4s, v6.4s\n"
+ "smin v20.4s, v20.4s, v6.4s\n"
+ "smin v21.4s, v21.4s, v6.4s\n"
+ "smax v19.4s, v19.4s, v5.4s\n"
+ "smax v20.4s, v20.4s, v5.4s\n"
+ "smax v21.4s, v21.4s, v5.4s\n"
+ "add v22.4s, v22.4s, v4.4s\n"
+ "add v23.4s, v23.4s, v4.4s\n"
+ "uzp1 v16.8h, v16.8h, v17.8h\n"
+ "smin v22.4s, v22.4s, v6.4s\n"
+ "smin v23.4s, v23.4s, v6.4s\n"
+ "uzp1 v17.8h, v18.8h, v19.8h\n"
+ "smax v22.4s, v22.4s, v5.4s\n"
+ "smax v23.4s, v23.4s, v5.4s\n"
+ "uzp1 v20.8h, v20.8h, v21.8h\n"
+ "uzp1 v16.16b, v16.16b, v17.16b\n"
+ "uzp1 v21.8h, v22.8h, v23.8h\n"
+ "uzp1 v20.16b, v20.16b, v21.16b\n"
+ "bge 61f\n"
+ "tbz x12, #3, 56f\n"
+ "str d16, [x9], #0x8\n"
+ "str d20, [x25], #0x8\n"
+ "tbz x12, #2, 54f\n"
+ "st1 { v16.s }[2], [x9], #0x4\n"
+ "st1 { v20.s }[2], [x25], #0x4\n"
+ "tbz x12, #1, 53f\n"
+ "st1 { v16.h }[6], [x9], #0x2\n"
+ "st1 { v20.h }[6], [x25], #0x2\n"
+ "tbz x12, #0, 60f\n"
+ "st1 { v16.b }[14], [x9]\n"
+ "st1 { v20.b }[14], [x25]\n"
+ "b 60f\n"
+ "53:" // Height 2: Partial direct writeback: partial_1_12
+ "tbz x12, #0, 60f\n"
+ "st1 { v16.b }[12], [x9]\n"
+ "st1 { v20.b }[12], [x25]\n"
+ "b 60f\n"
+ "54:" // Height 2: Partial direct writeback: partial_2_8
+ "tbz x12, #1, 55f\n"
+ "st1 { v16.h }[4], [x9], #0x2\n"
+ "st1 { v20.h }[4], [x25], #0x2\n"
+ "tbz x12, #0, 60f\n"
+ "st1 { v16.b }[10], [x9]\n"
+ "st1 { v20.b }[10], [x25]\n"
+ "b 60f\n"
+ "55:" // Height 2: Partial direct writeback: partial_1_8
+ "tbz x12, #0, 60f\n"
+ "st1 { v16.b }[8], [x9]\n"
+ "st1 { v20.b }[8], [x25]\n"
+ "b 60f\n"
+ "56:" // Height 2: Partial direct writeback: partial_4_0
+ "tbz x12, #2, 58f\n"
+ "str s16, [x9], #0x4\n"
+ "str s20, [x25], #0x4\n"
+ "tbz x12, #1, 57f\n"
+ "st1 { v16.h }[2], [x9], #0x2\n"
+ "st1 { v20.h }[2], [x25], #0x2\n"
+ "tbz x12, #0, 60f\n"
+ "st1 { v16.b }[6], [x9]\n"
+ "st1 { v20.b }[6], [x25]\n"
+ "b 60f\n"
+ "57:" // Height 2: Partial direct writeback: partial_1_4
+ "tbz x12, #0, 60f\n"
+ "st1 { v16.b }[4], [x9]\n"
+ "st1 { v20.b }[4], [x25]\n"
+ "b 60f\n"
+ "58:" // Height 2: Partial direct writeback: partial_2_0
+ "tbz x12, #1, 59f\n"
+ "str h16, [x9], #0x2\n"
+ "str h20, [x25], #0x2\n"
+ "tbz x12, #0, 60f\n"
+ "st1 { v16.b }[2], [x9]\n"
+ "st1 { v20.b }[2], [x25]\n"
+ "b 60f\n"
+ "59:" // Height 2: Partial direct writeback: partial_1_0
+ "str b16, [x9, #0x0]\n"
+ "str b20, [x25, #0x0]\n"
+ "60:" // Height 2: Partial direct writeback: Done
+ "b 62f\n"
+ "61:" // Height 2: Full writeback
+ "str q16, [x9, #0x0]\n"
+ "str q20, [x25, #0x0]\n"
+ "add x9, x9, #0x10\n"
+ "add x25, x25, #0x10\n"
+ "62:" // Height 2: Writeback done
+ "subs x12, x12, #0x10\n"
+ "bgt 34b\n"
+ "b 126f\n"
+ "63:" // Height 3
+ "movi v11.4s, #0x0\n"
+ "ldr x12, [%x[args_ptr], %[offsetof_N]]\n"
+ "mov x10, %x[col_bias]\n"
+ "movi v12.4s, #0x0\n"
+ "ldr x11, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "bic %x[flags], %x[flags], #0x80000000\n"
+ "movi v13.4s, #0x0\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "movi v14.4s, #0x0\n"
+ "movi v15.16b, #0x1\n"
+ "tbz %x[flags], #2, 64f\n"
+ "ldr x9, [%x[output_ptr], #0x0]\n"
+ "ldr x25, [%x[output_ptr], #0x8]\n"
+ "add x9, x9, x19\n"
+ "ldr x23, [%x[output_ptr], #0x10]\n"
+ "add x25, x25, x19\n"
+ "add x23, x23, x19\n"
+ "b 65f\n"
+ "64:" // Height 3: setup direct output
+ "mov x9, %x[output_ptr]\n"
+ "add x25, x9, x19\n"
+ "add x23, x25, x19\n"
+ "65:" // Height 3: Column loop
+ "movi v16.4s, #0x0\n"
+ "movi v17.4s, #0x0\n"
+ "movi v18.4s, #0x0\n"
+ "movi v19.4s, #0x0\n"
+ "movi v20.4s, #0x0\n"
+ "movi v21.4s, #0x0\n"
+ "movi v22.4s, #0x0\n"
+ "movi v23.4s, #0x0\n"
+ "movi v24.4s, #0x0\n"
+ "movi v25.4s, #0x0\n"
+ "movi v26.4s, #0x0\n"
+ "movi v27.4s, #0x0\n"
+ "66:" // Height 3: setup done
+ "mov x28, #0x0\n"
+ "67:" // Height 3: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w27, [x20, x28, LSL #0x2]\n"
+ "tbz %x[flags], #3, 68f\n"
+ "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x26, [x20, #0x0]\n"
+ "ldr x24, [x20, #0x8]\n"
+ "ldr x22, [x20, #0x10]\n"
+ "cbnz x28, 69f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x26, x26, x19\n"
+ "add x24, x24, x19\n"
+ "add x22, x22, x19\n"
+ "b 69f\n"
+ "68:" // Height 3: setup direct input
+ "mov x26, %x[input_ptr]\n"
+ "add x24, x26, x19\n"
+ "add x22, x24, x19\n"
+ "69:" // Height 3: input setup done
+ "cmp x27, #0x10\n"
+ "blt 74f\n"
+ "cmp x27, #0x20\n"
+ "blt 72f\n"
+ "70:" // Height 3: Multiply loop: Main loop head
+ "ldr q0, [x26, #0x0]\n"
+ "ldr q1, [x24, #0x0]\n"
+ "ldr q2, [x22, #0x0]\n"
+ "ldr q4, [x11, #0x0]\n"
+ ".inst 0x4f80e090 // sdot v16.4s, v4.16b, v0.4b[0]\n"
+ "ldr q5, [x11, #0x10]\n"
+ ".inst 0x4f81e094 // sdot v20.4s, v4.16b, v1.4b[0]\n"
+ "ldr q6, [x11, #0x20]\n"
+ ".inst 0x4f82e098 // sdot v24.4s, v4.16b, v2.4b[0]\n"
+ "ldr q7, [x11, #0x30]\n"
+ "ldr q8, [x11, #0x40]\n"
+ ".inst 0x4f80e0b1 // sdot v17.4s, v5.16b, v0.4b[0]\n"
+ "ldr q9, [x11, #0x50]\n"
+ ".inst 0x4f81e0b5 // sdot v21.4s, v5.16b, v1.4b[0]\n"
+ "ldr q10, [x11, #0x60]\n"
+ ".inst 0x4f82e0b9 // sdot v25.4s, v5.16b, v2.4b[0]\n"
+ "ldr q4, [x11, #0x70]\n"
+ ".inst 0x4f80e0d2 // sdot v18.4s, v6.16b, v0.4b[0]\n"
+ "ldr q5, [x11, #0x80]\n"
+ ".inst 0x4f81e0d6 // sdot v22.4s, v6.16b, v1.4b[0]\n"
+ "add x26, x26, #0x10\n"
+ ".inst 0x4f82e0da // sdot v26.4s, v6.16b, v2.4b[0]\n"
+ "ldr q6, [x11, #0x90]\n"
+ ".inst 0x4f80e0f3 // sdot v19.4s, v7.16b, v0.4b[0]\n"
+ "add x24, x24, #0x10\n"
+ ".inst 0x4f81e0f7 // sdot v23.4s, v7.16b, v1.4b[0]\n"
+ "add x22, x22, #0x10\n"
+ ".inst 0x4f82e0fb // sdot v27.4s, v7.16b, v2.4b[0]\n"
+ "ldr q7, [x11, #0xa0]\n"
+ ".inst 0x4fa0e110 // sdot v16.4s, v8.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e114 // sdot v20.4s, v8.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e118 // sdot v24.4s, v8.16b, v2.4b[1]\n"
+ "ldr q8, [x11, #0xb0]\n"
+ ".inst 0x4fa0e131 // sdot v17.4s, v9.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e135 // sdot v21.4s, v9.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e139 // sdot v25.4s, v9.16b, v2.4b[1]\n"
+ "ldr q9, [x11, #0xc0]\n"
+ ".inst 0x4fa0e152 // sdot v18.4s, v10.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e156 // sdot v22.4s, v10.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e15a // sdot v26.4s, v10.16b, v2.4b[1]\n"
+ "ldr q10, [x11, #0xd0]\n"
+ ".inst 0x4fa0e093 // sdot v19.4s, v4.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e097 // sdot v23.4s, v4.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e09b // sdot v27.4s, v4.16b, v2.4b[1]\n"
+ "ldr q4, [x11, #0xe0]\n"
+ ".inst 0x4f80e8b0 // sdot v16.4s, v5.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8b4 // sdot v20.4s, v5.16b, v1.4b[2]\n"
+ ".inst 0x4f82e8b8 // sdot v24.4s, v5.16b, v2.4b[2]\n"
+ "ldr q5, [x11, #0xf0]\n"
+ ".inst 0x4f80e8d1 // sdot v17.4s, v6.16b, v0.4b[2]\n"
+ "add x11, x11, #0x100\n"
+ ".inst 0x4f81e8d5 // sdot v21.4s, v6.16b, v1.4b[2]\n"
+ ".inst 0x4f82e8d9 // sdot v25.4s, v6.16b, v2.4b[2]\n"
+ ".inst 0x4f80e8f2 // sdot v18.4s, v7.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8f6 // sdot v22.4s, v7.16b, v1.4b[2]\n"
+ ".inst 0x4f82e8fa // sdot v26.4s, v7.16b, v2.4b[2]\n"
+ ".inst 0x4f80e913 // sdot v19.4s, v8.16b, v0.4b[2]\n"
+ ".inst 0x4f81e917 // sdot v23.4s, v8.16b, v1.4b[2]\n"
+ ".inst 0x4f82e91b // sdot v27.4s, v8.16b, v2.4b[2]\n"
+ ".inst 0x4fa0e930 // sdot v16.4s, v9.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e934 // sdot v20.4s, v9.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e938 // sdot v24.4s, v9.16b, v2.4b[3]\n"
+ ".inst 0x4fa0e951 // sdot v17.4s, v10.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e955 // sdot v21.4s, v10.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e959 // sdot v25.4s, v10.16b, v2.4b[3]\n"
+ ".inst 0x4fa0e892 // sdot v18.4s, v4.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e896 // sdot v22.4s, v4.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e89a // sdot v26.4s, v4.16b, v2.4b[3]\n"
+ ".inst 0x4fa0e8b3 // sdot v19.4s, v5.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e8b7 // sdot v23.4s, v5.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e8bb // sdot v27.4s, v5.16b, v2.4b[3]\n"
+ "tbnz %x[flags], #31, 71f\n"
+ ".inst 0x4e8f940b // sdot v11.4s, v0.16b, v15.16b\n"
+ ".inst 0x4e8f942c // sdot v12.4s, v1.16b, v15.16b\n"
+ ".inst 0x4e8f944d // sdot v13.4s, v2.16b, v15.16b\n"
+ "71:" // Height 3: Multiply loop: unique 9: skip row sum
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "sub x27, x27, #0x10\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "cmp x27, #0x20\n"
+ "prfm pldl1keep, [x22, #0x80]\n"
+ "bge 70b\n"
+ "72:" // Height 3: Multiply loop: Single iteration only
+ "sub x27, x27, #0x10\n"
+ "ldr q0, [x26, #0x0]\n"
+ "ldr q1, [x24, #0x0]\n"
+ "ldr q2, [x22, #0x0]\n"
+ "ldr q6, [x11, #0x0]\n"
+ ".inst 0x4f80e0d0 // sdot v16.4s, v6.16b, v0.4b[0]\n"
+ "ldr q7, [x11, #0x10]\n"
+ ".inst 0x4f81e0d4 // sdot v20.4s, v6.16b, v1.4b[0]\n"
+ "ldr q8, [x11, #0x20]\n"
+ ".inst 0x4f82e0d8 // sdot v24.4s, v6.16b, v2.4b[0]\n"
+ "ldr q9, [x11, #0x30]\n"
+ "ldr q10, [x11, #0x40]\n"
+ ".inst 0x4f80e0f1 // sdot v17.4s, v7.16b, v0.4b[0]\n"
+ "ldr q4, [x11, #0x50]\n"
+ ".inst 0x4f81e0f5 // sdot v21.4s, v7.16b, v1.4b[0]\n"
+ "ldr q5, [x11, #0x60]\n"
+ ".inst 0x4f82e0f9 // sdot v25.4s, v7.16b, v2.4b[0]\n"
+ "ldr q6, [x11, #0x70]\n"
+ ".inst 0x4f80e112 // sdot v18.4s, v8.16b, v0.4b[0]\n"
+ "ldr q7, [x11, #0x80]\n"
+ ".inst 0x4f81e116 // sdot v22.4s, v8.16b, v1.4b[0]\n"
+ "add x26, x26, #0x10\n"
+ ".inst 0x4f82e11a // sdot v26.4s, v8.16b, v2.4b[0]\n"
+ "ldr q8, [x11, #0x90]\n"
+ ".inst 0x4f80e133 // sdot v19.4s, v9.16b, v0.4b[0]\n"
+ "add x24, x24, #0x10\n"
+ ".inst 0x4f81e137 // sdot v23.4s, v9.16b, v1.4b[0]\n"
+ "add x22, x22, #0x10\n"
+ ".inst 0x4f82e13b // sdot v27.4s, v9.16b, v2.4b[0]\n"
+ "ldr q9, [x11, #0xa0]\n"
+ ".inst 0x4fa0e150 // sdot v16.4s, v10.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e154 // sdot v20.4s, v10.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e158 // sdot v24.4s, v10.16b, v2.4b[1]\n"
+ "ldr q10, [x11, #0xb0]\n"
+ ".inst 0x4fa0e091 // sdot v17.4s, v4.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e095 // sdot v21.4s, v4.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e099 // sdot v25.4s, v4.16b, v2.4b[1]\n"
+ "ldr q4, [x11, #0xc0]\n"
+ ".inst 0x4fa0e0b2 // sdot v18.4s, v5.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e0b6 // sdot v22.4s, v5.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e0ba // sdot v26.4s, v5.16b, v2.4b[1]\n"
+ "ldr q5, [x11, #0xd0]\n"
+ ".inst 0x4fa0e0d3 // sdot v19.4s, v6.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e0d7 // sdot v23.4s, v6.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e0db // sdot v27.4s, v6.16b, v2.4b[1]\n"
+ "ldr q6, [x11, #0xe0]\n"
+ ".inst 0x4f80e8f0 // sdot v16.4s, v7.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8f4 // sdot v20.4s, v7.16b, v1.4b[2]\n"
+ ".inst 0x4f82e8f8 // sdot v24.4s, v7.16b, v2.4b[2]\n"
+ "ldr q7, [x11, #0xf0]\n"
+ ".inst 0x4f80e911 // sdot v17.4s, v8.16b, v0.4b[2]\n"
+ "add x11, x11, #0x100\n"
+ ".inst 0x4f81e915 // sdot v21.4s, v8.16b, v1.4b[2]\n"
+ ".inst 0x4f82e919 // sdot v25.4s, v8.16b, v2.4b[2]\n"
+ ".inst 0x4f80e932 // sdot v18.4s, v9.16b, v0.4b[2]\n"
+ ".inst 0x4f81e936 // sdot v22.4s, v9.16b, v1.4b[2]\n"
+ ".inst 0x4f82e93a // sdot v26.4s, v9.16b, v2.4b[2]\n"
+ ".inst 0x4f80e953 // sdot v19.4s, v10.16b, v0.4b[2]\n"
+ ".inst 0x4f81e957 // sdot v23.4s, v10.16b, v1.4b[2]\n"
+ ".inst 0x4f82e95b // sdot v27.4s, v10.16b, v2.4b[2]\n"
+ ".inst 0x4fa0e890 // sdot v16.4s, v4.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e894 // sdot v20.4s, v4.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e898 // sdot v24.4s, v4.16b, v2.4b[3]\n"
+ ".inst 0x4fa0e8b1 // sdot v17.4s, v5.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e8b5 // sdot v21.4s, v5.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e8b9 // sdot v25.4s, v5.16b, v2.4b[3]\n"
+ ".inst 0x4fa0e8d2 // sdot v18.4s, v6.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e8d6 // sdot v22.4s, v6.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e8da // sdot v26.4s, v6.16b, v2.4b[3]\n"
+ ".inst 0x4fa0e8f3 // sdot v19.4s, v7.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e8f7 // sdot v23.4s, v7.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e8fb // sdot v27.4s, v7.16b, v2.4b[3]\n"
+ "tbnz %x[flags], #31, 73f\n"
+ ".inst 0x4e8f940b // sdot v11.4s, v0.16b, v15.16b\n"
+ ".inst 0x4e8f942c // sdot v12.4s, v1.16b, v15.16b\n"
+ ".inst 0x4e8f944d // sdot v13.4s, v2.16b, v15.16b\n"
+ "73:" // Height 3: Multiply loop: unique 10: skip row sum
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "prfm pldl1keep, [x22, #0x80]\n"
+ "74:" // Height 3: Multiply loop: Main loop skip
+ "cbz x27, 81f\n"
+ "cmp x27, #0x4\n"
+ "blt 77f\n"
+ "75:" // Height 3: Multiply loop: Odd block loop
+ "ldr s0, [x26], #0x4\n"
+ "ldr s1, [x24], #0x4\n"
+ "ldr s2, [x22], #0x4\n"
+ "tbnz %x[flags], #31, 76f\n"
+ ".inst 0x4e8f940b // sdot v11.4s, v0.16b, v15.16b\n"
+ ".inst 0x4e8f942c // sdot v12.4s, v1.16b, v15.16b\n"
+ ".inst 0x4e8f944d // sdot v13.4s, v2.16b, v15.16b\n"
+ "76:" // Height 3: Multiply loop: unique 11: skip row sum
+ "ldr q8, [x11, #0x0]\n"
+ ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
+ "ldr q9, [x11, #0x10]\n"
+ ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
+ "ldr q10, [x11, #0x20]\n"
+ ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n"
+ "ldr q4, [x11, #0x30]\n"
+ "sub x27, x27, #0x4\n"
+ ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
+ "cmp x27, #0x4\n"
+ ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
+ "add x11, x11, #0x40\n"
+ ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n"
+ ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
+ ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
+ ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n"
+ ".inst 0x4f80e093 // sdot v19.4s, v4.16b, v0.4b[0]\n"
+ ".inst 0x4f81e097 // sdot v23.4s, v4.16b, v1.4b[0]\n"
+ ".inst 0x4f82e09b // sdot v27.4s, v4.16b, v2.4b[0]\n"
+ "bge 75b\n"
+ "cbz x27, 81f\n"
+ "77:" // Height 3: Multiply loop: Skip odd blocks
+ "tbz x27, #1, 78f\n"
+ "ldr h0, [x26], #0x2\n"
+ "ldr h1, [x24], #0x2\n"
+ "ldr h2, [x22], #0x2\n"
+ "tbz x27, #0, 79f\n"
+ "ld1 { v0.b }[2], [x26]\n"
+ "ld1 { v1.b }[2], [x24]\n"
+ "ld1 { v2.b }[2], [x22]\n"
+ "b 79f\n"
+ "78:" // Height 3: Multiply loop: Ragged operand read: partial_1_0
+ "ldr b0, [x26, #0x0]\n"
+ "ldr b1, [x24, #0x0]\n"
+ "ldr b2, [x22, #0x0]\n"
+ "79:" // Height 3: Multiply loop: Ragged operand read: Done
+ "tbnz %x[flags], #31, 80f\n"
+ ".inst 0x4e8f940b // sdot v11.4s, v0.16b, v15.16b\n"
+ ".inst 0x4e8f942c // sdot v12.4s, v1.16b, v15.16b\n"
+ ".inst 0x4e8f944d // sdot v13.4s, v2.16b, v15.16b\n"
+ "80:" // Height 3: Multiply loop: unique 12: skip row sum
+ "ldr q5, [x11, #0x0]\n"
+ ".inst 0x4f80e0b0 // sdot v16.4s, v5.16b, v0.4b[0]\n"
+ "ldr q6, [x11, #0x10]\n"
+ ".inst 0x4f81e0b4 // sdot v20.4s, v5.16b, v1.4b[0]\n"
+ "ldr q7, [x11, #0x20]\n"
+ ".inst 0x4f82e0b8 // sdot v24.4s, v5.16b, v2.4b[0]\n"
+ "ldr q8, [x11, #0x30]\n"
+ "add x11, x11, #0x40\n"
+ ".inst 0x4f80e0d1 // sdot v17.4s, v6.16b, v0.4b[0]\n"
+ ".inst 0x4f81e0d5 // sdot v21.4s, v6.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0d9 // sdot v25.4s, v6.16b, v2.4b[0]\n"
+ ".inst 0x4f80e0f2 // sdot v18.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x4f81e0f6 // sdot v22.4s, v7.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0fa // sdot v26.4s, v7.16b, v2.4b[0]\n"
+ ".inst 0x4f80e113 // sdot v19.4s, v8.16b, v0.4b[0]\n"
+ ".inst 0x4f81e117 // sdot v23.4s, v8.16b, v1.4b[0]\n"
+ ".inst 0x4f82e11b // sdot v27.4s, v8.16b, v2.4b[0]\n"
+ "81:" // Height 3: Multiply loop: No odd multiplies
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "add x28, x28, #0x1\n"
+ "cmp x28, x19\n"
+ "bne 67b\n"
+ "prfm pstl1keep, [x9, #0x0]\n"
+ "prfm pstl1keep, [x25, #0x0]\n"
+ "prfm pstl1keep, [x23, #0x0]\n"
+ "tbnz %x[flags], #31, 82f\n"
+ "addp v11.4s, v11.4s, v11.4s\n"
+ "add x19, %x[qp], %[b_offset]\n"
+ "ld1r { v3.4s }, [x19]\n"
+ "addp v12.4s, v12.4s, v12.4s\n"
+ "addp v13.4s, v13.4s, v13.4s\n"
+ "addp v11.4s, v11.4s, v11.4s\n"
+ "addp v12.4s, v12.4s, v12.4s\n"
+ "addp v13.4s, v13.4s, v13.4s\n"
+ "neg v3.4s, v3.4s\n"
+ "mul v11.4s, v11.4s, v3.4s\n"
+ "mul v12.4s, v12.4s, v3.4s\n"
+ "mul v13.4s, v13.4s, v3.4s\n"
+ "82:" // Height 3: skip row sum fixup
+ "add v16.4s, v16.4s, v11.4s\n"
+ "orr %x[flags], %x[flags], #0x80000000\n"
+ "add v17.4s, v17.4s, v11.4s\n"
+ "ldr q0, [x10, #0x0]\n"
+ "add v18.4s, v18.4s, v11.4s\n"
+ "ldr q1, [x10, #0x10]\n"
+ "add v19.4s, v19.4s, v11.4s\n"
+ "ldr q2, [x10, #0x20]\n"
+ "add v20.4s, v20.4s, v12.4s\n"
+ "ldr q3, [x10, #0x30]\n"
+ "add v21.4s, v21.4s, v12.4s\n"
+ "add x20, %x[qp], %[per_layer_right_shift]\n"
+ "add v22.4s, v22.4s, v12.4s\n"
+ "add x19, %x[qp], %[per_layer_mul]\n"
+ "ld1r { v4.4s }, [x19]\n"
+ "add v23.4s, v23.4s, v12.4s\n"
+ "add x10, x10, #0x40\n"
+ "add v24.4s, v24.4s, v13.4s\n"
+ "add v25.4s, v25.4s, v13.4s\n"
+ "add v26.4s, v26.4s, v13.4s\n"
+ "add v27.4s, v27.4s, v13.4s\n"
+ "add v16.4s, v16.4s, v0.4s\n"
+ "add v17.4s, v17.4s, v1.4s\n"
+ "add v18.4s, v18.4s, v2.4s\n"
+ "add v19.4s, v19.4s, v3.4s\n"
+ "add v20.4s, v20.4s, v0.4s\n"
+ "add v21.4s, v21.4s, v1.4s\n"
+ "add v22.4s, v22.4s, v2.4s\n"
+ "add v23.4s, v23.4s, v3.4s\n"
+ "add v24.4s, v24.4s, v0.4s\n"
+ "ld1r { v0.4s }, [x20]\n"
+ "add v25.4s, v25.4s, v1.4s\n"
+ "add v26.4s, v26.4s, v2.4s\n"
+ "add v27.4s, v27.4s, v3.4s\n"
+ "sqrdmulh v16.4s, v16.4s, v4.4s\n"
+ "sqrdmulh v17.4s, v17.4s, v4.4s\n"
+ "sqrdmulh v18.4s, v18.4s, v4.4s\n"
+ "sqrdmulh v19.4s, v19.4s, v4.4s\n"
+ "sqrdmulh v20.4s, v20.4s, v4.4s\n"
+ "sqrdmulh v21.4s, v21.4s, v4.4s\n"
+ "sqrdmulh v22.4s, v22.4s, v4.4s\n"
+ "sqrdmulh v23.4s, v23.4s, v4.4s\n"
+ "sqrdmulh v24.4s, v24.4s, v4.4s\n"
+ "sqrdmulh v25.4s, v25.4s, v4.4s\n"
+ "sqrdmulh v26.4s, v26.4s, v4.4s\n"
+ "sqrdmulh v27.4s, v27.4s, v4.4s\n"
+ "tbz %x[flags], #5, 83f\n"
+ "and v4.16b, v16.16b, v0.16b\n"
+ "sshr v4.4s, v4.4s, #0x1f\n"
+ "and v5.16b, v17.16b, v0.16b\n"
+ "and v6.16b, v18.16b, v0.16b\n"
+ "sshr v5.4s, v5.4s, #0x1f\n"
+ "and v7.16b, v19.16b, v0.16b\n"
+ "and v8.16b, v20.16b, v0.16b\n"
+ "sshr v6.4s, v6.4s, #0x1f\n"
+ "and v9.16b, v21.16b, v0.16b\n"
+ "sshr v7.4s, v7.4s, #0x1f\n"
+ "sqadd v16.4s, v16.4s, v4.4s\n"
+ "and v10.16b, v22.16b, v0.16b\n"
+ "sshr v8.4s, v8.4s, #0x1f\n"
+ "and v4.16b, v23.16b, v0.16b\n"
+ "sshr v9.4s, v9.4s, #0x1f\n"
+ "sqadd v17.4s, v17.4s, v5.4s\n"
+ "sshr v10.4s, v10.4s, #0x1f\n"
+ "sqadd v18.4s, v18.4s, v6.4s\n"
+ "sshr v4.4s, v4.4s, #0x1f\n"
+ "and v5.16b, v24.16b, v0.16b\n"
+ "sshr v5.4s, v5.4s, #0x1f\n"
+ "sqadd v19.4s, v19.4s, v7.4s\n"
+ "sqadd v20.4s, v20.4s, v8.4s\n"
+ "sqadd v21.4s, v21.4s, v9.4s\n"
+ "sqadd v22.4s, v22.4s, v10.4s\n"
+ "sqadd v23.4s, v23.4s, v4.4s\n"
+ "and v6.16b, v25.16b, v0.16b\n"
+ "sshr v6.4s, v6.4s, #0x1f\n"
+ "sqadd v24.4s, v24.4s, v5.4s\n"
+ "and v7.16b, v26.16b, v0.16b\n"
+ "sshr v7.4s, v7.4s, #0x1f\n"
+ "and v8.16b, v27.16b, v0.16b\n"
+ "sqadd v25.4s, v25.4s, v6.4s\n"
+ "sshr v8.4s, v8.4s, #0x1f\n"
+ "sqadd v26.4s, v26.4s, v7.4s\n"
+ "sqadd v27.4s, v27.4s, v8.4s\n"
+ "83:" // Height 3: no shift correction
+ "srshl v16.4s, v16.4s, v0.4s\n"
+ "add x19, %x[qp], %[c_offset]\n"
+ "ld1r { v4.4s }, [x19]\n"
+ "srshl v17.4s, v17.4s, v0.4s\n"
+ "add x19, %x[qp], %[minval]\n"
+ "srshl v18.4s, v18.4s, v0.4s\n"
+ "ld1r { v5.4s }, [x19]\n"
+ "add x19, %x[qp], %[maxval]\n"
+ "srshl v19.4s, v19.4s, v0.4s\n"
+ "ld1r { v6.4s }, [x19]\n"
+ "cmp x12, #0x10\n"
+ "srshl v20.4s, v20.4s, v0.4s\n"
+ "srshl v21.4s, v21.4s, v0.4s\n"
+ "srshl v22.4s, v22.4s, v0.4s\n"
+ "srshl v23.4s, v23.4s, v0.4s\n"
+ "add v16.4s, v16.4s, v4.4s\n"
+ "add v17.4s, v17.4s, v4.4s\n"
+ "add v18.4s, v18.4s, v4.4s\n"
+ "smin v16.4s, v16.4s, v6.4s\n"
+ "smin v17.4s, v17.4s, v6.4s\n"
+ "smin v18.4s, v18.4s, v6.4s\n"
+ "smax v16.4s, v16.4s, v5.4s\n"
+ "smax v17.4s, v17.4s, v5.4s\n"
+ "smax v18.4s, v18.4s, v5.4s\n"
+ "add v19.4s, v19.4s, v4.4s\n"
+ "add v20.4s, v20.4s, v4.4s\n"
+ "add v21.4s, v21.4s, v4.4s\n"
+ "smin v19.4s, v19.4s, v6.4s\n"
+ "smin v20.4s, v20.4s, v6.4s\n"
+ "smin v21.4s, v21.4s, v6.4s\n"
+ "smax v19.4s, v19.4s, v5.4s\n"
+ "smax v20.4s, v20.4s, v5.4s\n"
+ "smax v21.4s, v21.4s, v5.4s\n"
+ "add v22.4s, v22.4s, v4.4s\n"
+ "add v23.4s, v23.4s, v4.4s\n"
+ "srshl v24.4s, v24.4s, v0.4s\n"
+ "smin v22.4s, v22.4s, v6.4s\n"
+ "smin v23.4s, v23.4s, v6.4s\n"
+ "srshl v25.4s, v25.4s, v0.4s\n"
+ "smax v22.4s, v22.4s, v5.4s\n"
+ "smax v23.4s, v23.4s, v5.4s\n"
+ "add v24.4s, v24.4s, v4.4s\n"
+ "add v25.4s, v25.4s, v4.4s\n"
+ "srshl v26.4s, v26.4s, v0.4s\n"
+ "smin v24.4s, v24.4s, v6.4s\n"
+ "smin v25.4s, v25.4s, v6.4s\n"
+ "srshl v27.4s, v27.4s, v0.4s\n"
+ "smax v24.4s, v24.4s, v5.4s\n"
+ "smax v25.4s, v25.4s, v5.4s\n"
+ "add v26.4s, v26.4s, v4.4s\n"
+ "add v27.4s, v27.4s, v4.4s\n"
+ "uzp1 v16.8h, v16.8h, v17.8h\n"
+ "smin v26.4s, v26.4s, v6.4s\n"
+ "smin v27.4s, v27.4s, v6.4s\n"
+ "uzp1 v17.8h, v18.8h, v19.8h\n"
+ "smax v26.4s, v26.4s, v5.4s\n"
+ "smax v27.4s, v27.4s, v5.4s\n"
+ "uzp1 v20.8h, v20.8h, v21.8h\n"
+ "uzp1 v21.8h, v22.8h, v23.8h\n"
+ "uzp1 v24.8h, v24.8h, v25.8h\n"
+ "uzp1 v25.8h, v26.8h, v27.8h\n"
+ "uzp1 v16.16b, v16.16b, v17.16b\n"
+ "uzp1 v20.16b, v20.16b, v21.16b\n"
+ "uzp1 v24.16b, v24.16b, v25.16b\n"
+ "bge 92f\n"
+ "tbz x12, #3, 87f\n"
+ "str d16, [x9], #0x8\n"
+ "str d20, [x25], #0x8\n"
+ "str d24, [x23], #0x8\n"
+ "tbz x12, #2, 85f\n"
+ "st1 { v16.s }[2], [x9], #0x4\n"
+ "st1 { v20.s }[2], [x25], #0x4\n"
+ "st1 { v24.s }[2], [x23], #0x4\n"
+ "tbz x12, #1, 84f\n"
+ "st1 { v16.h }[6], [x9], #0x2\n"
+ "st1 { v20.h }[6], [x25], #0x2\n"
+ "st1 { v24.h }[6], [x23], #0x2\n"
+ "tbz x12, #0, 91f\n"
+ "st1 { v16.b }[14], [x9]\n"
+ "st1 { v20.b }[14], [x25]\n"
+ "st1 { v24.b }[14], [x23]\n"
+ "b 91f\n"
+ "84:" // Height 3: Partial direct writeback: partial_1_12
+ "tbz x12, #0, 91f\n"
+ "st1 { v16.b }[12], [x9]\n"
+ "st1 { v20.b }[12], [x25]\n"
+ "st1 { v24.b }[12], [x23]\n"
+ "b 91f\n"
+ "85:" // Height 3: Partial direct writeback: partial_2_8
+ "tbz x12, #1, 86f\n"
+ "st1 { v16.h }[4], [x9], #0x2\n"
+ "st1 { v20.h }[4], [x25], #0x2\n"
+ "st1 { v24.h }[4], [x23], #0x2\n"
+ "tbz x12, #0, 91f\n"
+ "st1 { v16.b }[10], [x9]\n"
+ "st1 { v20.b }[10], [x25]\n"
+ "st1 { v24.b }[10], [x23]\n"
+ "b 91f\n"
+ "86:" // Height 3: Partial direct writeback: partial_1_8
+ "tbz x12, #0, 91f\n"
+ "st1 { v16.b }[8], [x9]\n"
+ "st1 { v20.b }[8], [x25]\n"
+ "st1 { v24.b }[8], [x23]\n"
+ "b 91f\n"
+ "87:" // Height 3: Partial direct writeback: partial_4_0
+ "tbz x12, #2, 89f\n"
+ "str s16, [x9], #0x4\n"
+ "str s20, [x25], #0x4\n"
+ "str s24, [x23], #0x4\n"
+ "tbz x12, #1, 88f\n"
+ "st1 { v16.h }[2], [x9], #0x2\n"
+ "st1 { v20.h }[2], [x25], #0x2\n"
+ "st1 { v24.h }[2], [x23], #0x2\n"
+ "tbz x12, #0, 91f\n"
+ "st1 { v16.b }[6], [x9]\n"
+ "st1 { v20.b }[6], [x25]\n"
+ "st1 { v24.b }[6], [x23]\n"
+ "b 91f\n"
+ "88:" // Height 3: Partial direct writeback: partial_1_4
+ "tbz x12, #0, 91f\n"
+ "st1 { v16.b }[4], [x9]\n"
+ "st1 { v20.b }[4], [x25]\n"
+ "st1 { v24.b }[4], [x23]\n"
+ "b 91f\n"
+ "89:" // Height 3: Partial direct writeback: partial_2_0
+ "tbz x12, #1, 90f\n"
+ "str h16, [x9], #0x2\n"
+ "str h20, [x25], #0x2\n"
+ "str h24, [x23], #0x2\n"
+ "tbz x12, #0, 91f\n"
+ "st1 { v16.b }[2], [x9]\n"
+ "st1 { v20.b }[2], [x25]\n"
+ "st1 { v24.b }[2], [x23]\n"
+ "b 91f\n"
+ "90:" // Height 3: Partial direct writeback: partial_1_0
+ "str b16, [x9, #0x0]\n"
+ "str b20, [x25, #0x0]\n"
+ "str b24, [x23, #0x0]\n"
+ "91:" // Height 3: Partial direct writeback: Done
+ "b 93f\n"
+ "92:" // Height 3: Full writeback
+ "str q16, [x9, #0x0]\n"
+ "str q20, [x25, #0x0]\n"
+ "str q24, [x23, #0x0]\n"
+ "add x9, x9, #0x10\n"
+ "add x25, x25, #0x10\n"
+ "add x23, x23, #0x10\n"
+ "93:" // Height 3: Writeback done
+ "subs x12, x12, #0x10\n"
+ "bgt 65b\n"
+ "b 126f\n"
+ "94:" // Height 4
+ "movi v11.4s, #0x0\n"
+ "ldr x12, [%x[args_ptr], %[offsetof_N]]\n"
+ "mov x10, %x[col_bias]\n"
+ "movi v12.4s, #0x0\n"
+ "ldr x11, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "bic %x[flags], %x[flags], #0x80000000\n"
+ "movi v13.4s, #0x0\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "movi v14.4s, #0x0\n"
+ "movi v15.16b, #0x1\n"
+ "tbz %x[flags], #2, 95f\n"
+ "ldr x9, [%x[output_ptr], #0x0]\n"
+ "ldr x25, [%x[output_ptr], #0x8]\n"
+ "add x9, x9, x19\n"
+ "ldr x23, [%x[output_ptr], #0x10]\n"
+ "ldr x21, [%x[output_ptr], #0x18]\n"
+ "add x25, x25, x19\n"
+ "add %x[output_ptr], %x[output_ptr], #0x20\n"
+ "add x23, x23, x19\n"
+ "add x21, x21, x19\n"
+ "b 96f\n"
+ "95:" // Height 4: setup direct output
+ "mov x9, %x[output_ptr]\n"
+ "add x25, x9, x19\n"
+ "add x23, x25, x19\n"
+ "add x21, x23, x19\n"
+ "add %x[output_ptr], x21, x19\n"
+ "96:" // Height 4: Column loop
+ "movi v16.4s, #0x0\n"
+ "movi v17.4s, #0x0\n"
+ "movi v18.4s, #0x0\n"
+ "movi v19.4s, #0x0\n"
+ "movi v20.4s, #0x0\n"
+ "movi v21.4s, #0x0\n"
+ "movi v22.4s, #0x0\n"
+ "movi v23.4s, #0x0\n"
+ "movi v24.4s, #0x0\n"
+ "movi v25.4s, #0x0\n"
+ "movi v26.4s, #0x0\n"
+ "movi v27.4s, #0x0\n"
+ "movi v28.4s, #0x0\n"
+ "movi v29.4s, #0x0\n"
+ "movi v30.4s, #0x0\n"
+ "movi v31.4s, #0x0\n"
+ "97:" // Height 4: setup done
+ "mov x28, #0x0\n"
+ "98:" // Height 4: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w27, [x20, x28, LSL #0x2]\n"
+ "tbz %x[flags], #3, 99f\n"
+ "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x26, [x20, #0x0]\n"
+ "ldr x24, [x20, #0x8]\n"
+ "ldr x22, [x20, #0x10]\n"
+ "ldr x20, [x20, #0x18]\n"
+ "cbnz x28, 100f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x26, x26, x19\n"
+ "add x24, x24, x19\n"
+ "add x22, x22, x19\n"
+ "add x20, x20, x19\n"
+ "b 100f\n"
+ "99:" // Height 4: setup direct input
+ "mov x26, %x[input_ptr]\n"
+ "add x24, x26, x19\n"
+ "add x22, x24, x19\n"
+ "add x20, x22, x19\n"
+ "100:" // Height 4: input setup done
+ "cmp x27, #0x10\n"
+ "blt 105f\n"
+ "cmp x27, #0x20\n"
+ "blt 103f\n"
+ "101:" // Height 4: Multiply loop: Main loop head
+ "ldr q0, [x26, #0x0]\n"
+ "ldr q1, [x24, #0x0]\n"
+ "ldr q2, [x22, #0x0]\n"
+ "ldr q3, [x20, #0x0]\n"
+ "ldr q4, [x11, #0x0]\n"
+ ".inst 0x4f80e090 // sdot v16.4s, v4.16b, v0.4b[0]\n"
+ "ldr q5, [x11, #0x10]\n"
+ ".inst 0x4f81e094 // sdot v20.4s, v4.16b, v1.4b[0]\n"
+ "ldr q6, [x11, #0x20]\n"
+ ".inst 0x4f82e098 // sdot v24.4s, v4.16b, v2.4b[0]\n"
+ "ldr q7, [x11, #0x30]\n"
+ ".inst 0x4f83e09c // sdot v28.4s, v4.16b, v3.4b[0]\n"
+ "ldr q8, [x11, #0x40]\n"
+ "ldr q9, [x11, #0x50]\n"
+ ".inst 0x4f80e0b1 // sdot v17.4s, v5.16b, v0.4b[0]\n"
+ "ldr q10, [x11, #0x60]\n"
+ ".inst 0x4f81e0b5 // sdot v21.4s, v5.16b, v1.4b[0]\n"
+ "ldr q4, [x11, #0x70]\n"
+ ".inst 0x4f82e0b9 // sdot v25.4s, v5.16b, v2.4b[0]\n"
+ "add x26, x26, #0x10\n"
+ ".inst 0x4f83e0bd // sdot v29.4s, v5.16b, v3.4b[0]\n"
+ "ldr q5, [x11, #0x80]\n"
+ ".inst 0x4f80e0d2 // sdot v18.4s, v6.16b, v0.4b[0]\n"
+ "add x24, x24, #0x10\n"
+ ".inst 0x4f81e0d6 // sdot v22.4s, v6.16b, v1.4b[0]\n"
+ "add x22, x22, #0x10\n"
+ ".inst 0x4f82e0da // sdot v26.4s, v6.16b, v2.4b[0]\n"
+ "add x20, x20, #0x10\n"
+ ".inst 0x4f83e0de // sdot v30.4s, v6.16b, v3.4b[0]\n"
+ "ldr q6, [x11, #0x90]\n"
+ ".inst 0x4f80e0f3 // sdot v19.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x4f81e0f7 // sdot v23.4s, v7.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0fb // sdot v27.4s, v7.16b, v2.4b[0]\n"
+ ".inst 0x4f83e0ff // sdot v31.4s, v7.16b, v3.4b[0]\n"
+ "ldr q7, [x11, #0xa0]\n"
+ ".inst 0x4fa0e110 // sdot v16.4s, v8.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e114 // sdot v20.4s, v8.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e118 // sdot v24.4s, v8.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e11c // sdot v28.4s, v8.16b, v3.4b[1]\n"
+ "ldr q8, [x11, #0xb0]\n"
+ ".inst 0x4fa0e131 // sdot v17.4s, v9.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e135 // sdot v21.4s, v9.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e139 // sdot v25.4s, v9.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e13d // sdot v29.4s, v9.16b, v3.4b[1]\n"
+ "ldr q9, [x11, #0xc0]\n"
+ ".inst 0x4fa0e152 // sdot v18.4s, v10.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e156 // sdot v22.4s, v10.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e15a // sdot v26.4s, v10.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e15e // sdot v30.4s, v10.16b, v3.4b[1]\n"
+ "ldr q10, [x11, #0xd0]\n"
+ ".inst 0x4fa0e093 // sdot v19.4s, v4.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e097 // sdot v23.4s, v4.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e09b // sdot v27.4s, v4.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e09f // sdot v31.4s, v4.16b, v3.4b[1]\n"
+ "ldr q4, [x11, #0xe0]\n"
+ ".inst 0x4f80e8b0 // sdot v16.4s, v5.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8b4 // sdot v20.4s, v5.16b, v1.4b[2]\n"
+ ".inst 0x4f82e8b8 // sdot v24.4s, v5.16b, v2.4b[2]\n"
+ ".inst 0x4f83e8bc // sdot v28.4s, v5.16b, v3.4b[2]\n"
+ "ldr q5, [x11, #0xf0]\n"
+ ".inst 0x4f80e8d1 // sdot v17.4s, v6.16b, v0.4b[2]\n"
+ "add x11, x11, #0x100\n"
+ ".inst 0x4f81e8d5 // sdot v21.4s, v6.16b, v1.4b[2]\n"
+ ".inst 0x4f82e8d9 // sdot v25.4s, v6.16b, v2.4b[2]\n"
+ ".inst 0x4f83e8dd // sdot v29.4s, v6.16b, v3.4b[2]\n"
+ ".inst 0x4f80e8f2 // sdot v18.4s, v7.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8f6 // sdot v22.4s, v7.16b, v1.4b[2]\n"
+ ".inst 0x4f82e8fa // sdot v26.4s, v7.16b, v2.4b[2]\n"
+ ".inst 0x4f83e8fe // sdot v30.4s, v7.16b, v3.4b[2]\n"
+ ".inst 0x4f80e913 // sdot v19.4s, v8.16b, v0.4b[2]\n"
+ ".inst 0x4f81e917 // sdot v23.4s, v8.16b, v1.4b[2]\n"
+ ".inst 0x4f82e91b // sdot v27.4s, v8.16b, v2.4b[2]\n"
+ ".inst 0x4f83e91f // sdot v31.4s, v8.16b, v3.4b[2]\n"
+ ".inst 0x4fa0e930 // sdot v16.4s, v9.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e934 // sdot v20.4s, v9.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e938 // sdot v24.4s, v9.16b, v2.4b[3]\n"
+ ".inst 0x4fa3e93c // sdot v28.4s, v9.16b, v3.4b[3]\n"
+ ".inst 0x4fa0e951 // sdot v17.4s, v10.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e955 // sdot v21.4s, v10.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e959 // sdot v25.4s, v10.16b, v2.4b[3]\n"
+ ".inst 0x4fa3e95d // sdot v29.4s, v10.16b, v3.4b[3]\n"
+ ".inst 0x4fa0e892 // sdot v18.4s, v4.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e896 // sdot v22.4s, v4.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e89a // sdot v26.4s, v4.16b, v2.4b[3]\n"
+ ".inst 0x4fa3e89e // sdot v30.4s, v4.16b, v3.4b[3]\n"
+ ".inst 0x4fa0e8b3 // sdot v19.4s, v5.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e8b7 // sdot v23.4s, v5.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e8bb // sdot v27.4s, v5.16b, v2.4b[3]\n"
+ ".inst 0x4fa3e8bf // sdot v31.4s, v5.16b, v3.4b[3]\n"
+ "tbnz %x[flags], #31, 102f\n"
+ ".inst 0x4e8f940b // sdot v11.4s, v0.16b, v15.16b\n"
+ ".inst 0x4e8f942c // sdot v12.4s, v1.16b, v15.16b\n"
+ ".inst 0x4e8f944d // sdot v13.4s, v2.16b, v15.16b\n"
+ ".inst 0x4e8f946e // sdot v14.4s, v3.16b, v15.16b\n"
+ "102:" // Height 4: Multiply loop: unique 13: skip row sum
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "sub x27, x27, #0x10\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "cmp x27, #0x20\n"
+ "prfm pldl1keep, [x22, #0x80]\n"
+ "prfm pldl1keep, [x20, #0x80]\n"
+ "bge 101b\n"
+ "103:" // Height 4: Multiply loop: Single iteration only
+ "sub x27, x27, #0x10\n"
+ "ldr q0, [x26, #0x0]\n"
+ "ldr q1, [x24, #0x0]\n"
+ "ldr q2, [x22, #0x0]\n"
+ "ldr q3, [x20, #0x0]\n"
+ "ldr q6, [x11, #0x0]\n"
+ ".inst 0x4f80e0d0 // sdot v16.4s, v6.16b, v0.4b[0]\n"
+ "ldr q7, [x11, #0x10]\n"
+ ".inst 0x4f81e0d4 // sdot v20.4s, v6.16b, v1.4b[0]\n"
+ "ldr q8, [x11, #0x20]\n"
+ ".inst 0x4f82e0d8 // sdot v24.4s, v6.16b, v2.4b[0]\n"
+ "ldr q9, [x11, #0x30]\n"
+ ".inst 0x4f83e0dc // sdot v28.4s, v6.16b, v3.4b[0]\n"
+ "ldr q10, [x11, #0x40]\n"
+ "ldr q4, [x11, #0x50]\n"
+ ".inst 0x4f80e0f1 // sdot v17.4s, v7.16b, v0.4b[0]\n"
+ "ldr q5, [x11, #0x60]\n"
+ ".inst 0x4f81e0f5 // sdot v21.4s, v7.16b, v1.4b[0]\n"
+ "ldr q6, [x11, #0x70]\n"
+ ".inst 0x4f82e0f9 // sdot v25.4s, v7.16b, v2.4b[0]\n"
+ "add x26, x26, #0x10\n"
+ ".inst 0x4f83e0fd // sdot v29.4s, v7.16b, v3.4b[0]\n"
+ "ldr q7, [x11, #0x80]\n"
+ ".inst 0x4f80e112 // sdot v18.4s, v8.16b, v0.4b[0]\n"
+ "add x24, x24, #0x10\n"
+ ".inst 0x4f81e116 // sdot v22.4s, v8.16b, v1.4b[0]\n"
+ "add x22, x22, #0x10\n"
+ ".inst 0x4f82e11a // sdot v26.4s, v8.16b, v2.4b[0]\n"
+ "add x20, x20, #0x10\n"
+ ".inst 0x4f83e11e // sdot v30.4s, v8.16b, v3.4b[0]\n"
+ "ldr q8, [x11, #0x90]\n"
+ ".inst 0x4f80e133 // sdot v19.4s, v9.16b, v0.4b[0]\n"
+ ".inst 0x4f81e137 // sdot v23.4s, v9.16b, v1.4b[0]\n"
+ ".inst 0x4f82e13b // sdot v27.4s, v9.16b, v2.4b[0]\n"
+ ".inst 0x4f83e13f // sdot v31.4s, v9.16b, v3.4b[0]\n"
+ "ldr q9, [x11, #0xa0]\n"
+ ".inst 0x4fa0e150 // sdot v16.4s, v10.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e154 // sdot v20.4s, v10.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e158 // sdot v24.4s, v10.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e15c // sdot v28.4s, v10.16b, v3.4b[1]\n"
+ "ldr q10, [x11, #0xb0]\n"
+ ".inst 0x4fa0e091 // sdot v17.4s, v4.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e095 // sdot v21.4s, v4.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e099 // sdot v25.4s, v4.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e09d // sdot v29.4s, v4.16b, v3.4b[1]\n"
+ "ldr q4, [x11, #0xc0]\n"
+ ".inst 0x4fa0e0b2 // sdot v18.4s, v5.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e0b6 // sdot v22.4s, v5.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e0ba // sdot v26.4s, v5.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e0be // sdot v30.4s, v5.16b, v3.4b[1]\n"
+ "ldr q5, [x11, #0xd0]\n"
+ ".inst 0x4fa0e0d3 // sdot v19.4s, v6.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e0d7 // sdot v23.4s, v6.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e0db // sdot v27.4s, v6.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e0df // sdot v31.4s, v6.16b, v3.4b[1]\n"
+ "ldr q6, [x11, #0xe0]\n"
+ ".inst 0x4f80e8f0 // sdot v16.4s, v7.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8f4 // sdot v20.4s, v7.16b, v1.4b[2]\n"
+ ".inst 0x4f82e8f8 // sdot v24.4s, v7.16b, v2.4b[2]\n"
+ ".inst 0x4f83e8fc // sdot v28.4s, v7.16b, v3.4b[2]\n"
+ "ldr q7, [x11, #0xf0]\n"
+ ".inst 0x4f80e911 // sdot v17.4s, v8.16b, v0.4b[2]\n"
+ "add x11, x11, #0x100\n"
+ ".inst 0x4f81e915 // sdot v21.4s, v8.16b, v1.4b[2]\n"
+ ".inst 0x4f82e919 // sdot v25.4s, v8.16b, v2.4b[2]\n"
+ ".inst 0x4f83e91d // sdot v29.4s, v8.16b, v3.4b[2]\n"
+ ".inst 0x4f80e932 // sdot v18.4s, v9.16b, v0.4b[2]\n"
+ ".inst 0x4f81e936 // sdot v22.4s, v9.16b, v1.4b[2]\n"
+ ".inst 0x4f82e93a // sdot v26.4s, v9.16b, v2.4b[2]\n"
+ ".inst 0x4f83e93e // sdot v30.4s, v9.16b, v3.4b[2]\n"
+ ".inst 0x4f80e953 // sdot v19.4s, v10.16b, v0.4b[2]\n"
+ ".inst 0x4f81e957 // sdot v23.4s, v10.16b, v1.4b[2]\n"
+ ".inst 0x4f82e95b // sdot v27.4s, v10.16b, v2.4b[2]\n"
+ ".inst 0x4f83e95f // sdot v31.4s, v10.16b, v3.4b[2]\n"
+ ".inst 0x4fa0e890 // sdot v16.4s, v4.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e894 // sdot v20.4s, v4.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e898 // sdot v24.4s, v4.16b, v2.4b[3]\n"
+ ".inst 0x4fa3e89c // sdot v28.4s, v4.16b, v3.4b[3]\n"
+ ".inst 0x4fa0e8b1 // sdot v17.4s, v5.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e8b5 // sdot v21.4s, v5.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e8b9 // sdot v25.4s, v5.16b, v2.4b[3]\n"
+ ".inst 0x4fa3e8bd // sdot v29.4s, v5.16b, v3.4b[3]\n"
+ ".inst 0x4fa0e8d2 // sdot v18.4s, v6.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e8d6 // sdot v22.4s, v6.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e8da // sdot v26.4s, v6.16b, v2.4b[3]\n"
+ ".inst 0x4fa3e8de // sdot v30.4s, v6.16b, v3.4b[3]\n"
+ ".inst 0x4fa0e8f3 // sdot v19.4s, v7.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e8f7 // sdot v23.4s, v7.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e8fb // sdot v27.4s, v7.16b, v2.4b[3]\n"
+ ".inst 0x4fa3e8ff // sdot v31.4s, v7.16b, v3.4b[3]\n"
+ "tbnz %x[flags], #31, 104f\n"
+ ".inst 0x4e8f940b // sdot v11.4s, v0.16b, v15.16b\n"
+ ".inst 0x4e8f942c // sdot v12.4s, v1.16b, v15.16b\n"
+ ".inst 0x4e8f944d // sdot v13.4s, v2.16b, v15.16b\n"
+ ".inst 0x4e8f946e // sdot v14.4s, v3.16b, v15.16b\n"
+ "104:" // Height 4: Multiply loop: unique 14: skip row sum
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "prfm pldl1keep, [x22, #0x80]\n"
+ "prfm pldl1keep, [x20, #0x80]\n"
+ "105:" // Height 4: Multiply loop: Main loop skip
+ "cbz x27, 112f\n"
+ "cmp x27, #0x4\n"
+ "blt 108f\n"
+ "106:" // Height 4: Multiply loop: Odd block loop
+ "ldr s0, [x26], #0x4\n"
+ "ldr s1, [x24], #0x4\n"
+ "ldr s2, [x22], #0x4\n"
+ "ldr s3, [x20], #0x4\n"
+ "tbnz %x[flags], #31, 107f\n"
+ ".inst 0x4e8f940b // sdot v11.4s, v0.16b, v15.16b\n"
+ ".inst 0x4e8f942c // sdot v12.4s, v1.16b, v15.16b\n"
+ ".inst 0x4e8f944d // sdot v13.4s, v2.16b, v15.16b\n"
+ ".inst 0x4e8f946e // sdot v14.4s, v3.16b, v15.16b\n"
+ "107:" // Height 4: Multiply loop: unique 15: skip row sum
+ "ldr q8, [x11, #0x0]\n"
+ ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
+ "ldr q9, [x11, #0x10]\n"
+ ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
+ "ldr q10, [x11, #0x20]\n"
+ ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n"
+ "ldr q4, [x11, #0x30]\n"
+ ".inst 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n"
+ "sub x27, x27, #0x4\n"
+ "add x11, x11, #0x40\n"
+ ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
+ "cmp x27, #0x4\n"
+ ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
+ ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n"
+ ".inst 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n"
+ ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
+ ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
+ ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n"
+ ".inst 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n"
+ ".inst 0x4f80e093 // sdot v19.4s, v4.16b, v0.4b[0]\n"
+ ".inst 0x4f81e097 // sdot v23.4s, v4.16b, v1.4b[0]\n"
+ ".inst 0x4f82e09b // sdot v27.4s, v4.16b, v2.4b[0]\n"
+ ".inst 0x4f83e09f // sdot v31.4s, v4.16b, v3.4b[0]\n"
+ "bge 106b\n"
+ "cbz x27, 112f\n"
+ "108:" // Height 4: Multiply loop: Skip odd blocks
+ "tbz x27, #1, 109f\n"
+ "ldr h0, [x26], #0x2\n"
+ "ldr h1, [x24], #0x2\n"
+ "ldr h2, [x22], #0x2\n"
+ "ldr h3, [x20], #0x2\n"
+ "tbz x27, #0, 110f\n"
+ "ld1 { v0.b }[2], [x26]\n"
+ "ld1 { v1.b }[2], [x24]\n"
+ "ld1 { v2.b }[2], [x22]\n"
+ "ld1 { v3.b }[2], [x20]\n"
+ "b 110f\n"
+ "109:" // Height 4: Multiply loop: Ragged operand read: partial_1_0
+ "ldr b0, [x26, #0x0]\n"
+ "ldr b1, [x24, #0x0]\n"
+ "ldr b2, [x22, #0x0]\n"
+ "ldr b3, [x20, #0x0]\n"
+ "110:" // Height 4: Multiply loop: Ragged operand read: Done
+ "tbnz %x[flags], #31, 111f\n"
+ ".inst 0x4e8f940b // sdot v11.4s, v0.16b, v15.16b\n"
+ ".inst 0x4e8f942c // sdot v12.4s, v1.16b, v15.16b\n"
+ ".inst 0x4e8f944d // sdot v13.4s, v2.16b, v15.16b\n"
+ ".inst 0x4e8f946e // sdot v14.4s, v3.16b, v15.16b\n"
+ "111:" // Height 4: Multiply loop: unique 16: skip row sum
+ "ldr q5, [x11, #0x0]\n"
+ ".inst 0x4f80e0b0 // sdot v16.4s, v5.16b, v0.4b[0]\n"
+ "ldr q6, [x11, #0x10]\n"
+ ".inst 0x4f81e0b4 // sdot v20.4s, v5.16b, v1.4b[0]\n"
+ "ldr q7, [x11, #0x20]\n"
+ ".inst 0x4f82e0b8 // sdot v24.4s, v5.16b, v2.4b[0]\n"
+ "ldr q8, [x11, #0x30]\n"
+ ".inst 0x4f83e0bc // sdot v28.4s, v5.16b, v3.4b[0]\n"
+ "add x11, x11, #0x40\n"
+ ".inst 0x4f80e0d1 // sdot v17.4s, v6.16b, v0.4b[0]\n"
+ ".inst 0x4f81e0d5 // sdot v21.4s, v6.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0d9 // sdot v25.4s, v6.16b, v2.4b[0]\n"
+ ".inst 0x4f83e0dd // sdot v29.4s, v6.16b, v3.4b[0]\n"
+ ".inst 0x4f80e0f2 // sdot v18.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x4f81e0f6 // sdot v22.4s, v7.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0fa // sdot v26.4s, v7.16b, v2.4b[0]\n"
+ ".inst 0x4f83e0fe // sdot v30.4s, v7.16b, v3.4b[0]\n"
+ ".inst 0x4f80e113 // sdot v19.4s, v8.16b, v0.4b[0]\n"
+ ".inst 0x4f81e117 // sdot v23.4s, v8.16b, v1.4b[0]\n"
+ ".inst 0x4f82e11b // sdot v27.4s, v8.16b, v2.4b[0]\n"
+ ".inst 0x4f83e11f // sdot v31.4s, v8.16b, v3.4b[0]\n"
+ "112:" // Height 4: Multiply loop: No odd multiplies
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "add x28, x28, #0x1\n"
+ "cmp x28, x19\n"
+ "bne 98b\n"
+ "prfm pstl1keep, [x9, #0x0]\n"
+ "prfm pstl1keep, [x25, #0x0]\n"
+ "prfm pstl1keep, [x23, #0x0]\n"
+ "prfm pstl1keep, [x21, #0x0]\n"
+ "tbnz %x[flags], #31, 113f\n"
+ "addp v11.4s, v11.4s, v11.4s\n"
+ "add x19, %x[qp], %[b_offset]\n"
+ "ld1r { v4.4s }, [x19]\n"
+ "addp v12.4s, v12.4s, v12.4s\n"
+ "addp v13.4s, v13.4s, v13.4s\n"
+ "addp v14.4s, v14.4s, v14.4s\n"
+ "addp v11.4s, v11.4s, v11.4s\n"
+ "addp v12.4s, v12.4s, v12.4s\n"
+ "addp v13.4s, v13.4s, v13.4s\n"
+ "addp v14.4s, v14.4s, v14.4s\n"
+ "neg v4.4s, v4.4s\n"
+ "mul v11.4s, v11.4s, v4.4s\n"
+ "mul v12.4s, v12.4s, v4.4s\n"
+ "mul v13.4s, v13.4s, v4.4s\n"
+ "mul v14.4s, v14.4s, v4.4s\n"
+ "113:" // Height 4: skip row sum fixup
+ "add v16.4s, v16.4s, v11.4s\n"
+ "orr %x[flags], %x[flags], #0x80000000\n"
+ "add v17.4s, v17.4s, v11.4s\n"
+ "ldr q0, [x10, #0x0]\n"
+ "add v18.4s, v18.4s, v11.4s\n"
+ "ldr q1, [x10, #0x10]\n"
+ "add v19.4s, v19.4s, v11.4s\n"
+ "ldr q2, [x10, #0x20]\n"
+ "add v20.4s, v20.4s, v12.4s\n"
+ "ldr q3, [x10, #0x30]\n"
+ "add v21.4s, v21.4s, v12.4s\n"
+ "add x20, %x[qp], %[per_layer_right_shift]\n"
+ "add v22.4s, v22.4s, v12.4s\n"
+ "add x19, %x[qp], %[per_layer_mul]\n"
+ "ld1r { v4.4s }, [x19]\n"
+ "add v23.4s, v23.4s, v12.4s\n"
+ "add x10, x10, #0x40\n"
+ "add v24.4s, v24.4s, v13.4s\n"
+ "add v25.4s, v25.4s, v13.4s\n"
+ "add v26.4s, v26.4s, v13.4s\n"
+ "add v27.4s, v27.4s, v13.4s\n"
+ "add v28.4s, v28.4s, v14.4s\n"
+ "add v29.4s, v29.4s, v14.4s\n"
+ "add v30.4s, v30.4s, v14.4s\n"
+ "add v31.4s, v31.4s, v14.4s\n"
+ "add v16.4s, v16.4s, v0.4s\n"
+ "add v17.4s, v17.4s, v1.4s\n"
+ "add v18.4s, v18.4s, v2.4s\n"
+ "add v19.4s, v19.4s, v3.4s\n"
+ "add v20.4s, v20.4s, v0.4s\n"
+ "add v21.4s, v21.4s, v1.4s\n"
+ "add v22.4s, v22.4s, v2.4s\n"
+ "add v23.4s, v23.4s, v3.4s\n"
+ "add v24.4s, v24.4s, v0.4s\n"
+ "add v25.4s, v25.4s, v1.4s\n"
+ "add v26.4s, v26.4s, v2.4s\n"
+ "add v27.4s, v27.4s, v3.4s\n"
+ "add v28.4s, v28.4s, v0.4s\n"
+ "ld1r { v0.4s }, [x20]\n"
+ "add v29.4s, v29.4s, v1.4s\n"
+ "add v30.4s, v30.4s, v2.4s\n"
+ "add v31.4s, v31.4s, v3.4s\n"
+ "sqrdmulh v16.4s, v16.4s, v4.4s\n"
+ "sqrdmulh v17.4s, v17.4s, v4.4s\n"
+ "sqrdmulh v18.4s, v18.4s, v4.4s\n"
+ "sqrdmulh v19.4s, v19.4s, v4.4s\n"
+ "sqrdmulh v20.4s, v20.4s, v4.4s\n"
+ "sqrdmulh v21.4s, v21.4s, v4.4s\n"
+ "sqrdmulh v22.4s, v22.4s, v4.4s\n"
+ "sqrdmulh v23.4s, v23.4s, v4.4s\n"
+ "sqrdmulh v24.4s, v24.4s, v4.4s\n"
+ "sqrdmulh v25.4s, v25.4s, v4.4s\n"
+ "sqrdmulh v26.4s, v26.4s, v4.4s\n"
+ "sqrdmulh v27.4s, v27.4s, v4.4s\n"
+ "sqrdmulh v28.4s, v28.4s, v4.4s\n"
+ "sqrdmulh v29.4s, v29.4s, v4.4s\n"
+ "sqrdmulh v30.4s, v30.4s, v4.4s\n"
+ "sqrdmulh v31.4s, v31.4s, v4.4s\n"
+ "tbz %x[flags], #5, 114f\n"
+ "and v4.16b, v16.16b, v0.16b\n"
+ "sshr v4.4s, v4.4s, #0x1f\n"
+ "and v5.16b, v17.16b, v0.16b\n"
+ "and v6.16b, v18.16b, v0.16b\n"
+ "sshr v5.4s, v5.4s, #0x1f\n"
+ "and v7.16b, v19.16b, v0.16b\n"
+ "and v8.16b, v20.16b, v0.16b\n"
+ "sshr v6.4s, v6.4s, #0x1f\n"
+ "and v9.16b, v21.16b, v0.16b\n"
+ "sshr v7.4s, v7.4s, #0x1f\n"
+ "sqadd v16.4s, v16.4s, v4.4s\n"
+ "and v10.16b, v22.16b, v0.16b\n"
+ "sshr v8.4s, v8.4s, #0x1f\n"
+ "and v4.16b, v23.16b, v0.16b\n"
+ "sshr v9.4s, v9.4s, #0x1f\n"
+ "sqadd v17.4s, v17.4s, v5.4s\n"
+ "sshr v10.4s, v10.4s, #0x1f\n"
+ "sqadd v18.4s, v18.4s, v6.4s\n"
+ "sshr v4.4s, v4.4s, #0x1f\n"
+ "and v5.16b, v24.16b, v0.16b\n"
+ "sshr v5.4s, v5.4s, #0x1f\n"
+ "sqadd v19.4s, v19.4s, v7.4s\n"
+ "sqadd v20.4s, v20.4s, v8.4s\n"
+ "sqadd v21.4s, v21.4s, v9.4s\n"
+ "sqadd v22.4s, v22.4s, v10.4s\n"
+ "sqadd v23.4s, v23.4s, v4.4s\n"
+ "and v6.16b, v25.16b, v0.16b\n"
+ "sshr v6.4s, v6.4s, #0x1f\n"
+ "sqadd v24.4s, v24.4s, v5.4s\n"
+ "and v7.16b, v26.16b, v0.16b\n"
+ "sshr v7.4s, v7.4s, #0x1f\n"
+ "and v8.16b, v27.16b, v0.16b\n"
+ "and v9.16b, v28.16b, v0.16b\n"
+ "sshr v8.4s, v8.4s, #0x1f\n"
+ "sqadd v25.4s, v25.4s, v6.4s\n"
+ "and v10.16b, v29.16b, v0.16b\n"
+ "sshr v9.4s, v9.4s, #0x1f\n"
+ "and v4.16b, v30.16b, v0.16b\n"
+ "sshr v10.4s, v10.4s, #0x1f\n"
+ "sqadd v26.4s, v26.4s, v7.4s\n"
+ "and v5.16b, v31.16b, v0.16b\n"
+ "sshr v4.4s, v4.4s, #0x1f\n"
+ "sqadd v27.4s, v27.4s, v8.4s\n"
+ "sshr v5.4s, v5.4s, #0x1f\n"
+ "sqadd v28.4s, v28.4s, v9.4s\n"
+ "sqadd v29.4s, v29.4s, v10.4s\n"
+ "sqadd v30.4s, v30.4s, v4.4s\n"
+ "sqadd v31.4s, v31.4s, v5.4s\n"
+ "114:" // Height 4: no shift correction
+ "srshl v16.4s, v16.4s, v0.4s\n"
+ "add x19, %x[qp], %[c_offset]\n"
+ "ld1r { v4.4s }, [x19]\n"
+ "srshl v17.4s, v17.4s, v0.4s\n"
+ "add x19, %x[qp], %[minval]\n"
+ "srshl v18.4s, v18.4s, v0.4s\n"
+ "ld1r { v5.4s }, [x19]\n"
+ "add x19, %x[qp], %[maxval]\n"
+ "srshl v19.4s, v19.4s, v0.4s\n"
+ "ld1r { v6.4s }, [x19]\n"
+ "cmp x12, #0x10\n"
+ "srshl v20.4s, v20.4s, v0.4s\n"
+ "srshl v21.4s, v21.4s, v0.4s\n"
+ "srshl v22.4s, v22.4s, v0.4s\n"
+ "srshl v23.4s, v23.4s, v0.4s\n"
+ "add v16.4s, v16.4s, v4.4s\n"
+ "add v17.4s, v17.4s, v4.4s\n"
+ "add v18.4s, v18.4s, v4.4s\n"
+ "smin v16.4s, v16.4s, v6.4s\n"
+ "smin v17.4s, v17.4s, v6.4s\n"
+ "smin v18.4s, v18.4s, v6.4s\n"
+ "smax v16.4s, v16.4s, v5.4s\n"
+ "smax v17.4s, v17.4s, v5.4s\n"
+ "smax v18.4s, v18.4s, v5.4s\n"
+ "add v19.4s, v19.4s, v4.4s\n"
+ "add v20.4s, v20.4s, v4.4s\n"
+ "add v21.4s, v21.4s, v4.4s\n"
+ "smin v19.4s, v19.4s, v6.4s\n"
+ "smin v20.4s, v20.4s, v6.4s\n"
+ "smin v21.4s, v21.4s, v6.4s\n"
+ "smax v19.4s, v19.4s, v5.4s\n"
+ "smax v20.4s, v20.4s, v5.4s\n"
+ "smax v21.4s, v21.4s, v5.4s\n"
+ "add v22.4s, v22.4s, v4.4s\n"
+ "add v23.4s, v23.4s, v4.4s\n"
+ "srshl v24.4s, v24.4s, v0.4s\n"
+ "smin v22.4s, v22.4s, v6.4s\n"
+ "smin v23.4s, v23.4s, v6.4s\n"
+ "srshl v25.4s, v25.4s, v0.4s\n"
+ "smax v22.4s, v22.4s, v5.4s\n"
+ "smax v23.4s, v23.4s, v5.4s\n"
+ "add v24.4s, v24.4s, v4.4s\n"
+ "add v25.4s, v25.4s, v4.4s\n"
+ "srshl v26.4s, v26.4s, v0.4s\n"
+ "smin v24.4s, v24.4s, v6.4s\n"
+ "smin v25.4s, v25.4s, v6.4s\n"
+ "srshl v27.4s, v27.4s, v0.4s\n"
+ "smax v24.4s, v24.4s, v5.4s\n"
+ "smax v25.4s, v25.4s, v5.4s\n"
+ "add v26.4s, v26.4s, v4.4s\n"
+ "add v27.4s, v27.4s, v4.4s\n"
+ "srshl v28.4s, v28.4s, v0.4s\n"
+ "smin v26.4s, v26.4s, v6.4s\n"
+ "smin v27.4s, v27.4s, v6.4s\n"
+ "srshl v29.4s, v29.4s, v0.4s\n"
+ "smax v26.4s, v26.4s, v5.4s\n"
+ "smax v27.4s, v27.4s, v5.4s\n"
+ "add v28.4s, v28.4s, v4.4s\n"
+ "add v29.4s, v29.4s, v4.4s\n"
+ "srshl v30.4s, v30.4s, v0.4s\n"
+ "smin v28.4s, v28.4s, v6.4s\n"
+ "smin v29.4s, v29.4s, v6.4s\n"
+ "srshl v31.4s, v31.4s, v0.4s\n"
+ "smax v28.4s, v28.4s, v5.4s\n"
+ "smax v29.4s, v29.4s, v5.4s\n"
+ "add v30.4s, v30.4s, v4.4s\n"
+ "add v31.4s, v31.4s, v4.4s\n"
+ "uzp1 v16.8h, v16.8h, v17.8h\n"
+ "smin v30.4s, v30.4s, v6.4s\n"
+ "smin v31.4s, v31.4s, v6.4s\n"
+ "uzp1 v17.8h, v18.8h, v19.8h\n"
+ "smax v30.4s, v30.4s, v5.4s\n"
+ "smax v31.4s, v31.4s, v5.4s\n"
+ "uzp1 v20.8h, v20.8h, v21.8h\n"
+ "uzp1 v21.8h, v22.8h, v23.8h\n"
+ "uzp1 v24.8h, v24.8h, v25.8h\n"
+ "uzp1 v25.8h, v26.8h, v27.8h\n"
+ "uzp1 v28.8h, v28.8h, v29.8h\n"
+ "uzp1 v29.8h, v30.8h, v31.8h\n"
+ "uzp1 v16.16b, v16.16b, v17.16b\n"
+ "uzp1 v20.16b, v20.16b, v21.16b\n"
+ "uzp1 v24.16b, v24.16b, v25.16b\n"
+ "uzp1 v28.16b, v28.16b, v29.16b\n"
+ "bge 123f\n"
+ "tbz x12, #3, 118f\n"
+ "str d16, [x9], #0x8\n"
+ "str d20, [x25], #0x8\n"
+ "str d24, [x23], #0x8\n"
+ "str d28, [x21], #0x8\n"
+ "tbz x12, #2, 116f\n"
+ "st1 { v16.s }[2], [x9], #0x4\n"
+ "st1 { v20.s }[2], [x25], #0x4\n"
+ "st1 { v24.s }[2], [x23], #0x4\n"
+ "st1 { v28.s }[2], [x21], #0x4\n"
+ "tbz x12, #1, 115f\n"
+ "st1 { v16.h }[6], [x9], #0x2\n"
+ "st1 { v20.h }[6], [x25], #0x2\n"
+ "st1 { v24.h }[6], [x23], #0x2\n"
+ "st1 { v28.h }[6], [x21], #0x2\n"
+ "tbz x12, #0, 122f\n"
+ "st1 { v16.b }[14], [x9]\n"
+ "st1 { v20.b }[14], [x25]\n"
+ "st1 { v24.b }[14], [x23]\n"
+ "st1 { v28.b }[14], [x21]\n"
+ "b 122f\n"
+ "115:" // Height 4: Partial direct writeback: partial_1_12
+ "tbz x12, #0, 122f\n"
+ "st1 { v16.b }[12], [x9]\n"
+ "st1 { v20.b }[12], [x25]\n"
+ "st1 { v24.b }[12], [x23]\n"
+ "st1 { v28.b }[12], [x21]\n"
+ "b 122f\n"
+ "116:" // Height 4: Partial direct writeback: partial_2_8
+ "tbz x12, #1, 117f\n"
+ "st1 { v16.h }[4], [x9], #0x2\n"
+ "st1 { v20.h }[4], [x25], #0x2\n"
+ "st1 { v24.h }[4], [x23], #0x2\n"
+ "st1 { v28.h }[4], [x21], #0x2\n"
+ "tbz x12, #0, 122f\n"
+ "st1 { v16.b }[10], [x9]\n"
+ "st1 { v20.b }[10], [x25]\n"
+ "st1 { v24.b }[10], [x23]\n"
+ "st1 { v28.b }[10], [x21]\n"
+ "b 122f\n"
+ "117:" // Height 4: Partial direct writeback: partial_1_8
+ "tbz x12, #0, 122f\n"
+ "st1 { v16.b }[8], [x9]\n"
+ "st1 { v20.b }[8], [x25]\n"
+ "st1 { v24.b }[8], [x23]\n"
+ "st1 { v28.b }[8], [x21]\n"
+ "b 122f\n"
+ "118:" // Height 4: Partial direct writeback: partial_4_0
+ "tbz x12, #2, 120f\n"
+ "str s16, [x9], #0x4\n"
+ "str s20, [x25], #0x4\n"
+ "str s24, [x23], #0x4\n"
+ "str s28, [x21], #0x4\n"
+ "tbz x12, #1, 119f\n"
+ "st1 { v16.h }[2], [x9], #0x2\n"
+ "st1 { v20.h }[2], [x25], #0x2\n"
+ "st1 { v24.h }[2], [x23], #0x2\n"
+ "st1 { v28.h }[2], [x21], #0x2\n"
+ "tbz x12, #0, 122f\n"
+ "st1 { v16.b }[6], [x9]\n"
+ "st1 { v20.b }[6], [x25]\n"
+ "st1 { v24.b }[6], [x23]\n"
+ "st1 { v28.b }[6], [x21]\n"
+ "b 122f\n"
+ "119:" // Height 4: Partial direct writeback: partial_1_4
+ "tbz x12, #0, 122f\n"
+ "st1 { v16.b }[4], [x9]\n"
+ "st1 { v20.b }[4], [x25]\n"
+ "st1 { v24.b }[4], [x23]\n"
+ "st1 { v28.b }[4], [x21]\n"
+ "b 122f\n"
+ "120:" // Height 4: Partial direct writeback: partial_2_0
+ "tbz x12, #1, 121f\n"
+ "str h16, [x9], #0x2\n"
+ "str h20, [x25], #0x2\n"
+ "str h24, [x23], #0x2\n"
+ "str h28, [x21], #0x2\n"
+ "tbz x12, #0, 122f\n"
+ "st1 { v16.b }[2], [x9]\n"
+ "st1 { v20.b }[2], [x25]\n"
+ "st1 { v24.b }[2], [x23]\n"
+ "st1 { v28.b }[2], [x21]\n"
+ "b 122f\n"
+ "121:" // Height 4: Partial direct writeback: partial_1_0
+ "str b16, [x9, #0x0]\n"
+ "str b20, [x25, #0x0]\n"
+ "str b24, [x23, #0x0]\n"
+ "str b28, [x21, #0x0]\n"
+ "122:" // Height 4: Partial direct writeback: Done
+ "b 124f\n"
+ "123:" // Height 4: Full writeback
+ "str q16, [x9, #0x0]\n"
+ "str q20, [x25, #0x0]\n"
+ "str q24, [x23, #0x0]\n"
+ "str q28, [x21, #0x0]\n"
+ "add x9, x9, #0x10\n"
+ "add x25, x25, #0x10\n"
+ "add x23, x23, #0x10\n"
+ "add x21, x21, #0x10\n"
+ "124:" // Height 4: Writeback done
+ "subs x12, x12, #0x10\n"
+ "bgt 96b\n"
+ "subs %x[M], %x[M], #0x4\n"
+ "beq 126f\n"
+ "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "tbz %x[flags], #3, 125f\n"
+ "add x20, x20, #0x4\n"
+ "str x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "b 1b\n"
+ "125:" // Update direct input
+ "mov x19, #0x4\n"
+ "madd %x[input_ptr], x19, x20, %x[input_ptr]\n"
+ "b 1b\n"
+ "126:" // Exit
+
+ : [M] "+r" (M), [flags] "+r" (flags), [input_ptr] "+r" (input_ptr), [output_ptr] "+r" (output_ptr)
+ : [args_ptr] "r" (&ka), [b_offset] "I" (offsetof(Requantize32, b_offset)), [c_offset] "I" (offsetof(Requantize32, c_offset)), [col_bias] "r" (col_bias), [maxval] "I" (offsetof(Requantize32, maxval)), [minval] "I" (offsetof(Requantize32, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths)), [per_layer_mul] "I" (offsetof(Requantize32, per_layer_mul)), [per_layer_right_shift] "I" (offsetof(Requantize32, per_layer_right_shift)), [qp] "r" (qp)
+ : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x12", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
+ );
+}
+
+} // namespace arm_gemm
+#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_4x6.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qs_dot_6x16.hpp
similarity index 62%
copy from src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_4x6.hpp
copy to src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qs_dot_6x16.hpp
index 6f31efe..6d4f3b2 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_4x6.hpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qs_dot_6x16.hpp
@@ -10,34 +10,43 @@
* sell copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
*/
#pragma once
-
#ifdef __aarch64__
+#include "../std_transforms_fixed.hpp"
+
+#define ARGLIST \
+ unsigned int, const unsigned int *, \
+ IndirectInputArg<int8_t>, \
+ size_t, size_t, \
+ const int8_t *, \
+ IndirectOutputArg<int8_t>, \
+ const Requantize32 *, const int32_t *, unsigned int
+
namespace arm_gemm
{
// Actual kernel implementations
-void a64_smallK_hybrid_fp32_mla_4x6(const float *, int, const float *, float *, int, int, int, int, const float *, Activation, bool);
+void a64_hybrid_s8qs_dot_6x16( ARGLIST );
-class smallK_hybrid_fp32_mla_4x6
+class cls_a64_hybrid_s8qs_dot_6x16
{
public:
- typedef float operand_type;
- typedef float result_type;
+ typedef int8_t operand_type;
+ typedef int8_t result_type;
- typedef void (*kern_type)(const float *, int, const float *, float *, int, int, int, int, const float *, Activation, bool);
+ typedef void (*kern_type)( ARGLIST );
/* Kernel blocking parameters */
static constexpr unsigned int out_height()
@@ -47,12 +56,12 @@
static unsigned int out_width()
{
- return 4;
+ return 16;
}
static constexpr unsigned int k_unroll()
{
- return 1;
+ return 4;
}
static constexpr bool supports_accumulate()
@@ -60,27 +69,17 @@
return false;
}
- static constexpr bool supports_bias()
- {
- return true;
- }
-
- static constexpr bool supports_activation()
- {
- return true;
- }
-
- StdTransformsFixed<operand_type, result_type, 6, 4, 1> transforms = {};
+ StdTransformsFixed<operand_type, result_type, 6, 16, 4> transforms = {};
// Default to the generic kernel
- kern_type kernel=a64_smallK_hybrid_fp32_mla_4x6;
+ kern_type kernel=a64_hybrid_s8qs_dot_6x16;
- smallK_hybrid_fp32_mla_4x6(const CPUInfo *)
+ cls_a64_hybrid_s8qs_dot_6x16(const CPUInfo *)
{
-
}
};
} // namespace arm_gemm
+#undef ARGLIST
#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qs_dot_6x16/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qs_dot_6x16/generic.cpp
new file mode 100644
index 0000000..0e98ab8
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qs_dot_6x16/generic.cpp
@@ -0,0 +1,3613 @@
+/*
+ * Copyright (c) 2019-2020 Arm Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+#ifdef __aarch64__
+
+#include "arm_gemm.hpp"
+#include "../../utils.hpp"
+
+#include <cassert>
+#include <cstdint>
+
+namespace arm_gemm {
+
+void a64_hybrid_s8qs_dot_6x16 (
+ unsigned int num_strings, const unsigned int *string_lengths, IndirectInputArg<int8_t> A_arg,
+ size_t M, size_t N, const int8_t *B_ptr, IndirectOutputArg<int8_t> output_arg,
+ const Requantize32 *qp, const int32_t *col_bias, unsigned int col_base
+)
+{
+ struct KernelArgs {
+ const int32_t *multiplier_ptr = {};
+ const int32_t *shift_ptr = {};
+ unsigned int num_strings = {};
+ const unsigned int *string_lengths = {};
+ size_t N = {};
+ const int8_t *B_ptr = {};
+ size_t output_offset = {};
+ size_t input_initial_col = {};
+ size_t input_offset = {};
+ } ka;
+
+ unsigned long flags=0;
+ void *output_ptr;
+ void *input_ptr;
+
+ if (output_arg.is_indirect) {
+ output_ptr=(void *)(output_arg.indirect.ptr);
+ ka.output_offset=output_arg.indirect.offset;
+ flags |= 0x4;
+ } else {
+ output_ptr=(void *)(output_arg.direct.base);
+ ka.output_offset=output_arg.direct.stride;
+ }
+
+ if (A_arg.is_indirect) {
+ input_ptr=(void *)(A_arg.indirect.ptr);
+ ka.input_offset=A_arg.indirect.start_row;
+ ka.input_initial_col=A_arg.indirect.start_col;
+ flags |= 0x8;
+ } else {
+ assert(num_strings==1);
+ input_ptr=(void *)(A_arg.direct.base);
+ ka.input_offset=A_arg.direct.stride;
+ }
+ ka.num_strings = num_strings;
+ ka.string_lengths = string_lengths;
+ ka.N = N;
+ ka.B_ptr = B_ptr;
+ if (qp->per_channel_requant) {
+ flags |= 0x10;
+ ka.multiplier_ptr=qp->per_channel_muls + col_base;
+ ka.shift_ptr=qp->per_channel_right_shifts + col_base;
+ }
+ if (qp->c_offset > qp->minval) {
+ flags |= 0x20;
+ }
+ __asm__ __volatile__(
+
+ "1:" // Row loop
+ "cmp %x[M], #0x6\n"
+ "bge 141f\n"
+ "cmp %x[M], #0x4\n"
+ "bgt 113f\n"
+ "beq 85f\n"
+ "cmp %x[M], #0x2\n"
+ "bgt 57f\n"
+ "beq 29f\n"
+ "ldr x8, [%x[args_ptr], %[offsetof_multiplier_ptr]]\n"
+ "ldr x17, [%x[args_ptr], %[offsetof_shift_ptr]]\n"
+ "mov x16, %x[col_bias]\n"
+ "ldr x15, [%x[args_ptr], %[offsetof_N]]\n"
+ "ldr x14, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 2f\n"
+ "ldr x13, [%x[output_ptr], #0x0]\n"
+ "add x13, x13, x19\n"
+ "b 3f\n"
+ "2:" // Height 1: setup direct output
+ "mov x13, %x[output_ptr]\n"
+ "3:" // Height 1: Column loop
+ "movi v8.4s, #0x0\n"
+ "movi v9.4s, #0x0\n"
+ "movi v10.4s, #0x0\n"
+ "movi v11.4s, #0x0\n"
+ "4:" // Height 1: setup done
+ "mov x12, #0x0\n"
+ "5:" // Height 1: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w11, [x20, x12, LSL #0x2]\n"
+ "tbz %x[flags], #3, 6f\n"
+ "ldr x20, [%x[input_ptr], x12, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x10, [x20, #0x0]\n"
+ "cbnz x12, 7f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x10, x10, x19\n"
+ "b 7f\n"
+ "6:" // Height 1: setup direct input
+ "mov x10, %x[input_ptr]\n"
+ "7:" // Height 1: input setup done
+ "cmp x11, #0x10\n"
+ "blt 10f\n"
+ "cmp x11, #0x20\n"
+ "blt 9f\n"
+ "8:" // Height 1: Multiply loop: Main loop head
+ "ldr q0, [x10, #0x0]\n"
+ "ldr q6, [x14, #0x0]\n"
+ ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n"
+ "ldr q7, [x14, #0x10]\n"
+ "ldr q6, [x14, #0x20]\n"
+ ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n"
+ "ldr q7, [x14, #0x30]\n"
+ ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n"
+ "ldr q6, [x14, #0x40]\n"
+ "add x10, x10, #0x10\n"
+ ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "ldr q7, [x14, #0x50]\n"
+ ".inst 0x4fa0e0c8 // sdot v8.4s, v6.16b, v0.4b[1]\n"
+ "ldr q6, [x14, #0x60]\n"
+ "sub x11, x11, #0x10\n"
+ ".inst 0x4fa0e0e9 // sdot v9.4s, v7.16b, v0.4b[1]\n"
+ "ldr q7, [x14, #0x70]\n"
+ "cmp x11, #0x20\n"
+ ".inst 0x4fa0e0ca // sdot v10.4s, v6.16b, v0.4b[1]\n"
+ "ldr q6, [x14, #0x80]\n"
+ ".inst 0x4fa0e0eb // sdot v11.4s, v7.16b, v0.4b[1]\n"
+ "ldr q7, [x14, #0x90]\n"
+ ".inst 0x4f80e8c8 // sdot v8.4s, v6.16b, v0.4b[2]\n"
+ "ldr q6, [x14, #0xa0]\n"
+ ".inst 0x4f80e8e9 // sdot v9.4s, v7.16b, v0.4b[2]\n"
+ "ldr q7, [x14, #0xb0]\n"
+ ".inst 0x4f80e8ca // sdot v10.4s, v6.16b, v0.4b[2]\n"
+ "ldr q6, [x14, #0xc0]\n"
+ ".inst 0x4f80e8eb // sdot v11.4s, v7.16b, v0.4b[2]\n"
+ "ldr q7, [x14, #0xd0]\n"
+ ".inst 0x4fa0e8c8 // sdot v8.4s, v6.16b, v0.4b[3]\n"
+ "ldr q6, [x14, #0xe0]\n"
+ ".inst 0x4fa0e8e9 // sdot v9.4s, v7.16b, v0.4b[3]\n"
+ "ldr q7, [x14, #0xf0]\n"
+ "add x14, x14, #0x100\n"
+ ".inst 0x4fa0e8ca // sdot v10.4s, v6.16b, v0.4b[3]\n"
+ ".inst 0x4fa0e8eb // sdot v11.4s, v7.16b, v0.4b[3]\n"
+ "bge 8b\n"
+ "9:" // Height 1: Multiply loop: Single iteration only
+ "sub x11, x11, #0x10\n"
+ "ldr q0, [x10, #0x0]\n"
+ "ldr q6, [x14, #0x0]\n"
+ ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n"
+ "ldr q7, [x14, #0x10]\n"
+ "ldr q6, [x14, #0x20]\n"
+ ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n"
+ "ldr q7, [x14, #0x30]\n"
+ ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n"
+ "ldr q6, [x14, #0x40]\n"
+ "add x10, x10, #0x10\n"
+ ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "ldr q7, [x14, #0x50]\n"
+ ".inst 0x4fa0e0c8 // sdot v8.4s, v6.16b, v0.4b[1]\n"
+ "ldr q6, [x14, #0x60]\n"
+ ".inst 0x4fa0e0e9 // sdot v9.4s, v7.16b, v0.4b[1]\n"
+ "ldr q7, [x14, #0x70]\n"
+ ".inst 0x4fa0e0ca // sdot v10.4s, v6.16b, v0.4b[1]\n"
+ "ldr q6, [x14, #0x80]\n"
+ ".inst 0x4fa0e0eb // sdot v11.4s, v7.16b, v0.4b[1]\n"
+ "ldr q7, [x14, #0x90]\n"
+ ".inst 0x4f80e8c8 // sdot v8.4s, v6.16b, v0.4b[2]\n"
+ "ldr q6, [x14, #0xa0]\n"
+ ".inst 0x4f80e8e9 // sdot v9.4s, v7.16b, v0.4b[2]\n"
+ "ldr q7, [x14, #0xb0]\n"
+ ".inst 0x4f80e8ca // sdot v10.4s, v6.16b, v0.4b[2]\n"
+ "ldr q6, [x14, #0xc0]\n"
+ ".inst 0x4f80e8eb // sdot v11.4s, v7.16b, v0.4b[2]\n"
+ "ldr q7, [x14, #0xd0]\n"
+ ".inst 0x4fa0e8c8 // sdot v8.4s, v6.16b, v0.4b[3]\n"
+ "ldr q6, [x14, #0xe0]\n"
+ ".inst 0x4fa0e8e9 // sdot v9.4s, v7.16b, v0.4b[3]\n"
+ "ldr q7, [x14, #0xf0]\n"
+ "add x14, x14, #0x100\n"
+ ".inst 0x4fa0e8ca // sdot v10.4s, v6.16b, v0.4b[3]\n"
+ ".inst 0x4fa0e8eb // sdot v11.4s, v7.16b, v0.4b[3]\n"
+ "10:" // Height 1: Multiply loop: Main loop skip
+ "cbz x11, 15f\n"
+ "cmp x11, #0x4\n"
+ "blt 12f\n"
+ "11:" // Height 1: Multiply loop: Odd block loop
+ "ldr s0, [x10], #0x4\n"
+ "ldr q6, [x14, #0x0]\n"
+ ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n"
+ "ldr q7, [x14, #0x10]\n"
+ "ldr q6, [x14, #0x20]\n"
+ ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n"
+ "ldr q7, [x14, #0x30]\n"
+ ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n"
+ "sub x11, x11, #0x4\n"
+ "add x14, x14, #0x40\n"
+ ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n"
+ "cmp x11, #0x4\n"
+ "bge 11b\n"
+ "cbz x11, 15f\n"
+ "12:" // Height 1: Multiply loop: Skip odd blocks
+ "tbz x11, #1, 13f\n"
+ "ldr h0, [x10], #0x2\n"
+ "tbz x11, #0, 14f\n"
+ "ld1 { v0.b }[2], [x10]\n"
+ "b 14f\n"
+ "13:" // Height 1: Multiply loop: Ragged operand read: partial_1_0
+ "ldr b0, [x10, #0x0]\n"
+ "14:" // Height 1: Multiply loop: Ragged operand read: Done
+ "ldr q6, [x14, #0x0]\n"
+ ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n"
+ "ldr q7, [x14, #0x10]\n"
+ "ldr q6, [x14, #0x20]\n"
+ ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n"
+ "ldr q7, [x14, #0x30]\n"
+ ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n"
+ "add x14, x14, #0x40\n"
+ ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n"
+ "15:" // Height 1: Multiply loop: No odd multiplies
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "add x12, x12, #0x1\n"
+ "cmp x12, x19\n"
+ "bne 5b\n"
+ "prfm pstl1keep, [x13, #0x0]\n"
+ "ldr q0, [x16, #0x0]\n"
+ "add v8.4s, v8.4s, v0.4s\n"
+ "ldr q1, [x16, #0x10]\n"
+ "ldr q2, [x16, #0x20]\n"
+ "add v9.4s, v9.4s, v1.4s\n"
+ "ldr q3, [x16, #0x30]\n"
+ "add v10.4s, v10.4s, v2.4s\n"
+ "add x16, x16, #0x40\n"
+ "add v11.4s, v11.4s, v3.4s\n"
+ "tbz %x[flags], #4, 16f\n"
+ "ldr q0, [x17, #0x0]\n"
+ "ldr q4, [x8, #0x0]\n"
+ "ldr q1, [x17, #0x10]\n"
+ "ldr q5, [x8, #0x10]\n"
+ "ldr q2, [x17, #0x20]\n"
+ "ldr q6, [x8, #0x20]\n"
+ "ldr q3, [x17, #0x30]\n"
+ "ldr q7, [x8, #0x30]\n"
+ "add x17, x17, #0x40\n"
+ "add x8, x8, #0x40\n"
+ "b 17f\n"
+ "16:" // Height 1: per layer parameters
+ "add x19, %x[qp], %[per_layer_right_shift]\n"
+ "ld1r { v0.4s }, [x19]\n"
+ "mov v1.16b, v0.16b\n"
+ "add x19, %x[qp], %[per_layer_mul]\n"
+ "ld1r { v4.4s }, [x19]\n"
+ "mov v2.16b, v0.16b\n"
+ "mov v3.16b, v0.16b\n"
+ "mov v5.16b, v4.16b\n"
+ "mov v6.16b, v4.16b\n"
+ "mov v7.16b, v4.16b\n"
+ "17:" // Height 1: parameters loaded
+ "sqrdmulh v8.4s, v8.4s, v4.4s\n"
+ "sqrdmulh v9.4s, v9.4s, v5.4s\n"
+ "sqrdmulh v10.4s, v10.4s, v6.4s\n"
+ "sqrdmulh v11.4s, v11.4s, v7.4s\n"
+ "tbz %x[flags], #5, 18f\n"
+ "and v4.16b, v8.16b, v0.16b\n"
+ "sshr v4.4s, v4.4s, #0x1f\n"
+ "and v5.16b, v9.16b, v1.16b\n"
+ "and v6.16b, v10.16b, v2.16b\n"
+ "sshr v5.4s, v5.4s, #0x1f\n"
+ "and v7.16b, v11.16b, v3.16b\n"
+ "sshr v6.4s, v6.4s, #0x1f\n"
+ "sqadd v8.4s, v8.4s, v4.4s\n"
+ "sshr v7.4s, v7.4s, #0x1f\n"
+ "sqadd v9.4s, v9.4s, v5.4s\n"
+ "sqadd v10.4s, v10.4s, v6.4s\n"
+ "sqadd v11.4s, v11.4s, v7.4s\n"
+ "18:" // Height 1: no shift correction
+ "srshl v8.4s, v8.4s, v0.4s\n"
+ "add x19, %x[qp], %[c_offset]\n"
+ "ld1r { v4.4s }, [x19]\n"
+ "srshl v9.4s, v9.4s, v1.4s\n"
+ "add x19, %x[qp], %[minval]\n"
+ "srshl v10.4s, v10.4s, v2.4s\n"
+ "ld1r { v5.4s }, [x19]\n"
+ "add x19, %x[qp], %[maxval]\n"
+ "srshl v11.4s, v11.4s, v3.4s\n"
+ "ld1r { v6.4s }, [x19]\n"
+ "cmp x15, #0x10\n"
+ "add v8.4s, v8.4s, v4.4s\n"
+ "add v9.4s, v9.4s, v4.4s\n"
+ "add v10.4s, v10.4s, v4.4s\n"
+ "add v11.4s, v11.4s, v4.4s\n"
+ "smin v8.4s, v8.4s, v6.4s\n"
+ "smin v9.4s, v9.4s, v6.4s\n"
+ "smin v10.4s, v10.4s, v6.4s\n"
+ "smax v8.4s, v8.4s, v5.4s\n"
+ "smax v9.4s, v9.4s, v5.4s\n"
+ "smax v10.4s, v10.4s, v5.4s\n"
+ "smin v11.4s, v11.4s, v6.4s\n"
+ "uzp1 v8.8h, v8.8h, v9.8h\n"
+ "smax v11.4s, v11.4s, v5.4s\n"
+ "uzp1 v9.8h, v10.8h, v11.8h\n"
+ "uzp1 v8.16b, v8.16b, v9.16b\n"
+ "bge 27f\n"
+ "tbz x15, #3, 22f\n"
+ "str d8, [x13], #0x8\n"
+ "tbz x15, #2, 20f\n"
+ "st1 { v8.s }[2], [x13], #0x4\n"
+ "tbz x15, #1, 19f\n"
+ "st1 { v8.h }[6], [x13], #0x2\n"
+ "tbz x15, #0, 26f\n"
+ "st1 { v8.b }[14], [x13]\n"
+ "b 26f\n"
+ "19:" // Height 1: Partial direct writeback: partial_1_12
+ "tbz x15, #0, 26f\n"
+ "st1 { v8.b }[12], [x13]\n"
+ "b 26f\n"
+ "20:" // Height 1: Partial direct writeback: partial_2_8
+ "tbz x15, #1, 21f\n"
+ "st1 { v8.h }[4], [x13], #0x2\n"
+ "tbz x15, #0, 26f\n"
+ "st1 { v8.b }[10], [x13]\n"
+ "b 26f\n"
+ "21:" // Height 1: Partial direct writeback: partial_1_8
+ "tbz x15, #0, 26f\n"
+ "st1 { v8.b }[8], [x13]\n"
+ "b 26f\n"
+ "22:" // Height 1: Partial direct writeback: partial_4_0
+ "tbz x15, #2, 24f\n"
+ "str s8, [x13], #0x4\n"
+ "tbz x15, #1, 23f\n"
+ "st1 { v8.h }[2], [x13], #0x2\n"
+ "tbz x15, #0, 26f\n"
+ "st1 { v8.b }[6], [x13]\n"
+ "b 26f\n"
+ "23:" // Height 1: Partial direct writeback: partial_1_4
+ "tbz x15, #0, 26f\n"
+ "st1 { v8.b }[4], [x13]\n"
+ "b 26f\n"
+ "24:" // Height 1: Partial direct writeback: partial_2_0
+ "tbz x15, #1, 25f\n"
+ "str h8, [x13], #0x2\n"
+ "tbz x15, #0, 26f\n"
+ "st1 { v8.b }[2], [x13]\n"
+ "b 26f\n"
+ "25:" // Height 1: Partial direct writeback: partial_1_0
+ "str b8, [x13, #0x0]\n"
+ "26:" // Height 1: Partial direct writeback: Done
+ "b 28f\n"
+ "27:" // Height 1: Full writeback
+ "str q8, [x13, #0x0]\n"
+ "add x13, x13, #0x10\n"
+ "28:" // Height 1: Writeback done
+ "subs x15, x15, #0x10\n"
+ "bgt 3b\n"
+ "b 170f\n"
+ "29:" // Height 2
+ "ldr x8, [%x[args_ptr], %[offsetof_multiplier_ptr]]\n"
+ "mov x16, %x[col_bias]\n"
+ "ldr x17, [%x[args_ptr], %[offsetof_shift_ptr]]\n"
+ "ldr x15, [%x[args_ptr], %[offsetof_N]]\n"
+ "ldr x14, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 30f\n"
+ "ldr x13, [%x[output_ptr], #0x0]\n"
+ "add x13, x13, x19\n"
+ "ldr x9, [%x[output_ptr], #0x8]\n"
+ "add x9, x9, x19\n"
+ "b 31f\n"
+ "30:" // Height 2: setup direct output
+ "mov x13, %x[output_ptr]\n"
+ "add x9, x13, x19\n"
+ "31:" // Height 2: Column loop
+ "movi v8.4s, #0x0\n"
+ "movi v9.4s, #0x0\n"
+ "movi v10.4s, #0x0\n"
+ "movi v11.4s, #0x0\n"
+ "movi v12.4s, #0x0\n"
+ "movi v13.4s, #0x0\n"
+ "movi v14.4s, #0x0\n"
+ "movi v15.4s, #0x0\n"
+ "32:" // Height 2: setup done
+ "mov x12, #0x0\n"
+ "33:" // Height 2: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w11, [x20, x12, LSL #0x2]\n"
+ "tbz %x[flags], #3, 34f\n"
+ "ldr x20, [%x[input_ptr], x12, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x10, [x20, #0x0]\n"
+ "ldr x28, [x20, #0x8]\n"
+ "cbnz x12, 35f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x10, x10, x19\n"
+ "add x28, x28, x19\n"
+ "b 35f\n"
+ "34:" // Height 2: setup direct input
+ "mov x10, %x[input_ptr]\n"
+ "add x28, x10, x19\n"
+ "35:" // Height 2: input setup done
+ "cmp x11, #0x10\n"
+ "blt 38f\n"
+ "cmp x11, #0x20\n"
+ "blt 37f\n"
+ "36:" // Height 2: Multiply loop: Main loop head
+ "ldr q0, [x10, #0x0]\n"
+ "ldr q1, [x28, #0x0]\n"
+ "ldr q6, [x14, #0x0]\n"
+ ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n"
+ "ldr q7, [x14, #0x10]\n"
+ ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n"
+ "ldr q6, [x14, #0x20]\n"
+ "add x10, x10, #0x10\n"
+ ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x28, x28, #0x10\n"
+ ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "ldr q7, [x14, #0x30]\n"
+ ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n"
+ "sub x11, x11, #0x10\n"
+ ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n"
+ "ldr q6, [x14, #0x40]\n"
+ "cmp x11, #0x20\n"
+ ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n"
+ "ldr q7, [x14, #0x50]\n"
+ ".inst 0x4fa0e0c8 // sdot v8.4s, v6.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e0cc // sdot v12.4s, v6.16b, v1.4b[1]\n"
+ "ldr q6, [x14, #0x60]\n"
+ ".inst 0x4fa0e0e9 // sdot v9.4s, v7.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e0ed // sdot v13.4s, v7.16b, v1.4b[1]\n"
+ "ldr q7, [x14, #0x70]\n"
+ ".inst 0x4fa0e0ca // sdot v10.4s, v6.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e0ce // sdot v14.4s, v6.16b, v1.4b[1]\n"
+ "ldr q6, [x14, #0x80]\n"
+ ".inst 0x4fa0e0eb // sdot v11.4s, v7.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e0ef // sdot v15.4s, v7.16b, v1.4b[1]\n"
+ "ldr q7, [x14, #0x90]\n"
+ ".inst 0x4f80e8c8 // sdot v8.4s, v6.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8cc // sdot v12.4s, v6.16b, v1.4b[2]\n"
+ "ldr q6, [x14, #0xa0]\n"
+ ".inst 0x4f80e8e9 // sdot v9.4s, v7.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8ed // sdot v13.4s, v7.16b, v1.4b[2]\n"
+ "ldr q7, [x14, #0xb0]\n"
+ ".inst 0x4f80e8ca // sdot v10.4s, v6.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8ce // sdot v14.4s, v6.16b, v1.4b[2]\n"
+ "ldr q6, [x14, #0xc0]\n"
+ ".inst 0x4f80e8eb // sdot v11.4s, v7.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8ef // sdot v15.4s, v7.16b, v1.4b[2]\n"
+ "ldr q7, [x14, #0xd0]\n"
+ ".inst 0x4fa0e8c8 // sdot v8.4s, v6.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e8cc // sdot v12.4s, v6.16b, v1.4b[3]\n"
+ "ldr q6, [x14, #0xe0]\n"
+ ".inst 0x4fa0e8e9 // sdot v9.4s, v7.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e8ed // sdot v13.4s, v7.16b, v1.4b[3]\n"
+ "ldr q7, [x14, #0xf0]\n"
+ "add x14, x14, #0x100\n"
+ ".inst 0x4fa0e8ca // sdot v10.4s, v6.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e8ce // sdot v14.4s, v6.16b, v1.4b[3]\n"
+ ".inst 0x4fa0e8eb // sdot v11.4s, v7.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e8ef // sdot v15.4s, v7.16b, v1.4b[3]\n"
+ "bge 36b\n"
+ "37:" // Height 2: Multiply loop: Single iteration only
+ "sub x11, x11, #0x10\n"
+ "ldr q0, [x10, #0x0]\n"
+ "ldr q1, [x28, #0x0]\n"
+ "ldr q6, [x14, #0x0]\n"
+ ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n"
+ "ldr q7, [x14, #0x10]\n"
+ ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n"
+ "ldr q6, [x14, #0x20]\n"
+ "add x10, x10, #0x10\n"
+ ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x28, x28, #0x10\n"
+ ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "ldr q7, [x14, #0x30]\n"
+ ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n"
+ ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n"
+ "ldr q6, [x14, #0x40]\n"
+ ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n"
+ "ldr q7, [x14, #0x50]\n"
+ ".inst 0x4fa0e0c8 // sdot v8.4s, v6.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e0cc // sdot v12.4s, v6.16b, v1.4b[1]\n"
+ "ldr q6, [x14, #0x60]\n"
+ ".inst 0x4fa0e0e9 // sdot v9.4s, v7.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e0ed // sdot v13.4s, v7.16b, v1.4b[1]\n"
+ "ldr q7, [x14, #0x70]\n"
+ ".inst 0x4fa0e0ca // sdot v10.4s, v6.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e0ce // sdot v14.4s, v6.16b, v1.4b[1]\n"
+ "ldr q6, [x14, #0x80]\n"
+ ".inst 0x4fa0e0eb // sdot v11.4s, v7.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e0ef // sdot v15.4s, v7.16b, v1.4b[1]\n"
+ "ldr q7, [x14, #0x90]\n"
+ ".inst 0x4f80e8c8 // sdot v8.4s, v6.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8cc // sdot v12.4s, v6.16b, v1.4b[2]\n"
+ "ldr q6, [x14, #0xa0]\n"
+ ".inst 0x4f80e8e9 // sdot v9.4s, v7.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8ed // sdot v13.4s, v7.16b, v1.4b[2]\n"
+ "ldr q7, [x14, #0xb0]\n"
+ ".inst 0x4f80e8ca // sdot v10.4s, v6.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8ce // sdot v14.4s, v6.16b, v1.4b[2]\n"
+ "ldr q6, [x14, #0xc0]\n"
+ ".inst 0x4f80e8eb // sdot v11.4s, v7.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8ef // sdot v15.4s, v7.16b, v1.4b[2]\n"
+ "ldr q7, [x14, #0xd0]\n"
+ ".inst 0x4fa0e8c8 // sdot v8.4s, v6.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e8cc // sdot v12.4s, v6.16b, v1.4b[3]\n"
+ "ldr q6, [x14, #0xe0]\n"
+ ".inst 0x4fa0e8e9 // sdot v9.4s, v7.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e8ed // sdot v13.4s, v7.16b, v1.4b[3]\n"
+ "ldr q7, [x14, #0xf0]\n"
+ "add x14, x14, #0x100\n"
+ ".inst 0x4fa0e8ca // sdot v10.4s, v6.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e8ce // sdot v14.4s, v6.16b, v1.4b[3]\n"
+ ".inst 0x4fa0e8eb // sdot v11.4s, v7.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e8ef // sdot v15.4s, v7.16b, v1.4b[3]\n"
+ "38:" // Height 2: Multiply loop: Main loop skip
+ "cbz x11, 43f\n"
+ "cmp x11, #0x4\n"
+ "blt 40f\n"
+ "39:" // Height 2: Multiply loop: Odd block loop
+ "ldr s0, [x10], #0x4\n"
+ "ldr s1, [x28], #0x4\n"
+ "ldr q6, [x14, #0x0]\n"
+ ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n"
+ "ldr q7, [x14, #0x10]\n"
+ ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n"
+ "ldr q6, [x14, #0x20]\n"
+ "sub x11, x11, #0x4\n"
+ ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n"
+ "cmp x11, #0x4\n"
+ ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n"
+ "ldr q7, [x14, #0x30]\n"
+ ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n"
+ "add x14, x14, #0x40\n"
+ ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n"
+ ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n"
+ "bge 39b\n"
+ "cbz x11, 43f\n"
+ "40:" // Height 2: Multiply loop: Skip odd blocks
+ "tbz x11, #1, 41f\n"
+ "ldr h0, [x10], #0x2\n"
+ "ldr h1, [x28], #0x2\n"
+ "tbz x11, #0, 42f\n"
+ "ld1 { v0.b }[2], [x10]\n"
+ "ld1 { v1.b }[2], [x28]\n"
+ "b 42f\n"
+ "41:" // Height 2: Multiply loop: Ragged operand read: partial_1_0
+ "ldr b0, [x10, #0x0]\n"
+ "ldr b1, [x28, #0x0]\n"
+ "42:" // Height 2: Multiply loop: Ragged operand read: Done
+ "ldr q6, [x14, #0x0]\n"
+ ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n"
+ "ldr q7, [x14, #0x10]\n"
+ ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n"
+ "ldr q6, [x14, #0x20]\n"
+ ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n"
+ "ldr q7, [x14, #0x30]\n"
+ ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n"
+ "add x14, x14, #0x40\n"
+ ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n"
+ ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n"
+ "43:" // Height 2: Multiply loop: No odd multiplies
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "add x12, x12, #0x1\n"
+ "cmp x12, x19\n"
+ "bne 33b\n"
+ "prfm pstl1keep, [x13, #0x0]\n"
+ "prfm pstl1keep, [x9, #0x0]\n"
+ "ldr q0, [x16, #0x0]\n"
+ "add v8.4s, v8.4s, v0.4s\n"
+ "ldr q1, [x16, #0x10]\n"
+ "add v12.4s, v12.4s, v0.4s\n"
+ "ldr q2, [x16, #0x20]\n"
+ "ldr q3, [x16, #0x30]\n"
+ "add v9.4s, v9.4s, v1.4s\n"
+ "add x16, x16, #0x40\n"
+ "add v13.4s, v13.4s, v1.4s\n"
+ "add v10.4s, v10.4s, v2.4s\n"
+ "add v11.4s, v11.4s, v3.4s\n"
+ "add v14.4s, v14.4s, v2.4s\n"
+ "add v15.4s, v15.4s, v3.4s\n"
+ "tbz %x[flags], #4, 44f\n"
+ "ldr q0, [x17, #0x0]\n"
+ "ldr q4, [x8, #0x0]\n"
+ "ldr q1, [x17, #0x10]\n"
+ "ldr q5, [x8, #0x10]\n"
+ "ldr q2, [x17, #0x20]\n"
+ "ldr q6, [x8, #0x20]\n"
+ "ldr q3, [x17, #0x30]\n"
+ "ldr q7, [x8, #0x30]\n"
+ "add x17, x17, #0x40\n"
+ "add x8, x8, #0x40\n"
+ "b 45f\n"
+ "44:" // Height 2: per layer parameters
+ "add x19, %x[qp], %[per_layer_right_shift]\n"
+ "ld1r { v0.4s }, [x19]\n"
+ "mov v1.16b, v0.16b\n"
+ "add x19, %x[qp], %[per_layer_mul]\n"
+ "ld1r { v4.4s }, [x19]\n"
+ "mov v2.16b, v0.16b\n"
+ "mov v3.16b, v0.16b\n"
+ "mov v5.16b, v4.16b\n"
+ "mov v6.16b, v4.16b\n"
+ "mov v7.16b, v4.16b\n"
+ "45:" // Height 2: parameters loaded
+ "sqrdmulh v8.4s, v8.4s, v4.4s\n"
+ "sqrdmulh v9.4s, v9.4s, v5.4s\n"
+ "sqrdmulh v10.4s, v10.4s, v6.4s\n"
+ "sqrdmulh v11.4s, v11.4s, v7.4s\n"
+ "sqrdmulh v12.4s, v12.4s, v4.4s\n"
+ "sqrdmulh v13.4s, v13.4s, v5.4s\n"
+ "sqrdmulh v14.4s, v14.4s, v6.4s\n"
+ "sqrdmulh v15.4s, v15.4s, v7.4s\n"
+ "tbz %x[flags], #5, 46f\n"
+ "and v4.16b, v8.16b, v0.16b\n"
+ "sshr v4.4s, v4.4s, #0x1f\n"
+ "and v5.16b, v9.16b, v1.16b\n"
+ "and v6.16b, v10.16b, v2.16b\n"
+ "sshr v5.4s, v5.4s, #0x1f\n"
+ "and v7.16b, v11.16b, v3.16b\n"
+ "sshr v6.4s, v6.4s, #0x1f\n"
+ "sqadd v8.4s, v8.4s, v4.4s\n"
+ "sshr v7.4s, v7.4s, #0x1f\n"
+ "and v4.16b, v12.16b, v0.16b\n"
+ "sqadd v9.4s, v9.4s, v5.4s\n"
+ "sshr v4.4s, v4.4s, #0x1f\n"
+ "sqadd v10.4s, v10.4s, v6.4s\n"
+ "and v5.16b, v13.16b, v1.16b\n"
+ "sshr v5.4s, v5.4s, #0x1f\n"
+ "sqadd v11.4s, v11.4s, v7.4s\n"
+ "and v6.16b, v14.16b, v2.16b\n"
+ "sshr v6.4s, v6.4s, #0x1f\n"
+ "sqadd v12.4s, v12.4s, v4.4s\n"
+ "and v7.16b, v15.16b, v3.16b\n"
+ "sshr v7.4s, v7.4s, #0x1f\n"
+ "sqadd v13.4s, v13.4s, v5.4s\n"
+ "sqadd v14.4s, v14.4s, v6.4s\n"
+ "sqadd v15.4s, v15.4s, v7.4s\n"
+ "46:" // Height 2: no shift correction
+ "srshl v8.4s, v8.4s, v0.4s\n"
+ "add x19, %x[qp], %[c_offset]\n"
+ "ld1r { v4.4s }, [x19]\n"
+ "srshl v9.4s, v9.4s, v1.4s\n"
+ "add x19, %x[qp], %[minval]\n"
+ "srshl v10.4s, v10.4s, v2.4s\n"
+ "ld1r { v5.4s }, [x19]\n"
+ "add x19, %x[qp], %[maxval]\n"
+ "srshl v11.4s, v11.4s, v3.4s\n"
+ "ld1r { v6.4s }, [x19]\n"
+ "cmp x15, #0x10\n"
+ "srshl v12.4s, v12.4s, v0.4s\n"
+ "srshl v13.4s, v13.4s, v1.4s\n"
+ "srshl v14.4s, v14.4s, v2.4s\n"
+ "srshl v15.4s, v15.4s, v3.4s\n"
+ "add v8.4s, v8.4s, v4.4s\n"
+ "add v9.4s, v9.4s, v4.4s\n"
+ "add v10.4s, v10.4s, v4.4s\n"
+ "smin v8.4s, v8.4s, v6.4s\n"
+ "smin v9.4s, v9.4s, v6.4s\n"
+ "smin v10.4s, v10.4s, v6.4s\n"
+ "smax v8.4s, v8.4s, v5.4s\n"
+ "smax v9.4s, v9.4s, v5.4s\n"
+ "smax v10.4s, v10.4s, v5.4s\n"
+ "add v11.4s, v11.4s, v4.4s\n"
+ "add v12.4s, v12.4s, v4.4s\n"
+ "add v13.4s, v13.4s, v4.4s\n"
+ "smin v11.4s, v11.4s, v6.4s\n"
+ "smin v12.4s, v12.4s, v6.4s\n"
+ "smin v13.4s, v13.4s, v6.4s\n"
+ "smax v11.4s, v11.4s, v5.4s\n"
+ "smax v12.4s, v12.4s, v5.4s\n"
+ "smax v13.4s, v13.4s, v5.4s\n"
+ "add v14.4s, v14.4s, v4.4s\n"
+ "add v15.4s, v15.4s, v4.4s\n"
+ "uzp1 v8.8h, v8.8h, v9.8h\n"
+ "smin v14.4s, v14.4s, v6.4s\n"
+ "smin v15.4s, v15.4s, v6.4s\n"
+ "uzp1 v9.8h, v10.8h, v11.8h\n"
+ "smax v14.4s, v14.4s, v5.4s\n"
+ "smax v15.4s, v15.4s, v5.4s\n"
+ "uzp1 v12.8h, v12.8h, v13.8h\n"
+ "uzp1 v8.16b, v8.16b, v9.16b\n"
+ "uzp1 v13.8h, v14.8h, v15.8h\n"
+ "uzp1 v12.16b, v12.16b, v13.16b\n"
+ "bge 55f\n"
+ "tbz x15, #3, 50f\n"
+ "str d8, [x13], #0x8\n"
+ "str d12, [x9], #0x8\n"
+ "tbz x15, #2, 48f\n"
+ "st1 { v8.s }[2], [x13], #0x4\n"
+ "st1 { v12.s }[2], [x9], #0x4\n"
+ "tbz x15, #1, 47f\n"
+ "st1 { v8.h }[6], [x13], #0x2\n"
+ "st1 { v12.h }[6], [x9], #0x2\n"
+ "tbz x15, #0, 54f\n"
+ "st1 { v8.b }[14], [x13]\n"
+ "st1 { v12.b }[14], [x9]\n"
+ "b 54f\n"
+ "47:" // Height 2: Partial direct writeback: partial_1_12
+ "tbz x15, #0, 54f\n"
+ "st1 { v8.b }[12], [x13]\n"
+ "st1 { v12.b }[12], [x9]\n"
+ "b 54f\n"
+ "48:" // Height 2: Partial direct writeback: partial_2_8
+ "tbz x15, #1, 49f\n"
+ "st1 { v8.h }[4], [x13], #0x2\n"
+ "st1 { v12.h }[4], [x9], #0x2\n"
+ "tbz x15, #0, 54f\n"
+ "st1 { v8.b }[10], [x13]\n"
+ "st1 { v12.b }[10], [x9]\n"
+ "b 54f\n"
+ "49:" // Height 2: Partial direct writeback: partial_1_8
+ "tbz x15, #0, 54f\n"
+ "st1 { v8.b }[8], [x13]\n"
+ "st1 { v12.b }[8], [x9]\n"
+ "b 54f\n"
+ "50:" // Height 2: Partial direct writeback: partial_4_0
+ "tbz x15, #2, 52f\n"
+ "str s8, [x13], #0x4\n"
+ "str s12, [x9], #0x4\n"
+ "tbz x15, #1, 51f\n"
+ "st1 { v8.h }[2], [x13], #0x2\n"
+ "st1 { v12.h }[2], [x9], #0x2\n"
+ "tbz x15, #0, 54f\n"
+ "st1 { v8.b }[6], [x13]\n"
+ "st1 { v12.b }[6], [x9]\n"
+ "b 54f\n"
+ "51:" // Height 2: Partial direct writeback: partial_1_4
+ "tbz x15, #0, 54f\n"
+ "st1 { v8.b }[4], [x13]\n"
+ "st1 { v12.b }[4], [x9]\n"
+ "b 54f\n"
+ "52:" // Height 2: Partial direct writeback: partial_2_0
+ "tbz x15, #1, 53f\n"
+ "str h8, [x13], #0x2\n"
+ "str h12, [x9], #0x2\n"
+ "tbz x15, #0, 54f\n"
+ "st1 { v8.b }[2], [x13]\n"
+ "st1 { v12.b }[2], [x9]\n"
+ "b 54f\n"
+ "53:" // Height 2: Partial direct writeback: partial_1_0
+ "str b8, [x13, #0x0]\n"
+ "str b12, [x9, #0x0]\n"
+ "54:" // Height 2: Partial direct writeback: Done
+ "b 56f\n"
+ "55:" // Height 2: Full writeback
+ "str q8, [x13, #0x0]\n"
+ "str q12, [x9, #0x0]\n"
+ "add x13, x13, #0x10\n"
+ "add x9, x9, #0x10\n"
+ "56:" // Height 2: Writeback done
+ "subs x15, x15, #0x10\n"
+ "bgt 31b\n"
+ "b 170f\n"
+ "57:" // Height 3
+ "ldr x8, [%x[args_ptr], %[offsetof_multiplier_ptr]]\n"
+ "mov x16, %x[col_bias]\n"
+ "ldr x17, [%x[args_ptr], %[offsetof_shift_ptr]]\n"
+ "ldr x15, [%x[args_ptr], %[offsetof_N]]\n"
+ "ldr x14, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 58f\n"
+ "ldr x13, [%x[output_ptr], #0x0]\n"
+ "add x13, x13, x19\n"
+ "ldr x9, [%x[output_ptr], #0x8]\n"
+ "ldr x27, [%x[output_ptr], #0x10]\n"
+ "add x9, x9, x19\n"
+ "add x27, x27, x19\n"
+ "b 59f\n"
+ "58:" // Height 3: setup direct output
+ "mov x13, %x[output_ptr]\n"
+ "add x9, x13, x19\n"
+ "add x27, x9, x19\n"
+ "59:" // Height 3: Column loop
+ "movi v8.4s, #0x0\n"
+ "movi v9.4s, #0x0\n"
+ "movi v10.4s, #0x0\n"
+ "movi v11.4s, #0x0\n"
+ "movi v12.4s, #0x0\n"
+ "movi v13.4s, #0x0\n"
+ "movi v14.4s, #0x0\n"
+ "movi v15.4s, #0x0\n"
+ "movi v16.4s, #0x0\n"
+ "movi v17.4s, #0x0\n"
+ "movi v18.4s, #0x0\n"
+ "movi v19.4s, #0x0\n"
+ "60:" // Height 3: setup done
+ "mov x12, #0x0\n"
+ "61:" // Height 3: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w11, [x20, x12, LSL #0x2]\n"
+ "tbz %x[flags], #3, 62f\n"
+ "ldr x20, [%x[input_ptr], x12, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x10, [x20, #0x0]\n"
+ "ldr x28, [x20, #0x8]\n"
+ "ldr x26, [x20, #0x10]\n"
+ "cbnz x12, 63f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x10, x10, x19\n"
+ "add x28, x28, x19\n"
+ "add x26, x26, x19\n"
+ "b 63f\n"
+ "62:" // Height 3: setup direct input
+ "mov x10, %x[input_ptr]\n"
+ "add x28, x10, x19\n"
+ "add x26, x28, x19\n"
+ "63:" // Height 3: input setup done
+ "cmp x11, #0x10\n"
+ "blt 66f\n"
+ "cmp x11, #0x20\n"
+ "blt 65f\n"
+ "64:" // Height 3: Multiply loop: Main loop head
+ "ldr q0, [x10, #0x0]\n"
+ "ldr q1, [x28, #0x0]\n"
+ "ldr q2, [x26, #0x0]\n"
+ "ldr q6, [x14, #0x0]\n"
+ ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n"
+ "ldr q7, [x14, #0x10]\n"
+ ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n"
+ "add x10, x10, #0x10\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n"
+ "ldr q6, [x14, #0x20]\n"
+ ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n"
+ "add x28, x28, #0x10\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n"
+ "add x26, x26, #0x10\n"
+ ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "ldr q7, [x14, #0x30]\n"
+ ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n"
+ "sub x11, x11, #0x10\n"
+ ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n"
+ "cmp x11, #0x20\n"
+ ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n"
+ "ldr q6, [x14, #0x40]\n"
+ ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n"
+ "ldr q7, [x14, #0x50]\n"
+ ".inst 0x4fa0e0c8 // sdot v8.4s, v6.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e0cc // sdot v12.4s, v6.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e0d0 // sdot v16.4s, v6.16b, v2.4b[1]\n"
+ "ldr q6, [x14, #0x60]\n"
+ ".inst 0x4fa0e0e9 // sdot v9.4s, v7.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e0ed // sdot v13.4s, v7.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e0f1 // sdot v17.4s, v7.16b, v2.4b[1]\n"
+ "ldr q7, [x14, #0x70]\n"
+ ".inst 0x4fa0e0ca // sdot v10.4s, v6.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e0ce // sdot v14.4s, v6.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e0d2 // sdot v18.4s, v6.16b, v2.4b[1]\n"
+ "ldr q6, [x14, #0x80]\n"
+ ".inst 0x4fa0e0eb // sdot v11.4s, v7.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e0ef // sdot v15.4s, v7.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e0f3 // sdot v19.4s, v7.16b, v2.4b[1]\n"
+ "ldr q7, [x14, #0x90]\n"
+ ".inst 0x4f80e8c8 // sdot v8.4s, v6.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8cc // sdot v12.4s, v6.16b, v1.4b[2]\n"
+ ".inst 0x4f82e8d0 // sdot v16.4s, v6.16b, v2.4b[2]\n"
+ "ldr q6, [x14, #0xa0]\n"
+ ".inst 0x4f80e8e9 // sdot v9.4s, v7.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8ed // sdot v13.4s, v7.16b, v1.4b[2]\n"
+ ".inst 0x4f82e8f1 // sdot v17.4s, v7.16b, v2.4b[2]\n"
+ "ldr q7, [x14, #0xb0]\n"
+ ".inst 0x4f80e8ca // sdot v10.4s, v6.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8ce // sdot v14.4s, v6.16b, v1.4b[2]\n"
+ ".inst 0x4f82e8d2 // sdot v18.4s, v6.16b, v2.4b[2]\n"
+ "ldr q6, [x14, #0xc0]\n"
+ ".inst 0x4f80e8eb // sdot v11.4s, v7.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8ef // sdot v15.4s, v7.16b, v1.4b[2]\n"
+ ".inst 0x4f82e8f3 // sdot v19.4s, v7.16b, v2.4b[2]\n"
+ "ldr q7, [x14, #0xd0]\n"
+ ".inst 0x4fa0e8c8 // sdot v8.4s, v6.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e8cc // sdot v12.4s, v6.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e8d0 // sdot v16.4s, v6.16b, v2.4b[3]\n"
+ "ldr q6, [x14, #0xe0]\n"
+ ".inst 0x4fa0e8e9 // sdot v9.4s, v7.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e8ed // sdot v13.4s, v7.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e8f1 // sdot v17.4s, v7.16b, v2.4b[3]\n"
+ "ldr q7, [x14, #0xf0]\n"
+ "add x14, x14, #0x100\n"
+ ".inst 0x4fa0e8ca // sdot v10.4s, v6.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e8ce // sdot v14.4s, v6.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e8d2 // sdot v18.4s, v6.16b, v2.4b[3]\n"
+ ".inst 0x4fa0e8eb // sdot v11.4s, v7.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e8ef // sdot v15.4s, v7.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e8f3 // sdot v19.4s, v7.16b, v2.4b[3]\n"
+ "bge 64b\n"
+ "65:" // Height 3: Multiply loop: Single iteration only
+ "sub x11, x11, #0x10\n"
+ "ldr q0, [x10, #0x0]\n"
+ "ldr q1, [x28, #0x0]\n"
+ "ldr q2, [x26, #0x0]\n"
+ "ldr q6, [x14, #0x0]\n"
+ ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n"
+ "ldr q7, [x14, #0x10]\n"
+ ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n"
+ "add x10, x10, #0x10\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n"
+ "ldr q6, [x14, #0x20]\n"
+ ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n"
+ "add x28, x28, #0x10\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n"
+ "add x26, x26, #0x10\n"
+ ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "ldr q7, [x14, #0x30]\n"
+ ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n"
+ ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n"
+ "ldr q6, [x14, #0x40]\n"
+ ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n"
+ "ldr q7, [x14, #0x50]\n"
+ ".inst 0x4fa0e0c8 // sdot v8.4s, v6.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e0cc // sdot v12.4s, v6.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e0d0 // sdot v16.4s, v6.16b, v2.4b[1]\n"
+ "ldr q6, [x14, #0x60]\n"
+ ".inst 0x4fa0e0e9 // sdot v9.4s, v7.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e0ed // sdot v13.4s, v7.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e0f1 // sdot v17.4s, v7.16b, v2.4b[1]\n"
+ "ldr q7, [x14, #0x70]\n"
+ ".inst 0x4fa0e0ca // sdot v10.4s, v6.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e0ce // sdot v14.4s, v6.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e0d2 // sdot v18.4s, v6.16b, v2.4b[1]\n"
+ "ldr q6, [x14, #0x80]\n"
+ ".inst 0x4fa0e0eb // sdot v11.4s, v7.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e0ef // sdot v15.4s, v7.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e0f3 // sdot v19.4s, v7.16b, v2.4b[1]\n"
+ "ldr q7, [x14, #0x90]\n"
+ ".inst 0x4f80e8c8 // sdot v8.4s, v6.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8cc // sdot v12.4s, v6.16b, v1.4b[2]\n"
+ ".inst 0x4f82e8d0 // sdot v16.4s, v6.16b, v2.4b[2]\n"
+ "ldr q6, [x14, #0xa0]\n"
+ ".inst 0x4f80e8e9 // sdot v9.4s, v7.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8ed // sdot v13.4s, v7.16b, v1.4b[2]\n"
+ ".inst 0x4f82e8f1 // sdot v17.4s, v7.16b, v2.4b[2]\n"
+ "ldr q7, [x14, #0xb0]\n"
+ ".inst 0x4f80e8ca // sdot v10.4s, v6.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8ce // sdot v14.4s, v6.16b, v1.4b[2]\n"
+ ".inst 0x4f82e8d2 // sdot v18.4s, v6.16b, v2.4b[2]\n"
+ "ldr q6, [x14, #0xc0]\n"
+ ".inst 0x4f80e8eb // sdot v11.4s, v7.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8ef // sdot v15.4s, v7.16b, v1.4b[2]\n"
+ ".inst 0x4f82e8f3 // sdot v19.4s, v7.16b, v2.4b[2]\n"
+ "ldr q7, [x14, #0xd0]\n"
+ ".inst 0x4fa0e8c8 // sdot v8.4s, v6.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e8cc // sdot v12.4s, v6.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e8d0 // sdot v16.4s, v6.16b, v2.4b[3]\n"
+ "ldr q6, [x14, #0xe0]\n"
+ ".inst 0x4fa0e8e9 // sdot v9.4s, v7.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e8ed // sdot v13.4s, v7.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e8f1 // sdot v17.4s, v7.16b, v2.4b[3]\n"
+ "ldr q7, [x14, #0xf0]\n"
+ "add x14, x14, #0x100\n"
+ ".inst 0x4fa0e8ca // sdot v10.4s, v6.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e8ce // sdot v14.4s, v6.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e8d2 // sdot v18.4s, v6.16b, v2.4b[3]\n"
+ ".inst 0x4fa0e8eb // sdot v11.4s, v7.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e8ef // sdot v15.4s, v7.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e8f3 // sdot v19.4s, v7.16b, v2.4b[3]\n"
+ "66:" // Height 3: Multiply loop: Main loop skip
+ "cbz x11, 71f\n"
+ "cmp x11, #0x4\n"
+ "blt 68f\n"
+ "67:" // Height 3: Multiply loop: Odd block loop
+ "ldr s0, [x10], #0x4\n"
+ "ldr s1, [x28], #0x4\n"
+ "ldr s2, [x26], #0x4\n"
+ "ldr q6, [x14, #0x0]\n"
+ ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n"
+ "ldr q7, [x14, #0x10]\n"
+ ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n"
+ "sub x11, x11, #0x4\n"
+ ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n"
+ "ldr q6, [x14, #0x20]\n"
+ "cmp x11, #0x4\n"
+ ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n"
+ "ldr q7, [x14, #0x30]\n"
+ ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n"
+ "add x14, x14, #0x40\n"
+ ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n"
+ ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n"
+ "bge 67b\n"
+ "cbz x11, 71f\n"
+ "68:" // Height 3: Multiply loop: Skip odd blocks
+ "tbz x11, #1, 69f\n"
+ "ldr h0, [x10], #0x2\n"
+ "ldr h1, [x28], #0x2\n"
+ "ldr h2, [x26], #0x2\n"
+ "tbz x11, #0, 70f\n"
+ "ld1 { v0.b }[2], [x10]\n"
+ "ld1 { v1.b }[2], [x28]\n"
+ "ld1 { v2.b }[2], [x26]\n"
+ "b 70f\n"
+ "69:" // Height 3: Multiply loop: Ragged operand read: partial_1_0
+ "ldr b0, [x10, #0x0]\n"
+ "ldr b1, [x28, #0x0]\n"
+ "ldr b2, [x26, #0x0]\n"
+ "70:" // Height 3: Multiply loop: Ragged operand read: Done
+ "ldr q6, [x14, #0x0]\n"
+ ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n"
+ "ldr q7, [x14, #0x10]\n"
+ ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n"
+ "ldr q6, [x14, #0x20]\n"
+ ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n"
+ "ldr q7, [x14, #0x30]\n"
+ ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n"
+ "add x14, x14, #0x40\n"
+ ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n"
+ ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n"
+ "71:" // Height 3: Multiply loop: No odd multiplies
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "add x12, x12, #0x1\n"
+ "cmp x12, x19\n"
+ "bne 61b\n"
+ "prfm pstl1keep, [x13, #0x0]\n"
+ "prfm pstl1keep, [x9, #0x0]\n"
+ "ldr q0, [x16, #0x0]\n"
+ "add v8.4s, v8.4s, v0.4s\n"
+ "prfm pstl1keep, [x27, #0x0]\n"
+ "ldr q1, [x16, #0x10]\n"
+ "add v12.4s, v12.4s, v0.4s\n"
+ "ldr q2, [x16, #0x20]\n"
+ "add v16.4s, v16.4s, v0.4s\n"
+ "ldr q3, [x16, #0x30]\n"
+ "add x16, x16, #0x40\n"
+ "add v9.4s, v9.4s, v1.4s\n"
+ "add v13.4s, v13.4s, v1.4s\n"
+ "add v10.4s, v10.4s, v2.4s\n"
+ "add v11.4s, v11.4s, v3.4s\n"
+ "add v14.4s, v14.4s, v2.4s\n"
+ "add v15.4s, v15.4s, v3.4s\n"
+ "add v17.4s, v17.4s, v1.4s\n"
+ "add v18.4s, v18.4s, v2.4s\n"
+ "add v19.4s, v19.4s, v3.4s\n"
+ "tbz %x[flags], #4, 72f\n"
+ "ldr q0, [x17, #0x0]\n"
+ "ldr q4, [x8, #0x0]\n"
+ "ldr q1, [x17, #0x10]\n"
+ "ldr q5, [x8, #0x10]\n"
+ "ldr q2, [x17, #0x20]\n"
+ "ldr q6, [x8, #0x20]\n"
+ "ldr q3, [x17, #0x30]\n"
+ "ldr q7, [x8, #0x30]\n"
+ "add x17, x17, #0x40\n"
+ "add x8, x8, #0x40\n"
+ "b 73f\n"
+ "72:" // Height 3: per layer parameters
+ "add x19, %x[qp], %[per_layer_right_shift]\n"
+ "ld1r { v0.4s }, [x19]\n"
+ "mov v1.16b, v0.16b\n"
+ "add x19, %x[qp], %[per_layer_mul]\n"
+ "ld1r { v4.4s }, [x19]\n"
+ "mov v2.16b, v0.16b\n"
+ "mov v3.16b, v0.16b\n"
+ "mov v5.16b, v4.16b\n"
+ "mov v6.16b, v4.16b\n"
+ "mov v7.16b, v4.16b\n"
+ "73:" // Height 3: parameters loaded
+ "sqrdmulh v8.4s, v8.4s, v4.4s\n"
+ "sqrdmulh v9.4s, v9.4s, v5.4s\n"
+ "sqrdmulh v10.4s, v10.4s, v6.4s\n"
+ "sqrdmulh v11.4s, v11.4s, v7.4s\n"
+ "sqrdmulh v12.4s, v12.4s, v4.4s\n"
+ "sqrdmulh v13.4s, v13.4s, v5.4s\n"
+ "sqrdmulh v14.4s, v14.4s, v6.4s\n"
+ "sqrdmulh v15.4s, v15.4s, v7.4s\n"
+ "sqrdmulh v16.4s, v16.4s, v4.4s\n"
+ "sqrdmulh v17.4s, v17.4s, v5.4s\n"
+ "sqrdmulh v18.4s, v18.4s, v6.4s\n"
+ "sqrdmulh v19.4s, v19.4s, v7.4s\n"
+ "tbz %x[flags], #5, 74f\n"
+ "and v4.16b, v8.16b, v0.16b\n"
+ "sshr v4.4s, v4.4s, #0x1f\n"
+ "and v5.16b, v9.16b, v1.16b\n"
+ "and v6.16b, v10.16b, v2.16b\n"
+ "sshr v5.4s, v5.4s, #0x1f\n"
+ "and v7.16b, v11.16b, v3.16b\n"
+ "sshr v6.4s, v6.4s, #0x1f\n"
+ "sqadd v8.4s, v8.4s, v4.4s\n"
+ "sshr v7.4s, v7.4s, #0x1f\n"
+ "and v4.16b, v12.16b, v0.16b\n"
+ "sqadd v9.4s, v9.4s, v5.4s\n"
+ "sshr v4.4s, v4.4s, #0x1f\n"
+ "sqadd v10.4s, v10.4s, v6.4s\n"
+ "and v5.16b, v13.16b, v1.16b\n"
+ "sshr v5.4s, v5.4s, #0x1f\n"
+ "sqadd v11.4s, v11.4s, v7.4s\n"
+ "and v6.16b, v14.16b, v2.16b\n"
+ "sshr v6.4s, v6.4s, #0x1f\n"
+ "sqadd v12.4s, v12.4s, v4.4s\n"
+ "and v7.16b, v15.16b, v3.16b\n"
+ "sshr v7.4s, v7.4s, #0x1f\n"
+ "sqadd v13.4s, v13.4s, v5.4s\n"
+ "and v4.16b, v16.16b, v0.16b\n"
+ "sshr v4.4s, v4.4s, #0x1f\n"
+ "sqadd v14.4s, v14.4s, v6.4s\n"
+ "and v5.16b, v17.16b, v1.16b\n"
+ "sshr v5.4s, v5.4s, #0x1f\n"
+ "sqadd v15.4s, v15.4s, v7.4s\n"
+ "and v6.16b, v18.16b, v2.16b\n"
+ "sshr v6.4s, v6.4s, #0x1f\n"
+ "sqadd v16.4s, v16.4s, v4.4s\n"
+ "and v7.16b, v19.16b, v3.16b\n"
+ "sshr v7.4s, v7.4s, #0x1f\n"
+ "sqadd v17.4s, v17.4s, v5.4s\n"
+ "sqadd v18.4s, v18.4s, v6.4s\n"
+ "sqadd v19.4s, v19.4s, v7.4s\n"
+ "74:" // Height 3: no shift correction
+ "srshl v8.4s, v8.4s, v0.4s\n"
+ "add x19, %x[qp], %[c_offset]\n"
+ "ld1r { v4.4s }, [x19]\n"
+ "srshl v9.4s, v9.4s, v1.4s\n"
+ "add x19, %x[qp], %[minval]\n"
+ "srshl v10.4s, v10.4s, v2.4s\n"
+ "ld1r { v5.4s }, [x19]\n"
+ "add x19, %x[qp], %[maxval]\n"
+ "srshl v11.4s, v11.4s, v3.4s\n"
+ "ld1r { v6.4s }, [x19]\n"
+ "cmp x15, #0x10\n"
+ "srshl v12.4s, v12.4s, v0.4s\n"
+ "srshl v13.4s, v13.4s, v1.4s\n"
+ "srshl v14.4s, v14.4s, v2.4s\n"
+ "srshl v15.4s, v15.4s, v3.4s\n"
+ "add v8.4s, v8.4s, v4.4s\n"
+ "add v9.4s, v9.4s, v4.4s\n"
+ "add v10.4s, v10.4s, v4.4s\n"
+ "smin v8.4s, v8.4s, v6.4s\n"
+ "smin v9.4s, v9.4s, v6.4s\n"
+ "smin v10.4s, v10.4s, v6.4s\n"
+ "smax v8.4s, v8.4s, v5.4s\n"
+ "smax v9.4s, v9.4s, v5.4s\n"
+ "smax v10.4s, v10.4s, v5.4s\n"
+ "add v11.4s, v11.4s, v4.4s\n"
+ "add v12.4s, v12.4s, v4.4s\n"
+ "add v13.4s, v13.4s, v4.4s\n"
+ "smin v11.4s, v11.4s, v6.4s\n"
+ "smin v12.4s, v12.4s, v6.4s\n"
+ "smin v13.4s, v13.4s, v6.4s\n"
+ "smax v11.4s, v11.4s, v5.4s\n"
+ "smax v12.4s, v12.4s, v5.4s\n"
+ "smax v13.4s, v13.4s, v5.4s\n"
+ "add v14.4s, v14.4s, v4.4s\n"
+ "add v15.4s, v15.4s, v4.4s\n"
+ "srshl v16.4s, v16.4s, v0.4s\n"
+ "smin v14.4s, v14.4s, v6.4s\n"
+ "smin v15.4s, v15.4s, v6.4s\n"
+ "srshl v17.4s, v17.4s, v1.4s\n"
+ "smax v14.4s, v14.4s, v5.4s\n"
+ "smax v15.4s, v15.4s, v5.4s\n"
+ "add v16.4s, v16.4s, v4.4s\n"
+ "add v17.4s, v17.4s, v4.4s\n"
+ "srshl v18.4s, v18.4s, v2.4s\n"
+ "smin v16.4s, v16.4s, v6.4s\n"
+ "smin v17.4s, v17.4s, v6.4s\n"
+ "srshl v19.4s, v19.4s, v3.4s\n"
+ "smax v16.4s, v16.4s, v5.4s\n"
+ "smax v17.4s, v17.4s, v5.4s\n"
+ "add v18.4s, v18.4s, v4.4s\n"
+ "add v19.4s, v19.4s, v4.4s\n"
+ "uzp1 v8.8h, v8.8h, v9.8h\n"
+ "smin v18.4s, v18.4s, v6.4s\n"
+ "smin v19.4s, v19.4s, v6.4s\n"
+ "uzp1 v9.8h, v10.8h, v11.8h\n"
+ "smax v18.4s, v18.4s, v5.4s\n"
+ "smax v19.4s, v19.4s, v5.4s\n"
+ "uzp1 v12.8h, v12.8h, v13.8h\n"
+ "uzp1 v13.8h, v14.8h, v15.8h\n"
+ "uzp1 v16.8h, v16.8h, v17.8h\n"
+ "uzp1 v17.8h, v18.8h, v19.8h\n"
+ "uzp1 v8.16b, v8.16b, v9.16b\n"
+ "uzp1 v12.16b, v12.16b, v13.16b\n"
+ "uzp1 v16.16b, v16.16b, v17.16b\n"
+ "bge 83f\n"
+ "tbz x15, #3, 78f\n"
+ "str d8, [x13], #0x8\n"
+ "str d12, [x9], #0x8\n"
+ "str d16, [x27], #0x8\n"
+ "tbz x15, #2, 76f\n"
+ "st1 { v8.s }[2], [x13], #0x4\n"
+ "st1 { v12.s }[2], [x9], #0x4\n"
+ "st1 { v16.s }[2], [x27], #0x4\n"
+ "tbz x15, #1, 75f\n"
+ "st1 { v8.h }[6], [x13], #0x2\n"
+ "st1 { v12.h }[6], [x9], #0x2\n"
+ "st1 { v16.h }[6], [x27], #0x2\n"
+ "tbz x15, #0, 82f\n"
+ "st1 { v8.b }[14], [x13]\n"
+ "st1 { v12.b }[14], [x9]\n"
+ "st1 { v16.b }[14], [x27]\n"
+ "b 82f\n"
+ "75:" // Height 3: Partial direct writeback: partial_1_12
+ "tbz x15, #0, 82f\n"
+ "st1 { v8.b }[12], [x13]\n"
+ "st1 { v12.b }[12], [x9]\n"
+ "st1 { v16.b }[12], [x27]\n"
+ "b 82f\n"
+ "76:" // Height 3: Partial direct writeback: partial_2_8
+ "tbz x15, #1, 77f\n"
+ "st1 { v8.h }[4], [x13], #0x2\n"
+ "st1 { v12.h }[4], [x9], #0x2\n"
+ "st1 { v16.h }[4], [x27], #0x2\n"
+ "tbz x15, #0, 82f\n"
+ "st1 { v8.b }[10], [x13]\n"
+ "st1 { v12.b }[10], [x9]\n"
+ "st1 { v16.b }[10], [x27]\n"
+ "b 82f\n"
+ "77:" // Height 3: Partial direct writeback: partial_1_8
+ "tbz x15, #0, 82f\n"
+ "st1 { v8.b }[8], [x13]\n"
+ "st1 { v12.b }[8], [x9]\n"
+ "st1 { v16.b }[8], [x27]\n"
+ "b 82f\n"
+ "78:" // Height 3: Partial direct writeback: partial_4_0
+ "tbz x15, #2, 80f\n"
+ "str s8, [x13], #0x4\n"
+ "str s12, [x9], #0x4\n"
+ "str s16, [x27], #0x4\n"
+ "tbz x15, #1, 79f\n"
+ "st1 { v8.h }[2], [x13], #0x2\n"
+ "st1 { v12.h }[2], [x9], #0x2\n"
+ "st1 { v16.h }[2], [x27], #0x2\n"
+ "tbz x15, #0, 82f\n"
+ "st1 { v8.b }[6], [x13]\n"
+ "st1 { v12.b }[6], [x9]\n"
+ "st1 { v16.b }[6], [x27]\n"
+ "b 82f\n"
+ "79:" // Height 3: Partial direct writeback: partial_1_4
+ "tbz x15, #0, 82f\n"
+ "st1 { v8.b }[4], [x13]\n"
+ "st1 { v12.b }[4], [x9]\n"
+ "st1 { v16.b }[4], [x27]\n"
+ "b 82f\n"
+ "80:" // Height 3: Partial direct writeback: partial_2_0
+ "tbz x15, #1, 81f\n"
+ "str h8, [x13], #0x2\n"
+ "str h12, [x9], #0x2\n"
+ "str h16, [x27], #0x2\n"
+ "tbz x15, #0, 82f\n"
+ "st1 { v8.b }[2], [x13]\n"
+ "st1 { v12.b }[2], [x9]\n"
+ "st1 { v16.b }[2], [x27]\n"
+ "b 82f\n"
+ "81:" // Height 3: Partial direct writeback: partial_1_0
+ "str b8, [x13, #0x0]\n"
+ "str b12, [x9, #0x0]\n"
+ "str b16, [x27, #0x0]\n"
+ "82:" // Height 3: Partial direct writeback: Done
+ "b 84f\n"
+ "83:" // Height 3: Full writeback
+ "str q8, [x13, #0x0]\n"
+ "str q12, [x9, #0x0]\n"
+ "str q16, [x27, #0x0]\n"
+ "add x13, x13, #0x10\n"
+ "add x9, x9, #0x10\n"
+ "add x27, x27, #0x10\n"
+ "84:" // Height 3: Writeback done
+ "subs x15, x15, #0x10\n"
+ "bgt 59b\n"
+ "b 170f\n"
+ "85:" // Height 4
+ "ldr x8, [%x[args_ptr], %[offsetof_multiplier_ptr]]\n"
+ "mov x16, %x[col_bias]\n"
+ "ldr x17, [%x[args_ptr], %[offsetof_shift_ptr]]\n"
+ "ldr x15, [%x[args_ptr], %[offsetof_N]]\n"
+ "ldr x14, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 86f\n"
+ "ldr x13, [%x[output_ptr], #0x0]\n"
+ "add x13, x13, x19\n"
+ "ldr x9, [%x[output_ptr], #0x8]\n"
+ "ldr x27, [%x[output_ptr], #0x10]\n"
+ "add x9, x9, x19\n"
+ "ldr x25, [%x[output_ptr], #0x18]\n"
+ "add x27, x27, x19\n"
+ "add x25, x25, x19\n"
+ "b 87f\n"
+ "86:" // Height 4: setup direct output
+ "mov x13, %x[output_ptr]\n"
+ "add x9, x13, x19\n"
+ "add x27, x9, x19\n"
+ "add x25, x27, x19\n"
+ "87:" // Height 4: Column loop
+ "movi v8.4s, #0x0\n"
+ "movi v9.4s, #0x0\n"
+ "movi v10.4s, #0x0\n"
+ "movi v11.4s, #0x0\n"
+ "movi v12.4s, #0x0\n"
+ "movi v13.4s, #0x0\n"
+ "movi v14.4s, #0x0\n"
+ "movi v15.4s, #0x0\n"
+ "movi v16.4s, #0x0\n"
+ "movi v17.4s, #0x0\n"
+ "movi v18.4s, #0x0\n"
+ "movi v19.4s, #0x0\n"
+ "movi v20.4s, #0x0\n"
+ "movi v21.4s, #0x0\n"
+ "movi v22.4s, #0x0\n"
+ "movi v23.4s, #0x0\n"
+ "88:" // Height 4: setup done
+ "mov x12, #0x0\n"
+ "89:" // Height 4: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w11, [x20, x12, LSL #0x2]\n"
+ "tbz %x[flags], #3, 90f\n"
+ "ldr x20, [%x[input_ptr], x12, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x10, [x20, #0x0]\n"
+ "ldr x28, [x20, #0x8]\n"
+ "ldr x26, [x20, #0x10]\n"
+ "ldr x24, [x20, #0x18]\n"
+ "cbnz x12, 91f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x10, x10, x19\n"
+ "add x28, x28, x19\n"
+ "add x26, x26, x19\n"
+ "add x24, x24, x19\n"
+ "b 91f\n"
+ "90:" // Height 4: setup direct input
+ "mov x10, %x[input_ptr]\n"
+ "add x28, x10, x19\n"
+ "add x26, x28, x19\n"
+ "add x24, x26, x19\n"
+ "91:" // Height 4: input setup done
+ "cmp x11, #0x10\n"
+ "blt 94f\n"
+ "cmp x11, #0x20\n"
+ "blt 93f\n"
+ "92:" // Height 4: Multiply loop: Main loop head
+ "ldr q0, [x10, #0x0]\n"
+ "ldr q1, [x28, #0x0]\n"
+ "ldr q2, [x26, #0x0]\n"
+ "ldr q3, [x24, #0x0]\n"
+ "ldr q6, [x14, #0x0]\n"
+ ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n"
+ "ldr q7, [x14, #0x10]\n"
+ ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n"
+ "add x10, x10, #0x10\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n"
+ "add x28, x28, #0x10\n"
+ ".inst 0x4f83e0d4 // sdot v20.4s, v6.16b, v3.4b[0]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "ldr q6, [x14, #0x20]\n"
+ ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n"
+ "add x26, x26, #0x10\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n"
+ "add x24, x24, #0x10\n"
+ ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "sub x11, x11, #0x10\n"
+ ".inst 0x4f83e0f5 // sdot v21.4s, v7.16b, v3.4b[0]\n"
+ "ldr q7, [x14, #0x30]\n"
+ ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n"
+ "cmp x11, #0x20\n"
+ ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n"
+ ".inst 0x4f83e0d6 // sdot v22.4s, v6.16b, v3.4b[0]\n"
+ "ldr q6, [x14, #0x40]\n"
+ ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n"
+ ".inst 0x4f83e0f7 // sdot v23.4s, v7.16b, v3.4b[0]\n"
+ "ldr q7, [x14, #0x50]\n"
+ ".inst 0x4fa0e0c8 // sdot v8.4s, v6.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e0cc // sdot v12.4s, v6.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e0d0 // sdot v16.4s, v6.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e0d4 // sdot v20.4s, v6.16b, v3.4b[1]\n"
+ "ldr q6, [x14, #0x60]\n"
+ ".inst 0x4fa0e0e9 // sdot v9.4s, v7.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e0ed // sdot v13.4s, v7.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e0f1 // sdot v17.4s, v7.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e0f5 // sdot v21.4s, v7.16b, v3.4b[1]\n"
+ "ldr q7, [x14, #0x70]\n"
+ ".inst 0x4fa0e0ca // sdot v10.4s, v6.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e0ce // sdot v14.4s, v6.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e0d2 // sdot v18.4s, v6.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e0d6 // sdot v22.4s, v6.16b, v3.4b[1]\n"
+ "ldr q6, [x14, #0x80]\n"
+ ".inst 0x4fa0e0eb // sdot v11.4s, v7.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e0ef // sdot v15.4s, v7.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e0f3 // sdot v19.4s, v7.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e0f7 // sdot v23.4s, v7.16b, v3.4b[1]\n"
+ "ldr q7, [x14, #0x90]\n"
+ ".inst 0x4f80e8c8 // sdot v8.4s, v6.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8cc // sdot v12.4s, v6.16b, v1.4b[2]\n"
+ ".inst 0x4f82e8d0 // sdot v16.4s, v6.16b, v2.4b[2]\n"
+ ".inst 0x4f83e8d4 // sdot v20.4s, v6.16b, v3.4b[2]\n"
+ "ldr q6, [x14, #0xa0]\n"
+ ".inst 0x4f80e8e9 // sdot v9.4s, v7.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8ed // sdot v13.4s, v7.16b, v1.4b[2]\n"
+ ".inst 0x4f82e8f1 // sdot v17.4s, v7.16b, v2.4b[2]\n"
+ ".inst 0x4f83e8f5 // sdot v21.4s, v7.16b, v3.4b[2]\n"
+ "ldr q7, [x14, #0xb0]\n"
+ ".inst 0x4f80e8ca // sdot v10.4s, v6.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8ce // sdot v14.4s, v6.16b, v1.4b[2]\n"
+ ".inst 0x4f82e8d2 // sdot v18.4s, v6.16b, v2.4b[2]\n"
+ ".inst 0x4f83e8d6 // sdot v22.4s, v6.16b, v3.4b[2]\n"
+ "ldr q6, [x14, #0xc0]\n"
+ ".inst 0x4f80e8eb // sdot v11.4s, v7.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8ef // sdot v15.4s, v7.16b, v1.4b[2]\n"
+ ".inst 0x4f82e8f3 // sdot v19.4s, v7.16b, v2.4b[2]\n"
+ ".inst 0x4f83e8f7 // sdot v23.4s, v7.16b, v3.4b[2]\n"
+ "ldr q7, [x14, #0xd0]\n"
+ ".inst 0x4fa0e8c8 // sdot v8.4s, v6.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e8cc // sdot v12.4s, v6.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e8d0 // sdot v16.4s, v6.16b, v2.4b[3]\n"
+ ".inst 0x4fa3e8d4 // sdot v20.4s, v6.16b, v3.4b[3]\n"
+ "ldr q6, [x14, #0xe0]\n"
+ ".inst 0x4fa0e8e9 // sdot v9.4s, v7.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e8ed // sdot v13.4s, v7.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e8f1 // sdot v17.4s, v7.16b, v2.4b[3]\n"
+ ".inst 0x4fa3e8f5 // sdot v21.4s, v7.16b, v3.4b[3]\n"
+ "ldr q7, [x14, #0xf0]\n"
+ "add x14, x14, #0x100\n"
+ ".inst 0x4fa0e8ca // sdot v10.4s, v6.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e8ce // sdot v14.4s, v6.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e8d2 // sdot v18.4s, v6.16b, v2.4b[3]\n"
+ ".inst 0x4fa3e8d6 // sdot v22.4s, v6.16b, v3.4b[3]\n"
+ ".inst 0x4fa0e8eb // sdot v11.4s, v7.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e8ef // sdot v15.4s, v7.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e8f3 // sdot v19.4s, v7.16b, v2.4b[3]\n"
+ ".inst 0x4fa3e8f7 // sdot v23.4s, v7.16b, v3.4b[3]\n"
+ "bge 92b\n"
+ "93:" // Height 4: Multiply loop: Single iteration only
+ "sub x11, x11, #0x10\n"
+ "ldr q0, [x10, #0x0]\n"
+ "ldr q1, [x28, #0x0]\n"
+ "ldr q2, [x26, #0x0]\n"
+ "ldr q3, [x24, #0x0]\n"
+ "ldr q6, [x14, #0x0]\n"
+ ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n"
+ "ldr q7, [x14, #0x10]\n"
+ ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n"
+ "add x10, x10, #0x10\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n"
+ "add x28, x28, #0x10\n"
+ ".inst 0x4f83e0d4 // sdot v20.4s, v6.16b, v3.4b[0]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "ldr q6, [x14, #0x20]\n"
+ ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n"
+ "add x26, x26, #0x10\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n"
+ "add x24, x24, #0x10\n"
+ ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ ".inst 0x4f83e0f5 // sdot v21.4s, v7.16b, v3.4b[0]\n"
+ "ldr q7, [x14, #0x30]\n"
+ ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n"
+ ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n"
+ ".inst 0x4f83e0d6 // sdot v22.4s, v6.16b, v3.4b[0]\n"
+ "ldr q6, [x14, #0x40]\n"
+ ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n"
+ ".inst 0x4f83e0f7 // sdot v23.4s, v7.16b, v3.4b[0]\n"
+ "ldr q7, [x14, #0x50]\n"
+ ".inst 0x4fa0e0c8 // sdot v8.4s, v6.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e0cc // sdot v12.4s, v6.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e0d0 // sdot v16.4s, v6.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e0d4 // sdot v20.4s, v6.16b, v3.4b[1]\n"
+ "ldr q6, [x14, #0x60]\n"
+ ".inst 0x4fa0e0e9 // sdot v9.4s, v7.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e0ed // sdot v13.4s, v7.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e0f1 // sdot v17.4s, v7.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e0f5 // sdot v21.4s, v7.16b, v3.4b[1]\n"
+ "ldr q7, [x14, #0x70]\n"
+ ".inst 0x4fa0e0ca // sdot v10.4s, v6.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e0ce // sdot v14.4s, v6.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e0d2 // sdot v18.4s, v6.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e0d6 // sdot v22.4s, v6.16b, v3.4b[1]\n"
+ "ldr q6, [x14, #0x80]\n"
+ ".inst 0x4fa0e0eb // sdot v11.4s, v7.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e0ef // sdot v15.4s, v7.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e0f3 // sdot v19.4s, v7.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e0f7 // sdot v23.4s, v7.16b, v3.4b[1]\n"
+ "ldr q7, [x14, #0x90]\n"
+ ".inst 0x4f80e8c8 // sdot v8.4s, v6.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8cc // sdot v12.4s, v6.16b, v1.4b[2]\n"
+ ".inst 0x4f82e8d0 // sdot v16.4s, v6.16b, v2.4b[2]\n"
+ ".inst 0x4f83e8d4 // sdot v20.4s, v6.16b, v3.4b[2]\n"
+ "ldr q6, [x14, #0xa0]\n"
+ ".inst 0x4f80e8e9 // sdot v9.4s, v7.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8ed // sdot v13.4s, v7.16b, v1.4b[2]\n"
+ ".inst 0x4f82e8f1 // sdot v17.4s, v7.16b, v2.4b[2]\n"
+ ".inst 0x4f83e8f5 // sdot v21.4s, v7.16b, v3.4b[2]\n"
+ "ldr q7, [x14, #0xb0]\n"
+ ".inst 0x4f80e8ca // sdot v10.4s, v6.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8ce // sdot v14.4s, v6.16b, v1.4b[2]\n"
+ ".inst 0x4f82e8d2 // sdot v18.4s, v6.16b, v2.4b[2]\n"
+ ".inst 0x4f83e8d6 // sdot v22.4s, v6.16b, v3.4b[2]\n"
+ "ldr q6, [x14, #0xc0]\n"
+ ".inst 0x4f80e8eb // sdot v11.4s, v7.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8ef // sdot v15.4s, v7.16b, v1.4b[2]\n"
+ ".inst 0x4f82e8f3 // sdot v19.4s, v7.16b, v2.4b[2]\n"
+ ".inst 0x4f83e8f7 // sdot v23.4s, v7.16b, v3.4b[2]\n"
+ "ldr q7, [x14, #0xd0]\n"
+ ".inst 0x4fa0e8c8 // sdot v8.4s, v6.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e8cc // sdot v12.4s, v6.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e8d0 // sdot v16.4s, v6.16b, v2.4b[3]\n"
+ ".inst 0x4fa3e8d4 // sdot v20.4s, v6.16b, v3.4b[3]\n"
+ "ldr q6, [x14, #0xe0]\n"
+ ".inst 0x4fa0e8e9 // sdot v9.4s, v7.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e8ed // sdot v13.4s, v7.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e8f1 // sdot v17.4s, v7.16b, v2.4b[3]\n"
+ ".inst 0x4fa3e8f5 // sdot v21.4s, v7.16b, v3.4b[3]\n"
+ "ldr q7, [x14, #0xf0]\n"
+ "add x14, x14, #0x100\n"
+ ".inst 0x4fa0e8ca // sdot v10.4s, v6.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e8ce // sdot v14.4s, v6.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e8d2 // sdot v18.4s, v6.16b, v2.4b[3]\n"
+ ".inst 0x4fa3e8d6 // sdot v22.4s, v6.16b, v3.4b[3]\n"
+ ".inst 0x4fa0e8eb // sdot v11.4s, v7.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e8ef // sdot v15.4s, v7.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e8f3 // sdot v19.4s, v7.16b, v2.4b[3]\n"
+ ".inst 0x4fa3e8f7 // sdot v23.4s, v7.16b, v3.4b[3]\n"
+ "94:" // Height 4: Multiply loop: Main loop skip
+ "cbz x11, 99f\n"
+ "cmp x11, #0x4\n"
+ "blt 96f\n"
+ "95:" // Height 4: Multiply loop: Odd block loop
+ "ldr s0, [x10], #0x4\n"
+ "ldr s1, [x28], #0x4\n"
+ "ldr s2, [x26], #0x4\n"
+ "ldr s3, [x24], #0x4\n"
+ "ldr q6, [x14, #0x0]\n"
+ ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n"
+ "ldr q7, [x14, #0x10]\n"
+ ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n"
+ "sub x11, x11, #0x4\n"
+ ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n"
+ "cmp x11, #0x4\n"
+ ".inst 0x4f83e0d4 // sdot v20.4s, v6.16b, v3.4b[0]\n"
+ "ldr q6, [x14, #0x20]\n"
+ ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n"
+ ".inst 0x4f83e0f5 // sdot v21.4s, v7.16b, v3.4b[0]\n"
+ "ldr q7, [x14, #0x30]\n"
+ ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n"
+ "add x14, x14, #0x40\n"
+ ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n"
+ ".inst 0x4f83e0d6 // sdot v22.4s, v6.16b, v3.4b[0]\n"
+ ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n"
+ ".inst 0x4f83e0f7 // sdot v23.4s, v7.16b, v3.4b[0]\n"
+ "bge 95b\n"
+ "cbz x11, 99f\n"
+ "96:" // Height 4: Multiply loop: Skip odd blocks
+ "tbz x11, #1, 97f\n"
+ "ldr h0, [x10], #0x2\n"
+ "ldr h1, [x28], #0x2\n"
+ "ldr h2, [x26], #0x2\n"
+ "ldr h3, [x24], #0x2\n"
+ "tbz x11, #0, 98f\n"
+ "ld1 { v0.b }[2], [x10]\n"
+ "ld1 { v1.b }[2], [x28]\n"
+ "ld1 { v2.b }[2], [x26]\n"
+ "ld1 { v3.b }[2], [x24]\n"
+ "b 98f\n"
+ "97:" // Height 4: Multiply loop: Ragged operand read: partial_1_0
+ "ldr b0, [x10, #0x0]\n"
+ "ldr b1, [x28, #0x0]\n"
+ "ldr b2, [x26, #0x0]\n"
+ "ldr b3, [x24, #0x0]\n"
+ "98:" // Height 4: Multiply loop: Ragged operand read: Done
+ "ldr q6, [x14, #0x0]\n"
+ ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n"
+ "ldr q7, [x14, #0x10]\n"
+ ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n"
+ ".inst 0x4f83e0d4 // sdot v20.4s, v6.16b, v3.4b[0]\n"
+ "ldr q6, [x14, #0x20]\n"
+ ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n"
+ ".inst 0x4f83e0f5 // sdot v21.4s, v7.16b, v3.4b[0]\n"
+ "ldr q7, [x14, #0x30]\n"
+ ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n"
+ "add x14, x14, #0x40\n"
+ ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n"
+ ".inst 0x4f83e0d6 // sdot v22.4s, v6.16b, v3.4b[0]\n"
+ ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n"
+ ".inst 0x4f83e0f7 // sdot v23.4s, v7.16b, v3.4b[0]\n"
+ "99:" // Height 4: Multiply loop: No odd multiplies
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "add x12, x12, #0x1\n"
+ "cmp x12, x19\n"
+ "bne 89b\n"
+ "prfm pstl1keep, [x13, #0x0]\n"
+ "prfm pstl1keep, [x9, #0x0]\n"
+ "ldr q0, [x16, #0x0]\n"
+ "add v8.4s, v8.4s, v0.4s\n"
+ "prfm pstl1keep, [x27, #0x0]\n"
+ "ldr q1, [x16, #0x10]\n"
+ "add v12.4s, v12.4s, v0.4s\n"
+ "prfm pstl1keep, [x25, #0x0]\n"
+ "ldr q2, [x16, #0x20]\n"
+ "add v16.4s, v16.4s, v0.4s\n"
+ "ldr q3, [x16, #0x30]\n"
+ "add v20.4s, v20.4s, v0.4s\n"
+ "add x16, x16, #0x40\n"
+ "add v9.4s, v9.4s, v1.4s\n"
+ "add v13.4s, v13.4s, v1.4s\n"
+ "add v10.4s, v10.4s, v2.4s\n"
+ "add v11.4s, v11.4s, v3.4s\n"
+ "add v14.4s, v14.4s, v2.4s\n"
+ "add v15.4s, v15.4s, v3.4s\n"
+ "add v17.4s, v17.4s, v1.4s\n"
+ "add v18.4s, v18.4s, v2.4s\n"
+ "add v19.4s, v19.4s, v3.4s\n"
+ "add v21.4s, v21.4s, v1.4s\n"
+ "add v22.4s, v22.4s, v2.4s\n"
+ "add v23.4s, v23.4s, v3.4s\n"
+ "tbz %x[flags], #4, 100f\n"
+ "ldr q0, [x17, #0x0]\n"
+ "ldr q4, [x8, #0x0]\n"
+ "ldr q1, [x17, #0x10]\n"
+ "ldr q5, [x8, #0x10]\n"
+ "ldr q2, [x17, #0x20]\n"
+ "ldr q6, [x8, #0x20]\n"
+ "ldr q3, [x17, #0x30]\n"
+ "ldr q7, [x8, #0x30]\n"
+ "add x17, x17, #0x40\n"
+ "add x8, x8, #0x40\n"
+ "b 101f\n"
+ "100:" // Height 4: per layer parameters
+ "add x19, %x[qp], %[per_layer_right_shift]\n"
+ "ld1r { v0.4s }, [x19]\n"
+ "mov v1.16b, v0.16b\n"
+ "add x19, %x[qp], %[per_layer_mul]\n"
+ "ld1r { v4.4s }, [x19]\n"
+ "mov v2.16b, v0.16b\n"
+ "mov v3.16b, v0.16b\n"
+ "mov v5.16b, v4.16b\n"
+ "mov v6.16b, v4.16b\n"
+ "mov v7.16b, v4.16b\n"
+ "101:" // Height 4: parameters loaded
+ "sqrdmulh v8.4s, v8.4s, v4.4s\n"
+ "sqrdmulh v9.4s, v9.4s, v5.4s\n"
+ "sqrdmulh v10.4s, v10.4s, v6.4s\n"
+ "sqrdmulh v11.4s, v11.4s, v7.4s\n"
+ "sqrdmulh v12.4s, v12.4s, v4.4s\n"
+ "sqrdmulh v13.4s, v13.4s, v5.4s\n"
+ "sqrdmulh v14.4s, v14.4s, v6.4s\n"
+ "sqrdmulh v15.4s, v15.4s, v7.4s\n"
+ "sqrdmulh v16.4s, v16.4s, v4.4s\n"
+ "sqrdmulh v17.4s, v17.4s, v5.4s\n"
+ "sqrdmulh v18.4s, v18.4s, v6.4s\n"
+ "sqrdmulh v19.4s, v19.4s, v7.4s\n"
+ "sqrdmulh v20.4s, v20.4s, v4.4s\n"
+ "sqrdmulh v21.4s, v21.4s, v5.4s\n"
+ "sqrdmulh v22.4s, v22.4s, v6.4s\n"
+ "sqrdmulh v23.4s, v23.4s, v7.4s\n"
+ "tbz %x[flags], #5, 102f\n"
+ "and v4.16b, v8.16b, v0.16b\n"
+ "sshr v4.4s, v4.4s, #0x1f\n"
+ "and v5.16b, v9.16b, v1.16b\n"
+ "and v6.16b, v10.16b, v2.16b\n"
+ "sshr v5.4s, v5.4s, #0x1f\n"
+ "and v7.16b, v11.16b, v3.16b\n"
+ "sshr v6.4s, v6.4s, #0x1f\n"
+ "sqadd v8.4s, v8.4s, v4.4s\n"
+ "sshr v7.4s, v7.4s, #0x1f\n"
+ "and v4.16b, v12.16b, v0.16b\n"
+ "sqadd v9.4s, v9.4s, v5.4s\n"
+ "sshr v4.4s, v4.4s, #0x1f\n"
+ "sqadd v10.4s, v10.4s, v6.4s\n"
+ "and v5.16b, v13.16b, v1.16b\n"
+ "sshr v5.4s, v5.4s, #0x1f\n"
+ "sqadd v11.4s, v11.4s, v7.4s\n"
+ "and v6.16b, v14.16b, v2.16b\n"
+ "sshr v6.4s, v6.4s, #0x1f\n"
+ "sqadd v12.4s, v12.4s, v4.4s\n"
+ "and v7.16b, v15.16b, v3.16b\n"
+ "sshr v7.4s, v7.4s, #0x1f\n"
+ "sqadd v13.4s, v13.4s, v5.4s\n"
+ "and v4.16b, v16.16b, v0.16b\n"
+ "sshr v4.4s, v4.4s, #0x1f\n"
+ "sqadd v14.4s, v14.4s, v6.4s\n"
+ "and v5.16b, v17.16b, v1.16b\n"
+ "sshr v5.4s, v5.4s, #0x1f\n"
+ "sqadd v15.4s, v15.4s, v7.4s\n"
+ "and v6.16b, v18.16b, v2.16b\n"
+ "sshr v6.4s, v6.4s, #0x1f\n"
+ "sqadd v16.4s, v16.4s, v4.4s\n"
+ "and v7.16b, v19.16b, v3.16b\n"
+ "sshr v7.4s, v7.4s, #0x1f\n"
+ "sqadd v17.4s, v17.4s, v5.4s\n"
+ "and v4.16b, v20.16b, v0.16b\n"
+ "sshr v4.4s, v4.4s, #0x1f\n"
+ "sqadd v18.4s, v18.4s, v6.4s\n"
+ "and v5.16b, v21.16b, v1.16b\n"
+ "sshr v5.4s, v5.4s, #0x1f\n"
+ "sqadd v19.4s, v19.4s, v7.4s\n"
+ "and v6.16b, v22.16b, v2.16b\n"
+ "sshr v6.4s, v6.4s, #0x1f\n"
+ "sqadd v20.4s, v20.4s, v4.4s\n"
+ "and v7.16b, v23.16b, v3.16b\n"
+ "sshr v7.4s, v7.4s, #0x1f\n"
+ "sqadd v21.4s, v21.4s, v5.4s\n"
+ "sqadd v22.4s, v22.4s, v6.4s\n"
+ "sqadd v23.4s, v23.4s, v7.4s\n"
+ "102:" // Height 4: no shift correction
+ "srshl v8.4s, v8.4s, v0.4s\n"
+ "add x19, %x[qp], %[c_offset]\n"
+ "ld1r { v4.4s }, [x19]\n"
+ "srshl v9.4s, v9.4s, v1.4s\n"
+ "add x19, %x[qp], %[minval]\n"
+ "srshl v10.4s, v10.4s, v2.4s\n"
+ "ld1r { v5.4s }, [x19]\n"
+ "add x19, %x[qp], %[maxval]\n"
+ "srshl v11.4s, v11.4s, v3.4s\n"
+ "ld1r { v6.4s }, [x19]\n"
+ "cmp x15, #0x10\n"
+ "srshl v12.4s, v12.4s, v0.4s\n"
+ "srshl v13.4s, v13.4s, v1.4s\n"
+ "srshl v14.4s, v14.4s, v2.4s\n"
+ "srshl v15.4s, v15.4s, v3.4s\n"
+ "add v8.4s, v8.4s, v4.4s\n"
+ "add v9.4s, v9.4s, v4.4s\n"
+ "add v10.4s, v10.4s, v4.4s\n"
+ "smin v8.4s, v8.4s, v6.4s\n"
+ "smin v9.4s, v9.4s, v6.4s\n"
+ "smin v10.4s, v10.4s, v6.4s\n"
+ "smax v8.4s, v8.4s, v5.4s\n"
+ "smax v9.4s, v9.4s, v5.4s\n"
+ "smax v10.4s, v10.4s, v5.4s\n"
+ "add v11.4s, v11.4s, v4.4s\n"
+ "add v12.4s, v12.4s, v4.4s\n"
+ "add v13.4s, v13.4s, v4.4s\n"
+ "smin v11.4s, v11.4s, v6.4s\n"
+ "smin v12.4s, v12.4s, v6.4s\n"
+ "smin v13.4s, v13.4s, v6.4s\n"
+ "smax v11.4s, v11.4s, v5.4s\n"
+ "smax v12.4s, v12.4s, v5.4s\n"
+ "smax v13.4s, v13.4s, v5.4s\n"
+ "add v14.4s, v14.4s, v4.4s\n"
+ "add v15.4s, v15.4s, v4.4s\n"
+ "srshl v16.4s, v16.4s, v0.4s\n"
+ "smin v14.4s, v14.4s, v6.4s\n"
+ "smin v15.4s, v15.4s, v6.4s\n"
+ "srshl v17.4s, v17.4s, v1.4s\n"
+ "smax v14.4s, v14.4s, v5.4s\n"
+ "smax v15.4s, v15.4s, v5.4s\n"
+ "add v16.4s, v16.4s, v4.4s\n"
+ "add v17.4s, v17.4s, v4.4s\n"
+ "srshl v18.4s, v18.4s, v2.4s\n"
+ "smin v16.4s, v16.4s, v6.4s\n"
+ "smin v17.4s, v17.4s, v6.4s\n"
+ "srshl v19.4s, v19.4s, v3.4s\n"
+ "smax v16.4s, v16.4s, v5.4s\n"
+ "smax v17.4s, v17.4s, v5.4s\n"
+ "add v18.4s, v18.4s, v4.4s\n"
+ "add v19.4s, v19.4s, v4.4s\n"
+ "srshl v20.4s, v20.4s, v0.4s\n"
+ "smin v18.4s, v18.4s, v6.4s\n"
+ "smin v19.4s, v19.4s, v6.4s\n"
+ "srshl v21.4s, v21.4s, v1.4s\n"
+ "smax v18.4s, v18.4s, v5.4s\n"
+ "smax v19.4s, v19.4s, v5.4s\n"
+ "add v20.4s, v20.4s, v4.4s\n"
+ "add v21.4s, v21.4s, v4.4s\n"
+ "srshl v22.4s, v22.4s, v2.4s\n"
+ "smin v20.4s, v20.4s, v6.4s\n"
+ "smin v21.4s, v21.4s, v6.4s\n"
+ "srshl v23.4s, v23.4s, v3.4s\n"
+ "smax v20.4s, v20.4s, v5.4s\n"
+ "smax v21.4s, v21.4s, v5.4s\n"
+ "add v22.4s, v22.4s, v4.4s\n"
+ "add v23.4s, v23.4s, v4.4s\n"
+ "uzp1 v8.8h, v8.8h, v9.8h\n"
+ "smin v22.4s, v22.4s, v6.4s\n"
+ "smin v23.4s, v23.4s, v6.4s\n"
+ "uzp1 v9.8h, v10.8h, v11.8h\n"
+ "smax v22.4s, v22.4s, v5.4s\n"
+ "smax v23.4s, v23.4s, v5.4s\n"
+ "uzp1 v12.8h, v12.8h, v13.8h\n"
+ "uzp1 v13.8h, v14.8h, v15.8h\n"
+ "uzp1 v16.8h, v16.8h, v17.8h\n"
+ "uzp1 v17.8h, v18.8h, v19.8h\n"
+ "uzp1 v20.8h, v20.8h, v21.8h\n"
+ "uzp1 v21.8h, v22.8h, v23.8h\n"
+ "uzp1 v8.16b, v8.16b, v9.16b\n"
+ "uzp1 v12.16b, v12.16b, v13.16b\n"
+ "uzp1 v16.16b, v16.16b, v17.16b\n"
+ "uzp1 v20.16b, v20.16b, v21.16b\n"
+ "bge 111f\n"
+ "tbz x15, #3, 106f\n"
+ "str d8, [x13], #0x8\n"
+ "str d12, [x9], #0x8\n"
+ "str d16, [x27], #0x8\n"
+ "str d20, [x25], #0x8\n"
+ "tbz x15, #2, 104f\n"
+ "st1 { v8.s }[2], [x13], #0x4\n"
+ "st1 { v12.s }[2], [x9], #0x4\n"
+ "st1 { v16.s }[2], [x27], #0x4\n"
+ "st1 { v20.s }[2], [x25], #0x4\n"
+ "tbz x15, #1, 103f\n"
+ "st1 { v8.h }[6], [x13], #0x2\n"
+ "st1 { v12.h }[6], [x9], #0x2\n"
+ "st1 { v16.h }[6], [x27], #0x2\n"
+ "st1 { v20.h }[6], [x25], #0x2\n"
+ "tbz x15, #0, 110f\n"
+ "st1 { v8.b }[14], [x13]\n"
+ "st1 { v12.b }[14], [x9]\n"
+ "st1 { v16.b }[14], [x27]\n"
+ "st1 { v20.b }[14], [x25]\n"
+ "b 110f\n"
+ "103:" // Height 4: Partial direct writeback: partial_1_12
+ "tbz x15, #0, 110f\n"
+ "st1 { v8.b }[12], [x13]\n"
+ "st1 { v12.b }[12], [x9]\n"
+ "st1 { v16.b }[12], [x27]\n"
+ "st1 { v20.b }[12], [x25]\n"
+ "b 110f\n"
+ "104:" // Height 4: Partial direct writeback: partial_2_8
+ "tbz x15, #1, 105f\n"
+ "st1 { v8.h }[4], [x13], #0x2\n"
+ "st1 { v12.h }[4], [x9], #0x2\n"
+ "st1 { v16.h }[4], [x27], #0x2\n"
+ "st1 { v20.h }[4], [x25], #0x2\n"
+ "tbz x15, #0, 110f\n"
+ "st1 { v8.b }[10], [x13]\n"
+ "st1 { v12.b }[10], [x9]\n"
+ "st1 { v16.b }[10], [x27]\n"
+ "st1 { v20.b }[10], [x25]\n"
+ "b 110f\n"
+ "105:" // Height 4: Partial direct writeback: partial_1_8
+ "tbz x15, #0, 110f\n"
+ "st1 { v8.b }[8], [x13]\n"
+ "st1 { v12.b }[8], [x9]\n"
+ "st1 { v16.b }[8], [x27]\n"
+ "st1 { v20.b }[8], [x25]\n"
+ "b 110f\n"
+ "106:" // Height 4: Partial direct writeback: partial_4_0
+ "tbz x15, #2, 108f\n"
+ "str s8, [x13], #0x4\n"
+ "str s12, [x9], #0x4\n"
+ "str s16, [x27], #0x4\n"
+ "str s20, [x25], #0x4\n"
+ "tbz x15, #1, 107f\n"
+ "st1 { v8.h }[2], [x13], #0x2\n"
+ "st1 { v12.h }[2], [x9], #0x2\n"
+ "st1 { v16.h }[2], [x27], #0x2\n"
+ "st1 { v20.h }[2], [x25], #0x2\n"
+ "tbz x15, #0, 110f\n"
+ "st1 { v8.b }[6], [x13]\n"
+ "st1 { v12.b }[6], [x9]\n"
+ "st1 { v16.b }[6], [x27]\n"
+ "st1 { v20.b }[6], [x25]\n"
+ "b 110f\n"
+ "107:" // Height 4: Partial direct writeback: partial_1_4
+ "tbz x15, #0, 110f\n"
+ "st1 { v8.b }[4], [x13]\n"
+ "st1 { v12.b }[4], [x9]\n"
+ "st1 { v16.b }[4], [x27]\n"
+ "st1 { v20.b }[4], [x25]\n"
+ "b 110f\n"
+ "108:" // Height 4: Partial direct writeback: partial_2_0
+ "tbz x15, #1, 109f\n"
+ "str h8, [x13], #0x2\n"
+ "str h12, [x9], #0x2\n"
+ "str h16, [x27], #0x2\n"
+ "str h20, [x25], #0x2\n"
+ "tbz x15, #0, 110f\n"
+ "st1 { v8.b }[2], [x13]\n"
+ "st1 { v12.b }[2], [x9]\n"
+ "st1 { v16.b }[2], [x27]\n"
+ "st1 { v20.b }[2], [x25]\n"
+ "b 110f\n"
+ "109:" // Height 4: Partial direct writeback: partial_1_0
+ "str b8, [x13, #0x0]\n"
+ "str b12, [x9, #0x0]\n"
+ "str b16, [x27, #0x0]\n"
+ "str b20, [x25, #0x0]\n"
+ "110:" // Height 4: Partial direct writeback: Done
+ "b 112f\n"
+ "111:" // Height 4: Full writeback
+ "str q8, [x13, #0x0]\n"
+ "str q12, [x9, #0x0]\n"
+ "str q16, [x27, #0x0]\n"
+ "str q20, [x25, #0x0]\n"
+ "add x13, x13, #0x10\n"
+ "add x9, x9, #0x10\n"
+ "add x27, x27, #0x10\n"
+ "add x25, x25, #0x10\n"
+ "112:" // Height 4: Writeback done
+ "subs x15, x15, #0x10\n"
+ "bgt 87b\n"
+ "b 170f\n"
+ "113:" // Height 5
+ "ldr x8, [%x[args_ptr], %[offsetof_multiplier_ptr]]\n"
+ "mov x16, %x[col_bias]\n"
+ "ldr x17, [%x[args_ptr], %[offsetof_shift_ptr]]\n"
+ "ldr x15, [%x[args_ptr], %[offsetof_N]]\n"
+ "ldr x14, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 114f\n"
+ "ldr x13, [%x[output_ptr], #0x0]\n"
+ "add x13, x13, x19\n"
+ "ldr x9, [%x[output_ptr], #0x8]\n"
+ "ldr x27, [%x[output_ptr], #0x10]\n"
+ "add x9, x9, x19\n"
+ "ldr x25, [%x[output_ptr], #0x18]\n"
+ "ldr x23, [%x[output_ptr], #0x20]\n"
+ "add x27, x27, x19\n"
+ "add x25, x25, x19\n"
+ "add x23, x23, x19\n"
+ "b 115f\n"
+ "114:" // Height 5: setup direct output
+ "mov x13, %x[output_ptr]\n"
+ "add x9, x13, x19\n"
+ "add x27, x9, x19\n"
+ "add x25, x27, x19\n"
+ "add x23, x25, x19\n"
+ "115:" // Height 5: Column loop
+ "movi v8.4s, #0x0\n"
+ "movi v9.4s, #0x0\n"
+ "movi v10.4s, #0x0\n"
+ "movi v11.4s, #0x0\n"
+ "movi v12.4s, #0x0\n"
+ "movi v13.4s, #0x0\n"
+ "movi v14.4s, #0x0\n"
+ "movi v15.4s, #0x0\n"
+ "movi v16.4s, #0x0\n"
+ "movi v17.4s, #0x0\n"
+ "movi v18.4s, #0x0\n"
+ "movi v19.4s, #0x0\n"
+ "movi v20.4s, #0x0\n"
+ "movi v21.4s, #0x0\n"
+ "movi v22.4s, #0x0\n"
+ "movi v23.4s, #0x0\n"
+ "movi v24.4s, #0x0\n"
+ "movi v25.4s, #0x0\n"
+ "movi v26.4s, #0x0\n"
+ "movi v27.4s, #0x0\n"
+ "116:" // Height 5: setup done
+ "mov x12, #0x0\n"
+ "117:" // Height 5: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w11, [x20, x12, LSL #0x2]\n"
+ "tbz %x[flags], #3, 118f\n"
+ "ldr x20, [%x[input_ptr], x12, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x10, [x20, #0x0]\n"
+ "ldr x28, [x20, #0x8]\n"
+ "ldr x26, [x20, #0x10]\n"
+ "ldr x24, [x20, #0x18]\n"
+ "ldr x22, [x20, #0x20]\n"
+ "cbnz x12, 119f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x10, x10, x19\n"
+ "add x28, x28, x19\n"
+ "add x26, x26, x19\n"
+ "add x24, x24, x19\n"
+ "add x22, x22, x19\n"
+ "b 119f\n"
+ "118:" // Height 5: setup direct input
+ "mov x10, %x[input_ptr]\n"
+ "add x28, x10, x19\n"
+ "add x26, x28, x19\n"
+ "add x24, x26, x19\n"
+ "add x22, x24, x19\n"
+ "119:" // Height 5: input setup done
+ "cmp x11, #0x10\n"
+ "blt 122f\n"
+ "cmp x11, #0x20\n"
+ "blt 121f\n"
+ "120:" // Height 5: Multiply loop: Main loop head
+ "ldr q0, [x10, #0x0]\n"
+ "ldr q1, [x28, #0x0]\n"
+ "ldr q2, [x26, #0x0]\n"
+ "ldr q3, [x24, #0x0]\n"
+ "ldr q4, [x22, #0x0]\n"
+ "ldr q6, [x14, #0x0]\n"
+ ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n"
+ "ldr q7, [x14, #0x10]\n"
+ ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n"
+ "add x10, x10, #0x10\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n"
+ "add x28, x28, #0x10\n"
+ ".inst 0x4f83e0d4 // sdot v20.4s, v6.16b, v3.4b[0]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "add x26, x26, #0x10\n"
+ ".inst 0x4f84e0d8 // sdot v24.4s, v6.16b, v4.4b[0]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "ldr q6, [x14, #0x20]\n"
+ ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n"
+ "add x24, x24, #0x10\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n"
+ "add x22, x22, #0x10\n"
+ ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n"
+ "prfm pldl1keep, [x22, #0x80]\n"
+ "sub x11, x11, #0x10\n"
+ ".inst 0x4f83e0f5 // sdot v21.4s, v7.16b, v3.4b[0]\n"
+ "cmp x11, #0x20\n"
+ ".inst 0x4f84e0f9 // sdot v25.4s, v7.16b, v4.4b[0]\n"
+ "ldr q7, [x14, #0x30]\n"
+ ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n"
+ ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n"
+ ".inst 0x4f83e0d6 // sdot v22.4s, v6.16b, v3.4b[0]\n"
+ ".inst 0x4f84e0da // sdot v26.4s, v6.16b, v4.4b[0]\n"
+ "ldr q6, [x14, #0x40]\n"
+ ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n"
+ ".inst 0x4f83e0f7 // sdot v23.4s, v7.16b, v3.4b[0]\n"
+ ".inst 0x4f84e0fb // sdot v27.4s, v7.16b, v4.4b[0]\n"
+ "ldr q7, [x14, #0x50]\n"
+ ".inst 0x4fa0e0c8 // sdot v8.4s, v6.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e0cc // sdot v12.4s, v6.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e0d0 // sdot v16.4s, v6.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e0d4 // sdot v20.4s, v6.16b, v3.4b[1]\n"
+ ".inst 0x4fa4e0d8 // sdot v24.4s, v6.16b, v4.4b[1]\n"
+ "ldr q6, [x14, #0x60]\n"
+ ".inst 0x4fa0e0e9 // sdot v9.4s, v7.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e0ed // sdot v13.4s, v7.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e0f1 // sdot v17.4s, v7.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e0f5 // sdot v21.4s, v7.16b, v3.4b[1]\n"
+ ".inst 0x4fa4e0f9 // sdot v25.4s, v7.16b, v4.4b[1]\n"
+ "ldr q7, [x14, #0x70]\n"
+ ".inst 0x4fa0e0ca // sdot v10.4s, v6.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e0ce // sdot v14.4s, v6.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e0d2 // sdot v18.4s, v6.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e0d6 // sdot v22.4s, v6.16b, v3.4b[1]\n"
+ ".inst 0x4fa4e0da // sdot v26.4s, v6.16b, v4.4b[1]\n"
+ "ldr q6, [x14, #0x80]\n"
+ ".inst 0x4fa0e0eb // sdot v11.4s, v7.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e0ef // sdot v15.4s, v7.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e0f3 // sdot v19.4s, v7.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e0f7 // sdot v23.4s, v7.16b, v3.4b[1]\n"
+ ".inst 0x4fa4e0fb // sdot v27.4s, v7.16b, v4.4b[1]\n"
+ "ldr q7, [x14, #0x90]\n"
+ ".inst 0x4f80e8c8 // sdot v8.4s, v6.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8cc // sdot v12.4s, v6.16b, v1.4b[2]\n"
+ ".inst 0x4f82e8d0 // sdot v16.4s, v6.16b, v2.4b[2]\n"
+ ".inst 0x4f83e8d4 // sdot v20.4s, v6.16b, v3.4b[2]\n"
+ ".inst 0x4f84e8d8 // sdot v24.4s, v6.16b, v4.4b[2]\n"
+ "ldr q6, [x14, #0xa0]\n"
+ ".inst 0x4f80e8e9 // sdot v9.4s, v7.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8ed // sdot v13.4s, v7.16b, v1.4b[2]\n"
+ ".inst 0x4f82e8f1 // sdot v17.4s, v7.16b, v2.4b[2]\n"
+ ".inst 0x4f83e8f5 // sdot v21.4s, v7.16b, v3.4b[2]\n"
+ ".inst 0x4f84e8f9 // sdot v25.4s, v7.16b, v4.4b[2]\n"
+ "ldr q7, [x14, #0xb0]\n"
+ ".inst 0x4f80e8ca // sdot v10.4s, v6.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8ce // sdot v14.4s, v6.16b, v1.4b[2]\n"
+ ".inst 0x4f82e8d2 // sdot v18.4s, v6.16b, v2.4b[2]\n"
+ ".inst 0x4f83e8d6 // sdot v22.4s, v6.16b, v3.4b[2]\n"
+ ".inst 0x4f84e8da // sdot v26.4s, v6.16b, v4.4b[2]\n"
+ "ldr q6, [x14, #0xc0]\n"
+ ".inst 0x4f80e8eb // sdot v11.4s, v7.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8ef // sdot v15.4s, v7.16b, v1.4b[2]\n"
+ ".inst 0x4f82e8f3 // sdot v19.4s, v7.16b, v2.4b[2]\n"
+ ".inst 0x4f83e8f7 // sdot v23.4s, v7.16b, v3.4b[2]\n"
+ ".inst 0x4f84e8fb // sdot v27.4s, v7.16b, v4.4b[2]\n"
+ "ldr q7, [x14, #0xd0]\n"
+ ".inst 0x4fa0e8c8 // sdot v8.4s, v6.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e8cc // sdot v12.4s, v6.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e8d0 // sdot v16.4s, v6.16b, v2.4b[3]\n"
+ ".inst 0x4fa3e8d4 // sdot v20.4s, v6.16b, v3.4b[3]\n"
+ ".inst 0x4fa4e8d8 // sdot v24.4s, v6.16b, v4.4b[3]\n"
+ "ldr q6, [x14, #0xe0]\n"
+ ".inst 0x4fa0e8e9 // sdot v9.4s, v7.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e8ed // sdot v13.4s, v7.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e8f1 // sdot v17.4s, v7.16b, v2.4b[3]\n"
+ ".inst 0x4fa3e8f5 // sdot v21.4s, v7.16b, v3.4b[3]\n"
+ ".inst 0x4fa4e8f9 // sdot v25.4s, v7.16b, v4.4b[3]\n"
+ "ldr q7, [x14, #0xf0]\n"
+ ".inst 0x4fa0e8ca // sdot v10.4s, v6.16b, v0.4b[3]\n"
+ "add x14, x14, #0x100\n"
+ ".inst 0x4fa1e8ce // sdot v14.4s, v6.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e8d2 // sdot v18.4s, v6.16b, v2.4b[3]\n"
+ ".inst 0x4fa3e8d6 // sdot v22.4s, v6.16b, v3.4b[3]\n"
+ ".inst 0x4fa4e8da // sdot v26.4s, v6.16b, v4.4b[3]\n"
+ ".inst 0x4fa0e8eb // sdot v11.4s, v7.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e8ef // sdot v15.4s, v7.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e8f3 // sdot v19.4s, v7.16b, v2.4b[3]\n"
+ ".inst 0x4fa3e8f7 // sdot v23.4s, v7.16b, v3.4b[3]\n"
+ ".inst 0x4fa4e8fb // sdot v27.4s, v7.16b, v4.4b[3]\n"
+ "bge 120b\n"
+ "121:" // Height 5: Multiply loop: Single iteration only
+ "sub x11, x11, #0x10\n"
+ "ldr q0, [x10, #0x0]\n"
+ "ldr q1, [x28, #0x0]\n"
+ "ldr q2, [x26, #0x0]\n"
+ "ldr q3, [x24, #0x0]\n"
+ "ldr q4, [x22, #0x0]\n"
+ "ldr q6, [x14, #0x0]\n"
+ ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n"
+ "ldr q7, [x14, #0x10]\n"
+ ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n"
+ "add x10, x10, #0x10\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n"
+ "add x28, x28, #0x10\n"
+ ".inst 0x4f83e0d4 // sdot v20.4s, v6.16b, v3.4b[0]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "add x26, x26, #0x10\n"
+ ".inst 0x4f84e0d8 // sdot v24.4s, v6.16b, v4.4b[0]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "ldr q6, [x14, #0x20]\n"
+ ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n"
+ "add x24, x24, #0x10\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n"
+ "add x22, x22, #0x10\n"
+ ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n"
+ "prfm pldl1keep, [x22, #0x80]\n"
+ ".inst 0x4f83e0f5 // sdot v21.4s, v7.16b, v3.4b[0]\n"
+ ".inst 0x4f84e0f9 // sdot v25.4s, v7.16b, v4.4b[0]\n"
+ "ldr q7, [x14, #0x30]\n"
+ ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n"
+ ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n"
+ ".inst 0x4f83e0d6 // sdot v22.4s, v6.16b, v3.4b[0]\n"
+ ".inst 0x4f84e0da // sdot v26.4s, v6.16b, v4.4b[0]\n"
+ "ldr q6, [x14, #0x40]\n"
+ ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n"
+ ".inst 0x4f83e0f7 // sdot v23.4s, v7.16b, v3.4b[0]\n"
+ ".inst 0x4f84e0fb // sdot v27.4s, v7.16b, v4.4b[0]\n"
+ "ldr q7, [x14, #0x50]\n"
+ ".inst 0x4fa0e0c8 // sdot v8.4s, v6.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e0cc // sdot v12.4s, v6.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e0d0 // sdot v16.4s, v6.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e0d4 // sdot v20.4s, v6.16b, v3.4b[1]\n"
+ ".inst 0x4fa4e0d8 // sdot v24.4s, v6.16b, v4.4b[1]\n"
+ "ldr q6, [x14, #0x60]\n"
+ ".inst 0x4fa0e0e9 // sdot v9.4s, v7.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e0ed // sdot v13.4s, v7.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e0f1 // sdot v17.4s, v7.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e0f5 // sdot v21.4s, v7.16b, v3.4b[1]\n"
+ ".inst 0x4fa4e0f9 // sdot v25.4s, v7.16b, v4.4b[1]\n"
+ "ldr q7, [x14, #0x70]\n"
+ ".inst 0x4fa0e0ca // sdot v10.4s, v6.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e0ce // sdot v14.4s, v6.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e0d2 // sdot v18.4s, v6.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e0d6 // sdot v22.4s, v6.16b, v3.4b[1]\n"
+ ".inst 0x4fa4e0da // sdot v26.4s, v6.16b, v4.4b[1]\n"
+ "ldr q6, [x14, #0x80]\n"
+ ".inst 0x4fa0e0eb // sdot v11.4s, v7.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e0ef // sdot v15.4s, v7.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e0f3 // sdot v19.4s, v7.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e0f7 // sdot v23.4s, v7.16b, v3.4b[1]\n"
+ ".inst 0x4fa4e0fb // sdot v27.4s, v7.16b, v4.4b[1]\n"
+ "ldr q7, [x14, #0x90]\n"
+ ".inst 0x4f80e8c8 // sdot v8.4s, v6.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8cc // sdot v12.4s, v6.16b, v1.4b[2]\n"
+ ".inst 0x4f82e8d0 // sdot v16.4s, v6.16b, v2.4b[2]\n"
+ ".inst 0x4f83e8d4 // sdot v20.4s, v6.16b, v3.4b[2]\n"
+ ".inst 0x4f84e8d8 // sdot v24.4s, v6.16b, v4.4b[2]\n"
+ "ldr q6, [x14, #0xa0]\n"
+ ".inst 0x4f80e8e9 // sdot v9.4s, v7.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8ed // sdot v13.4s, v7.16b, v1.4b[2]\n"
+ ".inst 0x4f82e8f1 // sdot v17.4s, v7.16b, v2.4b[2]\n"
+ ".inst 0x4f83e8f5 // sdot v21.4s, v7.16b, v3.4b[2]\n"
+ ".inst 0x4f84e8f9 // sdot v25.4s, v7.16b, v4.4b[2]\n"
+ "ldr q7, [x14, #0xb0]\n"
+ ".inst 0x4f80e8ca // sdot v10.4s, v6.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8ce // sdot v14.4s, v6.16b, v1.4b[2]\n"
+ ".inst 0x4f82e8d2 // sdot v18.4s, v6.16b, v2.4b[2]\n"
+ ".inst 0x4f83e8d6 // sdot v22.4s, v6.16b, v3.4b[2]\n"
+ ".inst 0x4f84e8da // sdot v26.4s, v6.16b, v4.4b[2]\n"
+ "ldr q6, [x14, #0xc0]\n"
+ ".inst 0x4f80e8eb // sdot v11.4s, v7.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8ef // sdot v15.4s, v7.16b, v1.4b[2]\n"
+ ".inst 0x4f82e8f3 // sdot v19.4s, v7.16b, v2.4b[2]\n"
+ ".inst 0x4f83e8f7 // sdot v23.4s, v7.16b, v3.4b[2]\n"
+ ".inst 0x4f84e8fb // sdot v27.4s, v7.16b, v4.4b[2]\n"
+ "ldr q7, [x14, #0xd0]\n"
+ ".inst 0x4fa0e8c8 // sdot v8.4s, v6.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e8cc // sdot v12.4s, v6.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e8d0 // sdot v16.4s, v6.16b, v2.4b[3]\n"
+ ".inst 0x4fa3e8d4 // sdot v20.4s, v6.16b, v3.4b[3]\n"
+ ".inst 0x4fa4e8d8 // sdot v24.4s, v6.16b, v4.4b[3]\n"
+ "ldr q6, [x14, #0xe0]\n"
+ ".inst 0x4fa0e8e9 // sdot v9.4s, v7.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e8ed // sdot v13.4s, v7.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e8f1 // sdot v17.4s, v7.16b, v2.4b[3]\n"
+ ".inst 0x4fa3e8f5 // sdot v21.4s, v7.16b, v3.4b[3]\n"
+ ".inst 0x4fa4e8f9 // sdot v25.4s, v7.16b, v4.4b[3]\n"
+ "ldr q7, [x14, #0xf0]\n"
+ ".inst 0x4fa0e8ca // sdot v10.4s, v6.16b, v0.4b[3]\n"
+ "add x14, x14, #0x100\n"
+ ".inst 0x4fa1e8ce // sdot v14.4s, v6.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e8d2 // sdot v18.4s, v6.16b, v2.4b[3]\n"
+ ".inst 0x4fa3e8d6 // sdot v22.4s, v6.16b, v3.4b[3]\n"
+ ".inst 0x4fa4e8da // sdot v26.4s, v6.16b, v4.4b[3]\n"
+ ".inst 0x4fa0e8eb // sdot v11.4s, v7.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e8ef // sdot v15.4s, v7.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e8f3 // sdot v19.4s, v7.16b, v2.4b[3]\n"
+ ".inst 0x4fa3e8f7 // sdot v23.4s, v7.16b, v3.4b[3]\n"
+ ".inst 0x4fa4e8fb // sdot v27.4s, v7.16b, v4.4b[3]\n"
+ "122:" // Height 5: Multiply loop: Main loop skip
+ "cbz x11, 127f\n"
+ "cmp x11, #0x4\n"
+ "blt 124f\n"
+ "123:" // Height 5: Multiply loop: Odd block loop
+ "ldr s0, [x10], #0x4\n"
+ "ldr s1, [x28], #0x4\n"
+ "ldr s2, [x26], #0x4\n"
+ "ldr s3, [x24], #0x4\n"
+ "ldr s4, [x22], #0x4\n"
+ "ldr q6, [x14, #0x0]\n"
+ ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n"
+ "ldr q7, [x14, #0x10]\n"
+ ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n"
+ "sub x11, x11, #0x4\n"
+ ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n"
+ "cmp x11, #0x4\n"
+ ".inst 0x4f83e0d4 // sdot v20.4s, v6.16b, v3.4b[0]\n"
+ ".inst 0x4f84e0d8 // sdot v24.4s, v6.16b, v4.4b[0]\n"
+ "ldr q6, [x14, #0x20]\n"
+ ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n"
+ ".inst 0x4f83e0f5 // sdot v21.4s, v7.16b, v3.4b[0]\n"
+ ".inst 0x4f84e0f9 // sdot v25.4s, v7.16b, v4.4b[0]\n"
+ "ldr q7, [x14, #0x30]\n"
+ ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n"
+ "add x14, x14, #0x40\n"
+ ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n"
+ ".inst 0x4f83e0d6 // sdot v22.4s, v6.16b, v3.4b[0]\n"
+ ".inst 0x4f84e0da // sdot v26.4s, v6.16b, v4.4b[0]\n"
+ ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n"
+ ".inst 0x4f83e0f7 // sdot v23.4s, v7.16b, v3.4b[0]\n"
+ ".inst 0x4f84e0fb // sdot v27.4s, v7.16b, v4.4b[0]\n"
+ "bge 123b\n"
+ "cbz x11, 127f\n"
+ "124:" // Height 5: Multiply loop: Skip odd blocks
+ "tbz x11, #1, 125f\n"
+ "ldr h0, [x10], #0x2\n"
+ "ldr h1, [x28], #0x2\n"
+ "ldr h2, [x26], #0x2\n"
+ "ldr h3, [x24], #0x2\n"
+ "ldr h4, [x22], #0x2\n"
+ "tbz x11, #0, 126f\n"
+ "ld1 { v0.b }[2], [x10]\n"
+ "ld1 { v1.b }[2], [x28]\n"
+ "ld1 { v2.b }[2], [x26]\n"
+ "ld1 { v3.b }[2], [x24]\n"
+ "ld1 { v4.b }[2], [x22]\n"
+ "b 126f\n"
+ "125:" // Height 5: Multiply loop: Ragged operand read: partial_1_0
+ "ldr b0, [x10, #0x0]\n"
+ "ldr b1, [x28, #0x0]\n"
+ "ldr b2, [x26, #0x0]\n"
+ "ldr b3, [x24, #0x0]\n"
+ "ldr b4, [x22, #0x0]\n"
+ "126:" // Height 5: Multiply loop: Ragged operand read: Done
+ "ldr q6, [x14, #0x0]\n"
+ ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n"
+ "ldr q7, [x14, #0x10]\n"
+ ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n"
+ ".inst 0x4f83e0d4 // sdot v20.4s, v6.16b, v3.4b[0]\n"
+ ".inst 0x4f84e0d8 // sdot v24.4s, v6.16b, v4.4b[0]\n"
+ "ldr q6, [x14, #0x20]\n"
+ ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n"
+ ".inst 0x4f83e0f5 // sdot v21.4s, v7.16b, v3.4b[0]\n"
+ ".inst 0x4f84e0f9 // sdot v25.4s, v7.16b, v4.4b[0]\n"
+ "ldr q7, [x14, #0x30]\n"
+ ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n"
+ "add x14, x14, #0x40\n"
+ ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n"
+ ".inst 0x4f83e0d6 // sdot v22.4s, v6.16b, v3.4b[0]\n"
+ ".inst 0x4f84e0da // sdot v26.4s, v6.16b, v4.4b[0]\n"
+ ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n"
+ ".inst 0x4f83e0f7 // sdot v23.4s, v7.16b, v3.4b[0]\n"
+ ".inst 0x4f84e0fb // sdot v27.4s, v7.16b, v4.4b[0]\n"
+ "127:" // Height 5: Multiply loop: No odd multiplies
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "add x12, x12, #0x1\n"
+ "cmp x12, x19\n"
+ "bne 117b\n"
+ "prfm pstl1keep, [x13, #0x0]\n"
+ "prfm pstl1keep, [x9, #0x0]\n"
+ "ldr q0, [x16, #0x0]\n"
+ "add v8.4s, v8.4s, v0.4s\n"
+ "prfm pstl1keep, [x27, #0x0]\n"
+ "ldr q1, [x16, #0x10]\n"
+ "add v12.4s, v12.4s, v0.4s\n"
+ "prfm pstl1keep, [x25, #0x0]\n"
+ "ldr q2, [x16, #0x20]\n"
+ "add v16.4s, v16.4s, v0.4s\n"
+ "prfm pstl1keep, [x23, #0x0]\n"
+ "ldr q3, [x16, #0x30]\n"
+ "add v20.4s, v20.4s, v0.4s\n"
+ "add x16, x16, #0x40\n"
+ "add v24.4s, v24.4s, v0.4s\n"
+ "add v9.4s, v9.4s, v1.4s\n"
+ "add v13.4s, v13.4s, v1.4s\n"
+ "add v10.4s, v10.4s, v2.4s\n"
+ "add v11.4s, v11.4s, v3.4s\n"
+ "add v14.4s, v14.4s, v2.4s\n"
+ "add v15.4s, v15.4s, v3.4s\n"
+ "add v17.4s, v17.4s, v1.4s\n"
+ "add v18.4s, v18.4s, v2.4s\n"
+ "add v19.4s, v19.4s, v3.4s\n"
+ "add v21.4s, v21.4s, v1.4s\n"
+ "add v22.4s, v22.4s, v2.4s\n"
+ "add v23.4s, v23.4s, v3.4s\n"
+ "add v25.4s, v25.4s, v1.4s\n"
+ "add v26.4s, v26.4s, v2.4s\n"
+ "add v27.4s, v27.4s, v3.4s\n"
+ "tbz %x[flags], #4, 128f\n"
+ "ldr q0, [x17, #0x0]\n"
+ "ldr q4, [x8, #0x0]\n"
+ "ldr q1, [x17, #0x10]\n"
+ "ldr q5, [x8, #0x10]\n"
+ "ldr q2, [x17, #0x20]\n"
+ "ldr q6, [x8, #0x20]\n"
+ "ldr q3, [x17, #0x30]\n"
+ "ldr q7, [x8, #0x30]\n"
+ "add x17, x17, #0x40\n"
+ "add x8, x8, #0x40\n"
+ "b 129f\n"
+ "128:" // Height 5: per layer parameters
+ "add x19, %x[qp], %[per_layer_right_shift]\n"
+ "ld1r { v0.4s }, [x19]\n"
+ "mov v1.16b, v0.16b\n"
+ "add x19, %x[qp], %[per_layer_mul]\n"
+ "ld1r { v4.4s }, [x19]\n"
+ "mov v2.16b, v0.16b\n"
+ "mov v3.16b, v0.16b\n"
+ "mov v5.16b, v4.16b\n"
+ "mov v6.16b, v4.16b\n"
+ "mov v7.16b, v4.16b\n"
+ "129:" // Height 5: parameters loaded
+ "sqrdmulh v8.4s, v8.4s, v4.4s\n"
+ "sqrdmulh v9.4s, v9.4s, v5.4s\n"
+ "sqrdmulh v10.4s, v10.4s, v6.4s\n"
+ "sqrdmulh v11.4s, v11.4s, v7.4s\n"
+ "sqrdmulh v12.4s, v12.4s, v4.4s\n"
+ "sqrdmulh v13.4s, v13.4s, v5.4s\n"
+ "sqrdmulh v14.4s, v14.4s, v6.4s\n"
+ "sqrdmulh v15.4s, v15.4s, v7.4s\n"
+ "sqrdmulh v16.4s, v16.4s, v4.4s\n"
+ "sqrdmulh v17.4s, v17.4s, v5.4s\n"
+ "sqrdmulh v18.4s, v18.4s, v6.4s\n"
+ "sqrdmulh v19.4s, v19.4s, v7.4s\n"
+ "sqrdmulh v20.4s, v20.4s, v4.4s\n"
+ "sqrdmulh v21.4s, v21.4s, v5.4s\n"
+ "sqrdmulh v22.4s, v22.4s, v6.4s\n"
+ "sqrdmulh v23.4s, v23.4s, v7.4s\n"
+ "sqrdmulh v24.4s, v24.4s, v4.4s\n"
+ "sqrdmulh v25.4s, v25.4s, v5.4s\n"
+ "sqrdmulh v26.4s, v26.4s, v6.4s\n"
+ "sqrdmulh v27.4s, v27.4s, v7.4s\n"
+ "tbz %x[flags], #5, 130f\n"
+ "and v4.16b, v8.16b, v0.16b\n"
+ "sshr v4.4s, v4.4s, #0x1f\n"
+ "and v5.16b, v9.16b, v1.16b\n"
+ "and v6.16b, v10.16b, v2.16b\n"
+ "sshr v5.4s, v5.4s, #0x1f\n"
+ "and v7.16b, v11.16b, v3.16b\n"
+ "sshr v6.4s, v6.4s, #0x1f\n"
+ "sqadd v8.4s, v8.4s, v4.4s\n"
+ "sshr v7.4s, v7.4s, #0x1f\n"
+ "and v4.16b, v12.16b, v0.16b\n"
+ "sqadd v9.4s, v9.4s, v5.4s\n"
+ "sshr v4.4s, v4.4s, #0x1f\n"
+ "sqadd v10.4s, v10.4s, v6.4s\n"
+ "and v5.16b, v13.16b, v1.16b\n"
+ "sshr v5.4s, v5.4s, #0x1f\n"
+ "sqadd v11.4s, v11.4s, v7.4s\n"
+ "and v6.16b, v14.16b, v2.16b\n"
+ "sshr v6.4s, v6.4s, #0x1f\n"
+ "sqadd v12.4s, v12.4s, v4.4s\n"
+ "and v7.16b, v15.16b, v3.16b\n"
+ "sshr v7.4s, v7.4s, #0x1f\n"
+ "sqadd v13.4s, v13.4s, v5.4s\n"
+ "and v4.16b, v16.16b, v0.16b\n"
+ "sshr v4.4s, v4.4s, #0x1f\n"
+ "sqadd v14.4s, v14.4s, v6.4s\n"
+ "and v5.16b, v17.16b, v1.16b\n"
+ "sshr v5.4s, v5.4s, #0x1f\n"
+ "sqadd v15.4s, v15.4s, v7.4s\n"
+ "and v6.16b, v18.16b, v2.16b\n"
+ "sshr v6.4s, v6.4s, #0x1f\n"
+ "sqadd v16.4s, v16.4s, v4.4s\n"
+ "and v7.16b, v19.16b, v3.16b\n"
+ "sshr v7.4s, v7.4s, #0x1f\n"
+ "sqadd v17.4s, v17.4s, v5.4s\n"
+ "and v4.16b, v20.16b, v0.16b\n"
+ "sshr v4.4s, v4.4s, #0x1f\n"
+ "sqadd v18.4s, v18.4s, v6.4s\n"
+ "and v5.16b, v21.16b, v1.16b\n"
+ "sshr v5.4s, v5.4s, #0x1f\n"
+ "sqadd v19.4s, v19.4s, v7.4s\n"
+ "and v6.16b, v22.16b, v2.16b\n"
+ "sshr v6.4s, v6.4s, #0x1f\n"
+ "sqadd v20.4s, v20.4s, v4.4s\n"
+ "and v7.16b, v23.16b, v3.16b\n"
+ "sshr v7.4s, v7.4s, #0x1f\n"
+ "sqadd v21.4s, v21.4s, v5.4s\n"
+ "and v4.16b, v24.16b, v0.16b\n"
+ "sshr v4.4s, v4.4s, #0x1f\n"
+ "sqadd v22.4s, v22.4s, v6.4s\n"
+ "and v5.16b, v25.16b, v1.16b\n"
+ "sshr v5.4s, v5.4s, #0x1f\n"
+ "sqadd v23.4s, v23.4s, v7.4s\n"
+ "and v6.16b, v26.16b, v2.16b\n"
+ "sshr v6.4s, v6.4s, #0x1f\n"
+ "sqadd v24.4s, v24.4s, v4.4s\n"
+ "and v7.16b, v27.16b, v3.16b\n"
+ "sshr v7.4s, v7.4s, #0x1f\n"
+ "sqadd v25.4s, v25.4s, v5.4s\n"
+ "sqadd v26.4s, v26.4s, v6.4s\n"
+ "sqadd v27.4s, v27.4s, v7.4s\n"
+ "130:" // Height 5: no shift correction
+ "srshl v8.4s, v8.4s, v0.4s\n"
+ "add x19, %x[qp], %[c_offset]\n"
+ "ld1r { v4.4s }, [x19]\n"
+ "srshl v9.4s, v9.4s, v1.4s\n"
+ "add x19, %x[qp], %[minval]\n"
+ "srshl v10.4s, v10.4s, v2.4s\n"
+ "ld1r { v5.4s }, [x19]\n"
+ "add x19, %x[qp], %[maxval]\n"
+ "srshl v11.4s, v11.4s, v3.4s\n"
+ "ld1r { v6.4s }, [x19]\n"
+ "cmp x15, #0x10\n"
+ "srshl v12.4s, v12.4s, v0.4s\n"
+ "srshl v13.4s, v13.4s, v1.4s\n"
+ "srshl v14.4s, v14.4s, v2.4s\n"
+ "srshl v15.4s, v15.4s, v3.4s\n"
+ "add v8.4s, v8.4s, v4.4s\n"
+ "add v9.4s, v9.4s, v4.4s\n"
+ "add v10.4s, v10.4s, v4.4s\n"
+ "smin v8.4s, v8.4s, v6.4s\n"
+ "smin v9.4s, v9.4s, v6.4s\n"
+ "smin v10.4s, v10.4s, v6.4s\n"
+ "smax v8.4s, v8.4s, v5.4s\n"
+ "smax v9.4s, v9.4s, v5.4s\n"
+ "smax v10.4s, v10.4s, v5.4s\n"
+ "add v11.4s, v11.4s, v4.4s\n"
+ "add v12.4s, v12.4s, v4.4s\n"
+ "add v13.4s, v13.4s, v4.4s\n"
+ "smin v11.4s, v11.4s, v6.4s\n"
+ "smin v12.4s, v12.4s, v6.4s\n"
+ "smin v13.4s, v13.4s, v6.4s\n"
+ "smax v11.4s, v11.4s, v5.4s\n"
+ "smax v12.4s, v12.4s, v5.4s\n"
+ "smax v13.4s, v13.4s, v5.4s\n"
+ "add v14.4s, v14.4s, v4.4s\n"
+ "add v15.4s, v15.4s, v4.4s\n"
+ "srshl v16.4s, v16.4s, v0.4s\n"
+ "smin v14.4s, v14.4s, v6.4s\n"
+ "smin v15.4s, v15.4s, v6.4s\n"
+ "srshl v17.4s, v17.4s, v1.4s\n"
+ "smax v14.4s, v14.4s, v5.4s\n"
+ "smax v15.4s, v15.4s, v5.4s\n"
+ "add v16.4s, v16.4s, v4.4s\n"
+ "add v17.4s, v17.4s, v4.4s\n"
+ "srshl v18.4s, v18.4s, v2.4s\n"
+ "smin v16.4s, v16.4s, v6.4s\n"
+ "smin v17.4s, v17.4s, v6.4s\n"
+ "srshl v19.4s, v19.4s, v3.4s\n"
+ "smax v16.4s, v16.4s, v5.4s\n"
+ "smax v17.4s, v17.4s, v5.4s\n"
+ "add v18.4s, v18.4s, v4.4s\n"
+ "add v19.4s, v19.4s, v4.4s\n"
+ "srshl v20.4s, v20.4s, v0.4s\n"
+ "smin v18.4s, v18.4s, v6.4s\n"
+ "smin v19.4s, v19.4s, v6.4s\n"
+ "srshl v21.4s, v21.4s, v1.4s\n"
+ "smax v18.4s, v18.4s, v5.4s\n"
+ "smax v19.4s, v19.4s, v5.4s\n"
+ "add v20.4s, v20.4s, v4.4s\n"
+ "add v21.4s, v21.4s, v4.4s\n"
+ "srshl v22.4s, v22.4s, v2.4s\n"
+ "smin v20.4s, v20.4s, v6.4s\n"
+ "smin v21.4s, v21.4s, v6.4s\n"
+ "srshl v23.4s, v23.4s, v3.4s\n"
+ "smax v20.4s, v20.4s, v5.4s\n"
+ "smax v21.4s, v21.4s, v5.4s\n"
+ "add v22.4s, v22.4s, v4.4s\n"
+ "add v23.4s, v23.4s, v4.4s\n"
+ "srshl v24.4s, v24.4s, v0.4s\n"
+ "smin v22.4s, v22.4s, v6.4s\n"
+ "smin v23.4s, v23.4s, v6.4s\n"
+ "srshl v25.4s, v25.4s, v1.4s\n"
+ "smax v22.4s, v22.4s, v5.4s\n"
+ "smax v23.4s, v23.4s, v5.4s\n"
+ "add v24.4s, v24.4s, v4.4s\n"
+ "add v25.4s, v25.4s, v4.4s\n"
+ "srshl v26.4s, v26.4s, v2.4s\n"
+ "smin v24.4s, v24.4s, v6.4s\n"
+ "smin v25.4s, v25.4s, v6.4s\n"
+ "srshl v27.4s, v27.4s, v3.4s\n"
+ "smax v24.4s, v24.4s, v5.4s\n"
+ "smax v25.4s, v25.4s, v5.4s\n"
+ "add v26.4s, v26.4s, v4.4s\n"
+ "add v27.4s, v27.4s, v4.4s\n"
+ "uzp1 v8.8h, v8.8h, v9.8h\n"
+ "smin v26.4s, v26.4s, v6.4s\n"
+ "smin v27.4s, v27.4s, v6.4s\n"
+ "uzp1 v9.8h, v10.8h, v11.8h\n"
+ "smax v26.4s, v26.4s, v5.4s\n"
+ "smax v27.4s, v27.4s, v5.4s\n"
+ "uzp1 v12.8h, v12.8h, v13.8h\n"
+ "uzp1 v13.8h, v14.8h, v15.8h\n"
+ "uzp1 v16.8h, v16.8h, v17.8h\n"
+ "uzp1 v17.8h, v18.8h, v19.8h\n"
+ "uzp1 v20.8h, v20.8h, v21.8h\n"
+ "uzp1 v21.8h, v22.8h, v23.8h\n"
+ "uzp1 v24.8h, v24.8h, v25.8h\n"
+ "uzp1 v25.8h, v26.8h, v27.8h\n"
+ "uzp1 v8.16b, v8.16b, v9.16b\n"
+ "uzp1 v12.16b, v12.16b, v13.16b\n"
+ "uzp1 v16.16b, v16.16b, v17.16b\n"
+ "uzp1 v20.16b, v20.16b, v21.16b\n"
+ "uzp1 v24.16b, v24.16b, v25.16b\n"
+ "bge 139f\n"
+ "tbz x15, #3, 134f\n"
+ "str d8, [x13], #0x8\n"
+ "str d12, [x9], #0x8\n"
+ "str d16, [x27], #0x8\n"
+ "str d20, [x25], #0x8\n"
+ "str d24, [x23], #0x8\n"
+ "tbz x15, #2, 132f\n"
+ "st1 { v8.s }[2], [x13], #0x4\n"
+ "st1 { v12.s }[2], [x9], #0x4\n"
+ "st1 { v16.s }[2], [x27], #0x4\n"
+ "st1 { v20.s }[2], [x25], #0x4\n"
+ "st1 { v24.s }[2], [x23], #0x4\n"
+ "tbz x15, #1, 131f\n"
+ "st1 { v8.h }[6], [x13], #0x2\n"
+ "st1 { v12.h }[6], [x9], #0x2\n"
+ "st1 { v16.h }[6], [x27], #0x2\n"
+ "st1 { v20.h }[6], [x25], #0x2\n"
+ "st1 { v24.h }[6], [x23], #0x2\n"
+ "tbz x15, #0, 138f\n"
+ "st1 { v8.b }[14], [x13]\n"
+ "st1 { v12.b }[14], [x9]\n"
+ "st1 { v16.b }[14], [x27]\n"
+ "st1 { v20.b }[14], [x25]\n"
+ "st1 { v24.b }[14], [x23]\n"
+ "b 138f\n"
+ "131:" // Height 5: Partial direct writeback: partial_1_12
+ "tbz x15, #0, 138f\n"
+ "st1 { v8.b }[12], [x13]\n"
+ "st1 { v12.b }[12], [x9]\n"
+ "st1 { v16.b }[12], [x27]\n"
+ "st1 { v20.b }[12], [x25]\n"
+ "st1 { v24.b }[12], [x23]\n"
+ "b 138f\n"
+ "132:" // Height 5: Partial direct writeback: partial_2_8
+ "tbz x15, #1, 133f\n"
+ "st1 { v8.h }[4], [x13], #0x2\n"
+ "st1 { v12.h }[4], [x9], #0x2\n"
+ "st1 { v16.h }[4], [x27], #0x2\n"
+ "st1 { v20.h }[4], [x25], #0x2\n"
+ "st1 { v24.h }[4], [x23], #0x2\n"
+ "tbz x15, #0, 138f\n"
+ "st1 { v8.b }[10], [x13]\n"
+ "st1 { v12.b }[10], [x9]\n"
+ "st1 { v16.b }[10], [x27]\n"
+ "st1 { v20.b }[10], [x25]\n"
+ "st1 { v24.b }[10], [x23]\n"
+ "b 138f\n"
+ "133:" // Height 5: Partial direct writeback: partial_1_8
+ "tbz x15, #0, 138f\n"
+ "st1 { v8.b }[8], [x13]\n"
+ "st1 { v12.b }[8], [x9]\n"
+ "st1 { v16.b }[8], [x27]\n"
+ "st1 { v20.b }[8], [x25]\n"
+ "st1 { v24.b }[8], [x23]\n"
+ "b 138f\n"
+ "134:" // Height 5: Partial direct writeback: partial_4_0
+ "tbz x15, #2, 136f\n"
+ "str s8, [x13], #0x4\n"
+ "str s12, [x9], #0x4\n"
+ "str s16, [x27], #0x4\n"
+ "str s20, [x25], #0x4\n"
+ "str s24, [x23], #0x4\n"
+ "tbz x15, #1, 135f\n"
+ "st1 { v8.h }[2], [x13], #0x2\n"
+ "st1 { v12.h }[2], [x9], #0x2\n"
+ "st1 { v16.h }[2], [x27], #0x2\n"
+ "st1 { v20.h }[2], [x25], #0x2\n"
+ "st1 { v24.h }[2], [x23], #0x2\n"
+ "tbz x15, #0, 138f\n"
+ "st1 { v8.b }[6], [x13]\n"
+ "st1 { v12.b }[6], [x9]\n"
+ "st1 { v16.b }[6], [x27]\n"
+ "st1 { v20.b }[6], [x25]\n"
+ "st1 { v24.b }[6], [x23]\n"
+ "b 138f\n"
+ "135:" // Height 5: Partial direct writeback: partial_1_4
+ "tbz x15, #0, 138f\n"
+ "st1 { v8.b }[4], [x13]\n"
+ "st1 { v12.b }[4], [x9]\n"
+ "st1 { v16.b }[4], [x27]\n"
+ "st1 { v20.b }[4], [x25]\n"
+ "st1 { v24.b }[4], [x23]\n"
+ "b 138f\n"
+ "136:" // Height 5: Partial direct writeback: partial_2_0
+ "tbz x15, #1, 137f\n"
+ "str h8, [x13], #0x2\n"
+ "str h12, [x9], #0x2\n"
+ "str h16, [x27], #0x2\n"
+ "str h20, [x25], #0x2\n"
+ "str h24, [x23], #0x2\n"
+ "tbz x15, #0, 138f\n"
+ "st1 { v8.b }[2], [x13]\n"
+ "st1 { v12.b }[2], [x9]\n"
+ "st1 { v16.b }[2], [x27]\n"
+ "st1 { v20.b }[2], [x25]\n"
+ "st1 { v24.b }[2], [x23]\n"
+ "b 138f\n"
+ "137:" // Height 5: Partial direct writeback: partial_1_0
+ "str b8, [x13, #0x0]\n"
+ "str b12, [x9, #0x0]\n"
+ "str b16, [x27, #0x0]\n"
+ "str b20, [x25, #0x0]\n"
+ "str b24, [x23, #0x0]\n"
+ "138:" // Height 5: Partial direct writeback: Done
+ "b 140f\n"
+ "139:" // Height 5: Full writeback
+ "str q8, [x13, #0x0]\n"
+ "str q12, [x9, #0x0]\n"
+ "str q16, [x27, #0x0]\n"
+ "str q20, [x25, #0x0]\n"
+ "str q24, [x23, #0x0]\n"
+ "add x13, x13, #0x10\n"
+ "add x9, x9, #0x10\n"
+ "add x27, x27, #0x10\n"
+ "add x25, x25, #0x10\n"
+ "add x23, x23, #0x10\n"
+ "140:" // Height 5: Writeback done
+ "subs x15, x15, #0x10\n"
+ "bgt 115b\n"
+ "b 170f\n"
+ "141:" // Height 6
+ "ldr x8, [%x[args_ptr], %[offsetof_multiplier_ptr]]\n"
+ "mov x16, %x[col_bias]\n"
+ "ldr x17, [%x[args_ptr], %[offsetof_shift_ptr]]\n"
+ "ldr x15, [%x[args_ptr], %[offsetof_N]]\n"
+ "ldr x14, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 142f\n"
+ "ldr x13, [%x[output_ptr], #0x0]\n"
+ "add x13, x13, x19\n"
+ "ldr x9, [%x[output_ptr], #0x8]\n"
+ "ldr x27, [%x[output_ptr], #0x10]\n"
+ "add x9, x9, x19\n"
+ "ldr x25, [%x[output_ptr], #0x18]\n"
+ "ldr x23, [%x[output_ptr], #0x20]\n"
+ "add x27, x27, x19\n"
+ "ldr x21, [%x[output_ptr], #0x28]\n"
+ "add %x[output_ptr], %x[output_ptr], #0x30\n"
+ "add x25, x25, x19\n"
+ "add x23, x23, x19\n"
+ "add x21, x21, x19\n"
+ "b 143f\n"
+ "142:" // Height 6: setup direct output
+ "mov x13, %x[output_ptr]\n"
+ "add x9, x13, x19\n"
+ "add x27, x9, x19\n"
+ "add x25, x27, x19\n"
+ "add x23, x25, x19\n"
+ "add x21, x23, x19\n"
+ "add %x[output_ptr], x21, x19\n"
+ "143:" // Height 6: Column loop
+ "movi v8.4s, #0x0\n"
+ "movi v9.4s, #0x0\n"
+ "movi v10.4s, #0x0\n"
+ "movi v11.4s, #0x0\n"
+ "movi v12.4s, #0x0\n"
+ "movi v13.4s, #0x0\n"
+ "movi v14.4s, #0x0\n"
+ "movi v15.4s, #0x0\n"
+ "movi v16.4s, #0x0\n"
+ "movi v17.4s, #0x0\n"
+ "movi v18.4s, #0x0\n"
+ "movi v19.4s, #0x0\n"
+ "movi v20.4s, #0x0\n"
+ "movi v21.4s, #0x0\n"
+ "movi v22.4s, #0x0\n"
+ "movi v23.4s, #0x0\n"
+ "movi v24.4s, #0x0\n"
+ "movi v25.4s, #0x0\n"
+ "movi v26.4s, #0x0\n"
+ "movi v27.4s, #0x0\n"
+ "movi v28.4s, #0x0\n"
+ "movi v29.4s, #0x0\n"
+ "movi v30.4s, #0x0\n"
+ "movi v31.4s, #0x0\n"
+ "144:" // Height 6: setup done
+ "mov x12, #0x0\n"
+ "145:" // Height 6: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w11, [x20, x12, LSL #0x2]\n"
+ "tbz %x[flags], #3, 146f\n"
+ "ldr x20, [%x[input_ptr], x12, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x10, [x20, #0x0]\n"
+ "ldr x28, [x20, #0x8]\n"
+ "ldr x26, [x20, #0x10]\n"
+ "ldr x24, [x20, #0x18]\n"
+ "ldr x22, [x20, #0x20]\n"
+ "ldr x20, [x20, #0x28]\n"
+ "cbnz x12, 147f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x10, x10, x19\n"
+ "add x28, x28, x19\n"
+ "add x26, x26, x19\n"
+ "add x24, x24, x19\n"
+ "add x22, x22, x19\n"
+ "add x20, x20, x19\n"
+ "b 147f\n"
+ "146:" // Height 6: setup direct input
+ "mov x10, %x[input_ptr]\n"
+ "add x28, x10, x19\n"
+ "add x26, x28, x19\n"
+ "add x24, x26, x19\n"
+ "add x22, x24, x19\n"
+ "add x20, x22, x19\n"
+ "147:" // Height 6: input setup done
+ "cmp x11, #0x10\n"
+ "blt 150f\n"
+ "cmp x11, #0x20\n"
+ "blt 149f\n"
+ "148:" // Height 6: Multiply loop: Main loop head
+ "ldr q0, [x10, #0x0]\n"
+ "ldr q1, [x28, #0x0]\n"
+ "ldr q2, [x26, #0x0]\n"
+ "ldr q3, [x24, #0x0]\n"
+ "ldr q4, [x22, #0x0]\n"
+ "ldr q5, [x20, #0x0]\n"
+ "ldr q6, [x14, #0x0]\n"
+ ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n"
+ "ldr q7, [x14, #0x10]\n"
+ ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n"
+ "add x10, x10, #0x10\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n"
+ "add x28, x28, #0x10\n"
+ ".inst 0x4f83e0d4 // sdot v20.4s, v6.16b, v3.4b[0]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "add x26, x26, #0x10\n"
+ ".inst 0x4f84e0d8 // sdot v24.4s, v6.16b, v4.4b[0]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "add x24, x24, #0x10\n"
+ ".inst 0x4f85e0dc // sdot v28.4s, v6.16b, v5.4b[0]\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "ldr q6, [x14, #0x20]\n"
+ ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n"
+ "add x22, x22, #0x10\n"
+ "prfm pldl1keep, [x22, #0x80]\n"
+ ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n"
+ "add x20, x20, #0x10\n"
+ ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n"
+ "prfm pldl1keep, [x20, #0x80]\n"
+ "sub x11, x11, #0x10\n"
+ ".inst 0x4f83e0f5 // sdot v21.4s, v7.16b, v3.4b[0]\n"
+ "cmp x11, #0x20\n"
+ ".inst 0x4f84e0f9 // sdot v25.4s, v7.16b, v4.4b[0]\n"
+ ".inst 0x4f85e0fd // sdot v29.4s, v7.16b, v5.4b[0]\n"
+ "ldr q7, [x14, #0x30]\n"
+ ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n"
+ ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n"
+ ".inst 0x4f83e0d6 // sdot v22.4s, v6.16b, v3.4b[0]\n"
+ ".inst 0x4f84e0da // sdot v26.4s, v6.16b, v4.4b[0]\n"
+ ".inst 0x4f85e0de // sdot v30.4s, v6.16b, v5.4b[0]\n"
+ "ldr q6, [x14, #0x40]\n"
+ ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n"
+ ".inst 0x4f83e0f7 // sdot v23.4s, v7.16b, v3.4b[0]\n"
+ ".inst 0x4f84e0fb // sdot v27.4s, v7.16b, v4.4b[0]\n"
+ ".inst 0x4f85e0ff // sdot v31.4s, v7.16b, v5.4b[0]\n"
+ "ldr q7, [x14, #0x50]\n"
+ ".inst 0x4fa0e0c8 // sdot v8.4s, v6.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e0cc // sdot v12.4s, v6.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e0d0 // sdot v16.4s, v6.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e0d4 // sdot v20.4s, v6.16b, v3.4b[1]\n"
+ ".inst 0x4fa4e0d8 // sdot v24.4s, v6.16b, v4.4b[1]\n"
+ ".inst 0x4fa5e0dc // sdot v28.4s, v6.16b, v5.4b[1]\n"
+ "ldr q6, [x14, #0x60]\n"
+ ".inst 0x4fa0e0e9 // sdot v9.4s, v7.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e0ed // sdot v13.4s, v7.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e0f1 // sdot v17.4s, v7.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e0f5 // sdot v21.4s, v7.16b, v3.4b[1]\n"
+ ".inst 0x4fa4e0f9 // sdot v25.4s, v7.16b, v4.4b[1]\n"
+ ".inst 0x4fa5e0fd // sdot v29.4s, v7.16b, v5.4b[1]\n"
+ "ldr q7, [x14, #0x70]\n"
+ ".inst 0x4fa0e0ca // sdot v10.4s, v6.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e0ce // sdot v14.4s, v6.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e0d2 // sdot v18.4s, v6.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e0d6 // sdot v22.4s, v6.16b, v3.4b[1]\n"
+ ".inst 0x4fa4e0da // sdot v26.4s, v6.16b, v4.4b[1]\n"
+ ".inst 0x4fa5e0de // sdot v30.4s, v6.16b, v5.4b[1]\n"
+ "ldr q6, [x14, #0x80]\n"
+ ".inst 0x4fa0e0eb // sdot v11.4s, v7.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e0ef // sdot v15.4s, v7.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e0f3 // sdot v19.4s, v7.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e0f7 // sdot v23.4s, v7.16b, v3.4b[1]\n"
+ ".inst 0x4fa4e0fb // sdot v27.4s, v7.16b, v4.4b[1]\n"
+ ".inst 0x4fa5e0ff // sdot v31.4s, v7.16b, v5.4b[1]\n"
+ "ldr q7, [x14, #0x90]\n"
+ ".inst 0x4f80e8c8 // sdot v8.4s, v6.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8cc // sdot v12.4s, v6.16b, v1.4b[2]\n"
+ ".inst 0x4f82e8d0 // sdot v16.4s, v6.16b, v2.4b[2]\n"
+ ".inst 0x4f83e8d4 // sdot v20.4s, v6.16b, v3.4b[2]\n"
+ ".inst 0x4f84e8d8 // sdot v24.4s, v6.16b, v4.4b[2]\n"
+ ".inst 0x4f85e8dc // sdot v28.4s, v6.16b, v5.4b[2]\n"
+ "ldr q6, [x14, #0xa0]\n"
+ ".inst 0x4f80e8e9 // sdot v9.4s, v7.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8ed // sdot v13.4s, v7.16b, v1.4b[2]\n"
+ ".inst 0x4f82e8f1 // sdot v17.4s, v7.16b, v2.4b[2]\n"
+ ".inst 0x4f83e8f5 // sdot v21.4s, v7.16b, v3.4b[2]\n"
+ ".inst 0x4f84e8f9 // sdot v25.4s, v7.16b, v4.4b[2]\n"
+ ".inst 0x4f85e8fd // sdot v29.4s, v7.16b, v5.4b[2]\n"
+ "ldr q7, [x14, #0xb0]\n"
+ ".inst 0x4f80e8ca // sdot v10.4s, v6.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8ce // sdot v14.4s, v6.16b, v1.4b[2]\n"
+ ".inst 0x4f82e8d2 // sdot v18.4s, v6.16b, v2.4b[2]\n"
+ ".inst 0x4f83e8d6 // sdot v22.4s, v6.16b, v3.4b[2]\n"
+ ".inst 0x4f84e8da // sdot v26.4s, v6.16b, v4.4b[2]\n"
+ ".inst 0x4f85e8de // sdot v30.4s, v6.16b, v5.4b[2]\n"
+ "ldr q6, [x14, #0xc0]\n"
+ ".inst 0x4f80e8eb // sdot v11.4s, v7.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8ef // sdot v15.4s, v7.16b, v1.4b[2]\n"
+ ".inst 0x4f82e8f3 // sdot v19.4s, v7.16b, v2.4b[2]\n"
+ ".inst 0x4f83e8f7 // sdot v23.4s, v7.16b, v3.4b[2]\n"
+ ".inst 0x4f84e8fb // sdot v27.4s, v7.16b, v4.4b[2]\n"
+ ".inst 0x4f85e8ff // sdot v31.4s, v7.16b, v5.4b[2]\n"
+ "ldr q7, [x14, #0xd0]\n"
+ ".inst 0x4fa0e8c8 // sdot v8.4s, v6.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e8cc // sdot v12.4s, v6.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e8d0 // sdot v16.4s, v6.16b, v2.4b[3]\n"
+ ".inst 0x4fa3e8d4 // sdot v20.4s, v6.16b, v3.4b[3]\n"
+ ".inst 0x4fa4e8d8 // sdot v24.4s, v6.16b, v4.4b[3]\n"
+ ".inst 0x4fa5e8dc // sdot v28.4s, v6.16b, v5.4b[3]\n"
+ "ldr q6, [x14, #0xe0]\n"
+ ".inst 0x4fa0e8e9 // sdot v9.4s, v7.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e8ed // sdot v13.4s, v7.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e8f1 // sdot v17.4s, v7.16b, v2.4b[3]\n"
+ ".inst 0x4fa3e8f5 // sdot v21.4s, v7.16b, v3.4b[3]\n"
+ ".inst 0x4fa4e8f9 // sdot v25.4s, v7.16b, v4.4b[3]\n"
+ ".inst 0x4fa5e8fd // sdot v29.4s, v7.16b, v5.4b[3]\n"
+ "ldr q7, [x14, #0xf0]\n"
+ ".inst 0x4fa0e8ca // sdot v10.4s, v6.16b, v0.4b[3]\n"
+ "add x14, x14, #0x100\n"
+ ".inst 0x4fa1e8ce // sdot v14.4s, v6.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e8d2 // sdot v18.4s, v6.16b, v2.4b[3]\n"
+ ".inst 0x4fa3e8d6 // sdot v22.4s, v6.16b, v3.4b[3]\n"
+ ".inst 0x4fa4e8da // sdot v26.4s, v6.16b, v4.4b[3]\n"
+ ".inst 0x4fa5e8de // sdot v30.4s, v6.16b, v5.4b[3]\n"
+ ".inst 0x4fa0e8eb // sdot v11.4s, v7.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e8ef // sdot v15.4s, v7.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e8f3 // sdot v19.4s, v7.16b, v2.4b[3]\n"
+ ".inst 0x4fa3e8f7 // sdot v23.4s, v7.16b, v3.4b[3]\n"
+ ".inst 0x4fa4e8fb // sdot v27.4s, v7.16b, v4.4b[3]\n"
+ ".inst 0x4fa5e8ff // sdot v31.4s, v7.16b, v5.4b[3]\n"
+ "bge 148b\n"
+ "149:" // Height 6: Multiply loop: Single iteration only
+ "sub x11, x11, #0x10\n"
+ "ldr q0, [x10, #0x0]\n"
+ "ldr q1, [x28, #0x0]\n"
+ "ldr q2, [x26, #0x0]\n"
+ "ldr q3, [x24, #0x0]\n"
+ "ldr q4, [x22, #0x0]\n"
+ "ldr q5, [x20, #0x0]\n"
+ "ldr q6, [x14, #0x0]\n"
+ ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n"
+ "ldr q7, [x14, #0x10]\n"
+ ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n"
+ "add x10, x10, #0x10\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n"
+ "add x28, x28, #0x10\n"
+ ".inst 0x4f83e0d4 // sdot v20.4s, v6.16b, v3.4b[0]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "add x26, x26, #0x10\n"
+ ".inst 0x4f84e0d8 // sdot v24.4s, v6.16b, v4.4b[0]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "add x24, x24, #0x10\n"
+ ".inst 0x4f85e0dc // sdot v28.4s, v6.16b, v5.4b[0]\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "ldr q6, [x14, #0x20]\n"
+ ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n"
+ "add x22, x22, #0x10\n"
+ "prfm pldl1keep, [x22, #0x80]\n"
+ ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n"
+ "add x20, x20, #0x10\n"
+ ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n"
+ "prfm pldl1keep, [x20, #0x80]\n"
+ ".inst 0x4f83e0f5 // sdot v21.4s, v7.16b, v3.4b[0]\n"
+ ".inst 0x4f84e0f9 // sdot v25.4s, v7.16b, v4.4b[0]\n"
+ ".inst 0x4f85e0fd // sdot v29.4s, v7.16b, v5.4b[0]\n"
+ "ldr q7, [x14, #0x30]\n"
+ ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n"
+ ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n"
+ ".inst 0x4f83e0d6 // sdot v22.4s, v6.16b, v3.4b[0]\n"
+ ".inst 0x4f84e0da // sdot v26.4s, v6.16b, v4.4b[0]\n"
+ ".inst 0x4f85e0de // sdot v30.4s, v6.16b, v5.4b[0]\n"
+ "ldr q6, [x14, #0x40]\n"
+ ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n"
+ ".inst 0x4f83e0f7 // sdot v23.4s, v7.16b, v3.4b[0]\n"
+ ".inst 0x4f84e0fb // sdot v27.4s, v7.16b, v4.4b[0]\n"
+ ".inst 0x4f85e0ff // sdot v31.4s, v7.16b, v5.4b[0]\n"
+ "ldr q7, [x14, #0x50]\n"
+ ".inst 0x4fa0e0c8 // sdot v8.4s, v6.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e0cc // sdot v12.4s, v6.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e0d0 // sdot v16.4s, v6.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e0d4 // sdot v20.4s, v6.16b, v3.4b[1]\n"
+ ".inst 0x4fa4e0d8 // sdot v24.4s, v6.16b, v4.4b[1]\n"
+ ".inst 0x4fa5e0dc // sdot v28.4s, v6.16b, v5.4b[1]\n"
+ "ldr q6, [x14, #0x60]\n"
+ ".inst 0x4fa0e0e9 // sdot v9.4s, v7.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e0ed // sdot v13.4s, v7.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e0f1 // sdot v17.4s, v7.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e0f5 // sdot v21.4s, v7.16b, v3.4b[1]\n"
+ ".inst 0x4fa4e0f9 // sdot v25.4s, v7.16b, v4.4b[1]\n"
+ ".inst 0x4fa5e0fd // sdot v29.4s, v7.16b, v5.4b[1]\n"
+ "ldr q7, [x14, #0x70]\n"
+ ".inst 0x4fa0e0ca // sdot v10.4s, v6.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e0ce // sdot v14.4s, v6.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e0d2 // sdot v18.4s, v6.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e0d6 // sdot v22.4s, v6.16b, v3.4b[1]\n"
+ ".inst 0x4fa4e0da // sdot v26.4s, v6.16b, v4.4b[1]\n"
+ ".inst 0x4fa5e0de // sdot v30.4s, v6.16b, v5.4b[1]\n"
+ "ldr q6, [x14, #0x80]\n"
+ ".inst 0x4fa0e0eb // sdot v11.4s, v7.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e0ef // sdot v15.4s, v7.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e0f3 // sdot v19.4s, v7.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e0f7 // sdot v23.4s, v7.16b, v3.4b[1]\n"
+ ".inst 0x4fa4e0fb // sdot v27.4s, v7.16b, v4.4b[1]\n"
+ ".inst 0x4fa5e0ff // sdot v31.4s, v7.16b, v5.4b[1]\n"
+ "ldr q7, [x14, #0x90]\n"
+ ".inst 0x4f80e8c8 // sdot v8.4s, v6.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8cc // sdot v12.4s, v6.16b, v1.4b[2]\n"
+ ".inst 0x4f82e8d0 // sdot v16.4s, v6.16b, v2.4b[2]\n"
+ ".inst 0x4f83e8d4 // sdot v20.4s, v6.16b, v3.4b[2]\n"
+ ".inst 0x4f84e8d8 // sdot v24.4s, v6.16b, v4.4b[2]\n"
+ ".inst 0x4f85e8dc // sdot v28.4s, v6.16b, v5.4b[2]\n"
+ "ldr q6, [x14, #0xa0]\n"
+ ".inst 0x4f80e8e9 // sdot v9.4s, v7.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8ed // sdot v13.4s, v7.16b, v1.4b[2]\n"
+ ".inst 0x4f82e8f1 // sdot v17.4s, v7.16b, v2.4b[2]\n"
+ ".inst 0x4f83e8f5 // sdot v21.4s, v7.16b, v3.4b[2]\n"
+ ".inst 0x4f84e8f9 // sdot v25.4s, v7.16b, v4.4b[2]\n"
+ ".inst 0x4f85e8fd // sdot v29.4s, v7.16b, v5.4b[2]\n"
+ "ldr q7, [x14, #0xb0]\n"
+ ".inst 0x4f80e8ca // sdot v10.4s, v6.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8ce // sdot v14.4s, v6.16b, v1.4b[2]\n"
+ ".inst 0x4f82e8d2 // sdot v18.4s, v6.16b, v2.4b[2]\n"
+ ".inst 0x4f83e8d6 // sdot v22.4s, v6.16b, v3.4b[2]\n"
+ ".inst 0x4f84e8da // sdot v26.4s, v6.16b, v4.4b[2]\n"
+ ".inst 0x4f85e8de // sdot v30.4s, v6.16b, v5.4b[2]\n"
+ "ldr q6, [x14, #0xc0]\n"
+ ".inst 0x4f80e8eb // sdot v11.4s, v7.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8ef // sdot v15.4s, v7.16b, v1.4b[2]\n"
+ ".inst 0x4f82e8f3 // sdot v19.4s, v7.16b, v2.4b[2]\n"
+ ".inst 0x4f83e8f7 // sdot v23.4s, v7.16b, v3.4b[2]\n"
+ ".inst 0x4f84e8fb // sdot v27.4s, v7.16b, v4.4b[2]\n"
+ ".inst 0x4f85e8ff // sdot v31.4s, v7.16b, v5.4b[2]\n"
+ "ldr q7, [x14, #0xd0]\n"
+ ".inst 0x4fa0e8c8 // sdot v8.4s, v6.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e8cc // sdot v12.4s, v6.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e8d0 // sdot v16.4s, v6.16b, v2.4b[3]\n"
+ ".inst 0x4fa3e8d4 // sdot v20.4s, v6.16b, v3.4b[3]\n"
+ ".inst 0x4fa4e8d8 // sdot v24.4s, v6.16b, v4.4b[3]\n"
+ ".inst 0x4fa5e8dc // sdot v28.4s, v6.16b, v5.4b[3]\n"
+ "ldr q6, [x14, #0xe0]\n"
+ ".inst 0x4fa0e8e9 // sdot v9.4s, v7.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e8ed // sdot v13.4s, v7.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e8f1 // sdot v17.4s, v7.16b, v2.4b[3]\n"
+ ".inst 0x4fa3e8f5 // sdot v21.4s, v7.16b, v3.4b[3]\n"
+ ".inst 0x4fa4e8f9 // sdot v25.4s, v7.16b, v4.4b[3]\n"
+ ".inst 0x4fa5e8fd // sdot v29.4s, v7.16b, v5.4b[3]\n"
+ "ldr q7, [x14, #0xf0]\n"
+ ".inst 0x4fa0e8ca // sdot v10.4s, v6.16b, v0.4b[3]\n"
+ "add x14, x14, #0x100\n"
+ ".inst 0x4fa1e8ce // sdot v14.4s, v6.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e8d2 // sdot v18.4s, v6.16b, v2.4b[3]\n"
+ ".inst 0x4fa3e8d6 // sdot v22.4s, v6.16b, v3.4b[3]\n"
+ ".inst 0x4fa4e8da // sdot v26.4s, v6.16b, v4.4b[3]\n"
+ ".inst 0x4fa5e8de // sdot v30.4s, v6.16b, v5.4b[3]\n"
+ ".inst 0x4fa0e8eb // sdot v11.4s, v7.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e8ef // sdot v15.4s, v7.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e8f3 // sdot v19.4s, v7.16b, v2.4b[3]\n"
+ ".inst 0x4fa3e8f7 // sdot v23.4s, v7.16b, v3.4b[3]\n"
+ ".inst 0x4fa4e8fb // sdot v27.4s, v7.16b, v4.4b[3]\n"
+ ".inst 0x4fa5e8ff // sdot v31.4s, v7.16b, v5.4b[3]\n"
+ "150:" // Height 6: Multiply loop: Main loop skip
+ "cbz x11, 155f\n"
+ "cmp x11, #0x4\n"
+ "blt 152f\n"
+ "151:" // Height 6: Multiply loop: Odd block loop
+ "ldr s0, [x10], #0x4\n"
+ "ldr s1, [x28], #0x4\n"
+ "ldr s2, [x26], #0x4\n"
+ "ldr s3, [x24], #0x4\n"
+ "ldr s4, [x22], #0x4\n"
+ "ldr s5, [x20], #0x4\n"
+ "ldr q6, [x14, #0x0]\n"
+ ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n"
+ "ldr q7, [x14, #0x10]\n"
+ ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n"
+ "sub x11, x11, #0x4\n"
+ ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n"
+ "cmp x11, #0x4\n"
+ ".inst 0x4f83e0d4 // sdot v20.4s, v6.16b, v3.4b[0]\n"
+ ".inst 0x4f84e0d8 // sdot v24.4s, v6.16b, v4.4b[0]\n"
+ ".inst 0x4f85e0dc // sdot v28.4s, v6.16b, v5.4b[0]\n"
+ "ldr q6, [x14, #0x20]\n"
+ ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n"
+ ".inst 0x4f83e0f5 // sdot v21.4s, v7.16b, v3.4b[0]\n"
+ ".inst 0x4f84e0f9 // sdot v25.4s, v7.16b, v4.4b[0]\n"
+ ".inst 0x4f85e0fd // sdot v29.4s, v7.16b, v5.4b[0]\n"
+ "ldr q7, [x14, #0x30]\n"
+ ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n"
+ "add x14, x14, #0x40\n"
+ ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n"
+ ".inst 0x4f83e0d6 // sdot v22.4s, v6.16b, v3.4b[0]\n"
+ ".inst 0x4f84e0da // sdot v26.4s, v6.16b, v4.4b[0]\n"
+ ".inst 0x4f85e0de // sdot v30.4s, v6.16b, v5.4b[0]\n"
+ ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n"
+ ".inst 0x4f83e0f7 // sdot v23.4s, v7.16b, v3.4b[0]\n"
+ ".inst 0x4f84e0fb // sdot v27.4s, v7.16b, v4.4b[0]\n"
+ ".inst 0x4f85e0ff // sdot v31.4s, v7.16b, v5.4b[0]\n"
+ "bge 151b\n"
+ "cbz x11, 155f\n"
+ "152:" // Height 6: Multiply loop: Skip odd blocks
+ "tbz x11, #1, 153f\n"
+ "ldr h0, [x10], #0x2\n"
+ "ldr h1, [x28], #0x2\n"
+ "ldr h2, [x26], #0x2\n"
+ "ldr h3, [x24], #0x2\n"
+ "ldr h4, [x22], #0x2\n"
+ "ldr h5, [x20], #0x2\n"
+ "tbz x11, #0, 154f\n"
+ "ld1 { v0.b }[2], [x10]\n"
+ "ld1 { v1.b }[2], [x28]\n"
+ "ld1 { v2.b }[2], [x26]\n"
+ "ld1 { v3.b }[2], [x24]\n"
+ "ld1 { v4.b }[2], [x22]\n"
+ "ld1 { v5.b }[2], [x20]\n"
+ "b 154f\n"
+ "153:" // Height 6: Multiply loop: Ragged operand read: partial_1_0
+ "ldr b0, [x10, #0x0]\n"
+ "ldr b1, [x28, #0x0]\n"
+ "ldr b2, [x26, #0x0]\n"
+ "ldr b3, [x24, #0x0]\n"
+ "ldr b4, [x22, #0x0]\n"
+ "ldr b5, [x20, #0x0]\n"
+ "154:" // Height 6: Multiply loop: Ragged operand read: Done
+ "ldr q6, [x14, #0x0]\n"
+ ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n"
+ "ldr q7, [x14, #0x10]\n"
+ ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n"
+ ".inst 0x4f83e0d4 // sdot v20.4s, v6.16b, v3.4b[0]\n"
+ ".inst 0x4f84e0d8 // sdot v24.4s, v6.16b, v4.4b[0]\n"
+ ".inst 0x4f85e0dc // sdot v28.4s, v6.16b, v5.4b[0]\n"
+ "ldr q6, [x14, #0x20]\n"
+ ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n"
+ ".inst 0x4f83e0f5 // sdot v21.4s, v7.16b, v3.4b[0]\n"
+ ".inst 0x4f84e0f9 // sdot v25.4s, v7.16b, v4.4b[0]\n"
+ ".inst 0x4f85e0fd // sdot v29.4s, v7.16b, v5.4b[0]\n"
+ "ldr q7, [x14, #0x30]\n"
+ ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n"
+ "add x14, x14, #0x40\n"
+ ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n"
+ ".inst 0x4f83e0d6 // sdot v22.4s, v6.16b, v3.4b[0]\n"
+ ".inst 0x4f84e0da // sdot v26.4s, v6.16b, v4.4b[0]\n"
+ ".inst 0x4f85e0de // sdot v30.4s, v6.16b, v5.4b[0]\n"
+ ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n"
+ ".inst 0x4f83e0f7 // sdot v23.4s, v7.16b, v3.4b[0]\n"
+ ".inst 0x4f84e0fb // sdot v27.4s, v7.16b, v4.4b[0]\n"
+ ".inst 0x4f85e0ff // sdot v31.4s, v7.16b, v5.4b[0]\n"
+ "155:" // Height 6: Multiply loop: No odd multiplies
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "add x12, x12, #0x1\n"
+ "cmp x12, x19\n"
+ "bne 145b\n"
+ "prfm pstl1keep, [x13, #0x0]\n"
+ "prfm pstl1keep, [x9, #0x0]\n"
+ "ldr q0, [x16, #0x0]\n"
+ "add v8.4s, v8.4s, v0.4s\n"
+ "prfm pstl1keep, [x27, #0x0]\n"
+ "ldr q1, [x16, #0x10]\n"
+ "add v12.4s, v12.4s, v0.4s\n"
+ "prfm pstl1keep, [x25, #0x0]\n"
+ "ldr q2, [x16, #0x20]\n"
+ "add v16.4s, v16.4s, v0.4s\n"
+ "prfm pstl1keep, [x23, #0x0]\n"
+ "ldr q3, [x16, #0x30]\n"
+ "add v20.4s, v20.4s, v0.4s\n"
+ "prfm pstl1keep, [x21, #0x0]\n"
+ "add x16, x16, #0x40\n"
+ "add v24.4s, v24.4s, v0.4s\n"
+ "add v28.4s, v28.4s, v0.4s\n"
+ "add v9.4s, v9.4s, v1.4s\n"
+ "add v10.4s, v10.4s, v2.4s\n"
+ "add v11.4s, v11.4s, v3.4s\n"
+ "add v13.4s, v13.4s, v1.4s\n"
+ "add v14.4s, v14.4s, v2.4s\n"
+ "add v15.4s, v15.4s, v3.4s\n"
+ "add v17.4s, v17.4s, v1.4s\n"
+ "add v18.4s, v18.4s, v2.4s\n"
+ "add v19.4s, v19.4s, v3.4s\n"
+ "add v21.4s, v21.4s, v1.4s\n"
+ "add v22.4s, v22.4s, v2.4s\n"
+ "add v23.4s, v23.4s, v3.4s\n"
+ "add v25.4s, v25.4s, v1.4s\n"
+ "add v26.4s, v26.4s, v2.4s\n"
+ "add v27.4s, v27.4s, v3.4s\n"
+ "add v29.4s, v29.4s, v1.4s\n"
+ "add v30.4s, v30.4s, v2.4s\n"
+ "add v31.4s, v31.4s, v3.4s\n"
+ "tbz %x[flags], #4, 156f\n"
+ "ldr q0, [x17, #0x0]\n"
+ "ldr q4, [x8, #0x0]\n"
+ "ldr q1, [x17, #0x10]\n"
+ "ldr q5, [x8, #0x10]\n"
+ "ldr q2, [x17, #0x20]\n"
+ "ldr q6, [x8, #0x20]\n"
+ "ldr q3, [x17, #0x30]\n"
+ "ldr q7, [x8, #0x30]\n"
+ "add x17, x17, #0x40\n"
+ "add x8, x8, #0x40\n"
+ "b 157f\n"
+ "156:" // Height 6: per layer parameters
+ "add x19, %x[qp], %[per_layer_right_shift]\n"
+ "ld1r { v0.4s }, [x19]\n"
+ "mov v1.16b, v0.16b\n"
+ "add x19, %x[qp], %[per_layer_mul]\n"
+ "ld1r { v4.4s }, [x19]\n"
+ "mov v2.16b, v0.16b\n"
+ "mov v3.16b, v0.16b\n"
+ "mov v5.16b, v4.16b\n"
+ "mov v6.16b, v4.16b\n"
+ "mov v7.16b, v4.16b\n"
+ "157:" // Height 6: parameters loaded
+ "sqrdmulh v8.4s, v8.4s, v4.4s\n"
+ "sqrdmulh v9.4s, v9.4s, v5.4s\n"
+ "sqrdmulh v10.4s, v10.4s, v6.4s\n"
+ "sqrdmulh v11.4s, v11.4s, v7.4s\n"
+ "sqrdmulh v12.4s, v12.4s, v4.4s\n"
+ "sqrdmulh v13.4s, v13.4s, v5.4s\n"
+ "sqrdmulh v14.4s, v14.4s, v6.4s\n"
+ "sqrdmulh v15.4s, v15.4s, v7.4s\n"
+ "sqrdmulh v16.4s, v16.4s, v4.4s\n"
+ "sqrdmulh v17.4s, v17.4s, v5.4s\n"
+ "sqrdmulh v18.4s, v18.4s, v6.4s\n"
+ "sqrdmulh v19.4s, v19.4s, v7.4s\n"
+ "sqrdmulh v20.4s, v20.4s, v4.4s\n"
+ "sqrdmulh v21.4s, v21.4s, v5.4s\n"
+ "sqrdmulh v22.4s, v22.4s, v6.4s\n"
+ "sqrdmulh v23.4s, v23.4s, v7.4s\n"
+ "sqrdmulh v24.4s, v24.4s, v4.4s\n"
+ "sqrdmulh v25.4s, v25.4s, v5.4s\n"
+ "sqrdmulh v26.4s, v26.4s, v6.4s\n"
+ "sqrdmulh v27.4s, v27.4s, v7.4s\n"
+ "sqrdmulh v28.4s, v28.4s, v4.4s\n"
+ "sqrdmulh v29.4s, v29.4s, v5.4s\n"
+ "sqrdmulh v30.4s, v30.4s, v6.4s\n"
+ "sqrdmulh v31.4s, v31.4s, v7.4s\n"
+ "tbz %x[flags], #5, 158f\n"
+ "and v4.16b, v8.16b, v0.16b\n"
+ "sshr v4.4s, v4.4s, #0x1f\n"
+ "and v5.16b, v9.16b, v1.16b\n"
+ "and v6.16b, v10.16b, v2.16b\n"
+ "sshr v5.4s, v5.4s, #0x1f\n"
+ "and v7.16b, v11.16b, v3.16b\n"
+ "sshr v6.4s, v6.4s, #0x1f\n"
+ "sqadd v8.4s, v8.4s, v4.4s\n"
+ "sshr v7.4s, v7.4s, #0x1f\n"
+ "and v4.16b, v12.16b, v0.16b\n"
+ "sqadd v9.4s, v9.4s, v5.4s\n"
+ "sshr v4.4s, v4.4s, #0x1f\n"
+ "sqadd v10.4s, v10.4s, v6.4s\n"
+ "and v5.16b, v13.16b, v1.16b\n"
+ "sshr v5.4s, v5.4s, #0x1f\n"
+ "sqadd v11.4s, v11.4s, v7.4s\n"
+ "and v6.16b, v14.16b, v2.16b\n"
+ "sshr v6.4s, v6.4s, #0x1f\n"
+ "sqadd v12.4s, v12.4s, v4.4s\n"
+ "and v7.16b, v15.16b, v3.16b\n"
+ "sshr v7.4s, v7.4s, #0x1f\n"
+ "sqadd v13.4s, v13.4s, v5.4s\n"
+ "and v4.16b, v16.16b, v0.16b\n"
+ "sshr v4.4s, v4.4s, #0x1f\n"
+ "sqadd v14.4s, v14.4s, v6.4s\n"
+ "and v5.16b, v17.16b, v1.16b\n"
+ "sshr v5.4s, v5.4s, #0x1f\n"
+ "sqadd v15.4s, v15.4s, v7.4s\n"
+ "and v6.16b, v18.16b, v2.16b\n"
+ "sshr v6.4s, v6.4s, #0x1f\n"
+ "sqadd v16.4s, v16.4s, v4.4s\n"
+ "and v7.16b, v19.16b, v3.16b\n"
+ "sshr v7.4s, v7.4s, #0x1f\n"
+ "sqadd v17.4s, v17.4s, v5.4s\n"
+ "and v4.16b, v20.16b, v0.16b\n"
+ "sshr v4.4s, v4.4s, #0x1f\n"
+ "sqadd v18.4s, v18.4s, v6.4s\n"
+ "and v5.16b, v21.16b, v1.16b\n"
+ "sshr v5.4s, v5.4s, #0x1f\n"
+ "sqadd v19.4s, v19.4s, v7.4s\n"
+ "and v6.16b, v22.16b, v2.16b\n"
+ "sshr v6.4s, v6.4s, #0x1f\n"
+ "sqadd v20.4s, v20.4s, v4.4s\n"
+ "and v7.16b, v23.16b, v3.16b\n"
+ "sshr v7.4s, v7.4s, #0x1f\n"
+ "sqadd v21.4s, v21.4s, v5.4s\n"
+ "and v4.16b, v24.16b, v0.16b\n"
+ "sshr v4.4s, v4.4s, #0x1f\n"
+ "sqadd v22.4s, v22.4s, v6.4s\n"
+ "and v5.16b, v25.16b, v1.16b\n"
+ "sshr v5.4s, v5.4s, #0x1f\n"
+ "sqadd v23.4s, v23.4s, v7.4s\n"
+ "and v6.16b, v26.16b, v2.16b\n"
+ "sshr v6.4s, v6.4s, #0x1f\n"
+ "sqadd v24.4s, v24.4s, v4.4s\n"
+ "and v7.16b, v27.16b, v3.16b\n"
+ "sshr v7.4s, v7.4s, #0x1f\n"
+ "sqadd v25.4s, v25.4s, v5.4s\n"
+ "and v4.16b, v28.16b, v0.16b\n"
+ "sshr v4.4s, v4.4s, #0x1f\n"
+ "sqadd v26.4s, v26.4s, v6.4s\n"
+ "and v5.16b, v29.16b, v1.16b\n"
+ "sshr v5.4s, v5.4s, #0x1f\n"
+ "sqadd v27.4s, v27.4s, v7.4s\n"
+ "and v6.16b, v30.16b, v2.16b\n"
+ "sshr v6.4s, v6.4s, #0x1f\n"
+ "sqadd v28.4s, v28.4s, v4.4s\n"
+ "and v7.16b, v31.16b, v3.16b\n"
+ "sshr v7.4s, v7.4s, #0x1f\n"
+ "sqadd v29.4s, v29.4s, v5.4s\n"
+ "sqadd v30.4s, v30.4s, v6.4s\n"
+ "sqadd v31.4s, v31.4s, v7.4s\n"
+ "158:" // Height 6: no shift correction
+ "srshl v8.4s, v8.4s, v0.4s\n"
+ "add x19, %x[qp], %[c_offset]\n"
+ "ld1r { v4.4s }, [x19]\n"
+ "srshl v9.4s, v9.4s, v1.4s\n"
+ "add x19, %x[qp], %[minval]\n"
+ "srshl v10.4s, v10.4s, v2.4s\n"
+ "ld1r { v5.4s }, [x19]\n"
+ "add x19, %x[qp], %[maxval]\n"
+ "srshl v11.4s, v11.4s, v3.4s\n"
+ "ld1r { v6.4s }, [x19]\n"
+ "cmp x15, #0x10\n"
+ "srshl v12.4s, v12.4s, v0.4s\n"
+ "srshl v13.4s, v13.4s, v1.4s\n"
+ "srshl v14.4s, v14.4s, v2.4s\n"
+ "srshl v15.4s, v15.4s, v3.4s\n"
+ "add v8.4s, v8.4s, v4.4s\n"
+ "add v9.4s, v9.4s, v4.4s\n"
+ "add v10.4s, v10.4s, v4.4s\n"
+ "smin v8.4s, v8.4s, v6.4s\n"
+ "smin v9.4s, v9.4s, v6.4s\n"
+ "smin v10.4s, v10.4s, v6.4s\n"
+ "smax v8.4s, v8.4s, v5.4s\n"
+ "smax v9.4s, v9.4s, v5.4s\n"
+ "smax v10.4s, v10.4s, v5.4s\n"
+ "add v11.4s, v11.4s, v4.4s\n"
+ "add v12.4s, v12.4s, v4.4s\n"
+ "add v13.4s, v13.4s, v4.4s\n"
+ "smin v11.4s, v11.4s, v6.4s\n"
+ "smin v12.4s, v12.4s, v6.4s\n"
+ "smin v13.4s, v13.4s, v6.4s\n"
+ "smax v11.4s, v11.4s, v5.4s\n"
+ "smax v12.4s, v12.4s, v5.4s\n"
+ "smax v13.4s, v13.4s, v5.4s\n"
+ "add v14.4s, v14.4s, v4.4s\n"
+ "add v15.4s, v15.4s, v4.4s\n"
+ "srshl v16.4s, v16.4s, v0.4s\n"
+ "smin v14.4s, v14.4s, v6.4s\n"
+ "smin v15.4s, v15.4s, v6.4s\n"
+ "srshl v17.4s, v17.4s, v1.4s\n"
+ "smax v14.4s, v14.4s, v5.4s\n"
+ "smax v15.4s, v15.4s, v5.4s\n"
+ "add v16.4s, v16.4s, v4.4s\n"
+ "add v17.4s, v17.4s, v4.4s\n"
+ "srshl v18.4s, v18.4s, v2.4s\n"
+ "smin v16.4s, v16.4s, v6.4s\n"
+ "smin v17.4s, v17.4s, v6.4s\n"
+ "srshl v19.4s, v19.4s, v3.4s\n"
+ "smax v16.4s, v16.4s, v5.4s\n"
+ "smax v17.4s, v17.4s, v5.4s\n"
+ "add v18.4s, v18.4s, v4.4s\n"
+ "add v19.4s, v19.4s, v4.4s\n"
+ "srshl v20.4s, v20.4s, v0.4s\n"
+ "smin v18.4s, v18.4s, v6.4s\n"
+ "smin v19.4s, v19.4s, v6.4s\n"
+ "srshl v21.4s, v21.4s, v1.4s\n"
+ "smax v18.4s, v18.4s, v5.4s\n"
+ "smax v19.4s, v19.4s, v5.4s\n"
+ "add v20.4s, v20.4s, v4.4s\n"
+ "add v21.4s, v21.4s, v4.4s\n"
+ "srshl v22.4s, v22.4s, v2.4s\n"
+ "smin v20.4s, v20.4s, v6.4s\n"
+ "smin v21.4s, v21.4s, v6.4s\n"
+ "srshl v23.4s, v23.4s, v3.4s\n"
+ "smax v20.4s, v20.4s, v5.4s\n"
+ "smax v21.4s, v21.4s, v5.4s\n"
+ "add v22.4s, v22.4s, v4.4s\n"
+ "add v23.4s, v23.4s, v4.4s\n"
+ "srshl v24.4s, v24.4s, v0.4s\n"
+ "smin v22.4s, v22.4s, v6.4s\n"
+ "smin v23.4s, v23.4s, v6.4s\n"
+ "srshl v25.4s, v25.4s, v1.4s\n"
+ "smax v22.4s, v22.4s, v5.4s\n"
+ "smax v23.4s, v23.4s, v5.4s\n"
+ "add v24.4s, v24.4s, v4.4s\n"
+ "add v25.4s, v25.4s, v4.4s\n"
+ "srshl v26.4s, v26.4s, v2.4s\n"
+ "smin v24.4s, v24.4s, v6.4s\n"
+ "smin v25.4s, v25.4s, v6.4s\n"
+ "srshl v27.4s, v27.4s, v3.4s\n"
+ "smax v24.4s, v24.4s, v5.4s\n"
+ "smax v25.4s, v25.4s, v5.4s\n"
+ "add v26.4s, v26.4s, v4.4s\n"
+ "add v27.4s, v27.4s, v4.4s\n"
+ "srshl v28.4s, v28.4s, v0.4s\n"
+ "smin v26.4s, v26.4s, v6.4s\n"
+ "smin v27.4s, v27.4s, v6.4s\n"
+ "srshl v29.4s, v29.4s, v1.4s\n"
+ "smax v26.4s, v26.4s, v5.4s\n"
+ "smax v27.4s, v27.4s, v5.4s\n"
+ "add v28.4s, v28.4s, v4.4s\n"
+ "add v29.4s, v29.4s, v4.4s\n"
+ "srshl v30.4s, v30.4s, v2.4s\n"
+ "smin v28.4s, v28.4s, v6.4s\n"
+ "smin v29.4s, v29.4s, v6.4s\n"
+ "srshl v31.4s, v31.4s, v3.4s\n"
+ "smax v28.4s, v28.4s, v5.4s\n"
+ "smax v29.4s, v29.4s, v5.4s\n"
+ "add v30.4s, v30.4s, v4.4s\n"
+ "add v31.4s, v31.4s, v4.4s\n"
+ "uzp1 v8.8h, v8.8h, v9.8h\n"
+ "smin v30.4s, v30.4s, v6.4s\n"
+ "smin v31.4s, v31.4s, v6.4s\n"
+ "uzp1 v9.8h, v10.8h, v11.8h\n"
+ "smax v30.4s, v30.4s, v5.4s\n"
+ "smax v31.4s, v31.4s, v5.4s\n"
+ "uzp1 v12.8h, v12.8h, v13.8h\n"
+ "uzp1 v13.8h, v14.8h, v15.8h\n"
+ "uzp1 v16.8h, v16.8h, v17.8h\n"
+ "uzp1 v17.8h, v18.8h, v19.8h\n"
+ "uzp1 v20.8h, v20.8h, v21.8h\n"
+ "uzp1 v21.8h, v22.8h, v23.8h\n"
+ "uzp1 v24.8h, v24.8h, v25.8h\n"
+ "uzp1 v25.8h, v26.8h, v27.8h\n"
+ "uzp1 v28.8h, v28.8h, v29.8h\n"
+ "uzp1 v29.8h, v30.8h, v31.8h\n"
+ "uzp1 v8.16b, v8.16b, v9.16b\n"
+ "uzp1 v12.16b, v12.16b, v13.16b\n"
+ "uzp1 v16.16b, v16.16b, v17.16b\n"
+ "uzp1 v20.16b, v20.16b, v21.16b\n"
+ "uzp1 v24.16b, v24.16b, v25.16b\n"
+ "uzp1 v28.16b, v28.16b, v29.16b\n"
+ "bge 167f\n"
+ "tbz x15, #3, 162f\n"
+ "str d8, [x13], #0x8\n"
+ "str d12, [x9], #0x8\n"
+ "str d16, [x27], #0x8\n"
+ "str d20, [x25], #0x8\n"
+ "str d24, [x23], #0x8\n"
+ "str d28, [x21], #0x8\n"
+ "tbz x15, #2, 160f\n"
+ "st1 { v8.s }[2], [x13], #0x4\n"
+ "st1 { v12.s }[2], [x9], #0x4\n"
+ "st1 { v16.s }[2], [x27], #0x4\n"
+ "st1 { v20.s }[2], [x25], #0x4\n"
+ "st1 { v24.s }[2], [x23], #0x4\n"
+ "st1 { v28.s }[2], [x21], #0x4\n"
+ "tbz x15, #1, 159f\n"
+ "st1 { v8.h }[6], [x13], #0x2\n"
+ "st1 { v12.h }[6], [x9], #0x2\n"
+ "st1 { v16.h }[6], [x27], #0x2\n"
+ "st1 { v20.h }[6], [x25], #0x2\n"
+ "st1 { v24.h }[6], [x23], #0x2\n"
+ "st1 { v28.h }[6], [x21], #0x2\n"
+ "tbz x15, #0, 166f\n"
+ "st1 { v8.b }[14], [x13]\n"
+ "st1 { v12.b }[14], [x9]\n"
+ "st1 { v16.b }[14], [x27]\n"
+ "st1 { v20.b }[14], [x25]\n"
+ "st1 { v24.b }[14], [x23]\n"
+ "st1 { v28.b }[14], [x21]\n"
+ "b 166f\n"
+ "159:" // Height 6: Partial direct writeback: partial_1_12
+ "tbz x15, #0, 166f\n"
+ "st1 { v8.b }[12], [x13]\n"
+ "st1 { v12.b }[12], [x9]\n"
+ "st1 { v16.b }[12], [x27]\n"
+ "st1 { v20.b }[12], [x25]\n"
+ "st1 { v24.b }[12], [x23]\n"
+ "st1 { v28.b }[12], [x21]\n"
+ "b 166f\n"
+ "160:" // Height 6: Partial direct writeback: partial_2_8
+ "tbz x15, #1, 161f\n"
+ "st1 { v8.h }[4], [x13], #0x2\n"
+ "st1 { v12.h }[4], [x9], #0x2\n"
+ "st1 { v16.h }[4], [x27], #0x2\n"
+ "st1 { v20.h }[4], [x25], #0x2\n"
+ "st1 { v24.h }[4], [x23], #0x2\n"
+ "st1 { v28.h }[4], [x21], #0x2\n"
+ "tbz x15, #0, 166f\n"
+ "st1 { v8.b }[10], [x13]\n"
+ "st1 { v12.b }[10], [x9]\n"
+ "st1 { v16.b }[10], [x27]\n"
+ "st1 { v20.b }[10], [x25]\n"
+ "st1 { v24.b }[10], [x23]\n"
+ "st1 { v28.b }[10], [x21]\n"
+ "b 166f\n"
+ "161:" // Height 6: Partial direct writeback: partial_1_8
+ "tbz x15, #0, 166f\n"
+ "st1 { v8.b }[8], [x13]\n"
+ "st1 { v12.b }[8], [x9]\n"
+ "st1 { v16.b }[8], [x27]\n"
+ "st1 { v20.b }[8], [x25]\n"
+ "st1 { v24.b }[8], [x23]\n"
+ "st1 { v28.b }[8], [x21]\n"
+ "b 166f\n"
+ "162:" // Height 6: Partial direct writeback: partial_4_0
+ "tbz x15, #2, 164f\n"
+ "str s8, [x13], #0x4\n"
+ "str s12, [x9], #0x4\n"
+ "str s16, [x27], #0x4\n"
+ "str s20, [x25], #0x4\n"
+ "str s24, [x23], #0x4\n"
+ "str s28, [x21], #0x4\n"
+ "tbz x15, #1, 163f\n"
+ "st1 { v8.h }[2], [x13], #0x2\n"
+ "st1 { v12.h }[2], [x9], #0x2\n"
+ "st1 { v16.h }[2], [x27], #0x2\n"
+ "st1 { v20.h }[2], [x25], #0x2\n"
+ "st1 { v24.h }[2], [x23], #0x2\n"
+ "st1 { v28.h }[2], [x21], #0x2\n"
+ "tbz x15, #0, 166f\n"
+ "st1 { v8.b }[6], [x13]\n"
+ "st1 { v12.b }[6], [x9]\n"
+ "st1 { v16.b }[6], [x27]\n"
+ "st1 { v20.b }[6], [x25]\n"
+ "st1 { v24.b }[6], [x23]\n"
+ "st1 { v28.b }[6], [x21]\n"
+ "b 166f\n"
+ "163:" // Height 6: Partial direct writeback: partial_1_4
+ "tbz x15, #0, 166f\n"
+ "st1 { v8.b }[4], [x13]\n"
+ "st1 { v12.b }[4], [x9]\n"
+ "st1 { v16.b }[4], [x27]\n"
+ "st1 { v20.b }[4], [x25]\n"
+ "st1 { v24.b }[4], [x23]\n"
+ "st1 { v28.b }[4], [x21]\n"
+ "b 166f\n"
+ "164:" // Height 6: Partial direct writeback: partial_2_0
+ "tbz x15, #1, 165f\n"
+ "str h8, [x13], #0x2\n"
+ "str h12, [x9], #0x2\n"
+ "str h16, [x27], #0x2\n"
+ "str h20, [x25], #0x2\n"
+ "str h24, [x23], #0x2\n"
+ "str h28, [x21], #0x2\n"
+ "tbz x15, #0, 166f\n"
+ "st1 { v8.b }[2], [x13]\n"
+ "st1 { v12.b }[2], [x9]\n"
+ "st1 { v16.b }[2], [x27]\n"
+ "st1 { v20.b }[2], [x25]\n"
+ "st1 { v24.b }[2], [x23]\n"
+ "st1 { v28.b }[2], [x21]\n"
+ "b 166f\n"
+ "165:" // Height 6: Partial direct writeback: partial_1_0
+ "str b8, [x13, #0x0]\n"
+ "str b12, [x9, #0x0]\n"
+ "str b16, [x27, #0x0]\n"
+ "str b20, [x25, #0x0]\n"
+ "str b24, [x23, #0x0]\n"
+ "str b28, [x21, #0x0]\n"
+ "166:" // Height 6: Partial direct writeback: Done
+ "b 168f\n"
+ "167:" // Height 6: Full writeback
+ "str q8, [x13, #0x0]\n"
+ "str q12, [x9, #0x0]\n"
+ "str q16, [x27, #0x0]\n"
+ "str q20, [x25, #0x0]\n"
+ "str q24, [x23, #0x0]\n"
+ "str q28, [x21, #0x0]\n"
+ "add x13, x13, #0x10\n"
+ "add x9, x9, #0x10\n"
+ "add x27, x27, #0x10\n"
+ "add x25, x25, #0x10\n"
+ "add x23, x23, #0x10\n"
+ "add x21, x21, #0x10\n"
+ "168:" // Height 6: Writeback done
+ "subs x15, x15, #0x10\n"
+ "bgt 143b\n"
+ "subs %x[M], %x[M], #0x6\n"
+ "beq 170f\n"
+ "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "tbz %x[flags], #3, 169f\n"
+ "add x20, x20, #0x6\n"
+ "str x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "b 1b\n"
+ "169:" // Update direct input
+ "mov x19, #0x6\n"
+ "madd %x[input_ptr], x19, x20, %x[input_ptr]\n"
+ "b 1b\n"
+ "170:" // Exit
+
+ : [M] "+r" (M), [input_ptr] "+r" (input_ptr), [output_ptr] "+r" (output_ptr)
+ : [args_ptr] "r" (&ka), [c_offset] "I" (offsetof(Requantize32, c_offset)), [col_bias] "r" (col_bias), [flags] "r" (flags), [maxval] "I" (offsetof(Requantize32, maxval)), [minval] "I" (offsetof(Requantize32, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_multiplier_ptr] "I" (offsetof(KernelArgs, multiplier_ptr)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_shift_ptr] "I" (offsetof(KernelArgs, shift_ptr)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths)), [per_layer_mul] "I" (offsetof(Requantize32, per_layer_mul)), [per_layer_right_shift] "I" (offsetof(Requantize32, per_layer_right_shift)), [qp] "r" (qp)
+ : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
+ );
+}
+
+} // namespace arm_gemm
+#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_16x4.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_16x4.hpp
deleted file mode 100644
index a23101a..0000000
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_16x4.hpp
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- * Copyright (c) 2018-2020 Arm Limited.
- *
- * SPDX-License-Identifier: MIT
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to
- * deal in the Software without restriction, including without limitation the
- * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
- * sell copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-#pragma once
-
-#ifdef __aarch64__
-
-#include <cstdint>
-#include "../std_transforms_fixed.hpp"
-
-namespace arm_gemm
-{
-
-// Actual kernel implementations
-void a64_hybrid_s8s32_dot_16x4(const int8_t *, int, const int8_t *, int32_t *, int, int, int, int, const int32_t *, Activation, bool);
-void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *, int, const int8_t *, int32_t *, int, int, int, int, const int32_t *, Activation, bool);
-
-class hybrid_s8s32_dot_16x4
-{
-public:
- typedef int8_t operand_type;
- typedef int32_t result_type;
-
- typedef void (*kern_type)(const int8_t *, int, const int8_t *, int32_t *, int, int, int, int, const int32_t *, Activation, bool);
-
- /* Kernel blocking parameters */
- static constexpr unsigned int out_height()
- {
- return 4;
- }
-
- static unsigned int out_width()
- {
- return 16;
- }
-
- static constexpr unsigned int k_unroll()
- {
- return 4;
- }
-
- static constexpr bool supports_accumulate()
- {
- return true;
- }
-
- static constexpr bool supports_bias()
- {
- return false;
- }
-
- static constexpr bool supports_activation()
- {
- return false;
- }
-
- StdTransformsFixed<operand_type, result_type, 4, 16, 4> transforms = {};
-
- // Default to the generic kernel
- kern_type kernel=a64_hybrid_s8s32_dot_16x4;
-
- hybrid_s8s32_dot_16x4(const CPUInfo *ci)
- {
- if (ci->get_cpu_model() == CPUModel::A55r1) {
- kernel = a64_hybrid_s8s32_dot_16x4_a55;
- }
- }
-};
-
-} // namespace arm_gemm
-
-#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_16x4/a55.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_16x4/a55.cpp
deleted file mode 100644
index 4a7cdc5..0000000
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_16x4/a55.cpp
+++ /dev/null
@@ -1,2434 +0,0 @@
-/*
- * Copyright (c) 2018-2020 Arm Limited.
- *
- * SPDX-License-Identifier: MIT
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to
- * deal in the Software without restriction, including without limitation the
- * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
- * sell copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-#ifdef __aarch64__
-
-#include <algorithm>
-
-#include "arm_gemm.hpp"
-#include <cstdint>
-#include "../../asmlib.hpp"
-#include "../../utils.hpp"
-
-namespace arm_gemm {
-
-void a64_hybrid_s8s32_dot_16x4_a55(const int8_t *A, int lda, const int8_t *B, int32_t *C, int ldc, int M, int N, int K, const int32_t *, Activation , bool accumulate) {
- const int K_stride = ((K + 3) / 4) * 4;
- const long loops_count = ((K + 16) / 32) - 1;
- K -= loops_count * 32;
- const long regs_count = (K / 16) - 1;
- K -= (regs_count + 1) * 16;
- const long blocks_count = K / 4;
- const long odds_count = K - (blocks_count * 4);
-
- int rows_to_compute;
-
- for (int y=0; y<M; y+=rows_to_compute) {
- const int8_t * const a_ptr0_base = A + (y * lda);
- const unsigned long ldab = lda * sizeof(int8_t);
-
- int32_t *c_ptr0 = C + (y * ldc);
-
- rows_to_compute = M-y;
- if (rows_to_compute > 4) {
- if (rows_to_compute % 4) {
- rows_to_compute = 4 - 1;
- } else {
- rows_to_compute = 4;
- }
- }
-
- for (int x0=0; x0<N; x0+=16ul) {
- const long width = std::min((unsigned long)N-x0, 16ul);
- long loops = loops_count;
- long regs = regs_count;
- long blocks = blocks_count;
- long odds = odds_count;
- const int8_t *a_ptr0 = a_ptr0_base;
- const int8_t *b_ptr0 = B + (K_stride * x0);
- const bool use_result_buffer = (width < 16);
- int32_t result_buffer[64];
- const unsigned long ldcb = (use_result_buffer ? 16 : ldc) * sizeof(int32_t);
- int32_t *c_ptr_real = c_ptr0;
- if (use_result_buffer && accumulate) {
- for(int cy=0; cy<std::min(M-y, 4); cy++) {
- for(unsigned int cx=0; cx<width; cx++) {
- result_buffer[cy * 16 + cx] = c_ptr_real[cy * ldc + cx];
- }
- }
- }
- if (use_result_buffer) {
- c_ptr0 = result_buffer;
- }
-
- switch(rows_to_compute) {
- case 1:
- __asm __volatile (
- "temploadreg0 .req X0\n"
- "temploadreg1 .req X1\n"
- "temploadreg2 .req X2\n"
- "temploadreg3 .req X3\n"
- "cbnz %[accumulate], 1f\n"
- "movi v16.4s, #0\n"
- "ldr q0, [%[a_ptr0]]\n"
- "movi v17.4s, #0\n"
- "ldr q8, [%[b_ptr0]]\n"
- "movi v18.4s, #0\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "movi v19.4s, #0\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "ldr q16, [%[c_ptr0]]\n"
- "ldr q17, [%[c_ptr0], #0x10]\n"
- "ldr q18, [%[c_ptr0], #0x20]\n"
- "ldr q19, [%[c_ptr0], #0x30]\n"
- "ldr q0, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ldr q8, [%[b_ptr0]]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "cbz %[loops], 2f\n"
- "3:\n"
- ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
- "ins v14.d[1], temploadreg2\n"
- ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
- "ldr d15, [%[b_ptr0], #-0x10]\n"
- ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
- "ldr temploadreg3, [%[b_ptr0], #-0x8]\n"
- ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
- "ldr d4, [%[a_ptr0]]\n"
- ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n"
- "ldr temploadreg0, [%[a_ptr0], #0x8]\n"
- ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n"
- "ldr d8, [%[b_ptr0]]\n"
- ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n"
- "ldr d9, [%[b_ptr0], #0x10]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ins v4.d[1], temploadreg0\n"
- "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
- "ldr d10, [%[b_ptr0], #0x20]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- "ldr d11, [%[b_ptr0], #0x30]\n"
- "ins v15.d[1], temploadreg3\n"
- "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- "ldr d12, [%[b_ptr0], #0x40]\n"
- "ins v8.d[1], temploadreg0\n"
- ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- "ldr d13, [%[b_ptr0], #0x50]\n"
- "ins v9.d[1], temploadreg1\n"
- ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- "ins v10.d[1], temploadreg2\n"
- ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- "ldr d15, [%[b_ptr0], #0x70]\n"
- "ins v11.d[1], temploadreg3\n"
- ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- "ins v12.d[1], temploadreg0\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n"
- "ldr d8, [%[b_ptr0], #-0x80]\n"
- "ldr temploadreg0, [%[b_ptr0], #-0x78]\n"
- ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n"
- "ldr d9, [%[b_ptr0], #-0x70]\n"
- "ins v13.d[1], temploadreg1\n"
- "ldr temploadreg1, [%[b_ptr0], #-0x68]\n"
- "ldr d10, [%[b_ptr0], #-0x60]\n"
- "ins v14.d[1], temploadreg2\n"
- ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n"
- "ldr temploadreg2, [%[b_ptr0], #-0x58]\n"
- "ldr d11, [%[b_ptr0], #-0x50]\n"
- "ins v15.d[1], temploadreg3\n"
- ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n"
- "ldr temploadreg3, [%[b_ptr0], #-0x48]\n"
- "ldr d12, [%[b_ptr0], #-0x40]\n"
- "ins v8.d[1], temploadreg0\n"
- ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n"
- "ldr temploadreg0, [%[b_ptr0], #-0x38]\n"
- "ldr d13, [%[b_ptr0], #-0x30]\n"
- "ins v9.d[1], temploadreg1\n"
- ".inst 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n"
- "ldr temploadreg1, [%[b_ptr0], #-0x28]\n"
- "ldr d14, [%[b_ptr0], #-0x20]\n"
- "ins v10.d[1], temploadreg2\n"
- ".inst 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n"
- "ldr temploadreg2, [%[b_ptr0], #-0x18]\n"
- "ldr d15, [%[b_ptr0], #-0x10]\n"
- "ins v11.d[1], temploadreg3\n"
- ".inst 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n"
- "ldr temploadreg3, [%[b_ptr0], #-0x8]\n"
- "ldr d0, [%[a_ptr0], #-0x10]\n"
- "ins v12.d[1], temploadreg0\n"
- ".inst 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n"
- "ldr temploadreg0, [%[a_ptr0], #-0x8]\n"
- "ldr d8, [%[b_ptr0]]\n"
- "ldr d9, [%[b_ptr0], #0x10]\n"
- ".inst 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n"
- "ins v0.d[1], temploadreg0\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- "ins v13.d[1], temploadreg1\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- "ldr d10, [%[b_ptr0], #0x20]\n"
- "ins v14.d[1], temploadreg2\n"
- ".inst 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- "ldr d11, [%[b_ptr0], #0x30]\n"
- "ins v15.d[1], temploadreg3\n"
- ".inst 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- "ldr d12, [%[b_ptr0], #0x40]\n"
- "ins v8.d[1], temploadreg0\n"
- ".inst 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- "ldr d13, [%[b_ptr0], #0x50]\n"
- "ins v9.d[1], temploadreg1\n"
- ".inst 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- "ins v10.d[1], temploadreg2\n"
- ".inst 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- "ldr d15, [%[b_ptr0], #0x70]\n"
- "ins v11.d[1], temploadreg3\n"
- ".inst 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- "ins v12.d[1], temploadreg0\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".inst 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n"
- "ldr d8, [%[b_ptr0], #-0x80]\n"
- "ldr temploadreg0, [%[b_ptr0], #-0x78]\n"
- ".inst 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n"
- "ldr d9, [%[b_ptr0], #-0x70]\n"
- "ins v13.d[1], temploadreg1\n"
- "ldr temploadreg1, [%[b_ptr0], #-0x68]\n"
- "ldr d10, [%[b_ptr0], #-0x60]\n"
- "ins v14.d[1], temploadreg2\n"
- ".inst 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n"
- "ldr temploadreg2, [%[b_ptr0], #-0x58]\n"
- "ldr d11, [%[b_ptr0], #-0x50]\n"
- "ins v15.d[1], temploadreg3\n"
- ".inst 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n"
- "ldr temploadreg3, [%[b_ptr0], #-0x48]\n"
- "ldr d12, [%[b_ptr0], #-0x40]\n"
- "ins v8.d[1], temploadreg0\n"
- ".inst 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n"
- "ldr temploadreg0, [%[b_ptr0], #-0x38]\n"
- "ldr d13, [%[b_ptr0], #-0x30]\n"
- "ins v9.d[1], temploadreg1\n"
- "ldr temploadreg1, [%[b_ptr0], #-0x28]\n"
- "ldr d14, [%[b_ptr0], #-0x20]\n"
- "ins v10.d[1], temploadreg2\n"
- "ldr temploadreg2, [%[b_ptr0], #-0x18]\n"
- "ins v11.d[1], temploadreg3\n"
- "ins v12.d[1], temploadreg0\n"
- "ins v13.d[1], temploadreg1\n"
- "b.ne 3b\n"
- "2:\n"
- "ins v14.d[1], temploadreg2\n"
- "prfm PSTL1KEEP, [%[c_ptr0]]\n"
- "ldr d15, [%[b_ptr0], #-0x10]\n"
- "ldr temploadreg3, [%[b_ptr0], #-0x8]\n"
- "ins v15.d[1], temploadreg3\n"
- "cbz %[regs], 4f\n"
- ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
- "ldr d4, [%[a_ptr0]]\n"
- ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
- "ldr temploadreg0, [%[a_ptr0], #0x8]\n"
- ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
- "ldr d8, [%[b_ptr0]]\n"
- ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
- "ldr d9, [%[b_ptr0], #0x10]\n"
- ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n"
- "ins v4.d[1], temploadreg0\n"
- ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n"
- "ldr d10, [%[b_ptr0], #0x20]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ldr d11, [%[b_ptr0], #0x30]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- "ldr d12, [%[b_ptr0], #0x40]\n"
- "ins v8.d[1], temploadreg0\n"
- "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- "ldr d13, [%[b_ptr0], #0x50]\n"
- "ins v9.d[1], temploadreg1\n"
- ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- "ins v10.d[1], temploadreg2\n"
- ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- "ldr d15, [%[b_ptr0], #0x70]\n"
- "ins v11.d[1], temploadreg3\n"
- ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- "ins v12.d[1], temploadreg0\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n"
- "ldr d8, [%[b_ptr0], #-0x80]\n"
- "ldr temploadreg0, [%[b_ptr0], #-0x78]\n"
- ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n"
- "ldr d9, [%[b_ptr0], #-0x70]\n"
- "ins v13.d[1], temploadreg1\n"
- "ldr temploadreg1, [%[b_ptr0], #-0x68]\n"
- "ldr d10, [%[b_ptr0], #-0x60]\n"
- "ins v14.d[1], temploadreg2\n"
- ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n"
- "ldr temploadreg2, [%[b_ptr0], #-0x58]\n"
- "ldr d11, [%[b_ptr0], #-0x50]\n"
- "ins v15.d[1], temploadreg3\n"
- ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n"
- "ldr temploadreg3, [%[b_ptr0], #-0x48]\n"
- "ldr d12, [%[b_ptr0], #-0x40]\n"
- "ins v8.d[1], temploadreg0\n"
- ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n"
- "ldr temploadreg0, [%[b_ptr0], #-0x38]\n"
- "ldr d13, [%[b_ptr0], #-0x30]\n"
- "ins v9.d[1], temploadreg1\n"
- ".inst 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n"
- "ldr temploadreg1, [%[b_ptr0], #-0x28]\n"
- "ldr d14, [%[b_ptr0], #-0x20]\n"
- "ins v10.d[1], temploadreg2\n"
- ".inst 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n"
- "ldr temploadreg2, [%[b_ptr0], #-0x18]\n"
- "ldr d15, [%[b_ptr0], #-0x10]\n"
- "ins v11.d[1], temploadreg3\n"
- ".inst 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n"
- "ldr temploadreg3, [%[b_ptr0], #-0x8]\n"
- "ldr d8, [%[b_ptr0]]\n"
- "ins v12.d[1], temploadreg0\n"
- ".inst 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- "ldr d9, [%[b_ptr0], #0x10]\n"
- "ins v13.d[1], temploadreg1\n"
- ".inst 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- "ldr d10, [%[b_ptr0], #0x20]\n"
- "ins v14.d[1], temploadreg2\n"
- ".inst 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- "ldr d11, [%[b_ptr0], #0x30]\n"
- "ins v15.d[1], temploadreg3\n"
- ".inst 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- "ldr d12, [%[b_ptr0], #0x40]\n"
- "ins v8.d[1], temploadreg0\n"
- ".inst 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- "ldr d13, [%[b_ptr0], #0x50]\n"
- "ins v9.d[1], temploadreg1\n"
- ".inst 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- "ins v10.d[1], temploadreg2\n"
- ".inst 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- "ldr d15, [%[b_ptr0], #0x70]\n"
- "ins v11.d[1], temploadreg3\n"
- ".inst 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- "ins v12.d[1], temploadreg0\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- ".inst 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n"
- "ins v13.d[1], temploadreg1\n"
- "ins v14.d[1], temploadreg2\n"
- ".inst 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n"
- "ins v15.d[1], temploadreg3\n"
- ".inst 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n"
- ".inst 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n"
- ".inst 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n"
- "b 5f\n"
- "4:\n"
- ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
- "ldr d8, [%[b_ptr0]]\n"
- ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
- "ldr d9, [%[b_ptr0], #0x10]\n"
- ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n"
- "ldr d10, [%[b_ptr0], #0x20]\n"
- ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n"
- "ldr d11, [%[b_ptr0], #0x30]\n"
- ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- "ldr d12, [%[b_ptr0], #0x40]\n"
- "ins v8.d[1], temploadreg0\n"
- "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- "ldr d13, [%[b_ptr0], #0x50]\n"
- "ins v9.d[1], temploadreg1\n"
- ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- "ins v10.d[1], temploadreg2\n"
- ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- "ldr d15, [%[b_ptr0], #0x70]\n"
- "ins v11.d[1], temploadreg3\n"
- ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- "ins v12.d[1], temploadreg0\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n"
- "ins v13.d[1], temploadreg1\n"
- "ins v14.d[1], temploadreg2\n"
- ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n"
- "ins v15.d[1], temploadreg3\n"
- ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n"
- ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n"
- ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n"
- "5:\n"
- "cbz %[blocks], 6f\n"
- "7:\n"
- "ldr q8, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "ldr s0, [%[a_ptr0]]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "add %[a_ptr0], %[a_ptr0], #0x4\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "add %[b_ptr0], %[b_ptr0], #0x40\n"
- ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
- ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
- ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
- ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
- "b.ne 7b\n"
- "6:\n"
- "cbz %[odds], 8f\n"
- "ld1 {v0.b}[0], [%[a_ptr0]], #1\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 9f\n"
- "ld1 {v0.b}[1], [%[a_ptr0]], #1\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 9f\n"
- "ld1 {v0.b}[2], [%[a_ptr0]]\n"
- "9:\n"
- "ldr q8, [%[b_ptr0]]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
- ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
- ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
- ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
- "8:\n"
- "str q16, [%[c_ptr0]]\n"
- "str q17, [%[c_ptr0], #0x10]\n"
- "str q18, [%[c_ptr0], #0x20]\n"
- "str q19, [%[c_ptr0], #0x30]\n"
- "add %[c_ptr0], %[c_ptr0], #0x40\n"
- ".unreq temploadreg0\n"
- ".unreq temploadreg1\n"
- ".unreq temploadreg2\n"
- ".unreq temploadreg3\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks), [odds] "+r" (odds)
- : [width] "r" (width), [accumulate] "r" (static_cast<uint64_t>(accumulate)), [lda] "r" (ldab), [ldc] "r" (ldcb)
- : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "cc", "memory"
- );
- break;
- case 2:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "c_ptr1 .req X1\n"
- "temploadreg0 .req X2\n"
- "temploadreg1 .req X3\n"
- "temploadreg2 .req X4\n"
- "temploadreg3 .req X5\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "cbnz %[accumulate], 1f\n"
- "movi v16.4s, #0\n"
- "ldr q0, [%[a_ptr0]]\n"
- "movi v17.4s, #0\n"
- "ldr q1, [a_ptr1]\n"
- "movi v18.4s, #0\n"
- "ldr q8, [%[b_ptr0]]\n"
- "movi v19.4s, #0\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "movi v20.4s, #0\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "movi v21.4s, #0\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "movi v22.4s, #0\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- "movi v23.4s, #0\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "ldr q16, [%[c_ptr0]]\n"
- "ldr q17, [%[c_ptr0], #0x10]\n"
- "ldr q18, [%[c_ptr0], #0x20]\n"
- "ldr q19, [%[c_ptr0], #0x30]\n"
- "ldr q20, [c_ptr1]\n"
- "ldr q21, [c_ptr1, #0x10]\n"
- "ldr q22, [c_ptr1, #0x20]\n"
- "ldr q23, [c_ptr1, #0x30]\n"
- "ldr q0, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ldr q1, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "ldr q8, [%[b_ptr0]]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "cbz %[loops], 2f\n"
- "3:\n"
- ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
- "ins v14.d[1], temploadreg2\n"
- ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
- "ldr d15, [%[b_ptr0], #-0x10]\n"
- ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
- "ldr temploadreg3, [%[b_ptr0], #-0x8]\n"
- ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
- "ldr d4, [%[a_ptr0]]\n"
- ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
- "ldr temploadreg0, [%[a_ptr0], #0x8]\n"
- ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
- "ldr d5, [a_ptr1]\n"
- ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
- "ldr temploadreg1, [a_ptr1, #0x8]\n"
- ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
- "ldr d8, [%[b_ptr0]]\n"
- ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n"
- "ins v4.d[1], temploadreg0\n"
- ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n"
- "ldr d9, [%[b_ptr0], #0x10]\n"
- ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n"
- "ins v5.d[1], temploadreg1\n"
- ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n"
- "ldr d10, [%[b_ptr0], #0x20]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ldr d11, [%[b_ptr0], #0x30]\n"
- "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n"
- "ins v15.d[1], temploadreg3\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
- "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- "add a_ptr1, a_ptr1, #0x20\n"
- ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n"
- "ldr d12, [%[b_ptr0], #0x40]\n"
- ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n"
- "ins v8.d[1], temploadreg0\n"
- "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- "prfm PLDL1KEEP, [a_ptr1, #0x40]\n"
- "ldr d13, [%[b_ptr0], #0x50]\n"
- ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n"
- "ins v9.d[1], temploadreg1\n"
- ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- "ins v10.d[1], temploadreg2\n"
- ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n"
- "ldr d15, [%[b_ptr0], #0x70]\n"
- "ins v11.d[1], temploadreg3\n"
- ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n"
- "ins v12.d[1], temploadreg0\n"
- "ins v13.d[1], temploadreg1\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n"
- "ldr d8, [%[b_ptr0], #-0x80]\n"
- ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n"
- "ldr temploadreg0, [%[b_ptr0], #-0x78]\n"
- ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n"
- "ldr d9, [%[b_ptr0], #-0x70]\n"
- ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n"
- "ldr temploadreg1, [%[b_ptr0], #-0x68]\n"
- ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n"
- "ldr d10, [%[b_ptr0], #-0x60]\n"
- ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n"
- "ins v14.d[1], temploadreg2\n"
- "ldr temploadreg2, [%[b_ptr0], #-0x58]\n"
- "ldr d11, [%[b_ptr0], #-0x50]\n"
- "ins v15.d[1], temploadreg3\n"
- ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n"
- "ldr temploadreg3, [%[b_ptr0], #-0x48]\n"
- ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n"
- "ldr d12, [%[b_ptr0], #-0x40]\n"
- "ins v8.d[1], temploadreg0\n"
- ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n"
- "ldr temploadreg0, [%[b_ptr0], #-0x38]\n"
- ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n"
- "ldr d13, [%[b_ptr0], #-0x30]\n"
- "ins v9.d[1], temploadreg1\n"
- ".inst 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n"
- "ldr temploadreg1, [%[b_ptr0], #-0x28]\n"
- ".inst 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n"
- "ldr d14, [%[b_ptr0], #-0x20]\n"
- "ins v10.d[1], temploadreg2\n"
- ".inst 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n"
- "ldr temploadreg2, [%[b_ptr0], #-0x18]\n"
- ".inst 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n"
- "ldr d15, [%[b_ptr0], #-0x10]\n"
- "ins v11.d[1], temploadreg3\n"
- ".inst 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n"
- "ldr temploadreg3, [%[b_ptr0], #-0x8]\n"
- ".inst 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n"
- "ldr d0, [%[a_ptr0], #-0x10]\n"
- "ins v12.d[1], temploadreg0\n"
- ".inst 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n"
- "ldr temploadreg0, [%[a_ptr0], #-0x8]\n"
- ".inst 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n"
- "ldr d1, [a_ptr1, #-0x10]\n"
- "ins v13.d[1], temploadreg1\n"
- ".inst 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n"
- "ldr temploadreg1, [a_ptr1, #-0x8]\n"
- ".inst 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n"
- "ldr d8, [%[b_ptr0]]\n"
- "ins v0.d[1], temploadreg0\n"
- ".inst 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".inst 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n"
- "ldr d9, [%[b_ptr0], #0x10]\n"
- "ins v1.d[1], temploadreg1\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- "ldr d10, [%[b_ptr0], #0x20]\n"
- "ins v14.d[1], temploadreg2\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- "ldr d11, [%[b_ptr0], #0x30]\n"
- "ins v15.d[1], temploadreg3\n"
- ".inst 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- ".inst 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n"
- "ldr d12, [%[b_ptr0], #0x40]\n"
- "ins v8.d[1], temploadreg0\n"
- ".inst 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- ".inst 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n"
- "ldr d13, [%[b_ptr0], #0x50]\n"
- "ins v9.d[1], temploadreg1\n"
- ".inst 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- ".inst 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- "ins v10.d[1], temploadreg2\n"
- ".inst 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- ".inst 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n"
- "ldr d15, [%[b_ptr0], #0x70]\n"
- "ins v11.d[1], temploadreg3\n"
- ".inst 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- ".inst 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n"
- "ins v12.d[1], temploadreg0\n"
- "ins v13.d[1], temploadreg1\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".inst 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n"
- "ldr d8, [%[b_ptr0], #-0x80]\n"
- ".inst 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n"
- "ldr temploadreg0, [%[b_ptr0], #-0x78]\n"
- ".inst 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n"
- "ldr d9, [%[b_ptr0], #-0x70]\n"
- ".inst 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n"
- "ldr temploadreg1, [%[b_ptr0], #-0x68]\n"
- ".inst 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n"
- "ldr d10, [%[b_ptr0], #-0x60]\n"
- ".inst 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n"
- "ins v14.d[1], temploadreg2\n"
- "ldr temploadreg2, [%[b_ptr0], #-0x58]\n"
- "ldr d11, [%[b_ptr0], #-0x50]\n"
- "ins v15.d[1], temploadreg3\n"
- ".inst 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n"
- "ldr temploadreg3, [%[b_ptr0], #-0x48]\n"
- ".inst 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n"
- "ldr d12, [%[b_ptr0], #-0x40]\n"
- "ins v8.d[1], temploadreg0\n"
- ".inst 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n"
- "ldr temploadreg0, [%[b_ptr0], #-0x38]\n"
- ".inst 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n"
- "ldr d13, [%[b_ptr0], #-0x30]\n"
- "ins v9.d[1], temploadreg1\n"
- "ldr temploadreg1, [%[b_ptr0], #-0x28]\n"
- "ldr d14, [%[b_ptr0], #-0x20]\n"
- "ins v10.d[1], temploadreg2\n"
- "ldr temploadreg2, [%[b_ptr0], #-0x18]\n"
- "ins v11.d[1], temploadreg3\n"
- "ins v12.d[1], temploadreg0\n"
- "ins v13.d[1], temploadreg1\n"
- "b.ne 3b\n"
- "2:\n"
- "ins v14.d[1], temploadreg2\n"
- "prfm PSTL1KEEP, [%[c_ptr0]]\n"
- "ldr d15, [%[b_ptr0], #-0x10]\n"
- "prfm PSTL1KEEP, [c_ptr1]\n"
- "ldr temploadreg3, [%[b_ptr0], #-0x8]\n"
- "ins v15.d[1], temploadreg3\n"
- "cbz %[regs], 4f\n"
- ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
- "ldr d4, [%[a_ptr0]]\n"
- ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
- "ldr temploadreg0, [%[a_ptr0], #0x8]\n"
- ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
- "ldr d5, [a_ptr1]\n"
- ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
- "ldr temploadreg1, [a_ptr1, #0x8]\n"
- ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
- "ldr d8, [%[b_ptr0]]\n"
- ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
- "ins v4.d[1], temploadreg0\n"
- ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
- "ldr d9, [%[b_ptr0], #0x10]\n"
- ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n"
- "ins v5.d[1], temploadreg1\n"
- ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n"
- "ldr d10, [%[b_ptr0], #0x20]\n"
- ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n"
- "ldr d11, [%[b_ptr0], #0x30]\n"
- ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n"
- "ldr d12, [%[b_ptr0], #0x40]\n"
- ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n"
- "ins v8.d[1], temploadreg0\n"
- "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ldr d13, [%[b_ptr0], #0x50]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n"
- "ins v9.d[1], temploadreg1\n"
- ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- "ins v10.d[1], temploadreg2\n"
- ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n"
- "ldr d15, [%[b_ptr0], #0x70]\n"
- "ins v11.d[1], temploadreg3\n"
- ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n"
- "ins v12.d[1], temploadreg0\n"
- "ins v13.d[1], temploadreg1\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n"
- "ldr d8, [%[b_ptr0], #-0x80]\n"
- ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n"
- "ldr temploadreg0, [%[b_ptr0], #-0x78]\n"
- ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n"
- "ldr d9, [%[b_ptr0], #-0x70]\n"
- ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n"
- "ldr temploadreg1, [%[b_ptr0], #-0x68]\n"
- ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n"
- "ldr d10, [%[b_ptr0], #-0x60]\n"
- ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n"
- "ins v14.d[1], temploadreg2\n"
- "ldr temploadreg2, [%[b_ptr0], #-0x58]\n"
- "ldr d11, [%[b_ptr0], #-0x50]\n"
- "ins v15.d[1], temploadreg3\n"
- ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n"
- "ldr temploadreg3, [%[b_ptr0], #-0x48]\n"
- ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n"
- "ldr d12, [%[b_ptr0], #-0x40]\n"
- "ins v8.d[1], temploadreg0\n"
- ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n"
- "ldr temploadreg0, [%[b_ptr0], #-0x38]\n"
- ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n"
- "ldr d13, [%[b_ptr0], #-0x30]\n"
- "ins v9.d[1], temploadreg1\n"
- ".inst 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n"
- "ldr temploadreg1, [%[b_ptr0], #-0x28]\n"
- ".inst 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n"
- "ldr d14, [%[b_ptr0], #-0x20]\n"
- "ins v10.d[1], temploadreg2\n"
- ".inst 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n"
- "ldr temploadreg2, [%[b_ptr0], #-0x18]\n"
- ".inst 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n"
- "ldr d15, [%[b_ptr0], #-0x10]\n"
- "ins v11.d[1], temploadreg3\n"
- ".inst 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n"
- "ldr temploadreg3, [%[b_ptr0], #-0x8]\n"
- ".inst 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n"
- "ldr d8, [%[b_ptr0]]\n"
- "ins v12.d[1], temploadreg0\n"
- ".inst 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".inst 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n"
- "ldr d9, [%[b_ptr0], #0x10]\n"
- "ins v13.d[1], temploadreg1\n"
- ".inst 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".inst 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n"
- "ldr d10, [%[b_ptr0], #0x20]\n"
- "ins v14.d[1], temploadreg2\n"
- ".inst 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- ".inst 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n"
- "ldr d11, [%[b_ptr0], #0x30]\n"
- "ins v15.d[1], temploadreg3\n"
- ".inst 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- ".inst 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n"
- "ldr d12, [%[b_ptr0], #0x40]\n"
- "ins v8.d[1], temploadreg0\n"
- ".inst 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- ".inst 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n"
- "ldr d13, [%[b_ptr0], #0x50]\n"
- "ins v9.d[1], temploadreg1\n"
- ".inst 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- ".inst 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- "ins v10.d[1], temploadreg2\n"
- ".inst 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- ".inst 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n"
- "ldr d15, [%[b_ptr0], #0x70]\n"
- "ins v11.d[1], temploadreg3\n"
- ".inst 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- ".inst 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n"
- "ins v12.d[1], temploadreg0\n"
- "ins v13.d[1], temploadreg1\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- ".inst 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n"
- "ins v14.d[1], temploadreg2\n"
- ".inst 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n"
- "ins v15.d[1], temploadreg3\n"
- ".inst 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n"
- ".inst 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n"
- ".inst 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n"
- ".inst 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n"
- ".inst 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n"
- ".inst 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n"
- ".inst 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n"
- ".inst 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n"
- "b 5f\n"
- "4:\n"
- ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
- "ldr d8, [%[b_ptr0]]\n"
- ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
- "ldr d9, [%[b_ptr0], #0x10]\n"
- ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
- "ldr d10, [%[b_ptr0], #0x20]\n"
- ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
- "ldr d11, [%[b_ptr0], #0x30]\n"
- ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n"
- "ins v8.d[1], temploadreg0\n"
- ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n"
- "ldr d12, [%[b_ptr0], #0x40]\n"
- ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n"
- "ldr d13, [%[b_ptr0], #0x50]\n"
- ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n"
- "ins v9.d[1], temploadreg1\n"
- ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n"
- "ins v10.d[1], temploadreg2\n"
- ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n"
- "ldr d15, [%[b_ptr0], #0x70]\n"
- ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n"
- "ins v11.d[1], temploadreg3\n"
- ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n"
- "ins v12.d[1], temploadreg0\n"
- ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n"
- "ins v13.d[1], temploadreg1\n"
- ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n"
- "ins v14.d[1], temploadreg2\n"
- ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n"
- "ins v15.d[1], temploadreg3\n"
- ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n"
- ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n"
- ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n"
- ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n"
- ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n"
- ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n"
- ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n"
- "5:\n"
- "cbz %[blocks], 6f\n"
- "7:\n"
- "ldr q8, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "ldr s0, [%[a_ptr0]]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "add %[a_ptr0], %[a_ptr0], #0x4\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "add %[b_ptr0], %[b_ptr0], #0x40\n"
- ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
- "ldr s1, [a_ptr1]\n"
- ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
- "add a_ptr1, a_ptr1, #0x4\n"
- ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
- ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
- ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
- ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
- ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
- ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
- "b.ne 7b\n"
- "6:\n"
- "cbz %[odds], 8f\n"
- "ld1 {v0.b}[0], [%[a_ptr0]], #1\n"
- "ld1 {v1.b}[0], [a_ptr1], #1\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 9f\n"
- "ld1 {v0.b}[1], [%[a_ptr0]], #1\n"
- "ld1 {v1.b}[1], [a_ptr1], #1\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 9f\n"
- "ld1 {v0.b}[2], [%[a_ptr0]]\n"
- "ld1 {v1.b}[2], [a_ptr1]\n"
- "9:\n"
- "ldr q8, [%[b_ptr0]]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
- ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
- ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
- ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
- ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
- ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
- ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
- ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
- "8:\n"
- "str q16, [%[c_ptr0]]\n"
- "str q17, [%[c_ptr0], #0x10]\n"
- "str q18, [%[c_ptr0], #0x20]\n"
- "str q19, [%[c_ptr0], #0x30]\n"
- "add %[c_ptr0], %[c_ptr0], #0x40\n"
- "str q20, [c_ptr1]\n"
- "str q21, [c_ptr1, #0x10]\n"
- "str q22, [c_ptr1, #0x20]\n"
- "str q23, [c_ptr1, #0x30]\n"
- ".unreq a_ptr1\n"
- ".unreq c_ptr1\n"
- ".unreq temploadreg0\n"
- ".unreq temploadreg1\n"
- ".unreq temploadreg2\n"
- ".unreq temploadreg3\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks), [odds] "+r" (odds)
- : [width] "r" (width), [accumulate] "r" (static_cast<uint64_t>(accumulate)), [lda] "r" (ldab), [ldc] "r" (ldcb)
- : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "x4", "x5", "cc", "memory"
- );
- break;
- case 3:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "c_ptr1 .req X2\n"
- "c_ptr2 .req X3\n"
- "temploadreg0 .req X4\n"
- "temploadreg1 .req X5\n"
- "temploadreg2 .req X6\n"
- "temploadreg3 .req X7\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "cbnz %[accumulate], 1f\n"
- "movi v16.4s, #0\n"
- "ldr q0, [%[a_ptr0]]\n"
- "movi v17.4s, #0\n"
- "ldr q1, [a_ptr1]\n"
- "movi v18.4s, #0\n"
- "ldr q2, [a_ptr2]\n"
- "movi v19.4s, #0\n"
- "ldr q8, [%[b_ptr0]]\n"
- "movi v20.4s, #0\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "movi v21.4s, #0\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "movi v22.4s, #0\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "movi v23.4s, #0\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- "movi v24.4s, #0\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- "movi v25.4s, #0\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- "movi v26.4s, #0\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- "movi v27.4s, #0\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "ins v14.d[1], temploadreg2\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "ldr q16, [%[c_ptr0]]\n"
- "ldr q17, [%[c_ptr0], #0x10]\n"
- "ldr q18, [%[c_ptr0], #0x20]\n"
- "ldr q19, [%[c_ptr0], #0x30]\n"
- "ldr q20, [c_ptr1]\n"
- "ldr q21, [c_ptr1, #0x10]\n"
- "ldr q22, [c_ptr1, #0x20]\n"
- "ldr q23, [c_ptr1, #0x30]\n"
- "ldr q24, [c_ptr2]\n"
- "ldr q25, [c_ptr2, #0x10]\n"
- "ldr q26, [c_ptr2, #0x20]\n"
- "ldr q27, [c_ptr2, #0x30]\n"
- "ldr q0, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ldr q1, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "ldr q2, [a_ptr2]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "ldr q8, [%[b_ptr0]]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "ins v14.d[1], temploadreg2\n"
- "cbz %[loops], 2f\n"
- "3:\n"
- ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
- "ldr d15, [%[b_ptr0], #-0x10]\n"
- ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
- "ldr temploadreg3, [%[b_ptr0], #-0x8]\n"
- ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n"
- "ldr d4, [%[a_ptr0]]\n"
- ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
- "ldr temploadreg0, [%[a_ptr0], #0x8]\n"
- ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
- "ldr d5, [a_ptr1]\n"
- ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n"
- "ldr temploadreg1, [a_ptr1, #0x8]\n"
- ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
- "ldr d6, [a_ptr2]\n"
- ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
- "ldr temploadreg2, [a_ptr2, #0x8]\n"
- ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n"
- "ldr d8, [%[b_ptr0]]\n"
- ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
- "ins v4.d[1], temploadreg0\n"
- ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n"
- "ldr d9, [%[b_ptr0], #0x10]\n"
- ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n"
- "ins v5.d[1], temploadreg1\n"
- ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".inst 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n"
- "ldr d10, [%[b_ptr0], #0x20]\n"
- ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n"
- "ins v6.d[1], temploadreg2\n"
- ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- ".inst 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n"
- "ldr d11, [%[b_ptr0], #0x30]\n"
- ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n"
- "ins v15.d[1], temploadreg3\n"
- ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- ".inst 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n"
- "ldr d12, [%[b_ptr0], #0x40]\n"
- "ins v8.d[1], temploadreg0\n"
- "subs %[loops], %[loops], #0x1\n"
- ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n"
- "ldr d13, [%[b_ptr0], #0x50]\n"
- ".inst 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n"
- "ins v9.d[1], temploadreg1\n"
- ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- ".inst 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n"
- "ins v10.d[1], temploadreg2\n"
- ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n"
- "ldr d15, [%[b_ptr0], #0x70]\n"
- ".inst 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n"
- "ins v11.d[1], temploadreg3\n"
- ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n"
- "ins v12.d[1], temploadreg0\n"
- ".inst 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n"
- "ins v13.d[1], temploadreg1\n"
- ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n"
- "ins v14.d[1], temploadreg2\n"
- ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n"
- "ins v15.d[1], temploadreg3\n"
- ".inst 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n"
- "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n"
- ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n"
- "ldr d8, [%[b_ptr0], #-0x80]\n"
- ".inst 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n"
- "ldr temploadreg0, [%[b_ptr0], #-0x78]\n"
- ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n"
- "ldr d9, [%[b_ptr0], #-0x70]\n"
- ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n"
- "ldr temploadreg1, [%[b_ptr0], #-0x68]\n"
- ".inst 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n"
- "ldr d10, [%[b_ptr0], #-0x60]\n"
- ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n"
- "ldr temploadreg2, [%[b_ptr0], #-0x58]\n"
- ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n"
- "ldr d11, [%[b_ptr0], #-0x50]\n"
- ".inst 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n"
- "ldr temploadreg3, [%[b_ptr0], #-0x48]\n"
- ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n"
- "ldr d12, [%[b_ptr0], #-0x40]\n"
- ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n"
- "ins v8.d[1], temploadreg0\n"
- ".inst 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n"
- "ldr temploadreg0, [%[b_ptr0], #-0x38]\n"
- "ldr d13, [%[b_ptr0], #-0x30]\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
- ".inst 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n"
- "ins v9.d[1], temploadreg1\n"
- ".inst 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n"
- "ldr temploadreg1, [%[b_ptr0], #-0x28]\n"
- ".inst 0x4f86e118 // sdot v24.4s, v8.16b, v6.4b[0]\n"
- "ldr d14, [%[b_ptr0], #-0x20]\n"
- "ins v10.d[1], temploadreg2\n"
- "add a_ptr1, a_ptr1, #0x20\n"
- ".inst 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n"
- "ldr temploadreg2, [%[b_ptr0], #-0x18]\n"
- ".inst 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n"
- "ldr d15, [%[b_ptr0], #-0x10]\n"
- ".inst 0x4f86e139 // sdot v25.4s, v9.16b, v6.4b[0]\n"
- "ins v11.d[1], temploadreg3\n"
- ".inst 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n"
- "ldr temploadreg3, [%[b_ptr0], #-0x8]\n"
- ".inst 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n"
- "ldr d0, [%[a_ptr0], #-0x10]\n"
- ".inst 0x4f86e15a // sdot v26.4s, v10.16b, v6.4b[0]\n"
- "ins v12.d[1], temploadreg0\n"
- ".inst 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n"
- "ldr temploadreg0, [%[a_ptr0], #-0x8]\n"
- ".inst 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n"
- "ldr d1, [a_ptr1, #-0x10]\n"
- ".inst 0x4f86e17b // sdot v27.4s, v11.16b, v6.4b[0]\n"
- "ins v13.d[1], temploadreg1\n"
- ".inst 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n"
- "ldr temploadreg1, [a_ptr1, #-0x8]\n"
- ".inst 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n"
- "ins v14.d[1], temploadreg2\n"
- ".inst 0x4fa6e198 // sdot v24.4s, v12.16b, v6.4b[1]\n"
- "ldr d8, [%[b_ptr0]]\n"
- ".inst 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n"
- "ins v0.d[1], temploadreg0\n"
- ".inst 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".inst 0x4fa6e1b9 // sdot v25.4s, v13.16b, v6.4b[1]\n"
- "ldr d9, [%[b_ptr0], #0x10]\n"
- ".inst 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n"
- "ins v1.d[1], temploadreg1\n"
- ".inst 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".inst 0x4fa6e1da // sdot v26.4s, v14.16b, v6.4b[1]\n"
- "ldr d10, [%[b_ptr0], #0x20]\n"
- "ldr d11, [%[b_ptr0], #0x30]\n"
- "add a_ptr2, a_ptr2, #0x20\n"
- "ins v15.d[1], temploadreg3\n"
- "prfm PLDL1KEEP, [a_ptr1, #0x40]\n"
- "ldr d2, [a_ptr2, #-0x10]\n"
- "prfm PLDL1KEEP, [a_ptr2, #0x40]\n"
- ".inst 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n"
- "ldr temploadreg2, [a_ptr2, #-0x8]\n"
- ".inst 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- ".inst 0x4fa6e1fb // sdot v27.4s, v15.16b, v6.4b[1]\n"
- "ldr d12, [%[b_ptr0], #0x40]\n"
- "ins v8.d[1], temploadreg0\n"
- "ins v2.d[1], temploadreg2\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- ".inst 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n"
- "ldr d13, [%[b_ptr0], #0x50]\n"
- ".inst 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n"
- "ins v9.d[1], temploadreg1\n"
- ".inst 0x4f86e918 // sdot v24.4s, v8.16b, v6.4b[2]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- "ins v10.d[1], temploadreg2\n"
- ".inst 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- ".inst 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n"
- "ldr d15, [%[b_ptr0], #0x70]\n"
- ".inst 0x4f86e939 // sdot v25.4s, v9.16b, v6.4b[2]\n"
- "ins v11.d[1], temploadreg3\n"
- ".inst 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- ".inst 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n"
- "ins v12.d[1], temploadreg0\n"
- ".inst 0x4f86e95a // sdot v26.4s, v10.16b, v6.4b[2]\n"
- "ins v13.d[1], temploadreg1\n"
- ".inst 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n"
- "ins v14.d[1], temploadreg2\n"
- ".inst 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n"
- "ins v15.d[1], temploadreg3\n"
- ".inst 0x4f86e97b // sdot v27.4s, v11.16b, v6.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".inst 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n"
- "ldr d8, [%[b_ptr0], #-0x80]\n"
- ".inst 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n"
- "ldr temploadreg0, [%[b_ptr0], #-0x78]\n"
- ".inst 0x4fa6e998 // sdot v24.4s, v12.16b, v6.4b[3]\n"
- "ldr d9, [%[b_ptr0], #-0x70]\n"
- ".inst 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n"
- "ldr temploadreg1, [%[b_ptr0], #-0x68]\n"
- ".inst 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n"
- "ldr d10, [%[b_ptr0], #-0x60]\n"
- ".inst 0x4fa6e9b9 // sdot v25.4s, v13.16b, v6.4b[3]\n"
- "ldr temploadreg2, [%[b_ptr0], #-0x58]\n"
- ".inst 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n"
- "ldr d11, [%[b_ptr0], #-0x50]\n"
- ".inst 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n"
- "ldr temploadreg3, [%[b_ptr0], #-0x48]\n"
- ".inst 0x4fa6e9da // sdot v26.4s, v14.16b, v6.4b[3]\n"
- "ldr d12, [%[b_ptr0], #-0x40]\n"
- ".inst 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n"
- "ins v8.d[1], temploadreg0\n"
- ".inst 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n"
- "ldr temploadreg0, [%[b_ptr0], #-0x38]\n"
- ".inst 0x4fa6e9fb // sdot v27.4s, v15.16b, v6.4b[3]\n"
- "ldr d13, [%[b_ptr0], #-0x30]\n"
- "ins v9.d[1], temploadreg1\n"
- "ldr temploadreg1, [%[b_ptr0], #-0x28]\n"
- "ldr d14, [%[b_ptr0], #-0x20]\n"
- "ins v10.d[1], temploadreg2\n"
- "ldr temploadreg2, [%[b_ptr0], #-0x18]\n"
- "ins v11.d[1], temploadreg3\n"
- "ins v12.d[1], temploadreg0\n"
- "ins v13.d[1], temploadreg1\n"
- "ins v14.d[1], temploadreg2\n"
- "b.ne 3b\n"
- "2:\n"
- "ldr d15, [%[b_ptr0], #-0x10]\n"
- "prfm PSTL1KEEP, [%[c_ptr0]]\n"
- "ldr temploadreg3, [%[b_ptr0], #-0x8]\n"
- "prfm PSTL1KEEP, [c_ptr1]\n"
- "prfm PSTL1KEEP, [c_ptr2]\n"
- "ins v15.d[1], temploadreg3\n"
- "cbz %[regs], 4f\n"
- ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
- "ldr d4, [%[a_ptr0]]\n"
- ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
- "ldr temploadreg0, [%[a_ptr0], #0x8]\n"
- ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n"
- "ldr d5, [a_ptr1]\n"
- ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
- "ldr temploadreg1, [a_ptr1, #0x8]\n"
- ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
- "ldr d6, [a_ptr2]\n"
- ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n"
- "ldr temploadreg2, [a_ptr2, #0x8]\n"
- ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
- "ldr d8, [%[b_ptr0]]\n"
- ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
- "ins v4.d[1], temploadreg0\n"
- ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
- "ldr d9, [%[b_ptr0], #0x10]\n"
- ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
- "ins v5.d[1], temploadreg1\n"
- ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n"
- "ldr d10, [%[b_ptr0], #0x20]\n"
- ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n"
- "ins v6.d[1], temploadreg2\n"
- ".inst 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n"
- "ldr d11, [%[b_ptr0], #0x30]\n"
- ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- ".inst 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n"
- "ldr d12, [%[b_ptr0], #0x40]\n"
- ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n"
- "ins v8.d[1], temploadreg0\n"
- ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- ".inst 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n"
- "ldr d13, [%[b_ptr0], #0x50]\n"
- ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n"
- "ins v9.d[1], temploadreg1\n"
- ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- ".inst 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n"
- "ins v10.d[1], temploadreg2\n"
- ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- ".inst 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n"
- "ldr d15, [%[b_ptr0], #0x70]\n"
- ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n"
- "ins v11.d[1], temploadreg3\n"
- ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- ".inst 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n"
- "ins v12.d[1], temploadreg0\n"
- ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n"
- "ins v13.d[1], temploadreg1\n"
- ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n"
- "ins v14.d[1], temploadreg2\n"
- ".inst 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n"
- "ins v15.d[1], temploadreg3\n"
- ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n"
- "ldr d8, [%[b_ptr0], #-0x80]\n"
- ".inst 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n"
- "ldr temploadreg0, [%[b_ptr0], #-0x78]\n"
- ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n"
- "ldr d9, [%[b_ptr0], #-0x70]\n"
- ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n"
- "ldr temploadreg1, [%[b_ptr0], #-0x68]\n"
- ".inst 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n"
- "ldr d10, [%[b_ptr0], #-0x60]\n"
- ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n"
- "ldr temploadreg2, [%[b_ptr0], #-0x58]\n"
- ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n"
- "ldr d11, [%[b_ptr0], #-0x50]\n"
- ".inst 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n"
- "ldr temploadreg3, [%[b_ptr0], #-0x48]\n"
- ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n"
- "ldr d12, [%[b_ptr0], #-0x40]\n"
- ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n"
- "ins v8.d[1], temploadreg0\n"
- ".inst 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n"
- "ldr temploadreg0, [%[b_ptr0], #-0x38]\n"
- ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n"
- "ldr d13, [%[b_ptr0], #-0x30]\n"
- ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n"
- "ins v9.d[1], temploadreg1\n"
- ".inst 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n"
- "ldr temploadreg1, [%[b_ptr0], #-0x28]\n"
- ".inst 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n"
- "ldr d14, [%[b_ptr0], #-0x20]\n"
- ".inst 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n"
- "ins v10.d[1], temploadreg2\n"
- ".inst 0x4f86e118 // sdot v24.4s, v8.16b, v6.4b[0]\n"
- "ldr temploadreg2, [%[b_ptr0], #-0x18]\n"
- ".inst 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n"
- "ldr d15, [%[b_ptr0], #-0x10]\n"
- ".inst 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n"
- "ins v11.d[1], temploadreg3\n"
- ".inst 0x4f86e139 // sdot v25.4s, v9.16b, v6.4b[0]\n"
- "ldr temploadreg3, [%[b_ptr0], #-0x8]\n"
- ".inst 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n"
- "ldr d8, [%[b_ptr0]]\n"
- ".inst 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n"
- "ins v12.d[1], temploadreg0\n"
- ".inst 0x4f86e15a // sdot v26.4s, v10.16b, v6.4b[0]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".inst 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n"
- "ldr d9, [%[b_ptr0], #0x10]\n"
- ".inst 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n"
- "ins v13.d[1], temploadreg1\n"
- ".inst 0x4f86e17b // sdot v27.4s, v11.16b, v6.4b[0]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".inst 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n"
- "ldr d10, [%[b_ptr0], #0x20]\n"
- ".inst 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n"
- "ins v14.d[1], temploadreg2\n"
- ".inst 0x4fa6e198 // sdot v24.4s, v12.16b, v6.4b[1]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- ".inst 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n"
- "ldr d11, [%[b_ptr0], #0x30]\n"
- ".inst 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n"
- "ins v15.d[1], temploadreg3\n"
- ".inst 0x4fa6e1b9 // sdot v25.4s, v13.16b, v6.4b[1]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- ".inst 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n"
- "ldr d12, [%[b_ptr0], #0x40]\n"
- ".inst 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n"
- "ins v8.d[1], temploadreg0\n"
- ".inst 0x4fa6e1da // sdot v26.4s, v14.16b, v6.4b[1]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- ".inst 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n"
- "ldr d13, [%[b_ptr0], #0x50]\n"
- ".inst 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n"
- "ins v9.d[1], temploadreg1\n"
- ".inst 0x4fa6e1fb // sdot v27.4s, v15.16b, v6.4b[1]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- ".inst 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- ".inst 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n"
- "ins v10.d[1], temploadreg2\n"
- ".inst 0x4f86e918 // sdot v24.4s, v8.16b, v6.4b[2]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- ".inst 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n"
- "ldr d15, [%[b_ptr0], #0x70]\n"
- ".inst 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n"
- "ins v11.d[1], temploadreg3\n"
- ".inst 0x4f86e939 // sdot v25.4s, v9.16b, v6.4b[2]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- ".inst 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n"
- "ins v12.d[1], temploadreg0\n"
- ".inst 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n"
- "ins v13.d[1], temploadreg1\n"
- ".inst 0x4f86e95a // sdot v26.4s, v10.16b, v6.4b[2]\n"
- "ins v14.d[1], temploadreg2\n"
- ".inst 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n"
- "ins v15.d[1], temploadreg3\n"
- ".inst 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- ".inst 0x4f86e97b // sdot v27.4s, v11.16b, v6.4b[2]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- ".inst 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- ".inst 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- ".inst 0x4fa6e998 // sdot v24.4s, v12.16b, v6.4b[3]\n"
- ".inst 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n"
- ".inst 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n"
- ".inst 0x4fa6e9b9 // sdot v25.4s, v13.16b, v6.4b[3]\n"
- ".inst 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n"
- ".inst 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n"
- ".inst 0x4fa6e9da // sdot v26.4s, v14.16b, v6.4b[3]\n"
- ".inst 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n"
- ".inst 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n"
- ".inst 0x4fa6e9fb // sdot v27.4s, v15.16b, v6.4b[3]\n"
- "b 5f\n"
- "4:\n"
- ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n"
- "ldr d8, [%[b_ptr0]]\n"
- ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n"
- "ldr d9, [%[b_ptr0], #0x10]\n"
- ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
- "ins v8.d[1], temploadreg0\n"
- ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n"
- "ldr d10, [%[b_ptr0], #0x20]\n"
- ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
- "ins v9.d[1], temploadreg1\n"
- ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n"
- "ldr d11, [%[b_ptr0], #0x30]\n"
- ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n"
- "ins v10.d[1], temploadreg2\n"
- ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- ".inst 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n"
- "ldr d12, [%[b_ptr0], #0x40]\n"
- ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n"
- "ins v11.d[1], temploadreg3\n"
- ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- ".inst 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n"
- "ldr d13, [%[b_ptr0], #0x50]\n"
- ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n"
- "ins v12.d[1], temploadreg0\n"
- ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n"
- ".inst 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n"
- "ins v13.d[1], temploadreg1\n"
- ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n"
- ".inst 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n"
- "ldr d15, [%[b_ptr0], #0x70]\n"
- ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n"
- "ins v14.d[1], temploadreg2\n"
- ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- ".inst 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n"
- "ins v15.d[1], temploadreg3\n"
- ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n"
- ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n"
- ".inst 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n"
- ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n"
- ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n"
- ".inst 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n"
- ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n"
- ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n"
- ".inst 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n"
- ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n"
- ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n"
- ".inst 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n"
- ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n"
- ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n"
- ".inst 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n"
- ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n"
- ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n"
- ".inst 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n"
- ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n"
- ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n"
- ".inst 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n"
- "5:\n"
- "cbz %[blocks], 6f\n"
- "7:\n"
- "ldr q8, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "ldr s0, [%[a_ptr0]]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "add %[a_ptr0], %[a_ptr0], #0x4\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "add %[b_ptr0], %[b_ptr0], #0x40\n"
- ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
- "ldr s1, [a_ptr1]\n"
- ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
- "add a_ptr1, a_ptr1, #0x4\n"
- ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
- "ldr s2, [a_ptr2]\n"
- ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
- "add a_ptr2, a_ptr2, #0x4\n"
- ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
- ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n"
- ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n"
- ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
- ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n"
- ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
- ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
- ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n"
- "b.ne 7b\n"
- "6:\n"
- "cbz %[odds], 8f\n"
- "ld1 {v0.b}[0], [%[a_ptr0]], #1\n"
- "ld1 {v1.b}[0], [a_ptr1], #1\n"
- "ld1 {v2.b}[0], [a_ptr2], #1\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 9f\n"
- "ld1 {v0.b}[1], [%[a_ptr0]], #1\n"
- "ld1 {v1.b}[1], [a_ptr1], #1\n"
- "ld1 {v2.b}[1], [a_ptr2], #1\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 9f\n"
- "ld1 {v0.b}[2], [%[a_ptr0]]\n"
- "ld1 {v1.b}[2], [a_ptr1]\n"
- "ld1 {v2.b}[2], [a_ptr2]\n"
- "9:\n"
- "ldr q8, [%[b_ptr0]]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
- ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
- ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n"
- ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
- ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
- ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n"
- ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
- ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
- ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n"
- ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
- ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
- ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n"
- "8:\n"
- "str q16, [%[c_ptr0]]\n"
- "str q17, [%[c_ptr0], #0x10]\n"
- "str q18, [%[c_ptr0], #0x20]\n"
- "str q19, [%[c_ptr0], #0x30]\n"
- "add %[c_ptr0], %[c_ptr0], #0x40\n"
- "str q20, [c_ptr1]\n"
- "str q21, [c_ptr1, #0x10]\n"
- "str q22, [c_ptr1, #0x20]\n"
- "str q23, [c_ptr1, #0x30]\n"
- "str q24, [c_ptr2]\n"
- "str q25, [c_ptr2, #0x10]\n"
- "str q26, [c_ptr2, #0x20]\n"
- "str q27, [c_ptr2, #0x30]\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- ".unreq temploadreg0\n"
- ".unreq temploadreg1\n"
- ".unreq temploadreg2\n"
- ".unreq temploadreg3\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks), [odds] "+r" (odds)
- : [width] "r" (width), [accumulate] "r" (static_cast<uint64_t>(accumulate)), [lda] "r" (ldab), [ldc] "r" (ldcb)
- : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "cc", "memory"
- );
- break;
- default:
- case 4:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "a_ptr3 .req X2\n"
- "c_ptr1 .req X3\n"
- "c_ptr2 .req X4\n"
- "c_ptr3 .req X5\n"
- "temploadreg0 .req X6\n"
- "temploadreg1 .req X7\n"
- "temploadreg2 .req X8\n"
- "temploadreg3 .req X9\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "add a_ptr3, a_ptr2, %[lda]\n"
- "add c_ptr3, c_ptr2, %[ldc]\n"
- "cbnz %[accumulate], 1f\n"
- "movi v16.4s, #0\n"
- "ldr q0, [%[a_ptr0]]\n"
- "movi v17.4s, #0\n"
- "ldr q1, [a_ptr1]\n"
- "movi v18.4s, #0\n"
- "ldr q2, [a_ptr2]\n"
- "movi v19.4s, #0\n"
- "ldr q3, [a_ptr3]\n"
- "movi v20.4s, #0\n"
- "ldr q8, [%[b_ptr0]]\n"
- "movi v21.4s, #0\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "movi v22.4s, #0\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "movi v23.4s, #0\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "movi v24.4s, #0\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- "movi v25.4s, #0\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- "movi v26.4s, #0\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- "movi v27.4s, #0\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- "movi v28.4s, #0\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "movi v29.4s, #0\n"
- "ins v14.d[1], temploadreg2\n"
- "movi v30.4s, #0\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "movi v31.4s, #0\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "add a_ptr3, a_ptr3, #0x10\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "ldr q16, [%[c_ptr0]]\n"
- "ldr q17, [%[c_ptr0], #0x10]\n"
- "ldr q18, [%[c_ptr0], #0x20]\n"
- "ldr q19, [%[c_ptr0], #0x30]\n"
- "ldr q20, [c_ptr1]\n"
- "ldr q21, [c_ptr1, #0x10]\n"
- "ldr q22, [c_ptr1, #0x20]\n"
- "ldr q23, [c_ptr1, #0x30]\n"
- "ldr q24, [c_ptr2]\n"
- "ldr q25, [c_ptr2, #0x10]\n"
- "ldr q26, [c_ptr2, #0x20]\n"
- "ldr q27, [c_ptr2, #0x30]\n"
- "ldr q28, [c_ptr3]\n"
- "ldr q29, [c_ptr3, #0x10]\n"
- "ldr q30, [c_ptr3, #0x20]\n"
- "ldr q31, [c_ptr3, #0x30]\n"
- "ldr q0, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ldr q1, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "ldr q2, [a_ptr2]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "ldr q3, [a_ptr3]\n"
- "add a_ptr3, a_ptr3, #0x10\n"
- "ldr q8, [%[b_ptr0]]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "ins v14.d[1], temploadreg2\n"
- "cbz %[loops], 2f\n"
- "3:\n"
- ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
- "ldr d15, [%[b_ptr0], #-0x10]\n"
- ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
- "ldr temploadreg3, [%[b_ptr0], #-0x8]\n"
- ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n"
- "ldr d4, [%[a_ptr0]]\n"
- ".inst 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n"
- "ldr temploadreg0, [%[a_ptr0], #0x8]\n"
- ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
- "ldr d5, [a_ptr1]\n"
- ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
- "ldr temploadreg1, [a_ptr1, #0x8]\n"
- ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n"
- "ldr d6, [a_ptr2]\n"
- ".inst 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n"
- "ldr temploadreg2, [a_ptr2, #0x8]\n"
- ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
- "ldr d7, [a_ptr3]\n"
- ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
- "ins v15.d[1], temploadreg3\n"
- ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n"
- "ldr temploadreg3, [a_ptr3, #0x8]\n"
- ".inst 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n"
- "ldr d8, [%[b_ptr0]]\n"
- ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
- "ins v4.d[1], temploadreg0\n"
- ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n"
- "ldr d9, [%[b_ptr0], #0x10]\n"
- ".inst 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n"
- "ins v5.d[1], temploadreg1\n"
- ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n"
- "ldr d10, [%[b_ptr0], #0x20]\n"
- ".inst 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n"
- "ins v6.d[1], temploadreg2\n"
- ".inst 0x4fa3e19c // sdot v28.4s, v12.16b, v3.4b[1]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n"
- "ldr d11, [%[b_ptr0], #0x30]\n"
- ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n"
- "ins v7.d[1], temploadreg3\n"
- ".inst 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- ".inst 0x4fa3e1bd // sdot v29.4s, v13.16b, v3.4b[1]\n"
- "ldr d12, [%[b_ptr0], #0x40]\n"
- ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n"
- "ins v8.d[1], temploadreg0\n"
- ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- ".inst 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n"
- "ldr d13, [%[b_ptr0], #0x50]\n"
- ".inst 0x4fa3e1de // sdot v30.4s, v14.16b, v3.4b[1]\n"
- "ins v9.d[1], temploadreg1\n"
- ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- ".inst 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n"
- "ins v10.d[1], temploadreg2\n"
- ".inst 0x4fa3e1ff // sdot v31.4s, v15.16b, v3.4b[1]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n"
- "ldr d15, [%[b_ptr0], #0x70]\n"
- ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n"
- "ins v11.d[1], temploadreg3\n"
- ".inst 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- ".inst 0x4f83e91c // sdot v28.4s, v8.16b, v3.4b[2]\n"
- "ins v12.d[1], temploadreg0\n"
- ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n"
- "ins v13.d[1], temploadreg1\n"
- ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n"
- "ins v14.d[1], temploadreg2\n"
- ".inst 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n"
- "ins v15.d[1], temploadreg3\n"
- ".inst 0x4f83e93d // sdot v29.4s, v9.16b, v3.4b[2]\n"
- "subs %[loops], %[loops], #0x1\n"
- ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n"
- "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n"
- ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".inst 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n"
- "ldr d8, [%[b_ptr0], #-0x80]\n"
- ".inst 0x4f83e95e // sdot v30.4s, v10.16b, v3.4b[2]\n"
- "ldr temploadreg0, [%[b_ptr0], #-0x78]\n"
- ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n"
- "ldr d9, [%[b_ptr0], #-0x70]\n"
- ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n"
- "ldr temploadreg1, [%[b_ptr0], #-0x68]\n"
- ".inst 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n"
- "ldr d10, [%[b_ptr0], #-0x60]\n"
- ".inst 0x4f83e97f // sdot v31.4s, v11.16b, v3.4b[2]\n"
- "ldr temploadreg2, [%[b_ptr0], #-0x58]\n"
- ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n"
- "ldr d11, [%[b_ptr0], #-0x50]\n"
- ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n"
- "ldr temploadreg3, [%[b_ptr0], #-0x48]\n"
- ".inst 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n"
- "ins v8.d[1], temploadreg0\n"
- ".inst 0x4fa3e99c // sdot v28.4s, v12.16b, v3.4b[3]\n"
- "ldr d12, [%[b_ptr0], #-0x40]\n"
- ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n"
- "ldr temploadreg0, [%[b_ptr0], #-0x38]\n"
- ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n"
- "ins v9.d[1], temploadreg1\n"
- ".inst 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n"
- "ldr temploadreg1, [%[b_ptr0], #-0x28]\n"
- ".inst 0x4fa3e9bd // sdot v29.4s, v13.16b, v3.4b[3]\n"
- "ldr d13, [%[b_ptr0], #-0x30]\n"
- ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n"
- "ins v10.d[1], temploadreg2\n"
- ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n"
- "ldr temploadreg2, [%[b_ptr0], #-0x18]\n"
- ".inst 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n"
- "ins v11.d[1], temploadreg3\n"
- ".inst 0x4fa3e9de // sdot v30.4s, v14.16b, v3.4b[3]\n"
- "ldr d14, [%[b_ptr0], #-0x20]\n"
- ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n"
- "ldr temploadreg3, [%[b_ptr0], #-0x8]\n"
- ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n"
- "ins v12.d[1], temploadreg0\n"
- ".inst 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n"
- "ins v13.d[1], temploadreg1\n"
- ".inst 0x4fa3e9ff // sdot v31.4s, v15.16b, v3.4b[3]\n"
- "ldr d15, [%[b_ptr0], #-0x10]\n"
- ".inst 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n"
- "ins v14.d[1], temploadreg2\n"
- ".inst 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
- ".inst 0x4f86e118 // sdot v24.4s, v8.16b, v6.4b[0]\n"
- "ldr d0, [%[a_ptr0], #-0x10]\n"
- ".inst 0x4f87e11c // sdot v28.4s, v8.16b, v7.4b[0]\n"
- "ldr temploadreg0, [%[a_ptr0], #-0x8]\n"
- ".inst 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n"
- "ins v15.d[1], temploadreg3\n"
- ".inst 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n"
- "ldr d8, [%[b_ptr0]]\n"
- ".inst 0x4f86e139 // sdot v25.4s, v9.16b, v6.4b[0]\n"
- "ins v0.d[1], temploadreg0\n"
- ".inst 0x4f87e13d // sdot v29.4s, v9.16b, v7.4b[0]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".inst 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n"
- "ldr d9, [%[b_ptr0], #0x10]\n"
- ".inst 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n"
- "add a_ptr1, a_ptr1, #0x20\n"
- ".inst 0x4f86e15a // sdot v26.4s, v10.16b, v6.4b[0]\n"
- "ldr d1, [a_ptr1, #-0x10]\n"
- ".inst 0x4f87e15e // sdot v30.4s, v10.16b, v7.4b[0]\n"
- "ldr temploadreg1, [a_ptr1, #-0x8]\n"
- ".inst 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n"
- "ldr d10, [%[b_ptr0], #0x20]\n"
- ".inst 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n"
- "ins v8.d[1], temploadreg0\n"
- ".inst 0x4f86e17b // sdot v27.4s, v11.16b, v6.4b[0]\n"
- "ins v1.d[1], temploadreg1\n"
- ".inst 0x4f87e17f // sdot v31.4s, v11.16b, v7.4b[0]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".inst 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n"
- "ldr d11, [%[b_ptr0], #0x30]\n"
- ".inst 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- ".inst 0x4fa6e198 // sdot v24.4s, v12.16b, v6.4b[1]\n"
- "ins v9.d[1], temploadreg1\n"
- ".inst 0x4fa7e19c // sdot v28.4s, v12.16b, v7.4b[1]\n"
- "ldr d12, [%[b_ptr0], #0x40]\n"
- ".inst 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- ".inst 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n"
- "add a_ptr2, a_ptr2, #0x20\n"
- ".inst 0x4fa6e1b9 // sdot v25.4s, v13.16b, v6.4b[1]\n"
- "ldr d2, [a_ptr2, #-0x10]\n"
- ".inst 0x4fa7e1bd // sdot v29.4s, v13.16b, v7.4b[1]\n"
- "ldr temploadreg2, [a_ptr2, #-0x8]\n"
- ".inst 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n"
- "ldr d13, [%[b_ptr0], #0x50]\n"
- ".inst 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n"
- "ins v12.d[1], temploadreg0\n"
- ".inst 0x4fa6e1da // sdot v26.4s, v14.16b, v6.4b[1]\n"
- "ins v2.d[1], temploadreg2\n"
- ".inst 0x4fa7e1de // sdot v30.4s, v14.16b, v7.4b[1]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- ".inst 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- ".inst 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n"
- "ins v13.d[1], temploadreg1\n"
- ".inst 0x4fa6e1fb // sdot v27.4s, v15.16b, v6.4b[1]\n"
- "ins v10.d[1], temploadreg2\n"
- ".inst 0x4fa7e1ff // sdot v31.4s, v15.16b, v7.4b[1]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- ".inst 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n"
- "ldr d15, [%[b_ptr0], #0x70]\n"
- ".inst 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n"
- "add a_ptr3, a_ptr3, #0x20\n"
- ".inst 0x4f86e918 // sdot v24.4s, v8.16b, v6.4b[2]\n"
- "ldr d3, [a_ptr3, #-0x10]\n"
- ".inst 0x4f87e91c // sdot v28.4s, v8.16b, v7.4b[2]\n"
- "ldr temploadreg3, [a_ptr3, #-0x8]\n"
- ".inst 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n"
- "ins v14.d[1], temploadreg2\n"
- ".inst 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n"
- "prfm PLDL1KEEP, [a_ptr1, #0x40]\n"
- ".inst 0x4f86e939 // sdot v25.4s, v9.16b, v6.4b[2]\n"
- "ins v3.d[1], temploadreg3\n"
- ".inst 0x4f87e93d // sdot v29.4s, v9.16b, v7.4b[2]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- ".inst 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n"
- "prfm PLDL1KEEP, [a_ptr2, #0x40]\n"
- ".inst 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n"
- "ins v11.d[1], temploadreg3\n"
- ".inst 0x4f86e95a // sdot v26.4s, v10.16b, v6.4b[2]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- ".inst 0x4f87e95e // sdot v30.4s, v10.16b, v7.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".inst 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n"
- "ldr d8, [%[b_ptr0], #-0x80]\n"
- ".inst 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n"
- "ldr temploadreg0, [%[b_ptr0], #-0x78]\n"
- ".inst 0x4f86e97b // sdot v27.4s, v11.16b, v6.4b[2]\n"
- "ldr d9, [%[b_ptr0], #-0x70]\n"
- ".inst 0x4f87e97f // sdot v31.4s, v11.16b, v7.4b[2]\n"
- "ldr temploadreg1, [%[b_ptr0], #-0x68]\n"
- ".inst 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n"
- "ldr d10, [%[b_ptr0], #-0x60]\n"
- ".inst 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n"
- "ldr temploadreg2, [%[b_ptr0], #-0x58]\n"
- ".inst 0x4fa6e998 // sdot v24.4s, v12.16b, v6.4b[3]\n"
- "ldr d11, [%[b_ptr0], #-0x50]\n"
- ".inst 0x4fa7e99c // sdot v28.4s, v12.16b, v7.4b[3]\n"
- "ins v15.d[1], temploadreg3\n"
- ".inst 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n"
- "ldr temploadreg3, [%[b_ptr0], #-0x48]\n"
- ".inst 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n"
- "ldr d12, [%[b_ptr0], #-0x40]\n"
- ".inst 0x4fa6e9b9 // sdot v25.4s, v13.16b, v6.4b[3]\n"
- "ins v8.d[1], temploadreg0\n"
- ".inst 0x4fa7e9bd // sdot v29.4s, v13.16b, v7.4b[3]\n"
- "ldr temploadreg0, [%[b_ptr0], #-0x38]\n"
- ".inst 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n"
- "ldr d13, [%[b_ptr0], #-0x30]\n"
- ".inst 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n"
- "ins v9.d[1], temploadreg1\n"
- ".inst 0x4fa6e9da // sdot v26.4s, v14.16b, v6.4b[3]\n"
- "ldr temploadreg1, [%[b_ptr0], #-0x28]\n"
- ".inst 0x4fa7e9de // sdot v30.4s, v14.16b, v7.4b[3]\n"
- "ldr d14, [%[b_ptr0], #-0x20]\n"
- ".inst 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n"
- "ins v10.d[1], temploadreg2\n"
- ".inst 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n"
- "ldr temploadreg2, [%[b_ptr0], #-0x18]\n"
- ".inst 0x4fa6e9fb // sdot v27.4s, v15.16b, v6.4b[3]\n"
- "ins v11.d[1], temploadreg3\n"
- ".inst 0x4fa7e9ff // sdot v31.4s, v15.16b, v7.4b[3]\n"
- "ins v12.d[1], temploadreg0\n"
- "ins v13.d[1], temploadreg1\n"
- "prfm PLDL1KEEP, [a_ptr3, #0x40]\n"
- "ins v14.d[1], temploadreg2\n"
- "b.ne 3b\n"
- "2:\n"
- "ldr d15, [%[b_ptr0], #-0x10]\n"
- "prfm PSTL1KEEP, [%[c_ptr0]]\n"
- "ldr temploadreg3, [%[b_ptr0], #-0x8]\n"
- "prfm PSTL1KEEP, [c_ptr1]\n"
- "prfm PSTL1KEEP, [c_ptr2]\n"
- "prfm PSTL1KEEP, [c_ptr3]\n"
- "ins v15.d[1], temploadreg3\n"
- "cbz %[regs], 4f\n"
- ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
- "ldr d4, [%[a_ptr0]]\n"
- ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
- "ldr temploadreg0, [%[a_ptr0], #0x8]\n"
- ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n"
- "ldr d5, [a_ptr1]\n"
- ".inst 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n"
- "ldr temploadreg1, [a_ptr1, #0x8]\n"
- ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
- "ldr d6, [a_ptr2]\n"
- ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
- "ldr temploadreg2, [a_ptr2, #0x8]\n"
- ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n"
- "ldr d7, [a_ptr3]\n"
- ".inst 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n"
- "ldr temploadreg3, [a_ptr3, #0x8]\n"
- ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
- "ldr d8, [%[b_ptr0]]\n"
- ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
- "ins v4.d[1], temploadreg0\n"
- ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".inst 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n"
- "ldr d9, [%[b_ptr0], #0x10]\n"
- ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
- "ins v5.d[1], temploadreg1\n"
- ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n"
- "ldr d10, [%[b_ptr0], #0x20]\n"
- ".inst 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n"
- "ins v6.d[1], temploadreg2\n"
- ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n"
- "ldr d11, [%[b_ptr0], #0x30]\n"
- ".inst 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n"
- "ins v7.d[1], temploadreg3\n"
- ".inst 0x4fa3e19c // sdot v28.4s, v12.16b, v3.4b[1]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n"
- "ldr d12, [%[b_ptr0], #0x40]\n"
- ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n"
- "ins v8.d[1], temploadreg0\n"
- ".inst 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- ".inst 0x4fa3e1bd // sdot v29.4s, v13.16b, v3.4b[1]\n"
- "ldr d13, [%[b_ptr0], #0x50]\n"
- ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n"
- "ins v9.d[1], temploadreg1\n"
- ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- ".inst 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n"
- "ins v10.d[1], temploadreg2\n"
- ".inst 0x4fa3e1de // sdot v30.4s, v14.16b, v3.4b[1]\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n"
- "ins v11.d[1], temploadreg3\n"
- ".inst 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- ".inst 0x4fa3e1ff // sdot v31.4s, v15.16b, v3.4b[1]\n"
- "ldr d15, [%[b_ptr0], #0x70]\n"
- ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n"
- "ins v12.d[1], temploadreg0\n"
- ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n"
- "ins v13.d[1], temploadreg1\n"
- ".inst 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n"
- "ins v14.d[1], temploadreg2\n"
- ".inst 0x4f83e91c // sdot v28.4s, v8.16b, v3.4b[2]\n"
- "ins v15.d[1], temploadreg3\n"
- ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n"
- "ldr d8, [%[b_ptr0], #-0x80]\n"
- ".inst 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n"
- "ldr temploadreg0, [%[b_ptr0], #-0x78]\n"
- ".inst 0x4f83e93d // sdot v29.4s, v9.16b, v3.4b[2]\n"
- "ldr d9, [%[b_ptr0], #-0x70]\n"
- ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n"
- "ldr temploadreg1, [%[b_ptr0], #-0x68]\n"
- ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n"
- "ldr temploadreg2, [%[b_ptr0], #-0x58]\n"
- ".inst 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n"
- "ldr temploadreg3, [%[b_ptr0], #-0x48]\n"
- ".inst 0x4f83e95e // sdot v30.4s, v10.16b, v3.4b[2]\n"
- "ldr d10, [%[b_ptr0], #-0x60]\n"
- ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n"
- "ins v8.d[1], temploadreg0\n"
- ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n"
- "ldr temploadreg0, [%[b_ptr0], #-0x38]\n"
- ".inst 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n"
- "ins v9.d[1], temploadreg1\n"
- ".inst 0x4f83e97f // sdot v31.4s, v11.16b, v3.4b[2]\n"
- "ldr d11, [%[b_ptr0], #-0x50]\n"
- ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n"
- "ldr temploadreg1, [%[b_ptr0], #-0x28]\n"
- ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n"
- "ins v10.d[1], temploadreg2\n"
- ".inst 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n"
- "ldr temploadreg2, [%[b_ptr0], #-0x18]\n"
- ".inst 0x4fa3e99c // sdot v28.4s, v12.16b, v3.4b[3]\n"
- "ldr d12, [%[b_ptr0], #-0x40]\n"
- ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n"
- "ins v11.d[1], temploadreg3\n"
- ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n"
- "ldr temploadreg3, [%[b_ptr0], #-0x8]\n"
- ".inst 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n"
- "ins v12.d[1], temploadreg0\n"
- ".inst 0x4fa3e9bd // sdot v29.4s, v13.16b, v3.4b[3]\n"
- "ldr d13, [%[b_ptr0], #-0x30]\n"
- ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- ".inst 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n"
- "ins v13.d[1], temploadreg1\n"
- ".inst 0x4fa3e9de // sdot v30.4s, v14.16b, v3.4b[3]\n"
- "ldr d14, [%[b_ptr0], #-0x20]\n"
- ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- ".inst 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n"
- "ins v14.d[1], temploadreg2\n"
- ".inst 0x4fa3e9ff // sdot v31.4s, v15.16b, v3.4b[3]\n"
- "ldr d15, [%[b_ptr0], #-0x10]\n"
- ".inst 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- ".inst 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- ".inst 0x4f86e118 // sdot v24.4s, v8.16b, v6.4b[0]\n"
- "ins v15.d[1], temploadreg3\n"
- ".inst 0x4f87e11c // sdot v28.4s, v8.16b, v7.4b[0]\n"
- "ldr d8, [%[b_ptr0]]\n"
- ".inst 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- ".inst 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n"
- "add a_ptr3, a_ptr3, #0x10\n"
- ".inst 0x4f86e139 // sdot v25.4s, v9.16b, v6.4b[0]\n"
- "ins v8.d[1], temploadreg0\n"
- ".inst 0x4f87e13d // sdot v29.4s, v9.16b, v7.4b[0]\n"
- "ldr d9, [%[b_ptr0], #0x10]\n"
- ".inst 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- ".inst 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n"
- ".inst 0x4f86e15a // sdot v26.4s, v10.16b, v6.4b[0]\n"
- "ins v9.d[1], temploadreg1\n"
- ".inst 0x4f87e15e // sdot v30.4s, v10.16b, v7.4b[0]\n"
- "ldr d10, [%[b_ptr0], #0x20]\n"
- ".inst 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- ".inst 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n"
- ".inst 0x4f86e17b // sdot v27.4s, v11.16b, v6.4b[0]\n"
- "ins v10.d[1], temploadreg2\n"
- ".inst 0x4f87e17f // sdot v31.4s, v11.16b, v7.4b[0]\n"
- "ldr d11, [%[b_ptr0], #0x30]\n"
- ".inst 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- ".inst 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n"
- ".inst 0x4fa6e198 // sdot v24.4s, v12.16b, v6.4b[1]\n"
- "ins v11.d[1], temploadreg3\n"
- ".inst 0x4fa7e19c // sdot v28.4s, v12.16b, v7.4b[1]\n"
- "ldr d12, [%[b_ptr0], #0x40]\n"
- ".inst 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- ".inst 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n"
- ".inst 0x4fa6e1b9 // sdot v25.4s, v13.16b, v6.4b[1]\n"
- "ins v12.d[1], temploadreg0\n"
- ".inst 0x4fa7e1bd // sdot v29.4s, v13.16b, v7.4b[1]\n"
- "ldr d13, [%[b_ptr0], #0x50]\n"
- ".inst 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n"
- ".inst 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n"
- ".inst 0x4fa6e1da // sdot v26.4s, v14.16b, v6.4b[1]\n"
- "ins v13.d[1], temploadreg1\n"
- ".inst 0x4fa7e1de // sdot v30.4s, v14.16b, v7.4b[1]\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- ".inst 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n"
- ".inst 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n"
- ".inst 0x4fa6e1fb // sdot v27.4s, v15.16b, v6.4b[1]\n"
- "ins v14.d[1], temploadreg2\n"
- ".inst 0x4fa7e1ff // sdot v31.4s, v15.16b, v7.4b[1]\n"
- "ldr d15, [%[b_ptr0], #0x70]\n"
- ".inst 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- ".inst 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n"
- "ins v15.d[1], temploadreg3\n"
- ".inst 0x4f86e918 // sdot v24.4s, v8.16b, v6.4b[2]\n"
- ".inst 0x4f87e91c // sdot v28.4s, v8.16b, v7.4b[2]\n"
- ".inst 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n"
- ".inst 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n"
- ".inst 0x4f86e939 // sdot v25.4s, v9.16b, v6.4b[2]\n"
- ".inst 0x4f87e93d // sdot v29.4s, v9.16b, v7.4b[2]\n"
- ".inst 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n"
- ".inst 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n"
- ".inst 0x4f86e95a // sdot v26.4s, v10.16b, v6.4b[2]\n"
- ".inst 0x4f87e95e // sdot v30.4s, v10.16b, v7.4b[2]\n"
- ".inst 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n"
- ".inst 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n"
- ".inst 0x4f86e97b // sdot v27.4s, v11.16b, v6.4b[2]\n"
- ".inst 0x4f87e97f // sdot v31.4s, v11.16b, v7.4b[2]\n"
- ".inst 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n"
- ".inst 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n"
- ".inst 0x4fa6e998 // sdot v24.4s, v12.16b, v6.4b[3]\n"
- ".inst 0x4fa7e99c // sdot v28.4s, v12.16b, v7.4b[3]\n"
- ".inst 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n"
- ".inst 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n"
- ".inst 0x4fa6e9b9 // sdot v25.4s, v13.16b, v6.4b[3]\n"
- ".inst 0x4fa7e9bd // sdot v29.4s, v13.16b, v7.4b[3]\n"
- ".inst 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n"
- ".inst 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n"
- ".inst 0x4fa6e9da // sdot v26.4s, v14.16b, v6.4b[3]\n"
- ".inst 0x4fa7e9de // sdot v30.4s, v14.16b, v7.4b[3]\n"
- ".inst 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n"
- ".inst 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n"
- ".inst 0x4fa6e9fb // sdot v27.4s, v15.16b, v6.4b[3]\n"
- ".inst 0x4fa7e9ff // sdot v31.4s, v15.16b, v7.4b[3]\n"
- "b 5f\n"
- "4:\n"
- ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- ".inst 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n"
- "ldr d8, [%[b_ptr0]]\n"
- ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
- ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n"
- "ins v8.d[1], temploadreg0\n"
- ".inst 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n"
- "ldr d9, [%[b_ptr0], #0x10]\n"
- ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
- ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n"
- "ins v9.d[1], temploadreg1\n"
- ".inst 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n"
- "ldr d10, [%[b_ptr0], #0x20]\n"
- ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
- ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n"
- "ins v10.d[1], temploadreg2\n"
- ".inst 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n"
- "ldr d11, [%[b_ptr0], #0x30]\n"
- ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n"
- ".inst 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n"
- "ins v11.d[1], temploadreg3\n"
- ".inst 0x4fa3e19c // sdot v28.4s, v12.16b, v3.4b[1]\n"
- "ldr d12, [%[b_ptr0], #0x40]\n"
- ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n"
- ".inst 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n"
- "ins v12.d[1], temploadreg0\n"
- ".inst 0x4fa3e1bd // sdot v29.4s, v13.16b, v3.4b[1]\n"
- "ldr d13, [%[b_ptr0], #0x50]\n"
- ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n"
- ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n"
- ".inst 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n"
- "ins v13.d[1], temploadreg1\n"
- ".inst 0x4fa3e1de // sdot v30.4s, v14.16b, v3.4b[1]\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n"
- ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n"
- ".inst 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n"
- "ins v14.d[1], temploadreg2\n"
- ".inst 0x4fa3e1ff // sdot v31.4s, v15.16b, v3.4b[1]\n"
- "ldr d15, [%[b_ptr0], #0x70]\n"
- ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n"
- "ins v15.d[1], temploadreg3\n"
- ".inst 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n"
- ".inst 0x4f83e91c // sdot v28.4s, v8.16b, v3.4b[2]\n"
- ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n"
- ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n"
- ".inst 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n"
- ".inst 0x4f83e93d // sdot v29.4s, v9.16b, v3.4b[2]\n"
- ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n"
- ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n"
- ".inst 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n"
- ".inst 0x4f83e95e // sdot v30.4s, v10.16b, v3.4b[2]\n"
- ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n"
- ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n"
- ".inst 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n"
- ".inst 0x4f83e97f // sdot v31.4s, v11.16b, v3.4b[2]\n"
- ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n"
- ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n"
- ".inst 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n"
- ".inst 0x4fa3e99c // sdot v28.4s, v12.16b, v3.4b[3]\n"
- ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n"
- ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n"
- ".inst 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n"
- ".inst 0x4fa3e9bd // sdot v29.4s, v13.16b, v3.4b[3]\n"
- ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n"
- ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n"
- ".inst 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n"
- ".inst 0x4fa3e9de // sdot v30.4s, v14.16b, v3.4b[3]\n"
- ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n"
- ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n"
- ".inst 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n"
- ".inst 0x4fa3e9ff // sdot v31.4s, v15.16b, v3.4b[3]\n"
- "5:\n"
- "cbz %[blocks], 6f\n"
- "7:\n"
- "ldr q8, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "ldr s0, [%[a_ptr0]]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "add %[a_ptr0], %[a_ptr0], #0x4\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "add %[b_ptr0], %[b_ptr0], #0x40\n"
- ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
- "ldr s1, [a_ptr1]\n"
- ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
- "add a_ptr1, a_ptr1, #0x4\n"
- ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
- "ldr s2, [a_ptr2]\n"
- ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
- "add a_ptr2, a_ptr2, #0x4\n"
- ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
- "ldr s3, [a_ptr3]\n"
- ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n"
- "add a_ptr3, a_ptr3, #0x4\n"
- ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n"
- ".inst 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n"
- ".inst 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n"
- ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
- ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n"
- ".inst 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n"
- ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
- ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
- ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n"
- ".inst 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n"
- "b.ne 7b\n"
- "6:\n"
- "cbz %[odds], 8f\n"
- "ld1 {v0.b}[0], [%[a_ptr0]], #1\n"
- "ld1 {v1.b}[0], [a_ptr1], #1\n"
- "ld1 {v2.b}[0], [a_ptr2], #1\n"
- "ld1 {v3.b}[0], [a_ptr3], #1\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 9f\n"
- "ld1 {v0.b}[1], [%[a_ptr0]], #1\n"
- "ld1 {v1.b}[1], [a_ptr1], #1\n"
- "ld1 {v2.b}[1], [a_ptr2], #1\n"
- "ld1 {v3.b}[1], [a_ptr3], #1\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 9f\n"
- "ld1 {v0.b}[2], [%[a_ptr0]]\n"
- "ld1 {v1.b}[2], [a_ptr1]\n"
- "ld1 {v2.b}[2], [a_ptr2]\n"
- "ld1 {v3.b}[2], [a_ptr3]\n"
- "9:\n"
- "ldr q8, [%[b_ptr0]]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
- ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
- ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n"
- ".inst 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n"
- ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
- ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
- ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n"
- ".inst 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n"
- ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
- ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
- ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n"
- ".inst 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n"
- ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
- ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
- ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n"
- ".inst 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n"
- "8:\n"
- "str q16, [%[c_ptr0]]\n"
- "str q17, [%[c_ptr0], #0x10]\n"
- "str q18, [%[c_ptr0], #0x20]\n"
- "str q19, [%[c_ptr0], #0x30]\n"
- "add %[c_ptr0], %[c_ptr0], #0x40\n"
- "str q20, [c_ptr1]\n"
- "str q21, [c_ptr1, #0x10]\n"
- "str q22, [c_ptr1, #0x20]\n"
- "str q23, [c_ptr1, #0x30]\n"
- "str q24, [c_ptr2]\n"
- "str q25, [c_ptr2, #0x10]\n"
- "str q26, [c_ptr2, #0x20]\n"
- "str q27, [c_ptr2, #0x30]\n"
- "str q28, [c_ptr3]\n"
- "str q29, [c_ptr3, #0x10]\n"
- "str q30, [c_ptr3, #0x20]\n"
- "str q31, [c_ptr3, #0x30]\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq a_ptr3\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- ".unreq c_ptr3\n"
- ".unreq temploadreg0\n"
- ".unreq temploadreg1\n"
- ".unreq temploadreg2\n"
- ".unreq temploadreg3\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks), [odds] "+r" (odds)
- : [width] "r" (width), [accumulate] "r" (static_cast<uint64_t>(accumulate)), [lda] "r" (ldab), [ldc] "r" (ldcb)
- : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "cc", "memory"
- );
- break;
- }
- if (use_result_buffer) {
- for(int cy=0; cy<std::min(M-y, 4); cy++) {
- for(unsigned int cx=0; cx<width; cx++) {
- c_ptr_real[cy * ldc + cx] = result_buffer[cy * 16 + cx];
- }
- }
- }
- }
- }
-}
-
-} // namespace arm_gemm
-
-#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_16x4/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_16x4/generic.cpp
deleted file mode 100644
index da39a32..0000000
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_16x4/generic.cpp
+++ /dev/null
@@ -1,1808 +0,0 @@
-/*
- * Copyright (c) 2018-2020 Arm Limited.
- *
- * SPDX-License-Identifier: MIT
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to
- * deal in the Software without restriction, including without limitation the
- * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
- * sell copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-#ifdef __aarch64__
-
-#include <algorithm>
-
-#include "arm_gemm.hpp"
-#include <cstdint>
-#include "../../asmlib.hpp"
-#include "../../utils.hpp"
-
-namespace arm_gemm {
-
-void a64_hybrid_s8s32_dot_16x4(const int8_t *A, int lda, const int8_t *B, int32_t *C, int ldc, int M, int N, int K, const int32_t *, Activation , bool accumulate) {
- const int K_stride = ((K + 3) / 4) * 4;
- const long loops_count = ((K + 16) / 32) - 1;
- K -= loops_count * 32;
- const long regs_count = (K / 16) - 1;
- K -= (regs_count + 1) * 16;
- const long blocks_count = K / 4;
- const long odds_count = K - (blocks_count * 4);
-
- int rows_to_compute;
-
- for (int y=0; y<M; y+=rows_to_compute) {
- const int8_t * const a_ptr0_base = A + (y * lda);
- const unsigned long ldab = lda * sizeof(int8_t);
-
- int32_t *c_ptr0 = C + (y * ldc);
-
- rows_to_compute = M-y;
- if (rows_to_compute > 4) {
- if (rows_to_compute % 4) {
- rows_to_compute = 4 - 1;
- } else {
- rows_to_compute = 4;
- }
- }
-
- for (int x0=0; x0<N; x0+=16ul) {
- const long width = std::min((unsigned long)N-x0, 16ul);
- long loops = loops_count;
- long regs = regs_count;
- long blocks = blocks_count;
- long odds = odds_count;
- const int8_t *a_ptr0 = a_ptr0_base;
- const int8_t *b_ptr0 = B + (K_stride * x0);
- const bool use_result_buffer = (width < 16);
- int32_t result_buffer[64];
- const unsigned long ldcb = (use_result_buffer ? 16 : ldc) * sizeof(int32_t);
- int32_t *c_ptr_real = c_ptr0;
- if (use_result_buffer && accumulate) {
- for(int cy=0; cy<std::min(M-y, 4); cy++) {
- for(unsigned int cx=0; cx<width; cx++) {
- result_buffer[cy * 16 + cx] = c_ptr_real[cy * ldc + cx];
- }
- }
- }
- if (use_result_buffer) {
- c_ptr0 = result_buffer;
- }
-
- switch(rows_to_compute) {
- case 1:
- __asm __volatile (
- "cbnz %[accumulate], 1f\n"
- "movi v16.4s, #0\n"
- "ldr q0, [%[a_ptr0]]\n"
- "movi v17.4s, #0\n"
- "ldr q8, [%[b_ptr0]]\n"
- "movi v18.4s, #0\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "movi v19.4s, #0\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "ldr q16, [%[c_ptr0]]\n"
- "ldr q17, [%[c_ptr0], #0x10]\n"
- "ldr q18, [%[c_ptr0], #0x20]\n"
- "ldr q19, [%[c_ptr0], #0x30]\n"
- "ldr q0, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ldr q8, [%[b_ptr0]]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "cbz %[loops], 2f\n"
- "3:\n"
- ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
- "ldr q15, [%[b_ptr0], #-0x10]\n"
- ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
- "ldr q4, [%[a_ptr0]]\n"
- ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
- "ldr q8, [%[b_ptr0]]\n"
- ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n"
- "ldr q15, [%[b_ptr0], #0x70]\n"
- ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n"
- "subs %[loops], %[loops], #0x1\n"
- ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n"
- "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n"
- ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n"
- "ldr q8, [%[b_ptr0], #-0x80]\n"
- ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n"
- "ldr q9, [%[b_ptr0], #-0x70]\n"
- ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n"
- "ldr q10, [%[b_ptr0], #-0x60]\n"
- "ldr q11, [%[b_ptr0], #-0x50]\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
- ".inst 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n"
- "ldr q12, [%[b_ptr0], #-0x40]\n"
- ".inst 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n"
- "ldr q13, [%[b_ptr0], #-0x30]\n"
- ".inst 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n"
- "ldr q14, [%[b_ptr0], #-0x20]\n"
- ".inst 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n"
- "ldr q15, [%[b_ptr0], #-0x10]\n"
- ".inst 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n"
- "ldr q0, [%[a_ptr0], #-0x10]\n"
- ".inst 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n"
- "ldr q8, [%[b_ptr0]]\n"
- ".inst 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- ".inst 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- ".inst 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- ".inst 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- ".inst 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- ".inst 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n"
- "ldr q15, [%[b_ptr0], #0x70]\n"
- ".inst 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".inst 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n"
- "ldr q8, [%[b_ptr0], #-0x80]\n"
- ".inst 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n"
- "ldr q9, [%[b_ptr0], #-0x70]\n"
- ".inst 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n"
- "ldr q10, [%[b_ptr0], #-0x60]\n"
- "ldr q11, [%[b_ptr0], #-0x50]\n"
- "ldr q12, [%[b_ptr0], #-0x40]\n"
- "ldr q13, [%[b_ptr0], #-0x30]\n"
- "ldr q14, [%[b_ptr0], #-0x20]\n"
- "b.ne 3b\n"
- "2:\n"
- "ldr q15, [%[b_ptr0], #-0x10]\n"
- "prfm PSTL1KEEP, [%[c_ptr0]]\n"
- "cbz %[regs], 4f\n"
- ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
- "ldr q4, [%[a_ptr0]]\n"
- ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
- "ldr q8, [%[b_ptr0]]\n"
- ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n"
- "ldr q15, [%[b_ptr0], #0x70]\n"
- ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n"
- "ldr q8, [%[b_ptr0], #-0x80]\n"
- ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n"
- "ldr q9, [%[b_ptr0], #-0x70]\n"
- ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n"
- "ldr q10, [%[b_ptr0], #-0x60]\n"
- ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n"
- "ldr q11, [%[b_ptr0], #-0x50]\n"
- ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n"
- "ldr q12, [%[b_ptr0], #-0x40]\n"
- ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n"
- "ldr q13, [%[b_ptr0], #-0x30]\n"
- ".inst 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n"
- "ldr q14, [%[b_ptr0], #-0x20]\n"
- ".inst 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n"
- "ldr q15, [%[b_ptr0], #-0x10]\n"
- ".inst 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n"
- "ldr q8, [%[b_ptr0]]\n"
- ".inst 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- ".inst 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- ".inst 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- ".inst 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- ".inst 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- ".inst 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- ".inst 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n"
- "ldr q15, [%[b_ptr0], #0x70]\n"
- ".inst 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- ".inst 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- ".inst 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n"
- ".inst 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n"
- ".inst 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n"
- ".inst 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n"
- "b 5f\n"
- "4:\n"
- ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
- "ldr q8, [%[b_ptr0]]\n"
- ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n"
- "ldr q15, [%[b_ptr0], #0x70]\n"
- ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n"
- ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n"
- ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n"
- ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n"
- ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n"
- ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n"
- ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n"
- "5:\n"
- "cbz %[blocks], 6f\n"
- "7:\n"
- "ldr q8, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "ldr s0, [%[a_ptr0]]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "add %[a_ptr0], %[a_ptr0], #0x4\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "add %[b_ptr0], %[b_ptr0], #0x40\n"
- ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
- ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
- ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
- ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
- "b.ne 7b\n"
- "6:\n"
- "cbz %[odds], 8f\n"
- "ld1 {v0.b}[0], [%[a_ptr0]], #1\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 9f\n"
- "ld1 {v0.b}[1], [%[a_ptr0]], #1\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 9f\n"
- "ld1 {v0.b}[2], [%[a_ptr0]]\n"
- "9:\n"
- "ldr q8, [%[b_ptr0]]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
- ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
- ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
- ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
- "8:\n"
- "str q16, [%[c_ptr0]]\n"
- "str q17, [%[c_ptr0], #0x10]\n"
- "str q18, [%[c_ptr0], #0x20]\n"
- "str q19, [%[c_ptr0], #0x30]\n"
- "add %[c_ptr0], %[c_ptr0], #0x40\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks), [odds] "+r" (odds)
- : [width] "r" (width), [accumulate] "r" (static_cast<uint64_t>(accumulate)), [lda] "r" (ldab), [ldc] "r" (ldcb)
- : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "cc", "memory"
- );
- break;
- case 2:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "c_ptr1 .req X1\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "cbnz %[accumulate], 1f\n"
- "movi v16.4s, #0\n"
- "ldr q0, [%[a_ptr0]]\n"
- "movi v17.4s, #0\n"
- "ldr q1, [a_ptr1]\n"
- "movi v18.4s, #0\n"
- "ldr q8, [%[b_ptr0]]\n"
- "movi v19.4s, #0\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "movi v20.4s, #0\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "movi v21.4s, #0\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "movi v22.4s, #0\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- "movi v23.4s, #0\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "ldr q16, [%[c_ptr0]]\n"
- "ldr q17, [%[c_ptr0], #0x10]\n"
- "ldr q18, [%[c_ptr0], #0x20]\n"
- "ldr q19, [%[c_ptr0], #0x30]\n"
- "ldr q20, [c_ptr1]\n"
- "ldr q21, [c_ptr1, #0x10]\n"
- "ldr q22, [c_ptr1, #0x20]\n"
- "ldr q23, [c_ptr1, #0x30]\n"
- "ldr q0, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ldr q1, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "ldr q8, [%[b_ptr0]]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "cbz %[loops], 2f\n"
- "3:\n"
- ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
- "ldr q15, [%[b_ptr0], #-0x10]\n"
- ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
- "ldr q4, [%[a_ptr0]]\n"
- ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
- "ldr q5, [a_ptr1]\n"
- ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
- "ldr q8, [%[b_ptr0]]\n"
- ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
- "subs %[loops], %[loops], #0x1\n"
- ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n"
- "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n"
- ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
- ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n"
- "add a_ptr1, a_ptr1, #0x20\n"
- ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n"
- "prfm PLDL1KEEP, [a_ptr1, #0x40]\n"
- ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n"
- "ldr q15, [%[b_ptr0], #0x70]\n"
- ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n"
- "ldr q8, [%[b_ptr0], #-0x80]\n"
- ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n"
- ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n"
- "ldr q9, [%[b_ptr0], #-0x70]\n"
- ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n"
- ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n"
- "ldr q10, [%[b_ptr0], #-0x60]\n"
- ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n"
- ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n"
- "ldr q11, [%[b_ptr0], #-0x50]\n"
- ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n"
- ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n"
- "ldr q12, [%[b_ptr0], #-0x40]\n"
- ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n"
- ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n"
- "ldr q13, [%[b_ptr0], #-0x30]\n"
- ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n"
- ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n"
- "ldr q14, [%[b_ptr0], #-0x20]\n"
- ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n"
- "ldr q0, [%[a_ptr0], #-0x10]\n"
- ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n"
- "ldr q15, [%[b_ptr0], #-0x10]\n"
- ".inst 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n"
- "ldr q1, [a_ptr1, #-0x10]\n"
- ".inst 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n"
- "ldr q8, [%[b_ptr0]]\n"
- ".inst 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n"
- ".inst 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- ".inst 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n"
- ".inst 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- ".inst 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n"
- ".inst 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- ".inst 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n"
- ".inst 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- ".inst 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n"
- ".inst 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- ".inst 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n"
- ".inst 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- ".inst 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n"
- ".inst 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n"
- "ldr q15, [%[b_ptr0], #0x70]\n"
- ".inst 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".inst 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n"
- "ldr q8, [%[b_ptr0], #-0x80]\n"
- ".inst 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n"
- ".inst 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n"
- "ldr q9, [%[b_ptr0], #-0x70]\n"
- ".inst 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n"
- ".inst 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n"
- "ldr q10, [%[b_ptr0], #-0x60]\n"
- ".inst 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n"
- ".inst 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n"
- "ldr q11, [%[b_ptr0], #-0x50]\n"
- ".inst 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n"
- ".inst 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n"
- "ldr q12, [%[b_ptr0], #-0x40]\n"
- ".inst 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n"
- ".inst 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n"
- "ldr q13, [%[b_ptr0], #-0x30]\n"
- ".inst 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n"
- ".inst 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n"
- "ldr q14, [%[b_ptr0], #-0x20]\n"
- ".inst 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n"
- ".inst 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n"
- "b.ne 3b\n"
- "2:\n"
- "ldr q15, [%[b_ptr0], #-0x10]\n"
- "prfm PSTL1KEEP, [%[c_ptr0]]\n"
- "prfm PSTL1KEEP, [c_ptr1]\n"
- "cbz %[regs], 4f\n"
- ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
- "ldr q4, [%[a_ptr0]]\n"
- ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
- "ldr q5, [a_ptr1]\n"
- ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
- "ldr q8, [%[b_ptr0]]\n"
- ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n"
- ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n"
- ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n"
- ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n"
- ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n"
- "ldr q15, [%[b_ptr0], #0x70]\n"
- ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n"
- "ldr q8, [%[b_ptr0], #-0x80]\n"
- ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n"
- ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n"
- "ldr q9, [%[b_ptr0], #-0x70]\n"
- ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n"
- ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n"
- "ldr q10, [%[b_ptr0], #-0x60]\n"
- ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n"
- ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n"
- "ldr q11, [%[b_ptr0], #-0x50]\n"
- ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n"
- ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n"
- "ldr q12, [%[b_ptr0], #-0x40]\n"
- ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n"
- ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n"
- "ldr q13, [%[b_ptr0], #-0x30]\n"
- ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n"
- ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n"
- "ldr q14, [%[b_ptr0], #-0x20]\n"
- ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n"
- ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n"
- "ldr q15, [%[b_ptr0], #-0x10]\n"
- ".inst 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n"
- ".inst 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n"
- "ldr q8, [%[b_ptr0]]\n"
- ".inst 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n"
- ".inst 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- ".inst 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n"
- ".inst 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- ".inst 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n"
- ".inst 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- ".inst 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n"
- ".inst 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- ".inst 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n"
- ".inst 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- ".inst 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n"
- ".inst 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- ".inst 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n"
- ".inst 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n"
- "ldr q15, [%[b_ptr0], #0x70]\n"
- ".inst 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- ".inst 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n"
- ".inst 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n"
- ".inst 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n"
- ".inst 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n"
- ".inst 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n"
- ".inst 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n"
- ".inst 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n"
- ".inst 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n"
- ".inst 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n"
- ".inst 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n"
- ".inst 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n"
- ".inst 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n"
- ".inst 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n"
- ".inst 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n"
- ".inst 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n"
- "b 5f\n"
- "4:\n"
- ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
- ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
- "ldr q8, [%[b_ptr0]]\n"
- ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
- ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
- ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
- ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n"
- ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n"
- ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n"
- ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n"
- ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n"
- "ldr q15, [%[b_ptr0], #0x70]\n"
- ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n"
- ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n"
- ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n"
- ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n"
- ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n"
- ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n"
- ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n"
- ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n"
- ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n"
- ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n"
- ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n"
- ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n"
- ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n"
- ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n"
- ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n"
- "5:\n"
- "cbz %[blocks], 6f\n"
- "7:\n"
- "ldr q8, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "ldr s0, [%[a_ptr0]]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "add %[a_ptr0], %[a_ptr0], #0x4\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "add %[b_ptr0], %[b_ptr0], #0x40\n"
- ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
- "ldr s1, [a_ptr1]\n"
- ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
- "add a_ptr1, a_ptr1, #0x4\n"
- ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
- ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
- ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
- ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
- ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
- ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
- "b.ne 7b\n"
- "6:\n"
- "cbz %[odds], 8f\n"
- "ld1 {v0.b}[0], [%[a_ptr0]], #1\n"
- "ld1 {v1.b}[0], [a_ptr1], #1\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 9f\n"
- "ld1 {v0.b}[1], [%[a_ptr0]], #1\n"
- "ld1 {v1.b}[1], [a_ptr1], #1\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 9f\n"
- "ld1 {v0.b}[2], [%[a_ptr0]]\n"
- "ld1 {v1.b}[2], [a_ptr1]\n"
- "9:\n"
- "ldr q8, [%[b_ptr0]]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
- ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
- ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
- ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
- ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
- ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
- ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
- ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
- "8:\n"
- "str q16, [%[c_ptr0]]\n"
- "str q17, [%[c_ptr0], #0x10]\n"
- "str q18, [%[c_ptr0], #0x20]\n"
- "str q19, [%[c_ptr0], #0x30]\n"
- "add %[c_ptr0], %[c_ptr0], #0x40\n"
- "str q20, [c_ptr1]\n"
- "str q21, [c_ptr1, #0x10]\n"
- "str q22, [c_ptr1, #0x20]\n"
- "str q23, [c_ptr1, #0x30]\n"
- ".unreq a_ptr1\n"
- ".unreq c_ptr1\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks), [odds] "+r" (odds)
- : [width] "r" (width), [accumulate] "r" (static_cast<uint64_t>(accumulate)), [lda] "r" (ldab), [ldc] "r" (ldcb)
- : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "cc", "memory"
- );
- break;
- case 3:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "c_ptr1 .req X2\n"
- "c_ptr2 .req X3\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "cbnz %[accumulate], 1f\n"
- "movi v16.4s, #0\n"
- "ldr q0, [%[a_ptr0]]\n"
- "movi v17.4s, #0\n"
- "ldr q1, [a_ptr1]\n"
- "movi v18.4s, #0\n"
- "ldr q2, [a_ptr2]\n"
- "movi v19.4s, #0\n"
- "ldr q8, [%[b_ptr0]]\n"
- "movi v20.4s, #0\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "movi v21.4s, #0\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "movi v22.4s, #0\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "movi v23.4s, #0\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- "movi v24.4s, #0\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- "movi v25.4s, #0\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- "movi v26.4s, #0\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "movi v27.4s, #0\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "ldr q16, [%[c_ptr0]]\n"
- "ldr q17, [%[c_ptr0], #0x10]\n"
- "ldr q18, [%[c_ptr0], #0x20]\n"
- "ldr q19, [%[c_ptr0], #0x30]\n"
- "ldr q20, [c_ptr1]\n"
- "ldr q21, [c_ptr1, #0x10]\n"
- "ldr q22, [c_ptr1, #0x20]\n"
- "ldr q23, [c_ptr1, #0x30]\n"
- "ldr q24, [c_ptr2]\n"
- "ldr q25, [c_ptr2, #0x10]\n"
- "ldr q26, [c_ptr2, #0x20]\n"
- "ldr q27, [c_ptr2, #0x30]\n"
- "ldr q0, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ldr q1, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "ldr q2, [a_ptr2]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "ldr q8, [%[b_ptr0]]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "cbz %[loops], 2f\n"
- "3:\n"
- ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
- "ldr q15, [%[b_ptr0], #-0x10]\n"
- ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
- "ldr q4, [%[a_ptr0]]\n"
- ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n"
- "ldr q5, [a_ptr1]\n"
- ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
- "ldr q6, [a_ptr2]\n"
- ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
- "ldr q8, [%[b_ptr0]]\n"
- ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
- "subs %[loops], %[loops], #0x1\n"
- ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
- "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n"
- ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
- ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
- "add a_ptr1, a_ptr1, #0x20\n"
- ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n"
- "add a_ptr2, a_ptr2, #0x20\n"
- ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n"
- "prfm PLDL1KEEP, [a_ptr1, #0x40]\n"
- ".inst 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n"
- "prfm PLDL1KEEP, [a_ptr2, #0x40]\n"
- ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n"
- ".inst 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n"
- ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n"
- ".inst 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n"
- ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n"
- ".inst 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n"
- "ldr q15, [%[b_ptr0], #0x70]\n"
- ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n"
- ".inst 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n"
- "ldr q8, [%[b_ptr0], #-0x80]\n"
- ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n"
- ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n"
- ".inst 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n"
- "ldr q9, [%[b_ptr0], #-0x70]\n"
- ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n"
- ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n"
- ".inst 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n"
- "ldr q10, [%[b_ptr0], #-0x60]\n"
- ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n"
- ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n"
- ".inst 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n"
- "ldr q11, [%[b_ptr0], #-0x50]\n"
- ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n"
- ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n"
- ".inst 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n"
- "ldr q12, [%[b_ptr0], #-0x40]\n"
- ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n"
- ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n"
- ".inst 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n"
- "ldr q13, [%[b_ptr0], #-0x30]\n"
- ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n"
- ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n"
- ".inst 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n"
- "ldr q14, [%[b_ptr0], #-0x20]\n"
- ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n"
- "ldr q0, [%[a_ptr0], #-0x10]\n"
- ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n"
- "ldr q1, [a_ptr1, #-0x10]\n"
- ".inst 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n"
- "ldr q15, [%[b_ptr0], #-0x10]\n"
- ".inst 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n"
- "ldr q2, [a_ptr2, #-0x10]\n"
- ".inst 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n"
- ".inst 0x4f86e118 // sdot v24.4s, v8.16b, v6.4b[0]\n"
- "ldr q8, [%[b_ptr0]]\n"
- ".inst 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n"
- ".inst 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n"
- ".inst 0x4f86e139 // sdot v25.4s, v9.16b, v6.4b[0]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- ".inst 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n"
- ".inst 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n"
- ".inst 0x4f86e15a // sdot v26.4s, v10.16b, v6.4b[0]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- ".inst 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n"
- ".inst 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n"
- ".inst 0x4f86e17b // sdot v27.4s, v11.16b, v6.4b[0]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- ".inst 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n"
- ".inst 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n"
- ".inst 0x4fa6e198 // sdot v24.4s, v12.16b, v6.4b[1]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- ".inst 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n"
- ".inst 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n"
- ".inst 0x4fa6e1b9 // sdot v25.4s, v13.16b, v6.4b[1]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- ".inst 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n"
- ".inst 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n"
- ".inst 0x4fa6e1da // sdot v26.4s, v14.16b, v6.4b[1]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- ".inst 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n"
- ".inst 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n"
- ".inst 0x4fa6e1fb // sdot v27.4s, v15.16b, v6.4b[1]\n"
- "ldr q15, [%[b_ptr0], #0x70]\n"
- ".inst 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".inst 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n"
- ".inst 0x4f86e918 // sdot v24.4s, v8.16b, v6.4b[2]\n"
- "ldr q8, [%[b_ptr0], #-0x80]\n"
- ".inst 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n"
- ".inst 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n"
- ".inst 0x4f86e939 // sdot v25.4s, v9.16b, v6.4b[2]\n"
- "ldr q9, [%[b_ptr0], #-0x70]\n"
- ".inst 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n"
- ".inst 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n"
- ".inst 0x4f86e95a // sdot v26.4s, v10.16b, v6.4b[2]\n"
- "ldr q10, [%[b_ptr0], #-0x60]\n"
- ".inst 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n"
- ".inst 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n"
- ".inst 0x4f86e97b // sdot v27.4s, v11.16b, v6.4b[2]\n"
- "ldr q11, [%[b_ptr0], #-0x50]\n"
- ".inst 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n"
- ".inst 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n"
- ".inst 0x4fa6e998 // sdot v24.4s, v12.16b, v6.4b[3]\n"
- "ldr q12, [%[b_ptr0], #-0x40]\n"
- ".inst 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n"
- ".inst 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n"
- ".inst 0x4fa6e9b9 // sdot v25.4s, v13.16b, v6.4b[3]\n"
- "ldr q13, [%[b_ptr0], #-0x30]\n"
- ".inst 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n"
- ".inst 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n"
- ".inst 0x4fa6e9da // sdot v26.4s, v14.16b, v6.4b[3]\n"
- "ldr q14, [%[b_ptr0], #-0x20]\n"
- ".inst 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n"
- ".inst 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n"
- ".inst 0x4fa6e9fb // sdot v27.4s, v15.16b, v6.4b[3]\n"
- "b.ne 3b\n"
- "2:\n"
- "ldr q15, [%[b_ptr0], #-0x10]\n"
- "prfm PSTL1KEEP, [%[c_ptr0]]\n"
- "prfm PSTL1KEEP, [c_ptr1]\n"
- "prfm PSTL1KEEP, [c_ptr2]\n"
- "cbz %[regs], 4f\n"
- ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
- "ldr q4, [%[a_ptr0]]\n"
- ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
- "ldr q5, [a_ptr1]\n"
- ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n"
- "ldr q6, [a_ptr2]\n"
- ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
- "ldr q8, [%[b_ptr0]]\n"
- ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
- ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
- ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n"
- ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n"
- ".inst 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n"
- ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n"
- ".inst 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n"
- ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n"
- ".inst 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n"
- ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n"
- ".inst 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n"
- "ldr q15, [%[b_ptr0], #0x70]\n"
- ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n"
- ".inst 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n"
- "ldr q8, [%[b_ptr0], #-0x80]\n"
- ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n"
- ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n"
- ".inst 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n"
- "ldr q9, [%[b_ptr0], #-0x70]\n"
- ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n"
- ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n"
- ".inst 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n"
- "ldr q10, [%[b_ptr0], #-0x60]\n"
- ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n"
- ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n"
- ".inst 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n"
- "ldr q11, [%[b_ptr0], #-0x50]\n"
- ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n"
- ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n"
- ".inst 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n"
- "ldr q12, [%[b_ptr0], #-0x40]\n"
- ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n"
- ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n"
- ".inst 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n"
- "ldr q13, [%[b_ptr0], #-0x30]\n"
- ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n"
- ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n"
- ".inst 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n"
- "ldr q14, [%[b_ptr0], #-0x20]\n"
- ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n"
- ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n"
- ".inst 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n"
- "ldr q15, [%[b_ptr0], #-0x10]\n"
- ".inst 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n"
- ".inst 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n"
- ".inst 0x4f86e118 // sdot v24.4s, v8.16b, v6.4b[0]\n"
- "ldr q8, [%[b_ptr0]]\n"
- ".inst 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n"
- ".inst 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n"
- ".inst 0x4f86e139 // sdot v25.4s, v9.16b, v6.4b[0]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- ".inst 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n"
- ".inst 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n"
- ".inst 0x4f86e15a // sdot v26.4s, v10.16b, v6.4b[0]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- ".inst 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n"
- ".inst 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n"
- ".inst 0x4f86e17b // sdot v27.4s, v11.16b, v6.4b[0]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- ".inst 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n"
- ".inst 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n"
- ".inst 0x4fa6e198 // sdot v24.4s, v12.16b, v6.4b[1]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- ".inst 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n"
- ".inst 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n"
- ".inst 0x4fa6e1b9 // sdot v25.4s, v13.16b, v6.4b[1]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- ".inst 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n"
- ".inst 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n"
- ".inst 0x4fa6e1da // sdot v26.4s, v14.16b, v6.4b[1]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- ".inst 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n"
- ".inst 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n"
- ".inst 0x4fa6e1fb // sdot v27.4s, v15.16b, v6.4b[1]\n"
- "ldr q15, [%[b_ptr0], #0x70]\n"
- ".inst 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- ".inst 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n"
- ".inst 0x4f86e918 // sdot v24.4s, v8.16b, v6.4b[2]\n"
- ".inst 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n"
- ".inst 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n"
- ".inst 0x4f86e939 // sdot v25.4s, v9.16b, v6.4b[2]\n"
- ".inst 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n"
- ".inst 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n"
- ".inst 0x4f86e95a // sdot v26.4s, v10.16b, v6.4b[2]\n"
- ".inst 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n"
- ".inst 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n"
- ".inst 0x4f86e97b // sdot v27.4s, v11.16b, v6.4b[2]\n"
- ".inst 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n"
- ".inst 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n"
- ".inst 0x4fa6e998 // sdot v24.4s, v12.16b, v6.4b[3]\n"
- ".inst 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n"
- ".inst 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n"
- ".inst 0x4fa6e9b9 // sdot v25.4s, v13.16b, v6.4b[3]\n"
- ".inst 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n"
- ".inst 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n"
- ".inst 0x4fa6e9da // sdot v26.4s, v14.16b, v6.4b[3]\n"
- ".inst 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n"
- ".inst 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n"
- ".inst 0x4fa6e9fb // sdot v27.4s, v15.16b, v6.4b[3]\n"
- "b 5f\n"
- "4:\n"
- ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
- ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
- ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n"
- "ldr q8, [%[b_ptr0]]\n"
- ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
- ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
- ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
- ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
- ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
- ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
- ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n"
- ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n"
- ".inst 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n"
- ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n"
- ".inst 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n"
- ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n"
- ".inst 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n"
- ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n"
- ".inst 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n"
- "ldr q15, [%[b_ptr0], #0x70]\n"
- ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n"
- ".inst 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n"
- ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n"
- ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n"
- ".inst 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n"
- ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n"
- ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n"
- ".inst 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n"
- ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n"
- ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n"
- ".inst 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n"
- ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n"
- ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n"
- ".inst 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n"
- ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n"
- ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n"
- ".inst 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n"
- ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n"
- ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n"
- ".inst 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n"
- ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n"
- ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n"
- ".inst 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n"
- "5:\n"
- "cbz %[blocks], 6f\n"
- "7:\n"
- "ldr q8, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "ldr s0, [%[a_ptr0]]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "add %[a_ptr0], %[a_ptr0], #0x4\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "add %[b_ptr0], %[b_ptr0], #0x40\n"
- ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
- "ldr s1, [a_ptr1]\n"
- ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
- "add a_ptr1, a_ptr1, #0x4\n"
- ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
- "ldr s2, [a_ptr2]\n"
- ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
- "add a_ptr2, a_ptr2, #0x4\n"
- ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
- ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n"
- ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n"
- ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
- ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n"
- ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
- ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
- ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n"
- "b.ne 7b\n"
- "6:\n"
- "cbz %[odds], 8f\n"
- "ld1 {v0.b}[0], [%[a_ptr0]], #1\n"
- "ld1 {v1.b}[0], [a_ptr1], #1\n"
- "ld1 {v2.b}[0], [a_ptr2], #1\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 9f\n"
- "ld1 {v0.b}[1], [%[a_ptr0]], #1\n"
- "ld1 {v1.b}[1], [a_ptr1], #1\n"
- "ld1 {v2.b}[1], [a_ptr2], #1\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 9f\n"
- "ld1 {v0.b}[2], [%[a_ptr0]]\n"
- "ld1 {v1.b}[2], [a_ptr1]\n"
- "ld1 {v2.b}[2], [a_ptr2]\n"
- "9:\n"
- "ldr q8, [%[b_ptr0]]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
- ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
- ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n"
- ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
- ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
- ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n"
- ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
- ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
- ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n"
- ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
- ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
- ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n"
- "8:\n"
- "str q16, [%[c_ptr0]]\n"
- "str q17, [%[c_ptr0], #0x10]\n"
- "str q18, [%[c_ptr0], #0x20]\n"
- "str q19, [%[c_ptr0], #0x30]\n"
- "add %[c_ptr0], %[c_ptr0], #0x40\n"
- "str q20, [c_ptr1]\n"
- "str q21, [c_ptr1, #0x10]\n"
- "str q22, [c_ptr1, #0x20]\n"
- "str q23, [c_ptr1, #0x30]\n"
- "str q24, [c_ptr2]\n"
- "str q25, [c_ptr2, #0x10]\n"
- "str q26, [c_ptr2, #0x20]\n"
- "str q27, [c_ptr2, #0x30]\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks), [odds] "+r" (odds)
- : [width] "r" (width), [accumulate] "r" (static_cast<uint64_t>(accumulate)), [lda] "r" (ldab), [ldc] "r" (ldcb)
- : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "cc", "memory"
- );
- break;
- default:
- case 4:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "a_ptr3 .req X2\n"
- "c_ptr1 .req X3\n"
- "c_ptr2 .req X4\n"
- "c_ptr3 .req X5\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "add a_ptr3, a_ptr2, %[lda]\n"
- "add c_ptr3, c_ptr2, %[ldc]\n"
- "cbnz %[accumulate], 1f\n"
- "movi v16.4s, #0\n"
- "ldr q0, [%[a_ptr0]]\n"
- "movi v17.4s, #0\n"
- "ldr q1, [a_ptr1]\n"
- "movi v18.4s, #0\n"
- "ldr q2, [a_ptr2]\n"
- "movi v19.4s, #0\n"
- "ldr q3, [a_ptr3]\n"
- "movi v20.4s, #0\n"
- "ldr q8, [%[b_ptr0]]\n"
- "movi v21.4s, #0\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "movi v22.4s, #0\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "movi v23.4s, #0\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "movi v24.4s, #0\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- "movi v25.4s, #0\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- "movi v26.4s, #0\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- "movi v27.4s, #0\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "movi v28.4s, #0\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "movi v29.4s, #0\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "movi v30.4s, #0\n"
- "add a_ptr3, a_ptr3, #0x10\n"
- "movi v31.4s, #0\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "ldr q16, [%[c_ptr0]]\n"
- "ldr q17, [%[c_ptr0], #0x10]\n"
- "ldr q18, [%[c_ptr0], #0x20]\n"
- "ldr q19, [%[c_ptr0], #0x30]\n"
- "ldr q20, [c_ptr1]\n"
- "ldr q21, [c_ptr1, #0x10]\n"
- "ldr q22, [c_ptr1, #0x20]\n"
- "ldr q23, [c_ptr1, #0x30]\n"
- "ldr q24, [c_ptr2]\n"
- "ldr q25, [c_ptr2, #0x10]\n"
- "ldr q26, [c_ptr2, #0x20]\n"
- "ldr q27, [c_ptr2, #0x30]\n"
- "ldr q28, [c_ptr3]\n"
- "ldr q29, [c_ptr3, #0x10]\n"
- "ldr q30, [c_ptr3, #0x20]\n"
- "ldr q31, [c_ptr3, #0x30]\n"
- "ldr q0, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ldr q1, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "ldr q2, [a_ptr2]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "ldr q3, [a_ptr3]\n"
- "add a_ptr3, a_ptr3, #0x10\n"
- "ldr q8, [%[b_ptr0]]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "cbz %[loops], 2f\n"
- "3:\n"
- ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
- "ldr q15, [%[b_ptr0], #-0x10]\n"
- ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
- "ldr q4, [%[a_ptr0]]\n"
- ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n"
- "ldr q5, [a_ptr1]\n"
- ".inst 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n"
- "ldr q6, [a_ptr2]\n"
- ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
- "ldr q7, [a_ptr3]\n"
- ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
- "ldr q8, [%[b_ptr0]]\n"
- ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n"
- "subs %[loops], %[loops], #0x1\n"
- ".inst 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
- "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n"
- ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
- ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n"
- "add a_ptr1, a_ptr1, #0x20\n"
- ".inst 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
- "add a_ptr2, a_ptr2, #0x20\n"
- ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
- "add a_ptr3, a_ptr3, #0x20\n"
- ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n"
- "prfm PLDL1KEEP, [a_ptr1, #0x40]\n"
- ".inst 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n"
- "prfm PLDL1KEEP, [a_ptr2, #0x40]\n"
- ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n"
- "prfm PLDL1KEEP, [a_ptr3, #0x40]\n"
- ".inst 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n"
- ".inst 0x4fa3e19c // sdot v28.4s, v12.16b, v3.4b[1]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n"
- ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n"
- ".inst 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n"
- ".inst 0x4fa3e1bd // sdot v29.4s, v13.16b, v3.4b[1]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n"
- ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n"
- ".inst 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n"
- ".inst 0x4fa3e1de // sdot v30.4s, v14.16b, v3.4b[1]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n"
- ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n"
- ".inst 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n"
- ".inst 0x4fa3e1ff // sdot v31.4s, v15.16b, v3.4b[1]\n"
- "ldr q15, [%[b_ptr0], #0x70]\n"
- ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n"
- ".inst 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n"
- ".inst 0x4f83e91c // sdot v28.4s, v8.16b, v3.4b[2]\n"
- "ldr q8, [%[b_ptr0], #-0x80]\n"
- ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n"
- ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n"
- ".inst 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n"
- ".inst 0x4f83e93d // sdot v29.4s, v9.16b, v3.4b[2]\n"
- "ldr q9, [%[b_ptr0], #-0x70]\n"
- ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n"
- ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n"
- ".inst 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n"
- ".inst 0x4f83e95e // sdot v30.4s, v10.16b, v3.4b[2]\n"
- "ldr q10, [%[b_ptr0], #-0x60]\n"
- ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n"
- ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n"
- ".inst 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n"
- ".inst 0x4f83e97f // sdot v31.4s, v11.16b, v3.4b[2]\n"
- "ldr q11, [%[b_ptr0], #-0x50]\n"
- ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n"
- ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n"
- ".inst 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n"
- ".inst 0x4fa3e99c // sdot v28.4s, v12.16b, v3.4b[3]\n"
- "ldr q12, [%[b_ptr0], #-0x40]\n"
- ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n"
- ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n"
- ".inst 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n"
- ".inst 0x4fa3e9bd // sdot v29.4s, v13.16b, v3.4b[3]\n"
- "ldr q13, [%[b_ptr0], #-0x30]\n"
- ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n"
- ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n"
- ".inst 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n"
- ".inst 0x4fa3e9de // sdot v30.4s, v14.16b, v3.4b[3]\n"
- "ldr q14, [%[b_ptr0], #-0x20]\n"
- ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n"
- "ldr q0, [%[a_ptr0], #-0x10]\n"
- ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n"
- "ldr q1, [a_ptr1, #-0x10]\n"
- ".inst 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n"
- "ldr q2, [a_ptr2, #-0x10]\n"
- ".inst 0x4fa3e9ff // sdot v31.4s, v15.16b, v3.4b[3]\n"
- "ldr q15, [%[b_ptr0], #-0x10]\n"
- ".inst 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n"
- "ldr q3, [a_ptr3, #-0x10]\n"
- ".inst 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n"
- ".inst 0x4f86e118 // sdot v24.4s, v8.16b, v6.4b[0]\n"
- ".inst 0x4f87e11c // sdot v28.4s, v8.16b, v7.4b[0]\n"
- "ldr q8, [%[b_ptr0]]\n"
- ".inst 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n"
- ".inst 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n"
- ".inst 0x4f86e139 // sdot v25.4s, v9.16b, v6.4b[0]\n"
- ".inst 0x4f87e13d // sdot v29.4s, v9.16b, v7.4b[0]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- ".inst 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n"
- ".inst 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n"
- ".inst 0x4f86e15a // sdot v26.4s, v10.16b, v6.4b[0]\n"
- ".inst 0x4f87e15e // sdot v30.4s, v10.16b, v7.4b[0]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- ".inst 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n"
- ".inst 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n"
- ".inst 0x4f86e17b // sdot v27.4s, v11.16b, v6.4b[0]\n"
- ".inst 0x4f87e17f // sdot v31.4s, v11.16b, v7.4b[0]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- ".inst 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n"
- ".inst 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n"
- ".inst 0x4fa6e198 // sdot v24.4s, v12.16b, v6.4b[1]\n"
- ".inst 0x4fa7e19c // sdot v28.4s, v12.16b, v7.4b[1]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- ".inst 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n"
- ".inst 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n"
- ".inst 0x4fa6e1b9 // sdot v25.4s, v13.16b, v6.4b[1]\n"
- ".inst 0x4fa7e1bd // sdot v29.4s, v13.16b, v7.4b[1]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- ".inst 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n"
- ".inst 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n"
- ".inst 0x4fa6e1da // sdot v26.4s, v14.16b, v6.4b[1]\n"
- ".inst 0x4fa7e1de // sdot v30.4s, v14.16b, v7.4b[1]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- ".inst 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n"
- ".inst 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n"
- ".inst 0x4fa6e1fb // sdot v27.4s, v15.16b, v6.4b[1]\n"
- ".inst 0x4fa7e1ff // sdot v31.4s, v15.16b, v7.4b[1]\n"
- "ldr q15, [%[b_ptr0], #0x70]\n"
- ".inst 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".inst 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n"
- ".inst 0x4f86e918 // sdot v24.4s, v8.16b, v6.4b[2]\n"
- ".inst 0x4f87e91c // sdot v28.4s, v8.16b, v7.4b[2]\n"
- "ldr q8, [%[b_ptr0], #-0x80]\n"
- ".inst 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n"
- ".inst 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n"
- ".inst 0x4f86e939 // sdot v25.4s, v9.16b, v6.4b[2]\n"
- ".inst 0x4f87e93d // sdot v29.4s, v9.16b, v7.4b[2]\n"
- "ldr q9, [%[b_ptr0], #-0x70]\n"
- ".inst 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n"
- ".inst 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n"
- ".inst 0x4f86e95a // sdot v26.4s, v10.16b, v6.4b[2]\n"
- ".inst 0x4f87e95e // sdot v30.4s, v10.16b, v7.4b[2]\n"
- "ldr q10, [%[b_ptr0], #-0x60]\n"
- ".inst 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n"
- ".inst 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n"
- ".inst 0x4f86e97b // sdot v27.4s, v11.16b, v6.4b[2]\n"
- ".inst 0x4f87e97f // sdot v31.4s, v11.16b, v7.4b[2]\n"
- "ldr q11, [%[b_ptr0], #-0x50]\n"
- ".inst 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n"
- ".inst 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n"
- ".inst 0x4fa6e998 // sdot v24.4s, v12.16b, v6.4b[3]\n"
- ".inst 0x4fa7e99c // sdot v28.4s, v12.16b, v7.4b[3]\n"
- "ldr q12, [%[b_ptr0], #-0x40]\n"
- ".inst 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n"
- ".inst 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n"
- ".inst 0x4fa6e9b9 // sdot v25.4s, v13.16b, v6.4b[3]\n"
- ".inst 0x4fa7e9bd // sdot v29.4s, v13.16b, v7.4b[3]\n"
- "ldr q13, [%[b_ptr0], #-0x30]\n"
- ".inst 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n"
- ".inst 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n"
- ".inst 0x4fa6e9da // sdot v26.4s, v14.16b, v6.4b[3]\n"
- ".inst 0x4fa7e9de // sdot v30.4s, v14.16b, v7.4b[3]\n"
- "ldr q14, [%[b_ptr0], #-0x20]\n"
- ".inst 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n"
- ".inst 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n"
- ".inst 0x4fa6e9fb // sdot v27.4s, v15.16b, v6.4b[3]\n"
- ".inst 0x4fa7e9ff // sdot v31.4s, v15.16b, v7.4b[3]\n"
- "b.ne 3b\n"
- "2:\n"
- "ldr q15, [%[b_ptr0], #-0x10]\n"
- "prfm PSTL1KEEP, [%[c_ptr0]]\n"
- "prfm PSTL1KEEP, [c_ptr1]\n"
- "prfm PSTL1KEEP, [c_ptr2]\n"
- "prfm PSTL1KEEP, [c_ptr3]\n"
- "cbz %[regs], 4f\n"
- ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
- "ldr q4, [%[a_ptr0]]\n"
- ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
- "ldr q5, [a_ptr1]\n"
- ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n"
- "ldr q6, [a_ptr2]\n"
- ".inst 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n"
- "ldr q7, [a_ptr3]\n"
- ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
- "ldr q8, [%[b_ptr0]]\n"
- ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- ".inst 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
- "add a_ptr3, a_ptr3, #0x10\n"
- ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n"
- ".inst 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
- ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
- ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n"
- ".inst 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n"
- ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n"
- ".inst 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n"
- ".inst 0x4fa3e19c // sdot v28.4s, v12.16b, v3.4b[1]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n"
- ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n"
- ".inst 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n"
- ".inst 0x4fa3e1bd // sdot v29.4s, v13.16b, v3.4b[1]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n"
- ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n"
- ".inst 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n"
- ".inst 0x4fa3e1de // sdot v30.4s, v14.16b, v3.4b[1]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n"
- ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n"
- ".inst 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n"
- ".inst 0x4fa3e1ff // sdot v31.4s, v15.16b, v3.4b[1]\n"
- "ldr q15, [%[b_ptr0], #0x70]\n"
- ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n"
- ".inst 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n"
- ".inst 0x4f83e91c // sdot v28.4s, v8.16b, v3.4b[2]\n"
- "ldr q8, [%[b_ptr0], #-0x80]\n"
- ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n"
- ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n"
- ".inst 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n"
- ".inst 0x4f83e93d // sdot v29.4s, v9.16b, v3.4b[2]\n"
- "ldr q9, [%[b_ptr0], #-0x70]\n"
- ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n"
- ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n"
- ".inst 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n"
- ".inst 0x4f83e95e // sdot v30.4s, v10.16b, v3.4b[2]\n"
- "ldr q10, [%[b_ptr0], #-0x60]\n"
- ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n"
- ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n"
- ".inst 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n"
- ".inst 0x4f83e97f // sdot v31.4s, v11.16b, v3.4b[2]\n"
- "ldr q11, [%[b_ptr0], #-0x50]\n"
- ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n"
- ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n"
- ".inst 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n"
- ".inst 0x4fa3e99c // sdot v28.4s, v12.16b, v3.4b[3]\n"
- "ldr q12, [%[b_ptr0], #-0x40]\n"
- ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n"
- ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n"
- ".inst 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n"
- ".inst 0x4fa3e9bd // sdot v29.4s, v13.16b, v3.4b[3]\n"
- "ldr q13, [%[b_ptr0], #-0x30]\n"
- ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n"
- ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n"
- ".inst 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n"
- ".inst 0x4fa3e9de // sdot v30.4s, v14.16b, v3.4b[3]\n"
- "ldr q14, [%[b_ptr0], #-0x20]\n"
- ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n"
- ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n"
- ".inst 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n"
- ".inst 0x4fa3e9ff // sdot v31.4s, v15.16b, v3.4b[3]\n"
- "ldr q15, [%[b_ptr0], #-0x10]\n"
- ".inst 0x4f84e110 // sdot v16.4s, v8.16b, v4.4b[0]\n"
- ".inst 0x4f85e114 // sdot v20.4s, v8.16b, v5.4b[0]\n"
- ".inst 0x4f86e118 // sdot v24.4s, v8.16b, v6.4b[0]\n"
- ".inst 0x4f87e11c // sdot v28.4s, v8.16b, v7.4b[0]\n"
- "ldr q8, [%[b_ptr0]]\n"
- ".inst 0x4f84e131 // sdot v17.4s, v9.16b, v4.4b[0]\n"
- ".inst 0x4f85e135 // sdot v21.4s, v9.16b, v5.4b[0]\n"
- ".inst 0x4f86e139 // sdot v25.4s, v9.16b, v6.4b[0]\n"
- ".inst 0x4f87e13d // sdot v29.4s, v9.16b, v7.4b[0]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- ".inst 0x4f84e152 // sdot v18.4s, v10.16b, v4.4b[0]\n"
- ".inst 0x4f85e156 // sdot v22.4s, v10.16b, v5.4b[0]\n"
- ".inst 0x4f86e15a // sdot v26.4s, v10.16b, v6.4b[0]\n"
- ".inst 0x4f87e15e // sdot v30.4s, v10.16b, v7.4b[0]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- ".inst 0x4f84e173 // sdot v19.4s, v11.16b, v4.4b[0]\n"
- ".inst 0x4f85e177 // sdot v23.4s, v11.16b, v5.4b[0]\n"
- ".inst 0x4f86e17b // sdot v27.4s, v11.16b, v6.4b[0]\n"
- ".inst 0x4f87e17f // sdot v31.4s, v11.16b, v7.4b[0]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- ".inst 0x4fa4e190 // sdot v16.4s, v12.16b, v4.4b[1]\n"
- ".inst 0x4fa5e194 // sdot v20.4s, v12.16b, v5.4b[1]\n"
- ".inst 0x4fa6e198 // sdot v24.4s, v12.16b, v6.4b[1]\n"
- ".inst 0x4fa7e19c // sdot v28.4s, v12.16b, v7.4b[1]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- ".inst 0x4fa4e1b1 // sdot v17.4s, v13.16b, v4.4b[1]\n"
- ".inst 0x4fa5e1b5 // sdot v21.4s, v13.16b, v5.4b[1]\n"
- ".inst 0x4fa6e1b9 // sdot v25.4s, v13.16b, v6.4b[1]\n"
- ".inst 0x4fa7e1bd // sdot v29.4s, v13.16b, v7.4b[1]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- ".inst 0x4fa4e1d2 // sdot v18.4s, v14.16b, v4.4b[1]\n"
- ".inst 0x4fa5e1d6 // sdot v22.4s, v14.16b, v5.4b[1]\n"
- ".inst 0x4fa6e1da // sdot v26.4s, v14.16b, v6.4b[1]\n"
- ".inst 0x4fa7e1de // sdot v30.4s, v14.16b, v7.4b[1]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- ".inst 0x4fa4e1f3 // sdot v19.4s, v15.16b, v4.4b[1]\n"
- ".inst 0x4fa5e1f7 // sdot v23.4s, v15.16b, v5.4b[1]\n"
- ".inst 0x4fa6e1fb // sdot v27.4s, v15.16b, v6.4b[1]\n"
- ".inst 0x4fa7e1ff // sdot v31.4s, v15.16b, v7.4b[1]\n"
- "ldr q15, [%[b_ptr0], #0x70]\n"
- ".inst 0x4f84e910 // sdot v16.4s, v8.16b, v4.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- ".inst 0x4f85e914 // sdot v20.4s, v8.16b, v5.4b[2]\n"
- ".inst 0x4f86e918 // sdot v24.4s, v8.16b, v6.4b[2]\n"
- ".inst 0x4f87e91c // sdot v28.4s, v8.16b, v7.4b[2]\n"
- ".inst 0x4f84e931 // sdot v17.4s, v9.16b, v4.4b[2]\n"
- ".inst 0x4f85e935 // sdot v21.4s, v9.16b, v5.4b[2]\n"
- ".inst 0x4f86e939 // sdot v25.4s, v9.16b, v6.4b[2]\n"
- ".inst 0x4f87e93d // sdot v29.4s, v9.16b, v7.4b[2]\n"
- ".inst 0x4f84e952 // sdot v18.4s, v10.16b, v4.4b[2]\n"
- ".inst 0x4f85e956 // sdot v22.4s, v10.16b, v5.4b[2]\n"
- ".inst 0x4f86e95a // sdot v26.4s, v10.16b, v6.4b[2]\n"
- ".inst 0x4f87e95e // sdot v30.4s, v10.16b, v7.4b[2]\n"
- ".inst 0x4f84e973 // sdot v19.4s, v11.16b, v4.4b[2]\n"
- ".inst 0x4f85e977 // sdot v23.4s, v11.16b, v5.4b[2]\n"
- ".inst 0x4f86e97b // sdot v27.4s, v11.16b, v6.4b[2]\n"
- ".inst 0x4f87e97f // sdot v31.4s, v11.16b, v7.4b[2]\n"
- ".inst 0x4fa4e990 // sdot v16.4s, v12.16b, v4.4b[3]\n"
- ".inst 0x4fa5e994 // sdot v20.4s, v12.16b, v5.4b[3]\n"
- ".inst 0x4fa6e998 // sdot v24.4s, v12.16b, v6.4b[3]\n"
- ".inst 0x4fa7e99c // sdot v28.4s, v12.16b, v7.4b[3]\n"
- ".inst 0x4fa4e9b1 // sdot v17.4s, v13.16b, v4.4b[3]\n"
- ".inst 0x4fa5e9b5 // sdot v21.4s, v13.16b, v5.4b[3]\n"
- ".inst 0x4fa6e9b9 // sdot v25.4s, v13.16b, v6.4b[3]\n"
- ".inst 0x4fa7e9bd // sdot v29.4s, v13.16b, v7.4b[3]\n"
- ".inst 0x4fa4e9d2 // sdot v18.4s, v14.16b, v4.4b[3]\n"
- ".inst 0x4fa5e9d6 // sdot v22.4s, v14.16b, v5.4b[3]\n"
- ".inst 0x4fa6e9da // sdot v26.4s, v14.16b, v6.4b[3]\n"
- ".inst 0x4fa7e9de // sdot v30.4s, v14.16b, v7.4b[3]\n"
- ".inst 0x4fa4e9f3 // sdot v19.4s, v15.16b, v4.4b[3]\n"
- ".inst 0x4fa5e9f7 // sdot v23.4s, v15.16b, v5.4b[3]\n"
- ".inst 0x4fa6e9fb // sdot v27.4s, v15.16b, v6.4b[3]\n"
- ".inst 0x4fa7e9ff // sdot v31.4s, v15.16b, v7.4b[3]\n"
- "b 5f\n"
- "4:\n"
- ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
- ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
- ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n"
- ".inst 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n"
- "ldr q8, [%[b_ptr0]]\n"
- ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
- ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
- ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n"
- ".inst 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
- ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
- ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n"
- ".inst 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
- ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
- ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n"
- ".inst 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- ".inst 0x4fa0e190 // sdot v16.4s, v12.16b, v0.4b[1]\n"
- ".inst 0x4fa1e194 // sdot v20.4s, v12.16b, v1.4b[1]\n"
- ".inst 0x4fa2e198 // sdot v24.4s, v12.16b, v2.4b[1]\n"
- ".inst 0x4fa3e19c // sdot v28.4s, v12.16b, v3.4b[1]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- ".inst 0x4fa0e1b1 // sdot v17.4s, v13.16b, v0.4b[1]\n"
- ".inst 0x4fa1e1b5 // sdot v21.4s, v13.16b, v1.4b[1]\n"
- ".inst 0x4fa2e1b9 // sdot v25.4s, v13.16b, v2.4b[1]\n"
- ".inst 0x4fa3e1bd // sdot v29.4s, v13.16b, v3.4b[1]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- ".inst 0x4fa0e1d2 // sdot v18.4s, v14.16b, v0.4b[1]\n"
- ".inst 0x4fa1e1d6 // sdot v22.4s, v14.16b, v1.4b[1]\n"
- ".inst 0x4fa2e1da // sdot v26.4s, v14.16b, v2.4b[1]\n"
- ".inst 0x4fa3e1de // sdot v30.4s, v14.16b, v3.4b[1]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- ".inst 0x4fa0e1f3 // sdot v19.4s, v15.16b, v0.4b[1]\n"
- ".inst 0x4fa1e1f7 // sdot v23.4s, v15.16b, v1.4b[1]\n"
- ".inst 0x4fa2e1fb // sdot v27.4s, v15.16b, v2.4b[1]\n"
- ".inst 0x4fa3e1ff // sdot v31.4s, v15.16b, v3.4b[1]\n"
- "ldr q15, [%[b_ptr0], #0x70]\n"
- ".inst 0x4f80e910 // sdot v16.4s, v8.16b, v0.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- ".inst 0x4f81e914 // sdot v20.4s, v8.16b, v1.4b[2]\n"
- ".inst 0x4f82e918 // sdot v24.4s, v8.16b, v2.4b[2]\n"
- ".inst 0x4f83e91c // sdot v28.4s, v8.16b, v3.4b[2]\n"
- ".inst 0x4f80e931 // sdot v17.4s, v9.16b, v0.4b[2]\n"
- ".inst 0x4f81e935 // sdot v21.4s, v9.16b, v1.4b[2]\n"
- ".inst 0x4f82e939 // sdot v25.4s, v9.16b, v2.4b[2]\n"
- ".inst 0x4f83e93d // sdot v29.4s, v9.16b, v3.4b[2]\n"
- ".inst 0x4f80e952 // sdot v18.4s, v10.16b, v0.4b[2]\n"
- ".inst 0x4f81e956 // sdot v22.4s, v10.16b, v1.4b[2]\n"
- ".inst 0x4f82e95a // sdot v26.4s, v10.16b, v2.4b[2]\n"
- ".inst 0x4f83e95e // sdot v30.4s, v10.16b, v3.4b[2]\n"
- ".inst 0x4f80e973 // sdot v19.4s, v11.16b, v0.4b[2]\n"
- ".inst 0x4f81e977 // sdot v23.4s, v11.16b, v1.4b[2]\n"
- ".inst 0x4f82e97b // sdot v27.4s, v11.16b, v2.4b[2]\n"
- ".inst 0x4f83e97f // sdot v31.4s, v11.16b, v3.4b[2]\n"
- ".inst 0x4fa0e990 // sdot v16.4s, v12.16b, v0.4b[3]\n"
- ".inst 0x4fa1e994 // sdot v20.4s, v12.16b, v1.4b[3]\n"
- ".inst 0x4fa2e998 // sdot v24.4s, v12.16b, v2.4b[3]\n"
- ".inst 0x4fa3e99c // sdot v28.4s, v12.16b, v3.4b[3]\n"
- ".inst 0x4fa0e9b1 // sdot v17.4s, v13.16b, v0.4b[3]\n"
- ".inst 0x4fa1e9b5 // sdot v21.4s, v13.16b, v1.4b[3]\n"
- ".inst 0x4fa2e9b9 // sdot v25.4s, v13.16b, v2.4b[3]\n"
- ".inst 0x4fa3e9bd // sdot v29.4s, v13.16b, v3.4b[3]\n"
- ".inst 0x4fa0e9d2 // sdot v18.4s, v14.16b, v0.4b[3]\n"
- ".inst 0x4fa1e9d6 // sdot v22.4s, v14.16b, v1.4b[3]\n"
- ".inst 0x4fa2e9da // sdot v26.4s, v14.16b, v2.4b[3]\n"
- ".inst 0x4fa3e9de // sdot v30.4s, v14.16b, v3.4b[3]\n"
- ".inst 0x4fa0e9f3 // sdot v19.4s, v15.16b, v0.4b[3]\n"
- ".inst 0x4fa1e9f7 // sdot v23.4s, v15.16b, v1.4b[3]\n"
- ".inst 0x4fa2e9fb // sdot v27.4s, v15.16b, v2.4b[3]\n"
- ".inst 0x4fa3e9ff // sdot v31.4s, v15.16b, v3.4b[3]\n"
- "5:\n"
- "cbz %[blocks], 6f\n"
- "7:\n"
- "ldr q8, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "ldr s0, [%[a_ptr0]]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "add %[a_ptr0], %[a_ptr0], #0x4\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "add %[b_ptr0], %[b_ptr0], #0x40\n"
- ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
- "ldr s1, [a_ptr1]\n"
- ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
- "add a_ptr1, a_ptr1, #0x4\n"
- ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
- "ldr s2, [a_ptr2]\n"
- ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
- "add a_ptr2, a_ptr2, #0x4\n"
- ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
- "ldr s3, [a_ptr3]\n"
- ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n"
- "add a_ptr3, a_ptr3, #0x4\n"
- ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n"
- ".inst 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n"
- ".inst 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n"
- ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
- ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n"
- ".inst 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n"
- ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
- ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
- ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n"
- ".inst 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n"
- "b.ne 7b\n"
- "6:\n"
- "cbz %[odds], 8f\n"
- "ld1 {v0.b}[0], [%[a_ptr0]], #1\n"
- "ld1 {v1.b}[0], [a_ptr1], #1\n"
- "ld1 {v2.b}[0], [a_ptr2], #1\n"
- "ld1 {v3.b}[0], [a_ptr3], #1\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 9f\n"
- "ld1 {v0.b}[1], [%[a_ptr0]], #1\n"
- "ld1 {v1.b}[1], [a_ptr1], #1\n"
- "ld1 {v2.b}[1], [a_ptr2], #1\n"
- "ld1 {v3.b}[1], [a_ptr3], #1\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 9f\n"
- "ld1 {v0.b}[2], [%[a_ptr0]]\n"
- "ld1 {v1.b}[2], [a_ptr1]\n"
- "ld1 {v2.b}[2], [a_ptr2]\n"
- "ld1 {v3.b}[2], [a_ptr3]\n"
- "9:\n"
- "ldr q8, [%[b_ptr0]]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- ".inst 0x4f80e110 // sdot v16.4s, v8.16b, v0.4b[0]\n"
- ".inst 0x4f81e114 // sdot v20.4s, v8.16b, v1.4b[0]\n"
- ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n"
- ".inst 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n"
- ".inst 0x4f80e131 // sdot v17.4s, v9.16b, v0.4b[0]\n"
- ".inst 0x4f81e135 // sdot v21.4s, v9.16b, v1.4b[0]\n"
- ".inst 0x4f82e139 // sdot v25.4s, v9.16b, v2.4b[0]\n"
- ".inst 0x4f83e13d // sdot v29.4s, v9.16b, v3.4b[0]\n"
- ".inst 0x4f80e152 // sdot v18.4s, v10.16b, v0.4b[0]\n"
- ".inst 0x4f81e156 // sdot v22.4s, v10.16b, v1.4b[0]\n"
- ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n"
- ".inst 0x4f83e15e // sdot v30.4s, v10.16b, v3.4b[0]\n"
- ".inst 0x4f80e173 // sdot v19.4s, v11.16b, v0.4b[0]\n"
- ".inst 0x4f81e177 // sdot v23.4s, v11.16b, v1.4b[0]\n"
- ".inst 0x4f82e17b // sdot v27.4s, v11.16b, v2.4b[0]\n"
- ".inst 0x4f83e17f // sdot v31.4s, v11.16b, v3.4b[0]\n"
- "8:\n"
- "str q16, [%[c_ptr0]]\n"
- "str q17, [%[c_ptr0], #0x10]\n"
- "str q18, [%[c_ptr0], #0x20]\n"
- "str q19, [%[c_ptr0], #0x30]\n"
- "add %[c_ptr0], %[c_ptr0], #0x40\n"
- "str q20, [c_ptr1]\n"
- "str q21, [c_ptr1, #0x10]\n"
- "str q22, [c_ptr1, #0x20]\n"
- "str q23, [c_ptr1, #0x30]\n"
- "str q24, [c_ptr2]\n"
- "str q25, [c_ptr2, #0x10]\n"
- "str q26, [c_ptr2, #0x20]\n"
- "str q27, [c_ptr2, #0x30]\n"
- "str q28, [c_ptr3]\n"
- "str q29, [c_ptr3, #0x10]\n"
- "str q30, [c_ptr3, #0x20]\n"
- "str q31, [c_ptr3, #0x30]\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq a_ptr3\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- ".unreq c_ptr3\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks), [odds] "+r" (odds)
- : [width] "r" (width), [accumulate] "r" (static_cast<uint64_t>(accumulate)), [lda] "r" (ldab), [ldc] "r" (ldcb)
- : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "x4", "x5", "cc", "memory"
- );
- break;
- }
- if (use_result_buffer) {
- for(int cy=0; cy<std::min(M-y, 4); cy++) {
- for(unsigned int cx=0; cx<width; cx++) {
- c_ptr_real[cy * ldc + cx] = result_buffer[cy * 16 + cx];
- }
- }
- }
- }
- }
-}
-
-} // namespace arm_gemm
-
-#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_4x6.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_6x16.hpp
similarity index 62%
copy from src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_4x6.hpp
copy to src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_6x16.hpp
index 6f31efe..16a6f92 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_4x6.hpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_6x16.hpp
@@ -10,34 +10,43 @@
* sell copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
*/
#pragma once
-
#ifdef __aarch64__
+#include "../std_transforms_fixed.hpp"
+
+#define ARGLIST \
+ unsigned int, const unsigned int *, \
+ IndirectInputArg<int8_t>, \
+ size_t, size_t, \
+ const int8_t *, \
+ IndirectOutputArg<int32_t>, \
+ const int32_t *, Activation, bool
+
namespace arm_gemm
{
// Actual kernel implementations
-void a64_smallK_hybrid_fp32_mla_4x6(const float *, int, const float *, float *, int, int, int, int, const float *, Activation, bool);
+void a64_hybrid_s8s32_dot_6x16( ARGLIST );
-class smallK_hybrid_fp32_mla_4x6
+class cls_a64_hybrid_s8s32_dot_6x16
{
public:
- typedef float operand_type;
- typedef float result_type;
+ typedef int8_t operand_type;
+ typedef int32_t result_type;
- typedef void (*kern_type)(const float *, int, const float *, float *, int, int, int, int, const float *, Activation, bool);
+ typedef void (*kern_type)( ARGLIST );
/* Kernel blocking parameters */
static constexpr unsigned int out_height()
@@ -47,40 +56,30 @@
static unsigned int out_width()
{
- return 4;
+ return 16;
}
static constexpr unsigned int k_unroll()
{
- return 1;
+ return 4;
}
static constexpr bool supports_accumulate()
{
- return false;
- }
-
- static constexpr bool supports_bias()
- {
return true;
}
- static constexpr bool supports_activation()
- {
- return true;
- }
-
- StdTransformsFixed<operand_type, result_type, 6, 4, 1> transforms = {};
+ StdTransformsFixed<operand_type, result_type, 6, 16, 4> transforms = {};
// Default to the generic kernel
- kern_type kernel=a64_smallK_hybrid_fp32_mla_4x6;
+ kern_type kernel=a64_hybrid_s8s32_dot_6x16;
- smallK_hybrid_fp32_mla_4x6(const CPUInfo *)
+ cls_a64_hybrid_s8s32_dot_6x16(const CPUInfo *)
{
-
}
};
} // namespace arm_gemm
+#undef ARGLIST
#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_6x16/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_6x16/generic.cpp
new file mode 100644
index 0000000..3257986
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_6x16/generic.cpp
@@ -0,0 +1,3335 @@
+/*
+ * Copyright (c) 2019-2020 Arm Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+#ifdef __aarch64__
+
+#include "arm_gemm.hpp"
+#include "../../utils.hpp"
+
+#include <cassert>
+
+namespace arm_gemm {
+
+void a64_hybrid_s8s32_dot_6x16 (
+ unsigned int num_strings, const unsigned int *string_lengths, IndirectInputArg<int8_t> A_arg,
+ size_t M, size_t N, const int8_t *B_ptr, IndirectOutputArg<int32_t> output_arg,
+ const int32_t *, Activation, bool accumulate
+)
+{
+ struct KernelArgs {
+ unsigned int num_strings = {};
+ const unsigned int *string_lengths = {};
+ size_t N = {};
+ const int8_t *B_ptr = {};
+ size_t output_offset = {};
+ size_t input_initial_col = {};
+ size_t input_offset = {};
+ } ka;
+
+ unsigned long flags=0;
+ void *output_ptr;
+ void *input_ptr;
+
+ if (output_arg.is_indirect) {
+ output_ptr=(void *)(output_arg.indirect.ptr);
+ ka.output_offset=output_arg.indirect.offset;
+ flags |= 0x4;
+ } else {
+ output_ptr=(void *)(output_arg.direct.base);
+ ka.output_offset=output_arg.direct.stride;
+ }
+
+ if (A_arg.is_indirect) {
+ input_ptr=(void *)(A_arg.indirect.ptr);
+ ka.input_offset=A_arg.indirect.start_row;
+ ka.input_initial_col=A_arg.indirect.start_col;
+ flags |= 0x8;
+ } else {
+ assert(num_strings==1);
+ input_ptr=(void *)(A_arg.direct.base);
+ ka.input_offset=A_arg.direct.stride;
+ }
+ if (accumulate) {
+ flags |= 0x1;
+ }
+ ka.num_strings = num_strings;
+ ka.string_lengths = string_lengths;
+ ka.N = N;
+ ka.B_ptr = B_ptr;
+ __asm__ __volatile__(
+
+ "1:" // Row loop
+ "cmp %x[M], #0x6\n"
+ "bge 176f\n"
+ "cmp %x[M], #0x4\n"
+ "bgt 141f\n"
+ "beq 106f\n"
+ "cmp %x[M], #0x2\n"
+ "bgt 71f\n"
+ "beq 36f\n"
+ "ldr x15, [%x[args_ptr], %[offsetof_N]]\n"
+ "ldr x14, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 2f\n"
+ "ldr x13, [%x[output_ptr], #0x0]\n"
+ "add x13, x13, x19, LSL #2\n"
+ "b 3f\n"
+ "2:" // Height 1: setup direct output
+ "mov x13, %x[output_ptr]\n"
+ "3:" // Height 1: Column loop
+ "tbz %x[flags], #0, 13f\n"
+ "cmp x15, #0x10\n"
+ "bge 12f\n"
+ "tbz x15, #3, 7f\n"
+ "ld1 { v8.4s }, [x13], #0x10\n"
+ "ld1 { v9.4s }, [x13], #0x10\n"
+ "tbz x15, #2, 5f\n"
+ "ld1 { v10.4s }, [x13], #0x10\n"
+ "tbz x15, #1, 4f\n"
+ "mov x19, #0x38\n"
+ "ldr d11, [x13], #0x8\n"
+ "tbz x15, #0, 11f\n"
+ "ld1 { v11.s }[2], [x13]\n"
+ "b 11f\n"
+ "4:" // Height 1: Partial accumulate: partial_1_12
+ "mov x19, #0x30\n"
+ "tbz x15, #0, 11f\n"
+ "ldr s11, [x13, #0x0]\n"
+ "b 11f\n"
+ "5:" // Height 1: Partial accumulate: partial_2_8
+ "tbz x15, #1, 6f\n"
+ "ldr d10, [x13], #0x8\n"
+ "mov x19, #0x28\n"
+ "tbz x15, #0, 11f\n"
+ "ld1 { v10.s }[2], [x13]\n"
+ "b 11f\n"
+ "6:" // Height 1: Partial accumulate: partial_1_8
+ "mov x19, #0x20\n"
+ "tbz x15, #0, 11f\n"
+ "ldr s10, [x13, #0x0]\n"
+ "b 11f\n"
+ "7:" // Height 1: Partial accumulate: partial_4_0
+ "tbz x15, #2, 9f\n"
+ "ld1 { v8.4s }, [x13], #0x10\n"
+ "tbz x15, #1, 8f\n"
+ "mov x19, #0x18\n"
+ "ldr d9, [x13], #0x8\n"
+ "tbz x15, #0, 11f\n"
+ "ld1 { v9.s }[2], [x13]\n"
+ "b 11f\n"
+ "8:" // Height 1: Partial accumulate: partial_1_4
+ "mov x19, #0x10\n"
+ "tbz x15, #0, 11f\n"
+ "ldr s9, [x13, #0x0]\n"
+ "b 11f\n"
+ "9:" // Height 1: Partial accumulate: partial_2_0
+ "tbz x15, #1, 10f\n"
+ "ldr d8, [x13], #0x8\n"
+ "mov x19, #0x8\n"
+ "tbz x15, #0, 11f\n"
+ "ld1 { v8.s }[2], [x13]\n"
+ "b 11f\n"
+ "10:" // Height 1: Partial accumulate: partial_1_0
+ "mov x19, #0x0\n"
+ "ldr s8, [x13, #0x0]\n"
+ "11:" // Height 1: Partial accumulate: Done
+ "sub x13, x13, x19\n"
+ "b 14f\n"
+ "12:" // Height 1: full accumulate
+ "ldr q8, [x13, #0x0]\n"
+ "ldr q9, [x13, #0x10]\n"
+ "ldr q10, [x13, #0x20]\n"
+ "ldr q11, [x13, #0x30]\n"
+ "b 14f\n"
+ "13:" // Height 1: no accumulate
+ "movi v8.4s, #0x0\n"
+ "movi v9.4s, #0x0\n"
+ "movi v10.4s, #0x0\n"
+ "movi v11.4s, #0x0\n"
+ "14:" // Height 1: setup done
+ "mov x12, #0x0\n"
+ "15:" // Height 1: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w11, [x20, x12, LSL #0x2]\n"
+ "tbz %x[flags], #3, 16f\n"
+ "ldr x20, [%x[input_ptr], x12, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x10, [x20, #0x0]\n"
+ "cbnz x12, 17f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x10, x10, x19\n"
+ "b 17f\n"
+ "16:" // Height 1: setup direct input
+ "mov x10, %x[input_ptr]\n"
+ "17:" // Height 1: input setup done
+ "cmp x11, #0x10\n"
+ "blt 20f\n"
+ "cmp x11, #0x20\n"
+ "blt 19f\n"
+ "18:" // Height 1: Multiply loop: Main loop head
+ "ldr q0, [x10, #0x0]\n"
+ "ldr q6, [x14, #0x0]\n"
+ ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n"
+ "ldr q7, [x14, #0x10]\n"
+ "ldr q6, [x14, #0x20]\n"
+ ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n"
+ "ldr q7, [x14, #0x30]\n"
+ ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n"
+ "ldr q6, [x14, #0x40]\n"
+ "add x10, x10, #0x10\n"
+ ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "ldr q7, [x14, #0x50]\n"
+ ".inst 0x4fa0e0c8 // sdot v8.4s, v6.16b, v0.4b[1]\n"
+ "ldr q6, [x14, #0x60]\n"
+ "sub x11, x11, #0x10\n"
+ ".inst 0x4fa0e0e9 // sdot v9.4s, v7.16b, v0.4b[1]\n"
+ "ldr q7, [x14, #0x70]\n"
+ "cmp x11, #0x20\n"
+ ".inst 0x4fa0e0ca // sdot v10.4s, v6.16b, v0.4b[1]\n"
+ "ldr q6, [x14, #0x80]\n"
+ ".inst 0x4fa0e0eb // sdot v11.4s, v7.16b, v0.4b[1]\n"
+ "ldr q7, [x14, #0x90]\n"
+ ".inst 0x4f80e8c8 // sdot v8.4s, v6.16b, v0.4b[2]\n"
+ "ldr q6, [x14, #0xa0]\n"
+ ".inst 0x4f80e8e9 // sdot v9.4s, v7.16b, v0.4b[2]\n"
+ "ldr q7, [x14, #0xb0]\n"
+ ".inst 0x4f80e8ca // sdot v10.4s, v6.16b, v0.4b[2]\n"
+ "ldr q6, [x14, #0xc0]\n"
+ ".inst 0x4f80e8eb // sdot v11.4s, v7.16b, v0.4b[2]\n"
+ "ldr q7, [x14, #0xd0]\n"
+ ".inst 0x4fa0e8c8 // sdot v8.4s, v6.16b, v0.4b[3]\n"
+ "ldr q6, [x14, #0xe0]\n"
+ ".inst 0x4fa0e8e9 // sdot v9.4s, v7.16b, v0.4b[3]\n"
+ "ldr q7, [x14, #0xf0]\n"
+ "add x14, x14, #0x100\n"
+ ".inst 0x4fa0e8ca // sdot v10.4s, v6.16b, v0.4b[3]\n"
+ ".inst 0x4fa0e8eb // sdot v11.4s, v7.16b, v0.4b[3]\n"
+ "bge 18b\n"
+ "19:" // Height 1: Multiply loop: Single iteration only
+ "sub x11, x11, #0x10\n"
+ "ldr q0, [x10, #0x0]\n"
+ "ldr q6, [x14, #0x0]\n"
+ ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n"
+ "ldr q7, [x14, #0x10]\n"
+ "ldr q6, [x14, #0x20]\n"
+ ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n"
+ "ldr q7, [x14, #0x30]\n"
+ ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n"
+ "ldr q6, [x14, #0x40]\n"
+ "add x10, x10, #0x10\n"
+ ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "ldr q7, [x14, #0x50]\n"
+ ".inst 0x4fa0e0c8 // sdot v8.4s, v6.16b, v0.4b[1]\n"
+ "ldr q6, [x14, #0x60]\n"
+ ".inst 0x4fa0e0e9 // sdot v9.4s, v7.16b, v0.4b[1]\n"
+ "ldr q7, [x14, #0x70]\n"
+ ".inst 0x4fa0e0ca // sdot v10.4s, v6.16b, v0.4b[1]\n"
+ "ldr q6, [x14, #0x80]\n"
+ ".inst 0x4fa0e0eb // sdot v11.4s, v7.16b, v0.4b[1]\n"
+ "ldr q7, [x14, #0x90]\n"
+ ".inst 0x4f80e8c8 // sdot v8.4s, v6.16b, v0.4b[2]\n"
+ "ldr q6, [x14, #0xa0]\n"
+ ".inst 0x4f80e8e9 // sdot v9.4s, v7.16b, v0.4b[2]\n"
+ "ldr q7, [x14, #0xb0]\n"
+ ".inst 0x4f80e8ca // sdot v10.4s, v6.16b, v0.4b[2]\n"
+ "ldr q6, [x14, #0xc0]\n"
+ ".inst 0x4f80e8eb // sdot v11.4s, v7.16b, v0.4b[2]\n"
+ "ldr q7, [x14, #0xd0]\n"
+ ".inst 0x4fa0e8c8 // sdot v8.4s, v6.16b, v0.4b[3]\n"
+ "ldr q6, [x14, #0xe0]\n"
+ ".inst 0x4fa0e8e9 // sdot v9.4s, v7.16b, v0.4b[3]\n"
+ "ldr q7, [x14, #0xf0]\n"
+ "add x14, x14, #0x100\n"
+ ".inst 0x4fa0e8ca // sdot v10.4s, v6.16b, v0.4b[3]\n"
+ ".inst 0x4fa0e8eb // sdot v11.4s, v7.16b, v0.4b[3]\n"
+ "20:" // Height 1: Multiply loop: Main loop skip
+ "cbz x11, 25f\n"
+ "cmp x11, #0x4\n"
+ "blt 22f\n"
+ "21:" // Height 1: Multiply loop: Odd block loop
+ "ldr s0, [x10], #0x4\n"
+ "ldr q6, [x14, #0x0]\n"
+ ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n"
+ "ldr q7, [x14, #0x10]\n"
+ "ldr q6, [x14, #0x20]\n"
+ ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n"
+ "ldr q7, [x14, #0x30]\n"
+ ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n"
+ "sub x11, x11, #0x4\n"
+ "add x14, x14, #0x40\n"
+ ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n"
+ "cmp x11, #0x4\n"
+ "bge 21b\n"
+ "cbz x11, 25f\n"
+ "22:" // Height 1: Multiply loop: Skip odd blocks
+ "tbz x11, #1, 23f\n"
+ "ldr h0, [x10], #0x2\n"
+ "tbz x11, #0, 24f\n"
+ "ld1 { v0.b }[2], [x10]\n"
+ "b 24f\n"
+ "23:" // Height 1: Multiply loop: Ragged operand read: partial_1_0
+ "ldr b0, [x10, #0x0]\n"
+ "24:" // Height 1: Multiply loop: Ragged operand read: Done
+ "ldr q6, [x14, #0x0]\n"
+ ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n"
+ "ldr q7, [x14, #0x10]\n"
+ "ldr q6, [x14, #0x20]\n"
+ ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n"
+ "ldr q7, [x14, #0x30]\n"
+ ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n"
+ "add x14, x14, #0x40\n"
+ ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n"
+ "25:" // Height 1: Multiply loop: No odd multiplies
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "add x12, x12, #0x1\n"
+ "cmp x12, x19\n"
+ "bne 15b\n"
+ "prfm pstl1keep, [x13, #0x0]\n"
+ "cmp x15, #0x10\n"
+ "bge 34f\n"
+ "tbz x15, #3, 29f\n"
+ "st1 { v8.4s }, [x13], #0x10\n"
+ "st1 { v9.4s }, [x13], #0x10\n"
+ "tbz x15, #2, 27f\n"
+ "st1 { v10.4s }, [x13], #0x10\n"
+ "tbz x15, #1, 26f\n"
+ "str d11, [x13], #0x8\n"
+ "tbz x15, #0, 33f\n"
+ "st1 { v11.s }[2], [x13]\n"
+ "b 33f\n"
+ "26:" // Height 1: Partial direct writeback: partial_1_12
+ "tbz x15, #0, 33f\n"
+ "str s11, [x13, #0x0]\n"
+ "b 33f\n"
+ "27:" // Height 1: Partial direct writeback: partial_2_8
+ "tbz x15, #1, 28f\n"
+ "str d10, [x13], #0x8\n"
+ "tbz x15, #0, 33f\n"
+ "st1 { v10.s }[2], [x13]\n"
+ "b 33f\n"
+ "28:" // Height 1: Partial direct writeback: partial_1_8
+ "tbz x15, #0, 33f\n"
+ "str s10, [x13, #0x0]\n"
+ "b 33f\n"
+ "29:" // Height 1: Partial direct writeback: partial_4_0
+ "tbz x15, #2, 31f\n"
+ "st1 { v8.4s }, [x13], #0x10\n"
+ "tbz x15, #1, 30f\n"
+ "str d9, [x13], #0x8\n"
+ "tbz x15, #0, 33f\n"
+ "st1 { v9.s }[2], [x13]\n"
+ "b 33f\n"
+ "30:" // Height 1: Partial direct writeback: partial_1_4
+ "tbz x15, #0, 33f\n"
+ "str s9, [x13, #0x0]\n"
+ "b 33f\n"
+ "31:" // Height 1: Partial direct writeback: partial_2_0
+ "tbz x15, #1, 32f\n"
+ "str d8, [x13], #0x8\n"
+ "tbz x15, #0, 33f\n"
+ "st1 { v8.s }[2], [x13]\n"
+ "b 33f\n"
+ "32:" // Height 1: Partial direct writeback: partial_1_0
+ "str s8, [x13, #0x0]\n"
+ "33:" // Height 1: Partial direct writeback: Done
+ "b 35f\n"
+ "34:" // Height 1: Full writeback
+ "str q8, [x13, #0x0]\n"
+ "str q9, [x13, #0x10]\n"
+ "str q10, [x13, #0x20]\n"
+ "str q11, [x13, #0x30]\n"
+ "add x13, x13, #0x40\n"
+ "35:" // Height 1: Writeback done
+ "subs x15, x15, #0x10\n"
+ "bgt 3b\n"
+ "b 212f\n"
+ "36:" // Height 2
+ "ldr x15, [%x[args_ptr], %[offsetof_N]]\n"
+ "ldr x14, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 37f\n"
+ "ldr x13, [%x[output_ptr], #0x0]\n"
+ "add x13, x13, x19, LSL #2\n"
+ "ldr x9, [%x[output_ptr], #0x8]\n"
+ "add x9, x9, x19, LSL #2\n"
+ "b 38f\n"
+ "37:" // Height 2: setup direct output
+ "mov x13, %x[output_ptr]\n"
+ "add x9, x13, x19, LSL #2\n"
+ "38:" // Height 2: Column loop
+ "tbz %x[flags], #0, 48f\n"
+ "cmp x15, #0x10\n"
+ "bge 47f\n"
+ "tbz x15, #3, 42f\n"
+ "ld1 { v8.4s }, [x13], #0x10\n"
+ "ld1 { v12.4s }, [x9], #0x10\n"
+ "ld1 { v9.4s }, [x13], #0x10\n"
+ "ld1 { v13.4s }, [x9], #0x10\n"
+ "tbz x15, #2, 40f\n"
+ "ld1 { v10.4s }, [x13], #0x10\n"
+ "ld1 { v14.4s }, [x9], #0x10\n"
+ "tbz x15, #1, 39f\n"
+ "mov x19, #0x38\n"
+ "ldr d11, [x13], #0x8\n"
+ "ldr d15, [x9], #0x8\n"
+ "tbz x15, #0, 46f\n"
+ "ld1 { v11.s }[2], [x13]\n"
+ "ld1 { v15.s }[2], [x9]\n"
+ "b 46f\n"
+ "39:" // Height 2: Partial accumulate: partial_1_12
+ "mov x19, #0x30\n"
+ "tbz x15, #0, 46f\n"
+ "ldr s11, [x13, #0x0]\n"
+ "ldr s15, [x9, #0x0]\n"
+ "b 46f\n"
+ "40:" // Height 2: Partial accumulate: partial_2_8
+ "tbz x15, #1, 41f\n"
+ "ldr d10, [x13], #0x8\n"
+ "ldr d14, [x9], #0x8\n"
+ "mov x19, #0x28\n"
+ "tbz x15, #0, 46f\n"
+ "ld1 { v10.s }[2], [x13]\n"
+ "ld1 { v14.s }[2], [x9]\n"
+ "b 46f\n"
+ "41:" // Height 2: Partial accumulate: partial_1_8
+ "mov x19, #0x20\n"
+ "tbz x15, #0, 46f\n"
+ "ldr s10, [x13, #0x0]\n"
+ "ldr s14, [x9, #0x0]\n"
+ "b 46f\n"
+ "42:" // Height 2: Partial accumulate: partial_4_0
+ "tbz x15, #2, 44f\n"
+ "ld1 { v8.4s }, [x13], #0x10\n"
+ "ld1 { v12.4s }, [x9], #0x10\n"
+ "tbz x15, #1, 43f\n"
+ "mov x19, #0x18\n"
+ "ldr d9, [x13], #0x8\n"
+ "ldr d13, [x9], #0x8\n"
+ "tbz x15, #0, 46f\n"
+ "ld1 { v9.s }[2], [x13]\n"
+ "ld1 { v13.s }[2], [x9]\n"
+ "b 46f\n"
+ "43:" // Height 2: Partial accumulate: partial_1_4
+ "mov x19, #0x10\n"
+ "tbz x15, #0, 46f\n"
+ "ldr s9, [x13, #0x0]\n"
+ "ldr s13, [x9, #0x0]\n"
+ "b 46f\n"
+ "44:" // Height 2: Partial accumulate: partial_2_0
+ "tbz x15, #1, 45f\n"
+ "ldr d8, [x13], #0x8\n"
+ "ldr d12, [x9], #0x8\n"
+ "mov x19, #0x8\n"
+ "tbz x15, #0, 46f\n"
+ "ld1 { v8.s }[2], [x13]\n"
+ "ld1 { v12.s }[2], [x9]\n"
+ "b 46f\n"
+ "45:" // Height 2: Partial accumulate: partial_1_0
+ "mov x19, #0x0\n"
+ "ldr s8, [x13, #0x0]\n"
+ "ldr s12, [x9, #0x0]\n"
+ "46:" // Height 2: Partial accumulate: Done
+ "sub x13, x13, x19\n"
+ "sub x9, x9, x19\n"
+ "b 49f\n"
+ "47:" // Height 2: full accumulate
+ "ldr q8, [x13, #0x0]\n"
+ "ldr q9, [x13, #0x10]\n"
+ "ldr q10, [x13, #0x20]\n"
+ "ldr q11, [x13, #0x30]\n"
+ "ldr q12, [x9, #0x0]\n"
+ "ldr q13, [x9, #0x10]\n"
+ "ldr q14, [x9, #0x20]\n"
+ "ldr q15, [x9, #0x30]\n"
+ "b 49f\n"
+ "48:" // Height 2: no accumulate
+ "movi v8.4s, #0x0\n"
+ "movi v9.4s, #0x0\n"
+ "movi v10.4s, #0x0\n"
+ "movi v11.4s, #0x0\n"
+ "movi v12.4s, #0x0\n"
+ "movi v13.4s, #0x0\n"
+ "movi v14.4s, #0x0\n"
+ "movi v15.4s, #0x0\n"
+ "49:" // Height 2: setup done
+ "mov x12, #0x0\n"
+ "50:" // Height 2: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w11, [x20, x12, LSL #0x2]\n"
+ "tbz %x[flags], #3, 51f\n"
+ "ldr x20, [%x[input_ptr], x12, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x10, [x20, #0x0]\n"
+ "ldr x28, [x20, #0x8]\n"
+ "cbnz x12, 52f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x10, x10, x19\n"
+ "add x28, x28, x19\n"
+ "b 52f\n"
+ "51:" // Height 2: setup direct input
+ "mov x10, %x[input_ptr]\n"
+ "add x28, x10, x19\n"
+ "52:" // Height 2: input setup done
+ "cmp x11, #0x10\n"
+ "blt 55f\n"
+ "cmp x11, #0x20\n"
+ "blt 54f\n"
+ "53:" // Height 2: Multiply loop: Main loop head
+ "ldr q0, [x10, #0x0]\n"
+ "ldr q1, [x28, #0x0]\n"
+ "ldr q6, [x14, #0x0]\n"
+ ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n"
+ "ldr q7, [x14, #0x10]\n"
+ ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n"
+ "ldr q6, [x14, #0x20]\n"
+ "add x10, x10, #0x10\n"
+ ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x28, x28, #0x10\n"
+ ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "ldr q7, [x14, #0x30]\n"
+ ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n"
+ "sub x11, x11, #0x10\n"
+ ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n"
+ "ldr q6, [x14, #0x40]\n"
+ "cmp x11, #0x20\n"
+ ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n"
+ "ldr q7, [x14, #0x50]\n"
+ ".inst 0x4fa0e0c8 // sdot v8.4s, v6.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e0cc // sdot v12.4s, v6.16b, v1.4b[1]\n"
+ "ldr q6, [x14, #0x60]\n"
+ ".inst 0x4fa0e0e9 // sdot v9.4s, v7.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e0ed // sdot v13.4s, v7.16b, v1.4b[1]\n"
+ "ldr q7, [x14, #0x70]\n"
+ ".inst 0x4fa0e0ca // sdot v10.4s, v6.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e0ce // sdot v14.4s, v6.16b, v1.4b[1]\n"
+ "ldr q6, [x14, #0x80]\n"
+ ".inst 0x4fa0e0eb // sdot v11.4s, v7.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e0ef // sdot v15.4s, v7.16b, v1.4b[1]\n"
+ "ldr q7, [x14, #0x90]\n"
+ ".inst 0x4f80e8c8 // sdot v8.4s, v6.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8cc // sdot v12.4s, v6.16b, v1.4b[2]\n"
+ "ldr q6, [x14, #0xa0]\n"
+ ".inst 0x4f80e8e9 // sdot v9.4s, v7.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8ed // sdot v13.4s, v7.16b, v1.4b[2]\n"
+ "ldr q7, [x14, #0xb0]\n"
+ ".inst 0x4f80e8ca // sdot v10.4s, v6.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8ce // sdot v14.4s, v6.16b, v1.4b[2]\n"
+ "ldr q6, [x14, #0xc0]\n"
+ ".inst 0x4f80e8eb // sdot v11.4s, v7.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8ef // sdot v15.4s, v7.16b, v1.4b[2]\n"
+ "ldr q7, [x14, #0xd0]\n"
+ ".inst 0x4fa0e8c8 // sdot v8.4s, v6.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e8cc // sdot v12.4s, v6.16b, v1.4b[3]\n"
+ "ldr q6, [x14, #0xe0]\n"
+ ".inst 0x4fa0e8e9 // sdot v9.4s, v7.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e8ed // sdot v13.4s, v7.16b, v1.4b[3]\n"
+ "ldr q7, [x14, #0xf0]\n"
+ "add x14, x14, #0x100\n"
+ ".inst 0x4fa0e8ca // sdot v10.4s, v6.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e8ce // sdot v14.4s, v6.16b, v1.4b[3]\n"
+ ".inst 0x4fa0e8eb // sdot v11.4s, v7.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e8ef // sdot v15.4s, v7.16b, v1.4b[3]\n"
+ "bge 53b\n"
+ "54:" // Height 2: Multiply loop: Single iteration only
+ "sub x11, x11, #0x10\n"
+ "ldr q0, [x10, #0x0]\n"
+ "ldr q1, [x28, #0x0]\n"
+ "ldr q6, [x14, #0x0]\n"
+ ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n"
+ "ldr q7, [x14, #0x10]\n"
+ ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n"
+ "ldr q6, [x14, #0x20]\n"
+ "add x10, x10, #0x10\n"
+ ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x28, x28, #0x10\n"
+ ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "ldr q7, [x14, #0x30]\n"
+ ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n"
+ ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n"
+ "ldr q6, [x14, #0x40]\n"
+ ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n"
+ "ldr q7, [x14, #0x50]\n"
+ ".inst 0x4fa0e0c8 // sdot v8.4s, v6.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e0cc // sdot v12.4s, v6.16b, v1.4b[1]\n"
+ "ldr q6, [x14, #0x60]\n"
+ ".inst 0x4fa0e0e9 // sdot v9.4s, v7.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e0ed // sdot v13.4s, v7.16b, v1.4b[1]\n"
+ "ldr q7, [x14, #0x70]\n"
+ ".inst 0x4fa0e0ca // sdot v10.4s, v6.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e0ce // sdot v14.4s, v6.16b, v1.4b[1]\n"
+ "ldr q6, [x14, #0x80]\n"
+ ".inst 0x4fa0e0eb // sdot v11.4s, v7.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e0ef // sdot v15.4s, v7.16b, v1.4b[1]\n"
+ "ldr q7, [x14, #0x90]\n"
+ ".inst 0x4f80e8c8 // sdot v8.4s, v6.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8cc // sdot v12.4s, v6.16b, v1.4b[2]\n"
+ "ldr q6, [x14, #0xa0]\n"
+ ".inst 0x4f80e8e9 // sdot v9.4s, v7.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8ed // sdot v13.4s, v7.16b, v1.4b[2]\n"
+ "ldr q7, [x14, #0xb0]\n"
+ ".inst 0x4f80e8ca // sdot v10.4s, v6.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8ce // sdot v14.4s, v6.16b, v1.4b[2]\n"
+ "ldr q6, [x14, #0xc0]\n"
+ ".inst 0x4f80e8eb // sdot v11.4s, v7.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8ef // sdot v15.4s, v7.16b, v1.4b[2]\n"
+ "ldr q7, [x14, #0xd0]\n"
+ ".inst 0x4fa0e8c8 // sdot v8.4s, v6.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e8cc // sdot v12.4s, v6.16b, v1.4b[3]\n"
+ "ldr q6, [x14, #0xe0]\n"
+ ".inst 0x4fa0e8e9 // sdot v9.4s, v7.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e8ed // sdot v13.4s, v7.16b, v1.4b[3]\n"
+ "ldr q7, [x14, #0xf0]\n"
+ "add x14, x14, #0x100\n"
+ ".inst 0x4fa0e8ca // sdot v10.4s, v6.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e8ce // sdot v14.4s, v6.16b, v1.4b[3]\n"
+ ".inst 0x4fa0e8eb // sdot v11.4s, v7.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e8ef // sdot v15.4s, v7.16b, v1.4b[3]\n"
+ "55:" // Height 2: Multiply loop: Main loop skip
+ "cbz x11, 60f\n"
+ "cmp x11, #0x4\n"
+ "blt 57f\n"
+ "56:" // Height 2: Multiply loop: Odd block loop
+ "ldr s0, [x10], #0x4\n"
+ "ldr s1, [x28], #0x4\n"
+ "ldr q6, [x14, #0x0]\n"
+ ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n"
+ "ldr q7, [x14, #0x10]\n"
+ ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n"
+ "ldr q6, [x14, #0x20]\n"
+ "sub x11, x11, #0x4\n"
+ ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n"
+ "cmp x11, #0x4\n"
+ ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n"
+ "ldr q7, [x14, #0x30]\n"
+ ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n"
+ "add x14, x14, #0x40\n"
+ ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n"
+ ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n"
+ "bge 56b\n"
+ "cbz x11, 60f\n"
+ "57:" // Height 2: Multiply loop: Skip odd blocks
+ "tbz x11, #1, 58f\n"
+ "ldr h0, [x10], #0x2\n"
+ "ldr h1, [x28], #0x2\n"
+ "tbz x11, #0, 59f\n"
+ "ld1 { v0.b }[2], [x10]\n"
+ "ld1 { v1.b }[2], [x28]\n"
+ "b 59f\n"
+ "58:" // Height 2: Multiply loop: Ragged operand read: partial_1_0
+ "ldr b0, [x10, #0x0]\n"
+ "ldr b1, [x28, #0x0]\n"
+ "59:" // Height 2: Multiply loop: Ragged operand read: Done
+ "ldr q6, [x14, #0x0]\n"
+ ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n"
+ "ldr q7, [x14, #0x10]\n"
+ ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n"
+ "ldr q6, [x14, #0x20]\n"
+ ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n"
+ "ldr q7, [x14, #0x30]\n"
+ ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n"
+ "add x14, x14, #0x40\n"
+ ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n"
+ ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n"
+ "60:" // Height 2: Multiply loop: No odd multiplies
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "add x12, x12, #0x1\n"
+ "cmp x12, x19\n"
+ "bne 50b\n"
+ "prfm pstl1keep, [x13, #0x0]\n"
+ "prfm pstl1keep, [x9, #0x0]\n"
+ "cmp x15, #0x10\n"
+ "bge 69f\n"
+ "tbz x15, #3, 64f\n"
+ "st1 { v8.4s }, [x13], #0x10\n"
+ "st1 { v9.4s }, [x13], #0x10\n"
+ "st1 { v12.4s }, [x9], #0x10\n"
+ "st1 { v13.4s }, [x9], #0x10\n"
+ "tbz x15, #2, 62f\n"
+ "st1 { v10.4s }, [x13], #0x10\n"
+ "st1 { v14.4s }, [x9], #0x10\n"
+ "tbz x15, #1, 61f\n"
+ "str d11, [x13], #0x8\n"
+ "str d15, [x9], #0x8\n"
+ "tbz x15, #0, 68f\n"
+ "st1 { v11.s }[2], [x13]\n"
+ "st1 { v15.s }[2], [x9]\n"
+ "b 68f\n"
+ "61:" // Height 2: Partial direct writeback: partial_1_12
+ "tbz x15, #0, 68f\n"
+ "str s11, [x13, #0x0]\n"
+ "str s15, [x9, #0x0]\n"
+ "b 68f\n"
+ "62:" // Height 2: Partial direct writeback: partial_2_8
+ "tbz x15, #1, 63f\n"
+ "str d10, [x13], #0x8\n"
+ "str d14, [x9], #0x8\n"
+ "tbz x15, #0, 68f\n"
+ "st1 { v10.s }[2], [x13]\n"
+ "st1 { v14.s }[2], [x9]\n"
+ "b 68f\n"
+ "63:" // Height 2: Partial direct writeback: partial_1_8
+ "tbz x15, #0, 68f\n"
+ "str s10, [x13, #0x0]\n"
+ "str s14, [x9, #0x0]\n"
+ "b 68f\n"
+ "64:" // Height 2: Partial direct writeback: partial_4_0
+ "tbz x15, #2, 66f\n"
+ "st1 { v8.4s }, [x13], #0x10\n"
+ "st1 { v12.4s }, [x9], #0x10\n"
+ "tbz x15, #1, 65f\n"
+ "str d9, [x13], #0x8\n"
+ "str d13, [x9], #0x8\n"
+ "tbz x15, #0, 68f\n"
+ "st1 { v9.s }[2], [x13]\n"
+ "st1 { v13.s }[2], [x9]\n"
+ "b 68f\n"
+ "65:" // Height 2: Partial direct writeback: partial_1_4
+ "tbz x15, #0, 68f\n"
+ "str s9, [x13, #0x0]\n"
+ "str s13, [x9, #0x0]\n"
+ "b 68f\n"
+ "66:" // Height 2: Partial direct writeback: partial_2_0
+ "tbz x15, #1, 67f\n"
+ "str d8, [x13], #0x8\n"
+ "str d12, [x9], #0x8\n"
+ "tbz x15, #0, 68f\n"
+ "st1 { v8.s }[2], [x13]\n"
+ "st1 { v12.s }[2], [x9]\n"
+ "b 68f\n"
+ "67:" // Height 2: Partial direct writeback: partial_1_0
+ "str s8, [x13, #0x0]\n"
+ "str s12, [x9, #0x0]\n"
+ "68:" // Height 2: Partial direct writeback: Done
+ "b 70f\n"
+ "69:" // Height 2: Full writeback
+ "str q8, [x13, #0x0]\n"
+ "str q9, [x13, #0x10]\n"
+ "str q10, [x13, #0x20]\n"
+ "str q11, [x13, #0x30]\n"
+ "str q12, [x9, #0x0]\n"
+ "str q13, [x9, #0x10]\n"
+ "str q14, [x9, #0x20]\n"
+ "str q15, [x9, #0x30]\n"
+ "add x13, x13, #0x40\n"
+ "add x9, x9, #0x40\n"
+ "70:" // Height 2: Writeback done
+ "subs x15, x15, #0x10\n"
+ "bgt 38b\n"
+ "b 212f\n"
+ "71:" // Height 3
+ "ldr x15, [%x[args_ptr], %[offsetof_N]]\n"
+ "ldr x14, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 72f\n"
+ "ldr x13, [%x[output_ptr], #0x0]\n"
+ "add x13, x13, x19, LSL #2\n"
+ "ldr x9, [%x[output_ptr], #0x8]\n"
+ "ldr x27, [%x[output_ptr], #0x10]\n"
+ "add x9, x9, x19, LSL #2\n"
+ "add x27, x27, x19, LSL #2\n"
+ "b 73f\n"
+ "72:" // Height 3: setup direct output
+ "mov x13, %x[output_ptr]\n"
+ "add x9, x13, x19, LSL #2\n"
+ "add x27, x9, x19, LSL #2\n"
+ "73:" // Height 3: Column loop
+ "tbz %x[flags], #0, 83f\n"
+ "cmp x15, #0x10\n"
+ "bge 82f\n"
+ "tbz x15, #3, 77f\n"
+ "ld1 { v8.4s }, [x13], #0x10\n"
+ "ld1 { v12.4s }, [x9], #0x10\n"
+ "ld1 { v16.4s }, [x27], #0x10\n"
+ "ld1 { v9.4s }, [x13], #0x10\n"
+ "ld1 { v13.4s }, [x9], #0x10\n"
+ "ld1 { v17.4s }, [x27], #0x10\n"
+ "tbz x15, #2, 75f\n"
+ "ld1 { v10.4s }, [x13], #0x10\n"
+ "ld1 { v14.4s }, [x9], #0x10\n"
+ "ld1 { v18.4s }, [x27], #0x10\n"
+ "tbz x15, #1, 74f\n"
+ "mov x19, #0x38\n"
+ "ldr d11, [x13], #0x8\n"
+ "ldr d15, [x9], #0x8\n"
+ "ldr d19, [x27], #0x8\n"
+ "tbz x15, #0, 81f\n"
+ "ld1 { v11.s }[2], [x13]\n"
+ "ld1 { v15.s }[2], [x9]\n"
+ "ld1 { v19.s }[2], [x27]\n"
+ "b 81f\n"
+ "74:" // Height 3: Partial accumulate: partial_1_12
+ "mov x19, #0x30\n"
+ "tbz x15, #0, 81f\n"
+ "ldr s11, [x13, #0x0]\n"
+ "ldr s15, [x9, #0x0]\n"
+ "ldr s19, [x27, #0x0]\n"
+ "b 81f\n"
+ "75:" // Height 3: Partial accumulate: partial_2_8
+ "tbz x15, #1, 76f\n"
+ "ldr d10, [x13], #0x8\n"
+ "ldr d14, [x9], #0x8\n"
+ "ldr d18, [x27], #0x8\n"
+ "mov x19, #0x28\n"
+ "tbz x15, #0, 81f\n"
+ "ld1 { v10.s }[2], [x13]\n"
+ "ld1 { v14.s }[2], [x9]\n"
+ "ld1 { v18.s }[2], [x27]\n"
+ "b 81f\n"
+ "76:" // Height 3: Partial accumulate: partial_1_8
+ "mov x19, #0x20\n"
+ "tbz x15, #0, 81f\n"
+ "ldr s10, [x13, #0x0]\n"
+ "ldr s14, [x9, #0x0]\n"
+ "ldr s18, [x27, #0x0]\n"
+ "b 81f\n"
+ "77:" // Height 3: Partial accumulate: partial_4_0
+ "tbz x15, #2, 79f\n"
+ "ld1 { v8.4s }, [x13], #0x10\n"
+ "ld1 { v12.4s }, [x9], #0x10\n"
+ "ld1 { v16.4s }, [x27], #0x10\n"
+ "tbz x15, #1, 78f\n"
+ "mov x19, #0x18\n"
+ "ldr d9, [x13], #0x8\n"
+ "ldr d13, [x9], #0x8\n"
+ "ldr d17, [x27], #0x8\n"
+ "tbz x15, #0, 81f\n"
+ "ld1 { v9.s }[2], [x13]\n"
+ "ld1 { v13.s }[2], [x9]\n"
+ "ld1 { v17.s }[2], [x27]\n"
+ "b 81f\n"
+ "78:" // Height 3: Partial accumulate: partial_1_4
+ "mov x19, #0x10\n"
+ "tbz x15, #0, 81f\n"
+ "ldr s9, [x13, #0x0]\n"
+ "ldr s13, [x9, #0x0]\n"
+ "ldr s17, [x27, #0x0]\n"
+ "b 81f\n"
+ "79:" // Height 3: Partial accumulate: partial_2_0
+ "tbz x15, #1, 80f\n"
+ "ldr d8, [x13], #0x8\n"
+ "ldr d12, [x9], #0x8\n"
+ "ldr d16, [x27], #0x8\n"
+ "mov x19, #0x8\n"
+ "tbz x15, #0, 81f\n"
+ "ld1 { v8.s }[2], [x13]\n"
+ "ld1 { v12.s }[2], [x9]\n"
+ "ld1 { v16.s }[2], [x27]\n"
+ "b 81f\n"
+ "80:" // Height 3: Partial accumulate: partial_1_0
+ "mov x19, #0x0\n"
+ "ldr s8, [x13, #0x0]\n"
+ "ldr s12, [x9, #0x0]\n"
+ "ldr s16, [x27, #0x0]\n"
+ "81:" // Height 3: Partial accumulate: Done
+ "sub x13, x13, x19\n"
+ "sub x9, x9, x19\n"
+ "sub x27, x27, x19\n"
+ "b 84f\n"
+ "82:" // Height 3: full accumulate
+ "ldr q8, [x13, #0x0]\n"
+ "ldr q9, [x13, #0x10]\n"
+ "ldr q10, [x13, #0x20]\n"
+ "ldr q11, [x13, #0x30]\n"
+ "ldr q12, [x9, #0x0]\n"
+ "ldr q13, [x9, #0x10]\n"
+ "ldr q14, [x9, #0x20]\n"
+ "ldr q15, [x9, #0x30]\n"
+ "ldr q16, [x27, #0x0]\n"
+ "ldr q17, [x27, #0x10]\n"
+ "ldr q18, [x27, #0x20]\n"
+ "ldr q19, [x27, #0x30]\n"
+ "b 84f\n"
+ "83:" // Height 3: no accumulate
+ "movi v8.4s, #0x0\n"
+ "movi v9.4s, #0x0\n"
+ "movi v10.4s, #0x0\n"
+ "movi v11.4s, #0x0\n"
+ "movi v12.4s, #0x0\n"
+ "movi v13.4s, #0x0\n"
+ "movi v14.4s, #0x0\n"
+ "movi v15.4s, #0x0\n"
+ "movi v16.4s, #0x0\n"
+ "movi v17.4s, #0x0\n"
+ "movi v18.4s, #0x0\n"
+ "movi v19.4s, #0x0\n"
+ "84:" // Height 3: setup done
+ "mov x12, #0x0\n"
+ "85:" // Height 3: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w11, [x20, x12, LSL #0x2]\n"
+ "tbz %x[flags], #3, 86f\n"
+ "ldr x20, [%x[input_ptr], x12, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x10, [x20, #0x0]\n"
+ "ldr x28, [x20, #0x8]\n"
+ "ldr x26, [x20, #0x10]\n"
+ "cbnz x12, 87f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x10, x10, x19\n"
+ "add x28, x28, x19\n"
+ "add x26, x26, x19\n"
+ "b 87f\n"
+ "86:" // Height 3: setup direct input
+ "mov x10, %x[input_ptr]\n"
+ "add x28, x10, x19\n"
+ "add x26, x28, x19\n"
+ "87:" // Height 3: input setup done
+ "cmp x11, #0x10\n"
+ "blt 90f\n"
+ "cmp x11, #0x20\n"
+ "blt 89f\n"
+ "88:" // Height 3: Multiply loop: Main loop head
+ "ldr q0, [x10, #0x0]\n"
+ "ldr q1, [x28, #0x0]\n"
+ "ldr q2, [x26, #0x0]\n"
+ "ldr q6, [x14, #0x0]\n"
+ ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n"
+ "ldr q7, [x14, #0x10]\n"
+ ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n"
+ "add x10, x10, #0x10\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n"
+ "ldr q6, [x14, #0x20]\n"
+ ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n"
+ "add x28, x28, #0x10\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n"
+ "add x26, x26, #0x10\n"
+ ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "ldr q7, [x14, #0x30]\n"
+ ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n"
+ "sub x11, x11, #0x10\n"
+ ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n"
+ "cmp x11, #0x20\n"
+ ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n"
+ "ldr q6, [x14, #0x40]\n"
+ ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n"
+ "ldr q7, [x14, #0x50]\n"
+ ".inst 0x4fa0e0c8 // sdot v8.4s, v6.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e0cc // sdot v12.4s, v6.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e0d0 // sdot v16.4s, v6.16b, v2.4b[1]\n"
+ "ldr q6, [x14, #0x60]\n"
+ ".inst 0x4fa0e0e9 // sdot v9.4s, v7.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e0ed // sdot v13.4s, v7.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e0f1 // sdot v17.4s, v7.16b, v2.4b[1]\n"
+ "ldr q7, [x14, #0x70]\n"
+ ".inst 0x4fa0e0ca // sdot v10.4s, v6.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e0ce // sdot v14.4s, v6.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e0d2 // sdot v18.4s, v6.16b, v2.4b[1]\n"
+ "ldr q6, [x14, #0x80]\n"
+ ".inst 0x4fa0e0eb // sdot v11.4s, v7.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e0ef // sdot v15.4s, v7.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e0f3 // sdot v19.4s, v7.16b, v2.4b[1]\n"
+ "ldr q7, [x14, #0x90]\n"
+ ".inst 0x4f80e8c8 // sdot v8.4s, v6.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8cc // sdot v12.4s, v6.16b, v1.4b[2]\n"
+ ".inst 0x4f82e8d0 // sdot v16.4s, v6.16b, v2.4b[2]\n"
+ "ldr q6, [x14, #0xa0]\n"
+ ".inst 0x4f80e8e9 // sdot v9.4s, v7.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8ed // sdot v13.4s, v7.16b, v1.4b[2]\n"
+ ".inst 0x4f82e8f1 // sdot v17.4s, v7.16b, v2.4b[2]\n"
+ "ldr q7, [x14, #0xb0]\n"
+ ".inst 0x4f80e8ca // sdot v10.4s, v6.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8ce // sdot v14.4s, v6.16b, v1.4b[2]\n"
+ ".inst 0x4f82e8d2 // sdot v18.4s, v6.16b, v2.4b[2]\n"
+ "ldr q6, [x14, #0xc0]\n"
+ ".inst 0x4f80e8eb // sdot v11.4s, v7.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8ef // sdot v15.4s, v7.16b, v1.4b[2]\n"
+ ".inst 0x4f82e8f3 // sdot v19.4s, v7.16b, v2.4b[2]\n"
+ "ldr q7, [x14, #0xd0]\n"
+ ".inst 0x4fa0e8c8 // sdot v8.4s, v6.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e8cc // sdot v12.4s, v6.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e8d0 // sdot v16.4s, v6.16b, v2.4b[3]\n"
+ "ldr q6, [x14, #0xe0]\n"
+ ".inst 0x4fa0e8e9 // sdot v9.4s, v7.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e8ed // sdot v13.4s, v7.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e8f1 // sdot v17.4s, v7.16b, v2.4b[3]\n"
+ "ldr q7, [x14, #0xf0]\n"
+ "add x14, x14, #0x100\n"
+ ".inst 0x4fa0e8ca // sdot v10.4s, v6.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e8ce // sdot v14.4s, v6.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e8d2 // sdot v18.4s, v6.16b, v2.4b[3]\n"
+ ".inst 0x4fa0e8eb // sdot v11.4s, v7.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e8ef // sdot v15.4s, v7.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e8f3 // sdot v19.4s, v7.16b, v2.4b[3]\n"
+ "bge 88b\n"
+ "89:" // Height 3: Multiply loop: Single iteration only
+ "sub x11, x11, #0x10\n"
+ "ldr q0, [x10, #0x0]\n"
+ "ldr q1, [x28, #0x0]\n"
+ "ldr q2, [x26, #0x0]\n"
+ "ldr q6, [x14, #0x0]\n"
+ ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n"
+ "ldr q7, [x14, #0x10]\n"
+ ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n"
+ "add x10, x10, #0x10\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n"
+ "ldr q6, [x14, #0x20]\n"
+ ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n"
+ "add x28, x28, #0x10\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n"
+ "add x26, x26, #0x10\n"
+ ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "ldr q7, [x14, #0x30]\n"
+ ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n"
+ ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n"
+ "ldr q6, [x14, #0x40]\n"
+ ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n"
+ "ldr q7, [x14, #0x50]\n"
+ ".inst 0x4fa0e0c8 // sdot v8.4s, v6.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e0cc // sdot v12.4s, v6.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e0d0 // sdot v16.4s, v6.16b, v2.4b[1]\n"
+ "ldr q6, [x14, #0x60]\n"
+ ".inst 0x4fa0e0e9 // sdot v9.4s, v7.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e0ed // sdot v13.4s, v7.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e0f1 // sdot v17.4s, v7.16b, v2.4b[1]\n"
+ "ldr q7, [x14, #0x70]\n"
+ ".inst 0x4fa0e0ca // sdot v10.4s, v6.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e0ce // sdot v14.4s, v6.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e0d2 // sdot v18.4s, v6.16b, v2.4b[1]\n"
+ "ldr q6, [x14, #0x80]\n"
+ ".inst 0x4fa0e0eb // sdot v11.4s, v7.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e0ef // sdot v15.4s, v7.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e0f3 // sdot v19.4s, v7.16b, v2.4b[1]\n"
+ "ldr q7, [x14, #0x90]\n"
+ ".inst 0x4f80e8c8 // sdot v8.4s, v6.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8cc // sdot v12.4s, v6.16b, v1.4b[2]\n"
+ ".inst 0x4f82e8d0 // sdot v16.4s, v6.16b, v2.4b[2]\n"
+ "ldr q6, [x14, #0xa0]\n"
+ ".inst 0x4f80e8e9 // sdot v9.4s, v7.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8ed // sdot v13.4s, v7.16b, v1.4b[2]\n"
+ ".inst 0x4f82e8f1 // sdot v17.4s, v7.16b, v2.4b[2]\n"
+ "ldr q7, [x14, #0xb0]\n"
+ ".inst 0x4f80e8ca // sdot v10.4s, v6.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8ce // sdot v14.4s, v6.16b, v1.4b[2]\n"
+ ".inst 0x4f82e8d2 // sdot v18.4s, v6.16b, v2.4b[2]\n"
+ "ldr q6, [x14, #0xc0]\n"
+ ".inst 0x4f80e8eb // sdot v11.4s, v7.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8ef // sdot v15.4s, v7.16b, v1.4b[2]\n"
+ ".inst 0x4f82e8f3 // sdot v19.4s, v7.16b, v2.4b[2]\n"
+ "ldr q7, [x14, #0xd0]\n"
+ ".inst 0x4fa0e8c8 // sdot v8.4s, v6.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e8cc // sdot v12.4s, v6.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e8d0 // sdot v16.4s, v6.16b, v2.4b[3]\n"
+ "ldr q6, [x14, #0xe0]\n"
+ ".inst 0x4fa0e8e9 // sdot v9.4s, v7.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e8ed // sdot v13.4s, v7.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e8f1 // sdot v17.4s, v7.16b, v2.4b[3]\n"
+ "ldr q7, [x14, #0xf0]\n"
+ "add x14, x14, #0x100\n"
+ ".inst 0x4fa0e8ca // sdot v10.4s, v6.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e8ce // sdot v14.4s, v6.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e8d2 // sdot v18.4s, v6.16b, v2.4b[3]\n"
+ ".inst 0x4fa0e8eb // sdot v11.4s, v7.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e8ef // sdot v15.4s, v7.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e8f3 // sdot v19.4s, v7.16b, v2.4b[3]\n"
+ "90:" // Height 3: Multiply loop: Main loop skip
+ "cbz x11, 95f\n"
+ "cmp x11, #0x4\n"
+ "blt 92f\n"
+ "91:" // Height 3: Multiply loop: Odd block loop
+ "ldr s0, [x10], #0x4\n"
+ "ldr s1, [x28], #0x4\n"
+ "ldr s2, [x26], #0x4\n"
+ "ldr q6, [x14, #0x0]\n"
+ ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n"
+ "ldr q7, [x14, #0x10]\n"
+ ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n"
+ "sub x11, x11, #0x4\n"
+ ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n"
+ "ldr q6, [x14, #0x20]\n"
+ "cmp x11, #0x4\n"
+ ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n"
+ "ldr q7, [x14, #0x30]\n"
+ ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n"
+ "add x14, x14, #0x40\n"
+ ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n"
+ ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n"
+ "bge 91b\n"
+ "cbz x11, 95f\n"
+ "92:" // Height 3: Multiply loop: Skip odd blocks
+ "tbz x11, #1, 93f\n"
+ "ldr h0, [x10], #0x2\n"
+ "ldr h1, [x28], #0x2\n"
+ "ldr h2, [x26], #0x2\n"
+ "tbz x11, #0, 94f\n"
+ "ld1 { v0.b }[2], [x10]\n"
+ "ld1 { v1.b }[2], [x28]\n"
+ "ld1 { v2.b }[2], [x26]\n"
+ "b 94f\n"
+ "93:" // Height 3: Multiply loop: Ragged operand read: partial_1_0
+ "ldr b0, [x10, #0x0]\n"
+ "ldr b1, [x28, #0x0]\n"
+ "ldr b2, [x26, #0x0]\n"
+ "94:" // Height 3: Multiply loop: Ragged operand read: Done
+ "ldr q6, [x14, #0x0]\n"
+ ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n"
+ "ldr q7, [x14, #0x10]\n"
+ ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n"
+ "ldr q6, [x14, #0x20]\n"
+ ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n"
+ "ldr q7, [x14, #0x30]\n"
+ ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n"
+ "add x14, x14, #0x40\n"
+ ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n"
+ ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n"
+ "95:" // Height 3: Multiply loop: No odd multiplies
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "add x12, x12, #0x1\n"
+ "cmp x12, x19\n"
+ "bne 85b\n"
+ "prfm pstl1keep, [x13, #0x0]\n"
+ "prfm pstl1keep, [x9, #0x0]\n"
+ "cmp x15, #0x10\n"
+ "prfm pstl1keep, [x27, #0x0]\n"
+ "bge 104f\n"
+ "tbz x15, #3, 99f\n"
+ "st1 { v8.4s }, [x13], #0x10\n"
+ "st1 { v9.4s }, [x13], #0x10\n"
+ "st1 { v12.4s }, [x9], #0x10\n"
+ "st1 { v13.4s }, [x9], #0x10\n"
+ "st1 { v16.4s }, [x27], #0x10\n"
+ "st1 { v17.4s }, [x27], #0x10\n"
+ "tbz x15, #2, 97f\n"
+ "st1 { v10.4s }, [x13], #0x10\n"
+ "st1 { v14.4s }, [x9], #0x10\n"
+ "st1 { v18.4s }, [x27], #0x10\n"
+ "tbz x15, #1, 96f\n"
+ "str d11, [x13], #0x8\n"
+ "str d15, [x9], #0x8\n"
+ "str d19, [x27], #0x8\n"
+ "tbz x15, #0, 103f\n"
+ "st1 { v11.s }[2], [x13]\n"
+ "st1 { v15.s }[2], [x9]\n"
+ "st1 { v19.s }[2], [x27]\n"
+ "b 103f\n"
+ "96:" // Height 3: Partial direct writeback: partial_1_12
+ "tbz x15, #0, 103f\n"
+ "str s11, [x13, #0x0]\n"
+ "str s15, [x9, #0x0]\n"
+ "str s19, [x27, #0x0]\n"
+ "b 103f\n"
+ "97:" // Height 3: Partial direct writeback: partial_2_8
+ "tbz x15, #1, 98f\n"
+ "str d10, [x13], #0x8\n"
+ "str d14, [x9], #0x8\n"
+ "str d18, [x27], #0x8\n"
+ "tbz x15, #0, 103f\n"
+ "st1 { v10.s }[2], [x13]\n"
+ "st1 { v14.s }[2], [x9]\n"
+ "st1 { v18.s }[2], [x27]\n"
+ "b 103f\n"
+ "98:" // Height 3: Partial direct writeback: partial_1_8
+ "tbz x15, #0, 103f\n"
+ "str s10, [x13, #0x0]\n"
+ "str s14, [x9, #0x0]\n"
+ "str s18, [x27, #0x0]\n"
+ "b 103f\n"
+ "99:" // Height 3: Partial direct writeback: partial_4_0
+ "tbz x15, #2, 101f\n"
+ "st1 { v8.4s }, [x13], #0x10\n"
+ "st1 { v12.4s }, [x9], #0x10\n"
+ "st1 { v16.4s }, [x27], #0x10\n"
+ "tbz x15, #1, 100f\n"
+ "str d9, [x13], #0x8\n"
+ "str d13, [x9], #0x8\n"
+ "str d17, [x27], #0x8\n"
+ "tbz x15, #0, 103f\n"
+ "st1 { v9.s }[2], [x13]\n"
+ "st1 { v13.s }[2], [x9]\n"
+ "st1 { v17.s }[2], [x27]\n"
+ "b 103f\n"
+ "100:" // Height 3: Partial direct writeback: partial_1_4
+ "tbz x15, #0, 103f\n"
+ "str s9, [x13, #0x0]\n"
+ "str s13, [x9, #0x0]\n"
+ "str s17, [x27, #0x0]\n"
+ "b 103f\n"
+ "101:" // Height 3: Partial direct writeback: partial_2_0
+ "tbz x15, #1, 102f\n"
+ "str d8, [x13], #0x8\n"
+ "str d12, [x9], #0x8\n"
+ "str d16, [x27], #0x8\n"
+ "tbz x15, #0, 103f\n"
+ "st1 { v8.s }[2], [x13]\n"
+ "st1 { v12.s }[2], [x9]\n"
+ "st1 { v16.s }[2], [x27]\n"
+ "b 103f\n"
+ "102:" // Height 3: Partial direct writeback: partial_1_0
+ "str s8, [x13, #0x0]\n"
+ "str s12, [x9, #0x0]\n"
+ "str s16, [x27, #0x0]\n"
+ "103:" // Height 3: Partial direct writeback: Done
+ "b 105f\n"
+ "104:" // Height 3: Full writeback
+ "str q8, [x13, #0x0]\n"
+ "str q9, [x13, #0x10]\n"
+ "str q10, [x13, #0x20]\n"
+ "str q11, [x13, #0x30]\n"
+ "str q12, [x9, #0x0]\n"
+ "str q13, [x9, #0x10]\n"
+ "str q14, [x9, #0x20]\n"
+ "str q15, [x9, #0x30]\n"
+ "str q16, [x27, #0x0]\n"
+ "str q17, [x27, #0x10]\n"
+ "str q18, [x27, #0x20]\n"
+ "str q19, [x27, #0x30]\n"
+ "add x13, x13, #0x40\n"
+ "add x9, x9, #0x40\n"
+ "add x27, x27, #0x40\n"
+ "105:" // Height 3: Writeback done
+ "subs x15, x15, #0x10\n"
+ "bgt 73b\n"
+ "b 212f\n"
+ "106:" // Height 4
+ "ldr x15, [%x[args_ptr], %[offsetof_N]]\n"
+ "ldr x14, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 107f\n"
+ "ldr x13, [%x[output_ptr], #0x0]\n"
+ "add x13, x13, x19, LSL #2\n"
+ "ldr x9, [%x[output_ptr], #0x8]\n"
+ "ldr x27, [%x[output_ptr], #0x10]\n"
+ "add x9, x9, x19, LSL #2\n"
+ "ldr x25, [%x[output_ptr], #0x18]\n"
+ "add x27, x27, x19, LSL #2\n"
+ "add x25, x25, x19, LSL #2\n"
+ "b 108f\n"
+ "107:" // Height 4: setup direct output
+ "mov x13, %x[output_ptr]\n"
+ "add x9, x13, x19, LSL #2\n"
+ "add x27, x9, x19, LSL #2\n"
+ "add x25, x27, x19, LSL #2\n"
+ "108:" // Height 4: Column loop
+ "tbz %x[flags], #0, 118f\n"
+ "cmp x15, #0x10\n"
+ "bge 117f\n"
+ "tbz x15, #3, 112f\n"
+ "ld1 { v8.4s }, [x13], #0x10\n"
+ "ld1 { v12.4s }, [x9], #0x10\n"
+ "ld1 { v16.4s }, [x27], #0x10\n"
+ "ld1 { v20.4s }, [x25], #0x10\n"
+ "ld1 { v9.4s }, [x13], #0x10\n"
+ "ld1 { v13.4s }, [x9], #0x10\n"
+ "ld1 { v17.4s }, [x27], #0x10\n"
+ "ld1 { v21.4s }, [x25], #0x10\n"
+ "tbz x15, #2, 110f\n"
+ "ld1 { v10.4s }, [x13], #0x10\n"
+ "ld1 { v14.4s }, [x9], #0x10\n"
+ "ld1 { v18.4s }, [x27], #0x10\n"
+ "ld1 { v22.4s }, [x25], #0x10\n"
+ "tbz x15, #1, 109f\n"
+ "mov x19, #0x38\n"
+ "ldr d11, [x13], #0x8\n"
+ "ldr d15, [x9], #0x8\n"
+ "ldr d19, [x27], #0x8\n"
+ "ldr d23, [x25], #0x8\n"
+ "tbz x15, #0, 116f\n"
+ "ld1 { v11.s }[2], [x13]\n"
+ "ld1 { v15.s }[2], [x9]\n"
+ "ld1 { v19.s }[2], [x27]\n"
+ "ld1 { v23.s }[2], [x25]\n"
+ "b 116f\n"
+ "109:" // Height 4: Partial accumulate: partial_1_12
+ "mov x19, #0x30\n"
+ "tbz x15, #0, 116f\n"
+ "ldr s11, [x13, #0x0]\n"
+ "ldr s15, [x9, #0x0]\n"
+ "ldr s19, [x27, #0x0]\n"
+ "ldr s23, [x25, #0x0]\n"
+ "b 116f\n"
+ "110:" // Height 4: Partial accumulate: partial_2_8
+ "tbz x15, #1, 111f\n"
+ "ldr d10, [x13], #0x8\n"
+ "ldr d14, [x9], #0x8\n"
+ "ldr d18, [x27], #0x8\n"
+ "ldr d22, [x25], #0x8\n"
+ "mov x19, #0x28\n"
+ "tbz x15, #0, 116f\n"
+ "ld1 { v10.s }[2], [x13]\n"
+ "ld1 { v14.s }[2], [x9]\n"
+ "ld1 { v18.s }[2], [x27]\n"
+ "ld1 { v22.s }[2], [x25]\n"
+ "b 116f\n"
+ "111:" // Height 4: Partial accumulate: partial_1_8
+ "mov x19, #0x20\n"
+ "tbz x15, #0, 116f\n"
+ "ldr s10, [x13, #0x0]\n"
+ "ldr s14, [x9, #0x0]\n"
+ "ldr s18, [x27, #0x0]\n"
+ "ldr s22, [x25, #0x0]\n"
+ "b 116f\n"
+ "112:" // Height 4: Partial accumulate: partial_4_0
+ "tbz x15, #2, 114f\n"
+ "ld1 { v8.4s }, [x13], #0x10\n"
+ "ld1 { v12.4s }, [x9], #0x10\n"
+ "ld1 { v16.4s }, [x27], #0x10\n"
+ "ld1 { v20.4s }, [x25], #0x10\n"
+ "tbz x15, #1, 113f\n"
+ "mov x19, #0x18\n"
+ "ldr d9, [x13], #0x8\n"
+ "ldr d13, [x9], #0x8\n"
+ "ldr d17, [x27], #0x8\n"
+ "ldr d21, [x25], #0x8\n"
+ "tbz x15, #0, 116f\n"
+ "ld1 { v9.s }[2], [x13]\n"
+ "ld1 { v13.s }[2], [x9]\n"
+ "ld1 { v17.s }[2], [x27]\n"
+ "ld1 { v21.s }[2], [x25]\n"
+ "b 116f\n"
+ "113:" // Height 4: Partial accumulate: partial_1_4
+ "mov x19, #0x10\n"
+ "tbz x15, #0, 116f\n"
+ "ldr s9, [x13, #0x0]\n"
+ "ldr s13, [x9, #0x0]\n"
+ "ldr s17, [x27, #0x0]\n"
+ "ldr s21, [x25, #0x0]\n"
+ "b 116f\n"
+ "114:" // Height 4: Partial accumulate: partial_2_0
+ "tbz x15, #1, 115f\n"
+ "ldr d8, [x13], #0x8\n"
+ "ldr d12, [x9], #0x8\n"
+ "ldr d16, [x27], #0x8\n"
+ "ldr d20, [x25], #0x8\n"
+ "mov x19, #0x8\n"
+ "tbz x15, #0, 116f\n"
+ "ld1 { v8.s }[2], [x13]\n"
+ "ld1 { v12.s }[2], [x9]\n"
+ "ld1 { v16.s }[2], [x27]\n"
+ "ld1 { v20.s }[2], [x25]\n"
+ "b 116f\n"
+ "115:" // Height 4: Partial accumulate: partial_1_0
+ "mov x19, #0x0\n"
+ "ldr s8, [x13, #0x0]\n"
+ "ldr s12, [x9, #0x0]\n"
+ "ldr s16, [x27, #0x0]\n"
+ "ldr s20, [x25, #0x0]\n"
+ "116:" // Height 4: Partial accumulate: Done
+ "sub x13, x13, x19\n"
+ "sub x9, x9, x19\n"
+ "sub x27, x27, x19\n"
+ "sub x25, x25, x19\n"
+ "b 119f\n"
+ "117:" // Height 4: full accumulate
+ "ldr q8, [x13, #0x0]\n"
+ "ldr q9, [x13, #0x10]\n"
+ "ldr q10, [x13, #0x20]\n"
+ "ldr q11, [x13, #0x30]\n"
+ "ldr q12, [x9, #0x0]\n"
+ "ldr q13, [x9, #0x10]\n"
+ "ldr q14, [x9, #0x20]\n"
+ "ldr q15, [x9, #0x30]\n"
+ "ldr q16, [x27, #0x0]\n"
+ "ldr q17, [x27, #0x10]\n"
+ "ldr q18, [x27, #0x20]\n"
+ "ldr q19, [x27, #0x30]\n"
+ "ldr q20, [x25, #0x0]\n"
+ "ldr q21, [x25, #0x10]\n"
+ "ldr q22, [x25, #0x20]\n"
+ "ldr q23, [x25, #0x30]\n"
+ "b 119f\n"
+ "118:" // Height 4: no accumulate
+ "movi v8.4s, #0x0\n"
+ "movi v9.4s, #0x0\n"
+ "movi v10.4s, #0x0\n"
+ "movi v11.4s, #0x0\n"
+ "movi v12.4s, #0x0\n"
+ "movi v13.4s, #0x0\n"
+ "movi v14.4s, #0x0\n"
+ "movi v15.4s, #0x0\n"
+ "movi v16.4s, #0x0\n"
+ "movi v17.4s, #0x0\n"
+ "movi v18.4s, #0x0\n"
+ "movi v19.4s, #0x0\n"
+ "movi v20.4s, #0x0\n"
+ "movi v21.4s, #0x0\n"
+ "movi v22.4s, #0x0\n"
+ "movi v23.4s, #0x0\n"
+ "119:" // Height 4: setup done
+ "mov x12, #0x0\n"
+ "120:" // Height 4: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w11, [x20, x12, LSL #0x2]\n"
+ "tbz %x[flags], #3, 121f\n"
+ "ldr x20, [%x[input_ptr], x12, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x10, [x20, #0x0]\n"
+ "ldr x28, [x20, #0x8]\n"
+ "ldr x26, [x20, #0x10]\n"
+ "ldr x24, [x20, #0x18]\n"
+ "cbnz x12, 122f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x10, x10, x19\n"
+ "add x28, x28, x19\n"
+ "add x26, x26, x19\n"
+ "add x24, x24, x19\n"
+ "b 122f\n"
+ "121:" // Height 4: setup direct input
+ "mov x10, %x[input_ptr]\n"
+ "add x28, x10, x19\n"
+ "add x26, x28, x19\n"
+ "add x24, x26, x19\n"
+ "122:" // Height 4: input setup done
+ "cmp x11, #0x10\n"
+ "blt 125f\n"
+ "cmp x11, #0x20\n"
+ "blt 124f\n"
+ "123:" // Height 4: Multiply loop: Main loop head
+ "ldr q0, [x10, #0x0]\n"
+ "ldr q1, [x28, #0x0]\n"
+ "ldr q2, [x26, #0x0]\n"
+ "ldr q3, [x24, #0x0]\n"
+ "ldr q6, [x14, #0x0]\n"
+ ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n"
+ "ldr q7, [x14, #0x10]\n"
+ ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n"
+ "add x10, x10, #0x10\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n"
+ "add x28, x28, #0x10\n"
+ ".inst 0x4f83e0d4 // sdot v20.4s, v6.16b, v3.4b[0]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "ldr q6, [x14, #0x20]\n"
+ ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n"
+ "add x26, x26, #0x10\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n"
+ "add x24, x24, #0x10\n"
+ ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "sub x11, x11, #0x10\n"
+ ".inst 0x4f83e0f5 // sdot v21.4s, v7.16b, v3.4b[0]\n"
+ "ldr q7, [x14, #0x30]\n"
+ ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n"
+ "cmp x11, #0x20\n"
+ ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n"
+ ".inst 0x4f83e0d6 // sdot v22.4s, v6.16b, v3.4b[0]\n"
+ "ldr q6, [x14, #0x40]\n"
+ ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n"
+ ".inst 0x4f83e0f7 // sdot v23.4s, v7.16b, v3.4b[0]\n"
+ "ldr q7, [x14, #0x50]\n"
+ ".inst 0x4fa0e0c8 // sdot v8.4s, v6.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e0cc // sdot v12.4s, v6.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e0d0 // sdot v16.4s, v6.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e0d4 // sdot v20.4s, v6.16b, v3.4b[1]\n"
+ "ldr q6, [x14, #0x60]\n"
+ ".inst 0x4fa0e0e9 // sdot v9.4s, v7.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e0ed // sdot v13.4s, v7.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e0f1 // sdot v17.4s, v7.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e0f5 // sdot v21.4s, v7.16b, v3.4b[1]\n"
+ "ldr q7, [x14, #0x70]\n"
+ ".inst 0x4fa0e0ca // sdot v10.4s, v6.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e0ce // sdot v14.4s, v6.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e0d2 // sdot v18.4s, v6.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e0d6 // sdot v22.4s, v6.16b, v3.4b[1]\n"
+ "ldr q6, [x14, #0x80]\n"
+ ".inst 0x4fa0e0eb // sdot v11.4s, v7.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e0ef // sdot v15.4s, v7.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e0f3 // sdot v19.4s, v7.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e0f7 // sdot v23.4s, v7.16b, v3.4b[1]\n"
+ "ldr q7, [x14, #0x90]\n"
+ ".inst 0x4f80e8c8 // sdot v8.4s, v6.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8cc // sdot v12.4s, v6.16b, v1.4b[2]\n"
+ ".inst 0x4f82e8d0 // sdot v16.4s, v6.16b, v2.4b[2]\n"
+ ".inst 0x4f83e8d4 // sdot v20.4s, v6.16b, v3.4b[2]\n"
+ "ldr q6, [x14, #0xa0]\n"
+ ".inst 0x4f80e8e9 // sdot v9.4s, v7.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8ed // sdot v13.4s, v7.16b, v1.4b[2]\n"
+ ".inst 0x4f82e8f1 // sdot v17.4s, v7.16b, v2.4b[2]\n"
+ ".inst 0x4f83e8f5 // sdot v21.4s, v7.16b, v3.4b[2]\n"
+ "ldr q7, [x14, #0xb0]\n"
+ ".inst 0x4f80e8ca // sdot v10.4s, v6.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8ce // sdot v14.4s, v6.16b, v1.4b[2]\n"
+ ".inst 0x4f82e8d2 // sdot v18.4s, v6.16b, v2.4b[2]\n"
+ ".inst 0x4f83e8d6 // sdot v22.4s, v6.16b, v3.4b[2]\n"
+ "ldr q6, [x14, #0xc0]\n"
+ ".inst 0x4f80e8eb // sdot v11.4s, v7.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8ef // sdot v15.4s, v7.16b, v1.4b[2]\n"
+ ".inst 0x4f82e8f3 // sdot v19.4s, v7.16b, v2.4b[2]\n"
+ ".inst 0x4f83e8f7 // sdot v23.4s, v7.16b, v3.4b[2]\n"
+ "ldr q7, [x14, #0xd0]\n"
+ ".inst 0x4fa0e8c8 // sdot v8.4s, v6.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e8cc // sdot v12.4s, v6.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e8d0 // sdot v16.4s, v6.16b, v2.4b[3]\n"
+ ".inst 0x4fa3e8d4 // sdot v20.4s, v6.16b, v3.4b[3]\n"
+ "ldr q6, [x14, #0xe0]\n"
+ ".inst 0x4fa0e8e9 // sdot v9.4s, v7.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e8ed // sdot v13.4s, v7.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e8f1 // sdot v17.4s, v7.16b, v2.4b[3]\n"
+ ".inst 0x4fa3e8f5 // sdot v21.4s, v7.16b, v3.4b[3]\n"
+ "ldr q7, [x14, #0xf0]\n"
+ "add x14, x14, #0x100\n"
+ ".inst 0x4fa0e8ca // sdot v10.4s, v6.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e8ce // sdot v14.4s, v6.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e8d2 // sdot v18.4s, v6.16b, v2.4b[3]\n"
+ ".inst 0x4fa3e8d6 // sdot v22.4s, v6.16b, v3.4b[3]\n"
+ ".inst 0x4fa0e8eb // sdot v11.4s, v7.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e8ef // sdot v15.4s, v7.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e8f3 // sdot v19.4s, v7.16b, v2.4b[3]\n"
+ ".inst 0x4fa3e8f7 // sdot v23.4s, v7.16b, v3.4b[3]\n"
+ "bge 123b\n"
+ "124:" // Height 4: Multiply loop: Single iteration only
+ "sub x11, x11, #0x10\n"
+ "ldr q0, [x10, #0x0]\n"
+ "ldr q1, [x28, #0x0]\n"
+ "ldr q2, [x26, #0x0]\n"
+ "ldr q3, [x24, #0x0]\n"
+ "ldr q6, [x14, #0x0]\n"
+ ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n"
+ "ldr q7, [x14, #0x10]\n"
+ ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n"
+ "add x10, x10, #0x10\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n"
+ "add x28, x28, #0x10\n"
+ ".inst 0x4f83e0d4 // sdot v20.4s, v6.16b, v3.4b[0]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "ldr q6, [x14, #0x20]\n"
+ ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n"
+ "add x26, x26, #0x10\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n"
+ "add x24, x24, #0x10\n"
+ ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ ".inst 0x4f83e0f5 // sdot v21.4s, v7.16b, v3.4b[0]\n"
+ "ldr q7, [x14, #0x30]\n"
+ ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n"
+ ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n"
+ ".inst 0x4f83e0d6 // sdot v22.4s, v6.16b, v3.4b[0]\n"
+ "ldr q6, [x14, #0x40]\n"
+ ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n"
+ ".inst 0x4f83e0f7 // sdot v23.4s, v7.16b, v3.4b[0]\n"
+ "ldr q7, [x14, #0x50]\n"
+ ".inst 0x4fa0e0c8 // sdot v8.4s, v6.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e0cc // sdot v12.4s, v6.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e0d0 // sdot v16.4s, v6.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e0d4 // sdot v20.4s, v6.16b, v3.4b[1]\n"
+ "ldr q6, [x14, #0x60]\n"
+ ".inst 0x4fa0e0e9 // sdot v9.4s, v7.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e0ed // sdot v13.4s, v7.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e0f1 // sdot v17.4s, v7.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e0f5 // sdot v21.4s, v7.16b, v3.4b[1]\n"
+ "ldr q7, [x14, #0x70]\n"
+ ".inst 0x4fa0e0ca // sdot v10.4s, v6.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e0ce // sdot v14.4s, v6.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e0d2 // sdot v18.4s, v6.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e0d6 // sdot v22.4s, v6.16b, v3.4b[1]\n"
+ "ldr q6, [x14, #0x80]\n"
+ ".inst 0x4fa0e0eb // sdot v11.4s, v7.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e0ef // sdot v15.4s, v7.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e0f3 // sdot v19.4s, v7.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e0f7 // sdot v23.4s, v7.16b, v3.4b[1]\n"
+ "ldr q7, [x14, #0x90]\n"
+ ".inst 0x4f80e8c8 // sdot v8.4s, v6.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8cc // sdot v12.4s, v6.16b, v1.4b[2]\n"
+ ".inst 0x4f82e8d0 // sdot v16.4s, v6.16b, v2.4b[2]\n"
+ ".inst 0x4f83e8d4 // sdot v20.4s, v6.16b, v3.4b[2]\n"
+ "ldr q6, [x14, #0xa0]\n"
+ ".inst 0x4f80e8e9 // sdot v9.4s, v7.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8ed // sdot v13.4s, v7.16b, v1.4b[2]\n"
+ ".inst 0x4f82e8f1 // sdot v17.4s, v7.16b, v2.4b[2]\n"
+ ".inst 0x4f83e8f5 // sdot v21.4s, v7.16b, v3.4b[2]\n"
+ "ldr q7, [x14, #0xb0]\n"
+ ".inst 0x4f80e8ca // sdot v10.4s, v6.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8ce // sdot v14.4s, v6.16b, v1.4b[2]\n"
+ ".inst 0x4f82e8d2 // sdot v18.4s, v6.16b, v2.4b[2]\n"
+ ".inst 0x4f83e8d6 // sdot v22.4s, v6.16b, v3.4b[2]\n"
+ "ldr q6, [x14, #0xc0]\n"
+ ".inst 0x4f80e8eb // sdot v11.4s, v7.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8ef // sdot v15.4s, v7.16b, v1.4b[2]\n"
+ ".inst 0x4f82e8f3 // sdot v19.4s, v7.16b, v2.4b[2]\n"
+ ".inst 0x4f83e8f7 // sdot v23.4s, v7.16b, v3.4b[2]\n"
+ "ldr q7, [x14, #0xd0]\n"
+ ".inst 0x4fa0e8c8 // sdot v8.4s, v6.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e8cc // sdot v12.4s, v6.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e8d0 // sdot v16.4s, v6.16b, v2.4b[3]\n"
+ ".inst 0x4fa3e8d4 // sdot v20.4s, v6.16b, v3.4b[3]\n"
+ "ldr q6, [x14, #0xe0]\n"
+ ".inst 0x4fa0e8e9 // sdot v9.4s, v7.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e8ed // sdot v13.4s, v7.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e8f1 // sdot v17.4s, v7.16b, v2.4b[3]\n"
+ ".inst 0x4fa3e8f5 // sdot v21.4s, v7.16b, v3.4b[3]\n"
+ "ldr q7, [x14, #0xf0]\n"
+ "add x14, x14, #0x100\n"
+ ".inst 0x4fa0e8ca // sdot v10.4s, v6.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e8ce // sdot v14.4s, v6.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e8d2 // sdot v18.4s, v6.16b, v2.4b[3]\n"
+ ".inst 0x4fa3e8d6 // sdot v22.4s, v6.16b, v3.4b[3]\n"
+ ".inst 0x4fa0e8eb // sdot v11.4s, v7.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e8ef // sdot v15.4s, v7.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e8f3 // sdot v19.4s, v7.16b, v2.4b[3]\n"
+ ".inst 0x4fa3e8f7 // sdot v23.4s, v7.16b, v3.4b[3]\n"
+ "125:" // Height 4: Multiply loop: Main loop skip
+ "cbz x11, 130f\n"
+ "cmp x11, #0x4\n"
+ "blt 127f\n"
+ "126:" // Height 4: Multiply loop: Odd block loop
+ "ldr s0, [x10], #0x4\n"
+ "ldr s1, [x28], #0x4\n"
+ "ldr s2, [x26], #0x4\n"
+ "ldr s3, [x24], #0x4\n"
+ "ldr q6, [x14, #0x0]\n"
+ ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n"
+ "ldr q7, [x14, #0x10]\n"
+ ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n"
+ "sub x11, x11, #0x4\n"
+ ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n"
+ "cmp x11, #0x4\n"
+ ".inst 0x4f83e0d4 // sdot v20.4s, v6.16b, v3.4b[0]\n"
+ "ldr q6, [x14, #0x20]\n"
+ ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n"
+ ".inst 0x4f83e0f5 // sdot v21.4s, v7.16b, v3.4b[0]\n"
+ "ldr q7, [x14, #0x30]\n"
+ ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n"
+ "add x14, x14, #0x40\n"
+ ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n"
+ ".inst 0x4f83e0d6 // sdot v22.4s, v6.16b, v3.4b[0]\n"
+ ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n"
+ ".inst 0x4f83e0f7 // sdot v23.4s, v7.16b, v3.4b[0]\n"
+ "bge 126b\n"
+ "cbz x11, 130f\n"
+ "127:" // Height 4: Multiply loop: Skip odd blocks
+ "tbz x11, #1, 128f\n"
+ "ldr h0, [x10], #0x2\n"
+ "ldr h1, [x28], #0x2\n"
+ "ldr h2, [x26], #0x2\n"
+ "ldr h3, [x24], #0x2\n"
+ "tbz x11, #0, 129f\n"
+ "ld1 { v0.b }[2], [x10]\n"
+ "ld1 { v1.b }[2], [x28]\n"
+ "ld1 { v2.b }[2], [x26]\n"
+ "ld1 { v3.b }[2], [x24]\n"
+ "b 129f\n"
+ "128:" // Height 4: Multiply loop: Ragged operand read: partial_1_0
+ "ldr b0, [x10, #0x0]\n"
+ "ldr b1, [x28, #0x0]\n"
+ "ldr b2, [x26, #0x0]\n"
+ "ldr b3, [x24, #0x0]\n"
+ "129:" // Height 4: Multiply loop: Ragged operand read: Done
+ "ldr q6, [x14, #0x0]\n"
+ ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n"
+ "ldr q7, [x14, #0x10]\n"
+ ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n"
+ ".inst 0x4f83e0d4 // sdot v20.4s, v6.16b, v3.4b[0]\n"
+ "ldr q6, [x14, #0x20]\n"
+ ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n"
+ ".inst 0x4f83e0f5 // sdot v21.4s, v7.16b, v3.4b[0]\n"
+ "ldr q7, [x14, #0x30]\n"
+ ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n"
+ "add x14, x14, #0x40\n"
+ ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n"
+ ".inst 0x4f83e0d6 // sdot v22.4s, v6.16b, v3.4b[0]\n"
+ ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n"
+ ".inst 0x4f83e0f7 // sdot v23.4s, v7.16b, v3.4b[0]\n"
+ "130:" // Height 4: Multiply loop: No odd multiplies
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "add x12, x12, #0x1\n"
+ "cmp x12, x19\n"
+ "bne 120b\n"
+ "prfm pstl1keep, [x13, #0x0]\n"
+ "prfm pstl1keep, [x9, #0x0]\n"
+ "cmp x15, #0x10\n"
+ "prfm pstl1keep, [x27, #0x0]\n"
+ "prfm pstl1keep, [x25, #0x0]\n"
+ "bge 139f\n"
+ "tbz x15, #3, 134f\n"
+ "st1 { v8.4s }, [x13], #0x10\n"
+ "st1 { v9.4s }, [x13], #0x10\n"
+ "st1 { v12.4s }, [x9], #0x10\n"
+ "st1 { v13.4s }, [x9], #0x10\n"
+ "st1 { v16.4s }, [x27], #0x10\n"
+ "st1 { v17.4s }, [x27], #0x10\n"
+ "st1 { v20.4s }, [x25], #0x10\n"
+ "st1 { v21.4s }, [x25], #0x10\n"
+ "tbz x15, #2, 132f\n"
+ "st1 { v10.4s }, [x13], #0x10\n"
+ "st1 { v14.4s }, [x9], #0x10\n"
+ "st1 { v18.4s }, [x27], #0x10\n"
+ "st1 { v22.4s }, [x25], #0x10\n"
+ "tbz x15, #1, 131f\n"
+ "str d11, [x13], #0x8\n"
+ "str d15, [x9], #0x8\n"
+ "str d19, [x27], #0x8\n"
+ "str d23, [x25], #0x8\n"
+ "tbz x15, #0, 138f\n"
+ "st1 { v11.s }[2], [x13]\n"
+ "st1 { v15.s }[2], [x9]\n"
+ "st1 { v19.s }[2], [x27]\n"
+ "st1 { v23.s }[2], [x25]\n"
+ "b 138f\n"
+ "131:" // Height 4: Partial direct writeback: partial_1_12
+ "tbz x15, #0, 138f\n"
+ "str s11, [x13, #0x0]\n"
+ "str s15, [x9, #0x0]\n"
+ "str s19, [x27, #0x0]\n"
+ "str s23, [x25, #0x0]\n"
+ "b 138f\n"
+ "132:" // Height 4: Partial direct writeback: partial_2_8
+ "tbz x15, #1, 133f\n"
+ "str d10, [x13], #0x8\n"
+ "str d14, [x9], #0x8\n"
+ "str d18, [x27], #0x8\n"
+ "str d22, [x25], #0x8\n"
+ "tbz x15, #0, 138f\n"
+ "st1 { v10.s }[2], [x13]\n"
+ "st1 { v14.s }[2], [x9]\n"
+ "st1 { v18.s }[2], [x27]\n"
+ "st1 { v22.s }[2], [x25]\n"
+ "b 138f\n"
+ "133:" // Height 4: Partial direct writeback: partial_1_8
+ "tbz x15, #0, 138f\n"
+ "str s10, [x13, #0x0]\n"
+ "str s14, [x9, #0x0]\n"
+ "str s18, [x27, #0x0]\n"
+ "str s22, [x25, #0x0]\n"
+ "b 138f\n"
+ "134:" // Height 4: Partial direct writeback: partial_4_0
+ "tbz x15, #2, 136f\n"
+ "st1 { v8.4s }, [x13], #0x10\n"
+ "st1 { v12.4s }, [x9], #0x10\n"
+ "st1 { v16.4s }, [x27], #0x10\n"
+ "st1 { v20.4s }, [x25], #0x10\n"
+ "tbz x15, #1, 135f\n"
+ "str d9, [x13], #0x8\n"
+ "str d13, [x9], #0x8\n"
+ "str d17, [x27], #0x8\n"
+ "str d21, [x25], #0x8\n"
+ "tbz x15, #0, 138f\n"
+ "st1 { v9.s }[2], [x13]\n"
+ "st1 { v13.s }[2], [x9]\n"
+ "st1 { v17.s }[2], [x27]\n"
+ "st1 { v21.s }[2], [x25]\n"
+ "b 138f\n"
+ "135:" // Height 4: Partial direct writeback: partial_1_4
+ "tbz x15, #0, 138f\n"
+ "str s9, [x13, #0x0]\n"
+ "str s13, [x9, #0x0]\n"
+ "str s17, [x27, #0x0]\n"
+ "str s21, [x25, #0x0]\n"
+ "b 138f\n"
+ "136:" // Height 4: Partial direct writeback: partial_2_0
+ "tbz x15, #1, 137f\n"
+ "str d8, [x13], #0x8\n"
+ "str d12, [x9], #0x8\n"
+ "str d16, [x27], #0x8\n"
+ "str d20, [x25], #0x8\n"
+ "tbz x15, #0, 138f\n"
+ "st1 { v8.s }[2], [x13]\n"
+ "st1 { v12.s }[2], [x9]\n"
+ "st1 { v16.s }[2], [x27]\n"
+ "st1 { v20.s }[2], [x25]\n"
+ "b 138f\n"
+ "137:" // Height 4: Partial direct writeback: partial_1_0
+ "str s8, [x13, #0x0]\n"
+ "str s12, [x9, #0x0]\n"
+ "str s16, [x27, #0x0]\n"
+ "str s20, [x25, #0x0]\n"
+ "138:" // Height 4: Partial direct writeback: Done
+ "b 140f\n"
+ "139:" // Height 4: Full writeback
+ "str q8, [x13, #0x0]\n"
+ "str q9, [x13, #0x10]\n"
+ "str q10, [x13, #0x20]\n"
+ "str q11, [x13, #0x30]\n"
+ "str q12, [x9, #0x0]\n"
+ "str q13, [x9, #0x10]\n"
+ "str q14, [x9, #0x20]\n"
+ "str q15, [x9, #0x30]\n"
+ "str q16, [x27, #0x0]\n"
+ "str q17, [x27, #0x10]\n"
+ "str q18, [x27, #0x20]\n"
+ "str q19, [x27, #0x30]\n"
+ "str q20, [x25, #0x0]\n"
+ "str q21, [x25, #0x10]\n"
+ "str q22, [x25, #0x20]\n"
+ "str q23, [x25, #0x30]\n"
+ "add x13, x13, #0x40\n"
+ "add x9, x9, #0x40\n"
+ "add x27, x27, #0x40\n"
+ "add x25, x25, #0x40\n"
+ "140:" // Height 4: Writeback done
+ "subs x15, x15, #0x10\n"
+ "bgt 108b\n"
+ "b 212f\n"
+ "141:" // Height 5
+ "ldr x15, [%x[args_ptr], %[offsetof_N]]\n"
+ "ldr x14, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 142f\n"
+ "ldr x13, [%x[output_ptr], #0x0]\n"
+ "add x13, x13, x19, LSL #2\n"
+ "ldr x9, [%x[output_ptr], #0x8]\n"
+ "ldr x27, [%x[output_ptr], #0x10]\n"
+ "add x9, x9, x19, LSL #2\n"
+ "ldr x25, [%x[output_ptr], #0x18]\n"
+ "ldr x23, [%x[output_ptr], #0x20]\n"
+ "add x27, x27, x19, LSL #2\n"
+ "add x25, x25, x19, LSL #2\n"
+ "add x23, x23, x19, LSL #2\n"
+ "b 143f\n"
+ "142:" // Height 5: setup direct output
+ "mov x13, %x[output_ptr]\n"
+ "add x9, x13, x19, LSL #2\n"
+ "add x27, x9, x19, LSL #2\n"
+ "add x25, x27, x19, LSL #2\n"
+ "add x23, x25, x19, LSL #2\n"
+ "143:" // Height 5: Column loop
+ "tbz %x[flags], #0, 153f\n"
+ "cmp x15, #0x10\n"
+ "bge 152f\n"
+ "tbz x15, #3, 147f\n"
+ "ld1 { v8.4s }, [x13], #0x10\n"
+ "ld1 { v12.4s }, [x9], #0x10\n"
+ "ld1 { v16.4s }, [x27], #0x10\n"
+ "ld1 { v20.4s }, [x25], #0x10\n"
+ "ld1 { v24.4s }, [x23], #0x10\n"
+ "ld1 { v9.4s }, [x13], #0x10\n"
+ "ld1 { v13.4s }, [x9], #0x10\n"
+ "ld1 { v17.4s }, [x27], #0x10\n"
+ "ld1 { v21.4s }, [x25], #0x10\n"
+ "ld1 { v25.4s }, [x23], #0x10\n"
+ "tbz x15, #2, 145f\n"
+ "ld1 { v10.4s }, [x13], #0x10\n"
+ "ld1 { v14.4s }, [x9], #0x10\n"
+ "ld1 { v18.4s }, [x27], #0x10\n"
+ "ld1 { v22.4s }, [x25], #0x10\n"
+ "ld1 { v26.4s }, [x23], #0x10\n"
+ "tbz x15, #1, 144f\n"
+ "mov x19, #0x38\n"
+ "ldr d11, [x13], #0x8\n"
+ "ldr d15, [x9], #0x8\n"
+ "ldr d19, [x27], #0x8\n"
+ "ldr d23, [x25], #0x8\n"
+ "ldr d27, [x23], #0x8\n"
+ "tbz x15, #0, 151f\n"
+ "ld1 { v11.s }[2], [x13]\n"
+ "ld1 { v15.s }[2], [x9]\n"
+ "ld1 { v19.s }[2], [x27]\n"
+ "ld1 { v23.s }[2], [x25]\n"
+ "ld1 { v27.s }[2], [x23]\n"
+ "b 151f\n"
+ "144:" // Height 5: Partial accumulate: partial_1_12
+ "mov x19, #0x30\n"
+ "tbz x15, #0, 151f\n"
+ "ldr s11, [x13, #0x0]\n"
+ "ldr s15, [x9, #0x0]\n"
+ "ldr s19, [x27, #0x0]\n"
+ "ldr s23, [x25, #0x0]\n"
+ "ldr s27, [x23, #0x0]\n"
+ "b 151f\n"
+ "145:" // Height 5: Partial accumulate: partial_2_8
+ "tbz x15, #1, 146f\n"
+ "ldr d10, [x13], #0x8\n"
+ "ldr d14, [x9], #0x8\n"
+ "ldr d18, [x27], #0x8\n"
+ "ldr d22, [x25], #0x8\n"
+ "ldr d26, [x23], #0x8\n"
+ "mov x19, #0x28\n"
+ "tbz x15, #0, 151f\n"
+ "ld1 { v10.s }[2], [x13]\n"
+ "ld1 { v14.s }[2], [x9]\n"
+ "ld1 { v18.s }[2], [x27]\n"
+ "ld1 { v22.s }[2], [x25]\n"
+ "ld1 { v26.s }[2], [x23]\n"
+ "b 151f\n"
+ "146:" // Height 5: Partial accumulate: partial_1_8
+ "mov x19, #0x20\n"
+ "tbz x15, #0, 151f\n"
+ "ldr s10, [x13, #0x0]\n"
+ "ldr s14, [x9, #0x0]\n"
+ "ldr s18, [x27, #0x0]\n"
+ "ldr s22, [x25, #0x0]\n"
+ "ldr s26, [x23, #0x0]\n"
+ "b 151f\n"
+ "147:" // Height 5: Partial accumulate: partial_4_0
+ "tbz x15, #2, 149f\n"
+ "ld1 { v8.4s }, [x13], #0x10\n"
+ "ld1 { v12.4s }, [x9], #0x10\n"
+ "ld1 { v16.4s }, [x27], #0x10\n"
+ "ld1 { v20.4s }, [x25], #0x10\n"
+ "ld1 { v24.4s }, [x23], #0x10\n"
+ "tbz x15, #1, 148f\n"
+ "mov x19, #0x18\n"
+ "ldr d9, [x13], #0x8\n"
+ "ldr d13, [x9], #0x8\n"
+ "ldr d17, [x27], #0x8\n"
+ "ldr d21, [x25], #0x8\n"
+ "ldr d25, [x23], #0x8\n"
+ "tbz x15, #0, 151f\n"
+ "ld1 { v9.s }[2], [x13]\n"
+ "ld1 { v13.s }[2], [x9]\n"
+ "ld1 { v17.s }[2], [x27]\n"
+ "ld1 { v21.s }[2], [x25]\n"
+ "ld1 { v25.s }[2], [x23]\n"
+ "b 151f\n"
+ "148:" // Height 5: Partial accumulate: partial_1_4
+ "mov x19, #0x10\n"
+ "tbz x15, #0, 151f\n"
+ "ldr s9, [x13, #0x0]\n"
+ "ldr s13, [x9, #0x0]\n"
+ "ldr s17, [x27, #0x0]\n"
+ "ldr s21, [x25, #0x0]\n"
+ "ldr s25, [x23, #0x0]\n"
+ "b 151f\n"
+ "149:" // Height 5: Partial accumulate: partial_2_0
+ "tbz x15, #1, 150f\n"
+ "ldr d8, [x13], #0x8\n"
+ "ldr d12, [x9], #0x8\n"
+ "ldr d16, [x27], #0x8\n"
+ "ldr d20, [x25], #0x8\n"
+ "ldr d24, [x23], #0x8\n"
+ "mov x19, #0x8\n"
+ "tbz x15, #0, 151f\n"
+ "ld1 { v8.s }[2], [x13]\n"
+ "ld1 { v12.s }[2], [x9]\n"
+ "ld1 { v16.s }[2], [x27]\n"
+ "ld1 { v20.s }[2], [x25]\n"
+ "ld1 { v24.s }[2], [x23]\n"
+ "b 151f\n"
+ "150:" // Height 5: Partial accumulate: partial_1_0
+ "mov x19, #0x0\n"
+ "ldr s8, [x13, #0x0]\n"
+ "ldr s12, [x9, #0x0]\n"
+ "ldr s16, [x27, #0x0]\n"
+ "ldr s20, [x25, #0x0]\n"
+ "ldr s24, [x23, #0x0]\n"
+ "151:" // Height 5: Partial accumulate: Done
+ "sub x13, x13, x19\n"
+ "sub x9, x9, x19\n"
+ "sub x27, x27, x19\n"
+ "sub x25, x25, x19\n"
+ "sub x23, x23, x19\n"
+ "b 154f\n"
+ "152:" // Height 5: full accumulate
+ "ldr q8, [x13, #0x0]\n"
+ "ldr q9, [x13, #0x10]\n"
+ "ldr q10, [x13, #0x20]\n"
+ "ldr q11, [x13, #0x30]\n"
+ "ldr q12, [x9, #0x0]\n"
+ "ldr q13, [x9, #0x10]\n"
+ "ldr q14, [x9, #0x20]\n"
+ "ldr q15, [x9, #0x30]\n"
+ "ldr q16, [x27, #0x0]\n"
+ "ldr q17, [x27, #0x10]\n"
+ "ldr q18, [x27, #0x20]\n"
+ "ldr q19, [x27, #0x30]\n"
+ "ldr q20, [x25, #0x0]\n"
+ "ldr q21, [x25, #0x10]\n"
+ "ldr q22, [x25, #0x20]\n"
+ "ldr q23, [x25, #0x30]\n"
+ "ldr q24, [x23, #0x0]\n"
+ "ldr q25, [x23, #0x10]\n"
+ "ldr q26, [x23, #0x20]\n"
+ "ldr q27, [x23, #0x30]\n"
+ "b 154f\n"
+ "153:" // Height 5: no accumulate
+ "movi v8.4s, #0x0\n"
+ "movi v9.4s, #0x0\n"
+ "movi v10.4s, #0x0\n"
+ "movi v11.4s, #0x0\n"
+ "movi v12.4s, #0x0\n"
+ "movi v13.4s, #0x0\n"
+ "movi v14.4s, #0x0\n"
+ "movi v15.4s, #0x0\n"
+ "movi v16.4s, #0x0\n"
+ "movi v17.4s, #0x0\n"
+ "movi v18.4s, #0x0\n"
+ "movi v19.4s, #0x0\n"
+ "movi v20.4s, #0x0\n"
+ "movi v21.4s, #0x0\n"
+ "movi v22.4s, #0x0\n"
+ "movi v23.4s, #0x0\n"
+ "movi v24.4s, #0x0\n"
+ "movi v25.4s, #0x0\n"
+ "movi v26.4s, #0x0\n"
+ "movi v27.4s, #0x0\n"
+ "154:" // Height 5: setup done
+ "mov x12, #0x0\n"
+ "155:" // Height 5: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w11, [x20, x12, LSL #0x2]\n"
+ "tbz %x[flags], #3, 156f\n"
+ "ldr x20, [%x[input_ptr], x12, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x10, [x20, #0x0]\n"
+ "ldr x28, [x20, #0x8]\n"
+ "ldr x26, [x20, #0x10]\n"
+ "ldr x24, [x20, #0x18]\n"
+ "ldr x22, [x20, #0x20]\n"
+ "cbnz x12, 157f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x10, x10, x19\n"
+ "add x28, x28, x19\n"
+ "add x26, x26, x19\n"
+ "add x24, x24, x19\n"
+ "add x22, x22, x19\n"
+ "b 157f\n"
+ "156:" // Height 5: setup direct input
+ "mov x10, %x[input_ptr]\n"
+ "add x28, x10, x19\n"
+ "add x26, x28, x19\n"
+ "add x24, x26, x19\n"
+ "add x22, x24, x19\n"
+ "157:" // Height 5: input setup done
+ "cmp x11, #0x10\n"
+ "blt 160f\n"
+ "cmp x11, #0x20\n"
+ "blt 159f\n"
+ "158:" // Height 5: Multiply loop: Main loop head
+ "ldr q0, [x10, #0x0]\n"
+ "ldr q1, [x28, #0x0]\n"
+ "ldr q2, [x26, #0x0]\n"
+ "ldr q3, [x24, #0x0]\n"
+ "ldr q4, [x22, #0x0]\n"
+ "ldr q6, [x14, #0x0]\n"
+ ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n"
+ "ldr q7, [x14, #0x10]\n"
+ ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n"
+ "add x10, x10, #0x10\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n"
+ "add x28, x28, #0x10\n"
+ ".inst 0x4f83e0d4 // sdot v20.4s, v6.16b, v3.4b[0]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "add x26, x26, #0x10\n"
+ ".inst 0x4f84e0d8 // sdot v24.4s, v6.16b, v4.4b[0]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "ldr q6, [x14, #0x20]\n"
+ ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n"
+ "add x24, x24, #0x10\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n"
+ "add x22, x22, #0x10\n"
+ ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n"
+ "prfm pldl1keep, [x22, #0x80]\n"
+ "sub x11, x11, #0x10\n"
+ ".inst 0x4f83e0f5 // sdot v21.4s, v7.16b, v3.4b[0]\n"
+ "cmp x11, #0x20\n"
+ ".inst 0x4f84e0f9 // sdot v25.4s, v7.16b, v4.4b[0]\n"
+ "ldr q7, [x14, #0x30]\n"
+ ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n"
+ ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n"
+ ".inst 0x4f83e0d6 // sdot v22.4s, v6.16b, v3.4b[0]\n"
+ ".inst 0x4f84e0da // sdot v26.4s, v6.16b, v4.4b[0]\n"
+ "ldr q6, [x14, #0x40]\n"
+ ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n"
+ ".inst 0x4f83e0f7 // sdot v23.4s, v7.16b, v3.4b[0]\n"
+ ".inst 0x4f84e0fb // sdot v27.4s, v7.16b, v4.4b[0]\n"
+ "ldr q7, [x14, #0x50]\n"
+ ".inst 0x4fa0e0c8 // sdot v8.4s, v6.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e0cc // sdot v12.4s, v6.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e0d0 // sdot v16.4s, v6.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e0d4 // sdot v20.4s, v6.16b, v3.4b[1]\n"
+ ".inst 0x4fa4e0d8 // sdot v24.4s, v6.16b, v4.4b[1]\n"
+ "ldr q6, [x14, #0x60]\n"
+ ".inst 0x4fa0e0e9 // sdot v9.4s, v7.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e0ed // sdot v13.4s, v7.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e0f1 // sdot v17.4s, v7.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e0f5 // sdot v21.4s, v7.16b, v3.4b[1]\n"
+ ".inst 0x4fa4e0f9 // sdot v25.4s, v7.16b, v4.4b[1]\n"
+ "ldr q7, [x14, #0x70]\n"
+ ".inst 0x4fa0e0ca // sdot v10.4s, v6.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e0ce // sdot v14.4s, v6.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e0d2 // sdot v18.4s, v6.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e0d6 // sdot v22.4s, v6.16b, v3.4b[1]\n"
+ ".inst 0x4fa4e0da // sdot v26.4s, v6.16b, v4.4b[1]\n"
+ "ldr q6, [x14, #0x80]\n"
+ ".inst 0x4fa0e0eb // sdot v11.4s, v7.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e0ef // sdot v15.4s, v7.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e0f3 // sdot v19.4s, v7.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e0f7 // sdot v23.4s, v7.16b, v3.4b[1]\n"
+ ".inst 0x4fa4e0fb // sdot v27.4s, v7.16b, v4.4b[1]\n"
+ "ldr q7, [x14, #0x90]\n"
+ ".inst 0x4f80e8c8 // sdot v8.4s, v6.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8cc // sdot v12.4s, v6.16b, v1.4b[2]\n"
+ ".inst 0x4f82e8d0 // sdot v16.4s, v6.16b, v2.4b[2]\n"
+ ".inst 0x4f83e8d4 // sdot v20.4s, v6.16b, v3.4b[2]\n"
+ ".inst 0x4f84e8d8 // sdot v24.4s, v6.16b, v4.4b[2]\n"
+ "ldr q6, [x14, #0xa0]\n"
+ ".inst 0x4f80e8e9 // sdot v9.4s, v7.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8ed // sdot v13.4s, v7.16b, v1.4b[2]\n"
+ ".inst 0x4f82e8f1 // sdot v17.4s, v7.16b, v2.4b[2]\n"
+ ".inst 0x4f83e8f5 // sdot v21.4s, v7.16b, v3.4b[2]\n"
+ ".inst 0x4f84e8f9 // sdot v25.4s, v7.16b, v4.4b[2]\n"
+ "ldr q7, [x14, #0xb0]\n"
+ ".inst 0x4f80e8ca // sdot v10.4s, v6.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8ce // sdot v14.4s, v6.16b, v1.4b[2]\n"
+ ".inst 0x4f82e8d2 // sdot v18.4s, v6.16b, v2.4b[2]\n"
+ ".inst 0x4f83e8d6 // sdot v22.4s, v6.16b, v3.4b[2]\n"
+ ".inst 0x4f84e8da // sdot v26.4s, v6.16b, v4.4b[2]\n"
+ "ldr q6, [x14, #0xc0]\n"
+ ".inst 0x4f80e8eb // sdot v11.4s, v7.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8ef // sdot v15.4s, v7.16b, v1.4b[2]\n"
+ ".inst 0x4f82e8f3 // sdot v19.4s, v7.16b, v2.4b[2]\n"
+ ".inst 0x4f83e8f7 // sdot v23.4s, v7.16b, v3.4b[2]\n"
+ ".inst 0x4f84e8fb // sdot v27.4s, v7.16b, v4.4b[2]\n"
+ "ldr q7, [x14, #0xd0]\n"
+ ".inst 0x4fa0e8c8 // sdot v8.4s, v6.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e8cc // sdot v12.4s, v6.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e8d0 // sdot v16.4s, v6.16b, v2.4b[3]\n"
+ ".inst 0x4fa3e8d4 // sdot v20.4s, v6.16b, v3.4b[3]\n"
+ ".inst 0x4fa4e8d8 // sdot v24.4s, v6.16b, v4.4b[3]\n"
+ "ldr q6, [x14, #0xe0]\n"
+ ".inst 0x4fa0e8e9 // sdot v9.4s, v7.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e8ed // sdot v13.4s, v7.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e8f1 // sdot v17.4s, v7.16b, v2.4b[3]\n"
+ ".inst 0x4fa3e8f5 // sdot v21.4s, v7.16b, v3.4b[3]\n"
+ ".inst 0x4fa4e8f9 // sdot v25.4s, v7.16b, v4.4b[3]\n"
+ "ldr q7, [x14, #0xf0]\n"
+ ".inst 0x4fa0e8ca // sdot v10.4s, v6.16b, v0.4b[3]\n"
+ "add x14, x14, #0x100\n"
+ ".inst 0x4fa1e8ce // sdot v14.4s, v6.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e8d2 // sdot v18.4s, v6.16b, v2.4b[3]\n"
+ ".inst 0x4fa3e8d6 // sdot v22.4s, v6.16b, v3.4b[3]\n"
+ ".inst 0x4fa4e8da // sdot v26.4s, v6.16b, v4.4b[3]\n"
+ ".inst 0x4fa0e8eb // sdot v11.4s, v7.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e8ef // sdot v15.4s, v7.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e8f3 // sdot v19.4s, v7.16b, v2.4b[3]\n"
+ ".inst 0x4fa3e8f7 // sdot v23.4s, v7.16b, v3.4b[3]\n"
+ ".inst 0x4fa4e8fb // sdot v27.4s, v7.16b, v4.4b[3]\n"
+ "bge 158b\n"
+ "159:" // Height 5: Multiply loop: Single iteration only
+ "sub x11, x11, #0x10\n"
+ "ldr q0, [x10, #0x0]\n"
+ "ldr q1, [x28, #0x0]\n"
+ "ldr q2, [x26, #0x0]\n"
+ "ldr q3, [x24, #0x0]\n"
+ "ldr q4, [x22, #0x0]\n"
+ "ldr q6, [x14, #0x0]\n"
+ ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n"
+ "ldr q7, [x14, #0x10]\n"
+ ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n"
+ "add x10, x10, #0x10\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n"
+ "add x28, x28, #0x10\n"
+ ".inst 0x4f83e0d4 // sdot v20.4s, v6.16b, v3.4b[0]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "add x26, x26, #0x10\n"
+ ".inst 0x4f84e0d8 // sdot v24.4s, v6.16b, v4.4b[0]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "ldr q6, [x14, #0x20]\n"
+ ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n"
+ "add x24, x24, #0x10\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n"
+ "add x22, x22, #0x10\n"
+ ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n"
+ "prfm pldl1keep, [x22, #0x80]\n"
+ ".inst 0x4f83e0f5 // sdot v21.4s, v7.16b, v3.4b[0]\n"
+ ".inst 0x4f84e0f9 // sdot v25.4s, v7.16b, v4.4b[0]\n"
+ "ldr q7, [x14, #0x30]\n"
+ ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n"
+ ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n"
+ ".inst 0x4f83e0d6 // sdot v22.4s, v6.16b, v3.4b[0]\n"
+ ".inst 0x4f84e0da // sdot v26.4s, v6.16b, v4.4b[0]\n"
+ "ldr q6, [x14, #0x40]\n"
+ ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n"
+ ".inst 0x4f83e0f7 // sdot v23.4s, v7.16b, v3.4b[0]\n"
+ ".inst 0x4f84e0fb // sdot v27.4s, v7.16b, v4.4b[0]\n"
+ "ldr q7, [x14, #0x50]\n"
+ ".inst 0x4fa0e0c8 // sdot v8.4s, v6.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e0cc // sdot v12.4s, v6.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e0d0 // sdot v16.4s, v6.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e0d4 // sdot v20.4s, v6.16b, v3.4b[1]\n"
+ ".inst 0x4fa4e0d8 // sdot v24.4s, v6.16b, v4.4b[1]\n"
+ "ldr q6, [x14, #0x60]\n"
+ ".inst 0x4fa0e0e9 // sdot v9.4s, v7.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e0ed // sdot v13.4s, v7.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e0f1 // sdot v17.4s, v7.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e0f5 // sdot v21.4s, v7.16b, v3.4b[1]\n"
+ ".inst 0x4fa4e0f9 // sdot v25.4s, v7.16b, v4.4b[1]\n"
+ "ldr q7, [x14, #0x70]\n"
+ ".inst 0x4fa0e0ca // sdot v10.4s, v6.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e0ce // sdot v14.4s, v6.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e0d2 // sdot v18.4s, v6.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e0d6 // sdot v22.4s, v6.16b, v3.4b[1]\n"
+ ".inst 0x4fa4e0da // sdot v26.4s, v6.16b, v4.4b[1]\n"
+ "ldr q6, [x14, #0x80]\n"
+ ".inst 0x4fa0e0eb // sdot v11.4s, v7.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e0ef // sdot v15.4s, v7.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e0f3 // sdot v19.4s, v7.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e0f7 // sdot v23.4s, v7.16b, v3.4b[1]\n"
+ ".inst 0x4fa4e0fb // sdot v27.4s, v7.16b, v4.4b[1]\n"
+ "ldr q7, [x14, #0x90]\n"
+ ".inst 0x4f80e8c8 // sdot v8.4s, v6.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8cc // sdot v12.4s, v6.16b, v1.4b[2]\n"
+ ".inst 0x4f82e8d0 // sdot v16.4s, v6.16b, v2.4b[2]\n"
+ ".inst 0x4f83e8d4 // sdot v20.4s, v6.16b, v3.4b[2]\n"
+ ".inst 0x4f84e8d8 // sdot v24.4s, v6.16b, v4.4b[2]\n"
+ "ldr q6, [x14, #0xa0]\n"
+ ".inst 0x4f80e8e9 // sdot v9.4s, v7.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8ed // sdot v13.4s, v7.16b, v1.4b[2]\n"
+ ".inst 0x4f82e8f1 // sdot v17.4s, v7.16b, v2.4b[2]\n"
+ ".inst 0x4f83e8f5 // sdot v21.4s, v7.16b, v3.4b[2]\n"
+ ".inst 0x4f84e8f9 // sdot v25.4s, v7.16b, v4.4b[2]\n"
+ "ldr q7, [x14, #0xb0]\n"
+ ".inst 0x4f80e8ca // sdot v10.4s, v6.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8ce // sdot v14.4s, v6.16b, v1.4b[2]\n"
+ ".inst 0x4f82e8d2 // sdot v18.4s, v6.16b, v2.4b[2]\n"
+ ".inst 0x4f83e8d6 // sdot v22.4s, v6.16b, v3.4b[2]\n"
+ ".inst 0x4f84e8da // sdot v26.4s, v6.16b, v4.4b[2]\n"
+ "ldr q6, [x14, #0xc0]\n"
+ ".inst 0x4f80e8eb // sdot v11.4s, v7.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8ef // sdot v15.4s, v7.16b, v1.4b[2]\n"
+ ".inst 0x4f82e8f3 // sdot v19.4s, v7.16b, v2.4b[2]\n"
+ ".inst 0x4f83e8f7 // sdot v23.4s, v7.16b, v3.4b[2]\n"
+ ".inst 0x4f84e8fb // sdot v27.4s, v7.16b, v4.4b[2]\n"
+ "ldr q7, [x14, #0xd0]\n"
+ ".inst 0x4fa0e8c8 // sdot v8.4s, v6.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e8cc // sdot v12.4s, v6.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e8d0 // sdot v16.4s, v6.16b, v2.4b[3]\n"
+ ".inst 0x4fa3e8d4 // sdot v20.4s, v6.16b, v3.4b[3]\n"
+ ".inst 0x4fa4e8d8 // sdot v24.4s, v6.16b, v4.4b[3]\n"
+ "ldr q6, [x14, #0xe0]\n"
+ ".inst 0x4fa0e8e9 // sdot v9.4s, v7.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e8ed // sdot v13.4s, v7.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e8f1 // sdot v17.4s, v7.16b, v2.4b[3]\n"
+ ".inst 0x4fa3e8f5 // sdot v21.4s, v7.16b, v3.4b[3]\n"
+ ".inst 0x4fa4e8f9 // sdot v25.4s, v7.16b, v4.4b[3]\n"
+ "ldr q7, [x14, #0xf0]\n"
+ ".inst 0x4fa0e8ca // sdot v10.4s, v6.16b, v0.4b[3]\n"
+ "add x14, x14, #0x100\n"
+ ".inst 0x4fa1e8ce // sdot v14.4s, v6.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e8d2 // sdot v18.4s, v6.16b, v2.4b[3]\n"
+ ".inst 0x4fa3e8d6 // sdot v22.4s, v6.16b, v3.4b[3]\n"
+ ".inst 0x4fa4e8da // sdot v26.4s, v6.16b, v4.4b[3]\n"
+ ".inst 0x4fa0e8eb // sdot v11.4s, v7.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e8ef // sdot v15.4s, v7.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e8f3 // sdot v19.4s, v7.16b, v2.4b[3]\n"
+ ".inst 0x4fa3e8f7 // sdot v23.4s, v7.16b, v3.4b[3]\n"
+ ".inst 0x4fa4e8fb // sdot v27.4s, v7.16b, v4.4b[3]\n"
+ "160:" // Height 5: Multiply loop: Main loop skip
+ "cbz x11, 165f\n"
+ "cmp x11, #0x4\n"
+ "blt 162f\n"
+ "161:" // Height 5: Multiply loop: Odd block loop
+ "ldr s0, [x10], #0x4\n"
+ "ldr s1, [x28], #0x4\n"
+ "ldr s2, [x26], #0x4\n"
+ "ldr s3, [x24], #0x4\n"
+ "ldr s4, [x22], #0x4\n"
+ "ldr q6, [x14, #0x0]\n"
+ ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n"
+ "ldr q7, [x14, #0x10]\n"
+ ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n"
+ "sub x11, x11, #0x4\n"
+ ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n"
+ "cmp x11, #0x4\n"
+ ".inst 0x4f83e0d4 // sdot v20.4s, v6.16b, v3.4b[0]\n"
+ ".inst 0x4f84e0d8 // sdot v24.4s, v6.16b, v4.4b[0]\n"
+ "ldr q6, [x14, #0x20]\n"
+ ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n"
+ ".inst 0x4f83e0f5 // sdot v21.4s, v7.16b, v3.4b[0]\n"
+ ".inst 0x4f84e0f9 // sdot v25.4s, v7.16b, v4.4b[0]\n"
+ "ldr q7, [x14, #0x30]\n"
+ ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n"
+ "add x14, x14, #0x40\n"
+ ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n"
+ ".inst 0x4f83e0d6 // sdot v22.4s, v6.16b, v3.4b[0]\n"
+ ".inst 0x4f84e0da // sdot v26.4s, v6.16b, v4.4b[0]\n"
+ ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n"
+ ".inst 0x4f83e0f7 // sdot v23.4s, v7.16b, v3.4b[0]\n"
+ ".inst 0x4f84e0fb // sdot v27.4s, v7.16b, v4.4b[0]\n"
+ "bge 161b\n"
+ "cbz x11, 165f\n"
+ "162:" // Height 5: Multiply loop: Skip odd blocks
+ "tbz x11, #1, 163f\n"
+ "ldr h0, [x10], #0x2\n"
+ "ldr h1, [x28], #0x2\n"
+ "ldr h2, [x26], #0x2\n"
+ "ldr h3, [x24], #0x2\n"
+ "ldr h4, [x22], #0x2\n"
+ "tbz x11, #0, 164f\n"
+ "ld1 { v0.b }[2], [x10]\n"
+ "ld1 { v1.b }[2], [x28]\n"
+ "ld1 { v2.b }[2], [x26]\n"
+ "ld1 { v3.b }[2], [x24]\n"
+ "ld1 { v4.b }[2], [x22]\n"
+ "b 164f\n"
+ "163:" // Height 5: Multiply loop: Ragged operand read: partial_1_0
+ "ldr b0, [x10, #0x0]\n"
+ "ldr b1, [x28, #0x0]\n"
+ "ldr b2, [x26, #0x0]\n"
+ "ldr b3, [x24, #0x0]\n"
+ "ldr b4, [x22, #0x0]\n"
+ "164:" // Height 5: Multiply loop: Ragged operand read: Done
+ "ldr q6, [x14, #0x0]\n"
+ ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n"
+ "ldr q7, [x14, #0x10]\n"
+ ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n"
+ ".inst 0x4f83e0d4 // sdot v20.4s, v6.16b, v3.4b[0]\n"
+ ".inst 0x4f84e0d8 // sdot v24.4s, v6.16b, v4.4b[0]\n"
+ "ldr q6, [x14, #0x20]\n"
+ ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n"
+ ".inst 0x4f83e0f5 // sdot v21.4s, v7.16b, v3.4b[0]\n"
+ ".inst 0x4f84e0f9 // sdot v25.4s, v7.16b, v4.4b[0]\n"
+ "ldr q7, [x14, #0x30]\n"
+ ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n"
+ "add x14, x14, #0x40\n"
+ ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n"
+ ".inst 0x4f83e0d6 // sdot v22.4s, v6.16b, v3.4b[0]\n"
+ ".inst 0x4f84e0da // sdot v26.4s, v6.16b, v4.4b[0]\n"
+ ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n"
+ ".inst 0x4f83e0f7 // sdot v23.4s, v7.16b, v3.4b[0]\n"
+ ".inst 0x4f84e0fb // sdot v27.4s, v7.16b, v4.4b[0]\n"
+ "165:" // Height 5: Multiply loop: No odd multiplies
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "add x12, x12, #0x1\n"
+ "cmp x12, x19\n"
+ "bne 155b\n"
+ "prfm pstl1keep, [x13, #0x0]\n"
+ "prfm pstl1keep, [x9, #0x0]\n"
+ "cmp x15, #0x10\n"
+ "prfm pstl1keep, [x27, #0x0]\n"
+ "prfm pstl1keep, [x25, #0x0]\n"
+ "prfm pstl1keep, [x23, #0x0]\n"
+ "bge 174f\n"
+ "tbz x15, #3, 169f\n"
+ "st1 { v8.4s }, [x13], #0x10\n"
+ "st1 { v9.4s }, [x13], #0x10\n"
+ "st1 { v12.4s }, [x9], #0x10\n"
+ "st1 { v13.4s }, [x9], #0x10\n"
+ "st1 { v16.4s }, [x27], #0x10\n"
+ "st1 { v17.4s }, [x27], #0x10\n"
+ "st1 { v20.4s }, [x25], #0x10\n"
+ "st1 { v21.4s }, [x25], #0x10\n"
+ "st1 { v24.4s }, [x23], #0x10\n"
+ "st1 { v25.4s }, [x23], #0x10\n"
+ "tbz x15, #2, 167f\n"
+ "st1 { v10.4s }, [x13], #0x10\n"
+ "st1 { v14.4s }, [x9], #0x10\n"
+ "st1 { v18.4s }, [x27], #0x10\n"
+ "st1 { v22.4s }, [x25], #0x10\n"
+ "st1 { v26.4s }, [x23], #0x10\n"
+ "tbz x15, #1, 166f\n"
+ "str d11, [x13], #0x8\n"
+ "str d15, [x9], #0x8\n"
+ "str d19, [x27], #0x8\n"
+ "str d23, [x25], #0x8\n"
+ "str d27, [x23], #0x8\n"
+ "tbz x15, #0, 173f\n"
+ "st1 { v11.s }[2], [x13]\n"
+ "st1 { v15.s }[2], [x9]\n"
+ "st1 { v19.s }[2], [x27]\n"
+ "st1 { v23.s }[2], [x25]\n"
+ "st1 { v27.s }[2], [x23]\n"
+ "b 173f\n"
+ "166:" // Height 5: Partial direct writeback: partial_1_12
+ "tbz x15, #0, 173f\n"
+ "str s11, [x13, #0x0]\n"
+ "str s15, [x9, #0x0]\n"
+ "str s19, [x27, #0x0]\n"
+ "str s23, [x25, #0x0]\n"
+ "str s27, [x23, #0x0]\n"
+ "b 173f\n"
+ "167:" // Height 5: Partial direct writeback: partial_2_8
+ "tbz x15, #1, 168f\n"
+ "str d10, [x13], #0x8\n"
+ "str d14, [x9], #0x8\n"
+ "str d18, [x27], #0x8\n"
+ "str d22, [x25], #0x8\n"
+ "str d26, [x23], #0x8\n"
+ "tbz x15, #0, 173f\n"
+ "st1 { v10.s }[2], [x13]\n"
+ "st1 { v14.s }[2], [x9]\n"
+ "st1 { v18.s }[2], [x27]\n"
+ "st1 { v22.s }[2], [x25]\n"
+ "st1 { v26.s }[2], [x23]\n"
+ "b 173f\n"
+ "168:" // Height 5: Partial direct writeback: partial_1_8
+ "tbz x15, #0, 173f\n"
+ "str s10, [x13, #0x0]\n"
+ "str s14, [x9, #0x0]\n"
+ "str s18, [x27, #0x0]\n"
+ "str s22, [x25, #0x0]\n"
+ "str s26, [x23, #0x0]\n"
+ "b 173f\n"
+ "169:" // Height 5: Partial direct writeback: partial_4_0
+ "tbz x15, #2, 171f\n"
+ "st1 { v8.4s }, [x13], #0x10\n"
+ "st1 { v12.4s }, [x9], #0x10\n"
+ "st1 { v16.4s }, [x27], #0x10\n"
+ "st1 { v20.4s }, [x25], #0x10\n"
+ "st1 { v24.4s }, [x23], #0x10\n"
+ "tbz x15, #1, 170f\n"
+ "str d9, [x13], #0x8\n"
+ "str d13, [x9], #0x8\n"
+ "str d17, [x27], #0x8\n"
+ "str d21, [x25], #0x8\n"
+ "str d25, [x23], #0x8\n"
+ "tbz x15, #0, 173f\n"
+ "st1 { v9.s }[2], [x13]\n"
+ "st1 { v13.s }[2], [x9]\n"
+ "st1 { v17.s }[2], [x27]\n"
+ "st1 { v21.s }[2], [x25]\n"
+ "st1 { v25.s }[2], [x23]\n"
+ "b 173f\n"
+ "170:" // Height 5: Partial direct writeback: partial_1_4
+ "tbz x15, #0, 173f\n"
+ "str s9, [x13, #0x0]\n"
+ "str s13, [x9, #0x0]\n"
+ "str s17, [x27, #0x0]\n"
+ "str s21, [x25, #0x0]\n"
+ "str s25, [x23, #0x0]\n"
+ "b 173f\n"
+ "171:" // Height 5: Partial direct writeback: partial_2_0
+ "tbz x15, #1, 172f\n"
+ "str d8, [x13], #0x8\n"
+ "str d12, [x9], #0x8\n"
+ "str d16, [x27], #0x8\n"
+ "str d20, [x25], #0x8\n"
+ "str d24, [x23], #0x8\n"
+ "tbz x15, #0, 173f\n"
+ "st1 { v8.s }[2], [x13]\n"
+ "st1 { v12.s }[2], [x9]\n"
+ "st1 { v16.s }[2], [x27]\n"
+ "st1 { v20.s }[2], [x25]\n"
+ "st1 { v24.s }[2], [x23]\n"
+ "b 173f\n"
+ "172:" // Height 5: Partial direct writeback: partial_1_0
+ "str s8, [x13, #0x0]\n"
+ "str s12, [x9, #0x0]\n"
+ "str s16, [x27, #0x0]\n"
+ "str s20, [x25, #0x0]\n"
+ "str s24, [x23, #0x0]\n"
+ "173:" // Height 5: Partial direct writeback: Done
+ "b 175f\n"
+ "174:" // Height 5: Full writeback
+ "str q8, [x13, #0x0]\n"
+ "str q9, [x13, #0x10]\n"
+ "str q10, [x13, #0x20]\n"
+ "str q11, [x13, #0x30]\n"
+ "str q12, [x9, #0x0]\n"
+ "str q13, [x9, #0x10]\n"
+ "str q14, [x9, #0x20]\n"
+ "str q15, [x9, #0x30]\n"
+ "str q16, [x27, #0x0]\n"
+ "str q17, [x27, #0x10]\n"
+ "str q18, [x27, #0x20]\n"
+ "str q19, [x27, #0x30]\n"
+ "str q20, [x25, #0x0]\n"
+ "str q21, [x25, #0x10]\n"
+ "str q22, [x25, #0x20]\n"
+ "str q23, [x25, #0x30]\n"
+ "str q24, [x23, #0x0]\n"
+ "str q25, [x23, #0x10]\n"
+ "str q26, [x23, #0x20]\n"
+ "str q27, [x23, #0x30]\n"
+ "add x13, x13, #0x40\n"
+ "add x9, x9, #0x40\n"
+ "add x27, x27, #0x40\n"
+ "add x25, x25, #0x40\n"
+ "add x23, x23, #0x40\n"
+ "175:" // Height 5: Writeback done
+ "subs x15, x15, #0x10\n"
+ "bgt 143b\n"
+ "b 212f\n"
+ "176:" // Height 6
+ "ldr x15, [%x[args_ptr], %[offsetof_N]]\n"
+ "ldr x14, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 177f\n"
+ "ldr x13, [%x[output_ptr], #0x0]\n"
+ "add x13, x13, x19, LSL #2\n"
+ "ldr x9, [%x[output_ptr], #0x8]\n"
+ "ldr x27, [%x[output_ptr], #0x10]\n"
+ "add x9, x9, x19, LSL #2\n"
+ "ldr x25, [%x[output_ptr], #0x18]\n"
+ "ldr x23, [%x[output_ptr], #0x20]\n"
+ "add x27, x27, x19, LSL #2\n"
+ "ldr x21, [%x[output_ptr], #0x28]\n"
+ "add %x[output_ptr], %x[output_ptr], #0x30\n"
+ "add x25, x25, x19, LSL #2\n"
+ "add x23, x23, x19, LSL #2\n"
+ "add x21, x21, x19, LSL #2\n"
+ "b 178f\n"
+ "177:" // Height 6: setup direct output
+ "mov x13, %x[output_ptr]\n"
+ "add x9, x13, x19, LSL #2\n"
+ "add x27, x9, x19, LSL #2\n"
+ "add x25, x27, x19, LSL #2\n"
+ "add x23, x25, x19, LSL #2\n"
+ "add x21, x23, x19, LSL #2\n"
+ "add %x[output_ptr], x21, x19, LSL #2\n"
+ "178:" // Height 6: Column loop
+ "tbz %x[flags], #0, 188f\n"
+ "cmp x15, #0x10\n"
+ "bge 187f\n"
+ "tbz x15, #3, 182f\n"
+ "ld1 { v8.4s }, [x13], #0x10\n"
+ "ld1 { v12.4s }, [x9], #0x10\n"
+ "ld1 { v16.4s }, [x27], #0x10\n"
+ "ld1 { v20.4s }, [x25], #0x10\n"
+ "ld1 { v24.4s }, [x23], #0x10\n"
+ "ld1 { v28.4s }, [x21], #0x10\n"
+ "ld1 { v9.4s }, [x13], #0x10\n"
+ "ld1 { v13.4s }, [x9], #0x10\n"
+ "ld1 { v17.4s }, [x27], #0x10\n"
+ "ld1 { v21.4s }, [x25], #0x10\n"
+ "ld1 { v25.4s }, [x23], #0x10\n"
+ "ld1 { v29.4s }, [x21], #0x10\n"
+ "tbz x15, #2, 180f\n"
+ "ld1 { v10.4s }, [x13], #0x10\n"
+ "ld1 { v14.4s }, [x9], #0x10\n"
+ "ld1 { v18.4s }, [x27], #0x10\n"
+ "ld1 { v22.4s }, [x25], #0x10\n"
+ "ld1 { v26.4s }, [x23], #0x10\n"
+ "ld1 { v30.4s }, [x21], #0x10\n"
+ "tbz x15, #1, 179f\n"
+ "mov x19, #0x38\n"
+ "ldr d11, [x13], #0x8\n"
+ "ldr d15, [x9], #0x8\n"
+ "ldr d19, [x27], #0x8\n"
+ "ldr d23, [x25], #0x8\n"
+ "ldr d27, [x23], #0x8\n"
+ "ldr d31, [x21], #0x8\n"
+ "tbz x15, #0, 186f\n"
+ "ld1 { v11.s }[2], [x13]\n"
+ "ld1 { v15.s }[2], [x9]\n"
+ "ld1 { v19.s }[2], [x27]\n"
+ "ld1 { v23.s }[2], [x25]\n"
+ "ld1 { v27.s }[2], [x23]\n"
+ "ld1 { v31.s }[2], [x21]\n"
+ "b 186f\n"
+ "179:" // Height 6: Partial accumulate: partial_1_12
+ "mov x19, #0x30\n"
+ "tbz x15, #0, 186f\n"
+ "ldr s11, [x13, #0x0]\n"
+ "ldr s15, [x9, #0x0]\n"
+ "ldr s19, [x27, #0x0]\n"
+ "ldr s23, [x25, #0x0]\n"
+ "ldr s27, [x23, #0x0]\n"
+ "ldr s31, [x21, #0x0]\n"
+ "b 186f\n"
+ "180:" // Height 6: Partial accumulate: partial_2_8
+ "tbz x15, #1, 181f\n"
+ "ldr d10, [x13], #0x8\n"
+ "ldr d14, [x9], #0x8\n"
+ "ldr d18, [x27], #0x8\n"
+ "ldr d22, [x25], #0x8\n"
+ "ldr d26, [x23], #0x8\n"
+ "ldr d30, [x21], #0x8\n"
+ "mov x19, #0x28\n"
+ "tbz x15, #0, 186f\n"
+ "ld1 { v10.s }[2], [x13]\n"
+ "ld1 { v14.s }[2], [x9]\n"
+ "ld1 { v18.s }[2], [x27]\n"
+ "ld1 { v22.s }[2], [x25]\n"
+ "ld1 { v26.s }[2], [x23]\n"
+ "ld1 { v30.s }[2], [x21]\n"
+ "b 186f\n"
+ "181:" // Height 6: Partial accumulate: partial_1_8
+ "mov x19, #0x20\n"
+ "tbz x15, #0, 186f\n"
+ "ldr s10, [x13, #0x0]\n"
+ "ldr s14, [x9, #0x0]\n"
+ "ldr s18, [x27, #0x0]\n"
+ "ldr s22, [x25, #0x0]\n"
+ "ldr s26, [x23, #0x0]\n"
+ "ldr s30, [x21, #0x0]\n"
+ "b 186f\n"
+ "182:" // Height 6: Partial accumulate: partial_4_0
+ "tbz x15, #2, 184f\n"
+ "ld1 { v8.4s }, [x13], #0x10\n"
+ "ld1 { v12.4s }, [x9], #0x10\n"
+ "ld1 { v16.4s }, [x27], #0x10\n"
+ "ld1 { v20.4s }, [x25], #0x10\n"
+ "ld1 { v24.4s }, [x23], #0x10\n"
+ "ld1 { v28.4s }, [x21], #0x10\n"
+ "tbz x15, #1, 183f\n"
+ "mov x19, #0x18\n"
+ "ldr d9, [x13], #0x8\n"
+ "ldr d13, [x9], #0x8\n"
+ "ldr d17, [x27], #0x8\n"
+ "ldr d21, [x25], #0x8\n"
+ "ldr d25, [x23], #0x8\n"
+ "ldr d29, [x21], #0x8\n"
+ "tbz x15, #0, 186f\n"
+ "ld1 { v9.s }[2], [x13]\n"
+ "ld1 { v13.s }[2], [x9]\n"
+ "ld1 { v17.s }[2], [x27]\n"
+ "ld1 { v21.s }[2], [x25]\n"
+ "ld1 { v25.s }[2], [x23]\n"
+ "ld1 { v29.s }[2], [x21]\n"
+ "b 186f\n"
+ "183:" // Height 6: Partial accumulate: partial_1_4
+ "mov x19, #0x10\n"
+ "tbz x15, #0, 186f\n"
+ "ldr s9, [x13, #0x0]\n"
+ "ldr s13, [x9, #0x0]\n"
+ "ldr s17, [x27, #0x0]\n"
+ "ldr s21, [x25, #0x0]\n"
+ "ldr s25, [x23, #0x0]\n"
+ "ldr s29, [x21, #0x0]\n"
+ "b 186f\n"
+ "184:" // Height 6: Partial accumulate: partial_2_0
+ "tbz x15, #1, 185f\n"
+ "ldr d8, [x13], #0x8\n"
+ "ldr d12, [x9], #0x8\n"
+ "ldr d16, [x27], #0x8\n"
+ "ldr d20, [x25], #0x8\n"
+ "ldr d24, [x23], #0x8\n"
+ "ldr d28, [x21], #0x8\n"
+ "mov x19, #0x8\n"
+ "tbz x15, #0, 186f\n"
+ "ld1 { v8.s }[2], [x13]\n"
+ "ld1 { v12.s }[2], [x9]\n"
+ "ld1 { v16.s }[2], [x27]\n"
+ "ld1 { v20.s }[2], [x25]\n"
+ "ld1 { v24.s }[2], [x23]\n"
+ "ld1 { v28.s }[2], [x21]\n"
+ "b 186f\n"
+ "185:" // Height 6: Partial accumulate: partial_1_0
+ "mov x19, #0x0\n"
+ "ldr s8, [x13, #0x0]\n"
+ "ldr s12, [x9, #0x0]\n"
+ "ldr s16, [x27, #0x0]\n"
+ "ldr s20, [x25, #0x0]\n"
+ "ldr s24, [x23, #0x0]\n"
+ "ldr s28, [x21, #0x0]\n"
+ "186:" // Height 6: Partial accumulate: Done
+ "sub x13, x13, x19\n"
+ "sub x9, x9, x19\n"
+ "sub x27, x27, x19\n"
+ "sub x25, x25, x19\n"
+ "sub x23, x23, x19\n"
+ "sub x21, x21, x19\n"
+ "b 189f\n"
+ "187:" // Height 6: full accumulate
+ "ldr q8, [x13, #0x0]\n"
+ "ldr q9, [x13, #0x10]\n"
+ "ldr q10, [x13, #0x20]\n"
+ "ldr q11, [x13, #0x30]\n"
+ "ldr q12, [x9, #0x0]\n"
+ "ldr q13, [x9, #0x10]\n"
+ "ldr q14, [x9, #0x20]\n"
+ "ldr q15, [x9, #0x30]\n"
+ "ldr q16, [x27, #0x0]\n"
+ "ldr q17, [x27, #0x10]\n"
+ "ldr q18, [x27, #0x20]\n"
+ "ldr q19, [x27, #0x30]\n"
+ "ldr q20, [x25, #0x0]\n"
+ "ldr q21, [x25, #0x10]\n"
+ "ldr q22, [x25, #0x20]\n"
+ "ldr q23, [x25, #0x30]\n"
+ "ldr q24, [x23, #0x0]\n"
+ "ldr q25, [x23, #0x10]\n"
+ "ldr q26, [x23, #0x20]\n"
+ "ldr q27, [x23, #0x30]\n"
+ "ldr q28, [x21, #0x0]\n"
+ "ldr q29, [x21, #0x10]\n"
+ "ldr q30, [x21, #0x20]\n"
+ "ldr q31, [x21, #0x30]\n"
+ "b 189f\n"
+ "188:" // Height 6: no accumulate
+ "movi v8.4s, #0x0\n"
+ "movi v9.4s, #0x0\n"
+ "movi v10.4s, #0x0\n"
+ "movi v11.4s, #0x0\n"
+ "movi v12.4s, #0x0\n"
+ "movi v13.4s, #0x0\n"
+ "movi v14.4s, #0x0\n"
+ "movi v15.4s, #0x0\n"
+ "movi v16.4s, #0x0\n"
+ "movi v17.4s, #0x0\n"
+ "movi v18.4s, #0x0\n"
+ "movi v19.4s, #0x0\n"
+ "movi v20.4s, #0x0\n"
+ "movi v21.4s, #0x0\n"
+ "movi v22.4s, #0x0\n"
+ "movi v23.4s, #0x0\n"
+ "movi v24.4s, #0x0\n"
+ "movi v25.4s, #0x0\n"
+ "movi v26.4s, #0x0\n"
+ "movi v27.4s, #0x0\n"
+ "movi v28.4s, #0x0\n"
+ "movi v29.4s, #0x0\n"
+ "movi v30.4s, #0x0\n"
+ "movi v31.4s, #0x0\n"
+ "189:" // Height 6: setup done
+ "mov x12, #0x0\n"
+ "190:" // Height 6: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w11, [x20, x12, LSL #0x2]\n"
+ "tbz %x[flags], #3, 191f\n"
+ "ldr x20, [%x[input_ptr], x12, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x10, [x20, #0x0]\n"
+ "ldr x28, [x20, #0x8]\n"
+ "ldr x26, [x20, #0x10]\n"
+ "ldr x24, [x20, #0x18]\n"
+ "ldr x22, [x20, #0x20]\n"
+ "ldr x20, [x20, #0x28]\n"
+ "cbnz x12, 192f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x10, x10, x19\n"
+ "add x28, x28, x19\n"
+ "add x26, x26, x19\n"
+ "add x24, x24, x19\n"
+ "add x22, x22, x19\n"
+ "add x20, x20, x19\n"
+ "b 192f\n"
+ "191:" // Height 6: setup direct input
+ "mov x10, %x[input_ptr]\n"
+ "add x28, x10, x19\n"
+ "add x26, x28, x19\n"
+ "add x24, x26, x19\n"
+ "add x22, x24, x19\n"
+ "add x20, x22, x19\n"
+ "192:" // Height 6: input setup done
+ "cmp x11, #0x10\n"
+ "blt 195f\n"
+ "cmp x11, #0x20\n"
+ "blt 194f\n"
+ "193:" // Height 6: Multiply loop: Main loop head
+ "ldr q0, [x10, #0x0]\n"
+ "ldr q1, [x28, #0x0]\n"
+ "ldr q2, [x26, #0x0]\n"
+ "ldr q3, [x24, #0x0]\n"
+ "ldr q4, [x22, #0x0]\n"
+ "ldr q5, [x20, #0x0]\n"
+ "ldr q6, [x14, #0x0]\n"
+ ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n"
+ "ldr q7, [x14, #0x10]\n"
+ ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n"
+ "add x10, x10, #0x10\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n"
+ "add x28, x28, #0x10\n"
+ ".inst 0x4f83e0d4 // sdot v20.4s, v6.16b, v3.4b[0]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "add x26, x26, #0x10\n"
+ ".inst 0x4f84e0d8 // sdot v24.4s, v6.16b, v4.4b[0]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "add x24, x24, #0x10\n"
+ ".inst 0x4f85e0dc // sdot v28.4s, v6.16b, v5.4b[0]\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "ldr q6, [x14, #0x20]\n"
+ ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n"
+ "add x22, x22, #0x10\n"
+ "prfm pldl1keep, [x22, #0x80]\n"
+ ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n"
+ "add x20, x20, #0x10\n"
+ ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n"
+ "prfm pldl1keep, [x20, #0x80]\n"
+ "sub x11, x11, #0x10\n"
+ ".inst 0x4f83e0f5 // sdot v21.4s, v7.16b, v3.4b[0]\n"
+ "cmp x11, #0x20\n"
+ ".inst 0x4f84e0f9 // sdot v25.4s, v7.16b, v4.4b[0]\n"
+ ".inst 0x4f85e0fd // sdot v29.4s, v7.16b, v5.4b[0]\n"
+ "ldr q7, [x14, #0x30]\n"
+ ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n"
+ ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n"
+ ".inst 0x4f83e0d6 // sdot v22.4s, v6.16b, v3.4b[0]\n"
+ ".inst 0x4f84e0da // sdot v26.4s, v6.16b, v4.4b[0]\n"
+ ".inst 0x4f85e0de // sdot v30.4s, v6.16b, v5.4b[0]\n"
+ "ldr q6, [x14, #0x40]\n"
+ ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n"
+ ".inst 0x4f83e0f7 // sdot v23.4s, v7.16b, v3.4b[0]\n"
+ ".inst 0x4f84e0fb // sdot v27.4s, v7.16b, v4.4b[0]\n"
+ ".inst 0x4f85e0ff // sdot v31.4s, v7.16b, v5.4b[0]\n"
+ "ldr q7, [x14, #0x50]\n"
+ ".inst 0x4fa0e0c8 // sdot v8.4s, v6.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e0cc // sdot v12.4s, v6.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e0d0 // sdot v16.4s, v6.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e0d4 // sdot v20.4s, v6.16b, v3.4b[1]\n"
+ ".inst 0x4fa4e0d8 // sdot v24.4s, v6.16b, v4.4b[1]\n"
+ ".inst 0x4fa5e0dc // sdot v28.4s, v6.16b, v5.4b[1]\n"
+ "ldr q6, [x14, #0x60]\n"
+ ".inst 0x4fa0e0e9 // sdot v9.4s, v7.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e0ed // sdot v13.4s, v7.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e0f1 // sdot v17.4s, v7.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e0f5 // sdot v21.4s, v7.16b, v3.4b[1]\n"
+ ".inst 0x4fa4e0f9 // sdot v25.4s, v7.16b, v4.4b[1]\n"
+ ".inst 0x4fa5e0fd // sdot v29.4s, v7.16b, v5.4b[1]\n"
+ "ldr q7, [x14, #0x70]\n"
+ ".inst 0x4fa0e0ca // sdot v10.4s, v6.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e0ce // sdot v14.4s, v6.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e0d2 // sdot v18.4s, v6.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e0d6 // sdot v22.4s, v6.16b, v3.4b[1]\n"
+ ".inst 0x4fa4e0da // sdot v26.4s, v6.16b, v4.4b[1]\n"
+ ".inst 0x4fa5e0de // sdot v30.4s, v6.16b, v5.4b[1]\n"
+ "ldr q6, [x14, #0x80]\n"
+ ".inst 0x4fa0e0eb // sdot v11.4s, v7.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e0ef // sdot v15.4s, v7.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e0f3 // sdot v19.4s, v7.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e0f7 // sdot v23.4s, v7.16b, v3.4b[1]\n"
+ ".inst 0x4fa4e0fb // sdot v27.4s, v7.16b, v4.4b[1]\n"
+ ".inst 0x4fa5e0ff // sdot v31.4s, v7.16b, v5.4b[1]\n"
+ "ldr q7, [x14, #0x90]\n"
+ ".inst 0x4f80e8c8 // sdot v8.4s, v6.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8cc // sdot v12.4s, v6.16b, v1.4b[2]\n"
+ ".inst 0x4f82e8d0 // sdot v16.4s, v6.16b, v2.4b[2]\n"
+ ".inst 0x4f83e8d4 // sdot v20.4s, v6.16b, v3.4b[2]\n"
+ ".inst 0x4f84e8d8 // sdot v24.4s, v6.16b, v4.4b[2]\n"
+ ".inst 0x4f85e8dc // sdot v28.4s, v6.16b, v5.4b[2]\n"
+ "ldr q6, [x14, #0xa0]\n"
+ ".inst 0x4f80e8e9 // sdot v9.4s, v7.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8ed // sdot v13.4s, v7.16b, v1.4b[2]\n"
+ ".inst 0x4f82e8f1 // sdot v17.4s, v7.16b, v2.4b[2]\n"
+ ".inst 0x4f83e8f5 // sdot v21.4s, v7.16b, v3.4b[2]\n"
+ ".inst 0x4f84e8f9 // sdot v25.4s, v7.16b, v4.4b[2]\n"
+ ".inst 0x4f85e8fd // sdot v29.4s, v7.16b, v5.4b[2]\n"
+ "ldr q7, [x14, #0xb0]\n"
+ ".inst 0x4f80e8ca // sdot v10.4s, v6.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8ce // sdot v14.4s, v6.16b, v1.4b[2]\n"
+ ".inst 0x4f82e8d2 // sdot v18.4s, v6.16b, v2.4b[2]\n"
+ ".inst 0x4f83e8d6 // sdot v22.4s, v6.16b, v3.4b[2]\n"
+ ".inst 0x4f84e8da // sdot v26.4s, v6.16b, v4.4b[2]\n"
+ ".inst 0x4f85e8de // sdot v30.4s, v6.16b, v5.4b[2]\n"
+ "ldr q6, [x14, #0xc0]\n"
+ ".inst 0x4f80e8eb // sdot v11.4s, v7.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8ef // sdot v15.4s, v7.16b, v1.4b[2]\n"
+ ".inst 0x4f82e8f3 // sdot v19.4s, v7.16b, v2.4b[2]\n"
+ ".inst 0x4f83e8f7 // sdot v23.4s, v7.16b, v3.4b[2]\n"
+ ".inst 0x4f84e8fb // sdot v27.4s, v7.16b, v4.4b[2]\n"
+ ".inst 0x4f85e8ff // sdot v31.4s, v7.16b, v5.4b[2]\n"
+ "ldr q7, [x14, #0xd0]\n"
+ ".inst 0x4fa0e8c8 // sdot v8.4s, v6.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e8cc // sdot v12.4s, v6.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e8d0 // sdot v16.4s, v6.16b, v2.4b[3]\n"
+ ".inst 0x4fa3e8d4 // sdot v20.4s, v6.16b, v3.4b[3]\n"
+ ".inst 0x4fa4e8d8 // sdot v24.4s, v6.16b, v4.4b[3]\n"
+ ".inst 0x4fa5e8dc // sdot v28.4s, v6.16b, v5.4b[3]\n"
+ "ldr q6, [x14, #0xe0]\n"
+ ".inst 0x4fa0e8e9 // sdot v9.4s, v7.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e8ed // sdot v13.4s, v7.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e8f1 // sdot v17.4s, v7.16b, v2.4b[3]\n"
+ ".inst 0x4fa3e8f5 // sdot v21.4s, v7.16b, v3.4b[3]\n"
+ ".inst 0x4fa4e8f9 // sdot v25.4s, v7.16b, v4.4b[3]\n"
+ ".inst 0x4fa5e8fd // sdot v29.4s, v7.16b, v5.4b[3]\n"
+ "ldr q7, [x14, #0xf0]\n"
+ ".inst 0x4fa0e8ca // sdot v10.4s, v6.16b, v0.4b[3]\n"
+ "add x14, x14, #0x100\n"
+ ".inst 0x4fa1e8ce // sdot v14.4s, v6.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e8d2 // sdot v18.4s, v6.16b, v2.4b[3]\n"
+ ".inst 0x4fa3e8d6 // sdot v22.4s, v6.16b, v3.4b[3]\n"
+ ".inst 0x4fa4e8da // sdot v26.4s, v6.16b, v4.4b[3]\n"
+ ".inst 0x4fa5e8de // sdot v30.4s, v6.16b, v5.4b[3]\n"
+ ".inst 0x4fa0e8eb // sdot v11.4s, v7.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e8ef // sdot v15.4s, v7.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e8f3 // sdot v19.4s, v7.16b, v2.4b[3]\n"
+ ".inst 0x4fa3e8f7 // sdot v23.4s, v7.16b, v3.4b[3]\n"
+ ".inst 0x4fa4e8fb // sdot v27.4s, v7.16b, v4.4b[3]\n"
+ ".inst 0x4fa5e8ff // sdot v31.4s, v7.16b, v5.4b[3]\n"
+ "bge 193b\n"
+ "194:" // Height 6: Multiply loop: Single iteration only
+ "sub x11, x11, #0x10\n"
+ "ldr q0, [x10, #0x0]\n"
+ "ldr q1, [x28, #0x0]\n"
+ "ldr q2, [x26, #0x0]\n"
+ "ldr q3, [x24, #0x0]\n"
+ "ldr q4, [x22, #0x0]\n"
+ "ldr q5, [x20, #0x0]\n"
+ "ldr q6, [x14, #0x0]\n"
+ ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n"
+ "ldr q7, [x14, #0x10]\n"
+ ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n"
+ "add x10, x10, #0x10\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n"
+ "add x28, x28, #0x10\n"
+ ".inst 0x4f83e0d4 // sdot v20.4s, v6.16b, v3.4b[0]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "add x26, x26, #0x10\n"
+ ".inst 0x4f84e0d8 // sdot v24.4s, v6.16b, v4.4b[0]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "add x24, x24, #0x10\n"
+ ".inst 0x4f85e0dc // sdot v28.4s, v6.16b, v5.4b[0]\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "ldr q6, [x14, #0x20]\n"
+ ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n"
+ "add x22, x22, #0x10\n"
+ "prfm pldl1keep, [x22, #0x80]\n"
+ ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n"
+ "add x20, x20, #0x10\n"
+ ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n"
+ "prfm pldl1keep, [x20, #0x80]\n"
+ ".inst 0x4f83e0f5 // sdot v21.4s, v7.16b, v3.4b[0]\n"
+ ".inst 0x4f84e0f9 // sdot v25.4s, v7.16b, v4.4b[0]\n"
+ ".inst 0x4f85e0fd // sdot v29.4s, v7.16b, v5.4b[0]\n"
+ "ldr q7, [x14, #0x30]\n"
+ ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n"
+ ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n"
+ ".inst 0x4f83e0d6 // sdot v22.4s, v6.16b, v3.4b[0]\n"
+ ".inst 0x4f84e0da // sdot v26.4s, v6.16b, v4.4b[0]\n"
+ ".inst 0x4f85e0de // sdot v30.4s, v6.16b, v5.4b[0]\n"
+ "ldr q6, [x14, #0x40]\n"
+ ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n"
+ ".inst 0x4f83e0f7 // sdot v23.4s, v7.16b, v3.4b[0]\n"
+ ".inst 0x4f84e0fb // sdot v27.4s, v7.16b, v4.4b[0]\n"
+ ".inst 0x4f85e0ff // sdot v31.4s, v7.16b, v5.4b[0]\n"
+ "ldr q7, [x14, #0x50]\n"
+ ".inst 0x4fa0e0c8 // sdot v8.4s, v6.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e0cc // sdot v12.4s, v6.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e0d0 // sdot v16.4s, v6.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e0d4 // sdot v20.4s, v6.16b, v3.4b[1]\n"
+ ".inst 0x4fa4e0d8 // sdot v24.4s, v6.16b, v4.4b[1]\n"
+ ".inst 0x4fa5e0dc // sdot v28.4s, v6.16b, v5.4b[1]\n"
+ "ldr q6, [x14, #0x60]\n"
+ ".inst 0x4fa0e0e9 // sdot v9.4s, v7.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e0ed // sdot v13.4s, v7.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e0f1 // sdot v17.4s, v7.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e0f5 // sdot v21.4s, v7.16b, v3.4b[1]\n"
+ ".inst 0x4fa4e0f9 // sdot v25.4s, v7.16b, v4.4b[1]\n"
+ ".inst 0x4fa5e0fd // sdot v29.4s, v7.16b, v5.4b[1]\n"
+ "ldr q7, [x14, #0x70]\n"
+ ".inst 0x4fa0e0ca // sdot v10.4s, v6.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e0ce // sdot v14.4s, v6.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e0d2 // sdot v18.4s, v6.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e0d6 // sdot v22.4s, v6.16b, v3.4b[1]\n"
+ ".inst 0x4fa4e0da // sdot v26.4s, v6.16b, v4.4b[1]\n"
+ ".inst 0x4fa5e0de // sdot v30.4s, v6.16b, v5.4b[1]\n"
+ "ldr q6, [x14, #0x80]\n"
+ ".inst 0x4fa0e0eb // sdot v11.4s, v7.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e0ef // sdot v15.4s, v7.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e0f3 // sdot v19.4s, v7.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e0f7 // sdot v23.4s, v7.16b, v3.4b[1]\n"
+ ".inst 0x4fa4e0fb // sdot v27.4s, v7.16b, v4.4b[1]\n"
+ ".inst 0x4fa5e0ff // sdot v31.4s, v7.16b, v5.4b[1]\n"
+ "ldr q7, [x14, #0x90]\n"
+ ".inst 0x4f80e8c8 // sdot v8.4s, v6.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8cc // sdot v12.4s, v6.16b, v1.4b[2]\n"
+ ".inst 0x4f82e8d0 // sdot v16.4s, v6.16b, v2.4b[2]\n"
+ ".inst 0x4f83e8d4 // sdot v20.4s, v6.16b, v3.4b[2]\n"
+ ".inst 0x4f84e8d8 // sdot v24.4s, v6.16b, v4.4b[2]\n"
+ ".inst 0x4f85e8dc // sdot v28.4s, v6.16b, v5.4b[2]\n"
+ "ldr q6, [x14, #0xa0]\n"
+ ".inst 0x4f80e8e9 // sdot v9.4s, v7.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8ed // sdot v13.4s, v7.16b, v1.4b[2]\n"
+ ".inst 0x4f82e8f1 // sdot v17.4s, v7.16b, v2.4b[2]\n"
+ ".inst 0x4f83e8f5 // sdot v21.4s, v7.16b, v3.4b[2]\n"
+ ".inst 0x4f84e8f9 // sdot v25.4s, v7.16b, v4.4b[2]\n"
+ ".inst 0x4f85e8fd // sdot v29.4s, v7.16b, v5.4b[2]\n"
+ "ldr q7, [x14, #0xb0]\n"
+ ".inst 0x4f80e8ca // sdot v10.4s, v6.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8ce // sdot v14.4s, v6.16b, v1.4b[2]\n"
+ ".inst 0x4f82e8d2 // sdot v18.4s, v6.16b, v2.4b[2]\n"
+ ".inst 0x4f83e8d6 // sdot v22.4s, v6.16b, v3.4b[2]\n"
+ ".inst 0x4f84e8da // sdot v26.4s, v6.16b, v4.4b[2]\n"
+ ".inst 0x4f85e8de // sdot v30.4s, v6.16b, v5.4b[2]\n"
+ "ldr q6, [x14, #0xc0]\n"
+ ".inst 0x4f80e8eb // sdot v11.4s, v7.16b, v0.4b[2]\n"
+ ".inst 0x4f81e8ef // sdot v15.4s, v7.16b, v1.4b[2]\n"
+ ".inst 0x4f82e8f3 // sdot v19.4s, v7.16b, v2.4b[2]\n"
+ ".inst 0x4f83e8f7 // sdot v23.4s, v7.16b, v3.4b[2]\n"
+ ".inst 0x4f84e8fb // sdot v27.4s, v7.16b, v4.4b[2]\n"
+ ".inst 0x4f85e8ff // sdot v31.4s, v7.16b, v5.4b[2]\n"
+ "ldr q7, [x14, #0xd0]\n"
+ ".inst 0x4fa0e8c8 // sdot v8.4s, v6.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e8cc // sdot v12.4s, v6.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e8d0 // sdot v16.4s, v6.16b, v2.4b[3]\n"
+ ".inst 0x4fa3e8d4 // sdot v20.4s, v6.16b, v3.4b[3]\n"
+ ".inst 0x4fa4e8d8 // sdot v24.4s, v6.16b, v4.4b[3]\n"
+ ".inst 0x4fa5e8dc // sdot v28.4s, v6.16b, v5.4b[3]\n"
+ "ldr q6, [x14, #0xe0]\n"
+ ".inst 0x4fa0e8e9 // sdot v9.4s, v7.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e8ed // sdot v13.4s, v7.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e8f1 // sdot v17.4s, v7.16b, v2.4b[3]\n"
+ ".inst 0x4fa3e8f5 // sdot v21.4s, v7.16b, v3.4b[3]\n"
+ ".inst 0x4fa4e8f9 // sdot v25.4s, v7.16b, v4.4b[3]\n"
+ ".inst 0x4fa5e8fd // sdot v29.4s, v7.16b, v5.4b[3]\n"
+ "ldr q7, [x14, #0xf0]\n"
+ ".inst 0x4fa0e8ca // sdot v10.4s, v6.16b, v0.4b[3]\n"
+ "add x14, x14, #0x100\n"
+ ".inst 0x4fa1e8ce // sdot v14.4s, v6.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e8d2 // sdot v18.4s, v6.16b, v2.4b[3]\n"
+ ".inst 0x4fa3e8d6 // sdot v22.4s, v6.16b, v3.4b[3]\n"
+ ".inst 0x4fa4e8da // sdot v26.4s, v6.16b, v4.4b[3]\n"
+ ".inst 0x4fa5e8de // sdot v30.4s, v6.16b, v5.4b[3]\n"
+ ".inst 0x4fa0e8eb // sdot v11.4s, v7.16b, v0.4b[3]\n"
+ ".inst 0x4fa1e8ef // sdot v15.4s, v7.16b, v1.4b[3]\n"
+ ".inst 0x4fa2e8f3 // sdot v19.4s, v7.16b, v2.4b[3]\n"
+ ".inst 0x4fa3e8f7 // sdot v23.4s, v7.16b, v3.4b[3]\n"
+ ".inst 0x4fa4e8fb // sdot v27.4s, v7.16b, v4.4b[3]\n"
+ ".inst 0x4fa5e8ff // sdot v31.4s, v7.16b, v5.4b[3]\n"
+ "195:" // Height 6: Multiply loop: Main loop skip
+ "cbz x11, 200f\n"
+ "cmp x11, #0x4\n"
+ "blt 197f\n"
+ "196:" // Height 6: Multiply loop: Odd block loop
+ "ldr s0, [x10], #0x4\n"
+ "ldr s1, [x28], #0x4\n"
+ "ldr s2, [x26], #0x4\n"
+ "ldr s3, [x24], #0x4\n"
+ "ldr s4, [x22], #0x4\n"
+ "ldr s5, [x20], #0x4\n"
+ "ldr q6, [x14, #0x0]\n"
+ ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n"
+ "ldr q7, [x14, #0x10]\n"
+ ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n"
+ "sub x11, x11, #0x4\n"
+ ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n"
+ "cmp x11, #0x4\n"
+ ".inst 0x4f83e0d4 // sdot v20.4s, v6.16b, v3.4b[0]\n"
+ ".inst 0x4f84e0d8 // sdot v24.4s, v6.16b, v4.4b[0]\n"
+ ".inst 0x4f85e0dc // sdot v28.4s, v6.16b, v5.4b[0]\n"
+ "ldr q6, [x14, #0x20]\n"
+ ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n"
+ ".inst 0x4f83e0f5 // sdot v21.4s, v7.16b, v3.4b[0]\n"
+ ".inst 0x4f84e0f9 // sdot v25.4s, v7.16b, v4.4b[0]\n"
+ ".inst 0x4f85e0fd // sdot v29.4s, v7.16b, v5.4b[0]\n"
+ "ldr q7, [x14, #0x30]\n"
+ ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n"
+ "add x14, x14, #0x40\n"
+ ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n"
+ ".inst 0x4f83e0d6 // sdot v22.4s, v6.16b, v3.4b[0]\n"
+ ".inst 0x4f84e0da // sdot v26.4s, v6.16b, v4.4b[0]\n"
+ ".inst 0x4f85e0de // sdot v30.4s, v6.16b, v5.4b[0]\n"
+ ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n"
+ ".inst 0x4f83e0f7 // sdot v23.4s, v7.16b, v3.4b[0]\n"
+ ".inst 0x4f84e0fb // sdot v27.4s, v7.16b, v4.4b[0]\n"
+ ".inst 0x4f85e0ff // sdot v31.4s, v7.16b, v5.4b[0]\n"
+ "bge 196b\n"
+ "cbz x11, 200f\n"
+ "197:" // Height 6: Multiply loop: Skip odd blocks
+ "tbz x11, #1, 198f\n"
+ "ldr h0, [x10], #0x2\n"
+ "ldr h1, [x28], #0x2\n"
+ "ldr h2, [x26], #0x2\n"
+ "ldr h3, [x24], #0x2\n"
+ "ldr h4, [x22], #0x2\n"
+ "ldr h5, [x20], #0x2\n"
+ "tbz x11, #0, 199f\n"
+ "ld1 { v0.b }[2], [x10]\n"
+ "ld1 { v1.b }[2], [x28]\n"
+ "ld1 { v2.b }[2], [x26]\n"
+ "ld1 { v3.b }[2], [x24]\n"
+ "ld1 { v4.b }[2], [x22]\n"
+ "ld1 { v5.b }[2], [x20]\n"
+ "b 199f\n"
+ "198:" // Height 6: Multiply loop: Ragged operand read: partial_1_0
+ "ldr b0, [x10, #0x0]\n"
+ "ldr b1, [x28, #0x0]\n"
+ "ldr b2, [x26, #0x0]\n"
+ "ldr b3, [x24, #0x0]\n"
+ "ldr b4, [x22, #0x0]\n"
+ "ldr b5, [x20, #0x0]\n"
+ "199:" // Height 6: Multiply loop: Ragged operand read: Done
+ "ldr q6, [x14, #0x0]\n"
+ ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n"
+ "ldr q7, [x14, #0x10]\n"
+ ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n"
+ ".inst 0x4f83e0d4 // sdot v20.4s, v6.16b, v3.4b[0]\n"
+ ".inst 0x4f84e0d8 // sdot v24.4s, v6.16b, v4.4b[0]\n"
+ ".inst 0x4f85e0dc // sdot v28.4s, v6.16b, v5.4b[0]\n"
+ "ldr q6, [x14, #0x20]\n"
+ ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n"
+ ".inst 0x4f83e0f5 // sdot v21.4s, v7.16b, v3.4b[0]\n"
+ ".inst 0x4f84e0f9 // sdot v25.4s, v7.16b, v4.4b[0]\n"
+ ".inst 0x4f85e0fd // sdot v29.4s, v7.16b, v5.4b[0]\n"
+ "ldr q7, [x14, #0x30]\n"
+ ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n"
+ "add x14, x14, #0x40\n"
+ ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n"
+ ".inst 0x4f83e0d6 // sdot v22.4s, v6.16b, v3.4b[0]\n"
+ ".inst 0x4f84e0da // sdot v26.4s, v6.16b, v4.4b[0]\n"
+ ".inst 0x4f85e0de // sdot v30.4s, v6.16b, v5.4b[0]\n"
+ ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n"
+ ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n"
+ ".inst 0x4f83e0f7 // sdot v23.4s, v7.16b, v3.4b[0]\n"
+ ".inst 0x4f84e0fb // sdot v27.4s, v7.16b, v4.4b[0]\n"
+ ".inst 0x4f85e0ff // sdot v31.4s, v7.16b, v5.4b[0]\n"
+ "200:" // Height 6: Multiply loop: No odd multiplies
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "add x12, x12, #0x1\n"
+ "cmp x12, x19\n"
+ "bne 190b\n"
+ "prfm pstl1keep, [x13, #0x0]\n"
+ "prfm pstl1keep, [x9, #0x0]\n"
+ "cmp x15, #0x10\n"
+ "prfm pstl1keep, [x27, #0x0]\n"
+ "prfm pstl1keep, [x25, #0x0]\n"
+ "prfm pstl1keep, [x23, #0x0]\n"
+ "prfm pstl1keep, [x21, #0x0]\n"
+ "bge 209f\n"
+ "tbz x15, #3, 204f\n"
+ "st1 { v8.4s }, [x13], #0x10\n"
+ "st1 { v9.4s }, [x13], #0x10\n"
+ "st1 { v12.4s }, [x9], #0x10\n"
+ "st1 { v13.4s }, [x9], #0x10\n"
+ "st1 { v16.4s }, [x27], #0x10\n"
+ "st1 { v17.4s }, [x27], #0x10\n"
+ "st1 { v20.4s }, [x25], #0x10\n"
+ "st1 { v21.4s }, [x25], #0x10\n"
+ "st1 { v24.4s }, [x23], #0x10\n"
+ "st1 { v25.4s }, [x23], #0x10\n"
+ "st1 { v28.4s }, [x21], #0x10\n"
+ "st1 { v29.4s }, [x21], #0x10\n"
+ "tbz x15, #2, 202f\n"
+ "st1 { v10.4s }, [x13], #0x10\n"
+ "st1 { v14.4s }, [x9], #0x10\n"
+ "st1 { v18.4s }, [x27], #0x10\n"
+ "st1 { v22.4s }, [x25], #0x10\n"
+ "st1 { v26.4s }, [x23], #0x10\n"
+ "st1 { v30.4s }, [x21], #0x10\n"
+ "tbz x15, #1, 201f\n"
+ "str d11, [x13], #0x8\n"
+ "str d15, [x9], #0x8\n"
+ "str d19, [x27], #0x8\n"
+ "str d23, [x25], #0x8\n"
+ "str d27, [x23], #0x8\n"
+ "str d31, [x21], #0x8\n"
+ "tbz x15, #0, 208f\n"
+ "st1 { v11.s }[2], [x13]\n"
+ "st1 { v15.s }[2], [x9]\n"
+ "st1 { v19.s }[2], [x27]\n"
+ "st1 { v23.s }[2], [x25]\n"
+ "st1 { v27.s }[2], [x23]\n"
+ "st1 { v31.s }[2], [x21]\n"
+ "b 208f\n"
+ "201:" // Height 6: Partial direct writeback: partial_1_12
+ "tbz x15, #0, 208f\n"
+ "str s11, [x13, #0x0]\n"
+ "str s15, [x9, #0x0]\n"
+ "str s19, [x27, #0x0]\n"
+ "str s23, [x25, #0x0]\n"
+ "str s27, [x23, #0x0]\n"
+ "str s31, [x21, #0x0]\n"
+ "b 208f\n"
+ "202:" // Height 6: Partial direct writeback: partial_2_8
+ "tbz x15, #1, 203f\n"
+ "str d10, [x13], #0x8\n"
+ "str d14, [x9], #0x8\n"
+ "str d18, [x27], #0x8\n"
+ "str d22, [x25], #0x8\n"
+ "str d26, [x23], #0x8\n"
+ "str d30, [x21], #0x8\n"
+ "tbz x15, #0, 208f\n"
+ "st1 { v10.s }[2], [x13]\n"
+ "st1 { v14.s }[2], [x9]\n"
+ "st1 { v18.s }[2], [x27]\n"
+ "st1 { v22.s }[2], [x25]\n"
+ "st1 { v26.s }[2], [x23]\n"
+ "st1 { v30.s }[2], [x21]\n"
+ "b 208f\n"
+ "203:" // Height 6: Partial direct writeback: partial_1_8
+ "tbz x15, #0, 208f\n"
+ "str s10, [x13, #0x0]\n"
+ "str s14, [x9, #0x0]\n"
+ "str s18, [x27, #0x0]\n"
+ "str s22, [x25, #0x0]\n"
+ "str s26, [x23, #0x0]\n"
+ "str s30, [x21, #0x0]\n"
+ "b 208f\n"
+ "204:" // Height 6: Partial direct writeback: partial_4_0
+ "tbz x15, #2, 206f\n"
+ "st1 { v8.4s }, [x13], #0x10\n"
+ "st1 { v12.4s }, [x9], #0x10\n"
+ "st1 { v16.4s }, [x27], #0x10\n"
+ "st1 { v20.4s }, [x25], #0x10\n"
+ "st1 { v24.4s }, [x23], #0x10\n"
+ "st1 { v28.4s }, [x21], #0x10\n"
+ "tbz x15, #1, 205f\n"
+ "str d9, [x13], #0x8\n"
+ "str d13, [x9], #0x8\n"
+ "str d17, [x27], #0x8\n"
+ "str d21, [x25], #0x8\n"
+ "str d25, [x23], #0x8\n"
+ "str d29, [x21], #0x8\n"
+ "tbz x15, #0, 208f\n"
+ "st1 { v9.s }[2], [x13]\n"
+ "st1 { v13.s }[2], [x9]\n"
+ "st1 { v17.s }[2], [x27]\n"
+ "st1 { v21.s }[2], [x25]\n"
+ "st1 { v25.s }[2], [x23]\n"
+ "st1 { v29.s }[2], [x21]\n"
+ "b 208f\n"
+ "205:" // Height 6: Partial direct writeback: partial_1_4
+ "tbz x15, #0, 208f\n"
+ "str s9, [x13, #0x0]\n"
+ "str s13, [x9, #0x0]\n"
+ "str s17, [x27, #0x0]\n"
+ "str s21, [x25, #0x0]\n"
+ "str s25, [x23, #0x0]\n"
+ "str s29, [x21, #0x0]\n"
+ "b 208f\n"
+ "206:" // Height 6: Partial direct writeback: partial_2_0
+ "tbz x15, #1, 207f\n"
+ "str d8, [x13], #0x8\n"
+ "str d12, [x9], #0x8\n"
+ "str d16, [x27], #0x8\n"
+ "str d20, [x25], #0x8\n"
+ "str d24, [x23], #0x8\n"
+ "str d28, [x21], #0x8\n"
+ "tbz x15, #0, 208f\n"
+ "st1 { v8.s }[2], [x13]\n"
+ "st1 { v12.s }[2], [x9]\n"
+ "st1 { v16.s }[2], [x27]\n"
+ "st1 { v20.s }[2], [x25]\n"
+ "st1 { v24.s }[2], [x23]\n"
+ "st1 { v28.s }[2], [x21]\n"
+ "b 208f\n"
+ "207:" // Height 6: Partial direct writeback: partial_1_0
+ "str s8, [x13, #0x0]\n"
+ "str s12, [x9, #0x0]\n"
+ "str s16, [x27, #0x0]\n"
+ "str s20, [x25, #0x0]\n"
+ "str s24, [x23, #0x0]\n"
+ "str s28, [x21, #0x0]\n"
+ "208:" // Height 6: Partial direct writeback: Done
+ "b 210f\n"
+ "209:" // Height 6: Full writeback
+ "str q8, [x13, #0x0]\n"
+ "str q9, [x13, #0x10]\n"
+ "str q10, [x13, #0x20]\n"
+ "str q11, [x13, #0x30]\n"
+ "str q12, [x9, #0x0]\n"
+ "str q13, [x9, #0x10]\n"
+ "str q14, [x9, #0x20]\n"
+ "str q15, [x9, #0x30]\n"
+ "str q16, [x27, #0x0]\n"
+ "str q17, [x27, #0x10]\n"
+ "str q18, [x27, #0x20]\n"
+ "str q19, [x27, #0x30]\n"
+ "str q20, [x25, #0x0]\n"
+ "str q21, [x25, #0x10]\n"
+ "str q22, [x25, #0x20]\n"
+ "str q23, [x25, #0x30]\n"
+ "str q24, [x23, #0x0]\n"
+ "str q25, [x23, #0x10]\n"
+ "str q26, [x23, #0x20]\n"
+ "str q27, [x23, #0x30]\n"
+ "str q28, [x21, #0x0]\n"
+ "str q29, [x21, #0x10]\n"
+ "str q30, [x21, #0x20]\n"
+ "str q31, [x21, #0x30]\n"
+ "add x13, x13, #0x40\n"
+ "add x9, x9, #0x40\n"
+ "add x27, x27, #0x40\n"
+ "add x25, x25, #0x40\n"
+ "add x23, x23, #0x40\n"
+ "add x21, x21, #0x40\n"
+ "210:" // Height 6: Writeback done
+ "subs x15, x15, #0x10\n"
+ "bgt 178b\n"
+ "subs %x[M], %x[M], #0x6\n"
+ "beq 212f\n"
+ "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "tbz %x[flags], #3, 211f\n"
+ "add x20, x20, #0x6\n"
+ "str x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "b 1b\n"
+ "211:" // Update direct input
+ "mov x19, #0x6\n"
+ "madd %x[input_ptr], x19, x20, %x[input_ptr]\n"
+ "b 1b\n"
+ "212:" // Exit
+
+ : [M] "+r" (M), [input_ptr] "+r" (input_ptr), [output_ptr] "+r" (output_ptr)
+ : [args_ptr] "r" (&ka), [flags] "r" (flags), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths))
+ : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
+ );
+}
+
+} // namespace arm_gemm
+#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_4x6.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8qa_dot_4x16.hpp
similarity index 62%
copy from src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_4x6.hpp
copy to src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8qa_dot_4x16.hpp
index 6f31efe..5b4a7f3 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_4x6.hpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8qa_dot_4x16.hpp
@@ -10,49 +10,58 @@
* sell copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
*/
#pragma once
-
#ifdef __aarch64__
+#include "../std_transforms_fixed.hpp"
+
+#define ARGLIST \
+ unsigned int, const unsigned int *, \
+ IndirectInputArg<uint8_t>, \
+ size_t, size_t, \
+ const uint8_t *, \
+ IndirectOutputArg<uint8_t>, \
+ const Requantize32 *, const int32_t *, unsigned int
+
namespace arm_gemm
{
// Actual kernel implementations
-void a64_smallK_hybrid_fp32_mla_4x6(const float *, int, const float *, float *, int, int, int, int, const float *, Activation, bool);
+void a64_hybrid_u8qa_dot_4x16( ARGLIST );
-class smallK_hybrid_fp32_mla_4x6
+class cls_a64_hybrid_u8qa_dot_4x16
{
public:
- typedef float operand_type;
- typedef float result_type;
+ typedef uint8_t operand_type;
+ typedef uint8_t result_type;
- typedef void (*kern_type)(const float *, int, const float *, float *, int, int, int, int, const float *, Activation, bool);
+ typedef void (*kern_type)( ARGLIST );
/* Kernel blocking parameters */
static constexpr unsigned int out_height()
{
- return 6;
+ return 4;
}
static unsigned int out_width()
{
- return 4;
+ return 16;
}
static constexpr unsigned int k_unroll()
{
- return 1;
+ return 4;
}
static constexpr bool supports_accumulate()
@@ -60,27 +69,17 @@
return false;
}
- static constexpr bool supports_bias()
- {
- return true;
- }
-
- static constexpr bool supports_activation()
- {
- return true;
- }
-
- StdTransformsFixed<operand_type, result_type, 6, 4, 1> transforms = {};
+ StdTransformsFixed<operand_type, result_type, 4, 16, 4> transforms = {};
// Default to the generic kernel
- kern_type kernel=a64_smallK_hybrid_fp32_mla_4x6;
+ kern_type kernel=a64_hybrid_u8qa_dot_4x16;
- smallK_hybrid_fp32_mla_4x6(const CPUInfo *)
+ cls_a64_hybrid_u8qa_dot_4x16(const CPUInfo *)
{
-
}
};
} // namespace arm_gemm
+#undef ARGLIST
#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8qa_dot_4x16/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8qa_dot_4x16/generic.cpp
new file mode 100644
index 0000000..ff12472
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8qa_dot_4x16/generic.cpp
@@ -0,0 +1,2072 @@
+/*
+ * Copyright (c) 2019-2020 Arm Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+#ifdef __aarch64__
+
+#include "arm_gemm.hpp"
+#include "../../utils.hpp"
+
+#include <cassert>
+#include <cstdint>
+
+namespace arm_gemm {
+
+void a64_hybrid_u8qa_dot_4x16 (
+ unsigned int num_strings, const unsigned int *string_lengths, IndirectInputArg<uint8_t> A_arg,
+ size_t M, size_t N, const uint8_t *B_ptr, IndirectOutputArg<uint8_t> output_arg,
+ const Requantize32 *qp, const int32_t *col_bias, unsigned int
+)
+{
+ struct KernelArgs {
+ unsigned int num_strings = {};
+ const unsigned int *string_lengths = {};
+ size_t N = {};
+ const uint8_t *B_ptr = {};
+ size_t output_offset = {};
+ size_t input_initial_col = {};
+ size_t input_offset = {};
+ } ka;
+
+ unsigned long flags=0;
+ void *output_ptr;
+ void *input_ptr;
+
+ if (output_arg.is_indirect) {
+ output_ptr=(void *)(output_arg.indirect.ptr);
+ ka.output_offset=output_arg.indirect.offset;
+ flags |= 0x4;
+ } else {
+ output_ptr=(void *)(output_arg.direct.base);
+ ka.output_offset=output_arg.direct.stride;
+ }
+
+ if (A_arg.is_indirect) {
+ input_ptr=(void *)(A_arg.indirect.ptr);
+ ka.input_offset=A_arg.indirect.start_row;
+ ka.input_initial_col=A_arg.indirect.start_col;
+ flags |= 0x8;
+ } else {
+ assert(num_strings==1);
+ input_ptr=(void *)(A_arg.direct.base);
+ ka.input_offset=A_arg.direct.stride;
+ }
+ ka.num_strings = num_strings;
+ ka.string_lengths = string_lengths;
+ ka.N = N;
+ ka.B_ptr = B_ptr;
+ if (qp->c_offset > qp->minval) {
+ flags |= 0x20;
+ }
+ __asm__ __volatile__(
+
+ "1:" // Row loop
+ "cmp %x[M], #0x4\n"
+ "bge 94f\n"
+ "cmp %x[M], #0x2\n"
+ "bgt 63f\n"
+ "beq 32f\n"
+ "movi v11.4s, #0x0\n"
+ "ldr x12, [%x[args_ptr], %[offsetof_N]]\n"
+ "movi v12.4s, #0x0\n"
+ "ldr x11, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "mov x10, %x[col_bias]\n"
+ "movi v13.4s, #0x0\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "bic %x[flags], %x[flags], #0x80000000\n"
+ "movi v14.4s, #0x0\n"
+ "movi v15.16b, #0x1\n"
+ "tbz %x[flags], #2, 2f\n"
+ "ldr x9, [%x[output_ptr], #0x0]\n"
+ "add x9, x9, x19\n"
+ "b 3f\n"
+ "2:" // Height 1: setup direct output
+ "mov x9, %x[output_ptr]\n"
+ "3:" // Height 1: Column loop
+ "movi v16.4s, #0x0\n"
+ "movi v17.4s, #0x0\n"
+ "movi v18.4s, #0x0\n"
+ "movi v19.4s, #0x0\n"
+ "4:" // Height 1: setup done
+ "mov x28, #0x0\n"
+ "5:" // Height 1: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w27, [x20, x28, LSL #0x2]\n"
+ "tbz %x[flags], #3, 6f\n"
+ "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x26, [x20, #0x0]\n"
+ "cbnz x28, 7f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x26, x26, x19\n"
+ "b 7f\n"
+ "6:" // Height 1: setup direct input
+ "mov x26, %x[input_ptr]\n"
+ "7:" // Height 1: input setup done
+ "cmp x27, #0x10\n"
+ "blt 12f\n"
+ "cmp x27, #0x20\n"
+ "blt 10f\n"
+ "8:" // Height 1: Multiply loop: Main loop head
+ "ldr q0, [x26, #0x0]\n"
+ "ldr q4, [x11, #0x0]\n"
+ ".inst 0x6f80e090 // udot v16.4s, v4.16b, v0.4b[0]\n"
+ "ldr q5, [x11, #0x10]\n"
+ "ldr q6, [x11, #0x20]\n"
+ ".inst 0x6f80e0b1 // udot v17.4s, v5.16b, v0.4b[0]\n"
+ "ldr q7, [x11, #0x30]\n"
+ ".inst 0x6f80e0d2 // udot v18.4s, v6.16b, v0.4b[0]\n"
+ "ldr q8, [x11, #0x40]\n"
+ "ldr q9, [x11, #0x50]\n"
+ ".inst 0x6f80e0f3 // udot v19.4s, v7.16b, v0.4b[0]\n"
+ "ldr q10, [x11, #0x60]\n"
+ "ldr q4, [x11, #0x70]\n"
+ ".inst 0x6fa0e110 // udot v16.4s, v8.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e131 // udot v17.4s, v9.16b, v0.4b[1]\n"
+ "ldr q5, [x11, #0x80]\n"
+ "ldr q6, [x11, #0x90]\n"
+ ".inst 0x6fa0e152 // udot v18.4s, v10.16b, v0.4b[1]\n"
+ "ldr q7, [x11, #0xa0]\n"
+ ".inst 0x6fa0e093 // udot v19.4s, v4.16b, v0.4b[1]\n"
+ "ldr q8, [x11, #0xb0]\n"
+ "ldr q9, [x11, #0xc0]\n"
+ ".inst 0x6f80e8b0 // udot v16.4s, v5.16b, v0.4b[2]\n"
+ ".inst 0x6f80e8d1 // udot v17.4s, v6.16b, v0.4b[2]\n"
+ "ldr q10, [x11, #0xd0]\n"
+ "ldr q4, [x11, #0xe0]\n"
+ ".inst 0x6f80e8f2 // udot v18.4s, v7.16b, v0.4b[2]\n"
+ "ldr q5, [x11, #0xf0]\n"
+ "add x26, x26, #0x10\n"
+ ".inst 0x6f80e913 // udot v19.4s, v8.16b, v0.4b[2]\n"
+ ".inst 0x6fa0e930 // udot v16.4s, v9.16b, v0.4b[3]\n"
+ "add x11, x11, #0x100\n"
+ ".inst 0x6fa0e951 // udot v17.4s, v10.16b, v0.4b[3]\n"
+ ".inst 0x6fa0e892 // udot v18.4s, v4.16b, v0.4b[3]\n"
+ ".inst 0x6fa0e8b3 // udot v19.4s, v5.16b, v0.4b[3]\n"
+ "tbnz %x[flags], #31, 9f\n"
+ ".inst 0x6e8f940b // udot v11.4s, v0.16b, v15.16b\n"
+ "9:" // Height 1: Multiply loop: unique 1: skip row sum
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "sub x27, x27, #0x10\n"
+ "cmp x27, #0x20\n"
+ "bge 8b\n"
+ "10:" // Height 1: Multiply loop: Single iteration only
+ "sub x27, x27, #0x10\n"
+ "ldr q0, [x26, #0x0]\n"
+ "ldr q6, [x11, #0x0]\n"
+ ".inst 0x6f80e0d0 // udot v16.4s, v6.16b, v0.4b[0]\n"
+ "ldr q7, [x11, #0x10]\n"
+ "ldr q8, [x11, #0x20]\n"
+ ".inst 0x6f80e0f1 // udot v17.4s, v7.16b, v0.4b[0]\n"
+ "ldr q9, [x11, #0x30]\n"
+ ".inst 0x6f80e112 // udot v18.4s, v8.16b, v0.4b[0]\n"
+ "ldr q10, [x11, #0x40]\n"
+ "ldr q4, [x11, #0x50]\n"
+ ".inst 0x6f80e133 // udot v19.4s, v9.16b, v0.4b[0]\n"
+ "ldr q5, [x11, #0x60]\n"
+ "ldr q6, [x11, #0x70]\n"
+ ".inst 0x6fa0e150 // udot v16.4s, v10.16b, v0.4b[1]\n"
+ ".inst 0x6fa0e091 // udot v17.4s, v4.16b, v0.4b[1]\n"
+ "ldr q7, [x11, #0x80]\n"
+ "ldr q8, [x11, #0x90]\n"
+ ".inst 0x6fa0e0b2 // udot v18.4s, v5.16b, v0.4b[1]\n"
+ "ldr q9, [x11, #0xa0]\n"
+ ".inst 0x6fa0e0d3 // udot v19.4s, v6.16b, v0.4b[1]\n"
+ "ldr q10, [x11, #0xb0]\n"
+ "ldr q4, [x11, #0xc0]\n"
+ ".inst 0x6f80e8f0 // udot v16.4s, v7.16b, v0.4b[2]\n"
+ ".inst 0x6f80e911 // udot v17.4s, v8.16b, v0.4b[2]\n"
+ "ldr q5, [x11, #0xd0]\n"
+ "ldr q6, [x11, #0xe0]\n"
+ ".inst 0x6f80e932 // udot v18.4s, v9.16b, v0.4b[2]\n"
+ "ldr q7, [x11, #0xf0]\n"
+ "add x26, x26, #0x10\n"
+ ".inst 0x6f80e953 // udot v19.4s, v10.16b, v0.4b[2]\n"
+ ".inst 0x6fa0e890 // udot v16.4s, v4.16b, v0.4b[3]\n"
+ "add x11, x11, #0x100\n"
+ ".inst 0x6fa0e8b1 // udot v17.4s, v5.16b, v0.4b[3]\n"
+ ".inst 0x6fa0e8d2 // udot v18.4s, v6.16b, v0.4b[3]\n"
+ ".inst 0x6fa0e8f3 // udot v19.4s, v7.16b, v0.4b[3]\n"
+ "tbnz %x[flags], #31, 11f\n"
+ ".inst 0x6e8f940b // udot v11.4s, v0.16b, v15.16b\n"
+ "11:" // Height 1: Multiply loop: unique 2: skip row sum
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "12:" // Height 1: Multiply loop: Main loop skip
+ "cbz x27, 19f\n"
+ "cmp x27, #0x4\n"
+ "blt 15f\n"
+ "13:" // Height 1: Multiply loop: Odd block loop
+ "ldr s0, [x26], #0x4\n"
+ "tbnz %x[flags], #31, 14f\n"
+ ".inst 0x6e8f940b // udot v11.4s, v0.16b, v15.16b\n"
+ "14:" // Height 1: Multiply loop: unique 3: skip row sum
+ "ldr q8, [x11, #0x0]\n"
+ ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
+ "ldr q9, [x11, #0x10]\n"
+ "ldr q10, [x11, #0x20]\n"
+ ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
+ "ldr q4, [x11, #0x30]\n"
+ ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
+ "sub x27, x27, #0x4\n"
+ "add x11, x11, #0x40\n"
+ ".inst 0x6f80e093 // udot v19.4s, v4.16b, v0.4b[0]\n"
+ "cmp x27, #0x4\n"
+ "bge 13b\n"
+ "cbz x27, 19f\n"
+ "15:" // Height 1: Multiply loop: Skip odd blocks
+ "tbz x27, #1, 16f\n"
+ "ldr h0, [x26], #0x2\n"
+ "tbz x27, #0, 17f\n"
+ "ld1 { v0.b }[2], [x26]\n"
+ "b 17f\n"
+ "16:" // Height 1: Multiply loop: Ragged operand read: partial_1_0
+ "ldr b0, [x26, #0x0]\n"
+ "17:" // Height 1: Multiply loop: Ragged operand read: Done
+ "tbnz %x[flags], #31, 18f\n"
+ ".inst 0x6e8f940b // udot v11.4s, v0.16b, v15.16b\n"
+ "18:" // Height 1: Multiply loop: unique 4: skip row sum
+ "ldr q5, [x11, #0x0]\n"
+ ".inst 0x6f80e0b0 // udot v16.4s, v5.16b, v0.4b[0]\n"
+ "ldr q6, [x11, #0x10]\n"
+ "ldr q7, [x11, #0x20]\n"
+ ".inst 0x6f80e0d1 // udot v17.4s, v6.16b, v0.4b[0]\n"
+ "ldr q8, [x11, #0x30]\n"
+ ".inst 0x6f80e0f2 // udot v18.4s, v7.16b, v0.4b[0]\n"
+ "add x11, x11, #0x40\n"
+ ".inst 0x6f80e113 // udot v19.4s, v8.16b, v0.4b[0]\n"
+ "19:" // Height 1: Multiply loop: No odd multiplies
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "add x28, x28, #0x1\n"
+ "cmp x28, x19\n"
+ "bne 5b\n"
+ "prfm pstl1keep, [x9, #0x0]\n"
+ "tbnz %x[flags], #31, 20f\n"
+ "addp v11.4s, v11.4s, v11.4s\n"
+ "add x19, %x[qp], %[b_offset]\n"
+ "addp v11.4s, v11.4s, v11.4s\n"
+ "ld1r { v1.4s }, [x19]\n"
+ "neg v1.4s, v1.4s\n"
+ "mul v11.4s, v11.4s, v1.4s\n"
+ "20:" // Height 1: skip row sum fixup
+ "add v16.4s, v16.4s, v11.4s\n"
+ "orr %x[flags], %x[flags], #0x80000000\n"
+ "add v17.4s, v17.4s, v11.4s\n"
+ "ldr q0, [x10, #0x0]\n"
+ "add v18.4s, v18.4s, v11.4s\n"
+ "ldr q1, [x10, #0x10]\n"
+ "add v19.4s, v19.4s, v11.4s\n"
+ "ldr q2, [x10, #0x20]\n"
+ "ldr q3, [x10, #0x30]\n"
+ "add v16.4s, v16.4s, v0.4s\n"
+ "add x20, %x[qp], %[per_layer_right_shift]\n"
+ "ld1r { v0.4s }, [x20]\n"
+ "add v17.4s, v17.4s, v1.4s\n"
+ "add x19, %x[qp], %[per_layer_mul]\n"
+ "add v18.4s, v18.4s, v2.4s\n"
+ "ld1r { v4.4s }, [x19]\n"
+ "add x10, x10, #0x40\n"
+ "add v19.4s, v19.4s, v3.4s\n"
+ "sqrdmulh v16.4s, v16.4s, v4.4s\n"
+ "sqrdmulh v17.4s, v17.4s, v4.4s\n"
+ "sqrdmulh v18.4s, v18.4s, v4.4s\n"
+ "sqrdmulh v19.4s, v19.4s, v4.4s\n"
+ "tbz %x[flags], #5, 21f\n"
+ "and v4.16b, v16.16b, v0.16b\n"
+ "sshr v4.4s, v4.4s, #0x1f\n"
+ "and v5.16b, v17.16b, v0.16b\n"
+ "and v6.16b, v18.16b, v0.16b\n"
+ "sshr v5.4s, v5.4s, #0x1f\n"
+ "and v7.16b, v19.16b, v0.16b\n"
+ "sshr v6.4s, v6.4s, #0x1f\n"
+ "sqadd v16.4s, v16.4s, v4.4s\n"
+ "sshr v7.4s, v7.4s, #0x1f\n"
+ "sqadd v17.4s, v17.4s, v5.4s\n"
+ "sqadd v18.4s, v18.4s, v6.4s\n"
+ "sqadd v19.4s, v19.4s, v7.4s\n"
+ "21:" // Height 1: no shift correction
+ "srshl v16.4s, v16.4s, v0.4s\n"
+ "add x19, %x[qp], %[c_offset]\n"
+ "ld1r { v4.4s }, [x19]\n"
+ "srshl v17.4s, v17.4s, v0.4s\n"
+ "add x19, %x[qp], %[minval]\n"
+ "srshl v18.4s, v18.4s, v0.4s\n"
+ "ld1r { v5.4s }, [x19]\n"
+ "add x19, %x[qp], %[maxval]\n"
+ "srshl v19.4s, v19.4s, v0.4s\n"
+ "ld1r { v6.4s }, [x19]\n"
+ "cmp x12, #0x10\n"
+ "add v16.4s, v16.4s, v4.4s\n"
+ "add v17.4s, v17.4s, v4.4s\n"
+ "add v18.4s, v18.4s, v4.4s\n"
+ "add v19.4s, v19.4s, v4.4s\n"
+ "smin v16.4s, v16.4s, v6.4s\n"
+ "smin v17.4s, v17.4s, v6.4s\n"
+ "smin v18.4s, v18.4s, v6.4s\n"
+ "smax v16.4s, v16.4s, v5.4s\n"
+ "smax v17.4s, v17.4s, v5.4s\n"
+ "smax v18.4s, v18.4s, v5.4s\n"
+ "smin v19.4s, v19.4s, v6.4s\n"
+ "uzp1 v16.8h, v16.8h, v17.8h\n"
+ "smax v19.4s, v19.4s, v5.4s\n"
+ "uzp1 v17.8h, v18.8h, v19.8h\n"
+ "uzp1 v16.16b, v16.16b, v17.16b\n"
+ "bge 30f\n"
+ "tbz x12, #3, 25f\n"
+ "str d16, [x9], #0x8\n"
+ "tbz x12, #2, 23f\n"
+ "st1 { v16.s }[2], [x9], #0x4\n"
+ "tbz x12, #1, 22f\n"
+ "st1 { v16.h }[6], [x9], #0x2\n"
+ "tbz x12, #0, 29f\n"
+ "st1 { v16.b }[14], [x9]\n"
+ "b 29f\n"
+ "22:" // Height 1: Partial direct writeback: partial_1_12
+ "tbz x12, #0, 29f\n"
+ "st1 { v16.b }[12], [x9]\n"
+ "b 29f\n"
+ "23:" // Height 1: Partial direct writeback: partial_2_8
+ "tbz x12, #1, 24f\n"
+ "st1 { v16.h }[4], [x9], #0x2\n"
+ "tbz x12, #0, 29f\n"
+ "st1 { v16.b }[10], [x9]\n"
+ "b 29f\n"
+ "24:" // Height 1: Partial direct writeback: partial_1_8
+ "tbz x12, #0, 29f\n"
+ "st1 { v16.b }[8], [x9]\n"
+ "b 29f\n"
+ "25:" // Height 1: Partial direct writeback: partial_4_0
+ "tbz x12, #2, 27f\n"
+ "str s16, [x9], #0x4\n"
+ "tbz x12, #1, 26f\n"
+ "st1 { v16.h }[2], [x9], #0x2\n"
+ "tbz x12, #0, 29f\n"
+ "st1 { v16.b }[6], [x9]\n"
+ "b 29f\n"
+ "26:" // Height 1: Partial direct writeback: partial_1_4
+ "tbz x12, #0, 29f\n"
+ "st1 { v16.b }[4], [x9]\n"
+ "b 29f\n"
+ "27:" // Height 1: Partial direct writeback: partial_2_0
+ "tbz x12, #1, 28f\n"
+ "str h16, [x9], #0x2\n"
+ "tbz x12, #0, 29f\n"
+ "st1 { v16.b }[2], [x9]\n"
+ "b 29f\n"
+ "28:" // Height 1: Partial direct writeback: partial_1_0
+ "str b16, [x9, #0x0]\n"
+ "29:" // Height 1: Partial direct writeback: Done
+ "b 31f\n"
+ "30:" // Height 1: Full writeback
+ "str q16, [x9, #0x0]\n"
+ "add x9, x9, #0x10\n"
+ "31:" // Height 1: Writeback done
+ "subs x12, x12, #0x10\n"
+ "bgt 3b\n"
+ "b 126f\n"
+ "32:" // Height 2
+ "movi v11.4s, #0x0\n"
+ "ldr x12, [%x[args_ptr], %[offsetof_N]]\n"
+ "mov x10, %x[col_bias]\n"
+ "movi v12.4s, #0x0\n"
+ "ldr x11, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "bic %x[flags], %x[flags], #0x80000000\n"
+ "movi v13.4s, #0x0\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "movi v14.4s, #0x0\n"
+ "movi v15.16b, #0x1\n"
+ "tbz %x[flags], #2, 33f\n"
+ "ldr x9, [%x[output_ptr], #0x0]\n"
+ "ldr x25, [%x[output_ptr], #0x8]\n"
+ "add x9, x9, x19\n"
+ "add x25, x25, x19\n"
+ "b 34f\n"
+ "33:" // Height 2: setup direct output
+ "mov x9, %x[output_ptr]\n"
+ "add x25, x9, x19\n"
+ "34:" // Height 2: Column loop
+ "movi v16.4s, #0x0\n"
+ "movi v17.4s, #0x0\n"
+ "movi v18.4s, #0x0\n"
+ "movi v19.4s, #0x0\n"
+ "movi v20.4s, #0x0\n"
+ "movi v21.4s, #0x0\n"
+ "movi v22.4s, #0x0\n"
+ "movi v23.4s, #0x0\n"
+ "35:" // Height 2: setup done
+ "mov x28, #0x0\n"
+ "36:" // Height 2: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w27, [x20, x28, LSL #0x2]\n"
+ "tbz %x[flags], #3, 37f\n"
+ "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x26, [x20, #0x0]\n"
+ "ldr x24, [x20, #0x8]\n"
+ "cbnz x28, 38f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x26, x26, x19\n"
+ "add x24, x24, x19\n"
+ "b 38f\n"
+ "37:" // Height 2: setup direct input
+ "mov x26, %x[input_ptr]\n"
+ "add x24, x26, x19\n"
+ "38:" // Height 2: input setup done
+ "cmp x27, #0x10\n"
+ "blt 43f\n"
+ "cmp x27, #0x20\n"
+ "blt 41f\n"
+ "39:" // Height 2: Multiply loop: Main loop head
+ "ldr q0, [x26, #0x0]\n"
+ "ldr q1, [x24, #0x0]\n"
+ "ldr q4, [x11, #0x0]\n"
+ ".inst 0x6f80e090 // udot v16.4s, v4.16b, v0.4b[0]\n"
+ "ldr q5, [x11, #0x10]\n"
+ ".inst 0x6f81e094 // udot v20.4s, v4.16b, v1.4b[0]\n"
+ "ldr q6, [x11, #0x20]\n"
+ "ldr q7, [x11, #0x30]\n"
+ ".inst 0x6f80e0b1 // udot v17.4s, v5.16b, v0.4b[0]\n"
+ "ldr q8, [x11, #0x40]\n"
+ ".inst 0x6f81e0b5 // udot v21.4s, v5.16b, v1.4b[0]\n"
+ "ldr q9, [x11, #0x50]\n"
+ ".inst 0x6f80e0d2 // udot v18.4s, v6.16b, v0.4b[0]\n"
+ "ldr q10, [x11, #0x60]\n"
+ ".inst 0x6f81e0d6 // udot v22.4s, v6.16b, v1.4b[0]\n"
+ "ldr q4, [x11, #0x70]\n"
+ ".inst 0x6f80e0f3 // udot v19.4s, v7.16b, v0.4b[0]\n"
+ "ldr q5, [x11, #0x80]\n"
+ ".inst 0x6f81e0f7 // udot v23.4s, v7.16b, v1.4b[0]\n"
+ "ldr q6, [x11, #0x90]\n"
+ ".inst 0x6fa0e110 // udot v16.4s, v8.16b, v0.4b[1]\n"
+ "ldr q7, [x11, #0xa0]\n"
+ ".inst 0x6fa1e114 // udot v20.4s, v8.16b, v1.4b[1]\n"
+ "ldr q8, [x11, #0xb0]\n"
+ ".inst 0x6fa0e131 // udot v17.4s, v9.16b, v0.4b[1]\n"
+ "add x26, x26, #0x10\n"
+ ".inst 0x6fa1e135 // udot v21.4s, v9.16b, v1.4b[1]\n"
+ "ldr q9, [x11, #0xc0]\n"
+ ".inst 0x6fa0e152 // udot v18.4s, v10.16b, v0.4b[1]\n"
+ "add x24, x24, #0x10\n"
+ ".inst 0x6fa1e156 // udot v22.4s, v10.16b, v1.4b[1]\n"
+ "ldr q10, [x11, #0xd0]\n"
+ ".inst 0x6fa0e093 // udot v19.4s, v4.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e097 // udot v23.4s, v4.16b, v1.4b[1]\n"
+ "ldr q4, [x11, #0xe0]\n"
+ ".inst 0x6f80e8b0 // udot v16.4s, v5.16b, v0.4b[2]\n"
+ ".inst 0x6f81e8b4 // udot v20.4s, v5.16b, v1.4b[2]\n"
+ "ldr q5, [x11, #0xf0]\n"
+ ".inst 0x6f80e8d1 // udot v17.4s, v6.16b, v0.4b[2]\n"
+ "add x11, x11, #0x100\n"
+ ".inst 0x6f81e8d5 // udot v21.4s, v6.16b, v1.4b[2]\n"
+ ".inst 0x6f80e8f2 // udot v18.4s, v7.16b, v0.4b[2]\n"
+ ".inst 0x6f81e8f6 // udot v22.4s, v7.16b, v1.4b[2]\n"
+ ".inst 0x6f80e913 // udot v19.4s, v8.16b, v0.4b[2]\n"
+ ".inst 0x6f81e917 // udot v23.4s, v8.16b, v1.4b[2]\n"
+ ".inst 0x6fa0e930 // udot v16.4s, v9.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e934 // udot v20.4s, v9.16b, v1.4b[3]\n"
+ ".inst 0x6fa0e951 // udot v17.4s, v10.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e955 // udot v21.4s, v10.16b, v1.4b[3]\n"
+ ".inst 0x6fa0e892 // udot v18.4s, v4.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e896 // udot v22.4s, v4.16b, v1.4b[3]\n"
+ ".inst 0x6fa0e8b3 // udot v19.4s, v5.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e8b7 // udot v23.4s, v5.16b, v1.4b[3]\n"
+ "tbnz %x[flags], #31, 40f\n"
+ ".inst 0x6e8f940b // udot v11.4s, v0.16b, v15.16b\n"
+ ".inst 0x6e8f942c // udot v12.4s, v1.16b, v15.16b\n"
+ "40:" // Height 2: Multiply loop: unique 5: skip row sum
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "sub x27, x27, #0x10\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "cmp x27, #0x20\n"
+ "bge 39b\n"
+ "41:" // Height 2: Multiply loop: Single iteration only
+ "sub x27, x27, #0x10\n"
+ "ldr q0, [x26, #0x0]\n"
+ "ldr q1, [x24, #0x0]\n"
+ "ldr q6, [x11, #0x0]\n"
+ ".inst 0x6f80e0d0 // udot v16.4s, v6.16b, v0.4b[0]\n"
+ "ldr q7, [x11, #0x10]\n"
+ ".inst 0x6f81e0d4 // udot v20.4s, v6.16b, v1.4b[0]\n"
+ "ldr q8, [x11, #0x20]\n"
+ "ldr q9, [x11, #0x30]\n"
+ ".inst 0x6f80e0f1 // udot v17.4s, v7.16b, v0.4b[0]\n"
+ "ldr q10, [x11, #0x40]\n"
+ ".inst 0x6f81e0f5 // udot v21.4s, v7.16b, v1.4b[0]\n"
+ "ldr q4, [x11, #0x50]\n"
+ ".inst 0x6f80e112 // udot v18.4s, v8.16b, v0.4b[0]\n"
+ "ldr q5, [x11, #0x60]\n"
+ ".inst 0x6f81e116 // udot v22.4s, v8.16b, v1.4b[0]\n"
+ "ldr q6, [x11, #0x70]\n"
+ ".inst 0x6f80e133 // udot v19.4s, v9.16b, v0.4b[0]\n"
+ "ldr q7, [x11, #0x80]\n"
+ ".inst 0x6f81e137 // udot v23.4s, v9.16b, v1.4b[0]\n"
+ "ldr q8, [x11, #0x90]\n"
+ ".inst 0x6fa0e150 // udot v16.4s, v10.16b, v0.4b[1]\n"
+ "ldr q9, [x11, #0xa0]\n"
+ ".inst 0x6fa1e154 // udot v20.4s, v10.16b, v1.4b[1]\n"
+ "ldr q10, [x11, #0xb0]\n"
+ ".inst 0x6fa0e091 // udot v17.4s, v4.16b, v0.4b[1]\n"
+ "add x26, x26, #0x10\n"
+ ".inst 0x6fa1e095 // udot v21.4s, v4.16b, v1.4b[1]\n"
+ "ldr q4, [x11, #0xc0]\n"
+ ".inst 0x6fa0e0b2 // udot v18.4s, v5.16b, v0.4b[1]\n"
+ "add x24, x24, #0x10\n"
+ ".inst 0x6fa1e0b6 // udot v22.4s, v5.16b, v1.4b[1]\n"
+ "ldr q5, [x11, #0xd0]\n"
+ ".inst 0x6fa0e0d3 // udot v19.4s, v6.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e0d7 // udot v23.4s, v6.16b, v1.4b[1]\n"
+ "ldr q6, [x11, #0xe0]\n"
+ ".inst 0x6f80e8f0 // udot v16.4s, v7.16b, v0.4b[2]\n"
+ ".inst 0x6f81e8f4 // udot v20.4s, v7.16b, v1.4b[2]\n"
+ "ldr q7, [x11, #0xf0]\n"
+ ".inst 0x6f80e911 // udot v17.4s, v8.16b, v0.4b[2]\n"
+ "add x11, x11, #0x100\n"
+ ".inst 0x6f81e915 // udot v21.4s, v8.16b, v1.4b[2]\n"
+ ".inst 0x6f80e932 // udot v18.4s, v9.16b, v0.4b[2]\n"
+ ".inst 0x6f81e936 // udot v22.4s, v9.16b, v1.4b[2]\n"
+ ".inst 0x6f80e953 // udot v19.4s, v10.16b, v0.4b[2]\n"
+ ".inst 0x6f81e957 // udot v23.4s, v10.16b, v1.4b[2]\n"
+ ".inst 0x6fa0e890 // udot v16.4s, v4.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e894 // udot v20.4s, v4.16b, v1.4b[3]\n"
+ ".inst 0x6fa0e8b1 // udot v17.4s, v5.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e8b5 // udot v21.4s, v5.16b, v1.4b[3]\n"
+ ".inst 0x6fa0e8d2 // udot v18.4s, v6.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e8d6 // udot v22.4s, v6.16b, v1.4b[3]\n"
+ ".inst 0x6fa0e8f3 // udot v19.4s, v7.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e8f7 // udot v23.4s, v7.16b, v1.4b[3]\n"
+ "tbnz %x[flags], #31, 42f\n"
+ ".inst 0x6e8f940b // udot v11.4s, v0.16b, v15.16b\n"
+ ".inst 0x6e8f942c // udot v12.4s, v1.16b, v15.16b\n"
+ "42:" // Height 2: Multiply loop: unique 6: skip row sum
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "43:" // Height 2: Multiply loop: Main loop skip
+ "cbz x27, 50f\n"
+ "cmp x27, #0x4\n"
+ "blt 46f\n"
+ "44:" // Height 2: Multiply loop: Odd block loop
+ "ldr s0, [x26], #0x4\n"
+ "ldr s1, [x24], #0x4\n"
+ "tbnz %x[flags], #31, 45f\n"
+ ".inst 0x6e8f940b // udot v11.4s, v0.16b, v15.16b\n"
+ ".inst 0x6e8f942c // udot v12.4s, v1.16b, v15.16b\n"
+ "45:" // Height 2: Multiply loop: unique 7: skip row sum
+ "ldr q8, [x11, #0x0]\n"
+ ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
+ "ldr q9, [x11, #0x10]\n"
+ ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
+ "ldr q10, [x11, #0x20]\n"
+ "ldr q4, [x11, #0x30]\n"
+ ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
+ "sub x27, x27, #0x4\n"
+ ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
+ "cmp x27, #0x4\n"
+ ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
+ "add x11, x11, #0x40\n"
+ ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
+ ".inst 0x6f80e093 // udot v19.4s, v4.16b, v0.4b[0]\n"
+ ".inst 0x6f81e097 // udot v23.4s, v4.16b, v1.4b[0]\n"
+ "bge 44b\n"
+ "cbz x27, 50f\n"
+ "46:" // Height 2: Multiply loop: Skip odd blocks
+ "tbz x27, #1, 47f\n"
+ "ldr h0, [x26], #0x2\n"
+ "ldr h1, [x24], #0x2\n"
+ "tbz x27, #0, 48f\n"
+ "ld1 { v0.b }[2], [x26]\n"
+ "ld1 { v1.b }[2], [x24]\n"
+ "b 48f\n"
+ "47:" // Height 2: Multiply loop: Ragged operand read: partial_1_0
+ "ldr b0, [x26, #0x0]\n"
+ "ldr b1, [x24, #0x0]\n"
+ "48:" // Height 2: Multiply loop: Ragged operand read: Done
+ "tbnz %x[flags], #31, 49f\n"
+ ".inst 0x6e8f940b // udot v11.4s, v0.16b, v15.16b\n"
+ ".inst 0x6e8f942c // udot v12.4s, v1.16b, v15.16b\n"
+ "49:" // Height 2: Multiply loop: unique 8: skip row sum
+ "ldr q5, [x11, #0x0]\n"
+ ".inst 0x6f80e0b0 // udot v16.4s, v5.16b, v0.4b[0]\n"
+ "ldr q6, [x11, #0x10]\n"
+ ".inst 0x6f81e0b4 // udot v20.4s, v5.16b, v1.4b[0]\n"
+ "ldr q7, [x11, #0x20]\n"
+ "ldr q8, [x11, #0x30]\n"
+ ".inst 0x6f80e0d1 // udot v17.4s, v6.16b, v0.4b[0]\n"
+ "add x11, x11, #0x40\n"
+ ".inst 0x6f81e0d5 // udot v21.4s, v6.16b, v1.4b[0]\n"
+ ".inst 0x6f80e0f2 // udot v18.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x6f81e0f6 // udot v22.4s, v7.16b, v1.4b[0]\n"
+ ".inst 0x6f80e113 // udot v19.4s, v8.16b, v0.4b[0]\n"
+ ".inst 0x6f81e117 // udot v23.4s, v8.16b, v1.4b[0]\n"
+ "50:" // Height 2: Multiply loop: No odd multiplies
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "add x28, x28, #0x1\n"
+ "cmp x28, x19\n"
+ "bne 36b\n"
+ "prfm pstl1keep, [x9, #0x0]\n"
+ "prfm pstl1keep, [x25, #0x0]\n"
+ "tbnz %x[flags], #31, 51f\n"
+ "addp v11.4s, v11.4s, v11.4s\n"
+ "add x19, %x[qp], %[b_offset]\n"
+ "ld1r { v2.4s }, [x19]\n"
+ "addp v12.4s, v12.4s, v12.4s\n"
+ "addp v11.4s, v11.4s, v11.4s\n"
+ "addp v12.4s, v12.4s, v12.4s\n"
+ "neg v2.4s, v2.4s\n"
+ "mul v11.4s, v11.4s, v2.4s\n"
+ "mul v12.4s, v12.4s, v2.4s\n"
+ "51:" // Height 2: skip row sum fixup
+ "add v16.4s, v16.4s, v11.4s\n"
+ "orr %x[flags], %x[flags], #0x80000000\n"
+ "add v17.4s, v17.4s, v11.4s\n"
+ "ldr q0, [x10, #0x0]\n"
+ "add v18.4s, v18.4s, v11.4s\n"
+ "ldr q1, [x10, #0x10]\n"
+ "add v19.4s, v19.4s, v11.4s\n"
+ "ldr q2, [x10, #0x20]\n"
+ "add v20.4s, v20.4s, v12.4s\n"
+ "ldr q3, [x10, #0x30]\n"
+ "add v21.4s, v21.4s, v12.4s\n"
+ "add x20, %x[qp], %[per_layer_right_shift]\n"
+ "add v22.4s, v22.4s, v12.4s\n"
+ "add x19, %x[qp], %[per_layer_mul]\n"
+ "ld1r { v4.4s }, [x19]\n"
+ "add v23.4s, v23.4s, v12.4s\n"
+ "add x10, x10, #0x40\n"
+ "add v16.4s, v16.4s, v0.4s\n"
+ "add v17.4s, v17.4s, v1.4s\n"
+ "add v18.4s, v18.4s, v2.4s\n"
+ "add v19.4s, v19.4s, v3.4s\n"
+ "add v20.4s, v20.4s, v0.4s\n"
+ "ld1r { v0.4s }, [x20]\n"
+ "add v21.4s, v21.4s, v1.4s\n"
+ "add v22.4s, v22.4s, v2.4s\n"
+ "add v23.4s, v23.4s, v3.4s\n"
+ "sqrdmulh v16.4s, v16.4s, v4.4s\n"
+ "sqrdmulh v17.4s, v17.4s, v4.4s\n"
+ "sqrdmulh v18.4s, v18.4s, v4.4s\n"
+ "sqrdmulh v19.4s, v19.4s, v4.4s\n"
+ "sqrdmulh v20.4s, v20.4s, v4.4s\n"
+ "sqrdmulh v21.4s, v21.4s, v4.4s\n"
+ "sqrdmulh v22.4s, v22.4s, v4.4s\n"
+ "sqrdmulh v23.4s, v23.4s, v4.4s\n"
+ "tbz %x[flags], #5, 52f\n"
+ "and v4.16b, v16.16b, v0.16b\n"
+ "sshr v4.4s, v4.4s, #0x1f\n"
+ "and v5.16b, v17.16b, v0.16b\n"
+ "and v6.16b, v18.16b, v0.16b\n"
+ "sshr v5.4s, v5.4s, #0x1f\n"
+ "and v7.16b, v19.16b, v0.16b\n"
+ "and v8.16b, v20.16b, v0.16b\n"
+ "sshr v6.4s, v6.4s, #0x1f\n"
+ "and v9.16b, v21.16b, v0.16b\n"
+ "sshr v7.4s, v7.4s, #0x1f\n"
+ "sqadd v16.4s, v16.4s, v4.4s\n"
+ "and v10.16b, v22.16b, v0.16b\n"
+ "sshr v8.4s, v8.4s, #0x1f\n"
+ "and v4.16b, v23.16b, v0.16b\n"
+ "sshr v9.4s, v9.4s, #0x1f\n"
+ "sqadd v17.4s, v17.4s, v5.4s\n"
+ "sshr v10.4s, v10.4s, #0x1f\n"
+ "sqadd v18.4s, v18.4s, v6.4s\n"
+ "sshr v4.4s, v4.4s, #0x1f\n"
+ "sqadd v19.4s, v19.4s, v7.4s\n"
+ "sqadd v20.4s, v20.4s, v8.4s\n"
+ "sqadd v21.4s, v21.4s, v9.4s\n"
+ "sqadd v22.4s, v22.4s, v10.4s\n"
+ "sqadd v23.4s, v23.4s, v4.4s\n"
+ "52:" // Height 2: no shift correction
+ "srshl v16.4s, v16.4s, v0.4s\n"
+ "add x19, %x[qp], %[c_offset]\n"
+ "ld1r { v4.4s }, [x19]\n"
+ "srshl v17.4s, v17.4s, v0.4s\n"
+ "add x19, %x[qp], %[minval]\n"
+ "srshl v18.4s, v18.4s, v0.4s\n"
+ "ld1r { v5.4s }, [x19]\n"
+ "add x19, %x[qp], %[maxval]\n"
+ "srshl v19.4s, v19.4s, v0.4s\n"
+ "ld1r { v6.4s }, [x19]\n"
+ "cmp x12, #0x10\n"
+ "srshl v20.4s, v20.4s, v0.4s\n"
+ "srshl v21.4s, v21.4s, v0.4s\n"
+ "srshl v22.4s, v22.4s, v0.4s\n"
+ "srshl v23.4s, v23.4s, v0.4s\n"
+ "add v16.4s, v16.4s, v4.4s\n"
+ "add v17.4s, v17.4s, v4.4s\n"
+ "add v18.4s, v18.4s, v4.4s\n"
+ "smin v16.4s, v16.4s, v6.4s\n"
+ "smin v17.4s, v17.4s, v6.4s\n"
+ "smin v18.4s, v18.4s, v6.4s\n"
+ "smax v16.4s, v16.4s, v5.4s\n"
+ "smax v17.4s, v17.4s, v5.4s\n"
+ "smax v18.4s, v18.4s, v5.4s\n"
+ "add v19.4s, v19.4s, v4.4s\n"
+ "add v20.4s, v20.4s, v4.4s\n"
+ "add v21.4s, v21.4s, v4.4s\n"
+ "smin v19.4s, v19.4s, v6.4s\n"
+ "smin v20.4s, v20.4s, v6.4s\n"
+ "smin v21.4s, v21.4s, v6.4s\n"
+ "smax v19.4s, v19.4s, v5.4s\n"
+ "smax v20.4s, v20.4s, v5.4s\n"
+ "smax v21.4s, v21.4s, v5.4s\n"
+ "add v22.4s, v22.4s, v4.4s\n"
+ "add v23.4s, v23.4s, v4.4s\n"
+ "uzp1 v16.8h, v16.8h, v17.8h\n"
+ "smin v22.4s, v22.4s, v6.4s\n"
+ "smin v23.4s, v23.4s, v6.4s\n"
+ "uzp1 v17.8h, v18.8h, v19.8h\n"
+ "smax v22.4s, v22.4s, v5.4s\n"
+ "smax v23.4s, v23.4s, v5.4s\n"
+ "uzp1 v20.8h, v20.8h, v21.8h\n"
+ "uzp1 v16.16b, v16.16b, v17.16b\n"
+ "uzp1 v21.8h, v22.8h, v23.8h\n"
+ "uzp1 v20.16b, v20.16b, v21.16b\n"
+ "bge 61f\n"
+ "tbz x12, #3, 56f\n"
+ "str d16, [x9], #0x8\n"
+ "str d20, [x25], #0x8\n"
+ "tbz x12, #2, 54f\n"
+ "st1 { v16.s }[2], [x9], #0x4\n"
+ "st1 { v20.s }[2], [x25], #0x4\n"
+ "tbz x12, #1, 53f\n"
+ "st1 { v16.h }[6], [x9], #0x2\n"
+ "st1 { v20.h }[6], [x25], #0x2\n"
+ "tbz x12, #0, 60f\n"
+ "st1 { v16.b }[14], [x9]\n"
+ "st1 { v20.b }[14], [x25]\n"
+ "b 60f\n"
+ "53:" // Height 2: Partial direct writeback: partial_1_12
+ "tbz x12, #0, 60f\n"
+ "st1 { v16.b }[12], [x9]\n"
+ "st1 { v20.b }[12], [x25]\n"
+ "b 60f\n"
+ "54:" // Height 2: Partial direct writeback: partial_2_8
+ "tbz x12, #1, 55f\n"
+ "st1 { v16.h }[4], [x9], #0x2\n"
+ "st1 { v20.h }[4], [x25], #0x2\n"
+ "tbz x12, #0, 60f\n"
+ "st1 { v16.b }[10], [x9]\n"
+ "st1 { v20.b }[10], [x25]\n"
+ "b 60f\n"
+ "55:" // Height 2: Partial direct writeback: partial_1_8
+ "tbz x12, #0, 60f\n"
+ "st1 { v16.b }[8], [x9]\n"
+ "st1 { v20.b }[8], [x25]\n"
+ "b 60f\n"
+ "56:" // Height 2: Partial direct writeback: partial_4_0
+ "tbz x12, #2, 58f\n"
+ "str s16, [x9], #0x4\n"
+ "str s20, [x25], #0x4\n"
+ "tbz x12, #1, 57f\n"
+ "st1 { v16.h }[2], [x9], #0x2\n"
+ "st1 { v20.h }[2], [x25], #0x2\n"
+ "tbz x12, #0, 60f\n"
+ "st1 { v16.b }[6], [x9]\n"
+ "st1 { v20.b }[6], [x25]\n"
+ "b 60f\n"
+ "57:" // Height 2: Partial direct writeback: partial_1_4
+ "tbz x12, #0, 60f\n"
+ "st1 { v16.b }[4], [x9]\n"
+ "st1 { v20.b }[4], [x25]\n"
+ "b 60f\n"
+ "58:" // Height 2: Partial direct writeback: partial_2_0
+ "tbz x12, #1, 59f\n"
+ "str h16, [x9], #0x2\n"
+ "str h20, [x25], #0x2\n"
+ "tbz x12, #0, 60f\n"
+ "st1 { v16.b }[2], [x9]\n"
+ "st1 { v20.b }[2], [x25]\n"
+ "b 60f\n"
+ "59:" // Height 2: Partial direct writeback: partial_1_0
+ "str b16, [x9, #0x0]\n"
+ "str b20, [x25, #0x0]\n"
+ "60:" // Height 2: Partial direct writeback: Done
+ "b 62f\n"
+ "61:" // Height 2: Full writeback
+ "str q16, [x9, #0x0]\n"
+ "str q20, [x25, #0x0]\n"
+ "add x9, x9, #0x10\n"
+ "add x25, x25, #0x10\n"
+ "62:" // Height 2: Writeback done
+ "subs x12, x12, #0x10\n"
+ "bgt 34b\n"
+ "b 126f\n"
+ "63:" // Height 3
+ "movi v11.4s, #0x0\n"
+ "ldr x12, [%x[args_ptr], %[offsetof_N]]\n"
+ "mov x10, %x[col_bias]\n"
+ "movi v12.4s, #0x0\n"
+ "ldr x11, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "bic %x[flags], %x[flags], #0x80000000\n"
+ "movi v13.4s, #0x0\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "movi v14.4s, #0x0\n"
+ "movi v15.16b, #0x1\n"
+ "tbz %x[flags], #2, 64f\n"
+ "ldr x9, [%x[output_ptr], #0x0]\n"
+ "ldr x25, [%x[output_ptr], #0x8]\n"
+ "add x9, x9, x19\n"
+ "ldr x23, [%x[output_ptr], #0x10]\n"
+ "add x25, x25, x19\n"
+ "add x23, x23, x19\n"
+ "b 65f\n"
+ "64:" // Height 3: setup direct output
+ "mov x9, %x[output_ptr]\n"
+ "add x25, x9, x19\n"
+ "add x23, x25, x19\n"
+ "65:" // Height 3: Column loop
+ "movi v16.4s, #0x0\n"
+ "movi v17.4s, #0x0\n"
+ "movi v18.4s, #0x0\n"
+ "movi v19.4s, #0x0\n"
+ "movi v20.4s, #0x0\n"
+ "movi v21.4s, #0x0\n"
+ "movi v22.4s, #0x0\n"
+ "movi v23.4s, #0x0\n"
+ "movi v24.4s, #0x0\n"
+ "movi v25.4s, #0x0\n"
+ "movi v26.4s, #0x0\n"
+ "movi v27.4s, #0x0\n"
+ "66:" // Height 3: setup done
+ "mov x28, #0x0\n"
+ "67:" // Height 3: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w27, [x20, x28, LSL #0x2]\n"
+ "tbz %x[flags], #3, 68f\n"
+ "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x26, [x20, #0x0]\n"
+ "ldr x24, [x20, #0x8]\n"
+ "ldr x22, [x20, #0x10]\n"
+ "cbnz x28, 69f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x26, x26, x19\n"
+ "add x24, x24, x19\n"
+ "add x22, x22, x19\n"
+ "b 69f\n"
+ "68:" // Height 3: setup direct input
+ "mov x26, %x[input_ptr]\n"
+ "add x24, x26, x19\n"
+ "add x22, x24, x19\n"
+ "69:" // Height 3: input setup done
+ "cmp x27, #0x10\n"
+ "blt 74f\n"
+ "cmp x27, #0x20\n"
+ "blt 72f\n"
+ "70:" // Height 3: Multiply loop: Main loop head
+ "ldr q0, [x26, #0x0]\n"
+ "ldr q1, [x24, #0x0]\n"
+ "ldr q2, [x22, #0x0]\n"
+ "ldr q4, [x11, #0x0]\n"
+ ".inst 0x6f80e090 // udot v16.4s, v4.16b, v0.4b[0]\n"
+ "ldr q5, [x11, #0x10]\n"
+ ".inst 0x6f81e094 // udot v20.4s, v4.16b, v1.4b[0]\n"
+ "ldr q6, [x11, #0x20]\n"
+ ".inst 0x6f82e098 // udot v24.4s, v4.16b, v2.4b[0]\n"
+ "ldr q7, [x11, #0x30]\n"
+ "ldr q8, [x11, #0x40]\n"
+ ".inst 0x6f80e0b1 // udot v17.4s, v5.16b, v0.4b[0]\n"
+ "ldr q9, [x11, #0x50]\n"
+ ".inst 0x6f81e0b5 // udot v21.4s, v5.16b, v1.4b[0]\n"
+ "ldr q10, [x11, #0x60]\n"
+ ".inst 0x6f82e0b9 // udot v25.4s, v5.16b, v2.4b[0]\n"
+ "ldr q4, [x11, #0x70]\n"
+ ".inst 0x6f80e0d2 // udot v18.4s, v6.16b, v0.4b[0]\n"
+ "ldr q5, [x11, #0x80]\n"
+ ".inst 0x6f81e0d6 // udot v22.4s, v6.16b, v1.4b[0]\n"
+ "add x26, x26, #0x10\n"
+ ".inst 0x6f82e0da // udot v26.4s, v6.16b, v2.4b[0]\n"
+ "ldr q6, [x11, #0x90]\n"
+ ".inst 0x6f80e0f3 // udot v19.4s, v7.16b, v0.4b[0]\n"
+ "add x24, x24, #0x10\n"
+ ".inst 0x6f81e0f7 // udot v23.4s, v7.16b, v1.4b[0]\n"
+ "add x22, x22, #0x10\n"
+ ".inst 0x6f82e0fb // udot v27.4s, v7.16b, v2.4b[0]\n"
+ "ldr q7, [x11, #0xa0]\n"
+ ".inst 0x6fa0e110 // udot v16.4s, v8.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e114 // udot v20.4s, v8.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e118 // udot v24.4s, v8.16b, v2.4b[1]\n"
+ "ldr q8, [x11, #0xb0]\n"
+ ".inst 0x6fa0e131 // udot v17.4s, v9.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e135 // udot v21.4s, v9.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e139 // udot v25.4s, v9.16b, v2.4b[1]\n"
+ "ldr q9, [x11, #0xc0]\n"
+ ".inst 0x6fa0e152 // udot v18.4s, v10.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e156 // udot v22.4s, v10.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e15a // udot v26.4s, v10.16b, v2.4b[1]\n"
+ "ldr q10, [x11, #0xd0]\n"
+ ".inst 0x6fa0e093 // udot v19.4s, v4.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e097 // udot v23.4s, v4.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e09b // udot v27.4s, v4.16b, v2.4b[1]\n"
+ "ldr q4, [x11, #0xe0]\n"
+ ".inst 0x6f80e8b0 // udot v16.4s, v5.16b, v0.4b[2]\n"
+ ".inst 0x6f81e8b4 // udot v20.4s, v5.16b, v1.4b[2]\n"
+ ".inst 0x6f82e8b8 // udot v24.4s, v5.16b, v2.4b[2]\n"
+ "ldr q5, [x11, #0xf0]\n"
+ ".inst 0x6f80e8d1 // udot v17.4s, v6.16b, v0.4b[2]\n"
+ "add x11, x11, #0x100\n"
+ ".inst 0x6f81e8d5 // udot v21.4s, v6.16b, v1.4b[2]\n"
+ ".inst 0x6f82e8d9 // udot v25.4s, v6.16b, v2.4b[2]\n"
+ ".inst 0x6f80e8f2 // udot v18.4s, v7.16b, v0.4b[2]\n"
+ ".inst 0x6f81e8f6 // udot v22.4s, v7.16b, v1.4b[2]\n"
+ ".inst 0x6f82e8fa // udot v26.4s, v7.16b, v2.4b[2]\n"
+ ".inst 0x6f80e913 // udot v19.4s, v8.16b, v0.4b[2]\n"
+ ".inst 0x6f81e917 // udot v23.4s, v8.16b, v1.4b[2]\n"
+ ".inst 0x6f82e91b // udot v27.4s, v8.16b, v2.4b[2]\n"
+ ".inst 0x6fa0e930 // udot v16.4s, v9.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e934 // udot v20.4s, v9.16b, v1.4b[3]\n"
+ ".inst 0x6fa2e938 // udot v24.4s, v9.16b, v2.4b[3]\n"
+ ".inst 0x6fa0e951 // udot v17.4s, v10.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e955 // udot v21.4s, v10.16b, v1.4b[3]\n"
+ ".inst 0x6fa2e959 // udot v25.4s, v10.16b, v2.4b[3]\n"
+ ".inst 0x6fa0e892 // udot v18.4s, v4.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e896 // udot v22.4s, v4.16b, v1.4b[3]\n"
+ ".inst 0x6fa2e89a // udot v26.4s, v4.16b, v2.4b[3]\n"
+ ".inst 0x6fa0e8b3 // udot v19.4s, v5.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e8b7 // udot v23.4s, v5.16b, v1.4b[3]\n"
+ ".inst 0x6fa2e8bb // udot v27.4s, v5.16b, v2.4b[3]\n"
+ "tbnz %x[flags], #31, 71f\n"
+ ".inst 0x6e8f940b // udot v11.4s, v0.16b, v15.16b\n"
+ ".inst 0x6e8f942c // udot v12.4s, v1.16b, v15.16b\n"
+ ".inst 0x6e8f944d // udot v13.4s, v2.16b, v15.16b\n"
+ "71:" // Height 3: Multiply loop: unique 9: skip row sum
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "sub x27, x27, #0x10\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "cmp x27, #0x20\n"
+ "prfm pldl1keep, [x22, #0x80]\n"
+ "bge 70b\n"
+ "72:" // Height 3: Multiply loop: Single iteration only
+ "sub x27, x27, #0x10\n"
+ "ldr q0, [x26, #0x0]\n"
+ "ldr q1, [x24, #0x0]\n"
+ "ldr q2, [x22, #0x0]\n"
+ "ldr q6, [x11, #0x0]\n"
+ ".inst 0x6f80e0d0 // udot v16.4s, v6.16b, v0.4b[0]\n"
+ "ldr q7, [x11, #0x10]\n"
+ ".inst 0x6f81e0d4 // udot v20.4s, v6.16b, v1.4b[0]\n"
+ "ldr q8, [x11, #0x20]\n"
+ ".inst 0x6f82e0d8 // udot v24.4s, v6.16b, v2.4b[0]\n"
+ "ldr q9, [x11, #0x30]\n"
+ "ldr q10, [x11, #0x40]\n"
+ ".inst 0x6f80e0f1 // udot v17.4s, v7.16b, v0.4b[0]\n"
+ "ldr q4, [x11, #0x50]\n"
+ ".inst 0x6f81e0f5 // udot v21.4s, v7.16b, v1.4b[0]\n"
+ "ldr q5, [x11, #0x60]\n"
+ ".inst 0x6f82e0f9 // udot v25.4s, v7.16b, v2.4b[0]\n"
+ "ldr q6, [x11, #0x70]\n"
+ ".inst 0x6f80e112 // udot v18.4s, v8.16b, v0.4b[0]\n"
+ "ldr q7, [x11, #0x80]\n"
+ ".inst 0x6f81e116 // udot v22.4s, v8.16b, v1.4b[0]\n"
+ "add x26, x26, #0x10\n"
+ ".inst 0x6f82e11a // udot v26.4s, v8.16b, v2.4b[0]\n"
+ "ldr q8, [x11, #0x90]\n"
+ ".inst 0x6f80e133 // udot v19.4s, v9.16b, v0.4b[0]\n"
+ "add x24, x24, #0x10\n"
+ ".inst 0x6f81e137 // udot v23.4s, v9.16b, v1.4b[0]\n"
+ "add x22, x22, #0x10\n"
+ ".inst 0x6f82e13b // udot v27.4s, v9.16b, v2.4b[0]\n"
+ "ldr q9, [x11, #0xa0]\n"
+ ".inst 0x6fa0e150 // udot v16.4s, v10.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e154 // udot v20.4s, v10.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e158 // udot v24.4s, v10.16b, v2.4b[1]\n"
+ "ldr q10, [x11, #0xb0]\n"
+ ".inst 0x6fa0e091 // udot v17.4s, v4.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e095 // udot v21.4s, v4.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e099 // udot v25.4s, v4.16b, v2.4b[1]\n"
+ "ldr q4, [x11, #0xc0]\n"
+ ".inst 0x6fa0e0b2 // udot v18.4s, v5.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e0b6 // udot v22.4s, v5.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e0ba // udot v26.4s, v5.16b, v2.4b[1]\n"
+ "ldr q5, [x11, #0xd0]\n"
+ ".inst 0x6fa0e0d3 // udot v19.4s, v6.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e0d7 // udot v23.4s, v6.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e0db // udot v27.4s, v6.16b, v2.4b[1]\n"
+ "ldr q6, [x11, #0xe0]\n"
+ ".inst 0x6f80e8f0 // udot v16.4s, v7.16b, v0.4b[2]\n"
+ ".inst 0x6f81e8f4 // udot v20.4s, v7.16b, v1.4b[2]\n"
+ ".inst 0x6f82e8f8 // udot v24.4s, v7.16b, v2.4b[2]\n"
+ "ldr q7, [x11, #0xf0]\n"
+ ".inst 0x6f80e911 // udot v17.4s, v8.16b, v0.4b[2]\n"
+ "add x11, x11, #0x100\n"
+ ".inst 0x6f81e915 // udot v21.4s, v8.16b, v1.4b[2]\n"
+ ".inst 0x6f82e919 // udot v25.4s, v8.16b, v2.4b[2]\n"
+ ".inst 0x6f80e932 // udot v18.4s, v9.16b, v0.4b[2]\n"
+ ".inst 0x6f81e936 // udot v22.4s, v9.16b, v1.4b[2]\n"
+ ".inst 0x6f82e93a // udot v26.4s, v9.16b, v2.4b[2]\n"
+ ".inst 0x6f80e953 // udot v19.4s, v10.16b, v0.4b[2]\n"
+ ".inst 0x6f81e957 // udot v23.4s, v10.16b, v1.4b[2]\n"
+ ".inst 0x6f82e95b // udot v27.4s, v10.16b, v2.4b[2]\n"
+ ".inst 0x6fa0e890 // udot v16.4s, v4.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e894 // udot v20.4s, v4.16b, v1.4b[3]\n"
+ ".inst 0x6fa2e898 // udot v24.4s, v4.16b, v2.4b[3]\n"
+ ".inst 0x6fa0e8b1 // udot v17.4s, v5.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e8b5 // udot v21.4s, v5.16b, v1.4b[3]\n"
+ ".inst 0x6fa2e8b9 // udot v25.4s, v5.16b, v2.4b[3]\n"
+ ".inst 0x6fa0e8d2 // udot v18.4s, v6.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e8d6 // udot v22.4s, v6.16b, v1.4b[3]\n"
+ ".inst 0x6fa2e8da // udot v26.4s, v6.16b, v2.4b[3]\n"
+ ".inst 0x6fa0e8f3 // udot v19.4s, v7.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e8f7 // udot v23.4s, v7.16b, v1.4b[3]\n"
+ ".inst 0x6fa2e8fb // udot v27.4s, v7.16b, v2.4b[3]\n"
+ "tbnz %x[flags], #31, 73f\n"
+ ".inst 0x6e8f940b // udot v11.4s, v0.16b, v15.16b\n"
+ ".inst 0x6e8f942c // udot v12.4s, v1.16b, v15.16b\n"
+ ".inst 0x6e8f944d // udot v13.4s, v2.16b, v15.16b\n"
+ "73:" // Height 3: Multiply loop: unique 10: skip row sum
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "prfm pldl1keep, [x22, #0x80]\n"
+ "74:" // Height 3: Multiply loop: Main loop skip
+ "cbz x27, 81f\n"
+ "cmp x27, #0x4\n"
+ "blt 77f\n"
+ "75:" // Height 3: Multiply loop: Odd block loop
+ "ldr s0, [x26], #0x4\n"
+ "ldr s1, [x24], #0x4\n"
+ "ldr s2, [x22], #0x4\n"
+ "tbnz %x[flags], #31, 76f\n"
+ ".inst 0x6e8f940b // udot v11.4s, v0.16b, v15.16b\n"
+ ".inst 0x6e8f942c // udot v12.4s, v1.16b, v15.16b\n"
+ ".inst 0x6e8f944d // udot v13.4s, v2.16b, v15.16b\n"
+ "76:" // Height 3: Multiply loop: unique 11: skip row sum
+ "ldr q8, [x11, #0x0]\n"
+ ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
+ "ldr q9, [x11, #0x10]\n"
+ ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
+ "ldr q10, [x11, #0x20]\n"
+ ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n"
+ "ldr q4, [x11, #0x30]\n"
+ "sub x27, x27, #0x4\n"
+ ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
+ "cmp x27, #0x4\n"
+ ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
+ "add x11, x11, #0x40\n"
+ ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n"
+ ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
+ ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
+ ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n"
+ ".inst 0x6f80e093 // udot v19.4s, v4.16b, v0.4b[0]\n"
+ ".inst 0x6f81e097 // udot v23.4s, v4.16b, v1.4b[0]\n"
+ ".inst 0x6f82e09b // udot v27.4s, v4.16b, v2.4b[0]\n"
+ "bge 75b\n"
+ "cbz x27, 81f\n"
+ "77:" // Height 3: Multiply loop: Skip odd blocks
+ "tbz x27, #1, 78f\n"
+ "ldr h0, [x26], #0x2\n"
+ "ldr h1, [x24], #0x2\n"
+ "ldr h2, [x22], #0x2\n"
+ "tbz x27, #0, 79f\n"
+ "ld1 { v0.b }[2], [x26]\n"
+ "ld1 { v1.b }[2], [x24]\n"
+ "ld1 { v2.b }[2], [x22]\n"
+ "b 79f\n"
+ "78:" // Height 3: Multiply loop: Ragged operand read: partial_1_0
+ "ldr b0, [x26, #0x0]\n"
+ "ldr b1, [x24, #0x0]\n"
+ "ldr b2, [x22, #0x0]\n"
+ "79:" // Height 3: Multiply loop: Ragged operand read: Done
+ "tbnz %x[flags], #31, 80f\n"
+ ".inst 0x6e8f940b // udot v11.4s, v0.16b, v15.16b\n"
+ ".inst 0x6e8f942c // udot v12.4s, v1.16b, v15.16b\n"
+ ".inst 0x6e8f944d // udot v13.4s, v2.16b, v15.16b\n"
+ "80:" // Height 3: Multiply loop: unique 12: skip row sum
+ "ldr q5, [x11, #0x0]\n"
+ ".inst 0x6f80e0b0 // udot v16.4s, v5.16b, v0.4b[0]\n"
+ "ldr q6, [x11, #0x10]\n"
+ ".inst 0x6f81e0b4 // udot v20.4s, v5.16b, v1.4b[0]\n"
+ "ldr q7, [x11, #0x20]\n"
+ ".inst 0x6f82e0b8 // udot v24.4s, v5.16b, v2.4b[0]\n"
+ "ldr q8, [x11, #0x30]\n"
+ "add x11, x11, #0x40\n"
+ ".inst 0x6f80e0d1 // udot v17.4s, v6.16b, v0.4b[0]\n"
+ ".inst 0x6f81e0d5 // udot v21.4s, v6.16b, v1.4b[0]\n"
+ ".inst 0x6f82e0d9 // udot v25.4s, v6.16b, v2.4b[0]\n"
+ ".inst 0x6f80e0f2 // udot v18.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x6f81e0f6 // udot v22.4s, v7.16b, v1.4b[0]\n"
+ ".inst 0x6f82e0fa // udot v26.4s, v7.16b, v2.4b[0]\n"
+ ".inst 0x6f80e113 // udot v19.4s, v8.16b, v0.4b[0]\n"
+ ".inst 0x6f81e117 // udot v23.4s, v8.16b, v1.4b[0]\n"
+ ".inst 0x6f82e11b // udot v27.4s, v8.16b, v2.4b[0]\n"
+ "81:" // Height 3: Multiply loop: No odd multiplies
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "add x28, x28, #0x1\n"
+ "cmp x28, x19\n"
+ "bne 67b\n"
+ "prfm pstl1keep, [x9, #0x0]\n"
+ "prfm pstl1keep, [x25, #0x0]\n"
+ "prfm pstl1keep, [x23, #0x0]\n"
+ "tbnz %x[flags], #31, 82f\n"
+ "addp v11.4s, v11.4s, v11.4s\n"
+ "add x19, %x[qp], %[b_offset]\n"
+ "ld1r { v3.4s }, [x19]\n"
+ "addp v12.4s, v12.4s, v12.4s\n"
+ "addp v13.4s, v13.4s, v13.4s\n"
+ "addp v11.4s, v11.4s, v11.4s\n"
+ "addp v12.4s, v12.4s, v12.4s\n"
+ "addp v13.4s, v13.4s, v13.4s\n"
+ "neg v3.4s, v3.4s\n"
+ "mul v11.4s, v11.4s, v3.4s\n"
+ "mul v12.4s, v12.4s, v3.4s\n"
+ "mul v13.4s, v13.4s, v3.4s\n"
+ "82:" // Height 3: skip row sum fixup
+ "add v16.4s, v16.4s, v11.4s\n"
+ "orr %x[flags], %x[flags], #0x80000000\n"
+ "add v17.4s, v17.4s, v11.4s\n"
+ "ldr q0, [x10, #0x0]\n"
+ "add v18.4s, v18.4s, v11.4s\n"
+ "ldr q1, [x10, #0x10]\n"
+ "add v19.4s, v19.4s, v11.4s\n"
+ "ldr q2, [x10, #0x20]\n"
+ "add v20.4s, v20.4s, v12.4s\n"
+ "ldr q3, [x10, #0x30]\n"
+ "add v21.4s, v21.4s, v12.4s\n"
+ "add x20, %x[qp], %[per_layer_right_shift]\n"
+ "add v22.4s, v22.4s, v12.4s\n"
+ "add x19, %x[qp], %[per_layer_mul]\n"
+ "ld1r { v4.4s }, [x19]\n"
+ "add v23.4s, v23.4s, v12.4s\n"
+ "add x10, x10, #0x40\n"
+ "add v24.4s, v24.4s, v13.4s\n"
+ "add v25.4s, v25.4s, v13.4s\n"
+ "add v26.4s, v26.4s, v13.4s\n"
+ "add v27.4s, v27.4s, v13.4s\n"
+ "add v16.4s, v16.4s, v0.4s\n"
+ "add v17.4s, v17.4s, v1.4s\n"
+ "add v18.4s, v18.4s, v2.4s\n"
+ "add v19.4s, v19.4s, v3.4s\n"
+ "add v20.4s, v20.4s, v0.4s\n"
+ "add v21.4s, v21.4s, v1.4s\n"
+ "add v22.4s, v22.4s, v2.4s\n"
+ "add v23.4s, v23.4s, v3.4s\n"
+ "add v24.4s, v24.4s, v0.4s\n"
+ "ld1r { v0.4s }, [x20]\n"
+ "add v25.4s, v25.4s, v1.4s\n"
+ "add v26.4s, v26.4s, v2.4s\n"
+ "add v27.4s, v27.4s, v3.4s\n"
+ "sqrdmulh v16.4s, v16.4s, v4.4s\n"
+ "sqrdmulh v17.4s, v17.4s, v4.4s\n"
+ "sqrdmulh v18.4s, v18.4s, v4.4s\n"
+ "sqrdmulh v19.4s, v19.4s, v4.4s\n"
+ "sqrdmulh v20.4s, v20.4s, v4.4s\n"
+ "sqrdmulh v21.4s, v21.4s, v4.4s\n"
+ "sqrdmulh v22.4s, v22.4s, v4.4s\n"
+ "sqrdmulh v23.4s, v23.4s, v4.4s\n"
+ "sqrdmulh v24.4s, v24.4s, v4.4s\n"
+ "sqrdmulh v25.4s, v25.4s, v4.4s\n"
+ "sqrdmulh v26.4s, v26.4s, v4.4s\n"
+ "sqrdmulh v27.4s, v27.4s, v4.4s\n"
+ "tbz %x[flags], #5, 83f\n"
+ "and v4.16b, v16.16b, v0.16b\n"
+ "sshr v4.4s, v4.4s, #0x1f\n"
+ "and v5.16b, v17.16b, v0.16b\n"
+ "and v6.16b, v18.16b, v0.16b\n"
+ "sshr v5.4s, v5.4s, #0x1f\n"
+ "and v7.16b, v19.16b, v0.16b\n"
+ "and v8.16b, v20.16b, v0.16b\n"
+ "sshr v6.4s, v6.4s, #0x1f\n"
+ "and v9.16b, v21.16b, v0.16b\n"
+ "sshr v7.4s, v7.4s, #0x1f\n"
+ "sqadd v16.4s, v16.4s, v4.4s\n"
+ "and v10.16b, v22.16b, v0.16b\n"
+ "sshr v8.4s, v8.4s, #0x1f\n"
+ "and v4.16b, v23.16b, v0.16b\n"
+ "sshr v9.4s, v9.4s, #0x1f\n"
+ "sqadd v17.4s, v17.4s, v5.4s\n"
+ "sshr v10.4s, v10.4s, #0x1f\n"
+ "sqadd v18.4s, v18.4s, v6.4s\n"
+ "sshr v4.4s, v4.4s, #0x1f\n"
+ "and v5.16b, v24.16b, v0.16b\n"
+ "sshr v5.4s, v5.4s, #0x1f\n"
+ "sqadd v19.4s, v19.4s, v7.4s\n"
+ "sqadd v20.4s, v20.4s, v8.4s\n"
+ "sqadd v21.4s, v21.4s, v9.4s\n"
+ "sqadd v22.4s, v22.4s, v10.4s\n"
+ "sqadd v23.4s, v23.4s, v4.4s\n"
+ "and v6.16b, v25.16b, v0.16b\n"
+ "sshr v6.4s, v6.4s, #0x1f\n"
+ "sqadd v24.4s, v24.4s, v5.4s\n"
+ "and v7.16b, v26.16b, v0.16b\n"
+ "sshr v7.4s, v7.4s, #0x1f\n"
+ "and v8.16b, v27.16b, v0.16b\n"
+ "sqadd v25.4s, v25.4s, v6.4s\n"
+ "sshr v8.4s, v8.4s, #0x1f\n"
+ "sqadd v26.4s, v26.4s, v7.4s\n"
+ "sqadd v27.4s, v27.4s, v8.4s\n"
+ "83:" // Height 3: no shift correction
+ "srshl v16.4s, v16.4s, v0.4s\n"
+ "add x19, %x[qp], %[c_offset]\n"
+ "ld1r { v4.4s }, [x19]\n"
+ "srshl v17.4s, v17.4s, v0.4s\n"
+ "add x19, %x[qp], %[minval]\n"
+ "srshl v18.4s, v18.4s, v0.4s\n"
+ "ld1r { v5.4s }, [x19]\n"
+ "add x19, %x[qp], %[maxval]\n"
+ "srshl v19.4s, v19.4s, v0.4s\n"
+ "ld1r { v6.4s }, [x19]\n"
+ "cmp x12, #0x10\n"
+ "srshl v20.4s, v20.4s, v0.4s\n"
+ "srshl v21.4s, v21.4s, v0.4s\n"
+ "srshl v22.4s, v22.4s, v0.4s\n"
+ "srshl v23.4s, v23.4s, v0.4s\n"
+ "add v16.4s, v16.4s, v4.4s\n"
+ "add v17.4s, v17.4s, v4.4s\n"
+ "add v18.4s, v18.4s, v4.4s\n"
+ "smin v16.4s, v16.4s, v6.4s\n"
+ "smin v17.4s, v17.4s, v6.4s\n"
+ "smin v18.4s, v18.4s, v6.4s\n"
+ "smax v16.4s, v16.4s, v5.4s\n"
+ "smax v17.4s, v17.4s, v5.4s\n"
+ "smax v18.4s, v18.4s, v5.4s\n"
+ "add v19.4s, v19.4s, v4.4s\n"
+ "add v20.4s, v20.4s, v4.4s\n"
+ "add v21.4s, v21.4s, v4.4s\n"
+ "smin v19.4s, v19.4s, v6.4s\n"
+ "smin v20.4s, v20.4s, v6.4s\n"
+ "smin v21.4s, v21.4s, v6.4s\n"
+ "smax v19.4s, v19.4s, v5.4s\n"
+ "smax v20.4s, v20.4s, v5.4s\n"
+ "smax v21.4s, v21.4s, v5.4s\n"
+ "add v22.4s, v22.4s, v4.4s\n"
+ "add v23.4s, v23.4s, v4.4s\n"
+ "srshl v24.4s, v24.4s, v0.4s\n"
+ "smin v22.4s, v22.4s, v6.4s\n"
+ "smin v23.4s, v23.4s, v6.4s\n"
+ "srshl v25.4s, v25.4s, v0.4s\n"
+ "smax v22.4s, v22.4s, v5.4s\n"
+ "smax v23.4s, v23.4s, v5.4s\n"
+ "add v24.4s, v24.4s, v4.4s\n"
+ "add v25.4s, v25.4s, v4.4s\n"
+ "srshl v26.4s, v26.4s, v0.4s\n"
+ "smin v24.4s, v24.4s, v6.4s\n"
+ "smin v25.4s, v25.4s, v6.4s\n"
+ "srshl v27.4s, v27.4s, v0.4s\n"
+ "smax v24.4s, v24.4s, v5.4s\n"
+ "smax v25.4s, v25.4s, v5.4s\n"
+ "add v26.4s, v26.4s, v4.4s\n"
+ "add v27.4s, v27.4s, v4.4s\n"
+ "uzp1 v16.8h, v16.8h, v17.8h\n"
+ "smin v26.4s, v26.4s, v6.4s\n"
+ "smin v27.4s, v27.4s, v6.4s\n"
+ "uzp1 v17.8h, v18.8h, v19.8h\n"
+ "smax v26.4s, v26.4s, v5.4s\n"
+ "smax v27.4s, v27.4s, v5.4s\n"
+ "uzp1 v20.8h, v20.8h, v21.8h\n"
+ "uzp1 v21.8h, v22.8h, v23.8h\n"
+ "uzp1 v24.8h, v24.8h, v25.8h\n"
+ "uzp1 v25.8h, v26.8h, v27.8h\n"
+ "uzp1 v16.16b, v16.16b, v17.16b\n"
+ "uzp1 v20.16b, v20.16b, v21.16b\n"
+ "uzp1 v24.16b, v24.16b, v25.16b\n"
+ "bge 92f\n"
+ "tbz x12, #3, 87f\n"
+ "str d16, [x9], #0x8\n"
+ "str d20, [x25], #0x8\n"
+ "str d24, [x23], #0x8\n"
+ "tbz x12, #2, 85f\n"
+ "st1 { v16.s }[2], [x9], #0x4\n"
+ "st1 { v20.s }[2], [x25], #0x4\n"
+ "st1 { v24.s }[2], [x23], #0x4\n"
+ "tbz x12, #1, 84f\n"
+ "st1 { v16.h }[6], [x9], #0x2\n"
+ "st1 { v20.h }[6], [x25], #0x2\n"
+ "st1 { v24.h }[6], [x23], #0x2\n"
+ "tbz x12, #0, 91f\n"
+ "st1 { v16.b }[14], [x9]\n"
+ "st1 { v20.b }[14], [x25]\n"
+ "st1 { v24.b }[14], [x23]\n"
+ "b 91f\n"
+ "84:" // Height 3: Partial direct writeback: partial_1_12
+ "tbz x12, #0, 91f\n"
+ "st1 { v16.b }[12], [x9]\n"
+ "st1 { v20.b }[12], [x25]\n"
+ "st1 { v24.b }[12], [x23]\n"
+ "b 91f\n"
+ "85:" // Height 3: Partial direct writeback: partial_2_8
+ "tbz x12, #1, 86f\n"
+ "st1 { v16.h }[4], [x9], #0x2\n"
+ "st1 { v20.h }[4], [x25], #0x2\n"
+ "st1 { v24.h }[4], [x23], #0x2\n"
+ "tbz x12, #0, 91f\n"
+ "st1 { v16.b }[10], [x9]\n"
+ "st1 { v20.b }[10], [x25]\n"
+ "st1 { v24.b }[10], [x23]\n"
+ "b 91f\n"
+ "86:" // Height 3: Partial direct writeback: partial_1_8
+ "tbz x12, #0, 91f\n"
+ "st1 { v16.b }[8], [x9]\n"
+ "st1 { v20.b }[8], [x25]\n"
+ "st1 { v24.b }[8], [x23]\n"
+ "b 91f\n"
+ "87:" // Height 3: Partial direct writeback: partial_4_0
+ "tbz x12, #2, 89f\n"
+ "str s16, [x9], #0x4\n"
+ "str s20, [x25], #0x4\n"
+ "str s24, [x23], #0x4\n"
+ "tbz x12, #1, 88f\n"
+ "st1 { v16.h }[2], [x9], #0x2\n"
+ "st1 { v20.h }[2], [x25], #0x2\n"
+ "st1 { v24.h }[2], [x23], #0x2\n"
+ "tbz x12, #0, 91f\n"
+ "st1 { v16.b }[6], [x9]\n"
+ "st1 { v20.b }[6], [x25]\n"
+ "st1 { v24.b }[6], [x23]\n"
+ "b 91f\n"
+ "88:" // Height 3: Partial direct writeback: partial_1_4
+ "tbz x12, #0, 91f\n"
+ "st1 { v16.b }[4], [x9]\n"
+ "st1 { v20.b }[4], [x25]\n"
+ "st1 { v24.b }[4], [x23]\n"
+ "b 91f\n"
+ "89:" // Height 3: Partial direct writeback: partial_2_0
+ "tbz x12, #1, 90f\n"
+ "str h16, [x9], #0x2\n"
+ "str h20, [x25], #0x2\n"
+ "str h24, [x23], #0x2\n"
+ "tbz x12, #0, 91f\n"
+ "st1 { v16.b }[2], [x9]\n"
+ "st1 { v20.b }[2], [x25]\n"
+ "st1 { v24.b }[2], [x23]\n"
+ "b 91f\n"
+ "90:" // Height 3: Partial direct writeback: partial_1_0
+ "str b16, [x9, #0x0]\n"
+ "str b20, [x25, #0x0]\n"
+ "str b24, [x23, #0x0]\n"
+ "91:" // Height 3: Partial direct writeback: Done
+ "b 93f\n"
+ "92:" // Height 3: Full writeback
+ "str q16, [x9, #0x0]\n"
+ "str q20, [x25, #0x0]\n"
+ "str q24, [x23, #0x0]\n"
+ "add x9, x9, #0x10\n"
+ "add x25, x25, #0x10\n"
+ "add x23, x23, #0x10\n"
+ "93:" // Height 3: Writeback done
+ "subs x12, x12, #0x10\n"
+ "bgt 65b\n"
+ "b 126f\n"
+ "94:" // Height 4
+ "movi v11.4s, #0x0\n"
+ "ldr x12, [%x[args_ptr], %[offsetof_N]]\n"
+ "mov x10, %x[col_bias]\n"
+ "movi v12.4s, #0x0\n"
+ "ldr x11, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "bic %x[flags], %x[flags], #0x80000000\n"
+ "movi v13.4s, #0x0\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "movi v14.4s, #0x0\n"
+ "movi v15.16b, #0x1\n"
+ "tbz %x[flags], #2, 95f\n"
+ "ldr x9, [%x[output_ptr], #0x0]\n"
+ "ldr x25, [%x[output_ptr], #0x8]\n"
+ "add x9, x9, x19\n"
+ "ldr x23, [%x[output_ptr], #0x10]\n"
+ "ldr x21, [%x[output_ptr], #0x18]\n"
+ "add x25, x25, x19\n"
+ "add %x[output_ptr], %x[output_ptr], #0x20\n"
+ "add x23, x23, x19\n"
+ "add x21, x21, x19\n"
+ "b 96f\n"
+ "95:" // Height 4: setup direct output
+ "mov x9, %x[output_ptr]\n"
+ "add x25, x9, x19\n"
+ "add x23, x25, x19\n"
+ "add x21, x23, x19\n"
+ "add %x[output_ptr], x21, x19\n"
+ "96:" // Height 4: Column loop
+ "movi v16.4s, #0x0\n"
+ "movi v17.4s, #0x0\n"
+ "movi v18.4s, #0x0\n"
+ "movi v19.4s, #0x0\n"
+ "movi v20.4s, #0x0\n"
+ "movi v21.4s, #0x0\n"
+ "movi v22.4s, #0x0\n"
+ "movi v23.4s, #0x0\n"
+ "movi v24.4s, #0x0\n"
+ "movi v25.4s, #0x0\n"
+ "movi v26.4s, #0x0\n"
+ "movi v27.4s, #0x0\n"
+ "movi v28.4s, #0x0\n"
+ "movi v29.4s, #0x0\n"
+ "movi v30.4s, #0x0\n"
+ "movi v31.4s, #0x0\n"
+ "97:" // Height 4: setup done
+ "mov x28, #0x0\n"
+ "98:" // Height 4: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w27, [x20, x28, LSL #0x2]\n"
+ "tbz %x[flags], #3, 99f\n"
+ "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x26, [x20, #0x0]\n"
+ "ldr x24, [x20, #0x8]\n"
+ "ldr x22, [x20, #0x10]\n"
+ "ldr x20, [x20, #0x18]\n"
+ "cbnz x28, 100f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x26, x26, x19\n"
+ "add x24, x24, x19\n"
+ "add x22, x22, x19\n"
+ "add x20, x20, x19\n"
+ "b 100f\n"
+ "99:" // Height 4: setup direct input
+ "mov x26, %x[input_ptr]\n"
+ "add x24, x26, x19\n"
+ "add x22, x24, x19\n"
+ "add x20, x22, x19\n"
+ "100:" // Height 4: input setup done
+ "cmp x27, #0x10\n"
+ "blt 105f\n"
+ "cmp x27, #0x20\n"
+ "blt 103f\n"
+ "101:" // Height 4: Multiply loop: Main loop head
+ "ldr q0, [x26, #0x0]\n"
+ "ldr q1, [x24, #0x0]\n"
+ "ldr q2, [x22, #0x0]\n"
+ "ldr q3, [x20, #0x0]\n"
+ "ldr q4, [x11, #0x0]\n"
+ ".inst 0x6f80e090 // udot v16.4s, v4.16b, v0.4b[0]\n"
+ "ldr q5, [x11, #0x10]\n"
+ ".inst 0x6f81e094 // udot v20.4s, v4.16b, v1.4b[0]\n"
+ "ldr q6, [x11, #0x20]\n"
+ ".inst 0x6f82e098 // udot v24.4s, v4.16b, v2.4b[0]\n"
+ "ldr q7, [x11, #0x30]\n"
+ ".inst 0x6f83e09c // udot v28.4s, v4.16b, v3.4b[0]\n"
+ "ldr q8, [x11, #0x40]\n"
+ "ldr q9, [x11, #0x50]\n"
+ ".inst 0x6f80e0b1 // udot v17.4s, v5.16b, v0.4b[0]\n"
+ "ldr q10, [x11, #0x60]\n"
+ ".inst 0x6f81e0b5 // udot v21.4s, v5.16b, v1.4b[0]\n"
+ "ldr q4, [x11, #0x70]\n"
+ ".inst 0x6f82e0b9 // udot v25.4s, v5.16b, v2.4b[0]\n"
+ "add x26, x26, #0x10\n"
+ ".inst 0x6f83e0bd // udot v29.4s, v5.16b, v3.4b[0]\n"
+ "ldr q5, [x11, #0x80]\n"
+ ".inst 0x6f80e0d2 // udot v18.4s, v6.16b, v0.4b[0]\n"
+ "add x24, x24, #0x10\n"
+ ".inst 0x6f81e0d6 // udot v22.4s, v6.16b, v1.4b[0]\n"
+ "add x22, x22, #0x10\n"
+ ".inst 0x6f82e0da // udot v26.4s, v6.16b, v2.4b[0]\n"
+ "add x20, x20, #0x10\n"
+ ".inst 0x6f83e0de // udot v30.4s, v6.16b, v3.4b[0]\n"
+ "ldr q6, [x11, #0x90]\n"
+ ".inst 0x6f80e0f3 // udot v19.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x6f81e0f7 // udot v23.4s, v7.16b, v1.4b[0]\n"
+ ".inst 0x6f82e0fb // udot v27.4s, v7.16b, v2.4b[0]\n"
+ ".inst 0x6f83e0ff // udot v31.4s, v7.16b, v3.4b[0]\n"
+ "ldr q7, [x11, #0xa0]\n"
+ ".inst 0x6fa0e110 // udot v16.4s, v8.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e114 // udot v20.4s, v8.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e118 // udot v24.4s, v8.16b, v2.4b[1]\n"
+ ".inst 0x6fa3e11c // udot v28.4s, v8.16b, v3.4b[1]\n"
+ "ldr q8, [x11, #0xb0]\n"
+ ".inst 0x6fa0e131 // udot v17.4s, v9.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e135 // udot v21.4s, v9.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e139 // udot v25.4s, v9.16b, v2.4b[1]\n"
+ ".inst 0x6fa3e13d // udot v29.4s, v9.16b, v3.4b[1]\n"
+ "ldr q9, [x11, #0xc0]\n"
+ ".inst 0x6fa0e152 // udot v18.4s, v10.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e156 // udot v22.4s, v10.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e15a // udot v26.4s, v10.16b, v2.4b[1]\n"
+ ".inst 0x6fa3e15e // udot v30.4s, v10.16b, v3.4b[1]\n"
+ "ldr q10, [x11, #0xd0]\n"
+ ".inst 0x6fa0e093 // udot v19.4s, v4.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e097 // udot v23.4s, v4.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e09b // udot v27.4s, v4.16b, v2.4b[1]\n"
+ ".inst 0x6fa3e09f // udot v31.4s, v4.16b, v3.4b[1]\n"
+ "ldr q4, [x11, #0xe0]\n"
+ ".inst 0x6f80e8b0 // udot v16.4s, v5.16b, v0.4b[2]\n"
+ ".inst 0x6f81e8b4 // udot v20.4s, v5.16b, v1.4b[2]\n"
+ ".inst 0x6f82e8b8 // udot v24.4s, v5.16b, v2.4b[2]\n"
+ ".inst 0x6f83e8bc // udot v28.4s, v5.16b, v3.4b[2]\n"
+ "ldr q5, [x11, #0xf0]\n"
+ ".inst 0x6f80e8d1 // udot v17.4s, v6.16b, v0.4b[2]\n"
+ "add x11, x11, #0x100\n"
+ ".inst 0x6f81e8d5 // udot v21.4s, v6.16b, v1.4b[2]\n"
+ ".inst 0x6f82e8d9 // udot v25.4s, v6.16b, v2.4b[2]\n"
+ ".inst 0x6f83e8dd // udot v29.4s, v6.16b, v3.4b[2]\n"
+ ".inst 0x6f80e8f2 // udot v18.4s, v7.16b, v0.4b[2]\n"
+ ".inst 0x6f81e8f6 // udot v22.4s, v7.16b, v1.4b[2]\n"
+ ".inst 0x6f82e8fa // udot v26.4s, v7.16b, v2.4b[2]\n"
+ ".inst 0x6f83e8fe // udot v30.4s, v7.16b, v3.4b[2]\n"
+ ".inst 0x6f80e913 // udot v19.4s, v8.16b, v0.4b[2]\n"
+ ".inst 0x6f81e917 // udot v23.4s, v8.16b, v1.4b[2]\n"
+ ".inst 0x6f82e91b // udot v27.4s, v8.16b, v2.4b[2]\n"
+ ".inst 0x6f83e91f // udot v31.4s, v8.16b, v3.4b[2]\n"
+ ".inst 0x6fa0e930 // udot v16.4s, v9.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e934 // udot v20.4s, v9.16b, v1.4b[3]\n"
+ ".inst 0x6fa2e938 // udot v24.4s, v9.16b, v2.4b[3]\n"
+ ".inst 0x6fa3e93c // udot v28.4s, v9.16b, v3.4b[3]\n"
+ ".inst 0x6fa0e951 // udot v17.4s, v10.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e955 // udot v21.4s, v10.16b, v1.4b[3]\n"
+ ".inst 0x6fa2e959 // udot v25.4s, v10.16b, v2.4b[3]\n"
+ ".inst 0x6fa3e95d // udot v29.4s, v10.16b, v3.4b[3]\n"
+ ".inst 0x6fa0e892 // udot v18.4s, v4.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e896 // udot v22.4s, v4.16b, v1.4b[3]\n"
+ ".inst 0x6fa2e89a // udot v26.4s, v4.16b, v2.4b[3]\n"
+ ".inst 0x6fa3e89e // udot v30.4s, v4.16b, v3.4b[3]\n"
+ ".inst 0x6fa0e8b3 // udot v19.4s, v5.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e8b7 // udot v23.4s, v5.16b, v1.4b[3]\n"
+ ".inst 0x6fa2e8bb // udot v27.4s, v5.16b, v2.4b[3]\n"
+ ".inst 0x6fa3e8bf // udot v31.4s, v5.16b, v3.4b[3]\n"
+ "tbnz %x[flags], #31, 102f\n"
+ ".inst 0x6e8f940b // udot v11.4s, v0.16b, v15.16b\n"
+ ".inst 0x6e8f942c // udot v12.4s, v1.16b, v15.16b\n"
+ ".inst 0x6e8f944d // udot v13.4s, v2.16b, v15.16b\n"
+ ".inst 0x6e8f946e // udot v14.4s, v3.16b, v15.16b\n"
+ "102:" // Height 4: Multiply loop: unique 13: skip row sum
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "sub x27, x27, #0x10\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "cmp x27, #0x20\n"
+ "prfm pldl1keep, [x22, #0x80]\n"
+ "prfm pldl1keep, [x20, #0x80]\n"
+ "bge 101b\n"
+ "103:" // Height 4: Multiply loop: Single iteration only
+ "sub x27, x27, #0x10\n"
+ "ldr q0, [x26, #0x0]\n"
+ "ldr q1, [x24, #0x0]\n"
+ "ldr q2, [x22, #0x0]\n"
+ "ldr q3, [x20, #0x0]\n"
+ "ldr q6, [x11, #0x0]\n"
+ ".inst 0x6f80e0d0 // udot v16.4s, v6.16b, v0.4b[0]\n"
+ "ldr q7, [x11, #0x10]\n"
+ ".inst 0x6f81e0d4 // udot v20.4s, v6.16b, v1.4b[0]\n"
+ "ldr q8, [x11, #0x20]\n"
+ ".inst 0x6f82e0d8 // udot v24.4s, v6.16b, v2.4b[0]\n"
+ "ldr q9, [x11, #0x30]\n"
+ ".inst 0x6f83e0dc // udot v28.4s, v6.16b, v3.4b[0]\n"
+ "ldr q10, [x11, #0x40]\n"
+ "ldr q4, [x11, #0x50]\n"
+ ".inst 0x6f80e0f1 // udot v17.4s, v7.16b, v0.4b[0]\n"
+ "ldr q5, [x11, #0x60]\n"
+ ".inst 0x6f81e0f5 // udot v21.4s, v7.16b, v1.4b[0]\n"
+ "ldr q6, [x11, #0x70]\n"
+ ".inst 0x6f82e0f9 // udot v25.4s, v7.16b, v2.4b[0]\n"
+ "add x26, x26, #0x10\n"
+ ".inst 0x6f83e0fd // udot v29.4s, v7.16b, v3.4b[0]\n"
+ "ldr q7, [x11, #0x80]\n"
+ ".inst 0x6f80e112 // udot v18.4s, v8.16b, v0.4b[0]\n"
+ "add x24, x24, #0x10\n"
+ ".inst 0x6f81e116 // udot v22.4s, v8.16b, v1.4b[0]\n"
+ "add x22, x22, #0x10\n"
+ ".inst 0x6f82e11a // udot v26.4s, v8.16b, v2.4b[0]\n"
+ "add x20, x20, #0x10\n"
+ ".inst 0x6f83e11e // udot v30.4s, v8.16b, v3.4b[0]\n"
+ "ldr q8, [x11, #0x90]\n"
+ ".inst 0x6f80e133 // udot v19.4s, v9.16b, v0.4b[0]\n"
+ ".inst 0x6f81e137 // udot v23.4s, v9.16b, v1.4b[0]\n"
+ ".inst 0x6f82e13b // udot v27.4s, v9.16b, v2.4b[0]\n"
+ ".inst 0x6f83e13f // udot v31.4s, v9.16b, v3.4b[0]\n"
+ "ldr q9, [x11, #0xa0]\n"
+ ".inst 0x6fa0e150 // udot v16.4s, v10.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e154 // udot v20.4s, v10.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e158 // udot v24.4s, v10.16b, v2.4b[1]\n"
+ ".inst 0x6fa3e15c // udot v28.4s, v10.16b, v3.4b[1]\n"
+ "ldr q10, [x11, #0xb0]\n"
+ ".inst 0x6fa0e091 // udot v17.4s, v4.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e095 // udot v21.4s, v4.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e099 // udot v25.4s, v4.16b, v2.4b[1]\n"
+ ".inst 0x6fa3e09d // udot v29.4s, v4.16b, v3.4b[1]\n"
+ "ldr q4, [x11, #0xc0]\n"
+ ".inst 0x6fa0e0b2 // udot v18.4s, v5.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e0b6 // udot v22.4s, v5.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e0ba // udot v26.4s, v5.16b, v2.4b[1]\n"
+ ".inst 0x6fa3e0be // udot v30.4s, v5.16b, v3.4b[1]\n"
+ "ldr q5, [x11, #0xd0]\n"
+ ".inst 0x6fa0e0d3 // udot v19.4s, v6.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e0d7 // udot v23.4s, v6.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e0db // udot v27.4s, v6.16b, v2.4b[1]\n"
+ ".inst 0x6fa3e0df // udot v31.4s, v6.16b, v3.4b[1]\n"
+ "ldr q6, [x11, #0xe0]\n"
+ ".inst 0x6f80e8f0 // udot v16.4s, v7.16b, v0.4b[2]\n"
+ ".inst 0x6f81e8f4 // udot v20.4s, v7.16b, v1.4b[2]\n"
+ ".inst 0x6f82e8f8 // udot v24.4s, v7.16b, v2.4b[2]\n"
+ ".inst 0x6f83e8fc // udot v28.4s, v7.16b, v3.4b[2]\n"
+ "ldr q7, [x11, #0xf0]\n"
+ ".inst 0x6f80e911 // udot v17.4s, v8.16b, v0.4b[2]\n"
+ "add x11, x11, #0x100\n"
+ ".inst 0x6f81e915 // udot v21.4s, v8.16b, v1.4b[2]\n"
+ ".inst 0x6f82e919 // udot v25.4s, v8.16b, v2.4b[2]\n"
+ ".inst 0x6f83e91d // udot v29.4s, v8.16b, v3.4b[2]\n"
+ ".inst 0x6f80e932 // udot v18.4s, v9.16b, v0.4b[2]\n"
+ ".inst 0x6f81e936 // udot v22.4s, v9.16b, v1.4b[2]\n"
+ ".inst 0x6f82e93a // udot v26.4s, v9.16b, v2.4b[2]\n"
+ ".inst 0x6f83e93e // udot v30.4s, v9.16b, v3.4b[2]\n"
+ ".inst 0x6f80e953 // udot v19.4s, v10.16b, v0.4b[2]\n"
+ ".inst 0x6f81e957 // udot v23.4s, v10.16b, v1.4b[2]\n"
+ ".inst 0x6f82e95b // udot v27.4s, v10.16b, v2.4b[2]\n"
+ ".inst 0x6f83e95f // udot v31.4s, v10.16b, v3.4b[2]\n"
+ ".inst 0x6fa0e890 // udot v16.4s, v4.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e894 // udot v20.4s, v4.16b, v1.4b[3]\n"
+ ".inst 0x6fa2e898 // udot v24.4s, v4.16b, v2.4b[3]\n"
+ ".inst 0x6fa3e89c // udot v28.4s, v4.16b, v3.4b[3]\n"
+ ".inst 0x6fa0e8b1 // udot v17.4s, v5.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e8b5 // udot v21.4s, v5.16b, v1.4b[3]\n"
+ ".inst 0x6fa2e8b9 // udot v25.4s, v5.16b, v2.4b[3]\n"
+ ".inst 0x6fa3e8bd // udot v29.4s, v5.16b, v3.4b[3]\n"
+ ".inst 0x6fa0e8d2 // udot v18.4s, v6.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e8d6 // udot v22.4s, v6.16b, v1.4b[3]\n"
+ ".inst 0x6fa2e8da // udot v26.4s, v6.16b, v2.4b[3]\n"
+ ".inst 0x6fa3e8de // udot v30.4s, v6.16b, v3.4b[3]\n"
+ ".inst 0x6fa0e8f3 // udot v19.4s, v7.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e8f7 // udot v23.4s, v7.16b, v1.4b[3]\n"
+ ".inst 0x6fa2e8fb // udot v27.4s, v7.16b, v2.4b[3]\n"
+ ".inst 0x6fa3e8ff // udot v31.4s, v7.16b, v3.4b[3]\n"
+ "tbnz %x[flags], #31, 104f\n"
+ ".inst 0x6e8f940b // udot v11.4s, v0.16b, v15.16b\n"
+ ".inst 0x6e8f942c // udot v12.4s, v1.16b, v15.16b\n"
+ ".inst 0x6e8f944d // udot v13.4s, v2.16b, v15.16b\n"
+ ".inst 0x6e8f946e // udot v14.4s, v3.16b, v15.16b\n"
+ "104:" // Height 4: Multiply loop: unique 14: skip row sum
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "prfm pldl1keep, [x22, #0x80]\n"
+ "prfm pldl1keep, [x20, #0x80]\n"
+ "105:" // Height 4: Multiply loop: Main loop skip
+ "cbz x27, 112f\n"
+ "cmp x27, #0x4\n"
+ "blt 108f\n"
+ "106:" // Height 4: Multiply loop: Odd block loop
+ "ldr s0, [x26], #0x4\n"
+ "ldr s1, [x24], #0x4\n"
+ "ldr s2, [x22], #0x4\n"
+ "ldr s3, [x20], #0x4\n"
+ "tbnz %x[flags], #31, 107f\n"
+ ".inst 0x6e8f940b // udot v11.4s, v0.16b, v15.16b\n"
+ ".inst 0x6e8f942c // udot v12.4s, v1.16b, v15.16b\n"
+ ".inst 0x6e8f944d // udot v13.4s, v2.16b, v15.16b\n"
+ ".inst 0x6e8f946e // udot v14.4s, v3.16b, v15.16b\n"
+ "107:" // Height 4: Multiply loop: unique 15: skip row sum
+ "ldr q8, [x11, #0x0]\n"
+ ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
+ "ldr q9, [x11, #0x10]\n"
+ ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
+ "ldr q10, [x11, #0x20]\n"
+ ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n"
+ "ldr q4, [x11, #0x30]\n"
+ ".inst 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n"
+ "sub x27, x27, #0x4\n"
+ "add x11, x11, #0x40\n"
+ ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
+ "cmp x27, #0x4\n"
+ ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
+ ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n"
+ ".inst 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n"
+ ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
+ ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
+ ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n"
+ ".inst 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n"
+ ".inst 0x6f80e093 // udot v19.4s, v4.16b, v0.4b[0]\n"
+ ".inst 0x6f81e097 // udot v23.4s, v4.16b, v1.4b[0]\n"
+ ".inst 0x6f82e09b // udot v27.4s, v4.16b, v2.4b[0]\n"
+ ".inst 0x6f83e09f // udot v31.4s, v4.16b, v3.4b[0]\n"
+ "bge 106b\n"
+ "cbz x27, 112f\n"
+ "108:" // Height 4: Multiply loop: Skip odd blocks
+ "tbz x27, #1, 109f\n"
+ "ldr h0, [x26], #0x2\n"
+ "ldr h1, [x24], #0x2\n"
+ "ldr h2, [x22], #0x2\n"
+ "ldr h3, [x20], #0x2\n"
+ "tbz x27, #0, 110f\n"
+ "ld1 { v0.b }[2], [x26]\n"
+ "ld1 { v1.b }[2], [x24]\n"
+ "ld1 { v2.b }[2], [x22]\n"
+ "ld1 { v3.b }[2], [x20]\n"
+ "b 110f\n"
+ "109:" // Height 4: Multiply loop: Ragged operand read: partial_1_0
+ "ldr b0, [x26, #0x0]\n"
+ "ldr b1, [x24, #0x0]\n"
+ "ldr b2, [x22, #0x0]\n"
+ "ldr b3, [x20, #0x0]\n"
+ "110:" // Height 4: Multiply loop: Ragged operand read: Done
+ "tbnz %x[flags], #31, 111f\n"
+ ".inst 0x6e8f940b // udot v11.4s, v0.16b, v15.16b\n"
+ ".inst 0x6e8f942c // udot v12.4s, v1.16b, v15.16b\n"
+ ".inst 0x6e8f944d // udot v13.4s, v2.16b, v15.16b\n"
+ ".inst 0x6e8f946e // udot v14.4s, v3.16b, v15.16b\n"
+ "111:" // Height 4: Multiply loop: unique 16: skip row sum
+ "ldr q5, [x11, #0x0]\n"
+ ".inst 0x6f80e0b0 // udot v16.4s, v5.16b, v0.4b[0]\n"
+ "ldr q6, [x11, #0x10]\n"
+ ".inst 0x6f81e0b4 // udot v20.4s, v5.16b, v1.4b[0]\n"
+ "ldr q7, [x11, #0x20]\n"
+ ".inst 0x6f82e0b8 // udot v24.4s, v5.16b, v2.4b[0]\n"
+ "ldr q8, [x11, #0x30]\n"
+ ".inst 0x6f83e0bc // udot v28.4s, v5.16b, v3.4b[0]\n"
+ "add x11, x11, #0x40\n"
+ ".inst 0x6f80e0d1 // udot v17.4s, v6.16b, v0.4b[0]\n"
+ ".inst 0x6f81e0d5 // udot v21.4s, v6.16b, v1.4b[0]\n"
+ ".inst 0x6f82e0d9 // udot v25.4s, v6.16b, v2.4b[0]\n"
+ ".inst 0x6f83e0dd // udot v29.4s, v6.16b, v3.4b[0]\n"
+ ".inst 0x6f80e0f2 // udot v18.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x6f81e0f6 // udot v22.4s, v7.16b, v1.4b[0]\n"
+ ".inst 0x6f82e0fa // udot v26.4s, v7.16b, v2.4b[0]\n"
+ ".inst 0x6f83e0fe // udot v30.4s, v7.16b, v3.4b[0]\n"
+ ".inst 0x6f80e113 // udot v19.4s, v8.16b, v0.4b[0]\n"
+ ".inst 0x6f81e117 // udot v23.4s, v8.16b, v1.4b[0]\n"
+ ".inst 0x6f82e11b // udot v27.4s, v8.16b, v2.4b[0]\n"
+ ".inst 0x6f83e11f // udot v31.4s, v8.16b, v3.4b[0]\n"
+ "112:" // Height 4: Multiply loop: No odd multiplies
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "add x28, x28, #0x1\n"
+ "cmp x28, x19\n"
+ "bne 98b\n"
+ "prfm pstl1keep, [x9, #0x0]\n"
+ "prfm pstl1keep, [x25, #0x0]\n"
+ "prfm pstl1keep, [x23, #0x0]\n"
+ "prfm pstl1keep, [x21, #0x0]\n"
+ "tbnz %x[flags], #31, 113f\n"
+ "addp v11.4s, v11.4s, v11.4s\n"
+ "add x19, %x[qp], %[b_offset]\n"
+ "ld1r { v4.4s }, [x19]\n"
+ "addp v12.4s, v12.4s, v12.4s\n"
+ "addp v13.4s, v13.4s, v13.4s\n"
+ "addp v14.4s, v14.4s, v14.4s\n"
+ "addp v11.4s, v11.4s, v11.4s\n"
+ "addp v12.4s, v12.4s, v12.4s\n"
+ "addp v13.4s, v13.4s, v13.4s\n"
+ "addp v14.4s, v14.4s, v14.4s\n"
+ "neg v4.4s, v4.4s\n"
+ "mul v11.4s, v11.4s, v4.4s\n"
+ "mul v12.4s, v12.4s, v4.4s\n"
+ "mul v13.4s, v13.4s, v4.4s\n"
+ "mul v14.4s, v14.4s, v4.4s\n"
+ "113:" // Height 4: skip row sum fixup
+ "add v16.4s, v16.4s, v11.4s\n"
+ "orr %x[flags], %x[flags], #0x80000000\n"
+ "add v17.4s, v17.4s, v11.4s\n"
+ "ldr q0, [x10, #0x0]\n"
+ "add v18.4s, v18.4s, v11.4s\n"
+ "ldr q1, [x10, #0x10]\n"
+ "add v19.4s, v19.4s, v11.4s\n"
+ "ldr q2, [x10, #0x20]\n"
+ "add v20.4s, v20.4s, v12.4s\n"
+ "ldr q3, [x10, #0x30]\n"
+ "add v21.4s, v21.4s, v12.4s\n"
+ "add x20, %x[qp], %[per_layer_right_shift]\n"
+ "add v22.4s, v22.4s, v12.4s\n"
+ "add x19, %x[qp], %[per_layer_mul]\n"
+ "ld1r { v4.4s }, [x19]\n"
+ "add v23.4s, v23.4s, v12.4s\n"
+ "add x10, x10, #0x40\n"
+ "add v24.4s, v24.4s, v13.4s\n"
+ "add v25.4s, v25.4s, v13.4s\n"
+ "add v26.4s, v26.4s, v13.4s\n"
+ "add v27.4s, v27.4s, v13.4s\n"
+ "add v28.4s, v28.4s, v14.4s\n"
+ "add v29.4s, v29.4s, v14.4s\n"
+ "add v30.4s, v30.4s, v14.4s\n"
+ "add v31.4s, v31.4s, v14.4s\n"
+ "add v16.4s, v16.4s, v0.4s\n"
+ "add v17.4s, v17.4s, v1.4s\n"
+ "add v18.4s, v18.4s, v2.4s\n"
+ "add v19.4s, v19.4s, v3.4s\n"
+ "add v20.4s, v20.4s, v0.4s\n"
+ "add v21.4s, v21.4s, v1.4s\n"
+ "add v22.4s, v22.4s, v2.4s\n"
+ "add v23.4s, v23.4s, v3.4s\n"
+ "add v24.4s, v24.4s, v0.4s\n"
+ "add v25.4s, v25.4s, v1.4s\n"
+ "add v26.4s, v26.4s, v2.4s\n"
+ "add v27.4s, v27.4s, v3.4s\n"
+ "add v28.4s, v28.4s, v0.4s\n"
+ "ld1r { v0.4s }, [x20]\n"
+ "add v29.4s, v29.4s, v1.4s\n"
+ "add v30.4s, v30.4s, v2.4s\n"
+ "add v31.4s, v31.4s, v3.4s\n"
+ "sqrdmulh v16.4s, v16.4s, v4.4s\n"
+ "sqrdmulh v17.4s, v17.4s, v4.4s\n"
+ "sqrdmulh v18.4s, v18.4s, v4.4s\n"
+ "sqrdmulh v19.4s, v19.4s, v4.4s\n"
+ "sqrdmulh v20.4s, v20.4s, v4.4s\n"
+ "sqrdmulh v21.4s, v21.4s, v4.4s\n"
+ "sqrdmulh v22.4s, v22.4s, v4.4s\n"
+ "sqrdmulh v23.4s, v23.4s, v4.4s\n"
+ "sqrdmulh v24.4s, v24.4s, v4.4s\n"
+ "sqrdmulh v25.4s, v25.4s, v4.4s\n"
+ "sqrdmulh v26.4s, v26.4s, v4.4s\n"
+ "sqrdmulh v27.4s, v27.4s, v4.4s\n"
+ "sqrdmulh v28.4s, v28.4s, v4.4s\n"
+ "sqrdmulh v29.4s, v29.4s, v4.4s\n"
+ "sqrdmulh v30.4s, v30.4s, v4.4s\n"
+ "sqrdmulh v31.4s, v31.4s, v4.4s\n"
+ "tbz %x[flags], #5, 114f\n"
+ "and v4.16b, v16.16b, v0.16b\n"
+ "sshr v4.4s, v4.4s, #0x1f\n"
+ "and v5.16b, v17.16b, v0.16b\n"
+ "and v6.16b, v18.16b, v0.16b\n"
+ "sshr v5.4s, v5.4s, #0x1f\n"
+ "and v7.16b, v19.16b, v0.16b\n"
+ "and v8.16b, v20.16b, v0.16b\n"
+ "sshr v6.4s, v6.4s, #0x1f\n"
+ "and v9.16b, v21.16b, v0.16b\n"
+ "sshr v7.4s, v7.4s, #0x1f\n"
+ "sqadd v16.4s, v16.4s, v4.4s\n"
+ "and v10.16b, v22.16b, v0.16b\n"
+ "sshr v8.4s, v8.4s, #0x1f\n"
+ "and v4.16b, v23.16b, v0.16b\n"
+ "sshr v9.4s, v9.4s, #0x1f\n"
+ "sqadd v17.4s, v17.4s, v5.4s\n"
+ "sshr v10.4s, v10.4s, #0x1f\n"
+ "sqadd v18.4s, v18.4s, v6.4s\n"
+ "sshr v4.4s, v4.4s, #0x1f\n"
+ "and v5.16b, v24.16b, v0.16b\n"
+ "sshr v5.4s, v5.4s, #0x1f\n"
+ "sqadd v19.4s, v19.4s, v7.4s\n"
+ "sqadd v20.4s, v20.4s, v8.4s\n"
+ "sqadd v21.4s, v21.4s, v9.4s\n"
+ "sqadd v22.4s, v22.4s, v10.4s\n"
+ "sqadd v23.4s, v23.4s, v4.4s\n"
+ "and v6.16b, v25.16b, v0.16b\n"
+ "sshr v6.4s, v6.4s, #0x1f\n"
+ "sqadd v24.4s, v24.4s, v5.4s\n"
+ "and v7.16b, v26.16b, v0.16b\n"
+ "sshr v7.4s, v7.4s, #0x1f\n"
+ "and v8.16b, v27.16b, v0.16b\n"
+ "and v9.16b, v28.16b, v0.16b\n"
+ "sshr v8.4s, v8.4s, #0x1f\n"
+ "sqadd v25.4s, v25.4s, v6.4s\n"
+ "and v10.16b, v29.16b, v0.16b\n"
+ "sshr v9.4s, v9.4s, #0x1f\n"
+ "and v4.16b, v30.16b, v0.16b\n"
+ "sshr v10.4s, v10.4s, #0x1f\n"
+ "sqadd v26.4s, v26.4s, v7.4s\n"
+ "and v5.16b, v31.16b, v0.16b\n"
+ "sshr v4.4s, v4.4s, #0x1f\n"
+ "sqadd v27.4s, v27.4s, v8.4s\n"
+ "sshr v5.4s, v5.4s, #0x1f\n"
+ "sqadd v28.4s, v28.4s, v9.4s\n"
+ "sqadd v29.4s, v29.4s, v10.4s\n"
+ "sqadd v30.4s, v30.4s, v4.4s\n"
+ "sqadd v31.4s, v31.4s, v5.4s\n"
+ "114:" // Height 4: no shift correction
+ "srshl v16.4s, v16.4s, v0.4s\n"
+ "add x19, %x[qp], %[c_offset]\n"
+ "ld1r { v4.4s }, [x19]\n"
+ "srshl v17.4s, v17.4s, v0.4s\n"
+ "add x19, %x[qp], %[minval]\n"
+ "srshl v18.4s, v18.4s, v0.4s\n"
+ "ld1r { v5.4s }, [x19]\n"
+ "add x19, %x[qp], %[maxval]\n"
+ "srshl v19.4s, v19.4s, v0.4s\n"
+ "ld1r { v6.4s }, [x19]\n"
+ "cmp x12, #0x10\n"
+ "srshl v20.4s, v20.4s, v0.4s\n"
+ "srshl v21.4s, v21.4s, v0.4s\n"
+ "srshl v22.4s, v22.4s, v0.4s\n"
+ "srshl v23.4s, v23.4s, v0.4s\n"
+ "add v16.4s, v16.4s, v4.4s\n"
+ "add v17.4s, v17.4s, v4.4s\n"
+ "add v18.4s, v18.4s, v4.4s\n"
+ "smin v16.4s, v16.4s, v6.4s\n"
+ "smin v17.4s, v17.4s, v6.4s\n"
+ "smin v18.4s, v18.4s, v6.4s\n"
+ "smax v16.4s, v16.4s, v5.4s\n"
+ "smax v17.4s, v17.4s, v5.4s\n"
+ "smax v18.4s, v18.4s, v5.4s\n"
+ "add v19.4s, v19.4s, v4.4s\n"
+ "add v20.4s, v20.4s, v4.4s\n"
+ "add v21.4s, v21.4s, v4.4s\n"
+ "smin v19.4s, v19.4s, v6.4s\n"
+ "smin v20.4s, v20.4s, v6.4s\n"
+ "smin v21.4s, v21.4s, v6.4s\n"
+ "smax v19.4s, v19.4s, v5.4s\n"
+ "smax v20.4s, v20.4s, v5.4s\n"
+ "smax v21.4s, v21.4s, v5.4s\n"
+ "add v22.4s, v22.4s, v4.4s\n"
+ "add v23.4s, v23.4s, v4.4s\n"
+ "srshl v24.4s, v24.4s, v0.4s\n"
+ "smin v22.4s, v22.4s, v6.4s\n"
+ "smin v23.4s, v23.4s, v6.4s\n"
+ "srshl v25.4s, v25.4s, v0.4s\n"
+ "smax v22.4s, v22.4s, v5.4s\n"
+ "smax v23.4s, v23.4s, v5.4s\n"
+ "add v24.4s, v24.4s, v4.4s\n"
+ "add v25.4s, v25.4s, v4.4s\n"
+ "srshl v26.4s, v26.4s, v0.4s\n"
+ "smin v24.4s, v24.4s, v6.4s\n"
+ "smin v25.4s, v25.4s, v6.4s\n"
+ "srshl v27.4s, v27.4s, v0.4s\n"
+ "smax v24.4s, v24.4s, v5.4s\n"
+ "smax v25.4s, v25.4s, v5.4s\n"
+ "add v26.4s, v26.4s, v4.4s\n"
+ "add v27.4s, v27.4s, v4.4s\n"
+ "srshl v28.4s, v28.4s, v0.4s\n"
+ "smin v26.4s, v26.4s, v6.4s\n"
+ "smin v27.4s, v27.4s, v6.4s\n"
+ "srshl v29.4s, v29.4s, v0.4s\n"
+ "smax v26.4s, v26.4s, v5.4s\n"
+ "smax v27.4s, v27.4s, v5.4s\n"
+ "add v28.4s, v28.4s, v4.4s\n"
+ "add v29.4s, v29.4s, v4.4s\n"
+ "srshl v30.4s, v30.4s, v0.4s\n"
+ "smin v28.4s, v28.4s, v6.4s\n"
+ "smin v29.4s, v29.4s, v6.4s\n"
+ "srshl v31.4s, v31.4s, v0.4s\n"
+ "smax v28.4s, v28.4s, v5.4s\n"
+ "smax v29.4s, v29.4s, v5.4s\n"
+ "add v30.4s, v30.4s, v4.4s\n"
+ "add v31.4s, v31.4s, v4.4s\n"
+ "uzp1 v16.8h, v16.8h, v17.8h\n"
+ "smin v30.4s, v30.4s, v6.4s\n"
+ "smin v31.4s, v31.4s, v6.4s\n"
+ "uzp1 v17.8h, v18.8h, v19.8h\n"
+ "smax v30.4s, v30.4s, v5.4s\n"
+ "smax v31.4s, v31.4s, v5.4s\n"
+ "uzp1 v20.8h, v20.8h, v21.8h\n"
+ "uzp1 v21.8h, v22.8h, v23.8h\n"
+ "uzp1 v24.8h, v24.8h, v25.8h\n"
+ "uzp1 v25.8h, v26.8h, v27.8h\n"
+ "uzp1 v28.8h, v28.8h, v29.8h\n"
+ "uzp1 v29.8h, v30.8h, v31.8h\n"
+ "uzp1 v16.16b, v16.16b, v17.16b\n"
+ "uzp1 v20.16b, v20.16b, v21.16b\n"
+ "uzp1 v24.16b, v24.16b, v25.16b\n"
+ "uzp1 v28.16b, v28.16b, v29.16b\n"
+ "bge 123f\n"
+ "tbz x12, #3, 118f\n"
+ "str d16, [x9], #0x8\n"
+ "str d20, [x25], #0x8\n"
+ "str d24, [x23], #0x8\n"
+ "str d28, [x21], #0x8\n"
+ "tbz x12, #2, 116f\n"
+ "st1 { v16.s }[2], [x9], #0x4\n"
+ "st1 { v20.s }[2], [x25], #0x4\n"
+ "st1 { v24.s }[2], [x23], #0x4\n"
+ "st1 { v28.s }[2], [x21], #0x4\n"
+ "tbz x12, #1, 115f\n"
+ "st1 { v16.h }[6], [x9], #0x2\n"
+ "st1 { v20.h }[6], [x25], #0x2\n"
+ "st1 { v24.h }[6], [x23], #0x2\n"
+ "st1 { v28.h }[6], [x21], #0x2\n"
+ "tbz x12, #0, 122f\n"
+ "st1 { v16.b }[14], [x9]\n"
+ "st1 { v20.b }[14], [x25]\n"
+ "st1 { v24.b }[14], [x23]\n"
+ "st1 { v28.b }[14], [x21]\n"
+ "b 122f\n"
+ "115:" // Height 4: Partial direct writeback: partial_1_12
+ "tbz x12, #0, 122f\n"
+ "st1 { v16.b }[12], [x9]\n"
+ "st1 { v20.b }[12], [x25]\n"
+ "st1 { v24.b }[12], [x23]\n"
+ "st1 { v28.b }[12], [x21]\n"
+ "b 122f\n"
+ "116:" // Height 4: Partial direct writeback: partial_2_8
+ "tbz x12, #1, 117f\n"
+ "st1 { v16.h }[4], [x9], #0x2\n"
+ "st1 { v20.h }[4], [x25], #0x2\n"
+ "st1 { v24.h }[4], [x23], #0x2\n"
+ "st1 { v28.h }[4], [x21], #0x2\n"
+ "tbz x12, #0, 122f\n"
+ "st1 { v16.b }[10], [x9]\n"
+ "st1 { v20.b }[10], [x25]\n"
+ "st1 { v24.b }[10], [x23]\n"
+ "st1 { v28.b }[10], [x21]\n"
+ "b 122f\n"
+ "117:" // Height 4: Partial direct writeback: partial_1_8
+ "tbz x12, #0, 122f\n"
+ "st1 { v16.b }[8], [x9]\n"
+ "st1 { v20.b }[8], [x25]\n"
+ "st1 { v24.b }[8], [x23]\n"
+ "st1 { v28.b }[8], [x21]\n"
+ "b 122f\n"
+ "118:" // Height 4: Partial direct writeback: partial_4_0
+ "tbz x12, #2, 120f\n"
+ "str s16, [x9], #0x4\n"
+ "str s20, [x25], #0x4\n"
+ "str s24, [x23], #0x4\n"
+ "str s28, [x21], #0x4\n"
+ "tbz x12, #1, 119f\n"
+ "st1 { v16.h }[2], [x9], #0x2\n"
+ "st1 { v20.h }[2], [x25], #0x2\n"
+ "st1 { v24.h }[2], [x23], #0x2\n"
+ "st1 { v28.h }[2], [x21], #0x2\n"
+ "tbz x12, #0, 122f\n"
+ "st1 { v16.b }[6], [x9]\n"
+ "st1 { v20.b }[6], [x25]\n"
+ "st1 { v24.b }[6], [x23]\n"
+ "st1 { v28.b }[6], [x21]\n"
+ "b 122f\n"
+ "119:" // Height 4: Partial direct writeback: partial_1_4
+ "tbz x12, #0, 122f\n"
+ "st1 { v16.b }[4], [x9]\n"
+ "st1 { v20.b }[4], [x25]\n"
+ "st1 { v24.b }[4], [x23]\n"
+ "st1 { v28.b }[4], [x21]\n"
+ "b 122f\n"
+ "120:" // Height 4: Partial direct writeback: partial_2_0
+ "tbz x12, #1, 121f\n"
+ "str h16, [x9], #0x2\n"
+ "str h20, [x25], #0x2\n"
+ "str h24, [x23], #0x2\n"
+ "str h28, [x21], #0x2\n"
+ "tbz x12, #0, 122f\n"
+ "st1 { v16.b }[2], [x9]\n"
+ "st1 { v20.b }[2], [x25]\n"
+ "st1 { v24.b }[2], [x23]\n"
+ "st1 { v28.b }[2], [x21]\n"
+ "b 122f\n"
+ "121:" // Height 4: Partial direct writeback: partial_1_0
+ "str b16, [x9, #0x0]\n"
+ "str b20, [x25, #0x0]\n"
+ "str b24, [x23, #0x0]\n"
+ "str b28, [x21, #0x0]\n"
+ "122:" // Height 4: Partial direct writeback: Done
+ "b 124f\n"
+ "123:" // Height 4: Full writeback
+ "str q16, [x9, #0x0]\n"
+ "str q20, [x25, #0x0]\n"
+ "str q24, [x23, #0x0]\n"
+ "str q28, [x21, #0x0]\n"
+ "add x9, x9, #0x10\n"
+ "add x25, x25, #0x10\n"
+ "add x23, x23, #0x10\n"
+ "add x21, x21, #0x10\n"
+ "124:" // Height 4: Writeback done
+ "subs x12, x12, #0x10\n"
+ "bgt 96b\n"
+ "subs %x[M], %x[M], #0x4\n"
+ "beq 126f\n"
+ "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "tbz %x[flags], #3, 125f\n"
+ "add x20, x20, #0x4\n"
+ "str x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "b 1b\n"
+ "125:" // Update direct input
+ "mov x19, #0x4\n"
+ "madd %x[input_ptr], x19, x20, %x[input_ptr]\n"
+ "b 1b\n"
+ "126:" // Exit
+
+ : [M] "+r" (M), [flags] "+r" (flags), [input_ptr] "+r" (input_ptr), [output_ptr] "+r" (output_ptr)
+ : [args_ptr] "r" (&ka), [b_offset] "I" (offsetof(Requantize32, b_offset)), [c_offset] "I" (offsetof(Requantize32, c_offset)), [col_bias] "r" (col_bias), [maxval] "I" (offsetof(Requantize32, maxval)), [minval] "I" (offsetof(Requantize32, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths)), [per_layer_mul] "I" (offsetof(Requantize32, per_layer_mul)), [per_layer_right_shift] "I" (offsetof(Requantize32, per_layer_right_shift)), [qp] "r" (qp)
+ : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x12", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
+ );
+}
+
+} // namespace arm_gemm
+#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_dot_16x4.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_dot_16x4.hpp
deleted file mode 100644
index e5a88b4..0000000
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_dot_16x4.hpp
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- * Copyright (c) 2018-2020 Arm Limited.
- *
- * SPDX-License-Identifier: MIT
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to
- * deal in the Software without restriction, including without limitation the
- * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
- * sell copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-#pragma once
-
-#ifdef __aarch64__
-
-#include <cstdint>
-#include "../std_transforms_fixed.hpp"
-
-namespace arm_gemm
-{
-
-// Actual kernel implementations
-void a64_hybrid_u8u32_dot_16x4(const uint8_t *, int, const uint8_t *, uint32_t *, int, int, int, int, const uint32_t *, Activation, bool);
-void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *, int, const uint8_t *, uint32_t *, int, int, int, int, const uint32_t *, Activation, bool);
-
-class hybrid_u8u32_dot_16x4
-{
-public:
- typedef uint8_t operand_type;
- typedef uint32_t result_type;
-
- typedef void (*kern_type)(const uint8_t *, int, const uint8_t *, uint32_t *, int, int, int, int, const uint32_t *, Activation, bool);
-
- /* Kernel blocking parameters */
- static constexpr unsigned int out_height()
- {
- return 4;
- }
-
- static unsigned int out_width()
- {
- return 16;
- }
-
- static constexpr unsigned int k_unroll()
- {
- return 4;
- }
-
- static constexpr bool supports_accumulate()
- {
- return true;
- }
-
- static constexpr bool supports_bias()
- {
- return false;
- }
-
- static constexpr bool supports_activation()
- {
- return false;
- }
-
- StdTransformsFixed<operand_type, result_type, 4, 16, 4> transforms = {};
-
- // Default to the generic kernel
- kern_type kernel=a64_hybrid_u8u32_dot_16x4;
-
- hybrid_u8u32_dot_16x4(const CPUInfo *ci)
- {
- if (ci->get_cpu_model() == CPUModel::A55r1) {
- kernel = a64_hybrid_u8u32_dot_16x4_a55;
- }
- }
-};
-
-} // namespace arm_gemm
-
-#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_dot_16x4/a55.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_dot_16x4/a55.cpp
deleted file mode 100644
index 735e5fd..0000000
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_dot_16x4/a55.cpp
+++ /dev/null
@@ -1,2434 +0,0 @@
-/*
- * Copyright (c) 2018-2020 Arm Limited.
- *
- * SPDX-License-Identifier: MIT
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to
- * deal in the Software without restriction, including without limitation the
- * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
- * sell copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-#ifdef __aarch64__
-
-#include <algorithm>
-
-#include "arm_gemm.hpp"
-#include <cstdint>
-#include "../../asmlib.hpp"
-#include "../../utils.hpp"
-
-namespace arm_gemm {
-
-void a64_hybrid_u8u32_dot_16x4_a55(const uint8_t *A, int lda, const uint8_t *B, uint32_t *C, int ldc, int M, int N, int K, const uint32_t *, Activation , bool accumulate) {
- const int K_stride = ((K + 3) / 4) * 4;
- const long loops_count = ((K + 16) / 32) - 1;
- K -= loops_count * 32;
- const long regs_count = (K / 16) - 1;
- K -= (regs_count + 1) * 16;
- const long blocks_count = K / 4;
- const long odds_count = K - (blocks_count * 4);
-
- int rows_to_compute;
-
- for (int y=0; y<M; y+=rows_to_compute) {
- const uint8_t * const a_ptr0_base = A + (y * lda);
- const unsigned long ldab = lda * sizeof(uint8_t);
-
- uint32_t *c_ptr0 = C + (y * ldc);
-
- rows_to_compute = M-y;
- if (rows_to_compute > 4) {
- if (rows_to_compute % 4) {
- rows_to_compute = 4 - 1;
- } else {
- rows_to_compute = 4;
- }
- }
-
- for (int x0=0; x0<N; x0+=16ul) {
- const long width = std::min((unsigned long)N-x0, 16ul);
- long loops = loops_count;
- long regs = regs_count;
- long blocks = blocks_count;
- long odds = odds_count;
- const uint8_t *a_ptr0 = a_ptr0_base;
- const uint8_t *b_ptr0 = B + (K_stride * x0);
- const bool use_result_buffer = (width < 16);
- uint32_t result_buffer[64];
- const unsigned long ldcb = (use_result_buffer ? 16 : ldc) * sizeof(uint32_t);
- uint32_t *c_ptr_real = c_ptr0;
- if (use_result_buffer && accumulate) {
- for(int cy=0; cy<std::min(M-y, 4); cy++) {
- for(unsigned int cx=0; cx<width; cx++) {
- result_buffer[cy * 16 + cx] = c_ptr_real[cy * ldc + cx];
- }
- }
- }
- if (use_result_buffer) {
- c_ptr0 = result_buffer;
- }
-
- switch(rows_to_compute) {
- case 1:
- __asm __volatile (
- "temploadreg0 .req X0\n"
- "temploadreg1 .req X1\n"
- "temploadreg2 .req X2\n"
- "temploadreg3 .req X3\n"
- "cbnz %[accumulate], 1f\n"
- "movi v16.4s, #0\n"
- "ldr q0, [%[a_ptr0]]\n"
- "movi v17.4s, #0\n"
- "ldr q8, [%[b_ptr0]]\n"
- "movi v18.4s, #0\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "movi v19.4s, #0\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "ldr q16, [%[c_ptr0]]\n"
- "ldr q17, [%[c_ptr0], #0x10]\n"
- "ldr q18, [%[c_ptr0], #0x20]\n"
- "ldr q19, [%[c_ptr0], #0x30]\n"
- "ldr q0, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ldr q8, [%[b_ptr0]]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "cbz %[loops], 2f\n"
- "3:\n"
- ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
- "ins v14.d[1], temploadreg2\n"
- ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
- "ldr d15, [%[b_ptr0], #-0x10]\n"
- ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
- "ldr temploadreg3, [%[b_ptr0], #-0x8]\n"
- ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
- "ldr d4, [%[a_ptr0]]\n"
- ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n"
- "ldr temploadreg0, [%[a_ptr0], #0x8]\n"
- ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n"
- "ldr d8, [%[b_ptr0]]\n"
- ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n"
- "ldr d9, [%[b_ptr0], #0x10]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ins v4.d[1], temploadreg0\n"
- "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
- "ldr d10, [%[b_ptr0], #0x20]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- "ldr d11, [%[b_ptr0], #0x30]\n"
- "ins v15.d[1], temploadreg3\n"
- "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- "ldr d12, [%[b_ptr0], #0x40]\n"
- "ins v8.d[1], temploadreg0\n"
- ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- "ldr d13, [%[b_ptr0], #0x50]\n"
- "ins v9.d[1], temploadreg1\n"
- ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- "ins v10.d[1], temploadreg2\n"
- ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- "ldr d15, [%[b_ptr0], #0x70]\n"
- "ins v11.d[1], temploadreg3\n"
- ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- "ins v12.d[1], temploadreg0\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n"
- "ldr d8, [%[b_ptr0], #-0x80]\n"
- "ldr temploadreg0, [%[b_ptr0], #-0x78]\n"
- ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n"
- "ldr d9, [%[b_ptr0], #-0x70]\n"
- "ins v13.d[1], temploadreg1\n"
- "ldr temploadreg1, [%[b_ptr0], #-0x68]\n"
- "ldr d10, [%[b_ptr0], #-0x60]\n"
- "ins v14.d[1], temploadreg2\n"
- ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n"
- "ldr temploadreg2, [%[b_ptr0], #-0x58]\n"
- "ldr d11, [%[b_ptr0], #-0x50]\n"
- "ins v15.d[1], temploadreg3\n"
- ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n"
- "ldr temploadreg3, [%[b_ptr0], #-0x48]\n"
- "ldr d12, [%[b_ptr0], #-0x40]\n"
- "ins v8.d[1], temploadreg0\n"
- ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n"
- "ldr temploadreg0, [%[b_ptr0], #-0x38]\n"
- "ldr d13, [%[b_ptr0], #-0x30]\n"
- "ins v9.d[1], temploadreg1\n"
- ".inst 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n"
- "ldr temploadreg1, [%[b_ptr0], #-0x28]\n"
- "ldr d14, [%[b_ptr0], #-0x20]\n"
- "ins v10.d[1], temploadreg2\n"
- ".inst 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n"
- "ldr temploadreg2, [%[b_ptr0], #-0x18]\n"
- "ldr d15, [%[b_ptr0], #-0x10]\n"
- "ins v11.d[1], temploadreg3\n"
- ".inst 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n"
- "ldr temploadreg3, [%[b_ptr0], #-0x8]\n"
- "ldr d0, [%[a_ptr0], #-0x10]\n"
- "ins v12.d[1], temploadreg0\n"
- ".inst 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n"
- "ldr temploadreg0, [%[a_ptr0], #-0x8]\n"
- "ldr d8, [%[b_ptr0]]\n"
- "ldr d9, [%[b_ptr0], #0x10]\n"
- ".inst 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n"
- "ins v0.d[1], temploadreg0\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- "ins v13.d[1], temploadreg1\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- "ldr d10, [%[b_ptr0], #0x20]\n"
- "ins v14.d[1], temploadreg2\n"
- ".inst 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- "ldr d11, [%[b_ptr0], #0x30]\n"
- "ins v15.d[1], temploadreg3\n"
- ".inst 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- "ldr d12, [%[b_ptr0], #0x40]\n"
- "ins v8.d[1], temploadreg0\n"
- ".inst 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- "ldr d13, [%[b_ptr0], #0x50]\n"
- "ins v9.d[1], temploadreg1\n"
- ".inst 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- "ins v10.d[1], temploadreg2\n"
- ".inst 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- "ldr d15, [%[b_ptr0], #0x70]\n"
- "ins v11.d[1], temploadreg3\n"
- ".inst 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- "ins v12.d[1], temploadreg0\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".inst 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n"
- "ldr d8, [%[b_ptr0], #-0x80]\n"
- "ldr temploadreg0, [%[b_ptr0], #-0x78]\n"
- ".inst 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n"
- "ldr d9, [%[b_ptr0], #-0x70]\n"
- "ins v13.d[1], temploadreg1\n"
- "ldr temploadreg1, [%[b_ptr0], #-0x68]\n"
- "ldr d10, [%[b_ptr0], #-0x60]\n"
- "ins v14.d[1], temploadreg2\n"
- ".inst 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n"
- "ldr temploadreg2, [%[b_ptr0], #-0x58]\n"
- "ldr d11, [%[b_ptr0], #-0x50]\n"
- "ins v15.d[1], temploadreg3\n"
- ".inst 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n"
- "ldr temploadreg3, [%[b_ptr0], #-0x48]\n"
- "ldr d12, [%[b_ptr0], #-0x40]\n"
- "ins v8.d[1], temploadreg0\n"
- ".inst 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n"
- "ldr temploadreg0, [%[b_ptr0], #-0x38]\n"
- "ldr d13, [%[b_ptr0], #-0x30]\n"
- "ins v9.d[1], temploadreg1\n"
- "ldr temploadreg1, [%[b_ptr0], #-0x28]\n"
- "ldr d14, [%[b_ptr0], #-0x20]\n"
- "ins v10.d[1], temploadreg2\n"
- "ldr temploadreg2, [%[b_ptr0], #-0x18]\n"
- "ins v11.d[1], temploadreg3\n"
- "ins v12.d[1], temploadreg0\n"
- "ins v13.d[1], temploadreg1\n"
- "b.ne 3b\n"
- "2:\n"
- "ins v14.d[1], temploadreg2\n"
- "prfm PSTL1KEEP, [%[c_ptr0]]\n"
- "ldr d15, [%[b_ptr0], #-0x10]\n"
- "ldr temploadreg3, [%[b_ptr0], #-0x8]\n"
- "ins v15.d[1], temploadreg3\n"
- "cbz %[regs], 4f\n"
- ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
- "ldr d4, [%[a_ptr0]]\n"
- ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
- "ldr temploadreg0, [%[a_ptr0], #0x8]\n"
- ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
- "ldr d8, [%[b_ptr0]]\n"
- ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
- "ldr d9, [%[b_ptr0], #0x10]\n"
- ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n"
- "ins v4.d[1], temploadreg0\n"
- ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n"
- "ldr d10, [%[b_ptr0], #0x20]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ldr d11, [%[b_ptr0], #0x30]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- "ldr d12, [%[b_ptr0], #0x40]\n"
- "ins v8.d[1], temploadreg0\n"
- "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- "ldr d13, [%[b_ptr0], #0x50]\n"
- "ins v9.d[1], temploadreg1\n"
- ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- "ins v10.d[1], temploadreg2\n"
- ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- "ldr d15, [%[b_ptr0], #0x70]\n"
- "ins v11.d[1], temploadreg3\n"
- ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- "ins v12.d[1], temploadreg0\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n"
- "ldr d8, [%[b_ptr0], #-0x80]\n"
- "ldr temploadreg0, [%[b_ptr0], #-0x78]\n"
- ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n"
- "ldr d9, [%[b_ptr0], #-0x70]\n"
- "ins v13.d[1], temploadreg1\n"
- "ldr temploadreg1, [%[b_ptr0], #-0x68]\n"
- "ldr d10, [%[b_ptr0], #-0x60]\n"
- "ins v14.d[1], temploadreg2\n"
- ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n"
- "ldr temploadreg2, [%[b_ptr0], #-0x58]\n"
- "ldr d11, [%[b_ptr0], #-0x50]\n"
- "ins v15.d[1], temploadreg3\n"
- ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n"
- "ldr temploadreg3, [%[b_ptr0], #-0x48]\n"
- "ldr d12, [%[b_ptr0], #-0x40]\n"
- "ins v8.d[1], temploadreg0\n"
- ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n"
- "ldr temploadreg0, [%[b_ptr0], #-0x38]\n"
- "ldr d13, [%[b_ptr0], #-0x30]\n"
- "ins v9.d[1], temploadreg1\n"
- ".inst 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n"
- "ldr temploadreg1, [%[b_ptr0], #-0x28]\n"
- "ldr d14, [%[b_ptr0], #-0x20]\n"
- "ins v10.d[1], temploadreg2\n"
- ".inst 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n"
- "ldr temploadreg2, [%[b_ptr0], #-0x18]\n"
- "ldr d15, [%[b_ptr0], #-0x10]\n"
- "ins v11.d[1], temploadreg3\n"
- ".inst 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n"
- "ldr temploadreg3, [%[b_ptr0], #-0x8]\n"
- "ldr d8, [%[b_ptr0]]\n"
- "ins v12.d[1], temploadreg0\n"
- ".inst 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- "ldr d9, [%[b_ptr0], #0x10]\n"
- "ins v13.d[1], temploadreg1\n"
- ".inst 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- "ldr d10, [%[b_ptr0], #0x20]\n"
- "ins v14.d[1], temploadreg2\n"
- ".inst 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- "ldr d11, [%[b_ptr0], #0x30]\n"
- "ins v15.d[1], temploadreg3\n"
- ".inst 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- "ldr d12, [%[b_ptr0], #0x40]\n"
- "ins v8.d[1], temploadreg0\n"
- ".inst 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- "ldr d13, [%[b_ptr0], #0x50]\n"
- "ins v9.d[1], temploadreg1\n"
- ".inst 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- "ins v10.d[1], temploadreg2\n"
- ".inst 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- "ldr d15, [%[b_ptr0], #0x70]\n"
- "ins v11.d[1], temploadreg3\n"
- ".inst 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- "ins v12.d[1], temploadreg0\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- ".inst 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n"
- "ins v13.d[1], temploadreg1\n"
- "ins v14.d[1], temploadreg2\n"
- ".inst 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n"
- "ins v15.d[1], temploadreg3\n"
- ".inst 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n"
- ".inst 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n"
- ".inst 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n"
- "b 5f\n"
- "4:\n"
- ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
- "ldr d8, [%[b_ptr0]]\n"
- ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
- "ldr d9, [%[b_ptr0], #0x10]\n"
- ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n"
- "ldr d10, [%[b_ptr0], #0x20]\n"
- ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n"
- "ldr d11, [%[b_ptr0], #0x30]\n"
- ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- "ldr d12, [%[b_ptr0], #0x40]\n"
- "ins v8.d[1], temploadreg0\n"
- "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- "ldr d13, [%[b_ptr0], #0x50]\n"
- "ins v9.d[1], temploadreg1\n"
- ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- "ins v10.d[1], temploadreg2\n"
- ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- "ldr d15, [%[b_ptr0], #0x70]\n"
- "ins v11.d[1], temploadreg3\n"
- ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- "ins v12.d[1], temploadreg0\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n"
- "ins v13.d[1], temploadreg1\n"
- "ins v14.d[1], temploadreg2\n"
- ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n"
- "ins v15.d[1], temploadreg3\n"
- ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n"
- ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n"
- ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n"
- "5:\n"
- "cbz %[blocks], 6f\n"
- "7:\n"
- "ldr q8, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "ldr s0, [%[a_ptr0]]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "add %[a_ptr0], %[a_ptr0], #0x4\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "add %[b_ptr0], %[b_ptr0], #0x40\n"
- ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
- ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
- ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
- ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
- "b.ne 7b\n"
- "6:\n"
- "cbz %[odds], 8f\n"
- "ld1 {v0.b}[0], [%[a_ptr0]], #1\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 9f\n"
- "ld1 {v0.b}[1], [%[a_ptr0]], #1\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 9f\n"
- "ld1 {v0.b}[2], [%[a_ptr0]]\n"
- "9:\n"
- "ldr q8, [%[b_ptr0]]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
- ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
- ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
- ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
- "8:\n"
- "str q16, [%[c_ptr0]]\n"
- "str q17, [%[c_ptr0], #0x10]\n"
- "str q18, [%[c_ptr0], #0x20]\n"
- "str q19, [%[c_ptr0], #0x30]\n"
- "add %[c_ptr0], %[c_ptr0], #0x40\n"
- ".unreq temploadreg0\n"
- ".unreq temploadreg1\n"
- ".unreq temploadreg2\n"
- ".unreq temploadreg3\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks), [odds] "+r" (odds)
- : [width] "r" (width), [accumulate] "r" (static_cast<uint64_t>(accumulate)), [lda] "r" (ldab), [ldc] "r" (ldcb)
- : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "cc", "memory"
- );
- break;
- case 2:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "c_ptr1 .req X1\n"
- "temploadreg0 .req X2\n"
- "temploadreg1 .req X3\n"
- "temploadreg2 .req X4\n"
- "temploadreg3 .req X5\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "cbnz %[accumulate], 1f\n"
- "movi v16.4s, #0\n"
- "ldr q0, [%[a_ptr0]]\n"
- "movi v17.4s, #0\n"
- "ldr q1, [a_ptr1]\n"
- "movi v18.4s, #0\n"
- "ldr q8, [%[b_ptr0]]\n"
- "movi v19.4s, #0\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "movi v20.4s, #0\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "movi v21.4s, #0\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "movi v22.4s, #0\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- "movi v23.4s, #0\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "ldr q16, [%[c_ptr0]]\n"
- "ldr q17, [%[c_ptr0], #0x10]\n"
- "ldr q18, [%[c_ptr0], #0x20]\n"
- "ldr q19, [%[c_ptr0], #0x30]\n"
- "ldr q20, [c_ptr1]\n"
- "ldr q21, [c_ptr1, #0x10]\n"
- "ldr q22, [c_ptr1, #0x20]\n"
- "ldr q23, [c_ptr1, #0x30]\n"
- "ldr q0, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ldr q1, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "ldr q8, [%[b_ptr0]]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "cbz %[loops], 2f\n"
- "3:\n"
- ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
- "ins v14.d[1], temploadreg2\n"
- ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
- "ldr d15, [%[b_ptr0], #-0x10]\n"
- ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
- "ldr temploadreg3, [%[b_ptr0], #-0x8]\n"
- ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
- "ldr d4, [%[a_ptr0]]\n"
- ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
- "ldr temploadreg0, [%[a_ptr0], #0x8]\n"
- ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
- "ldr d5, [a_ptr1]\n"
- ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
- "ldr temploadreg1, [a_ptr1, #0x8]\n"
- ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
- "ldr d8, [%[b_ptr0]]\n"
- ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n"
- "ins v4.d[1], temploadreg0\n"
- ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n"
- "ldr d9, [%[b_ptr0], #0x10]\n"
- ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n"
- "ins v5.d[1], temploadreg1\n"
- ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n"
- "ldr d10, [%[b_ptr0], #0x20]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ldr d11, [%[b_ptr0], #0x30]\n"
- "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n"
- "ins v15.d[1], temploadreg3\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
- "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- "add a_ptr1, a_ptr1, #0x20\n"
- ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n"
- "ldr d12, [%[b_ptr0], #0x40]\n"
- ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n"
- "ins v8.d[1], temploadreg0\n"
- "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- "prfm PLDL1KEEP, [a_ptr1, #0x40]\n"
- "ldr d13, [%[b_ptr0], #0x50]\n"
- ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n"
- "ins v9.d[1], temploadreg1\n"
- ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- "ins v10.d[1], temploadreg2\n"
- ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n"
- "ldr d15, [%[b_ptr0], #0x70]\n"
- "ins v11.d[1], temploadreg3\n"
- ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n"
- "ins v12.d[1], temploadreg0\n"
- "ins v13.d[1], temploadreg1\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n"
- "ldr d8, [%[b_ptr0], #-0x80]\n"
- ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n"
- "ldr temploadreg0, [%[b_ptr0], #-0x78]\n"
- ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n"
- "ldr d9, [%[b_ptr0], #-0x70]\n"
- ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n"
- "ldr temploadreg1, [%[b_ptr0], #-0x68]\n"
- ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n"
- "ldr d10, [%[b_ptr0], #-0x60]\n"
- ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n"
- "ins v14.d[1], temploadreg2\n"
- "ldr temploadreg2, [%[b_ptr0], #-0x58]\n"
- "ldr d11, [%[b_ptr0], #-0x50]\n"
- "ins v15.d[1], temploadreg3\n"
- ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n"
- "ldr temploadreg3, [%[b_ptr0], #-0x48]\n"
- ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n"
- "ldr d12, [%[b_ptr0], #-0x40]\n"
- "ins v8.d[1], temploadreg0\n"
- ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n"
- "ldr temploadreg0, [%[b_ptr0], #-0x38]\n"
- ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n"
- "ldr d13, [%[b_ptr0], #-0x30]\n"
- "ins v9.d[1], temploadreg1\n"
- ".inst 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n"
- "ldr temploadreg1, [%[b_ptr0], #-0x28]\n"
- ".inst 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n"
- "ldr d14, [%[b_ptr0], #-0x20]\n"
- "ins v10.d[1], temploadreg2\n"
- ".inst 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n"
- "ldr temploadreg2, [%[b_ptr0], #-0x18]\n"
- ".inst 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n"
- "ldr d15, [%[b_ptr0], #-0x10]\n"
- "ins v11.d[1], temploadreg3\n"
- ".inst 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n"
- "ldr temploadreg3, [%[b_ptr0], #-0x8]\n"
- ".inst 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n"
- "ldr d0, [%[a_ptr0], #-0x10]\n"
- "ins v12.d[1], temploadreg0\n"
- ".inst 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n"
- "ldr temploadreg0, [%[a_ptr0], #-0x8]\n"
- ".inst 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n"
- "ldr d1, [a_ptr1, #-0x10]\n"
- "ins v13.d[1], temploadreg1\n"
- ".inst 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n"
- "ldr temploadreg1, [a_ptr1, #-0x8]\n"
- ".inst 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n"
- "ldr d8, [%[b_ptr0]]\n"
- "ins v0.d[1], temploadreg0\n"
- ".inst 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".inst 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n"
- "ldr d9, [%[b_ptr0], #0x10]\n"
- "ins v1.d[1], temploadreg1\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- "ldr d10, [%[b_ptr0], #0x20]\n"
- "ins v14.d[1], temploadreg2\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- "ldr d11, [%[b_ptr0], #0x30]\n"
- "ins v15.d[1], temploadreg3\n"
- ".inst 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- ".inst 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n"
- "ldr d12, [%[b_ptr0], #0x40]\n"
- "ins v8.d[1], temploadreg0\n"
- ".inst 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- ".inst 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n"
- "ldr d13, [%[b_ptr0], #0x50]\n"
- "ins v9.d[1], temploadreg1\n"
- ".inst 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- ".inst 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- "ins v10.d[1], temploadreg2\n"
- ".inst 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- ".inst 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n"
- "ldr d15, [%[b_ptr0], #0x70]\n"
- "ins v11.d[1], temploadreg3\n"
- ".inst 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- ".inst 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n"
- "ins v12.d[1], temploadreg0\n"
- "ins v13.d[1], temploadreg1\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".inst 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n"
- "ldr d8, [%[b_ptr0], #-0x80]\n"
- ".inst 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n"
- "ldr temploadreg0, [%[b_ptr0], #-0x78]\n"
- ".inst 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n"
- "ldr d9, [%[b_ptr0], #-0x70]\n"
- ".inst 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n"
- "ldr temploadreg1, [%[b_ptr0], #-0x68]\n"
- ".inst 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n"
- "ldr d10, [%[b_ptr0], #-0x60]\n"
- ".inst 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n"
- "ins v14.d[1], temploadreg2\n"
- "ldr temploadreg2, [%[b_ptr0], #-0x58]\n"
- "ldr d11, [%[b_ptr0], #-0x50]\n"
- "ins v15.d[1], temploadreg3\n"
- ".inst 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n"
- "ldr temploadreg3, [%[b_ptr0], #-0x48]\n"
- ".inst 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n"
- "ldr d12, [%[b_ptr0], #-0x40]\n"
- "ins v8.d[1], temploadreg0\n"
- ".inst 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n"
- "ldr temploadreg0, [%[b_ptr0], #-0x38]\n"
- ".inst 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n"
- "ldr d13, [%[b_ptr0], #-0x30]\n"
- "ins v9.d[1], temploadreg1\n"
- "ldr temploadreg1, [%[b_ptr0], #-0x28]\n"
- "ldr d14, [%[b_ptr0], #-0x20]\n"
- "ins v10.d[1], temploadreg2\n"
- "ldr temploadreg2, [%[b_ptr0], #-0x18]\n"
- "ins v11.d[1], temploadreg3\n"
- "ins v12.d[1], temploadreg0\n"
- "ins v13.d[1], temploadreg1\n"
- "b.ne 3b\n"
- "2:\n"
- "ins v14.d[1], temploadreg2\n"
- "prfm PSTL1KEEP, [%[c_ptr0]]\n"
- "ldr d15, [%[b_ptr0], #-0x10]\n"
- "prfm PSTL1KEEP, [c_ptr1]\n"
- "ldr temploadreg3, [%[b_ptr0], #-0x8]\n"
- "ins v15.d[1], temploadreg3\n"
- "cbz %[regs], 4f\n"
- ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
- "ldr d4, [%[a_ptr0]]\n"
- ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
- "ldr temploadreg0, [%[a_ptr0], #0x8]\n"
- ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
- "ldr d5, [a_ptr1]\n"
- ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
- "ldr temploadreg1, [a_ptr1, #0x8]\n"
- ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
- "ldr d8, [%[b_ptr0]]\n"
- ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
- "ins v4.d[1], temploadreg0\n"
- ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
- "ldr d9, [%[b_ptr0], #0x10]\n"
- ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n"
- "ins v5.d[1], temploadreg1\n"
- ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n"
- "ldr d10, [%[b_ptr0], #0x20]\n"
- ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n"
- "ldr d11, [%[b_ptr0], #0x30]\n"
- ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n"
- "ldr d12, [%[b_ptr0], #0x40]\n"
- ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n"
- "ins v8.d[1], temploadreg0\n"
- "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ldr d13, [%[b_ptr0], #0x50]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n"
- "ins v9.d[1], temploadreg1\n"
- ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- "ins v10.d[1], temploadreg2\n"
- ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n"
- "ldr d15, [%[b_ptr0], #0x70]\n"
- "ins v11.d[1], temploadreg3\n"
- ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n"
- "ins v12.d[1], temploadreg0\n"
- "ins v13.d[1], temploadreg1\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n"
- "ldr d8, [%[b_ptr0], #-0x80]\n"
- ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n"
- "ldr temploadreg0, [%[b_ptr0], #-0x78]\n"
- ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n"
- "ldr d9, [%[b_ptr0], #-0x70]\n"
- ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n"
- "ldr temploadreg1, [%[b_ptr0], #-0x68]\n"
- ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n"
- "ldr d10, [%[b_ptr0], #-0x60]\n"
- ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n"
- "ins v14.d[1], temploadreg2\n"
- "ldr temploadreg2, [%[b_ptr0], #-0x58]\n"
- "ldr d11, [%[b_ptr0], #-0x50]\n"
- "ins v15.d[1], temploadreg3\n"
- ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n"
- "ldr temploadreg3, [%[b_ptr0], #-0x48]\n"
- ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n"
- "ldr d12, [%[b_ptr0], #-0x40]\n"
- "ins v8.d[1], temploadreg0\n"
- ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n"
- "ldr temploadreg0, [%[b_ptr0], #-0x38]\n"
- ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n"
- "ldr d13, [%[b_ptr0], #-0x30]\n"
- "ins v9.d[1], temploadreg1\n"
- ".inst 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n"
- "ldr temploadreg1, [%[b_ptr0], #-0x28]\n"
- ".inst 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n"
- "ldr d14, [%[b_ptr0], #-0x20]\n"
- "ins v10.d[1], temploadreg2\n"
- ".inst 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n"
- "ldr temploadreg2, [%[b_ptr0], #-0x18]\n"
- ".inst 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n"
- "ldr d15, [%[b_ptr0], #-0x10]\n"
- "ins v11.d[1], temploadreg3\n"
- ".inst 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n"
- "ldr temploadreg3, [%[b_ptr0], #-0x8]\n"
- ".inst 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n"
- "ldr d8, [%[b_ptr0]]\n"
- "ins v12.d[1], temploadreg0\n"
- ".inst 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".inst 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n"
- "ldr d9, [%[b_ptr0], #0x10]\n"
- "ins v13.d[1], temploadreg1\n"
- ".inst 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".inst 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n"
- "ldr d10, [%[b_ptr0], #0x20]\n"
- "ins v14.d[1], temploadreg2\n"
- ".inst 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- ".inst 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n"
- "ldr d11, [%[b_ptr0], #0x30]\n"
- "ins v15.d[1], temploadreg3\n"
- ".inst 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- ".inst 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n"
- "ldr d12, [%[b_ptr0], #0x40]\n"
- "ins v8.d[1], temploadreg0\n"
- ".inst 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- ".inst 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n"
- "ldr d13, [%[b_ptr0], #0x50]\n"
- "ins v9.d[1], temploadreg1\n"
- ".inst 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- ".inst 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- "ins v10.d[1], temploadreg2\n"
- ".inst 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- ".inst 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n"
- "ldr d15, [%[b_ptr0], #0x70]\n"
- "ins v11.d[1], temploadreg3\n"
- ".inst 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- ".inst 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n"
- "ins v12.d[1], temploadreg0\n"
- "ins v13.d[1], temploadreg1\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- ".inst 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n"
- "ins v14.d[1], temploadreg2\n"
- ".inst 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n"
- "ins v15.d[1], temploadreg3\n"
- ".inst 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n"
- ".inst 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n"
- ".inst 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n"
- ".inst 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n"
- ".inst 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n"
- ".inst 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n"
- ".inst 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n"
- ".inst 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n"
- "b 5f\n"
- "4:\n"
- ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
- "ldr d8, [%[b_ptr0]]\n"
- ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
- "ldr d9, [%[b_ptr0], #0x10]\n"
- ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
- "ldr d10, [%[b_ptr0], #0x20]\n"
- ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
- "ldr d11, [%[b_ptr0], #0x30]\n"
- ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n"
- "ins v8.d[1], temploadreg0\n"
- ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n"
- "ldr d12, [%[b_ptr0], #0x40]\n"
- ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n"
- "ldr d13, [%[b_ptr0], #0x50]\n"
- ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n"
- "ins v9.d[1], temploadreg1\n"
- ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n"
- "ins v10.d[1], temploadreg2\n"
- ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n"
- "ldr d15, [%[b_ptr0], #0x70]\n"
- ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n"
- "ins v11.d[1], temploadreg3\n"
- ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n"
- "ins v12.d[1], temploadreg0\n"
- ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n"
- "ins v13.d[1], temploadreg1\n"
- ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n"
- "ins v14.d[1], temploadreg2\n"
- ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n"
- "ins v15.d[1], temploadreg3\n"
- ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n"
- ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n"
- ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n"
- ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n"
- ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n"
- ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n"
- ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n"
- "5:\n"
- "cbz %[blocks], 6f\n"
- "7:\n"
- "ldr q8, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "ldr s0, [%[a_ptr0]]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "add %[a_ptr0], %[a_ptr0], #0x4\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "add %[b_ptr0], %[b_ptr0], #0x40\n"
- ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
- "ldr s1, [a_ptr1]\n"
- ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
- "add a_ptr1, a_ptr1, #0x4\n"
- ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
- ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
- ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
- ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
- ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
- ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
- "b.ne 7b\n"
- "6:\n"
- "cbz %[odds], 8f\n"
- "ld1 {v0.b}[0], [%[a_ptr0]], #1\n"
- "ld1 {v1.b}[0], [a_ptr1], #1\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 9f\n"
- "ld1 {v0.b}[1], [%[a_ptr0]], #1\n"
- "ld1 {v1.b}[1], [a_ptr1], #1\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 9f\n"
- "ld1 {v0.b}[2], [%[a_ptr0]]\n"
- "ld1 {v1.b}[2], [a_ptr1]\n"
- "9:\n"
- "ldr q8, [%[b_ptr0]]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
- ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
- ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
- ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
- ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
- ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
- ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
- ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
- "8:\n"
- "str q16, [%[c_ptr0]]\n"
- "str q17, [%[c_ptr0], #0x10]\n"
- "str q18, [%[c_ptr0], #0x20]\n"
- "str q19, [%[c_ptr0], #0x30]\n"
- "add %[c_ptr0], %[c_ptr0], #0x40\n"
- "str q20, [c_ptr1]\n"
- "str q21, [c_ptr1, #0x10]\n"
- "str q22, [c_ptr1, #0x20]\n"
- "str q23, [c_ptr1, #0x30]\n"
- ".unreq a_ptr1\n"
- ".unreq c_ptr1\n"
- ".unreq temploadreg0\n"
- ".unreq temploadreg1\n"
- ".unreq temploadreg2\n"
- ".unreq temploadreg3\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks), [odds] "+r" (odds)
- : [width] "r" (width), [accumulate] "r" (static_cast<uint64_t>(accumulate)), [lda] "r" (ldab), [ldc] "r" (ldcb)
- : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "x4", "x5", "cc", "memory"
- );
- break;
- case 3:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "c_ptr1 .req X2\n"
- "c_ptr2 .req X3\n"
- "temploadreg0 .req X4\n"
- "temploadreg1 .req X5\n"
- "temploadreg2 .req X6\n"
- "temploadreg3 .req X7\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "cbnz %[accumulate], 1f\n"
- "movi v16.4s, #0\n"
- "ldr q0, [%[a_ptr0]]\n"
- "movi v17.4s, #0\n"
- "ldr q1, [a_ptr1]\n"
- "movi v18.4s, #0\n"
- "ldr q2, [a_ptr2]\n"
- "movi v19.4s, #0\n"
- "ldr q8, [%[b_ptr0]]\n"
- "movi v20.4s, #0\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "movi v21.4s, #0\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "movi v22.4s, #0\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "movi v23.4s, #0\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- "movi v24.4s, #0\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- "movi v25.4s, #0\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- "movi v26.4s, #0\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- "movi v27.4s, #0\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "ins v14.d[1], temploadreg2\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "ldr q16, [%[c_ptr0]]\n"
- "ldr q17, [%[c_ptr0], #0x10]\n"
- "ldr q18, [%[c_ptr0], #0x20]\n"
- "ldr q19, [%[c_ptr0], #0x30]\n"
- "ldr q20, [c_ptr1]\n"
- "ldr q21, [c_ptr1, #0x10]\n"
- "ldr q22, [c_ptr1, #0x20]\n"
- "ldr q23, [c_ptr1, #0x30]\n"
- "ldr q24, [c_ptr2]\n"
- "ldr q25, [c_ptr2, #0x10]\n"
- "ldr q26, [c_ptr2, #0x20]\n"
- "ldr q27, [c_ptr2, #0x30]\n"
- "ldr q0, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ldr q1, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "ldr q2, [a_ptr2]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "ldr q8, [%[b_ptr0]]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "ins v14.d[1], temploadreg2\n"
- "cbz %[loops], 2f\n"
- "3:\n"
- ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
- "ldr d15, [%[b_ptr0], #-0x10]\n"
- ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
- "ldr temploadreg3, [%[b_ptr0], #-0x8]\n"
- ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n"
- "ldr d4, [%[a_ptr0]]\n"
- ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
- "ldr temploadreg0, [%[a_ptr0], #0x8]\n"
- ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
- "ldr d5, [a_ptr1]\n"
- ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n"
- "ldr temploadreg1, [a_ptr1, #0x8]\n"
- ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
- "ldr d6, [a_ptr2]\n"
- ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
- "ldr temploadreg2, [a_ptr2, #0x8]\n"
- ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n"
- "ldr d8, [%[b_ptr0]]\n"
- ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
- "ins v4.d[1], temploadreg0\n"
- ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n"
- "ldr d9, [%[b_ptr0], #0x10]\n"
- ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n"
- "ins v5.d[1], temploadreg1\n"
- ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".inst 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n"
- "ldr d10, [%[b_ptr0], #0x20]\n"
- ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n"
- "ins v6.d[1], temploadreg2\n"
- ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- ".inst 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n"
- "ldr d11, [%[b_ptr0], #0x30]\n"
- ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n"
- "ins v15.d[1], temploadreg3\n"
- ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- ".inst 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n"
- "ldr d12, [%[b_ptr0], #0x40]\n"
- "ins v8.d[1], temploadreg0\n"
- "subs %[loops], %[loops], #0x1\n"
- ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n"
- "ldr d13, [%[b_ptr0], #0x50]\n"
- ".inst 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n"
- "ins v9.d[1], temploadreg1\n"
- ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- ".inst 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n"
- "ins v10.d[1], temploadreg2\n"
- ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n"
- "ldr d15, [%[b_ptr0], #0x70]\n"
- ".inst 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n"
- "ins v11.d[1], temploadreg3\n"
- ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n"
- "ins v12.d[1], temploadreg0\n"
- ".inst 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n"
- "ins v13.d[1], temploadreg1\n"
- ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n"
- "ins v14.d[1], temploadreg2\n"
- ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n"
- "ins v15.d[1], temploadreg3\n"
- ".inst 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n"
- "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n"
- ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n"
- "ldr d8, [%[b_ptr0], #-0x80]\n"
- ".inst 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n"
- "ldr temploadreg0, [%[b_ptr0], #-0x78]\n"
- ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n"
- "ldr d9, [%[b_ptr0], #-0x70]\n"
- ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n"
- "ldr temploadreg1, [%[b_ptr0], #-0x68]\n"
- ".inst 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n"
- "ldr d10, [%[b_ptr0], #-0x60]\n"
- ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n"
- "ldr temploadreg2, [%[b_ptr0], #-0x58]\n"
- ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n"
- "ldr d11, [%[b_ptr0], #-0x50]\n"
- ".inst 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n"
- "ldr temploadreg3, [%[b_ptr0], #-0x48]\n"
- ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n"
- "ldr d12, [%[b_ptr0], #-0x40]\n"
- ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n"
- "ins v8.d[1], temploadreg0\n"
- ".inst 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n"
- "ldr temploadreg0, [%[b_ptr0], #-0x38]\n"
- "ldr d13, [%[b_ptr0], #-0x30]\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
- ".inst 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n"
- "ins v9.d[1], temploadreg1\n"
- ".inst 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n"
- "ldr temploadreg1, [%[b_ptr0], #-0x28]\n"
- ".inst 0x6f86e118 // udot v24.4s, v8.16b, v6.4b[0]\n"
- "ldr d14, [%[b_ptr0], #-0x20]\n"
- "ins v10.d[1], temploadreg2\n"
- "add a_ptr1, a_ptr1, #0x20\n"
- ".inst 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n"
- "ldr temploadreg2, [%[b_ptr0], #-0x18]\n"
- ".inst 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n"
- "ldr d15, [%[b_ptr0], #-0x10]\n"
- ".inst 0x6f86e139 // udot v25.4s, v9.16b, v6.4b[0]\n"
- "ins v11.d[1], temploadreg3\n"
- ".inst 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n"
- "ldr temploadreg3, [%[b_ptr0], #-0x8]\n"
- ".inst 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n"
- "ldr d0, [%[a_ptr0], #-0x10]\n"
- ".inst 0x6f86e15a // udot v26.4s, v10.16b, v6.4b[0]\n"
- "ins v12.d[1], temploadreg0\n"
- ".inst 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n"
- "ldr temploadreg0, [%[a_ptr0], #-0x8]\n"
- ".inst 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n"
- "ldr d1, [a_ptr1, #-0x10]\n"
- ".inst 0x6f86e17b // udot v27.4s, v11.16b, v6.4b[0]\n"
- "ins v13.d[1], temploadreg1\n"
- ".inst 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n"
- "ldr temploadreg1, [a_ptr1, #-0x8]\n"
- ".inst 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n"
- "ins v14.d[1], temploadreg2\n"
- ".inst 0x6fa6e198 // udot v24.4s, v12.16b, v6.4b[1]\n"
- "ldr d8, [%[b_ptr0]]\n"
- ".inst 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n"
- "ins v0.d[1], temploadreg0\n"
- ".inst 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".inst 0x6fa6e1b9 // udot v25.4s, v13.16b, v6.4b[1]\n"
- "ldr d9, [%[b_ptr0], #0x10]\n"
- ".inst 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n"
- "ins v1.d[1], temploadreg1\n"
- ".inst 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".inst 0x6fa6e1da // udot v26.4s, v14.16b, v6.4b[1]\n"
- "ldr d10, [%[b_ptr0], #0x20]\n"
- "ldr d11, [%[b_ptr0], #0x30]\n"
- "add a_ptr2, a_ptr2, #0x20\n"
- "ins v15.d[1], temploadreg3\n"
- "prfm PLDL1KEEP, [a_ptr1, #0x40]\n"
- "ldr d2, [a_ptr2, #-0x10]\n"
- "prfm PLDL1KEEP, [a_ptr2, #0x40]\n"
- ".inst 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n"
- "ldr temploadreg2, [a_ptr2, #-0x8]\n"
- ".inst 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- ".inst 0x6fa6e1fb // udot v27.4s, v15.16b, v6.4b[1]\n"
- "ldr d12, [%[b_ptr0], #0x40]\n"
- "ins v8.d[1], temploadreg0\n"
- "ins v2.d[1], temploadreg2\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- ".inst 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n"
- "ldr d13, [%[b_ptr0], #0x50]\n"
- ".inst 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n"
- "ins v9.d[1], temploadreg1\n"
- ".inst 0x6f86e918 // udot v24.4s, v8.16b, v6.4b[2]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- "ins v10.d[1], temploadreg2\n"
- ".inst 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- ".inst 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n"
- "ldr d15, [%[b_ptr0], #0x70]\n"
- ".inst 0x6f86e939 // udot v25.4s, v9.16b, v6.4b[2]\n"
- "ins v11.d[1], temploadreg3\n"
- ".inst 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- ".inst 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n"
- "ins v12.d[1], temploadreg0\n"
- ".inst 0x6f86e95a // udot v26.4s, v10.16b, v6.4b[2]\n"
- "ins v13.d[1], temploadreg1\n"
- ".inst 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n"
- "ins v14.d[1], temploadreg2\n"
- ".inst 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n"
- "ins v15.d[1], temploadreg3\n"
- ".inst 0x6f86e97b // udot v27.4s, v11.16b, v6.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".inst 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n"
- "ldr d8, [%[b_ptr0], #-0x80]\n"
- ".inst 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n"
- "ldr temploadreg0, [%[b_ptr0], #-0x78]\n"
- ".inst 0x6fa6e998 // udot v24.4s, v12.16b, v6.4b[3]\n"
- "ldr d9, [%[b_ptr0], #-0x70]\n"
- ".inst 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n"
- "ldr temploadreg1, [%[b_ptr0], #-0x68]\n"
- ".inst 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n"
- "ldr d10, [%[b_ptr0], #-0x60]\n"
- ".inst 0x6fa6e9b9 // udot v25.4s, v13.16b, v6.4b[3]\n"
- "ldr temploadreg2, [%[b_ptr0], #-0x58]\n"
- ".inst 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n"
- "ldr d11, [%[b_ptr0], #-0x50]\n"
- ".inst 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n"
- "ldr temploadreg3, [%[b_ptr0], #-0x48]\n"
- ".inst 0x6fa6e9da // udot v26.4s, v14.16b, v6.4b[3]\n"
- "ldr d12, [%[b_ptr0], #-0x40]\n"
- ".inst 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n"
- "ins v8.d[1], temploadreg0\n"
- ".inst 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n"
- "ldr temploadreg0, [%[b_ptr0], #-0x38]\n"
- ".inst 0x6fa6e9fb // udot v27.4s, v15.16b, v6.4b[3]\n"
- "ldr d13, [%[b_ptr0], #-0x30]\n"
- "ins v9.d[1], temploadreg1\n"
- "ldr temploadreg1, [%[b_ptr0], #-0x28]\n"
- "ldr d14, [%[b_ptr0], #-0x20]\n"
- "ins v10.d[1], temploadreg2\n"
- "ldr temploadreg2, [%[b_ptr0], #-0x18]\n"
- "ins v11.d[1], temploadreg3\n"
- "ins v12.d[1], temploadreg0\n"
- "ins v13.d[1], temploadreg1\n"
- "ins v14.d[1], temploadreg2\n"
- "b.ne 3b\n"
- "2:\n"
- "ldr d15, [%[b_ptr0], #-0x10]\n"
- "prfm PSTL1KEEP, [%[c_ptr0]]\n"
- "ldr temploadreg3, [%[b_ptr0], #-0x8]\n"
- "prfm PSTL1KEEP, [c_ptr1]\n"
- "prfm PSTL1KEEP, [c_ptr2]\n"
- "ins v15.d[1], temploadreg3\n"
- "cbz %[regs], 4f\n"
- ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
- "ldr d4, [%[a_ptr0]]\n"
- ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
- "ldr temploadreg0, [%[a_ptr0], #0x8]\n"
- ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n"
- "ldr d5, [a_ptr1]\n"
- ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
- "ldr temploadreg1, [a_ptr1, #0x8]\n"
- ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
- "ldr d6, [a_ptr2]\n"
- ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n"
- "ldr temploadreg2, [a_ptr2, #0x8]\n"
- ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
- "ldr d8, [%[b_ptr0]]\n"
- ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
- "ins v4.d[1], temploadreg0\n"
- ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
- "ldr d9, [%[b_ptr0], #0x10]\n"
- ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
- "ins v5.d[1], temploadreg1\n"
- ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n"
- "ldr d10, [%[b_ptr0], #0x20]\n"
- ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n"
- "ins v6.d[1], temploadreg2\n"
- ".inst 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n"
- "ldr d11, [%[b_ptr0], #0x30]\n"
- ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- ".inst 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n"
- "ldr d12, [%[b_ptr0], #0x40]\n"
- ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n"
- "ins v8.d[1], temploadreg0\n"
- ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- ".inst 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n"
- "ldr d13, [%[b_ptr0], #0x50]\n"
- ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n"
- "ins v9.d[1], temploadreg1\n"
- ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- ".inst 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n"
- "ins v10.d[1], temploadreg2\n"
- ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- ".inst 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n"
- "ldr d15, [%[b_ptr0], #0x70]\n"
- ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n"
- "ins v11.d[1], temploadreg3\n"
- ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- ".inst 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n"
- "ins v12.d[1], temploadreg0\n"
- ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n"
- "ins v13.d[1], temploadreg1\n"
- ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n"
- "ins v14.d[1], temploadreg2\n"
- ".inst 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n"
- "ins v15.d[1], temploadreg3\n"
- ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n"
- "ldr d8, [%[b_ptr0], #-0x80]\n"
- ".inst 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n"
- "ldr temploadreg0, [%[b_ptr0], #-0x78]\n"
- ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n"
- "ldr d9, [%[b_ptr0], #-0x70]\n"
- ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n"
- "ldr temploadreg1, [%[b_ptr0], #-0x68]\n"
- ".inst 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n"
- "ldr d10, [%[b_ptr0], #-0x60]\n"
- ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n"
- "ldr temploadreg2, [%[b_ptr0], #-0x58]\n"
- ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n"
- "ldr d11, [%[b_ptr0], #-0x50]\n"
- ".inst 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n"
- "ldr temploadreg3, [%[b_ptr0], #-0x48]\n"
- ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n"
- "ldr d12, [%[b_ptr0], #-0x40]\n"
- ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n"
- "ins v8.d[1], temploadreg0\n"
- ".inst 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n"
- "ldr temploadreg0, [%[b_ptr0], #-0x38]\n"
- ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n"
- "ldr d13, [%[b_ptr0], #-0x30]\n"
- ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n"
- "ins v9.d[1], temploadreg1\n"
- ".inst 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n"
- "ldr temploadreg1, [%[b_ptr0], #-0x28]\n"
- ".inst 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n"
- "ldr d14, [%[b_ptr0], #-0x20]\n"
- ".inst 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n"
- "ins v10.d[1], temploadreg2\n"
- ".inst 0x6f86e118 // udot v24.4s, v8.16b, v6.4b[0]\n"
- "ldr temploadreg2, [%[b_ptr0], #-0x18]\n"
- ".inst 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n"
- "ldr d15, [%[b_ptr0], #-0x10]\n"
- ".inst 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n"
- "ins v11.d[1], temploadreg3\n"
- ".inst 0x6f86e139 // udot v25.4s, v9.16b, v6.4b[0]\n"
- "ldr temploadreg3, [%[b_ptr0], #-0x8]\n"
- ".inst 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n"
- "ldr d8, [%[b_ptr0]]\n"
- ".inst 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n"
- "ins v12.d[1], temploadreg0\n"
- ".inst 0x6f86e15a // udot v26.4s, v10.16b, v6.4b[0]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".inst 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n"
- "ldr d9, [%[b_ptr0], #0x10]\n"
- ".inst 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n"
- "ins v13.d[1], temploadreg1\n"
- ".inst 0x6f86e17b // udot v27.4s, v11.16b, v6.4b[0]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".inst 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n"
- "ldr d10, [%[b_ptr0], #0x20]\n"
- ".inst 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n"
- "ins v14.d[1], temploadreg2\n"
- ".inst 0x6fa6e198 // udot v24.4s, v12.16b, v6.4b[1]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- ".inst 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n"
- "ldr d11, [%[b_ptr0], #0x30]\n"
- ".inst 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n"
- "ins v15.d[1], temploadreg3\n"
- ".inst 0x6fa6e1b9 // udot v25.4s, v13.16b, v6.4b[1]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- ".inst 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n"
- "ldr d12, [%[b_ptr0], #0x40]\n"
- ".inst 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n"
- "ins v8.d[1], temploadreg0\n"
- ".inst 0x6fa6e1da // udot v26.4s, v14.16b, v6.4b[1]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- ".inst 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n"
- "ldr d13, [%[b_ptr0], #0x50]\n"
- ".inst 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n"
- "ins v9.d[1], temploadreg1\n"
- ".inst 0x6fa6e1fb // udot v27.4s, v15.16b, v6.4b[1]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- ".inst 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- ".inst 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n"
- "ins v10.d[1], temploadreg2\n"
- ".inst 0x6f86e918 // udot v24.4s, v8.16b, v6.4b[2]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- ".inst 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n"
- "ldr d15, [%[b_ptr0], #0x70]\n"
- ".inst 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n"
- "ins v11.d[1], temploadreg3\n"
- ".inst 0x6f86e939 // udot v25.4s, v9.16b, v6.4b[2]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- ".inst 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n"
- "ins v12.d[1], temploadreg0\n"
- ".inst 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n"
- "ins v13.d[1], temploadreg1\n"
- ".inst 0x6f86e95a // udot v26.4s, v10.16b, v6.4b[2]\n"
- "ins v14.d[1], temploadreg2\n"
- ".inst 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n"
- "ins v15.d[1], temploadreg3\n"
- ".inst 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- ".inst 0x6f86e97b // udot v27.4s, v11.16b, v6.4b[2]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- ".inst 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- ".inst 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- ".inst 0x6fa6e998 // udot v24.4s, v12.16b, v6.4b[3]\n"
- ".inst 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n"
- ".inst 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n"
- ".inst 0x6fa6e9b9 // udot v25.4s, v13.16b, v6.4b[3]\n"
- ".inst 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n"
- ".inst 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n"
- ".inst 0x6fa6e9da // udot v26.4s, v14.16b, v6.4b[3]\n"
- ".inst 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n"
- ".inst 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n"
- ".inst 0x6fa6e9fb // udot v27.4s, v15.16b, v6.4b[3]\n"
- "b 5f\n"
- "4:\n"
- ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n"
- "ldr d8, [%[b_ptr0]]\n"
- ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n"
- "ldr d9, [%[b_ptr0], #0x10]\n"
- ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
- "ins v8.d[1], temploadreg0\n"
- ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n"
- "ldr d10, [%[b_ptr0], #0x20]\n"
- ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
- "ins v9.d[1], temploadreg1\n"
- ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n"
- "ldr d11, [%[b_ptr0], #0x30]\n"
- ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n"
- "ins v10.d[1], temploadreg2\n"
- ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- ".inst 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n"
- "ldr d12, [%[b_ptr0], #0x40]\n"
- ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n"
- "ins v11.d[1], temploadreg3\n"
- ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- ".inst 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n"
- "ldr d13, [%[b_ptr0], #0x50]\n"
- ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n"
- "ins v12.d[1], temploadreg0\n"
- ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n"
- ".inst 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n"
- "ins v13.d[1], temploadreg1\n"
- ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n"
- ".inst 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n"
- "ldr d15, [%[b_ptr0], #0x70]\n"
- ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n"
- "ins v14.d[1], temploadreg2\n"
- ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- ".inst 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n"
- "ins v15.d[1], temploadreg3\n"
- ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n"
- ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n"
- ".inst 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n"
- ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n"
- ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n"
- ".inst 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n"
- ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n"
- ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n"
- ".inst 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n"
- ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n"
- ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n"
- ".inst 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n"
- ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n"
- ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n"
- ".inst 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n"
- ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n"
- ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n"
- ".inst 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n"
- ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n"
- ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n"
- ".inst 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n"
- "5:\n"
- "cbz %[blocks], 6f\n"
- "7:\n"
- "ldr q8, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "ldr s0, [%[a_ptr0]]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "add %[a_ptr0], %[a_ptr0], #0x4\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "add %[b_ptr0], %[b_ptr0], #0x40\n"
- ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
- "ldr s1, [a_ptr1]\n"
- ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
- "add a_ptr1, a_ptr1, #0x4\n"
- ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
- "ldr s2, [a_ptr2]\n"
- ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
- "add a_ptr2, a_ptr2, #0x4\n"
- ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
- ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n"
- ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n"
- ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
- ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n"
- ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
- ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
- ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n"
- "b.ne 7b\n"
- "6:\n"
- "cbz %[odds], 8f\n"
- "ld1 {v0.b}[0], [%[a_ptr0]], #1\n"
- "ld1 {v1.b}[0], [a_ptr1], #1\n"
- "ld1 {v2.b}[0], [a_ptr2], #1\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 9f\n"
- "ld1 {v0.b}[1], [%[a_ptr0]], #1\n"
- "ld1 {v1.b}[1], [a_ptr1], #1\n"
- "ld1 {v2.b}[1], [a_ptr2], #1\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 9f\n"
- "ld1 {v0.b}[2], [%[a_ptr0]]\n"
- "ld1 {v1.b}[2], [a_ptr1]\n"
- "ld1 {v2.b}[2], [a_ptr2]\n"
- "9:\n"
- "ldr q8, [%[b_ptr0]]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
- ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
- ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n"
- ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
- ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
- ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n"
- ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
- ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
- ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n"
- ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
- ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
- ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n"
- "8:\n"
- "str q16, [%[c_ptr0]]\n"
- "str q17, [%[c_ptr0], #0x10]\n"
- "str q18, [%[c_ptr0], #0x20]\n"
- "str q19, [%[c_ptr0], #0x30]\n"
- "add %[c_ptr0], %[c_ptr0], #0x40\n"
- "str q20, [c_ptr1]\n"
- "str q21, [c_ptr1, #0x10]\n"
- "str q22, [c_ptr1, #0x20]\n"
- "str q23, [c_ptr1, #0x30]\n"
- "str q24, [c_ptr2]\n"
- "str q25, [c_ptr2, #0x10]\n"
- "str q26, [c_ptr2, #0x20]\n"
- "str q27, [c_ptr2, #0x30]\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- ".unreq temploadreg0\n"
- ".unreq temploadreg1\n"
- ".unreq temploadreg2\n"
- ".unreq temploadreg3\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks), [odds] "+r" (odds)
- : [width] "r" (width), [accumulate] "r" (static_cast<uint64_t>(accumulate)), [lda] "r" (ldab), [ldc] "r" (ldcb)
- : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "cc", "memory"
- );
- break;
- default:
- case 4:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "a_ptr3 .req X2\n"
- "c_ptr1 .req X3\n"
- "c_ptr2 .req X4\n"
- "c_ptr3 .req X5\n"
- "temploadreg0 .req X6\n"
- "temploadreg1 .req X7\n"
- "temploadreg2 .req X8\n"
- "temploadreg3 .req X9\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "add a_ptr3, a_ptr2, %[lda]\n"
- "add c_ptr3, c_ptr2, %[ldc]\n"
- "cbnz %[accumulate], 1f\n"
- "movi v16.4s, #0\n"
- "ldr q0, [%[a_ptr0]]\n"
- "movi v17.4s, #0\n"
- "ldr q1, [a_ptr1]\n"
- "movi v18.4s, #0\n"
- "ldr q2, [a_ptr2]\n"
- "movi v19.4s, #0\n"
- "ldr q3, [a_ptr3]\n"
- "movi v20.4s, #0\n"
- "ldr q8, [%[b_ptr0]]\n"
- "movi v21.4s, #0\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "movi v22.4s, #0\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "movi v23.4s, #0\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "movi v24.4s, #0\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- "movi v25.4s, #0\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- "movi v26.4s, #0\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- "movi v27.4s, #0\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- "movi v28.4s, #0\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "movi v29.4s, #0\n"
- "ins v14.d[1], temploadreg2\n"
- "movi v30.4s, #0\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "movi v31.4s, #0\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "add a_ptr3, a_ptr3, #0x10\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "ldr q16, [%[c_ptr0]]\n"
- "ldr q17, [%[c_ptr0], #0x10]\n"
- "ldr q18, [%[c_ptr0], #0x20]\n"
- "ldr q19, [%[c_ptr0], #0x30]\n"
- "ldr q20, [c_ptr1]\n"
- "ldr q21, [c_ptr1, #0x10]\n"
- "ldr q22, [c_ptr1, #0x20]\n"
- "ldr q23, [c_ptr1, #0x30]\n"
- "ldr q24, [c_ptr2]\n"
- "ldr q25, [c_ptr2, #0x10]\n"
- "ldr q26, [c_ptr2, #0x20]\n"
- "ldr q27, [c_ptr2, #0x30]\n"
- "ldr q28, [c_ptr3]\n"
- "ldr q29, [c_ptr3, #0x10]\n"
- "ldr q30, [c_ptr3, #0x20]\n"
- "ldr q31, [c_ptr3, #0x30]\n"
- "ldr q0, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ldr q1, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "ldr q2, [a_ptr2]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "ldr q3, [a_ptr3]\n"
- "add a_ptr3, a_ptr3, #0x10\n"
- "ldr q8, [%[b_ptr0]]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "ins v14.d[1], temploadreg2\n"
- "cbz %[loops], 2f\n"
- "3:\n"
- ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
- "ldr d15, [%[b_ptr0], #-0x10]\n"
- ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
- "ldr temploadreg3, [%[b_ptr0], #-0x8]\n"
- ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n"
- "ldr d4, [%[a_ptr0]]\n"
- ".inst 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n"
- "ldr temploadreg0, [%[a_ptr0], #0x8]\n"
- ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
- "ldr d5, [a_ptr1]\n"
- ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
- "ldr temploadreg1, [a_ptr1, #0x8]\n"
- ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n"
- "ldr d6, [a_ptr2]\n"
- ".inst 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n"
- "ldr temploadreg2, [a_ptr2, #0x8]\n"
- ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
- "ldr d7, [a_ptr3]\n"
- ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
- "ins v15.d[1], temploadreg3\n"
- ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n"
- "ldr temploadreg3, [a_ptr3, #0x8]\n"
- ".inst 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n"
- "ldr d8, [%[b_ptr0]]\n"
- ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
- "ins v4.d[1], temploadreg0\n"
- ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n"
- "ldr d9, [%[b_ptr0], #0x10]\n"
- ".inst 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n"
- "ins v5.d[1], temploadreg1\n"
- ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n"
- "ldr d10, [%[b_ptr0], #0x20]\n"
- ".inst 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n"
- "ins v6.d[1], temploadreg2\n"
- ".inst 0x6fa3e19c // udot v28.4s, v12.16b, v3.4b[1]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n"
- "ldr d11, [%[b_ptr0], #0x30]\n"
- ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n"
- "ins v7.d[1], temploadreg3\n"
- ".inst 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- ".inst 0x6fa3e1bd // udot v29.4s, v13.16b, v3.4b[1]\n"
- "ldr d12, [%[b_ptr0], #0x40]\n"
- ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n"
- "ins v8.d[1], temploadreg0\n"
- ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- ".inst 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n"
- "ldr d13, [%[b_ptr0], #0x50]\n"
- ".inst 0x6fa3e1de // udot v30.4s, v14.16b, v3.4b[1]\n"
- "ins v9.d[1], temploadreg1\n"
- ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- ".inst 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n"
- "ins v10.d[1], temploadreg2\n"
- ".inst 0x6fa3e1ff // udot v31.4s, v15.16b, v3.4b[1]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n"
- "ldr d15, [%[b_ptr0], #0x70]\n"
- ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n"
- "ins v11.d[1], temploadreg3\n"
- ".inst 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- ".inst 0x6f83e91c // udot v28.4s, v8.16b, v3.4b[2]\n"
- "ins v12.d[1], temploadreg0\n"
- ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n"
- "ins v13.d[1], temploadreg1\n"
- ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n"
- "ins v14.d[1], temploadreg2\n"
- ".inst 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n"
- "ins v15.d[1], temploadreg3\n"
- ".inst 0x6f83e93d // udot v29.4s, v9.16b, v3.4b[2]\n"
- "subs %[loops], %[loops], #0x1\n"
- ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n"
- "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n"
- ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".inst 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n"
- "ldr d8, [%[b_ptr0], #-0x80]\n"
- ".inst 0x6f83e95e // udot v30.4s, v10.16b, v3.4b[2]\n"
- "ldr temploadreg0, [%[b_ptr0], #-0x78]\n"
- ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n"
- "ldr d9, [%[b_ptr0], #-0x70]\n"
- ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n"
- "ldr temploadreg1, [%[b_ptr0], #-0x68]\n"
- ".inst 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n"
- "ldr d10, [%[b_ptr0], #-0x60]\n"
- ".inst 0x6f83e97f // udot v31.4s, v11.16b, v3.4b[2]\n"
- "ldr temploadreg2, [%[b_ptr0], #-0x58]\n"
- ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n"
- "ldr d11, [%[b_ptr0], #-0x50]\n"
- ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n"
- "ldr temploadreg3, [%[b_ptr0], #-0x48]\n"
- ".inst 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n"
- "ins v8.d[1], temploadreg0\n"
- ".inst 0x6fa3e99c // udot v28.4s, v12.16b, v3.4b[3]\n"
- "ldr d12, [%[b_ptr0], #-0x40]\n"
- ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n"
- "ldr temploadreg0, [%[b_ptr0], #-0x38]\n"
- ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n"
- "ins v9.d[1], temploadreg1\n"
- ".inst 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n"
- "ldr temploadreg1, [%[b_ptr0], #-0x28]\n"
- ".inst 0x6fa3e9bd // udot v29.4s, v13.16b, v3.4b[3]\n"
- "ldr d13, [%[b_ptr0], #-0x30]\n"
- ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n"
- "ins v10.d[1], temploadreg2\n"
- ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n"
- "ldr temploadreg2, [%[b_ptr0], #-0x18]\n"
- ".inst 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n"
- "ins v11.d[1], temploadreg3\n"
- ".inst 0x6fa3e9de // udot v30.4s, v14.16b, v3.4b[3]\n"
- "ldr d14, [%[b_ptr0], #-0x20]\n"
- ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n"
- "ldr temploadreg3, [%[b_ptr0], #-0x8]\n"
- ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n"
- "ins v12.d[1], temploadreg0\n"
- ".inst 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n"
- "ins v13.d[1], temploadreg1\n"
- ".inst 0x6fa3e9ff // udot v31.4s, v15.16b, v3.4b[3]\n"
- "ldr d15, [%[b_ptr0], #-0x10]\n"
- ".inst 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n"
- "ins v14.d[1], temploadreg2\n"
- ".inst 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
- ".inst 0x6f86e118 // udot v24.4s, v8.16b, v6.4b[0]\n"
- "ldr d0, [%[a_ptr0], #-0x10]\n"
- ".inst 0x6f87e11c // udot v28.4s, v8.16b, v7.4b[0]\n"
- "ldr temploadreg0, [%[a_ptr0], #-0x8]\n"
- ".inst 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n"
- "ins v15.d[1], temploadreg3\n"
- ".inst 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n"
- "ldr d8, [%[b_ptr0]]\n"
- ".inst 0x6f86e139 // udot v25.4s, v9.16b, v6.4b[0]\n"
- "ins v0.d[1], temploadreg0\n"
- ".inst 0x6f87e13d // udot v29.4s, v9.16b, v7.4b[0]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".inst 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n"
- "ldr d9, [%[b_ptr0], #0x10]\n"
- ".inst 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n"
- "add a_ptr1, a_ptr1, #0x20\n"
- ".inst 0x6f86e15a // udot v26.4s, v10.16b, v6.4b[0]\n"
- "ldr d1, [a_ptr1, #-0x10]\n"
- ".inst 0x6f87e15e // udot v30.4s, v10.16b, v7.4b[0]\n"
- "ldr temploadreg1, [a_ptr1, #-0x8]\n"
- ".inst 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n"
- "ldr d10, [%[b_ptr0], #0x20]\n"
- ".inst 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n"
- "ins v8.d[1], temploadreg0\n"
- ".inst 0x6f86e17b // udot v27.4s, v11.16b, v6.4b[0]\n"
- "ins v1.d[1], temploadreg1\n"
- ".inst 0x6f87e17f // udot v31.4s, v11.16b, v7.4b[0]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".inst 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n"
- "ldr d11, [%[b_ptr0], #0x30]\n"
- ".inst 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- ".inst 0x6fa6e198 // udot v24.4s, v12.16b, v6.4b[1]\n"
- "ins v9.d[1], temploadreg1\n"
- ".inst 0x6fa7e19c // udot v28.4s, v12.16b, v7.4b[1]\n"
- "ldr d12, [%[b_ptr0], #0x40]\n"
- ".inst 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- ".inst 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n"
- "add a_ptr2, a_ptr2, #0x20\n"
- ".inst 0x6fa6e1b9 // udot v25.4s, v13.16b, v6.4b[1]\n"
- "ldr d2, [a_ptr2, #-0x10]\n"
- ".inst 0x6fa7e1bd // udot v29.4s, v13.16b, v7.4b[1]\n"
- "ldr temploadreg2, [a_ptr2, #-0x8]\n"
- ".inst 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n"
- "ldr d13, [%[b_ptr0], #0x50]\n"
- ".inst 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n"
- "ins v12.d[1], temploadreg0\n"
- ".inst 0x6fa6e1da // udot v26.4s, v14.16b, v6.4b[1]\n"
- "ins v2.d[1], temploadreg2\n"
- ".inst 0x6fa7e1de // udot v30.4s, v14.16b, v7.4b[1]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- ".inst 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- ".inst 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n"
- "ins v13.d[1], temploadreg1\n"
- ".inst 0x6fa6e1fb // udot v27.4s, v15.16b, v6.4b[1]\n"
- "ins v10.d[1], temploadreg2\n"
- ".inst 0x6fa7e1ff // udot v31.4s, v15.16b, v7.4b[1]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- ".inst 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n"
- "ldr d15, [%[b_ptr0], #0x70]\n"
- ".inst 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n"
- "add a_ptr3, a_ptr3, #0x20\n"
- ".inst 0x6f86e918 // udot v24.4s, v8.16b, v6.4b[2]\n"
- "ldr d3, [a_ptr3, #-0x10]\n"
- ".inst 0x6f87e91c // udot v28.4s, v8.16b, v7.4b[2]\n"
- "ldr temploadreg3, [a_ptr3, #-0x8]\n"
- ".inst 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n"
- "ins v14.d[1], temploadreg2\n"
- ".inst 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n"
- "prfm PLDL1KEEP, [a_ptr1, #0x40]\n"
- ".inst 0x6f86e939 // udot v25.4s, v9.16b, v6.4b[2]\n"
- "ins v3.d[1], temploadreg3\n"
- ".inst 0x6f87e93d // udot v29.4s, v9.16b, v7.4b[2]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- ".inst 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n"
- "prfm PLDL1KEEP, [a_ptr2, #0x40]\n"
- ".inst 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n"
- "ins v11.d[1], temploadreg3\n"
- ".inst 0x6f86e95a // udot v26.4s, v10.16b, v6.4b[2]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- ".inst 0x6f87e95e // udot v30.4s, v10.16b, v7.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".inst 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n"
- "ldr d8, [%[b_ptr0], #-0x80]\n"
- ".inst 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n"
- "ldr temploadreg0, [%[b_ptr0], #-0x78]\n"
- ".inst 0x6f86e97b // udot v27.4s, v11.16b, v6.4b[2]\n"
- "ldr d9, [%[b_ptr0], #-0x70]\n"
- ".inst 0x6f87e97f // udot v31.4s, v11.16b, v7.4b[2]\n"
- "ldr temploadreg1, [%[b_ptr0], #-0x68]\n"
- ".inst 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n"
- "ldr d10, [%[b_ptr0], #-0x60]\n"
- ".inst 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n"
- "ldr temploadreg2, [%[b_ptr0], #-0x58]\n"
- ".inst 0x6fa6e998 // udot v24.4s, v12.16b, v6.4b[3]\n"
- "ldr d11, [%[b_ptr0], #-0x50]\n"
- ".inst 0x6fa7e99c // udot v28.4s, v12.16b, v7.4b[3]\n"
- "ins v15.d[1], temploadreg3\n"
- ".inst 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n"
- "ldr temploadreg3, [%[b_ptr0], #-0x48]\n"
- ".inst 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n"
- "ldr d12, [%[b_ptr0], #-0x40]\n"
- ".inst 0x6fa6e9b9 // udot v25.4s, v13.16b, v6.4b[3]\n"
- "ins v8.d[1], temploadreg0\n"
- ".inst 0x6fa7e9bd // udot v29.4s, v13.16b, v7.4b[3]\n"
- "ldr temploadreg0, [%[b_ptr0], #-0x38]\n"
- ".inst 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n"
- "ldr d13, [%[b_ptr0], #-0x30]\n"
- ".inst 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n"
- "ins v9.d[1], temploadreg1\n"
- ".inst 0x6fa6e9da // udot v26.4s, v14.16b, v6.4b[3]\n"
- "ldr temploadreg1, [%[b_ptr0], #-0x28]\n"
- ".inst 0x6fa7e9de // udot v30.4s, v14.16b, v7.4b[3]\n"
- "ldr d14, [%[b_ptr0], #-0x20]\n"
- ".inst 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n"
- "ins v10.d[1], temploadreg2\n"
- ".inst 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n"
- "ldr temploadreg2, [%[b_ptr0], #-0x18]\n"
- ".inst 0x6fa6e9fb // udot v27.4s, v15.16b, v6.4b[3]\n"
- "ins v11.d[1], temploadreg3\n"
- ".inst 0x6fa7e9ff // udot v31.4s, v15.16b, v7.4b[3]\n"
- "ins v12.d[1], temploadreg0\n"
- "ins v13.d[1], temploadreg1\n"
- "prfm PLDL1KEEP, [a_ptr3, #0x40]\n"
- "ins v14.d[1], temploadreg2\n"
- "b.ne 3b\n"
- "2:\n"
- "ldr d15, [%[b_ptr0], #-0x10]\n"
- "prfm PSTL1KEEP, [%[c_ptr0]]\n"
- "ldr temploadreg3, [%[b_ptr0], #-0x8]\n"
- "prfm PSTL1KEEP, [c_ptr1]\n"
- "prfm PSTL1KEEP, [c_ptr2]\n"
- "prfm PSTL1KEEP, [c_ptr3]\n"
- "ins v15.d[1], temploadreg3\n"
- "cbz %[regs], 4f\n"
- ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
- "ldr d4, [%[a_ptr0]]\n"
- ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
- "ldr temploadreg0, [%[a_ptr0], #0x8]\n"
- ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n"
- "ldr d5, [a_ptr1]\n"
- ".inst 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n"
- "ldr temploadreg1, [a_ptr1, #0x8]\n"
- ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
- "ldr d6, [a_ptr2]\n"
- ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
- "ldr temploadreg2, [a_ptr2, #0x8]\n"
- ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n"
- "ldr d7, [a_ptr3]\n"
- ".inst 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n"
- "ldr temploadreg3, [a_ptr3, #0x8]\n"
- ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
- "ldr d8, [%[b_ptr0]]\n"
- ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
- "ins v4.d[1], temploadreg0\n"
- ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".inst 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n"
- "ldr d9, [%[b_ptr0], #0x10]\n"
- ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
- "ins v5.d[1], temploadreg1\n"
- ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n"
- "ldr d10, [%[b_ptr0], #0x20]\n"
- ".inst 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n"
- "ins v6.d[1], temploadreg2\n"
- ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n"
- "ldr d11, [%[b_ptr0], #0x30]\n"
- ".inst 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n"
- "ins v7.d[1], temploadreg3\n"
- ".inst 0x6fa3e19c // udot v28.4s, v12.16b, v3.4b[1]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n"
- "ldr d12, [%[b_ptr0], #0x40]\n"
- ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n"
- "ins v8.d[1], temploadreg0\n"
- ".inst 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- ".inst 0x6fa3e1bd // udot v29.4s, v13.16b, v3.4b[1]\n"
- "ldr d13, [%[b_ptr0], #0x50]\n"
- ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n"
- "ins v9.d[1], temploadreg1\n"
- ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- ".inst 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n"
- "ins v10.d[1], temploadreg2\n"
- ".inst 0x6fa3e1de // udot v30.4s, v14.16b, v3.4b[1]\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n"
- "ins v11.d[1], temploadreg3\n"
- ".inst 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- ".inst 0x6fa3e1ff // udot v31.4s, v15.16b, v3.4b[1]\n"
- "ldr d15, [%[b_ptr0], #0x70]\n"
- ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n"
- "ins v12.d[1], temploadreg0\n"
- ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n"
- "ins v13.d[1], temploadreg1\n"
- ".inst 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n"
- "ins v14.d[1], temploadreg2\n"
- ".inst 0x6f83e91c // udot v28.4s, v8.16b, v3.4b[2]\n"
- "ins v15.d[1], temploadreg3\n"
- ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n"
- "ldr d8, [%[b_ptr0], #-0x80]\n"
- ".inst 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n"
- "ldr temploadreg0, [%[b_ptr0], #-0x78]\n"
- ".inst 0x6f83e93d // udot v29.4s, v9.16b, v3.4b[2]\n"
- "ldr d9, [%[b_ptr0], #-0x70]\n"
- ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n"
- "ldr temploadreg1, [%[b_ptr0], #-0x68]\n"
- ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n"
- "ldr temploadreg2, [%[b_ptr0], #-0x58]\n"
- ".inst 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n"
- "ldr temploadreg3, [%[b_ptr0], #-0x48]\n"
- ".inst 0x6f83e95e // udot v30.4s, v10.16b, v3.4b[2]\n"
- "ldr d10, [%[b_ptr0], #-0x60]\n"
- ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n"
- "ins v8.d[1], temploadreg0\n"
- ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n"
- "ldr temploadreg0, [%[b_ptr0], #-0x38]\n"
- ".inst 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n"
- "ins v9.d[1], temploadreg1\n"
- ".inst 0x6f83e97f // udot v31.4s, v11.16b, v3.4b[2]\n"
- "ldr d11, [%[b_ptr0], #-0x50]\n"
- ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n"
- "ldr temploadreg1, [%[b_ptr0], #-0x28]\n"
- ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n"
- "ins v10.d[1], temploadreg2\n"
- ".inst 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n"
- "ldr temploadreg2, [%[b_ptr0], #-0x18]\n"
- ".inst 0x6fa3e99c // udot v28.4s, v12.16b, v3.4b[3]\n"
- "ldr d12, [%[b_ptr0], #-0x40]\n"
- ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n"
- "ins v11.d[1], temploadreg3\n"
- ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n"
- "ldr temploadreg3, [%[b_ptr0], #-0x8]\n"
- ".inst 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n"
- "ins v12.d[1], temploadreg0\n"
- ".inst 0x6fa3e9bd // udot v29.4s, v13.16b, v3.4b[3]\n"
- "ldr d13, [%[b_ptr0], #-0x30]\n"
- ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- ".inst 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n"
- "ins v13.d[1], temploadreg1\n"
- ".inst 0x6fa3e9de // udot v30.4s, v14.16b, v3.4b[3]\n"
- "ldr d14, [%[b_ptr0], #-0x20]\n"
- ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- ".inst 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n"
- "ins v14.d[1], temploadreg2\n"
- ".inst 0x6fa3e9ff // udot v31.4s, v15.16b, v3.4b[3]\n"
- "ldr d15, [%[b_ptr0], #-0x10]\n"
- ".inst 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- ".inst 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- ".inst 0x6f86e118 // udot v24.4s, v8.16b, v6.4b[0]\n"
- "ins v15.d[1], temploadreg3\n"
- ".inst 0x6f87e11c // udot v28.4s, v8.16b, v7.4b[0]\n"
- "ldr d8, [%[b_ptr0]]\n"
- ".inst 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- ".inst 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n"
- "add a_ptr3, a_ptr3, #0x10\n"
- ".inst 0x6f86e139 // udot v25.4s, v9.16b, v6.4b[0]\n"
- "ins v8.d[1], temploadreg0\n"
- ".inst 0x6f87e13d // udot v29.4s, v9.16b, v7.4b[0]\n"
- "ldr d9, [%[b_ptr0], #0x10]\n"
- ".inst 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- ".inst 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n"
- ".inst 0x6f86e15a // udot v26.4s, v10.16b, v6.4b[0]\n"
- "ins v9.d[1], temploadreg1\n"
- ".inst 0x6f87e15e // udot v30.4s, v10.16b, v7.4b[0]\n"
- "ldr d10, [%[b_ptr0], #0x20]\n"
- ".inst 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- ".inst 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n"
- ".inst 0x6f86e17b // udot v27.4s, v11.16b, v6.4b[0]\n"
- "ins v10.d[1], temploadreg2\n"
- ".inst 0x6f87e17f // udot v31.4s, v11.16b, v7.4b[0]\n"
- "ldr d11, [%[b_ptr0], #0x30]\n"
- ".inst 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- ".inst 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n"
- ".inst 0x6fa6e198 // udot v24.4s, v12.16b, v6.4b[1]\n"
- "ins v11.d[1], temploadreg3\n"
- ".inst 0x6fa7e19c // udot v28.4s, v12.16b, v7.4b[1]\n"
- "ldr d12, [%[b_ptr0], #0x40]\n"
- ".inst 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- ".inst 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n"
- ".inst 0x6fa6e1b9 // udot v25.4s, v13.16b, v6.4b[1]\n"
- "ins v12.d[1], temploadreg0\n"
- ".inst 0x6fa7e1bd // udot v29.4s, v13.16b, v7.4b[1]\n"
- "ldr d13, [%[b_ptr0], #0x50]\n"
- ".inst 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n"
- ".inst 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n"
- ".inst 0x6fa6e1da // udot v26.4s, v14.16b, v6.4b[1]\n"
- "ins v13.d[1], temploadreg1\n"
- ".inst 0x6fa7e1de // udot v30.4s, v14.16b, v7.4b[1]\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- ".inst 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n"
- ".inst 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n"
- ".inst 0x6fa6e1fb // udot v27.4s, v15.16b, v6.4b[1]\n"
- "ins v14.d[1], temploadreg2\n"
- ".inst 0x6fa7e1ff // udot v31.4s, v15.16b, v7.4b[1]\n"
- "ldr d15, [%[b_ptr0], #0x70]\n"
- ".inst 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- ".inst 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n"
- "ins v15.d[1], temploadreg3\n"
- ".inst 0x6f86e918 // udot v24.4s, v8.16b, v6.4b[2]\n"
- ".inst 0x6f87e91c // udot v28.4s, v8.16b, v7.4b[2]\n"
- ".inst 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n"
- ".inst 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n"
- ".inst 0x6f86e939 // udot v25.4s, v9.16b, v6.4b[2]\n"
- ".inst 0x6f87e93d // udot v29.4s, v9.16b, v7.4b[2]\n"
- ".inst 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n"
- ".inst 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n"
- ".inst 0x6f86e95a // udot v26.4s, v10.16b, v6.4b[2]\n"
- ".inst 0x6f87e95e // udot v30.4s, v10.16b, v7.4b[2]\n"
- ".inst 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n"
- ".inst 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n"
- ".inst 0x6f86e97b // udot v27.4s, v11.16b, v6.4b[2]\n"
- ".inst 0x6f87e97f // udot v31.4s, v11.16b, v7.4b[2]\n"
- ".inst 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n"
- ".inst 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n"
- ".inst 0x6fa6e998 // udot v24.4s, v12.16b, v6.4b[3]\n"
- ".inst 0x6fa7e99c // udot v28.4s, v12.16b, v7.4b[3]\n"
- ".inst 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n"
- ".inst 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n"
- ".inst 0x6fa6e9b9 // udot v25.4s, v13.16b, v6.4b[3]\n"
- ".inst 0x6fa7e9bd // udot v29.4s, v13.16b, v7.4b[3]\n"
- ".inst 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n"
- ".inst 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n"
- ".inst 0x6fa6e9da // udot v26.4s, v14.16b, v6.4b[3]\n"
- ".inst 0x6fa7e9de // udot v30.4s, v14.16b, v7.4b[3]\n"
- ".inst 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n"
- ".inst 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n"
- ".inst 0x6fa6e9fb // udot v27.4s, v15.16b, v6.4b[3]\n"
- ".inst 0x6fa7e9ff // udot v31.4s, v15.16b, v7.4b[3]\n"
- "b 5f\n"
- "4:\n"
- ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- ".inst 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n"
- "ldr d8, [%[b_ptr0]]\n"
- ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
- ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n"
- "ins v8.d[1], temploadreg0\n"
- ".inst 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n"
- "ldr d9, [%[b_ptr0], #0x10]\n"
- ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
- ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n"
- "ins v9.d[1], temploadreg1\n"
- ".inst 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n"
- "ldr d10, [%[b_ptr0], #0x20]\n"
- ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
- ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n"
- "ins v10.d[1], temploadreg2\n"
- ".inst 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n"
- "ldr d11, [%[b_ptr0], #0x30]\n"
- ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n"
- ".inst 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n"
- "ins v11.d[1], temploadreg3\n"
- ".inst 0x6fa3e19c // udot v28.4s, v12.16b, v3.4b[1]\n"
- "ldr d12, [%[b_ptr0], #0x40]\n"
- ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n"
- ".inst 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n"
- "ins v12.d[1], temploadreg0\n"
- ".inst 0x6fa3e1bd // udot v29.4s, v13.16b, v3.4b[1]\n"
- "ldr d13, [%[b_ptr0], #0x50]\n"
- ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n"
- ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n"
- ".inst 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n"
- "ins v13.d[1], temploadreg1\n"
- ".inst 0x6fa3e1de // udot v30.4s, v14.16b, v3.4b[1]\n"
- "ldr d14, [%[b_ptr0], #0x60]\n"
- ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n"
- ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n"
- ".inst 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n"
- "ins v14.d[1], temploadreg2\n"
- ".inst 0x6fa3e1ff // udot v31.4s, v15.16b, v3.4b[1]\n"
- "ldr d15, [%[b_ptr0], #0x70]\n"
- ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n"
- "ins v15.d[1], temploadreg3\n"
- ".inst 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n"
- ".inst 0x6f83e91c // udot v28.4s, v8.16b, v3.4b[2]\n"
- ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n"
- ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n"
- ".inst 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n"
- ".inst 0x6f83e93d // udot v29.4s, v9.16b, v3.4b[2]\n"
- ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n"
- ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n"
- ".inst 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n"
- ".inst 0x6f83e95e // udot v30.4s, v10.16b, v3.4b[2]\n"
- ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n"
- ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n"
- ".inst 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n"
- ".inst 0x6f83e97f // udot v31.4s, v11.16b, v3.4b[2]\n"
- ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n"
- ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n"
- ".inst 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n"
- ".inst 0x6fa3e99c // udot v28.4s, v12.16b, v3.4b[3]\n"
- ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n"
- ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n"
- ".inst 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n"
- ".inst 0x6fa3e9bd // udot v29.4s, v13.16b, v3.4b[3]\n"
- ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n"
- ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n"
- ".inst 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n"
- ".inst 0x6fa3e9de // udot v30.4s, v14.16b, v3.4b[3]\n"
- ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n"
- ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n"
- ".inst 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n"
- ".inst 0x6fa3e9ff // udot v31.4s, v15.16b, v3.4b[3]\n"
- "5:\n"
- "cbz %[blocks], 6f\n"
- "7:\n"
- "ldr q8, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "ldr s0, [%[a_ptr0]]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "add %[a_ptr0], %[a_ptr0], #0x4\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "add %[b_ptr0], %[b_ptr0], #0x40\n"
- ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
- "ldr s1, [a_ptr1]\n"
- ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
- "add a_ptr1, a_ptr1, #0x4\n"
- ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
- "ldr s2, [a_ptr2]\n"
- ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
- "add a_ptr2, a_ptr2, #0x4\n"
- ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
- "ldr s3, [a_ptr3]\n"
- ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n"
- "add a_ptr3, a_ptr3, #0x4\n"
- ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n"
- ".inst 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n"
- ".inst 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n"
- ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
- ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n"
- ".inst 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n"
- ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
- ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
- ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n"
- ".inst 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n"
- "b.ne 7b\n"
- "6:\n"
- "cbz %[odds], 8f\n"
- "ld1 {v0.b}[0], [%[a_ptr0]], #1\n"
- "ld1 {v1.b}[0], [a_ptr1], #1\n"
- "ld1 {v2.b}[0], [a_ptr2], #1\n"
- "ld1 {v3.b}[0], [a_ptr3], #1\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 9f\n"
- "ld1 {v0.b}[1], [%[a_ptr0]], #1\n"
- "ld1 {v1.b}[1], [a_ptr1], #1\n"
- "ld1 {v2.b}[1], [a_ptr2], #1\n"
- "ld1 {v3.b}[1], [a_ptr3], #1\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 9f\n"
- "ld1 {v0.b}[2], [%[a_ptr0]]\n"
- "ld1 {v1.b}[2], [a_ptr1]\n"
- "ld1 {v2.b}[2], [a_ptr2]\n"
- "ld1 {v3.b}[2], [a_ptr3]\n"
- "9:\n"
- "ldr q8, [%[b_ptr0]]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
- ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
- ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n"
- ".inst 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n"
- ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
- ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
- ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n"
- ".inst 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n"
- ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
- ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
- ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n"
- ".inst 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n"
- ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
- ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
- ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n"
- ".inst 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n"
- "8:\n"
- "str q16, [%[c_ptr0]]\n"
- "str q17, [%[c_ptr0], #0x10]\n"
- "str q18, [%[c_ptr0], #0x20]\n"
- "str q19, [%[c_ptr0], #0x30]\n"
- "add %[c_ptr0], %[c_ptr0], #0x40\n"
- "str q20, [c_ptr1]\n"
- "str q21, [c_ptr1, #0x10]\n"
- "str q22, [c_ptr1, #0x20]\n"
- "str q23, [c_ptr1, #0x30]\n"
- "str q24, [c_ptr2]\n"
- "str q25, [c_ptr2, #0x10]\n"
- "str q26, [c_ptr2, #0x20]\n"
- "str q27, [c_ptr2, #0x30]\n"
- "str q28, [c_ptr3]\n"
- "str q29, [c_ptr3, #0x10]\n"
- "str q30, [c_ptr3, #0x20]\n"
- "str q31, [c_ptr3, #0x30]\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq a_ptr3\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- ".unreq c_ptr3\n"
- ".unreq temploadreg0\n"
- ".unreq temploadreg1\n"
- ".unreq temploadreg2\n"
- ".unreq temploadreg3\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks), [odds] "+r" (odds)
- : [width] "r" (width), [accumulate] "r" (static_cast<uint64_t>(accumulate)), [lda] "r" (ldab), [ldc] "r" (ldcb)
- : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "cc", "memory"
- );
- break;
- }
- if (use_result_buffer) {
- for(int cy=0; cy<std::min(M-y, 4); cy++) {
- for(unsigned int cx=0; cx<width; cx++) {
- c_ptr_real[cy * ldc + cx] = result_buffer[cy * 16 + cx];
- }
- }
- }
- }
- }
-}
-
-} // namespace arm_gemm
-
-#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_dot_16x4/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_dot_16x4/generic.cpp
deleted file mode 100644
index 2e86233..0000000
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_dot_16x4/generic.cpp
+++ /dev/null
@@ -1,1808 +0,0 @@
-/*
- * Copyright (c) 2018-2020 Arm Limited.
- *
- * SPDX-License-Identifier: MIT
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to
- * deal in the Software without restriction, including without limitation the
- * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
- * sell copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-#ifdef __aarch64__
-
-#include <algorithm>
-
-#include "arm_gemm.hpp"
-#include <cstdint>
-#include "../../asmlib.hpp"
-#include "../../utils.hpp"
-
-namespace arm_gemm {
-
-void a64_hybrid_u8u32_dot_16x4(const uint8_t *A, int lda, const uint8_t *B, uint32_t *C, int ldc, int M, int N, int K, const uint32_t *, Activation , bool accumulate) {
- const int K_stride = ((K + 3) / 4) * 4;
- const long loops_count = ((K + 16) / 32) - 1;
- K -= loops_count * 32;
- const long regs_count = (K / 16) - 1;
- K -= (regs_count + 1) * 16;
- const long blocks_count = K / 4;
- const long odds_count = K - (blocks_count * 4);
-
- int rows_to_compute;
-
- for (int y=0; y<M; y+=rows_to_compute) {
- const uint8_t * const a_ptr0_base = A + (y * lda);
- const unsigned long ldab = lda * sizeof(uint8_t);
-
- uint32_t *c_ptr0 = C + (y * ldc);
-
- rows_to_compute = M-y;
- if (rows_to_compute > 4) {
- if (rows_to_compute % 4) {
- rows_to_compute = 4 - 1;
- } else {
- rows_to_compute = 4;
- }
- }
-
- for (int x0=0; x0<N; x0+=16ul) {
- const long width = std::min((unsigned long)N-x0, 16ul);
- long loops = loops_count;
- long regs = regs_count;
- long blocks = blocks_count;
- long odds = odds_count;
- const uint8_t *a_ptr0 = a_ptr0_base;
- const uint8_t *b_ptr0 = B + (K_stride * x0);
- const bool use_result_buffer = (width < 16);
- uint32_t result_buffer[64];
- const unsigned long ldcb = (use_result_buffer ? 16 : ldc) * sizeof(uint32_t);
- uint32_t *c_ptr_real = c_ptr0;
- if (use_result_buffer && accumulate) {
- for(int cy=0; cy<std::min(M-y, 4); cy++) {
- for(unsigned int cx=0; cx<width; cx++) {
- result_buffer[cy * 16 + cx] = c_ptr_real[cy * ldc + cx];
- }
- }
- }
- if (use_result_buffer) {
- c_ptr0 = result_buffer;
- }
-
- switch(rows_to_compute) {
- case 1:
- __asm __volatile (
- "cbnz %[accumulate], 1f\n"
- "movi v16.4s, #0\n"
- "ldr q0, [%[a_ptr0]]\n"
- "movi v17.4s, #0\n"
- "ldr q8, [%[b_ptr0]]\n"
- "movi v18.4s, #0\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "movi v19.4s, #0\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "ldr q16, [%[c_ptr0]]\n"
- "ldr q17, [%[c_ptr0], #0x10]\n"
- "ldr q18, [%[c_ptr0], #0x20]\n"
- "ldr q19, [%[c_ptr0], #0x30]\n"
- "ldr q0, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ldr q8, [%[b_ptr0]]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "cbz %[loops], 2f\n"
- "3:\n"
- ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
- "ldr q15, [%[b_ptr0], #-0x10]\n"
- ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
- "ldr q4, [%[a_ptr0]]\n"
- ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
- "ldr q8, [%[b_ptr0]]\n"
- ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n"
- "ldr q15, [%[b_ptr0], #0x70]\n"
- ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n"
- "subs %[loops], %[loops], #0x1\n"
- ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n"
- "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n"
- ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n"
- "ldr q8, [%[b_ptr0], #-0x80]\n"
- ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n"
- "ldr q9, [%[b_ptr0], #-0x70]\n"
- ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n"
- "ldr q10, [%[b_ptr0], #-0x60]\n"
- "ldr q11, [%[b_ptr0], #-0x50]\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
- ".inst 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n"
- "ldr q12, [%[b_ptr0], #-0x40]\n"
- ".inst 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n"
- "ldr q13, [%[b_ptr0], #-0x30]\n"
- ".inst 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n"
- "ldr q14, [%[b_ptr0], #-0x20]\n"
- ".inst 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n"
- "ldr q15, [%[b_ptr0], #-0x10]\n"
- ".inst 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n"
- "ldr q0, [%[a_ptr0], #-0x10]\n"
- ".inst 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n"
- "ldr q8, [%[b_ptr0]]\n"
- ".inst 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- ".inst 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- ".inst 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- ".inst 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- ".inst 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- ".inst 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n"
- "ldr q15, [%[b_ptr0], #0x70]\n"
- ".inst 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".inst 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n"
- "ldr q8, [%[b_ptr0], #-0x80]\n"
- ".inst 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n"
- "ldr q9, [%[b_ptr0], #-0x70]\n"
- ".inst 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n"
- "ldr q10, [%[b_ptr0], #-0x60]\n"
- "ldr q11, [%[b_ptr0], #-0x50]\n"
- "ldr q12, [%[b_ptr0], #-0x40]\n"
- "ldr q13, [%[b_ptr0], #-0x30]\n"
- "ldr q14, [%[b_ptr0], #-0x20]\n"
- "b.ne 3b\n"
- "2:\n"
- "ldr q15, [%[b_ptr0], #-0x10]\n"
- "prfm PSTL1KEEP, [%[c_ptr0]]\n"
- "cbz %[regs], 4f\n"
- ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
- "ldr q4, [%[a_ptr0]]\n"
- ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
- "ldr q8, [%[b_ptr0]]\n"
- ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n"
- "ldr q15, [%[b_ptr0], #0x70]\n"
- ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n"
- "ldr q8, [%[b_ptr0], #-0x80]\n"
- ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n"
- "ldr q9, [%[b_ptr0], #-0x70]\n"
- ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n"
- "ldr q10, [%[b_ptr0], #-0x60]\n"
- ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n"
- "ldr q11, [%[b_ptr0], #-0x50]\n"
- ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n"
- "ldr q12, [%[b_ptr0], #-0x40]\n"
- ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n"
- "ldr q13, [%[b_ptr0], #-0x30]\n"
- ".inst 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n"
- "ldr q14, [%[b_ptr0], #-0x20]\n"
- ".inst 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n"
- "ldr q15, [%[b_ptr0], #-0x10]\n"
- ".inst 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n"
- "ldr q8, [%[b_ptr0]]\n"
- ".inst 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- ".inst 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- ".inst 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- ".inst 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- ".inst 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- ".inst 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- ".inst 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n"
- "ldr q15, [%[b_ptr0], #0x70]\n"
- ".inst 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- ".inst 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- ".inst 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n"
- ".inst 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n"
- ".inst 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n"
- ".inst 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n"
- "b 5f\n"
- "4:\n"
- ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
- "ldr q8, [%[b_ptr0]]\n"
- ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n"
- "ldr q15, [%[b_ptr0], #0x70]\n"
- ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n"
- ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n"
- ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n"
- ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n"
- ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n"
- ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n"
- ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n"
- "5:\n"
- "cbz %[blocks], 6f\n"
- "7:\n"
- "ldr q8, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "ldr s0, [%[a_ptr0]]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "add %[a_ptr0], %[a_ptr0], #0x4\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "add %[b_ptr0], %[b_ptr0], #0x40\n"
- ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
- ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
- ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
- ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
- "b.ne 7b\n"
- "6:\n"
- "cbz %[odds], 8f\n"
- "ld1 {v0.b}[0], [%[a_ptr0]], #1\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 9f\n"
- "ld1 {v0.b}[1], [%[a_ptr0]], #1\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 9f\n"
- "ld1 {v0.b}[2], [%[a_ptr0]]\n"
- "9:\n"
- "ldr q8, [%[b_ptr0]]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
- ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
- ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
- ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
- "8:\n"
- "str q16, [%[c_ptr0]]\n"
- "str q17, [%[c_ptr0], #0x10]\n"
- "str q18, [%[c_ptr0], #0x20]\n"
- "str q19, [%[c_ptr0], #0x30]\n"
- "add %[c_ptr0], %[c_ptr0], #0x40\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks), [odds] "+r" (odds)
- : [width] "r" (width), [accumulate] "r" (static_cast<uint64_t>(accumulate)), [lda] "r" (ldab), [ldc] "r" (ldcb)
- : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "cc", "memory"
- );
- break;
- case 2:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "c_ptr1 .req X1\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "cbnz %[accumulate], 1f\n"
- "movi v16.4s, #0\n"
- "ldr q0, [%[a_ptr0]]\n"
- "movi v17.4s, #0\n"
- "ldr q1, [a_ptr1]\n"
- "movi v18.4s, #0\n"
- "ldr q8, [%[b_ptr0]]\n"
- "movi v19.4s, #0\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "movi v20.4s, #0\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "movi v21.4s, #0\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "movi v22.4s, #0\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- "movi v23.4s, #0\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "ldr q16, [%[c_ptr0]]\n"
- "ldr q17, [%[c_ptr0], #0x10]\n"
- "ldr q18, [%[c_ptr0], #0x20]\n"
- "ldr q19, [%[c_ptr0], #0x30]\n"
- "ldr q20, [c_ptr1]\n"
- "ldr q21, [c_ptr1, #0x10]\n"
- "ldr q22, [c_ptr1, #0x20]\n"
- "ldr q23, [c_ptr1, #0x30]\n"
- "ldr q0, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ldr q1, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "ldr q8, [%[b_ptr0]]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "cbz %[loops], 2f\n"
- "3:\n"
- ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
- "ldr q15, [%[b_ptr0], #-0x10]\n"
- ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
- "ldr q4, [%[a_ptr0]]\n"
- ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
- "ldr q5, [a_ptr1]\n"
- ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
- "ldr q8, [%[b_ptr0]]\n"
- ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
- "subs %[loops], %[loops], #0x1\n"
- ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n"
- "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n"
- ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
- ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n"
- "add a_ptr1, a_ptr1, #0x20\n"
- ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n"
- "prfm PLDL1KEEP, [a_ptr1, #0x40]\n"
- ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n"
- "ldr q15, [%[b_ptr0], #0x70]\n"
- ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n"
- "ldr q8, [%[b_ptr0], #-0x80]\n"
- ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n"
- ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n"
- "ldr q9, [%[b_ptr0], #-0x70]\n"
- ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n"
- ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n"
- "ldr q10, [%[b_ptr0], #-0x60]\n"
- ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n"
- ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n"
- "ldr q11, [%[b_ptr0], #-0x50]\n"
- ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n"
- ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n"
- "ldr q12, [%[b_ptr0], #-0x40]\n"
- ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n"
- ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n"
- "ldr q13, [%[b_ptr0], #-0x30]\n"
- ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n"
- ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n"
- "ldr q14, [%[b_ptr0], #-0x20]\n"
- ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n"
- "ldr q0, [%[a_ptr0], #-0x10]\n"
- ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n"
- "ldr q15, [%[b_ptr0], #-0x10]\n"
- ".inst 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n"
- "ldr q1, [a_ptr1, #-0x10]\n"
- ".inst 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n"
- "ldr q8, [%[b_ptr0]]\n"
- ".inst 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n"
- ".inst 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- ".inst 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n"
- ".inst 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- ".inst 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n"
- ".inst 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- ".inst 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n"
- ".inst 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- ".inst 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n"
- ".inst 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- ".inst 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n"
- ".inst 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- ".inst 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n"
- ".inst 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n"
- "ldr q15, [%[b_ptr0], #0x70]\n"
- ".inst 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".inst 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n"
- "ldr q8, [%[b_ptr0], #-0x80]\n"
- ".inst 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n"
- ".inst 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n"
- "ldr q9, [%[b_ptr0], #-0x70]\n"
- ".inst 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n"
- ".inst 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n"
- "ldr q10, [%[b_ptr0], #-0x60]\n"
- ".inst 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n"
- ".inst 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n"
- "ldr q11, [%[b_ptr0], #-0x50]\n"
- ".inst 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n"
- ".inst 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n"
- "ldr q12, [%[b_ptr0], #-0x40]\n"
- ".inst 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n"
- ".inst 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n"
- "ldr q13, [%[b_ptr0], #-0x30]\n"
- ".inst 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n"
- ".inst 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n"
- "ldr q14, [%[b_ptr0], #-0x20]\n"
- ".inst 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n"
- ".inst 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n"
- "b.ne 3b\n"
- "2:\n"
- "ldr q15, [%[b_ptr0], #-0x10]\n"
- "prfm PSTL1KEEP, [%[c_ptr0]]\n"
- "prfm PSTL1KEEP, [c_ptr1]\n"
- "cbz %[regs], 4f\n"
- ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
- "ldr q4, [%[a_ptr0]]\n"
- ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
- "ldr q5, [a_ptr1]\n"
- ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
- "ldr q8, [%[b_ptr0]]\n"
- ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n"
- ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n"
- ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n"
- ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n"
- ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n"
- "ldr q15, [%[b_ptr0], #0x70]\n"
- ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n"
- "ldr q8, [%[b_ptr0], #-0x80]\n"
- ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n"
- ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n"
- "ldr q9, [%[b_ptr0], #-0x70]\n"
- ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n"
- ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n"
- "ldr q10, [%[b_ptr0], #-0x60]\n"
- ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n"
- ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n"
- "ldr q11, [%[b_ptr0], #-0x50]\n"
- ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n"
- ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n"
- "ldr q12, [%[b_ptr0], #-0x40]\n"
- ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n"
- ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n"
- "ldr q13, [%[b_ptr0], #-0x30]\n"
- ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n"
- ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n"
- "ldr q14, [%[b_ptr0], #-0x20]\n"
- ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n"
- ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n"
- "ldr q15, [%[b_ptr0], #-0x10]\n"
- ".inst 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n"
- ".inst 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n"
- "ldr q8, [%[b_ptr0]]\n"
- ".inst 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n"
- ".inst 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- ".inst 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n"
- ".inst 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- ".inst 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n"
- ".inst 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- ".inst 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n"
- ".inst 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- ".inst 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n"
- ".inst 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- ".inst 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n"
- ".inst 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- ".inst 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n"
- ".inst 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n"
- "ldr q15, [%[b_ptr0], #0x70]\n"
- ".inst 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- ".inst 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n"
- ".inst 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n"
- ".inst 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n"
- ".inst 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n"
- ".inst 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n"
- ".inst 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n"
- ".inst 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n"
- ".inst 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n"
- ".inst 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n"
- ".inst 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n"
- ".inst 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n"
- ".inst 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n"
- ".inst 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n"
- ".inst 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n"
- ".inst 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n"
- "b 5f\n"
- "4:\n"
- ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
- ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
- "ldr q8, [%[b_ptr0]]\n"
- ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
- ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
- ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
- ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n"
- ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n"
- ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n"
- ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n"
- ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n"
- "ldr q15, [%[b_ptr0], #0x70]\n"
- ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n"
- ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n"
- ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n"
- ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n"
- ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n"
- ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n"
- ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n"
- ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n"
- ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n"
- ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n"
- ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n"
- ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n"
- ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n"
- ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n"
- ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n"
- "5:\n"
- "cbz %[blocks], 6f\n"
- "7:\n"
- "ldr q8, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "ldr s0, [%[a_ptr0]]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "add %[a_ptr0], %[a_ptr0], #0x4\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "add %[b_ptr0], %[b_ptr0], #0x40\n"
- ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
- "ldr s1, [a_ptr1]\n"
- ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
- "add a_ptr1, a_ptr1, #0x4\n"
- ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
- ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
- ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
- ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
- ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
- ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
- "b.ne 7b\n"
- "6:\n"
- "cbz %[odds], 8f\n"
- "ld1 {v0.b}[0], [%[a_ptr0]], #1\n"
- "ld1 {v1.b}[0], [a_ptr1], #1\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 9f\n"
- "ld1 {v0.b}[1], [%[a_ptr0]], #1\n"
- "ld1 {v1.b}[1], [a_ptr1], #1\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 9f\n"
- "ld1 {v0.b}[2], [%[a_ptr0]]\n"
- "ld1 {v1.b}[2], [a_ptr1]\n"
- "9:\n"
- "ldr q8, [%[b_ptr0]]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
- ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
- ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
- ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
- ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
- ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
- ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
- ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
- "8:\n"
- "str q16, [%[c_ptr0]]\n"
- "str q17, [%[c_ptr0], #0x10]\n"
- "str q18, [%[c_ptr0], #0x20]\n"
- "str q19, [%[c_ptr0], #0x30]\n"
- "add %[c_ptr0], %[c_ptr0], #0x40\n"
- "str q20, [c_ptr1]\n"
- "str q21, [c_ptr1, #0x10]\n"
- "str q22, [c_ptr1, #0x20]\n"
- "str q23, [c_ptr1, #0x30]\n"
- ".unreq a_ptr1\n"
- ".unreq c_ptr1\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks), [odds] "+r" (odds)
- : [width] "r" (width), [accumulate] "r" (static_cast<uint64_t>(accumulate)), [lda] "r" (ldab), [ldc] "r" (ldcb)
- : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "cc", "memory"
- );
- break;
- case 3:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "c_ptr1 .req X2\n"
- "c_ptr2 .req X3\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "cbnz %[accumulate], 1f\n"
- "movi v16.4s, #0\n"
- "ldr q0, [%[a_ptr0]]\n"
- "movi v17.4s, #0\n"
- "ldr q1, [a_ptr1]\n"
- "movi v18.4s, #0\n"
- "ldr q2, [a_ptr2]\n"
- "movi v19.4s, #0\n"
- "ldr q8, [%[b_ptr0]]\n"
- "movi v20.4s, #0\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "movi v21.4s, #0\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "movi v22.4s, #0\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "movi v23.4s, #0\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- "movi v24.4s, #0\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- "movi v25.4s, #0\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- "movi v26.4s, #0\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "movi v27.4s, #0\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "ldr q16, [%[c_ptr0]]\n"
- "ldr q17, [%[c_ptr0], #0x10]\n"
- "ldr q18, [%[c_ptr0], #0x20]\n"
- "ldr q19, [%[c_ptr0], #0x30]\n"
- "ldr q20, [c_ptr1]\n"
- "ldr q21, [c_ptr1, #0x10]\n"
- "ldr q22, [c_ptr1, #0x20]\n"
- "ldr q23, [c_ptr1, #0x30]\n"
- "ldr q24, [c_ptr2]\n"
- "ldr q25, [c_ptr2, #0x10]\n"
- "ldr q26, [c_ptr2, #0x20]\n"
- "ldr q27, [c_ptr2, #0x30]\n"
- "ldr q0, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ldr q1, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "ldr q2, [a_ptr2]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "ldr q8, [%[b_ptr0]]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "cbz %[loops], 2f\n"
- "3:\n"
- ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
- "ldr q15, [%[b_ptr0], #-0x10]\n"
- ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
- "ldr q4, [%[a_ptr0]]\n"
- ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n"
- "ldr q5, [a_ptr1]\n"
- ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
- "ldr q6, [a_ptr2]\n"
- ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
- "ldr q8, [%[b_ptr0]]\n"
- ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
- "subs %[loops], %[loops], #0x1\n"
- ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
- "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n"
- ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
- ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
- "add a_ptr1, a_ptr1, #0x20\n"
- ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n"
- "add a_ptr2, a_ptr2, #0x20\n"
- ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n"
- "prfm PLDL1KEEP, [a_ptr1, #0x40]\n"
- ".inst 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n"
- "prfm PLDL1KEEP, [a_ptr2, #0x40]\n"
- ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n"
- ".inst 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n"
- ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n"
- ".inst 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n"
- ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n"
- ".inst 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n"
- "ldr q15, [%[b_ptr0], #0x70]\n"
- ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n"
- ".inst 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n"
- "ldr q8, [%[b_ptr0], #-0x80]\n"
- ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n"
- ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n"
- ".inst 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n"
- "ldr q9, [%[b_ptr0], #-0x70]\n"
- ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n"
- ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n"
- ".inst 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n"
- "ldr q10, [%[b_ptr0], #-0x60]\n"
- ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n"
- ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n"
- ".inst 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n"
- "ldr q11, [%[b_ptr0], #-0x50]\n"
- ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n"
- ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n"
- ".inst 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n"
- "ldr q12, [%[b_ptr0], #-0x40]\n"
- ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n"
- ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n"
- ".inst 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n"
- "ldr q13, [%[b_ptr0], #-0x30]\n"
- ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n"
- ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n"
- ".inst 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n"
- "ldr q14, [%[b_ptr0], #-0x20]\n"
- ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n"
- "ldr q0, [%[a_ptr0], #-0x10]\n"
- ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n"
- "ldr q1, [a_ptr1, #-0x10]\n"
- ".inst 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n"
- "ldr q15, [%[b_ptr0], #-0x10]\n"
- ".inst 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n"
- "ldr q2, [a_ptr2, #-0x10]\n"
- ".inst 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n"
- ".inst 0x6f86e118 // udot v24.4s, v8.16b, v6.4b[0]\n"
- "ldr q8, [%[b_ptr0]]\n"
- ".inst 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n"
- ".inst 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n"
- ".inst 0x6f86e139 // udot v25.4s, v9.16b, v6.4b[0]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- ".inst 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n"
- ".inst 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n"
- ".inst 0x6f86e15a // udot v26.4s, v10.16b, v6.4b[0]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- ".inst 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n"
- ".inst 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n"
- ".inst 0x6f86e17b // udot v27.4s, v11.16b, v6.4b[0]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- ".inst 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n"
- ".inst 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n"
- ".inst 0x6fa6e198 // udot v24.4s, v12.16b, v6.4b[1]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- ".inst 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n"
- ".inst 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n"
- ".inst 0x6fa6e1b9 // udot v25.4s, v13.16b, v6.4b[1]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- ".inst 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n"
- ".inst 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n"
- ".inst 0x6fa6e1da // udot v26.4s, v14.16b, v6.4b[1]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- ".inst 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n"
- ".inst 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n"
- ".inst 0x6fa6e1fb // udot v27.4s, v15.16b, v6.4b[1]\n"
- "ldr q15, [%[b_ptr0], #0x70]\n"
- ".inst 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".inst 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n"
- ".inst 0x6f86e918 // udot v24.4s, v8.16b, v6.4b[2]\n"
- "ldr q8, [%[b_ptr0], #-0x80]\n"
- ".inst 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n"
- ".inst 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n"
- ".inst 0x6f86e939 // udot v25.4s, v9.16b, v6.4b[2]\n"
- "ldr q9, [%[b_ptr0], #-0x70]\n"
- ".inst 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n"
- ".inst 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n"
- ".inst 0x6f86e95a // udot v26.4s, v10.16b, v6.4b[2]\n"
- "ldr q10, [%[b_ptr0], #-0x60]\n"
- ".inst 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n"
- ".inst 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n"
- ".inst 0x6f86e97b // udot v27.4s, v11.16b, v6.4b[2]\n"
- "ldr q11, [%[b_ptr0], #-0x50]\n"
- ".inst 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n"
- ".inst 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n"
- ".inst 0x6fa6e998 // udot v24.4s, v12.16b, v6.4b[3]\n"
- "ldr q12, [%[b_ptr0], #-0x40]\n"
- ".inst 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n"
- ".inst 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n"
- ".inst 0x6fa6e9b9 // udot v25.4s, v13.16b, v6.4b[3]\n"
- "ldr q13, [%[b_ptr0], #-0x30]\n"
- ".inst 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n"
- ".inst 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n"
- ".inst 0x6fa6e9da // udot v26.4s, v14.16b, v6.4b[3]\n"
- "ldr q14, [%[b_ptr0], #-0x20]\n"
- ".inst 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n"
- ".inst 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n"
- ".inst 0x6fa6e9fb // udot v27.4s, v15.16b, v6.4b[3]\n"
- "b.ne 3b\n"
- "2:\n"
- "ldr q15, [%[b_ptr0], #-0x10]\n"
- "prfm PSTL1KEEP, [%[c_ptr0]]\n"
- "prfm PSTL1KEEP, [c_ptr1]\n"
- "prfm PSTL1KEEP, [c_ptr2]\n"
- "cbz %[regs], 4f\n"
- ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
- "ldr q4, [%[a_ptr0]]\n"
- ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
- "ldr q5, [a_ptr1]\n"
- ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n"
- "ldr q6, [a_ptr2]\n"
- ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
- "ldr q8, [%[b_ptr0]]\n"
- ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
- ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
- ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n"
- ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n"
- ".inst 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n"
- ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n"
- ".inst 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n"
- ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n"
- ".inst 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n"
- ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n"
- ".inst 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n"
- "ldr q15, [%[b_ptr0], #0x70]\n"
- ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n"
- ".inst 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n"
- "ldr q8, [%[b_ptr0], #-0x80]\n"
- ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n"
- ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n"
- ".inst 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n"
- "ldr q9, [%[b_ptr0], #-0x70]\n"
- ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n"
- ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n"
- ".inst 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n"
- "ldr q10, [%[b_ptr0], #-0x60]\n"
- ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n"
- ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n"
- ".inst 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n"
- "ldr q11, [%[b_ptr0], #-0x50]\n"
- ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n"
- ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n"
- ".inst 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n"
- "ldr q12, [%[b_ptr0], #-0x40]\n"
- ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n"
- ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n"
- ".inst 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n"
- "ldr q13, [%[b_ptr0], #-0x30]\n"
- ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n"
- ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n"
- ".inst 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n"
- "ldr q14, [%[b_ptr0], #-0x20]\n"
- ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n"
- ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n"
- ".inst 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n"
- "ldr q15, [%[b_ptr0], #-0x10]\n"
- ".inst 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n"
- ".inst 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n"
- ".inst 0x6f86e118 // udot v24.4s, v8.16b, v6.4b[0]\n"
- "ldr q8, [%[b_ptr0]]\n"
- ".inst 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n"
- ".inst 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n"
- ".inst 0x6f86e139 // udot v25.4s, v9.16b, v6.4b[0]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- ".inst 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n"
- ".inst 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n"
- ".inst 0x6f86e15a // udot v26.4s, v10.16b, v6.4b[0]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- ".inst 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n"
- ".inst 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n"
- ".inst 0x6f86e17b // udot v27.4s, v11.16b, v6.4b[0]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- ".inst 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n"
- ".inst 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n"
- ".inst 0x6fa6e198 // udot v24.4s, v12.16b, v6.4b[1]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- ".inst 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n"
- ".inst 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n"
- ".inst 0x6fa6e1b9 // udot v25.4s, v13.16b, v6.4b[1]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- ".inst 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n"
- ".inst 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n"
- ".inst 0x6fa6e1da // udot v26.4s, v14.16b, v6.4b[1]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- ".inst 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n"
- ".inst 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n"
- ".inst 0x6fa6e1fb // udot v27.4s, v15.16b, v6.4b[1]\n"
- "ldr q15, [%[b_ptr0], #0x70]\n"
- ".inst 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- ".inst 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n"
- ".inst 0x6f86e918 // udot v24.4s, v8.16b, v6.4b[2]\n"
- ".inst 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n"
- ".inst 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n"
- ".inst 0x6f86e939 // udot v25.4s, v9.16b, v6.4b[2]\n"
- ".inst 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n"
- ".inst 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n"
- ".inst 0x6f86e95a // udot v26.4s, v10.16b, v6.4b[2]\n"
- ".inst 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n"
- ".inst 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n"
- ".inst 0x6f86e97b // udot v27.4s, v11.16b, v6.4b[2]\n"
- ".inst 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n"
- ".inst 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n"
- ".inst 0x6fa6e998 // udot v24.4s, v12.16b, v6.4b[3]\n"
- ".inst 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n"
- ".inst 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n"
- ".inst 0x6fa6e9b9 // udot v25.4s, v13.16b, v6.4b[3]\n"
- ".inst 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n"
- ".inst 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n"
- ".inst 0x6fa6e9da // udot v26.4s, v14.16b, v6.4b[3]\n"
- ".inst 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n"
- ".inst 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n"
- ".inst 0x6fa6e9fb // udot v27.4s, v15.16b, v6.4b[3]\n"
- "b 5f\n"
- "4:\n"
- ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
- ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
- ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n"
- "ldr q8, [%[b_ptr0]]\n"
- ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
- ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
- ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
- ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
- ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
- ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
- ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n"
- ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n"
- ".inst 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n"
- ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n"
- ".inst 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n"
- ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n"
- ".inst 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n"
- ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n"
- ".inst 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n"
- "ldr q15, [%[b_ptr0], #0x70]\n"
- ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n"
- ".inst 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n"
- ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n"
- ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n"
- ".inst 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n"
- ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n"
- ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n"
- ".inst 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n"
- ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n"
- ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n"
- ".inst 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n"
- ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n"
- ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n"
- ".inst 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n"
- ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n"
- ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n"
- ".inst 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n"
- ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n"
- ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n"
- ".inst 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n"
- ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n"
- ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n"
- ".inst 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n"
- "5:\n"
- "cbz %[blocks], 6f\n"
- "7:\n"
- "ldr q8, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "ldr s0, [%[a_ptr0]]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "add %[a_ptr0], %[a_ptr0], #0x4\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "add %[b_ptr0], %[b_ptr0], #0x40\n"
- ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
- "ldr s1, [a_ptr1]\n"
- ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
- "add a_ptr1, a_ptr1, #0x4\n"
- ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
- "ldr s2, [a_ptr2]\n"
- ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
- "add a_ptr2, a_ptr2, #0x4\n"
- ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
- ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n"
- ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n"
- ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
- ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n"
- ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
- ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
- ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n"
- "b.ne 7b\n"
- "6:\n"
- "cbz %[odds], 8f\n"
- "ld1 {v0.b}[0], [%[a_ptr0]], #1\n"
- "ld1 {v1.b}[0], [a_ptr1], #1\n"
- "ld1 {v2.b}[0], [a_ptr2], #1\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 9f\n"
- "ld1 {v0.b}[1], [%[a_ptr0]], #1\n"
- "ld1 {v1.b}[1], [a_ptr1], #1\n"
- "ld1 {v2.b}[1], [a_ptr2], #1\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 9f\n"
- "ld1 {v0.b}[2], [%[a_ptr0]]\n"
- "ld1 {v1.b}[2], [a_ptr1]\n"
- "ld1 {v2.b}[2], [a_ptr2]\n"
- "9:\n"
- "ldr q8, [%[b_ptr0]]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
- ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
- ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n"
- ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
- ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
- ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n"
- ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
- ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
- ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n"
- ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
- ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
- ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n"
- "8:\n"
- "str q16, [%[c_ptr0]]\n"
- "str q17, [%[c_ptr0], #0x10]\n"
- "str q18, [%[c_ptr0], #0x20]\n"
- "str q19, [%[c_ptr0], #0x30]\n"
- "add %[c_ptr0], %[c_ptr0], #0x40\n"
- "str q20, [c_ptr1]\n"
- "str q21, [c_ptr1, #0x10]\n"
- "str q22, [c_ptr1, #0x20]\n"
- "str q23, [c_ptr1, #0x30]\n"
- "str q24, [c_ptr2]\n"
- "str q25, [c_ptr2, #0x10]\n"
- "str q26, [c_ptr2, #0x20]\n"
- "str q27, [c_ptr2, #0x30]\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks), [odds] "+r" (odds)
- : [width] "r" (width), [accumulate] "r" (static_cast<uint64_t>(accumulate)), [lda] "r" (ldab), [ldc] "r" (ldcb)
- : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "cc", "memory"
- );
- break;
- default:
- case 4:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "a_ptr3 .req X2\n"
- "c_ptr1 .req X3\n"
- "c_ptr2 .req X4\n"
- "c_ptr3 .req X5\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "add a_ptr3, a_ptr2, %[lda]\n"
- "add c_ptr3, c_ptr2, %[ldc]\n"
- "cbnz %[accumulate], 1f\n"
- "movi v16.4s, #0\n"
- "ldr q0, [%[a_ptr0]]\n"
- "movi v17.4s, #0\n"
- "ldr q1, [a_ptr1]\n"
- "movi v18.4s, #0\n"
- "ldr q2, [a_ptr2]\n"
- "movi v19.4s, #0\n"
- "ldr q3, [a_ptr3]\n"
- "movi v20.4s, #0\n"
- "ldr q8, [%[b_ptr0]]\n"
- "movi v21.4s, #0\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "movi v22.4s, #0\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "movi v23.4s, #0\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "movi v24.4s, #0\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- "movi v25.4s, #0\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- "movi v26.4s, #0\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- "movi v27.4s, #0\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "movi v28.4s, #0\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "movi v29.4s, #0\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "movi v30.4s, #0\n"
- "add a_ptr3, a_ptr3, #0x10\n"
- "movi v31.4s, #0\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "ldr q16, [%[c_ptr0]]\n"
- "ldr q17, [%[c_ptr0], #0x10]\n"
- "ldr q18, [%[c_ptr0], #0x20]\n"
- "ldr q19, [%[c_ptr0], #0x30]\n"
- "ldr q20, [c_ptr1]\n"
- "ldr q21, [c_ptr1, #0x10]\n"
- "ldr q22, [c_ptr1, #0x20]\n"
- "ldr q23, [c_ptr1, #0x30]\n"
- "ldr q24, [c_ptr2]\n"
- "ldr q25, [c_ptr2, #0x10]\n"
- "ldr q26, [c_ptr2, #0x20]\n"
- "ldr q27, [c_ptr2, #0x30]\n"
- "ldr q28, [c_ptr3]\n"
- "ldr q29, [c_ptr3, #0x10]\n"
- "ldr q30, [c_ptr3, #0x20]\n"
- "ldr q31, [c_ptr3, #0x30]\n"
- "ldr q0, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ldr q1, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "ldr q2, [a_ptr2]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "ldr q3, [a_ptr3]\n"
- "add a_ptr3, a_ptr3, #0x10\n"
- "ldr q8, [%[b_ptr0]]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "cbz %[loops], 2f\n"
- "3:\n"
- ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
- "ldr q15, [%[b_ptr0], #-0x10]\n"
- ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
- "ldr q4, [%[a_ptr0]]\n"
- ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n"
- "ldr q5, [a_ptr1]\n"
- ".inst 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n"
- "ldr q6, [a_ptr2]\n"
- ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
- "ldr q7, [a_ptr3]\n"
- ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
- "ldr q8, [%[b_ptr0]]\n"
- ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n"
- "subs %[loops], %[loops], #0x1\n"
- ".inst 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
- "prfm PLDL1KEEP, [%[a_ptr0], #0x40]\n"
- ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
- ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n"
- "add a_ptr1, a_ptr1, #0x20\n"
- ".inst 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
- "add a_ptr2, a_ptr2, #0x20\n"
- ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
- "add a_ptr3, a_ptr3, #0x20\n"
- ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n"
- "prfm PLDL1KEEP, [a_ptr1, #0x40]\n"
- ".inst 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n"
- "prfm PLDL1KEEP, [a_ptr2, #0x40]\n"
- ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n"
- "prfm PLDL1KEEP, [a_ptr3, #0x40]\n"
- ".inst 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n"
- ".inst 0x6fa3e19c // udot v28.4s, v12.16b, v3.4b[1]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n"
- ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n"
- ".inst 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n"
- ".inst 0x6fa3e1bd // udot v29.4s, v13.16b, v3.4b[1]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n"
- ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n"
- ".inst 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n"
- ".inst 0x6fa3e1de // udot v30.4s, v14.16b, v3.4b[1]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n"
- ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n"
- ".inst 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n"
- ".inst 0x6fa3e1ff // udot v31.4s, v15.16b, v3.4b[1]\n"
- "ldr q15, [%[b_ptr0], #0x70]\n"
- ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n"
- ".inst 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n"
- ".inst 0x6f83e91c // udot v28.4s, v8.16b, v3.4b[2]\n"
- "ldr q8, [%[b_ptr0], #-0x80]\n"
- ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n"
- ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n"
- ".inst 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n"
- ".inst 0x6f83e93d // udot v29.4s, v9.16b, v3.4b[2]\n"
- "ldr q9, [%[b_ptr0], #-0x70]\n"
- ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n"
- ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n"
- ".inst 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n"
- ".inst 0x6f83e95e // udot v30.4s, v10.16b, v3.4b[2]\n"
- "ldr q10, [%[b_ptr0], #-0x60]\n"
- ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n"
- ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n"
- ".inst 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n"
- ".inst 0x6f83e97f // udot v31.4s, v11.16b, v3.4b[2]\n"
- "ldr q11, [%[b_ptr0], #-0x50]\n"
- ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n"
- ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n"
- ".inst 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n"
- ".inst 0x6fa3e99c // udot v28.4s, v12.16b, v3.4b[3]\n"
- "ldr q12, [%[b_ptr0], #-0x40]\n"
- ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n"
- ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n"
- ".inst 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n"
- ".inst 0x6fa3e9bd // udot v29.4s, v13.16b, v3.4b[3]\n"
- "ldr q13, [%[b_ptr0], #-0x30]\n"
- ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n"
- ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n"
- ".inst 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n"
- ".inst 0x6fa3e9de // udot v30.4s, v14.16b, v3.4b[3]\n"
- "ldr q14, [%[b_ptr0], #-0x20]\n"
- ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n"
- "ldr q0, [%[a_ptr0], #-0x10]\n"
- ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n"
- "ldr q1, [a_ptr1, #-0x10]\n"
- ".inst 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n"
- "ldr q2, [a_ptr2, #-0x10]\n"
- ".inst 0x6fa3e9ff // udot v31.4s, v15.16b, v3.4b[3]\n"
- "ldr q15, [%[b_ptr0], #-0x10]\n"
- ".inst 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n"
- "ldr q3, [a_ptr3, #-0x10]\n"
- ".inst 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n"
- ".inst 0x6f86e118 // udot v24.4s, v8.16b, v6.4b[0]\n"
- ".inst 0x6f87e11c // udot v28.4s, v8.16b, v7.4b[0]\n"
- "ldr q8, [%[b_ptr0]]\n"
- ".inst 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n"
- ".inst 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n"
- ".inst 0x6f86e139 // udot v25.4s, v9.16b, v6.4b[0]\n"
- ".inst 0x6f87e13d // udot v29.4s, v9.16b, v7.4b[0]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- ".inst 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n"
- ".inst 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n"
- ".inst 0x6f86e15a // udot v26.4s, v10.16b, v6.4b[0]\n"
- ".inst 0x6f87e15e // udot v30.4s, v10.16b, v7.4b[0]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- ".inst 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n"
- ".inst 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n"
- ".inst 0x6f86e17b // udot v27.4s, v11.16b, v6.4b[0]\n"
- ".inst 0x6f87e17f // udot v31.4s, v11.16b, v7.4b[0]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- ".inst 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n"
- ".inst 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n"
- ".inst 0x6fa6e198 // udot v24.4s, v12.16b, v6.4b[1]\n"
- ".inst 0x6fa7e19c // udot v28.4s, v12.16b, v7.4b[1]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- ".inst 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n"
- ".inst 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n"
- ".inst 0x6fa6e1b9 // udot v25.4s, v13.16b, v6.4b[1]\n"
- ".inst 0x6fa7e1bd // udot v29.4s, v13.16b, v7.4b[1]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- ".inst 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n"
- ".inst 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n"
- ".inst 0x6fa6e1da // udot v26.4s, v14.16b, v6.4b[1]\n"
- ".inst 0x6fa7e1de // udot v30.4s, v14.16b, v7.4b[1]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- ".inst 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n"
- ".inst 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n"
- ".inst 0x6fa6e1fb // udot v27.4s, v15.16b, v6.4b[1]\n"
- ".inst 0x6fa7e1ff // udot v31.4s, v15.16b, v7.4b[1]\n"
- "ldr q15, [%[b_ptr0], #0x70]\n"
- ".inst 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".inst 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n"
- ".inst 0x6f86e918 // udot v24.4s, v8.16b, v6.4b[2]\n"
- ".inst 0x6f87e91c // udot v28.4s, v8.16b, v7.4b[2]\n"
- "ldr q8, [%[b_ptr0], #-0x80]\n"
- ".inst 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n"
- ".inst 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n"
- ".inst 0x6f86e939 // udot v25.4s, v9.16b, v6.4b[2]\n"
- ".inst 0x6f87e93d // udot v29.4s, v9.16b, v7.4b[2]\n"
- "ldr q9, [%[b_ptr0], #-0x70]\n"
- ".inst 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n"
- ".inst 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n"
- ".inst 0x6f86e95a // udot v26.4s, v10.16b, v6.4b[2]\n"
- ".inst 0x6f87e95e // udot v30.4s, v10.16b, v7.4b[2]\n"
- "ldr q10, [%[b_ptr0], #-0x60]\n"
- ".inst 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n"
- ".inst 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n"
- ".inst 0x6f86e97b // udot v27.4s, v11.16b, v6.4b[2]\n"
- ".inst 0x6f87e97f // udot v31.4s, v11.16b, v7.4b[2]\n"
- "ldr q11, [%[b_ptr0], #-0x50]\n"
- ".inst 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n"
- ".inst 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n"
- ".inst 0x6fa6e998 // udot v24.4s, v12.16b, v6.4b[3]\n"
- ".inst 0x6fa7e99c // udot v28.4s, v12.16b, v7.4b[3]\n"
- "ldr q12, [%[b_ptr0], #-0x40]\n"
- ".inst 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n"
- ".inst 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n"
- ".inst 0x6fa6e9b9 // udot v25.4s, v13.16b, v6.4b[3]\n"
- ".inst 0x6fa7e9bd // udot v29.4s, v13.16b, v7.4b[3]\n"
- "ldr q13, [%[b_ptr0], #-0x30]\n"
- ".inst 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n"
- ".inst 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n"
- ".inst 0x6fa6e9da // udot v26.4s, v14.16b, v6.4b[3]\n"
- ".inst 0x6fa7e9de // udot v30.4s, v14.16b, v7.4b[3]\n"
- "ldr q14, [%[b_ptr0], #-0x20]\n"
- ".inst 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n"
- ".inst 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n"
- ".inst 0x6fa6e9fb // udot v27.4s, v15.16b, v6.4b[3]\n"
- ".inst 0x6fa7e9ff // udot v31.4s, v15.16b, v7.4b[3]\n"
- "b.ne 3b\n"
- "2:\n"
- "ldr q15, [%[b_ptr0], #-0x10]\n"
- "prfm PSTL1KEEP, [%[c_ptr0]]\n"
- "prfm PSTL1KEEP, [c_ptr1]\n"
- "prfm PSTL1KEEP, [c_ptr2]\n"
- "prfm PSTL1KEEP, [c_ptr3]\n"
- "cbz %[regs], 4f\n"
- ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
- "ldr q4, [%[a_ptr0]]\n"
- ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
- "ldr q5, [a_ptr1]\n"
- ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n"
- "ldr q6, [a_ptr2]\n"
- ".inst 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n"
- "ldr q7, [a_ptr3]\n"
- ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
- "ldr q8, [%[b_ptr0]]\n"
- ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- ".inst 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
- "add a_ptr3, a_ptr3, #0x10\n"
- ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n"
- ".inst 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
- ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
- ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n"
- ".inst 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n"
- ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n"
- ".inst 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n"
- ".inst 0x6fa3e19c // udot v28.4s, v12.16b, v3.4b[1]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n"
- ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n"
- ".inst 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n"
- ".inst 0x6fa3e1bd // udot v29.4s, v13.16b, v3.4b[1]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n"
- ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n"
- ".inst 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n"
- ".inst 0x6fa3e1de // udot v30.4s, v14.16b, v3.4b[1]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n"
- ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n"
- ".inst 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n"
- ".inst 0x6fa3e1ff // udot v31.4s, v15.16b, v3.4b[1]\n"
- "ldr q15, [%[b_ptr0], #0x70]\n"
- ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x100\n"
- ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n"
- ".inst 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n"
- ".inst 0x6f83e91c // udot v28.4s, v8.16b, v3.4b[2]\n"
- "ldr q8, [%[b_ptr0], #-0x80]\n"
- ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n"
- ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n"
- ".inst 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n"
- ".inst 0x6f83e93d // udot v29.4s, v9.16b, v3.4b[2]\n"
- "ldr q9, [%[b_ptr0], #-0x70]\n"
- ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n"
- ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n"
- ".inst 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n"
- ".inst 0x6f83e95e // udot v30.4s, v10.16b, v3.4b[2]\n"
- "ldr q10, [%[b_ptr0], #-0x60]\n"
- ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n"
- ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n"
- ".inst 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n"
- ".inst 0x6f83e97f // udot v31.4s, v11.16b, v3.4b[2]\n"
- "ldr q11, [%[b_ptr0], #-0x50]\n"
- ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n"
- ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n"
- ".inst 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n"
- ".inst 0x6fa3e99c // udot v28.4s, v12.16b, v3.4b[3]\n"
- "ldr q12, [%[b_ptr0], #-0x40]\n"
- ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n"
- ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n"
- ".inst 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n"
- ".inst 0x6fa3e9bd // udot v29.4s, v13.16b, v3.4b[3]\n"
- "ldr q13, [%[b_ptr0], #-0x30]\n"
- ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n"
- ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n"
- ".inst 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n"
- ".inst 0x6fa3e9de // udot v30.4s, v14.16b, v3.4b[3]\n"
- "ldr q14, [%[b_ptr0], #-0x20]\n"
- ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n"
- ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n"
- ".inst 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n"
- ".inst 0x6fa3e9ff // udot v31.4s, v15.16b, v3.4b[3]\n"
- "ldr q15, [%[b_ptr0], #-0x10]\n"
- ".inst 0x6f84e110 // udot v16.4s, v8.16b, v4.4b[0]\n"
- ".inst 0x6f85e114 // udot v20.4s, v8.16b, v5.4b[0]\n"
- ".inst 0x6f86e118 // udot v24.4s, v8.16b, v6.4b[0]\n"
- ".inst 0x6f87e11c // udot v28.4s, v8.16b, v7.4b[0]\n"
- "ldr q8, [%[b_ptr0]]\n"
- ".inst 0x6f84e131 // udot v17.4s, v9.16b, v4.4b[0]\n"
- ".inst 0x6f85e135 // udot v21.4s, v9.16b, v5.4b[0]\n"
- ".inst 0x6f86e139 // udot v25.4s, v9.16b, v6.4b[0]\n"
- ".inst 0x6f87e13d // udot v29.4s, v9.16b, v7.4b[0]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- ".inst 0x6f84e152 // udot v18.4s, v10.16b, v4.4b[0]\n"
- ".inst 0x6f85e156 // udot v22.4s, v10.16b, v5.4b[0]\n"
- ".inst 0x6f86e15a // udot v26.4s, v10.16b, v6.4b[0]\n"
- ".inst 0x6f87e15e // udot v30.4s, v10.16b, v7.4b[0]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- ".inst 0x6f84e173 // udot v19.4s, v11.16b, v4.4b[0]\n"
- ".inst 0x6f85e177 // udot v23.4s, v11.16b, v5.4b[0]\n"
- ".inst 0x6f86e17b // udot v27.4s, v11.16b, v6.4b[0]\n"
- ".inst 0x6f87e17f // udot v31.4s, v11.16b, v7.4b[0]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- ".inst 0x6fa4e190 // udot v16.4s, v12.16b, v4.4b[1]\n"
- ".inst 0x6fa5e194 // udot v20.4s, v12.16b, v5.4b[1]\n"
- ".inst 0x6fa6e198 // udot v24.4s, v12.16b, v6.4b[1]\n"
- ".inst 0x6fa7e19c // udot v28.4s, v12.16b, v7.4b[1]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- ".inst 0x6fa4e1b1 // udot v17.4s, v13.16b, v4.4b[1]\n"
- ".inst 0x6fa5e1b5 // udot v21.4s, v13.16b, v5.4b[1]\n"
- ".inst 0x6fa6e1b9 // udot v25.4s, v13.16b, v6.4b[1]\n"
- ".inst 0x6fa7e1bd // udot v29.4s, v13.16b, v7.4b[1]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- ".inst 0x6fa4e1d2 // udot v18.4s, v14.16b, v4.4b[1]\n"
- ".inst 0x6fa5e1d6 // udot v22.4s, v14.16b, v5.4b[1]\n"
- ".inst 0x6fa6e1da // udot v26.4s, v14.16b, v6.4b[1]\n"
- ".inst 0x6fa7e1de // udot v30.4s, v14.16b, v7.4b[1]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- ".inst 0x6fa4e1f3 // udot v19.4s, v15.16b, v4.4b[1]\n"
- ".inst 0x6fa5e1f7 // udot v23.4s, v15.16b, v5.4b[1]\n"
- ".inst 0x6fa6e1fb // udot v27.4s, v15.16b, v6.4b[1]\n"
- ".inst 0x6fa7e1ff // udot v31.4s, v15.16b, v7.4b[1]\n"
- "ldr q15, [%[b_ptr0], #0x70]\n"
- ".inst 0x6f84e910 // udot v16.4s, v8.16b, v4.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- ".inst 0x6f85e914 // udot v20.4s, v8.16b, v5.4b[2]\n"
- ".inst 0x6f86e918 // udot v24.4s, v8.16b, v6.4b[2]\n"
- ".inst 0x6f87e91c // udot v28.4s, v8.16b, v7.4b[2]\n"
- ".inst 0x6f84e931 // udot v17.4s, v9.16b, v4.4b[2]\n"
- ".inst 0x6f85e935 // udot v21.4s, v9.16b, v5.4b[2]\n"
- ".inst 0x6f86e939 // udot v25.4s, v9.16b, v6.4b[2]\n"
- ".inst 0x6f87e93d // udot v29.4s, v9.16b, v7.4b[2]\n"
- ".inst 0x6f84e952 // udot v18.4s, v10.16b, v4.4b[2]\n"
- ".inst 0x6f85e956 // udot v22.4s, v10.16b, v5.4b[2]\n"
- ".inst 0x6f86e95a // udot v26.4s, v10.16b, v6.4b[2]\n"
- ".inst 0x6f87e95e // udot v30.4s, v10.16b, v7.4b[2]\n"
- ".inst 0x6f84e973 // udot v19.4s, v11.16b, v4.4b[2]\n"
- ".inst 0x6f85e977 // udot v23.4s, v11.16b, v5.4b[2]\n"
- ".inst 0x6f86e97b // udot v27.4s, v11.16b, v6.4b[2]\n"
- ".inst 0x6f87e97f // udot v31.4s, v11.16b, v7.4b[2]\n"
- ".inst 0x6fa4e990 // udot v16.4s, v12.16b, v4.4b[3]\n"
- ".inst 0x6fa5e994 // udot v20.4s, v12.16b, v5.4b[3]\n"
- ".inst 0x6fa6e998 // udot v24.4s, v12.16b, v6.4b[3]\n"
- ".inst 0x6fa7e99c // udot v28.4s, v12.16b, v7.4b[3]\n"
- ".inst 0x6fa4e9b1 // udot v17.4s, v13.16b, v4.4b[3]\n"
- ".inst 0x6fa5e9b5 // udot v21.4s, v13.16b, v5.4b[3]\n"
- ".inst 0x6fa6e9b9 // udot v25.4s, v13.16b, v6.4b[3]\n"
- ".inst 0x6fa7e9bd // udot v29.4s, v13.16b, v7.4b[3]\n"
- ".inst 0x6fa4e9d2 // udot v18.4s, v14.16b, v4.4b[3]\n"
- ".inst 0x6fa5e9d6 // udot v22.4s, v14.16b, v5.4b[3]\n"
- ".inst 0x6fa6e9da // udot v26.4s, v14.16b, v6.4b[3]\n"
- ".inst 0x6fa7e9de // udot v30.4s, v14.16b, v7.4b[3]\n"
- ".inst 0x6fa4e9f3 // udot v19.4s, v15.16b, v4.4b[3]\n"
- ".inst 0x6fa5e9f7 // udot v23.4s, v15.16b, v5.4b[3]\n"
- ".inst 0x6fa6e9fb // udot v27.4s, v15.16b, v6.4b[3]\n"
- ".inst 0x6fa7e9ff // udot v31.4s, v15.16b, v7.4b[3]\n"
- "b 5f\n"
- "4:\n"
- ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
- ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
- ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n"
- ".inst 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n"
- "ldr q8, [%[b_ptr0]]\n"
- ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
- ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
- ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n"
- ".inst 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
- ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
- ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n"
- ".inst 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
- ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
- ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n"
- ".inst 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- ".inst 0x6fa0e190 // udot v16.4s, v12.16b, v0.4b[1]\n"
- ".inst 0x6fa1e194 // udot v20.4s, v12.16b, v1.4b[1]\n"
- ".inst 0x6fa2e198 // udot v24.4s, v12.16b, v2.4b[1]\n"
- ".inst 0x6fa3e19c // udot v28.4s, v12.16b, v3.4b[1]\n"
- "ldr q12, [%[b_ptr0], #0x40]\n"
- ".inst 0x6fa0e1b1 // udot v17.4s, v13.16b, v0.4b[1]\n"
- ".inst 0x6fa1e1b5 // udot v21.4s, v13.16b, v1.4b[1]\n"
- ".inst 0x6fa2e1b9 // udot v25.4s, v13.16b, v2.4b[1]\n"
- ".inst 0x6fa3e1bd // udot v29.4s, v13.16b, v3.4b[1]\n"
- "ldr q13, [%[b_ptr0], #0x50]\n"
- ".inst 0x6fa0e1d2 // udot v18.4s, v14.16b, v0.4b[1]\n"
- ".inst 0x6fa1e1d6 // udot v22.4s, v14.16b, v1.4b[1]\n"
- ".inst 0x6fa2e1da // udot v26.4s, v14.16b, v2.4b[1]\n"
- ".inst 0x6fa3e1de // udot v30.4s, v14.16b, v3.4b[1]\n"
- "ldr q14, [%[b_ptr0], #0x60]\n"
- ".inst 0x6fa0e1f3 // udot v19.4s, v15.16b, v0.4b[1]\n"
- ".inst 0x6fa1e1f7 // udot v23.4s, v15.16b, v1.4b[1]\n"
- ".inst 0x6fa2e1fb // udot v27.4s, v15.16b, v2.4b[1]\n"
- ".inst 0x6fa3e1ff // udot v31.4s, v15.16b, v3.4b[1]\n"
- "ldr q15, [%[b_ptr0], #0x70]\n"
- ".inst 0x6f80e910 // udot v16.4s, v8.16b, v0.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- ".inst 0x6f81e914 // udot v20.4s, v8.16b, v1.4b[2]\n"
- ".inst 0x6f82e918 // udot v24.4s, v8.16b, v2.4b[2]\n"
- ".inst 0x6f83e91c // udot v28.4s, v8.16b, v3.4b[2]\n"
- ".inst 0x6f80e931 // udot v17.4s, v9.16b, v0.4b[2]\n"
- ".inst 0x6f81e935 // udot v21.4s, v9.16b, v1.4b[2]\n"
- ".inst 0x6f82e939 // udot v25.4s, v9.16b, v2.4b[2]\n"
- ".inst 0x6f83e93d // udot v29.4s, v9.16b, v3.4b[2]\n"
- ".inst 0x6f80e952 // udot v18.4s, v10.16b, v0.4b[2]\n"
- ".inst 0x6f81e956 // udot v22.4s, v10.16b, v1.4b[2]\n"
- ".inst 0x6f82e95a // udot v26.4s, v10.16b, v2.4b[2]\n"
- ".inst 0x6f83e95e // udot v30.4s, v10.16b, v3.4b[2]\n"
- ".inst 0x6f80e973 // udot v19.4s, v11.16b, v0.4b[2]\n"
- ".inst 0x6f81e977 // udot v23.4s, v11.16b, v1.4b[2]\n"
- ".inst 0x6f82e97b // udot v27.4s, v11.16b, v2.4b[2]\n"
- ".inst 0x6f83e97f // udot v31.4s, v11.16b, v3.4b[2]\n"
- ".inst 0x6fa0e990 // udot v16.4s, v12.16b, v0.4b[3]\n"
- ".inst 0x6fa1e994 // udot v20.4s, v12.16b, v1.4b[3]\n"
- ".inst 0x6fa2e998 // udot v24.4s, v12.16b, v2.4b[3]\n"
- ".inst 0x6fa3e99c // udot v28.4s, v12.16b, v3.4b[3]\n"
- ".inst 0x6fa0e9b1 // udot v17.4s, v13.16b, v0.4b[3]\n"
- ".inst 0x6fa1e9b5 // udot v21.4s, v13.16b, v1.4b[3]\n"
- ".inst 0x6fa2e9b9 // udot v25.4s, v13.16b, v2.4b[3]\n"
- ".inst 0x6fa3e9bd // udot v29.4s, v13.16b, v3.4b[3]\n"
- ".inst 0x6fa0e9d2 // udot v18.4s, v14.16b, v0.4b[3]\n"
- ".inst 0x6fa1e9d6 // udot v22.4s, v14.16b, v1.4b[3]\n"
- ".inst 0x6fa2e9da // udot v26.4s, v14.16b, v2.4b[3]\n"
- ".inst 0x6fa3e9de // udot v30.4s, v14.16b, v3.4b[3]\n"
- ".inst 0x6fa0e9f3 // udot v19.4s, v15.16b, v0.4b[3]\n"
- ".inst 0x6fa1e9f7 // udot v23.4s, v15.16b, v1.4b[3]\n"
- ".inst 0x6fa2e9fb // udot v27.4s, v15.16b, v2.4b[3]\n"
- ".inst 0x6fa3e9ff // udot v31.4s, v15.16b, v3.4b[3]\n"
- "5:\n"
- "cbz %[blocks], 6f\n"
- "7:\n"
- "ldr q8, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "ldr s0, [%[a_ptr0]]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "add %[a_ptr0], %[a_ptr0], #0x4\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- "add %[b_ptr0], %[b_ptr0], #0x40\n"
- ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
- "ldr s1, [a_ptr1]\n"
- ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
- "add a_ptr1, a_ptr1, #0x4\n"
- ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
- "ldr s2, [a_ptr2]\n"
- ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
- "add a_ptr2, a_ptr2, #0x4\n"
- ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
- "ldr s3, [a_ptr3]\n"
- ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n"
- "add a_ptr3, a_ptr3, #0x4\n"
- ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n"
- ".inst 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n"
- ".inst 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n"
- ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
- ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n"
- ".inst 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n"
- ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
- ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
- ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n"
- ".inst 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n"
- "b.ne 7b\n"
- "6:\n"
- "cbz %[odds], 8f\n"
- "ld1 {v0.b}[0], [%[a_ptr0]], #1\n"
- "ld1 {v1.b}[0], [a_ptr1], #1\n"
- "ld1 {v2.b}[0], [a_ptr2], #1\n"
- "ld1 {v3.b}[0], [a_ptr3], #1\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 9f\n"
- "ld1 {v0.b}[1], [%[a_ptr0]], #1\n"
- "ld1 {v1.b}[1], [a_ptr1], #1\n"
- "ld1 {v2.b}[1], [a_ptr2], #1\n"
- "ld1 {v3.b}[1], [a_ptr3], #1\n"
- "subs %[odds], %[odds], #0x1\n"
- "b.eq 9f\n"
- "ld1 {v0.b}[2], [%[a_ptr0]]\n"
- "ld1 {v1.b}[2], [a_ptr1]\n"
- "ld1 {v2.b}[2], [a_ptr2]\n"
- "ld1 {v3.b}[2], [a_ptr3]\n"
- "9:\n"
- "ldr q8, [%[b_ptr0]]\n"
- "ldr q9, [%[b_ptr0], #0x10]\n"
- "ldr q10, [%[b_ptr0], #0x20]\n"
- "ldr q11, [%[b_ptr0], #0x30]\n"
- ".inst 0x6f80e110 // udot v16.4s, v8.16b, v0.4b[0]\n"
- ".inst 0x6f81e114 // udot v20.4s, v8.16b, v1.4b[0]\n"
- ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n"
- ".inst 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n"
- ".inst 0x6f80e131 // udot v17.4s, v9.16b, v0.4b[0]\n"
- ".inst 0x6f81e135 // udot v21.4s, v9.16b, v1.4b[0]\n"
- ".inst 0x6f82e139 // udot v25.4s, v9.16b, v2.4b[0]\n"
- ".inst 0x6f83e13d // udot v29.4s, v9.16b, v3.4b[0]\n"
- ".inst 0x6f80e152 // udot v18.4s, v10.16b, v0.4b[0]\n"
- ".inst 0x6f81e156 // udot v22.4s, v10.16b, v1.4b[0]\n"
- ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n"
- ".inst 0x6f83e15e // udot v30.4s, v10.16b, v3.4b[0]\n"
- ".inst 0x6f80e173 // udot v19.4s, v11.16b, v0.4b[0]\n"
- ".inst 0x6f81e177 // udot v23.4s, v11.16b, v1.4b[0]\n"
- ".inst 0x6f82e17b // udot v27.4s, v11.16b, v2.4b[0]\n"
- ".inst 0x6f83e17f // udot v31.4s, v11.16b, v3.4b[0]\n"
- "8:\n"
- "str q16, [%[c_ptr0]]\n"
- "str q17, [%[c_ptr0], #0x10]\n"
- "str q18, [%[c_ptr0], #0x20]\n"
- "str q19, [%[c_ptr0], #0x30]\n"
- "add %[c_ptr0], %[c_ptr0], #0x40\n"
- "str q20, [c_ptr1]\n"
- "str q21, [c_ptr1, #0x10]\n"
- "str q22, [c_ptr1, #0x20]\n"
- "str q23, [c_ptr1, #0x30]\n"
- "str q24, [c_ptr2]\n"
- "str q25, [c_ptr2, #0x10]\n"
- "str q26, [c_ptr2, #0x20]\n"
- "str q27, [c_ptr2, #0x30]\n"
- "str q28, [c_ptr3]\n"
- "str q29, [c_ptr3, #0x10]\n"
- "str q30, [c_ptr3, #0x20]\n"
- "str q31, [c_ptr3, #0x30]\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq a_ptr3\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- ".unreq c_ptr3\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [blocks] "+r" (blocks), [odds] "+r" (odds)
- : [width] "r" (width), [accumulate] "r" (static_cast<uint64_t>(accumulate)), [lda] "r" (ldab), [ldc] "r" (ldcb)
- : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "x4", "x5", "cc", "memory"
- );
- break;
- }
- if (use_result_buffer) {
- for(int cy=0; cy<std::min(M-y, 4); cy++) {
- for(unsigned int cx=0; cx<width; cx++) {
- c_ptr_real[cy * ldc + cx] = result_buffer[cy * 16 + cx];
- }
- }
- }
- }
- }
-}
-
-} // namespace arm_gemm
-
-#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_4x6.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_dot_6x16.hpp
similarity index 62%
copy from src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_4x6.hpp
copy to src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_dot_6x16.hpp
index 6f31efe..238c182 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_4x6.hpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_dot_6x16.hpp
@@ -10,34 +10,43 @@
* sell copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
*/
#pragma once
-
#ifdef __aarch64__
+#include "../std_transforms_fixed.hpp"
+
+#define ARGLIST \
+ unsigned int, const unsigned int *, \
+ IndirectInputArg<uint8_t>, \
+ size_t, size_t, \
+ const uint8_t *, \
+ IndirectOutputArg<uint32_t>, \
+ const uint32_t *, Activation, bool
+
namespace arm_gemm
{
// Actual kernel implementations
-void a64_smallK_hybrid_fp32_mla_4x6(const float *, int, const float *, float *, int, int, int, int, const float *, Activation, bool);
+void a64_hybrid_u8u32_dot_6x16( ARGLIST );
-class smallK_hybrid_fp32_mla_4x6
+class cls_a64_hybrid_u8u32_dot_6x16
{
public:
- typedef float operand_type;
- typedef float result_type;
+ typedef uint8_t operand_type;
+ typedef uint32_t result_type;
- typedef void (*kern_type)(const float *, int, const float *, float *, int, int, int, int, const float *, Activation, bool);
+ typedef void (*kern_type)( ARGLIST );
/* Kernel blocking parameters */
static constexpr unsigned int out_height()
@@ -47,40 +56,30 @@
static unsigned int out_width()
{
- return 4;
+ return 16;
}
static constexpr unsigned int k_unroll()
{
- return 1;
+ return 4;
}
static constexpr bool supports_accumulate()
{
- return false;
- }
-
- static constexpr bool supports_bias()
- {
return true;
}
- static constexpr bool supports_activation()
- {
- return true;
- }
-
- StdTransformsFixed<operand_type, result_type, 6, 4, 1> transforms = {};
+ StdTransformsFixed<operand_type, result_type, 6, 16, 4> transforms = {};
// Default to the generic kernel
- kern_type kernel=a64_smallK_hybrid_fp32_mla_4x6;
+ kern_type kernel=a64_hybrid_u8u32_dot_6x16;
- smallK_hybrid_fp32_mla_4x6(const CPUInfo *)
+ cls_a64_hybrid_u8u32_dot_6x16(const CPUInfo *)
{
-
}
};
} // namespace arm_gemm
+#undef ARGLIST
#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_dot_6x16/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_dot_6x16/generic.cpp
new file mode 100644
index 0000000..3c86541
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_dot_6x16/generic.cpp
@@ -0,0 +1,3335 @@
+/*
+ * Copyright (c) 2019-2020 Arm Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+#ifdef __aarch64__
+
+#include "arm_gemm.hpp"
+#include "../../utils.hpp"
+
+#include <cassert>
+
+namespace arm_gemm {
+
+void a64_hybrid_u8u32_dot_6x16 (
+ unsigned int num_strings, const unsigned int *string_lengths, IndirectInputArg<uint8_t> A_arg,
+ size_t M, size_t N, const uint8_t *B_ptr, IndirectOutputArg<uint32_t> output_arg,
+ const uint32_t *, Activation, bool accumulate
+)
+{
+ struct KernelArgs {
+ unsigned int num_strings = {};
+ const unsigned int *string_lengths = {};
+ size_t N = {};
+ const uint8_t *B_ptr = {};
+ size_t output_offset = {};
+ size_t input_initial_col = {};
+ size_t input_offset = {};
+ } ka;
+
+ unsigned long flags=0;
+ void *output_ptr;
+ void *input_ptr;
+
+ if (output_arg.is_indirect) {
+ output_ptr=(void *)(output_arg.indirect.ptr);
+ ka.output_offset=output_arg.indirect.offset;
+ flags |= 0x4;
+ } else {
+ output_ptr=(void *)(output_arg.direct.base);
+ ka.output_offset=output_arg.direct.stride;
+ }
+
+ if (A_arg.is_indirect) {
+ input_ptr=(void *)(A_arg.indirect.ptr);
+ ka.input_offset=A_arg.indirect.start_row;
+ ka.input_initial_col=A_arg.indirect.start_col;
+ flags |= 0x8;
+ } else {
+ assert(num_strings==1);
+ input_ptr=(void *)(A_arg.direct.base);
+ ka.input_offset=A_arg.direct.stride;
+ }
+ if (accumulate) {
+ flags |= 0x1;
+ }
+ ka.num_strings = num_strings;
+ ka.string_lengths = string_lengths;
+ ka.N = N;
+ ka.B_ptr = B_ptr;
+ __asm__ __volatile__(
+
+ "1:" // Row loop
+ "cmp %x[M], #0x6\n"
+ "bge 176f\n"
+ "cmp %x[M], #0x4\n"
+ "bgt 141f\n"
+ "beq 106f\n"
+ "cmp %x[M], #0x2\n"
+ "bgt 71f\n"
+ "beq 36f\n"
+ "ldr x15, [%x[args_ptr], %[offsetof_N]]\n"
+ "ldr x14, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 2f\n"
+ "ldr x13, [%x[output_ptr], #0x0]\n"
+ "add x13, x13, x19, LSL #2\n"
+ "b 3f\n"
+ "2:" // Height 1: setup direct output
+ "mov x13, %x[output_ptr]\n"
+ "3:" // Height 1: Column loop
+ "tbz %x[flags], #0, 13f\n"
+ "cmp x15, #0x10\n"
+ "bge 12f\n"
+ "tbz x15, #3, 7f\n"
+ "ld1 { v8.4s }, [x13], #0x10\n"
+ "ld1 { v9.4s }, [x13], #0x10\n"
+ "tbz x15, #2, 5f\n"
+ "ld1 { v10.4s }, [x13], #0x10\n"
+ "tbz x15, #1, 4f\n"
+ "mov x19, #0x38\n"
+ "ldr d11, [x13], #0x8\n"
+ "tbz x15, #0, 11f\n"
+ "ld1 { v11.s }[2], [x13]\n"
+ "b 11f\n"
+ "4:" // Height 1: Partial accumulate: partial_1_12
+ "mov x19, #0x30\n"
+ "tbz x15, #0, 11f\n"
+ "ldr s11, [x13, #0x0]\n"
+ "b 11f\n"
+ "5:" // Height 1: Partial accumulate: partial_2_8
+ "tbz x15, #1, 6f\n"
+ "ldr d10, [x13], #0x8\n"
+ "mov x19, #0x28\n"
+ "tbz x15, #0, 11f\n"
+ "ld1 { v10.s }[2], [x13]\n"
+ "b 11f\n"
+ "6:" // Height 1: Partial accumulate: partial_1_8
+ "mov x19, #0x20\n"
+ "tbz x15, #0, 11f\n"
+ "ldr s10, [x13, #0x0]\n"
+ "b 11f\n"
+ "7:" // Height 1: Partial accumulate: partial_4_0
+ "tbz x15, #2, 9f\n"
+ "ld1 { v8.4s }, [x13], #0x10\n"
+ "tbz x15, #1, 8f\n"
+ "mov x19, #0x18\n"
+ "ldr d9, [x13], #0x8\n"
+ "tbz x15, #0, 11f\n"
+ "ld1 { v9.s }[2], [x13]\n"
+ "b 11f\n"
+ "8:" // Height 1: Partial accumulate: partial_1_4
+ "mov x19, #0x10\n"
+ "tbz x15, #0, 11f\n"
+ "ldr s9, [x13, #0x0]\n"
+ "b 11f\n"
+ "9:" // Height 1: Partial accumulate: partial_2_0
+ "tbz x15, #1, 10f\n"
+ "ldr d8, [x13], #0x8\n"
+ "mov x19, #0x8\n"
+ "tbz x15, #0, 11f\n"
+ "ld1 { v8.s }[2], [x13]\n"
+ "b 11f\n"
+ "10:" // Height 1: Partial accumulate: partial_1_0
+ "mov x19, #0x0\n"
+ "ldr s8, [x13, #0x0]\n"
+ "11:" // Height 1: Partial accumulate: Done
+ "sub x13, x13, x19\n"
+ "b 14f\n"
+ "12:" // Height 1: full accumulate
+ "ldr q8, [x13, #0x0]\n"
+ "ldr q9, [x13, #0x10]\n"
+ "ldr q10, [x13, #0x20]\n"
+ "ldr q11, [x13, #0x30]\n"
+ "b 14f\n"
+ "13:" // Height 1: no accumulate
+ "movi v8.4s, #0x0\n"
+ "movi v9.4s, #0x0\n"
+ "movi v10.4s, #0x0\n"
+ "movi v11.4s, #0x0\n"
+ "14:" // Height 1: setup done
+ "mov x12, #0x0\n"
+ "15:" // Height 1: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w11, [x20, x12, LSL #0x2]\n"
+ "tbz %x[flags], #3, 16f\n"
+ "ldr x20, [%x[input_ptr], x12, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x10, [x20, #0x0]\n"
+ "cbnz x12, 17f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x10, x10, x19\n"
+ "b 17f\n"
+ "16:" // Height 1: setup direct input
+ "mov x10, %x[input_ptr]\n"
+ "17:" // Height 1: input setup done
+ "cmp x11, #0x10\n"
+ "blt 20f\n"
+ "cmp x11, #0x20\n"
+ "blt 19f\n"
+ "18:" // Height 1: Multiply loop: Main loop head
+ "ldr q0, [x10, #0x0]\n"
+ "ldr q6, [x14, #0x0]\n"
+ ".inst 0x6f80e0c8 // udot v8.4s, v6.16b, v0.4b[0]\n"
+ "ldr q7, [x14, #0x10]\n"
+ "ldr q6, [x14, #0x20]\n"
+ ".inst 0x6f80e0e9 // udot v9.4s, v7.16b, v0.4b[0]\n"
+ "ldr q7, [x14, #0x30]\n"
+ ".inst 0x6f80e0ca // udot v10.4s, v6.16b, v0.4b[0]\n"
+ "ldr q6, [x14, #0x40]\n"
+ "add x10, x10, #0x10\n"
+ ".inst 0x6f80e0eb // udot v11.4s, v7.16b, v0.4b[0]\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "ldr q7, [x14, #0x50]\n"
+ ".inst 0x6fa0e0c8 // udot v8.4s, v6.16b, v0.4b[1]\n"
+ "ldr q6, [x14, #0x60]\n"
+ "sub x11, x11, #0x10\n"
+ ".inst 0x6fa0e0e9 // udot v9.4s, v7.16b, v0.4b[1]\n"
+ "ldr q7, [x14, #0x70]\n"
+ "cmp x11, #0x20\n"
+ ".inst 0x6fa0e0ca // udot v10.4s, v6.16b, v0.4b[1]\n"
+ "ldr q6, [x14, #0x80]\n"
+ ".inst 0x6fa0e0eb // udot v11.4s, v7.16b, v0.4b[1]\n"
+ "ldr q7, [x14, #0x90]\n"
+ ".inst 0x6f80e8c8 // udot v8.4s, v6.16b, v0.4b[2]\n"
+ "ldr q6, [x14, #0xa0]\n"
+ ".inst 0x6f80e8e9 // udot v9.4s, v7.16b, v0.4b[2]\n"
+ "ldr q7, [x14, #0xb0]\n"
+ ".inst 0x6f80e8ca // udot v10.4s, v6.16b, v0.4b[2]\n"
+ "ldr q6, [x14, #0xc0]\n"
+ ".inst 0x6f80e8eb // udot v11.4s, v7.16b, v0.4b[2]\n"
+ "ldr q7, [x14, #0xd0]\n"
+ ".inst 0x6fa0e8c8 // udot v8.4s, v6.16b, v0.4b[3]\n"
+ "ldr q6, [x14, #0xe0]\n"
+ ".inst 0x6fa0e8e9 // udot v9.4s, v7.16b, v0.4b[3]\n"
+ "ldr q7, [x14, #0xf0]\n"
+ "add x14, x14, #0x100\n"
+ ".inst 0x6fa0e8ca // udot v10.4s, v6.16b, v0.4b[3]\n"
+ ".inst 0x6fa0e8eb // udot v11.4s, v7.16b, v0.4b[3]\n"
+ "bge 18b\n"
+ "19:" // Height 1: Multiply loop: Single iteration only
+ "sub x11, x11, #0x10\n"
+ "ldr q0, [x10, #0x0]\n"
+ "ldr q6, [x14, #0x0]\n"
+ ".inst 0x6f80e0c8 // udot v8.4s, v6.16b, v0.4b[0]\n"
+ "ldr q7, [x14, #0x10]\n"
+ "ldr q6, [x14, #0x20]\n"
+ ".inst 0x6f80e0e9 // udot v9.4s, v7.16b, v0.4b[0]\n"
+ "ldr q7, [x14, #0x30]\n"
+ ".inst 0x6f80e0ca // udot v10.4s, v6.16b, v0.4b[0]\n"
+ "ldr q6, [x14, #0x40]\n"
+ "add x10, x10, #0x10\n"
+ ".inst 0x6f80e0eb // udot v11.4s, v7.16b, v0.4b[0]\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "ldr q7, [x14, #0x50]\n"
+ ".inst 0x6fa0e0c8 // udot v8.4s, v6.16b, v0.4b[1]\n"
+ "ldr q6, [x14, #0x60]\n"
+ ".inst 0x6fa0e0e9 // udot v9.4s, v7.16b, v0.4b[1]\n"
+ "ldr q7, [x14, #0x70]\n"
+ ".inst 0x6fa0e0ca // udot v10.4s, v6.16b, v0.4b[1]\n"
+ "ldr q6, [x14, #0x80]\n"
+ ".inst 0x6fa0e0eb // udot v11.4s, v7.16b, v0.4b[1]\n"
+ "ldr q7, [x14, #0x90]\n"
+ ".inst 0x6f80e8c8 // udot v8.4s, v6.16b, v0.4b[2]\n"
+ "ldr q6, [x14, #0xa0]\n"
+ ".inst 0x6f80e8e9 // udot v9.4s, v7.16b, v0.4b[2]\n"
+ "ldr q7, [x14, #0xb0]\n"
+ ".inst 0x6f80e8ca // udot v10.4s, v6.16b, v0.4b[2]\n"
+ "ldr q6, [x14, #0xc0]\n"
+ ".inst 0x6f80e8eb // udot v11.4s, v7.16b, v0.4b[2]\n"
+ "ldr q7, [x14, #0xd0]\n"
+ ".inst 0x6fa0e8c8 // udot v8.4s, v6.16b, v0.4b[3]\n"
+ "ldr q6, [x14, #0xe0]\n"
+ ".inst 0x6fa0e8e9 // udot v9.4s, v7.16b, v0.4b[3]\n"
+ "ldr q7, [x14, #0xf0]\n"
+ "add x14, x14, #0x100\n"
+ ".inst 0x6fa0e8ca // udot v10.4s, v6.16b, v0.4b[3]\n"
+ ".inst 0x6fa0e8eb // udot v11.4s, v7.16b, v0.4b[3]\n"
+ "20:" // Height 1: Multiply loop: Main loop skip
+ "cbz x11, 25f\n"
+ "cmp x11, #0x4\n"
+ "blt 22f\n"
+ "21:" // Height 1: Multiply loop: Odd block loop
+ "ldr s0, [x10], #0x4\n"
+ "ldr q6, [x14, #0x0]\n"
+ ".inst 0x6f80e0c8 // udot v8.4s, v6.16b, v0.4b[0]\n"
+ "ldr q7, [x14, #0x10]\n"
+ "ldr q6, [x14, #0x20]\n"
+ ".inst 0x6f80e0e9 // udot v9.4s, v7.16b, v0.4b[0]\n"
+ "ldr q7, [x14, #0x30]\n"
+ ".inst 0x6f80e0ca // udot v10.4s, v6.16b, v0.4b[0]\n"
+ "sub x11, x11, #0x4\n"
+ "add x14, x14, #0x40\n"
+ ".inst 0x6f80e0eb // udot v11.4s, v7.16b, v0.4b[0]\n"
+ "cmp x11, #0x4\n"
+ "bge 21b\n"
+ "cbz x11, 25f\n"
+ "22:" // Height 1: Multiply loop: Skip odd blocks
+ "tbz x11, #1, 23f\n"
+ "ldr h0, [x10], #0x2\n"
+ "tbz x11, #0, 24f\n"
+ "ld1 { v0.b }[2], [x10]\n"
+ "b 24f\n"
+ "23:" // Height 1: Multiply loop: Ragged operand read: partial_1_0
+ "ldr b0, [x10, #0x0]\n"
+ "24:" // Height 1: Multiply loop: Ragged operand read: Done
+ "ldr q6, [x14, #0x0]\n"
+ ".inst 0x6f80e0c8 // udot v8.4s, v6.16b, v0.4b[0]\n"
+ "ldr q7, [x14, #0x10]\n"
+ "ldr q6, [x14, #0x20]\n"
+ ".inst 0x6f80e0e9 // udot v9.4s, v7.16b, v0.4b[0]\n"
+ "ldr q7, [x14, #0x30]\n"
+ ".inst 0x6f80e0ca // udot v10.4s, v6.16b, v0.4b[0]\n"
+ "add x14, x14, #0x40\n"
+ ".inst 0x6f80e0eb // udot v11.4s, v7.16b, v0.4b[0]\n"
+ "25:" // Height 1: Multiply loop: No odd multiplies
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "add x12, x12, #0x1\n"
+ "cmp x12, x19\n"
+ "bne 15b\n"
+ "prfm pstl1keep, [x13, #0x0]\n"
+ "cmp x15, #0x10\n"
+ "bge 34f\n"
+ "tbz x15, #3, 29f\n"
+ "st1 { v8.4s }, [x13], #0x10\n"
+ "st1 { v9.4s }, [x13], #0x10\n"
+ "tbz x15, #2, 27f\n"
+ "st1 { v10.4s }, [x13], #0x10\n"
+ "tbz x15, #1, 26f\n"
+ "str d11, [x13], #0x8\n"
+ "tbz x15, #0, 33f\n"
+ "st1 { v11.s }[2], [x13]\n"
+ "b 33f\n"
+ "26:" // Height 1: Partial direct writeback: partial_1_12
+ "tbz x15, #0, 33f\n"
+ "str s11, [x13, #0x0]\n"
+ "b 33f\n"
+ "27:" // Height 1: Partial direct writeback: partial_2_8
+ "tbz x15, #1, 28f\n"
+ "str d10, [x13], #0x8\n"
+ "tbz x15, #0, 33f\n"
+ "st1 { v10.s }[2], [x13]\n"
+ "b 33f\n"
+ "28:" // Height 1: Partial direct writeback: partial_1_8
+ "tbz x15, #0, 33f\n"
+ "str s10, [x13, #0x0]\n"
+ "b 33f\n"
+ "29:" // Height 1: Partial direct writeback: partial_4_0
+ "tbz x15, #2, 31f\n"
+ "st1 { v8.4s }, [x13], #0x10\n"
+ "tbz x15, #1, 30f\n"
+ "str d9, [x13], #0x8\n"
+ "tbz x15, #0, 33f\n"
+ "st1 { v9.s }[2], [x13]\n"
+ "b 33f\n"
+ "30:" // Height 1: Partial direct writeback: partial_1_4
+ "tbz x15, #0, 33f\n"
+ "str s9, [x13, #0x0]\n"
+ "b 33f\n"
+ "31:" // Height 1: Partial direct writeback: partial_2_0
+ "tbz x15, #1, 32f\n"
+ "str d8, [x13], #0x8\n"
+ "tbz x15, #0, 33f\n"
+ "st1 { v8.s }[2], [x13]\n"
+ "b 33f\n"
+ "32:" // Height 1: Partial direct writeback: partial_1_0
+ "str s8, [x13, #0x0]\n"
+ "33:" // Height 1: Partial direct writeback: Done
+ "b 35f\n"
+ "34:" // Height 1: Full writeback
+ "str q8, [x13, #0x0]\n"
+ "str q9, [x13, #0x10]\n"
+ "str q10, [x13, #0x20]\n"
+ "str q11, [x13, #0x30]\n"
+ "add x13, x13, #0x40\n"
+ "35:" // Height 1: Writeback done
+ "subs x15, x15, #0x10\n"
+ "bgt 3b\n"
+ "b 212f\n"
+ "36:" // Height 2
+ "ldr x15, [%x[args_ptr], %[offsetof_N]]\n"
+ "ldr x14, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 37f\n"
+ "ldr x13, [%x[output_ptr], #0x0]\n"
+ "add x13, x13, x19, LSL #2\n"
+ "ldr x9, [%x[output_ptr], #0x8]\n"
+ "add x9, x9, x19, LSL #2\n"
+ "b 38f\n"
+ "37:" // Height 2: setup direct output
+ "mov x13, %x[output_ptr]\n"
+ "add x9, x13, x19, LSL #2\n"
+ "38:" // Height 2: Column loop
+ "tbz %x[flags], #0, 48f\n"
+ "cmp x15, #0x10\n"
+ "bge 47f\n"
+ "tbz x15, #3, 42f\n"
+ "ld1 { v8.4s }, [x13], #0x10\n"
+ "ld1 { v12.4s }, [x9], #0x10\n"
+ "ld1 { v9.4s }, [x13], #0x10\n"
+ "ld1 { v13.4s }, [x9], #0x10\n"
+ "tbz x15, #2, 40f\n"
+ "ld1 { v10.4s }, [x13], #0x10\n"
+ "ld1 { v14.4s }, [x9], #0x10\n"
+ "tbz x15, #1, 39f\n"
+ "mov x19, #0x38\n"
+ "ldr d11, [x13], #0x8\n"
+ "ldr d15, [x9], #0x8\n"
+ "tbz x15, #0, 46f\n"
+ "ld1 { v11.s }[2], [x13]\n"
+ "ld1 { v15.s }[2], [x9]\n"
+ "b 46f\n"
+ "39:" // Height 2: Partial accumulate: partial_1_12
+ "mov x19, #0x30\n"
+ "tbz x15, #0, 46f\n"
+ "ldr s11, [x13, #0x0]\n"
+ "ldr s15, [x9, #0x0]\n"
+ "b 46f\n"
+ "40:" // Height 2: Partial accumulate: partial_2_8
+ "tbz x15, #1, 41f\n"
+ "ldr d10, [x13], #0x8\n"
+ "ldr d14, [x9], #0x8\n"
+ "mov x19, #0x28\n"
+ "tbz x15, #0, 46f\n"
+ "ld1 { v10.s }[2], [x13]\n"
+ "ld1 { v14.s }[2], [x9]\n"
+ "b 46f\n"
+ "41:" // Height 2: Partial accumulate: partial_1_8
+ "mov x19, #0x20\n"
+ "tbz x15, #0, 46f\n"
+ "ldr s10, [x13, #0x0]\n"
+ "ldr s14, [x9, #0x0]\n"
+ "b 46f\n"
+ "42:" // Height 2: Partial accumulate: partial_4_0
+ "tbz x15, #2, 44f\n"
+ "ld1 { v8.4s }, [x13], #0x10\n"
+ "ld1 { v12.4s }, [x9], #0x10\n"
+ "tbz x15, #1, 43f\n"
+ "mov x19, #0x18\n"
+ "ldr d9, [x13], #0x8\n"
+ "ldr d13, [x9], #0x8\n"
+ "tbz x15, #0, 46f\n"
+ "ld1 { v9.s }[2], [x13]\n"
+ "ld1 { v13.s }[2], [x9]\n"
+ "b 46f\n"
+ "43:" // Height 2: Partial accumulate: partial_1_4
+ "mov x19, #0x10\n"
+ "tbz x15, #0, 46f\n"
+ "ldr s9, [x13, #0x0]\n"
+ "ldr s13, [x9, #0x0]\n"
+ "b 46f\n"
+ "44:" // Height 2: Partial accumulate: partial_2_0
+ "tbz x15, #1, 45f\n"
+ "ldr d8, [x13], #0x8\n"
+ "ldr d12, [x9], #0x8\n"
+ "mov x19, #0x8\n"
+ "tbz x15, #0, 46f\n"
+ "ld1 { v8.s }[2], [x13]\n"
+ "ld1 { v12.s }[2], [x9]\n"
+ "b 46f\n"
+ "45:" // Height 2: Partial accumulate: partial_1_0
+ "mov x19, #0x0\n"
+ "ldr s8, [x13, #0x0]\n"
+ "ldr s12, [x9, #0x0]\n"
+ "46:" // Height 2: Partial accumulate: Done
+ "sub x13, x13, x19\n"
+ "sub x9, x9, x19\n"
+ "b 49f\n"
+ "47:" // Height 2: full accumulate
+ "ldr q8, [x13, #0x0]\n"
+ "ldr q9, [x13, #0x10]\n"
+ "ldr q10, [x13, #0x20]\n"
+ "ldr q11, [x13, #0x30]\n"
+ "ldr q12, [x9, #0x0]\n"
+ "ldr q13, [x9, #0x10]\n"
+ "ldr q14, [x9, #0x20]\n"
+ "ldr q15, [x9, #0x30]\n"
+ "b 49f\n"
+ "48:" // Height 2: no accumulate
+ "movi v8.4s, #0x0\n"
+ "movi v9.4s, #0x0\n"
+ "movi v10.4s, #0x0\n"
+ "movi v11.4s, #0x0\n"
+ "movi v12.4s, #0x0\n"
+ "movi v13.4s, #0x0\n"
+ "movi v14.4s, #0x0\n"
+ "movi v15.4s, #0x0\n"
+ "49:" // Height 2: setup done
+ "mov x12, #0x0\n"
+ "50:" // Height 2: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w11, [x20, x12, LSL #0x2]\n"
+ "tbz %x[flags], #3, 51f\n"
+ "ldr x20, [%x[input_ptr], x12, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x10, [x20, #0x0]\n"
+ "ldr x28, [x20, #0x8]\n"
+ "cbnz x12, 52f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x10, x10, x19\n"
+ "add x28, x28, x19\n"
+ "b 52f\n"
+ "51:" // Height 2: setup direct input
+ "mov x10, %x[input_ptr]\n"
+ "add x28, x10, x19\n"
+ "52:" // Height 2: input setup done
+ "cmp x11, #0x10\n"
+ "blt 55f\n"
+ "cmp x11, #0x20\n"
+ "blt 54f\n"
+ "53:" // Height 2: Multiply loop: Main loop head
+ "ldr q0, [x10, #0x0]\n"
+ "ldr q1, [x28, #0x0]\n"
+ "ldr q6, [x14, #0x0]\n"
+ ".inst 0x6f80e0c8 // udot v8.4s, v6.16b, v0.4b[0]\n"
+ "ldr q7, [x14, #0x10]\n"
+ ".inst 0x6f81e0cc // udot v12.4s, v6.16b, v1.4b[0]\n"
+ "ldr q6, [x14, #0x20]\n"
+ "add x10, x10, #0x10\n"
+ ".inst 0x6f80e0e9 // udot v9.4s, v7.16b, v0.4b[0]\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x28, x28, #0x10\n"
+ ".inst 0x6f81e0ed // udot v13.4s, v7.16b, v1.4b[0]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "ldr q7, [x14, #0x30]\n"
+ ".inst 0x6f80e0ca // udot v10.4s, v6.16b, v0.4b[0]\n"
+ "sub x11, x11, #0x10\n"
+ ".inst 0x6f81e0ce // udot v14.4s, v6.16b, v1.4b[0]\n"
+ "ldr q6, [x14, #0x40]\n"
+ "cmp x11, #0x20\n"
+ ".inst 0x6f80e0eb // udot v11.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x6f81e0ef // udot v15.4s, v7.16b, v1.4b[0]\n"
+ "ldr q7, [x14, #0x50]\n"
+ ".inst 0x6fa0e0c8 // udot v8.4s, v6.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e0cc // udot v12.4s, v6.16b, v1.4b[1]\n"
+ "ldr q6, [x14, #0x60]\n"
+ ".inst 0x6fa0e0e9 // udot v9.4s, v7.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e0ed // udot v13.4s, v7.16b, v1.4b[1]\n"
+ "ldr q7, [x14, #0x70]\n"
+ ".inst 0x6fa0e0ca // udot v10.4s, v6.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e0ce // udot v14.4s, v6.16b, v1.4b[1]\n"
+ "ldr q6, [x14, #0x80]\n"
+ ".inst 0x6fa0e0eb // udot v11.4s, v7.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e0ef // udot v15.4s, v7.16b, v1.4b[1]\n"
+ "ldr q7, [x14, #0x90]\n"
+ ".inst 0x6f80e8c8 // udot v8.4s, v6.16b, v0.4b[2]\n"
+ ".inst 0x6f81e8cc // udot v12.4s, v6.16b, v1.4b[2]\n"
+ "ldr q6, [x14, #0xa0]\n"
+ ".inst 0x6f80e8e9 // udot v9.4s, v7.16b, v0.4b[2]\n"
+ ".inst 0x6f81e8ed // udot v13.4s, v7.16b, v1.4b[2]\n"
+ "ldr q7, [x14, #0xb0]\n"
+ ".inst 0x6f80e8ca // udot v10.4s, v6.16b, v0.4b[2]\n"
+ ".inst 0x6f81e8ce // udot v14.4s, v6.16b, v1.4b[2]\n"
+ "ldr q6, [x14, #0xc0]\n"
+ ".inst 0x6f80e8eb // udot v11.4s, v7.16b, v0.4b[2]\n"
+ ".inst 0x6f81e8ef // udot v15.4s, v7.16b, v1.4b[2]\n"
+ "ldr q7, [x14, #0xd0]\n"
+ ".inst 0x6fa0e8c8 // udot v8.4s, v6.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e8cc // udot v12.4s, v6.16b, v1.4b[3]\n"
+ "ldr q6, [x14, #0xe0]\n"
+ ".inst 0x6fa0e8e9 // udot v9.4s, v7.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e8ed // udot v13.4s, v7.16b, v1.4b[3]\n"
+ "ldr q7, [x14, #0xf0]\n"
+ "add x14, x14, #0x100\n"
+ ".inst 0x6fa0e8ca // udot v10.4s, v6.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e8ce // udot v14.4s, v6.16b, v1.4b[3]\n"
+ ".inst 0x6fa0e8eb // udot v11.4s, v7.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e8ef // udot v15.4s, v7.16b, v1.4b[3]\n"
+ "bge 53b\n"
+ "54:" // Height 2: Multiply loop: Single iteration only
+ "sub x11, x11, #0x10\n"
+ "ldr q0, [x10, #0x0]\n"
+ "ldr q1, [x28, #0x0]\n"
+ "ldr q6, [x14, #0x0]\n"
+ ".inst 0x6f80e0c8 // udot v8.4s, v6.16b, v0.4b[0]\n"
+ "ldr q7, [x14, #0x10]\n"
+ ".inst 0x6f81e0cc // udot v12.4s, v6.16b, v1.4b[0]\n"
+ "ldr q6, [x14, #0x20]\n"
+ "add x10, x10, #0x10\n"
+ ".inst 0x6f80e0e9 // udot v9.4s, v7.16b, v0.4b[0]\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x28, x28, #0x10\n"
+ ".inst 0x6f81e0ed // udot v13.4s, v7.16b, v1.4b[0]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "ldr q7, [x14, #0x30]\n"
+ ".inst 0x6f80e0ca // udot v10.4s, v6.16b, v0.4b[0]\n"
+ ".inst 0x6f81e0ce // udot v14.4s, v6.16b, v1.4b[0]\n"
+ "ldr q6, [x14, #0x40]\n"
+ ".inst 0x6f80e0eb // udot v11.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x6f81e0ef // udot v15.4s, v7.16b, v1.4b[0]\n"
+ "ldr q7, [x14, #0x50]\n"
+ ".inst 0x6fa0e0c8 // udot v8.4s, v6.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e0cc // udot v12.4s, v6.16b, v1.4b[1]\n"
+ "ldr q6, [x14, #0x60]\n"
+ ".inst 0x6fa0e0e9 // udot v9.4s, v7.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e0ed // udot v13.4s, v7.16b, v1.4b[1]\n"
+ "ldr q7, [x14, #0x70]\n"
+ ".inst 0x6fa0e0ca // udot v10.4s, v6.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e0ce // udot v14.4s, v6.16b, v1.4b[1]\n"
+ "ldr q6, [x14, #0x80]\n"
+ ".inst 0x6fa0e0eb // udot v11.4s, v7.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e0ef // udot v15.4s, v7.16b, v1.4b[1]\n"
+ "ldr q7, [x14, #0x90]\n"
+ ".inst 0x6f80e8c8 // udot v8.4s, v6.16b, v0.4b[2]\n"
+ ".inst 0x6f81e8cc // udot v12.4s, v6.16b, v1.4b[2]\n"
+ "ldr q6, [x14, #0xa0]\n"
+ ".inst 0x6f80e8e9 // udot v9.4s, v7.16b, v0.4b[2]\n"
+ ".inst 0x6f81e8ed // udot v13.4s, v7.16b, v1.4b[2]\n"
+ "ldr q7, [x14, #0xb0]\n"
+ ".inst 0x6f80e8ca // udot v10.4s, v6.16b, v0.4b[2]\n"
+ ".inst 0x6f81e8ce // udot v14.4s, v6.16b, v1.4b[2]\n"
+ "ldr q6, [x14, #0xc0]\n"
+ ".inst 0x6f80e8eb // udot v11.4s, v7.16b, v0.4b[2]\n"
+ ".inst 0x6f81e8ef // udot v15.4s, v7.16b, v1.4b[2]\n"
+ "ldr q7, [x14, #0xd0]\n"
+ ".inst 0x6fa0e8c8 // udot v8.4s, v6.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e8cc // udot v12.4s, v6.16b, v1.4b[3]\n"
+ "ldr q6, [x14, #0xe0]\n"
+ ".inst 0x6fa0e8e9 // udot v9.4s, v7.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e8ed // udot v13.4s, v7.16b, v1.4b[3]\n"
+ "ldr q7, [x14, #0xf0]\n"
+ "add x14, x14, #0x100\n"
+ ".inst 0x6fa0e8ca // udot v10.4s, v6.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e8ce // udot v14.4s, v6.16b, v1.4b[3]\n"
+ ".inst 0x6fa0e8eb // udot v11.4s, v7.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e8ef // udot v15.4s, v7.16b, v1.4b[3]\n"
+ "55:" // Height 2: Multiply loop: Main loop skip
+ "cbz x11, 60f\n"
+ "cmp x11, #0x4\n"
+ "blt 57f\n"
+ "56:" // Height 2: Multiply loop: Odd block loop
+ "ldr s0, [x10], #0x4\n"
+ "ldr s1, [x28], #0x4\n"
+ "ldr q6, [x14, #0x0]\n"
+ ".inst 0x6f80e0c8 // udot v8.4s, v6.16b, v0.4b[0]\n"
+ "ldr q7, [x14, #0x10]\n"
+ ".inst 0x6f81e0cc // udot v12.4s, v6.16b, v1.4b[0]\n"
+ "ldr q6, [x14, #0x20]\n"
+ "sub x11, x11, #0x4\n"
+ ".inst 0x6f80e0e9 // udot v9.4s, v7.16b, v0.4b[0]\n"
+ "cmp x11, #0x4\n"
+ ".inst 0x6f81e0ed // udot v13.4s, v7.16b, v1.4b[0]\n"
+ "ldr q7, [x14, #0x30]\n"
+ ".inst 0x6f80e0ca // udot v10.4s, v6.16b, v0.4b[0]\n"
+ "add x14, x14, #0x40\n"
+ ".inst 0x6f81e0ce // udot v14.4s, v6.16b, v1.4b[0]\n"
+ ".inst 0x6f80e0eb // udot v11.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x6f81e0ef // udot v15.4s, v7.16b, v1.4b[0]\n"
+ "bge 56b\n"
+ "cbz x11, 60f\n"
+ "57:" // Height 2: Multiply loop: Skip odd blocks
+ "tbz x11, #1, 58f\n"
+ "ldr h0, [x10], #0x2\n"
+ "ldr h1, [x28], #0x2\n"
+ "tbz x11, #0, 59f\n"
+ "ld1 { v0.b }[2], [x10]\n"
+ "ld1 { v1.b }[2], [x28]\n"
+ "b 59f\n"
+ "58:" // Height 2: Multiply loop: Ragged operand read: partial_1_0
+ "ldr b0, [x10, #0x0]\n"
+ "ldr b1, [x28, #0x0]\n"
+ "59:" // Height 2: Multiply loop: Ragged operand read: Done
+ "ldr q6, [x14, #0x0]\n"
+ ".inst 0x6f80e0c8 // udot v8.4s, v6.16b, v0.4b[0]\n"
+ "ldr q7, [x14, #0x10]\n"
+ ".inst 0x6f81e0cc // udot v12.4s, v6.16b, v1.4b[0]\n"
+ "ldr q6, [x14, #0x20]\n"
+ ".inst 0x6f80e0e9 // udot v9.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x6f81e0ed // udot v13.4s, v7.16b, v1.4b[0]\n"
+ "ldr q7, [x14, #0x30]\n"
+ ".inst 0x6f80e0ca // udot v10.4s, v6.16b, v0.4b[0]\n"
+ "add x14, x14, #0x40\n"
+ ".inst 0x6f81e0ce // udot v14.4s, v6.16b, v1.4b[0]\n"
+ ".inst 0x6f80e0eb // udot v11.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x6f81e0ef // udot v15.4s, v7.16b, v1.4b[0]\n"
+ "60:" // Height 2: Multiply loop: No odd multiplies
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "add x12, x12, #0x1\n"
+ "cmp x12, x19\n"
+ "bne 50b\n"
+ "prfm pstl1keep, [x13, #0x0]\n"
+ "prfm pstl1keep, [x9, #0x0]\n"
+ "cmp x15, #0x10\n"
+ "bge 69f\n"
+ "tbz x15, #3, 64f\n"
+ "st1 { v8.4s }, [x13], #0x10\n"
+ "st1 { v9.4s }, [x13], #0x10\n"
+ "st1 { v12.4s }, [x9], #0x10\n"
+ "st1 { v13.4s }, [x9], #0x10\n"
+ "tbz x15, #2, 62f\n"
+ "st1 { v10.4s }, [x13], #0x10\n"
+ "st1 { v14.4s }, [x9], #0x10\n"
+ "tbz x15, #1, 61f\n"
+ "str d11, [x13], #0x8\n"
+ "str d15, [x9], #0x8\n"
+ "tbz x15, #0, 68f\n"
+ "st1 { v11.s }[2], [x13]\n"
+ "st1 { v15.s }[2], [x9]\n"
+ "b 68f\n"
+ "61:" // Height 2: Partial direct writeback: partial_1_12
+ "tbz x15, #0, 68f\n"
+ "str s11, [x13, #0x0]\n"
+ "str s15, [x9, #0x0]\n"
+ "b 68f\n"
+ "62:" // Height 2: Partial direct writeback: partial_2_8
+ "tbz x15, #1, 63f\n"
+ "str d10, [x13], #0x8\n"
+ "str d14, [x9], #0x8\n"
+ "tbz x15, #0, 68f\n"
+ "st1 { v10.s }[2], [x13]\n"
+ "st1 { v14.s }[2], [x9]\n"
+ "b 68f\n"
+ "63:" // Height 2: Partial direct writeback: partial_1_8
+ "tbz x15, #0, 68f\n"
+ "str s10, [x13, #0x0]\n"
+ "str s14, [x9, #0x0]\n"
+ "b 68f\n"
+ "64:" // Height 2: Partial direct writeback: partial_4_0
+ "tbz x15, #2, 66f\n"
+ "st1 { v8.4s }, [x13], #0x10\n"
+ "st1 { v12.4s }, [x9], #0x10\n"
+ "tbz x15, #1, 65f\n"
+ "str d9, [x13], #0x8\n"
+ "str d13, [x9], #0x8\n"
+ "tbz x15, #0, 68f\n"
+ "st1 { v9.s }[2], [x13]\n"
+ "st1 { v13.s }[2], [x9]\n"
+ "b 68f\n"
+ "65:" // Height 2: Partial direct writeback: partial_1_4
+ "tbz x15, #0, 68f\n"
+ "str s9, [x13, #0x0]\n"
+ "str s13, [x9, #0x0]\n"
+ "b 68f\n"
+ "66:" // Height 2: Partial direct writeback: partial_2_0
+ "tbz x15, #1, 67f\n"
+ "str d8, [x13], #0x8\n"
+ "str d12, [x9], #0x8\n"
+ "tbz x15, #0, 68f\n"
+ "st1 { v8.s }[2], [x13]\n"
+ "st1 { v12.s }[2], [x9]\n"
+ "b 68f\n"
+ "67:" // Height 2: Partial direct writeback: partial_1_0
+ "str s8, [x13, #0x0]\n"
+ "str s12, [x9, #0x0]\n"
+ "68:" // Height 2: Partial direct writeback: Done
+ "b 70f\n"
+ "69:" // Height 2: Full writeback
+ "str q8, [x13, #0x0]\n"
+ "str q9, [x13, #0x10]\n"
+ "str q10, [x13, #0x20]\n"
+ "str q11, [x13, #0x30]\n"
+ "str q12, [x9, #0x0]\n"
+ "str q13, [x9, #0x10]\n"
+ "str q14, [x9, #0x20]\n"
+ "str q15, [x9, #0x30]\n"
+ "add x13, x13, #0x40\n"
+ "add x9, x9, #0x40\n"
+ "70:" // Height 2: Writeback done
+ "subs x15, x15, #0x10\n"
+ "bgt 38b\n"
+ "b 212f\n"
+ "71:" // Height 3
+ "ldr x15, [%x[args_ptr], %[offsetof_N]]\n"
+ "ldr x14, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 72f\n"
+ "ldr x13, [%x[output_ptr], #0x0]\n"
+ "add x13, x13, x19, LSL #2\n"
+ "ldr x9, [%x[output_ptr], #0x8]\n"
+ "ldr x27, [%x[output_ptr], #0x10]\n"
+ "add x9, x9, x19, LSL #2\n"
+ "add x27, x27, x19, LSL #2\n"
+ "b 73f\n"
+ "72:" // Height 3: setup direct output
+ "mov x13, %x[output_ptr]\n"
+ "add x9, x13, x19, LSL #2\n"
+ "add x27, x9, x19, LSL #2\n"
+ "73:" // Height 3: Column loop
+ "tbz %x[flags], #0, 83f\n"
+ "cmp x15, #0x10\n"
+ "bge 82f\n"
+ "tbz x15, #3, 77f\n"
+ "ld1 { v8.4s }, [x13], #0x10\n"
+ "ld1 { v12.4s }, [x9], #0x10\n"
+ "ld1 { v16.4s }, [x27], #0x10\n"
+ "ld1 { v9.4s }, [x13], #0x10\n"
+ "ld1 { v13.4s }, [x9], #0x10\n"
+ "ld1 { v17.4s }, [x27], #0x10\n"
+ "tbz x15, #2, 75f\n"
+ "ld1 { v10.4s }, [x13], #0x10\n"
+ "ld1 { v14.4s }, [x9], #0x10\n"
+ "ld1 { v18.4s }, [x27], #0x10\n"
+ "tbz x15, #1, 74f\n"
+ "mov x19, #0x38\n"
+ "ldr d11, [x13], #0x8\n"
+ "ldr d15, [x9], #0x8\n"
+ "ldr d19, [x27], #0x8\n"
+ "tbz x15, #0, 81f\n"
+ "ld1 { v11.s }[2], [x13]\n"
+ "ld1 { v15.s }[2], [x9]\n"
+ "ld1 { v19.s }[2], [x27]\n"
+ "b 81f\n"
+ "74:" // Height 3: Partial accumulate: partial_1_12
+ "mov x19, #0x30\n"
+ "tbz x15, #0, 81f\n"
+ "ldr s11, [x13, #0x0]\n"
+ "ldr s15, [x9, #0x0]\n"
+ "ldr s19, [x27, #0x0]\n"
+ "b 81f\n"
+ "75:" // Height 3: Partial accumulate: partial_2_8
+ "tbz x15, #1, 76f\n"
+ "ldr d10, [x13], #0x8\n"
+ "ldr d14, [x9], #0x8\n"
+ "ldr d18, [x27], #0x8\n"
+ "mov x19, #0x28\n"
+ "tbz x15, #0, 81f\n"
+ "ld1 { v10.s }[2], [x13]\n"
+ "ld1 { v14.s }[2], [x9]\n"
+ "ld1 { v18.s }[2], [x27]\n"
+ "b 81f\n"
+ "76:" // Height 3: Partial accumulate: partial_1_8
+ "mov x19, #0x20\n"
+ "tbz x15, #0, 81f\n"
+ "ldr s10, [x13, #0x0]\n"
+ "ldr s14, [x9, #0x0]\n"
+ "ldr s18, [x27, #0x0]\n"
+ "b 81f\n"
+ "77:" // Height 3: Partial accumulate: partial_4_0
+ "tbz x15, #2, 79f\n"
+ "ld1 { v8.4s }, [x13], #0x10\n"
+ "ld1 { v12.4s }, [x9], #0x10\n"
+ "ld1 { v16.4s }, [x27], #0x10\n"
+ "tbz x15, #1, 78f\n"
+ "mov x19, #0x18\n"
+ "ldr d9, [x13], #0x8\n"
+ "ldr d13, [x9], #0x8\n"
+ "ldr d17, [x27], #0x8\n"
+ "tbz x15, #0, 81f\n"
+ "ld1 { v9.s }[2], [x13]\n"
+ "ld1 { v13.s }[2], [x9]\n"
+ "ld1 { v17.s }[2], [x27]\n"
+ "b 81f\n"
+ "78:" // Height 3: Partial accumulate: partial_1_4
+ "mov x19, #0x10\n"
+ "tbz x15, #0, 81f\n"
+ "ldr s9, [x13, #0x0]\n"
+ "ldr s13, [x9, #0x0]\n"
+ "ldr s17, [x27, #0x0]\n"
+ "b 81f\n"
+ "79:" // Height 3: Partial accumulate: partial_2_0
+ "tbz x15, #1, 80f\n"
+ "ldr d8, [x13], #0x8\n"
+ "ldr d12, [x9], #0x8\n"
+ "ldr d16, [x27], #0x8\n"
+ "mov x19, #0x8\n"
+ "tbz x15, #0, 81f\n"
+ "ld1 { v8.s }[2], [x13]\n"
+ "ld1 { v12.s }[2], [x9]\n"
+ "ld1 { v16.s }[2], [x27]\n"
+ "b 81f\n"
+ "80:" // Height 3: Partial accumulate: partial_1_0
+ "mov x19, #0x0\n"
+ "ldr s8, [x13, #0x0]\n"
+ "ldr s12, [x9, #0x0]\n"
+ "ldr s16, [x27, #0x0]\n"
+ "81:" // Height 3: Partial accumulate: Done
+ "sub x13, x13, x19\n"
+ "sub x9, x9, x19\n"
+ "sub x27, x27, x19\n"
+ "b 84f\n"
+ "82:" // Height 3: full accumulate
+ "ldr q8, [x13, #0x0]\n"
+ "ldr q9, [x13, #0x10]\n"
+ "ldr q10, [x13, #0x20]\n"
+ "ldr q11, [x13, #0x30]\n"
+ "ldr q12, [x9, #0x0]\n"
+ "ldr q13, [x9, #0x10]\n"
+ "ldr q14, [x9, #0x20]\n"
+ "ldr q15, [x9, #0x30]\n"
+ "ldr q16, [x27, #0x0]\n"
+ "ldr q17, [x27, #0x10]\n"
+ "ldr q18, [x27, #0x20]\n"
+ "ldr q19, [x27, #0x30]\n"
+ "b 84f\n"
+ "83:" // Height 3: no accumulate
+ "movi v8.4s, #0x0\n"
+ "movi v9.4s, #0x0\n"
+ "movi v10.4s, #0x0\n"
+ "movi v11.4s, #0x0\n"
+ "movi v12.4s, #0x0\n"
+ "movi v13.4s, #0x0\n"
+ "movi v14.4s, #0x0\n"
+ "movi v15.4s, #0x0\n"
+ "movi v16.4s, #0x0\n"
+ "movi v17.4s, #0x0\n"
+ "movi v18.4s, #0x0\n"
+ "movi v19.4s, #0x0\n"
+ "84:" // Height 3: setup done
+ "mov x12, #0x0\n"
+ "85:" // Height 3: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w11, [x20, x12, LSL #0x2]\n"
+ "tbz %x[flags], #3, 86f\n"
+ "ldr x20, [%x[input_ptr], x12, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x10, [x20, #0x0]\n"
+ "ldr x28, [x20, #0x8]\n"
+ "ldr x26, [x20, #0x10]\n"
+ "cbnz x12, 87f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x10, x10, x19\n"
+ "add x28, x28, x19\n"
+ "add x26, x26, x19\n"
+ "b 87f\n"
+ "86:" // Height 3: setup direct input
+ "mov x10, %x[input_ptr]\n"
+ "add x28, x10, x19\n"
+ "add x26, x28, x19\n"
+ "87:" // Height 3: input setup done
+ "cmp x11, #0x10\n"
+ "blt 90f\n"
+ "cmp x11, #0x20\n"
+ "blt 89f\n"
+ "88:" // Height 3: Multiply loop: Main loop head
+ "ldr q0, [x10, #0x0]\n"
+ "ldr q1, [x28, #0x0]\n"
+ "ldr q2, [x26, #0x0]\n"
+ "ldr q6, [x14, #0x0]\n"
+ ".inst 0x6f80e0c8 // udot v8.4s, v6.16b, v0.4b[0]\n"
+ "ldr q7, [x14, #0x10]\n"
+ ".inst 0x6f81e0cc // udot v12.4s, v6.16b, v1.4b[0]\n"
+ "add x10, x10, #0x10\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ ".inst 0x6f82e0d0 // udot v16.4s, v6.16b, v2.4b[0]\n"
+ "ldr q6, [x14, #0x20]\n"
+ ".inst 0x6f80e0e9 // udot v9.4s, v7.16b, v0.4b[0]\n"
+ "add x28, x28, #0x10\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ ".inst 0x6f81e0ed // udot v13.4s, v7.16b, v1.4b[0]\n"
+ "add x26, x26, #0x10\n"
+ ".inst 0x6f82e0f1 // udot v17.4s, v7.16b, v2.4b[0]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "ldr q7, [x14, #0x30]\n"
+ ".inst 0x6f80e0ca // udot v10.4s, v6.16b, v0.4b[0]\n"
+ "sub x11, x11, #0x10\n"
+ ".inst 0x6f81e0ce // udot v14.4s, v6.16b, v1.4b[0]\n"
+ "cmp x11, #0x20\n"
+ ".inst 0x6f82e0d2 // udot v18.4s, v6.16b, v2.4b[0]\n"
+ "ldr q6, [x14, #0x40]\n"
+ ".inst 0x6f80e0eb // udot v11.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x6f81e0ef // udot v15.4s, v7.16b, v1.4b[0]\n"
+ ".inst 0x6f82e0f3 // udot v19.4s, v7.16b, v2.4b[0]\n"
+ "ldr q7, [x14, #0x50]\n"
+ ".inst 0x6fa0e0c8 // udot v8.4s, v6.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e0cc // udot v12.4s, v6.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e0d0 // udot v16.4s, v6.16b, v2.4b[1]\n"
+ "ldr q6, [x14, #0x60]\n"
+ ".inst 0x6fa0e0e9 // udot v9.4s, v7.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e0ed // udot v13.4s, v7.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e0f1 // udot v17.4s, v7.16b, v2.4b[1]\n"
+ "ldr q7, [x14, #0x70]\n"
+ ".inst 0x6fa0e0ca // udot v10.4s, v6.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e0ce // udot v14.4s, v6.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e0d2 // udot v18.4s, v6.16b, v2.4b[1]\n"
+ "ldr q6, [x14, #0x80]\n"
+ ".inst 0x6fa0e0eb // udot v11.4s, v7.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e0ef // udot v15.4s, v7.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e0f3 // udot v19.4s, v7.16b, v2.4b[1]\n"
+ "ldr q7, [x14, #0x90]\n"
+ ".inst 0x6f80e8c8 // udot v8.4s, v6.16b, v0.4b[2]\n"
+ ".inst 0x6f81e8cc // udot v12.4s, v6.16b, v1.4b[2]\n"
+ ".inst 0x6f82e8d0 // udot v16.4s, v6.16b, v2.4b[2]\n"
+ "ldr q6, [x14, #0xa0]\n"
+ ".inst 0x6f80e8e9 // udot v9.4s, v7.16b, v0.4b[2]\n"
+ ".inst 0x6f81e8ed // udot v13.4s, v7.16b, v1.4b[2]\n"
+ ".inst 0x6f82e8f1 // udot v17.4s, v7.16b, v2.4b[2]\n"
+ "ldr q7, [x14, #0xb0]\n"
+ ".inst 0x6f80e8ca // udot v10.4s, v6.16b, v0.4b[2]\n"
+ ".inst 0x6f81e8ce // udot v14.4s, v6.16b, v1.4b[2]\n"
+ ".inst 0x6f82e8d2 // udot v18.4s, v6.16b, v2.4b[2]\n"
+ "ldr q6, [x14, #0xc0]\n"
+ ".inst 0x6f80e8eb // udot v11.4s, v7.16b, v0.4b[2]\n"
+ ".inst 0x6f81e8ef // udot v15.4s, v7.16b, v1.4b[2]\n"
+ ".inst 0x6f82e8f3 // udot v19.4s, v7.16b, v2.4b[2]\n"
+ "ldr q7, [x14, #0xd0]\n"
+ ".inst 0x6fa0e8c8 // udot v8.4s, v6.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e8cc // udot v12.4s, v6.16b, v1.4b[3]\n"
+ ".inst 0x6fa2e8d0 // udot v16.4s, v6.16b, v2.4b[3]\n"
+ "ldr q6, [x14, #0xe0]\n"
+ ".inst 0x6fa0e8e9 // udot v9.4s, v7.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e8ed // udot v13.4s, v7.16b, v1.4b[3]\n"
+ ".inst 0x6fa2e8f1 // udot v17.4s, v7.16b, v2.4b[3]\n"
+ "ldr q7, [x14, #0xf0]\n"
+ "add x14, x14, #0x100\n"
+ ".inst 0x6fa0e8ca // udot v10.4s, v6.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e8ce // udot v14.4s, v6.16b, v1.4b[3]\n"
+ ".inst 0x6fa2e8d2 // udot v18.4s, v6.16b, v2.4b[3]\n"
+ ".inst 0x6fa0e8eb // udot v11.4s, v7.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e8ef // udot v15.4s, v7.16b, v1.4b[3]\n"
+ ".inst 0x6fa2e8f3 // udot v19.4s, v7.16b, v2.4b[3]\n"
+ "bge 88b\n"
+ "89:" // Height 3: Multiply loop: Single iteration only
+ "sub x11, x11, #0x10\n"
+ "ldr q0, [x10, #0x0]\n"
+ "ldr q1, [x28, #0x0]\n"
+ "ldr q2, [x26, #0x0]\n"
+ "ldr q6, [x14, #0x0]\n"
+ ".inst 0x6f80e0c8 // udot v8.4s, v6.16b, v0.4b[0]\n"
+ "ldr q7, [x14, #0x10]\n"
+ ".inst 0x6f81e0cc // udot v12.4s, v6.16b, v1.4b[0]\n"
+ "add x10, x10, #0x10\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ ".inst 0x6f82e0d0 // udot v16.4s, v6.16b, v2.4b[0]\n"
+ "ldr q6, [x14, #0x20]\n"
+ ".inst 0x6f80e0e9 // udot v9.4s, v7.16b, v0.4b[0]\n"
+ "add x28, x28, #0x10\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ ".inst 0x6f81e0ed // udot v13.4s, v7.16b, v1.4b[0]\n"
+ "add x26, x26, #0x10\n"
+ ".inst 0x6f82e0f1 // udot v17.4s, v7.16b, v2.4b[0]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "ldr q7, [x14, #0x30]\n"
+ ".inst 0x6f80e0ca // udot v10.4s, v6.16b, v0.4b[0]\n"
+ ".inst 0x6f81e0ce // udot v14.4s, v6.16b, v1.4b[0]\n"
+ ".inst 0x6f82e0d2 // udot v18.4s, v6.16b, v2.4b[0]\n"
+ "ldr q6, [x14, #0x40]\n"
+ ".inst 0x6f80e0eb // udot v11.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x6f81e0ef // udot v15.4s, v7.16b, v1.4b[0]\n"
+ ".inst 0x6f82e0f3 // udot v19.4s, v7.16b, v2.4b[0]\n"
+ "ldr q7, [x14, #0x50]\n"
+ ".inst 0x6fa0e0c8 // udot v8.4s, v6.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e0cc // udot v12.4s, v6.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e0d0 // udot v16.4s, v6.16b, v2.4b[1]\n"
+ "ldr q6, [x14, #0x60]\n"
+ ".inst 0x6fa0e0e9 // udot v9.4s, v7.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e0ed // udot v13.4s, v7.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e0f1 // udot v17.4s, v7.16b, v2.4b[1]\n"
+ "ldr q7, [x14, #0x70]\n"
+ ".inst 0x6fa0e0ca // udot v10.4s, v6.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e0ce // udot v14.4s, v6.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e0d2 // udot v18.4s, v6.16b, v2.4b[1]\n"
+ "ldr q6, [x14, #0x80]\n"
+ ".inst 0x6fa0e0eb // udot v11.4s, v7.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e0ef // udot v15.4s, v7.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e0f3 // udot v19.4s, v7.16b, v2.4b[1]\n"
+ "ldr q7, [x14, #0x90]\n"
+ ".inst 0x6f80e8c8 // udot v8.4s, v6.16b, v0.4b[2]\n"
+ ".inst 0x6f81e8cc // udot v12.4s, v6.16b, v1.4b[2]\n"
+ ".inst 0x6f82e8d0 // udot v16.4s, v6.16b, v2.4b[2]\n"
+ "ldr q6, [x14, #0xa0]\n"
+ ".inst 0x6f80e8e9 // udot v9.4s, v7.16b, v0.4b[2]\n"
+ ".inst 0x6f81e8ed // udot v13.4s, v7.16b, v1.4b[2]\n"
+ ".inst 0x6f82e8f1 // udot v17.4s, v7.16b, v2.4b[2]\n"
+ "ldr q7, [x14, #0xb0]\n"
+ ".inst 0x6f80e8ca // udot v10.4s, v6.16b, v0.4b[2]\n"
+ ".inst 0x6f81e8ce // udot v14.4s, v6.16b, v1.4b[2]\n"
+ ".inst 0x6f82e8d2 // udot v18.4s, v6.16b, v2.4b[2]\n"
+ "ldr q6, [x14, #0xc0]\n"
+ ".inst 0x6f80e8eb // udot v11.4s, v7.16b, v0.4b[2]\n"
+ ".inst 0x6f81e8ef // udot v15.4s, v7.16b, v1.4b[2]\n"
+ ".inst 0x6f82e8f3 // udot v19.4s, v7.16b, v2.4b[2]\n"
+ "ldr q7, [x14, #0xd0]\n"
+ ".inst 0x6fa0e8c8 // udot v8.4s, v6.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e8cc // udot v12.4s, v6.16b, v1.4b[3]\n"
+ ".inst 0x6fa2e8d0 // udot v16.4s, v6.16b, v2.4b[3]\n"
+ "ldr q6, [x14, #0xe0]\n"
+ ".inst 0x6fa0e8e9 // udot v9.4s, v7.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e8ed // udot v13.4s, v7.16b, v1.4b[3]\n"
+ ".inst 0x6fa2e8f1 // udot v17.4s, v7.16b, v2.4b[3]\n"
+ "ldr q7, [x14, #0xf0]\n"
+ "add x14, x14, #0x100\n"
+ ".inst 0x6fa0e8ca // udot v10.4s, v6.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e8ce // udot v14.4s, v6.16b, v1.4b[3]\n"
+ ".inst 0x6fa2e8d2 // udot v18.4s, v6.16b, v2.4b[3]\n"
+ ".inst 0x6fa0e8eb // udot v11.4s, v7.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e8ef // udot v15.4s, v7.16b, v1.4b[3]\n"
+ ".inst 0x6fa2e8f3 // udot v19.4s, v7.16b, v2.4b[3]\n"
+ "90:" // Height 3: Multiply loop: Main loop skip
+ "cbz x11, 95f\n"
+ "cmp x11, #0x4\n"
+ "blt 92f\n"
+ "91:" // Height 3: Multiply loop: Odd block loop
+ "ldr s0, [x10], #0x4\n"
+ "ldr s1, [x28], #0x4\n"
+ "ldr s2, [x26], #0x4\n"
+ "ldr q6, [x14, #0x0]\n"
+ ".inst 0x6f80e0c8 // udot v8.4s, v6.16b, v0.4b[0]\n"
+ "ldr q7, [x14, #0x10]\n"
+ ".inst 0x6f81e0cc // udot v12.4s, v6.16b, v1.4b[0]\n"
+ "sub x11, x11, #0x4\n"
+ ".inst 0x6f82e0d0 // udot v16.4s, v6.16b, v2.4b[0]\n"
+ "ldr q6, [x14, #0x20]\n"
+ "cmp x11, #0x4\n"
+ ".inst 0x6f80e0e9 // udot v9.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x6f81e0ed // udot v13.4s, v7.16b, v1.4b[0]\n"
+ ".inst 0x6f82e0f1 // udot v17.4s, v7.16b, v2.4b[0]\n"
+ "ldr q7, [x14, #0x30]\n"
+ ".inst 0x6f80e0ca // udot v10.4s, v6.16b, v0.4b[0]\n"
+ "add x14, x14, #0x40\n"
+ ".inst 0x6f81e0ce // udot v14.4s, v6.16b, v1.4b[0]\n"
+ ".inst 0x6f82e0d2 // udot v18.4s, v6.16b, v2.4b[0]\n"
+ ".inst 0x6f80e0eb // udot v11.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x6f81e0ef // udot v15.4s, v7.16b, v1.4b[0]\n"
+ ".inst 0x6f82e0f3 // udot v19.4s, v7.16b, v2.4b[0]\n"
+ "bge 91b\n"
+ "cbz x11, 95f\n"
+ "92:" // Height 3: Multiply loop: Skip odd blocks
+ "tbz x11, #1, 93f\n"
+ "ldr h0, [x10], #0x2\n"
+ "ldr h1, [x28], #0x2\n"
+ "ldr h2, [x26], #0x2\n"
+ "tbz x11, #0, 94f\n"
+ "ld1 { v0.b }[2], [x10]\n"
+ "ld1 { v1.b }[2], [x28]\n"
+ "ld1 { v2.b }[2], [x26]\n"
+ "b 94f\n"
+ "93:" // Height 3: Multiply loop: Ragged operand read: partial_1_0
+ "ldr b0, [x10, #0x0]\n"
+ "ldr b1, [x28, #0x0]\n"
+ "ldr b2, [x26, #0x0]\n"
+ "94:" // Height 3: Multiply loop: Ragged operand read: Done
+ "ldr q6, [x14, #0x0]\n"
+ ".inst 0x6f80e0c8 // udot v8.4s, v6.16b, v0.4b[0]\n"
+ "ldr q7, [x14, #0x10]\n"
+ ".inst 0x6f81e0cc // udot v12.4s, v6.16b, v1.4b[0]\n"
+ ".inst 0x6f82e0d0 // udot v16.4s, v6.16b, v2.4b[0]\n"
+ "ldr q6, [x14, #0x20]\n"
+ ".inst 0x6f80e0e9 // udot v9.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x6f81e0ed // udot v13.4s, v7.16b, v1.4b[0]\n"
+ ".inst 0x6f82e0f1 // udot v17.4s, v7.16b, v2.4b[0]\n"
+ "ldr q7, [x14, #0x30]\n"
+ ".inst 0x6f80e0ca // udot v10.4s, v6.16b, v0.4b[0]\n"
+ "add x14, x14, #0x40\n"
+ ".inst 0x6f81e0ce // udot v14.4s, v6.16b, v1.4b[0]\n"
+ ".inst 0x6f82e0d2 // udot v18.4s, v6.16b, v2.4b[0]\n"
+ ".inst 0x6f80e0eb // udot v11.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x6f81e0ef // udot v15.4s, v7.16b, v1.4b[0]\n"
+ ".inst 0x6f82e0f3 // udot v19.4s, v7.16b, v2.4b[0]\n"
+ "95:" // Height 3: Multiply loop: No odd multiplies
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "add x12, x12, #0x1\n"
+ "cmp x12, x19\n"
+ "bne 85b\n"
+ "prfm pstl1keep, [x13, #0x0]\n"
+ "prfm pstl1keep, [x9, #0x0]\n"
+ "cmp x15, #0x10\n"
+ "prfm pstl1keep, [x27, #0x0]\n"
+ "bge 104f\n"
+ "tbz x15, #3, 99f\n"
+ "st1 { v8.4s }, [x13], #0x10\n"
+ "st1 { v9.4s }, [x13], #0x10\n"
+ "st1 { v12.4s }, [x9], #0x10\n"
+ "st1 { v13.4s }, [x9], #0x10\n"
+ "st1 { v16.4s }, [x27], #0x10\n"
+ "st1 { v17.4s }, [x27], #0x10\n"
+ "tbz x15, #2, 97f\n"
+ "st1 { v10.4s }, [x13], #0x10\n"
+ "st1 { v14.4s }, [x9], #0x10\n"
+ "st1 { v18.4s }, [x27], #0x10\n"
+ "tbz x15, #1, 96f\n"
+ "str d11, [x13], #0x8\n"
+ "str d15, [x9], #0x8\n"
+ "str d19, [x27], #0x8\n"
+ "tbz x15, #0, 103f\n"
+ "st1 { v11.s }[2], [x13]\n"
+ "st1 { v15.s }[2], [x9]\n"
+ "st1 { v19.s }[2], [x27]\n"
+ "b 103f\n"
+ "96:" // Height 3: Partial direct writeback: partial_1_12
+ "tbz x15, #0, 103f\n"
+ "str s11, [x13, #0x0]\n"
+ "str s15, [x9, #0x0]\n"
+ "str s19, [x27, #0x0]\n"
+ "b 103f\n"
+ "97:" // Height 3: Partial direct writeback: partial_2_8
+ "tbz x15, #1, 98f\n"
+ "str d10, [x13], #0x8\n"
+ "str d14, [x9], #0x8\n"
+ "str d18, [x27], #0x8\n"
+ "tbz x15, #0, 103f\n"
+ "st1 { v10.s }[2], [x13]\n"
+ "st1 { v14.s }[2], [x9]\n"
+ "st1 { v18.s }[2], [x27]\n"
+ "b 103f\n"
+ "98:" // Height 3: Partial direct writeback: partial_1_8
+ "tbz x15, #0, 103f\n"
+ "str s10, [x13, #0x0]\n"
+ "str s14, [x9, #0x0]\n"
+ "str s18, [x27, #0x0]\n"
+ "b 103f\n"
+ "99:" // Height 3: Partial direct writeback: partial_4_0
+ "tbz x15, #2, 101f\n"
+ "st1 { v8.4s }, [x13], #0x10\n"
+ "st1 { v12.4s }, [x9], #0x10\n"
+ "st1 { v16.4s }, [x27], #0x10\n"
+ "tbz x15, #1, 100f\n"
+ "str d9, [x13], #0x8\n"
+ "str d13, [x9], #0x8\n"
+ "str d17, [x27], #0x8\n"
+ "tbz x15, #0, 103f\n"
+ "st1 { v9.s }[2], [x13]\n"
+ "st1 { v13.s }[2], [x9]\n"
+ "st1 { v17.s }[2], [x27]\n"
+ "b 103f\n"
+ "100:" // Height 3: Partial direct writeback: partial_1_4
+ "tbz x15, #0, 103f\n"
+ "str s9, [x13, #0x0]\n"
+ "str s13, [x9, #0x0]\n"
+ "str s17, [x27, #0x0]\n"
+ "b 103f\n"
+ "101:" // Height 3: Partial direct writeback: partial_2_0
+ "tbz x15, #1, 102f\n"
+ "str d8, [x13], #0x8\n"
+ "str d12, [x9], #0x8\n"
+ "str d16, [x27], #0x8\n"
+ "tbz x15, #0, 103f\n"
+ "st1 { v8.s }[2], [x13]\n"
+ "st1 { v12.s }[2], [x9]\n"
+ "st1 { v16.s }[2], [x27]\n"
+ "b 103f\n"
+ "102:" // Height 3: Partial direct writeback: partial_1_0
+ "str s8, [x13, #0x0]\n"
+ "str s12, [x9, #0x0]\n"
+ "str s16, [x27, #0x0]\n"
+ "103:" // Height 3: Partial direct writeback: Done
+ "b 105f\n"
+ "104:" // Height 3: Full writeback
+ "str q8, [x13, #0x0]\n"
+ "str q9, [x13, #0x10]\n"
+ "str q10, [x13, #0x20]\n"
+ "str q11, [x13, #0x30]\n"
+ "str q12, [x9, #0x0]\n"
+ "str q13, [x9, #0x10]\n"
+ "str q14, [x9, #0x20]\n"
+ "str q15, [x9, #0x30]\n"
+ "str q16, [x27, #0x0]\n"
+ "str q17, [x27, #0x10]\n"
+ "str q18, [x27, #0x20]\n"
+ "str q19, [x27, #0x30]\n"
+ "add x13, x13, #0x40\n"
+ "add x9, x9, #0x40\n"
+ "add x27, x27, #0x40\n"
+ "105:" // Height 3: Writeback done
+ "subs x15, x15, #0x10\n"
+ "bgt 73b\n"
+ "b 212f\n"
+ "106:" // Height 4
+ "ldr x15, [%x[args_ptr], %[offsetof_N]]\n"
+ "ldr x14, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 107f\n"
+ "ldr x13, [%x[output_ptr], #0x0]\n"
+ "add x13, x13, x19, LSL #2\n"
+ "ldr x9, [%x[output_ptr], #0x8]\n"
+ "ldr x27, [%x[output_ptr], #0x10]\n"
+ "add x9, x9, x19, LSL #2\n"
+ "ldr x25, [%x[output_ptr], #0x18]\n"
+ "add x27, x27, x19, LSL #2\n"
+ "add x25, x25, x19, LSL #2\n"
+ "b 108f\n"
+ "107:" // Height 4: setup direct output
+ "mov x13, %x[output_ptr]\n"
+ "add x9, x13, x19, LSL #2\n"
+ "add x27, x9, x19, LSL #2\n"
+ "add x25, x27, x19, LSL #2\n"
+ "108:" // Height 4: Column loop
+ "tbz %x[flags], #0, 118f\n"
+ "cmp x15, #0x10\n"
+ "bge 117f\n"
+ "tbz x15, #3, 112f\n"
+ "ld1 { v8.4s }, [x13], #0x10\n"
+ "ld1 { v12.4s }, [x9], #0x10\n"
+ "ld1 { v16.4s }, [x27], #0x10\n"
+ "ld1 { v20.4s }, [x25], #0x10\n"
+ "ld1 { v9.4s }, [x13], #0x10\n"
+ "ld1 { v13.4s }, [x9], #0x10\n"
+ "ld1 { v17.4s }, [x27], #0x10\n"
+ "ld1 { v21.4s }, [x25], #0x10\n"
+ "tbz x15, #2, 110f\n"
+ "ld1 { v10.4s }, [x13], #0x10\n"
+ "ld1 { v14.4s }, [x9], #0x10\n"
+ "ld1 { v18.4s }, [x27], #0x10\n"
+ "ld1 { v22.4s }, [x25], #0x10\n"
+ "tbz x15, #1, 109f\n"
+ "mov x19, #0x38\n"
+ "ldr d11, [x13], #0x8\n"
+ "ldr d15, [x9], #0x8\n"
+ "ldr d19, [x27], #0x8\n"
+ "ldr d23, [x25], #0x8\n"
+ "tbz x15, #0, 116f\n"
+ "ld1 { v11.s }[2], [x13]\n"
+ "ld1 { v15.s }[2], [x9]\n"
+ "ld1 { v19.s }[2], [x27]\n"
+ "ld1 { v23.s }[2], [x25]\n"
+ "b 116f\n"
+ "109:" // Height 4: Partial accumulate: partial_1_12
+ "mov x19, #0x30\n"
+ "tbz x15, #0, 116f\n"
+ "ldr s11, [x13, #0x0]\n"
+ "ldr s15, [x9, #0x0]\n"
+ "ldr s19, [x27, #0x0]\n"
+ "ldr s23, [x25, #0x0]\n"
+ "b 116f\n"
+ "110:" // Height 4: Partial accumulate: partial_2_8
+ "tbz x15, #1, 111f\n"
+ "ldr d10, [x13], #0x8\n"
+ "ldr d14, [x9], #0x8\n"
+ "ldr d18, [x27], #0x8\n"
+ "ldr d22, [x25], #0x8\n"
+ "mov x19, #0x28\n"
+ "tbz x15, #0, 116f\n"
+ "ld1 { v10.s }[2], [x13]\n"
+ "ld1 { v14.s }[2], [x9]\n"
+ "ld1 { v18.s }[2], [x27]\n"
+ "ld1 { v22.s }[2], [x25]\n"
+ "b 116f\n"
+ "111:" // Height 4: Partial accumulate: partial_1_8
+ "mov x19, #0x20\n"
+ "tbz x15, #0, 116f\n"
+ "ldr s10, [x13, #0x0]\n"
+ "ldr s14, [x9, #0x0]\n"
+ "ldr s18, [x27, #0x0]\n"
+ "ldr s22, [x25, #0x0]\n"
+ "b 116f\n"
+ "112:" // Height 4: Partial accumulate: partial_4_0
+ "tbz x15, #2, 114f\n"
+ "ld1 { v8.4s }, [x13], #0x10\n"
+ "ld1 { v12.4s }, [x9], #0x10\n"
+ "ld1 { v16.4s }, [x27], #0x10\n"
+ "ld1 { v20.4s }, [x25], #0x10\n"
+ "tbz x15, #1, 113f\n"
+ "mov x19, #0x18\n"
+ "ldr d9, [x13], #0x8\n"
+ "ldr d13, [x9], #0x8\n"
+ "ldr d17, [x27], #0x8\n"
+ "ldr d21, [x25], #0x8\n"
+ "tbz x15, #0, 116f\n"
+ "ld1 { v9.s }[2], [x13]\n"
+ "ld1 { v13.s }[2], [x9]\n"
+ "ld1 { v17.s }[2], [x27]\n"
+ "ld1 { v21.s }[2], [x25]\n"
+ "b 116f\n"
+ "113:" // Height 4: Partial accumulate: partial_1_4
+ "mov x19, #0x10\n"
+ "tbz x15, #0, 116f\n"
+ "ldr s9, [x13, #0x0]\n"
+ "ldr s13, [x9, #0x0]\n"
+ "ldr s17, [x27, #0x0]\n"
+ "ldr s21, [x25, #0x0]\n"
+ "b 116f\n"
+ "114:" // Height 4: Partial accumulate: partial_2_0
+ "tbz x15, #1, 115f\n"
+ "ldr d8, [x13], #0x8\n"
+ "ldr d12, [x9], #0x8\n"
+ "ldr d16, [x27], #0x8\n"
+ "ldr d20, [x25], #0x8\n"
+ "mov x19, #0x8\n"
+ "tbz x15, #0, 116f\n"
+ "ld1 { v8.s }[2], [x13]\n"
+ "ld1 { v12.s }[2], [x9]\n"
+ "ld1 { v16.s }[2], [x27]\n"
+ "ld1 { v20.s }[2], [x25]\n"
+ "b 116f\n"
+ "115:" // Height 4: Partial accumulate: partial_1_0
+ "mov x19, #0x0\n"
+ "ldr s8, [x13, #0x0]\n"
+ "ldr s12, [x9, #0x0]\n"
+ "ldr s16, [x27, #0x0]\n"
+ "ldr s20, [x25, #0x0]\n"
+ "116:" // Height 4: Partial accumulate: Done
+ "sub x13, x13, x19\n"
+ "sub x9, x9, x19\n"
+ "sub x27, x27, x19\n"
+ "sub x25, x25, x19\n"
+ "b 119f\n"
+ "117:" // Height 4: full accumulate
+ "ldr q8, [x13, #0x0]\n"
+ "ldr q9, [x13, #0x10]\n"
+ "ldr q10, [x13, #0x20]\n"
+ "ldr q11, [x13, #0x30]\n"
+ "ldr q12, [x9, #0x0]\n"
+ "ldr q13, [x9, #0x10]\n"
+ "ldr q14, [x9, #0x20]\n"
+ "ldr q15, [x9, #0x30]\n"
+ "ldr q16, [x27, #0x0]\n"
+ "ldr q17, [x27, #0x10]\n"
+ "ldr q18, [x27, #0x20]\n"
+ "ldr q19, [x27, #0x30]\n"
+ "ldr q20, [x25, #0x0]\n"
+ "ldr q21, [x25, #0x10]\n"
+ "ldr q22, [x25, #0x20]\n"
+ "ldr q23, [x25, #0x30]\n"
+ "b 119f\n"
+ "118:" // Height 4: no accumulate
+ "movi v8.4s, #0x0\n"
+ "movi v9.4s, #0x0\n"
+ "movi v10.4s, #0x0\n"
+ "movi v11.4s, #0x0\n"
+ "movi v12.4s, #0x0\n"
+ "movi v13.4s, #0x0\n"
+ "movi v14.4s, #0x0\n"
+ "movi v15.4s, #0x0\n"
+ "movi v16.4s, #0x0\n"
+ "movi v17.4s, #0x0\n"
+ "movi v18.4s, #0x0\n"
+ "movi v19.4s, #0x0\n"
+ "movi v20.4s, #0x0\n"
+ "movi v21.4s, #0x0\n"
+ "movi v22.4s, #0x0\n"
+ "movi v23.4s, #0x0\n"
+ "119:" // Height 4: setup done
+ "mov x12, #0x0\n"
+ "120:" // Height 4: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w11, [x20, x12, LSL #0x2]\n"
+ "tbz %x[flags], #3, 121f\n"
+ "ldr x20, [%x[input_ptr], x12, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x10, [x20, #0x0]\n"
+ "ldr x28, [x20, #0x8]\n"
+ "ldr x26, [x20, #0x10]\n"
+ "ldr x24, [x20, #0x18]\n"
+ "cbnz x12, 122f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x10, x10, x19\n"
+ "add x28, x28, x19\n"
+ "add x26, x26, x19\n"
+ "add x24, x24, x19\n"
+ "b 122f\n"
+ "121:" // Height 4: setup direct input
+ "mov x10, %x[input_ptr]\n"
+ "add x28, x10, x19\n"
+ "add x26, x28, x19\n"
+ "add x24, x26, x19\n"
+ "122:" // Height 4: input setup done
+ "cmp x11, #0x10\n"
+ "blt 125f\n"
+ "cmp x11, #0x20\n"
+ "blt 124f\n"
+ "123:" // Height 4: Multiply loop: Main loop head
+ "ldr q0, [x10, #0x0]\n"
+ "ldr q1, [x28, #0x0]\n"
+ "ldr q2, [x26, #0x0]\n"
+ "ldr q3, [x24, #0x0]\n"
+ "ldr q6, [x14, #0x0]\n"
+ ".inst 0x6f80e0c8 // udot v8.4s, v6.16b, v0.4b[0]\n"
+ "ldr q7, [x14, #0x10]\n"
+ ".inst 0x6f81e0cc // udot v12.4s, v6.16b, v1.4b[0]\n"
+ "add x10, x10, #0x10\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ ".inst 0x6f82e0d0 // udot v16.4s, v6.16b, v2.4b[0]\n"
+ "add x28, x28, #0x10\n"
+ ".inst 0x6f83e0d4 // udot v20.4s, v6.16b, v3.4b[0]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "ldr q6, [x14, #0x20]\n"
+ ".inst 0x6f80e0e9 // udot v9.4s, v7.16b, v0.4b[0]\n"
+ "add x26, x26, #0x10\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ ".inst 0x6f81e0ed // udot v13.4s, v7.16b, v1.4b[0]\n"
+ "add x24, x24, #0x10\n"
+ ".inst 0x6f82e0f1 // udot v17.4s, v7.16b, v2.4b[0]\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "sub x11, x11, #0x10\n"
+ ".inst 0x6f83e0f5 // udot v21.4s, v7.16b, v3.4b[0]\n"
+ "ldr q7, [x14, #0x30]\n"
+ ".inst 0x6f80e0ca // udot v10.4s, v6.16b, v0.4b[0]\n"
+ "cmp x11, #0x20\n"
+ ".inst 0x6f81e0ce // udot v14.4s, v6.16b, v1.4b[0]\n"
+ ".inst 0x6f82e0d2 // udot v18.4s, v6.16b, v2.4b[0]\n"
+ ".inst 0x6f83e0d6 // udot v22.4s, v6.16b, v3.4b[0]\n"
+ "ldr q6, [x14, #0x40]\n"
+ ".inst 0x6f80e0eb // udot v11.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x6f81e0ef // udot v15.4s, v7.16b, v1.4b[0]\n"
+ ".inst 0x6f82e0f3 // udot v19.4s, v7.16b, v2.4b[0]\n"
+ ".inst 0x6f83e0f7 // udot v23.4s, v7.16b, v3.4b[0]\n"
+ "ldr q7, [x14, #0x50]\n"
+ ".inst 0x6fa0e0c8 // udot v8.4s, v6.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e0cc // udot v12.4s, v6.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e0d0 // udot v16.4s, v6.16b, v2.4b[1]\n"
+ ".inst 0x6fa3e0d4 // udot v20.4s, v6.16b, v3.4b[1]\n"
+ "ldr q6, [x14, #0x60]\n"
+ ".inst 0x6fa0e0e9 // udot v9.4s, v7.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e0ed // udot v13.4s, v7.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e0f1 // udot v17.4s, v7.16b, v2.4b[1]\n"
+ ".inst 0x6fa3e0f5 // udot v21.4s, v7.16b, v3.4b[1]\n"
+ "ldr q7, [x14, #0x70]\n"
+ ".inst 0x6fa0e0ca // udot v10.4s, v6.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e0ce // udot v14.4s, v6.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e0d2 // udot v18.4s, v6.16b, v2.4b[1]\n"
+ ".inst 0x6fa3e0d6 // udot v22.4s, v6.16b, v3.4b[1]\n"
+ "ldr q6, [x14, #0x80]\n"
+ ".inst 0x6fa0e0eb // udot v11.4s, v7.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e0ef // udot v15.4s, v7.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e0f3 // udot v19.4s, v7.16b, v2.4b[1]\n"
+ ".inst 0x6fa3e0f7 // udot v23.4s, v7.16b, v3.4b[1]\n"
+ "ldr q7, [x14, #0x90]\n"
+ ".inst 0x6f80e8c8 // udot v8.4s, v6.16b, v0.4b[2]\n"
+ ".inst 0x6f81e8cc // udot v12.4s, v6.16b, v1.4b[2]\n"
+ ".inst 0x6f82e8d0 // udot v16.4s, v6.16b, v2.4b[2]\n"
+ ".inst 0x6f83e8d4 // udot v20.4s, v6.16b, v3.4b[2]\n"
+ "ldr q6, [x14, #0xa0]\n"
+ ".inst 0x6f80e8e9 // udot v9.4s, v7.16b, v0.4b[2]\n"
+ ".inst 0x6f81e8ed // udot v13.4s, v7.16b, v1.4b[2]\n"
+ ".inst 0x6f82e8f1 // udot v17.4s, v7.16b, v2.4b[2]\n"
+ ".inst 0x6f83e8f5 // udot v21.4s, v7.16b, v3.4b[2]\n"
+ "ldr q7, [x14, #0xb0]\n"
+ ".inst 0x6f80e8ca // udot v10.4s, v6.16b, v0.4b[2]\n"
+ ".inst 0x6f81e8ce // udot v14.4s, v6.16b, v1.4b[2]\n"
+ ".inst 0x6f82e8d2 // udot v18.4s, v6.16b, v2.4b[2]\n"
+ ".inst 0x6f83e8d6 // udot v22.4s, v6.16b, v3.4b[2]\n"
+ "ldr q6, [x14, #0xc0]\n"
+ ".inst 0x6f80e8eb // udot v11.4s, v7.16b, v0.4b[2]\n"
+ ".inst 0x6f81e8ef // udot v15.4s, v7.16b, v1.4b[2]\n"
+ ".inst 0x6f82e8f3 // udot v19.4s, v7.16b, v2.4b[2]\n"
+ ".inst 0x6f83e8f7 // udot v23.4s, v7.16b, v3.4b[2]\n"
+ "ldr q7, [x14, #0xd0]\n"
+ ".inst 0x6fa0e8c8 // udot v8.4s, v6.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e8cc // udot v12.4s, v6.16b, v1.4b[3]\n"
+ ".inst 0x6fa2e8d0 // udot v16.4s, v6.16b, v2.4b[3]\n"
+ ".inst 0x6fa3e8d4 // udot v20.4s, v6.16b, v3.4b[3]\n"
+ "ldr q6, [x14, #0xe0]\n"
+ ".inst 0x6fa0e8e9 // udot v9.4s, v7.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e8ed // udot v13.4s, v7.16b, v1.4b[3]\n"
+ ".inst 0x6fa2e8f1 // udot v17.4s, v7.16b, v2.4b[3]\n"
+ ".inst 0x6fa3e8f5 // udot v21.4s, v7.16b, v3.4b[3]\n"
+ "ldr q7, [x14, #0xf0]\n"
+ "add x14, x14, #0x100\n"
+ ".inst 0x6fa0e8ca // udot v10.4s, v6.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e8ce // udot v14.4s, v6.16b, v1.4b[3]\n"
+ ".inst 0x6fa2e8d2 // udot v18.4s, v6.16b, v2.4b[3]\n"
+ ".inst 0x6fa3e8d6 // udot v22.4s, v6.16b, v3.4b[3]\n"
+ ".inst 0x6fa0e8eb // udot v11.4s, v7.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e8ef // udot v15.4s, v7.16b, v1.4b[3]\n"
+ ".inst 0x6fa2e8f3 // udot v19.4s, v7.16b, v2.4b[3]\n"
+ ".inst 0x6fa3e8f7 // udot v23.4s, v7.16b, v3.4b[3]\n"
+ "bge 123b\n"
+ "124:" // Height 4: Multiply loop: Single iteration only
+ "sub x11, x11, #0x10\n"
+ "ldr q0, [x10, #0x0]\n"
+ "ldr q1, [x28, #0x0]\n"
+ "ldr q2, [x26, #0x0]\n"
+ "ldr q3, [x24, #0x0]\n"
+ "ldr q6, [x14, #0x0]\n"
+ ".inst 0x6f80e0c8 // udot v8.4s, v6.16b, v0.4b[0]\n"
+ "ldr q7, [x14, #0x10]\n"
+ ".inst 0x6f81e0cc // udot v12.4s, v6.16b, v1.4b[0]\n"
+ "add x10, x10, #0x10\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ ".inst 0x6f82e0d0 // udot v16.4s, v6.16b, v2.4b[0]\n"
+ "add x28, x28, #0x10\n"
+ ".inst 0x6f83e0d4 // udot v20.4s, v6.16b, v3.4b[0]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "ldr q6, [x14, #0x20]\n"
+ ".inst 0x6f80e0e9 // udot v9.4s, v7.16b, v0.4b[0]\n"
+ "add x26, x26, #0x10\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ ".inst 0x6f81e0ed // udot v13.4s, v7.16b, v1.4b[0]\n"
+ "add x24, x24, #0x10\n"
+ ".inst 0x6f82e0f1 // udot v17.4s, v7.16b, v2.4b[0]\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ ".inst 0x6f83e0f5 // udot v21.4s, v7.16b, v3.4b[0]\n"
+ "ldr q7, [x14, #0x30]\n"
+ ".inst 0x6f80e0ca // udot v10.4s, v6.16b, v0.4b[0]\n"
+ ".inst 0x6f81e0ce // udot v14.4s, v6.16b, v1.4b[0]\n"
+ ".inst 0x6f82e0d2 // udot v18.4s, v6.16b, v2.4b[0]\n"
+ ".inst 0x6f83e0d6 // udot v22.4s, v6.16b, v3.4b[0]\n"
+ "ldr q6, [x14, #0x40]\n"
+ ".inst 0x6f80e0eb // udot v11.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x6f81e0ef // udot v15.4s, v7.16b, v1.4b[0]\n"
+ ".inst 0x6f82e0f3 // udot v19.4s, v7.16b, v2.4b[0]\n"
+ ".inst 0x6f83e0f7 // udot v23.4s, v7.16b, v3.4b[0]\n"
+ "ldr q7, [x14, #0x50]\n"
+ ".inst 0x6fa0e0c8 // udot v8.4s, v6.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e0cc // udot v12.4s, v6.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e0d0 // udot v16.4s, v6.16b, v2.4b[1]\n"
+ ".inst 0x6fa3e0d4 // udot v20.4s, v6.16b, v3.4b[1]\n"
+ "ldr q6, [x14, #0x60]\n"
+ ".inst 0x6fa0e0e9 // udot v9.4s, v7.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e0ed // udot v13.4s, v7.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e0f1 // udot v17.4s, v7.16b, v2.4b[1]\n"
+ ".inst 0x6fa3e0f5 // udot v21.4s, v7.16b, v3.4b[1]\n"
+ "ldr q7, [x14, #0x70]\n"
+ ".inst 0x6fa0e0ca // udot v10.4s, v6.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e0ce // udot v14.4s, v6.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e0d2 // udot v18.4s, v6.16b, v2.4b[1]\n"
+ ".inst 0x6fa3e0d6 // udot v22.4s, v6.16b, v3.4b[1]\n"
+ "ldr q6, [x14, #0x80]\n"
+ ".inst 0x6fa0e0eb // udot v11.4s, v7.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e0ef // udot v15.4s, v7.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e0f3 // udot v19.4s, v7.16b, v2.4b[1]\n"
+ ".inst 0x6fa3e0f7 // udot v23.4s, v7.16b, v3.4b[1]\n"
+ "ldr q7, [x14, #0x90]\n"
+ ".inst 0x6f80e8c8 // udot v8.4s, v6.16b, v0.4b[2]\n"
+ ".inst 0x6f81e8cc // udot v12.4s, v6.16b, v1.4b[2]\n"
+ ".inst 0x6f82e8d0 // udot v16.4s, v6.16b, v2.4b[2]\n"
+ ".inst 0x6f83e8d4 // udot v20.4s, v6.16b, v3.4b[2]\n"
+ "ldr q6, [x14, #0xa0]\n"
+ ".inst 0x6f80e8e9 // udot v9.4s, v7.16b, v0.4b[2]\n"
+ ".inst 0x6f81e8ed // udot v13.4s, v7.16b, v1.4b[2]\n"
+ ".inst 0x6f82e8f1 // udot v17.4s, v7.16b, v2.4b[2]\n"
+ ".inst 0x6f83e8f5 // udot v21.4s, v7.16b, v3.4b[2]\n"
+ "ldr q7, [x14, #0xb0]\n"
+ ".inst 0x6f80e8ca // udot v10.4s, v6.16b, v0.4b[2]\n"
+ ".inst 0x6f81e8ce // udot v14.4s, v6.16b, v1.4b[2]\n"
+ ".inst 0x6f82e8d2 // udot v18.4s, v6.16b, v2.4b[2]\n"
+ ".inst 0x6f83e8d6 // udot v22.4s, v6.16b, v3.4b[2]\n"
+ "ldr q6, [x14, #0xc0]\n"
+ ".inst 0x6f80e8eb // udot v11.4s, v7.16b, v0.4b[2]\n"
+ ".inst 0x6f81e8ef // udot v15.4s, v7.16b, v1.4b[2]\n"
+ ".inst 0x6f82e8f3 // udot v19.4s, v7.16b, v2.4b[2]\n"
+ ".inst 0x6f83e8f7 // udot v23.4s, v7.16b, v3.4b[2]\n"
+ "ldr q7, [x14, #0xd0]\n"
+ ".inst 0x6fa0e8c8 // udot v8.4s, v6.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e8cc // udot v12.4s, v6.16b, v1.4b[3]\n"
+ ".inst 0x6fa2e8d0 // udot v16.4s, v6.16b, v2.4b[3]\n"
+ ".inst 0x6fa3e8d4 // udot v20.4s, v6.16b, v3.4b[3]\n"
+ "ldr q6, [x14, #0xe0]\n"
+ ".inst 0x6fa0e8e9 // udot v9.4s, v7.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e8ed // udot v13.4s, v7.16b, v1.4b[3]\n"
+ ".inst 0x6fa2e8f1 // udot v17.4s, v7.16b, v2.4b[3]\n"
+ ".inst 0x6fa3e8f5 // udot v21.4s, v7.16b, v3.4b[3]\n"
+ "ldr q7, [x14, #0xf0]\n"
+ "add x14, x14, #0x100\n"
+ ".inst 0x6fa0e8ca // udot v10.4s, v6.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e8ce // udot v14.4s, v6.16b, v1.4b[3]\n"
+ ".inst 0x6fa2e8d2 // udot v18.4s, v6.16b, v2.4b[3]\n"
+ ".inst 0x6fa3e8d6 // udot v22.4s, v6.16b, v3.4b[3]\n"
+ ".inst 0x6fa0e8eb // udot v11.4s, v7.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e8ef // udot v15.4s, v7.16b, v1.4b[3]\n"
+ ".inst 0x6fa2e8f3 // udot v19.4s, v7.16b, v2.4b[3]\n"
+ ".inst 0x6fa3e8f7 // udot v23.4s, v7.16b, v3.4b[3]\n"
+ "125:" // Height 4: Multiply loop: Main loop skip
+ "cbz x11, 130f\n"
+ "cmp x11, #0x4\n"
+ "blt 127f\n"
+ "126:" // Height 4: Multiply loop: Odd block loop
+ "ldr s0, [x10], #0x4\n"
+ "ldr s1, [x28], #0x4\n"
+ "ldr s2, [x26], #0x4\n"
+ "ldr s3, [x24], #0x4\n"
+ "ldr q6, [x14, #0x0]\n"
+ ".inst 0x6f80e0c8 // udot v8.4s, v6.16b, v0.4b[0]\n"
+ "ldr q7, [x14, #0x10]\n"
+ ".inst 0x6f81e0cc // udot v12.4s, v6.16b, v1.4b[0]\n"
+ "sub x11, x11, #0x4\n"
+ ".inst 0x6f82e0d0 // udot v16.4s, v6.16b, v2.4b[0]\n"
+ "cmp x11, #0x4\n"
+ ".inst 0x6f83e0d4 // udot v20.4s, v6.16b, v3.4b[0]\n"
+ "ldr q6, [x14, #0x20]\n"
+ ".inst 0x6f80e0e9 // udot v9.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x6f81e0ed // udot v13.4s, v7.16b, v1.4b[0]\n"
+ ".inst 0x6f82e0f1 // udot v17.4s, v7.16b, v2.4b[0]\n"
+ ".inst 0x6f83e0f5 // udot v21.4s, v7.16b, v3.4b[0]\n"
+ "ldr q7, [x14, #0x30]\n"
+ ".inst 0x6f80e0ca // udot v10.4s, v6.16b, v0.4b[0]\n"
+ "add x14, x14, #0x40\n"
+ ".inst 0x6f81e0ce // udot v14.4s, v6.16b, v1.4b[0]\n"
+ ".inst 0x6f82e0d2 // udot v18.4s, v6.16b, v2.4b[0]\n"
+ ".inst 0x6f83e0d6 // udot v22.4s, v6.16b, v3.4b[0]\n"
+ ".inst 0x6f80e0eb // udot v11.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x6f81e0ef // udot v15.4s, v7.16b, v1.4b[0]\n"
+ ".inst 0x6f82e0f3 // udot v19.4s, v7.16b, v2.4b[0]\n"
+ ".inst 0x6f83e0f7 // udot v23.4s, v7.16b, v3.4b[0]\n"
+ "bge 126b\n"
+ "cbz x11, 130f\n"
+ "127:" // Height 4: Multiply loop: Skip odd blocks
+ "tbz x11, #1, 128f\n"
+ "ldr h0, [x10], #0x2\n"
+ "ldr h1, [x28], #0x2\n"
+ "ldr h2, [x26], #0x2\n"
+ "ldr h3, [x24], #0x2\n"
+ "tbz x11, #0, 129f\n"
+ "ld1 { v0.b }[2], [x10]\n"
+ "ld1 { v1.b }[2], [x28]\n"
+ "ld1 { v2.b }[2], [x26]\n"
+ "ld1 { v3.b }[2], [x24]\n"
+ "b 129f\n"
+ "128:" // Height 4: Multiply loop: Ragged operand read: partial_1_0
+ "ldr b0, [x10, #0x0]\n"
+ "ldr b1, [x28, #0x0]\n"
+ "ldr b2, [x26, #0x0]\n"
+ "ldr b3, [x24, #0x0]\n"
+ "129:" // Height 4: Multiply loop: Ragged operand read: Done
+ "ldr q6, [x14, #0x0]\n"
+ ".inst 0x6f80e0c8 // udot v8.4s, v6.16b, v0.4b[0]\n"
+ "ldr q7, [x14, #0x10]\n"
+ ".inst 0x6f81e0cc // udot v12.4s, v6.16b, v1.4b[0]\n"
+ ".inst 0x6f82e0d0 // udot v16.4s, v6.16b, v2.4b[0]\n"
+ ".inst 0x6f83e0d4 // udot v20.4s, v6.16b, v3.4b[0]\n"
+ "ldr q6, [x14, #0x20]\n"
+ ".inst 0x6f80e0e9 // udot v9.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x6f81e0ed // udot v13.4s, v7.16b, v1.4b[0]\n"
+ ".inst 0x6f82e0f1 // udot v17.4s, v7.16b, v2.4b[0]\n"
+ ".inst 0x6f83e0f5 // udot v21.4s, v7.16b, v3.4b[0]\n"
+ "ldr q7, [x14, #0x30]\n"
+ ".inst 0x6f80e0ca // udot v10.4s, v6.16b, v0.4b[0]\n"
+ "add x14, x14, #0x40\n"
+ ".inst 0x6f81e0ce // udot v14.4s, v6.16b, v1.4b[0]\n"
+ ".inst 0x6f82e0d2 // udot v18.4s, v6.16b, v2.4b[0]\n"
+ ".inst 0x6f83e0d6 // udot v22.4s, v6.16b, v3.4b[0]\n"
+ ".inst 0x6f80e0eb // udot v11.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x6f81e0ef // udot v15.4s, v7.16b, v1.4b[0]\n"
+ ".inst 0x6f82e0f3 // udot v19.4s, v7.16b, v2.4b[0]\n"
+ ".inst 0x6f83e0f7 // udot v23.4s, v7.16b, v3.4b[0]\n"
+ "130:" // Height 4: Multiply loop: No odd multiplies
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "add x12, x12, #0x1\n"
+ "cmp x12, x19\n"
+ "bne 120b\n"
+ "prfm pstl1keep, [x13, #0x0]\n"
+ "prfm pstl1keep, [x9, #0x0]\n"
+ "cmp x15, #0x10\n"
+ "prfm pstl1keep, [x27, #0x0]\n"
+ "prfm pstl1keep, [x25, #0x0]\n"
+ "bge 139f\n"
+ "tbz x15, #3, 134f\n"
+ "st1 { v8.4s }, [x13], #0x10\n"
+ "st1 { v9.4s }, [x13], #0x10\n"
+ "st1 { v12.4s }, [x9], #0x10\n"
+ "st1 { v13.4s }, [x9], #0x10\n"
+ "st1 { v16.4s }, [x27], #0x10\n"
+ "st1 { v17.4s }, [x27], #0x10\n"
+ "st1 { v20.4s }, [x25], #0x10\n"
+ "st1 { v21.4s }, [x25], #0x10\n"
+ "tbz x15, #2, 132f\n"
+ "st1 { v10.4s }, [x13], #0x10\n"
+ "st1 { v14.4s }, [x9], #0x10\n"
+ "st1 { v18.4s }, [x27], #0x10\n"
+ "st1 { v22.4s }, [x25], #0x10\n"
+ "tbz x15, #1, 131f\n"
+ "str d11, [x13], #0x8\n"
+ "str d15, [x9], #0x8\n"
+ "str d19, [x27], #0x8\n"
+ "str d23, [x25], #0x8\n"
+ "tbz x15, #0, 138f\n"
+ "st1 { v11.s }[2], [x13]\n"
+ "st1 { v15.s }[2], [x9]\n"
+ "st1 { v19.s }[2], [x27]\n"
+ "st1 { v23.s }[2], [x25]\n"
+ "b 138f\n"
+ "131:" // Height 4: Partial direct writeback: partial_1_12
+ "tbz x15, #0, 138f\n"
+ "str s11, [x13, #0x0]\n"
+ "str s15, [x9, #0x0]\n"
+ "str s19, [x27, #0x0]\n"
+ "str s23, [x25, #0x0]\n"
+ "b 138f\n"
+ "132:" // Height 4: Partial direct writeback: partial_2_8
+ "tbz x15, #1, 133f\n"
+ "str d10, [x13], #0x8\n"
+ "str d14, [x9], #0x8\n"
+ "str d18, [x27], #0x8\n"
+ "str d22, [x25], #0x8\n"
+ "tbz x15, #0, 138f\n"
+ "st1 { v10.s }[2], [x13]\n"
+ "st1 { v14.s }[2], [x9]\n"
+ "st1 { v18.s }[2], [x27]\n"
+ "st1 { v22.s }[2], [x25]\n"
+ "b 138f\n"
+ "133:" // Height 4: Partial direct writeback: partial_1_8
+ "tbz x15, #0, 138f\n"
+ "str s10, [x13, #0x0]\n"
+ "str s14, [x9, #0x0]\n"
+ "str s18, [x27, #0x0]\n"
+ "str s22, [x25, #0x0]\n"
+ "b 138f\n"
+ "134:" // Height 4: Partial direct writeback: partial_4_0
+ "tbz x15, #2, 136f\n"
+ "st1 { v8.4s }, [x13], #0x10\n"
+ "st1 { v12.4s }, [x9], #0x10\n"
+ "st1 { v16.4s }, [x27], #0x10\n"
+ "st1 { v20.4s }, [x25], #0x10\n"
+ "tbz x15, #1, 135f\n"
+ "str d9, [x13], #0x8\n"
+ "str d13, [x9], #0x8\n"
+ "str d17, [x27], #0x8\n"
+ "str d21, [x25], #0x8\n"
+ "tbz x15, #0, 138f\n"
+ "st1 { v9.s }[2], [x13]\n"
+ "st1 { v13.s }[2], [x9]\n"
+ "st1 { v17.s }[2], [x27]\n"
+ "st1 { v21.s }[2], [x25]\n"
+ "b 138f\n"
+ "135:" // Height 4: Partial direct writeback: partial_1_4
+ "tbz x15, #0, 138f\n"
+ "str s9, [x13, #0x0]\n"
+ "str s13, [x9, #0x0]\n"
+ "str s17, [x27, #0x0]\n"
+ "str s21, [x25, #0x0]\n"
+ "b 138f\n"
+ "136:" // Height 4: Partial direct writeback: partial_2_0
+ "tbz x15, #1, 137f\n"
+ "str d8, [x13], #0x8\n"
+ "str d12, [x9], #0x8\n"
+ "str d16, [x27], #0x8\n"
+ "str d20, [x25], #0x8\n"
+ "tbz x15, #0, 138f\n"
+ "st1 { v8.s }[2], [x13]\n"
+ "st1 { v12.s }[2], [x9]\n"
+ "st1 { v16.s }[2], [x27]\n"
+ "st1 { v20.s }[2], [x25]\n"
+ "b 138f\n"
+ "137:" // Height 4: Partial direct writeback: partial_1_0
+ "str s8, [x13, #0x0]\n"
+ "str s12, [x9, #0x0]\n"
+ "str s16, [x27, #0x0]\n"
+ "str s20, [x25, #0x0]\n"
+ "138:" // Height 4: Partial direct writeback: Done
+ "b 140f\n"
+ "139:" // Height 4: Full writeback
+ "str q8, [x13, #0x0]\n"
+ "str q9, [x13, #0x10]\n"
+ "str q10, [x13, #0x20]\n"
+ "str q11, [x13, #0x30]\n"
+ "str q12, [x9, #0x0]\n"
+ "str q13, [x9, #0x10]\n"
+ "str q14, [x9, #0x20]\n"
+ "str q15, [x9, #0x30]\n"
+ "str q16, [x27, #0x0]\n"
+ "str q17, [x27, #0x10]\n"
+ "str q18, [x27, #0x20]\n"
+ "str q19, [x27, #0x30]\n"
+ "str q20, [x25, #0x0]\n"
+ "str q21, [x25, #0x10]\n"
+ "str q22, [x25, #0x20]\n"
+ "str q23, [x25, #0x30]\n"
+ "add x13, x13, #0x40\n"
+ "add x9, x9, #0x40\n"
+ "add x27, x27, #0x40\n"
+ "add x25, x25, #0x40\n"
+ "140:" // Height 4: Writeback done
+ "subs x15, x15, #0x10\n"
+ "bgt 108b\n"
+ "b 212f\n"
+ "141:" // Height 5
+ "ldr x15, [%x[args_ptr], %[offsetof_N]]\n"
+ "ldr x14, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 142f\n"
+ "ldr x13, [%x[output_ptr], #0x0]\n"
+ "add x13, x13, x19, LSL #2\n"
+ "ldr x9, [%x[output_ptr], #0x8]\n"
+ "ldr x27, [%x[output_ptr], #0x10]\n"
+ "add x9, x9, x19, LSL #2\n"
+ "ldr x25, [%x[output_ptr], #0x18]\n"
+ "ldr x23, [%x[output_ptr], #0x20]\n"
+ "add x27, x27, x19, LSL #2\n"
+ "add x25, x25, x19, LSL #2\n"
+ "add x23, x23, x19, LSL #2\n"
+ "b 143f\n"
+ "142:" // Height 5: setup direct output
+ "mov x13, %x[output_ptr]\n"
+ "add x9, x13, x19, LSL #2\n"
+ "add x27, x9, x19, LSL #2\n"
+ "add x25, x27, x19, LSL #2\n"
+ "add x23, x25, x19, LSL #2\n"
+ "143:" // Height 5: Column loop
+ "tbz %x[flags], #0, 153f\n"
+ "cmp x15, #0x10\n"
+ "bge 152f\n"
+ "tbz x15, #3, 147f\n"
+ "ld1 { v8.4s }, [x13], #0x10\n"
+ "ld1 { v12.4s }, [x9], #0x10\n"
+ "ld1 { v16.4s }, [x27], #0x10\n"
+ "ld1 { v20.4s }, [x25], #0x10\n"
+ "ld1 { v24.4s }, [x23], #0x10\n"
+ "ld1 { v9.4s }, [x13], #0x10\n"
+ "ld1 { v13.4s }, [x9], #0x10\n"
+ "ld1 { v17.4s }, [x27], #0x10\n"
+ "ld1 { v21.4s }, [x25], #0x10\n"
+ "ld1 { v25.4s }, [x23], #0x10\n"
+ "tbz x15, #2, 145f\n"
+ "ld1 { v10.4s }, [x13], #0x10\n"
+ "ld1 { v14.4s }, [x9], #0x10\n"
+ "ld1 { v18.4s }, [x27], #0x10\n"
+ "ld1 { v22.4s }, [x25], #0x10\n"
+ "ld1 { v26.4s }, [x23], #0x10\n"
+ "tbz x15, #1, 144f\n"
+ "mov x19, #0x38\n"
+ "ldr d11, [x13], #0x8\n"
+ "ldr d15, [x9], #0x8\n"
+ "ldr d19, [x27], #0x8\n"
+ "ldr d23, [x25], #0x8\n"
+ "ldr d27, [x23], #0x8\n"
+ "tbz x15, #0, 151f\n"
+ "ld1 { v11.s }[2], [x13]\n"
+ "ld1 { v15.s }[2], [x9]\n"
+ "ld1 { v19.s }[2], [x27]\n"
+ "ld1 { v23.s }[2], [x25]\n"
+ "ld1 { v27.s }[2], [x23]\n"
+ "b 151f\n"
+ "144:" // Height 5: Partial accumulate: partial_1_12
+ "mov x19, #0x30\n"
+ "tbz x15, #0, 151f\n"
+ "ldr s11, [x13, #0x0]\n"
+ "ldr s15, [x9, #0x0]\n"
+ "ldr s19, [x27, #0x0]\n"
+ "ldr s23, [x25, #0x0]\n"
+ "ldr s27, [x23, #0x0]\n"
+ "b 151f\n"
+ "145:" // Height 5: Partial accumulate: partial_2_8
+ "tbz x15, #1, 146f\n"
+ "ldr d10, [x13], #0x8\n"
+ "ldr d14, [x9], #0x8\n"
+ "ldr d18, [x27], #0x8\n"
+ "ldr d22, [x25], #0x8\n"
+ "ldr d26, [x23], #0x8\n"
+ "mov x19, #0x28\n"
+ "tbz x15, #0, 151f\n"
+ "ld1 { v10.s }[2], [x13]\n"
+ "ld1 { v14.s }[2], [x9]\n"
+ "ld1 { v18.s }[2], [x27]\n"
+ "ld1 { v22.s }[2], [x25]\n"
+ "ld1 { v26.s }[2], [x23]\n"
+ "b 151f\n"
+ "146:" // Height 5: Partial accumulate: partial_1_8
+ "mov x19, #0x20\n"
+ "tbz x15, #0, 151f\n"
+ "ldr s10, [x13, #0x0]\n"
+ "ldr s14, [x9, #0x0]\n"
+ "ldr s18, [x27, #0x0]\n"
+ "ldr s22, [x25, #0x0]\n"
+ "ldr s26, [x23, #0x0]\n"
+ "b 151f\n"
+ "147:" // Height 5: Partial accumulate: partial_4_0
+ "tbz x15, #2, 149f\n"
+ "ld1 { v8.4s }, [x13], #0x10\n"
+ "ld1 { v12.4s }, [x9], #0x10\n"
+ "ld1 { v16.4s }, [x27], #0x10\n"
+ "ld1 { v20.4s }, [x25], #0x10\n"
+ "ld1 { v24.4s }, [x23], #0x10\n"
+ "tbz x15, #1, 148f\n"
+ "mov x19, #0x18\n"
+ "ldr d9, [x13], #0x8\n"
+ "ldr d13, [x9], #0x8\n"
+ "ldr d17, [x27], #0x8\n"
+ "ldr d21, [x25], #0x8\n"
+ "ldr d25, [x23], #0x8\n"
+ "tbz x15, #0, 151f\n"
+ "ld1 { v9.s }[2], [x13]\n"
+ "ld1 { v13.s }[2], [x9]\n"
+ "ld1 { v17.s }[2], [x27]\n"
+ "ld1 { v21.s }[2], [x25]\n"
+ "ld1 { v25.s }[2], [x23]\n"
+ "b 151f\n"
+ "148:" // Height 5: Partial accumulate: partial_1_4
+ "mov x19, #0x10\n"
+ "tbz x15, #0, 151f\n"
+ "ldr s9, [x13, #0x0]\n"
+ "ldr s13, [x9, #0x0]\n"
+ "ldr s17, [x27, #0x0]\n"
+ "ldr s21, [x25, #0x0]\n"
+ "ldr s25, [x23, #0x0]\n"
+ "b 151f\n"
+ "149:" // Height 5: Partial accumulate: partial_2_0
+ "tbz x15, #1, 150f\n"
+ "ldr d8, [x13], #0x8\n"
+ "ldr d12, [x9], #0x8\n"
+ "ldr d16, [x27], #0x8\n"
+ "ldr d20, [x25], #0x8\n"
+ "ldr d24, [x23], #0x8\n"
+ "mov x19, #0x8\n"
+ "tbz x15, #0, 151f\n"
+ "ld1 { v8.s }[2], [x13]\n"
+ "ld1 { v12.s }[2], [x9]\n"
+ "ld1 { v16.s }[2], [x27]\n"
+ "ld1 { v20.s }[2], [x25]\n"
+ "ld1 { v24.s }[2], [x23]\n"
+ "b 151f\n"
+ "150:" // Height 5: Partial accumulate: partial_1_0
+ "mov x19, #0x0\n"
+ "ldr s8, [x13, #0x0]\n"
+ "ldr s12, [x9, #0x0]\n"
+ "ldr s16, [x27, #0x0]\n"
+ "ldr s20, [x25, #0x0]\n"
+ "ldr s24, [x23, #0x0]\n"
+ "151:" // Height 5: Partial accumulate: Done
+ "sub x13, x13, x19\n"
+ "sub x9, x9, x19\n"
+ "sub x27, x27, x19\n"
+ "sub x25, x25, x19\n"
+ "sub x23, x23, x19\n"
+ "b 154f\n"
+ "152:" // Height 5: full accumulate
+ "ldr q8, [x13, #0x0]\n"
+ "ldr q9, [x13, #0x10]\n"
+ "ldr q10, [x13, #0x20]\n"
+ "ldr q11, [x13, #0x30]\n"
+ "ldr q12, [x9, #0x0]\n"
+ "ldr q13, [x9, #0x10]\n"
+ "ldr q14, [x9, #0x20]\n"
+ "ldr q15, [x9, #0x30]\n"
+ "ldr q16, [x27, #0x0]\n"
+ "ldr q17, [x27, #0x10]\n"
+ "ldr q18, [x27, #0x20]\n"
+ "ldr q19, [x27, #0x30]\n"
+ "ldr q20, [x25, #0x0]\n"
+ "ldr q21, [x25, #0x10]\n"
+ "ldr q22, [x25, #0x20]\n"
+ "ldr q23, [x25, #0x30]\n"
+ "ldr q24, [x23, #0x0]\n"
+ "ldr q25, [x23, #0x10]\n"
+ "ldr q26, [x23, #0x20]\n"
+ "ldr q27, [x23, #0x30]\n"
+ "b 154f\n"
+ "153:" // Height 5: no accumulate
+ "movi v8.4s, #0x0\n"
+ "movi v9.4s, #0x0\n"
+ "movi v10.4s, #0x0\n"
+ "movi v11.4s, #0x0\n"
+ "movi v12.4s, #0x0\n"
+ "movi v13.4s, #0x0\n"
+ "movi v14.4s, #0x0\n"
+ "movi v15.4s, #0x0\n"
+ "movi v16.4s, #0x0\n"
+ "movi v17.4s, #0x0\n"
+ "movi v18.4s, #0x0\n"
+ "movi v19.4s, #0x0\n"
+ "movi v20.4s, #0x0\n"
+ "movi v21.4s, #0x0\n"
+ "movi v22.4s, #0x0\n"
+ "movi v23.4s, #0x0\n"
+ "movi v24.4s, #0x0\n"
+ "movi v25.4s, #0x0\n"
+ "movi v26.4s, #0x0\n"
+ "movi v27.4s, #0x0\n"
+ "154:" // Height 5: setup done
+ "mov x12, #0x0\n"
+ "155:" // Height 5: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w11, [x20, x12, LSL #0x2]\n"
+ "tbz %x[flags], #3, 156f\n"
+ "ldr x20, [%x[input_ptr], x12, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x10, [x20, #0x0]\n"
+ "ldr x28, [x20, #0x8]\n"
+ "ldr x26, [x20, #0x10]\n"
+ "ldr x24, [x20, #0x18]\n"
+ "ldr x22, [x20, #0x20]\n"
+ "cbnz x12, 157f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x10, x10, x19\n"
+ "add x28, x28, x19\n"
+ "add x26, x26, x19\n"
+ "add x24, x24, x19\n"
+ "add x22, x22, x19\n"
+ "b 157f\n"
+ "156:" // Height 5: setup direct input
+ "mov x10, %x[input_ptr]\n"
+ "add x28, x10, x19\n"
+ "add x26, x28, x19\n"
+ "add x24, x26, x19\n"
+ "add x22, x24, x19\n"
+ "157:" // Height 5: input setup done
+ "cmp x11, #0x10\n"
+ "blt 160f\n"
+ "cmp x11, #0x20\n"
+ "blt 159f\n"
+ "158:" // Height 5: Multiply loop: Main loop head
+ "ldr q0, [x10, #0x0]\n"
+ "ldr q1, [x28, #0x0]\n"
+ "ldr q2, [x26, #0x0]\n"
+ "ldr q3, [x24, #0x0]\n"
+ "ldr q4, [x22, #0x0]\n"
+ "ldr q6, [x14, #0x0]\n"
+ ".inst 0x6f80e0c8 // udot v8.4s, v6.16b, v0.4b[0]\n"
+ "ldr q7, [x14, #0x10]\n"
+ ".inst 0x6f81e0cc // udot v12.4s, v6.16b, v1.4b[0]\n"
+ "add x10, x10, #0x10\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ ".inst 0x6f82e0d0 // udot v16.4s, v6.16b, v2.4b[0]\n"
+ "add x28, x28, #0x10\n"
+ ".inst 0x6f83e0d4 // udot v20.4s, v6.16b, v3.4b[0]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "add x26, x26, #0x10\n"
+ ".inst 0x6f84e0d8 // udot v24.4s, v6.16b, v4.4b[0]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "ldr q6, [x14, #0x20]\n"
+ ".inst 0x6f80e0e9 // udot v9.4s, v7.16b, v0.4b[0]\n"
+ "add x24, x24, #0x10\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ ".inst 0x6f81e0ed // udot v13.4s, v7.16b, v1.4b[0]\n"
+ "add x22, x22, #0x10\n"
+ ".inst 0x6f82e0f1 // udot v17.4s, v7.16b, v2.4b[0]\n"
+ "prfm pldl1keep, [x22, #0x80]\n"
+ "sub x11, x11, #0x10\n"
+ ".inst 0x6f83e0f5 // udot v21.4s, v7.16b, v3.4b[0]\n"
+ "cmp x11, #0x20\n"
+ ".inst 0x6f84e0f9 // udot v25.4s, v7.16b, v4.4b[0]\n"
+ "ldr q7, [x14, #0x30]\n"
+ ".inst 0x6f80e0ca // udot v10.4s, v6.16b, v0.4b[0]\n"
+ ".inst 0x6f81e0ce // udot v14.4s, v6.16b, v1.4b[0]\n"
+ ".inst 0x6f82e0d2 // udot v18.4s, v6.16b, v2.4b[0]\n"
+ ".inst 0x6f83e0d6 // udot v22.4s, v6.16b, v3.4b[0]\n"
+ ".inst 0x6f84e0da // udot v26.4s, v6.16b, v4.4b[0]\n"
+ "ldr q6, [x14, #0x40]\n"
+ ".inst 0x6f80e0eb // udot v11.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x6f81e0ef // udot v15.4s, v7.16b, v1.4b[0]\n"
+ ".inst 0x6f82e0f3 // udot v19.4s, v7.16b, v2.4b[0]\n"
+ ".inst 0x6f83e0f7 // udot v23.4s, v7.16b, v3.4b[0]\n"
+ ".inst 0x6f84e0fb // udot v27.4s, v7.16b, v4.4b[0]\n"
+ "ldr q7, [x14, #0x50]\n"
+ ".inst 0x6fa0e0c8 // udot v8.4s, v6.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e0cc // udot v12.4s, v6.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e0d0 // udot v16.4s, v6.16b, v2.4b[1]\n"
+ ".inst 0x6fa3e0d4 // udot v20.4s, v6.16b, v3.4b[1]\n"
+ ".inst 0x6fa4e0d8 // udot v24.4s, v6.16b, v4.4b[1]\n"
+ "ldr q6, [x14, #0x60]\n"
+ ".inst 0x6fa0e0e9 // udot v9.4s, v7.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e0ed // udot v13.4s, v7.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e0f1 // udot v17.4s, v7.16b, v2.4b[1]\n"
+ ".inst 0x6fa3e0f5 // udot v21.4s, v7.16b, v3.4b[1]\n"
+ ".inst 0x6fa4e0f9 // udot v25.4s, v7.16b, v4.4b[1]\n"
+ "ldr q7, [x14, #0x70]\n"
+ ".inst 0x6fa0e0ca // udot v10.4s, v6.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e0ce // udot v14.4s, v6.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e0d2 // udot v18.4s, v6.16b, v2.4b[1]\n"
+ ".inst 0x6fa3e0d6 // udot v22.4s, v6.16b, v3.4b[1]\n"
+ ".inst 0x6fa4e0da // udot v26.4s, v6.16b, v4.4b[1]\n"
+ "ldr q6, [x14, #0x80]\n"
+ ".inst 0x6fa0e0eb // udot v11.4s, v7.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e0ef // udot v15.4s, v7.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e0f3 // udot v19.4s, v7.16b, v2.4b[1]\n"
+ ".inst 0x6fa3e0f7 // udot v23.4s, v7.16b, v3.4b[1]\n"
+ ".inst 0x6fa4e0fb // udot v27.4s, v7.16b, v4.4b[1]\n"
+ "ldr q7, [x14, #0x90]\n"
+ ".inst 0x6f80e8c8 // udot v8.4s, v6.16b, v0.4b[2]\n"
+ ".inst 0x6f81e8cc // udot v12.4s, v6.16b, v1.4b[2]\n"
+ ".inst 0x6f82e8d0 // udot v16.4s, v6.16b, v2.4b[2]\n"
+ ".inst 0x6f83e8d4 // udot v20.4s, v6.16b, v3.4b[2]\n"
+ ".inst 0x6f84e8d8 // udot v24.4s, v6.16b, v4.4b[2]\n"
+ "ldr q6, [x14, #0xa0]\n"
+ ".inst 0x6f80e8e9 // udot v9.4s, v7.16b, v0.4b[2]\n"
+ ".inst 0x6f81e8ed // udot v13.4s, v7.16b, v1.4b[2]\n"
+ ".inst 0x6f82e8f1 // udot v17.4s, v7.16b, v2.4b[2]\n"
+ ".inst 0x6f83e8f5 // udot v21.4s, v7.16b, v3.4b[2]\n"
+ ".inst 0x6f84e8f9 // udot v25.4s, v7.16b, v4.4b[2]\n"
+ "ldr q7, [x14, #0xb0]\n"
+ ".inst 0x6f80e8ca // udot v10.4s, v6.16b, v0.4b[2]\n"
+ ".inst 0x6f81e8ce // udot v14.4s, v6.16b, v1.4b[2]\n"
+ ".inst 0x6f82e8d2 // udot v18.4s, v6.16b, v2.4b[2]\n"
+ ".inst 0x6f83e8d6 // udot v22.4s, v6.16b, v3.4b[2]\n"
+ ".inst 0x6f84e8da // udot v26.4s, v6.16b, v4.4b[2]\n"
+ "ldr q6, [x14, #0xc0]\n"
+ ".inst 0x6f80e8eb // udot v11.4s, v7.16b, v0.4b[2]\n"
+ ".inst 0x6f81e8ef // udot v15.4s, v7.16b, v1.4b[2]\n"
+ ".inst 0x6f82e8f3 // udot v19.4s, v7.16b, v2.4b[2]\n"
+ ".inst 0x6f83e8f7 // udot v23.4s, v7.16b, v3.4b[2]\n"
+ ".inst 0x6f84e8fb // udot v27.4s, v7.16b, v4.4b[2]\n"
+ "ldr q7, [x14, #0xd0]\n"
+ ".inst 0x6fa0e8c8 // udot v8.4s, v6.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e8cc // udot v12.4s, v6.16b, v1.4b[3]\n"
+ ".inst 0x6fa2e8d0 // udot v16.4s, v6.16b, v2.4b[3]\n"
+ ".inst 0x6fa3e8d4 // udot v20.4s, v6.16b, v3.4b[3]\n"
+ ".inst 0x6fa4e8d8 // udot v24.4s, v6.16b, v4.4b[3]\n"
+ "ldr q6, [x14, #0xe0]\n"
+ ".inst 0x6fa0e8e9 // udot v9.4s, v7.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e8ed // udot v13.4s, v7.16b, v1.4b[3]\n"
+ ".inst 0x6fa2e8f1 // udot v17.4s, v7.16b, v2.4b[3]\n"
+ ".inst 0x6fa3e8f5 // udot v21.4s, v7.16b, v3.4b[3]\n"
+ ".inst 0x6fa4e8f9 // udot v25.4s, v7.16b, v4.4b[3]\n"
+ "ldr q7, [x14, #0xf0]\n"
+ ".inst 0x6fa0e8ca // udot v10.4s, v6.16b, v0.4b[3]\n"
+ "add x14, x14, #0x100\n"
+ ".inst 0x6fa1e8ce // udot v14.4s, v6.16b, v1.4b[3]\n"
+ ".inst 0x6fa2e8d2 // udot v18.4s, v6.16b, v2.4b[3]\n"
+ ".inst 0x6fa3e8d6 // udot v22.4s, v6.16b, v3.4b[3]\n"
+ ".inst 0x6fa4e8da // udot v26.4s, v6.16b, v4.4b[3]\n"
+ ".inst 0x6fa0e8eb // udot v11.4s, v7.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e8ef // udot v15.4s, v7.16b, v1.4b[3]\n"
+ ".inst 0x6fa2e8f3 // udot v19.4s, v7.16b, v2.4b[3]\n"
+ ".inst 0x6fa3e8f7 // udot v23.4s, v7.16b, v3.4b[3]\n"
+ ".inst 0x6fa4e8fb // udot v27.4s, v7.16b, v4.4b[3]\n"
+ "bge 158b\n"
+ "159:" // Height 5: Multiply loop: Single iteration only
+ "sub x11, x11, #0x10\n"
+ "ldr q0, [x10, #0x0]\n"
+ "ldr q1, [x28, #0x0]\n"
+ "ldr q2, [x26, #0x0]\n"
+ "ldr q3, [x24, #0x0]\n"
+ "ldr q4, [x22, #0x0]\n"
+ "ldr q6, [x14, #0x0]\n"
+ ".inst 0x6f80e0c8 // udot v8.4s, v6.16b, v0.4b[0]\n"
+ "ldr q7, [x14, #0x10]\n"
+ ".inst 0x6f81e0cc // udot v12.4s, v6.16b, v1.4b[0]\n"
+ "add x10, x10, #0x10\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ ".inst 0x6f82e0d0 // udot v16.4s, v6.16b, v2.4b[0]\n"
+ "add x28, x28, #0x10\n"
+ ".inst 0x6f83e0d4 // udot v20.4s, v6.16b, v3.4b[0]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "add x26, x26, #0x10\n"
+ ".inst 0x6f84e0d8 // udot v24.4s, v6.16b, v4.4b[0]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "ldr q6, [x14, #0x20]\n"
+ ".inst 0x6f80e0e9 // udot v9.4s, v7.16b, v0.4b[0]\n"
+ "add x24, x24, #0x10\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ ".inst 0x6f81e0ed // udot v13.4s, v7.16b, v1.4b[0]\n"
+ "add x22, x22, #0x10\n"
+ ".inst 0x6f82e0f1 // udot v17.4s, v7.16b, v2.4b[0]\n"
+ "prfm pldl1keep, [x22, #0x80]\n"
+ ".inst 0x6f83e0f5 // udot v21.4s, v7.16b, v3.4b[0]\n"
+ ".inst 0x6f84e0f9 // udot v25.4s, v7.16b, v4.4b[0]\n"
+ "ldr q7, [x14, #0x30]\n"
+ ".inst 0x6f80e0ca // udot v10.4s, v6.16b, v0.4b[0]\n"
+ ".inst 0x6f81e0ce // udot v14.4s, v6.16b, v1.4b[0]\n"
+ ".inst 0x6f82e0d2 // udot v18.4s, v6.16b, v2.4b[0]\n"
+ ".inst 0x6f83e0d6 // udot v22.4s, v6.16b, v3.4b[0]\n"
+ ".inst 0x6f84e0da // udot v26.4s, v6.16b, v4.4b[0]\n"
+ "ldr q6, [x14, #0x40]\n"
+ ".inst 0x6f80e0eb // udot v11.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x6f81e0ef // udot v15.4s, v7.16b, v1.4b[0]\n"
+ ".inst 0x6f82e0f3 // udot v19.4s, v7.16b, v2.4b[0]\n"
+ ".inst 0x6f83e0f7 // udot v23.4s, v7.16b, v3.4b[0]\n"
+ ".inst 0x6f84e0fb // udot v27.4s, v7.16b, v4.4b[0]\n"
+ "ldr q7, [x14, #0x50]\n"
+ ".inst 0x6fa0e0c8 // udot v8.4s, v6.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e0cc // udot v12.4s, v6.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e0d0 // udot v16.4s, v6.16b, v2.4b[1]\n"
+ ".inst 0x6fa3e0d4 // udot v20.4s, v6.16b, v3.4b[1]\n"
+ ".inst 0x6fa4e0d8 // udot v24.4s, v6.16b, v4.4b[1]\n"
+ "ldr q6, [x14, #0x60]\n"
+ ".inst 0x6fa0e0e9 // udot v9.4s, v7.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e0ed // udot v13.4s, v7.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e0f1 // udot v17.4s, v7.16b, v2.4b[1]\n"
+ ".inst 0x6fa3e0f5 // udot v21.4s, v7.16b, v3.4b[1]\n"
+ ".inst 0x6fa4e0f9 // udot v25.4s, v7.16b, v4.4b[1]\n"
+ "ldr q7, [x14, #0x70]\n"
+ ".inst 0x6fa0e0ca // udot v10.4s, v6.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e0ce // udot v14.4s, v6.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e0d2 // udot v18.4s, v6.16b, v2.4b[1]\n"
+ ".inst 0x6fa3e0d6 // udot v22.4s, v6.16b, v3.4b[1]\n"
+ ".inst 0x6fa4e0da // udot v26.4s, v6.16b, v4.4b[1]\n"
+ "ldr q6, [x14, #0x80]\n"
+ ".inst 0x6fa0e0eb // udot v11.4s, v7.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e0ef // udot v15.4s, v7.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e0f3 // udot v19.4s, v7.16b, v2.4b[1]\n"
+ ".inst 0x6fa3e0f7 // udot v23.4s, v7.16b, v3.4b[1]\n"
+ ".inst 0x6fa4e0fb // udot v27.4s, v7.16b, v4.4b[1]\n"
+ "ldr q7, [x14, #0x90]\n"
+ ".inst 0x6f80e8c8 // udot v8.4s, v6.16b, v0.4b[2]\n"
+ ".inst 0x6f81e8cc // udot v12.4s, v6.16b, v1.4b[2]\n"
+ ".inst 0x6f82e8d0 // udot v16.4s, v6.16b, v2.4b[2]\n"
+ ".inst 0x6f83e8d4 // udot v20.4s, v6.16b, v3.4b[2]\n"
+ ".inst 0x6f84e8d8 // udot v24.4s, v6.16b, v4.4b[2]\n"
+ "ldr q6, [x14, #0xa0]\n"
+ ".inst 0x6f80e8e9 // udot v9.4s, v7.16b, v0.4b[2]\n"
+ ".inst 0x6f81e8ed // udot v13.4s, v7.16b, v1.4b[2]\n"
+ ".inst 0x6f82e8f1 // udot v17.4s, v7.16b, v2.4b[2]\n"
+ ".inst 0x6f83e8f5 // udot v21.4s, v7.16b, v3.4b[2]\n"
+ ".inst 0x6f84e8f9 // udot v25.4s, v7.16b, v4.4b[2]\n"
+ "ldr q7, [x14, #0xb0]\n"
+ ".inst 0x6f80e8ca // udot v10.4s, v6.16b, v0.4b[2]\n"
+ ".inst 0x6f81e8ce // udot v14.4s, v6.16b, v1.4b[2]\n"
+ ".inst 0x6f82e8d2 // udot v18.4s, v6.16b, v2.4b[2]\n"
+ ".inst 0x6f83e8d6 // udot v22.4s, v6.16b, v3.4b[2]\n"
+ ".inst 0x6f84e8da // udot v26.4s, v6.16b, v4.4b[2]\n"
+ "ldr q6, [x14, #0xc0]\n"
+ ".inst 0x6f80e8eb // udot v11.4s, v7.16b, v0.4b[2]\n"
+ ".inst 0x6f81e8ef // udot v15.4s, v7.16b, v1.4b[2]\n"
+ ".inst 0x6f82e8f3 // udot v19.4s, v7.16b, v2.4b[2]\n"
+ ".inst 0x6f83e8f7 // udot v23.4s, v7.16b, v3.4b[2]\n"
+ ".inst 0x6f84e8fb // udot v27.4s, v7.16b, v4.4b[2]\n"
+ "ldr q7, [x14, #0xd0]\n"
+ ".inst 0x6fa0e8c8 // udot v8.4s, v6.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e8cc // udot v12.4s, v6.16b, v1.4b[3]\n"
+ ".inst 0x6fa2e8d0 // udot v16.4s, v6.16b, v2.4b[3]\n"
+ ".inst 0x6fa3e8d4 // udot v20.4s, v6.16b, v3.4b[3]\n"
+ ".inst 0x6fa4e8d8 // udot v24.4s, v6.16b, v4.4b[3]\n"
+ "ldr q6, [x14, #0xe0]\n"
+ ".inst 0x6fa0e8e9 // udot v9.4s, v7.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e8ed // udot v13.4s, v7.16b, v1.4b[3]\n"
+ ".inst 0x6fa2e8f1 // udot v17.4s, v7.16b, v2.4b[3]\n"
+ ".inst 0x6fa3e8f5 // udot v21.4s, v7.16b, v3.4b[3]\n"
+ ".inst 0x6fa4e8f9 // udot v25.4s, v7.16b, v4.4b[3]\n"
+ "ldr q7, [x14, #0xf0]\n"
+ ".inst 0x6fa0e8ca // udot v10.4s, v6.16b, v0.4b[3]\n"
+ "add x14, x14, #0x100\n"
+ ".inst 0x6fa1e8ce // udot v14.4s, v6.16b, v1.4b[3]\n"
+ ".inst 0x6fa2e8d2 // udot v18.4s, v6.16b, v2.4b[3]\n"
+ ".inst 0x6fa3e8d6 // udot v22.4s, v6.16b, v3.4b[3]\n"
+ ".inst 0x6fa4e8da // udot v26.4s, v6.16b, v4.4b[3]\n"
+ ".inst 0x6fa0e8eb // udot v11.4s, v7.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e8ef // udot v15.4s, v7.16b, v1.4b[3]\n"
+ ".inst 0x6fa2e8f3 // udot v19.4s, v7.16b, v2.4b[3]\n"
+ ".inst 0x6fa3e8f7 // udot v23.4s, v7.16b, v3.4b[3]\n"
+ ".inst 0x6fa4e8fb // udot v27.4s, v7.16b, v4.4b[3]\n"
+ "160:" // Height 5: Multiply loop: Main loop skip
+ "cbz x11, 165f\n"
+ "cmp x11, #0x4\n"
+ "blt 162f\n"
+ "161:" // Height 5: Multiply loop: Odd block loop
+ "ldr s0, [x10], #0x4\n"
+ "ldr s1, [x28], #0x4\n"
+ "ldr s2, [x26], #0x4\n"
+ "ldr s3, [x24], #0x4\n"
+ "ldr s4, [x22], #0x4\n"
+ "ldr q6, [x14, #0x0]\n"
+ ".inst 0x6f80e0c8 // udot v8.4s, v6.16b, v0.4b[0]\n"
+ "ldr q7, [x14, #0x10]\n"
+ ".inst 0x6f81e0cc // udot v12.4s, v6.16b, v1.4b[0]\n"
+ "sub x11, x11, #0x4\n"
+ ".inst 0x6f82e0d0 // udot v16.4s, v6.16b, v2.4b[0]\n"
+ "cmp x11, #0x4\n"
+ ".inst 0x6f83e0d4 // udot v20.4s, v6.16b, v3.4b[0]\n"
+ ".inst 0x6f84e0d8 // udot v24.4s, v6.16b, v4.4b[0]\n"
+ "ldr q6, [x14, #0x20]\n"
+ ".inst 0x6f80e0e9 // udot v9.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x6f81e0ed // udot v13.4s, v7.16b, v1.4b[0]\n"
+ ".inst 0x6f82e0f1 // udot v17.4s, v7.16b, v2.4b[0]\n"
+ ".inst 0x6f83e0f5 // udot v21.4s, v7.16b, v3.4b[0]\n"
+ ".inst 0x6f84e0f9 // udot v25.4s, v7.16b, v4.4b[0]\n"
+ "ldr q7, [x14, #0x30]\n"
+ ".inst 0x6f80e0ca // udot v10.4s, v6.16b, v0.4b[0]\n"
+ "add x14, x14, #0x40\n"
+ ".inst 0x6f81e0ce // udot v14.4s, v6.16b, v1.4b[0]\n"
+ ".inst 0x6f82e0d2 // udot v18.4s, v6.16b, v2.4b[0]\n"
+ ".inst 0x6f83e0d6 // udot v22.4s, v6.16b, v3.4b[0]\n"
+ ".inst 0x6f84e0da // udot v26.4s, v6.16b, v4.4b[0]\n"
+ ".inst 0x6f80e0eb // udot v11.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x6f81e0ef // udot v15.4s, v7.16b, v1.4b[0]\n"
+ ".inst 0x6f82e0f3 // udot v19.4s, v7.16b, v2.4b[0]\n"
+ ".inst 0x6f83e0f7 // udot v23.4s, v7.16b, v3.4b[0]\n"
+ ".inst 0x6f84e0fb // udot v27.4s, v7.16b, v4.4b[0]\n"
+ "bge 161b\n"
+ "cbz x11, 165f\n"
+ "162:" // Height 5: Multiply loop: Skip odd blocks
+ "tbz x11, #1, 163f\n"
+ "ldr h0, [x10], #0x2\n"
+ "ldr h1, [x28], #0x2\n"
+ "ldr h2, [x26], #0x2\n"
+ "ldr h3, [x24], #0x2\n"
+ "ldr h4, [x22], #0x2\n"
+ "tbz x11, #0, 164f\n"
+ "ld1 { v0.b }[2], [x10]\n"
+ "ld1 { v1.b }[2], [x28]\n"
+ "ld1 { v2.b }[2], [x26]\n"
+ "ld1 { v3.b }[2], [x24]\n"
+ "ld1 { v4.b }[2], [x22]\n"
+ "b 164f\n"
+ "163:" // Height 5: Multiply loop: Ragged operand read: partial_1_0
+ "ldr b0, [x10, #0x0]\n"
+ "ldr b1, [x28, #0x0]\n"
+ "ldr b2, [x26, #0x0]\n"
+ "ldr b3, [x24, #0x0]\n"
+ "ldr b4, [x22, #0x0]\n"
+ "164:" // Height 5: Multiply loop: Ragged operand read: Done
+ "ldr q6, [x14, #0x0]\n"
+ ".inst 0x6f80e0c8 // udot v8.4s, v6.16b, v0.4b[0]\n"
+ "ldr q7, [x14, #0x10]\n"
+ ".inst 0x6f81e0cc // udot v12.4s, v6.16b, v1.4b[0]\n"
+ ".inst 0x6f82e0d0 // udot v16.4s, v6.16b, v2.4b[0]\n"
+ ".inst 0x6f83e0d4 // udot v20.4s, v6.16b, v3.4b[0]\n"
+ ".inst 0x6f84e0d8 // udot v24.4s, v6.16b, v4.4b[0]\n"
+ "ldr q6, [x14, #0x20]\n"
+ ".inst 0x6f80e0e9 // udot v9.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x6f81e0ed // udot v13.4s, v7.16b, v1.4b[0]\n"
+ ".inst 0x6f82e0f1 // udot v17.4s, v7.16b, v2.4b[0]\n"
+ ".inst 0x6f83e0f5 // udot v21.4s, v7.16b, v3.4b[0]\n"
+ ".inst 0x6f84e0f9 // udot v25.4s, v7.16b, v4.4b[0]\n"
+ "ldr q7, [x14, #0x30]\n"
+ ".inst 0x6f80e0ca // udot v10.4s, v6.16b, v0.4b[0]\n"
+ "add x14, x14, #0x40\n"
+ ".inst 0x6f81e0ce // udot v14.4s, v6.16b, v1.4b[0]\n"
+ ".inst 0x6f82e0d2 // udot v18.4s, v6.16b, v2.4b[0]\n"
+ ".inst 0x6f83e0d6 // udot v22.4s, v6.16b, v3.4b[0]\n"
+ ".inst 0x6f84e0da // udot v26.4s, v6.16b, v4.4b[0]\n"
+ ".inst 0x6f80e0eb // udot v11.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x6f81e0ef // udot v15.4s, v7.16b, v1.4b[0]\n"
+ ".inst 0x6f82e0f3 // udot v19.4s, v7.16b, v2.4b[0]\n"
+ ".inst 0x6f83e0f7 // udot v23.4s, v7.16b, v3.4b[0]\n"
+ ".inst 0x6f84e0fb // udot v27.4s, v7.16b, v4.4b[0]\n"
+ "165:" // Height 5: Multiply loop: No odd multiplies
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "add x12, x12, #0x1\n"
+ "cmp x12, x19\n"
+ "bne 155b\n"
+ "prfm pstl1keep, [x13, #0x0]\n"
+ "prfm pstl1keep, [x9, #0x0]\n"
+ "cmp x15, #0x10\n"
+ "prfm pstl1keep, [x27, #0x0]\n"
+ "prfm pstl1keep, [x25, #0x0]\n"
+ "prfm pstl1keep, [x23, #0x0]\n"
+ "bge 174f\n"
+ "tbz x15, #3, 169f\n"
+ "st1 { v8.4s }, [x13], #0x10\n"
+ "st1 { v9.4s }, [x13], #0x10\n"
+ "st1 { v12.4s }, [x9], #0x10\n"
+ "st1 { v13.4s }, [x9], #0x10\n"
+ "st1 { v16.4s }, [x27], #0x10\n"
+ "st1 { v17.4s }, [x27], #0x10\n"
+ "st1 { v20.4s }, [x25], #0x10\n"
+ "st1 { v21.4s }, [x25], #0x10\n"
+ "st1 { v24.4s }, [x23], #0x10\n"
+ "st1 { v25.4s }, [x23], #0x10\n"
+ "tbz x15, #2, 167f\n"
+ "st1 { v10.4s }, [x13], #0x10\n"
+ "st1 { v14.4s }, [x9], #0x10\n"
+ "st1 { v18.4s }, [x27], #0x10\n"
+ "st1 { v22.4s }, [x25], #0x10\n"
+ "st1 { v26.4s }, [x23], #0x10\n"
+ "tbz x15, #1, 166f\n"
+ "str d11, [x13], #0x8\n"
+ "str d15, [x9], #0x8\n"
+ "str d19, [x27], #0x8\n"
+ "str d23, [x25], #0x8\n"
+ "str d27, [x23], #0x8\n"
+ "tbz x15, #0, 173f\n"
+ "st1 { v11.s }[2], [x13]\n"
+ "st1 { v15.s }[2], [x9]\n"
+ "st1 { v19.s }[2], [x27]\n"
+ "st1 { v23.s }[2], [x25]\n"
+ "st1 { v27.s }[2], [x23]\n"
+ "b 173f\n"
+ "166:" // Height 5: Partial direct writeback: partial_1_12
+ "tbz x15, #0, 173f\n"
+ "str s11, [x13, #0x0]\n"
+ "str s15, [x9, #0x0]\n"
+ "str s19, [x27, #0x0]\n"
+ "str s23, [x25, #0x0]\n"
+ "str s27, [x23, #0x0]\n"
+ "b 173f\n"
+ "167:" // Height 5: Partial direct writeback: partial_2_8
+ "tbz x15, #1, 168f\n"
+ "str d10, [x13], #0x8\n"
+ "str d14, [x9], #0x8\n"
+ "str d18, [x27], #0x8\n"
+ "str d22, [x25], #0x8\n"
+ "str d26, [x23], #0x8\n"
+ "tbz x15, #0, 173f\n"
+ "st1 { v10.s }[2], [x13]\n"
+ "st1 { v14.s }[2], [x9]\n"
+ "st1 { v18.s }[2], [x27]\n"
+ "st1 { v22.s }[2], [x25]\n"
+ "st1 { v26.s }[2], [x23]\n"
+ "b 173f\n"
+ "168:" // Height 5: Partial direct writeback: partial_1_8
+ "tbz x15, #0, 173f\n"
+ "str s10, [x13, #0x0]\n"
+ "str s14, [x9, #0x0]\n"
+ "str s18, [x27, #0x0]\n"
+ "str s22, [x25, #0x0]\n"
+ "str s26, [x23, #0x0]\n"
+ "b 173f\n"
+ "169:" // Height 5: Partial direct writeback: partial_4_0
+ "tbz x15, #2, 171f\n"
+ "st1 { v8.4s }, [x13], #0x10\n"
+ "st1 { v12.4s }, [x9], #0x10\n"
+ "st1 { v16.4s }, [x27], #0x10\n"
+ "st1 { v20.4s }, [x25], #0x10\n"
+ "st1 { v24.4s }, [x23], #0x10\n"
+ "tbz x15, #1, 170f\n"
+ "str d9, [x13], #0x8\n"
+ "str d13, [x9], #0x8\n"
+ "str d17, [x27], #0x8\n"
+ "str d21, [x25], #0x8\n"
+ "str d25, [x23], #0x8\n"
+ "tbz x15, #0, 173f\n"
+ "st1 { v9.s }[2], [x13]\n"
+ "st1 { v13.s }[2], [x9]\n"
+ "st1 { v17.s }[2], [x27]\n"
+ "st1 { v21.s }[2], [x25]\n"
+ "st1 { v25.s }[2], [x23]\n"
+ "b 173f\n"
+ "170:" // Height 5: Partial direct writeback: partial_1_4
+ "tbz x15, #0, 173f\n"
+ "str s9, [x13, #0x0]\n"
+ "str s13, [x9, #0x0]\n"
+ "str s17, [x27, #0x0]\n"
+ "str s21, [x25, #0x0]\n"
+ "str s25, [x23, #0x0]\n"
+ "b 173f\n"
+ "171:" // Height 5: Partial direct writeback: partial_2_0
+ "tbz x15, #1, 172f\n"
+ "str d8, [x13], #0x8\n"
+ "str d12, [x9], #0x8\n"
+ "str d16, [x27], #0x8\n"
+ "str d20, [x25], #0x8\n"
+ "str d24, [x23], #0x8\n"
+ "tbz x15, #0, 173f\n"
+ "st1 { v8.s }[2], [x13]\n"
+ "st1 { v12.s }[2], [x9]\n"
+ "st1 { v16.s }[2], [x27]\n"
+ "st1 { v20.s }[2], [x25]\n"
+ "st1 { v24.s }[2], [x23]\n"
+ "b 173f\n"
+ "172:" // Height 5: Partial direct writeback: partial_1_0
+ "str s8, [x13, #0x0]\n"
+ "str s12, [x9, #0x0]\n"
+ "str s16, [x27, #0x0]\n"
+ "str s20, [x25, #0x0]\n"
+ "str s24, [x23, #0x0]\n"
+ "173:" // Height 5: Partial direct writeback: Done
+ "b 175f\n"
+ "174:" // Height 5: Full writeback
+ "str q8, [x13, #0x0]\n"
+ "str q9, [x13, #0x10]\n"
+ "str q10, [x13, #0x20]\n"
+ "str q11, [x13, #0x30]\n"
+ "str q12, [x9, #0x0]\n"
+ "str q13, [x9, #0x10]\n"
+ "str q14, [x9, #0x20]\n"
+ "str q15, [x9, #0x30]\n"
+ "str q16, [x27, #0x0]\n"
+ "str q17, [x27, #0x10]\n"
+ "str q18, [x27, #0x20]\n"
+ "str q19, [x27, #0x30]\n"
+ "str q20, [x25, #0x0]\n"
+ "str q21, [x25, #0x10]\n"
+ "str q22, [x25, #0x20]\n"
+ "str q23, [x25, #0x30]\n"
+ "str q24, [x23, #0x0]\n"
+ "str q25, [x23, #0x10]\n"
+ "str q26, [x23, #0x20]\n"
+ "str q27, [x23, #0x30]\n"
+ "add x13, x13, #0x40\n"
+ "add x9, x9, #0x40\n"
+ "add x27, x27, #0x40\n"
+ "add x25, x25, #0x40\n"
+ "add x23, x23, #0x40\n"
+ "175:" // Height 5: Writeback done
+ "subs x15, x15, #0x10\n"
+ "bgt 143b\n"
+ "b 212f\n"
+ "176:" // Height 6
+ "ldr x15, [%x[args_ptr], %[offsetof_N]]\n"
+ "ldr x14, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 177f\n"
+ "ldr x13, [%x[output_ptr], #0x0]\n"
+ "add x13, x13, x19, LSL #2\n"
+ "ldr x9, [%x[output_ptr], #0x8]\n"
+ "ldr x27, [%x[output_ptr], #0x10]\n"
+ "add x9, x9, x19, LSL #2\n"
+ "ldr x25, [%x[output_ptr], #0x18]\n"
+ "ldr x23, [%x[output_ptr], #0x20]\n"
+ "add x27, x27, x19, LSL #2\n"
+ "ldr x21, [%x[output_ptr], #0x28]\n"
+ "add %x[output_ptr], %x[output_ptr], #0x30\n"
+ "add x25, x25, x19, LSL #2\n"
+ "add x23, x23, x19, LSL #2\n"
+ "add x21, x21, x19, LSL #2\n"
+ "b 178f\n"
+ "177:" // Height 6: setup direct output
+ "mov x13, %x[output_ptr]\n"
+ "add x9, x13, x19, LSL #2\n"
+ "add x27, x9, x19, LSL #2\n"
+ "add x25, x27, x19, LSL #2\n"
+ "add x23, x25, x19, LSL #2\n"
+ "add x21, x23, x19, LSL #2\n"
+ "add %x[output_ptr], x21, x19, LSL #2\n"
+ "178:" // Height 6: Column loop
+ "tbz %x[flags], #0, 188f\n"
+ "cmp x15, #0x10\n"
+ "bge 187f\n"
+ "tbz x15, #3, 182f\n"
+ "ld1 { v8.4s }, [x13], #0x10\n"
+ "ld1 { v12.4s }, [x9], #0x10\n"
+ "ld1 { v16.4s }, [x27], #0x10\n"
+ "ld1 { v20.4s }, [x25], #0x10\n"
+ "ld1 { v24.4s }, [x23], #0x10\n"
+ "ld1 { v28.4s }, [x21], #0x10\n"
+ "ld1 { v9.4s }, [x13], #0x10\n"
+ "ld1 { v13.4s }, [x9], #0x10\n"
+ "ld1 { v17.4s }, [x27], #0x10\n"
+ "ld1 { v21.4s }, [x25], #0x10\n"
+ "ld1 { v25.4s }, [x23], #0x10\n"
+ "ld1 { v29.4s }, [x21], #0x10\n"
+ "tbz x15, #2, 180f\n"
+ "ld1 { v10.4s }, [x13], #0x10\n"
+ "ld1 { v14.4s }, [x9], #0x10\n"
+ "ld1 { v18.4s }, [x27], #0x10\n"
+ "ld1 { v22.4s }, [x25], #0x10\n"
+ "ld1 { v26.4s }, [x23], #0x10\n"
+ "ld1 { v30.4s }, [x21], #0x10\n"
+ "tbz x15, #1, 179f\n"
+ "mov x19, #0x38\n"
+ "ldr d11, [x13], #0x8\n"
+ "ldr d15, [x9], #0x8\n"
+ "ldr d19, [x27], #0x8\n"
+ "ldr d23, [x25], #0x8\n"
+ "ldr d27, [x23], #0x8\n"
+ "ldr d31, [x21], #0x8\n"
+ "tbz x15, #0, 186f\n"
+ "ld1 { v11.s }[2], [x13]\n"
+ "ld1 { v15.s }[2], [x9]\n"
+ "ld1 { v19.s }[2], [x27]\n"
+ "ld1 { v23.s }[2], [x25]\n"
+ "ld1 { v27.s }[2], [x23]\n"
+ "ld1 { v31.s }[2], [x21]\n"
+ "b 186f\n"
+ "179:" // Height 6: Partial accumulate: partial_1_12
+ "mov x19, #0x30\n"
+ "tbz x15, #0, 186f\n"
+ "ldr s11, [x13, #0x0]\n"
+ "ldr s15, [x9, #0x0]\n"
+ "ldr s19, [x27, #0x0]\n"
+ "ldr s23, [x25, #0x0]\n"
+ "ldr s27, [x23, #0x0]\n"
+ "ldr s31, [x21, #0x0]\n"
+ "b 186f\n"
+ "180:" // Height 6: Partial accumulate: partial_2_8
+ "tbz x15, #1, 181f\n"
+ "ldr d10, [x13], #0x8\n"
+ "ldr d14, [x9], #0x8\n"
+ "ldr d18, [x27], #0x8\n"
+ "ldr d22, [x25], #0x8\n"
+ "ldr d26, [x23], #0x8\n"
+ "ldr d30, [x21], #0x8\n"
+ "mov x19, #0x28\n"
+ "tbz x15, #0, 186f\n"
+ "ld1 { v10.s }[2], [x13]\n"
+ "ld1 { v14.s }[2], [x9]\n"
+ "ld1 { v18.s }[2], [x27]\n"
+ "ld1 { v22.s }[2], [x25]\n"
+ "ld1 { v26.s }[2], [x23]\n"
+ "ld1 { v30.s }[2], [x21]\n"
+ "b 186f\n"
+ "181:" // Height 6: Partial accumulate: partial_1_8
+ "mov x19, #0x20\n"
+ "tbz x15, #0, 186f\n"
+ "ldr s10, [x13, #0x0]\n"
+ "ldr s14, [x9, #0x0]\n"
+ "ldr s18, [x27, #0x0]\n"
+ "ldr s22, [x25, #0x0]\n"
+ "ldr s26, [x23, #0x0]\n"
+ "ldr s30, [x21, #0x0]\n"
+ "b 186f\n"
+ "182:" // Height 6: Partial accumulate: partial_4_0
+ "tbz x15, #2, 184f\n"
+ "ld1 { v8.4s }, [x13], #0x10\n"
+ "ld1 { v12.4s }, [x9], #0x10\n"
+ "ld1 { v16.4s }, [x27], #0x10\n"
+ "ld1 { v20.4s }, [x25], #0x10\n"
+ "ld1 { v24.4s }, [x23], #0x10\n"
+ "ld1 { v28.4s }, [x21], #0x10\n"
+ "tbz x15, #1, 183f\n"
+ "mov x19, #0x18\n"
+ "ldr d9, [x13], #0x8\n"
+ "ldr d13, [x9], #0x8\n"
+ "ldr d17, [x27], #0x8\n"
+ "ldr d21, [x25], #0x8\n"
+ "ldr d25, [x23], #0x8\n"
+ "ldr d29, [x21], #0x8\n"
+ "tbz x15, #0, 186f\n"
+ "ld1 { v9.s }[2], [x13]\n"
+ "ld1 { v13.s }[2], [x9]\n"
+ "ld1 { v17.s }[2], [x27]\n"
+ "ld1 { v21.s }[2], [x25]\n"
+ "ld1 { v25.s }[2], [x23]\n"
+ "ld1 { v29.s }[2], [x21]\n"
+ "b 186f\n"
+ "183:" // Height 6: Partial accumulate: partial_1_4
+ "mov x19, #0x10\n"
+ "tbz x15, #0, 186f\n"
+ "ldr s9, [x13, #0x0]\n"
+ "ldr s13, [x9, #0x0]\n"
+ "ldr s17, [x27, #0x0]\n"
+ "ldr s21, [x25, #0x0]\n"
+ "ldr s25, [x23, #0x0]\n"
+ "ldr s29, [x21, #0x0]\n"
+ "b 186f\n"
+ "184:" // Height 6: Partial accumulate: partial_2_0
+ "tbz x15, #1, 185f\n"
+ "ldr d8, [x13], #0x8\n"
+ "ldr d12, [x9], #0x8\n"
+ "ldr d16, [x27], #0x8\n"
+ "ldr d20, [x25], #0x8\n"
+ "ldr d24, [x23], #0x8\n"
+ "ldr d28, [x21], #0x8\n"
+ "mov x19, #0x8\n"
+ "tbz x15, #0, 186f\n"
+ "ld1 { v8.s }[2], [x13]\n"
+ "ld1 { v12.s }[2], [x9]\n"
+ "ld1 { v16.s }[2], [x27]\n"
+ "ld1 { v20.s }[2], [x25]\n"
+ "ld1 { v24.s }[2], [x23]\n"
+ "ld1 { v28.s }[2], [x21]\n"
+ "b 186f\n"
+ "185:" // Height 6: Partial accumulate: partial_1_0
+ "mov x19, #0x0\n"
+ "ldr s8, [x13, #0x0]\n"
+ "ldr s12, [x9, #0x0]\n"
+ "ldr s16, [x27, #0x0]\n"
+ "ldr s20, [x25, #0x0]\n"
+ "ldr s24, [x23, #0x0]\n"
+ "ldr s28, [x21, #0x0]\n"
+ "186:" // Height 6: Partial accumulate: Done
+ "sub x13, x13, x19\n"
+ "sub x9, x9, x19\n"
+ "sub x27, x27, x19\n"
+ "sub x25, x25, x19\n"
+ "sub x23, x23, x19\n"
+ "sub x21, x21, x19\n"
+ "b 189f\n"
+ "187:" // Height 6: full accumulate
+ "ldr q8, [x13, #0x0]\n"
+ "ldr q9, [x13, #0x10]\n"
+ "ldr q10, [x13, #0x20]\n"
+ "ldr q11, [x13, #0x30]\n"
+ "ldr q12, [x9, #0x0]\n"
+ "ldr q13, [x9, #0x10]\n"
+ "ldr q14, [x9, #0x20]\n"
+ "ldr q15, [x9, #0x30]\n"
+ "ldr q16, [x27, #0x0]\n"
+ "ldr q17, [x27, #0x10]\n"
+ "ldr q18, [x27, #0x20]\n"
+ "ldr q19, [x27, #0x30]\n"
+ "ldr q20, [x25, #0x0]\n"
+ "ldr q21, [x25, #0x10]\n"
+ "ldr q22, [x25, #0x20]\n"
+ "ldr q23, [x25, #0x30]\n"
+ "ldr q24, [x23, #0x0]\n"
+ "ldr q25, [x23, #0x10]\n"
+ "ldr q26, [x23, #0x20]\n"
+ "ldr q27, [x23, #0x30]\n"
+ "ldr q28, [x21, #0x0]\n"
+ "ldr q29, [x21, #0x10]\n"
+ "ldr q30, [x21, #0x20]\n"
+ "ldr q31, [x21, #0x30]\n"
+ "b 189f\n"
+ "188:" // Height 6: no accumulate
+ "movi v8.4s, #0x0\n"
+ "movi v9.4s, #0x0\n"
+ "movi v10.4s, #0x0\n"
+ "movi v11.4s, #0x0\n"
+ "movi v12.4s, #0x0\n"
+ "movi v13.4s, #0x0\n"
+ "movi v14.4s, #0x0\n"
+ "movi v15.4s, #0x0\n"
+ "movi v16.4s, #0x0\n"
+ "movi v17.4s, #0x0\n"
+ "movi v18.4s, #0x0\n"
+ "movi v19.4s, #0x0\n"
+ "movi v20.4s, #0x0\n"
+ "movi v21.4s, #0x0\n"
+ "movi v22.4s, #0x0\n"
+ "movi v23.4s, #0x0\n"
+ "movi v24.4s, #0x0\n"
+ "movi v25.4s, #0x0\n"
+ "movi v26.4s, #0x0\n"
+ "movi v27.4s, #0x0\n"
+ "movi v28.4s, #0x0\n"
+ "movi v29.4s, #0x0\n"
+ "movi v30.4s, #0x0\n"
+ "movi v31.4s, #0x0\n"
+ "189:" // Height 6: setup done
+ "mov x12, #0x0\n"
+ "190:" // Height 6: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w11, [x20, x12, LSL #0x2]\n"
+ "tbz %x[flags], #3, 191f\n"
+ "ldr x20, [%x[input_ptr], x12, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x10, [x20, #0x0]\n"
+ "ldr x28, [x20, #0x8]\n"
+ "ldr x26, [x20, #0x10]\n"
+ "ldr x24, [x20, #0x18]\n"
+ "ldr x22, [x20, #0x20]\n"
+ "ldr x20, [x20, #0x28]\n"
+ "cbnz x12, 192f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x10, x10, x19\n"
+ "add x28, x28, x19\n"
+ "add x26, x26, x19\n"
+ "add x24, x24, x19\n"
+ "add x22, x22, x19\n"
+ "add x20, x20, x19\n"
+ "b 192f\n"
+ "191:" // Height 6: setup direct input
+ "mov x10, %x[input_ptr]\n"
+ "add x28, x10, x19\n"
+ "add x26, x28, x19\n"
+ "add x24, x26, x19\n"
+ "add x22, x24, x19\n"
+ "add x20, x22, x19\n"
+ "192:" // Height 6: input setup done
+ "cmp x11, #0x10\n"
+ "blt 195f\n"
+ "cmp x11, #0x20\n"
+ "blt 194f\n"
+ "193:" // Height 6: Multiply loop: Main loop head
+ "ldr q0, [x10, #0x0]\n"
+ "ldr q1, [x28, #0x0]\n"
+ "ldr q2, [x26, #0x0]\n"
+ "ldr q3, [x24, #0x0]\n"
+ "ldr q4, [x22, #0x0]\n"
+ "ldr q5, [x20, #0x0]\n"
+ "ldr q6, [x14, #0x0]\n"
+ ".inst 0x6f80e0c8 // udot v8.4s, v6.16b, v0.4b[0]\n"
+ "ldr q7, [x14, #0x10]\n"
+ ".inst 0x6f81e0cc // udot v12.4s, v6.16b, v1.4b[0]\n"
+ "add x10, x10, #0x10\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ ".inst 0x6f82e0d0 // udot v16.4s, v6.16b, v2.4b[0]\n"
+ "add x28, x28, #0x10\n"
+ ".inst 0x6f83e0d4 // udot v20.4s, v6.16b, v3.4b[0]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "add x26, x26, #0x10\n"
+ ".inst 0x6f84e0d8 // udot v24.4s, v6.16b, v4.4b[0]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "add x24, x24, #0x10\n"
+ ".inst 0x6f85e0dc // udot v28.4s, v6.16b, v5.4b[0]\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "ldr q6, [x14, #0x20]\n"
+ ".inst 0x6f80e0e9 // udot v9.4s, v7.16b, v0.4b[0]\n"
+ "add x22, x22, #0x10\n"
+ "prfm pldl1keep, [x22, #0x80]\n"
+ ".inst 0x6f81e0ed // udot v13.4s, v7.16b, v1.4b[0]\n"
+ "add x20, x20, #0x10\n"
+ ".inst 0x6f82e0f1 // udot v17.4s, v7.16b, v2.4b[0]\n"
+ "prfm pldl1keep, [x20, #0x80]\n"
+ "sub x11, x11, #0x10\n"
+ ".inst 0x6f83e0f5 // udot v21.4s, v7.16b, v3.4b[0]\n"
+ "cmp x11, #0x20\n"
+ ".inst 0x6f84e0f9 // udot v25.4s, v7.16b, v4.4b[0]\n"
+ ".inst 0x6f85e0fd // udot v29.4s, v7.16b, v5.4b[0]\n"
+ "ldr q7, [x14, #0x30]\n"
+ ".inst 0x6f80e0ca // udot v10.4s, v6.16b, v0.4b[0]\n"
+ ".inst 0x6f81e0ce // udot v14.4s, v6.16b, v1.4b[0]\n"
+ ".inst 0x6f82e0d2 // udot v18.4s, v6.16b, v2.4b[0]\n"
+ ".inst 0x6f83e0d6 // udot v22.4s, v6.16b, v3.4b[0]\n"
+ ".inst 0x6f84e0da // udot v26.4s, v6.16b, v4.4b[0]\n"
+ ".inst 0x6f85e0de // udot v30.4s, v6.16b, v5.4b[0]\n"
+ "ldr q6, [x14, #0x40]\n"
+ ".inst 0x6f80e0eb // udot v11.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x6f81e0ef // udot v15.4s, v7.16b, v1.4b[0]\n"
+ ".inst 0x6f82e0f3 // udot v19.4s, v7.16b, v2.4b[0]\n"
+ ".inst 0x6f83e0f7 // udot v23.4s, v7.16b, v3.4b[0]\n"
+ ".inst 0x6f84e0fb // udot v27.4s, v7.16b, v4.4b[0]\n"
+ ".inst 0x6f85e0ff // udot v31.4s, v7.16b, v5.4b[0]\n"
+ "ldr q7, [x14, #0x50]\n"
+ ".inst 0x6fa0e0c8 // udot v8.4s, v6.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e0cc // udot v12.4s, v6.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e0d0 // udot v16.4s, v6.16b, v2.4b[1]\n"
+ ".inst 0x6fa3e0d4 // udot v20.4s, v6.16b, v3.4b[1]\n"
+ ".inst 0x6fa4e0d8 // udot v24.4s, v6.16b, v4.4b[1]\n"
+ ".inst 0x6fa5e0dc // udot v28.4s, v6.16b, v5.4b[1]\n"
+ "ldr q6, [x14, #0x60]\n"
+ ".inst 0x6fa0e0e9 // udot v9.4s, v7.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e0ed // udot v13.4s, v7.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e0f1 // udot v17.4s, v7.16b, v2.4b[1]\n"
+ ".inst 0x6fa3e0f5 // udot v21.4s, v7.16b, v3.4b[1]\n"
+ ".inst 0x6fa4e0f9 // udot v25.4s, v7.16b, v4.4b[1]\n"
+ ".inst 0x6fa5e0fd // udot v29.4s, v7.16b, v5.4b[1]\n"
+ "ldr q7, [x14, #0x70]\n"
+ ".inst 0x6fa0e0ca // udot v10.4s, v6.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e0ce // udot v14.4s, v6.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e0d2 // udot v18.4s, v6.16b, v2.4b[1]\n"
+ ".inst 0x6fa3e0d6 // udot v22.4s, v6.16b, v3.4b[1]\n"
+ ".inst 0x6fa4e0da // udot v26.4s, v6.16b, v4.4b[1]\n"
+ ".inst 0x6fa5e0de // udot v30.4s, v6.16b, v5.4b[1]\n"
+ "ldr q6, [x14, #0x80]\n"
+ ".inst 0x6fa0e0eb // udot v11.4s, v7.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e0ef // udot v15.4s, v7.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e0f3 // udot v19.4s, v7.16b, v2.4b[1]\n"
+ ".inst 0x6fa3e0f7 // udot v23.4s, v7.16b, v3.4b[1]\n"
+ ".inst 0x6fa4e0fb // udot v27.4s, v7.16b, v4.4b[1]\n"
+ ".inst 0x6fa5e0ff // udot v31.4s, v7.16b, v5.4b[1]\n"
+ "ldr q7, [x14, #0x90]\n"
+ ".inst 0x6f80e8c8 // udot v8.4s, v6.16b, v0.4b[2]\n"
+ ".inst 0x6f81e8cc // udot v12.4s, v6.16b, v1.4b[2]\n"
+ ".inst 0x6f82e8d0 // udot v16.4s, v6.16b, v2.4b[2]\n"
+ ".inst 0x6f83e8d4 // udot v20.4s, v6.16b, v3.4b[2]\n"
+ ".inst 0x6f84e8d8 // udot v24.4s, v6.16b, v4.4b[2]\n"
+ ".inst 0x6f85e8dc // udot v28.4s, v6.16b, v5.4b[2]\n"
+ "ldr q6, [x14, #0xa0]\n"
+ ".inst 0x6f80e8e9 // udot v9.4s, v7.16b, v0.4b[2]\n"
+ ".inst 0x6f81e8ed // udot v13.4s, v7.16b, v1.4b[2]\n"
+ ".inst 0x6f82e8f1 // udot v17.4s, v7.16b, v2.4b[2]\n"
+ ".inst 0x6f83e8f5 // udot v21.4s, v7.16b, v3.4b[2]\n"
+ ".inst 0x6f84e8f9 // udot v25.4s, v7.16b, v4.4b[2]\n"
+ ".inst 0x6f85e8fd // udot v29.4s, v7.16b, v5.4b[2]\n"
+ "ldr q7, [x14, #0xb0]\n"
+ ".inst 0x6f80e8ca // udot v10.4s, v6.16b, v0.4b[2]\n"
+ ".inst 0x6f81e8ce // udot v14.4s, v6.16b, v1.4b[2]\n"
+ ".inst 0x6f82e8d2 // udot v18.4s, v6.16b, v2.4b[2]\n"
+ ".inst 0x6f83e8d6 // udot v22.4s, v6.16b, v3.4b[2]\n"
+ ".inst 0x6f84e8da // udot v26.4s, v6.16b, v4.4b[2]\n"
+ ".inst 0x6f85e8de // udot v30.4s, v6.16b, v5.4b[2]\n"
+ "ldr q6, [x14, #0xc0]\n"
+ ".inst 0x6f80e8eb // udot v11.4s, v7.16b, v0.4b[2]\n"
+ ".inst 0x6f81e8ef // udot v15.4s, v7.16b, v1.4b[2]\n"
+ ".inst 0x6f82e8f3 // udot v19.4s, v7.16b, v2.4b[2]\n"
+ ".inst 0x6f83e8f7 // udot v23.4s, v7.16b, v3.4b[2]\n"
+ ".inst 0x6f84e8fb // udot v27.4s, v7.16b, v4.4b[2]\n"
+ ".inst 0x6f85e8ff // udot v31.4s, v7.16b, v5.4b[2]\n"
+ "ldr q7, [x14, #0xd0]\n"
+ ".inst 0x6fa0e8c8 // udot v8.4s, v6.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e8cc // udot v12.4s, v6.16b, v1.4b[3]\n"
+ ".inst 0x6fa2e8d0 // udot v16.4s, v6.16b, v2.4b[3]\n"
+ ".inst 0x6fa3e8d4 // udot v20.4s, v6.16b, v3.4b[3]\n"
+ ".inst 0x6fa4e8d8 // udot v24.4s, v6.16b, v4.4b[3]\n"
+ ".inst 0x6fa5e8dc // udot v28.4s, v6.16b, v5.4b[3]\n"
+ "ldr q6, [x14, #0xe0]\n"
+ ".inst 0x6fa0e8e9 // udot v9.4s, v7.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e8ed // udot v13.4s, v7.16b, v1.4b[3]\n"
+ ".inst 0x6fa2e8f1 // udot v17.4s, v7.16b, v2.4b[3]\n"
+ ".inst 0x6fa3e8f5 // udot v21.4s, v7.16b, v3.4b[3]\n"
+ ".inst 0x6fa4e8f9 // udot v25.4s, v7.16b, v4.4b[3]\n"
+ ".inst 0x6fa5e8fd // udot v29.4s, v7.16b, v5.4b[3]\n"
+ "ldr q7, [x14, #0xf0]\n"
+ ".inst 0x6fa0e8ca // udot v10.4s, v6.16b, v0.4b[3]\n"
+ "add x14, x14, #0x100\n"
+ ".inst 0x6fa1e8ce // udot v14.4s, v6.16b, v1.4b[3]\n"
+ ".inst 0x6fa2e8d2 // udot v18.4s, v6.16b, v2.4b[3]\n"
+ ".inst 0x6fa3e8d6 // udot v22.4s, v6.16b, v3.4b[3]\n"
+ ".inst 0x6fa4e8da // udot v26.4s, v6.16b, v4.4b[3]\n"
+ ".inst 0x6fa5e8de // udot v30.4s, v6.16b, v5.4b[3]\n"
+ ".inst 0x6fa0e8eb // udot v11.4s, v7.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e8ef // udot v15.4s, v7.16b, v1.4b[3]\n"
+ ".inst 0x6fa2e8f3 // udot v19.4s, v7.16b, v2.4b[3]\n"
+ ".inst 0x6fa3e8f7 // udot v23.4s, v7.16b, v3.4b[3]\n"
+ ".inst 0x6fa4e8fb // udot v27.4s, v7.16b, v4.4b[3]\n"
+ ".inst 0x6fa5e8ff // udot v31.4s, v7.16b, v5.4b[3]\n"
+ "bge 193b\n"
+ "194:" // Height 6: Multiply loop: Single iteration only
+ "sub x11, x11, #0x10\n"
+ "ldr q0, [x10, #0x0]\n"
+ "ldr q1, [x28, #0x0]\n"
+ "ldr q2, [x26, #0x0]\n"
+ "ldr q3, [x24, #0x0]\n"
+ "ldr q4, [x22, #0x0]\n"
+ "ldr q5, [x20, #0x0]\n"
+ "ldr q6, [x14, #0x0]\n"
+ ".inst 0x6f80e0c8 // udot v8.4s, v6.16b, v0.4b[0]\n"
+ "ldr q7, [x14, #0x10]\n"
+ ".inst 0x6f81e0cc // udot v12.4s, v6.16b, v1.4b[0]\n"
+ "add x10, x10, #0x10\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ ".inst 0x6f82e0d0 // udot v16.4s, v6.16b, v2.4b[0]\n"
+ "add x28, x28, #0x10\n"
+ ".inst 0x6f83e0d4 // udot v20.4s, v6.16b, v3.4b[0]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "add x26, x26, #0x10\n"
+ ".inst 0x6f84e0d8 // udot v24.4s, v6.16b, v4.4b[0]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "add x24, x24, #0x10\n"
+ ".inst 0x6f85e0dc // udot v28.4s, v6.16b, v5.4b[0]\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "ldr q6, [x14, #0x20]\n"
+ ".inst 0x6f80e0e9 // udot v9.4s, v7.16b, v0.4b[0]\n"
+ "add x22, x22, #0x10\n"
+ "prfm pldl1keep, [x22, #0x80]\n"
+ ".inst 0x6f81e0ed // udot v13.4s, v7.16b, v1.4b[0]\n"
+ "add x20, x20, #0x10\n"
+ ".inst 0x6f82e0f1 // udot v17.4s, v7.16b, v2.4b[0]\n"
+ "prfm pldl1keep, [x20, #0x80]\n"
+ ".inst 0x6f83e0f5 // udot v21.4s, v7.16b, v3.4b[0]\n"
+ ".inst 0x6f84e0f9 // udot v25.4s, v7.16b, v4.4b[0]\n"
+ ".inst 0x6f85e0fd // udot v29.4s, v7.16b, v5.4b[0]\n"
+ "ldr q7, [x14, #0x30]\n"
+ ".inst 0x6f80e0ca // udot v10.4s, v6.16b, v0.4b[0]\n"
+ ".inst 0x6f81e0ce // udot v14.4s, v6.16b, v1.4b[0]\n"
+ ".inst 0x6f82e0d2 // udot v18.4s, v6.16b, v2.4b[0]\n"
+ ".inst 0x6f83e0d6 // udot v22.4s, v6.16b, v3.4b[0]\n"
+ ".inst 0x6f84e0da // udot v26.4s, v6.16b, v4.4b[0]\n"
+ ".inst 0x6f85e0de // udot v30.4s, v6.16b, v5.4b[0]\n"
+ "ldr q6, [x14, #0x40]\n"
+ ".inst 0x6f80e0eb // udot v11.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x6f81e0ef // udot v15.4s, v7.16b, v1.4b[0]\n"
+ ".inst 0x6f82e0f3 // udot v19.4s, v7.16b, v2.4b[0]\n"
+ ".inst 0x6f83e0f7 // udot v23.4s, v7.16b, v3.4b[0]\n"
+ ".inst 0x6f84e0fb // udot v27.4s, v7.16b, v4.4b[0]\n"
+ ".inst 0x6f85e0ff // udot v31.4s, v7.16b, v5.4b[0]\n"
+ "ldr q7, [x14, #0x50]\n"
+ ".inst 0x6fa0e0c8 // udot v8.4s, v6.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e0cc // udot v12.4s, v6.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e0d0 // udot v16.4s, v6.16b, v2.4b[1]\n"
+ ".inst 0x6fa3e0d4 // udot v20.4s, v6.16b, v3.4b[1]\n"
+ ".inst 0x6fa4e0d8 // udot v24.4s, v6.16b, v4.4b[1]\n"
+ ".inst 0x6fa5e0dc // udot v28.4s, v6.16b, v5.4b[1]\n"
+ "ldr q6, [x14, #0x60]\n"
+ ".inst 0x6fa0e0e9 // udot v9.4s, v7.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e0ed // udot v13.4s, v7.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e0f1 // udot v17.4s, v7.16b, v2.4b[1]\n"
+ ".inst 0x6fa3e0f5 // udot v21.4s, v7.16b, v3.4b[1]\n"
+ ".inst 0x6fa4e0f9 // udot v25.4s, v7.16b, v4.4b[1]\n"
+ ".inst 0x6fa5e0fd // udot v29.4s, v7.16b, v5.4b[1]\n"
+ "ldr q7, [x14, #0x70]\n"
+ ".inst 0x6fa0e0ca // udot v10.4s, v6.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e0ce // udot v14.4s, v6.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e0d2 // udot v18.4s, v6.16b, v2.4b[1]\n"
+ ".inst 0x6fa3e0d6 // udot v22.4s, v6.16b, v3.4b[1]\n"
+ ".inst 0x6fa4e0da // udot v26.4s, v6.16b, v4.4b[1]\n"
+ ".inst 0x6fa5e0de // udot v30.4s, v6.16b, v5.4b[1]\n"
+ "ldr q6, [x14, #0x80]\n"
+ ".inst 0x6fa0e0eb // udot v11.4s, v7.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e0ef // udot v15.4s, v7.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e0f3 // udot v19.4s, v7.16b, v2.4b[1]\n"
+ ".inst 0x6fa3e0f7 // udot v23.4s, v7.16b, v3.4b[1]\n"
+ ".inst 0x6fa4e0fb // udot v27.4s, v7.16b, v4.4b[1]\n"
+ ".inst 0x6fa5e0ff // udot v31.4s, v7.16b, v5.4b[1]\n"
+ "ldr q7, [x14, #0x90]\n"
+ ".inst 0x6f80e8c8 // udot v8.4s, v6.16b, v0.4b[2]\n"
+ ".inst 0x6f81e8cc // udot v12.4s, v6.16b, v1.4b[2]\n"
+ ".inst 0x6f82e8d0 // udot v16.4s, v6.16b, v2.4b[2]\n"
+ ".inst 0x6f83e8d4 // udot v20.4s, v6.16b, v3.4b[2]\n"
+ ".inst 0x6f84e8d8 // udot v24.4s, v6.16b, v4.4b[2]\n"
+ ".inst 0x6f85e8dc // udot v28.4s, v6.16b, v5.4b[2]\n"
+ "ldr q6, [x14, #0xa0]\n"
+ ".inst 0x6f80e8e9 // udot v9.4s, v7.16b, v0.4b[2]\n"
+ ".inst 0x6f81e8ed // udot v13.4s, v7.16b, v1.4b[2]\n"
+ ".inst 0x6f82e8f1 // udot v17.4s, v7.16b, v2.4b[2]\n"
+ ".inst 0x6f83e8f5 // udot v21.4s, v7.16b, v3.4b[2]\n"
+ ".inst 0x6f84e8f9 // udot v25.4s, v7.16b, v4.4b[2]\n"
+ ".inst 0x6f85e8fd // udot v29.4s, v7.16b, v5.4b[2]\n"
+ "ldr q7, [x14, #0xb0]\n"
+ ".inst 0x6f80e8ca // udot v10.4s, v6.16b, v0.4b[2]\n"
+ ".inst 0x6f81e8ce // udot v14.4s, v6.16b, v1.4b[2]\n"
+ ".inst 0x6f82e8d2 // udot v18.4s, v6.16b, v2.4b[2]\n"
+ ".inst 0x6f83e8d6 // udot v22.4s, v6.16b, v3.4b[2]\n"
+ ".inst 0x6f84e8da // udot v26.4s, v6.16b, v4.4b[2]\n"
+ ".inst 0x6f85e8de // udot v30.4s, v6.16b, v5.4b[2]\n"
+ "ldr q6, [x14, #0xc0]\n"
+ ".inst 0x6f80e8eb // udot v11.4s, v7.16b, v0.4b[2]\n"
+ ".inst 0x6f81e8ef // udot v15.4s, v7.16b, v1.4b[2]\n"
+ ".inst 0x6f82e8f3 // udot v19.4s, v7.16b, v2.4b[2]\n"
+ ".inst 0x6f83e8f7 // udot v23.4s, v7.16b, v3.4b[2]\n"
+ ".inst 0x6f84e8fb // udot v27.4s, v7.16b, v4.4b[2]\n"
+ ".inst 0x6f85e8ff // udot v31.4s, v7.16b, v5.4b[2]\n"
+ "ldr q7, [x14, #0xd0]\n"
+ ".inst 0x6fa0e8c8 // udot v8.4s, v6.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e8cc // udot v12.4s, v6.16b, v1.4b[3]\n"
+ ".inst 0x6fa2e8d0 // udot v16.4s, v6.16b, v2.4b[3]\n"
+ ".inst 0x6fa3e8d4 // udot v20.4s, v6.16b, v3.4b[3]\n"
+ ".inst 0x6fa4e8d8 // udot v24.4s, v6.16b, v4.4b[3]\n"
+ ".inst 0x6fa5e8dc // udot v28.4s, v6.16b, v5.4b[3]\n"
+ "ldr q6, [x14, #0xe0]\n"
+ ".inst 0x6fa0e8e9 // udot v9.4s, v7.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e8ed // udot v13.4s, v7.16b, v1.4b[3]\n"
+ ".inst 0x6fa2e8f1 // udot v17.4s, v7.16b, v2.4b[3]\n"
+ ".inst 0x6fa3e8f5 // udot v21.4s, v7.16b, v3.4b[3]\n"
+ ".inst 0x6fa4e8f9 // udot v25.4s, v7.16b, v4.4b[3]\n"
+ ".inst 0x6fa5e8fd // udot v29.4s, v7.16b, v5.4b[3]\n"
+ "ldr q7, [x14, #0xf0]\n"
+ ".inst 0x6fa0e8ca // udot v10.4s, v6.16b, v0.4b[3]\n"
+ "add x14, x14, #0x100\n"
+ ".inst 0x6fa1e8ce // udot v14.4s, v6.16b, v1.4b[3]\n"
+ ".inst 0x6fa2e8d2 // udot v18.4s, v6.16b, v2.4b[3]\n"
+ ".inst 0x6fa3e8d6 // udot v22.4s, v6.16b, v3.4b[3]\n"
+ ".inst 0x6fa4e8da // udot v26.4s, v6.16b, v4.4b[3]\n"
+ ".inst 0x6fa5e8de // udot v30.4s, v6.16b, v5.4b[3]\n"
+ ".inst 0x6fa0e8eb // udot v11.4s, v7.16b, v0.4b[3]\n"
+ ".inst 0x6fa1e8ef // udot v15.4s, v7.16b, v1.4b[3]\n"
+ ".inst 0x6fa2e8f3 // udot v19.4s, v7.16b, v2.4b[3]\n"
+ ".inst 0x6fa3e8f7 // udot v23.4s, v7.16b, v3.4b[3]\n"
+ ".inst 0x6fa4e8fb // udot v27.4s, v7.16b, v4.4b[3]\n"
+ ".inst 0x6fa5e8ff // udot v31.4s, v7.16b, v5.4b[3]\n"
+ "195:" // Height 6: Multiply loop: Main loop skip
+ "cbz x11, 200f\n"
+ "cmp x11, #0x4\n"
+ "blt 197f\n"
+ "196:" // Height 6: Multiply loop: Odd block loop
+ "ldr s0, [x10], #0x4\n"
+ "ldr s1, [x28], #0x4\n"
+ "ldr s2, [x26], #0x4\n"
+ "ldr s3, [x24], #0x4\n"
+ "ldr s4, [x22], #0x4\n"
+ "ldr s5, [x20], #0x4\n"
+ "ldr q6, [x14, #0x0]\n"
+ ".inst 0x6f80e0c8 // udot v8.4s, v6.16b, v0.4b[0]\n"
+ "ldr q7, [x14, #0x10]\n"
+ ".inst 0x6f81e0cc // udot v12.4s, v6.16b, v1.4b[0]\n"
+ "sub x11, x11, #0x4\n"
+ ".inst 0x6f82e0d0 // udot v16.4s, v6.16b, v2.4b[0]\n"
+ "cmp x11, #0x4\n"
+ ".inst 0x6f83e0d4 // udot v20.4s, v6.16b, v3.4b[0]\n"
+ ".inst 0x6f84e0d8 // udot v24.4s, v6.16b, v4.4b[0]\n"
+ ".inst 0x6f85e0dc // udot v28.4s, v6.16b, v5.4b[0]\n"
+ "ldr q6, [x14, #0x20]\n"
+ ".inst 0x6f80e0e9 // udot v9.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x6f81e0ed // udot v13.4s, v7.16b, v1.4b[0]\n"
+ ".inst 0x6f82e0f1 // udot v17.4s, v7.16b, v2.4b[0]\n"
+ ".inst 0x6f83e0f5 // udot v21.4s, v7.16b, v3.4b[0]\n"
+ ".inst 0x6f84e0f9 // udot v25.4s, v7.16b, v4.4b[0]\n"
+ ".inst 0x6f85e0fd // udot v29.4s, v7.16b, v5.4b[0]\n"
+ "ldr q7, [x14, #0x30]\n"
+ ".inst 0x6f80e0ca // udot v10.4s, v6.16b, v0.4b[0]\n"
+ "add x14, x14, #0x40\n"
+ ".inst 0x6f81e0ce // udot v14.4s, v6.16b, v1.4b[0]\n"
+ ".inst 0x6f82e0d2 // udot v18.4s, v6.16b, v2.4b[0]\n"
+ ".inst 0x6f83e0d6 // udot v22.4s, v6.16b, v3.4b[0]\n"
+ ".inst 0x6f84e0da // udot v26.4s, v6.16b, v4.4b[0]\n"
+ ".inst 0x6f85e0de // udot v30.4s, v6.16b, v5.4b[0]\n"
+ ".inst 0x6f80e0eb // udot v11.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x6f81e0ef // udot v15.4s, v7.16b, v1.4b[0]\n"
+ ".inst 0x6f82e0f3 // udot v19.4s, v7.16b, v2.4b[0]\n"
+ ".inst 0x6f83e0f7 // udot v23.4s, v7.16b, v3.4b[0]\n"
+ ".inst 0x6f84e0fb // udot v27.4s, v7.16b, v4.4b[0]\n"
+ ".inst 0x6f85e0ff // udot v31.4s, v7.16b, v5.4b[0]\n"
+ "bge 196b\n"
+ "cbz x11, 200f\n"
+ "197:" // Height 6: Multiply loop: Skip odd blocks
+ "tbz x11, #1, 198f\n"
+ "ldr h0, [x10], #0x2\n"
+ "ldr h1, [x28], #0x2\n"
+ "ldr h2, [x26], #0x2\n"
+ "ldr h3, [x24], #0x2\n"
+ "ldr h4, [x22], #0x2\n"
+ "ldr h5, [x20], #0x2\n"
+ "tbz x11, #0, 199f\n"
+ "ld1 { v0.b }[2], [x10]\n"
+ "ld1 { v1.b }[2], [x28]\n"
+ "ld1 { v2.b }[2], [x26]\n"
+ "ld1 { v3.b }[2], [x24]\n"
+ "ld1 { v4.b }[2], [x22]\n"
+ "ld1 { v5.b }[2], [x20]\n"
+ "b 199f\n"
+ "198:" // Height 6: Multiply loop: Ragged operand read: partial_1_0
+ "ldr b0, [x10, #0x0]\n"
+ "ldr b1, [x28, #0x0]\n"
+ "ldr b2, [x26, #0x0]\n"
+ "ldr b3, [x24, #0x0]\n"
+ "ldr b4, [x22, #0x0]\n"
+ "ldr b5, [x20, #0x0]\n"
+ "199:" // Height 6: Multiply loop: Ragged operand read: Done
+ "ldr q6, [x14, #0x0]\n"
+ ".inst 0x6f80e0c8 // udot v8.4s, v6.16b, v0.4b[0]\n"
+ "ldr q7, [x14, #0x10]\n"
+ ".inst 0x6f81e0cc // udot v12.4s, v6.16b, v1.4b[0]\n"
+ ".inst 0x6f82e0d0 // udot v16.4s, v6.16b, v2.4b[0]\n"
+ ".inst 0x6f83e0d4 // udot v20.4s, v6.16b, v3.4b[0]\n"
+ ".inst 0x6f84e0d8 // udot v24.4s, v6.16b, v4.4b[0]\n"
+ ".inst 0x6f85e0dc // udot v28.4s, v6.16b, v5.4b[0]\n"
+ "ldr q6, [x14, #0x20]\n"
+ ".inst 0x6f80e0e9 // udot v9.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x6f81e0ed // udot v13.4s, v7.16b, v1.4b[0]\n"
+ ".inst 0x6f82e0f1 // udot v17.4s, v7.16b, v2.4b[0]\n"
+ ".inst 0x6f83e0f5 // udot v21.4s, v7.16b, v3.4b[0]\n"
+ ".inst 0x6f84e0f9 // udot v25.4s, v7.16b, v4.4b[0]\n"
+ ".inst 0x6f85e0fd // udot v29.4s, v7.16b, v5.4b[0]\n"
+ "ldr q7, [x14, #0x30]\n"
+ ".inst 0x6f80e0ca // udot v10.4s, v6.16b, v0.4b[0]\n"
+ "add x14, x14, #0x40\n"
+ ".inst 0x6f81e0ce // udot v14.4s, v6.16b, v1.4b[0]\n"
+ ".inst 0x6f82e0d2 // udot v18.4s, v6.16b, v2.4b[0]\n"
+ ".inst 0x6f83e0d6 // udot v22.4s, v6.16b, v3.4b[0]\n"
+ ".inst 0x6f84e0da // udot v26.4s, v6.16b, v4.4b[0]\n"
+ ".inst 0x6f85e0de // udot v30.4s, v6.16b, v5.4b[0]\n"
+ ".inst 0x6f80e0eb // udot v11.4s, v7.16b, v0.4b[0]\n"
+ ".inst 0x6f81e0ef // udot v15.4s, v7.16b, v1.4b[0]\n"
+ ".inst 0x6f82e0f3 // udot v19.4s, v7.16b, v2.4b[0]\n"
+ ".inst 0x6f83e0f7 // udot v23.4s, v7.16b, v3.4b[0]\n"
+ ".inst 0x6f84e0fb // udot v27.4s, v7.16b, v4.4b[0]\n"
+ ".inst 0x6f85e0ff // udot v31.4s, v7.16b, v5.4b[0]\n"
+ "200:" // Height 6: Multiply loop: No odd multiplies
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "add x12, x12, #0x1\n"
+ "cmp x12, x19\n"
+ "bne 190b\n"
+ "prfm pstl1keep, [x13, #0x0]\n"
+ "prfm pstl1keep, [x9, #0x0]\n"
+ "cmp x15, #0x10\n"
+ "prfm pstl1keep, [x27, #0x0]\n"
+ "prfm pstl1keep, [x25, #0x0]\n"
+ "prfm pstl1keep, [x23, #0x0]\n"
+ "prfm pstl1keep, [x21, #0x0]\n"
+ "bge 209f\n"
+ "tbz x15, #3, 204f\n"
+ "st1 { v8.4s }, [x13], #0x10\n"
+ "st1 { v9.4s }, [x13], #0x10\n"
+ "st1 { v12.4s }, [x9], #0x10\n"
+ "st1 { v13.4s }, [x9], #0x10\n"
+ "st1 { v16.4s }, [x27], #0x10\n"
+ "st1 { v17.4s }, [x27], #0x10\n"
+ "st1 { v20.4s }, [x25], #0x10\n"
+ "st1 { v21.4s }, [x25], #0x10\n"
+ "st1 { v24.4s }, [x23], #0x10\n"
+ "st1 { v25.4s }, [x23], #0x10\n"
+ "st1 { v28.4s }, [x21], #0x10\n"
+ "st1 { v29.4s }, [x21], #0x10\n"
+ "tbz x15, #2, 202f\n"
+ "st1 { v10.4s }, [x13], #0x10\n"
+ "st1 { v14.4s }, [x9], #0x10\n"
+ "st1 { v18.4s }, [x27], #0x10\n"
+ "st1 { v22.4s }, [x25], #0x10\n"
+ "st1 { v26.4s }, [x23], #0x10\n"
+ "st1 { v30.4s }, [x21], #0x10\n"
+ "tbz x15, #1, 201f\n"
+ "str d11, [x13], #0x8\n"
+ "str d15, [x9], #0x8\n"
+ "str d19, [x27], #0x8\n"
+ "str d23, [x25], #0x8\n"
+ "str d27, [x23], #0x8\n"
+ "str d31, [x21], #0x8\n"
+ "tbz x15, #0, 208f\n"
+ "st1 { v11.s }[2], [x13]\n"
+ "st1 { v15.s }[2], [x9]\n"
+ "st1 { v19.s }[2], [x27]\n"
+ "st1 { v23.s }[2], [x25]\n"
+ "st1 { v27.s }[2], [x23]\n"
+ "st1 { v31.s }[2], [x21]\n"
+ "b 208f\n"
+ "201:" // Height 6: Partial direct writeback: partial_1_12
+ "tbz x15, #0, 208f\n"
+ "str s11, [x13, #0x0]\n"
+ "str s15, [x9, #0x0]\n"
+ "str s19, [x27, #0x0]\n"
+ "str s23, [x25, #0x0]\n"
+ "str s27, [x23, #0x0]\n"
+ "str s31, [x21, #0x0]\n"
+ "b 208f\n"
+ "202:" // Height 6: Partial direct writeback: partial_2_8
+ "tbz x15, #1, 203f\n"
+ "str d10, [x13], #0x8\n"
+ "str d14, [x9], #0x8\n"
+ "str d18, [x27], #0x8\n"
+ "str d22, [x25], #0x8\n"
+ "str d26, [x23], #0x8\n"
+ "str d30, [x21], #0x8\n"
+ "tbz x15, #0, 208f\n"
+ "st1 { v10.s }[2], [x13]\n"
+ "st1 { v14.s }[2], [x9]\n"
+ "st1 { v18.s }[2], [x27]\n"
+ "st1 { v22.s }[2], [x25]\n"
+ "st1 { v26.s }[2], [x23]\n"
+ "st1 { v30.s }[2], [x21]\n"
+ "b 208f\n"
+ "203:" // Height 6: Partial direct writeback: partial_1_8
+ "tbz x15, #0, 208f\n"
+ "str s10, [x13, #0x0]\n"
+ "str s14, [x9, #0x0]\n"
+ "str s18, [x27, #0x0]\n"
+ "str s22, [x25, #0x0]\n"
+ "str s26, [x23, #0x0]\n"
+ "str s30, [x21, #0x0]\n"
+ "b 208f\n"
+ "204:" // Height 6: Partial direct writeback: partial_4_0
+ "tbz x15, #2, 206f\n"
+ "st1 { v8.4s }, [x13], #0x10\n"
+ "st1 { v12.4s }, [x9], #0x10\n"
+ "st1 { v16.4s }, [x27], #0x10\n"
+ "st1 { v20.4s }, [x25], #0x10\n"
+ "st1 { v24.4s }, [x23], #0x10\n"
+ "st1 { v28.4s }, [x21], #0x10\n"
+ "tbz x15, #1, 205f\n"
+ "str d9, [x13], #0x8\n"
+ "str d13, [x9], #0x8\n"
+ "str d17, [x27], #0x8\n"
+ "str d21, [x25], #0x8\n"
+ "str d25, [x23], #0x8\n"
+ "str d29, [x21], #0x8\n"
+ "tbz x15, #0, 208f\n"
+ "st1 { v9.s }[2], [x13]\n"
+ "st1 { v13.s }[2], [x9]\n"
+ "st1 { v17.s }[2], [x27]\n"
+ "st1 { v21.s }[2], [x25]\n"
+ "st1 { v25.s }[2], [x23]\n"
+ "st1 { v29.s }[2], [x21]\n"
+ "b 208f\n"
+ "205:" // Height 6: Partial direct writeback: partial_1_4
+ "tbz x15, #0, 208f\n"
+ "str s9, [x13, #0x0]\n"
+ "str s13, [x9, #0x0]\n"
+ "str s17, [x27, #0x0]\n"
+ "str s21, [x25, #0x0]\n"
+ "str s25, [x23, #0x0]\n"
+ "str s29, [x21, #0x0]\n"
+ "b 208f\n"
+ "206:" // Height 6: Partial direct writeback: partial_2_0
+ "tbz x15, #1, 207f\n"
+ "str d8, [x13], #0x8\n"
+ "str d12, [x9], #0x8\n"
+ "str d16, [x27], #0x8\n"
+ "str d20, [x25], #0x8\n"
+ "str d24, [x23], #0x8\n"
+ "str d28, [x21], #0x8\n"
+ "tbz x15, #0, 208f\n"
+ "st1 { v8.s }[2], [x13]\n"
+ "st1 { v12.s }[2], [x9]\n"
+ "st1 { v16.s }[2], [x27]\n"
+ "st1 { v20.s }[2], [x25]\n"
+ "st1 { v24.s }[2], [x23]\n"
+ "st1 { v28.s }[2], [x21]\n"
+ "b 208f\n"
+ "207:" // Height 6: Partial direct writeback: partial_1_0
+ "str s8, [x13, #0x0]\n"
+ "str s12, [x9, #0x0]\n"
+ "str s16, [x27, #0x0]\n"
+ "str s20, [x25, #0x0]\n"
+ "str s24, [x23, #0x0]\n"
+ "str s28, [x21, #0x0]\n"
+ "208:" // Height 6: Partial direct writeback: Done
+ "b 210f\n"
+ "209:" // Height 6: Full writeback
+ "str q8, [x13, #0x0]\n"
+ "str q9, [x13, #0x10]\n"
+ "str q10, [x13, #0x20]\n"
+ "str q11, [x13, #0x30]\n"
+ "str q12, [x9, #0x0]\n"
+ "str q13, [x9, #0x10]\n"
+ "str q14, [x9, #0x20]\n"
+ "str q15, [x9, #0x30]\n"
+ "str q16, [x27, #0x0]\n"
+ "str q17, [x27, #0x10]\n"
+ "str q18, [x27, #0x20]\n"
+ "str q19, [x27, #0x30]\n"
+ "str q20, [x25, #0x0]\n"
+ "str q21, [x25, #0x10]\n"
+ "str q22, [x25, #0x20]\n"
+ "str q23, [x25, #0x30]\n"
+ "str q24, [x23, #0x0]\n"
+ "str q25, [x23, #0x10]\n"
+ "str q26, [x23, #0x20]\n"
+ "str q27, [x23, #0x30]\n"
+ "str q28, [x21, #0x0]\n"
+ "str q29, [x21, #0x10]\n"
+ "str q30, [x21, #0x20]\n"
+ "str q31, [x21, #0x30]\n"
+ "add x13, x13, #0x40\n"
+ "add x9, x9, #0x40\n"
+ "add x27, x27, #0x40\n"
+ "add x25, x25, #0x40\n"
+ "add x23, x23, #0x40\n"
+ "add x21, x21, #0x40\n"
+ "210:" // Height 6: Writeback done
+ "subs x15, x15, #0x10\n"
+ "bgt 178b\n"
+ "subs %x[M], %x[M], #0x6\n"
+ "beq 212f\n"
+ "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "tbz %x[flags], #3, 211f\n"
+ "add x20, x20, #0x6\n"
+ "str x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "b 1b\n"
+ "211:" // Update direct input
+ "mov x19, #0x6\n"
+ "madd %x[input_ptr], x19, x20, %x[input_ptr]\n"
+ "b 1b\n"
+ "212:" // Exit
+
+ : [M] "+r" (M), [input_ptr] "+r" (input_ptr), [output_ptr] "+r" (output_ptr)
+ : [args_ptr] "r" (&ka), [flags] "r" (flags), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths))
+ : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
+ );
+}
+
+} // namespace arm_gemm
+#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_bf16fp32_dot_12x8.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_bf16fp32_dot_12x8.hpp
deleted file mode 100644
index 95fed86..0000000
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_bf16fp32_dot_12x8.hpp
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Copyright (c) 2019-2020 Arm Limited.
- *
- * SPDX-License-Identifier: MIT
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to
- * deal in the Software without restriction, including without limitation the
- * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
- * sell copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-#pragma once
-
-#ifdef __aarch64__
-
-#include "../bfloat.hpp"
-#include "../std_transforms_fixed.hpp"
-
-namespace arm_gemm {
-
-// Actual kernel implementations
-void a64_interleaved_bf16fp32_dot_12x8(const bfloat16 *, const bfloat16 *, float *, int, int, int);
-void a64_interleaved_bf16fp32_dot_12x8_x1(const bfloat16 *, const bfloat16 *, float *, int, int, int);
-
-class interleaved_bf16fp32_dot_12x8 {
-public:
- typedef bfloat16 operand_type;
- typedef float result_type;
-
- typedef void (*kern_type)(const bfloat16 *, const bfloat16 *, float *, int, int, int);
-
- /* Kernel blocking parameters */
- static unsigned int out_width()
- {
- return 12;
- }
-
- static unsigned int out_height()
- {
- return 8;
- }
-
- static unsigned int k_unroll()
- {
- return 2;
- }
-
- // Use the standard fixed size transforms.
- StdTransformsFixed<operand_type, result_type, 8, 12, 2> transforms = {};
-
- kern_type kernel=a64_interleaved_bf16fp32_dot_12x8;
-
- interleaved_bf16fp32_dot_12x8(const CPUInfo *ci)
- {
- if (ci->get_cpu_model() == CPUModel::X1) {
- kernel = a64_interleaved_bf16fp32_dot_12x8_x1;
- }
- }
-};
-
-} // namespace arm_gemm
-
-#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_bf16fp32_dot_12x8/x1.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_bf16fp32_dot_12x8/x1.cpp
deleted file mode 100644
index 58a5143..0000000
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_bf16fp32_dot_12x8/x1.cpp
+++ /dev/null
@@ -1,328 +0,0 @@
-/*
- * Copyright (c) 2019 Arm Limited.
- *
- * SPDX-License-Identifier: MIT
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to
- * deal in the Software without restriction, including without limitation the
- * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
- * sell copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-#ifdef __aarch64__
-
-#include "../../bfloat.hpp"
-#include "../../asmlib.hpp"
-
-namespace arm_gemm {
-
-void a64_interleaved_bf16fp32_dot_12x8_x1(const bfloat16 *Apanel, const bfloat16 *Bpanel, float *Cpanel, int ablocks, int bblocks, int K) {
- const bfloat16 *a_ptr = Apanel;
- float *c_ptr = Cpanel;
-
- K /= 2;
- const long loops_count = (K / 2) - 1;
- const long tails_count = K % 2;
-
- for (int yb=0; yb<ablocks; yb++) {
- const bfloat16 *a_ptr0 = a_ptr;
- const bfloat16 *b_ptr = Bpanel;
-
- for (int xb=0; xb<bblocks; xb++) {
- a_ptr = a_ptr0;
- long loops = loops_count;
- long tails = tails_count;
-
- __asm __volatile (
- "movi v8.4s, #0\n"
- "ldr q0, [%[a_ptr]]\n"
- "movi v9.4s, #0\n"
- "ldr q2, [%[b_ptr]]\n"
- "movi v10.4s, #0\n"
- "ldr q1, [%[a_ptr], #0x10]\n"
- "movi v11.4s, #0\n"
- "ldr q3, [%[b_ptr], #0x10]\n"
- "movi v12.4s, #0\n"
- "ldr q4, [%[b_ptr], #0x20]\n"
- "movi v13.4s, #0\n"
- "add %[a_ptr], %[a_ptr], #0x20\n"
- "movi v14.4s, #0\n"
- "add %[b_ptr], %[b_ptr], #0x30\n"
- "movi v15.4s, #0\n"
- "movi v16.4s, #0\n"
- "movi v17.4s, #0\n"
- "movi v18.4s, #0\n"
- "movi v19.4s, #0\n"
- "movi v20.4s, #0\n"
- "movi v21.4s, #0\n"
- "movi v22.4s, #0\n"
- "movi v23.4s, #0\n"
- "movi v24.4s, #0\n"
- "movi v25.4s, #0\n"
- "movi v26.4s, #0\n"
- "movi v27.4s, #0\n"
- "movi v28.4s, #0\n"
- "movi v29.4s, #0\n"
- "movi v30.4s, #0\n"
- "movi v31.4s, #0\n"
- "cbz %[loops], 1f\n"
- "2:\n"
- ".inst 0x4f40f048 // bfdot v8.4s, v2.8h, v0.h[0]\n"
- "subs %[loops], %[loops], #0x1\n"
- ".inst 0x4f60f049 // bfdot v9.4s, v2.8h, v0.h[1]\n"
- ".inst 0x4f40f84a // bfdot v10.4s, v2.8h, v0.h[2]\n"
- ".inst 0x4f60f84b // bfdot v11.4s, v2.8h, v0.h[3]\n"
- ".inst 0x4f41f054 // bfdot v20.4s, v2.8h, v1.h[0]\n"
- ".inst 0x4f61f055 // bfdot v21.4s, v2.8h, v1.h[1]\n"
- ".inst 0x4f41f856 // bfdot v22.4s, v2.8h, v1.h[2]\n"
- ".inst 0x4f61f857 // bfdot v23.4s, v2.8h, v1.h[3]\n"
- "ldr q2, [%[b_ptr]]\n"
- ".inst 0x4f40f06c // bfdot v12.4s, v3.8h, v0.h[0]\n"
- ".inst 0x4f60f06d // bfdot v13.4s, v3.8h, v0.h[1]\n"
- ".inst 0x4f40f86e // bfdot v14.4s, v3.8h, v0.h[2]\n"
- ".inst 0x4f60f86f // bfdot v15.4s, v3.8h, v0.h[3]\n"
- ".inst 0x4f41f078 // bfdot v24.4s, v3.8h, v1.h[0]\n"
- ".inst 0x4f61f079 // bfdot v25.4s, v3.8h, v1.h[1]\n"
- ".inst 0x4f41f87a // bfdot v26.4s, v3.8h, v1.h[2]\n"
- ".inst 0x4f61f87b // bfdot v27.4s, v3.8h, v1.h[3]\n"
- "ldr q3, [%[b_ptr], #0x10]\n"
- ".inst 0x4f40f090 // bfdot v16.4s, v4.8h, v0.h[0]\n"
- ".inst 0x4f60f091 // bfdot v17.4s, v4.8h, v0.h[1]\n"
- ".inst 0x4f40f892 // bfdot v18.4s, v4.8h, v0.h[2]\n"
- ".inst 0x4f60f893 // bfdot v19.4s, v4.8h, v0.h[3]\n"
- "ldr q0, [%[a_ptr]]\n"
- ".inst 0x4f41f09c // bfdot v28.4s, v4.8h, v1.h[0]\n"
- ".inst 0x4f61f09d // bfdot v29.4s, v4.8h, v1.h[1]\n"
- ".inst 0x4f41f89e // bfdot v30.4s, v4.8h, v1.h[2]\n"
- ".inst 0x4f61f89f // bfdot v31.4s, v4.8h, v1.h[3]\n"
- "ldr q4, [%[b_ptr], #0x20]\n"
- ".inst 0x4f40f048 // bfdot v8.4s, v2.8h, v0.h[0]\n"
- "ldr q1, [%[a_ptr], #0x10]\n"
- ".inst 0x4f60f049 // bfdot v9.4s, v2.8h, v0.h[1]\n"
- "add %[a_ptr], %[a_ptr], #0x40\n"
- ".inst 0x4f40f84a // bfdot v10.4s, v2.8h, v0.h[2]\n"
- "add %[b_ptr], %[b_ptr], #0x60\n"
- ".inst 0x4f60f84b // bfdot v11.4s, v2.8h, v0.h[3]\n"
- ".inst 0x4f41f054 // bfdot v20.4s, v2.8h, v1.h[0]\n"
- ".inst 0x4f61f055 // bfdot v21.4s, v2.8h, v1.h[1]\n"
- ".inst 0x4f41f856 // bfdot v22.4s, v2.8h, v1.h[2]\n"
- ".inst 0x4f61f857 // bfdot v23.4s, v2.8h, v1.h[3]\n"
- "ldr q2, [%[b_ptr], #-0x30]\n"
- ".inst 0x4f40f06c // bfdot v12.4s, v3.8h, v0.h[0]\n"
- ".inst 0x4f60f06d // bfdot v13.4s, v3.8h, v0.h[1]\n"
- ".inst 0x4f40f86e // bfdot v14.4s, v3.8h, v0.h[2]\n"
- ".inst 0x4f60f86f // bfdot v15.4s, v3.8h, v0.h[3]\n"
- ".inst 0x4f41f078 // bfdot v24.4s, v3.8h, v1.h[0]\n"
- ".inst 0x4f61f079 // bfdot v25.4s, v3.8h, v1.h[1]\n"
- ".inst 0x4f41f87a // bfdot v26.4s, v3.8h, v1.h[2]\n"
- ".inst 0x4f61f87b // bfdot v27.4s, v3.8h, v1.h[3]\n"
- "ldr q3, [%[b_ptr], #-0x20]\n"
- ".inst 0x4f40f090 // bfdot v16.4s, v4.8h, v0.h[0]\n"
- ".inst 0x4f60f091 // bfdot v17.4s, v4.8h, v0.h[1]\n"
- ".inst 0x4f40f892 // bfdot v18.4s, v4.8h, v0.h[2]\n"
- ".inst 0x4f60f893 // bfdot v19.4s, v4.8h, v0.h[3]\n"
- "ldr q0, [%[a_ptr], #-0x20]\n"
- ".inst 0x4f41f09c // bfdot v28.4s, v4.8h, v1.h[0]\n"
- ".inst 0x4f61f09d // bfdot v29.4s, v4.8h, v1.h[1]\n"
- ".inst 0x4f41f89e // bfdot v30.4s, v4.8h, v1.h[2]\n"
- ".inst 0x4f61f89f // bfdot v31.4s, v4.8h, v1.h[3]\n"
- "ldr q4, [%[b_ptr], #-0x10]\n"
- "ldr q1, [%[a_ptr], #-0x10]\n"
- "b.ne 2b\n"
- "1:\n"
- "cbz %[tails], 3f\n"
- ".inst 0x4f40f048 // bfdot v8.4s, v2.8h, v0.h[0]\n"
- ".inst 0x4f60f049 // bfdot v9.4s, v2.8h, v0.h[1]\n"
- ".inst 0x4f40f84a // bfdot v10.4s, v2.8h, v0.h[2]\n"
- ".inst 0x4f60f84b // bfdot v11.4s, v2.8h, v0.h[3]\n"
- ".inst 0x4f41f054 // bfdot v20.4s, v2.8h, v1.h[0]\n"
- ".inst 0x4f61f055 // bfdot v21.4s, v2.8h, v1.h[1]\n"
- ".inst 0x4f41f856 // bfdot v22.4s, v2.8h, v1.h[2]\n"
- ".inst 0x4f61f857 // bfdot v23.4s, v2.8h, v1.h[3]\n"
- "ldr q2, [%[b_ptr]]\n"
- ".inst 0x4f40f06c // bfdot v12.4s, v3.8h, v0.h[0]\n"
- ".inst 0x4f60f06d // bfdot v13.4s, v3.8h, v0.h[1]\n"
- ".inst 0x4f40f86e // bfdot v14.4s, v3.8h, v0.h[2]\n"
- ".inst 0x4f60f86f // bfdot v15.4s, v3.8h, v0.h[3]\n"
- ".inst 0x4f41f078 // bfdot v24.4s, v3.8h, v1.h[0]\n"
- ".inst 0x4f61f079 // bfdot v25.4s, v3.8h, v1.h[1]\n"
- ".inst 0x4f41f87a // bfdot v26.4s, v3.8h, v1.h[2]\n"
- ".inst 0x4f61f87b // bfdot v27.4s, v3.8h, v1.h[3]\n"
- "ldr q3, [%[b_ptr], #0x10]\n"
- ".inst 0x4f40f090 // bfdot v16.4s, v4.8h, v0.h[0]\n"
- ".inst 0x4f60f091 // bfdot v17.4s, v4.8h, v0.h[1]\n"
- ".inst 0x4f40f892 // bfdot v18.4s, v4.8h, v0.h[2]\n"
- ".inst 0x4f60f893 // bfdot v19.4s, v4.8h, v0.h[3]\n"
- "ldr q0, [%[a_ptr]]\n"
- ".inst 0x4f41f09c // bfdot v28.4s, v4.8h, v1.h[0]\n"
- ".inst 0x4f61f09d // bfdot v29.4s, v4.8h, v1.h[1]\n"
- ".inst 0x4f41f89e // bfdot v30.4s, v4.8h, v1.h[2]\n"
- ".inst 0x4f61f89f // bfdot v31.4s, v4.8h, v1.h[3]\n"
- "ldr q4, [%[b_ptr], #0x20]\n"
- ".inst 0x4f40f048 // bfdot v8.4s, v2.8h, v0.h[0]\n"
- "ldr q1, [%[a_ptr], #0x10]\n"
- ".inst 0x4f60f049 // bfdot v9.4s, v2.8h, v0.h[1]\n"
- "add %[a_ptr], %[a_ptr], #0x40\n"
- ".inst 0x4f40f84a // bfdot v10.4s, v2.8h, v0.h[2]\n"
- "add %[b_ptr], %[b_ptr], #0x60\n"
- ".inst 0x4f60f84b // bfdot v11.4s, v2.8h, v0.h[3]\n"
- ".inst 0x4f41f054 // bfdot v20.4s, v2.8h, v1.h[0]\n"
- ".inst 0x4f61f055 // bfdot v21.4s, v2.8h, v1.h[1]\n"
- ".inst 0x4f41f856 // bfdot v22.4s, v2.8h, v1.h[2]\n"
- ".inst 0x4f61f857 // bfdot v23.4s, v2.8h, v1.h[3]\n"
- "ldr q2, [%[b_ptr], #-0x30]\n"
- ".inst 0x4f40f06c // bfdot v12.4s, v3.8h, v0.h[0]\n"
- ".inst 0x4f60f06d // bfdot v13.4s, v3.8h, v0.h[1]\n"
- ".inst 0x4f40f86e // bfdot v14.4s, v3.8h, v0.h[2]\n"
- ".inst 0x4f60f86f // bfdot v15.4s, v3.8h, v0.h[3]\n"
- ".inst 0x4f41f078 // bfdot v24.4s, v3.8h, v1.h[0]\n"
- ".inst 0x4f61f079 // bfdot v25.4s, v3.8h, v1.h[1]\n"
- ".inst 0x4f41f87a // bfdot v26.4s, v3.8h, v1.h[2]\n"
- ".inst 0x4f61f87b // bfdot v27.4s, v3.8h, v1.h[3]\n"
- "ldr q3, [%[b_ptr], #-0x20]\n"
- ".inst 0x4f40f090 // bfdot v16.4s, v4.8h, v0.h[0]\n"
- ".inst 0x4f60f091 // bfdot v17.4s, v4.8h, v0.h[1]\n"
- ".inst 0x4f40f892 // bfdot v18.4s, v4.8h, v0.h[2]\n"
- ".inst 0x4f60f893 // bfdot v19.4s, v4.8h, v0.h[3]\n"
- "ldr q0, [%[a_ptr], #-0x20]\n"
- ".inst 0x4f41f09c // bfdot v28.4s, v4.8h, v1.h[0]\n"
- ".inst 0x4f61f09d // bfdot v29.4s, v4.8h, v1.h[1]\n"
- ".inst 0x4f41f89e // bfdot v30.4s, v4.8h, v1.h[2]\n"
- ".inst 0x4f61f89f // bfdot v31.4s, v4.8h, v1.h[3]\n"
- "ldr q4, [%[b_ptr], #-0x10]\n"
- ".inst 0x4f40f048 // bfdot v8.4s, v2.8h, v0.h[0]\n"
- "ldr q1, [%[a_ptr], #-0x10]\n"
- ".inst 0x4f60f049 // bfdot v9.4s, v2.8h, v0.h[1]\n"
- ".inst 0x4f40f84a // bfdot v10.4s, v2.8h, v0.h[2]\n"
- ".inst 0x4f60f84b // bfdot v11.4s, v2.8h, v0.h[3]\n"
- "str q8, [%[c_ptr]]\n"
- ".inst 0x4f41f054 // bfdot v20.4s, v2.8h, v1.h[0]\n"
- ".inst 0x4f61f055 // bfdot v21.4s, v2.8h, v1.h[1]\n"
- ".inst 0x4f41f856 // bfdot v22.4s, v2.8h, v1.h[2]\n"
- ".inst 0x4f61f857 // bfdot v23.4s, v2.8h, v1.h[3]\n"
- ".inst 0x4f40f06c // bfdot v12.4s, v3.8h, v0.h[0]\n"
- ".inst 0x4f60f06d // bfdot v13.4s, v3.8h, v0.h[1]\n"
- ".inst 0x4f40f86e // bfdot v14.4s, v3.8h, v0.h[2]\n"
- ".inst 0x4f60f86f // bfdot v15.4s, v3.8h, v0.h[3]\n"
- "str q12, [%[c_ptr], #0x10]\n"
- ".inst 0x4f41f078 // bfdot v24.4s, v3.8h, v1.h[0]\n"
- ".inst 0x4f61f079 // bfdot v25.4s, v3.8h, v1.h[1]\n"
- ".inst 0x4f41f87a // bfdot v26.4s, v3.8h, v1.h[2]\n"
- ".inst 0x4f61f87b // bfdot v27.4s, v3.8h, v1.h[3]\n"
- ".inst 0x4f40f090 // bfdot v16.4s, v4.8h, v0.h[0]\n"
- ".inst 0x4f60f091 // bfdot v17.4s, v4.8h, v0.h[1]\n"
- ".inst 0x4f40f892 // bfdot v18.4s, v4.8h, v0.h[2]\n"
- ".inst 0x4f60f893 // bfdot v19.4s, v4.8h, v0.h[3]\n"
- "str q16, [%[c_ptr], #0x20]\n"
- ".inst 0x4f41f09c // bfdot v28.4s, v4.8h, v1.h[0]\n"
- ".inst 0x4f61f09d // bfdot v29.4s, v4.8h, v1.h[1]\n"
- ".inst 0x4f41f89e // bfdot v30.4s, v4.8h, v1.h[2]\n"
- "str q9, [%[c_ptr], #0x30]\n"
- ".inst 0x4f61f89f // bfdot v31.4s, v4.8h, v1.h[3]\n"
- "b 4f\n"
- "3:\n"
- ".inst 0x4f40f048 // bfdot v8.4s, v2.8h, v0.h[0]\n"
- "add %[a_ptr], %[a_ptr], #0x20\n"
- ".inst 0x4f60f049 // bfdot v9.4s, v2.8h, v0.h[1]\n"
- "add %[b_ptr], %[b_ptr], #0x30\n"
- ".inst 0x4f40f84a // bfdot v10.4s, v2.8h, v0.h[2]\n"
- ".inst 0x4f60f84b // bfdot v11.4s, v2.8h, v0.h[3]\n"
- ".inst 0x4f41f054 // bfdot v20.4s, v2.8h, v1.h[0]\n"
- ".inst 0x4f61f055 // bfdot v21.4s, v2.8h, v1.h[1]\n"
- ".inst 0x4f41f856 // bfdot v22.4s, v2.8h, v1.h[2]\n"
- ".inst 0x4f61f857 // bfdot v23.4s, v2.8h, v1.h[3]\n"
- "ldr q2, [%[b_ptr], #-0x30]\n"
- ".inst 0x4f40f06c // bfdot v12.4s, v3.8h, v0.h[0]\n"
- ".inst 0x4f60f06d // bfdot v13.4s, v3.8h, v0.h[1]\n"
- ".inst 0x4f40f86e // bfdot v14.4s, v3.8h, v0.h[2]\n"
- ".inst 0x4f60f86f // bfdot v15.4s, v3.8h, v0.h[3]\n"
- ".inst 0x4f41f078 // bfdot v24.4s, v3.8h, v1.h[0]\n"
- ".inst 0x4f61f079 // bfdot v25.4s, v3.8h, v1.h[1]\n"
- ".inst 0x4f41f87a // bfdot v26.4s, v3.8h, v1.h[2]\n"
- ".inst 0x4f61f87b // bfdot v27.4s, v3.8h, v1.h[3]\n"
- "ldr q3, [%[b_ptr], #-0x20]\n"
- ".inst 0x4f40f090 // bfdot v16.4s, v4.8h, v0.h[0]\n"
- ".inst 0x4f60f091 // bfdot v17.4s, v4.8h, v0.h[1]\n"
- ".inst 0x4f40f892 // bfdot v18.4s, v4.8h, v0.h[2]\n"
- ".inst 0x4f60f893 // bfdot v19.4s, v4.8h, v0.h[3]\n"
- "ldr q0, [%[a_ptr], #-0x20]\n"
- ".inst 0x4f41f09c // bfdot v28.4s, v4.8h, v1.h[0]\n"
- ".inst 0x4f61f09d // bfdot v29.4s, v4.8h, v1.h[1]\n"
- ".inst 0x4f41f89e // bfdot v30.4s, v4.8h, v1.h[2]\n"
- ".inst 0x4f61f89f // bfdot v31.4s, v4.8h, v1.h[3]\n"
- "ldr q4, [%[b_ptr], #-0x10]\n"
- ".inst 0x4f40f048 // bfdot v8.4s, v2.8h, v0.h[0]\n"
- "ldr q1, [%[a_ptr], #-0x10]\n"
- ".inst 0x4f60f049 // bfdot v9.4s, v2.8h, v0.h[1]\n"
- ".inst 0x4f40f84a // bfdot v10.4s, v2.8h, v0.h[2]\n"
- ".inst 0x4f60f84b // bfdot v11.4s, v2.8h, v0.h[3]\n"
- "str q8, [%[c_ptr]]\n"
- ".inst 0x4f41f054 // bfdot v20.4s, v2.8h, v1.h[0]\n"
- ".inst 0x4f61f055 // bfdot v21.4s, v2.8h, v1.h[1]\n"
- ".inst 0x4f41f856 // bfdot v22.4s, v2.8h, v1.h[2]\n"
- ".inst 0x4f61f857 // bfdot v23.4s, v2.8h, v1.h[3]\n"
- ".inst 0x4f40f06c // bfdot v12.4s, v3.8h, v0.h[0]\n"
- ".inst 0x4f60f06d // bfdot v13.4s, v3.8h, v0.h[1]\n"
- ".inst 0x4f40f86e // bfdot v14.4s, v3.8h, v0.h[2]\n"
- ".inst 0x4f60f86f // bfdot v15.4s, v3.8h, v0.h[3]\n"
- "str q12, [%[c_ptr], #0x10]\n"
- ".inst 0x4f41f078 // bfdot v24.4s, v3.8h, v1.h[0]\n"
- ".inst 0x4f61f079 // bfdot v25.4s, v3.8h, v1.h[1]\n"
- ".inst 0x4f41f87a // bfdot v26.4s, v3.8h, v1.h[2]\n"
- ".inst 0x4f61f87b // bfdot v27.4s, v3.8h, v1.h[3]\n"
- ".inst 0x4f40f090 // bfdot v16.4s, v4.8h, v0.h[0]\n"
- ".inst 0x4f60f091 // bfdot v17.4s, v4.8h, v0.h[1]\n"
- ".inst 0x4f40f892 // bfdot v18.4s, v4.8h, v0.h[2]\n"
- ".inst 0x4f60f893 // bfdot v19.4s, v4.8h, v0.h[3]\n"
- "str q16, [%[c_ptr], #0x20]\n"
- ".inst 0x4f41f09c // bfdot v28.4s, v4.8h, v1.h[0]\n"
- ".inst 0x4f61f09d // bfdot v29.4s, v4.8h, v1.h[1]\n"
- ".inst 0x4f41f89e // bfdot v30.4s, v4.8h, v1.h[2]\n"
- "str q9, [%[c_ptr], #0x30]\n"
- ".inst 0x4f61f89f // bfdot v31.4s, v4.8h, v1.h[3]\n"
- "4:\n"
- "str q13, [%[c_ptr], #0x40]\n"
- "str q17, [%[c_ptr], #0x50]\n"
- "str q10, [%[c_ptr], #0x60]\n"
- "str q14, [%[c_ptr], #0x70]\n"
- "str q18, [%[c_ptr], #0x80]\n"
- "str q11, [%[c_ptr], #0x90]\n"
- "str q15, [%[c_ptr], #0xa0]\n"
- "str q19, [%[c_ptr], #0xb0]\n"
- "str q20, [%[c_ptr], #0xc0]\n"
- "str q24, [%[c_ptr], #0xd0]\n"
- "str q28, [%[c_ptr], #0xe0]\n"
- "str q21, [%[c_ptr], #0xf0]\n"
- "str q25, [%[c_ptr], #0x100]\n"
- "str q29, [%[c_ptr], #0x110]\n"
- "str q22, [%[c_ptr], #0x120]\n"
- "str q26, [%[c_ptr], #0x130]\n"
- "str q30, [%[c_ptr], #0x140]\n"
- "str q23, [%[c_ptr], #0x150]\n"
- "str q27, [%[c_ptr], #0x160]\n"
- "str q31, [%[c_ptr], #0x170]\n"
- "add %[c_ptr], %[c_ptr], #0x180\n"
- : [a_ptr] "+r" (a_ptr), [b_ptr] "+r" (b_ptr), [c_ptr] "+r" (c_ptr),
- [loops] "+r" (loops), [tails] "+r" (tails)
- :
- : "v0", "v1", "v2", "v3", "v4", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "cc", "memory"
- );
- }
- }
-}
-
-} // namespace arm_gemm
-
-#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_bf16fp32_mmla_12x8.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_bf16fp32_dot_8x12.hpp
similarity index 83%
copy from src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_bf16fp32_mmla_12x8.hpp
copy to src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_bf16fp32_dot_8x12.hpp
index 7fac599..2fea5ad2 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_bf16fp32_mmla_12x8.hpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_bf16fp32_dot_8x12.hpp
@@ -31,9 +31,9 @@
namespace arm_gemm {
// Actual kernel implementations
-void a64_interleaved_bf16fp32_mmla_12x8(const bfloat16 *, const bfloat16 *, float *, int, int, int);
+void a64_interleaved_bf16fp32_dot_8x12(const bfloat16 *, const bfloat16 *, float *, int, int, int);
-class interleaved_bf16fp32_mmla_12x8 {
+class cls_a64_interleaved_bf16fp32_dot_8x12 {
public:
typedef bfloat16 operand_type;
typedef float result_type;
@@ -53,15 +53,15 @@
static unsigned int k_unroll()
{
- return 4;
+ return 2;
}
// Use the standard fixed size transforms.
- StdTransformsFixed<operand_type, result_type, 8, 12, 4> transforms = {};
+ StdTransformsFixed<operand_type, result_type, 8, 12, 2> transforms = {};
- kern_type kernel=a64_interleaved_bf16fp32_mmla_12x8;
+ kern_type kernel=a64_interleaved_bf16fp32_dot_8x12;
- interleaved_bf16fp32_mmla_12x8(const CPUInfo *)
+ cls_a64_interleaved_bf16fp32_dot_8x12(const CPUInfo *)
{
}
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_bf16fp32_dot_12x8/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_bf16fp32_dot_8x12/generic.cpp
similarity index 99%
rename from src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_bf16fp32_dot_12x8/generic.cpp
rename to src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_bf16fp32_dot_8x12/generic.cpp
index 7ffae52..92149a5 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_bf16fp32_dot_12x8/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_bf16fp32_dot_8x12/generic.cpp
@@ -28,7 +28,7 @@
namespace arm_gemm {
-void a64_interleaved_bf16fp32_dot_12x8(const bfloat16 *Apanel, const bfloat16 *Bpanel, float *Cpanel, int ablocks, int bblocks, int K) {
+void a64_interleaved_bf16fp32_dot_8x12(const bfloat16 *Apanel, const bfloat16 *Bpanel, float *Cpanel, int ablocks, int bblocks, int K) {
const bfloat16 *a_ptr = Apanel;
float *c_ptr = Cpanel;
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_bf16fp32_mmla_12x8.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_bf16fp32_mmla_8x12.hpp
similarity index 89%
rename from src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_bf16fp32_mmla_12x8.hpp
rename to src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_bf16fp32_mmla_8x12.hpp
index 7fac599..b2c2407 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_bf16fp32_mmla_12x8.hpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_bf16fp32_mmla_8x12.hpp
@@ -31,9 +31,9 @@
namespace arm_gemm {
// Actual kernel implementations
-void a64_interleaved_bf16fp32_mmla_12x8(const bfloat16 *, const bfloat16 *, float *, int, int, int);
+void a64_interleaved_bf16fp32_mmla_8x12(const bfloat16 *, const bfloat16 *, float *, int, int, int);
-class interleaved_bf16fp32_mmla_12x8 {
+class cls_a64_interleaved_bf16fp32_mmla_8x12 {
public:
typedef bfloat16 operand_type;
typedef float result_type;
@@ -59,9 +59,9 @@
// Use the standard fixed size transforms.
StdTransformsFixed<operand_type, result_type, 8, 12, 4> transforms = {};
- kern_type kernel=a64_interleaved_bf16fp32_mmla_12x8;
+ kern_type kernel=a64_interleaved_bf16fp32_mmla_8x12;
- interleaved_bf16fp32_mmla_12x8(const CPUInfo *)
+ cls_a64_interleaved_bf16fp32_mmla_8x12(const CPUInfo *)
{
}
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_bf16fp32_mmla_12x8/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_bf16fp32_mmla_8x12/generic.cpp
similarity index 96%
rename from src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_bf16fp32_mmla_12x8/generic.cpp
rename to src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_bf16fp32_mmla_8x12/generic.cpp
index 7f0eff2..c476fcf 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_bf16fp32_mmla_12x8/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_bf16fp32_mmla_8x12/generic.cpp
@@ -28,7 +28,7 @@
namespace arm_gemm {
-void a64_interleaved_bf16fp32_mmla_12x8(const bfloat16 *Apanel, const bfloat16 *Bpanel, float *Cpanel, int ablocks, int bblocks, int K) {
+void a64_interleaved_bf16fp32_mmla_8x12(const bfloat16 *Apanel, const bfloat16 *Bpanel, float *Cpanel, int ablocks, int bblocks, int K) {
const bfloat16 *a_ptr = Apanel;
float *c_ptr = Cpanel;
@@ -87,13 +87,23 @@
"movi v27.4s, #0\n"
"prfm PLDL1KEEP, [%[b_ptr], #0x1c0]\n"
"movi v28.4s, #0\n"
- "prfm PLDL1KEEP, [%[b_ptr], #0x200]\n"
+ "prfm PLDL1KEEP, [%[a_ptr], #0x200]\n"
"movi v29.4s, #0\n"
- "prfm PLDL1KEEP, [%[b_ptr], #0x240]\n"
+ "prfm PLDL1KEEP, [%[b_ptr], #0x200]\n"
"movi v30.4s, #0\n"
- "prfm PLDL1KEEP, [%[b_ptr], #0x280]\n"
+ "prfm PLDL1KEEP, [%[a_ptr], #0x240]\n"
"movi v31.4s, #0\n"
+ "prfm PLDL1KEEP, [%[b_ptr], #0x240]\n"
+ "prfm PLDL1KEEP, [%[a_ptr], #0x280]\n"
+ "prfm PLDL1KEEP, [%[b_ptr], #0x280]\n"
+ "prfm PLDL1KEEP, [%[a_ptr], #0x2c0]\n"
"prfm PLDL1KEEP, [%[b_ptr], #0x2c0]\n"
+ "prfm PLDL1KEEP, [%[b_ptr], #0x300]\n"
+ "prfm PLDL1KEEP, [%[b_ptr], #0x340]\n"
+ "prfm PLDL1KEEP, [%[b_ptr], #0x380]\n"
+ "prfm PLDL1KEEP, [%[b_ptr], #0x3c0]\n"
+ "prfm PLDL1KEEP, [%[b_ptr], #0x400]\n"
+ "prfm PLDL1KEEP, [%[b_ptr], #0x440]\n"
"add %[a_ptr], %[a_ptr], #0x40\n"
"add %[b_ptr], %[b_ptr], #0x40\n"
"cbz %[loops], 1f\n"
@@ -105,19 +115,19 @@
".inst 0x6e44ec54 // bfmmla v20.4s, v2.8h, v4.8h\n"
"subs %[loops], %[loops], #0x1\n"
".inst 0x6e45ec09 // bfmmla v9.4s, v0.8h, v5.8h\n"
- "prfm PLDL1KEEP, [%[a_ptr], #0x1c0]\n"
+ "prfm PLDL1KEEP, [%[a_ptr], #0x2c0]\n"
".inst 0x6e44ec7a // bfmmla v26.4s, v3.8h, v4.8h\n"
"ldr q4, [%[b_ptr]]\n"
".inst 0x6e45ec2f // bfmmla v15.4s, v1.8h, v5.8h\n"
- "prfm PLDL1KEEP, [%[b_ptr], #0x2c0]\n"
+ "prfm PLDL1KEEP, [%[b_ptr], #0x440]\n"
".inst 0x6e45ec55 // bfmmla v21.4s, v2.8h, v5.8h\n"
- "prfm PLDL1KEEP, [%[a_ptr], #0x200]\n"
+ "prfm PLDL1KEEP, [%[a_ptr], #0x300]\n"
".inst 0x6e45ec7b // bfmmla v27.4s, v3.8h, v5.8h\n"
"ldr q5, [%[b_ptr], #0x10]\n"
".inst 0x6e46ec0a // bfmmla v10.4s, v0.8h, v6.8h\n"
- "prfm PLDL1KEEP, [%[b_ptr], #0x300]\n"
+ "prfm PLDL1KEEP, [%[b_ptr], #0x480]\n"
".inst 0x6e46ec30 // bfmmla v16.4s, v1.8h, v6.8h\n"
- "prfm PLDL1KEEP, [%[b_ptr], #0x340]\n"
+ "prfm PLDL1KEEP, [%[b_ptr], #0x4c0]\n"
".inst 0x6e46ec56 // bfmmla v22.4s, v2.8h, v6.8h\n"
".inst 0x6e46ec7c // bfmmla v28.4s, v3.8h, v6.8h\n"
"ldr q6, [%[b_ptr], #0x20]\n"
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_s8s32_mmla_12x8.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_s8s32_mmla_8x12.hpp
similarity index 85%
rename from src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_s8s32_mmla_12x8.hpp
rename to src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_s8s32_mmla_8x12.hpp
index 7bfb229..b17b76f 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_s8s32_mmla_12x8.hpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_s8s32_mmla_8x12.hpp
@@ -31,9 +31,9 @@
namespace arm_gemm {
// Actual kernel implementations
-void a64_interleaved_s8s32_mmla_12x8(const int8_t *, const int8_t *, int32_t *, int, int, int);
+void a64_interleaved_s8s32_mmla_8x12(const int8_t *, const int8_t *, int32_t *, int, int, int);
-class interleaved_s8s32_mmla_12x8 {
+class cls_a64_interleaved_s8s32_mmla_8x12 {
public:
typedef int8_t operand_type;
typedef int32_t result_type;
@@ -58,10 +58,11 @@
// Use the standard fixed size transforms.
StdTransformsFixed<operand_type, result_type, 8, 12, 8> transforms = {};
+ StdTransformsFixed<operand_type, result_type, 8, 12, 8, true> transforms_quantized = {};
- kern_type kernel=a64_interleaved_s8s32_mmla_12x8;
+ kern_type kernel=a64_interleaved_s8s32_mmla_8x12;
- interleaved_s8s32_mmla_12x8(const CPUInfo *)
+ cls_a64_interleaved_s8s32_mmla_8x12(const CPUInfo *)
{
}
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_s8s32_mmla_12x8/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_s8s32_mmla_8x12/generic.cpp
similarity index 99%
rename from src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_s8s32_mmla_12x8/generic.cpp
rename to src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_s8s32_mmla_8x12/generic.cpp
index 7953510..2093e75 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_s8s32_mmla_12x8/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_s8s32_mmla_8x12/generic.cpp
@@ -28,7 +28,7 @@
namespace arm_gemm {
-void a64_interleaved_s8s32_mmla_12x8(const int8_t *Apanel, const int8_t *Bpanel, int32_t *Cpanel, int ablocks, int bblocks, int K) {
+void a64_interleaved_s8s32_mmla_8x12(const int8_t *Apanel, const int8_t *Bpanel, int32_t *Cpanel, int ablocks, int bblocks, int K) {
const int8_t *a_ptr = Apanel;
int32_t *c_ptr = Cpanel;
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_u8u32_mmla_12x8.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_u8u32_mmla_8x12.hpp
similarity index 85%
rename from src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_u8u32_mmla_12x8.hpp
rename to src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_u8u32_mmla_8x12.hpp
index d493517..99dd0be 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_u8u32_mmla_12x8.hpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_u8u32_mmla_8x12.hpp
@@ -31,9 +31,9 @@
namespace arm_gemm {
// Actual kernel implementations
-void a64_interleaved_u8u32_mmla_12x8(const uint8_t *, const uint8_t *, uint32_t *, int, int, int);
+void a64_interleaved_u8u32_mmla_8x12(const uint8_t *, const uint8_t *, uint32_t *, int, int, int);
-class interleaved_u8u32_mmla_12x8 {
+class cls_a64_interleaved_u8u32_mmla_8x12 {
public:
typedef uint8_t operand_type;
typedef uint32_t result_type;
@@ -58,10 +58,11 @@
// Use the standard fixed size transforms.
StdTransformsFixed<operand_type, result_type, 8, 12, 8> transforms = {};
+ StdTransformsFixed<operand_type, result_type, 8, 12, 8, true> transforms_quantized = {};
- kern_type kernel=a64_interleaved_u8u32_mmla_12x8;
+ kern_type kernel=a64_interleaved_u8u32_mmla_8x12;
- interleaved_u8u32_mmla_12x8(const CPUInfo *)
+ cls_a64_interleaved_u8u32_mmla_8x12(const CPUInfo *)
{
}
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_u8u32_mmla_12x8/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_u8u32_mmla_8x12/generic.cpp
similarity index 99%
rename from src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_u8u32_mmla_12x8/generic.cpp
rename to src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_u8u32_mmla_8x12/generic.cpp
index dcd15f0..568e5d1 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_u8u32_mmla_12x8/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_u8u32_mmla_8x12/generic.cpp
@@ -28,7 +28,7 @@
namespace arm_gemm {
-void a64_interleaved_u8u32_mmla_12x8(const uint8_t *Apanel, const uint8_t *Bpanel, uint32_t *Cpanel, int ablocks, int bblocks, int K) {
+void a64_interleaved_u8u32_mmla_8x12(const uint8_t *Apanel, const uint8_t *Bpanel, uint32_t *Cpanel, int ablocks, int bblocks, int K) {
const uint8_t *a_ptr = Apanel;
uint32_t *c_ptr = Cpanel;
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_12x8.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_8x12.hpp
similarity index 82%
rename from src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_12x8.hpp
rename to src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_8x12.hpp
index 981ce34..d77e1b0 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_12x8.hpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_8x12.hpp
@@ -30,13 +30,13 @@
namespace arm_gemm {
// Actual kernel implementations
-void a64_sgemm_asimd_12x8(const float *, const float *, float *, int, int, int);
-void a64_sgemm_asimd_12x8_a53(const float *, const float *, float *, int, int, int);
-void a64_sgemm_asimd_12x8_a55(const float *, const float *, float *, int, int, int);
-void a64_sgemm_asimd_12x8_a55r1(const float *, const float *, float *, int, int, int);
-void a64_sgemm_asimd_12x8_x1(const float *, const float *, float *, int, int, int);
+void a64_sgemm_asimd_8x12(const float *, const float *, float *, int, int, int);
+void a64_sgemm_asimd_8x12_a53(const float *, const float *, float *, int, int, int);
+void a64_sgemm_asimd_8x12_a55(const float *, const float *, float *, int, int, int);
+void a64_sgemm_asimd_8x12_a55r1(const float *, const float *, float *, int, int, int);
+void a64_sgemm_asimd_8x12_x1(const float *, const float *, float *, int, int, int);
-// 12x8 SGEMM "strategy" class.
+// 8x12 SGEMM "strategy" class.
//
// This describes the characteristics of a family of kernels, in terms of
// the required interleave properties and the output block size.
@@ -44,7 +44,7 @@
// All kernels in the family must share these characteristics. The actual
// kernel to be used can be chosen at runtime, based on the CPU_type
// structure.
-class sgemm_12x8 {
+class cls_a64_sgemm_8x12 {
public:
typedef float operand_type;
typedef float result_type;
@@ -83,25 +83,25 @@
}
}
- kern_type kernel=a64_sgemm_asimd_12x8;
+ kern_type kernel=a64_sgemm_asimd_8x12;
- sgemm_12x8(const CPUInfo *ci) {
+ cls_a64_sgemm_8x12(const CPUInfo *ci) {
// Select specific kernel if available
switch(ci->get_cpu_model()) {
case CPUModel::A53:
- kernel = a64_sgemm_asimd_12x8_a53;
+ kernel = a64_sgemm_asimd_8x12_a53;
break;
case CPUModel::A55r0:
- kernel = a64_sgemm_asimd_12x8_a55;
+ kernel = a64_sgemm_asimd_8x12_a55;
break;
case CPUModel::A55r1:
- kernel = a64_sgemm_asimd_12x8_a55r1;
+ kernel = a64_sgemm_asimd_8x12_a55r1;
break;
case CPUModel::X1:
- kernel = a64_sgemm_asimd_12x8_x1;
+ kernel = a64_sgemm_asimd_8x12_x1;
break;
default:
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_12x8/a53.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_8x12/a53.cpp
similarity index 99%
rename from src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_12x8/a53.cpp
rename to src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_8x12/a53.cpp
index 5532485..f4b6e7b 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_12x8/a53.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_8x12/a53.cpp
@@ -29,7 +29,7 @@
namespace arm_gemm {
-void a64_sgemm_asimd_12x8_a53(const float *Apanel, const float *Bpanel, float *Cpanel, int ablocks, int bblocks, int K) {
+void a64_sgemm_asimd_8x12_a53(const float *Apanel, const float *Bpanel, float *Cpanel, int ablocks, int bblocks, int K) {
const float *a_ptr = Apanel;
float *c_ptr = Cpanel;
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_12x8/a55.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_8x12/a55.cpp
similarity index 99%
rename from src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_12x8/a55.cpp
rename to src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_8x12/a55.cpp
index e9f071f..5f86da8 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_12x8/a55.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_8x12/a55.cpp
@@ -29,7 +29,7 @@
namespace arm_gemm {
-void a64_sgemm_asimd_12x8_a55(const float *Apanel, const float *Bpanel, float *Cpanel, int ablocks, int bblocks, int K) {
+void a64_sgemm_asimd_8x12_a55(const float *Apanel, const float *Bpanel, float *Cpanel, int ablocks, int bblocks, int K) {
const float *a_ptr = Apanel;
float *c_ptr = Cpanel;
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_12x8/a55r1.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_8x12/a55r1.cpp
similarity index 99%
rename from src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_12x8/a55r1.cpp
rename to src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_8x12/a55r1.cpp
index 8a6fbac..7709ad1 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_12x8/a55r1.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_8x12/a55r1.cpp
@@ -29,7 +29,7 @@
namespace arm_gemm {
-void a64_sgemm_asimd_12x8_a55r1(const float *Apanel, const float *Bpanel, float *Cpanel, const int ablocks, const int bblocks, const int K) {
+void a64_sgemm_asimd_8x12_a55r1(const float *Apanel, const float *Bpanel, float *Cpanel, const int ablocks, const int bblocks, const int K) {
const float *a_ptr = Apanel;
float *c_ptr = Cpanel;
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_12x8/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_8x12/generic.cpp
similarity index 99%
rename from src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_12x8/generic.cpp
rename to src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_8x12/generic.cpp
index 48dc467..dc72095 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_12x8/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_8x12/generic.cpp
@@ -39,7 +39,7 @@
namespace arm_gemm {
-void a64_sgemm_asimd_12x8(const float *Apanel, const float *Bpanel, float *Cpanel, int ablocks, int bblocks, int K) {
+void a64_sgemm_asimd_8x12(const float *Apanel, const float *Bpanel, float *Cpanel, int ablocks, int bblocks, int K) {
const float *a_ptr = Apanel;
float *c_ptr = Cpanel;
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_12x8/x1.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_8x12/x1.cpp
similarity index 99%
rename from src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_12x8/x1.cpp
rename to src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_8x12/x1.cpp
index 63fdf4d..89f8ac2 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_12x8/x1.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_8x12/x1.cpp
@@ -39,7 +39,7 @@
namespace arm_gemm {
-void a64_sgemm_asimd_12x8_x1(const float *Apanel, const float *Bpanel, float *Cpanel, int ablocks, int bblocks, int K) {
+void a64_sgemm_asimd_8x12_x1(const float *Apanel, const float *Bpanel, float *Cpanel, int ablocks, int bblocks, int K) {
const float *a_ptr = Apanel;
float *c_ptr = Cpanel;
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_4x6.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_6x4.hpp
similarity index 90%
rename from src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_4x6.hpp
rename to src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_6x4.hpp
index 6f31efe..5f7252f 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_4x6.hpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_6x4.hpp
@@ -25,13 +25,15 @@
#ifdef __aarch64__
+
+
namespace arm_gemm
{
// Actual kernel implementations
-void a64_smallK_hybrid_fp32_mla_4x6(const float *, int, const float *, float *, int, int, int, int, const float *, Activation, bool);
+void a64_smallK_hybrid_fp32_mla_6x4(const float *, int, const float *, float *, int, int, int, int, const float *, Activation, bool);
-class smallK_hybrid_fp32_mla_4x6
+class cls_a64_smallK_hybrid_fp32_mla_6x4
{
public:
typedef float operand_type;
@@ -73,9 +75,9 @@
StdTransformsFixed<operand_type, result_type, 6, 4, 1> transforms = {};
// Default to the generic kernel
- kern_type kernel=a64_smallK_hybrid_fp32_mla_4x6;
+ kern_type kernel=a64_smallK_hybrid_fp32_mla_6x4;
- smallK_hybrid_fp32_mla_4x6(const CPUInfo *)
+ cls_a64_smallK_hybrid_fp32_mla_6x4(const CPUInfo *)
{
}
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_4x6/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_6x4/generic.cpp
similarity index 99%
rename from src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_4x6/generic.cpp
rename to src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_6x4/generic.cpp
index e2fec6a..52548b4 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_4x6/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_6x4/generic.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2019 Arm Limited.
+ * Copyright (c) 2019-2020 Arm Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -33,7 +33,7 @@
namespace arm_gemm {
-void a64_smallK_hybrid_fp32_mla_4x6(const float *A, int lda, const float *B, float *C, int ldc, int M, int N, int K, const float *bias, Activation act, bool) {
+void a64_smallK_hybrid_fp32_mla_6x4(const float *A, int lda, const float *B, float *C, int ldc, int M, int N, int K, const float *bias, Activation act, bool) {
const long loops_count = iceildiv(N, (int)4) - 1;
const long ldab = lda * sizeof(float);
const long ldcb = ldc * sizeof(float);
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_4x8.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_8x4.hpp
similarity index 90%
rename from src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_4x8.hpp
rename to src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_8x4.hpp
index e9a0948..a8e0c24 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_4x8.hpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_8x4.hpp
@@ -25,13 +25,15 @@
#ifdef __aarch64__
+
+
namespace arm_gemm
{
// Actual kernel implementations
-void a64_smallK_hybrid_fp32_mla_4x8(const float *, int, const float *, float *, int, int, int, int, const float *, Activation, bool);
+void a64_smallK_hybrid_fp32_mla_8x4(const float *, int, const float *, float *, int, int, int, int, const float *, Activation, bool);
-class smallK_hybrid_fp32_mla_4x8
+class cls_a64_smallK_hybrid_fp32_mla_8x4
{
public:
typedef float operand_type;
@@ -73,9 +75,9 @@
StdTransformsFixed<operand_type, result_type, 8, 4, 1> transforms = {};
// Default to the generic kernel
- kern_type kernel=a64_smallK_hybrid_fp32_mla_4x8;
+ kern_type kernel=a64_smallK_hybrid_fp32_mla_8x4;
- smallK_hybrid_fp32_mla_4x8(const CPUInfo *)
+ cls_a64_smallK_hybrid_fp32_mla_8x4(const CPUInfo *)
{
}
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_4x8/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_8x4/generic.cpp
similarity index 99%
rename from src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_4x8/generic.cpp
rename to src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_8x4/generic.cpp
index 11888bc..deaef27 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_4x8/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_fp32_mla_8x4/generic.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2019 Arm Limited.
+ * Copyright (c) 2019-2020 Arm Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -33,7 +33,7 @@
namespace arm_gemm {
-void a64_smallK_hybrid_fp32_mla_4x8(const float *A, int lda, const float *B, float *C, int ldc, int M, int N, int K, const float *bias, Activation act, bool) {
+void a64_smallK_hybrid_fp32_mla_8x4(const float *A, int lda, const float *B, float *C, int ldc, int M, int N, int K, const float *bias, Activation act, bool) {
const long loops_count = iceildiv(N, (int)4) - 1;
const long ldab = lda * sizeof(float);
const long ldcb = ldc * sizeof(float);
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x6.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_6x4.hpp
similarity index 87%
rename from src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x6.hpp
rename to src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_6x4.hpp
index fc087b7..abf0eda 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x6.hpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_6x4.hpp
@@ -31,10 +31,10 @@
{
// Actual kernel implementations
-void a64_smallK_hybrid_s8s32_dot_4x6(const int8_t *, int, const int8_t *, int32_t *, int, int, int, int, const int32_t *, Activation, bool);
-void a64_smallK_hybrid_s8s32_dot_4x6_a55(const int8_t *, int, const int8_t *, int32_t *, int, int, int, int, const int32_t *, Activation, bool);
+void a64_smallK_hybrid_s8s32_dot_6x4(const int8_t *, int, const int8_t *, int32_t *, int, int, int, int, const int32_t *, Activation, bool);
+void a64_smallK_hybrid_s8s32_dot_6x4_a55(const int8_t *, int, const int8_t *, int32_t *, int, int, int, int, const int32_t *, Activation, bool);
-class smallK_hybrid_s8s32_dot_4x6
+class cls_a64_smallK_hybrid_s8s32_dot_6x4
{
public:
typedef int8_t operand_type;
@@ -76,12 +76,12 @@
StdTransformsFixed<operand_type, result_type, 6, 4, 4> transforms = {};
// Default to the generic kernel
- kern_type kernel=a64_smallK_hybrid_s8s32_dot_4x6;
+ kern_type kernel=a64_smallK_hybrid_s8s32_dot_6x4;
- smallK_hybrid_s8s32_dot_4x6(const CPUInfo *ci)
+ cls_a64_smallK_hybrid_s8s32_dot_6x4(const CPUInfo *ci)
{
if (ci->get_cpu_model() == CPUModel::A55r1) {
- kernel = a64_smallK_hybrid_s8s32_dot_4x6_a55;
+ kernel = a64_smallK_hybrid_s8s32_dot_6x4_a55;
}
}
};
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x6/a55.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_6x4/a55.cpp
similarity index 80%
rename from src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x6/a55.cpp
rename to src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_6x4/a55.cpp
index 2d6d2f0..a992660 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x6/a55.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_6x4/a55.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2019 Arm Limited.
+ * Copyright (c) 2019-2020 Arm Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -33,7 +33,7 @@
namespace arm_gemm {
-void a64_smallK_hybrid_s8s32_dot_4x6_a55(const int8_t *A, int lda, const int8_t *B, int32_t *C, int ldc, int M, int N, int K, const int32_t *, Activation, bool) {
+void a64_smallK_hybrid_s8s32_dot_6x4_a55(const int8_t *A, int lda, const int8_t *B, int32_t *C, int ldc, int M, int N, int K, const int32_t *, Activation, bool) {
const long loops_count = iceildiv(N, (int)4) - 1;
const long ldab = lda * sizeof(int8_t);
const long ldcb = ldc * sizeof(int32_t);
@@ -97,6 +97,7 @@
"add c_ptr1, %[c_ptr0], #0x0\n"
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
+ "cbnz %[odds], 2f\n"
"ldr q0, [%[a_ptr0]], #0x10\n"
"ldr q3, [a_ptr1], #0x10\n"
"ldr q6, [a_ptr2], #0x10\n"
@@ -107,18 +108,29 @@
"ldr q4, [a_ptr1], #0x10\n"
"ldr q7, [a_ptr2], #0x10\n"
"ldr q10, [a_ptr3], #0x10\n"
+ "ldr s2, [%[a_ptr0]]\n"
+ "ldr q13, [a_ptr4], #0x10\n"
+ "ldr s5, [a_ptr1]\n"
+ "ldr q16, [a_ptr5], #0x10\n"
+ "ldr s8, [a_ptr2]\n"
+ "ldr s11, [a_ptr3]\n"
+ "ldr s14, [a_ptr4]\n"
+ "ldr s17, [a_ptr5]\n"
+ "b 3f\n"
+ "2:\n"
+ "ldr q0, [%[a_ptr0]], #0x10\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "ldr q3, [a_ptr1], #0x10\n"
+ "ldr q6, [a_ptr2], #0x10\n"
+ "ldr q9, [a_ptr3], #0x10\n"
+ "ldr q12, [a_ptr4], #0x10\n"
+ "ldr q15, [a_ptr5], #0x10\n"
+ "ldr q1, [%[a_ptr0]], #0x10\n"
+ "ldr q4, [a_ptr1], #0x10\n"
+ "ldr q7, [a_ptr2], #0x10\n"
+ "ldr q10, [a_ptr3], #0x10\n"
"ldr q13, [a_ptr4], #0x10\n"
"ldr q16, [a_ptr5], #0x10\n"
- "cbnz %[odds], 2f\n"
- "ldr s2, [%[a_ptr0]]\n"
- "ldr s5, [a_ptr1]\n"
- "ldr s8, [a_ptr2]\n"
- "ldr s11, [a_ptr3]\n"
- "ldr s14, [a_ptr4]\n"
- "ldr s17, [a_ptr5]\n"
- "b 3f\n"
- "2:\n"
- "subs %[odds], %[odds], #0x1\n"
"b.ne 4f\n"
"ldr b2, [%[a_ptr0]]\n"
"ldr b5, [a_ptr1]\n"
@@ -145,40 +157,42 @@
"ld1 {v14.b}[2], [a_ptr4]\n"
"ld1 {v17.b}[2], [a_ptr5]\n"
"3:\n"
- "movi v26.4s, #0\n"
"ldr q18, [%[b_ptr0]]\n"
- "movi v27.4s, #0\n"
- "ldr q19, [%[b_ptr0], #0x10]\n"
- "movi v28.4s, #0\n"
- "ldr q20, [%[b_ptr0], #0x20]\n"
- "movi v29.4s, #0\n"
- "ldr q21, [%[b_ptr0], #0x30]\n"
- "movi v30.4s, #0\n"
- "ldr q22, [%[b_ptr0], #0x40]\n"
- "movi v31.4s, #0\n"
- "ldr q23, [%[b_ptr0], #0x50]\n"
- ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n"
- "ldr q24, [%[b_ptr0], #0x60]\n"
- ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n"
- "ldr q25, [%[b_ptr0], #0x70]\n"
- ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x40]\n"
- ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x80]\n"
- ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
"prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
- ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x100]\n"
- ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n"
+ "ldr q22, [%[b_ptr0], #0x40]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x140]\n"
- ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n"
+ "ldr q23, [%[b_ptr0], #0x50]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x180]\n"
- ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n"
+ "ldr q24, [%[b_ptr0], #0x60]\n"
+ "ldr q25, [%[b_ptr0], #0x70]\n"
"add %[b_ptr0], %[b_ptr0], #0x80\n"
- ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n"
+ "cbz %[loops], 6f\n"
+ "movi v26.4s, #0\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n"
+ ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n"
+ ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n"
+ ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n"
+ ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n"
+ ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n"
"ldr q18, [%[b_ptr0]]\n"
- ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n"
+ ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n"
"add %[b_ptr0], %[b_ptr0], #0x10\n"
+ ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n"
+ ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n"
+ ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n"
+ ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n"
".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n"
".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n"
".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n"
@@ -222,173 +236,219 @@
".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n"
".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n"
".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n"
- "cbz %[loops], 6f\n"
- "ldr d18, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ldr temploadreg2, [%[b_ptr0], #0x8]\n"
- "ldr d19, [%[b_ptr0], #0x10]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x18]\n"
- "ldr d20, [%[b_ptr0], #0x20]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x28]\n"
- "ldr d21, [%[b_ptr0], #0x30]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x38]\n"
- "ldr d22, [%[b_ptr0], #0x40]\n"
- "ins v18.d[1], temploadreg2\n"
- "ldr temploadreg2, [%[b_ptr0], #0x48]\n"
- "ldr d23, [%[b_ptr0], #0x50]\n"
- "ins v19.d[1], temploadreg3\n"
- "ldr temploadreg3, [%[b_ptr0], #0x58]\n"
- "ldr d24, [%[b_ptr0], #0x60]\n"
- "ins v20.d[1], temploadreg0\n"
- "ldr temploadreg0, [%[b_ptr0], #0x68]\n"
- "ldr d25, [%[b_ptr0], #0x70]\n"
- "ins v21.d[1], temploadreg1\n"
- "ldr temploadreg1, [%[b_ptr0], #0x78]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "ins v22.d[1], temploadreg2\n"
- "ins v23.d[1], temploadreg3\n"
- "ins v24.d[1], temploadreg0\n"
- "ins v25.d[1], temploadreg1\n"
"b.eq 7f\n"
"8:\n"
"str q26, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"movi v26.4s, #0\n"
+ "ldr d18, [%[b_ptr0]]\n"
"ldr temploadreg2, [%[b_ptr0], #0x8]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q27, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
- ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n"
- "str q28, [c_ptr2]\n"
- "movi v28.4s, #0\n"
- "add c_ptr2, c_ptr2, #0x10\n"
- ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n"
- "str q29, [c_ptr3]\n"
- "movi v29.4s, #0\n"
- "add c_ptr3, c_ptr3, #0x10\n"
- ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n"
- "str q30, [c_ptr4]\n"
- "movi v30.4s, #0\n"
- "add c_ptr4, c_ptr4, #0x10\n"
- ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n"
- "str q31, [c_ptr5]\n"
- "movi v31.4s, #0\n"
- "add c_ptr5, c_ptr5, #0x10\n"
- ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n"
- "ldr d18, [%[b_ptr0]]\n"
- ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n"
- "add %[b_ptr0], %[b_ptr0], #0x10\n"
- ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n"
- "ins v18.d[1], temploadreg2\n"
- ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x8]\n"
- ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x18]\n"
- ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x28]\n"
- ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n"
"ldr d19, [%[b_ptr0], #0x10]\n"
- ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x38]\n"
- ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n"
- "ins v19.d[1], temploadreg3\n"
- ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x58]\n"
- ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n"
+ "ldr temploadreg3, [%[b_ptr0], #0x18]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "str q28, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ "movi v28.4s, #0\n"
"ldr d20, [%[b_ptr0], #0x20]\n"
+ "ldr temploadreg0, [%[b_ptr0], #0x28]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ "str q29, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ "movi v29.4s, #0\n"
+ "ldr d21, [%[b_ptr0], #0x30]\n"
+ "ldr temploadreg1, [%[b_ptr0], #0x38]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ "str q30, [c_ptr4]\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ "movi v30.4s, #0\n"
+ "ldr d22, [%[b_ptr0], #0x40]\n"
+ "ins v18.d[1], temploadreg2\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
+ "str q31, [c_ptr5]\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ "movi v31.4s, #0\n"
+ "ldr temploadreg2, [%[b_ptr0], #0x48]\n"
+ ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n"
+ "ldr d23, [%[b_ptr0], #0x50]\n"
+ ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n"
+ "ins v19.d[1], temploadreg3\n"
+ ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n"
+ "ldr temploadreg3, [%[b_ptr0], #0x58]\n"
+ ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n"
+ "ldr d24, [%[b_ptr0], #0x60]\n"
+ ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n"
+ "ins v20.d[1], temploadreg0\n"
+ ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n"
+ "ldr temploadreg0, [%[b_ptr0], #0x68]\n"
+ ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n"
+ "ldr d25, [%[b_ptr0], #0x70]\n"
+ ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n"
+ "ins v21.d[1], temploadreg1\n"
+ ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n"
+ "ldr temploadreg1, [%[b_ptr0], #0x78]\n"
+ ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n"
+ "ins v22.d[1], temploadreg2\n"
+ ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n"
+ "ins v23.d[1], temploadreg3\n"
+ ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n"
+ "ins v24.d[1], temploadreg0\n"
+ ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n"
+ "ins v25.d[1], temploadreg1\n"
+ ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
+ ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n"
+ "ldr d18, [%[b_ptr0]]\n"
+ ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n"
+ "ldr temploadreg2, [%[b_ptr0], #0x8]\n"
+ ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x10\n"
+ ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n"
+ "ins v18.d[1], temploadreg2\n"
".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n"
"prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n"
- "ins v20.d[1], temploadreg0\n"
- ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x68]\n"
- ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n"
"prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
+ ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n"
+ ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n"
".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n"
".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n"
- "ldr d21, [%[b_ptr0], #0x30]\n"
".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n"
".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n"
".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n"
- "ins v21.d[1], temploadreg1\n"
".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x78]\n"
".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n"
".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n"
- "ldr d22, [%[b_ptr0], #0x40]\n"
".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n"
".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n"
".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n"
".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n"
".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n"
".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n"
- "ldr d23, [%[b_ptr0], #0x50]\n"
".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n"
- "ins v23.d[1], temploadreg3\n"
".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n"
".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n"
".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n"
- "ldr d24, [%[b_ptr0], #0x60]\n"
".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n"
- "ins v24.d[1], temploadreg0\n"
".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n"
".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n"
".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n"
- "ldr d25, [%[b_ptr0], #0x70]\n"
".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n"
".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n"
".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n"
- "ins v25.d[1], temploadreg1\n"
".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n"
".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n"
".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n"
- "ldr d18, [%[b_ptr0]]\n"
- "ins v18.d[1], temploadreg2\n"
- "ldr temploadreg2, [%[b_ptr0], #0x48]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "ins v22.d[1], temploadreg2\n"
"b.ne 8b\n"
"7:\n"
"str q26, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v26.4s, #0\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
"str q27, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n"
"str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
+ "ldr q22, [%[b_ptr0], #0x40]\n"
"add c_ptr2, c_ptr2, #0x10\n"
".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n"
"str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
- "add c_ptr3, c_ptr3, #0x10\n"
+ "ldr q23, [%[b_ptr0], #0x50]\n"
".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n"
+ "ldr q24, [%[b_ptr0], #0x60]\n"
+ ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n"
"str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
- "add c_ptr4, c_ptr4, #0x10\n"
+ "ldr q25, [%[b_ptr0], #0x70]\n"
".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n"
"str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
- "add c_ptr5, c_ptr5, #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n"
- ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
+ ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x10\n"
+ ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n"
+ ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n"
+ ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n"
+ ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n"
+ ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n"
+ ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n"
+ ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n"
+ ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n"
+ ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n"
+ ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n"
+ ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n"
+ ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n"
+ ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n"
+ ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n"
+ ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n"
+ ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n"
+ ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n"
+ ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n"
+ ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n"
+ ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n"
+ ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n"
+ ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n"
+ ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n"
+ ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n"
+ ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n"
+ ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n"
+ ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n"
+ ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n"
+ ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n"
+ ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n"
+ ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n"
+ ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n"
+ ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n"
+ "b 9f\n"
+ "6:\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n"
+ ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n"
+ ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n"
+ ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n"
+ ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n"
".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n"
"ldr q18, [%[b_ptr0]]\n"
- ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n"
+ ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n"
"add %[b_ptr0], %[b_ptr0], #0x10\n"
+ ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n"
".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n"
".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n"
".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n"
@@ -435,19 +495,14 @@
".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n"
".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n"
".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n"
- "6:\n"
+ "9:\n"
"str q26, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q27, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, #0x10\n"
"str q28, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, #0x10\n"
"str q29, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, #0x10\n"
"str q30, [c_ptr4]\n"
- "add c_ptr4, c_ptr4, #0x10\n"
"str q31, [c_ptr5]\n"
- "add c_ptr5, c_ptr5, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -514,6 +569,7 @@
"add c_ptr1, %[c_ptr0], #0x0\n"
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
+ "cbnz %[odds], 2f\n"
"ldr q0, [%[a_ptr0]], #0x10\n"
"ldr q3, [a_ptr1], #0x10\n"
"ldr q6, [a_ptr2], #0x10\n"
@@ -524,24 +580,35 @@
"ldr q4, [a_ptr1], #0x10\n"
"ldr q7, [a_ptr2], #0x10\n"
"ldr q10, [a_ptr3], #0x10\n"
- "ldr q13, [a_ptr4], #0x10\n"
- "ldr q16, [a_ptr5], #0x10\n"
- "cbnz %[odds], 2f\n"
"ldr d2, [%[a_ptr0]]\n"
+ "ldr q13, [a_ptr4], #0x10\n"
"ldr d5, [a_ptr1]\n"
+ "ldr q16, [a_ptr5], #0x10\n"
"ldr d8, [a_ptr2]\n"
"ldr d11, [a_ptr3]\n"
"ldr d14, [a_ptr4]\n"
"ldr d17, [a_ptr5]\n"
"b 3f\n"
"2:\n"
+ "ldr q0, [%[a_ptr0]], #0x10\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "ldr q3, [a_ptr1], #0x10\n"
+ "ldr q6, [a_ptr2], #0x10\n"
+ "ldr q9, [a_ptr3], #0x10\n"
+ "ldr q12, [a_ptr4], #0x10\n"
+ "ldr q15, [a_ptr5], #0x10\n"
+ "ldr q1, [%[a_ptr0]], #0x10\n"
+ "ldr q4, [a_ptr1], #0x10\n"
+ "ldr q7, [a_ptr2], #0x10\n"
+ "ldr q10, [a_ptr3], #0x10\n"
"ldr s2, [%[a_ptr0]], #0x4\n"
+ "ldr q13, [a_ptr4], #0x10\n"
"ldr s5, [a_ptr1], #0x4\n"
+ "ldr q16, [a_ptr5], #0x10\n"
"ldr s8, [a_ptr2], #0x4\n"
"ldr s11, [a_ptr3], #0x4\n"
"ldr s14, [a_ptr4], #0x4\n"
"ldr s17, [a_ptr5], #0x4\n"
- "subs %[odds], %[odds], #0x1\n"
"b.ne 4f\n"
"ld1 {v2.b}[4], [%[a_ptr0]]\n"
"ld1 {v5.b}[4], [a_ptr1]\n"
@@ -568,38 +635,40 @@
"ld1 {v14.b}[6], [a_ptr4]\n"
"ld1 {v17.b}[6], [a_ptr5]\n"
"3:\n"
- "movi v26.4s, #0\n"
"ldr q18, [%[b_ptr0]]\n"
- "movi v27.4s, #0\n"
- "ldr q19, [%[b_ptr0], #0x10]\n"
- "movi v28.4s, #0\n"
- "ldr q20, [%[b_ptr0], #0x20]\n"
- "movi v29.4s, #0\n"
- "ldr q21, [%[b_ptr0], #0x30]\n"
- "movi v30.4s, #0\n"
- "ldr q22, [%[b_ptr0], #0x40]\n"
- "movi v31.4s, #0\n"
- "ldr q23, [%[b_ptr0], #0x50]\n"
- ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n"
- "ldr q24, [%[b_ptr0], #0x60]\n"
- ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n"
- "ldr q25, [%[b_ptr0], #0x70]\n"
- ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x40]\n"
- ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x80]\n"
- ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
"prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
- ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x100]\n"
- ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n"
+ "ldr q22, [%[b_ptr0], #0x40]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x140]\n"
- ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n"
+ "ldr q23, [%[b_ptr0], #0x50]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x180]\n"
- ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n"
+ "ldr q24, [%[b_ptr0], #0x60]\n"
+ "ldr q25, [%[b_ptr0], #0x70]\n"
"add %[b_ptr0], %[b_ptr0], #0x80\n"
- ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n"
+ "cbz %[loops], 6f\n"
+ "movi v26.4s, #0\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n"
+ ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n"
+ ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n"
+ ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n"
+ ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n"
+ ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n"
"ldr q18, [%[b_ptr0]]\n"
+ ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n"
+ ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n"
+ ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n"
+ ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n"
".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n"
".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n"
"ldr q19, [%[b_ptr0], #0x10]\n"
@@ -652,180 +721,233 @@
".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n"
".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n"
".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n"
- "cbz %[loops], 6f\n"
- "ldr d18, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ldr temploadreg2, [%[b_ptr0], #0x8]\n"
- "ldr d19, [%[b_ptr0], #0x10]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x18]\n"
- "ldr d20, [%[b_ptr0], #0x20]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x28]\n"
- "ldr d21, [%[b_ptr0], #0x30]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x38]\n"
- "ldr d22, [%[b_ptr0], #0x40]\n"
- "ins v18.d[1], temploadreg2\n"
- "ldr temploadreg2, [%[b_ptr0], #0x48]\n"
- "ldr d23, [%[b_ptr0], #0x50]\n"
- "ins v19.d[1], temploadreg3\n"
- "ldr temploadreg3, [%[b_ptr0], #0x58]\n"
- "ldr d24, [%[b_ptr0], #0x60]\n"
- "ins v20.d[1], temploadreg0\n"
- "ldr temploadreg0, [%[b_ptr0], #0x68]\n"
- "ldr d25, [%[b_ptr0], #0x70]\n"
- "ins v21.d[1], temploadreg1\n"
- "ldr temploadreg1, [%[b_ptr0], #0x78]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "ins v22.d[1], temploadreg2\n"
- "ins v23.d[1], temploadreg3\n"
- "ins v24.d[1], temploadreg0\n"
- "ins v25.d[1], temploadreg1\n"
"b.eq 7f\n"
"8:\n"
"str q26, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"movi v26.4s, #0\n"
+ "ldr d18, [%[b_ptr0]]\n"
"ldr temploadreg2, [%[b_ptr0], #0x8]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x18]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q27, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
- ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n"
- "str q28, [c_ptr2]\n"
- "movi v28.4s, #0\n"
- "add c_ptr2, c_ptr2, #0x10\n"
- ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n"
- "str q29, [c_ptr3]\n"
- "movi v29.4s, #0\n"
- "add c_ptr3, c_ptr3, #0x10\n"
- ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n"
- "str q30, [c_ptr4]\n"
- "movi v30.4s, #0\n"
- "add c_ptr4, c_ptr4, #0x10\n"
- ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n"
- "str q31, [c_ptr5]\n"
- "movi v31.4s, #0\n"
- "add c_ptr5, c_ptr5, #0x10\n"
- ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n"
- "ldr d18, [%[b_ptr0]]\n"
- ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n"
- "ins v18.d[1], temploadreg2\n"
- ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n"
"ldr d19, [%[b_ptr0], #0x10]\n"
- ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n"
- "ins v19.d[1], temploadreg3\n"
- ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x8]\n"
- ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n"
"ldr temploadreg3, [%[b_ptr0], #0x18]\n"
- ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x28]\n"
- ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "str q28, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ "movi v28.4s, #0\n"
"ldr d20, [%[b_ptr0], #0x20]\n"
- ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x38]\n"
- ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n"
- ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n"
- "ins v20.d[1], temploadreg0\n"
- ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x68]\n"
- ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n"
- ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n"
+ "ldr temploadreg0, [%[b_ptr0], #0x28]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ "str q29, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ "movi v29.4s, #0\n"
"ldr d21, [%[b_ptr0], #0x30]\n"
+ "ldr temploadreg1, [%[b_ptr0], #0x38]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ "str q30, [c_ptr4]\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ "movi v30.4s, #0\n"
+ "ldr d22, [%[b_ptr0], #0x40]\n"
+ "ins v18.d[1], temploadreg2\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
+ "str q31, [c_ptr5]\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ "movi v31.4s, #0\n"
+ "ldr temploadreg2, [%[b_ptr0], #0x48]\n"
+ ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n"
+ "ldr d23, [%[b_ptr0], #0x50]\n"
+ ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n"
+ "ins v19.d[1], temploadreg3\n"
+ ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n"
+ "ldr temploadreg3, [%[b_ptr0], #0x58]\n"
+ ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n"
+ "ldr d24, [%[b_ptr0], #0x60]\n"
+ ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n"
+ "ins v20.d[1], temploadreg0\n"
+ ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n"
+ "ldr temploadreg0, [%[b_ptr0], #0x68]\n"
+ ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n"
+ "ldr d25, [%[b_ptr0], #0x70]\n"
+ ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n"
+ "ins v21.d[1], temploadreg1\n"
+ ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n"
+ "ldr temploadreg1, [%[b_ptr0], #0x78]\n"
+ ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n"
+ "ins v22.d[1], temploadreg2\n"
+ ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n"
+ "ins v23.d[1], temploadreg3\n"
+ ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n"
+ "ins v24.d[1], temploadreg0\n"
+ ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n"
+ "ins v25.d[1], temploadreg1\n"
+ ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
+ ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n"
+ "ldr d18, [%[b_ptr0]]\n"
+ ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n"
+ "ldr temploadreg2, [%[b_ptr0], #0x8]\n"
+ ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n"
+ "ldr d19, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n"
+ "ldr temploadreg3, [%[b_ptr0], #0x18]\n"
+ ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n"
+ "ins v18.d[1], temploadreg2\n"
+ ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n"
+ "ins v19.d[1], temploadreg3\n"
+ ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n"
+ "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
+ ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n"
+ "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
+ ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n"
".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n"
".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n"
".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n"
- "ins v21.d[1], temploadreg1\n"
".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x78]\n"
".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n"
".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n"
- "ldr d22, [%[b_ptr0], #0x40]\n"
".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n"
".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n"
".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n"
".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n"
".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n"
".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n"
- "ldr d23, [%[b_ptr0], #0x50]\n"
".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n"
".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n"
".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n"
".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n"
- "ldr d24, [%[b_ptr0], #0x60]\n"
".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n"
- "ins v24.d[1], temploadreg0\n"
".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n"
".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n"
".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n"
- "ldr d25, [%[b_ptr0], #0x70]\n"
".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n"
".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n"
".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n"
- "ins v25.d[1], temploadreg1\n"
".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n"
".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n"
".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n"
- "ldr d18, [%[b_ptr0]]\n"
".inst 0x4fa2e27a // sdot v26.4s, v19.16b, v2.4b[1]\n"
".inst 0x4fa5e27b // sdot v27.4s, v19.16b, v5.4b[1]\n"
".inst 0x4fa8e27c // sdot v28.4s, v19.16b, v8.4b[1]\n"
- "ins v18.d[1], temploadreg2\n"
".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x48]\n"
".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n"
".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n"
- "ldr d19, [%[b_ptr0], #0x10]\n"
- "ins v22.d[1], temploadreg2\n"
- "ins v19.d[1], temploadreg3\n"
- "ldr temploadreg3, [%[b_ptr0], #0x58]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "ins v23.d[1], temploadreg3\n"
"b.ne 8b\n"
"7:\n"
"str q26, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v26.4s, #0\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
"str q27, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n"
"str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
+ "ldr q22, [%[b_ptr0], #0x40]\n"
"add c_ptr2, c_ptr2, #0x10\n"
".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n"
"str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
- "add c_ptr3, c_ptr3, #0x10\n"
+ "ldr q23, [%[b_ptr0], #0x50]\n"
".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n"
+ "ldr q24, [%[b_ptr0], #0x60]\n"
+ ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n"
"str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
- "add c_ptr4, c_ptr4, #0x10\n"
+ "ldr q25, [%[b_ptr0], #0x70]\n"
".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n"
"str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
- "add c_ptr5, c_ptr5, #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n"
- ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
+ ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n"
+ ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n"
+ ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n"
+ ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n"
+ ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n"
+ ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n"
+ ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n"
+ ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n"
+ ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n"
+ ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n"
+ ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n"
+ ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n"
+ ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n"
+ ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n"
+ ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n"
+ ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n"
+ ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n"
+ ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n"
+ ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n"
+ ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n"
+ ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n"
+ ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n"
+ ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n"
+ ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n"
+ ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n"
+ ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n"
+ ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n"
+ ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n"
+ ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n"
+ ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n"
+ ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n"
+ ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n"
+ ".inst 0x4fa2e27a // sdot v26.4s, v19.16b, v2.4b[1]\n"
+ ".inst 0x4fa5e27b // sdot v27.4s, v19.16b, v5.4b[1]\n"
+ ".inst 0x4fa8e27c // sdot v28.4s, v19.16b, v8.4b[1]\n"
+ ".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n"
+ ".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n"
+ ".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n"
+ "b 9f\n"
+ "6:\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n"
+ ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n"
+ ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n"
+ ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n"
+ ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n"
".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n"
"ldr q18, [%[b_ptr0]]\n"
+ ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n"
".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n"
".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n"
".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n"
@@ -881,19 +1003,14 @@
".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n"
".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n"
".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n"
- "6:\n"
+ "9:\n"
"str q26, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q27, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, #0x10\n"
"str q28, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, #0x10\n"
"str q29, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, #0x10\n"
"str q30, [c_ptr4]\n"
- "add c_ptr4, c_ptr4, #0x10\n"
"str q31, [c_ptr5]\n"
- "add c_ptr5, c_ptr5, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -1014,38 +1131,40 @@
"ld1 {v14.b}[10], [a_ptr4]\n"
"ld1 {v17.b}[10], [a_ptr5]\n"
"3:\n"
- "movi v26.4s, #0\n"
"ldr q18, [%[b_ptr0]]\n"
- "movi v27.4s, #0\n"
- "ldr q19, [%[b_ptr0], #0x10]\n"
- "movi v28.4s, #0\n"
- "ldr q20, [%[b_ptr0], #0x20]\n"
- "movi v29.4s, #0\n"
- "ldr q21, [%[b_ptr0], #0x30]\n"
- "movi v30.4s, #0\n"
- "ldr q22, [%[b_ptr0], #0x40]\n"
- "movi v31.4s, #0\n"
- "ldr q23, [%[b_ptr0], #0x50]\n"
- ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n"
- "ldr q24, [%[b_ptr0], #0x60]\n"
- ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n"
- "ldr q25, [%[b_ptr0], #0x70]\n"
- ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x40]\n"
- ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x80]\n"
- ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
"prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
- ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x100]\n"
- ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n"
+ "ldr q22, [%[b_ptr0], #0x40]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x140]\n"
- ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n"
+ "ldr q23, [%[b_ptr0], #0x50]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x180]\n"
- ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n"
+ "ldr q24, [%[b_ptr0], #0x60]\n"
+ "ldr q25, [%[b_ptr0], #0x70]\n"
"add %[b_ptr0], %[b_ptr0], #0x80\n"
- ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n"
+ "cbz %[loops], 6f\n"
+ "movi v26.4s, #0\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n"
+ ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n"
+ ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n"
+ ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n"
+ ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n"
+ ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n"
"ldr q18, [%[b_ptr0]]\n"
+ ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n"
+ ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n"
+ ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n"
+ ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n"
".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n"
".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n"
"ldr q19, [%[b_ptr0], #0x10]\n"
@@ -1105,189 +1224,249 @@
".inst 0x4f8bea9d // sdot v29.4s, v20.16b, v11.4b[2]\n"
".inst 0x4f8eea9e // sdot v30.4s, v20.16b, v14.4b[2]\n"
".inst 0x4f91ea9f // sdot v31.4s, v20.16b, v17.4b[2]\n"
- "cbz %[loops], 6f\n"
- "ldr d18, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ldr temploadreg2, [%[b_ptr0], #0x8]\n"
- "ldr d19, [%[b_ptr0], #0x10]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x18]\n"
- "ldr d20, [%[b_ptr0], #0x20]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x28]\n"
- "ldr d21, [%[b_ptr0], #0x30]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x38]\n"
- "ldr d22, [%[b_ptr0], #0x40]\n"
- "ins v18.d[1], temploadreg2\n"
- "ldr temploadreg2, [%[b_ptr0], #0x48]\n"
- "ldr d23, [%[b_ptr0], #0x50]\n"
- "ins v19.d[1], temploadreg3\n"
- "ldr temploadreg3, [%[b_ptr0], #0x58]\n"
- "ldr d24, [%[b_ptr0], #0x60]\n"
- "ins v20.d[1], temploadreg0\n"
- "ldr temploadreg0, [%[b_ptr0], #0x68]\n"
- "ldr d25, [%[b_ptr0], #0x70]\n"
- "ins v21.d[1], temploadreg1\n"
- "ldr temploadreg1, [%[b_ptr0], #0x78]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "ins v22.d[1], temploadreg2\n"
- "ins v23.d[1], temploadreg3\n"
- "ins v24.d[1], temploadreg0\n"
- "ins v25.d[1], temploadreg1\n"
"b.eq 7f\n"
"8:\n"
"str q26, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"movi v26.4s, #0\n"
+ "ldr d18, [%[b_ptr0]]\n"
"ldr temploadreg2, [%[b_ptr0], #0x8]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x18]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q27, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
- "ldr temploadreg0, [%[b_ptr0], #0x28]\n"
- ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n"
- "str q28, [c_ptr2]\n"
- "movi v28.4s, #0\n"
- "add c_ptr2, c_ptr2, #0x10\n"
- ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n"
- "str q29, [c_ptr3]\n"
- "movi v29.4s, #0\n"
- "add c_ptr3, c_ptr3, #0x10\n"
- ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n"
- "str q30, [c_ptr4]\n"
- "movi v30.4s, #0\n"
- "add c_ptr4, c_ptr4, #0x10\n"
- ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n"
- "str q31, [c_ptr5]\n"
- "movi v31.4s, #0\n"
- "add c_ptr5, c_ptr5, #0x10\n"
- ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
- ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n"
- "ldr d18, [%[b_ptr0]]\n"
- ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n"
- "ins v18.d[1], temploadreg2\n"
- ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n"
"ldr d19, [%[b_ptr0], #0x10]\n"
- ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n"
- "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n"
- "ins v19.d[1], temploadreg3\n"
- ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n"
- ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n"
- ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n"
- ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n"
- "ldr d20, [%[b_ptr0], #0x20]\n"
- ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n"
- "add %[b_ptr0], %[b_ptr0], #0x30\n"
- ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n"
- "ins v20.d[1], temploadreg0\n"
- ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x8]\n"
- ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n"
"ldr temploadreg3, [%[b_ptr0], #0x18]\n"
- ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "str q28, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ "movi v28.4s, #0\n"
+ "ldr d20, [%[b_ptr0], #0x20]\n"
"ldr temploadreg0, [%[b_ptr0], #0x28]\n"
- ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ "str q29, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ "movi v29.4s, #0\n"
"ldr d21, [%[b_ptr0], #0x30]\n"
- ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n"
"ldr temploadreg1, [%[b_ptr0], #0x38]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ "str q30, [c_ptr4]\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ "movi v30.4s, #0\n"
+ "ldr d22, [%[b_ptr0], #0x40]\n"
+ "ins v18.d[1], temploadreg2\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
+ "str q31, [c_ptr5]\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ "movi v31.4s, #0\n"
+ "ldr temploadreg2, [%[b_ptr0], #0x48]\n"
+ ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n"
+ "ldr d23, [%[b_ptr0], #0x50]\n"
+ ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n"
+ "ins v19.d[1], temploadreg3\n"
+ ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n"
+ "ldr temploadreg3, [%[b_ptr0], #0x58]\n"
+ ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n"
+ "ldr d24, [%[b_ptr0], #0x60]\n"
+ ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n"
+ "ins v20.d[1], temploadreg0\n"
+ ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n"
+ "ldr temploadreg0, [%[b_ptr0], #0x68]\n"
+ ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n"
+ "ldr d25, [%[b_ptr0], #0x70]\n"
+ ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n"
+ "ins v21.d[1], temploadreg1\n"
+ ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n"
+ "ldr temploadreg1, [%[b_ptr0], #0x78]\n"
+ ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n"
+ "ins v22.d[1], temploadreg2\n"
+ ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n"
+ "ins v23.d[1], temploadreg3\n"
+ ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n"
+ "ins v24.d[1], temploadreg0\n"
+ ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n"
+ "ins v25.d[1], temploadreg1\n"
+ ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
+ ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n"
+ "ldr d18, [%[b_ptr0]]\n"
+ ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n"
+ "ldr temploadreg2, [%[b_ptr0], #0x8]\n"
+ ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n"
+ "ldr d19, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n"
+ "ldr temploadreg3, [%[b_ptr0], #0x18]\n"
+ ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n"
+ "ldr d20, [%[b_ptr0], #0x20]\n"
+ ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n"
+ "ldr temploadreg0, [%[b_ptr0], #0x28]\n"
+ ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n"
+ "ins v18.d[1], temploadreg2\n"
+ ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n"
+ "ins v19.d[1], temploadreg3\n"
+ ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n"
+ "ins v20.d[1], temploadreg0\n"
+ ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x30\n"
+ ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n"
+ "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n"
+ "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n"
".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n"
- "ins v21.d[1], temploadreg1\n"
".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x78]\n"
".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n"
- "ldr d22, [%[b_ptr0], #0x40]\n"
".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n"
".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n"
".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n"
".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n"
".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n"
".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n"
- "ldr d23, [%[b_ptr0], #0x50]\n"
".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n"
".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n"
".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n"
".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n"
- "ldr d24, [%[b_ptr0], #0x60]\n"
".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n"
".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n"
".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n"
".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n"
- "ldr d25, [%[b_ptr0], #0x70]\n"
".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n"
".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n"
".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n"
- "ins v25.d[1], temploadreg1\n"
".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n"
".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n"
".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n"
- "ldr d18, [%[b_ptr0]]\n"
".inst 0x4fa2e27a // sdot v26.4s, v19.16b, v2.4b[1]\n"
".inst 0x4fa5e27b // sdot v27.4s, v19.16b, v5.4b[1]\n"
".inst 0x4fa8e27c // sdot v28.4s, v19.16b, v8.4b[1]\n"
- "ins v18.d[1], temploadreg2\n"
".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x48]\n"
".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n"
".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n"
- "ldr d19, [%[b_ptr0], #0x10]\n"
".inst 0x4f82ea9a // sdot v26.4s, v20.16b, v2.4b[2]\n"
- "ins v22.d[1], temploadreg2\n"
".inst 0x4f85ea9b // sdot v27.4s, v20.16b, v5.4b[2]\n"
".inst 0x4f88ea9c // sdot v28.4s, v20.16b, v8.4b[2]\n"
- "ins v19.d[1], temploadreg3\n"
".inst 0x4f8bea9d // sdot v29.4s, v20.16b, v11.4b[2]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x58]\n"
".inst 0x4f8eea9e // sdot v30.4s, v20.16b, v14.4b[2]\n"
".inst 0x4f91ea9f // sdot v31.4s, v20.16b, v17.4b[2]\n"
- "ldr d20, [%[b_ptr0], #0x20]\n"
- "ins v23.d[1], temploadreg3\n"
- "ins v20.d[1], temploadreg0\n"
- "ldr temploadreg0, [%[b_ptr0], #0x68]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "ins v24.d[1], temploadreg0\n"
"b.ne 8b\n"
"7:\n"
"str q26, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v26.4s, #0\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
"str q27, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n"
"str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
+ "ldr q22, [%[b_ptr0], #0x40]\n"
"add c_ptr2, c_ptr2, #0x10\n"
".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n"
"str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
- "add c_ptr3, c_ptr3, #0x10\n"
+ "ldr q23, [%[b_ptr0], #0x50]\n"
".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n"
+ "ldr q24, [%[b_ptr0], #0x60]\n"
+ ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n"
"str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
- "add c_ptr4, c_ptr4, #0x10\n"
+ "ldr q25, [%[b_ptr0], #0x70]\n"
".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n"
"str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
- "add c_ptr5, c_ptr5, #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n"
- ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
+ ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n"
+ ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n"
+ ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n"
+ ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n"
+ ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n"
+ ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n"
+ ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n"
+ ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
+ ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x30\n"
+ ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n"
+ ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n"
+ ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n"
+ ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n"
+ ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n"
+ ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n"
+ ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n"
+ ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n"
+ ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n"
+ ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n"
+ ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n"
+ ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n"
+ ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n"
+ ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n"
+ ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n"
+ ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n"
+ ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n"
+ ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n"
+ ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n"
+ ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n"
+ ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n"
+ ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n"
+ ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n"
+ ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n"
+ ".inst 0x4fa2e27a // sdot v26.4s, v19.16b, v2.4b[1]\n"
+ ".inst 0x4fa5e27b // sdot v27.4s, v19.16b, v5.4b[1]\n"
+ ".inst 0x4fa8e27c // sdot v28.4s, v19.16b, v8.4b[1]\n"
+ ".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n"
+ ".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n"
+ ".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n"
+ ".inst 0x4f82ea9a // sdot v26.4s, v20.16b, v2.4b[2]\n"
+ ".inst 0x4f85ea9b // sdot v27.4s, v20.16b, v5.4b[2]\n"
+ ".inst 0x4f88ea9c // sdot v28.4s, v20.16b, v8.4b[2]\n"
+ ".inst 0x4f8bea9d // sdot v29.4s, v20.16b, v11.4b[2]\n"
+ ".inst 0x4f8eea9e // sdot v30.4s, v20.16b, v14.4b[2]\n"
+ ".inst 0x4f91ea9f // sdot v31.4s, v20.16b, v17.4b[2]\n"
+ "b 9f\n"
+ "6:\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n"
+ ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n"
+ ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n"
+ ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n"
+ ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n"
".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n"
"ldr q18, [%[b_ptr0]]\n"
+ ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n"
".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n"
".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n"
".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n"
@@ -1350,19 +1529,14 @@
".inst 0x4f8bea9d // sdot v29.4s, v20.16b, v11.4b[2]\n"
".inst 0x4f8eea9e // sdot v30.4s, v20.16b, v14.4b[2]\n"
".inst 0x4f91ea9f // sdot v31.4s, v20.16b, v17.4b[2]\n"
- "6:\n"
+ "9:\n"
"str q26, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q27, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, #0x10\n"
"str q28, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, #0x10\n"
"str q29, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, #0x10\n"
"str q30, [c_ptr4]\n"
- "add c_ptr4, c_ptr4, #0x10\n"
"str q31, [c_ptr5]\n"
- "add c_ptr5, c_ptr5, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -1429,6 +1603,7 @@
"add c_ptr1, %[c_ptr0], #0x0\n"
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
+ "cbnz %[odds], 2f\n"
"ldr q0, [%[a_ptr0]], #0x10\n"
"ldr q3, [a_ptr1], #0x10\n"
"ldr q6, [a_ptr2], #0x10\n"
@@ -1441,7 +1616,6 @@
"ldr q10, [a_ptr3], #0x10\n"
"ldr q13, [a_ptr4], #0x10\n"
"ldr q16, [a_ptr5], #0x10\n"
- "cbnz %[odds], 2f\n"
"ldr q2, [%[a_ptr0]]\n"
"ldr q5, [a_ptr1]\n"
"ldr q8, [a_ptr2]\n"
@@ -1450,8 +1624,21 @@
"ldr q17, [a_ptr5]\n"
"b 3f\n"
"2:\n"
+ "ldr q0, [%[a_ptr0]], #0x10\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "ldr q3, [a_ptr1], #0x10\n"
+ "ldr q6, [a_ptr2], #0x10\n"
+ "ldr q9, [a_ptr3], #0x10\n"
+ "ldr q12, [a_ptr4], #0x10\n"
+ "ldr q15, [a_ptr5], #0x10\n"
+ "ldr q1, [%[a_ptr0]], #0x10\n"
+ "ldr q4, [a_ptr1], #0x10\n"
+ "ldr q7, [a_ptr2], #0x10\n"
+ "ldr q10, [a_ptr3], #0x10\n"
"ldr d2, [%[a_ptr0]], #0x8\n"
+ "ldr q13, [a_ptr4], #0x10\n"
"ldr d5, [a_ptr1], #0x8\n"
+ "ldr q16, [a_ptr5], #0x10\n"
"ldr d8, [a_ptr2], #0x8\n"
"ldr d11, [a_ptr3], #0x8\n"
"ldr d14, [a_ptr4], #0x8\n"
@@ -1462,7 +1649,6 @@
"ld1 {v11.s}[2], [a_ptr3], #4\n"
"ld1 {v14.s}[2], [a_ptr4], #4\n"
"ld1 {v17.s}[2], [a_ptr5], #4\n"
- "subs %[odds], %[odds], #0x1\n"
"b.ne 4f\n"
"ld1 {v2.b}[12], [%[a_ptr0]]\n"
"ld1 {v5.b}[12], [a_ptr1]\n"
@@ -1489,38 +1675,40 @@
"ld1 {v14.b}[14], [a_ptr4]\n"
"ld1 {v17.b}[14], [a_ptr5]\n"
"3:\n"
- "movi v26.4s, #0\n"
"ldr q18, [%[b_ptr0]]\n"
- "movi v27.4s, #0\n"
- "ldr q19, [%[b_ptr0], #0x10]\n"
- "movi v28.4s, #0\n"
- "ldr q20, [%[b_ptr0], #0x20]\n"
- "movi v29.4s, #0\n"
- "ldr q21, [%[b_ptr0], #0x30]\n"
- "movi v30.4s, #0\n"
- "ldr q22, [%[b_ptr0], #0x40]\n"
- "movi v31.4s, #0\n"
- "ldr q23, [%[b_ptr0], #0x50]\n"
- ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n"
- "ldr q24, [%[b_ptr0], #0x60]\n"
- ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n"
- "ldr q25, [%[b_ptr0], #0x70]\n"
- ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x40]\n"
- ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x80]\n"
- ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
"prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
- ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x100]\n"
- ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n"
+ "ldr q22, [%[b_ptr0], #0x40]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x140]\n"
- ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n"
+ "ldr q23, [%[b_ptr0], #0x50]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x180]\n"
- ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n"
+ "ldr q24, [%[b_ptr0], #0x60]\n"
+ "ldr q25, [%[b_ptr0], #0x70]\n"
"add %[b_ptr0], %[b_ptr0], #0x80\n"
- ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n"
+ "cbz %[loops], 6f\n"
+ "movi v26.4s, #0\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n"
+ ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n"
+ ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n"
+ ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n"
+ ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n"
+ ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n"
"ldr q18, [%[b_ptr0]]\n"
+ ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n"
+ ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n"
+ ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n"
+ ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n"
".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n"
".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n"
"ldr q19, [%[b_ptr0], #0x10]\n"
@@ -1587,198 +1775,265 @@
".inst 0x4fabeabd // sdot v29.4s, v21.16b, v11.4b[3]\n"
".inst 0x4faeeabe // sdot v30.4s, v21.16b, v14.4b[3]\n"
".inst 0x4fb1eabf // sdot v31.4s, v21.16b, v17.4b[3]\n"
- "cbz %[loops], 6f\n"
- "ldr d18, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ldr temploadreg2, [%[b_ptr0], #0x8]\n"
- "ldr d19, [%[b_ptr0], #0x10]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x18]\n"
- "ldr d20, [%[b_ptr0], #0x20]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x28]\n"
- "ldr d21, [%[b_ptr0], #0x30]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x38]\n"
- "ldr d22, [%[b_ptr0], #0x40]\n"
- "ins v18.d[1], temploadreg2\n"
- "ldr temploadreg2, [%[b_ptr0], #0x48]\n"
- "ldr d23, [%[b_ptr0], #0x50]\n"
- "ins v19.d[1], temploadreg3\n"
- "ldr temploadreg3, [%[b_ptr0], #0x58]\n"
- "ldr d24, [%[b_ptr0], #0x60]\n"
- "ins v20.d[1], temploadreg0\n"
- "ldr temploadreg0, [%[b_ptr0], #0x68]\n"
- "ldr d25, [%[b_ptr0], #0x70]\n"
- "ins v21.d[1], temploadreg1\n"
- "ldr temploadreg1, [%[b_ptr0], #0x78]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "ins v22.d[1], temploadreg2\n"
- "ins v23.d[1], temploadreg3\n"
- "ins v24.d[1], temploadreg0\n"
"b.eq 7f\n"
"8:\n"
"str q26, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"movi v26.4s, #0\n"
- "ins v25.d[1], temploadreg1\n"
+ "ldr d18, [%[b_ptr0]]\n"
"ldr temploadreg2, [%[b_ptr0], #0x8]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q27, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
- "ldr temploadreg3, [%[b_ptr0], #0x18]\n"
- ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n"
- "str q28, [c_ptr2]\n"
- "movi v28.4s, #0\n"
- "ldr temploadreg0, [%[b_ptr0], #0x28]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x38]\n"
- "add c_ptr2, c_ptr2, #0x10\n"
- ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n"
- "str q29, [c_ptr3]\n"
- "movi v29.4s, #0\n"
- "add c_ptr3, c_ptr3, #0x10\n"
- ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n"
- "str q30, [c_ptr4]\n"
- "movi v30.4s, #0\n"
- "add c_ptr4, c_ptr4, #0x10\n"
- ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n"
- "str q31, [c_ptr5]\n"
- "movi v31.4s, #0\n"
- "add c_ptr5, c_ptr5, #0x10\n"
- ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
- ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n"
- "ldr d18, [%[b_ptr0]]\n"
- ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n"
- "ins v18.d[1], temploadreg2\n"
- ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n"
"ldr d19, [%[b_ptr0], #0x10]\n"
- ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n"
- "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n"
- "ins v19.d[1], temploadreg3\n"
- ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n"
- ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n"
- ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n"
- ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n"
+ "ldr temploadreg3, [%[b_ptr0], #0x18]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "str q28, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ "movi v28.4s, #0\n"
"ldr d20, [%[b_ptr0], #0x20]\n"
- ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n"
- ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n"
- ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n"
+ "ldr temploadreg0, [%[b_ptr0], #0x28]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ "str q29, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ "movi v29.4s, #0\n"
+ "ldr d21, [%[b_ptr0], #0x30]\n"
+ "ldr temploadreg1, [%[b_ptr0], #0x38]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ "str q30, [c_ptr4]\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ "movi v30.4s, #0\n"
+ "ldr d22, [%[b_ptr0], #0x40]\n"
+ "ins v18.d[1], temploadreg2\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
+ "str q31, [c_ptr5]\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ "movi v31.4s, #0\n"
+ "ldr temploadreg2, [%[b_ptr0], #0x48]\n"
+ ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n"
+ "ldr d23, [%[b_ptr0], #0x50]\n"
+ ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n"
+ "ins v19.d[1], temploadreg3\n"
+ ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n"
+ "ldr temploadreg3, [%[b_ptr0], #0x58]\n"
+ ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n"
+ "ldr d24, [%[b_ptr0], #0x60]\n"
+ ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n"
"ins v20.d[1], temploadreg0\n"
+ ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n"
+ "ldr temploadreg0, [%[b_ptr0], #0x68]\n"
+ ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n"
+ "ldr d25, [%[b_ptr0], #0x70]\n"
+ ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n"
+ "ins v21.d[1], temploadreg1\n"
+ ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n"
+ "ldr temploadreg1, [%[b_ptr0], #0x78]\n"
+ ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n"
+ "ins v22.d[1], temploadreg2\n"
+ ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n"
+ "ins v23.d[1], temploadreg3\n"
+ ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n"
+ "ins v24.d[1], temploadreg0\n"
+ ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n"
+ "ins v25.d[1], temploadreg1\n"
+ ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
+ ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n"
+ "ldr d18, [%[b_ptr0]]\n"
+ ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n"
+ "ldr temploadreg2, [%[b_ptr0], #0x8]\n"
+ ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n"
+ "ldr d19, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n"
+ "ldr temploadreg3, [%[b_ptr0], #0x18]\n"
+ ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n"
+ "ldr d20, [%[b_ptr0], #0x20]\n"
+ ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n"
+ "ldr temploadreg0, [%[b_ptr0], #0x28]\n"
+ ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n"
+ "ldr temploadreg1, [%[b_ptr0], #0x38]\n"
".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n"
+ "ins v18.d[1], temploadreg2\n"
".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n"
+ "ins v19.d[1], temploadreg3\n"
".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n"
"ldr d21, [%[b_ptr0], #0x30]\n"
".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x40\n"
+ "ins v20.d[1], temploadreg0\n"
".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n"
- "ins v21.d[1], temploadreg1\n"
+ "add %[b_ptr0], %[b_ptr0], #0x40\n"
".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x8]\n"
+ "ins v21.d[1], temploadreg1\n"
".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x18]\n"
+ "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x28]\n"
+ "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x38]\n"
".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n"
- "ldr d22, [%[b_ptr0], #0x40]\n"
".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n"
".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n"
".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n"
".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n"
".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n"
- "ldr d23, [%[b_ptr0], #0x50]\n"
".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n"
".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n"
".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n"
".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n"
- "ldr d24, [%[b_ptr0], #0x60]\n"
".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n"
".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n"
".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n"
".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n"
- "ldr d25, [%[b_ptr0], #0x70]\n"
".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n"
".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n"
".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n"
".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n"
".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n"
".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n"
- "ldr d18, [%[b_ptr0]]\n"
".inst 0x4fa2e27a // sdot v26.4s, v19.16b, v2.4b[1]\n"
".inst 0x4fa5e27b // sdot v27.4s, v19.16b, v5.4b[1]\n"
".inst 0x4fa8e27c // sdot v28.4s, v19.16b, v8.4b[1]\n"
- "ins v18.d[1], temploadreg2\n"
".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x48]\n"
".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n"
".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n"
- "ldr d19, [%[b_ptr0], #0x10]\n"
".inst 0x4f82ea9a // sdot v26.4s, v20.16b, v2.4b[2]\n"
- "ins v22.d[1], temploadreg2\n"
".inst 0x4f85ea9b // sdot v27.4s, v20.16b, v5.4b[2]\n"
".inst 0x4f88ea9c // sdot v28.4s, v20.16b, v8.4b[2]\n"
- "ins v19.d[1], temploadreg3\n"
".inst 0x4f8bea9d // sdot v29.4s, v20.16b, v11.4b[2]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x58]\n"
".inst 0x4f8eea9e // sdot v30.4s, v20.16b, v14.4b[2]\n"
".inst 0x4f91ea9f // sdot v31.4s, v20.16b, v17.4b[2]\n"
- "ldr d20, [%[b_ptr0], #0x20]\n"
".inst 0x4fa2eaba // sdot v26.4s, v21.16b, v2.4b[3]\n"
- "ins v23.d[1], temploadreg3\n"
".inst 0x4fa5eabb // sdot v27.4s, v21.16b, v5.4b[3]\n"
".inst 0x4fa8eabc // sdot v28.4s, v21.16b, v8.4b[3]\n"
- "ins v20.d[1], temploadreg0\n"
".inst 0x4fabeabd // sdot v29.4s, v21.16b, v11.4b[3]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x68]\n"
".inst 0x4faeeabe // sdot v30.4s, v21.16b, v14.4b[3]\n"
".inst 0x4fb1eabf // sdot v31.4s, v21.16b, v17.4b[3]\n"
- "ldr d21, [%[b_ptr0], #0x30]\n"
- "ins v24.d[1], temploadreg0\n"
- "ins v21.d[1], temploadreg1\n"
- "ldr temploadreg1, [%[b_ptr0], #0x78]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
"b.ne 8b\n"
"7:\n"
"str q26, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v26.4s, #0\n"
- "ins v25.d[1], temploadreg1\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
"str q27, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n"
"str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
+ "ldr q22, [%[b_ptr0], #0x40]\n"
"add c_ptr2, c_ptr2, #0x10\n"
".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n"
"str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
- "add c_ptr3, c_ptr3, #0x10\n"
+ "ldr q23, [%[b_ptr0], #0x50]\n"
".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n"
+ "ldr q24, [%[b_ptr0], #0x60]\n"
+ ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n"
"str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
- "add c_ptr4, c_ptr4, #0x10\n"
+ "ldr q25, [%[b_ptr0], #0x70]\n"
".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n"
"str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
- "add c_ptr5, c_ptr5, #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n"
- ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
+ ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n"
+ ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n"
+ ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n"
+ ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n"
+ ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n"
+ ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n"
+ ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n"
+ ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
+ ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n"
+ ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n"
+ ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n"
+ ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n"
+ ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n"
+ ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
+ ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x40\n"
+ ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n"
+ ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n"
+ ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n"
+ ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n"
+ ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n"
+ ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n"
+ ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n"
+ ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n"
+ ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n"
+ ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n"
+ ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n"
+ ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n"
+ ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n"
+ ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n"
+ ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n"
+ ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n"
+ ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n"
+ ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n"
+ ".inst 0x4fa2e27a // sdot v26.4s, v19.16b, v2.4b[1]\n"
+ ".inst 0x4fa5e27b // sdot v27.4s, v19.16b, v5.4b[1]\n"
+ ".inst 0x4fa8e27c // sdot v28.4s, v19.16b, v8.4b[1]\n"
+ ".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n"
+ ".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n"
+ ".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n"
+ ".inst 0x4f82ea9a // sdot v26.4s, v20.16b, v2.4b[2]\n"
+ ".inst 0x4f85ea9b // sdot v27.4s, v20.16b, v5.4b[2]\n"
+ ".inst 0x4f88ea9c // sdot v28.4s, v20.16b, v8.4b[2]\n"
+ ".inst 0x4f8bea9d // sdot v29.4s, v20.16b, v11.4b[2]\n"
+ ".inst 0x4f8eea9e // sdot v30.4s, v20.16b, v14.4b[2]\n"
+ ".inst 0x4f91ea9f // sdot v31.4s, v20.16b, v17.4b[2]\n"
+ ".inst 0x4fa2eaba // sdot v26.4s, v21.16b, v2.4b[3]\n"
+ ".inst 0x4fa5eabb // sdot v27.4s, v21.16b, v5.4b[3]\n"
+ ".inst 0x4fa8eabc // sdot v28.4s, v21.16b, v8.4b[3]\n"
+ ".inst 0x4fabeabd // sdot v29.4s, v21.16b, v11.4b[3]\n"
+ ".inst 0x4faeeabe // sdot v30.4s, v21.16b, v14.4b[3]\n"
+ ".inst 0x4fb1eabf // sdot v31.4s, v21.16b, v17.4b[3]\n"
+ "b 9f\n"
+ "6:\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n"
+ ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n"
+ ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n"
+ ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n"
+ ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n"
".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n"
"ldr q18, [%[b_ptr0]]\n"
+ ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n"
".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n"
".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n"
".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n"
@@ -1848,19 +2103,14 @@
".inst 0x4fabeabd // sdot v29.4s, v21.16b, v11.4b[3]\n"
".inst 0x4faeeabe // sdot v30.4s, v21.16b, v14.4b[3]\n"
".inst 0x4fb1eabf // sdot v31.4s, v21.16b, v17.4b[3]\n"
- "6:\n"
+ "9:\n"
"str q26, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q27, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, #0x10\n"
"str q28, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, #0x10\n"
"str q29, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, #0x10\n"
"str q30, [c_ptr4]\n"
- "add c_ptr4, c_ptr4, #0x10\n"
"str q31, [c_ptr5]\n"
- "add c_ptr5, c_ptr5, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -1927,6 +2177,7 @@
"add c_ptr1, %[c_ptr0], #0x0\n"
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
+ "cbnz %[odds], 2f\n"
"ldr q0, [%[a_ptr0]], #0x10\n"
"ldr q4, [a_ptr1], #0x10\n"
"ldr q8, [a_ptr2], #0x10\n"
@@ -1943,18 +2194,35 @@
"ldr q6, [a_ptr1], #0x10\n"
"ldr q10, [a_ptr2], #0x10\n"
"ldr q14, [a_ptr3], #0x10\n"
+ "ldr s3, [%[a_ptr0]]\n"
+ "ldr q18, [a_ptr4], #0x10\n"
+ "ldr s7, [a_ptr1]\n"
+ "ldr q22, [a_ptr5], #0x10\n"
+ "ldr s11, [a_ptr2]\n"
+ "ldr s15, [a_ptr3]\n"
+ "ldr s19, [a_ptr4]\n"
+ "ldr s23, [a_ptr5]\n"
+ "b 3f\n"
+ "2:\n"
+ "ldr q0, [%[a_ptr0]], #0x10\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "ldr q4, [a_ptr1], #0x10\n"
+ "ldr q8, [a_ptr2], #0x10\n"
+ "ldr q12, [a_ptr3], #0x10\n"
+ "ldr q16, [a_ptr4], #0x10\n"
+ "ldr q20, [a_ptr5], #0x10\n"
+ "ldr q1, [%[a_ptr0]], #0x10\n"
+ "ldr q5, [a_ptr1], #0x10\n"
+ "ldr q9, [a_ptr2], #0x10\n"
+ "ldr q13, [a_ptr3], #0x10\n"
+ "ldr q17, [a_ptr4], #0x10\n"
+ "ldr q21, [a_ptr5], #0x10\n"
+ "ldr q2, [%[a_ptr0]], #0x10\n"
+ "ldr q6, [a_ptr1], #0x10\n"
+ "ldr q10, [a_ptr2], #0x10\n"
+ "ldr q14, [a_ptr3], #0x10\n"
"ldr q18, [a_ptr4], #0x10\n"
"ldr q22, [a_ptr5], #0x10\n"
- "cbnz %[odds], 2f\n"
- "ldr s3, [%[a_ptr0]]\n"
- "ldr s7, [a_ptr1]\n"
- "ldr s11, [a_ptr2]\n"
- "ldr s15, [a_ptr3]\n"
- "ldr s19, [a_ptr4]\n"
- "ldr s23, [a_ptr5]\n"
- "b 3f\n"
- "2:\n"
- "subs %[odds], %[odds], #0x1\n"
"b.ne 4f\n"
"ldr b3, [%[a_ptr0]]\n"
"ldr b7, [a_ptr1]\n"
@@ -1981,24 +2249,26 @@
"ld1 {v19.b}[2], [a_ptr4]\n"
"ld1 {v23.b}[2], [a_ptr5]\n"
"3:\n"
- "movi v26.4s, #0\n"
"ldr q24, [%[b_ptr0]]\n"
- "movi v27.4s, #0\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- "movi v28.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr5, #0x40]\n"
- "movi v29.4s, #0\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x80]\n"
- "movi v30.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
- "movi v31.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr5, #0x100]\n"
- ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x140]\n"
- ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x180]\n"
- ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "cbz %[loops], 6f\n"
+ "movi v26.4s, #0\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
@@ -2091,57 +2361,55 @@
".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n"
".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n"
".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n"
- "cbz %[loops], 6f\n"
- "ldr d24, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- "ldr d25, [%[b_ptr0], #0x10]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- "ins v24.d[1], temploadreg0\n"
- "ins v25.d[1], temploadreg1\n"
"b.eq 7f\n"
"8:\n"
"str q26, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"movi v26.4s, #0\n"
+ "ldr d24, [%[b_ptr0]]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q27, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
- ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
+ "ldr d25, [%[b_ptr0], #0x10]\n"
+ "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
"str q28, [c_ptr2]\n"
- "movi v28.4s, #0\n"
"add c_ptr2, c_ptr2, #0x10\n"
- ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
+ "movi v28.4s, #0\n"
+ "ins v24.d[1], temploadreg0\n"
+ "ins v25.d[1], temploadreg1\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
"str q29, [c_ptr3]\n"
- "movi v29.4s, #0\n"
"add c_ptr3, c_ptr3, #0x10\n"
- ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
+ "movi v29.4s, #0\n"
+ "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
+ ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
"str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
+ "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
+ ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
"add c_ptr4, c_ptr4, #0x10\n"
- ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
+ ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
"str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
"add c_ptr5, c_ptr5, #0x10\n"
- ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
+ ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
"prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
"ldr d24, [%[b_ptr0]]\n"
".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n"
"ins v24.d[1], temploadreg0\n"
".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
+ ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
+ ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n"
"ldr d25, [%[b_ptr0], #0x10]\n"
".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n"
@@ -2235,27 +2503,23 @@
".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n"
"ins v24.d[1], temploadreg0\n"
".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n"
".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n"
- "ldr d25, [%[b_ptr0], #0x10]\n"
".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n"
".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n"
".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n"
- "ins v25.d[1], temploadreg1\n"
".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n"
".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n"
".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n"
- "ldr d24, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- "ins v24.d[1], temploadreg0\n"
"b.ne 8b\n"
"7:\n"
"str q26, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v26.4s, #0\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
"str q27, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
@@ -2366,19 +2630,117 @@
".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n"
".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n"
".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n"
+ "b 9f\n"
"6:\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
+ ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
+ ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
+ ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
+ ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n"
+ ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n"
+ ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
+ ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
+ ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n"
+ ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n"
+ ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n"
+ ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n"
+ ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n"
+ ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n"
+ ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n"
+ ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n"
+ ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n"
+ ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n"
+ ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n"
+ ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n"
+ ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n"
+ ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n"
+ ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n"
+ ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n"
+ ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n"
+ ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n"
+ ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n"
+ ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n"
+ ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n"
+ ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n"
+ ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n"
+ ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n"
+ ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n"
+ ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n"
+ ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n"
+ ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n"
+ ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n"
+ ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n"
+ ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n"
+ ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n"
+ ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x10\n"
+ ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n"
+ ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n"
+ ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n"
+ ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n"
+ ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n"
+ ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n"
+ ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n"
+ ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n"
+ ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n"
+ ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n"
+ "9:\n"
"str q26, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q27, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, #0x10\n"
"str q28, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, #0x10\n"
"str q29, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, #0x10\n"
"str q30, [c_ptr4]\n"
- "add c_ptr4, c_ptr4, #0x10\n"
"str q31, [c_ptr5]\n"
- "add c_ptr5, c_ptr5, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -2445,6 +2807,7 @@
"add c_ptr1, %[c_ptr0], #0x0\n"
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
+ "cbnz %[odds], 2f\n"
"ldr q0, [%[a_ptr0]], #0x10\n"
"ldr q4, [a_ptr1], #0x10\n"
"ldr q8, [a_ptr2], #0x10\n"
@@ -2461,24 +2824,41 @@
"ldr q6, [a_ptr1], #0x10\n"
"ldr q10, [a_ptr2], #0x10\n"
"ldr q14, [a_ptr3], #0x10\n"
- "ldr q18, [a_ptr4], #0x10\n"
- "ldr q22, [a_ptr5], #0x10\n"
- "cbnz %[odds], 2f\n"
"ldr d3, [%[a_ptr0]]\n"
+ "ldr q18, [a_ptr4], #0x10\n"
"ldr d7, [a_ptr1]\n"
+ "ldr q22, [a_ptr5], #0x10\n"
"ldr d11, [a_ptr2]\n"
"ldr d15, [a_ptr3]\n"
"ldr d19, [a_ptr4]\n"
"ldr d23, [a_ptr5]\n"
"b 3f\n"
"2:\n"
+ "ldr q0, [%[a_ptr0]], #0x10\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "ldr q4, [a_ptr1], #0x10\n"
+ "ldr q8, [a_ptr2], #0x10\n"
+ "ldr q12, [a_ptr3], #0x10\n"
+ "ldr q16, [a_ptr4], #0x10\n"
+ "ldr q20, [a_ptr5], #0x10\n"
+ "ldr q1, [%[a_ptr0]], #0x10\n"
+ "ldr q5, [a_ptr1], #0x10\n"
+ "ldr q9, [a_ptr2], #0x10\n"
+ "ldr q13, [a_ptr3], #0x10\n"
+ "ldr q17, [a_ptr4], #0x10\n"
+ "ldr q21, [a_ptr5], #0x10\n"
+ "ldr q2, [%[a_ptr0]], #0x10\n"
+ "ldr q6, [a_ptr1], #0x10\n"
+ "ldr q10, [a_ptr2], #0x10\n"
+ "ldr q14, [a_ptr3], #0x10\n"
"ldr s3, [%[a_ptr0]], #0x4\n"
+ "ldr q18, [a_ptr4], #0x10\n"
"ldr s7, [a_ptr1], #0x4\n"
+ "ldr q22, [a_ptr5], #0x10\n"
"ldr s11, [a_ptr2], #0x4\n"
"ldr s15, [a_ptr3], #0x4\n"
"ldr s19, [a_ptr4], #0x4\n"
"ldr s23, [a_ptr5], #0x4\n"
- "subs %[odds], %[odds], #0x1\n"
"b.ne 4f\n"
"ld1 {v3.b}[4], [%[a_ptr0]]\n"
"ld1 {v7.b}[4], [a_ptr1]\n"
@@ -2505,24 +2885,26 @@
"ld1 {v19.b}[6], [a_ptr4]\n"
"ld1 {v23.b}[6], [a_ptr5]\n"
"3:\n"
- "movi v26.4s, #0\n"
"ldr q24, [%[b_ptr0]]\n"
- "movi v27.4s, #0\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- "movi v28.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr5, #0x40]\n"
- "movi v29.4s, #0\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x80]\n"
- "movi v30.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
- "movi v31.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr5, #0x100]\n"
- ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x140]\n"
- ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x180]\n"
- ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "cbz %[loops], 6f\n"
+ "movi v26.4s, #0\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
@@ -2622,68 +3004,66 @@
".inst 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n"
".inst 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n"
".inst 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n"
- "cbz %[loops], 6f\n"
- "ldr d24, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- "ldr d25, [%[b_ptr0], #0x10]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- "ins v24.d[1], temploadreg0\n"
"b.eq 7f\n"
"8:\n"
"str q26, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"movi v26.4s, #0\n"
- "ins v25.d[1], temploadreg1\n"
+ "ldr d24, [%[b_ptr0]]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q27, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
+ "ldr d25, [%[b_ptr0], #0x10]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
"str q28, [c_ptr2]\n"
- "movi v28.4s, #0\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
- ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
- "str q29, [c_ptr3]\n"
- "movi v29.4s, #0\n"
"add c_ptr2, c_ptr2, #0x10\n"
- ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
+ "movi v28.4s, #0\n"
+ "ins v24.d[1], temploadreg0\n"
+ "ins v25.d[1], temploadreg1\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "str q29, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ "movi v29.4s, #0\n"
+ "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
+ ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
"str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
- "add c_ptr3, c_ptr3, #0x10\n"
- ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
+ "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
+ ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
"str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
- "add c_ptr4, c_ptr4, #0x10\n"
- ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
"add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
"ldr d24, [%[b_ptr0]]\n"
".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n"
"ins v24.d[1], temploadreg0\n"
".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
+ ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
+ ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n"
"ldr d25, [%[b_ptr0], #0x10]\n"
".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n"
- "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
"ins v25.d[1], temploadreg1\n"
".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
+ ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n"
".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n"
"ldr d24, [%[b_ptr0]]\n"
".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n"
@@ -2775,27 +3155,23 @@
".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n"
"ins v25.d[1], temploadreg1\n"
".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n"
".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n"
- "ldr d24, [%[b_ptr0]]\n"
".inst 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n"
".inst 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n"
".inst 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n"
- "ins v24.d[1], temploadreg0\n"
".inst 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n"
".inst 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n"
".inst 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n"
- "ldr d25, [%[b_ptr0], #0x10]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
"b.ne 8b\n"
"7:\n"
"str q26, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v26.4s, #0\n"
- "ins v25.d[1], temploadreg1\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
"str q27, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
@@ -2913,19 +3289,124 @@
".inst 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n"
".inst 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n"
".inst 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n"
+ "b 9f\n"
"6:\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
+ ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
+ ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
+ ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
+ ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n"
+ ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n"
+ ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
+ ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
+ ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n"
+ ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n"
+ ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n"
+ ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n"
+ ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n"
+ ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n"
+ ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n"
+ ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n"
+ ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n"
+ ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n"
+ ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n"
+ ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n"
+ ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n"
+ ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n"
+ ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n"
+ ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n"
+ ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n"
+ ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n"
+ ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n"
+ ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n"
+ ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n"
+ ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n"
+ ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n"
+ ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n"
+ ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n"
+ ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n"
+ ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n"
+ ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n"
+ ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n"
+ ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n"
+ ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n"
+ ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n"
+ ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n"
+ ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n"
+ ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n"
+ ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n"
+ ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n"
+ ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n"
+ ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n"
+ ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n"
+ ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n"
+ ".inst 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n"
+ ".inst 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n"
+ ".inst 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n"
+ ".inst 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n"
+ ".inst 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n"
+ ".inst 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n"
+ "9:\n"
"str q26, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q27, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, #0x10\n"
"str q28, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, #0x10\n"
"str q29, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, #0x10\n"
"str q30, [c_ptr4]\n"
- "add c_ptr4, c_ptr4, #0x10\n"
"str q31, [c_ptr5]\n"
- "add c_ptr5, c_ptr5, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -3052,24 +3533,26 @@
"ld1 {v19.b}[10], [a_ptr4]\n"
"ld1 {v23.b}[10], [a_ptr5]\n"
"3:\n"
- "movi v26.4s, #0\n"
"ldr q24, [%[b_ptr0]]\n"
- "movi v27.4s, #0\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- "movi v28.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr5, #0x40]\n"
- "movi v29.4s, #0\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x80]\n"
- "movi v30.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
- "movi v31.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr5, #0x100]\n"
- ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x140]\n"
- ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x180]\n"
- ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "cbz %[loops], 6f\n"
+ "movi v26.4s, #0\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
@@ -3177,57 +3660,55 @@
".inst 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n"
".inst 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n"
".inst 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n"
- "cbz %[loops], 6f\n"
- "ldr d24, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- "ldr d25, [%[b_ptr0], #0x10]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- "ins v24.d[1], temploadreg0\n"
- "ins v25.d[1], temploadreg1\n"
"b.eq 7f\n"
"8:\n"
"str q26, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"movi v26.4s, #0\n"
+ "ldr d24, [%[b_ptr0]]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q27, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
- ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
+ "ldr d25, [%[b_ptr0], #0x10]\n"
+ "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
"str q28, [c_ptr2]\n"
- "movi v28.4s, #0\n"
"add c_ptr2, c_ptr2, #0x10\n"
- ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
+ "movi v28.4s, #0\n"
+ "ins v24.d[1], temploadreg0\n"
+ "ins v25.d[1], temploadreg1\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
"str q29, [c_ptr3]\n"
- "movi v29.4s, #0\n"
"add c_ptr3, c_ptr3, #0x10\n"
- ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
+ "movi v29.4s, #0\n"
+ "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
+ ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
"str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
+ "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
+ ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
"add c_ptr4, c_ptr4, #0x10\n"
- ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
+ ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
"str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
"add c_ptr5, c_ptr5, #0x10\n"
- ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
+ ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
"prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
"ldr d24, [%[b_ptr0]]\n"
".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n"
"ins v24.d[1], temploadreg0\n"
".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
+ ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
+ ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n"
"ldr d25, [%[b_ptr0], #0x10]\n"
".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n"
@@ -3340,27 +3821,23 @@
".inst 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n"
"ins v24.d[1], temploadreg0\n"
".inst 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
".inst 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
".inst 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n"
".inst 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n"
- "ldr d25, [%[b_ptr0], #0x10]\n"
".inst 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n"
".inst 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n"
".inst 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n"
- "ins v25.d[1], temploadreg1\n"
".inst 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n"
".inst 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n"
".inst 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n"
- "ldr d24, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- "ins v24.d[1], temploadreg0\n"
"b.ne 8b\n"
"7:\n"
"str q26, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v26.4s, #0\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
"str q27, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
@@ -3486,19 +3963,132 @@
".inst 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n"
".inst 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n"
".inst 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n"
+ "b 9f\n"
"6:\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
+ ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
+ ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
+ ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
+ ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n"
+ ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n"
+ ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
+ ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
+ ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n"
+ ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n"
+ ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n"
+ ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n"
+ ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n"
+ ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n"
+ ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n"
+ ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n"
+ ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n"
+ ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n"
+ ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n"
+ ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n"
+ ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n"
+ ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n"
+ ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n"
+ ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n"
+ ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n"
+ ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n"
+ ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n"
+ ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n"
+ ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n"
+ ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n"
+ ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n"
+ ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n"
+ ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n"
+ ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n"
+ ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n"
+ ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n"
+ ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n"
+ ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n"
+ ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n"
+ ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n"
+ ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n"
+ ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n"
+ ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n"
+ ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n"
+ ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n"
+ ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n"
+ ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n"
+ ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n"
+ ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x10\n"
+ ".inst 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n"
+ ".inst 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n"
+ ".inst 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n"
+ ".inst 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n"
+ ".inst 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n"
+ ".inst 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n"
+ ".inst 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n"
+ ".inst 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n"
+ ".inst 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n"
+ ".inst 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n"
+ "9:\n"
"str q26, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q27, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, #0x10\n"
"str q28, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, #0x10\n"
"str q29, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, #0x10\n"
"str q30, [c_ptr4]\n"
- "add c_ptr4, c_ptr4, #0x10\n"
"str q31, [c_ptr5]\n"
- "add c_ptr5, c_ptr5, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -3566,6 +4156,7 @@
"add c_ptr1, %[c_ptr0], #0x0\n"
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
+ "cbnz %[odds], 2f\n"
"ldr q0, [%[a_ptr0]], #0x10\n"
"ldr q4, [a_ptr1], #0x10\n"
"ldr q8, [a_ptr2], #0x10\n"
@@ -3584,7 +4175,6 @@
"ldr q14, [a_ptr3], #0x10\n"
"ldr q18, [a_ptr4], #0x10\n"
"ldr q22, [a_ptr5], #0x10\n"
- "cbnz %[odds], 2f\n"
"ldr q3, [%[a_ptr0]]\n"
"ldr q7, [a_ptr1]\n"
"ldr q11, [a_ptr2]\n"
@@ -3593,8 +4183,27 @@
"ldr q23, [a_ptr5]\n"
"b 3f\n"
"2:\n"
+ "ldr q0, [%[a_ptr0]], #0x10\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "ldr q4, [a_ptr1], #0x10\n"
+ "ldr q8, [a_ptr2], #0x10\n"
+ "ldr q12, [a_ptr3], #0x10\n"
+ "ldr q16, [a_ptr4], #0x10\n"
+ "ldr q20, [a_ptr5], #0x10\n"
+ "ldr q1, [%[a_ptr0]], #0x10\n"
+ "ldr q5, [a_ptr1], #0x10\n"
+ "ldr q9, [a_ptr2], #0x10\n"
+ "ldr q13, [a_ptr3], #0x10\n"
+ "ldr q17, [a_ptr4], #0x10\n"
+ "ldr q21, [a_ptr5], #0x10\n"
+ "ldr q2, [%[a_ptr0]], #0x10\n"
+ "ldr q6, [a_ptr1], #0x10\n"
+ "ldr q10, [a_ptr2], #0x10\n"
+ "ldr q14, [a_ptr3], #0x10\n"
"ldr d3, [%[a_ptr0]], #0x8\n"
+ "ldr q18, [a_ptr4], #0x10\n"
"ldr d7, [a_ptr1], #0x8\n"
+ "ldr q22, [a_ptr5], #0x10\n"
"ldr d11, [a_ptr2], #0x8\n"
"ldr d15, [a_ptr3], #0x8\n"
"ldr d19, [a_ptr4], #0x8\n"
@@ -3605,7 +4214,6 @@
"ld1 {v15.s}[2], [a_ptr3], #4\n"
"ld1 {v19.s}[2], [a_ptr4], #4\n"
"ld1 {v23.s}[2], [a_ptr5], #4\n"
- "subs %[odds], %[odds], #0x1\n"
"b.ne 4f\n"
"ld1 {v3.b}[12], [%[a_ptr0]]\n"
"ld1 {v7.b}[12], [a_ptr1]\n"
@@ -3632,24 +4240,26 @@
"ld1 {v19.b}[14], [a_ptr4]\n"
"ld1 {v23.b}[14], [a_ptr5]\n"
"3:\n"
- "movi v26.4s, #0\n"
"ldr q24, [%[b_ptr0]]\n"
- "movi v27.4s, #0\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- "movi v28.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr5, #0x40]\n"
- "movi v29.4s, #0\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x80]\n"
- "movi v30.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
- "movi v31.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr5, #0x100]\n"
- ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x140]\n"
- ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x180]\n"
- ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "cbz %[loops], 6f\n"
+ "movi v26.4s, #0\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
@@ -3764,68 +4374,66 @@
".inst 0x4fafeb3d // sdot v29.4s, v25.16b, v15.4b[3]\n"
".inst 0x4fb3eb3e // sdot v30.4s, v25.16b, v19.4b[3]\n"
".inst 0x4fb7eb3f // sdot v31.4s, v25.16b, v23.4b[3]\n"
- "cbz %[loops], 6f\n"
- "ldr d24, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- "ldr d25, [%[b_ptr0], #0x10]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- "ins v24.d[1], temploadreg0\n"
"b.eq 7f\n"
"8:\n"
"str q26, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"movi v26.4s, #0\n"
- "ins v25.d[1], temploadreg1\n"
+ "ldr d24, [%[b_ptr0]]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q27, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
+ "ldr d25, [%[b_ptr0], #0x10]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
"str q28, [c_ptr2]\n"
- "movi v28.4s, #0\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
- ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
- "str q29, [c_ptr3]\n"
- "movi v29.4s, #0\n"
"add c_ptr2, c_ptr2, #0x10\n"
- ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
+ "movi v28.4s, #0\n"
+ "ins v24.d[1], temploadreg0\n"
+ "ins v25.d[1], temploadreg1\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "str q29, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ "movi v29.4s, #0\n"
+ "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
+ ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
"str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
- "add c_ptr3, c_ptr3, #0x10\n"
- ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
+ "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
+ ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
"str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
- "add c_ptr4, c_ptr4, #0x10\n"
- ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
"add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
"ldr d24, [%[b_ptr0]]\n"
".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n"
"ins v24.d[1], temploadreg0\n"
".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
+ ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
+ ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n"
"ldr d25, [%[b_ptr0], #0x10]\n"
".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n"
- "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
"ins v25.d[1], temploadreg1\n"
".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
+ ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n"
".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n"
"ldr d24, [%[b_ptr0]]\n"
".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n"
@@ -3936,27 +4544,23 @@
".inst 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n"
"ins v25.d[1], temploadreg1\n"
".inst 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
".inst 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
".inst 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n"
".inst 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n"
- "ldr d24, [%[b_ptr0]]\n"
".inst 0x4fa3eb3a // sdot v26.4s, v25.16b, v3.4b[3]\n"
".inst 0x4fa7eb3b // sdot v27.4s, v25.16b, v7.4b[3]\n"
".inst 0x4fabeb3c // sdot v28.4s, v25.16b, v11.4b[3]\n"
- "ins v24.d[1], temploadreg0\n"
".inst 0x4fafeb3d // sdot v29.4s, v25.16b, v15.4b[3]\n"
".inst 0x4fb3eb3e // sdot v30.4s, v25.16b, v19.4b[3]\n"
".inst 0x4fb7eb3f // sdot v31.4s, v25.16b, v23.4b[3]\n"
- "ldr d25, [%[b_ptr0], #0x10]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
"b.ne 8b\n"
"7:\n"
"str q26, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v26.4s, #0\n"
- "ins v25.d[1], temploadreg1\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
"str q27, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
@@ -4089,19 +4693,139 @@
".inst 0x4fafeb3d // sdot v29.4s, v25.16b, v15.4b[3]\n"
".inst 0x4fb3eb3e // sdot v30.4s, v25.16b, v19.4b[3]\n"
".inst 0x4fb7eb3f // sdot v31.4s, v25.16b, v23.4b[3]\n"
+ "b 9f\n"
"6:\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
+ ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
+ ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
+ ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
+ ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n"
+ ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n"
+ ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
+ ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
+ ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n"
+ ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n"
+ ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n"
+ ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n"
+ ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n"
+ ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n"
+ ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n"
+ ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n"
+ ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n"
+ ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n"
+ ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n"
+ ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n"
+ ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n"
+ ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n"
+ ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n"
+ ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n"
+ ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n"
+ ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n"
+ ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n"
+ ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n"
+ ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n"
+ ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n"
+ ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n"
+ ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n"
+ ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n"
+ ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n"
+ ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n"
+ ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n"
+ ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n"
+ ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n"
+ ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n"
+ ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n"
+ ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n"
+ ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n"
+ ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n"
+ ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n"
+ ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n"
+ ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n"
+ ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n"
+ ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n"
+ ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n"
+ ".inst 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n"
+ ".inst 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n"
+ ".inst 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n"
+ ".inst 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n"
+ ".inst 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n"
+ ".inst 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n"
+ ".inst 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n"
+ ".inst 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n"
+ ".inst 0x4fa3eb3a // sdot v26.4s, v25.16b, v3.4b[3]\n"
+ ".inst 0x4fa7eb3b // sdot v27.4s, v25.16b, v7.4b[3]\n"
+ ".inst 0x4fabeb3c // sdot v28.4s, v25.16b, v11.4b[3]\n"
+ ".inst 0x4fafeb3d // sdot v29.4s, v25.16b, v15.4b[3]\n"
+ ".inst 0x4fb3eb3e // sdot v30.4s, v25.16b, v19.4b[3]\n"
+ ".inst 0x4fb7eb3f // sdot v31.4s, v25.16b, v23.4b[3]\n"
+ "9:\n"
"str q26, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q27, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, #0x10\n"
"str q28, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, #0x10\n"
"str q29, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, #0x10\n"
"str q30, [c_ptr4]\n"
- "add c_ptr4, c_ptr4, #0x10\n"
"str q31, [c_ptr5]\n"
- "add c_ptr5, c_ptr5, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x6/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_6x4/generic.cpp
similarity index 80%
rename from src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x6/generic.cpp
rename to src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_6x4/generic.cpp
index 88ad36a..9ff3971 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x6/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_6x4/generic.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2019 Arm Limited.
+ * Copyright (c) 2019-2020 Arm Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -33,7 +33,7 @@
namespace arm_gemm {
-void a64_smallK_hybrid_s8s32_dot_4x6(const int8_t *A, int lda, const int8_t *B, int32_t *C, int ldc, int M, int N, int K, const int32_t *, Activation, bool) {
+void a64_smallK_hybrid_s8s32_dot_6x4(const int8_t *A, int lda, const int8_t *B, int32_t *C, int ldc, int M, int N, int K, const int32_t *, Activation, bool) {
const long loops_count = iceildiv(N, (int)4) - 1;
const long ldab = lda * sizeof(int8_t);
const long ldcb = ldc * sizeof(int32_t);
@@ -93,6 +93,7 @@
"add c_ptr1, %[c_ptr0], #0x0\n"
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
+ "cbnz %[odds], 2f\n"
"ldr q0, [%[a_ptr0]], #0x10\n"
"ldr q3, [a_ptr1], #0x10\n"
"ldr q6, [a_ptr2], #0x10\n"
@@ -103,18 +104,29 @@
"ldr q4, [a_ptr1], #0x10\n"
"ldr q7, [a_ptr2], #0x10\n"
"ldr q10, [a_ptr3], #0x10\n"
+ "ldr s2, [%[a_ptr0]]\n"
+ "ldr q13, [a_ptr4], #0x10\n"
+ "ldr s5, [a_ptr1]\n"
+ "ldr q16, [a_ptr5], #0x10\n"
+ "ldr s8, [a_ptr2]\n"
+ "ldr s11, [a_ptr3]\n"
+ "ldr s14, [a_ptr4]\n"
+ "ldr s17, [a_ptr5]\n"
+ "b 3f\n"
+ "2:\n"
+ "ldr q0, [%[a_ptr0]], #0x10\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "ldr q3, [a_ptr1], #0x10\n"
+ "ldr q6, [a_ptr2], #0x10\n"
+ "ldr q9, [a_ptr3], #0x10\n"
+ "ldr q12, [a_ptr4], #0x10\n"
+ "ldr q15, [a_ptr5], #0x10\n"
+ "ldr q1, [%[a_ptr0]], #0x10\n"
+ "ldr q4, [a_ptr1], #0x10\n"
+ "ldr q7, [a_ptr2], #0x10\n"
+ "ldr q10, [a_ptr3], #0x10\n"
"ldr q13, [a_ptr4], #0x10\n"
"ldr q16, [a_ptr5], #0x10\n"
- "cbnz %[odds], 2f\n"
- "ldr s2, [%[a_ptr0]]\n"
- "ldr s5, [a_ptr1]\n"
- "ldr s8, [a_ptr2]\n"
- "ldr s11, [a_ptr3]\n"
- "ldr s14, [a_ptr4]\n"
- "ldr s17, [a_ptr5]\n"
- "b 3f\n"
- "2:\n"
- "subs %[odds], %[odds], #0x1\n"
"b.ne 4f\n"
"ldr b2, [%[a_ptr0]]\n"
"ldr b5, [a_ptr1]\n"
@@ -141,40 +153,42 @@
"ld1 {v14.b}[2], [a_ptr4]\n"
"ld1 {v17.b}[2], [a_ptr5]\n"
"3:\n"
- "movi v26.4s, #0\n"
"ldr q18, [%[b_ptr0]]\n"
- "movi v27.4s, #0\n"
- "ldr q19, [%[b_ptr0], #0x10]\n"
- "movi v28.4s, #0\n"
- "ldr q20, [%[b_ptr0], #0x20]\n"
- "movi v29.4s, #0\n"
- "ldr q21, [%[b_ptr0], #0x30]\n"
- "movi v30.4s, #0\n"
- "ldr q22, [%[b_ptr0], #0x40]\n"
- "movi v31.4s, #0\n"
- "ldr q23, [%[b_ptr0], #0x50]\n"
- ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n"
- "ldr q24, [%[b_ptr0], #0x60]\n"
- ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n"
- "ldr q25, [%[b_ptr0], #0x70]\n"
- ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x40]\n"
- ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x80]\n"
- ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
"prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
- ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x100]\n"
- ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n"
+ "ldr q22, [%[b_ptr0], #0x40]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x140]\n"
- ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n"
+ "ldr q23, [%[b_ptr0], #0x50]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x180]\n"
- ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n"
+ "ldr q24, [%[b_ptr0], #0x60]\n"
+ "ldr q25, [%[b_ptr0], #0x70]\n"
"add %[b_ptr0], %[b_ptr0], #0x80\n"
- ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n"
+ "cbz %[loops], 6f\n"
+ "movi v26.4s, #0\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n"
+ ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n"
+ ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n"
+ ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n"
+ ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n"
+ ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n"
"ldr q18, [%[b_ptr0]]\n"
- ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n"
+ ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n"
"add %[b_ptr0], %[b_ptr0], #0x10\n"
+ ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n"
+ ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n"
+ ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n"
+ ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n"
".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n"
".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n"
".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n"
@@ -218,139 +232,201 @@
".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n"
".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n"
".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n"
- "cbz %[loops], 6f\n"
- "ldr q18, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ldr q19, [%[b_ptr0], #0x10]\n"
- "ldr q20, [%[b_ptr0], #0x20]\n"
- "ldr q21, [%[b_ptr0], #0x30]\n"
- "ldr q22, [%[b_ptr0], #0x40]\n"
- "ldr q23, [%[b_ptr0], #0x50]\n"
- "ldr q24, [%[b_ptr0], #0x60]\n"
- "ldr q25, [%[b_ptr0], #0x70]\n"
"b.eq 7f\n"
"8:\n"
"str q26, [%[c_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "movi v26.4s, #0\n"
"subs %[loops], %[loops], #0x1\n"
- "str q27, [c_ptr1]\n"
+ "movi v26.4s, #0\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
- "movi v27.4s, #0\n"
+ "str q27, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
+ "movi v27.4s, #0\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n"
"str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
+ "ldr q22, [%[b_ptr0], #0x40]\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n"
"str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
- "add c_ptr2, c_ptr2, #0x10\n"
+ "ldr q23, [%[b_ptr0], #0x50]\n"
".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n"
+ "ldr q24, [%[b_ptr0], #0x60]\n"
+ ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n"
"str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
- "add c_ptr3, c_ptr3, #0x10\n"
+ "ldr q25, [%[b_ptr0], #0x70]\n"
".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n"
"str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n"
"add c_ptr5, c_ptr5, #0x10\n"
".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n"
- "ldr q18, [%[b_ptr0]]\n"
- ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n"
- "add %[b_ptr0], %[b_ptr0], #0x10\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ "ldr q18, [%[b_ptr0]]\n"
".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x10\n"
".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n"
- "ldr q19, [%[b_ptr0], #0x10]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n"
- "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n"
+ "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n"
+ "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n"
".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n"
- "ldr q20, [%[b_ptr0], #0x20]\n"
".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n"
".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n"
".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n"
".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n"
".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n"
".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n"
- "ldr q21, [%[b_ptr0], #0x30]\n"
".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n"
".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n"
".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n"
".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n"
".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n"
".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n"
- "ldr q22, [%[b_ptr0], #0x40]\n"
".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n"
".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n"
".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n"
".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n"
".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n"
".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n"
- "ldr q23, [%[b_ptr0], #0x50]\n"
".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n"
".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n"
".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n"
".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n"
- "ldr q24, [%[b_ptr0], #0x60]\n"
".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n"
".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n"
".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n"
".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n"
- "ldr q25, [%[b_ptr0], #0x70]\n"
".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n"
".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n"
".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n"
".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n"
".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n"
".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n"
- "ldr q18, [%[b_ptr0]]\n"
"b.ne 8b\n"
"7:\n"
"str q26, [%[c_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "movi v26.4s, #0\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
+ "movi v26.4s, #0\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
"str q27, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n"
"str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
+ "ldr q22, [%[b_ptr0], #0x40]\n"
"add c_ptr2, c_ptr2, #0x10\n"
".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n"
"str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
- "add c_ptr3, c_ptr3, #0x10\n"
+ "ldr q23, [%[b_ptr0], #0x50]\n"
".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n"
+ "ldr q24, [%[b_ptr0], #0x60]\n"
+ ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n"
"str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
- "add c_ptr4, c_ptr4, #0x10\n"
+ "ldr q25, [%[b_ptr0], #0x70]\n"
".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n"
"str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
- "add c_ptr5, c_ptr5, #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n"
- ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
+ ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x10\n"
+ ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n"
+ ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n"
+ ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n"
+ ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n"
+ ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n"
+ ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n"
+ ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n"
+ ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n"
+ ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n"
+ ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n"
+ ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n"
+ ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n"
+ ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n"
+ ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n"
+ ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n"
+ ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n"
+ ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n"
+ ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n"
+ ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n"
+ ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n"
+ ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n"
+ ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n"
+ ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n"
+ ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n"
+ ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n"
+ ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n"
+ ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n"
+ ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n"
+ ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n"
+ ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n"
+ ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n"
+ ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n"
+ ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n"
+ "b 9f\n"
+ "6:\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n"
+ ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n"
+ ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n"
+ ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n"
+ ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n"
".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n"
"ldr q18, [%[b_ptr0]]\n"
- ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n"
+ ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n"
"add %[b_ptr0], %[b_ptr0], #0x10\n"
+ ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n"
".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n"
".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n"
".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n"
@@ -397,19 +473,14 @@
".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n"
".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n"
".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n"
- "6:\n"
+ "9:\n"
"str q26, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q27, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, #0x10\n"
"str q28, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, #0x10\n"
"str q29, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, #0x10\n"
"str q30, [c_ptr4]\n"
- "add c_ptr4, c_ptr4, #0x10\n"
"str q31, [c_ptr5]\n"
- "add c_ptr5, c_ptr5, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -468,6 +539,7 @@
"add c_ptr1, %[c_ptr0], #0x0\n"
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
+ "cbnz %[odds], 2f\n"
"ldr q0, [%[a_ptr0]], #0x10\n"
"ldr q3, [a_ptr1], #0x10\n"
"ldr q6, [a_ptr2], #0x10\n"
@@ -478,24 +550,35 @@
"ldr q4, [a_ptr1], #0x10\n"
"ldr q7, [a_ptr2], #0x10\n"
"ldr q10, [a_ptr3], #0x10\n"
- "ldr q13, [a_ptr4], #0x10\n"
- "ldr q16, [a_ptr5], #0x10\n"
- "cbnz %[odds], 2f\n"
"ldr d2, [%[a_ptr0]]\n"
+ "ldr q13, [a_ptr4], #0x10\n"
"ldr d5, [a_ptr1]\n"
+ "ldr q16, [a_ptr5], #0x10\n"
"ldr d8, [a_ptr2]\n"
"ldr d11, [a_ptr3]\n"
"ldr d14, [a_ptr4]\n"
"ldr d17, [a_ptr5]\n"
"b 3f\n"
"2:\n"
+ "ldr q0, [%[a_ptr0]], #0x10\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "ldr q3, [a_ptr1], #0x10\n"
+ "ldr q6, [a_ptr2], #0x10\n"
+ "ldr q9, [a_ptr3], #0x10\n"
+ "ldr q12, [a_ptr4], #0x10\n"
+ "ldr q15, [a_ptr5], #0x10\n"
+ "ldr q1, [%[a_ptr0]], #0x10\n"
+ "ldr q4, [a_ptr1], #0x10\n"
+ "ldr q7, [a_ptr2], #0x10\n"
+ "ldr q10, [a_ptr3], #0x10\n"
"ldr s2, [%[a_ptr0]], #0x4\n"
+ "ldr q13, [a_ptr4], #0x10\n"
"ldr s5, [a_ptr1], #0x4\n"
+ "ldr q16, [a_ptr5], #0x10\n"
"ldr s8, [a_ptr2], #0x4\n"
"ldr s11, [a_ptr3], #0x4\n"
"ldr s14, [a_ptr4], #0x4\n"
"ldr s17, [a_ptr5], #0x4\n"
- "subs %[odds], %[odds], #0x1\n"
"b.ne 4f\n"
"ld1 {v2.b}[4], [%[a_ptr0]]\n"
"ld1 {v5.b}[4], [a_ptr1]\n"
@@ -522,38 +605,40 @@
"ld1 {v14.b}[6], [a_ptr4]\n"
"ld1 {v17.b}[6], [a_ptr5]\n"
"3:\n"
- "movi v26.4s, #0\n"
"ldr q18, [%[b_ptr0]]\n"
- "movi v27.4s, #0\n"
- "ldr q19, [%[b_ptr0], #0x10]\n"
- "movi v28.4s, #0\n"
- "ldr q20, [%[b_ptr0], #0x20]\n"
- "movi v29.4s, #0\n"
- "ldr q21, [%[b_ptr0], #0x30]\n"
- "movi v30.4s, #0\n"
- "ldr q22, [%[b_ptr0], #0x40]\n"
- "movi v31.4s, #0\n"
- "ldr q23, [%[b_ptr0], #0x50]\n"
- ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n"
- "ldr q24, [%[b_ptr0], #0x60]\n"
- ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n"
- "ldr q25, [%[b_ptr0], #0x70]\n"
- ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x40]\n"
- ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x80]\n"
- ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
"prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
- ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x100]\n"
- ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n"
+ "ldr q22, [%[b_ptr0], #0x40]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x140]\n"
- ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n"
+ "ldr q23, [%[b_ptr0], #0x50]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x180]\n"
- ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n"
+ "ldr q24, [%[b_ptr0], #0x60]\n"
+ "ldr q25, [%[b_ptr0], #0x70]\n"
"add %[b_ptr0], %[b_ptr0], #0x80\n"
- ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n"
+ "cbz %[loops], 6f\n"
+ "movi v26.4s, #0\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n"
+ ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n"
+ ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n"
+ ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n"
+ ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n"
+ ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n"
"ldr q18, [%[b_ptr0]]\n"
+ ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n"
+ ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n"
+ ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n"
+ ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n"
".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n"
".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n"
"ldr q19, [%[b_ptr0], #0x10]\n"
@@ -606,144 +691,213 @@
".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n"
".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n"
".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n"
- "cbz %[loops], 6f\n"
- "ldr q18, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ldr q19, [%[b_ptr0], #0x10]\n"
- "ldr q20, [%[b_ptr0], #0x20]\n"
- "ldr q21, [%[b_ptr0], #0x30]\n"
- "ldr q22, [%[b_ptr0], #0x40]\n"
- "ldr q23, [%[b_ptr0], #0x50]\n"
- "ldr q24, [%[b_ptr0], #0x60]\n"
- "ldr q25, [%[b_ptr0], #0x70]\n"
"b.eq 7f\n"
"8:\n"
"str q26, [%[c_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "movi v26.4s, #0\n"
"subs %[loops], %[loops], #0x1\n"
- "str q27, [c_ptr1]\n"
+ "movi v26.4s, #0\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
- "movi v27.4s, #0\n"
+ "str q27, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
+ "movi v27.4s, #0\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n"
"str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
+ "ldr q22, [%[b_ptr0], #0x40]\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n"
"str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
- "add c_ptr2, c_ptr2, #0x10\n"
+ "ldr q23, [%[b_ptr0], #0x50]\n"
".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n"
+ "ldr q24, [%[b_ptr0], #0x60]\n"
+ ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n"
"str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
- "add c_ptr3, c_ptr3, #0x10\n"
+ "ldr q25, [%[b_ptr0], #0x70]\n"
".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n"
"str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n"
"add c_ptr5, c_ptr5, #0x10\n"
".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n"
- "ldr q18, [%[b_ptr0]]\n"
- ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
+ "ldr q18, [%[b_ptr0]]\n"
".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n"
"ldr q19, [%[b_ptr0], #0x10]\n"
".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n"
+ "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n"
+ "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n"
- "ldr q20, [%[b_ptr0], #0x20]\n"
".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n"
".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n"
".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n"
".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n"
".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n"
".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n"
- "ldr q21, [%[b_ptr0], #0x30]\n"
".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n"
".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n"
".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n"
".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n"
".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n"
".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n"
- "ldr q22, [%[b_ptr0], #0x40]\n"
".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n"
".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n"
".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n"
".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n"
".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n"
".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n"
- "ldr q23, [%[b_ptr0], #0x50]\n"
".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n"
".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n"
".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n"
".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n"
- "ldr q24, [%[b_ptr0], #0x60]\n"
".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n"
".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n"
".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n"
".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n"
- "ldr q25, [%[b_ptr0], #0x70]\n"
".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n"
".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n"
".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n"
".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n"
".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n"
".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n"
- "ldr q18, [%[b_ptr0]]\n"
".inst 0x4fa2e27a // sdot v26.4s, v19.16b, v2.4b[1]\n"
".inst 0x4fa5e27b // sdot v27.4s, v19.16b, v5.4b[1]\n"
".inst 0x4fa8e27c // sdot v28.4s, v19.16b, v8.4b[1]\n"
".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n"
".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n"
".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n"
- "ldr q19, [%[b_ptr0], #0x10]\n"
"b.ne 8b\n"
"7:\n"
"str q26, [%[c_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "movi v26.4s, #0\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
+ "movi v26.4s, #0\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
"str q27, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n"
"str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
+ "ldr q22, [%[b_ptr0], #0x40]\n"
"add c_ptr2, c_ptr2, #0x10\n"
".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n"
"str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
- "add c_ptr3, c_ptr3, #0x10\n"
+ "ldr q23, [%[b_ptr0], #0x50]\n"
".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n"
+ "ldr q24, [%[b_ptr0], #0x60]\n"
+ ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n"
"str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
- "add c_ptr4, c_ptr4, #0x10\n"
+ "ldr q25, [%[b_ptr0], #0x70]\n"
".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n"
"str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
- "add c_ptr5, c_ptr5, #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n"
- ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
+ ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n"
+ ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n"
+ ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n"
+ ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n"
+ ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n"
+ ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n"
+ ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n"
+ ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n"
+ ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n"
+ ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n"
+ ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n"
+ ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n"
+ ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n"
+ ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n"
+ ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n"
+ ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n"
+ ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n"
+ ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n"
+ ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n"
+ ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n"
+ ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n"
+ ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n"
+ ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n"
+ ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n"
+ ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n"
+ ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n"
+ ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n"
+ ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n"
+ ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n"
+ ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n"
+ ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n"
+ ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n"
+ ".inst 0x4fa2e27a // sdot v26.4s, v19.16b, v2.4b[1]\n"
+ ".inst 0x4fa5e27b // sdot v27.4s, v19.16b, v5.4b[1]\n"
+ ".inst 0x4fa8e27c // sdot v28.4s, v19.16b, v8.4b[1]\n"
+ ".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n"
+ ".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n"
+ ".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n"
+ "b 9f\n"
+ "6:\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n"
+ ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n"
+ ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n"
+ ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n"
+ ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n"
".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n"
"ldr q18, [%[b_ptr0]]\n"
+ ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n"
".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n"
".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n"
".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n"
@@ -799,19 +953,14 @@
".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n"
".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n"
".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n"
- "6:\n"
+ "9:\n"
"str q26, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q27, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, #0x10\n"
"str q28, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, #0x10\n"
"str q29, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, #0x10\n"
"str q30, [c_ptr4]\n"
- "add c_ptr4, c_ptr4, #0x10\n"
"str q31, [c_ptr5]\n"
- "add c_ptr5, c_ptr5, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -924,38 +1073,40 @@
"ld1 {v14.b}[10], [a_ptr4]\n"
"ld1 {v17.b}[10], [a_ptr5]\n"
"3:\n"
- "movi v26.4s, #0\n"
"ldr q18, [%[b_ptr0]]\n"
- "movi v27.4s, #0\n"
- "ldr q19, [%[b_ptr0], #0x10]\n"
- "movi v28.4s, #0\n"
- "ldr q20, [%[b_ptr0], #0x20]\n"
- "movi v29.4s, #0\n"
- "ldr q21, [%[b_ptr0], #0x30]\n"
- "movi v30.4s, #0\n"
- "ldr q22, [%[b_ptr0], #0x40]\n"
- "movi v31.4s, #0\n"
- "ldr q23, [%[b_ptr0], #0x50]\n"
- ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n"
- "ldr q24, [%[b_ptr0], #0x60]\n"
- ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n"
- "ldr q25, [%[b_ptr0], #0x70]\n"
- ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x40]\n"
- ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x80]\n"
- ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
"prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
- ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x100]\n"
- ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n"
+ "ldr q22, [%[b_ptr0], #0x40]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x140]\n"
- ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n"
+ "ldr q23, [%[b_ptr0], #0x50]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x180]\n"
- ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n"
+ "ldr q24, [%[b_ptr0], #0x60]\n"
+ "ldr q25, [%[b_ptr0], #0x70]\n"
"add %[b_ptr0], %[b_ptr0], #0x80\n"
- ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n"
+ "cbz %[loops], 6f\n"
+ "movi v26.4s, #0\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n"
+ ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n"
+ ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n"
+ ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n"
+ ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n"
+ ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n"
"ldr q18, [%[b_ptr0]]\n"
+ ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n"
+ ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n"
+ ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n"
+ ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n"
".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n"
".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n"
"ldr q19, [%[b_ptr0], #0x10]\n"
@@ -1015,62 +1166,60 @@
".inst 0x4f8bea9d // sdot v29.4s, v20.16b, v11.4b[2]\n"
".inst 0x4f8eea9e // sdot v30.4s, v20.16b, v14.4b[2]\n"
".inst 0x4f91ea9f // sdot v31.4s, v20.16b, v17.4b[2]\n"
- "cbz %[loops], 6f\n"
- "ldr q18, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ldr q19, [%[b_ptr0], #0x10]\n"
- "ldr q20, [%[b_ptr0], #0x20]\n"
- "ldr q21, [%[b_ptr0], #0x30]\n"
- "ldr q22, [%[b_ptr0], #0x40]\n"
- "ldr q23, [%[b_ptr0], #0x50]\n"
- "ldr q24, [%[b_ptr0], #0x60]\n"
- "ldr q25, [%[b_ptr0], #0x70]\n"
"b.eq 7f\n"
"8:\n"
"str q26, [%[c_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "movi v26.4s, #0\n"
"subs %[loops], %[loops], #0x1\n"
- "str q27, [c_ptr1]\n"
+ "movi v26.4s, #0\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
- "movi v27.4s, #0\n"
+ "str q27, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
+ "movi v27.4s, #0\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n"
"str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
+ "ldr q22, [%[b_ptr0], #0x40]\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n"
"str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
- "add c_ptr2, c_ptr2, #0x10\n"
+ "ldr q23, [%[b_ptr0], #0x50]\n"
".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n"
+ "ldr q24, [%[b_ptr0], #0x60]\n"
+ ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n"
"str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
- "add c_ptr3, c_ptr3, #0x10\n"
+ "ldr q25, [%[b_ptr0], #0x70]\n"
".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n"
"str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n"
"add c_ptr5, c_ptr5, #0x10\n"
".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n"
- "ldr q18, [%[b_ptr0]]\n"
- ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
+ "ldr q18, [%[b_ptr0]]\n"
".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n"
"ldr q19, [%[b_ptr0], #0x10]\n"
".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n"
+ "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n"
+ "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n"
".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n"
"ldr q20, [%[b_ptr0], #0x20]\n"
@@ -1081,85 +1230,163 @@
".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n"
".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n"
".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n"
- "ldr q21, [%[b_ptr0], #0x30]\n"
".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n"
".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n"
".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n"
".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n"
".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n"
".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n"
- "ldr q22, [%[b_ptr0], #0x40]\n"
".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n"
".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n"
".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n"
".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n"
".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n"
".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n"
- "ldr q23, [%[b_ptr0], #0x50]\n"
".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n"
".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n"
".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n"
".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n"
- "ldr q24, [%[b_ptr0], #0x60]\n"
".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n"
".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n"
".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n"
".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n"
- "ldr q25, [%[b_ptr0], #0x70]\n"
".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n"
".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n"
".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n"
".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n"
".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n"
".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n"
- "ldr q18, [%[b_ptr0]]\n"
".inst 0x4fa2e27a // sdot v26.4s, v19.16b, v2.4b[1]\n"
".inst 0x4fa5e27b // sdot v27.4s, v19.16b, v5.4b[1]\n"
".inst 0x4fa8e27c // sdot v28.4s, v19.16b, v8.4b[1]\n"
".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n"
".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n"
".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n"
- "ldr q19, [%[b_ptr0], #0x10]\n"
".inst 0x4f82ea9a // sdot v26.4s, v20.16b, v2.4b[2]\n"
".inst 0x4f85ea9b // sdot v27.4s, v20.16b, v5.4b[2]\n"
".inst 0x4f88ea9c // sdot v28.4s, v20.16b, v8.4b[2]\n"
".inst 0x4f8bea9d // sdot v29.4s, v20.16b, v11.4b[2]\n"
".inst 0x4f8eea9e // sdot v30.4s, v20.16b, v14.4b[2]\n"
".inst 0x4f91ea9f // sdot v31.4s, v20.16b, v17.4b[2]\n"
- "ldr q20, [%[b_ptr0], #0x20]\n"
"b.ne 8b\n"
"7:\n"
"str q26, [%[c_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "movi v26.4s, #0\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
+ "movi v26.4s, #0\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
"str q27, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n"
"str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
+ "ldr q22, [%[b_ptr0], #0x40]\n"
"add c_ptr2, c_ptr2, #0x10\n"
".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n"
"str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
- "add c_ptr3, c_ptr3, #0x10\n"
+ "ldr q23, [%[b_ptr0], #0x50]\n"
".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n"
+ "ldr q24, [%[b_ptr0], #0x60]\n"
+ ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n"
"str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
- "add c_ptr4, c_ptr4, #0x10\n"
+ "ldr q25, [%[b_ptr0], #0x70]\n"
".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n"
"str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
- "add c_ptr5, c_ptr5, #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n"
- ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
+ ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n"
+ ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n"
+ ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n"
+ ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n"
+ ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n"
+ ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n"
+ ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n"
+ ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
+ ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x30\n"
+ ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n"
+ ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n"
+ ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n"
+ ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n"
+ ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n"
+ ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n"
+ ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n"
+ ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n"
+ ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n"
+ ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n"
+ ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n"
+ ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n"
+ ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n"
+ ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n"
+ ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n"
+ ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n"
+ ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n"
+ ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n"
+ ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n"
+ ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n"
+ ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n"
+ ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n"
+ ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n"
+ ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n"
+ ".inst 0x4fa2e27a // sdot v26.4s, v19.16b, v2.4b[1]\n"
+ ".inst 0x4fa5e27b // sdot v27.4s, v19.16b, v5.4b[1]\n"
+ ".inst 0x4fa8e27c // sdot v28.4s, v19.16b, v8.4b[1]\n"
+ ".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n"
+ ".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n"
+ ".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n"
+ ".inst 0x4f82ea9a // sdot v26.4s, v20.16b, v2.4b[2]\n"
+ ".inst 0x4f85ea9b // sdot v27.4s, v20.16b, v5.4b[2]\n"
+ ".inst 0x4f88ea9c // sdot v28.4s, v20.16b, v8.4b[2]\n"
+ ".inst 0x4f8bea9d // sdot v29.4s, v20.16b, v11.4b[2]\n"
+ ".inst 0x4f8eea9e // sdot v30.4s, v20.16b, v14.4b[2]\n"
+ ".inst 0x4f91ea9f // sdot v31.4s, v20.16b, v17.4b[2]\n"
+ "b 9f\n"
+ "6:\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n"
+ ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n"
+ ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n"
+ ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n"
+ ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n"
".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n"
"ldr q18, [%[b_ptr0]]\n"
+ ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n"
".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n"
".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n"
".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n"
@@ -1222,19 +1449,14 @@
".inst 0x4f8bea9d // sdot v29.4s, v20.16b, v11.4b[2]\n"
".inst 0x4f8eea9e // sdot v30.4s, v20.16b, v14.4b[2]\n"
".inst 0x4f91ea9f // sdot v31.4s, v20.16b, v17.4b[2]\n"
- "6:\n"
+ "9:\n"
"str q26, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q27, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, #0x10\n"
"str q28, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, #0x10\n"
"str q29, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, #0x10\n"
"str q30, [c_ptr4]\n"
- "add c_ptr4, c_ptr4, #0x10\n"
"str q31, [c_ptr5]\n"
- "add c_ptr5, c_ptr5, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -1293,6 +1515,7 @@
"add c_ptr1, %[c_ptr0], #0x0\n"
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
+ "cbnz %[odds], 2f\n"
"ldr q0, [%[a_ptr0]], #0x10\n"
"ldr q3, [a_ptr1], #0x10\n"
"ldr q6, [a_ptr2], #0x10\n"
@@ -1305,7 +1528,6 @@
"ldr q10, [a_ptr3], #0x10\n"
"ldr q13, [a_ptr4], #0x10\n"
"ldr q16, [a_ptr5], #0x10\n"
- "cbnz %[odds], 2f\n"
"ldr q2, [%[a_ptr0]]\n"
"ldr q5, [a_ptr1]\n"
"ldr q8, [a_ptr2]\n"
@@ -1314,8 +1536,21 @@
"ldr q17, [a_ptr5]\n"
"b 3f\n"
"2:\n"
+ "ldr q0, [%[a_ptr0]], #0x10\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "ldr q3, [a_ptr1], #0x10\n"
+ "ldr q6, [a_ptr2], #0x10\n"
+ "ldr q9, [a_ptr3], #0x10\n"
+ "ldr q12, [a_ptr4], #0x10\n"
+ "ldr q15, [a_ptr5], #0x10\n"
+ "ldr q1, [%[a_ptr0]], #0x10\n"
+ "ldr q4, [a_ptr1], #0x10\n"
+ "ldr q7, [a_ptr2], #0x10\n"
+ "ldr q10, [a_ptr3], #0x10\n"
"ldr d2, [%[a_ptr0]], #0x8\n"
+ "ldr q13, [a_ptr4], #0x10\n"
"ldr d5, [a_ptr1], #0x8\n"
+ "ldr q16, [a_ptr5], #0x10\n"
"ldr d8, [a_ptr2], #0x8\n"
"ldr d11, [a_ptr3], #0x8\n"
"ldr d14, [a_ptr4], #0x8\n"
@@ -1326,7 +1561,6 @@
"ld1 {v11.s}[2], [a_ptr3], #4\n"
"ld1 {v14.s}[2], [a_ptr4], #4\n"
"ld1 {v17.s}[2], [a_ptr5], #4\n"
- "subs %[odds], %[odds], #0x1\n"
"b.ne 4f\n"
"ld1 {v2.b}[12], [%[a_ptr0]]\n"
"ld1 {v5.b}[12], [a_ptr1]\n"
@@ -1353,38 +1587,40 @@
"ld1 {v14.b}[14], [a_ptr4]\n"
"ld1 {v17.b}[14], [a_ptr5]\n"
"3:\n"
- "movi v26.4s, #0\n"
"ldr q18, [%[b_ptr0]]\n"
- "movi v27.4s, #0\n"
- "ldr q19, [%[b_ptr0], #0x10]\n"
- "movi v28.4s, #0\n"
- "ldr q20, [%[b_ptr0], #0x20]\n"
- "movi v29.4s, #0\n"
- "ldr q21, [%[b_ptr0], #0x30]\n"
- "movi v30.4s, #0\n"
- "ldr q22, [%[b_ptr0], #0x40]\n"
- "movi v31.4s, #0\n"
- "ldr q23, [%[b_ptr0], #0x50]\n"
- ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n"
- "ldr q24, [%[b_ptr0], #0x60]\n"
- ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n"
- "ldr q25, [%[b_ptr0], #0x70]\n"
- ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x40]\n"
- ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x80]\n"
- ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
"prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
- ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x100]\n"
- ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n"
+ "ldr q22, [%[b_ptr0], #0x40]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x140]\n"
- ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n"
+ "ldr q23, [%[b_ptr0], #0x50]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x180]\n"
- ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n"
+ "ldr q24, [%[b_ptr0], #0x60]\n"
+ "ldr q25, [%[b_ptr0], #0x70]\n"
"add %[b_ptr0], %[b_ptr0], #0x80\n"
- ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n"
+ "cbz %[loops], 6f\n"
+ "movi v26.4s, #0\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n"
+ ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n"
+ ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n"
+ ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n"
+ ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n"
+ ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n"
"ldr q18, [%[b_ptr0]]\n"
+ ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n"
+ ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n"
+ ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n"
+ ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n"
".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n"
".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n"
"ldr q19, [%[b_ptr0], #0x10]\n"
@@ -1451,62 +1687,60 @@
".inst 0x4fabeabd // sdot v29.4s, v21.16b, v11.4b[3]\n"
".inst 0x4faeeabe // sdot v30.4s, v21.16b, v14.4b[3]\n"
".inst 0x4fb1eabf // sdot v31.4s, v21.16b, v17.4b[3]\n"
- "cbz %[loops], 6f\n"
- "ldr q18, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ldr q19, [%[b_ptr0], #0x10]\n"
- "ldr q20, [%[b_ptr0], #0x20]\n"
- "ldr q21, [%[b_ptr0], #0x30]\n"
- "ldr q22, [%[b_ptr0], #0x40]\n"
- "ldr q23, [%[b_ptr0], #0x50]\n"
- "ldr q24, [%[b_ptr0], #0x60]\n"
- "ldr q25, [%[b_ptr0], #0x70]\n"
"b.eq 7f\n"
"8:\n"
"str q26, [%[c_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "movi v26.4s, #0\n"
"subs %[loops], %[loops], #0x1\n"
- "str q27, [c_ptr1]\n"
+ "movi v26.4s, #0\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
- "movi v27.4s, #0\n"
+ "str q27, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
+ "movi v27.4s, #0\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n"
"str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
+ "ldr q22, [%[b_ptr0], #0x40]\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n"
"str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
- "add c_ptr2, c_ptr2, #0x10\n"
+ "ldr q23, [%[b_ptr0], #0x50]\n"
".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n"
+ "ldr q24, [%[b_ptr0], #0x60]\n"
+ ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n"
"str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
- "add c_ptr3, c_ptr3, #0x10\n"
+ "ldr q25, [%[b_ptr0], #0x70]\n"
".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n"
"str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n"
"add c_ptr5, c_ptr5, #0x10\n"
".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n"
- "ldr q18, [%[b_ptr0]]\n"
- ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
+ "ldr q18, [%[b_ptr0]]\n"
".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n"
"ldr q19, [%[b_ptr0], #0x10]\n"
".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n"
+ "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n"
+ "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n"
".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n"
"ldr q20, [%[b_ptr0], #0x20]\n"
@@ -1524,85 +1758,170 @@
".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n"
".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n"
".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n"
- "ldr q22, [%[b_ptr0], #0x40]\n"
".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n"
".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n"
".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n"
".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n"
".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n"
".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n"
- "ldr q23, [%[b_ptr0], #0x50]\n"
".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n"
".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n"
".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n"
".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n"
- "ldr q24, [%[b_ptr0], #0x60]\n"
".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n"
".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n"
".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n"
".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n"
- "ldr q25, [%[b_ptr0], #0x70]\n"
".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n"
".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n"
".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n"
".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n"
".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n"
".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n"
- "ldr q18, [%[b_ptr0]]\n"
".inst 0x4fa2e27a // sdot v26.4s, v19.16b, v2.4b[1]\n"
".inst 0x4fa5e27b // sdot v27.4s, v19.16b, v5.4b[1]\n"
".inst 0x4fa8e27c // sdot v28.4s, v19.16b, v8.4b[1]\n"
".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n"
".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n"
".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n"
- "ldr q19, [%[b_ptr0], #0x10]\n"
".inst 0x4f82ea9a // sdot v26.4s, v20.16b, v2.4b[2]\n"
".inst 0x4f85ea9b // sdot v27.4s, v20.16b, v5.4b[2]\n"
".inst 0x4f88ea9c // sdot v28.4s, v20.16b, v8.4b[2]\n"
".inst 0x4f8bea9d // sdot v29.4s, v20.16b, v11.4b[2]\n"
".inst 0x4f8eea9e // sdot v30.4s, v20.16b, v14.4b[2]\n"
".inst 0x4f91ea9f // sdot v31.4s, v20.16b, v17.4b[2]\n"
- "ldr q20, [%[b_ptr0], #0x20]\n"
".inst 0x4fa2eaba // sdot v26.4s, v21.16b, v2.4b[3]\n"
".inst 0x4fa5eabb // sdot v27.4s, v21.16b, v5.4b[3]\n"
".inst 0x4fa8eabc // sdot v28.4s, v21.16b, v8.4b[3]\n"
".inst 0x4fabeabd // sdot v29.4s, v21.16b, v11.4b[3]\n"
".inst 0x4faeeabe // sdot v30.4s, v21.16b, v14.4b[3]\n"
".inst 0x4fb1eabf // sdot v31.4s, v21.16b, v17.4b[3]\n"
- "ldr q21, [%[b_ptr0], #0x30]\n"
"b.ne 8b\n"
"7:\n"
"str q26, [%[c_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "movi v26.4s, #0\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
+ "movi v26.4s, #0\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
"str q27, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n"
"str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
+ "ldr q22, [%[b_ptr0], #0x40]\n"
"add c_ptr2, c_ptr2, #0x10\n"
".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n"
"str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
- "add c_ptr3, c_ptr3, #0x10\n"
+ "ldr q23, [%[b_ptr0], #0x50]\n"
".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n"
+ "ldr q24, [%[b_ptr0], #0x60]\n"
+ ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n"
"str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
- "add c_ptr4, c_ptr4, #0x10\n"
+ "ldr q25, [%[b_ptr0], #0x70]\n"
".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n"
"str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
- "add c_ptr5, c_ptr5, #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n"
- ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
+ ".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ ".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n"
+ ".inst 0x4face27e // sdot v30.4s, v19.16b, v12.4b[1]\n"
+ ".inst 0x4fafe27f // sdot v31.4s, v19.16b, v15.4b[1]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f80ea9a // sdot v26.4s, v20.16b, v0.4b[2]\n"
+ ".inst 0x4f83ea9b // sdot v27.4s, v20.16b, v3.4b[2]\n"
+ ".inst 0x4f86ea9c // sdot v28.4s, v20.16b, v6.4b[2]\n"
+ ".inst 0x4f89ea9d // sdot v29.4s, v20.16b, v9.4b[2]\n"
+ ".inst 0x4f8cea9e // sdot v30.4s, v20.16b, v12.4b[2]\n"
+ ".inst 0x4f8fea9f // sdot v31.4s, v20.16b, v15.4b[2]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
+ ".inst 0x4fa0eaba // sdot v26.4s, v21.16b, v0.4b[3]\n"
+ ".inst 0x4fa3eabb // sdot v27.4s, v21.16b, v3.4b[3]\n"
+ ".inst 0x4fa6eabc // sdot v28.4s, v21.16b, v6.4b[3]\n"
+ ".inst 0x4fa9eabd // sdot v29.4s, v21.16b, v9.4b[3]\n"
+ ".inst 0x4faceabe // sdot v30.4s, v21.16b, v12.4b[3]\n"
+ ".inst 0x4fafeabf // sdot v31.4s, v21.16b, v15.4b[3]\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
+ ".inst 0x4f81e2da // sdot v26.4s, v22.16b, v1.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x40\n"
+ ".inst 0x4f84e2db // sdot v27.4s, v22.16b, v4.4b[0]\n"
+ ".inst 0x4f87e2dc // sdot v28.4s, v22.16b, v7.4b[0]\n"
+ ".inst 0x4f8ae2dd // sdot v29.4s, v22.16b, v10.4b[0]\n"
+ ".inst 0x4f8de2de // sdot v30.4s, v22.16b, v13.4b[0]\n"
+ ".inst 0x4f90e2df // sdot v31.4s, v22.16b, v16.4b[0]\n"
+ ".inst 0x4fa1e2fa // sdot v26.4s, v23.16b, v1.4b[1]\n"
+ ".inst 0x4fa4e2fb // sdot v27.4s, v23.16b, v4.4b[1]\n"
+ ".inst 0x4fa7e2fc // sdot v28.4s, v23.16b, v7.4b[1]\n"
+ ".inst 0x4faae2fd // sdot v29.4s, v23.16b, v10.4b[1]\n"
+ ".inst 0x4fade2fe // sdot v30.4s, v23.16b, v13.4b[1]\n"
+ ".inst 0x4fb0e2ff // sdot v31.4s, v23.16b, v16.4b[1]\n"
+ ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x4f87eb1c // sdot v28.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x4f8aeb1d // sdot v29.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x4f8deb1e // sdot v30.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x4f90eb1f // sdot v31.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x4fa7eb3c // sdot v28.4s, v25.16b, v7.4b[3]\n"
+ ".inst 0x4faaeb3d // sdot v29.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x4fadeb3e // sdot v30.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x4fb0eb3f // sdot v31.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x4f82e25a // sdot v26.4s, v18.16b, v2.4b[0]\n"
+ ".inst 0x4f85e25b // sdot v27.4s, v18.16b, v5.4b[0]\n"
+ ".inst 0x4f88e25c // sdot v28.4s, v18.16b, v8.4b[0]\n"
+ ".inst 0x4f8be25d // sdot v29.4s, v18.16b, v11.4b[0]\n"
+ ".inst 0x4f8ee25e // sdot v30.4s, v18.16b, v14.4b[0]\n"
+ ".inst 0x4f91e25f // sdot v31.4s, v18.16b, v17.4b[0]\n"
+ ".inst 0x4fa2e27a // sdot v26.4s, v19.16b, v2.4b[1]\n"
+ ".inst 0x4fa5e27b // sdot v27.4s, v19.16b, v5.4b[1]\n"
+ ".inst 0x4fa8e27c // sdot v28.4s, v19.16b, v8.4b[1]\n"
+ ".inst 0x4fabe27d // sdot v29.4s, v19.16b, v11.4b[1]\n"
+ ".inst 0x4faee27e // sdot v30.4s, v19.16b, v14.4b[1]\n"
+ ".inst 0x4fb1e27f // sdot v31.4s, v19.16b, v17.4b[1]\n"
+ ".inst 0x4f82ea9a // sdot v26.4s, v20.16b, v2.4b[2]\n"
+ ".inst 0x4f85ea9b // sdot v27.4s, v20.16b, v5.4b[2]\n"
+ ".inst 0x4f88ea9c // sdot v28.4s, v20.16b, v8.4b[2]\n"
+ ".inst 0x4f8bea9d // sdot v29.4s, v20.16b, v11.4b[2]\n"
+ ".inst 0x4f8eea9e // sdot v30.4s, v20.16b, v14.4b[2]\n"
+ ".inst 0x4f91ea9f // sdot v31.4s, v20.16b, v17.4b[2]\n"
+ ".inst 0x4fa2eaba // sdot v26.4s, v21.16b, v2.4b[3]\n"
+ ".inst 0x4fa5eabb // sdot v27.4s, v21.16b, v5.4b[3]\n"
+ ".inst 0x4fa8eabc // sdot v28.4s, v21.16b, v8.4b[3]\n"
+ ".inst 0x4fabeabd // sdot v29.4s, v21.16b, v11.4b[3]\n"
+ ".inst 0x4faeeabe // sdot v30.4s, v21.16b, v14.4b[3]\n"
+ ".inst 0x4fb1eabf // sdot v31.4s, v21.16b, v17.4b[3]\n"
+ "b 9f\n"
+ "6:\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x4f80e25a // sdot v26.4s, v18.16b, v0.4b[0]\n"
+ ".inst 0x4f83e25b // sdot v27.4s, v18.16b, v3.4b[0]\n"
+ ".inst 0x4f86e25c // sdot v28.4s, v18.16b, v6.4b[0]\n"
+ ".inst 0x4f89e25d // sdot v29.4s, v18.16b, v9.4b[0]\n"
+ ".inst 0x4f8ce25e // sdot v30.4s, v18.16b, v12.4b[0]\n"
".inst 0x4f8fe25f // sdot v31.4s, v18.16b, v15.4b[0]\n"
"ldr q18, [%[b_ptr0]]\n"
+ ".inst 0x4fa0e27a // sdot v26.4s, v19.16b, v0.4b[1]\n"
".inst 0x4fa3e27b // sdot v27.4s, v19.16b, v3.4b[1]\n"
".inst 0x4fa6e27c // sdot v28.4s, v19.16b, v6.4b[1]\n"
".inst 0x4fa9e27d // sdot v29.4s, v19.16b, v9.4b[1]\n"
@@ -1672,19 +1991,14 @@
".inst 0x4fabeabd // sdot v29.4s, v21.16b, v11.4b[3]\n"
".inst 0x4faeeabe // sdot v30.4s, v21.16b, v14.4b[3]\n"
".inst 0x4fb1eabf // sdot v31.4s, v21.16b, v17.4b[3]\n"
- "6:\n"
+ "9:\n"
"str q26, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q27, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, #0x10\n"
"str q28, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, #0x10\n"
"str q29, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, #0x10\n"
"str q30, [c_ptr4]\n"
- "add c_ptr4, c_ptr4, #0x10\n"
"str q31, [c_ptr5]\n"
- "add c_ptr5, c_ptr5, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -1743,6 +2057,7 @@
"add c_ptr1, %[c_ptr0], #0x0\n"
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
+ "cbnz %[odds], 2f\n"
"ldr q0, [%[a_ptr0]], #0x10\n"
"ldr q4, [a_ptr1], #0x10\n"
"ldr q8, [a_ptr2], #0x10\n"
@@ -1759,18 +2074,35 @@
"ldr q6, [a_ptr1], #0x10\n"
"ldr q10, [a_ptr2], #0x10\n"
"ldr q14, [a_ptr3], #0x10\n"
+ "ldr s3, [%[a_ptr0]]\n"
+ "ldr q18, [a_ptr4], #0x10\n"
+ "ldr s7, [a_ptr1]\n"
+ "ldr q22, [a_ptr5], #0x10\n"
+ "ldr s11, [a_ptr2]\n"
+ "ldr s15, [a_ptr3]\n"
+ "ldr s19, [a_ptr4]\n"
+ "ldr s23, [a_ptr5]\n"
+ "b 3f\n"
+ "2:\n"
+ "ldr q0, [%[a_ptr0]], #0x10\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "ldr q4, [a_ptr1], #0x10\n"
+ "ldr q8, [a_ptr2], #0x10\n"
+ "ldr q12, [a_ptr3], #0x10\n"
+ "ldr q16, [a_ptr4], #0x10\n"
+ "ldr q20, [a_ptr5], #0x10\n"
+ "ldr q1, [%[a_ptr0]], #0x10\n"
+ "ldr q5, [a_ptr1], #0x10\n"
+ "ldr q9, [a_ptr2], #0x10\n"
+ "ldr q13, [a_ptr3], #0x10\n"
+ "ldr q17, [a_ptr4], #0x10\n"
+ "ldr q21, [a_ptr5], #0x10\n"
+ "ldr q2, [%[a_ptr0]], #0x10\n"
+ "ldr q6, [a_ptr1], #0x10\n"
+ "ldr q10, [a_ptr2], #0x10\n"
+ "ldr q14, [a_ptr3], #0x10\n"
"ldr q18, [a_ptr4], #0x10\n"
"ldr q22, [a_ptr5], #0x10\n"
- "cbnz %[odds], 2f\n"
- "ldr s3, [%[a_ptr0]]\n"
- "ldr s7, [a_ptr1]\n"
- "ldr s11, [a_ptr2]\n"
- "ldr s15, [a_ptr3]\n"
- "ldr s19, [a_ptr4]\n"
- "ldr s23, [a_ptr5]\n"
- "b 3f\n"
- "2:\n"
- "subs %[odds], %[odds], #0x1\n"
"b.ne 4f\n"
"ldr b3, [%[a_ptr0]]\n"
"ldr b7, [a_ptr1]\n"
@@ -1797,24 +2129,26 @@
"ld1 {v19.b}[2], [a_ptr4]\n"
"ld1 {v23.b}[2], [a_ptr5]\n"
"3:\n"
- "movi v26.4s, #0\n"
"ldr q24, [%[b_ptr0]]\n"
- "movi v27.4s, #0\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- "movi v28.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr5, #0x40]\n"
- "movi v29.4s, #0\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x80]\n"
- "movi v30.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
- "movi v31.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr5, #0x100]\n"
- ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x140]\n"
- ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x180]\n"
- ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "cbz %[loops], 6f\n"
+ "movi v26.4s, #0\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
@@ -1907,38 +2241,36 @@
".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n"
".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n"
".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n"
- "cbz %[loops], 6f\n"
- "ldr q24, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
"b.eq 7f\n"
"8:\n"
"str q26, [%[c_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- "movi v26.4s, #0\n"
"subs %[loops], %[loops], #0x1\n"
- "str q27, [c_ptr1]\n"
+ "movi v26.4s, #0\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
- "movi v27.4s, #0\n"
+ "str q27, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
+ "movi v27.4s, #0\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
"str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
"str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
- "add c_ptr2, c_ptr2, #0x10\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
"str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
- "add c_ptr3, c_ptr3, #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
"str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
- "add c_ptr4, c_ptr4, #0x10\n"
- ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
"add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
@@ -2028,20 +2360,20 @@
".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n"
".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n"
".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n"
".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n"
".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n"
".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n"
".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n"
".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
"b.ne 8b\n"
"7:\n"
"str q26, [%[c_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- "movi v26.4s, #0\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
+ "movi v26.4s, #0\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
"str q27, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
@@ -2152,19 +2484,117 @@
".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n"
".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n"
".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n"
+ "b 9f\n"
"6:\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
+ ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
+ ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
+ ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
+ ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n"
+ ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n"
+ ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
+ ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
+ ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n"
+ ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n"
+ ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n"
+ ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n"
+ ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n"
+ ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n"
+ ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n"
+ ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n"
+ ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n"
+ ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n"
+ ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n"
+ ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n"
+ ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n"
+ ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n"
+ ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n"
+ ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n"
+ ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n"
+ ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n"
+ ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n"
+ ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n"
+ ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n"
+ ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n"
+ ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n"
+ ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n"
+ ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n"
+ ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n"
+ ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n"
+ ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n"
+ ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n"
+ ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n"
+ ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n"
+ ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n"
+ ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x10\n"
+ ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n"
+ ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n"
+ ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n"
+ ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n"
+ ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n"
+ ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n"
+ ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n"
+ ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n"
+ ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n"
+ ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n"
+ "9:\n"
"str q26, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q27, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, #0x10\n"
"str q28, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, #0x10\n"
"str q29, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, #0x10\n"
"str q30, [c_ptr4]\n"
- "add c_ptr4, c_ptr4, #0x10\n"
"str q31, [c_ptr5]\n"
- "add c_ptr5, c_ptr5, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -2223,6 +2653,7 @@
"add c_ptr1, %[c_ptr0], #0x0\n"
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
+ "cbnz %[odds], 2f\n"
"ldr q0, [%[a_ptr0]], #0x10\n"
"ldr q4, [a_ptr1], #0x10\n"
"ldr q8, [a_ptr2], #0x10\n"
@@ -2239,24 +2670,41 @@
"ldr q6, [a_ptr1], #0x10\n"
"ldr q10, [a_ptr2], #0x10\n"
"ldr q14, [a_ptr3], #0x10\n"
- "ldr q18, [a_ptr4], #0x10\n"
- "ldr q22, [a_ptr5], #0x10\n"
- "cbnz %[odds], 2f\n"
"ldr d3, [%[a_ptr0]]\n"
+ "ldr q18, [a_ptr4], #0x10\n"
"ldr d7, [a_ptr1]\n"
+ "ldr q22, [a_ptr5], #0x10\n"
"ldr d11, [a_ptr2]\n"
"ldr d15, [a_ptr3]\n"
"ldr d19, [a_ptr4]\n"
"ldr d23, [a_ptr5]\n"
"b 3f\n"
"2:\n"
+ "ldr q0, [%[a_ptr0]], #0x10\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "ldr q4, [a_ptr1], #0x10\n"
+ "ldr q8, [a_ptr2], #0x10\n"
+ "ldr q12, [a_ptr3], #0x10\n"
+ "ldr q16, [a_ptr4], #0x10\n"
+ "ldr q20, [a_ptr5], #0x10\n"
+ "ldr q1, [%[a_ptr0]], #0x10\n"
+ "ldr q5, [a_ptr1], #0x10\n"
+ "ldr q9, [a_ptr2], #0x10\n"
+ "ldr q13, [a_ptr3], #0x10\n"
+ "ldr q17, [a_ptr4], #0x10\n"
+ "ldr q21, [a_ptr5], #0x10\n"
+ "ldr q2, [%[a_ptr0]], #0x10\n"
+ "ldr q6, [a_ptr1], #0x10\n"
+ "ldr q10, [a_ptr2], #0x10\n"
+ "ldr q14, [a_ptr3], #0x10\n"
"ldr s3, [%[a_ptr0]], #0x4\n"
+ "ldr q18, [a_ptr4], #0x10\n"
"ldr s7, [a_ptr1], #0x4\n"
+ "ldr q22, [a_ptr5], #0x10\n"
"ldr s11, [a_ptr2], #0x4\n"
"ldr s15, [a_ptr3], #0x4\n"
"ldr s19, [a_ptr4], #0x4\n"
"ldr s23, [a_ptr5], #0x4\n"
- "subs %[odds], %[odds], #0x1\n"
"b.ne 4f\n"
"ld1 {v3.b}[4], [%[a_ptr0]]\n"
"ld1 {v7.b}[4], [a_ptr1]\n"
@@ -2283,24 +2731,26 @@
"ld1 {v19.b}[6], [a_ptr4]\n"
"ld1 {v23.b}[6], [a_ptr5]\n"
"3:\n"
- "movi v26.4s, #0\n"
"ldr q24, [%[b_ptr0]]\n"
- "movi v27.4s, #0\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- "movi v28.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr5, #0x40]\n"
- "movi v29.4s, #0\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x80]\n"
- "movi v30.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
- "movi v31.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr5, #0x100]\n"
- ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x140]\n"
- ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x180]\n"
- ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "cbz %[loops], 6f\n"
+ "movi v26.4s, #0\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
@@ -2400,38 +2850,36 @@
".inst 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n"
".inst 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n"
".inst 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n"
- "cbz %[loops], 6f\n"
- "ldr q24, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
"b.eq 7f\n"
"8:\n"
"str q26, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"movi v26.4s, #0\n"
+ "ldr q24, [%[b_ptr0]]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- "str q27, [c_ptr1]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
- "movi v27.4s, #0\n"
+ "str q27, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
+ "movi v27.4s, #0\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
"str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
"str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
- "add c_ptr2, c_ptr2, #0x10\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
"str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
- "add c_ptr3, c_ptr3, #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
"str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
- "add c_ptr4, c_ptr4, #0x10\n"
- ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
"add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
@@ -2528,7 +2976,6 @@
".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n"
".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n"
".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
".inst 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n"
".inst 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n"
".inst 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n"
@@ -2540,6 +2987,7 @@
"str q26, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v26.4s, #0\n"
+ "ldr q24, [%[b_ptr0]]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
"str q27, [c_ptr1]\n"
@@ -2659,19 +3107,124 @@
".inst 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n"
".inst 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n"
".inst 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n"
+ "b 9f\n"
"6:\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
+ ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
+ ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
+ ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
+ ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n"
+ ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n"
+ ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
+ ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
+ ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n"
+ ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n"
+ ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n"
+ ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n"
+ ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n"
+ ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n"
+ ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n"
+ ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n"
+ ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n"
+ ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n"
+ ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n"
+ ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n"
+ ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n"
+ ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n"
+ ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n"
+ ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n"
+ ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n"
+ ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n"
+ ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n"
+ ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n"
+ ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n"
+ ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n"
+ ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n"
+ ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n"
+ ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n"
+ ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n"
+ ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n"
+ ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n"
+ ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n"
+ ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n"
+ ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n"
+ ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n"
+ ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n"
+ ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n"
+ ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n"
+ ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n"
+ ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n"
+ ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n"
+ ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n"
+ ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n"
+ ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n"
+ ".inst 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n"
+ ".inst 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n"
+ ".inst 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n"
+ ".inst 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n"
+ ".inst 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n"
+ ".inst 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n"
+ "9:\n"
"str q26, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q27, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, #0x10\n"
"str q28, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, #0x10\n"
"str q29, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, #0x10\n"
"str q30, [c_ptr4]\n"
- "add c_ptr4, c_ptr4, #0x10\n"
"str q31, [c_ptr5]\n"
- "add c_ptr5, c_ptr5, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -2790,24 +3343,26 @@
"ld1 {v19.b}[10], [a_ptr4]\n"
"ld1 {v23.b}[10], [a_ptr5]\n"
"3:\n"
- "movi v26.4s, #0\n"
"ldr q24, [%[b_ptr0]]\n"
- "movi v27.4s, #0\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- "movi v28.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr5, #0x40]\n"
- "movi v29.4s, #0\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x80]\n"
- "movi v30.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
- "movi v31.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr5, #0x100]\n"
- ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x140]\n"
- ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x180]\n"
- ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "cbz %[loops], 6f\n"
+ "movi v26.4s, #0\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
@@ -2915,38 +3470,36 @@
".inst 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n"
".inst 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n"
".inst 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n"
- "cbz %[loops], 6f\n"
- "ldr q24, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
"b.eq 7f\n"
"8:\n"
"str q26, [%[c_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- "movi v26.4s, #0\n"
"subs %[loops], %[loops], #0x1\n"
- "str q27, [c_ptr1]\n"
+ "movi v26.4s, #0\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
- "movi v27.4s, #0\n"
+ "str q27, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
+ "movi v27.4s, #0\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
"str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
"str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
- "add c_ptr2, c_ptr2, #0x10\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
"str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
- "add c_ptr3, c_ptr3, #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
"str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
- "add c_ptr4, c_ptr4, #0x10\n"
- ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
"add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
@@ -3051,20 +3604,20 @@
".inst 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n"
".inst 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n"
".inst 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
".inst 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n"
".inst 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n"
".inst 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n"
".inst 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n"
".inst 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n"
".inst 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
"b.ne 8b\n"
"7:\n"
"str q26, [%[c_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- "movi v26.4s, #0\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
+ "movi v26.4s, #0\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
"str q27, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
@@ -3190,19 +3743,132 @@
".inst 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n"
".inst 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n"
".inst 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n"
+ "b 9f\n"
"6:\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
+ ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
+ ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
+ ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
+ ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n"
+ ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n"
+ ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
+ ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
+ ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n"
+ ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n"
+ ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n"
+ ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n"
+ ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n"
+ ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n"
+ ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n"
+ ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n"
+ ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n"
+ ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n"
+ ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n"
+ ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n"
+ ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n"
+ ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n"
+ ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n"
+ ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n"
+ ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n"
+ ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n"
+ ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n"
+ ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n"
+ ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n"
+ ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n"
+ ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n"
+ ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n"
+ ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n"
+ ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n"
+ ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n"
+ ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n"
+ ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n"
+ ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n"
+ ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n"
+ ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n"
+ ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n"
+ ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n"
+ ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n"
+ ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n"
+ ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n"
+ ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n"
+ ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n"
+ ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n"
+ ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x10\n"
+ ".inst 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n"
+ ".inst 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n"
+ ".inst 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n"
+ ".inst 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n"
+ ".inst 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n"
+ ".inst 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n"
+ ".inst 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n"
+ ".inst 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n"
+ ".inst 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n"
+ ".inst 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n"
+ "9:\n"
"str q26, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q27, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, #0x10\n"
"str q28, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, #0x10\n"
"str q29, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, #0x10\n"
"str q30, [c_ptr4]\n"
- "add c_ptr4, c_ptr4, #0x10\n"
"str q31, [c_ptr5]\n"
- "add c_ptr5, c_ptr5, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -3262,6 +3928,7 @@
"add c_ptr1, %[c_ptr0], #0x0\n"
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
+ "cbnz %[odds], 2f\n"
"ldr q0, [%[a_ptr0]], #0x10\n"
"ldr q4, [a_ptr1], #0x10\n"
"ldr q8, [a_ptr2], #0x10\n"
@@ -3280,7 +3947,6 @@
"ldr q14, [a_ptr3], #0x10\n"
"ldr q18, [a_ptr4], #0x10\n"
"ldr q22, [a_ptr5], #0x10\n"
- "cbnz %[odds], 2f\n"
"ldr q3, [%[a_ptr0]]\n"
"ldr q7, [a_ptr1]\n"
"ldr q11, [a_ptr2]\n"
@@ -3289,8 +3955,27 @@
"ldr q23, [a_ptr5]\n"
"b 3f\n"
"2:\n"
+ "ldr q0, [%[a_ptr0]], #0x10\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "ldr q4, [a_ptr1], #0x10\n"
+ "ldr q8, [a_ptr2], #0x10\n"
+ "ldr q12, [a_ptr3], #0x10\n"
+ "ldr q16, [a_ptr4], #0x10\n"
+ "ldr q20, [a_ptr5], #0x10\n"
+ "ldr q1, [%[a_ptr0]], #0x10\n"
+ "ldr q5, [a_ptr1], #0x10\n"
+ "ldr q9, [a_ptr2], #0x10\n"
+ "ldr q13, [a_ptr3], #0x10\n"
+ "ldr q17, [a_ptr4], #0x10\n"
+ "ldr q21, [a_ptr5], #0x10\n"
+ "ldr q2, [%[a_ptr0]], #0x10\n"
+ "ldr q6, [a_ptr1], #0x10\n"
+ "ldr q10, [a_ptr2], #0x10\n"
+ "ldr q14, [a_ptr3], #0x10\n"
"ldr d3, [%[a_ptr0]], #0x8\n"
+ "ldr q18, [a_ptr4], #0x10\n"
"ldr d7, [a_ptr1], #0x8\n"
+ "ldr q22, [a_ptr5], #0x10\n"
"ldr d11, [a_ptr2], #0x8\n"
"ldr d15, [a_ptr3], #0x8\n"
"ldr d19, [a_ptr4], #0x8\n"
@@ -3301,7 +3986,6 @@
"ld1 {v15.s}[2], [a_ptr3], #4\n"
"ld1 {v19.s}[2], [a_ptr4], #4\n"
"ld1 {v23.s}[2], [a_ptr5], #4\n"
- "subs %[odds], %[odds], #0x1\n"
"b.ne 4f\n"
"ld1 {v3.b}[12], [%[a_ptr0]]\n"
"ld1 {v7.b}[12], [a_ptr1]\n"
@@ -3328,24 +4012,26 @@
"ld1 {v19.b}[14], [a_ptr4]\n"
"ld1 {v23.b}[14], [a_ptr5]\n"
"3:\n"
- "movi v26.4s, #0\n"
"ldr q24, [%[b_ptr0]]\n"
- "movi v27.4s, #0\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- "movi v28.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr5, #0x40]\n"
- "movi v29.4s, #0\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x80]\n"
- "movi v30.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
- "movi v31.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr5, #0x100]\n"
- ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x140]\n"
- ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x180]\n"
- ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "cbz %[loops], 6f\n"
+ "movi v26.4s, #0\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
@@ -3460,38 +4146,36 @@
".inst 0x4fafeb3d // sdot v29.4s, v25.16b, v15.4b[3]\n"
".inst 0x4fb3eb3e // sdot v30.4s, v25.16b, v19.4b[3]\n"
".inst 0x4fb7eb3f // sdot v31.4s, v25.16b, v23.4b[3]\n"
- "cbz %[loops], 6f\n"
- "ldr q24, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
"b.eq 7f\n"
"8:\n"
"str q26, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"movi v26.4s, #0\n"
+ "ldr q24, [%[b_ptr0]]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- "str q27, [c_ptr1]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
- "movi v27.4s, #0\n"
+ "str q27, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
+ "movi v27.4s, #0\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
"str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
"str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
- "add c_ptr2, c_ptr2, #0x10\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
"str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
- "add c_ptr3, c_ptr3, #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
"str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
- "add c_ptr4, c_ptr4, #0x10\n"
- ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
"add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
@@ -3603,7 +4287,6 @@
".inst 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n"
".inst 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n"
".inst 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
".inst 0x4fa3eb3a // sdot v26.4s, v25.16b, v3.4b[3]\n"
".inst 0x4fa7eb3b // sdot v27.4s, v25.16b, v7.4b[3]\n"
".inst 0x4fabeb3c // sdot v28.4s, v25.16b, v11.4b[3]\n"
@@ -3615,6 +4298,7 @@
"str q26, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v26.4s, #0\n"
+ "ldr q24, [%[b_ptr0]]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
"str q27, [c_ptr1]\n"
@@ -3749,19 +4433,139 @@
".inst 0x4fafeb3d // sdot v29.4s, v25.16b, v15.4b[3]\n"
".inst 0x4fb3eb3e // sdot v30.4s, v25.16b, v19.4b[3]\n"
".inst 0x4fb7eb3f // sdot v31.4s, v25.16b, v23.4b[3]\n"
+ "b 9f\n"
"6:\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x4f80e31a // sdot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x4f84e31b // sdot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x4f88e31c // sdot v28.4s, v24.16b, v8.4b[0]\n"
+ ".inst 0x4f8ce31d // sdot v29.4s, v24.16b, v12.4b[0]\n"
+ ".inst 0x4f90e31e // sdot v30.4s, v24.16b, v16.4b[0]\n"
+ ".inst 0x4f94e31f // sdot v31.4s, v24.16b, v20.4b[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x4fa0e33a // sdot v26.4s, v25.16b, v0.4b[1]\n"
+ ".inst 0x4fa4e33b // sdot v27.4s, v25.16b, v4.4b[1]\n"
+ ".inst 0x4fa8e33c // sdot v28.4s, v25.16b, v8.4b[1]\n"
+ ".inst 0x4face33d // sdot v29.4s, v25.16b, v12.4b[1]\n"
+ ".inst 0x4fb0e33e // sdot v30.4s, v25.16b, v16.4b[1]\n"
+ ".inst 0x4fb4e33f // sdot v31.4s, v25.16b, v20.4b[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f80eb1a // sdot v26.4s, v24.16b, v0.4b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x4f84eb1b // sdot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x4f88eb1c // sdot v28.4s, v24.16b, v8.4b[2]\n"
+ ".inst 0x4f8ceb1d // sdot v29.4s, v24.16b, v12.4b[2]\n"
+ ".inst 0x4f90eb1e // sdot v30.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x4f94eb1f // sdot v31.4s, v24.16b, v20.4b[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x4fa0eb3a // sdot v26.4s, v25.16b, v0.4b[3]\n"
+ ".inst 0x4fa4eb3b // sdot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x4fa8eb3c // sdot v28.4s, v25.16b, v8.4b[3]\n"
+ ".inst 0x4faceb3d // sdot v29.4s, v25.16b, v12.4b[3]\n"
+ ".inst 0x4fb0eb3e // sdot v30.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x4fb4eb3f // sdot v31.4s, v25.16b, v20.4b[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f81e31a // sdot v26.4s, v24.16b, v1.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x4f85e31b // sdot v27.4s, v24.16b, v5.4b[0]\n"
+ ".inst 0x4f89e31c // sdot v28.4s, v24.16b, v9.4b[0]\n"
+ ".inst 0x4f8de31d // sdot v29.4s, v24.16b, v13.4b[0]\n"
+ ".inst 0x4f91e31e // sdot v30.4s, v24.16b, v17.4b[0]\n"
+ ".inst 0x4f95e31f // sdot v31.4s, v24.16b, v21.4b[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x4fa1e33a // sdot v26.4s, v25.16b, v1.4b[1]\n"
+ ".inst 0x4fa5e33b // sdot v27.4s, v25.16b, v5.4b[1]\n"
+ ".inst 0x4fa9e33c // sdot v28.4s, v25.16b, v9.4b[1]\n"
+ ".inst 0x4fade33d // sdot v29.4s, v25.16b, v13.4b[1]\n"
+ ".inst 0x4fb1e33e // sdot v30.4s, v25.16b, v17.4b[1]\n"
+ ".inst 0x4fb5e33f // sdot v31.4s, v25.16b, v21.4b[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f81eb1a // sdot v26.4s, v24.16b, v1.4b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x4f85eb1b // sdot v27.4s, v24.16b, v5.4b[2]\n"
+ ".inst 0x4f89eb1c // sdot v28.4s, v24.16b, v9.4b[2]\n"
+ ".inst 0x4f8deb1d // sdot v29.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x4f91eb1e // sdot v30.4s, v24.16b, v17.4b[2]\n"
+ ".inst 0x4f95eb1f // sdot v31.4s, v24.16b, v21.4b[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x4fa1eb3a // sdot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x4fa5eb3b // sdot v27.4s, v25.16b, v5.4b[3]\n"
+ ".inst 0x4fa9eb3c // sdot v28.4s, v25.16b, v9.4b[3]\n"
+ ".inst 0x4fadeb3d // sdot v29.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x4fb1eb3e // sdot v30.4s, v25.16b, v17.4b[3]\n"
+ ".inst 0x4fb5eb3f // sdot v31.4s, v25.16b, v21.4b[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f82e31a // sdot v26.4s, v24.16b, v2.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x4f86e31b // sdot v27.4s, v24.16b, v6.4b[0]\n"
+ ".inst 0x4f8ae31c // sdot v28.4s, v24.16b, v10.4b[0]\n"
+ ".inst 0x4f8ee31d // sdot v29.4s, v24.16b, v14.4b[0]\n"
+ ".inst 0x4f92e31e // sdot v30.4s, v24.16b, v18.4b[0]\n"
+ ".inst 0x4f96e31f // sdot v31.4s, v24.16b, v22.4b[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x4fa2e33a // sdot v26.4s, v25.16b, v2.4b[1]\n"
+ ".inst 0x4fa6e33b // sdot v27.4s, v25.16b, v6.4b[1]\n"
+ ".inst 0x4faae33c // sdot v28.4s, v25.16b, v10.4b[1]\n"
+ ".inst 0x4faee33d // sdot v29.4s, v25.16b, v14.4b[1]\n"
+ ".inst 0x4fb2e33e // sdot v30.4s, v25.16b, v18.4b[1]\n"
+ ".inst 0x4fb6e33f // sdot v31.4s, v25.16b, v22.4b[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f82eb1a // sdot v26.4s, v24.16b, v2.4b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x4f86eb1b // sdot v27.4s, v24.16b, v6.4b[2]\n"
+ ".inst 0x4f8aeb1c // sdot v28.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x4f8eeb1d // sdot v29.4s, v24.16b, v14.4b[2]\n"
+ ".inst 0x4f92eb1e // sdot v30.4s, v24.16b, v18.4b[2]\n"
+ ".inst 0x4f96eb1f // sdot v31.4s, v24.16b, v22.4b[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x4fa2eb3a // sdot v26.4s, v25.16b, v2.4b[3]\n"
+ ".inst 0x4fa6eb3b // sdot v27.4s, v25.16b, v6.4b[3]\n"
+ ".inst 0x4faaeb3c // sdot v28.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x4faeeb3d // sdot v29.4s, v25.16b, v14.4b[3]\n"
+ ".inst 0x4fb2eb3e // sdot v30.4s, v25.16b, v18.4b[3]\n"
+ ".inst 0x4fb6eb3f // sdot v31.4s, v25.16b, v22.4b[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f83e31a // sdot v26.4s, v24.16b, v3.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x4f87e31b // sdot v27.4s, v24.16b, v7.4b[0]\n"
+ ".inst 0x4f8be31c // sdot v28.4s, v24.16b, v11.4b[0]\n"
+ ".inst 0x4f8fe31d // sdot v29.4s, v24.16b, v15.4b[0]\n"
+ ".inst 0x4f93e31e // sdot v30.4s, v24.16b, v19.4b[0]\n"
+ ".inst 0x4f97e31f // sdot v31.4s, v24.16b, v23.4b[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x4fa3e33a // sdot v26.4s, v25.16b, v3.4b[1]\n"
+ ".inst 0x4fa7e33b // sdot v27.4s, v25.16b, v7.4b[1]\n"
+ ".inst 0x4fabe33c // sdot v28.4s, v25.16b, v11.4b[1]\n"
+ ".inst 0x4fafe33d // sdot v29.4s, v25.16b, v15.4b[1]\n"
+ ".inst 0x4fb3e33e // sdot v30.4s, v25.16b, v19.4b[1]\n"
+ ".inst 0x4fb7e33f // sdot v31.4s, v25.16b, v23.4b[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4f83eb1a // sdot v26.4s, v24.16b, v3.4b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x4f87eb1b // sdot v27.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x4f8beb1c // sdot v28.4s, v24.16b, v11.4b[2]\n"
+ ".inst 0x4f8feb1d // sdot v29.4s, v24.16b, v15.4b[2]\n"
+ ".inst 0x4f93eb1e // sdot v30.4s, v24.16b, v19.4b[2]\n"
+ ".inst 0x4f97eb1f // sdot v31.4s, v24.16b, v23.4b[2]\n"
+ ".inst 0x4fa3eb3a // sdot v26.4s, v25.16b, v3.4b[3]\n"
+ ".inst 0x4fa7eb3b // sdot v27.4s, v25.16b, v7.4b[3]\n"
+ ".inst 0x4fabeb3c // sdot v28.4s, v25.16b, v11.4b[3]\n"
+ ".inst 0x4fafeb3d // sdot v29.4s, v25.16b, v15.4b[3]\n"
+ ".inst 0x4fb3eb3e // sdot v30.4s, v25.16b, v19.4b[3]\n"
+ ".inst 0x4fb7eb3f // sdot v31.4s, v25.16b, v23.4b[3]\n"
+ "9:\n"
"str q26, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q27, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, #0x10\n"
"str q28, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, #0x10\n"
"str q29, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, #0x10\n"
"str q30, [c_ptr4]\n"
- "add c_ptr4, c_ptr4, #0x10\n"
"str q31, [c_ptr5]\n"
- "add c_ptr5, c_ptr5, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x8.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_8x4.hpp
similarity index 87%
rename from src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x8.hpp
rename to src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_8x4.hpp
index 3de708c..9f9c2a4 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x8.hpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_8x4.hpp
@@ -31,10 +31,10 @@
{
// Actual kernel implementations
-void a64_smallK_hybrid_s8s32_dot_4x8(const int8_t *, int, const int8_t *, int32_t *, int, int, int, int, const int32_t *, Activation, bool);
-void a64_smallK_hybrid_s8s32_dot_4x8_a55(const int8_t *, int, const int8_t *, int32_t *, int, int, int, int, const int32_t *, Activation, bool);
+void a64_smallK_hybrid_s8s32_dot_8x4(const int8_t *, int, const int8_t *, int32_t *, int, int, int, int, const int32_t *, Activation, bool);
+void a64_smallK_hybrid_s8s32_dot_8x4_a55(const int8_t *, int, const int8_t *, int32_t *, int, int, int, int, const int32_t *, Activation, bool);
-class smallK_hybrid_s8s32_dot_4x8
+class cls_a64_smallK_hybrid_s8s32_dot_8x4
{
public:
typedef int8_t operand_type;
@@ -76,12 +76,12 @@
StdTransformsFixed<operand_type, result_type, 8, 4, 4> transforms = {};
// Default to the generic kernel
- kern_type kernel=a64_smallK_hybrid_s8s32_dot_4x8;
+ kern_type kernel=a64_smallK_hybrid_s8s32_dot_8x4;
- smallK_hybrid_s8s32_dot_4x8(const CPUInfo *ci)
+ cls_a64_smallK_hybrid_s8s32_dot_8x4(const CPUInfo *ci)
{
if (ci->get_cpu_model() == CPUModel::A55r1) {
- kernel = a64_smallK_hybrid_s8s32_dot_4x8_a55;
+ kernel = a64_smallK_hybrid_s8s32_dot_8x4_a55;
}
}
};
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x8/a55.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_8x4/a55.cpp
similarity index 85%
rename from src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x8/a55.cpp
rename to src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_8x4/a55.cpp
index 7135f2e..aba6e0d 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x8/a55.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_8x4/a55.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2019 Arm Limited.
+ * Copyright (c) 2019-2020 Arm Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -33,7 +33,7 @@
namespace arm_gemm {
-void a64_smallK_hybrid_s8s32_dot_4x8_a55(const int8_t *A, int lda, const int8_t *B, int32_t *C, int ldc, int M, int N, int K, const int32_t *, Activation, bool) {
+void a64_smallK_hybrid_s8s32_dot_8x4_a55(const int8_t *A, int lda, const int8_t *B, int32_t *C, int ldc, int M, int N, int K, const int32_t *, Activation, bool) {
const long loops_count = iceildiv(N, (int)4) - 1;
const long ldab = lda * sizeof(int8_t);
const long ldcb = ldc * sizeof(int32_t);
@@ -157,22 +157,24 @@
"ld1 {v6.b}[2], [a_ptr6]\n"
"ld1 {v7.b}[2], [a_ptr7]\n"
"3:\n"
- "movi v24.4s, #0\n"
"ldr q16, [%[b_ptr0]]\n"
- "movi v25.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
- "movi v26.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
- "movi v27.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
- "movi v28.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
- "movi v29.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- "movi v30.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- "movi v31.4s, #0\n"
"add %[b_ptr0], %[b_ptr0], #0x10\n"
+ "cbz %[loops], 6f\n"
+ "movi v24.4s, #0\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "movi v25.4s, #0\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n"
".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n"
@@ -181,55 +183,49 @@
".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n"
".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n"
".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n"
- "cbz %[loops], 6f\n"
- "ldr d16, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- "add %[b_ptr0], %[b_ptr0], #0x10\n"
- "ins v16.d[1], temploadreg0\n"
"b.eq 7f\n"
"8:\n"
"str q24, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"movi v24.4s, #0\n"
+ "ldr d16, [%[b_ptr0]]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q25, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
+ "ins v16.d[1], temploadreg0\n"
+ "add %[b_ptr0], %[b_ptr0], #0x10\n"
+ "str q26, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ "movi v26.4s, #0\n"
"prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
- "str q26, [c_ptr2]\n"
- "movi v26.4s, #0\n"
- "add c_ptr2, c_ptr2, #0x10\n"
- ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n"
"str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
"add c_ptr3, c_ptr3, #0x10\n"
- ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n"
"str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
- ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n"
+ ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n"
"str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
"add c_ptr5, c_ptr5, #0x10\n"
- ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n"
"str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
"add c_ptr6, c_ptr6, #0x10\n"
- ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n"
+ ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n"
"str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
"add c_ptr7, c_ptr7, #0x10\n"
- ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n"
"prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n"
- "ldr d16, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], #0x10\n"
+ ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n"
"prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n"
"prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- "ins v16.d[1], temploadreg0\n"
"prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
"prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
"prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
@@ -239,6 +235,8 @@
"str q24, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v24.4s, #0\n"
+ "ldr q16, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x10\n"
"str q25, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
@@ -268,23 +266,34 @@
"add c_ptr7, c_ptr7, #0x10\n"
".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n"
".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n"
+ "b 9f\n"
"6:\n"
+ "movi v24.4s, #0\n"
+ "movi v25.4s, #0\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n"
+ ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n"
+ ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n"
+ ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n"
+ "9:\n"
"str q24, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q25, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, #0x10\n"
"str q26, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, #0x10\n"
"str q27, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, #0x10\n"
"str q28, [c_ptr4]\n"
- "add c_ptr4, c_ptr4, #0x10\n"
"str q29, [c_ptr5]\n"
- "add c_ptr5, c_ptr5, #0x10\n"
"str q30, [c_ptr6]\n"
- "add c_ptr6, c_ptr6, #0x10\n"
"str q31, [c_ptr7]\n"
- "add c_ptr7, c_ptr7, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -423,24 +432,26 @@
"ld1 {v6.b}[6], [a_ptr6]\n"
"ld1 {v7.b}[6], [a_ptr7]\n"
"3:\n"
- "movi v24.4s, #0\n"
"ldr q16, [%[b_ptr0]]\n"
- "movi v25.4s, #0\n"
- "ldr q17, [%[b_ptr0], #0x10]\n"
- "movi v26.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
- "movi v27.4s, #0\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
- "movi v28.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
- "movi v29.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
- "movi v30.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- "movi v31.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "cbz %[loops], 6f\n"
+ "movi v24.4s, #0\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "movi v25.4s, #0\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n"
".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n"
".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n"
@@ -456,78 +467,72 @@
".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n"
".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n"
".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n"
- "cbz %[loops], 6f\n"
- "ldr d16, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- "ldr d17, [%[b_ptr0], #0x10]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- "ins v16.d[1], temploadreg0\n"
"b.eq 7f\n"
"8:\n"
"str q24, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"movi v24.4s, #0\n"
- "ins v17.d[1], temploadreg1\n"
+ "ldr d16, [%[b_ptr0]]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q25, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
+ "ldr d17, [%[b_ptr0], #0x10]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
"str q26, [c_ptr2]\n"
- "movi v26.4s, #0\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
- ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n"
- "str q27, [c_ptr3]\n"
- "movi v27.4s, #0\n"
"add c_ptr2, c_ptr2, #0x10\n"
- ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n"
+ "movi v26.4s, #0\n"
+ "ins v16.d[1], temploadreg0\n"
+ "ins v17.d[1], temploadreg1\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "str q27, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ "movi v27.4s, #0\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
"str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
- "add c_ptr3, c_ptr3, #0x10\n"
- ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n"
"str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
- "add c_ptr4, c_ptr4, #0x10\n"
- ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n"
"str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
- "add c_ptr5, c_ptr5, #0x10\n"
- ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
+ ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n"
"str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
- "add c_ptr6, c_ptr6, #0x10\n"
- ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n"
"add c_ptr7, c_ptr7, #0x10\n"
- ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n"
- "ldr d16, [%[b_ptr0]]\n"
- ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n"
- "ins v16.d[1], temploadreg0\n"
- ".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n"
"prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n"
+ ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n"
"prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".inst 0x4fa4e23c // sdot v28.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n"
"prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- ".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n"
+ ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n"
"prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- ".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
- ".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n"
- "ldr d17, [%[b_ptr0], #0x10]\n"
+ ".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n"
+ ".inst 0x4fa4e23c // sdot v28.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n"
+ ".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n"
"b.ne 8b\n"
"7:\n"
"str q24, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v24.4s, #0\n"
- "ins v17.d[1], temploadreg1\n"
+ "ldr q16, [%[b_ptr0]]\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
"str q25, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
@@ -565,23 +570,42 @@
".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n"
".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n"
".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n"
+ "b 9f\n"
"6:\n"
+ "movi v24.4s, #0\n"
+ "movi v25.4s, #0\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n"
+ ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n"
+ ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n"
+ ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n"
+ ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n"
+ ".inst 0x4fa4e23c // sdot v28.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n"
+ ".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n"
+ "9:\n"
"str q24, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q25, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, #0x10\n"
"str q26, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, #0x10\n"
"str q27, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, #0x10\n"
"str q28, [c_ptr4]\n"
- "add c_ptr4, c_ptr4, #0x10\n"
"str q29, [c_ptr5]\n"
- "add c_ptr5, c_ptr5, #0x10\n"
"str q30, [c_ptr6]\n"
- "add c_ptr6, c_ptr6, #0x10\n"
"str q31, [c_ptr7]\n"
- "add c_ptr7, c_ptr7, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -720,26 +744,28 @@
"ld1 {v6.b}[10], [a_ptr6]\n"
"ld1 {v7.b}[10], [a_ptr7]\n"
"3:\n"
- "movi v24.4s, #0\n"
"ldr q16, [%[b_ptr0]]\n"
- "movi v25.4s, #0\n"
- "ldr q17, [%[b_ptr0], #0x10]\n"
- "movi v26.4s, #0\n"
- "ldr q18, [%[b_ptr0], #0x20]\n"
- "movi v27.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
- "movi v28.4s, #0\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
- "movi v29.4s, #0\n"
+ "ldr q18, [%[b_ptr0], #0x20]\n"
"prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
- "movi v30.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
- "movi v31.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x30\n"
+ "cbz %[loops], 6f\n"
+ "movi v24.4s, #0\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "movi v25.4s, #0\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n"
".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n"
".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n"
".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n"
@@ -762,95 +788,86 @@
".inst 0x4f85ea5d // sdot v29.4s, v18.16b, v5.4b[2]\n"
".inst 0x4f86ea5e // sdot v30.4s, v18.16b, v6.4b[2]\n"
".inst 0x4f87ea5f // sdot v31.4s, v18.16b, v7.4b[2]\n"
- "cbz %[loops], 6f\n"
- "ldr d16, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- "ldr d17, [%[b_ptr0], #0x10]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- "ldr d18, [%[b_ptr0], #0x20]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- "add %[b_ptr0], %[b_ptr0], #0x30\n"
- "ins v16.d[1], temploadreg0\n"
- "ins v17.d[1], temploadreg1\n"
"b.eq 7f\n"
"8:\n"
"str q24, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"movi v24.4s, #0\n"
- "ins v18.d[1], temploadreg2\n"
+ "ldr d16, [%[b_ptr0]]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q25, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
+ "ldr d17, [%[b_ptr0], #0x10]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
"str q26, [c_ptr2]\n"
- "movi v26.4s, #0\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
"add c_ptr2, c_ptr2, #0x10\n"
- ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n"
+ "movi v26.4s, #0\n"
+ "ldr d18, [%[b_ptr0], #0x20]\n"
+ "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x30\n"
"str q27, [c_ptr3]\n"
- "movi v27.4s, #0\n"
"add c_ptr3, c_ptr3, #0x10\n"
- ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n"
+ "movi v27.4s, #0\n"
+ "ins v16.d[1], temploadreg0\n"
+ "ins v17.d[1], temploadreg1\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
"str q28, [c_ptr4]\n"
- "movi v28.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
- ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n"
+ "movi v28.4s, #0\n"
+ "ins v18.d[1], temploadreg2\n"
+ ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
"str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
"add c_ptr5, c_ptr5, #0x10\n"
- ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n"
"str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
"add c_ptr6, c_ptr6, #0x10\n"
- ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n"
+ ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n"
"str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
"add c_ptr7, c_ptr7, #0x10\n"
- ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
- ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n"
- "ldr d16, [%[b_ptr0]]\n"
- ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n"
- "ins v16.d[1], temploadreg0\n"
- ".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n"
"prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n"
+ ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n"
"prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".inst 0x4fa4e23c // sdot v28.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n"
"prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- ".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n"
+ ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n"
"prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- ".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n"
"prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
- ".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n"
- "ldr d17, [%[b_ptr0], #0x10]\n"
- ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
+ ".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n"
+ ".inst 0x4fa4e23c // sdot v28.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n"
+ ".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n"
+ ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
".inst 0x4f81ea59 // sdot v25.4s, v18.16b, v1.4b[2]\n"
- "ins v17.d[1], temploadreg1\n"
".inst 0x4f82ea5a // sdot v26.4s, v18.16b, v2.4b[2]\n"
".inst 0x4f83ea5b // sdot v27.4s, v18.16b, v3.4b[2]\n"
".inst 0x4f84ea5c // sdot v28.4s, v18.16b, v4.4b[2]\n"
".inst 0x4f85ea5d // sdot v29.4s, v18.16b, v5.4b[2]\n"
".inst 0x4f86ea5e // sdot v30.4s, v18.16b, v6.4b[2]\n"
".inst 0x4f87ea5f // sdot v31.4s, v18.16b, v7.4b[2]\n"
- "ldr d18, [%[b_ptr0], #0x20]\n"
- "add %[b_ptr0], %[b_ptr0], #0x30\n"
"b.ne 8b\n"
"7:\n"
"str q24, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v24.4s, #0\n"
- "ins v18.d[1], temploadreg2\n"
+ "ldr q16, [%[b_ptr0]]\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
"str q25, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
+ "ldr q18, [%[b_ptr0], #0x20]\n"
".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
"str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
@@ -876,8 +893,9 @@
"movi v31.4s, #0\n"
"add c_ptr7, c_ptr7, #0x10\n"
".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n"
- ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x30\n"
".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n"
+ ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n"
".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n"
".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n"
@@ -893,23 +911,50 @@
".inst 0x4f85ea5d // sdot v29.4s, v18.16b, v5.4b[2]\n"
".inst 0x4f86ea5e // sdot v30.4s, v18.16b, v6.4b[2]\n"
".inst 0x4f87ea5f // sdot v31.4s, v18.16b, v7.4b[2]\n"
+ "b 9f\n"
"6:\n"
+ "movi v24.4s, #0\n"
+ "movi v25.4s, #0\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n"
+ ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n"
+ ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n"
+ ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n"
+ ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n"
+ ".inst 0x4fa4e23c // sdot v28.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n"
+ ".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n"
+ ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x4f81ea59 // sdot v25.4s, v18.16b, v1.4b[2]\n"
+ ".inst 0x4f82ea5a // sdot v26.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x4f83ea5b // sdot v27.4s, v18.16b, v3.4b[2]\n"
+ ".inst 0x4f84ea5c // sdot v28.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x4f85ea5d // sdot v29.4s, v18.16b, v5.4b[2]\n"
+ ".inst 0x4f86ea5e // sdot v30.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x4f87ea5f // sdot v31.4s, v18.16b, v7.4b[2]\n"
+ "9:\n"
"str q24, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q25, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, #0x10\n"
"str q26, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, #0x10\n"
"str q27, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, #0x10\n"
"str q28, [c_ptr4]\n"
- "add c_ptr4, c_ptr4, #0x10\n"
"str q29, [c_ptr5]\n"
- "add c_ptr5, c_ptr5, #0x10\n"
"str q30, [c_ptr6]\n"
- "add c_ptr6, c_ptr6, #0x10\n"
"str q31, [c_ptr7]\n"
- "add c_ptr7, c_ptr7, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -1056,28 +1101,30 @@
"ld1 {v6.b}[14], [a_ptr6]\n"
"ld1 {v7.b}[14], [a_ptr7]\n"
"3:\n"
- "movi v24.4s, #0\n"
"ldr q16, [%[b_ptr0]]\n"
- "movi v25.4s, #0\n"
- "ldr q17, [%[b_ptr0], #0x10]\n"
- "movi v26.4s, #0\n"
- "ldr q18, [%[b_ptr0], #0x20]\n"
- "movi v27.4s, #0\n"
- "ldr q19, [%[b_ptr0], #0x30]\n"
- "movi v28.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
- "movi v29.4s, #0\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
- "movi v30.4s, #0\n"
+ "ldr q18, [%[b_ptr0], #0x20]\n"
"prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
- "movi v31.4s, #0\n"
+ "ldr q19, [%[b_ptr0], #0x30]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
- ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x40\n"
+ "cbz %[loops], 6f\n"
+ "movi v24.4s, #0\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "movi v25.4s, #0\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n"
+ ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n"
".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n"
".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n"
".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n"
@@ -1107,112 +1154,101 @@
".inst 0x4fa5ea7d // sdot v29.4s, v19.16b, v5.4b[3]\n"
".inst 0x4fa6ea7e // sdot v30.4s, v19.16b, v6.4b[3]\n"
".inst 0x4fa7ea7f // sdot v31.4s, v19.16b, v7.4b[3]\n"
- "cbz %[loops], 6f\n"
- "ldr d16, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- "ldr d17, [%[b_ptr0], #0x10]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- "ldr d18, [%[b_ptr0], #0x20]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- "ldr d19, [%[b_ptr0], #0x30]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- "add %[b_ptr0], %[b_ptr0], #0x40\n"
- "ins v16.d[1], temploadreg0\n"
- "ins v17.d[1], temploadreg1\n"
- "ins v18.d[1], temploadreg2\n"
"b.eq 7f\n"
"8:\n"
"str q24, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"movi v24.4s, #0\n"
- "ins v19.d[1], temploadreg3\n"
+ "ldr d16, [%[b_ptr0]]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q25, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
+ "ldr d17, [%[b_ptr0], #0x10]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
"str q26, [c_ptr2]\n"
- "movi v26.4s, #0\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
"add c_ptr2, c_ptr2, #0x10\n"
- ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n"
+ "movi v26.4s, #0\n"
+ "ldr d18, [%[b_ptr0], #0x20]\n"
+ "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
"str q27, [c_ptr3]\n"
- "movi v27.4s, #0\n"
"add c_ptr3, c_ptr3, #0x10\n"
- ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n"
+ "movi v27.4s, #0\n"
+ "ldr d19, [%[b_ptr0], #0x30]\n"
+ "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x40\n"
"str q28, [c_ptr4]\n"
- "movi v28.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
- ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n"
+ "movi v28.4s, #0\n"
+ "ins v16.d[1], temploadreg0\n"
+ "ins v17.d[1], temploadreg1\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
"str q29, [c_ptr5]\n"
- "movi v29.4s, #0\n"
"add c_ptr5, c_ptr5, #0x10\n"
- ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n"
+ "movi v29.4s, #0\n"
+ "ins v18.d[1], temploadreg2\n"
+ ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
"str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
+ "ins v19.d[1], temploadreg3\n"
+ ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n"
"add c_ptr6, c_ptr6, #0x10\n"
- ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n"
+ ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n"
"str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
"add c_ptr7, c_ptr7, #0x10\n"
- ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
- ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n"
- "ldr d16, [%[b_ptr0]]\n"
- ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n"
- "ins v16.d[1], temploadreg0\n"
- ".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n"
+ ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n"
"prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".inst 0x4fa4e23c // sdot v28.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n"
"prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- ".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n"
+ ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n"
"prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- ".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n"
"prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
- ".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n"
- "ldr d17, [%[b_ptr0], #0x10]\n"
- ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n"
"prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
+ ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n"
+ ".inst 0x4fa4e23c // sdot v28.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n"
+ ".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n"
+ ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
".inst 0x4f81ea59 // sdot v25.4s, v18.16b, v1.4b[2]\n"
- "ins v17.d[1], temploadreg1\n"
".inst 0x4f82ea5a // sdot v26.4s, v18.16b, v2.4b[2]\n"
".inst 0x4f83ea5b // sdot v27.4s, v18.16b, v3.4b[2]\n"
".inst 0x4f84ea5c // sdot v28.4s, v18.16b, v4.4b[2]\n"
".inst 0x4f85ea5d // sdot v29.4s, v18.16b, v5.4b[2]\n"
".inst 0x4f86ea5e // sdot v30.4s, v18.16b, v6.4b[2]\n"
".inst 0x4f87ea5f // sdot v31.4s, v18.16b, v7.4b[2]\n"
- "ldr d18, [%[b_ptr0], #0x20]\n"
".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n"
".inst 0x4fa1ea79 // sdot v25.4s, v19.16b, v1.4b[3]\n"
".inst 0x4fa2ea7a // sdot v26.4s, v19.16b, v2.4b[3]\n"
- "ins v18.d[1], temploadreg2\n"
".inst 0x4fa3ea7b // sdot v27.4s, v19.16b, v3.4b[3]\n"
".inst 0x4fa4ea7c // sdot v28.4s, v19.16b, v4.4b[3]\n"
".inst 0x4fa5ea7d // sdot v29.4s, v19.16b, v5.4b[3]\n"
".inst 0x4fa6ea7e // sdot v30.4s, v19.16b, v6.4b[3]\n"
".inst 0x4fa7ea7f // sdot v31.4s, v19.16b, v7.4b[3]\n"
- "ldr d19, [%[b_ptr0], #0x30]\n"
- "add %[b_ptr0], %[b_ptr0], #0x40\n"
"b.ne 8b\n"
"7:\n"
"str q24, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v24.4s, #0\n"
- "ins v19.d[1], temploadreg3\n"
+ "ldr q16, [%[b_ptr0]]\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
"str q25, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
+ "ldr q18, [%[b_ptr0], #0x20]\n"
".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
"str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
+ "ldr q19, [%[b_ptr0], #0x30]\n"
"add c_ptr2, c_ptr2, #0x10\n"
".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n"
"str q27, [c_ptr3]\n"
@@ -1235,8 +1271,9 @@
"movi v31.4s, #0\n"
"add c_ptr7, c_ptr7, #0x10\n"
".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n"
- ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x40\n"
".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n"
+ ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n"
".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n"
".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n"
@@ -1260,23 +1297,58 @@
".inst 0x4fa5ea7d // sdot v29.4s, v19.16b, v5.4b[3]\n"
".inst 0x4fa6ea7e // sdot v30.4s, v19.16b, v6.4b[3]\n"
".inst 0x4fa7ea7f // sdot v31.4s, v19.16b, v7.4b[3]\n"
+ "b 9f\n"
"6:\n"
+ "movi v24.4s, #0\n"
+ "movi v25.4s, #0\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n"
+ ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n"
+ ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n"
+ ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n"
+ ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n"
+ ".inst 0x4fa4e23c // sdot v28.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n"
+ ".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n"
+ ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x4f81ea59 // sdot v25.4s, v18.16b, v1.4b[2]\n"
+ ".inst 0x4f82ea5a // sdot v26.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x4f83ea5b // sdot v27.4s, v18.16b, v3.4b[2]\n"
+ ".inst 0x4f84ea5c // sdot v28.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x4f85ea5d // sdot v29.4s, v18.16b, v5.4b[2]\n"
+ ".inst 0x4f86ea5e // sdot v30.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x4f87ea5f // sdot v31.4s, v18.16b, v7.4b[2]\n"
+ ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n"
+ ".inst 0x4fa1ea79 // sdot v25.4s, v19.16b, v1.4b[3]\n"
+ ".inst 0x4fa2ea7a // sdot v26.4s, v19.16b, v2.4b[3]\n"
+ ".inst 0x4fa3ea7b // sdot v27.4s, v19.16b, v3.4b[3]\n"
+ ".inst 0x4fa4ea7c // sdot v28.4s, v19.16b, v4.4b[3]\n"
+ ".inst 0x4fa5ea7d // sdot v29.4s, v19.16b, v5.4b[3]\n"
+ ".inst 0x4fa6ea7e // sdot v30.4s, v19.16b, v6.4b[3]\n"
+ ".inst 0x4fa7ea7f // sdot v31.4s, v19.16b, v7.4b[3]\n"
+ "9:\n"
"str q24, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q25, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, #0x10\n"
"str q26, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, #0x10\n"
"str q27, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, #0x10\n"
"str q28, [c_ptr4]\n"
- "add c_ptr4, c_ptr4, #0x10\n"
"str q29, [c_ptr5]\n"
- "add c_ptr5, c_ptr5, #0x10\n"
"str q30, [c_ptr6]\n"
- "add c_ptr6, c_ptr6, #0x10\n"
"str q31, [c_ptr7]\n"
- "add c_ptr7, c_ptr7, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -1363,26 +1435,34 @@
"add c_ptr1, %[c_ptr0], #0x0\n"
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
+ "cbnz %[odds], 2f\n"
"ldr q0, [%[a_ptr0]], #0x10\n"
"ldr q2, [a_ptr1], #0x10\n"
"ldr q4, [a_ptr2], #0x10\n"
"ldr q6, [a_ptr3], #0x10\n"
+ "ldr s1, [%[a_ptr0]]\n"
+ "ldr q8, [a_ptr4], #0x10\n"
+ "ldr s3, [a_ptr1]\n"
+ "ldr q10, [a_ptr5], #0x10\n"
+ "ldr s5, [a_ptr2]\n"
+ "ldr q12, [a_ptr6], #0x10\n"
+ "ldr s7, [a_ptr3]\n"
+ "ldr q14, [a_ptr7], #0x10\n"
+ "ldr s9, [a_ptr4]\n"
+ "ldr s11, [a_ptr5]\n"
+ "ldr s13, [a_ptr6]\n"
+ "ldr s15, [a_ptr7]\n"
+ "b 3f\n"
+ "2:\n"
+ "ldr q0, [%[a_ptr0]], #0x10\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "ldr q2, [a_ptr1], #0x10\n"
+ "ldr q4, [a_ptr2], #0x10\n"
+ "ldr q6, [a_ptr3], #0x10\n"
"ldr q8, [a_ptr4], #0x10\n"
"ldr q10, [a_ptr5], #0x10\n"
"ldr q12, [a_ptr6], #0x10\n"
"ldr q14, [a_ptr7], #0x10\n"
- "cbnz %[odds], 2f\n"
- "ldr s1, [%[a_ptr0]]\n"
- "ldr s3, [a_ptr1]\n"
- "ldr s5, [a_ptr2]\n"
- "ldr s7, [a_ptr3]\n"
- "ldr s9, [a_ptr4]\n"
- "ldr s11, [a_ptr5]\n"
- "ldr s13, [a_ptr6]\n"
- "ldr s15, [a_ptr7]\n"
- "b 3f\n"
- "2:\n"
- "subs %[odds], %[odds], #0x1\n"
"b.ne 4f\n"
"ldr b1, [%[a_ptr0]]\n"
"ldr b3, [a_ptr1]\n"
@@ -1415,30 +1495,32 @@
"ld1 {v13.b}[2], [a_ptr6]\n"
"ld1 {v15.b}[2], [a_ptr7]\n"
"3:\n"
- "movi v24.4s, #0\n"
"ldr q16, [%[b_ptr0]]\n"
- "movi v25.4s, #0\n"
- "ldr q17, [%[b_ptr0], #0x10]\n"
- "movi v26.4s, #0\n"
- "ldr q18, [%[b_ptr0], #0x20]\n"
- "movi v27.4s, #0\n"
- "ldr q19, [%[b_ptr0], #0x30]\n"
- "movi v28.4s, #0\n"
- "ldr q20, [%[b_ptr0], #0x40]\n"
- "movi v29.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
- "movi v30.4s, #0\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
- "movi v31.4s, #0\n"
+ "ldr q18, [%[b_ptr0], #0x20]\n"
"prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
- ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ "ldr q19, [%[b_ptr0], #0x30]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
- ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
+ "ldr q20, [%[b_ptr0], #0x40]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x50\n"
+ "cbz %[loops], 6f\n"
+ "movi v24.4s, #0\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "movi v25.4s, #0\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
@@ -1475,126 +1557,113 @@
".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n"
".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n"
".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n"
- "cbz %[loops], 6f\n"
- "ldr d16, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- "ldr d17, [%[b_ptr0], #0x10]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- "ldr d18, [%[b_ptr0], #0x20]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- "ldr d19, [%[b_ptr0], #0x30]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- "ldr d20, [%[b_ptr0], #0x40]\n"
- "ins v16.d[1], temploadreg0\n"
- "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- "add %[b_ptr0], %[b_ptr0], #0x50\n"
- "ins v17.d[1], temploadreg1\n"
- "ins v18.d[1], temploadreg2\n"
- "ins v19.d[1], temploadreg3\n"
"b.eq 7f\n"
"8:\n"
"str q24, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"movi v24.4s, #0\n"
- "ins v20.d[1], temploadreg0\n"
+ "ldr d16, [%[b_ptr0]]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q25, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
+ "ldr d17, [%[b_ptr0], #0x10]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
"str q26, [c_ptr2]\n"
- "movi v26.4s, #0\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
"add c_ptr2, c_ptr2, #0x10\n"
- ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
+ "movi v26.4s, #0\n"
+ "ldr d18, [%[b_ptr0], #0x20]\n"
+ "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
"str q27, [c_ptr3]\n"
- "movi v27.4s, #0\n"
"add c_ptr3, c_ptr3, #0x10\n"
- ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
+ "movi v27.4s, #0\n"
+ "ldr d19, [%[b_ptr0], #0x30]\n"
+ "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
"str q28, [c_ptr4]\n"
- "movi v28.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
- ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
+ "movi v28.4s, #0\n"
+ "ldr d20, [%[b_ptr0], #0x40]\n"
+ "ins v16.d[1], temploadreg0\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
"str q29, [c_ptr5]\n"
- "movi v29.4s, #0\n"
"add c_ptr5, c_ptr5, #0x10\n"
- ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
+ "movi v29.4s, #0\n"
+ "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
+ ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
"str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
- "add c_ptr6, c_ptr6, #0x10\n"
- ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
+ "ins v17.d[1], temploadreg1\n"
+ ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
+ "ins v18.d[1], temploadreg2\n"
+ ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
"str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
+ "ins v19.d[1], temploadreg3\n"
+ ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
+ "ins v20.d[1], temploadreg0\n"
+ ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
+ ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
"add c_ptr7, c_ptr7, #0x10\n"
".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x50\n"
".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
- "ldr d16, [%[b_ptr0]]\n"
- ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
- "ins v16.d[1], temploadreg0\n"
- ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n"
+ ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n"
- "ldr d17, [%[b_ptr0], #0x10]\n"
- ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
- ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n"
- "ins v17.d[1], temploadreg1\n"
- ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
+ ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n"
+ ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n"
+ ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n"
+ ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n"
+ ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n"
".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n"
".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n"
".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n"
".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n"
".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n"
- "ldr d18, [%[b_ptr0], #0x20]\n"
".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n"
".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n"
".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n"
- "ins v18.d[1], temploadreg2\n"
".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n"
".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n"
".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n"
".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n"
".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n"
- "ldr d19, [%[b_ptr0], #0x30]\n"
".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n"
".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n"
".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n"
- "ins v19.d[1], temploadreg3\n"
".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n"
".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n"
".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n"
".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n"
".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n"
- "ldr d20, [%[b_ptr0], #0x40]\n"
- "add %[b_ptr0], %[b_ptr0], #0x50\n"
"b.ne 8b\n"
"7:\n"
"str q24, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v24.4s, #0\n"
- "ins v20.d[1], temploadreg0\n"
+ "ldr q16, [%[b_ptr0]]\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
"str q25, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
+ "ldr q18, [%[b_ptr0], #0x20]\n"
".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
"str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
+ "ldr q19, [%[b_ptr0], #0x30]\n"
+ "ldr q20, [%[b_ptr0], #0x40]\n"
"add c_ptr2, c_ptr2, #0x10\n"
".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
"str q27, [c_ptr3]\n"
@@ -1617,8 +1686,9 @@
"movi v31.4s, #0\n"
"add c_ptr7, c_ptr7, #0x10\n"
".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
- ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x50\n"
".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
+ ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
@@ -1650,23 +1720,66 @@
".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n"
".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n"
".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n"
+ "b 9f\n"
"6:\n"
+ "movi v24.4s, #0\n"
+ "movi v25.4s, #0\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
+ ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
+ ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
+ ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
+ ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n"
+ ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n"
+ ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n"
+ ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n"
+ ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n"
+ ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n"
+ ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n"
+ ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n"
+ ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n"
+ ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n"
+ ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n"
+ ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n"
+ ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n"
+ ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n"
+ ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n"
+ ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n"
+ ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n"
+ ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n"
+ ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n"
+ ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n"
+ ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n"
+ ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n"
+ ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n"
+ ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n"
+ "9:\n"
"str q24, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q25, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, #0x10\n"
"str q26, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, #0x10\n"
"str q27, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, #0x10\n"
"str q28, [c_ptr4]\n"
- "add c_ptr4, c_ptr4, #0x10\n"
"str q29, [c_ptr5]\n"
- "add c_ptr5, c_ptr5, #0x10\n"
"str q30, [c_ptr6]\n"
- "add c_ptr6, c_ptr6, #0x10\n"
"str q31, [c_ptr7]\n"
- "add c_ptr7, c_ptr7, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -1753,34 +1866,42 @@
"add c_ptr1, %[c_ptr0], #0x0\n"
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
+ "cbnz %[odds], 2f\n"
"ldr q0, [%[a_ptr0]], #0x10\n"
"ldr q2, [a_ptr1], #0x10\n"
"ldr q4, [a_ptr2], #0x10\n"
"ldr q6, [a_ptr3], #0x10\n"
- "ldr q8, [a_ptr4], #0x10\n"
- "ldr q10, [a_ptr5], #0x10\n"
- "ldr q12, [a_ptr6], #0x10\n"
- "ldr q14, [a_ptr7], #0x10\n"
- "cbnz %[odds], 2f\n"
"ldr d1, [%[a_ptr0]]\n"
+ "ldr q8, [a_ptr4], #0x10\n"
"ldr d3, [a_ptr1]\n"
+ "ldr q10, [a_ptr5], #0x10\n"
"ldr d5, [a_ptr2]\n"
+ "ldr q12, [a_ptr6], #0x10\n"
"ldr d7, [a_ptr3]\n"
+ "ldr q14, [a_ptr7], #0x10\n"
"ldr d9, [a_ptr4]\n"
"ldr d11, [a_ptr5]\n"
"ldr d13, [a_ptr6]\n"
"ldr d15, [a_ptr7]\n"
"b 3f\n"
"2:\n"
+ "ldr q0, [%[a_ptr0]], #0x10\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "ldr q2, [a_ptr1], #0x10\n"
+ "ldr q4, [a_ptr2], #0x10\n"
"ldr s1, [%[a_ptr0]], #0x4\n"
+ "ldr q6, [a_ptr3], #0x10\n"
"ldr s3, [a_ptr1], #0x4\n"
+ "ldr q8, [a_ptr4], #0x10\n"
"ldr s5, [a_ptr2], #0x4\n"
+ "ldr q10, [a_ptr5], #0x10\n"
"ldr s7, [a_ptr3], #0x4\n"
+ "ldr q12, [a_ptr6], #0x10\n"
"ldr s9, [a_ptr4], #0x4\n"
+ "ldr q14, [a_ptr7], #0x10\n"
"ldr s11, [a_ptr5], #0x4\n"
"ldr s13, [a_ptr6], #0x4\n"
"ldr s15, [a_ptr7], #0x4\n"
- "subs %[odds], %[odds], #0x1\n"
"b.ne 4f\n"
"ld1 {v1.b}[4], [%[a_ptr0]]\n"
"ld1 {v3.b}[4], [a_ptr1]\n"
@@ -1813,32 +1934,34 @@
"ld1 {v13.b}[6], [a_ptr6]\n"
"ld1 {v15.b}[6], [a_ptr7]\n"
"3:\n"
- "movi v24.4s, #0\n"
"ldr q16, [%[b_ptr0]]\n"
- "movi v25.4s, #0\n"
- "ldr q17, [%[b_ptr0], #0x10]\n"
- "movi v26.4s, #0\n"
- "ldr q18, [%[b_ptr0], #0x20]\n"
- "movi v27.4s, #0\n"
- "ldr q19, [%[b_ptr0], #0x30]\n"
- "movi v28.4s, #0\n"
- "ldr q20, [%[b_ptr0], #0x40]\n"
- "movi v29.4s, #0\n"
- "ldr q21, [%[b_ptr0], #0x50]\n"
- "movi v30.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
- "movi v31.4s, #0\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
- ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ "ldr q18, [%[b_ptr0], #0x20]\n"
"prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
- ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
+ "ldr q19, [%[b_ptr0], #0x30]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
- ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
+ "ldr q20, [%[b_ptr0], #0x40]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
+ "ldr q21, [%[b_ptr0], #0x50]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x60\n"
+ "cbz %[loops], 6f\n"
+ "movi v24.4s, #0\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "movi v25.4s, #0\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
@@ -1882,146 +2005,132 @@
".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n"
".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n"
".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n"
- "cbz %[loops], 6f\n"
- "ldr d16, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- "ldr d17, [%[b_ptr0], #0x10]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- "ldr d18, [%[b_ptr0], #0x20]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- "ldr d19, [%[b_ptr0], #0x30]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- "ldr d20, [%[b_ptr0], #0x40]\n"
- "ins v16.d[1], temploadreg0\n"
- "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- "ldr d21, [%[b_ptr0], #0x50]\n"
- "ins v17.d[1], temploadreg1\n"
- "ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- "add %[b_ptr0], %[b_ptr0], #0x60\n"
- "ins v18.d[1], temploadreg2\n"
- "ins v19.d[1], temploadreg3\n"
- "ins v20.d[1], temploadreg0\n"
"b.eq 7f\n"
"8:\n"
"str q24, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"movi v24.4s, #0\n"
- "ins v21.d[1], temploadreg1\n"
+ "ldr d16, [%[b_ptr0]]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q25, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
+ "ldr d17, [%[b_ptr0], #0x10]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
"str q26, [c_ptr2]\n"
- "movi v26.4s, #0\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
"add c_ptr2, c_ptr2, #0x10\n"
- ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
+ "movi v26.4s, #0\n"
+ "ldr d18, [%[b_ptr0], #0x20]\n"
+ "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
"str q27, [c_ptr3]\n"
- "movi v27.4s, #0\n"
"add c_ptr3, c_ptr3, #0x10\n"
- ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
+ "movi v27.4s, #0\n"
+ "ldr d19, [%[b_ptr0], #0x30]\n"
+ "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
"str q28, [c_ptr4]\n"
- "movi v28.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
- ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
+ "movi v28.4s, #0\n"
+ "ldr d20, [%[b_ptr0], #0x40]\n"
+ "ins v16.d[1], temploadreg0\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
"str q29, [c_ptr5]\n"
- "movi v29.4s, #0\n"
"add c_ptr5, c_ptr5, #0x10\n"
- ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
+ "movi v29.4s, #0\n"
+ "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
+ ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
"str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
- "add c_ptr6, c_ptr6, #0x10\n"
- ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
+ "ldr d21, [%[b_ptr0], #0x50]\n"
+ ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
+ "ins v17.d[1], temploadreg1\n"
+ ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
"str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
- "add c_ptr7, c_ptr7, #0x10\n"
- ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
- ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
- "ldr d16, [%[b_ptr0]]\n"
- ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
- "ins v16.d[1], temploadreg0\n"
- ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n"
- "ldr d17, [%[b_ptr0], #0x10]\n"
- ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
- "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
- ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n"
- "ins v17.d[1], temploadreg1\n"
- ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n"
"ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
+ "ins v18.d[1], temploadreg2\n"
+ ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
+ "ins v19.d[1], temploadreg3\n"
+ ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
+ "ins v20.d[1], temploadreg0\n"
+ ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
+ "ins v21.d[1], temploadreg1\n"
+ ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
+ ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
+ "add c_ptr7, c_ptr7, #0x10\n"
+ ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x60\n"
+ ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
+ ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
+ ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
+ ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
+ ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n"
+ ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n"
+ ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n"
".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n"
".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n"
".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n"
".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n"
- "ldr d18, [%[b_ptr0], #0x20]\n"
".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n"
".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n"
".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n"
- "ins v18.d[1], temploadreg2\n"
".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n"
".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n"
".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n"
".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n"
".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n"
- "ldr d19, [%[b_ptr0], #0x30]\n"
".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n"
".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n"
".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n"
- "ins v19.d[1], temploadreg3\n"
".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n"
".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n"
".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n"
".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n"
".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n"
- "ldr d20, [%[b_ptr0], #0x40]\n"
".inst 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n"
".inst 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n"
".inst 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n"
- "ins v20.d[1], temploadreg0\n"
".inst 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n"
".inst 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n"
".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n"
".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n"
".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n"
- "ldr d21, [%[b_ptr0], #0x50]\n"
- "add %[b_ptr0], %[b_ptr0], #0x60\n"
"b.ne 8b\n"
"7:\n"
"str q24, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v24.4s, #0\n"
- "ins v21.d[1], temploadreg1\n"
+ "ldr q16, [%[b_ptr0]]\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
"str q25, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
+ "ldr q18, [%[b_ptr0], #0x20]\n"
".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
"str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
+ "ldr q19, [%[b_ptr0], #0x30]\n"
+ "ldr q20, [%[b_ptr0], #0x40]\n"
"add c_ptr2, c_ptr2, #0x10\n"
".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
"str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
- "add c_ptr3, c_ptr3, #0x10\n"
+ "ldr q21, [%[b_ptr0], #0x50]\n"
".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
"str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
@@ -2038,7 +2147,7 @@
"movi v31.4s, #0\n"
"add c_ptr7, c_ptr7, #0x10\n"
".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
- ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x60\n"
".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
@@ -2079,23 +2188,74 @@
".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n"
".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n"
".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n"
+ "b 9f\n"
"6:\n"
+ "movi v24.4s, #0\n"
+ "movi v25.4s, #0\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
+ ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
+ ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
+ ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
+ ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n"
+ ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n"
+ ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n"
+ ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n"
+ ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n"
+ ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n"
+ ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n"
+ ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n"
+ ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n"
+ ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n"
+ ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n"
+ ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n"
+ ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n"
+ ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n"
+ ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n"
+ ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n"
+ ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n"
+ ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n"
+ ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n"
+ ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n"
+ ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n"
+ ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n"
+ ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n"
+ ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n"
+ ".inst 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n"
+ ".inst 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n"
+ ".inst 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n"
+ ".inst 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n"
+ ".inst 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n"
+ ".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n"
+ ".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n"
+ ".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n"
+ "9:\n"
"str q24, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q25, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, #0x10\n"
"str q26, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, #0x10\n"
"str q27, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, #0x10\n"
"str q28, [c_ptr4]\n"
- "add c_ptr4, c_ptr4, #0x10\n"
"str q29, [c_ptr5]\n"
- "add c_ptr5, c_ptr5, #0x10\n"
"str q30, [c_ptr6]\n"
- "add c_ptr6, c_ptr6, #0x10\n"
"str q31, [c_ptr7]\n"
- "add c_ptr7, c_ptr7, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -2242,34 +2402,36 @@
"ld1 {v13.b}[10], [a_ptr6]\n"
"ld1 {v15.b}[10], [a_ptr7]\n"
"3:\n"
- "movi v24.4s, #0\n"
"ldr q16, [%[b_ptr0]]\n"
- "movi v25.4s, #0\n"
- "ldr q17, [%[b_ptr0], #0x10]\n"
- "movi v26.4s, #0\n"
- "ldr q18, [%[b_ptr0], #0x20]\n"
- "movi v27.4s, #0\n"
- "ldr q19, [%[b_ptr0], #0x30]\n"
- "movi v28.4s, #0\n"
- "ldr q20, [%[b_ptr0], #0x40]\n"
- "movi v29.4s, #0\n"
- "ldr q21, [%[b_ptr0], #0x50]\n"
- "movi v30.4s, #0\n"
- "ldr q22, [%[b_ptr0], #0x60]\n"
- "movi v31.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
- ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
- ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
+ "ldr q18, [%[b_ptr0], #0x20]\n"
"prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
- ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
+ "ldr q19, [%[b_ptr0], #0x30]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
- ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
+ "ldr q20, [%[b_ptr0], #0x40]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
+ "ldr q21, [%[b_ptr0], #0x50]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
+ "ldr q22, [%[b_ptr0], #0x60]\n"
"add %[b_ptr0], %[b_ptr0], #0x70\n"
+ "cbz %[loops], 6f\n"
+ "movi v24.4s, #0\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "movi v25.4s, #0\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
+ ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
@@ -2320,178 +2482,162 @@
".inst 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n"
".inst 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n"
".inst 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n"
- "cbz %[loops], 6f\n"
- "ldr d16, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- "ldr d17, [%[b_ptr0], #0x10]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- "ldr d18, [%[b_ptr0], #0x20]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- "ldr d19, [%[b_ptr0], #0x30]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- "ldr d20, [%[b_ptr0], #0x40]\n"
- "ins v16.d[1], temploadreg0\n"
- "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- "ldr d21, [%[b_ptr0], #0x50]\n"
- "ins v17.d[1], temploadreg1\n"
- "ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- "ldr d22, [%[b_ptr0], #0x60]\n"
- "ins v18.d[1], temploadreg2\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- "add %[b_ptr0], %[b_ptr0], #0x70\n"
- "ins v19.d[1], temploadreg3\n"
- "ins v20.d[1], temploadreg0\n"
- "ins v21.d[1], temploadreg1\n"
"b.eq 7f\n"
"8:\n"
"str q24, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"movi v24.4s, #0\n"
- "ins v22.d[1], temploadreg2\n"
+ "ldr d16, [%[b_ptr0]]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q25, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
+ "ldr d17, [%[b_ptr0], #0x10]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
"str q26, [c_ptr2]\n"
- "movi v26.4s, #0\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
"add c_ptr2, c_ptr2, #0x10\n"
- ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
+ "movi v26.4s, #0\n"
+ "ldr d18, [%[b_ptr0], #0x20]\n"
+ "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
"str q27, [c_ptr3]\n"
- "movi v27.4s, #0\n"
"add c_ptr3, c_ptr3, #0x10\n"
- ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
+ "movi v27.4s, #0\n"
+ "ldr d19, [%[b_ptr0], #0x30]\n"
+ "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
"str q28, [c_ptr4]\n"
- "movi v28.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
- ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
+ "movi v28.4s, #0\n"
+ "ldr d20, [%[b_ptr0], #0x40]\n"
+ "ins v16.d[1], temploadreg0\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
"str q29, [c_ptr5]\n"
- "movi v29.4s, #0\n"
"add c_ptr5, c_ptr5, #0x10\n"
- ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
+ "movi v29.4s, #0\n"
+ "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
+ ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
"str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
- "add c_ptr6, c_ptr6, #0x10\n"
- ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
+ "ldr d21, [%[b_ptr0], #0x50]\n"
+ ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
+ "ins v17.d[1], temploadreg1\n"
+ ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
"str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
- "add c_ptr7, c_ptr7, #0x10\n"
+ "ldr temploadreg1, [%[b_ptr0], #0x58]\n"
+ ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
+ "ldr d22, [%[b_ptr0], #0x60]\n"
+ ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
+ "ins v18.d[1], temploadreg2\n"
+ ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
+ "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "ins v19.d[1], temploadreg3\n"
".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
- "ldr d16, [%[b_ptr0]]\n"
+ "ins v20.d[1], temploadreg0\n"
".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ "ins v21.d[1], temploadreg1\n"
".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
- "ins v16.d[1], temploadreg0\n"
+ "ins v22.d[1], temploadreg2\n"
".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ "add c_ptr7, c_ptr7, #0x10\n"
".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x70\n"
".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n"
- "ldr d17, [%[b_ptr0], #0x10]\n"
- ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
"prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
- ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n"
- "ins v17.d[1], temploadreg1\n"
- ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
"prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
+ ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n"
".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n"
".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n"
".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n"
".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n"
- "ldr d18, [%[b_ptr0], #0x20]\n"
".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n"
".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n"
".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n"
- "ins v18.d[1], temploadreg2\n"
".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n"
".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n"
".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n"
".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n"
- "ldr d19, [%[b_ptr0], #0x30]\n"
".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n"
".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n"
".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n"
- "ins v19.d[1], temploadreg3\n"
".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n"
".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n"
".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n"
".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n"
".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n"
- "ldr d20, [%[b_ptr0], #0x40]\n"
".inst 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n"
".inst 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n"
".inst 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n"
- "ins v20.d[1], temploadreg0\n"
".inst 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n"
".inst 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n"
".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n"
".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n"
".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n"
- "ldr d21, [%[b_ptr0], #0x50]\n"
".inst 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n"
".inst 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n"
".inst 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n"
- "ins v21.d[1], temploadreg1\n"
".inst 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n"
".inst 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n"
".inst 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n"
".inst 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n"
".inst 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n"
- "ldr d22, [%[b_ptr0], #0x60]\n"
- "add %[b_ptr0], %[b_ptr0], #0x70\n"
"b.ne 8b\n"
"7:\n"
"str q24, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v24.4s, #0\n"
- "ins v22.d[1], temploadreg2\n"
+ "ldr q16, [%[b_ptr0]]\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
"str q25, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
+ "ldr q18, [%[b_ptr0], #0x20]\n"
".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
"str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
+ "ldr q19, [%[b_ptr0], #0x30]\n"
+ "ldr q20, [%[b_ptr0], #0x40]\n"
"add c_ptr2, c_ptr2, #0x10\n"
".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
"str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
- "add c_ptr3, c_ptr3, #0x10\n"
+ "ldr q21, [%[b_ptr0], #0x50]\n"
".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
+ "ldr q22, [%[b_ptr0], #0x60]\n"
+ ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
"str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
- "add c_ptr4, c_ptr4, #0x10\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
"str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
- "add c_ptr5, c_ptr5, #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
"str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
- "add c_ptr6, c_ptr6, #0x10\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
"str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
- "add c_ptr7, c_ptr7, #0x10\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
- ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
+ "add c_ptr7, c_ptr7, #0x10\n"
".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x70\n"
".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
@@ -2539,23 +2685,82 @@
".inst 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n"
".inst 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n"
".inst 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n"
+ "b 9f\n"
"6:\n"
+ "movi v24.4s, #0\n"
+ "movi v25.4s, #0\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
+ ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
+ ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
+ ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
+ ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n"
+ ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n"
+ ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n"
+ ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n"
+ ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n"
+ ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n"
+ ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n"
+ ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n"
+ ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n"
+ ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n"
+ ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n"
+ ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n"
+ ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n"
+ ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n"
+ ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n"
+ ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n"
+ ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n"
+ ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n"
+ ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n"
+ ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n"
+ ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n"
+ ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n"
+ ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n"
+ ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n"
+ ".inst 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n"
+ ".inst 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n"
+ ".inst 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n"
+ ".inst 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n"
+ ".inst 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n"
+ ".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n"
+ ".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n"
+ ".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n"
+ ".inst 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n"
+ ".inst 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n"
+ ".inst 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n"
+ ".inst 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n"
+ ".inst 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n"
+ ".inst 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n"
+ ".inst 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n"
+ ".inst 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n"
+ "9:\n"
"str q24, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q25, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, #0x10\n"
"str q26, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, #0x10\n"
"str q27, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, #0x10\n"
"str q28, [c_ptr4]\n"
- "add c_ptr4, c_ptr4, #0x10\n"
"str q29, [c_ptr5]\n"
- "add c_ptr5, c_ptr5, #0x10\n"
"str q30, [c_ptr6]\n"
- "add c_ptr6, c_ptr6, #0x10\n"
"str q31, [c_ptr7]\n"
- "add c_ptr7, c_ptr7, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -2643,6 +2848,7 @@
"add c_ptr1, %[c_ptr0], #0x0\n"
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
+ "cbnz %[odds], 2f\n"
"ldr q0, [%[a_ptr0]], #0x10\n"
"ldr q2, [a_ptr1], #0x10\n"
"ldr q4, [a_ptr2], #0x10\n"
@@ -2651,7 +2857,6 @@
"ldr q10, [a_ptr5], #0x10\n"
"ldr q12, [a_ptr6], #0x10\n"
"ldr q14, [a_ptr7], #0x10\n"
- "cbnz %[odds], 2f\n"
"ldr q1, [%[a_ptr0]]\n"
"ldr q3, [a_ptr1]\n"
"ldr q5, [a_ptr2]\n"
@@ -2662,15 +2867,24 @@
"ldr q15, [a_ptr7]\n"
"b 3f\n"
"2:\n"
+ "ldr q0, [%[a_ptr0]], #0x10\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "ldr q2, [a_ptr1], #0x10\n"
+ "ldr q4, [a_ptr2], #0x10\n"
"ldr d1, [%[a_ptr0]], #0x8\n"
+ "ldr q6, [a_ptr3], #0x10\n"
"ldr d3, [a_ptr1], #0x8\n"
+ "ldr q8, [a_ptr4], #0x10\n"
"ldr d5, [a_ptr2], #0x8\n"
+ "ldr q10, [a_ptr5], #0x10\n"
"ldr d7, [a_ptr3], #0x8\n"
+ "ldr q12, [a_ptr6], #0x10\n"
"ldr d9, [a_ptr4], #0x8\n"
+ "ldr q14, [a_ptr7], #0x10\n"
"ldr d11, [a_ptr5], #0x8\n"
"ldr d13, [a_ptr6], #0x8\n"
- "ldr d15, [a_ptr7], #0x8\n"
"ld1 {v1.s}[2], [%[a_ptr0]], #4\n"
+ "ldr d15, [a_ptr7], #0x8\n"
"ld1 {v3.s}[2], [a_ptr1], #4\n"
"ld1 {v5.s}[2], [a_ptr2], #4\n"
"ld1 {v7.s}[2], [a_ptr3], #4\n"
@@ -2678,7 +2892,6 @@
"ld1 {v11.s}[2], [a_ptr5], #4\n"
"ld1 {v13.s}[2], [a_ptr6], #4\n"
"ld1 {v15.s}[2], [a_ptr7], #4\n"
- "subs %[odds], %[odds], #0x1\n"
"b.ne 4f\n"
"ld1 {v1.b}[12], [%[a_ptr0]]\n"
"ld1 {v3.b}[12], [a_ptr1]\n"
@@ -2711,36 +2924,38 @@
"ld1 {v13.b}[14], [a_ptr6]\n"
"ld1 {v15.b}[14], [a_ptr7]\n"
"3:\n"
- "movi v24.4s, #0\n"
"ldr q16, [%[b_ptr0]]\n"
- "movi v25.4s, #0\n"
- "ldr q17, [%[b_ptr0], #0x10]\n"
- "movi v26.4s, #0\n"
- "ldr q18, [%[b_ptr0], #0x20]\n"
- "movi v27.4s, #0\n"
- "ldr q19, [%[b_ptr0], #0x30]\n"
- "movi v28.4s, #0\n"
- "ldr q20, [%[b_ptr0], #0x40]\n"
- "movi v29.4s, #0\n"
- "ldr q21, [%[b_ptr0], #0x50]\n"
- "movi v30.4s, #0\n"
- "ldr q22, [%[b_ptr0], #0x60]\n"
- "movi v31.4s, #0\n"
- "ldr q23, [%[b_ptr0], #0x70]\n"
- ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
- ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
- ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
+ "ldr q18, [%[b_ptr0], #0x20]\n"
"prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
- ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
+ "ldr q19, [%[b_ptr0], #0x30]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
- ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
+ "ldr q20, [%[b_ptr0], #0x40]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
+ "ldr q21, [%[b_ptr0], #0x50]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
+ "ldr q22, [%[b_ptr0], #0x60]\n"
+ "ldr q23, [%[b_ptr0], #0x70]\n"
"add %[b_ptr0], %[b_ptr0], #0x80\n"
+ "cbz %[loops], 6f\n"
+ "movi v24.4s, #0\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "movi v25.4s, #0\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
+ ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
+ ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
@@ -2798,192 +3013,248 @@
".inst 0x4fabeafd // sdot v29.4s, v23.16b, v11.4b[3]\n"
".inst 0x4fadeafe // sdot v30.4s, v23.16b, v13.4b[3]\n"
".inst 0x4fafeaff // sdot v31.4s, v23.16b, v15.4b[3]\n"
- "cbz %[loops], 6f\n"
- "ldr d16, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- "ldr d17, [%[b_ptr0], #0x10]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- "ldr d18, [%[b_ptr0], #0x20]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- "ldr d19, [%[b_ptr0], #0x30]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- "ldr d20, [%[b_ptr0], #0x40]\n"
- "ins v16.d[1], temploadreg0\n"
- "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- "ldr d21, [%[b_ptr0], #0x50]\n"
- "ins v17.d[1], temploadreg1\n"
- "ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- "ldr d22, [%[b_ptr0], #0x60]\n"
- "ins v18.d[1], temploadreg2\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- "ldr d23, [%[b_ptr0], #0x70]\n"
- "ins v19.d[1], temploadreg3\n"
- "ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "ins v20.d[1], temploadreg0\n"
- "ins v21.d[1], temploadreg1\n"
- "ins v22.d[1], temploadreg2\n"
"b.eq 7f\n"
"8:\n"
"str q24, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"movi v24.4s, #0\n"
- "ins v23.d[1], temploadreg3\n"
+ "ldr d16, [%[b_ptr0]]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q25, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
+ "ldr d17, [%[b_ptr0], #0x10]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
"str q26, [c_ptr2]\n"
- "movi v26.4s, #0\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
"add c_ptr2, c_ptr2, #0x10\n"
- ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
+ "movi v26.4s, #0\n"
+ "ldr d18, [%[b_ptr0], #0x20]\n"
+ "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
"str q27, [c_ptr3]\n"
- "movi v27.4s, #0\n"
"add c_ptr3, c_ptr3, #0x10\n"
- ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
+ "movi v27.4s, #0\n"
+ "ldr d19, [%[b_ptr0], #0x30]\n"
+ "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
"str q28, [c_ptr4]\n"
- "movi v28.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
- ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
+ "movi v28.4s, #0\n"
+ "ldr d20, [%[b_ptr0], #0x40]\n"
+ "ins v16.d[1], temploadreg0\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
"str q29, [c_ptr5]\n"
- "movi v29.4s, #0\n"
"add c_ptr5, c_ptr5, #0x10\n"
- ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
+ "movi v29.4s, #0\n"
+ "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
+ ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
"str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
- "add c_ptr6, c_ptr6, #0x10\n"
- ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
+ "ldr d21, [%[b_ptr0], #0x50]\n"
+ ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
+ "ins v17.d[1], temploadreg1\n"
+ ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
"str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
- "add c_ptr7, c_ptr7, #0x10\n"
- ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
- ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
- "ldr d16, [%[b_ptr0]]\n"
- ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
- "ins v16.d[1], temploadreg0\n"
- ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n"
- "ldr d17, [%[b_ptr0], #0x10]\n"
- ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
- "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
- ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n"
- "ins v17.d[1], temploadreg1\n"
- ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n"
"ldr temploadreg1, [%[b_ptr0], #0x58]\n"
+ ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
+ "ldr d22, [%[b_ptr0], #0x60]\n"
+ ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
+ "ins v18.d[1], temploadreg2\n"
+ ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
+ "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
+ ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
+ "ldr d23, [%[b_ptr0], #0x70]\n"
+ ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
+ "ins v19.d[1], temploadreg3\n"
+ ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
+ "ldr temploadreg3, [%[b_ptr0], #0x78]\n"
+ ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
+ "ins v20.d[1], temploadreg0\n"
+ ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
+ "ins v21.d[1], temploadreg1\n"
+ ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
+ "ins v22.d[1], temploadreg2\n"
+ ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n"
+ "ins v23.d[1], temploadreg3\n"
+ ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
+ ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n"
+ "add c_ptr7, c_ptr7, #0x10\n"
+ ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
+ ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
+ "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
+ ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n"
+ "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
+ ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n"
+ "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n"
"prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n"
".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n"
".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n"
".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n"
- "ldr d18, [%[b_ptr0], #0x20]\n"
".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n"
".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n"
".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n"
- "ins v18.d[1], temploadreg2\n"
".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n"
".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n"
".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n"
".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n"
- "ldr d19, [%[b_ptr0], #0x30]\n"
".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n"
".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n"
".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n"
- "ins v19.d[1], temploadreg3\n"
".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x78]\n"
".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n"
".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n"
".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n"
".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n"
- "ldr d20, [%[b_ptr0], #0x40]\n"
".inst 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n"
".inst 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n"
".inst 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n"
- "ins v20.d[1], temploadreg0\n"
".inst 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n"
".inst 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n"
".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n"
".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n"
".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n"
- "ldr d21, [%[b_ptr0], #0x50]\n"
".inst 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n"
".inst 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n"
".inst 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n"
- "ins v21.d[1], temploadreg1\n"
".inst 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n"
".inst 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n"
".inst 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n"
".inst 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n"
".inst 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n"
- "ldr d22, [%[b_ptr0], #0x60]\n"
".inst 0x4fa1eaf8 // sdot v24.4s, v23.16b, v1.4b[3]\n"
".inst 0x4fa3eaf9 // sdot v25.4s, v23.16b, v3.4b[3]\n"
".inst 0x4fa5eafa // sdot v26.4s, v23.16b, v5.4b[3]\n"
- "ins v22.d[1], temploadreg2\n"
".inst 0x4fa7eafb // sdot v27.4s, v23.16b, v7.4b[3]\n"
".inst 0x4fa9eafc // sdot v28.4s, v23.16b, v9.4b[3]\n"
".inst 0x4fabeafd // sdot v29.4s, v23.16b, v11.4b[3]\n"
".inst 0x4fadeafe // sdot v30.4s, v23.16b, v13.4b[3]\n"
".inst 0x4fafeaff // sdot v31.4s, v23.16b, v15.4b[3]\n"
- "ldr d23, [%[b_ptr0], #0x70]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
"b.ne 8b\n"
"7:\n"
"str q24, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v24.4s, #0\n"
- "ins v23.d[1], temploadreg3\n"
+ "ldr q16, [%[b_ptr0]]\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
"str q25, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
+ "ldr q18, [%[b_ptr0], #0x20]\n"
".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
"str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
+ "ldr q19, [%[b_ptr0], #0x30]\n"
+ "ldr q20, [%[b_ptr0], #0x40]\n"
"add c_ptr2, c_ptr2, #0x10\n"
".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
"str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
- "add c_ptr3, c_ptr3, #0x10\n"
+ "ldr q21, [%[b_ptr0], #0x50]\n"
".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
+ "ldr q22, [%[b_ptr0], #0x60]\n"
+ ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
"str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
- "add c_ptr4, c_ptr4, #0x10\n"
+ "ldr q23, [%[b_ptr0], #0x70]\n"
".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
"str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
- "add c_ptr5, c_ptr5, #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
"str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
- "add c_ptr6, c_ptr6, #0x10\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
"str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
- "add c_ptr7, c_ptr7, #0x10\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
- ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
+ "add c_ptr7, c_ptr7, #0x10\n"
".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
+ ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n"
+ ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n"
+ ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n"
+ ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n"
+ ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n"
+ ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n"
+ ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n"
+ ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n"
+ ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n"
+ ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n"
+ ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n"
+ ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n"
+ ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n"
+ ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n"
+ ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n"
+ ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n"
+ ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n"
+ ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n"
+ ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n"
+ ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n"
+ ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n"
+ ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n"
+ ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n"
+ ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n"
+ ".inst 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n"
+ ".inst 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n"
+ ".inst 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n"
+ ".inst 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n"
+ ".inst 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n"
+ ".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n"
+ ".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n"
+ ".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n"
+ ".inst 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n"
+ ".inst 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n"
+ ".inst 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n"
+ ".inst 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n"
+ ".inst 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n"
+ ".inst 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n"
+ ".inst 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n"
+ ".inst 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n"
+ ".inst 0x4fa1eaf8 // sdot v24.4s, v23.16b, v1.4b[3]\n"
+ ".inst 0x4fa3eaf9 // sdot v25.4s, v23.16b, v3.4b[3]\n"
+ ".inst 0x4fa5eafa // sdot v26.4s, v23.16b, v5.4b[3]\n"
+ ".inst 0x4fa7eafb // sdot v27.4s, v23.16b, v7.4b[3]\n"
+ ".inst 0x4fa9eafc // sdot v28.4s, v23.16b, v9.4b[3]\n"
+ ".inst 0x4fabeafd // sdot v29.4s, v23.16b, v11.4b[3]\n"
+ ".inst 0x4fadeafe // sdot v30.4s, v23.16b, v13.4b[3]\n"
+ ".inst 0x4fafeaff // sdot v31.4s, v23.16b, v15.4b[3]\n"
+ "b 9f\n"
+ "6:\n"
+ "movi v24.4s, #0\n"
+ "movi v25.4s, #0\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
+ ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
+ ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
+ ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
+ ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
@@ -3039,23 +3310,16 @@
".inst 0x4fabeafd // sdot v29.4s, v23.16b, v11.4b[3]\n"
".inst 0x4fadeafe // sdot v30.4s, v23.16b, v13.4b[3]\n"
".inst 0x4fafeaff // sdot v31.4s, v23.16b, v15.4b[3]\n"
- "6:\n"
+ "9:\n"
"str q24, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q25, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, #0x10\n"
"str q26, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, #0x10\n"
"str q27, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, #0x10\n"
"str q28, [c_ptr4]\n"
- "add c_ptr4, c_ptr4, #0x10\n"
"str q29, [c_ptr5]\n"
- "add c_ptr5, c_ptr5, #0x10\n"
"str q30, [c_ptr6]\n"
- "add c_ptr6, c_ptr6, #0x10\n"
"str q31, [c_ptr7]\n"
- "add c_ptr7, c_ptr7, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x8/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_8x4/generic.cpp
similarity index 85%
rename from src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x8/generic.cpp
rename to src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_8x4/generic.cpp
index c94e975..7fcf853 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_4x8/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_s8s32_dot_8x4/generic.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2019 Arm Limited.
+ * Copyright (c) 2019-2020 Arm Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -33,7 +33,7 @@
namespace arm_gemm {
-void a64_smallK_hybrid_s8s32_dot_4x8(const int8_t *A, int lda, const int8_t *B, int32_t *C, int ldc, int M, int N, int K, const int32_t *, Activation, bool) {
+void a64_smallK_hybrid_s8s32_dot_8x4(const int8_t *A, int lda, const int8_t *B, int32_t *C, int ldc, int M, int N, int K, const int32_t *, Activation, bool) {
const long loops_count = iceildiv(N, (int)4) - 1;
const long ldab = lda * sizeof(int8_t);
const long ldcb = ldc * sizeof(int32_t);
@@ -153,22 +153,24 @@
"ld1 {v6.b}[2], [a_ptr6]\n"
"ld1 {v7.b}[2], [a_ptr7]\n"
"3:\n"
- "movi v24.4s, #0\n"
"ldr q16, [%[b_ptr0]]\n"
- "movi v25.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
- "movi v26.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
- "movi v27.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
- "movi v28.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
- "movi v29.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- "movi v30.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- "movi v31.4s, #0\n"
"add %[b_ptr0], %[b_ptr0], #0x10\n"
+ "cbz %[loops], 6f\n"
+ "movi v24.4s, #0\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "movi v25.4s, #0\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n"
".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n"
@@ -177,20 +179,17 @@
".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n"
".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n"
".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n"
- "cbz %[loops], 6f\n"
- "ldr q16, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "add %[b_ptr0], %[b_ptr0], #0x10\n"
"b.eq 7f\n"
"8:\n"
"str q24, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"movi v24.4s, #0\n"
+ "ldr q16, [%[b_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q25, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x10\n"
".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
"str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
@@ -216,10 +215,9 @@
"movi v31.4s, #0\n"
"add c_ptr7, c_ptr7, #0x10\n"
".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n"
- "ldr q16, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], #0x10\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
"prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
"prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
"prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
@@ -231,6 +229,8 @@
"str q24, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v24.4s, #0\n"
+ "ldr q16, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x10\n"
"str q25, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
@@ -260,23 +260,34 @@
"add c_ptr7, c_ptr7, #0x10\n"
".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n"
".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n"
+ "b 9f\n"
"6:\n"
+ "movi v24.4s, #0\n"
+ "movi v25.4s, #0\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n"
+ ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n"
+ ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n"
+ ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n"
+ "9:\n"
"str q24, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q25, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, #0x10\n"
"str q26, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, #0x10\n"
"str q27, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, #0x10\n"
"str q28, [c_ptr4]\n"
- "add c_ptr4, c_ptr4, #0x10\n"
"str q29, [c_ptr5]\n"
- "add c_ptr5, c_ptr5, #0x10\n"
"str q30, [c_ptr6]\n"
- "add c_ptr6, c_ptr6, #0x10\n"
"str q31, [c_ptr7]\n"
- "add c_ptr7, c_ptr7, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -407,24 +418,26 @@
"ld1 {v6.b}[6], [a_ptr6]\n"
"ld1 {v7.b}[6], [a_ptr7]\n"
"3:\n"
- "movi v24.4s, #0\n"
"ldr q16, [%[b_ptr0]]\n"
- "movi v25.4s, #0\n"
- "ldr q17, [%[b_ptr0], #0x10]\n"
- "movi v26.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
- "movi v27.4s, #0\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
- "movi v28.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
- "movi v29.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
- "movi v30.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- "movi v31.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "cbz %[loops], 6f\n"
+ "movi v24.4s, #0\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "movi v25.4s, #0\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n"
".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n"
".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n"
@@ -440,68 +453,66 @@
".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n"
".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n"
".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n"
- "cbz %[loops], 6f\n"
- "ldr q16, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
"b.eq 7f\n"
"8:\n"
"str q24, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"movi v24.4s, #0\n"
+ "ldr q16, [%[b_ptr0]]\n"
"ldr q17, [%[b_ptr0], #0x10]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- "str q25, [c_ptr1]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
- "movi v25.4s, #0\n"
+ "str q25, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
+ "movi v25.4s, #0\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
"str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n"
"str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
- "add c_ptr2, c_ptr2, #0x10\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n"
"str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
- "add c_ptr3, c_ptr3, #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n"
"str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
- "add c_ptr4, c_ptr4, #0x10\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n"
"str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
- "add c_ptr5, c_ptr5, #0x10\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n"
"str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
- "add c_ptr6, c_ptr6, #0x10\n"
- ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n"
"add c_ptr7, c_ptr7, #0x10\n"
+ ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n"
- "ldr q16, [%[b_ptr0]]\n"
- ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n"
+ ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n"
+ ".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- ".inst 0x4fa4e23c // sdot v28.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- ".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n"
+ ".inst 0x4fa4e23c // sdot v28.4s, v17.16b, v4.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
- ".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
+ ".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n"
".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n"
"b.ne 8b\n"
"7:\n"
"str q24, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v24.4s, #0\n"
+ "ldr q16, [%[b_ptr0]]\n"
"ldr q17, [%[b_ptr0], #0x10]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
"str q25, [c_ptr1]\n"
@@ -541,23 +552,42 @@
".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n"
".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n"
".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n"
+ "b 9f\n"
"6:\n"
+ "movi v24.4s, #0\n"
+ "movi v25.4s, #0\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n"
+ ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n"
+ ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n"
+ ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n"
+ ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n"
+ ".inst 0x4fa4e23c // sdot v28.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n"
+ ".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n"
+ "9:\n"
"str q24, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q25, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, #0x10\n"
"str q26, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, #0x10\n"
"str q27, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, #0x10\n"
"str q28, [c_ptr4]\n"
- "add c_ptr4, c_ptr4, #0x10\n"
"str q29, [c_ptr5]\n"
- "add c_ptr5, c_ptr5, #0x10\n"
"str q30, [c_ptr6]\n"
- "add c_ptr6, c_ptr6, #0x10\n"
"str q31, [c_ptr7]\n"
- "add c_ptr7, c_ptr7, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -688,26 +718,28 @@
"ld1 {v6.b}[10], [a_ptr6]\n"
"ld1 {v7.b}[10], [a_ptr7]\n"
"3:\n"
- "movi v24.4s, #0\n"
"ldr q16, [%[b_ptr0]]\n"
- "movi v25.4s, #0\n"
- "ldr q17, [%[b_ptr0], #0x10]\n"
- "movi v26.4s, #0\n"
- "ldr q18, [%[b_ptr0], #0x20]\n"
- "movi v27.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
- "movi v28.4s, #0\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
- "movi v29.4s, #0\n"
+ "ldr q18, [%[b_ptr0], #0x20]\n"
"prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
- "movi v30.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
- "movi v31.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x30\n"
+ "cbz %[loops], 6f\n"
+ "movi v24.4s, #0\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "movi v25.4s, #0\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n"
".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n"
".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n"
".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n"
@@ -730,49 +762,46 @@
".inst 0x4f85ea5d // sdot v29.4s, v18.16b, v5.4b[2]\n"
".inst 0x4f86ea5e // sdot v30.4s, v18.16b, v6.4b[2]\n"
".inst 0x4f87ea5f // sdot v31.4s, v18.16b, v7.4b[2]\n"
- "cbz %[loops], 6f\n"
- "ldr q16, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ldr q17, [%[b_ptr0], #0x10]\n"
"b.eq 7f\n"
"8:\n"
"str q24, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"movi v24.4s, #0\n"
- "ldr q18, [%[b_ptr0], #0x20]\n"
- "add %[b_ptr0], %[b_ptr0], #0x30\n"
- "str q25, [c_ptr1]\n"
+ "ldr q16, [%[b_ptr0]]\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
- "movi v25.4s, #0\n"
+ "str q25, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
+ "movi v25.4s, #0\n"
+ "ldr q18, [%[b_ptr0], #0x20]\n"
".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
"str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n"
"str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
- "add c_ptr2, c_ptr2, #0x10\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n"
"str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
- "add c_ptr3, c_ptr3, #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n"
"str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
- "add c_ptr4, c_ptr4, #0x10\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n"
"str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
- "add c_ptr5, c_ptr5, #0x10\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n"
"str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
- "add c_ptr6, c_ptr6, #0x10\n"
- ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n"
"add c_ptr7, c_ptr7, #0x10\n"
+ ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x30\n"
".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n"
- "ldr q16, [%[b_ptr0]]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n"
@@ -788,7 +817,6 @@
".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n"
- "ldr q17, [%[b_ptr0], #0x10]\n"
".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
".inst 0x4f81ea59 // sdot v25.4s, v18.16b, v1.4b[2]\n"
".inst 0x4f82ea5a // sdot v26.4s, v18.16b, v2.4b[2]\n"
@@ -802,11 +830,12 @@
"str q24, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v24.4s, #0\n"
- "ldr q18, [%[b_ptr0], #0x20]\n"
- "add %[b_ptr0], %[b_ptr0], #0x30\n"
+ "ldr q16, [%[b_ptr0]]\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
"str q25, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
+ "ldr q18, [%[b_ptr0], #0x20]\n"
".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
"str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
@@ -832,8 +861,9 @@
"movi v31.4s, #0\n"
"add c_ptr7, c_ptr7, #0x10\n"
".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n"
- ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x30\n"
".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n"
+ ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n"
".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n"
".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n"
@@ -849,23 +879,50 @@
".inst 0x4f85ea5d // sdot v29.4s, v18.16b, v5.4b[2]\n"
".inst 0x4f86ea5e // sdot v30.4s, v18.16b, v6.4b[2]\n"
".inst 0x4f87ea5f // sdot v31.4s, v18.16b, v7.4b[2]\n"
+ "b 9f\n"
"6:\n"
+ "movi v24.4s, #0\n"
+ "movi v25.4s, #0\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n"
+ ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n"
+ ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n"
+ ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n"
+ ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n"
+ ".inst 0x4fa4e23c // sdot v28.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n"
+ ".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n"
+ ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x4f81ea59 // sdot v25.4s, v18.16b, v1.4b[2]\n"
+ ".inst 0x4f82ea5a // sdot v26.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x4f83ea5b // sdot v27.4s, v18.16b, v3.4b[2]\n"
+ ".inst 0x4f84ea5c // sdot v28.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x4f85ea5d // sdot v29.4s, v18.16b, v5.4b[2]\n"
+ ".inst 0x4f86ea5e // sdot v30.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x4f87ea5f // sdot v31.4s, v18.16b, v7.4b[2]\n"
+ "9:\n"
"str q24, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q25, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, #0x10\n"
"str q26, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, #0x10\n"
"str q27, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, #0x10\n"
"str q28, [c_ptr4]\n"
- "add c_ptr4, c_ptr4, #0x10\n"
"str q29, [c_ptr5]\n"
- "add c_ptr5, c_ptr5, #0x10\n"
"str q30, [c_ptr6]\n"
- "add c_ptr6, c_ptr6, #0x10\n"
"str q31, [c_ptr7]\n"
- "add c_ptr7, c_ptr7, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -1004,28 +1061,30 @@
"ld1 {v6.b}[14], [a_ptr6]\n"
"ld1 {v7.b}[14], [a_ptr7]\n"
"3:\n"
- "movi v24.4s, #0\n"
"ldr q16, [%[b_ptr0]]\n"
- "movi v25.4s, #0\n"
- "ldr q17, [%[b_ptr0], #0x10]\n"
- "movi v26.4s, #0\n"
- "ldr q18, [%[b_ptr0], #0x20]\n"
- "movi v27.4s, #0\n"
- "ldr q19, [%[b_ptr0], #0x30]\n"
- "movi v28.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
- "movi v29.4s, #0\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
- "movi v30.4s, #0\n"
+ "ldr q18, [%[b_ptr0], #0x20]\n"
"prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
- "movi v31.4s, #0\n"
+ "ldr q19, [%[b_ptr0], #0x30]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
- ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x40\n"
+ "cbz %[loops], 6f\n"
+ "movi v24.4s, #0\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "movi v25.4s, #0\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n"
+ ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n"
".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n"
".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n"
".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n"
@@ -1055,50 +1114,47 @@
".inst 0x4fa5ea7d // sdot v29.4s, v19.16b, v5.4b[3]\n"
".inst 0x4fa6ea7e // sdot v30.4s, v19.16b, v6.4b[3]\n"
".inst 0x4fa7ea7f // sdot v31.4s, v19.16b, v7.4b[3]\n"
- "cbz %[loops], 6f\n"
- "ldr q16, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ldr q17, [%[b_ptr0], #0x10]\n"
- "ldr q18, [%[b_ptr0], #0x20]\n"
"b.eq 7f\n"
"8:\n"
"str q24, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"movi v24.4s, #0\n"
- "ldr q19, [%[b_ptr0], #0x30]\n"
- "add %[b_ptr0], %[b_ptr0], #0x40\n"
- "str q25, [c_ptr1]\n"
+ "ldr q16, [%[b_ptr0]]\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
- "movi v25.4s, #0\n"
+ "str q25, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
+ "movi v25.4s, #0\n"
+ "ldr q18, [%[b_ptr0], #0x20]\n"
".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
"str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "ldr q19, [%[b_ptr0], #0x30]\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n"
"str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
- "add c_ptr2, c_ptr2, #0x10\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n"
"str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
- "add c_ptr3, c_ptr3, #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n"
"str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
- "add c_ptr4, c_ptr4, #0x10\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n"
"str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
- "add c_ptr5, c_ptr5, #0x10\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n"
"str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
- "add c_ptr6, c_ptr6, #0x10\n"
- ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n"
"add c_ptr7, c_ptr7, #0x10\n"
+ ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x40\n"
".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n"
- "ldr q16, [%[b_ptr0]]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n"
@@ -1114,7 +1170,6 @@
".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n"
- "ldr q17, [%[b_ptr0], #0x10]\n"
".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
".inst 0x4f81ea59 // sdot v25.4s, v18.16b, v1.4b[2]\n"
".inst 0x4f82ea5a // sdot v26.4s, v18.16b, v2.4b[2]\n"
@@ -1123,7 +1178,6 @@
".inst 0x4f85ea5d // sdot v29.4s, v18.16b, v5.4b[2]\n"
".inst 0x4f86ea5e // sdot v30.4s, v18.16b, v6.4b[2]\n"
".inst 0x4f87ea5f // sdot v31.4s, v18.16b, v7.4b[2]\n"
- "ldr q18, [%[b_ptr0], #0x20]\n"
".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n"
".inst 0x4fa1ea79 // sdot v25.4s, v19.16b, v1.4b[3]\n"
".inst 0x4fa2ea7a // sdot v26.4s, v19.16b, v2.4b[3]\n"
@@ -1137,14 +1191,16 @@
"str q24, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v24.4s, #0\n"
- "ldr q19, [%[b_ptr0], #0x30]\n"
- "add %[b_ptr0], %[b_ptr0], #0x40\n"
+ "ldr q16, [%[b_ptr0]]\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
"str q25, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
+ "ldr q18, [%[b_ptr0], #0x20]\n"
".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
"str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
+ "ldr q19, [%[b_ptr0], #0x30]\n"
"add c_ptr2, c_ptr2, #0x10\n"
".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n"
"str q27, [c_ptr3]\n"
@@ -1167,8 +1223,9 @@
"movi v31.4s, #0\n"
"add c_ptr7, c_ptr7, #0x10\n"
".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n"
- ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x40\n"
".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n"
+ ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n"
".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n"
".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n"
@@ -1192,23 +1249,58 @@
".inst 0x4fa5ea7d // sdot v29.4s, v19.16b, v5.4b[3]\n"
".inst 0x4fa6ea7e // sdot v30.4s, v19.16b, v6.4b[3]\n"
".inst 0x4fa7ea7f // sdot v31.4s, v19.16b, v7.4b[3]\n"
+ "b 9f\n"
"6:\n"
+ "movi v24.4s, #0\n"
+ "movi v25.4s, #0\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x4f81e219 // sdot v25.4s, v16.16b, v1.4b[0]\n"
+ ".inst 0x4f82e21a // sdot v26.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x4f83e21b // sdot v27.4s, v16.16b, v3.4b[0]\n"
+ ".inst 0x4f84e21c // sdot v28.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x4f85e21d // sdot v29.4s, v16.16b, v5.4b[0]\n"
+ ".inst 0x4f86e21e // sdot v30.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x4f87e21f // sdot v31.4s, v16.16b, v7.4b[0]\n"
+ ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x4fa1e239 // sdot v25.4s, v17.16b, v1.4b[1]\n"
+ ".inst 0x4fa2e23a // sdot v26.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x4fa3e23b // sdot v27.4s, v17.16b, v3.4b[1]\n"
+ ".inst 0x4fa4e23c // sdot v28.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x4fa5e23d // sdot v29.4s, v17.16b, v5.4b[1]\n"
+ ".inst 0x4fa6e23e // sdot v30.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x4fa7e23f // sdot v31.4s, v17.16b, v7.4b[1]\n"
+ ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x4f81ea59 // sdot v25.4s, v18.16b, v1.4b[2]\n"
+ ".inst 0x4f82ea5a // sdot v26.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x4f83ea5b // sdot v27.4s, v18.16b, v3.4b[2]\n"
+ ".inst 0x4f84ea5c // sdot v28.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x4f85ea5d // sdot v29.4s, v18.16b, v5.4b[2]\n"
+ ".inst 0x4f86ea5e // sdot v30.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x4f87ea5f // sdot v31.4s, v18.16b, v7.4b[2]\n"
+ ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n"
+ ".inst 0x4fa1ea79 // sdot v25.4s, v19.16b, v1.4b[3]\n"
+ ".inst 0x4fa2ea7a // sdot v26.4s, v19.16b, v2.4b[3]\n"
+ ".inst 0x4fa3ea7b // sdot v27.4s, v19.16b, v3.4b[3]\n"
+ ".inst 0x4fa4ea7c // sdot v28.4s, v19.16b, v4.4b[3]\n"
+ ".inst 0x4fa5ea7d // sdot v29.4s, v19.16b, v5.4b[3]\n"
+ ".inst 0x4fa6ea7e // sdot v30.4s, v19.16b, v6.4b[3]\n"
+ ".inst 0x4fa7ea7f // sdot v31.4s, v19.16b, v7.4b[3]\n"
+ "9:\n"
"str q24, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q25, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, #0x10\n"
"str q26, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, #0x10\n"
"str q27, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, #0x10\n"
"str q28, [c_ptr4]\n"
- "add c_ptr4, c_ptr4, #0x10\n"
"str q29, [c_ptr5]\n"
- "add c_ptr5, c_ptr5, #0x10\n"
"str q30, [c_ptr6]\n"
- "add c_ptr6, c_ptr6, #0x10\n"
"str q31, [c_ptr7]\n"
- "add c_ptr7, c_ptr7, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -1287,26 +1379,34 @@
"add c_ptr1, %[c_ptr0], #0x0\n"
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
+ "cbnz %[odds], 2f\n"
"ldr q0, [%[a_ptr0]], #0x10\n"
"ldr q2, [a_ptr1], #0x10\n"
"ldr q4, [a_ptr2], #0x10\n"
"ldr q6, [a_ptr3], #0x10\n"
+ "ldr s1, [%[a_ptr0]]\n"
+ "ldr q8, [a_ptr4], #0x10\n"
+ "ldr s3, [a_ptr1]\n"
+ "ldr q10, [a_ptr5], #0x10\n"
+ "ldr s5, [a_ptr2]\n"
+ "ldr q12, [a_ptr6], #0x10\n"
+ "ldr s7, [a_ptr3]\n"
+ "ldr q14, [a_ptr7], #0x10\n"
+ "ldr s9, [a_ptr4]\n"
+ "ldr s11, [a_ptr5]\n"
+ "ldr s13, [a_ptr6]\n"
+ "ldr s15, [a_ptr7]\n"
+ "b 3f\n"
+ "2:\n"
+ "ldr q0, [%[a_ptr0]], #0x10\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "ldr q2, [a_ptr1], #0x10\n"
+ "ldr q4, [a_ptr2], #0x10\n"
+ "ldr q6, [a_ptr3], #0x10\n"
"ldr q8, [a_ptr4], #0x10\n"
"ldr q10, [a_ptr5], #0x10\n"
"ldr q12, [a_ptr6], #0x10\n"
"ldr q14, [a_ptr7], #0x10\n"
- "cbnz %[odds], 2f\n"
- "ldr s1, [%[a_ptr0]]\n"
- "ldr s3, [a_ptr1]\n"
- "ldr s5, [a_ptr2]\n"
- "ldr s7, [a_ptr3]\n"
- "ldr s9, [a_ptr4]\n"
- "ldr s11, [a_ptr5]\n"
- "ldr s13, [a_ptr6]\n"
- "ldr s15, [a_ptr7]\n"
- "b 3f\n"
- "2:\n"
- "subs %[odds], %[odds], #0x1\n"
"b.ne 4f\n"
"ldr b1, [%[a_ptr0]]\n"
"ldr b3, [a_ptr1]\n"
@@ -1339,30 +1439,32 @@
"ld1 {v13.b}[2], [a_ptr6]\n"
"ld1 {v15.b}[2], [a_ptr7]\n"
"3:\n"
- "movi v24.4s, #0\n"
"ldr q16, [%[b_ptr0]]\n"
- "movi v25.4s, #0\n"
- "ldr q17, [%[b_ptr0], #0x10]\n"
- "movi v26.4s, #0\n"
- "ldr q18, [%[b_ptr0], #0x20]\n"
- "movi v27.4s, #0\n"
- "ldr q19, [%[b_ptr0], #0x30]\n"
- "movi v28.4s, #0\n"
- "ldr q20, [%[b_ptr0], #0x40]\n"
- "movi v29.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
- "movi v30.4s, #0\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
- "movi v31.4s, #0\n"
+ "ldr q18, [%[b_ptr0], #0x20]\n"
"prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
- ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ "ldr q19, [%[b_ptr0], #0x30]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
- ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
+ "ldr q20, [%[b_ptr0], #0x40]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x50\n"
+ "cbz %[loops], 6f\n"
+ "movi v24.4s, #0\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "movi v25.4s, #0\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
@@ -1399,51 +1501,48 @@
".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n"
".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n"
".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n"
- "cbz %[loops], 6f\n"
- "ldr q16, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ldr q17, [%[b_ptr0], #0x10]\n"
- "ldr q18, [%[b_ptr0], #0x20]\n"
- "ldr q19, [%[b_ptr0], #0x30]\n"
"b.eq 7f\n"
"8:\n"
"str q24, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"movi v24.4s, #0\n"
- "ldr q20, [%[b_ptr0], #0x40]\n"
- "add %[b_ptr0], %[b_ptr0], #0x50\n"
- "str q25, [c_ptr1]\n"
+ "ldr q16, [%[b_ptr0]]\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
- "movi v25.4s, #0\n"
+ "str q25, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
+ "movi v25.4s, #0\n"
+ "ldr q18, [%[b_ptr0], #0x20]\n"
".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
"str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "ldr q19, [%[b_ptr0], #0x30]\n"
+ "ldr q20, [%[b_ptr0], #0x40]\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
"str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
- "add c_ptr2, c_ptr2, #0x10\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
"str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
- "add c_ptr3, c_ptr3, #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
"str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
- "add c_ptr4, c_ptr4, #0x10\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
"str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
- "add c_ptr5, c_ptr5, #0x10\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
"str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
- "add c_ptr6, c_ptr6, #0x10\n"
- ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
"add c_ptr7, c_ptr7, #0x10\n"
+ ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x50\n"
".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
- "ldr q16, [%[b_ptr0]]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
@@ -1459,7 +1558,6 @@
".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n"
- "ldr q17, [%[b_ptr0], #0x10]\n"
".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n"
".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n"
@@ -1468,7 +1566,6 @@
".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n"
".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n"
".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n"
- "ldr q18, [%[b_ptr0], #0x20]\n"
".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n"
".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n"
".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n"
@@ -1477,7 +1574,6 @@
".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n"
".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n"
".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n"
- "ldr q19, [%[b_ptr0], #0x30]\n"
".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n"
".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n"
".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n"
@@ -1491,14 +1587,17 @@
"str q24, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v24.4s, #0\n"
- "ldr q20, [%[b_ptr0], #0x40]\n"
- "add %[b_ptr0], %[b_ptr0], #0x50\n"
+ "ldr q16, [%[b_ptr0]]\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
"str q25, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
+ "ldr q18, [%[b_ptr0], #0x20]\n"
".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
"str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
+ "ldr q19, [%[b_ptr0], #0x30]\n"
+ "ldr q20, [%[b_ptr0], #0x40]\n"
"add c_ptr2, c_ptr2, #0x10\n"
".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
"str q27, [c_ptr3]\n"
@@ -1521,8 +1620,9 @@
"movi v31.4s, #0\n"
"add c_ptr7, c_ptr7, #0x10\n"
".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
- ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x50\n"
".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
+ ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
@@ -1554,23 +1654,66 @@
".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n"
".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n"
".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n"
+ "b 9f\n"
"6:\n"
+ "movi v24.4s, #0\n"
+ "movi v25.4s, #0\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
+ ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
+ ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
+ ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
+ ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n"
+ ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n"
+ ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n"
+ ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n"
+ ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n"
+ ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n"
+ ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n"
+ ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n"
+ ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n"
+ ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n"
+ ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n"
+ ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n"
+ ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n"
+ ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n"
+ ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n"
+ ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n"
+ ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n"
+ ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n"
+ ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n"
+ ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n"
+ ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n"
+ ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n"
+ ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n"
+ ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n"
+ "9:\n"
"str q24, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q25, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, #0x10\n"
"str q26, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, #0x10\n"
"str q27, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, #0x10\n"
"str q28, [c_ptr4]\n"
- "add c_ptr4, c_ptr4, #0x10\n"
"str q29, [c_ptr5]\n"
- "add c_ptr5, c_ptr5, #0x10\n"
"str q30, [c_ptr6]\n"
- "add c_ptr6, c_ptr6, #0x10\n"
"str q31, [c_ptr7]\n"
- "add c_ptr7, c_ptr7, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -1649,34 +1792,42 @@
"add c_ptr1, %[c_ptr0], #0x0\n"
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
+ "cbnz %[odds], 2f\n"
"ldr q0, [%[a_ptr0]], #0x10\n"
"ldr q2, [a_ptr1], #0x10\n"
"ldr q4, [a_ptr2], #0x10\n"
"ldr q6, [a_ptr3], #0x10\n"
- "ldr q8, [a_ptr4], #0x10\n"
- "ldr q10, [a_ptr5], #0x10\n"
- "ldr q12, [a_ptr6], #0x10\n"
- "ldr q14, [a_ptr7], #0x10\n"
- "cbnz %[odds], 2f\n"
"ldr d1, [%[a_ptr0]]\n"
+ "ldr q8, [a_ptr4], #0x10\n"
"ldr d3, [a_ptr1]\n"
+ "ldr q10, [a_ptr5], #0x10\n"
"ldr d5, [a_ptr2]\n"
+ "ldr q12, [a_ptr6], #0x10\n"
"ldr d7, [a_ptr3]\n"
+ "ldr q14, [a_ptr7], #0x10\n"
"ldr d9, [a_ptr4]\n"
"ldr d11, [a_ptr5]\n"
"ldr d13, [a_ptr6]\n"
"ldr d15, [a_ptr7]\n"
"b 3f\n"
"2:\n"
+ "ldr q0, [%[a_ptr0]], #0x10\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "ldr q2, [a_ptr1], #0x10\n"
+ "ldr q4, [a_ptr2], #0x10\n"
"ldr s1, [%[a_ptr0]], #0x4\n"
+ "ldr q6, [a_ptr3], #0x10\n"
"ldr s3, [a_ptr1], #0x4\n"
+ "ldr q8, [a_ptr4], #0x10\n"
"ldr s5, [a_ptr2], #0x4\n"
+ "ldr q10, [a_ptr5], #0x10\n"
"ldr s7, [a_ptr3], #0x4\n"
+ "ldr q12, [a_ptr6], #0x10\n"
"ldr s9, [a_ptr4], #0x4\n"
+ "ldr q14, [a_ptr7], #0x10\n"
"ldr s11, [a_ptr5], #0x4\n"
"ldr s13, [a_ptr6], #0x4\n"
"ldr s15, [a_ptr7], #0x4\n"
- "subs %[odds], %[odds], #0x1\n"
"b.ne 4f\n"
"ld1 {v1.b}[4], [%[a_ptr0]]\n"
"ld1 {v3.b}[4], [a_ptr1]\n"
@@ -1709,32 +1860,34 @@
"ld1 {v13.b}[6], [a_ptr6]\n"
"ld1 {v15.b}[6], [a_ptr7]\n"
"3:\n"
- "movi v24.4s, #0\n"
"ldr q16, [%[b_ptr0]]\n"
- "movi v25.4s, #0\n"
- "ldr q17, [%[b_ptr0], #0x10]\n"
- "movi v26.4s, #0\n"
- "ldr q18, [%[b_ptr0], #0x20]\n"
- "movi v27.4s, #0\n"
- "ldr q19, [%[b_ptr0], #0x30]\n"
- "movi v28.4s, #0\n"
- "ldr q20, [%[b_ptr0], #0x40]\n"
- "movi v29.4s, #0\n"
- "ldr q21, [%[b_ptr0], #0x50]\n"
- "movi v30.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
- "movi v31.4s, #0\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
- ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ "ldr q18, [%[b_ptr0], #0x20]\n"
"prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
- ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
+ "ldr q19, [%[b_ptr0], #0x30]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
- ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
+ "ldr q20, [%[b_ptr0], #0x40]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
+ "ldr q21, [%[b_ptr0], #0x50]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x60\n"
+ "cbz %[loops], 6f\n"
+ "movi v24.4s, #0\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "movi v25.4s, #0\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
@@ -1778,68 +1931,64 @@
".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n"
".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n"
".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n"
- "cbz %[loops], 6f\n"
- "ldr q16, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ldr q17, [%[b_ptr0], #0x10]\n"
- "ldr q18, [%[b_ptr0], #0x20]\n"
- "ldr q19, [%[b_ptr0], #0x30]\n"
- "ldr q20, [%[b_ptr0], #0x40]\n"
"b.eq 7f\n"
"8:\n"
"str q24, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"movi v24.4s, #0\n"
- "ldr q21, [%[b_ptr0], #0x50]\n"
- "add %[b_ptr0], %[b_ptr0], #0x60\n"
- "str q25, [c_ptr1]\n"
+ "ldr q16, [%[b_ptr0]]\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
- "movi v25.4s, #0\n"
+ "str q25, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
+ "movi v25.4s, #0\n"
+ "ldr q18, [%[b_ptr0], #0x20]\n"
".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
"str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "ldr q19, [%[b_ptr0], #0x30]\n"
+ "ldr q20, [%[b_ptr0], #0x40]\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
"str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
- "add c_ptr2, c_ptr2, #0x10\n"
+ "ldr q21, [%[b_ptr0], #0x50]\n"
".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
"str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
- "add c_ptr3, c_ptr3, #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
"str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
- "add c_ptr4, c_ptr4, #0x10\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
"str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
- "add c_ptr5, c_ptr5, #0x10\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
"str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
- "add c_ptr6, c_ptr6, #0x10\n"
- ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
"add c_ptr7, c_ptr7, #0x10\n"
+ ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x60\n"
".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
- "ldr q16, [%[b_ptr0]]\n"
- ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
+ "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
+ "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
+ "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n"
- "ldr q17, [%[b_ptr0], #0x10]\n"
+ "prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n"
".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n"
@@ -1848,7 +1997,6 @@
".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n"
".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n"
".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n"
- "ldr q18, [%[b_ptr0], #0x20]\n"
".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n"
".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n"
".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n"
@@ -1857,7 +2005,6 @@
".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n"
".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n"
".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n"
- "ldr q19, [%[b_ptr0], #0x30]\n"
".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n"
".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n"
".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n"
@@ -1866,7 +2013,6 @@
".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n"
".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n"
".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n"
- "ldr q20, [%[b_ptr0], #0x40]\n"
".inst 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n"
".inst 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n"
".inst 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n"
@@ -1880,20 +2026,25 @@
"str q24, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v24.4s, #0\n"
- "ldr q21, [%[b_ptr0], #0x50]\n"
- "add %[b_ptr0], %[b_ptr0], #0x60\n"
+ "ldr q16, [%[b_ptr0]]\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
"str q25, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
+ "ldr q18, [%[b_ptr0], #0x20]\n"
".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
"str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
+ "ldr q19, [%[b_ptr0], #0x30]\n"
+ "ldr q20, [%[b_ptr0], #0x40]\n"
"add c_ptr2, c_ptr2, #0x10\n"
".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
"str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
- "add c_ptr3, c_ptr3, #0x10\n"
+ "ldr q21, [%[b_ptr0], #0x50]\n"
".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
"str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
@@ -1910,7 +2061,7 @@
"movi v31.4s, #0\n"
"add c_ptr7, c_ptr7, #0x10\n"
".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
- ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x60\n"
".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
@@ -1951,23 +2102,74 @@
".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n"
".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n"
".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n"
+ "b 9f\n"
"6:\n"
+ "movi v24.4s, #0\n"
+ "movi v25.4s, #0\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
+ ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
+ ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
+ ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
+ ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n"
+ ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n"
+ ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n"
+ ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n"
+ ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n"
+ ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n"
+ ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n"
+ ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n"
+ ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n"
+ ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n"
+ ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n"
+ ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n"
+ ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n"
+ ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n"
+ ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n"
+ ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n"
+ ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n"
+ ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n"
+ ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n"
+ ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n"
+ ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n"
+ ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n"
+ ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n"
+ ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n"
+ ".inst 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n"
+ ".inst 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n"
+ ".inst 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n"
+ ".inst 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n"
+ ".inst 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n"
+ ".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n"
+ ".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n"
+ ".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n"
+ "9:\n"
"str q24, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q25, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, #0x10\n"
"str q26, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, #0x10\n"
"str q27, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, #0x10\n"
"str q28, [c_ptr4]\n"
- "add c_ptr4, c_ptr4, #0x10\n"
"str q29, [c_ptr5]\n"
- "add c_ptr5, c_ptr5, #0x10\n"
"str q30, [c_ptr6]\n"
- "add c_ptr6, c_ptr6, #0x10\n"
"str q31, [c_ptr7]\n"
- "add c_ptr7, c_ptr7, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -2106,34 +2308,36 @@
"ld1 {v13.b}[10], [a_ptr6]\n"
"ld1 {v15.b}[10], [a_ptr7]\n"
"3:\n"
- "movi v24.4s, #0\n"
"ldr q16, [%[b_ptr0]]\n"
- "movi v25.4s, #0\n"
- "ldr q17, [%[b_ptr0], #0x10]\n"
- "movi v26.4s, #0\n"
- "ldr q18, [%[b_ptr0], #0x20]\n"
- "movi v27.4s, #0\n"
- "ldr q19, [%[b_ptr0], #0x30]\n"
- "movi v28.4s, #0\n"
- "ldr q20, [%[b_ptr0], #0x40]\n"
- "movi v29.4s, #0\n"
- "ldr q21, [%[b_ptr0], #0x50]\n"
- "movi v30.4s, #0\n"
- "ldr q22, [%[b_ptr0], #0x60]\n"
- "movi v31.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
- ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
- ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
+ "ldr q18, [%[b_ptr0], #0x20]\n"
"prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
- ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
+ "ldr q19, [%[b_ptr0], #0x30]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
- ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
+ "ldr q20, [%[b_ptr0], #0x40]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
+ "ldr q21, [%[b_ptr0], #0x50]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
+ "ldr q22, [%[b_ptr0], #0x60]\n"
"add %[b_ptr0], %[b_ptr0], #0x70\n"
+ "cbz %[loops], 6f\n"
+ "movi v24.4s, #0\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "movi v25.4s, #0\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
+ ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
@@ -2184,34 +2388,31 @@
".inst 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n"
".inst 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n"
".inst 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n"
- "cbz %[loops], 6f\n"
- "ldr q16, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ldr q17, [%[b_ptr0], #0x10]\n"
- "ldr q18, [%[b_ptr0], #0x20]\n"
- "ldr q19, [%[b_ptr0], #0x30]\n"
- "ldr q20, [%[b_ptr0], #0x40]\n"
- "ldr q21, [%[b_ptr0], #0x50]\n"
"b.eq 7f\n"
"8:\n"
"str q24, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"movi v24.4s, #0\n"
- "ldr q22, [%[b_ptr0], #0x60]\n"
- "add %[b_ptr0], %[b_ptr0], #0x70\n"
- "str q25, [c_ptr1]\n"
+ "ldr q16, [%[b_ptr0]]\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
- "movi v25.4s, #0\n"
+ "str q25, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
+ "movi v25.4s, #0\n"
+ "ldr q18, [%[b_ptr0], #0x20]\n"
".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
"str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "ldr q19, [%[b_ptr0], #0x30]\n"
+ "ldr q20, [%[b_ptr0], #0x40]\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
"str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
- "add c_ptr2, c_ptr2, #0x10\n"
+ "ldr q21, [%[b_ptr0], #0x50]\n"
".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
+ "ldr q22, [%[b_ptr0], #0x60]\n"
+ ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
"str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
"add c_ptr3, c_ptr3, #0x10\n"
@@ -2230,24 +2431,23 @@
".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
"add c_ptr7, c_ptr7, #0x10\n"
".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
- "ldr q16, [%[b_ptr0]]\n"
- ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x70\n"
".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
+ "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
+ "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n"
- "ldr q17, [%[b_ptr0], #0x10]\n"
+ "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
+ "prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n"
".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n"
".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n"
@@ -2255,7 +2455,6 @@
".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n"
".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n"
".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n"
- "ldr q18, [%[b_ptr0], #0x20]\n"
".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n"
".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n"
".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n"
@@ -2264,7 +2463,6 @@
".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n"
".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n"
".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n"
- "ldr q19, [%[b_ptr0], #0x30]\n"
".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n"
".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n"
".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n"
@@ -2273,7 +2471,6 @@
".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n"
".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n"
".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n"
- "ldr q20, [%[b_ptr0], #0x40]\n"
".inst 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n"
".inst 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n"
".inst 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n"
@@ -2282,7 +2479,6 @@
".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n"
".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n"
".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n"
- "ldr q21, [%[b_ptr0], #0x50]\n"
".inst 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n"
".inst 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n"
".inst 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n"
@@ -2296,38 +2492,44 @@
"str q24, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v24.4s, #0\n"
- "ldr q22, [%[b_ptr0], #0x60]\n"
- "add %[b_ptr0], %[b_ptr0], #0x70\n"
+ "ldr q16, [%[b_ptr0]]\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
"str q25, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
+ "ldr q18, [%[b_ptr0], #0x20]\n"
".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
"str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
+ "ldr q19, [%[b_ptr0], #0x30]\n"
+ "ldr q20, [%[b_ptr0], #0x40]\n"
"add c_ptr2, c_ptr2, #0x10\n"
".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
"str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
- "add c_ptr3, c_ptr3, #0x10\n"
+ "ldr q21, [%[b_ptr0], #0x50]\n"
".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
+ "ldr q22, [%[b_ptr0], #0x60]\n"
+ ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
"str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
- "add c_ptr4, c_ptr4, #0x10\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
"str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
- "add c_ptr5, c_ptr5, #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
"str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
- "add c_ptr6, c_ptr6, #0x10\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
"str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
- "add c_ptr7, c_ptr7, #0x10\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
- ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
+ "add c_ptr7, c_ptr7, #0x10\n"
".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x70\n"
".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
@@ -2375,23 +2577,82 @@
".inst 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n"
".inst 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n"
".inst 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n"
+ "b 9f\n"
"6:\n"
+ "movi v24.4s, #0\n"
+ "movi v25.4s, #0\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
+ ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
+ ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
+ ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
+ ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n"
+ ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n"
+ ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n"
+ ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n"
+ ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n"
+ ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n"
+ ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n"
+ ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n"
+ ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n"
+ ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n"
+ ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n"
+ ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n"
+ ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n"
+ ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n"
+ ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n"
+ ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n"
+ ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n"
+ ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n"
+ ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n"
+ ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n"
+ ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n"
+ ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n"
+ ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n"
+ ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n"
+ ".inst 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n"
+ ".inst 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n"
+ ".inst 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n"
+ ".inst 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n"
+ ".inst 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n"
+ ".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n"
+ ".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n"
+ ".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n"
+ ".inst 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n"
+ ".inst 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n"
+ ".inst 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n"
+ ".inst 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n"
+ ".inst 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n"
+ ".inst 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n"
+ ".inst 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n"
+ ".inst 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n"
+ "9:\n"
"str q24, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q25, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, #0x10\n"
"str q26, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, #0x10\n"
"str q27, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, #0x10\n"
"str q28, [c_ptr4]\n"
- "add c_ptr4, c_ptr4, #0x10\n"
"str q29, [c_ptr5]\n"
- "add c_ptr5, c_ptr5, #0x10\n"
"str q30, [c_ptr6]\n"
- "add c_ptr6, c_ptr6, #0x10\n"
"str q31, [c_ptr7]\n"
- "add c_ptr7, c_ptr7, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -2471,6 +2732,7 @@
"add c_ptr1, %[c_ptr0], #0x0\n"
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
+ "cbnz %[odds], 2f\n"
"ldr q0, [%[a_ptr0]], #0x10\n"
"ldr q2, [a_ptr1], #0x10\n"
"ldr q4, [a_ptr2], #0x10\n"
@@ -2479,7 +2741,6 @@
"ldr q10, [a_ptr5], #0x10\n"
"ldr q12, [a_ptr6], #0x10\n"
"ldr q14, [a_ptr7], #0x10\n"
- "cbnz %[odds], 2f\n"
"ldr q1, [%[a_ptr0]]\n"
"ldr q3, [a_ptr1]\n"
"ldr q5, [a_ptr2]\n"
@@ -2490,15 +2751,24 @@
"ldr q15, [a_ptr7]\n"
"b 3f\n"
"2:\n"
+ "ldr q0, [%[a_ptr0]], #0x10\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "ldr q2, [a_ptr1], #0x10\n"
+ "ldr q4, [a_ptr2], #0x10\n"
"ldr d1, [%[a_ptr0]], #0x8\n"
+ "ldr q6, [a_ptr3], #0x10\n"
"ldr d3, [a_ptr1], #0x8\n"
+ "ldr q8, [a_ptr4], #0x10\n"
"ldr d5, [a_ptr2], #0x8\n"
+ "ldr q10, [a_ptr5], #0x10\n"
"ldr d7, [a_ptr3], #0x8\n"
+ "ldr q12, [a_ptr6], #0x10\n"
"ldr d9, [a_ptr4], #0x8\n"
+ "ldr q14, [a_ptr7], #0x10\n"
"ldr d11, [a_ptr5], #0x8\n"
"ldr d13, [a_ptr6], #0x8\n"
- "ldr d15, [a_ptr7], #0x8\n"
"ld1 {v1.s}[2], [%[a_ptr0]], #4\n"
+ "ldr d15, [a_ptr7], #0x8\n"
"ld1 {v3.s}[2], [a_ptr1], #4\n"
"ld1 {v5.s}[2], [a_ptr2], #4\n"
"ld1 {v7.s}[2], [a_ptr3], #4\n"
@@ -2506,7 +2776,6 @@
"ld1 {v11.s}[2], [a_ptr5], #4\n"
"ld1 {v13.s}[2], [a_ptr6], #4\n"
"ld1 {v15.s}[2], [a_ptr7], #4\n"
- "subs %[odds], %[odds], #0x1\n"
"b.ne 4f\n"
"ld1 {v1.b}[12], [%[a_ptr0]]\n"
"ld1 {v3.b}[12], [a_ptr1]\n"
@@ -2539,36 +2808,38 @@
"ld1 {v13.b}[14], [a_ptr6]\n"
"ld1 {v15.b}[14], [a_ptr7]\n"
"3:\n"
- "movi v24.4s, #0\n"
"ldr q16, [%[b_ptr0]]\n"
- "movi v25.4s, #0\n"
- "ldr q17, [%[b_ptr0], #0x10]\n"
- "movi v26.4s, #0\n"
- "ldr q18, [%[b_ptr0], #0x20]\n"
- "movi v27.4s, #0\n"
- "ldr q19, [%[b_ptr0], #0x30]\n"
- "movi v28.4s, #0\n"
- "ldr q20, [%[b_ptr0], #0x40]\n"
- "movi v29.4s, #0\n"
- "ldr q21, [%[b_ptr0], #0x50]\n"
- "movi v30.4s, #0\n"
- "ldr q22, [%[b_ptr0], #0x60]\n"
- "movi v31.4s, #0\n"
- "ldr q23, [%[b_ptr0], #0x70]\n"
- ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
- ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
- ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
+ "ldr q18, [%[b_ptr0], #0x20]\n"
"prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
- ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
+ "ldr q19, [%[b_ptr0], #0x30]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
- ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
+ "ldr q20, [%[b_ptr0], #0x40]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
+ "ldr q21, [%[b_ptr0], #0x50]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
+ "ldr q22, [%[b_ptr0], #0x60]\n"
+ "ldr q23, [%[b_ptr0], #0x70]\n"
"add %[b_ptr0], %[b_ptr0], #0x80\n"
+ "cbz %[loops], 6f\n"
+ "movi v24.4s, #0\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "movi v25.4s, #0\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
+ ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
+ ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
@@ -2626,39 +2897,37 @@
".inst 0x4fabeafd // sdot v29.4s, v23.16b, v11.4b[3]\n"
".inst 0x4fadeafe // sdot v30.4s, v23.16b, v13.4b[3]\n"
".inst 0x4fafeaff // sdot v31.4s, v23.16b, v15.4b[3]\n"
- "cbz %[loops], 6f\n"
- "ldr q16, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ldr q17, [%[b_ptr0], #0x10]\n"
- "ldr q18, [%[b_ptr0], #0x20]\n"
- "ldr q19, [%[b_ptr0], #0x30]\n"
- "ldr q20, [%[b_ptr0], #0x40]\n"
- "ldr q21, [%[b_ptr0], #0x50]\n"
- "ldr q22, [%[b_ptr0], #0x60]\n"
"b.eq 7f\n"
"8:\n"
"str q24, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"movi v24.4s, #0\n"
- "ldr q23, [%[b_ptr0], #0x70]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "str q25, [c_ptr1]\n"
+ "ldr q16, [%[b_ptr0]]\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
- "movi v25.4s, #0\n"
+ "str q25, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
+ "movi v25.4s, #0\n"
+ "ldr q18, [%[b_ptr0], #0x20]\n"
".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
"str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "ldr q19, [%[b_ptr0], #0x30]\n"
+ "ldr q20, [%[b_ptr0], #0x40]\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
"str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
- "add c_ptr2, c_ptr2, #0x10\n"
+ "ldr q21, [%[b_ptr0], #0x50]\n"
".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
+ "ldr q22, [%[b_ptr0], #0x60]\n"
+ ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
"str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
- "add c_ptr3, c_ptr3, #0x10\n"
+ "ldr q23, [%[b_ptr0], #0x70]\n"
".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
"str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
@@ -2673,32 +2942,29 @@
".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
"add c_ptr7, c_ptr7, #0x10\n"
".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
- "ldr q16, [%[b_ptr0]]\n"
- ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
+ "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n"
- "ldr q17, [%[b_ptr0], #0x10]\n"
+ "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
+ "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n"
+ "prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n"
".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n"
".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n"
".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n"
".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n"
".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n"
- "ldr q18, [%[b_ptr0], #0x20]\n"
".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n"
".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n"
".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n"
@@ -2707,7 +2973,6 @@
".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n"
".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n"
".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n"
- "ldr q19, [%[b_ptr0], #0x30]\n"
".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n"
".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n"
".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n"
@@ -2716,7 +2981,6 @@
".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n"
".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n"
".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n"
- "ldr q20, [%[b_ptr0], #0x40]\n"
".inst 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n"
".inst 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n"
".inst 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n"
@@ -2725,7 +2989,6 @@
".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n"
".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n"
".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n"
- "ldr q21, [%[b_ptr0], #0x50]\n"
".inst 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n"
".inst 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n"
".inst 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n"
@@ -2734,7 +2997,6 @@
".inst 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n"
".inst 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n"
".inst 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n"
- "ldr q22, [%[b_ptr0], #0x60]\n"
".inst 0x4fa1eaf8 // sdot v24.4s, v23.16b, v1.4b[3]\n"
".inst 0x4fa3eaf9 // sdot v25.4s, v23.16b, v3.4b[3]\n"
".inst 0x4fa5eafa // sdot v26.4s, v23.16b, v5.4b[3]\n"
@@ -2748,38 +3010,119 @@
"str q24, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v24.4s, #0\n"
- "ldr q23, [%[b_ptr0], #0x70]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
+ "ldr q16, [%[b_ptr0]]\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
"str q25, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
+ "ldr q18, [%[b_ptr0], #0x20]\n"
".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
"str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
+ "ldr q19, [%[b_ptr0], #0x30]\n"
+ "ldr q20, [%[b_ptr0], #0x40]\n"
"add c_ptr2, c_ptr2, #0x10\n"
".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
"str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
- "add c_ptr3, c_ptr3, #0x10\n"
+ "ldr q21, [%[b_ptr0], #0x50]\n"
".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
+ "ldr q22, [%[b_ptr0], #0x60]\n"
+ ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
"str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
- "add c_ptr4, c_ptr4, #0x10\n"
+ "ldr q23, [%[b_ptr0], #0x70]\n"
".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
"str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
- "add c_ptr5, c_ptr5, #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
"str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
- "add c_ptr6, c_ptr6, #0x10\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
"str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
- "add c_ptr7, c_ptr7, #0x10\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
- ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
+ "add c_ptr7, c_ptr7, #0x10\n"
".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
+ ".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x4fa8e23c // sdot v28.4s, v17.16b, v8.4b[1]\n"
+ ".inst 0x4faae23d // sdot v29.4s, v17.16b, v10.4b[1]\n"
+ ".inst 0x4face23e // sdot v30.4s, v17.16b, v12.4b[1]\n"
+ ".inst 0x4faee23f // sdot v31.4s, v17.16b, v14.4b[1]\n"
+ ".inst 0x4f80ea58 // sdot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x4f82ea59 // sdot v25.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x4f84ea5a // sdot v26.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x4f86ea5b // sdot v27.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x4f88ea5c // sdot v28.4s, v18.16b, v8.4b[2]\n"
+ ".inst 0x4f8aea5d // sdot v29.4s, v18.16b, v10.4b[2]\n"
+ ".inst 0x4f8cea5e // sdot v30.4s, v18.16b, v12.4b[2]\n"
+ ".inst 0x4f8eea5f // sdot v31.4s, v18.16b, v14.4b[2]\n"
+ ".inst 0x4fa0ea78 // sdot v24.4s, v19.16b, v0.4b[3]\n"
+ ".inst 0x4fa2ea79 // sdot v25.4s, v19.16b, v2.4b[3]\n"
+ ".inst 0x4fa4ea7a // sdot v26.4s, v19.16b, v4.4b[3]\n"
+ ".inst 0x4fa6ea7b // sdot v27.4s, v19.16b, v6.4b[3]\n"
+ ".inst 0x4fa8ea7c // sdot v28.4s, v19.16b, v8.4b[3]\n"
+ ".inst 0x4faaea7d // sdot v29.4s, v19.16b, v10.4b[3]\n"
+ ".inst 0x4facea7e // sdot v30.4s, v19.16b, v12.4b[3]\n"
+ ".inst 0x4faeea7f // sdot v31.4s, v19.16b, v14.4b[3]\n"
+ ".inst 0x4f81e298 // sdot v24.4s, v20.16b, v1.4b[0]\n"
+ ".inst 0x4f83e299 // sdot v25.4s, v20.16b, v3.4b[0]\n"
+ ".inst 0x4f85e29a // sdot v26.4s, v20.16b, v5.4b[0]\n"
+ ".inst 0x4f87e29b // sdot v27.4s, v20.16b, v7.4b[0]\n"
+ ".inst 0x4f89e29c // sdot v28.4s, v20.16b, v9.4b[0]\n"
+ ".inst 0x4f8be29d // sdot v29.4s, v20.16b, v11.4b[0]\n"
+ ".inst 0x4f8de29e // sdot v30.4s, v20.16b, v13.4b[0]\n"
+ ".inst 0x4f8fe29f // sdot v31.4s, v20.16b, v15.4b[0]\n"
+ ".inst 0x4fa1e2b8 // sdot v24.4s, v21.16b, v1.4b[1]\n"
+ ".inst 0x4fa3e2b9 // sdot v25.4s, v21.16b, v3.4b[1]\n"
+ ".inst 0x4fa5e2ba // sdot v26.4s, v21.16b, v5.4b[1]\n"
+ ".inst 0x4fa7e2bb // sdot v27.4s, v21.16b, v7.4b[1]\n"
+ ".inst 0x4fa9e2bc // sdot v28.4s, v21.16b, v9.4b[1]\n"
+ ".inst 0x4fabe2bd // sdot v29.4s, v21.16b, v11.4b[1]\n"
+ ".inst 0x4fade2be // sdot v30.4s, v21.16b, v13.4b[1]\n"
+ ".inst 0x4fafe2bf // sdot v31.4s, v21.16b, v15.4b[1]\n"
+ ".inst 0x4f81ead8 // sdot v24.4s, v22.16b, v1.4b[2]\n"
+ ".inst 0x4f83ead9 // sdot v25.4s, v22.16b, v3.4b[2]\n"
+ ".inst 0x4f85eada // sdot v26.4s, v22.16b, v5.4b[2]\n"
+ ".inst 0x4f87eadb // sdot v27.4s, v22.16b, v7.4b[2]\n"
+ ".inst 0x4f89eadc // sdot v28.4s, v22.16b, v9.4b[2]\n"
+ ".inst 0x4f8beadd // sdot v29.4s, v22.16b, v11.4b[2]\n"
+ ".inst 0x4f8deade // sdot v30.4s, v22.16b, v13.4b[2]\n"
+ ".inst 0x4f8feadf // sdot v31.4s, v22.16b, v15.4b[2]\n"
+ ".inst 0x4fa1eaf8 // sdot v24.4s, v23.16b, v1.4b[3]\n"
+ ".inst 0x4fa3eaf9 // sdot v25.4s, v23.16b, v3.4b[3]\n"
+ ".inst 0x4fa5eafa // sdot v26.4s, v23.16b, v5.4b[3]\n"
+ ".inst 0x4fa7eafb // sdot v27.4s, v23.16b, v7.4b[3]\n"
+ ".inst 0x4fa9eafc // sdot v28.4s, v23.16b, v9.4b[3]\n"
+ ".inst 0x4fabeafd // sdot v29.4s, v23.16b, v11.4b[3]\n"
+ ".inst 0x4fadeafe // sdot v30.4s, v23.16b, v13.4b[3]\n"
+ ".inst 0x4fafeaff // sdot v31.4s, v23.16b, v15.4b[3]\n"
+ "b 9f\n"
+ "6:\n"
+ "movi v24.4s, #0\n"
+ "movi v25.4s, #0\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x4f80e218 // sdot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x4f82e219 // sdot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x4f84e21a // sdot v26.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x4f86e21b // sdot v27.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x4f88e21c // sdot v28.4s, v16.16b, v8.4b[0]\n"
+ ".inst 0x4f8ae21d // sdot v29.4s, v16.16b, v10.4b[0]\n"
+ ".inst 0x4f8ce21e // sdot v30.4s, v16.16b, v12.4b[0]\n"
+ ".inst 0x4f8ee21f // sdot v31.4s, v16.16b, v14.4b[0]\n"
+ ".inst 0x4fa0e238 // sdot v24.4s, v17.16b, v0.4b[1]\n"
".inst 0x4fa2e239 // sdot v25.4s, v17.16b, v2.4b[1]\n"
".inst 0x4fa4e23a // sdot v26.4s, v17.16b, v4.4b[1]\n"
".inst 0x4fa6e23b // sdot v27.4s, v17.16b, v6.4b[1]\n"
@@ -2835,23 +3178,16 @@
".inst 0x4fabeafd // sdot v29.4s, v23.16b, v11.4b[3]\n"
".inst 0x4fadeafe // sdot v30.4s, v23.16b, v13.4b[3]\n"
".inst 0x4fafeaff // sdot v31.4s, v23.16b, v15.4b[3]\n"
- "6:\n"
+ "9:\n"
"str q24, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q25, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, #0x10\n"
"str q26, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, #0x10\n"
"str q27, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, #0x10\n"
"str q28, [c_ptr4]\n"
- "add c_ptr4, c_ptr4, #0x10\n"
"str q29, [c_ptr5]\n"
- "add c_ptr5, c_ptr5, #0x10\n"
"str q30, [c_ptr6]\n"
- "add c_ptr6, c_ptr6, #0x10\n"
"str q31, [c_ptr7]\n"
- "add c_ptr7, c_ptr7, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x6.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_6x4.hpp
similarity index 87%
rename from src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x6.hpp
rename to src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_6x4.hpp
index 76931db..5d48a52 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x6.hpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_6x4.hpp
@@ -31,10 +31,10 @@
{
// Actual kernel implementations
-void a64_smallK_hybrid_u8u32_dot_4x6(const uint8_t *, int, const uint8_t *, uint32_t *, int, int, int, int, const uint32_t *, Activation, bool);
-void a64_smallK_hybrid_u8u32_dot_4x6_a55(const uint8_t *, int, const uint8_t *, uint32_t *, int, int, int, int, const uint32_t *, Activation, bool);
+void a64_smallK_hybrid_u8u32_dot_6x4(const uint8_t *, int, const uint8_t *, uint32_t *, int, int, int, int, const uint32_t *, Activation, bool);
+void a64_smallK_hybrid_u8u32_dot_6x4_a55(const uint8_t *, int, const uint8_t *, uint32_t *, int, int, int, int, const uint32_t *, Activation, bool);
-class smallK_hybrid_u8u32_dot_4x6
+class cls_a64_smallK_hybrid_u8u32_dot_6x4
{
public:
typedef uint8_t operand_type;
@@ -76,12 +76,12 @@
StdTransformsFixed<operand_type, result_type, 6, 4, 4> transforms = {};
// Default to the generic kernel
- kern_type kernel=a64_smallK_hybrid_u8u32_dot_4x6;
+ kern_type kernel=a64_smallK_hybrid_u8u32_dot_6x4;
- smallK_hybrid_u8u32_dot_4x6(const CPUInfo *ci)
+ cls_a64_smallK_hybrid_u8u32_dot_6x4(const CPUInfo *ci)
{
if (ci->get_cpu_model() == CPUModel::A55r1) {
- kernel = a64_smallK_hybrid_u8u32_dot_4x6_a55;
+ kernel = a64_smallK_hybrid_u8u32_dot_6x4_a55;
}
}
};
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x6/a55.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_6x4/a55.cpp
similarity index 80%
rename from src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x6/a55.cpp
rename to src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_6x4/a55.cpp
index 02894d8..dddf4c5 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x6/a55.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_6x4/a55.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2019 Arm Limited.
+ * Copyright (c) 2019-2020 Arm Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -33,7 +33,7 @@
namespace arm_gemm {
-void a64_smallK_hybrid_u8u32_dot_4x6_a55(const uint8_t *A, int lda, const uint8_t *B, uint32_t *C, int ldc, int M, int N, int K, const uint32_t *, Activation, bool) {
+void a64_smallK_hybrid_u8u32_dot_6x4_a55(const uint8_t *A, int lda, const uint8_t *B, uint32_t *C, int ldc, int M, int N, int K, const uint32_t *, Activation, bool) {
const long loops_count = iceildiv(N, (int)4) - 1;
const long ldab = lda * sizeof(uint8_t);
const long ldcb = ldc * sizeof(uint32_t);
@@ -97,6 +97,7 @@
"add c_ptr1, %[c_ptr0], #0x0\n"
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
+ "cbnz %[odds], 2f\n"
"ldr q0, [%[a_ptr0]], #0x10\n"
"ldr q3, [a_ptr1], #0x10\n"
"ldr q6, [a_ptr2], #0x10\n"
@@ -107,18 +108,29 @@
"ldr q4, [a_ptr1], #0x10\n"
"ldr q7, [a_ptr2], #0x10\n"
"ldr q10, [a_ptr3], #0x10\n"
+ "ldr s2, [%[a_ptr0]]\n"
+ "ldr q13, [a_ptr4], #0x10\n"
+ "ldr s5, [a_ptr1]\n"
+ "ldr q16, [a_ptr5], #0x10\n"
+ "ldr s8, [a_ptr2]\n"
+ "ldr s11, [a_ptr3]\n"
+ "ldr s14, [a_ptr4]\n"
+ "ldr s17, [a_ptr5]\n"
+ "b 3f\n"
+ "2:\n"
+ "ldr q0, [%[a_ptr0]], #0x10\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "ldr q3, [a_ptr1], #0x10\n"
+ "ldr q6, [a_ptr2], #0x10\n"
+ "ldr q9, [a_ptr3], #0x10\n"
+ "ldr q12, [a_ptr4], #0x10\n"
+ "ldr q15, [a_ptr5], #0x10\n"
+ "ldr q1, [%[a_ptr0]], #0x10\n"
+ "ldr q4, [a_ptr1], #0x10\n"
+ "ldr q7, [a_ptr2], #0x10\n"
+ "ldr q10, [a_ptr3], #0x10\n"
"ldr q13, [a_ptr4], #0x10\n"
"ldr q16, [a_ptr5], #0x10\n"
- "cbnz %[odds], 2f\n"
- "ldr s2, [%[a_ptr0]]\n"
- "ldr s5, [a_ptr1]\n"
- "ldr s8, [a_ptr2]\n"
- "ldr s11, [a_ptr3]\n"
- "ldr s14, [a_ptr4]\n"
- "ldr s17, [a_ptr5]\n"
- "b 3f\n"
- "2:\n"
- "subs %[odds], %[odds], #0x1\n"
"b.ne 4f\n"
"ldr b2, [%[a_ptr0]]\n"
"ldr b5, [a_ptr1]\n"
@@ -145,40 +157,42 @@
"ld1 {v14.b}[2], [a_ptr4]\n"
"ld1 {v17.b}[2], [a_ptr5]\n"
"3:\n"
- "movi v26.4s, #0\n"
"ldr q18, [%[b_ptr0]]\n"
- "movi v27.4s, #0\n"
- "ldr q19, [%[b_ptr0], #0x10]\n"
- "movi v28.4s, #0\n"
- "ldr q20, [%[b_ptr0], #0x20]\n"
- "movi v29.4s, #0\n"
- "ldr q21, [%[b_ptr0], #0x30]\n"
- "movi v30.4s, #0\n"
- "ldr q22, [%[b_ptr0], #0x40]\n"
- "movi v31.4s, #0\n"
- "ldr q23, [%[b_ptr0], #0x50]\n"
- ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n"
- "ldr q24, [%[b_ptr0], #0x60]\n"
- ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n"
- "ldr q25, [%[b_ptr0], #0x70]\n"
- ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x40]\n"
- ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x80]\n"
- ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
"prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
- ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x100]\n"
- ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n"
+ "ldr q22, [%[b_ptr0], #0x40]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x140]\n"
- ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n"
+ "ldr q23, [%[b_ptr0], #0x50]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x180]\n"
- ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n"
+ "ldr q24, [%[b_ptr0], #0x60]\n"
+ "ldr q25, [%[b_ptr0], #0x70]\n"
"add %[b_ptr0], %[b_ptr0], #0x80\n"
- ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n"
+ "cbz %[loops], 6f\n"
+ "movi v26.4s, #0\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n"
+ ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n"
+ ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n"
+ ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n"
+ ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n"
+ ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n"
"ldr q18, [%[b_ptr0]]\n"
- ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n"
+ ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n"
"add %[b_ptr0], %[b_ptr0], #0x10\n"
+ ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n"
+ ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n"
+ ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n"
+ ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n"
".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n"
".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n"
".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n"
@@ -222,173 +236,219 @@
".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n"
".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n"
".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n"
- "cbz %[loops], 6f\n"
- "ldr d18, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ldr temploadreg2, [%[b_ptr0], #0x8]\n"
- "ldr d19, [%[b_ptr0], #0x10]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x18]\n"
- "ldr d20, [%[b_ptr0], #0x20]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x28]\n"
- "ldr d21, [%[b_ptr0], #0x30]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x38]\n"
- "ldr d22, [%[b_ptr0], #0x40]\n"
- "ins v18.d[1], temploadreg2\n"
- "ldr temploadreg2, [%[b_ptr0], #0x48]\n"
- "ldr d23, [%[b_ptr0], #0x50]\n"
- "ins v19.d[1], temploadreg3\n"
- "ldr temploadreg3, [%[b_ptr0], #0x58]\n"
- "ldr d24, [%[b_ptr0], #0x60]\n"
- "ins v20.d[1], temploadreg0\n"
- "ldr temploadreg0, [%[b_ptr0], #0x68]\n"
- "ldr d25, [%[b_ptr0], #0x70]\n"
- "ins v21.d[1], temploadreg1\n"
- "ldr temploadreg1, [%[b_ptr0], #0x78]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "ins v22.d[1], temploadreg2\n"
- "ins v23.d[1], temploadreg3\n"
- "ins v24.d[1], temploadreg0\n"
- "ins v25.d[1], temploadreg1\n"
"b.eq 7f\n"
"8:\n"
"str q26, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"movi v26.4s, #0\n"
+ "ldr d18, [%[b_ptr0]]\n"
"ldr temploadreg2, [%[b_ptr0], #0x8]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q27, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
- ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n"
- "str q28, [c_ptr2]\n"
- "movi v28.4s, #0\n"
- "add c_ptr2, c_ptr2, #0x10\n"
- ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n"
- "str q29, [c_ptr3]\n"
- "movi v29.4s, #0\n"
- "add c_ptr3, c_ptr3, #0x10\n"
- ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n"
- "str q30, [c_ptr4]\n"
- "movi v30.4s, #0\n"
- "add c_ptr4, c_ptr4, #0x10\n"
- ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n"
- "str q31, [c_ptr5]\n"
- "movi v31.4s, #0\n"
- "add c_ptr5, c_ptr5, #0x10\n"
- ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n"
- "ldr d18, [%[b_ptr0]]\n"
- ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n"
- "add %[b_ptr0], %[b_ptr0], #0x10\n"
- ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n"
- "ins v18.d[1], temploadreg2\n"
- ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x8]\n"
- ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x18]\n"
- ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x28]\n"
- ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n"
"ldr d19, [%[b_ptr0], #0x10]\n"
- ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x38]\n"
- ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n"
- "ins v19.d[1], temploadreg3\n"
- ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x58]\n"
- ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n"
+ "ldr temploadreg3, [%[b_ptr0], #0x18]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "str q28, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ "movi v28.4s, #0\n"
"ldr d20, [%[b_ptr0], #0x20]\n"
+ "ldr temploadreg0, [%[b_ptr0], #0x28]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ "str q29, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ "movi v29.4s, #0\n"
+ "ldr d21, [%[b_ptr0], #0x30]\n"
+ "ldr temploadreg1, [%[b_ptr0], #0x38]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ "str q30, [c_ptr4]\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ "movi v30.4s, #0\n"
+ "ldr d22, [%[b_ptr0], #0x40]\n"
+ "ins v18.d[1], temploadreg2\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
+ "str q31, [c_ptr5]\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ "movi v31.4s, #0\n"
+ "ldr temploadreg2, [%[b_ptr0], #0x48]\n"
+ ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n"
+ "ldr d23, [%[b_ptr0], #0x50]\n"
+ ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n"
+ "ins v19.d[1], temploadreg3\n"
+ ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n"
+ "ldr temploadreg3, [%[b_ptr0], #0x58]\n"
+ ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n"
+ "ldr d24, [%[b_ptr0], #0x60]\n"
+ ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n"
+ "ins v20.d[1], temploadreg0\n"
+ ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n"
+ "ldr temploadreg0, [%[b_ptr0], #0x68]\n"
+ ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n"
+ "ldr d25, [%[b_ptr0], #0x70]\n"
+ ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n"
+ "ins v21.d[1], temploadreg1\n"
+ ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n"
+ "ldr temploadreg1, [%[b_ptr0], #0x78]\n"
+ ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n"
+ "ins v22.d[1], temploadreg2\n"
+ ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n"
+ "ins v23.d[1], temploadreg3\n"
+ ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n"
+ "ins v24.d[1], temploadreg0\n"
+ ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n"
+ "ins v25.d[1], temploadreg1\n"
+ ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
+ ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n"
+ "ldr d18, [%[b_ptr0]]\n"
+ ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n"
+ "ldr temploadreg2, [%[b_ptr0], #0x8]\n"
+ ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x10\n"
+ ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n"
+ "ins v18.d[1], temploadreg2\n"
".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n"
"prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n"
- "ins v20.d[1], temploadreg0\n"
- ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x68]\n"
- ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n"
"prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
+ ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n"
+ ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n"
".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n"
".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n"
- "ldr d21, [%[b_ptr0], #0x30]\n"
".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n"
".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n"
".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n"
- "ins v21.d[1], temploadreg1\n"
".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x78]\n"
".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n"
".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n"
- "ldr d22, [%[b_ptr0], #0x40]\n"
".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n"
".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n"
".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n"
".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n"
".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n"
".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n"
- "ldr d23, [%[b_ptr0], #0x50]\n"
".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n"
- "ins v23.d[1], temploadreg3\n"
".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n"
".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n"
".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n"
- "ldr d24, [%[b_ptr0], #0x60]\n"
".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n"
- "ins v24.d[1], temploadreg0\n"
".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n"
".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n"
".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n"
- "ldr d25, [%[b_ptr0], #0x70]\n"
".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n"
".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n"
".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n"
- "ins v25.d[1], temploadreg1\n"
".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n"
".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n"
".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n"
- "ldr d18, [%[b_ptr0]]\n"
- "ins v18.d[1], temploadreg2\n"
- "ldr temploadreg2, [%[b_ptr0], #0x48]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "ins v22.d[1], temploadreg2\n"
"b.ne 8b\n"
"7:\n"
"str q26, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v26.4s, #0\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
"str q27, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n"
"str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
+ "ldr q22, [%[b_ptr0], #0x40]\n"
"add c_ptr2, c_ptr2, #0x10\n"
".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n"
"str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
- "add c_ptr3, c_ptr3, #0x10\n"
+ "ldr q23, [%[b_ptr0], #0x50]\n"
".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n"
+ "ldr q24, [%[b_ptr0], #0x60]\n"
+ ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n"
"str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
- "add c_ptr4, c_ptr4, #0x10\n"
+ "ldr q25, [%[b_ptr0], #0x70]\n"
".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n"
"str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
- "add c_ptr5, c_ptr5, #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n"
- ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
+ ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x10\n"
+ ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n"
+ ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n"
+ ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n"
+ ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n"
+ ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n"
+ ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n"
+ ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n"
+ ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n"
+ ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n"
+ ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n"
+ ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n"
+ ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n"
+ ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n"
+ ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n"
+ ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n"
+ ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n"
+ ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n"
+ ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n"
+ ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n"
+ ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n"
+ ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n"
+ ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n"
+ ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n"
+ ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n"
+ ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n"
+ ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n"
+ ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n"
+ ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n"
+ ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n"
+ ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n"
+ ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n"
+ ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n"
+ ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n"
+ "b 9f\n"
+ "6:\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n"
+ ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n"
+ ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n"
+ ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n"
+ ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n"
".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n"
"ldr q18, [%[b_ptr0]]\n"
- ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n"
+ ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n"
"add %[b_ptr0], %[b_ptr0], #0x10\n"
+ ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n"
".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n"
".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n"
".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n"
@@ -435,19 +495,14 @@
".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n"
".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n"
".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n"
- "6:\n"
+ "9:\n"
"str q26, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q27, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, #0x10\n"
"str q28, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, #0x10\n"
"str q29, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, #0x10\n"
"str q30, [c_ptr4]\n"
- "add c_ptr4, c_ptr4, #0x10\n"
"str q31, [c_ptr5]\n"
- "add c_ptr5, c_ptr5, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -514,6 +569,7 @@
"add c_ptr1, %[c_ptr0], #0x0\n"
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
+ "cbnz %[odds], 2f\n"
"ldr q0, [%[a_ptr0]], #0x10\n"
"ldr q3, [a_ptr1], #0x10\n"
"ldr q6, [a_ptr2], #0x10\n"
@@ -524,24 +580,35 @@
"ldr q4, [a_ptr1], #0x10\n"
"ldr q7, [a_ptr2], #0x10\n"
"ldr q10, [a_ptr3], #0x10\n"
- "ldr q13, [a_ptr4], #0x10\n"
- "ldr q16, [a_ptr5], #0x10\n"
- "cbnz %[odds], 2f\n"
"ldr d2, [%[a_ptr0]]\n"
+ "ldr q13, [a_ptr4], #0x10\n"
"ldr d5, [a_ptr1]\n"
+ "ldr q16, [a_ptr5], #0x10\n"
"ldr d8, [a_ptr2]\n"
"ldr d11, [a_ptr3]\n"
"ldr d14, [a_ptr4]\n"
"ldr d17, [a_ptr5]\n"
"b 3f\n"
"2:\n"
+ "ldr q0, [%[a_ptr0]], #0x10\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "ldr q3, [a_ptr1], #0x10\n"
+ "ldr q6, [a_ptr2], #0x10\n"
+ "ldr q9, [a_ptr3], #0x10\n"
+ "ldr q12, [a_ptr4], #0x10\n"
+ "ldr q15, [a_ptr5], #0x10\n"
+ "ldr q1, [%[a_ptr0]], #0x10\n"
+ "ldr q4, [a_ptr1], #0x10\n"
+ "ldr q7, [a_ptr2], #0x10\n"
+ "ldr q10, [a_ptr3], #0x10\n"
"ldr s2, [%[a_ptr0]], #0x4\n"
+ "ldr q13, [a_ptr4], #0x10\n"
"ldr s5, [a_ptr1], #0x4\n"
+ "ldr q16, [a_ptr5], #0x10\n"
"ldr s8, [a_ptr2], #0x4\n"
"ldr s11, [a_ptr3], #0x4\n"
"ldr s14, [a_ptr4], #0x4\n"
"ldr s17, [a_ptr5], #0x4\n"
- "subs %[odds], %[odds], #0x1\n"
"b.ne 4f\n"
"ld1 {v2.b}[4], [%[a_ptr0]]\n"
"ld1 {v5.b}[4], [a_ptr1]\n"
@@ -568,38 +635,40 @@
"ld1 {v14.b}[6], [a_ptr4]\n"
"ld1 {v17.b}[6], [a_ptr5]\n"
"3:\n"
- "movi v26.4s, #0\n"
"ldr q18, [%[b_ptr0]]\n"
- "movi v27.4s, #0\n"
- "ldr q19, [%[b_ptr0], #0x10]\n"
- "movi v28.4s, #0\n"
- "ldr q20, [%[b_ptr0], #0x20]\n"
- "movi v29.4s, #0\n"
- "ldr q21, [%[b_ptr0], #0x30]\n"
- "movi v30.4s, #0\n"
- "ldr q22, [%[b_ptr0], #0x40]\n"
- "movi v31.4s, #0\n"
- "ldr q23, [%[b_ptr0], #0x50]\n"
- ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n"
- "ldr q24, [%[b_ptr0], #0x60]\n"
- ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n"
- "ldr q25, [%[b_ptr0], #0x70]\n"
- ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x40]\n"
- ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x80]\n"
- ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
"prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
- ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x100]\n"
- ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n"
+ "ldr q22, [%[b_ptr0], #0x40]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x140]\n"
- ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n"
+ "ldr q23, [%[b_ptr0], #0x50]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x180]\n"
- ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n"
+ "ldr q24, [%[b_ptr0], #0x60]\n"
+ "ldr q25, [%[b_ptr0], #0x70]\n"
"add %[b_ptr0], %[b_ptr0], #0x80\n"
- ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n"
+ "cbz %[loops], 6f\n"
+ "movi v26.4s, #0\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n"
+ ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n"
+ ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n"
+ ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n"
+ ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n"
+ ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n"
"ldr q18, [%[b_ptr0]]\n"
+ ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n"
+ ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n"
+ ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n"
+ ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n"
".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n"
".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n"
"ldr q19, [%[b_ptr0], #0x10]\n"
@@ -652,180 +721,233 @@
".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n"
".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n"
".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n"
- "cbz %[loops], 6f\n"
- "ldr d18, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ldr temploadreg2, [%[b_ptr0], #0x8]\n"
- "ldr d19, [%[b_ptr0], #0x10]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x18]\n"
- "ldr d20, [%[b_ptr0], #0x20]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x28]\n"
- "ldr d21, [%[b_ptr0], #0x30]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x38]\n"
- "ldr d22, [%[b_ptr0], #0x40]\n"
- "ins v18.d[1], temploadreg2\n"
- "ldr temploadreg2, [%[b_ptr0], #0x48]\n"
- "ldr d23, [%[b_ptr0], #0x50]\n"
- "ins v19.d[1], temploadreg3\n"
- "ldr temploadreg3, [%[b_ptr0], #0x58]\n"
- "ldr d24, [%[b_ptr0], #0x60]\n"
- "ins v20.d[1], temploadreg0\n"
- "ldr temploadreg0, [%[b_ptr0], #0x68]\n"
- "ldr d25, [%[b_ptr0], #0x70]\n"
- "ins v21.d[1], temploadreg1\n"
- "ldr temploadreg1, [%[b_ptr0], #0x78]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "ins v22.d[1], temploadreg2\n"
- "ins v23.d[1], temploadreg3\n"
- "ins v24.d[1], temploadreg0\n"
- "ins v25.d[1], temploadreg1\n"
"b.eq 7f\n"
"8:\n"
"str q26, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"movi v26.4s, #0\n"
+ "ldr d18, [%[b_ptr0]]\n"
"ldr temploadreg2, [%[b_ptr0], #0x8]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x18]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q27, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
- ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n"
- "str q28, [c_ptr2]\n"
- "movi v28.4s, #0\n"
- "add c_ptr2, c_ptr2, #0x10\n"
- ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n"
- "str q29, [c_ptr3]\n"
- "movi v29.4s, #0\n"
- "add c_ptr3, c_ptr3, #0x10\n"
- ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n"
- "str q30, [c_ptr4]\n"
- "movi v30.4s, #0\n"
- "add c_ptr4, c_ptr4, #0x10\n"
- ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n"
- "str q31, [c_ptr5]\n"
- "movi v31.4s, #0\n"
- "add c_ptr5, c_ptr5, #0x10\n"
- ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n"
- "ldr d18, [%[b_ptr0]]\n"
- ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n"
- "ins v18.d[1], temploadreg2\n"
- ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n"
"ldr d19, [%[b_ptr0], #0x10]\n"
- ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n"
- "ins v19.d[1], temploadreg3\n"
- ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x8]\n"
- ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n"
"ldr temploadreg3, [%[b_ptr0], #0x18]\n"
- ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x28]\n"
- ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "str q28, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ "movi v28.4s, #0\n"
"ldr d20, [%[b_ptr0], #0x20]\n"
- ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x38]\n"
- ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n"
- ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n"
- "ins v20.d[1], temploadreg0\n"
- ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x68]\n"
- ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n"
- ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n"
+ "ldr temploadreg0, [%[b_ptr0], #0x28]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ "str q29, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ "movi v29.4s, #0\n"
"ldr d21, [%[b_ptr0], #0x30]\n"
+ "ldr temploadreg1, [%[b_ptr0], #0x38]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ "str q30, [c_ptr4]\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ "movi v30.4s, #0\n"
+ "ldr d22, [%[b_ptr0], #0x40]\n"
+ "ins v18.d[1], temploadreg2\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
+ "str q31, [c_ptr5]\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ "movi v31.4s, #0\n"
+ "ldr temploadreg2, [%[b_ptr0], #0x48]\n"
+ ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n"
+ "ldr d23, [%[b_ptr0], #0x50]\n"
+ ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n"
+ "ins v19.d[1], temploadreg3\n"
+ ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n"
+ "ldr temploadreg3, [%[b_ptr0], #0x58]\n"
+ ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n"
+ "ldr d24, [%[b_ptr0], #0x60]\n"
+ ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n"
+ "ins v20.d[1], temploadreg0\n"
+ ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n"
+ "ldr temploadreg0, [%[b_ptr0], #0x68]\n"
+ ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n"
+ "ldr d25, [%[b_ptr0], #0x70]\n"
+ ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n"
+ "ins v21.d[1], temploadreg1\n"
+ ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n"
+ "ldr temploadreg1, [%[b_ptr0], #0x78]\n"
+ ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n"
+ "ins v22.d[1], temploadreg2\n"
+ ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n"
+ "ins v23.d[1], temploadreg3\n"
+ ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n"
+ "ins v24.d[1], temploadreg0\n"
+ ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n"
+ "ins v25.d[1], temploadreg1\n"
+ ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
+ ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n"
+ "ldr d18, [%[b_ptr0]]\n"
+ ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n"
+ "ldr temploadreg2, [%[b_ptr0], #0x8]\n"
+ ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n"
+ "ldr d19, [%[b_ptr0], #0x10]\n"
+ ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n"
+ "ldr temploadreg3, [%[b_ptr0], #0x18]\n"
+ ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n"
+ "ins v18.d[1], temploadreg2\n"
+ ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n"
+ "ins v19.d[1], temploadreg3\n"
+ ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n"
+ "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
+ ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n"
+ "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
+ ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n"
".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n"
".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n"
".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n"
- "ins v21.d[1], temploadreg1\n"
".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x78]\n"
".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n"
".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n"
- "ldr d22, [%[b_ptr0], #0x40]\n"
".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n"
".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n"
".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n"
".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n"
".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n"
".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n"
- "ldr d23, [%[b_ptr0], #0x50]\n"
".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n"
".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n"
".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n"
".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n"
- "ldr d24, [%[b_ptr0], #0x60]\n"
".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n"
- "ins v24.d[1], temploadreg0\n"
".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n"
".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n"
".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n"
- "ldr d25, [%[b_ptr0], #0x70]\n"
".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n"
".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n"
".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n"
- "ins v25.d[1], temploadreg1\n"
".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n"
".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n"
".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n"
- "ldr d18, [%[b_ptr0]]\n"
".inst 0x6fa2e27a // udot v26.4s, v19.16b, v2.4b[1]\n"
".inst 0x6fa5e27b // udot v27.4s, v19.16b, v5.4b[1]\n"
".inst 0x6fa8e27c // udot v28.4s, v19.16b, v8.4b[1]\n"
- "ins v18.d[1], temploadreg2\n"
".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x48]\n"
".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n"
".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n"
- "ldr d19, [%[b_ptr0], #0x10]\n"
- "ins v22.d[1], temploadreg2\n"
- "ins v19.d[1], temploadreg3\n"
- "ldr temploadreg3, [%[b_ptr0], #0x58]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "ins v23.d[1], temploadreg3\n"
"b.ne 8b\n"
"7:\n"
"str q26, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v26.4s, #0\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
"str q27, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n"
"str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
+ "ldr q22, [%[b_ptr0], #0x40]\n"
"add c_ptr2, c_ptr2, #0x10\n"
".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n"
"str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
- "add c_ptr3, c_ptr3, #0x10\n"
+ "ldr q23, [%[b_ptr0], #0x50]\n"
".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n"
+ "ldr q24, [%[b_ptr0], #0x60]\n"
+ ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n"
"str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
- "add c_ptr4, c_ptr4, #0x10\n"
+ "ldr q25, [%[b_ptr0], #0x70]\n"
".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n"
"str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
- "add c_ptr5, c_ptr5, #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n"
- ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
+ ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n"
+ ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n"
+ ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
+ ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n"
+ ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n"
+ ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n"
+ ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n"
+ ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n"
+ ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n"
+ ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n"
+ ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n"
+ ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n"
+ ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n"
+ ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n"
+ ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n"
+ ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n"
+ ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n"
+ ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n"
+ ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n"
+ ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n"
+ ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n"
+ ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n"
+ ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n"
+ ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n"
+ ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n"
+ ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n"
+ ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n"
+ ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n"
+ ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n"
+ ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n"
+ ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n"
+ ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n"
+ ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n"
+ ".inst 0x6fa2e27a // udot v26.4s, v19.16b, v2.4b[1]\n"
+ ".inst 0x6fa5e27b // udot v27.4s, v19.16b, v5.4b[1]\n"
+ ".inst 0x6fa8e27c // udot v28.4s, v19.16b, v8.4b[1]\n"
+ ".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n"
+ ".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n"
+ ".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n"
+ "b 9f\n"
+ "6:\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n"
+ ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n"
+ ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n"
+ ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n"
+ ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n"
".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n"
"ldr q18, [%[b_ptr0]]\n"
+ ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n"
".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n"
".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n"
".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n"
@@ -881,19 +1003,14 @@
".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n"
".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n"
".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n"
- "6:\n"
+ "9:\n"
"str q26, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q27, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, #0x10\n"
"str q28, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, #0x10\n"
"str q29, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, #0x10\n"
"str q30, [c_ptr4]\n"
- "add c_ptr4, c_ptr4, #0x10\n"
"str q31, [c_ptr5]\n"
- "add c_ptr5, c_ptr5, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -1014,38 +1131,40 @@
"ld1 {v14.b}[10], [a_ptr4]\n"
"ld1 {v17.b}[10], [a_ptr5]\n"
"3:\n"
- "movi v26.4s, #0\n"
"ldr q18, [%[b_ptr0]]\n"
- "movi v27.4s, #0\n"
- "ldr q19, [%[b_ptr0], #0x10]\n"
- "movi v28.4s, #0\n"
- "ldr q20, [%[b_ptr0], #0x20]\n"
- "movi v29.4s, #0\n"
- "ldr q21, [%[b_ptr0], #0x30]\n"
- "movi v30.4s, #0\n"
- "ldr q22, [%[b_ptr0], #0x40]\n"
- "movi v31.4s, #0\n"
- "ldr q23, [%[b_ptr0], #0x50]\n"
- ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n"
- "ldr q24, [%[b_ptr0], #0x60]\n"
- ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n"
- "ldr q25, [%[b_ptr0], #0x70]\n"
- ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x40]\n"
- ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x80]\n"
- ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
"prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
- ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x100]\n"
- ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n"
+ "ldr q22, [%[b_ptr0], #0x40]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x140]\n"
- ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n"
+ "ldr q23, [%[b_ptr0], #0x50]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x180]\n"
- ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n"
+ "ldr q24, [%[b_ptr0], #0x60]\n"
+ "ldr q25, [%[b_ptr0], #0x70]\n"
"add %[b_ptr0], %[b_ptr0], #0x80\n"
- ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n"
+ "cbz %[loops], 6f\n"
+ "movi v26.4s, #0\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n"
+ ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n"
+ ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n"
+ ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n"
+ ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n"
+ ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n"
"ldr q18, [%[b_ptr0]]\n"
+ ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n"
+ ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n"
+ ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n"
+ ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n"
".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n"
".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n"
"ldr q19, [%[b_ptr0], #0x10]\n"
@@ -1105,189 +1224,249 @@
".inst 0x6f8bea9d // udot v29.4s, v20.16b, v11.4b[2]\n"
".inst 0x6f8eea9e // udot v30.4s, v20.16b, v14.4b[2]\n"
".inst 0x6f91ea9f // udot v31.4s, v20.16b, v17.4b[2]\n"
- "cbz %[loops], 6f\n"
- "ldr d18, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ldr temploadreg2, [%[b_ptr0], #0x8]\n"
- "ldr d19, [%[b_ptr0], #0x10]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x18]\n"
- "ldr d20, [%[b_ptr0], #0x20]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x28]\n"
- "ldr d21, [%[b_ptr0], #0x30]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x38]\n"
- "ldr d22, [%[b_ptr0], #0x40]\n"
- "ins v18.d[1], temploadreg2\n"
- "ldr temploadreg2, [%[b_ptr0], #0x48]\n"
- "ldr d23, [%[b_ptr0], #0x50]\n"
- "ins v19.d[1], temploadreg3\n"
- "ldr temploadreg3, [%[b_ptr0], #0x58]\n"
- "ldr d24, [%[b_ptr0], #0x60]\n"
- "ins v20.d[1], temploadreg0\n"
- "ldr temploadreg0, [%[b_ptr0], #0x68]\n"
- "ldr d25, [%[b_ptr0], #0x70]\n"
- "ins v21.d[1], temploadreg1\n"
- "ldr temploadreg1, [%[b_ptr0], #0x78]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "ins v22.d[1], temploadreg2\n"
- "ins v23.d[1], temploadreg3\n"
- "ins v24.d[1], temploadreg0\n"
- "ins v25.d[1], temploadreg1\n"
"b.eq 7f\n"
"8:\n"
"str q26, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"movi v26.4s, #0\n"
+ "ldr d18, [%[b_ptr0]]\n"
"ldr temploadreg2, [%[b_ptr0], #0x8]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x18]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q27, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
- "ldr temploadreg0, [%[b_ptr0], #0x28]\n"
- ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n"
- "str q28, [c_ptr2]\n"
- "movi v28.4s, #0\n"
- "add c_ptr2, c_ptr2, #0x10\n"
- ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n"
- "str q29, [c_ptr3]\n"
- "movi v29.4s, #0\n"
- "add c_ptr3, c_ptr3, #0x10\n"
- ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n"
- "str q30, [c_ptr4]\n"
- "movi v30.4s, #0\n"
- "add c_ptr4, c_ptr4, #0x10\n"
- ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n"
- "str q31, [c_ptr5]\n"
- "movi v31.4s, #0\n"
- "add c_ptr5, c_ptr5, #0x10\n"
- ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
- ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n"
- "ldr d18, [%[b_ptr0]]\n"
- ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n"
- "ins v18.d[1], temploadreg2\n"
- ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n"
"ldr d19, [%[b_ptr0], #0x10]\n"
- ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n"
- "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n"
- "ins v19.d[1], temploadreg3\n"
- ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n"
- ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n"
- ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n"
- ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n"
- "ldr d20, [%[b_ptr0], #0x20]\n"
- ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n"
- "add %[b_ptr0], %[b_ptr0], #0x30\n"
- ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n"
- "ins v20.d[1], temploadreg0\n"
- ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x8]\n"
- ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n"
"ldr temploadreg3, [%[b_ptr0], #0x18]\n"
- ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "str q28, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ "movi v28.4s, #0\n"
+ "ldr d20, [%[b_ptr0], #0x20]\n"
"ldr temploadreg0, [%[b_ptr0], #0x28]\n"
- ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ "str q29, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ "movi v29.4s, #0\n"
"ldr d21, [%[b_ptr0], #0x30]\n"
- ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n"
"ldr temploadreg1, [%[b_ptr0], #0x38]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ "str q30, [c_ptr4]\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ "movi v30.4s, #0\n"
+ "ldr d22, [%[b_ptr0], #0x40]\n"
+ "ins v18.d[1], temploadreg2\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
+ "str q31, [c_ptr5]\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ "movi v31.4s, #0\n"
+ "ldr temploadreg2, [%[b_ptr0], #0x48]\n"
+ ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n"
+ "ldr d23, [%[b_ptr0], #0x50]\n"
+ ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n"
+ "ins v19.d[1], temploadreg3\n"
+ ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n"
+ "ldr temploadreg3, [%[b_ptr0], #0x58]\n"
+ ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n"
+ "ldr d24, [%[b_ptr0], #0x60]\n"
+ ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n"
+ "ins v20.d[1], temploadreg0\n"
+ ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n"
+ "ldr temploadreg0, [%[b_ptr0], #0x68]\n"
+ ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n"
+ "ldr d25, [%[b_ptr0], #0x70]\n"
+ ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n"
+ "ins v21.d[1], temploadreg1\n"
+ ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n"
+ "ldr temploadreg1, [%[b_ptr0], #0x78]\n"
+ ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n"
+ "ins v22.d[1], temploadreg2\n"
+ ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n"
+ "ins v23.d[1], temploadreg3\n"
+ ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n"
+ "ins v24.d[1], temploadreg0\n"
+ ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n"
+ "ins v25.d[1], temploadreg1\n"
+ ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
+ ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n"
+ "ldr d18, [%[b_ptr0]]\n"
+ ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n"
+ "ldr temploadreg2, [%[b_ptr0], #0x8]\n"
+ ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n"
+ "ldr d19, [%[b_ptr0], #0x10]\n"
+ ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n"
+ "ldr temploadreg3, [%[b_ptr0], #0x18]\n"
+ ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n"
+ "ldr d20, [%[b_ptr0], #0x20]\n"
+ ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n"
+ "ldr temploadreg0, [%[b_ptr0], #0x28]\n"
+ ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n"
+ "ins v18.d[1], temploadreg2\n"
+ ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n"
+ "ins v19.d[1], temploadreg3\n"
+ ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n"
+ "ins v20.d[1], temploadreg0\n"
+ ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x30\n"
+ ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n"
+ "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n"
+ "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n"
".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n"
- "ins v21.d[1], temploadreg1\n"
".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x78]\n"
".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n"
- "ldr d22, [%[b_ptr0], #0x40]\n"
".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n"
".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n"
".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n"
".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n"
".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n"
".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n"
- "ldr d23, [%[b_ptr0], #0x50]\n"
".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n"
".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n"
".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n"
".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n"
- "ldr d24, [%[b_ptr0], #0x60]\n"
".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n"
".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n"
".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n"
".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n"
- "ldr d25, [%[b_ptr0], #0x70]\n"
".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n"
".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n"
".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n"
- "ins v25.d[1], temploadreg1\n"
".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n"
".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n"
".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n"
- "ldr d18, [%[b_ptr0]]\n"
".inst 0x6fa2e27a // udot v26.4s, v19.16b, v2.4b[1]\n"
".inst 0x6fa5e27b // udot v27.4s, v19.16b, v5.4b[1]\n"
".inst 0x6fa8e27c // udot v28.4s, v19.16b, v8.4b[1]\n"
- "ins v18.d[1], temploadreg2\n"
".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x48]\n"
".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n"
".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n"
- "ldr d19, [%[b_ptr0], #0x10]\n"
".inst 0x6f82ea9a // udot v26.4s, v20.16b, v2.4b[2]\n"
- "ins v22.d[1], temploadreg2\n"
".inst 0x6f85ea9b // udot v27.4s, v20.16b, v5.4b[2]\n"
".inst 0x6f88ea9c // udot v28.4s, v20.16b, v8.4b[2]\n"
- "ins v19.d[1], temploadreg3\n"
".inst 0x6f8bea9d // udot v29.4s, v20.16b, v11.4b[2]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x58]\n"
".inst 0x6f8eea9e // udot v30.4s, v20.16b, v14.4b[2]\n"
".inst 0x6f91ea9f // udot v31.4s, v20.16b, v17.4b[2]\n"
- "ldr d20, [%[b_ptr0], #0x20]\n"
- "ins v23.d[1], temploadreg3\n"
- "ins v20.d[1], temploadreg0\n"
- "ldr temploadreg0, [%[b_ptr0], #0x68]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "ins v24.d[1], temploadreg0\n"
"b.ne 8b\n"
"7:\n"
"str q26, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v26.4s, #0\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
"str q27, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n"
"str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
+ "ldr q22, [%[b_ptr0], #0x40]\n"
"add c_ptr2, c_ptr2, #0x10\n"
".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n"
"str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
- "add c_ptr3, c_ptr3, #0x10\n"
+ "ldr q23, [%[b_ptr0], #0x50]\n"
".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n"
+ "ldr q24, [%[b_ptr0], #0x60]\n"
+ ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n"
"str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
- "add c_ptr4, c_ptr4, #0x10\n"
+ "ldr q25, [%[b_ptr0], #0x70]\n"
".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n"
"str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
- "add c_ptr5, c_ptr5, #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n"
- ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
+ ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n"
+ ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n"
+ ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
+ ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n"
+ ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n"
+ ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n"
+ ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n"
+ ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n"
+ ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
+ ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x30\n"
+ ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n"
+ ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n"
+ ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n"
+ ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n"
+ ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n"
+ ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n"
+ ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n"
+ ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n"
+ ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n"
+ ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n"
+ ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n"
+ ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n"
+ ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n"
+ ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n"
+ ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n"
+ ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n"
+ ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n"
+ ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n"
+ ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n"
+ ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n"
+ ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n"
+ ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n"
+ ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n"
+ ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n"
+ ".inst 0x6fa2e27a // udot v26.4s, v19.16b, v2.4b[1]\n"
+ ".inst 0x6fa5e27b // udot v27.4s, v19.16b, v5.4b[1]\n"
+ ".inst 0x6fa8e27c // udot v28.4s, v19.16b, v8.4b[1]\n"
+ ".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n"
+ ".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n"
+ ".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n"
+ ".inst 0x6f82ea9a // udot v26.4s, v20.16b, v2.4b[2]\n"
+ ".inst 0x6f85ea9b // udot v27.4s, v20.16b, v5.4b[2]\n"
+ ".inst 0x6f88ea9c // udot v28.4s, v20.16b, v8.4b[2]\n"
+ ".inst 0x6f8bea9d // udot v29.4s, v20.16b, v11.4b[2]\n"
+ ".inst 0x6f8eea9e // udot v30.4s, v20.16b, v14.4b[2]\n"
+ ".inst 0x6f91ea9f // udot v31.4s, v20.16b, v17.4b[2]\n"
+ "b 9f\n"
+ "6:\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n"
+ ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n"
+ ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n"
+ ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n"
+ ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n"
".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n"
"ldr q18, [%[b_ptr0]]\n"
+ ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n"
".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n"
".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n"
".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n"
@@ -1350,19 +1529,14 @@
".inst 0x6f8bea9d // udot v29.4s, v20.16b, v11.4b[2]\n"
".inst 0x6f8eea9e // udot v30.4s, v20.16b, v14.4b[2]\n"
".inst 0x6f91ea9f // udot v31.4s, v20.16b, v17.4b[2]\n"
- "6:\n"
+ "9:\n"
"str q26, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q27, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, #0x10\n"
"str q28, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, #0x10\n"
"str q29, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, #0x10\n"
"str q30, [c_ptr4]\n"
- "add c_ptr4, c_ptr4, #0x10\n"
"str q31, [c_ptr5]\n"
- "add c_ptr5, c_ptr5, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -1429,6 +1603,7 @@
"add c_ptr1, %[c_ptr0], #0x0\n"
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
+ "cbnz %[odds], 2f\n"
"ldr q0, [%[a_ptr0]], #0x10\n"
"ldr q3, [a_ptr1], #0x10\n"
"ldr q6, [a_ptr2], #0x10\n"
@@ -1441,7 +1616,6 @@
"ldr q10, [a_ptr3], #0x10\n"
"ldr q13, [a_ptr4], #0x10\n"
"ldr q16, [a_ptr5], #0x10\n"
- "cbnz %[odds], 2f\n"
"ldr q2, [%[a_ptr0]]\n"
"ldr q5, [a_ptr1]\n"
"ldr q8, [a_ptr2]\n"
@@ -1450,8 +1624,21 @@
"ldr q17, [a_ptr5]\n"
"b 3f\n"
"2:\n"
+ "ldr q0, [%[a_ptr0]], #0x10\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "ldr q3, [a_ptr1], #0x10\n"
+ "ldr q6, [a_ptr2], #0x10\n"
+ "ldr q9, [a_ptr3], #0x10\n"
+ "ldr q12, [a_ptr4], #0x10\n"
+ "ldr q15, [a_ptr5], #0x10\n"
+ "ldr q1, [%[a_ptr0]], #0x10\n"
+ "ldr q4, [a_ptr1], #0x10\n"
+ "ldr q7, [a_ptr2], #0x10\n"
+ "ldr q10, [a_ptr3], #0x10\n"
"ldr d2, [%[a_ptr0]], #0x8\n"
+ "ldr q13, [a_ptr4], #0x10\n"
"ldr d5, [a_ptr1], #0x8\n"
+ "ldr q16, [a_ptr5], #0x10\n"
"ldr d8, [a_ptr2], #0x8\n"
"ldr d11, [a_ptr3], #0x8\n"
"ldr d14, [a_ptr4], #0x8\n"
@@ -1462,7 +1649,6 @@
"ld1 {v11.s}[2], [a_ptr3], #4\n"
"ld1 {v14.s}[2], [a_ptr4], #4\n"
"ld1 {v17.s}[2], [a_ptr5], #4\n"
- "subs %[odds], %[odds], #0x1\n"
"b.ne 4f\n"
"ld1 {v2.b}[12], [%[a_ptr0]]\n"
"ld1 {v5.b}[12], [a_ptr1]\n"
@@ -1489,38 +1675,40 @@
"ld1 {v14.b}[14], [a_ptr4]\n"
"ld1 {v17.b}[14], [a_ptr5]\n"
"3:\n"
- "movi v26.4s, #0\n"
"ldr q18, [%[b_ptr0]]\n"
- "movi v27.4s, #0\n"
- "ldr q19, [%[b_ptr0], #0x10]\n"
- "movi v28.4s, #0\n"
- "ldr q20, [%[b_ptr0], #0x20]\n"
- "movi v29.4s, #0\n"
- "ldr q21, [%[b_ptr0], #0x30]\n"
- "movi v30.4s, #0\n"
- "ldr q22, [%[b_ptr0], #0x40]\n"
- "movi v31.4s, #0\n"
- "ldr q23, [%[b_ptr0], #0x50]\n"
- ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n"
- "ldr q24, [%[b_ptr0], #0x60]\n"
- ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n"
- "ldr q25, [%[b_ptr0], #0x70]\n"
- ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x40]\n"
- ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x80]\n"
- ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
"prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
- ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x100]\n"
- ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n"
+ "ldr q22, [%[b_ptr0], #0x40]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x140]\n"
- ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n"
+ "ldr q23, [%[b_ptr0], #0x50]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x180]\n"
- ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n"
+ "ldr q24, [%[b_ptr0], #0x60]\n"
+ "ldr q25, [%[b_ptr0], #0x70]\n"
"add %[b_ptr0], %[b_ptr0], #0x80\n"
- ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n"
+ "cbz %[loops], 6f\n"
+ "movi v26.4s, #0\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n"
+ ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n"
+ ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n"
+ ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n"
+ ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n"
+ ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n"
"ldr q18, [%[b_ptr0]]\n"
+ ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n"
+ ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n"
+ ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n"
+ ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n"
".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n"
".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n"
"ldr q19, [%[b_ptr0], #0x10]\n"
@@ -1587,198 +1775,265 @@
".inst 0x6fabeabd // udot v29.4s, v21.16b, v11.4b[3]\n"
".inst 0x6faeeabe // udot v30.4s, v21.16b, v14.4b[3]\n"
".inst 0x6fb1eabf // udot v31.4s, v21.16b, v17.4b[3]\n"
- "cbz %[loops], 6f\n"
- "ldr d18, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ldr temploadreg2, [%[b_ptr0], #0x8]\n"
- "ldr d19, [%[b_ptr0], #0x10]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x18]\n"
- "ldr d20, [%[b_ptr0], #0x20]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x28]\n"
- "ldr d21, [%[b_ptr0], #0x30]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x38]\n"
- "ldr d22, [%[b_ptr0], #0x40]\n"
- "ins v18.d[1], temploadreg2\n"
- "ldr temploadreg2, [%[b_ptr0], #0x48]\n"
- "ldr d23, [%[b_ptr0], #0x50]\n"
- "ins v19.d[1], temploadreg3\n"
- "ldr temploadreg3, [%[b_ptr0], #0x58]\n"
- "ldr d24, [%[b_ptr0], #0x60]\n"
- "ins v20.d[1], temploadreg0\n"
- "ldr temploadreg0, [%[b_ptr0], #0x68]\n"
- "ldr d25, [%[b_ptr0], #0x70]\n"
- "ins v21.d[1], temploadreg1\n"
- "ldr temploadreg1, [%[b_ptr0], #0x78]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "ins v22.d[1], temploadreg2\n"
- "ins v23.d[1], temploadreg3\n"
- "ins v24.d[1], temploadreg0\n"
"b.eq 7f\n"
"8:\n"
"str q26, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"movi v26.4s, #0\n"
- "ins v25.d[1], temploadreg1\n"
+ "ldr d18, [%[b_ptr0]]\n"
"ldr temploadreg2, [%[b_ptr0], #0x8]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q27, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
- "ldr temploadreg3, [%[b_ptr0], #0x18]\n"
- ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n"
- "str q28, [c_ptr2]\n"
- "movi v28.4s, #0\n"
- "ldr temploadreg0, [%[b_ptr0], #0x28]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x38]\n"
- "add c_ptr2, c_ptr2, #0x10\n"
- ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n"
- "str q29, [c_ptr3]\n"
- "movi v29.4s, #0\n"
- "add c_ptr3, c_ptr3, #0x10\n"
- ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n"
- "str q30, [c_ptr4]\n"
- "movi v30.4s, #0\n"
- "add c_ptr4, c_ptr4, #0x10\n"
- ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n"
- "str q31, [c_ptr5]\n"
- "movi v31.4s, #0\n"
- "add c_ptr5, c_ptr5, #0x10\n"
- ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
- ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n"
- "ldr d18, [%[b_ptr0]]\n"
- ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n"
- "ins v18.d[1], temploadreg2\n"
- ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n"
"ldr d19, [%[b_ptr0], #0x10]\n"
- ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n"
- "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n"
- "ins v19.d[1], temploadreg3\n"
- ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n"
- ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n"
- ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n"
- ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n"
+ "ldr temploadreg3, [%[b_ptr0], #0x18]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "str q28, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ "movi v28.4s, #0\n"
"ldr d20, [%[b_ptr0], #0x20]\n"
- ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n"
- ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n"
- ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n"
+ "ldr temploadreg0, [%[b_ptr0], #0x28]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ "str q29, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ "movi v29.4s, #0\n"
+ "ldr d21, [%[b_ptr0], #0x30]\n"
+ "ldr temploadreg1, [%[b_ptr0], #0x38]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ "str q30, [c_ptr4]\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ "movi v30.4s, #0\n"
+ "ldr d22, [%[b_ptr0], #0x40]\n"
+ "ins v18.d[1], temploadreg2\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
+ "str q31, [c_ptr5]\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ "movi v31.4s, #0\n"
+ "ldr temploadreg2, [%[b_ptr0], #0x48]\n"
+ ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n"
+ "ldr d23, [%[b_ptr0], #0x50]\n"
+ ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n"
+ "ins v19.d[1], temploadreg3\n"
+ ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n"
+ "ldr temploadreg3, [%[b_ptr0], #0x58]\n"
+ ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n"
+ "ldr d24, [%[b_ptr0], #0x60]\n"
+ ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n"
"ins v20.d[1], temploadreg0\n"
+ ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n"
+ "ldr temploadreg0, [%[b_ptr0], #0x68]\n"
+ ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n"
+ "ldr d25, [%[b_ptr0], #0x70]\n"
+ ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n"
+ "ins v21.d[1], temploadreg1\n"
+ ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n"
+ "ldr temploadreg1, [%[b_ptr0], #0x78]\n"
+ ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n"
+ "ins v22.d[1], temploadreg2\n"
+ ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n"
+ "ins v23.d[1], temploadreg3\n"
+ ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n"
+ "ins v24.d[1], temploadreg0\n"
+ ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n"
+ "ins v25.d[1], temploadreg1\n"
+ ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
+ ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n"
+ "ldr d18, [%[b_ptr0]]\n"
+ ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n"
+ "ldr temploadreg2, [%[b_ptr0], #0x8]\n"
+ ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n"
+ "ldr d19, [%[b_ptr0], #0x10]\n"
+ ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n"
+ "ldr temploadreg3, [%[b_ptr0], #0x18]\n"
+ ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n"
+ "ldr d20, [%[b_ptr0], #0x20]\n"
+ ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n"
+ "ldr temploadreg0, [%[b_ptr0], #0x28]\n"
+ ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n"
+ "ldr temploadreg1, [%[b_ptr0], #0x38]\n"
".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n"
+ "ins v18.d[1], temploadreg2\n"
".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n"
+ "ins v19.d[1], temploadreg3\n"
".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n"
"ldr d21, [%[b_ptr0], #0x30]\n"
".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n"
- "add %[b_ptr0], %[b_ptr0], #0x40\n"
+ "ins v20.d[1], temploadreg0\n"
".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n"
- "ins v21.d[1], temploadreg1\n"
+ "add %[b_ptr0], %[b_ptr0], #0x40\n"
".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x8]\n"
+ "ins v21.d[1], temploadreg1\n"
".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x18]\n"
+ "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x28]\n"
+ "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x38]\n"
".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n"
- "ldr d22, [%[b_ptr0], #0x40]\n"
".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n"
".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n"
".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n"
".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n"
".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n"
- "ldr d23, [%[b_ptr0], #0x50]\n"
".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n"
".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n"
".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n"
".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n"
- "ldr d24, [%[b_ptr0], #0x60]\n"
".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n"
".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n"
".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n"
".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n"
- "ldr d25, [%[b_ptr0], #0x70]\n"
".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n"
".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n"
".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n"
".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n"
".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n"
".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n"
- "ldr d18, [%[b_ptr0]]\n"
".inst 0x6fa2e27a // udot v26.4s, v19.16b, v2.4b[1]\n"
".inst 0x6fa5e27b // udot v27.4s, v19.16b, v5.4b[1]\n"
".inst 0x6fa8e27c // udot v28.4s, v19.16b, v8.4b[1]\n"
- "ins v18.d[1], temploadreg2\n"
".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x48]\n"
".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n"
".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n"
- "ldr d19, [%[b_ptr0], #0x10]\n"
".inst 0x6f82ea9a // udot v26.4s, v20.16b, v2.4b[2]\n"
- "ins v22.d[1], temploadreg2\n"
".inst 0x6f85ea9b // udot v27.4s, v20.16b, v5.4b[2]\n"
".inst 0x6f88ea9c // udot v28.4s, v20.16b, v8.4b[2]\n"
- "ins v19.d[1], temploadreg3\n"
".inst 0x6f8bea9d // udot v29.4s, v20.16b, v11.4b[2]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x58]\n"
".inst 0x6f8eea9e // udot v30.4s, v20.16b, v14.4b[2]\n"
".inst 0x6f91ea9f // udot v31.4s, v20.16b, v17.4b[2]\n"
- "ldr d20, [%[b_ptr0], #0x20]\n"
".inst 0x6fa2eaba // udot v26.4s, v21.16b, v2.4b[3]\n"
- "ins v23.d[1], temploadreg3\n"
".inst 0x6fa5eabb // udot v27.4s, v21.16b, v5.4b[3]\n"
".inst 0x6fa8eabc // udot v28.4s, v21.16b, v8.4b[3]\n"
- "ins v20.d[1], temploadreg0\n"
".inst 0x6fabeabd // udot v29.4s, v21.16b, v11.4b[3]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x68]\n"
".inst 0x6faeeabe // udot v30.4s, v21.16b, v14.4b[3]\n"
".inst 0x6fb1eabf // udot v31.4s, v21.16b, v17.4b[3]\n"
- "ldr d21, [%[b_ptr0], #0x30]\n"
- "ins v24.d[1], temploadreg0\n"
- "ins v21.d[1], temploadreg1\n"
- "ldr temploadreg1, [%[b_ptr0], #0x78]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
"b.ne 8b\n"
"7:\n"
"str q26, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v26.4s, #0\n"
- "ins v25.d[1], temploadreg1\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
"str q27, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n"
"str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
+ "ldr q22, [%[b_ptr0], #0x40]\n"
"add c_ptr2, c_ptr2, #0x10\n"
".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n"
"str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
- "add c_ptr3, c_ptr3, #0x10\n"
+ "ldr q23, [%[b_ptr0], #0x50]\n"
".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n"
+ "ldr q24, [%[b_ptr0], #0x60]\n"
+ ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n"
"str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
- "add c_ptr4, c_ptr4, #0x10\n"
+ "ldr q25, [%[b_ptr0], #0x70]\n"
".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n"
"str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
- "add c_ptr5, c_ptr5, #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n"
- ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
+ ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n"
+ ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n"
+ ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
+ ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n"
+ ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n"
+ ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n"
+ ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n"
+ ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n"
+ ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
+ ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n"
+ ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n"
+ ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n"
+ ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n"
+ ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n"
+ ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
+ ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x40\n"
+ ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n"
+ ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n"
+ ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n"
+ ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n"
+ ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n"
+ ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n"
+ ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n"
+ ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n"
+ ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n"
+ ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n"
+ ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n"
+ ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n"
+ ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n"
+ ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n"
+ ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n"
+ ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n"
+ ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n"
+ ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n"
+ ".inst 0x6fa2e27a // udot v26.4s, v19.16b, v2.4b[1]\n"
+ ".inst 0x6fa5e27b // udot v27.4s, v19.16b, v5.4b[1]\n"
+ ".inst 0x6fa8e27c // udot v28.4s, v19.16b, v8.4b[1]\n"
+ ".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n"
+ ".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n"
+ ".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n"
+ ".inst 0x6f82ea9a // udot v26.4s, v20.16b, v2.4b[2]\n"
+ ".inst 0x6f85ea9b // udot v27.4s, v20.16b, v5.4b[2]\n"
+ ".inst 0x6f88ea9c // udot v28.4s, v20.16b, v8.4b[2]\n"
+ ".inst 0x6f8bea9d // udot v29.4s, v20.16b, v11.4b[2]\n"
+ ".inst 0x6f8eea9e // udot v30.4s, v20.16b, v14.4b[2]\n"
+ ".inst 0x6f91ea9f // udot v31.4s, v20.16b, v17.4b[2]\n"
+ ".inst 0x6fa2eaba // udot v26.4s, v21.16b, v2.4b[3]\n"
+ ".inst 0x6fa5eabb // udot v27.4s, v21.16b, v5.4b[3]\n"
+ ".inst 0x6fa8eabc // udot v28.4s, v21.16b, v8.4b[3]\n"
+ ".inst 0x6fabeabd // udot v29.4s, v21.16b, v11.4b[3]\n"
+ ".inst 0x6faeeabe // udot v30.4s, v21.16b, v14.4b[3]\n"
+ ".inst 0x6fb1eabf // udot v31.4s, v21.16b, v17.4b[3]\n"
+ "b 9f\n"
+ "6:\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n"
+ ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n"
+ ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n"
+ ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n"
+ ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n"
".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n"
"ldr q18, [%[b_ptr0]]\n"
+ ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n"
".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n"
".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n"
".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n"
@@ -1848,19 +2103,14 @@
".inst 0x6fabeabd // udot v29.4s, v21.16b, v11.4b[3]\n"
".inst 0x6faeeabe // udot v30.4s, v21.16b, v14.4b[3]\n"
".inst 0x6fb1eabf // udot v31.4s, v21.16b, v17.4b[3]\n"
- "6:\n"
+ "9:\n"
"str q26, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q27, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, #0x10\n"
"str q28, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, #0x10\n"
"str q29, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, #0x10\n"
"str q30, [c_ptr4]\n"
- "add c_ptr4, c_ptr4, #0x10\n"
"str q31, [c_ptr5]\n"
- "add c_ptr5, c_ptr5, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -1927,6 +2177,7 @@
"add c_ptr1, %[c_ptr0], #0x0\n"
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
+ "cbnz %[odds], 2f\n"
"ldr q0, [%[a_ptr0]], #0x10\n"
"ldr q4, [a_ptr1], #0x10\n"
"ldr q8, [a_ptr2], #0x10\n"
@@ -1943,18 +2194,35 @@
"ldr q6, [a_ptr1], #0x10\n"
"ldr q10, [a_ptr2], #0x10\n"
"ldr q14, [a_ptr3], #0x10\n"
+ "ldr s3, [%[a_ptr0]]\n"
+ "ldr q18, [a_ptr4], #0x10\n"
+ "ldr s7, [a_ptr1]\n"
+ "ldr q22, [a_ptr5], #0x10\n"
+ "ldr s11, [a_ptr2]\n"
+ "ldr s15, [a_ptr3]\n"
+ "ldr s19, [a_ptr4]\n"
+ "ldr s23, [a_ptr5]\n"
+ "b 3f\n"
+ "2:\n"
+ "ldr q0, [%[a_ptr0]], #0x10\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "ldr q4, [a_ptr1], #0x10\n"
+ "ldr q8, [a_ptr2], #0x10\n"
+ "ldr q12, [a_ptr3], #0x10\n"
+ "ldr q16, [a_ptr4], #0x10\n"
+ "ldr q20, [a_ptr5], #0x10\n"
+ "ldr q1, [%[a_ptr0]], #0x10\n"
+ "ldr q5, [a_ptr1], #0x10\n"
+ "ldr q9, [a_ptr2], #0x10\n"
+ "ldr q13, [a_ptr3], #0x10\n"
+ "ldr q17, [a_ptr4], #0x10\n"
+ "ldr q21, [a_ptr5], #0x10\n"
+ "ldr q2, [%[a_ptr0]], #0x10\n"
+ "ldr q6, [a_ptr1], #0x10\n"
+ "ldr q10, [a_ptr2], #0x10\n"
+ "ldr q14, [a_ptr3], #0x10\n"
"ldr q18, [a_ptr4], #0x10\n"
"ldr q22, [a_ptr5], #0x10\n"
- "cbnz %[odds], 2f\n"
- "ldr s3, [%[a_ptr0]]\n"
- "ldr s7, [a_ptr1]\n"
- "ldr s11, [a_ptr2]\n"
- "ldr s15, [a_ptr3]\n"
- "ldr s19, [a_ptr4]\n"
- "ldr s23, [a_ptr5]\n"
- "b 3f\n"
- "2:\n"
- "subs %[odds], %[odds], #0x1\n"
"b.ne 4f\n"
"ldr b3, [%[a_ptr0]]\n"
"ldr b7, [a_ptr1]\n"
@@ -1981,24 +2249,26 @@
"ld1 {v19.b}[2], [a_ptr4]\n"
"ld1 {v23.b}[2], [a_ptr5]\n"
"3:\n"
- "movi v26.4s, #0\n"
"ldr q24, [%[b_ptr0]]\n"
- "movi v27.4s, #0\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- "movi v28.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr5, #0x40]\n"
- "movi v29.4s, #0\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x80]\n"
- "movi v30.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
- "movi v31.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr5, #0x100]\n"
- ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x140]\n"
- ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x180]\n"
- ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "cbz %[loops], 6f\n"
+ "movi v26.4s, #0\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
@@ -2091,57 +2361,55 @@
".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n"
".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n"
".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n"
- "cbz %[loops], 6f\n"
- "ldr d24, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- "ldr d25, [%[b_ptr0], #0x10]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- "ins v24.d[1], temploadreg0\n"
- "ins v25.d[1], temploadreg1\n"
"b.eq 7f\n"
"8:\n"
"str q26, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"movi v26.4s, #0\n"
+ "ldr d24, [%[b_ptr0]]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q27, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
- ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
+ "ldr d25, [%[b_ptr0], #0x10]\n"
+ "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
"str q28, [c_ptr2]\n"
- "movi v28.4s, #0\n"
"add c_ptr2, c_ptr2, #0x10\n"
- ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
+ "movi v28.4s, #0\n"
+ "ins v24.d[1], temploadreg0\n"
+ "ins v25.d[1], temploadreg1\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
"str q29, [c_ptr3]\n"
- "movi v29.4s, #0\n"
"add c_ptr3, c_ptr3, #0x10\n"
- ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
+ "movi v29.4s, #0\n"
+ "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
+ ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
"str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
+ "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
+ ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
"add c_ptr4, c_ptr4, #0x10\n"
- ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
+ ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
"str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
"add c_ptr5, c_ptr5, #0x10\n"
- ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
+ ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
"prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
"ldr d24, [%[b_ptr0]]\n"
".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n"
"ins v24.d[1], temploadreg0\n"
".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
+ ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
+ ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n"
"ldr d25, [%[b_ptr0], #0x10]\n"
".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n"
@@ -2235,27 +2503,23 @@
".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n"
"ins v24.d[1], temploadreg0\n"
".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n"
".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n"
- "ldr d25, [%[b_ptr0], #0x10]\n"
".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n"
".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n"
".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n"
- "ins v25.d[1], temploadreg1\n"
".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n"
".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n"
".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n"
- "ldr d24, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- "ins v24.d[1], temploadreg0\n"
"b.ne 8b\n"
"7:\n"
"str q26, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v26.4s, #0\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
"str q27, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
@@ -2366,19 +2630,117 @@
".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n"
".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n"
".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n"
+ "b 9f\n"
"6:\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
+ ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
+ ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
+ ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
+ ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n"
+ ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n"
+ ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
+ ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
+ ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n"
+ ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n"
+ ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n"
+ ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n"
+ ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n"
+ ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n"
+ ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n"
+ ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n"
+ ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n"
+ ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n"
+ ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n"
+ ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n"
+ ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n"
+ ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n"
+ ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n"
+ ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n"
+ ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n"
+ ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n"
+ ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n"
+ ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n"
+ ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n"
+ ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n"
+ ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n"
+ ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n"
+ ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n"
+ ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n"
+ ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n"
+ ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n"
+ ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n"
+ ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n"
+ ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n"
+ ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n"
+ ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x10\n"
+ ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n"
+ ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n"
+ ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n"
+ ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n"
+ ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n"
+ ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n"
+ ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n"
+ ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n"
+ ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n"
+ ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n"
+ "9:\n"
"str q26, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q27, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, #0x10\n"
"str q28, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, #0x10\n"
"str q29, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, #0x10\n"
"str q30, [c_ptr4]\n"
- "add c_ptr4, c_ptr4, #0x10\n"
"str q31, [c_ptr5]\n"
- "add c_ptr5, c_ptr5, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -2445,6 +2807,7 @@
"add c_ptr1, %[c_ptr0], #0x0\n"
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
+ "cbnz %[odds], 2f\n"
"ldr q0, [%[a_ptr0]], #0x10\n"
"ldr q4, [a_ptr1], #0x10\n"
"ldr q8, [a_ptr2], #0x10\n"
@@ -2461,24 +2824,41 @@
"ldr q6, [a_ptr1], #0x10\n"
"ldr q10, [a_ptr2], #0x10\n"
"ldr q14, [a_ptr3], #0x10\n"
- "ldr q18, [a_ptr4], #0x10\n"
- "ldr q22, [a_ptr5], #0x10\n"
- "cbnz %[odds], 2f\n"
"ldr d3, [%[a_ptr0]]\n"
+ "ldr q18, [a_ptr4], #0x10\n"
"ldr d7, [a_ptr1]\n"
+ "ldr q22, [a_ptr5], #0x10\n"
"ldr d11, [a_ptr2]\n"
"ldr d15, [a_ptr3]\n"
"ldr d19, [a_ptr4]\n"
"ldr d23, [a_ptr5]\n"
"b 3f\n"
"2:\n"
+ "ldr q0, [%[a_ptr0]], #0x10\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "ldr q4, [a_ptr1], #0x10\n"
+ "ldr q8, [a_ptr2], #0x10\n"
+ "ldr q12, [a_ptr3], #0x10\n"
+ "ldr q16, [a_ptr4], #0x10\n"
+ "ldr q20, [a_ptr5], #0x10\n"
+ "ldr q1, [%[a_ptr0]], #0x10\n"
+ "ldr q5, [a_ptr1], #0x10\n"
+ "ldr q9, [a_ptr2], #0x10\n"
+ "ldr q13, [a_ptr3], #0x10\n"
+ "ldr q17, [a_ptr4], #0x10\n"
+ "ldr q21, [a_ptr5], #0x10\n"
+ "ldr q2, [%[a_ptr0]], #0x10\n"
+ "ldr q6, [a_ptr1], #0x10\n"
+ "ldr q10, [a_ptr2], #0x10\n"
+ "ldr q14, [a_ptr3], #0x10\n"
"ldr s3, [%[a_ptr0]], #0x4\n"
+ "ldr q18, [a_ptr4], #0x10\n"
"ldr s7, [a_ptr1], #0x4\n"
+ "ldr q22, [a_ptr5], #0x10\n"
"ldr s11, [a_ptr2], #0x4\n"
"ldr s15, [a_ptr3], #0x4\n"
"ldr s19, [a_ptr4], #0x4\n"
"ldr s23, [a_ptr5], #0x4\n"
- "subs %[odds], %[odds], #0x1\n"
"b.ne 4f\n"
"ld1 {v3.b}[4], [%[a_ptr0]]\n"
"ld1 {v7.b}[4], [a_ptr1]\n"
@@ -2505,24 +2885,26 @@
"ld1 {v19.b}[6], [a_ptr4]\n"
"ld1 {v23.b}[6], [a_ptr5]\n"
"3:\n"
- "movi v26.4s, #0\n"
"ldr q24, [%[b_ptr0]]\n"
- "movi v27.4s, #0\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- "movi v28.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr5, #0x40]\n"
- "movi v29.4s, #0\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x80]\n"
- "movi v30.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
- "movi v31.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr5, #0x100]\n"
- ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x140]\n"
- ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x180]\n"
- ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "cbz %[loops], 6f\n"
+ "movi v26.4s, #0\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
@@ -2622,68 +3004,66 @@
".inst 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n"
".inst 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n"
".inst 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n"
- "cbz %[loops], 6f\n"
- "ldr d24, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- "ldr d25, [%[b_ptr0], #0x10]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- "ins v24.d[1], temploadreg0\n"
"b.eq 7f\n"
"8:\n"
"str q26, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"movi v26.4s, #0\n"
- "ins v25.d[1], temploadreg1\n"
+ "ldr d24, [%[b_ptr0]]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q27, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
+ "ldr d25, [%[b_ptr0], #0x10]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
"str q28, [c_ptr2]\n"
- "movi v28.4s, #0\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
- ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
- "str q29, [c_ptr3]\n"
- "movi v29.4s, #0\n"
"add c_ptr2, c_ptr2, #0x10\n"
- ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
+ "movi v28.4s, #0\n"
+ "ins v24.d[1], temploadreg0\n"
+ "ins v25.d[1], temploadreg1\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "str q29, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ "movi v29.4s, #0\n"
+ "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
+ ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
"str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
- "add c_ptr3, c_ptr3, #0x10\n"
- ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
+ "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
+ ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
"str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
- "add c_ptr4, c_ptr4, #0x10\n"
- ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
"add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
"ldr d24, [%[b_ptr0]]\n"
".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n"
"ins v24.d[1], temploadreg0\n"
".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
+ ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
+ ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n"
"ldr d25, [%[b_ptr0], #0x10]\n"
".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n"
- "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
"ins v25.d[1], temploadreg1\n"
".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
+ ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n"
".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n"
"ldr d24, [%[b_ptr0]]\n"
".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n"
@@ -2775,27 +3155,23 @@
".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n"
"ins v25.d[1], temploadreg1\n"
".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n"
".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n"
- "ldr d24, [%[b_ptr0]]\n"
".inst 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n"
".inst 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n"
".inst 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n"
- "ins v24.d[1], temploadreg0\n"
".inst 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n"
".inst 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n"
".inst 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n"
- "ldr d25, [%[b_ptr0], #0x10]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
"b.ne 8b\n"
"7:\n"
"str q26, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v26.4s, #0\n"
- "ins v25.d[1], temploadreg1\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
"str q27, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
@@ -2913,19 +3289,124 @@
".inst 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n"
".inst 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n"
".inst 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n"
+ "b 9f\n"
"6:\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
+ ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
+ ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
+ ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
+ ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n"
+ ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n"
+ ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
+ ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
+ ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n"
+ ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n"
+ ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n"
+ ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n"
+ ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n"
+ ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n"
+ ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n"
+ ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n"
+ ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n"
+ ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n"
+ ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n"
+ ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n"
+ ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n"
+ ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n"
+ ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n"
+ ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n"
+ ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n"
+ ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n"
+ ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n"
+ ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n"
+ ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n"
+ ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n"
+ ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n"
+ ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n"
+ ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n"
+ ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n"
+ ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n"
+ ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n"
+ ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n"
+ ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n"
+ ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n"
+ ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n"
+ ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n"
+ ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n"
+ ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n"
+ ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n"
+ ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n"
+ ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n"
+ ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n"
+ ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n"
+ ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n"
+ ".inst 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n"
+ ".inst 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n"
+ ".inst 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n"
+ ".inst 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n"
+ ".inst 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n"
+ ".inst 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n"
+ "9:\n"
"str q26, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q27, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, #0x10\n"
"str q28, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, #0x10\n"
"str q29, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, #0x10\n"
"str q30, [c_ptr4]\n"
- "add c_ptr4, c_ptr4, #0x10\n"
"str q31, [c_ptr5]\n"
- "add c_ptr5, c_ptr5, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -3052,24 +3533,26 @@
"ld1 {v19.b}[10], [a_ptr4]\n"
"ld1 {v23.b}[10], [a_ptr5]\n"
"3:\n"
- "movi v26.4s, #0\n"
"ldr q24, [%[b_ptr0]]\n"
- "movi v27.4s, #0\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- "movi v28.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr5, #0x40]\n"
- "movi v29.4s, #0\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x80]\n"
- "movi v30.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
- "movi v31.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr5, #0x100]\n"
- ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x140]\n"
- ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x180]\n"
- ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "cbz %[loops], 6f\n"
+ "movi v26.4s, #0\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
@@ -3177,57 +3660,55 @@
".inst 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n"
".inst 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n"
".inst 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n"
- "cbz %[loops], 6f\n"
- "ldr d24, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- "ldr d25, [%[b_ptr0], #0x10]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- "ins v24.d[1], temploadreg0\n"
- "ins v25.d[1], temploadreg1\n"
"b.eq 7f\n"
"8:\n"
"str q26, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"movi v26.4s, #0\n"
+ "ldr d24, [%[b_ptr0]]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q27, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
- ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
+ "ldr d25, [%[b_ptr0], #0x10]\n"
+ "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
"str q28, [c_ptr2]\n"
- "movi v28.4s, #0\n"
"add c_ptr2, c_ptr2, #0x10\n"
- ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
+ "movi v28.4s, #0\n"
+ "ins v24.d[1], temploadreg0\n"
+ "ins v25.d[1], temploadreg1\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
"str q29, [c_ptr3]\n"
- "movi v29.4s, #0\n"
"add c_ptr3, c_ptr3, #0x10\n"
- ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
+ "movi v29.4s, #0\n"
+ "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
+ ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
"str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
+ "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
+ ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
"add c_ptr4, c_ptr4, #0x10\n"
- ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
+ ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
"str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
"add c_ptr5, c_ptr5, #0x10\n"
- ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
+ ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
"prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
"ldr d24, [%[b_ptr0]]\n"
".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n"
"ins v24.d[1], temploadreg0\n"
".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
+ ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
+ ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n"
"ldr d25, [%[b_ptr0], #0x10]\n"
".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n"
@@ -3340,27 +3821,23 @@
".inst 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n"
"ins v24.d[1], temploadreg0\n"
".inst 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
".inst 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
".inst 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n"
".inst 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n"
- "ldr d25, [%[b_ptr0], #0x10]\n"
".inst 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n"
".inst 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n"
".inst 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n"
- "ins v25.d[1], temploadreg1\n"
".inst 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n"
".inst 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n"
".inst 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n"
- "ldr d24, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- "ins v24.d[1], temploadreg0\n"
"b.ne 8b\n"
"7:\n"
"str q26, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v26.4s, #0\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
"str q27, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
@@ -3486,19 +3963,132 @@
".inst 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n"
".inst 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n"
".inst 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n"
+ "b 9f\n"
"6:\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
+ ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
+ ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
+ ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
+ ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n"
+ ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n"
+ ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
+ ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
+ ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n"
+ ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n"
+ ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n"
+ ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n"
+ ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n"
+ ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n"
+ ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n"
+ ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n"
+ ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n"
+ ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n"
+ ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n"
+ ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n"
+ ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n"
+ ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n"
+ ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n"
+ ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n"
+ ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n"
+ ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n"
+ ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n"
+ ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n"
+ ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n"
+ ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n"
+ ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n"
+ ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n"
+ ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n"
+ ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n"
+ ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n"
+ ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n"
+ ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n"
+ ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n"
+ ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n"
+ ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n"
+ ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n"
+ ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n"
+ ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n"
+ ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n"
+ ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n"
+ ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n"
+ ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n"
+ ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n"
+ ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x10\n"
+ ".inst 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n"
+ ".inst 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n"
+ ".inst 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n"
+ ".inst 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n"
+ ".inst 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n"
+ ".inst 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n"
+ ".inst 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n"
+ ".inst 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n"
+ ".inst 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n"
+ ".inst 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n"
+ "9:\n"
"str q26, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q27, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, #0x10\n"
"str q28, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, #0x10\n"
"str q29, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, #0x10\n"
"str q30, [c_ptr4]\n"
- "add c_ptr4, c_ptr4, #0x10\n"
"str q31, [c_ptr5]\n"
- "add c_ptr5, c_ptr5, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -3566,6 +4156,7 @@
"add c_ptr1, %[c_ptr0], #0x0\n"
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
+ "cbnz %[odds], 2f\n"
"ldr q0, [%[a_ptr0]], #0x10\n"
"ldr q4, [a_ptr1], #0x10\n"
"ldr q8, [a_ptr2], #0x10\n"
@@ -3584,7 +4175,6 @@
"ldr q14, [a_ptr3], #0x10\n"
"ldr q18, [a_ptr4], #0x10\n"
"ldr q22, [a_ptr5], #0x10\n"
- "cbnz %[odds], 2f\n"
"ldr q3, [%[a_ptr0]]\n"
"ldr q7, [a_ptr1]\n"
"ldr q11, [a_ptr2]\n"
@@ -3593,8 +4183,27 @@
"ldr q23, [a_ptr5]\n"
"b 3f\n"
"2:\n"
+ "ldr q0, [%[a_ptr0]], #0x10\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "ldr q4, [a_ptr1], #0x10\n"
+ "ldr q8, [a_ptr2], #0x10\n"
+ "ldr q12, [a_ptr3], #0x10\n"
+ "ldr q16, [a_ptr4], #0x10\n"
+ "ldr q20, [a_ptr5], #0x10\n"
+ "ldr q1, [%[a_ptr0]], #0x10\n"
+ "ldr q5, [a_ptr1], #0x10\n"
+ "ldr q9, [a_ptr2], #0x10\n"
+ "ldr q13, [a_ptr3], #0x10\n"
+ "ldr q17, [a_ptr4], #0x10\n"
+ "ldr q21, [a_ptr5], #0x10\n"
+ "ldr q2, [%[a_ptr0]], #0x10\n"
+ "ldr q6, [a_ptr1], #0x10\n"
+ "ldr q10, [a_ptr2], #0x10\n"
+ "ldr q14, [a_ptr3], #0x10\n"
"ldr d3, [%[a_ptr0]], #0x8\n"
+ "ldr q18, [a_ptr4], #0x10\n"
"ldr d7, [a_ptr1], #0x8\n"
+ "ldr q22, [a_ptr5], #0x10\n"
"ldr d11, [a_ptr2], #0x8\n"
"ldr d15, [a_ptr3], #0x8\n"
"ldr d19, [a_ptr4], #0x8\n"
@@ -3605,7 +4214,6 @@
"ld1 {v15.s}[2], [a_ptr3], #4\n"
"ld1 {v19.s}[2], [a_ptr4], #4\n"
"ld1 {v23.s}[2], [a_ptr5], #4\n"
- "subs %[odds], %[odds], #0x1\n"
"b.ne 4f\n"
"ld1 {v3.b}[12], [%[a_ptr0]]\n"
"ld1 {v7.b}[12], [a_ptr1]\n"
@@ -3632,24 +4240,26 @@
"ld1 {v19.b}[14], [a_ptr4]\n"
"ld1 {v23.b}[14], [a_ptr5]\n"
"3:\n"
- "movi v26.4s, #0\n"
"ldr q24, [%[b_ptr0]]\n"
- "movi v27.4s, #0\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- "movi v28.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr5, #0x40]\n"
- "movi v29.4s, #0\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x80]\n"
- "movi v30.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
- "movi v31.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr5, #0x100]\n"
- ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x140]\n"
- ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x180]\n"
- ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "cbz %[loops], 6f\n"
+ "movi v26.4s, #0\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
@@ -3764,68 +4374,66 @@
".inst 0x6fafeb3d // udot v29.4s, v25.16b, v15.4b[3]\n"
".inst 0x6fb3eb3e // udot v30.4s, v25.16b, v19.4b[3]\n"
".inst 0x6fb7eb3f // udot v31.4s, v25.16b, v23.4b[3]\n"
- "cbz %[loops], 6f\n"
- "ldr d24, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- "ldr d25, [%[b_ptr0], #0x10]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- "ins v24.d[1], temploadreg0\n"
"b.eq 7f\n"
"8:\n"
"str q26, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"movi v26.4s, #0\n"
- "ins v25.d[1], temploadreg1\n"
+ "ldr d24, [%[b_ptr0]]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q27, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
+ "ldr d25, [%[b_ptr0], #0x10]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
"str q28, [c_ptr2]\n"
- "movi v28.4s, #0\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
- ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
- "str q29, [c_ptr3]\n"
- "movi v29.4s, #0\n"
"add c_ptr2, c_ptr2, #0x10\n"
- ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
+ "movi v28.4s, #0\n"
+ "ins v24.d[1], temploadreg0\n"
+ "ins v25.d[1], temploadreg1\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "str q29, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ "movi v29.4s, #0\n"
+ "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
+ ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
"str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
- "add c_ptr3, c_ptr3, #0x10\n"
- ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
+ "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
+ ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
"str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
- "add c_ptr4, c_ptr4, #0x10\n"
- ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
"add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
"ldr d24, [%[b_ptr0]]\n"
".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n"
"ins v24.d[1], temploadreg0\n"
".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
+ ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
+ ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n"
"ldr d25, [%[b_ptr0], #0x10]\n"
".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n"
- "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
"ins v25.d[1], temploadreg1\n"
".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
+ ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n"
".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n"
"ldr d24, [%[b_ptr0]]\n"
".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n"
@@ -3936,27 +4544,23 @@
".inst 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n"
"ins v25.d[1], temploadreg1\n"
".inst 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
".inst 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
".inst 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n"
".inst 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n"
- "ldr d24, [%[b_ptr0]]\n"
".inst 0x6fa3eb3a // udot v26.4s, v25.16b, v3.4b[3]\n"
".inst 0x6fa7eb3b // udot v27.4s, v25.16b, v7.4b[3]\n"
".inst 0x6fabeb3c // udot v28.4s, v25.16b, v11.4b[3]\n"
- "ins v24.d[1], temploadreg0\n"
".inst 0x6fafeb3d // udot v29.4s, v25.16b, v15.4b[3]\n"
".inst 0x6fb3eb3e // udot v30.4s, v25.16b, v19.4b[3]\n"
".inst 0x6fb7eb3f // udot v31.4s, v25.16b, v23.4b[3]\n"
- "ldr d25, [%[b_ptr0], #0x10]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
"b.ne 8b\n"
"7:\n"
"str q26, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v26.4s, #0\n"
- "ins v25.d[1], temploadreg1\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
"str q27, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
@@ -4089,19 +4693,139 @@
".inst 0x6fafeb3d // udot v29.4s, v25.16b, v15.4b[3]\n"
".inst 0x6fb3eb3e // udot v30.4s, v25.16b, v19.4b[3]\n"
".inst 0x6fb7eb3f // udot v31.4s, v25.16b, v23.4b[3]\n"
+ "b 9f\n"
"6:\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
+ ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
+ ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
+ ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
+ ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n"
+ ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n"
+ ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
+ ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
+ ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n"
+ ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n"
+ ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n"
+ ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n"
+ ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n"
+ ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n"
+ ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n"
+ ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n"
+ ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n"
+ ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n"
+ ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n"
+ ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n"
+ ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n"
+ ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n"
+ ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n"
+ ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n"
+ ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n"
+ ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n"
+ ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n"
+ ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n"
+ ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n"
+ ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n"
+ ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n"
+ ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n"
+ ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n"
+ ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n"
+ ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n"
+ ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n"
+ ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n"
+ ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n"
+ ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n"
+ ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n"
+ ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n"
+ ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n"
+ ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n"
+ ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n"
+ ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n"
+ ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n"
+ ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n"
+ ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n"
+ ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n"
+ ".inst 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n"
+ ".inst 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n"
+ ".inst 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n"
+ ".inst 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n"
+ ".inst 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n"
+ ".inst 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n"
+ ".inst 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n"
+ ".inst 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n"
+ ".inst 0x6fa3eb3a // udot v26.4s, v25.16b, v3.4b[3]\n"
+ ".inst 0x6fa7eb3b // udot v27.4s, v25.16b, v7.4b[3]\n"
+ ".inst 0x6fabeb3c // udot v28.4s, v25.16b, v11.4b[3]\n"
+ ".inst 0x6fafeb3d // udot v29.4s, v25.16b, v15.4b[3]\n"
+ ".inst 0x6fb3eb3e // udot v30.4s, v25.16b, v19.4b[3]\n"
+ ".inst 0x6fb7eb3f // udot v31.4s, v25.16b, v23.4b[3]\n"
+ "9:\n"
"str q26, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q27, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, #0x10\n"
"str q28, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, #0x10\n"
"str q29, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, #0x10\n"
"str q30, [c_ptr4]\n"
- "add c_ptr4, c_ptr4, #0x10\n"
"str q31, [c_ptr5]\n"
- "add c_ptr5, c_ptr5, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x6/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_6x4/generic.cpp
similarity index 80%
rename from src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x6/generic.cpp
rename to src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_6x4/generic.cpp
index fe69f74..10bd16a 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x6/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_6x4/generic.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2019 Arm Limited.
+ * Copyright (c) 2019-2020 Arm Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -33,7 +33,7 @@
namespace arm_gemm {
-void a64_smallK_hybrid_u8u32_dot_4x6(const uint8_t *A, int lda, const uint8_t *B, uint32_t *C, int ldc, int M, int N, int K, const uint32_t *, Activation, bool) {
+void a64_smallK_hybrid_u8u32_dot_6x4(const uint8_t *A, int lda, const uint8_t *B, uint32_t *C, int ldc, int M, int N, int K, const uint32_t *, Activation, bool) {
const long loops_count = iceildiv(N, (int)4) - 1;
const long ldab = lda * sizeof(uint8_t);
const long ldcb = ldc * sizeof(uint32_t);
@@ -93,6 +93,7 @@
"add c_ptr1, %[c_ptr0], #0x0\n"
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
+ "cbnz %[odds], 2f\n"
"ldr q0, [%[a_ptr0]], #0x10\n"
"ldr q3, [a_ptr1], #0x10\n"
"ldr q6, [a_ptr2], #0x10\n"
@@ -103,18 +104,29 @@
"ldr q4, [a_ptr1], #0x10\n"
"ldr q7, [a_ptr2], #0x10\n"
"ldr q10, [a_ptr3], #0x10\n"
+ "ldr s2, [%[a_ptr0]]\n"
+ "ldr q13, [a_ptr4], #0x10\n"
+ "ldr s5, [a_ptr1]\n"
+ "ldr q16, [a_ptr5], #0x10\n"
+ "ldr s8, [a_ptr2]\n"
+ "ldr s11, [a_ptr3]\n"
+ "ldr s14, [a_ptr4]\n"
+ "ldr s17, [a_ptr5]\n"
+ "b 3f\n"
+ "2:\n"
+ "ldr q0, [%[a_ptr0]], #0x10\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "ldr q3, [a_ptr1], #0x10\n"
+ "ldr q6, [a_ptr2], #0x10\n"
+ "ldr q9, [a_ptr3], #0x10\n"
+ "ldr q12, [a_ptr4], #0x10\n"
+ "ldr q15, [a_ptr5], #0x10\n"
+ "ldr q1, [%[a_ptr0]], #0x10\n"
+ "ldr q4, [a_ptr1], #0x10\n"
+ "ldr q7, [a_ptr2], #0x10\n"
+ "ldr q10, [a_ptr3], #0x10\n"
"ldr q13, [a_ptr4], #0x10\n"
"ldr q16, [a_ptr5], #0x10\n"
- "cbnz %[odds], 2f\n"
- "ldr s2, [%[a_ptr0]]\n"
- "ldr s5, [a_ptr1]\n"
- "ldr s8, [a_ptr2]\n"
- "ldr s11, [a_ptr3]\n"
- "ldr s14, [a_ptr4]\n"
- "ldr s17, [a_ptr5]\n"
- "b 3f\n"
- "2:\n"
- "subs %[odds], %[odds], #0x1\n"
"b.ne 4f\n"
"ldr b2, [%[a_ptr0]]\n"
"ldr b5, [a_ptr1]\n"
@@ -141,40 +153,42 @@
"ld1 {v14.b}[2], [a_ptr4]\n"
"ld1 {v17.b}[2], [a_ptr5]\n"
"3:\n"
- "movi v26.4s, #0\n"
"ldr q18, [%[b_ptr0]]\n"
- "movi v27.4s, #0\n"
- "ldr q19, [%[b_ptr0], #0x10]\n"
- "movi v28.4s, #0\n"
- "ldr q20, [%[b_ptr0], #0x20]\n"
- "movi v29.4s, #0\n"
- "ldr q21, [%[b_ptr0], #0x30]\n"
- "movi v30.4s, #0\n"
- "ldr q22, [%[b_ptr0], #0x40]\n"
- "movi v31.4s, #0\n"
- "ldr q23, [%[b_ptr0], #0x50]\n"
- ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n"
- "ldr q24, [%[b_ptr0], #0x60]\n"
- ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n"
- "ldr q25, [%[b_ptr0], #0x70]\n"
- ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x40]\n"
- ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x80]\n"
- ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
"prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
- ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x100]\n"
- ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n"
+ "ldr q22, [%[b_ptr0], #0x40]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x140]\n"
- ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n"
+ "ldr q23, [%[b_ptr0], #0x50]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x180]\n"
- ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n"
+ "ldr q24, [%[b_ptr0], #0x60]\n"
+ "ldr q25, [%[b_ptr0], #0x70]\n"
"add %[b_ptr0], %[b_ptr0], #0x80\n"
- ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n"
+ "cbz %[loops], 6f\n"
+ "movi v26.4s, #0\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n"
+ ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n"
+ ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n"
+ ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n"
+ ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n"
+ ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n"
"ldr q18, [%[b_ptr0]]\n"
- ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n"
+ ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n"
"add %[b_ptr0], %[b_ptr0], #0x10\n"
+ ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n"
+ ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n"
+ ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n"
+ ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n"
".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n"
".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n"
".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n"
@@ -218,139 +232,201 @@
".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n"
".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n"
".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n"
- "cbz %[loops], 6f\n"
- "ldr q18, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ldr q19, [%[b_ptr0], #0x10]\n"
- "ldr q20, [%[b_ptr0], #0x20]\n"
- "ldr q21, [%[b_ptr0], #0x30]\n"
- "ldr q22, [%[b_ptr0], #0x40]\n"
- "ldr q23, [%[b_ptr0], #0x50]\n"
- "ldr q24, [%[b_ptr0], #0x60]\n"
- "ldr q25, [%[b_ptr0], #0x70]\n"
"b.eq 7f\n"
"8:\n"
"str q26, [%[c_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "movi v26.4s, #0\n"
"subs %[loops], %[loops], #0x1\n"
- "str q27, [c_ptr1]\n"
+ "movi v26.4s, #0\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
- "movi v27.4s, #0\n"
+ "str q27, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
+ "movi v27.4s, #0\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n"
"str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
+ "ldr q22, [%[b_ptr0], #0x40]\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n"
"str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
- "add c_ptr2, c_ptr2, #0x10\n"
+ "ldr q23, [%[b_ptr0], #0x50]\n"
".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n"
+ "ldr q24, [%[b_ptr0], #0x60]\n"
+ ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n"
"str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
- "add c_ptr3, c_ptr3, #0x10\n"
+ "ldr q25, [%[b_ptr0], #0x70]\n"
".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n"
"str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n"
"add c_ptr5, c_ptr5, #0x10\n"
".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n"
- "ldr q18, [%[b_ptr0]]\n"
- ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n"
- "add %[b_ptr0], %[b_ptr0], #0x10\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ "ldr q18, [%[b_ptr0]]\n"
".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x10\n"
".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n"
- "ldr q19, [%[b_ptr0], #0x10]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n"
- "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n"
+ "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n"
+ "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n"
".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n"
- "ldr q20, [%[b_ptr0], #0x20]\n"
".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n"
".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n"
".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n"
".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n"
".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n"
".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n"
- "ldr q21, [%[b_ptr0], #0x30]\n"
".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n"
".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n"
".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n"
".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n"
".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n"
".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n"
- "ldr q22, [%[b_ptr0], #0x40]\n"
".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n"
".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n"
".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n"
".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n"
".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n"
".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n"
- "ldr q23, [%[b_ptr0], #0x50]\n"
".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n"
".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n"
".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n"
".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n"
- "ldr q24, [%[b_ptr0], #0x60]\n"
".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n"
".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n"
".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n"
".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n"
- "ldr q25, [%[b_ptr0], #0x70]\n"
".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n"
".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n"
".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n"
".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n"
".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n"
".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n"
- "ldr q18, [%[b_ptr0]]\n"
"b.ne 8b\n"
"7:\n"
"str q26, [%[c_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "movi v26.4s, #0\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
+ "movi v26.4s, #0\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
"str q27, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n"
"str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
+ "ldr q22, [%[b_ptr0], #0x40]\n"
"add c_ptr2, c_ptr2, #0x10\n"
".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n"
"str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
- "add c_ptr3, c_ptr3, #0x10\n"
+ "ldr q23, [%[b_ptr0], #0x50]\n"
".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n"
+ "ldr q24, [%[b_ptr0], #0x60]\n"
+ ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n"
"str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
- "add c_ptr4, c_ptr4, #0x10\n"
+ "ldr q25, [%[b_ptr0], #0x70]\n"
".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n"
"str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
- "add c_ptr5, c_ptr5, #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n"
- ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
+ ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x10\n"
+ ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n"
+ ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n"
+ ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n"
+ ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n"
+ ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n"
+ ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n"
+ ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n"
+ ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n"
+ ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n"
+ ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n"
+ ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n"
+ ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n"
+ ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n"
+ ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n"
+ ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n"
+ ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n"
+ ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n"
+ ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n"
+ ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n"
+ ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n"
+ ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n"
+ ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n"
+ ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n"
+ ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n"
+ ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n"
+ ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n"
+ ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n"
+ ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n"
+ ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n"
+ ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n"
+ ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n"
+ ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n"
+ ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n"
+ "b 9f\n"
+ "6:\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n"
+ ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n"
+ ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n"
+ ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n"
+ ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n"
".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n"
"ldr q18, [%[b_ptr0]]\n"
- ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n"
+ ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n"
"add %[b_ptr0], %[b_ptr0], #0x10\n"
+ ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n"
".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n"
".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n"
".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n"
@@ -397,19 +473,14 @@
".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n"
".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n"
".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n"
- "6:\n"
+ "9:\n"
"str q26, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q27, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, #0x10\n"
"str q28, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, #0x10\n"
"str q29, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, #0x10\n"
"str q30, [c_ptr4]\n"
- "add c_ptr4, c_ptr4, #0x10\n"
"str q31, [c_ptr5]\n"
- "add c_ptr5, c_ptr5, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -468,6 +539,7 @@
"add c_ptr1, %[c_ptr0], #0x0\n"
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
+ "cbnz %[odds], 2f\n"
"ldr q0, [%[a_ptr0]], #0x10\n"
"ldr q3, [a_ptr1], #0x10\n"
"ldr q6, [a_ptr2], #0x10\n"
@@ -478,24 +550,35 @@
"ldr q4, [a_ptr1], #0x10\n"
"ldr q7, [a_ptr2], #0x10\n"
"ldr q10, [a_ptr3], #0x10\n"
- "ldr q13, [a_ptr4], #0x10\n"
- "ldr q16, [a_ptr5], #0x10\n"
- "cbnz %[odds], 2f\n"
"ldr d2, [%[a_ptr0]]\n"
+ "ldr q13, [a_ptr4], #0x10\n"
"ldr d5, [a_ptr1]\n"
+ "ldr q16, [a_ptr5], #0x10\n"
"ldr d8, [a_ptr2]\n"
"ldr d11, [a_ptr3]\n"
"ldr d14, [a_ptr4]\n"
"ldr d17, [a_ptr5]\n"
"b 3f\n"
"2:\n"
+ "ldr q0, [%[a_ptr0]], #0x10\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "ldr q3, [a_ptr1], #0x10\n"
+ "ldr q6, [a_ptr2], #0x10\n"
+ "ldr q9, [a_ptr3], #0x10\n"
+ "ldr q12, [a_ptr4], #0x10\n"
+ "ldr q15, [a_ptr5], #0x10\n"
+ "ldr q1, [%[a_ptr0]], #0x10\n"
+ "ldr q4, [a_ptr1], #0x10\n"
+ "ldr q7, [a_ptr2], #0x10\n"
+ "ldr q10, [a_ptr3], #0x10\n"
"ldr s2, [%[a_ptr0]], #0x4\n"
+ "ldr q13, [a_ptr4], #0x10\n"
"ldr s5, [a_ptr1], #0x4\n"
+ "ldr q16, [a_ptr5], #0x10\n"
"ldr s8, [a_ptr2], #0x4\n"
"ldr s11, [a_ptr3], #0x4\n"
"ldr s14, [a_ptr4], #0x4\n"
"ldr s17, [a_ptr5], #0x4\n"
- "subs %[odds], %[odds], #0x1\n"
"b.ne 4f\n"
"ld1 {v2.b}[4], [%[a_ptr0]]\n"
"ld1 {v5.b}[4], [a_ptr1]\n"
@@ -522,38 +605,40 @@
"ld1 {v14.b}[6], [a_ptr4]\n"
"ld1 {v17.b}[6], [a_ptr5]\n"
"3:\n"
- "movi v26.4s, #0\n"
"ldr q18, [%[b_ptr0]]\n"
- "movi v27.4s, #0\n"
- "ldr q19, [%[b_ptr0], #0x10]\n"
- "movi v28.4s, #0\n"
- "ldr q20, [%[b_ptr0], #0x20]\n"
- "movi v29.4s, #0\n"
- "ldr q21, [%[b_ptr0], #0x30]\n"
- "movi v30.4s, #0\n"
- "ldr q22, [%[b_ptr0], #0x40]\n"
- "movi v31.4s, #0\n"
- "ldr q23, [%[b_ptr0], #0x50]\n"
- ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n"
- "ldr q24, [%[b_ptr0], #0x60]\n"
- ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n"
- "ldr q25, [%[b_ptr0], #0x70]\n"
- ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x40]\n"
- ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x80]\n"
- ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
"prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
- ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x100]\n"
- ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n"
+ "ldr q22, [%[b_ptr0], #0x40]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x140]\n"
- ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n"
+ "ldr q23, [%[b_ptr0], #0x50]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x180]\n"
- ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n"
+ "ldr q24, [%[b_ptr0], #0x60]\n"
+ "ldr q25, [%[b_ptr0], #0x70]\n"
"add %[b_ptr0], %[b_ptr0], #0x80\n"
- ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n"
+ "cbz %[loops], 6f\n"
+ "movi v26.4s, #0\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n"
+ ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n"
+ ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n"
+ ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n"
+ ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n"
+ ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n"
"ldr q18, [%[b_ptr0]]\n"
+ ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n"
+ ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n"
+ ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n"
+ ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n"
".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n"
".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n"
"ldr q19, [%[b_ptr0], #0x10]\n"
@@ -606,144 +691,213 @@
".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n"
".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n"
".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n"
- "cbz %[loops], 6f\n"
- "ldr q18, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ldr q19, [%[b_ptr0], #0x10]\n"
- "ldr q20, [%[b_ptr0], #0x20]\n"
- "ldr q21, [%[b_ptr0], #0x30]\n"
- "ldr q22, [%[b_ptr0], #0x40]\n"
- "ldr q23, [%[b_ptr0], #0x50]\n"
- "ldr q24, [%[b_ptr0], #0x60]\n"
- "ldr q25, [%[b_ptr0], #0x70]\n"
"b.eq 7f\n"
"8:\n"
"str q26, [%[c_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "movi v26.4s, #0\n"
"subs %[loops], %[loops], #0x1\n"
- "str q27, [c_ptr1]\n"
+ "movi v26.4s, #0\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
- "movi v27.4s, #0\n"
+ "str q27, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
+ "movi v27.4s, #0\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n"
"str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
+ "ldr q22, [%[b_ptr0], #0x40]\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n"
"str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
- "add c_ptr2, c_ptr2, #0x10\n"
+ "ldr q23, [%[b_ptr0], #0x50]\n"
".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n"
+ "ldr q24, [%[b_ptr0], #0x60]\n"
+ ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n"
"str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
- "add c_ptr3, c_ptr3, #0x10\n"
+ "ldr q25, [%[b_ptr0], #0x70]\n"
".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n"
"str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n"
"add c_ptr5, c_ptr5, #0x10\n"
".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n"
- "ldr q18, [%[b_ptr0]]\n"
- ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
+ "ldr q18, [%[b_ptr0]]\n"
".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n"
"ldr q19, [%[b_ptr0], #0x10]\n"
".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n"
+ "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n"
+ "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n"
- "ldr q20, [%[b_ptr0], #0x20]\n"
".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n"
".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n"
".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n"
".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n"
".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n"
".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n"
- "ldr q21, [%[b_ptr0], #0x30]\n"
".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n"
".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n"
".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n"
".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n"
".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n"
".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n"
- "ldr q22, [%[b_ptr0], #0x40]\n"
".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n"
".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n"
".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n"
".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n"
".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n"
".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n"
- "ldr q23, [%[b_ptr0], #0x50]\n"
".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n"
".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n"
".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n"
".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n"
- "ldr q24, [%[b_ptr0], #0x60]\n"
".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n"
".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n"
".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n"
".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n"
- "ldr q25, [%[b_ptr0], #0x70]\n"
".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n"
".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n"
".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n"
".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n"
".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n"
".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n"
- "ldr q18, [%[b_ptr0]]\n"
".inst 0x6fa2e27a // udot v26.4s, v19.16b, v2.4b[1]\n"
".inst 0x6fa5e27b // udot v27.4s, v19.16b, v5.4b[1]\n"
".inst 0x6fa8e27c // udot v28.4s, v19.16b, v8.4b[1]\n"
".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n"
".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n"
".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n"
- "ldr q19, [%[b_ptr0], #0x10]\n"
"b.ne 8b\n"
"7:\n"
"str q26, [%[c_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "movi v26.4s, #0\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
+ "movi v26.4s, #0\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
"str q27, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n"
"str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
+ "ldr q22, [%[b_ptr0], #0x40]\n"
"add c_ptr2, c_ptr2, #0x10\n"
".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n"
"str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
- "add c_ptr3, c_ptr3, #0x10\n"
+ "ldr q23, [%[b_ptr0], #0x50]\n"
".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n"
+ "ldr q24, [%[b_ptr0], #0x60]\n"
+ ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n"
"str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
- "add c_ptr4, c_ptr4, #0x10\n"
+ "ldr q25, [%[b_ptr0], #0x70]\n"
".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n"
"str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
- "add c_ptr5, c_ptr5, #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n"
- ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
+ ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n"
+ ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n"
+ ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
+ ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n"
+ ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n"
+ ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n"
+ ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n"
+ ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n"
+ ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n"
+ ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n"
+ ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n"
+ ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n"
+ ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n"
+ ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n"
+ ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n"
+ ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n"
+ ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n"
+ ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n"
+ ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n"
+ ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n"
+ ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n"
+ ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n"
+ ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n"
+ ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n"
+ ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n"
+ ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n"
+ ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n"
+ ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n"
+ ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n"
+ ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n"
+ ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n"
+ ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n"
+ ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n"
+ ".inst 0x6fa2e27a // udot v26.4s, v19.16b, v2.4b[1]\n"
+ ".inst 0x6fa5e27b // udot v27.4s, v19.16b, v5.4b[1]\n"
+ ".inst 0x6fa8e27c // udot v28.4s, v19.16b, v8.4b[1]\n"
+ ".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n"
+ ".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n"
+ ".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n"
+ "b 9f\n"
+ "6:\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n"
+ ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n"
+ ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n"
+ ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n"
+ ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n"
".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n"
"ldr q18, [%[b_ptr0]]\n"
+ ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n"
".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n"
".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n"
".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n"
@@ -799,19 +953,14 @@
".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n"
".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n"
".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n"
- "6:\n"
+ "9:\n"
"str q26, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q27, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, #0x10\n"
"str q28, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, #0x10\n"
"str q29, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, #0x10\n"
"str q30, [c_ptr4]\n"
- "add c_ptr4, c_ptr4, #0x10\n"
"str q31, [c_ptr5]\n"
- "add c_ptr5, c_ptr5, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -924,38 +1073,40 @@
"ld1 {v14.b}[10], [a_ptr4]\n"
"ld1 {v17.b}[10], [a_ptr5]\n"
"3:\n"
- "movi v26.4s, #0\n"
"ldr q18, [%[b_ptr0]]\n"
- "movi v27.4s, #0\n"
- "ldr q19, [%[b_ptr0], #0x10]\n"
- "movi v28.4s, #0\n"
- "ldr q20, [%[b_ptr0], #0x20]\n"
- "movi v29.4s, #0\n"
- "ldr q21, [%[b_ptr0], #0x30]\n"
- "movi v30.4s, #0\n"
- "ldr q22, [%[b_ptr0], #0x40]\n"
- "movi v31.4s, #0\n"
- "ldr q23, [%[b_ptr0], #0x50]\n"
- ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n"
- "ldr q24, [%[b_ptr0], #0x60]\n"
- ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n"
- "ldr q25, [%[b_ptr0], #0x70]\n"
- ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x40]\n"
- ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x80]\n"
- ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
"prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
- ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x100]\n"
- ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n"
+ "ldr q22, [%[b_ptr0], #0x40]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x140]\n"
- ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n"
+ "ldr q23, [%[b_ptr0], #0x50]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x180]\n"
- ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n"
+ "ldr q24, [%[b_ptr0], #0x60]\n"
+ "ldr q25, [%[b_ptr0], #0x70]\n"
"add %[b_ptr0], %[b_ptr0], #0x80\n"
- ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n"
+ "cbz %[loops], 6f\n"
+ "movi v26.4s, #0\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n"
+ ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n"
+ ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n"
+ ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n"
+ ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n"
+ ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n"
"ldr q18, [%[b_ptr0]]\n"
+ ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n"
+ ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n"
+ ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n"
+ ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n"
".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n"
".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n"
"ldr q19, [%[b_ptr0], #0x10]\n"
@@ -1015,62 +1166,60 @@
".inst 0x6f8bea9d // udot v29.4s, v20.16b, v11.4b[2]\n"
".inst 0x6f8eea9e // udot v30.4s, v20.16b, v14.4b[2]\n"
".inst 0x6f91ea9f // udot v31.4s, v20.16b, v17.4b[2]\n"
- "cbz %[loops], 6f\n"
- "ldr q18, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ldr q19, [%[b_ptr0], #0x10]\n"
- "ldr q20, [%[b_ptr0], #0x20]\n"
- "ldr q21, [%[b_ptr0], #0x30]\n"
- "ldr q22, [%[b_ptr0], #0x40]\n"
- "ldr q23, [%[b_ptr0], #0x50]\n"
- "ldr q24, [%[b_ptr0], #0x60]\n"
- "ldr q25, [%[b_ptr0], #0x70]\n"
"b.eq 7f\n"
"8:\n"
"str q26, [%[c_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "movi v26.4s, #0\n"
"subs %[loops], %[loops], #0x1\n"
- "str q27, [c_ptr1]\n"
+ "movi v26.4s, #0\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
- "movi v27.4s, #0\n"
+ "str q27, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
+ "movi v27.4s, #0\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n"
"str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
+ "ldr q22, [%[b_ptr0], #0x40]\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n"
"str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
- "add c_ptr2, c_ptr2, #0x10\n"
+ "ldr q23, [%[b_ptr0], #0x50]\n"
".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n"
+ "ldr q24, [%[b_ptr0], #0x60]\n"
+ ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n"
"str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
- "add c_ptr3, c_ptr3, #0x10\n"
+ "ldr q25, [%[b_ptr0], #0x70]\n"
".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n"
"str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n"
"add c_ptr5, c_ptr5, #0x10\n"
".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n"
- "ldr q18, [%[b_ptr0]]\n"
- ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
+ "ldr q18, [%[b_ptr0]]\n"
".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n"
"ldr q19, [%[b_ptr0], #0x10]\n"
".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n"
+ "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n"
+ "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n"
".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n"
"ldr q20, [%[b_ptr0], #0x20]\n"
@@ -1081,85 +1230,163 @@
".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n"
".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n"
".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n"
- "ldr q21, [%[b_ptr0], #0x30]\n"
".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n"
".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n"
".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n"
".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n"
".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n"
".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n"
- "ldr q22, [%[b_ptr0], #0x40]\n"
".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n"
".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n"
".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n"
".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n"
".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n"
".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n"
- "ldr q23, [%[b_ptr0], #0x50]\n"
".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n"
".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n"
".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n"
".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n"
- "ldr q24, [%[b_ptr0], #0x60]\n"
".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n"
".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n"
".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n"
".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n"
- "ldr q25, [%[b_ptr0], #0x70]\n"
".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n"
".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n"
".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n"
".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n"
".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n"
".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n"
- "ldr q18, [%[b_ptr0]]\n"
".inst 0x6fa2e27a // udot v26.4s, v19.16b, v2.4b[1]\n"
".inst 0x6fa5e27b // udot v27.4s, v19.16b, v5.4b[1]\n"
".inst 0x6fa8e27c // udot v28.4s, v19.16b, v8.4b[1]\n"
".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n"
".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n"
".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n"
- "ldr q19, [%[b_ptr0], #0x10]\n"
".inst 0x6f82ea9a // udot v26.4s, v20.16b, v2.4b[2]\n"
".inst 0x6f85ea9b // udot v27.4s, v20.16b, v5.4b[2]\n"
".inst 0x6f88ea9c // udot v28.4s, v20.16b, v8.4b[2]\n"
".inst 0x6f8bea9d // udot v29.4s, v20.16b, v11.4b[2]\n"
".inst 0x6f8eea9e // udot v30.4s, v20.16b, v14.4b[2]\n"
".inst 0x6f91ea9f // udot v31.4s, v20.16b, v17.4b[2]\n"
- "ldr q20, [%[b_ptr0], #0x20]\n"
"b.ne 8b\n"
"7:\n"
"str q26, [%[c_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "movi v26.4s, #0\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
+ "movi v26.4s, #0\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
"str q27, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n"
"str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
+ "ldr q22, [%[b_ptr0], #0x40]\n"
"add c_ptr2, c_ptr2, #0x10\n"
".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n"
"str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
- "add c_ptr3, c_ptr3, #0x10\n"
+ "ldr q23, [%[b_ptr0], #0x50]\n"
".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n"
+ "ldr q24, [%[b_ptr0], #0x60]\n"
+ ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n"
"str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
- "add c_ptr4, c_ptr4, #0x10\n"
+ "ldr q25, [%[b_ptr0], #0x70]\n"
".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n"
"str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
- "add c_ptr5, c_ptr5, #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n"
- ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
+ ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n"
+ ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n"
+ ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
+ ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n"
+ ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n"
+ ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n"
+ ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n"
+ ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n"
+ ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
+ ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x30\n"
+ ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n"
+ ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n"
+ ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n"
+ ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n"
+ ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n"
+ ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n"
+ ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n"
+ ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n"
+ ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n"
+ ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n"
+ ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n"
+ ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n"
+ ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n"
+ ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n"
+ ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n"
+ ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n"
+ ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n"
+ ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n"
+ ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n"
+ ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n"
+ ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n"
+ ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n"
+ ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n"
+ ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n"
+ ".inst 0x6fa2e27a // udot v26.4s, v19.16b, v2.4b[1]\n"
+ ".inst 0x6fa5e27b // udot v27.4s, v19.16b, v5.4b[1]\n"
+ ".inst 0x6fa8e27c // udot v28.4s, v19.16b, v8.4b[1]\n"
+ ".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n"
+ ".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n"
+ ".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n"
+ ".inst 0x6f82ea9a // udot v26.4s, v20.16b, v2.4b[2]\n"
+ ".inst 0x6f85ea9b // udot v27.4s, v20.16b, v5.4b[2]\n"
+ ".inst 0x6f88ea9c // udot v28.4s, v20.16b, v8.4b[2]\n"
+ ".inst 0x6f8bea9d // udot v29.4s, v20.16b, v11.4b[2]\n"
+ ".inst 0x6f8eea9e // udot v30.4s, v20.16b, v14.4b[2]\n"
+ ".inst 0x6f91ea9f // udot v31.4s, v20.16b, v17.4b[2]\n"
+ "b 9f\n"
+ "6:\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n"
+ ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n"
+ ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n"
+ ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n"
+ ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n"
".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n"
"ldr q18, [%[b_ptr0]]\n"
+ ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n"
".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n"
".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n"
".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n"
@@ -1222,19 +1449,14 @@
".inst 0x6f8bea9d // udot v29.4s, v20.16b, v11.4b[2]\n"
".inst 0x6f8eea9e // udot v30.4s, v20.16b, v14.4b[2]\n"
".inst 0x6f91ea9f // udot v31.4s, v20.16b, v17.4b[2]\n"
- "6:\n"
+ "9:\n"
"str q26, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q27, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, #0x10\n"
"str q28, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, #0x10\n"
"str q29, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, #0x10\n"
"str q30, [c_ptr4]\n"
- "add c_ptr4, c_ptr4, #0x10\n"
"str q31, [c_ptr5]\n"
- "add c_ptr5, c_ptr5, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -1293,6 +1515,7 @@
"add c_ptr1, %[c_ptr0], #0x0\n"
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
+ "cbnz %[odds], 2f\n"
"ldr q0, [%[a_ptr0]], #0x10\n"
"ldr q3, [a_ptr1], #0x10\n"
"ldr q6, [a_ptr2], #0x10\n"
@@ -1305,7 +1528,6 @@
"ldr q10, [a_ptr3], #0x10\n"
"ldr q13, [a_ptr4], #0x10\n"
"ldr q16, [a_ptr5], #0x10\n"
- "cbnz %[odds], 2f\n"
"ldr q2, [%[a_ptr0]]\n"
"ldr q5, [a_ptr1]\n"
"ldr q8, [a_ptr2]\n"
@@ -1314,8 +1536,21 @@
"ldr q17, [a_ptr5]\n"
"b 3f\n"
"2:\n"
+ "ldr q0, [%[a_ptr0]], #0x10\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "ldr q3, [a_ptr1], #0x10\n"
+ "ldr q6, [a_ptr2], #0x10\n"
+ "ldr q9, [a_ptr3], #0x10\n"
+ "ldr q12, [a_ptr4], #0x10\n"
+ "ldr q15, [a_ptr5], #0x10\n"
+ "ldr q1, [%[a_ptr0]], #0x10\n"
+ "ldr q4, [a_ptr1], #0x10\n"
+ "ldr q7, [a_ptr2], #0x10\n"
+ "ldr q10, [a_ptr3], #0x10\n"
"ldr d2, [%[a_ptr0]], #0x8\n"
+ "ldr q13, [a_ptr4], #0x10\n"
"ldr d5, [a_ptr1], #0x8\n"
+ "ldr q16, [a_ptr5], #0x10\n"
"ldr d8, [a_ptr2], #0x8\n"
"ldr d11, [a_ptr3], #0x8\n"
"ldr d14, [a_ptr4], #0x8\n"
@@ -1326,7 +1561,6 @@
"ld1 {v11.s}[2], [a_ptr3], #4\n"
"ld1 {v14.s}[2], [a_ptr4], #4\n"
"ld1 {v17.s}[2], [a_ptr5], #4\n"
- "subs %[odds], %[odds], #0x1\n"
"b.ne 4f\n"
"ld1 {v2.b}[12], [%[a_ptr0]]\n"
"ld1 {v5.b}[12], [a_ptr1]\n"
@@ -1353,38 +1587,40 @@
"ld1 {v14.b}[14], [a_ptr4]\n"
"ld1 {v17.b}[14], [a_ptr5]\n"
"3:\n"
- "movi v26.4s, #0\n"
"ldr q18, [%[b_ptr0]]\n"
- "movi v27.4s, #0\n"
- "ldr q19, [%[b_ptr0], #0x10]\n"
- "movi v28.4s, #0\n"
- "ldr q20, [%[b_ptr0], #0x20]\n"
- "movi v29.4s, #0\n"
- "ldr q21, [%[b_ptr0], #0x30]\n"
- "movi v30.4s, #0\n"
- "ldr q22, [%[b_ptr0], #0x40]\n"
- "movi v31.4s, #0\n"
- "ldr q23, [%[b_ptr0], #0x50]\n"
- ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n"
- "ldr q24, [%[b_ptr0], #0x60]\n"
- ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n"
- "ldr q25, [%[b_ptr0], #0x70]\n"
- ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x40]\n"
- ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x80]\n"
- ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
"prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
- ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x100]\n"
- ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n"
+ "ldr q22, [%[b_ptr0], #0x40]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x140]\n"
- ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n"
+ "ldr q23, [%[b_ptr0], #0x50]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x180]\n"
- ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n"
+ "ldr q24, [%[b_ptr0], #0x60]\n"
+ "ldr q25, [%[b_ptr0], #0x70]\n"
"add %[b_ptr0], %[b_ptr0], #0x80\n"
- ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n"
+ "cbz %[loops], 6f\n"
+ "movi v26.4s, #0\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n"
+ ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n"
+ ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n"
+ ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n"
+ ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n"
+ ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n"
"ldr q18, [%[b_ptr0]]\n"
+ ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n"
+ ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n"
+ ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n"
+ ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n"
".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n"
".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n"
"ldr q19, [%[b_ptr0], #0x10]\n"
@@ -1451,62 +1687,60 @@
".inst 0x6fabeabd // udot v29.4s, v21.16b, v11.4b[3]\n"
".inst 0x6faeeabe // udot v30.4s, v21.16b, v14.4b[3]\n"
".inst 0x6fb1eabf // udot v31.4s, v21.16b, v17.4b[3]\n"
- "cbz %[loops], 6f\n"
- "ldr q18, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ldr q19, [%[b_ptr0], #0x10]\n"
- "ldr q20, [%[b_ptr0], #0x20]\n"
- "ldr q21, [%[b_ptr0], #0x30]\n"
- "ldr q22, [%[b_ptr0], #0x40]\n"
- "ldr q23, [%[b_ptr0], #0x50]\n"
- "ldr q24, [%[b_ptr0], #0x60]\n"
- "ldr q25, [%[b_ptr0], #0x70]\n"
"b.eq 7f\n"
"8:\n"
"str q26, [%[c_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "movi v26.4s, #0\n"
"subs %[loops], %[loops], #0x1\n"
- "str q27, [c_ptr1]\n"
+ "movi v26.4s, #0\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
- "movi v27.4s, #0\n"
+ "str q27, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
+ "movi v27.4s, #0\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n"
"str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
+ "ldr q22, [%[b_ptr0], #0x40]\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n"
"str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
- "add c_ptr2, c_ptr2, #0x10\n"
+ "ldr q23, [%[b_ptr0], #0x50]\n"
".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n"
+ "ldr q24, [%[b_ptr0], #0x60]\n"
+ ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n"
"str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
- "add c_ptr3, c_ptr3, #0x10\n"
+ "ldr q25, [%[b_ptr0], #0x70]\n"
".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n"
"str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n"
"add c_ptr5, c_ptr5, #0x10\n"
".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n"
- "ldr q18, [%[b_ptr0]]\n"
- ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
+ "ldr q18, [%[b_ptr0]]\n"
".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n"
"ldr q19, [%[b_ptr0], #0x10]\n"
".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n"
+ "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n"
+ "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n"
".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n"
"ldr q20, [%[b_ptr0], #0x20]\n"
@@ -1524,85 +1758,170 @@
".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n"
".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n"
".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n"
- "ldr q22, [%[b_ptr0], #0x40]\n"
".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n"
".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n"
".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n"
".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n"
".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n"
".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n"
- "ldr q23, [%[b_ptr0], #0x50]\n"
".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n"
".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n"
".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n"
".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n"
- "ldr q24, [%[b_ptr0], #0x60]\n"
".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n"
".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n"
".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n"
".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n"
- "ldr q25, [%[b_ptr0], #0x70]\n"
".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n"
".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n"
".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n"
".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n"
".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n"
".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n"
- "ldr q18, [%[b_ptr0]]\n"
".inst 0x6fa2e27a // udot v26.4s, v19.16b, v2.4b[1]\n"
".inst 0x6fa5e27b // udot v27.4s, v19.16b, v5.4b[1]\n"
".inst 0x6fa8e27c // udot v28.4s, v19.16b, v8.4b[1]\n"
".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n"
".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n"
".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n"
- "ldr q19, [%[b_ptr0], #0x10]\n"
".inst 0x6f82ea9a // udot v26.4s, v20.16b, v2.4b[2]\n"
".inst 0x6f85ea9b // udot v27.4s, v20.16b, v5.4b[2]\n"
".inst 0x6f88ea9c // udot v28.4s, v20.16b, v8.4b[2]\n"
".inst 0x6f8bea9d // udot v29.4s, v20.16b, v11.4b[2]\n"
".inst 0x6f8eea9e // udot v30.4s, v20.16b, v14.4b[2]\n"
".inst 0x6f91ea9f // udot v31.4s, v20.16b, v17.4b[2]\n"
- "ldr q20, [%[b_ptr0], #0x20]\n"
".inst 0x6fa2eaba // udot v26.4s, v21.16b, v2.4b[3]\n"
".inst 0x6fa5eabb // udot v27.4s, v21.16b, v5.4b[3]\n"
".inst 0x6fa8eabc // udot v28.4s, v21.16b, v8.4b[3]\n"
".inst 0x6fabeabd // udot v29.4s, v21.16b, v11.4b[3]\n"
".inst 0x6faeeabe // udot v30.4s, v21.16b, v14.4b[3]\n"
".inst 0x6fb1eabf // udot v31.4s, v21.16b, v17.4b[3]\n"
- "ldr q21, [%[b_ptr0], #0x30]\n"
"b.ne 8b\n"
"7:\n"
"str q26, [%[c_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "movi v26.4s, #0\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
+ "movi v26.4s, #0\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
"str q27, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n"
"str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
+ "ldr q22, [%[b_ptr0], #0x40]\n"
"add c_ptr2, c_ptr2, #0x10\n"
".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n"
"str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
- "add c_ptr3, c_ptr3, #0x10\n"
+ "ldr q23, [%[b_ptr0], #0x50]\n"
".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n"
+ "ldr q24, [%[b_ptr0], #0x60]\n"
+ ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n"
"str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
- "add c_ptr4, c_ptr4, #0x10\n"
+ "ldr q25, [%[b_ptr0], #0x70]\n"
".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n"
"str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
- "add c_ptr5, c_ptr5, #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n"
- ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
+ ".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n"
+ "ldr q18, [%[b_ptr0]]\n"
+ ".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n"
+ ".inst 0x6face27e // udot v30.4s, v19.16b, v12.4b[1]\n"
+ ".inst 0x6fafe27f // udot v31.4s, v19.16b, v15.4b[1]\n"
+ "ldr q19, [%[b_ptr0], #0x10]\n"
+ ".inst 0x6f80ea9a // udot v26.4s, v20.16b, v0.4b[2]\n"
+ ".inst 0x6f83ea9b // udot v27.4s, v20.16b, v3.4b[2]\n"
+ ".inst 0x6f86ea9c // udot v28.4s, v20.16b, v6.4b[2]\n"
+ ".inst 0x6f89ea9d // udot v29.4s, v20.16b, v9.4b[2]\n"
+ ".inst 0x6f8cea9e // udot v30.4s, v20.16b, v12.4b[2]\n"
+ ".inst 0x6f8fea9f // udot v31.4s, v20.16b, v15.4b[2]\n"
+ "ldr q20, [%[b_ptr0], #0x20]\n"
+ ".inst 0x6fa0eaba // udot v26.4s, v21.16b, v0.4b[3]\n"
+ ".inst 0x6fa3eabb // udot v27.4s, v21.16b, v3.4b[3]\n"
+ ".inst 0x6fa6eabc // udot v28.4s, v21.16b, v6.4b[3]\n"
+ ".inst 0x6fa9eabd // udot v29.4s, v21.16b, v9.4b[3]\n"
+ ".inst 0x6faceabe // udot v30.4s, v21.16b, v12.4b[3]\n"
+ ".inst 0x6fafeabf // udot v31.4s, v21.16b, v15.4b[3]\n"
+ "ldr q21, [%[b_ptr0], #0x30]\n"
+ ".inst 0x6f81e2da // udot v26.4s, v22.16b, v1.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x40\n"
+ ".inst 0x6f84e2db // udot v27.4s, v22.16b, v4.4b[0]\n"
+ ".inst 0x6f87e2dc // udot v28.4s, v22.16b, v7.4b[0]\n"
+ ".inst 0x6f8ae2dd // udot v29.4s, v22.16b, v10.4b[0]\n"
+ ".inst 0x6f8de2de // udot v30.4s, v22.16b, v13.4b[0]\n"
+ ".inst 0x6f90e2df // udot v31.4s, v22.16b, v16.4b[0]\n"
+ ".inst 0x6fa1e2fa // udot v26.4s, v23.16b, v1.4b[1]\n"
+ ".inst 0x6fa4e2fb // udot v27.4s, v23.16b, v4.4b[1]\n"
+ ".inst 0x6fa7e2fc // udot v28.4s, v23.16b, v7.4b[1]\n"
+ ".inst 0x6faae2fd // udot v29.4s, v23.16b, v10.4b[1]\n"
+ ".inst 0x6fade2fe // udot v30.4s, v23.16b, v13.4b[1]\n"
+ ".inst 0x6fb0e2ff // udot v31.4s, v23.16b, v16.4b[1]\n"
+ ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
+ ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x6f87eb1c // udot v28.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x6f8aeb1d // udot v29.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x6f8deb1e // udot v30.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x6f90eb1f // udot v31.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x6fa7eb3c // udot v28.4s, v25.16b, v7.4b[3]\n"
+ ".inst 0x6faaeb3d // udot v29.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x6fadeb3e // udot v30.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x6fb0eb3f // udot v31.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x6f82e25a // udot v26.4s, v18.16b, v2.4b[0]\n"
+ ".inst 0x6f85e25b // udot v27.4s, v18.16b, v5.4b[0]\n"
+ ".inst 0x6f88e25c // udot v28.4s, v18.16b, v8.4b[0]\n"
+ ".inst 0x6f8be25d // udot v29.4s, v18.16b, v11.4b[0]\n"
+ ".inst 0x6f8ee25e // udot v30.4s, v18.16b, v14.4b[0]\n"
+ ".inst 0x6f91e25f // udot v31.4s, v18.16b, v17.4b[0]\n"
+ ".inst 0x6fa2e27a // udot v26.4s, v19.16b, v2.4b[1]\n"
+ ".inst 0x6fa5e27b // udot v27.4s, v19.16b, v5.4b[1]\n"
+ ".inst 0x6fa8e27c // udot v28.4s, v19.16b, v8.4b[1]\n"
+ ".inst 0x6fabe27d // udot v29.4s, v19.16b, v11.4b[1]\n"
+ ".inst 0x6faee27e // udot v30.4s, v19.16b, v14.4b[1]\n"
+ ".inst 0x6fb1e27f // udot v31.4s, v19.16b, v17.4b[1]\n"
+ ".inst 0x6f82ea9a // udot v26.4s, v20.16b, v2.4b[2]\n"
+ ".inst 0x6f85ea9b // udot v27.4s, v20.16b, v5.4b[2]\n"
+ ".inst 0x6f88ea9c // udot v28.4s, v20.16b, v8.4b[2]\n"
+ ".inst 0x6f8bea9d // udot v29.4s, v20.16b, v11.4b[2]\n"
+ ".inst 0x6f8eea9e // udot v30.4s, v20.16b, v14.4b[2]\n"
+ ".inst 0x6f91ea9f // udot v31.4s, v20.16b, v17.4b[2]\n"
+ ".inst 0x6fa2eaba // udot v26.4s, v21.16b, v2.4b[3]\n"
+ ".inst 0x6fa5eabb // udot v27.4s, v21.16b, v5.4b[3]\n"
+ ".inst 0x6fa8eabc // udot v28.4s, v21.16b, v8.4b[3]\n"
+ ".inst 0x6fabeabd // udot v29.4s, v21.16b, v11.4b[3]\n"
+ ".inst 0x6faeeabe // udot v30.4s, v21.16b, v14.4b[3]\n"
+ ".inst 0x6fb1eabf // udot v31.4s, v21.16b, v17.4b[3]\n"
+ "b 9f\n"
+ "6:\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x6f80e25a // udot v26.4s, v18.16b, v0.4b[0]\n"
+ ".inst 0x6f83e25b // udot v27.4s, v18.16b, v3.4b[0]\n"
+ ".inst 0x6f86e25c // udot v28.4s, v18.16b, v6.4b[0]\n"
+ ".inst 0x6f89e25d // udot v29.4s, v18.16b, v9.4b[0]\n"
+ ".inst 0x6f8ce25e // udot v30.4s, v18.16b, v12.4b[0]\n"
".inst 0x6f8fe25f // udot v31.4s, v18.16b, v15.4b[0]\n"
"ldr q18, [%[b_ptr0]]\n"
+ ".inst 0x6fa0e27a // udot v26.4s, v19.16b, v0.4b[1]\n"
".inst 0x6fa3e27b // udot v27.4s, v19.16b, v3.4b[1]\n"
".inst 0x6fa6e27c // udot v28.4s, v19.16b, v6.4b[1]\n"
".inst 0x6fa9e27d // udot v29.4s, v19.16b, v9.4b[1]\n"
@@ -1672,19 +1991,14 @@
".inst 0x6fabeabd // udot v29.4s, v21.16b, v11.4b[3]\n"
".inst 0x6faeeabe // udot v30.4s, v21.16b, v14.4b[3]\n"
".inst 0x6fb1eabf // udot v31.4s, v21.16b, v17.4b[3]\n"
- "6:\n"
+ "9:\n"
"str q26, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q27, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, #0x10\n"
"str q28, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, #0x10\n"
"str q29, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, #0x10\n"
"str q30, [c_ptr4]\n"
- "add c_ptr4, c_ptr4, #0x10\n"
"str q31, [c_ptr5]\n"
- "add c_ptr5, c_ptr5, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -1743,6 +2057,7 @@
"add c_ptr1, %[c_ptr0], #0x0\n"
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
+ "cbnz %[odds], 2f\n"
"ldr q0, [%[a_ptr0]], #0x10\n"
"ldr q4, [a_ptr1], #0x10\n"
"ldr q8, [a_ptr2], #0x10\n"
@@ -1759,18 +2074,35 @@
"ldr q6, [a_ptr1], #0x10\n"
"ldr q10, [a_ptr2], #0x10\n"
"ldr q14, [a_ptr3], #0x10\n"
+ "ldr s3, [%[a_ptr0]]\n"
+ "ldr q18, [a_ptr4], #0x10\n"
+ "ldr s7, [a_ptr1]\n"
+ "ldr q22, [a_ptr5], #0x10\n"
+ "ldr s11, [a_ptr2]\n"
+ "ldr s15, [a_ptr3]\n"
+ "ldr s19, [a_ptr4]\n"
+ "ldr s23, [a_ptr5]\n"
+ "b 3f\n"
+ "2:\n"
+ "ldr q0, [%[a_ptr0]], #0x10\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "ldr q4, [a_ptr1], #0x10\n"
+ "ldr q8, [a_ptr2], #0x10\n"
+ "ldr q12, [a_ptr3], #0x10\n"
+ "ldr q16, [a_ptr4], #0x10\n"
+ "ldr q20, [a_ptr5], #0x10\n"
+ "ldr q1, [%[a_ptr0]], #0x10\n"
+ "ldr q5, [a_ptr1], #0x10\n"
+ "ldr q9, [a_ptr2], #0x10\n"
+ "ldr q13, [a_ptr3], #0x10\n"
+ "ldr q17, [a_ptr4], #0x10\n"
+ "ldr q21, [a_ptr5], #0x10\n"
+ "ldr q2, [%[a_ptr0]], #0x10\n"
+ "ldr q6, [a_ptr1], #0x10\n"
+ "ldr q10, [a_ptr2], #0x10\n"
+ "ldr q14, [a_ptr3], #0x10\n"
"ldr q18, [a_ptr4], #0x10\n"
"ldr q22, [a_ptr5], #0x10\n"
- "cbnz %[odds], 2f\n"
- "ldr s3, [%[a_ptr0]]\n"
- "ldr s7, [a_ptr1]\n"
- "ldr s11, [a_ptr2]\n"
- "ldr s15, [a_ptr3]\n"
- "ldr s19, [a_ptr4]\n"
- "ldr s23, [a_ptr5]\n"
- "b 3f\n"
- "2:\n"
- "subs %[odds], %[odds], #0x1\n"
"b.ne 4f\n"
"ldr b3, [%[a_ptr0]]\n"
"ldr b7, [a_ptr1]\n"
@@ -1797,24 +2129,26 @@
"ld1 {v19.b}[2], [a_ptr4]\n"
"ld1 {v23.b}[2], [a_ptr5]\n"
"3:\n"
- "movi v26.4s, #0\n"
"ldr q24, [%[b_ptr0]]\n"
- "movi v27.4s, #0\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- "movi v28.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr5, #0x40]\n"
- "movi v29.4s, #0\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x80]\n"
- "movi v30.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
- "movi v31.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr5, #0x100]\n"
- ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x140]\n"
- ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x180]\n"
- ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "cbz %[loops], 6f\n"
+ "movi v26.4s, #0\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
@@ -1907,38 +2241,36 @@
".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n"
".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n"
".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n"
- "cbz %[loops], 6f\n"
- "ldr q24, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
"b.eq 7f\n"
"8:\n"
"str q26, [%[c_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- "movi v26.4s, #0\n"
"subs %[loops], %[loops], #0x1\n"
- "str q27, [c_ptr1]\n"
+ "movi v26.4s, #0\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
- "movi v27.4s, #0\n"
+ "str q27, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
+ "movi v27.4s, #0\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
"str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
"str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
- "add c_ptr2, c_ptr2, #0x10\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
"str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
- "add c_ptr3, c_ptr3, #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
"str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
- "add c_ptr4, c_ptr4, #0x10\n"
- ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
"add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
@@ -2028,20 +2360,20 @@
".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n"
".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n"
".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n"
".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n"
".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n"
".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n"
".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n"
".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
"b.ne 8b\n"
"7:\n"
"str q26, [%[c_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- "movi v26.4s, #0\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
+ "movi v26.4s, #0\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
"str q27, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
@@ -2152,19 +2484,117 @@
".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n"
".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n"
".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n"
+ "b 9f\n"
"6:\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
+ ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
+ ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
+ ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
+ ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n"
+ ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n"
+ ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
+ ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
+ ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n"
+ ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n"
+ ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n"
+ ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n"
+ ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n"
+ ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n"
+ ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n"
+ ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n"
+ ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n"
+ ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n"
+ ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n"
+ ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n"
+ ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n"
+ ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n"
+ ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n"
+ ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n"
+ ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n"
+ ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n"
+ ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n"
+ ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n"
+ ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n"
+ ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n"
+ ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n"
+ ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n"
+ ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n"
+ ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n"
+ ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n"
+ ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n"
+ ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n"
+ ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n"
+ ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n"
+ ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n"
+ ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x10\n"
+ ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n"
+ ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n"
+ ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n"
+ ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n"
+ ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n"
+ ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n"
+ ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n"
+ ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n"
+ ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n"
+ ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n"
+ "9:\n"
"str q26, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q27, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, #0x10\n"
"str q28, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, #0x10\n"
"str q29, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, #0x10\n"
"str q30, [c_ptr4]\n"
- "add c_ptr4, c_ptr4, #0x10\n"
"str q31, [c_ptr5]\n"
- "add c_ptr5, c_ptr5, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -2223,6 +2653,7 @@
"add c_ptr1, %[c_ptr0], #0x0\n"
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
+ "cbnz %[odds], 2f\n"
"ldr q0, [%[a_ptr0]], #0x10\n"
"ldr q4, [a_ptr1], #0x10\n"
"ldr q8, [a_ptr2], #0x10\n"
@@ -2239,24 +2670,41 @@
"ldr q6, [a_ptr1], #0x10\n"
"ldr q10, [a_ptr2], #0x10\n"
"ldr q14, [a_ptr3], #0x10\n"
- "ldr q18, [a_ptr4], #0x10\n"
- "ldr q22, [a_ptr5], #0x10\n"
- "cbnz %[odds], 2f\n"
"ldr d3, [%[a_ptr0]]\n"
+ "ldr q18, [a_ptr4], #0x10\n"
"ldr d7, [a_ptr1]\n"
+ "ldr q22, [a_ptr5], #0x10\n"
"ldr d11, [a_ptr2]\n"
"ldr d15, [a_ptr3]\n"
"ldr d19, [a_ptr4]\n"
"ldr d23, [a_ptr5]\n"
"b 3f\n"
"2:\n"
+ "ldr q0, [%[a_ptr0]], #0x10\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "ldr q4, [a_ptr1], #0x10\n"
+ "ldr q8, [a_ptr2], #0x10\n"
+ "ldr q12, [a_ptr3], #0x10\n"
+ "ldr q16, [a_ptr4], #0x10\n"
+ "ldr q20, [a_ptr5], #0x10\n"
+ "ldr q1, [%[a_ptr0]], #0x10\n"
+ "ldr q5, [a_ptr1], #0x10\n"
+ "ldr q9, [a_ptr2], #0x10\n"
+ "ldr q13, [a_ptr3], #0x10\n"
+ "ldr q17, [a_ptr4], #0x10\n"
+ "ldr q21, [a_ptr5], #0x10\n"
+ "ldr q2, [%[a_ptr0]], #0x10\n"
+ "ldr q6, [a_ptr1], #0x10\n"
+ "ldr q10, [a_ptr2], #0x10\n"
+ "ldr q14, [a_ptr3], #0x10\n"
"ldr s3, [%[a_ptr0]], #0x4\n"
+ "ldr q18, [a_ptr4], #0x10\n"
"ldr s7, [a_ptr1], #0x4\n"
+ "ldr q22, [a_ptr5], #0x10\n"
"ldr s11, [a_ptr2], #0x4\n"
"ldr s15, [a_ptr3], #0x4\n"
"ldr s19, [a_ptr4], #0x4\n"
"ldr s23, [a_ptr5], #0x4\n"
- "subs %[odds], %[odds], #0x1\n"
"b.ne 4f\n"
"ld1 {v3.b}[4], [%[a_ptr0]]\n"
"ld1 {v7.b}[4], [a_ptr1]\n"
@@ -2283,24 +2731,26 @@
"ld1 {v19.b}[6], [a_ptr4]\n"
"ld1 {v23.b}[6], [a_ptr5]\n"
"3:\n"
- "movi v26.4s, #0\n"
"ldr q24, [%[b_ptr0]]\n"
- "movi v27.4s, #0\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- "movi v28.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr5, #0x40]\n"
- "movi v29.4s, #0\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x80]\n"
- "movi v30.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
- "movi v31.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr5, #0x100]\n"
- ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x140]\n"
- ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x180]\n"
- ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "cbz %[loops], 6f\n"
+ "movi v26.4s, #0\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
@@ -2400,38 +2850,36 @@
".inst 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n"
".inst 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n"
".inst 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n"
- "cbz %[loops], 6f\n"
- "ldr q24, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
"b.eq 7f\n"
"8:\n"
"str q26, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"movi v26.4s, #0\n"
+ "ldr q24, [%[b_ptr0]]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- "str q27, [c_ptr1]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
- "movi v27.4s, #0\n"
+ "str q27, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
+ "movi v27.4s, #0\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
"str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
"str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
- "add c_ptr2, c_ptr2, #0x10\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
"str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
- "add c_ptr3, c_ptr3, #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
"str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
- "add c_ptr4, c_ptr4, #0x10\n"
- ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
"add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
@@ -2528,7 +2976,6 @@
".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n"
".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n"
".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n"
- "ldr q24, [%[b_ptr0]]\n"
".inst 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n"
".inst 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n"
".inst 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n"
@@ -2540,6 +2987,7 @@
"str q26, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v26.4s, #0\n"
+ "ldr q24, [%[b_ptr0]]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
"str q27, [c_ptr1]\n"
@@ -2659,19 +3107,124 @@
".inst 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n"
".inst 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n"
".inst 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n"
+ "b 9f\n"
"6:\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
+ ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
+ ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
+ ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
+ ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n"
+ ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n"
+ ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
+ ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
+ ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n"
+ ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n"
+ ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n"
+ ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n"
+ ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n"
+ ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n"
+ ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n"
+ ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n"
+ ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n"
+ ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n"
+ ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n"
+ ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n"
+ ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n"
+ ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n"
+ ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n"
+ ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n"
+ ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n"
+ ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n"
+ ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n"
+ ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n"
+ ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n"
+ ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n"
+ ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n"
+ ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n"
+ ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n"
+ ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n"
+ ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n"
+ ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n"
+ ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n"
+ ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n"
+ ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n"
+ ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n"
+ ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n"
+ ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n"
+ ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n"
+ ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n"
+ ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n"
+ ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n"
+ ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n"
+ ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n"
+ ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n"
+ ".inst 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n"
+ ".inst 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n"
+ ".inst 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n"
+ ".inst 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n"
+ ".inst 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n"
+ ".inst 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n"
+ "9:\n"
"str q26, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q27, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, #0x10\n"
"str q28, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, #0x10\n"
"str q29, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, #0x10\n"
"str q30, [c_ptr4]\n"
- "add c_ptr4, c_ptr4, #0x10\n"
"str q31, [c_ptr5]\n"
- "add c_ptr5, c_ptr5, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -2790,24 +3343,26 @@
"ld1 {v19.b}[10], [a_ptr4]\n"
"ld1 {v23.b}[10], [a_ptr5]\n"
"3:\n"
- "movi v26.4s, #0\n"
"ldr q24, [%[b_ptr0]]\n"
- "movi v27.4s, #0\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- "movi v28.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr5, #0x40]\n"
- "movi v29.4s, #0\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x80]\n"
- "movi v30.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
- "movi v31.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr5, #0x100]\n"
- ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x140]\n"
- ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x180]\n"
- ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "cbz %[loops], 6f\n"
+ "movi v26.4s, #0\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
@@ -2915,38 +3470,36 @@
".inst 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n"
".inst 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n"
".inst 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n"
- "cbz %[loops], 6f\n"
- "ldr q24, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
"b.eq 7f\n"
"8:\n"
"str q26, [%[c_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- "movi v26.4s, #0\n"
"subs %[loops], %[loops], #0x1\n"
- "str q27, [c_ptr1]\n"
+ "movi v26.4s, #0\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
- "movi v27.4s, #0\n"
+ "str q27, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
+ "movi v27.4s, #0\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
"str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
"str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
- "add c_ptr2, c_ptr2, #0x10\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
"str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
- "add c_ptr3, c_ptr3, #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
"str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
- "add c_ptr4, c_ptr4, #0x10\n"
- ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
"add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
@@ -3051,20 +3604,20 @@
".inst 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n"
".inst 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n"
".inst 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
".inst 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n"
".inst 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n"
".inst 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n"
".inst 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n"
".inst 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n"
".inst 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
"b.ne 8b\n"
"7:\n"
"str q26, [%[c_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- "movi v26.4s, #0\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
+ "movi v26.4s, #0\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
"str q27, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v27.4s, #0\n"
@@ -3190,19 +3743,132 @@
".inst 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n"
".inst 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n"
".inst 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n"
+ "b 9f\n"
"6:\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
+ ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
+ ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
+ ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
+ ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n"
+ ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n"
+ ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
+ ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
+ ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n"
+ ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n"
+ ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n"
+ ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n"
+ ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n"
+ ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n"
+ ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n"
+ ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n"
+ ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n"
+ ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n"
+ ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n"
+ ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n"
+ ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n"
+ ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n"
+ ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n"
+ ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n"
+ ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n"
+ ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n"
+ ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n"
+ ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n"
+ ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n"
+ ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n"
+ ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n"
+ ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n"
+ ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n"
+ ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n"
+ ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n"
+ ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n"
+ ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n"
+ ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n"
+ ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n"
+ ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n"
+ ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n"
+ ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n"
+ ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n"
+ ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n"
+ ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n"
+ ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n"
+ ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n"
+ ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n"
+ ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x10\n"
+ ".inst 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n"
+ ".inst 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n"
+ ".inst 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n"
+ ".inst 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n"
+ ".inst 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n"
+ ".inst 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n"
+ ".inst 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n"
+ ".inst 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n"
+ ".inst 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n"
+ ".inst 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n"
+ "9:\n"
"str q26, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q27, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, #0x10\n"
"str q28, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, #0x10\n"
"str q29, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, #0x10\n"
"str q30, [c_ptr4]\n"
- "add c_ptr4, c_ptr4, #0x10\n"
"str q31, [c_ptr5]\n"
- "add c_ptr5, c_ptr5, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -3262,6 +3928,7 @@
"add c_ptr1, %[c_ptr0], #0x0\n"
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
+ "cbnz %[odds], 2f\n"
"ldr q0, [%[a_ptr0]], #0x10\n"
"ldr q4, [a_ptr1], #0x10\n"
"ldr q8, [a_ptr2], #0x10\n"
@@ -3280,7 +3947,6 @@
"ldr q14, [a_ptr3], #0x10\n"
"ldr q18, [a_ptr4], #0x10\n"
"ldr q22, [a_ptr5], #0x10\n"
- "cbnz %[odds], 2f\n"
"ldr q3, [%[a_ptr0]]\n"
"ldr q7, [a_ptr1]\n"
"ldr q11, [a_ptr2]\n"
@@ -3289,8 +3955,27 @@
"ldr q23, [a_ptr5]\n"
"b 3f\n"
"2:\n"
+ "ldr q0, [%[a_ptr0]], #0x10\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "ldr q4, [a_ptr1], #0x10\n"
+ "ldr q8, [a_ptr2], #0x10\n"
+ "ldr q12, [a_ptr3], #0x10\n"
+ "ldr q16, [a_ptr4], #0x10\n"
+ "ldr q20, [a_ptr5], #0x10\n"
+ "ldr q1, [%[a_ptr0]], #0x10\n"
+ "ldr q5, [a_ptr1], #0x10\n"
+ "ldr q9, [a_ptr2], #0x10\n"
+ "ldr q13, [a_ptr3], #0x10\n"
+ "ldr q17, [a_ptr4], #0x10\n"
+ "ldr q21, [a_ptr5], #0x10\n"
+ "ldr q2, [%[a_ptr0]], #0x10\n"
+ "ldr q6, [a_ptr1], #0x10\n"
+ "ldr q10, [a_ptr2], #0x10\n"
+ "ldr q14, [a_ptr3], #0x10\n"
"ldr d3, [%[a_ptr0]], #0x8\n"
+ "ldr q18, [a_ptr4], #0x10\n"
"ldr d7, [a_ptr1], #0x8\n"
+ "ldr q22, [a_ptr5], #0x10\n"
"ldr d11, [a_ptr2], #0x8\n"
"ldr d15, [a_ptr3], #0x8\n"
"ldr d19, [a_ptr4], #0x8\n"
@@ -3301,7 +3986,6 @@
"ld1 {v15.s}[2], [a_ptr3], #4\n"
"ld1 {v19.s}[2], [a_ptr4], #4\n"
"ld1 {v23.s}[2], [a_ptr5], #4\n"
- "subs %[odds], %[odds], #0x1\n"
"b.ne 4f\n"
"ld1 {v3.b}[12], [%[a_ptr0]]\n"
"ld1 {v7.b}[12], [a_ptr1]\n"
@@ -3328,24 +4012,26 @@
"ld1 {v19.b}[14], [a_ptr4]\n"
"ld1 {v23.b}[14], [a_ptr5]\n"
"3:\n"
- "movi v26.4s, #0\n"
"ldr q24, [%[b_ptr0]]\n"
- "movi v27.4s, #0\n"
- "ldr q25, [%[b_ptr0], #0x10]\n"
- "movi v28.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr5, #0x40]\n"
- "movi v29.4s, #0\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x80]\n"
- "movi v30.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr5, #0xc0]\n"
- "movi v31.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr5, #0x100]\n"
- ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x140]\n"
- ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr5, #0x180]\n"
- ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "cbz %[loops], 6f\n"
+ "movi v26.4s, #0\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
@@ -3460,38 +4146,36 @@
".inst 0x6fafeb3d // udot v29.4s, v25.16b, v15.4b[3]\n"
".inst 0x6fb3eb3e // udot v30.4s, v25.16b, v19.4b[3]\n"
".inst 0x6fb7eb3f // udot v31.4s, v25.16b, v23.4b[3]\n"
- "cbz %[loops], 6f\n"
- "ldr q24, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
"b.eq 7f\n"
"8:\n"
"str q26, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"movi v26.4s, #0\n"
+ "ldr q24, [%[b_ptr0]]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- "str q27, [c_ptr1]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
- "movi v27.4s, #0\n"
+ "str q27, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
+ "movi v27.4s, #0\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
"str q28, [c_ptr2]\n"
"movi v28.4s, #0\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
"str q29, [c_ptr3]\n"
"movi v29.4s, #0\n"
- "add c_ptr2, c_ptr2, #0x10\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
"str q30, [c_ptr4]\n"
"movi v30.4s, #0\n"
- "add c_ptr3, c_ptr3, #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
"str q31, [c_ptr5]\n"
"movi v31.4s, #0\n"
- "add c_ptr4, c_ptr4, #0x10\n"
- ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
"add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
"ldr q24, [%[b_ptr0]]\n"
".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
@@ -3603,7 +4287,6 @@
".inst 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n"
".inst 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n"
".inst 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n"
- "ldr q24, [%[b_ptr0]]\n"
".inst 0x6fa3eb3a // udot v26.4s, v25.16b, v3.4b[3]\n"
".inst 0x6fa7eb3b // udot v27.4s, v25.16b, v7.4b[3]\n"
".inst 0x6fabeb3c // udot v28.4s, v25.16b, v11.4b[3]\n"
@@ -3615,6 +4298,7 @@
"str q26, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v26.4s, #0\n"
+ "ldr q24, [%[b_ptr0]]\n"
"ldr q25, [%[b_ptr0], #0x10]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
"str q27, [c_ptr1]\n"
@@ -3749,19 +4433,139 @@
".inst 0x6fafeb3d // udot v29.4s, v25.16b, v15.4b[3]\n"
".inst 0x6fb3eb3e // udot v30.4s, v25.16b, v19.4b[3]\n"
".inst 0x6fb7eb3f // udot v31.4s, v25.16b, v23.4b[3]\n"
+ "b 9f\n"
"6:\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x6f80e31a // udot v26.4s, v24.16b, v0.4b[0]\n"
+ ".inst 0x6f84e31b // udot v27.4s, v24.16b, v4.4b[0]\n"
+ ".inst 0x6f88e31c // udot v28.4s, v24.16b, v8.4b[0]\n"
+ ".inst 0x6f8ce31d // udot v29.4s, v24.16b, v12.4b[0]\n"
+ ".inst 0x6f90e31e // udot v30.4s, v24.16b, v16.4b[0]\n"
+ ".inst 0x6f94e31f // udot v31.4s, v24.16b, v20.4b[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x6fa0e33a // udot v26.4s, v25.16b, v0.4b[1]\n"
+ ".inst 0x6fa4e33b // udot v27.4s, v25.16b, v4.4b[1]\n"
+ ".inst 0x6fa8e33c // udot v28.4s, v25.16b, v8.4b[1]\n"
+ ".inst 0x6face33d // udot v29.4s, v25.16b, v12.4b[1]\n"
+ ".inst 0x6fb0e33e // udot v30.4s, v25.16b, v16.4b[1]\n"
+ ".inst 0x6fb4e33f // udot v31.4s, v25.16b, v20.4b[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x6f80eb1a // udot v26.4s, v24.16b, v0.4b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x6f84eb1b // udot v27.4s, v24.16b, v4.4b[2]\n"
+ ".inst 0x6f88eb1c // udot v28.4s, v24.16b, v8.4b[2]\n"
+ ".inst 0x6f8ceb1d // udot v29.4s, v24.16b, v12.4b[2]\n"
+ ".inst 0x6f90eb1e // udot v30.4s, v24.16b, v16.4b[2]\n"
+ ".inst 0x6f94eb1f // udot v31.4s, v24.16b, v20.4b[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x6fa0eb3a // udot v26.4s, v25.16b, v0.4b[3]\n"
+ ".inst 0x6fa4eb3b // udot v27.4s, v25.16b, v4.4b[3]\n"
+ ".inst 0x6fa8eb3c // udot v28.4s, v25.16b, v8.4b[3]\n"
+ ".inst 0x6faceb3d // udot v29.4s, v25.16b, v12.4b[3]\n"
+ ".inst 0x6fb0eb3e // udot v30.4s, v25.16b, v16.4b[3]\n"
+ ".inst 0x6fb4eb3f // udot v31.4s, v25.16b, v20.4b[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x6f81e31a // udot v26.4s, v24.16b, v1.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x6f85e31b // udot v27.4s, v24.16b, v5.4b[0]\n"
+ ".inst 0x6f89e31c // udot v28.4s, v24.16b, v9.4b[0]\n"
+ ".inst 0x6f8de31d // udot v29.4s, v24.16b, v13.4b[0]\n"
+ ".inst 0x6f91e31e // udot v30.4s, v24.16b, v17.4b[0]\n"
+ ".inst 0x6f95e31f // udot v31.4s, v24.16b, v21.4b[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x6fa1e33a // udot v26.4s, v25.16b, v1.4b[1]\n"
+ ".inst 0x6fa5e33b // udot v27.4s, v25.16b, v5.4b[1]\n"
+ ".inst 0x6fa9e33c // udot v28.4s, v25.16b, v9.4b[1]\n"
+ ".inst 0x6fade33d // udot v29.4s, v25.16b, v13.4b[1]\n"
+ ".inst 0x6fb1e33e // udot v30.4s, v25.16b, v17.4b[1]\n"
+ ".inst 0x6fb5e33f // udot v31.4s, v25.16b, v21.4b[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x6f81eb1a // udot v26.4s, v24.16b, v1.4b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x6f85eb1b // udot v27.4s, v24.16b, v5.4b[2]\n"
+ ".inst 0x6f89eb1c // udot v28.4s, v24.16b, v9.4b[2]\n"
+ ".inst 0x6f8deb1d // udot v29.4s, v24.16b, v13.4b[2]\n"
+ ".inst 0x6f91eb1e // udot v30.4s, v24.16b, v17.4b[2]\n"
+ ".inst 0x6f95eb1f // udot v31.4s, v24.16b, v21.4b[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x6fa1eb3a // udot v26.4s, v25.16b, v1.4b[3]\n"
+ ".inst 0x6fa5eb3b // udot v27.4s, v25.16b, v5.4b[3]\n"
+ ".inst 0x6fa9eb3c // udot v28.4s, v25.16b, v9.4b[3]\n"
+ ".inst 0x6fadeb3d // udot v29.4s, v25.16b, v13.4b[3]\n"
+ ".inst 0x6fb1eb3e // udot v30.4s, v25.16b, v17.4b[3]\n"
+ ".inst 0x6fb5eb3f // udot v31.4s, v25.16b, v21.4b[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x6f82e31a // udot v26.4s, v24.16b, v2.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x6f86e31b // udot v27.4s, v24.16b, v6.4b[0]\n"
+ ".inst 0x6f8ae31c // udot v28.4s, v24.16b, v10.4b[0]\n"
+ ".inst 0x6f8ee31d // udot v29.4s, v24.16b, v14.4b[0]\n"
+ ".inst 0x6f92e31e // udot v30.4s, v24.16b, v18.4b[0]\n"
+ ".inst 0x6f96e31f // udot v31.4s, v24.16b, v22.4b[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x6fa2e33a // udot v26.4s, v25.16b, v2.4b[1]\n"
+ ".inst 0x6fa6e33b // udot v27.4s, v25.16b, v6.4b[1]\n"
+ ".inst 0x6faae33c // udot v28.4s, v25.16b, v10.4b[1]\n"
+ ".inst 0x6faee33d // udot v29.4s, v25.16b, v14.4b[1]\n"
+ ".inst 0x6fb2e33e // udot v30.4s, v25.16b, v18.4b[1]\n"
+ ".inst 0x6fb6e33f // udot v31.4s, v25.16b, v22.4b[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x6f82eb1a // udot v26.4s, v24.16b, v2.4b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x6f86eb1b // udot v27.4s, v24.16b, v6.4b[2]\n"
+ ".inst 0x6f8aeb1c // udot v28.4s, v24.16b, v10.4b[2]\n"
+ ".inst 0x6f8eeb1d // udot v29.4s, v24.16b, v14.4b[2]\n"
+ ".inst 0x6f92eb1e // udot v30.4s, v24.16b, v18.4b[2]\n"
+ ".inst 0x6f96eb1f // udot v31.4s, v24.16b, v22.4b[2]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x6fa2eb3a // udot v26.4s, v25.16b, v2.4b[3]\n"
+ ".inst 0x6fa6eb3b // udot v27.4s, v25.16b, v6.4b[3]\n"
+ ".inst 0x6faaeb3c // udot v28.4s, v25.16b, v10.4b[3]\n"
+ ".inst 0x6faeeb3d // udot v29.4s, v25.16b, v14.4b[3]\n"
+ ".inst 0x6fb2eb3e // udot v30.4s, v25.16b, v18.4b[3]\n"
+ ".inst 0x6fb6eb3f // udot v31.4s, v25.16b, v22.4b[3]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x6f83e31a // udot v26.4s, v24.16b, v3.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x6f87e31b // udot v27.4s, v24.16b, v7.4b[0]\n"
+ ".inst 0x6f8be31c // udot v28.4s, v24.16b, v11.4b[0]\n"
+ ".inst 0x6f8fe31d // udot v29.4s, v24.16b, v15.4b[0]\n"
+ ".inst 0x6f93e31e // udot v30.4s, v24.16b, v19.4b[0]\n"
+ ".inst 0x6f97e31f // udot v31.4s, v24.16b, v23.4b[0]\n"
+ "ldr q24, [%[b_ptr0]]\n"
+ ".inst 0x6fa3e33a // udot v26.4s, v25.16b, v3.4b[1]\n"
+ ".inst 0x6fa7e33b // udot v27.4s, v25.16b, v7.4b[1]\n"
+ ".inst 0x6fabe33c // udot v28.4s, v25.16b, v11.4b[1]\n"
+ ".inst 0x6fafe33d // udot v29.4s, v25.16b, v15.4b[1]\n"
+ ".inst 0x6fb3e33e // udot v30.4s, v25.16b, v19.4b[1]\n"
+ ".inst 0x6fb7e33f // udot v31.4s, v25.16b, v23.4b[1]\n"
+ "ldr q25, [%[b_ptr0], #0x10]\n"
+ ".inst 0x6f83eb1a // udot v26.4s, v24.16b, v3.4b[2]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x6f87eb1b // udot v27.4s, v24.16b, v7.4b[2]\n"
+ ".inst 0x6f8beb1c // udot v28.4s, v24.16b, v11.4b[2]\n"
+ ".inst 0x6f8feb1d // udot v29.4s, v24.16b, v15.4b[2]\n"
+ ".inst 0x6f93eb1e // udot v30.4s, v24.16b, v19.4b[2]\n"
+ ".inst 0x6f97eb1f // udot v31.4s, v24.16b, v23.4b[2]\n"
+ ".inst 0x6fa3eb3a // udot v26.4s, v25.16b, v3.4b[3]\n"
+ ".inst 0x6fa7eb3b // udot v27.4s, v25.16b, v7.4b[3]\n"
+ ".inst 0x6fabeb3c // udot v28.4s, v25.16b, v11.4b[3]\n"
+ ".inst 0x6fafeb3d // udot v29.4s, v25.16b, v15.4b[3]\n"
+ ".inst 0x6fb3eb3e // udot v30.4s, v25.16b, v19.4b[3]\n"
+ ".inst 0x6fb7eb3f // udot v31.4s, v25.16b, v23.4b[3]\n"
+ "9:\n"
"str q26, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q27, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, #0x10\n"
"str q28, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, #0x10\n"
"str q29, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, #0x10\n"
"str q30, [c_ptr4]\n"
- "add c_ptr4, c_ptr4, #0x10\n"
"str q31, [c_ptr5]\n"
- "add c_ptr5, c_ptr5, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x8.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_8x4.hpp
similarity index 87%
rename from src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x8.hpp
rename to src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_8x4.hpp
index d91416c..942f94b 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x8.hpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_8x4.hpp
@@ -31,10 +31,10 @@
{
// Actual kernel implementations
-void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *, int, const uint8_t *, uint32_t *, int, int, int, int, const uint32_t *, Activation, bool);
-void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *, int, const uint8_t *, uint32_t *, int, int, int, int, const uint32_t *, Activation, bool);
+void a64_smallK_hybrid_u8u32_dot_8x4(const uint8_t *, int, const uint8_t *, uint32_t *, int, int, int, int, const uint32_t *, Activation, bool);
+void a64_smallK_hybrid_u8u32_dot_8x4_a55(const uint8_t *, int, const uint8_t *, uint32_t *, int, int, int, int, const uint32_t *, Activation, bool);
-class smallK_hybrid_u8u32_dot_4x8
+class cls_a64_smallK_hybrid_u8u32_dot_8x4
{
public:
typedef uint8_t operand_type;
@@ -76,12 +76,12 @@
StdTransformsFixed<operand_type, result_type, 8, 4, 4> transforms = {};
// Default to the generic kernel
- kern_type kernel=a64_smallK_hybrid_u8u32_dot_4x8;
+ kern_type kernel=a64_smallK_hybrid_u8u32_dot_8x4;
- smallK_hybrid_u8u32_dot_4x8(const CPUInfo *ci)
+ cls_a64_smallK_hybrid_u8u32_dot_8x4(const CPUInfo *ci)
{
if (ci->get_cpu_model() == CPUModel::A55r1) {
- kernel = a64_smallK_hybrid_u8u32_dot_4x8_a55;
+ kernel = a64_smallK_hybrid_u8u32_dot_8x4_a55;
}
}
};
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x8/a55.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_8x4/a55.cpp
similarity index 85%
rename from src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x8/a55.cpp
rename to src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_8x4/a55.cpp
index e70fb69..fcb546f 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x8/a55.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_8x4/a55.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2019 Arm Limited.
+ * Copyright (c) 2019-2020 Arm Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -33,7 +33,7 @@
namespace arm_gemm {
-void a64_smallK_hybrid_u8u32_dot_4x8_a55(const uint8_t *A, int lda, const uint8_t *B, uint32_t *C, int ldc, int M, int N, int K, const uint32_t *, Activation, bool) {
+void a64_smallK_hybrid_u8u32_dot_8x4_a55(const uint8_t *A, int lda, const uint8_t *B, uint32_t *C, int ldc, int M, int N, int K, const uint32_t *, Activation, bool) {
const long loops_count = iceildiv(N, (int)4) - 1;
const long ldab = lda * sizeof(uint8_t);
const long ldcb = ldc * sizeof(uint32_t);
@@ -157,22 +157,24 @@
"ld1 {v6.b}[2], [a_ptr6]\n"
"ld1 {v7.b}[2], [a_ptr7]\n"
"3:\n"
- "movi v24.4s, #0\n"
"ldr q16, [%[b_ptr0]]\n"
- "movi v25.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
- "movi v26.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
- "movi v27.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
- "movi v28.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
- "movi v29.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- "movi v30.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- "movi v31.4s, #0\n"
"add %[b_ptr0], %[b_ptr0], #0x10\n"
+ "cbz %[loops], 6f\n"
+ "movi v24.4s, #0\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "movi v25.4s, #0\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n"
".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n"
@@ -181,55 +183,49 @@
".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n"
".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n"
".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n"
- "cbz %[loops], 6f\n"
- "ldr d16, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- "add %[b_ptr0], %[b_ptr0], #0x10\n"
- "ins v16.d[1], temploadreg0\n"
"b.eq 7f\n"
"8:\n"
"str q24, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"movi v24.4s, #0\n"
+ "ldr d16, [%[b_ptr0]]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q25, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
+ "ins v16.d[1], temploadreg0\n"
+ "add %[b_ptr0], %[b_ptr0], #0x10\n"
+ "str q26, [c_ptr2]\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
+ "movi v26.4s, #0\n"
"prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
- "str q26, [c_ptr2]\n"
- "movi v26.4s, #0\n"
- "add c_ptr2, c_ptr2, #0x10\n"
- ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n"
"str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
"add c_ptr3, c_ptr3, #0x10\n"
- ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n"
"str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
- ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n"
+ ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n"
"str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
"add c_ptr5, c_ptr5, #0x10\n"
- ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n"
"str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
"add c_ptr6, c_ptr6, #0x10\n"
- ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n"
+ ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n"
"str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
"add c_ptr7, c_ptr7, #0x10\n"
- ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n"
"prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n"
- "ldr d16, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], #0x10\n"
+ ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n"
"prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n"
"prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- "ins v16.d[1], temploadreg0\n"
"prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
"prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
"prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
@@ -239,6 +235,8 @@
"str q24, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v24.4s, #0\n"
+ "ldr q16, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x10\n"
"str q25, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
@@ -268,23 +266,34 @@
"add c_ptr7, c_ptr7, #0x10\n"
".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n"
".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n"
+ "b 9f\n"
"6:\n"
+ "movi v24.4s, #0\n"
+ "movi v25.4s, #0\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n"
+ ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n"
+ ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n"
+ ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n"
+ "9:\n"
"str q24, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q25, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, #0x10\n"
"str q26, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, #0x10\n"
"str q27, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, #0x10\n"
"str q28, [c_ptr4]\n"
- "add c_ptr4, c_ptr4, #0x10\n"
"str q29, [c_ptr5]\n"
- "add c_ptr5, c_ptr5, #0x10\n"
"str q30, [c_ptr6]\n"
- "add c_ptr6, c_ptr6, #0x10\n"
"str q31, [c_ptr7]\n"
- "add c_ptr7, c_ptr7, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -423,24 +432,26 @@
"ld1 {v6.b}[6], [a_ptr6]\n"
"ld1 {v7.b}[6], [a_ptr7]\n"
"3:\n"
- "movi v24.4s, #0\n"
"ldr q16, [%[b_ptr0]]\n"
- "movi v25.4s, #0\n"
- "ldr q17, [%[b_ptr0], #0x10]\n"
- "movi v26.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
- "movi v27.4s, #0\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
- "movi v28.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
- "movi v29.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
- "movi v30.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- "movi v31.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "cbz %[loops], 6f\n"
+ "movi v24.4s, #0\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "movi v25.4s, #0\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n"
".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n"
".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n"
@@ -456,78 +467,72 @@
".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n"
".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n"
".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n"
- "cbz %[loops], 6f\n"
- "ldr d16, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- "ldr d17, [%[b_ptr0], #0x10]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- "ins v16.d[1], temploadreg0\n"
"b.eq 7f\n"
"8:\n"
"str q24, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"movi v24.4s, #0\n"
- "ins v17.d[1], temploadreg1\n"
+ "ldr d16, [%[b_ptr0]]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q25, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
+ "ldr d17, [%[b_ptr0], #0x10]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
"str q26, [c_ptr2]\n"
- "movi v26.4s, #0\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
- ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n"
- "str q27, [c_ptr3]\n"
- "movi v27.4s, #0\n"
"add c_ptr2, c_ptr2, #0x10\n"
- ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n"
+ "movi v26.4s, #0\n"
+ "ins v16.d[1], temploadreg0\n"
+ "ins v17.d[1], temploadreg1\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "str q27, [c_ptr3]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ "movi v27.4s, #0\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
"str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
- "add c_ptr3, c_ptr3, #0x10\n"
- ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
+ ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n"
"str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
- "add c_ptr4, c_ptr4, #0x10\n"
- ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
+ ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n"
"str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
- "add c_ptr5, c_ptr5, #0x10\n"
- ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
+ ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n"
"str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
- "add c_ptr6, c_ptr6, #0x10\n"
- ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n"
"add c_ptr7, c_ptr7, #0x10\n"
- ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n"
- "ldr d16, [%[b_ptr0]]\n"
- ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n"
- "ins v16.d[1], temploadreg0\n"
- ".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n"
"prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n"
+ ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n"
"prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".inst 0x6fa4e23c // udot v28.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n"
"prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- ".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n"
+ ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n"
"prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- ".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
- ".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n"
- "ldr d17, [%[b_ptr0], #0x10]\n"
+ ".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
+ ".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n"
+ ".inst 0x6fa4e23c // udot v28.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n"
+ ".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n"
"b.ne 8b\n"
"7:\n"
"str q24, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v24.4s, #0\n"
- "ins v17.d[1], temploadreg1\n"
+ "ldr q16, [%[b_ptr0]]\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
"str q25, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
@@ -565,23 +570,42 @@
".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n"
".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n"
".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n"
+ "b 9f\n"
"6:\n"
+ "movi v24.4s, #0\n"
+ "movi v25.4s, #0\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n"
+ ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n"
+ ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n"
+ ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n"
+ ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n"
+ ".inst 0x6fa4e23c // udot v28.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n"
+ ".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n"
+ "9:\n"
"str q24, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q25, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, #0x10\n"
"str q26, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, #0x10\n"
"str q27, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, #0x10\n"
"str q28, [c_ptr4]\n"
- "add c_ptr4, c_ptr4, #0x10\n"
"str q29, [c_ptr5]\n"
- "add c_ptr5, c_ptr5, #0x10\n"
"str q30, [c_ptr6]\n"
- "add c_ptr6, c_ptr6, #0x10\n"
"str q31, [c_ptr7]\n"
- "add c_ptr7, c_ptr7, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -720,26 +744,28 @@
"ld1 {v6.b}[10], [a_ptr6]\n"
"ld1 {v7.b}[10], [a_ptr7]\n"
"3:\n"
- "movi v24.4s, #0\n"
"ldr q16, [%[b_ptr0]]\n"
- "movi v25.4s, #0\n"
- "ldr q17, [%[b_ptr0], #0x10]\n"
- "movi v26.4s, #0\n"
- "ldr q18, [%[b_ptr0], #0x20]\n"
- "movi v27.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
- "movi v28.4s, #0\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
- "movi v29.4s, #0\n"
+ "ldr q18, [%[b_ptr0], #0x20]\n"
"prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
- "movi v30.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
- "movi v31.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x30\n"
+ "cbz %[loops], 6f\n"
+ "movi v24.4s, #0\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "movi v25.4s, #0\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n"
".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n"
".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n"
".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n"
@@ -762,95 +788,86 @@
".inst 0x6f85ea5d // udot v29.4s, v18.16b, v5.4b[2]\n"
".inst 0x6f86ea5e // udot v30.4s, v18.16b, v6.4b[2]\n"
".inst 0x6f87ea5f // udot v31.4s, v18.16b, v7.4b[2]\n"
- "cbz %[loops], 6f\n"
- "ldr d16, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- "ldr d17, [%[b_ptr0], #0x10]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- "ldr d18, [%[b_ptr0], #0x20]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- "add %[b_ptr0], %[b_ptr0], #0x30\n"
- "ins v16.d[1], temploadreg0\n"
- "ins v17.d[1], temploadreg1\n"
"b.eq 7f\n"
"8:\n"
"str q24, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"movi v24.4s, #0\n"
- "ins v18.d[1], temploadreg2\n"
+ "ldr d16, [%[b_ptr0]]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q25, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
+ "ldr d17, [%[b_ptr0], #0x10]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
"str q26, [c_ptr2]\n"
- "movi v26.4s, #0\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
"add c_ptr2, c_ptr2, #0x10\n"
- ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n"
+ "movi v26.4s, #0\n"
+ "ldr d18, [%[b_ptr0], #0x20]\n"
+ "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x30\n"
"str q27, [c_ptr3]\n"
- "movi v27.4s, #0\n"
"add c_ptr3, c_ptr3, #0x10\n"
- ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n"
+ "movi v27.4s, #0\n"
+ "ins v16.d[1], temploadreg0\n"
+ "ins v17.d[1], temploadreg1\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
"str q28, [c_ptr4]\n"
- "movi v28.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
- ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n"
+ "movi v28.4s, #0\n"
+ "ins v18.d[1], temploadreg2\n"
+ ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
"str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
"add c_ptr5, c_ptr5, #0x10\n"
- ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n"
"str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
"add c_ptr6, c_ptr6, #0x10\n"
- ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n"
+ ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n"
"str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
"add c_ptr7, c_ptr7, #0x10\n"
- ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
- ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n"
- "ldr d16, [%[b_ptr0]]\n"
- ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n"
- "ins v16.d[1], temploadreg0\n"
- ".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n"
"prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n"
+ ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n"
"prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".inst 0x6fa4e23c // udot v28.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n"
"prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- ".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n"
+ ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n"
"prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- ".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n"
"prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
- ".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n"
- "ldr d17, [%[b_ptr0], #0x10]\n"
- ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
+ ".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n"
+ ".inst 0x6fa4e23c // udot v28.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n"
+ ".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n"
+ ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
".inst 0x6f81ea59 // udot v25.4s, v18.16b, v1.4b[2]\n"
- "ins v17.d[1], temploadreg1\n"
".inst 0x6f82ea5a // udot v26.4s, v18.16b, v2.4b[2]\n"
".inst 0x6f83ea5b // udot v27.4s, v18.16b, v3.4b[2]\n"
".inst 0x6f84ea5c // udot v28.4s, v18.16b, v4.4b[2]\n"
".inst 0x6f85ea5d // udot v29.4s, v18.16b, v5.4b[2]\n"
".inst 0x6f86ea5e // udot v30.4s, v18.16b, v6.4b[2]\n"
".inst 0x6f87ea5f // udot v31.4s, v18.16b, v7.4b[2]\n"
- "ldr d18, [%[b_ptr0], #0x20]\n"
- "add %[b_ptr0], %[b_ptr0], #0x30\n"
"b.ne 8b\n"
"7:\n"
"str q24, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v24.4s, #0\n"
- "ins v18.d[1], temploadreg2\n"
+ "ldr q16, [%[b_ptr0]]\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
"str q25, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
+ "ldr q18, [%[b_ptr0], #0x20]\n"
".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
"str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
@@ -876,8 +893,9 @@
"movi v31.4s, #0\n"
"add c_ptr7, c_ptr7, #0x10\n"
".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n"
- ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x30\n"
".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n"
+ ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n"
".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n"
".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n"
@@ -893,23 +911,50 @@
".inst 0x6f85ea5d // udot v29.4s, v18.16b, v5.4b[2]\n"
".inst 0x6f86ea5e // udot v30.4s, v18.16b, v6.4b[2]\n"
".inst 0x6f87ea5f // udot v31.4s, v18.16b, v7.4b[2]\n"
+ "b 9f\n"
"6:\n"
+ "movi v24.4s, #0\n"
+ "movi v25.4s, #0\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n"
+ ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n"
+ ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n"
+ ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n"
+ ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n"
+ ".inst 0x6fa4e23c // udot v28.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n"
+ ".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n"
+ ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x6f81ea59 // udot v25.4s, v18.16b, v1.4b[2]\n"
+ ".inst 0x6f82ea5a // udot v26.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x6f83ea5b // udot v27.4s, v18.16b, v3.4b[2]\n"
+ ".inst 0x6f84ea5c // udot v28.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x6f85ea5d // udot v29.4s, v18.16b, v5.4b[2]\n"
+ ".inst 0x6f86ea5e // udot v30.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x6f87ea5f // udot v31.4s, v18.16b, v7.4b[2]\n"
+ "9:\n"
"str q24, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q25, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, #0x10\n"
"str q26, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, #0x10\n"
"str q27, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, #0x10\n"
"str q28, [c_ptr4]\n"
- "add c_ptr4, c_ptr4, #0x10\n"
"str q29, [c_ptr5]\n"
- "add c_ptr5, c_ptr5, #0x10\n"
"str q30, [c_ptr6]\n"
- "add c_ptr6, c_ptr6, #0x10\n"
"str q31, [c_ptr7]\n"
- "add c_ptr7, c_ptr7, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -1056,28 +1101,30 @@
"ld1 {v6.b}[14], [a_ptr6]\n"
"ld1 {v7.b}[14], [a_ptr7]\n"
"3:\n"
- "movi v24.4s, #0\n"
"ldr q16, [%[b_ptr0]]\n"
- "movi v25.4s, #0\n"
- "ldr q17, [%[b_ptr0], #0x10]\n"
- "movi v26.4s, #0\n"
- "ldr q18, [%[b_ptr0], #0x20]\n"
- "movi v27.4s, #0\n"
- "ldr q19, [%[b_ptr0], #0x30]\n"
- "movi v28.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
- "movi v29.4s, #0\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
- "movi v30.4s, #0\n"
+ "ldr q18, [%[b_ptr0], #0x20]\n"
"prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
- "movi v31.4s, #0\n"
+ "ldr q19, [%[b_ptr0], #0x30]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
- ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x40\n"
+ "cbz %[loops], 6f\n"
+ "movi v24.4s, #0\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "movi v25.4s, #0\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n"
+ ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n"
".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n"
".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n"
".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n"
@@ -1107,112 +1154,101 @@
".inst 0x6fa5ea7d // udot v29.4s, v19.16b, v5.4b[3]\n"
".inst 0x6fa6ea7e // udot v30.4s, v19.16b, v6.4b[3]\n"
".inst 0x6fa7ea7f // udot v31.4s, v19.16b, v7.4b[3]\n"
- "cbz %[loops], 6f\n"
- "ldr d16, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- "ldr d17, [%[b_ptr0], #0x10]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- "ldr d18, [%[b_ptr0], #0x20]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- "ldr d19, [%[b_ptr0], #0x30]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- "add %[b_ptr0], %[b_ptr0], #0x40\n"
- "ins v16.d[1], temploadreg0\n"
- "ins v17.d[1], temploadreg1\n"
- "ins v18.d[1], temploadreg2\n"
"b.eq 7f\n"
"8:\n"
"str q24, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"movi v24.4s, #0\n"
- "ins v19.d[1], temploadreg3\n"
+ "ldr d16, [%[b_ptr0]]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q25, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
+ "ldr d17, [%[b_ptr0], #0x10]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
"str q26, [c_ptr2]\n"
- "movi v26.4s, #0\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
"add c_ptr2, c_ptr2, #0x10\n"
- ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n"
+ "movi v26.4s, #0\n"
+ "ldr d18, [%[b_ptr0], #0x20]\n"
+ "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
"str q27, [c_ptr3]\n"
- "movi v27.4s, #0\n"
"add c_ptr3, c_ptr3, #0x10\n"
- ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n"
+ "movi v27.4s, #0\n"
+ "ldr d19, [%[b_ptr0], #0x30]\n"
+ "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x40\n"
"str q28, [c_ptr4]\n"
- "movi v28.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
- ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n"
+ "movi v28.4s, #0\n"
+ "ins v16.d[1], temploadreg0\n"
+ "ins v17.d[1], temploadreg1\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
"str q29, [c_ptr5]\n"
- "movi v29.4s, #0\n"
"add c_ptr5, c_ptr5, #0x10\n"
- ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n"
+ "movi v29.4s, #0\n"
+ "ins v18.d[1], temploadreg2\n"
+ ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
"str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
+ "ins v19.d[1], temploadreg3\n"
+ ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n"
"add c_ptr6, c_ptr6, #0x10\n"
- ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n"
+ ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n"
"str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
"add c_ptr7, c_ptr7, #0x10\n"
- ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
- ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n"
- "ldr d16, [%[b_ptr0]]\n"
- ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n"
- "ins v16.d[1], temploadreg0\n"
- ".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n"
+ ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n"
"prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".inst 0x6fa4e23c // udot v28.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n"
"prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- ".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n"
+ ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n"
"prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- ".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n"
"prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
- ".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n"
- "ldr d17, [%[b_ptr0], #0x10]\n"
- ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n"
"prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
+ ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n"
+ ".inst 0x6fa4e23c // udot v28.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n"
+ ".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n"
+ ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
".inst 0x6f81ea59 // udot v25.4s, v18.16b, v1.4b[2]\n"
- "ins v17.d[1], temploadreg1\n"
".inst 0x6f82ea5a // udot v26.4s, v18.16b, v2.4b[2]\n"
".inst 0x6f83ea5b // udot v27.4s, v18.16b, v3.4b[2]\n"
".inst 0x6f84ea5c // udot v28.4s, v18.16b, v4.4b[2]\n"
".inst 0x6f85ea5d // udot v29.4s, v18.16b, v5.4b[2]\n"
".inst 0x6f86ea5e // udot v30.4s, v18.16b, v6.4b[2]\n"
".inst 0x6f87ea5f // udot v31.4s, v18.16b, v7.4b[2]\n"
- "ldr d18, [%[b_ptr0], #0x20]\n"
".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n"
".inst 0x6fa1ea79 // udot v25.4s, v19.16b, v1.4b[3]\n"
".inst 0x6fa2ea7a // udot v26.4s, v19.16b, v2.4b[3]\n"
- "ins v18.d[1], temploadreg2\n"
".inst 0x6fa3ea7b // udot v27.4s, v19.16b, v3.4b[3]\n"
".inst 0x6fa4ea7c // udot v28.4s, v19.16b, v4.4b[3]\n"
".inst 0x6fa5ea7d // udot v29.4s, v19.16b, v5.4b[3]\n"
".inst 0x6fa6ea7e // udot v30.4s, v19.16b, v6.4b[3]\n"
".inst 0x6fa7ea7f // udot v31.4s, v19.16b, v7.4b[3]\n"
- "ldr d19, [%[b_ptr0], #0x30]\n"
- "add %[b_ptr0], %[b_ptr0], #0x40\n"
"b.ne 8b\n"
"7:\n"
"str q24, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v24.4s, #0\n"
- "ins v19.d[1], temploadreg3\n"
+ "ldr q16, [%[b_ptr0]]\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
"str q25, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
+ "ldr q18, [%[b_ptr0], #0x20]\n"
".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
"str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
+ "ldr q19, [%[b_ptr0], #0x30]\n"
"add c_ptr2, c_ptr2, #0x10\n"
".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n"
"str q27, [c_ptr3]\n"
@@ -1235,8 +1271,9 @@
"movi v31.4s, #0\n"
"add c_ptr7, c_ptr7, #0x10\n"
".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n"
- ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x40\n"
".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n"
+ ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n"
".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n"
".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n"
@@ -1260,23 +1297,58 @@
".inst 0x6fa5ea7d // udot v29.4s, v19.16b, v5.4b[3]\n"
".inst 0x6fa6ea7e // udot v30.4s, v19.16b, v6.4b[3]\n"
".inst 0x6fa7ea7f // udot v31.4s, v19.16b, v7.4b[3]\n"
+ "b 9f\n"
"6:\n"
+ "movi v24.4s, #0\n"
+ "movi v25.4s, #0\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n"
+ ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n"
+ ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n"
+ ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n"
+ ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n"
+ ".inst 0x6fa4e23c // udot v28.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n"
+ ".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n"
+ ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x6f81ea59 // udot v25.4s, v18.16b, v1.4b[2]\n"
+ ".inst 0x6f82ea5a // udot v26.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x6f83ea5b // udot v27.4s, v18.16b, v3.4b[2]\n"
+ ".inst 0x6f84ea5c // udot v28.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x6f85ea5d // udot v29.4s, v18.16b, v5.4b[2]\n"
+ ".inst 0x6f86ea5e // udot v30.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x6f87ea5f // udot v31.4s, v18.16b, v7.4b[2]\n"
+ ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n"
+ ".inst 0x6fa1ea79 // udot v25.4s, v19.16b, v1.4b[3]\n"
+ ".inst 0x6fa2ea7a // udot v26.4s, v19.16b, v2.4b[3]\n"
+ ".inst 0x6fa3ea7b // udot v27.4s, v19.16b, v3.4b[3]\n"
+ ".inst 0x6fa4ea7c // udot v28.4s, v19.16b, v4.4b[3]\n"
+ ".inst 0x6fa5ea7d // udot v29.4s, v19.16b, v5.4b[3]\n"
+ ".inst 0x6fa6ea7e // udot v30.4s, v19.16b, v6.4b[3]\n"
+ ".inst 0x6fa7ea7f // udot v31.4s, v19.16b, v7.4b[3]\n"
+ "9:\n"
"str q24, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q25, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, #0x10\n"
"str q26, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, #0x10\n"
"str q27, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, #0x10\n"
"str q28, [c_ptr4]\n"
- "add c_ptr4, c_ptr4, #0x10\n"
"str q29, [c_ptr5]\n"
- "add c_ptr5, c_ptr5, #0x10\n"
"str q30, [c_ptr6]\n"
- "add c_ptr6, c_ptr6, #0x10\n"
"str q31, [c_ptr7]\n"
- "add c_ptr7, c_ptr7, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -1363,26 +1435,34 @@
"add c_ptr1, %[c_ptr0], #0x0\n"
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
+ "cbnz %[odds], 2f\n"
"ldr q0, [%[a_ptr0]], #0x10\n"
"ldr q2, [a_ptr1], #0x10\n"
"ldr q4, [a_ptr2], #0x10\n"
"ldr q6, [a_ptr3], #0x10\n"
+ "ldr s1, [%[a_ptr0]]\n"
+ "ldr q8, [a_ptr4], #0x10\n"
+ "ldr s3, [a_ptr1]\n"
+ "ldr q10, [a_ptr5], #0x10\n"
+ "ldr s5, [a_ptr2]\n"
+ "ldr q12, [a_ptr6], #0x10\n"
+ "ldr s7, [a_ptr3]\n"
+ "ldr q14, [a_ptr7], #0x10\n"
+ "ldr s9, [a_ptr4]\n"
+ "ldr s11, [a_ptr5]\n"
+ "ldr s13, [a_ptr6]\n"
+ "ldr s15, [a_ptr7]\n"
+ "b 3f\n"
+ "2:\n"
+ "ldr q0, [%[a_ptr0]], #0x10\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "ldr q2, [a_ptr1], #0x10\n"
+ "ldr q4, [a_ptr2], #0x10\n"
+ "ldr q6, [a_ptr3], #0x10\n"
"ldr q8, [a_ptr4], #0x10\n"
"ldr q10, [a_ptr5], #0x10\n"
"ldr q12, [a_ptr6], #0x10\n"
"ldr q14, [a_ptr7], #0x10\n"
- "cbnz %[odds], 2f\n"
- "ldr s1, [%[a_ptr0]]\n"
- "ldr s3, [a_ptr1]\n"
- "ldr s5, [a_ptr2]\n"
- "ldr s7, [a_ptr3]\n"
- "ldr s9, [a_ptr4]\n"
- "ldr s11, [a_ptr5]\n"
- "ldr s13, [a_ptr6]\n"
- "ldr s15, [a_ptr7]\n"
- "b 3f\n"
- "2:\n"
- "subs %[odds], %[odds], #0x1\n"
"b.ne 4f\n"
"ldr b1, [%[a_ptr0]]\n"
"ldr b3, [a_ptr1]\n"
@@ -1415,30 +1495,32 @@
"ld1 {v13.b}[2], [a_ptr6]\n"
"ld1 {v15.b}[2], [a_ptr7]\n"
"3:\n"
- "movi v24.4s, #0\n"
"ldr q16, [%[b_ptr0]]\n"
- "movi v25.4s, #0\n"
- "ldr q17, [%[b_ptr0], #0x10]\n"
- "movi v26.4s, #0\n"
- "ldr q18, [%[b_ptr0], #0x20]\n"
- "movi v27.4s, #0\n"
- "ldr q19, [%[b_ptr0], #0x30]\n"
- "movi v28.4s, #0\n"
- "ldr q20, [%[b_ptr0], #0x40]\n"
- "movi v29.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
- "movi v30.4s, #0\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
- "movi v31.4s, #0\n"
+ "ldr q18, [%[b_ptr0], #0x20]\n"
"prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
- ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ "ldr q19, [%[b_ptr0], #0x30]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
- ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
+ "ldr q20, [%[b_ptr0], #0x40]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x50\n"
+ "cbz %[loops], 6f\n"
+ "movi v24.4s, #0\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "movi v25.4s, #0\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
@@ -1475,126 +1557,113 @@
".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n"
".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n"
".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n"
- "cbz %[loops], 6f\n"
- "ldr d16, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- "ldr d17, [%[b_ptr0], #0x10]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- "ldr d18, [%[b_ptr0], #0x20]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- "ldr d19, [%[b_ptr0], #0x30]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- "ldr d20, [%[b_ptr0], #0x40]\n"
- "ins v16.d[1], temploadreg0\n"
- "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- "add %[b_ptr0], %[b_ptr0], #0x50\n"
- "ins v17.d[1], temploadreg1\n"
- "ins v18.d[1], temploadreg2\n"
- "ins v19.d[1], temploadreg3\n"
"b.eq 7f\n"
"8:\n"
"str q24, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"movi v24.4s, #0\n"
- "ins v20.d[1], temploadreg0\n"
+ "ldr d16, [%[b_ptr0]]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q25, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
+ "ldr d17, [%[b_ptr0], #0x10]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
"str q26, [c_ptr2]\n"
- "movi v26.4s, #0\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
"add c_ptr2, c_ptr2, #0x10\n"
- ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
+ "movi v26.4s, #0\n"
+ "ldr d18, [%[b_ptr0], #0x20]\n"
+ "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
"str q27, [c_ptr3]\n"
- "movi v27.4s, #0\n"
"add c_ptr3, c_ptr3, #0x10\n"
- ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
+ "movi v27.4s, #0\n"
+ "ldr d19, [%[b_ptr0], #0x30]\n"
+ "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
"str q28, [c_ptr4]\n"
- "movi v28.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
- ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
+ "movi v28.4s, #0\n"
+ "ldr d20, [%[b_ptr0], #0x40]\n"
+ "ins v16.d[1], temploadreg0\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
"str q29, [c_ptr5]\n"
- "movi v29.4s, #0\n"
"add c_ptr5, c_ptr5, #0x10\n"
- ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
+ "movi v29.4s, #0\n"
+ "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
+ ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
"str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
- "add c_ptr6, c_ptr6, #0x10\n"
- ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
+ "ins v17.d[1], temploadreg1\n"
+ ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
+ "ins v18.d[1], temploadreg2\n"
+ ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
"str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
+ "ins v19.d[1], temploadreg3\n"
+ ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
+ "ins v20.d[1], temploadreg0\n"
+ ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
+ ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
"add c_ptr7, c_ptr7, #0x10\n"
".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x50\n"
".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
- "ldr d16, [%[b_ptr0]]\n"
- ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
- "ins v16.d[1], temploadreg0\n"
- ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n"
+ ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n"
- "ldr d17, [%[b_ptr0], #0x10]\n"
- ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
- ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n"
- "ins v17.d[1], temploadreg1\n"
- ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
+ ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n"
+ ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n"
+ ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n"
+ ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n"
+ ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n"
".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n"
".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n"
".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n"
".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n"
".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n"
- "ldr d18, [%[b_ptr0], #0x20]\n"
".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n"
".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n"
".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n"
- "ins v18.d[1], temploadreg2\n"
".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n"
".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n"
".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n"
".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n"
".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n"
- "ldr d19, [%[b_ptr0], #0x30]\n"
".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n"
".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n"
".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n"
- "ins v19.d[1], temploadreg3\n"
".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n"
".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n"
".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n"
".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n"
".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n"
- "ldr d20, [%[b_ptr0], #0x40]\n"
- "add %[b_ptr0], %[b_ptr0], #0x50\n"
"b.ne 8b\n"
"7:\n"
"str q24, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v24.4s, #0\n"
- "ins v20.d[1], temploadreg0\n"
+ "ldr q16, [%[b_ptr0]]\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
"str q25, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
+ "ldr q18, [%[b_ptr0], #0x20]\n"
".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
"str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
+ "ldr q19, [%[b_ptr0], #0x30]\n"
+ "ldr q20, [%[b_ptr0], #0x40]\n"
"add c_ptr2, c_ptr2, #0x10\n"
".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
"str q27, [c_ptr3]\n"
@@ -1617,8 +1686,9 @@
"movi v31.4s, #0\n"
"add c_ptr7, c_ptr7, #0x10\n"
".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
- ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x50\n"
".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
+ ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
@@ -1650,23 +1720,66 @@
".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n"
".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n"
".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n"
+ "b 9f\n"
"6:\n"
+ "movi v24.4s, #0\n"
+ "movi v25.4s, #0\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
+ ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
+ ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
+ ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
+ ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n"
+ ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n"
+ ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n"
+ ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n"
+ ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n"
+ ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n"
+ ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n"
+ ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n"
+ ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n"
+ ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n"
+ ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n"
+ ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n"
+ ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n"
+ ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n"
+ ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n"
+ ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n"
+ ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n"
+ ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n"
+ ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n"
+ ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n"
+ ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n"
+ ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n"
+ ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n"
+ ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n"
+ "9:\n"
"str q24, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q25, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, #0x10\n"
"str q26, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, #0x10\n"
"str q27, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, #0x10\n"
"str q28, [c_ptr4]\n"
- "add c_ptr4, c_ptr4, #0x10\n"
"str q29, [c_ptr5]\n"
- "add c_ptr5, c_ptr5, #0x10\n"
"str q30, [c_ptr6]\n"
- "add c_ptr6, c_ptr6, #0x10\n"
"str q31, [c_ptr7]\n"
- "add c_ptr7, c_ptr7, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -1753,34 +1866,42 @@
"add c_ptr1, %[c_ptr0], #0x0\n"
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
+ "cbnz %[odds], 2f\n"
"ldr q0, [%[a_ptr0]], #0x10\n"
"ldr q2, [a_ptr1], #0x10\n"
"ldr q4, [a_ptr2], #0x10\n"
"ldr q6, [a_ptr3], #0x10\n"
- "ldr q8, [a_ptr4], #0x10\n"
- "ldr q10, [a_ptr5], #0x10\n"
- "ldr q12, [a_ptr6], #0x10\n"
- "ldr q14, [a_ptr7], #0x10\n"
- "cbnz %[odds], 2f\n"
"ldr d1, [%[a_ptr0]]\n"
+ "ldr q8, [a_ptr4], #0x10\n"
"ldr d3, [a_ptr1]\n"
+ "ldr q10, [a_ptr5], #0x10\n"
"ldr d5, [a_ptr2]\n"
+ "ldr q12, [a_ptr6], #0x10\n"
"ldr d7, [a_ptr3]\n"
+ "ldr q14, [a_ptr7], #0x10\n"
"ldr d9, [a_ptr4]\n"
"ldr d11, [a_ptr5]\n"
"ldr d13, [a_ptr6]\n"
"ldr d15, [a_ptr7]\n"
"b 3f\n"
"2:\n"
+ "ldr q0, [%[a_ptr0]], #0x10\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "ldr q2, [a_ptr1], #0x10\n"
+ "ldr q4, [a_ptr2], #0x10\n"
"ldr s1, [%[a_ptr0]], #0x4\n"
+ "ldr q6, [a_ptr3], #0x10\n"
"ldr s3, [a_ptr1], #0x4\n"
+ "ldr q8, [a_ptr4], #0x10\n"
"ldr s5, [a_ptr2], #0x4\n"
+ "ldr q10, [a_ptr5], #0x10\n"
"ldr s7, [a_ptr3], #0x4\n"
+ "ldr q12, [a_ptr6], #0x10\n"
"ldr s9, [a_ptr4], #0x4\n"
+ "ldr q14, [a_ptr7], #0x10\n"
"ldr s11, [a_ptr5], #0x4\n"
"ldr s13, [a_ptr6], #0x4\n"
"ldr s15, [a_ptr7], #0x4\n"
- "subs %[odds], %[odds], #0x1\n"
"b.ne 4f\n"
"ld1 {v1.b}[4], [%[a_ptr0]]\n"
"ld1 {v3.b}[4], [a_ptr1]\n"
@@ -1813,32 +1934,34 @@
"ld1 {v13.b}[6], [a_ptr6]\n"
"ld1 {v15.b}[6], [a_ptr7]\n"
"3:\n"
- "movi v24.4s, #0\n"
"ldr q16, [%[b_ptr0]]\n"
- "movi v25.4s, #0\n"
- "ldr q17, [%[b_ptr0], #0x10]\n"
- "movi v26.4s, #0\n"
- "ldr q18, [%[b_ptr0], #0x20]\n"
- "movi v27.4s, #0\n"
- "ldr q19, [%[b_ptr0], #0x30]\n"
- "movi v28.4s, #0\n"
- "ldr q20, [%[b_ptr0], #0x40]\n"
- "movi v29.4s, #0\n"
- "ldr q21, [%[b_ptr0], #0x50]\n"
- "movi v30.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
- "movi v31.4s, #0\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
- ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ "ldr q18, [%[b_ptr0], #0x20]\n"
"prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
- ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
+ "ldr q19, [%[b_ptr0], #0x30]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
- ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
+ "ldr q20, [%[b_ptr0], #0x40]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
+ "ldr q21, [%[b_ptr0], #0x50]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x60\n"
+ "cbz %[loops], 6f\n"
+ "movi v24.4s, #0\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "movi v25.4s, #0\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
@@ -1882,146 +2005,132 @@
".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n"
".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n"
".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n"
- "cbz %[loops], 6f\n"
- "ldr d16, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- "ldr d17, [%[b_ptr0], #0x10]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- "ldr d18, [%[b_ptr0], #0x20]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- "ldr d19, [%[b_ptr0], #0x30]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- "ldr d20, [%[b_ptr0], #0x40]\n"
- "ins v16.d[1], temploadreg0\n"
- "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- "ldr d21, [%[b_ptr0], #0x50]\n"
- "ins v17.d[1], temploadreg1\n"
- "ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- "add %[b_ptr0], %[b_ptr0], #0x60\n"
- "ins v18.d[1], temploadreg2\n"
- "ins v19.d[1], temploadreg3\n"
- "ins v20.d[1], temploadreg0\n"
"b.eq 7f\n"
"8:\n"
"str q24, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"movi v24.4s, #0\n"
- "ins v21.d[1], temploadreg1\n"
+ "ldr d16, [%[b_ptr0]]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q25, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
+ "ldr d17, [%[b_ptr0], #0x10]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
"str q26, [c_ptr2]\n"
- "movi v26.4s, #0\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
"add c_ptr2, c_ptr2, #0x10\n"
- ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
+ "movi v26.4s, #0\n"
+ "ldr d18, [%[b_ptr0], #0x20]\n"
+ "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
"str q27, [c_ptr3]\n"
- "movi v27.4s, #0\n"
"add c_ptr3, c_ptr3, #0x10\n"
- ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
+ "movi v27.4s, #0\n"
+ "ldr d19, [%[b_ptr0], #0x30]\n"
+ "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
"str q28, [c_ptr4]\n"
- "movi v28.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
- ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
+ "movi v28.4s, #0\n"
+ "ldr d20, [%[b_ptr0], #0x40]\n"
+ "ins v16.d[1], temploadreg0\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
"str q29, [c_ptr5]\n"
- "movi v29.4s, #0\n"
"add c_ptr5, c_ptr5, #0x10\n"
- ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
+ "movi v29.4s, #0\n"
+ "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
+ ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
"str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
- "add c_ptr6, c_ptr6, #0x10\n"
- ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
+ "ldr d21, [%[b_ptr0], #0x50]\n"
+ ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
+ "ins v17.d[1], temploadreg1\n"
+ ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
"str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
- "add c_ptr7, c_ptr7, #0x10\n"
- ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
- ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
- "ldr d16, [%[b_ptr0]]\n"
- ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
- "ins v16.d[1], temploadreg0\n"
- ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n"
- "ldr d17, [%[b_ptr0], #0x10]\n"
- ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
- "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
- ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n"
- "ins v17.d[1], temploadreg1\n"
- ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n"
"ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
+ "ins v18.d[1], temploadreg2\n"
+ ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
+ "ins v19.d[1], temploadreg3\n"
+ ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
+ "ins v20.d[1], temploadreg0\n"
+ ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
+ "ins v21.d[1], temploadreg1\n"
+ ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
+ ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
+ "add c_ptr7, c_ptr7, #0x10\n"
+ ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x60\n"
+ ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
+ ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
+ ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
+ ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
+ ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n"
+ ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n"
+ ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n"
".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n"
".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n"
".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n"
".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n"
- "ldr d18, [%[b_ptr0], #0x20]\n"
".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n"
".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n"
".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n"
- "ins v18.d[1], temploadreg2\n"
".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n"
".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n"
".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n"
".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n"
".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n"
- "ldr d19, [%[b_ptr0], #0x30]\n"
".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n"
".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n"
".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n"
- "ins v19.d[1], temploadreg3\n"
".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n"
".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n"
".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n"
".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n"
".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n"
- "ldr d20, [%[b_ptr0], #0x40]\n"
".inst 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n"
".inst 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n"
".inst 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n"
- "ins v20.d[1], temploadreg0\n"
".inst 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n"
".inst 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n"
".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n"
".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n"
".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n"
- "ldr d21, [%[b_ptr0], #0x50]\n"
- "add %[b_ptr0], %[b_ptr0], #0x60\n"
"b.ne 8b\n"
"7:\n"
"str q24, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v24.4s, #0\n"
- "ins v21.d[1], temploadreg1\n"
+ "ldr q16, [%[b_ptr0]]\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
"str q25, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
+ "ldr q18, [%[b_ptr0], #0x20]\n"
".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
"str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
+ "ldr q19, [%[b_ptr0], #0x30]\n"
+ "ldr q20, [%[b_ptr0], #0x40]\n"
"add c_ptr2, c_ptr2, #0x10\n"
".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
"str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
- "add c_ptr3, c_ptr3, #0x10\n"
+ "ldr q21, [%[b_ptr0], #0x50]\n"
".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
"str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
@@ -2038,7 +2147,7 @@
"movi v31.4s, #0\n"
"add c_ptr7, c_ptr7, #0x10\n"
".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
- ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x60\n"
".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
@@ -2079,23 +2188,74 @@
".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n"
".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n"
".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n"
+ "b 9f\n"
"6:\n"
+ "movi v24.4s, #0\n"
+ "movi v25.4s, #0\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
+ ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
+ ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
+ ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
+ ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n"
+ ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n"
+ ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n"
+ ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n"
+ ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n"
+ ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n"
+ ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n"
+ ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n"
+ ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n"
+ ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n"
+ ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n"
+ ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n"
+ ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n"
+ ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n"
+ ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n"
+ ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n"
+ ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n"
+ ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n"
+ ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n"
+ ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n"
+ ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n"
+ ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n"
+ ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n"
+ ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n"
+ ".inst 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n"
+ ".inst 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n"
+ ".inst 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n"
+ ".inst 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n"
+ ".inst 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n"
+ ".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n"
+ ".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n"
+ ".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n"
+ "9:\n"
"str q24, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q25, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, #0x10\n"
"str q26, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, #0x10\n"
"str q27, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, #0x10\n"
"str q28, [c_ptr4]\n"
- "add c_ptr4, c_ptr4, #0x10\n"
"str q29, [c_ptr5]\n"
- "add c_ptr5, c_ptr5, #0x10\n"
"str q30, [c_ptr6]\n"
- "add c_ptr6, c_ptr6, #0x10\n"
"str q31, [c_ptr7]\n"
- "add c_ptr7, c_ptr7, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -2242,34 +2402,36 @@
"ld1 {v13.b}[10], [a_ptr6]\n"
"ld1 {v15.b}[10], [a_ptr7]\n"
"3:\n"
- "movi v24.4s, #0\n"
"ldr q16, [%[b_ptr0]]\n"
- "movi v25.4s, #0\n"
- "ldr q17, [%[b_ptr0], #0x10]\n"
- "movi v26.4s, #0\n"
- "ldr q18, [%[b_ptr0], #0x20]\n"
- "movi v27.4s, #0\n"
- "ldr q19, [%[b_ptr0], #0x30]\n"
- "movi v28.4s, #0\n"
- "ldr q20, [%[b_ptr0], #0x40]\n"
- "movi v29.4s, #0\n"
- "ldr q21, [%[b_ptr0], #0x50]\n"
- "movi v30.4s, #0\n"
- "ldr q22, [%[b_ptr0], #0x60]\n"
- "movi v31.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
- ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
- ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
+ "ldr q18, [%[b_ptr0], #0x20]\n"
"prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
- ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
+ "ldr q19, [%[b_ptr0], #0x30]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
- ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
+ "ldr q20, [%[b_ptr0], #0x40]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
+ "ldr q21, [%[b_ptr0], #0x50]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
+ "ldr q22, [%[b_ptr0], #0x60]\n"
"add %[b_ptr0], %[b_ptr0], #0x70\n"
+ "cbz %[loops], 6f\n"
+ "movi v24.4s, #0\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "movi v25.4s, #0\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
+ ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
@@ -2320,178 +2482,162 @@
".inst 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n"
".inst 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n"
".inst 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n"
- "cbz %[loops], 6f\n"
- "ldr d16, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- "ldr d17, [%[b_ptr0], #0x10]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- "ldr d18, [%[b_ptr0], #0x20]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- "ldr d19, [%[b_ptr0], #0x30]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- "ldr d20, [%[b_ptr0], #0x40]\n"
- "ins v16.d[1], temploadreg0\n"
- "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- "ldr d21, [%[b_ptr0], #0x50]\n"
- "ins v17.d[1], temploadreg1\n"
- "ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- "ldr d22, [%[b_ptr0], #0x60]\n"
- "ins v18.d[1], temploadreg2\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- "add %[b_ptr0], %[b_ptr0], #0x70\n"
- "ins v19.d[1], temploadreg3\n"
- "ins v20.d[1], temploadreg0\n"
- "ins v21.d[1], temploadreg1\n"
"b.eq 7f\n"
"8:\n"
"str q24, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"movi v24.4s, #0\n"
- "ins v22.d[1], temploadreg2\n"
+ "ldr d16, [%[b_ptr0]]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q25, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
+ "ldr d17, [%[b_ptr0], #0x10]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
"str q26, [c_ptr2]\n"
- "movi v26.4s, #0\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
"add c_ptr2, c_ptr2, #0x10\n"
- ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
+ "movi v26.4s, #0\n"
+ "ldr d18, [%[b_ptr0], #0x20]\n"
+ "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
"str q27, [c_ptr3]\n"
- "movi v27.4s, #0\n"
"add c_ptr3, c_ptr3, #0x10\n"
- ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
+ "movi v27.4s, #0\n"
+ "ldr d19, [%[b_ptr0], #0x30]\n"
+ "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
"str q28, [c_ptr4]\n"
- "movi v28.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
- ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
+ "movi v28.4s, #0\n"
+ "ldr d20, [%[b_ptr0], #0x40]\n"
+ "ins v16.d[1], temploadreg0\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
"str q29, [c_ptr5]\n"
- "movi v29.4s, #0\n"
"add c_ptr5, c_ptr5, #0x10\n"
- ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
+ "movi v29.4s, #0\n"
+ "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
+ ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
"str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
- "add c_ptr6, c_ptr6, #0x10\n"
- ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
+ "ldr d21, [%[b_ptr0], #0x50]\n"
+ ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
+ "ins v17.d[1], temploadreg1\n"
+ ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
"str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
- "add c_ptr7, c_ptr7, #0x10\n"
+ "ldr temploadreg1, [%[b_ptr0], #0x58]\n"
+ ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
+ "ldr d22, [%[b_ptr0], #0x60]\n"
+ ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
+ "ins v18.d[1], temploadreg2\n"
+ ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
+ "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "ins v19.d[1], temploadreg3\n"
".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
- "ldr d16, [%[b_ptr0]]\n"
+ "ins v20.d[1], temploadreg0\n"
".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ "ins v21.d[1], temploadreg1\n"
".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
- "ins v16.d[1], temploadreg0\n"
+ "ins v22.d[1], temploadreg2\n"
".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ "add c_ptr7, c_ptr7, #0x10\n"
".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x70\n"
".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n"
- "ldr d17, [%[b_ptr0], #0x10]\n"
- ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
"prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
- ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n"
- "ins v17.d[1], temploadreg1\n"
- ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
"prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
+ ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n"
".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n"
".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n"
".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n"
".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n"
- "ldr d18, [%[b_ptr0], #0x20]\n"
".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n"
".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n"
".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n"
- "ins v18.d[1], temploadreg2\n"
".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n"
".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n"
".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n"
".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n"
- "ldr d19, [%[b_ptr0], #0x30]\n"
".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n"
".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n"
".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n"
- "ins v19.d[1], temploadreg3\n"
".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n"
".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n"
".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n"
".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n"
".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n"
- "ldr d20, [%[b_ptr0], #0x40]\n"
".inst 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n"
".inst 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n"
".inst 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n"
- "ins v20.d[1], temploadreg0\n"
".inst 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n"
".inst 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n"
".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n"
".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n"
".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n"
- "ldr d21, [%[b_ptr0], #0x50]\n"
".inst 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n"
".inst 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n"
".inst 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n"
- "ins v21.d[1], temploadreg1\n"
".inst 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n"
".inst 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n"
".inst 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n"
".inst 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n"
".inst 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n"
- "ldr d22, [%[b_ptr0], #0x60]\n"
- "add %[b_ptr0], %[b_ptr0], #0x70\n"
"b.ne 8b\n"
"7:\n"
"str q24, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v24.4s, #0\n"
- "ins v22.d[1], temploadreg2\n"
+ "ldr q16, [%[b_ptr0]]\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
"str q25, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
+ "ldr q18, [%[b_ptr0], #0x20]\n"
".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
"str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
+ "ldr q19, [%[b_ptr0], #0x30]\n"
+ "ldr q20, [%[b_ptr0], #0x40]\n"
"add c_ptr2, c_ptr2, #0x10\n"
".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
"str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
- "add c_ptr3, c_ptr3, #0x10\n"
+ "ldr q21, [%[b_ptr0], #0x50]\n"
".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
+ "ldr q22, [%[b_ptr0], #0x60]\n"
+ ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
"str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
- "add c_ptr4, c_ptr4, #0x10\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
"str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
- "add c_ptr5, c_ptr5, #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
"str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
- "add c_ptr6, c_ptr6, #0x10\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
"str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
- "add c_ptr7, c_ptr7, #0x10\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
- ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
+ "add c_ptr7, c_ptr7, #0x10\n"
".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x70\n"
".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
@@ -2539,23 +2685,82 @@
".inst 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n"
".inst 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n"
".inst 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n"
+ "b 9f\n"
"6:\n"
+ "movi v24.4s, #0\n"
+ "movi v25.4s, #0\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
+ ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
+ ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
+ ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
+ ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n"
+ ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n"
+ ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n"
+ ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n"
+ ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n"
+ ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n"
+ ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n"
+ ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n"
+ ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n"
+ ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n"
+ ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n"
+ ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n"
+ ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n"
+ ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n"
+ ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n"
+ ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n"
+ ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n"
+ ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n"
+ ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n"
+ ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n"
+ ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n"
+ ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n"
+ ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n"
+ ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n"
+ ".inst 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n"
+ ".inst 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n"
+ ".inst 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n"
+ ".inst 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n"
+ ".inst 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n"
+ ".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n"
+ ".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n"
+ ".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n"
+ ".inst 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n"
+ ".inst 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n"
+ ".inst 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n"
+ ".inst 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n"
+ ".inst 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n"
+ ".inst 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n"
+ ".inst 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n"
+ ".inst 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n"
+ "9:\n"
"str q24, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q25, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, #0x10\n"
"str q26, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, #0x10\n"
"str q27, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, #0x10\n"
"str q28, [c_ptr4]\n"
- "add c_ptr4, c_ptr4, #0x10\n"
"str q29, [c_ptr5]\n"
- "add c_ptr5, c_ptr5, #0x10\n"
"str q30, [c_ptr6]\n"
- "add c_ptr6, c_ptr6, #0x10\n"
"str q31, [c_ptr7]\n"
- "add c_ptr7, c_ptr7, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -2643,6 +2848,7 @@
"add c_ptr1, %[c_ptr0], #0x0\n"
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
+ "cbnz %[odds], 2f\n"
"ldr q0, [%[a_ptr0]], #0x10\n"
"ldr q2, [a_ptr1], #0x10\n"
"ldr q4, [a_ptr2], #0x10\n"
@@ -2651,7 +2857,6 @@
"ldr q10, [a_ptr5], #0x10\n"
"ldr q12, [a_ptr6], #0x10\n"
"ldr q14, [a_ptr7], #0x10\n"
- "cbnz %[odds], 2f\n"
"ldr q1, [%[a_ptr0]]\n"
"ldr q3, [a_ptr1]\n"
"ldr q5, [a_ptr2]\n"
@@ -2662,15 +2867,24 @@
"ldr q15, [a_ptr7]\n"
"b 3f\n"
"2:\n"
+ "ldr q0, [%[a_ptr0]], #0x10\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "ldr q2, [a_ptr1], #0x10\n"
+ "ldr q4, [a_ptr2], #0x10\n"
"ldr d1, [%[a_ptr0]], #0x8\n"
+ "ldr q6, [a_ptr3], #0x10\n"
"ldr d3, [a_ptr1], #0x8\n"
+ "ldr q8, [a_ptr4], #0x10\n"
"ldr d5, [a_ptr2], #0x8\n"
+ "ldr q10, [a_ptr5], #0x10\n"
"ldr d7, [a_ptr3], #0x8\n"
+ "ldr q12, [a_ptr6], #0x10\n"
"ldr d9, [a_ptr4], #0x8\n"
+ "ldr q14, [a_ptr7], #0x10\n"
"ldr d11, [a_ptr5], #0x8\n"
"ldr d13, [a_ptr6], #0x8\n"
- "ldr d15, [a_ptr7], #0x8\n"
"ld1 {v1.s}[2], [%[a_ptr0]], #4\n"
+ "ldr d15, [a_ptr7], #0x8\n"
"ld1 {v3.s}[2], [a_ptr1], #4\n"
"ld1 {v5.s}[2], [a_ptr2], #4\n"
"ld1 {v7.s}[2], [a_ptr3], #4\n"
@@ -2678,7 +2892,6 @@
"ld1 {v11.s}[2], [a_ptr5], #4\n"
"ld1 {v13.s}[2], [a_ptr6], #4\n"
"ld1 {v15.s}[2], [a_ptr7], #4\n"
- "subs %[odds], %[odds], #0x1\n"
"b.ne 4f\n"
"ld1 {v1.b}[12], [%[a_ptr0]]\n"
"ld1 {v3.b}[12], [a_ptr1]\n"
@@ -2711,36 +2924,38 @@
"ld1 {v13.b}[14], [a_ptr6]\n"
"ld1 {v15.b}[14], [a_ptr7]\n"
"3:\n"
- "movi v24.4s, #0\n"
"ldr q16, [%[b_ptr0]]\n"
- "movi v25.4s, #0\n"
- "ldr q17, [%[b_ptr0], #0x10]\n"
- "movi v26.4s, #0\n"
- "ldr q18, [%[b_ptr0], #0x20]\n"
- "movi v27.4s, #0\n"
- "ldr q19, [%[b_ptr0], #0x30]\n"
- "movi v28.4s, #0\n"
- "ldr q20, [%[b_ptr0], #0x40]\n"
- "movi v29.4s, #0\n"
- "ldr q21, [%[b_ptr0], #0x50]\n"
- "movi v30.4s, #0\n"
- "ldr q22, [%[b_ptr0], #0x60]\n"
- "movi v31.4s, #0\n"
- "ldr q23, [%[b_ptr0], #0x70]\n"
- ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
- ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
- ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
+ "ldr q18, [%[b_ptr0], #0x20]\n"
"prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
- ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
+ "ldr q19, [%[b_ptr0], #0x30]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
- ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
+ "ldr q20, [%[b_ptr0], #0x40]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
+ "ldr q21, [%[b_ptr0], #0x50]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
+ "ldr q22, [%[b_ptr0], #0x60]\n"
+ "ldr q23, [%[b_ptr0], #0x70]\n"
"add %[b_ptr0], %[b_ptr0], #0x80\n"
+ "cbz %[loops], 6f\n"
+ "movi v24.4s, #0\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "movi v25.4s, #0\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
+ ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
+ ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
@@ -2798,192 +3013,248 @@
".inst 0x6fabeafd // udot v29.4s, v23.16b, v11.4b[3]\n"
".inst 0x6fadeafe // udot v30.4s, v23.16b, v13.4b[3]\n"
".inst 0x6fafeaff // udot v31.4s, v23.16b, v15.4b[3]\n"
- "cbz %[loops], 6f\n"
- "ldr d16, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ldr temploadreg0, [%[b_ptr0], #0x8]\n"
- "ldr d17, [%[b_ptr0], #0x10]\n"
- "ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- "ldr d18, [%[b_ptr0], #0x20]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- "ldr d19, [%[b_ptr0], #0x30]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
- "ldr d20, [%[b_ptr0], #0x40]\n"
- "ins v16.d[1], temploadreg0\n"
- "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- "ldr d21, [%[b_ptr0], #0x50]\n"
- "ins v17.d[1], temploadreg1\n"
- "ldr temploadreg1, [%[b_ptr0], #0x58]\n"
- "ldr d22, [%[b_ptr0], #0x60]\n"
- "ins v18.d[1], temploadreg2\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
- "ldr d23, [%[b_ptr0], #0x70]\n"
- "ins v19.d[1], temploadreg3\n"
- "ldr temploadreg3, [%[b_ptr0], #0x78]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "ins v20.d[1], temploadreg0\n"
- "ins v21.d[1], temploadreg1\n"
- "ins v22.d[1], temploadreg2\n"
"b.eq 7f\n"
"8:\n"
"str q24, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"movi v24.4s, #0\n"
- "ins v23.d[1], temploadreg3\n"
+ "ldr d16, [%[b_ptr0]]\n"
"ldr temploadreg0, [%[b_ptr0], #0x8]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q25, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
+ "ldr d17, [%[b_ptr0], #0x10]\n"
"ldr temploadreg1, [%[b_ptr0], #0x18]\n"
- ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
"str q26, [c_ptr2]\n"
- "movi v26.4s, #0\n"
- "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
"add c_ptr2, c_ptr2, #0x10\n"
- ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
+ "movi v26.4s, #0\n"
+ "ldr d18, [%[b_ptr0], #0x20]\n"
+ "ldr temploadreg2, [%[b_ptr0], #0x28]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
"str q27, [c_ptr3]\n"
- "movi v27.4s, #0\n"
"add c_ptr3, c_ptr3, #0x10\n"
- ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
+ "movi v27.4s, #0\n"
+ "ldr d19, [%[b_ptr0], #0x30]\n"
+ "ldr temploadreg3, [%[b_ptr0], #0x38]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
"str q28, [c_ptr4]\n"
- "movi v28.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
- ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
+ "movi v28.4s, #0\n"
+ "ldr d20, [%[b_ptr0], #0x40]\n"
+ "ins v16.d[1], temploadreg0\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
"str q29, [c_ptr5]\n"
- "movi v29.4s, #0\n"
"add c_ptr5, c_ptr5, #0x10\n"
- ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
+ "movi v29.4s, #0\n"
+ "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
+ ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
"str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
- "add c_ptr6, c_ptr6, #0x10\n"
- ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
+ "ldr d21, [%[b_ptr0], #0x50]\n"
+ ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
+ "ins v17.d[1], temploadreg1\n"
+ ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
"str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
- "add c_ptr7, c_ptr7, #0x10\n"
- ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
- ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
- "ldr d16, [%[b_ptr0]]\n"
- ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
- "ins v16.d[1], temploadreg0\n"
- ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
- "ldr temploadreg0, [%[b_ptr0], #0x48]\n"
- ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n"
- "ldr d17, [%[b_ptr0], #0x10]\n"
- ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
- "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
- ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n"
- "ins v17.d[1], temploadreg1\n"
- ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n"
"ldr temploadreg1, [%[b_ptr0], #0x58]\n"
+ ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
+ "ldr d22, [%[b_ptr0], #0x60]\n"
+ ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
+ "ins v18.d[1], temploadreg2\n"
+ ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
+ "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
+ ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
+ "ldr d23, [%[b_ptr0], #0x70]\n"
+ ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
+ "ins v19.d[1], temploadreg3\n"
+ ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
+ "ldr temploadreg3, [%[b_ptr0], #0x78]\n"
+ ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
+ "ins v20.d[1], temploadreg0\n"
+ ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
+ "ins v21.d[1], temploadreg1\n"
+ ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
+ "ins v22.d[1], temploadreg2\n"
+ ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n"
+ "ins v23.d[1], temploadreg3\n"
+ ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
+ ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n"
+ "add c_ptr7, c_ptr7, #0x10\n"
+ ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
+ ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
+ "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
+ ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n"
+ "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
+ ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n"
+ "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n"
"prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n"
".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n"
".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n"
".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n"
- "ldr d18, [%[b_ptr0], #0x20]\n"
".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n"
".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n"
".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n"
- "ins v18.d[1], temploadreg2\n"
".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n"
- "ldr temploadreg2, [%[b_ptr0], #0x68]\n"
".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n"
".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n"
".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n"
".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n"
- "ldr d19, [%[b_ptr0], #0x30]\n"
".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n"
".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n"
".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n"
- "ins v19.d[1], temploadreg3\n"
".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n"
- "ldr temploadreg3, [%[b_ptr0], #0x78]\n"
".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n"
".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n"
".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n"
".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n"
- "ldr d20, [%[b_ptr0], #0x40]\n"
".inst 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n"
".inst 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n"
".inst 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n"
- "ins v20.d[1], temploadreg0\n"
".inst 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n"
".inst 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n"
".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n"
".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n"
".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n"
- "ldr d21, [%[b_ptr0], #0x50]\n"
".inst 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n"
".inst 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n"
".inst 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n"
- "ins v21.d[1], temploadreg1\n"
".inst 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n"
".inst 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n"
".inst 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n"
".inst 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n"
".inst 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n"
- "ldr d22, [%[b_ptr0], #0x60]\n"
".inst 0x6fa1eaf8 // udot v24.4s, v23.16b, v1.4b[3]\n"
".inst 0x6fa3eaf9 // udot v25.4s, v23.16b, v3.4b[3]\n"
".inst 0x6fa5eafa // udot v26.4s, v23.16b, v5.4b[3]\n"
- "ins v22.d[1], temploadreg2\n"
".inst 0x6fa7eafb // udot v27.4s, v23.16b, v7.4b[3]\n"
".inst 0x6fa9eafc // udot v28.4s, v23.16b, v9.4b[3]\n"
".inst 0x6fabeafd // udot v29.4s, v23.16b, v11.4b[3]\n"
".inst 0x6fadeafe // udot v30.4s, v23.16b, v13.4b[3]\n"
".inst 0x6fafeaff // udot v31.4s, v23.16b, v15.4b[3]\n"
- "ldr d23, [%[b_ptr0], #0x70]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
"b.ne 8b\n"
"7:\n"
"str q24, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v24.4s, #0\n"
- "ins v23.d[1], temploadreg3\n"
+ "ldr q16, [%[b_ptr0]]\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
"str q25, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
+ "ldr q18, [%[b_ptr0], #0x20]\n"
".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
"str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
+ "ldr q19, [%[b_ptr0], #0x30]\n"
+ "ldr q20, [%[b_ptr0], #0x40]\n"
"add c_ptr2, c_ptr2, #0x10\n"
".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
"str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
- "add c_ptr3, c_ptr3, #0x10\n"
+ "ldr q21, [%[b_ptr0], #0x50]\n"
".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
+ "ldr q22, [%[b_ptr0], #0x60]\n"
+ ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
"str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
- "add c_ptr4, c_ptr4, #0x10\n"
+ "ldr q23, [%[b_ptr0], #0x70]\n"
".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
"str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
- "add c_ptr5, c_ptr5, #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
"str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
- "add c_ptr6, c_ptr6, #0x10\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
"str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
- "add c_ptr7, c_ptr7, #0x10\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
- ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
+ "add c_ptr7, c_ptr7, #0x10\n"
".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
+ ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n"
+ ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n"
+ ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n"
+ ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n"
+ ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n"
+ ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n"
+ ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n"
+ ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n"
+ ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n"
+ ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n"
+ ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n"
+ ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n"
+ ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n"
+ ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n"
+ ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n"
+ ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n"
+ ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n"
+ ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n"
+ ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n"
+ ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n"
+ ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n"
+ ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n"
+ ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n"
+ ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n"
+ ".inst 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n"
+ ".inst 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n"
+ ".inst 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n"
+ ".inst 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n"
+ ".inst 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n"
+ ".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n"
+ ".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n"
+ ".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n"
+ ".inst 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n"
+ ".inst 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n"
+ ".inst 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n"
+ ".inst 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n"
+ ".inst 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n"
+ ".inst 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n"
+ ".inst 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n"
+ ".inst 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n"
+ ".inst 0x6fa1eaf8 // udot v24.4s, v23.16b, v1.4b[3]\n"
+ ".inst 0x6fa3eaf9 // udot v25.4s, v23.16b, v3.4b[3]\n"
+ ".inst 0x6fa5eafa // udot v26.4s, v23.16b, v5.4b[3]\n"
+ ".inst 0x6fa7eafb // udot v27.4s, v23.16b, v7.4b[3]\n"
+ ".inst 0x6fa9eafc // udot v28.4s, v23.16b, v9.4b[3]\n"
+ ".inst 0x6fabeafd // udot v29.4s, v23.16b, v11.4b[3]\n"
+ ".inst 0x6fadeafe // udot v30.4s, v23.16b, v13.4b[3]\n"
+ ".inst 0x6fafeaff // udot v31.4s, v23.16b, v15.4b[3]\n"
+ "b 9f\n"
+ "6:\n"
+ "movi v24.4s, #0\n"
+ "movi v25.4s, #0\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
+ ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
+ ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
+ ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
+ ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
@@ -3039,23 +3310,16 @@
".inst 0x6fabeafd // udot v29.4s, v23.16b, v11.4b[3]\n"
".inst 0x6fadeafe // udot v30.4s, v23.16b, v13.4b[3]\n"
".inst 0x6fafeaff // udot v31.4s, v23.16b, v15.4b[3]\n"
- "6:\n"
+ "9:\n"
"str q24, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q25, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, #0x10\n"
"str q26, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, #0x10\n"
"str q27, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, #0x10\n"
"str q28, [c_ptr4]\n"
- "add c_ptr4, c_ptr4, #0x10\n"
"str q29, [c_ptr5]\n"
- "add c_ptr5, c_ptr5, #0x10\n"
"str q30, [c_ptr6]\n"
- "add c_ptr6, c_ptr6, #0x10\n"
"str q31, [c_ptr7]\n"
- "add c_ptr7, c_ptr7, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x8/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_8x4/generic.cpp
similarity index 85%
rename from src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x8/generic.cpp
rename to src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_8x4/generic.cpp
index 2a7dd3d..aeea051 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_4x8/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_smallK_hybrid_u8u32_dot_8x4/generic.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2019 Arm Limited.
+ * Copyright (c) 2019-2020 Arm Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -33,7 +33,7 @@
namespace arm_gemm {
-void a64_smallK_hybrid_u8u32_dot_4x8(const uint8_t *A, int lda, const uint8_t *B, uint32_t *C, int ldc, int M, int N, int K, const uint32_t *, Activation, bool) {
+void a64_smallK_hybrid_u8u32_dot_8x4(const uint8_t *A, int lda, const uint8_t *B, uint32_t *C, int ldc, int M, int N, int K, const uint32_t *, Activation, bool) {
const long loops_count = iceildiv(N, (int)4) - 1;
const long ldab = lda * sizeof(uint8_t);
const long ldcb = ldc * sizeof(uint32_t);
@@ -153,22 +153,24 @@
"ld1 {v6.b}[2], [a_ptr6]\n"
"ld1 {v7.b}[2], [a_ptr7]\n"
"3:\n"
- "movi v24.4s, #0\n"
"ldr q16, [%[b_ptr0]]\n"
- "movi v25.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
- "movi v26.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
- "movi v27.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
- "movi v28.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
- "movi v29.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- "movi v30.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- "movi v31.4s, #0\n"
"add %[b_ptr0], %[b_ptr0], #0x10\n"
+ "cbz %[loops], 6f\n"
+ "movi v24.4s, #0\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "movi v25.4s, #0\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n"
".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n"
@@ -177,20 +179,17 @@
".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n"
".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n"
".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n"
- "cbz %[loops], 6f\n"
- "ldr q16, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "add %[b_ptr0], %[b_ptr0], #0x10\n"
"b.eq 7f\n"
"8:\n"
"str q24, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"movi v24.4s, #0\n"
+ "ldr q16, [%[b_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q25, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x10\n"
".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
"str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
@@ -216,10 +215,9 @@
"movi v31.4s, #0\n"
"add c_ptr7, c_ptr7, #0x10\n"
".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n"
- "ldr q16, [%[b_ptr0]]\n"
- "add %[b_ptr0], %[b_ptr0], #0x10\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
"prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
"prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
"prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
@@ -231,6 +229,8 @@
"str q24, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v24.4s, #0\n"
+ "ldr q16, [%[b_ptr0]]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x10\n"
"str q25, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
@@ -260,23 +260,34 @@
"add c_ptr7, c_ptr7, #0x10\n"
".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n"
".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n"
+ "b 9f\n"
"6:\n"
+ "movi v24.4s, #0\n"
+ "movi v25.4s, #0\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n"
+ ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n"
+ ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n"
+ ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n"
+ "9:\n"
"str q24, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q25, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, #0x10\n"
"str q26, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, #0x10\n"
"str q27, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, #0x10\n"
"str q28, [c_ptr4]\n"
- "add c_ptr4, c_ptr4, #0x10\n"
"str q29, [c_ptr5]\n"
- "add c_ptr5, c_ptr5, #0x10\n"
"str q30, [c_ptr6]\n"
- "add c_ptr6, c_ptr6, #0x10\n"
"str q31, [c_ptr7]\n"
- "add c_ptr7, c_ptr7, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -407,24 +418,26 @@
"ld1 {v6.b}[6], [a_ptr6]\n"
"ld1 {v7.b}[6], [a_ptr7]\n"
"3:\n"
- "movi v24.4s, #0\n"
"ldr q16, [%[b_ptr0]]\n"
- "movi v25.4s, #0\n"
- "ldr q17, [%[b_ptr0], #0x10]\n"
- "movi v26.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
- "movi v27.4s, #0\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
- "movi v28.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
- "movi v29.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
- "movi v30.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- "movi v31.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
+ "cbz %[loops], 6f\n"
+ "movi v24.4s, #0\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "movi v25.4s, #0\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n"
".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n"
".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n"
@@ -440,68 +453,66 @@
".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n"
".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n"
".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n"
- "cbz %[loops], 6f\n"
- "ldr q16, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
"b.eq 7f\n"
"8:\n"
"str q24, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"movi v24.4s, #0\n"
+ "ldr q16, [%[b_ptr0]]\n"
"ldr q17, [%[b_ptr0], #0x10]\n"
- "add %[b_ptr0], %[b_ptr0], #0x20\n"
- "str q25, [c_ptr1]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
- "movi v25.4s, #0\n"
+ "str q25, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
+ "movi v25.4s, #0\n"
+ "add %[b_ptr0], %[b_ptr0], #0x20\n"
".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
"str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n"
"str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
- "add c_ptr2, c_ptr2, #0x10\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n"
"str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
- "add c_ptr3, c_ptr3, #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n"
"str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
- "add c_ptr4, c_ptr4, #0x10\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n"
"str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
- "add c_ptr5, c_ptr5, #0x10\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n"
"str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
- "add c_ptr6, c_ptr6, #0x10\n"
- ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n"
"add c_ptr7, c_ptr7, #0x10\n"
+ ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n"
- "ldr q16, [%[b_ptr0]]\n"
- ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n"
+ ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- ".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- ".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n"
+ ".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- ".inst 0x6fa4e23c // udot v28.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- ".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n"
+ ".inst 0x6fa4e23c // udot v28.4s, v17.16b, v4.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
- ".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
+ ".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n"
".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n"
"b.ne 8b\n"
"7:\n"
"str q24, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v24.4s, #0\n"
+ "ldr q16, [%[b_ptr0]]\n"
"ldr q17, [%[b_ptr0], #0x10]\n"
"add %[b_ptr0], %[b_ptr0], #0x20\n"
"str q25, [c_ptr1]\n"
@@ -541,23 +552,42 @@
".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n"
".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n"
".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n"
+ "b 9f\n"
"6:\n"
+ "movi v24.4s, #0\n"
+ "movi v25.4s, #0\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n"
+ ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n"
+ ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n"
+ ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n"
+ ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n"
+ ".inst 0x6fa4e23c // udot v28.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n"
+ ".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n"
+ "9:\n"
"str q24, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q25, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, #0x10\n"
"str q26, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, #0x10\n"
"str q27, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, #0x10\n"
"str q28, [c_ptr4]\n"
- "add c_ptr4, c_ptr4, #0x10\n"
"str q29, [c_ptr5]\n"
- "add c_ptr5, c_ptr5, #0x10\n"
"str q30, [c_ptr6]\n"
- "add c_ptr6, c_ptr6, #0x10\n"
"str q31, [c_ptr7]\n"
- "add c_ptr7, c_ptr7, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -688,26 +718,28 @@
"ld1 {v6.b}[10], [a_ptr6]\n"
"ld1 {v7.b}[10], [a_ptr7]\n"
"3:\n"
- "movi v24.4s, #0\n"
"ldr q16, [%[b_ptr0]]\n"
- "movi v25.4s, #0\n"
- "ldr q17, [%[b_ptr0], #0x10]\n"
- "movi v26.4s, #0\n"
- "ldr q18, [%[b_ptr0], #0x20]\n"
- "movi v27.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
- "movi v28.4s, #0\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
- "movi v29.4s, #0\n"
+ "ldr q18, [%[b_ptr0], #0x20]\n"
"prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
- "movi v30.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
- "movi v31.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x30\n"
+ "cbz %[loops], 6f\n"
+ "movi v24.4s, #0\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "movi v25.4s, #0\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n"
".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n"
".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n"
".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n"
@@ -730,49 +762,46 @@
".inst 0x6f85ea5d // udot v29.4s, v18.16b, v5.4b[2]\n"
".inst 0x6f86ea5e // udot v30.4s, v18.16b, v6.4b[2]\n"
".inst 0x6f87ea5f // udot v31.4s, v18.16b, v7.4b[2]\n"
- "cbz %[loops], 6f\n"
- "ldr q16, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ldr q17, [%[b_ptr0], #0x10]\n"
"b.eq 7f\n"
"8:\n"
"str q24, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"movi v24.4s, #0\n"
- "ldr q18, [%[b_ptr0], #0x20]\n"
- "add %[b_ptr0], %[b_ptr0], #0x30\n"
- "str q25, [c_ptr1]\n"
+ "ldr q16, [%[b_ptr0]]\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
- "movi v25.4s, #0\n"
+ "str q25, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
+ "movi v25.4s, #0\n"
+ "ldr q18, [%[b_ptr0], #0x20]\n"
".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
"str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n"
"str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
- "add c_ptr2, c_ptr2, #0x10\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n"
"str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
- "add c_ptr3, c_ptr3, #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n"
"str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
- "add c_ptr4, c_ptr4, #0x10\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n"
"str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
- "add c_ptr5, c_ptr5, #0x10\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n"
"str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
- "add c_ptr6, c_ptr6, #0x10\n"
- ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n"
"add c_ptr7, c_ptr7, #0x10\n"
+ ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x30\n"
".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n"
- "ldr q16, [%[b_ptr0]]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n"
@@ -788,7 +817,6 @@
".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n"
- "ldr q17, [%[b_ptr0], #0x10]\n"
".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
".inst 0x6f81ea59 // udot v25.4s, v18.16b, v1.4b[2]\n"
".inst 0x6f82ea5a // udot v26.4s, v18.16b, v2.4b[2]\n"
@@ -802,11 +830,12 @@
"str q24, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v24.4s, #0\n"
- "ldr q18, [%[b_ptr0], #0x20]\n"
- "add %[b_ptr0], %[b_ptr0], #0x30\n"
+ "ldr q16, [%[b_ptr0]]\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
"str q25, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
+ "ldr q18, [%[b_ptr0], #0x20]\n"
".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
"str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
@@ -832,8 +861,9 @@
"movi v31.4s, #0\n"
"add c_ptr7, c_ptr7, #0x10\n"
".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n"
- ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x30\n"
".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n"
+ ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n"
".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n"
".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n"
@@ -849,23 +879,50 @@
".inst 0x6f85ea5d // udot v29.4s, v18.16b, v5.4b[2]\n"
".inst 0x6f86ea5e // udot v30.4s, v18.16b, v6.4b[2]\n"
".inst 0x6f87ea5f // udot v31.4s, v18.16b, v7.4b[2]\n"
+ "b 9f\n"
"6:\n"
+ "movi v24.4s, #0\n"
+ "movi v25.4s, #0\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n"
+ ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n"
+ ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n"
+ ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n"
+ ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n"
+ ".inst 0x6fa4e23c // udot v28.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n"
+ ".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n"
+ ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x6f81ea59 // udot v25.4s, v18.16b, v1.4b[2]\n"
+ ".inst 0x6f82ea5a // udot v26.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x6f83ea5b // udot v27.4s, v18.16b, v3.4b[2]\n"
+ ".inst 0x6f84ea5c // udot v28.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x6f85ea5d // udot v29.4s, v18.16b, v5.4b[2]\n"
+ ".inst 0x6f86ea5e // udot v30.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x6f87ea5f // udot v31.4s, v18.16b, v7.4b[2]\n"
+ "9:\n"
"str q24, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q25, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, #0x10\n"
"str q26, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, #0x10\n"
"str q27, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, #0x10\n"
"str q28, [c_ptr4]\n"
- "add c_ptr4, c_ptr4, #0x10\n"
"str q29, [c_ptr5]\n"
- "add c_ptr5, c_ptr5, #0x10\n"
"str q30, [c_ptr6]\n"
- "add c_ptr6, c_ptr6, #0x10\n"
"str q31, [c_ptr7]\n"
- "add c_ptr7, c_ptr7, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -1004,28 +1061,30 @@
"ld1 {v6.b}[14], [a_ptr6]\n"
"ld1 {v7.b}[14], [a_ptr7]\n"
"3:\n"
- "movi v24.4s, #0\n"
"ldr q16, [%[b_ptr0]]\n"
- "movi v25.4s, #0\n"
- "ldr q17, [%[b_ptr0], #0x10]\n"
- "movi v26.4s, #0\n"
- "ldr q18, [%[b_ptr0], #0x20]\n"
- "movi v27.4s, #0\n"
- "ldr q19, [%[b_ptr0], #0x30]\n"
- "movi v28.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
- "movi v29.4s, #0\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
- "movi v30.4s, #0\n"
+ "ldr q18, [%[b_ptr0], #0x20]\n"
"prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
- "movi v31.4s, #0\n"
+ "ldr q19, [%[b_ptr0], #0x30]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
- ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x40\n"
+ "cbz %[loops], 6f\n"
+ "movi v24.4s, #0\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "movi v25.4s, #0\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n"
+ ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n"
".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n"
".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n"
".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n"
@@ -1055,50 +1114,47 @@
".inst 0x6fa5ea7d // udot v29.4s, v19.16b, v5.4b[3]\n"
".inst 0x6fa6ea7e // udot v30.4s, v19.16b, v6.4b[3]\n"
".inst 0x6fa7ea7f // udot v31.4s, v19.16b, v7.4b[3]\n"
- "cbz %[loops], 6f\n"
- "ldr q16, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ldr q17, [%[b_ptr0], #0x10]\n"
- "ldr q18, [%[b_ptr0], #0x20]\n"
"b.eq 7f\n"
"8:\n"
"str q24, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"movi v24.4s, #0\n"
- "ldr q19, [%[b_ptr0], #0x30]\n"
- "add %[b_ptr0], %[b_ptr0], #0x40\n"
- "str q25, [c_ptr1]\n"
+ "ldr q16, [%[b_ptr0]]\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
- "movi v25.4s, #0\n"
+ "str q25, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
+ "movi v25.4s, #0\n"
+ "ldr q18, [%[b_ptr0], #0x20]\n"
".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
"str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "ldr q19, [%[b_ptr0], #0x30]\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n"
"str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
- "add c_ptr2, c_ptr2, #0x10\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n"
"str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
- "add c_ptr3, c_ptr3, #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n"
"str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
- "add c_ptr4, c_ptr4, #0x10\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n"
"str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
- "add c_ptr5, c_ptr5, #0x10\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n"
"str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
- "add c_ptr6, c_ptr6, #0x10\n"
- ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n"
"add c_ptr7, c_ptr7, #0x10\n"
+ ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x40\n"
".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n"
- "ldr q16, [%[b_ptr0]]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n"
@@ -1114,7 +1170,6 @@
".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n"
- "ldr q17, [%[b_ptr0], #0x10]\n"
".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
".inst 0x6f81ea59 // udot v25.4s, v18.16b, v1.4b[2]\n"
".inst 0x6f82ea5a // udot v26.4s, v18.16b, v2.4b[2]\n"
@@ -1123,7 +1178,6 @@
".inst 0x6f85ea5d // udot v29.4s, v18.16b, v5.4b[2]\n"
".inst 0x6f86ea5e // udot v30.4s, v18.16b, v6.4b[2]\n"
".inst 0x6f87ea5f // udot v31.4s, v18.16b, v7.4b[2]\n"
- "ldr q18, [%[b_ptr0], #0x20]\n"
".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n"
".inst 0x6fa1ea79 // udot v25.4s, v19.16b, v1.4b[3]\n"
".inst 0x6fa2ea7a // udot v26.4s, v19.16b, v2.4b[3]\n"
@@ -1137,14 +1191,16 @@
"str q24, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v24.4s, #0\n"
- "ldr q19, [%[b_ptr0], #0x30]\n"
- "add %[b_ptr0], %[b_ptr0], #0x40\n"
+ "ldr q16, [%[b_ptr0]]\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
"str q25, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
+ "ldr q18, [%[b_ptr0], #0x20]\n"
".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
"str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
+ "ldr q19, [%[b_ptr0], #0x30]\n"
"add c_ptr2, c_ptr2, #0x10\n"
".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n"
"str q27, [c_ptr3]\n"
@@ -1167,8 +1223,9 @@
"movi v31.4s, #0\n"
"add c_ptr7, c_ptr7, #0x10\n"
".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n"
- ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x40\n"
".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n"
+ ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n"
".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n"
".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n"
@@ -1192,23 +1249,58 @@
".inst 0x6fa5ea7d // udot v29.4s, v19.16b, v5.4b[3]\n"
".inst 0x6fa6ea7e // udot v30.4s, v19.16b, v6.4b[3]\n"
".inst 0x6fa7ea7f // udot v31.4s, v19.16b, v7.4b[3]\n"
+ "b 9f\n"
"6:\n"
+ "movi v24.4s, #0\n"
+ "movi v25.4s, #0\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x6f81e219 // udot v25.4s, v16.16b, v1.4b[0]\n"
+ ".inst 0x6f82e21a // udot v26.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x6f83e21b // udot v27.4s, v16.16b, v3.4b[0]\n"
+ ".inst 0x6f84e21c // udot v28.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x6f85e21d // udot v29.4s, v16.16b, v5.4b[0]\n"
+ ".inst 0x6f86e21e // udot v30.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x6f87e21f // udot v31.4s, v16.16b, v7.4b[0]\n"
+ ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x6fa1e239 // udot v25.4s, v17.16b, v1.4b[1]\n"
+ ".inst 0x6fa2e23a // udot v26.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x6fa3e23b // udot v27.4s, v17.16b, v3.4b[1]\n"
+ ".inst 0x6fa4e23c // udot v28.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x6fa5e23d // udot v29.4s, v17.16b, v5.4b[1]\n"
+ ".inst 0x6fa6e23e // udot v30.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x6fa7e23f // udot v31.4s, v17.16b, v7.4b[1]\n"
+ ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x6f81ea59 // udot v25.4s, v18.16b, v1.4b[2]\n"
+ ".inst 0x6f82ea5a // udot v26.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x6f83ea5b // udot v27.4s, v18.16b, v3.4b[2]\n"
+ ".inst 0x6f84ea5c // udot v28.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x6f85ea5d // udot v29.4s, v18.16b, v5.4b[2]\n"
+ ".inst 0x6f86ea5e // udot v30.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x6f87ea5f // udot v31.4s, v18.16b, v7.4b[2]\n"
+ ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n"
+ ".inst 0x6fa1ea79 // udot v25.4s, v19.16b, v1.4b[3]\n"
+ ".inst 0x6fa2ea7a // udot v26.4s, v19.16b, v2.4b[3]\n"
+ ".inst 0x6fa3ea7b // udot v27.4s, v19.16b, v3.4b[3]\n"
+ ".inst 0x6fa4ea7c // udot v28.4s, v19.16b, v4.4b[3]\n"
+ ".inst 0x6fa5ea7d // udot v29.4s, v19.16b, v5.4b[3]\n"
+ ".inst 0x6fa6ea7e // udot v30.4s, v19.16b, v6.4b[3]\n"
+ ".inst 0x6fa7ea7f // udot v31.4s, v19.16b, v7.4b[3]\n"
+ "9:\n"
"str q24, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q25, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, #0x10\n"
"str q26, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, #0x10\n"
"str q27, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, #0x10\n"
"str q28, [c_ptr4]\n"
- "add c_ptr4, c_ptr4, #0x10\n"
"str q29, [c_ptr5]\n"
- "add c_ptr5, c_ptr5, #0x10\n"
"str q30, [c_ptr6]\n"
- "add c_ptr6, c_ptr6, #0x10\n"
"str q31, [c_ptr7]\n"
- "add c_ptr7, c_ptr7, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -1287,26 +1379,34 @@
"add c_ptr1, %[c_ptr0], #0x0\n"
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
+ "cbnz %[odds], 2f\n"
"ldr q0, [%[a_ptr0]], #0x10\n"
"ldr q2, [a_ptr1], #0x10\n"
"ldr q4, [a_ptr2], #0x10\n"
"ldr q6, [a_ptr3], #0x10\n"
+ "ldr s1, [%[a_ptr0]]\n"
+ "ldr q8, [a_ptr4], #0x10\n"
+ "ldr s3, [a_ptr1]\n"
+ "ldr q10, [a_ptr5], #0x10\n"
+ "ldr s5, [a_ptr2]\n"
+ "ldr q12, [a_ptr6], #0x10\n"
+ "ldr s7, [a_ptr3]\n"
+ "ldr q14, [a_ptr7], #0x10\n"
+ "ldr s9, [a_ptr4]\n"
+ "ldr s11, [a_ptr5]\n"
+ "ldr s13, [a_ptr6]\n"
+ "ldr s15, [a_ptr7]\n"
+ "b 3f\n"
+ "2:\n"
+ "ldr q0, [%[a_ptr0]], #0x10\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "ldr q2, [a_ptr1], #0x10\n"
+ "ldr q4, [a_ptr2], #0x10\n"
+ "ldr q6, [a_ptr3], #0x10\n"
"ldr q8, [a_ptr4], #0x10\n"
"ldr q10, [a_ptr5], #0x10\n"
"ldr q12, [a_ptr6], #0x10\n"
"ldr q14, [a_ptr7], #0x10\n"
- "cbnz %[odds], 2f\n"
- "ldr s1, [%[a_ptr0]]\n"
- "ldr s3, [a_ptr1]\n"
- "ldr s5, [a_ptr2]\n"
- "ldr s7, [a_ptr3]\n"
- "ldr s9, [a_ptr4]\n"
- "ldr s11, [a_ptr5]\n"
- "ldr s13, [a_ptr6]\n"
- "ldr s15, [a_ptr7]\n"
- "b 3f\n"
- "2:\n"
- "subs %[odds], %[odds], #0x1\n"
"b.ne 4f\n"
"ldr b1, [%[a_ptr0]]\n"
"ldr b3, [a_ptr1]\n"
@@ -1339,30 +1439,32 @@
"ld1 {v13.b}[2], [a_ptr6]\n"
"ld1 {v15.b}[2], [a_ptr7]\n"
"3:\n"
- "movi v24.4s, #0\n"
"ldr q16, [%[b_ptr0]]\n"
- "movi v25.4s, #0\n"
- "ldr q17, [%[b_ptr0], #0x10]\n"
- "movi v26.4s, #0\n"
- "ldr q18, [%[b_ptr0], #0x20]\n"
- "movi v27.4s, #0\n"
- "ldr q19, [%[b_ptr0], #0x30]\n"
- "movi v28.4s, #0\n"
- "ldr q20, [%[b_ptr0], #0x40]\n"
- "movi v29.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
- "movi v30.4s, #0\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
- "movi v31.4s, #0\n"
+ "ldr q18, [%[b_ptr0], #0x20]\n"
"prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
- ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ "ldr q19, [%[b_ptr0], #0x30]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
- ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
+ "ldr q20, [%[b_ptr0], #0x40]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x50\n"
+ "cbz %[loops], 6f\n"
+ "movi v24.4s, #0\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "movi v25.4s, #0\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
@@ -1399,51 +1501,48 @@
".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n"
".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n"
".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n"
- "cbz %[loops], 6f\n"
- "ldr q16, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ldr q17, [%[b_ptr0], #0x10]\n"
- "ldr q18, [%[b_ptr0], #0x20]\n"
- "ldr q19, [%[b_ptr0], #0x30]\n"
"b.eq 7f\n"
"8:\n"
"str q24, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"movi v24.4s, #0\n"
- "ldr q20, [%[b_ptr0], #0x40]\n"
- "add %[b_ptr0], %[b_ptr0], #0x50\n"
- "str q25, [c_ptr1]\n"
+ "ldr q16, [%[b_ptr0]]\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
- "movi v25.4s, #0\n"
+ "str q25, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
+ "movi v25.4s, #0\n"
+ "ldr q18, [%[b_ptr0], #0x20]\n"
".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
"str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "ldr q19, [%[b_ptr0], #0x30]\n"
+ "ldr q20, [%[b_ptr0], #0x40]\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
"str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
- "add c_ptr2, c_ptr2, #0x10\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
"str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
- "add c_ptr3, c_ptr3, #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
"str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
- "add c_ptr4, c_ptr4, #0x10\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
"str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
- "add c_ptr5, c_ptr5, #0x10\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
"str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
- "add c_ptr6, c_ptr6, #0x10\n"
- ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
"add c_ptr7, c_ptr7, #0x10\n"
+ ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x50\n"
".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
- "ldr q16, [%[b_ptr0]]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
@@ -1459,7 +1558,6 @@
".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n"
"prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n"
- "ldr q17, [%[b_ptr0], #0x10]\n"
".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n"
".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n"
@@ -1468,7 +1566,6 @@
".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n"
".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n"
".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n"
- "ldr q18, [%[b_ptr0], #0x20]\n"
".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n"
".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n"
".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n"
@@ -1477,7 +1574,6 @@
".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n"
".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n"
".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n"
- "ldr q19, [%[b_ptr0], #0x30]\n"
".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n"
".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n"
".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n"
@@ -1491,14 +1587,17 @@
"str q24, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v24.4s, #0\n"
- "ldr q20, [%[b_ptr0], #0x40]\n"
- "add %[b_ptr0], %[b_ptr0], #0x50\n"
+ "ldr q16, [%[b_ptr0]]\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
"str q25, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
+ "ldr q18, [%[b_ptr0], #0x20]\n"
".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
"str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
+ "ldr q19, [%[b_ptr0], #0x30]\n"
+ "ldr q20, [%[b_ptr0], #0x40]\n"
"add c_ptr2, c_ptr2, #0x10\n"
".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
"str q27, [c_ptr3]\n"
@@ -1521,8 +1620,9 @@
"movi v31.4s, #0\n"
"add c_ptr7, c_ptr7, #0x10\n"
".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
- ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x50\n"
".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
+ ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
@@ -1554,23 +1654,66 @@
".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n"
".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n"
".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n"
+ "b 9f\n"
"6:\n"
+ "movi v24.4s, #0\n"
+ "movi v25.4s, #0\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
+ ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
+ ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
+ ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
+ ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n"
+ ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n"
+ ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n"
+ ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n"
+ ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n"
+ ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n"
+ ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n"
+ ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n"
+ ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n"
+ ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n"
+ ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n"
+ ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n"
+ ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n"
+ ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n"
+ ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n"
+ ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n"
+ ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n"
+ ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n"
+ ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n"
+ ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n"
+ ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n"
+ ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n"
+ ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n"
+ ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n"
+ "9:\n"
"str q24, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q25, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, #0x10\n"
"str q26, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, #0x10\n"
"str q27, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, #0x10\n"
"str q28, [c_ptr4]\n"
- "add c_ptr4, c_ptr4, #0x10\n"
"str q29, [c_ptr5]\n"
- "add c_ptr5, c_ptr5, #0x10\n"
"str q30, [c_ptr6]\n"
- "add c_ptr6, c_ptr6, #0x10\n"
"str q31, [c_ptr7]\n"
- "add c_ptr7, c_ptr7, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -1649,34 +1792,42 @@
"add c_ptr1, %[c_ptr0], #0x0\n"
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
+ "cbnz %[odds], 2f\n"
"ldr q0, [%[a_ptr0]], #0x10\n"
"ldr q2, [a_ptr1], #0x10\n"
"ldr q4, [a_ptr2], #0x10\n"
"ldr q6, [a_ptr3], #0x10\n"
- "ldr q8, [a_ptr4], #0x10\n"
- "ldr q10, [a_ptr5], #0x10\n"
- "ldr q12, [a_ptr6], #0x10\n"
- "ldr q14, [a_ptr7], #0x10\n"
- "cbnz %[odds], 2f\n"
"ldr d1, [%[a_ptr0]]\n"
+ "ldr q8, [a_ptr4], #0x10\n"
"ldr d3, [a_ptr1]\n"
+ "ldr q10, [a_ptr5], #0x10\n"
"ldr d5, [a_ptr2]\n"
+ "ldr q12, [a_ptr6], #0x10\n"
"ldr d7, [a_ptr3]\n"
+ "ldr q14, [a_ptr7], #0x10\n"
"ldr d9, [a_ptr4]\n"
"ldr d11, [a_ptr5]\n"
"ldr d13, [a_ptr6]\n"
"ldr d15, [a_ptr7]\n"
"b 3f\n"
"2:\n"
+ "ldr q0, [%[a_ptr0]], #0x10\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "ldr q2, [a_ptr1], #0x10\n"
+ "ldr q4, [a_ptr2], #0x10\n"
"ldr s1, [%[a_ptr0]], #0x4\n"
+ "ldr q6, [a_ptr3], #0x10\n"
"ldr s3, [a_ptr1], #0x4\n"
+ "ldr q8, [a_ptr4], #0x10\n"
"ldr s5, [a_ptr2], #0x4\n"
+ "ldr q10, [a_ptr5], #0x10\n"
"ldr s7, [a_ptr3], #0x4\n"
+ "ldr q12, [a_ptr6], #0x10\n"
"ldr s9, [a_ptr4], #0x4\n"
+ "ldr q14, [a_ptr7], #0x10\n"
"ldr s11, [a_ptr5], #0x4\n"
"ldr s13, [a_ptr6], #0x4\n"
"ldr s15, [a_ptr7], #0x4\n"
- "subs %[odds], %[odds], #0x1\n"
"b.ne 4f\n"
"ld1 {v1.b}[4], [%[a_ptr0]]\n"
"ld1 {v3.b}[4], [a_ptr1]\n"
@@ -1709,32 +1860,34 @@
"ld1 {v13.b}[6], [a_ptr6]\n"
"ld1 {v15.b}[6], [a_ptr7]\n"
"3:\n"
- "movi v24.4s, #0\n"
"ldr q16, [%[b_ptr0]]\n"
- "movi v25.4s, #0\n"
- "ldr q17, [%[b_ptr0], #0x10]\n"
- "movi v26.4s, #0\n"
- "ldr q18, [%[b_ptr0], #0x20]\n"
- "movi v27.4s, #0\n"
- "ldr q19, [%[b_ptr0], #0x30]\n"
- "movi v28.4s, #0\n"
- "ldr q20, [%[b_ptr0], #0x40]\n"
- "movi v29.4s, #0\n"
- "ldr q21, [%[b_ptr0], #0x50]\n"
- "movi v30.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
- "movi v31.4s, #0\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
- ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ "ldr q18, [%[b_ptr0], #0x20]\n"
"prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
- ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
+ "ldr q19, [%[b_ptr0], #0x30]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
- ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
+ "ldr q20, [%[b_ptr0], #0x40]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
+ "ldr q21, [%[b_ptr0], #0x50]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
"add %[b_ptr0], %[b_ptr0], #0x60\n"
+ "cbz %[loops], 6f\n"
+ "movi v24.4s, #0\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "movi v25.4s, #0\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
@@ -1778,68 +1931,64 @@
".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n"
".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n"
".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n"
- "cbz %[loops], 6f\n"
- "ldr q16, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ldr q17, [%[b_ptr0], #0x10]\n"
- "ldr q18, [%[b_ptr0], #0x20]\n"
- "ldr q19, [%[b_ptr0], #0x30]\n"
- "ldr q20, [%[b_ptr0], #0x40]\n"
"b.eq 7f\n"
"8:\n"
"str q24, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"movi v24.4s, #0\n"
- "ldr q21, [%[b_ptr0], #0x50]\n"
- "add %[b_ptr0], %[b_ptr0], #0x60\n"
- "str q25, [c_ptr1]\n"
+ "ldr q16, [%[b_ptr0]]\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
- "movi v25.4s, #0\n"
+ "str q25, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
+ "movi v25.4s, #0\n"
+ "ldr q18, [%[b_ptr0], #0x20]\n"
".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
"str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "ldr q19, [%[b_ptr0], #0x30]\n"
+ "ldr q20, [%[b_ptr0], #0x40]\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
"str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
- "add c_ptr2, c_ptr2, #0x10\n"
+ "ldr q21, [%[b_ptr0], #0x50]\n"
".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
"str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
- "add c_ptr3, c_ptr3, #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
"str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
- "add c_ptr4, c_ptr4, #0x10\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
"str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
- "add c_ptr5, c_ptr5, #0x10\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
"str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
- "add c_ptr6, c_ptr6, #0x10\n"
- ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
"add c_ptr7, c_ptr7, #0x10\n"
+ ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x60\n"
".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
- "ldr q16, [%[b_ptr0]]\n"
- ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
+ "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
+ "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
+ "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n"
- "ldr q17, [%[b_ptr0], #0x10]\n"
+ "prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n"
".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n"
@@ -1848,7 +1997,6 @@
".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n"
".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n"
".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n"
- "ldr q18, [%[b_ptr0], #0x20]\n"
".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n"
".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n"
".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n"
@@ -1857,7 +2005,6 @@
".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n"
".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n"
".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n"
- "ldr q19, [%[b_ptr0], #0x30]\n"
".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n"
".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n"
".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n"
@@ -1866,7 +2013,6 @@
".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n"
".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n"
".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n"
- "ldr q20, [%[b_ptr0], #0x40]\n"
".inst 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n"
".inst 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n"
".inst 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n"
@@ -1880,20 +2026,25 @@
"str q24, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v24.4s, #0\n"
- "ldr q21, [%[b_ptr0], #0x50]\n"
- "add %[b_ptr0], %[b_ptr0], #0x60\n"
+ "ldr q16, [%[b_ptr0]]\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
"str q25, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
+ "ldr q18, [%[b_ptr0], #0x20]\n"
".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
"str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
+ "ldr q19, [%[b_ptr0], #0x30]\n"
+ "ldr q20, [%[b_ptr0], #0x40]\n"
"add c_ptr2, c_ptr2, #0x10\n"
".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
"str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
- "add c_ptr3, c_ptr3, #0x10\n"
+ "ldr q21, [%[b_ptr0], #0x50]\n"
".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
"str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
@@ -1910,7 +2061,7 @@
"movi v31.4s, #0\n"
"add c_ptr7, c_ptr7, #0x10\n"
".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
- ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x60\n"
".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
@@ -1951,23 +2102,74 @@
".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n"
".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n"
".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n"
+ "b 9f\n"
"6:\n"
+ "movi v24.4s, #0\n"
+ "movi v25.4s, #0\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
+ ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
+ ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
+ ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
+ ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n"
+ ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n"
+ ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n"
+ ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n"
+ ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n"
+ ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n"
+ ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n"
+ ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n"
+ ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n"
+ ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n"
+ ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n"
+ ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n"
+ ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n"
+ ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n"
+ ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n"
+ ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n"
+ ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n"
+ ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n"
+ ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n"
+ ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n"
+ ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n"
+ ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n"
+ ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n"
+ ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n"
+ ".inst 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n"
+ ".inst 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n"
+ ".inst 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n"
+ ".inst 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n"
+ ".inst 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n"
+ ".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n"
+ ".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n"
+ ".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n"
+ "9:\n"
"str q24, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q25, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, #0x10\n"
"str q26, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, #0x10\n"
"str q27, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, #0x10\n"
"str q28, [c_ptr4]\n"
- "add c_ptr4, c_ptr4, #0x10\n"
"str q29, [c_ptr5]\n"
- "add c_ptr5, c_ptr5, #0x10\n"
"str q30, [c_ptr6]\n"
- "add c_ptr6, c_ptr6, #0x10\n"
"str q31, [c_ptr7]\n"
- "add c_ptr7, c_ptr7, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -2106,34 +2308,36 @@
"ld1 {v13.b}[10], [a_ptr6]\n"
"ld1 {v15.b}[10], [a_ptr7]\n"
"3:\n"
- "movi v24.4s, #0\n"
"ldr q16, [%[b_ptr0]]\n"
- "movi v25.4s, #0\n"
- "ldr q17, [%[b_ptr0], #0x10]\n"
- "movi v26.4s, #0\n"
- "ldr q18, [%[b_ptr0], #0x20]\n"
- "movi v27.4s, #0\n"
- "ldr q19, [%[b_ptr0], #0x30]\n"
- "movi v28.4s, #0\n"
- "ldr q20, [%[b_ptr0], #0x40]\n"
- "movi v29.4s, #0\n"
- "ldr q21, [%[b_ptr0], #0x50]\n"
- "movi v30.4s, #0\n"
- "ldr q22, [%[b_ptr0], #0x60]\n"
- "movi v31.4s, #0\n"
"prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
- ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
- ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
+ "ldr q18, [%[b_ptr0], #0x20]\n"
"prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
- ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
+ "ldr q19, [%[b_ptr0], #0x30]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
- ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
+ "ldr q20, [%[b_ptr0], #0x40]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
+ "ldr q21, [%[b_ptr0], #0x50]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
+ "ldr q22, [%[b_ptr0], #0x60]\n"
"add %[b_ptr0], %[b_ptr0], #0x70\n"
+ "cbz %[loops], 6f\n"
+ "movi v24.4s, #0\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "movi v25.4s, #0\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
+ ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
@@ -2184,34 +2388,31 @@
".inst 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n"
".inst 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n"
".inst 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n"
- "cbz %[loops], 6f\n"
- "ldr q16, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ldr q17, [%[b_ptr0], #0x10]\n"
- "ldr q18, [%[b_ptr0], #0x20]\n"
- "ldr q19, [%[b_ptr0], #0x30]\n"
- "ldr q20, [%[b_ptr0], #0x40]\n"
- "ldr q21, [%[b_ptr0], #0x50]\n"
"b.eq 7f\n"
"8:\n"
"str q24, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"movi v24.4s, #0\n"
- "ldr q22, [%[b_ptr0], #0x60]\n"
- "add %[b_ptr0], %[b_ptr0], #0x70\n"
- "str q25, [c_ptr1]\n"
+ "ldr q16, [%[b_ptr0]]\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
- "movi v25.4s, #0\n"
+ "str q25, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
+ "movi v25.4s, #0\n"
+ "ldr q18, [%[b_ptr0], #0x20]\n"
".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
"str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "ldr q19, [%[b_ptr0], #0x30]\n"
+ "ldr q20, [%[b_ptr0], #0x40]\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
"str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
- "add c_ptr2, c_ptr2, #0x10\n"
+ "ldr q21, [%[b_ptr0], #0x50]\n"
".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
+ "ldr q22, [%[b_ptr0], #0x60]\n"
+ ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
"str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
"add c_ptr3, c_ptr3, #0x10\n"
@@ -2230,24 +2431,23 @@
".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
"add c_ptr7, c_ptr7, #0x10\n"
".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
- "ldr q16, [%[b_ptr0]]\n"
- ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x70\n"
".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
+ "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
+ "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n"
- "ldr q17, [%[b_ptr0], #0x10]\n"
+ "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
+ "prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n"
".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n"
".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n"
@@ -2255,7 +2455,6 @@
".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n"
".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n"
".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n"
- "ldr q18, [%[b_ptr0], #0x20]\n"
".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n"
".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n"
".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n"
@@ -2264,7 +2463,6 @@
".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n"
".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n"
".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n"
- "ldr q19, [%[b_ptr0], #0x30]\n"
".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n"
".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n"
".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n"
@@ -2273,7 +2471,6 @@
".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n"
".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n"
".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n"
- "ldr q20, [%[b_ptr0], #0x40]\n"
".inst 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n"
".inst 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n"
".inst 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n"
@@ -2282,7 +2479,6 @@
".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n"
".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n"
".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n"
- "ldr q21, [%[b_ptr0], #0x50]\n"
".inst 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n"
".inst 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n"
".inst 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n"
@@ -2296,38 +2492,44 @@
"str q24, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v24.4s, #0\n"
- "ldr q22, [%[b_ptr0], #0x60]\n"
- "add %[b_ptr0], %[b_ptr0], #0x70\n"
+ "ldr q16, [%[b_ptr0]]\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
"str q25, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
+ "ldr q18, [%[b_ptr0], #0x20]\n"
".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
"str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
+ "ldr q19, [%[b_ptr0], #0x30]\n"
+ "ldr q20, [%[b_ptr0], #0x40]\n"
"add c_ptr2, c_ptr2, #0x10\n"
".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
"str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
- "add c_ptr3, c_ptr3, #0x10\n"
+ "ldr q21, [%[b_ptr0], #0x50]\n"
".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
+ "ldr q22, [%[b_ptr0], #0x60]\n"
+ ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
"str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
- "add c_ptr4, c_ptr4, #0x10\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
"str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
- "add c_ptr5, c_ptr5, #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
"str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
- "add c_ptr6, c_ptr6, #0x10\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
"str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
- "add c_ptr7, c_ptr7, #0x10\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
- ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
+ "add c_ptr7, c_ptr7, #0x10\n"
".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x70\n"
".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
@@ -2375,23 +2577,82 @@
".inst 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n"
".inst 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n"
".inst 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n"
+ "b 9f\n"
"6:\n"
+ "movi v24.4s, #0\n"
+ "movi v25.4s, #0\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
+ ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
+ ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
+ ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
+ ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
+ ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
+ ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n"
+ ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n"
+ ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n"
+ ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n"
+ ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n"
+ ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n"
+ ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n"
+ ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n"
+ ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n"
+ ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n"
+ ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n"
+ ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n"
+ ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n"
+ ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n"
+ ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n"
+ ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n"
+ ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n"
+ ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n"
+ ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n"
+ ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n"
+ ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n"
+ ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n"
+ ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n"
+ ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n"
+ ".inst 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n"
+ ".inst 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n"
+ ".inst 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n"
+ ".inst 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n"
+ ".inst 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n"
+ ".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n"
+ ".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n"
+ ".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n"
+ ".inst 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n"
+ ".inst 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n"
+ ".inst 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n"
+ ".inst 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n"
+ ".inst 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n"
+ ".inst 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n"
+ ".inst 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n"
+ ".inst 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n"
+ "9:\n"
"str q24, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q25, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, #0x10\n"
"str q26, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, #0x10\n"
"str q27, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, #0x10\n"
"str q28, [c_ptr4]\n"
- "add c_ptr4, c_ptr4, #0x10\n"
"str q29, [c_ptr5]\n"
- "add c_ptr5, c_ptr5, #0x10\n"
"str q30, [c_ptr6]\n"
- "add c_ptr6, c_ptr6, #0x10\n"
"str q31, [c_ptr7]\n"
- "add c_ptr7, c_ptr7, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -2471,6 +2732,7 @@
"add c_ptr1, %[c_ptr0], #0x0\n"
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
+ "cbnz %[odds], 2f\n"
"ldr q0, [%[a_ptr0]], #0x10\n"
"ldr q2, [a_ptr1], #0x10\n"
"ldr q4, [a_ptr2], #0x10\n"
@@ -2479,7 +2741,6 @@
"ldr q10, [a_ptr5], #0x10\n"
"ldr q12, [a_ptr6], #0x10\n"
"ldr q14, [a_ptr7], #0x10\n"
- "cbnz %[odds], 2f\n"
"ldr q1, [%[a_ptr0]]\n"
"ldr q3, [a_ptr1]\n"
"ldr q5, [a_ptr2]\n"
@@ -2490,15 +2751,24 @@
"ldr q15, [a_ptr7]\n"
"b 3f\n"
"2:\n"
+ "ldr q0, [%[a_ptr0]], #0x10\n"
+ "subs %[odds], %[odds], #0x1\n"
+ "ldr q2, [a_ptr1], #0x10\n"
+ "ldr q4, [a_ptr2], #0x10\n"
"ldr d1, [%[a_ptr0]], #0x8\n"
+ "ldr q6, [a_ptr3], #0x10\n"
"ldr d3, [a_ptr1], #0x8\n"
+ "ldr q8, [a_ptr4], #0x10\n"
"ldr d5, [a_ptr2], #0x8\n"
+ "ldr q10, [a_ptr5], #0x10\n"
"ldr d7, [a_ptr3], #0x8\n"
+ "ldr q12, [a_ptr6], #0x10\n"
"ldr d9, [a_ptr4], #0x8\n"
+ "ldr q14, [a_ptr7], #0x10\n"
"ldr d11, [a_ptr5], #0x8\n"
"ldr d13, [a_ptr6], #0x8\n"
- "ldr d15, [a_ptr7], #0x8\n"
"ld1 {v1.s}[2], [%[a_ptr0]], #4\n"
+ "ldr d15, [a_ptr7], #0x8\n"
"ld1 {v3.s}[2], [a_ptr1], #4\n"
"ld1 {v5.s}[2], [a_ptr2], #4\n"
"ld1 {v7.s}[2], [a_ptr3], #4\n"
@@ -2506,7 +2776,6 @@
"ld1 {v11.s}[2], [a_ptr5], #4\n"
"ld1 {v13.s}[2], [a_ptr6], #4\n"
"ld1 {v15.s}[2], [a_ptr7], #4\n"
- "subs %[odds], %[odds], #0x1\n"
"b.ne 4f\n"
"ld1 {v1.b}[12], [%[a_ptr0]]\n"
"ld1 {v3.b}[12], [a_ptr1]\n"
@@ -2539,36 +2808,38 @@
"ld1 {v13.b}[14], [a_ptr6]\n"
"ld1 {v15.b}[14], [a_ptr7]\n"
"3:\n"
- "movi v24.4s, #0\n"
"ldr q16, [%[b_ptr0]]\n"
- "movi v25.4s, #0\n"
- "ldr q17, [%[b_ptr0], #0x10]\n"
- "movi v26.4s, #0\n"
- "ldr q18, [%[b_ptr0], #0x20]\n"
- "movi v27.4s, #0\n"
- "ldr q19, [%[b_ptr0], #0x30]\n"
- "movi v28.4s, #0\n"
- "ldr q20, [%[b_ptr0], #0x40]\n"
- "movi v29.4s, #0\n"
- "ldr q21, [%[b_ptr0], #0x50]\n"
- "movi v30.4s, #0\n"
- "ldr q22, [%[b_ptr0], #0x60]\n"
- "movi v31.4s, #0\n"
- "ldr q23, [%[b_ptr0], #0x70]\n"
- ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
- ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
- ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
+ "ldr q18, [%[b_ptr0], #0x20]\n"
"prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
- ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
+ "ldr q19, [%[b_ptr0], #0x30]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
- ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
+ "ldr q20, [%[b_ptr0], #0x40]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
+ "ldr q21, [%[b_ptr0], #0x50]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
+ "ldr q22, [%[b_ptr0], #0x60]\n"
+ "ldr q23, [%[b_ptr0], #0x70]\n"
"add %[b_ptr0], %[b_ptr0], #0x80\n"
+ "cbz %[loops], 6f\n"
+ "movi v24.4s, #0\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "movi v25.4s, #0\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
+ ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
+ ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
@@ -2626,39 +2897,37 @@
".inst 0x6fabeafd // udot v29.4s, v23.16b, v11.4b[3]\n"
".inst 0x6fadeafe // udot v30.4s, v23.16b, v13.4b[3]\n"
".inst 0x6fafeaff // udot v31.4s, v23.16b, v15.4b[3]\n"
- "cbz %[loops], 6f\n"
- "ldr q16, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ldr q17, [%[b_ptr0], #0x10]\n"
- "ldr q18, [%[b_ptr0], #0x20]\n"
- "ldr q19, [%[b_ptr0], #0x30]\n"
- "ldr q20, [%[b_ptr0], #0x40]\n"
- "ldr q21, [%[b_ptr0], #0x50]\n"
- "ldr q22, [%[b_ptr0], #0x60]\n"
"b.eq 7f\n"
"8:\n"
"str q24, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"movi v24.4s, #0\n"
- "ldr q23, [%[b_ptr0], #0x70]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
- "str q25, [c_ptr1]\n"
+ "ldr q16, [%[b_ptr0]]\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
- "movi v25.4s, #0\n"
+ "str q25, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
+ "movi v25.4s, #0\n"
+ "ldr q18, [%[b_ptr0], #0x20]\n"
".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
"str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "ldr q19, [%[b_ptr0], #0x30]\n"
+ "ldr q20, [%[b_ptr0], #0x40]\n"
+ "add c_ptr2, c_ptr2, #0x10\n"
".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
"str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
- "add c_ptr2, c_ptr2, #0x10\n"
+ "ldr q21, [%[b_ptr0], #0x50]\n"
".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
+ "ldr q22, [%[b_ptr0], #0x60]\n"
+ ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
"str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
- "add c_ptr3, c_ptr3, #0x10\n"
+ "ldr q23, [%[b_ptr0], #0x70]\n"
".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
"str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
"add c_ptr4, c_ptr4, #0x10\n"
@@ -2673,32 +2942,29 @@
".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
"add c_ptr7, c_ptr7, #0x10\n"
".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
- "ldr q16, [%[b_ptr0]]\n"
- ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n"
- "prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
+ "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n"
- "ldr q17, [%[b_ptr0], #0x10]\n"
+ "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
+ "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n"
+ "prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n"
".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n"
".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n"
".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n"
".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n"
".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n"
- "ldr q18, [%[b_ptr0], #0x20]\n"
".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n"
".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n"
".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n"
@@ -2707,7 +2973,6 @@
".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n"
".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n"
".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n"
- "ldr q19, [%[b_ptr0], #0x30]\n"
".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n"
".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n"
".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n"
@@ -2716,7 +2981,6 @@
".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n"
".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n"
".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n"
- "ldr q20, [%[b_ptr0], #0x40]\n"
".inst 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n"
".inst 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n"
".inst 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n"
@@ -2725,7 +2989,6 @@
".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n"
".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n"
".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n"
- "ldr q21, [%[b_ptr0], #0x50]\n"
".inst 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n"
".inst 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n"
".inst 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n"
@@ -2734,7 +2997,6 @@
".inst 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n"
".inst 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n"
".inst 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n"
- "ldr q22, [%[b_ptr0], #0x60]\n"
".inst 0x6fa1eaf8 // udot v24.4s, v23.16b, v1.4b[3]\n"
".inst 0x6fa3eaf9 // udot v25.4s, v23.16b, v3.4b[3]\n"
".inst 0x6fa5eafa // udot v26.4s, v23.16b, v5.4b[3]\n"
@@ -2748,38 +3010,119 @@
"str q24, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"movi v24.4s, #0\n"
- "ldr q23, [%[b_ptr0], #0x70]\n"
- "add %[b_ptr0], %[b_ptr0], #0x80\n"
+ "ldr q16, [%[b_ptr0]]\n"
+ "ldr q17, [%[b_ptr0], #0x10]\n"
"str q25, [c_ptr1]\n"
"add c_ptr1, c_ptr1, #0x10\n"
"movi v25.4s, #0\n"
+ "ldr q18, [%[b_ptr0], #0x20]\n"
".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
"str q26, [c_ptr2]\n"
"movi v26.4s, #0\n"
+ "ldr q19, [%[b_ptr0], #0x30]\n"
+ "ldr q20, [%[b_ptr0], #0x40]\n"
"add c_ptr2, c_ptr2, #0x10\n"
".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
"str q27, [c_ptr3]\n"
"movi v27.4s, #0\n"
- "add c_ptr3, c_ptr3, #0x10\n"
+ "ldr q21, [%[b_ptr0], #0x50]\n"
".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
+ "ldr q22, [%[b_ptr0], #0x60]\n"
+ ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
"str q28, [c_ptr4]\n"
"movi v28.4s, #0\n"
- "add c_ptr4, c_ptr4, #0x10\n"
+ "ldr q23, [%[b_ptr0], #0x70]\n"
".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
+ "add c_ptr3, c_ptr3, #0x10\n"
+ ".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
"str q29, [c_ptr5]\n"
"movi v29.4s, #0\n"
- "add c_ptr5, c_ptr5, #0x10\n"
+ "add c_ptr4, c_ptr4, #0x10\n"
".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
"str q30, [c_ptr6]\n"
"movi v30.4s, #0\n"
- "add c_ptr6, c_ptr6, #0x10\n"
+ "add c_ptr5, c_ptr5, #0x10\n"
".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
"str q31, [c_ptr7]\n"
"movi v31.4s, #0\n"
- "add c_ptr7, c_ptr7, #0x10\n"
+ "add c_ptr6, c_ptr6, #0x10\n"
".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
- ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
+ "add c_ptr7, c_ptr7, #0x10\n"
".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
+ "add %[b_ptr0], %[b_ptr0], #0x80\n"
+ ".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
+ ".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
+ ".inst 0x6fa8e23c // udot v28.4s, v17.16b, v8.4b[1]\n"
+ ".inst 0x6faae23d // udot v29.4s, v17.16b, v10.4b[1]\n"
+ ".inst 0x6face23e // udot v30.4s, v17.16b, v12.4b[1]\n"
+ ".inst 0x6faee23f // udot v31.4s, v17.16b, v14.4b[1]\n"
+ ".inst 0x6f80ea58 // udot v24.4s, v18.16b, v0.4b[2]\n"
+ ".inst 0x6f82ea59 // udot v25.4s, v18.16b, v2.4b[2]\n"
+ ".inst 0x6f84ea5a // udot v26.4s, v18.16b, v4.4b[2]\n"
+ ".inst 0x6f86ea5b // udot v27.4s, v18.16b, v6.4b[2]\n"
+ ".inst 0x6f88ea5c // udot v28.4s, v18.16b, v8.4b[2]\n"
+ ".inst 0x6f8aea5d // udot v29.4s, v18.16b, v10.4b[2]\n"
+ ".inst 0x6f8cea5e // udot v30.4s, v18.16b, v12.4b[2]\n"
+ ".inst 0x6f8eea5f // udot v31.4s, v18.16b, v14.4b[2]\n"
+ ".inst 0x6fa0ea78 // udot v24.4s, v19.16b, v0.4b[3]\n"
+ ".inst 0x6fa2ea79 // udot v25.4s, v19.16b, v2.4b[3]\n"
+ ".inst 0x6fa4ea7a // udot v26.4s, v19.16b, v4.4b[3]\n"
+ ".inst 0x6fa6ea7b // udot v27.4s, v19.16b, v6.4b[3]\n"
+ ".inst 0x6fa8ea7c // udot v28.4s, v19.16b, v8.4b[3]\n"
+ ".inst 0x6faaea7d // udot v29.4s, v19.16b, v10.4b[3]\n"
+ ".inst 0x6facea7e // udot v30.4s, v19.16b, v12.4b[3]\n"
+ ".inst 0x6faeea7f // udot v31.4s, v19.16b, v14.4b[3]\n"
+ ".inst 0x6f81e298 // udot v24.4s, v20.16b, v1.4b[0]\n"
+ ".inst 0x6f83e299 // udot v25.4s, v20.16b, v3.4b[0]\n"
+ ".inst 0x6f85e29a // udot v26.4s, v20.16b, v5.4b[0]\n"
+ ".inst 0x6f87e29b // udot v27.4s, v20.16b, v7.4b[0]\n"
+ ".inst 0x6f89e29c // udot v28.4s, v20.16b, v9.4b[0]\n"
+ ".inst 0x6f8be29d // udot v29.4s, v20.16b, v11.4b[0]\n"
+ ".inst 0x6f8de29e // udot v30.4s, v20.16b, v13.4b[0]\n"
+ ".inst 0x6f8fe29f // udot v31.4s, v20.16b, v15.4b[0]\n"
+ ".inst 0x6fa1e2b8 // udot v24.4s, v21.16b, v1.4b[1]\n"
+ ".inst 0x6fa3e2b9 // udot v25.4s, v21.16b, v3.4b[1]\n"
+ ".inst 0x6fa5e2ba // udot v26.4s, v21.16b, v5.4b[1]\n"
+ ".inst 0x6fa7e2bb // udot v27.4s, v21.16b, v7.4b[1]\n"
+ ".inst 0x6fa9e2bc // udot v28.4s, v21.16b, v9.4b[1]\n"
+ ".inst 0x6fabe2bd // udot v29.4s, v21.16b, v11.4b[1]\n"
+ ".inst 0x6fade2be // udot v30.4s, v21.16b, v13.4b[1]\n"
+ ".inst 0x6fafe2bf // udot v31.4s, v21.16b, v15.4b[1]\n"
+ ".inst 0x6f81ead8 // udot v24.4s, v22.16b, v1.4b[2]\n"
+ ".inst 0x6f83ead9 // udot v25.4s, v22.16b, v3.4b[2]\n"
+ ".inst 0x6f85eada // udot v26.4s, v22.16b, v5.4b[2]\n"
+ ".inst 0x6f87eadb // udot v27.4s, v22.16b, v7.4b[2]\n"
+ ".inst 0x6f89eadc // udot v28.4s, v22.16b, v9.4b[2]\n"
+ ".inst 0x6f8beadd // udot v29.4s, v22.16b, v11.4b[2]\n"
+ ".inst 0x6f8deade // udot v30.4s, v22.16b, v13.4b[2]\n"
+ ".inst 0x6f8feadf // udot v31.4s, v22.16b, v15.4b[2]\n"
+ ".inst 0x6fa1eaf8 // udot v24.4s, v23.16b, v1.4b[3]\n"
+ ".inst 0x6fa3eaf9 // udot v25.4s, v23.16b, v3.4b[3]\n"
+ ".inst 0x6fa5eafa // udot v26.4s, v23.16b, v5.4b[3]\n"
+ ".inst 0x6fa7eafb // udot v27.4s, v23.16b, v7.4b[3]\n"
+ ".inst 0x6fa9eafc // udot v28.4s, v23.16b, v9.4b[3]\n"
+ ".inst 0x6fabeafd // udot v29.4s, v23.16b, v11.4b[3]\n"
+ ".inst 0x6fadeafe // udot v30.4s, v23.16b, v13.4b[3]\n"
+ ".inst 0x6fafeaff // udot v31.4s, v23.16b, v15.4b[3]\n"
+ "b 9f\n"
+ "6:\n"
+ "movi v24.4s, #0\n"
+ "movi v25.4s, #0\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ ".inst 0x6f80e218 // udot v24.4s, v16.16b, v0.4b[0]\n"
+ ".inst 0x6f82e219 // udot v25.4s, v16.16b, v2.4b[0]\n"
+ ".inst 0x6f84e21a // udot v26.4s, v16.16b, v4.4b[0]\n"
+ ".inst 0x6f86e21b // udot v27.4s, v16.16b, v6.4b[0]\n"
+ ".inst 0x6f88e21c // udot v28.4s, v16.16b, v8.4b[0]\n"
+ ".inst 0x6f8ae21d // udot v29.4s, v16.16b, v10.4b[0]\n"
+ ".inst 0x6f8ce21e // udot v30.4s, v16.16b, v12.4b[0]\n"
+ ".inst 0x6f8ee21f // udot v31.4s, v16.16b, v14.4b[0]\n"
+ ".inst 0x6fa0e238 // udot v24.4s, v17.16b, v0.4b[1]\n"
".inst 0x6fa2e239 // udot v25.4s, v17.16b, v2.4b[1]\n"
".inst 0x6fa4e23a // udot v26.4s, v17.16b, v4.4b[1]\n"
".inst 0x6fa6e23b // udot v27.4s, v17.16b, v6.4b[1]\n"
@@ -2835,23 +3178,16 @@
".inst 0x6fabeafd // udot v29.4s, v23.16b, v11.4b[3]\n"
".inst 0x6fadeafe // udot v30.4s, v23.16b, v13.4b[3]\n"
".inst 0x6fafeaff // udot v31.4s, v23.16b, v15.4b[3]\n"
- "6:\n"
+ "9:\n"
"str q24, [%[c_ptr0]]\n"
"add %[c_ptr0], %[c_ptr0], #0x10\n"
"str q25, [c_ptr1]\n"
- "add c_ptr1, c_ptr1, #0x10\n"
"str q26, [c_ptr2]\n"
- "add c_ptr2, c_ptr2, #0x10\n"
"str q27, [c_ptr3]\n"
- "add c_ptr3, c_ptr3, #0x10\n"
"str q28, [c_ptr4]\n"
- "add c_ptr4, c_ptr4, #0x10\n"
"str q29, [c_ptr5]\n"
- "add c_ptr5, c_ptr5, #0x10\n"
"str q30, [c_ptr6]\n"
- "add c_ptr6, c_ptr6, #0x10\n"
"str q31, [c_ptr7]\n"
- "add c_ptr7, c_ptr7, #0x10\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_4x8.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_gemv_fp32_mla_8VL.hpp
similarity index 74%
copy from src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_4x8.hpp
copy to src/core/NEON/kernels/arm_gemm/kernels/sve_gemv_fp32_mla_8VL.hpp
index b60401b..57fd9c9 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_4x8.hpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_gemv_fp32_mla_8VL.hpp
@@ -25,32 +25,26 @@
#ifdef __aarch64__
-
-#include "../std_transforms_fixed.hpp"
+#include "../performance_parameters.hpp"
+#include "../std_transforms_sve.hpp"
namespace arm_gemm
{
// Actual kernel implementations
-void a64_hybrid_fp32_mla_4x8(const float *, int, const float *, float *, int, int, int, int, const float *, Activation, bool);
+void sve_gemv_fp32_mla_8VL(const float *, const float *, float *, size_t, size_t, const float *, Activation, bool);
-class hybrid_fp32_mla_4x8
+class cls_sve_gemv_fp32_mla_8VL
{
public:
typedef float operand_type;
typedef float result_type;
- typedef void (*kern_type)(const float *, int, const float *, float *, int, int, int, int, const float *, Activation, bool);
-
- /* Kernel blocking parameters */
- static constexpr unsigned int out_height()
- {
- return 8;
- }
+ typedef void (*kern_type)(const float *, const float *, float *, size_t, size_t, const float *, Activation, bool);
static unsigned int out_width()
{
- return 4;
+ return 8 * get_vector_length<float>();
}
static constexpr unsigned int k_unroll()
@@ -73,14 +67,13 @@
return true;
}
- StdTransformsFixed<operand_type, result_type, 8, 4, 1> transforms = {};
+ StdTransformsSVE<operand_type, result_type, 1, 8, 1> transforms = {};
// Default to the generic kernel
- kern_type kernel=a64_hybrid_fp32_mla_4x8;
+ kern_type kernel=sve_gemv_fp32_mla_8VL;
- hybrid_fp32_mla_4x8(const CPUInfo *)
+ cls_sve_gemv_fp32_mla_8VL(const CPUInfo *)
{
-
}
};
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_gemv_fp32_mla_8VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_gemv_fp32_mla_8VL/generic.cpp
new file mode 100644
index 0000000..c62e319
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_gemv_fp32_mla_8VL/generic.cpp
@@ -0,0 +1,1372 @@
+/*
+ * Copyright (c) 2019-2020 Arm Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+#ifdef __ARM_FEATURE_SVE
+
+#include "arm_gemm.hpp"
+#include "../../utils.hpp"
+
+#include <cassert>
+
+namespace arm_gemm {
+
+void sve_gemv_fp32_mla_8VL (
+ const float *A_ptr, const float *B_ptr, float *output_ptr,
+ size_t N, size_t K,
+ const float *bias, Activation act, bool
+)
+{
+ struct KernelArgs {
+ float maxval = static_cast<float>(std::numeric_limits<float>::infinity());
+ float minval = - static_cast<float>(std::numeric_limits<float>::infinity());
+ const float *B_ptr = {};
+ size_t output_offset = {};
+ unsigned int input_initial_col = {};
+ } ka;
+
+ unsigned long flags=0;
+ ka.B_ptr = B_ptr;
+ switch(act.type) {
+ default:
+ case Activation::Type::None:
+ break;
+ case Activation::Type::BoundedReLU:
+ ka.maxval = static_cast<float>(act.param1);
+ /* fall through */
+ case Activation::Type::ReLU:
+ ka.minval = 0;
+ flags |= 0x2;
+ break;
+ }
+ __asm__ __volatile__(
+ "ptrue p2.b\n"
+ "cntw x24\n"
+ "add x23, %x[N], x24\n"
+ "sub x23, x23, #0x1\n"
+ "udiv x23, x23, x24\n"
+ "mov x22, %x[bias]\n"
+ "1:" // Column loop
+ "cmp x23, #0x8\n"
+ "bge 50f\n"
+ "cmp x23, #0x6\n"
+ "bgt 43f\n"
+ "beq 36f\n"
+ "cmp x23, #0x4\n"
+ "bgt 29f\n"
+ "beq 22f\n"
+ "cmp x23, #0x2\n"
+ "bgt 15f\n"
+ "beq 8f\n"
+ "mov x21, %x[K]\n"
+ "mov x20, %x[A_ptr]\n"
+ "whilelt p1.s, XZR, %x[N]\n"
+ "cbz x22, 2f\n"
+ "ld1w { z24.s }, p2/Z, [x22]\n"
+ "addvl x22, x22, #1\n"
+ "b 3f\n"
+ "2:" // Width 1: no bias
+ "mov z24.b, #0x0\n"
+ "3:" // Width 1: setup done
+ "cmp x21, #0x4\n"
+ "ble 5f\n"
+ "4:" // Width 1: Multiply loop: Main loop head
+ "ld1w { z1.s }, p2/Z, [%x[B_ptr]]\n"
+ "whilelt p0.s, XZR, x21\n"
+ "addvl %x[B_ptr], %x[B_ptr], #8\n"
+ "ld1rqw { z0.s }, p0/Z, [x20]\n"
+ "fmla z24.s, z1.s, z0.s[0]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "add x20, x20, #0x10\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "sub x21, x21, #0x4\n"
+ "ld1w { z2.s }, p2/Z, [%x[B_ptr]]\n"
+ "fmla z24.s, z2.s, z0.s[1]\n"
+ "addvl %x[B_ptr], %x[B_ptr], #8\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "cmp x21, #0x4\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "ld1w { z3.s }, p2/Z, [%x[B_ptr]]\n"
+ "fmla z24.s, z3.s, z0.s[2]\n"
+ "addvl %x[B_ptr], %x[B_ptr], #8\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "ld1w { z4.s }, p2/Z, [%x[B_ptr]]\n"
+ "fmla z24.s, z4.s, z0.s[3]\n"
+ "addvl %x[B_ptr], %x[B_ptr], #8\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "prfm pldl1keep, [x20, #0x80]\n"
+ "bgt 4b\n"
+ "5:" // Width 1: Multiply loop: Single iteration only
+ "ld1w { z5.s }, p2/Z, [%x[B_ptr]]\n"
+ "whilelt p0.s, XZR, x21\n"
+ "addvl %x[B_ptr], %x[B_ptr], #8\n"
+ "ld1rqw { z0.s }, p0/Z, [x20]\n"
+ "fmla z24.s, z5.s, z0.s[0]\n"
+ "add x20, x20, #0x10\n"
+ "subs x21, x21, #0x1\n"
+ "ble 6f\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "subs x21, x21, #0x1\n"
+ "ld1w { z6.s }, p2/Z, [%x[B_ptr]]\n"
+ "fmla z24.s, z6.s, z0.s[1]\n"
+ "addvl %x[B_ptr], %x[B_ptr], #8\n"
+ "ble 6f\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "subs x21, x21, #0x1\n"
+ "ld1w { z7.s }, p2/Z, [%x[B_ptr]]\n"
+ "fmla z24.s, z7.s, z0.s[2]\n"
+ "addvl %x[B_ptr], %x[B_ptr], #8\n"
+ "ble 6f\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "ld1w { z8.s }, p2/Z, [%x[B_ptr]]\n"
+ "fmla z24.s, z8.s, z0.s[3]\n"
+ "addvl %x[B_ptr], %x[B_ptr], #8\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "6:" // Width 1: Multiply loop: multiply skip
+ "prfm pldl1keep, [x20, #0x80]\n"
+ "prfm pstl1keep, [%x[output_ptr], #0x0]\n"
+ "tbz %x[flags], #1, 7f\n"
+ "add x19, %x[args_ptr], %[offset_min]\n"
+ "ld1rw { z17.s }, p2/Z, [x19]\n"
+ "add x19, %x[args_ptr], %[offset_max]\n"
+ "ld1rw { z16.s }, p2/Z, [x19]\n"
+ "fmin z24.s, p2/M, z24.s, z16.s\n"
+ "fmax z24.s, p2/M, z24.s, z17.s\n"
+ "7:" // Width 1: No activation
+ "st1w { z24.s }, p1, [%x[output_ptr]]\n"
+ "addvl %x[output_ptr], %x[output_ptr], #1\n"
+ "b 57f\n"
+ "8:" // Width 2
+ "mov x21, %x[K]\n"
+ "mov x20, %x[A_ptr]\n"
+ "sub x19, %x[N], x24\n"
+ "whilelt p1.s, XZR, x19\n"
+ "cbz x22, 9f\n"
+ "ld1w { z24.s }, p2/Z, [x22]\n"
+ "ld1w { z25.s }, p2/Z, [x22, #1, MUL VL]\n"
+ "addvl x22, x22, #2\n"
+ "b 10f\n"
+ "9:" // Width 2: no bias
+ "mov z24.b, #0x0\n"
+ "mov z25.b, #0x0\n"
+ "10:" // Width 2: setup done
+ "cmp x21, #0x4\n"
+ "ble 12f\n"
+ "11:" // Width 2: Multiply loop: Main loop head
+ "ld1w { z1.s }, p2/Z, [%x[B_ptr]]\n"
+ "whilelt p0.s, XZR, x21\n"
+ "ld1w { z2.s }, p2/Z, [%x[B_ptr], #1, MUL VL]\n"
+ "addvl %x[B_ptr], %x[B_ptr], #8\n"
+ "ld1rqw { z0.s }, p0/Z, [x20]\n"
+ "fmla z24.s, z1.s, z0.s[0]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "add x20, x20, #0x10\n"
+ "fmla z25.s, z2.s, z0.s[0]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "sub x21, x21, #0x4\n"
+ "ld1w { z3.s }, p2/Z, [%x[B_ptr]]\n"
+ "fmla z24.s, z3.s, z0.s[1]\n"
+ "ld1w { z4.s }, p2/Z, [%x[B_ptr], #1, MUL VL]\n"
+ "addvl %x[B_ptr], %x[B_ptr], #8\n"
+ "fmla z25.s, z4.s, z0.s[1]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "cmp x21, #0x4\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "ld1w { z5.s }, p2/Z, [%x[B_ptr]]\n"
+ "fmla z24.s, z5.s, z0.s[2]\n"
+ "ld1w { z6.s }, p2/Z, [%x[B_ptr], #1, MUL VL]\n"
+ "addvl %x[B_ptr], %x[B_ptr], #8\n"
+ "fmla z25.s, z6.s, z0.s[2]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "ld1w { z7.s }, p2/Z, [%x[B_ptr]]\n"
+ "fmla z24.s, z7.s, z0.s[3]\n"
+ "ld1w { z8.s }, p2/Z, [%x[B_ptr], #1, MUL VL]\n"
+ "addvl %x[B_ptr], %x[B_ptr], #8\n"
+ "fmla z25.s, z8.s, z0.s[3]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "prfm pldl1keep, [x20, #0x80]\n"
+ "bgt 11b\n"
+ "12:" // Width 2: Multiply loop: Single iteration only
+ "ld1w { z9.s }, p2/Z, [%x[B_ptr]]\n"
+ "whilelt p0.s, XZR, x21\n"
+ "ld1w { z10.s }, p2/Z, [%x[B_ptr], #1, MUL VL]\n"
+ "addvl %x[B_ptr], %x[B_ptr], #8\n"
+ "ld1rqw { z0.s }, p0/Z, [x20]\n"
+ "fmla z24.s, z9.s, z0.s[0]\n"
+ "add x20, x20, #0x10\n"
+ "fmla z25.s, z10.s, z0.s[0]\n"
+ "subs x21, x21, #0x1\n"
+ "ble 13f\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "subs x21, x21, #0x1\n"
+ "ld1w { z11.s }, p2/Z, [%x[B_ptr]]\n"
+ "fmla z24.s, z11.s, z0.s[1]\n"
+ "ld1w { z12.s }, p2/Z, [%x[B_ptr], #1, MUL VL]\n"
+ "addvl %x[B_ptr], %x[B_ptr], #8\n"
+ "fmla z25.s, z12.s, z0.s[1]\n"
+ "ble 13f\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "subs x21, x21, #0x1\n"
+ "ld1w { z13.s }, p2/Z, [%x[B_ptr]]\n"
+ "fmla z24.s, z13.s, z0.s[2]\n"
+ "ld1w { z14.s }, p2/Z, [%x[B_ptr], #1, MUL VL]\n"
+ "addvl %x[B_ptr], %x[B_ptr], #8\n"
+ "fmla z25.s, z14.s, z0.s[2]\n"
+ "ble 13f\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "ld1w { z15.s }, p2/Z, [%x[B_ptr]]\n"
+ "fmla z24.s, z15.s, z0.s[3]\n"
+ "ld1w { z16.s }, p2/Z, [%x[B_ptr], #1, MUL VL]\n"
+ "addvl %x[B_ptr], %x[B_ptr], #8\n"
+ "fmla z25.s, z16.s, z0.s[3]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "13:" // Width 2: Multiply loop: multiply skip
+ "prfm pldl1keep, [x20, #0x80]\n"
+ "prfm pstl1keep, [%x[output_ptr], #0x0]\n"
+ "tbz %x[flags], #1, 14f\n"
+ "add x19, %x[args_ptr], %[offset_min]\n"
+ "ld1rw { z17.s }, p2/Z, [x19]\n"
+ "add x19, %x[args_ptr], %[offset_max]\n"
+ "ld1rw { z16.s }, p2/Z, [x19]\n"
+ "fmin z24.s, p2/M, z24.s, z16.s\n"
+ "fmin z25.s, p2/M, z25.s, z16.s\n"
+ "fmax z24.s, p2/M, z24.s, z17.s\n"
+ "fmax z25.s, p2/M, z25.s, z17.s\n"
+ "14:" // Width 2: No activation
+ "st1w { z24.s }, p2, [%x[output_ptr]]\n"
+ "st1w { z25.s }, p1, [%x[output_ptr], #1, MUL VL]\n"
+ "addvl %x[output_ptr], %x[output_ptr], #2\n"
+ "b 57f\n"
+ "15:" // Width 3
+ "mov x21, %x[K]\n"
+ "mov x20, %x[A_ptr]\n"
+ "mov x19, #0x2\n"
+ "msub x19, x24, x19, %x[N]\n"
+ "whilelt p1.s, XZR, x19\n"
+ "cbz x22, 16f\n"
+ "ld1w { z24.s }, p2/Z, [x22]\n"
+ "ld1w { z25.s }, p2/Z, [x22, #1, MUL VL]\n"
+ "ld1w { z26.s }, p2/Z, [x22, #2, MUL VL]\n"
+ "addvl x22, x22, #3\n"
+ "b 17f\n"
+ "16:" // Width 3: no bias
+ "mov z24.b, #0x0\n"
+ "mov z25.b, #0x0\n"
+ "mov z26.b, #0x0\n"
+ "17:" // Width 3: setup done
+ "cmp x21, #0x4\n"
+ "ble 19f\n"
+ "18:" // Width 3: Multiply loop: Main loop head
+ "ld1w { z1.s }, p2/Z, [%x[B_ptr]]\n"
+ "whilelt p0.s, XZR, x21\n"
+ "ld1w { z2.s }, p2/Z, [%x[B_ptr], #1, MUL VL]\n"
+ "sub x21, x21, #0x4\n"
+ "ld1rqw { z0.s }, p0/Z, [x20]\n"
+ "fmla z24.s, z1.s, z0.s[0]\n"
+ "ld1w { z3.s }, p2/Z, [%x[B_ptr], #2, MUL VL]\n"
+ "add x20, x20, #0x10\n"
+ "fmla z25.s, z2.s, z0.s[0]\n"
+ "addvl %x[B_ptr], %x[B_ptr], #8\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "fmla z26.s, z3.s, z0.s[0]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "cmp x21, #0x4\n"
+ "ld1w { z4.s }, p2/Z, [%x[B_ptr]]\n"
+ "fmla z24.s, z4.s, z0.s[1]\n"
+ "ld1w { z5.s }, p2/Z, [%x[B_ptr], #1, MUL VL]\n"
+ "ld1w { z6.s }, p2/Z, [%x[B_ptr], #2, MUL VL]\n"
+ "fmla z25.s, z5.s, z0.s[1]\n"
+ "addvl %x[B_ptr], %x[B_ptr], #8\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "fmla z26.s, z6.s, z0.s[1]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "ld1w { z7.s }, p2/Z, [%x[B_ptr]]\n"
+ "fmla z24.s, z7.s, z0.s[2]\n"
+ "ld1w { z8.s }, p2/Z, [%x[B_ptr], #1, MUL VL]\n"
+ "ld1w { z9.s }, p2/Z, [%x[B_ptr], #2, MUL VL]\n"
+ "fmla z25.s, z8.s, z0.s[2]\n"
+ "addvl %x[B_ptr], %x[B_ptr], #8\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "fmla z26.s, z9.s, z0.s[2]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "ld1w { z10.s }, p2/Z, [%x[B_ptr]]\n"
+ "fmla z24.s, z10.s, z0.s[3]\n"
+ "ld1w { z11.s }, p2/Z, [%x[B_ptr], #1, MUL VL]\n"
+ "ld1w { z12.s }, p2/Z, [%x[B_ptr], #2, MUL VL]\n"
+ "fmla z25.s, z11.s, z0.s[3]\n"
+ "addvl %x[B_ptr], %x[B_ptr], #8\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "fmla z26.s, z12.s, z0.s[3]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "prfm pldl1keep, [x20, #0x80]\n"
+ "bgt 18b\n"
+ "19:" // Width 3: Multiply loop: Single iteration only
+ "ld1w { z13.s }, p2/Z, [%x[B_ptr]]\n"
+ "whilelt p0.s, XZR, x21\n"
+ "ld1w { z14.s }, p2/Z, [%x[B_ptr], #1, MUL VL]\n"
+ "subs x21, x21, #0x1\n"
+ "ld1rqw { z0.s }, p0/Z, [x20]\n"
+ "fmla z24.s, z13.s, z0.s[0]\n"
+ "ld1w { z15.s }, p2/Z, [%x[B_ptr], #2, MUL VL]\n"
+ "add x20, x20, #0x10\n"
+ "fmla z25.s, z14.s, z0.s[0]\n"
+ "addvl %x[B_ptr], %x[B_ptr], #8\n"
+ "fmla z26.s, z15.s, z0.s[0]\n"
+ "ble 20f\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "subs x21, x21, #0x1\n"
+ "ld1w { z16.s }, p2/Z, [%x[B_ptr]]\n"
+ "fmla z24.s, z16.s, z0.s[1]\n"
+ "ld1w { z17.s }, p2/Z, [%x[B_ptr], #1, MUL VL]\n"
+ "ld1w { z18.s }, p2/Z, [%x[B_ptr], #2, MUL VL]\n"
+ "fmla z25.s, z17.s, z0.s[1]\n"
+ "addvl %x[B_ptr], %x[B_ptr], #8\n"
+ "fmla z26.s, z18.s, z0.s[1]\n"
+ "ble 20f\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "subs x21, x21, #0x1\n"
+ "ld1w { z19.s }, p2/Z, [%x[B_ptr]]\n"
+ "fmla z24.s, z19.s, z0.s[2]\n"
+ "ld1w { z20.s }, p2/Z, [%x[B_ptr], #1, MUL VL]\n"
+ "ld1w { z21.s }, p2/Z, [%x[B_ptr], #2, MUL VL]\n"
+ "fmla z25.s, z20.s, z0.s[2]\n"
+ "addvl %x[B_ptr], %x[B_ptr], #8\n"
+ "fmla z26.s, z21.s, z0.s[2]\n"
+ "ble 20f\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "ld1w { z22.s }, p2/Z, [%x[B_ptr]]\n"
+ "fmla z24.s, z22.s, z0.s[3]\n"
+ "ld1w { z23.s }, p2/Z, [%x[B_ptr], #1, MUL VL]\n"
+ "ld1w { z1.s }, p2/Z, [%x[B_ptr], #2, MUL VL]\n"
+ "fmla z25.s, z23.s, z0.s[3]\n"
+ "addvl %x[B_ptr], %x[B_ptr], #8\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "fmla z26.s, z1.s, z0.s[3]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "20:" // Width 3: Multiply loop: multiply skip
+ "prfm pldl1keep, [x20, #0x80]\n"
+ "prfm pstl1keep, [%x[output_ptr], #0x0]\n"
+ "tbz %x[flags], #1, 21f\n"
+ "add x19, %x[args_ptr], %[offset_min]\n"
+ "ld1rw { z17.s }, p2/Z, [x19]\n"
+ "add x19, %x[args_ptr], %[offset_max]\n"
+ "ld1rw { z16.s }, p2/Z, [x19]\n"
+ "fmin z24.s, p2/M, z24.s, z16.s\n"
+ "fmin z25.s, p2/M, z25.s, z16.s\n"
+ "fmin z26.s, p2/M, z26.s, z16.s\n"
+ "fmax z24.s, p2/M, z24.s, z17.s\n"
+ "fmax z25.s, p2/M, z25.s, z17.s\n"
+ "fmax z26.s, p2/M, z26.s, z17.s\n"
+ "21:" // Width 3: No activation
+ "st1w { z24.s }, p2, [%x[output_ptr]]\n"
+ "st1w { z25.s }, p2, [%x[output_ptr], #1, MUL VL]\n"
+ "st1w { z26.s }, p1, [%x[output_ptr], #2, MUL VL]\n"
+ "addvl %x[output_ptr], %x[output_ptr], #3\n"
+ "b 57f\n"
+ "22:" // Width 4
+ "mov x21, %x[K]\n"
+ "mov x20, %x[A_ptr]\n"
+ "mov x19, #0x3\n"
+ "msub x19, x24, x19, %x[N]\n"
+ "whilelt p1.s, XZR, x19\n"
+ "cbz x22, 23f\n"
+ "ld1w { z24.s }, p2/Z, [x22]\n"
+ "ld1w { z25.s }, p2/Z, [x22, #1, MUL VL]\n"
+ "ld1w { z26.s }, p2/Z, [x22, #2, MUL VL]\n"
+ "ld1w { z27.s }, p2/Z, [x22, #3, MUL VL]\n"
+ "addvl x22, x22, #4\n"
+ "b 24f\n"
+ "23:" // Width 4: no bias
+ "mov z24.b, #0x0\n"
+ "mov z25.b, #0x0\n"
+ "mov z26.b, #0x0\n"
+ "mov z27.b, #0x0\n"
+ "24:" // Width 4: setup done
+ "cmp x21, #0x4\n"
+ "ble 26f\n"
+ "25:" // Width 4: Multiply loop: Main loop head
+ "ld1w { z1.s }, p2/Z, [%x[B_ptr]]\n"
+ "whilelt p0.s, XZR, x21\n"
+ "ld1w { z2.s }, p2/Z, [%x[B_ptr], #1, MUL VL]\n"
+ "sub x21, x21, #0x4\n"
+ "ld1rqw { z0.s }, p0/Z, [x20]\n"
+ "fmla z24.s, z1.s, z0.s[0]\n"
+ "ld1w { z3.s }, p2/Z, [%x[B_ptr], #2, MUL VL]\n"
+ "add x20, x20, #0x10\n"
+ "fmla z25.s, z2.s, z0.s[0]\n"
+ "ld1w { z4.s }, p2/Z, [%x[B_ptr], #3, MUL VL]\n"
+ "addvl %x[B_ptr], %x[B_ptr], #8\n"
+ "fmla z26.s, z3.s, z0.s[0]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "cmp x21, #0x4\n"
+ "fmla z27.s, z4.s, z0.s[0]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "ld1w { z5.s }, p2/Z, [%x[B_ptr]]\n"
+ "fmla z24.s, z5.s, z0.s[1]\n"
+ "ld1w { z6.s }, p2/Z, [%x[B_ptr], #1, MUL VL]\n"
+ "ld1w { z7.s }, p2/Z, [%x[B_ptr], #2, MUL VL]\n"
+ "fmla z25.s, z6.s, z0.s[1]\n"
+ "ld1w { z8.s }, p2/Z, [%x[B_ptr], #3, MUL VL]\n"
+ "addvl %x[B_ptr], %x[B_ptr], #8\n"
+ "fmla z26.s, z7.s, z0.s[1]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "fmla z27.s, z8.s, z0.s[1]\n"
+ "ld1w { z9.s }, p2/Z, [%x[B_ptr]]\n"
+ "ld1w { z10.s }, p2/Z, [%x[B_ptr], #1, MUL VL]\n"
+ "fmla z24.s, z9.s, z0.s[2]\n"
+ "ld1w { z11.s }, p2/Z, [%x[B_ptr], #2, MUL VL]\n"
+ "fmla z25.s, z10.s, z0.s[2]\n"
+ "ld1w { z12.s }, p2/Z, [%x[B_ptr], #3, MUL VL]\n"
+ "addvl %x[B_ptr], %x[B_ptr], #8\n"
+ "fmla z26.s, z11.s, z0.s[2]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "fmla z27.s, z12.s, z0.s[2]\n"
+ "ld1w { z13.s }, p2/Z, [%x[B_ptr]]\n"
+ "fmla z24.s, z13.s, z0.s[3]\n"
+ "ld1w { z14.s }, p2/Z, [%x[B_ptr], #1, MUL VL]\n"
+ "ld1w { z15.s }, p2/Z, [%x[B_ptr], #2, MUL VL]\n"
+ "fmla z25.s, z14.s, z0.s[3]\n"
+ "ld1w { z16.s }, p2/Z, [%x[B_ptr], #3, MUL VL]\n"
+ "addvl %x[B_ptr], %x[B_ptr], #8\n"
+ "fmla z26.s, z15.s, z0.s[3]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "fmla z27.s, z16.s, z0.s[3]\n"
+ "prfm pldl1keep, [x20, #0x80]\n"
+ "bgt 25b\n"
+ "26:" // Width 4: Multiply loop: Single iteration only
+ "ld1w { z17.s }, p2/Z, [%x[B_ptr]]\n"
+ "whilelt p0.s, XZR, x21\n"
+ "ld1w { z18.s }, p2/Z, [%x[B_ptr], #1, MUL VL]\n"
+ "subs x21, x21, #0x1\n"
+ "ld1rqw { z0.s }, p0/Z, [x20]\n"
+ "fmla z24.s, z17.s, z0.s[0]\n"
+ "ld1w { z19.s }, p2/Z, [%x[B_ptr], #2, MUL VL]\n"
+ "add x20, x20, #0x10\n"
+ "fmla z25.s, z18.s, z0.s[0]\n"
+ "ld1w { z20.s }, p2/Z, [%x[B_ptr], #3, MUL VL]\n"
+ "addvl %x[B_ptr], %x[B_ptr], #8\n"
+ "fmla z26.s, z19.s, z0.s[0]\n"
+ "fmla z27.s, z20.s, z0.s[0]\n"
+ "ble 27f\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "subs x21, x21, #0x1\n"
+ "ld1w { z21.s }, p2/Z, [%x[B_ptr]]\n"
+ "fmla z24.s, z21.s, z0.s[1]\n"
+ "ld1w { z22.s }, p2/Z, [%x[B_ptr], #1, MUL VL]\n"
+ "ld1w { z23.s }, p2/Z, [%x[B_ptr], #2, MUL VL]\n"
+ "fmla z25.s, z22.s, z0.s[1]\n"
+ "ld1w { z1.s }, p2/Z, [%x[B_ptr], #3, MUL VL]\n"
+ "addvl %x[B_ptr], %x[B_ptr], #8\n"
+ "fmla z26.s, z23.s, z0.s[1]\n"
+ "fmla z27.s, z1.s, z0.s[1]\n"
+ "ble 27f\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "subs x21, x21, #0x1\n"
+ "ld1w { z2.s }, p2/Z, [%x[B_ptr]]\n"
+ "fmla z24.s, z2.s, z0.s[2]\n"
+ "ld1w { z3.s }, p2/Z, [%x[B_ptr], #1, MUL VL]\n"
+ "ld1w { z4.s }, p2/Z, [%x[B_ptr], #2, MUL VL]\n"
+ "fmla z25.s, z3.s, z0.s[2]\n"
+ "ld1w { z5.s }, p2/Z, [%x[B_ptr], #3, MUL VL]\n"
+ "addvl %x[B_ptr], %x[B_ptr], #8\n"
+ "fmla z26.s, z4.s, z0.s[2]\n"
+ "fmla z27.s, z5.s, z0.s[2]\n"
+ "ble 27f\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "ld1w { z6.s }, p2/Z, [%x[B_ptr]]\n"
+ "fmla z24.s, z6.s, z0.s[3]\n"
+ "ld1w { z7.s }, p2/Z, [%x[B_ptr], #1, MUL VL]\n"
+ "ld1w { z8.s }, p2/Z, [%x[B_ptr], #2, MUL VL]\n"
+ "fmla z25.s, z7.s, z0.s[3]\n"
+ "ld1w { z9.s }, p2/Z, [%x[B_ptr], #3, MUL VL]\n"
+ "addvl %x[B_ptr], %x[B_ptr], #8\n"
+ "fmla z26.s, z8.s, z0.s[3]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "fmla z27.s, z9.s, z0.s[3]\n"
+ "27:" // Width 4: Multiply loop: multiply skip
+ "prfm pldl1keep, [x20, #0x80]\n"
+ "prfm pstl1keep, [%x[output_ptr], #0x0]\n"
+ "tbz %x[flags], #1, 28f\n"
+ "add x19, %x[args_ptr], %[offset_min]\n"
+ "ld1rw { z17.s }, p2/Z, [x19]\n"
+ "add x19, %x[args_ptr], %[offset_max]\n"
+ "ld1rw { z16.s }, p2/Z, [x19]\n"
+ "fmin z24.s, p2/M, z24.s, z16.s\n"
+ "fmin z25.s, p2/M, z25.s, z16.s\n"
+ "fmin z26.s, p2/M, z26.s, z16.s\n"
+ "fmin z27.s, p2/M, z27.s, z16.s\n"
+ "fmax z24.s, p2/M, z24.s, z17.s\n"
+ "fmax z25.s, p2/M, z25.s, z17.s\n"
+ "fmax z26.s, p2/M, z26.s, z17.s\n"
+ "fmax z27.s, p2/M, z27.s, z17.s\n"
+ "28:" // Width 4: No activation
+ "st1w { z24.s }, p2, [%x[output_ptr]]\n"
+ "st1w { z25.s }, p2, [%x[output_ptr], #1, MUL VL]\n"
+ "st1w { z26.s }, p2, [%x[output_ptr], #2, MUL VL]\n"
+ "st1w { z27.s }, p1, [%x[output_ptr], #3, MUL VL]\n"
+ "addvl %x[output_ptr], %x[output_ptr], #4\n"
+ "b 57f\n"
+ "29:" // Width 5
+ "mov x21, %x[K]\n"
+ "mov x20, %x[A_ptr]\n"
+ "mov x19, #0x4\n"
+ "msub x19, x24, x19, %x[N]\n"
+ "whilelt p1.s, XZR, x19\n"
+ "cbz x22, 30f\n"
+ "ld1w { z24.s }, p2/Z, [x22]\n"
+ "ld1w { z25.s }, p2/Z, [x22, #1, MUL VL]\n"
+ "ld1w { z26.s }, p2/Z, [x22, #2, MUL VL]\n"
+ "ld1w { z27.s }, p2/Z, [x22, #3, MUL VL]\n"
+ "ld1w { z28.s }, p2/Z, [x22, #4, MUL VL]\n"
+ "addvl x22, x22, #5\n"
+ "b 31f\n"
+ "30:" // Width 5: no bias
+ "mov z24.b, #0x0\n"
+ "mov z25.b, #0x0\n"
+ "mov z26.b, #0x0\n"
+ "mov z27.b, #0x0\n"
+ "mov z28.b, #0x0\n"
+ "31:" // Width 5: setup done
+ "cmp x21, #0x4\n"
+ "ble 33f\n"
+ "32:" // Width 5: Multiply loop: Main loop head
+ "ld1w { z1.s }, p2/Z, [%x[B_ptr]]\n"
+ "whilelt p0.s, XZR, x21\n"
+ "ld1w { z2.s }, p2/Z, [%x[B_ptr], #1, MUL VL]\n"
+ "sub x21, x21, #0x4\n"
+ "ld1rqw { z0.s }, p0/Z, [x20]\n"
+ "fmla z24.s, z1.s, z0.s[0]\n"
+ "ld1w { z3.s }, p2/Z, [%x[B_ptr], #2, MUL VL]\n"
+ "add x20, x20, #0x10\n"
+ "fmla z25.s, z2.s, z0.s[0]\n"
+ "ld1w { z4.s }, p2/Z, [%x[B_ptr], #3, MUL VL]\n"
+ "cmp x21, #0x4\n"
+ "fmla z26.s, z3.s, z0.s[0]\n"
+ "ld1w { z5.s }, p2/Z, [%x[B_ptr], #4, MUL VL]\n"
+ "addvl %x[B_ptr], %x[B_ptr], #8\n"
+ "fmla z27.s, z4.s, z0.s[0]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "fmla z28.s, z5.s, z0.s[0]\n"
+ "ld1w { z6.s }, p2/Z, [%x[B_ptr]]\n"
+ "ld1w { z7.s }, p2/Z, [%x[B_ptr], #1, MUL VL]\n"
+ "fmla z24.s, z6.s, z0.s[1]\n"
+ "ld1w { z8.s }, p2/Z, [%x[B_ptr], #2, MUL VL]\n"
+ "fmla z25.s, z7.s, z0.s[1]\n"
+ "ld1w { z9.s }, p2/Z, [%x[B_ptr], #3, MUL VL]\n"
+ "ld1w { z10.s }, p2/Z, [%x[B_ptr], #4, MUL VL]\n"
+ "fmla z26.s, z8.s, z0.s[1]\n"
+ "addvl %x[B_ptr], %x[B_ptr], #8\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "fmla z27.s, z9.s, z0.s[1]\n"
+ "fmla z28.s, z10.s, z0.s[1]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "ld1w { z11.s }, p2/Z, [%x[B_ptr]]\n"
+ "fmla z24.s, z11.s, z0.s[2]\n"
+ "ld1w { z12.s }, p2/Z, [%x[B_ptr], #1, MUL VL]\n"
+ "ld1w { z13.s }, p2/Z, [%x[B_ptr], #2, MUL VL]\n"
+ "fmla z25.s, z12.s, z0.s[2]\n"
+ "ld1w { z14.s }, p2/Z, [%x[B_ptr], #3, MUL VL]\n"
+ "fmla z26.s, z13.s, z0.s[2]\n"
+ "ld1w { z15.s }, p2/Z, [%x[B_ptr], #4, MUL VL]\n"
+ "addvl %x[B_ptr], %x[B_ptr], #8\n"
+ "fmla z27.s, z14.s, z0.s[2]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "fmla z28.s, z15.s, z0.s[2]\n"
+ "ld1w { z16.s }, p2/Z, [%x[B_ptr]]\n"
+ "fmla z24.s, z16.s, z0.s[3]\n"
+ "ld1w { z17.s }, p2/Z, [%x[B_ptr], #1, MUL VL]\n"
+ "ld1w { z18.s }, p2/Z, [%x[B_ptr], #2, MUL VL]\n"
+ "fmla z25.s, z17.s, z0.s[3]\n"
+ "ld1w { z19.s }, p2/Z, [%x[B_ptr], #3, MUL VL]\n"
+ "fmla z26.s, z18.s, z0.s[3]\n"
+ "ld1w { z20.s }, p2/Z, [%x[B_ptr], #4, MUL VL]\n"
+ "addvl %x[B_ptr], %x[B_ptr], #8\n"
+ "fmla z27.s, z19.s, z0.s[3]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "fmla z28.s, z20.s, z0.s[3]\n"
+ "prfm pldl1keep, [x20, #0x80]\n"
+ "bgt 32b\n"
+ "33:" // Width 5: Multiply loop: Single iteration only
+ "ld1w { z21.s }, p2/Z, [%x[B_ptr]]\n"
+ "whilelt p0.s, XZR, x21\n"
+ "ld1w { z22.s }, p2/Z, [%x[B_ptr], #1, MUL VL]\n"
+ "subs x21, x21, #0x1\n"
+ "ld1rqw { z0.s }, p0/Z, [x20]\n"
+ "fmla z24.s, z21.s, z0.s[0]\n"
+ "ld1w { z23.s }, p2/Z, [%x[B_ptr], #2, MUL VL]\n"
+ "add x20, x20, #0x10\n"
+ "fmla z25.s, z22.s, z0.s[0]\n"
+ "ld1w { z1.s }, p2/Z, [%x[B_ptr], #3, MUL VL]\n"
+ "ld1w { z2.s }, p2/Z, [%x[B_ptr], #4, MUL VL]\n"
+ "fmla z26.s, z23.s, z0.s[0]\n"
+ "addvl %x[B_ptr], %x[B_ptr], #8\n"
+ "fmla z27.s, z1.s, z0.s[0]\n"
+ "fmla z28.s, z2.s, z0.s[0]\n"
+ "ble 34f\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "subs x21, x21, #0x1\n"
+ "ld1w { z3.s }, p2/Z, [%x[B_ptr]]\n"
+ "fmla z24.s, z3.s, z0.s[1]\n"
+ "ld1w { z4.s }, p2/Z, [%x[B_ptr], #1, MUL VL]\n"
+ "ld1w { z5.s }, p2/Z, [%x[B_ptr], #2, MUL VL]\n"
+ "fmla z25.s, z4.s, z0.s[1]\n"
+ "ld1w { z6.s }, p2/Z, [%x[B_ptr], #3, MUL VL]\n"
+ "fmla z26.s, z5.s, z0.s[1]\n"
+ "ld1w { z7.s }, p2/Z, [%x[B_ptr], #4, MUL VL]\n"
+ "addvl %x[B_ptr], %x[B_ptr], #8\n"
+ "fmla z27.s, z6.s, z0.s[1]\n"
+ "fmla z28.s, z7.s, z0.s[1]\n"
+ "ble 34f\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "subs x21, x21, #0x1\n"
+ "ld1w { z8.s }, p2/Z, [%x[B_ptr]]\n"
+ "fmla z24.s, z8.s, z0.s[2]\n"
+ "ld1w { z9.s }, p2/Z, [%x[B_ptr], #1, MUL VL]\n"
+ "ld1w { z10.s }, p2/Z, [%x[B_ptr], #2, MUL VL]\n"
+ "fmla z25.s, z9.s, z0.s[2]\n"
+ "ld1w { z11.s }, p2/Z, [%x[B_ptr], #3, MUL VL]\n"
+ "fmla z26.s, z10.s, z0.s[2]\n"
+ "ld1w { z12.s }, p2/Z, [%x[B_ptr], #4, MUL VL]\n"
+ "addvl %x[B_ptr], %x[B_ptr], #8\n"
+ "fmla z27.s, z11.s, z0.s[2]\n"
+ "fmla z28.s, z12.s, z0.s[2]\n"
+ "ble 34f\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "ld1w { z13.s }, p2/Z, [%x[B_ptr]]\n"
+ "fmla z24.s, z13.s, z0.s[3]\n"
+ "ld1w { z14.s }, p2/Z, [%x[B_ptr], #1, MUL VL]\n"
+ "ld1w { z15.s }, p2/Z, [%x[B_ptr], #2, MUL VL]\n"
+ "fmla z25.s, z14.s, z0.s[3]\n"
+ "ld1w { z16.s }, p2/Z, [%x[B_ptr], #3, MUL VL]\n"
+ "fmla z26.s, z15.s, z0.s[3]\n"
+ "ld1w { z17.s }, p2/Z, [%x[B_ptr], #4, MUL VL]\n"
+ "addvl %x[B_ptr], %x[B_ptr], #8\n"
+ "fmla z27.s, z16.s, z0.s[3]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "fmla z28.s, z17.s, z0.s[3]\n"
+ "34:" // Width 5: Multiply loop: multiply skip
+ "prfm pldl1keep, [x20, #0x80]\n"
+ "prfm pstl1keep, [%x[output_ptr], #0x0]\n"
+ "tbz %x[flags], #1, 35f\n"
+ "add x19, %x[args_ptr], %[offset_min]\n"
+ "ld1rw { z17.s }, p2/Z, [x19]\n"
+ "add x19, %x[args_ptr], %[offset_max]\n"
+ "ld1rw { z16.s }, p2/Z, [x19]\n"
+ "fmin z24.s, p2/M, z24.s, z16.s\n"
+ "fmin z25.s, p2/M, z25.s, z16.s\n"
+ "fmin z26.s, p2/M, z26.s, z16.s\n"
+ "fmin z27.s, p2/M, z27.s, z16.s\n"
+ "fmin z28.s, p2/M, z28.s, z16.s\n"
+ "fmax z24.s, p2/M, z24.s, z17.s\n"
+ "fmax z25.s, p2/M, z25.s, z17.s\n"
+ "fmax z26.s, p2/M, z26.s, z17.s\n"
+ "fmax z27.s, p2/M, z27.s, z17.s\n"
+ "fmax z28.s, p2/M, z28.s, z17.s\n"
+ "35:" // Width 5: No activation
+ "st1w { z24.s }, p2, [%x[output_ptr]]\n"
+ "st1w { z25.s }, p2, [%x[output_ptr], #1, MUL VL]\n"
+ "st1w { z26.s }, p2, [%x[output_ptr], #2, MUL VL]\n"
+ "st1w { z27.s }, p2, [%x[output_ptr], #3, MUL VL]\n"
+ "st1w { z28.s }, p1, [%x[output_ptr], #4, MUL VL]\n"
+ "addvl %x[output_ptr], %x[output_ptr], #5\n"
+ "b 57f\n"
+ "36:" // Width 6
+ "mov x21, %x[K]\n"
+ "mov x20, %x[A_ptr]\n"
+ "mov x19, #0x5\n"
+ "msub x19, x24, x19, %x[N]\n"
+ "whilelt p1.s, XZR, x19\n"
+ "cbz x22, 37f\n"
+ "ld1w { z24.s }, p2/Z, [x22]\n"
+ "ld1w { z25.s }, p2/Z, [x22, #1, MUL VL]\n"
+ "ld1w { z26.s }, p2/Z, [x22, #2, MUL VL]\n"
+ "ld1w { z27.s }, p2/Z, [x22, #3, MUL VL]\n"
+ "ld1w { z28.s }, p2/Z, [x22, #4, MUL VL]\n"
+ "ld1w { z29.s }, p2/Z, [x22, #5, MUL VL]\n"
+ "addvl x22, x22, #6\n"
+ "b 38f\n"
+ "37:" // Width 6: no bias
+ "mov z24.b, #0x0\n"
+ "mov z25.b, #0x0\n"
+ "mov z26.b, #0x0\n"
+ "mov z27.b, #0x0\n"
+ "mov z28.b, #0x0\n"
+ "mov z29.b, #0x0\n"
+ "38:" // Width 6: setup done
+ "cmp x21, #0x4\n"
+ "ble 40f\n"
+ "39:" // Width 6: Multiply loop: Main loop head
+ "ld1w { z1.s }, p2/Z, [%x[B_ptr]]\n"
+ "whilelt p0.s, XZR, x21\n"
+ "ld1w { z2.s }, p2/Z, [%x[B_ptr], #1, MUL VL]\n"
+ "sub x21, x21, #0x4\n"
+ "ld1rqw { z0.s }, p0/Z, [x20]\n"
+ "fmla z24.s, z1.s, z0.s[0]\n"
+ "ld1w { z3.s }, p2/Z, [%x[B_ptr], #2, MUL VL]\n"
+ "add x20, x20, #0x10\n"
+ "fmla z25.s, z2.s, z0.s[0]\n"
+ "ld1w { z4.s }, p2/Z, [%x[B_ptr], #3, MUL VL]\n"
+ "cmp x21, #0x4\n"
+ "fmla z26.s, z3.s, z0.s[0]\n"
+ "ld1w { z5.s }, p2/Z, [%x[B_ptr], #4, MUL VL]\n"
+ "ld1w { z6.s }, p2/Z, [%x[B_ptr], #5, MUL VL]\n"
+ "fmla z27.s, z4.s, z0.s[0]\n"
+ "addvl %x[B_ptr], %x[B_ptr], #8\n"
+ "fmla z28.s, z5.s, z0.s[0]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "fmla z29.s, z6.s, z0.s[0]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "ld1w { z7.s }, p2/Z, [%x[B_ptr]]\n"
+ "fmla z24.s, z7.s, z0.s[1]\n"
+ "ld1w { z8.s }, p2/Z, [%x[B_ptr], #1, MUL VL]\n"
+ "ld1w { z9.s }, p2/Z, [%x[B_ptr], #2, MUL VL]\n"
+ "fmla z25.s, z8.s, z0.s[1]\n"
+ "ld1w { z10.s }, p2/Z, [%x[B_ptr], #3, MUL VL]\n"
+ "fmla z26.s, z9.s, z0.s[1]\n"
+ "ld1w { z11.s }, p2/Z, [%x[B_ptr], #4, MUL VL]\n"
+ "ld1w { z12.s }, p2/Z, [%x[B_ptr], #5, MUL VL]\n"
+ "fmla z27.s, z10.s, z0.s[1]\n"
+ "addvl %x[B_ptr], %x[B_ptr], #8\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "fmla z28.s, z11.s, z0.s[1]\n"
+ "fmla z29.s, z12.s, z0.s[1]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "ld1w { z13.s }, p2/Z, [%x[B_ptr]]\n"
+ "fmla z24.s, z13.s, z0.s[2]\n"
+ "ld1w { z14.s }, p2/Z, [%x[B_ptr], #1, MUL VL]\n"
+ "ld1w { z15.s }, p2/Z, [%x[B_ptr], #2, MUL VL]\n"
+ "fmla z25.s, z14.s, z0.s[2]\n"
+ "ld1w { z16.s }, p2/Z, [%x[B_ptr], #3, MUL VL]\n"
+ "fmla z26.s, z15.s, z0.s[2]\n"
+ "ld1w { z17.s }, p2/Z, [%x[B_ptr], #4, MUL VL]\n"
+ "ld1w { z18.s }, p2/Z, [%x[B_ptr], #5, MUL VL]\n"
+ "fmla z27.s, z16.s, z0.s[2]\n"
+ "addvl %x[B_ptr], %x[B_ptr], #8\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "fmla z28.s, z17.s, z0.s[2]\n"
+ "fmla z29.s, z18.s, z0.s[2]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "ld1w { z19.s }, p2/Z, [%x[B_ptr]]\n"
+ "fmla z24.s, z19.s, z0.s[3]\n"
+ "ld1w { z20.s }, p2/Z, [%x[B_ptr], #1, MUL VL]\n"
+ "ld1w { z21.s }, p2/Z, [%x[B_ptr], #2, MUL VL]\n"
+ "fmla z25.s, z20.s, z0.s[3]\n"
+ "ld1w { z22.s }, p2/Z, [%x[B_ptr], #3, MUL VL]\n"
+ "fmla z26.s, z21.s, z0.s[3]\n"
+ "ld1w { z23.s }, p2/Z, [%x[B_ptr], #4, MUL VL]\n"
+ "ld1w { z1.s }, p2/Z, [%x[B_ptr], #5, MUL VL]\n"
+ "fmla z27.s, z22.s, z0.s[3]\n"
+ "addvl %x[B_ptr], %x[B_ptr], #8\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "fmla z28.s, z23.s, z0.s[3]\n"
+ "fmla z29.s, z1.s, z0.s[3]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "prfm pldl1keep, [x20, #0x80]\n"
+ "bgt 39b\n"
+ "40:" // Width 6: Multiply loop: Single iteration only
+ "ld1w { z2.s }, p2/Z, [%x[B_ptr]]\n"
+ "whilelt p0.s, XZR, x21\n"
+ "ld1w { z3.s }, p2/Z, [%x[B_ptr], #1, MUL VL]\n"
+ "subs x21, x21, #0x1\n"
+ "ld1rqw { z0.s }, p0/Z, [x20]\n"
+ "fmla z24.s, z2.s, z0.s[0]\n"
+ "ld1w { z4.s }, p2/Z, [%x[B_ptr], #2, MUL VL]\n"
+ "add x20, x20, #0x10\n"
+ "fmla z25.s, z3.s, z0.s[0]\n"
+ "ld1w { z5.s }, p2/Z, [%x[B_ptr], #3, MUL VL]\n"
+ "ld1w { z6.s }, p2/Z, [%x[B_ptr], #4, MUL VL]\n"
+ "fmla z26.s, z4.s, z0.s[0]\n"
+ "ld1w { z7.s }, p2/Z, [%x[B_ptr], #5, MUL VL]\n"
+ "addvl %x[B_ptr], %x[B_ptr], #8\n"
+ "fmla z27.s, z5.s, z0.s[0]\n"
+ "fmla z28.s, z6.s, z0.s[0]\n"
+ "fmla z29.s, z7.s, z0.s[0]\n"
+ "ble 41f\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "subs x21, x21, #0x1\n"
+ "ld1w { z8.s }, p2/Z, [%x[B_ptr]]\n"
+ "fmla z24.s, z8.s, z0.s[1]\n"
+ "ld1w { z9.s }, p2/Z, [%x[B_ptr], #1, MUL VL]\n"
+ "ld1w { z10.s }, p2/Z, [%x[B_ptr], #2, MUL VL]\n"
+ "fmla z25.s, z9.s, z0.s[1]\n"
+ "ld1w { z11.s }, p2/Z, [%x[B_ptr], #3, MUL VL]\n"
+ "fmla z26.s, z10.s, z0.s[1]\n"
+ "ld1w { z12.s }, p2/Z, [%x[B_ptr], #4, MUL VL]\n"
+ "ld1w { z13.s }, p2/Z, [%x[B_ptr], #5, MUL VL]\n"
+ "fmla z27.s, z11.s, z0.s[1]\n"
+ "addvl %x[B_ptr], %x[B_ptr], #8\n"
+ "fmla z28.s, z12.s, z0.s[1]\n"
+ "fmla z29.s, z13.s, z0.s[1]\n"
+ "ble 41f\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "subs x21, x21, #0x1\n"
+ "ld1w { z14.s }, p2/Z, [%x[B_ptr]]\n"
+ "fmla z24.s, z14.s, z0.s[2]\n"
+ "ld1w { z15.s }, p2/Z, [%x[B_ptr], #1, MUL VL]\n"
+ "ld1w { z16.s }, p2/Z, [%x[B_ptr], #2, MUL VL]\n"
+ "fmla z25.s, z15.s, z0.s[2]\n"
+ "ld1w { z17.s }, p2/Z, [%x[B_ptr], #3, MUL VL]\n"
+ "fmla z26.s, z16.s, z0.s[2]\n"
+ "ld1w { z18.s }, p2/Z, [%x[B_ptr], #4, MUL VL]\n"
+ "ld1w { z19.s }, p2/Z, [%x[B_ptr], #5, MUL VL]\n"
+ "fmla z27.s, z17.s, z0.s[2]\n"
+ "addvl %x[B_ptr], %x[B_ptr], #8\n"
+ "fmla z28.s, z18.s, z0.s[2]\n"
+ "fmla z29.s, z19.s, z0.s[2]\n"
+ "ble 41f\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "ld1w { z20.s }, p2/Z, [%x[B_ptr]]\n"
+ "fmla z24.s, z20.s, z0.s[3]\n"
+ "ld1w { z21.s }, p2/Z, [%x[B_ptr], #1, MUL VL]\n"
+ "ld1w { z22.s }, p2/Z, [%x[B_ptr], #2, MUL VL]\n"
+ "fmla z25.s, z21.s, z0.s[3]\n"
+ "ld1w { z23.s }, p2/Z, [%x[B_ptr], #3, MUL VL]\n"
+ "fmla z26.s, z22.s, z0.s[3]\n"
+ "ld1w { z1.s }, p2/Z, [%x[B_ptr], #4, MUL VL]\n"
+ "ld1w { z2.s }, p2/Z, [%x[B_ptr], #5, MUL VL]\n"
+ "fmla z27.s, z23.s, z0.s[3]\n"
+ "addvl %x[B_ptr], %x[B_ptr], #8\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "fmla z28.s, z1.s, z0.s[3]\n"
+ "fmla z29.s, z2.s, z0.s[3]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "41:" // Width 6: Multiply loop: multiply skip
+ "prfm pldl1keep, [x20, #0x80]\n"
+ "prfm pstl1keep, [%x[output_ptr], #0x0]\n"
+ "tbz %x[flags], #1, 42f\n"
+ "add x19, %x[args_ptr], %[offset_min]\n"
+ "ld1rw { z17.s }, p2/Z, [x19]\n"
+ "add x19, %x[args_ptr], %[offset_max]\n"
+ "ld1rw { z16.s }, p2/Z, [x19]\n"
+ "fmin z24.s, p2/M, z24.s, z16.s\n"
+ "fmin z25.s, p2/M, z25.s, z16.s\n"
+ "fmin z26.s, p2/M, z26.s, z16.s\n"
+ "fmin z27.s, p2/M, z27.s, z16.s\n"
+ "fmin z28.s, p2/M, z28.s, z16.s\n"
+ "fmax z24.s, p2/M, z24.s, z17.s\n"
+ "fmax z25.s, p2/M, z25.s, z17.s\n"
+ "fmax z26.s, p2/M, z26.s, z17.s\n"
+ "fmax z27.s, p2/M, z27.s, z17.s\n"
+ "fmax z28.s, p2/M, z28.s, z17.s\n"
+ "fmin z29.s, p2/M, z29.s, z16.s\n"
+ "fmax z29.s, p2/M, z29.s, z17.s\n"
+ "42:" // Width 6: No activation
+ "st1w { z24.s }, p2, [%x[output_ptr]]\n"
+ "st1w { z25.s }, p2, [%x[output_ptr], #1, MUL VL]\n"
+ "st1w { z26.s }, p2, [%x[output_ptr], #2, MUL VL]\n"
+ "st1w { z27.s }, p2, [%x[output_ptr], #3, MUL VL]\n"
+ "st1w { z28.s }, p2, [%x[output_ptr], #4, MUL VL]\n"
+ "st1w { z29.s }, p1, [%x[output_ptr], #5, MUL VL]\n"
+ "addvl %x[output_ptr], %x[output_ptr], #6\n"
+ "b 57f\n"
+ "43:" // Width 7
+ "mov x21, %x[K]\n"
+ "mov x20, %x[A_ptr]\n"
+ "mov x19, #0x6\n"
+ "msub x19, x24, x19, %x[N]\n"
+ "whilelt p1.s, XZR, x19\n"
+ "cbz x22, 44f\n"
+ "ld1w { z24.s }, p2/Z, [x22]\n"
+ "ld1w { z25.s }, p2/Z, [x22, #1, MUL VL]\n"
+ "ld1w { z26.s }, p2/Z, [x22, #2, MUL VL]\n"
+ "ld1w { z27.s }, p2/Z, [x22, #3, MUL VL]\n"
+ "ld1w { z28.s }, p2/Z, [x22, #4, MUL VL]\n"
+ "ld1w { z29.s }, p2/Z, [x22, #5, MUL VL]\n"
+ "ld1w { z30.s }, p2/Z, [x22, #6, MUL VL]\n"
+ "addvl x22, x22, #7\n"
+ "b 45f\n"
+ "44:" // Width 7: no bias
+ "mov z24.b, #0x0\n"
+ "mov z25.b, #0x0\n"
+ "mov z26.b, #0x0\n"
+ "mov z27.b, #0x0\n"
+ "mov z28.b, #0x0\n"
+ "mov z29.b, #0x0\n"
+ "mov z30.b, #0x0\n"
+ "45:" // Width 7: setup done
+ "cmp x21, #0x4\n"
+ "ble 47f\n"
+ "46:" // Width 7: Multiply loop: Main loop head
+ "ld1w { z1.s }, p2/Z, [%x[B_ptr]]\n"
+ "whilelt p0.s, XZR, x21\n"
+ "ld1w { z2.s }, p2/Z, [%x[B_ptr], #1, MUL VL]\n"
+ "sub x21, x21, #0x4\n"
+ "ld1rqw { z0.s }, p0/Z, [x20]\n"
+ "fmla z24.s, z1.s, z0.s[0]\n"
+ "ld1w { z3.s }, p2/Z, [%x[B_ptr], #2, MUL VL]\n"
+ "add x20, x20, #0x10\n"
+ "fmla z25.s, z2.s, z0.s[0]\n"
+ "ld1w { z4.s }, p2/Z, [%x[B_ptr], #3, MUL VL]\n"
+ "cmp x21, #0x4\n"
+ "fmla z26.s, z3.s, z0.s[0]\n"
+ "ld1w { z5.s }, p2/Z, [%x[B_ptr], #4, MUL VL]\n"
+ "ld1w { z6.s }, p2/Z, [%x[B_ptr], #5, MUL VL]\n"
+ "fmla z27.s, z4.s, z0.s[0]\n"
+ "ld1w { z7.s }, p2/Z, [%x[B_ptr], #6, MUL VL]\n"
+ "fmla z28.s, z5.s, z0.s[0]\n"
+ "addvl %x[B_ptr], %x[B_ptr], #8\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "fmla z29.s, z6.s, z0.s[0]\n"
+ "fmla z30.s, z7.s, z0.s[0]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "ld1w { z8.s }, p2/Z, [%x[B_ptr]]\n"
+ "fmla z24.s, z8.s, z0.s[1]\n"
+ "ld1w { z9.s }, p2/Z, [%x[B_ptr], #1, MUL VL]\n"
+ "ld1w { z10.s }, p2/Z, [%x[B_ptr], #2, MUL VL]\n"
+ "fmla z25.s, z9.s, z0.s[1]\n"
+ "ld1w { z11.s }, p2/Z, [%x[B_ptr], #3, MUL VL]\n"
+ "fmla z26.s, z10.s, z0.s[1]\n"
+ "ld1w { z12.s }, p2/Z, [%x[B_ptr], #4, MUL VL]\n"
+ "ld1w { z13.s }, p2/Z, [%x[B_ptr], #5, MUL VL]\n"
+ "fmla z27.s, z11.s, z0.s[1]\n"
+ "ld1w { z14.s }, p2/Z, [%x[B_ptr], #6, MUL VL]\n"
+ "addvl %x[B_ptr], %x[B_ptr], #8\n"
+ "fmla z28.s, z12.s, z0.s[1]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "fmla z29.s, z13.s, z0.s[1]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "ld1w { z15.s }, p2/Z, [%x[B_ptr]]\n"
+ "fmla z30.s, z14.s, z0.s[1]\n"
+ "ld1w { z16.s }, p2/Z, [%x[B_ptr], #1, MUL VL]\n"
+ "fmla z24.s, z15.s, z0.s[2]\n"
+ "ld1w { z17.s }, p2/Z, [%x[B_ptr], #2, MUL VL]\n"
+ "ld1w { z18.s }, p2/Z, [%x[B_ptr], #3, MUL VL]\n"
+ "fmla z25.s, z16.s, z0.s[2]\n"
+ "ld1w { z19.s }, p2/Z, [%x[B_ptr], #4, MUL VL]\n"
+ "fmla z26.s, z17.s, z0.s[2]\n"
+ "ld1w { z20.s }, p2/Z, [%x[B_ptr], #5, MUL VL]\n"
+ "fmla z27.s, z18.s, z0.s[2]\n"
+ "ld1w { z21.s }, p2/Z, [%x[B_ptr], #6, MUL VL]\n"
+ "addvl %x[B_ptr], %x[B_ptr], #8\n"
+ "fmla z28.s, z19.s, z0.s[2]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "fmla z29.s, z20.s, z0.s[2]\n"
+ "ld1w { z22.s }, p2/Z, [%x[B_ptr]]\n"
+ "fmla z30.s, z21.s, z0.s[2]\n"
+ "ld1w { z23.s }, p2/Z, [%x[B_ptr], #1, MUL VL]\n"
+ "fmla z24.s, z22.s, z0.s[3]\n"
+ "ld1w { z1.s }, p2/Z, [%x[B_ptr], #2, MUL VL]\n"
+ "ld1w { z2.s }, p2/Z, [%x[B_ptr], #3, MUL VL]\n"
+ "fmla z25.s, z23.s, z0.s[3]\n"
+ "ld1w { z3.s }, p2/Z, [%x[B_ptr], #4, MUL VL]\n"
+ "fmla z26.s, z1.s, z0.s[3]\n"
+ "ld1w { z4.s }, p2/Z, [%x[B_ptr], #5, MUL VL]\n"
+ "fmla z27.s, z2.s, z0.s[3]\n"
+ "ld1w { z5.s }, p2/Z, [%x[B_ptr], #6, MUL VL]\n"
+ "addvl %x[B_ptr], %x[B_ptr], #8\n"
+ "fmla z28.s, z3.s, z0.s[3]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "fmla z29.s, z4.s, z0.s[3]\n"
+ "prfm pldl1keep, [x20, #0x80]\n"
+ "fmla z30.s, z5.s, z0.s[3]\n"
+ "bgt 46b\n"
+ "47:" // Width 7: Multiply loop: Single iteration only
+ "ld1w { z6.s }, p2/Z, [%x[B_ptr]]\n"
+ "whilelt p0.s, XZR, x21\n"
+ "ld1w { z7.s }, p2/Z, [%x[B_ptr], #1, MUL VL]\n"
+ "subs x21, x21, #0x1\n"
+ "ld1rqw { z0.s }, p0/Z, [x20]\n"
+ "fmla z24.s, z6.s, z0.s[0]\n"
+ "ld1w { z8.s }, p2/Z, [%x[B_ptr], #2, MUL VL]\n"
+ "add x20, x20, #0x10\n"
+ "fmla z25.s, z7.s, z0.s[0]\n"
+ "ld1w { z9.s }, p2/Z, [%x[B_ptr], #3, MUL VL]\n"
+ "ld1w { z10.s }, p2/Z, [%x[B_ptr], #4, MUL VL]\n"
+ "fmla z26.s, z8.s, z0.s[0]\n"
+ "ld1w { z11.s }, p2/Z, [%x[B_ptr], #5, MUL VL]\n"
+ "ld1w { z12.s }, p2/Z, [%x[B_ptr], #6, MUL VL]\n"
+ "fmla z27.s, z9.s, z0.s[0]\n"
+ "addvl %x[B_ptr], %x[B_ptr], #8\n"
+ "fmla z28.s, z10.s, z0.s[0]\n"
+ "fmla z29.s, z11.s, z0.s[0]\n"
+ "fmla z30.s, z12.s, z0.s[0]\n"
+ "ble 48f\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "subs x21, x21, #0x1\n"
+ "ld1w { z13.s }, p2/Z, [%x[B_ptr]]\n"
+ "fmla z24.s, z13.s, z0.s[1]\n"
+ "ld1w { z14.s }, p2/Z, [%x[B_ptr], #1, MUL VL]\n"
+ "ld1w { z15.s }, p2/Z, [%x[B_ptr], #2, MUL VL]\n"
+ "fmla z25.s, z14.s, z0.s[1]\n"
+ "ld1w { z16.s }, p2/Z, [%x[B_ptr], #3, MUL VL]\n"
+ "fmla z26.s, z15.s, z0.s[1]\n"
+ "ld1w { z17.s }, p2/Z, [%x[B_ptr], #4, MUL VL]\n"
+ "ld1w { z18.s }, p2/Z, [%x[B_ptr], #5, MUL VL]\n"
+ "fmla z27.s, z16.s, z0.s[1]\n"
+ "ld1w { z19.s }, p2/Z, [%x[B_ptr], #6, MUL VL]\n"
+ "addvl %x[B_ptr], %x[B_ptr], #8\n"
+ "fmla z28.s, z17.s, z0.s[1]\n"
+ "fmla z29.s, z18.s, z0.s[1]\n"
+ "fmla z30.s, z19.s, z0.s[1]\n"
+ "ble 48f\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "subs x21, x21, #0x1\n"
+ "ld1w { z20.s }, p2/Z, [%x[B_ptr]]\n"
+ "fmla z24.s, z20.s, z0.s[2]\n"
+ "ld1w { z21.s }, p2/Z, [%x[B_ptr], #1, MUL VL]\n"
+ "ld1w { z22.s }, p2/Z, [%x[B_ptr], #2, MUL VL]\n"
+ "fmla z25.s, z21.s, z0.s[2]\n"
+ "ld1w { z23.s }, p2/Z, [%x[B_ptr], #3, MUL VL]\n"
+ "fmla z26.s, z22.s, z0.s[2]\n"
+ "ld1w { z1.s }, p2/Z, [%x[B_ptr], #4, MUL VL]\n"
+ "ld1w { z2.s }, p2/Z, [%x[B_ptr], #5, MUL VL]\n"
+ "fmla z27.s, z23.s, z0.s[2]\n"
+ "ld1w { z3.s }, p2/Z, [%x[B_ptr], #6, MUL VL]\n"
+ "addvl %x[B_ptr], %x[B_ptr], #8\n"
+ "fmla z28.s, z1.s, z0.s[2]\n"
+ "fmla z29.s, z2.s, z0.s[2]\n"
+ "fmla z30.s, z3.s, z0.s[2]\n"
+ "ble 48f\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "ld1w { z4.s }, p2/Z, [%x[B_ptr]]\n"
+ "fmla z24.s, z4.s, z0.s[3]\n"
+ "ld1w { z5.s }, p2/Z, [%x[B_ptr], #1, MUL VL]\n"
+ "ld1w { z6.s }, p2/Z, [%x[B_ptr], #2, MUL VL]\n"
+ "fmla z25.s, z5.s, z0.s[3]\n"
+ "ld1w { z7.s }, p2/Z, [%x[B_ptr], #3, MUL VL]\n"
+ "fmla z26.s, z6.s, z0.s[3]\n"
+ "ld1w { z8.s }, p2/Z, [%x[B_ptr], #4, MUL VL]\n"
+ "ld1w { z9.s }, p2/Z, [%x[B_ptr], #5, MUL VL]\n"
+ "fmla z27.s, z7.s, z0.s[3]\n"
+ "ld1w { z10.s }, p2/Z, [%x[B_ptr], #6, MUL VL]\n"
+ "addvl %x[B_ptr], %x[B_ptr], #8\n"
+ "fmla z28.s, z8.s, z0.s[3]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "fmla z29.s, z9.s, z0.s[3]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "fmla z30.s, z10.s, z0.s[3]\n"
+ "48:" // Width 7: Multiply loop: multiply skip
+ "prfm pldl1keep, [x20, #0x80]\n"
+ "prfm pstl1keep, [%x[output_ptr], #0x0]\n"
+ "tbz %x[flags], #1, 49f\n"
+ "add x19, %x[args_ptr], %[offset_min]\n"
+ "ld1rw { z17.s }, p2/Z, [x19]\n"
+ "add x19, %x[args_ptr], %[offset_max]\n"
+ "ld1rw { z16.s }, p2/Z, [x19]\n"
+ "fmin z24.s, p2/M, z24.s, z16.s\n"
+ "fmin z25.s, p2/M, z25.s, z16.s\n"
+ "fmin z26.s, p2/M, z26.s, z16.s\n"
+ "fmin z27.s, p2/M, z27.s, z16.s\n"
+ "fmin z28.s, p2/M, z28.s, z16.s\n"
+ "fmax z24.s, p2/M, z24.s, z17.s\n"
+ "fmax z25.s, p2/M, z25.s, z17.s\n"
+ "fmax z26.s, p2/M, z26.s, z17.s\n"
+ "fmax z27.s, p2/M, z27.s, z17.s\n"
+ "fmax z28.s, p2/M, z28.s, z17.s\n"
+ "fmin z29.s, p2/M, z29.s, z16.s\n"
+ "fmin z30.s, p2/M, z30.s, z16.s\n"
+ "fmax z29.s, p2/M, z29.s, z17.s\n"
+ "fmax z30.s, p2/M, z30.s, z17.s\n"
+ "49:" // Width 7: No activation
+ "st1w { z24.s }, p2, [%x[output_ptr]]\n"
+ "st1w { z25.s }, p2, [%x[output_ptr], #1, MUL VL]\n"
+ "st1w { z26.s }, p2, [%x[output_ptr], #2, MUL VL]\n"
+ "st1w { z27.s }, p2, [%x[output_ptr], #3, MUL VL]\n"
+ "st1w { z28.s }, p2, [%x[output_ptr], #4, MUL VL]\n"
+ "st1w { z29.s }, p2, [%x[output_ptr], #5, MUL VL]\n"
+ "st1w { z30.s }, p1, [%x[output_ptr], #6, MUL VL]\n"
+ "addvl %x[output_ptr], %x[output_ptr], #7\n"
+ "b 57f\n"
+ "50:" // Width 8
+ "mov x21, %x[K]\n"
+ "mov x20, %x[A_ptr]\n"
+ "mov x19, #0x7\n"
+ "msub x19, x24, x19, %x[N]\n"
+ "whilelt p1.s, XZR, x19\n"
+ "cbz x22, 51f\n"
+ "ld1w { z24.s }, p2/Z, [x22]\n"
+ "ld1w { z25.s }, p2/Z, [x22, #1, MUL VL]\n"
+ "ld1w { z26.s }, p2/Z, [x22, #2, MUL VL]\n"
+ "ld1w { z27.s }, p2/Z, [x22, #3, MUL VL]\n"
+ "ld1w { z28.s }, p2/Z, [x22, #4, MUL VL]\n"
+ "ld1w { z29.s }, p2/Z, [x22, #5, MUL VL]\n"
+ "ld1w { z30.s }, p2/Z, [x22, #6, MUL VL]\n"
+ "ld1w { z31.s }, p2/Z, [x22, #7, MUL VL]\n"
+ "addvl x22, x22, #8\n"
+ "b 52f\n"
+ "51:" // Width 8: no bias
+ "mov z24.b, #0x0\n"
+ "mov z25.b, #0x0\n"
+ "mov z26.b, #0x0\n"
+ "mov z27.b, #0x0\n"
+ "mov z28.b, #0x0\n"
+ "mov z29.b, #0x0\n"
+ "mov z30.b, #0x0\n"
+ "mov z31.b, #0x0\n"
+ "52:" // Width 8: setup done
+ "cmp x21, #0x4\n"
+ "ble 54f\n"
+ "53:" // Width 8: Multiply loop: Main loop head
+ "ld1w { z1.s }, p2/Z, [%x[B_ptr]]\n"
+ "whilelt p0.s, XZR, x21\n"
+ "ld1w { z2.s }, p2/Z, [%x[B_ptr], #1, MUL VL]\n"
+ "sub x21, x21, #0x4\n"
+ "ld1rqw { z0.s }, p0/Z, [x20]\n"
+ "fmla z24.s, z1.s, z0.s[0]\n"
+ "ld1w { z3.s }, p2/Z, [%x[B_ptr], #2, MUL VL]\n"
+ "add x20, x20, #0x10\n"
+ "fmla z25.s, z2.s, z0.s[0]\n"
+ "ld1w { z4.s }, p2/Z, [%x[B_ptr], #3, MUL VL]\n"
+ "cmp x21, #0x4\n"
+ "fmla z26.s, z3.s, z0.s[0]\n"
+ "ld1w { z5.s }, p2/Z, [%x[B_ptr], #4, MUL VL]\n"
+ "ld1w { z6.s }, p2/Z, [%x[B_ptr], #5, MUL VL]\n"
+ "fmla z27.s, z4.s, z0.s[0]\n"
+ "ld1w { z7.s }, p2/Z, [%x[B_ptr], #6, MUL VL]\n"
+ "fmla z28.s, z5.s, z0.s[0]\n"
+ "ld1w { z8.s }, p2/Z, [%x[B_ptr], #7, MUL VL]\n"
+ "addvl %x[B_ptr], %x[B_ptr], #8\n"
+ "fmla z29.s, z6.s, z0.s[0]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "fmla z30.s, z7.s, z0.s[0]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "ld1w { z9.s }, p2/Z, [%x[B_ptr]]\n"
+ "fmla z31.s, z8.s, z0.s[0]\n"
+ "ld1w { z10.s }, p2/Z, [%x[B_ptr], #1, MUL VL]\n"
+ "fmla z24.s, z9.s, z0.s[1]\n"
+ "ld1w { z11.s }, p2/Z, [%x[B_ptr], #2, MUL VL]\n"
+ "ld1w { z12.s }, p2/Z, [%x[B_ptr], #3, MUL VL]\n"
+ "fmla z25.s, z10.s, z0.s[1]\n"
+ "ld1w { z13.s }, p2/Z, [%x[B_ptr], #4, MUL VL]\n"
+ "fmla z26.s, z11.s, z0.s[1]\n"
+ "ld1w { z14.s }, p2/Z, [%x[B_ptr], #5, MUL VL]\n"
+ "fmla z27.s, z12.s, z0.s[1]\n"
+ "ld1w { z15.s }, p2/Z, [%x[B_ptr], #6, MUL VL]\n"
+ "fmla z28.s, z13.s, z0.s[1]\n"
+ "ld1w { z16.s }, p2/Z, [%x[B_ptr], #7, MUL VL]\n"
+ "addvl %x[B_ptr], %x[B_ptr], #8\n"
+ "fmla z29.s, z14.s, z0.s[1]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "fmla z30.s, z15.s, z0.s[1]\n"
+ "ld1w { z17.s }, p2/Z, [%x[B_ptr]]\n"
+ "fmla z31.s, z16.s, z0.s[1]\n"
+ "ld1w { z18.s }, p2/Z, [%x[B_ptr], #1, MUL VL]\n"
+ "fmla z24.s, z17.s, z0.s[2]\n"
+ "ld1w { z19.s }, p2/Z, [%x[B_ptr], #2, MUL VL]\n"
+ "ld1w { z20.s }, p2/Z, [%x[B_ptr], #3, MUL VL]\n"
+ "fmla z25.s, z18.s, z0.s[2]\n"
+ "ld1w { z21.s }, p2/Z, [%x[B_ptr], #4, MUL VL]\n"
+ "fmla z26.s, z19.s, z0.s[2]\n"
+ "ld1w { z22.s }, p2/Z, [%x[B_ptr], #5, MUL VL]\n"
+ "fmla z27.s, z20.s, z0.s[2]\n"
+ "ld1w { z23.s }, p2/Z, [%x[B_ptr], #6, MUL VL]\n"
+ "fmla z28.s, z21.s, z0.s[2]\n"
+ "ld1w { z1.s }, p2/Z, [%x[B_ptr], #7, MUL VL]\n"
+ "addvl %x[B_ptr], %x[B_ptr], #8\n"
+ "fmla z29.s, z22.s, z0.s[2]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "fmla z30.s, z23.s, z0.s[2]\n"
+ "ld1w { z2.s }, p2/Z, [%x[B_ptr]]\n"
+ "fmla z31.s, z1.s, z0.s[2]\n"
+ "ld1w { z3.s }, p2/Z, [%x[B_ptr], #1, MUL VL]\n"
+ "fmla z24.s, z2.s, z0.s[3]\n"
+ "ld1w { z4.s }, p2/Z, [%x[B_ptr], #2, MUL VL]\n"
+ "ld1w { z5.s }, p2/Z, [%x[B_ptr], #3, MUL VL]\n"
+ "fmla z25.s, z3.s, z0.s[3]\n"
+ "ld1w { z6.s }, p2/Z, [%x[B_ptr], #4, MUL VL]\n"
+ "fmla z26.s, z4.s, z0.s[3]\n"
+ "ld1w { z7.s }, p2/Z, [%x[B_ptr], #5, MUL VL]\n"
+ "fmla z27.s, z5.s, z0.s[3]\n"
+ "ld1w { z8.s }, p2/Z, [%x[B_ptr], #6, MUL VL]\n"
+ "fmla z28.s, z6.s, z0.s[3]\n"
+ "ld1w { z9.s }, p2/Z, [%x[B_ptr], #7, MUL VL]\n"
+ "addvl %x[B_ptr], %x[B_ptr], #8\n"
+ "fmla z29.s, z7.s, z0.s[3]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "fmla z30.s, z8.s, z0.s[3]\n"
+ "prfm pldl1keep, [x20, #0x80]\n"
+ "fmla z31.s, z9.s, z0.s[3]\n"
+ "bgt 53b\n"
+ "54:" // Width 8: Multiply loop: Single iteration only
+ "ld1w { z10.s }, p2/Z, [%x[B_ptr]]\n"
+ "whilelt p0.s, XZR, x21\n"
+ "ld1w { z11.s }, p2/Z, [%x[B_ptr], #1, MUL VL]\n"
+ "subs x21, x21, #0x1\n"
+ "ld1rqw { z0.s }, p0/Z, [x20]\n"
+ "fmla z24.s, z10.s, z0.s[0]\n"
+ "ld1w { z12.s }, p2/Z, [%x[B_ptr], #2, MUL VL]\n"
+ "add x20, x20, #0x10\n"
+ "fmla z25.s, z11.s, z0.s[0]\n"
+ "ld1w { z13.s }, p2/Z, [%x[B_ptr], #3, MUL VL]\n"
+ "ld1w { z14.s }, p2/Z, [%x[B_ptr], #4, MUL VL]\n"
+ "fmla z26.s, z12.s, z0.s[0]\n"
+ "ld1w { z15.s }, p2/Z, [%x[B_ptr], #5, MUL VL]\n"
+ "ld1w { z16.s }, p2/Z, [%x[B_ptr], #6, MUL VL]\n"
+ "fmla z27.s, z13.s, z0.s[0]\n"
+ "fmla z28.s, z14.s, z0.s[0]\n"
+ "ld1w { z17.s }, p2/Z, [%x[B_ptr], #7, MUL VL]\n"
+ "addvl %x[B_ptr], %x[B_ptr], #8\n"
+ "fmla z29.s, z15.s, z0.s[0]\n"
+ "fmla z30.s, z16.s, z0.s[0]\n"
+ "fmla z31.s, z17.s, z0.s[0]\n"
+ "ble 55f\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "subs x21, x21, #0x1\n"
+ "ld1w { z18.s }, p2/Z, [%x[B_ptr]]\n"
+ "fmla z24.s, z18.s, z0.s[1]\n"
+ "ld1w { z19.s }, p2/Z, [%x[B_ptr], #1, MUL VL]\n"
+ "ld1w { z20.s }, p2/Z, [%x[B_ptr], #2, MUL VL]\n"
+ "fmla z25.s, z19.s, z0.s[1]\n"
+ "ld1w { z21.s }, p2/Z, [%x[B_ptr], #3, MUL VL]\n"
+ "fmla z26.s, z20.s, z0.s[1]\n"
+ "ld1w { z22.s }, p2/Z, [%x[B_ptr], #4, MUL VL]\n"
+ "ld1w { z23.s }, p2/Z, [%x[B_ptr], #5, MUL VL]\n"
+ "fmla z27.s, z21.s, z0.s[1]\n"
+ "ld1w { z1.s }, p2/Z, [%x[B_ptr], #6, MUL VL]\n"
+ "ld1w { z2.s }, p2/Z, [%x[B_ptr], #7, MUL VL]\n"
+ "fmla z28.s, z22.s, z0.s[1]\n"
+ "addvl %x[B_ptr], %x[B_ptr], #8\n"
+ "fmla z29.s, z23.s, z0.s[1]\n"
+ "fmla z30.s, z1.s, z0.s[1]\n"
+ "fmla z31.s, z2.s, z0.s[1]\n"
+ "ble 55f\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "subs x21, x21, #0x1\n"
+ "ld1w { z3.s }, p2/Z, [%x[B_ptr]]\n"
+ "fmla z24.s, z3.s, z0.s[2]\n"
+ "ld1w { z4.s }, p2/Z, [%x[B_ptr], #1, MUL VL]\n"
+ "ld1w { z5.s }, p2/Z, [%x[B_ptr], #2, MUL VL]\n"
+ "fmla z25.s, z4.s, z0.s[2]\n"
+ "ld1w { z6.s }, p2/Z, [%x[B_ptr], #3, MUL VL]\n"
+ "fmla z26.s, z5.s, z0.s[2]\n"
+ "ld1w { z7.s }, p2/Z, [%x[B_ptr], #4, MUL VL]\n"
+ "ld1w { z8.s }, p2/Z, [%x[B_ptr], #5, MUL VL]\n"
+ "fmla z27.s, z6.s, z0.s[2]\n"
+ "ld1w { z9.s }, p2/Z, [%x[B_ptr], #6, MUL VL]\n"
+ "ld1w { z10.s }, p2/Z, [%x[B_ptr], #7, MUL VL]\n"
+ "fmla z28.s, z7.s, z0.s[2]\n"
+ "addvl %x[B_ptr], %x[B_ptr], #8\n"
+ "fmla z29.s, z8.s, z0.s[2]\n"
+ "fmla z30.s, z9.s, z0.s[2]\n"
+ "fmla z31.s, z10.s, z0.s[2]\n"
+ "ble 55f\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "ld1w { z11.s }, p2/Z, [%x[B_ptr]]\n"
+ "fmla z24.s, z11.s, z0.s[3]\n"
+ "ld1w { z12.s }, p2/Z, [%x[B_ptr], #1, MUL VL]\n"
+ "ld1w { z13.s }, p2/Z, [%x[B_ptr], #2, MUL VL]\n"
+ "fmla z25.s, z12.s, z0.s[3]\n"
+ "ld1w { z14.s }, p2/Z, [%x[B_ptr], #3, MUL VL]\n"
+ "fmla z26.s, z13.s, z0.s[3]\n"
+ "ld1w { z15.s }, p2/Z, [%x[B_ptr], #4, MUL VL]\n"
+ "ld1w { z16.s }, p2/Z, [%x[B_ptr], #5, MUL VL]\n"
+ "fmla z27.s, z14.s, z0.s[3]\n"
+ "ld1w { z17.s }, p2/Z, [%x[B_ptr], #6, MUL VL]\n"
+ "ld1w { z18.s }, p2/Z, [%x[B_ptr], #7, MUL VL]\n"
+ "fmla z28.s, z15.s, z0.s[3]\n"
+ "addvl %x[B_ptr], %x[B_ptr], #8\n"
+ "fmla z29.s, z16.s, z0.s[3]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x400]\n"
+ "prfm pldl1keep, [%x[B_ptr], #0x440]\n"
+ "fmla z30.s, z17.s, z0.s[3]\n"
+ "fmla z31.s, z18.s, z0.s[3]\n"
+ "55:" // Width 8: Multiply loop: multiply skip
+ "prfm pldl1keep, [x20, #0x80]\n"
+ "prfm pstl1keep, [%x[output_ptr], #0x0]\n"
+ "tbz %x[flags], #1, 56f\n"
+ "add x19, %x[args_ptr], %[offset_min]\n"
+ "ld1rw { z17.s }, p2/Z, [x19]\n"
+ "add x19, %x[args_ptr], %[offset_max]\n"
+ "ld1rw { z16.s }, p2/Z, [x19]\n"
+ "fmin z24.s, p2/M, z24.s, z16.s\n"
+ "fmin z25.s, p2/M, z25.s, z16.s\n"
+ "fmin z26.s, p2/M, z26.s, z16.s\n"
+ "fmin z27.s, p2/M, z27.s, z16.s\n"
+ "fmin z28.s, p2/M, z28.s, z16.s\n"
+ "fmax z24.s, p2/M, z24.s, z17.s\n"
+ "fmax z25.s, p2/M, z25.s, z17.s\n"
+ "fmax z26.s, p2/M, z26.s, z17.s\n"
+ "fmax z27.s, p2/M, z27.s, z17.s\n"
+ "fmax z28.s, p2/M, z28.s, z17.s\n"
+ "fmin z29.s, p2/M, z29.s, z16.s\n"
+ "fmin z30.s, p2/M, z30.s, z16.s\n"
+ "fmin z31.s, p2/M, z31.s, z16.s\n"
+ "fmax z29.s, p2/M, z29.s, z17.s\n"
+ "fmax z30.s, p2/M, z30.s, z17.s\n"
+ "fmax z31.s, p2/M, z31.s, z17.s\n"
+ "56:" // Width 8: No activation
+ "st1w { z24.s }, p2, [%x[output_ptr]]\n"
+ "subs x23, x23, #0x8\n"
+ "st1w { z25.s }, p2, [%x[output_ptr], #1, MUL VL]\n"
+ "sub %x[N], %x[N], x24, LSL #3\n"
+ "st1w { z26.s }, p2, [%x[output_ptr], #2, MUL VL]\n"
+ "st1w { z27.s }, p2, [%x[output_ptr], #3, MUL VL]\n"
+ "st1w { z28.s }, p2, [%x[output_ptr], #4, MUL VL]\n"
+ "st1w { z29.s }, p2, [%x[output_ptr], #5, MUL VL]\n"
+ "st1w { z30.s }, p2, [%x[output_ptr], #6, MUL VL]\n"
+ "st1w { z31.s }, p1, [%x[output_ptr], #7, MUL VL]\n"
+ "addvl %x[output_ptr], %x[output_ptr], #8\n"
+ "bgt 1b\n"
+ "57:" // Exit
+
+ : [B_ptr] "+r" (B_ptr), [N] "+r" (N), [output_ptr] "+r" (output_ptr)
+ : [A_ptr] "r" (A_ptr), [K] "r" (K), [args_ptr] "r" (&ka), [bias] "r" (bias), [flags] "r" (flags), [offset_max] "I" (offsetof(KernelArgs, maxval)), [offset_min] "I" (offsetof(KernelArgs, minval))
+ : "cc", "memory", "p0", "p1", "p2", "x19", "x20", "x21", "x22", "x23", "x24", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+ );
+}
+
+} // namespace arm_gemm
+
+#endif
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_bf16fp32_dot_4VLx4/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_bf16fp32_dot_4VLx4/generic.cpp
deleted file mode 100644
index 385a16f..0000000
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_bf16fp32_dot_4VLx4/generic.cpp
+++ /dev/null
@@ -1,2247 +0,0 @@
-/*
- * Copyright (c) 2018-2020 Arm Limited.
- *
- * SPDX-License-Identifier: MIT
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to
- * deal in the Software without restriction, including without limitation the
- * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
- * sell copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-#ifdef __ARM_FEATURE_SVE
-
-#include <algorithm>
-
-#include "arm_gemm.hpp"
-#include "../../bfloat.hpp"
-#include "../../asmlib.hpp"
-#include "../../utils.hpp"
-
-namespace arm_gemm {
-
-void sve_hybrid_bf16fp32_dot_4VLx4(const bfloat16 *A, int lda, const bfloat16 *B, float *C, int ldc, int M, int N, int K, const float *bias, Activation act, bool accumulate) {
- const int K_stride = ((K + 1) / 2) * 2;
- const long loops_count = ((K + 8) / 16) - 1;
- K -= loops_count * 16;
- const long regs_count = (K / 8) - 1;
- K -= (regs_count + 1) * 8;
- const long leftovers = K;
- const long blocks_count = (K + 1) / 2;
- float nullbias[256];
- if (!accumulate && !bias) {
- memset(nullbias, 0, (4 * get_vector_length<float>() * sizeof(float)));
- }
- float minval = - static_cast<float>(std::numeric_limits<float>::infinity());
- float maxval = static_cast<float>(std::numeric_limits<float>::infinity());
- const float * const minptr = &minval;
- const float * const maxptr = &maxval;
-
- switch(act.type)
- {
- default:
- case Activation::Type::None:
- break;
- case Activation::Type::BoundedReLU:
- maxval = static_cast<float>(act.param1);
- /* fall through */
- case Activation::Type::ReLU:
- minval = 0.0f;
- break;
- }
-
- int rows_to_compute;
-
- for (int y=0; y<M; y+=rows_to_compute) {
- const bfloat16 * const a_ptr0_base = A + (y * lda);
- const unsigned long ldab = lda * sizeof(bfloat16);
-
- float *c_ptr0 = C + (y * ldc);
-
- rows_to_compute = M-y;
- if (rows_to_compute > 4) {
- if (rows_to_compute % 4) {
- rows_to_compute = 4 - 1;
- } else {
- rows_to_compute = 4;
- }
- }
-
- for (int x0=0; x0<N; x0+=(4 * get_vector_length<float>())) {
- const long width = std::min((unsigned long)N-x0, (4 * get_vector_length<float>()));
- long loops = loops_count;
- long regs = regs_count;
- long temp = 0;
- long blocks = blocks_count;
- const bfloat16 *a_ptr0 = a_ptr0_base;
- const bfloat16 *b_ptr0 = B + (K_stride * x0);
- const unsigned long ldcb = ldc * sizeof(float);
- const float *biasptr = bias ? bias+x0 : nullbias;
-
- switch(rows_to_compute) {
- case 1:
- __asm __volatile (
- "whilelt p6.h, %[temp], %[leftovers]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "ptrue p7.h\n"
- "whilelt p1.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "whilelt p2.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "whilelt p3.s, %[temp], %[width]\n"
- "cbnz %[accumulate], 1f\n"
- "ld1w z16.s, p0/z, [%[biasptr]]\n"
- "ld1w z17.s, p1/z, [%[biasptr], #1, MUL VL]\n"
- "ld1w z18.s, p2/z, [%[biasptr], #2, MUL VL]\n"
- "ld1w z19.s, p3/z, [%[biasptr], #3, MUL VL]\n"
- "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "ld1w z16.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n"
- "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n"
- "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n"
- "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "cbz %[loops], 2f\n"
- "3:\n"
- ".inst 0x64604110 // bfdot z16.s, z8.h, z0.h[0]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- ".inst 0x64604131 // bfdot z17.s, z9.h, z0.h[0]\n"
- "ld1rqh z4.h, p7/z, [%[a_ptr0]]\n"
- ".inst 0x64604152 // bfdot z18.s, z10.h, z0.h[0]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- ".inst 0x64604173 // bfdot z19.s, z11.h, z0.h[0]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x64684190 // bfdot z16.s, z12.h, z0.h[1]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x646841b1 // bfdot z17.s, z13.h, z0.h[1]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x646841d2 // bfdot z18.s, z14.h, z0.h[1]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x646841f3 // bfdot z19.s, z15.h, z0.h[1]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x64704110 // bfdot z16.s, z8.h, z0.h[2]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x64704131 // bfdot z17.s, z9.h, z0.h[2]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x64704152 // bfdot z18.s, z10.h, z0.h[2]\n"
- "subs %[loops], %[loops], #0x1\n"
- ".inst 0x64704173 // bfdot z19.s, z11.h, z0.h[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- ".inst 0x64784190 // bfdot z16.s, z12.h, z0.h[3]\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
- ".inst 0x647841b1 // bfdot z17.s, z13.h, z0.h[3]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- ".inst 0x647841d2 // bfdot z18.s, z14.h, z0.h[3]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- ".inst 0x647841f3 // bfdot z19.s, z15.h, z0.h[3]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- ".inst 0x64644110 // bfdot z16.s, z8.h, z4.h[0]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- ".inst 0x64644131 // bfdot z17.s, z9.h, z4.h[0]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- ".inst 0x64644152 // bfdot z18.s, z10.h, z4.h[0]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- ".inst 0x64644173 // bfdot z19.s, z11.h, z4.h[0]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- ".inst 0x646c4190 // bfdot z16.s, z12.h, z4.h[1]\n"
- "ld1rqh z0.h, p7/z, [%[a_ptr0], #-0x10]\n"
- ".inst 0x646c41b1 // bfdot z17.s, z13.h, z4.h[1]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- ".inst 0x646c41d2 // bfdot z18.s, z14.h, z4.h[1]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x646c41f3 // bfdot z19.s, z15.h, z4.h[1]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x64744110 // bfdot z16.s, z8.h, z4.h[2]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x64744131 // bfdot z17.s, z9.h, z4.h[2]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x64744152 // bfdot z18.s, z10.h, z4.h[2]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x64744173 // bfdot z19.s, z11.h, z4.h[2]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x647c4190 // bfdot z16.s, z12.h, z4.h[3]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- ".inst 0x647c41b1 // bfdot z17.s, z13.h, z4.h[3]\n"
- ".inst 0x647c41d2 // bfdot z18.s, z14.h, z4.h[3]\n"
- ".inst 0x647c41f3 // bfdot z19.s, z15.h, z4.h[3]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "b.ne 3b\n"
- "2:\n"
- "cbz %[regs], 4f\n"
- ".inst 0x64604110 // bfdot z16.s, z8.h, z0.h[0]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- ".inst 0x64604131 // bfdot z17.s, z9.h, z0.h[0]\n"
- "ld1rqh z4.h, p7/z, [%[a_ptr0]]\n"
- ".inst 0x64604152 // bfdot z18.s, z10.h, z0.h[0]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- ".inst 0x64604173 // bfdot z19.s, z11.h, z0.h[0]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x64684190 // bfdot z16.s, z12.h, z0.h[1]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x646841b1 // bfdot z17.s, z13.h, z0.h[1]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x646841d2 // bfdot z18.s, z14.h, z0.h[1]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x646841f3 // bfdot z19.s, z15.h, z0.h[1]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x64704110 // bfdot z16.s, z8.h, z0.h[2]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x64704131 // bfdot z17.s, z9.h, z0.h[2]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x64704152 // bfdot z18.s, z10.h, z0.h[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- ".inst 0x64704173 // bfdot z19.s, z11.h, z0.h[2]\n"
- ".inst 0x64784190 // bfdot z16.s, z12.h, z0.h[3]\n"
- ".inst 0x647841b1 // bfdot z17.s, z13.h, z0.h[3]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- ".inst 0x647841d2 // bfdot z18.s, z14.h, z0.h[3]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- ".inst 0x647841f3 // bfdot z19.s, z15.h, z0.h[3]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- ".inst 0x64644110 // bfdot z16.s, z8.h, z4.h[0]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- ".inst 0x64644131 // bfdot z17.s, z9.h, z4.h[0]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- ".inst 0x64644152 // bfdot z18.s, z10.h, z4.h[0]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- ".inst 0x64644173 // bfdot z19.s, z11.h, z4.h[0]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- ".inst 0x646c4190 // bfdot z16.s, z12.h, z4.h[1]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- ".inst 0x646c41b1 // bfdot z17.s, z13.h, z4.h[1]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x646c41d2 // bfdot z18.s, z14.h, z4.h[1]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x646c41f3 // bfdot z19.s, z15.h, z4.h[1]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x64744110 // bfdot z16.s, z8.h, z4.h[2]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x64744131 // bfdot z17.s, z9.h, z4.h[2]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x64744152 // bfdot z18.s, z10.h, z4.h[2]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x64744173 // bfdot z19.s, z11.h, z4.h[2]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x647c4190 // bfdot z16.s, z12.h, z4.h[3]\n"
- "ld1rqh z0.h, p6/z, [%[a_ptr0], #0x10]\n"
- ".inst 0x647c41b1 // bfdot z17.s, z13.h, z4.h[3]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- ".inst 0x647c41d2 // bfdot z18.s, z14.h, z4.h[3]\n"
- "addvl %[a_ptr0], %[a_ptr0], #2\n"
- ".inst 0x647c41f3 // bfdot z19.s, z15.h, z4.h[3]\n"
- "cbz %[blocks], 5f\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x64604110 // bfdot z16.s, z8.h, z0.h[0]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x64604131 // bfdot z17.s, z9.h, z0.h[0]\n"
- ".inst 0x64604152 // bfdot z18.s, z10.h, z0.h[0]\n"
- ".inst 0x64604173 // bfdot z19.s, z11.h, z0.h[0]\n"
- "b.eq 5f\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x64684190 // bfdot z16.s, z12.h, z0.h[1]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x646841b1 // bfdot z17.s, z13.h, z0.h[1]\n"
- ".inst 0x646841d2 // bfdot z18.s, z14.h, z0.h[1]\n"
- ".inst 0x646841f3 // bfdot z19.s, z15.h, z0.h[1]\n"
- "b.eq 5f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- ".inst 0x64704110 // bfdot z16.s, z8.h, z0.h[2]\n"
- ".inst 0x64704131 // bfdot z17.s, z9.h, z0.h[2]\n"
- ".inst 0x64704152 // bfdot z18.s, z10.h, z0.h[2]\n"
- ".inst 0x64704173 // bfdot z19.s, z11.h, z0.h[2]\n"
- "b.eq 5f\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- ".inst 0x64784190 // bfdot z16.s, z12.h, z0.h[3]\n"
- ".inst 0x647841b1 // bfdot z17.s, z13.h, z0.h[3]\n"
- ".inst 0x647841d2 // bfdot z18.s, z14.h, z0.h[3]\n"
- ".inst 0x647841f3 // bfdot z19.s, z15.h, z0.h[3]\n"
- "b 5f\n"
- "4:\n"
- ".inst 0x64604110 // bfdot z16.s, z8.h, z0.h[0]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- ".inst 0x64604131 // bfdot z17.s, z9.h, z0.h[0]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- ".inst 0x64604152 // bfdot z18.s, z10.h, z0.h[0]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x64604173 // bfdot z19.s, z11.h, z0.h[0]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x64684190 // bfdot z16.s, z12.h, z0.h[1]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x646841b1 // bfdot z17.s, z13.h, z0.h[1]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x646841d2 // bfdot z18.s, z14.h, z0.h[1]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x646841f3 // bfdot z19.s, z15.h, z0.h[1]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x64704110 // bfdot z16.s, z8.h, z0.h[2]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x64704131 // bfdot z17.s, z9.h, z0.h[2]\n"
- "ld1rqh z4.h, p6/z, [%[a_ptr0]]\n"
- ".inst 0x64704152 // bfdot z18.s, z10.h, z0.h[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- ".inst 0x64704173 // bfdot z19.s, z11.h, z0.h[2]\n"
- "addvl %[a_ptr0], %[a_ptr0], #1\n"
- ".inst 0x64784190 // bfdot z16.s, z12.h, z0.h[3]\n"
- ".inst 0x647841b1 // bfdot z17.s, z13.h, z0.h[3]\n"
- ".inst 0x647841d2 // bfdot z18.s, z14.h, z0.h[3]\n"
- ".inst 0x647841f3 // bfdot z19.s, z15.h, z0.h[3]\n"
- "cbz %[blocks], 5f\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x64644110 // bfdot z16.s, z8.h, z4.h[0]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x64644131 // bfdot z17.s, z9.h, z4.h[0]\n"
- ".inst 0x64644152 // bfdot z18.s, z10.h, z4.h[0]\n"
- ".inst 0x64644173 // bfdot z19.s, z11.h, z4.h[0]\n"
- "b.eq 5f\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x646c4190 // bfdot z16.s, z12.h, z4.h[1]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x646c41b1 // bfdot z17.s, z13.h, z4.h[1]\n"
- ".inst 0x646c41d2 // bfdot z18.s, z14.h, z4.h[1]\n"
- ".inst 0x646c41f3 // bfdot z19.s, z15.h, z4.h[1]\n"
- "b.eq 5f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- ".inst 0x64744110 // bfdot z16.s, z8.h, z4.h[2]\n"
- ".inst 0x64744131 // bfdot z17.s, z9.h, z4.h[2]\n"
- ".inst 0x64744152 // bfdot z18.s, z10.h, z4.h[2]\n"
- ".inst 0x64744173 // bfdot z19.s, z11.h, z4.h[2]\n"
- "b.eq 5f\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- ".inst 0x647c4190 // bfdot z16.s, z12.h, z4.h[3]\n"
- ".inst 0x647c41b1 // bfdot z17.s, z13.h, z4.h[3]\n"
- ".inst 0x647c41d2 // bfdot z18.s, z14.h, z4.h[3]\n"
- ".inst 0x647c41f3 // bfdot z19.s, z15.h, z4.h[3]\n"
- "5:\n"
- "ld1rw z14.s, p7/z, [%[minptr]]\n"
- "ld1rw z15.s, p7/z, [%[maxptr]]\n"
- "fmax z16.s, p7/m, z16.s, z14.s\n"
- "fmax z17.s, p7/m, z17.s, z14.s\n"
- "fmax z18.s, p7/m, z18.s, z14.s\n"
- "fmax z19.s, p7/m, z19.s, z14.s\n"
- "fmin z16.s, p7/m, z16.s, z15.s\n"
- "fmin z17.s, p7/m, z17.s, z15.s\n"
- "fmin z18.s, p7/m, z18.s, z15.s\n"
- "fmin z19.s, p7/m, z19.s, z15.s\n"
- "st1w z16.s, p0, [%[c_ptr0]]\n"
- "st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n"
- "st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n"
- "st1w z19.s, p3, [%[c_ptr0], #3, MUL VL]\n"
- "addvl %[c_ptr0], %[c_ptr0], #4\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks)
- : [width] "r" (width), [accumulate] "r" (static_cast<uint64_t>(accumulate)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers)
- : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
- );
- break;
- case 2:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "c_ptr1 .req X1\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "whilelt p6.h, %[temp], %[leftovers]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "ptrue p7.h\n"
- "whilelt p1.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "whilelt p2.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "whilelt p3.s, %[temp], %[width]\n"
- "cbnz %[accumulate], 1f\n"
- "ld1w z16.s, p0/z, [%[biasptr]]\n"
- "ld1w z17.s, p1/z, [%[biasptr], #1, MUL VL]\n"
- "ld1w z18.s, p2/z, [%[biasptr], #2, MUL VL]\n"
- "ld1w z19.s, p3/z, [%[biasptr], #3, MUL VL]\n"
- "mov z20.d, z16.d\n"
- "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n"
- "mov z21.d, z17.d\n"
- "ld1rqh z1.h, p7/z, [a_ptr1]\n"
- "mov z22.d, z18.d\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "mov z23.d, z19.d\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "ld1w z16.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n"
- "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n"
- "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n"
- "ld1w z20.s, p0/z, [c_ptr1]\n"
- "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n"
- "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n"
- "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n"
- "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ld1rqh z1.h, p7/z, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "cbz %[loops], 2f\n"
- "3:\n"
- ".inst 0x64604110 // bfdot z16.s, z8.h, z0.h[0]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- ".inst 0x64614114 // bfdot z20.s, z8.h, z1.h[0]\n"
- "ld1rqh z4.h, p7/z, [%[a_ptr0]]\n"
- ".inst 0x64604131 // bfdot z17.s, z9.h, z0.h[0]\n"
- "ld1rqh z5.h, p7/z, [a_ptr1]\n"
- ".inst 0x64614135 // bfdot z21.s, z9.h, z1.h[0]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- ".inst 0x64604152 // bfdot z18.s, z10.h, z0.h[0]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x64614156 // bfdot z22.s, z10.h, z1.h[0]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x64604173 // bfdot z19.s, z11.h, z0.h[0]\n"
- "subs %[loops], %[loops], #0x1\n"
- ".inst 0x64614177 // bfdot z23.s, z11.h, z1.h[0]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x64684190 // bfdot z16.s, z12.h, z0.h[1]\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
- ".inst 0x64694194 // bfdot z20.s, z12.h, z1.h[1]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x646841b1 // bfdot z17.s, z13.h, z0.h[1]\n"
- "add a_ptr1, a_ptr1, #0x20\n"
- ".inst 0x646941b5 // bfdot z21.s, z13.h, z1.h[1]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x646841d2 // bfdot z18.s, z14.h, z0.h[1]\n"
- ".inst 0x646941d6 // bfdot z22.s, z14.h, z1.h[1]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x646841f3 // bfdot z19.s, z15.h, z0.h[1]\n"
- ".inst 0x646941f7 // bfdot z23.s, z15.h, z1.h[1]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x64704110 // bfdot z16.s, z8.h, z0.h[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- ".inst 0x64714114 // bfdot z20.s, z8.h, z1.h[2]\n"
- ".inst 0x64704131 // bfdot z17.s, z9.h, z0.h[2]\n"
- ".inst 0x64714135 // bfdot z21.s, z9.h, z1.h[2]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- ".inst 0x64704152 // bfdot z18.s, z10.h, z0.h[2]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- ".inst 0x64714156 // bfdot z22.s, z10.h, z1.h[2]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- ".inst 0x64704173 // bfdot z19.s, z11.h, z0.h[2]\n"
- ".inst 0x64714177 // bfdot z23.s, z11.h, z1.h[2]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- ".inst 0x64784190 // bfdot z16.s, z12.h, z0.h[3]\n"
- ".inst 0x64794194 // bfdot z20.s, z12.h, z1.h[3]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- ".inst 0x647841b1 // bfdot z17.s, z13.h, z0.h[3]\n"
- ".inst 0x647941b5 // bfdot z21.s, z13.h, z1.h[3]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- ".inst 0x647841d2 // bfdot z18.s, z14.h, z0.h[3]\n"
- ".inst 0x647941d6 // bfdot z22.s, z14.h, z1.h[3]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- ".inst 0x647841f3 // bfdot z19.s, z15.h, z0.h[3]\n"
- "ld1rqh z0.h, p7/z, [%[a_ptr0], #-0x10]\n"
- ".inst 0x647941f7 // bfdot z23.s, z15.h, z1.h[3]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- ".inst 0x64644110 // bfdot z16.s, z8.h, z4.h[0]\n"
- "ld1rqh z1.h, p7/z, [a_ptr1, #-0x10]\n"
- ".inst 0x64654114 // bfdot z20.s, z8.h, z5.h[0]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- ".inst 0x64644131 // bfdot z17.s, z9.h, z4.h[0]\n"
- ".inst 0x64654135 // bfdot z21.s, z9.h, z5.h[0]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x64644152 // bfdot z18.s, z10.h, z4.h[0]\n"
- ".inst 0x64654156 // bfdot z22.s, z10.h, z5.h[0]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x64644173 // bfdot z19.s, z11.h, z4.h[0]\n"
- ".inst 0x64654177 // bfdot z23.s, z11.h, z5.h[0]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x646c4190 // bfdot z16.s, z12.h, z4.h[1]\n"
- ".inst 0x646d4194 // bfdot z20.s, z12.h, z5.h[1]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x646c41b1 // bfdot z17.s, z13.h, z4.h[1]\n"
- ".inst 0x646d41b5 // bfdot z21.s, z13.h, z5.h[1]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x646c41d2 // bfdot z18.s, z14.h, z4.h[1]\n"
- ".inst 0x646d41d6 // bfdot z22.s, z14.h, z5.h[1]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x646c41f3 // bfdot z19.s, z15.h, z4.h[1]\n"
- ".inst 0x646d41f7 // bfdot z23.s, z15.h, z5.h[1]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x64744110 // bfdot z16.s, z8.h, z4.h[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- ".inst 0x64754114 // bfdot z20.s, z8.h, z5.h[2]\n"
- ".inst 0x64744131 // bfdot z17.s, z9.h, z4.h[2]\n"
- ".inst 0x64754135 // bfdot z21.s, z9.h, z5.h[2]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- ".inst 0x64744152 // bfdot z18.s, z10.h, z4.h[2]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- ".inst 0x64754156 // bfdot z22.s, z10.h, z5.h[2]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- ".inst 0x64744173 // bfdot z19.s, z11.h, z4.h[2]\n"
- ".inst 0x64754177 // bfdot z23.s, z11.h, z5.h[2]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- ".inst 0x647c4190 // bfdot z16.s, z12.h, z4.h[3]\n"
- ".inst 0x647d4194 // bfdot z20.s, z12.h, z5.h[3]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- ".inst 0x647c41b1 // bfdot z17.s, z13.h, z4.h[3]\n"
- ".inst 0x647d41b5 // bfdot z21.s, z13.h, z5.h[3]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- ".inst 0x647c41d2 // bfdot z18.s, z14.h, z4.h[3]\n"
- ".inst 0x647d41d6 // bfdot z22.s, z14.h, z5.h[3]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- ".inst 0x647c41f3 // bfdot z19.s, z15.h, z4.h[3]\n"
- ".inst 0x647d41f7 // bfdot z23.s, z15.h, z5.h[3]\n"
- "b.ne 3b\n"
- "2:\n"
- "cbz %[regs], 4f\n"
- ".inst 0x64604110 // bfdot z16.s, z8.h, z0.h[0]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- ".inst 0x64614114 // bfdot z20.s, z8.h, z1.h[0]\n"
- "ld1rqh z4.h, p7/z, [%[a_ptr0]]\n"
- ".inst 0x64604131 // bfdot z17.s, z9.h, z0.h[0]\n"
- "ld1rqh z5.h, p7/z, [a_ptr1]\n"
- ".inst 0x64614135 // bfdot z21.s, z9.h, z1.h[0]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- ".inst 0x64604152 // bfdot z18.s, z10.h, z0.h[0]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x64614156 // bfdot z22.s, z10.h, z1.h[0]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x64604173 // bfdot z19.s, z11.h, z0.h[0]\n"
- ".inst 0x64614177 // bfdot z23.s, z11.h, z1.h[0]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x64684190 // bfdot z16.s, z12.h, z0.h[1]\n"
- ".inst 0x64694194 // bfdot z20.s, z12.h, z1.h[1]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x646841b1 // bfdot z17.s, z13.h, z0.h[1]\n"
- ".inst 0x646941b5 // bfdot z21.s, z13.h, z1.h[1]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x646841d2 // bfdot z18.s, z14.h, z0.h[1]\n"
- ".inst 0x646941d6 // bfdot z22.s, z14.h, z1.h[1]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x646841f3 // bfdot z19.s, z15.h, z0.h[1]\n"
- ".inst 0x646941f7 // bfdot z23.s, z15.h, z1.h[1]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x64704110 // bfdot z16.s, z8.h, z0.h[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- ".inst 0x64714114 // bfdot z20.s, z8.h, z1.h[2]\n"
- ".inst 0x64704131 // bfdot z17.s, z9.h, z0.h[2]\n"
- ".inst 0x64714135 // bfdot z21.s, z9.h, z1.h[2]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- ".inst 0x64704152 // bfdot z18.s, z10.h, z0.h[2]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- ".inst 0x64714156 // bfdot z22.s, z10.h, z1.h[2]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- ".inst 0x64704173 // bfdot z19.s, z11.h, z0.h[2]\n"
- ".inst 0x64714177 // bfdot z23.s, z11.h, z1.h[2]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- ".inst 0x64784190 // bfdot z16.s, z12.h, z0.h[3]\n"
- ".inst 0x64794194 // bfdot z20.s, z12.h, z1.h[3]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- ".inst 0x647841b1 // bfdot z17.s, z13.h, z0.h[3]\n"
- ".inst 0x647941b5 // bfdot z21.s, z13.h, z1.h[3]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- ".inst 0x647841d2 // bfdot z18.s, z14.h, z0.h[3]\n"
- ".inst 0x647941d6 // bfdot z22.s, z14.h, z1.h[3]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- ".inst 0x647841f3 // bfdot z19.s, z15.h, z0.h[3]\n"
- "ld1rqh z0.h, p6/z, [%[a_ptr0], #0x10]\n"
- ".inst 0x647941f7 // bfdot z23.s, z15.h, z1.h[3]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- ".inst 0x64644110 // bfdot z16.s, z8.h, z4.h[0]\n"
- "ld1rqh z1.h, p6/z, [a_ptr1, #0x10]\n"
- ".inst 0x64654114 // bfdot z20.s, z8.h, z5.h[0]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- ".inst 0x64644131 // bfdot z17.s, z9.h, z4.h[0]\n"
- "addvl %[a_ptr0], %[a_ptr0], #2\n"
- ".inst 0x64654135 // bfdot z21.s, z9.h, z5.h[0]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x64644152 // bfdot z18.s, z10.h, z4.h[0]\n"
- "addvl a_ptr1, a_ptr1, #2\n"
- ".inst 0x64654156 // bfdot z22.s, z10.h, z5.h[0]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x64644173 // bfdot z19.s, z11.h, z4.h[0]\n"
- ".inst 0x64654177 // bfdot z23.s, z11.h, z5.h[0]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x646c4190 // bfdot z16.s, z12.h, z4.h[1]\n"
- ".inst 0x646d4194 // bfdot z20.s, z12.h, z5.h[1]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x646c41b1 // bfdot z17.s, z13.h, z4.h[1]\n"
- ".inst 0x646d41b5 // bfdot z21.s, z13.h, z5.h[1]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x646c41d2 // bfdot z18.s, z14.h, z4.h[1]\n"
- ".inst 0x646d41d6 // bfdot z22.s, z14.h, z5.h[1]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x646c41f3 // bfdot z19.s, z15.h, z4.h[1]\n"
- ".inst 0x646d41f7 // bfdot z23.s, z15.h, z5.h[1]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x64744110 // bfdot z16.s, z8.h, z4.h[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- ".inst 0x64754114 // bfdot z20.s, z8.h, z5.h[2]\n"
- ".inst 0x64744131 // bfdot z17.s, z9.h, z4.h[2]\n"
- ".inst 0x64754135 // bfdot z21.s, z9.h, z5.h[2]\n"
- ".inst 0x64744152 // bfdot z18.s, z10.h, z4.h[2]\n"
- ".inst 0x64754156 // bfdot z22.s, z10.h, z5.h[2]\n"
- ".inst 0x64744173 // bfdot z19.s, z11.h, z4.h[2]\n"
- ".inst 0x64754177 // bfdot z23.s, z11.h, z5.h[2]\n"
- ".inst 0x647c4190 // bfdot z16.s, z12.h, z4.h[3]\n"
- ".inst 0x647d4194 // bfdot z20.s, z12.h, z5.h[3]\n"
- ".inst 0x647c41b1 // bfdot z17.s, z13.h, z4.h[3]\n"
- ".inst 0x647d41b5 // bfdot z21.s, z13.h, z5.h[3]\n"
- ".inst 0x647c41d2 // bfdot z18.s, z14.h, z4.h[3]\n"
- ".inst 0x647d41d6 // bfdot z22.s, z14.h, z5.h[3]\n"
- ".inst 0x647c41f3 // bfdot z19.s, z15.h, z4.h[3]\n"
- ".inst 0x647d41f7 // bfdot z23.s, z15.h, z5.h[3]\n"
- "cbz %[blocks], 5f\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x64604110 // bfdot z16.s, z8.h, z0.h[0]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x64614114 // bfdot z20.s, z8.h, z1.h[0]\n"
- ".inst 0x64604131 // bfdot z17.s, z9.h, z0.h[0]\n"
- ".inst 0x64614135 // bfdot z21.s, z9.h, z1.h[0]\n"
- ".inst 0x64604152 // bfdot z18.s, z10.h, z0.h[0]\n"
- ".inst 0x64614156 // bfdot z22.s, z10.h, z1.h[0]\n"
- ".inst 0x64604173 // bfdot z19.s, z11.h, z0.h[0]\n"
- ".inst 0x64614177 // bfdot z23.s, z11.h, z1.h[0]\n"
- "b.eq 5f\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x64684190 // bfdot z16.s, z12.h, z0.h[1]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x64694194 // bfdot z20.s, z12.h, z1.h[1]\n"
- ".inst 0x646841b1 // bfdot z17.s, z13.h, z0.h[1]\n"
- ".inst 0x646941b5 // bfdot z21.s, z13.h, z1.h[1]\n"
- ".inst 0x646841d2 // bfdot z18.s, z14.h, z0.h[1]\n"
- ".inst 0x646941d6 // bfdot z22.s, z14.h, z1.h[1]\n"
- ".inst 0x646841f3 // bfdot z19.s, z15.h, z0.h[1]\n"
- ".inst 0x646941f7 // bfdot z23.s, z15.h, z1.h[1]\n"
- "b.eq 5f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- ".inst 0x64704110 // bfdot z16.s, z8.h, z0.h[2]\n"
- ".inst 0x64714114 // bfdot z20.s, z8.h, z1.h[2]\n"
- ".inst 0x64704131 // bfdot z17.s, z9.h, z0.h[2]\n"
- ".inst 0x64714135 // bfdot z21.s, z9.h, z1.h[2]\n"
- ".inst 0x64704152 // bfdot z18.s, z10.h, z0.h[2]\n"
- ".inst 0x64714156 // bfdot z22.s, z10.h, z1.h[2]\n"
- ".inst 0x64704173 // bfdot z19.s, z11.h, z0.h[2]\n"
- ".inst 0x64714177 // bfdot z23.s, z11.h, z1.h[2]\n"
- "b.eq 5f\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- ".inst 0x64784190 // bfdot z16.s, z12.h, z0.h[3]\n"
- ".inst 0x64794194 // bfdot z20.s, z12.h, z1.h[3]\n"
- ".inst 0x647841b1 // bfdot z17.s, z13.h, z0.h[3]\n"
- ".inst 0x647941b5 // bfdot z21.s, z13.h, z1.h[3]\n"
- ".inst 0x647841d2 // bfdot z18.s, z14.h, z0.h[3]\n"
- ".inst 0x647941d6 // bfdot z22.s, z14.h, z1.h[3]\n"
- ".inst 0x647841f3 // bfdot z19.s, z15.h, z0.h[3]\n"
- ".inst 0x647941f7 // bfdot z23.s, z15.h, z1.h[3]\n"
- "b 5f\n"
- "4:\n"
- ".inst 0x64604110 // bfdot z16.s, z8.h, z0.h[0]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- ".inst 0x64614114 // bfdot z20.s, z8.h, z1.h[0]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- ".inst 0x64604131 // bfdot z17.s, z9.h, z0.h[0]\n"
- "ld1rqh z4.h, p6/z, [%[a_ptr0]]\n"
- ".inst 0x64614135 // bfdot z21.s, z9.h, z1.h[0]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x64604152 // bfdot z18.s, z10.h, z0.h[0]\n"
- "ld1rqh z5.h, p6/z, [a_ptr1]\n"
- ".inst 0x64614156 // bfdot z22.s, z10.h, z1.h[0]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x64604173 // bfdot z19.s, z11.h, z0.h[0]\n"
- "addvl %[a_ptr0], %[a_ptr0], #1\n"
- ".inst 0x64614177 // bfdot z23.s, z11.h, z1.h[0]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x64684190 // bfdot z16.s, z12.h, z0.h[1]\n"
- "addvl a_ptr1, a_ptr1, #1\n"
- ".inst 0x64694194 // bfdot z20.s, z12.h, z1.h[1]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x646841b1 // bfdot z17.s, z13.h, z0.h[1]\n"
- ".inst 0x646941b5 // bfdot z21.s, z13.h, z1.h[1]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x646841d2 // bfdot z18.s, z14.h, z0.h[1]\n"
- ".inst 0x646941d6 // bfdot z22.s, z14.h, z1.h[1]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x646841f3 // bfdot z19.s, z15.h, z0.h[1]\n"
- ".inst 0x646941f7 // bfdot z23.s, z15.h, z1.h[1]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x64704110 // bfdot z16.s, z8.h, z0.h[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- ".inst 0x64714114 // bfdot z20.s, z8.h, z1.h[2]\n"
- ".inst 0x64704131 // bfdot z17.s, z9.h, z0.h[2]\n"
- ".inst 0x64714135 // bfdot z21.s, z9.h, z1.h[2]\n"
- ".inst 0x64704152 // bfdot z18.s, z10.h, z0.h[2]\n"
- ".inst 0x64714156 // bfdot z22.s, z10.h, z1.h[2]\n"
- ".inst 0x64704173 // bfdot z19.s, z11.h, z0.h[2]\n"
- ".inst 0x64714177 // bfdot z23.s, z11.h, z1.h[2]\n"
- ".inst 0x64784190 // bfdot z16.s, z12.h, z0.h[3]\n"
- ".inst 0x64794194 // bfdot z20.s, z12.h, z1.h[3]\n"
- ".inst 0x647841b1 // bfdot z17.s, z13.h, z0.h[3]\n"
- ".inst 0x647941b5 // bfdot z21.s, z13.h, z1.h[3]\n"
- ".inst 0x647841d2 // bfdot z18.s, z14.h, z0.h[3]\n"
- ".inst 0x647941d6 // bfdot z22.s, z14.h, z1.h[3]\n"
- ".inst 0x647841f3 // bfdot z19.s, z15.h, z0.h[3]\n"
- ".inst 0x647941f7 // bfdot z23.s, z15.h, z1.h[3]\n"
- "cbz %[blocks], 5f\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x64644110 // bfdot z16.s, z8.h, z4.h[0]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x64654114 // bfdot z20.s, z8.h, z5.h[0]\n"
- ".inst 0x64644131 // bfdot z17.s, z9.h, z4.h[0]\n"
- ".inst 0x64654135 // bfdot z21.s, z9.h, z5.h[0]\n"
- ".inst 0x64644152 // bfdot z18.s, z10.h, z4.h[0]\n"
- ".inst 0x64654156 // bfdot z22.s, z10.h, z5.h[0]\n"
- ".inst 0x64644173 // bfdot z19.s, z11.h, z4.h[0]\n"
- ".inst 0x64654177 // bfdot z23.s, z11.h, z5.h[0]\n"
- "b.eq 5f\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x646c4190 // bfdot z16.s, z12.h, z4.h[1]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x646d4194 // bfdot z20.s, z12.h, z5.h[1]\n"
- ".inst 0x646c41b1 // bfdot z17.s, z13.h, z4.h[1]\n"
- ".inst 0x646d41b5 // bfdot z21.s, z13.h, z5.h[1]\n"
- ".inst 0x646c41d2 // bfdot z18.s, z14.h, z4.h[1]\n"
- ".inst 0x646d41d6 // bfdot z22.s, z14.h, z5.h[1]\n"
- ".inst 0x646c41f3 // bfdot z19.s, z15.h, z4.h[1]\n"
- ".inst 0x646d41f7 // bfdot z23.s, z15.h, z5.h[1]\n"
- "b.eq 5f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- ".inst 0x64744110 // bfdot z16.s, z8.h, z4.h[2]\n"
- ".inst 0x64754114 // bfdot z20.s, z8.h, z5.h[2]\n"
- ".inst 0x64744131 // bfdot z17.s, z9.h, z4.h[2]\n"
- ".inst 0x64754135 // bfdot z21.s, z9.h, z5.h[2]\n"
- ".inst 0x64744152 // bfdot z18.s, z10.h, z4.h[2]\n"
- ".inst 0x64754156 // bfdot z22.s, z10.h, z5.h[2]\n"
- ".inst 0x64744173 // bfdot z19.s, z11.h, z4.h[2]\n"
- ".inst 0x64754177 // bfdot z23.s, z11.h, z5.h[2]\n"
- "b.eq 5f\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- ".inst 0x647c4190 // bfdot z16.s, z12.h, z4.h[3]\n"
- ".inst 0x647d4194 // bfdot z20.s, z12.h, z5.h[3]\n"
- ".inst 0x647c41b1 // bfdot z17.s, z13.h, z4.h[3]\n"
- ".inst 0x647d41b5 // bfdot z21.s, z13.h, z5.h[3]\n"
- ".inst 0x647c41d2 // bfdot z18.s, z14.h, z4.h[3]\n"
- ".inst 0x647d41d6 // bfdot z22.s, z14.h, z5.h[3]\n"
- ".inst 0x647c41f3 // bfdot z19.s, z15.h, z4.h[3]\n"
- ".inst 0x647d41f7 // bfdot z23.s, z15.h, z5.h[3]\n"
- "5:\n"
- "ld1rw z14.s, p7/z, [%[minptr]]\n"
- "ld1rw z15.s, p7/z, [%[maxptr]]\n"
- "fmax z16.s, p7/m, z16.s, z14.s\n"
- "fmax z17.s, p7/m, z17.s, z14.s\n"
- "fmax z18.s, p7/m, z18.s, z14.s\n"
- "fmax z19.s, p7/m, z19.s, z14.s\n"
- "fmin z16.s, p7/m, z16.s, z15.s\n"
- "fmin z17.s, p7/m, z17.s, z15.s\n"
- "fmin z18.s, p7/m, z18.s, z15.s\n"
- "fmin z19.s, p7/m, z19.s, z15.s\n"
- "st1w z16.s, p0, [%[c_ptr0]]\n"
- "fmax z20.s, p7/m, z20.s, z14.s\n"
- "fmax z21.s, p7/m, z21.s, z14.s\n"
- "fmax z22.s, p7/m, z22.s, z14.s\n"
- "st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n"
- "fmax z23.s, p7/m, z23.s, z14.s\n"
- "fmin z20.s, p7/m, z20.s, z15.s\n"
- "fmin z21.s, p7/m, z21.s, z15.s\n"
- "st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n"
- "fmin z22.s, p7/m, z22.s, z15.s\n"
- "fmin z23.s, p7/m, z23.s, z15.s\n"
- "st1w z19.s, p3, [%[c_ptr0], #3, MUL VL]\n"
- "addvl %[c_ptr0], %[c_ptr0], #4\n"
- "st1w z20.s, p0, [c_ptr1]\n"
- "st1w z21.s, p1, [c_ptr1, #1, MUL VL]\n"
- "st1w z22.s, p2, [c_ptr1, #2, MUL VL]\n"
- "st1w z23.s, p3, [c_ptr1, #3, MUL VL]\n"
- ".unreq a_ptr1\n"
- ".unreq c_ptr1\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks)
- : [width] "r" (width), [accumulate] "r" (static_cast<uint64_t>(accumulate)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers)
- : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "cc", "memory"
- );
- break;
- case 3:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "c_ptr1 .req X2\n"
- "c_ptr2 .req X3\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "whilelt p6.h, %[temp], %[leftovers]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "ptrue p7.h\n"
- "whilelt p1.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "whilelt p2.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "whilelt p3.s, %[temp], %[width]\n"
- "cbnz %[accumulate], 1f\n"
- "ld1w z16.s, p0/z, [%[biasptr]]\n"
- "ld1w z17.s, p1/z, [%[biasptr], #1, MUL VL]\n"
- "ld1w z18.s, p2/z, [%[biasptr], #2, MUL VL]\n"
- "ld1w z19.s, p3/z, [%[biasptr], #3, MUL VL]\n"
- "mov z20.d, z16.d\n"
- "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n"
- "mov z21.d, z17.d\n"
- "ld1rqh z1.h, p7/z, [a_ptr1]\n"
- "mov z22.d, z18.d\n"
- "ld1rqh z2.h, p7/z, [a_ptr2]\n"
- "mov z23.d, z19.d\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "mov z24.d, z16.d\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "mov z25.d, z17.d\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "mov z26.d, z18.d\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "mov z27.d, z19.d\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "ld1w z16.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n"
- "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n"
- "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n"
- "ld1w z20.s, p0/z, [c_ptr1]\n"
- "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n"
- "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n"
- "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n"
- "ld1w z24.s, p0/z, [c_ptr2]\n"
- "ld1w z25.s, p1/z, [c_ptr2, #1, MUL VL]\n"
- "ld1w z26.s, p2/z, [c_ptr2, #2, MUL VL]\n"
- "ld1w z27.s, p3/z, [c_ptr2, #3, MUL VL]\n"
- "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ld1rqh z1.h, p7/z, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "ld1rqh z2.h, p7/z, [a_ptr2]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "cbz %[loops], 2f\n"
- "3:\n"
- ".inst 0x64604110 // bfdot z16.s, z8.h, z0.h[0]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- ".inst 0x64614114 // bfdot z20.s, z8.h, z1.h[0]\n"
- "ld1rqh z4.h, p7/z, [%[a_ptr0]]\n"
- ".inst 0x64624118 // bfdot z24.s, z8.h, z2.h[0]\n"
- "ld1rqh z5.h, p7/z, [a_ptr1]\n"
- ".inst 0x64604131 // bfdot z17.s, z9.h, z0.h[0]\n"
- "ld1rqh z6.h, p7/z, [a_ptr2]\n"
- ".inst 0x64614135 // bfdot z21.s, z9.h, z1.h[0]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- ".inst 0x64624139 // bfdot z25.s, z9.h, z2.h[0]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x64604152 // bfdot z18.s, z10.h, z0.h[0]\n"
- "subs %[loops], %[loops], #0x1\n"
- ".inst 0x64614156 // bfdot z22.s, z10.h, z1.h[0]\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
- ".inst 0x6462415a // bfdot z26.s, z10.h, z2.h[0]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x64604173 // bfdot z19.s, z11.h, z0.h[0]\n"
- "add a_ptr1, a_ptr1, #0x20\n"
- ".inst 0x64614177 // bfdot z23.s, z11.h, z1.h[0]\n"
- "add a_ptr2, a_ptr2, #0x20\n"
- ".inst 0x6462417b // bfdot z27.s, z11.h, z2.h[0]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x64684190 // bfdot z16.s, z12.h, z0.h[1]\n"
- ".inst 0x64694194 // bfdot z20.s, z12.h, z1.h[1]\n"
- ".inst 0x646a4198 // bfdot z24.s, z12.h, z2.h[1]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x646841b1 // bfdot z17.s, z13.h, z0.h[1]\n"
- ".inst 0x646941b5 // bfdot z21.s, z13.h, z1.h[1]\n"
- ".inst 0x646a41b9 // bfdot z25.s, z13.h, z2.h[1]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x646841d2 // bfdot z18.s, z14.h, z0.h[1]\n"
- ".inst 0x646941d6 // bfdot z22.s, z14.h, z1.h[1]\n"
- ".inst 0x646a41da // bfdot z26.s, z14.h, z2.h[1]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x646841f3 // bfdot z19.s, z15.h, z0.h[1]\n"
- ".inst 0x646941f7 // bfdot z23.s, z15.h, z1.h[1]\n"
- ".inst 0x646a41fb // bfdot z27.s, z15.h, z2.h[1]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x64704110 // bfdot z16.s, z8.h, z0.h[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- ".inst 0x64714114 // bfdot z20.s, z8.h, z1.h[2]\n"
- ".inst 0x64724118 // bfdot z24.s, z8.h, z2.h[2]\n"
- ".inst 0x64704131 // bfdot z17.s, z9.h, z0.h[2]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- ".inst 0x64714135 // bfdot z21.s, z9.h, z1.h[2]\n"
- ".inst 0x64724139 // bfdot z25.s, z9.h, z2.h[2]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- ".inst 0x64704152 // bfdot z18.s, z10.h, z0.h[2]\n"
- ".inst 0x64714156 // bfdot z22.s, z10.h, z1.h[2]\n"
- ".inst 0x6472415a // bfdot z26.s, z10.h, z2.h[2]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- ".inst 0x64704173 // bfdot z19.s, z11.h, z0.h[2]\n"
- ".inst 0x64714177 // bfdot z23.s, z11.h, z1.h[2]\n"
- ".inst 0x6472417b // bfdot z27.s, z11.h, z2.h[2]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- ".inst 0x64784190 // bfdot z16.s, z12.h, z0.h[3]\n"
- ".inst 0x64794194 // bfdot z20.s, z12.h, z1.h[3]\n"
- ".inst 0x647a4198 // bfdot z24.s, z12.h, z2.h[3]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- ".inst 0x647841b1 // bfdot z17.s, z13.h, z0.h[3]\n"
- ".inst 0x647941b5 // bfdot z21.s, z13.h, z1.h[3]\n"
- ".inst 0x647a41b9 // bfdot z25.s, z13.h, z2.h[3]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- ".inst 0x647841d2 // bfdot z18.s, z14.h, z0.h[3]\n"
- ".inst 0x647941d6 // bfdot z22.s, z14.h, z1.h[3]\n"
- ".inst 0x647a41da // bfdot z26.s, z14.h, z2.h[3]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- ".inst 0x647841f3 // bfdot z19.s, z15.h, z0.h[3]\n"
- "ld1rqh z0.h, p7/z, [%[a_ptr0], #-0x10]\n"
- ".inst 0x647941f7 // bfdot z23.s, z15.h, z1.h[3]\n"
- "ld1rqh z1.h, p7/z, [a_ptr1, #-0x10]\n"
- ".inst 0x647a41fb // bfdot z27.s, z15.h, z2.h[3]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- ".inst 0x64644110 // bfdot z16.s, z8.h, z4.h[0]\n"
- "ld1rqh z2.h, p7/z, [a_ptr2, #-0x10]\n"
- ".inst 0x64654114 // bfdot z20.s, z8.h, z5.h[0]\n"
- ".inst 0x64664118 // bfdot z24.s, z8.h, z6.h[0]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- ".inst 0x64644131 // bfdot z17.s, z9.h, z4.h[0]\n"
- ".inst 0x64654135 // bfdot z21.s, z9.h, z5.h[0]\n"
- ".inst 0x64664139 // bfdot z25.s, z9.h, z6.h[0]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x64644152 // bfdot z18.s, z10.h, z4.h[0]\n"
- ".inst 0x64654156 // bfdot z22.s, z10.h, z5.h[0]\n"
- ".inst 0x6466415a // bfdot z26.s, z10.h, z6.h[0]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x64644173 // bfdot z19.s, z11.h, z4.h[0]\n"
- ".inst 0x64654177 // bfdot z23.s, z11.h, z5.h[0]\n"
- ".inst 0x6466417b // bfdot z27.s, z11.h, z6.h[0]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x646c4190 // bfdot z16.s, z12.h, z4.h[1]\n"
- ".inst 0x646d4194 // bfdot z20.s, z12.h, z5.h[1]\n"
- ".inst 0x646e4198 // bfdot z24.s, z12.h, z6.h[1]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x646c41b1 // bfdot z17.s, z13.h, z4.h[1]\n"
- ".inst 0x646d41b5 // bfdot z21.s, z13.h, z5.h[1]\n"
- ".inst 0x646e41b9 // bfdot z25.s, z13.h, z6.h[1]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x646c41d2 // bfdot z18.s, z14.h, z4.h[1]\n"
- ".inst 0x646d41d6 // bfdot z22.s, z14.h, z5.h[1]\n"
- ".inst 0x646e41da // bfdot z26.s, z14.h, z6.h[1]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x646c41f3 // bfdot z19.s, z15.h, z4.h[1]\n"
- ".inst 0x646d41f7 // bfdot z23.s, z15.h, z5.h[1]\n"
- ".inst 0x646e41fb // bfdot z27.s, z15.h, z6.h[1]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x64744110 // bfdot z16.s, z8.h, z4.h[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- ".inst 0x64754114 // bfdot z20.s, z8.h, z5.h[2]\n"
- ".inst 0x64764118 // bfdot z24.s, z8.h, z6.h[2]\n"
- ".inst 0x64744131 // bfdot z17.s, z9.h, z4.h[2]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- ".inst 0x64754135 // bfdot z21.s, z9.h, z5.h[2]\n"
- ".inst 0x64764139 // bfdot z25.s, z9.h, z6.h[2]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- ".inst 0x64744152 // bfdot z18.s, z10.h, z4.h[2]\n"
- ".inst 0x64754156 // bfdot z22.s, z10.h, z5.h[2]\n"
- ".inst 0x6476415a // bfdot z26.s, z10.h, z6.h[2]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- ".inst 0x64744173 // bfdot z19.s, z11.h, z4.h[2]\n"
- ".inst 0x64754177 // bfdot z23.s, z11.h, z5.h[2]\n"
- ".inst 0x6476417b // bfdot z27.s, z11.h, z6.h[2]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- ".inst 0x647c4190 // bfdot z16.s, z12.h, z4.h[3]\n"
- ".inst 0x647d4194 // bfdot z20.s, z12.h, z5.h[3]\n"
- ".inst 0x647e4198 // bfdot z24.s, z12.h, z6.h[3]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- ".inst 0x647c41b1 // bfdot z17.s, z13.h, z4.h[3]\n"
- ".inst 0x647d41b5 // bfdot z21.s, z13.h, z5.h[3]\n"
- ".inst 0x647e41b9 // bfdot z25.s, z13.h, z6.h[3]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- ".inst 0x647c41d2 // bfdot z18.s, z14.h, z4.h[3]\n"
- ".inst 0x647d41d6 // bfdot z22.s, z14.h, z5.h[3]\n"
- ".inst 0x647e41da // bfdot z26.s, z14.h, z6.h[3]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- ".inst 0x647c41f3 // bfdot z19.s, z15.h, z4.h[3]\n"
- ".inst 0x647d41f7 // bfdot z23.s, z15.h, z5.h[3]\n"
- ".inst 0x647e41fb // bfdot z27.s, z15.h, z6.h[3]\n"
- "b.ne 3b\n"
- "2:\n"
- "cbz %[regs], 4f\n"
- ".inst 0x64604110 // bfdot z16.s, z8.h, z0.h[0]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- ".inst 0x64614114 // bfdot z20.s, z8.h, z1.h[0]\n"
- "ld1rqh z4.h, p7/z, [%[a_ptr0]]\n"
- ".inst 0x64624118 // bfdot z24.s, z8.h, z2.h[0]\n"
- "ld1rqh z5.h, p7/z, [a_ptr1]\n"
- ".inst 0x64604131 // bfdot z17.s, z9.h, z0.h[0]\n"
- "ld1rqh z6.h, p7/z, [a_ptr2]\n"
- ".inst 0x64614135 // bfdot z21.s, z9.h, z1.h[0]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- ".inst 0x64624139 // bfdot z25.s, z9.h, z2.h[0]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x64604152 // bfdot z18.s, z10.h, z0.h[0]\n"
- ".inst 0x64614156 // bfdot z22.s, z10.h, z1.h[0]\n"
- ".inst 0x6462415a // bfdot z26.s, z10.h, z2.h[0]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x64604173 // bfdot z19.s, z11.h, z0.h[0]\n"
- ".inst 0x64614177 // bfdot z23.s, z11.h, z1.h[0]\n"
- ".inst 0x6462417b // bfdot z27.s, z11.h, z2.h[0]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x64684190 // bfdot z16.s, z12.h, z0.h[1]\n"
- ".inst 0x64694194 // bfdot z20.s, z12.h, z1.h[1]\n"
- ".inst 0x646a4198 // bfdot z24.s, z12.h, z2.h[1]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x646841b1 // bfdot z17.s, z13.h, z0.h[1]\n"
- ".inst 0x646941b5 // bfdot z21.s, z13.h, z1.h[1]\n"
- ".inst 0x646a41b9 // bfdot z25.s, z13.h, z2.h[1]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x646841d2 // bfdot z18.s, z14.h, z0.h[1]\n"
- ".inst 0x646941d6 // bfdot z22.s, z14.h, z1.h[1]\n"
- ".inst 0x646a41da // bfdot z26.s, z14.h, z2.h[1]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x646841f3 // bfdot z19.s, z15.h, z0.h[1]\n"
- ".inst 0x646941f7 // bfdot z23.s, z15.h, z1.h[1]\n"
- ".inst 0x646a41fb // bfdot z27.s, z15.h, z2.h[1]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x64704110 // bfdot z16.s, z8.h, z0.h[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- ".inst 0x64714114 // bfdot z20.s, z8.h, z1.h[2]\n"
- ".inst 0x64724118 // bfdot z24.s, z8.h, z2.h[2]\n"
- ".inst 0x64704131 // bfdot z17.s, z9.h, z0.h[2]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- ".inst 0x64714135 // bfdot z21.s, z9.h, z1.h[2]\n"
- ".inst 0x64724139 // bfdot z25.s, z9.h, z2.h[2]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- ".inst 0x64704152 // bfdot z18.s, z10.h, z0.h[2]\n"
- ".inst 0x64714156 // bfdot z22.s, z10.h, z1.h[2]\n"
- ".inst 0x6472415a // bfdot z26.s, z10.h, z2.h[2]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- ".inst 0x64704173 // bfdot z19.s, z11.h, z0.h[2]\n"
- ".inst 0x64714177 // bfdot z23.s, z11.h, z1.h[2]\n"
- ".inst 0x6472417b // bfdot z27.s, z11.h, z2.h[2]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- ".inst 0x64784190 // bfdot z16.s, z12.h, z0.h[3]\n"
- ".inst 0x64794194 // bfdot z20.s, z12.h, z1.h[3]\n"
- ".inst 0x647a4198 // bfdot z24.s, z12.h, z2.h[3]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- ".inst 0x647841b1 // bfdot z17.s, z13.h, z0.h[3]\n"
- ".inst 0x647941b5 // bfdot z21.s, z13.h, z1.h[3]\n"
- ".inst 0x647a41b9 // bfdot z25.s, z13.h, z2.h[3]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- ".inst 0x647841d2 // bfdot z18.s, z14.h, z0.h[3]\n"
- ".inst 0x647941d6 // bfdot z22.s, z14.h, z1.h[3]\n"
- ".inst 0x647a41da // bfdot z26.s, z14.h, z2.h[3]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- ".inst 0x647841f3 // bfdot z19.s, z15.h, z0.h[3]\n"
- "ld1rqh z0.h, p6/z, [%[a_ptr0], #0x10]\n"
- ".inst 0x647941f7 // bfdot z23.s, z15.h, z1.h[3]\n"
- "ld1rqh z1.h, p6/z, [a_ptr1, #0x10]\n"
- ".inst 0x647a41fb // bfdot z27.s, z15.h, z2.h[3]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- ".inst 0x64644110 // bfdot z16.s, z8.h, z4.h[0]\n"
- "ld1rqh z2.h, p6/z, [a_ptr2, #0x10]\n"
- ".inst 0x64654114 // bfdot z20.s, z8.h, z5.h[0]\n"
- "addvl %[a_ptr0], %[a_ptr0], #2\n"
- ".inst 0x64664118 // bfdot z24.s, z8.h, z6.h[0]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- ".inst 0x64644131 // bfdot z17.s, z9.h, z4.h[0]\n"
- "addvl a_ptr1, a_ptr1, #2\n"
- ".inst 0x64654135 // bfdot z21.s, z9.h, z5.h[0]\n"
- "addvl a_ptr2, a_ptr2, #2\n"
- ".inst 0x64664139 // bfdot z25.s, z9.h, z6.h[0]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x64644152 // bfdot z18.s, z10.h, z4.h[0]\n"
- ".inst 0x64654156 // bfdot z22.s, z10.h, z5.h[0]\n"
- ".inst 0x6466415a // bfdot z26.s, z10.h, z6.h[0]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x64644173 // bfdot z19.s, z11.h, z4.h[0]\n"
- ".inst 0x64654177 // bfdot z23.s, z11.h, z5.h[0]\n"
- ".inst 0x6466417b // bfdot z27.s, z11.h, z6.h[0]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x646c4190 // bfdot z16.s, z12.h, z4.h[1]\n"
- ".inst 0x646d4194 // bfdot z20.s, z12.h, z5.h[1]\n"
- ".inst 0x646e4198 // bfdot z24.s, z12.h, z6.h[1]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x646c41b1 // bfdot z17.s, z13.h, z4.h[1]\n"
- ".inst 0x646d41b5 // bfdot z21.s, z13.h, z5.h[1]\n"
- ".inst 0x646e41b9 // bfdot z25.s, z13.h, z6.h[1]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x646c41d2 // bfdot z18.s, z14.h, z4.h[1]\n"
- ".inst 0x646d41d6 // bfdot z22.s, z14.h, z5.h[1]\n"
- ".inst 0x646e41da // bfdot z26.s, z14.h, z6.h[1]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x646c41f3 // bfdot z19.s, z15.h, z4.h[1]\n"
- ".inst 0x646d41f7 // bfdot z23.s, z15.h, z5.h[1]\n"
- ".inst 0x646e41fb // bfdot z27.s, z15.h, z6.h[1]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x64744110 // bfdot z16.s, z8.h, z4.h[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- ".inst 0x64754114 // bfdot z20.s, z8.h, z5.h[2]\n"
- ".inst 0x64764118 // bfdot z24.s, z8.h, z6.h[2]\n"
- ".inst 0x64744131 // bfdot z17.s, z9.h, z4.h[2]\n"
- ".inst 0x64754135 // bfdot z21.s, z9.h, z5.h[2]\n"
- ".inst 0x64764139 // bfdot z25.s, z9.h, z6.h[2]\n"
- ".inst 0x64744152 // bfdot z18.s, z10.h, z4.h[2]\n"
- ".inst 0x64754156 // bfdot z22.s, z10.h, z5.h[2]\n"
- ".inst 0x6476415a // bfdot z26.s, z10.h, z6.h[2]\n"
- ".inst 0x64744173 // bfdot z19.s, z11.h, z4.h[2]\n"
- ".inst 0x64754177 // bfdot z23.s, z11.h, z5.h[2]\n"
- ".inst 0x6476417b // bfdot z27.s, z11.h, z6.h[2]\n"
- ".inst 0x647c4190 // bfdot z16.s, z12.h, z4.h[3]\n"
- ".inst 0x647d4194 // bfdot z20.s, z12.h, z5.h[3]\n"
- ".inst 0x647e4198 // bfdot z24.s, z12.h, z6.h[3]\n"
- ".inst 0x647c41b1 // bfdot z17.s, z13.h, z4.h[3]\n"
- ".inst 0x647d41b5 // bfdot z21.s, z13.h, z5.h[3]\n"
- ".inst 0x647e41b9 // bfdot z25.s, z13.h, z6.h[3]\n"
- ".inst 0x647c41d2 // bfdot z18.s, z14.h, z4.h[3]\n"
- ".inst 0x647d41d6 // bfdot z22.s, z14.h, z5.h[3]\n"
- ".inst 0x647e41da // bfdot z26.s, z14.h, z6.h[3]\n"
- ".inst 0x647c41f3 // bfdot z19.s, z15.h, z4.h[3]\n"
- ".inst 0x647d41f7 // bfdot z23.s, z15.h, z5.h[3]\n"
- ".inst 0x647e41fb // bfdot z27.s, z15.h, z6.h[3]\n"
- "cbz %[blocks], 5f\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x64604110 // bfdot z16.s, z8.h, z0.h[0]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x64614114 // bfdot z20.s, z8.h, z1.h[0]\n"
- ".inst 0x64624118 // bfdot z24.s, z8.h, z2.h[0]\n"
- ".inst 0x64604131 // bfdot z17.s, z9.h, z0.h[0]\n"
- ".inst 0x64614135 // bfdot z21.s, z9.h, z1.h[0]\n"
- ".inst 0x64624139 // bfdot z25.s, z9.h, z2.h[0]\n"
- ".inst 0x64604152 // bfdot z18.s, z10.h, z0.h[0]\n"
- ".inst 0x64614156 // bfdot z22.s, z10.h, z1.h[0]\n"
- ".inst 0x6462415a // bfdot z26.s, z10.h, z2.h[0]\n"
- ".inst 0x64604173 // bfdot z19.s, z11.h, z0.h[0]\n"
- ".inst 0x64614177 // bfdot z23.s, z11.h, z1.h[0]\n"
- ".inst 0x6462417b // bfdot z27.s, z11.h, z2.h[0]\n"
- "b.eq 5f\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x64684190 // bfdot z16.s, z12.h, z0.h[1]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x64694194 // bfdot z20.s, z12.h, z1.h[1]\n"
- ".inst 0x646a4198 // bfdot z24.s, z12.h, z2.h[1]\n"
- ".inst 0x646841b1 // bfdot z17.s, z13.h, z0.h[1]\n"
- ".inst 0x646941b5 // bfdot z21.s, z13.h, z1.h[1]\n"
- ".inst 0x646a41b9 // bfdot z25.s, z13.h, z2.h[1]\n"
- ".inst 0x646841d2 // bfdot z18.s, z14.h, z0.h[1]\n"
- ".inst 0x646941d6 // bfdot z22.s, z14.h, z1.h[1]\n"
- ".inst 0x646a41da // bfdot z26.s, z14.h, z2.h[1]\n"
- ".inst 0x646841f3 // bfdot z19.s, z15.h, z0.h[1]\n"
- ".inst 0x646941f7 // bfdot z23.s, z15.h, z1.h[1]\n"
- ".inst 0x646a41fb // bfdot z27.s, z15.h, z2.h[1]\n"
- "b.eq 5f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- ".inst 0x64704110 // bfdot z16.s, z8.h, z0.h[2]\n"
- ".inst 0x64714114 // bfdot z20.s, z8.h, z1.h[2]\n"
- ".inst 0x64724118 // bfdot z24.s, z8.h, z2.h[2]\n"
- ".inst 0x64704131 // bfdot z17.s, z9.h, z0.h[2]\n"
- ".inst 0x64714135 // bfdot z21.s, z9.h, z1.h[2]\n"
- ".inst 0x64724139 // bfdot z25.s, z9.h, z2.h[2]\n"
- ".inst 0x64704152 // bfdot z18.s, z10.h, z0.h[2]\n"
- ".inst 0x64714156 // bfdot z22.s, z10.h, z1.h[2]\n"
- ".inst 0x6472415a // bfdot z26.s, z10.h, z2.h[2]\n"
- ".inst 0x64704173 // bfdot z19.s, z11.h, z0.h[2]\n"
- ".inst 0x64714177 // bfdot z23.s, z11.h, z1.h[2]\n"
- ".inst 0x6472417b // bfdot z27.s, z11.h, z2.h[2]\n"
- "b.eq 5f\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- ".inst 0x64784190 // bfdot z16.s, z12.h, z0.h[3]\n"
- ".inst 0x64794194 // bfdot z20.s, z12.h, z1.h[3]\n"
- ".inst 0x647a4198 // bfdot z24.s, z12.h, z2.h[3]\n"
- ".inst 0x647841b1 // bfdot z17.s, z13.h, z0.h[3]\n"
- ".inst 0x647941b5 // bfdot z21.s, z13.h, z1.h[3]\n"
- ".inst 0x647a41b9 // bfdot z25.s, z13.h, z2.h[3]\n"
- ".inst 0x647841d2 // bfdot z18.s, z14.h, z0.h[3]\n"
- ".inst 0x647941d6 // bfdot z22.s, z14.h, z1.h[3]\n"
- ".inst 0x647a41da // bfdot z26.s, z14.h, z2.h[3]\n"
- ".inst 0x647841f3 // bfdot z19.s, z15.h, z0.h[3]\n"
- ".inst 0x647941f7 // bfdot z23.s, z15.h, z1.h[3]\n"
- ".inst 0x647a41fb // bfdot z27.s, z15.h, z2.h[3]\n"
- "b 5f\n"
- "4:\n"
- ".inst 0x64604110 // bfdot z16.s, z8.h, z0.h[0]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- ".inst 0x64614114 // bfdot z20.s, z8.h, z1.h[0]\n"
- "ld1rqh z4.h, p6/z, [%[a_ptr0]]\n"
- ".inst 0x64624118 // bfdot z24.s, z8.h, z2.h[0]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- ".inst 0x64604131 // bfdot z17.s, z9.h, z0.h[0]\n"
- "ld1rqh z5.h, p6/z, [a_ptr1]\n"
- ".inst 0x64614135 // bfdot z21.s, z9.h, z1.h[0]\n"
- "ld1rqh z6.h, p6/z, [a_ptr2]\n"
- ".inst 0x64624139 // bfdot z25.s, z9.h, z2.h[0]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x64604152 // bfdot z18.s, z10.h, z0.h[0]\n"
- "addvl %[a_ptr0], %[a_ptr0], #1\n"
- ".inst 0x64614156 // bfdot z22.s, z10.h, z1.h[0]\n"
- "addvl a_ptr1, a_ptr1, #1\n"
- ".inst 0x6462415a // bfdot z26.s, z10.h, z2.h[0]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x64604173 // bfdot z19.s, z11.h, z0.h[0]\n"
- "addvl a_ptr2, a_ptr2, #1\n"
- ".inst 0x64614177 // bfdot z23.s, z11.h, z1.h[0]\n"
- ".inst 0x6462417b // bfdot z27.s, z11.h, z2.h[0]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x64684190 // bfdot z16.s, z12.h, z0.h[1]\n"
- ".inst 0x64694194 // bfdot z20.s, z12.h, z1.h[1]\n"
- ".inst 0x646a4198 // bfdot z24.s, z12.h, z2.h[1]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x646841b1 // bfdot z17.s, z13.h, z0.h[1]\n"
- ".inst 0x646941b5 // bfdot z21.s, z13.h, z1.h[1]\n"
- ".inst 0x646a41b9 // bfdot z25.s, z13.h, z2.h[1]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x646841d2 // bfdot z18.s, z14.h, z0.h[1]\n"
- ".inst 0x646941d6 // bfdot z22.s, z14.h, z1.h[1]\n"
- ".inst 0x646a41da // bfdot z26.s, z14.h, z2.h[1]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x646841f3 // bfdot z19.s, z15.h, z0.h[1]\n"
- ".inst 0x646941f7 // bfdot z23.s, z15.h, z1.h[1]\n"
- ".inst 0x646a41fb // bfdot z27.s, z15.h, z2.h[1]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x64704110 // bfdot z16.s, z8.h, z0.h[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- ".inst 0x64714114 // bfdot z20.s, z8.h, z1.h[2]\n"
- ".inst 0x64724118 // bfdot z24.s, z8.h, z2.h[2]\n"
- ".inst 0x64704131 // bfdot z17.s, z9.h, z0.h[2]\n"
- ".inst 0x64714135 // bfdot z21.s, z9.h, z1.h[2]\n"
- ".inst 0x64724139 // bfdot z25.s, z9.h, z2.h[2]\n"
- ".inst 0x64704152 // bfdot z18.s, z10.h, z0.h[2]\n"
- ".inst 0x64714156 // bfdot z22.s, z10.h, z1.h[2]\n"
- ".inst 0x6472415a // bfdot z26.s, z10.h, z2.h[2]\n"
- ".inst 0x64704173 // bfdot z19.s, z11.h, z0.h[2]\n"
- ".inst 0x64714177 // bfdot z23.s, z11.h, z1.h[2]\n"
- ".inst 0x6472417b // bfdot z27.s, z11.h, z2.h[2]\n"
- ".inst 0x64784190 // bfdot z16.s, z12.h, z0.h[3]\n"
- ".inst 0x64794194 // bfdot z20.s, z12.h, z1.h[3]\n"
- ".inst 0x647a4198 // bfdot z24.s, z12.h, z2.h[3]\n"
- ".inst 0x647841b1 // bfdot z17.s, z13.h, z0.h[3]\n"
- ".inst 0x647941b5 // bfdot z21.s, z13.h, z1.h[3]\n"
- ".inst 0x647a41b9 // bfdot z25.s, z13.h, z2.h[3]\n"
- ".inst 0x647841d2 // bfdot z18.s, z14.h, z0.h[3]\n"
- ".inst 0x647941d6 // bfdot z22.s, z14.h, z1.h[3]\n"
- ".inst 0x647a41da // bfdot z26.s, z14.h, z2.h[3]\n"
- ".inst 0x647841f3 // bfdot z19.s, z15.h, z0.h[3]\n"
- ".inst 0x647941f7 // bfdot z23.s, z15.h, z1.h[3]\n"
- ".inst 0x647a41fb // bfdot z27.s, z15.h, z2.h[3]\n"
- "cbz %[blocks], 5f\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x64644110 // bfdot z16.s, z8.h, z4.h[0]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x64654114 // bfdot z20.s, z8.h, z5.h[0]\n"
- ".inst 0x64664118 // bfdot z24.s, z8.h, z6.h[0]\n"
- ".inst 0x64644131 // bfdot z17.s, z9.h, z4.h[0]\n"
- ".inst 0x64654135 // bfdot z21.s, z9.h, z5.h[0]\n"
- ".inst 0x64664139 // bfdot z25.s, z9.h, z6.h[0]\n"
- ".inst 0x64644152 // bfdot z18.s, z10.h, z4.h[0]\n"
- ".inst 0x64654156 // bfdot z22.s, z10.h, z5.h[0]\n"
- ".inst 0x6466415a // bfdot z26.s, z10.h, z6.h[0]\n"
- ".inst 0x64644173 // bfdot z19.s, z11.h, z4.h[0]\n"
- ".inst 0x64654177 // bfdot z23.s, z11.h, z5.h[0]\n"
- ".inst 0x6466417b // bfdot z27.s, z11.h, z6.h[0]\n"
- "b.eq 5f\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x646c4190 // bfdot z16.s, z12.h, z4.h[1]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x646d4194 // bfdot z20.s, z12.h, z5.h[1]\n"
- ".inst 0x646e4198 // bfdot z24.s, z12.h, z6.h[1]\n"
- ".inst 0x646c41b1 // bfdot z17.s, z13.h, z4.h[1]\n"
- ".inst 0x646d41b5 // bfdot z21.s, z13.h, z5.h[1]\n"
- ".inst 0x646e41b9 // bfdot z25.s, z13.h, z6.h[1]\n"
- ".inst 0x646c41d2 // bfdot z18.s, z14.h, z4.h[1]\n"
- ".inst 0x646d41d6 // bfdot z22.s, z14.h, z5.h[1]\n"
- ".inst 0x646e41da // bfdot z26.s, z14.h, z6.h[1]\n"
- ".inst 0x646c41f3 // bfdot z19.s, z15.h, z4.h[1]\n"
- ".inst 0x646d41f7 // bfdot z23.s, z15.h, z5.h[1]\n"
- ".inst 0x646e41fb // bfdot z27.s, z15.h, z6.h[1]\n"
- "b.eq 5f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- ".inst 0x64744110 // bfdot z16.s, z8.h, z4.h[2]\n"
- ".inst 0x64754114 // bfdot z20.s, z8.h, z5.h[2]\n"
- ".inst 0x64764118 // bfdot z24.s, z8.h, z6.h[2]\n"
- ".inst 0x64744131 // bfdot z17.s, z9.h, z4.h[2]\n"
- ".inst 0x64754135 // bfdot z21.s, z9.h, z5.h[2]\n"
- ".inst 0x64764139 // bfdot z25.s, z9.h, z6.h[2]\n"
- ".inst 0x64744152 // bfdot z18.s, z10.h, z4.h[2]\n"
- ".inst 0x64754156 // bfdot z22.s, z10.h, z5.h[2]\n"
- ".inst 0x6476415a // bfdot z26.s, z10.h, z6.h[2]\n"
- ".inst 0x64744173 // bfdot z19.s, z11.h, z4.h[2]\n"
- ".inst 0x64754177 // bfdot z23.s, z11.h, z5.h[2]\n"
- ".inst 0x6476417b // bfdot z27.s, z11.h, z6.h[2]\n"
- "b.eq 5f\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- ".inst 0x647c4190 // bfdot z16.s, z12.h, z4.h[3]\n"
- ".inst 0x647d4194 // bfdot z20.s, z12.h, z5.h[3]\n"
- ".inst 0x647e4198 // bfdot z24.s, z12.h, z6.h[3]\n"
- ".inst 0x647c41b1 // bfdot z17.s, z13.h, z4.h[3]\n"
- ".inst 0x647d41b5 // bfdot z21.s, z13.h, z5.h[3]\n"
- ".inst 0x647e41b9 // bfdot z25.s, z13.h, z6.h[3]\n"
- ".inst 0x647c41d2 // bfdot z18.s, z14.h, z4.h[3]\n"
- ".inst 0x647d41d6 // bfdot z22.s, z14.h, z5.h[3]\n"
- ".inst 0x647e41da // bfdot z26.s, z14.h, z6.h[3]\n"
- ".inst 0x647c41f3 // bfdot z19.s, z15.h, z4.h[3]\n"
- ".inst 0x647d41f7 // bfdot z23.s, z15.h, z5.h[3]\n"
- ".inst 0x647e41fb // bfdot z27.s, z15.h, z6.h[3]\n"
- "5:\n"
- "ld1rw z14.s, p7/z, [%[minptr]]\n"
- "ld1rw z15.s, p7/z, [%[maxptr]]\n"
- "fmax z16.s, p7/m, z16.s, z14.s\n"
- "fmax z17.s, p7/m, z17.s, z14.s\n"
- "fmax z18.s, p7/m, z18.s, z14.s\n"
- "fmax z19.s, p7/m, z19.s, z14.s\n"
- "fmin z16.s, p7/m, z16.s, z15.s\n"
- "fmin z17.s, p7/m, z17.s, z15.s\n"
- "fmin z18.s, p7/m, z18.s, z15.s\n"
- "fmin z19.s, p7/m, z19.s, z15.s\n"
- "st1w z16.s, p0, [%[c_ptr0]]\n"
- "fmax z20.s, p7/m, z20.s, z14.s\n"
- "fmax z21.s, p7/m, z21.s, z14.s\n"
- "fmax z22.s, p7/m, z22.s, z14.s\n"
- "st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n"
- "fmax z23.s, p7/m, z23.s, z14.s\n"
- "fmin z20.s, p7/m, z20.s, z15.s\n"
- "fmin z21.s, p7/m, z21.s, z15.s\n"
- "st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n"
- "fmin z22.s, p7/m, z22.s, z15.s\n"
- "fmin z23.s, p7/m, z23.s, z15.s\n"
- "fmax z24.s, p7/m, z24.s, z14.s\n"
- "st1w z19.s, p3, [%[c_ptr0], #3, MUL VL]\n"
- "fmax z25.s, p7/m, z25.s, z14.s\n"
- "addvl %[c_ptr0], %[c_ptr0], #4\n"
- "fmax z26.s, p7/m, z26.s, z14.s\n"
- "st1w z20.s, p0, [c_ptr1]\n"
- "fmin z24.s, p7/m, z24.s, z15.s\n"
- "fmin z25.s, p7/m, z25.s, z15.s\n"
- "fmax z27.s, p7/m, z27.s, z14.s\n"
- "st1w z21.s, p1, [c_ptr1, #1, MUL VL]\n"
- "fmin z26.s, p7/m, z26.s, z15.s\n"
- "fmin z27.s, p7/m, z27.s, z15.s\n"
- "st1w z22.s, p2, [c_ptr1, #2, MUL VL]\n"
- "st1w z23.s, p3, [c_ptr1, #3, MUL VL]\n"
- "st1w z24.s, p0, [c_ptr2]\n"
- "st1w z25.s, p1, [c_ptr2, #1, MUL VL]\n"
- "st1w z26.s, p2, [c_ptr2, #2, MUL VL]\n"
- "st1w z27.s, p3, [c_ptr2, #3, MUL VL]\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks)
- : [width] "r" (width), [accumulate] "r" (static_cast<uint64_t>(accumulate)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers)
- : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "cc", "memory"
- );
- break;
- default:
- case 4:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "a_ptr3 .req X2\n"
- "c_ptr1 .req X3\n"
- "c_ptr2 .req X4\n"
- "c_ptr3 .req X5\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "add a_ptr3, a_ptr2, %[lda]\n"
- "add c_ptr3, c_ptr2, %[ldc]\n"
- "whilelt p6.h, %[temp], %[leftovers]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "ptrue p7.h\n"
- "whilelt p1.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "whilelt p2.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "whilelt p3.s, %[temp], %[width]\n"
- "cbnz %[accumulate], 1f\n"
- "ld1w z16.s, p0/z, [%[biasptr]]\n"
- "ld1w z17.s, p1/z, [%[biasptr], #1, MUL VL]\n"
- "ld1w z18.s, p2/z, [%[biasptr], #2, MUL VL]\n"
- "ld1w z19.s, p3/z, [%[biasptr], #3, MUL VL]\n"
- "mov z20.d, z16.d\n"
- "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n"
- "mov z21.d, z17.d\n"
- "ld1rqh z1.h, p7/z, [a_ptr1]\n"
- "mov z22.d, z18.d\n"
- "ld1rqh z2.h, p7/z, [a_ptr2]\n"
- "mov z23.d, z19.d\n"
- "ld1rqh z3.h, p7/z, [a_ptr3]\n"
- "mov z24.d, z16.d\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "mov z25.d, z17.d\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "mov z26.d, z18.d\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "mov z27.d, z19.d\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "mov z28.d, z16.d\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "mov z29.d, z17.d\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "mov z30.d, z18.d\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "mov z31.d, z19.d\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "add a_ptr3, a_ptr3, #0x10\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "ld1w z16.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n"
- "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n"
- "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n"
- "ld1w z20.s, p0/z, [c_ptr1]\n"
- "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n"
- "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n"
- "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n"
- "ld1w z24.s, p0/z, [c_ptr2]\n"
- "ld1w z25.s, p1/z, [c_ptr2, #1, MUL VL]\n"
- "ld1w z26.s, p2/z, [c_ptr2, #2, MUL VL]\n"
- "ld1w z27.s, p3/z, [c_ptr2, #3, MUL VL]\n"
- "ld1w z28.s, p0/z, [c_ptr3]\n"
- "ld1w z29.s, p1/z, [c_ptr3, #1, MUL VL]\n"
- "ld1w z30.s, p2/z, [c_ptr3, #2, MUL VL]\n"
- "ld1w z31.s, p3/z, [c_ptr3, #3, MUL VL]\n"
- "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ld1rqh z1.h, p7/z, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "ld1rqh z2.h, p7/z, [a_ptr2]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "ld1rqh z3.h, p7/z, [a_ptr3]\n"
- "add a_ptr3, a_ptr3, #0x10\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "cbz %[loops], 2f\n"
- "3:\n"
- ".inst 0x64604110 // bfdot z16.s, z8.h, z0.h[0]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- ".inst 0x64614114 // bfdot z20.s, z8.h, z1.h[0]\n"
- "ld1rqh z4.h, p7/z, [%[a_ptr0]]\n"
- ".inst 0x64624118 // bfdot z24.s, z8.h, z2.h[0]\n"
- "ld1rqh z5.h, p7/z, [a_ptr1]\n"
- ".inst 0x6463411c // bfdot z28.s, z8.h, z3.h[0]\n"
- "ld1rqh z6.h, p7/z, [a_ptr2]\n"
- ".inst 0x64604131 // bfdot z17.s, z9.h, z0.h[0]\n"
- "ld1rqh z7.h, p7/z, [a_ptr3]\n"
- ".inst 0x64614135 // bfdot z21.s, z9.h, z1.h[0]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- ".inst 0x64624139 // bfdot z25.s, z9.h, z2.h[0]\n"
- "subs %[loops], %[loops], #0x1\n"
- ".inst 0x6463413d // bfdot z29.s, z9.h, z3.h[0]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x64604152 // bfdot z18.s, z10.h, z0.h[0]\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
- ".inst 0x64614156 // bfdot z22.s, z10.h, z1.h[0]\n"
- "add a_ptr1, a_ptr1, #0x20\n"
- ".inst 0x6462415a // bfdot z26.s, z10.h, z2.h[0]\n"
- "add a_ptr2, a_ptr2, #0x20\n"
- ".inst 0x6463415e // bfdot z30.s, z10.h, z3.h[0]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x64604173 // bfdot z19.s, z11.h, z0.h[0]\n"
- "add a_ptr3, a_ptr3, #0x20\n"
- ".inst 0x64614177 // bfdot z23.s, z11.h, z1.h[0]\n"
- ".inst 0x6462417b // bfdot z27.s, z11.h, z2.h[0]\n"
- ".inst 0x6463417f // bfdot z31.s, z11.h, z3.h[0]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x64684190 // bfdot z16.s, z12.h, z0.h[1]\n"
- ".inst 0x64694194 // bfdot z20.s, z12.h, z1.h[1]\n"
- ".inst 0x646a4198 // bfdot z24.s, z12.h, z2.h[1]\n"
- ".inst 0x646b419c // bfdot z28.s, z12.h, z3.h[1]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x646841b1 // bfdot z17.s, z13.h, z0.h[1]\n"
- ".inst 0x646941b5 // bfdot z21.s, z13.h, z1.h[1]\n"
- ".inst 0x646a41b9 // bfdot z25.s, z13.h, z2.h[1]\n"
- ".inst 0x646b41bd // bfdot z29.s, z13.h, z3.h[1]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x646841d2 // bfdot z18.s, z14.h, z0.h[1]\n"
- ".inst 0x646941d6 // bfdot z22.s, z14.h, z1.h[1]\n"
- ".inst 0x646a41da // bfdot z26.s, z14.h, z2.h[1]\n"
- ".inst 0x646b41de // bfdot z30.s, z14.h, z3.h[1]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x646841f3 // bfdot z19.s, z15.h, z0.h[1]\n"
- ".inst 0x646941f7 // bfdot z23.s, z15.h, z1.h[1]\n"
- ".inst 0x646a41fb // bfdot z27.s, z15.h, z2.h[1]\n"
- ".inst 0x646b41ff // bfdot z31.s, z15.h, z3.h[1]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x64704110 // bfdot z16.s, z8.h, z0.h[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- ".inst 0x64714114 // bfdot z20.s, z8.h, z1.h[2]\n"
- ".inst 0x64724118 // bfdot z24.s, z8.h, z2.h[2]\n"
- ".inst 0x6473411c // bfdot z28.s, z8.h, z3.h[2]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- ".inst 0x64704131 // bfdot z17.s, z9.h, z0.h[2]\n"
- ".inst 0x64714135 // bfdot z21.s, z9.h, z1.h[2]\n"
- ".inst 0x64724139 // bfdot z25.s, z9.h, z2.h[2]\n"
- ".inst 0x6473413d // bfdot z29.s, z9.h, z3.h[2]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- ".inst 0x64704152 // bfdot z18.s, z10.h, z0.h[2]\n"
- ".inst 0x64714156 // bfdot z22.s, z10.h, z1.h[2]\n"
- ".inst 0x6472415a // bfdot z26.s, z10.h, z2.h[2]\n"
- ".inst 0x6473415e // bfdot z30.s, z10.h, z3.h[2]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- ".inst 0x64704173 // bfdot z19.s, z11.h, z0.h[2]\n"
- ".inst 0x64714177 // bfdot z23.s, z11.h, z1.h[2]\n"
- ".inst 0x6472417b // bfdot z27.s, z11.h, z2.h[2]\n"
- ".inst 0x6473417f // bfdot z31.s, z11.h, z3.h[2]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- ".inst 0x64784190 // bfdot z16.s, z12.h, z0.h[3]\n"
- ".inst 0x64794194 // bfdot z20.s, z12.h, z1.h[3]\n"
- ".inst 0x647a4198 // bfdot z24.s, z12.h, z2.h[3]\n"
- ".inst 0x647b419c // bfdot z28.s, z12.h, z3.h[3]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- ".inst 0x647841b1 // bfdot z17.s, z13.h, z0.h[3]\n"
- ".inst 0x647941b5 // bfdot z21.s, z13.h, z1.h[3]\n"
- ".inst 0x647a41b9 // bfdot z25.s, z13.h, z2.h[3]\n"
- ".inst 0x647b41bd // bfdot z29.s, z13.h, z3.h[3]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- ".inst 0x647841d2 // bfdot z18.s, z14.h, z0.h[3]\n"
- ".inst 0x647941d6 // bfdot z22.s, z14.h, z1.h[3]\n"
- ".inst 0x647a41da // bfdot z26.s, z14.h, z2.h[3]\n"
- ".inst 0x647b41de // bfdot z30.s, z14.h, z3.h[3]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- ".inst 0x647841f3 // bfdot z19.s, z15.h, z0.h[3]\n"
- "ld1rqh z0.h, p7/z, [%[a_ptr0], #-0x10]\n"
- ".inst 0x647941f7 // bfdot z23.s, z15.h, z1.h[3]\n"
- "ld1rqh z1.h, p7/z, [a_ptr1, #-0x10]\n"
- ".inst 0x647a41fb // bfdot z27.s, z15.h, z2.h[3]\n"
- "ld1rqh z2.h, p7/z, [a_ptr2, #-0x10]\n"
- ".inst 0x647b41ff // bfdot z31.s, z15.h, z3.h[3]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- ".inst 0x64644110 // bfdot z16.s, z8.h, z4.h[0]\n"
- "ld1rqh z3.h, p7/z, [a_ptr3, #-0x10]\n"
- ".inst 0x64654114 // bfdot z20.s, z8.h, z5.h[0]\n"
- ".inst 0x64664118 // bfdot z24.s, z8.h, z6.h[0]\n"
- ".inst 0x6467411c // bfdot z28.s, z8.h, z7.h[0]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- ".inst 0x64644131 // bfdot z17.s, z9.h, z4.h[0]\n"
- ".inst 0x64654135 // bfdot z21.s, z9.h, z5.h[0]\n"
- ".inst 0x64664139 // bfdot z25.s, z9.h, z6.h[0]\n"
- ".inst 0x6467413d // bfdot z29.s, z9.h, z7.h[0]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x64644152 // bfdot z18.s, z10.h, z4.h[0]\n"
- ".inst 0x64654156 // bfdot z22.s, z10.h, z5.h[0]\n"
- ".inst 0x6466415a // bfdot z26.s, z10.h, z6.h[0]\n"
- ".inst 0x6467415e // bfdot z30.s, z10.h, z7.h[0]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x64644173 // bfdot z19.s, z11.h, z4.h[0]\n"
- ".inst 0x64654177 // bfdot z23.s, z11.h, z5.h[0]\n"
- ".inst 0x6466417b // bfdot z27.s, z11.h, z6.h[0]\n"
- ".inst 0x6467417f // bfdot z31.s, z11.h, z7.h[0]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x646c4190 // bfdot z16.s, z12.h, z4.h[1]\n"
- ".inst 0x646d4194 // bfdot z20.s, z12.h, z5.h[1]\n"
- ".inst 0x646e4198 // bfdot z24.s, z12.h, z6.h[1]\n"
- ".inst 0x646f419c // bfdot z28.s, z12.h, z7.h[1]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x646c41b1 // bfdot z17.s, z13.h, z4.h[1]\n"
- ".inst 0x646d41b5 // bfdot z21.s, z13.h, z5.h[1]\n"
- ".inst 0x646e41b9 // bfdot z25.s, z13.h, z6.h[1]\n"
- ".inst 0x646f41bd // bfdot z29.s, z13.h, z7.h[1]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x646c41d2 // bfdot z18.s, z14.h, z4.h[1]\n"
- ".inst 0x646d41d6 // bfdot z22.s, z14.h, z5.h[1]\n"
- ".inst 0x646e41da // bfdot z26.s, z14.h, z6.h[1]\n"
- ".inst 0x646f41de // bfdot z30.s, z14.h, z7.h[1]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x646c41f3 // bfdot z19.s, z15.h, z4.h[1]\n"
- ".inst 0x646d41f7 // bfdot z23.s, z15.h, z5.h[1]\n"
- ".inst 0x646e41fb // bfdot z27.s, z15.h, z6.h[1]\n"
- ".inst 0x646f41ff // bfdot z31.s, z15.h, z7.h[1]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x64744110 // bfdot z16.s, z8.h, z4.h[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- ".inst 0x64754114 // bfdot z20.s, z8.h, z5.h[2]\n"
- ".inst 0x64764118 // bfdot z24.s, z8.h, z6.h[2]\n"
- ".inst 0x6477411c // bfdot z28.s, z8.h, z7.h[2]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- ".inst 0x64744131 // bfdot z17.s, z9.h, z4.h[2]\n"
- ".inst 0x64754135 // bfdot z21.s, z9.h, z5.h[2]\n"
- ".inst 0x64764139 // bfdot z25.s, z9.h, z6.h[2]\n"
- ".inst 0x6477413d // bfdot z29.s, z9.h, z7.h[2]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- ".inst 0x64744152 // bfdot z18.s, z10.h, z4.h[2]\n"
- ".inst 0x64754156 // bfdot z22.s, z10.h, z5.h[2]\n"
- ".inst 0x6476415a // bfdot z26.s, z10.h, z6.h[2]\n"
- ".inst 0x6477415e // bfdot z30.s, z10.h, z7.h[2]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- ".inst 0x64744173 // bfdot z19.s, z11.h, z4.h[2]\n"
- ".inst 0x64754177 // bfdot z23.s, z11.h, z5.h[2]\n"
- ".inst 0x6476417b // bfdot z27.s, z11.h, z6.h[2]\n"
- ".inst 0x6477417f // bfdot z31.s, z11.h, z7.h[2]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- ".inst 0x647c4190 // bfdot z16.s, z12.h, z4.h[3]\n"
- ".inst 0x647d4194 // bfdot z20.s, z12.h, z5.h[3]\n"
- ".inst 0x647e4198 // bfdot z24.s, z12.h, z6.h[3]\n"
- ".inst 0x647f419c // bfdot z28.s, z12.h, z7.h[3]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- ".inst 0x647c41b1 // bfdot z17.s, z13.h, z4.h[3]\n"
- ".inst 0x647d41b5 // bfdot z21.s, z13.h, z5.h[3]\n"
- ".inst 0x647e41b9 // bfdot z25.s, z13.h, z6.h[3]\n"
- ".inst 0x647f41bd // bfdot z29.s, z13.h, z7.h[3]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- ".inst 0x647c41d2 // bfdot z18.s, z14.h, z4.h[3]\n"
- ".inst 0x647d41d6 // bfdot z22.s, z14.h, z5.h[3]\n"
- ".inst 0x647e41da // bfdot z26.s, z14.h, z6.h[3]\n"
- ".inst 0x647f41de // bfdot z30.s, z14.h, z7.h[3]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- ".inst 0x647c41f3 // bfdot z19.s, z15.h, z4.h[3]\n"
- ".inst 0x647d41f7 // bfdot z23.s, z15.h, z5.h[3]\n"
- ".inst 0x647e41fb // bfdot z27.s, z15.h, z6.h[3]\n"
- ".inst 0x647f41ff // bfdot z31.s, z15.h, z7.h[3]\n"
- "b.ne 3b\n"
- "2:\n"
- "cbz %[regs], 4f\n"
- ".inst 0x64604110 // bfdot z16.s, z8.h, z0.h[0]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- ".inst 0x64614114 // bfdot z20.s, z8.h, z1.h[0]\n"
- "ld1rqh z4.h, p7/z, [%[a_ptr0]]\n"
- ".inst 0x64624118 // bfdot z24.s, z8.h, z2.h[0]\n"
- "ld1rqh z5.h, p7/z, [a_ptr1]\n"
- ".inst 0x6463411c // bfdot z28.s, z8.h, z3.h[0]\n"
- "ld1rqh z6.h, p7/z, [a_ptr2]\n"
- ".inst 0x64604131 // bfdot z17.s, z9.h, z0.h[0]\n"
- "ld1rqh z7.h, p7/z, [a_ptr3]\n"
- ".inst 0x64614135 // bfdot z21.s, z9.h, z1.h[0]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- ".inst 0x64624139 // bfdot z25.s, z9.h, z2.h[0]\n"
- ".inst 0x6463413d // bfdot z29.s, z9.h, z3.h[0]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x64604152 // bfdot z18.s, z10.h, z0.h[0]\n"
- ".inst 0x64614156 // bfdot z22.s, z10.h, z1.h[0]\n"
- ".inst 0x6462415a // bfdot z26.s, z10.h, z2.h[0]\n"
- ".inst 0x6463415e // bfdot z30.s, z10.h, z3.h[0]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x64604173 // bfdot z19.s, z11.h, z0.h[0]\n"
- ".inst 0x64614177 // bfdot z23.s, z11.h, z1.h[0]\n"
- ".inst 0x6462417b // bfdot z27.s, z11.h, z2.h[0]\n"
- ".inst 0x6463417f // bfdot z31.s, z11.h, z3.h[0]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x64684190 // bfdot z16.s, z12.h, z0.h[1]\n"
- ".inst 0x64694194 // bfdot z20.s, z12.h, z1.h[1]\n"
- ".inst 0x646a4198 // bfdot z24.s, z12.h, z2.h[1]\n"
- ".inst 0x646b419c // bfdot z28.s, z12.h, z3.h[1]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x646841b1 // bfdot z17.s, z13.h, z0.h[1]\n"
- ".inst 0x646941b5 // bfdot z21.s, z13.h, z1.h[1]\n"
- ".inst 0x646a41b9 // bfdot z25.s, z13.h, z2.h[1]\n"
- ".inst 0x646b41bd // bfdot z29.s, z13.h, z3.h[1]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x646841d2 // bfdot z18.s, z14.h, z0.h[1]\n"
- ".inst 0x646941d6 // bfdot z22.s, z14.h, z1.h[1]\n"
- ".inst 0x646a41da // bfdot z26.s, z14.h, z2.h[1]\n"
- ".inst 0x646b41de // bfdot z30.s, z14.h, z3.h[1]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x646841f3 // bfdot z19.s, z15.h, z0.h[1]\n"
- ".inst 0x646941f7 // bfdot z23.s, z15.h, z1.h[1]\n"
- ".inst 0x646a41fb // bfdot z27.s, z15.h, z2.h[1]\n"
- ".inst 0x646b41ff // bfdot z31.s, z15.h, z3.h[1]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x64704110 // bfdot z16.s, z8.h, z0.h[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- ".inst 0x64714114 // bfdot z20.s, z8.h, z1.h[2]\n"
- ".inst 0x64724118 // bfdot z24.s, z8.h, z2.h[2]\n"
- ".inst 0x6473411c // bfdot z28.s, z8.h, z3.h[2]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- ".inst 0x64704131 // bfdot z17.s, z9.h, z0.h[2]\n"
- ".inst 0x64714135 // bfdot z21.s, z9.h, z1.h[2]\n"
- ".inst 0x64724139 // bfdot z25.s, z9.h, z2.h[2]\n"
- ".inst 0x6473413d // bfdot z29.s, z9.h, z3.h[2]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- ".inst 0x64704152 // bfdot z18.s, z10.h, z0.h[2]\n"
- ".inst 0x64714156 // bfdot z22.s, z10.h, z1.h[2]\n"
- ".inst 0x6472415a // bfdot z26.s, z10.h, z2.h[2]\n"
- ".inst 0x6473415e // bfdot z30.s, z10.h, z3.h[2]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- ".inst 0x64704173 // bfdot z19.s, z11.h, z0.h[2]\n"
- ".inst 0x64714177 // bfdot z23.s, z11.h, z1.h[2]\n"
- ".inst 0x6472417b // bfdot z27.s, z11.h, z2.h[2]\n"
- ".inst 0x6473417f // bfdot z31.s, z11.h, z3.h[2]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- ".inst 0x64784190 // bfdot z16.s, z12.h, z0.h[3]\n"
- ".inst 0x64794194 // bfdot z20.s, z12.h, z1.h[3]\n"
- ".inst 0x647a4198 // bfdot z24.s, z12.h, z2.h[3]\n"
- ".inst 0x647b419c // bfdot z28.s, z12.h, z3.h[3]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- ".inst 0x647841b1 // bfdot z17.s, z13.h, z0.h[3]\n"
- ".inst 0x647941b5 // bfdot z21.s, z13.h, z1.h[3]\n"
- ".inst 0x647a41b9 // bfdot z25.s, z13.h, z2.h[3]\n"
- ".inst 0x647b41bd // bfdot z29.s, z13.h, z3.h[3]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- ".inst 0x647841d2 // bfdot z18.s, z14.h, z0.h[3]\n"
- ".inst 0x647941d6 // bfdot z22.s, z14.h, z1.h[3]\n"
- ".inst 0x647a41da // bfdot z26.s, z14.h, z2.h[3]\n"
- ".inst 0x647b41de // bfdot z30.s, z14.h, z3.h[3]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- ".inst 0x647841f3 // bfdot z19.s, z15.h, z0.h[3]\n"
- "ld1rqh z0.h, p6/z, [%[a_ptr0], #0x10]\n"
- ".inst 0x647941f7 // bfdot z23.s, z15.h, z1.h[3]\n"
- "ld1rqh z1.h, p6/z, [a_ptr1, #0x10]\n"
- ".inst 0x647a41fb // bfdot z27.s, z15.h, z2.h[3]\n"
- "ld1rqh z2.h, p6/z, [a_ptr2, #0x10]\n"
- ".inst 0x647b41ff // bfdot z31.s, z15.h, z3.h[3]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- ".inst 0x64644110 // bfdot z16.s, z8.h, z4.h[0]\n"
- "ld1rqh z3.h, p6/z, [a_ptr3, #0x10]\n"
- ".inst 0x64654114 // bfdot z20.s, z8.h, z5.h[0]\n"
- "addvl %[a_ptr0], %[a_ptr0], #2\n"
- ".inst 0x64664118 // bfdot z24.s, z8.h, z6.h[0]\n"
- "addvl a_ptr1, a_ptr1, #2\n"
- ".inst 0x6467411c // bfdot z28.s, z8.h, z7.h[0]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- ".inst 0x64644131 // bfdot z17.s, z9.h, z4.h[0]\n"
- "addvl a_ptr2, a_ptr2, #2\n"
- ".inst 0x64654135 // bfdot z21.s, z9.h, z5.h[0]\n"
- "addvl a_ptr3, a_ptr3, #2\n"
- ".inst 0x64664139 // bfdot z25.s, z9.h, z6.h[0]\n"
- ".inst 0x6467413d // bfdot z29.s, z9.h, z7.h[0]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x64644152 // bfdot z18.s, z10.h, z4.h[0]\n"
- ".inst 0x64654156 // bfdot z22.s, z10.h, z5.h[0]\n"
- ".inst 0x6466415a // bfdot z26.s, z10.h, z6.h[0]\n"
- ".inst 0x6467415e // bfdot z30.s, z10.h, z7.h[0]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x64644173 // bfdot z19.s, z11.h, z4.h[0]\n"
- ".inst 0x64654177 // bfdot z23.s, z11.h, z5.h[0]\n"
- ".inst 0x6466417b // bfdot z27.s, z11.h, z6.h[0]\n"
- ".inst 0x6467417f // bfdot z31.s, z11.h, z7.h[0]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x646c4190 // bfdot z16.s, z12.h, z4.h[1]\n"
- ".inst 0x646d4194 // bfdot z20.s, z12.h, z5.h[1]\n"
- ".inst 0x646e4198 // bfdot z24.s, z12.h, z6.h[1]\n"
- ".inst 0x646f419c // bfdot z28.s, z12.h, z7.h[1]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x646c41b1 // bfdot z17.s, z13.h, z4.h[1]\n"
- ".inst 0x646d41b5 // bfdot z21.s, z13.h, z5.h[1]\n"
- ".inst 0x646e41b9 // bfdot z25.s, z13.h, z6.h[1]\n"
- ".inst 0x646f41bd // bfdot z29.s, z13.h, z7.h[1]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x646c41d2 // bfdot z18.s, z14.h, z4.h[1]\n"
- ".inst 0x646d41d6 // bfdot z22.s, z14.h, z5.h[1]\n"
- ".inst 0x646e41da // bfdot z26.s, z14.h, z6.h[1]\n"
- ".inst 0x646f41de // bfdot z30.s, z14.h, z7.h[1]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x646c41f3 // bfdot z19.s, z15.h, z4.h[1]\n"
- ".inst 0x646d41f7 // bfdot z23.s, z15.h, z5.h[1]\n"
- ".inst 0x646e41fb // bfdot z27.s, z15.h, z6.h[1]\n"
- ".inst 0x646f41ff // bfdot z31.s, z15.h, z7.h[1]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x64744110 // bfdot z16.s, z8.h, z4.h[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- ".inst 0x64754114 // bfdot z20.s, z8.h, z5.h[2]\n"
- ".inst 0x64764118 // bfdot z24.s, z8.h, z6.h[2]\n"
- ".inst 0x6477411c // bfdot z28.s, z8.h, z7.h[2]\n"
- ".inst 0x64744131 // bfdot z17.s, z9.h, z4.h[2]\n"
- ".inst 0x64754135 // bfdot z21.s, z9.h, z5.h[2]\n"
- ".inst 0x64764139 // bfdot z25.s, z9.h, z6.h[2]\n"
- ".inst 0x6477413d // bfdot z29.s, z9.h, z7.h[2]\n"
- ".inst 0x64744152 // bfdot z18.s, z10.h, z4.h[2]\n"
- ".inst 0x64754156 // bfdot z22.s, z10.h, z5.h[2]\n"
- ".inst 0x6476415a // bfdot z26.s, z10.h, z6.h[2]\n"
- ".inst 0x6477415e // bfdot z30.s, z10.h, z7.h[2]\n"
- ".inst 0x64744173 // bfdot z19.s, z11.h, z4.h[2]\n"
- ".inst 0x64754177 // bfdot z23.s, z11.h, z5.h[2]\n"
- ".inst 0x6476417b // bfdot z27.s, z11.h, z6.h[2]\n"
- ".inst 0x6477417f // bfdot z31.s, z11.h, z7.h[2]\n"
- ".inst 0x647c4190 // bfdot z16.s, z12.h, z4.h[3]\n"
- ".inst 0x647d4194 // bfdot z20.s, z12.h, z5.h[3]\n"
- ".inst 0x647e4198 // bfdot z24.s, z12.h, z6.h[3]\n"
- ".inst 0x647f419c // bfdot z28.s, z12.h, z7.h[3]\n"
- ".inst 0x647c41b1 // bfdot z17.s, z13.h, z4.h[3]\n"
- ".inst 0x647d41b5 // bfdot z21.s, z13.h, z5.h[3]\n"
- ".inst 0x647e41b9 // bfdot z25.s, z13.h, z6.h[3]\n"
- ".inst 0x647f41bd // bfdot z29.s, z13.h, z7.h[3]\n"
- ".inst 0x647c41d2 // bfdot z18.s, z14.h, z4.h[3]\n"
- ".inst 0x647d41d6 // bfdot z22.s, z14.h, z5.h[3]\n"
- ".inst 0x647e41da // bfdot z26.s, z14.h, z6.h[3]\n"
- ".inst 0x647f41de // bfdot z30.s, z14.h, z7.h[3]\n"
- ".inst 0x647c41f3 // bfdot z19.s, z15.h, z4.h[3]\n"
- ".inst 0x647d41f7 // bfdot z23.s, z15.h, z5.h[3]\n"
- ".inst 0x647e41fb // bfdot z27.s, z15.h, z6.h[3]\n"
- ".inst 0x647f41ff // bfdot z31.s, z15.h, z7.h[3]\n"
- "cbz %[blocks], 5f\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x64604110 // bfdot z16.s, z8.h, z0.h[0]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x64614114 // bfdot z20.s, z8.h, z1.h[0]\n"
- ".inst 0x64624118 // bfdot z24.s, z8.h, z2.h[0]\n"
- ".inst 0x6463411c // bfdot z28.s, z8.h, z3.h[0]\n"
- ".inst 0x64604131 // bfdot z17.s, z9.h, z0.h[0]\n"
- ".inst 0x64614135 // bfdot z21.s, z9.h, z1.h[0]\n"
- ".inst 0x64624139 // bfdot z25.s, z9.h, z2.h[0]\n"
- ".inst 0x6463413d // bfdot z29.s, z9.h, z3.h[0]\n"
- ".inst 0x64604152 // bfdot z18.s, z10.h, z0.h[0]\n"
- ".inst 0x64614156 // bfdot z22.s, z10.h, z1.h[0]\n"
- ".inst 0x6462415a // bfdot z26.s, z10.h, z2.h[0]\n"
- ".inst 0x6463415e // bfdot z30.s, z10.h, z3.h[0]\n"
- ".inst 0x64604173 // bfdot z19.s, z11.h, z0.h[0]\n"
- ".inst 0x64614177 // bfdot z23.s, z11.h, z1.h[0]\n"
- ".inst 0x6462417b // bfdot z27.s, z11.h, z2.h[0]\n"
- ".inst 0x6463417f // bfdot z31.s, z11.h, z3.h[0]\n"
- "b.eq 5f\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x64684190 // bfdot z16.s, z12.h, z0.h[1]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x64694194 // bfdot z20.s, z12.h, z1.h[1]\n"
- ".inst 0x646a4198 // bfdot z24.s, z12.h, z2.h[1]\n"
- ".inst 0x646b419c // bfdot z28.s, z12.h, z3.h[1]\n"
- ".inst 0x646841b1 // bfdot z17.s, z13.h, z0.h[1]\n"
- ".inst 0x646941b5 // bfdot z21.s, z13.h, z1.h[1]\n"
- ".inst 0x646a41b9 // bfdot z25.s, z13.h, z2.h[1]\n"
- ".inst 0x646b41bd // bfdot z29.s, z13.h, z3.h[1]\n"
- ".inst 0x646841d2 // bfdot z18.s, z14.h, z0.h[1]\n"
- ".inst 0x646941d6 // bfdot z22.s, z14.h, z1.h[1]\n"
- ".inst 0x646a41da // bfdot z26.s, z14.h, z2.h[1]\n"
- ".inst 0x646b41de // bfdot z30.s, z14.h, z3.h[1]\n"
- ".inst 0x646841f3 // bfdot z19.s, z15.h, z0.h[1]\n"
- ".inst 0x646941f7 // bfdot z23.s, z15.h, z1.h[1]\n"
- ".inst 0x646a41fb // bfdot z27.s, z15.h, z2.h[1]\n"
- ".inst 0x646b41ff // bfdot z31.s, z15.h, z3.h[1]\n"
- "b.eq 5f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- ".inst 0x64704110 // bfdot z16.s, z8.h, z0.h[2]\n"
- ".inst 0x64714114 // bfdot z20.s, z8.h, z1.h[2]\n"
- ".inst 0x64724118 // bfdot z24.s, z8.h, z2.h[2]\n"
- ".inst 0x6473411c // bfdot z28.s, z8.h, z3.h[2]\n"
- ".inst 0x64704131 // bfdot z17.s, z9.h, z0.h[2]\n"
- ".inst 0x64714135 // bfdot z21.s, z9.h, z1.h[2]\n"
- ".inst 0x64724139 // bfdot z25.s, z9.h, z2.h[2]\n"
- ".inst 0x6473413d // bfdot z29.s, z9.h, z3.h[2]\n"
- ".inst 0x64704152 // bfdot z18.s, z10.h, z0.h[2]\n"
- ".inst 0x64714156 // bfdot z22.s, z10.h, z1.h[2]\n"
- ".inst 0x6472415a // bfdot z26.s, z10.h, z2.h[2]\n"
- ".inst 0x6473415e // bfdot z30.s, z10.h, z3.h[2]\n"
- ".inst 0x64704173 // bfdot z19.s, z11.h, z0.h[2]\n"
- ".inst 0x64714177 // bfdot z23.s, z11.h, z1.h[2]\n"
- ".inst 0x6472417b // bfdot z27.s, z11.h, z2.h[2]\n"
- ".inst 0x6473417f // bfdot z31.s, z11.h, z3.h[2]\n"
- "b.eq 5f\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- ".inst 0x64784190 // bfdot z16.s, z12.h, z0.h[3]\n"
- ".inst 0x64794194 // bfdot z20.s, z12.h, z1.h[3]\n"
- ".inst 0x647a4198 // bfdot z24.s, z12.h, z2.h[3]\n"
- ".inst 0x647b419c // bfdot z28.s, z12.h, z3.h[3]\n"
- ".inst 0x647841b1 // bfdot z17.s, z13.h, z0.h[3]\n"
- ".inst 0x647941b5 // bfdot z21.s, z13.h, z1.h[3]\n"
- ".inst 0x647a41b9 // bfdot z25.s, z13.h, z2.h[3]\n"
- ".inst 0x647b41bd // bfdot z29.s, z13.h, z3.h[3]\n"
- ".inst 0x647841d2 // bfdot z18.s, z14.h, z0.h[3]\n"
- ".inst 0x647941d6 // bfdot z22.s, z14.h, z1.h[3]\n"
- ".inst 0x647a41da // bfdot z26.s, z14.h, z2.h[3]\n"
- ".inst 0x647b41de // bfdot z30.s, z14.h, z3.h[3]\n"
- ".inst 0x647841f3 // bfdot z19.s, z15.h, z0.h[3]\n"
- ".inst 0x647941f7 // bfdot z23.s, z15.h, z1.h[3]\n"
- ".inst 0x647a41fb // bfdot z27.s, z15.h, z2.h[3]\n"
- ".inst 0x647b41ff // bfdot z31.s, z15.h, z3.h[3]\n"
- "b 5f\n"
- "4:\n"
- ".inst 0x64604110 // bfdot z16.s, z8.h, z0.h[0]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- ".inst 0x64614114 // bfdot z20.s, z8.h, z1.h[0]\n"
- "ld1rqh z4.h, p6/z, [%[a_ptr0]]\n"
- ".inst 0x64624118 // bfdot z24.s, z8.h, z2.h[0]\n"
- "ld1rqh z5.h, p6/z, [a_ptr1]\n"
- ".inst 0x6463411c // bfdot z28.s, z8.h, z3.h[0]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- ".inst 0x64604131 // bfdot z17.s, z9.h, z0.h[0]\n"
- "ld1rqh z6.h, p6/z, [a_ptr2]\n"
- ".inst 0x64614135 // bfdot z21.s, z9.h, z1.h[0]\n"
- "ld1rqh z7.h, p6/z, [a_ptr3]\n"
- ".inst 0x64624139 // bfdot z25.s, z9.h, z2.h[0]\n"
- "addvl %[a_ptr0], %[a_ptr0], #1\n"
- ".inst 0x6463413d // bfdot z29.s, z9.h, z3.h[0]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x64604152 // bfdot z18.s, z10.h, z0.h[0]\n"
- "addvl a_ptr1, a_ptr1, #1\n"
- ".inst 0x64614156 // bfdot z22.s, z10.h, z1.h[0]\n"
- "addvl a_ptr2, a_ptr2, #1\n"
- ".inst 0x6462415a // bfdot z26.s, z10.h, z2.h[0]\n"
- "addvl a_ptr3, a_ptr3, #1\n"
- ".inst 0x6463415e // bfdot z30.s, z10.h, z3.h[0]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x64604173 // bfdot z19.s, z11.h, z0.h[0]\n"
- ".inst 0x64614177 // bfdot z23.s, z11.h, z1.h[0]\n"
- ".inst 0x6462417b // bfdot z27.s, z11.h, z2.h[0]\n"
- ".inst 0x6463417f // bfdot z31.s, z11.h, z3.h[0]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x64684190 // bfdot z16.s, z12.h, z0.h[1]\n"
- ".inst 0x64694194 // bfdot z20.s, z12.h, z1.h[1]\n"
- ".inst 0x646a4198 // bfdot z24.s, z12.h, z2.h[1]\n"
- ".inst 0x646b419c // bfdot z28.s, z12.h, z3.h[1]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x646841b1 // bfdot z17.s, z13.h, z0.h[1]\n"
- ".inst 0x646941b5 // bfdot z21.s, z13.h, z1.h[1]\n"
- ".inst 0x646a41b9 // bfdot z25.s, z13.h, z2.h[1]\n"
- ".inst 0x646b41bd // bfdot z29.s, z13.h, z3.h[1]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x646841d2 // bfdot z18.s, z14.h, z0.h[1]\n"
- ".inst 0x646941d6 // bfdot z22.s, z14.h, z1.h[1]\n"
- ".inst 0x646a41da // bfdot z26.s, z14.h, z2.h[1]\n"
- ".inst 0x646b41de // bfdot z30.s, z14.h, z3.h[1]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x646841f3 // bfdot z19.s, z15.h, z0.h[1]\n"
- ".inst 0x646941f7 // bfdot z23.s, z15.h, z1.h[1]\n"
- ".inst 0x646a41fb // bfdot z27.s, z15.h, z2.h[1]\n"
- ".inst 0x646b41ff // bfdot z31.s, z15.h, z3.h[1]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x64704110 // bfdot z16.s, z8.h, z0.h[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- ".inst 0x64714114 // bfdot z20.s, z8.h, z1.h[2]\n"
- ".inst 0x64724118 // bfdot z24.s, z8.h, z2.h[2]\n"
- ".inst 0x6473411c // bfdot z28.s, z8.h, z3.h[2]\n"
- ".inst 0x64704131 // bfdot z17.s, z9.h, z0.h[2]\n"
- ".inst 0x64714135 // bfdot z21.s, z9.h, z1.h[2]\n"
- ".inst 0x64724139 // bfdot z25.s, z9.h, z2.h[2]\n"
- ".inst 0x6473413d // bfdot z29.s, z9.h, z3.h[2]\n"
- ".inst 0x64704152 // bfdot z18.s, z10.h, z0.h[2]\n"
- ".inst 0x64714156 // bfdot z22.s, z10.h, z1.h[2]\n"
- ".inst 0x6472415a // bfdot z26.s, z10.h, z2.h[2]\n"
- ".inst 0x6473415e // bfdot z30.s, z10.h, z3.h[2]\n"
- ".inst 0x64704173 // bfdot z19.s, z11.h, z0.h[2]\n"
- ".inst 0x64714177 // bfdot z23.s, z11.h, z1.h[2]\n"
- ".inst 0x6472417b // bfdot z27.s, z11.h, z2.h[2]\n"
- ".inst 0x6473417f // bfdot z31.s, z11.h, z3.h[2]\n"
- ".inst 0x64784190 // bfdot z16.s, z12.h, z0.h[3]\n"
- ".inst 0x64794194 // bfdot z20.s, z12.h, z1.h[3]\n"
- ".inst 0x647a4198 // bfdot z24.s, z12.h, z2.h[3]\n"
- ".inst 0x647b419c // bfdot z28.s, z12.h, z3.h[3]\n"
- ".inst 0x647841b1 // bfdot z17.s, z13.h, z0.h[3]\n"
- ".inst 0x647941b5 // bfdot z21.s, z13.h, z1.h[3]\n"
- ".inst 0x647a41b9 // bfdot z25.s, z13.h, z2.h[3]\n"
- ".inst 0x647b41bd // bfdot z29.s, z13.h, z3.h[3]\n"
- ".inst 0x647841d2 // bfdot z18.s, z14.h, z0.h[3]\n"
- ".inst 0x647941d6 // bfdot z22.s, z14.h, z1.h[3]\n"
- ".inst 0x647a41da // bfdot z26.s, z14.h, z2.h[3]\n"
- ".inst 0x647b41de // bfdot z30.s, z14.h, z3.h[3]\n"
- ".inst 0x647841f3 // bfdot z19.s, z15.h, z0.h[3]\n"
- ".inst 0x647941f7 // bfdot z23.s, z15.h, z1.h[3]\n"
- ".inst 0x647a41fb // bfdot z27.s, z15.h, z2.h[3]\n"
- ".inst 0x647b41ff // bfdot z31.s, z15.h, z3.h[3]\n"
- "cbz %[blocks], 5f\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x64644110 // bfdot z16.s, z8.h, z4.h[0]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x64654114 // bfdot z20.s, z8.h, z5.h[0]\n"
- ".inst 0x64664118 // bfdot z24.s, z8.h, z6.h[0]\n"
- ".inst 0x6467411c // bfdot z28.s, z8.h, z7.h[0]\n"
- ".inst 0x64644131 // bfdot z17.s, z9.h, z4.h[0]\n"
- ".inst 0x64654135 // bfdot z21.s, z9.h, z5.h[0]\n"
- ".inst 0x64664139 // bfdot z25.s, z9.h, z6.h[0]\n"
- ".inst 0x6467413d // bfdot z29.s, z9.h, z7.h[0]\n"
- ".inst 0x64644152 // bfdot z18.s, z10.h, z4.h[0]\n"
- ".inst 0x64654156 // bfdot z22.s, z10.h, z5.h[0]\n"
- ".inst 0x6466415a // bfdot z26.s, z10.h, z6.h[0]\n"
- ".inst 0x6467415e // bfdot z30.s, z10.h, z7.h[0]\n"
- ".inst 0x64644173 // bfdot z19.s, z11.h, z4.h[0]\n"
- ".inst 0x64654177 // bfdot z23.s, z11.h, z5.h[0]\n"
- ".inst 0x6466417b // bfdot z27.s, z11.h, z6.h[0]\n"
- ".inst 0x6467417f // bfdot z31.s, z11.h, z7.h[0]\n"
- "b.eq 5f\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x646c4190 // bfdot z16.s, z12.h, z4.h[1]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x646d4194 // bfdot z20.s, z12.h, z5.h[1]\n"
- ".inst 0x646e4198 // bfdot z24.s, z12.h, z6.h[1]\n"
- ".inst 0x646f419c // bfdot z28.s, z12.h, z7.h[1]\n"
- ".inst 0x646c41b1 // bfdot z17.s, z13.h, z4.h[1]\n"
- ".inst 0x646d41b5 // bfdot z21.s, z13.h, z5.h[1]\n"
- ".inst 0x646e41b9 // bfdot z25.s, z13.h, z6.h[1]\n"
- ".inst 0x646f41bd // bfdot z29.s, z13.h, z7.h[1]\n"
- ".inst 0x646c41d2 // bfdot z18.s, z14.h, z4.h[1]\n"
- ".inst 0x646d41d6 // bfdot z22.s, z14.h, z5.h[1]\n"
- ".inst 0x646e41da // bfdot z26.s, z14.h, z6.h[1]\n"
- ".inst 0x646f41de // bfdot z30.s, z14.h, z7.h[1]\n"
- ".inst 0x646c41f3 // bfdot z19.s, z15.h, z4.h[1]\n"
- ".inst 0x646d41f7 // bfdot z23.s, z15.h, z5.h[1]\n"
- ".inst 0x646e41fb // bfdot z27.s, z15.h, z6.h[1]\n"
- ".inst 0x646f41ff // bfdot z31.s, z15.h, z7.h[1]\n"
- "b.eq 5f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- ".inst 0x64744110 // bfdot z16.s, z8.h, z4.h[2]\n"
- ".inst 0x64754114 // bfdot z20.s, z8.h, z5.h[2]\n"
- ".inst 0x64764118 // bfdot z24.s, z8.h, z6.h[2]\n"
- ".inst 0x6477411c // bfdot z28.s, z8.h, z7.h[2]\n"
- ".inst 0x64744131 // bfdot z17.s, z9.h, z4.h[2]\n"
- ".inst 0x64754135 // bfdot z21.s, z9.h, z5.h[2]\n"
- ".inst 0x64764139 // bfdot z25.s, z9.h, z6.h[2]\n"
- ".inst 0x6477413d // bfdot z29.s, z9.h, z7.h[2]\n"
- ".inst 0x64744152 // bfdot z18.s, z10.h, z4.h[2]\n"
- ".inst 0x64754156 // bfdot z22.s, z10.h, z5.h[2]\n"
- ".inst 0x6476415a // bfdot z26.s, z10.h, z6.h[2]\n"
- ".inst 0x6477415e // bfdot z30.s, z10.h, z7.h[2]\n"
- ".inst 0x64744173 // bfdot z19.s, z11.h, z4.h[2]\n"
- ".inst 0x64754177 // bfdot z23.s, z11.h, z5.h[2]\n"
- ".inst 0x6476417b // bfdot z27.s, z11.h, z6.h[2]\n"
- ".inst 0x6477417f // bfdot z31.s, z11.h, z7.h[2]\n"
- "b.eq 5f\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- ".inst 0x647c4190 // bfdot z16.s, z12.h, z4.h[3]\n"
- ".inst 0x647d4194 // bfdot z20.s, z12.h, z5.h[3]\n"
- ".inst 0x647e4198 // bfdot z24.s, z12.h, z6.h[3]\n"
- ".inst 0x647f419c // bfdot z28.s, z12.h, z7.h[3]\n"
- ".inst 0x647c41b1 // bfdot z17.s, z13.h, z4.h[3]\n"
- ".inst 0x647d41b5 // bfdot z21.s, z13.h, z5.h[3]\n"
- ".inst 0x647e41b9 // bfdot z25.s, z13.h, z6.h[3]\n"
- ".inst 0x647f41bd // bfdot z29.s, z13.h, z7.h[3]\n"
- ".inst 0x647c41d2 // bfdot z18.s, z14.h, z4.h[3]\n"
- ".inst 0x647d41d6 // bfdot z22.s, z14.h, z5.h[3]\n"
- ".inst 0x647e41da // bfdot z26.s, z14.h, z6.h[3]\n"
- ".inst 0x647f41de // bfdot z30.s, z14.h, z7.h[3]\n"
- ".inst 0x647c41f3 // bfdot z19.s, z15.h, z4.h[3]\n"
- ".inst 0x647d41f7 // bfdot z23.s, z15.h, z5.h[3]\n"
- ".inst 0x647e41fb // bfdot z27.s, z15.h, z6.h[3]\n"
- ".inst 0x647f41ff // bfdot z31.s, z15.h, z7.h[3]\n"
- "5:\n"
- "ld1rw z14.s, p7/z, [%[minptr]]\n"
- "ld1rw z15.s, p7/z, [%[maxptr]]\n"
- "fmax z16.s, p7/m, z16.s, z14.s\n"
- "fmax z17.s, p7/m, z17.s, z14.s\n"
- "fmax z18.s, p7/m, z18.s, z14.s\n"
- "fmax z19.s, p7/m, z19.s, z14.s\n"
- "fmin z16.s, p7/m, z16.s, z15.s\n"
- "fmin z17.s, p7/m, z17.s, z15.s\n"
- "fmin z18.s, p7/m, z18.s, z15.s\n"
- "fmin z19.s, p7/m, z19.s, z15.s\n"
- "st1w z16.s, p0, [%[c_ptr0]]\n"
- "fmax z20.s, p7/m, z20.s, z14.s\n"
- "fmax z21.s, p7/m, z21.s, z14.s\n"
- "fmax z22.s, p7/m, z22.s, z14.s\n"
- "st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n"
- "fmax z23.s, p7/m, z23.s, z14.s\n"
- "fmin z20.s, p7/m, z20.s, z15.s\n"
- "fmin z21.s, p7/m, z21.s, z15.s\n"
- "st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n"
- "fmin z22.s, p7/m, z22.s, z15.s\n"
- "fmin z23.s, p7/m, z23.s, z15.s\n"
- "fmax z24.s, p7/m, z24.s, z14.s\n"
- "st1w z19.s, p3, [%[c_ptr0], #3, MUL VL]\n"
- "fmax z25.s, p7/m, z25.s, z14.s\n"
- "addvl %[c_ptr0], %[c_ptr0], #4\n"
- "fmax z26.s, p7/m, z26.s, z14.s\n"
- "st1w z20.s, p0, [c_ptr1]\n"
- "fmin z24.s, p7/m, z24.s, z15.s\n"
- "fmin z25.s, p7/m, z25.s, z15.s\n"
- "fmax z27.s, p7/m, z27.s, z14.s\n"
- "st1w z21.s, p1, [c_ptr1, #1, MUL VL]\n"
- "fmin z26.s, p7/m, z26.s, z15.s\n"
- "fmax z28.s, p7/m, z28.s, z14.s\n"
- "fmax z29.s, p7/m, z29.s, z14.s\n"
- "st1w z22.s, p2, [c_ptr1, #2, MUL VL]\n"
- "fmin z27.s, p7/m, z27.s, z15.s\n"
- "fmax z30.s, p7/m, z30.s, z14.s\n"
- "fmin z28.s, p7/m, z28.s, z15.s\n"
- "st1w z23.s, p3, [c_ptr1, #3, MUL VL]\n"
- "fmin z29.s, p7/m, z29.s, z15.s\n"
- "fmax z31.s, p7/m, z31.s, z14.s\n"
- "fmin z30.s, p7/m, z30.s, z15.s\n"
- "st1w z24.s, p0, [c_ptr2]\n"
- "fmin z31.s, p7/m, z31.s, z15.s\n"
- "st1w z25.s, p1, [c_ptr2, #1, MUL VL]\n"
- "st1w z26.s, p2, [c_ptr2, #2, MUL VL]\n"
- "st1w z27.s, p3, [c_ptr2, #3, MUL VL]\n"
- "st1w z28.s, p0, [c_ptr3]\n"
- "st1w z29.s, p1, [c_ptr3, #1, MUL VL]\n"
- "st1w z30.s, p2, [c_ptr3, #2, MUL VL]\n"
- "st1w z31.s, p3, [c_ptr3, #3, MUL VL]\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq a_ptr3\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- ".unreq c_ptr3\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks)
- : [width] "r" (width), [accumulate] "r" (static_cast<uint64_t>(accumulate)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers)
- : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "x4", "x5", "cc", "memory"
- );
- break;
- }
-
- }
- }
-}
-
-} // namespace arm_gemm
-
-#endif // __ARM_FEATURE_SVE
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_bf16fp32_dot_4VLx4.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_bf16fp32_dot_6x4VL.hpp
similarity index 65%
rename from src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_bf16fp32_dot_4VLx4.hpp
rename to src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_bf16fp32_dot_6x4VL.hpp
index eba98bb..e344d82 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_bf16fp32_dot_4VLx4.hpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_bf16fp32_dot_6x4VL.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2018-2020 Arm Limited.
+ * Copyright (c) 2019-2020 Arm Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -10,42 +10,49 @@
* sell copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
*/
#pragma once
-
#ifdef __ARM_FEATURE_SVE
-#include "../bfloat.hpp"
#include "../std_transforms_sve.hpp"
+#include "../bfloat.hpp"
+
+#define ARGLIST \
+ unsigned int, const unsigned int *, \
+ IndirectInputArg<bfloat16>, \
+ size_t, size_t, \
+ const bfloat16 *, \
+ IndirectOutputArg<float>, \
+ const float *, Activation, bool
namespace arm_gemm
{
// Actual kernel implementations
-void sve_hybrid_bf16fp32_dot_4VLx4(const bfloat16 *, int, const bfloat16 *, float *, int, int, int, int, const float *, Activation, bool);
+void sve_hybrid_bf16fp32_dot_6x4VL( ARGLIST );
-class hybrid_bf16fp32_dot_4VLx4
+class cls_sve_hybrid_bf16fp32_dot_6x4VL
{
public:
typedef bfloat16 operand_type;
typedef float result_type;
- typedef void (*kern_type)(const bfloat16 *, int, const bfloat16 *, float *, int, int, int, int, const float *, Activation, bool);
+ typedef void (*kern_type)( ARGLIST );
/* Kernel blocking parameters */
static constexpr unsigned int out_height()
{
- return 4;
+ return 6;
}
static unsigned int out_width()
@@ -63,27 +70,17 @@
return true;
}
- static constexpr bool supports_bias()
- {
- return true;
- }
-
- static constexpr bool supports_activation()
- {
- return true;
- }
-
- StdTransformsSVE<operand_type, result_type, 4, 4, 2> transforms = {};
+ StdTransformsSVE<operand_type, result_type, 6, 4, 2> transforms = {};
// Default to the generic kernel
- kern_type kernel=sve_hybrid_bf16fp32_dot_4VLx4;
+ kern_type kernel=sve_hybrid_bf16fp32_dot_6x4VL;
- hybrid_bf16fp32_dot_4VLx4(const CPUInfo *)
+ cls_sve_hybrid_bf16fp32_dot_6x4VL(const CPUInfo *)
{
-
}
};
} // namespace arm_gemm
+#undef ARGLIST
#endif // __ARM_FEATURE_SVE
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_bf16fp32_dot_6x4VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_bf16fp32_dot_6x4VL/generic.cpp
new file mode 100644
index 0000000..19385e5
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_bf16fp32_dot_6x4VL/generic.cpp
@@ -0,0 +1,2237 @@
+/*
+ * Copyright (c) 2019-2020 Arm Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+#ifdef __ARM_FEATURE_SVE
+
+#include "arm_gemm.hpp"
+#include "../../utils.hpp"
+#include "../../bfloat.hpp"
+
+#include <cassert>
+
+namespace arm_gemm {
+
+void sve_hybrid_bf16fp32_dot_6x4VL (
+ unsigned int num_strings, const unsigned int *string_lengths, IndirectInputArg<bfloat16> A_arg,
+ size_t M, size_t N, const bfloat16 *B_ptr, IndirectOutputArg<float> output_arg,
+ const float *bias, Activation act, bool accumulate
+)
+{
+ struct KernelArgs {
+ float maxval = static_cast<float>(std::numeric_limits<float>::infinity());
+ float minval = - static_cast<float>(std::numeric_limits<float>::infinity());
+ unsigned int num_strings = {};
+ const unsigned int *string_lengths = {};
+ size_t N = {};
+ const bfloat16 *B_ptr = {};
+ size_t output_offset = {};
+ size_t input_initial_col = {};
+ size_t input_offset = {};
+ } ka;
+
+ unsigned long flags=0;
+ void *output_ptr;
+ void *input_ptr;
+
+ if (output_arg.is_indirect) {
+ output_ptr=(void *)(output_arg.indirect.ptr);
+ ka.output_offset=output_arg.indirect.offset;
+ flags |= 0x4;
+ } else {
+ output_ptr=(void *)(output_arg.direct.base);
+ ka.output_offset=output_arg.direct.stride;
+ }
+
+ if (A_arg.is_indirect) {
+ input_ptr=(void *)(A_arg.indirect.ptr);
+ ka.input_offset=A_arg.indirect.start_row;
+ ka.input_initial_col=A_arg.indirect.start_col;
+ flags |= 0x8;
+ } else {
+ assert(num_strings==1);
+ input_ptr=(void *)(A_arg.direct.base);
+ ka.input_offset=A_arg.direct.stride;
+ }
+ if (accumulate) {
+ flags |= 0x1;
+ }
+ ka.num_strings = num_strings;
+ ka.string_lengths = string_lengths;
+ ka.N = N;
+ ka.B_ptr = B_ptr;
+ switch(act.type) {
+ default:
+ case Activation::Type::None:
+ break;
+ case Activation::Type::BoundedReLU:
+ ka.maxval = static_cast<float>(act.param1);
+ /* fall through */
+ case Activation::Type::ReLU:
+ ka.minval = 0;
+ flags |= 0x2;
+ break;
+ }
+ __asm__ __volatile__(
+ "ptrue p5.b\n"
+ "1:" // Row loop
+ "cmp %x[M], #0x6\n"
+ "bge 71f\n"
+ "cmp %x[M], #0x4\n"
+ "bgt 57f\n"
+ "beq 43f\n"
+ "cmp %x[M], #0x2\n"
+ "bgt 29f\n"
+ "beq 15f\n"
+ "ldr x16, [%x[args_ptr], %[offsetof_N]]\n"
+ "ldr x15, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "mov x14, %x[bias]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 2f\n"
+ "ldr x13, [%x[output_ptr], #0x0]\n"
+ "add x13, x13, x19, LSL #2\n"
+ "b 3f\n"
+ "2:" // Height 1: setup direct output
+ "mov x13, %x[output_ptr]\n"
+ "3:" // Height 1: Column loop
+ "mov x19, #0x0\n"
+ "whilelt p4.s, x19, x16\n"
+ "incw x19\n"
+ "whilelt p3.s, x19, x16\n"
+ "incw x19\n"
+ "whilelt p2.s, x19, x16\n"
+ "incw x19\n"
+ "whilelt p1.s, x19, x16\n"
+ "cbz x14, 4f\n"
+ "ld1w { z8.s }, p5/Z, [x14]\n"
+ "ld1w { z9.s }, p5/Z, [x14, #1, MUL VL]\n"
+ "ld1w { z10.s }, p5/Z, [x14, #2, MUL VL]\n"
+ "ld1w { z11.s }, p5/Z, [x14, #3, MUL VL]\n"
+ "addvl x14, x14, #4\n"
+ "b 6f\n"
+ "4:" // Height 1: no bias
+ "tbz %x[flags], #0, 5f\n"
+ "ld1w { z8.s }, p4/Z, [x13]\n"
+ "ld1w { z9.s }, p3/Z, [x13, #1, MUL VL]\n"
+ "ld1w { z10.s }, p2/Z, [x13, #2, MUL VL]\n"
+ "ld1w { z11.s }, p1/Z, [x13, #3, MUL VL]\n"
+ "b 6f\n"
+ "5:" // Height 1: no accumulate
+ "mov z8.b, #0x0\n"
+ "mov z9.b, #0x0\n"
+ "mov z10.b, #0x0\n"
+ "mov z11.b, #0x0\n"
+ "6:" // Height 1: setup done
+ "mov x12, #0x0\n"
+ "7:" // Height 1: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w11, [x20, x12, LSL #0x2]\n"
+ "tbz %x[flags], #3, 8f\n"
+ "ldr x20, [%x[input_ptr], x12, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x10, [x20, #0x0]\n"
+ "cbnz x12, 9f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x10, x10, x19, LSL #1\n"
+ "b 9f\n"
+ "8:" // Height 1: setup direct input
+ "mov x10, %x[input_ptr]\n"
+ "9:" // Height 1: input setup done
+ "cmp x11, #0x8\n"
+ "ble 11f\n"
+ "10:" // Height 1: Multiply loop: Main loop head
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ "whilelt p0.h, XZR, x11\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ "sub x11, x11, #0x8\n"
+ "ld1rqh { z0.h }, p0/Z, [x10]\n"
+ ".inst 0x646040c8 // bfdot z8.s, z6.h, z0.h[0]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ "add x10, x10, #0x10\n"
+ ".inst 0x646040e9 // bfdot z9.s, z7.h, z0.h[0]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ "cmp x11, #0x8\n"
+ ".inst 0x646040ca // bfdot z10.s, z6.h, z0.h[0]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #4, MUL VL]\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ ".inst 0x646040eb // bfdot z11.s, z7.h, z0.h[0]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #5, MUL VL]\n"
+ ".inst 0x646840c8 // bfdot z8.s, z6.h, z0.h[1]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #6, MUL VL]\n"
+ ".inst 0x646840e9 // bfdot z9.s, z7.h, z0.h[1]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #7, MUL VL]\n"
+ "addvl x15, x15, #16\n"
+ ".inst 0x646840ca // bfdot z10.s, z6.h, z0.h[1]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #-8, MUL VL]\n"
+ ".inst 0x646840eb // bfdot z11.s, z7.h, z0.h[1]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #-7, MUL VL]\n"
+ ".inst 0x647040c8 // bfdot z8.s, z6.h, z0.h[2]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #-6, MUL VL]\n"
+ ".inst 0x647040e9 // bfdot z9.s, z7.h, z0.h[2]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #-5, MUL VL]\n"
+ ".inst 0x647040ca // bfdot z10.s, z6.h, z0.h[2]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #-4, MUL VL]\n"
+ ".inst 0x647040eb // bfdot z11.s, z7.h, z0.h[2]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #-3, MUL VL]\n"
+ ".inst 0x647840c8 // bfdot z8.s, z6.h, z0.h[3]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #-2, MUL VL]\n"
+ ".inst 0x647840e9 // bfdot z9.s, z7.h, z0.h[3]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #-1, MUL VL]\n"
+ ".inst 0x647840ca // bfdot z10.s, z6.h, z0.h[3]\n"
+ ".inst 0x647840eb // bfdot z11.s, z7.h, z0.h[3]\n"
+ "bgt 10b\n"
+ "11:" // Height 1: Multiply loop: Single iteration only
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ "whilelt p0.h, XZR, x11\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ "subs x11, x11, #0x2\n"
+ "ld1rqh { z0.h }, p0/Z, [x10]\n"
+ ".inst 0x646040c8 // bfdot z8.s, z6.h, z0.h[0]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ "add x10, x10, #0x10\n"
+ ".inst 0x646040e9 // bfdot z9.s, z7.h, z0.h[0]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ "addvl x15, x15, #4\n"
+ ".inst 0x646040ca // bfdot z10.s, z6.h, z0.h[0]\n"
+ ".inst 0x646040eb // bfdot z11.s, z7.h, z0.h[0]\n"
+ "ble 12f\n"
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ ".inst 0x646840c8 // bfdot z8.s, z6.h, z0.h[1]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ "subs x11, x11, #0x2\n"
+ ".inst 0x646840e9 // bfdot z9.s, z7.h, z0.h[1]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ ".inst 0x646840ca // bfdot z10.s, z6.h, z0.h[1]\n"
+ "addvl x15, x15, #4\n"
+ ".inst 0x646840eb // bfdot z11.s, z7.h, z0.h[1]\n"
+ "ble 12f\n"
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ ".inst 0x647040c8 // bfdot z8.s, z6.h, z0.h[2]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ "subs x11, x11, #0x2\n"
+ ".inst 0x647040e9 // bfdot z9.s, z7.h, z0.h[2]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ ".inst 0x647040ca // bfdot z10.s, z6.h, z0.h[2]\n"
+ "addvl x15, x15, #4\n"
+ ".inst 0x647040eb // bfdot z11.s, z7.h, z0.h[2]\n"
+ "ble 12f\n"
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ ".inst 0x647840c8 // bfdot z8.s, z6.h, z0.h[3]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ ".inst 0x647840e9 // bfdot z9.s, z7.h, z0.h[3]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ "addvl x15, x15, #4\n"
+ ".inst 0x647840ca // bfdot z10.s, z6.h, z0.h[3]\n"
+ ".inst 0x647840eb // bfdot z11.s, z7.h, z0.h[3]\n"
+ "12:" // Height 1: Multiply loop: multiply skip
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x12, x12, #0x1\n"
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "cmp x12, x19\n"
+ "bne 7b\n"
+ "prfm pstl1keep, [x13, #0x0]\n"
+ "tbz %x[flags], #1, 13f\n"
+ "add x19, %x[args_ptr], %[offset_min]\n"
+ "ld1rw { z1.s }, p5/Z, [x19]\n"
+ "add x19, %x[args_ptr], %[offset_max]\n"
+ "ld1rw { z0.s }, p5/Z, [x19]\n"
+ "fmin z8.s, p5/M, z8.s, z0.s\n"
+ "fmin z9.s, p5/M, z9.s, z0.s\n"
+ "fmin z10.s, p5/M, z10.s, z0.s\n"
+ "fmin z11.s, p5/M, z11.s, z0.s\n"
+ "fmax z8.s, p5/M, z8.s, z1.s\n"
+ "fmax z9.s, p5/M, z9.s, z1.s\n"
+ "fmax z10.s, p5/M, z10.s, z1.s\n"
+ "fmax z11.s, p5/M, z11.s, z1.s\n"
+ "13:" // Height 1: No activation
+ "st1w { z8.s }, p4, [x13]\n"
+ "st1w { z9.s }, p3, [x13, #1, MUL VL]\n"
+ "st1w { z10.s }, p2, [x13, #2, MUL VL]\n"
+ "st1w { z11.s }, p1, [x13, #3, MUL VL]\n"
+ "addvl x13, x13, #4\n"
+ "14:" // Height 1: Writeback done
+ "mov x19, #0x0\n"
+ "incw x19, ALL, MUL #4\n"
+ "subs x16, x16, x19\n"
+ "bgt 3b\n"
+ "b 86f\n"
+ "15:" // Height 2
+ "ldr x16, [%x[args_ptr], %[offsetof_N]]\n"
+ "mov x14, %x[bias]\n"
+ "ldr x15, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 16f\n"
+ "ldr x13, [%x[output_ptr], #0x0]\n"
+ "add x13, x13, x19, LSL #2\n"
+ "ldr x9, [%x[output_ptr], #0x8]\n"
+ "add x9, x9, x19, LSL #2\n"
+ "b 17f\n"
+ "16:" // Height 2: setup direct output
+ "mov x13, %x[output_ptr]\n"
+ "add x9, x13, x19, LSL #2\n"
+ "17:" // Height 2: Column loop
+ "mov x19, #0x0\n"
+ "whilelt p4.s, x19, x16\n"
+ "incw x19\n"
+ "whilelt p3.s, x19, x16\n"
+ "incw x19\n"
+ "whilelt p2.s, x19, x16\n"
+ "incw x19\n"
+ "whilelt p1.s, x19, x16\n"
+ "cbz x14, 18f\n"
+ "ld1w { z8.s }, p5/Z, [x14]\n"
+ "mov z12.d, z8.d\n"
+ "ld1w { z9.s }, p5/Z, [x14, #1, MUL VL]\n"
+ "ld1w { z10.s }, p5/Z, [x14, #2, MUL VL]\n"
+ "mov z13.d, z9.d\n"
+ "ld1w { z11.s }, p5/Z, [x14, #3, MUL VL]\n"
+ "addvl x14, x14, #4\n"
+ "mov z14.d, z10.d\n"
+ "mov z15.d, z11.d\n"
+ "b 20f\n"
+ "18:" // Height 2: no bias
+ "tbz %x[flags], #0, 19f\n"
+ "ld1w { z8.s }, p4/Z, [x13]\n"
+ "ld1w { z9.s }, p3/Z, [x13, #1, MUL VL]\n"
+ "ld1w { z10.s }, p2/Z, [x13, #2, MUL VL]\n"
+ "ld1w { z11.s }, p1/Z, [x13, #3, MUL VL]\n"
+ "ld1w { z12.s }, p4/Z, [x9]\n"
+ "ld1w { z13.s }, p3/Z, [x9, #1, MUL VL]\n"
+ "ld1w { z14.s }, p2/Z, [x9, #2, MUL VL]\n"
+ "ld1w { z15.s }, p1/Z, [x9, #3, MUL VL]\n"
+ "b 20f\n"
+ "19:" // Height 2: no accumulate
+ "mov z8.b, #0x0\n"
+ "mov z9.b, #0x0\n"
+ "mov z10.b, #0x0\n"
+ "mov z11.b, #0x0\n"
+ "mov z12.b, #0x0\n"
+ "mov z13.b, #0x0\n"
+ "mov z14.b, #0x0\n"
+ "mov z15.b, #0x0\n"
+ "20:" // Height 2: setup done
+ "mov x12, #0x0\n"
+ "21:" // Height 2: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w11, [x20, x12, LSL #0x2]\n"
+ "tbz %x[flags], #3, 22f\n"
+ "ldr x20, [%x[input_ptr], x12, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x10, [x20, #0x0]\n"
+ "ldr x28, [x20, #0x8]\n"
+ "cbnz x12, 23f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x10, x10, x19, LSL #1\n"
+ "add x28, x28, x19, LSL #1\n"
+ "b 23f\n"
+ "22:" // Height 2: setup direct input
+ "mov x10, %x[input_ptr]\n"
+ "add x28, x10, x19, LSL #1\n"
+ "23:" // Height 2: input setup done
+ "cmp x11, #0x8\n"
+ "ble 25f\n"
+ "24:" // Height 2: Multiply loop: Main loop head
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ "whilelt p0.h, XZR, x11\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ "sub x11, x11, #0x8\n"
+ "ld1rqh { z0.h }, p0/Z, [x10]\n"
+ ".inst 0x646040c8 // bfdot z8.s, z6.h, z0.h[0]\n"
+ "ld1rqh { z1.h }, p0/Z, [x28]\n"
+ "add x10, x10, #0x10\n"
+ ".inst 0x646040e9 // bfdot z9.s, z7.h, z0.h[0]\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x28, x28, #0x10\n"
+ ".inst 0x646140cc // bfdot z12.s, z6.h, z1.h[0]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ "cmp x11, #0x8\n"
+ ".inst 0x646140ed // bfdot z13.s, z7.h, z1.h[0]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ ".inst 0x646040ca // bfdot z10.s, z6.h, z0.h[0]\n"
+ ".inst 0x646140ce // bfdot z14.s, z6.h, z1.h[0]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #4, MUL VL]\n"
+ ".inst 0x646040eb // bfdot z11.s, z7.h, z0.h[0]\n"
+ ".inst 0x646140ef // bfdot z15.s, z7.h, z1.h[0]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #5, MUL VL]\n"
+ ".inst 0x646840c8 // bfdot z8.s, z6.h, z0.h[1]\n"
+ ".inst 0x646940cc // bfdot z12.s, z6.h, z1.h[1]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #6, MUL VL]\n"
+ ".inst 0x646840e9 // bfdot z9.s, z7.h, z0.h[1]\n"
+ ".inst 0x646940ed // bfdot z13.s, z7.h, z1.h[1]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #7, MUL VL]\n"
+ "addvl x15, x15, #16\n"
+ ".inst 0x646840ca // bfdot z10.s, z6.h, z0.h[1]\n"
+ ".inst 0x646940ce // bfdot z14.s, z6.h, z1.h[1]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #-8, MUL VL]\n"
+ ".inst 0x646840eb // bfdot z11.s, z7.h, z0.h[1]\n"
+ ".inst 0x646940ef // bfdot z15.s, z7.h, z1.h[1]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #-7, MUL VL]\n"
+ ".inst 0x647040c8 // bfdot z8.s, z6.h, z0.h[2]\n"
+ ".inst 0x647140cc // bfdot z12.s, z6.h, z1.h[2]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #-6, MUL VL]\n"
+ ".inst 0x647040e9 // bfdot z9.s, z7.h, z0.h[2]\n"
+ ".inst 0x647140ed // bfdot z13.s, z7.h, z1.h[2]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #-5, MUL VL]\n"
+ ".inst 0x647040ca // bfdot z10.s, z6.h, z0.h[2]\n"
+ ".inst 0x647140ce // bfdot z14.s, z6.h, z1.h[2]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #-4, MUL VL]\n"
+ ".inst 0x647040eb // bfdot z11.s, z7.h, z0.h[2]\n"
+ ".inst 0x647140ef // bfdot z15.s, z7.h, z1.h[2]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #-3, MUL VL]\n"
+ ".inst 0x647840c8 // bfdot z8.s, z6.h, z0.h[3]\n"
+ ".inst 0x647940cc // bfdot z12.s, z6.h, z1.h[3]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #-2, MUL VL]\n"
+ ".inst 0x647840e9 // bfdot z9.s, z7.h, z0.h[3]\n"
+ ".inst 0x647940ed // bfdot z13.s, z7.h, z1.h[3]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #-1, MUL VL]\n"
+ ".inst 0x647840ca // bfdot z10.s, z6.h, z0.h[3]\n"
+ ".inst 0x647940ce // bfdot z14.s, z6.h, z1.h[3]\n"
+ ".inst 0x647840eb // bfdot z11.s, z7.h, z0.h[3]\n"
+ ".inst 0x647940ef // bfdot z15.s, z7.h, z1.h[3]\n"
+ "bgt 24b\n"
+ "25:" // Height 2: Multiply loop: Single iteration only
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ "whilelt p0.h, XZR, x11\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ "subs x11, x11, #0x2\n"
+ "ld1rqh { z0.h }, p0/Z, [x10]\n"
+ ".inst 0x646040c8 // bfdot z8.s, z6.h, z0.h[0]\n"
+ "ld1rqh { z1.h }, p0/Z, [x28]\n"
+ "add x10, x10, #0x10\n"
+ ".inst 0x646040e9 // bfdot z9.s, z7.h, z0.h[0]\n"
+ "add x28, x28, #0x10\n"
+ ".inst 0x646140cc // bfdot z12.s, z6.h, z1.h[0]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ ".inst 0x646140ed // bfdot z13.s, z7.h, z1.h[0]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ "addvl x15, x15, #4\n"
+ ".inst 0x646040ca // bfdot z10.s, z6.h, z0.h[0]\n"
+ ".inst 0x646140ce // bfdot z14.s, z6.h, z1.h[0]\n"
+ ".inst 0x646040eb // bfdot z11.s, z7.h, z0.h[0]\n"
+ ".inst 0x646140ef // bfdot z15.s, z7.h, z1.h[0]\n"
+ "ble 26f\n"
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ ".inst 0x646840c8 // bfdot z8.s, z6.h, z0.h[1]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ "subs x11, x11, #0x2\n"
+ ".inst 0x646940cc // bfdot z12.s, z6.h, z1.h[1]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ ".inst 0x646840e9 // bfdot z9.s, z7.h, z0.h[1]\n"
+ ".inst 0x646940ed // bfdot z13.s, z7.h, z1.h[1]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ "addvl x15, x15, #4\n"
+ ".inst 0x646840ca // bfdot z10.s, z6.h, z0.h[1]\n"
+ ".inst 0x646940ce // bfdot z14.s, z6.h, z1.h[1]\n"
+ ".inst 0x646840eb // bfdot z11.s, z7.h, z0.h[1]\n"
+ ".inst 0x646940ef // bfdot z15.s, z7.h, z1.h[1]\n"
+ "ble 26f\n"
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ ".inst 0x647040c8 // bfdot z8.s, z6.h, z0.h[2]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ "subs x11, x11, #0x2\n"
+ ".inst 0x647140cc // bfdot z12.s, z6.h, z1.h[2]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ ".inst 0x647040e9 // bfdot z9.s, z7.h, z0.h[2]\n"
+ ".inst 0x647140ed // bfdot z13.s, z7.h, z1.h[2]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ "addvl x15, x15, #4\n"
+ ".inst 0x647040ca // bfdot z10.s, z6.h, z0.h[2]\n"
+ ".inst 0x647140ce // bfdot z14.s, z6.h, z1.h[2]\n"
+ ".inst 0x647040eb // bfdot z11.s, z7.h, z0.h[2]\n"
+ ".inst 0x647140ef // bfdot z15.s, z7.h, z1.h[2]\n"
+ "ble 26f\n"
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ ".inst 0x647840c8 // bfdot z8.s, z6.h, z0.h[3]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ ".inst 0x647940cc // bfdot z12.s, z6.h, z1.h[3]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ ".inst 0x647840e9 // bfdot z9.s, z7.h, z0.h[3]\n"
+ ".inst 0x647940ed // bfdot z13.s, z7.h, z1.h[3]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ "addvl x15, x15, #4\n"
+ ".inst 0x647840ca // bfdot z10.s, z6.h, z0.h[3]\n"
+ ".inst 0x647940ce // bfdot z14.s, z6.h, z1.h[3]\n"
+ ".inst 0x647840eb // bfdot z11.s, z7.h, z0.h[3]\n"
+ ".inst 0x647940ef // bfdot z15.s, z7.h, z1.h[3]\n"
+ "26:" // Height 2: Multiply loop: multiply skip
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x12, x12, #0x1\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "cmp x12, x19\n"
+ "bne 21b\n"
+ "prfm pstl1keep, [x13, #0x0]\n"
+ "prfm pstl1keep, [x9, #0x0]\n"
+ "tbz %x[flags], #1, 27f\n"
+ "add x19, %x[args_ptr], %[offset_min]\n"
+ "ld1rw { z1.s }, p5/Z, [x19]\n"
+ "add x19, %x[args_ptr], %[offset_max]\n"
+ "ld1rw { z0.s }, p5/Z, [x19]\n"
+ "fmin z8.s, p5/M, z8.s, z0.s\n"
+ "fmin z9.s, p5/M, z9.s, z0.s\n"
+ "fmin z10.s, p5/M, z10.s, z0.s\n"
+ "fmin z11.s, p5/M, z11.s, z0.s\n"
+ "fmin z12.s, p5/M, z12.s, z0.s\n"
+ "fmax z8.s, p5/M, z8.s, z1.s\n"
+ "fmax z9.s, p5/M, z9.s, z1.s\n"
+ "fmax z10.s, p5/M, z10.s, z1.s\n"
+ "fmax z11.s, p5/M, z11.s, z1.s\n"
+ "fmax z12.s, p5/M, z12.s, z1.s\n"
+ "fmin z13.s, p5/M, z13.s, z0.s\n"
+ "fmin z14.s, p5/M, z14.s, z0.s\n"
+ "fmin z15.s, p5/M, z15.s, z0.s\n"
+ "fmax z13.s, p5/M, z13.s, z1.s\n"
+ "fmax z14.s, p5/M, z14.s, z1.s\n"
+ "fmax z15.s, p5/M, z15.s, z1.s\n"
+ "27:" // Height 2: No activation
+ "st1w { z8.s }, p4, [x13]\n"
+ "st1w { z9.s }, p3, [x13, #1, MUL VL]\n"
+ "st1w { z10.s }, p2, [x13, #2, MUL VL]\n"
+ "st1w { z11.s }, p1, [x13, #3, MUL VL]\n"
+ "addvl x13, x13, #4\n"
+ "st1w { z12.s }, p4, [x9]\n"
+ "st1w { z13.s }, p3, [x9, #1, MUL VL]\n"
+ "st1w { z14.s }, p2, [x9, #2, MUL VL]\n"
+ "st1w { z15.s }, p1, [x9, #3, MUL VL]\n"
+ "addvl x9, x9, #4\n"
+ "28:" // Height 2: Writeback done
+ "mov x19, #0x0\n"
+ "incw x19, ALL, MUL #4\n"
+ "subs x16, x16, x19\n"
+ "bgt 17b\n"
+ "b 86f\n"
+ "29:" // Height 3
+ "ldr x16, [%x[args_ptr], %[offsetof_N]]\n"
+ "mov x14, %x[bias]\n"
+ "ldr x15, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 30f\n"
+ "ldr x13, [%x[output_ptr], #0x0]\n"
+ "add x13, x13, x19, LSL #2\n"
+ "ldr x9, [%x[output_ptr], #0x8]\n"
+ "ldr x27, [%x[output_ptr], #0x10]\n"
+ "add x9, x9, x19, LSL #2\n"
+ "add x27, x27, x19, LSL #2\n"
+ "b 31f\n"
+ "30:" // Height 3: setup direct output
+ "mov x13, %x[output_ptr]\n"
+ "add x9, x13, x19, LSL #2\n"
+ "add x27, x9, x19, LSL #2\n"
+ "31:" // Height 3: Column loop
+ "mov x19, #0x0\n"
+ "whilelt p4.s, x19, x16\n"
+ "incw x19\n"
+ "whilelt p3.s, x19, x16\n"
+ "incw x19\n"
+ "whilelt p2.s, x19, x16\n"
+ "incw x19\n"
+ "whilelt p1.s, x19, x16\n"
+ "cbz x14, 32f\n"
+ "ld1w { z8.s }, p5/Z, [x14]\n"
+ "mov z12.d, z8.d\n"
+ "ld1w { z9.s }, p5/Z, [x14, #1, MUL VL]\n"
+ "mov z16.d, z8.d\n"
+ "ld1w { z10.s }, p5/Z, [x14, #2, MUL VL]\n"
+ "ld1w { z11.s }, p5/Z, [x14, #3, MUL VL]\n"
+ "mov z13.d, z9.d\n"
+ "addvl x14, x14, #4\n"
+ "mov z17.d, z9.d\n"
+ "mov z14.d, z10.d\n"
+ "mov z15.d, z11.d\n"
+ "mov z18.d, z10.d\n"
+ "mov z19.d, z11.d\n"
+ "b 34f\n"
+ "32:" // Height 3: no bias
+ "tbz %x[flags], #0, 33f\n"
+ "ld1w { z8.s }, p4/Z, [x13]\n"
+ "ld1w { z9.s }, p3/Z, [x13, #1, MUL VL]\n"
+ "ld1w { z10.s }, p2/Z, [x13, #2, MUL VL]\n"
+ "ld1w { z11.s }, p1/Z, [x13, #3, MUL VL]\n"
+ "ld1w { z12.s }, p4/Z, [x9]\n"
+ "ld1w { z13.s }, p3/Z, [x9, #1, MUL VL]\n"
+ "ld1w { z14.s }, p2/Z, [x9, #2, MUL VL]\n"
+ "ld1w { z15.s }, p1/Z, [x9, #3, MUL VL]\n"
+ "ld1w { z16.s }, p4/Z, [x27]\n"
+ "ld1w { z17.s }, p3/Z, [x27, #1, MUL VL]\n"
+ "ld1w { z18.s }, p2/Z, [x27, #2, MUL VL]\n"
+ "ld1w { z19.s }, p1/Z, [x27, #3, MUL VL]\n"
+ "b 34f\n"
+ "33:" // Height 3: no accumulate
+ "mov z8.b, #0x0\n"
+ "mov z9.b, #0x0\n"
+ "mov z10.b, #0x0\n"
+ "mov z11.b, #0x0\n"
+ "mov z12.b, #0x0\n"
+ "mov z13.b, #0x0\n"
+ "mov z14.b, #0x0\n"
+ "mov z15.b, #0x0\n"
+ "mov z16.b, #0x0\n"
+ "mov z17.b, #0x0\n"
+ "mov z18.b, #0x0\n"
+ "mov z19.b, #0x0\n"
+ "34:" // Height 3: setup done
+ "mov x12, #0x0\n"
+ "35:" // Height 3: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w11, [x20, x12, LSL #0x2]\n"
+ "tbz %x[flags], #3, 36f\n"
+ "ldr x20, [%x[input_ptr], x12, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x10, [x20, #0x0]\n"
+ "ldr x28, [x20, #0x8]\n"
+ "ldr x26, [x20, #0x10]\n"
+ "cbnz x12, 37f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x10, x10, x19, LSL #1\n"
+ "add x28, x28, x19, LSL #1\n"
+ "add x26, x26, x19, LSL #1\n"
+ "b 37f\n"
+ "36:" // Height 3: setup direct input
+ "mov x10, %x[input_ptr]\n"
+ "add x28, x10, x19, LSL #1\n"
+ "add x26, x28, x19, LSL #1\n"
+ "37:" // Height 3: input setup done
+ "cmp x11, #0x8\n"
+ "ble 39f\n"
+ "38:" // Height 3: Multiply loop: Main loop head
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ "whilelt p0.h, XZR, x11\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ "sub x11, x11, #0x8\n"
+ "ld1rqh { z0.h }, p0/Z, [x10]\n"
+ ".inst 0x646040c8 // bfdot z8.s, z6.h, z0.h[0]\n"
+ "ld1rqh { z1.h }, p0/Z, [x28]\n"
+ "add x10, x10, #0x10\n"
+ ".inst 0x646040e9 // bfdot z9.s, z7.h, z0.h[0]\n"
+ "ld1rqh { z2.h }, p0/Z, [x26]\n"
+ "add x28, x28, #0x10\n"
+ ".inst 0x646140cc // bfdot z12.s, z6.h, z1.h[0]\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x26, x26, #0x10\n"
+ ".inst 0x646240d0 // bfdot z16.s, z6.h, z2.h[0]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ "cmp x11, #0x8\n"
+ ".inst 0x646140ed // bfdot z13.s, z7.h, z1.h[0]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ ".inst 0x646240f1 // bfdot z17.s, z7.h, z2.h[0]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ ".inst 0x646040ca // bfdot z10.s, z6.h, z0.h[0]\n"
+ ".inst 0x646140ce // bfdot z14.s, z6.h, z1.h[0]\n"
+ ".inst 0x646240d2 // bfdot z18.s, z6.h, z2.h[0]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #4, MUL VL]\n"
+ ".inst 0x646040eb // bfdot z11.s, z7.h, z0.h[0]\n"
+ ".inst 0x646140ef // bfdot z15.s, z7.h, z1.h[0]\n"
+ ".inst 0x646240f3 // bfdot z19.s, z7.h, z2.h[0]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #5, MUL VL]\n"
+ ".inst 0x646840c8 // bfdot z8.s, z6.h, z0.h[1]\n"
+ ".inst 0x646940cc // bfdot z12.s, z6.h, z1.h[1]\n"
+ ".inst 0x646a40d0 // bfdot z16.s, z6.h, z2.h[1]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #6, MUL VL]\n"
+ ".inst 0x646840e9 // bfdot z9.s, z7.h, z0.h[1]\n"
+ ".inst 0x646940ed // bfdot z13.s, z7.h, z1.h[1]\n"
+ ".inst 0x646a40f1 // bfdot z17.s, z7.h, z2.h[1]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #7, MUL VL]\n"
+ "addvl x15, x15, #16\n"
+ ".inst 0x646840ca // bfdot z10.s, z6.h, z0.h[1]\n"
+ ".inst 0x646940ce // bfdot z14.s, z6.h, z1.h[1]\n"
+ ".inst 0x646a40d2 // bfdot z18.s, z6.h, z2.h[1]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #-8, MUL VL]\n"
+ ".inst 0x646840eb // bfdot z11.s, z7.h, z0.h[1]\n"
+ ".inst 0x646940ef // bfdot z15.s, z7.h, z1.h[1]\n"
+ ".inst 0x646a40f3 // bfdot z19.s, z7.h, z2.h[1]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #-7, MUL VL]\n"
+ ".inst 0x647040c8 // bfdot z8.s, z6.h, z0.h[2]\n"
+ ".inst 0x647140cc // bfdot z12.s, z6.h, z1.h[2]\n"
+ ".inst 0x647240d0 // bfdot z16.s, z6.h, z2.h[2]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #-6, MUL VL]\n"
+ ".inst 0x647040e9 // bfdot z9.s, z7.h, z0.h[2]\n"
+ ".inst 0x647140ed // bfdot z13.s, z7.h, z1.h[2]\n"
+ ".inst 0x647240f1 // bfdot z17.s, z7.h, z2.h[2]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #-5, MUL VL]\n"
+ ".inst 0x647040ca // bfdot z10.s, z6.h, z0.h[2]\n"
+ ".inst 0x647140ce // bfdot z14.s, z6.h, z1.h[2]\n"
+ ".inst 0x647240d2 // bfdot z18.s, z6.h, z2.h[2]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #-4, MUL VL]\n"
+ ".inst 0x647040eb // bfdot z11.s, z7.h, z0.h[2]\n"
+ ".inst 0x647140ef // bfdot z15.s, z7.h, z1.h[2]\n"
+ ".inst 0x647240f3 // bfdot z19.s, z7.h, z2.h[2]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #-3, MUL VL]\n"
+ ".inst 0x647840c8 // bfdot z8.s, z6.h, z0.h[3]\n"
+ ".inst 0x647940cc // bfdot z12.s, z6.h, z1.h[3]\n"
+ ".inst 0x647a40d0 // bfdot z16.s, z6.h, z2.h[3]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #-2, MUL VL]\n"
+ ".inst 0x647840e9 // bfdot z9.s, z7.h, z0.h[3]\n"
+ ".inst 0x647940ed // bfdot z13.s, z7.h, z1.h[3]\n"
+ ".inst 0x647a40f1 // bfdot z17.s, z7.h, z2.h[3]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #-1, MUL VL]\n"
+ ".inst 0x647840ca // bfdot z10.s, z6.h, z0.h[3]\n"
+ ".inst 0x647940ce // bfdot z14.s, z6.h, z1.h[3]\n"
+ ".inst 0x647a40d2 // bfdot z18.s, z6.h, z2.h[3]\n"
+ ".inst 0x647840eb // bfdot z11.s, z7.h, z0.h[3]\n"
+ ".inst 0x647940ef // bfdot z15.s, z7.h, z1.h[3]\n"
+ ".inst 0x647a40f3 // bfdot z19.s, z7.h, z2.h[3]\n"
+ "bgt 38b\n"
+ "39:" // Height 3: Multiply loop: Single iteration only
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ "whilelt p0.h, XZR, x11\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ "subs x11, x11, #0x2\n"
+ "ld1rqh { z0.h }, p0/Z, [x10]\n"
+ ".inst 0x646040c8 // bfdot z8.s, z6.h, z0.h[0]\n"
+ "ld1rqh { z1.h }, p0/Z, [x28]\n"
+ "add x10, x10, #0x10\n"
+ ".inst 0x646040e9 // bfdot z9.s, z7.h, z0.h[0]\n"
+ "ld1rqh { z2.h }, p0/Z, [x26]\n"
+ "add x28, x28, #0x10\n"
+ ".inst 0x646140cc // bfdot z12.s, z6.h, z1.h[0]\n"
+ "add x26, x26, #0x10\n"
+ ".inst 0x646140ed // bfdot z13.s, z7.h, z1.h[0]\n"
+ ".inst 0x646240d0 // bfdot z16.s, z6.h, z2.h[0]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ ".inst 0x646240f1 // bfdot z17.s, z7.h, z2.h[0]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ "addvl x15, x15, #4\n"
+ ".inst 0x646040ca // bfdot z10.s, z6.h, z0.h[0]\n"
+ ".inst 0x646140ce // bfdot z14.s, z6.h, z1.h[0]\n"
+ ".inst 0x646240d2 // bfdot z18.s, z6.h, z2.h[0]\n"
+ ".inst 0x646040eb // bfdot z11.s, z7.h, z0.h[0]\n"
+ ".inst 0x646140ef // bfdot z15.s, z7.h, z1.h[0]\n"
+ ".inst 0x646240f3 // bfdot z19.s, z7.h, z2.h[0]\n"
+ "ble 40f\n"
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ ".inst 0x646840c8 // bfdot z8.s, z6.h, z0.h[1]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ "subs x11, x11, #0x2\n"
+ ".inst 0x646940cc // bfdot z12.s, z6.h, z1.h[1]\n"
+ ".inst 0x646a40d0 // bfdot z16.s, z6.h, z2.h[1]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ ".inst 0x646840e9 // bfdot z9.s, z7.h, z0.h[1]\n"
+ ".inst 0x646940ed // bfdot z13.s, z7.h, z1.h[1]\n"
+ ".inst 0x646a40f1 // bfdot z17.s, z7.h, z2.h[1]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ "addvl x15, x15, #4\n"
+ ".inst 0x646840ca // bfdot z10.s, z6.h, z0.h[1]\n"
+ ".inst 0x646940ce // bfdot z14.s, z6.h, z1.h[1]\n"
+ ".inst 0x646a40d2 // bfdot z18.s, z6.h, z2.h[1]\n"
+ ".inst 0x646840eb // bfdot z11.s, z7.h, z0.h[1]\n"
+ ".inst 0x646940ef // bfdot z15.s, z7.h, z1.h[1]\n"
+ ".inst 0x646a40f3 // bfdot z19.s, z7.h, z2.h[1]\n"
+ "ble 40f\n"
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ ".inst 0x647040c8 // bfdot z8.s, z6.h, z0.h[2]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ "subs x11, x11, #0x2\n"
+ ".inst 0x647140cc // bfdot z12.s, z6.h, z1.h[2]\n"
+ ".inst 0x647240d0 // bfdot z16.s, z6.h, z2.h[2]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ ".inst 0x647040e9 // bfdot z9.s, z7.h, z0.h[2]\n"
+ ".inst 0x647140ed // bfdot z13.s, z7.h, z1.h[2]\n"
+ ".inst 0x647240f1 // bfdot z17.s, z7.h, z2.h[2]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ "addvl x15, x15, #4\n"
+ ".inst 0x647040ca // bfdot z10.s, z6.h, z0.h[2]\n"
+ ".inst 0x647140ce // bfdot z14.s, z6.h, z1.h[2]\n"
+ ".inst 0x647240d2 // bfdot z18.s, z6.h, z2.h[2]\n"
+ ".inst 0x647040eb // bfdot z11.s, z7.h, z0.h[2]\n"
+ ".inst 0x647140ef // bfdot z15.s, z7.h, z1.h[2]\n"
+ ".inst 0x647240f3 // bfdot z19.s, z7.h, z2.h[2]\n"
+ "ble 40f\n"
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ ".inst 0x647840c8 // bfdot z8.s, z6.h, z0.h[3]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ ".inst 0x647940cc // bfdot z12.s, z6.h, z1.h[3]\n"
+ ".inst 0x647a40d0 // bfdot z16.s, z6.h, z2.h[3]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ ".inst 0x647840e9 // bfdot z9.s, z7.h, z0.h[3]\n"
+ ".inst 0x647940ed // bfdot z13.s, z7.h, z1.h[3]\n"
+ ".inst 0x647a40f1 // bfdot z17.s, z7.h, z2.h[3]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ "addvl x15, x15, #4\n"
+ ".inst 0x647840ca // bfdot z10.s, z6.h, z0.h[3]\n"
+ ".inst 0x647940ce // bfdot z14.s, z6.h, z1.h[3]\n"
+ ".inst 0x647a40d2 // bfdot z18.s, z6.h, z2.h[3]\n"
+ ".inst 0x647840eb // bfdot z11.s, z7.h, z0.h[3]\n"
+ ".inst 0x647940ef // bfdot z15.s, z7.h, z1.h[3]\n"
+ ".inst 0x647a40f3 // bfdot z19.s, z7.h, z2.h[3]\n"
+ "40:" // Height 3: Multiply loop: multiply skip
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x12, x12, #0x1\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "cmp x12, x19\n"
+ "bne 35b\n"
+ "prfm pstl1keep, [x13, #0x0]\n"
+ "prfm pstl1keep, [x9, #0x0]\n"
+ "prfm pstl1keep, [x27, #0x0]\n"
+ "tbz %x[flags], #1, 41f\n"
+ "add x19, %x[args_ptr], %[offset_min]\n"
+ "ld1rw { z1.s }, p5/Z, [x19]\n"
+ "add x19, %x[args_ptr], %[offset_max]\n"
+ "ld1rw { z0.s }, p5/Z, [x19]\n"
+ "fmin z8.s, p5/M, z8.s, z0.s\n"
+ "fmin z9.s, p5/M, z9.s, z0.s\n"
+ "fmin z10.s, p5/M, z10.s, z0.s\n"
+ "fmin z11.s, p5/M, z11.s, z0.s\n"
+ "fmin z12.s, p5/M, z12.s, z0.s\n"
+ "fmax z8.s, p5/M, z8.s, z1.s\n"
+ "fmax z9.s, p5/M, z9.s, z1.s\n"
+ "fmax z10.s, p5/M, z10.s, z1.s\n"
+ "fmax z11.s, p5/M, z11.s, z1.s\n"
+ "fmax z12.s, p5/M, z12.s, z1.s\n"
+ "fmin z13.s, p5/M, z13.s, z0.s\n"
+ "fmin z14.s, p5/M, z14.s, z0.s\n"
+ "fmin z15.s, p5/M, z15.s, z0.s\n"
+ "fmin z16.s, p5/M, z16.s, z0.s\n"
+ "fmax z13.s, p5/M, z13.s, z1.s\n"
+ "fmax z14.s, p5/M, z14.s, z1.s\n"
+ "fmax z15.s, p5/M, z15.s, z1.s\n"
+ "fmax z16.s, p5/M, z16.s, z1.s\n"
+ "fmin z17.s, p5/M, z17.s, z0.s\n"
+ "fmin z18.s, p5/M, z18.s, z0.s\n"
+ "fmin z19.s, p5/M, z19.s, z0.s\n"
+ "fmax z17.s, p5/M, z17.s, z1.s\n"
+ "fmax z18.s, p5/M, z18.s, z1.s\n"
+ "fmax z19.s, p5/M, z19.s, z1.s\n"
+ "41:" // Height 3: No activation
+ "st1w { z8.s }, p4, [x13]\n"
+ "st1w { z9.s }, p3, [x13, #1, MUL VL]\n"
+ "st1w { z10.s }, p2, [x13, #2, MUL VL]\n"
+ "st1w { z11.s }, p1, [x13, #3, MUL VL]\n"
+ "addvl x13, x13, #4\n"
+ "st1w { z12.s }, p4, [x9]\n"
+ "st1w { z13.s }, p3, [x9, #1, MUL VL]\n"
+ "st1w { z14.s }, p2, [x9, #2, MUL VL]\n"
+ "st1w { z15.s }, p1, [x9, #3, MUL VL]\n"
+ "addvl x9, x9, #4\n"
+ "st1w { z16.s }, p4, [x27]\n"
+ "st1w { z17.s }, p3, [x27, #1, MUL VL]\n"
+ "st1w { z18.s }, p2, [x27, #2, MUL VL]\n"
+ "st1w { z19.s }, p1, [x27, #3, MUL VL]\n"
+ "addvl x27, x27, #4\n"
+ "42:" // Height 3: Writeback done
+ "mov x19, #0x0\n"
+ "incw x19, ALL, MUL #4\n"
+ "subs x16, x16, x19\n"
+ "bgt 31b\n"
+ "b 86f\n"
+ "43:" // Height 4
+ "ldr x16, [%x[args_ptr], %[offsetof_N]]\n"
+ "mov x14, %x[bias]\n"
+ "ldr x15, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 44f\n"
+ "ldr x13, [%x[output_ptr], #0x0]\n"
+ "add x13, x13, x19, LSL #2\n"
+ "ldr x9, [%x[output_ptr], #0x8]\n"
+ "ldr x27, [%x[output_ptr], #0x10]\n"
+ "add x9, x9, x19, LSL #2\n"
+ "ldr x25, [%x[output_ptr], #0x18]\n"
+ "add x27, x27, x19, LSL #2\n"
+ "add x25, x25, x19, LSL #2\n"
+ "b 45f\n"
+ "44:" // Height 4: setup direct output
+ "mov x13, %x[output_ptr]\n"
+ "add x9, x13, x19, LSL #2\n"
+ "add x27, x9, x19, LSL #2\n"
+ "add x25, x27, x19, LSL #2\n"
+ "45:" // Height 4: Column loop
+ "mov x19, #0x0\n"
+ "whilelt p4.s, x19, x16\n"
+ "incw x19\n"
+ "whilelt p3.s, x19, x16\n"
+ "incw x19\n"
+ "whilelt p2.s, x19, x16\n"
+ "incw x19\n"
+ "whilelt p1.s, x19, x16\n"
+ "cbz x14, 46f\n"
+ "ld1w { z8.s }, p5/Z, [x14]\n"
+ "mov z12.d, z8.d\n"
+ "ld1w { z9.s }, p5/Z, [x14, #1, MUL VL]\n"
+ "mov z16.d, z8.d\n"
+ "ld1w { z10.s }, p5/Z, [x14, #2, MUL VL]\n"
+ "mov z20.d, z8.d\n"
+ "ld1w { z11.s }, p5/Z, [x14, #3, MUL VL]\n"
+ "addvl x14, x14, #4\n"
+ "mov z13.d, z9.d\n"
+ "mov z17.d, z9.d\n"
+ "mov z14.d, z10.d\n"
+ "mov z15.d, z11.d\n"
+ "mov z18.d, z10.d\n"
+ "mov z19.d, z11.d\n"
+ "mov z21.d, z9.d\n"
+ "mov z22.d, z10.d\n"
+ "mov z23.d, z11.d\n"
+ "b 48f\n"
+ "46:" // Height 4: no bias
+ "tbz %x[flags], #0, 47f\n"
+ "ld1w { z8.s }, p4/Z, [x13]\n"
+ "ld1w { z9.s }, p3/Z, [x13, #1, MUL VL]\n"
+ "ld1w { z10.s }, p2/Z, [x13, #2, MUL VL]\n"
+ "ld1w { z11.s }, p1/Z, [x13, #3, MUL VL]\n"
+ "ld1w { z12.s }, p4/Z, [x9]\n"
+ "ld1w { z13.s }, p3/Z, [x9, #1, MUL VL]\n"
+ "ld1w { z14.s }, p2/Z, [x9, #2, MUL VL]\n"
+ "ld1w { z15.s }, p1/Z, [x9, #3, MUL VL]\n"
+ "ld1w { z16.s }, p4/Z, [x27]\n"
+ "ld1w { z17.s }, p3/Z, [x27, #1, MUL VL]\n"
+ "ld1w { z18.s }, p2/Z, [x27, #2, MUL VL]\n"
+ "ld1w { z19.s }, p1/Z, [x27, #3, MUL VL]\n"
+ "ld1w { z20.s }, p4/Z, [x25]\n"
+ "ld1w { z21.s }, p3/Z, [x25, #1, MUL VL]\n"
+ "ld1w { z22.s }, p2/Z, [x25, #2, MUL VL]\n"
+ "ld1w { z23.s }, p1/Z, [x25, #3, MUL VL]\n"
+ "b 48f\n"
+ "47:" // Height 4: no accumulate
+ "mov z8.b, #0x0\n"
+ "mov z9.b, #0x0\n"
+ "mov z10.b, #0x0\n"
+ "mov z11.b, #0x0\n"
+ "mov z12.b, #0x0\n"
+ "mov z13.b, #0x0\n"
+ "mov z14.b, #0x0\n"
+ "mov z15.b, #0x0\n"
+ "mov z16.b, #0x0\n"
+ "mov z17.b, #0x0\n"
+ "mov z18.b, #0x0\n"
+ "mov z19.b, #0x0\n"
+ "mov z20.b, #0x0\n"
+ "mov z21.b, #0x0\n"
+ "mov z22.b, #0x0\n"
+ "mov z23.b, #0x0\n"
+ "48:" // Height 4: setup done
+ "mov x12, #0x0\n"
+ "49:" // Height 4: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w11, [x20, x12, LSL #0x2]\n"
+ "tbz %x[flags], #3, 50f\n"
+ "ldr x20, [%x[input_ptr], x12, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x10, [x20, #0x0]\n"
+ "ldr x28, [x20, #0x8]\n"
+ "ldr x26, [x20, #0x10]\n"
+ "ldr x24, [x20, #0x18]\n"
+ "cbnz x12, 51f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x10, x10, x19, LSL #1\n"
+ "add x28, x28, x19, LSL #1\n"
+ "add x26, x26, x19, LSL #1\n"
+ "add x24, x24, x19, LSL #1\n"
+ "b 51f\n"
+ "50:" // Height 4: setup direct input
+ "mov x10, %x[input_ptr]\n"
+ "add x28, x10, x19, LSL #1\n"
+ "add x26, x28, x19, LSL #1\n"
+ "add x24, x26, x19, LSL #1\n"
+ "51:" // Height 4: input setup done
+ "cmp x11, #0x8\n"
+ "ble 53f\n"
+ "52:" // Height 4: Multiply loop: Main loop head
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ "whilelt p0.h, XZR, x11\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ "sub x11, x11, #0x8\n"
+ "ld1rqh { z0.h }, p0/Z, [x10]\n"
+ ".inst 0x646040c8 // bfdot z8.s, z6.h, z0.h[0]\n"
+ "ld1rqh { z1.h }, p0/Z, [x28]\n"
+ "add x10, x10, #0x10\n"
+ ".inst 0x646040e9 // bfdot z9.s, z7.h, z0.h[0]\n"
+ "ld1rqh { z2.h }, p0/Z, [x26]\n"
+ "add x28, x28, #0x10\n"
+ ".inst 0x646140cc // bfdot z12.s, z6.h, z1.h[0]\n"
+ "ld1rqh { z3.h }, p0/Z, [x24]\n"
+ "add x26, x26, #0x10\n"
+ ".inst 0x646240d0 // bfdot z16.s, z6.h, z2.h[0]\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x24, x24, #0x10\n"
+ ".inst 0x646140ed // bfdot z13.s, z7.h, z1.h[0]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "cmp x11, #0x8\n"
+ ".inst 0x646340d4 // bfdot z20.s, z6.h, z3.h[0]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ ".inst 0x646240f1 // bfdot z17.s, z7.h, z2.h[0]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ ".inst 0x646340f5 // bfdot z21.s, z7.h, z3.h[0]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ ".inst 0x646040ca // bfdot z10.s, z6.h, z0.h[0]\n"
+ ".inst 0x646140ce // bfdot z14.s, z6.h, z1.h[0]\n"
+ ".inst 0x646240d2 // bfdot z18.s, z6.h, z2.h[0]\n"
+ ".inst 0x646340d6 // bfdot z22.s, z6.h, z3.h[0]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #4, MUL VL]\n"
+ ".inst 0x646040eb // bfdot z11.s, z7.h, z0.h[0]\n"
+ ".inst 0x646140ef // bfdot z15.s, z7.h, z1.h[0]\n"
+ ".inst 0x646240f3 // bfdot z19.s, z7.h, z2.h[0]\n"
+ ".inst 0x646340f7 // bfdot z23.s, z7.h, z3.h[0]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #5, MUL VL]\n"
+ ".inst 0x646840c8 // bfdot z8.s, z6.h, z0.h[1]\n"
+ ".inst 0x646940cc // bfdot z12.s, z6.h, z1.h[1]\n"
+ ".inst 0x646a40d0 // bfdot z16.s, z6.h, z2.h[1]\n"
+ ".inst 0x646b40d4 // bfdot z20.s, z6.h, z3.h[1]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #6, MUL VL]\n"
+ ".inst 0x646840e9 // bfdot z9.s, z7.h, z0.h[1]\n"
+ ".inst 0x646940ed // bfdot z13.s, z7.h, z1.h[1]\n"
+ ".inst 0x646a40f1 // bfdot z17.s, z7.h, z2.h[1]\n"
+ ".inst 0x646b40f5 // bfdot z21.s, z7.h, z3.h[1]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #7, MUL VL]\n"
+ "addvl x15, x15, #16\n"
+ ".inst 0x646840ca // bfdot z10.s, z6.h, z0.h[1]\n"
+ ".inst 0x646940ce // bfdot z14.s, z6.h, z1.h[1]\n"
+ ".inst 0x646a40d2 // bfdot z18.s, z6.h, z2.h[1]\n"
+ ".inst 0x646b40d6 // bfdot z22.s, z6.h, z3.h[1]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #-8, MUL VL]\n"
+ ".inst 0x646840eb // bfdot z11.s, z7.h, z0.h[1]\n"
+ ".inst 0x646940ef // bfdot z15.s, z7.h, z1.h[1]\n"
+ ".inst 0x646a40f3 // bfdot z19.s, z7.h, z2.h[1]\n"
+ ".inst 0x646b40f7 // bfdot z23.s, z7.h, z3.h[1]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #-7, MUL VL]\n"
+ ".inst 0x647040c8 // bfdot z8.s, z6.h, z0.h[2]\n"
+ ".inst 0x647140cc // bfdot z12.s, z6.h, z1.h[2]\n"
+ ".inst 0x647240d0 // bfdot z16.s, z6.h, z2.h[2]\n"
+ ".inst 0x647340d4 // bfdot z20.s, z6.h, z3.h[2]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #-6, MUL VL]\n"
+ ".inst 0x647040e9 // bfdot z9.s, z7.h, z0.h[2]\n"
+ ".inst 0x647140ed // bfdot z13.s, z7.h, z1.h[2]\n"
+ ".inst 0x647240f1 // bfdot z17.s, z7.h, z2.h[2]\n"
+ ".inst 0x647340f5 // bfdot z21.s, z7.h, z3.h[2]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #-5, MUL VL]\n"
+ ".inst 0x647040ca // bfdot z10.s, z6.h, z0.h[2]\n"
+ ".inst 0x647140ce // bfdot z14.s, z6.h, z1.h[2]\n"
+ ".inst 0x647240d2 // bfdot z18.s, z6.h, z2.h[2]\n"
+ ".inst 0x647340d6 // bfdot z22.s, z6.h, z3.h[2]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #-4, MUL VL]\n"
+ ".inst 0x647040eb // bfdot z11.s, z7.h, z0.h[2]\n"
+ ".inst 0x647140ef // bfdot z15.s, z7.h, z1.h[2]\n"
+ ".inst 0x647240f3 // bfdot z19.s, z7.h, z2.h[2]\n"
+ ".inst 0x647340f7 // bfdot z23.s, z7.h, z3.h[2]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #-3, MUL VL]\n"
+ ".inst 0x647840c8 // bfdot z8.s, z6.h, z0.h[3]\n"
+ ".inst 0x647940cc // bfdot z12.s, z6.h, z1.h[3]\n"
+ ".inst 0x647a40d0 // bfdot z16.s, z6.h, z2.h[3]\n"
+ ".inst 0x647b40d4 // bfdot z20.s, z6.h, z3.h[3]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #-2, MUL VL]\n"
+ ".inst 0x647840e9 // bfdot z9.s, z7.h, z0.h[3]\n"
+ ".inst 0x647940ed // bfdot z13.s, z7.h, z1.h[3]\n"
+ ".inst 0x647a40f1 // bfdot z17.s, z7.h, z2.h[3]\n"
+ ".inst 0x647b40f5 // bfdot z21.s, z7.h, z3.h[3]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #-1, MUL VL]\n"
+ ".inst 0x647840ca // bfdot z10.s, z6.h, z0.h[3]\n"
+ ".inst 0x647940ce // bfdot z14.s, z6.h, z1.h[3]\n"
+ ".inst 0x647a40d2 // bfdot z18.s, z6.h, z2.h[3]\n"
+ ".inst 0x647b40d6 // bfdot z22.s, z6.h, z3.h[3]\n"
+ ".inst 0x647840eb // bfdot z11.s, z7.h, z0.h[3]\n"
+ ".inst 0x647940ef // bfdot z15.s, z7.h, z1.h[3]\n"
+ ".inst 0x647a40f3 // bfdot z19.s, z7.h, z2.h[3]\n"
+ ".inst 0x647b40f7 // bfdot z23.s, z7.h, z3.h[3]\n"
+ "bgt 52b\n"
+ "53:" // Height 4: Multiply loop: Single iteration only
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ "whilelt p0.h, XZR, x11\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ "subs x11, x11, #0x2\n"
+ "ld1rqh { z0.h }, p0/Z, [x10]\n"
+ ".inst 0x646040c8 // bfdot z8.s, z6.h, z0.h[0]\n"
+ "ld1rqh { z1.h }, p0/Z, [x28]\n"
+ "add x10, x10, #0x10\n"
+ ".inst 0x646040e9 // bfdot z9.s, z7.h, z0.h[0]\n"
+ "ld1rqh { z2.h }, p0/Z, [x26]\n"
+ "add x28, x28, #0x10\n"
+ ".inst 0x646140cc // bfdot z12.s, z6.h, z1.h[0]\n"
+ "ld1rqh { z3.h }, p0/Z, [x24]\n"
+ "add x26, x26, #0x10\n"
+ ".inst 0x646240d0 // bfdot z16.s, z6.h, z2.h[0]\n"
+ "add x24, x24, #0x10\n"
+ ".inst 0x646140ed // bfdot z13.s, z7.h, z1.h[0]\n"
+ ".inst 0x646240f1 // bfdot z17.s, z7.h, z2.h[0]\n"
+ ".inst 0x646340d4 // bfdot z20.s, z6.h, z3.h[0]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ ".inst 0x646340f5 // bfdot z21.s, z7.h, z3.h[0]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ "addvl x15, x15, #4\n"
+ ".inst 0x646040ca // bfdot z10.s, z6.h, z0.h[0]\n"
+ ".inst 0x646140ce // bfdot z14.s, z6.h, z1.h[0]\n"
+ ".inst 0x646240d2 // bfdot z18.s, z6.h, z2.h[0]\n"
+ ".inst 0x646340d6 // bfdot z22.s, z6.h, z3.h[0]\n"
+ ".inst 0x646040eb // bfdot z11.s, z7.h, z0.h[0]\n"
+ ".inst 0x646140ef // bfdot z15.s, z7.h, z1.h[0]\n"
+ ".inst 0x646240f3 // bfdot z19.s, z7.h, z2.h[0]\n"
+ ".inst 0x646340f7 // bfdot z23.s, z7.h, z3.h[0]\n"
+ "ble 54f\n"
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ ".inst 0x646840c8 // bfdot z8.s, z6.h, z0.h[1]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ "subs x11, x11, #0x2\n"
+ ".inst 0x646940cc // bfdot z12.s, z6.h, z1.h[1]\n"
+ ".inst 0x646a40d0 // bfdot z16.s, z6.h, z2.h[1]\n"
+ ".inst 0x646b40d4 // bfdot z20.s, z6.h, z3.h[1]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ ".inst 0x646840e9 // bfdot z9.s, z7.h, z0.h[1]\n"
+ ".inst 0x646940ed // bfdot z13.s, z7.h, z1.h[1]\n"
+ ".inst 0x646a40f1 // bfdot z17.s, z7.h, z2.h[1]\n"
+ ".inst 0x646b40f5 // bfdot z21.s, z7.h, z3.h[1]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ "addvl x15, x15, #4\n"
+ ".inst 0x646840ca // bfdot z10.s, z6.h, z0.h[1]\n"
+ ".inst 0x646940ce // bfdot z14.s, z6.h, z1.h[1]\n"
+ ".inst 0x646a40d2 // bfdot z18.s, z6.h, z2.h[1]\n"
+ ".inst 0x646b40d6 // bfdot z22.s, z6.h, z3.h[1]\n"
+ ".inst 0x646840eb // bfdot z11.s, z7.h, z0.h[1]\n"
+ ".inst 0x646940ef // bfdot z15.s, z7.h, z1.h[1]\n"
+ ".inst 0x646a40f3 // bfdot z19.s, z7.h, z2.h[1]\n"
+ ".inst 0x646b40f7 // bfdot z23.s, z7.h, z3.h[1]\n"
+ "ble 54f\n"
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ ".inst 0x647040c8 // bfdot z8.s, z6.h, z0.h[2]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ "subs x11, x11, #0x2\n"
+ ".inst 0x647140cc // bfdot z12.s, z6.h, z1.h[2]\n"
+ ".inst 0x647240d0 // bfdot z16.s, z6.h, z2.h[2]\n"
+ ".inst 0x647340d4 // bfdot z20.s, z6.h, z3.h[2]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ ".inst 0x647040e9 // bfdot z9.s, z7.h, z0.h[2]\n"
+ ".inst 0x647140ed // bfdot z13.s, z7.h, z1.h[2]\n"
+ ".inst 0x647240f1 // bfdot z17.s, z7.h, z2.h[2]\n"
+ ".inst 0x647340f5 // bfdot z21.s, z7.h, z3.h[2]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ "addvl x15, x15, #4\n"
+ ".inst 0x647040ca // bfdot z10.s, z6.h, z0.h[2]\n"
+ ".inst 0x647140ce // bfdot z14.s, z6.h, z1.h[2]\n"
+ ".inst 0x647240d2 // bfdot z18.s, z6.h, z2.h[2]\n"
+ ".inst 0x647340d6 // bfdot z22.s, z6.h, z3.h[2]\n"
+ ".inst 0x647040eb // bfdot z11.s, z7.h, z0.h[2]\n"
+ ".inst 0x647140ef // bfdot z15.s, z7.h, z1.h[2]\n"
+ ".inst 0x647240f3 // bfdot z19.s, z7.h, z2.h[2]\n"
+ ".inst 0x647340f7 // bfdot z23.s, z7.h, z3.h[2]\n"
+ "ble 54f\n"
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ ".inst 0x647840c8 // bfdot z8.s, z6.h, z0.h[3]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ ".inst 0x647940cc // bfdot z12.s, z6.h, z1.h[3]\n"
+ ".inst 0x647a40d0 // bfdot z16.s, z6.h, z2.h[3]\n"
+ ".inst 0x647b40d4 // bfdot z20.s, z6.h, z3.h[3]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ ".inst 0x647840e9 // bfdot z9.s, z7.h, z0.h[3]\n"
+ ".inst 0x647940ed // bfdot z13.s, z7.h, z1.h[3]\n"
+ ".inst 0x647a40f1 // bfdot z17.s, z7.h, z2.h[3]\n"
+ ".inst 0x647b40f5 // bfdot z21.s, z7.h, z3.h[3]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ "addvl x15, x15, #4\n"
+ ".inst 0x647840ca // bfdot z10.s, z6.h, z0.h[3]\n"
+ ".inst 0x647940ce // bfdot z14.s, z6.h, z1.h[3]\n"
+ ".inst 0x647a40d2 // bfdot z18.s, z6.h, z2.h[3]\n"
+ ".inst 0x647b40d6 // bfdot z22.s, z6.h, z3.h[3]\n"
+ ".inst 0x647840eb // bfdot z11.s, z7.h, z0.h[3]\n"
+ ".inst 0x647940ef // bfdot z15.s, z7.h, z1.h[3]\n"
+ ".inst 0x647a40f3 // bfdot z19.s, z7.h, z2.h[3]\n"
+ ".inst 0x647b40f7 // bfdot z23.s, z7.h, z3.h[3]\n"
+ "54:" // Height 4: Multiply loop: multiply skip
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x12, x12, #0x1\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "cmp x12, x19\n"
+ "bne 49b\n"
+ "prfm pstl1keep, [x13, #0x0]\n"
+ "prfm pstl1keep, [x9, #0x0]\n"
+ "prfm pstl1keep, [x27, #0x0]\n"
+ "prfm pstl1keep, [x25, #0x0]\n"
+ "tbz %x[flags], #1, 55f\n"
+ "add x19, %x[args_ptr], %[offset_min]\n"
+ "ld1rw { z1.s }, p5/Z, [x19]\n"
+ "add x19, %x[args_ptr], %[offset_max]\n"
+ "ld1rw { z0.s }, p5/Z, [x19]\n"
+ "fmin z8.s, p5/M, z8.s, z0.s\n"
+ "fmin z9.s, p5/M, z9.s, z0.s\n"
+ "fmin z10.s, p5/M, z10.s, z0.s\n"
+ "fmin z11.s, p5/M, z11.s, z0.s\n"
+ "fmin z12.s, p5/M, z12.s, z0.s\n"
+ "fmax z8.s, p5/M, z8.s, z1.s\n"
+ "fmax z9.s, p5/M, z9.s, z1.s\n"
+ "fmax z10.s, p5/M, z10.s, z1.s\n"
+ "fmax z11.s, p5/M, z11.s, z1.s\n"
+ "fmax z12.s, p5/M, z12.s, z1.s\n"
+ "fmin z13.s, p5/M, z13.s, z0.s\n"
+ "fmin z14.s, p5/M, z14.s, z0.s\n"
+ "fmin z15.s, p5/M, z15.s, z0.s\n"
+ "fmin z16.s, p5/M, z16.s, z0.s\n"
+ "fmax z13.s, p5/M, z13.s, z1.s\n"
+ "fmax z14.s, p5/M, z14.s, z1.s\n"
+ "fmax z15.s, p5/M, z15.s, z1.s\n"
+ "fmax z16.s, p5/M, z16.s, z1.s\n"
+ "fmin z17.s, p5/M, z17.s, z0.s\n"
+ "fmin z18.s, p5/M, z18.s, z0.s\n"
+ "fmin z19.s, p5/M, z19.s, z0.s\n"
+ "fmin z20.s, p5/M, z20.s, z0.s\n"
+ "fmax z17.s, p5/M, z17.s, z1.s\n"
+ "fmax z18.s, p5/M, z18.s, z1.s\n"
+ "fmax z19.s, p5/M, z19.s, z1.s\n"
+ "fmax z20.s, p5/M, z20.s, z1.s\n"
+ "fmin z21.s, p5/M, z21.s, z0.s\n"
+ "fmin z22.s, p5/M, z22.s, z0.s\n"
+ "fmin z23.s, p5/M, z23.s, z0.s\n"
+ "fmax z21.s, p5/M, z21.s, z1.s\n"
+ "fmax z22.s, p5/M, z22.s, z1.s\n"
+ "fmax z23.s, p5/M, z23.s, z1.s\n"
+ "55:" // Height 4: No activation
+ "st1w { z8.s }, p4, [x13]\n"
+ "st1w { z9.s }, p3, [x13, #1, MUL VL]\n"
+ "st1w { z10.s }, p2, [x13, #2, MUL VL]\n"
+ "st1w { z11.s }, p1, [x13, #3, MUL VL]\n"
+ "addvl x13, x13, #4\n"
+ "st1w { z12.s }, p4, [x9]\n"
+ "st1w { z13.s }, p3, [x9, #1, MUL VL]\n"
+ "st1w { z14.s }, p2, [x9, #2, MUL VL]\n"
+ "st1w { z15.s }, p1, [x9, #3, MUL VL]\n"
+ "addvl x9, x9, #4\n"
+ "st1w { z16.s }, p4, [x27]\n"
+ "st1w { z17.s }, p3, [x27, #1, MUL VL]\n"
+ "st1w { z18.s }, p2, [x27, #2, MUL VL]\n"
+ "st1w { z19.s }, p1, [x27, #3, MUL VL]\n"
+ "addvl x27, x27, #4\n"
+ "st1w { z20.s }, p4, [x25]\n"
+ "st1w { z21.s }, p3, [x25, #1, MUL VL]\n"
+ "st1w { z22.s }, p2, [x25, #2, MUL VL]\n"
+ "st1w { z23.s }, p1, [x25, #3, MUL VL]\n"
+ "addvl x25, x25, #4\n"
+ "56:" // Height 4: Writeback done
+ "mov x19, #0x0\n"
+ "incw x19, ALL, MUL #4\n"
+ "subs x16, x16, x19\n"
+ "bgt 45b\n"
+ "b 86f\n"
+ "57:" // Height 5
+ "ldr x16, [%x[args_ptr], %[offsetof_N]]\n"
+ "mov x14, %x[bias]\n"
+ "ldr x15, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 58f\n"
+ "ldr x13, [%x[output_ptr], #0x0]\n"
+ "add x13, x13, x19, LSL #2\n"
+ "ldr x9, [%x[output_ptr], #0x8]\n"
+ "ldr x27, [%x[output_ptr], #0x10]\n"
+ "add x9, x9, x19, LSL #2\n"
+ "ldr x25, [%x[output_ptr], #0x18]\n"
+ "ldr x23, [%x[output_ptr], #0x20]\n"
+ "add x27, x27, x19, LSL #2\n"
+ "add x25, x25, x19, LSL #2\n"
+ "add x23, x23, x19, LSL #2\n"
+ "b 59f\n"
+ "58:" // Height 5: setup direct output
+ "mov x13, %x[output_ptr]\n"
+ "add x9, x13, x19, LSL #2\n"
+ "add x27, x9, x19, LSL #2\n"
+ "add x25, x27, x19, LSL #2\n"
+ "add x23, x25, x19, LSL #2\n"
+ "59:" // Height 5: Column loop
+ "mov x19, #0x0\n"
+ "whilelt p4.s, x19, x16\n"
+ "incw x19\n"
+ "whilelt p3.s, x19, x16\n"
+ "incw x19\n"
+ "whilelt p2.s, x19, x16\n"
+ "incw x19\n"
+ "whilelt p1.s, x19, x16\n"
+ "cbz x14, 60f\n"
+ "ld1w { z8.s }, p5/Z, [x14]\n"
+ "mov z12.d, z8.d\n"
+ "ld1w { z9.s }, p5/Z, [x14, #1, MUL VL]\n"
+ "mov z16.d, z8.d\n"
+ "ld1w { z10.s }, p5/Z, [x14, #2, MUL VL]\n"
+ "mov z20.d, z8.d\n"
+ "ld1w { z11.s }, p5/Z, [x14, #3, MUL VL]\n"
+ "addvl x14, x14, #4\n"
+ "mov z13.d, z9.d\n"
+ "mov z17.d, z9.d\n"
+ "mov z14.d, z10.d\n"
+ "mov z15.d, z11.d\n"
+ "mov z18.d, z10.d\n"
+ "mov z19.d, z11.d\n"
+ "mov z21.d, z9.d\n"
+ "mov z22.d, z10.d\n"
+ "mov z23.d, z11.d\n"
+ "mov z24.d, z8.d\n"
+ "mov z25.d, z9.d\n"
+ "mov z26.d, z10.d\n"
+ "mov z27.d, z11.d\n"
+ "b 62f\n"
+ "60:" // Height 5: no bias
+ "tbz %x[flags], #0, 61f\n"
+ "ld1w { z8.s }, p4/Z, [x13]\n"
+ "ld1w { z9.s }, p3/Z, [x13, #1, MUL VL]\n"
+ "ld1w { z10.s }, p2/Z, [x13, #2, MUL VL]\n"
+ "ld1w { z11.s }, p1/Z, [x13, #3, MUL VL]\n"
+ "ld1w { z12.s }, p4/Z, [x9]\n"
+ "ld1w { z13.s }, p3/Z, [x9, #1, MUL VL]\n"
+ "ld1w { z14.s }, p2/Z, [x9, #2, MUL VL]\n"
+ "ld1w { z15.s }, p1/Z, [x9, #3, MUL VL]\n"
+ "ld1w { z16.s }, p4/Z, [x27]\n"
+ "ld1w { z17.s }, p3/Z, [x27, #1, MUL VL]\n"
+ "ld1w { z18.s }, p2/Z, [x27, #2, MUL VL]\n"
+ "ld1w { z19.s }, p1/Z, [x27, #3, MUL VL]\n"
+ "ld1w { z20.s }, p4/Z, [x25]\n"
+ "ld1w { z21.s }, p3/Z, [x25, #1, MUL VL]\n"
+ "ld1w { z22.s }, p2/Z, [x25, #2, MUL VL]\n"
+ "ld1w { z23.s }, p1/Z, [x25, #3, MUL VL]\n"
+ "ld1w { z24.s }, p4/Z, [x23]\n"
+ "ld1w { z25.s }, p3/Z, [x23, #1, MUL VL]\n"
+ "ld1w { z26.s }, p2/Z, [x23, #2, MUL VL]\n"
+ "ld1w { z27.s }, p1/Z, [x23, #3, MUL VL]\n"
+ "b 62f\n"
+ "61:" // Height 5: no accumulate
+ "mov z8.b, #0x0\n"
+ "mov z9.b, #0x0\n"
+ "mov z10.b, #0x0\n"
+ "mov z11.b, #0x0\n"
+ "mov z12.b, #0x0\n"
+ "mov z13.b, #0x0\n"
+ "mov z14.b, #0x0\n"
+ "mov z15.b, #0x0\n"
+ "mov z16.b, #0x0\n"
+ "mov z17.b, #0x0\n"
+ "mov z18.b, #0x0\n"
+ "mov z19.b, #0x0\n"
+ "mov z20.b, #0x0\n"
+ "mov z21.b, #0x0\n"
+ "mov z22.b, #0x0\n"
+ "mov z23.b, #0x0\n"
+ "mov z24.b, #0x0\n"
+ "mov z25.b, #0x0\n"
+ "mov z26.b, #0x0\n"
+ "mov z27.b, #0x0\n"
+ "62:" // Height 5: setup done
+ "mov x12, #0x0\n"
+ "63:" // Height 5: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w11, [x20, x12, LSL #0x2]\n"
+ "tbz %x[flags], #3, 64f\n"
+ "ldr x20, [%x[input_ptr], x12, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x10, [x20, #0x0]\n"
+ "ldr x28, [x20, #0x8]\n"
+ "ldr x26, [x20, #0x10]\n"
+ "ldr x24, [x20, #0x18]\n"
+ "ldr x22, [x20, #0x20]\n"
+ "cbnz x12, 65f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x10, x10, x19, LSL #1\n"
+ "add x28, x28, x19, LSL #1\n"
+ "add x26, x26, x19, LSL #1\n"
+ "add x24, x24, x19, LSL #1\n"
+ "add x22, x22, x19, LSL #1\n"
+ "b 65f\n"
+ "64:" // Height 5: setup direct input
+ "mov x10, %x[input_ptr]\n"
+ "add x28, x10, x19, LSL #1\n"
+ "add x26, x28, x19, LSL #1\n"
+ "add x24, x26, x19, LSL #1\n"
+ "add x22, x24, x19, LSL #1\n"
+ "65:" // Height 5: input setup done
+ "cmp x11, #0x8\n"
+ "ble 67f\n"
+ "66:" // Height 5: Multiply loop: Main loop head
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ "whilelt p0.h, XZR, x11\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ "sub x11, x11, #0x8\n"
+ "ld1rqh { z0.h }, p0/Z, [x10]\n"
+ ".inst 0x646040c8 // bfdot z8.s, z6.h, z0.h[0]\n"
+ "ld1rqh { z1.h }, p0/Z, [x28]\n"
+ "add x10, x10, #0x10\n"
+ ".inst 0x646040e9 // bfdot z9.s, z7.h, z0.h[0]\n"
+ "ld1rqh { z2.h }, p0/Z, [x26]\n"
+ "add x28, x28, #0x10\n"
+ ".inst 0x646140cc // bfdot z12.s, z6.h, z1.h[0]\n"
+ "ld1rqh { z3.h }, p0/Z, [x24]\n"
+ "add x26, x26, #0x10\n"
+ ".inst 0x646240d0 // bfdot z16.s, z6.h, z2.h[0]\n"
+ "ld1rqh { z4.h }, p0/Z, [x22]\n"
+ "add x24, x24, #0x10\n"
+ ".inst 0x646140ed // bfdot z13.s, z7.h, z1.h[0]\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x22, x22, #0x10\n"
+ ".inst 0x646340d4 // bfdot z20.s, z6.h, z3.h[0]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "cmp x11, #0x8\n"
+ ".inst 0x646440d8 // bfdot z24.s, z6.h, z4.h[0]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ ".inst 0x646240f1 // bfdot z17.s, z7.h, z2.h[0]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ ".inst 0x646340f5 // bfdot z21.s, z7.h, z3.h[0]\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ ".inst 0x646440f9 // bfdot z25.s, z7.h, z4.h[0]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ ".inst 0x646040ca // bfdot z10.s, z6.h, z0.h[0]\n"
+ "prfm pldl1keep, [x22, #0x80]\n"
+ ".inst 0x646140ce // bfdot z14.s, z6.h, z1.h[0]\n"
+ ".inst 0x646240d2 // bfdot z18.s, z6.h, z2.h[0]\n"
+ ".inst 0x646340d6 // bfdot z22.s, z6.h, z3.h[0]\n"
+ ".inst 0x646440da // bfdot z26.s, z6.h, z4.h[0]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #4, MUL VL]\n"
+ ".inst 0x646040eb // bfdot z11.s, z7.h, z0.h[0]\n"
+ ".inst 0x646140ef // bfdot z15.s, z7.h, z1.h[0]\n"
+ ".inst 0x646240f3 // bfdot z19.s, z7.h, z2.h[0]\n"
+ ".inst 0x646340f7 // bfdot z23.s, z7.h, z3.h[0]\n"
+ ".inst 0x646440fb // bfdot z27.s, z7.h, z4.h[0]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #5, MUL VL]\n"
+ ".inst 0x646840c8 // bfdot z8.s, z6.h, z0.h[1]\n"
+ ".inst 0x646940cc // bfdot z12.s, z6.h, z1.h[1]\n"
+ ".inst 0x646a40d0 // bfdot z16.s, z6.h, z2.h[1]\n"
+ ".inst 0x646b40d4 // bfdot z20.s, z6.h, z3.h[1]\n"
+ ".inst 0x646c40d8 // bfdot z24.s, z6.h, z4.h[1]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #6, MUL VL]\n"
+ ".inst 0x646840e9 // bfdot z9.s, z7.h, z0.h[1]\n"
+ ".inst 0x646940ed // bfdot z13.s, z7.h, z1.h[1]\n"
+ ".inst 0x646a40f1 // bfdot z17.s, z7.h, z2.h[1]\n"
+ ".inst 0x646b40f5 // bfdot z21.s, z7.h, z3.h[1]\n"
+ ".inst 0x646c40f9 // bfdot z25.s, z7.h, z4.h[1]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #7, MUL VL]\n"
+ "addvl x15, x15, #16\n"
+ ".inst 0x646840ca // bfdot z10.s, z6.h, z0.h[1]\n"
+ ".inst 0x646940ce // bfdot z14.s, z6.h, z1.h[1]\n"
+ ".inst 0x646a40d2 // bfdot z18.s, z6.h, z2.h[1]\n"
+ ".inst 0x646b40d6 // bfdot z22.s, z6.h, z3.h[1]\n"
+ ".inst 0x646c40da // bfdot z26.s, z6.h, z4.h[1]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #-8, MUL VL]\n"
+ ".inst 0x646840eb // bfdot z11.s, z7.h, z0.h[1]\n"
+ ".inst 0x646940ef // bfdot z15.s, z7.h, z1.h[1]\n"
+ ".inst 0x646a40f3 // bfdot z19.s, z7.h, z2.h[1]\n"
+ ".inst 0x646b40f7 // bfdot z23.s, z7.h, z3.h[1]\n"
+ ".inst 0x646c40fb // bfdot z27.s, z7.h, z4.h[1]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #-7, MUL VL]\n"
+ ".inst 0x647040c8 // bfdot z8.s, z6.h, z0.h[2]\n"
+ ".inst 0x647140cc // bfdot z12.s, z6.h, z1.h[2]\n"
+ ".inst 0x647240d0 // bfdot z16.s, z6.h, z2.h[2]\n"
+ ".inst 0x647340d4 // bfdot z20.s, z6.h, z3.h[2]\n"
+ ".inst 0x647440d8 // bfdot z24.s, z6.h, z4.h[2]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #-6, MUL VL]\n"
+ ".inst 0x647040e9 // bfdot z9.s, z7.h, z0.h[2]\n"
+ ".inst 0x647140ed // bfdot z13.s, z7.h, z1.h[2]\n"
+ ".inst 0x647240f1 // bfdot z17.s, z7.h, z2.h[2]\n"
+ ".inst 0x647340f5 // bfdot z21.s, z7.h, z3.h[2]\n"
+ ".inst 0x647440f9 // bfdot z25.s, z7.h, z4.h[2]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #-5, MUL VL]\n"
+ ".inst 0x647040ca // bfdot z10.s, z6.h, z0.h[2]\n"
+ ".inst 0x647140ce // bfdot z14.s, z6.h, z1.h[2]\n"
+ ".inst 0x647240d2 // bfdot z18.s, z6.h, z2.h[2]\n"
+ ".inst 0x647340d6 // bfdot z22.s, z6.h, z3.h[2]\n"
+ ".inst 0x647440da // bfdot z26.s, z6.h, z4.h[2]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #-4, MUL VL]\n"
+ ".inst 0x647040eb // bfdot z11.s, z7.h, z0.h[2]\n"
+ ".inst 0x647140ef // bfdot z15.s, z7.h, z1.h[2]\n"
+ ".inst 0x647240f3 // bfdot z19.s, z7.h, z2.h[2]\n"
+ ".inst 0x647340f7 // bfdot z23.s, z7.h, z3.h[2]\n"
+ ".inst 0x647440fb // bfdot z27.s, z7.h, z4.h[2]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #-3, MUL VL]\n"
+ ".inst 0x647840c8 // bfdot z8.s, z6.h, z0.h[3]\n"
+ ".inst 0x647940cc // bfdot z12.s, z6.h, z1.h[3]\n"
+ ".inst 0x647a40d0 // bfdot z16.s, z6.h, z2.h[3]\n"
+ ".inst 0x647b40d4 // bfdot z20.s, z6.h, z3.h[3]\n"
+ ".inst 0x647c40d8 // bfdot z24.s, z6.h, z4.h[3]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #-2, MUL VL]\n"
+ ".inst 0x647840e9 // bfdot z9.s, z7.h, z0.h[3]\n"
+ ".inst 0x647940ed // bfdot z13.s, z7.h, z1.h[3]\n"
+ ".inst 0x647a40f1 // bfdot z17.s, z7.h, z2.h[3]\n"
+ ".inst 0x647b40f5 // bfdot z21.s, z7.h, z3.h[3]\n"
+ ".inst 0x647c40f9 // bfdot z25.s, z7.h, z4.h[3]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #-1, MUL VL]\n"
+ ".inst 0x647840ca // bfdot z10.s, z6.h, z0.h[3]\n"
+ ".inst 0x647940ce // bfdot z14.s, z6.h, z1.h[3]\n"
+ ".inst 0x647a40d2 // bfdot z18.s, z6.h, z2.h[3]\n"
+ ".inst 0x647b40d6 // bfdot z22.s, z6.h, z3.h[3]\n"
+ ".inst 0x647c40da // bfdot z26.s, z6.h, z4.h[3]\n"
+ ".inst 0x647840eb // bfdot z11.s, z7.h, z0.h[3]\n"
+ ".inst 0x647940ef // bfdot z15.s, z7.h, z1.h[3]\n"
+ ".inst 0x647a40f3 // bfdot z19.s, z7.h, z2.h[3]\n"
+ ".inst 0x647b40f7 // bfdot z23.s, z7.h, z3.h[3]\n"
+ ".inst 0x647c40fb // bfdot z27.s, z7.h, z4.h[3]\n"
+ "bgt 66b\n"
+ "67:" // Height 5: Multiply loop: Single iteration only
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ "whilelt p0.h, XZR, x11\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ "subs x11, x11, #0x2\n"
+ "ld1rqh { z0.h }, p0/Z, [x10]\n"
+ ".inst 0x646040c8 // bfdot z8.s, z6.h, z0.h[0]\n"
+ "ld1rqh { z1.h }, p0/Z, [x28]\n"
+ "add x10, x10, #0x10\n"
+ ".inst 0x646040e9 // bfdot z9.s, z7.h, z0.h[0]\n"
+ "ld1rqh { z2.h }, p0/Z, [x26]\n"
+ "add x28, x28, #0x10\n"
+ ".inst 0x646140cc // bfdot z12.s, z6.h, z1.h[0]\n"
+ "ld1rqh { z3.h }, p0/Z, [x24]\n"
+ "add x26, x26, #0x10\n"
+ ".inst 0x646240d0 // bfdot z16.s, z6.h, z2.h[0]\n"
+ "ld1rqh { z4.h }, p0/Z, [x22]\n"
+ "add x24, x24, #0x10\n"
+ ".inst 0x646140ed // bfdot z13.s, z7.h, z1.h[0]\n"
+ "add x22, x22, #0x10\n"
+ ".inst 0x646240f1 // bfdot z17.s, z7.h, z2.h[0]\n"
+ ".inst 0x646340d4 // bfdot z20.s, z6.h, z3.h[0]\n"
+ ".inst 0x646440d8 // bfdot z24.s, z6.h, z4.h[0]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ ".inst 0x646340f5 // bfdot z21.s, z7.h, z3.h[0]\n"
+ ".inst 0x646440f9 // bfdot z25.s, z7.h, z4.h[0]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ "addvl x15, x15, #4\n"
+ ".inst 0x646040ca // bfdot z10.s, z6.h, z0.h[0]\n"
+ ".inst 0x646140ce // bfdot z14.s, z6.h, z1.h[0]\n"
+ ".inst 0x646240d2 // bfdot z18.s, z6.h, z2.h[0]\n"
+ ".inst 0x646340d6 // bfdot z22.s, z6.h, z3.h[0]\n"
+ ".inst 0x646440da // bfdot z26.s, z6.h, z4.h[0]\n"
+ ".inst 0x646040eb // bfdot z11.s, z7.h, z0.h[0]\n"
+ ".inst 0x646140ef // bfdot z15.s, z7.h, z1.h[0]\n"
+ ".inst 0x646240f3 // bfdot z19.s, z7.h, z2.h[0]\n"
+ ".inst 0x646340f7 // bfdot z23.s, z7.h, z3.h[0]\n"
+ ".inst 0x646440fb // bfdot z27.s, z7.h, z4.h[0]\n"
+ "ble 68f\n"
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ ".inst 0x646840c8 // bfdot z8.s, z6.h, z0.h[1]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ "subs x11, x11, #0x2\n"
+ ".inst 0x646940cc // bfdot z12.s, z6.h, z1.h[1]\n"
+ ".inst 0x646a40d0 // bfdot z16.s, z6.h, z2.h[1]\n"
+ ".inst 0x646b40d4 // bfdot z20.s, z6.h, z3.h[1]\n"
+ ".inst 0x646c40d8 // bfdot z24.s, z6.h, z4.h[1]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ ".inst 0x646840e9 // bfdot z9.s, z7.h, z0.h[1]\n"
+ ".inst 0x646940ed // bfdot z13.s, z7.h, z1.h[1]\n"
+ ".inst 0x646a40f1 // bfdot z17.s, z7.h, z2.h[1]\n"
+ ".inst 0x646b40f5 // bfdot z21.s, z7.h, z3.h[1]\n"
+ ".inst 0x646c40f9 // bfdot z25.s, z7.h, z4.h[1]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ "addvl x15, x15, #4\n"
+ ".inst 0x646840ca // bfdot z10.s, z6.h, z0.h[1]\n"
+ ".inst 0x646940ce // bfdot z14.s, z6.h, z1.h[1]\n"
+ ".inst 0x646a40d2 // bfdot z18.s, z6.h, z2.h[1]\n"
+ ".inst 0x646b40d6 // bfdot z22.s, z6.h, z3.h[1]\n"
+ ".inst 0x646c40da // bfdot z26.s, z6.h, z4.h[1]\n"
+ ".inst 0x646840eb // bfdot z11.s, z7.h, z0.h[1]\n"
+ ".inst 0x646940ef // bfdot z15.s, z7.h, z1.h[1]\n"
+ ".inst 0x646a40f3 // bfdot z19.s, z7.h, z2.h[1]\n"
+ ".inst 0x646b40f7 // bfdot z23.s, z7.h, z3.h[1]\n"
+ ".inst 0x646c40fb // bfdot z27.s, z7.h, z4.h[1]\n"
+ "ble 68f\n"
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ ".inst 0x647040c8 // bfdot z8.s, z6.h, z0.h[2]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ "subs x11, x11, #0x2\n"
+ ".inst 0x647140cc // bfdot z12.s, z6.h, z1.h[2]\n"
+ ".inst 0x647240d0 // bfdot z16.s, z6.h, z2.h[2]\n"
+ ".inst 0x647340d4 // bfdot z20.s, z6.h, z3.h[2]\n"
+ ".inst 0x647440d8 // bfdot z24.s, z6.h, z4.h[2]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ ".inst 0x647040e9 // bfdot z9.s, z7.h, z0.h[2]\n"
+ ".inst 0x647140ed // bfdot z13.s, z7.h, z1.h[2]\n"
+ ".inst 0x647240f1 // bfdot z17.s, z7.h, z2.h[2]\n"
+ ".inst 0x647340f5 // bfdot z21.s, z7.h, z3.h[2]\n"
+ ".inst 0x647440f9 // bfdot z25.s, z7.h, z4.h[2]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ "addvl x15, x15, #4\n"
+ ".inst 0x647040ca // bfdot z10.s, z6.h, z0.h[2]\n"
+ ".inst 0x647140ce // bfdot z14.s, z6.h, z1.h[2]\n"
+ ".inst 0x647240d2 // bfdot z18.s, z6.h, z2.h[2]\n"
+ ".inst 0x647340d6 // bfdot z22.s, z6.h, z3.h[2]\n"
+ ".inst 0x647440da // bfdot z26.s, z6.h, z4.h[2]\n"
+ ".inst 0x647040eb // bfdot z11.s, z7.h, z0.h[2]\n"
+ ".inst 0x647140ef // bfdot z15.s, z7.h, z1.h[2]\n"
+ ".inst 0x647240f3 // bfdot z19.s, z7.h, z2.h[2]\n"
+ ".inst 0x647340f7 // bfdot z23.s, z7.h, z3.h[2]\n"
+ ".inst 0x647440fb // bfdot z27.s, z7.h, z4.h[2]\n"
+ "ble 68f\n"
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ ".inst 0x647840c8 // bfdot z8.s, z6.h, z0.h[3]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ ".inst 0x647940cc // bfdot z12.s, z6.h, z1.h[3]\n"
+ ".inst 0x647a40d0 // bfdot z16.s, z6.h, z2.h[3]\n"
+ ".inst 0x647b40d4 // bfdot z20.s, z6.h, z3.h[3]\n"
+ ".inst 0x647c40d8 // bfdot z24.s, z6.h, z4.h[3]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ ".inst 0x647840e9 // bfdot z9.s, z7.h, z0.h[3]\n"
+ ".inst 0x647940ed // bfdot z13.s, z7.h, z1.h[3]\n"
+ ".inst 0x647a40f1 // bfdot z17.s, z7.h, z2.h[3]\n"
+ ".inst 0x647b40f5 // bfdot z21.s, z7.h, z3.h[3]\n"
+ ".inst 0x647c40f9 // bfdot z25.s, z7.h, z4.h[3]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ "addvl x15, x15, #4\n"
+ ".inst 0x647840ca // bfdot z10.s, z6.h, z0.h[3]\n"
+ ".inst 0x647940ce // bfdot z14.s, z6.h, z1.h[3]\n"
+ ".inst 0x647a40d2 // bfdot z18.s, z6.h, z2.h[3]\n"
+ ".inst 0x647b40d6 // bfdot z22.s, z6.h, z3.h[3]\n"
+ ".inst 0x647c40da // bfdot z26.s, z6.h, z4.h[3]\n"
+ ".inst 0x647840eb // bfdot z11.s, z7.h, z0.h[3]\n"
+ ".inst 0x647940ef // bfdot z15.s, z7.h, z1.h[3]\n"
+ ".inst 0x647a40f3 // bfdot z19.s, z7.h, z2.h[3]\n"
+ ".inst 0x647b40f7 // bfdot z23.s, z7.h, z3.h[3]\n"
+ ".inst 0x647c40fb // bfdot z27.s, z7.h, z4.h[3]\n"
+ "68:" // Height 5: Multiply loop: multiply skip
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x12, x12, #0x1\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "prfm pldl1keep, [x22, #0x80]\n"
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "cmp x12, x19\n"
+ "bne 63b\n"
+ "prfm pstl1keep, [x13, #0x0]\n"
+ "prfm pstl1keep, [x9, #0x0]\n"
+ "prfm pstl1keep, [x27, #0x0]\n"
+ "prfm pstl1keep, [x25, #0x0]\n"
+ "prfm pstl1keep, [x23, #0x0]\n"
+ "tbz %x[flags], #1, 69f\n"
+ "add x19, %x[args_ptr], %[offset_min]\n"
+ "ld1rw { z1.s }, p5/Z, [x19]\n"
+ "add x19, %x[args_ptr], %[offset_max]\n"
+ "ld1rw { z0.s }, p5/Z, [x19]\n"
+ "fmin z8.s, p5/M, z8.s, z0.s\n"
+ "fmin z9.s, p5/M, z9.s, z0.s\n"
+ "fmin z10.s, p5/M, z10.s, z0.s\n"
+ "fmin z11.s, p5/M, z11.s, z0.s\n"
+ "fmin z12.s, p5/M, z12.s, z0.s\n"
+ "fmax z8.s, p5/M, z8.s, z1.s\n"
+ "fmax z9.s, p5/M, z9.s, z1.s\n"
+ "fmax z10.s, p5/M, z10.s, z1.s\n"
+ "fmax z11.s, p5/M, z11.s, z1.s\n"
+ "fmax z12.s, p5/M, z12.s, z1.s\n"
+ "fmin z13.s, p5/M, z13.s, z0.s\n"
+ "fmin z14.s, p5/M, z14.s, z0.s\n"
+ "fmin z15.s, p5/M, z15.s, z0.s\n"
+ "fmin z16.s, p5/M, z16.s, z0.s\n"
+ "fmax z13.s, p5/M, z13.s, z1.s\n"
+ "fmax z14.s, p5/M, z14.s, z1.s\n"
+ "fmax z15.s, p5/M, z15.s, z1.s\n"
+ "fmax z16.s, p5/M, z16.s, z1.s\n"
+ "fmin z17.s, p5/M, z17.s, z0.s\n"
+ "fmin z18.s, p5/M, z18.s, z0.s\n"
+ "fmin z19.s, p5/M, z19.s, z0.s\n"
+ "fmin z20.s, p5/M, z20.s, z0.s\n"
+ "fmax z17.s, p5/M, z17.s, z1.s\n"
+ "fmax z18.s, p5/M, z18.s, z1.s\n"
+ "fmax z19.s, p5/M, z19.s, z1.s\n"
+ "fmax z20.s, p5/M, z20.s, z1.s\n"
+ "fmin z21.s, p5/M, z21.s, z0.s\n"
+ "fmin z22.s, p5/M, z22.s, z0.s\n"
+ "fmin z23.s, p5/M, z23.s, z0.s\n"
+ "fmin z24.s, p5/M, z24.s, z0.s\n"
+ "fmax z21.s, p5/M, z21.s, z1.s\n"
+ "fmax z22.s, p5/M, z22.s, z1.s\n"
+ "fmax z23.s, p5/M, z23.s, z1.s\n"
+ "fmax z24.s, p5/M, z24.s, z1.s\n"
+ "fmin z25.s, p5/M, z25.s, z0.s\n"
+ "fmin z26.s, p5/M, z26.s, z0.s\n"
+ "fmin z27.s, p5/M, z27.s, z0.s\n"
+ "fmax z25.s, p5/M, z25.s, z1.s\n"
+ "fmax z26.s, p5/M, z26.s, z1.s\n"
+ "fmax z27.s, p5/M, z27.s, z1.s\n"
+ "69:" // Height 5: No activation
+ "st1w { z8.s }, p4, [x13]\n"
+ "st1w { z9.s }, p3, [x13, #1, MUL VL]\n"
+ "st1w { z10.s }, p2, [x13, #2, MUL VL]\n"
+ "st1w { z11.s }, p1, [x13, #3, MUL VL]\n"
+ "addvl x13, x13, #4\n"
+ "st1w { z12.s }, p4, [x9]\n"
+ "st1w { z13.s }, p3, [x9, #1, MUL VL]\n"
+ "st1w { z14.s }, p2, [x9, #2, MUL VL]\n"
+ "st1w { z15.s }, p1, [x9, #3, MUL VL]\n"
+ "addvl x9, x9, #4\n"
+ "st1w { z16.s }, p4, [x27]\n"
+ "st1w { z17.s }, p3, [x27, #1, MUL VL]\n"
+ "st1w { z18.s }, p2, [x27, #2, MUL VL]\n"
+ "st1w { z19.s }, p1, [x27, #3, MUL VL]\n"
+ "addvl x27, x27, #4\n"
+ "st1w { z20.s }, p4, [x25]\n"
+ "st1w { z21.s }, p3, [x25, #1, MUL VL]\n"
+ "st1w { z22.s }, p2, [x25, #2, MUL VL]\n"
+ "st1w { z23.s }, p1, [x25, #3, MUL VL]\n"
+ "addvl x25, x25, #4\n"
+ "st1w { z24.s }, p4, [x23]\n"
+ "st1w { z25.s }, p3, [x23, #1, MUL VL]\n"
+ "st1w { z26.s }, p2, [x23, #2, MUL VL]\n"
+ "st1w { z27.s }, p1, [x23, #3, MUL VL]\n"
+ "addvl x23, x23, #4\n"
+ "70:" // Height 5: Writeback done
+ "mov x19, #0x0\n"
+ "incw x19, ALL, MUL #4\n"
+ "subs x16, x16, x19\n"
+ "bgt 59b\n"
+ "b 86f\n"
+ "71:" // Height 6
+ "ldr x16, [%x[args_ptr], %[offsetof_N]]\n"
+ "mov x14, %x[bias]\n"
+ "ldr x15, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 72f\n"
+ "ldr x13, [%x[output_ptr], #0x0]\n"
+ "add x13, x13, x19, LSL #2\n"
+ "ldr x9, [%x[output_ptr], #0x8]\n"
+ "ldr x27, [%x[output_ptr], #0x10]\n"
+ "add x9, x9, x19, LSL #2\n"
+ "ldr x25, [%x[output_ptr], #0x18]\n"
+ "ldr x23, [%x[output_ptr], #0x20]\n"
+ "add x27, x27, x19, LSL #2\n"
+ "ldr x21, [%x[output_ptr], #0x28]\n"
+ "add %x[output_ptr], %x[output_ptr], #0x30\n"
+ "add x25, x25, x19, LSL #2\n"
+ "add x23, x23, x19, LSL #2\n"
+ "add x21, x21, x19, LSL #2\n"
+ "b 73f\n"
+ "72:" // Height 6: setup direct output
+ "mov x13, %x[output_ptr]\n"
+ "add x9, x13, x19, LSL #2\n"
+ "add x27, x9, x19, LSL #2\n"
+ "add x25, x27, x19, LSL #2\n"
+ "add x23, x25, x19, LSL #2\n"
+ "add x21, x23, x19, LSL #2\n"
+ "add %x[output_ptr], x21, x19, LSL #2\n"
+ "73:" // Height 6: Column loop
+ "mov x19, #0x0\n"
+ "whilelt p4.s, x19, x16\n"
+ "incw x19\n"
+ "whilelt p3.s, x19, x16\n"
+ "incw x19\n"
+ "whilelt p2.s, x19, x16\n"
+ "incw x19\n"
+ "whilelt p1.s, x19, x16\n"
+ "cbz x14, 74f\n"
+ "ld1w { z8.s }, p5/Z, [x14]\n"
+ "mov z12.d, z8.d\n"
+ "ld1w { z9.s }, p5/Z, [x14, #1, MUL VL]\n"
+ "mov z16.d, z8.d\n"
+ "ld1w { z10.s }, p5/Z, [x14, #2, MUL VL]\n"
+ "mov z20.d, z8.d\n"
+ "ld1w { z11.s }, p5/Z, [x14, #3, MUL VL]\n"
+ "addvl x14, x14, #4\n"
+ "mov z13.d, z9.d\n"
+ "mov z17.d, z9.d\n"
+ "mov z14.d, z10.d\n"
+ "mov z15.d, z11.d\n"
+ "mov z18.d, z10.d\n"
+ "mov z19.d, z11.d\n"
+ "mov z21.d, z9.d\n"
+ "mov z22.d, z10.d\n"
+ "mov z23.d, z11.d\n"
+ "mov z24.d, z8.d\n"
+ "mov z25.d, z9.d\n"
+ "mov z26.d, z10.d\n"
+ "mov z27.d, z11.d\n"
+ "mov z28.d, z8.d\n"
+ "mov z29.d, z9.d\n"
+ "mov z30.d, z10.d\n"
+ "mov z31.d, z11.d\n"
+ "b 76f\n"
+ "74:" // Height 6: no bias
+ "tbz %x[flags], #0, 75f\n"
+ "ld1w { z8.s }, p4/Z, [x13]\n"
+ "ld1w { z9.s }, p3/Z, [x13, #1, MUL VL]\n"
+ "ld1w { z10.s }, p2/Z, [x13, #2, MUL VL]\n"
+ "ld1w { z11.s }, p1/Z, [x13, #3, MUL VL]\n"
+ "ld1w { z12.s }, p4/Z, [x9]\n"
+ "ld1w { z13.s }, p3/Z, [x9, #1, MUL VL]\n"
+ "ld1w { z14.s }, p2/Z, [x9, #2, MUL VL]\n"
+ "ld1w { z15.s }, p1/Z, [x9, #3, MUL VL]\n"
+ "ld1w { z16.s }, p4/Z, [x27]\n"
+ "ld1w { z17.s }, p3/Z, [x27, #1, MUL VL]\n"
+ "ld1w { z18.s }, p2/Z, [x27, #2, MUL VL]\n"
+ "ld1w { z19.s }, p1/Z, [x27, #3, MUL VL]\n"
+ "ld1w { z20.s }, p4/Z, [x25]\n"
+ "ld1w { z21.s }, p3/Z, [x25, #1, MUL VL]\n"
+ "ld1w { z22.s }, p2/Z, [x25, #2, MUL VL]\n"
+ "ld1w { z23.s }, p1/Z, [x25, #3, MUL VL]\n"
+ "ld1w { z24.s }, p4/Z, [x23]\n"
+ "ld1w { z25.s }, p3/Z, [x23, #1, MUL VL]\n"
+ "ld1w { z26.s }, p2/Z, [x23, #2, MUL VL]\n"
+ "ld1w { z27.s }, p1/Z, [x23, #3, MUL VL]\n"
+ "ld1w { z28.s }, p4/Z, [x21]\n"
+ "ld1w { z29.s }, p3/Z, [x21, #1, MUL VL]\n"
+ "ld1w { z30.s }, p2/Z, [x21, #2, MUL VL]\n"
+ "ld1w { z31.s }, p1/Z, [x21, #3, MUL VL]\n"
+ "b 76f\n"
+ "75:" // Height 6: no accumulate
+ "mov z8.b, #0x0\n"
+ "mov z9.b, #0x0\n"
+ "mov z10.b, #0x0\n"
+ "mov z11.b, #0x0\n"
+ "mov z12.b, #0x0\n"
+ "mov z13.b, #0x0\n"
+ "mov z14.b, #0x0\n"
+ "mov z15.b, #0x0\n"
+ "mov z16.b, #0x0\n"
+ "mov z17.b, #0x0\n"
+ "mov z18.b, #0x0\n"
+ "mov z19.b, #0x0\n"
+ "mov z20.b, #0x0\n"
+ "mov z21.b, #0x0\n"
+ "mov z22.b, #0x0\n"
+ "mov z23.b, #0x0\n"
+ "mov z24.b, #0x0\n"
+ "mov z25.b, #0x0\n"
+ "mov z26.b, #0x0\n"
+ "mov z27.b, #0x0\n"
+ "mov z28.b, #0x0\n"
+ "mov z29.b, #0x0\n"
+ "mov z30.b, #0x0\n"
+ "mov z31.b, #0x0\n"
+ "76:" // Height 6: setup done
+ "mov x12, #0x0\n"
+ "77:" // Height 6: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w11, [x20, x12, LSL #0x2]\n"
+ "tbz %x[flags], #3, 78f\n"
+ "ldr x20, [%x[input_ptr], x12, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x10, [x20, #0x0]\n"
+ "ldr x28, [x20, #0x8]\n"
+ "ldr x26, [x20, #0x10]\n"
+ "ldr x24, [x20, #0x18]\n"
+ "ldr x22, [x20, #0x20]\n"
+ "ldr x20, [x20, #0x28]\n"
+ "cbnz x12, 79f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x10, x10, x19, LSL #1\n"
+ "add x28, x28, x19, LSL #1\n"
+ "add x26, x26, x19, LSL #1\n"
+ "add x24, x24, x19, LSL #1\n"
+ "add x22, x22, x19, LSL #1\n"
+ "add x20, x20, x19, LSL #1\n"
+ "b 79f\n"
+ "78:" // Height 6: setup direct input
+ "mov x10, %x[input_ptr]\n"
+ "add x28, x10, x19, LSL #1\n"
+ "add x26, x28, x19, LSL #1\n"
+ "add x24, x26, x19, LSL #1\n"
+ "add x22, x24, x19, LSL #1\n"
+ "add x20, x22, x19, LSL #1\n"
+ "79:" // Height 6: input setup done
+ "cmp x11, #0x8\n"
+ "ble 81f\n"
+ "80:" // Height 6: Multiply loop: Main loop head
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ "whilelt p0.h, XZR, x11\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ "sub x11, x11, #0x8\n"
+ "ld1rqh { z0.h }, p0/Z, [x10]\n"
+ ".inst 0x646040c8 // bfdot z8.s, z6.h, z0.h[0]\n"
+ "ld1rqh { z1.h }, p0/Z, [x28]\n"
+ "add x10, x10, #0x10\n"
+ ".inst 0x646040e9 // bfdot z9.s, z7.h, z0.h[0]\n"
+ "ld1rqh { z2.h }, p0/Z, [x26]\n"
+ "add x28, x28, #0x10\n"
+ ".inst 0x646140cc // bfdot z12.s, z6.h, z1.h[0]\n"
+ "ld1rqh { z3.h }, p0/Z, [x24]\n"
+ "add x26, x26, #0x10\n"
+ ".inst 0x646240d0 // bfdot z16.s, z6.h, z2.h[0]\n"
+ "ld1rqh { z4.h }, p0/Z, [x22]\n"
+ "add x24, x24, #0x10\n"
+ ".inst 0x646140ed // bfdot z13.s, z7.h, z1.h[0]\n"
+ "ld1rqh { z5.h }, p0/Z, [x20]\n"
+ "add x22, x22, #0x10\n"
+ ".inst 0x646340d4 // bfdot z20.s, z6.h, z3.h[0]\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x20, x20, #0x10\n"
+ ".inst 0x646440d8 // bfdot z24.s, z6.h, z4.h[0]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "cmp x11, #0x8\n"
+ ".inst 0x646540dc // bfdot z28.s, z6.h, z5.h[0]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ ".inst 0x646240f1 // bfdot z17.s, z7.h, z2.h[0]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ ".inst 0x646340f5 // bfdot z21.s, z7.h, z3.h[0]\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ ".inst 0x646440f9 // bfdot z25.s, z7.h, z4.h[0]\n"
+ "prfm pldl1keep, [x22, #0x80]\n"
+ ".inst 0x646540fd // bfdot z29.s, z7.h, z5.h[0]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ ".inst 0x646040ca // bfdot z10.s, z6.h, z0.h[0]\n"
+ "prfm pldl1keep, [x20, #0x80]\n"
+ ".inst 0x646140ce // bfdot z14.s, z6.h, z1.h[0]\n"
+ ".inst 0x646240d2 // bfdot z18.s, z6.h, z2.h[0]\n"
+ ".inst 0x646340d6 // bfdot z22.s, z6.h, z3.h[0]\n"
+ ".inst 0x646440da // bfdot z26.s, z6.h, z4.h[0]\n"
+ ".inst 0x646540de // bfdot z30.s, z6.h, z5.h[0]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #4, MUL VL]\n"
+ ".inst 0x646040eb // bfdot z11.s, z7.h, z0.h[0]\n"
+ ".inst 0x646140ef // bfdot z15.s, z7.h, z1.h[0]\n"
+ ".inst 0x646240f3 // bfdot z19.s, z7.h, z2.h[0]\n"
+ ".inst 0x646340f7 // bfdot z23.s, z7.h, z3.h[0]\n"
+ ".inst 0x646440fb // bfdot z27.s, z7.h, z4.h[0]\n"
+ ".inst 0x646540ff // bfdot z31.s, z7.h, z5.h[0]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #5, MUL VL]\n"
+ ".inst 0x646840c8 // bfdot z8.s, z6.h, z0.h[1]\n"
+ ".inst 0x646940cc // bfdot z12.s, z6.h, z1.h[1]\n"
+ ".inst 0x646a40d0 // bfdot z16.s, z6.h, z2.h[1]\n"
+ ".inst 0x646b40d4 // bfdot z20.s, z6.h, z3.h[1]\n"
+ ".inst 0x646c40d8 // bfdot z24.s, z6.h, z4.h[1]\n"
+ ".inst 0x646d40dc // bfdot z28.s, z6.h, z5.h[1]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #6, MUL VL]\n"
+ ".inst 0x646840e9 // bfdot z9.s, z7.h, z0.h[1]\n"
+ ".inst 0x646940ed // bfdot z13.s, z7.h, z1.h[1]\n"
+ ".inst 0x646a40f1 // bfdot z17.s, z7.h, z2.h[1]\n"
+ ".inst 0x646b40f5 // bfdot z21.s, z7.h, z3.h[1]\n"
+ ".inst 0x646c40f9 // bfdot z25.s, z7.h, z4.h[1]\n"
+ ".inst 0x646d40fd // bfdot z29.s, z7.h, z5.h[1]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #7, MUL VL]\n"
+ "addvl x15, x15, #16\n"
+ ".inst 0x646840ca // bfdot z10.s, z6.h, z0.h[1]\n"
+ ".inst 0x646940ce // bfdot z14.s, z6.h, z1.h[1]\n"
+ ".inst 0x646a40d2 // bfdot z18.s, z6.h, z2.h[1]\n"
+ ".inst 0x646b40d6 // bfdot z22.s, z6.h, z3.h[1]\n"
+ ".inst 0x646c40da // bfdot z26.s, z6.h, z4.h[1]\n"
+ ".inst 0x646d40de // bfdot z30.s, z6.h, z5.h[1]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #-8, MUL VL]\n"
+ ".inst 0x646840eb // bfdot z11.s, z7.h, z0.h[1]\n"
+ ".inst 0x646940ef // bfdot z15.s, z7.h, z1.h[1]\n"
+ ".inst 0x646a40f3 // bfdot z19.s, z7.h, z2.h[1]\n"
+ ".inst 0x646b40f7 // bfdot z23.s, z7.h, z3.h[1]\n"
+ ".inst 0x646c40fb // bfdot z27.s, z7.h, z4.h[1]\n"
+ ".inst 0x646d40ff // bfdot z31.s, z7.h, z5.h[1]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #-7, MUL VL]\n"
+ ".inst 0x647040c8 // bfdot z8.s, z6.h, z0.h[2]\n"
+ ".inst 0x647140cc // bfdot z12.s, z6.h, z1.h[2]\n"
+ ".inst 0x647240d0 // bfdot z16.s, z6.h, z2.h[2]\n"
+ ".inst 0x647340d4 // bfdot z20.s, z6.h, z3.h[2]\n"
+ ".inst 0x647440d8 // bfdot z24.s, z6.h, z4.h[2]\n"
+ ".inst 0x647540dc // bfdot z28.s, z6.h, z5.h[2]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #-6, MUL VL]\n"
+ ".inst 0x647040e9 // bfdot z9.s, z7.h, z0.h[2]\n"
+ ".inst 0x647140ed // bfdot z13.s, z7.h, z1.h[2]\n"
+ ".inst 0x647240f1 // bfdot z17.s, z7.h, z2.h[2]\n"
+ ".inst 0x647340f5 // bfdot z21.s, z7.h, z3.h[2]\n"
+ ".inst 0x647440f9 // bfdot z25.s, z7.h, z4.h[2]\n"
+ ".inst 0x647540fd // bfdot z29.s, z7.h, z5.h[2]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #-5, MUL VL]\n"
+ ".inst 0x647040ca // bfdot z10.s, z6.h, z0.h[2]\n"
+ ".inst 0x647140ce // bfdot z14.s, z6.h, z1.h[2]\n"
+ ".inst 0x647240d2 // bfdot z18.s, z6.h, z2.h[2]\n"
+ ".inst 0x647340d6 // bfdot z22.s, z6.h, z3.h[2]\n"
+ ".inst 0x647440da // bfdot z26.s, z6.h, z4.h[2]\n"
+ ".inst 0x647540de // bfdot z30.s, z6.h, z5.h[2]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #-4, MUL VL]\n"
+ ".inst 0x647040eb // bfdot z11.s, z7.h, z0.h[2]\n"
+ ".inst 0x647140ef // bfdot z15.s, z7.h, z1.h[2]\n"
+ ".inst 0x647240f3 // bfdot z19.s, z7.h, z2.h[2]\n"
+ ".inst 0x647340f7 // bfdot z23.s, z7.h, z3.h[2]\n"
+ ".inst 0x647440fb // bfdot z27.s, z7.h, z4.h[2]\n"
+ ".inst 0x647540ff // bfdot z31.s, z7.h, z5.h[2]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #-3, MUL VL]\n"
+ ".inst 0x647840c8 // bfdot z8.s, z6.h, z0.h[3]\n"
+ ".inst 0x647940cc // bfdot z12.s, z6.h, z1.h[3]\n"
+ ".inst 0x647a40d0 // bfdot z16.s, z6.h, z2.h[3]\n"
+ ".inst 0x647b40d4 // bfdot z20.s, z6.h, z3.h[3]\n"
+ ".inst 0x647c40d8 // bfdot z24.s, z6.h, z4.h[3]\n"
+ ".inst 0x647d40dc // bfdot z28.s, z6.h, z5.h[3]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #-2, MUL VL]\n"
+ ".inst 0x647840e9 // bfdot z9.s, z7.h, z0.h[3]\n"
+ ".inst 0x647940ed // bfdot z13.s, z7.h, z1.h[3]\n"
+ ".inst 0x647a40f1 // bfdot z17.s, z7.h, z2.h[3]\n"
+ ".inst 0x647b40f5 // bfdot z21.s, z7.h, z3.h[3]\n"
+ ".inst 0x647c40f9 // bfdot z25.s, z7.h, z4.h[3]\n"
+ ".inst 0x647d40fd // bfdot z29.s, z7.h, z5.h[3]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #-1, MUL VL]\n"
+ ".inst 0x647840ca // bfdot z10.s, z6.h, z0.h[3]\n"
+ ".inst 0x647940ce // bfdot z14.s, z6.h, z1.h[3]\n"
+ ".inst 0x647a40d2 // bfdot z18.s, z6.h, z2.h[3]\n"
+ ".inst 0x647b40d6 // bfdot z22.s, z6.h, z3.h[3]\n"
+ ".inst 0x647c40da // bfdot z26.s, z6.h, z4.h[3]\n"
+ ".inst 0x647d40de // bfdot z30.s, z6.h, z5.h[3]\n"
+ ".inst 0x647840eb // bfdot z11.s, z7.h, z0.h[3]\n"
+ ".inst 0x647940ef // bfdot z15.s, z7.h, z1.h[3]\n"
+ ".inst 0x647a40f3 // bfdot z19.s, z7.h, z2.h[3]\n"
+ ".inst 0x647b40f7 // bfdot z23.s, z7.h, z3.h[3]\n"
+ ".inst 0x647c40fb // bfdot z27.s, z7.h, z4.h[3]\n"
+ ".inst 0x647d40ff // bfdot z31.s, z7.h, z5.h[3]\n"
+ "bgt 80b\n"
+ "81:" // Height 6: Multiply loop: Single iteration only
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ "whilelt p0.h, XZR, x11\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ "subs x11, x11, #0x2\n"
+ "ld1rqh { z0.h }, p0/Z, [x10]\n"
+ ".inst 0x646040c8 // bfdot z8.s, z6.h, z0.h[0]\n"
+ "ld1rqh { z1.h }, p0/Z, [x28]\n"
+ "add x10, x10, #0x10\n"
+ ".inst 0x646040e9 // bfdot z9.s, z7.h, z0.h[0]\n"
+ "ld1rqh { z2.h }, p0/Z, [x26]\n"
+ "add x28, x28, #0x10\n"
+ ".inst 0x646140cc // bfdot z12.s, z6.h, z1.h[0]\n"
+ "ld1rqh { z3.h }, p0/Z, [x24]\n"
+ "add x26, x26, #0x10\n"
+ ".inst 0x646240d0 // bfdot z16.s, z6.h, z2.h[0]\n"
+ "ld1rqh { z4.h }, p0/Z, [x22]\n"
+ "add x24, x24, #0x10\n"
+ ".inst 0x646140ed // bfdot z13.s, z7.h, z1.h[0]\n"
+ "ld1rqh { z5.h }, p0/Z, [x20]\n"
+ "add x22, x22, #0x10\n"
+ ".inst 0x646340d4 // bfdot z20.s, z6.h, z3.h[0]\n"
+ "add x20, x20, #0x10\n"
+ ".inst 0x646240f1 // bfdot z17.s, z7.h, z2.h[0]\n"
+ ".inst 0x646440d8 // bfdot z24.s, z6.h, z4.h[0]\n"
+ ".inst 0x646540dc // bfdot z28.s, z6.h, z5.h[0]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ ".inst 0x646340f5 // bfdot z21.s, z7.h, z3.h[0]\n"
+ ".inst 0x646440f9 // bfdot z25.s, z7.h, z4.h[0]\n"
+ ".inst 0x646540fd // bfdot z29.s, z7.h, z5.h[0]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ "addvl x15, x15, #4\n"
+ ".inst 0x646040ca // bfdot z10.s, z6.h, z0.h[0]\n"
+ ".inst 0x646140ce // bfdot z14.s, z6.h, z1.h[0]\n"
+ ".inst 0x646240d2 // bfdot z18.s, z6.h, z2.h[0]\n"
+ ".inst 0x646340d6 // bfdot z22.s, z6.h, z3.h[0]\n"
+ ".inst 0x646440da // bfdot z26.s, z6.h, z4.h[0]\n"
+ ".inst 0x646540de // bfdot z30.s, z6.h, z5.h[0]\n"
+ ".inst 0x646040eb // bfdot z11.s, z7.h, z0.h[0]\n"
+ ".inst 0x646140ef // bfdot z15.s, z7.h, z1.h[0]\n"
+ ".inst 0x646240f3 // bfdot z19.s, z7.h, z2.h[0]\n"
+ ".inst 0x646340f7 // bfdot z23.s, z7.h, z3.h[0]\n"
+ ".inst 0x646440fb // bfdot z27.s, z7.h, z4.h[0]\n"
+ ".inst 0x646540ff // bfdot z31.s, z7.h, z5.h[0]\n"
+ "ble 82f\n"
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ ".inst 0x646840c8 // bfdot z8.s, z6.h, z0.h[1]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ "subs x11, x11, #0x2\n"
+ ".inst 0x646940cc // bfdot z12.s, z6.h, z1.h[1]\n"
+ ".inst 0x646a40d0 // bfdot z16.s, z6.h, z2.h[1]\n"
+ ".inst 0x646b40d4 // bfdot z20.s, z6.h, z3.h[1]\n"
+ ".inst 0x646c40d8 // bfdot z24.s, z6.h, z4.h[1]\n"
+ ".inst 0x646d40dc // bfdot z28.s, z6.h, z5.h[1]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ ".inst 0x646840e9 // bfdot z9.s, z7.h, z0.h[1]\n"
+ ".inst 0x646940ed // bfdot z13.s, z7.h, z1.h[1]\n"
+ ".inst 0x646a40f1 // bfdot z17.s, z7.h, z2.h[1]\n"
+ ".inst 0x646b40f5 // bfdot z21.s, z7.h, z3.h[1]\n"
+ ".inst 0x646c40f9 // bfdot z25.s, z7.h, z4.h[1]\n"
+ ".inst 0x646d40fd // bfdot z29.s, z7.h, z5.h[1]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ "addvl x15, x15, #4\n"
+ ".inst 0x646840ca // bfdot z10.s, z6.h, z0.h[1]\n"
+ ".inst 0x646940ce // bfdot z14.s, z6.h, z1.h[1]\n"
+ ".inst 0x646a40d2 // bfdot z18.s, z6.h, z2.h[1]\n"
+ ".inst 0x646b40d6 // bfdot z22.s, z6.h, z3.h[1]\n"
+ ".inst 0x646c40da // bfdot z26.s, z6.h, z4.h[1]\n"
+ ".inst 0x646d40de // bfdot z30.s, z6.h, z5.h[1]\n"
+ ".inst 0x646840eb // bfdot z11.s, z7.h, z0.h[1]\n"
+ ".inst 0x646940ef // bfdot z15.s, z7.h, z1.h[1]\n"
+ ".inst 0x646a40f3 // bfdot z19.s, z7.h, z2.h[1]\n"
+ ".inst 0x646b40f7 // bfdot z23.s, z7.h, z3.h[1]\n"
+ ".inst 0x646c40fb // bfdot z27.s, z7.h, z4.h[1]\n"
+ ".inst 0x646d40ff // bfdot z31.s, z7.h, z5.h[1]\n"
+ "ble 82f\n"
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ ".inst 0x647040c8 // bfdot z8.s, z6.h, z0.h[2]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ "subs x11, x11, #0x2\n"
+ ".inst 0x647140cc // bfdot z12.s, z6.h, z1.h[2]\n"
+ ".inst 0x647240d0 // bfdot z16.s, z6.h, z2.h[2]\n"
+ ".inst 0x647340d4 // bfdot z20.s, z6.h, z3.h[2]\n"
+ ".inst 0x647440d8 // bfdot z24.s, z6.h, z4.h[2]\n"
+ ".inst 0x647540dc // bfdot z28.s, z6.h, z5.h[2]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ ".inst 0x647040e9 // bfdot z9.s, z7.h, z0.h[2]\n"
+ ".inst 0x647140ed // bfdot z13.s, z7.h, z1.h[2]\n"
+ ".inst 0x647240f1 // bfdot z17.s, z7.h, z2.h[2]\n"
+ ".inst 0x647340f5 // bfdot z21.s, z7.h, z3.h[2]\n"
+ ".inst 0x647440f9 // bfdot z25.s, z7.h, z4.h[2]\n"
+ ".inst 0x647540fd // bfdot z29.s, z7.h, z5.h[2]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ "addvl x15, x15, #4\n"
+ ".inst 0x647040ca // bfdot z10.s, z6.h, z0.h[2]\n"
+ ".inst 0x647140ce // bfdot z14.s, z6.h, z1.h[2]\n"
+ ".inst 0x647240d2 // bfdot z18.s, z6.h, z2.h[2]\n"
+ ".inst 0x647340d6 // bfdot z22.s, z6.h, z3.h[2]\n"
+ ".inst 0x647440da // bfdot z26.s, z6.h, z4.h[2]\n"
+ ".inst 0x647540de // bfdot z30.s, z6.h, z5.h[2]\n"
+ ".inst 0x647040eb // bfdot z11.s, z7.h, z0.h[2]\n"
+ ".inst 0x647140ef // bfdot z15.s, z7.h, z1.h[2]\n"
+ ".inst 0x647240f3 // bfdot z19.s, z7.h, z2.h[2]\n"
+ ".inst 0x647340f7 // bfdot z23.s, z7.h, z3.h[2]\n"
+ ".inst 0x647440fb // bfdot z27.s, z7.h, z4.h[2]\n"
+ ".inst 0x647540ff // bfdot z31.s, z7.h, z5.h[2]\n"
+ "ble 82f\n"
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ ".inst 0x647840c8 // bfdot z8.s, z6.h, z0.h[3]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ ".inst 0x647940cc // bfdot z12.s, z6.h, z1.h[3]\n"
+ ".inst 0x647a40d0 // bfdot z16.s, z6.h, z2.h[3]\n"
+ ".inst 0x647b40d4 // bfdot z20.s, z6.h, z3.h[3]\n"
+ ".inst 0x647c40d8 // bfdot z24.s, z6.h, z4.h[3]\n"
+ ".inst 0x647d40dc // bfdot z28.s, z6.h, z5.h[3]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ ".inst 0x647840e9 // bfdot z9.s, z7.h, z0.h[3]\n"
+ ".inst 0x647940ed // bfdot z13.s, z7.h, z1.h[3]\n"
+ ".inst 0x647a40f1 // bfdot z17.s, z7.h, z2.h[3]\n"
+ ".inst 0x647b40f5 // bfdot z21.s, z7.h, z3.h[3]\n"
+ ".inst 0x647c40f9 // bfdot z25.s, z7.h, z4.h[3]\n"
+ ".inst 0x647d40fd // bfdot z29.s, z7.h, z5.h[3]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ "addvl x15, x15, #4\n"
+ ".inst 0x647840ca // bfdot z10.s, z6.h, z0.h[3]\n"
+ ".inst 0x647940ce // bfdot z14.s, z6.h, z1.h[3]\n"
+ ".inst 0x647a40d2 // bfdot z18.s, z6.h, z2.h[3]\n"
+ ".inst 0x647b40d6 // bfdot z22.s, z6.h, z3.h[3]\n"
+ ".inst 0x647c40da // bfdot z26.s, z6.h, z4.h[3]\n"
+ ".inst 0x647d40de // bfdot z30.s, z6.h, z5.h[3]\n"
+ ".inst 0x647840eb // bfdot z11.s, z7.h, z0.h[3]\n"
+ ".inst 0x647940ef // bfdot z15.s, z7.h, z1.h[3]\n"
+ ".inst 0x647a40f3 // bfdot z19.s, z7.h, z2.h[3]\n"
+ ".inst 0x647b40f7 // bfdot z23.s, z7.h, z3.h[3]\n"
+ ".inst 0x647c40fb // bfdot z27.s, z7.h, z4.h[3]\n"
+ ".inst 0x647d40ff // bfdot z31.s, z7.h, z5.h[3]\n"
+ "82:" // Height 6: Multiply loop: multiply skip
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x12, x12, #0x1\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "prfm pldl1keep, [x22, #0x80]\n"
+ "prfm pldl1keep, [x20, #0x80]\n"
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "cmp x12, x19\n"
+ "bne 77b\n"
+ "prfm pstl1keep, [x13, #0x0]\n"
+ "prfm pstl1keep, [x9, #0x0]\n"
+ "prfm pstl1keep, [x27, #0x0]\n"
+ "prfm pstl1keep, [x25, #0x0]\n"
+ "prfm pstl1keep, [x23, #0x0]\n"
+ "prfm pstl1keep, [x21, #0x0]\n"
+ "tbz %x[flags], #1, 83f\n"
+ "add x19, %x[args_ptr], %[offset_min]\n"
+ "ld1rw { z1.s }, p5/Z, [x19]\n"
+ "add x19, %x[args_ptr], %[offset_max]\n"
+ "ld1rw { z0.s }, p5/Z, [x19]\n"
+ "fmin z8.s, p5/M, z8.s, z0.s\n"
+ "fmin z9.s, p5/M, z9.s, z0.s\n"
+ "fmin z10.s, p5/M, z10.s, z0.s\n"
+ "fmin z11.s, p5/M, z11.s, z0.s\n"
+ "fmin z12.s, p5/M, z12.s, z0.s\n"
+ "fmax z8.s, p5/M, z8.s, z1.s\n"
+ "fmax z9.s, p5/M, z9.s, z1.s\n"
+ "fmax z10.s, p5/M, z10.s, z1.s\n"
+ "fmax z11.s, p5/M, z11.s, z1.s\n"
+ "fmax z12.s, p5/M, z12.s, z1.s\n"
+ "fmin z13.s, p5/M, z13.s, z0.s\n"
+ "fmin z14.s, p5/M, z14.s, z0.s\n"
+ "fmin z15.s, p5/M, z15.s, z0.s\n"
+ "fmin z16.s, p5/M, z16.s, z0.s\n"
+ "fmax z13.s, p5/M, z13.s, z1.s\n"
+ "fmax z14.s, p5/M, z14.s, z1.s\n"
+ "fmax z15.s, p5/M, z15.s, z1.s\n"
+ "fmax z16.s, p5/M, z16.s, z1.s\n"
+ "fmin z17.s, p5/M, z17.s, z0.s\n"
+ "fmin z18.s, p5/M, z18.s, z0.s\n"
+ "fmin z19.s, p5/M, z19.s, z0.s\n"
+ "fmin z20.s, p5/M, z20.s, z0.s\n"
+ "fmax z17.s, p5/M, z17.s, z1.s\n"
+ "fmax z18.s, p5/M, z18.s, z1.s\n"
+ "fmax z19.s, p5/M, z19.s, z1.s\n"
+ "fmax z20.s, p5/M, z20.s, z1.s\n"
+ "fmin z21.s, p5/M, z21.s, z0.s\n"
+ "fmin z22.s, p5/M, z22.s, z0.s\n"
+ "fmin z23.s, p5/M, z23.s, z0.s\n"
+ "fmin z24.s, p5/M, z24.s, z0.s\n"
+ "fmax z21.s, p5/M, z21.s, z1.s\n"
+ "fmax z22.s, p5/M, z22.s, z1.s\n"
+ "fmax z23.s, p5/M, z23.s, z1.s\n"
+ "fmax z24.s, p5/M, z24.s, z1.s\n"
+ "fmin z25.s, p5/M, z25.s, z0.s\n"
+ "fmin z26.s, p5/M, z26.s, z0.s\n"
+ "fmin z27.s, p5/M, z27.s, z0.s\n"
+ "fmin z28.s, p5/M, z28.s, z0.s\n"
+ "fmax z25.s, p5/M, z25.s, z1.s\n"
+ "fmax z26.s, p5/M, z26.s, z1.s\n"
+ "fmax z27.s, p5/M, z27.s, z1.s\n"
+ "fmax z28.s, p5/M, z28.s, z1.s\n"
+ "fmin z29.s, p5/M, z29.s, z0.s\n"
+ "fmin z30.s, p5/M, z30.s, z0.s\n"
+ "fmin z31.s, p5/M, z31.s, z0.s\n"
+ "fmax z29.s, p5/M, z29.s, z1.s\n"
+ "fmax z30.s, p5/M, z30.s, z1.s\n"
+ "fmax z31.s, p5/M, z31.s, z1.s\n"
+ "83:" // Height 6: No activation
+ "st1w { z8.s }, p4, [x13]\n"
+ "st1w { z9.s }, p3, [x13, #1, MUL VL]\n"
+ "st1w { z10.s }, p2, [x13, #2, MUL VL]\n"
+ "st1w { z11.s }, p1, [x13, #3, MUL VL]\n"
+ "addvl x13, x13, #4\n"
+ "st1w { z12.s }, p4, [x9]\n"
+ "st1w { z13.s }, p3, [x9, #1, MUL VL]\n"
+ "st1w { z14.s }, p2, [x9, #2, MUL VL]\n"
+ "st1w { z15.s }, p1, [x9, #3, MUL VL]\n"
+ "addvl x9, x9, #4\n"
+ "st1w { z16.s }, p4, [x27]\n"
+ "st1w { z17.s }, p3, [x27, #1, MUL VL]\n"
+ "st1w { z18.s }, p2, [x27, #2, MUL VL]\n"
+ "st1w { z19.s }, p1, [x27, #3, MUL VL]\n"
+ "addvl x27, x27, #4\n"
+ "st1w { z20.s }, p4, [x25]\n"
+ "st1w { z21.s }, p3, [x25, #1, MUL VL]\n"
+ "st1w { z22.s }, p2, [x25, #2, MUL VL]\n"
+ "st1w { z23.s }, p1, [x25, #3, MUL VL]\n"
+ "addvl x25, x25, #4\n"
+ "st1w { z24.s }, p4, [x23]\n"
+ "st1w { z25.s }, p3, [x23, #1, MUL VL]\n"
+ "st1w { z26.s }, p2, [x23, #2, MUL VL]\n"
+ "st1w { z27.s }, p1, [x23, #3, MUL VL]\n"
+ "addvl x23, x23, #4\n"
+ "st1w { z28.s }, p4, [x21]\n"
+ "st1w { z29.s }, p3, [x21, #1, MUL VL]\n"
+ "st1w { z30.s }, p2, [x21, #2, MUL VL]\n"
+ "st1w { z31.s }, p1, [x21, #3, MUL VL]\n"
+ "addvl x21, x21, #4\n"
+ "84:" // Height 6: Writeback done
+ "mov x19, #0x0\n"
+ "incw x19, ALL, MUL #4\n"
+ "subs x16, x16, x19\n"
+ "bgt 73b\n"
+ "subs %x[M], %x[M], #0x6\n"
+ "beq 86f\n"
+ "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "tbz %x[flags], #3, 85f\n"
+ "add x20, x20, #0x6\n"
+ "str x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "b 1b\n"
+ "85:" // Update direct input
+ "mov x19, #0xc\n"
+ "madd %x[input_ptr], x19, x20, %x[input_ptr]\n"
+ "b 1b\n"
+ "86:" // Exit
+
+ : [M] "+r" (M), [input_ptr] "+r" (input_ptr), [output_ptr] "+r" (output_ptr)
+ : [args_ptr] "r" (&ka), [bias] "r" (bias), [flags] "r" (flags), [offset_max] "I" (offsetof(KernelArgs, maxval)), [offset_min] "I" (offsetof(KernelArgs, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths))
+ : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+ );
+}
+
+} // namespace arm_gemm
+#endif // __ARM_FEATURE_SVE
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_bf16fp32_mmla_4VLx4.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_bf16fp32_mmla_4VLx4.hpp
deleted file mode 100644
index 641e5c1..0000000
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_bf16fp32_mmla_4VLx4.hpp
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Copyright (c) 2018-2020 Arm Limited.
- *
- * SPDX-License-Identifier: MIT
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to
- * deal in the Software without restriction, including without limitation the
- * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
- * sell copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-#pragma once
-
-#ifdef __ARM_FEATURE_SVE
-
-#include "../bfloat.hpp"
-#include "../std_transforms_sve.hpp"
-
-namespace arm_gemm
-{
-
-// Actual kernel implementations
-void sve_hybrid_bf16fp32_mmla_4VLx4(const bfloat16 *, int, const bfloat16 *, float *, int, int, int, int, const float *, Activation, bool);
-
-class hybrid_bf16fp32_mmla_4VLx4
-{
-public:
- typedef bfloat16 operand_type;
- typedef float result_type;
-
- typedef void (*kern_type)(const bfloat16 *, int, const bfloat16 *, float *, int, int, int, int, const float *, Activation, bool);
-
- /* Kernel blocking parameters */
- static constexpr unsigned int out_height()
- {
- return 8;
- }
-
- static unsigned int out_width()
- {
- return get_vector_length<float>() * 2;
- }
-
- static constexpr unsigned int k_unroll()
- {
- return 4;
- }
-
- static constexpr bool supports_accumulate()
- {
- return true;
- }
-
- static constexpr bool supports_bias()
- {
- return true;
- }
-
- static constexpr bool supports_activation()
- {
- return true;
- }
-
- StdTransformsSVE<operand_type, result_type, 4, 4, 4> transforms = {};
-
- // Default to the generic kernel
- kern_type kernel=sve_hybrid_bf16fp32_mmla_4VLx4;
-
- hybrid_bf16fp32_mmla_4VLx4(const CPUInfo *)
- {
-
- }
-};
-
-} // namespace arm_gemm
-
-#endif // __ARM_FEATURE_SVE
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_bf16fp32_mmla_4VLx4/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_bf16fp32_mmla_4VLx4/generic.cpp
deleted file mode 100644
index 76e3546..0000000
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_bf16fp32_mmla_4VLx4/generic.cpp
+++ /dev/null
@@ -1,3459 +0,0 @@
-/*
- * Copyright (c) 2018-2020 Arm Limited.
- *
- * SPDX-License-Identifier: MIT
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to
- * deal in the Software without restriction, including without limitation the
- * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
- * sell copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-#ifdef __ARM_FEATURE_SVE
-
-#include <algorithm>
-
-#include "arm_gemm.hpp"
-#include "../../bfloat.hpp"
-#include "../../asmlib.hpp"
-#include "../../utils.hpp"
-
-namespace arm_gemm {
-
-void sve_hybrid_bf16fp32_mmla_4VLx4(const bfloat16 *A, int lda, const bfloat16 *B, float *C, int ldc, int M, int N, int K, const float *bias, Activation act, bool accumulate) {
- const int K_stride = ((K + 3) / 4) * 4;
- const long loops_count = ((K + 8) / 16) - 1;
- K -= loops_count * 16;
- const long regs_count = (K / 8) - 1;
- K -= (regs_count + 1) * 8;
- const long leftovers = K;
- const long blocks_count = (K + 3) / 4;
- float nullbias[128];
- if (!accumulate && !bias) {
- memset(nullbias, 0, (2 * get_vector_length<float>() * sizeof(float)));
- }
- float minval = - static_cast<float>(std::numeric_limits<float>::infinity());
- float maxval = static_cast<float>(std::numeric_limits<float>::infinity());
- const float * const minptr = &minval;
- const float * const maxptr = &maxval;
-
- switch(act.type)
- {
- default:
- case Activation::Type::None:
- break;
- case Activation::Type::BoundedReLU:
- maxval = static_cast<float>(act.param1);
- /* fall through */
- case Activation::Type::ReLU:
- minval = 0.0f;
- break;
- }
-
- int rows_to_compute;
-
- for (int y=0; y<M; y+=rows_to_compute) {
- const bfloat16 * const a_ptr0_base = A + (y * lda);
- const unsigned long ldab = lda * sizeof(bfloat16);
-
- float *c_ptr0 = C + (y * ldc);
-
- rows_to_compute = M-y;
- if (rows_to_compute > 8) {
- if (rows_to_compute % 8) {
- rows_to_compute = 8 - 1;
- } else {
- rows_to_compute = 8;
- }
- }
-
- for (int x0=0; x0<N; x0+=(2 * get_vector_length<float>())) {
- const long width = std::min((unsigned long)N-x0, (2 * get_vector_length<float>()));
- long loops = loops_count;
- long regs = regs_count;
- long temp = 0;
- long blocks = blocks_count;
- const bfloat16 *a_ptr0 = a_ptr0_base;
- const bfloat16 *b_ptr0 = B + (K_stride * x0);
- const unsigned long ldcb = ldc * sizeof(float);
- const float *biasptr = bias ? bias+x0 : nullbias;
-
- switch(rows_to_compute) {
- case 1:
- __asm __volatile (
- "whilelt p6.h, %[temp], %[leftovers]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "ptrue p7.h\n"
- "whilelt p1.s, %[temp], %[width]\n"
- "cbnz %[accumulate], 1f\n"
- "mov z1.h, #0\n"
- "ld1w z15.s, p0/z, [%[biasptr]]\n"
- "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ld1h z12.h, p7/z, [%[b_ptr0]]\n"
- "zip1 z16.s, z15.s, z15.s\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "zip2 z17.s, z15.s, z15.s\n"
- "ld1w z15.s, p1/z, [%[biasptr], #1, MUL VL]\n"
- "trn1 z8.d, z0.d, z1.d\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "zip1 z18.s, z15.s, z15.s\n"
- "zip2 z19.s, z15.s, z15.s\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #4\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "mov z14.s, #0\n"
- "ld1w z13.s, p0/z, [%[c_ptr0]]\n"
- "mov z1.h, #0\n"
- "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "zip1 z16.s, z13.s, z14.s\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "zip2 z17.s, z13.s, z14.s\n"
- "ld1w z13.s, p1/z, [%[c_ptr0], #1, MUL VL]\n"
- "trn1 z8.d, z0.d, z1.d\n"
- "mov z14.s, #0\n"
- "zip1 z18.s, z13.s, z14.s\n"
- "zip2 z19.s, z13.s, z14.s\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #4\n"
- "cbz %[loops], 2f\n"
- "3:\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "ld1rqh z4.h, p7/z, [%[a_ptr0]]\n"
- ".inst 0x6468e590 // bfmmla z16.s, z12.h, z8.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0]]\n"
- ".inst 0x6468e5b1 // bfmmla z17.s, z13.h, z8.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x6468e5d2 // bfmmla z18.s, z14.h, z8.h\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x6468e5f3 // bfmmla z19.s, z15.h, z8.h\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "mov z5.h, #0\n"
- "subs %[loops], %[loops], #0x1\n"
- ".inst 0x6460e590 // bfmmla z16.s, z12.h, z0.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x6460e5b1 // bfmmla z17.s, z13.h, z0.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "trn2 z8.d, z4.d, z5.d\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
- ".inst 0x6460e5d2 // bfmmla z18.s, z14.h, z0.h\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x6460e5f3 // bfmmla z19.s, z15.h, z0.h\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "trn1 z0.d, z4.d, z5.d\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "mov z1.h, #0\n"
- ".inst 0x6460e590 // bfmmla z16.s, z12.h, z0.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- ".inst 0x6460e5b1 // bfmmla z17.s, z13.h, z0.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- ".inst 0x6460e5d2 // bfmmla z18.s, z14.h, z0.h\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- ".inst 0x6460e5f3 // bfmmla z19.s, z15.h, z0.h\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- ".inst 0x6468e590 // bfmmla z16.s, z12.h, z8.h\n"
- "ld1rqh z0.h, p7/z, [%[a_ptr0], #-0x10]\n"
- ".inst 0x6468e5b1 // bfmmla z17.s, z13.h, z8.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- ".inst 0x6468e5d2 // bfmmla z18.s, z14.h, z8.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- ".inst 0x6468e5f3 // bfmmla z19.s, z15.h, z8.h\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "trn1 z8.d, z0.d, z1.d\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "b.ne 3b\n"
- "2:\n"
- "cbz %[regs], 4f\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "ld1rqh z4.h, p7/z, [%[a_ptr0]]\n"
- ".inst 0x6468e590 // bfmmla z16.s, z12.h, z8.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0]]\n"
- ".inst 0x6468e5b1 // bfmmla z17.s, z13.h, z8.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x6468e5d2 // bfmmla z18.s, z14.h, z8.h\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x6468e5f3 // bfmmla z19.s, z15.h, z8.h\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "mov z5.h, #0\n"
- ".inst 0x6460e590 // bfmmla z16.s, z12.h, z0.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x6460e5b1 // bfmmla z17.s, z13.h, z0.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x6460e5d2 // bfmmla z18.s, z14.h, z0.h\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "trn2 z8.d, z4.d, z5.d\n"
- ".inst 0x6460e5f3 // bfmmla z19.s, z15.h, z0.h\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "trn1 z0.d, z4.d, z5.d\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "mov z1.h, #0\n"
- ".inst 0x6460e590 // bfmmla z16.s, z12.h, z0.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- ".inst 0x6460e5b1 // bfmmla z17.s, z13.h, z0.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- ".inst 0x6460e5d2 // bfmmla z18.s, z14.h, z0.h\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- ".inst 0x6460e5f3 // bfmmla z19.s, z15.h, z0.h\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- ".inst 0x6468e590 // bfmmla z16.s, z12.h, z8.h\n"
- "ld1rqh z0.h, p6/z, [%[a_ptr0], #0x10]\n"
- ".inst 0x6468e5b1 // bfmmla z17.s, z13.h, z8.h\n"
- "addvl %[b_ptr0], %[b_ptr0], #-4\n"
- ".inst 0x6468e5d2 // bfmmla z18.s, z14.h, z8.h\n"
- "addvl %[a_ptr0], %[a_ptr0], #2\n"
- ".inst 0x6468e5f3 // bfmmla z19.s, z15.h, z8.h\n"
- "trn1 z8.d, z0.d, z1.d\n"
- "cbz %[blocks], 5f\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "ld1h z12.h, p7/z, [%[b_ptr0]]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x6468e590 // bfmmla z16.s, z12.h, z8.h\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x6468e5b1 // bfmmla z17.s, z13.h, z8.h\n"
- ".inst 0x6468e5d2 // bfmmla z18.s, z14.h, z8.h\n"
- ".inst 0x6468e5f3 // bfmmla z19.s, z15.h, z8.h\n"
- "b.eq 5f\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x6460e590 // bfmmla z16.s, z12.h, z0.h\n"
- ".inst 0x6460e5b1 // bfmmla z17.s, z13.h, z0.h\n"
- ".inst 0x6460e5d2 // bfmmla z18.s, z14.h, z0.h\n"
- ".inst 0x6460e5f3 // bfmmla z19.s, z15.h, z0.h\n"
- "b 5f\n"
- "4:\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "ld1rqh z4.h, p6/z, [%[a_ptr0]]\n"
- ".inst 0x6468e590 // bfmmla z16.s, z12.h, z8.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0]]\n"
- ".inst 0x6468e5b1 // bfmmla z17.s, z13.h, z8.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x6468e5d2 // bfmmla z18.s, z14.h, z8.h\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x6468e5f3 // bfmmla z19.s, z15.h, z8.h\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "mov z5.h, #0\n"
- "addvl %[b_ptr0], %[b_ptr0], #4\n"
- ".inst 0x6460e590 // bfmmla z16.s, z12.h, z0.h\n"
- "addvl %[a_ptr0], %[a_ptr0], #1\n"
- ".inst 0x6460e5b1 // bfmmla z17.s, z13.h, z0.h\n"
- ".inst 0x6460e5d2 // bfmmla z18.s, z14.h, z0.h\n"
- ".inst 0x6460e5f3 // bfmmla z19.s, z15.h, z0.h\n"
- "trn1 z0.d, z4.d, z5.d\n"
- "cbz %[blocks], 5f\n"
- "trn2 z8.d, z4.d, z5.d\n"
- "ld1h z12.h, p7/z, [%[b_ptr0]]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x6460e590 // bfmmla z16.s, z12.h, z0.h\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x6460e5b1 // bfmmla z17.s, z13.h, z0.h\n"
- ".inst 0x6460e5d2 // bfmmla z18.s, z14.h, z0.h\n"
- ".inst 0x6460e5f3 // bfmmla z19.s, z15.h, z0.h\n"
- "b.eq 5f\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x6468e590 // bfmmla z16.s, z12.h, z8.h\n"
- ".inst 0x6468e5b1 // bfmmla z17.s, z13.h, z8.h\n"
- ".inst 0x6468e5d2 // bfmmla z18.s, z14.h, z8.h\n"
- ".inst 0x6468e5f3 // bfmmla z19.s, z15.h, z8.h\n"
- "5:\n"
- "ld1rw z14.s, p7/z, [%[minptr]]\n"
- "ld1rw z15.s, p7/z, [%[maxptr]]\n"
- "fmax z16.s, p7/m, z16.s, z14.s\n"
- "fmax z17.s, p7/m, z17.s, z14.s\n"
- "fmax z18.s, p7/m, z18.s, z14.s\n"
- "fmax z19.s, p7/m, z19.s, z14.s\n"
- "fmin z16.s, p7/m, z16.s, z15.s\n"
- "fmin z17.s, p7/m, z17.s, z15.s\n"
- "fmin z18.s, p7/m, z18.s, z15.s\n"
- "fmin z19.s, p7/m, z19.s, z15.s\n"
- "uzp1 z0.s, z16.s, z17.s\n"
- "uzp1 z1.s, z18.s, z19.s\n"
- "st1w z0.s, p0, [%[c_ptr0]]\n"
- "st1w z1.s, p1, [%[c_ptr0], #1, MUL VL]\n"
- "addvl %[c_ptr0], %[c_ptr0], #2\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks)
- : [width] "r" (width), [accumulate] "r" (static_cast<uint64_t>(accumulate)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers)
- : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
- );
- break;
- case 2:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "c_ptr1 .req X1\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "whilelt p6.h, %[temp], %[leftovers]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "ptrue p7.h\n"
- "whilelt p1.s, %[temp], %[width]\n"
- "cbnz %[accumulate], 1f\n"
- "ld1w z15.s, p0/z, [%[biasptr]]\n"
- "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ld1rqh z1.h, p7/z, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "zip1 z16.s, z15.s, z15.s\n"
- "ld1h z12.h, p7/z, [%[b_ptr0]]\n"
- "zip2 z17.s, z15.s, z15.s\n"
- "ld1w z15.s, p1/z, [%[biasptr], #1, MUL VL]\n"
- "trn1 z8.d, z0.d, z1.d\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "zip1 z18.s, z15.s, z15.s\n"
- "zip2 z19.s, z15.s, z15.s\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #4\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "ld1w z13.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z14.s, p0/z, [c_ptr1]\n"
- "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ld1rqh z1.h, p7/z, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "zip1 z16.s, z13.s, z14.s\n"
- "ld1h z12.h, p7/z, [%[b_ptr0]]\n"
- "zip2 z17.s, z13.s, z14.s\n"
- "ld1w z13.s, p1/z, [%[c_ptr0], #1, MUL VL]\n"
- "trn1 z8.d, z0.d, z1.d\n"
- "ld1w z14.s, p1/z, [c_ptr1, #1, MUL VL]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "zip1 z18.s, z13.s, z14.s\n"
- "zip2 z19.s, z13.s, z14.s\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #4\n"
- "cbz %[loops], 2f\n"
- "3:\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "ld1rqh z4.h, p7/z, [%[a_ptr0]]\n"
- ".inst 0x6468e590 // bfmmla z16.s, z12.h, z8.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0]]\n"
- ".inst 0x6468e5b1 // bfmmla z17.s, z13.h, z8.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x6468e5d2 // bfmmla z18.s, z14.h, z8.h\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x6468e5f3 // bfmmla z19.s, z15.h, z8.h\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x6460e590 // bfmmla z16.s, z12.h, z0.h\n"
- "ld1rqh z5.h, p7/z, [a_ptr1]\n"
- ".inst 0x6460e5b1 // bfmmla z17.s, z13.h, z0.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x6460e5d2 // bfmmla z18.s, z14.h, z0.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x6460e5f3 // bfmmla z19.s, z15.h, z0.h\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "trn1 z0.d, z4.d, z5.d\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "trn2 z8.d, z4.d, z5.d\n"
- "subs %[loops], %[loops], #0x1\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
- ".inst 0x6460e590 // bfmmla z16.s, z12.h, z0.h\n"
- "add a_ptr1, a_ptr1, #0x20\n"
- ".inst 0x6460e5b1 // bfmmla z17.s, z13.h, z0.h\n"
- "ld1rqh z1.h, p7/z, [a_ptr1, #-0x10]\n"
- ".inst 0x6460e5d2 // bfmmla z18.s, z14.h, z0.h\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- ".inst 0x6460e5f3 // bfmmla z19.s, z15.h, z0.h\n"
- "ld1rqh z0.h, p7/z, [%[a_ptr0], #-0x10]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- ".inst 0x6468e590 // bfmmla z16.s, z12.h, z8.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- ".inst 0x6468e5b1 // bfmmla z17.s, z13.h, z8.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- ".inst 0x6468e5d2 // bfmmla z18.s, z14.h, z8.h\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- ".inst 0x6468e5f3 // bfmmla z19.s, z15.h, z8.h\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "trn1 z8.d, z0.d, z1.d\n"
- "b.ne 3b\n"
- "2:\n"
- "cbz %[regs], 4f\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "ld1rqh z4.h, p7/z, [%[a_ptr0]]\n"
- ".inst 0x6468e590 // bfmmla z16.s, z12.h, z8.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0]]\n"
- ".inst 0x6468e5b1 // bfmmla z17.s, z13.h, z8.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x6468e5d2 // bfmmla z18.s, z14.h, z8.h\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x6468e5f3 // bfmmla z19.s, z15.h, z8.h\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x6460e590 // bfmmla z16.s, z12.h, z0.h\n"
- "ld1rqh z5.h, p7/z, [a_ptr1]\n"
- ".inst 0x6460e5b1 // bfmmla z17.s, z13.h, z0.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x6460e5d2 // bfmmla z18.s, z14.h, z0.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x6460e5f3 // bfmmla z19.s, z15.h, z0.h\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "trn1 z0.d, z4.d, z5.d\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "trn2 z8.d, z4.d, z5.d\n"
- "ld1rqh z1.h, p6/z, [a_ptr1, #0x10]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "addvl a_ptr1, a_ptr1, #2\n"
- ".inst 0x6460e590 // bfmmla z16.s, z12.h, z0.h\n"
- ".inst 0x6460e5b1 // bfmmla z17.s, z13.h, z0.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- ".inst 0x6460e5d2 // bfmmla z18.s, z14.h, z0.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- ".inst 0x6460e5f3 // bfmmla z19.s, z15.h, z0.h\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #-4\n"
- ".inst 0x6468e590 // bfmmla z16.s, z12.h, z8.h\n"
- "ld1rqh z0.h, p6/z, [%[a_ptr0], #0x10]\n"
- ".inst 0x6468e5b1 // bfmmla z17.s, z13.h, z8.h\n"
- "addvl %[a_ptr0], %[a_ptr0], #2\n"
- ".inst 0x6468e5d2 // bfmmla z18.s, z14.h, z8.h\n"
- ".inst 0x6468e5f3 // bfmmla z19.s, z15.h, z8.h\n"
- "trn1 z8.d, z0.d, z1.d\n"
- "cbz %[blocks], 5f\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "ld1h z12.h, p7/z, [%[b_ptr0]]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x6468e590 // bfmmla z16.s, z12.h, z8.h\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x6468e5b1 // bfmmla z17.s, z13.h, z8.h\n"
- ".inst 0x6468e5d2 // bfmmla z18.s, z14.h, z8.h\n"
- ".inst 0x6468e5f3 // bfmmla z19.s, z15.h, z8.h\n"
- "b.eq 5f\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x6460e590 // bfmmla z16.s, z12.h, z0.h\n"
- ".inst 0x6460e5b1 // bfmmla z17.s, z13.h, z0.h\n"
- ".inst 0x6460e5d2 // bfmmla z18.s, z14.h, z0.h\n"
- ".inst 0x6460e5f3 // bfmmla z19.s, z15.h, z0.h\n"
- "b 5f\n"
- "4:\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "ld1rqh z4.h, p6/z, [%[a_ptr0]]\n"
- ".inst 0x6468e590 // bfmmla z16.s, z12.h, z8.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0]]\n"
- ".inst 0x6468e5b1 // bfmmla z17.s, z13.h, z8.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x6468e5d2 // bfmmla z18.s, z14.h, z8.h\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x6468e5f3 // bfmmla z19.s, z15.h, z8.h\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x6460e590 // bfmmla z16.s, z12.h, z0.h\n"
- "ld1rqh z5.h, p6/z, [a_ptr1]\n"
- ".inst 0x6460e5b1 // bfmmla z17.s, z13.h, z0.h\n"
- "addvl %[b_ptr0], %[b_ptr0], #4\n"
- ".inst 0x6460e5d2 // bfmmla z18.s, z14.h, z0.h\n"
- "addvl %[a_ptr0], %[a_ptr0], #1\n"
- ".inst 0x6460e5f3 // bfmmla z19.s, z15.h, z0.h\n"
- "addvl a_ptr1, a_ptr1, #1\n"
- "trn1 z0.d, z4.d, z5.d\n"
- "cbz %[blocks], 5f\n"
- "trn2 z8.d, z4.d, z5.d\n"
- "ld1h z12.h, p7/z, [%[b_ptr0]]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x6460e590 // bfmmla z16.s, z12.h, z0.h\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x6460e5b1 // bfmmla z17.s, z13.h, z0.h\n"
- ".inst 0x6460e5d2 // bfmmla z18.s, z14.h, z0.h\n"
- ".inst 0x6460e5f3 // bfmmla z19.s, z15.h, z0.h\n"
- "b.eq 5f\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x6468e590 // bfmmla z16.s, z12.h, z8.h\n"
- ".inst 0x6468e5b1 // bfmmla z17.s, z13.h, z8.h\n"
- ".inst 0x6468e5d2 // bfmmla z18.s, z14.h, z8.h\n"
- ".inst 0x6468e5f3 // bfmmla z19.s, z15.h, z8.h\n"
- "5:\n"
- "ld1rw z14.s, p7/z, [%[minptr]]\n"
- "ld1rw z15.s, p7/z, [%[maxptr]]\n"
- "fmax z16.s, p7/m, z16.s, z14.s\n"
- "fmax z17.s, p7/m, z17.s, z14.s\n"
- "fmax z18.s, p7/m, z18.s, z14.s\n"
- "fmax z19.s, p7/m, z19.s, z14.s\n"
- "fmin z16.s, p7/m, z16.s, z15.s\n"
- "fmin z17.s, p7/m, z17.s, z15.s\n"
- "fmin z18.s, p7/m, z18.s, z15.s\n"
- "fmin z19.s, p7/m, z19.s, z15.s\n"
- "uzp1 z0.s, z16.s, z17.s\n"
- "uzp2 z1.s, z16.s, z17.s\n"
- "uzp1 z2.s, z18.s, z19.s\n"
- "uzp2 z3.s, z18.s, z19.s\n"
- "st1w z0.s, p0, [%[c_ptr0]]\n"
- "st1w z1.s, p0, [c_ptr1]\n"
- "st1w z2.s, p1, [%[c_ptr0], #1, MUL VL]\n"
- "addvl %[c_ptr0], %[c_ptr0], #2\n"
- "st1w z3.s, p1, [c_ptr1, #1, MUL VL]\n"
- ".unreq a_ptr1\n"
- ".unreq c_ptr1\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks)
- : [width] "r" (width), [accumulate] "r" (static_cast<uint64_t>(accumulate)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers)
- : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "cc", "memory"
- );
- break;
- case 3:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "c_ptr1 .req X2\n"
- "c_ptr2 .req X3\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "whilelt p6.h, %[temp], %[leftovers]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "ptrue p7.h\n"
- "whilelt p1.s, %[temp], %[width]\n"
- "cbnz %[accumulate], 1f\n"
- "mov z3.h, #0\n"
- "ld1w z15.s, p0/z, [%[biasptr]]\n"
- "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ld1rqh z1.h, p7/z, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "zip1 z16.s, z15.s, z15.s\n"
- "ld1rqh z2.h, p7/z, [a_ptr2]\n"
- "zip2 z17.s, z15.s, z15.s\n"
- "ld1w z15.s, p1/z, [%[biasptr], #1, MUL VL]\n"
- "trn1 z8.d, z0.d, z1.d\n"
- "ld1h z12.h, p7/z, [%[b_ptr0]]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "zip1 z18.s, z15.s, z15.s\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "zip2 z19.s, z15.s, z15.s\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "trn1 z9.d, z2.d, z3.d\n"
- "addvl %[b_ptr0], %[b_ptr0], #4\n"
- "mov z20.d, z16.d\n"
- "mov z21.d, z17.d\n"
- "mov z22.d, z18.d\n"
- "mov z23.d, z19.d\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "mov z3.h, #0\n"
- "ld1w z13.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z14.s, p0/z, [c_ptr1]\n"
- "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ld1rqh z1.h, p7/z, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "zip1 z16.s, z13.s, z14.s\n"
- "ld1rqh z2.h, p7/z, [a_ptr2]\n"
- "zip2 z17.s, z13.s, z14.s\n"
- "ld1w z13.s, p1/z, [%[c_ptr0], #1, MUL VL]\n"
- "trn1 z8.d, z0.d, z1.d\n"
- "ld1w z14.s, p1/z, [c_ptr1, #1, MUL VL]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0]]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "trn1 z9.d, z2.d, z3.d\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "zip1 z18.s, z13.s, z14.s\n"
- "zip2 z19.s, z13.s, z14.s\n"
- "ld1w z13.s, p0/z, [c_ptr2]\n"
- "mov z14.s, #0\n"
- "zip1 z20.s, z13.s, z14.s\n"
- "zip2 z21.s, z13.s, z14.s\n"
- "ld1w z13.s, p1/z, [c_ptr2, #1, MUL VL]\n"
- "mov z14.s, #0\n"
- "zip1 z22.s, z13.s, z14.s\n"
- "zip2 z23.s, z13.s, z14.s\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #4\n"
- "cbz %[loops], 2f\n"
- "3:\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "ld1rqh z4.h, p7/z, [%[a_ptr0]]\n"
- "trn2 z1.d, z2.d, z3.d\n"
- "ld1rqh z5.h, p7/z, [a_ptr1]\n"
- ".inst 0x6468e590 // bfmmla z16.s, z12.h, z8.h\n"
- "ld1rqh z6.h, p7/z, [a_ptr2]\n"
- ".inst 0x6468e5b1 // bfmmla z17.s, z13.h, z8.h\n"
- "subs %[loops], %[loops], #0x1\n"
- ".inst 0x6468e5d2 // bfmmla z18.s, z14.h, z8.h\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
- ".inst 0x6468e5f3 // bfmmla z19.s, z15.h, z8.h\n"
- "add a_ptr1, a_ptr1, #0x20\n"
- "trn2 z8.d, z4.d, z5.d\n"
- "add a_ptr2, a_ptr2, #0x20\n"
- ".inst 0x6469e594 // bfmmla z20.s, z12.h, z9.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0]]\n"
- ".inst 0x6469e5b5 // bfmmla z21.s, z13.h, z9.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x6469e5d6 // bfmmla z22.s, z14.h, z9.h\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x6469e5f7 // bfmmla z23.s, z15.h, z9.h\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "mov z7.h, #0\n"
- "ld1rqh z2.h, p7/z, [a_ptr2, #-0x10]\n"
- ".inst 0x6460e590 // bfmmla z16.s, z12.h, z0.h\n"
- ".inst 0x6460e5b1 // bfmmla z17.s, z13.h, z0.h\n"
- ".inst 0x6460e5d2 // bfmmla z18.s, z14.h, z0.h\n"
- "trn2 z9.d, z6.d, z7.d\n"
- ".inst 0x6460e5f3 // bfmmla z19.s, z15.h, z0.h\n"
- "trn1 z0.d, z4.d, z5.d\n"
- ".inst 0x6461e594 // bfmmla z20.s, z12.h, z1.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x6461e5b5 // bfmmla z21.s, z13.h, z1.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x6461e5d6 // bfmmla z22.s, z14.h, z1.h\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x6461e5f7 // bfmmla z23.s, z15.h, z1.h\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "trn1 z1.d, z6.d, z7.d\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- ".inst 0x6460e590 // bfmmla z16.s, z12.h, z0.h\n"
- ".inst 0x6460e5b1 // bfmmla z17.s, z13.h, z0.h\n"
- ".inst 0x6460e5d2 // bfmmla z18.s, z14.h, z0.h\n"
- ".inst 0x6460e5f3 // bfmmla z19.s, z15.h, z0.h\n"
- "ld1rqh z0.h, p7/z, [%[a_ptr0], #-0x10]\n"
- ".inst 0x6461e594 // bfmmla z20.s, z12.h, z1.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- ".inst 0x6461e5b5 // bfmmla z21.s, z13.h, z1.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- ".inst 0x6461e5d6 // bfmmla z22.s, z14.h, z1.h\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- ".inst 0x6461e5f7 // bfmmla z23.s, z15.h, z1.h\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "mov z3.h, #0\n"
- "ld1rqh z1.h, p7/z, [a_ptr1, #-0x10]\n"
- ".inst 0x6468e590 // bfmmla z16.s, z12.h, z8.h\n"
- ".inst 0x6468e5b1 // bfmmla z17.s, z13.h, z8.h\n"
- ".inst 0x6468e5d2 // bfmmla z18.s, z14.h, z8.h\n"
- ".inst 0x6468e5f3 // bfmmla z19.s, z15.h, z8.h\n"
- "trn1 z8.d, z0.d, z1.d\n"
- ".inst 0x6469e594 // bfmmla z20.s, z12.h, z9.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- ".inst 0x6469e5b5 // bfmmla z21.s, z13.h, z9.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- ".inst 0x6469e5d6 // bfmmla z22.s, z14.h, z9.h\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- ".inst 0x6469e5f7 // bfmmla z23.s, z15.h, z9.h\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "trn1 z9.d, z2.d, z3.d\n"
- "b.ne 3b\n"
- "2:\n"
- "cbz %[regs], 4f\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "ld1rqh z4.h, p7/z, [%[a_ptr0]]\n"
- "trn2 z1.d, z2.d, z3.d\n"
- "ld1rqh z5.h, p7/z, [a_ptr1]\n"
- ".inst 0x6468e590 // bfmmla z16.s, z12.h, z8.h\n"
- "ld1rqh z6.h, p7/z, [a_ptr2]\n"
- ".inst 0x6468e5b1 // bfmmla z17.s, z13.h, z8.h\n"
- "ld1rqh z2.h, p6/z, [a_ptr2, #0x10]\n"
- ".inst 0x6468e5d2 // bfmmla z18.s, z14.h, z8.h\n"
- "addvl a_ptr2, a_ptr2, #2\n"
- ".inst 0x6468e5f3 // bfmmla z19.s, z15.h, z8.h\n"
- "trn2 z8.d, z4.d, z5.d\n"
- ".inst 0x6469e594 // bfmmla z20.s, z12.h, z9.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0]]\n"
- ".inst 0x6469e5b5 // bfmmla z21.s, z13.h, z9.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x6469e5d6 // bfmmla z22.s, z14.h, z9.h\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x6469e5f7 // bfmmla z23.s, z15.h, z9.h\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "mov z7.h, #0\n"
- ".inst 0x6460e590 // bfmmla z16.s, z12.h, z0.h\n"
- ".inst 0x6460e5b1 // bfmmla z17.s, z13.h, z0.h\n"
- ".inst 0x6460e5d2 // bfmmla z18.s, z14.h, z0.h\n"
- "trn2 z9.d, z6.d, z7.d\n"
- ".inst 0x6460e5f3 // bfmmla z19.s, z15.h, z0.h\n"
- "trn1 z0.d, z4.d, z5.d\n"
- ".inst 0x6461e594 // bfmmla z20.s, z12.h, z1.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x6461e5b5 // bfmmla z21.s, z13.h, z1.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x6461e5d6 // bfmmla z22.s, z14.h, z1.h\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x6461e5f7 // bfmmla z23.s, z15.h, z1.h\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "trn1 z1.d, z6.d, z7.d\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- ".inst 0x6460e590 // bfmmla z16.s, z12.h, z0.h\n"
- ".inst 0x6460e5b1 // bfmmla z17.s, z13.h, z0.h\n"
- ".inst 0x6460e5d2 // bfmmla z18.s, z14.h, z0.h\n"
- ".inst 0x6460e5f3 // bfmmla z19.s, z15.h, z0.h\n"
- "ld1rqh z0.h, p6/z, [%[a_ptr0], #0x10]\n"
- ".inst 0x6461e594 // bfmmla z20.s, z12.h, z1.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- ".inst 0x6461e5b5 // bfmmla z21.s, z13.h, z1.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- ".inst 0x6461e5d6 // bfmmla z22.s, z14.h, z1.h\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- ".inst 0x6461e5f7 // bfmmla z23.s, z15.h, z1.h\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "mov z3.h, #0\n"
- "ld1rqh z1.h, p6/z, [a_ptr1, #0x10]\n"
- ".inst 0x6468e590 // bfmmla z16.s, z12.h, z8.h\n"
- "addvl %[b_ptr0], %[b_ptr0], #-4\n"
- ".inst 0x6468e5b1 // bfmmla z17.s, z13.h, z8.h\n"
- "addvl %[a_ptr0], %[a_ptr0], #2\n"
- ".inst 0x6468e5d2 // bfmmla z18.s, z14.h, z8.h\n"
- "addvl a_ptr1, a_ptr1, #2\n"
- ".inst 0x6468e5f3 // bfmmla z19.s, z15.h, z8.h\n"
- "trn1 z8.d, z0.d, z1.d\n"
- ".inst 0x6469e594 // bfmmla z20.s, z12.h, z9.h\n"
- ".inst 0x6469e5b5 // bfmmla z21.s, z13.h, z9.h\n"
- ".inst 0x6469e5d6 // bfmmla z22.s, z14.h, z9.h\n"
- ".inst 0x6469e5f7 // bfmmla z23.s, z15.h, z9.h\n"
- "trn1 z9.d, z2.d, z3.d\n"
- "cbz %[blocks], 5f\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "ld1h z12.h, p7/z, [%[b_ptr0]]\n"
- "trn2 z1.d, z2.d, z3.d\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- ".inst 0x6468e590 // bfmmla z16.s, z12.h, z8.h\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x6468e5b1 // bfmmla z17.s, z13.h, z8.h\n"
- ".inst 0x6468e5d2 // bfmmla z18.s, z14.h, z8.h\n"
- ".inst 0x6469e594 // bfmmla z20.s, z12.h, z9.h\n"
- ".inst 0x6468e5f3 // bfmmla z19.s, z15.h, z8.h\n"
- ".inst 0x6469e5b5 // bfmmla z21.s, z13.h, z9.h\n"
- ".inst 0x6469e5d6 // bfmmla z22.s, z14.h, z9.h\n"
- ".inst 0x6469e5f7 // bfmmla z23.s, z15.h, z9.h\n"
- "b.eq 5f\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x6460e590 // bfmmla z16.s, z12.h, z0.h\n"
- ".inst 0x6460e5b1 // bfmmla z17.s, z13.h, z0.h\n"
- ".inst 0x6460e5d2 // bfmmla z18.s, z14.h, z0.h\n"
- ".inst 0x6460e5f3 // bfmmla z19.s, z15.h, z0.h\n"
- ".inst 0x6461e594 // bfmmla z20.s, z12.h, z1.h\n"
- ".inst 0x6461e5b5 // bfmmla z21.s, z13.h, z1.h\n"
- ".inst 0x6461e5d6 // bfmmla z22.s, z14.h, z1.h\n"
- ".inst 0x6461e5f7 // bfmmla z23.s, z15.h, z1.h\n"
- "b 5f\n"
- "4:\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "ld1rqh z4.h, p6/z, [%[a_ptr0]]\n"
- "trn2 z1.d, z2.d, z3.d\n"
- "ld1rqh z5.h, p6/z, [a_ptr1]\n"
- ".inst 0x6468e590 // bfmmla z16.s, z12.h, z8.h\n"
- "ld1rqh z6.h, p6/z, [a_ptr2]\n"
- ".inst 0x6468e5b1 // bfmmla z17.s, z13.h, z8.h\n"
- "addvl %[a_ptr0], %[a_ptr0], #1\n"
- ".inst 0x6468e5d2 // bfmmla z18.s, z14.h, z8.h\n"
- "addvl a_ptr1, a_ptr1, #1\n"
- ".inst 0x6468e5f3 // bfmmla z19.s, z15.h, z8.h\n"
- "addvl a_ptr2, a_ptr2, #1\n"
- ".inst 0x6469e594 // bfmmla z20.s, z12.h, z9.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0]]\n"
- ".inst 0x6469e5b5 // bfmmla z21.s, z13.h, z9.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x6469e5d6 // bfmmla z22.s, z14.h, z9.h\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x6469e5f7 // bfmmla z23.s, z15.h, z9.h\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "mov z7.h, #0\n"
- "addvl %[b_ptr0], %[b_ptr0], #4\n"
- ".inst 0x6460e590 // bfmmla z16.s, z12.h, z0.h\n"
- ".inst 0x6460e5b1 // bfmmla z17.s, z13.h, z0.h\n"
- ".inst 0x6460e5d2 // bfmmla z18.s, z14.h, z0.h\n"
- ".inst 0x6460e5f3 // bfmmla z19.s, z15.h, z0.h\n"
- "trn1 z0.d, z4.d, z5.d\n"
- ".inst 0x6461e594 // bfmmla z20.s, z12.h, z1.h\n"
- ".inst 0x6461e5b5 // bfmmla z21.s, z13.h, z1.h\n"
- ".inst 0x6461e5d6 // bfmmla z22.s, z14.h, z1.h\n"
- ".inst 0x6461e5f7 // bfmmla z23.s, z15.h, z1.h\n"
- "trn1 z1.d, z6.d, z7.d\n"
- "cbz %[blocks], 5f\n"
- "trn2 z9.d, z6.d, z7.d\n"
- "ld1h z12.h, p7/z, [%[b_ptr0]]\n"
- "trn2 z8.d, z4.d, z5.d\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- ".inst 0x6460e590 // bfmmla z16.s, z12.h, z0.h\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x6460e5b1 // bfmmla z17.s, z13.h, z0.h\n"
- ".inst 0x6460e5d2 // bfmmla z18.s, z14.h, z0.h\n"
- ".inst 0x6461e594 // bfmmla z20.s, z12.h, z1.h\n"
- ".inst 0x6460e5f3 // bfmmla z19.s, z15.h, z0.h\n"
- ".inst 0x6461e5b5 // bfmmla z21.s, z13.h, z1.h\n"
- ".inst 0x6461e5d6 // bfmmla z22.s, z14.h, z1.h\n"
- ".inst 0x6461e5f7 // bfmmla z23.s, z15.h, z1.h\n"
- "b.eq 5f\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x6468e590 // bfmmla z16.s, z12.h, z8.h\n"
- ".inst 0x6468e5b1 // bfmmla z17.s, z13.h, z8.h\n"
- ".inst 0x6468e5d2 // bfmmla z18.s, z14.h, z8.h\n"
- ".inst 0x6468e5f3 // bfmmla z19.s, z15.h, z8.h\n"
- ".inst 0x6469e594 // bfmmla z20.s, z12.h, z9.h\n"
- ".inst 0x6469e5b5 // bfmmla z21.s, z13.h, z9.h\n"
- ".inst 0x6469e5d6 // bfmmla z22.s, z14.h, z9.h\n"
- ".inst 0x6469e5f7 // bfmmla z23.s, z15.h, z9.h\n"
- "5:\n"
- "ld1rw z14.s, p7/z, [%[minptr]]\n"
- "ld1rw z15.s, p7/z, [%[maxptr]]\n"
- "fmax z16.s, p7/m, z16.s, z14.s\n"
- "fmax z17.s, p7/m, z17.s, z14.s\n"
- "fmax z18.s, p7/m, z18.s, z14.s\n"
- "fmax z19.s, p7/m, z19.s, z14.s\n"
- "fmin z16.s, p7/m, z16.s, z15.s\n"
- "fmin z17.s, p7/m, z17.s, z15.s\n"
- "fmin z18.s, p7/m, z18.s, z15.s\n"
- "fmin z19.s, p7/m, z19.s, z15.s\n"
- "fmax z20.s, p7/m, z20.s, z14.s\n"
- "uzp1 z0.s, z16.s, z17.s\n"
- "uzp2 z1.s, z16.s, z17.s\n"
- "uzp1 z2.s, z18.s, z19.s\n"
- "uzp2 z3.s, z18.s, z19.s\n"
- "st1w z0.s, p0, [%[c_ptr0]]\n"
- "fmin z20.s, p7/m, z20.s, z15.s\n"
- "fmax z21.s, p7/m, z21.s, z14.s\n"
- "fmax z22.s, p7/m, z22.s, z14.s\n"
- "st1w z1.s, p0, [c_ptr1]\n"
- "fmax z23.s, p7/m, z23.s, z14.s\n"
- "fmin z21.s, p7/m, z21.s, z15.s\n"
- "st1w z2.s, p1, [%[c_ptr0], #1, MUL VL]\n"
- "fmin z22.s, p7/m, z22.s, z15.s\n"
- "addvl %[c_ptr0], %[c_ptr0], #2\n"
- "fmin z23.s, p7/m, z23.s, z15.s\n"
- "st1w z3.s, p1, [c_ptr1, #1, MUL VL]\n"
- "uzp1 z4.s, z20.s, z21.s\n"
- "uzp1 z5.s, z22.s, z23.s\n"
- "st1w z4.s, p0, [c_ptr2]\n"
- "st1w z5.s, p1, [c_ptr2, #1, MUL VL]\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks)
- : [width] "r" (width), [accumulate] "r" (static_cast<uint64_t>(accumulate)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers)
- : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "cc", "memory"
- );
- break;
- case 4:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "a_ptr3 .req X2\n"
- "c_ptr1 .req X3\n"
- "c_ptr2 .req X4\n"
- "c_ptr3 .req X5\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "add a_ptr3, a_ptr2, %[lda]\n"
- "add c_ptr3, c_ptr2, %[ldc]\n"
- "whilelt p6.h, %[temp], %[leftovers]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "ptrue p7.h\n"
- "whilelt p1.s, %[temp], %[width]\n"
- "cbnz %[accumulate], 1f\n"
- "ld1w z15.s, p0/z, [%[biasptr]]\n"
- "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ld1rqh z1.h, p7/z, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "zip1 z16.s, z15.s, z15.s\n"
- "ld1rqh z2.h, p7/z, [a_ptr2]\n"
- "zip2 z17.s, z15.s, z15.s\n"
- "ld1w z15.s, p1/z, [%[biasptr], #1, MUL VL]\n"
- "trn1 z8.d, z0.d, z1.d\n"
- "ld1rqh z3.h, p7/z, [a_ptr3]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0]]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "zip1 z18.s, z15.s, z15.s\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "zip2 z19.s, z15.s, z15.s\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "trn1 z9.d, z2.d, z3.d\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "mov z20.d, z16.d\n"
- "add a_ptr3, a_ptr3, #0x10\n"
- "mov z21.d, z17.d\n"
- "addvl %[b_ptr0], %[b_ptr0], #4\n"
- "mov z22.d, z18.d\n"
- "mov z23.d, z19.d\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "ld1w z13.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z14.s, p0/z, [c_ptr1]\n"
- "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ld1rqh z1.h, p7/z, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "zip1 z16.s, z13.s, z14.s\n"
- "ld1rqh z2.h, p7/z, [a_ptr2]\n"
- "zip2 z17.s, z13.s, z14.s\n"
- "ld1w z13.s, p1/z, [%[c_ptr0], #1, MUL VL]\n"
- "trn1 z8.d, z0.d, z1.d\n"
- "ld1w z14.s, p1/z, [c_ptr1, #1, MUL VL]\n"
- "ld1rqh z3.h, p7/z, [a_ptr3]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "ld1h z12.h, p7/z, [%[b_ptr0]]\n"
- "add a_ptr3, a_ptr3, #0x10\n"
- "zip1 z18.s, z13.s, z14.s\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "zip2 z19.s, z13.s, z14.s\n"
- "ld1w z13.s, p0/z, [c_ptr2]\n"
- "trn1 z9.d, z2.d, z3.d\n"
- "ld1w z14.s, p0/z, [c_ptr3]\n"
- "zip1 z20.s, z13.s, z14.s\n"
- "zip2 z21.s, z13.s, z14.s\n"
- "ld1w z13.s, p1/z, [c_ptr2, #1, MUL VL]\n"
- "ld1w z14.s, p1/z, [c_ptr3, #1, MUL VL]\n"
- "zip1 z22.s, z13.s, z14.s\n"
- "zip2 z23.s, z13.s, z14.s\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #4\n"
- "cbz %[loops], 2f\n"
- "3:\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "ld1rqh z4.h, p7/z, [%[a_ptr0]]\n"
- "trn2 z1.d, z2.d, z3.d\n"
- "ld1rqh z5.h, p7/z, [a_ptr1]\n"
- ".inst 0x6468e590 // bfmmla z16.s, z12.h, z8.h\n"
- "ld1rqh z6.h, p7/z, [a_ptr2]\n"
- ".inst 0x6468e5b1 // bfmmla z17.s, z13.h, z8.h\n"
- "ld1rqh z7.h, p7/z, [a_ptr3]\n"
- ".inst 0x6468e5d2 // bfmmla z18.s, z14.h, z8.h\n"
- "subs %[loops], %[loops], #0x1\n"
- ".inst 0x6468e5f3 // bfmmla z19.s, z15.h, z8.h\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
- "trn2 z8.d, z4.d, z5.d\n"
- "add a_ptr1, a_ptr1, #0x20\n"
- ".inst 0x6469e594 // bfmmla z20.s, z12.h, z9.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0]]\n"
- ".inst 0x6469e5b5 // bfmmla z21.s, z13.h, z9.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x6469e5d6 // bfmmla z22.s, z14.h, z9.h\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x6469e5f7 // bfmmla z23.s, z15.h, z9.h\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "trn2 z9.d, z6.d, z7.d\n"
- "add a_ptr2, a_ptr2, #0x20\n"
- ".inst 0x6460e590 // bfmmla z16.s, z12.h, z0.h\n"
- "ld1rqh z2.h, p7/z, [a_ptr2, #-0x10]\n"
- ".inst 0x6460e5b1 // bfmmla z17.s, z13.h, z0.h\n"
- "add a_ptr3, a_ptr3, #0x20\n"
- ".inst 0x6460e5d2 // bfmmla z18.s, z14.h, z0.h\n"
- "ld1rqh z3.h, p7/z, [a_ptr3, #-0x10]\n"
- ".inst 0x6460e5f3 // bfmmla z19.s, z15.h, z0.h\n"
- "trn1 z0.d, z4.d, z5.d\n"
- ".inst 0x6461e594 // bfmmla z20.s, z12.h, z1.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x6461e5b5 // bfmmla z21.s, z13.h, z1.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x6461e5d6 // bfmmla z22.s, z14.h, z1.h\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x6461e5f7 // bfmmla z23.s, z15.h, z1.h\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "trn1 z1.d, z6.d, z7.d\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- ".inst 0x6460e590 // bfmmla z16.s, z12.h, z0.h\n"
- ".inst 0x6460e5b1 // bfmmla z17.s, z13.h, z0.h\n"
- ".inst 0x6460e5d2 // bfmmla z18.s, z14.h, z0.h\n"
- ".inst 0x6460e5f3 // bfmmla z19.s, z15.h, z0.h\n"
- "ld1rqh z0.h, p7/z, [%[a_ptr0], #-0x10]\n"
- ".inst 0x6461e594 // bfmmla z20.s, z12.h, z1.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- ".inst 0x6461e5b5 // bfmmla z21.s, z13.h, z1.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- ".inst 0x6461e5d6 // bfmmla z22.s, z14.h, z1.h\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- ".inst 0x6461e5f7 // bfmmla z23.s, z15.h, z1.h\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- ".inst 0x6468e590 // bfmmla z16.s, z12.h, z8.h\n"
- "ld1rqh z1.h, p7/z, [a_ptr1, #-0x10]\n"
- ".inst 0x6468e5b1 // bfmmla z17.s, z13.h, z8.h\n"
- ".inst 0x6468e5d2 // bfmmla z18.s, z14.h, z8.h\n"
- ".inst 0x6468e5f3 // bfmmla z19.s, z15.h, z8.h\n"
- "trn1 z8.d, z0.d, z1.d\n"
- ".inst 0x6469e594 // bfmmla z20.s, z12.h, z9.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- ".inst 0x6469e5b5 // bfmmla z21.s, z13.h, z9.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- ".inst 0x6469e5d6 // bfmmla z22.s, z14.h, z9.h\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- ".inst 0x6469e5f7 // bfmmla z23.s, z15.h, z9.h\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "trn1 z9.d, z2.d, z3.d\n"
- "b.ne 3b\n"
- "2:\n"
- "cbz %[regs], 4f\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "ld1rqh z4.h, p7/z, [%[a_ptr0]]\n"
- "trn2 z1.d, z2.d, z3.d\n"
- "ld1rqh z5.h, p7/z, [a_ptr1]\n"
- ".inst 0x6468e590 // bfmmla z16.s, z12.h, z8.h\n"
- "ld1rqh z6.h, p7/z, [a_ptr2]\n"
- ".inst 0x6468e5b1 // bfmmla z17.s, z13.h, z8.h\n"
- "ld1rqh z7.h, p7/z, [a_ptr3]\n"
- ".inst 0x6468e5d2 // bfmmla z18.s, z14.h, z8.h\n"
- "ld1rqh z2.h, p6/z, [a_ptr2, #0x10]\n"
- ".inst 0x6468e5f3 // bfmmla z19.s, z15.h, z8.h\n"
- "ld1rqh z3.h, p6/z, [a_ptr3, #0x10]\n"
- "trn2 z8.d, z4.d, z5.d\n"
- "addvl a_ptr2, a_ptr2, #2\n"
- ".inst 0x6469e594 // bfmmla z20.s, z12.h, z9.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0]]\n"
- ".inst 0x6469e5b5 // bfmmla z21.s, z13.h, z9.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x6469e5d6 // bfmmla z22.s, z14.h, z9.h\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x6469e5f7 // bfmmla z23.s, z15.h, z9.h\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "trn2 z9.d, z6.d, z7.d\n"
- "addvl a_ptr3, a_ptr3, #2\n"
- ".inst 0x6460e590 // bfmmla z16.s, z12.h, z0.h\n"
- ".inst 0x6460e5b1 // bfmmla z17.s, z13.h, z0.h\n"
- ".inst 0x6460e5d2 // bfmmla z18.s, z14.h, z0.h\n"
- ".inst 0x6460e5f3 // bfmmla z19.s, z15.h, z0.h\n"
- "trn1 z0.d, z4.d, z5.d\n"
- ".inst 0x6461e594 // bfmmla z20.s, z12.h, z1.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x6461e5b5 // bfmmla z21.s, z13.h, z1.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x6461e5d6 // bfmmla z22.s, z14.h, z1.h\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x6461e5f7 // bfmmla z23.s, z15.h, z1.h\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "trn1 z1.d, z6.d, z7.d\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- ".inst 0x6460e590 // bfmmla z16.s, z12.h, z0.h\n"
- ".inst 0x6460e5b1 // bfmmla z17.s, z13.h, z0.h\n"
- ".inst 0x6460e5d2 // bfmmla z18.s, z14.h, z0.h\n"
- ".inst 0x6460e5f3 // bfmmla z19.s, z15.h, z0.h\n"
- "ld1rqh z0.h, p6/z, [%[a_ptr0], #0x10]\n"
- ".inst 0x6461e594 // bfmmla z20.s, z12.h, z1.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- ".inst 0x6461e5b5 // bfmmla z21.s, z13.h, z1.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- ".inst 0x6461e5d6 // bfmmla z22.s, z14.h, z1.h\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- ".inst 0x6461e5f7 // bfmmla z23.s, z15.h, z1.h\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- ".inst 0x6468e590 // bfmmla z16.s, z12.h, z8.h\n"
- "ld1rqh z1.h, p6/z, [a_ptr1, #0x10]\n"
- ".inst 0x6468e5b1 // bfmmla z17.s, z13.h, z8.h\n"
- "addvl %[b_ptr0], %[b_ptr0], #-4\n"
- ".inst 0x6468e5d2 // bfmmla z18.s, z14.h, z8.h\n"
- "addvl %[a_ptr0], %[a_ptr0], #2\n"
- ".inst 0x6468e5f3 // bfmmla z19.s, z15.h, z8.h\n"
- "addvl a_ptr1, a_ptr1, #2\n"
- "trn1 z8.d, z0.d, z1.d\n"
- ".inst 0x6469e594 // bfmmla z20.s, z12.h, z9.h\n"
- ".inst 0x6469e5b5 // bfmmla z21.s, z13.h, z9.h\n"
- ".inst 0x6469e5d6 // bfmmla z22.s, z14.h, z9.h\n"
- ".inst 0x6469e5f7 // bfmmla z23.s, z15.h, z9.h\n"
- "trn1 z9.d, z2.d, z3.d\n"
- "cbz %[blocks], 5f\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "ld1h z12.h, p7/z, [%[b_ptr0]]\n"
- "trn2 z1.d, z2.d, z3.d\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- ".inst 0x6468e590 // bfmmla z16.s, z12.h, z8.h\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x6468e5b1 // bfmmla z17.s, z13.h, z8.h\n"
- ".inst 0x6468e5d2 // bfmmla z18.s, z14.h, z8.h\n"
- ".inst 0x6469e594 // bfmmla z20.s, z12.h, z9.h\n"
- ".inst 0x6468e5f3 // bfmmla z19.s, z15.h, z8.h\n"
- ".inst 0x6469e5b5 // bfmmla z21.s, z13.h, z9.h\n"
- ".inst 0x6469e5d6 // bfmmla z22.s, z14.h, z9.h\n"
- ".inst 0x6469e5f7 // bfmmla z23.s, z15.h, z9.h\n"
- "b.eq 5f\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x6460e590 // bfmmla z16.s, z12.h, z0.h\n"
- ".inst 0x6460e5b1 // bfmmla z17.s, z13.h, z0.h\n"
- ".inst 0x6460e5d2 // bfmmla z18.s, z14.h, z0.h\n"
- ".inst 0x6460e5f3 // bfmmla z19.s, z15.h, z0.h\n"
- ".inst 0x6461e594 // bfmmla z20.s, z12.h, z1.h\n"
- ".inst 0x6461e5b5 // bfmmla z21.s, z13.h, z1.h\n"
- ".inst 0x6461e5d6 // bfmmla z22.s, z14.h, z1.h\n"
- ".inst 0x6461e5f7 // bfmmla z23.s, z15.h, z1.h\n"
- "b 5f\n"
- "4:\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "ld1rqh z4.h, p6/z, [%[a_ptr0]]\n"
- "trn2 z1.d, z2.d, z3.d\n"
- "ld1rqh z5.h, p6/z, [a_ptr1]\n"
- ".inst 0x6468e590 // bfmmla z16.s, z12.h, z8.h\n"
- "ld1rqh z6.h, p6/z, [a_ptr2]\n"
- ".inst 0x6468e5b1 // bfmmla z17.s, z13.h, z8.h\n"
- "ld1rqh z7.h, p6/z, [a_ptr3]\n"
- ".inst 0x6468e5d2 // bfmmla z18.s, z14.h, z8.h\n"
- "addvl %[a_ptr0], %[a_ptr0], #1\n"
- ".inst 0x6468e5f3 // bfmmla z19.s, z15.h, z8.h\n"
- "addvl a_ptr1, a_ptr1, #1\n"
- ".inst 0x6469e594 // bfmmla z20.s, z12.h, z9.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0]]\n"
- ".inst 0x6469e5b5 // bfmmla z21.s, z13.h, z9.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x6469e5d6 // bfmmla z22.s, z14.h, z9.h\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x6469e5f7 // bfmmla z23.s, z15.h, z9.h\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x6460e590 // bfmmla z16.s, z12.h, z0.h\n"
- "addvl %[b_ptr0], %[b_ptr0], #4\n"
- ".inst 0x6460e5b1 // bfmmla z17.s, z13.h, z0.h\n"
- "addvl a_ptr2, a_ptr2, #1\n"
- ".inst 0x6460e5d2 // bfmmla z18.s, z14.h, z0.h\n"
- "addvl a_ptr3, a_ptr3, #1\n"
- ".inst 0x6460e5f3 // bfmmla z19.s, z15.h, z0.h\n"
- "trn1 z0.d, z4.d, z5.d\n"
- ".inst 0x6461e594 // bfmmla z20.s, z12.h, z1.h\n"
- ".inst 0x6461e5b5 // bfmmla z21.s, z13.h, z1.h\n"
- ".inst 0x6461e5d6 // bfmmla z22.s, z14.h, z1.h\n"
- ".inst 0x6461e5f7 // bfmmla z23.s, z15.h, z1.h\n"
- "trn1 z1.d, z6.d, z7.d\n"
- "cbz %[blocks], 5f\n"
- "trn2 z9.d, z6.d, z7.d\n"
- "ld1h z12.h, p7/z, [%[b_ptr0]]\n"
- "trn2 z8.d, z4.d, z5.d\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- ".inst 0x6460e590 // bfmmla z16.s, z12.h, z0.h\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x6460e5b1 // bfmmla z17.s, z13.h, z0.h\n"
- ".inst 0x6460e5d2 // bfmmla z18.s, z14.h, z0.h\n"
- ".inst 0x6461e594 // bfmmla z20.s, z12.h, z1.h\n"
- ".inst 0x6460e5f3 // bfmmla z19.s, z15.h, z0.h\n"
- ".inst 0x6461e5b5 // bfmmla z21.s, z13.h, z1.h\n"
- ".inst 0x6461e5d6 // bfmmla z22.s, z14.h, z1.h\n"
- ".inst 0x6461e5f7 // bfmmla z23.s, z15.h, z1.h\n"
- "b.eq 5f\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x6468e590 // bfmmla z16.s, z12.h, z8.h\n"
- ".inst 0x6468e5b1 // bfmmla z17.s, z13.h, z8.h\n"
- ".inst 0x6468e5d2 // bfmmla z18.s, z14.h, z8.h\n"
- ".inst 0x6468e5f3 // bfmmla z19.s, z15.h, z8.h\n"
- ".inst 0x6469e594 // bfmmla z20.s, z12.h, z9.h\n"
- ".inst 0x6469e5b5 // bfmmla z21.s, z13.h, z9.h\n"
- ".inst 0x6469e5d6 // bfmmla z22.s, z14.h, z9.h\n"
- ".inst 0x6469e5f7 // bfmmla z23.s, z15.h, z9.h\n"
- "5:\n"
- "ld1rw z14.s, p7/z, [%[minptr]]\n"
- "ld1rw z15.s, p7/z, [%[maxptr]]\n"
- "fmax z16.s, p7/m, z16.s, z14.s\n"
- "fmax z17.s, p7/m, z17.s, z14.s\n"
- "fmax z18.s, p7/m, z18.s, z14.s\n"
- "fmax z19.s, p7/m, z19.s, z14.s\n"
- "fmin z16.s, p7/m, z16.s, z15.s\n"
- "fmin z17.s, p7/m, z17.s, z15.s\n"
- "fmin z18.s, p7/m, z18.s, z15.s\n"
- "fmin z19.s, p7/m, z19.s, z15.s\n"
- "fmax z20.s, p7/m, z20.s, z14.s\n"
- "uzp1 z0.s, z16.s, z17.s\n"
- "uzp2 z1.s, z16.s, z17.s\n"
- "uzp1 z2.s, z18.s, z19.s\n"
- "uzp2 z3.s, z18.s, z19.s\n"
- "st1w z0.s, p0, [%[c_ptr0]]\n"
- "fmin z20.s, p7/m, z20.s, z15.s\n"
- "fmax z21.s, p7/m, z21.s, z14.s\n"
- "fmax z22.s, p7/m, z22.s, z14.s\n"
- "st1w z1.s, p0, [c_ptr1]\n"
- "fmax z23.s, p7/m, z23.s, z14.s\n"
- "fmin z21.s, p7/m, z21.s, z15.s\n"
- "st1w z2.s, p1, [%[c_ptr0], #1, MUL VL]\n"
- "fmin z22.s, p7/m, z22.s, z15.s\n"
- "addvl %[c_ptr0], %[c_ptr0], #2\n"
- "fmin z23.s, p7/m, z23.s, z15.s\n"
- "st1w z3.s, p1, [c_ptr1, #1, MUL VL]\n"
- "uzp1 z4.s, z20.s, z21.s\n"
- "uzp2 z5.s, z20.s, z21.s\n"
- "uzp1 z6.s, z22.s, z23.s\n"
- "st1w z4.s, p0, [c_ptr2]\n"
- "uzp2 z7.s, z22.s, z23.s\n"
- "st1w z5.s, p0, [c_ptr3]\n"
- "st1w z6.s, p1, [c_ptr2, #1, MUL VL]\n"
- "st1w z7.s, p1, [c_ptr3, #1, MUL VL]\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq a_ptr3\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- ".unreq c_ptr3\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks)
- : [width] "r" (width), [accumulate] "r" (static_cast<uint64_t>(accumulate)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers)
- : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "x4", "x5", "cc", "memory"
- );
- break;
- case 5:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "a_ptr3 .req X2\n"
- "a_ptr4 .req X3\n"
- "c_ptr1 .req X4\n"
- "c_ptr2 .req X5\n"
- "c_ptr3 .req X6\n"
- "c_ptr4 .req X7\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "add a_ptr3, a_ptr2, %[lda]\n"
- "add c_ptr3, c_ptr2, %[ldc]\n"
- "add a_ptr4, a_ptr3, %[lda]\n"
- "add c_ptr4, c_ptr3, %[ldc]\n"
- "whilelt p6.h, %[temp], %[leftovers]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "ptrue p7.h\n"
- "whilelt p1.s, %[temp], %[width]\n"
- "cbnz %[accumulate], 1f\n"
- "mov z5.h, #0\n"
- "ld1w z15.s, p0/z, [%[biasptr]]\n"
- "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ld1rqh z1.h, p7/z, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "zip1 z16.s, z15.s, z15.s\n"
- "ld1rqh z2.h, p7/z, [a_ptr2]\n"
- "zip2 z17.s, z15.s, z15.s\n"
- "ld1w z15.s, p1/z, [%[biasptr], #1, MUL VL]\n"
- "trn1 z8.d, z0.d, z1.d\n"
- "ld1rqh z3.h, p7/z, [a_ptr3]\n"
- "ld1rqh z4.h, p7/z, [a_ptr4]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "zip1 z18.s, z15.s, z15.s\n"
- "ld1h z12.h, p7/z, [%[b_ptr0]]\n"
- "zip2 z19.s, z15.s, z15.s\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "trn1 z9.d, z2.d, z3.d\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "trn1 z10.d, z4.d, z5.d\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "mov z20.d, z16.d\n"
- "add a_ptr3, a_ptr3, #0x10\n"
- "mov z21.d, z17.d\n"
- "add a_ptr4, a_ptr4, #0x10\n"
- "mov z22.d, z18.d\n"
- "addvl %[b_ptr0], %[b_ptr0], #4\n"
- "mov z23.d, z19.d\n"
- "mov z24.d, z16.d\n"
- "mov z25.d, z17.d\n"
- "mov z26.d, z18.d\n"
- "mov z27.d, z19.d\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "mov z5.h, #0\n"
- "ld1w z13.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z14.s, p0/z, [c_ptr1]\n"
- "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ld1rqh z1.h, p7/z, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "zip1 z16.s, z13.s, z14.s\n"
- "ld1rqh z2.h, p7/z, [a_ptr2]\n"
- "zip2 z17.s, z13.s, z14.s\n"
- "ld1w z13.s, p1/z, [%[c_ptr0], #1, MUL VL]\n"
- "trn1 z8.d, z0.d, z1.d\n"
- "ld1w z14.s, p1/z, [c_ptr1, #1, MUL VL]\n"
- "ld1rqh z3.h, p7/z, [a_ptr3]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "ld1rqh z4.h, p7/z, [a_ptr4]\n"
- "add a_ptr3, a_ptr3, #0x10\n"
- "zip1 z18.s, z13.s, z14.s\n"
- "ld1h z12.h, p7/z, [%[b_ptr0]]\n"
- "zip2 z19.s, z13.s, z14.s\n"
- "ld1w z13.s, p0/z, [c_ptr2]\n"
- "trn1 z9.d, z2.d, z3.d\n"
- "ld1w z14.s, p0/z, [c_ptr3]\n"
- "trn1 z10.d, z4.d, z5.d\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "add a_ptr4, a_ptr4, #0x10\n"
- "zip1 z20.s, z13.s, z14.s\n"
- "zip2 z21.s, z13.s, z14.s\n"
- "ld1w z13.s, p1/z, [c_ptr2, #1, MUL VL]\n"
- "ld1w z14.s, p1/z, [c_ptr3, #1, MUL VL]\n"
- "zip1 z22.s, z13.s, z14.s\n"
- "zip2 z23.s, z13.s, z14.s\n"
- "ld1w z13.s, p0/z, [c_ptr4]\n"
- "mov z14.s, #0\n"
- "zip1 z24.s, z13.s, z14.s\n"
- "zip2 z25.s, z13.s, z14.s\n"
- "ld1w z13.s, p1/z, [c_ptr4, #1, MUL VL]\n"
- "mov z14.s, #0\n"
- "zip1 z26.s, z13.s, z14.s\n"
- "zip2 z27.s, z13.s, z14.s\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #4\n"
- "cbz %[loops], 2f\n"
- "3:\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "ld1rqh z6.h, p7/z, [a_ptr2]\n"
- "trn2 z1.d, z2.d, z3.d\n"
- "ld1rqh z7.h, p7/z, [a_ptr3]\n"
- "trn2 z2.d, z4.d, z5.d\n"
- "ld1rqh z4.h, p7/z, [%[a_ptr0]]\n"
- ".inst 0x6468e590 // bfmmla z16.s, z12.h, z8.h\n"
- "ld1rqh z5.h, p7/z, [a_ptr1]\n"
- ".inst 0x6468e5b1 // bfmmla z17.s, z13.h, z8.h\n"
- "subs %[loops], %[loops], #0x1\n"
- ".inst 0x6468e5d2 // bfmmla z18.s, z14.h, z8.h\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
- ".inst 0x6468e5f3 // bfmmla z19.s, z15.h, z8.h\n"
- "ld1rqh z8.h, p7/z, [a_ptr4]\n"
- ".inst 0x6469e594 // bfmmla z20.s, z12.h, z9.h\n"
- "add a_ptr1, a_ptr1, #0x20\n"
- ".inst 0x6469e5b5 // bfmmla z21.s, z13.h, z9.h\n"
- "add a_ptr2, a_ptr2, #0x20\n"
- ".inst 0x6469e5d6 // bfmmla z22.s, z14.h, z9.h\n"
- "add a_ptr3, a_ptr3, #0x20\n"
- ".inst 0x6469e5f7 // bfmmla z23.s, z15.h, z9.h\n"
- "ld1rqh z3.h, p7/z, [a_ptr3, #-0x10]\n"
- ".inst 0x646ae598 // bfmmla z24.s, z12.h, z10.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0]]\n"
- ".inst 0x646ae5b9 // bfmmla z25.s, z13.h, z10.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x646ae5da // bfmmla z26.s, z14.h, z10.h\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x646ae5fb // bfmmla z27.s, z15.h, z10.h\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "mov z9.h, #0\n"
- "add a_ptr4, a_ptr4, #0x20\n"
- ".inst 0x6460e590 // bfmmla z16.s, z12.h, z0.h\n"
- ".inst 0x6460e5b1 // bfmmla z17.s, z13.h, z0.h\n"
- "trn2 z10.d, z8.d, z9.d\n"
- ".inst 0x6460e5d2 // bfmmla z18.s, z14.h, z0.h\n"
- ".inst 0x6460e5f3 // bfmmla z19.s, z15.h, z0.h\n"
- "trn1 z0.d, z4.d, z5.d\n"
- ".inst 0x6461e594 // bfmmla z20.s, z12.h, z1.h\n"
- ".inst 0x6461e5b5 // bfmmla z21.s, z13.h, z1.h\n"
- ".inst 0x6461e5d6 // bfmmla z22.s, z14.h, z1.h\n"
- ".inst 0x6461e5f7 // bfmmla z23.s, z15.h, z1.h\n"
- "trn1 z1.d, z6.d, z7.d\n"
- ".inst 0x6462e598 // bfmmla z24.s, z12.h, z2.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x6462e5b9 // bfmmla z25.s, z13.h, z2.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x6462e5da // bfmmla z26.s, z14.h, z2.h\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x6462e5fb // bfmmla z27.s, z15.h, z2.h\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "trn1 z2.d, z8.d, z9.d\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "trn2 z9.d, z6.d, z7.d\n"
- "trn2 z8.d, z4.d, z5.d\n"
- "ld1rqh z4.h, p7/z, [a_ptr4, #-0x10]\n"
- ".inst 0x6460e590 // bfmmla z16.s, z12.h, z0.h\n"
- ".inst 0x6460e5b1 // bfmmla z17.s, z13.h, z0.h\n"
- ".inst 0x6460e5d2 // bfmmla z18.s, z14.h, z0.h\n"
- ".inst 0x6460e5f3 // bfmmla z19.s, z15.h, z0.h\n"
- "ld1rqh z0.h, p7/z, [%[a_ptr0], #-0x10]\n"
- ".inst 0x6461e594 // bfmmla z20.s, z12.h, z1.h\n"
- ".inst 0x6461e5b5 // bfmmla z21.s, z13.h, z1.h\n"
- ".inst 0x6461e5d6 // bfmmla z22.s, z14.h, z1.h\n"
- ".inst 0x6461e5f7 // bfmmla z23.s, z15.h, z1.h\n"
- "ld1rqh z1.h, p7/z, [a_ptr1, #-0x10]\n"
- ".inst 0x6462e598 // bfmmla z24.s, z12.h, z2.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- ".inst 0x6462e5b9 // bfmmla z25.s, z13.h, z2.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- ".inst 0x6462e5da // bfmmla z26.s, z14.h, z2.h\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- ".inst 0x6462e5fb // bfmmla z27.s, z15.h, z2.h\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "mov z5.h, #0\n"
- "ld1rqh z2.h, p7/z, [a_ptr2, #-0x10]\n"
- ".inst 0x6468e590 // bfmmla z16.s, z12.h, z8.h\n"
- ".inst 0x6468e5b1 // bfmmla z17.s, z13.h, z8.h\n"
- ".inst 0x6468e5d2 // bfmmla z18.s, z14.h, z8.h\n"
- ".inst 0x6468e5f3 // bfmmla z19.s, z15.h, z8.h\n"
- "trn1 z8.d, z0.d, z1.d\n"
- ".inst 0x6469e594 // bfmmla z20.s, z12.h, z9.h\n"
- ".inst 0x6469e5b5 // bfmmla z21.s, z13.h, z9.h\n"
- ".inst 0x6469e5d6 // bfmmla z22.s, z14.h, z9.h\n"
- ".inst 0x6469e5f7 // bfmmla z23.s, z15.h, z9.h\n"
- "trn1 z9.d, z2.d, z3.d\n"
- ".inst 0x646ae598 // bfmmla z24.s, z12.h, z10.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- ".inst 0x646ae5b9 // bfmmla z25.s, z13.h, z10.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- ".inst 0x646ae5da // bfmmla z26.s, z14.h, z10.h\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- ".inst 0x646ae5fb // bfmmla z27.s, z15.h, z10.h\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "trn1 z10.d, z4.d, z5.d\n"
- "b.ne 3b\n"
- "2:\n"
- "cbz %[regs], 4f\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "ld1rqh z6.h, p7/z, [a_ptr2]\n"
- "trn2 z1.d, z2.d, z3.d\n"
- "ld1rqh z7.h, p7/z, [a_ptr3]\n"
- "trn2 z2.d, z4.d, z5.d\n"
- "ld1rqh z4.h, p7/z, [%[a_ptr0]]\n"
- ".inst 0x6468e590 // bfmmla z16.s, z12.h, z8.h\n"
- "ld1rqh z5.h, p7/z, [a_ptr1]\n"
- ".inst 0x6468e5b1 // bfmmla z17.s, z13.h, z8.h\n"
- "ld1rqh z3.h, p6/z, [a_ptr3, #0x10]\n"
- ".inst 0x6468e5d2 // bfmmla z18.s, z14.h, z8.h\n"
- "addvl a_ptr3, a_ptr3, #2\n"
- ".inst 0x6468e5f3 // bfmmla z19.s, z15.h, z8.h\n"
- "ld1rqh z8.h, p7/z, [a_ptr4]\n"
- ".inst 0x6469e594 // bfmmla z20.s, z12.h, z9.h\n"
- ".inst 0x6469e5b5 // bfmmla z21.s, z13.h, z9.h\n"
- ".inst 0x6469e5d6 // bfmmla z22.s, z14.h, z9.h\n"
- ".inst 0x6469e5f7 // bfmmla z23.s, z15.h, z9.h\n"
- ".inst 0x646ae598 // bfmmla z24.s, z12.h, z10.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0]]\n"
- ".inst 0x646ae5b9 // bfmmla z25.s, z13.h, z10.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x646ae5da // bfmmla z26.s, z14.h, z10.h\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x646ae5fb // bfmmla z27.s, z15.h, z10.h\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "mov z9.h, #0\n"
- ".inst 0x6460e590 // bfmmla z16.s, z12.h, z0.h\n"
- ".inst 0x6460e5b1 // bfmmla z17.s, z13.h, z0.h\n"
- ".inst 0x6460e5d2 // bfmmla z18.s, z14.h, z0.h\n"
- "trn2 z10.d, z8.d, z9.d\n"
- ".inst 0x6460e5f3 // bfmmla z19.s, z15.h, z0.h\n"
- "trn1 z0.d, z4.d, z5.d\n"
- ".inst 0x6461e594 // bfmmla z20.s, z12.h, z1.h\n"
- ".inst 0x6461e5b5 // bfmmla z21.s, z13.h, z1.h\n"
- ".inst 0x6461e5d6 // bfmmla z22.s, z14.h, z1.h\n"
- ".inst 0x6461e5f7 // bfmmla z23.s, z15.h, z1.h\n"
- "trn1 z1.d, z6.d, z7.d\n"
- ".inst 0x6462e598 // bfmmla z24.s, z12.h, z2.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x6462e5b9 // bfmmla z25.s, z13.h, z2.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x6462e5da // bfmmla z26.s, z14.h, z2.h\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x6462e5fb // bfmmla z27.s, z15.h, z2.h\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "trn1 z2.d, z8.d, z9.d\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "trn2 z9.d, z6.d, z7.d\n"
- "trn2 z8.d, z4.d, z5.d\n"
- "ld1rqh z4.h, p6/z, [a_ptr4, #0x10]\n"
- ".inst 0x6460e590 // bfmmla z16.s, z12.h, z0.h\n"
- "addvl a_ptr4, a_ptr4, #2\n"
- ".inst 0x6460e5b1 // bfmmla z17.s, z13.h, z0.h\n"
- ".inst 0x6460e5d2 // bfmmla z18.s, z14.h, z0.h\n"
- ".inst 0x6460e5f3 // bfmmla z19.s, z15.h, z0.h\n"
- "ld1rqh z0.h, p6/z, [%[a_ptr0], #0x10]\n"
- ".inst 0x6461e594 // bfmmla z20.s, z12.h, z1.h\n"
- "addvl %[a_ptr0], %[a_ptr0], #2\n"
- ".inst 0x6461e5b5 // bfmmla z21.s, z13.h, z1.h\n"
- ".inst 0x6461e5d6 // bfmmla z22.s, z14.h, z1.h\n"
- ".inst 0x6461e5f7 // bfmmla z23.s, z15.h, z1.h\n"
- "ld1rqh z1.h, p6/z, [a_ptr1, #0x10]\n"
- ".inst 0x6462e598 // bfmmla z24.s, z12.h, z2.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- ".inst 0x6462e5b9 // bfmmla z25.s, z13.h, z2.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- ".inst 0x6462e5da // bfmmla z26.s, z14.h, z2.h\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- ".inst 0x6462e5fb // bfmmla z27.s, z15.h, z2.h\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "mov z5.h, #0\n"
- "ld1rqh z2.h, p6/z, [a_ptr2, #0x10]\n"
- ".inst 0x6468e590 // bfmmla z16.s, z12.h, z8.h\n"
- "addvl %[b_ptr0], %[b_ptr0], #-4\n"
- ".inst 0x6468e5b1 // bfmmla z17.s, z13.h, z8.h\n"
- "addvl a_ptr1, a_ptr1, #2\n"
- ".inst 0x6468e5d2 // bfmmla z18.s, z14.h, z8.h\n"
- "addvl a_ptr2, a_ptr2, #2\n"
- ".inst 0x6468e5f3 // bfmmla z19.s, z15.h, z8.h\n"
- "trn1 z8.d, z0.d, z1.d\n"
- ".inst 0x6469e594 // bfmmla z20.s, z12.h, z9.h\n"
- ".inst 0x6469e5b5 // bfmmla z21.s, z13.h, z9.h\n"
- ".inst 0x6469e5d6 // bfmmla z22.s, z14.h, z9.h\n"
- ".inst 0x6469e5f7 // bfmmla z23.s, z15.h, z9.h\n"
- "trn1 z9.d, z2.d, z3.d\n"
- ".inst 0x646ae598 // bfmmla z24.s, z12.h, z10.h\n"
- ".inst 0x646ae5b9 // bfmmla z25.s, z13.h, z10.h\n"
- ".inst 0x646ae5da // bfmmla z26.s, z14.h, z10.h\n"
- ".inst 0x646ae5fb // bfmmla z27.s, z15.h, z10.h\n"
- "trn1 z10.d, z4.d, z5.d\n"
- "cbz %[blocks], 5f\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "ld1h z12.h, p7/z, [%[b_ptr0]]\n"
- "trn2 z1.d, z2.d, z3.d\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "trn2 z2.d, z4.d, z5.d\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- ".inst 0x6468e590 // bfmmla z16.s, z12.h, z8.h\n"
- ".inst 0x6468e5b1 // bfmmla z17.s, z13.h, z8.h\n"
- ".inst 0x6468e5d2 // bfmmla z18.s, z14.h, z8.h\n"
- ".inst 0x6468e5f3 // bfmmla z19.s, z15.h, z8.h\n"
- ".inst 0x6469e594 // bfmmla z20.s, z12.h, z9.h\n"
- ".inst 0x6469e5b5 // bfmmla z21.s, z13.h, z9.h\n"
- ".inst 0x6469e5d6 // bfmmla z22.s, z14.h, z9.h\n"
- ".inst 0x6469e5f7 // bfmmla z23.s, z15.h, z9.h\n"
- ".inst 0x646ae598 // bfmmla z24.s, z12.h, z10.h\n"
- ".inst 0x646ae5b9 // bfmmla z25.s, z13.h, z10.h\n"
- ".inst 0x646ae5da // bfmmla z26.s, z14.h, z10.h\n"
- ".inst 0x646ae5fb // bfmmla z27.s, z15.h, z10.h\n"
- "b.eq 5f\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x6460e590 // bfmmla z16.s, z12.h, z0.h\n"
- ".inst 0x6460e5b1 // bfmmla z17.s, z13.h, z0.h\n"
- ".inst 0x6460e5d2 // bfmmla z18.s, z14.h, z0.h\n"
- ".inst 0x6460e5f3 // bfmmla z19.s, z15.h, z0.h\n"
- ".inst 0x6461e594 // bfmmla z20.s, z12.h, z1.h\n"
- ".inst 0x6461e5b5 // bfmmla z21.s, z13.h, z1.h\n"
- ".inst 0x6461e5d6 // bfmmla z22.s, z14.h, z1.h\n"
- ".inst 0x6461e5f7 // bfmmla z23.s, z15.h, z1.h\n"
- ".inst 0x6462e598 // bfmmla z24.s, z12.h, z2.h\n"
- ".inst 0x6462e5b9 // bfmmla z25.s, z13.h, z2.h\n"
- ".inst 0x6462e5da // bfmmla z26.s, z14.h, z2.h\n"
- ".inst 0x6462e5fb // bfmmla z27.s, z15.h, z2.h\n"
- "b 5f\n"
- "4:\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "ld1rqh z6.h, p6/z, [a_ptr2]\n"
- "trn2 z1.d, z2.d, z3.d\n"
- "ld1rqh z7.h, p6/z, [a_ptr3]\n"
- "trn2 z2.d, z4.d, z5.d\n"
- "ld1rqh z4.h, p6/z, [%[a_ptr0]]\n"
- ".inst 0x6468e590 // bfmmla z16.s, z12.h, z8.h\n"
- "ld1rqh z5.h, p6/z, [a_ptr1]\n"
- ".inst 0x6468e5b1 // bfmmla z17.s, z13.h, z8.h\n"
- "addvl %[a_ptr0], %[a_ptr0], #1\n"
- ".inst 0x6468e5d2 // bfmmla z18.s, z14.h, z8.h\n"
- "addvl a_ptr1, a_ptr1, #1\n"
- ".inst 0x6468e5f3 // bfmmla z19.s, z15.h, z8.h\n"
- "ld1rqh z8.h, p6/z, [a_ptr4]\n"
- ".inst 0x6469e594 // bfmmla z20.s, z12.h, z9.h\n"
- "addvl a_ptr2, a_ptr2, #1\n"
- ".inst 0x6469e5b5 // bfmmla z21.s, z13.h, z9.h\n"
- "addvl a_ptr3, a_ptr3, #1\n"
- ".inst 0x6469e5d6 // bfmmla z22.s, z14.h, z9.h\n"
- "addvl a_ptr4, a_ptr4, #1\n"
- ".inst 0x6469e5f7 // bfmmla z23.s, z15.h, z9.h\n"
- ".inst 0x646ae598 // bfmmla z24.s, z12.h, z10.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0]]\n"
- ".inst 0x646ae5b9 // bfmmla z25.s, z13.h, z10.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x646ae5da // bfmmla z26.s, z14.h, z10.h\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x646ae5fb // bfmmla z27.s, z15.h, z10.h\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "mov z9.h, #0\n"
- "addvl %[b_ptr0], %[b_ptr0], #4\n"
- ".inst 0x6460e590 // bfmmla z16.s, z12.h, z0.h\n"
- ".inst 0x6460e5b1 // bfmmla z17.s, z13.h, z0.h\n"
- ".inst 0x6460e5d2 // bfmmla z18.s, z14.h, z0.h\n"
- ".inst 0x6460e5f3 // bfmmla z19.s, z15.h, z0.h\n"
- "trn1 z0.d, z4.d, z5.d\n"
- ".inst 0x6461e594 // bfmmla z20.s, z12.h, z1.h\n"
- ".inst 0x6461e5b5 // bfmmla z21.s, z13.h, z1.h\n"
- ".inst 0x6461e5d6 // bfmmla z22.s, z14.h, z1.h\n"
- ".inst 0x6461e5f7 // bfmmla z23.s, z15.h, z1.h\n"
- "trn1 z1.d, z6.d, z7.d\n"
- ".inst 0x6462e598 // bfmmla z24.s, z12.h, z2.h\n"
- ".inst 0x6462e5b9 // bfmmla z25.s, z13.h, z2.h\n"
- ".inst 0x6462e5da // bfmmla z26.s, z14.h, z2.h\n"
- ".inst 0x6462e5fb // bfmmla z27.s, z15.h, z2.h\n"
- "trn1 z2.d, z8.d, z9.d\n"
- "cbz %[blocks], 5f\n"
- "trn2 z10.d, z8.d, z9.d\n"
- "ld1h z12.h, p7/z, [%[b_ptr0]]\n"
- "trn2 z9.d, z6.d, z7.d\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "trn2 z8.d, z4.d, z5.d\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- ".inst 0x6460e590 // bfmmla z16.s, z12.h, z0.h\n"
- ".inst 0x6460e5b1 // bfmmla z17.s, z13.h, z0.h\n"
- ".inst 0x6460e5d2 // bfmmla z18.s, z14.h, z0.h\n"
- ".inst 0x6460e5f3 // bfmmla z19.s, z15.h, z0.h\n"
- ".inst 0x6461e594 // bfmmla z20.s, z12.h, z1.h\n"
- ".inst 0x6461e5b5 // bfmmla z21.s, z13.h, z1.h\n"
- ".inst 0x6461e5d6 // bfmmla z22.s, z14.h, z1.h\n"
- ".inst 0x6461e5f7 // bfmmla z23.s, z15.h, z1.h\n"
- ".inst 0x6462e598 // bfmmla z24.s, z12.h, z2.h\n"
- ".inst 0x6462e5b9 // bfmmla z25.s, z13.h, z2.h\n"
- ".inst 0x6462e5da // bfmmla z26.s, z14.h, z2.h\n"
- ".inst 0x6462e5fb // bfmmla z27.s, z15.h, z2.h\n"
- "b.eq 5f\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x6468e590 // bfmmla z16.s, z12.h, z8.h\n"
- ".inst 0x6468e5b1 // bfmmla z17.s, z13.h, z8.h\n"
- ".inst 0x6468e5d2 // bfmmla z18.s, z14.h, z8.h\n"
- ".inst 0x6468e5f3 // bfmmla z19.s, z15.h, z8.h\n"
- ".inst 0x6469e594 // bfmmla z20.s, z12.h, z9.h\n"
- ".inst 0x6469e5b5 // bfmmla z21.s, z13.h, z9.h\n"
- ".inst 0x6469e5d6 // bfmmla z22.s, z14.h, z9.h\n"
- ".inst 0x6469e5f7 // bfmmla z23.s, z15.h, z9.h\n"
- ".inst 0x646ae598 // bfmmla z24.s, z12.h, z10.h\n"
- ".inst 0x646ae5b9 // bfmmla z25.s, z13.h, z10.h\n"
- ".inst 0x646ae5da // bfmmla z26.s, z14.h, z10.h\n"
- ".inst 0x646ae5fb // bfmmla z27.s, z15.h, z10.h\n"
- "5:\n"
- "ld1rw z14.s, p7/z, [%[minptr]]\n"
- "ld1rw z15.s, p7/z, [%[maxptr]]\n"
- "fmax z16.s, p7/m, z16.s, z14.s\n"
- "fmax z17.s, p7/m, z17.s, z14.s\n"
- "fmax z18.s, p7/m, z18.s, z14.s\n"
- "fmax z19.s, p7/m, z19.s, z14.s\n"
- "fmin z16.s, p7/m, z16.s, z15.s\n"
- "fmin z17.s, p7/m, z17.s, z15.s\n"
- "fmin z18.s, p7/m, z18.s, z15.s\n"
- "fmin z19.s, p7/m, z19.s, z15.s\n"
- "fmax z20.s, p7/m, z20.s, z14.s\n"
- "uzp1 z0.s, z16.s, z17.s\n"
- "uzp2 z1.s, z16.s, z17.s\n"
- "uzp1 z2.s, z18.s, z19.s\n"
- "uzp2 z3.s, z18.s, z19.s\n"
- "st1w z0.s, p0, [%[c_ptr0]]\n"
- "fmin z20.s, p7/m, z20.s, z15.s\n"
- "fmax z21.s, p7/m, z21.s, z14.s\n"
- "fmax z22.s, p7/m, z22.s, z14.s\n"
- "st1w z1.s, p0, [c_ptr1]\n"
- "fmax z23.s, p7/m, z23.s, z14.s\n"
- "fmax z24.s, p7/m, z24.s, z14.s\n"
- "fmin z21.s, p7/m, z21.s, z15.s\n"
- "st1w z2.s, p1, [%[c_ptr0], #1, MUL VL]\n"
- "fmin z22.s, p7/m, z22.s, z15.s\n"
- "addvl %[c_ptr0], %[c_ptr0], #2\n"
- "fmin z23.s, p7/m, z23.s, z15.s\n"
- "st1w z3.s, p1, [c_ptr1, #1, MUL VL]\n"
- "uzp1 z4.s, z20.s, z21.s\n"
- "uzp2 z5.s, z20.s, z21.s\n"
- "fmin z24.s, p7/m, z24.s, z15.s\n"
- "uzp1 z6.s, z22.s, z23.s\n"
- "st1w z4.s, p0, [c_ptr2]\n"
- "uzp2 z7.s, z22.s, z23.s\n"
- "fmax z25.s, p7/m, z25.s, z14.s\n"
- "fmax z26.s, p7/m, z26.s, z14.s\n"
- "st1w z5.s, p0, [c_ptr3]\n"
- "fmax z27.s, p7/m, z27.s, z14.s\n"
- "fmin z25.s, p7/m, z25.s, z15.s\n"
- "st1w z6.s, p1, [c_ptr2, #1, MUL VL]\n"
- "fmin z26.s, p7/m, z26.s, z15.s\n"
- "fmin z27.s, p7/m, z27.s, z15.s\n"
- "uzp1 z8.s, z24.s, z25.s\n"
- "st1w z7.s, p1, [c_ptr3, #1, MUL VL]\n"
- "uzp1 z9.s, z26.s, z27.s\n"
- "st1w z8.s, p0, [c_ptr4]\n"
- "st1w z9.s, p1, [c_ptr4, #1, MUL VL]\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq a_ptr3\n"
- ".unreq a_ptr4\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- ".unreq c_ptr3\n"
- ".unreq c_ptr4\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks)
- : [width] "r" (width), [accumulate] "r" (static_cast<uint64_t>(accumulate)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers)
- : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "cc", "memory"
- );
- break;
- case 6:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "a_ptr3 .req X2\n"
- "a_ptr4 .req X3\n"
- "a_ptr5 .req X4\n"
- "c_ptr1 .req X5\n"
- "c_ptr2 .req X6\n"
- "c_ptr3 .req X7\n"
- "c_ptr4 .req X8\n"
- "c_ptr5 .req X9\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "add a_ptr3, a_ptr2, %[lda]\n"
- "add c_ptr3, c_ptr2, %[ldc]\n"
- "add a_ptr4, a_ptr3, %[lda]\n"
- "add c_ptr4, c_ptr3, %[ldc]\n"
- "add a_ptr5, a_ptr4, %[lda]\n"
- "add c_ptr5, c_ptr4, %[ldc]\n"
- "whilelt p6.h, %[temp], %[leftovers]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "ptrue p7.h\n"
- "whilelt p1.s, %[temp], %[width]\n"
- "cbnz %[accumulate], 1f\n"
- "ld1w z15.s, p0/z, [%[biasptr]]\n"
- "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ld1rqh z1.h, p7/z, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "zip1 z16.s, z15.s, z15.s\n"
- "ld1rqh z2.h, p7/z, [a_ptr2]\n"
- "zip2 z17.s, z15.s, z15.s\n"
- "ld1w z15.s, p1/z, [%[biasptr], #1, MUL VL]\n"
- "trn1 z8.d, z0.d, z1.d\n"
- "ld1rqh z3.h, p7/z, [a_ptr3]\n"
- "ld1rqh z4.h, p7/z, [a_ptr4]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "zip1 z18.s, z15.s, z15.s\n"
- "ld1rqh z5.h, p7/z, [a_ptr5]\n"
- "zip2 z19.s, z15.s, z15.s\n"
- "ld1h z12.h, p7/z, [%[b_ptr0]]\n"
- "trn1 z9.d, z2.d, z3.d\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "mov z20.d, z16.d\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "trn1 z10.d, z4.d, z5.d\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "mov z21.d, z17.d\n"
- "add a_ptr3, a_ptr3, #0x10\n"
- "mov z22.d, z18.d\n"
- "add a_ptr4, a_ptr4, #0x10\n"
- "mov z23.d, z19.d\n"
- "add a_ptr5, a_ptr5, #0x10\n"
- "mov z24.d, z16.d\n"
- "addvl %[b_ptr0], %[b_ptr0], #4\n"
- "mov z25.d, z17.d\n"
- "mov z26.d, z18.d\n"
- "mov z27.d, z19.d\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "ld1w z13.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z14.s, p0/z, [c_ptr1]\n"
- "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ld1rqh z1.h, p7/z, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "zip1 z16.s, z13.s, z14.s\n"
- "ld1rqh z2.h, p7/z, [a_ptr2]\n"
- "zip2 z17.s, z13.s, z14.s\n"
- "ld1w z13.s, p1/z, [%[c_ptr0], #1, MUL VL]\n"
- "trn1 z8.d, z0.d, z1.d\n"
- "ld1w z14.s, p1/z, [c_ptr1, #1, MUL VL]\n"
- "ld1rqh z3.h, p7/z, [a_ptr3]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "ld1rqh z4.h, p7/z, [a_ptr4]\n"
- "add a_ptr3, a_ptr3, #0x10\n"
- "zip1 z18.s, z13.s, z14.s\n"
- "ld1rqh z5.h, p7/z, [a_ptr5]\n"
- "zip2 z19.s, z13.s, z14.s\n"
- "ld1w z13.s, p0/z, [c_ptr2]\n"
- "trn1 z9.d, z2.d, z3.d\n"
- "ld1w z14.s, p0/z, [c_ptr3]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0]]\n"
- "add a_ptr4, a_ptr4, #0x10\n"
- "trn1 z10.d, z4.d, z5.d\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "zip1 z20.s, z13.s, z14.s\n"
- "add a_ptr5, a_ptr5, #0x10\n"
- "zip2 z21.s, z13.s, z14.s\n"
- "ld1w z13.s, p1/z, [c_ptr2, #1, MUL VL]\n"
- "ld1w z14.s, p1/z, [c_ptr3, #1, MUL VL]\n"
- "zip1 z22.s, z13.s, z14.s\n"
- "zip2 z23.s, z13.s, z14.s\n"
- "ld1w z13.s, p0/z, [c_ptr4]\n"
- "ld1w z14.s, p0/z, [c_ptr5]\n"
- "zip1 z24.s, z13.s, z14.s\n"
- "zip2 z25.s, z13.s, z14.s\n"
- "ld1w z13.s, p1/z, [c_ptr4, #1, MUL VL]\n"
- "ld1w z14.s, p1/z, [c_ptr5, #1, MUL VL]\n"
- "zip1 z26.s, z13.s, z14.s\n"
- "zip2 z27.s, z13.s, z14.s\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #4\n"
- "cbz %[loops], 2f\n"
- "3:\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "ld1rqh z6.h, p7/z, [a_ptr2]\n"
- "trn2 z1.d, z2.d, z3.d\n"
- "ld1rqh z7.h, p7/z, [a_ptr3]\n"
- "trn2 z2.d, z4.d, z5.d\n"
- "ld1rqh z4.h, p7/z, [%[a_ptr0]]\n"
- ".inst 0x6468e590 // bfmmla z16.s, z12.h, z8.h\n"
- "ld1rqh z5.h, p7/z, [a_ptr1]\n"
- ".inst 0x6468e5b1 // bfmmla z17.s, z13.h, z8.h\n"
- "subs %[loops], %[loops], #0x1\n"
- ".inst 0x6468e5d2 // bfmmla z18.s, z14.h, z8.h\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
- ".inst 0x6468e5f3 // bfmmla z19.s, z15.h, z8.h\n"
- "ld1rqh z8.h, p7/z, [a_ptr4]\n"
- ".inst 0x6469e594 // bfmmla z20.s, z12.h, z9.h\n"
- "add a_ptr1, a_ptr1, #0x20\n"
- ".inst 0x6469e5b5 // bfmmla z21.s, z13.h, z9.h\n"
- "add a_ptr2, a_ptr2, #0x20\n"
- ".inst 0x6469e5d6 // bfmmla z22.s, z14.h, z9.h\n"
- "add a_ptr3, a_ptr3, #0x20\n"
- ".inst 0x6469e5f7 // bfmmla z23.s, z15.h, z9.h\n"
- "ld1rqh z9.h, p7/z, [a_ptr5]\n"
- ".inst 0x646ae598 // bfmmla z24.s, z12.h, z10.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0]]\n"
- ".inst 0x646ae5b9 // bfmmla z25.s, z13.h, z10.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x646ae5da // bfmmla z26.s, z14.h, z10.h\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x646ae5fb // bfmmla z27.s, z15.h, z10.h\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "trn2 z10.d, z8.d, z9.d\n"
- "ld1rqh z3.h, p7/z, [a_ptr3, #-0x10]\n"
- ".inst 0x6460e590 // bfmmla z16.s, z12.h, z0.h\n"
- "add a_ptr4, a_ptr4, #0x20\n"
- ".inst 0x6460e5b1 // bfmmla z17.s, z13.h, z0.h\n"
- "add a_ptr5, a_ptr5, #0x20\n"
- ".inst 0x6460e5d2 // bfmmla z18.s, z14.h, z0.h\n"
- ".inst 0x6460e5f3 // bfmmla z19.s, z15.h, z0.h\n"
- "trn1 z0.d, z4.d, z5.d\n"
- ".inst 0x6461e594 // bfmmla z20.s, z12.h, z1.h\n"
- ".inst 0x6461e5b5 // bfmmla z21.s, z13.h, z1.h\n"
- ".inst 0x6461e5d6 // bfmmla z22.s, z14.h, z1.h\n"
- ".inst 0x6461e5f7 // bfmmla z23.s, z15.h, z1.h\n"
- "trn1 z1.d, z6.d, z7.d\n"
- ".inst 0x6462e598 // bfmmla z24.s, z12.h, z2.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x6462e5b9 // bfmmla z25.s, z13.h, z2.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x6462e5da // bfmmla z26.s, z14.h, z2.h\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x6462e5fb // bfmmla z27.s, z15.h, z2.h\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "trn1 z2.d, z8.d, z9.d\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "trn2 z9.d, z6.d, z7.d\n"
- "trn2 z8.d, z4.d, z5.d\n"
- "ld1rqh z4.h, p7/z, [a_ptr4, #-0x10]\n"
- ".inst 0x6460e590 // bfmmla z16.s, z12.h, z0.h\n"
- "ld1rqh z5.h, p7/z, [a_ptr5, #-0x10]\n"
- ".inst 0x6460e5b1 // bfmmla z17.s, z13.h, z0.h\n"
- ".inst 0x6460e5d2 // bfmmla z18.s, z14.h, z0.h\n"
- ".inst 0x6460e5f3 // bfmmla z19.s, z15.h, z0.h\n"
- "ld1rqh z0.h, p7/z, [%[a_ptr0], #-0x10]\n"
- ".inst 0x6461e594 // bfmmla z20.s, z12.h, z1.h\n"
- ".inst 0x6461e5b5 // bfmmla z21.s, z13.h, z1.h\n"
- ".inst 0x6461e5d6 // bfmmla z22.s, z14.h, z1.h\n"
- ".inst 0x6461e5f7 // bfmmla z23.s, z15.h, z1.h\n"
- "ld1rqh z1.h, p7/z, [a_ptr1, #-0x10]\n"
- ".inst 0x6462e598 // bfmmla z24.s, z12.h, z2.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- ".inst 0x6462e5b9 // bfmmla z25.s, z13.h, z2.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- ".inst 0x6462e5da // bfmmla z26.s, z14.h, z2.h\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- ".inst 0x6462e5fb // bfmmla z27.s, z15.h, z2.h\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- ".inst 0x6468e590 // bfmmla z16.s, z12.h, z8.h\n"
- "ld1rqh z2.h, p7/z, [a_ptr2, #-0x10]\n"
- ".inst 0x6468e5b1 // bfmmla z17.s, z13.h, z8.h\n"
- ".inst 0x6468e5d2 // bfmmla z18.s, z14.h, z8.h\n"
- ".inst 0x6468e5f3 // bfmmla z19.s, z15.h, z8.h\n"
- "trn1 z8.d, z0.d, z1.d\n"
- ".inst 0x6469e594 // bfmmla z20.s, z12.h, z9.h\n"
- ".inst 0x6469e5b5 // bfmmla z21.s, z13.h, z9.h\n"
- ".inst 0x6469e5d6 // bfmmla z22.s, z14.h, z9.h\n"
- ".inst 0x6469e5f7 // bfmmla z23.s, z15.h, z9.h\n"
- "trn1 z9.d, z2.d, z3.d\n"
- ".inst 0x646ae598 // bfmmla z24.s, z12.h, z10.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- ".inst 0x646ae5b9 // bfmmla z25.s, z13.h, z10.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- ".inst 0x646ae5da // bfmmla z26.s, z14.h, z10.h\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- ".inst 0x646ae5fb // bfmmla z27.s, z15.h, z10.h\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "trn1 z10.d, z4.d, z5.d\n"
- "b.ne 3b\n"
- "2:\n"
- "cbz %[regs], 4f\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "ld1rqh z6.h, p7/z, [a_ptr2]\n"
- "trn2 z1.d, z2.d, z3.d\n"
- "ld1rqh z7.h, p7/z, [a_ptr3]\n"
- "trn2 z2.d, z4.d, z5.d\n"
- "ld1rqh z4.h, p7/z, [%[a_ptr0]]\n"
- ".inst 0x6468e590 // bfmmla z16.s, z12.h, z8.h\n"
- "ld1rqh z5.h, p7/z, [a_ptr1]\n"
- ".inst 0x6468e5b1 // bfmmla z17.s, z13.h, z8.h\n"
- "ld1rqh z3.h, p6/z, [a_ptr3, #0x10]\n"
- ".inst 0x6468e5d2 // bfmmla z18.s, z14.h, z8.h\n"
- "addvl a_ptr3, a_ptr3, #2\n"
- ".inst 0x6468e5f3 // bfmmla z19.s, z15.h, z8.h\n"
- "ld1rqh z8.h, p7/z, [a_ptr4]\n"
- ".inst 0x6469e594 // bfmmla z20.s, z12.h, z9.h\n"
- ".inst 0x6469e5b5 // bfmmla z21.s, z13.h, z9.h\n"
- ".inst 0x6469e5d6 // bfmmla z22.s, z14.h, z9.h\n"
- ".inst 0x6469e5f7 // bfmmla z23.s, z15.h, z9.h\n"
- "ld1rqh z9.h, p7/z, [a_ptr5]\n"
- ".inst 0x646ae598 // bfmmla z24.s, z12.h, z10.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0]]\n"
- ".inst 0x646ae5b9 // bfmmla z25.s, z13.h, z10.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x646ae5da // bfmmla z26.s, z14.h, z10.h\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x646ae5fb // bfmmla z27.s, z15.h, z10.h\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "trn2 z10.d, z8.d, z9.d\n"
- ".inst 0x6460e590 // bfmmla z16.s, z12.h, z0.h\n"
- ".inst 0x6460e5b1 // bfmmla z17.s, z13.h, z0.h\n"
- ".inst 0x6460e5d2 // bfmmla z18.s, z14.h, z0.h\n"
- ".inst 0x6460e5f3 // bfmmla z19.s, z15.h, z0.h\n"
- "trn1 z0.d, z4.d, z5.d\n"
- ".inst 0x6461e594 // bfmmla z20.s, z12.h, z1.h\n"
- ".inst 0x6461e5b5 // bfmmla z21.s, z13.h, z1.h\n"
- ".inst 0x6461e5d6 // bfmmla z22.s, z14.h, z1.h\n"
- ".inst 0x6461e5f7 // bfmmla z23.s, z15.h, z1.h\n"
- "trn1 z1.d, z6.d, z7.d\n"
- ".inst 0x6462e598 // bfmmla z24.s, z12.h, z2.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x6462e5b9 // bfmmla z25.s, z13.h, z2.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x6462e5da // bfmmla z26.s, z14.h, z2.h\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x6462e5fb // bfmmla z27.s, z15.h, z2.h\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "trn1 z2.d, z8.d, z9.d\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "trn2 z9.d, z6.d, z7.d\n"
- "trn2 z8.d, z4.d, z5.d\n"
- "ld1rqh z4.h, p6/z, [a_ptr4, #0x10]\n"
- ".inst 0x6460e590 // bfmmla z16.s, z12.h, z0.h\n"
- "ld1rqh z5.h, p6/z, [a_ptr5, #0x10]\n"
- ".inst 0x6460e5b1 // bfmmla z17.s, z13.h, z0.h\n"
- "addvl a_ptr4, a_ptr4, #2\n"
- ".inst 0x6460e5d2 // bfmmla z18.s, z14.h, z0.h\n"
- "addvl a_ptr5, a_ptr5, #2\n"
- ".inst 0x6460e5f3 // bfmmla z19.s, z15.h, z0.h\n"
- "ld1rqh z0.h, p6/z, [%[a_ptr0], #0x10]\n"
- ".inst 0x6461e594 // bfmmla z20.s, z12.h, z1.h\n"
- "addvl %[a_ptr0], %[a_ptr0], #2\n"
- ".inst 0x6461e5b5 // bfmmla z21.s, z13.h, z1.h\n"
- ".inst 0x6461e5d6 // bfmmla z22.s, z14.h, z1.h\n"
- ".inst 0x6461e5f7 // bfmmla z23.s, z15.h, z1.h\n"
- "ld1rqh z1.h, p6/z, [a_ptr1, #0x10]\n"
- ".inst 0x6462e598 // bfmmla z24.s, z12.h, z2.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- ".inst 0x6462e5b9 // bfmmla z25.s, z13.h, z2.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- ".inst 0x6462e5da // bfmmla z26.s, z14.h, z2.h\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- ".inst 0x6462e5fb // bfmmla z27.s, z15.h, z2.h\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- ".inst 0x6468e590 // bfmmla z16.s, z12.h, z8.h\n"
- "ld1rqh z2.h, p6/z, [a_ptr2, #0x10]\n"
- ".inst 0x6468e5b1 // bfmmla z17.s, z13.h, z8.h\n"
- "addvl %[b_ptr0], %[b_ptr0], #-4\n"
- ".inst 0x6468e5d2 // bfmmla z18.s, z14.h, z8.h\n"
- "addvl a_ptr1, a_ptr1, #2\n"
- ".inst 0x6468e5f3 // bfmmla z19.s, z15.h, z8.h\n"
- "addvl a_ptr2, a_ptr2, #2\n"
- "trn1 z8.d, z0.d, z1.d\n"
- ".inst 0x6469e594 // bfmmla z20.s, z12.h, z9.h\n"
- ".inst 0x6469e5b5 // bfmmla z21.s, z13.h, z9.h\n"
- ".inst 0x6469e5d6 // bfmmla z22.s, z14.h, z9.h\n"
- ".inst 0x6469e5f7 // bfmmla z23.s, z15.h, z9.h\n"
- "trn1 z9.d, z2.d, z3.d\n"
- ".inst 0x646ae598 // bfmmla z24.s, z12.h, z10.h\n"
- ".inst 0x646ae5b9 // bfmmla z25.s, z13.h, z10.h\n"
- ".inst 0x646ae5da // bfmmla z26.s, z14.h, z10.h\n"
- ".inst 0x646ae5fb // bfmmla z27.s, z15.h, z10.h\n"
- "trn1 z10.d, z4.d, z5.d\n"
- "cbz %[blocks], 5f\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "ld1h z12.h, p7/z, [%[b_ptr0]]\n"
- "trn2 z1.d, z2.d, z3.d\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "trn2 z2.d, z4.d, z5.d\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- ".inst 0x6468e590 // bfmmla z16.s, z12.h, z8.h\n"
- ".inst 0x6468e5b1 // bfmmla z17.s, z13.h, z8.h\n"
- ".inst 0x6468e5d2 // bfmmla z18.s, z14.h, z8.h\n"
- ".inst 0x6468e5f3 // bfmmla z19.s, z15.h, z8.h\n"
- ".inst 0x6469e594 // bfmmla z20.s, z12.h, z9.h\n"
- ".inst 0x6469e5b5 // bfmmla z21.s, z13.h, z9.h\n"
- ".inst 0x6469e5d6 // bfmmla z22.s, z14.h, z9.h\n"
- ".inst 0x6469e5f7 // bfmmla z23.s, z15.h, z9.h\n"
- ".inst 0x646ae598 // bfmmla z24.s, z12.h, z10.h\n"
- ".inst 0x646ae5b9 // bfmmla z25.s, z13.h, z10.h\n"
- ".inst 0x646ae5da // bfmmla z26.s, z14.h, z10.h\n"
- ".inst 0x646ae5fb // bfmmla z27.s, z15.h, z10.h\n"
- "b.eq 5f\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x6460e590 // bfmmla z16.s, z12.h, z0.h\n"
- ".inst 0x6460e5b1 // bfmmla z17.s, z13.h, z0.h\n"
- ".inst 0x6460e5d2 // bfmmla z18.s, z14.h, z0.h\n"
- ".inst 0x6460e5f3 // bfmmla z19.s, z15.h, z0.h\n"
- ".inst 0x6461e594 // bfmmla z20.s, z12.h, z1.h\n"
- ".inst 0x6461e5b5 // bfmmla z21.s, z13.h, z1.h\n"
- ".inst 0x6461e5d6 // bfmmla z22.s, z14.h, z1.h\n"
- ".inst 0x6461e5f7 // bfmmla z23.s, z15.h, z1.h\n"
- ".inst 0x6462e598 // bfmmla z24.s, z12.h, z2.h\n"
- ".inst 0x6462e5b9 // bfmmla z25.s, z13.h, z2.h\n"
- ".inst 0x6462e5da // bfmmla z26.s, z14.h, z2.h\n"
- ".inst 0x6462e5fb // bfmmla z27.s, z15.h, z2.h\n"
- "b 5f\n"
- "4:\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "ld1rqh z6.h, p6/z, [a_ptr2]\n"
- "trn2 z1.d, z2.d, z3.d\n"
- "ld1rqh z7.h, p6/z, [a_ptr3]\n"
- "trn2 z2.d, z4.d, z5.d\n"
- "ld1rqh z4.h, p6/z, [%[a_ptr0]]\n"
- ".inst 0x6468e590 // bfmmla z16.s, z12.h, z8.h\n"
- "ld1rqh z5.h, p6/z, [a_ptr1]\n"
- ".inst 0x6468e5b1 // bfmmla z17.s, z13.h, z8.h\n"
- "addvl %[a_ptr0], %[a_ptr0], #1\n"
- ".inst 0x6468e5d2 // bfmmla z18.s, z14.h, z8.h\n"
- "addvl a_ptr1, a_ptr1, #1\n"
- ".inst 0x6468e5f3 // bfmmla z19.s, z15.h, z8.h\n"
- "ld1rqh z8.h, p6/z, [a_ptr4]\n"
- ".inst 0x6469e594 // bfmmla z20.s, z12.h, z9.h\n"
- "addvl a_ptr2, a_ptr2, #1\n"
- ".inst 0x6469e5b5 // bfmmla z21.s, z13.h, z9.h\n"
- "addvl a_ptr3, a_ptr3, #1\n"
- ".inst 0x6469e5d6 // bfmmla z22.s, z14.h, z9.h\n"
- "addvl a_ptr4, a_ptr4, #1\n"
- ".inst 0x6469e5f7 // bfmmla z23.s, z15.h, z9.h\n"
- "ld1rqh z9.h, p6/z, [a_ptr5]\n"
- ".inst 0x646ae598 // bfmmla z24.s, z12.h, z10.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0]]\n"
- ".inst 0x646ae5b9 // bfmmla z25.s, z13.h, z10.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x646ae5da // bfmmla z26.s, z14.h, z10.h\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x646ae5fb // bfmmla z27.s, z15.h, z10.h\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x6460e590 // bfmmla z16.s, z12.h, z0.h\n"
- "addvl %[b_ptr0], %[b_ptr0], #4\n"
- ".inst 0x6460e5b1 // bfmmla z17.s, z13.h, z0.h\n"
- "addvl a_ptr5, a_ptr5, #1\n"
- ".inst 0x6460e5d2 // bfmmla z18.s, z14.h, z0.h\n"
- ".inst 0x6460e5f3 // bfmmla z19.s, z15.h, z0.h\n"
- "trn1 z0.d, z4.d, z5.d\n"
- ".inst 0x6461e594 // bfmmla z20.s, z12.h, z1.h\n"
- ".inst 0x6461e5b5 // bfmmla z21.s, z13.h, z1.h\n"
- ".inst 0x6461e5d6 // bfmmla z22.s, z14.h, z1.h\n"
- ".inst 0x6461e5f7 // bfmmla z23.s, z15.h, z1.h\n"
- "trn1 z1.d, z6.d, z7.d\n"
- ".inst 0x6462e598 // bfmmla z24.s, z12.h, z2.h\n"
- ".inst 0x6462e5b9 // bfmmla z25.s, z13.h, z2.h\n"
- ".inst 0x6462e5da // bfmmla z26.s, z14.h, z2.h\n"
- ".inst 0x6462e5fb // bfmmla z27.s, z15.h, z2.h\n"
- "trn1 z2.d, z8.d, z9.d\n"
- "cbz %[blocks], 5f\n"
- "trn2 z10.d, z8.d, z9.d\n"
- "ld1h z12.h, p7/z, [%[b_ptr0]]\n"
- "trn2 z9.d, z6.d, z7.d\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "trn2 z8.d, z4.d, z5.d\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- ".inst 0x6460e590 // bfmmla z16.s, z12.h, z0.h\n"
- ".inst 0x6460e5b1 // bfmmla z17.s, z13.h, z0.h\n"
- ".inst 0x6460e5d2 // bfmmla z18.s, z14.h, z0.h\n"
- ".inst 0x6460e5f3 // bfmmla z19.s, z15.h, z0.h\n"
- ".inst 0x6461e594 // bfmmla z20.s, z12.h, z1.h\n"
- ".inst 0x6461e5b5 // bfmmla z21.s, z13.h, z1.h\n"
- ".inst 0x6461e5d6 // bfmmla z22.s, z14.h, z1.h\n"
- ".inst 0x6461e5f7 // bfmmla z23.s, z15.h, z1.h\n"
- ".inst 0x6462e598 // bfmmla z24.s, z12.h, z2.h\n"
- ".inst 0x6462e5b9 // bfmmla z25.s, z13.h, z2.h\n"
- ".inst 0x6462e5da // bfmmla z26.s, z14.h, z2.h\n"
- ".inst 0x6462e5fb // bfmmla z27.s, z15.h, z2.h\n"
- "b.eq 5f\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x6468e590 // bfmmla z16.s, z12.h, z8.h\n"
- ".inst 0x6468e5b1 // bfmmla z17.s, z13.h, z8.h\n"
- ".inst 0x6468e5d2 // bfmmla z18.s, z14.h, z8.h\n"
- ".inst 0x6468e5f3 // bfmmla z19.s, z15.h, z8.h\n"
- ".inst 0x6469e594 // bfmmla z20.s, z12.h, z9.h\n"
- ".inst 0x6469e5b5 // bfmmla z21.s, z13.h, z9.h\n"
- ".inst 0x6469e5d6 // bfmmla z22.s, z14.h, z9.h\n"
- ".inst 0x6469e5f7 // bfmmla z23.s, z15.h, z9.h\n"
- ".inst 0x646ae598 // bfmmla z24.s, z12.h, z10.h\n"
- ".inst 0x646ae5b9 // bfmmla z25.s, z13.h, z10.h\n"
- ".inst 0x646ae5da // bfmmla z26.s, z14.h, z10.h\n"
- ".inst 0x646ae5fb // bfmmla z27.s, z15.h, z10.h\n"
- "5:\n"
- "ld1rw z14.s, p7/z, [%[minptr]]\n"
- "ld1rw z15.s, p7/z, [%[maxptr]]\n"
- "fmax z16.s, p7/m, z16.s, z14.s\n"
- "fmax z17.s, p7/m, z17.s, z14.s\n"
- "fmax z18.s, p7/m, z18.s, z14.s\n"
- "fmax z19.s, p7/m, z19.s, z14.s\n"
- "fmin z16.s, p7/m, z16.s, z15.s\n"
- "fmin z17.s, p7/m, z17.s, z15.s\n"
- "fmin z18.s, p7/m, z18.s, z15.s\n"
- "fmin z19.s, p7/m, z19.s, z15.s\n"
- "fmax z20.s, p7/m, z20.s, z14.s\n"
- "uzp1 z0.s, z16.s, z17.s\n"
- "uzp2 z1.s, z16.s, z17.s\n"
- "uzp1 z2.s, z18.s, z19.s\n"
- "uzp2 z3.s, z18.s, z19.s\n"
- "st1w z0.s, p0, [%[c_ptr0]]\n"
- "fmin z20.s, p7/m, z20.s, z15.s\n"
- "fmax z21.s, p7/m, z21.s, z14.s\n"
- "fmax z22.s, p7/m, z22.s, z14.s\n"
- "st1w z1.s, p0, [c_ptr1]\n"
- "fmax z23.s, p7/m, z23.s, z14.s\n"
- "fmax z24.s, p7/m, z24.s, z14.s\n"
- "fmin z21.s, p7/m, z21.s, z15.s\n"
- "st1w z2.s, p1, [%[c_ptr0], #1, MUL VL]\n"
- "fmin z22.s, p7/m, z22.s, z15.s\n"
- "addvl %[c_ptr0], %[c_ptr0], #2\n"
- "fmin z23.s, p7/m, z23.s, z15.s\n"
- "st1w z3.s, p1, [c_ptr1, #1, MUL VL]\n"
- "uzp1 z4.s, z20.s, z21.s\n"
- "uzp2 z5.s, z20.s, z21.s\n"
- "fmin z24.s, p7/m, z24.s, z15.s\n"
- "uzp1 z6.s, z22.s, z23.s\n"
- "st1w z4.s, p0, [c_ptr2]\n"
- "uzp2 z7.s, z22.s, z23.s\n"
- "fmax z25.s, p7/m, z25.s, z14.s\n"
- "fmax z26.s, p7/m, z26.s, z14.s\n"
- "st1w z5.s, p0, [c_ptr3]\n"
- "fmax z27.s, p7/m, z27.s, z14.s\n"
- "fmin z25.s, p7/m, z25.s, z15.s\n"
- "st1w z6.s, p1, [c_ptr2, #1, MUL VL]\n"
- "fmin z26.s, p7/m, z26.s, z15.s\n"
- "fmin z27.s, p7/m, z27.s, z15.s\n"
- "uzp1 z8.s, z24.s, z25.s\n"
- "st1w z7.s, p1, [c_ptr3, #1, MUL VL]\n"
- "uzp2 z9.s, z24.s, z25.s\n"
- "uzp1 z10.s, z26.s, z27.s\n"
- "uzp2 z11.s, z26.s, z27.s\n"
- "st1w z8.s, p0, [c_ptr4]\n"
- "st1w z9.s, p0, [c_ptr5]\n"
- "st1w z10.s, p1, [c_ptr4, #1, MUL VL]\n"
- "st1w z11.s, p1, [c_ptr5, #1, MUL VL]\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq a_ptr3\n"
- ".unreq a_ptr4\n"
- ".unreq a_ptr5\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- ".unreq c_ptr3\n"
- ".unreq c_ptr4\n"
- ".unreq c_ptr5\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks)
- : [width] "r" (width), [accumulate] "r" (static_cast<uint64_t>(accumulate)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers)
- : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "cc", "memory"
- );
- break;
- case 7:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "a_ptr3 .req X2\n"
- "a_ptr4 .req X3\n"
- "a_ptr5 .req X4\n"
- "a_ptr6 .req X5\n"
- "c_ptr1 .req X6\n"
- "c_ptr2 .req X7\n"
- "c_ptr3 .req X8\n"
- "c_ptr4 .req X9\n"
- "c_ptr5 .req X10\n"
- "c_ptr6 .req X11\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "add a_ptr3, a_ptr2, %[lda]\n"
- "add c_ptr3, c_ptr2, %[ldc]\n"
- "add a_ptr4, a_ptr3, %[lda]\n"
- "add c_ptr4, c_ptr3, %[ldc]\n"
- "add a_ptr5, a_ptr4, %[lda]\n"
- "add c_ptr5, c_ptr4, %[ldc]\n"
- "add a_ptr6, a_ptr5, %[lda]\n"
- "add c_ptr6, c_ptr5, %[ldc]\n"
- "whilelt p6.h, %[temp], %[leftovers]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "ptrue p7.h\n"
- "whilelt p1.s, %[temp], %[width]\n"
- "cbnz %[accumulate], 1f\n"
- "mov z7.h, #0\n"
- "ld1w z15.s, p0/z, [%[biasptr]]\n"
- "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ld1rqh z1.h, p7/z, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "zip1 z16.s, z15.s, z15.s\n"
- "ld1rqh z2.h, p7/z, [a_ptr2]\n"
- "zip2 z17.s, z15.s, z15.s\n"
- "ld1w z15.s, p1/z, [%[biasptr], #1, MUL VL]\n"
- "trn1 z8.d, z0.d, z1.d\n"
- "ld1rqh z3.h, p7/z, [a_ptr3]\n"
- "ld1rqh z4.h, p7/z, [a_ptr4]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "zip1 z18.s, z15.s, z15.s\n"
- "ld1rqh z5.h, p7/z, [a_ptr5]\n"
- "zip2 z19.s, z15.s, z15.s\n"
- "ld1rqh z6.h, p7/z, [a_ptr6]\n"
- "trn1 z9.d, z2.d, z3.d\n"
- "ld1h z12.h, p7/z, [%[b_ptr0]]\n"
- "mov z20.d, z16.d\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "trn1 z10.d, z4.d, z5.d\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "trn1 z11.d, z6.d, z7.d\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "mov z21.d, z17.d\n"
- "add a_ptr3, a_ptr3, #0x10\n"
- "mov z22.d, z18.d\n"
- "add a_ptr4, a_ptr4, #0x10\n"
- "mov z23.d, z19.d\n"
- "add a_ptr5, a_ptr5, #0x10\n"
- "mov z24.d, z16.d\n"
- "add a_ptr6, a_ptr6, #0x10\n"
- "mov z25.d, z17.d\n"
- "addvl %[b_ptr0], %[b_ptr0], #4\n"
- "mov z26.d, z18.d\n"
- "mov z27.d, z19.d\n"
- "mov z28.d, z16.d\n"
- "mov z29.d, z17.d\n"
- "mov z30.d, z18.d\n"
- "mov z31.d, z19.d\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "mov z7.h, #0\n"
- "ld1w z13.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z14.s, p0/z, [c_ptr1]\n"
- "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ld1rqh z1.h, p7/z, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "zip1 z16.s, z13.s, z14.s\n"
- "ld1rqh z2.h, p7/z, [a_ptr2]\n"
- "zip2 z17.s, z13.s, z14.s\n"
- "ld1w z13.s, p1/z, [%[c_ptr0], #1, MUL VL]\n"
- "trn1 z8.d, z0.d, z1.d\n"
- "ld1w z14.s, p1/z, [c_ptr1, #1, MUL VL]\n"
- "ld1rqh z3.h, p7/z, [a_ptr3]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "ld1rqh z4.h, p7/z, [a_ptr4]\n"
- "add a_ptr3, a_ptr3, #0x10\n"
- "zip1 z18.s, z13.s, z14.s\n"
- "ld1rqh z5.h, p7/z, [a_ptr5]\n"
- "zip2 z19.s, z13.s, z14.s\n"
- "ld1w z13.s, p0/z, [c_ptr2]\n"
- "trn1 z9.d, z2.d, z3.d\n"
- "ld1w z14.s, p0/z, [c_ptr3]\n"
- "ld1rqh z6.h, p7/z, [a_ptr6]\n"
- "add a_ptr4, a_ptr4, #0x10\n"
- "trn1 z10.d, z4.d, z5.d\n"
- "ld1h z12.h, p7/z, [%[b_ptr0]]\n"
- "zip1 z20.s, z13.s, z14.s\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "zip2 z21.s, z13.s, z14.s\n"
- "ld1w z13.s, p1/z, [c_ptr2, #1, MUL VL]\n"
- "trn1 z11.d, z6.d, z7.d\n"
- "ld1w z14.s, p1/z, [c_ptr3, #1, MUL VL]\n"
- "add a_ptr5, a_ptr5, #0x10\n"
- "add a_ptr6, a_ptr6, #0x10\n"
- "zip1 z22.s, z13.s, z14.s\n"
- "zip2 z23.s, z13.s, z14.s\n"
- "ld1w z13.s, p0/z, [c_ptr4]\n"
- "ld1w z14.s, p0/z, [c_ptr5]\n"
- "zip1 z24.s, z13.s, z14.s\n"
- "zip2 z25.s, z13.s, z14.s\n"
- "ld1w z13.s, p1/z, [c_ptr4, #1, MUL VL]\n"
- "ld1w z14.s, p1/z, [c_ptr5, #1, MUL VL]\n"
- "zip1 z26.s, z13.s, z14.s\n"
- "zip2 z27.s, z13.s, z14.s\n"
- "ld1w z13.s, p0/z, [c_ptr6]\n"
- "mov z14.s, #0\n"
- "zip1 z28.s, z13.s, z14.s\n"
- "zip2 z29.s, z13.s, z14.s\n"
- "ld1w z13.s, p1/z, [c_ptr6, #1, MUL VL]\n"
- "mov z14.s, #0\n"
- "zip1 z30.s, z13.s, z14.s\n"
- "zip2 z31.s, z13.s, z14.s\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #4\n"
- "cbz %[loops], 2f\n"
- "3:\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "subs %[loops], %[loops], #0x1\n"
- "trn2 z1.d, z2.d, z3.d\n"
- "trn2 z2.d, z4.d, z5.d\n"
- "ld1rqh z4.h, p7/z, [%[a_ptr0]]\n"
- "trn2 z3.d, z6.d, z7.d\n"
- "ld1rqh z5.h, p7/z, [a_ptr1]\n"
- ".inst 0x6468e590 // bfmmla z16.s, z12.h, z8.h\n"
- "ld1rqh z6.h, p7/z, [a_ptr2]\n"
- ".inst 0x6468e5b1 // bfmmla z17.s, z13.h, z8.h\n"
- "ld1rqh z7.h, p7/z, [a_ptr3]\n"
- ".inst 0x6468e5d2 // bfmmla z18.s, z14.h, z8.h\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
- ".inst 0x6468e5f3 // bfmmla z19.s, z15.h, z8.h\n"
- "ld1rqh z8.h, p7/z, [a_ptr4]\n"
- ".inst 0x6469e594 // bfmmla z20.s, z12.h, z9.h\n"
- "add a_ptr1, a_ptr1, #0x20\n"
- ".inst 0x6469e5b5 // bfmmla z21.s, z13.h, z9.h\n"
- "add a_ptr2, a_ptr2, #0x20\n"
- ".inst 0x6469e5d6 // bfmmla z22.s, z14.h, z9.h\n"
- "add a_ptr3, a_ptr3, #0x20\n"
- ".inst 0x6469e5f7 // bfmmla z23.s, z15.h, z9.h\n"
- "ld1rqh z9.h, p7/z, [a_ptr5]\n"
- ".inst 0x646ae598 // bfmmla z24.s, z12.h, z10.h\n"
- "add a_ptr4, a_ptr4, #0x20\n"
- ".inst 0x646ae5b9 // bfmmla z25.s, z13.h, z10.h\n"
- "add a_ptr5, a_ptr5, #0x20\n"
- ".inst 0x646ae5da // bfmmla z26.s, z14.h, z10.h\n"
- ".inst 0x646ae5fb // bfmmla z27.s, z15.h, z10.h\n"
- "ld1rqh z10.h, p7/z, [a_ptr6]\n"
- ".inst 0x646be59c // bfmmla z28.s, z12.h, z11.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0]]\n"
- ".inst 0x646be5bd // bfmmla z29.s, z13.h, z11.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x646be5de // bfmmla z30.s, z14.h, z11.h\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x646be5ff // bfmmla z31.s, z15.h, z11.h\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "mov z11.h, #0\n"
- "add a_ptr6, a_ptr6, #0x20\n"
- ".inst 0x6460e590 // bfmmla z16.s, z12.h, z0.h\n"
- ".inst 0x6460e5b1 // bfmmla z17.s, z13.h, z0.h\n"
- ".inst 0x6460e5d2 // bfmmla z18.s, z14.h, z0.h\n"
- ".inst 0x6460e5f3 // bfmmla z19.s, z15.h, z0.h\n"
- "trn1 z0.d, z4.d, z5.d\n"
- ".inst 0x6461e594 // bfmmla z20.s, z12.h, z1.h\n"
- ".inst 0x6461e5b5 // bfmmla z21.s, z13.h, z1.h\n"
- ".inst 0x6461e5d6 // bfmmla z22.s, z14.h, z1.h\n"
- ".inst 0x6461e5f7 // bfmmla z23.s, z15.h, z1.h\n"
- "trn1 z1.d, z6.d, z7.d\n"
- ".inst 0x6462e598 // bfmmla z24.s, z12.h, z2.h\n"
- ".inst 0x6462e5b9 // bfmmla z25.s, z13.h, z2.h\n"
- ".inst 0x6462e5da // bfmmla z26.s, z14.h, z2.h\n"
- ".inst 0x6462e5fb // bfmmla z27.s, z15.h, z2.h\n"
- "trn1 z2.d, z8.d, z9.d\n"
- ".inst 0x6463e59c // bfmmla z28.s, z12.h, z3.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x6463e5bd // bfmmla z29.s, z13.h, z3.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x6463e5de // bfmmla z30.s, z14.h, z3.h\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x6463e5ff // bfmmla z31.s, z15.h, z3.h\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "trn1 z3.d, z10.d, z11.d\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "trn2 z11.d, z10.d, z11.d\n"
- "trn2 z10.d, z8.d, z9.d\n"
- "trn2 z9.d, z6.d, z7.d\n"
- "ld1rqh z6.h, p7/z, [a_ptr6, #-0x10]\n"
- "trn2 z8.d, z4.d, z5.d\n"
- "ld1rqh z4.h, p7/z, [a_ptr4, #-0x10]\n"
- ".inst 0x6460e590 // bfmmla z16.s, z12.h, z0.h\n"
- "ld1rqh z5.h, p7/z, [a_ptr5, #-0x10]\n"
- ".inst 0x6460e5b1 // bfmmla z17.s, z13.h, z0.h\n"
- ".inst 0x6460e5d2 // bfmmla z18.s, z14.h, z0.h\n"
- ".inst 0x6460e5f3 // bfmmla z19.s, z15.h, z0.h\n"
- "ld1rqh z0.h, p7/z, [%[a_ptr0], #-0x10]\n"
- ".inst 0x6461e594 // bfmmla z20.s, z12.h, z1.h\n"
- ".inst 0x6461e5b5 // bfmmla z21.s, z13.h, z1.h\n"
- ".inst 0x6461e5d6 // bfmmla z22.s, z14.h, z1.h\n"
- ".inst 0x6461e5f7 // bfmmla z23.s, z15.h, z1.h\n"
- "ld1rqh z1.h, p7/z, [a_ptr1, #-0x10]\n"
- ".inst 0x6462e598 // bfmmla z24.s, z12.h, z2.h\n"
- ".inst 0x6462e5b9 // bfmmla z25.s, z13.h, z2.h\n"
- ".inst 0x6462e5da // bfmmla z26.s, z14.h, z2.h\n"
- ".inst 0x6462e5fb // bfmmla z27.s, z15.h, z2.h\n"
- "ld1rqh z2.h, p7/z, [a_ptr2, #-0x10]\n"
- ".inst 0x6463e59c // bfmmla z28.s, z12.h, z3.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- ".inst 0x6463e5bd // bfmmla z29.s, z13.h, z3.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- ".inst 0x6463e5de // bfmmla z30.s, z14.h, z3.h\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- ".inst 0x6463e5ff // bfmmla z31.s, z15.h, z3.h\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "mov z7.h, #0\n"
- "ld1rqh z3.h, p7/z, [a_ptr3, #-0x10]\n"
- ".inst 0x6468e590 // bfmmla z16.s, z12.h, z8.h\n"
- ".inst 0x6468e5b1 // bfmmla z17.s, z13.h, z8.h\n"
- ".inst 0x6468e5d2 // bfmmla z18.s, z14.h, z8.h\n"
- ".inst 0x6468e5f3 // bfmmla z19.s, z15.h, z8.h\n"
- "trn1 z8.d, z0.d, z1.d\n"
- ".inst 0x6469e594 // bfmmla z20.s, z12.h, z9.h\n"
- ".inst 0x6469e5b5 // bfmmla z21.s, z13.h, z9.h\n"
- ".inst 0x6469e5d6 // bfmmla z22.s, z14.h, z9.h\n"
- ".inst 0x6469e5f7 // bfmmla z23.s, z15.h, z9.h\n"
- "trn1 z9.d, z2.d, z3.d\n"
- ".inst 0x646ae598 // bfmmla z24.s, z12.h, z10.h\n"
- ".inst 0x646ae5b9 // bfmmla z25.s, z13.h, z10.h\n"
- ".inst 0x646ae5da // bfmmla z26.s, z14.h, z10.h\n"
- ".inst 0x646ae5fb // bfmmla z27.s, z15.h, z10.h\n"
- "trn1 z10.d, z4.d, z5.d\n"
- ".inst 0x646be59c // bfmmla z28.s, z12.h, z11.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- ".inst 0x646be5bd // bfmmla z29.s, z13.h, z11.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- ".inst 0x646be5de // bfmmla z30.s, z14.h, z11.h\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- ".inst 0x646be5ff // bfmmla z31.s, z15.h, z11.h\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "trn1 z11.d, z6.d, z7.d\n"
- "b.ne 3b\n"
- "2:\n"
- "cbz %[regs], 4f\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "trn2 z1.d, z2.d, z3.d\n"
- "trn2 z2.d, z4.d, z5.d\n"
- "ld1rqh z4.h, p7/z, [%[a_ptr0]]\n"
- "trn2 z3.d, z6.d, z7.d\n"
- "ld1rqh z5.h, p7/z, [a_ptr1]\n"
- ".inst 0x6468e590 // bfmmla z16.s, z12.h, z8.h\n"
- "ld1rqh z6.h, p7/z, [a_ptr2]\n"
- ".inst 0x6468e5b1 // bfmmla z17.s, z13.h, z8.h\n"
- "ld1rqh z7.h, p7/z, [a_ptr3]\n"
- ".inst 0x6468e5d2 // bfmmla z18.s, z14.h, z8.h\n"
- ".inst 0x6468e5f3 // bfmmla z19.s, z15.h, z8.h\n"
- "ld1rqh z8.h, p7/z, [a_ptr4]\n"
- ".inst 0x6469e594 // bfmmla z20.s, z12.h, z9.h\n"
- ".inst 0x6469e5b5 // bfmmla z21.s, z13.h, z9.h\n"
- ".inst 0x6469e5d6 // bfmmla z22.s, z14.h, z9.h\n"
- ".inst 0x6469e5f7 // bfmmla z23.s, z15.h, z9.h\n"
- "ld1rqh z9.h, p7/z, [a_ptr5]\n"
- ".inst 0x646ae598 // bfmmla z24.s, z12.h, z10.h\n"
- ".inst 0x646ae5b9 // bfmmla z25.s, z13.h, z10.h\n"
- ".inst 0x646ae5da // bfmmla z26.s, z14.h, z10.h\n"
- ".inst 0x646ae5fb // bfmmla z27.s, z15.h, z10.h\n"
- "ld1rqh z10.h, p7/z, [a_ptr6]\n"
- ".inst 0x646be59c // bfmmla z28.s, z12.h, z11.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0]]\n"
- ".inst 0x646be5bd // bfmmla z29.s, z13.h, z11.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x646be5de // bfmmla z30.s, z14.h, z11.h\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x646be5ff // bfmmla z31.s, z15.h, z11.h\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "mov z11.h, #0\n"
- ".inst 0x6460e590 // bfmmla z16.s, z12.h, z0.h\n"
- ".inst 0x6460e5b1 // bfmmla z17.s, z13.h, z0.h\n"
- ".inst 0x6460e5d2 // bfmmla z18.s, z14.h, z0.h\n"
- ".inst 0x6460e5f3 // bfmmla z19.s, z15.h, z0.h\n"
- "trn1 z0.d, z4.d, z5.d\n"
- ".inst 0x6461e594 // bfmmla z20.s, z12.h, z1.h\n"
- ".inst 0x6461e5b5 // bfmmla z21.s, z13.h, z1.h\n"
- ".inst 0x6461e5d6 // bfmmla z22.s, z14.h, z1.h\n"
- ".inst 0x6461e5f7 // bfmmla z23.s, z15.h, z1.h\n"
- "trn1 z1.d, z6.d, z7.d\n"
- ".inst 0x6462e598 // bfmmla z24.s, z12.h, z2.h\n"
- ".inst 0x6462e5b9 // bfmmla z25.s, z13.h, z2.h\n"
- ".inst 0x6462e5da // bfmmla z26.s, z14.h, z2.h\n"
- ".inst 0x6462e5fb // bfmmla z27.s, z15.h, z2.h\n"
- "trn1 z2.d, z8.d, z9.d\n"
- ".inst 0x6463e59c // bfmmla z28.s, z12.h, z3.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x6463e5bd // bfmmla z29.s, z13.h, z3.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x6463e5de // bfmmla z30.s, z14.h, z3.h\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x6463e5ff // bfmmla z31.s, z15.h, z3.h\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "trn1 z3.d, z10.d, z11.d\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "trn2 z11.d, z10.d, z11.d\n"
- "trn2 z10.d, z8.d, z9.d\n"
- "trn2 z9.d, z6.d, z7.d\n"
- "ld1rqh z6.h, p6/z, [a_ptr6, #0x10]\n"
- "trn2 z8.d, z4.d, z5.d\n"
- "ld1rqh z4.h, p6/z, [a_ptr4, #0x10]\n"
- ".inst 0x6460e590 // bfmmla z16.s, z12.h, z0.h\n"
- "ld1rqh z5.h, p6/z, [a_ptr5, #0x10]\n"
- ".inst 0x6460e5b1 // bfmmla z17.s, z13.h, z0.h\n"
- "addvl a_ptr4, a_ptr4, #2\n"
- ".inst 0x6460e5d2 // bfmmla z18.s, z14.h, z0.h\n"
- "addvl a_ptr5, a_ptr5, #2\n"
- ".inst 0x6460e5f3 // bfmmla z19.s, z15.h, z0.h\n"
- "ld1rqh z0.h, p6/z, [%[a_ptr0], #0x10]\n"
- ".inst 0x6461e594 // bfmmla z20.s, z12.h, z1.h\n"
- "addvl %[a_ptr0], %[a_ptr0], #2\n"
- ".inst 0x6461e5b5 // bfmmla z21.s, z13.h, z1.h\n"
- "addvl a_ptr6, a_ptr6, #2\n"
- ".inst 0x6461e5d6 // bfmmla z22.s, z14.h, z1.h\n"
- ".inst 0x6461e5f7 // bfmmla z23.s, z15.h, z1.h\n"
- "ld1rqh z1.h, p6/z, [a_ptr1, #0x10]\n"
- ".inst 0x6462e598 // bfmmla z24.s, z12.h, z2.h\n"
- "addvl a_ptr1, a_ptr1, #2\n"
- ".inst 0x6462e5b9 // bfmmla z25.s, z13.h, z2.h\n"
- ".inst 0x6462e5da // bfmmla z26.s, z14.h, z2.h\n"
- ".inst 0x6462e5fb // bfmmla z27.s, z15.h, z2.h\n"
- "ld1rqh z2.h, p6/z, [a_ptr2, #0x10]\n"
- ".inst 0x6463e59c // bfmmla z28.s, z12.h, z3.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- ".inst 0x6463e5bd // bfmmla z29.s, z13.h, z3.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- ".inst 0x6463e5de // bfmmla z30.s, z14.h, z3.h\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- ".inst 0x6463e5ff // bfmmla z31.s, z15.h, z3.h\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "mov z7.h, #0\n"
- "ld1rqh z3.h, p6/z, [a_ptr3, #0x10]\n"
- ".inst 0x6468e590 // bfmmla z16.s, z12.h, z8.h\n"
- "addvl %[b_ptr0], %[b_ptr0], #-4\n"
- ".inst 0x6468e5b1 // bfmmla z17.s, z13.h, z8.h\n"
- "addvl a_ptr2, a_ptr2, #2\n"
- ".inst 0x6468e5d2 // bfmmla z18.s, z14.h, z8.h\n"
- "addvl a_ptr3, a_ptr3, #2\n"
- ".inst 0x6468e5f3 // bfmmla z19.s, z15.h, z8.h\n"
- "trn1 z8.d, z0.d, z1.d\n"
- ".inst 0x6469e594 // bfmmla z20.s, z12.h, z9.h\n"
- ".inst 0x6469e5b5 // bfmmla z21.s, z13.h, z9.h\n"
- ".inst 0x6469e5d6 // bfmmla z22.s, z14.h, z9.h\n"
- ".inst 0x6469e5f7 // bfmmla z23.s, z15.h, z9.h\n"
- "trn1 z9.d, z2.d, z3.d\n"
- ".inst 0x646ae598 // bfmmla z24.s, z12.h, z10.h\n"
- ".inst 0x646ae5b9 // bfmmla z25.s, z13.h, z10.h\n"
- ".inst 0x646ae5da // bfmmla z26.s, z14.h, z10.h\n"
- ".inst 0x646ae5fb // bfmmla z27.s, z15.h, z10.h\n"
- "trn1 z10.d, z4.d, z5.d\n"
- ".inst 0x646be59c // bfmmla z28.s, z12.h, z11.h\n"
- ".inst 0x646be5bd // bfmmla z29.s, z13.h, z11.h\n"
- ".inst 0x646be5de // bfmmla z30.s, z14.h, z11.h\n"
- ".inst 0x646be5ff // bfmmla z31.s, z15.h, z11.h\n"
- "trn1 z11.d, z6.d, z7.d\n"
- "cbz %[blocks], 5f\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "ld1h z12.h, p7/z, [%[b_ptr0]]\n"
- "trn2 z1.d, z2.d, z3.d\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "trn2 z2.d, z4.d, z5.d\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "trn2 z3.d, z6.d, z7.d\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x6468e590 // bfmmla z16.s, z12.h, z8.h\n"
- "subs %[blocks], %[blocks], #0x1\n"
- ".inst 0x6468e5b1 // bfmmla z17.s, z13.h, z8.h\n"
- ".inst 0x6468e5d2 // bfmmla z18.s, z14.h, z8.h\n"
- ".inst 0x6468e5f3 // bfmmla z19.s, z15.h, z8.h\n"
- ".inst 0x6469e594 // bfmmla z20.s, z12.h, z9.h\n"
- ".inst 0x6469e5b5 // bfmmla z21.s, z13.h, z9.h\n"
- ".inst 0x6469e5d6 // bfmmla z22.s, z14.h, z9.h\n"
- ".inst 0x6469e5f7 // bfmmla z23.s, z15.h, z9.h\n"
- ".inst 0x646ae598 // bfmmla z24.s, z12.h, z10.h\n"
- ".inst 0x646ae5b9 // bfmmla z25.s, z13.h, z10.h\n"
- ".inst 0x646ae5da // bfmmla z26.s, z14.h, z10.h\n"
- ".inst 0x646ae5fb // bfmmla z27.s, z15.h, z10.h\n"
- ".inst 0x646be59c // bfmmla z28.s, z12.h, z11.h\n"
- ".inst 0x646be5bd // bfmmla z29.s, z13.h, z11.h\n"
- ".inst 0x646be5de // bfmmla z30.s, z14.h, z11.h\n"
- ".inst 0x646be5ff // bfmmla z31.s, z15.h, z11.h\n"
- "b.eq 5f\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x6460e590 // bfmmla z16.s, z12.h, z0.h\n"
- ".inst 0x6460e5b1 // bfmmla z17.s, z13.h, z0.h\n"
- ".inst 0x6460e5d2 // bfmmla z18.s, z14.h, z0.h\n"
- ".inst 0x6460e5f3 // bfmmla z19.s, z15.h, z0.h\n"
- ".inst 0x6461e594 // bfmmla z20.s, z12.h, z1.h\n"
- ".inst 0x6461e5b5 // bfmmla z21.s, z13.h, z1.h\n"
- ".inst 0x6461e5d6 // bfmmla z22.s, z14.h, z1.h\n"
- ".inst 0x6461e5f7 // bfmmla z23.s, z15.h, z1.h\n"
- ".inst 0x6462e598 // bfmmla z24.s, z12.h, z2.h\n"
- ".inst 0x6462e5b9 // bfmmla z25.s, z13.h, z2.h\n"
- ".inst 0x6462e5da // bfmmla z26.s, z14.h, z2.h\n"
- ".inst 0x6462e5fb // bfmmla z27.s, z15.h, z2.h\n"
- ".inst 0x6463e59c // bfmmla z28.s, z12.h, z3.h\n"
- ".inst 0x6463e5bd // bfmmla z29.s, z13.h, z3.h\n"
- ".inst 0x6463e5de // bfmmla z30.s, z14.h, z3.h\n"
- ".inst 0x6463e5ff // bfmmla z31.s, z15.h, z3.h\n"
- "b 5f\n"
- "4:\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "trn2 z1.d, z2.d, z3.d\n"
- "trn2 z2.d, z4.d, z5.d\n"
- "ld1rqh z4.h, p6/z, [%[a_ptr0]]\n"
- "trn2 z3.d, z6.d, z7.d\n"
- "ld1rqh z5.h, p6/z, [a_ptr1]\n"
- ".inst 0x6468e590 // bfmmla z16.s, z12.h, z8.h\n"
- "ld1rqh z6.h, p6/z, [a_ptr2]\n"
- ".inst 0x6468e5b1 // bfmmla z17.s, z13.h, z8.h\n"
- "ld1rqh z7.h, p6/z, [a_ptr3]\n"
- ".inst 0x6468e5d2 // bfmmla z18.s, z14.h, z8.h\n"
- "addvl %[a_ptr0], %[a_ptr0], #1\n"
- ".inst 0x6468e5f3 // bfmmla z19.s, z15.h, z8.h\n"
- "ld1rqh z8.h, p6/z, [a_ptr4]\n"
- ".inst 0x6469e594 // bfmmla z20.s, z12.h, z9.h\n"
- "addvl a_ptr1, a_ptr1, #1\n"
- ".inst 0x6469e5b5 // bfmmla z21.s, z13.h, z9.h\n"
- "addvl a_ptr2, a_ptr2, #1\n"
- ".inst 0x6469e5d6 // bfmmla z22.s, z14.h, z9.h\n"
- "addvl a_ptr3, a_ptr3, #1\n"
- ".inst 0x6469e5f7 // bfmmla z23.s, z15.h, z9.h\n"
- "ld1rqh z9.h, p6/z, [a_ptr5]\n"
- ".inst 0x646ae598 // bfmmla z24.s, z12.h, z10.h\n"
- "addvl a_ptr4, a_ptr4, #1\n"
- ".inst 0x646ae5b9 // bfmmla z25.s, z13.h, z10.h\n"
- "addvl a_ptr5, a_ptr5, #1\n"
- ".inst 0x646ae5da // bfmmla z26.s, z14.h, z10.h\n"
- ".inst 0x646ae5fb // bfmmla z27.s, z15.h, z10.h\n"
- "ld1rqh z10.h, p6/z, [a_ptr6]\n"
- ".inst 0x646be59c // bfmmla z28.s, z12.h, z11.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0]]\n"
- ".inst 0x646be5bd // bfmmla z29.s, z13.h, z11.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x646be5de // bfmmla z30.s, z14.h, z11.h\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x646be5ff // bfmmla z31.s, z15.h, z11.h\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "mov z11.h, #0\n"
- "addvl %[b_ptr0], %[b_ptr0], #4\n"
- ".inst 0x6460e590 // bfmmla z16.s, z12.h, z0.h\n"
- "addvl a_ptr6, a_ptr6, #1\n"
- ".inst 0x6460e5b1 // bfmmla z17.s, z13.h, z0.h\n"
- ".inst 0x6460e5d2 // bfmmla z18.s, z14.h, z0.h\n"
- ".inst 0x6460e5f3 // bfmmla z19.s, z15.h, z0.h\n"
- "trn1 z0.d, z4.d, z5.d\n"
- ".inst 0x6461e594 // bfmmla z20.s, z12.h, z1.h\n"
- ".inst 0x6461e5b5 // bfmmla z21.s, z13.h, z1.h\n"
- ".inst 0x6461e5d6 // bfmmla z22.s, z14.h, z1.h\n"
- ".inst 0x6461e5f7 // bfmmla z23.s, z15.h, z1.h\n"
- "trn1 z1.d, z6.d, z7.d\n"
- ".inst 0x6462e598 // bfmmla z24.s, z12.h, z2.h\n"
- ".inst 0x6462e5b9 // bfmmla z25.s, z13.h, z2.h\n"
- ".inst 0x6462e5da // bfmmla z26.s, z14.h, z2.h\n"
- ".inst 0x6462e5fb // bfmmla z27.s, z15.h, z2.h\n"
- "trn1 z2.d, z8.d, z9.d\n"
- ".inst 0x6463e59c // bfmmla z28.s, z12.h, z3.h\n"
- ".inst 0x6463e5bd // bfmmla z29.s, z13.h, z3.h\n"
- ".inst 0x6463e5de // bfmmla z30.s, z14.h, z3.h\n"
- ".inst 0x6463e5ff // bfmmla z31.s, z15.h, z3.h\n"
- "trn1 z3.d, z10.d, z11.d\n"
- "cbz %[blocks], 5f\n"
- "trn2 z11.d, z10.d, z11.d\n"
- "ld1h z12.h, p7/z, [%[b_ptr0]]\n"
- "trn2 z10.d, z8.d, z9.d\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "trn2 z9.d, z6.d, z7.d\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "trn2 z8.d, z4.d, z5.d\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x6460e590 // bfmmla z16.s, z12.h, z0.h\n"
- "subs %[blocks], %[blocks], #0x1\n"
- ".inst 0x6460e5b1 // bfmmla z17.s, z13.h, z0.h\n"
- ".inst 0x6460e5d2 // bfmmla z18.s, z14.h, z0.h\n"
- ".inst 0x6460e5f3 // bfmmla z19.s, z15.h, z0.h\n"
- ".inst 0x6461e594 // bfmmla z20.s, z12.h, z1.h\n"
- ".inst 0x6461e5b5 // bfmmla z21.s, z13.h, z1.h\n"
- ".inst 0x6461e5d6 // bfmmla z22.s, z14.h, z1.h\n"
- ".inst 0x6461e5f7 // bfmmla z23.s, z15.h, z1.h\n"
- ".inst 0x6462e598 // bfmmla z24.s, z12.h, z2.h\n"
- ".inst 0x6462e5b9 // bfmmla z25.s, z13.h, z2.h\n"
- ".inst 0x6462e5da // bfmmla z26.s, z14.h, z2.h\n"
- ".inst 0x6462e5fb // bfmmla z27.s, z15.h, z2.h\n"
- ".inst 0x6463e59c // bfmmla z28.s, z12.h, z3.h\n"
- ".inst 0x6463e5bd // bfmmla z29.s, z13.h, z3.h\n"
- ".inst 0x6463e5de // bfmmla z30.s, z14.h, z3.h\n"
- ".inst 0x6463e5ff // bfmmla z31.s, z15.h, z3.h\n"
- "b.eq 5f\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x6468e590 // bfmmla z16.s, z12.h, z8.h\n"
- ".inst 0x6468e5b1 // bfmmla z17.s, z13.h, z8.h\n"
- ".inst 0x6468e5d2 // bfmmla z18.s, z14.h, z8.h\n"
- ".inst 0x6468e5f3 // bfmmla z19.s, z15.h, z8.h\n"
- ".inst 0x6469e594 // bfmmla z20.s, z12.h, z9.h\n"
- ".inst 0x6469e5b5 // bfmmla z21.s, z13.h, z9.h\n"
- ".inst 0x6469e5d6 // bfmmla z22.s, z14.h, z9.h\n"
- ".inst 0x6469e5f7 // bfmmla z23.s, z15.h, z9.h\n"
- ".inst 0x646ae598 // bfmmla z24.s, z12.h, z10.h\n"
- ".inst 0x646ae5b9 // bfmmla z25.s, z13.h, z10.h\n"
- ".inst 0x646ae5da // bfmmla z26.s, z14.h, z10.h\n"
- ".inst 0x646ae5fb // bfmmla z27.s, z15.h, z10.h\n"
- ".inst 0x646be59c // bfmmla z28.s, z12.h, z11.h\n"
- ".inst 0x646be5bd // bfmmla z29.s, z13.h, z11.h\n"
- ".inst 0x646be5de // bfmmla z30.s, z14.h, z11.h\n"
- ".inst 0x646be5ff // bfmmla z31.s, z15.h, z11.h\n"
- "5:\n"
- "ld1rw z14.s, p7/z, [%[minptr]]\n"
- "ld1rw z15.s, p7/z, [%[maxptr]]\n"
- "fmax z16.s, p7/m, z16.s, z14.s\n"
- "fmax z17.s, p7/m, z17.s, z14.s\n"
- "fmax z18.s, p7/m, z18.s, z14.s\n"
- "fmax z19.s, p7/m, z19.s, z14.s\n"
- "fmin z16.s, p7/m, z16.s, z15.s\n"
- "fmin z17.s, p7/m, z17.s, z15.s\n"
- "fmin z18.s, p7/m, z18.s, z15.s\n"
- "fmin z19.s, p7/m, z19.s, z15.s\n"
- "fmax z20.s, p7/m, z20.s, z14.s\n"
- "uzp1 z0.s, z16.s, z17.s\n"
- "uzp2 z1.s, z16.s, z17.s\n"
- "uzp1 z2.s, z18.s, z19.s\n"
- "uzp2 z3.s, z18.s, z19.s\n"
- "st1w z0.s, p0, [%[c_ptr0]]\n"
- "fmin z20.s, p7/m, z20.s, z15.s\n"
- "fmax z21.s, p7/m, z21.s, z14.s\n"
- "fmax z22.s, p7/m, z22.s, z14.s\n"
- "st1w z1.s, p0, [c_ptr1]\n"
- "fmax z23.s, p7/m, z23.s, z14.s\n"
- "fmax z24.s, p7/m, z24.s, z14.s\n"
- "fmin z21.s, p7/m, z21.s, z15.s\n"
- "st1w z2.s, p1, [%[c_ptr0], #1, MUL VL]\n"
- "fmin z22.s, p7/m, z22.s, z15.s\n"
- "addvl %[c_ptr0], %[c_ptr0], #2\n"
- "fmin z23.s, p7/m, z23.s, z15.s\n"
- "st1w z3.s, p1, [c_ptr1, #1, MUL VL]\n"
- "uzp1 z4.s, z20.s, z21.s\n"
- "uzp2 z5.s, z20.s, z21.s\n"
- "fmin z24.s, p7/m, z24.s, z15.s\n"
- "uzp1 z6.s, z22.s, z23.s\n"
- "st1w z4.s, p0, [c_ptr2]\n"
- "uzp2 z7.s, z22.s, z23.s\n"
- "fmax z25.s, p7/m, z25.s, z14.s\n"
- "fmax z26.s, p7/m, z26.s, z14.s\n"
- "st1w z5.s, p0, [c_ptr3]\n"
- "fmax z27.s, p7/m, z27.s, z14.s\n"
- "fmax z28.s, p7/m, z28.s, z14.s\n"
- "fmin z25.s, p7/m, z25.s, z15.s\n"
- "st1w z6.s, p1, [c_ptr2, #1, MUL VL]\n"
- "fmin z26.s, p7/m, z26.s, z15.s\n"
- "fmin z27.s, p7/m, z27.s, z15.s\n"
- "fmin z28.s, p7/m, z28.s, z15.s\n"
- "st1w z7.s, p1, [c_ptr3, #1, MUL VL]\n"
- "uzp1 z8.s, z24.s, z25.s\n"
- "uzp2 z9.s, z24.s, z25.s\n"
- "uzp1 z10.s, z26.s, z27.s\n"
- "uzp2 z11.s, z26.s, z27.s\n"
- "st1w z8.s, p0, [c_ptr4]\n"
- "fmax z29.s, p7/m, z29.s, z14.s\n"
- "fmax z30.s, p7/m, z30.s, z14.s\n"
- "fmax z31.s, p7/m, z31.s, z14.s\n"
- "st1w z9.s, p0, [c_ptr5]\n"
- "fmin z29.s, p7/m, z29.s, z15.s\n"
- "fmin z30.s, p7/m, z30.s, z15.s\n"
- "st1w z10.s, p1, [c_ptr4, #1, MUL VL]\n"
- "fmin z31.s, p7/m, z31.s, z15.s\n"
- "uzp1 z12.s, z28.s, z29.s\n"
- "st1w z11.s, p1, [c_ptr5, #1, MUL VL]\n"
- "uzp1 z13.s, z30.s, z31.s\n"
- "st1w z12.s, p0, [c_ptr6]\n"
- "st1w z13.s, p1, [c_ptr6, #1, MUL VL]\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq a_ptr3\n"
- ".unreq a_ptr4\n"
- ".unreq a_ptr5\n"
- ".unreq a_ptr6\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- ".unreq c_ptr3\n"
- ".unreq c_ptr4\n"
- ".unreq c_ptr5\n"
- ".unreq c_ptr6\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks)
- : [width] "r" (width), [accumulate] "r" (static_cast<uint64_t>(accumulate)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers)
- : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "cc", "memory"
- );
- break;
- default:
- case 8:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "a_ptr3 .req X2\n"
- "a_ptr4 .req X3\n"
- "a_ptr5 .req X4\n"
- "a_ptr6 .req X5\n"
- "a_ptr7 .req X6\n"
- "c_ptr1 .req X7\n"
- "c_ptr2 .req X8\n"
- "c_ptr3 .req X9\n"
- "c_ptr4 .req X10\n"
- "c_ptr5 .req X11\n"
- "c_ptr6 .req X12\n"
- "c_ptr7 .req X13\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "add a_ptr3, a_ptr2, %[lda]\n"
- "add c_ptr3, c_ptr2, %[ldc]\n"
- "add a_ptr4, a_ptr3, %[lda]\n"
- "add c_ptr4, c_ptr3, %[ldc]\n"
- "add a_ptr5, a_ptr4, %[lda]\n"
- "add c_ptr5, c_ptr4, %[ldc]\n"
- "add a_ptr6, a_ptr5, %[lda]\n"
- "add c_ptr6, c_ptr5, %[ldc]\n"
- "add a_ptr7, a_ptr6, %[lda]\n"
- "add c_ptr7, c_ptr6, %[ldc]\n"
- "whilelt p6.h, %[temp], %[leftovers]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "ptrue p7.h\n"
- "whilelt p1.s, %[temp], %[width]\n"
- "cbnz %[accumulate], 1f\n"
- "ld1w z15.s, p0/z, [%[biasptr]]\n"
- "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ld1rqh z1.h, p7/z, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "zip1 z16.s, z15.s, z15.s\n"
- "ld1rqh z2.h, p7/z, [a_ptr2]\n"
- "zip2 z17.s, z15.s, z15.s\n"
- "ld1w z15.s, p1/z, [%[biasptr], #1, MUL VL]\n"
- "trn1 z8.d, z0.d, z1.d\n"
- "ld1rqh z3.h, p7/z, [a_ptr3]\n"
- "ld1rqh z4.h, p7/z, [a_ptr4]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "zip1 z18.s, z15.s, z15.s\n"
- "ld1rqh z5.h, p7/z, [a_ptr5]\n"
- "zip2 z19.s, z15.s, z15.s\n"
- "ld1rqh z6.h, p7/z, [a_ptr6]\n"
- "trn1 z9.d, z2.d, z3.d\n"
- "ld1rqh z7.h, p7/z, [a_ptr7]\n"
- "mov z20.d, z16.d\n"
- "ld1h z12.h, p7/z, [%[b_ptr0]]\n"
- "trn1 z10.d, z4.d, z5.d\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "mov z21.d, z17.d\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "trn1 z11.d, z6.d, z7.d\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "mov z22.d, z18.d\n"
- "add a_ptr3, a_ptr3, #0x10\n"
- "mov z23.d, z19.d\n"
- "add a_ptr4, a_ptr4, #0x10\n"
- "mov z24.d, z16.d\n"
- "add a_ptr5, a_ptr5, #0x10\n"
- "mov z25.d, z17.d\n"
- "add a_ptr6, a_ptr6, #0x10\n"
- "mov z26.d, z18.d\n"
- "add a_ptr7, a_ptr7, #0x10\n"
- "mov z27.d, z19.d\n"
- "addvl %[b_ptr0], %[b_ptr0], #4\n"
- "mov z28.d, z16.d\n"
- "mov z29.d, z17.d\n"
- "mov z30.d, z18.d\n"
- "mov z31.d, z19.d\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "ld1w z13.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z14.s, p0/z, [c_ptr1]\n"
- "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ld1rqh z1.h, p7/z, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "zip1 z16.s, z13.s, z14.s\n"
- "ld1rqh z2.h, p7/z, [a_ptr2]\n"
- "zip2 z17.s, z13.s, z14.s\n"
- "ld1w z13.s, p1/z, [%[c_ptr0], #1, MUL VL]\n"
- "trn1 z8.d, z0.d, z1.d\n"
- "ld1w z14.s, p1/z, [c_ptr1, #1, MUL VL]\n"
- "ld1rqh z3.h, p7/z, [a_ptr3]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "ld1rqh z4.h, p7/z, [a_ptr4]\n"
- "add a_ptr3, a_ptr3, #0x10\n"
- "zip1 z18.s, z13.s, z14.s\n"
- "ld1rqh z5.h, p7/z, [a_ptr5]\n"
- "zip2 z19.s, z13.s, z14.s\n"
- "ld1w z13.s, p0/z, [c_ptr2]\n"
- "trn1 z9.d, z2.d, z3.d\n"
- "ld1w z14.s, p0/z, [c_ptr3]\n"
- "ld1rqh z6.h, p7/z, [a_ptr6]\n"
- "add a_ptr4, a_ptr4, #0x10\n"
- "trn1 z10.d, z4.d, z5.d\n"
- "ld1rqh z7.h, p7/z, [a_ptr7]\n"
- "zip1 z20.s, z13.s, z14.s\n"
- "ld1h z12.h, p7/z, [%[b_ptr0]]\n"
- "zip2 z21.s, z13.s, z14.s\n"
- "ld1w z13.s, p1/z, [c_ptr2, #1, MUL VL]\n"
- "ld1w z14.s, p1/z, [c_ptr3, #1, MUL VL]\n"
- "add a_ptr5, a_ptr5, #0x10\n"
- "trn1 z11.d, z6.d, z7.d\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "add a_ptr6, a_ptr6, #0x10\n"
- "zip1 z22.s, z13.s, z14.s\n"
- "add a_ptr7, a_ptr7, #0x10\n"
- "zip2 z23.s, z13.s, z14.s\n"
- "ld1w z13.s, p0/z, [c_ptr4]\n"
- "ld1w z14.s, p0/z, [c_ptr5]\n"
- "zip1 z24.s, z13.s, z14.s\n"
- "zip2 z25.s, z13.s, z14.s\n"
- "ld1w z13.s, p1/z, [c_ptr4, #1, MUL VL]\n"
- "ld1w z14.s, p1/z, [c_ptr5, #1, MUL VL]\n"
- "zip1 z26.s, z13.s, z14.s\n"
- "zip2 z27.s, z13.s, z14.s\n"
- "ld1w z13.s, p0/z, [c_ptr6]\n"
- "ld1w z14.s, p0/z, [c_ptr7]\n"
- "zip1 z28.s, z13.s, z14.s\n"
- "zip2 z29.s, z13.s, z14.s\n"
- "ld1w z13.s, p1/z, [c_ptr6, #1, MUL VL]\n"
- "ld1w z14.s, p1/z, [c_ptr7, #1, MUL VL]\n"
- "zip1 z30.s, z13.s, z14.s\n"
- "zip2 z31.s, z13.s, z14.s\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #4\n"
- "cbz %[loops], 2f\n"
- "3:\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "subs %[loops], %[loops], #0x1\n"
- "trn2 z1.d, z2.d, z3.d\n"
- "trn2 z2.d, z4.d, z5.d\n"
- "ld1rqh z4.h, p7/z, [%[a_ptr0]]\n"
- "trn2 z3.d, z6.d, z7.d\n"
- "ld1rqh z5.h, p7/z, [a_ptr1]\n"
- ".inst 0x6468e590 // bfmmla z16.s, z12.h, z8.h\n"
- "ld1rqh z6.h, p7/z, [a_ptr2]\n"
- ".inst 0x6468e5b1 // bfmmla z17.s, z13.h, z8.h\n"
- "ld1rqh z7.h, p7/z, [a_ptr3]\n"
- ".inst 0x6468e5d2 // bfmmla z18.s, z14.h, z8.h\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
- ".inst 0x6468e5f3 // bfmmla z19.s, z15.h, z8.h\n"
- "ld1rqh z8.h, p7/z, [a_ptr4]\n"
- ".inst 0x6469e594 // bfmmla z20.s, z12.h, z9.h\n"
- "add a_ptr1, a_ptr1, #0x20\n"
- ".inst 0x6469e5b5 // bfmmla z21.s, z13.h, z9.h\n"
- "add a_ptr2, a_ptr2, #0x20\n"
- ".inst 0x6469e5d6 // bfmmla z22.s, z14.h, z9.h\n"
- "add a_ptr3, a_ptr3, #0x20\n"
- ".inst 0x6469e5f7 // bfmmla z23.s, z15.h, z9.h\n"
- "ld1rqh z9.h, p7/z, [a_ptr5]\n"
- ".inst 0x646ae598 // bfmmla z24.s, z12.h, z10.h\n"
- "add a_ptr4, a_ptr4, #0x20\n"
- ".inst 0x646ae5b9 // bfmmla z25.s, z13.h, z10.h\n"
- "add a_ptr5, a_ptr5, #0x20\n"
- ".inst 0x646ae5da // bfmmla z26.s, z14.h, z10.h\n"
- ".inst 0x646ae5fb // bfmmla z27.s, z15.h, z10.h\n"
- "ld1rqh z10.h, p7/z, [a_ptr6]\n"
- ".inst 0x646be59c // bfmmla z28.s, z12.h, z11.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0]]\n"
- ".inst 0x646be5bd // bfmmla z29.s, z13.h, z11.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x646be5de // bfmmla z30.s, z14.h, z11.h\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x646be5ff // bfmmla z31.s, z15.h, z11.h\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x6460e590 // bfmmla z16.s, z12.h, z0.h\n"
- "ld1rqh z11.h, p7/z, [a_ptr7]\n"
- ".inst 0x6460e5b1 // bfmmla z17.s, z13.h, z0.h\n"
- "add a_ptr6, a_ptr6, #0x20\n"
- ".inst 0x6460e5d2 // bfmmla z18.s, z14.h, z0.h\n"
- "add a_ptr7, a_ptr7, #0x20\n"
- ".inst 0x6460e5f3 // bfmmla z19.s, z15.h, z0.h\n"
- "trn1 z0.d, z4.d, z5.d\n"
- ".inst 0x6461e594 // bfmmla z20.s, z12.h, z1.h\n"
- ".inst 0x6461e5b5 // bfmmla z21.s, z13.h, z1.h\n"
- ".inst 0x6461e5d6 // bfmmla z22.s, z14.h, z1.h\n"
- ".inst 0x6461e5f7 // bfmmla z23.s, z15.h, z1.h\n"
- "trn1 z1.d, z6.d, z7.d\n"
- ".inst 0x6462e598 // bfmmla z24.s, z12.h, z2.h\n"
- ".inst 0x6462e5b9 // bfmmla z25.s, z13.h, z2.h\n"
- ".inst 0x6462e5da // bfmmla z26.s, z14.h, z2.h\n"
- ".inst 0x6462e5fb // bfmmla z27.s, z15.h, z2.h\n"
- "trn1 z2.d, z8.d, z9.d\n"
- ".inst 0x6463e59c // bfmmla z28.s, z12.h, z3.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x6463e5bd // bfmmla z29.s, z13.h, z3.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x6463e5de // bfmmla z30.s, z14.h, z3.h\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x6463e5ff // bfmmla z31.s, z15.h, z3.h\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "trn1 z3.d, z10.d, z11.d\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "trn2 z11.d, z10.d, z11.d\n"
- "trn2 z10.d, z8.d, z9.d\n"
- "trn2 z9.d, z6.d, z7.d\n"
- "ld1rqh z6.h, p7/z, [a_ptr6, #-0x10]\n"
- "trn2 z8.d, z4.d, z5.d\n"
- "ld1rqh z4.h, p7/z, [a_ptr4, #-0x10]\n"
- ".inst 0x6460e590 // bfmmla z16.s, z12.h, z0.h\n"
- "ld1rqh z5.h, p7/z, [a_ptr5, #-0x10]\n"
- ".inst 0x6460e5b1 // bfmmla z17.s, z13.h, z0.h\n"
- "ld1rqh z7.h, p7/z, [a_ptr7, #-0x10]\n"
- ".inst 0x6460e5d2 // bfmmla z18.s, z14.h, z0.h\n"
- ".inst 0x6460e5f3 // bfmmla z19.s, z15.h, z0.h\n"
- "ld1rqh z0.h, p7/z, [%[a_ptr0], #-0x10]\n"
- ".inst 0x6461e594 // bfmmla z20.s, z12.h, z1.h\n"
- ".inst 0x6461e5b5 // bfmmla z21.s, z13.h, z1.h\n"
- ".inst 0x6461e5d6 // bfmmla z22.s, z14.h, z1.h\n"
- ".inst 0x6461e5f7 // bfmmla z23.s, z15.h, z1.h\n"
- "ld1rqh z1.h, p7/z, [a_ptr1, #-0x10]\n"
- ".inst 0x6462e598 // bfmmla z24.s, z12.h, z2.h\n"
- ".inst 0x6462e5b9 // bfmmla z25.s, z13.h, z2.h\n"
- ".inst 0x6462e5da // bfmmla z26.s, z14.h, z2.h\n"
- ".inst 0x6462e5fb // bfmmla z27.s, z15.h, z2.h\n"
- "ld1rqh z2.h, p7/z, [a_ptr2, #-0x10]\n"
- ".inst 0x6463e59c // bfmmla z28.s, z12.h, z3.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- ".inst 0x6463e5bd // bfmmla z29.s, z13.h, z3.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- ".inst 0x6463e5de // bfmmla z30.s, z14.h, z3.h\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- ".inst 0x6463e5ff // bfmmla z31.s, z15.h, z3.h\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- ".inst 0x6468e590 // bfmmla z16.s, z12.h, z8.h\n"
- "ld1rqh z3.h, p7/z, [a_ptr3, #-0x10]\n"
- ".inst 0x6468e5b1 // bfmmla z17.s, z13.h, z8.h\n"
- ".inst 0x6468e5d2 // bfmmla z18.s, z14.h, z8.h\n"
- ".inst 0x6468e5f3 // bfmmla z19.s, z15.h, z8.h\n"
- "trn1 z8.d, z0.d, z1.d\n"
- ".inst 0x6469e594 // bfmmla z20.s, z12.h, z9.h\n"
- ".inst 0x6469e5b5 // bfmmla z21.s, z13.h, z9.h\n"
- ".inst 0x6469e5d6 // bfmmla z22.s, z14.h, z9.h\n"
- ".inst 0x6469e5f7 // bfmmla z23.s, z15.h, z9.h\n"
- "trn1 z9.d, z2.d, z3.d\n"
- ".inst 0x646ae598 // bfmmla z24.s, z12.h, z10.h\n"
- ".inst 0x646ae5b9 // bfmmla z25.s, z13.h, z10.h\n"
- ".inst 0x646ae5da // bfmmla z26.s, z14.h, z10.h\n"
- ".inst 0x646ae5fb // bfmmla z27.s, z15.h, z10.h\n"
- "trn1 z10.d, z4.d, z5.d\n"
- ".inst 0x646be59c // bfmmla z28.s, z12.h, z11.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- ".inst 0x646be5bd // bfmmla z29.s, z13.h, z11.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- ".inst 0x646be5de // bfmmla z30.s, z14.h, z11.h\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- ".inst 0x646be5ff // bfmmla z31.s, z15.h, z11.h\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "trn1 z11.d, z6.d, z7.d\n"
- "b.ne 3b\n"
- "2:\n"
- "cbz %[regs], 4f\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "trn2 z1.d, z2.d, z3.d\n"
- "trn2 z2.d, z4.d, z5.d\n"
- "ld1rqh z4.h, p7/z, [%[a_ptr0]]\n"
- "trn2 z3.d, z6.d, z7.d\n"
- "ld1rqh z5.h, p7/z, [a_ptr1]\n"
- ".inst 0x6468e590 // bfmmla z16.s, z12.h, z8.h\n"
- "ld1rqh z6.h, p7/z, [a_ptr2]\n"
- ".inst 0x6468e5b1 // bfmmla z17.s, z13.h, z8.h\n"
- "ld1rqh z7.h, p7/z, [a_ptr3]\n"
- ".inst 0x6468e5d2 // bfmmla z18.s, z14.h, z8.h\n"
- ".inst 0x6468e5f3 // bfmmla z19.s, z15.h, z8.h\n"
- "ld1rqh z8.h, p7/z, [a_ptr4]\n"
- ".inst 0x6469e594 // bfmmla z20.s, z12.h, z9.h\n"
- ".inst 0x6469e5b5 // bfmmla z21.s, z13.h, z9.h\n"
- ".inst 0x6469e5d6 // bfmmla z22.s, z14.h, z9.h\n"
- ".inst 0x6469e5f7 // bfmmla z23.s, z15.h, z9.h\n"
- "ld1rqh z9.h, p7/z, [a_ptr5]\n"
- ".inst 0x646ae598 // bfmmla z24.s, z12.h, z10.h\n"
- ".inst 0x646ae5b9 // bfmmla z25.s, z13.h, z10.h\n"
- ".inst 0x646ae5da // bfmmla z26.s, z14.h, z10.h\n"
- ".inst 0x646ae5fb // bfmmla z27.s, z15.h, z10.h\n"
- "ld1rqh z10.h, p7/z, [a_ptr6]\n"
- ".inst 0x646be59c // bfmmla z28.s, z12.h, z11.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0]]\n"
- ".inst 0x646be5bd // bfmmla z29.s, z13.h, z11.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x646be5de // bfmmla z30.s, z14.h, z11.h\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x646be5ff // bfmmla z31.s, z15.h, z11.h\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x6460e590 // bfmmla z16.s, z12.h, z0.h\n"
- "ld1rqh z11.h, p7/z, [a_ptr7]\n"
- ".inst 0x6460e5b1 // bfmmla z17.s, z13.h, z0.h\n"
- ".inst 0x6460e5d2 // bfmmla z18.s, z14.h, z0.h\n"
- ".inst 0x6460e5f3 // bfmmla z19.s, z15.h, z0.h\n"
- "trn1 z0.d, z4.d, z5.d\n"
- ".inst 0x6461e594 // bfmmla z20.s, z12.h, z1.h\n"
- ".inst 0x6461e5b5 // bfmmla z21.s, z13.h, z1.h\n"
- ".inst 0x6461e5d6 // bfmmla z22.s, z14.h, z1.h\n"
- ".inst 0x6461e5f7 // bfmmla z23.s, z15.h, z1.h\n"
- "trn1 z1.d, z6.d, z7.d\n"
- ".inst 0x6462e598 // bfmmla z24.s, z12.h, z2.h\n"
- ".inst 0x6462e5b9 // bfmmla z25.s, z13.h, z2.h\n"
- ".inst 0x6462e5da // bfmmla z26.s, z14.h, z2.h\n"
- ".inst 0x6462e5fb // bfmmla z27.s, z15.h, z2.h\n"
- "trn1 z2.d, z8.d, z9.d\n"
- ".inst 0x6463e59c // bfmmla z28.s, z12.h, z3.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x6463e5bd // bfmmla z29.s, z13.h, z3.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x6463e5de // bfmmla z30.s, z14.h, z3.h\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x6463e5ff // bfmmla z31.s, z15.h, z3.h\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "trn1 z3.d, z10.d, z11.d\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "trn2 z11.d, z10.d, z11.d\n"
- "trn2 z10.d, z8.d, z9.d\n"
- "trn2 z9.d, z6.d, z7.d\n"
- "ld1rqh z6.h, p6/z, [a_ptr6, #0x10]\n"
- "trn2 z8.d, z4.d, z5.d\n"
- "ld1rqh z4.h, p6/z, [a_ptr4, #0x10]\n"
- ".inst 0x6460e590 // bfmmla z16.s, z12.h, z0.h\n"
- "ld1rqh z5.h, p6/z, [a_ptr5, #0x10]\n"
- ".inst 0x6460e5b1 // bfmmla z17.s, z13.h, z0.h\n"
- "ld1rqh z7.h, p6/z, [a_ptr7, #0x10]\n"
- ".inst 0x6460e5d2 // bfmmla z18.s, z14.h, z0.h\n"
- "addvl a_ptr4, a_ptr4, #2\n"
- ".inst 0x6460e5f3 // bfmmla z19.s, z15.h, z0.h\n"
- "ld1rqh z0.h, p6/z, [%[a_ptr0], #0x10]\n"
- ".inst 0x6461e594 // bfmmla z20.s, z12.h, z1.h\n"
- "addvl %[a_ptr0], %[a_ptr0], #2\n"
- ".inst 0x6461e5b5 // bfmmla z21.s, z13.h, z1.h\n"
- "addvl a_ptr5, a_ptr5, #2\n"
- ".inst 0x6461e5d6 // bfmmla z22.s, z14.h, z1.h\n"
- "addvl a_ptr6, a_ptr6, #2\n"
- ".inst 0x6461e5f7 // bfmmla z23.s, z15.h, z1.h\n"
- "ld1rqh z1.h, p6/z, [a_ptr1, #0x10]\n"
- ".inst 0x6462e598 // bfmmla z24.s, z12.h, z2.h\n"
- "addvl a_ptr1, a_ptr1, #2\n"
- ".inst 0x6462e5b9 // bfmmla z25.s, z13.h, z2.h\n"
- "addvl a_ptr7, a_ptr7, #2\n"
- ".inst 0x6462e5da // bfmmla z26.s, z14.h, z2.h\n"
- ".inst 0x6462e5fb // bfmmla z27.s, z15.h, z2.h\n"
- "ld1rqh z2.h, p6/z, [a_ptr2, #0x10]\n"
- ".inst 0x6463e59c // bfmmla z28.s, z12.h, z3.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- ".inst 0x6463e5bd // bfmmla z29.s, z13.h, z3.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- ".inst 0x6463e5de // bfmmla z30.s, z14.h, z3.h\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- ".inst 0x6463e5ff // bfmmla z31.s, z15.h, z3.h\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- ".inst 0x6468e590 // bfmmla z16.s, z12.h, z8.h\n"
- "ld1rqh z3.h, p6/z, [a_ptr3, #0x10]\n"
- ".inst 0x6468e5b1 // bfmmla z17.s, z13.h, z8.h\n"
- "addvl %[b_ptr0], %[b_ptr0], #-4\n"
- ".inst 0x6468e5d2 // bfmmla z18.s, z14.h, z8.h\n"
- "addvl a_ptr2, a_ptr2, #2\n"
- ".inst 0x6468e5f3 // bfmmla z19.s, z15.h, z8.h\n"
- "addvl a_ptr3, a_ptr3, #2\n"
- "trn1 z8.d, z0.d, z1.d\n"
- ".inst 0x6469e594 // bfmmla z20.s, z12.h, z9.h\n"
- ".inst 0x6469e5b5 // bfmmla z21.s, z13.h, z9.h\n"
- ".inst 0x6469e5d6 // bfmmla z22.s, z14.h, z9.h\n"
- ".inst 0x6469e5f7 // bfmmla z23.s, z15.h, z9.h\n"
- "trn1 z9.d, z2.d, z3.d\n"
- ".inst 0x646ae598 // bfmmla z24.s, z12.h, z10.h\n"
- ".inst 0x646ae5b9 // bfmmla z25.s, z13.h, z10.h\n"
- ".inst 0x646ae5da // bfmmla z26.s, z14.h, z10.h\n"
- ".inst 0x646ae5fb // bfmmla z27.s, z15.h, z10.h\n"
- "trn1 z10.d, z4.d, z5.d\n"
- ".inst 0x646be59c // bfmmla z28.s, z12.h, z11.h\n"
- ".inst 0x646be5bd // bfmmla z29.s, z13.h, z11.h\n"
- ".inst 0x646be5de // bfmmla z30.s, z14.h, z11.h\n"
- ".inst 0x646be5ff // bfmmla z31.s, z15.h, z11.h\n"
- "trn1 z11.d, z6.d, z7.d\n"
- "cbz %[blocks], 5f\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "ld1h z12.h, p7/z, [%[b_ptr0]]\n"
- "trn2 z1.d, z2.d, z3.d\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "trn2 z2.d, z4.d, z5.d\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "trn2 z3.d, z6.d, z7.d\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x6468e590 // bfmmla z16.s, z12.h, z8.h\n"
- "subs %[blocks], %[blocks], #0x1\n"
- ".inst 0x6468e5b1 // bfmmla z17.s, z13.h, z8.h\n"
- ".inst 0x6468e5d2 // bfmmla z18.s, z14.h, z8.h\n"
- ".inst 0x6468e5f3 // bfmmla z19.s, z15.h, z8.h\n"
- ".inst 0x6469e594 // bfmmla z20.s, z12.h, z9.h\n"
- ".inst 0x6469e5b5 // bfmmla z21.s, z13.h, z9.h\n"
- ".inst 0x6469e5d6 // bfmmla z22.s, z14.h, z9.h\n"
- ".inst 0x6469e5f7 // bfmmla z23.s, z15.h, z9.h\n"
- ".inst 0x646ae598 // bfmmla z24.s, z12.h, z10.h\n"
- ".inst 0x646ae5b9 // bfmmla z25.s, z13.h, z10.h\n"
- ".inst 0x646ae5da // bfmmla z26.s, z14.h, z10.h\n"
- ".inst 0x646ae5fb // bfmmla z27.s, z15.h, z10.h\n"
- ".inst 0x646be59c // bfmmla z28.s, z12.h, z11.h\n"
- ".inst 0x646be5bd // bfmmla z29.s, z13.h, z11.h\n"
- ".inst 0x646be5de // bfmmla z30.s, z14.h, z11.h\n"
- ".inst 0x646be5ff // bfmmla z31.s, z15.h, z11.h\n"
- "b.eq 5f\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x6460e590 // bfmmla z16.s, z12.h, z0.h\n"
- ".inst 0x6460e5b1 // bfmmla z17.s, z13.h, z0.h\n"
- ".inst 0x6460e5d2 // bfmmla z18.s, z14.h, z0.h\n"
- ".inst 0x6460e5f3 // bfmmla z19.s, z15.h, z0.h\n"
- ".inst 0x6461e594 // bfmmla z20.s, z12.h, z1.h\n"
- ".inst 0x6461e5b5 // bfmmla z21.s, z13.h, z1.h\n"
- ".inst 0x6461e5d6 // bfmmla z22.s, z14.h, z1.h\n"
- ".inst 0x6461e5f7 // bfmmla z23.s, z15.h, z1.h\n"
- ".inst 0x6462e598 // bfmmla z24.s, z12.h, z2.h\n"
- ".inst 0x6462e5b9 // bfmmla z25.s, z13.h, z2.h\n"
- ".inst 0x6462e5da // bfmmla z26.s, z14.h, z2.h\n"
- ".inst 0x6462e5fb // bfmmla z27.s, z15.h, z2.h\n"
- ".inst 0x6463e59c // bfmmla z28.s, z12.h, z3.h\n"
- ".inst 0x6463e5bd // bfmmla z29.s, z13.h, z3.h\n"
- ".inst 0x6463e5de // bfmmla z30.s, z14.h, z3.h\n"
- ".inst 0x6463e5ff // bfmmla z31.s, z15.h, z3.h\n"
- "b 5f\n"
- "4:\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "trn2 z1.d, z2.d, z3.d\n"
- "trn2 z2.d, z4.d, z5.d\n"
- "ld1rqh z4.h, p6/z, [%[a_ptr0]]\n"
- "trn2 z3.d, z6.d, z7.d\n"
- "ld1rqh z5.h, p6/z, [a_ptr1]\n"
- ".inst 0x6468e590 // bfmmla z16.s, z12.h, z8.h\n"
- "ld1rqh z6.h, p6/z, [a_ptr2]\n"
- ".inst 0x6468e5b1 // bfmmla z17.s, z13.h, z8.h\n"
- "ld1rqh z7.h, p6/z, [a_ptr3]\n"
- ".inst 0x6468e5d2 // bfmmla z18.s, z14.h, z8.h\n"
- "addvl %[a_ptr0], %[a_ptr0], #1\n"
- ".inst 0x6468e5f3 // bfmmla z19.s, z15.h, z8.h\n"
- "ld1rqh z8.h, p6/z, [a_ptr4]\n"
- ".inst 0x6469e594 // bfmmla z20.s, z12.h, z9.h\n"
- "addvl a_ptr1, a_ptr1, #1\n"
- ".inst 0x6469e5b5 // bfmmla z21.s, z13.h, z9.h\n"
- "addvl a_ptr2, a_ptr2, #1\n"
- ".inst 0x6469e5d6 // bfmmla z22.s, z14.h, z9.h\n"
- "addvl a_ptr3, a_ptr3, #1\n"
- ".inst 0x6469e5f7 // bfmmla z23.s, z15.h, z9.h\n"
- "ld1rqh z9.h, p6/z, [a_ptr5]\n"
- ".inst 0x646ae598 // bfmmla z24.s, z12.h, z10.h\n"
- "addvl a_ptr4, a_ptr4, #1\n"
- ".inst 0x646ae5b9 // bfmmla z25.s, z13.h, z10.h\n"
- "addvl a_ptr5, a_ptr5, #1\n"
- ".inst 0x646ae5da // bfmmla z26.s, z14.h, z10.h\n"
- ".inst 0x646ae5fb // bfmmla z27.s, z15.h, z10.h\n"
- "ld1rqh z10.h, p6/z, [a_ptr6]\n"
- ".inst 0x646be59c // bfmmla z28.s, z12.h, z11.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0]]\n"
- ".inst 0x646be5bd // bfmmla z29.s, z13.h, z11.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x646be5de // bfmmla z30.s, z14.h, z11.h\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x646be5ff // bfmmla z31.s, z15.h, z11.h\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x6460e590 // bfmmla z16.s, z12.h, z0.h\n"
- "ld1rqh z11.h, p6/z, [a_ptr7]\n"
- ".inst 0x6460e5b1 // bfmmla z17.s, z13.h, z0.h\n"
- "addvl %[b_ptr0], %[b_ptr0], #4\n"
- ".inst 0x6460e5d2 // bfmmla z18.s, z14.h, z0.h\n"
- "addvl a_ptr6, a_ptr6, #1\n"
- ".inst 0x6460e5f3 // bfmmla z19.s, z15.h, z0.h\n"
- "addvl a_ptr7, a_ptr7, #1\n"
- "trn1 z0.d, z4.d, z5.d\n"
- ".inst 0x6461e594 // bfmmla z20.s, z12.h, z1.h\n"
- ".inst 0x6461e5b5 // bfmmla z21.s, z13.h, z1.h\n"
- ".inst 0x6461e5d6 // bfmmla z22.s, z14.h, z1.h\n"
- ".inst 0x6461e5f7 // bfmmla z23.s, z15.h, z1.h\n"
- "trn1 z1.d, z6.d, z7.d\n"
- ".inst 0x6462e598 // bfmmla z24.s, z12.h, z2.h\n"
- ".inst 0x6462e5b9 // bfmmla z25.s, z13.h, z2.h\n"
- ".inst 0x6462e5da // bfmmla z26.s, z14.h, z2.h\n"
- ".inst 0x6462e5fb // bfmmla z27.s, z15.h, z2.h\n"
- "trn1 z2.d, z8.d, z9.d\n"
- ".inst 0x6463e59c // bfmmla z28.s, z12.h, z3.h\n"
- ".inst 0x6463e5bd // bfmmla z29.s, z13.h, z3.h\n"
- ".inst 0x6463e5de // bfmmla z30.s, z14.h, z3.h\n"
- ".inst 0x6463e5ff // bfmmla z31.s, z15.h, z3.h\n"
- "trn1 z3.d, z10.d, z11.d\n"
- "cbz %[blocks], 5f\n"
- "trn2 z11.d, z10.d, z11.d\n"
- "ld1h z12.h, p7/z, [%[b_ptr0]]\n"
- "trn2 z10.d, z8.d, z9.d\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "trn2 z9.d, z6.d, z7.d\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "trn2 z8.d, z4.d, z5.d\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x6460e590 // bfmmla z16.s, z12.h, z0.h\n"
- "subs %[blocks], %[blocks], #0x1\n"
- ".inst 0x6460e5b1 // bfmmla z17.s, z13.h, z0.h\n"
- ".inst 0x6460e5d2 // bfmmla z18.s, z14.h, z0.h\n"
- ".inst 0x6460e5f3 // bfmmla z19.s, z15.h, z0.h\n"
- ".inst 0x6461e594 // bfmmla z20.s, z12.h, z1.h\n"
- ".inst 0x6461e5b5 // bfmmla z21.s, z13.h, z1.h\n"
- ".inst 0x6461e5d6 // bfmmla z22.s, z14.h, z1.h\n"
- ".inst 0x6461e5f7 // bfmmla z23.s, z15.h, z1.h\n"
- ".inst 0x6462e598 // bfmmla z24.s, z12.h, z2.h\n"
- ".inst 0x6462e5b9 // bfmmla z25.s, z13.h, z2.h\n"
- ".inst 0x6462e5da // bfmmla z26.s, z14.h, z2.h\n"
- ".inst 0x6462e5fb // bfmmla z27.s, z15.h, z2.h\n"
- ".inst 0x6463e59c // bfmmla z28.s, z12.h, z3.h\n"
- ".inst 0x6463e5bd // bfmmla z29.s, z13.h, z3.h\n"
- ".inst 0x6463e5de // bfmmla z30.s, z14.h, z3.h\n"
- ".inst 0x6463e5ff // bfmmla z31.s, z15.h, z3.h\n"
- "b.eq 5f\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x6468e590 // bfmmla z16.s, z12.h, z8.h\n"
- ".inst 0x6468e5b1 // bfmmla z17.s, z13.h, z8.h\n"
- ".inst 0x6468e5d2 // bfmmla z18.s, z14.h, z8.h\n"
- ".inst 0x6468e5f3 // bfmmla z19.s, z15.h, z8.h\n"
- ".inst 0x6469e594 // bfmmla z20.s, z12.h, z9.h\n"
- ".inst 0x6469e5b5 // bfmmla z21.s, z13.h, z9.h\n"
- ".inst 0x6469e5d6 // bfmmla z22.s, z14.h, z9.h\n"
- ".inst 0x6469e5f7 // bfmmla z23.s, z15.h, z9.h\n"
- ".inst 0x646ae598 // bfmmla z24.s, z12.h, z10.h\n"
- ".inst 0x646ae5b9 // bfmmla z25.s, z13.h, z10.h\n"
- ".inst 0x646ae5da // bfmmla z26.s, z14.h, z10.h\n"
- ".inst 0x646ae5fb // bfmmla z27.s, z15.h, z10.h\n"
- ".inst 0x646be59c // bfmmla z28.s, z12.h, z11.h\n"
- ".inst 0x646be5bd // bfmmla z29.s, z13.h, z11.h\n"
- ".inst 0x646be5de // bfmmla z30.s, z14.h, z11.h\n"
- ".inst 0x646be5ff // bfmmla z31.s, z15.h, z11.h\n"
- "5:\n"
- "ld1rw z14.s, p7/z, [%[minptr]]\n"
- "ld1rw z15.s, p7/z, [%[maxptr]]\n"
- "fmax z16.s, p7/m, z16.s, z14.s\n"
- "fmax z17.s, p7/m, z17.s, z14.s\n"
- "fmax z18.s, p7/m, z18.s, z14.s\n"
- "fmax z19.s, p7/m, z19.s, z14.s\n"
- "fmin z16.s, p7/m, z16.s, z15.s\n"
- "fmin z17.s, p7/m, z17.s, z15.s\n"
- "fmin z18.s, p7/m, z18.s, z15.s\n"
- "fmin z19.s, p7/m, z19.s, z15.s\n"
- "fmax z20.s, p7/m, z20.s, z14.s\n"
- "uzp1 z0.s, z16.s, z17.s\n"
- "uzp2 z1.s, z16.s, z17.s\n"
- "uzp1 z2.s, z18.s, z19.s\n"
- "uzp2 z3.s, z18.s, z19.s\n"
- "st1w z0.s, p0, [%[c_ptr0]]\n"
- "fmin z20.s, p7/m, z20.s, z15.s\n"
- "fmax z21.s, p7/m, z21.s, z14.s\n"
- "fmax z22.s, p7/m, z22.s, z14.s\n"
- "st1w z1.s, p0, [c_ptr1]\n"
- "fmax z23.s, p7/m, z23.s, z14.s\n"
- "fmax z24.s, p7/m, z24.s, z14.s\n"
- "fmin z21.s, p7/m, z21.s, z15.s\n"
- "st1w z2.s, p1, [%[c_ptr0], #1, MUL VL]\n"
- "fmin z22.s, p7/m, z22.s, z15.s\n"
- "addvl %[c_ptr0], %[c_ptr0], #2\n"
- "fmin z23.s, p7/m, z23.s, z15.s\n"
- "st1w z3.s, p1, [c_ptr1, #1, MUL VL]\n"
- "uzp1 z4.s, z20.s, z21.s\n"
- "uzp2 z5.s, z20.s, z21.s\n"
- "fmin z24.s, p7/m, z24.s, z15.s\n"
- "uzp1 z6.s, z22.s, z23.s\n"
- "st1w z4.s, p0, [c_ptr2]\n"
- "uzp2 z7.s, z22.s, z23.s\n"
- "fmax z25.s, p7/m, z25.s, z14.s\n"
- "fmax z26.s, p7/m, z26.s, z14.s\n"
- "st1w z5.s, p0, [c_ptr3]\n"
- "fmax z27.s, p7/m, z27.s, z14.s\n"
- "fmax z28.s, p7/m, z28.s, z14.s\n"
- "fmin z25.s, p7/m, z25.s, z15.s\n"
- "st1w z6.s, p1, [c_ptr2, #1, MUL VL]\n"
- "fmin z26.s, p7/m, z26.s, z15.s\n"
- "fmin z27.s, p7/m, z27.s, z15.s\n"
- "fmin z28.s, p7/m, z28.s, z15.s\n"
- "st1w z7.s, p1, [c_ptr3, #1, MUL VL]\n"
- "uzp1 z8.s, z24.s, z25.s\n"
- "uzp2 z9.s, z24.s, z25.s\n"
- "uzp1 z10.s, z26.s, z27.s\n"
- "uzp2 z11.s, z26.s, z27.s\n"
- "st1w z8.s, p0, [c_ptr4]\n"
- "fmax z29.s, p7/m, z29.s, z14.s\n"
- "fmax z30.s, p7/m, z30.s, z14.s\n"
- "fmax z31.s, p7/m, z31.s, z14.s\n"
- "st1w z9.s, p0, [c_ptr5]\n"
- "fmin z29.s, p7/m, z29.s, z15.s\n"
- "fmin z30.s, p7/m, z30.s, z15.s\n"
- "st1w z10.s, p1, [c_ptr4, #1, MUL VL]\n"
- "fmin z31.s, p7/m, z31.s, z15.s\n"
- "uzp1 z12.s, z28.s, z29.s\n"
- "st1w z11.s, p1, [c_ptr5, #1, MUL VL]\n"
- "uzp2 z13.s, z28.s, z29.s\n"
- "uzp1 z14.s, z30.s, z31.s\n"
- "uzp2 z15.s, z30.s, z31.s\n"
- "st1w z12.s, p0, [c_ptr6]\n"
- "st1w z13.s, p0, [c_ptr7]\n"
- "st1w z14.s, p1, [c_ptr6, #1, MUL VL]\n"
- "st1w z15.s, p1, [c_ptr7, #1, MUL VL]\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq a_ptr3\n"
- ".unreq a_ptr4\n"
- ".unreq a_ptr5\n"
- ".unreq a_ptr6\n"
- ".unreq a_ptr7\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- ".unreq c_ptr3\n"
- ".unreq c_ptr4\n"
- ".unreq c_ptr5\n"
- ".unreq c_ptr6\n"
- ".unreq c_ptr7\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks)
- : [width] "r" (width), [accumulate] "r" (static_cast<uint64_t>(accumulate)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers)
- : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "cc", "memory"
- );
- break;
- }
-
- }
- }
-}
-
-} // namespace arm_gemm
-
-#endif // __ARM_FEATURE_SVE
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_bf16fp32_mmla_6VLx2.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_bf16fp32_mmla_6VLx2.hpp
deleted file mode 100644
index bd457e9..0000000
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_bf16fp32_mmla_6VLx2.hpp
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Copyright (c) 2018-2020 Arm Limited.
- *
- * SPDX-License-Identifier: MIT
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to
- * deal in the Software without restriction, including without limitation the
- * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
- * sell copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-#pragma once
-
-#ifdef __ARM_FEATURE_SVE
-
-#include "../bfloat.hpp"
-#include "../std_transforms_sve.hpp"
-
-namespace arm_gemm
-{
-
-// Actual kernel implementations
-void sve_hybrid_bf16fp32_mmla_6VLx2(const bfloat16 *, int, const bfloat16 *, float *, int, int, int, int, const float *, Activation, bool);
-
-class hybrid_bf16fp32_mmla_6VLx2
-{
-public:
- typedef bfloat16 operand_type;
- typedef float result_type;
-
- typedef void (*kern_type)(const bfloat16 *, int, const bfloat16 *, float *, int, int, int, int, const float *, Activation, bool);
-
- /* Kernel blocking parameters */
- static constexpr unsigned int out_height()
- {
- return 4;
- }
-
- static unsigned int out_width()
- {
- return get_vector_length<float>() * 3;
- }
-
- static constexpr unsigned int k_unroll()
- {
- return 4;
- }
-
- static constexpr bool supports_accumulate()
- {
- return true;
- }
-
- static constexpr bool supports_bias()
- {
- return true;
- }
-
- static constexpr bool supports_activation()
- {
- return true;
- }
-
- StdTransformsSVE<operand_type, result_type, 2, 6, 4> transforms = {};
-
- // Default to the generic kernel
- kern_type kernel=sve_hybrid_bf16fp32_mmla_6VLx2;
-
- hybrid_bf16fp32_mmla_6VLx2(const CPUInfo *)
- {
-
- }
-};
-
-} // namespace arm_gemm
-
-#endif // __ARM_FEATURE_SVE
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_bf16fp32_mmla_6VLx2/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_bf16fp32_mmla_6VLx2/generic.cpp
deleted file mode 100644
index 59dc6dc..0000000
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_bf16fp32_mmla_6VLx2/generic.cpp
+++ /dev/null
@@ -1,1633 +0,0 @@
-/*
- * Copyright (c) 2018-2020 Arm Limited.
- *
- * SPDX-License-Identifier: MIT
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to
- * deal in the Software without restriction, including without limitation the
- * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
- * sell copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-#ifdef __ARM_FEATURE_SVE
-
-#include <algorithm>
-
-#include "arm_gemm.hpp"
-#include "../../bfloat.hpp"
-#include "../../asmlib.hpp"
-#include "../../utils.hpp"
-
-namespace arm_gemm {
-
-void sve_hybrid_bf16fp32_mmla_6VLx2(const bfloat16 *A, int lda, const bfloat16 *B, float *C, int ldc, int M, int N, int K, const float *bias, Activation act, bool accumulate) {
- const int K_stride = ((K + 3) / 4) * 4;
- const long loops_count = ((K + 8) / 16) - 1;
- K -= loops_count * 16;
- const long regs_count = (K / 8) - 1;
- K -= (regs_count + 1) * 8;
- const long leftovers = K;
- const long blocks_count = (K + 3) / 4;
- float nullbias[192];
- if (!accumulate && !bias) {
- memset(nullbias, 0, (3 * get_vector_length<float>() * sizeof(float)));
- }
- float minval = - static_cast<float>(std::numeric_limits<float>::infinity());
- float maxval = static_cast<float>(std::numeric_limits<float>::infinity());
- const float * const minptr = &minval;
- const float * const maxptr = &maxval;
-
- switch(act.type)
- {
- default:
- case Activation::Type::None:
- break;
- case Activation::Type::BoundedReLU:
- maxval = static_cast<float>(act.param1);
- /* fall through */
- case Activation::Type::ReLU:
- minval = 0.0f;
- break;
- }
-
- int rows_to_compute;
-
- for (int y=0; y<M; y+=rows_to_compute) {
- const bfloat16 * const a_ptr0_base = A + (y * lda);
- const unsigned long ldab = lda * sizeof(bfloat16);
-
- float *c_ptr0 = C + (y * ldc);
-
- rows_to_compute = M-y;
- if (rows_to_compute > 4) {
- if (rows_to_compute % 4) {
- rows_to_compute = 4 - 1;
- } else {
- rows_to_compute = 4;
- }
- }
-
- for (int x0=0; x0<N; x0+=(3 * get_vector_length<float>())) {
- const long width = std::min((unsigned long)N-x0, (3 * get_vector_length<float>()));
- long loops = loops_count;
- long regs = regs_count;
- long temp = 0;
- long blocks = blocks_count;
- const bfloat16 *a_ptr0 = a_ptr0_base;
- const bfloat16 *b_ptr0 = B + (K_stride * x0);
- const unsigned long ldcb = ldc * sizeof(float);
- const float *biasptr = bias ? bias+x0 : nullbias;
-
- switch(rows_to_compute) {
- case 1:
- __asm __volatile (
- "whilelt p6.h, %[temp], %[leftovers]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "ptrue p7.h\n"
- "whilelt p1.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "whilelt p2.s, %[temp], %[width]\n"
- "cbnz %[accumulate], 1f\n"
- "mov z1.h, #0\n"
- "ld1w z19.s, p0/z, [%[biasptr]]\n"
- "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ld1h z6.h, p7/z, [%[b_ptr0]]\n"
- "zip1 z20.s, z19.s, z19.s\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "zip2 z21.s, z19.s, z19.s\n"
- "ld1w z19.s, p1/z, [%[biasptr], #1, MUL VL]\n"
- "trn1 z4.d, z0.d, z1.d\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "zip1 z22.s, z19.s, z19.s\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "zip2 z23.s, z19.s, z19.s\n"
- "ld1w z19.s, p2/z, [%[biasptr], #2, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #6\n"
- "zip1 z24.s, z19.s, z19.s\n"
- "zip2 z25.s, z19.s, z19.s\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "mov z18.s, #0\n"
- "ld1w z17.s, p0/z, [%[c_ptr0]]\n"
- "mov z1.h, #0\n"
- "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n"
- "ld1h z6.h, p7/z, [%[b_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "zip1 z20.s, z17.s, z18.s\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "zip2 z21.s, z17.s, z18.s\n"
- "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n"
- "trn1 z4.d, z0.d, z1.d\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "mov z18.s, #0\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #6\n"
- "zip1 z22.s, z17.s, z18.s\n"
- "zip2 z23.s, z17.s, z18.s\n"
- "ld1w z17.s, p2/z, [%[c_ptr0], #2, MUL VL]\n"
- "mov z18.s, #0\n"
- "zip1 z24.s, z17.s, z18.s\n"
- "zip2 z25.s, z17.s, z18.s\n"
- "cbz %[loops], 2f\n"
- "3:\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "ld1rqh z2.h, p7/z, [%[a_ptr0]]\n"
- ".inst 0x6464e4d4 // bfmmla z20.s, z6.h, z4.h\n"
- "ld1h z6.h, p7/z, [%[b_ptr0]]\n"
- ".inst 0x6464e4f5 // bfmmla z21.s, z7.h, z4.h\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x6464e516 // bfmmla z22.s, z8.h, z4.h\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x6464e537 // bfmmla z23.s, z9.h, z4.h\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x6464e558 // bfmmla z24.s, z10.h, z4.h\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x6464e579 // bfmmla z25.s, z11.h, z4.h\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "mov z3.h, #0\n"
- "subs %[loops], %[loops], #0x1\n"
- ".inst 0x6460e4d4 // bfmmla z20.s, z6.h, z0.h\n"
- "ld1h z6.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x6460e4f5 // bfmmla z21.s, z7.h, z0.h\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "trn2 z4.d, z2.d, z3.d\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
- ".inst 0x6460e516 // bfmmla z22.s, z8.h, z0.h\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- ".inst 0x6460e537 // bfmmla z23.s, z9.h, z0.h\n"
- ".inst 0x6460e558 // bfmmla z24.s, z10.h, z0.h\n"
- ".inst 0x6460e579 // bfmmla z25.s, z11.h, z0.h\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "trn1 z0.d, z2.d, z3.d\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "mov z1.h, #0\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- ".inst 0x6460e4d4 // bfmmla z20.s, z6.h, z0.h\n"
- "ld1h z6.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- ".inst 0x6460e4f5 // bfmmla z21.s, z7.h, z0.h\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- ".inst 0x6460e516 // bfmmla z22.s, z8.h, z0.h\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- ".inst 0x6460e537 // bfmmla z23.s, z9.h, z0.h\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- ".inst 0x6460e558 // bfmmla z24.s, z10.h, z0.h\n"
- "ld1h z10.h, p7/z, [%[b_ptr0]]\n"
- ".inst 0x6460e579 // bfmmla z25.s, z11.h, z0.h\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x6464e4d4 // bfmmla z20.s, z6.h, z4.h\n"
- "ld1rqh z0.h, p7/z, [%[a_ptr0], #-0x10]\n"
- ".inst 0x6464e4f5 // bfmmla z21.s, z7.h, z4.h\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- ".inst 0x6464e516 // bfmmla z22.s, z8.h, z4.h\n"
- ".inst 0x6464e537 // bfmmla z23.s, z9.h, z4.h\n"
- ".inst 0x6464e558 // bfmmla z24.s, z10.h, z4.h\n"
- "ld1h z6.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- ".inst 0x6464e579 // bfmmla z25.s, z11.h, z4.h\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "trn1 z4.d, z0.d, z1.d\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "b.ne 3b\n"
- "2:\n"
- "cbz %[regs], 4f\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "ld1rqh z2.h, p7/z, [%[a_ptr0]]\n"
- ".inst 0x6464e4d4 // bfmmla z20.s, z6.h, z4.h\n"
- "ld1h z6.h, p7/z, [%[b_ptr0]]\n"
- ".inst 0x6464e4f5 // bfmmla z21.s, z7.h, z4.h\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x6464e516 // bfmmla z22.s, z8.h, z4.h\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x6464e537 // bfmmla z23.s, z9.h, z4.h\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x6464e558 // bfmmla z24.s, z10.h, z4.h\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x6464e579 // bfmmla z25.s, z11.h, z4.h\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "mov z3.h, #0\n"
- ".inst 0x6460e4d4 // bfmmla z20.s, z6.h, z0.h\n"
- "ld1h z6.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x6460e4f5 // bfmmla z21.s, z7.h, z0.h\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x6460e516 // bfmmla z22.s, z8.h, z0.h\n"
- "addvl %[b_ptr0], %[b_ptr0], #6\n"
- "trn2 z4.d, z2.d, z3.d\n"
- ".inst 0x6460e537 // bfmmla z23.s, z9.h, z0.h\n"
- ".inst 0x6460e558 // bfmmla z24.s, z10.h, z0.h\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x6460e579 // bfmmla z25.s, z11.h, z0.h\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "trn1 z0.d, z2.d, z3.d\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "mov z1.h, #0\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x6460e4d4 // bfmmla z20.s, z6.h, z0.h\n"
- "ld1h z6.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x6460e4f5 // bfmmla z21.s, z7.h, z0.h\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x6460e516 // bfmmla z22.s, z8.h, z0.h\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- ".inst 0x6460e537 // bfmmla z23.s, z9.h, z0.h\n"
- ".inst 0x6460e558 // bfmmla z24.s, z10.h, z0.h\n"
- ".inst 0x6460e579 // bfmmla z25.s, z11.h, z0.h\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- ".inst 0x6464e4d4 // bfmmla z20.s, z6.h, z4.h\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- ".inst 0x6464e4f5 // bfmmla z21.s, z7.h, z4.h\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #-4\n"
- ".inst 0x6464e516 // bfmmla z22.s, z8.h, z4.h\n"
- "ld1rqh z0.h, p6/z, [%[a_ptr0], #0x10]\n"
- ".inst 0x6464e537 // bfmmla z23.s, z9.h, z4.h\n"
- "addvl %[a_ptr0], %[a_ptr0], #2\n"
- ".inst 0x6464e558 // bfmmla z24.s, z10.h, z4.h\n"
- ".inst 0x6464e579 // bfmmla z25.s, z11.h, z4.h\n"
- "trn1 z4.d, z0.d, z1.d\n"
- "cbz %[blocks], 5f\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "ld1h z6.h, p7/z, [%[b_ptr0]]\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x6464e4d4 // bfmmla z20.s, z6.h, z4.h\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x6464e4f5 // bfmmla z21.s, z7.h, z4.h\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x6464e516 // bfmmla z22.s, z8.h, z4.h\n"
- ".inst 0x6464e537 // bfmmla z23.s, z9.h, z4.h\n"
- ".inst 0x6464e558 // bfmmla z24.s, z10.h, z4.h\n"
- ".inst 0x6464e579 // bfmmla z25.s, z11.h, z4.h\n"
- "b.eq 5f\n"
- "ld1h z6.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #12\n"
- ".inst 0x6460e4d4 // bfmmla z20.s, z6.h, z0.h\n"
- ".inst 0x6460e4f5 // bfmmla z21.s, z7.h, z0.h\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- ".inst 0x6460e516 // bfmmla z22.s, z8.h, z0.h\n"
- ".inst 0x6460e537 // bfmmla z23.s, z9.h, z0.h\n"
- ".inst 0x6460e558 // bfmmla z24.s, z10.h, z0.h\n"
- ".inst 0x6460e579 // bfmmla z25.s, z11.h, z0.h\n"
- "b 5f\n"
- "4:\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "ld1rqh z2.h, p6/z, [%[a_ptr0]]\n"
- ".inst 0x6464e4d4 // bfmmla z20.s, z6.h, z4.h\n"
- "ld1h z6.h, p7/z, [%[b_ptr0]]\n"
- ".inst 0x6464e4f5 // bfmmla z21.s, z7.h, z4.h\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x6464e516 // bfmmla z22.s, z8.h, z4.h\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x6464e537 // bfmmla z23.s, z9.h, z4.h\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x6464e558 // bfmmla z24.s, z10.h, z4.h\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x6464e579 // bfmmla z25.s, z11.h, z4.h\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "mov z3.h, #0\n"
- "addvl %[b_ptr0], %[b_ptr0], #6\n"
- ".inst 0x6460e4d4 // bfmmla z20.s, z6.h, z0.h\n"
- "addvl %[a_ptr0], %[a_ptr0], #1\n"
- ".inst 0x6460e4f5 // bfmmla z21.s, z7.h, z0.h\n"
- ".inst 0x6460e516 // bfmmla z22.s, z8.h, z0.h\n"
- ".inst 0x6460e537 // bfmmla z23.s, z9.h, z0.h\n"
- ".inst 0x6460e558 // bfmmla z24.s, z10.h, z0.h\n"
- ".inst 0x6460e579 // bfmmla z25.s, z11.h, z0.h\n"
- "trn1 z0.d, z2.d, z3.d\n"
- "cbz %[blocks], 5f\n"
- "trn2 z4.d, z2.d, z3.d\n"
- "ld1h z6.h, p7/z, [%[b_ptr0]]\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x6460e4d4 // bfmmla z20.s, z6.h, z0.h\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x6460e4f5 // bfmmla z21.s, z7.h, z0.h\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x6460e516 // bfmmla z22.s, z8.h, z0.h\n"
- ".inst 0x6460e537 // bfmmla z23.s, z9.h, z0.h\n"
- ".inst 0x6460e558 // bfmmla z24.s, z10.h, z0.h\n"
- ".inst 0x6460e579 // bfmmla z25.s, z11.h, z0.h\n"
- "b.eq 5f\n"
- "ld1h z6.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- ".inst 0x6464e4d4 // bfmmla z20.s, z6.h, z4.h\n"
- ".inst 0x6464e4f5 // bfmmla z21.s, z7.h, z4.h\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- ".inst 0x6464e516 // bfmmla z22.s, z8.h, z4.h\n"
- ".inst 0x6464e537 // bfmmla z23.s, z9.h, z4.h\n"
- ".inst 0x6464e558 // bfmmla z24.s, z10.h, z4.h\n"
- ".inst 0x6464e579 // bfmmla z25.s, z11.h, z4.h\n"
- "5:\n"
- "ld1rw z18.s, p7/z, [%[minptr]]\n"
- "ld1rw z19.s, p7/z, [%[maxptr]]\n"
- "fmax z20.s, p7/m, z20.s, z18.s\n"
- "fmax z21.s, p7/m, z21.s, z18.s\n"
- "fmax z22.s, p7/m, z22.s, z18.s\n"
- "fmax z23.s, p7/m, z23.s, z18.s\n"
- "fmin z20.s, p7/m, z20.s, z19.s\n"
- "fmin z21.s, p7/m, z21.s, z19.s\n"
- "fmin z22.s, p7/m, z22.s, z19.s\n"
- "fmin z23.s, p7/m, z23.s, z19.s\n"
- "fmax z24.s, p7/m, z24.s, z18.s\n"
- "uzp1 z0.s, z20.s, z21.s\n"
- "fmax z25.s, p7/m, z25.s, z18.s\n"
- "uzp1 z1.s, z22.s, z23.s\n"
- "fmin z24.s, p7/m, z24.s, z19.s\n"
- "st1w z0.s, p0, [%[c_ptr0]]\n"
- "fmin z25.s, p7/m, z25.s, z19.s\n"
- "st1w z1.s, p1, [%[c_ptr0], #1, MUL VL]\n"
- "uzp1 z2.s, z24.s, z25.s\n"
- "st1w z2.s, p2, [%[c_ptr0], #2, MUL VL]\n"
- "addvl %[c_ptr0], %[c_ptr0], #3\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks)
- : [width] "r" (width), [accumulate] "r" (static_cast<uint64_t>(accumulate)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers)
- : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
- );
- break;
- case 2:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "c_ptr1 .req X1\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "whilelt p6.h, %[temp], %[leftovers]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "ptrue p7.h\n"
- "whilelt p1.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "whilelt p2.s, %[temp], %[width]\n"
- "cbnz %[accumulate], 1f\n"
- "ld1w z19.s, p0/z, [%[biasptr]]\n"
- "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ld1rqh z1.h, p7/z, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "zip1 z20.s, z19.s, z19.s\n"
- "ld1h z6.h, p7/z, [%[b_ptr0]]\n"
- "zip2 z21.s, z19.s, z19.s\n"
- "ld1w z19.s, p1/z, [%[biasptr], #1, MUL VL]\n"
- "trn1 z4.d, z0.d, z1.d\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "zip1 z22.s, z19.s, z19.s\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "zip2 z23.s, z19.s, z19.s\n"
- "ld1w z19.s, p2/z, [%[biasptr], #2, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #6\n"
- "zip1 z24.s, z19.s, z19.s\n"
- "zip2 z25.s, z19.s, z19.s\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "ld1w z17.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z18.s, p0/z, [c_ptr1]\n"
- "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ld1rqh z1.h, p7/z, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "zip1 z20.s, z17.s, z18.s\n"
- "ld1h z6.h, p7/z, [%[b_ptr0]]\n"
- "zip2 z21.s, z17.s, z18.s\n"
- "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n"
- "trn1 z4.d, z0.d, z1.d\n"
- "ld1w z18.s, p1/z, [c_ptr1, #1, MUL VL]\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "zip1 z22.s, z17.s, z18.s\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "zip2 z23.s, z17.s, z18.s\n"
- "ld1w z17.s, p2/z, [%[c_ptr0], #2, MUL VL]\n"
- "ld1w z18.s, p2/z, [c_ptr1, #2, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #6\n"
- "zip1 z24.s, z17.s, z18.s\n"
- "zip2 z25.s, z17.s, z18.s\n"
- "cbz %[loops], 2f\n"
- "3:\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "ld1rqh z2.h, p7/z, [%[a_ptr0]]\n"
- ".inst 0x6464e4d4 // bfmmla z20.s, z6.h, z4.h\n"
- "ld1h z6.h, p7/z, [%[b_ptr0]]\n"
- ".inst 0x6464e4f5 // bfmmla z21.s, z7.h, z4.h\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x6464e516 // bfmmla z22.s, z8.h, z4.h\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x6464e537 // bfmmla z23.s, z9.h, z4.h\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x6464e558 // bfmmla z24.s, z10.h, z4.h\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x6464e579 // bfmmla z25.s, z11.h, z4.h\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x6460e4d4 // bfmmla z20.s, z6.h, z0.h\n"
- "ld1rqh z3.h, p7/z, [a_ptr1]\n"
- ".inst 0x6460e4f5 // bfmmla z21.s, z7.h, z0.h\n"
- "ld1h z6.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x6460e516 // bfmmla z22.s, z8.h, z0.h\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x6460e537 // bfmmla z23.s, z9.h, z0.h\n"
- "subs %[loops], %[loops], #0x1\n"
- "trn2 z4.d, z2.d, z3.d\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
- ".inst 0x6460e558 // bfmmla z24.s, z10.h, z0.h\n"
- "add a_ptr1, a_ptr1, #0x20\n"
- ".inst 0x6460e579 // bfmmla z25.s, z11.h, z0.h\n"
- "ld1rqh z1.h, p7/z, [a_ptr1, #-0x10]\n"
- "trn1 z0.d, z2.d, z3.d\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- ".inst 0x6460e4d4 // bfmmla z20.s, z6.h, z0.h\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- ".inst 0x6460e4f5 // bfmmla z21.s, z7.h, z0.h\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- ".inst 0x6460e516 // bfmmla z22.s, z8.h, z0.h\n"
- "ld1h z6.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- ".inst 0x6460e537 // bfmmla z23.s, z9.h, z0.h\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- ".inst 0x6460e558 // bfmmla z24.s, z10.h, z0.h\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- ".inst 0x6460e579 // bfmmla z25.s, z11.h, z0.h\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- ".inst 0x6464e4d4 // bfmmla z20.s, z6.h, z4.h\n"
- "ld1h z10.h, p7/z, [%[b_ptr0]]\n"
- ".inst 0x6464e4f5 // bfmmla z21.s, z7.h, z4.h\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x6464e516 // bfmmla z22.s, z8.h, z4.h\n"
- "ld1rqh z0.h, p7/z, [%[a_ptr0], #-0x10]\n"
- ".inst 0x6464e537 // bfmmla z23.s, z9.h, z4.h\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- ".inst 0x6464e558 // bfmmla z24.s, z10.h, z4.h\n"
- ".inst 0x6464e579 // bfmmla z25.s, z11.h, z4.h\n"
- "trn1 z4.d, z0.d, z1.d\n"
- "ld1h z6.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "b.ne 3b\n"
- "2:\n"
- "cbz %[regs], 4f\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "ld1rqh z2.h, p7/z, [%[a_ptr0]]\n"
- ".inst 0x6464e4d4 // bfmmla z20.s, z6.h, z4.h\n"
- "ld1h z6.h, p7/z, [%[b_ptr0]]\n"
- ".inst 0x6464e4f5 // bfmmla z21.s, z7.h, z4.h\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x6464e516 // bfmmla z22.s, z8.h, z4.h\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x6464e537 // bfmmla z23.s, z9.h, z4.h\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x6464e558 // bfmmla z24.s, z10.h, z4.h\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x6464e579 // bfmmla z25.s, z11.h, z4.h\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x6460e4d4 // bfmmla z20.s, z6.h, z0.h\n"
- "ld1rqh z3.h, p7/z, [a_ptr1]\n"
- ".inst 0x6460e4f5 // bfmmla z21.s, z7.h, z0.h\n"
- "ld1h z6.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x6460e516 // bfmmla z22.s, z8.h, z0.h\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x6460e537 // bfmmla z23.s, z9.h, z0.h\n"
- "ld1rqh z1.h, p6/z, [a_ptr1, #0x10]\n"
- "trn2 z4.d, z2.d, z3.d\n"
- "addvl %[b_ptr0], %[b_ptr0], #6\n"
- ".inst 0x6460e558 // bfmmla z24.s, z10.h, z0.h\n"
- "addvl a_ptr1, a_ptr1, #2\n"
- ".inst 0x6460e579 // bfmmla z25.s, z11.h, z0.h\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "trn1 z0.d, z2.d, z3.d\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x6460e4d4 // bfmmla z20.s, z6.h, z0.h\n"
- "ld1h z6.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x6460e4f5 // bfmmla z21.s, z7.h, z0.h\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x6460e516 // bfmmla z22.s, z8.h, z0.h\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- ".inst 0x6460e537 // bfmmla z23.s, z9.h, z0.h\n"
- ".inst 0x6460e558 // bfmmla z24.s, z10.h, z0.h\n"
- ".inst 0x6460e579 // bfmmla z25.s, z11.h, z0.h\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- ".inst 0x6464e4d4 // bfmmla z20.s, z6.h, z4.h\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- ".inst 0x6464e4f5 // bfmmla z21.s, z7.h, z4.h\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #-4\n"
- ".inst 0x6464e516 // bfmmla z22.s, z8.h, z4.h\n"
- "ld1rqh z0.h, p6/z, [%[a_ptr0], #0x10]\n"
- ".inst 0x6464e537 // bfmmla z23.s, z9.h, z4.h\n"
- "addvl %[a_ptr0], %[a_ptr0], #2\n"
- ".inst 0x6464e558 // bfmmla z24.s, z10.h, z4.h\n"
- ".inst 0x6464e579 // bfmmla z25.s, z11.h, z4.h\n"
- "trn1 z4.d, z0.d, z1.d\n"
- "cbz %[blocks], 5f\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "ld1h z6.h, p7/z, [%[b_ptr0]]\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x6464e4d4 // bfmmla z20.s, z6.h, z4.h\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x6464e4f5 // bfmmla z21.s, z7.h, z4.h\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x6464e516 // bfmmla z22.s, z8.h, z4.h\n"
- ".inst 0x6464e537 // bfmmla z23.s, z9.h, z4.h\n"
- ".inst 0x6464e558 // bfmmla z24.s, z10.h, z4.h\n"
- ".inst 0x6464e579 // bfmmla z25.s, z11.h, z4.h\n"
- "b.eq 5f\n"
- "ld1h z6.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #12\n"
- ".inst 0x6460e4d4 // bfmmla z20.s, z6.h, z0.h\n"
- ".inst 0x6460e4f5 // bfmmla z21.s, z7.h, z0.h\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- ".inst 0x6460e516 // bfmmla z22.s, z8.h, z0.h\n"
- ".inst 0x6460e537 // bfmmla z23.s, z9.h, z0.h\n"
- ".inst 0x6460e558 // bfmmla z24.s, z10.h, z0.h\n"
- ".inst 0x6460e579 // bfmmla z25.s, z11.h, z0.h\n"
- "b 5f\n"
- "4:\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "ld1rqh z2.h, p6/z, [%[a_ptr0]]\n"
- ".inst 0x6464e4d4 // bfmmla z20.s, z6.h, z4.h\n"
- "ld1h z6.h, p7/z, [%[b_ptr0]]\n"
- ".inst 0x6464e4f5 // bfmmla z21.s, z7.h, z4.h\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x6464e516 // bfmmla z22.s, z8.h, z4.h\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x6464e537 // bfmmla z23.s, z9.h, z4.h\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x6464e558 // bfmmla z24.s, z10.h, z4.h\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x6464e579 // bfmmla z25.s, z11.h, z4.h\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x6460e4d4 // bfmmla z20.s, z6.h, z0.h\n"
- "ld1rqh z3.h, p6/z, [a_ptr1]\n"
- ".inst 0x6460e4f5 // bfmmla z21.s, z7.h, z0.h\n"
- "addvl %[b_ptr0], %[b_ptr0], #6\n"
- ".inst 0x6460e516 // bfmmla z22.s, z8.h, z0.h\n"
- "addvl %[a_ptr0], %[a_ptr0], #1\n"
- ".inst 0x6460e537 // bfmmla z23.s, z9.h, z0.h\n"
- "addvl a_ptr1, a_ptr1, #1\n"
- ".inst 0x6460e558 // bfmmla z24.s, z10.h, z0.h\n"
- ".inst 0x6460e579 // bfmmla z25.s, z11.h, z0.h\n"
- "trn1 z0.d, z2.d, z3.d\n"
- "cbz %[blocks], 5f\n"
- "trn2 z4.d, z2.d, z3.d\n"
- "ld1h z6.h, p7/z, [%[b_ptr0]]\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x6460e4d4 // bfmmla z20.s, z6.h, z0.h\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x6460e4f5 // bfmmla z21.s, z7.h, z0.h\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x6460e516 // bfmmla z22.s, z8.h, z0.h\n"
- ".inst 0x6460e537 // bfmmla z23.s, z9.h, z0.h\n"
- ".inst 0x6460e558 // bfmmla z24.s, z10.h, z0.h\n"
- ".inst 0x6460e579 // bfmmla z25.s, z11.h, z0.h\n"
- "b.eq 5f\n"
- "ld1h z6.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- ".inst 0x6464e4d4 // bfmmla z20.s, z6.h, z4.h\n"
- ".inst 0x6464e4f5 // bfmmla z21.s, z7.h, z4.h\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- ".inst 0x6464e516 // bfmmla z22.s, z8.h, z4.h\n"
- ".inst 0x6464e537 // bfmmla z23.s, z9.h, z4.h\n"
- ".inst 0x6464e558 // bfmmla z24.s, z10.h, z4.h\n"
- ".inst 0x6464e579 // bfmmla z25.s, z11.h, z4.h\n"
- "5:\n"
- "ld1rw z18.s, p7/z, [%[minptr]]\n"
- "ld1rw z19.s, p7/z, [%[maxptr]]\n"
- "fmax z20.s, p7/m, z20.s, z18.s\n"
- "fmax z21.s, p7/m, z21.s, z18.s\n"
- "fmax z22.s, p7/m, z22.s, z18.s\n"
- "fmax z23.s, p7/m, z23.s, z18.s\n"
- "fmin z20.s, p7/m, z20.s, z19.s\n"
- "fmin z21.s, p7/m, z21.s, z19.s\n"
- "fmin z22.s, p7/m, z22.s, z19.s\n"
- "fmin z23.s, p7/m, z23.s, z19.s\n"
- "fmax z24.s, p7/m, z24.s, z18.s\n"
- "uzp1 z0.s, z20.s, z21.s\n"
- "uzp2 z1.s, z20.s, z21.s\n"
- "uzp1 z2.s, z22.s, z23.s\n"
- "uzp2 z3.s, z22.s, z23.s\n"
- "st1w z0.s, p0, [%[c_ptr0]]\n"
- "fmin z24.s, p7/m, z24.s, z19.s\n"
- "fmax z25.s, p7/m, z25.s, z18.s\n"
- "st1w z1.s, p0, [c_ptr1]\n"
- "fmin z25.s, p7/m, z25.s, z19.s\n"
- "st1w z2.s, p1, [%[c_ptr0], #1, MUL VL]\n"
- "uzp1 z4.s, z24.s, z25.s\n"
- "uzp2 z5.s, z24.s, z25.s\n"
- "st1w z3.s, p1, [c_ptr1, #1, MUL VL]\n"
- "st1w z4.s, p2, [%[c_ptr0], #2, MUL VL]\n"
- "addvl %[c_ptr0], %[c_ptr0], #3\n"
- "st1w z5.s, p2, [c_ptr1, #2, MUL VL]\n"
- ".unreq a_ptr1\n"
- ".unreq c_ptr1\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks)
- : [width] "r" (width), [accumulate] "r" (static_cast<uint64_t>(accumulate)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers)
- : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "cc", "memory"
- );
- break;
- case 3:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "c_ptr1 .req X2\n"
- "c_ptr2 .req X3\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "whilelt p6.h, %[temp], %[leftovers]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "ptrue p7.h\n"
- "whilelt p1.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "whilelt p2.s, %[temp], %[width]\n"
- "cbnz %[accumulate], 1f\n"
- "mov z3.h, #0\n"
- "ld1w z19.s, p0/z, [%[biasptr]]\n"
- "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ld1rqh z1.h, p7/z, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "zip1 z20.s, z19.s, z19.s\n"
- "ld1rqh z2.h, p7/z, [a_ptr2]\n"
- "zip2 z21.s, z19.s, z19.s\n"
- "ld1w z19.s, p1/z, [%[biasptr], #1, MUL VL]\n"
- "trn1 z4.d, z0.d, z1.d\n"
- "ld1h z6.h, p7/z, [%[b_ptr0]]\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "zip1 z22.s, z19.s, z19.s\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "zip2 z23.s, z19.s, z19.s\n"
- "ld1w z19.s, p2/z, [%[biasptr], #2, MUL VL]\n"
- "trn1 z5.d, z2.d, z3.d\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "mov z26.d, z20.d\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "mov z27.d, z21.d\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "zip1 z24.s, z19.s, z19.s\n"
- "addvl %[b_ptr0], %[b_ptr0], #6\n"
- "zip2 z25.s, z19.s, z19.s\n"
- "mov z28.d, z22.d\n"
- "mov z29.d, z23.d\n"
- "mov z30.d, z24.d\n"
- "mov z31.d, z25.d\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "mov z3.h, #0\n"
- "ld1w z17.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z18.s, p0/z, [c_ptr1]\n"
- "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ld1rqh z1.h, p7/z, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "zip1 z20.s, z17.s, z18.s\n"
- "ld1rqh z2.h, p7/z, [a_ptr2]\n"
- "zip2 z21.s, z17.s, z18.s\n"
- "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n"
- "trn1 z4.d, z0.d, z1.d\n"
- "ld1w z18.s, p1/z, [c_ptr1, #1, MUL VL]\n"
- "ld1h z6.h, p7/z, [%[b_ptr0]]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "trn1 z5.d, z2.d, z3.d\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "zip1 z22.s, z17.s, z18.s\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "zip2 z23.s, z17.s, z18.s\n"
- "ld1w z17.s, p2/z, [%[c_ptr0], #2, MUL VL]\n"
- "ld1w z18.s, p2/z, [c_ptr1, #2, MUL VL]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #6\n"
- "zip1 z24.s, z17.s, z18.s\n"
- "zip2 z25.s, z17.s, z18.s\n"
- "ld1w z17.s, p0/z, [c_ptr2]\n"
- "mov z18.s, #0\n"
- "zip1 z26.s, z17.s, z18.s\n"
- "zip2 z27.s, z17.s, z18.s\n"
- "ld1w z17.s, p1/z, [c_ptr2, #1, MUL VL]\n"
- "mov z18.s, #0\n"
- "zip1 z28.s, z17.s, z18.s\n"
- "zip2 z29.s, z17.s, z18.s\n"
- "ld1w z17.s, p2/z, [c_ptr2, #2, MUL VL]\n"
- "mov z18.s, #0\n"
- "zip1 z30.s, z17.s, z18.s\n"
- "zip2 z31.s, z17.s, z18.s\n"
- "cbz %[loops], 2f\n"
- "3:\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "subs %[loops], %[loops], #0x1\n"
- "trn2 z1.d, z2.d, z3.d\n"
- "ld1rqh z2.h, p7/z, [%[a_ptr0]]\n"
- ".inst 0x6464e4d4 // bfmmla z20.s, z6.h, z4.h\n"
- "ld1rqh z3.h, p7/z, [a_ptr1]\n"
- ".inst 0x6464e4f5 // bfmmla z21.s, z7.h, z4.h\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
- ".inst 0x6464e516 // bfmmla z22.s, z8.h, z4.h\n"
- "add a_ptr1, a_ptr1, #0x20\n"
- ".inst 0x6464e537 // bfmmla z23.s, z9.h, z4.h\n"
- ".inst 0x6464e558 // bfmmla z24.s, z10.h, z4.h\n"
- ".inst 0x6464e579 // bfmmla z25.s, z11.h, z4.h\n"
- "ld1rqh z4.h, p7/z, [a_ptr2]\n"
- ".inst 0x6465e4da // bfmmla z26.s, z6.h, z5.h\n"
- "ld1h z6.h, p7/z, [%[b_ptr0]]\n"
- ".inst 0x6465e4fb // bfmmla z27.s, z7.h, z5.h\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x6465e51c // bfmmla z28.s, z8.h, z5.h\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x6465e53d // bfmmla z29.s, z9.h, z5.h\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x6465e55e // bfmmla z30.s, z10.h, z5.h\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x6465e57f // bfmmla z31.s, z11.h, z5.h\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "mov z5.h, #0\n"
- "add a_ptr2, a_ptr2, #0x20\n"
- ".inst 0x6460e4d4 // bfmmla z20.s, z6.h, z0.h\n"
- ".inst 0x6460e4f5 // bfmmla z21.s, z7.h, z0.h\n"
- ".inst 0x6460e516 // bfmmla z22.s, z8.h, z0.h\n"
- ".inst 0x6460e537 // bfmmla z23.s, z9.h, z0.h\n"
- ".inst 0x6460e558 // bfmmla z24.s, z10.h, z0.h\n"
- ".inst 0x6460e579 // bfmmla z25.s, z11.h, z0.h\n"
- "trn1 z0.d, z2.d, z3.d\n"
- ".inst 0x6461e4da // bfmmla z26.s, z6.h, z1.h\n"
- "ld1h z6.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x6461e4fb // bfmmla z27.s, z7.h, z1.h\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x6461e51c // bfmmla z28.s, z8.h, z1.h\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- ".inst 0x6461e53d // bfmmla z29.s, z9.h, z1.h\n"
- ".inst 0x6461e55e // bfmmla z30.s, z10.h, z1.h\n"
- ".inst 0x6461e57f // bfmmla z31.s, z11.h, z1.h\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "trn1 z1.d, z4.d, z5.d\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "trn2 z5.d, z4.d, z5.d\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "trn2 z4.d, z2.d, z3.d\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- ".inst 0x6460e4d4 // bfmmla z20.s, z6.h, z0.h\n"
- "ld1rqh z2.h, p7/z, [a_ptr2, #-0x10]\n"
- ".inst 0x6460e4f5 // bfmmla z21.s, z7.h, z0.h\n"
- ".inst 0x6460e516 // bfmmla z22.s, z8.h, z0.h\n"
- ".inst 0x6460e537 // bfmmla z23.s, z9.h, z0.h\n"
- ".inst 0x6460e558 // bfmmla z24.s, z10.h, z0.h\n"
- ".inst 0x6460e579 // bfmmla z25.s, z11.h, z0.h\n"
- "ld1rqh z0.h, p7/z, [%[a_ptr0], #-0x10]\n"
- ".inst 0x6461e4da // bfmmla z26.s, z6.h, z1.h\n"
- "ld1h z6.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- ".inst 0x6461e4fb // bfmmla z27.s, z7.h, z1.h\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- ".inst 0x6461e51c // bfmmla z28.s, z8.h, z1.h\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- ".inst 0x6461e53d // bfmmla z29.s, z9.h, z1.h\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- ".inst 0x6461e55e // bfmmla z30.s, z10.h, z1.h\n"
- "ld1h z10.h, p7/z, [%[b_ptr0]]\n"
- ".inst 0x6461e57f // bfmmla z31.s, z11.h, z1.h\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "mov z3.h, #0\n"
- "ld1rqh z1.h, p7/z, [a_ptr1, #-0x10]\n"
- ".inst 0x6464e4d4 // bfmmla z20.s, z6.h, z4.h\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- ".inst 0x6464e4f5 // bfmmla z21.s, z7.h, z4.h\n"
- ".inst 0x6464e516 // bfmmla z22.s, z8.h, z4.h\n"
- ".inst 0x6464e537 // bfmmla z23.s, z9.h, z4.h\n"
- ".inst 0x6464e558 // bfmmla z24.s, z10.h, z4.h\n"
- ".inst 0x6464e579 // bfmmla z25.s, z11.h, z4.h\n"
- "trn1 z4.d, z0.d, z1.d\n"
- ".inst 0x6465e4da // bfmmla z26.s, z6.h, z5.h\n"
- "ld1h z6.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- ".inst 0x6465e4fb // bfmmla z27.s, z7.h, z5.h\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- ".inst 0x6465e51c // bfmmla z28.s, z8.h, z5.h\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- ".inst 0x6465e53d // bfmmla z29.s, z9.h, z5.h\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- ".inst 0x6465e55e // bfmmla z30.s, z10.h, z5.h\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- ".inst 0x6465e57f // bfmmla z31.s, z11.h, z5.h\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "trn1 z5.d, z2.d, z3.d\n"
- "b.ne 3b\n"
- "2:\n"
- "cbz %[regs], 4f\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "trn2 z1.d, z2.d, z3.d\n"
- "ld1rqh z2.h, p7/z, [%[a_ptr0]]\n"
- ".inst 0x6464e4d4 // bfmmla z20.s, z6.h, z4.h\n"
- "ld1rqh z3.h, p7/z, [a_ptr1]\n"
- ".inst 0x6464e4f5 // bfmmla z21.s, z7.h, z4.h\n"
- ".inst 0x6464e516 // bfmmla z22.s, z8.h, z4.h\n"
- ".inst 0x6464e537 // bfmmla z23.s, z9.h, z4.h\n"
- ".inst 0x6464e558 // bfmmla z24.s, z10.h, z4.h\n"
- ".inst 0x6464e579 // bfmmla z25.s, z11.h, z4.h\n"
- "ld1rqh z4.h, p7/z, [a_ptr2]\n"
- ".inst 0x6465e4da // bfmmla z26.s, z6.h, z5.h\n"
- "ld1h z6.h, p7/z, [%[b_ptr0]]\n"
- ".inst 0x6465e4fb // bfmmla z27.s, z7.h, z5.h\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x6465e51c // bfmmla z28.s, z8.h, z5.h\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x6465e53d // bfmmla z29.s, z9.h, z5.h\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x6465e55e // bfmmla z30.s, z10.h, z5.h\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x6465e57f // bfmmla z31.s, z11.h, z5.h\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "mov z5.h, #0\n"
- ".inst 0x6460e4d4 // bfmmla z20.s, z6.h, z0.h\n"
- ".inst 0x6460e4f5 // bfmmla z21.s, z7.h, z0.h\n"
- ".inst 0x6460e516 // bfmmla z22.s, z8.h, z0.h\n"
- ".inst 0x6460e537 // bfmmla z23.s, z9.h, z0.h\n"
- ".inst 0x6460e558 // bfmmla z24.s, z10.h, z0.h\n"
- ".inst 0x6460e579 // bfmmla z25.s, z11.h, z0.h\n"
- "trn1 z0.d, z2.d, z3.d\n"
- ".inst 0x6461e4da // bfmmla z26.s, z6.h, z1.h\n"
- "ld1h z6.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x6461e4fb // bfmmla z27.s, z7.h, z1.h\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x6461e51c // bfmmla z28.s, z8.h, z1.h\n"
- "addvl %[b_ptr0], %[b_ptr0], #6\n"
- ".inst 0x6461e53d // bfmmla z29.s, z9.h, z1.h\n"
- ".inst 0x6461e55e // bfmmla z30.s, z10.h, z1.h\n"
- ".inst 0x6461e57f // bfmmla z31.s, z11.h, z1.h\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "trn1 z1.d, z4.d, z5.d\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "trn2 z5.d, z4.d, z5.d\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "trn2 z4.d, z2.d, z3.d\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x6460e4d4 // bfmmla z20.s, z6.h, z0.h\n"
- "ld1rqh z2.h, p6/z, [a_ptr2, #0x10]\n"
- ".inst 0x6460e4f5 // bfmmla z21.s, z7.h, z0.h\n"
- "addvl a_ptr2, a_ptr2, #2\n"
- ".inst 0x6460e516 // bfmmla z22.s, z8.h, z0.h\n"
- ".inst 0x6460e537 // bfmmla z23.s, z9.h, z0.h\n"
- ".inst 0x6460e558 // bfmmla z24.s, z10.h, z0.h\n"
- ".inst 0x6460e579 // bfmmla z25.s, z11.h, z0.h\n"
- "ld1rqh z0.h, p6/z, [%[a_ptr0], #0x10]\n"
- ".inst 0x6461e4da // bfmmla z26.s, z6.h, z1.h\n"
- "ld1h z6.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x6461e4fb // bfmmla z27.s, z7.h, z1.h\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x6461e51c // bfmmla z28.s, z8.h, z1.h\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- ".inst 0x6461e53d // bfmmla z29.s, z9.h, z1.h\n"
- "addvl %[a_ptr0], %[a_ptr0], #2\n"
- ".inst 0x6461e55e // bfmmla z30.s, z10.h, z1.h\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- ".inst 0x6461e57f // bfmmla z31.s, z11.h, z1.h\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "mov z3.h, #0\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- ".inst 0x6464e4d4 // bfmmla z20.s, z6.h, z4.h\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- ".inst 0x6464e4f5 // bfmmla z21.s, z7.h, z4.h\n"
- "ld1rqh z1.h, p6/z, [a_ptr1, #0x10]\n"
- ".inst 0x6464e516 // bfmmla z22.s, z8.h, z4.h\n"
- "addvl %[b_ptr0], %[b_ptr0], #-4\n"
- ".inst 0x6464e537 // bfmmla z23.s, z9.h, z4.h\n"
- "addvl a_ptr1, a_ptr1, #2\n"
- ".inst 0x6464e558 // bfmmla z24.s, z10.h, z4.h\n"
- ".inst 0x6464e579 // bfmmla z25.s, z11.h, z4.h\n"
- "trn1 z4.d, z0.d, z1.d\n"
- ".inst 0x6465e4da // bfmmla z26.s, z6.h, z5.h\n"
- ".inst 0x6465e4fb // bfmmla z27.s, z7.h, z5.h\n"
- ".inst 0x6465e51c // bfmmla z28.s, z8.h, z5.h\n"
- ".inst 0x6465e53d // bfmmla z29.s, z9.h, z5.h\n"
- ".inst 0x6465e55e // bfmmla z30.s, z10.h, z5.h\n"
- ".inst 0x6465e57f // bfmmla z31.s, z11.h, z5.h\n"
- "trn1 z5.d, z2.d, z3.d\n"
- "cbz %[blocks], 5f\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "ld1h z6.h, p7/z, [%[b_ptr0]]\n"
- "trn2 z1.d, z2.d, z3.d\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- ".inst 0x6464e4d4 // bfmmla z20.s, z6.h, z4.h\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x6464e4f5 // bfmmla z21.s, z7.h, z4.h\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x6464e516 // bfmmla z22.s, z8.h, z4.h\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x6465e4da // bfmmla z26.s, z6.h, z5.h\n"
- ".inst 0x6464e537 // bfmmla z23.s, z9.h, z4.h\n"
- ".inst 0x6464e558 // bfmmla z24.s, z10.h, z4.h\n"
- ".inst 0x6464e579 // bfmmla z25.s, z11.h, z4.h\n"
- ".inst 0x6465e4fb // bfmmla z27.s, z7.h, z5.h\n"
- ".inst 0x6465e51c // bfmmla z28.s, z8.h, z5.h\n"
- ".inst 0x6465e53d // bfmmla z29.s, z9.h, z5.h\n"
- ".inst 0x6465e55e // bfmmla z30.s, z10.h, z5.h\n"
- ".inst 0x6465e57f // bfmmla z31.s, z11.h, z5.h\n"
- "b.eq 5f\n"
- "ld1h z6.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #12\n"
- ".inst 0x6460e4d4 // bfmmla z20.s, z6.h, z0.h\n"
- ".inst 0x6460e4f5 // bfmmla z21.s, z7.h, z0.h\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- ".inst 0x6461e4da // bfmmla z26.s, z6.h, z1.h\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- ".inst 0x6461e4fb // bfmmla z27.s, z7.h, z1.h\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- ".inst 0x6460e516 // bfmmla z22.s, z8.h, z0.h\n"
- ".inst 0x6460e537 // bfmmla z23.s, z9.h, z0.h\n"
- ".inst 0x6460e558 // bfmmla z24.s, z10.h, z0.h\n"
- ".inst 0x6460e579 // bfmmla z25.s, z11.h, z0.h\n"
- ".inst 0x6461e51c // bfmmla z28.s, z8.h, z1.h\n"
- ".inst 0x6461e53d // bfmmla z29.s, z9.h, z1.h\n"
- ".inst 0x6461e55e // bfmmla z30.s, z10.h, z1.h\n"
- ".inst 0x6461e57f // bfmmla z31.s, z11.h, z1.h\n"
- "b 5f\n"
- "4:\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "trn2 z1.d, z2.d, z3.d\n"
- "ld1rqh z2.h, p6/z, [%[a_ptr0]]\n"
- ".inst 0x6464e4d4 // bfmmla z20.s, z6.h, z4.h\n"
- "ld1rqh z3.h, p6/z, [a_ptr1]\n"
- ".inst 0x6464e4f5 // bfmmla z21.s, z7.h, z4.h\n"
- "addvl %[a_ptr0], %[a_ptr0], #1\n"
- ".inst 0x6464e516 // bfmmla z22.s, z8.h, z4.h\n"
- "addvl a_ptr1, a_ptr1, #1\n"
- ".inst 0x6464e537 // bfmmla z23.s, z9.h, z4.h\n"
- ".inst 0x6464e558 // bfmmla z24.s, z10.h, z4.h\n"
- ".inst 0x6464e579 // bfmmla z25.s, z11.h, z4.h\n"
- "ld1rqh z4.h, p6/z, [a_ptr2]\n"
- ".inst 0x6465e4da // bfmmla z26.s, z6.h, z5.h\n"
- "ld1h z6.h, p7/z, [%[b_ptr0]]\n"
- ".inst 0x6465e4fb // bfmmla z27.s, z7.h, z5.h\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x6465e51c // bfmmla z28.s, z8.h, z5.h\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x6465e53d // bfmmla z29.s, z9.h, z5.h\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x6465e55e // bfmmla z30.s, z10.h, z5.h\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x6465e57f // bfmmla z31.s, z11.h, z5.h\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "mov z5.h, #0\n"
- "addvl %[b_ptr0], %[b_ptr0], #6\n"
- ".inst 0x6460e4d4 // bfmmla z20.s, z6.h, z0.h\n"
- "addvl a_ptr2, a_ptr2, #1\n"
- ".inst 0x6460e4f5 // bfmmla z21.s, z7.h, z0.h\n"
- ".inst 0x6460e516 // bfmmla z22.s, z8.h, z0.h\n"
- ".inst 0x6460e537 // bfmmla z23.s, z9.h, z0.h\n"
- ".inst 0x6460e558 // bfmmla z24.s, z10.h, z0.h\n"
- ".inst 0x6460e579 // bfmmla z25.s, z11.h, z0.h\n"
- "trn1 z0.d, z2.d, z3.d\n"
- ".inst 0x6461e4da // bfmmla z26.s, z6.h, z1.h\n"
- ".inst 0x6461e4fb // bfmmla z27.s, z7.h, z1.h\n"
- ".inst 0x6461e51c // bfmmla z28.s, z8.h, z1.h\n"
- ".inst 0x6461e53d // bfmmla z29.s, z9.h, z1.h\n"
- ".inst 0x6461e55e // bfmmla z30.s, z10.h, z1.h\n"
- ".inst 0x6461e57f // bfmmla z31.s, z11.h, z1.h\n"
- "trn1 z1.d, z4.d, z5.d\n"
- "cbz %[blocks], 5f\n"
- "trn2 z5.d, z4.d, z5.d\n"
- "ld1h z6.h, p7/z, [%[b_ptr0]]\n"
- "trn2 z4.d, z2.d, z3.d\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- ".inst 0x6460e4d4 // bfmmla z20.s, z6.h, z0.h\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x6460e4f5 // bfmmla z21.s, z7.h, z0.h\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x6460e516 // bfmmla z22.s, z8.h, z0.h\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x6461e4da // bfmmla z26.s, z6.h, z1.h\n"
- ".inst 0x6460e537 // bfmmla z23.s, z9.h, z0.h\n"
- ".inst 0x6460e558 // bfmmla z24.s, z10.h, z0.h\n"
- ".inst 0x6460e579 // bfmmla z25.s, z11.h, z0.h\n"
- ".inst 0x6461e4fb // bfmmla z27.s, z7.h, z1.h\n"
- ".inst 0x6461e51c // bfmmla z28.s, z8.h, z1.h\n"
- ".inst 0x6461e53d // bfmmla z29.s, z9.h, z1.h\n"
- ".inst 0x6461e55e // bfmmla z30.s, z10.h, z1.h\n"
- ".inst 0x6461e57f // bfmmla z31.s, z11.h, z1.h\n"
- "b.eq 5f\n"
- "ld1h z6.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- ".inst 0x6464e4d4 // bfmmla z20.s, z6.h, z4.h\n"
- ".inst 0x6464e4f5 // bfmmla z21.s, z7.h, z4.h\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- ".inst 0x6465e4da // bfmmla z26.s, z6.h, z5.h\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- ".inst 0x6465e4fb // bfmmla z27.s, z7.h, z5.h\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- ".inst 0x6464e516 // bfmmla z22.s, z8.h, z4.h\n"
- ".inst 0x6464e537 // bfmmla z23.s, z9.h, z4.h\n"
- ".inst 0x6464e558 // bfmmla z24.s, z10.h, z4.h\n"
- ".inst 0x6464e579 // bfmmla z25.s, z11.h, z4.h\n"
- ".inst 0x6465e51c // bfmmla z28.s, z8.h, z5.h\n"
- ".inst 0x6465e53d // bfmmla z29.s, z9.h, z5.h\n"
- ".inst 0x6465e55e // bfmmla z30.s, z10.h, z5.h\n"
- ".inst 0x6465e57f // bfmmla z31.s, z11.h, z5.h\n"
- "5:\n"
- "ld1rw z18.s, p7/z, [%[minptr]]\n"
- "ld1rw z19.s, p7/z, [%[maxptr]]\n"
- "fmax z20.s, p7/m, z20.s, z18.s\n"
- "fmax z21.s, p7/m, z21.s, z18.s\n"
- "fmax z22.s, p7/m, z22.s, z18.s\n"
- "fmax z23.s, p7/m, z23.s, z18.s\n"
- "fmin z20.s, p7/m, z20.s, z19.s\n"
- "fmin z21.s, p7/m, z21.s, z19.s\n"
- "fmin z22.s, p7/m, z22.s, z19.s\n"
- "fmin z23.s, p7/m, z23.s, z19.s\n"
- "fmax z24.s, p7/m, z24.s, z18.s\n"
- "uzp1 z0.s, z20.s, z21.s\n"
- "uzp2 z1.s, z20.s, z21.s\n"
- "uzp1 z2.s, z22.s, z23.s\n"
- "uzp2 z3.s, z22.s, z23.s\n"
- "st1w z0.s, p0, [%[c_ptr0]]\n"
- "fmin z24.s, p7/m, z24.s, z19.s\n"
- "fmax z25.s, p7/m, z25.s, z18.s\n"
- "fmax z26.s, p7/m, z26.s, z18.s\n"
- "st1w z1.s, p0, [c_ptr1]\n"
- "fmax z27.s, p7/m, z27.s, z18.s\n"
- "fmax z28.s, p7/m, z28.s, z18.s\n"
- "fmin z25.s, p7/m, z25.s, z19.s\n"
- "st1w z2.s, p1, [%[c_ptr0], #1, MUL VL]\n"
- "fmin z26.s, p7/m, z26.s, z19.s\n"
- "fmin z27.s, p7/m, z27.s, z19.s\n"
- "fmin z28.s, p7/m, z28.s, z19.s\n"
- "st1w z3.s, p1, [c_ptr1, #1, MUL VL]\n"
- "uzp1 z4.s, z24.s, z25.s\n"
- "uzp2 z5.s, z24.s, z25.s\n"
- "uzp1 z6.s, z26.s, z27.s\n"
- "fmax z29.s, p7/m, z29.s, z18.s\n"
- "st1w z4.s, p2, [%[c_ptr0], #2, MUL VL]\n"
- "fmax z30.s, p7/m, z30.s, z18.s\n"
- "addvl %[c_ptr0], %[c_ptr0], #3\n"
- "fmax z31.s, p7/m, z31.s, z18.s\n"
- "st1w z5.s, p2, [c_ptr1, #2, MUL VL]\n"
- "fmin z29.s, p7/m, z29.s, z19.s\n"
- "fmin z30.s, p7/m, z30.s, z19.s\n"
- "fmin z31.s, p7/m, z31.s, z19.s\n"
- "st1w z6.s, p0, [c_ptr2]\n"
- "uzp1 z7.s, z28.s, z29.s\n"
- "uzp1 z8.s, z30.s, z31.s\n"
- "st1w z7.s, p1, [c_ptr2, #1, MUL VL]\n"
- "st1w z8.s, p2, [c_ptr2, #2, MUL VL]\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks)
- : [width] "r" (width), [accumulate] "r" (static_cast<uint64_t>(accumulate)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers)
- : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "cc", "memory"
- );
- break;
- default:
- case 4:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "a_ptr3 .req X2\n"
- "c_ptr1 .req X3\n"
- "c_ptr2 .req X4\n"
- "c_ptr3 .req X5\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "add a_ptr3, a_ptr2, %[lda]\n"
- "add c_ptr3, c_ptr2, %[ldc]\n"
- "whilelt p6.h, %[temp], %[leftovers]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "ptrue p7.h\n"
- "whilelt p1.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "whilelt p2.s, %[temp], %[width]\n"
- "cbnz %[accumulate], 1f\n"
- "ld1w z19.s, p0/z, [%[biasptr]]\n"
- "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ld1rqh z1.h, p7/z, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "zip1 z20.s, z19.s, z19.s\n"
- "ld1rqh z2.h, p7/z, [a_ptr2]\n"
- "zip2 z21.s, z19.s, z19.s\n"
- "ld1w z19.s, p1/z, [%[biasptr], #1, MUL VL]\n"
- "trn1 z4.d, z0.d, z1.d\n"
- "ld1rqh z3.h, p7/z, [a_ptr3]\n"
- "ld1h z6.h, p7/z, [%[b_ptr0]]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "zip1 z22.s, z19.s, z19.s\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "zip2 z23.s, z19.s, z19.s\n"
- "ld1w z19.s, p2/z, [%[biasptr], #2, MUL VL]\n"
- "trn1 z5.d, z2.d, z3.d\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "mov z26.d, z20.d\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "mov z27.d, z21.d\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "zip1 z24.s, z19.s, z19.s\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "zip2 z25.s, z19.s, z19.s\n"
- "add a_ptr3, a_ptr3, #0x10\n"
- "mov z28.d, z22.d\n"
- "addvl %[b_ptr0], %[b_ptr0], #6\n"
- "mov z29.d, z23.d\n"
- "mov z30.d, z24.d\n"
- "mov z31.d, z25.d\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "ld1w z17.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z18.s, p0/z, [c_ptr1]\n"
- "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ld1rqh z1.h, p7/z, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "zip1 z20.s, z17.s, z18.s\n"
- "ld1rqh z2.h, p7/z, [a_ptr2]\n"
- "zip2 z21.s, z17.s, z18.s\n"
- "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n"
- "trn1 z4.d, z0.d, z1.d\n"
- "ld1w z18.s, p1/z, [c_ptr1, #1, MUL VL]\n"
- "ld1rqh z3.h, p7/z, [a_ptr3]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "ld1h z6.h, p7/z, [%[b_ptr0]]\n"
- "add a_ptr3, a_ptr3, #0x10\n"
- "zip1 z22.s, z17.s, z18.s\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "zip2 z23.s, z17.s, z18.s\n"
- "ld1w z17.s, p2/z, [%[c_ptr0], #2, MUL VL]\n"
- "trn1 z5.d, z2.d, z3.d\n"
- "ld1w z18.s, p2/z, [c_ptr1, #2, MUL VL]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "zip1 z24.s, z17.s, z18.s\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "zip2 z25.s, z17.s, z18.s\n"
- "ld1w z17.s, p0/z, [c_ptr2]\n"
- "ld1w z18.s, p0/z, [c_ptr3]\n"
- "addvl %[b_ptr0], %[b_ptr0], #6\n"
- "zip1 z26.s, z17.s, z18.s\n"
- "zip2 z27.s, z17.s, z18.s\n"
- "ld1w z17.s, p1/z, [c_ptr2, #1, MUL VL]\n"
- "ld1w z18.s, p1/z, [c_ptr3, #1, MUL VL]\n"
- "zip1 z28.s, z17.s, z18.s\n"
- "zip2 z29.s, z17.s, z18.s\n"
- "ld1w z17.s, p2/z, [c_ptr2, #2, MUL VL]\n"
- "ld1w z18.s, p2/z, [c_ptr3, #2, MUL VL]\n"
- "zip1 z30.s, z17.s, z18.s\n"
- "zip2 z31.s, z17.s, z18.s\n"
- "cbz %[loops], 2f\n"
- "3:\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "subs %[loops], %[loops], #0x1\n"
- "trn2 z1.d, z2.d, z3.d\n"
- "ld1rqh z2.h, p7/z, [%[a_ptr0]]\n"
- ".inst 0x6464e4d4 // bfmmla z20.s, z6.h, z4.h\n"
- "ld1rqh z3.h, p7/z, [a_ptr1]\n"
- ".inst 0x6464e4f5 // bfmmla z21.s, z7.h, z4.h\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
- ".inst 0x6464e516 // bfmmla z22.s, z8.h, z4.h\n"
- "add a_ptr1, a_ptr1, #0x20\n"
- ".inst 0x6464e537 // bfmmla z23.s, z9.h, z4.h\n"
- ".inst 0x6464e558 // bfmmla z24.s, z10.h, z4.h\n"
- ".inst 0x6464e579 // bfmmla z25.s, z11.h, z4.h\n"
- "ld1rqh z4.h, p7/z, [a_ptr2]\n"
- ".inst 0x6465e4da // bfmmla z26.s, z6.h, z5.h\n"
- "ld1h z6.h, p7/z, [%[b_ptr0]]\n"
- ".inst 0x6465e4fb // bfmmla z27.s, z7.h, z5.h\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x6465e51c // bfmmla z28.s, z8.h, z5.h\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x6465e53d // bfmmla z29.s, z9.h, z5.h\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x6465e55e // bfmmla z30.s, z10.h, z5.h\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x6465e57f // bfmmla z31.s, z11.h, z5.h\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x6460e4d4 // bfmmla z20.s, z6.h, z0.h\n"
- "ld1rqh z5.h, p7/z, [a_ptr3]\n"
- ".inst 0x6460e4f5 // bfmmla z21.s, z7.h, z0.h\n"
- "add a_ptr2, a_ptr2, #0x20\n"
- ".inst 0x6460e516 // bfmmla z22.s, z8.h, z0.h\n"
- "add a_ptr3, a_ptr3, #0x20\n"
- ".inst 0x6460e537 // bfmmla z23.s, z9.h, z0.h\n"
- ".inst 0x6460e558 // bfmmla z24.s, z10.h, z0.h\n"
- ".inst 0x6460e579 // bfmmla z25.s, z11.h, z0.h\n"
- "trn1 z0.d, z2.d, z3.d\n"
- ".inst 0x6461e4da // bfmmla z26.s, z6.h, z1.h\n"
- "ld1h z6.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x6461e4fb // bfmmla z27.s, z7.h, z1.h\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x6461e51c // bfmmla z28.s, z8.h, z1.h\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- ".inst 0x6461e53d // bfmmla z29.s, z9.h, z1.h\n"
- ".inst 0x6461e55e // bfmmla z30.s, z10.h, z1.h\n"
- ".inst 0x6461e57f // bfmmla z31.s, z11.h, z1.h\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "trn1 z1.d, z4.d, z5.d\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "trn2 z5.d, z4.d, z5.d\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "trn2 z4.d, z2.d, z3.d\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- ".inst 0x6460e4d4 // bfmmla z20.s, z6.h, z0.h\n"
- "ld1rqh z2.h, p7/z, [a_ptr2, #-0x10]\n"
- ".inst 0x6460e4f5 // bfmmla z21.s, z7.h, z0.h\n"
- "ld1rqh z3.h, p7/z, [a_ptr3, #-0x10]\n"
- ".inst 0x6460e516 // bfmmla z22.s, z8.h, z0.h\n"
- ".inst 0x6460e537 // bfmmla z23.s, z9.h, z0.h\n"
- ".inst 0x6460e558 // bfmmla z24.s, z10.h, z0.h\n"
- ".inst 0x6460e579 // bfmmla z25.s, z11.h, z0.h\n"
- "ld1rqh z0.h, p7/z, [%[a_ptr0], #-0x10]\n"
- ".inst 0x6461e4da // bfmmla z26.s, z6.h, z1.h\n"
- "ld1h z6.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- ".inst 0x6461e4fb // bfmmla z27.s, z7.h, z1.h\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- ".inst 0x6461e51c // bfmmla z28.s, z8.h, z1.h\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- ".inst 0x6461e53d // bfmmla z29.s, z9.h, z1.h\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- ".inst 0x6461e55e // bfmmla z30.s, z10.h, z1.h\n"
- "ld1h z10.h, p7/z, [%[b_ptr0]]\n"
- ".inst 0x6461e57f // bfmmla z31.s, z11.h, z1.h\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x6464e4d4 // bfmmla z20.s, z6.h, z4.h\n"
- "ld1rqh z1.h, p7/z, [a_ptr1, #-0x10]\n"
- ".inst 0x6464e4f5 // bfmmla z21.s, z7.h, z4.h\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- ".inst 0x6464e516 // bfmmla z22.s, z8.h, z4.h\n"
- ".inst 0x6464e537 // bfmmla z23.s, z9.h, z4.h\n"
- ".inst 0x6464e558 // bfmmla z24.s, z10.h, z4.h\n"
- ".inst 0x6464e579 // bfmmla z25.s, z11.h, z4.h\n"
- "trn1 z4.d, z0.d, z1.d\n"
- ".inst 0x6465e4da // bfmmla z26.s, z6.h, z5.h\n"
- "ld1h z6.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- ".inst 0x6465e4fb // bfmmla z27.s, z7.h, z5.h\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- ".inst 0x6465e51c // bfmmla z28.s, z8.h, z5.h\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- ".inst 0x6465e53d // bfmmla z29.s, z9.h, z5.h\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- ".inst 0x6465e55e // bfmmla z30.s, z10.h, z5.h\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- ".inst 0x6465e57f // bfmmla z31.s, z11.h, z5.h\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "trn1 z5.d, z2.d, z3.d\n"
- "b.ne 3b\n"
- "2:\n"
- "cbz %[regs], 4f\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "trn2 z1.d, z2.d, z3.d\n"
- "ld1rqh z2.h, p7/z, [%[a_ptr0]]\n"
- ".inst 0x6464e4d4 // bfmmla z20.s, z6.h, z4.h\n"
- "ld1rqh z3.h, p7/z, [a_ptr1]\n"
- ".inst 0x6464e4f5 // bfmmla z21.s, z7.h, z4.h\n"
- ".inst 0x6464e516 // bfmmla z22.s, z8.h, z4.h\n"
- ".inst 0x6464e537 // bfmmla z23.s, z9.h, z4.h\n"
- ".inst 0x6464e558 // bfmmla z24.s, z10.h, z4.h\n"
- ".inst 0x6464e579 // bfmmla z25.s, z11.h, z4.h\n"
- "ld1rqh z4.h, p7/z, [a_ptr2]\n"
- ".inst 0x6465e4da // bfmmla z26.s, z6.h, z5.h\n"
- "ld1h z6.h, p7/z, [%[b_ptr0]]\n"
- ".inst 0x6465e4fb // bfmmla z27.s, z7.h, z5.h\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x6465e51c // bfmmla z28.s, z8.h, z5.h\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x6465e53d // bfmmla z29.s, z9.h, z5.h\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x6465e55e // bfmmla z30.s, z10.h, z5.h\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x6465e57f // bfmmla z31.s, z11.h, z5.h\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x6460e4d4 // bfmmla z20.s, z6.h, z0.h\n"
- "ld1rqh z5.h, p7/z, [a_ptr3]\n"
- ".inst 0x6460e4f5 // bfmmla z21.s, z7.h, z0.h\n"
- ".inst 0x6460e516 // bfmmla z22.s, z8.h, z0.h\n"
- ".inst 0x6460e537 // bfmmla z23.s, z9.h, z0.h\n"
- ".inst 0x6460e558 // bfmmla z24.s, z10.h, z0.h\n"
- ".inst 0x6460e579 // bfmmla z25.s, z11.h, z0.h\n"
- "trn1 z0.d, z2.d, z3.d\n"
- ".inst 0x6461e4da // bfmmla z26.s, z6.h, z1.h\n"
- "ld1h z6.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x6461e4fb // bfmmla z27.s, z7.h, z1.h\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x6461e51c // bfmmla z28.s, z8.h, z1.h\n"
- "addvl %[b_ptr0], %[b_ptr0], #6\n"
- ".inst 0x6461e53d // bfmmla z29.s, z9.h, z1.h\n"
- ".inst 0x6461e55e // bfmmla z30.s, z10.h, z1.h\n"
- ".inst 0x6461e57f // bfmmla z31.s, z11.h, z1.h\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "trn1 z1.d, z4.d, z5.d\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "trn2 z5.d, z4.d, z5.d\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "trn2 z4.d, z2.d, z3.d\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x6460e4d4 // bfmmla z20.s, z6.h, z0.h\n"
- "ld1rqh z2.h, p6/z, [a_ptr2, #0x10]\n"
- ".inst 0x6460e4f5 // bfmmla z21.s, z7.h, z0.h\n"
- "ld1rqh z3.h, p6/z, [a_ptr3, #0x10]\n"
- ".inst 0x6460e516 // bfmmla z22.s, z8.h, z0.h\n"
- "addvl a_ptr2, a_ptr2, #2\n"
- ".inst 0x6460e537 // bfmmla z23.s, z9.h, z0.h\n"
- "addvl a_ptr3, a_ptr3, #2\n"
- ".inst 0x6460e558 // bfmmla z24.s, z10.h, z0.h\n"
- ".inst 0x6460e579 // bfmmla z25.s, z11.h, z0.h\n"
- "ld1rqh z0.h, p6/z, [%[a_ptr0], #0x10]\n"
- ".inst 0x6461e4da // bfmmla z26.s, z6.h, z1.h\n"
- "ld1h z6.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x6461e4fb // bfmmla z27.s, z7.h, z1.h\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x6461e51c // bfmmla z28.s, z8.h, z1.h\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- ".inst 0x6461e53d // bfmmla z29.s, z9.h, z1.h\n"
- "addvl %[a_ptr0], %[a_ptr0], #2\n"
- ".inst 0x6461e55e // bfmmla z30.s, z10.h, z1.h\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- ".inst 0x6461e57f // bfmmla z31.s, z11.h, z1.h\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- ".inst 0x6464e4d4 // bfmmla z20.s, z6.h, z4.h\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- ".inst 0x6464e4f5 // bfmmla z21.s, z7.h, z4.h\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- ".inst 0x6464e516 // bfmmla z22.s, z8.h, z4.h\n"
- "ld1rqh z1.h, p6/z, [a_ptr1, #0x10]\n"
- ".inst 0x6464e537 // bfmmla z23.s, z9.h, z4.h\n"
- "addvl %[b_ptr0], %[b_ptr0], #-4\n"
- ".inst 0x6464e558 // bfmmla z24.s, z10.h, z4.h\n"
- "addvl a_ptr1, a_ptr1, #2\n"
- ".inst 0x6464e579 // bfmmla z25.s, z11.h, z4.h\n"
- "trn1 z4.d, z0.d, z1.d\n"
- ".inst 0x6465e4da // bfmmla z26.s, z6.h, z5.h\n"
- ".inst 0x6465e4fb // bfmmla z27.s, z7.h, z5.h\n"
- ".inst 0x6465e51c // bfmmla z28.s, z8.h, z5.h\n"
- ".inst 0x6465e53d // bfmmla z29.s, z9.h, z5.h\n"
- ".inst 0x6465e55e // bfmmla z30.s, z10.h, z5.h\n"
- ".inst 0x6465e57f // bfmmla z31.s, z11.h, z5.h\n"
- "trn1 z5.d, z2.d, z3.d\n"
- "cbz %[blocks], 5f\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "ld1h z6.h, p7/z, [%[b_ptr0]]\n"
- "trn2 z1.d, z2.d, z3.d\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- ".inst 0x6464e4d4 // bfmmla z20.s, z6.h, z4.h\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x6464e4f5 // bfmmla z21.s, z7.h, z4.h\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x6464e516 // bfmmla z22.s, z8.h, z4.h\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x6465e4da // bfmmla z26.s, z6.h, z5.h\n"
- ".inst 0x6464e537 // bfmmla z23.s, z9.h, z4.h\n"
- ".inst 0x6464e558 // bfmmla z24.s, z10.h, z4.h\n"
- ".inst 0x6464e579 // bfmmla z25.s, z11.h, z4.h\n"
- ".inst 0x6465e4fb // bfmmla z27.s, z7.h, z5.h\n"
- ".inst 0x6465e51c // bfmmla z28.s, z8.h, z5.h\n"
- ".inst 0x6465e53d // bfmmla z29.s, z9.h, z5.h\n"
- ".inst 0x6465e55e // bfmmla z30.s, z10.h, z5.h\n"
- ".inst 0x6465e57f // bfmmla z31.s, z11.h, z5.h\n"
- "b.eq 5f\n"
- "ld1h z6.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #12\n"
- ".inst 0x6460e4d4 // bfmmla z20.s, z6.h, z0.h\n"
- ".inst 0x6460e4f5 // bfmmla z21.s, z7.h, z0.h\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- ".inst 0x6461e4da // bfmmla z26.s, z6.h, z1.h\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- ".inst 0x6461e4fb // bfmmla z27.s, z7.h, z1.h\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- ".inst 0x6460e516 // bfmmla z22.s, z8.h, z0.h\n"
- ".inst 0x6460e537 // bfmmla z23.s, z9.h, z0.h\n"
- ".inst 0x6460e558 // bfmmla z24.s, z10.h, z0.h\n"
- ".inst 0x6460e579 // bfmmla z25.s, z11.h, z0.h\n"
- ".inst 0x6461e51c // bfmmla z28.s, z8.h, z1.h\n"
- ".inst 0x6461e53d // bfmmla z29.s, z9.h, z1.h\n"
- ".inst 0x6461e55e // bfmmla z30.s, z10.h, z1.h\n"
- ".inst 0x6461e57f // bfmmla z31.s, z11.h, z1.h\n"
- "b 5f\n"
- "4:\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "trn2 z1.d, z2.d, z3.d\n"
- "ld1rqh z2.h, p6/z, [%[a_ptr0]]\n"
- ".inst 0x6464e4d4 // bfmmla z20.s, z6.h, z4.h\n"
- "ld1rqh z3.h, p6/z, [a_ptr1]\n"
- ".inst 0x6464e4f5 // bfmmla z21.s, z7.h, z4.h\n"
- "addvl %[a_ptr0], %[a_ptr0], #1\n"
- ".inst 0x6464e516 // bfmmla z22.s, z8.h, z4.h\n"
- "addvl a_ptr1, a_ptr1, #1\n"
- ".inst 0x6464e537 // bfmmla z23.s, z9.h, z4.h\n"
- ".inst 0x6464e558 // bfmmla z24.s, z10.h, z4.h\n"
- ".inst 0x6464e579 // bfmmla z25.s, z11.h, z4.h\n"
- "ld1rqh z4.h, p6/z, [a_ptr2]\n"
- ".inst 0x6465e4da // bfmmla z26.s, z6.h, z5.h\n"
- "ld1h z6.h, p7/z, [%[b_ptr0]]\n"
- ".inst 0x6465e4fb // bfmmla z27.s, z7.h, z5.h\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x6465e51c // bfmmla z28.s, z8.h, z5.h\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x6465e53d // bfmmla z29.s, z9.h, z5.h\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x6465e55e // bfmmla z30.s, z10.h, z5.h\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x6465e57f // bfmmla z31.s, z11.h, z5.h\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x6460e4d4 // bfmmla z20.s, z6.h, z0.h\n"
- "ld1rqh z5.h, p6/z, [a_ptr3]\n"
- ".inst 0x6460e4f5 // bfmmla z21.s, z7.h, z0.h\n"
- "addvl %[b_ptr0], %[b_ptr0], #6\n"
- ".inst 0x6460e516 // bfmmla z22.s, z8.h, z0.h\n"
- "addvl a_ptr2, a_ptr2, #1\n"
- ".inst 0x6460e537 // bfmmla z23.s, z9.h, z0.h\n"
- "addvl a_ptr3, a_ptr3, #1\n"
- ".inst 0x6460e558 // bfmmla z24.s, z10.h, z0.h\n"
- ".inst 0x6460e579 // bfmmla z25.s, z11.h, z0.h\n"
- "trn1 z0.d, z2.d, z3.d\n"
- ".inst 0x6461e4da // bfmmla z26.s, z6.h, z1.h\n"
- ".inst 0x6461e4fb // bfmmla z27.s, z7.h, z1.h\n"
- ".inst 0x6461e51c // bfmmla z28.s, z8.h, z1.h\n"
- ".inst 0x6461e53d // bfmmla z29.s, z9.h, z1.h\n"
- ".inst 0x6461e55e // bfmmla z30.s, z10.h, z1.h\n"
- ".inst 0x6461e57f // bfmmla z31.s, z11.h, z1.h\n"
- "trn1 z1.d, z4.d, z5.d\n"
- "cbz %[blocks], 5f\n"
- "trn2 z5.d, z4.d, z5.d\n"
- "ld1h z6.h, p7/z, [%[b_ptr0]]\n"
- "trn2 z4.d, z2.d, z3.d\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- ".inst 0x6460e4d4 // bfmmla z20.s, z6.h, z0.h\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x6460e4f5 // bfmmla z21.s, z7.h, z0.h\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x6460e516 // bfmmla z22.s, z8.h, z0.h\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x6461e4da // bfmmla z26.s, z6.h, z1.h\n"
- ".inst 0x6460e537 // bfmmla z23.s, z9.h, z0.h\n"
- ".inst 0x6460e558 // bfmmla z24.s, z10.h, z0.h\n"
- ".inst 0x6460e579 // bfmmla z25.s, z11.h, z0.h\n"
- ".inst 0x6461e4fb // bfmmla z27.s, z7.h, z1.h\n"
- ".inst 0x6461e51c // bfmmla z28.s, z8.h, z1.h\n"
- ".inst 0x6461e53d // bfmmla z29.s, z9.h, z1.h\n"
- ".inst 0x6461e55e // bfmmla z30.s, z10.h, z1.h\n"
- ".inst 0x6461e57f // bfmmla z31.s, z11.h, z1.h\n"
- "b.eq 5f\n"
- "ld1h z6.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- ".inst 0x6464e4d4 // bfmmla z20.s, z6.h, z4.h\n"
- ".inst 0x6464e4f5 // bfmmla z21.s, z7.h, z4.h\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- ".inst 0x6465e4da // bfmmla z26.s, z6.h, z5.h\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- ".inst 0x6465e4fb // bfmmla z27.s, z7.h, z5.h\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- ".inst 0x6464e516 // bfmmla z22.s, z8.h, z4.h\n"
- ".inst 0x6464e537 // bfmmla z23.s, z9.h, z4.h\n"
- ".inst 0x6464e558 // bfmmla z24.s, z10.h, z4.h\n"
- ".inst 0x6464e579 // bfmmla z25.s, z11.h, z4.h\n"
- ".inst 0x6465e51c // bfmmla z28.s, z8.h, z5.h\n"
- ".inst 0x6465e53d // bfmmla z29.s, z9.h, z5.h\n"
- ".inst 0x6465e55e // bfmmla z30.s, z10.h, z5.h\n"
- ".inst 0x6465e57f // bfmmla z31.s, z11.h, z5.h\n"
- "5:\n"
- "ld1rw z18.s, p7/z, [%[minptr]]\n"
- "ld1rw z19.s, p7/z, [%[maxptr]]\n"
- "fmax z20.s, p7/m, z20.s, z18.s\n"
- "fmax z21.s, p7/m, z21.s, z18.s\n"
- "fmax z22.s, p7/m, z22.s, z18.s\n"
- "fmax z23.s, p7/m, z23.s, z18.s\n"
- "fmin z20.s, p7/m, z20.s, z19.s\n"
- "fmin z21.s, p7/m, z21.s, z19.s\n"
- "fmin z22.s, p7/m, z22.s, z19.s\n"
- "fmin z23.s, p7/m, z23.s, z19.s\n"
- "fmax z24.s, p7/m, z24.s, z18.s\n"
- "uzp1 z0.s, z20.s, z21.s\n"
- "uzp2 z1.s, z20.s, z21.s\n"
- "uzp1 z2.s, z22.s, z23.s\n"
- "uzp2 z3.s, z22.s, z23.s\n"
- "st1w z0.s, p0, [%[c_ptr0]]\n"
- "fmin z24.s, p7/m, z24.s, z19.s\n"
- "fmax z25.s, p7/m, z25.s, z18.s\n"
- "fmax z26.s, p7/m, z26.s, z18.s\n"
- "st1w z1.s, p0, [c_ptr1]\n"
- "fmax z27.s, p7/m, z27.s, z18.s\n"
- "fmax z28.s, p7/m, z28.s, z18.s\n"
- "fmin z25.s, p7/m, z25.s, z19.s\n"
- "st1w z2.s, p1, [%[c_ptr0], #1, MUL VL]\n"
- "fmin z26.s, p7/m, z26.s, z19.s\n"
- "fmin z27.s, p7/m, z27.s, z19.s\n"
- "fmin z28.s, p7/m, z28.s, z19.s\n"
- "st1w z3.s, p1, [c_ptr1, #1, MUL VL]\n"
- "uzp1 z4.s, z24.s, z25.s\n"
- "uzp2 z5.s, z24.s, z25.s\n"
- "uzp1 z6.s, z26.s, z27.s\n"
- "uzp2 z7.s, z26.s, z27.s\n"
- "st1w z4.s, p2, [%[c_ptr0], #2, MUL VL]\n"
- "fmax z29.s, p7/m, z29.s, z18.s\n"
- "addvl %[c_ptr0], %[c_ptr0], #3\n"
- "fmax z30.s, p7/m, z30.s, z18.s\n"
- "st1w z5.s, p2, [c_ptr1, #2, MUL VL]\n"
- "fmax z31.s, p7/m, z31.s, z18.s\n"
- "fmin z29.s, p7/m, z29.s, z19.s\n"
- "fmin z30.s, p7/m, z30.s, z19.s\n"
- "st1w z6.s, p0, [c_ptr2]\n"
- "fmin z31.s, p7/m, z31.s, z19.s\n"
- "uzp1 z8.s, z28.s, z29.s\n"
- "uzp2 z9.s, z28.s, z29.s\n"
- "st1w z7.s, p0, [c_ptr3]\n"
- "uzp1 z10.s, z30.s, z31.s\n"
- "uzp2 z11.s, z30.s, z31.s\n"
- "st1w z8.s, p1, [c_ptr2, #1, MUL VL]\n"
- "st1w z9.s, p1, [c_ptr3, #1, MUL VL]\n"
- "st1w z10.s, p2, [c_ptr2, #2, MUL VL]\n"
- "st1w z11.s, p2, [c_ptr3, #2, MUL VL]\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq a_ptr3\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- ".unreq c_ptr3\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks)
- : [width] "r" (width), [accumulate] "r" (static_cast<uint64_t>(accumulate)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers)
- : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "x4", "x5", "cc", "memory"
- );
- break;
- }
-
- }
- }
-}
-
-} // namespace arm_gemm
-
-#endif // __ARM_FEATURE_SVE
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_bf16fp32_mmla_8VLx2.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_bf16fp32_mmla_8VLx2.hpp
deleted file mode 100644
index f25f747..0000000
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_bf16fp32_mmla_8VLx2.hpp
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Copyright (c) 2018-2020 Arm Limited.
- *
- * SPDX-License-Identifier: MIT
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to
- * deal in the Software without restriction, including without limitation the
- * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
- * sell copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-#pragma once
-
-#ifdef __ARM_FEATURE_SVE
-
-#include "../bfloat.hpp"
-#include "../std_transforms_sve.hpp"
-
-namespace arm_gemm
-{
-
-// Actual kernel implementations
-void sve_hybrid_bf16fp32_mmla_8VLx2(const bfloat16 *, int, const bfloat16 *, float *, int, int, int, int, const float *, Activation, bool);
-
-class hybrid_bf16fp32_mmla_8VLx2
-{
-public:
- typedef bfloat16 operand_type;
- typedef float result_type;
-
- typedef void (*kern_type)(const bfloat16 *, int, const bfloat16 *, float *, int, int, int, int, const float *, Activation, bool);
-
- /* Kernel blocking parameters */
- static constexpr unsigned int out_height()
- {
- return 4;
- }
-
- static unsigned int out_width()
- {
- return get_vector_length<float>() * 4;
- }
-
- static constexpr unsigned int k_unroll()
- {
- return 4;
- }
-
- static constexpr bool supports_accumulate()
- {
- return true;
- }
-
- static constexpr bool supports_bias()
- {
- return true;
- }
-
- static constexpr bool supports_activation()
- {
- return true;
- }
-
- StdTransformsSVE<operand_type, result_type, 2, 8, 4> transforms = {};
-
- // Default to the generic kernel
- kern_type kernel=sve_hybrid_bf16fp32_mmla_8VLx2;
-
- hybrid_bf16fp32_mmla_8VLx2(const CPUInfo *)
- {
-
- }
-};
-
-} // namespace arm_gemm
-
-#endif // __ARM_FEATURE_SVE
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_bf16fp32_mmla_8VLx2/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_bf16fp32_mmla_8VLx2/generic.cpp
deleted file mode 100644
index f38a2ea..0000000
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_bf16fp32_mmla_8VLx2/generic.cpp
+++ /dev/null
@@ -1,2001 +0,0 @@
-/*
- * Copyright (c) 2018-2020 Arm Limited.
- *
- * SPDX-License-Identifier: MIT
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to
- * deal in the Software without restriction, including without limitation the
- * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
- * sell copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-#ifdef __ARM_FEATURE_SVE
-
-#include <algorithm>
-
-#include "arm_gemm.hpp"
-#include "../../bfloat.hpp"
-#include "../../asmlib.hpp"
-#include "../../utils.hpp"
-
-namespace arm_gemm {
-
-void sve_hybrid_bf16fp32_mmla_8VLx2(const bfloat16 *A, int lda, const bfloat16 *B, float *C, int ldc, int M, int N, int K, const float *bias, Activation act, bool accumulate) {
- const int K_stride = ((K + 3) / 4) * 4;
- const long loops_count = ((K + 8) / 16) - 1;
- K -= loops_count * 16;
- const long regs_count = (K / 8) - 1;
- K -= (regs_count + 1) * 8;
- const long leftovers = K;
- const long blocks_count = (K + 3) / 4;
- float nullbias[256];
- if (!accumulate && !bias) {
- memset(nullbias, 0, (4 * get_vector_length<float>() * sizeof(float)));
- }
- float minval = - static_cast<float>(std::numeric_limits<float>::infinity());
- float maxval = static_cast<float>(std::numeric_limits<float>::infinity());
- const float * const minptr = &minval;
- const float * const maxptr = &maxval;
-
- switch(act.type)
- {
- default:
- case Activation::Type::None:
- break;
- case Activation::Type::BoundedReLU:
- maxval = static_cast<float>(act.param1);
- /* fall through */
- case Activation::Type::ReLU:
- minval = 0.0f;
- break;
- }
-
- int rows_to_compute;
-
- for (int y=0; y<M; y+=rows_to_compute) {
- const bfloat16 * const a_ptr0_base = A + (y * lda);
- const unsigned long ldab = lda * sizeof(bfloat16);
-
- float *c_ptr0 = C + (y * ldc);
-
- rows_to_compute = M-y;
- if (rows_to_compute > 4) {
- if (rows_to_compute % 4) {
- rows_to_compute = 4 - 1;
- } else {
- rows_to_compute = 4;
- }
- }
-
- for (int x0=0; x0<N; x0+=(4 * get_vector_length<float>())) {
- const long width = std::min((unsigned long)N-x0, (4 * get_vector_length<float>()));
- long loops = loops_count;
- long regs = regs_count;
- long temp = 0;
- long blocks = blocks_count;
- const bfloat16 *a_ptr0 = a_ptr0_base;
- const bfloat16 *b_ptr0 = B + (K_stride * x0);
- const unsigned long ldcb = ldc * sizeof(float);
- const float *biasptr = bias ? bias+x0 : nullbias;
-
- switch(rows_to_compute) {
- case 1:
- __asm __volatile (
- "whilelt p6.h, %[temp], %[leftovers]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "ptrue p7.h\n"
- "whilelt p1.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "whilelt p2.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "whilelt p3.s, %[temp], %[width]\n"
- "cbnz %[accumulate], 1f\n"
- "mov z1.h, #0\n"
- "ld1w z15.s, p0/z, [%[biasptr]]\n"
- "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ld1h z6.h, p7/z, [%[b_ptr0]]\n"
- "zip1 z16.s, z15.s, z15.s\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "zip2 z17.s, z15.s, z15.s\n"
- "ld1w z15.s, p1/z, [%[biasptr], #1, MUL VL]\n"
- "trn1 z4.d, z0.d, z1.d\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "zip1 z18.s, z15.s, z15.s\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "zip2 z19.s, z15.s, z15.s\n"
- "ld1w z15.s, p2/z, [%[biasptr], #2, MUL VL]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "zip1 z20.s, z15.s, z15.s\n"
- "zip2 z21.s, z15.s, z15.s\n"
- "ld1w z15.s, p3/z, [%[biasptr], #3, MUL VL]\n"
- "zip1 z22.s, z15.s, z15.s\n"
- "zip2 z23.s, z15.s, z15.s\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "mov z14.s, #0\n"
- "ld1w z13.s, p0/z, [%[c_ptr0]]\n"
- "mov z1.h, #0\n"
- "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n"
- "ld1h z6.h, p7/z, [%[b_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "zip1 z16.s, z13.s, z14.s\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "zip2 z17.s, z13.s, z14.s\n"
- "ld1w z13.s, p1/z, [%[c_ptr0], #1, MUL VL]\n"
- "trn1 z4.d, z0.d, z1.d\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "mov z14.s, #0\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "zip1 z18.s, z13.s, z14.s\n"
- "zip2 z19.s, z13.s, z14.s\n"
- "ld1w z13.s, p2/z, [%[c_ptr0], #2, MUL VL]\n"
- "mov z14.s, #0\n"
- "zip1 z20.s, z13.s, z14.s\n"
- "zip2 z21.s, z13.s, z14.s\n"
- "ld1w z13.s, p3/z, [%[c_ptr0], #3, MUL VL]\n"
- "mov z14.s, #0\n"
- "zip1 z22.s, z13.s, z14.s\n"
- "zip2 z23.s, z13.s, z14.s\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "cbz %[loops], 2f\n"
- "3:\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "ld1rqh z2.h, p7/z, [%[a_ptr0]]\n"
- ".inst 0x6464e4d0 // bfmmla z16.s, z6.h, z4.h\n"
- "ld1h z6.h, p7/z, [%[b_ptr0]]\n"
- ".inst 0x6464e4f1 // bfmmla z17.s, z7.h, z4.h\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x6464e512 // bfmmla z18.s, z8.h, z4.h\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x6464e533 // bfmmla z19.s, z9.h, z4.h\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x6464e554 // bfmmla z20.s, z10.h, z4.h\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x6464e575 // bfmmla z21.s, z11.h, z4.h\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x6464e596 // bfmmla z22.s, z12.h, z4.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x6464e5b7 // bfmmla z23.s, z13.h, z4.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "mov z3.h, #0\n"
- "subs %[loops], %[loops], #0x1\n"
- ".inst 0x6460e4d0 // bfmmla z16.s, z6.h, z0.h\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
- "trn2 z4.d, z2.d, z3.d\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- ".inst 0x6460e4f1 // bfmmla z17.s, z7.h, z0.h\n"
- ".inst 0x6460e512 // bfmmla z18.s, z8.h, z0.h\n"
- ".inst 0x6460e533 // bfmmla z19.s, z9.h, z0.h\n"
- "ld1h z6.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- ".inst 0x6460e554 // bfmmla z20.s, z10.h, z0.h\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- ".inst 0x6460e575 // bfmmla z21.s, z11.h, z0.h\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- ".inst 0x6460e596 // bfmmla z22.s, z12.h, z0.h\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- ".inst 0x6460e5b7 // bfmmla z23.s, z13.h, z0.h\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "trn1 z0.d, z2.d, z3.d\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "mov z1.h, #0\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- ".inst 0x6460e4d0 // bfmmla z16.s, z6.h, z0.h\n"
- "ld1h z6.h, p7/z, [%[b_ptr0]]\n"
- ".inst 0x6460e4f1 // bfmmla z17.s, z7.h, z0.h\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x6460e512 // bfmmla z18.s, z8.h, z0.h\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x6460e533 // bfmmla z19.s, z9.h, z0.h\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x6460e554 // bfmmla z20.s, z10.h, z0.h\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x6460e575 // bfmmla z21.s, z11.h, z0.h\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x6460e596 // bfmmla z22.s, z12.h, z0.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x6460e5b7 // bfmmla z23.s, z13.h, z0.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x6464e4d0 // bfmmla z16.s, z6.h, z4.h\n"
- "ld1rqh z0.h, p7/z, [%[a_ptr0], #-0x10]\n"
- ".inst 0x6464e4f1 // bfmmla z17.s, z7.h, z4.h\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- ".inst 0x6464e512 // bfmmla z18.s, z8.h, z4.h\n"
- ".inst 0x6464e533 // bfmmla z19.s, z9.h, z4.h\n"
- ".inst 0x6464e554 // bfmmla z20.s, z10.h, z4.h\n"
- "ld1h z6.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- ".inst 0x6464e575 // bfmmla z21.s, z11.h, z4.h\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- ".inst 0x6464e596 // bfmmla z22.s, z12.h, z4.h\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- ".inst 0x6464e5b7 // bfmmla z23.s, z13.h, z4.h\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "trn1 z4.d, z0.d, z1.d\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "b.ne 3b\n"
- "2:\n"
- "cbz %[regs], 4f\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "ld1rqh z2.h, p7/z, [%[a_ptr0]]\n"
- ".inst 0x6464e4d0 // bfmmla z16.s, z6.h, z4.h\n"
- "ld1h z6.h, p7/z, [%[b_ptr0]]\n"
- ".inst 0x6464e4f1 // bfmmla z17.s, z7.h, z4.h\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x6464e512 // bfmmla z18.s, z8.h, z4.h\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x6464e533 // bfmmla z19.s, z9.h, z4.h\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x6464e554 // bfmmla z20.s, z10.h, z4.h\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x6464e575 // bfmmla z21.s, z11.h, z4.h\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x6464e596 // bfmmla z22.s, z12.h, z4.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x6464e5b7 // bfmmla z23.s, z13.h, z4.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "mov z3.h, #0\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- ".inst 0x6460e4d0 // bfmmla z16.s, z6.h, z0.h\n"
- ".inst 0x6460e4f1 // bfmmla z17.s, z7.h, z0.h\n"
- "trn2 z4.d, z2.d, z3.d\n"
- "ld1h z6.h, p7/z, [%[b_ptr0]]\n"
- ".inst 0x6460e512 // bfmmla z18.s, z8.h, z0.h\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x6460e533 // bfmmla z19.s, z9.h, z0.h\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x6460e554 // bfmmla z20.s, z10.h, z0.h\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x6460e575 // bfmmla z21.s, z11.h, z0.h\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x6460e596 // bfmmla z22.s, z12.h, z0.h\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x6460e5b7 // bfmmla z23.s, z13.h, z0.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "trn1 z0.d, z2.d, z3.d\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "mov z1.h, #0\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- ".inst 0x6460e4d0 // bfmmla z16.s, z6.h, z0.h\n"
- ".inst 0x6460e4f1 // bfmmla z17.s, z7.h, z0.h\n"
- "ld1h z6.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- ".inst 0x6460e512 // bfmmla z18.s, z8.h, z0.h\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- ".inst 0x6460e533 // bfmmla z19.s, z9.h, z0.h\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- ".inst 0x6460e554 // bfmmla z20.s, z10.h, z0.h\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- ".inst 0x6460e575 // bfmmla z21.s, z11.h, z0.h\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- ".inst 0x6460e596 // bfmmla z22.s, z12.h, z0.h\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- ".inst 0x6460e5b7 // bfmmla z23.s, z13.h, z0.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- ".inst 0x6464e4d0 // bfmmla z16.s, z6.h, z4.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- ".inst 0x6464e4f1 // bfmmla z17.s, z7.h, z4.h\n"
- "ld1rqh z0.h, p6/z, [%[a_ptr0], #0x10]\n"
- ".inst 0x6464e512 // bfmmla z18.s, z8.h, z4.h\n"
- "addvl %[a_ptr0], %[a_ptr0], #2\n"
- ".inst 0x6464e533 // bfmmla z19.s, z9.h, z4.h\n"
- ".inst 0x6464e554 // bfmmla z20.s, z10.h, z4.h\n"
- ".inst 0x6464e575 // bfmmla z21.s, z11.h, z4.h\n"
- ".inst 0x6464e596 // bfmmla z22.s, z12.h, z4.h\n"
- ".inst 0x6464e5b7 // bfmmla z23.s, z13.h, z4.h\n"
- "trn1 z4.d, z0.d, z1.d\n"
- "cbz %[blocks], 5f\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "ld1h z6.h, p7/z, [%[b_ptr0]]\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x6464e4d0 // bfmmla z16.s, z6.h, z4.h\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x6464e4f1 // bfmmla z17.s, z7.h, z4.h\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x6464e512 // bfmmla z18.s, z8.h, z4.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x6464e533 // bfmmla z19.s, z9.h, z4.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x6464e554 // bfmmla z20.s, z10.h, z4.h\n"
- ".inst 0x6464e575 // bfmmla z21.s, z11.h, z4.h\n"
- ".inst 0x6464e596 // bfmmla z22.s, z12.h, z4.h\n"
- ".inst 0x6464e5b7 // bfmmla z23.s, z13.h, z4.h\n"
- "b.eq 5f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "ld1h z6.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- ".inst 0x6460e4d0 // bfmmla z16.s, z6.h, z0.h\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- ".inst 0x6460e4f1 // bfmmla z17.s, z7.h, z0.h\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- ".inst 0x6460e512 // bfmmla z18.s, z8.h, z0.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- ".inst 0x6460e533 // bfmmla z19.s, z9.h, z0.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- ".inst 0x6460e554 // bfmmla z20.s, z10.h, z0.h\n"
- ".inst 0x6460e575 // bfmmla z21.s, z11.h, z0.h\n"
- ".inst 0x6460e596 // bfmmla z22.s, z12.h, z0.h\n"
- ".inst 0x6460e5b7 // bfmmla z23.s, z13.h, z0.h\n"
- "b 5f\n"
- "4:\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "ld1rqh z2.h, p6/z, [%[a_ptr0]]\n"
- ".inst 0x6464e4d0 // bfmmla z16.s, z6.h, z4.h\n"
- "ld1h z6.h, p7/z, [%[b_ptr0]]\n"
- ".inst 0x6464e4f1 // bfmmla z17.s, z7.h, z4.h\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x6464e512 // bfmmla z18.s, z8.h, z4.h\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x6464e533 // bfmmla z19.s, z9.h, z4.h\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x6464e554 // bfmmla z20.s, z10.h, z4.h\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x6464e575 // bfmmla z21.s, z11.h, z4.h\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x6464e596 // bfmmla z22.s, z12.h, z4.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x6464e5b7 // bfmmla z23.s, z13.h, z4.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "mov z3.h, #0\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- ".inst 0x6460e4d0 // bfmmla z16.s, z6.h, z0.h\n"
- "addvl %[a_ptr0], %[a_ptr0], #1\n"
- ".inst 0x6460e4f1 // bfmmla z17.s, z7.h, z0.h\n"
- ".inst 0x6460e512 // bfmmla z18.s, z8.h, z0.h\n"
- ".inst 0x6460e533 // bfmmla z19.s, z9.h, z0.h\n"
- ".inst 0x6460e554 // bfmmla z20.s, z10.h, z0.h\n"
- ".inst 0x6460e575 // bfmmla z21.s, z11.h, z0.h\n"
- ".inst 0x6460e596 // bfmmla z22.s, z12.h, z0.h\n"
- ".inst 0x6460e5b7 // bfmmla z23.s, z13.h, z0.h\n"
- "trn1 z0.d, z2.d, z3.d\n"
- "cbz %[blocks], 5f\n"
- "trn2 z4.d, z2.d, z3.d\n"
- "ld1h z6.h, p7/z, [%[b_ptr0]]\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x6460e4d0 // bfmmla z16.s, z6.h, z0.h\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x6460e4f1 // bfmmla z17.s, z7.h, z0.h\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x6460e512 // bfmmla z18.s, z8.h, z0.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x6460e533 // bfmmla z19.s, z9.h, z0.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x6460e554 // bfmmla z20.s, z10.h, z0.h\n"
- ".inst 0x6460e575 // bfmmla z21.s, z11.h, z0.h\n"
- ".inst 0x6460e596 // bfmmla z22.s, z12.h, z0.h\n"
- ".inst 0x6460e5b7 // bfmmla z23.s, z13.h, z0.h\n"
- "b.eq 5f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "ld1h z6.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- ".inst 0x6464e4d0 // bfmmla z16.s, z6.h, z4.h\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- ".inst 0x6464e4f1 // bfmmla z17.s, z7.h, z4.h\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- ".inst 0x6464e512 // bfmmla z18.s, z8.h, z4.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- ".inst 0x6464e533 // bfmmla z19.s, z9.h, z4.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- ".inst 0x6464e554 // bfmmla z20.s, z10.h, z4.h\n"
- ".inst 0x6464e575 // bfmmla z21.s, z11.h, z4.h\n"
- ".inst 0x6464e596 // bfmmla z22.s, z12.h, z4.h\n"
- ".inst 0x6464e5b7 // bfmmla z23.s, z13.h, z4.h\n"
- "5:\n"
- "ld1rw z14.s, p7/z, [%[minptr]]\n"
- "ld1rw z15.s, p7/z, [%[maxptr]]\n"
- "fmax z16.s, p7/m, z16.s, z14.s\n"
- "fmax z17.s, p7/m, z17.s, z14.s\n"
- "fmax z18.s, p7/m, z18.s, z14.s\n"
- "fmax z19.s, p7/m, z19.s, z14.s\n"
- "fmin z16.s, p7/m, z16.s, z15.s\n"
- "fmin z17.s, p7/m, z17.s, z15.s\n"
- "fmin z18.s, p7/m, z18.s, z15.s\n"
- "fmin z19.s, p7/m, z19.s, z15.s\n"
- "fmax z20.s, p7/m, z20.s, z14.s\n"
- "uzp1 z0.s, z16.s, z17.s\n"
- "fmax z21.s, p7/m, z21.s, z14.s\n"
- "uzp1 z1.s, z18.s, z19.s\n"
- "fmin z20.s, p7/m, z20.s, z15.s\n"
- "st1w z0.s, p0, [%[c_ptr0]]\n"
- "fmax z22.s, p7/m, z22.s, z14.s\n"
- "fmin z21.s, p7/m, z21.s, z15.s\n"
- "fmax z23.s, p7/m, z23.s, z14.s\n"
- "st1w z1.s, p1, [%[c_ptr0], #1, MUL VL]\n"
- "fmin z22.s, p7/m, z22.s, z15.s\n"
- "uzp1 z2.s, z20.s, z21.s\n"
- "fmin z23.s, p7/m, z23.s, z15.s\n"
- "st1w z2.s, p2, [%[c_ptr0], #2, MUL VL]\n"
- "uzp1 z3.s, z22.s, z23.s\n"
- "st1w z3.s, p3, [%[c_ptr0], #3, MUL VL]\n"
- "addvl %[c_ptr0], %[c_ptr0], #4\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks)
- : [width] "r" (width), [accumulate] "r" (static_cast<uint64_t>(accumulate)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers)
- : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
- );
- break;
- case 2:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "c_ptr1 .req X1\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "whilelt p6.h, %[temp], %[leftovers]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "ptrue p7.h\n"
- "whilelt p1.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "whilelt p2.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "whilelt p3.s, %[temp], %[width]\n"
- "cbnz %[accumulate], 1f\n"
- "ld1w z15.s, p0/z, [%[biasptr]]\n"
- "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ld1rqh z1.h, p7/z, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "zip1 z16.s, z15.s, z15.s\n"
- "ld1h z6.h, p7/z, [%[b_ptr0]]\n"
- "zip2 z17.s, z15.s, z15.s\n"
- "ld1w z15.s, p1/z, [%[biasptr], #1, MUL VL]\n"
- "trn1 z4.d, z0.d, z1.d\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "zip1 z18.s, z15.s, z15.s\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "zip2 z19.s, z15.s, z15.s\n"
- "ld1w z15.s, p2/z, [%[biasptr], #2, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "zip1 z20.s, z15.s, z15.s\n"
- "zip2 z21.s, z15.s, z15.s\n"
- "ld1w z15.s, p3/z, [%[biasptr], #3, MUL VL]\n"
- "zip1 z22.s, z15.s, z15.s\n"
- "zip2 z23.s, z15.s, z15.s\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "ld1w z13.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z14.s, p0/z, [c_ptr1]\n"
- "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ld1rqh z1.h, p7/z, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "zip1 z16.s, z13.s, z14.s\n"
- "ld1h z6.h, p7/z, [%[b_ptr0]]\n"
- "zip2 z17.s, z13.s, z14.s\n"
- "ld1w z13.s, p1/z, [%[c_ptr0], #1, MUL VL]\n"
- "trn1 z4.d, z0.d, z1.d\n"
- "ld1w z14.s, p1/z, [c_ptr1, #1, MUL VL]\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "zip1 z18.s, z13.s, z14.s\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "zip2 z19.s, z13.s, z14.s\n"
- "ld1w z13.s, p2/z, [%[c_ptr0], #2, MUL VL]\n"
- "ld1w z14.s, p2/z, [c_ptr1, #2, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "zip1 z20.s, z13.s, z14.s\n"
- "zip2 z21.s, z13.s, z14.s\n"
- "ld1w z13.s, p3/z, [%[c_ptr0], #3, MUL VL]\n"
- "ld1w z14.s, p3/z, [c_ptr1, #3, MUL VL]\n"
- "zip1 z22.s, z13.s, z14.s\n"
- "zip2 z23.s, z13.s, z14.s\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "cbz %[loops], 2f\n"
- "3:\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "ld1rqh z2.h, p7/z, [%[a_ptr0]]\n"
- ".inst 0x6464e4d0 // bfmmla z16.s, z6.h, z4.h\n"
- "ld1h z6.h, p7/z, [%[b_ptr0]]\n"
- ".inst 0x6464e4f1 // bfmmla z17.s, z7.h, z4.h\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x6464e512 // bfmmla z18.s, z8.h, z4.h\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x6464e533 // bfmmla z19.s, z9.h, z4.h\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x6464e554 // bfmmla z20.s, z10.h, z4.h\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x6464e575 // bfmmla z21.s, z11.h, z4.h\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x6464e596 // bfmmla z22.s, z12.h, z4.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x6464e5b7 // bfmmla z23.s, z13.h, z4.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x6460e4d0 // bfmmla z16.s, z6.h, z0.h\n"
- "ld1rqh z3.h, p7/z, [a_ptr1]\n"
- ".inst 0x6460e4f1 // bfmmla z17.s, z7.h, z0.h\n"
- "subs %[loops], %[loops], #0x1\n"
- ".inst 0x6460e512 // bfmmla z18.s, z8.h, z0.h\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
- "trn2 z4.d, z2.d, z3.d\n"
- "add a_ptr1, a_ptr1, #0x20\n"
- ".inst 0x6460e533 // bfmmla z19.s, z9.h, z0.h\n"
- "ld1rqh z1.h, p7/z, [a_ptr1, #-0x10]\n"
- ".inst 0x6460e554 // bfmmla z20.s, z10.h, z0.h\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- ".inst 0x6460e575 // bfmmla z21.s, z11.h, z0.h\n"
- ".inst 0x6460e596 // bfmmla z22.s, z12.h, z0.h\n"
- ".inst 0x6460e5b7 // bfmmla z23.s, z13.h, z0.h\n"
- "ld1h z6.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "trn1 z0.d, z2.d, z3.d\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- ".inst 0x6460e4d0 // bfmmla z16.s, z6.h, z0.h\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- ".inst 0x6460e4f1 // bfmmla z17.s, z7.h, z0.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- ".inst 0x6460e512 // bfmmla z18.s, z8.h, z0.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- ".inst 0x6460e533 // bfmmla z19.s, z9.h, z0.h\n"
- "ld1h z6.h, p7/z, [%[b_ptr0]]\n"
- ".inst 0x6460e554 // bfmmla z20.s, z10.h, z0.h\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x6460e575 // bfmmla z21.s, z11.h, z0.h\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x6460e596 // bfmmla z22.s, z12.h, z0.h\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x6460e5b7 // bfmmla z23.s, z13.h, z0.h\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x6464e4d0 // bfmmla z16.s, z6.h, z4.h\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x6464e4f1 // bfmmla z17.s, z7.h, z4.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x6464e512 // bfmmla z18.s, z8.h, z4.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x6464e533 // bfmmla z19.s, z9.h, z4.h\n"
- "ld1rqh z0.h, p7/z, [%[a_ptr0], #-0x10]\n"
- ".inst 0x6464e554 // bfmmla z20.s, z10.h, z4.h\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- ".inst 0x6464e575 // bfmmla z21.s, z11.h, z4.h\n"
- ".inst 0x6464e596 // bfmmla z22.s, z12.h, z4.h\n"
- ".inst 0x6464e5b7 // bfmmla z23.s, z13.h, z4.h\n"
- "ld1h z6.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "trn1 z4.d, z0.d, z1.d\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "b.ne 3b\n"
- "2:\n"
- "cbz %[regs], 4f\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "ld1rqh z2.h, p7/z, [%[a_ptr0]]\n"
- ".inst 0x6464e4d0 // bfmmla z16.s, z6.h, z4.h\n"
- "ld1h z6.h, p7/z, [%[b_ptr0]]\n"
- ".inst 0x6464e4f1 // bfmmla z17.s, z7.h, z4.h\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x6464e512 // bfmmla z18.s, z8.h, z4.h\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x6464e533 // bfmmla z19.s, z9.h, z4.h\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x6464e554 // bfmmla z20.s, z10.h, z4.h\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x6464e575 // bfmmla z21.s, z11.h, z4.h\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x6464e596 // bfmmla z22.s, z12.h, z4.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x6464e5b7 // bfmmla z23.s, z13.h, z4.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x6460e4d0 // bfmmla z16.s, z6.h, z0.h\n"
- "ld1rqh z3.h, p7/z, [a_ptr1]\n"
- ".inst 0x6460e4f1 // bfmmla z17.s, z7.h, z0.h\n"
- "ld1rqh z1.h, p6/z, [a_ptr1, #0x10]\n"
- ".inst 0x6460e512 // bfmmla z18.s, z8.h, z0.h\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "trn2 z4.d, z2.d, z3.d\n"
- "addvl a_ptr1, a_ptr1, #2\n"
- ".inst 0x6460e533 // bfmmla z19.s, z9.h, z0.h\n"
- "ld1h z6.h, p7/z, [%[b_ptr0]]\n"
- ".inst 0x6460e554 // bfmmla z20.s, z10.h, z0.h\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x6460e575 // bfmmla z21.s, z11.h, z0.h\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x6460e596 // bfmmla z22.s, z12.h, z0.h\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x6460e5b7 // bfmmla z23.s, z13.h, z0.h\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "trn1 z0.d, z2.d, z3.d\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- ".inst 0x6460e4d0 // bfmmla z16.s, z6.h, z0.h\n"
- ".inst 0x6460e4f1 // bfmmla z17.s, z7.h, z0.h\n"
- ".inst 0x6460e512 // bfmmla z18.s, z8.h, z0.h\n"
- "ld1h z6.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- ".inst 0x6460e533 // bfmmla z19.s, z9.h, z0.h\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- ".inst 0x6460e554 // bfmmla z20.s, z10.h, z0.h\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- ".inst 0x6460e575 // bfmmla z21.s, z11.h, z0.h\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- ".inst 0x6460e596 // bfmmla z22.s, z12.h, z0.h\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- ".inst 0x6460e5b7 // bfmmla z23.s, z13.h, z0.h\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- ".inst 0x6464e4d0 // bfmmla z16.s, z6.h, z4.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- ".inst 0x6464e4f1 // bfmmla z17.s, z7.h, z4.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- ".inst 0x6464e512 // bfmmla z18.s, z8.h, z4.h\n"
- "ld1rqh z0.h, p6/z, [%[a_ptr0], #0x10]\n"
- ".inst 0x6464e533 // bfmmla z19.s, z9.h, z4.h\n"
- "addvl %[a_ptr0], %[a_ptr0], #2\n"
- ".inst 0x6464e554 // bfmmla z20.s, z10.h, z4.h\n"
- ".inst 0x6464e575 // bfmmla z21.s, z11.h, z4.h\n"
- ".inst 0x6464e596 // bfmmla z22.s, z12.h, z4.h\n"
- ".inst 0x6464e5b7 // bfmmla z23.s, z13.h, z4.h\n"
- "trn1 z4.d, z0.d, z1.d\n"
- "cbz %[blocks], 5f\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "ld1h z6.h, p7/z, [%[b_ptr0]]\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x6464e4d0 // bfmmla z16.s, z6.h, z4.h\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x6464e4f1 // bfmmla z17.s, z7.h, z4.h\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x6464e512 // bfmmla z18.s, z8.h, z4.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x6464e533 // bfmmla z19.s, z9.h, z4.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x6464e554 // bfmmla z20.s, z10.h, z4.h\n"
- ".inst 0x6464e575 // bfmmla z21.s, z11.h, z4.h\n"
- ".inst 0x6464e596 // bfmmla z22.s, z12.h, z4.h\n"
- ".inst 0x6464e5b7 // bfmmla z23.s, z13.h, z4.h\n"
- "b.eq 5f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "ld1h z6.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- ".inst 0x6460e4d0 // bfmmla z16.s, z6.h, z0.h\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- ".inst 0x6460e4f1 // bfmmla z17.s, z7.h, z0.h\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- ".inst 0x6460e512 // bfmmla z18.s, z8.h, z0.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- ".inst 0x6460e533 // bfmmla z19.s, z9.h, z0.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- ".inst 0x6460e554 // bfmmla z20.s, z10.h, z0.h\n"
- ".inst 0x6460e575 // bfmmla z21.s, z11.h, z0.h\n"
- ".inst 0x6460e596 // bfmmla z22.s, z12.h, z0.h\n"
- ".inst 0x6460e5b7 // bfmmla z23.s, z13.h, z0.h\n"
- "b 5f\n"
- "4:\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "ld1rqh z2.h, p6/z, [%[a_ptr0]]\n"
- ".inst 0x6464e4d0 // bfmmla z16.s, z6.h, z4.h\n"
- "ld1h z6.h, p7/z, [%[b_ptr0]]\n"
- ".inst 0x6464e4f1 // bfmmla z17.s, z7.h, z4.h\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x6464e512 // bfmmla z18.s, z8.h, z4.h\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x6464e533 // bfmmla z19.s, z9.h, z4.h\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x6464e554 // bfmmla z20.s, z10.h, z4.h\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x6464e575 // bfmmla z21.s, z11.h, z4.h\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x6464e596 // bfmmla z22.s, z12.h, z4.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x6464e5b7 // bfmmla z23.s, z13.h, z4.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x6460e4d0 // bfmmla z16.s, z6.h, z0.h\n"
- "ld1rqh z3.h, p6/z, [a_ptr1]\n"
- ".inst 0x6460e4f1 // bfmmla z17.s, z7.h, z0.h\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- ".inst 0x6460e512 // bfmmla z18.s, z8.h, z0.h\n"
- "addvl %[a_ptr0], %[a_ptr0], #1\n"
- ".inst 0x6460e533 // bfmmla z19.s, z9.h, z0.h\n"
- "addvl a_ptr1, a_ptr1, #1\n"
- ".inst 0x6460e554 // bfmmla z20.s, z10.h, z0.h\n"
- ".inst 0x6460e575 // bfmmla z21.s, z11.h, z0.h\n"
- ".inst 0x6460e596 // bfmmla z22.s, z12.h, z0.h\n"
- ".inst 0x6460e5b7 // bfmmla z23.s, z13.h, z0.h\n"
- "trn1 z0.d, z2.d, z3.d\n"
- "cbz %[blocks], 5f\n"
- "trn2 z4.d, z2.d, z3.d\n"
- "ld1h z6.h, p7/z, [%[b_ptr0]]\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x6460e4d0 // bfmmla z16.s, z6.h, z0.h\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x6460e4f1 // bfmmla z17.s, z7.h, z0.h\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x6460e512 // bfmmla z18.s, z8.h, z0.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x6460e533 // bfmmla z19.s, z9.h, z0.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x6460e554 // bfmmla z20.s, z10.h, z0.h\n"
- ".inst 0x6460e575 // bfmmla z21.s, z11.h, z0.h\n"
- ".inst 0x6460e596 // bfmmla z22.s, z12.h, z0.h\n"
- ".inst 0x6460e5b7 // bfmmla z23.s, z13.h, z0.h\n"
- "b.eq 5f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "ld1h z6.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- ".inst 0x6464e4d0 // bfmmla z16.s, z6.h, z4.h\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- ".inst 0x6464e4f1 // bfmmla z17.s, z7.h, z4.h\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- ".inst 0x6464e512 // bfmmla z18.s, z8.h, z4.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- ".inst 0x6464e533 // bfmmla z19.s, z9.h, z4.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- ".inst 0x6464e554 // bfmmla z20.s, z10.h, z4.h\n"
- ".inst 0x6464e575 // bfmmla z21.s, z11.h, z4.h\n"
- ".inst 0x6464e596 // bfmmla z22.s, z12.h, z4.h\n"
- ".inst 0x6464e5b7 // bfmmla z23.s, z13.h, z4.h\n"
- "5:\n"
- "ld1rw z14.s, p7/z, [%[minptr]]\n"
- "ld1rw z15.s, p7/z, [%[maxptr]]\n"
- "fmax z16.s, p7/m, z16.s, z14.s\n"
- "fmax z17.s, p7/m, z17.s, z14.s\n"
- "fmax z18.s, p7/m, z18.s, z14.s\n"
- "fmax z19.s, p7/m, z19.s, z14.s\n"
- "fmin z16.s, p7/m, z16.s, z15.s\n"
- "fmin z17.s, p7/m, z17.s, z15.s\n"
- "fmin z18.s, p7/m, z18.s, z15.s\n"
- "fmin z19.s, p7/m, z19.s, z15.s\n"
- "fmax z20.s, p7/m, z20.s, z14.s\n"
- "uzp1 z0.s, z16.s, z17.s\n"
- "uzp2 z1.s, z16.s, z17.s\n"
- "uzp1 z2.s, z18.s, z19.s\n"
- "uzp2 z3.s, z18.s, z19.s\n"
- "st1w z0.s, p0, [%[c_ptr0]]\n"
- "fmin z20.s, p7/m, z20.s, z15.s\n"
- "fmax z21.s, p7/m, z21.s, z14.s\n"
- "fmax z22.s, p7/m, z22.s, z14.s\n"
- "st1w z1.s, p0, [c_ptr1]\n"
- "fmax z23.s, p7/m, z23.s, z14.s\n"
- "fmin z21.s, p7/m, z21.s, z15.s\n"
- "st1w z2.s, p1, [%[c_ptr0], #1, MUL VL]\n"
- "fmin z22.s, p7/m, z22.s, z15.s\n"
- "fmin z23.s, p7/m, z23.s, z15.s\n"
- "uzp1 z4.s, z20.s, z21.s\n"
- "st1w z3.s, p1, [c_ptr1, #1, MUL VL]\n"
- "uzp2 z5.s, z20.s, z21.s\n"
- "uzp1 z6.s, z22.s, z23.s\n"
- "uzp2 z7.s, z22.s, z23.s\n"
- "st1w z4.s, p2, [%[c_ptr0], #2, MUL VL]\n"
- "st1w z5.s, p2, [c_ptr1, #2, MUL VL]\n"
- "st1w z6.s, p3, [%[c_ptr0], #3, MUL VL]\n"
- "addvl %[c_ptr0], %[c_ptr0], #4\n"
- "st1w z7.s, p3, [c_ptr1, #3, MUL VL]\n"
- ".unreq a_ptr1\n"
- ".unreq c_ptr1\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks)
- : [width] "r" (width), [accumulate] "r" (static_cast<uint64_t>(accumulate)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers)
- : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "cc", "memory"
- );
- break;
- case 3:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "c_ptr1 .req X2\n"
- "c_ptr2 .req X3\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "whilelt p6.h, %[temp], %[leftovers]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "ptrue p7.h\n"
- "whilelt p1.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "whilelt p2.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "whilelt p3.s, %[temp], %[width]\n"
- "cbnz %[accumulate], 1f\n"
- "mov z3.h, #0\n"
- "ld1w z15.s, p0/z, [%[biasptr]]\n"
- "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ld1rqh z1.h, p7/z, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "zip1 z16.s, z15.s, z15.s\n"
- "ld1rqh z2.h, p7/z, [a_ptr2]\n"
- "zip2 z17.s, z15.s, z15.s\n"
- "ld1w z15.s, p1/z, [%[biasptr], #1, MUL VL]\n"
- "trn1 z4.d, z0.d, z1.d\n"
- "ld1h z6.h, p7/z, [%[b_ptr0]]\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "zip1 z18.s, z15.s, z15.s\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "zip2 z19.s, z15.s, z15.s\n"
- "ld1w z15.s, p2/z, [%[biasptr], #2, MUL VL]\n"
- "trn1 z5.d, z2.d, z3.d\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "mov z24.d, z16.d\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "mov z25.d, z17.d\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "zip1 z20.s, z15.s, z15.s\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "zip2 z21.s, z15.s, z15.s\n"
- "ld1w z15.s, p3/z, [%[biasptr], #3, MUL VL]\n"
- "mov z26.d, z18.d\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "mov z27.d, z19.d\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "zip1 z22.s, z15.s, z15.s\n"
- "zip2 z23.s, z15.s, z15.s\n"
- "mov z28.d, z20.d\n"
- "mov z29.d, z21.d\n"
- "mov z30.d, z22.d\n"
- "mov z31.d, z23.d\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "mov z3.h, #0\n"
- "ld1w z13.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z14.s, p0/z, [c_ptr1]\n"
- "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ld1rqh z1.h, p7/z, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "zip1 z16.s, z13.s, z14.s\n"
- "ld1rqh z2.h, p7/z, [a_ptr2]\n"
- "zip2 z17.s, z13.s, z14.s\n"
- "ld1w z13.s, p1/z, [%[c_ptr0], #1, MUL VL]\n"
- "trn1 z4.d, z0.d, z1.d\n"
- "ld1w z14.s, p1/z, [c_ptr1, #1, MUL VL]\n"
- "ld1h z6.h, p7/z, [%[b_ptr0]]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "trn1 z5.d, z2.d, z3.d\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "zip1 z18.s, z13.s, z14.s\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "zip2 z19.s, z13.s, z14.s\n"
- "ld1w z13.s, p2/z, [%[c_ptr0], #2, MUL VL]\n"
- "ld1w z14.s, p2/z, [c_ptr1, #2, MUL VL]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "zip1 z20.s, z13.s, z14.s\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "zip2 z21.s, z13.s, z14.s\n"
- "ld1w z13.s, p3/z, [%[c_ptr0], #3, MUL VL]\n"
- "ld1w z14.s, p3/z, [c_ptr1, #3, MUL VL]\n"
- "zip1 z22.s, z13.s, z14.s\n"
- "zip2 z23.s, z13.s, z14.s\n"
- "ld1w z13.s, p0/z, [c_ptr2]\n"
- "mov z14.s, #0\n"
- "zip1 z24.s, z13.s, z14.s\n"
- "zip2 z25.s, z13.s, z14.s\n"
- "ld1w z13.s, p1/z, [c_ptr2, #1, MUL VL]\n"
- "mov z14.s, #0\n"
- "zip1 z26.s, z13.s, z14.s\n"
- "zip2 z27.s, z13.s, z14.s\n"
- "ld1w z13.s, p2/z, [c_ptr2, #2, MUL VL]\n"
- "mov z14.s, #0\n"
- "zip1 z28.s, z13.s, z14.s\n"
- "zip2 z29.s, z13.s, z14.s\n"
- "ld1w z13.s, p3/z, [c_ptr2, #3, MUL VL]\n"
- "mov z14.s, #0\n"
- "zip1 z30.s, z13.s, z14.s\n"
- "zip2 z31.s, z13.s, z14.s\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "cbz %[loops], 2f\n"
- "3:\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "subs %[loops], %[loops], #0x1\n"
- "trn2 z1.d, z2.d, z3.d\n"
- "ld1rqh z2.h, p7/z, [%[a_ptr0]]\n"
- ".inst 0x6464e4d0 // bfmmla z16.s, z6.h, z4.h\n"
- "ld1rqh z3.h, p7/z, [a_ptr1]\n"
- ".inst 0x6464e4f1 // bfmmla z17.s, z7.h, z4.h\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
- ".inst 0x6464e512 // bfmmla z18.s, z8.h, z4.h\n"
- "add a_ptr1, a_ptr1, #0x20\n"
- ".inst 0x6464e533 // bfmmla z19.s, z9.h, z4.h\n"
- ".inst 0x6464e554 // bfmmla z20.s, z10.h, z4.h\n"
- ".inst 0x6464e575 // bfmmla z21.s, z11.h, z4.h\n"
- ".inst 0x6464e596 // bfmmla z22.s, z12.h, z4.h\n"
- ".inst 0x6464e5b7 // bfmmla z23.s, z13.h, z4.h\n"
- "ld1rqh z4.h, p7/z, [a_ptr2]\n"
- ".inst 0x6465e4d8 // bfmmla z24.s, z6.h, z5.h\n"
- "ld1h z6.h, p7/z, [%[b_ptr0]]\n"
- ".inst 0x6465e4f9 // bfmmla z25.s, z7.h, z5.h\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x6465e51a // bfmmla z26.s, z8.h, z5.h\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x6465e53b // bfmmla z27.s, z9.h, z5.h\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x6465e55c // bfmmla z28.s, z10.h, z5.h\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x6465e57d // bfmmla z29.s, z11.h, z5.h\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x6465e59e // bfmmla z30.s, z12.h, z5.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x6465e5bf // bfmmla z31.s, z13.h, z5.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "mov z5.h, #0\n"
- "add a_ptr2, a_ptr2, #0x20\n"
- ".inst 0x6460e4d0 // bfmmla z16.s, z6.h, z0.h\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- ".inst 0x6460e4f1 // bfmmla z17.s, z7.h, z0.h\n"
- ".inst 0x6460e512 // bfmmla z18.s, z8.h, z0.h\n"
- ".inst 0x6460e533 // bfmmla z19.s, z9.h, z0.h\n"
- ".inst 0x6460e554 // bfmmla z20.s, z10.h, z0.h\n"
- ".inst 0x6460e575 // bfmmla z21.s, z11.h, z0.h\n"
- ".inst 0x6460e596 // bfmmla z22.s, z12.h, z0.h\n"
- ".inst 0x6460e5b7 // bfmmla z23.s, z13.h, z0.h\n"
- "trn1 z0.d, z2.d, z3.d\n"
- ".inst 0x6461e4d8 // bfmmla z24.s, z6.h, z1.h\n"
- "ld1h z6.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- ".inst 0x6461e4f9 // bfmmla z25.s, z7.h, z1.h\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- ".inst 0x6461e51a // bfmmla z26.s, z8.h, z1.h\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- ".inst 0x6461e53b // bfmmla z27.s, z9.h, z1.h\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- ".inst 0x6461e55c // bfmmla z28.s, z10.h, z1.h\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- ".inst 0x6461e57d // bfmmla z29.s, z11.h, z1.h\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- ".inst 0x6461e59e // bfmmla z30.s, z12.h, z1.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- ".inst 0x6461e5bf // bfmmla z31.s, z13.h, z1.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "trn1 z1.d, z4.d, z5.d\n"
- "trn2 z5.d, z4.d, z5.d\n"
- "trn2 z4.d, z2.d, z3.d\n"
- "ld1rqh z2.h, p7/z, [a_ptr2, #-0x10]\n"
- ".inst 0x6460e4d0 // bfmmla z16.s, z6.h, z0.h\n"
- ".inst 0x6460e4f1 // bfmmla z17.s, z7.h, z0.h\n"
- ".inst 0x6460e512 // bfmmla z18.s, z8.h, z0.h\n"
- ".inst 0x6460e533 // bfmmla z19.s, z9.h, z0.h\n"
- ".inst 0x6460e554 // bfmmla z20.s, z10.h, z0.h\n"
- ".inst 0x6460e575 // bfmmla z21.s, z11.h, z0.h\n"
- ".inst 0x6460e596 // bfmmla z22.s, z12.h, z0.h\n"
- ".inst 0x6460e5b7 // bfmmla z23.s, z13.h, z0.h\n"
- "ld1rqh z0.h, p7/z, [%[a_ptr0], #-0x10]\n"
- ".inst 0x6461e4d8 // bfmmla z24.s, z6.h, z1.h\n"
- "ld1h z6.h, p7/z, [%[b_ptr0]]\n"
- ".inst 0x6461e4f9 // bfmmla z25.s, z7.h, z1.h\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x6461e51a // bfmmla z26.s, z8.h, z1.h\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x6461e53b // bfmmla z27.s, z9.h, z1.h\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x6461e55c // bfmmla z28.s, z10.h, z1.h\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x6461e57d // bfmmla z29.s, z11.h, z1.h\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x6461e59e // bfmmla z30.s, z12.h, z1.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x6461e5bf // bfmmla z31.s, z13.h, z1.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "mov z3.h, #0\n"
- "ld1rqh z1.h, p7/z, [a_ptr1, #-0x10]\n"
- ".inst 0x6464e4d0 // bfmmla z16.s, z6.h, z4.h\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- ".inst 0x6464e4f1 // bfmmla z17.s, z7.h, z4.h\n"
- ".inst 0x6464e512 // bfmmla z18.s, z8.h, z4.h\n"
- ".inst 0x6464e533 // bfmmla z19.s, z9.h, z4.h\n"
- ".inst 0x6464e554 // bfmmla z20.s, z10.h, z4.h\n"
- ".inst 0x6464e575 // bfmmla z21.s, z11.h, z4.h\n"
- ".inst 0x6464e596 // bfmmla z22.s, z12.h, z4.h\n"
- ".inst 0x6464e5b7 // bfmmla z23.s, z13.h, z4.h\n"
- "trn1 z4.d, z0.d, z1.d\n"
- ".inst 0x6465e4d8 // bfmmla z24.s, z6.h, z5.h\n"
- "ld1h z6.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- ".inst 0x6465e4f9 // bfmmla z25.s, z7.h, z5.h\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- ".inst 0x6465e51a // bfmmla z26.s, z8.h, z5.h\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- ".inst 0x6465e53b // bfmmla z27.s, z9.h, z5.h\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- ".inst 0x6465e55c // bfmmla z28.s, z10.h, z5.h\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- ".inst 0x6465e57d // bfmmla z29.s, z11.h, z5.h\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- ".inst 0x6465e59e // bfmmla z30.s, z12.h, z5.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- ".inst 0x6465e5bf // bfmmla z31.s, z13.h, z5.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "trn1 z5.d, z2.d, z3.d\n"
- "b.ne 3b\n"
- "2:\n"
- "cbz %[regs], 4f\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "trn2 z1.d, z2.d, z3.d\n"
- "ld1rqh z2.h, p7/z, [%[a_ptr0]]\n"
- ".inst 0x6464e4d0 // bfmmla z16.s, z6.h, z4.h\n"
- "ld1rqh z3.h, p7/z, [a_ptr1]\n"
- ".inst 0x6464e4f1 // bfmmla z17.s, z7.h, z4.h\n"
- ".inst 0x6464e512 // bfmmla z18.s, z8.h, z4.h\n"
- ".inst 0x6464e533 // bfmmla z19.s, z9.h, z4.h\n"
- ".inst 0x6464e554 // bfmmla z20.s, z10.h, z4.h\n"
- ".inst 0x6464e575 // bfmmla z21.s, z11.h, z4.h\n"
- ".inst 0x6464e596 // bfmmla z22.s, z12.h, z4.h\n"
- ".inst 0x6464e5b7 // bfmmla z23.s, z13.h, z4.h\n"
- "ld1rqh z4.h, p7/z, [a_ptr2]\n"
- ".inst 0x6465e4d8 // bfmmla z24.s, z6.h, z5.h\n"
- "ld1h z6.h, p7/z, [%[b_ptr0]]\n"
- ".inst 0x6465e4f9 // bfmmla z25.s, z7.h, z5.h\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x6465e51a // bfmmla z26.s, z8.h, z5.h\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x6465e53b // bfmmla z27.s, z9.h, z5.h\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x6465e55c // bfmmla z28.s, z10.h, z5.h\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x6465e57d // bfmmla z29.s, z11.h, z5.h\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x6465e59e // bfmmla z30.s, z12.h, z5.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x6465e5bf // bfmmla z31.s, z13.h, z5.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "mov z5.h, #0\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- ".inst 0x6460e4d0 // bfmmla z16.s, z6.h, z0.h\n"
- ".inst 0x6460e4f1 // bfmmla z17.s, z7.h, z0.h\n"
- ".inst 0x6460e512 // bfmmla z18.s, z8.h, z0.h\n"
- ".inst 0x6460e533 // bfmmla z19.s, z9.h, z0.h\n"
- ".inst 0x6460e554 // bfmmla z20.s, z10.h, z0.h\n"
- ".inst 0x6460e575 // bfmmla z21.s, z11.h, z0.h\n"
- ".inst 0x6460e596 // bfmmla z22.s, z12.h, z0.h\n"
- ".inst 0x6460e5b7 // bfmmla z23.s, z13.h, z0.h\n"
- "trn1 z0.d, z2.d, z3.d\n"
- ".inst 0x6461e4d8 // bfmmla z24.s, z6.h, z1.h\n"
- "ld1h z6.h, p7/z, [%[b_ptr0]]\n"
- ".inst 0x6461e4f9 // bfmmla z25.s, z7.h, z1.h\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x6461e51a // bfmmla z26.s, z8.h, z1.h\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x6461e53b // bfmmla z27.s, z9.h, z1.h\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x6461e55c // bfmmla z28.s, z10.h, z1.h\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x6461e57d // bfmmla z29.s, z11.h, z1.h\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x6461e59e // bfmmla z30.s, z12.h, z1.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x6461e5bf // bfmmla z31.s, z13.h, z1.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "trn1 z1.d, z4.d, z5.d\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "trn2 z5.d, z4.d, z5.d\n"
- "trn2 z4.d, z2.d, z3.d\n"
- "ld1rqh z2.h, p6/z, [a_ptr2, #0x10]\n"
- ".inst 0x6460e4d0 // bfmmla z16.s, z6.h, z0.h\n"
- "addvl a_ptr2, a_ptr2, #2\n"
- ".inst 0x6460e4f1 // bfmmla z17.s, z7.h, z0.h\n"
- ".inst 0x6460e512 // bfmmla z18.s, z8.h, z0.h\n"
- ".inst 0x6460e533 // bfmmla z19.s, z9.h, z0.h\n"
- ".inst 0x6460e554 // bfmmla z20.s, z10.h, z0.h\n"
- ".inst 0x6460e575 // bfmmla z21.s, z11.h, z0.h\n"
- ".inst 0x6460e596 // bfmmla z22.s, z12.h, z0.h\n"
- ".inst 0x6460e5b7 // bfmmla z23.s, z13.h, z0.h\n"
- "ld1rqh z0.h, p6/z, [%[a_ptr0], #0x10]\n"
- ".inst 0x6461e4d8 // bfmmla z24.s, z6.h, z1.h\n"
- "ld1h z6.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- ".inst 0x6461e4f9 // bfmmla z25.s, z7.h, z1.h\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- ".inst 0x6461e51a // bfmmla z26.s, z8.h, z1.h\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- ".inst 0x6461e53b // bfmmla z27.s, z9.h, z1.h\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- ".inst 0x6461e55c // bfmmla z28.s, z10.h, z1.h\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- ".inst 0x6461e57d // bfmmla z29.s, z11.h, z1.h\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- ".inst 0x6461e59e // bfmmla z30.s, z12.h, z1.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- ".inst 0x6461e5bf // bfmmla z31.s, z13.h, z1.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "mov z3.h, #0\n"
- "ld1rqh z1.h, p6/z, [a_ptr1, #0x10]\n"
- ".inst 0x6464e4d0 // bfmmla z16.s, z6.h, z4.h\n"
- "addvl %[a_ptr0], %[a_ptr0], #2\n"
- ".inst 0x6464e4f1 // bfmmla z17.s, z7.h, z4.h\n"
- "addvl a_ptr1, a_ptr1, #2\n"
- ".inst 0x6464e512 // bfmmla z18.s, z8.h, z4.h\n"
- ".inst 0x6464e533 // bfmmla z19.s, z9.h, z4.h\n"
- ".inst 0x6464e554 // bfmmla z20.s, z10.h, z4.h\n"
- ".inst 0x6464e575 // bfmmla z21.s, z11.h, z4.h\n"
- ".inst 0x6464e596 // bfmmla z22.s, z12.h, z4.h\n"
- ".inst 0x6464e5b7 // bfmmla z23.s, z13.h, z4.h\n"
- "trn1 z4.d, z0.d, z1.d\n"
- ".inst 0x6465e4d8 // bfmmla z24.s, z6.h, z5.h\n"
- ".inst 0x6465e4f9 // bfmmla z25.s, z7.h, z5.h\n"
- ".inst 0x6465e51a // bfmmla z26.s, z8.h, z5.h\n"
- ".inst 0x6465e53b // bfmmla z27.s, z9.h, z5.h\n"
- ".inst 0x6465e55c // bfmmla z28.s, z10.h, z5.h\n"
- ".inst 0x6465e57d // bfmmla z29.s, z11.h, z5.h\n"
- ".inst 0x6465e59e // bfmmla z30.s, z12.h, z5.h\n"
- ".inst 0x6465e5bf // bfmmla z31.s, z13.h, z5.h\n"
- "trn1 z5.d, z2.d, z3.d\n"
- "cbz %[blocks], 5f\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "ld1h z6.h, p7/z, [%[b_ptr0]]\n"
- "trn2 z1.d, z2.d, z3.d\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- ".inst 0x6464e4d0 // bfmmla z16.s, z6.h, z4.h\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x6464e4f1 // bfmmla z17.s, z7.h, z4.h\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x6464e512 // bfmmla z18.s, z8.h, z4.h\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x6465e4d8 // bfmmla z24.s, z6.h, z5.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x6464e533 // bfmmla z19.s, z9.h, z4.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x6464e554 // bfmmla z20.s, z10.h, z4.h\n"
- ".inst 0x6464e575 // bfmmla z21.s, z11.h, z4.h\n"
- ".inst 0x6464e596 // bfmmla z22.s, z12.h, z4.h\n"
- ".inst 0x6464e5b7 // bfmmla z23.s, z13.h, z4.h\n"
- ".inst 0x6465e4f9 // bfmmla z25.s, z7.h, z5.h\n"
- ".inst 0x6465e51a // bfmmla z26.s, z8.h, z5.h\n"
- ".inst 0x6465e53b // bfmmla z27.s, z9.h, z5.h\n"
- ".inst 0x6465e55c // bfmmla z28.s, z10.h, z5.h\n"
- ".inst 0x6465e57d // bfmmla z29.s, z11.h, z5.h\n"
- ".inst 0x6465e59e // bfmmla z30.s, z12.h, z5.h\n"
- ".inst 0x6465e5bf // bfmmla z31.s, z13.h, z5.h\n"
- "b.eq 5f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "ld1h z6.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- ".inst 0x6460e4d0 // bfmmla z16.s, z6.h, z0.h\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- ".inst 0x6460e4f1 // bfmmla z17.s, z7.h, z0.h\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- ".inst 0x6460e512 // bfmmla z18.s, z8.h, z0.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- ".inst 0x6460e533 // bfmmla z19.s, z9.h, z0.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- ".inst 0x6460e554 // bfmmla z20.s, z10.h, z0.h\n"
- ".inst 0x6460e575 // bfmmla z21.s, z11.h, z0.h\n"
- ".inst 0x6460e596 // bfmmla z22.s, z12.h, z0.h\n"
- ".inst 0x6460e5b7 // bfmmla z23.s, z13.h, z0.h\n"
- ".inst 0x6461e4d8 // bfmmla z24.s, z6.h, z1.h\n"
- ".inst 0x6461e4f9 // bfmmla z25.s, z7.h, z1.h\n"
- ".inst 0x6461e51a // bfmmla z26.s, z8.h, z1.h\n"
- ".inst 0x6461e53b // bfmmla z27.s, z9.h, z1.h\n"
- ".inst 0x6461e55c // bfmmla z28.s, z10.h, z1.h\n"
- ".inst 0x6461e57d // bfmmla z29.s, z11.h, z1.h\n"
- ".inst 0x6461e59e // bfmmla z30.s, z12.h, z1.h\n"
- ".inst 0x6461e5bf // bfmmla z31.s, z13.h, z1.h\n"
- "b 5f\n"
- "4:\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "trn2 z1.d, z2.d, z3.d\n"
- "ld1rqh z2.h, p6/z, [%[a_ptr0]]\n"
- ".inst 0x6464e4d0 // bfmmla z16.s, z6.h, z4.h\n"
- "ld1rqh z3.h, p6/z, [a_ptr1]\n"
- ".inst 0x6464e4f1 // bfmmla z17.s, z7.h, z4.h\n"
- "addvl %[a_ptr0], %[a_ptr0], #1\n"
- ".inst 0x6464e512 // bfmmla z18.s, z8.h, z4.h\n"
- "addvl a_ptr1, a_ptr1, #1\n"
- ".inst 0x6464e533 // bfmmla z19.s, z9.h, z4.h\n"
- ".inst 0x6464e554 // bfmmla z20.s, z10.h, z4.h\n"
- ".inst 0x6464e575 // bfmmla z21.s, z11.h, z4.h\n"
- ".inst 0x6464e596 // bfmmla z22.s, z12.h, z4.h\n"
- ".inst 0x6464e5b7 // bfmmla z23.s, z13.h, z4.h\n"
- "ld1rqh z4.h, p6/z, [a_ptr2]\n"
- ".inst 0x6465e4d8 // bfmmla z24.s, z6.h, z5.h\n"
- "ld1h z6.h, p7/z, [%[b_ptr0]]\n"
- ".inst 0x6465e4f9 // bfmmla z25.s, z7.h, z5.h\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x6465e51a // bfmmla z26.s, z8.h, z5.h\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x6465e53b // bfmmla z27.s, z9.h, z5.h\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x6465e55c // bfmmla z28.s, z10.h, z5.h\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x6465e57d // bfmmla z29.s, z11.h, z5.h\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x6465e59e // bfmmla z30.s, z12.h, z5.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x6465e5bf // bfmmla z31.s, z13.h, z5.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "mov z5.h, #0\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- ".inst 0x6460e4d0 // bfmmla z16.s, z6.h, z0.h\n"
- "addvl a_ptr2, a_ptr2, #1\n"
- ".inst 0x6460e4f1 // bfmmla z17.s, z7.h, z0.h\n"
- ".inst 0x6460e512 // bfmmla z18.s, z8.h, z0.h\n"
- ".inst 0x6460e533 // bfmmla z19.s, z9.h, z0.h\n"
- ".inst 0x6460e554 // bfmmla z20.s, z10.h, z0.h\n"
- ".inst 0x6460e575 // bfmmla z21.s, z11.h, z0.h\n"
- ".inst 0x6460e596 // bfmmla z22.s, z12.h, z0.h\n"
- ".inst 0x6460e5b7 // bfmmla z23.s, z13.h, z0.h\n"
- "trn1 z0.d, z2.d, z3.d\n"
- ".inst 0x6461e4d8 // bfmmla z24.s, z6.h, z1.h\n"
- ".inst 0x6461e4f9 // bfmmla z25.s, z7.h, z1.h\n"
- ".inst 0x6461e51a // bfmmla z26.s, z8.h, z1.h\n"
- ".inst 0x6461e53b // bfmmla z27.s, z9.h, z1.h\n"
- ".inst 0x6461e55c // bfmmla z28.s, z10.h, z1.h\n"
- ".inst 0x6461e57d // bfmmla z29.s, z11.h, z1.h\n"
- ".inst 0x6461e59e // bfmmla z30.s, z12.h, z1.h\n"
- ".inst 0x6461e5bf // bfmmla z31.s, z13.h, z1.h\n"
- "trn1 z1.d, z4.d, z5.d\n"
- "cbz %[blocks], 5f\n"
- "trn2 z5.d, z4.d, z5.d\n"
- "ld1h z6.h, p7/z, [%[b_ptr0]]\n"
- "trn2 z4.d, z2.d, z3.d\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- ".inst 0x6460e4d0 // bfmmla z16.s, z6.h, z0.h\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x6460e4f1 // bfmmla z17.s, z7.h, z0.h\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x6460e512 // bfmmla z18.s, z8.h, z0.h\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x6461e4d8 // bfmmla z24.s, z6.h, z1.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x6460e533 // bfmmla z19.s, z9.h, z0.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x6460e554 // bfmmla z20.s, z10.h, z0.h\n"
- ".inst 0x6460e575 // bfmmla z21.s, z11.h, z0.h\n"
- ".inst 0x6460e596 // bfmmla z22.s, z12.h, z0.h\n"
- ".inst 0x6460e5b7 // bfmmla z23.s, z13.h, z0.h\n"
- ".inst 0x6461e4f9 // bfmmla z25.s, z7.h, z1.h\n"
- ".inst 0x6461e51a // bfmmla z26.s, z8.h, z1.h\n"
- ".inst 0x6461e53b // bfmmla z27.s, z9.h, z1.h\n"
- ".inst 0x6461e55c // bfmmla z28.s, z10.h, z1.h\n"
- ".inst 0x6461e57d // bfmmla z29.s, z11.h, z1.h\n"
- ".inst 0x6461e59e // bfmmla z30.s, z12.h, z1.h\n"
- ".inst 0x6461e5bf // bfmmla z31.s, z13.h, z1.h\n"
- "b.eq 5f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "ld1h z6.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- ".inst 0x6464e4d0 // bfmmla z16.s, z6.h, z4.h\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- ".inst 0x6464e4f1 // bfmmla z17.s, z7.h, z4.h\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- ".inst 0x6464e512 // bfmmla z18.s, z8.h, z4.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- ".inst 0x6464e533 // bfmmla z19.s, z9.h, z4.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- ".inst 0x6464e554 // bfmmla z20.s, z10.h, z4.h\n"
- ".inst 0x6464e575 // bfmmla z21.s, z11.h, z4.h\n"
- ".inst 0x6464e596 // bfmmla z22.s, z12.h, z4.h\n"
- ".inst 0x6464e5b7 // bfmmla z23.s, z13.h, z4.h\n"
- ".inst 0x6465e4d8 // bfmmla z24.s, z6.h, z5.h\n"
- ".inst 0x6465e4f9 // bfmmla z25.s, z7.h, z5.h\n"
- ".inst 0x6465e51a // bfmmla z26.s, z8.h, z5.h\n"
- ".inst 0x6465e53b // bfmmla z27.s, z9.h, z5.h\n"
- ".inst 0x6465e55c // bfmmla z28.s, z10.h, z5.h\n"
- ".inst 0x6465e57d // bfmmla z29.s, z11.h, z5.h\n"
- ".inst 0x6465e59e // bfmmla z30.s, z12.h, z5.h\n"
- ".inst 0x6465e5bf // bfmmla z31.s, z13.h, z5.h\n"
- "5:\n"
- "ld1rw z14.s, p7/z, [%[minptr]]\n"
- "ld1rw z15.s, p7/z, [%[maxptr]]\n"
- "fmax z16.s, p7/m, z16.s, z14.s\n"
- "fmax z17.s, p7/m, z17.s, z14.s\n"
- "fmax z18.s, p7/m, z18.s, z14.s\n"
- "fmax z19.s, p7/m, z19.s, z14.s\n"
- "fmin z16.s, p7/m, z16.s, z15.s\n"
- "fmin z17.s, p7/m, z17.s, z15.s\n"
- "fmin z18.s, p7/m, z18.s, z15.s\n"
- "fmin z19.s, p7/m, z19.s, z15.s\n"
- "fmax z20.s, p7/m, z20.s, z14.s\n"
- "uzp1 z0.s, z16.s, z17.s\n"
- "uzp2 z1.s, z16.s, z17.s\n"
- "uzp1 z2.s, z18.s, z19.s\n"
- "uzp2 z3.s, z18.s, z19.s\n"
- "st1w z0.s, p0, [%[c_ptr0]]\n"
- "fmin z20.s, p7/m, z20.s, z15.s\n"
- "fmax z21.s, p7/m, z21.s, z14.s\n"
- "fmax z22.s, p7/m, z22.s, z14.s\n"
- "st1w z1.s, p0, [c_ptr1]\n"
- "fmax z23.s, p7/m, z23.s, z14.s\n"
- "fmax z24.s, p7/m, z24.s, z14.s\n"
- "fmin z21.s, p7/m, z21.s, z15.s\n"
- "st1w z2.s, p1, [%[c_ptr0], #1, MUL VL]\n"
- "fmin z22.s, p7/m, z22.s, z15.s\n"
- "fmin z23.s, p7/m, z23.s, z15.s\n"
- "fmin z24.s, p7/m, z24.s, z15.s\n"
- "st1w z3.s, p1, [c_ptr1, #1, MUL VL]\n"
- "uzp1 z4.s, z20.s, z21.s\n"
- "uzp2 z5.s, z20.s, z21.s\n"
- "uzp1 z6.s, z22.s, z23.s\n"
- "uzp2 z7.s, z22.s, z23.s\n"
- "st1w z4.s, p2, [%[c_ptr0], #2, MUL VL]\n"
- "fmax z25.s, p7/m, z25.s, z14.s\n"
- "fmax z26.s, p7/m, z26.s, z14.s\n"
- "fmax z27.s, p7/m, z27.s, z14.s\n"
- "st1w z5.s, p2, [c_ptr1, #2, MUL VL]\n"
- "fmax z28.s, p7/m, z28.s, z14.s\n"
- "fmin z25.s, p7/m, z25.s, z15.s\n"
- "fmin z26.s, p7/m, z26.s, z15.s\n"
- "st1w z6.s, p3, [%[c_ptr0], #3, MUL VL]\n"
- "fmin z27.s, p7/m, z27.s, z15.s\n"
- "addvl %[c_ptr0], %[c_ptr0], #4\n"
- "uzp1 z8.s, z24.s, z25.s\n"
- "st1w z7.s, p3, [c_ptr1, #3, MUL VL]\n"
- "fmin z28.s, p7/m, z28.s, z15.s\n"
- "uzp1 z9.s, z26.s, z27.s\n"
- "fmax z29.s, p7/m, z29.s, z14.s\n"
- "st1w z8.s, p0, [c_ptr2]\n"
- "fmax z30.s, p7/m, z30.s, z14.s\n"
- "fmax z31.s, p7/m, z31.s, z14.s\n"
- "fmin z29.s, p7/m, z29.s, z15.s\n"
- "st1w z9.s, p1, [c_ptr2, #1, MUL VL]\n"
- "fmin z30.s, p7/m, z30.s, z15.s\n"
- "fmin z31.s, p7/m, z31.s, z15.s\n"
- "uzp1 z10.s, z28.s, z29.s\n"
- "uzp1 z11.s, z30.s, z31.s\n"
- "st1w z10.s, p2, [c_ptr2, #2, MUL VL]\n"
- "st1w z11.s, p3, [c_ptr2, #3, MUL VL]\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks)
- : [width] "r" (width), [accumulate] "r" (static_cast<uint64_t>(accumulate)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers)
- : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "cc", "memory"
- );
- break;
- default:
- case 4:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "a_ptr3 .req X2\n"
- "c_ptr1 .req X3\n"
- "c_ptr2 .req X4\n"
- "c_ptr3 .req X5\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "add a_ptr3, a_ptr2, %[lda]\n"
- "add c_ptr3, c_ptr2, %[ldc]\n"
- "whilelt p6.h, %[temp], %[leftovers]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "ptrue p7.h\n"
- "whilelt p1.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "whilelt p2.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "whilelt p3.s, %[temp], %[width]\n"
- "cbnz %[accumulate], 1f\n"
- "ld1w z15.s, p0/z, [%[biasptr]]\n"
- "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ld1rqh z1.h, p7/z, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "zip1 z16.s, z15.s, z15.s\n"
- "ld1rqh z2.h, p7/z, [a_ptr2]\n"
- "zip2 z17.s, z15.s, z15.s\n"
- "ld1w z15.s, p1/z, [%[biasptr], #1, MUL VL]\n"
- "trn1 z4.d, z0.d, z1.d\n"
- "ld1rqh z3.h, p7/z, [a_ptr3]\n"
- "ld1h z6.h, p7/z, [%[b_ptr0]]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "zip1 z18.s, z15.s, z15.s\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "zip2 z19.s, z15.s, z15.s\n"
- "ld1w z15.s, p2/z, [%[biasptr], #2, MUL VL]\n"
- "trn1 z5.d, z2.d, z3.d\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "mov z24.d, z16.d\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "mov z25.d, z17.d\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "zip1 z20.s, z15.s, z15.s\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "zip2 z21.s, z15.s, z15.s\n"
- "ld1w z15.s, p3/z, [%[biasptr], #3, MUL VL]\n"
- "mov z26.d, z18.d\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "mov z27.d, z19.d\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "mov z28.d, z20.d\n"
- "add a_ptr3, a_ptr3, #0x10\n"
- "zip1 z22.s, z15.s, z15.s\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "zip2 z23.s, z15.s, z15.s\n"
- "mov z29.d, z21.d\n"
- "mov z30.d, z22.d\n"
- "mov z31.d, z23.d\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "ld1w z13.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z14.s, p0/z, [c_ptr1]\n"
- "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ld1rqh z1.h, p7/z, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "zip1 z16.s, z13.s, z14.s\n"
- "ld1rqh z2.h, p7/z, [a_ptr2]\n"
- "zip2 z17.s, z13.s, z14.s\n"
- "ld1w z13.s, p1/z, [%[c_ptr0], #1, MUL VL]\n"
- "trn1 z4.d, z0.d, z1.d\n"
- "ld1w z14.s, p1/z, [c_ptr1, #1, MUL VL]\n"
- "ld1rqh z3.h, p7/z, [a_ptr3]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "ld1h z6.h, p7/z, [%[b_ptr0]]\n"
- "add a_ptr3, a_ptr3, #0x10\n"
- "zip1 z18.s, z13.s, z14.s\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "zip2 z19.s, z13.s, z14.s\n"
- "ld1w z13.s, p2/z, [%[c_ptr0], #2, MUL VL]\n"
- "trn1 z5.d, z2.d, z3.d\n"
- "ld1w z14.s, p2/z, [c_ptr1, #2, MUL VL]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "zip1 z20.s, z13.s, z14.s\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "zip2 z21.s, z13.s, z14.s\n"
- "ld1w z13.s, p3/z, [%[c_ptr0], #3, MUL VL]\n"
- "ld1w z14.s, p3/z, [c_ptr1, #3, MUL VL]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "zip1 z22.s, z13.s, z14.s\n"
- "zip2 z23.s, z13.s, z14.s\n"
- "ld1w z13.s, p0/z, [c_ptr2]\n"
- "ld1w z14.s, p0/z, [c_ptr3]\n"
- "zip1 z24.s, z13.s, z14.s\n"
- "zip2 z25.s, z13.s, z14.s\n"
- "ld1w z13.s, p1/z, [c_ptr2, #1, MUL VL]\n"
- "ld1w z14.s, p1/z, [c_ptr3, #1, MUL VL]\n"
- "zip1 z26.s, z13.s, z14.s\n"
- "zip2 z27.s, z13.s, z14.s\n"
- "ld1w z13.s, p2/z, [c_ptr2, #2, MUL VL]\n"
- "ld1w z14.s, p2/z, [c_ptr3, #2, MUL VL]\n"
- "zip1 z28.s, z13.s, z14.s\n"
- "zip2 z29.s, z13.s, z14.s\n"
- "ld1w z13.s, p3/z, [c_ptr2, #3, MUL VL]\n"
- "ld1w z14.s, p3/z, [c_ptr3, #3, MUL VL]\n"
- "zip1 z30.s, z13.s, z14.s\n"
- "zip2 z31.s, z13.s, z14.s\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "cbz %[loops], 2f\n"
- "3:\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "subs %[loops], %[loops], #0x1\n"
- "trn2 z1.d, z2.d, z3.d\n"
- "ld1rqh z2.h, p7/z, [%[a_ptr0]]\n"
- ".inst 0x6464e4d0 // bfmmla z16.s, z6.h, z4.h\n"
- "ld1rqh z3.h, p7/z, [a_ptr1]\n"
- ".inst 0x6464e4f1 // bfmmla z17.s, z7.h, z4.h\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
- ".inst 0x6464e512 // bfmmla z18.s, z8.h, z4.h\n"
- "add a_ptr1, a_ptr1, #0x20\n"
- ".inst 0x6464e533 // bfmmla z19.s, z9.h, z4.h\n"
- ".inst 0x6464e554 // bfmmla z20.s, z10.h, z4.h\n"
- ".inst 0x6464e575 // bfmmla z21.s, z11.h, z4.h\n"
- ".inst 0x6464e596 // bfmmla z22.s, z12.h, z4.h\n"
- ".inst 0x6464e5b7 // bfmmla z23.s, z13.h, z4.h\n"
- "ld1rqh z4.h, p7/z, [a_ptr2]\n"
- ".inst 0x6465e4d8 // bfmmla z24.s, z6.h, z5.h\n"
- "ld1h z6.h, p7/z, [%[b_ptr0]]\n"
- ".inst 0x6465e4f9 // bfmmla z25.s, z7.h, z5.h\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x6465e51a // bfmmla z26.s, z8.h, z5.h\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x6465e53b // bfmmla z27.s, z9.h, z5.h\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x6465e55c // bfmmla z28.s, z10.h, z5.h\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x6465e57d // bfmmla z29.s, z11.h, z5.h\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x6465e59e // bfmmla z30.s, z12.h, z5.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x6465e5bf // bfmmla z31.s, z13.h, z5.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x6460e4d0 // bfmmla z16.s, z6.h, z0.h\n"
- "ld1rqh z5.h, p7/z, [a_ptr3]\n"
- ".inst 0x6460e4f1 // bfmmla z17.s, z7.h, z0.h\n"
- "add a_ptr2, a_ptr2, #0x20\n"
- ".inst 0x6460e512 // bfmmla z18.s, z8.h, z0.h\n"
- "add a_ptr3, a_ptr3, #0x20\n"
- ".inst 0x6460e533 // bfmmla z19.s, z9.h, z0.h\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- ".inst 0x6460e554 // bfmmla z20.s, z10.h, z0.h\n"
- ".inst 0x6460e575 // bfmmla z21.s, z11.h, z0.h\n"
- ".inst 0x6460e596 // bfmmla z22.s, z12.h, z0.h\n"
- ".inst 0x6460e5b7 // bfmmla z23.s, z13.h, z0.h\n"
- "trn1 z0.d, z2.d, z3.d\n"
- ".inst 0x6461e4d8 // bfmmla z24.s, z6.h, z1.h\n"
- "ld1h z6.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- ".inst 0x6461e4f9 // bfmmla z25.s, z7.h, z1.h\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- ".inst 0x6461e51a // bfmmla z26.s, z8.h, z1.h\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- ".inst 0x6461e53b // bfmmla z27.s, z9.h, z1.h\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- ".inst 0x6461e55c // bfmmla z28.s, z10.h, z1.h\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- ".inst 0x6461e57d // bfmmla z29.s, z11.h, z1.h\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- ".inst 0x6461e59e // bfmmla z30.s, z12.h, z1.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- ".inst 0x6461e5bf // bfmmla z31.s, z13.h, z1.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "trn1 z1.d, z4.d, z5.d\n"
- "trn2 z5.d, z4.d, z5.d\n"
- "trn2 z4.d, z2.d, z3.d\n"
- "ld1rqh z2.h, p7/z, [a_ptr2, #-0x10]\n"
- ".inst 0x6460e4d0 // bfmmla z16.s, z6.h, z0.h\n"
- "ld1rqh z3.h, p7/z, [a_ptr3, #-0x10]\n"
- ".inst 0x6460e4f1 // bfmmla z17.s, z7.h, z0.h\n"
- ".inst 0x6460e512 // bfmmla z18.s, z8.h, z0.h\n"
- ".inst 0x6460e533 // bfmmla z19.s, z9.h, z0.h\n"
- ".inst 0x6460e554 // bfmmla z20.s, z10.h, z0.h\n"
- ".inst 0x6460e575 // bfmmla z21.s, z11.h, z0.h\n"
- ".inst 0x6460e596 // bfmmla z22.s, z12.h, z0.h\n"
- ".inst 0x6460e5b7 // bfmmla z23.s, z13.h, z0.h\n"
- "ld1rqh z0.h, p7/z, [%[a_ptr0], #-0x10]\n"
- ".inst 0x6461e4d8 // bfmmla z24.s, z6.h, z1.h\n"
- "ld1h z6.h, p7/z, [%[b_ptr0]]\n"
- ".inst 0x6461e4f9 // bfmmla z25.s, z7.h, z1.h\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x6461e51a // bfmmla z26.s, z8.h, z1.h\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x6461e53b // bfmmla z27.s, z9.h, z1.h\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x6461e55c // bfmmla z28.s, z10.h, z1.h\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x6461e57d // bfmmla z29.s, z11.h, z1.h\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x6461e59e // bfmmla z30.s, z12.h, z1.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x6461e5bf // bfmmla z31.s, z13.h, z1.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x6464e4d0 // bfmmla z16.s, z6.h, z4.h\n"
- "ld1rqh z1.h, p7/z, [a_ptr1, #-0x10]\n"
- ".inst 0x6464e4f1 // bfmmla z17.s, z7.h, z4.h\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- ".inst 0x6464e512 // bfmmla z18.s, z8.h, z4.h\n"
- ".inst 0x6464e533 // bfmmla z19.s, z9.h, z4.h\n"
- ".inst 0x6464e554 // bfmmla z20.s, z10.h, z4.h\n"
- ".inst 0x6464e575 // bfmmla z21.s, z11.h, z4.h\n"
- ".inst 0x6464e596 // bfmmla z22.s, z12.h, z4.h\n"
- ".inst 0x6464e5b7 // bfmmla z23.s, z13.h, z4.h\n"
- "trn1 z4.d, z0.d, z1.d\n"
- ".inst 0x6465e4d8 // bfmmla z24.s, z6.h, z5.h\n"
- "ld1h z6.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- ".inst 0x6465e4f9 // bfmmla z25.s, z7.h, z5.h\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- ".inst 0x6465e51a // bfmmla z26.s, z8.h, z5.h\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- ".inst 0x6465e53b // bfmmla z27.s, z9.h, z5.h\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- ".inst 0x6465e55c // bfmmla z28.s, z10.h, z5.h\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- ".inst 0x6465e57d // bfmmla z29.s, z11.h, z5.h\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- ".inst 0x6465e59e // bfmmla z30.s, z12.h, z5.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- ".inst 0x6465e5bf // bfmmla z31.s, z13.h, z5.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "trn1 z5.d, z2.d, z3.d\n"
- "b.ne 3b\n"
- "2:\n"
- "cbz %[regs], 4f\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "trn2 z1.d, z2.d, z3.d\n"
- "ld1rqh z2.h, p7/z, [%[a_ptr0]]\n"
- ".inst 0x6464e4d0 // bfmmla z16.s, z6.h, z4.h\n"
- "ld1rqh z3.h, p7/z, [a_ptr1]\n"
- ".inst 0x6464e4f1 // bfmmla z17.s, z7.h, z4.h\n"
- ".inst 0x6464e512 // bfmmla z18.s, z8.h, z4.h\n"
- ".inst 0x6464e533 // bfmmla z19.s, z9.h, z4.h\n"
- ".inst 0x6464e554 // bfmmla z20.s, z10.h, z4.h\n"
- ".inst 0x6464e575 // bfmmla z21.s, z11.h, z4.h\n"
- ".inst 0x6464e596 // bfmmla z22.s, z12.h, z4.h\n"
- ".inst 0x6464e5b7 // bfmmla z23.s, z13.h, z4.h\n"
- "ld1rqh z4.h, p7/z, [a_ptr2]\n"
- ".inst 0x6465e4d8 // bfmmla z24.s, z6.h, z5.h\n"
- "ld1h z6.h, p7/z, [%[b_ptr0]]\n"
- ".inst 0x6465e4f9 // bfmmla z25.s, z7.h, z5.h\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x6465e51a // bfmmla z26.s, z8.h, z5.h\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x6465e53b // bfmmla z27.s, z9.h, z5.h\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x6465e55c // bfmmla z28.s, z10.h, z5.h\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x6465e57d // bfmmla z29.s, z11.h, z5.h\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x6465e59e // bfmmla z30.s, z12.h, z5.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x6465e5bf // bfmmla z31.s, z13.h, z5.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x6460e4d0 // bfmmla z16.s, z6.h, z0.h\n"
- "ld1rqh z5.h, p7/z, [a_ptr3]\n"
- ".inst 0x6460e4f1 // bfmmla z17.s, z7.h, z0.h\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- ".inst 0x6460e512 // bfmmla z18.s, z8.h, z0.h\n"
- ".inst 0x6460e533 // bfmmla z19.s, z9.h, z0.h\n"
- ".inst 0x6460e554 // bfmmla z20.s, z10.h, z0.h\n"
- ".inst 0x6460e575 // bfmmla z21.s, z11.h, z0.h\n"
- ".inst 0x6460e596 // bfmmla z22.s, z12.h, z0.h\n"
- ".inst 0x6460e5b7 // bfmmla z23.s, z13.h, z0.h\n"
- "trn1 z0.d, z2.d, z3.d\n"
- ".inst 0x6461e4d8 // bfmmla z24.s, z6.h, z1.h\n"
- "ld1h z6.h, p7/z, [%[b_ptr0]]\n"
- ".inst 0x6461e4f9 // bfmmla z25.s, z7.h, z1.h\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x6461e51a // bfmmla z26.s, z8.h, z1.h\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x6461e53b // bfmmla z27.s, z9.h, z1.h\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x6461e55c // bfmmla z28.s, z10.h, z1.h\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x6461e57d // bfmmla z29.s, z11.h, z1.h\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x6461e59e // bfmmla z30.s, z12.h, z1.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x6461e5bf // bfmmla z31.s, z13.h, z1.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "trn1 z1.d, z4.d, z5.d\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "trn2 z5.d, z4.d, z5.d\n"
- "trn2 z4.d, z2.d, z3.d\n"
- "ld1rqh z2.h, p6/z, [a_ptr2, #0x10]\n"
- ".inst 0x6460e4d0 // bfmmla z16.s, z6.h, z0.h\n"
- "ld1rqh z3.h, p6/z, [a_ptr3, #0x10]\n"
- ".inst 0x6460e4f1 // bfmmla z17.s, z7.h, z0.h\n"
- "addvl a_ptr2, a_ptr2, #2\n"
- ".inst 0x6460e512 // bfmmla z18.s, z8.h, z0.h\n"
- "addvl a_ptr3, a_ptr3, #2\n"
- ".inst 0x6460e533 // bfmmla z19.s, z9.h, z0.h\n"
- ".inst 0x6460e554 // bfmmla z20.s, z10.h, z0.h\n"
- ".inst 0x6460e575 // bfmmla z21.s, z11.h, z0.h\n"
- ".inst 0x6460e596 // bfmmla z22.s, z12.h, z0.h\n"
- ".inst 0x6460e5b7 // bfmmla z23.s, z13.h, z0.h\n"
- "ld1rqh z0.h, p6/z, [%[a_ptr0], #0x10]\n"
- ".inst 0x6461e4d8 // bfmmla z24.s, z6.h, z1.h\n"
- "ld1h z6.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- ".inst 0x6461e4f9 // bfmmla z25.s, z7.h, z1.h\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- ".inst 0x6461e51a // bfmmla z26.s, z8.h, z1.h\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- ".inst 0x6461e53b // bfmmla z27.s, z9.h, z1.h\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- ".inst 0x6461e55c // bfmmla z28.s, z10.h, z1.h\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- ".inst 0x6461e57d // bfmmla z29.s, z11.h, z1.h\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- ".inst 0x6461e59e // bfmmla z30.s, z12.h, z1.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- ".inst 0x6461e5bf // bfmmla z31.s, z13.h, z1.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- ".inst 0x6464e4d0 // bfmmla z16.s, z6.h, z4.h\n"
- "ld1rqh z1.h, p6/z, [a_ptr1, #0x10]\n"
- ".inst 0x6464e4f1 // bfmmla z17.s, z7.h, z4.h\n"
- "addvl %[a_ptr0], %[a_ptr0], #2\n"
- ".inst 0x6464e512 // bfmmla z18.s, z8.h, z4.h\n"
- "addvl a_ptr1, a_ptr1, #2\n"
- ".inst 0x6464e533 // bfmmla z19.s, z9.h, z4.h\n"
- ".inst 0x6464e554 // bfmmla z20.s, z10.h, z4.h\n"
- ".inst 0x6464e575 // bfmmla z21.s, z11.h, z4.h\n"
- ".inst 0x6464e596 // bfmmla z22.s, z12.h, z4.h\n"
- ".inst 0x6464e5b7 // bfmmla z23.s, z13.h, z4.h\n"
- "trn1 z4.d, z0.d, z1.d\n"
- ".inst 0x6465e4d8 // bfmmla z24.s, z6.h, z5.h\n"
- ".inst 0x6465e4f9 // bfmmla z25.s, z7.h, z5.h\n"
- ".inst 0x6465e51a // bfmmla z26.s, z8.h, z5.h\n"
- ".inst 0x6465e53b // bfmmla z27.s, z9.h, z5.h\n"
- ".inst 0x6465e55c // bfmmla z28.s, z10.h, z5.h\n"
- ".inst 0x6465e57d // bfmmla z29.s, z11.h, z5.h\n"
- ".inst 0x6465e59e // bfmmla z30.s, z12.h, z5.h\n"
- ".inst 0x6465e5bf // bfmmla z31.s, z13.h, z5.h\n"
- "trn1 z5.d, z2.d, z3.d\n"
- "cbz %[blocks], 5f\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "ld1h z6.h, p7/z, [%[b_ptr0]]\n"
- "trn2 z1.d, z2.d, z3.d\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- ".inst 0x6464e4d0 // bfmmla z16.s, z6.h, z4.h\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x6464e4f1 // bfmmla z17.s, z7.h, z4.h\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x6464e512 // bfmmla z18.s, z8.h, z4.h\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x6465e4d8 // bfmmla z24.s, z6.h, z5.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x6464e533 // bfmmla z19.s, z9.h, z4.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x6464e554 // bfmmla z20.s, z10.h, z4.h\n"
- ".inst 0x6464e575 // bfmmla z21.s, z11.h, z4.h\n"
- ".inst 0x6464e596 // bfmmla z22.s, z12.h, z4.h\n"
- ".inst 0x6464e5b7 // bfmmla z23.s, z13.h, z4.h\n"
- ".inst 0x6465e4f9 // bfmmla z25.s, z7.h, z5.h\n"
- ".inst 0x6465e51a // bfmmla z26.s, z8.h, z5.h\n"
- ".inst 0x6465e53b // bfmmla z27.s, z9.h, z5.h\n"
- ".inst 0x6465e55c // bfmmla z28.s, z10.h, z5.h\n"
- ".inst 0x6465e57d // bfmmla z29.s, z11.h, z5.h\n"
- ".inst 0x6465e59e // bfmmla z30.s, z12.h, z5.h\n"
- ".inst 0x6465e5bf // bfmmla z31.s, z13.h, z5.h\n"
- "b.eq 5f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "ld1h z6.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- ".inst 0x6460e4d0 // bfmmla z16.s, z6.h, z0.h\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- ".inst 0x6460e4f1 // bfmmla z17.s, z7.h, z0.h\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- ".inst 0x6460e512 // bfmmla z18.s, z8.h, z0.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- ".inst 0x6460e533 // bfmmla z19.s, z9.h, z0.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- ".inst 0x6460e554 // bfmmla z20.s, z10.h, z0.h\n"
- ".inst 0x6460e575 // bfmmla z21.s, z11.h, z0.h\n"
- ".inst 0x6460e596 // bfmmla z22.s, z12.h, z0.h\n"
- ".inst 0x6460e5b7 // bfmmla z23.s, z13.h, z0.h\n"
- ".inst 0x6461e4d8 // bfmmla z24.s, z6.h, z1.h\n"
- ".inst 0x6461e4f9 // bfmmla z25.s, z7.h, z1.h\n"
- ".inst 0x6461e51a // bfmmla z26.s, z8.h, z1.h\n"
- ".inst 0x6461e53b // bfmmla z27.s, z9.h, z1.h\n"
- ".inst 0x6461e55c // bfmmla z28.s, z10.h, z1.h\n"
- ".inst 0x6461e57d // bfmmla z29.s, z11.h, z1.h\n"
- ".inst 0x6461e59e // bfmmla z30.s, z12.h, z1.h\n"
- ".inst 0x6461e5bf // bfmmla z31.s, z13.h, z1.h\n"
- "b 5f\n"
- "4:\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "trn2 z1.d, z2.d, z3.d\n"
- "ld1rqh z2.h, p6/z, [%[a_ptr0]]\n"
- ".inst 0x6464e4d0 // bfmmla z16.s, z6.h, z4.h\n"
- "ld1rqh z3.h, p6/z, [a_ptr1]\n"
- ".inst 0x6464e4f1 // bfmmla z17.s, z7.h, z4.h\n"
- "addvl %[a_ptr0], %[a_ptr0], #1\n"
- ".inst 0x6464e512 // bfmmla z18.s, z8.h, z4.h\n"
- "addvl a_ptr1, a_ptr1, #1\n"
- ".inst 0x6464e533 // bfmmla z19.s, z9.h, z4.h\n"
- ".inst 0x6464e554 // bfmmla z20.s, z10.h, z4.h\n"
- ".inst 0x6464e575 // bfmmla z21.s, z11.h, z4.h\n"
- ".inst 0x6464e596 // bfmmla z22.s, z12.h, z4.h\n"
- ".inst 0x6464e5b7 // bfmmla z23.s, z13.h, z4.h\n"
- "ld1rqh z4.h, p6/z, [a_ptr2]\n"
- ".inst 0x6465e4d8 // bfmmla z24.s, z6.h, z5.h\n"
- "ld1h z6.h, p7/z, [%[b_ptr0]]\n"
- ".inst 0x6465e4f9 // bfmmla z25.s, z7.h, z5.h\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x6465e51a // bfmmla z26.s, z8.h, z5.h\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x6465e53b // bfmmla z27.s, z9.h, z5.h\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x6465e55c // bfmmla z28.s, z10.h, z5.h\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x6465e57d // bfmmla z29.s, z11.h, z5.h\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x6465e59e // bfmmla z30.s, z12.h, z5.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x6465e5bf // bfmmla z31.s, z13.h, z5.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x6460e4d0 // bfmmla z16.s, z6.h, z0.h\n"
- "ld1rqh z5.h, p6/z, [a_ptr3]\n"
- ".inst 0x6460e4f1 // bfmmla z17.s, z7.h, z0.h\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- ".inst 0x6460e512 // bfmmla z18.s, z8.h, z0.h\n"
- "addvl a_ptr2, a_ptr2, #1\n"
- ".inst 0x6460e533 // bfmmla z19.s, z9.h, z0.h\n"
- "addvl a_ptr3, a_ptr3, #1\n"
- ".inst 0x6460e554 // bfmmla z20.s, z10.h, z0.h\n"
- ".inst 0x6460e575 // bfmmla z21.s, z11.h, z0.h\n"
- ".inst 0x6460e596 // bfmmla z22.s, z12.h, z0.h\n"
- ".inst 0x6460e5b7 // bfmmla z23.s, z13.h, z0.h\n"
- "trn1 z0.d, z2.d, z3.d\n"
- ".inst 0x6461e4d8 // bfmmla z24.s, z6.h, z1.h\n"
- ".inst 0x6461e4f9 // bfmmla z25.s, z7.h, z1.h\n"
- ".inst 0x6461e51a // bfmmla z26.s, z8.h, z1.h\n"
- ".inst 0x6461e53b // bfmmla z27.s, z9.h, z1.h\n"
- ".inst 0x6461e55c // bfmmla z28.s, z10.h, z1.h\n"
- ".inst 0x6461e57d // bfmmla z29.s, z11.h, z1.h\n"
- ".inst 0x6461e59e // bfmmla z30.s, z12.h, z1.h\n"
- ".inst 0x6461e5bf // bfmmla z31.s, z13.h, z1.h\n"
- "trn1 z1.d, z4.d, z5.d\n"
- "cbz %[blocks], 5f\n"
- "trn2 z5.d, z4.d, z5.d\n"
- "ld1h z6.h, p7/z, [%[b_ptr0]]\n"
- "trn2 z4.d, z2.d, z3.d\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- ".inst 0x6460e4d0 // bfmmla z16.s, z6.h, z0.h\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x6460e4f1 // bfmmla z17.s, z7.h, z0.h\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x6460e512 // bfmmla z18.s, z8.h, z0.h\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x6461e4d8 // bfmmla z24.s, z6.h, z1.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x6460e533 // bfmmla z19.s, z9.h, z0.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x6460e554 // bfmmla z20.s, z10.h, z0.h\n"
- ".inst 0x6460e575 // bfmmla z21.s, z11.h, z0.h\n"
- ".inst 0x6460e596 // bfmmla z22.s, z12.h, z0.h\n"
- ".inst 0x6460e5b7 // bfmmla z23.s, z13.h, z0.h\n"
- ".inst 0x6461e4f9 // bfmmla z25.s, z7.h, z1.h\n"
- ".inst 0x6461e51a // bfmmla z26.s, z8.h, z1.h\n"
- ".inst 0x6461e53b // bfmmla z27.s, z9.h, z1.h\n"
- ".inst 0x6461e55c // bfmmla z28.s, z10.h, z1.h\n"
- ".inst 0x6461e57d // bfmmla z29.s, z11.h, z1.h\n"
- ".inst 0x6461e59e // bfmmla z30.s, z12.h, z1.h\n"
- ".inst 0x6461e5bf // bfmmla z31.s, z13.h, z1.h\n"
- "b.eq 5f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "ld1h z6.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1h z7.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- ".inst 0x6464e4d0 // bfmmla z16.s, z6.h, z4.h\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- ".inst 0x6464e4f1 // bfmmla z17.s, z7.h, z4.h\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- ".inst 0x6464e512 // bfmmla z18.s, z8.h, z4.h\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- ".inst 0x6464e533 // bfmmla z19.s, z9.h, z4.h\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- ".inst 0x6464e554 // bfmmla z20.s, z10.h, z4.h\n"
- ".inst 0x6464e575 // bfmmla z21.s, z11.h, z4.h\n"
- ".inst 0x6464e596 // bfmmla z22.s, z12.h, z4.h\n"
- ".inst 0x6464e5b7 // bfmmla z23.s, z13.h, z4.h\n"
- ".inst 0x6465e4d8 // bfmmla z24.s, z6.h, z5.h\n"
- ".inst 0x6465e4f9 // bfmmla z25.s, z7.h, z5.h\n"
- ".inst 0x6465e51a // bfmmla z26.s, z8.h, z5.h\n"
- ".inst 0x6465e53b // bfmmla z27.s, z9.h, z5.h\n"
- ".inst 0x6465e55c // bfmmla z28.s, z10.h, z5.h\n"
- ".inst 0x6465e57d // bfmmla z29.s, z11.h, z5.h\n"
- ".inst 0x6465e59e // bfmmla z30.s, z12.h, z5.h\n"
- ".inst 0x6465e5bf // bfmmla z31.s, z13.h, z5.h\n"
- "5:\n"
- "ld1rw z14.s, p7/z, [%[minptr]]\n"
- "ld1rw z15.s, p7/z, [%[maxptr]]\n"
- "fmax z16.s, p7/m, z16.s, z14.s\n"
- "fmax z17.s, p7/m, z17.s, z14.s\n"
- "fmax z18.s, p7/m, z18.s, z14.s\n"
- "fmax z19.s, p7/m, z19.s, z14.s\n"
- "fmin z16.s, p7/m, z16.s, z15.s\n"
- "fmin z17.s, p7/m, z17.s, z15.s\n"
- "fmin z18.s, p7/m, z18.s, z15.s\n"
- "fmin z19.s, p7/m, z19.s, z15.s\n"
- "fmax z20.s, p7/m, z20.s, z14.s\n"
- "uzp1 z0.s, z16.s, z17.s\n"
- "uzp2 z1.s, z16.s, z17.s\n"
- "uzp1 z2.s, z18.s, z19.s\n"
- "uzp2 z3.s, z18.s, z19.s\n"
- "st1w z0.s, p0, [%[c_ptr0]]\n"
- "fmin z20.s, p7/m, z20.s, z15.s\n"
- "fmax z21.s, p7/m, z21.s, z14.s\n"
- "fmax z22.s, p7/m, z22.s, z14.s\n"
- "st1w z1.s, p0, [c_ptr1]\n"
- "fmax z23.s, p7/m, z23.s, z14.s\n"
- "fmax z24.s, p7/m, z24.s, z14.s\n"
- "fmin z21.s, p7/m, z21.s, z15.s\n"
- "st1w z2.s, p1, [%[c_ptr0], #1, MUL VL]\n"
- "fmin z22.s, p7/m, z22.s, z15.s\n"
- "fmin z23.s, p7/m, z23.s, z15.s\n"
- "fmin z24.s, p7/m, z24.s, z15.s\n"
- "st1w z3.s, p1, [c_ptr1, #1, MUL VL]\n"
- "uzp1 z4.s, z20.s, z21.s\n"
- "uzp2 z5.s, z20.s, z21.s\n"
- "uzp1 z6.s, z22.s, z23.s\n"
- "uzp2 z7.s, z22.s, z23.s\n"
- "st1w z4.s, p2, [%[c_ptr0], #2, MUL VL]\n"
- "fmax z25.s, p7/m, z25.s, z14.s\n"
- "fmax z26.s, p7/m, z26.s, z14.s\n"
- "fmax z27.s, p7/m, z27.s, z14.s\n"
- "st1w z5.s, p2, [c_ptr1, #2, MUL VL]\n"
- "fmax z28.s, p7/m, z28.s, z14.s\n"
- "fmin z25.s, p7/m, z25.s, z15.s\n"
- "fmin z26.s, p7/m, z26.s, z15.s\n"
- "st1w z6.s, p3, [%[c_ptr0], #3, MUL VL]\n"
- "fmin z27.s, p7/m, z27.s, z15.s\n"
- "addvl %[c_ptr0], %[c_ptr0], #4\n"
- "uzp1 z8.s, z24.s, z25.s\n"
- "st1w z7.s, p3, [c_ptr1, #3, MUL VL]\n"
- "uzp2 z9.s, z24.s, z25.s\n"
- "uzp1 z10.s, z26.s, z27.s\n"
- "uzp2 z11.s, z26.s, z27.s\n"
- "st1w z8.s, p0, [c_ptr2]\n"
- "fmin z28.s, p7/m, z28.s, z15.s\n"
- "fmax z29.s, p7/m, z29.s, z14.s\n"
- "fmax z30.s, p7/m, z30.s, z14.s\n"
- "st1w z9.s, p0, [c_ptr3]\n"
- "fmax z31.s, p7/m, z31.s, z14.s\n"
- "fmin z29.s, p7/m, z29.s, z15.s\n"
- "st1w z10.s, p1, [c_ptr2, #1, MUL VL]\n"
- "fmin z30.s, p7/m, z30.s, z15.s\n"
- "fmin z31.s, p7/m, z31.s, z15.s\n"
- "uzp1 z12.s, z28.s, z29.s\n"
- "st1w z11.s, p1, [c_ptr3, #1, MUL VL]\n"
- "uzp2 z13.s, z28.s, z29.s\n"
- "uzp1 z14.s, z30.s, z31.s\n"
- "uzp2 z15.s, z30.s, z31.s\n"
- "st1w z12.s, p2, [c_ptr2, #2, MUL VL]\n"
- "st1w z13.s, p2, [c_ptr3, #2, MUL VL]\n"
- "st1w z14.s, p3, [c_ptr2, #3, MUL VL]\n"
- "st1w z15.s, p3, [c_ptr3, #3, MUL VL]\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq a_ptr3\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- ".unreq c_ptr3\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks)
- : [width] "r" (width), [accumulate] "r" (static_cast<uint64_t>(accumulate)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers)
- : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "x4", "x5", "cc", "memory"
- );
- break;
- }
-
- }
- }
-}
-
-} // namespace arm_gemm
-
-#endif // __ARM_FEATURE_SVE
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp16_mla_4VLx4/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp16_mla_4VLx4/generic.cpp
deleted file mode 100644
index 7610a20..0000000
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp16_mla_4VLx4/generic.cpp
+++ /dev/null
@@ -1,3778 +0,0 @@
-/*
- * Copyright (c) 2018-2020 Arm Limited.
- *
- * SPDX-License-Identifier: MIT
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to
- * deal in the Software without restriction, including without limitation the
- * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
- * sell copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-#ifdef __ARM_FEATURE_SVE
-
-#include <algorithm>
-
-#include "arm_gemm.hpp"
-
-#include "../../asmlib.hpp"
-#include "../../utils.hpp"
-
-namespace arm_gemm {
-
-void sve_hybrid_fp16_mla_4VLx4(const __fp16 *A, int lda, const __fp16 *B, __fp16 *C, int ldc, int M, int N, int K, const __fp16 *bias, Activation act, bool accumulate) {
- const int K_stride = K;
- const long loops_count = ((K + 8) / 16) - 1;
- K -= loops_count * 16;
- const long regs_count = (K / 8) - 1;
- K -= (regs_count + 1) * 8;
- const long leftovers = K;
- __fp16 nullbias[512];
- if (!accumulate && !bias) {
- memset(nullbias, 0, (4 * get_vector_length<__fp16>() * sizeof(__fp16)));
- }
- __fp16 minval = - static_cast<__fp16>(std::numeric_limits<float>::infinity());
- __fp16 maxval = static_cast<__fp16>(std::numeric_limits<float>::infinity());
- const __fp16 * const minptr = &minval;
- const __fp16 * const maxptr = &maxval;
-
- switch(act.type)
- {
- default:
- case Activation::Type::None:
- break;
- case Activation::Type::BoundedReLU:
- maxval = static_cast<__fp16>(act.param1);
- /* fall through */
- case Activation::Type::ReLU:
- minval = 0.0f;
- break;
- }
-
- int rows_to_compute;
-
- for (int y=0; y<M; y+=rows_to_compute) {
- const __fp16 * const a_ptr0_base = A + (y * lda);
- const unsigned long ldab = lda * sizeof(__fp16);
-
- __fp16 *c_ptr0 = C + (y * ldc);
-
- rows_to_compute = M-y;
- if (rows_to_compute > 4) {
- if (rows_to_compute % 4) {
- rows_to_compute = 4 - 1;
- } else {
- rows_to_compute = 4;
- }
- }
-
- for (int x0=0; x0<N; x0+=(4 * get_vector_length<__fp16>())) {
- const long width = std::min((unsigned long)N-x0, (4 * get_vector_length<__fp16>()));
- long loops = loops_count;
- long regs = regs_count;
- long temp = 0;
- long blocks = leftovers;
- const __fp16 *a_ptr0 = a_ptr0_base;
- const __fp16 *b_ptr0 = B + (K_stride * x0);
- const unsigned long ldcb = ldc * sizeof(__fp16);
- const __fp16 *biasptr = bias ? bias+x0 : nullbias;
-
- switch(rows_to_compute) {
- case 1:
- __asm __volatile (
- "whilelt p6.h, %[temp], %[leftovers]\n"
- "whilelt p0.h, %[temp], %[width]\n"
- "inch %[temp], all, mul #1\n"
- "ptrue p7.h\n"
- "whilelt p1.h, %[temp], %[width]\n"
- "inch %[temp], all, mul #1\n"
- "whilelt p2.h, %[temp], %[width]\n"
- "inch %[temp], all, mul #1\n"
- "whilelt p3.h, %[temp], %[width]\n"
- "cbnz %[accumulate], 1f\n"
- "ld1h z16.h, p0/z, [%[biasptr]]\n"
- "ld1h z17.h, p1/z, [%[biasptr], #1, MUL VL]\n"
- "ld1h z18.h, p2/z, [%[biasptr], #2, MUL VL]\n"
- "ld1h z19.h, p3/z, [%[biasptr], #3, MUL VL]\n"
- "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "ld1h z16.h, p0/z, [%[c_ptr0]]\n"
- "ld1h z17.h, p1/z, [%[c_ptr0], #1, MUL VL]\n"
- "ld1h z18.h, p2/z, [%[c_ptr0], #2, MUL VL]\n"
- "ld1h z19.h, p3/z, [%[c_ptr0], #3, MUL VL]\n"
- "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "cbz %[loops], 2f\n"
- "3:\n"
- "fmla z16.h, z8.h, z0.h[0]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "fmla z17.h, z9.h, z0.h[0]\n"
- "ld1rqh z4.h, p7/z, [%[a_ptr0]]\n"
- "fmla z18.h, z10.h, z0.h[0]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "fmla z19.h, z11.h, z0.h[0]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z16.h, z12.h, z0.h[1]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z17.h, z13.h, z0.h[1]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z18.h, z14.h, z0.h[1]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "fmla z19.h, z15.h, z0.h[1]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "fmla z16.h, z8.h, z0.h[2]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z17.h, z9.h, z0.h[2]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z18.h, z10.h, z0.h[2]\n"
- "subs %[loops], %[loops], #0x1\n"
- "fmla z19.h, z11.h, z0.h[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "fmla z16.h, z12.h, z0.h[3]\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
- "fmla z17.h, z13.h, z0.h[3]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "fmla z18.h, z14.h, z0.h[3]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "fmla z19.h, z15.h, z0.h[3]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "fmla z16.h, z8.h, z0.h[4]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "fmla z17.h, z9.h, z0.h[4]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "fmla z18.h, z10.h, z0.h[4]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "fmla z19.h, z11.h, z0.h[4]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "fmla z16.h, z12.h, z0.h[5]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "fmla z17.h, z13.h, z0.h[5]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z18.h, z14.h, z0.h[5]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z19.h, z15.h, z0.h[5]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z16.h, z8.h, z0.h[6]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "fmla z17.h, z9.h, z0.h[6]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "fmla z18.h, z10.h, z0.h[6]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z19.h, z11.h, z0.h[6]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z16.h, z12.h, z0.h[7]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "fmla z17.h, z13.h, z0.h[7]\n"
- "fmla z18.h, z14.h, z0.h[7]\n"
- "fmla z19.h, z15.h, z0.h[7]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "fmla z16.h, z8.h, z4.h[0]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "fmla z17.h, z9.h, z4.h[0]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "fmla z18.h, z10.h, z4.h[0]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "fmla z19.h, z11.h, z4.h[0]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "fmla z16.h, z12.h, z4.h[1]\n"
- "ld1rqh z0.h, p7/z, [%[a_ptr0], #-0x10]\n"
- "fmla z17.h, z13.h, z4.h[1]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "fmla z18.h, z14.h, z4.h[1]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z19.h, z15.h, z4.h[1]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z16.h, z8.h, z4.h[2]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "fmla z17.h, z9.h, z4.h[2]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "fmla z18.h, z10.h, z4.h[2]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z19.h, z11.h, z4.h[2]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z16.h, z12.h, z4.h[3]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "fmla z17.h, z13.h, z4.h[3]\n"
- "fmla z18.h, z14.h, z4.h[3]\n"
- "fmla z19.h, z15.h, z4.h[3]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "fmla z16.h, z8.h, z4.h[4]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "fmla z17.h, z9.h, z4.h[4]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "fmla z18.h, z10.h, z4.h[4]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "fmla z19.h, z11.h, z4.h[4]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "fmla z16.h, z12.h, z4.h[5]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "fmla z17.h, z13.h, z4.h[5]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z18.h, z14.h, z4.h[5]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z19.h, z15.h, z4.h[5]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z16.h, z8.h, z4.h[6]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "fmla z17.h, z9.h, z4.h[6]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "fmla z18.h, z10.h, z4.h[6]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z19.h, z11.h, z4.h[6]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z16.h, z12.h, z4.h[7]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "fmla z17.h, z13.h, z4.h[7]\n"
- "fmla z18.h, z14.h, z4.h[7]\n"
- "fmla z19.h, z15.h, z4.h[7]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "b.ne 3b\n"
- "2:\n"
- "cbz %[regs], 4f\n"
- "fmla z16.h, z8.h, z0.h[0]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "fmla z17.h, z9.h, z0.h[0]\n"
- "ld1rqh z4.h, p7/z, [%[a_ptr0]]\n"
- "fmla z18.h, z10.h, z0.h[0]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "fmla z19.h, z11.h, z0.h[0]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z16.h, z12.h, z0.h[1]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z17.h, z13.h, z0.h[1]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z18.h, z14.h, z0.h[1]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "fmla z19.h, z15.h, z0.h[1]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "fmla z16.h, z8.h, z0.h[2]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z17.h, z9.h, z0.h[2]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z18.h, z10.h, z0.h[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "fmla z19.h, z11.h, z0.h[2]\n"
- "fmla z16.h, z12.h, z0.h[3]\n"
- "fmla z17.h, z13.h, z0.h[3]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "fmla z18.h, z14.h, z0.h[3]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "fmla z19.h, z15.h, z0.h[3]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "fmla z16.h, z8.h, z0.h[4]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "fmla z17.h, z9.h, z0.h[4]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "fmla z18.h, z10.h, z0.h[4]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "fmla z19.h, z11.h, z0.h[4]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "fmla z16.h, z12.h, z0.h[5]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "fmla z17.h, z13.h, z0.h[5]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z18.h, z14.h, z0.h[5]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z19.h, z15.h, z0.h[5]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z16.h, z8.h, z0.h[6]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "fmla z17.h, z9.h, z0.h[6]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "fmla z18.h, z10.h, z0.h[6]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z19.h, z11.h, z0.h[6]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z16.h, z12.h, z0.h[7]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "fmla z17.h, z13.h, z0.h[7]\n"
- "fmla z18.h, z14.h, z0.h[7]\n"
- "fmla z19.h, z15.h, z0.h[7]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "fmla z16.h, z8.h, z4.h[0]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "fmla z17.h, z9.h, z4.h[0]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "fmla z18.h, z10.h, z4.h[0]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "fmla z19.h, z11.h, z4.h[0]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "fmla z16.h, z12.h, z4.h[1]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "fmla z17.h, z13.h, z4.h[1]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z18.h, z14.h, z4.h[1]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z19.h, z15.h, z4.h[1]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z16.h, z8.h, z4.h[2]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "fmla z17.h, z9.h, z4.h[2]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "fmla z18.h, z10.h, z4.h[2]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z19.h, z11.h, z4.h[2]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z16.h, z12.h, z4.h[3]\n"
- "ld1rqh z0.h, p6/z, [%[a_ptr0], #0x10]\n"
- "fmla z17.h, z13.h, z4.h[3]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "fmla z18.h, z14.h, z4.h[3]\n"
- "addvl %[a_ptr0], %[a_ptr0], #2\n"
- "fmla z19.h, z15.h, z4.h[3]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "fmla z16.h, z8.h, z4.h[4]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "fmla z17.h, z9.h, z4.h[4]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "fmla z18.h, z10.h, z4.h[4]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "fmla z19.h, z11.h, z4.h[4]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "fmla z16.h, z12.h, z4.h[5]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "fmla z17.h, z13.h, z4.h[5]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z18.h, z14.h, z4.h[5]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z19.h, z15.h, z4.h[5]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z16.h, z8.h, z4.h[6]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "fmla z17.h, z9.h, z4.h[6]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "fmla z18.h, z10.h, z4.h[6]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z19.h, z11.h, z4.h[6]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z16.h, z12.h, z4.h[7]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "fmla z17.h, z13.h, z4.h[7]\n"
- "fmla z18.h, z14.h, z4.h[7]\n"
- "fmla z19.h, z15.h, z4.h[7]\n"
- "cbz %[blocks], 5f\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z16.h, z8.h, z0.h[0]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z17.h, z9.h, z0.h[0]\n"
- "fmla z18.h, z10.h, z0.h[0]\n"
- "fmla z19.h, z11.h, z0.h[0]\n"
- "b.eq 5f\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z16.h, z12.h, z0.h[1]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z17.h, z13.h, z0.h[1]\n"
- "fmla z18.h, z14.h, z0.h[1]\n"
- "fmla z19.h, z15.h, z0.h[1]\n"
- "b.eq 5f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "fmla z16.h, z8.h, z0.h[2]\n"
- "fmla z17.h, z9.h, z0.h[2]\n"
- "fmla z18.h, z10.h, z0.h[2]\n"
- "fmla z19.h, z11.h, z0.h[2]\n"
- "b.eq 5f\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "fmla z16.h, z12.h, z0.h[3]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "fmla z17.h, z13.h, z0.h[3]\n"
- "fmla z18.h, z14.h, z0.h[3]\n"
- "fmla z19.h, z15.h, z0.h[3]\n"
- "b.eq 5f\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z16.h, z8.h, z0.h[4]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z17.h, z9.h, z0.h[4]\n"
- "fmla z18.h, z10.h, z0.h[4]\n"
- "fmla z19.h, z11.h, z0.h[4]\n"
- "b.eq 5f\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z16.h, z12.h, z0.h[5]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z17.h, z13.h, z0.h[5]\n"
- "fmla z18.h, z14.h, z0.h[5]\n"
- "fmla z19.h, z15.h, z0.h[5]\n"
- "b.eq 5f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "fmla z16.h, z8.h, z0.h[6]\n"
- "fmla z17.h, z9.h, z0.h[6]\n"
- "fmla z18.h, z10.h, z0.h[6]\n"
- "fmla z19.h, z11.h, z0.h[6]\n"
- "b 5f\n"
- "4:\n"
- "fmla z16.h, z8.h, z0.h[0]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "fmla z17.h, z9.h, z0.h[0]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "fmla z18.h, z10.h, z0.h[0]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z19.h, z11.h, z0.h[0]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z16.h, z12.h, z0.h[1]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z17.h, z13.h, z0.h[1]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "fmla z18.h, z14.h, z0.h[1]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "fmla z19.h, z15.h, z0.h[1]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z16.h, z8.h, z0.h[2]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z17.h, z9.h, z0.h[2]\n"
- "ld1rqh z4.h, p6/z, [%[a_ptr0]]\n"
- "fmla z18.h, z10.h, z0.h[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "fmla z19.h, z11.h, z0.h[2]\n"
- "addvl %[a_ptr0], %[a_ptr0], #1\n"
- "fmla z16.h, z12.h, z0.h[3]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "fmla z17.h, z13.h, z0.h[3]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "fmla z18.h, z14.h, z0.h[3]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "fmla z19.h, z15.h, z0.h[3]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "fmla z16.h, z8.h, z0.h[4]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "fmla z17.h, z9.h, z0.h[4]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "fmla z18.h, z10.h, z0.h[4]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "fmla z19.h, z11.h, z0.h[4]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "fmla z16.h, z12.h, z0.h[5]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "fmla z17.h, z13.h, z0.h[5]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z18.h, z14.h, z0.h[5]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z19.h, z15.h, z0.h[5]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z16.h, z8.h, z0.h[6]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "fmla z17.h, z9.h, z0.h[6]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "fmla z18.h, z10.h, z0.h[6]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z19.h, z11.h, z0.h[6]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z16.h, z12.h, z0.h[7]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "fmla z17.h, z13.h, z0.h[7]\n"
- "fmla z18.h, z14.h, z0.h[7]\n"
- "fmla z19.h, z15.h, z0.h[7]\n"
- "cbz %[blocks], 5f\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z16.h, z8.h, z4.h[0]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z17.h, z9.h, z4.h[0]\n"
- "fmla z18.h, z10.h, z4.h[0]\n"
- "fmla z19.h, z11.h, z4.h[0]\n"
- "b.eq 5f\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z16.h, z12.h, z4.h[1]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z17.h, z13.h, z4.h[1]\n"
- "fmla z18.h, z14.h, z4.h[1]\n"
- "fmla z19.h, z15.h, z4.h[1]\n"
- "b.eq 5f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "fmla z16.h, z8.h, z4.h[2]\n"
- "fmla z17.h, z9.h, z4.h[2]\n"
- "fmla z18.h, z10.h, z4.h[2]\n"
- "fmla z19.h, z11.h, z4.h[2]\n"
- "b.eq 5f\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "fmla z16.h, z12.h, z4.h[3]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "fmla z17.h, z13.h, z4.h[3]\n"
- "fmla z18.h, z14.h, z4.h[3]\n"
- "fmla z19.h, z15.h, z4.h[3]\n"
- "b.eq 5f\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z16.h, z8.h, z4.h[4]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z17.h, z9.h, z4.h[4]\n"
- "fmla z18.h, z10.h, z4.h[4]\n"
- "fmla z19.h, z11.h, z4.h[4]\n"
- "b.eq 5f\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z16.h, z12.h, z4.h[5]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z17.h, z13.h, z4.h[5]\n"
- "fmla z18.h, z14.h, z4.h[5]\n"
- "fmla z19.h, z15.h, z4.h[5]\n"
- "b.eq 5f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "fmla z16.h, z8.h, z4.h[6]\n"
- "fmla z17.h, z9.h, z4.h[6]\n"
- "fmla z18.h, z10.h, z4.h[6]\n"
- "fmla z19.h, z11.h, z4.h[6]\n"
- "5:\n"
- "ld1rh z14.h, p7/z, [%[minptr]]\n"
- "ld1rh z15.h, p7/z, [%[maxptr]]\n"
- "fmax z16.h, p7/m, z16.h, z14.h\n"
- "fmax z17.h, p7/m, z17.h, z14.h\n"
- "fmax z18.h, p7/m, z18.h, z14.h\n"
- "fmax z19.h, p7/m, z19.h, z14.h\n"
- "fmin z16.h, p7/m, z16.h, z15.h\n"
- "fmin z17.h, p7/m, z17.h, z15.h\n"
- "fmin z18.h, p7/m, z18.h, z15.h\n"
- "fmin z19.h, p7/m, z19.h, z15.h\n"
- "st1h z16.h, p0, [%[c_ptr0]]\n"
- "st1h z17.h, p1, [%[c_ptr0], #1, MUL VL]\n"
- "st1h z18.h, p2, [%[c_ptr0], #2, MUL VL]\n"
- "st1h z19.h, p3, [%[c_ptr0], #3, MUL VL]\n"
- "addvl %[c_ptr0], %[c_ptr0], #4\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks)
- : [width] "r" (width), [accumulate] "r" (static_cast<uint64_t>(accumulate)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers)
- : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
- );
- break;
- case 2:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "c_ptr1 .req X1\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "whilelt p6.h, %[temp], %[leftovers]\n"
- "whilelt p0.h, %[temp], %[width]\n"
- "inch %[temp], all, mul #1\n"
- "ptrue p7.h\n"
- "whilelt p1.h, %[temp], %[width]\n"
- "inch %[temp], all, mul #1\n"
- "whilelt p2.h, %[temp], %[width]\n"
- "inch %[temp], all, mul #1\n"
- "whilelt p3.h, %[temp], %[width]\n"
- "cbnz %[accumulate], 1f\n"
- "ld1h z16.h, p0/z, [%[biasptr]]\n"
- "ld1h z17.h, p1/z, [%[biasptr], #1, MUL VL]\n"
- "ld1h z18.h, p2/z, [%[biasptr], #2, MUL VL]\n"
- "ld1h z19.h, p3/z, [%[biasptr], #3, MUL VL]\n"
- "mov z20.d, z16.d\n"
- "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n"
- "mov z21.d, z17.d\n"
- "ld1rqh z1.h, p7/z, [a_ptr1]\n"
- "mov z22.d, z18.d\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "mov z23.d, z19.d\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "ld1h z16.h, p0/z, [%[c_ptr0]]\n"
- "ld1h z17.h, p1/z, [%[c_ptr0], #1, MUL VL]\n"
- "ld1h z18.h, p2/z, [%[c_ptr0], #2, MUL VL]\n"
- "ld1h z19.h, p3/z, [%[c_ptr0], #3, MUL VL]\n"
- "ld1h z20.h, p0/z, [c_ptr1]\n"
- "ld1h z21.h, p1/z, [c_ptr1, #1, MUL VL]\n"
- "ld1h z22.h, p2/z, [c_ptr1, #2, MUL VL]\n"
- "ld1h z23.h, p3/z, [c_ptr1, #3, MUL VL]\n"
- "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ld1rqh z1.h, p7/z, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "cbz %[loops], 2f\n"
- "3:\n"
- "fmla z16.h, z8.h, z0.h[0]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "fmla z20.h, z8.h, z1.h[0]\n"
- "ld1rqh z4.h, p7/z, [%[a_ptr0]]\n"
- "fmla z17.h, z9.h, z0.h[0]\n"
- "ld1rqh z5.h, p7/z, [a_ptr1]\n"
- "fmla z21.h, z9.h, z1.h[0]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "fmla z18.h, z10.h, z0.h[0]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z22.h, z10.h, z1.h[0]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z19.h, z11.h, z0.h[0]\n"
- "subs %[loops], %[loops], #0x1\n"
- "fmla z23.h, z11.h, z1.h[0]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z16.h, z12.h, z0.h[1]\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
- "fmla z20.h, z12.h, z1.h[1]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "fmla z17.h, z13.h, z0.h[1]\n"
- "add a_ptr1, a_ptr1, #0x20\n"
- "fmla z21.h, z13.h, z1.h[1]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "fmla z18.h, z14.h, z0.h[1]\n"
- "fmla z22.h, z14.h, z1.h[1]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z19.h, z15.h, z0.h[1]\n"
- "fmla z23.h, z15.h, z1.h[1]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z16.h, z8.h, z0.h[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "fmla z20.h, z8.h, z1.h[2]\n"
- "fmla z17.h, z9.h, z0.h[2]\n"
- "fmla z21.h, z9.h, z1.h[2]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "fmla z18.h, z10.h, z0.h[2]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "fmla z22.h, z10.h, z1.h[2]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "fmla z19.h, z11.h, z0.h[2]\n"
- "fmla z23.h, z11.h, z1.h[2]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "fmla z16.h, z12.h, z0.h[3]\n"
- "fmla z20.h, z12.h, z1.h[3]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "fmla z17.h, z13.h, z0.h[3]\n"
- "fmla z21.h, z13.h, z1.h[3]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "fmla z18.h, z14.h, z0.h[3]\n"
- "fmla z22.h, z14.h, z1.h[3]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "fmla z19.h, z15.h, z0.h[3]\n"
- "fmla z23.h, z15.h, z1.h[3]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "fmla z16.h, z8.h, z0.h[4]\n"
- "fmla z20.h, z8.h, z1.h[4]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "fmla z17.h, z9.h, z0.h[4]\n"
- "fmla z21.h, z9.h, z1.h[4]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z18.h, z10.h, z0.h[4]\n"
- "fmla z22.h, z10.h, z1.h[4]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z19.h, z11.h, z0.h[4]\n"
- "fmla z23.h, z11.h, z1.h[4]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z16.h, z12.h, z0.h[5]\n"
- "fmla z20.h, z12.h, z1.h[5]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "fmla z17.h, z13.h, z0.h[5]\n"
- "fmla z21.h, z13.h, z1.h[5]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "fmla z18.h, z14.h, z0.h[5]\n"
- "fmla z22.h, z14.h, z1.h[5]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z19.h, z15.h, z0.h[5]\n"
- "fmla z23.h, z15.h, z1.h[5]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z16.h, z8.h, z0.h[6]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "fmla z20.h, z8.h, z1.h[6]\n"
- "fmla z17.h, z9.h, z0.h[6]\n"
- "fmla z21.h, z9.h, z1.h[6]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "fmla z18.h, z10.h, z0.h[6]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "fmla z22.h, z10.h, z1.h[6]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "fmla z19.h, z11.h, z0.h[6]\n"
- "fmla z23.h, z11.h, z1.h[6]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "fmla z16.h, z12.h, z0.h[7]\n"
- "fmla z20.h, z12.h, z1.h[7]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "fmla z17.h, z13.h, z0.h[7]\n"
- "fmla z21.h, z13.h, z1.h[7]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "fmla z18.h, z14.h, z0.h[7]\n"
- "fmla z22.h, z14.h, z1.h[7]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "fmla z19.h, z15.h, z0.h[7]\n"
- "ld1rqh z0.h, p7/z, [%[a_ptr0], #-0x10]\n"
- "fmla z23.h, z15.h, z1.h[7]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "fmla z16.h, z8.h, z4.h[0]\n"
- "ld1rqh z1.h, p7/z, [a_ptr1, #-0x10]\n"
- "fmla z20.h, z8.h, z5.h[0]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "fmla z17.h, z9.h, z4.h[0]\n"
- "fmla z21.h, z9.h, z5.h[0]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z18.h, z10.h, z4.h[0]\n"
- "fmla z22.h, z10.h, z5.h[0]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z19.h, z11.h, z4.h[0]\n"
- "fmla z23.h, z11.h, z5.h[0]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z16.h, z12.h, z4.h[1]\n"
- "fmla z20.h, z12.h, z5.h[1]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "fmla z17.h, z13.h, z4.h[1]\n"
- "fmla z21.h, z13.h, z5.h[1]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "fmla z18.h, z14.h, z4.h[1]\n"
- "fmla z22.h, z14.h, z5.h[1]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z19.h, z15.h, z4.h[1]\n"
- "fmla z23.h, z15.h, z5.h[1]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z16.h, z8.h, z4.h[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "fmla z20.h, z8.h, z5.h[2]\n"
- "fmla z17.h, z9.h, z4.h[2]\n"
- "fmla z21.h, z9.h, z5.h[2]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "fmla z18.h, z10.h, z4.h[2]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "fmla z22.h, z10.h, z5.h[2]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "fmla z19.h, z11.h, z4.h[2]\n"
- "fmla z23.h, z11.h, z5.h[2]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "fmla z16.h, z12.h, z4.h[3]\n"
- "fmla z20.h, z12.h, z5.h[3]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "fmla z17.h, z13.h, z4.h[3]\n"
- "fmla z21.h, z13.h, z5.h[3]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "fmla z18.h, z14.h, z4.h[3]\n"
- "fmla z22.h, z14.h, z5.h[3]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "fmla z19.h, z15.h, z4.h[3]\n"
- "fmla z23.h, z15.h, z5.h[3]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "fmla z16.h, z8.h, z4.h[4]\n"
- "fmla z20.h, z8.h, z5.h[4]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "fmla z17.h, z9.h, z4.h[4]\n"
- "fmla z21.h, z9.h, z5.h[4]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z18.h, z10.h, z4.h[4]\n"
- "fmla z22.h, z10.h, z5.h[4]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z19.h, z11.h, z4.h[4]\n"
- "fmla z23.h, z11.h, z5.h[4]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z16.h, z12.h, z4.h[5]\n"
- "fmla z20.h, z12.h, z5.h[5]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "fmla z17.h, z13.h, z4.h[5]\n"
- "fmla z21.h, z13.h, z5.h[5]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "fmla z18.h, z14.h, z4.h[5]\n"
- "fmla z22.h, z14.h, z5.h[5]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z19.h, z15.h, z4.h[5]\n"
- "fmla z23.h, z15.h, z5.h[5]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z16.h, z8.h, z4.h[6]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "fmla z20.h, z8.h, z5.h[6]\n"
- "fmla z17.h, z9.h, z4.h[6]\n"
- "fmla z21.h, z9.h, z5.h[6]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "fmla z18.h, z10.h, z4.h[6]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "fmla z22.h, z10.h, z5.h[6]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "fmla z19.h, z11.h, z4.h[6]\n"
- "fmla z23.h, z11.h, z5.h[6]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "fmla z16.h, z12.h, z4.h[7]\n"
- "fmla z20.h, z12.h, z5.h[7]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "fmla z17.h, z13.h, z4.h[7]\n"
- "fmla z21.h, z13.h, z5.h[7]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "fmla z18.h, z14.h, z4.h[7]\n"
- "fmla z22.h, z14.h, z5.h[7]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "fmla z19.h, z15.h, z4.h[7]\n"
- "fmla z23.h, z15.h, z5.h[7]\n"
- "b.ne 3b\n"
- "2:\n"
- "cbz %[regs], 4f\n"
- "fmla z16.h, z8.h, z0.h[0]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "fmla z20.h, z8.h, z1.h[0]\n"
- "ld1rqh z4.h, p7/z, [%[a_ptr0]]\n"
- "fmla z17.h, z9.h, z0.h[0]\n"
- "ld1rqh z5.h, p7/z, [a_ptr1]\n"
- "fmla z21.h, z9.h, z1.h[0]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "fmla z18.h, z10.h, z0.h[0]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z22.h, z10.h, z1.h[0]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z19.h, z11.h, z0.h[0]\n"
- "fmla z23.h, z11.h, z1.h[0]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z16.h, z12.h, z0.h[1]\n"
- "fmla z20.h, z12.h, z1.h[1]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "fmla z17.h, z13.h, z0.h[1]\n"
- "fmla z21.h, z13.h, z1.h[1]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "fmla z18.h, z14.h, z0.h[1]\n"
- "fmla z22.h, z14.h, z1.h[1]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z19.h, z15.h, z0.h[1]\n"
- "fmla z23.h, z15.h, z1.h[1]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z16.h, z8.h, z0.h[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "fmla z20.h, z8.h, z1.h[2]\n"
- "fmla z17.h, z9.h, z0.h[2]\n"
- "fmla z21.h, z9.h, z1.h[2]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "fmla z18.h, z10.h, z0.h[2]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "fmla z22.h, z10.h, z1.h[2]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "fmla z19.h, z11.h, z0.h[2]\n"
- "fmla z23.h, z11.h, z1.h[2]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "fmla z16.h, z12.h, z0.h[3]\n"
- "fmla z20.h, z12.h, z1.h[3]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "fmla z17.h, z13.h, z0.h[3]\n"
- "fmla z21.h, z13.h, z1.h[3]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "fmla z18.h, z14.h, z0.h[3]\n"
- "fmla z22.h, z14.h, z1.h[3]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "fmla z19.h, z15.h, z0.h[3]\n"
- "fmla z23.h, z15.h, z1.h[3]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "fmla z16.h, z8.h, z0.h[4]\n"
- "fmla z20.h, z8.h, z1.h[4]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "fmla z17.h, z9.h, z0.h[4]\n"
- "fmla z21.h, z9.h, z1.h[4]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z18.h, z10.h, z0.h[4]\n"
- "fmla z22.h, z10.h, z1.h[4]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z19.h, z11.h, z0.h[4]\n"
- "fmla z23.h, z11.h, z1.h[4]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z16.h, z12.h, z0.h[5]\n"
- "fmla z20.h, z12.h, z1.h[5]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "fmla z17.h, z13.h, z0.h[5]\n"
- "fmla z21.h, z13.h, z1.h[5]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "fmla z18.h, z14.h, z0.h[5]\n"
- "fmla z22.h, z14.h, z1.h[5]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z19.h, z15.h, z0.h[5]\n"
- "fmla z23.h, z15.h, z1.h[5]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z16.h, z8.h, z0.h[6]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "fmla z20.h, z8.h, z1.h[6]\n"
- "fmla z17.h, z9.h, z0.h[6]\n"
- "fmla z21.h, z9.h, z1.h[6]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "fmla z18.h, z10.h, z0.h[6]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "fmla z22.h, z10.h, z1.h[6]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "fmla z19.h, z11.h, z0.h[6]\n"
- "fmla z23.h, z11.h, z1.h[6]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "fmla z16.h, z12.h, z0.h[7]\n"
- "fmla z20.h, z12.h, z1.h[7]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "fmla z17.h, z13.h, z0.h[7]\n"
- "fmla z21.h, z13.h, z1.h[7]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "fmla z18.h, z14.h, z0.h[7]\n"
- "fmla z22.h, z14.h, z1.h[7]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "fmla z19.h, z15.h, z0.h[7]\n"
- "ld1rqh z0.h, p6/z, [%[a_ptr0], #0x10]\n"
- "fmla z23.h, z15.h, z1.h[7]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "fmla z16.h, z8.h, z4.h[0]\n"
- "ld1rqh z1.h, p6/z, [a_ptr1, #0x10]\n"
- "fmla z20.h, z8.h, z5.h[0]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "fmla z17.h, z9.h, z4.h[0]\n"
- "addvl %[a_ptr0], %[a_ptr0], #2\n"
- "fmla z21.h, z9.h, z5.h[0]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z18.h, z10.h, z4.h[0]\n"
- "addvl a_ptr1, a_ptr1, #2\n"
- "fmla z22.h, z10.h, z5.h[0]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z19.h, z11.h, z4.h[0]\n"
- "fmla z23.h, z11.h, z5.h[0]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z16.h, z12.h, z4.h[1]\n"
- "fmla z20.h, z12.h, z5.h[1]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "fmla z17.h, z13.h, z4.h[1]\n"
- "fmla z21.h, z13.h, z5.h[1]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "fmla z18.h, z14.h, z4.h[1]\n"
- "fmla z22.h, z14.h, z5.h[1]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z19.h, z15.h, z4.h[1]\n"
- "fmla z23.h, z15.h, z5.h[1]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z16.h, z8.h, z4.h[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "fmla z20.h, z8.h, z5.h[2]\n"
- "fmla z17.h, z9.h, z4.h[2]\n"
- "fmla z21.h, z9.h, z5.h[2]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "fmla z18.h, z10.h, z4.h[2]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "fmla z22.h, z10.h, z5.h[2]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "fmla z19.h, z11.h, z4.h[2]\n"
- "fmla z23.h, z11.h, z5.h[2]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "fmla z16.h, z12.h, z4.h[3]\n"
- "fmla z20.h, z12.h, z5.h[3]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "fmla z17.h, z13.h, z4.h[3]\n"
- "fmla z21.h, z13.h, z5.h[3]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "fmla z18.h, z14.h, z4.h[3]\n"
- "fmla z22.h, z14.h, z5.h[3]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "fmla z19.h, z15.h, z4.h[3]\n"
- "fmla z23.h, z15.h, z5.h[3]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "fmla z16.h, z8.h, z4.h[4]\n"
- "fmla z20.h, z8.h, z5.h[4]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "fmla z17.h, z9.h, z4.h[4]\n"
- "fmla z21.h, z9.h, z5.h[4]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z18.h, z10.h, z4.h[4]\n"
- "fmla z22.h, z10.h, z5.h[4]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z19.h, z11.h, z4.h[4]\n"
- "fmla z23.h, z11.h, z5.h[4]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z16.h, z12.h, z4.h[5]\n"
- "fmla z20.h, z12.h, z5.h[5]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "fmla z17.h, z13.h, z4.h[5]\n"
- "fmla z21.h, z13.h, z5.h[5]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "fmla z18.h, z14.h, z4.h[5]\n"
- "fmla z22.h, z14.h, z5.h[5]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z19.h, z15.h, z4.h[5]\n"
- "fmla z23.h, z15.h, z5.h[5]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z16.h, z8.h, z4.h[6]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "fmla z20.h, z8.h, z5.h[6]\n"
- "fmla z17.h, z9.h, z4.h[6]\n"
- "fmla z21.h, z9.h, z5.h[6]\n"
- "fmla z18.h, z10.h, z4.h[6]\n"
- "fmla z22.h, z10.h, z5.h[6]\n"
- "fmla z19.h, z11.h, z4.h[6]\n"
- "fmla z23.h, z11.h, z5.h[6]\n"
- "fmla z16.h, z12.h, z4.h[7]\n"
- "fmla z20.h, z12.h, z5.h[7]\n"
- "fmla z17.h, z13.h, z4.h[7]\n"
- "fmla z21.h, z13.h, z5.h[7]\n"
- "fmla z18.h, z14.h, z4.h[7]\n"
- "fmla z22.h, z14.h, z5.h[7]\n"
- "fmla z19.h, z15.h, z4.h[7]\n"
- "fmla z23.h, z15.h, z5.h[7]\n"
- "cbz %[blocks], 5f\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z16.h, z8.h, z0.h[0]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z20.h, z8.h, z1.h[0]\n"
- "fmla z17.h, z9.h, z0.h[0]\n"
- "fmla z21.h, z9.h, z1.h[0]\n"
- "fmla z18.h, z10.h, z0.h[0]\n"
- "fmla z22.h, z10.h, z1.h[0]\n"
- "fmla z19.h, z11.h, z0.h[0]\n"
- "fmla z23.h, z11.h, z1.h[0]\n"
- "b.eq 5f\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z16.h, z12.h, z0.h[1]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z20.h, z12.h, z1.h[1]\n"
- "fmla z17.h, z13.h, z0.h[1]\n"
- "fmla z21.h, z13.h, z1.h[1]\n"
- "fmla z18.h, z14.h, z0.h[1]\n"
- "fmla z22.h, z14.h, z1.h[1]\n"
- "fmla z19.h, z15.h, z0.h[1]\n"
- "fmla z23.h, z15.h, z1.h[1]\n"
- "b.eq 5f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "fmla z16.h, z8.h, z0.h[2]\n"
- "fmla z20.h, z8.h, z1.h[2]\n"
- "fmla z17.h, z9.h, z0.h[2]\n"
- "fmla z21.h, z9.h, z1.h[2]\n"
- "fmla z18.h, z10.h, z0.h[2]\n"
- "fmla z22.h, z10.h, z1.h[2]\n"
- "fmla z19.h, z11.h, z0.h[2]\n"
- "fmla z23.h, z11.h, z1.h[2]\n"
- "b.eq 5f\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "fmla z16.h, z12.h, z0.h[3]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "fmla z20.h, z12.h, z1.h[3]\n"
- "fmla z17.h, z13.h, z0.h[3]\n"
- "fmla z21.h, z13.h, z1.h[3]\n"
- "fmla z18.h, z14.h, z0.h[3]\n"
- "fmla z22.h, z14.h, z1.h[3]\n"
- "fmla z19.h, z15.h, z0.h[3]\n"
- "fmla z23.h, z15.h, z1.h[3]\n"
- "b.eq 5f\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z16.h, z8.h, z0.h[4]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z20.h, z8.h, z1.h[4]\n"
- "fmla z17.h, z9.h, z0.h[4]\n"
- "fmla z21.h, z9.h, z1.h[4]\n"
- "fmla z18.h, z10.h, z0.h[4]\n"
- "fmla z22.h, z10.h, z1.h[4]\n"
- "fmla z19.h, z11.h, z0.h[4]\n"
- "fmla z23.h, z11.h, z1.h[4]\n"
- "b.eq 5f\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z16.h, z12.h, z0.h[5]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z20.h, z12.h, z1.h[5]\n"
- "fmla z17.h, z13.h, z0.h[5]\n"
- "fmla z21.h, z13.h, z1.h[5]\n"
- "fmla z18.h, z14.h, z0.h[5]\n"
- "fmla z22.h, z14.h, z1.h[5]\n"
- "fmla z19.h, z15.h, z0.h[5]\n"
- "fmla z23.h, z15.h, z1.h[5]\n"
- "b.eq 5f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "fmla z16.h, z8.h, z0.h[6]\n"
- "fmla z20.h, z8.h, z1.h[6]\n"
- "fmla z17.h, z9.h, z0.h[6]\n"
- "fmla z21.h, z9.h, z1.h[6]\n"
- "fmla z18.h, z10.h, z0.h[6]\n"
- "fmla z22.h, z10.h, z1.h[6]\n"
- "fmla z19.h, z11.h, z0.h[6]\n"
- "fmla z23.h, z11.h, z1.h[6]\n"
- "b 5f\n"
- "4:\n"
- "fmla z16.h, z8.h, z0.h[0]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "fmla z20.h, z8.h, z1.h[0]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "fmla z17.h, z9.h, z0.h[0]\n"
- "ld1rqh z4.h, p6/z, [%[a_ptr0]]\n"
- "fmla z21.h, z9.h, z1.h[0]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z18.h, z10.h, z0.h[0]\n"
- "ld1rqh z5.h, p6/z, [a_ptr1]\n"
- "fmla z22.h, z10.h, z1.h[0]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z19.h, z11.h, z0.h[0]\n"
- "addvl %[a_ptr0], %[a_ptr0], #1\n"
- "fmla z23.h, z11.h, z1.h[0]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z16.h, z12.h, z0.h[1]\n"
- "addvl a_ptr1, a_ptr1, #1\n"
- "fmla z20.h, z12.h, z1.h[1]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "fmla z17.h, z13.h, z0.h[1]\n"
- "fmla z21.h, z13.h, z1.h[1]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "fmla z18.h, z14.h, z0.h[1]\n"
- "fmla z22.h, z14.h, z1.h[1]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z19.h, z15.h, z0.h[1]\n"
- "fmla z23.h, z15.h, z1.h[1]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z16.h, z8.h, z0.h[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "fmla z20.h, z8.h, z1.h[2]\n"
- "fmla z17.h, z9.h, z0.h[2]\n"
- "fmla z21.h, z9.h, z1.h[2]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "fmla z18.h, z10.h, z0.h[2]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "fmla z22.h, z10.h, z1.h[2]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "fmla z19.h, z11.h, z0.h[2]\n"
- "fmla z23.h, z11.h, z1.h[2]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "fmla z16.h, z12.h, z0.h[3]\n"
- "fmla z20.h, z12.h, z1.h[3]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "fmla z17.h, z13.h, z0.h[3]\n"
- "fmla z21.h, z13.h, z1.h[3]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "fmla z18.h, z14.h, z0.h[3]\n"
- "fmla z22.h, z14.h, z1.h[3]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "fmla z19.h, z15.h, z0.h[3]\n"
- "fmla z23.h, z15.h, z1.h[3]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "fmla z16.h, z8.h, z0.h[4]\n"
- "fmla z20.h, z8.h, z1.h[4]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "fmla z17.h, z9.h, z0.h[4]\n"
- "fmla z21.h, z9.h, z1.h[4]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z18.h, z10.h, z0.h[4]\n"
- "fmla z22.h, z10.h, z1.h[4]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z19.h, z11.h, z0.h[4]\n"
- "fmla z23.h, z11.h, z1.h[4]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z16.h, z12.h, z0.h[5]\n"
- "fmla z20.h, z12.h, z1.h[5]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "fmla z17.h, z13.h, z0.h[5]\n"
- "fmla z21.h, z13.h, z1.h[5]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "fmla z18.h, z14.h, z0.h[5]\n"
- "fmla z22.h, z14.h, z1.h[5]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z19.h, z15.h, z0.h[5]\n"
- "fmla z23.h, z15.h, z1.h[5]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z16.h, z8.h, z0.h[6]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "fmla z20.h, z8.h, z1.h[6]\n"
- "fmla z17.h, z9.h, z0.h[6]\n"
- "fmla z21.h, z9.h, z1.h[6]\n"
- "fmla z18.h, z10.h, z0.h[6]\n"
- "fmla z22.h, z10.h, z1.h[6]\n"
- "fmla z19.h, z11.h, z0.h[6]\n"
- "fmla z23.h, z11.h, z1.h[6]\n"
- "fmla z16.h, z12.h, z0.h[7]\n"
- "fmla z20.h, z12.h, z1.h[7]\n"
- "fmla z17.h, z13.h, z0.h[7]\n"
- "fmla z21.h, z13.h, z1.h[7]\n"
- "fmla z18.h, z14.h, z0.h[7]\n"
- "fmla z22.h, z14.h, z1.h[7]\n"
- "fmla z19.h, z15.h, z0.h[7]\n"
- "fmla z23.h, z15.h, z1.h[7]\n"
- "cbz %[blocks], 5f\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z16.h, z8.h, z4.h[0]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z20.h, z8.h, z5.h[0]\n"
- "fmla z17.h, z9.h, z4.h[0]\n"
- "fmla z21.h, z9.h, z5.h[0]\n"
- "fmla z18.h, z10.h, z4.h[0]\n"
- "fmla z22.h, z10.h, z5.h[0]\n"
- "fmla z19.h, z11.h, z4.h[0]\n"
- "fmla z23.h, z11.h, z5.h[0]\n"
- "b.eq 5f\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z16.h, z12.h, z4.h[1]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z20.h, z12.h, z5.h[1]\n"
- "fmla z17.h, z13.h, z4.h[1]\n"
- "fmla z21.h, z13.h, z5.h[1]\n"
- "fmla z18.h, z14.h, z4.h[1]\n"
- "fmla z22.h, z14.h, z5.h[1]\n"
- "fmla z19.h, z15.h, z4.h[1]\n"
- "fmla z23.h, z15.h, z5.h[1]\n"
- "b.eq 5f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "fmla z16.h, z8.h, z4.h[2]\n"
- "fmla z20.h, z8.h, z5.h[2]\n"
- "fmla z17.h, z9.h, z4.h[2]\n"
- "fmla z21.h, z9.h, z5.h[2]\n"
- "fmla z18.h, z10.h, z4.h[2]\n"
- "fmla z22.h, z10.h, z5.h[2]\n"
- "fmla z19.h, z11.h, z4.h[2]\n"
- "fmla z23.h, z11.h, z5.h[2]\n"
- "b.eq 5f\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "fmla z16.h, z12.h, z4.h[3]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "fmla z20.h, z12.h, z5.h[3]\n"
- "fmla z17.h, z13.h, z4.h[3]\n"
- "fmla z21.h, z13.h, z5.h[3]\n"
- "fmla z18.h, z14.h, z4.h[3]\n"
- "fmla z22.h, z14.h, z5.h[3]\n"
- "fmla z19.h, z15.h, z4.h[3]\n"
- "fmla z23.h, z15.h, z5.h[3]\n"
- "b.eq 5f\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z16.h, z8.h, z4.h[4]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z20.h, z8.h, z5.h[4]\n"
- "fmla z17.h, z9.h, z4.h[4]\n"
- "fmla z21.h, z9.h, z5.h[4]\n"
- "fmla z18.h, z10.h, z4.h[4]\n"
- "fmla z22.h, z10.h, z5.h[4]\n"
- "fmla z19.h, z11.h, z4.h[4]\n"
- "fmla z23.h, z11.h, z5.h[4]\n"
- "b.eq 5f\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z16.h, z12.h, z4.h[5]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z20.h, z12.h, z5.h[5]\n"
- "fmla z17.h, z13.h, z4.h[5]\n"
- "fmla z21.h, z13.h, z5.h[5]\n"
- "fmla z18.h, z14.h, z4.h[5]\n"
- "fmla z22.h, z14.h, z5.h[5]\n"
- "fmla z19.h, z15.h, z4.h[5]\n"
- "fmla z23.h, z15.h, z5.h[5]\n"
- "b.eq 5f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "fmla z16.h, z8.h, z4.h[6]\n"
- "fmla z20.h, z8.h, z5.h[6]\n"
- "fmla z17.h, z9.h, z4.h[6]\n"
- "fmla z21.h, z9.h, z5.h[6]\n"
- "fmla z18.h, z10.h, z4.h[6]\n"
- "fmla z22.h, z10.h, z5.h[6]\n"
- "fmla z19.h, z11.h, z4.h[6]\n"
- "fmla z23.h, z11.h, z5.h[6]\n"
- "5:\n"
- "ld1rh z14.h, p7/z, [%[minptr]]\n"
- "ld1rh z15.h, p7/z, [%[maxptr]]\n"
- "fmax z16.h, p7/m, z16.h, z14.h\n"
- "fmax z17.h, p7/m, z17.h, z14.h\n"
- "fmax z18.h, p7/m, z18.h, z14.h\n"
- "fmax z19.h, p7/m, z19.h, z14.h\n"
- "fmin z16.h, p7/m, z16.h, z15.h\n"
- "fmin z17.h, p7/m, z17.h, z15.h\n"
- "fmin z18.h, p7/m, z18.h, z15.h\n"
- "fmin z19.h, p7/m, z19.h, z15.h\n"
- "st1h z16.h, p0, [%[c_ptr0]]\n"
- "fmax z20.h, p7/m, z20.h, z14.h\n"
- "fmax z21.h, p7/m, z21.h, z14.h\n"
- "fmax z22.h, p7/m, z22.h, z14.h\n"
- "st1h z17.h, p1, [%[c_ptr0], #1, MUL VL]\n"
- "fmax z23.h, p7/m, z23.h, z14.h\n"
- "fmin z20.h, p7/m, z20.h, z15.h\n"
- "fmin z21.h, p7/m, z21.h, z15.h\n"
- "st1h z18.h, p2, [%[c_ptr0], #2, MUL VL]\n"
- "fmin z22.h, p7/m, z22.h, z15.h\n"
- "fmin z23.h, p7/m, z23.h, z15.h\n"
- "st1h z19.h, p3, [%[c_ptr0], #3, MUL VL]\n"
- "addvl %[c_ptr0], %[c_ptr0], #4\n"
- "st1h z20.h, p0, [c_ptr1]\n"
- "st1h z21.h, p1, [c_ptr1, #1, MUL VL]\n"
- "st1h z22.h, p2, [c_ptr1, #2, MUL VL]\n"
- "st1h z23.h, p3, [c_ptr1, #3, MUL VL]\n"
- ".unreq a_ptr1\n"
- ".unreq c_ptr1\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks)
- : [width] "r" (width), [accumulate] "r" (static_cast<uint64_t>(accumulate)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers)
- : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "cc", "memory"
- );
- break;
- case 3:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "c_ptr1 .req X2\n"
- "c_ptr2 .req X3\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "whilelt p6.h, %[temp], %[leftovers]\n"
- "whilelt p0.h, %[temp], %[width]\n"
- "inch %[temp], all, mul #1\n"
- "ptrue p7.h\n"
- "whilelt p1.h, %[temp], %[width]\n"
- "inch %[temp], all, mul #1\n"
- "whilelt p2.h, %[temp], %[width]\n"
- "inch %[temp], all, mul #1\n"
- "whilelt p3.h, %[temp], %[width]\n"
- "cbnz %[accumulate], 1f\n"
- "ld1h z16.h, p0/z, [%[biasptr]]\n"
- "ld1h z17.h, p1/z, [%[biasptr], #1, MUL VL]\n"
- "ld1h z18.h, p2/z, [%[biasptr], #2, MUL VL]\n"
- "ld1h z19.h, p3/z, [%[biasptr], #3, MUL VL]\n"
- "mov z20.d, z16.d\n"
- "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n"
- "mov z21.d, z17.d\n"
- "ld1rqh z1.h, p7/z, [a_ptr1]\n"
- "mov z22.d, z18.d\n"
- "ld1rqh z2.h, p7/z, [a_ptr2]\n"
- "mov z23.d, z19.d\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "mov z24.d, z16.d\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "mov z25.d, z17.d\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "mov z26.d, z18.d\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "mov z27.d, z19.d\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "ld1h z16.h, p0/z, [%[c_ptr0]]\n"
- "ld1h z17.h, p1/z, [%[c_ptr0], #1, MUL VL]\n"
- "ld1h z18.h, p2/z, [%[c_ptr0], #2, MUL VL]\n"
- "ld1h z19.h, p3/z, [%[c_ptr0], #3, MUL VL]\n"
- "ld1h z20.h, p0/z, [c_ptr1]\n"
- "ld1h z21.h, p1/z, [c_ptr1, #1, MUL VL]\n"
- "ld1h z22.h, p2/z, [c_ptr1, #2, MUL VL]\n"
- "ld1h z23.h, p3/z, [c_ptr1, #3, MUL VL]\n"
- "ld1h z24.h, p0/z, [c_ptr2]\n"
- "ld1h z25.h, p1/z, [c_ptr2, #1, MUL VL]\n"
- "ld1h z26.h, p2/z, [c_ptr2, #2, MUL VL]\n"
- "ld1h z27.h, p3/z, [c_ptr2, #3, MUL VL]\n"
- "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ld1rqh z1.h, p7/z, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "ld1rqh z2.h, p7/z, [a_ptr2]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "cbz %[loops], 2f\n"
- "3:\n"
- "fmla z16.h, z8.h, z0.h[0]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "fmla z20.h, z8.h, z1.h[0]\n"
- "ld1rqh z4.h, p7/z, [%[a_ptr0]]\n"
- "fmla z24.h, z8.h, z2.h[0]\n"
- "ld1rqh z5.h, p7/z, [a_ptr1]\n"
- "fmla z17.h, z9.h, z0.h[0]\n"
- "ld1rqh z6.h, p7/z, [a_ptr2]\n"
- "fmla z21.h, z9.h, z1.h[0]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "fmla z25.h, z9.h, z2.h[0]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z18.h, z10.h, z0.h[0]\n"
- "subs %[loops], %[loops], #0x1\n"
- "fmla z22.h, z10.h, z1.h[0]\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
- "fmla z26.h, z10.h, z2.h[0]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z19.h, z11.h, z0.h[0]\n"
- "add a_ptr1, a_ptr1, #0x20\n"
- "fmla z23.h, z11.h, z1.h[0]\n"
- "add a_ptr2, a_ptr2, #0x20\n"
- "fmla z27.h, z11.h, z2.h[0]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z16.h, z12.h, z0.h[1]\n"
- "fmla z20.h, z12.h, z1.h[1]\n"
- "fmla z24.h, z12.h, z2.h[1]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "fmla z17.h, z13.h, z0.h[1]\n"
- "fmla z21.h, z13.h, z1.h[1]\n"
- "fmla z25.h, z13.h, z2.h[1]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "fmla z18.h, z14.h, z0.h[1]\n"
- "fmla z22.h, z14.h, z1.h[1]\n"
- "fmla z26.h, z14.h, z2.h[1]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z19.h, z15.h, z0.h[1]\n"
- "fmla z23.h, z15.h, z1.h[1]\n"
- "fmla z27.h, z15.h, z2.h[1]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z16.h, z8.h, z0.h[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "fmla z20.h, z8.h, z1.h[2]\n"
- "fmla z24.h, z8.h, z2.h[2]\n"
- "fmla z17.h, z9.h, z0.h[2]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "fmla z21.h, z9.h, z1.h[2]\n"
- "fmla z25.h, z9.h, z2.h[2]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "fmla z18.h, z10.h, z0.h[2]\n"
- "fmla z22.h, z10.h, z1.h[2]\n"
- "fmla z26.h, z10.h, z2.h[2]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "fmla z19.h, z11.h, z0.h[2]\n"
- "fmla z23.h, z11.h, z1.h[2]\n"
- "fmla z27.h, z11.h, z2.h[2]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "fmla z16.h, z12.h, z0.h[3]\n"
- "fmla z20.h, z12.h, z1.h[3]\n"
- "fmla z24.h, z12.h, z2.h[3]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "fmla z17.h, z13.h, z0.h[3]\n"
- "fmla z21.h, z13.h, z1.h[3]\n"
- "fmla z25.h, z13.h, z2.h[3]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "fmla z18.h, z14.h, z0.h[3]\n"
- "fmla z22.h, z14.h, z1.h[3]\n"
- "fmla z26.h, z14.h, z2.h[3]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "fmla z19.h, z15.h, z0.h[3]\n"
- "fmla z23.h, z15.h, z1.h[3]\n"
- "fmla z27.h, z15.h, z2.h[3]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "fmla z16.h, z8.h, z0.h[4]\n"
- "fmla z20.h, z8.h, z1.h[4]\n"
- "fmla z24.h, z8.h, z2.h[4]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "fmla z17.h, z9.h, z0.h[4]\n"
- "fmla z21.h, z9.h, z1.h[4]\n"
- "fmla z25.h, z9.h, z2.h[4]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z18.h, z10.h, z0.h[4]\n"
- "fmla z22.h, z10.h, z1.h[4]\n"
- "fmla z26.h, z10.h, z2.h[4]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z19.h, z11.h, z0.h[4]\n"
- "fmla z23.h, z11.h, z1.h[4]\n"
- "fmla z27.h, z11.h, z2.h[4]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z16.h, z12.h, z0.h[5]\n"
- "fmla z20.h, z12.h, z1.h[5]\n"
- "fmla z24.h, z12.h, z2.h[5]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "fmla z17.h, z13.h, z0.h[5]\n"
- "fmla z21.h, z13.h, z1.h[5]\n"
- "fmla z25.h, z13.h, z2.h[5]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "fmla z18.h, z14.h, z0.h[5]\n"
- "fmla z22.h, z14.h, z1.h[5]\n"
- "fmla z26.h, z14.h, z2.h[5]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z19.h, z15.h, z0.h[5]\n"
- "fmla z23.h, z15.h, z1.h[5]\n"
- "fmla z27.h, z15.h, z2.h[5]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z16.h, z8.h, z0.h[6]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "fmla z20.h, z8.h, z1.h[6]\n"
- "fmla z24.h, z8.h, z2.h[6]\n"
- "fmla z17.h, z9.h, z0.h[6]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "fmla z21.h, z9.h, z1.h[6]\n"
- "fmla z25.h, z9.h, z2.h[6]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "fmla z18.h, z10.h, z0.h[6]\n"
- "fmla z22.h, z10.h, z1.h[6]\n"
- "fmla z26.h, z10.h, z2.h[6]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "fmla z19.h, z11.h, z0.h[6]\n"
- "fmla z23.h, z11.h, z1.h[6]\n"
- "fmla z27.h, z11.h, z2.h[6]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "fmla z16.h, z12.h, z0.h[7]\n"
- "fmla z20.h, z12.h, z1.h[7]\n"
- "fmla z24.h, z12.h, z2.h[7]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "fmla z17.h, z13.h, z0.h[7]\n"
- "fmla z21.h, z13.h, z1.h[7]\n"
- "fmla z25.h, z13.h, z2.h[7]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "fmla z18.h, z14.h, z0.h[7]\n"
- "fmla z22.h, z14.h, z1.h[7]\n"
- "fmla z26.h, z14.h, z2.h[7]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "fmla z19.h, z15.h, z0.h[7]\n"
- "ld1rqh z0.h, p7/z, [%[a_ptr0], #-0x10]\n"
- "fmla z23.h, z15.h, z1.h[7]\n"
- "ld1rqh z1.h, p7/z, [a_ptr1, #-0x10]\n"
- "fmla z27.h, z15.h, z2.h[7]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "fmla z16.h, z8.h, z4.h[0]\n"
- "ld1rqh z2.h, p7/z, [a_ptr2, #-0x10]\n"
- "fmla z20.h, z8.h, z5.h[0]\n"
- "fmla z24.h, z8.h, z6.h[0]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "fmla z17.h, z9.h, z4.h[0]\n"
- "fmla z21.h, z9.h, z5.h[0]\n"
- "fmla z25.h, z9.h, z6.h[0]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z18.h, z10.h, z4.h[0]\n"
- "fmla z22.h, z10.h, z5.h[0]\n"
- "fmla z26.h, z10.h, z6.h[0]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z19.h, z11.h, z4.h[0]\n"
- "fmla z23.h, z11.h, z5.h[0]\n"
- "fmla z27.h, z11.h, z6.h[0]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z16.h, z12.h, z4.h[1]\n"
- "fmla z20.h, z12.h, z5.h[1]\n"
- "fmla z24.h, z12.h, z6.h[1]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "fmla z17.h, z13.h, z4.h[1]\n"
- "fmla z21.h, z13.h, z5.h[1]\n"
- "fmla z25.h, z13.h, z6.h[1]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "fmla z18.h, z14.h, z4.h[1]\n"
- "fmla z22.h, z14.h, z5.h[1]\n"
- "fmla z26.h, z14.h, z6.h[1]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z19.h, z15.h, z4.h[1]\n"
- "fmla z23.h, z15.h, z5.h[1]\n"
- "fmla z27.h, z15.h, z6.h[1]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z16.h, z8.h, z4.h[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "fmla z20.h, z8.h, z5.h[2]\n"
- "fmla z24.h, z8.h, z6.h[2]\n"
- "fmla z17.h, z9.h, z4.h[2]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "fmla z21.h, z9.h, z5.h[2]\n"
- "fmla z25.h, z9.h, z6.h[2]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "fmla z18.h, z10.h, z4.h[2]\n"
- "fmla z22.h, z10.h, z5.h[2]\n"
- "fmla z26.h, z10.h, z6.h[2]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "fmla z19.h, z11.h, z4.h[2]\n"
- "fmla z23.h, z11.h, z5.h[2]\n"
- "fmla z27.h, z11.h, z6.h[2]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "fmla z16.h, z12.h, z4.h[3]\n"
- "fmla z20.h, z12.h, z5.h[3]\n"
- "fmla z24.h, z12.h, z6.h[3]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "fmla z17.h, z13.h, z4.h[3]\n"
- "fmla z21.h, z13.h, z5.h[3]\n"
- "fmla z25.h, z13.h, z6.h[3]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "fmla z18.h, z14.h, z4.h[3]\n"
- "fmla z22.h, z14.h, z5.h[3]\n"
- "fmla z26.h, z14.h, z6.h[3]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "fmla z19.h, z15.h, z4.h[3]\n"
- "fmla z23.h, z15.h, z5.h[3]\n"
- "fmla z27.h, z15.h, z6.h[3]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "fmla z16.h, z8.h, z4.h[4]\n"
- "fmla z20.h, z8.h, z5.h[4]\n"
- "fmla z24.h, z8.h, z6.h[4]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "fmla z17.h, z9.h, z4.h[4]\n"
- "fmla z21.h, z9.h, z5.h[4]\n"
- "fmla z25.h, z9.h, z6.h[4]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z18.h, z10.h, z4.h[4]\n"
- "fmla z22.h, z10.h, z5.h[4]\n"
- "fmla z26.h, z10.h, z6.h[4]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z19.h, z11.h, z4.h[4]\n"
- "fmla z23.h, z11.h, z5.h[4]\n"
- "fmla z27.h, z11.h, z6.h[4]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z16.h, z12.h, z4.h[5]\n"
- "fmla z20.h, z12.h, z5.h[5]\n"
- "fmla z24.h, z12.h, z6.h[5]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "fmla z17.h, z13.h, z4.h[5]\n"
- "fmla z21.h, z13.h, z5.h[5]\n"
- "fmla z25.h, z13.h, z6.h[5]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "fmla z18.h, z14.h, z4.h[5]\n"
- "fmla z22.h, z14.h, z5.h[5]\n"
- "fmla z26.h, z14.h, z6.h[5]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z19.h, z15.h, z4.h[5]\n"
- "fmla z23.h, z15.h, z5.h[5]\n"
- "fmla z27.h, z15.h, z6.h[5]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z16.h, z8.h, z4.h[6]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "fmla z20.h, z8.h, z5.h[6]\n"
- "fmla z24.h, z8.h, z6.h[6]\n"
- "fmla z17.h, z9.h, z4.h[6]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "fmla z21.h, z9.h, z5.h[6]\n"
- "fmla z25.h, z9.h, z6.h[6]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "fmla z18.h, z10.h, z4.h[6]\n"
- "fmla z22.h, z10.h, z5.h[6]\n"
- "fmla z26.h, z10.h, z6.h[6]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "fmla z19.h, z11.h, z4.h[6]\n"
- "fmla z23.h, z11.h, z5.h[6]\n"
- "fmla z27.h, z11.h, z6.h[6]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "fmla z16.h, z12.h, z4.h[7]\n"
- "fmla z20.h, z12.h, z5.h[7]\n"
- "fmla z24.h, z12.h, z6.h[7]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "fmla z17.h, z13.h, z4.h[7]\n"
- "fmla z21.h, z13.h, z5.h[7]\n"
- "fmla z25.h, z13.h, z6.h[7]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "fmla z18.h, z14.h, z4.h[7]\n"
- "fmla z22.h, z14.h, z5.h[7]\n"
- "fmla z26.h, z14.h, z6.h[7]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "fmla z19.h, z15.h, z4.h[7]\n"
- "fmla z23.h, z15.h, z5.h[7]\n"
- "fmla z27.h, z15.h, z6.h[7]\n"
- "b.ne 3b\n"
- "2:\n"
- "cbz %[regs], 4f\n"
- "fmla z16.h, z8.h, z0.h[0]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "fmla z20.h, z8.h, z1.h[0]\n"
- "ld1rqh z4.h, p7/z, [%[a_ptr0]]\n"
- "fmla z24.h, z8.h, z2.h[0]\n"
- "ld1rqh z5.h, p7/z, [a_ptr1]\n"
- "fmla z17.h, z9.h, z0.h[0]\n"
- "ld1rqh z6.h, p7/z, [a_ptr2]\n"
- "fmla z21.h, z9.h, z1.h[0]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "fmla z25.h, z9.h, z2.h[0]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z18.h, z10.h, z0.h[0]\n"
- "fmla z22.h, z10.h, z1.h[0]\n"
- "fmla z26.h, z10.h, z2.h[0]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z19.h, z11.h, z0.h[0]\n"
- "fmla z23.h, z11.h, z1.h[0]\n"
- "fmla z27.h, z11.h, z2.h[0]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z16.h, z12.h, z0.h[1]\n"
- "fmla z20.h, z12.h, z1.h[1]\n"
- "fmla z24.h, z12.h, z2.h[1]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "fmla z17.h, z13.h, z0.h[1]\n"
- "fmla z21.h, z13.h, z1.h[1]\n"
- "fmla z25.h, z13.h, z2.h[1]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "fmla z18.h, z14.h, z0.h[1]\n"
- "fmla z22.h, z14.h, z1.h[1]\n"
- "fmla z26.h, z14.h, z2.h[1]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z19.h, z15.h, z0.h[1]\n"
- "fmla z23.h, z15.h, z1.h[1]\n"
- "fmla z27.h, z15.h, z2.h[1]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z16.h, z8.h, z0.h[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "fmla z20.h, z8.h, z1.h[2]\n"
- "fmla z24.h, z8.h, z2.h[2]\n"
- "fmla z17.h, z9.h, z0.h[2]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "fmla z21.h, z9.h, z1.h[2]\n"
- "fmla z25.h, z9.h, z2.h[2]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "fmla z18.h, z10.h, z0.h[2]\n"
- "fmla z22.h, z10.h, z1.h[2]\n"
- "fmla z26.h, z10.h, z2.h[2]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "fmla z19.h, z11.h, z0.h[2]\n"
- "fmla z23.h, z11.h, z1.h[2]\n"
- "fmla z27.h, z11.h, z2.h[2]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "fmla z16.h, z12.h, z0.h[3]\n"
- "fmla z20.h, z12.h, z1.h[3]\n"
- "fmla z24.h, z12.h, z2.h[3]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "fmla z17.h, z13.h, z0.h[3]\n"
- "fmla z21.h, z13.h, z1.h[3]\n"
- "fmla z25.h, z13.h, z2.h[3]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "fmla z18.h, z14.h, z0.h[3]\n"
- "fmla z22.h, z14.h, z1.h[3]\n"
- "fmla z26.h, z14.h, z2.h[3]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "fmla z19.h, z15.h, z0.h[3]\n"
- "fmla z23.h, z15.h, z1.h[3]\n"
- "fmla z27.h, z15.h, z2.h[3]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "fmla z16.h, z8.h, z0.h[4]\n"
- "fmla z20.h, z8.h, z1.h[4]\n"
- "fmla z24.h, z8.h, z2.h[4]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "fmla z17.h, z9.h, z0.h[4]\n"
- "fmla z21.h, z9.h, z1.h[4]\n"
- "fmla z25.h, z9.h, z2.h[4]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z18.h, z10.h, z0.h[4]\n"
- "fmla z22.h, z10.h, z1.h[4]\n"
- "fmla z26.h, z10.h, z2.h[4]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z19.h, z11.h, z0.h[4]\n"
- "fmla z23.h, z11.h, z1.h[4]\n"
- "fmla z27.h, z11.h, z2.h[4]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z16.h, z12.h, z0.h[5]\n"
- "fmla z20.h, z12.h, z1.h[5]\n"
- "fmla z24.h, z12.h, z2.h[5]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "fmla z17.h, z13.h, z0.h[5]\n"
- "fmla z21.h, z13.h, z1.h[5]\n"
- "fmla z25.h, z13.h, z2.h[5]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "fmla z18.h, z14.h, z0.h[5]\n"
- "fmla z22.h, z14.h, z1.h[5]\n"
- "fmla z26.h, z14.h, z2.h[5]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z19.h, z15.h, z0.h[5]\n"
- "fmla z23.h, z15.h, z1.h[5]\n"
- "fmla z27.h, z15.h, z2.h[5]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z16.h, z8.h, z0.h[6]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "fmla z20.h, z8.h, z1.h[6]\n"
- "fmla z24.h, z8.h, z2.h[6]\n"
- "fmla z17.h, z9.h, z0.h[6]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "fmla z21.h, z9.h, z1.h[6]\n"
- "fmla z25.h, z9.h, z2.h[6]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "fmla z18.h, z10.h, z0.h[6]\n"
- "fmla z22.h, z10.h, z1.h[6]\n"
- "fmla z26.h, z10.h, z2.h[6]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "fmla z19.h, z11.h, z0.h[6]\n"
- "fmla z23.h, z11.h, z1.h[6]\n"
- "fmla z27.h, z11.h, z2.h[6]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "fmla z16.h, z12.h, z0.h[7]\n"
- "fmla z20.h, z12.h, z1.h[7]\n"
- "fmla z24.h, z12.h, z2.h[7]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "fmla z17.h, z13.h, z0.h[7]\n"
- "fmla z21.h, z13.h, z1.h[7]\n"
- "fmla z25.h, z13.h, z2.h[7]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "fmla z18.h, z14.h, z0.h[7]\n"
- "fmla z22.h, z14.h, z1.h[7]\n"
- "fmla z26.h, z14.h, z2.h[7]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "fmla z19.h, z15.h, z0.h[7]\n"
- "ld1rqh z0.h, p6/z, [%[a_ptr0], #0x10]\n"
- "fmla z23.h, z15.h, z1.h[7]\n"
- "ld1rqh z1.h, p6/z, [a_ptr1, #0x10]\n"
- "fmla z27.h, z15.h, z2.h[7]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "fmla z16.h, z8.h, z4.h[0]\n"
- "ld1rqh z2.h, p6/z, [a_ptr2, #0x10]\n"
- "fmla z20.h, z8.h, z5.h[0]\n"
- "addvl %[a_ptr0], %[a_ptr0], #2\n"
- "fmla z24.h, z8.h, z6.h[0]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "fmla z17.h, z9.h, z4.h[0]\n"
- "addvl a_ptr1, a_ptr1, #2\n"
- "fmla z21.h, z9.h, z5.h[0]\n"
- "addvl a_ptr2, a_ptr2, #2\n"
- "fmla z25.h, z9.h, z6.h[0]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z18.h, z10.h, z4.h[0]\n"
- "fmla z22.h, z10.h, z5.h[0]\n"
- "fmla z26.h, z10.h, z6.h[0]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z19.h, z11.h, z4.h[0]\n"
- "fmla z23.h, z11.h, z5.h[0]\n"
- "fmla z27.h, z11.h, z6.h[0]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z16.h, z12.h, z4.h[1]\n"
- "fmla z20.h, z12.h, z5.h[1]\n"
- "fmla z24.h, z12.h, z6.h[1]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "fmla z17.h, z13.h, z4.h[1]\n"
- "fmla z21.h, z13.h, z5.h[1]\n"
- "fmla z25.h, z13.h, z6.h[1]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "fmla z18.h, z14.h, z4.h[1]\n"
- "fmla z22.h, z14.h, z5.h[1]\n"
- "fmla z26.h, z14.h, z6.h[1]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z19.h, z15.h, z4.h[1]\n"
- "fmla z23.h, z15.h, z5.h[1]\n"
- "fmla z27.h, z15.h, z6.h[1]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z16.h, z8.h, z4.h[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "fmla z20.h, z8.h, z5.h[2]\n"
- "fmla z24.h, z8.h, z6.h[2]\n"
- "fmla z17.h, z9.h, z4.h[2]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "fmla z21.h, z9.h, z5.h[2]\n"
- "fmla z25.h, z9.h, z6.h[2]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "fmla z18.h, z10.h, z4.h[2]\n"
- "fmla z22.h, z10.h, z5.h[2]\n"
- "fmla z26.h, z10.h, z6.h[2]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "fmla z19.h, z11.h, z4.h[2]\n"
- "fmla z23.h, z11.h, z5.h[2]\n"
- "fmla z27.h, z11.h, z6.h[2]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "fmla z16.h, z12.h, z4.h[3]\n"
- "fmla z20.h, z12.h, z5.h[3]\n"
- "fmla z24.h, z12.h, z6.h[3]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "fmla z17.h, z13.h, z4.h[3]\n"
- "fmla z21.h, z13.h, z5.h[3]\n"
- "fmla z25.h, z13.h, z6.h[3]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "fmla z18.h, z14.h, z4.h[3]\n"
- "fmla z22.h, z14.h, z5.h[3]\n"
- "fmla z26.h, z14.h, z6.h[3]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "fmla z19.h, z15.h, z4.h[3]\n"
- "fmla z23.h, z15.h, z5.h[3]\n"
- "fmla z27.h, z15.h, z6.h[3]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "fmla z16.h, z8.h, z4.h[4]\n"
- "fmla z20.h, z8.h, z5.h[4]\n"
- "fmla z24.h, z8.h, z6.h[4]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "fmla z17.h, z9.h, z4.h[4]\n"
- "fmla z21.h, z9.h, z5.h[4]\n"
- "fmla z25.h, z9.h, z6.h[4]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z18.h, z10.h, z4.h[4]\n"
- "fmla z22.h, z10.h, z5.h[4]\n"
- "fmla z26.h, z10.h, z6.h[4]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z19.h, z11.h, z4.h[4]\n"
- "fmla z23.h, z11.h, z5.h[4]\n"
- "fmla z27.h, z11.h, z6.h[4]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z16.h, z12.h, z4.h[5]\n"
- "fmla z20.h, z12.h, z5.h[5]\n"
- "fmla z24.h, z12.h, z6.h[5]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "fmla z17.h, z13.h, z4.h[5]\n"
- "fmla z21.h, z13.h, z5.h[5]\n"
- "fmla z25.h, z13.h, z6.h[5]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "fmla z18.h, z14.h, z4.h[5]\n"
- "fmla z22.h, z14.h, z5.h[5]\n"
- "fmla z26.h, z14.h, z6.h[5]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z19.h, z15.h, z4.h[5]\n"
- "fmla z23.h, z15.h, z5.h[5]\n"
- "fmla z27.h, z15.h, z6.h[5]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z16.h, z8.h, z4.h[6]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "fmla z20.h, z8.h, z5.h[6]\n"
- "fmla z24.h, z8.h, z6.h[6]\n"
- "fmla z17.h, z9.h, z4.h[6]\n"
- "fmla z21.h, z9.h, z5.h[6]\n"
- "fmla z25.h, z9.h, z6.h[6]\n"
- "fmla z18.h, z10.h, z4.h[6]\n"
- "fmla z22.h, z10.h, z5.h[6]\n"
- "fmla z26.h, z10.h, z6.h[6]\n"
- "fmla z19.h, z11.h, z4.h[6]\n"
- "fmla z23.h, z11.h, z5.h[6]\n"
- "fmla z27.h, z11.h, z6.h[6]\n"
- "fmla z16.h, z12.h, z4.h[7]\n"
- "fmla z20.h, z12.h, z5.h[7]\n"
- "fmla z24.h, z12.h, z6.h[7]\n"
- "fmla z17.h, z13.h, z4.h[7]\n"
- "fmla z21.h, z13.h, z5.h[7]\n"
- "fmla z25.h, z13.h, z6.h[7]\n"
- "fmla z18.h, z14.h, z4.h[7]\n"
- "fmla z22.h, z14.h, z5.h[7]\n"
- "fmla z26.h, z14.h, z6.h[7]\n"
- "fmla z19.h, z15.h, z4.h[7]\n"
- "fmla z23.h, z15.h, z5.h[7]\n"
- "fmla z27.h, z15.h, z6.h[7]\n"
- "cbz %[blocks], 5f\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z16.h, z8.h, z0.h[0]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z20.h, z8.h, z1.h[0]\n"
- "fmla z24.h, z8.h, z2.h[0]\n"
- "fmla z17.h, z9.h, z0.h[0]\n"
- "fmla z21.h, z9.h, z1.h[0]\n"
- "fmla z25.h, z9.h, z2.h[0]\n"
- "fmla z18.h, z10.h, z0.h[0]\n"
- "fmla z22.h, z10.h, z1.h[0]\n"
- "fmla z26.h, z10.h, z2.h[0]\n"
- "fmla z19.h, z11.h, z0.h[0]\n"
- "fmla z23.h, z11.h, z1.h[0]\n"
- "fmla z27.h, z11.h, z2.h[0]\n"
- "b.eq 5f\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z16.h, z12.h, z0.h[1]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z20.h, z12.h, z1.h[1]\n"
- "fmla z24.h, z12.h, z2.h[1]\n"
- "fmla z17.h, z13.h, z0.h[1]\n"
- "fmla z21.h, z13.h, z1.h[1]\n"
- "fmla z25.h, z13.h, z2.h[1]\n"
- "fmla z18.h, z14.h, z0.h[1]\n"
- "fmla z22.h, z14.h, z1.h[1]\n"
- "fmla z26.h, z14.h, z2.h[1]\n"
- "fmla z19.h, z15.h, z0.h[1]\n"
- "fmla z23.h, z15.h, z1.h[1]\n"
- "fmla z27.h, z15.h, z2.h[1]\n"
- "b.eq 5f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "fmla z16.h, z8.h, z0.h[2]\n"
- "fmla z20.h, z8.h, z1.h[2]\n"
- "fmla z24.h, z8.h, z2.h[2]\n"
- "fmla z17.h, z9.h, z0.h[2]\n"
- "fmla z21.h, z9.h, z1.h[2]\n"
- "fmla z25.h, z9.h, z2.h[2]\n"
- "fmla z18.h, z10.h, z0.h[2]\n"
- "fmla z22.h, z10.h, z1.h[2]\n"
- "fmla z26.h, z10.h, z2.h[2]\n"
- "fmla z19.h, z11.h, z0.h[2]\n"
- "fmla z23.h, z11.h, z1.h[2]\n"
- "fmla z27.h, z11.h, z2.h[2]\n"
- "b.eq 5f\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "fmla z16.h, z12.h, z0.h[3]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "fmla z20.h, z12.h, z1.h[3]\n"
- "fmla z24.h, z12.h, z2.h[3]\n"
- "fmla z17.h, z13.h, z0.h[3]\n"
- "fmla z21.h, z13.h, z1.h[3]\n"
- "fmla z25.h, z13.h, z2.h[3]\n"
- "fmla z18.h, z14.h, z0.h[3]\n"
- "fmla z22.h, z14.h, z1.h[3]\n"
- "fmla z26.h, z14.h, z2.h[3]\n"
- "fmla z19.h, z15.h, z0.h[3]\n"
- "fmla z23.h, z15.h, z1.h[3]\n"
- "fmla z27.h, z15.h, z2.h[3]\n"
- "b.eq 5f\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z16.h, z8.h, z0.h[4]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z20.h, z8.h, z1.h[4]\n"
- "fmla z24.h, z8.h, z2.h[4]\n"
- "fmla z17.h, z9.h, z0.h[4]\n"
- "fmla z21.h, z9.h, z1.h[4]\n"
- "fmla z25.h, z9.h, z2.h[4]\n"
- "fmla z18.h, z10.h, z0.h[4]\n"
- "fmla z22.h, z10.h, z1.h[4]\n"
- "fmla z26.h, z10.h, z2.h[4]\n"
- "fmla z19.h, z11.h, z0.h[4]\n"
- "fmla z23.h, z11.h, z1.h[4]\n"
- "fmla z27.h, z11.h, z2.h[4]\n"
- "b.eq 5f\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z16.h, z12.h, z0.h[5]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z20.h, z12.h, z1.h[5]\n"
- "fmla z24.h, z12.h, z2.h[5]\n"
- "fmla z17.h, z13.h, z0.h[5]\n"
- "fmla z21.h, z13.h, z1.h[5]\n"
- "fmla z25.h, z13.h, z2.h[5]\n"
- "fmla z18.h, z14.h, z0.h[5]\n"
- "fmla z22.h, z14.h, z1.h[5]\n"
- "fmla z26.h, z14.h, z2.h[5]\n"
- "fmla z19.h, z15.h, z0.h[5]\n"
- "fmla z23.h, z15.h, z1.h[5]\n"
- "fmla z27.h, z15.h, z2.h[5]\n"
- "b.eq 5f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "fmla z16.h, z8.h, z0.h[6]\n"
- "fmla z20.h, z8.h, z1.h[6]\n"
- "fmla z24.h, z8.h, z2.h[6]\n"
- "fmla z17.h, z9.h, z0.h[6]\n"
- "fmla z21.h, z9.h, z1.h[6]\n"
- "fmla z25.h, z9.h, z2.h[6]\n"
- "fmla z18.h, z10.h, z0.h[6]\n"
- "fmla z22.h, z10.h, z1.h[6]\n"
- "fmla z26.h, z10.h, z2.h[6]\n"
- "fmla z19.h, z11.h, z0.h[6]\n"
- "fmla z23.h, z11.h, z1.h[6]\n"
- "fmla z27.h, z11.h, z2.h[6]\n"
- "b 5f\n"
- "4:\n"
- "fmla z16.h, z8.h, z0.h[0]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "fmla z20.h, z8.h, z1.h[0]\n"
- "ld1rqh z4.h, p6/z, [%[a_ptr0]]\n"
- "fmla z24.h, z8.h, z2.h[0]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "fmla z17.h, z9.h, z0.h[0]\n"
- "ld1rqh z5.h, p6/z, [a_ptr1]\n"
- "fmla z21.h, z9.h, z1.h[0]\n"
- "ld1rqh z6.h, p6/z, [a_ptr2]\n"
- "fmla z25.h, z9.h, z2.h[0]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z18.h, z10.h, z0.h[0]\n"
- "addvl %[a_ptr0], %[a_ptr0], #1\n"
- "fmla z22.h, z10.h, z1.h[0]\n"
- "addvl a_ptr1, a_ptr1, #1\n"
- "fmla z26.h, z10.h, z2.h[0]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z19.h, z11.h, z0.h[0]\n"
- "addvl a_ptr2, a_ptr2, #1\n"
- "fmla z23.h, z11.h, z1.h[0]\n"
- "fmla z27.h, z11.h, z2.h[0]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z16.h, z12.h, z0.h[1]\n"
- "fmla z20.h, z12.h, z1.h[1]\n"
- "fmla z24.h, z12.h, z2.h[1]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "fmla z17.h, z13.h, z0.h[1]\n"
- "fmla z21.h, z13.h, z1.h[1]\n"
- "fmla z25.h, z13.h, z2.h[1]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "fmla z18.h, z14.h, z0.h[1]\n"
- "fmla z22.h, z14.h, z1.h[1]\n"
- "fmla z26.h, z14.h, z2.h[1]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z19.h, z15.h, z0.h[1]\n"
- "fmla z23.h, z15.h, z1.h[1]\n"
- "fmla z27.h, z15.h, z2.h[1]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z16.h, z8.h, z0.h[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "fmla z20.h, z8.h, z1.h[2]\n"
- "fmla z24.h, z8.h, z2.h[2]\n"
- "fmla z17.h, z9.h, z0.h[2]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "fmla z21.h, z9.h, z1.h[2]\n"
- "fmla z25.h, z9.h, z2.h[2]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "fmla z18.h, z10.h, z0.h[2]\n"
- "fmla z22.h, z10.h, z1.h[2]\n"
- "fmla z26.h, z10.h, z2.h[2]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "fmla z19.h, z11.h, z0.h[2]\n"
- "fmla z23.h, z11.h, z1.h[2]\n"
- "fmla z27.h, z11.h, z2.h[2]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "fmla z16.h, z12.h, z0.h[3]\n"
- "fmla z20.h, z12.h, z1.h[3]\n"
- "fmla z24.h, z12.h, z2.h[3]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "fmla z17.h, z13.h, z0.h[3]\n"
- "fmla z21.h, z13.h, z1.h[3]\n"
- "fmla z25.h, z13.h, z2.h[3]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "fmla z18.h, z14.h, z0.h[3]\n"
- "fmla z22.h, z14.h, z1.h[3]\n"
- "fmla z26.h, z14.h, z2.h[3]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "fmla z19.h, z15.h, z0.h[3]\n"
- "fmla z23.h, z15.h, z1.h[3]\n"
- "fmla z27.h, z15.h, z2.h[3]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "fmla z16.h, z8.h, z0.h[4]\n"
- "fmla z20.h, z8.h, z1.h[4]\n"
- "fmla z24.h, z8.h, z2.h[4]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "fmla z17.h, z9.h, z0.h[4]\n"
- "fmla z21.h, z9.h, z1.h[4]\n"
- "fmla z25.h, z9.h, z2.h[4]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z18.h, z10.h, z0.h[4]\n"
- "fmla z22.h, z10.h, z1.h[4]\n"
- "fmla z26.h, z10.h, z2.h[4]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z19.h, z11.h, z0.h[4]\n"
- "fmla z23.h, z11.h, z1.h[4]\n"
- "fmla z27.h, z11.h, z2.h[4]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z16.h, z12.h, z0.h[5]\n"
- "fmla z20.h, z12.h, z1.h[5]\n"
- "fmla z24.h, z12.h, z2.h[5]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "fmla z17.h, z13.h, z0.h[5]\n"
- "fmla z21.h, z13.h, z1.h[5]\n"
- "fmla z25.h, z13.h, z2.h[5]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "fmla z18.h, z14.h, z0.h[5]\n"
- "fmla z22.h, z14.h, z1.h[5]\n"
- "fmla z26.h, z14.h, z2.h[5]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z19.h, z15.h, z0.h[5]\n"
- "fmla z23.h, z15.h, z1.h[5]\n"
- "fmla z27.h, z15.h, z2.h[5]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z16.h, z8.h, z0.h[6]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "fmla z20.h, z8.h, z1.h[6]\n"
- "fmla z24.h, z8.h, z2.h[6]\n"
- "fmla z17.h, z9.h, z0.h[6]\n"
- "fmla z21.h, z9.h, z1.h[6]\n"
- "fmla z25.h, z9.h, z2.h[6]\n"
- "fmla z18.h, z10.h, z0.h[6]\n"
- "fmla z22.h, z10.h, z1.h[6]\n"
- "fmla z26.h, z10.h, z2.h[6]\n"
- "fmla z19.h, z11.h, z0.h[6]\n"
- "fmla z23.h, z11.h, z1.h[6]\n"
- "fmla z27.h, z11.h, z2.h[6]\n"
- "fmla z16.h, z12.h, z0.h[7]\n"
- "fmla z20.h, z12.h, z1.h[7]\n"
- "fmla z24.h, z12.h, z2.h[7]\n"
- "fmla z17.h, z13.h, z0.h[7]\n"
- "fmla z21.h, z13.h, z1.h[7]\n"
- "fmla z25.h, z13.h, z2.h[7]\n"
- "fmla z18.h, z14.h, z0.h[7]\n"
- "fmla z22.h, z14.h, z1.h[7]\n"
- "fmla z26.h, z14.h, z2.h[7]\n"
- "fmla z19.h, z15.h, z0.h[7]\n"
- "fmla z23.h, z15.h, z1.h[7]\n"
- "fmla z27.h, z15.h, z2.h[7]\n"
- "cbz %[blocks], 5f\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z16.h, z8.h, z4.h[0]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z20.h, z8.h, z5.h[0]\n"
- "fmla z24.h, z8.h, z6.h[0]\n"
- "fmla z17.h, z9.h, z4.h[0]\n"
- "fmla z21.h, z9.h, z5.h[0]\n"
- "fmla z25.h, z9.h, z6.h[0]\n"
- "fmla z18.h, z10.h, z4.h[0]\n"
- "fmla z22.h, z10.h, z5.h[0]\n"
- "fmla z26.h, z10.h, z6.h[0]\n"
- "fmla z19.h, z11.h, z4.h[0]\n"
- "fmla z23.h, z11.h, z5.h[0]\n"
- "fmla z27.h, z11.h, z6.h[0]\n"
- "b.eq 5f\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z16.h, z12.h, z4.h[1]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z20.h, z12.h, z5.h[1]\n"
- "fmla z24.h, z12.h, z6.h[1]\n"
- "fmla z17.h, z13.h, z4.h[1]\n"
- "fmla z21.h, z13.h, z5.h[1]\n"
- "fmla z25.h, z13.h, z6.h[1]\n"
- "fmla z18.h, z14.h, z4.h[1]\n"
- "fmla z22.h, z14.h, z5.h[1]\n"
- "fmla z26.h, z14.h, z6.h[1]\n"
- "fmla z19.h, z15.h, z4.h[1]\n"
- "fmla z23.h, z15.h, z5.h[1]\n"
- "fmla z27.h, z15.h, z6.h[1]\n"
- "b.eq 5f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "fmla z16.h, z8.h, z4.h[2]\n"
- "fmla z20.h, z8.h, z5.h[2]\n"
- "fmla z24.h, z8.h, z6.h[2]\n"
- "fmla z17.h, z9.h, z4.h[2]\n"
- "fmla z21.h, z9.h, z5.h[2]\n"
- "fmla z25.h, z9.h, z6.h[2]\n"
- "fmla z18.h, z10.h, z4.h[2]\n"
- "fmla z22.h, z10.h, z5.h[2]\n"
- "fmla z26.h, z10.h, z6.h[2]\n"
- "fmla z19.h, z11.h, z4.h[2]\n"
- "fmla z23.h, z11.h, z5.h[2]\n"
- "fmla z27.h, z11.h, z6.h[2]\n"
- "b.eq 5f\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "fmla z16.h, z12.h, z4.h[3]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "fmla z20.h, z12.h, z5.h[3]\n"
- "fmla z24.h, z12.h, z6.h[3]\n"
- "fmla z17.h, z13.h, z4.h[3]\n"
- "fmla z21.h, z13.h, z5.h[3]\n"
- "fmla z25.h, z13.h, z6.h[3]\n"
- "fmla z18.h, z14.h, z4.h[3]\n"
- "fmla z22.h, z14.h, z5.h[3]\n"
- "fmla z26.h, z14.h, z6.h[3]\n"
- "fmla z19.h, z15.h, z4.h[3]\n"
- "fmla z23.h, z15.h, z5.h[3]\n"
- "fmla z27.h, z15.h, z6.h[3]\n"
- "b.eq 5f\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z16.h, z8.h, z4.h[4]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z20.h, z8.h, z5.h[4]\n"
- "fmla z24.h, z8.h, z6.h[4]\n"
- "fmla z17.h, z9.h, z4.h[4]\n"
- "fmla z21.h, z9.h, z5.h[4]\n"
- "fmla z25.h, z9.h, z6.h[4]\n"
- "fmla z18.h, z10.h, z4.h[4]\n"
- "fmla z22.h, z10.h, z5.h[4]\n"
- "fmla z26.h, z10.h, z6.h[4]\n"
- "fmla z19.h, z11.h, z4.h[4]\n"
- "fmla z23.h, z11.h, z5.h[4]\n"
- "fmla z27.h, z11.h, z6.h[4]\n"
- "b.eq 5f\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z16.h, z12.h, z4.h[5]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z20.h, z12.h, z5.h[5]\n"
- "fmla z24.h, z12.h, z6.h[5]\n"
- "fmla z17.h, z13.h, z4.h[5]\n"
- "fmla z21.h, z13.h, z5.h[5]\n"
- "fmla z25.h, z13.h, z6.h[5]\n"
- "fmla z18.h, z14.h, z4.h[5]\n"
- "fmla z22.h, z14.h, z5.h[5]\n"
- "fmla z26.h, z14.h, z6.h[5]\n"
- "fmla z19.h, z15.h, z4.h[5]\n"
- "fmla z23.h, z15.h, z5.h[5]\n"
- "fmla z27.h, z15.h, z6.h[5]\n"
- "b.eq 5f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "fmla z16.h, z8.h, z4.h[6]\n"
- "fmla z20.h, z8.h, z5.h[6]\n"
- "fmla z24.h, z8.h, z6.h[6]\n"
- "fmla z17.h, z9.h, z4.h[6]\n"
- "fmla z21.h, z9.h, z5.h[6]\n"
- "fmla z25.h, z9.h, z6.h[6]\n"
- "fmla z18.h, z10.h, z4.h[6]\n"
- "fmla z22.h, z10.h, z5.h[6]\n"
- "fmla z26.h, z10.h, z6.h[6]\n"
- "fmla z19.h, z11.h, z4.h[6]\n"
- "fmla z23.h, z11.h, z5.h[6]\n"
- "fmla z27.h, z11.h, z6.h[6]\n"
- "5:\n"
- "ld1rh z14.h, p7/z, [%[minptr]]\n"
- "ld1rh z15.h, p7/z, [%[maxptr]]\n"
- "fmax z16.h, p7/m, z16.h, z14.h\n"
- "fmax z17.h, p7/m, z17.h, z14.h\n"
- "fmax z18.h, p7/m, z18.h, z14.h\n"
- "fmax z19.h, p7/m, z19.h, z14.h\n"
- "fmin z16.h, p7/m, z16.h, z15.h\n"
- "fmin z17.h, p7/m, z17.h, z15.h\n"
- "fmin z18.h, p7/m, z18.h, z15.h\n"
- "fmin z19.h, p7/m, z19.h, z15.h\n"
- "st1h z16.h, p0, [%[c_ptr0]]\n"
- "fmax z20.h, p7/m, z20.h, z14.h\n"
- "fmax z21.h, p7/m, z21.h, z14.h\n"
- "fmax z22.h, p7/m, z22.h, z14.h\n"
- "st1h z17.h, p1, [%[c_ptr0], #1, MUL VL]\n"
- "fmax z23.h, p7/m, z23.h, z14.h\n"
- "fmin z20.h, p7/m, z20.h, z15.h\n"
- "fmin z21.h, p7/m, z21.h, z15.h\n"
- "st1h z18.h, p2, [%[c_ptr0], #2, MUL VL]\n"
- "fmin z22.h, p7/m, z22.h, z15.h\n"
- "fmin z23.h, p7/m, z23.h, z15.h\n"
- "fmax z24.h, p7/m, z24.h, z14.h\n"
- "st1h z19.h, p3, [%[c_ptr0], #3, MUL VL]\n"
- "fmax z25.h, p7/m, z25.h, z14.h\n"
- "addvl %[c_ptr0], %[c_ptr0], #4\n"
- "fmax z26.h, p7/m, z26.h, z14.h\n"
- "st1h z20.h, p0, [c_ptr1]\n"
- "fmin z24.h, p7/m, z24.h, z15.h\n"
- "fmin z25.h, p7/m, z25.h, z15.h\n"
- "fmax z27.h, p7/m, z27.h, z14.h\n"
- "st1h z21.h, p1, [c_ptr1, #1, MUL VL]\n"
- "fmin z26.h, p7/m, z26.h, z15.h\n"
- "fmin z27.h, p7/m, z27.h, z15.h\n"
- "st1h z22.h, p2, [c_ptr1, #2, MUL VL]\n"
- "st1h z23.h, p3, [c_ptr1, #3, MUL VL]\n"
- "st1h z24.h, p0, [c_ptr2]\n"
- "st1h z25.h, p1, [c_ptr2, #1, MUL VL]\n"
- "st1h z26.h, p2, [c_ptr2, #2, MUL VL]\n"
- "st1h z27.h, p3, [c_ptr2, #3, MUL VL]\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks)
- : [width] "r" (width), [accumulate] "r" (static_cast<uint64_t>(accumulate)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers)
- : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "cc", "memory"
- );
- break;
- default:
- case 4:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "a_ptr3 .req X2\n"
- "c_ptr1 .req X3\n"
- "c_ptr2 .req X4\n"
- "c_ptr3 .req X5\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "add a_ptr3, a_ptr2, %[lda]\n"
- "add c_ptr3, c_ptr2, %[ldc]\n"
- "whilelt p6.h, %[temp], %[leftovers]\n"
- "whilelt p0.h, %[temp], %[width]\n"
- "inch %[temp], all, mul #1\n"
- "ptrue p7.h\n"
- "whilelt p1.h, %[temp], %[width]\n"
- "inch %[temp], all, mul #1\n"
- "whilelt p2.h, %[temp], %[width]\n"
- "inch %[temp], all, mul #1\n"
- "whilelt p3.h, %[temp], %[width]\n"
- "cbnz %[accumulate], 1f\n"
- "ld1h z16.h, p0/z, [%[biasptr]]\n"
- "ld1h z17.h, p1/z, [%[biasptr], #1, MUL VL]\n"
- "ld1h z18.h, p2/z, [%[biasptr], #2, MUL VL]\n"
- "ld1h z19.h, p3/z, [%[biasptr], #3, MUL VL]\n"
- "mov z20.d, z16.d\n"
- "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n"
- "mov z21.d, z17.d\n"
- "ld1rqh z1.h, p7/z, [a_ptr1]\n"
- "mov z22.d, z18.d\n"
- "ld1rqh z2.h, p7/z, [a_ptr2]\n"
- "mov z23.d, z19.d\n"
- "ld1rqh z3.h, p7/z, [a_ptr3]\n"
- "mov z24.d, z16.d\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "mov z25.d, z17.d\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "mov z26.d, z18.d\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "mov z27.d, z19.d\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "mov z28.d, z16.d\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "mov z29.d, z17.d\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "mov z30.d, z18.d\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "mov z31.d, z19.d\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "add a_ptr3, a_ptr3, #0x10\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "ld1h z16.h, p0/z, [%[c_ptr0]]\n"
- "ld1h z17.h, p1/z, [%[c_ptr0], #1, MUL VL]\n"
- "ld1h z18.h, p2/z, [%[c_ptr0], #2, MUL VL]\n"
- "ld1h z19.h, p3/z, [%[c_ptr0], #3, MUL VL]\n"
- "ld1h z20.h, p0/z, [c_ptr1]\n"
- "ld1h z21.h, p1/z, [c_ptr1, #1, MUL VL]\n"
- "ld1h z22.h, p2/z, [c_ptr1, #2, MUL VL]\n"
- "ld1h z23.h, p3/z, [c_ptr1, #3, MUL VL]\n"
- "ld1h z24.h, p0/z, [c_ptr2]\n"
- "ld1h z25.h, p1/z, [c_ptr2, #1, MUL VL]\n"
- "ld1h z26.h, p2/z, [c_ptr2, #2, MUL VL]\n"
- "ld1h z27.h, p3/z, [c_ptr2, #3, MUL VL]\n"
- "ld1h z28.h, p0/z, [c_ptr3]\n"
- "ld1h z29.h, p1/z, [c_ptr3, #1, MUL VL]\n"
- "ld1h z30.h, p2/z, [c_ptr3, #2, MUL VL]\n"
- "ld1h z31.h, p3/z, [c_ptr3, #3, MUL VL]\n"
- "ld1rqh z0.h, p7/z, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ld1rqh z1.h, p7/z, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "ld1rqh z2.h, p7/z, [a_ptr2]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "ld1rqh z3.h, p7/z, [a_ptr3]\n"
- "add a_ptr3, a_ptr3, #0x10\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "cbz %[loops], 2f\n"
- "3:\n"
- "fmla z16.h, z8.h, z0.h[0]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "fmla z20.h, z8.h, z1.h[0]\n"
- "ld1rqh z4.h, p7/z, [%[a_ptr0]]\n"
- "fmla z24.h, z8.h, z2.h[0]\n"
- "ld1rqh z5.h, p7/z, [a_ptr1]\n"
- "fmla z28.h, z8.h, z3.h[0]\n"
- "ld1rqh z6.h, p7/z, [a_ptr2]\n"
- "fmla z17.h, z9.h, z0.h[0]\n"
- "ld1rqh z7.h, p7/z, [a_ptr3]\n"
- "fmla z21.h, z9.h, z1.h[0]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "fmla z25.h, z9.h, z2.h[0]\n"
- "subs %[loops], %[loops], #0x1\n"
- "fmla z29.h, z9.h, z3.h[0]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z18.h, z10.h, z0.h[0]\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
- "fmla z22.h, z10.h, z1.h[0]\n"
- "add a_ptr1, a_ptr1, #0x20\n"
- "fmla z26.h, z10.h, z2.h[0]\n"
- "add a_ptr2, a_ptr2, #0x20\n"
- "fmla z30.h, z10.h, z3.h[0]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z19.h, z11.h, z0.h[0]\n"
- "add a_ptr3, a_ptr3, #0x20\n"
- "fmla z23.h, z11.h, z1.h[0]\n"
- "fmla z27.h, z11.h, z2.h[0]\n"
- "fmla z31.h, z11.h, z3.h[0]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z16.h, z12.h, z0.h[1]\n"
- "fmla z20.h, z12.h, z1.h[1]\n"
- "fmla z24.h, z12.h, z2.h[1]\n"
- "fmla z28.h, z12.h, z3.h[1]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "fmla z17.h, z13.h, z0.h[1]\n"
- "fmla z21.h, z13.h, z1.h[1]\n"
- "fmla z25.h, z13.h, z2.h[1]\n"
- "fmla z29.h, z13.h, z3.h[1]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "fmla z18.h, z14.h, z0.h[1]\n"
- "fmla z22.h, z14.h, z1.h[1]\n"
- "fmla z26.h, z14.h, z2.h[1]\n"
- "fmla z30.h, z14.h, z3.h[1]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z19.h, z15.h, z0.h[1]\n"
- "fmla z23.h, z15.h, z1.h[1]\n"
- "fmla z27.h, z15.h, z2.h[1]\n"
- "fmla z31.h, z15.h, z3.h[1]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z16.h, z8.h, z0.h[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "fmla z20.h, z8.h, z1.h[2]\n"
- "fmla z24.h, z8.h, z2.h[2]\n"
- "fmla z28.h, z8.h, z3.h[2]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "fmla z17.h, z9.h, z0.h[2]\n"
- "fmla z21.h, z9.h, z1.h[2]\n"
- "fmla z25.h, z9.h, z2.h[2]\n"
- "fmla z29.h, z9.h, z3.h[2]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "fmla z18.h, z10.h, z0.h[2]\n"
- "fmla z22.h, z10.h, z1.h[2]\n"
- "fmla z26.h, z10.h, z2.h[2]\n"
- "fmla z30.h, z10.h, z3.h[2]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "fmla z19.h, z11.h, z0.h[2]\n"
- "fmla z23.h, z11.h, z1.h[2]\n"
- "fmla z27.h, z11.h, z2.h[2]\n"
- "fmla z31.h, z11.h, z3.h[2]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "fmla z16.h, z12.h, z0.h[3]\n"
- "fmla z20.h, z12.h, z1.h[3]\n"
- "fmla z24.h, z12.h, z2.h[3]\n"
- "fmla z28.h, z12.h, z3.h[3]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "fmla z17.h, z13.h, z0.h[3]\n"
- "fmla z21.h, z13.h, z1.h[3]\n"
- "fmla z25.h, z13.h, z2.h[3]\n"
- "fmla z29.h, z13.h, z3.h[3]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "fmla z18.h, z14.h, z0.h[3]\n"
- "fmla z22.h, z14.h, z1.h[3]\n"
- "fmla z26.h, z14.h, z2.h[3]\n"
- "fmla z30.h, z14.h, z3.h[3]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "fmla z19.h, z15.h, z0.h[3]\n"
- "fmla z23.h, z15.h, z1.h[3]\n"
- "fmla z27.h, z15.h, z2.h[3]\n"
- "fmla z31.h, z15.h, z3.h[3]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "fmla z16.h, z8.h, z0.h[4]\n"
- "fmla z20.h, z8.h, z1.h[4]\n"
- "fmla z24.h, z8.h, z2.h[4]\n"
- "fmla z28.h, z8.h, z3.h[4]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "fmla z17.h, z9.h, z0.h[4]\n"
- "fmla z21.h, z9.h, z1.h[4]\n"
- "fmla z25.h, z9.h, z2.h[4]\n"
- "fmla z29.h, z9.h, z3.h[4]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z18.h, z10.h, z0.h[4]\n"
- "fmla z22.h, z10.h, z1.h[4]\n"
- "fmla z26.h, z10.h, z2.h[4]\n"
- "fmla z30.h, z10.h, z3.h[4]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z19.h, z11.h, z0.h[4]\n"
- "fmla z23.h, z11.h, z1.h[4]\n"
- "fmla z27.h, z11.h, z2.h[4]\n"
- "fmla z31.h, z11.h, z3.h[4]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z16.h, z12.h, z0.h[5]\n"
- "fmla z20.h, z12.h, z1.h[5]\n"
- "fmla z24.h, z12.h, z2.h[5]\n"
- "fmla z28.h, z12.h, z3.h[5]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "fmla z17.h, z13.h, z0.h[5]\n"
- "fmla z21.h, z13.h, z1.h[5]\n"
- "fmla z25.h, z13.h, z2.h[5]\n"
- "fmla z29.h, z13.h, z3.h[5]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "fmla z18.h, z14.h, z0.h[5]\n"
- "fmla z22.h, z14.h, z1.h[5]\n"
- "fmla z26.h, z14.h, z2.h[5]\n"
- "fmla z30.h, z14.h, z3.h[5]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z19.h, z15.h, z0.h[5]\n"
- "fmla z23.h, z15.h, z1.h[5]\n"
- "fmla z27.h, z15.h, z2.h[5]\n"
- "fmla z31.h, z15.h, z3.h[5]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z16.h, z8.h, z0.h[6]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "fmla z20.h, z8.h, z1.h[6]\n"
- "fmla z24.h, z8.h, z2.h[6]\n"
- "fmla z28.h, z8.h, z3.h[6]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "fmla z17.h, z9.h, z0.h[6]\n"
- "fmla z21.h, z9.h, z1.h[6]\n"
- "fmla z25.h, z9.h, z2.h[6]\n"
- "fmla z29.h, z9.h, z3.h[6]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "fmla z18.h, z10.h, z0.h[6]\n"
- "fmla z22.h, z10.h, z1.h[6]\n"
- "fmla z26.h, z10.h, z2.h[6]\n"
- "fmla z30.h, z10.h, z3.h[6]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "fmla z19.h, z11.h, z0.h[6]\n"
- "fmla z23.h, z11.h, z1.h[6]\n"
- "fmla z27.h, z11.h, z2.h[6]\n"
- "fmla z31.h, z11.h, z3.h[6]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "fmla z16.h, z12.h, z0.h[7]\n"
- "fmla z20.h, z12.h, z1.h[7]\n"
- "fmla z24.h, z12.h, z2.h[7]\n"
- "fmla z28.h, z12.h, z3.h[7]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "fmla z17.h, z13.h, z0.h[7]\n"
- "fmla z21.h, z13.h, z1.h[7]\n"
- "fmla z25.h, z13.h, z2.h[7]\n"
- "fmla z29.h, z13.h, z3.h[7]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "fmla z18.h, z14.h, z0.h[7]\n"
- "fmla z22.h, z14.h, z1.h[7]\n"
- "fmla z26.h, z14.h, z2.h[7]\n"
- "fmla z30.h, z14.h, z3.h[7]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "fmla z19.h, z15.h, z0.h[7]\n"
- "ld1rqh z0.h, p7/z, [%[a_ptr0], #-0x10]\n"
- "fmla z23.h, z15.h, z1.h[7]\n"
- "ld1rqh z1.h, p7/z, [a_ptr1, #-0x10]\n"
- "fmla z27.h, z15.h, z2.h[7]\n"
- "ld1rqh z2.h, p7/z, [a_ptr2, #-0x10]\n"
- "fmla z31.h, z15.h, z3.h[7]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "fmla z16.h, z8.h, z4.h[0]\n"
- "ld1rqh z3.h, p7/z, [a_ptr3, #-0x10]\n"
- "fmla z20.h, z8.h, z5.h[0]\n"
- "fmla z24.h, z8.h, z6.h[0]\n"
- "fmla z28.h, z8.h, z7.h[0]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "fmla z17.h, z9.h, z4.h[0]\n"
- "fmla z21.h, z9.h, z5.h[0]\n"
- "fmla z25.h, z9.h, z6.h[0]\n"
- "fmla z29.h, z9.h, z7.h[0]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z18.h, z10.h, z4.h[0]\n"
- "fmla z22.h, z10.h, z5.h[0]\n"
- "fmla z26.h, z10.h, z6.h[0]\n"
- "fmla z30.h, z10.h, z7.h[0]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z19.h, z11.h, z4.h[0]\n"
- "fmla z23.h, z11.h, z5.h[0]\n"
- "fmla z27.h, z11.h, z6.h[0]\n"
- "fmla z31.h, z11.h, z7.h[0]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z16.h, z12.h, z4.h[1]\n"
- "fmla z20.h, z12.h, z5.h[1]\n"
- "fmla z24.h, z12.h, z6.h[1]\n"
- "fmla z28.h, z12.h, z7.h[1]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "fmla z17.h, z13.h, z4.h[1]\n"
- "fmla z21.h, z13.h, z5.h[1]\n"
- "fmla z25.h, z13.h, z6.h[1]\n"
- "fmla z29.h, z13.h, z7.h[1]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "fmla z18.h, z14.h, z4.h[1]\n"
- "fmla z22.h, z14.h, z5.h[1]\n"
- "fmla z26.h, z14.h, z6.h[1]\n"
- "fmla z30.h, z14.h, z7.h[1]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z19.h, z15.h, z4.h[1]\n"
- "fmla z23.h, z15.h, z5.h[1]\n"
- "fmla z27.h, z15.h, z6.h[1]\n"
- "fmla z31.h, z15.h, z7.h[1]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z16.h, z8.h, z4.h[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "fmla z20.h, z8.h, z5.h[2]\n"
- "fmla z24.h, z8.h, z6.h[2]\n"
- "fmla z28.h, z8.h, z7.h[2]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "fmla z17.h, z9.h, z4.h[2]\n"
- "fmla z21.h, z9.h, z5.h[2]\n"
- "fmla z25.h, z9.h, z6.h[2]\n"
- "fmla z29.h, z9.h, z7.h[2]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "fmla z18.h, z10.h, z4.h[2]\n"
- "fmla z22.h, z10.h, z5.h[2]\n"
- "fmla z26.h, z10.h, z6.h[2]\n"
- "fmla z30.h, z10.h, z7.h[2]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "fmla z19.h, z11.h, z4.h[2]\n"
- "fmla z23.h, z11.h, z5.h[2]\n"
- "fmla z27.h, z11.h, z6.h[2]\n"
- "fmla z31.h, z11.h, z7.h[2]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "fmla z16.h, z12.h, z4.h[3]\n"
- "fmla z20.h, z12.h, z5.h[3]\n"
- "fmla z24.h, z12.h, z6.h[3]\n"
- "fmla z28.h, z12.h, z7.h[3]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "fmla z17.h, z13.h, z4.h[3]\n"
- "fmla z21.h, z13.h, z5.h[3]\n"
- "fmla z25.h, z13.h, z6.h[3]\n"
- "fmla z29.h, z13.h, z7.h[3]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "fmla z18.h, z14.h, z4.h[3]\n"
- "fmla z22.h, z14.h, z5.h[3]\n"
- "fmla z26.h, z14.h, z6.h[3]\n"
- "fmla z30.h, z14.h, z7.h[3]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "fmla z19.h, z15.h, z4.h[3]\n"
- "fmla z23.h, z15.h, z5.h[3]\n"
- "fmla z27.h, z15.h, z6.h[3]\n"
- "fmla z31.h, z15.h, z7.h[3]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "fmla z16.h, z8.h, z4.h[4]\n"
- "fmla z20.h, z8.h, z5.h[4]\n"
- "fmla z24.h, z8.h, z6.h[4]\n"
- "fmla z28.h, z8.h, z7.h[4]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "fmla z17.h, z9.h, z4.h[4]\n"
- "fmla z21.h, z9.h, z5.h[4]\n"
- "fmla z25.h, z9.h, z6.h[4]\n"
- "fmla z29.h, z9.h, z7.h[4]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z18.h, z10.h, z4.h[4]\n"
- "fmla z22.h, z10.h, z5.h[4]\n"
- "fmla z26.h, z10.h, z6.h[4]\n"
- "fmla z30.h, z10.h, z7.h[4]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z19.h, z11.h, z4.h[4]\n"
- "fmla z23.h, z11.h, z5.h[4]\n"
- "fmla z27.h, z11.h, z6.h[4]\n"
- "fmla z31.h, z11.h, z7.h[4]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z16.h, z12.h, z4.h[5]\n"
- "fmla z20.h, z12.h, z5.h[5]\n"
- "fmla z24.h, z12.h, z6.h[5]\n"
- "fmla z28.h, z12.h, z7.h[5]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "fmla z17.h, z13.h, z4.h[5]\n"
- "fmla z21.h, z13.h, z5.h[5]\n"
- "fmla z25.h, z13.h, z6.h[5]\n"
- "fmla z29.h, z13.h, z7.h[5]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "fmla z18.h, z14.h, z4.h[5]\n"
- "fmla z22.h, z14.h, z5.h[5]\n"
- "fmla z26.h, z14.h, z6.h[5]\n"
- "fmla z30.h, z14.h, z7.h[5]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z19.h, z15.h, z4.h[5]\n"
- "fmla z23.h, z15.h, z5.h[5]\n"
- "fmla z27.h, z15.h, z6.h[5]\n"
- "fmla z31.h, z15.h, z7.h[5]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z16.h, z8.h, z4.h[6]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "fmla z20.h, z8.h, z5.h[6]\n"
- "fmla z24.h, z8.h, z6.h[6]\n"
- "fmla z28.h, z8.h, z7.h[6]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "fmla z17.h, z9.h, z4.h[6]\n"
- "fmla z21.h, z9.h, z5.h[6]\n"
- "fmla z25.h, z9.h, z6.h[6]\n"
- "fmla z29.h, z9.h, z7.h[6]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "fmla z18.h, z10.h, z4.h[6]\n"
- "fmla z22.h, z10.h, z5.h[6]\n"
- "fmla z26.h, z10.h, z6.h[6]\n"
- "fmla z30.h, z10.h, z7.h[6]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "fmla z19.h, z11.h, z4.h[6]\n"
- "fmla z23.h, z11.h, z5.h[6]\n"
- "fmla z27.h, z11.h, z6.h[6]\n"
- "fmla z31.h, z11.h, z7.h[6]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "fmla z16.h, z12.h, z4.h[7]\n"
- "fmla z20.h, z12.h, z5.h[7]\n"
- "fmla z24.h, z12.h, z6.h[7]\n"
- "fmla z28.h, z12.h, z7.h[7]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "fmla z17.h, z13.h, z4.h[7]\n"
- "fmla z21.h, z13.h, z5.h[7]\n"
- "fmla z25.h, z13.h, z6.h[7]\n"
- "fmla z29.h, z13.h, z7.h[7]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "fmla z18.h, z14.h, z4.h[7]\n"
- "fmla z22.h, z14.h, z5.h[7]\n"
- "fmla z26.h, z14.h, z6.h[7]\n"
- "fmla z30.h, z14.h, z7.h[7]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "fmla z19.h, z15.h, z4.h[7]\n"
- "fmla z23.h, z15.h, z5.h[7]\n"
- "fmla z27.h, z15.h, z6.h[7]\n"
- "fmla z31.h, z15.h, z7.h[7]\n"
- "b.ne 3b\n"
- "2:\n"
- "cbz %[regs], 4f\n"
- "fmla z16.h, z8.h, z0.h[0]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "fmla z20.h, z8.h, z1.h[0]\n"
- "ld1rqh z4.h, p7/z, [%[a_ptr0]]\n"
- "fmla z24.h, z8.h, z2.h[0]\n"
- "ld1rqh z5.h, p7/z, [a_ptr1]\n"
- "fmla z28.h, z8.h, z3.h[0]\n"
- "ld1rqh z6.h, p7/z, [a_ptr2]\n"
- "fmla z17.h, z9.h, z0.h[0]\n"
- "ld1rqh z7.h, p7/z, [a_ptr3]\n"
- "fmla z21.h, z9.h, z1.h[0]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "fmla z25.h, z9.h, z2.h[0]\n"
- "fmla z29.h, z9.h, z3.h[0]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z18.h, z10.h, z0.h[0]\n"
- "fmla z22.h, z10.h, z1.h[0]\n"
- "fmla z26.h, z10.h, z2.h[0]\n"
- "fmla z30.h, z10.h, z3.h[0]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z19.h, z11.h, z0.h[0]\n"
- "fmla z23.h, z11.h, z1.h[0]\n"
- "fmla z27.h, z11.h, z2.h[0]\n"
- "fmla z31.h, z11.h, z3.h[0]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z16.h, z12.h, z0.h[1]\n"
- "fmla z20.h, z12.h, z1.h[1]\n"
- "fmla z24.h, z12.h, z2.h[1]\n"
- "fmla z28.h, z12.h, z3.h[1]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "fmla z17.h, z13.h, z0.h[1]\n"
- "fmla z21.h, z13.h, z1.h[1]\n"
- "fmla z25.h, z13.h, z2.h[1]\n"
- "fmla z29.h, z13.h, z3.h[1]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "fmla z18.h, z14.h, z0.h[1]\n"
- "fmla z22.h, z14.h, z1.h[1]\n"
- "fmla z26.h, z14.h, z2.h[1]\n"
- "fmla z30.h, z14.h, z3.h[1]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z19.h, z15.h, z0.h[1]\n"
- "fmla z23.h, z15.h, z1.h[1]\n"
- "fmla z27.h, z15.h, z2.h[1]\n"
- "fmla z31.h, z15.h, z3.h[1]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z16.h, z8.h, z0.h[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "fmla z20.h, z8.h, z1.h[2]\n"
- "fmla z24.h, z8.h, z2.h[2]\n"
- "fmla z28.h, z8.h, z3.h[2]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "fmla z17.h, z9.h, z0.h[2]\n"
- "fmla z21.h, z9.h, z1.h[2]\n"
- "fmla z25.h, z9.h, z2.h[2]\n"
- "fmla z29.h, z9.h, z3.h[2]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "fmla z18.h, z10.h, z0.h[2]\n"
- "fmla z22.h, z10.h, z1.h[2]\n"
- "fmla z26.h, z10.h, z2.h[2]\n"
- "fmla z30.h, z10.h, z3.h[2]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "fmla z19.h, z11.h, z0.h[2]\n"
- "fmla z23.h, z11.h, z1.h[2]\n"
- "fmla z27.h, z11.h, z2.h[2]\n"
- "fmla z31.h, z11.h, z3.h[2]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "fmla z16.h, z12.h, z0.h[3]\n"
- "fmla z20.h, z12.h, z1.h[3]\n"
- "fmla z24.h, z12.h, z2.h[3]\n"
- "fmla z28.h, z12.h, z3.h[3]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "fmla z17.h, z13.h, z0.h[3]\n"
- "fmla z21.h, z13.h, z1.h[3]\n"
- "fmla z25.h, z13.h, z2.h[3]\n"
- "fmla z29.h, z13.h, z3.h[3]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "fmla z18.h, z14.h, z0.h[3]\n"
- "fmla z22.h, z14.h, z1.h[3]\n"
- "fmla z26.h, z14.h, z2.h[3]\n"
- "fmla z30.h, z14.h, z3.h[3]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "fmla z19.h, z15.h, z0.h[3]\n"
- "fmla z23.h, z15.h, z1.h[3]\n"
- "fmla z27.h, z15.h, z2.h[3]\n"
- "fmla z31.h, z15.h, z3.h[3]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "fmla z16.h, z8.h, z0.h[4]\n"
- "fmla z20.h, z8.h, z1.h[4]\n"
- "fmla z24.h, z8.h, z2.h[4]\n"
- "fmla z28.h, z8.h, z3.h[4]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "fmla z17.h, z9.h, z0.h[4]\n"
- "fmla z21.h, z9.h, z1.h[4]\n"
- "fmla z25.h, z9.h, z2.h[4]\n"
- "fmla z29.h, z9.h, z3.h[4]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z18.h, z10.h, z0.h[4]\n"
- "fmla z22.h, z10.h, z1.h[4]\n"
- "fmla z26.h, z10.h, z2.h[4]\n"
- "fmla z30.h, z10.h, z3.h[4]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z19.h, z11.h, z0.h[4]\n"
- "fmla z23.h, z11.h, z1.h[4]\n"
- "fmla z27.h, z11.h, z2.h[4]\n"
- "fmla z31.h, z11.h, z3.h[4]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z16.h, z12.h, z0.h[5]\n"
- "fmla z20.h, z12.h, z1.h[5]\n"
- "fmla z24.h, z12.h, z2.h[5]\n"
- "fmla z28.h, z12.h, z3.h[5]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "fmla z17.h, z13.h, z0.h[5]\n"
- "fmla z21.h, z13.h, z1.h[5]\n"
- "fmla z25.h, z13.h, z2.h[5]\n"
- "fmla z29.h, z13.h, z3.h[5]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "fmla z18.h, z14.h, z0.h[5]\n"
- "fmla z22.h, z14.h, z1.h[5]\n"
- "fmla z26.h, z14.h, z2.h[5]\n"
- "fmla z30.h, z14.h, z3.h[5]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z19.h, z15.h, z0.h[5]\n"
- "fmla z23.h, z15.h, z1.h[5]\n"
- "fmla z27.h, z15.h, z2.h[5]\n"
- "fmla z31.h, z15.h, z3.h[5]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z16.h, z8.h, z0.h[6]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "fmla z20.h, z8.h, z1.h[6]\n"
- "fmla z24.h, z8.h, z2.h[6]\n"
- "fmla z28.h, z8.h, z3.h[6]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "fmla z17.h, z9.h, z0.h[6]\n"
- "fmla z21.h, z9.h, z1.h[6]\n"
- "fmla z25.h, z9.h, z2.h[6]\n"
- "fmla z29.h, z9.h, z3.h[6]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "fmla z18.h, z10.h, z0.h[6]\n"
- "fmla z22.h, z10.h, z1.h[6]\n"
- "fmla z26.h, z10.h, z2.h[6]\n"
- "fmla z30.h, z10.h, z3.h[6]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "fmla z19.h, z11.h, z0.h[6]\n"
- "fmla z23.h, z11.h, z1.h[6]\n"
- "fmla z27.h, z11.h, z2.h[6]\n"
- "fmla z31.h, z11.h, z3.h[6]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "fmla z16.h, z12.h, z0.h[7]\n"
- "fmla z20.h, z12.h, z1.h[7]\n"
- "fmla z24.h, z12.h, z2.h[7]\n"
- "fmla z28.h, z12.h, z3.h[7]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "fmla z17.h, z13.h, z0.h[7]\n"
- "fmla z21.h, z13.h, z1.h[7]\n"
- "fmla z25.h, z13.h, z2.h[7]\n"
- "fmla z29.h, z13.h, z3.h[7]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "fmla z18.h, z14.h, z0.h[7]\n"
- "fmla z22.h, z14.h, z1.h[7]\n"
- "fmla z26.h, z14.h, z2.h[7]\n"
- "fmla z30.h, z14.h, z3.h[7]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "fmla z19.h, z15.h, z0.h[7]\n"
- "ld1rqh z0.h, p6/z, [%[a_ptr0], #0x10]\n"
- "fmla z23.h, z15.h, z1.h[7]\n"
- "ld1rqh z1.h, p6/z, [a_ptr1, #0x10]\n"
- "fmla z27.h, z15.h, z2.h[7]\n"
- "ld1rqh z2.h, p6/z, [a_ptr2, #0x10]\n"
- "fmla z31.h, z15.h, z3.h[7]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "fmla z16.h, z8.h, z4.h[0]\n"
- "ld1rqh z3.h, p6/z, [a_ptr3, #0x10]\n"
- "fmla z20.h, z8.h, z5.h[0]\n"
- "addvl %[a_ptr0], %[a_ptr0], #2\n"
- "fmla z24.h, z8.h, z6.h[0]\n"
- "addvl a_ptr1, a_ptr1, #2\n"
- "fmla z28.h, z8.h, z7.h[0]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "fmla z17.h, z9.h, z4.h[0]\n"
- "addvl a_ptr2, a_ptr2, #2\n"
- "fmla z21.h, z9.h, z5.h[0]\n"
- "addvl a_ptr3, a_ptr3, #2\n"
- "fmla z25.h, z9.h, z6.h[0]\n"
- "fmla z29.h, z9.h, z7.h[0]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z18.h, z10.h, z4.h[0]\n"
- "fmla z22.h, z10.h, z5.h[0]\n"
- "fmla z26.h, z10.h, z6.h[0]\n"
- "fmla z30.h, z10.h, z7.h[0]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z19.h, z11.h, z4.h[0]\n"
- "fmla z23.h, z11.h, z5.h[0]\n"
- "fmla z27.h, z11.h, z6.h[0]\n"
- "fmla z31.h, z11.h, z7.h[0]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z16.h, z12.h, z4.h[1]\n"
- "fmla z20.h, z12.h, z5.h[1]\n"
- "fmla z24.h, z12.h, z6.h[1]\n"
- "fmla z28.h, z12.h, z7.h[1]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "fmla z17.h, z13.h, z4.h[1]\n"
- "fmla z21.h, z13.h, z5.h[1]\n"
- "fmla z25.h, z13.h, z6.h[1]\n"
- "fmla z29.h, z13.h, z7.h[1]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "fmla z18.h, z14.h, z4.h[1]\n"
- "fmla z22.h, z14.h, z5.h[1]\n"
- "fmla z26.h, z14.h, z6.h[1]\n"
- "fmla z30.h, z14.h, z7.h[1]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z19.h, z15.h, z4.h[1]\n"
- "fmla z23.h, z15.h, z5.h[1]\n"
- "fmla z27.h, z15.h, z6.h[1]\n"
- "fmla z31.h, z15.h, z7.h[1]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z16.h, z8.h, z4.h[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "fmla z20.h, z8.h, z5.h[2]\n"
- "fmla z24.h, z8.h, z6.h[2]\n"
- "fmla z28.h, z8.h, z7.h[2]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "fmla z17.h, z9.h, z4.h[2]\n"
- "fmla z21.h, z9.h, z5.h[2]\n"
- "fmla z25.h, z9.h, z6.h[2]\n"
- "fmla z29.h, z9.h, z7.h[2]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "fmla z18.h, z10.h, z4.h[2]\n"
- "fmla z22.h, z10.h, z5.h[2]\n"
- "fmla z26.h, z10.h, z6.h[2]\n"
- "fmla z30.h, z10.h, z7.h[2]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "fmla z19.h, z11.h, z4.h[2]\n"
- "fmla z23.h, z11.h, z5.h[2]\n"
- "fmla z27.h, z11.h, z6.h[2]\n"
- "fmla z31.h, z11.h, z7.h[2]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "fmla z16.h, z12.h, z4.h[3]\n"
- "fmla z20.h, z12.h, z5.h[3]\n"
- "fmla z24.h, z12.h, z6.h[3]\n"
- "fmla z28.h, z12.h, z7.h[3]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "fmla z17.h, z13.h, z4.h[3]\n"
- "fmla z21.h, z13.h, z5.h[3]\n"
- "fmla z25.h, z13.h, z6.h[3]\n"
- "fmla z29.h, z13.h, z7.h[3]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "fmla z18.h, z14.h, z4.h[3]\n"
- "fmla z22.h, z14.h, z5.h[3]\n"
- "fmla z26.h, z14.h, z6.h[3]\n"
- "fmla z30.h, z14.h, z7.h[3]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "fmla z19.h, z15.h, z4.h[3]\n"
- "fmla z23.h, z15.h, z5.h[3]\n"
- "fmla z27.h, z15.h, z6.h[3]\n"
- "fmla z31.h, z15.h, z7.h[3]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "fmla z16.h, z8.h, z4.h[4]\n"
- "fmla z20.h, z8.h, z5.h[4]\n"
- "fmla z24.h, z8.h, z6.h[4]\n"
- "fmla z28.h, z8.h, z7.h[4]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "fmla z17.h, z9.h, z4.h[4]\n"
- "fmla z21.h, z9.h, z5.h[4]\n"
- "fmla z25.h, z9.h, z6.h[4]\n"
- "fmla z29.h, z9.h, z7.h[4]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z18.h, z10.h, z4.h[4]\n"
- "fmla z22.h, z10.h, z5.h[4]\n"
- "fmla z26.h, z10.h, z6.h[4]\n"
- "fmla z30.h, z10.h, z7.h[4]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z19.h, z11.h, z4.h[4]\n"
- "fmla z23.h, z11.h, z5.h[4]\n"
- "fmla z27.h, z11.h, z6.h[4]\n"
- "fmla z31.h, z11.h, z7.h[4]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z16.h, z12.h, z4.h[5]\n"
- "fmla z20.h, z12.h, z5.h[5]\n"
- "fmla z24.h, z12.h, z6.h[5]\n"
- "fmla z28.h, z12.h, z7.h[5]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "fmla z17.h, z13.h, z4.h[5]\n"
- "fmla z21.h, z13.h, z5.h[5]\n"
- "fmla z25.h, z13.h, z6.h[5]\n"
- "fmla z29.h, z13.h, z7.h[5]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "fmla z18.h, z14.h, z4.h[5]\n"
- "fmla z22.h, z14.h, z5.h[5]\n"
- "fmla z26.h, z14.h, z6.h[5]\n"
- "fmla z30.h, z14.h, z7.h[5]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z19.h, z15.h, z4.h[5]\n"
- "fmla z23.h, z15.h, z5.h[5]\n"
- "fmla z27.h, z15.h, z6.h[5]\n"
- "fmla z31.h, z15.h, z7.h[5]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z16.h, z8.h, z4.h[6]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "fmla z20.h, z8.h, z5.h[6]\n"
- "fmla z24.h, z8.h, z6.h[6]\n"
- "fmla z28.h, z8.h, z7.h[6]\n"
- "fmla z17.h, z9.h, z4.h[6]\n"
- "fmla z21.h, z9.h, z5.h[6]\n"
- "fmla z25.h, z9.h, z6.h[6]\n"
- "fmla z29.h, z9.h, z7.h[6]\n"
- "fmla z18.h, z10.h, z4.h[6]\n"
- "fmla z22.h, z10.h, z5.h[6]\n"
- "fmla z26.h, z10.h, z6.h[6]\n"
- "fmla z30.h, z10.h, z7.h[6]\n"
- "fmla z19.h, z11.h, z4.h[6]\n"
- "fmla z23.h, z11.h, z5.h[6]\n"
- "fmla z27.h, z11.h, z6.h[6]\n"
- "fmla z31.h, z11.h, z7.h[6]\n"
- "fmla z16.h, z12.h, z4.h[7]\n"
- "fmla z20.h, z12.h, z5.h[7]\n"
- "fmla z24.h, z12.h, z6.h[7]\n"
- "fmla z28.h, z12.h, z7.h[7]\n"
- "fmla z17.h, z13.h, z4.h[7]\n"
- "fmla z21.h, z13.h, z5.h[7]\n"
- "fmla z25.h, z13.h, z6.h[7]\n"
- "fmla z29.h, z13.h, z7.h[7]\n"
- "fmla z18.h, z14.h, z4.h[7]\n"
- "fmla z22.h, z14.h, z5.h[7]\n"
- "fmla z26.h, z14.h, z6.h[7]\n"
- "fmla z30.h, z14.h, z7.h[7]\n"
- "fmla z19.h, z15.h, z4.h[7]\n"
- "fmla z23.h, z15.h, z5.h[7]\n"
- "fmla z27.h, z15.h, z6.h[7]\n"
- "fmla z31.h, z15.h, z7.h[7]\n"
- "cbz %[blocks], 5f\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z16.h, z8.h, z0.h[0]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z20.h, z8.h, z1.h[0]\n"
- "fmla z24.h, z8.h, z2.h[0]\n"
- "fmla z28.h, z8.h, z3.h[0]\n"
- "fmla z17.h, z9.h, z0.h[0]\n"
- "fmla z21.h, z9.h, z1.h[0]\n"
- "fmla z25.h, z9.h, z2.h[0]\n"
- "fmla z29.h, z9.h, z3.h[0]\n"
- "fmla z18.h, z10.h, z0.h[0]\n"
- "fmla z22.h, z10.h, z1.h[0]\n"
- "fmla z26.h, z10.h, z2.h[0]\n"
- "fmla z30.h, z10.h, z3.h[0]\n"
- "fmla z19.h, z11.h, z0.h[0]\n"
- "fmla z23.h, z11.h, z1.h[0]\n"
- "fmla z27.h, z11.h, z2.h[0]\n"
- "fmla z31.h, z11.h, z3.h[0]\n"
- "b.eq 5f\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z16.h, z12.h, z0.h[1]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z20.h, z12.h, z1.h[1]\n"
- "fmla z24.h, z12.h, z2.h[1]\n"
- "fmla z28.h, z12.h, z3.h[1]\n"
- "fmla z17.h, z13.h, z0.h[1]\n"
- "fmla z21.h, z13.h, z1.h[1]\n"
- "fmla z25.h, z13.h, z2.h[1]\n"
- "fmla z29.h, z13.h, z3.h[1]\n"
- "fmla z18.h, z14.h, z0.h[1]\n"
- "fmla z22.h, z14.h, z1.h[1]\n"
- "fmla z26.h, z14.h, z2.h[1]\n"
- "fmla z30.h, z14.h, z3.h[1]\n"
- "fmla z19.h, z15.h, z0.h[1]\n"
- "fmla z23.h, z15.h, z1.h[1]\n"
- "fmla z27.h, z15.h, z2.h[1]\n"
- "fmla z31.h, z15.h, z3.h[1]\n"
- "b.eq 5f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "fmla z16.h, z8.h, z0.h[2]\n"
- "fmla z20.h, z8.h, z1.h[2]\n"
- "fmla z24.h, z8.h, z2.h[2]\n"
- "fmla z28.h, z8.h, z3.h[2]\n"
- "fmla z17.h, z9.h, z0.h[2]\n"
- "fmla z21.h, z9.h, z1.h[2]\n"
- "fmla z25.h, z9.h, z2.h[2]\n"
- "fmla z29.h, z9.h, z3.h[2]\n"
- "fmla z18.h, z10.h, z0.h[2]\n"
- "fmla z22.h, z10.h, z1.h[2]\n"
- "fmla z26.h, z10.h, z2.h[2]\n"
- "fmla z30.h, z10.h, z3.h[2]\n"
- "fmla z19.h, z11.h, z0.h[2]\n"
- "fmla z23.h, z11.h, z1.h[2]\n"
- "fmla z27.h, z11.h, z2.h[2]\n"
- "fmla z31.h, z11.h, z3.h[2]\n"
- "b.eq 5f\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "fmla z16.h, z12.h, z0.h[3]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "fmla z20.h, z12.h, z1.h[3]\n"
- "fmla z24.h, z12.h, z2.h[3]\n"
- "fmla z28.h, z12.h, z3.h[3]\n"
- "fmla z17.h, z13.h, z0.h[3]\n"
- "fmla z21.h, z13.h, z1.h[3]\n"
- "fmla z25.h, z13.h, z2.h[3]\n"
- "fmla z29.h, z13.h, z3.h[3]\n"
- "fmla z18.h, z14.h, z0.h[3]\n"
- "fmla z22.h, z14.h, z1.h[3]\n"
- "fmla z26.h, z14.h, z2.h[3]\n"
- "fmla z30.h, z14.h, z3.h[3]\n"
- "fmla z19.h, z15.h, z0.h[3]\n"
- "fmla z23.h, z15.h, z1.h[3]\n"
- "fmla z27.h, z15.h, z2.h[3]\n"
- "fmla z31.h, z15.h, z3.h[3]\n"
- "b.eq 5f\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z16.h, z8.h, z0.h[4]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z20.h, z8.h, z1.h[4]\n"
- "fmla z24.h, z8.h, z2.h[4]\n"
- "fmla z28.h, z8.h, z3.h[4]\n"
- "fmla z17.h, z9.h, z0.h[4]\n"
- "fmla z21.h, z9.h, z1.h[4]\n"
- "fmla z25.h, z9.h, z2.h[4]\n"
- "fmla z29.h, z9.h, z3.h[4]\n"
- "fmla z18.h, z10.h, z0.h[4]\n"
- "fmla z22.h, z10.h, z1.h[4]\n"
- "fmla z26.h, z10.h, z2.h[4]\n"
- "fmla z30.h, z10.h, z3.h[4]\n"
- "fmla z19.h, z11.h, z0.h[4]\n"
- "fmla z23.h, z11.h, z1.h[4]\n"
- "fmla z27.h, z11.h, z2.h[4]\n"
- "fmla z31.h, z11.h, z3.h[4]\n"
- "b.eq 5f\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z16.h, z12.h, z0.h[5]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z20.h, z12.h, z1.h[5]\n"
- "fmla z24.h, z12.h, z2.h[5]\n"
- "fmla z28.h, z12.h, z3.h[5]\n"
- "fmla z17.h, z13.h, z0.h[5]\n"
- "fmla z21.h, z13.h, z1.h[5]\n"
- "fmla z25.h, z13.h, z2.h[5]\n"
- "fmla z29.h, z13.h, z3.h[5]\n"
- "fmla z18.h, z14.h, z0.h[5]\n"
- "fmla z22.h, z14.h, z1.h[5]\n"
- "fmla z26.h, z14.h, z2.h[5]\n"
- "fmla z30.h, z14.h, z3.h[5]\n"
- "fmla z19.h, z15.h, z0.h[5]\n"
- "fmla z23.h, z15.h, z1.h[5]\n"
- "fmla z27.h, z15.h, z2.h[5]\n"
- "fmla z31.h, z15.h, z3.h[5]\n"
- "b.eq 5f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "fmla z16.h, z8.h, z0.h[6]\n"
- "fmla z20.h, z8.h, z1.h[6]\n"
- "fmla z24.h, z8.h, z2.h[6]\n"
- "fmla z28.h, z8.h, z3.h[6]\n"
- "fmla z17.h, z9.h, z0.h[6]\n"
- "fmla z21.h, z9.h, z1.h[6]\n"
- "fmla z25.h, z9.h, z2.h[6]\n"
- "fmla z29.h, z9.h, z3.h[6]\n"
- "fmla z18.h, z10.h, z0.h[6]\n"
- "fmla z22.h, z10.h, z1.h[6]\n"
- "fmla z26.h, z10.h, z2.h[6]\n"
- "fmla z30.h, z10.h, z3.h[6]\n"
- "fmla z19.h, z11.h, z0.h[6]\n"
- "fmla z23.h, z11.h, z1.h[6]\n"
- "fmla z27.h, z11.h, z2.h[6]\n"
- "fmla z31.h, z11.h, z3.h[6]\n"
- "b 5f\n"
- "4:\n"
- "fmla z16.h, z8.h, z0.h[0]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "fmla z20.h, z8.h, z1.h[0]\n"
- "ld1rqh z4.h, p6/z, [%[a_ptr0]]\n"
- "fmla z24.h, z8.h, z2.h[0]\n"
- "ld1rqh z5.h, p6/z, [a_ptr1]\n"
- "fmla z28.h, z8.h, z3.h[0]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "fmla z17.h, z9.h, z0.h[0]\n"
- "ld1rqh z6.h, p6/z, [a_ptr2]\n"
- "fmla z21.h, z9.h, z1.h[0]\n"
- "ld1rqh z7.h, p6/z, [a_ptr3]\n"
- "fmla z25.h, z9.h, z2.h[0]\n"
- "addvl %[a_ptr0], %[a_ptr0], #1\n"
- "fmla z29.h, z9.h, z3.h[0]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z18.h, z10.h, z0.h[0]\n"
- "addvl a_ptr1, a_ptr1, #1\n"
- "fmla z22.h, z10.h, z1.h[0]\n"
- "addvl a_ptr2, a_ptr2, #1\n"
- "fmla z26.h, z10.h, z2.h[0]\n"
- "addvl a_ptr3, a_ptr3, #1\n"
- "fmla z30.h, z10.h, z3.h[0]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z19.h, z11.h, z0.h[0]\n"
- "fmla z23.h, z11.h, z1.h[0]\n"
- "fmla z27.h, z11.h, z2.h[0]\n"
- "fmla z31.h, z11.h, z3.h[0]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z16.h, z12.h, z0.h[1]\n"
- "fmla z20.h, z12.h, z1.h[1]\n"
- "fmla z24.h, z12.h, z2.h[1]\n"
- "fmla z28.h, z12.h, z3.h[1]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "fmla z17.h, z13.h, z0.h[1]\n"
- "fmla z21.h, z13.h, z1.h[1]\n"
- "fmla z25.h, z13.h, z2.h[1]\n"
- "fmla z29.h, z13.h, z3.h[1]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "fmla z18.h, z14.h, z0.h[1]\n"
- "fmla z22.h, z14.h, z1.h[1]\n"
- "fmla z26.h, z14.h, z2.h[1]\n"
- "fmla z30.h, z14.h, z3.h[1]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z19.h, z15.h, z0.h[1]\n"
- "fmla z23.h, z15.h, z1.h[1]\n"
- "fmla z27.h, z15.h, z2.h[1]\n"
- "fmla z31.h, z15.h, z3.h[1]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z16.h, z8.h, z0.h[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "fmla z20.h, z8.h, z1.h[2]\n"
- "fmla z24.h, z8.h, z2.h[2]\n"
- "fmla z28.h, z8.h, z3.h[2]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "fmla z17.h, z9.h, z0.h[2]\n"
- "fmla z21.h, z9.h, z1.h[2]\n"
- "fmla z25.h, z9.h, z2.h[2]\n"
- "fmla z29.h, z9.h, z3.h[2]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "fmla z18.h, z10.h, z0.h[2]\n"
- "fmla z22.h, z10.h, z1.h[2]\n"
- "fmla z26.h, z10.h, z2.h[2]\n"
- "fmla z30.h, z10.h, z3.h[2]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "fmla z19.h, z11.h, z0.h[2]\n"
- "fmla z23.h, z11.h, z1.h[2]\n"
- "fmla z27.h, z11.h, z2.h[2]\n"
- "fmla z31.h, z11.h, z3.h[2]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "fmla z16.h, z12.h, z0.h[3]\n"
- "fmla z20.h, z12.h, z1.h[3]\n"
- "fmla z24.h, z12.h, z2.h[3]\n"
- "fmla z28.h, z12.h, z3.h[3]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "fmla z17.h, z13.h, z0.h[3]\n"
- "fmla z21.h, z13.h, z1.h[3]\n"
- "fmla z25.h, z13.h, z2.h[3]\n"
- "fmla z29.h, z13.h, z3.h[3]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "fmla z18.h, z14.h, z0.h[3]\n"
- "fmla z22.h, z14.h, z1.h[3]\n"
- "fmla z26.h, z14.h, z2.h[3]\n"
- "fmla z30.h, z14.h, z3.h[3]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "fmla z19.h, z15.h, z0.h[3]\n"
- "fmla z23.h, z15.h, z1.h[3]\n"
- "fmla z27.h, z15.h, z2.h[3]\n"
- "fmla z31.h, z15.h, z3.h[3]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "fmla z16.h, z8.h, z0.h[4]\n"
- "fmla z20.h, z8.h, z1.h[4]\n"
- "fmla z24.h, z8.h, z2.h[4]\n"
- "fmla z28.h, z8.h, z3.h[4]\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "fmla z17.h, z9.h, z0.h[4]\n"
- "fmla z21.h, z9.h, z1.h[4]\n"
- "fmla z25.h, z9.h, z2.h[4]\n"
- "fmla z29.h, z9.h, z3.h[4]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z18.h, z10.h, z0.h[4]\n"
- "fmla z22.h, z10.h, z1.h[4]\n"
- "fmla z26.h, z10.h, z2.h[4]\n"
- "fmla z30.h, z10.h, z3.h[4]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z19.h, z11.h, z0.h[4]\n"
- "fmla z23.h, z11.h, z1.h[4]\n"
- "fmla z27.h, z11.h, z2.h[4]\n"
- "fmla z31.h, z11.h, z3.h[4]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z16.h, z12.h, z0.h[5]\n"
- "fmla z20.h, z12.h, z1.h[5]\n"
- "fmla z24.h, z12.h, z2.h[5]\n"
- "fmla z28.h, z12.h, z3.h[5]\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "fmla z17.h, z13.h, z0.h[5]\n"
- "fmla z21.h, z13.h, z1.h[5]\n"
- "fmla z25.h, z13.h, z2.h[5]\n"
- "fmla z29.h, z13.h, z3.h[5]\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "fmla z18.h, z14.h, z0.h[5]\n"
- "fmla z22.h, z14.h, z1.h[5]\n"
- "fmla z26.h, z14.h, z2.h[5]\n"
- "fmla z30.h, z14.h, z3.h[5]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z19.h, z15.h, z0.h[5]\n"
- "fmla z23.h, z15.h, z1.h[5]\n"
- "fmla z27.h, z15.h, z2.h[5]\n"
- "fmla z31.h, z15.h, z3.h[5]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z16.h, z8.h, z0.h[6]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "fmla z20.h, z8.h, z1.h[6]\n"
- "fmla z24.h, z8.h, z2.h[6]\n"
- "fmla z28.h, z8.h, z3.h[6]\n"
- "fmla z17.h, z9.h, z0.h[6]\n"
- "fmla z21.h, z9.h, z1.h[6]\n"
- "fmla z25.h, z9.h, z2.h[6]\n"
- "fmla z29.h, z9.h, z3.h[6]\n"
- "fmla z18.h, z10.h, z0.h[6]\n"
- "fmla z22.h, z10.h, z1.h[6]\n"
- "fmla z26.h, z10.h, z2.h[6]\n"
- "fmla z30.h, z10.h, z3.h[6]\n"
- "fmla z19.h, z11.h, z0.h[6]\n"
- "fmla z23.h, z11.h, z1.h[6]\n"
- "fmla z27.h, z11.h, z2.h[6]\n"
- "fmla z31.h, z11.h, z3.h[6]\n"
- "fmla z16.h, z12.h, z0.h[7]\n"
- "fmla z20.h, z12.h, z1.h[7]\n"
- "fmla z24.h, z12.h, z2.h[7]\n"
- "fmla z28.h, z12.h, z3.h[7]\n"
- "fmla z17.h, z13.h, z0.h[7]\n"
- "fmla z21.h, z13.h, z1.h[7]\n"
- "fmla z25.h, z13.h, z2.h[7]\n"
- "fmla z29.h, z13.h, z3.h[7]\n"
- "fmla z18.h, z14.h, z0.h[7]\n"
- "fmla z22.h, z14.h, z1.h[7]\n"
- "fmla z26.h, z14.h, z2.h[7]\n"
- "fmla z30.h, z14.h, z3.h[7]\n"
- "fmla z19.h, z15.h, z0.h[7]\n"
- "fmla z23.h, z15.h, z1.h[7]\n"
- "fmla z27.h, z15.h, z2.h[7]\n"
- "fmla z31.h, z15.h, z3.h[7]\n"
- "cbz %[blocks], 5f\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z16.h, z8.h, z4.h[0]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z20.h, z8.h, z5.h[0]\n"
- "fmla z24.h, z8.h, z6.h[0]\n"
- "fmla z28.h, z8.h, z7.h[0]\n"
- "fmla z17.h, z9.h, z4.h[0]\n"
- "fmla z21.h, z9.h, z5.h[0]\n"
- "fmla z25.h, z9.h, z6.h[0]\n"
- "fmla z29.h, z9.h, z7.h[0]\n"
- "fmla z18.h, z10.h, z4.h[0]\n"
- "fmla z22.h, z10.h, z5.h[0]\n"
- "fmla z26.h, z10.h, z6.h[0]\n"
- "fmla z30.h, z10.h, z7.h[0]\n"
- "fmla z19.h, z11.h, z4.h[0]\n"
- "fmla z23.h, z11.h, z5.h[0]\n"
- "fmla z27.h, z11.h, z6.h[0]\n"
- "fmla z31.h, z11.h, z7.h[0]\n"
- "b.eq 5f\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z16.h, z12.h, z4.h[1]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z20.h, z12.h, z5.h[1]\n"
- "fmla z24.h, z12.h, z6.h[1]\n"
- "fmla z28.h, z12.h, z7.h[1]\n"
- "fmla z17.h, z13.h, z4.h[1]\n"
- "fmla z21.h, z13.h, z5.h[1]\n"
- "fmla z25.h, z13.h, z6.h[1]\n"
- "fmla z29.h, z13.h, z7.h[1]\n"
- "fmla z18.h, z14.h, z4.h[1]\n"
- "fmla z22.h, z14.h, z5.h[1]\n"
- "fmla z26.h, z14.h, z6.h[1]\n"
- "fmla z30.h, z14.h, z7.h[1]\n"
- "fmla z19.h, z15.h, z4.h[1]\n"
- "fmla z23.h, z15.h, z5.h[1]\n"
- "fmla z27.h, z15.h, z6.h[1]\n"
- "fmla z31.h, z15.h, z7.h[1]\n"
- "b.eq 5f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "fmla z16.h, z8.h, z4.h[2]\n"
- "fmla z20.h, z8.h, z5.h[2]\n"
- "fmla z24.h, z8.h, z6.h[2]\n"
- "fmla z28.h, z8.h, z7.h[2]\n"
- "fmla z17.h, z9.h, z4.h[2]\n"
- "fmla z21.h, z9.h, z5.h[2]\n"
- "fmla z25.h, z9.h, z6.h[2]\n"
- "fmla z29.h, z9.h, z7.h[2]\n"
- "fmla z18.h, z10.h, z4.h[2]\n"
- "fmla z22.h, z10.h, z5.h[2]\n"
- "fmla z26.h, z10.h, z6.h[2]\n"
- "fmla z30.h, z10.h, z7.h[2]\n"
- "fmla z19.h, z11.h, z4.h[2]\n"
- "fmla z23.h, z11.h, z5.h[2]\n"
- "fmla z27.h, z11.h, z6.h[2]\n"
- "fmla z31.h, z11.h, z7.h[2]\n"
- "b.eq 5f\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "fmla z16.h, z12.h, z4.h[3]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "fmla z20.h, z12.h, z5.h[3]\n"
- "fmla z24.h, z12.h, z6.h[3]\n"
- "fmla z28.h, z12.h, z7.h[3]\n"
- "fmla z17.h, z13.h, z4.h[3]\n"
- "fmla z21.h, z13.h, z5.h[3]\n"
- "fmla z25.h, z13.h, z6.h[3]\n"
- "fmla z29.h, z13.h, z7.h[3]\n"
- "fmla z18.h, z14.h, z4.h[3]\n"
- "fmla z22.h, z14.h, z5.h[3]\n"
- "fmla z26.h, z14.h, z6.h[3]\n"
- "fmla z30.h, z14.h, z7.h[3]\n"
- "fmla z19.h, z15.h, z4.h[3]\n"
- "fmla z23.h, z15.h, z5.h[3]\n"
- "fmla z27.h, z15.h, z6.h[3]\n"
- "fmla z31.h, z15.h, z7.h[3]\n"
- "b.eq 5f\n"
- "ld1h z8.h, p7/z, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z16.h, z8.h, z4.h[4]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z20.h, z8.h, z5.h[4]\n"
- "fmla z24.h, z8.h, z6.h[4]\n"
- "fmla z28.h, z8.h, z7.h[4]\n"
- "fmla z17.h, z9.h, z4.h[4]\n"
- "fmla z21.h, z9.h, z5.h[4]\n"
- "fmla z25.h, z9.h, z6.h[4]\n"
- "fmla z29.h, z9.h, z7.h[4]\n"
- "fmla z18.h, z10.h, z4.h[4]\n"
- "fmla z22.h, z10.h, z5.h[4]\n"
- "fmla z26.h, z10.h, z6.h[4]\n"
- "fmla z30.h, z10.h, z7.h[4]\n"
- "fmla z19.h, z11.h, z4.h[4]\n"
- "fmla z23.h, z11.h, z5.h[4]\n"
- "fmla z27.h, z11.h, z6.h[4]\n"
- "fmla z31.h, z11.h, z7.h[4]\n"
- "b.eq 5f\n"
- "ld1h z12.h, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1h z13.h, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1h z14.h, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z16.h, z12.h, z4.h[5]\n"
- "ld1h z15.h, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z20.h, z12.h, z5.h[5]\n"
- "fmla z24.h, z12.h, z6.h[5]\n"
- "fmla z28.h, z12.h, z7.h[5]\n"
- "fmla z17.h, z13.h, z4.h[5]\n"
- "fmla z21.h, z13.h, z5.h[5]\n"
- "fmla z25.h, z13.h, z6.h[5]\n"
- "fmla z29.h, z13.h, z7.h[5]\n"
- "fmla z18.h, z14.h, z4.h[5]\n"
- "fmla z22.h, z14.h, z5.h[5]\n"
- "fmla z26.h, z14.h, z6.h[5]\n"
- "fmla z30.h, z14.h, z7.h[5]\n"
- "fmla z19.h, z15.h, z4.h[5]\n"
- "fmla z23.h, z15.h, z5.h[5]\n"
- "fmla z27.h, z15.h, z6.h[5]\n"
- "fmla z31.h, z15.h, z7.h[5]\n"
- "b.eq 5f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "ld1h z8.h, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1h z9.h, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1h z10.h, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1h z11.h, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "fmla z16.h, z8.h, z4.h[6]\n"
- "fmla z20.h, z8.h, z5.h[6]\n"
- "fmla z24.h, z8.h, z6.h[6]\n"
- "fmla z28.h, z8.h, z7.h[6]\n"
- "fmla z17.h, z9.h, z4.h[6]\n"
- "fmla z21.h, z9.h, z5.h[6]\n"
- "fmla z25.h, z9.h, z6.h[6]\n"
- "fmla z29.h, z9.h, z7.h[6]\n"
- "fmla z18.h, z10.h, z4.h[6]\n"
- "fmla z22.h, z10.h, z5.h[6]\n"
- "fmla z26.h, z10.h, z6.h[6]\n"
- "fmla z30.h, z10.h, z7.h[6]\n"
- "fmla z19.h, z11.h, z4.h[6]\n"
- "fmla z23.h, z11.h, z5.h[6]\n"
- "fmla z27.h, z11.h, z6.h[6]\n"
- "fmla z31.h, z11.h, z7.h[6]\n"
- "5:\n"
- "ld1rh z14.h, p7/z, [%[minptr]]\n"
- "ld1rh z15.h, p7/z, [%[maxptr]]\n"
- "fmax z16.h, p7/m, z16.h, z14.h\n"
- "fmax z17.h, p7/m, z17.h, z14.h\n"
- "fmax z18.h, p7/m, z18.h, z14.h\n"
- "fmax z19.h, p7/m, z19.h, z14.h\n"
- "fmin z16.h, p7/m, z16.h, z15.h\n"
- "fmin z17.h, p7/m, z17.h, z15.h\n"
- "fmin z18.h, p7/m, z18.h, z15.h\n"
- "fmin z19.h, p7/m, z19.h, z15.h\n"
- "st1h z16.h, p0, [%[c_ptr0]]\n"
- "fmax z20.h, p7/m, z20.h, z14.h\n"
- "fmax z21.h, p7/m, z21.h, z14.h\n"
- "fmax z22.h, p7/m, z22.h, z14.h\n"
- "st1h z17.h, p1, [%[c_ptr0], #1, MUL VL]\n"
- "fmax z23.h, p7/m, z23.h, z14.h\n"
- "fmin z20.h, p7/m, z20.h, z15.h\n"
- "fmin z21.h, p7/m, z21.h, z15.h\n"
- "st1h z18.h, p2, [%[c_ptr0], #2, MUL VL]\n"
- "fmin z22.h, p7/m, z22.h, z15.h\n"
- "fmin z23.h, p7/m, z23.h, z15.h\n"
- "fmax z24.h, p7/m, z24.h, z14.h\n"
- "st1h z19.h, p3, [%[c_ptr0], #3, MUL VL]\n"
- "fmax z25.h, p7/m, z25.h, z14.h\n"
- "addvl %[c_ptr0], %[c_ptr0], #4\n"
- "fmax z26.h, p7/m, z26.h, z14.h\n"
- "st1h z20.h, p0, [c_ptr1]\n"
- "fmin z24.h, p7/m, z24.h, z15.h\n"
- "fmin z25.h, p7/m, z25.h, z15.h\n"
- "fmax z27.h, p7/m, z27.h, z14.h\n"
- "st1h z21.h, p1, [c_ptr1, #1, MUL VL]\n"
- "fmin z26.h, p7/m, z26.h, z15.h\n"
- "fmax z28.h, p7/m, z28.h, z14.h\n"
- "fmax z29.h, p7/m, z29.h, z14.h\n"
- "st1h z22.h, p2, [c_ptr1, #2, MUL VL]\n"
- "fmin z27.h, p7/m, z27.h, z15.h\n"
- "fmax z30.h, p7/m, z30.h, z14.h\n"
- "fmin z28.h, p7/m, z28.h, z15.h\n"
- "st1h z23.h, p3, [c_ptr1, #3, MUL VL]\n"
- "fmin z29.h, p7/m, z29.h, z15.h\n"
- "fmax z31.h, p7/m, z31.h, z14.h\n"
- "fmin z30.h, p7/m, z30.h, z15.h\n"
- "st1h z24.h, p0, [c_ptr2]\n"
- "fmin z31.h, p7/m, z31.h, z15.h\n"
- "st1h z25.h, p1, [c_ptr2, #1, MUL VL]\n"
- "st1h z26.h, p2, [c_ptr2, #2, MUL VL]\n"
- "st1h z27.h, p3, [c_ptr2, #3, MUL VL]\n"
- "st1h z28.h, p0, [c_ptr3]\n"
- "st1h z29.h, p1, [c_ptr3, #1, MUL VL]\n"
- "st1h z30.h, p2, [c_ptr3, #2, MUL VL]\n"
- "st1h z31.h, p3, [c_ptr3, #3, MUL VL]\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq a_ptr3\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- ".unreq c_ptr3\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks)
- : [width] "r" (width), [accumulate] "r" (static_cast<uint64_t>(accumulate)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers)
- : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "x4", "x5", "cc", "memory"
- );
- break;
- }
-
- }
- }
-}
-
-} // namespace arm_gemm
-
-#endif // __ARM_FEATURE_SVE
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp16_mla_4VLx4.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp16_mla_6x4VL.hpp
similarity index 65%
rename from src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp16_mla_4VLx4.hpp
rename to src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp16_mla_6x4VL.hpp
index ebef413..0260050 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp16_mla_4VLx4.hpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp16_mla_6x4VL.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2018-2020 Arm Limited.
+ * Copyright (c) 2019-2020 Arm Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -10,42 +10,48 @@
* sell copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
*/
#pragma once
-
#ifdef __ARM_FEATURE_SVE
-
#include "../std_transforms_sve.hpp"
+#define ARGLIST \
+ unsigned int, const unsigned int *, \
+ IndirectInputArg<__fp16>, \
+ size_t, size_t, \
+ const __fp16 *, \
+ IndirectOutputArg<__fp16>, \
+ const __fp16 *, Activation, bool
+
namespace arm_gemm
{
// Actual kernel implementations
-void sve_hybrid_fp16_mla_4VLx4(const __fp16 *, int, const __fp16 *, __fp16 *, int, int, int, int, const __fp16 *, Activation, bool);
+void sve_hybrid_fp16_mla_6x4VL( ARGLIST );
-class hybrid_fp16_mla_4VLx4
+class cls_sve_hybrid_fp16_mla_6x4VL
{
public:
typedef __fp16 operand_type;
typedef __fp16 result_type;
- typedef void (*kern_type)(const __fp16 *, int, const __fp16 *, __fp16 *, int, int, int, int, const __fp16 *, Activation, bool);
+ typedef void (*kern_type)( ARGLIST );
/* Kernel blocking parameters */
static constexpr unsigned int out_height()
{
- return 4;
+ return 6;
}
static unsigned int out_width()
@@ -63,27 +69,17 @@
return true;
}
- static constexpr bool supports_bias()
- {
- return true;
- }
-
- static constexpr bool supports_activation()
- {
- return true;
- }
-
- StdTransformsSVE<operand_type, result_type, 4, 4, 1> transforms = {};
+ StdTransformsSVE<operand_type, result_type, 6, 4, 1> transforms = {};
// Default to the generic kernel
- kern_type kernel=sve_hybrid_fp16_mla_4VLx4;
+ kern_type kernel=sve_hybrid_fp16_mla_6x4VL;
- hybrid_fp16_mla_4VLx4(const CPUInfo *)
+ cls_sve_hybrid_fp16_mla_6x4VL(const CPUInfo *)
{
-
}
};
} // namespace arm_gemm
+#undef ARGLIST
#endif // __ARM_FEATURE_SVE
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp16_mla_6x4VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp16_mla_6x4VL/generic.cpp
new file mode 100644
index 0000000..b19842b
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp16_mla_6x4VL/generic.cpp
@@ -0,0 +1,3178 @@
+/*
+ * Copyright (c) 2019-2020 Arm Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+#ifdef __ARM_FEATURE_SVE
+
+#include "arm_gemm.hpp"
+#include "../../utils.hpp"
+
+#include <cassert>
+
+namespace arm_gemm {
+
+void sve_hybrid_fp16_mla_6x4VL (
+ unsigned int num_strings, const unsigned int *string_lengths, IndirectInputArg<__fp16> A_arg,
+ size_t M, size_t N, const __fp16 *B_ptr, IndirectOutputArg<__fp16> output_arg,
+ const __fp16 *bias, Activation act, bool accumulate
+)
+{
+ struct KernelArgs {
+ __fp16 maxval = static_cast<__fp16>(std::numeric_limits<float>::infinity());
+ __fp16 minval = - static_cast<__fp16>(std::numeric_limits<float>::infinity());
+ unsigned int num_strings = {};
+ const unsigned int *string_lengths = {};
+ size_t N = {};
+ const __fp16 *B_ptr = {};
+ size_t output_offset = {};
+ size_t input_initial_col = {};
+ size_t input_offset = {};
+ } ka;
+
+ unsigned long flags=0;
+ void *output_ptr;
+ void *input_ptr;
+
+ if (output_arg.is_indirect) {
+ output_ptr=(void *)(output_arg.indirect.ptr);
+ ka.output_offset=output_arg.indirect.offset;
+ flags |= 0x4;
+ } else {
+ output_ptr=(void *)(output_arg.direct.base);
+ ka.output_offset=output_arg.direct.stride;
+ }
+
+ if (A_arg.is_indirect) {
+ input_ptr=(void *)(A_arg.indirect.ptr);
+ ka.input_offset=A_arg.indirect.start_row;
+ ka.input_initial_col=A_arg.indirect.start_col;
+ flags |= 0x8;
+ } else {
+ assert(num_strings==1);
+ input_ptr=(void *)(A_arg.direct.base);
+ ka.input_offset=A_arg.direct.stride;
+ }
+ if (accumulate) {
+ flags |= 0x1;
+ }
+ ka.num_strings = num_strings;
+ ka.string_lengths = string_lengths;
+ ka.N = N;
+ ka.B_ptr = B_ptr;
+ switch(act.type) {
+ default:
+ case Activation::Type::None:
+ break;
+ case Activation::Type::BoundedReLU:
+ ka.maxval = static_cast<__fp16>(act.param1);
+ /* fall through */
+ case Activation::Type::ReLU:
+ ka.minval = 0;
+ flags |= 0x2;
+ break;
+ }
+ __asm__ __volatile__(
+ "ptrue p5.b\n"
+ "1:" // Row loop
+ "cmp %x[M], #0x6\n"
+ "bge 71f\n"
+ "cmp %x[M], #0x4\n"
+ "bgt 57f\n"
+ "beq 43f\n"
+ "cmp %x[M], #0x2\n"
+ "bgt 29f\n"
+ "beq 15f\n"
+ "ldr x16, [%x[args_ptr], %[offsetof_N]]\n"
+ "ldr x15, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "mov x14, %x[bias]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 2f\n"
+ "ldr x13, [%x[output_ptr], #0x0]\n"
+ "add x13, x13, x19, LSL #1\n"
+ "b 3f\n"
+ "2:" // Height 1: setup direct output
+ "mov x13, %x[output_ptr]\n"
+ "3:" // Height 1: Column loop
+ "mov x19, #0x0\n"
+ "whilelt p4.h, x19, x16\n"
+ "inch x19\n"
+ "whilelt p3.h, x19, x16\n"
+ "inch x19\n"
+ "whilelt p2.h, x19, x16\n"
+ "inch x19\n"
+ "whilelt p1.h, x19, x16\n"
+ "cbz x14, 4f\n"
+ "ld1h { z8.h }, p5/Z, [x14]\n"
+ "ld1h { z9.h }, p5/Z, [x14, #1, MUL VL]\n"
+ "ld1h { z10.h }, p5/Z, [x14, #2, MUL VL]\n"
+ "ld1h { z11.h }, p5/Z, [x14, #3, MUL VL]\n"
+ "addvl x14, x14, #4\n"
+ "b 6f\n"
+ "4:" // Height 1: no bias
+ "tbz %x[flags], #0, 5f\n"
+ "ld1h { z8.h }, p4/Z, [x13]\n"
+ "ld1h { z9.h }, p3/Z, [x13, #1, MUL VL]\n"
+ "ld1h { z10.h }, p2/Z, [x13, #2, MUL VL]\n"
+ "ld1h { z11.h }, p1/Z, [x13, #3, MUL VL]\n"
+ "b 6f\n"
+ "5:" // Height 1: no accumulate
+ "mov z8.b, #0x0\n"
+ "mov z9.b, #0x0\n"
+ "mov z10.b, #0x0\n"
+ "mov z11.b, #0x0\n"
+ "6:" // Height 1: setup done
+ "mov x12, #0x0\n"
+ "7:" // Height 1: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w11, [x20, x12, LSL #0x2]\n"
+ "tbz %x[flags], #3, 8f\n"
+ "ldr x20, [%x[input_ptr], x12, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x10, [x20, #0x0]\n"
+ "cbnz x12, 9f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x10, x10, x19, LSL #1\n"
+ "b 9f\n"
+ "8:" // Height 1: setup direct input
+ "mov x10, %x[input_ptr]\n"
+ "9:" // Height 1: input setup done
+ "cmp x11, #0x8\n"
+ "ble 11f\n"
+ "10:" // Height 1: Multiply loop: Main loop head
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ "whilelt p0.h, XZR, x11\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ "sub x11, x11, #0x8\n"
+ "ld1rqh { z0.h }, p0/Z, [x10]\n"
+ "fmla z8.h, z6.h, z0.h[0]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ "add x10, x10, #0x10\n"
+ "fmla z9.h, z7.h, z0.h[0]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ "cmp x11, #0x8\n"
+ "fmla z10.h, z6.h, z0.h[0]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #4, MUL VL]\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "fmla z11.h, z7.h, z0.h[0]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #5, MUL VL]\n"
+ "fmla z8.h, z6.h, z0.h[1]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #6, MUL VL]\n"
+ "fmla z9.h, z7.h, z0.h[1]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #7, MUL VL]\n"
+ "addvl x15, x15, #16\n"
+ "fmla z10.h, z6.h, z0.h[1]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #-8, MUL VL]\n"
+ "fmla z11.h, z7.h, z0.h[1]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #-7, MUL VL]\n"
+ "fmla z8.h, z6.h, z0.h[2]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #-6, MUL VL]\n"
+ "fmla z9.h, z7.h, z0.h[2]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #-5, MUL VL]\n"
+ "fmla z10.h, z6.h, z0.h[2]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #-4, MUL VL]\n"
+ "fmla z11.h, z7.h, z0.h[2]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #-3, MUL VL]\n"
+ "fmla z8.h, z6.h, z0.h[3]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #-2, MUL VL]\n"
+ "fmla z9.h, z7.h, z0.h[3]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #-1, MUL VL]\n"
+ "fmla z10.h, z6.h, z0.h[3]\n"
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ "fmla z11.h, z7.h, z0.h[3]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ "fmla z8.h, z6.h, z0.h[4]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ "fmla z9.h, z7.h, z0.h[4]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ "fmla z10.h, z6.h, z0.h[4]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #4, MUL VL]\n"
+ "fmla z11.h, z7.h, z0.h[4]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #5, MUL VL]\n"
+ "fmla z8.h, z6.h, z0.h[5]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #6, MUL VL]\n"
+ "fmla z9.h, z7.h, z0.h[5]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #7, MUL VL]\n"
+ "addvl x15, x15, #16\n"
+ "fmla z10.h, z6.h, z0.h[5]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #-8, MUL VL]\n"
+ "fmla z11.h, z7.h, z0.h[5]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #-7, MUL VL]\n"
+ "fmla z8.h, z6.h, z0.h[6]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #-6, MUL VL]\n"
+ "fmla z9.h, z7.h, z0.h[6]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #-5, MUL VL]\n"
+ "fmla z10.h, z6.h, z0.h[6]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #-4, MUL VL]\n"
+ "fmla z11.h, z7.h, z0.h[6]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #-3, MUL VL]\n"
+ "fmla z8.h, z6.h, z0.h[7]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #-2, MUL VL]\n"
+ "fmla z9.h, z7.h, z0.h[7]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #-1, MUL VL]\n"
+ "fmla z10.h, z6.h, z0.h[7]\n"
+ "fmla z11.h, z7.h, z0.h[7]\n"
+ "bgt 10b\n"
+ "11:" // Height 1: Multiply loop: Single iteration only
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ "whilelt p0.h, XZR, x11\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ "subs x11, x11, #0x1\n"
+ "ld1rqh { z0.h }, p0/Z, [x10]\n"
+ "fmla z8.h, z6.h, z0.h[0]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ "add x10, x10, #0x10\n"
+ "fmla z9.h, z7.h, z0.h[0]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ "addvl x15, x15, #4\n"
+ "fmla z10.h, z6.h, z0.h[0]\n"
+ "fmla z11.h, z7.h, z0.h[0]\n"
+ "ble 12f\n"
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ "fmla z8.h, z6.h, z0.h[1]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ "subs x11, x11, #0x1\n"
+ "fmla z9.h, z7.h, z0.h[1]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ "fmla z10.h, z6.h, z0.h[1]\n"
+ "addvl x15, x15, #4\n"
+ "fmla z11.h, z7.h, z0.h[1]\n"
+ "ble 12f\n"
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ "fmla z8.h, z6.h, z0.h[2]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ "subs x11, x11, #0x1\n"
+ "fmla z9.h, z7.h, z0.h[2]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ "fmla z10.h, z6.h, z0.h[2]\n"
+ "addvl x15, x15, #4\n"
+ "fmla z11.h, z7.h, z0.h[2]\n"
+ "ble 12f\n"
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ "fmla z8.h, z6.h, z0.h[3]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ "subs x11, x11, #0x1\n"
+ "fmla z9.h, z7.h, z0.h[3]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ "fmla z10.h, z6.h, z0.h[3]\n"
+ "addvl x15, x15, #4\n"
+ "fmla z11.h, z7.h, z0.h[3]\n"
+ "ble 12f\n"
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ "fmla z8.h, z6.h, z0.h[4]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ "subs x11, x11, #0x1\n"
+ "fmla z9.h, z7.h, z0.h[4]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ "fmla z10.h, z6.h, z0.h[4]\n"
+ "addvl x15, x15, #4\n"
+ "fmla z11.h, z7.h, z0.h[4]\n"
+ "ble 12f\n"
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ "fmla z8.h, z6.h, z0.h[5]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ "subs x11, x11, #0x1\n"
+ "fmla z9.h, z7.h, z0.h[5]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ "fmla z10.h, z6.h, z0.h[5]\n"
+ "addvl x15, x15, #4\n"
+ "fmla z11.h, z7.h, z0.h[5]\n"
+ "ble 12f\n"
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ "fmla z8.h, z6.h, z0.h[6]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ "subs x11, x11, #0x1\n"
+ "fmla z9.h, z7.h, z0.h[6]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ "fmla z10.h, z6.h, z0.h[6]\n"
+ "addvl x15, x15, #4\n"
+ "fmla z11.h, z7.h, z0.h[6]\n"
+ "ble 12f\n"
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ "fmla z8.h, z6.h, z0.h[7]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ "fmla z9.h, z7.h, z0.h[7]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ "addvl x15, x15, #4\n"
+ "fmla z10.h, z6.h, z0.h[7]\n"
+ "fmla z11.h, z7.h, z0.h[7]\n"
+ "12:" // Height 1: Multiply loop: multiply skip
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x12, x12, #0x1\n"
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "cmp x12, x19\n"
+ "bne 7b\n"
+ "prfm pstl1keep, [x13, #0x0]\n"
+ "tbz %x[flags], #1, 13f\n"
+ "add x19, %x[args_ptr], %[offset_min]\n"
+ "ld1rh { z1.h }, p5/Z, [x19]\n"
+ "add x19, %x[args_ptr], %[offset_max]\n"
+ "ld1rh { z0.h }, p5/Z, [x19]\n"
+ "fmin z8.h, p5/M, z8.h, z0.h\n"
+ "fmin z9.h, p5/M, z9.h, z0.h\n"
+ "fmin z10.h, p5/M, z10.h, z0.h\n"
+ "fmin z11.h, p5/M, z11.h, z0.h\n"
+ "fmax z8.h, p5/M, z8.h, z1.h\n"
+ "fmax z9.h, p5/M, z9.h, z1.h\n"
+ "fmax z10.h, p5/M, z10.h, z1.h\n"
+ "fmax z11.h, p5/M, z11.h, z1.h\n"
+ "13:" // Height 1: No activation
+ "st1h { z8.h }, p4, [x13]\n"
+ "st1h { z9.h }, p3, [x13, #1, MUL VL]\n"
+ "st1h { z10.h }, p2, [x13, #2, MUL VL]\n"
+ "st1h { z11.h }, p1, [x13, #3, MUL VL]\n"
+ "addvl x13, x13, #4\n"
+ "14:" // Height 1: Writeback done
+ "mov x19, #0x0\n"
+ "inch x19, ALL, MUL #4\n"
+ "subs x16, x16, x19\n"
+ "bgt 3b\n"
+ "b 86f\n"
+ "15:" // Height 2
+ "ldr x16, [%x[args_ptr], %[offsetof_N]]\n"
+ "mov x14, %x[bias]\n"
+ "ldr x15, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 16f\n"
+ "ldr x13, [%x[output_ptr], #0x0]\n"
+ "add x13, x13, x19, LSL #1\n"
+ "ldr x9, [%x[output_ptr], #0x8]\n"
+ "add x9, x9, x19, LSL #1\n"
+ "b 17f\n"
+ "16:" // Height 2: setup direct output
+ "mov x13, %x[output_ptr]\n"
+ "add x9, x13, x19, LSL #1\n"
+ "17:" // Height 2: Column loop
+ "mov x19, #0x0\n"
+ "whilelt p4.h, x19, x16\n"
+ "inch x19\n"
+ "whilelt p3.h, x19, x16\n"
+ "inch x19\n"
+ "whilelt p2.h, x19, x16\n"
+ "inch x19\n"
+ "whilelt p1.h, x19, x16\n"
+ "cbz x14, 18f\n"
+ "ld1h { z8.h }, p5/Z, [x14]\n"
+ "mov z12.d, z8.d\n"
+ "ld1h { z9.h }, p5/Z, [x14, #1, MUL VL]\n"
+ "ld1h { z10.h }, p5/Z, [x14, #2, MUL VL]\n"
+ "mov z13.d, z9.d\n"
+ "ld1h { z11.h }, p5/Z, [x14, #3, MUL VL]\n"
+ "addvl x14, x14, #4\n"
+ "mov z14.d, z10.d\n"
+ "mov z15.d, z11.d\n"
+ "b 20f\n"
+ "18:" // Height 2: no bias
+ "tbz %x[flags], #0, 19f\n"
+ "ld1h { z8.h }, p4/Z, [x13]\n"
+ "ld1h { z9.h }, p3/Z, [x13, #1, MUL VL]\n"
+ "ld1h { z10.h }, p2/Z, [x13, #2, MUL VL]\n"
+ "ld1h { z11.h }, p1/Z, [x13, #3, MUL VL]\n"
+ "ld1h { z12.h }, p4/Z, [x9]\n"
+ "ld1h { z13.h }, p3/Z, [x9, #1, MUL VL]\n"
+ "ld1h { z14.h }, p2/Z, [x9, #2, MUL VL]\n"
+ "ld1h { z15.h }, p1/Z, [x9, #3, MUL VL]\n"
+ "b 20f\n"
+ "19:" // Height 2: no accumulate
+ "mov z8.b, #0x0\n"
+ "mov z9.b, #0x0\n"
+ "mov z10.b, #0x0\n"
+ "mov z11.b, #0x0\n"
+ "mov z12.b, #0x0\n"
+ "mov z13.b, #0x0\n"
+ "mov z14.b, #0x0\n"
+ "mov z15.b, #0x0\n"
+ "20:" // Height 2: setup done
+ "mov x12, #0x0\n"
+ "21:" // Height 2: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w11, [x20, x12, LSL #0x2]\n"
+ "tbz %x[flags], #3, 22f\n"
+ "ldr x20, [%x[input_ptr], x12, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x10, [x20, #0x0]\n"
+ "ldr x28, [x20, #0x8]\n"
+ "cbnz x12, 23f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x10, x10, x19, LSL #1\n"
+ "add x28, x28, x19, LSL #1\n"
+ "b 23f\n"
+ "22:" // Height 2: setup direct input
+ "mov x10, %x[input_ptr]\n"
+ "add x28, x10, x19, LSL #1\n"
+ "23:" // Height 2: input setup done
+ "cmp x11, #0x8\n"
+ "ble 25f\n"
+ "24:" // Height 2: Multiply loop: Main loop head
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ "whilelt p0.h, XZR, x11\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ "sub x11, x11, #0x8\n"
+ "ld1rqh { z0.h }, p0/Z, [x10]\n"
+ "fmla z8.h, z6.h, z0.h[0]\n"
+ "ld1rqh { z1.h }, p0/Z, [x28]\n"
+ "add x10, x10, #0x10\n"
+ "fmla z9.h, z7.h, z0.h[0]\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x28, x28, #0x10\n"
+ "fmla z12.h, z6.h, z1.h[0]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ "cmp x11, #0x8\n"
+ "fmla z13.h, z7.h, z1.h[0]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "fmla z10.h, z6.h, z0.h[0]\n"
+ "fmla z14.h, z6.h, z1.h[0]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #4, MUL VL]\n"
+ "fmla z11.h, z7.h, z0.h[0]\n"
+ "fmla z15.h, z7.h, z1.h[0]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #5, MUL VL]\n"
+ "fmla z8.h, z6.h, z0.h[1]\n"
+ "fmla z12.h, z6.h, z1.h[1]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #6, MUL VL]\n"
+ "fmla z9.h, z7.h, z0.h[1]\n"
+ "fmla z13.h, z7.h, z1.h[1]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #7, MUL VL]\n"
+ "addvl x15, x15, #16\n"
+ "fmla z10.h, z6.h, z0.h[1]\n"
+ "fmla z14.h, z6.h, z1.h[1]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #-8, MUL VL]\n"
+ "fmla z11.h, z7.h, z0.h[1]\n"
+ "fmla z15.h, z7.h, z1.h[1]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #-7, MUL VL]\n"
+ "fmla z8.h, z6.h, z0.h[2]\n"
+ "fmla z12.h, z6.h, z1.h[2]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #-6, MUL VL]\n"
+ "fmla z9.h, z7.h, z0.h[2]\n"
+ "fmla z13.h, z7.h, z1.h[2]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #-5, MUL VL]\n"
+ "fmla z10.h, z6.h, z0.h[2]\n"
+ "fmla z14.h, z6.h, z1.h[2]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #-4, MUL VL]\n"
+ "fmla z11.h, z7.h, z0.h[2]\n"
+ "fmla z15.h, z7.h, z1.h[2]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #-3, MUL VL]\n"
+ "fmla z8.h, z6.h, z0.h[3]\n"
+ "fmla z12.h, z6.h, z1.h[3]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #-2, MUL VL]\n"
+ "fmla z9.h, z7.h, z0.h[3]\n"
+ "fmla z13.h, z7.h, z1.h[3]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #-1, MUL VL]\n"
+ "fmla z10.h, z6.h, z0.h[3]\n"
+ "fmla z14.h, z6.h, z1.h[3]\n"
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ "fmla z11.h, z7.h, z0.h[3]\n"
+ "fmla z15.h, z7.h, z1.h[3]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ "fmla z8.h, z6.h, z0.h[4]\n"
+ "fmla z12.h, z6.h, z1.h[4]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ "fmla z9.h, z7.h, z0.h[4]\n"
+ "fmla z13.h, z7.h, z1.h[4]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ "fmla z10.h, z6.h, z0.h[4]\n"
+ "fmla z14.h, z6.h, z1.h[4]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #4, MUL VL]\n"
+ "fmla z11.h, z7.h, z0.h[4]\n"
+ "fmla z15.h, z7.h, z1.h[4]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #5, MUL VL]\n"
+ "fmla z8.h, z6.h, z0.h[5]\n"
+ "fmla z12.h, z6.h, z1.h[5]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #6, MUL VL]\n"
+ "fmla z9.h, z7.h, z0.h[5]\n"
+ "fmla z13.h, z7.h, z1.h[5]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #7, MUL VL]\n"
+ "addvl x15, x15, #16\n"
+ "fmla z10.h, z6.h, z0.h[5]\n"
+ "fmla z14.h, z6.h, z1.h[5]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #-8, MUL VL]\n"
+ "fmla z11.h, z7.h, z0.h[5]\n"
+ "fmla z15.h, z7.h, z1.h[5]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #-7, MUL VL]\n"
+ "fmla z8.h, z6.h, z0.h[6]\n"
+ "fmla z12.h, z6.h, z1.h[6]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #-6, MUL VL]\n"
+ "fmla z9.h, z7.h, z0.h[6]\n"
+ "fmla z13.h, z7.h, z1.h[6]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #-5, MUL VL]\n"
+ "fmla z10.h, z6.h, z0.h[6]\n"
+ "fmla z14.h, z6.h, z1.h[6]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #-4, MUL VL]\n"
+ "fmla z11.h, z7.h, z0.h[6]\n"
+ "fmla z15.h, z7.h, z1.h[6]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #-3, MUL VL]\n"
+ "fmla z8.h, z6.h, z0.h[7]\n"
+ "fmla z12.h, z6.h, z1.h[7]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #-2, MUL VL]\n"
+ "fmla z9.h, z7.h, z0.h[7]\n"
+ "fmla z13.h, z7.h, z1.h[7]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #-1, MUL VL]\n"
+ "fmla z10.h, z6.h, z0.h[7]\n"
+ "fmla z14.h, z6.h, z1.h[7]\n"
+ "fmla z11.h, z7.h, z0.h[7]\n"
+ "fmla z15.h, z7.h, z1.h[7]\n"
+ "bgt 24b\n"
+ "25:" // Height 2: Multiply loop: Single iteration only
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ "whilelt p0.h, XZR, x11\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ "subs x11, x11, #0x1\n"
+ "ld1rqh { z0.h }, p0/Z, [x10]\n"
+ "fmla z8.h, z6.h, z0.h[0]\n"
+ "ld1rqh { z1.h }, p0/Z, [x28]\n"
+ "add x10, x10, #0x10\n"
+ "fmla z9.h, z7.h, z0.h[0]\n"
+ "add x28, x28, #0x10\n"
+ "fmla z12.h, z6.h, z1.h[0]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ "fmla z13.h, z7.h, z1.h[0]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ "addvl x15, x15, #4\n"
+ "fmla z10.h, z6.h, z0.h[0]\n"
+ "fmla z14.h, z6.h, z1.h[0]\n"
+ "fmla z11.h, z7.h, z0.h[0]\n"
+ "fmla z15.h, z7.h, z1.h[0]\n"
+ "ble 26f\n"
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ "fmla z8.h, z6.h, z0.h[1]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ "subs x11, x11, #0x1\n"
+ "fmla z12.h, z6.h, z1.h[1]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ "fmla z9.h, z7.h, z0.h[1]\n"
+ "fmla z13.h, z7.h, z1.h[1]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ "addvl x15, x15, #4\n"
+ "fmla z10.h, z6.h, z0.h[1]\n"
+ "fmla z14.h, z6.h, z1.h[1]\n"
+ "fmla z11.h, z7.h, z0.h[1]\n"
+ "fmla z15.h, z7.h, z1.h[1]\n"
+ "ble 26f\n"
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ "fmla z8.h, z6.h, z0.h[2]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ "subs x11, x11, #0x1\n"
+ "fmla z12.h, z6.h, z1.h[2]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ "fmla z9.h, z7.h, z0.h[2]\n"
+ "fmla z13.h, z7.h, z1.h[2]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ "addvl x15, x15, #4\n"
+ "fmla z10.h, z6.h, z0.h[2]\n"
+ "fmla z14.h, z6.h, z1.h[2]\n"
+ "fmla z11.h, z7.h, z0.h[2]\n"
+ "fmla z15.h, z7.h, z1.h[2]\n"
+ "ble 26f\n"
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ "fmla z8.h, z6.h, z0.h[3]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ "subs x11, x11, #0x1\n"
+ "fmla z12.h, z6.h, z1.h[3]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ "fmla z9.h, z7.h, z0.h[3]\n"
+ "fmla z13.h, z7.h, z1.h[3]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ "addvl x15, x15, #4\n"
+ "fmla z10.h, z6.h, z0.h[3]\n"
+ "fmla z14.h, z6.h, z1.h[3]\n"
+ "fmla z11.h, z7.h, z0.h[3]\n"
+ "fmla z15.h, z7.h, z1.h[3]\n"
+ "ble 26f\n"
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ "fmla z8.h, z6.h, z0.h[4]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ "subs x11, x11, #0x1\n"
+ "fmla z12.h, z6.h, z1.h[4]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ "fmla z9.h, z7.h, z0.h[4]\n"
+ "fmla z13.h, z7.h, z1.h[4]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ "addvl x15, x15, #4\n"
+ "fmla z10.h, z6.h, z0.h[4]\n"
+ "fmla z14.h, z6.h, z1.h[4]\n"
+ "fmla z11.h, z7.h, z0.h[4]\n"
+ "fmla z15.h, z7.h, z1.h[4]\n"
+ "ble 26f\n"
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ "fmla z8.h, z6.h, z0.h[5]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ "subs x11, x11, #0x1\n"
+ "fmla z12.h, z6.h, z1.h[5]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ "fmla z9.h, z7.h, z0.h[5]\n"
+ "fmla z13.h, z7.h, z1.h[5]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ "addvl x15, x15, #4\n"
+ "fmla z10.h, z6.h, z0.h[5]\n"
+ "fmla z14.h, z6.h, z1.h[5]\n"
+ "fmla z11.h, z7.h, z0.h[5]\n"
+ "fmla z15.h, z7.h, z1.h[5]\n"
+ "ble 26f\n"
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ "fmla z8.h, z6.h, z0.h[6]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ "subs x11, x11, #0x1\n"
+ "fmla z12.h, z6.h, z1.h[6]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ "fmla z9.h, z7.h, z0.h[6]\n"
+ "fmla z13.h, z7.h, z1.h[6]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ "addvl x15, x15, #4\n"
+ "fmla z10.h, z6.h, z0.h[6]\n"
+ "fmla z14.h, z6.h, z1.h[6]\n"
+ "fmla z11.h, z7.h, z0.h[6]\n"
+ "fmla z15.h, z7.h, z1.h[6]\n"
+ "ble 26f\n"
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ "fmla z8.h, z6.h, z0.h[7]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ "fmla z12.h, z6.h, z1.h[7]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ "fmla z9.h, z7.h, z0.h[7]\n"
+ "fmla z13.h, z7.h, z1.h[7]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ "addvl x15, x15, #4\n"
+ "fmla z10.h, z6.h, z0.h[7]\n"
+ "fmla z14.h, z6.h, z1.h[7]\n"
+ "fmla z11.h, z7.h, z0.h[7]\n"
+ "fmla z15.h, z7.h, z1.h[7]\n"
+ "26:" // Height 2: Multiply loop: multiply skip
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x12, x12, #0x1\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "cmp x12, x19\n"
+ "bne 21b\n"
+ "prfm pstl1keep, [x13, #0x0]\n"
+ "prfm pstl1keep, [x9, #0x0]\n"
+ "tbz %x[flags], #1, 27f\n"
+ "add x19, %x[args_ptr], %[offset_min]\n"
+ "ld1rh { z1.h }, p5/Z, [x19]\n"
+ "add x19, %x[args_ptr], %[offset_max]\n"
+ "ld1rh { z0.h }, p5/Z, [x19]\n"
+ "fmin z8.h, p5/M, z8.h, z0.h\n"
+ "fmin z9.h, p5/M, z9.h, z0.h\n"
+ "fmin z10.h, p5/M, z10.h, z0.h\n"
+ "fmin z11.h, p5/M, z11.h, z0.h\n"
+ "fmin z12.h, p5/M, z12.h, z0.h\n"
+ "fmax z8.h, p5/M, z8.h, z1.h\n"
+ "fmax z9.h, p5/M, z9.h, z1.h\n"
+ "fmax z10.h, p5/M, z10.h, z1.h\n"
+ "fmax z11.h, p5/M, z11.h, z1.h\n"
+ "fmax z12.h, p5/M, z12.h, z1.h\n"
+ "fmin z13.h, p5/M, z13.h, z0.h\n"
+ "fmin z14.h, p5/M, z14.h, z0.h\n"
+ "fmin z15.h, p5/M, z15.h, z0.h\n"
+ "fmax z13.h, p5/M, z13.h, z1.h\n"
+ "fmax z14.h, p5/M, z14.h, z1.h\n"
+ "fmax z15.h, p5/M, z15.h, z1.h\n"
+ "27:" // Height 2: No activation
+ "st1h { z8.h }, p4, [x13]\n"
+ "st1h { z9.h }, p3, [x13, #1, MUL VL]\n"
+ "st1h { z10.h }, p2, [x13, #2, MUL VL]\n"
+ "st1h { z11.h }, p1, [x13, #3, MUL VL]\n"
+ "addvl x13, x13, #4\n"
+ "st1h { z12.h }, p4, [x9]\n"
+ "st1h { z13.h }, p3, [x9, #1, MUL VL]\n"
+ "st1h { z14.h }, p2, [x9, #2, MUL VL]\n"
+ "st1h { z15.h }, p1, [x9, #3, MUL VL]\n"
+ "addvl x9, x9, #4\n"
+ "28:" // Height 2: Writeback done
+ "mov x19, #0x0\n"
+ "inch x19, ALL, MUL #4\n"
+ "subs x16, x16, x19\n"
+ "bgt 17b\n"
+ "b 86f\n"
+ "29:" // Height 3
+ "ldr x16, [%x[args_ptr], %[offsetof_N]]\n"
+ "mov x14, %x[bias]\n"
+ "ldr x15, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 30f\n"
+ "ldr x13, [%x[output_ptr], #0x0]\n"
+ "add x13, x13, x19, LSL #1\n"
+ "ldr x9, [%x[output_ptr], #0x8]\n"
+ "ldr x27, [%x[output_ptr], #0x10]\n"
+ "add x9, x9, x19, LSL #1\n"
+ "add x27, x27, x19, LSL #1\n"
+ "b 31f\n"
+ "30:" // Height 3: setup direct output
+ "mov x13, %x[output_ptr]\n"
+ "add x9, x13, x19, LSL #1\n"
+ "add x27, x9, x19, LSL #1\n"
+ "31:" // Height 3: Column loop
+ "mov x19, #0x0\n"
+ "whilelt p4.h, x19, x16\n"
+ "inch x19\n"
+ "whilelt p3.h, x19, x16\n"
+ "inch x19\n"
+ "whilelt p2.h, x19, x16\n"
+ "inch x19\n"
+ "whilelt p1.h, x19, x16\n"
+ "cbz x14, 32f\n"
+ "ld1h { z8.h }, p5/Z, [x14]\n"
+ "mov z12.d, z8.d\n"
+ "ld1h { z9.h }, p5/Z, [x14, #1, MUL VL]\n"
+ "mov z16.d, z8.d\n"
+ "ld1h { z10.h }, p5/Z, [x14, #2, MUL VL]\n"
+ "ld1h { z11.h }, p5/Z, [x14, #3, MUL VL]\n"
+ "mov z13.d, z9.d\n"
+ "addvl x14, x14, #4\n"
+ "mov z17.d, z9.d\n"
+ "mov z14.d, z10.d\n"
+ "mov z15.d, z11.d\n"
+ "mov z18.d, z10.d\n"
+ "mov z19.d, z11.d\n"
+ "b 34f\n"
+ "32:" // Height 3: no bias
+ "tbz %x[flags], #0, 33f\n"
+ "ld1h { z8.h }, p4/Z, [x13]\n"
+ "ld1h { z9.h }, p3/Z, [x13, #1, MUL VL]\n"
+ "ld1h { z10.h }, p2/Z, [x13, #2, MUL VL]\n"
+ "ld1h { z11.h }, p1/Z, [x13, #3, MUL VL]\n"
+ "ld1h { z12.h }, p4/Z, [x9]\n"
+ "ld1h { z13.h }, p3/Z, [x9, #1, MUL VL]\n"
+ "ld1h { z14.h }, p2/Z, [x9, #2, MUL VL]\n"
+ "ld1h { z15.h }, p1/Z, [x9, #3, MUL VL]\n"
+ "ld1h { z16.h }, p4/Z, [x27]\n"
+ "ld1h { z17.h }, p3/Z, [x27, #1, MUL VL]\n"
+ "ld1h { z18.h }, p2/Z, [x27, #2, MUL VL]\n"
+ "ld1h { z19.h }, p1/Z, [x27, #3, MUL VL]\n"
+ "b 34f\n"
+ "33:" // Height 3: no accumulate
+ "mov z8.b, #0x0\n"
+ "mov z9.b, #0x0\n"
+ "mov z10.b, #0x0\n"
+ "mov z11.b, #0x0\n"
+ "mov z12.b, #0x0\n"
+ "mov z13.b, #0x0\n"
+ "mov z14.b, #0x0\n"
+ "mov z15.b, #0x0\n"
+ "mov z16.b, #0x0\n"
+ "mov z17.b, #0x0\n"
+ "mov z18.b, #0x0\n"
+ "mov z19.b, #0x0\n"
+ "34:" // Height 3: setup done
+ "mov x12, #0x0\n"
+ "35:" // Height 3: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w11, [x20, x12, LSL #0x2]\n"
+ "tbz %x[flags], #3, 36f\n"
+ "ldr x20, [%x[input_ptr], x12, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x10, [x20, #0x0]\n"
+ "ldr x28, [x20, #0x8]\n"
+ "ldr x26, [x20, #0x10]\n"
+ "cbnz x12, 37f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x10, x10, x19, LSL #1\n"
+ "add x28, x28, x19, LSL #1\n"
+ "add x26, x26, x19, LSL #1\n"
+ "b 37f\n"
+ "36:" // Height 3: setup direct input
+ "mov x10, %x[input_ptr]\n"
+ "add x28, x10, x19, LSL #1\n"
+ "add x26, x28, x19, LSL #1\n"
+ "37:" // Height 3: input setup done
+ "cmp x11, #0x8\n"
+ "ble 39f\n"
+ "38:" // Height 3: Multiply loop: Main loop head
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ "whilelt p0.h, XZR, x11\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ "sub x11, x11, #0x8\n"
+ "ld1rqh { z0.h }, p0/Z, [x10]\n"
+ "fmla z8.h, z6.h, z0.h[0]\n"
+ "ld1rqh { z1.h }, p0/Z, [x28]\n"
+ "add x10, x10, #0x10\n"
+ "fmla z9.h, z7.h, z0.h[0]\n"
+ "ld1rqh { z2.h }, p0/Z, [x26]\n"
+ "add x28, x28, #0x10\n"
+ "fmla z12.h, z6.h, z1.h[0]\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x26, x26, #0x10\n"
+ "fmla z16.h, z6.h, z2.h[0]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ "cmp x11, #0x8\n"
+ "fmla z13.h, z7.h, z1.h[0]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "fmla z17.h, z7.h, z2.h[0]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "fmla z10.h, z6.h, z0.h[0]\n"
+ "fmla z14.h, z6.h, z1.h[0]\n"
+ "fmla z18.h, z6.h, z2.h[0]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #4, MUL VL]\n"
+ "fmla z11.h, z7.h, z0.h[0]\n"
+ "fmla z15.h, z7.h, z1.h[0]\n"
+ "fmla z19.h, z7.h, z2.h[0]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #5, MUL VL]\n"
+ "fmla z8.h, z6.h, z0.h[1]\n"
+ "fmla z12.h, z6.h, z1.h[1]\n"
+ "fmla z16.h, z6.h, z2.h[1]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #6, MUL VL]\n"
+ "fmla z9.h, z7.h, z0.h[1]\n"
+ "fmla z13.h, z7.h, z1.h[1]\n"
+ "fmla z17.h, z7.h, z2.h[1]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #7, MUL VL]\n"
+ "addvl x15, x15, #16\n"
+ "fmla z10.h, z6.h, z0.h[1]\n"
+ "fmla z14.h, z6.h, z1.h[1]\n"
+ "fmla z18.h, z6.h, z2.h[1]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #-8, MUL VL]\n"
+ "fmla z11.h, z7.h, z0.h[1]\n"
+ "fmla z15.h, z7.h, z1.h[1]\n"
+ "fmla z19.h, z7.h, z2.h[1]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #-7, MUL VL]\n"
+ "fmla z8.h, z6.h, z0.h[2]\n"
+ "fmla z12.h, z6.h, z1.h[2]\n"
+ "fmla z16.h, z6.h, z2.h[2]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #-6, MUL VL]\n"
+ "fmla z9.h, z7.h, z0.h[2]\n"
+ "fmla z13.h, z7.h, z1.h[2]\n"
+ "fmla z17.h, z7.h, z2.h[2]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #-5, MUL VL]\n"
+ "fmla z10.h, z6.h, z0.h[2]\n"
+ "fmla z14.h, z6.h, z1.h[2]\n"
+ "fmla z18.h, z6.h, z2.h[2]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #-4, MUL VL]\n"
+ "fmla z11.h, z7.h, z0.h[2]\n"
+ "fmla z15.h, z7.h, z1.h[2]\n"
+ "fmla z19.h, z7.h, z2.h[2]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #-3, MUL VL]\n"
+ "fmla z8.h, z6.h, z0.h[3]\n"
+ "fmla z12.h, z6.h, z1.h[3]\n"
+ "fmla z16.h, z6.h, z2.h[3]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #-2, MUL VL]\n"
+ "fmla z9.h, z7.h, z0.h[3]\n"
+ "fmla z13.h, z7.h, z1.h[3]\n"
+ "fmla z17.h, z7.h, z2.h[3]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #-1, MUL VL]\n"
+ "fmla z10.h, z6.h, z0.h[3]\n"
+ "fmla z14.h, z6.h, z1.h[3]\n"
+ "fmla z18.h, z6.h, z2.h[3]\n"
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ "fmla z11.h, z7.h, z0.h[3]\n"
+ "fmla z15.h, z7.h, z1.h[3]\n"
+ "fmla z19.h, z7.h, z2.h[3]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ "fmla z8.h, z6.h, z0.h[4]\n"
+ "fmla z12.h, z6.h, z1.h[4]\n"
+ "fmla z16.h, z6.h, z2.h[4]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ "fmla z9.h, z7.h, z0.h[4]\n"
+ "fmla z13.h, z7.h, z1.h[4]\n"
+ "fmla z17.h, z7.h, z2.h[4]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ "fmla z10.h, z6.h, z0.h[4]\n"
+ "fmla z14.h, z6.h, z1.h[4]\n"
+ "fmla z18.h, z6.h, z2.h[4]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #4, MUL VL]\n"
+ "fmla z11.h, z7.h, z0.h[4]\n"
+ "fmla z15.h, z7.h, z1.h[4]\n"
+ "fmla z19.h, z7.h, z2.h[4]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #5, MUL VL]\n"
+ "fmla z8.h, z6.h, z0.h[5]\n"
+ "fmla z12.h, z6.h, z1.h[5]\n"
+ "fmla z16.h, z6.h, z2.h[5]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #6, MUL VL]\n"
+ "fmla z9.h, z7.h, z0.h[5]\n"
+ "fmla z13.h, z7.h, z1.h[5]\n"
+ "fmla z17.h, z7.h, z2.h[5]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #7, MUL VL]\n"
+ "addvl x15, x15, #16\n"
+ "fmla z10.h, z6.h, z0.h[5]\n"
+ "fmla z14.h, z6.h, z1.h[5]\n"
+ "fmla z18.h, z6.h, z2.h[5]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #-8, MUL VL]\n"
+ "fmla z11.h, z7.h, z0.h[5]\n"
+ "fmla z15.h, z7.h, z1.h[5]\n"
+ "fmla z19.h, z7.h, z2.h[5]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #-7, MUL VL]\n"
+ "fmla z8.h, z6.h, z0.h[6]\n"
+ "fmla z12.h, z6.h, z1.h[6]\n"
+ "fmla z16.h, z6.h, z2.h[6]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #-6, MUL VL]\n"
+ "fmla z9.h, z7.h, z0.h[6]\n"
+ "fmla z13.h, z7.h, z1.h[6]\n"
+ "fmla z17.h, z7.h, z2.h[6]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #-5, MUL VL]\n"
+ "fmla z10.h, z6.h, z0.h[6]\n"
+ "fmla z14.h, z6.h, z1.h[6]\n"
+ "fmla z18.h, z6.h, z2.h[6]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #-4, MUL VL]\n"
+ "fmla z11.h, z7.h, z0.h[6]\n"
+ "fmla z15.h, z7.h, z1.h[6]\n"
+ "fmla z19.h, z7.h, z2.h[6]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #-3, MUL VL]\n"
+ "fmla z8.h, z6.h, z0.h[7]\n"
+ "fmla z12.h, z6.h, z1.h[7]\n"
+ "fmla z16.h, z6.h, z2.h[7]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #-2, MUL VL]\n"
+ "fmla z9.h, z7.h, z0.h[7]\n"
+ "fmla z13.h, z7.h, z1.h[7]\n"
+ "fmla z17.h, z7.h, z2.h[7]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #-1, MUL VL]\n"
+ "fmla z10.h, z6.h, z0.h[7]\n"
+ "fmla z14.h, z6.h, z1.h[7]\n"
+ "fmla z18.h, z6.h, z2.h[7]\n"
+ "fmla z11.h, z7.h, z0.h[7]\n"
+ "fmla z15.h, z7.h, z1.h[7]\n"
+ "fmla z19.h, z7.h, z2.h[7]\n"
+ "bgt 38b\n"
+ "39:" // Height 3: Multiply loop: Single iteration only
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ "whilelt p0.h, XZR, x11\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ "subs x11, x11, #0x1\n"
+ "ld1rqh { z0.h }, p0/Z, [x10]\n"
+ "fmla z8.h, z6.h, z0.h[0]\n"
+ "ld1rqh { z1.h }, p0/Z, [x28]\n"
+ "add x10, x10, #0x10\n"
+ "fmla z9.h, z7.h, z0.h[0]\n"
+ "ld1rqh { z2.h }, p0/Z, [x26]\n"
+ "add x28, x28, #0x10\n"
+ "fmla z12.h, z6.h, z1.h[0]\n"
+ "add x26, x26, #0x10\n"
+ "fmla z13.h, z7.h, z1.h[0]\n"
+ "fmla z16.h, z6.h, z2.h[0]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ "fmla z17.h, z7.h, z2.h[0]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ "addvl x15, x15, #4\n"
+ "fmla z10.h, z6.h, z0.h[0]\n"
+ "fmla z14.h, z6.h, z1.h[0]\n"
+ "fmla z18.h, z6.h, z2.h[0]\n"
+ "fmla z11.h, z7.h, z0.h[0]\n"
+ "fmla z15.h, z7.h, z1.h[0]\n"
+ "fmla z19.h, z7.h, z2.h[0]\n"
+ "ble 40f\n"
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ "fmla z8.h, z6.h, z0.h[1]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ "subs x11, x11, #0x1\n"
+ "fmla z12.h, z6.h, z1.h[1]\n"
+ "fmla z16.h, z6.h, z2.h[1]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ "fmla z9.h, z7.h, z0.h[1]\n"
+ "fmla z13.h, z7.h, z1.h[1]\n"
+ "fmla z17.h, z7.h, z2.h[1]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ "addvl x15, x15, #4\n"
+ "fmla z10.h, z6.h, z0.h[1]\n"
+ "fmla z14.h, z6.h, z1.h[1]\n"
+ "fmla z18.h, z6.h, z2.h[1]\n"
+ "fmla z11.h, z7.h, z0.h[1]\n"
+ "fmla z15.h, z7.h, z1.h[1]\n"
+ "fmla z19.h, z7.h, z2.h[1]\n"
+ "ble 40f\n"
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ "fmla z8.h, z6.h, z0.h[2]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ "subs x11, x11, #0x1\n"
+ "fmla z12.h, z6.h, z1.h[2]\n"
+ "fmla z16.h, z6.h, z2.h[2]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ "fmla z9.h, z7.h, z0.h[2]\n"
+ "fmla z13.h, z7.h, z1.h[2]\n"
+ "fmla z17.h, z7.h, z2.h[2]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ "addvl x15, x15, #4\n"
+ "fmla z10.h, z6.h, z0.h[2]\n"
+ "fmla z14.h, z6.h, z1.h[2]\n"
+ "fmla z18.h, z6.h, z2.h[2]\n"
+ "fmla z11.h, z7.h, z0.h[2]\n"
+ "fmla z15.h, z7.h, z1.h[2]\n"
+ "fmla z19.h, z7.h, z2.h[2]\n"
+ "ble 40f\n"
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ "fmla z8.h, z6.h, z0.h[3]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ "subs x11, x11, #0x1\n"
+ "fmla z12.h, z6.h, z1.h[3]\n"
+ "fmla z16.h, z6.h, z2.h[3]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ "fmla z9.h, z7.h, z0.h[3]\n"
+ "fmla z13.h, z7.h, z1.h[3]\n"
+ "fmla z17.h, z7.h, z2.h[3]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ "addvl x15, x15, #4\n"
+ "fmla z10.h, z6.h, z0.h[3]\n"
+ "fmla z14.h, z6.h, z1.h[3]\n"
+ "fmla z18.h, z6.h, z2.h[3]\n"
+ "fmla z11.h, z7.h, z0.h[3]\n"
+ "fmla z15.h, z7.h, z1.h[3]\n"
+ "fmla z19.h, z7.h, z2.h[3]\n"
+ "ble 40f\n"
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ "fmla z8.h, z6.h, z0.h[4]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ "subs x11, x11, #0x1\n"
+ "fmla z12.h, z6.h, z1.h[4]\n"
+ "fmla z16.h, z6.h, z2.h[4]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ "fmla z9.h, z7.h, z0.h[4]\n"
+ "fmla z13.h, z7.h, z1.h[4]\n"
+ "fmla z17.h, z7.h, z2.h[4]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ "addvl x15, x15, #4\n"
+ "fmla z10.h, z6.h, z0.h[4]\n"
+ "fmla z14.h, z6.h, z1.h[4]\n"
+ "fmla z18.h, z6.h, z2.h[4]\n"
+ "fmla z11.h, z7.h, z0.h[4]\n"
+ "fmla z15.h, z7.h, z1.h[4]\n"
+ "fmla z19.h, z7.h, z2.h[4]\n"
+ "ble 40f\n"
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ "fmla z8.h, z6.h, z0.h[5]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ "subs x11, x11, #0x1\n"
+ "fmla z12.h, z6.h, z1.h[5]\n"
+ "fmla z16.h, z6.h, z2.h[5]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ "fmla z9.h, z7.h, z0.h[5]\n"
+ "fmla z13.h, z7.h, z1.h[5]\n"
+ "fmla z17.h, z7.h, z2.h[5]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ "addvl x15, x15, #4\n"
+ "fmla z10.h, z6.h, z0.h[5]\n"
+ "fmla z14.h, z6.h, z1.h[5]\n"
+ "fmla z18.h, z6.h, z2.h[5]\n"
+ "fmla z11.h, z7.h, z0.h[5]\n"
+ "fmla z15.h, z7.h, z1.h[5]\n"
+ "fmla z19.h, z7.h, z2.h[5]\n"
+ "ble 40f\n"
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ "fmla z8.h, z6.h, z0.h[6]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ "subs x11, x11, #0x1\n"
+ "fmla z12.h, z6.h, z1.h[6]\n"
+ "fmla z16.h, z6.h, z2.h[6]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ "fmla z9.h, z7.h, z0.h[6]\n"
+ "fmla z13.h, z7.h, z1.h[6]\n"
+ "fmla z17.h, z7.h, z2.h[6]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ "addvl x15, x15, #4\n"
+ "fmla z10.h, z6.h, z0.h[6]\n"
+ "fmla z14.h, z6.h, z1.h[6]\n"
+ "fmla z18.h, z6.h, z2.h[6]\n"
+ "fmla z11.h, z7.h, z0.h[6]\n"
+ "fmla z15.h, z7.h, z1.h[6]\n"
+ "fmla z19.h, z7.h, z2.h[6]\n"
+ "ble 40f\n"
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ "fmla z8.h, z6.h, z0.h[7]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ "fmla z12.h, z6.h, z1.h[7]\n"
+ "fmla z16.h, z6.h, z2.h[7]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ "fmla z9.h, z7.h, z0.h[7]\n"
+ "fmla z13.h, z7.h, z1.h[7]\n"
+ "fmla z17.h, z7.h, z2.h[7]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ "addvl x15, x15, #4\n"
+ "fmla z10.h, z6.h, z0.h[7]\n"
+ "fmla z14.h, z6.h, z1.h[7]\n"
+ "fmla z18.h, z6.h, z2.h[7]\n"
+ "fmla z11.h, z7.h, z0.h[7]\n"
+ "fmla z15.h, z7.h, z1.h[7]\n"
+ "fmla z19.h, z7.h, z2.h[7]\n"
+ "40:" // Height 3: Multiply loop: multiply skip
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x12, x12, #0x1\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "cmp x12, x19\n"
+ "bne 35b\n"
+ "prfm pstl1keep, [x13, #0x0]\n"
+ "prfm pstl1keep, [x9, #0x0]\n"
+ "prfm pstl1keep, [x27, #0x0]\n"
+ "tbz %x[flags], #1, 41f\n"
+ "add x19, %x[args_ptr], %[offset_min]\n"
+ "ld1rh { z1.h }, p5/Z, [x19]\n"
+ "add x19, %x[args_ptr], %[offset_max]\n"
+ "ld1rh { z0.h }, p5/Z, [x19]\n"
+ "fmin z8.h, p5/M, z8.h, z0.h\n"
+ "fmin z9.h, p5/M, z9.h, z0.h\n"
+ "fmin z10.h, p5/M, z10.h, z0.h\n"
+ "fmin z11.h, p5/M, z11.h, z0.h\n"
+ "fmin z12.h, p5/M, z12.h, z0.h\n"
+ "fmax z8.h, p5/M, z8.h, z1.h\n"
+ "fmax z9.h, p5/M, z9.h, z1.h\n"
+ "fmax z10.h, p5/M, z10.h, z1.h\n"
+ "fmax z11.h, p5/M, z11.h, z1.h\n"
+ "fmax z12.h, p5/M, z12.h, z1.h\n"
+ "fmin z13.h, p5/M, z13.h, z0.h\n"
+ "fmin z14.h, p5/M, z14.h, z0.h\n"
+ "fmin z15.h, p5/M, z15.h, z0.h\n"
+ "fmin z16.h, p5/M, z16.h, z0.h\n"
+ "fmax z13.h, p5/M, z13.h, z1.h\n"
+ "fmax z14.h, p5/M, z14.h, z1.h\n"
+ "fmax z15.h, p5/M, z15.h, z1.h\n"
+ "fmax z16.h, p5/M, z16.h, z1.h\n"
+ "fmin z17.h, p5/M, z17.h, z0.h\n"
+ "fmin z18.h, p5/M, z18.h, z0.h\n"
+ "fmin z19.h, p5/M, z19.h, z0.h\n"
+ "fmax z17.h, p5/M, z17.h, z1.h\n"
+ "fmax z18.h, p5/M, z18.h, z1.h\n"
+ "fmax z19.h, p5/M, z19.h, z1.h\n"
+ "41:" // Height 3: No activation
+ "st1h { z8.h }, p4, [x13]\n"
+ "st1h { z9.h }, p3, [x13, #1, MUL VL]\n"
+ "st1h { z10.h }, p2, [x13, #2, MUL VL]\n"
+ "st1h { z11.h }, p1, [x13, #3, MUL VL]\n"
+ "addvl x13, x13, #4\n"
+ "st1h { z12.h }, p4, [x9]\n"
+ "st1h { z13.h }, p3, [x9, #1, MUL VL]\n"
+ "st1h { z14.h }, p2, [x9, #2, MUL VL]\n"
+ "st1h { z15.h }, p1, [x9, #3, MUL VL]\n"
+ "addvl x9, x9, #4\n"
+ "st1h { z16.h }, p4, [x27]\n"
+ "st1h { z17.h }, p3, [x27, #1, MUL VL]\n"
+ "st1h { z18.h }, p2, [x27, #2, MUL VL]\n"
+ "st1h { z19.h }, p1, [x27, #3, MUL VL]\n"
+ "addvl x27, x27, #4\n"
+ "42:" // Height 3: Writeback done
+ "mov x19, #0x0\n"
+ "inch x19, ALL, MUL #4\n"
+ "subs x16, x16, x19\n"
+ "bgt 31b\n"
+ "b 86f\n"
+ "43:" // Height 4
+ "ldr x16, [%x[args_ptr], %[offsetof_N]]\n"
+ "mov x14, %x[bias]\n"
+ "ldr x15, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 44f\n"
+ "ldr x13, [%x[output_ptr], #0x0]\n"
+ "add x13, x13, x19, LSL #1\n"
+ "ldr x9, [%x[output_ptr], #0x8]\n"
+ "ldr x27, [%x[output_ptr], #0x10]\n"
+ "add x9, x9, x19, LSL #1\n"
+ "ldr x25, [%x[output_ptr], #0x18]\n"
+ "add x27, x27, x19, LSL #1\n"
+ "add x25, x25, x19, LSL #1\n"
+ "b 45f\n"
+ "44:" // Height 4: setup direct output
+ "mov x13, %x[output_ptr]\n"
+ "add x9, x13, x19, LSL #1\n"
+ "add x27, x9, x19, LSL #1\n"
+ "add x25, x27, x19, LSL #1\n"
+ "45:" // Height 4: Column loop
+ "mov x19, #0x0\n"
+ "whilelt p4.h, x19, x16\n"
+ "inch x19\n"
+ "whilelt p3.h, x19, x16\n"
+ "inch x19\n"
+ "whilelt p2.h, x19, x16\n"
+ "inch x19\n"
+ "whilelt p1.h, x19, x16\n"
+ "cbz x14, 46f\n"
+ "ld1h { z8.h }, p5/Z, [x14]\n"
+ "mov z12.d, z8.d\n"
+ "ld1h { z9.h }, p5/Z, [x14, #1, MUL VL]\n"
+ "mov z16.d, z8.d\n"
+ "ld1h { z10.h }, p5/Z, [x14, #2, MUL VL]\n"
+ "mov z20.d, z8.d\n"
+ "ld1h { z11.h }, p5/Z, [x14, #3, MUL VL]\n"
+ "addvl x14, x14, #4\n"
+ "mov z13.d, z9.d\n"
+ "mov z17.d, z9.d\n"
+ "mov z14.d, z10.d\n"
+ "mov z15.d, z11.d\n"
+ "mov z18.d, z10.d\n"
+ "mov z19.d, z11.d\n"
+ "mov z21.d, z9.d\n"
+ "mov z22.d, z10.d\n"
+ "mov z23.d, z11.d\n"
+ "b 48f\n"
+ "46:" // Height 4: no bias
+ "tbz %x[flags], #0, 47f\n"
+ "ld1h { z8.h }, p4/Z, [x13]\n"
+ "ld1h { z9.h }, p3/Z, [x13, #1, MUL VL]\n"
+ "ld1h { z10.h }, p2/Z, [x13, #2, MUL VL]\n"
+ "ld1h { z11.h }, p1/Z, [x13, #3, MUL VL]\n"
+ "ld1h { z12.h }, p4/Z, [x9]\n"
+ "ld1h { z13.h }, p3/Z, [x9, #1, MUL VL]\n"
+ "ld1h { z14.h }, p2/Z, [x9, #2, MUL VL]\n"
+ "ld1h { z15.h }, p1/Z, [x9, #3, MUL VL]\n"
+ "ld1h { z16.h }, p4/Z, [x27]\n"
+ "ld1h { z17.h }, p3/Z, [x27, #1, MUL VL]\n"
+ "ld1h { z18.h }, p2/Z, [x27, #2, MUL VL]\n"
+ "ld1h { z19.h }, p1/Z, [x27, #3, MUL VL]\n"
+ "ld1h { z20.h }, p4/Z, [x25]\n"
+ "ld1h { z21.h }, p3/Z, [x25, #1, MUL VL]\n"
+ "ld1h { z22.h }, p2/Z, [x25, #2, MUL VL]\n"
+ "ld1h { z23.h }, p1/Z, [x25, #3, MUL VL]\n"
+ "b 48f\n"
+ "47:" // Height 4: no accumulate
+ "mov z8.b, #0x0\n"
+ "mov z9.b, #0x0\n"
+ "mov z10.b, #0x0\n"
+ "mov z11.b, #0x0\n"
+ "mov z12.b, #0x0\n"
+ "mov z13.b, #0x0\n"
+ "mov z14.b, #0x0\n"
+ "mov z15.b, #0x0\n"
+ "mov z16.b, #0x0\n"
+ "mov z17.b, #0x0\n"
+ "mov z18.b, #0x0\n"
+ "mov z19.b, #0x0\n"
+ "mov z20.b, #0x0\n"
+ "mov z21.b, #0x0\n"
+ "mov z22.b, #0x0\n"
+ "mov z23.b, #0x0\n"
+ "48:" // Height 4: setup done
+ "mov x12, #0x0\n"
+ "49:" // Height 4: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w11, [x20, x12, LSL #0x2]\n"
+ "tbz %x[flags], #3, 50f\n"
+ "ldr x20, [%x[input_ptr], x12, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x10, [x20, #0x0]\n"
+ "ldr x28, [x20, #0x8]\n"
+ "ldr x26, [x20, #0x10]\n"
+ "ldr x24, [x20, #0x18]\n"
+ "cbnz x12, 51f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x10, x10, x19, LSL #1\n"
+ "add x28, x28, x19, LSL #1\n"
+ "add x26, x26, x19, LSL #1\n"
+ "add x24, x24, x19, LSL #1\n"
+ "b 51f\n"
+ "50:" // Height 4: setup direct input
+ "mov x10, %x[input_ptr]\n"
+ "add x28, x10, x19, LSL #1\n"
+ "add x26, x28, x19, LSL #1\n"
+ "add x24, x26, x19, LSL #1\n"
+ "51:" // Height 4: input setup done
+ "cmp x11, #0x8\n"
+ "ble 53f\n"
+ "52:" // Height 4: Multiply loop: Main loop head
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ "whilelt p0.h, XZR, x11\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ "sub x11, x11, #0x8\n"
+ "ld1rqh { z0.h }, p0/Z, [x10]\n"
+ "fmla z8.h, z6.h, z0.h[0]\n"
+ "ld1rqh { z1.h }, p0/Z, [x28]\n"
+ "add x10, x10, #0x10\n"
+ "fmla z9.h, z7.h, z0.h[0]\n"
+ "ld1rqh { z2.h }, p0/Z, [x26]\n"
+ "add x28, x28, #0x10\n"
+ "fmla z12.h, z6.h, z1.h[0]\n"
+ "ld1rqh { z3.h }, p0/Z, [x24]\n"
+ "add x26, x26, #0x10\n"
+ "fmla z16.h, z6.h, z2.h[0]\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x24, x24, #0x10\n"
+ "fmla z13.h, z7.h, z1.h[0]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "cmp x11, #0x8\n"
+ "fmla z20.h, z6.h, z3.h[0]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ "fmla z17.h, z7.h, z2.h[0]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "fmla z21.h, z7.h, z3.h[0]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "fmla z10.h, z6.h, z0.h[0]\n"
+ "fmla z14.h, z6.h, z1.h[0]\n"
+ "fmla z18.h, z6.h, z2.h[0]\n"
+ "fmla z22.h, z6.h, z3.h[0]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #4, MUL VL]\n"
+ "fmla z11.h, z7.h, z0.h[0]\n"
+ "fmla z15.h, z7.h, z1.h[0]\n"
+ "fmla z19.h, z7.h, z2.h[0]\n"
+ "fmla z23.h, z7.h, z3.h[0]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #5, MUL VL]\n"
+ "fmla z8.h, z6.h, z0.h[1]\n"
+ "fmla z12.h, z6.h, z1.h[1]\n"
+ "fmla z16.h, z6.h, z2.h[1]\n"
+ "fmla z20.h, z6.h, z3.h[1]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #6, MUL VL]\n"
+ "fmla z9.h, z7.h, z0.h[1]\n"
+ "fmla z13.h, z7.h, z1.h[1]\n"
+ "fmla z17.h, z7.h, z2.h[1]\n"
+ "fmla z21.h, z7.h, z3.h[1]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #7, MUL VL]\n"
+ "addvl x15, x15, #16\n"
+ "fmla z10.h, z6.h, z0.h[1]\n"
+ "fmla z14.h, z6.h, z1.h[1]\n"
+ "fmla z18.h, z6.h, z2.h[1]\n"
+ "fmla z22.h, z6.h, z3.h[1]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #-8, MUL VL]\n"
+ "fmla z11.h, z7.h, z0.h[1]\n"
+ "fmla z15.h, z7.h, z1.h[1]\n"
+ "fmla z19.h, z7.h, z2.h[1]\n"
+ "fmla z23.h, z7.h, z3.h[1]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #-7, MUL VL]\n"
+ "fmla z8.h, z6.h, z0.h[2]\n"
+ "fmla z12.h, z6.h, z1.h[2]\n"
+ "fmla z16.h, z6.h, z2.h[2]\n"
+ "fmla z20.h, z6.h, z3.h[2]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #-6, MUL VL]\n"
+ "fmla z9.h, z7.h, z0.h[2]\n"
+ "fmla z13.h, z7.h, z1.h[2]\n"
+ "fmla z17.h, z7.h, z2.h[2]\n"
+ "fmla z21.h, z7.h, z3.h[2]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #-5, MUL VL]\n"
+ "fmla z10.h, z6.h, z0.h[2]\n"
+ "fmla z14.h, z6.h, z1.h[2]\n"
+ "fmla z18.h, z6.h, z2.h[2]\n"
+ "fmla z22.h, z6.h, z3.h[2]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #-4, MUL VL]\n"
+ "fmla z11.h, z7.h, z0.h[2]\n"
+ "fmla z15.h, z7.h, z1.h[2]\n"
+ "fmla z19.h, z7.h, z2.h[2]\n"
+ "fmla z23.h, z7.h, z3.h[2]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #-3, MUL VL]\n"
+ "fmla z8.h, z6.h, z0.h[3]\n"
+ "fmla z12.h, z6.h, z1.h[3]\n"
+ "fmla z16.h, z6.h, z2.h[3]\n"
+ "fmla z20.h, z6.h, z3.h[3]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #-2, MUL VL]\n"
+ "fmla z9.h, z7.h, z0.h[3]\n"
+ "fmla z13.h, z7.h, z1.h[3]\n"
+ "fmla z17.h, z7.h, z2.h[3]\n"
+ "fmla z21.h, z7.h, z3.h[3]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #-1, MUL VL]\n"
+ "fmla z10.h, z6.h, z0.h[3]\n"
+ "fmla z14.h, z6.h, z1.h[3]\n"
+ "fmla z18.h, z6.h, z2.h[3]\n"
+ "fmla z22.h, z6.h, z3.h[3]\n"
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ "fmla z11.h, z7.h, z0.h[3]\n"
+ "fmla z15.h, z7.h, z1.h[3]\n"
+ "fmla z19.h, z7.h, z2.h[3]\n"
+ "fmla z23.h, z7.h, z3.h[3]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ "fmla z8.h, z6.h, z0.h[4]\n"
+ "fmla z12.h, z6.h, z1.h[4]\n"
+ "fmla z16.h, z6.h, z2.h[4]\n"
+ "fmla z20.h, z6.h, z3.h[4]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ "fmla z9.h, z7.h, z0.h[4]\n"
+ "fmla z13.h, z7.h, z1.h[4]\n"
+ "fmla z17.h, z7.h, z2.h[4]\n"
+ "fmla z21.h, z7.h, z3.h[4]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ "fmla z10.h, z6.h, z0.h[4]\n"
+ "fmla z14.h, z6.h, z1.h[4]\n"
+ "fmla z18.h, z6.h, z2.h[4]\n"
+ "fmla z22.h, z6.h, z3.h[4]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #4, MUL VL]\n"
+ "fmla z11.h, z7.h, z0.h[4]\n"
+ "fmla z15.h, z7.h, z1.h[4]\n"
+ "fmla z19.h, z7.h, z2.h[4]\n"
+ "fmla z23.h, z7.h, z3.h[4]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #5, MUL VL]\n"
+ "fmla z8.h, z6.h, z0.h[5]\n"
+ "fmla z12.h, z6.h, z1.h[5]\n"
+ "fmla z16.h, z6.h, z2.h[5]\n"
+ "fmla z20.h, z6.h, z3.h[5]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #6, MUL VL]\n"
+ "fmla z9.h, z7.h, z0.h[5]\n"
+ "fmla z13.h, z7.h, z1.h[5]\n"
+ "fmla z17.h, z7.h, z2.h[5]\n"
+ "fmla z21.h, z7.h, z3.h[5]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #7, MUL VL]\n"
+ "addvl x15, x15, #16\n"
+ "fmla z10.h, z6.h, z0.h[5]\n"
+ "fmla z14.h, z6.h, z1.h[5]\n"
+ "fmla z18.h, z6.h, z2.h[5]\n"
+ "fmla z22.h, z6.h, z3.h[5]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #-8, MUL VL]\n"
+ "fmla z11.h, z7.h, z0.h[5]\n"
+ "fmla z15.h, z7.h, z1.h[5]\n"
+ "fmla z19.h, z7.h, z2.h[5]\n"
+ "fmla z23.h, z7.h, z3.h[5]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #-7, MUL VL]\n"
+ "fmla z8.h, z6.h, z0.h[6]\n"
+ "fmla z12.h, z6.h, z1.h[6]\n"
+ "fmla z16.h, z6.h, z2.h[6]\n"
+ "fmla z20.h, z6.h, z3.h[6]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #-6, MUL VL]\n"
+ "fmla z9.h, z7.h, z0.h[6]\n"
+ "fmla z13.h, z7.h, z1.h[6]\n"
+ "fmla z17.h, z7.h, z2.h[6]\n"
+ "fmla z21.h, z7.h, z3.h[6]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #-5, MUL VL]\n"
+ "fmla z10.h, z6.h, z0.h[6]\n"
+ "fmla z14.h, z6.h, z1.h[6]\n"
+ "fmla z18.h, z6.h, z2.h[6]\n"
+ "fmla z22.h, z6.h, z3.h[6]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #-4, MUL VL]\n"
+ "fmla z11.h, z7.h, z0.h[6]\n"
+ "fmla z15.h, z7.h, z1.h[6]\n"
+ "fmla z19.h, z7.h, z2.h[6]\n"
+ "fmla z23.h, z7.h, z3.h[6]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #-3, MUL VL]\n"
+ "fmla z8.h, z6.h, z0.h[7]\n"
+ "fmla z12.h, z6.h, z1.h[7]\n"
+ "fmla z16.h, z6.h, z2.h[7]\n"
+ "fmla z20.h, z6.h, z3.h[7]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #-2, MUL VL]\n"
+ "fmla z9.h, z7.h, z0.h[7]\n"
+ "fmla z13.h, z7.h, z1.h[7]\n"
+ "fmla z17.h, z7.h, z2.h[7]\n"
+ "fmla z21.h, z7.h, z3.h[7]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #-1, MUL VL]\n"
+ "fmla z10.h, z6.h, z0.h[7]\n"
+ "fmla z14.h, z6.h, z1.h[7]\n"
+ "fmla z18.h, z6.h, z2.h[7]\n"
+ "fmla z22.h, z6.h, z3.h[7]\n"
+ "fmla z11.h, z7.h, z0.h[7]\n"
+ "fmla z15.h, z7.h, z1.h[7]\n"
+ "fmla z19.h, z7.h, z2.h[7]\n"
+ "fmla z23.h, z7.h, z3.h[7]\n"
+ "bgt 52b\n"
+ "53:" // Height 4: Multiply loop: Single iteration only
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ "whilelt p0.h, XZR, x11\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ "subs x11, x11, #0x1\n"
+ "ld1rqh { z0.h }, p0/Z, [x10]\n"
+ "fmla z8.h, z6.h, z0.h[0]\n"
+ "ld1rqh { z1.h }, p0/Z, [x28]\n"
+ "add x10, x10, #0x10\n"
+ "fmla z9.h, z7.h, z0.h[0]\n"
+ "ld1rqh { z2.h }, p0/Z, [x26]\n"
+ "add x28, x28, #0x10\n"
+ "fmla z12.h, z6.h, z1.h[0]\n"
+ "ld1rqh { z3.h }, p0/Z, [x24]\n"
+ "add x26, x26, #0x10\n"
+ "fmla z16.h, z6.h, z2.h[0]\n"
+ "add x24, x24, #0x10\n"
+ "fmla z13.h, z7.h, z1.h[0]\n"
+ "fmla z17.h, z7.h, z2.h[0]\n"
+ "fmla z20.h, z6.h, z3.h[0]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ "fmla z21.h, z7.h, z3.h[0]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ "addvl x15, x15, #4\n"
+ "fmla z10.h, z6.h, z0.h[0]\n"
+ "fmla z14.h, z6.h, z1.h[0]\n"
+ "fmla z18.h, z6.h, z2.h[0]\n"
+ "fmla z22.h, z6.h, z3.h[0]\n"
+ "fmla z11.h, z7.h, z0.h[0]\n"
+ "fmla z15.h, z7.h, z1.h[0]\n"
+ "fmla z19.h, z7.h, z2.h[0]\n"
+ "fmla z23.h, z7.h, z3.h[0]\n"
+ "ble 54f\n"
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ "fmla z8.h, z6.h, z0.h[1]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ "subs x11, x11, #0x1\n"
+ "fmla z12.h, z6.h, z1.h[1]\n"
+ "fmla z16.h, z6.h, z2.h[1]\n"
+ "fmla z20.h, z6.h, z3.h[1]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ "fmla z9.h, z7.h, z0.h[1]\n"
+ "fmla z13.h, z7.h, z1.h[1]\n"
+ "fmla z17.h, z7.h, z2.h[1]\n"
+ "fmla z21.h, z7.h, z3.h[1]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ "addvl x15, x15, #4\n"
+ "fmla z10.h, z6.h, z0.h[1]\n"
+ "fmla z14.h, z6.h, z1.h[1]\n"
+ "fmla z18.h, z6.h, z2.h[1]\n"
+ "fmla z22.h, z6.h, z3.h[1]\n"
+ "fmla z11.h, z7.h, z0.h[1]\n"
+ "fmla z15.h, z7.h, z1.h[1]\n"
+ "fmla z19.h, z7.h, z2.h[1]\n"
+ "fmla z23.h, z7.h, z3.h[1]\n"
+ "ble 54f\n"
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ "fmla z8.h, z6.h, z0.h[2]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ "subs x11, x11, #0x1\n"
+ "fmla z12.h, z6.h, z1.h[2]\n"
+ "fmla z16.h, z6.h, z2.h[2]\n"
+ "fmla z20.h, z6.h, z3.h[2]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ "fmla z9.h, z7.h, z0.h[2]\n"
+ "fmla z13.h, z7.h, z1.h[2]\n"
+ "fmla z17.h, z7.h, z2.h[2]\n"
+ "fmla z21.h, z7.h, z3.h[2]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ "addvl x15, x15, #4\n"
+ "fmla z10.h, z6.h, z0.h[2]\n"
+ "fmla z14.h, z6.h, z1.h[2]\n"
+ "fmla z18.h, z6.h, z2.h[2]\n"
+ "fmla z22.h, z6.h, z3.h[2]\n"
+ "fmla z11.h, z7.h, z0.h[2]\n"
+ "fmla z15.h, z7.h, z1.h[2]\n"
+ "fmla z19.h, z7.h, z2.h[2]\n"
+ "fmla z23.h, z7.h, z3.h[2]\n"
+ "ble 54f\n"
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ "fmla z8.h, z6.h, z0.h[3]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ "subs x11, x11, #0x1\n"
+ "fmla z12.h, z6.h, z1.h[3]\n"
+ "fmla z16.h, z6.h, z2.h[3]\n"
+ "fmla z20.h, z6.h, z3.h[3]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ "fmla z9.h, z7.h, z0.h[3]\n"
+ "fmla z13.h, z7.h, z1.h[3]\n"
+ "fmla z17.h, z7.h, z2.h[3]\n"
+ "fmla z21.h, z7.h, z3.h[3]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ "addvl x15, x15, #4\n"
+ "fmla z10.h, z6.h, z0.h[3]\n"
+ "fmla z14.h, z6.h, z1.h[3]\n"
+ "fmla z18.h, z6.h, z2.h[3]\n"
+ "fmla z22.h, z6.h, z3.h[3]\n"
+ "fmla z11.h, z7.h, z0.h[3]\n"
+ "fmla z15.h, z7.h, z1.h[3]\n"
+ "fmla z19.h, z7.h, z2.h[3]\n"
+ "fmla z23.h, z7.h, z3.h[3]\n"
+ "ble 54f\n"
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ "fmla z8.h, z6.h, z0.h[4]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ "subs x11, x11, #0x1\n"
+ "fmla z12.h, z6.h, z1.h[4]\n"
+ "fmla z16.h, z6.h, z2.h[4]\n"
+ "fmla z20.h, z6.h, z3.h[4]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ "fmla z9.h, z7.h, z0.h[4]\n"
+ "fmla z13.h, z7.h, z1.h[4]\n"
+ "fmla z17.h, z7.h, z2.h[4]\n"
+ "fmla z21.h, z7.h, z3.h[4]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ "addvl x15, x15, #4\n"
+ "fmla z10.h, z6.h, z0.h[4]\n"
+ "fmla z14.h, z6.h, z1.h[4]\n"
+ "fmla z18.h, z6.h, z2.h[4]\n"
+ "fmla z22.h, z6.h, z3.h[4]\n"
+ "fmla z11.h, z7.h, z0.h[4]\n"
+ "fmla z15.h, z7.h, z1.h[4]\n"
+ "fmla z19.h, z7.h, z2.h[4]\n"
+ "fmla z23.h, z7.h, z3.h[4]\n"
+ "ble 54f\n"
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ "fmla z8.h, z6.h, z0.h[5]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ "subs x11, x11, #0x1\n"
+ "fmla z12.h, z6.h, z1.h[5]\n"
+ "fmla z16.h, z6.h, z2.h[5]\n"
+ "fmla z20.h, z6.h, z3.h[5]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ "fmla z9.h, z7.h, z0.h[5]\n"
+ "fmla z13.h, z7.h, z1.h[5]\n"
+ "fmla z17.h, z7.h, z2.h[5]\n"
+ "fmla z21.h, z7.h, z3.h[5]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ "addvl x15, x15, #4\n"
+ "fmla z10.h, z6.h, z0.h[5]\n"
+ "fmla z14.h, z6.h, z1.h[5]\n"
+ "fmla z18.h, z6.h, z2.h[5]\n"
+ "fmla z22.h, z6.h, z3.h[5]\n"
+ "fmla z11.h, z7.h, z0.h[5]\n"
+ "fmla z15.h, z7.h, z1.h[5]\n"
+ "fmla z19.h, z7.h, z2.h[5]\n"
+ "fmla z23.h, z7.h, z3.h[5]\n"
+ "ble 54f\n"
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ "fmla z8.h, z6.h, z0.h[6]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ "subs x11, x11, #0x1\n"
+ "fmla z12.h, z6.h, z1.h[6]\n"
+ "fmla z16.h, z6.h, z2.h[6]\n"
+ "fmla z20.h, z6.h, z3.h[6]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ "fmla z9.h, z7.h, z0.h[6]\n"
+ "fmla z13.h, z7.h, z1.h[6]\n"
+ "fmla z17.h, z7.h, z2.h[6]\n"
+ "fmla z21.h, z7.h, z3.h[6]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ "addvl x15, x15, #4\n"
+ "fmla z10.h, z6.h, z0.h[6]\n"
+ "fmla z14.h, z6.h, z1.h[6]\n"
+ "fmla z18.h, z6.h, z2.h[6]\n"
+ "fmla z22.h, z6.h, z3.h[6]\n"
+ "fmla z11.h, z7.h, z0.h[6]\n"
+ "fmla z15.h, z7.h, z1.h[6]\n"
+ "fmla z19.h, z7.h, z2.h[6]\n"
+ "fmla z23.h, z7.h, z3.h[6]\n"
+ "ble 54f\n"
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ "fmla z8.h, z6.h, z0.h[7]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ "fmla z12.h, z6.h, z1.h[7]\n"
+ "fmla z16.h, z6.h, z2.h[7]\n"
+ "fmla z20.h, z6.h, z3.h[7]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ "fmla z9.h, z7.h, z0.h[7]\n"
+ "fmla z13.h, z7.h, z1.h[7]\n"
+ "fmla z17.h, z7.h, z2.h[7]\n"
+ "fmla z21.h, z7.h, z3.h[7]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ "addvl x15, x15, #4\n"
+ "fmla z10.h, z6.h, z0.h[7]\n"
+ "fmla z14.h, z6.h, z1.h[7]\n"
+ "fmla z18.h, z6.h, z2.h[7]\n"
+ "fmla z22.h, z6.h, z3.h[7]\n"
+ "fmla z11.h, z7.h, z0.h[7]\n"
+ "fmla z15.h, z7.h, z1.h[7]\n"
+ "fmla z19.h, z7.h, z2.h[7]\n"
+ "fmla z23.h, z7.h, z3.h[7]\n"
+ "54:" // Height 4: Multiply loop: multiply skip
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x12, x12, #0x1\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "cmp x12, x19\n"
+ "bne 49b\n"
+ "prfm pstl1keep, [x13, #0x0]\n"
+ "prfm pstl1keep, [x9, #0x0]\n"
+ "prfm pstl1keep, [x27, #0x0]\n"
+ "prfm pstl1keep, [x25, #0x0]\n"
+ "tbz %x[flags], #1, 55f\n"
+ "add x19, %x[args_ptr], %[offset_min]\n"
+ "ld1rh { z1.h }, p5/Z, [x19]\n"
+ "add x19, %x[args_ptr], %[offset_max]\n"
+ "ld1rh { z0.h }, p5/Z, [x19]\n"
+ "fmin z8.h, p5/M, z8.h, z0.h\n"
+ "fmin z9.h, p5/M, z9.h, z0.h\n"
+ "fmin z10.h, p5/M, z10.h, z0.h\n"
+ "fmin z11.h, p5/M, z11.h, z0.h\n"
+ "fmin z12.h, p5/M, z12.h, z0.h\n"
+ "fmax z8.h, p5/M, z8.h, z1.h\n"
+ "fmax z9.h, p5/M, z9.h, z1.h\n"
+ "fmax z10.h, p5/M, z10.h, z1.h\n"
+ "fmax z11.h, p5/M, z11.h, z1.h\n"
+ "fmax z12.h, p5/M, z12.h, z1.h\n"
+ "fmin z13.h, p5/M, z13.h, z0.h\n"
+ "fmin z14.h, p5/M, z14.h, z0.h\n"
+ "fmin z15.h, p5/M, z15.h, z0.h\n"
+ "fmin z16.h, p5/M, z16.h, z0.h\n"
+ "fmax z13.h, p5/M, z13.h, z1.h\n"
+ "fmax z14.h, p5/M, z14.h, z1.h\n"
+ "fmax z15.h, p5/M, z15.h, z1.h\n"
+ "fmax z16.h, p5/M, z16.h, z1.h\n"
+ "fmin z17.h, p5/M, z17.h, z0.h\n"
+ "fmin z18.h, p5/M, z18.h, z0.h\n"
+ "fmin z19.h, p5/M, z19.h, z0.h\n"
+ "fmin z20.h, p5/M, z20.h, z0.h\n"
+ "fmax z17.h, p5/M, z17.h, z1.h\n"
+ "fmax z18.h, p5/M, z18.h, z1.h\n"
+ "fmax z19.h, p5/M, z19.h, z1.h\n"
+ "fmax z20.h, p5/M, z20.h, z1.h\n"
+ "fmin z21.h, p5/M, z21.h, z0.h\n"
+ "fmin z22.h, p5/M, z22.h, z0.h\n"
+ "fmin z23.h, p5/M, z23.h, z0.h\n"
+ "fmax z21.h, p5/M, z21.h, z1.h\n"
+ "fmax z22.h, p5/M, z22.h, z1.h\n"
+ "fmax z23.h, p5/M, z23.h, z1.h\n"
+ "55:" // Height 4: No activation
+ "st1h { z8.h }, p4, [x13]\n"
+ "st1h { z9.h }, p3, [x13, #1, MUL VL]\n"
+ "st1h { z10.h }, p2, [x13, #2, MUL VL]\n"
+ "st1h { z11.h }, p1, [x13, #3, MUL VL]\n"
+ "addvl x13, x13, #4\n"
+ "st1h { z12.h }, p4, [x9]\n"
+ "st1h { z13.h }, p3, [x9, #1, MUL VL]\n"
+ "st1h { z14.h }, p2, [x9, #2, MUL VL]\n"
+ "st1h { z15.h }, p1, [x9, #3, MUL VL]\n"
+ "addvl x9, x9, #4\n"
+ "st1h { z16.h }, p4, [x27]\n"
+ "st1h { z17.h }, p3, [x27, #1, MUL VL]\n"
+ "st1h { z18.h }, p2, [x27, #2, MUL VL]\n"
+ "st1h { z19.h }, p1, [x27, #3, MUL VL]\n"
+ "addvl x27, x27, #4\n"
+ "st1h { z20.h }, p4, [x25]\n"
+ "st1h { z21.h }, p3, [x25, #1, MUL VL]\n"
+ "st1h { z22.h }, p2, [x25, #2, MUL VL]\n"
+ "st1h { z23.h }, p1, [x25, #3, MUL VL]\n"
+ "addvl x25, x25, #4\n"
+ "56:" // Height 4: Writeback done
+ "mov x19, #0x0\n"
+ "inch x19, ALL, MUL #4\n"
+ "subs x16, x16, x19\n"
+ "bgt 45b\n"
+ "b 86f\n"
+ "57:" // Height 5
+ "ldr x16, [%x[args_ptr], %[offsetof_N]]\n"
+ "mov x14, %x[bias]\n"
+ "ldr x15, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 58f\n"
+ "ldr x13, [%x[output_ptr], #0x0]\n"
+ "add x13, x13, x19, LSL #1\n"
+ "ldr x9, [%x[output_ptr], #0x8]\n"
+ "ldr x27, [%x[output_ptr], #0x10]\n"
+ "add x9, x9, x19, LSL #1\n"
+ "ldr x25, [%x[output_ptr], #0x18]\n"
+ "ldr x23, [%x[output_ptr], #0x20]\n"
+ "add x27, x27, x19, LSL #1\n"
+ "add x25, x25, x19, LSL #1\n"
+ "add x23, x23, x19, LSL #1\n"
+ "b 59f\n"
+ "58:" // Height 5: setup direct output
+ "mov x13, %x[output_ptr]\n"
+ "add x9, x13, x19, LSL #1\n"
+ "add x27, x9, x19, LSL #1\n"
+ "add x25, x27, x19, LSL #1\n"
+ "add x23, x25, x19, LSL #1\n"
+ "59:" // Height 5: Column loop
+ "mov x19, #0x0\n"
+ "whilelt p4.h, x19, x16\n"
+ "inch x19\n"
+ "whilelt p3.h, x19, x16\n"
+ "inch x19\n"
+ "whilelt p2.h, x19, x16\n"
+ "inch x19\n"
+ "whilelt p1.h, x19, x16\n"
+ "cbz x14, 60f\n"
+ "ld1h { z8.h }, p5/Z, [x14]\n"
+ "mov z12.d, z8.d\n"
+ "ld1h { z9.h }, p5/Z, [x14, #1, MUL VL]\n"
+ "mov z16.d, z8.d\n"
+ "ld1h { z10.h }, p5/Z, [x14, #2, MUL VL]\n"
+ "mov z20.d, z8.d\n"
+ "ld1h { z11.h }, p5/Z, [x14, #3, MUL VL]\n"
+ "addvl x14, x14, #4\n"
+ "mov z13.d, z9.d\n"
+ "mov z17.d, z9.d\n"
+ "mov z14.d, z10.d\n"
+ "mov z15.d, z11.d\n"
+ "mov z18.d, z10.d\n"
+ "mov z19.d, z11.d\n"
+ "mov z21.d, z9.d\n"
+ "mov z22.d, z10.d\n"
+ "mov z23.d, z11.d\n"
+ "mov z24.d, z8.d\n"
+ "mov z25.d, z9.d\n"
+ "mov z26.d, z10.d\n"
+ "mov z27.d, z11.d\n"
+ "b 62f\n"
+ "60:" // Height 5: no bias
+ "tbz %x[flags], #0, 61f\n"
+ "ld1h { z8.h }, p4/Z, [x13]\n"
+ "ld1h { z9.h }, p3/Z, [x13, #1, MUL VL]\n"
+ "ld1h { z10.h }, p2/Z, [x13, #2, MUL VL]\n"
+ "ld1h { z11.h }, p1/Z, [x13, #3, MUL VL]\n"
+ "ld1h { z12.h }, p4/Z, [x9]\n"
+ "ld1h { z13.h }, p3/Z, [x9, #1, MUL VL]\n"
+ "ld1h { z14.h }, p2/Z, [x9, #2, MUL VL]\n"
+ "ld1h { z15.h }, p1/Z, [x9, #3, MUL VL]\n"
+ "ld1h { z16.h }, p4/Z, [x27]\n"
+ "ld1h { z17.h }, p3/Z, [x27, #1, MUL VL]\n"
+ "ld1h { z18.h }, p2/Z, [x27, #2, MUL VL]\n"
+ "ld1h { z19.h }, p1/Z, [x27, #3, MUL VL]\n"
+ "ld1h { z20.h }, p4/Z, [x25]\n"
+ "ld1h { z21.h }, p3/Z, [x25, #1, MUL VL]\n"
+ "ld1h { z22.h }, p2/Z, [x25, #2, MUL VL]\n"
+ "ld1h { z23.h }, p1/Z, [x25, #3, MUL VL]\n"
+ "ld1h { z24.h }, p4/Z, [x23]\n"
+ "ld1h { z25.h }, p3/Z, [x23, #1, MUL VL]\n"
+ "ld1h { z26.h }, p2/Z, [x23, #2, MUL VL]\n"
+ "ld1h { z27.h }, p1/Z, [x23, #3, MUL VL]\n"
+ "b 62f\n"
+ "61:" // Height 5: no accumulate
+ "mov z8.b, #0x0\n"
+ "mov z9.b, #0x0\n"
+ "mov z10.b, #0x0\n"
+ "mov z11.b, #0x0\n"
+ "mov z12.b, #0x0\n"
+ "mov z13.b, #0x0\n"
+ "mov z14.b, #0x0\n"
+ "mov z15.b, #0x0\n"
+ "mov z16.b, #0x0\n"
+ "mov z17.b, #0x0\n"
+ "mov z18.b, #0x0\n"
+ "mov z19.b, #0x0\n"
+ "mov z20.b, #0x0\n"
+ "mov z21.b, #0x0\n"
+ "mov z22.b, #0x0\n"
+ "mov z23.b, #0x0\n"
+ "mov z24.b, #0x0\n"
+ "mov z25.b, #0x0\n"
+ "mov z26.b, #0x0\n"
+ "mov z27.b, #0x0\n"
+ "62:" // Height 5: setup done
+ "mov x12, #0x0\n"
+ "63:" // Height 5: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w11, [x20, x12, LSL #0x2]\n"
+ "tbz %x[flags], #3, 64f\n"
+ "ldr x20, [%x[input_ptr], x12, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x10, [x20, #0x0]\n"
+ "ldr x28, [x20, #0x8]\n"
+ "ldr x26, [x20, #0x10]\n"
+ "ldr x24, [x20, #0x18]\n"
+ "ldr x22, [x20, #0x20]\n"
+ "cbnz x12, 65f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x10, x10, x19, LSL #1\n"
+ "add x28, x28, x19, LSL #1\n"
+ "add x26, x26, x19, LSL #1\n"
+ "add x24, x24, x19, LSL #1\n"
+ "add x22, x22, x19, LSL #1\n"
+ "b 65f\n"
+ "64:" // Height 5: setup direct input
+ "mov x10, %x[input_ptr]\n"
+ "add x28, x10, x19, LSL #1\n"
+ "add x26, x28, x19, LSL #1\n"
+ "add x24, x26, x19, LSL #1\n"
+ "add x22, x24, x19, LSL #1\n"
+ "65:" // Height 5: input setup done
+ "cmp x11, #0x8\n"
+ "ble 67f\n"
+ "66:" // Height 5: Multiply loop: Main loop head
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ "whilelt p0.h, XZR, x11\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ "sub x11, x11, #0x8\n"
+ "ld1rqh { z0.h }, p0/Z, [x10]\n"
+ "fmla z8.h, z6.h, z0.h[0]\n"
+ "ld1rqh { z1.h }, p0/Z, [x28]\n"
+ "add x10, x10, #0x10\n"
+ "fmla z9.h, z7.h, z0.h[0]\n"
+ "ld1rqh { z2.h }, p0/Z, [x26]\n"
+ "add x28, x28, #0x10\n"
+ "fmla z12.h, z6.h, z1.h[0]\n"
+ "ld1rqh { z3.h }, p0/Z, [x24]\n"
+ "add x26, x26, #0x10\n"
+ "fmla z16.h, z6.h, z2.h[0]\n"
+ "ld1rqh { z4.h }, p0/Z, [x22]\n"
+ "add x24, x24, #0x10\n"
+ "fmla z13.h, z7.h, z1.h[0]\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x22, x22, #0x10\n"
+ "fmla z20.h, z6.h, z3.h[0]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "cmp x11, #0x8\n"
+ "fmla z24.h, z6.h, z4.h[0]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ "fmla z17.h, z7.h, z2.h[0]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "fmla z21.h, z7.h, z3.h[0]\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "fmla z25.h, z7.h, z4.h[0]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ "fmla z10.h, z6.h, z0.h[0]\n"
+ "prfm pldl1keep, [x22, #0x80]\n"
+ "fmla z14.h, z6.h, z1.h[0]\n"
+ "fmla z18.h, z6.h, z2.h[0]\n"
+ "fmla z22.h, z6.h, z3.h[0]\n"
+ "fmla z26.h, z6.h, z4.h[0]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #4, MUL VL]\n"
+ "fmla z11.h, z7.h, z0.h[0]\n"
+ "fmla z15.h, z7.h, z1.h[0]\n"
+ "fmla z19.h, z7.h, z2.h[0]\n"
+ "fmla z23.h, z7.h, z3.h[0]\n"
+ "fmla z27.h, z7.h, z4.h[0]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #5, MUL VL]\n"
+ "fmla z8.h, z6.h, z0.h[1]\n"
+ "fmla z12.h, z6.h, z1.h[1]\n"
+ "fmla z16.h, z6.h, z2.h[1]\n"
+ "fmla z20.h, z6.h, z3.h[1]\n"
+ "fmla z24.h, z6.h, z4.h[1]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #6, MUL VL]\n"
+ "fmla z9.h, z7.h, z0.h[1]\n"
+ "fmla z13.h, z7.h, z1.h[1]\n"
+ "fmla z17.h, z7.h, z2.h[1]\n"
+ "fmla z21.h, z7.h, z3.h[1]\n"
+ "fmla z25.h, z7.h, z4.h[1]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #7, MUL VL]\n"
+ "addvl x15, x15, #16\n"
+ "fmla z10.h, z6.h, z0.h[1]\n"
+ "fmla z14.h, z6.h, z1.h[1]\n"
+ "fmla z18.h, z6.h, z2.h[1]\n"
+ "fmla z22.h, z6.h, z3.h[1]\n"
+ "fmla z26.h, z6.h, z4.h[1]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #-8, MUL VL]\n"
+ "fmla z11.h, z7.h, z0.h[1]\n"
+ "fmla z15.h, z7.h, z1.h[1]\n"
+ "fmla z19.h, z7.h, z2.h[1]\n"
+ "fmla z23.h, z7.h, z3.h[1]\n"
+ "fmla z27.h, z7.h, z4.h[1]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #-7, MUL VL]\n"
+ "fmla z8.h, z6.h, z0.h[2]\n"
+ "fmla z12.h, z6.h, z1.h[2]\n"
+ "fmla z16.h, z6.h, z2.h[2]\n"
+ "fmla z20.h, z6.h, z3.h[2]\n"
+ "fmla z24.h, z6.h, z4.h[2]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #-6, MUL VL]\n"
+ "fmla z9.h, z7.h, z0.h[2]\n"
+ "fmla z13.h, z7.h, z1.h[2]\n"
+ "fmla z17.h, z7.h, z2.h[2]\n"
+ "fmla z21.h, z7.h, z3.h[2]\n"
+ "fmla z25.h, z7.h, z4.h[2]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #-5, MUL VL]\n"
+ "fmla z10.h, z6.h, z0.h[2]\n"
+ "fmla z14.h, z6.h, z1.h[2]\n"
+ "fmla z18.h, z6.h, z2.h[2]\n"
+ "fmla z22.h, z6.h, z3.h[2]\n"
+ "fmla z26.h, z6.h, z4.h[2]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #-4, MUL VL]\n"
+ "fmla z11.h, z7.h, z0.h[2]\n"
+ "fmla z15.h, z7.h, z1.h[2]\n"
+ "fmla z19.h, z7.h, z2.h[2]\n"
+ "fmla z23.h, z7.h, z3.h[2]\n"
+ "fmla z27.h, z7.h, z4.h[2]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #-3, MUL VL]\n"
+ "fmla z8.h, z6.h, z0.h[3]\n"
+ "fmla z12.h, z6.h, z1.h[3]\n"
+ "fmla z16.h, z6.h, z2.h[3]\n"
+ "fmla z20.h, z6.h, z3.h[3]\n"
+ "fmla z24.h, z6.h, z4.h[3]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #-2, MUL VL]\n"
+ "fmla z9.h, z7.h, z0.h[3]\n"
+ "fmla z13.h, z7.h, z1.h[3]\n"
+ "fmla z17.h, z7.h, z2.h[3]\n"
+ "fmla z21.h, z7.h, z3.h[3]\n"
+ "fmla z25.h, z7.h, z4.h[3]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #-1, MUL VL]\n"
+ "fmla z10.h, z6.h, z0.h[3]\n"
+ "fmla z14.h, z6.h, z1.h[3]\n"
+ "fmla z18.h, z6.h, z2.h[3]\n"
+ "fmla z22.h, z6.h, z3.h[3]\n"
+ "fmla z26.h, z6.h, z4.h[3]\n"
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ "fmla z11.h, z7.h, z0.h[3]\n"
+ "fmla z15.h, z7.h, z1.h[3]\n"
+ "fmla z19.h, z7.h, z2.h[3]\n"
+ "fmla z23.h, z7.h, z3.h[3]\n"
+ "fmla z27.h, z7.h, z4.h[3]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ "fmla z8.h, z6.h, z0.h[4]\n"
+ "fmla z12.h, z6.h, z1.h[4]\n"
+ "fmla z16.h, z6.h, z2.h[4]\n"
+ "fmla z20.h, z6.h, z3.h[4]\n"
+ "fmla z24.h, z6.h, z4.h[4]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ "fmla z9.h, z7.h, z0.h[4]\n"
+ "fmla z13.h, z7.h, z1.h[4]\n"
+ "fmla z17.h, z7.h, z2.h[4]\n"
+ "fmla z21.h, z7.h, z3.h[4]\n"
+ "fmla z25.h, z7.h, z4.h[4]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ "fmla z10.h, z6.h, z0.h[4]\n"
+ "fmla z14.h, z6.h, z1.h[4]\n"
+ "fmla z18.h, z6.h, z2.h[4]\n"
+ "fmla z22.h, z6.h, z3.h[4]\n"
+ "fmla z26.h, z6.h, z4.h[4]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #4, MUL VL]\n"
+ "fmla z11.h, z7.h, z0.h[4]\n"
+ "fmla z15.h, z7.h, z1.h[4]\n"
+ "fmla z19.h, z7.h, z2.h[4]\n"
+ "fmla z23.h, z7.h, z3.h[4]\n"
+ "fmla z27.h, z7.h, z4.h[4]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #5, MUL VL]\n"
+ "fmla z8.h, z6.h, z0.h[5]\n"
+ "fmla z12.h, z6.h, z1.h[5]\n"
+ "fmla z16.h, z6.h, z2.h[5]\n"
+ "fmla z20.h, z6.h, z3.h[5]\n"
+ "fmla z24.h, z6.h, z4.h[5]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #6, MUL VL]\n"
+ "fmla z9.h, z7.h, z0.h[5]\n"
+ "fmla z13.h, z7.h, z1.h[5]\n"
+ "fmla z17.h, z7.h, z2.h[5]\n"
+ "fmla z21.h, z7.h, z3.h[5]\n"
+ "fmla z25.h, z7.h, z4.h[5]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #7, MUL VL]\n"
+ "addvl x15, x15, #16\n"
+ "fmla z10.h, z6.h, z0.h[5]\n"
+ "fmla z14.h, z6.h, z1.h[5]\n"
+ "fmla z18.h, z6.h, z2.h[5]\n"
+ "fmla z22.h, z6.h, z3.h[5]\n"
+ "fmla z26.h, z6.h, z4.h[5]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #-8, MUL VL]\n"
+ "fmla z11.h, z7.h, z0.h[5]\n"
+ "fmla z15.h, z7.h, z1.h[5]\n"
+ "fmla z19.h, z7.h, z2.h[5]\n"
+ "fmla z23.h, z7.h, z3.h[5]\n"
+ "fmla z27.h, z7.h, z4.h[5]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #-7, MUL VL]\n"
+ "fmla z8.h, z6.h, z0.h[6]\n"
+ "fmla z12.h, z6.h, z1.h[6]\n"
+ "fmla z16.h, z6.h, z2.h[6]\n"
+ "fmla z20.h, z6.h, z3.h[6]\n"
+ "fmla z24.h, z6.h, z4.h[6]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #-6, MUL VL]\n"
+ "fmla z9.h, z7.h, z0.h[6]\n"
+ "fmla z13.h, z7.h, z1.h[6]\n"
+ "fmla z17.h, z7.h, z2.h[6]\n"
+ "fmla z21.h, z7.h, z3.h[6]\n"
+ "fmla z25.h, z7.h, z4.h[6]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #-5, MUL VL]\n"
+ "fmla z10.h, z6.h, z0.h[6]\n"
+ "fmla z14.h, z6.h, z1.h[6]\n"
+ "fmla z18.h, z6.h, z2.h[6]\n"
+ "fmla z22.h, z6.h, z3.h[6]\n"
+ "fmla z26.h, z6.h, z4.h[6]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #-4, MUL VL]\n"
+ "fmla z11.h, z7.h, z0.h[6]\n"
+ "fmla z15.h, z7.h, z1.h[6]\n"
+ "fmla z19.h, z7.h, z2.h[6]\n"
+ "fmla z23.h, z7.h, z3.h[6]\n"
+ "fmla z27.h, z7.h, z4.h[6]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #-3, MUL VL]\n"
+ "fmla z8.h, z6.h, z0.h[7]\n"
+ "fmla z12.h, z6.h, z1.h[7]\n"
+ "fmla z16.h, z6.h, z2.h[7]\n"
+ "fmla z20.h, z6.h, z3.h[7]\n"
+ "fmla z24.h, z6.h, z4.h[7]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #-2, MUL VL]\n"
+ "fmla z9.h, z7.h, z0.h[7]\n"
+ "fmla z13.h, z7.h, z1.h[7]\n"
+ "fmla z17.h, z7.h, z2.h[7]\n"
+ "fmla z21.h, z7.h, z3.h[7]\n"
+ "fmla z25.h, z7.h, z4.h[7]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #-1, MUL VL]\n"
+ "fmla z10.h, z6.h, z0.h[7]\n"
+ "fmla z14.h, z6.h, z1.h[7]\n"
+ "fmla z18.h, z6.h, z2.h[7]\n"
+ "fmla z22.h, z6.h, z3.h[7]\n"
+ "fmla z26.h, z6.h, z4.h[7]\n"
+ "fmla z11.h, z7.h, z0.h[7]\n"
+ "fmla z15.h, z7.h, z1.h[7]\n"
+ "fmla z19.h, z7.h, z2.h[7]\n"
+ "fmla z23.h, z7.h, z3.h[7]\n"
+ "fmla z27.h, z7.h, z4.h[7]\n"
+ "bgt 66b\n"
+ "67:" // Height 5: Multiply loop: Single iteration only
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ "whilelt p0.h, XZR, x11\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ "subs x11, x11, #0x1\n"
+ "ld1rqh { z0.h }, p0/Z, [x10]\n"
+ "fmla z8.h, z6.h, z0.h[0]\n"
+ "ld1rqh { z1.h }, p0/Z, [x28]\n"
+ "add x10, x10, #0x10\n"
+ "fmla z9.h, z7.h, z0.h[0]\n"
+ "ld1rqh { z2.h }, p0/Z, [x26]\n"
+ "add x28, x28, #0x10\n"
+ "fmla z12.h, z6.h, z1.h[0]\n"
+ "ld1rqh { z3.h }, p0/Z, [x24]\n"
+ "add x26, x26, #0x10\n"
+ "fmla z16.h, z6.h, z2.h[0]\n"
+ "ld1rqh { z4.h }, p0/Z, [x22]\n"
+ "add x24, x24, #0x10\n"
+ "fmla z13.h, z7.h, z1.h[0]\n"
+ "add x22, x22, #0x10\n"
+ "fmla z17.h, z7.h, z2.h[0]\n"
+ "fmla z20.h, z6.h, z3.h[0]\n"
+ "fmla z24.h, z6.h, z4.h[0]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ "fmla z21.h, z7.h, z3.h[0]\n"
+ "fmla z25.h, z7.h, z4.h[0]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ "addvl x15, x15, #4\n"
+ "fmla z10.h, z6.h, z0.h[0]\n"
+ "fmla z14.h, z6.h, z1.h[0]\n"
+ "fmla z18.h, z6.h, z2.h[0]\n"
+ "fmla z22.h, z6.h, z3.h[0]\n"
+ "fmla z26.h, z6.h, z4.h[0]\n"
+ "fmla z11.h, z7.h, z0.h[0]\n"
+ "fmla z15.h, z7.h, z1.h[0]\n"
+ "fmla z19.h, z7.h, z2.h[0]\n"
+ "fmla z23.h, z7.h, z3.h[0]\n"
+ "fmla z27.h, z7.h, z4.h[0]\n"
+ "ble 68f\n"
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ "fmla z8.h, z6.h, z0.h[1]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ "subs x11, x11, #0x1\n"
+ "fmla z12.h, z6.h, z1.h[1]\n"
+ "fmla z16.h, z6.h, z2.h[1]\n"
+ "fmla z20.h, z6.h, z3.h[1]\n"
+ "fmla z24.h, z6.h, z4.h[1]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ "fmla z9.h, z7.h, z0.h[1]\n"
+ "fmla z13.h, z7.h, z1.h[1]\n"
+ "fmla z17.h, z7.h, z2.h[1]\n"
+ "fmla z21.h, z7.h, z3.h[1]\n"
+ "fmla z25.h, z7.h, z4.h[1]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ "addvl x15, x15, #4\n"
+ "fmla z10.h, z6.h, z0.h[1]\n"
+ "fmla z14.h, z6.h, z1.h[1]\n"
+ "fmla z18.h, z6.h, z2.h[1]\n"
+ "fmla z22.h, z6.h, z3.h[1]\n"
+ "fmla z26.h, z6.h, z4.h[1]\n"
+ "fmla z11.h, z7.h, z0.h[1]\n"
+ "fmla z15.h, z7.h, z1.h[1]\n"
+ "fmla z19.h, z7.h, z2.h[1]\n"
+ "fmla z23.h, z7.h, z3.h[1]\n"
+ "fmla z27.h, z7.h, z4.h[1]\n"
+ "ble 68f\n"
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ "fmla z8.h, z6.h, z0.h[2]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ "subs x11, x11, #0x1\n"
+ "fmla z12.h, z6.h, z1.h[2]\n"
+ "fmla z16.h, z6.h, z2.h[2]\n"
+ "fmla z20.h, z6.h, z3.h[2]\n"
+ "fmla z24.h, z6.h, z4.h[2]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ "fmla z9.h, z7.h, z0.h[2]\n"
+ "fmla z13.h, z7.h, z1.h[2]\n"
+ "fmla z17.h, z7.h, z2.h[2]\n"
+ "fmla z21.h, z7.h, z3.h[2]\n"
+ "fmla z25.h, z7.h, z4.h[2]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ "addvl x15, x15, #4\n"
+ "fmla z10.h, z6.h, z0.h[2]\n"
+ "fmla z14.h, z6.h, z1.h[2]\n"
+ "fmla z18.h, z6.h, z2.h[2]\n"
+ "fmla z22.h, z6.h, z3.h[2]\n"
+ "fmla z26.h, z6.h, z4.h[2]\n"
+ "fmla z11.h, z7.h, z0.h[2]\n"
+ "fmla z15.h, z7.h, z1.h[2]\n"
+ "fmla z19.h, z7.h, z2.h[2]\n"
+ "fmla z23.h, z7.h, z3.h[2]\n"
+ "fmla z27.h, z7.h, z4.h[2]\n"
+ "ble 68f\n"
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ "fmla z8.h, z6.h, z0.h[3]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ "subs x11, x11, #0x1\n"
+ "fmla z12.h, z6.h, z1.h[3]\n"
+ "fmla z16.h, z6.h, z2.h[3]\n"
+ "fmla z20.h, z6.h, z3.h[3]\n"
+ "fmla z24.h, z6.h, z4.h[3]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ "fmla z9.h, z7.h, z0.h[3]\n"
+ "fmla z13.h, z7.h, z1.h[3]\n"
+ "fmla z17.h, z7.h, z2.h[3]\n"
+ "fmla z21.h, z7.h, z3.h[3]\n"
+ "fmla z25.h, z7.h, z4.h[3]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ "addvl x15, x15, #4\n"
+ "fmla z10.h, z6.h, z0.h[3]\n"
+ "fmla z14.h, z6.h, z1.h[3]\n"
+ "fmla z18.h, z6.h, z2.h[3]\n"
+ "fmla z22.h, z6.h, z3.h[3]\n"
+ "fmla z26.h, z6.h, z4.h[3]\n"
+ "fmla z11.h, z7.h, z0.h[3]\n"
+ "fmla z15.h, z7.h, z1.h[3]\n"
+ "fmla z19.h, z7.h, z2.h[3]\n"
+ "fmla z23.h, z7.h, z3.h[3]\n"
+ "fmla z27.h, z7.h, z4.h[3]\n"
+ "ble 68f\n"
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ "fmla z8.h, z6.h, z0.h[4]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ "subs x11, x11, #0x1\n"
+ "fmla z12.h, z6.h, z1.h[4]\n"
+ "fmla z16.h, z6.h, z2.h[4]\n"
+ "fmla z20.h, z6.h, z3.h[4]\n"
+ "fmla z24.h, z6.h, z4.h[4]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ "fmla z9.h, z7.h, z0.h[4]\n"
+ "fmla z13.h, z7.h, z1.h[4]\n"
+ "fmla z17.h, z7.h, z2.h[4]\n"
+ "fmla z21.h, z7.h, z3.h[4]\n"
+ "fmla z25.h, z7.h, z4.h[4]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ "addvl x15, x15, #4\n"
+ "fmla z10.h, z6.h, z0.h[4]\n"
+ "fmla z14.h, z6.h, z1.h[4]\n"
+ "fmla z18.h, z6.h, z2.h[4]\n"
+ "fmla z22.h, z6.h, z3.h[4]\n"
+ "fmla z26.h, z6.h, z4.h[4]\n"
+ "fmla z11.h, z7.h, z0.h[4]\n"
+ "fmla z15.h, z7.h, z1.h[4]\n"
+ "fmla z19.h, z7.h, z2.h[4]\n"
+ "fmla z23.h, z7.h, z3.h[4]\n"
+ "fmla z27.h, z7.h, z4.h[4]\n"
+ "ble 68f\n"
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ "fmla z8.h, z6.h, z0.h[5]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ "subs x11, x11, #0x1\n"
+ "fmla z12.h, z6.h, z1.h[5]\n"
+ "fmla z16.h, z6.h, z2.h[5]\n"
+ "fmla z20.h, z6.h, z3.h[5]\n"
+ "fmla z24.h, z6.h, z4.h[5]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ "fmla z9.h, z7.h, z0.h[5]\n"
+ "fmla z13.h, z7.h, z1.h[5]\n"
+ "fmla z17.h, z7.h, z2.h[5]\n"
+ "fmla z21.h, z7.h, z3.h[5]\n"
+ "fmla z25.h, z7.h, z4.h[5]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ "addvl x15, x15, #4\n"
+ "fmla z10.h, z6.h, z0.h[5]\n"
+ "fmla z14.h, z6.h, z1.h[5]\n"
+ "fmla z18.h, z6.h, z2.h[5]\n"
+ "fmla z22.h, z6.h, z3.h[5]\n"
+ "fmla z26.h, z6.h, z4.h[5]\n"
+ "fmla z11.h, z7.h, z0.h[5]\n"
+ "fmla z15.h, z7.h, z1.h[5]\n"
+ "fmla z19.h, z7.h, z2.h[5]\n"
+ "fmla z23.h, z7.h, z3.h[5]\n"
+ "fmla z27.h, z7.h, z4.h[5]\n"
+ "ble 68f\n"
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ "fmla z8.h, z6.h, z0.h[6]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ "subs x11, x11, #0x1\n"
+ "fmla z12.h, z6.h, z1.h[6]\n"
+ "fmla z16.h, z6.h, z2.h[6]\n"
+ "fmla z20.h, z6.h, z3.h[6]\n"
+ "fmla z24.h, z6.h, z4.h[6]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ "fmla z9.h, z7.h, z0.h[6]\n"
+ "fmla z13.h, z7.h, z1.h[6]\n"
+ "fmla z17.h, z7.h, z2.h[6]\n"
+ "fmla z21.h, z7.h, z3.h[6]\n"
+ "fmla z25.h, z7.h, z4.h[6]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ "addvl x15, x15, #4\n"
+ "fmla z10.h, z6.h, z0.h[6]\n"
+ "fmla z14.h, z6.h, z1.h[6]\n"
+ "fmla z18.h, z6.h, z2.h[6]\n"
+ "fmla z22.h, z6.h, z3.h[6]\n"
+ "fmla z26.h, z6.h, z4.h[6]\n"
+ "fmla z11.h, z7.h, z0.h[6]\n"
+ "fmla z15.h, z7.h, z1.h[6]\n"
+ "fmla z19.h, z7.h, z2.h[6]\n"
+ "fmla z23.h, z7.h, z3.h[6]\n"
+ "fmla z27.h, z7.h, z4.h[6]\n"
+ "ble 68f\n"
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ "fmla z8.h, z6.h, z0.h[7]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ "fmla z12.h, z6.h, z1.h[7]\n"
+ "fmla z16.h, z6.h, z2.h[7]\n"
+ "fmla z20.h, z6.h, z3.h[7]\n"
+ "fmla z24.h, z6.h, z4.h[7]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ "fmla z9.h, z7.h, z0.h[7]\n"
+ "fmla z13.h, z7.h, z1.h[7]\n"
+ "fmla z17.h, z7.h, z2.h[7]\n"
+ "fmla z21.h, z7.h, z3.h[7]\n"
+ "fmla z25.h, z7.h, z4.h[7]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ "addvl x15, x15, #4\n"
+ "fmla z10.h, z6.h, z0.h[7]\n"
+ "fmla z14.h, z6.h, z1.h[7]\n"
+ "fmla z18.h, z6.h, z2.h[7]\n"
+ "fmla z22.h, z6.h, z3.h[7]\n"
+ "fmla z26.h, z6.h, z4.h[7]\n"
+ "fmla z11.h, z7.h, z0.h[7]\n"
+ "fmla z15.h, z7.h, z1.h[7]\n"
+ "fmla z19.h, z7.h, z2.h[7]\n"
+ "fmla z23.h, z7.h, z3.h[7]\n"
+ "fmla z27.h, z7.h, z4.h[7]\n"
+ "68:" // Height 5: Multiply loop: multiply skip
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x12, x12, #0x1\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "prfm pldl1keep, [x22, #0x80]\n"
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "cmp x12, x19\n"
+ "bne 63b\n"
+ "prfm pstl1keep, [x13, #0x0]\n"
+ "prfm pstl1keep, [x9, #0x0]\n"
+ "prfm pstl1keep, [x27, #0x0]\n"
+ "prfm pstl1keep, [x25, #0x0]\n"
+ "prfm pstl1keep, [x23, #0x0]\n"
+ "tbz %x[flags], #1, 69f\n"
+ "add x19, %x[args_ptr], %[offset_min]\n"
+ "ld1rh { z1.h }, p5/Z, [x19]\n"
+ "add x19, %x[args_ptr], %[offset_max]\n"
+ "ld1rh { z0.h }, p5/Z, [x19]\n"
+ "fmin z8.h, p5/M, z8.h, z0.h\n"
+ "fmin z9.h, p5/M, z9.h, z0.h\n"
+ "fmin z10.h, p5/M, z10.h, z0.h\n"
+ "fmin z11.h, p5/M, z11.h, z0.h\n"
+ "fmin z12.h, p5/M, z12.h, z0.h\n"
+ "fmax z8.h, p5/M, z8.h, z1.h\n"
+ "fmax z9.h, p5/M, z9.h, z1.h\n"
+ "fmax z10.h, p5/M, z10.h, z1.h\n"
+ "fmax z11.h, p5/M, z11.h, z1.h\n"
+ "fmax z12.h, p5/M, z12.h, z1.h\n"
+ "fmin z13.h, p5/M, z13.h, z0.h\n"
+ "fmin z14.h, p5/M, z14.h, z0.h\n"
+ "fmin z15.h, p5/M, z15.h, z0.h\n"
+ "fmin z16.h, p5/M, z16.h, z0.h\n"
+ "fmax z13.h, p5/M, z13.h, z1.h\n"
+ "fmax z14.h, p5/M, z14.h, z1.h\n"
+ "fmax z15.h, p5/M, z15.h, z1.h\n"
+ "fmax z16.h, p5/M, z16.h, z1.h\n"
+ "fmin z17.h, p5/M, z17.h, z0.h\n"
+ "fmin z18.h, p5/M, z18.h, z0.h\n"
+ "fmin z19.h, p5/M, z19.h, z0.h\n"
+ "fmin z20.h, p5/M, z20.h, z0.h\n"
+ "fmax z17.h, p5/M, z17.h, z1.h\n"
+ "fmax z18.h, p5/M, z18.h, z1.h\n"
+ "fmax z19.h, p5/M, z19.h, z1.h\n"
+ "fmax z20.h, p5/M, z20.h, z1.h\n"
+ "fmin z21.h, p5/M, z21.h, z0.h\n"
+ "fmin z22.h, p5/M, z22.h, z0.h\n"
+ "fmin z23.h, p5/M, z23.h, z0.h\n"
+ "fmin z24.h, p5/M, z24.h, z0.h\n"
+ "fmax z21.h, p5/M, z21.h, z1.h\n"
+ "fmax z22.h, p5/M, z22.h, z1.h\n"
+ "fmax z23.h, p5/M, z23.h, z1.h\n"
+ "fmax z24.h, p5/M, z24.h, z1.h\n"
+ "fmin z25.h, p5/M, z25.h, z0.h\n"
+ "fmin z26.h, p5/M, z26.h, z0.h\n"
+ "fmin z27.h, p5/M, z27.h, z0.h\n"
+ "fmax z25.h, p5/M, z25.h, z1.h\n"
+ "fmax z26.h, p5/M, z26.h, z1.h\n"
+ "fmax z27.h, p5/M, z27.h, z1.h\n"
+ "69:" // Height 5: No activation
+ "st1h { z8.h }, p4, [x13]\n"
+ "st1h { z9.h }, p3, [x13, #1, MUL VL]\n"
+ "st1h { z10.h }, p2, [x13, #2, MUL VL]\n"
+ "st1h { z11.h }, p1, [x13, #3, MUL VL]\n"
+ "addvl x13, x13, #4\n"
+ "st1h { z12.h }, p4, [x9]\n"
+ "st1h { z13.h }, p3, [x9, #1, MUL VL]\n"
+ "st1h { z14.h }, p2, [x9, #2, MUL VL]\n"
+ "st1h { z15.h }, p1, [x9, #3, MUL VL]\n"
+ "addvl x9, x9, #4\n"
+ "st1h { z16.h }, p4, [x27]\n"
+ "st1h { z17.h }, p3, [x27, #1, MUL VL]\n"
+ "st1h { z18.h }, p2, [x27, #2, MUL VL]\n"
+ "st1h { z19.h }, p1, [x27, #3, MUL VL]\n"
+ "addvl x27, x27, #4\n"
+ "st1h { z20.h }, p4, [x25]\n"
+ "st1h { z21.h }, p3, [x25, #1, MUL VL]\n"
+ "st1h { z22.h }, p2, [x25, #2, MUL VL]\n"
+ "st1h { z23.h }, p1, [x25, #3, MUL VL]\n"
+ "addvl x25, x25, #4\n"
+ "st1h { z24.h }, p4, [x23]\n"
+ "st1h { z25.h }, p3, [x23, #1, MUL VL]\n"
+ "st1h { z26.h }, p2, [x23, #2, MUL VL]\n"
+ "st1h { z27.h }, p1, [x23, #3, MUL VL]\n"
+ "addvl x23, x23, #4\n"
+ "70:" // Height 5: Writeback done
+ "mov x19, #0x0\n"
+ "inch x19, ALL, MUL #4\n"
+ "subs x16, x16, x19\n"
+ "bgt 59b\n"
+ "b 86f\n"
+ "71:" // Height 6
+ "ldr x16, [%x[args_ptr], %[offsetof_N]]\n"
+ "mov x14, %x[bias]\n"
+ "ldr x15, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 72f\n"
+ "ldr x13, [%x[output_ptr], #0x0]\n"
+ "add x13, x13, x19, LSL #1\n"
+ "ldr x9, [%x[output_ptr], #0x8]\n"
+ "ldr x27, [%x[output_ptr], #0x10]\n"
+ "add x9, x9, x19, LSL #1\n"
+ "ldr x25, [%x[output_ptr], #0x18]\n"
+ "ldr x23, [%x[output_ptr], #0x20]\n"
+ "add x27, x27, x19, LSL #1\n"
+ "ldr x21, [%x[output_ptr], #0x28]\n"
+ "add %x[output_ptr], %x[output_ptr], #0x30\n"
+ "add x25, x25, x19, LSL #1\n"
+ "add x23, x23, x19, LSL #1\n"
+ "add x21, x21, x19, LSL #1\n"
+ "b 73f\n"
+ "72:" // Height 6: setup direct output
+ "mov x13, %x[output_ptr]\n"
+ "add x9, x13, x19, LSL #1\n"
+ "add x27, x9, x19, LSL #1\n"
+ "add x25, x27, x19, LSL #1\n"
+ "add x23, x25, x19, LSL #1\n"
+ "add x21, x23, x19, LSL #1\n"
+ "add %x[output_ptr], x21, x19, LSL #1\n"
+ "73:" // Height 6: Column loop
+ "mov x19, #0x0\n"
+ "whilelt p4.h, x19, x16\n"
+ "inch x19\n"
+ "whilelt p3.h, x19, x16\n"
+ "inch x19\n"
+ "whilelt p2.h, x19, x16\n"
+ "inch x19\n"
+ "whilelt p1.h, x19, x16\n"
+ "cbz x14, 74f\n"
+ "ld1h { z8.h }, p5/Z, [x14]\n"
+ "mov z12.d, z8.d\n"
+ "ld1h { z9.h }, p5/Z, [x14, #1, MUL VL]\n"
+ "mov z16.d, z8.d\n"
+ "ld1h { z10.h }, p5/Z, [x14, #2, MUL VL]\n"
+ "mov z20.d, z8.d\n"
+ "ld1h { z11.h }, p5/Z, [x14, #3, MUL VL]\n"
+ "addvl x14, x14, #4\n"
+ "mov z13.d, z9.d\n"
+ "mov z17.d, z9.d\n"
+ "mov z14.d, z10.d\n"
+ "mov z15.d, z11.d\n"
+ "mov z18.d, z10.d\n"
+ "mov z19.d, z11.d\n"
+ "mov z21.d, z9.d\n"
+ "mov z22.d, z10.d\n"
+ "mov z23.d, z11.d\n"
+ "mov z24.d, z8.d\n"
+ "mov z25.d, z9.d\n"
+ "mov z26.d, z10.d\n"
+ "mov z27.d, z11.d\n"
+ "mov z28.d, z8.d\n"
+ "mov z29.d, z9.d\n"
+ "mov z30.d, z10.d\n"
+ "mov z31.d, z11.d\n"
+ "b 76f\n"
+ "74:" // Height 6: no bias
+ "tbz %x[flags], #0, 75f\n"
+ "ld1h { z8.h }, p4/Z, [x13]\n"
+ "ld1h { z9.h }, p3/Z, [x13, #1, MUL VL]\n"
+ "ld1h { z10.h }, p2/Z, [x13, #2, MUL VL]\n"
+ "ld1h { z11.h }, p1/Z, [x13, #3, MUL VL]\n"
+ "ld1h { z12.h }, p4/Z, [x9]\n"
+ "ld1h { z13.h }, p3/Z, [x9, #1, MUL VL]\n"
+ "ld1h { z14.h }, p2/Z, [x9, #2, MUL VL]\n"
+ "ld1h { z15.h }, p1/Z, [x9, #3, MUL VL]\n"
+ "ld1h { z16.h }, p4/Z, [x27]\n"
+ "ld1h { z17.h }, p3/Z, [x27, #1, MUL VL]\n"
+ "ld1h { z18.h }, p2/Z, [x27, #2, MUL VL]\n"
+ "ld1h { z19.h }, p1/Z, [x27, #3, MUL VL]\n"
+ "ld1h { z20.h }, p4/Z, [x25]\n"
+ "ld1h { z21.h }, p3/Z, [x25, #1, MUL VL]\n"
+ "ld1h { z22.h }, p2/Z, [x25, #2, MUL VL]\n"
+ "ld1h { z23.h }, p1/Z, [x25, #3, MUL VL]\n"
+ "ld1h { z24.h }, p4/Z, [x23]\n"
+ "ld1h { z25.h }, p3/Z, [x23, #1, MUL VL]\n"
+ "ld1h { z26.h }, p2/Z, [x23, #2, MUL VL]\n"
+ "ld1h { z27.h }, p1/Z, [x23, #3, MUL VL]\n"
+ "ld1h { z28.h }, p4/Z, [x21]\n"
+ "ld1h { z29.h }, p3/Z, [x21, #1, MUL VL]\n"
+ "ld1h { z30.h }, p2/Z, [x21, #2, MUL VL]\n"
+ "ld1h { z31.h }, p1/Z, [x21, #3, MUL VL]\n"
+ "b 76f\n"
+ "75:" // Height 6: no accumulate
+ "mov z8.b, #0x0\n"
+ "mov z9.b, #0x0\n"
+ "mov z10.b, #0x0\n"
+ "mov z11.b, #0x0\n"
+ "mov z12.b, #0x0\n"
+ "mov z13.b, #0x0\n"
+ "mov z14.b, #0x0\n"
+ "mov z15.b, #0x0\n"
+ "mov z16.b, #0x0\n"
+ "mov z17.b, #0x0\n"
+ "mov z18.b, #0x0\n"
+ "mov z19.b, #0x0\n"
+ "mov z20.b, #0x0\n"
+ "mov z21.b, #0x0\n"
+ "mov z22.b, #0x0\n"
+ "mov z23.b, #0x0\n"
+ "mov z24.b, #0x0\n"
+ "mov z25.b, #0x0\n"
+ "mov z26.b, #0x0\n"
+ "mov z27.b, #0x0\n"
+ "mov z28.b, #0x0\n"
+ "mov z29.b, #0x0\n"
+ "mov z30.b, #0x0\n"
+ "mov z31.b, #0x0\n"
+ "76:" // Height 6: setup done
+ "mov x12, #0x0\n"
+ "77:" // Height 6: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w11, [x20, x12, LSL #0x2]\n"
+ "tbz %x[flags], #3, 78f\n"
+ "ldr x20, [%x[input_ptr], x12, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x10, [x20, #0x0]\n"
+ "ldr x28, [x20, #0x8]\n"
+ "ldr x26, [x20, #0x10]\n"
+ "ldr x24, [x20, #0x18]\n"
+ "ldr x22, [x20, #0x20]\n"
+ "ldr x20, [x20, #0x28]\n"
+ "cbnz x12, 79f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x10, x10, x19, LSL #1\n"
+ "add x28, x28, x19, LSL #1\n"
+ "add x26, x26, x19, LSL #1\n"
+ "add x24, x24, x19, LSL #1\n"
+ "add x22, x22, x19, LSL #1\n"
+ "add x20, x20, x19, LSL #1\n"
+ "b 79f\n"
+ "78:" // Height 6: setup direct input
+ "mov x10, %x[input_ptr]\n"
+ "add x28, x10, x19, LSL #1\n"
+ "add x26, x28, x19, LSL #1\n"
+ "add x24, x26, x19, LSL #1\n"
+ "add x22, x24, x19, LSL #1\n"
+ "add x20, x22, x19, LSL #1\n"
+ "79:" // Height 6: input setup done
+ "cmp x11, #0x8\n"
+ "ble 81f\n"
+ "80:" // Height 6: Multiply loop: Main loop head
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ "whilelt p0.h, XZR, x11\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ "sub x11, x11, #0x8\n"
+ "ld1rqh { z0.h }, p0/Z, [x10]\n"
+ "fmla z8.h, z6.h, z0.h[0]\n"
+ "ld1rqh { z1.h }, p0/Z, [x28]\n"
+ "add x10, x10, #0x10\n"
+ "fmla z9.h, z7.h, z0.h[0]\n"
+ "ld1rqh { z2.h }, p0/Z, [x26]\n"
+ "add x28, x28, #0x10\n"
+ "fmla z12.h, z6.h, z1.h[0]\n"
+ "ld1rqh { z3.h }, p0/Z, [x24]\n"
+ "add x26, x26, #0x10\n"
+ "fmla z16.h, z6.h, z2.h[0]\n"
+ "ld1rqh { z4.h }, p0/Z, [x22]\n"
+ "add x24, x24, #0x10\n"
+ "fmla z13.h, z7.h, z1.h[0]\n"
+ "ld1rqh { z5.h }, p0/Z, [x20]\n"
+ "add x22, x22, #0x10\n"
+ "fmla z20.h, z6.h, z3.h[0]\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x20, x20, #0x10\n"
+ "fmla z24.h, z6.h, z4.h[0]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "cmp x11, #0x8\n"
+ "fmla z28.h, z6.h, z5.h[0]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ "fmla z17.h, z7.h, z2.h[0]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "fmla z21.h, z7.h, z3.h[0]\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "fmla z25.h, z7.h, z4.h[0]\n"
+ "prfm pldl1keep, [x22, #0x80]\n"
+ "fmla z29.h, z7.h, z5.h[0]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ "fmla z10.h, z6.h, z0.h[0]\n"
+ "prfm pldl1keep, [x20, #0x80]\n"
+ "fmla z14.h, z6.h, z1.h[0]\n"
+ "fmla z18.h, z6.h, z2.h[0]\n"
+ "fmla z22.h, z6.h, z3.h[0]\n"
+ "fmla z26.h, z6.h, z4.h[0]\n"
+ "fmla z30.h, z6.h, z5.h[0]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #4, MUL VL]\n"
+ "fmla z11.h, z7.h, z0.h[0]\n"
+ "fmla z15.h, z7.h, z1.h[0]\n"
+ "fmla z19.h, z7.h, z2.h[0]\n"
+ "fmla z23.h, z7.h, z3.h[0]\n"
+ "fmla z27.h, z7.h, z4.h[0]\n"
+ "fmla z31.h, z7.h, z5.h[0]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #5, MUL VL]\n"
+ "fmla z8.h, z6.h, z0.h[1]\n"
+ "fmla z12.h, z6.h, z1.h[1]\n"
+ "fmla z16.h, z6.h, z2.h[1]\n"
+ "fmla z20.h, z6.h, z3.h[1]\n"
+ "fmla z24.h, z6.h, z4.h[1]\n"
+ "fmla z28.h, z6.h, z5.h[1]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #6, MUL VL]\n"
+ "fmla z9.h, z7.h, z0.h[1]\n"
+ "fmla z13.h, z7.h, z1.h[1]\n"
+ "fmla z17.h, z7.h, z2.h[1]\n"
+ "fmla z21.h, z7.h, z3.h[1]\n"
+ "fmla z25.h, z7.h, z4.h[1]\n"
+ "fmla z29.h, z7.h, z5.h[1]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #7, MUL VL]\n"
+ "addvl x15, x15, #16\n"
+ "fmla z10.h, z6.h, z0.h[1]\n"
+ "fmla z14.h, z6.h, z1.h[1]\n"
+ "fmla z18.h, z6.h, z2.h[1]\n"
+ "fmla z22.h, z6.h, z3.h[1]\n"
+ "fmla z26.h, z6.h, z4.h[1]\n"
+ "fmla z30.h, z6.h, z5.h[1]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #-8, MUL VL]\n"
+ "fmla z11.h, z7.h, z0.h[1]\n"
+ "fmla z15.h, z7.h, z1.h[1]\n"
+ "fmla z19.h, z7.h, z2.h[1]\n"
+ "fmla z23.h, z7.h, z3.h[1]\n"
+ "fmla z27.h, z7.h, z4.h[1]\n"
+ "fmla z31.h, z7.h, z5.h[1]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #-7, MUL VL]\n"
+ "fmla z8.h, z6.h, z0.h[2]\n"
+ "fmla z12.h, z6.h, z1.h[2]\n"
+ "fmla z16.h, z6.h, z2.h[2]\n"
+ "fmla z20.h, z6.h, z3.h[2]\n"
+ "fmla z24.h, z6.h, z4.h[2]\n"
+ "fmla z28.h, z6.h, z5.h[2]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #-6, MUL VL]\n"
+ "fmla z9.h, z7.h, z0.h[2]\n"
+ "fmla z13.h, z7.h, z1.h[2]\n"
+ "fmla z17.h, z7.h, z2.h[2]\n"
+ "fmla z21.h, z7.h, z3.h[2]\n"
+ "fmla z25.h, z7.h, z4.h[2]\n"
+ "fmla z29.h, z7.h, z5.h[2]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #-5, MUL VL]\n"
+ "fmla z10.h, z6.h, z0.h[2]\n"
+ "fmla z14.h, z6.h, z1.h[2]\n"
+ "fmla z18.h, z6.h, z2.h[2]\n"
+ "fmla z22.h, z6.h, z3.h[2]\n"
+ "fmla z26.h, z6.h, z4.h[2]\n"
+ "fmla z30.h, z6.h, z5.h[2]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #-4, MUL VL]\n"
+ "fmla z11.h, z7.h, z0.h[2]\n"
+ "fmla z15.h, z7.h, z1.h[2]\n"
+ "fmla z19.h, z7.h, z2.h[2]\n"
+ "fmla z23.h, z7.h, z3.h[2]\n"
+ "fmla z27.h, z7.h, z4.h[2]\n"
+ "fmla z31.h, z7.h, z5.h[2]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #-3, MUL VL]\n"
+ "fmla z8.h, z6.h, z0.h[3]\n"
+ "fmla z12.h, z6.h, z1.h[3]\n"
+ "fmla z16.h, z6.h, z2.h[3]\n"
+ "fmla z20.h, z6.h, z3.h[3]\n"
+ "fmla z24.h, z6.h, z4.h[3]\n"
+ "fmla z28.h, z6.h, z5.h[3]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #-2, MUL VL]\n"
+ "fmla z9.h, z7.h, z0.h[3]\n"
+ "fmla z13.h, z7.h, z1.h[3]\n"
+ "fmla z17.h, z7.h, z2.h[3]\n"
+ "fmla z21.h, z7.h, z3.h[3]\n"
+ "fmla z25.h, z7.h, z4.h[3]\n"
+ "fmla z29.h, z7.h, z5.h[3]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #-1, MUL VL]\n"
+ "fmla z10.h, z6.h, z0.h[3]\n"
+ "fmla z14.h, z6.h, z1.h[3]\n"
+ "fmla z18.h, z6.h, z2.h[3]\n"
+ "fmla z22.h, z6.h, z3.h[3]\n"
+ "fmla z26.h, z6.h, z4.h[3]\n"
+ "fmla z30.h, z6.h, z5.h[3]\n"
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ "fmla z11.h, z7.h, z0.h[3]\n"
+ "fmla z15.h, z7.h, z1.h[3]\n"
+ "fmla z19.h, z7.h, z2.h[3]\n"
+ "fmla z23.h, z7.h, z3.h[3]\n"
+ "fmla z27.h, z7.h, z4.h[3]\n"
+ "fmla z31.h, z7.h, z5.h[3]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ "fmla z8.h, z6.h, z0.h[4]\n"
+ "fmla z12.h, z6.h, z1.h[4]\n"
+ "fmla z16.h, z6.h, z2.h[4]\n"
+ "fmla z20.h, z6.h, z3.h[4]\n"
+ "fmla z24.h, z6.h, z4.h[4]\n"
+ "fmla z28.h, z6.h, z5.h[4]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ "fmla z9.h, z7.h, z0.h[4]\n"
+ "fmla z13.h, z7.h, z1.h[4]\n"
+ "fmla z17.h, z7.h, z2.h[4]\n"
+ "fmla z21.h, z7.h, z3.h[4]\n"
+ "fmla z25.h, z7.h, z4.h[4]\n"
+ "fmla z29.h, z7.h, z5.h[4]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ "fmla z10.h, z6.h, z0.h[4]\n"
+ "fmla z14.h, z6.h, z1.h[4]\n"
+ "fmla z18.h, z6.h, z2.h[4]\n"
+ "fmla z22.h, z6.h, z3.h[4]\n"
+ "fmla z26.h, z6.h, z4.h[4]\n"
+ "fmla z30.h, z6.h, z5.h[4]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #4, MUL VL]\n"
+ "fmla z11.h, z7.h, z0.h[4]\n"
+ "fmla z15.h, z7.h, z1.h[4]\n"
+ "fmla z19.h, z7.h, z2.h[4]\n"
+ "fmla z23.h, z7.h, z3.h[4]\n"
+ "fmla z27.h, z7.h, z4.h[4]\n"
+ "fmla z31.h, z7.h, z5.h[4]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #5, MUL VL]\n"
+ "fmla z8.h, z6.h, z0.h[5]\n"
+ "fmla z12.h, z6.h, z1.h[5]\n"
+ "fmla z16.h, z6.h, z2.h[5]\n"
+ "fmla z20.h, z6.h, z3.h[5]\n"
+ "fmla z24.h, z6.h, z4.h[5]\n"
+ "fmla z28.h, z6.h, z5.h[5]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #6, MUL VL]\n"
+ "fmla z9.h, z7.h, z0.h[5]\n"
+ "fmla z13.h, z7.h, z1.h[5]\n"
+ "fmla z17.h, z7.h, z2.h[5]\n"
+ "fmla z21.h, z7.h, z3.h[5]\n"
+ "fmla z25.h, z7.h, z4.h[5]\n"
+ "fmla z29.h, z7.h, z5.h[5]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #7, MUL VL]\n"
+ "addvl x15, x15, #16\n"
+ "fmla z10.h, z6.h, z0.h[5]\n"
+ "fmla z14.h, z6.h, z1.h[5]\n"
+ "fmla z18.h, z6.h, z2.h[5]\n"
+ "fmla z22.h, z6.h, z3.h[5]\n"
+ "fmla z26.h, z6.h, z4.h[5]\n"
+ "fmla z30.h, z6.h, z5.h[5]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #-8, MUL VL]\n"
+ "fmla z11.h, z7.h, z0.h[5]\n"
+ "fmla z15.h, z7.h, z1.h[5]\n"
+ "fmla z19.h, z7.h, z2.h[5]\n"
+ "fmla z23.h, z7.h, z3.h[5]\n"
+ "fmla z27.h, z7.h, z4.h[5]\n"
+ "fmla z31.h, z7.h, z5.h[5]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #-7, MUL VL]\n"
+ "fmla z8.h, z6.h, z0.h[6]\n"
+ "fmla z12.h, z6.h, z1.h[6]\n"
+ "fmla z16.h, z6.h, z2.h[6]\n"
+ "fmla z20.h, z6.h, z3.h[6]\n"
+ "fmla z24.h, z6.h, z4.h[6]\n"
+ "fmla z28.h, z6.h, z5.h[6]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #-6, MUL VL]\n"
+ "fmla z9.h, z7.h, z0.h[6]\n"
+ "fmla z13.h, z7.h, z1.h[6]\n"
+ "fmla z17.h, z7.h, z2.h[6]\n"
+ "fmla z21.h, z7.h, z3.h[6]\n"
+ "fmla z25.h, z7.h, z4.h[6]\n"
+ "fmla z29.h, z7.h, z5.h[6]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #-5, MUL VL]\n"
+ "fmla z10.h, z6.h, z0.h[6]\n"
+ "fmla z14.h, z6.h, z1.h[6]\n"
+ "fmla z18.h, z6.h, z2.h[6]\n"
+ "fmla z22.h, z6.h, z3.h[6]\n"
+ "fmla z26.h, z6.h, z4.h[6]\n"
+ "fmla z30.h, z6.h, z5.h[6]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #-4, MUL VL]\n"
+ "fmla z11.h, z7.h, z0.h[6]\n"
+ "fmla z15.h, z7.h, z1.h[6]\n"
+ "fmla z19.h, z7.h, z2.h[6]\n"
+ "fmla z23.h, z7.h, z3.h[6]\n"
+ "fmla z27.h, z7.h, z4.h[6]\n"
+ "fmla z31.h, z7.h, z5.h[6]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #-3, MUL VL]\n"
+ "fmla z8.h, z6.h, z0.h[7]\n"
+ "fmla z12.h, z6.h, z1.h[7]\n"
+ "fmla z16.h, z6.h, z2.h[7]\n"
+ "fmla z20.h, z6.h, z3.h[7]\n"
+ "fmla z24.h, z6.h, z4.h[7]\n"
+ "fmla z28.h, z6.h, z5.h[7]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #-2, MUL VL]\n"
+ "fmla z9.h, z7.h, z0.h[7]\n"
+ "fmla z13.h, z7.h, z1.h[7]\n"
+ "fmla z17.h, z7.h, z2.h[7]\n"
+ "fmla z21.h, z7.h, z3.h[7]\n"
+ "fmla z25.h, z7.h, z4.h[7]\n"
+ "fmla z29.h, z7.h, z5.h[7]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #-1, MUL VL]\n"
+ "fmla z10.h, z6.h, z0.h[7]\n"
+ "fmla z14.h, z6.h, z1.h[7]\n"
+ "fmla z18.h, z6.h, z2.h[7]\n"
+ "fmla z22.h, z6.h, z3.h[7]\n"
+ "fmla z26.h, z6.h, z4.h[7]\n"
+ "fmla z30.h, z6.h, z5.h[7]\n"
+ "fmla z11.h, z7.h, z0.h[7]\n"
+ "fmla z15.h, z7.h, z1.h[7]\n"
+ "fmla z19.h, z7.h, z2.h[7]\n"
+ "fmla z23.h, z7.h, z3.h[7]\n"
+ "fmla z27.h, z7.h, z4.h[7]\n"
+ "fmla z31.h, z7.h, z5.h[7]\n"
+ "bgt 80b\n"
+ "81:" // Height 6: Multiply loop: Single iteration only
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ "whilelt p0.h, XZR, x11\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ "subs x11, x11, #0x1\n"
+ "ld1rqh { z0.h }, p0/Z, [x10]\n"
+ "fmla z8.h, z6.h, z0.h[0]\n"
+ "ld1rqh { z1.h }, p0/Z, [x28]\n"
+ "add x10, x10, #0x10\n"
+ "fmla z9.h, z7.h, z0.h[0]\n"
+ "ld1rqh { z2.h }, p0/Z, [x26]\n"
+ "add x28, x28, #0x10\n"
+ "fmla z12.h, z6.h, z1.h[0]\n"
+ "ld1rqh { z3.h }, p0/Z, [x24]\n"
+ "add x26, x26, #0x10\n"
+ "fmla z16.h, z6.h, z2.h[0]\n"
+ "ld1rqh { z4.h }, p0/Z, [x22]\n"
+ "add x24, x24, #0x10\n"
+ "fmla z13.h, z7.h, z1.h[0]\n"
+ "ld1rqh { z5.h }, p0/Z, [x20]\n"
+ "add x22, x22, #0x10\n"
+ "fmla z20.h, z6.h, z3.h[0]\n"
+ "add x20, x20, #0x10\n"
+ "fmla z17.h, z7.h, z2.h[0]\n"
+ "fmla z24.h, z6.h, z4.h[0]\n"
+ "fmla z28.h, z6.h, z5.h[0]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ "fmla z21.h, z7.h, z3.h[0]\n"
+ "fmla z25.h, z7.h, z4.h[0]\n"
+ "fmla z29.h, z7.h, z5.h[0]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ "addvl x15, x15, #4\n"
+ "fmla z10.h, z6.h, z0.h[0]\n"
+ "fmla z14.h, z6.h, z1.h[0]\n"
+ "fmla z18.h, z6.h, z2.h[0]\n"
+ "fmla z22.h, z6.h, z3.h[0]\n"
+ "fmla z26.h, z6.h, z4.h[0]\n"
+ "fmla z30.h, z6.h, z5.h[0]\n"
+ "fmla z11.h, z7.h, z0.h[0]\n"
+ "fmla z15.h, z7.h, z1.h[0]\n"
+ "fmla z19.h, z7.h, z2.h[0]\n"
+ "fmla z23.h, z7.h, z3.h[0]\n"
+ "fmla z27.h, z7.h, z4.h[0]\n"
+ "fmla z31.h, z7.h, z5.h[0]\n"
+ "ble 82f\n"
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ "fmla z8.h, z6.h, z0.h[1]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ "subs x11, x11, #0x1\n"
+ "fmla z12.h, z6.h, z1.h[1]\n"
+ "fmla z16.h, z6.h, z2.h[1]\n"
+ "fmla z20.h, z6.h, z3.h[1]\n"
+ "fmla z24.h, z6.h, z4.h[1]\n"
+ "fmla z28.h, z6.h, z5.h[1]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ "fmla z9.h, z7.h, z0.h[1]\n"
+ "fmla z13.h, z7.h, z1.h[1]\n"
+ "fmla z17.h, z7.h, z2.h[1]\n"
+ "fmla z21.h, z7.h, z3.h[1]\n"
+ "fmla z25.h, z7.h, z4.h[1]\n"
+ "fmla z29.h, z7.h, z5.h[1]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ "addvl x15, x15, #4\n"
+ "fmla z10.h, z6.h, z0.h[1]\n"
+ "fmla z14.h, z6.h, z1.h[1]\n"
+ "fmla z18.h, z6.h, z2.h[1]\n"
+ "fmla z22.h, z6.h, z3.h[1]\n"
+ "fmla z26.h, z6.h, z4.h[1]\n"
+ "fmla z30.h, z6.h, z5.h[1]\n"
+ "fmla z11.h, z7.h, z0.h[1]\n"
+ "fmla z15.h, z7.h, z1.h[1]\n"
+ "fmla z19.h, z7.h, z2.h[1]\n"
+ "fmla z23.h, z7.h, z3.h[1]\n"
+ "fmla z27.h, z7.h, z4.h[1]\n"
+ "fmla z31.h, z7.h, z5.h[1]\n"
+ "ble 82f\n"
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ "fmla z8.h, z6.h, z0.h[2]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ "subs x11, x11, #0x1\n"
+ "fmla z12.h, z6.h, z1.h[2]\n"
+ "fmla z16.h, z6.h, z2.h[2]\n"
+ "fmla z20.h, z6.h, z3.h[2]\n"
+ "fmla z24.h, z6.h, z4.h[2]\n"
+ "fmla z28.h, z6.h, z5.h[2]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ "fmla z9.h, z7.h, z0.h[2]\n"
+ "fmla z13.h, z7.h, z1.h[2]\n"
+ "fmla z17.h, z7.h, z2.h[2]\n"
+ "fmla z21.h, z7.h, z3.h[2]\n"
+ "fmla z25.h, z7.h, z4.h[2]\n"
+ "fmla z29.h, z7.h, z5.h[2]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ "addvl x15, x15, #4\n"
+ "fmla z10.h, z6.h, z0.h[2]\n"
+ "fmla z14.h, z6.h, z1.h[2]\n"
+ "fmla z18.h, z6.h, z2.h[2]\n"
+ "fmla z22.h, z6.h, z3.h[2]\n"
+ "fmla z26.h, z6.h, z4.h[2]\n"
+ "fmla z30.h, z6.h, z5.h[2]\n"
+ "fmla z11.h, z7.h, z0.h[2]\n"
+ "fmla z15.h, z7.h, z1.h[2]\n"
+ "fmla z19.h, z7.h, z2.h[2]\n"
+ "fmla z23.h, z7.h, z3.h[2]\n"
+ "fmla z27.h, z7.h, z4.h[2]\n"
+ "fmla z31.h, z7.h, z5.h[2]\n"
+ "ble 82f\n"
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ "fmla z8.h, z6.h, z0.h[3]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ "subs x11, x11, #0x1\n"
+ "fmla z12.h, z6.h, z1.h[3]\n"
+ "fmla z16.h, z6.h, z2.h[3]\n"
+ "fmla z20.h, z6.h, z3.h[3]\n"
+ "fmla z24.h, z6.h, z4.h[3]\n"
+ "fmla z28.h, z6.h, z5.h[3]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ "fmla z9.h, z7.h, z0.h[3]\n"
+ "fmla z13.h, z7.h, z1.h[3]\n"
+ "fmla z17.h, z7.h, z2.h[3]\n"
+ "fmla z21.h, z7.h, z3.h[3]\n"
+ "fmla z25.h, z7.h, z4.h[3]\n"
+ "fmla z29.h, z7.h, z5.h[3]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ "addvl x15, x15, #4\n"
+ "fmla z10.h, z6.h, z0.h[3]\n"
+ "fmla z14.h, z6.h, z1.h[3]\n"
+ "fmla z18.h, z6.h, z2.h[3]\n"
+ "fmla z22.h, z6.h, z3.h[3]\n"
+ "fmla z26.h, z6.h, z4.h[3]\n"
+ "fmla z30.h, z6.h, z5.h[3]\n"
+ "fmla z11.h, z7.h, z0.h[3]\n"
+ "fmla z15.h, z7.h, z1.h[3]\n"
+ "fmla z19.h, z7.h, z2.h[3]\n"
+ "fmla z23.h, z7.h, z3.h[3]\n"
+ "fmla z27.h, z7.h, z4.h[3]\n"
+ "fmla z31.h, z7.h, z5.h[3]\n"
+ "ble 82f\n"
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ "fmla z8.h, z6.h, z0.h[4]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ "subs x11, x11, #0x1\n"
+ "fmla z12.h, z6.h, z1.h[4]\n"
+ "fmla z16.h, z6.h, z2.h[4]\n"
+ "fmla z20.h, z6.h, z3.h[4]\n"
+ "fmla z24.h, z6.h, z4.h[4]\n"
+ "fmla z28.h, z6.h, z5.h[4]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ "fmla z9.h, z7.h, z0.h[4]\n"
+ "fmla z13.h, z7.h, z1.h[4]\n"
+ "fmla z17.h, z7.h, z2.h[4]\n"
+ "fmla z21.h, z7.h, z3.h[4]\n"
+ "fmla z25.h, z7.h, z4.h[4]\n"
+ "fmla z29.h, z7.h, z5.h[4]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ "addvl x15, x15, #4\n"
+ "fmla z10.h, z6.h, z0.h[4]\n"
+ "fmla z14.h, z6.h, z1.h[4]\n"
+ "fmla z18.h, z6.h, z2.h[4]\n"
+ "fmla z22.h, z6.h, z3.h[4]\n"
+ "fmla z26.h, z6.h, z4.h[4]\n"
+ "fmla z30.h, z6.h, z5.h[4]\n"
+ "fmla z11.h, z7.h, z0.h[4]\n"
+ "fmla z15.h, z7.h, z1.h[4]\n"
+ "fmla z19.h, z7.h, z2.h[4]\n"
+ "fmla z23.h, z7.h, z3.h[4]\n"
+ "fmla z27.h, z7.h, z4.h[4]\n"
+ "fmla z31.h, z7.h, z5.h[4]\n"
+ "ble 82f\n"
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ "fmla z8.h, z6.h, z0.h[5]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ "subs x11, x11, #0x1\n"
+ "fmla z12.h, z6.h, z1.h[5]\n"
+ "fmla z16.h, z6.h, z2.h[5]\n"
+ "fmla z20.h, z6.h, z3.h[5]\n"
+ "fmla z24.h, z6.h, z4.h[5]\n"
+ "fmla z28.h, z6.h, z5.h[5]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ "fmla z9.h, z7.h, z0.h[5]\n"
+ "fmla z13.h, z7.h, z1.h[5]\n"
+ "fmla z17.h, z7.h, z2.h[5]\n"
+ "fmla z21.h, z7.h, z3.h[5]\n"
+ "fmla z25.h, z7.h, z4.h[5]\n"
+ "fmla z29.h, z7.h, z5.h[5]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ "addvl x15, x15, #4\n"
+ "fmla z10.h, z6.h, z0.h[5]\n"
+ "fmla z14.h, z6.h, z1.h[5]\n"
+ "fmla z18.h, z6.h, z2.h[5]\n"
+ "fmla z22.h, z6.h, z3.h[5]\n"
+ "fmla z26.h, z6.h, z4.h[5]\n"
+ "fmla z30.h, z6.h, z5.h[5]\n"
+ "fmla z11.h, z7.h, z0.h[5]\n"
+ "fmla z15.h, z7.h, z1.h[5]\n"
+ "fmla z19.h, z7.h, z2.h[5]\n"
+ "fmla z23.h, z7.h, z3.h[5]\n"
+ "fmla z27.h, z7.h, z4.h[5]\n"
+ "fmla z31.h, z7.h, z5.h[5]\n"
+ "ble 82f\n"
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ "fmla z8.h, z6.h, z0.h[6]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ "subs x11, x11, #0x1\n"
+ "fmla z12.h, z6.h, z1.h[6]\n"
+ "fmla z16.h, z6.h, z2.h[6]\n"
+ "fmla z20.h, z6.h, z3.h[6]\n"
+ "fmla z24.h, z6.h, z4.h[6]\n"
+ "fmla z28.h, z6.h, z5.h[6]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ "fmla z9.h, z7.h, z0.h[6]\n"
+ "fmla z13.h, z7.h, z1.h[6]\n"
+ "fmla z17.h, z7.h, z2.h[6]\n"
+ "fmla z21.h, z7.h, z3.h[6]\n"
+ "fmla z25.h, z7.h, z4.h[6]\n"
+ "fmla z29.h, z7.h, z5.h[6]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ "addvl x15, x15, #4\n"
+ "fmla z10.h, z6.h, z0.h[6]\n"
+ "fmla z14.h, z6.h, z1.h[6]\n"
+ "fmla z18.h, z6.h, z2.h[6]\n"
+ "fmla z22.h, z6.h, z3.h[6]\n"
+ "fmla z26.h, z6.h, z4.h[6]\n"
+ "fmla z30.h, z6.h, z5.h[6]\n"
+ "fmla z11.h, z7.h, z0.h[6]\n"
+ "fmla z15.h, z7.h, z1.h[6]\n"
+ "fmla z19.h, z7.h, z2.h[6]\n"
+ "fmla z23.h, z7.h, z3.h[6]\n"
+ "fmla z27.h, z7.h, z4.h[6]\n"
+ "fmla z31.h, z7.h, z5.h[6]\n"
+ "ble 82f\n"
+ "ld1h { z6.h }, p5/Z, [x15]\n"
+ "fmla z8.h, z6.h, z0.h[7]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #1, MUL VL]\n"
+ "fmla z12.h, z6.h, z1.h[7]\n"
+ "fmla z16.h, z6.h, z2.h[7]\n"
+ "fmla z20.h, z6.h, z3.h[7]\n"
+ "fmla z24.h, z6.h, z4.h[7]\n"
+ "fmla z28.h, z6.h, z5.h[7]\n"
+ "ld1h { z6.h }, p5/Z, [x15, #2, MUL VL]\n"
+ "fmla z9.h, z7.h, z0.h[7]\n"
+ "fmla z13.h, z7.h, z1.h[7]\n"
+ "fmla z17.h, z7.h, z2.h[7]\n"
+ "fmla z21.h, z7.h, z3.h[7]\n"
+ "fmla z25.h, z7.h, z4.h[7]\n"
+ "fmla z29.h, z7.h, z5.h[7]\n"
+ "ld1h { z7.h }, p5/Z, [x15, #3, MUL VL]\n"
+ "addvl x15, x15, #4\n"
+ "fmla z10.h, z6.h, z0.h[7]\n"
+ "fmla z14.h, z6.h, z1.h[7]\n"
+ "fmla z18.h, z6.h, z2.h[7]\n"
+ "fmla z22.h, z6.h, z3.h[7]\n"
+ "fmla z26.h, z6.h, z4.h[7]\n"
+ "fmla z30.h, z6.h, z5.h[7]\n"
+ "fmla z11.h, z7.h, z0.h[7]\n"
+ "fmla z15.h, z7.h, z1.h[7]\n"
+ "fmla z19.h, z7.h, z2.h[7]\n"
+ "fmla z23.h, z7.h, z3.h[7]\n"
+ "fmla z27.h, z7.h, z4.h[7]\n"
+ "fmla z31.h, z7.h, z5.h[7]\n"
+ "82:" // Height 6: Multiply loop: multiply skip
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x12, x12, #0x1\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "prfm pldl1keep, [x22, #0x80]\n"
+ "prfm pldl1keep, [x20, #0x80]\n"
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "cmp x12, x19\n"
+ "bne 77b\n"
+ "prfm pstl1keep, [x13, #0x0]\n"
+ "prfm pstl1keep, [x9, #0x0]\n"
+ "prfm pstl1keep, [x27, #0x0]\n"
+ "prfm pstl1keep, [x25, #0x0]\n"
+ "prfm pstl1keep, [x23, #0x0]\n"
+ "prfm pstl1keep, [x21, #0x0]\n"
+ "tbz %x[flags], #1, 83f\n"
+ "add x19, %x[args_ptr], %[offset_min]\n"
+ "ld1rh { z1.h }, p5/Z, [x19]\n"
+ "add x19, %x[args_ptr], %[offset_max]\n"
+ "ld1rh { z0.h }, p5/Z, [x19]\n"
+ "fmin z8.h, p5/M, z8.h, z0.h\n"
+ "fmin z9.h, p5/M, z9.h, z0.h\n"
+ "fmin z10.h, p5/M, z10.h, z0.h\n"
+ "fmin z11.h, p5/M, z11.h, z0.h\n"
+ "fmin z12.h, p5/M, z12.h, z0.h\n"
+ "fmax z8.h, p5/M, z8.h, z1.h\n"
+ "fmax z9.h, p5/M, z9.h, z1.h\n"
+ "fmax z10.h, p5/M, z10.h, z1.h\n"
+ "fmax z11.h, p5/M, z11.h, z1.h\n"
+ "fmax z12.h, p5/M, z12.h, z1.h\n"
+ "fmin z13.h, p5/M, z13.h, z0.h\n"
+ "fmin z14.h, p5/M, z14.h, z0.h\n"
+ "fmin z15.h, p5/M, z15.h, z0.h\n"
+ "fmin z16.h, p5/M, z16.h, z0.h\n"
+ "fmax z13.h, p5/M, z13.h, z1.h\n"
+ "fmax z14.h, p5/M, z14.h, z1.h\n"
+ "fmax z15.h, p5/M, z15.h, z1.h\n"
+ "fmax z16.h, p5/M, z16.h, z1.h\n"
+ "fmin z17.h, p5/M, z17.h, z0.h\n"
+ "fmin z18.h, p5/M, z18.h, z0.h\n"
+ "fmin z19.h, p5/M, z19.h, z0.h\n"
+ "fmin z20.h, p5/M, z20.h, z0.h\n"
+ "fmax z17.h, p5/M, z17.h, z1.h\n"
+ "fmax z18.h, p5/M, z18.h, z1.h\n"
+ "fmax z19.h, p5/M, z19.h, z1.h\n"
+ "fmax z20.h, p5/M, z20.h, z1.h\n"
+ "fmin z21.h, p5/M, z21.h, z0.h\n"
+ "fmin z22.h, p5/M, z22.h, z0.h\n"
+ "fmin z23.h, p5/M, z23.h, z0.h\n"
+ "fmin z24.h, p5/M, z24.h, z0.h\n"
+ "fmax z21.h, p5/M, z21.h, z1.h\n"
+ "fmax z22.h, p5/M, z22.h, z1.h\n"
+ "fmax z23.h, p5/M, z23.h, z1.h\n"
+ "fmax z24.h, p5/M, z24.h, z1.h\n"
+ "fmin z25.h, p5/M, z25.h, z0.h\n"
+ "fmin z26.h, p5/M, z26.h, z0.h\n"
+ "fmin z27.h, p5/M, z27.h, z0.h\n"
+ "fmin z28.h, p5/M, z28.h, z0.h\n"
+ "fmax z25.h, p5/M, z25.h, z1.h\n"
+ "fmax z26.h, p5/M, z26.h, z1.h\n"
+ "fmax z27.h, p5/M, z27.h, z1.h\n"
+ "fmax z28.h, p5/M, z28.h, z1.h\n"
+ "fmin z29.h, p5/M, z29.h, z0.h\n"
+ "fmin z30.h, p5/M, z30.h, z0.h\n"
+ "fmin z31.h, p5/M, z31.h, z0.h\n"
+ "fmax z29.h, p5/M, z29.h, z1.h\n"
+ "fmax z30.h, p5/M, z30.h, z1.h\n"
+ "fmax z31.h, p5/M, z31.h, z1.h\n"
+ "83:" // Height 6: No activation
+ "st1h { z8.h }, p4, [x13]\n"
+ "st1h { z9.h }, p3, [x13, #1, MUL VL]\n"
+ "st1h { z10.h }, p2, [x13, #2, MUL VL]\n"
+ "st1h { z11.h }, p1, [x13, #3, MUL VL]\n"
+ "addvl x13, x13, #4\n"
+ "st1h { z12.h }, p4, [x9]\n"
+ "st1h { z13.h }, p3, [x9, #1, MUL VL]\n"
+ "st1h { z14.h }, p2, [x9, #2, MUL VL]\n"
+ "st1h { z15.h }, p1, [x9, #3, MUL VL]\n"
+ "addvl x9, x9, #4\n"
+ "st1h { z16.h }, p4, [x27]\n"
+ "st1h { z17.h }, p3, [x27, #1, MUL VL]\n"
+ "st1h { z18.h }, p2, [x27, #2, MUL VL]\n"
+ "st1h { z19.h }, p1, [x27, #3, MUL VL]\n"
+ "addvl x27, x27, #4\n"
+ "st1h { z20.h }, p4, [x25]\n"
+ "st1h { z21.h }, p3, [x25, #1, MUL VL]\n"
+ "st1h { z22.h }, p2, [x25, #2, MUL VL]\n"
+ "st1h { z23.h }, p1, [x25, #3, MUL VL]\n"
+ "addvl x25, x25, #4\n"
+ "st1h { z24.h }, p4, [x23]\n"
+ "st1h { z25.h }, p3, [x23, #1, MUL VL]\n"
+ "st1h { z26.h }, p2, [x23, #2, MUL VL]\n"
+ "st1h { z27.h }, p1, [x23, #3, MUL VL]\n"
+ "addvl x23, x23, #4\n"
+ "st1h { z28.h }, p4, [x21]\n"
+ "st1h { z29.h }, p3, [x21, #1, MUL VL]\n"
+ "st1h { z30.h }, p2, [x21, #2, MUL VL]\n"
+ "st1h { z31.h }, p1, [x21, #3, MUL VL]\n"
+ "addvl x21, x21, #4\n"
+ "84:" // Height 6: Writeback done
+ "mov x19, #0x0\n"
+ "inch x19, ALL, MUL #4\n"
+ "subs x16, x16, x19\n"
+ "bgt 73b\n"
+ "subs %x[M], %x[M], #0x6\n"
+ "beq 86f\n"
+ "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "tbz %x[flags], #3, 85f\n"
+ "add x20, x20, #0x6\n"
+ "str x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "b 1b\n"
+ "85:" // Update direct input
+ "mov x19, #0xc\n"
+ "madd %x[input_ptr], x19, x20, %x[input_ptr]\n"
+ "b 1b\n"
+ "86:" // Exit
+
+ : [M] "+r" (M), [input_ptr] "+r" (input_ptr), [output_ptr] "+r" (output_ptr)
+ : [args_ptr] "r" (&ka), [bias] "r" (bias), [flags] "r" (flags), [offset_max] "I" (offsetof(KernelArgs, maxval)), [offset_min] "I" (offsetof(KernelArgs, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths))
+ : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+ );
+}
+
+} // namespace arm_gemm
+#endif // __ARM_FEATURE_SVE
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_4VLx4/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_4VLx4/generic.cpp
deleted file mode 100644
index ce36243..0000000
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_4VLx4/generic.cpp
+++ /dev/null
@@ -1,2118 +0,0 @@
-/*
- * Copyright (c) 2018-2020 Arm Limited.
- *
- * SPDX-License-Identifier: MIT
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to
- * deal in the Software without restriction, including without limitation the
- * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
- * sell copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-#ifdef __ARM_FEATURE_SVE
-
-#include <algorithm>
-
-#include "arm_gemm.hpp"
-
-#include "../../asmlib.hpp"
-#include "../../utils.hpp"
-
-namespace arm_gemm {
-
-void sve_hybrid_fp32_mla_4VLx4(const float *A, int lda, const float *B, float *C, int ldc, int M, int N, int K, const float *bias, Activation act, bool accumulate) {
- const int K_stride = K;
- const long loops_count = ((K + 4) / 8) - 1;
- K -= loops_count * 8;
- const long regs_count = (K / 4) - 1;
- K -= (regs_count + 1) * 4;
- const long leftovers = K;
- float nullbias[256];
- if (!accumulate && !bias) {
- memset(nullbias, 0, (4 * get_vector_length<float>() * sizeof(float)));
- }
- float minval = - static_cast<float>(std::numeric_limits<float>::infinity());
- float maxval = static_cast<float>(std::numeric_limits<float>::infinity());
- const float * const minptr = &minval;
- const float * const maxptr = &maxval;
-
- switch(act.type)
- {
- default:
- case Activation::Type::None:
- break;
- case Activation::Type::BoundedReLU:
- maxval = static_cast<float>(act.param1);
- /* fall through */
- case Activation::Type::ReLU:
- minval = 0.0f;
- break;
- }
-
- int rows_to_compute;
-
- for (int y=0; y<M; y+=rows_to_compute) {
- const float * const a_ptr0_base = A + (y * lda);
- const unsigned long ldab = lda * sizeof(float);
-
- float *c_ptr0 = C + (y * ldc);
-
- rows_to_compute = M-y;
- if (rows_to_compute > 4) {
- if (rows_to_compute % 4) {
- rows_to_compute = 4 - 1;
- } else {
- rows_to_compute = 4;
- }
- }
-
- for (int x0=0; x0<N; x0+=(4 * get_vector_length<float>())) {
- const long width = std::min((unsigned long)N-x0, (4 * get_vector_length<float>()));
- long loops = loops_count;
- long regs = regs_count;
- long temp = 0;
- long blocks = leftovers;
- const float *a_ptr0 = a_ptr0_base;
- const float *b_ptr0 = B + (K_stride * x0);
- const unsigned long ldcb = ldc * sizeof(float);
- const float *biasptr = bias ? bias+x0 : nullbias;
-
- switch(rows_to_compute) {
- case 1:
- __asm __volatile (
- "whilelt p6.s, %[temp], %[leftovers]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "ptrue p7.s\n"
- "whilelt p1.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "whilelt p2.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "whilelt p3.s, %[temp], %[width]\n"
- "cbnz %[accumulate], 1f\n"
- "ld1w z16.s, p0/z, [%[biasptr]]\n"
- "ld1w z17.s, p1/z, [%[biasptr], #1, MUL VL]\n"
- "ld1w z18.s, p2/z, [%[biasptr], #2, MUL VL]\n"
- "ld1w z19.s, p3/z, [%[biasptr], #3, MUL VL]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "ld1w z16.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n"
- "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n"
- "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "cbz %[loops], 2f\n"
- "3:\n"
- "fmla z16.s, z8.s, z0.s[0]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "fmla z17.s, z9.s, z0.s[0]\n"
- "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n"
- "fmla z18.s, z10.s, z0.s[0]\n"
- "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
- "fmla z19.s, z11.s, z0.s[0]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z16.s, z12.s, z0.s[1]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z17.s, z13.s, z0.s[1]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z18.s, z14.s, z0.s[1]\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "fmla z19.s, z15.s, z0.s[1]\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "fmla z16.s, z8.s, z0.s[2]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z17.s, z9.s, z0.s[2]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z18.s, z10.s, z0.s[2]\n"
- "subs %[loops], %[loops], #0x1\n"
- "fmla z19.s, z11.s, z0.s[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "fmla z16.s, z12.s, z0.s[3]\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
- "fmla z17.s, z13.s, z0.s[3]\n"
- "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "fmla z18.s, z14.s, z0.s[3]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "fmla z19.s, z15.s, z0.s[3]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "fmla z16.s, z8.s, z4.s[0]\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "fmla z17.s, z9.s, z4.s[0]\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "fmla z18.s, z10.s, z4.s[0]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "fmla z19.s, z11.s, z4.s[0]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "fmla z16.s, z12.s, z4.s[1]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #-0x10]\n"
- "fmla z17.s, z13.s, z4.s[1]\n"
- "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
- "fmla z18.s, z14.s, z4.s[1]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z19.s, z15.s, z4.s[1]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z16.s, z8.s, z4.s[2]\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "fmla z17.s, z9.s, z4.s[2]\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "fmla z18.s, z10.s, z4.s[2]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z19.s, z11.s, z4.s[2]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z16.s, z12.s, z4.s[3]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "fmla z17.s, z13.s, z4.s[3]\n"
- "fmla z18.s, z14.s, z4.s[3]\n"
- "fmla z19.s, z15.s, z4.s[3]\n"
- "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "b.ne 3b\n"
- "2:\n"
- "cbz %[regs], 4f\n"
- "fmla z16.s, z8.s, z0.s[0]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "fmla z17.s, z9.s, z0.s[0]\n"
- "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n"
- "fmla z18.s, z10.s, z0.s[0]\n"
- "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
- "fmla z19.s, z11.s, z0.s[0]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z16.s, z12.s, z0.s[1]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z17.s, z13.s, z0.s[1]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z18.s, z14.s, z0.s[1]\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "fmla z19.s, z15.s, z0.s[1]\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "fmla z16.s, z8.s, z0.s[2]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z17.s, z9.s, z0.s[2]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z18.s, z10.s, z0.s[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "fmla z19.s, z11.s, z0.s[2]\n"
- "fmla z16.s, z12.s, z0.s[3]\n"
- "fmla z17.s, z13.s, z0.s[3]\n"
- "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "fmla z18.s, z14.s, z0.s[3]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "fmla z19.s, z15.s, z0.s[3]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "fmla z16.s, z8.s, z4.s[0]\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "fmla z17.s, z9.s, z4.s[0]\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "fmla z18.s, z10.s, z4.s[0]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "fmla z19.s, z11.s, z4.s[0]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "fmla z16.s, z12.s, z4.s[1]\n"
- "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
- "fmla z17.s, z13.s, z4.s[1]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z18.s, z14.s, z4.s[1]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z19.s, z15.s, z4.s[1]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z16.s, z8.s, z4.s[2]\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "fmla z17.s, z9.s, z4.s[2]\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "fmla z18.s, z10.s, z4.s[2]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z19.s, z11.s, z4.s[2]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z16.s, z12.s, z4.s[3]\n"
- "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x10]\n"
- "fmla z17.s, z13.s, z4.s[3]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "fmla z18.s, z14.s, z4.s[3]\n"
- "addvl %[a_ptr0], %[a_ptr0], #2\n"
- "fmla z19.s, z15.s, z4.s[3]\n"
- "cbz %[blocks], 5f\n"
- "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z16.s, z8.s, z0.s[0]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z17.s, z9.s, z0.s[0]\n"
- "fmla z18.s, z10.s, z0.s[0]\n"
- "fmla z19.s, z11.s, z0.s[0]\n"
- "b.eq 5f\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z16.s, z12.s, z0.s[1]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z17.s, z13.s, z0.s[1]\n"
- "fmla z18.s, z14.s, z0.s[1]\n"
- "fmla z19.s, z15.s, z0.s[1]\n"
- "b.eq 5f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "fmla z16.s, z8.s, z0.s[2]\n"
- "fmla z17.s, z9.s, z0.s[2]\n"
- "fmla z18.s, z10.s, z0.s[2]\n"
- "fmla z19.s, z11.s, z0.s[2]\n"
- "b 5f\n"
- "4:\n"
- "fmla z16.s, z8.s, z0.s[0]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "fmla z17.s, z9.s, z0.s[0]\n"
- "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
- "fmla z18.s, z10.s, z0.s[0]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z19.s, z11.s, z0.s[0]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z16.s, z12.s, z0.s[1]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z17.s, z13.s, z0.s[1]\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "fmla z18.s, z14.s, z0.s[1]\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "fmla z19.s, z15.s, z0.s[1]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z16.s, z8.s, z0.s[2]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z17.s, z9.s, z0.s[2]\n"
- "ld1rqw z4.s, p6/z, [%[a_ptr0]]\n"
- "fmla z18.s, z10.s, z0.s[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "fmla z19.s, z11.s, z0.s[2]\n"
- "addvl %[a_ptr0], %[a_ptr0], #1\n"
- "fmla z16.s, z12.s, z0.s[3]\n"
- "fmla z17.s, z13.s, z0.s[3]\n"
- "fmla z18.s, z14.s, z0.s[3]\n"
- "fmla z19.s, z15.s, z0.s[3]\n"
- "cbz %[blocks], 5f\n"
- "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z16.s, z8.s, z4.s[0]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z17.s, z9.s, z4.s[0]\n"
- "fmla z18.s, z10.s, z4.s[0]\n"
- "fmla z19.s, z11.s, z4.s[0]\n"
- "b.eq 5f\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z16.s, z12.s, z4.s[1]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z17.s, z13.s, z4.s[1]\n"
- "fmla z18.s, z14.s, z4.s[1]\n"
- "fmla z19.s, z15.s, z4.s[1]\n"
- "b.eq 5f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "fmla z16.s, z8.s, z4.s[2]\n"
- "fmla z17.s, z9.s, z4.s[2]\n"
- "fmla z18.s, z10.s, z4.s[2]\n"
- "fmla z19.s, z11.s, z4.s[2]\n"
- "5:\n"
- "ld1rw z14.s, p7/z, [%[minptr]]\n"
- "ld1rw z15.s, p7/z, [%[maxptr]]\n"
- "fmax z16.s, p7/m, z16.s, z14.s\n"
- "fmax z17.s, p7/m, z17.s, z14.s\n"
- "fmax z18.s, p7/m, z18.s, z14.s\n"
- "fmax z19.s, p7/m, z19.s, z14.s\n"
- "fmin z16.s, p7/m, z16.s, z15.s\n"
- "fmin z17.s, p7/m, z17.s, z15.s\n"
- "fmin z18.s, p7/m, z18.s, z15.s\n"
- "fmin z19.s, p7/m, z19.s, z15.s\n"
- "st1w z16.s, p0, [%[c_ptr0]]\n"
- "st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n"
- "st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n"
- "st1w z19.s, p3, [%[c_ptr0], #3, MUL VL]\n"
- "addvl %[c_ptr0], %[c_ptr0], #4\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks)
- : [width] "r" (width), [accumulate] "r" (static_cast<uint64_t>(accumulate)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers)
- : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
- );
- break;
- case 2:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "c_ptr1 .req X1\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "whilelt p6.s, %[temp], %[leftovers]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "ptrue p7.s\n"
- "whilelt p1.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "whilelt p2.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "whilelt p3.s, %[temp], %[width]\n"
- "cbnz %[accumulate], 1f\n"
- "ld1w z16.s, p0/z, [%[biasptr]]\n"
- "ld1w z17.s, p1/z, [%[biasptr], #1, MUL VL]\n"
- "ld1w z18.s, p2/z, [%[biasptr], #2, MUL VL]\n"
- "ld1w z19.s, p3/z, [%[biasptr], #3, MUL VL]\n"
- "mov z20.d, z16.d\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "mov z21.d, z17.d\n"
- "ld1rqw z1.s, p7/z, [a_ptr1]\n"
- "mov z22.d, z18.d\n"
- "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
- "mov z23.d, z19.d\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "ld1w z16.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n"
- "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n"
- "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n"
- "ld1w z20.s, p0/z, [c_ptr1]\n"
- "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n"
- "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n"
- "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ld1rqw z1.s, p7/z, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "cbz %[loops], 2f\n"
- "3:\n"
- "fmla z16.s, z8.s, z0.s[0]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "fmla z20.s, z8.s, z1.s[0]\n"
- "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n"
- "fmla z17.s, z9.s, z0.s[0]\n"
- "ld1rqw z5.s, p7/z, [a_ptr1]\n"
- "fmla z21.s, z9.s, z1.s[0]\n"
- "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
- "fmla z18.s, z10.s, z0.s[0]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z22.s, z10.s, z1.s[0]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z19.s, z11.s, z0.s[0]\n"
- "subs %[loops], %[loops], #0x1\n"
- "fmla z23.s, z11.s, z1.s[0]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z16.s, z12.s, z0.s[1]\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
- "fmla z20.s, z12.s, z1.s[1]\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "fmla z17.s, z13.s, z0.s[1]\n"
- "add a_ptr1, a_ptr1, #0x20\n"
- "fmla z21.s, z13.s, z1.s[1]\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "fmla z18.s, z14.s, z0.s[1]\n"
- "fmla z22.s, z14.s, z1.s[1]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z19.s, z15.s, z0.s[1]\n"
- "fmla z23.s, z15.s, z1.s[1]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z16.s, z8.s, z0.s[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "fmla z20.s, z8.s, z1.s[2]\n"
- "fmla z17.s, z9.s, z0.s[2]\n"
- "fmla z21.s, z9.s, z1.s[2]\n"
- "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "fmla z18.s, z10.s, z0.s[2]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "fmla z22.s, z10.s, z1.s[2]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "fmla z19.s, z11.s, z0.s[2]\n"
- "fmla z23.s, z11.s, z1.s[2]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "fmla z16.s, z12.s, z0.s[3]\n"
- "fmla z20.s, z12.s, z1.s[3]\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "fmla z17.s, z13.s, z0.s[3]\n"
- "fmla z21.s, z13.s, z1.s[3]\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "fmla z18.s, z14.s, z0.s[3]\n"
- "fmla z22.s, z14.s, z1.s[3]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "fmla z19.s, z15.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #-0x10]\n"
- "fmla z23.s, z15.s, z1.s[3]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "fmla z16.s, z8.s, z4.s[0]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #-0x10]\n"
- "fmla z20.s, z8.s, z5.s[0]\n"
- "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
- "fmla z17.s, z9.s, z4.s[0]\n"
- "fmla z21.s, z9.s, z5.s[0]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z18.s, z10.s, z4.s[0]\n"
- "fmla z22.s, z10.s, z5.s[0]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z19.s, z11.s, z4.s[0]\n"
- "fmla z23.s, z11.s, z5.s[0]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z16.s, z12.s, z4.s[1]\n"
- "fmla z20.s, z12.s, z5.s[1]\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "fmla z17.s, z13.s, z4.s[1]\n"
- "fmla z21.s, z13.s, z5.s[1]\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "fmla z18.s, z14.s, z4.s[1]\n"
- "fmla z22.s, z14.s, z5.s[1]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z19.s, z15.s, z4.s[1]\n"
- "fmla z23.s, z15.s, z5.s[1]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z16.s, z8.s, z4.s[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "fmla z20.s, z8.s, z5.s[2]\n"
- "fmla z17.s, z9.s, z4.s[2]\n"
- "fmla z21.s, z9.s, z5.s[2]\n"
- "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "fmla z18.s, z10.s, z4.s[2]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "fmla z22.s, z10.s, z5.s[2]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "fmla z19.s, z11.s, z4.s[2]\n"
- "fmla z23.s, z11.s, z5.s[2]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "fmla z16.s, z12.s, z4.s[3]\n"
- "fmla z20.s, z12.s, z5.s[3]\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "fmla z17.s, z13.s, z4.s[3]\n"
- "fmla z21.s, z13.s, z5.s[3]\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "fmla z18.s, z14.s, z4.s[3]\n"
- "fmla z22.s, z14.s, z5.s[3]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "fmla z19.s, z15.s, z4.s[3]\n"
- "fmla z23.s, z15.s, z5.s[3]\n"
- "b.ne 3b\n"
- "2:\n"
- "cbz %[regs], 4f\n"
- "fmla z16.s, z8.s, z0.s[0]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "fmla z20.s, z8.s, z1.s[0]\n"
- "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n"
- "fmla z17.s, z9.s, z0.s[0]\n"
- "ld1rqw z5.s, p7/z, [a_ptr1]\n"
- "fmla z21.s, z9.s, z1.s[0]\n"
- "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
- "fmla z18.s, z10.s, z0.s[0]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z22.s, z10.s, z1.s[0]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z19.s, z11.s, z0.s[0]\n"
- "fmla z23.s, z11.s, z1.s[0]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z16.s, z12.s, z0.s[1]\n"
- "fmla z20.s, z12.s, z1.s[1]\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "fmla z17.s, z13.s, z0.s[1]\n"
- "fmla z21.s, z13.s, z1.s[1]\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "fmla z18.s, z14.s, z0.s[1]\n"
- "fmla z22.s, z14.s, z1.s[1]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z19.s, z15.s, z0.s[1]\n"
- "fmla z23.s, z15.s, z1.s[1]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z16.s, z8.s, z0.s[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "fmla z20.s, z8.s, z1.s[2]\n"
- "fmla z17.s, z9.s, z0.s[2]\n"
- "fmla z21.s, z9.s, z1.s[2]\n"
- "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "fmla z18.s, z10.s, z0.s[2]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "fmla z22.s, z10.s, z1.s[2]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "fmla z19.s, z11.s, z0.s[2]\n"
- "fmla z23.s, z11.s, z1.s[2]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "fmla z16.s, z12.s, z0.s[3]\n"
- "fmla z20.s, z12.s, z1.s[3]\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "fmla z17.s, z13.s, z0.s[3]\n"
- "fmla z21.s, z13.s, z1.s[3]\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "fmla z18.s, z14.s, z0.s[3]\n"
- "fmla z22.s, z14.s, z1.s[3]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "fmla z19.s, z15.s, z0.s[3]\n"
- "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x10]\n"
- "fmla z23.s, z15.s, z1.s[3]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "fmla z16.s, z8.s, z4.s[0]\n"
- "ld1rqw z1.s, p6/z, [a_ptr1, #0x10]\n"
- "fmla z20.s, z8.s, z5.s[0]\n"
- "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
- "fmla z17.s, z9.s, z4.s[0]\n"
- "addvl %[a_ptr0], %[a_ptr0], #2\n"
- "fmla z21.s, z9.s, z5.s[0]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z18.s, z10.s, z4.s[0]\n"
- "addvl a_ptr1, a_ptr1, #2\n"
- "fmla z22.s, z10.s, z5.s[0]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z19.s, z11.s, z4.s[0]\n"
- "fmla z23.s, z11.s, z5.s[0]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z16.s, z12.s, z4.s[1]\n"
- "fmla z20.s, z12.s, z5.s[1]\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "fmla z17.s, z13.s, z4.s[1]\n"
- "fmla z21.s, z13.s, z5.s[1]\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "fmla z18.s, z14.s, z4.s[1]\n"
- "fmla z22.s, z14.s, z5.s[1]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z19.s, z15.s, z4.s[1]\n"
- "fmla z23.s, z15.s, z5.s[1]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z16.s, z8.s, z4.s[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "fmla z20.s, z8.s, z5.s[2]\n"
- "fmla z17.s, z9.s, z4.s[2]\n"
- "fmla z21.s, z9.s, z5.s[2]\n"
- "fmla z18.s, z10.s, z4.s[2]\n"
- "fmla z22.s, z10.s, z5.s[2]\n"
- "fmla z19.s, z11.s, z4.s[2]\n"
- "fmla z23.s, z11.s, z5.s[2]\n"
- "fmla z16.s, z12.s, z4.s[3]\n"
- "fmla z20.s, z12.s, z5.s[3]\n"
- "fmla z17.s, z13.s, z4.s[3]\n"
- "fmla z21.s, z13.s, z5.s[3]\n"
- "fmla z18.s, z14.s, z4.s[3]\n"
- "fmla z22.s, z14.s, z5.s[3]\n"
- "fmla z19.s, z15.s, z4.s[3]\n"
- "fmla z23.s, z15.s, z5.s[3]\n"
- "cbz %[blocks], 5f\n"
- "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z16.s, z8.s, z0.s[0]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z20.s, z8.s, z1.s[0]\n"
- "fmla z17.s, z9.s, z0.s[0]\n"
- "fmla z21.s, z9.s, z1.s[0]\n"
- "fmla z18.s, z10.s, z0.s[0]\n"
- "fmla z22.s, z10.s, z1.s[0]\n"
- "fmla z19.s, z11.s, z0.s[0]\n"
- "fmla z23.s, z11.s, z1.s[0]\n"
- "b.eq 5f\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z16.s, z12.s, z0.s[1]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z20.s, z12.s, z1.s[1]\n"
- "fmla z17.s, z13.s, z0.s[1]\n"
- "fmla z21.s, z13.s, z1.s[1]\n"
- "fmla z18.s, z14.s, z0.s[1]\n"
- "fmla z22.s, z14.s, z1.s[1]\n"
- "fmla z19.s, z15.s, z0.s[1]\n"
- "fmla z23.s, z15.s, z1.s[1]\n"
- "b.eq 5f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "fmla z16.s, z8.s, z0.s[2]\n"
- "fmla z20.s, z8.s, z1.s[2]\n"
- "fmla z17.s, z9.s, z0.s[2]\n"
- "fmla z21.s, z9.s, z1.s[2]\n"
- "fmla z18.s, z10.s, z0.s[2]\n"
- "fmla z22.s, z10.s, z1.s[2]\n"
- "fmla z19.s, z11.s, z0.s[2]\n"
- "fmla z23.s, z11.s, z1.s[2]\n"
- "b 5f\n"
- "4:\n"
- "fmla z16.s, z8.s, z0.s[0]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "fmla z20.s, z8.s, z1.s[0]\n"
- "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
- "fmla z17.s, z9.s, z0.s[0]\n"
- "ld1rqw z4.s, p6/z, [%[a_ptr0]]\n"
- "fmla z21.s, z9.s, z1.s[0]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z18.s, z10.s, z0.s[0]\n"
- "ld1rqw z5.s, p6/z, [a_ptr1]\n"
- "fmla z22.s, z10.s, z1.s[0]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z19.s, z11.s, z0.s[0]\n"
- "addvl %[a_ptr0], %[a_ptr0], #1\n"
- "fmla z23.s, z11.s, z1.s[0]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z16.s, z12.s, z0.s[1]\n"
- "addvl a_ptr1, a_ptr1, #1\n"
- "fmla z20.s, z12.s, z1.s[1]\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "fmla z17.s, z13.s, z0.s[1]\n"
- "fmla z21.s, z13.s, z1.s[1]\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "fmla z18.s, z14.s, z0.s[1]\n"
- "fmla z22.s, z14.s, z1.s[1]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z19.s, z15.s, z0.s[1]\n"
- "fmla z23.s, z15.s, z1.s[1]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z16.s, z8.s, z0.s[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "fmla z20.s, z8.s, z1.s[2]\n"
- "fmla z17.s, z9.s, z0.s[2]\n"
- "fmla z21.s, z9.s, z1.s[2]\n"
- "fmla z18.s, z10.s, z0.s[2]\n"
- "fmla z22.s, z10.s, z1.s[2]\n"
- "fmla z19.s, z11.s, z0.s[2]\n"
- "fmla z23.s, z11.s, z1.s[2]\n"
- "fmla z16.s, z12.s, z0.s[3]\n"
- "fmla z20.s, z12.s, z1.s[3]\n"
- "fmla z17.s, z13.s, z0.s[3]\n"
- "fmla z21.s, z13.s, z1.s[3]\n"
- "fmla z18.s, z14.s, z0.s[3]\n"
- "fmla z22.s, z14.s, z1.s[3]\n"
- "fmla z19.s, z15.s, z0.s[3]\n"
- "fmla z23.s, z15.s, z1.s[3]\n"
- "cbz %[blocks], 5f\n"
- "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z16.s, z8.s, z4.s[0]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z20.s, z8.s, z5.s[0]\n"
- "fmla z17.s, z9.s, z4.s[0]\n"
- "fmla z21.s, z9.s, z5.s[0]\n"
- "fmla z18.s, z10.s, z4.s[0]\n"
- "fmla z22.s, z10.s, z5.s[0]\n"
- "fmla z19.s, z11.s, z4.s[0]\n"
- "fmla z23.s, z11.s, z5.s[0]\n"
- "b.eq 5f\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z16.s, z12.s, z4.s[1]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z20.s, z12.s, z5.s[1]\n"
- "fmla z17.s, z13.s, z4.s[1]\n"
- "fmla z21.s, z13.s, z5.s[1]\n"
- "fmla z18.s, z14.s, z4.s[1]\n"
- "fmla z22.s, z14.s, z5.s[1]\n"
- "fmla z19.s, z15.s, z4.s[1]\n"
- "fmla z23.s, z15.s, z5.s[1]\n"
- "b.eq 5f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "fmla z16.s, z8.s, z4.s[2]\n"
- "fmla z20.s, z8.s, z5.s[2]\n"
- "fmla z17.s, z9.s, z4.s[2]\n"
- "fmla z21.s, z9.s, z5.s[2]\n"
- "fmla z18.s, z10.s, z4.s[2]\n"
- "fmla z22.s, z10.s, z5.s[2]\n"
- "fmla z19.s, z11.s, z4.s[2]\n"
- "fmla z23.s, z11.s, z5.s[2]\n"
- "5:\n"
- "ld1rw z14.s, p7/z, [%[minptr]]\n"
- "ld1rw z15.s, p7/z, [%[maxptr]]\n"
- "fmax z16.s, p7/m, z16.s, z14.s\n"
- "fmax z17.s, p7/m, z17.s, z14.s\n"
- "fmax z18.s, p7/m, z18.s, z14.s\n"
- "fmax z19.s, p7/m, z19.s, z14.s\n"
- "fmin z16.s, p7/m, z16.s, z15.s\n"
- "fmin z17.s, p7/m, z17.s, z15.s\n"
- "fmin z18.s, p7/m, z18.s, z15.s\n"
- "fmin z19.s, p7/m, z19.s, z15.s\n"
- "st1w z16.s, p0, [%[c_ptr0]]\n"
- "fmax z20.s, p7/m, z20.s, z14.s\n"
- "fmax z21.s, p7/m, z21.s, z14.s\n"
- "fmax z22.s, p7/m, z22.s, z14.s\n"
- "st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n"
- "fmax z23.s, p7/m, z23.s, z14.s\n"
- "fmin z20.s, p7/m, z20.s, z15.s\n"
- "fmin z21.s, p7/m, z21.s, z15.s\n"
- "st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n"
- "fmin z22.s, p7/m, z22.s, z15.s\n"
- "fmin z23.s, p7/m, z23.s, z15.s\n"
- "st1w z19.s, p3, [%[c_ptr0], #3, MUL VL]\n"
- "addvl %[c_ptr0], %[c_ptr0], #4\n"
- "st1w z20.s, p0, [c_ptr1]\n"
- "st1w z21.s, p1, [c_ptr1, #1, MUL VL]\n"
- "st1w z22.s, p2, [c_ptr1, #2, MUL VL]\n"
- "st1w z23.s, p3, [c_ptr1, #3, MUL VL]\n"
- ".unreq a_ptr1\n"
- ".unreq c_ptr1\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks)
- : [width] "r" (width), [accumulate] "r" (static_cast<uint64_t>(accumulate)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers)
- : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "cc", "memory"
- );
- break;
- case 3:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "c_ptr1 .req X2\n"
- "c_ptr2 .req X3\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "whilelt p6.s, %[temp], %[leftovers]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "ptrue p7.s\n"
- "whilelt p1.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "whilelt p2.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "whilelt p3.s, %[temp], %[width]\n"
- "cbnz %[accumulate], 1f\n"
- "ld1w z16.s, p0/z, [%[biasptr]]\n"
- "ld1w z17.s, p1/z, [%[biasptr], #1, MUL VL]\n"
- "ld1w z18.s, p2/z, [%[biasptr], #2, MUL VL]\n"
- "ld1w z19.s, p3/z, [%[biasptr], #3, MUL VL]\n"
- "mov z20.d, z16.d\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "mov z21.d, z17.d\n"
- "ld1rqw z1.s, p7/z, [a_ptr1]\n"
- "mov z22.d, z18.d\n"
- "ld1rqw z2.s, p7/z, [a_ptr2]\n"
- "mov z23.d, z19.d\n"
- "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
- "mov z24.d, z16.d\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "mov z25.d, z17.d\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "mov z26.d, z18.d\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "mov z27.d, z19.d\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "ld1w z16.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n"
- "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n"
- "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n"
- "ld1w z20.s, p0/z, [c_ptr1]\n"
- "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n"
- "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n"
- "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n"
- "ld1w z24.s, p0/z, [c_ptr2]\n"
- "ld1w z25.s, p1/z, [c_ptr2, #1, MUL VL]\n"
- "ld1w z26.s, p2/z, [c_ptr2, #2, MUL VL]\n"
- "ld1w z27.s, p3/z, [c_ptr2, #3, MUL VL]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ld1rqw z1.s, p7/z, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "ld1rqw z2.s, p7/z, [a_ptr2]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "cbz %[loops], 2f\n"
- "3:\n"
- "fmla z16.s, z8.s, z0.s[0]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "fmla z20.s, z8.s, z1.s[0]\n"
- "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n"
- "fmla z24.s, z8.s, z2.s[0]\n"
- "ld1rqw z5.s, p7/z, [a_ptr1]\n"
- "fmla z17.s, z9.s, z0.s[0]\n"
- "ld1rqw z6.s, p7/z, [a_ptr2]\n"
- "fmla z21.s, z9.s, z1.s[0]\n"
- "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
- "fmla z25.s, z9.s, z2.s[0]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z18.s, z10.s, z0.s[0]\n"
- "subs %[loops], %[loops], #0x1\n"
- "fmla z22.s, z10.s, z1.s[0]\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
- "fmla z26.s, z10.s, z2.s[0]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z19.s, z11.s, z0.s[0]\n"
- "add a_ptr1, a_ptr1, #0x20\n"
- "fmla z23.s, z11.s, z1.s[0]\n"
- "add a_ptr2, a_ptr2, #0x20\n"
- "fmla z27.s, z11.s, z2.s[0]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z16.s, z12.s, z0.s[1]\n"
- "fmla z20.s, z12.s, z1.s[1]\n"
- "fmla z24.s, z12.s, z2.s[1]\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "fmla z17.s, z13.s, z0.s[1]\n"
- "fmla z21.s, z13.s, z1.s[1]\n"
- "fmla z25.s, z13.s, z2.s[1]\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "fmla z18.s, z14.s, z0.s[1]\n"
- "fmla z22.s, z14.s, z1.s[1]\n"
- "fmla z26.s, z14.s, z2.s[1]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z19.s, z15.s, z0.s[1]\n"
- "fmla z23.s, z15.s, z1.s[1]\n"
- "fmla z27.s, z15.s, z2.s[1]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z16.s, z8.s, z0.s[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "fmla z20.s, z8.s, z1.s[2]\n"
- "fmla z24.s, z8.s, z2.s[2]\n"
- "fmla z17.s, z9.s, z0.s[2]\n"
- "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "fmla z21.s, z9.s, z1.s[2]\n"
- "fmla z25.s, z9.s, z2.s[2]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "fmla z18.s, z10.s, z0.s[2]\n"
- "fmla z22.s, z10.s, z1.s[2]\n"
- "fmla z26.s, z10.s, z2.s[2]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "fmla z19.s, z11.s, z0.s[2]\n"
- "fmla z23.s, z11.s, z1.s[2]\n"
- "fmla z27.s, z11.s, z2.s[2]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "fmla z16.s, z12.s, z0.s[3]\n"
- "fmla z20.s, z12.s, z1.s[3]\n"
- "fmla z24.s, z12.s, z2.s[3]\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "fmla z17.s, z13.s, z0.s[3]\n"
- "fmla z21.s, z13.s, z1.s[3]\n"
- "fmla z25.s, z13.s, z2.s[3]\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "fmla z18.s, z14.s, z0.s[3]\n"
- "fmla z22.s, z14.s, z1.s[3]\n"
- "fmla z26.s, z14.s, z2.s[3]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "fmla z19.s, z15.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #-0x10]\n"
- "fmla z23.s, z15.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #-0x10]\n"
- "fmla z27.s, z15.s, z2.s[3]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "fmla z16.s, z8.s, z4.s[0]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #-0x10]\n"
- "fmla z20.s, z8.s, z5.s[0]\n"
- "fmla z24.s, z8.s, z6.s[0]\n"
- "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
- "fmla z17.s, z9.s, z4.s[0]\n"
- "fmla z21.s, z9.s, z5.s[0]\n"
- "fmla z25.s, z9.s, z6.s[0]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z18.s, z10.s, z4.s[0]\n"
- "fmla z22.s, z10.s, z5.s[0]\n"
- "fmla z26.s, z10.s, z6.s[0]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z19.s, z11.s, z4.s[0]\n"
- "fmla z23.s, z11.s, z5.s[0]\n"
- "fmla z27.s, z11.s, z6.s[0]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z16.s, z12.s, z4.s[1]\n"
- "fmla z20.s, z12.s, z5.s[1]\n"
- "fmla z24.s, z12.s, z6.s[1]\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "fmla z17.s, z13.s, z4.s[1]\n"
- "fmla z21.s, z13.s, z5.s[1]\n"
- "fmla z25.s, z13.s, z6.s[1]\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "fmla z18.s, z14.s, z4.s[1]\n"
- "fmla z22.s, z14.s, z5.s[1]\n"
- "fmla z26.s, z14.s, z6.s[1]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z19.s, z15.s, z4.s[1]\n"
- "fmla z23.s, z15.s, z5.s[1]\n"
- "fmla z27.s, z15.s, z6.s[1]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z16.s, z8.s, z4.s[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "fmla z20.s, z8.s, z5.s[2]\n"
- "fmla z24.s, z8.s, z6.s[2]\n"
- "fmla z17.s, z9.s, z4.s[2]\n"
- "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "fmla z21.s, z9.s, z5.s[2]\n"
- "fmla z25.s, z9.s, z6.s[2]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "fmla z18.s, z10.s, z4.s[2]\n"
- "fmla z22.s, z10.s, z5.s[2]\n"
- "fmla z26.s, z10.s, z6.s[2]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "fmla z19.s, z11.s, z4.s[2]\n"
- "fmla z23.s, z11.s, z5.s[2]\n"
- "fmla z27.s, z11.s, z6.s[2]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "fmla z16.s, z12.s, z4.s[3]\n"
- "fmla z20.s, z12.s, z5.s[3]\n"
- "fmla z24.s, z12.s, z6.s[3]\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "fmla z17.s, z13.s, z4.s[3]\n"
- "fmla z21.s, z13.s, z5.s[3]\n"
- "fmla z25.s, z13.s, z6.s[3]\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "fmla z18.s, z14.s, z4.s[3]\n"
- "fmla z22.s, z14.s, z5.s[3]\n"
- "fmla z26.s, z14.s, z6.s[3]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "fmla z19.s, z15.s, z4.s[3]\n"
- "fmla z23.s, z15.s, z5.s[3]\n"
- "fmla z27.s, z15.s, z6.s[3]\n"
- "b.ne 3b\n"
- "2:\n"
- "cbz %[regs], 4f\n"
- "fmla z16.s, z8.s, z0.s[0]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "fmla z20.s, z8.s, z1.s[0]\n"
- "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n"
- "fmla z24.s, z8.s, z2.s[0]\n"
- "ld1rqw z5.s, p7/z, [a_ptr1]\n"
- "fmla z17.s, z9.s, z0.s[0]\n"
- "ld1rqw z6.s, p7/z, [a_ptr2]\n"
- "fmla z21.s, z9.s, z1.s[0]\n"
- "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
- "fmla z25.s, z9.s, z2.s[0]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z18.s, z10.s, z0.s[0]\n"
- "fmla z22.s, z10.s, z1.s[0]\n"
- "fmla z26.s, z10.s, z2.s[0]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z19.s, z11.s, z0.s[0]\n"
- "fmla z23.s, z11.s, z1.s[0]\n"
- "fmla z27.s, z11.s, z2.s[0]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z16.s, z12.s, z0.s[1]\n"
- "fmla z20.s, z12.s, z1.s[1]\n"
- "fmla z24.s, z12.s, z2.s[1]\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "fmla z17.s, z13.s, z0.s[1]\n"
- "fmla z21.s, z13.s, z1.s[1]\n"
- "fmla z25.s, z13.s, z2.s[1]\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "fmla z18.s, z14.s, z0.s[1]\n"
- "fmla z22.s, z14.s, z1.s[1]\n"
- "fmla z26.s, z14.s, z2.s[1]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z19.s, z15.s, z0.s[1]\n"
- "fmla z23.s, z15.s, z1.s[1]\n"
- "fmla z27.s, z15.s, z2.s[1]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z16.s, z8.s, z0.s[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "fmla z20.s, z8.s, z1.s[2]\n"
- "fmla z24.s, z8.s, z2.s[2]\n"
- "fmla z17.s, z9.s, z0.s[2]\n"
- "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "fmla z21.s, z9.s, z1.s[2]\n"
- "fmla z25.s, z9.s, z2.s[2]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "fmla z18.s, z10.s, z0.s[2]\n"
- "fmla z22.s, z10.s, z1.s[2]\n"
- "fmla z26.s, z10.s, z2.s[2]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "fmla z19.s, z11.s, z0.s[2]\n"
- "fmla z23.s, z11.s, z1.s[2]\n"
- "fmla z27.s, z11.s, z2.s[2]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "fmla z16.s, z12.s, z0.s[3]\n"
- "fmla z20.s, z12.s, z1.s[3]\n"
- "fmla z24.s, z12.s, z2.s[3]\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "fmla z17.s, z13.s, z0.s[3]\n"
- "fmla z21.s, z13.s, z1.s[3]\n"
- "fmla z25.s, z13.s, z2.s[3]\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "fmla z18.s, z14.s, z0.s[3]\n"
- "fmla z22.s, z14.s, z1.s[3]\n"
- "fmla z26.s, z14.s, z2.s[3]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "fmla z19.s, z15.s, z0.s[3]\n"
- "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x10]\n"
- "fmla z23.s, z15.s, z1.s[3]\n"
- "ld1rqw z1.s, p6/z, [a_ptr1, #0x10]\n"
- "fmla z27.s, z15.s, z2.s[3]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "fmla z16.s, z8.s, z4.s[0]\n"
- "ld1rqw z2.s, p6/z, [a_ptr2, #0x10]\n"
- "fmla z20.s, z8.s, z5.s[0]\n"
- "addvl %[a_ptr0], %[a_ptr0], #2\n"
- "fmla z24.s, z8.s, z6.s[0]\n"
- "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
- "fmla z17.s, z9.s, z4.s[0]\n"
- "addvl a_ptr1, a_ptr1, #2\n"
- "fmla z21.s, z9.s, z5.s[0]\n"
- "addvl a_ptr2, a_ptr2, #2\n"
- "fmla z25.s, z9.s, z6.s[0]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z18.s, z10.s, z4.s[0]\n"
- "fmla z22.s, z10.s, z5.s[0]\n"
- "fmla z26.s, z10.s, z6.s[0]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z19.s, z11.s, z4.s[0]\n"
- "fmla z23.s, z11.s, z5.s[0]\n"
- "fmla z27.s, z11.s, z6.s[0]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z16.s, z12.s, z4.s[1]\n"
- "fmla z20.s, z12.s, z5.s[1]\n"
- "fmla z24.s, z12.s, z6.s[1]\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "fmla z17.s, z13.s, z4.s[1]\n"
- "fmla z21.s, z13.s, z5.s[1]\n"
- "fmla z25.s, z13.s, z6.s[1]\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "fmla z18.s, z14.s, z4.s[1]\n"
- "fmla z22.s, z14.s, z5.s[1]\n"
- "fmla z26.s, z14.s, z6.s[1]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z19.s, z15.s, z4.s[1]\n"
- "fmla z23.s, z15.s, z5.s[1]\n"
- "fmla z27.s, z15.s, z6.s[1]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z16.s, z8.s, z4.s[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "fmla z20.s, z8.s, z5.s[2]\n"
- "fmla z24.s, z8.s, z6.s[2]\n"
- "fmla z17.s, z9.s, z4.s[2]\n"
- "fmla z21.s, z9.s, z5.s[2]\n"
- "fmla z25.s, z9.s, z6.s[2]\n"
- "fmla z18.s, z10.s, z4.s[2]\n"
- "fmla z22.s, z10.s, z5.s[2]\n"
- "fmla z26.s, z10.s, z6.s[2]\n"
- "fmla z19.s, z11.s, z4.s[2]\n"
- "fmla z23.s, z11.s, z5.s[2]\n"
- "fmla z27.s, z11.s, z6.s[2]\n"
- "fmla z16.s, z12.s, z4.s[3]\n"
- "fmla z20.s, z12.s, z5.s[3]\n"
- "fmla z24.s, z12.s, z6.s[3]\n"
- "fmla z17.s, z13.s, z4.s[3]\n"
- "fmla z21.s, z13.s, z5.s[3]\n"
- "fmla z25.s, z13.s, z6.s[3]\n"
- "fmla z18.s, z14.s, z4.s[3]\n"
- "fmla z22.s, z14.s, z5.s[3]\n"
- "fmla z26.s, z14.s, z6.s[3]\n"
- "fmla z19.s, z15.s, z4.s[3]\n"
- "fmla z23.s, z15.s, z5.s[3]\n"
- "fmla z27.s, z15.s, z6.s[3]\n"
- "cbz %[blocks], 5f\n"
- "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z16.s, z8.s, z0.s[0]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z20.s, z8.s, z1.s[0]\n"
- "fmla z24.s, z8.s, z2.s[0]\n"
- "fmla z17.s, z9.s, z0.s[0]\n"
- "fmla z21.s, z9.s, z1.s[0]\n"
- "fmla z25.s, z9.s, z2.s[0]\n"
- "fmla z18.s, z10.s, z0.s[0]\n"
- "fmla z22.s, z10.s, z1.s[0]\n"
- "fmla z26.s, z10.s, z2.s[0]\n"
- "fmla z19.s, z11.s, z0.s[0]\n"
- "fmla z23.s, z11.s, z1.s[0]\n"
- "fmla z27.s, z11.s, z2.s[0]\n"
- "b.eq 5f\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z16.s, z12.s, z0.s[1]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z20.s, z12.s, z1.s[1]\n"
- "fmla z24.s, z12.s, z2.s[1]\n"
- "fmla z17.s, z13.s, z0.s[1]\n"
- "fmla z21.s, z13.s, z1.s[1]\n"
- "fmla z25.s, z13.s, z2.s[1]\n"
- "fmla z18.s, z14.s, z0.s[1]\n"
- "fmla z22.s, z14.s, z1.s[1]\n"
- "fmla z26.s, z14.s, z2.s[1]\n"
- "fmla z19.s, z15.s, z0.s[1]\n"
- "fmla z23.s, z15.s, z1.s[1]\n"
- "fmla z27.s, z15.s, z2.s[1]\n"
- "b.eq 5f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "fmla z16.s, z8.s, z0.s[2]\n"
- "fmla z20.s, z8.s, z1.s[2]\n"
- "fmla z24.s, z8.s, z2.s[2]\n"
- "fmla z17.s, z9.s, z0.s[2]\n"
- "fmla z21.s, z9.s, z1.s[2]\n"
- "fmla z25.s, z9.s, z2.s[2]\n"
- "fmla z18.s, z10.s, z0.s[2]\n"
- "fmla z22.s, z10.s, z1.s[2]\n"
- "fmla z26.s, z10.s, z2.s[2]\n"
- "fmla z19.s, z11.s, z0.s[2]\n"
- "fmla z23.s, z11.s, z1.s[2]\n"
- "fmla z27.s, z11.s, z2.s[2]\n"
- "b 5f\n"
- "4:\n"
- "fmla z16.s, z8.s, z0.s[0]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "fmla z20.s, z8.s, z1.s[0]\n"
- "ld1rqw z4.s, p6/z, [%[a_ptr0]]\n"
- "fmla z24.s, z8.s, z2.s[0]\n"
- "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
- "fmla z17.s, z9.s, z0.s[0]\n"
- "ld1rqw z5.s, p6/z, [a_ptr1]\n"
- "fmla z21.s, z9.s, z1.s[0]\n"
- "ld1rqw z6.s, p6/z, [a_ptr2]\n"
- "fmla z25.s, z9.s, z2.s[0]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z18.s, z10.s, z0.s[0]\n"
- "addvl %[a_ptr0], %[a_ptr0], #1\n"
- "fmla z22.s, z10.s, z1.s[0]\n"
- "addvl a_ptr1, a_ptr1, #1\n"
- "fmla z26.s, z10.s, z2.s[0]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z19.s, z11.s, z0.s[0]\n"
- "addvl a_ptr2, a_ptr2, #1\n"
- "fmla z23.s, z11.s, z1.s[0]\n"
- "fmla z27.s, z11.s, z2.s[0]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z16.s, z12.s, z0.s[1]\n"
- "fmla z20.s, z12.s, z1.s[1]\n"
- "fmla z24.s, z12.s, z2.s[1]\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "fmla z17.s, z13.s, z0.s[1]\n"
- "fmla z21.s, z13.s, z1.s[1]\n"
- "fmla z25.s, z13.s, z2.s[1]\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "fmla z18.s, z14.s, z0.s[1]\n"
- "fmla z22.s, z14.s, z1.s[1]\n"
- "fmla z26.s, z14.s, z2.s[1]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z19.s, z15.s, z0.s[1]\n"
- "fmla z23.s, z15.s, z1.s[1]\n"
- "fmla z27.s, z15.s, z2.s[1]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z16.s, z8.s, z0.s[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "fmla z20.s, z8.s, z1.s[2]\n"
- "fmla z24.s, z8.s, z2.s[2]\n"
- "fmla z17.s, z9.s, z0.s[2]\n"
- "fmla z21.s, z9.s, z1.s[2]\n"
- "fmla z25.s, z9.s, z2.s[2]\n"
- "fmla z18.s, z10.s, z0.s[2]\n"
- "fmla z22.s, z10.s, z1.s[2]\n"
- "fmla z26.s, z10.s, z2.s[2]\n"
- "fmla z19.s, z11.s, z0.s[2]\n"
- "fmla z23.s, z11.s, z1.s[2]\n"
- "fmla z27.s, z11.s, z2.s[2]\n"
- "fmla z16.s, z12.s, z0.s[3]\n"
- "fmla z20.s, z12.s, z1.s[3]\n"
- "fmla z24.s, z12.s, z2.s[3]\n"
- "fmla z17.s, z13.s, z0.s[3]\n"
- "fmla z21.s, z13.s, z1.s[3]\n"
- "fmla z25.s, z13.s, z2.s[3]\n"
- "fmla z18.s, z14.s, z0.s[3]\n"
- "fmla z22.s, z14.s, z1.s[3]\n"
- "fmla z26.s, z14.s, z2.s[3]\n"
- "fmla z19.s, z15.s, z0.s[3]\n"
- "fmla z23.s, z15.s, z1.s[3]\n"
- "fmla z27.s, z15.s, z2.s[3]\n"
- "cbz %[blocks], 5f\n"
- "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z16.s, z8.s, z4.s[0]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z20.s, z8.s, z5.s[0]\n"
- "fmla z24.s, z8.s, z6.s[0]\n"
- "fmla z17.s, z9.s, z4.s[0]\n"
- "fmla z21.s, z9.s, z5.s[0]\n"
- "fmla z25.s, z9.s, z6.s[0]\n"
- "fmla z18.s, z10.s, z4.s[0]\n"
- "fmla z22.s, z10.s, z5.s[0]\n"
- "fmla z26.s, z10.s, z6.s[0]\n"
- "fmla z19.s, z11.s, z4.s[0]\n"
- "fmla z23.s, z11.s, z5.s[0]\n"
- "fmla z27.s, z11.s, z6.s[0]\n"
- "b.eq 5f\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z16.s, z12.s, z4.s[1]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z20.s, z12.s, z5.s[1]\n"
- "fmla z24.s, z12.s, z6.s[1]\n"
- "fmla z17.s, z13.s, z4.s[1]\n"
- "fmla z21.s, z13.s, z5.s[1]\n"
- "fmla z25.s, z13.s, z6.s[1]\n"
- "fmla z18.s, z14.s, z4.s[1]\n"
- "fmla z22.s, z14.s, z5.s[1]\n"
- "fmla z26.s, z14.s, z6.s[1]\n"
- "fmla z19.s, z15.s, z4.s[1]\n"
- "fmla z23.s, z15.s, z5.s[1]\n"
- "fmla z27.s, z15.s, z6.s[1]\n"
- "b.eq 5f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "fmla z16.s, z8.s, z4.s[2]\n"
- "fmla z20.s, z8.s, z5.s[2]\n"
- "fmla z24.s, z8.s, z6.s[2]\n"
- "fmla z17.s, z9.s, z4.s[2]\n"
- "fmla z21.s, z9.s, z5.s[2]\n"
- "fmla z25.s, z9.s, z6.s[2]\n"
- "fmla z18.s, z10.s, z4.s[2]\n"
- "fmla z22.s, z10.s, z5.s[2]\n"
- "fmla z26.s, z10.s, z6.s[2]\n"
- "fmla z19.s, z11.s, z4.s[2]\n"
- "fmla z23.s, z11.s, z5.s[2]\n"
- "fmla z27.s, z11.s, z6.s[2]\n"
- "5:\n"
- "ld1rw z14.s, p7/z, [%[minptr]]\n"
- "ld1rw z15.s, p7/z, [%[maxptr]]\n"
- "fmax z16.s, p7/m, z16.s, z14.s\n"
- "fmax z17.s, p7/m, z17.s, z14.s\n"
- "fmax z18.s, p7/m, z18.s, z14.s\n"
- "fmax z19.s, p7/m, z19.s, z14.s\n"
- "fmin z16.s, p7/m, z16.s, z15.s\n"
- "fmin z17.s, p7/m, z17.s, z15.s\n"
- "fmin z18.s, p7/m, z18.s, z15.s\n"
- "fmin z19.s, p7/m, z19.s, z15.s\n"
- "st1w z16.s, p0, [%[c_ptr0]]\n"
- "fmax z20.s, p7/m, z20.s, z14.s\n"
- "fmax z21.s, p7/m, z21.s, z14.s\n"
- "fmax z22.s, p7/m, z22.s, z14.s\n"
- "st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n"
- "fmax z23.s, p7/m, z23.s, z14.s\n"
- "fmin z20.s, p7/m, z20.s, z15.s\n"
- "fmin z21.s, p7/m, z21.s, z15.s\n"
- "st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n"
- "fmin z22.s, p7/m, z22.s, z15.s\n"
- "fmin z23.s, p7/m, z23.s, z15.s\n"
- "fmax z24.s, p7/m, z24.s, z14.s\n"
- "st1w z19.s, p3, [%[c_ptr0], #3, MUL VL]\n"
- "fmax z25.s, p7/m, z25.s, z14.s\n"
- "addvl %[c_ptr0], %[c_ptr0], #4\n"
- "fmax z26.s, p7/m, z26.s, z14.s\n"
- "st1w z20.s, p0, [c_ptr1]\n"
- "fmin z24.s, p7/m, z24.s, z15.s\n"
- "fmin z25.s, p7/m, z25.s, z15.s\n"
- "fmax z27.s, p7/m, z27.s, z14.s\n"
- "st1w z21.s, p1, [c_ptr1, #1, MUL VL]\n"
- "fmin z26.s, p7/m, z26.s, z15.s\n"
- "fmin z27.s, p7/m, z27.s, z15.s\n"
- "st1w z22.s, p2, [c_ptr1, #2, MUL VL]\n"
- "st1w z23.s, p3, [c_ptr1, #3, MUL VL]\n"
- "st1w z24.s, p0, [c_ptr2]\n"
- "st1w z25.s, p1, [c_ptr2, #1, MUL VL]\n"
- "st1w z26.s, p2, [c_ptr2, #2, MUL VL]\n"
- "st1w z27.s, p3, [c_ptr2, #3, MUL VL]\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks)
- : [width] "r" (width), [accumulate] "r" (static_cast<uint64_t>(accumulate)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers)
- : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "cc", "memory"
- );
- break;
- default:
- case 4:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "a_ptr3 .req X2\n"
- "c_ptr1 .req X3\n"
- "c_ptr2 .req X4\n"
- "c_ptr3 .req X5\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "add a_ptr3, a_ptr2, %[lda]\n"
- "add c_ptr3, c_ptr2, %[ldc]\n"
- "whilelt p6.s, %[temp], %[leftovers]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "ptrue p7.s\n"
- "whilelt p1.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "whilelt p2.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "whilelt p3.s, %[temp], %[width]\n"
- "cbnz %[accumulate], 1f\n"
- "ld1w z16.s, p0/z, [%[biasptr]]\n"
- "ld1w z17.s, p1/z, [%[biasptr], #1, MUL VL]\n"
- "ld1w z18.s, p2/z, [%[biasptr], #2, MUL VL]\n"
- "ld1w z19.s, p3/z, [%[biasptr], #3, MUL VL]\n"
- "mov z20.d, z16.d\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "mov z21.d, z17.d\n"
- "ld1rqw z1.s, p7/z, [a_ptr1]\n"
- "mov z22.d, z18.d\n"
- "ld1rqw z2.s, p7/z, [a_ptr2]\n"
- "mov z23.d, z19.d\n"
- "ld1rqw z3.s, p7/z, [a_ptr3]\n"
- "mov z24.d, z16.d\n"
- "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
- "mov z25.d, z17.d\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "mov z26.d, z18.d\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "mov z27.d, z19.d\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "mov z28.d, z16.d\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "mov z29.d, z17.d\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "mov z30.d, z18.d\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "mov z31.d, z19.d\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "add a_ptr3, a_ptr3, #0x10\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "ld1w z16.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n"
- "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n"
- "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n"
- "ld1w z20.s, p0/z, [c_ptr1]\n"
- "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n"
- "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n"
- "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n"
- "ld1w z24.s, p0/z, [c_ptr2]\n"
- "ld1w z25.s, p1/z, [c_ptr2, #1, MUL VL]\n"
- "ld1w z26.s, p2/z, [c_ptr2, #2, MUL VL]\n"
- "ld1w z27.s, p3/z, [c_ptr2, #3, MUL VL]\n"
- "ld1w z28.s, p0/z, [c_ptr3]\n"
- "ld1w z29.s, p1/z, [c_ptr3, #1, MUL VL]\n"
- "ld1w z30.s, p2/z, [c_ptr3, #2, MUL VL]\n"
- "ld1w z31.s, p3/z, [c_ptr3, #3, MUL VL]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ld1rqw z1.s, p7/z, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "ld1rqw z2.s, p7/z, [a_ptr2]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "ld1rqw z3.s, p7/z, [a_ptr3]\n"
- "add a_ptr3, a_ptr3, #0x10\n"
- "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "cbz %[loops], 2f\n"
- "3:\n"
- "fmla z16.s, z8.s, z0.s[0]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "fmla z20.s, z8.s, z1.s[0]\n"
- "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n"
- "fmla z24.s, z8.s, z2.s[0]\n"
- "ld1rqw z5.s, p7/z, [a_ptr1]\n"
- "fmla z28.s, z8.s, z3.s[0]\n"
- "ld1rqw z6.s, p7/z, [a_ptr2]\n"
- "fmla z17.s, z9.s, z0.s[0]\n"
- "ld1rqw z7.s, p7/z, [a_ptr3]\n"
- "fmla z21.s, z9.s, z1.s[0]\n"
- "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
- "fmla z25.s, z9.s, z2.s[0]\n"
- "subs %[loops], %[loops], #0x1\n"
- "fmla z29.s, z9.s, z3.s[0]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z18.s, z10.s, z0.s[0]\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
- "fmla z22.s, z10.s, z1.s[0]\n"
- "add a_ptr1, a_ptr1, #0x20\n"
- "fmla z26.s, z10.s, z2.s[0]\n"
- "add a_ptr2, a_ptr2, #0x20\n"
- "fmla z30.s, z10.s, z3.s[0]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z19.s, z11.s, z0.s[0]\n"
- "add a_ptr3, a_ptr3, #0x20\n"
- "fmla z23.s, z11.s, z1.s[0]\n"
- "fmla z27.s, z11.s, z2.s[0]\n"
- "fmla z31.s, z11.s, z3.s[0]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z16.s, z12.s, z0.s[1]\n"
- "fmla z20.s, z12.s, z1.s[1]\n"
- "fmla z24.s, z12.s, z2.s[1]\n"
- "fmla z28.s, z12.s, z3.s[1]\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "fmla z17.s, z13.s, z0.s[1]\n"
- "fmla z21.s, z13.s, z1.s[1]\n"
- "fmla z25.s, z13.s, z2.s[1]\n"
- "fmla z29.s, z13.s, z3.s[1]\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "fmla z18.s, z14.s, z0.s[1]\n"
- "fmla z22.s, z14.s, z1.s[1]\n"
- "fmla z26.s, z14.s, z2.s[1]\n"
- "fmla z30.s, z14.s, z3.s[1]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z19.s, z15.s, z0.s[1]\n"
- "fmla z23.s, z15.s, z1.s[1]\n"
- "fmla z27.s, z15.s, z2.s[1]\n"
- "fmla z31.s, z15.s, z3.s[1]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z16.s, z8.s, z0.s[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "fmla z20.s, z8.s, z1.s[2]\n"
- "fmla z24.s, z8.s, z2.s[2]\n"
- "fmla z28.s, z8.s, z3.s[2]\n"
- "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "fmla z17.s, z9.s, z0.s[2]\n"
- "fmla z21.s, z9.s, z1.s[2]\n"
- "fmla z25.s, z9.s, z2.s[2]\n"
- "fmla z29.s, z9.s, z3.s[2]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "fmla z18.s, z10.s, z0.s[2]\n"
- "fmla z22.s, z10.s, z1.s[2]\n"
- "fmla z26.s, z10.s, z2.s[2]\n"
- "fmla z30.s, z10.s, z3.s[2]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "fmla z19.s, z11.s, z0.s[2]\n"
- "fmla z23.s, z11.s, z1.s[2]\n"
- "fmla z27.s, z11.s, z2.s[2]\n"
- "fmla z31.s, z11.s, z3.s[2]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "fmla z16.s, z12.s, z0.s[3]\n"
- "fmla z20.s, z12.s, z1.s[3]\n"
- "fmla z24.s, z12.s, z2.s[3]\n"
- "fmla z28.s, z12.s, z3.s[3]\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "fmla z17.s, z13.s, z0.s[3]\n"
- "fmla z21.s, z13.s, z1.s[3]\n"
- "fmla z25.s, z13.s, z2.s[3]\n"
- "fmla z29.s, z13.s, z3.s[3]\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "fmla z18.s, z14.s, z0.s[3]\n"
- "fmla z22.s, z14.s, z1.s[3]\n"
- "fmla z26.s, z14.s, z2.s[3]\n"
- "fmla z30.s, z14.s, z3.s[3]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "fmla z19.s, z15.s, z0.s[3]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #-0x10]\n"
- "fmla z23.s, z15.s, z1.s[3]\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #-0x10]\n"
- "fmla z27.s, z15.s, z2.s[3]\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #-0x10]\n"
- "fmla z31.s, z15.s, z3.s[3]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "fmla z16.s, z8.s, z4.s[0]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #-0x10]\n"
- "fmla z20.s, z8.s, z5.s[0]\n"
- "fmla z24.s, z8.s, z6.s[0]\n"
- "fmla z28.s, z8.s, z7.s[0]\n"
- "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
- "fmla z17.s, z9.s, z4.s[0]\n"
- "fmla z21.s, z9.s, z5.s[0]\n"
- "fmla z25.s, z9.s, z6.s[0]\n"
- "fmla z29.s, z9.s, z7.s[0]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z18.s, z10.s, z4.s[0]\n"
- "fmla z22.s, z10.s, z5.s[0]\n"
- "fmla z26.s, z10.s, z6.s[0]\n"
- "fmla z30.s, z10.s, z7.s[0]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z19.s, z11.s, z4.s[0]\n"
- "fmla z23.s, z11.s, z5.s[0]\n"
- "fmla z27.s, z11.s, z6.s[0]\n"
- "fmla z31.s, z11.s, z7.s[0]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z16.s, z12.s, z4.s[1]\n"
- "fmla z20.s, z12.s, z5.s[1]\n"
- "fmla z24.s, z12.s, z6.s[1]\n"
- "fmla z28.s, z12.s, z7.s[1]\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "fmla z17.s, z13.s, z4.s[1]\n"
- "fmla z21.s, z13.s, z5.s[1]\n"
- "fmla z25.s, z13.s, z6.s[1]\n"
- "fmla z29.s, z13.s, z7.s[1]\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "fmla z18.s, z14.s, z4.s[1]\n"
- "fmla z22.s, z14.s, z5.s[1]\n"
- "fmla z26.s, z14.s, z6.s[1]\n"
- "fmla z30.s, z14.s, z7.s[1]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z19.s, z15.s, z4.s[1]\n"
- "fmla z23.s, z15.s, z5.s[1]\n"
- "fmla z27.s, z15.s, z6.s[1]\n"
- "fmla z31.s, z15.s, z7.s[1]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z16.s, z8.s, z4.s[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "fmla z20.s, z8.s, z5.s[2]\n"
- "fmla z24.s, z8.s, z6.s[2]\n"
- "fmla z28.s, z8.s, z7.s[2]\n"
- "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "fmla z17.s, z9.s, z4.s[2]\n"
- "fmla z21.s, z9.s, z5.s[2]\n"
- "fmla z25.s, z9.s, z6.s[2]\n"
- "fmla z29.s, z9.s, z7.s[2]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "fmla z18.s, z10.s, z4.s[2]\n"
- "fmla z22.s, z10.s, z5.s[2]\n"
- "fmla z26.s, z10.s, z6.s[2]\n"
- "fmla z30.s, z10.s, z7.s[2]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "fmla z19.s, z11.s, z4.s[2]\n"
- "fmla z23.s, z11.s, z5.s[2]\n"
- "fmla z27.s, z11.s, z6.s[2]\n"
- "fmla z31.s, z11.s, z7.s[2]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "fmla z16.s, z12.s, z4.s[3]\n"
- "fmla z20.s, z12.s, z5.s[3]\n"
- "fmla z24.s, z12.s, z6.s[3]\n"
- "fmla z28.s, z12.s, z7.s[3]\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "fmla z17.s, z13.s, z4.s[3]\n"
- "fmla z21.s, z13.s, z5.s[3]\n"
- "fmla z25.s, z13.s, z6.s[3]\n"
- "fmla z29.s, z13.s, z7.s[3]\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "fmla z18.s, z14.s, z4.s[3]\n"
- "fmla z22.s, z14.s, z5.s[3]\n"
- "fmla z26.s, z14.s, z6.s[3]\n"
- "fmla z30.s, z14.s, z7.s[3]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "fmla z19.s, z15.s, z4.s[3]\n"
- "fmla z23.s, z15.s, z5.s[3]\n"
- "fmla z27.s, z15.s, z6.s[3]\n"
- "fmla z31.s, z15.s, z7.s[3]\n"
- "b.ne 3b\n"
- "2:\n"
- "cbz %[regs], 4f\n"
- "fmla z16.s, z8.s, z0.s[0]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "fmla z20.s, z8.s, z1.s[0]\n"
- "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n"
- "fmla z24.s, z8.s, z2.s[0]\n"
- "ld1rqw z5.s, p7/z, [a_ptr1]\n"
- "fmla z28.s, z8.s, z3.s[0]\n"
- "ld1rqw z6.s, p7/z, [a_ptr2]\n"
- "fmla z17.s, z9.s, z0.s[0]\n"
- "ld1rqw z7.s, p7/z, [a_ptr3]\n"
- "fmla z21.s, z9.s, z1.s[0]\n"
- "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
- "fmla z25.s, z9.s, z2.s[0]\n"
- "fmla z29.s, z9.s, z3.s[0]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z18.s, z10.s, z0.s[0]\n"
- "fmla z22.s, z10.s, z1.s[0]\n"
- "fmla z26.s, z10.s, z2.s[0]\n"
- "fmla z30.s, z10.s, z3.s[0]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z19.s, z11.s, z0.s[0]\n"
- "fmla z23.s, z11.s, z1.s[0]\n"
- "fmla z27.s, z11.s, z2.s[0]\n"
- "fmla z31.s, z11.s, z3.s[0]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z16.s, z12.s, z0.s[1]\n"
- "fmla z20.s, z12.s, z1.s[1]\n"
- "fmla z24.s, z12.s, z2.s[1]\n"
- "fmla z28.s, z12.s, z3.s[1]\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "fmla z17.s, z13.s, z0.s[1]\n"
- "fmla z21.s, z13.s, z1.s[1]\n"
- "fmla z25.s, z13.s, z2.s[1]\n"
- "fmla z29.s, z13.s, z3.s[1]\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "fmla z18.s, z14.s, z0.s[1]\n"
- "fmla z22.s, z14.s, z1.s[1]\n"
- "fmla z26.s, z14.s, z2.s[1]\n"
- "fmla z30.s, z14.s, z3.s[1]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z19.s, z15.s, z0.s[1]\n"
- "fmla z23.s, z15.s, z1.s[1]\n"
- "fmla z27.s, z15.s, z2.s[1]\n"
- "fmla z31.s, z15.s, z3.s[1]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z16.s, z8.s, z0.s[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "fmla z20.s, z8.s, z1.s[2]\n"
- "fmla z24.s, z8.s, z2.s[2]\n"
- "fmla z28.s, z8.s, z3.s[2]\n"
- "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "fmla z17.s, z9.s, z0.s[2]\n"
- "fmla z21.s, z9.s, z1.s[2]\n"
- "fmla z25.s, z9.s, z2.s[2]\n"
- "fmla z29.s, z9.s, z3.s[2]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "fmla z18.s, z10.s, z0.s[2]\n"
- "fmla z22.s, z10.s, z1.s[2]\n"
- "fmla z26.s, z10.s, z2.s[2]\n"
- "fmla z30.s, z10.s, z3.s[2]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "fmla z19.s, z11.s, z0.s[2]\n"
- "fmla z23.s, z11.s, z1.s[2]\n"
- "fmla z27.s, z11.s, z2.s[2]\n"
- "fmla z31.s, z11.s, z3.s[2]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "fmla z16.s, z12.s, z0.s[3]\n"
- "fmla z20.s, z12.s, z1.s[3]\n"
- "fmla z24.s, z12.s, z2.s[3]\n"
- "fmla z28.s, z12.s, z3.s[3]\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "fmla z17.s, z13.s, z0.s[3]\n"
- "fmla z21.s, z13.s, z1.s[3]\n"
- "fmla z25.s, z13.s, z2.s[3]\n"
- "fmla z29.s, z13.s, z3.s[3]\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "fmla z18.s, z14.s, z0.s[3]\n"
- "fmla z22.s, z14.s, z1.s[3]\n"
- "fmla z26.s, z14.s, z2.s[3]\n"
- "fmla z30.s, z14.s, z3.s[3]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "fmla z19.s, z15.s, z0.s[3]\n"
- "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x10]\n"
- "fmla z23.s, z15.s, z1.s[3]\n"
- "ld1rqw z1.s, p6/z, [a_ptr1, #0x10]\n"
- "fmla z27.s, z15.s, z2.s[3]\n"
- "ld1rqw z2.s, p6/z, [a_ptr2, #0x10]\n"
- "fmla z31.s, z15.s, z3.s[3]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "fmla z16.s, z8.s, z4.s[0]\n"
- "ld1rqw z3.s, p6/z, [a_ptr3, #0x10]\n"
- "fmla z20.s, z8.s, z5.s[0]\n"
- "addvl %[a_ptr0], %[a_ptr0], #2\n"
- "fmla z24.s, z8.s, z6.s[0]\n"
- "addvl a_ptr1, a_ptr1, #2\n"
- "fmla z28.s, z8.s, z7.s[0]\n"
- "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
- "fmla z17.s, z9.s, z4.s[0]\n"
- "addvl a_ptr2, a_ptr2, #2\n"
- "fmla z21.s, z9.s, z5.s[0]\n"
- "addvl a_ptr3, a_ptr3, #2\n"
- "fmla z25.s, z9.s, z6.s[0]\n"
- "fmla z29.s, z9.s, z7.s[0]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z18.s, z10.s, z4.s[0]\n"
- "fmla z22.s, z10.s, z5.s[0]\n"
- "fmla z26.s, z10.s, z6.s[0]\n"
- "fmla z30.s, z10.s, z7.s[0]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z19.s, z11.s, z4.s[0]\n"
- "fmla z23.s, z11.s, z5.s[0]\n"
- "fmla z27.s, z11.s, z6.s[0]\n"
- "fmla z31.s, z11.s, z7.s[0]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z16.s, z12.s, z4.s[1]\n"
- "fmla z20.s, z12.s, z5.s[1]\n"
- "fmla z24.s, z12.s, z6.s[1]\n"
- "fmla z28.s, z12.s, z7.s[1]\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "fmla z17.s, z13.s, z4.s[1]\n"
- "fmla z21.s, z13.s, z5.s[1]\n"
- "fmla z25.s, z13.s, z6.s[1]\n"
- "fmla z29.s, z13.s, z7.s[1]\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "fmla z18.s, z14.s, z4.s[1]\n"
- "fmla z22.s, z14.s, z5.s[1]\n"
- "fmla z26.s, z14.s, z6.s[1]\n"
- "fmla z30.s, z14.s, z7.s[1]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z19.s, z15.s, z4.s[1]\n"
- "fmla z23.s, z15.s, z5.s[1]\n"
- "fmla z27.s, z15.s, z6.s[1]\n"
- "fmla z31.s, z15.s, z7.s[1]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z16.s, z8.s, z4.s[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "fmla z20.s, z8.s, z5.s[2]\n"
- "fmla z24.s, z8.s, z6.s[2]\n"
- "fmla z28.s, z8.s, z7.s[2]\n"
- "fmla z17.s, z9.s, z4.s[2]\n"
- "fmla z21.s, z9.s, z5.s[2]\n"
- "fmla z25.s, z9.s, z6.s[2]\n"
- "fmla z29.s, z9.s, z7.s[2]\n"
- "fmla z18.s, z10.s, z4.s[2]\n"
- "fmla z22.s, z10.s, z5.s[2]\n"
- "fmla z26.s, z10.s, z6.s[2]\n"
- "fmla z30.s, z10.s, z7.s[2]\n"
- "fmla z19.s, z11.s, z4.s[2]\n"
- "fmla z23.s, z11.s, z5.s[2]\n"
- "fmla z27.s, z11.s, z6.s[2]\n"
- "fmla z31.s, z11.s, z7.s[2]\n"
- "fmla z16.s, z12.s, z4.s[3]\n"
- "fmla z20.s, z12.s, z5.s[3]\n"
- "fmla z24.s, z12.s, z6.s[3]\n"
- "fmla z28.s, z12.s, z7.s[3]\n"
- "fmla z17.s, z13.s, z4.s[3]\n"
- "fmla z21.s, z13.s, z5.s[3]\n"
- "fmla z25.s, z13.s, z6.s[3]\n"
- "fmla z29.s, z13.s, z7.s[3]\n"
- "fmla z18.s, z14.s, z4.s[3]\n"
- "fmla z22.s, z14.s, z5.s[3]\n"
- "fmla z26.s, z14.s, z6.s[3]\n"
- "fmla z30.s, z14.s, z7.s[3]\n"
- "fmla z19.s, z15.s, z4.s[3]\n"
- "fmla z23.s, z15.s, z5.s[3]\n"
- "fmla z27.s, z15.s, z6.s[3]\n"
- "fmla z31.s, z15.s, z7.s[3]\n"
- "cbz %[blocks], 5f\n"
- "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z16.s, z8.s, z0.s[0]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z20.s, z8.s, z1.s[0]\n"
- "fmla z24.s, z8.s, z2.s[0]\n"
- "fmla z28.s, z8.s, z3.s[0]\n"
- "fmla z17.s, z9.s, z0.s[0]\n"
- "fmla z21.s, z9.s, z1.s[0]\n"
- "fmla z25.s, z9.s, z2.s[0]\n"
- "fmla z29.s, z9.s, z3.s[0]\n"
- "fmla z18.s, z10.s, z0.s[0]\n"
- "fmla z22.s, z10.s, z1.s[0]\n"
- "fmla z26.s, z10.s, z2.s[0]\n"
- "fmla z30.s, z10.s, z3.s[0]\n"
- "fmla z19.s, z11.s, z0.s[0]\n"
- "fmla z23.s, z11.s, z1.s[0]\n"
- "fmla z27.s, z11.s, z2.s[0]\n"
- "fmla z31.s, z11.s, z3.s[0]\n"
- "b.eq 5f\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z16.s, z12.s, z0.s[1]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z20.s, z12.s, z1.s[1]\n"
- "fmla z24.s, z12.s, z2.s[1]\n"
- "fmla z28.s, z12.s, z3.s[1]\n"
- "fmla z17.s, z13.s, z0.s[1]\n"
- "fmla z21.s, z13.s, z1.s[1]\n"
- "fmla z25.s, z13.s, z2.s[1]\n"
- "fmla z29.s, z13.s, z3.s[1]\n"
- "fmla z18.s, z14.s, z0.s[1]\n"
- "fmla z22.s, z14.s, z1.s[1]\n"
- "fmla z26.s, z14.s, z2.s[1]\n"
- "fmla z30.s, z14.s, z3.s[1]\n"
- "fmla z19.s, z15.s, z0.s[1]\n"
- "fmla z23.s, z15.s, z1.s[1]\n"
- "fmla z27.s, z15.s, z2.s[1]\n"
- "fmla z31.s, z15.s, z3.s[1]\n"
- "b.eq 5f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "fmla z16.s, z8.s, z0.s[2]\n"
- "fmla z20.s, z8.s, z1.s[2]\n"
- "fmla z24.s, z8.s, z2.s[2]\n"
- "fmla z28.s, z8.s, z3.s[2]\n"
- "fmla z17.s, z9.s, z0.s[2]\n"
- "fmla z21.s, z9.s, z1.s[2]\n"
- "fmla z25.s, z9.s, z2.s[2]\n"
- "fmla z29.s, z9.s, z3.s[2]\n"
- "fmla z18.s, z10.s, z0.s[2]\n"
- "fmla z22.s, z10.s, z1.s[2]\n"
- "fmla z26.s, z10.s, z2.s[2]\n"
- "fmla z30.s, z10.s, z3.s[2]\n"
- "fmla z19.s, z11.s, z0.s[2]\n"
- "fmla z23.s, z11.s, z1.s[2]\n"
- "fmla z27.s, z11.s, z2.s[2]\n"
- "fmla z31.s, z11.s, z3.s[2]\n"
- "b 5f\n"
- "4:\n"
- "fmla z16.s, z8.s, z0.s[0]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "fmla z20.s, z8.s, z1.s[0]\n"
- "ld1rqw z4.s, p6/z, [%[a_ptr0]]\n"
- "fmla z24.s, z8.s, z2.s[0]\n"
- "ld1rqw z5.s, p6/z, [a_ptr1]\n"
- "fmla z28.s, z8.s, z3.s[0]\n"
- "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
- "fmla z17.s, z9.s, z0.s[0]\n"
- "ld1rqw z6.s, p6/z, [a_ptr2]\n"
- "fmla z21.s, z9.s, z1.s[0]\n"
- "ld1rqw z7.s, p6/z, [a_ptr3]\n"
- "fmla z25.s, z9.s, z2.s[0]\n"
- "addvl %[a_ptr0], %[a_ptr0], #1\n"
- "fmla z29.s, z9.s, z3.s[0]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "fmla z18.s, z10.s, z0.s[0]\n"
- "addvl a_ptr1, a_ptr1, #1\n"
- "fmla z22.s, z10.s, z1.s[0]\n"
- "addvl a_ptr2, a_ptr2, #1\n"
- "fmla z26.s, z10.s, z2.s[0]\n"
- "addvl a_ptr3, a_ptr3, #1\n"
- "fmla z30.s, z10.s, z3.s[0]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z19.s, z11.s, z0.s[0]\n"
- "fmla z23.s, z11.s, z1.s[0]\n"
- "fmla z27.s, z11.s, z2.s[0]\n"
- "fmla z31.s, z11.s, z3.s[0]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z16.s, z12.s, z0.s[1]\n"
- "fmla z20.s, z12.s, z1.s[1]\n"
- "fmla z24.s, z12.s, z2.s[1]\n"
- "fmla z28.s, z12.s, z3.s[1]\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "fmla z17.s, z13.s, z0.s[1]\n"
- "fmla z21.s, z13.s, z1.s[1]\n"
- "fmla z25.s, z13.s, z2.s[1]\n"
- "fmla z29.s, z13.s, z3.s[1]\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "fmla z18.s, z14.s, z0.s[1]\n"
- "fmla z22.s, z14.s, z1.s[1]\n"
- "fmla z26.s, z14.s, z2.s[1]\n"
- "fmla z30.s, z14.s, z3.s[1]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z19.s, z15.s, z0.s[1]\n"
- "fmla z23.s, z15.s, z1.s[1]\n"
- "fmla z27.s, z15.s, z2.s[1]\n"
- "fmla z31.s, z15.s, z3.s[1]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z16.s, z8.s, z0.s[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "fmla z20.s, z8.s, z1.s[2]\n"
- "fmla z24.s, z8.s, z2.s[2]\n"
- "fmla z28.s, z8.s, z3.s[2]\n"
- "fmla z17.s, z9.s, z0.s[2]\n"
- "fmla z21.s, z9.s, z1.s[2]\n"
- "fmla z25.s, z9.s, z2.s[2]\n"
- "fmla z29.s, z9.s, z3.s[2]\n"
- "fmla z18.s, z10.s, z0.s[2]\n"
- "fmla z22.s, z10.s, z1.s[2]\n"
- "fmla z26.s, z10.s, z2.s[2]\n"
- "fmla z30.s, z10.s, z3.s[2]\n"
- "fmla z19.s, z11.s, z0.s[2]\n"
- "fmla z23.s, z11.s, z1.s[2]\n"
- "fmla z27.s, z11.s, z2.s[2]\n"
- "fmla z31.s, z11.s, z3.s[2]\n"
- "fmla z16.s, z12.s, z0.s[3]\n"
- "fmla z20.s, z12.s, z1.s[3]\n"
- "fmla z24.s, z12.s, z2.s[3]\n"
- "fmla z28.s, z12.s, z3.s[3]\n"
- "fmla z17.s, z13.s, z0.s[3]\n"
- "fmla z21.s, z13.s, z1.s[3]\n"
- "fmla z25.s, z13.s, z2.s[3]\n"
- "fmla z29.s, z13.s, z3.s[3]\n"
- "fmla z18.s, z14.s, z0.s[3]\n"
- "fmla z22.s, z14.s, z1.s[3]\n"
- "fmla z26.s, z14.s, z2.s[3]\n"
- "fmla z30.s, z14.s, z3.s[3]\n"
- "fmla z19.s, z15.s, z0.s[3]\n"
- "fmla z23.s, z15.s, z1.s[3]\n"
- "fmla z27.s, z15.s, z2.s[3]\n"
- "fmla z31.s, z15.s, z3.s[3]\n"
- "cbz %[blocks], 5f\n"
- "ld1w z8.s, p7/z, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "fmla z16.s, z8.s, z4.s[0]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "fmla z20.s, z8.s, z5.s[0]\n"
- "fmla z24.s, z8.s, z6.s[0]\n"
- "fmla z28.s, z8.s, z7.s[0]\n"
- "fmla z17.s, z9.s, z4.s[0]\n"
- "fmla z21.s, z9.s, z5.s[0]\n"
- "fmla z25.s, z9.s, z6.s[0]\n"
- "fmla z29.s, z9.s, z7.s[0]\n"
- "fmla z18.s, z10.s, z4.s[0]\n"
- "fmla z22.s, z10.s, z5.s[0]\n"
- "fmla z26.s, z10.s, z6.s[0]\n"
- "fmla z30.s, z10.s, z7.s[0]\n"
- "fmla z19.s, z11.s, z4.s[0]\n"
- "fmla z23.s, z11.s, z5.s[0]\n"
- "fmla z27.s, z11.s, z6.s[0]\n"
- "fmla z31.s, z11.s, z7.s[0]\n"
- "b.eq 5f\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "fmla z16.s, z12.s, z4.s[1]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "fmla z20.s, z12.s, z5.s[1]\n"
- "fmla z24.s, z12.s, z6.s[1]\n"
- "fmla z28.s, z12.s, z7.s[1]\n"
- "fmla z17.s, z13.s, z4.s[1]\n"
- "fmla z21.s, z13.s, z5.s[1]\n"
- "fmla z25.s, z13.s, z6.s[1]\n"
- "fmla z29.s, z13.s, z7.s[1]\n"
- "fmla z18.s, z14.s, z4.s[1]\n"
- "fmla z22.s, z14.s, z5.s[1]\n"
- "fmla z26.s, z14.s, z6.s[1]\n"
- "fmla z30.s, z14.s, z7.s[1]\n"
- "fmla z19.s, z15.s, z4.s[1]\n"
- "fmla z23.s, z15.s, z5.s[1]\n"
- "fmla z27.s, z15.s, z6.s[1]\n"
- "fmla z31.s, z15.s, z7.s[1]\n"
- "b.eq 5f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "ld1w z8.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1w z9.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1w z10.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1w z11.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "fmla z16.s, z8.s, z4.s[2]\n"
- "fmla z20.s, z8.s, z5.s[2]\n"
- "fmla z24.s, z8.s, z6.s[2]\n"
- "fmla z28.s, z8.s, z7.s[2]\n"
- "fmla z17.s, z9.s, z4.s[2]\n"
- "fmla z21.s, z9.s, z5.s[2]\n"
- "fmla z25.s, z9.s, z6.s[2]\n"
- "fmla z29.s, z9.s, z7.s[2]\n"
- "fmla z18.s, z10.s, z4.s[2]\n"
- "fmla z22.s, z10.s, z5.s[2]\n"
- "fmla z26.s, z10.s, z6.s[2]\n"
- "fmla z30.s, z10.s, z7.s[2]\n"
- "fmla z19.s, z11.s, z4.s[2]\n"
- "fmla z23.s, z11.s, z5.s[2]\n"
- "fmla z27.s, z11.s, z6.s[2]\n"
- "fmla z31.s, z11.s, z7.s[2]\n"
- "5:\n"
- "ld1rw z14.s, p7/z, [%[minptr]]\n"
- "ld1rw z15.s, p7/z, [%[maxptr]]\n"
- "fmax z16.s, p7/m, z16.s, z14.s\n"
- "fmax z17.s, p7/m, z17.s, z14.s\n"
- "fmax z18.s, p7/m, z18.s, z14.s\n"
- "fmax z19.s, p7/m, z19.s, z14.s\n"
- "fmin z16.s, p7/m, z16.s, z15.s\n"
- "fmin z17.s, p7/m, z17.s, z15.s\n"
- "fmin z18.s, p7/m, z18.s, z15.s\n"
- "fmin z19.s, p7/m, z19.s, z15.s\n"
- "st1w z16.s, p0, [%[c_ptr0]]\n"
- "fmax z20.s, p7/m, z20.s, z14.s\n"
- "fmax z21.s, p7/m, z21.s, z14.s\n"
- "fmax z22.s, p7/m, z22.s, z14.s\n"
- "st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n"
- "fmax z23.s, p7/m, z23.s, z14.s\n"
- "fmin z20.s, p7/m, z20.s, z15.s\n"
- "fmin z21.s, p7/m, z21.s, z15.s\n"
- "st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n"
- "fmin z22.s, p7/m, z22.s, z15.s\n"
- "fmin z23.s, p7/m, z23.s, z15.s\n"
- "fmax z24.s, p7/m, z24.s, z14.s\n"
- "st1w z19.s, p3, [%[c_ptr0], #3, MUL VL]\n"
- "fmax z25.s, p7/m, z25.s, z14.s\n"
- "addvl %[c_ptr0], %[c_ptr0], #4\n"
- "fmax z26.s, p7/m, z26.s, z14.s\n"
- "st1w z20.s, p0, [c_ptr1]\n"
- "fmin z24.s, p7/m, z24.s, z15.s\n"
- "fmin z25.s, p7/m, z25.s, z15.s\n"
- "fmax z27.s, p7/m, z27.s, z14.s\n"
- "st1w z21.s, p1, [c_ptr1, #1, MUL VL]\n"
- "fmin z26.s, p7/m, z26.s, z15.s\n"
- "fmax z28.s, p7/m, z28.s, z14.s\n"
- "fmax z29.s, p7/m, z29.s, z14.s\n"
- "st1w z22.s, p2, [c_ptr1, #2, MUL VL]\n"
- "fmin z27.s, p7/m, z27.s, z15.s\n"
- "fmax z30.s, p7/m, z30.s, z14.s\n"
- "fmin z28.s, p7/m, z28.s, z15.s\n"
- "st1w z23.s, p3, [c_ptr1, #3, MUL VL]\n"
- "fmin z29.s, p7/m, z29.s, z15.s\n"
- "fmax z31.s, p7/m, z31.s, z14.s\n"
- "fmin z30.s, p7/m, z30.s, z15.s\n"
- "st1w z24.s, p0, [c_ptr2]\n"
- "fmin z31.s, p7/m, z31.s, z15.s\n"
- "st1w z25.s, p1, [c_ptr2, #1, MUL VL]\n"
- "st1w z26.s, p2, [c_ptr2, #2, MUL VL]\n"
- "st1w z27.s, p3, [c_ptr2, #3, MUL VL]\n"
- "st1w z28.s, p0, [c_ptr3]\n"
- "st1w z29.s, p1, [c_ptr3, #1, MUL VL]\n"
- "st1w z30.s, p2, [c_ptr3, #2, MUL VL]\n"
- "st1w z31.s, p3, [c_ptr3, #3, MUL VL]\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq a_ptr3\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- ".unreq c_ptr3\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks)
- : [width] "r" (width), [accumulate] "r" (static_cast<uint64_t>(accumulate)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers)
- : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "x4", "x5", "cc", "memory"
- );
- break;
- }
-
- }
- }
-}
-
-} // namespace arm_gemm
-
-#endif // __ARM_FEATURE_SVE
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_4VLx4.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_6x4VL.hpp
similarity index 65%
rename from src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_4VLx4.hpp
rename to src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_6x4VL.hpp
index 1bc8021..f0cc70b 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_4VLx4.hpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_6x4VL.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2018-2020 Arm Limited.
+ * Copyright (c) 2019-2020 Arm Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -10,42 +10,48 @@
* sell copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
*/
#pragma once
-
#ifdef __ARM_FEATURE_SVE
-
#include "../std_transforms_sve.hpp"
+#define ARGLIST \
+ unsigned int, const unsigned int *, \
+ IndirectInputArg<float>, \
+ size_t, size_t, \
+ const float *, \
+ IndirectOutputArg<float>, \
+ const float *, Activation, bool
+
namespace arm_gemm
{
// Actual kernel implementations
-void sve_hybrid_fp32_mla_4VLx4(const float *, int, const float *, float *, int, int, int, int, const float *, Activation, bool);
+void sve_hybrid_fp32_mla_6x4VL( ARGLIST );
-class hybrid_fp32_mla_4VLx4
+class cls_sve_hybrid_fp32_mla_6x4VL
{
public:
typedef float operand_type;
typedef float result_type;
- typedef void (*kern_type)(const float *, int, const float *, float *, int, int, int, int, const float *, Activation, bool);
+ typedef void (*kern_type)( ARGLIST );
/* Kernel blocking parameters */
static constexpr unsigned int out_height()
{
- return 4;
+ return 6;
}
static unsigned int out_width()
@@ -63,27 +69,17 @@
return true;
}
- static constexpr bool supports_bias()
- {
- return true;
- }
-
- static constexpr bool supports_activation()
- {
- return true;
- }
-
- StdTransformsSVE<operand_type, result_type, 4, 4, 1> transforms = {};
+ StdTransformsSVE<operand_type, result_type, 6, 4, 1> transforms = {};
// Default to the generic kernel
- kern_type kernel=sve_hybrid_fp32_mla_4VLx4;
+ kern_type kernel=sve_hybrid_fp32_mla_6x4VL;
- hybrid_fp32_mla_4VLx4(const CPUInfo *)
+ cls_sve_hybrid_fp32_mla_6x4VL(const CPUInfo *)
{
-
}
};
} // namespace arm_gemm
+#undef ARGLIST
#endif // __ARM_FEATURE_SVE
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_6x4VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_6x4VL/generic.cpp
new file mode 100644
index 0000000..3a6422a
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_6x4VL/generic.cpp
@@ -0,0 +1,2236 @@
+/*
+ * Copyright (c) 2019-2020 Arm Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+#ifdef __ARM_FEATURE_SVE
+
+#include "arm_gemm.hpp"
+#include "../../utils.hpp"
+
+#include <cassert>
+
+namespace arm_gemm {
+
+void sve_hybrid_fp32_mla_6x4VL (
+ unsigned int num_strings, const unsigned int *string_lengths, IndirectInputArg<float> A_arg,
+ size_t M, size_t N, const float *B_ptr, IndirectOutputArg<float> output_arg,
+ const float *bias, Activation act, bool accumulate
+)
+{
+ struct KernelArgs {
+ float maxval = static_cast<float>(std::numeric_limits<float>::infinity());
+ float minval = - static_cast<float>(std::numeric_limits<float>::infinity());
+ unsigned int num_strings = {};
+ const unsigned int *string_lengths = {};
+ size_t N = {};
+ const float *B_ptr = {};
+ size_t output_offset = {};
+ size_t input_initial_col = {};
+ size_t input_offset = {};
+ } ka;
+
+ unsigned long flags=0;
+ void *output_ptr;
+ void *input_ptr;
+
+ if (output_arg.is_indirect) {
+ output_ptr=(void *)(output_arg.indirect.ptr);
+ ka.output_offset=output_arg.indirect.offset;
+ flags |= 0x4;
+ } else {
+ output_ptr=(void *)(output_arg.direct.base);
+ ka.output_offset=output_arg.direct.stride;
+ }
+
+ if (A_arg.is_indirect) {
+ input_ptr=(void *)(A_arg.indirect.ptr);
+ ka.input_offset=A_arg.indirect.start_row;
+ ka.input_initial_col=A_arg.indirect.start_col;
+ flags |= 0x8;
+ } else {
+ assert(num_strings==1);
+ input_ptr=(void *)(A_arg.direct.base);
+ ka.input_offset=A_arg.direct.stride;
+ }
+ if (accumulate) {
+ flags |= 0x1;
+ }
+ ka.num_strings = num_strings;
+ ka.string_lengths = string_lengths;
+ ka.N = N;
+ ka.B_ptr = B_ptr;
+ switch(act.type) {
+ default:
+ case Activation::Type::None:
+ break;
+ case Activation::Type::BoundedReLU:
+ ka.maxval = static_cast<float>(act.param1);
+ /* fall through */
+ case Activation::Type::ReLU:
+ ka.minval = 0;
+ flags |= 0x2;
+ break;
+ }
+ __asm__ __volatile__(
+ "ptrue p5.b\n"
+ "1:" // Row loop
+ "cmp %x[M], #0x6\n"
+ "bge 71f\n"
+ "cmp %x[M], #0x4\n"
+ "bgt 57f\n"
+ "beq 43f\n"
+ "cmp %x[M], #0x2\n"
+ "bgt 29f\n"
+ "beq 15f\n"
+ "ldr x16, [%x[args_ptr], %[offsetof_N]]\n"
+ "ldr x15, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "mov x14, %x[bias]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 2f\n"
+ "ldr x13, [%x[output_ptr], #0x0]\n"
+ "add x13, x13, x19, LSL #2\n"
+ "b 3f\n"
+ "2:" // Height 1: setup direct output
+ "mov x13, %x[output_ptr]\n"
+ "3:" // Height 1: Column loop
+ "mov x19, #0x0\n"
+ "whilelt p4.s, x19, x16\n"
+ "incw x19\n"
+ "whilelt p3.s, x19, x16\n"
+ "incw x19\n"
+ "whilelt p2.s, x19, x16\n"
+ "incw x19\n"
+ "whilelt p1.s, x19, x16\n"
+ "cbz x14, 4f\n"
+ "ld1w { z8.s }, p5/Z, [x14]\n"
+ "ld1w { z9.s }, p5/Z, [x14, #1, MUL VL]\n"
+ "ld1w { z10.s }, p5/Z, [x14, #2, MUL VL]\n"
+ "ld1w { z11.s }, p5/Z, [x14, #3, MUL VL]\n"
+ "addvl x14, x14, #4\n"
+ "b 6f\n"
+ "4:" // Height 1: no bias
+ "tbz %x[flags], #0, 5f\n"
+ "ld1w { z8.s }, p4/Z, [x13]\n"
+ "ld1w { z9.s }, p3/Z, [x13, #1, MUL VL]\n"
+ "ld1w { z10.s }, p2/Z, [x13, #2, MUL VL]\n"
+ "ld1w { z11.s }, p1/Z, [x13, #3, MUL VL]\n"
+ "b 6f\n"
+ "5:" // Height 1: no accumulate
+ "mov z8.b, #0x0\n"
+ "mov z9.b, #0x0\n"
+ "mov z10.b, #0x0\n"
+ "mov z11.b, #0x0\n"
+ "6:" // Height 1: setup done
+ "mov x12, #0x0\n"
+ "7:" // Height 1: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w11, [x20, x12, LSL #0x2]\n"
+ "tbz %x[flags], #3, 8f\n"
+ "ldr x20, [%x[input_ptr], x12, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x10, [x20, #0x0]\n"
+ "cbnz x12, 9f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x10, x10, x19, LSL #2\n"
+ "b 9f\n"
+ "8:" // Height 1: setup direct input
+ "mov x10, %x[input_ptr]\n"
+ "9:" // Height 1: input setup done
+ "cmp x11, #0x4\n"
+ "ble 11f\n"
+ "10:" // Height 1: Multiply loop: Main loop head
+ "ld1w { z6.s }, p5/Z, [x15]\n"
+ "whilelt p0.s, XZR, x11\n"
+ "ld1w { z7.s }, p5/Z, [x15, #1, MUL VL]\n"
+ "sub x11, x11, #0x4\n"
+ "ld1rqw { z0.s }, p0/Z, [x10]\n"
+ "fmla z8.s, z6.s, z0.s[0]\n"
+ "ld1w { z6.s }, p5/Z, [x15, #2, MUL VL]\n"
+ "add x10, x10, #0x10\n"
+ "fmla z9.s, z7.s, z0.s[0]\n"
+ "ld1w { z7.s }, p5/Z, [x15, #3, MUL VL]\n"
+ "cmp x11, #0x4\n"
+ "fmla z10.s, z6.s, z0.s[0]\n"
+ "ld1w { z6.s }, p5/Z, [x15, #4, MUL VL]\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "fmla z11.s, z7.s, z0.s[0]\n"
+ "ld1w { z7.s }, p5/Z, [x15, #5, MUL VL]\n"
+ "fmla z8.s, z6.s, z0.s[1]\n"
+ "ld1w { z6.s }, p5/Z, [x15, #6, MUL VL]\n"
+ "fmla z9.s, z7.s, z0.s[1]\n"
+ "ld1w { z7.s }, p5/Z, [x15, #7, MUL VL]\n"
+ "addvl x15, x15, #16\n"
+ "fmla z10.s, z6.s, z0.s[1]\n"
+ "ld1w { z6.s }, p5/Z, [x15, #-8, MUL VL]\n"
+ "fmla z11.s, z7.s, z0.s[1]\n"
+ "ld1w { z7.s }, p5/Z, [x15, #-7, MUL VL]\n"
+ "fmla z8.s, z6.s, z0.s[2]\n"
+ "ld1w { z6.s }, p5/Z, [x15, #-6, MUL VL]\n"
+ "fmla z9.s, z7.s, z0.s[2]\n"
+ "ld1w { z7.s }, p5/Z, [x15, #-5, MUL VL]\n"
+ "fmla z10.s, z6.s, z0.s[2]\n"
+ "ld1w { z6.s }, p5/Z, [x15, #-4, MUL VL]\n"
+ "fmla z11.s, z7.s, z0.s[2]\n"
+ "ld1w { z7.s }, p5/Z, [x15, #-3, MUL VL]\n"
+ "fmla z8.s, z6.s, z0.s[3]\n"
+ "ld1w { z6.s }, p5/Z, [x15, #-2, MUL VL]\n"
+ "fmla z9.s, z7.s, z0.s[3]\n"
+ "ld1w { z7.s }, p5/Z, [x15, #-1, MUL VL]\n"
+ "fmla z10.s, z6.s, z0.s[3]\n"
+ "fmla z11.s, z7.s, z0.s[3]\n"
+ "bgt 10b\n"
+ "11:" // Height 1: Multiply loop: Single iteration only
+ "ld1w { z6.s }, p5/Z, [x15]\n"
+ "whilelt p0.s, XZR, x11\n"
+ "ld1w { z7.s }, p5/Z, [x15, #1, MUL VL]\n"
+ "subs x11, x11, #0x1\n"
+ "ld1rqw { z0.s }, p0/Z, [x10]\n"
+ "fmla z8.s, z6.s, z0.s[0]\n"
+ "ld1w { z6.s }, p5/Z, [x15, #2, MUL VL]\n"
+ "add x10, x10, #0x10\n"
+ "fmla z9.s, z7.s, z0.s[0]\n"
+ "ld1w { z7.s }, p5/Z, [x15, #3, MUL VL]\n"
+ "addvl x15, x15, #4\n"
+ "fmla z10.s, z6.s, z0.s[0]\n"
+ "fmla z11.s, z7.s, z0.s[0]\n"
+ "ble 12f\n"
+ "ld1w { z6.s }, p5/Z, [x15]\n"
+ "fmla z8.s, z6.s, z0.s[1]\n"
+ "ld1w { z7.s }, p5/Z, [x15, #1, MUL VL]\n"
+ "subs x11, x11, #0x1\n"
+ "fmla z9.s, z7.s, z0.s[1]\n"
+ "ld1w { z6.s }, p5/Z, [x15, #2, MUL VL]\n"
+ "ld1w { z7.s }, p5/Z, [x15, #3, MUL VL]\n"
+ "fmla z10.s, z6.s, z0.s[1]\n"
+ "addvl x15, x15, #4\n"
+ "fmla z11.s, z7.s, z0.s[1]\n"
+ "ble 12f\n"
+ "ld1w { z6.s }, p5/Z, [x15]\n"
+ "fmla z8.s, z6.s, z0.s[2]\n"
+ "ld1w { z7.s }, p5/Z, [x15, #1, MUL VL]\n"
+ "subs x11, x11, #0x1\n"
+ "fmla z9.s, z7.s, z0.s[2]\n"
+ "ld1w { z6.s }, p5/Z, [x15, #2, MUL VL]\n"
+ "ld1w { z7.s }, p5/Z, [x15, #3, MUL VL]\n"
+ "fmla z10.s, z6.s, z0.s[2]\n"
+ "addvl x15, x15, #4\n"
+ "fmla z11.s, z7.s, z0.s[2]\n"
+ "ble 12f\n"
+ "ld1w { z6.s }, p5/Z, [x15]\n"
+ "fmla z8.s, z6.s, z0.s[3]\n"
+ "ld1w { z7.s }, p5/Z, [x15, #1, MUL VL]\n"
+ "ld1w { z6.s }, p5/Z, [x15, #2, MUL VL]\n"
+ "fmla z9.s, z7.s, z0.s[3]\n"
+ "ld1w { z7.s }, p5/Z, [x15, #3, MUL VL]\n"
+ "addvl x15, x15, #4\n"
+ "fmla z10.s, z6.s, z0.s[3]\n"
+ "fmla z11.s, z7.s, z0.s[3]\n"
+ "12:" // Height 1: Multiply loop: multiply skip
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x12, x12, #0x1\n"
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "cmp x12, x19\n"
+ "bne 7b\n"
+ "prfm pstl1keep, [x13, #0x0]\n"
+ "tbz %x[flags], #1, 13f\n"
+ "add x19, %x[args_ptr], %[offset_min]\n"
+ "ld1rw { z1.s }, p5/Z, [x19]\n"
+ "add x19, %x[args_ptr], %[offset_max]\n"
+ "ld1rw { z0.s }, p5/Z, [x19]\n"
+ "fmin z8.s, p5/M, z8.s, z0.s\n"
+ "fmin z9.s, p5/M, z9.s, z0.s\n"
+ "fmin z10.s, p5/M, z10.s, z0.s\n"
+ "fmin z11.s, p5/M, z11.s, z0.s\n"
+ "fmax z8.s, p5/M, z8.s, z1.s\n"
+ "fmax z9.s, p5/M, z9.s, z1.s\n"
+ "fmax z10.s, p5/M, z10.s, z1.s\n"
+ "fmax z11.s, p5/M, z11.s, z1.s\n"
+ "13:" // Height 1: No activation
+ "st1w { z8.s }, p4, [x13]\n"
+ "st1w { z9.s }, p3, [x13, #1, MUL VL]\n"
+ "st1w { z10.s }, p2, [x13, #2, MUL VL]\n"
+ "st1w { z11.s }, p1, [x13, #3, MUL VL]\n"
+ "addvl x13, x13, #4\n"
+ "14:" // Height 1: Writeback done
+ "mov x19, #0x0\n"
+ "incw x19, ALL, MUL #4\n"
+ "subs x16, x16, x19\n"
+ "bgt 3b\n"
+ "b 86f\n"
+ "15:" // Height 2
+ "ldr x16, [%x[args_ptr], %[offsetof_N]]\n"
+ "mov x14, %x[bias]\n"
+ "ldr x15, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 16f\n"
+ "ldr x13, [%x[output_ptr], #0x0]\n"
+ "add x13, x13, x19, LSL #2\n"
+ "ldr x9, [%x[output_ptr], #0x8]\n"
+ "add x9, x9, x19, LSL #2\n"
+ "b 17f\n"
+ "16:" // Height 2: setup direct output
+ "mov x13, %x[output_ptr]\n"
+ "add x9, x13, x19, LSL #2\n"
+ "17:" // Height 2: Column loop
+ "mov x19, #0x0\n"
+ "whilelt p4.s, x19, x16\n"
+ "incw x19\n"
+ "whilelt p3.s, x19, x16\n"
+ "incw x19\n"
+ "whilelt p2.s, x19, x16\n"
+ "incw x19\n"
+ "whilelt p1.s, x19, x16\n"
+ "cbz x14, 18f\n"
+ "ld1w { z8.s }, p5/Z, [x14]\n"
+ "mov z12.d, z8.d\n"
+ "ld1w { z9.s }, p5/Z, [x14, #1, MUL VL]\n"
+ "ld1w { z10.s }, p5/Z, [x14, #2, MUL VL]\n"
+ "mov z13.d, z9.d\n"
+ "ld1w { z11.s }, p5/Z, [x14, #3, MUL VL]\n"
+ "addvl x14, x14, #4\n"
+ "mov z14.d, z10.d\n"
+ "mov z15.d, z11.d\n"
+ "b 20f\n"
+ "18:" // Height 2: no bias
+ "tbz %x[flags], #0, 19f\n"
+ "ld1w { z8.s }, p4/Z, [x13]\n"
+ "ld1w { z9.s }, p3/Z, [x13, #1, MUL VL]\n"
+ "ld1w { z10.s }, p2/Z, [x13, #2, MUL VL]\n"
+ "ld1w { z11.s }, p1/Z, [x13, #3, MUL VL]\n"
+ "ld1w { z12.s }, p4/Z, [x9]\n"
+ "ld1w { z13.s }, p3/Z, [x9, #1, MUL VL]\n"
+ "ld1w { z14.s }, p2/Z, [x9, #2, MUL VL]\n"
+ "ld1w { z15.s }, p1/Z, [x9, #3, MUL VL]\n"
+ "b 20f\n"
+ "19:" // Height 2: no accumulate
+ "mov z8.b, #0x0\n"
+ "mov z9.b, #0x0\n"
+ "mov z10.b, #0x0\n"
+ "mov z11.b, #0x0\n"
+ "mov z12.b, #0x0\n"
+ "mov z13.b, #0x0\n"
+ "mov z14.b, #0x0\n"
+ "mov z15.b, #0x0\n"
+ "20:" // Height 2: setup done
+ "mov x12, #0x0\n"
+ "21:" // Height 2: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w11, [x20, x12, LSL #0x2]\n"
+ "tbz %x[flags], #3, 22f\n"
+ "ldr x20, [%x[input_ptr], x12, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x10, [x20, #0x0]\n"
+ "ldr x28, [x20, #0x8]\n"
+ "cbnz x12, 23f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x10, x10, x19, LSL #2\n"
+ "add x28, x28, x19, LSL #2\n"
+ "b 23f\n"
+ "22:" // Height 2: setup direct input
+ "mov x10, %x[input_ptr]\n"
+ "add x28, x10, x19, LSL #2\n"
+ "23:" // Height 2: input setup done
+ "cmp x11, #0x4\n"
+ "ble 25f\n"
+ "24:" // Height 2: Multiply loop: Main loop head
+ "ld1w { z6.s }, p5/Z, [x15]\n"
+ "whilelt p0.s, XZR, x11\n"
+ "ld1w { z7.s }, p5/Z, [x15, #1, MUL VL]\n"
+ "sub x11, x11, #0x4\n"
+ "ld1rqw { z0.s }, p0/Z, [x10]\n"
+ "fmla z8.s, z6.s, z0.s[0]\n"
+ "ld1rqw { z1.s }, p0/Z, [x28]\n"
+ "add x10, x10, #0x10\n"
+ "fmla z9.s, z7.s, z0.s[0]\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x28, x28, #0x10\n"
+ "fmla z12.s, z6.s, z1.s[0]\n"
+ "ld1w { z6.s }, p5/Z, [x15, #2, MUL VL]\n"
+ "cmp x11, #0x4\n"
+ "fmla z13.s, z7.s, z1.s[0]\n"
+ "ld1w { z7.s }, p5/Z, [x15, #3, MUL VL]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "fmla z10.s, z6.s, z0.s[0]\n"
+ "fmla z14.s, z6.s, z1.s[0]\n"
+ "ld1w { z6.s }, p5/Z, [x15, #4, MUL VL]\n"
+ "fmla z11.s, z7.s, z0.s[0]\n"
+ "fmla z15.s, z7.s, z1.s[0]\n"
+ "ld1w { z7.s }, p5/Z, [x15, #5, MUL VL]\n"
+ "fmla z8.s, z6.s, z0.s[1]\n"
+ "fmla z12.s, z6.s, z1.s[1]\n"
+ "ld1w { z6.s }, p5/Z, [x15, #6, MUL VL]\n"
+ "fmla z9.s, z7.s, z0.s[1]\n"
+ "fmla z13.s, z7.s, z1.s[1]\n"
+ "ld1w { z7.s }, p5/Z, [x15, #7, MUL VL]\n"
+ "addvl x15, x15, #16\n"
+ "fmla z10.s, z6.s, z0.s[1]\n"
+ "fmla z14.s, z6.s, z1.s[1]\n"
+ "ld1w { z6.s }, p5/Z, [x15, #-8, MUL VL]\n"
+ "fmla z11.s, z7.s, z0.s[1]\n"
+ "fmla z15.s, z7.s, z1.s[1]\n"
+ "ld1w { z7.s }, p5/Z, [x15, #-7, MUL VL]\n"
+ "fmla z8.s, z6.s, z0.s[2]\n"
+ "fmla z12.s, z6.s, z1.s[2]\n"
+ "ld1w { z6.s }, p5/Z, [x15, #-6, MUL VL]\n"
+ "fmla z9.s, z7.s, z0.s[2]\n"
+ "fmla z13.s, z7.s, z1.s[2]\n"
+ "ld1w { z7.s }, p5/Z, [x15, #-5, MUL VL]\n"
+ "fmla z10.s, z6.s, z0.s[2]\n"
+ "fmla z14.s, z6.s, z1.s[2]\n"
+ "ld1w { z6.s }, p5/Z, [x15, #-4, MUL VL]\n"
+ "fmla z11.s, z7.s, z0.s[2]\n"
+ "fmla z15.s, z7.s, z1.s[2]\n"
+ "ld1w { z7.s }, p5/Z, [x15, #-3, MUL VL]\n"
+ "fmla z8.s, z6.s, z0.s[3]\n"
+ "fmla z12.s, z6.s, z1.s[3]\n"
+ "ld1w { z6.s }, p5/Z, [x15, #-2, MUL VL]\n"
+ "fmla z9.s, z7.s, z0.s[3]\n"
+ "fmla z13.s, z7.s, z1.s[3]\n"
+ "ld1w { z7.s }, p5/Z, [x15, #-1, MUL VL]\n"
+ "fmla z10.s, z6.s, z0.s[3]\n"
+ "fmla z14.s, z6.s, z1.s[3]\n"
+ "fmla z11.s, z7.s, z0.s[3]\n"
+ "fmla z15.s, z7.s, z1.s[3]\n"
+ "bgt 24b\n"
+ "25:" // Height 2: Multiply loop: Single iteration only
+ "ld1w { z6.s }, p5/Z, [x15]\n"
+ "whilelt p0.s, XZR, x11\n"
+ "ld1w { z7.s }, p5/Z, [x15, #1, MUL VL]\n"
+ "subs x11, x11, #0x1\n"
+ "ld1rqw { z0.s }, p0/Z, [x10]\n"
+ "fmla z8.s, z6.s, z0.s[0]\n"
+ "ld1rqw { z1.s }, p0/Z, [x28]\n"
+ "add x10, x10, #0x10\n"
+ "fmla z9.s, z7.s, z0.s[0]\n"
+ "add x28, x28, #0x10\n"
+ "fmla z12.s, z6.s, z1.s[0]\n"
+ "ld1w { z6.s }, p5/Z, [x15, #2, MUL VL]\n"
+ "fmla z13.s, z7.s, z1.s[0]\n"
+ "ld1w { z7.s }, p5/Z, [x15, #3, MUL VL]\n"
+ "addvl x15, x15, #4\n"
+ "fmla z10.s, z6.s, z0.s[0]\n"
+ "fmla z14.s, z6.s, z1.s[0]\n"
+ "fmla z11.s, z7.s, z0.s[0]\n"
+ "fmla z15.s, z7.s, z1.s[0]\n"
+ "ble 26f\n"
+ "ld1w { z6.s }, p5/Z, [x15]\n"
+ "fmla z8.s, z6.s, z0.s[1]\n"
+ "ld1w { z7.s }, p5/Z, [x15, #1, MUL VL]\n"
+ "subs x11, x11, #0x1\n"
+ "fmla z12.s, z6.s, z1.s[1]\n"
+ "ld1w { z6.s }, p5/Z, [x15, #2, MUL VL]\n"
+ "fmla z9.s, z7.s, z0.s[1]\n"
+ "fmla z13.s, z7.s, z1.s[1]\n"
+ "ld1w { z7.s }, p5/Z, [x15, #3, MUL VL]\n"
+ "addvl x15, x15, #4\n"
+ "fmla z10.s, z6.s, z0.s[1]\n"
+ "fmla z14.s, z6.s, z1.s[1]\n"
+ "fmla z11.s, z7.s, z0.s[1]\n"
+ "fmla z15.s, z7.s, z1.s[1]\n"
+ "ble 26f\n"
+ "ld1w { z6.s }, p5/Z, [x15]\n"
+ "fmla z8.s, z6.s, z0.s[2]\n"
+ "ld1w { z7.s }, p5/Z, [x15, #1, MUL VL]\n"
+ "subs x11, x11, #0x1\n"
+ "fmla z12.s, z6.s, z1.s[2]\n"
+ "ld1w { z6.s }, p5/Z, [x15, #2, MUL VL]\n"
+ "fmla z9.s, z7.s, z0.s[2]\n"
+ "fmla z13.s, z7.s, z1.s[2]\n"
+ "ld1w { z7.s }, p5/Z, [x15, #3, MUL VL]\n"
+ "addvl x15, x15, #4\n"
+ "fmla z10.s, z6.s, z0.s[2]\n"
+ "fmla z14.s, z6.s, z1.s[2]\n"
+ "fmla z11.s, z7.s, z0.s[2]\n"
+ "fmla z15.s, z7.s, z1.s[2]\n"
+ "ble 26f\n"
+ "ld1w { z6.s }, p5/Z, [x15]\n"
+ "fmla z8.s, z6.s, z0.s[3]\n"
+ "ld1w { z7.s }, p5/Z, [x15, #1, MUL VL]\n"
+ "fmla z12.s, z6.s, z1.s[3]\n"
+ "ld1w { z6.s }, p5/Z, [x15, #2, MUL VL]\n"
+ "fmla z9.s, z7.s, z0.s[3]\n"
+ "fmla z13.s, z7.s, z1.s[3]\n"
+ "ld1w { z7.s }, p5/Z, [x15, #3, MUL VL]\n"
+ "addvl x15, x15, #4\n"
+ "fmla z10.s, z6.s, z0.s[3]\n"
+ "fmla z14.s, z6.s, z1.s[3]\n"
+ "fmla z11.s, z7.s, z0.s[3]\n"
+ "fmla z15.s, z7.s, z1.s[3]\n"
+ "26:" // Height 2: Multiply loop: multiply skip
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x12, x12, #0x1\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "cmp x12, x19\n"
+ "bne 21b\n"
+ "prfm pstl1keep, [x13, #0x0]\n"
+ "prfm pstl1keep, [x9, #0x0]\n"
+ "tbz %x[flags], #1, 27f\n"
+ "add x19, %x[args_ptr], %[offset_min]\n"
+ "ld1rw { z1.s }, p5/Z, [x19]\n"
+ "add x19, %x[args_ptr], %[offset_max]\n"
+ "ld1rw { z0.s }, p5/Z, [x19]\n"
+ "fmin z8.s, p5/M, z8.s, z0.s\n"
+ "fmin z9.s, p5/M, z9.s, z0.s\n"
+ "fmin z10.s, p5/M, z10.s, z0.s\n"
+ "fmin z11.s, p5/M, z11.s, z0.s\n"
+ "fmin z12.s, p5/M, z12.s, z0.s\n"
+ "fmax z8.s, p5/M, z8.s, z1.s\n"
+ "fmax z9.s, p5/M, z9.s, z1.s\n"
+ "fmax z10.s, p5/M, z10.s, z1.s\n"
+ "fmax z11.s, p5/M, z11.s, z1.s\n"
+ "fmax z12.s, p5/M, z12.s, z1.s\n"
+ "fmin z13.s, p5/M, z13.s, z0.s\n"
+ "fmin z14.s, p5/M, z14.s, z0.s\n"
+ "fmin z15.s, p5/M, z15.s, z0.s\n"
+ "fmax z13.s, p5/M, z13.s, z1.s\n"
+ "fmax z14.s, p5/M, z14.s, z1.s\n"
+ "fmax z15.s, p5/M, z15.s, z1.s\n"
+ "27:" // Height 2: No activation
+ "st1w { z8.s }, p4, [x13]\n"
+ "st1w { z9.s }, p3, [x13, #1, MUL VL]\n"
+ "st1w { z10.s }, p2, [x13, #2, MUL VL]\n"
+ "st1w { z11.s }, p1, [x13, #3, MUL VL]\n"
+ "addvl x13, x13, #4\n"
+ "st1w { z12.s }, p4, [x9]\n"
+ "st1w { z13.s }, p3, [x9, #1, MUL VL]\n"
+ "st1w { z14.s }, p2, [x9, #2, MUL VL]\n"
+ "st1w { z15.s }, p1, [x9, #3, MUL VL]\n"
+ "addvl x9, x9, #4\n"
+ "28:" // Height 2: Writeback done
+ "mov x19, #0x0\n"
+ "incw x19, ALL, MUL #4\n"
+ "subs x16, x16, x19\n"
+ "bgt 17b\n"
+ "b 86f\n"
+ "29:" // Height 3
+ "ldr x16, [%x[args_ptr], %[offsetof_N]]\n"
+ "mov x14, %x[bias]\n"
+ "ldr x15, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 30f\n"
+ "ldr x13, [%x[output_ptr], #0x0]\n"
+ "add x13, x13, x19, LSL #2\n"
+ "ldr x9, [%x[output_ptr], #0x8]\n"
+ "ldr x27, [%x[output_ptr], #0x10]\n"
+ "add x9, x9, x19, LSL #2\n"
+ "add x27, x27, x19, LSL #2\n"
+ "b 31f\n"
+ "30:" // Height 3: setup direct output
+ "mov x13, %x[output_ptr]\n"
+ "add x9, x13, x19, LSL #2\n"
+ "add x27, x9, x19, LSL #2\n"
+ "31:" // Height 3: Column loop
+ "mov x19, #0x0\n"
+ "whilelt p4.s, x19, x16\n"
+ "incw x19\n"
+ "whilelt p3.s, x19, x16\n"
+ "incw x19\n"
+ "whilelt p2.s, x19, x16\n"
+ "incw x19\n"
+ "whilelt p1.s, x19, x16\n"
+ "cbz x14, 32f\n"
+ "ld1w { z8.s }, p5/Z, [x14]\n"
+ "mov z12.d, z8.d\n"
+ "ld1w { z9.s }, p5/Z, [x14, #1, MUL VL]\n"
+ "mov z16.d, z8.d\n"
+ "ld1w { z10.s }, p5/Z, [x14, #2, MUL VL]\n"
+ "ld1w { z11.s }, p5/Z, [x14, #3, MUL VL]\n"
+ "mov z13.d, z9.d\n"
+ "addvl x14, x14, #4\n"
+ "mov z17.d, z9.d\n"
+ "mov z14.d, z10.d\n"
+ "mov z15.d, z11.d\n"
+ "mov z18.d, z10.d\n"
+ "mov z19.d, z11.d\n"
+ "b 34f\n"
+ "32:" // Height 3: no bias
+ "tbz %x[flags], #0, 33f\n"
+ "ld1w { z8.s }, p4/Z, [x13]\n"
+ "ld1w { z9.s }, p3/Z, [x13, #1, MUL VL]\n"
+ "ld1w { z10.s }, p2/Z, [x13, #2, MUL VL]\n"
+ "ld1w { z11.s }, p1/Z, [x13, #3, MUL VL]\n"
+ "ld1w { z12.s }, p4/Z, [x9]\n"
+ "ld1w { z13.s }, p3/Z, [x9, #1, MUL VL]\n"
+ "ld1w { z14.s }, p2/Z, [x9, #2, MUL VL]\n"
+ "ld1w { z15.s }, p1/Z, [x9, #3, MUL VL]\n"
+ "ld1w { z16.s }, p4/Z, [x27]\n"
+ "ld1w { z17.s }, p3/Z, [x27, #1, MUL VL]\n"
+ "ld1w { z18.s }, p2/Z, [x27, #2, MUL VL]\n"
+ "ld1w { z19.s }, p1/Z, [x27, #3, MUL VL]\n"
+ "b 34f\n"
+ "33:" // Height 3: no accumulate
+ "mov z8.b, #0x0\n"
+ "mov z9.b, #0x0\n"
+ "mov z10.b, #0x0\n"
+ "mov z11.b, #0x0\n"
+ "mov z12.b, #0x0\n"
+ "mov z13.b, #0x0\n"
+ "mov z14.b, #0x0\n"
+ "mov z15.b, #0x0\n"
+ "mov z16.b, #0x0\n"
+ "mov z17.b, #0x0\n"
+ "mov z18.b, #0x0\n"
+ "mov z19.b, #0x0\n"
+ "34:" // Height 3: setup done
+ "mov x12, #0x0\n"
+ "35:" // Height 3: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w11, [x20, x12, LSL #0x2]\n"
+ "tbz %x[flags], #3, 36f\n"
+ "ldr x20, [%x[input_ptr], x12, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x10, [x20, #0x0]\n"
+ "ldr x28, [x20, #0x8]\n"
+ "ldr x26, [x20, #0x10]\n"
+ "cbnz x12, 37f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x10, x10, x19, LSL #2\n"
+ "add x28, x28, x19, LSL #2\n"
+ "add x26, x26, x19, LSL #2\n"
+ "b 37f\n"
+ "36:" // Height 3: setup direct input
+ "mov x10, %x[input_ptr]\n"
+ "add x28, x10, x19, LSL #2\n"
+ "add x26, x28, x19, LSL #2\n"
+ "37:" // Height 3: input setup done
+ "cmp x11, #0x4\n"
+ "ble 39f\n"
+ "38:" // Height 3: Multiply loop: Main loop head
+ "ld1w { z6.s }, p5/Z, [x15]\n"
+ "whilelt p0.s, XZR, x11\n"
+ "ld1w { z7.s }, p5/Z, [x15, #1, MUL VL]\n"
+ "sub x11, x11, #0x4\n"
+ "ld1rqw { z0.s }, p0/Z, [x10]\n"
+ "fmla z8.s, z6.s, z0.s[0]\n"
+ "ld1rqw { z1.s }, p0/Z, [x28]\n"
+ "add x10, x10, #0x10\n"
+ "fmla z9.s, z7.s, z0.s[0]\n"
+ "ld1rqw { z2.s }, p0/Z, [x26]\n"
+ "add x28, x28, #0x10\n"
+ "fmla z12.s, z6.s, z1.s[0]\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x26, x26, #0x10\n"
+ "fmla z16.s, z6.s, z2.s[0]\n"
+ "ld1w { z6.s }, p5/Z, [x15, #2, MUL VL]\n"
+ "cmp x11, #0x4\n"
+ "fmla z13.s, z7.s, z1.s[0]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "fmla z17.s, z7.s, z2.s[0]\n"
+ "ld1w { z7.s }, p5/Z, [x15, #3, MUL VL]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "fmla z10.s, z6.s, z0.s[0]\n"
+ "fmla z14.s, z6.s, z1.s[0]\n"
+ "fmla z18.s, z6.s, z2.s[0]\n"
+ "ld1w { z6.s }, p5/Z, [x15, #4, MUL VL]\n"
+ "fmla z11.s, z7.s, z0.s[0]\n"
+ "fmla z15.s, z7.s, z1.s[0]\n"
+ "fmla z19.s, z7.s, z2.s[0]\n"
+ "ld1w { z7.s }, p5/Z, [x15, #5, MUL VL]\n"
+ "fmla z8.s, z6.s, z0.s[1]\n"
+ "fmla z12.s, z6.s, z1.s[1]\n"
+ "fmla z16.s, z6.s, z2.s[1]\n"
+ "ld1w { z6.s }, p5/Z, [x15, #6, MUL VL]\n"
+ "fmla z9.s, z7.s, z0.s[1]\n"
+ "fmla z13.s, z7.s, z1.s[1]\n"
+ "fmla z17.s, z7.s, z2.s[1]\n"
+ "ld1w { z7.s }, p5/Z, [x15, #7, MUL VL]\n"
+ "addvl x15, x15, #16\n"
+ "fmla z10.s, z6.s, z0.s[1]\n"
+ "fmla z14.s, z6.s, z1.s[1]\n"
+ "fmla z18.s, z6.s, z2.s[1]\n"
+ "ld1w { z6.s }, p5/Z, [x15, #-8, MUL VL]\n"
+ "fmla z11.s, z7.s, z0.s[1]\n"
+ "fmla z15.s, z7.s, z1.s[1]\n"
+ "fmla z19.s, z7.s, z2.s[1]\n"
+ "ld1w { z7.s }, p5/Z, [x15, #-7, MUL VL]\n"
+ "fmla z8.s, z6.s, z0.s[2]\n"
+ "fmla z12.s, z6.s, z1.s[2]\n"
+ "fmla z16.s, z6.s, z2.s[2]\n"
+ "ld1w { z6.s }, p5/Z, [x15, #-6, MUL VL]\n"
+ "fmla z9.s, z7.s, z0.s[2]\n"
+ "fmla z13.s, z7.s, z1.s[2]\n"
+ "fmla z17.s, z7.s, z2.s[2]\n"
+ "ld1w { z7.s }, p5/Z, [x15, #-5, MUL VL]\n"
+ "fmla z10.s, z6.s, z0.s[2]\n"
+ "fmla z14.s, z6.s, z1.s[2]\n"
+ "fmla z18.s, z6.s, z2.s[2]\n"
+ "ld1w { z6.s }, p5/Z, [x15, #-4, MUL VL]\n"
+ "fmla z11.s, z7.s, z0.s[2]\n"
+ "fmla z15.s, z7.s, z1.s[2]\n"
+ "fmla z19.s, z7.s, z2.s[2]\n"
+ "ld1w { z7.s }, p5/Z, [x15, #-3, MUL VL]\n"
+ "fmla z8.s, z6.s, z0.s[3]\n"
+ "fmla z12.s, z6.s, z1.s[3]\n"
+ "fmla z16.s, z6.s, z2.s[3]\n"
+ "ld1w { z6.s }, p5/Z, [x15, #-2, MUL VL]\n"
+ "fmla z9.s, z7.s, z0.s[3]\n"
+ "fmla z13.s, z7.s, z1.s[3]\n"
+ "fmla z17.s, z7.s, z2.s[3]\n"
+ "ld1w { z7.s }, p5/Z, [x15, #-1, MUL VL]\n"
+ "fmla z10.s, z6.s, z0.s[3]\n"
+ "fmla z14.s, z6.s, z1.s[3]\n"
+ "fmla z18.s, z6.s, z2.s[3]\n"
+ "fmla z11.s, z7.s, z0.s[3]\n"
+ "fmla z15.s, z7.s, z1.s[3]\n"
+ "fmla z19.s, z7.s, z2.s[3]\n"
+ "bgt 38b\n"
+ "39:" // Height 3: Multiply loop: Single iteration only
+ "ld1w { z6.s }, p5/Z, [x15]\n"
+ "whilelt p0.s, XZR, x11\n"
+ "ld1w { z7.s }, p5/Z, [x15, #1, MUL VL]\n"
+ "subs x11, x11, #0x1\n"
+ "ld1rqw { z0.s }, p0/Z, [x10]\n"
+ "fmla z8.s, z6.s, z0.s[0]\n"
+ "ld1rqw { z1.s }, p0/Z, [x28]\n"
+ "add x10, x10, #0x10\n"
+ "fmla z9.s, z7.s, z0.s[0]\n"
+ "ld1rqw { z2.s }, p0/Z, [x26]\n"
+ "add x28, x28, #0x10\n"
+ "fmla z12.s, z6.s, z1.s[0]\n"
+ "add x26, x26, #0x10\n"
+ "fmla z13.s, z7.s, z1.s[0]\n"
+ "fmla z16.s, z6.s, z2.s[0]\n"
+ "ld1w { z6.s }, p5/Z, [x15, #2, MUL VL]\n"
+ "fmla z17.s, z7.s, z2.s[0]\n"
+ "ld1w { z7.s }, p5/Z, [x15, #3, MUL VL]\n"
+ "addvl x15, x15, #4\n"
+ "fmla z10.s, z6.s, z0.s[0]\n"
+ "fmla z14.s, z6.s, z1.s[0]\n"
+ "fmla z18.s, z6.s, z2.s[0]\n"
+ "fmla z11.s, z7.s, z0.s[0]\n"
+ "fmla z15.s, z7.s, z1.s[0]\n"
+ "fmla z19.s, z7.s, z2.s[0]\n"
+ "ble 40f\n"
+ "ld1w { z6.s }, p5/Z, [x15]\n"
+ "fmla z8.s, z6.s, z0.s[1]\n"
+ "ld1w { z7.s }, p5/Z, [x15, #1, MUL VL]\n"
+ "subs x11, x11, #0x1\n"
+ "fmla z12.s, z6.s, z1.s[1]\n"
+ "fmla z16.s, z6.s, z2.s[1]\n"
+ "ld1w { z6.s }, p5/Z, [x15, #2, MUL VL]\n"
+ "fmla z9.s, z7.s, z0.s[1]\n"
+ "fmla z13.s, z7.s, z1.s[1]\n"
+ "fmla z17.s, z7.s, z2.s[1]\n"
+ "ld1w { z7.s }, p5/Z, [x15, #3, MUL VL]\n"
+ "addvl x15, x15, #4\n"
+ "fmla z10.s, z6.s, z0.s[1]\n"
+ "fmla z14.s, z6.s, z1.s[1]\n"
+ "fmla z18.s, z6.s, z2.s[1]\n"
+ "fmla z11.s, z7.s, z0.s[1]\n"
+ "fmla z15.s, z7.s, z1.s[1]\n"
+ "fmla z19.s, z7.s, z2.s[1]\n"
+ "ble 40f\n"
+ "ld1w { z6.s }, p5/Z, [x15]\n"
+ "fmla z8.s, z6.s, z0.s[2]\n"
+ "ld1w { z7.s }, p5/Z, [x15, #1, MUL VL]\n"
+ "subs x11, x11, #0x1\n"
+ "fmla z12.s, z6.s, z1.s[2]\n"
+ "fmla z16.s, z6.s, z2.s[2]\n"
+ "ld1w { z6.s }, p5/Z, [x15, #2, MUL VL]\n"
+ "fmla z9.s, z7.s, z0.s[2]\n"
+ "fmla z13.s, z7.s, z1.s[2]\n"
+ "fmla z17.s, z7.s, z2.s[2]\n"
+ "ld1w { z7.s }, p5/Z, [x15, #3, MUL VL]\n"
+ "addvl x15, x15, #4\n"
+ "fmla z10.s, z6.s, z0.s[2]\n"
+ "fmla z14.s, z6.s, z1.s[2]\n"
+ "fmla z18.s, z6.s, z2.s[2]\n"
+ "fmla z11.s, z7.s, z0.s[2]\n"
+ "fmla z15.s, z7.s, z1.s[2]\n"
+ "fmla z19.s, z7.s, z2.s[2]\n"
+ "ble 40f\n"
+ "ld1w { z6.s }, p5/Z, [x15]\n"
+ "fmla z8.s, z6.s, z0.s[3]\n"
+ "ld1w { z7.s }, p5/Z, [x15, #1, MUL VL]\n"
+ "fmla z12.s, z6.s, z1.s[3]\n"
+ "fmla z16.s, z6.s, z2.s[3]\n"
+ "ld1w { z6.s }, p5/Z, [x15, #2, MUL VL]\n"
+ "fmla z9.s, z7.s, z0.s[3]\n"
+ "fmla z13.s, z7.s, z1.s[3]\n"
+ "fmla z17.s, z7.s, z2.s[3]\n"
+ "ld1w { z7.s }, p5/Z, [x15, #3, MUL VL]\n"
+ "addvl x15, x15, #4\n"
+ "fmla z10.s, z6.s, z0.s[3]\n"
+ "fmla z14.s, z6.s, z1.s[3]\n"
+ "fmla z18.s, z6.s, z2.s[3]\n"
+ "fmla z11.s, z7.s, z0.s[3]\n"
+ "fmla z15.s, z7.s, z1.s[3]\n"
+ "fmla z19.s, z7.s, z2.s[3]\n"
+ "40:" // Height 3: Multiply loop: multiply skip
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x12, x12, #0x1\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "cmp x12, x19\n"
+ "bne 35b\n"
+ "prfm pstl1keep, [x13, #0x0]\n"
+ "prfm pstl1keep, [x9, #0x0]\n"
+ "prfm pstl1keep, [x27, #0x0]\n"
+ "tbz %x[flags], #1, 41f\n"
+ "add x19, %x[args_ptr], %[offset_min]\n"
+ "ld1rw { z1.s }, p5/Z, [x19]\n"
+ "add x19, %x[args_ptr], %[offset_max]\n"
+ "ld1rw { z0.s }, p5/Z, [x19]\n"
+ "fmin z8.s, p5/M, z8.s, z0.s\n"
+ "fmin z9.s, p5/M, z9.s, z0.s\n"
+ "fmin z10.s, p5/M, z10.s, z0.s\n"
+ "fmin z11.s, p5/M, z11.s, z0.s\n"
+ "fmin z12.s, p5/M, z12.s, z0.s\n"
+ "fmax z8.s, p5/M, z8.s, z1.s\n"
+ "fmax z9.s, p5/M, z9.s, z1.s\n"
+ "fmax z10.s, p5/M, z10.s, z1.s\n"
+ "fmax z11.s, p5/M, z11.s, z1.s\n"
+ "fmax z12.s, p5/M, z12.s, z1.s\n"
+ "fmin z13.s, p5/M, z13.s, z0.s\n"
+ "fmin z14.s, p5/M, z14.s, z0.s\n"
+ "fmin z15.s, p5/M, z15.s, z0.s\n"
+ "fmin z16.s, p5/M, z16.s, z0.s\n"
+ "fmax z13.s, p5/M, z13.s, z1.s\n"
+ "fmax z14.s, p5/M, z14.s, z1.s\n"
+ "fmax z15.s, p5/M, z15.s, z1.s\n"
+ "fmax z16.s, p5/M, z16.s, z1.s\n"
+ "fmin z17.s, p5/M, z17.s, z0.s\n"
+ "fmin z18.s, p5/M, z18.s, z0.s\n"
+ "fmin z19.s, p5/M, z19.s, z0.s\n"
+ "fmax z17.s, p5/M, z17.s, z1.s\n"
+ "fmax z18.s, p5/M, z18.s, z1.s\n"
+ "fmax z19.s, p5/M, z19.s, z1.s\n"
+ "41:" // Height 3: No activation
+ "st1w { z8.s }, p4, [x13]\n"
+ "st1w { z9.s }, p3, [x13, #1, MUL VL]\n"
+ "st1w { z10.s }, p2, [x13, #2, MUL VL]\n"
+ "st1w { z11.s }, p1, [x13, #3, MUL VL]\n"
+ "addvl x13, x13, #4\n"
+ "st1w { z12.s }, p4, [x9]\n"
+ "st1w { z13.s }, p3, [x9, #1, MUL VL]\n"
+ "st1w { z14.s }, p2, [x9, #2, MUL VL]\n"
+ "st1w { z15.s }, p1, [x9, #3, MUL VL]\n"
+ "addvl x9, x9, #4\n"
+ "st1w { z16.s }, p4, [x27]\n"
+ "st1w { z17.s }, p3, [x27, #1, MUL VL]\n"
+ "st1w { z18.s }, p2, [x27, #2, MUL VL]\n"
+ "st1w { z19.s }, p1, [x27, #3, MUL VL]\n"
+ "addvl x27, x27, #4\n"
+ "42:" // Height 3: Writeback done
+ "mov x19, #0x0\n"
+ "incw x19, ALL, MUL #4\n"
+ "subs x16, x16, x19\n"
+ "bgt 31b\n"
+ "b 86f\n"
+ "43:" // Height 4
+ "ldr x16, [%x[args_ptr], %[offsetof_N]]\n"
+ "mov x14, %x[bias]\n"
+ "ldr x15, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 44f\n"
+ "ldr x13, [%x[output_ptr], #0x0]\n"
+ "add x13, x13, x19, LSL #2\n"
+ "ldr x9, [%x[output_ptr], #0x8]\n"
+ "ldr x27, [%x[output_ptr], #0x10]\n"
+ "add x9, x9, x19, LSL #2\n"
+ "ldr x25, [%x[output_ptr], #0x18]\n"
+ "add x27, x27, x19, LSL #2\n"
+ "add x25, x25, x19, LSL #2\n"
+ "b 45f\n"
+ "44:" // Height 4: setup direct output
+ "mov x13, %x[output_ptr]\n"
+ "add x9, x13, x19, LSL #2\n"
+ "add x27, x9, x19, LSL #2\n"
+ "add x25, x27, x19, LSL #2\n"
+ "45:" // Height 4: Column loop
+ "mov x19, #0x0\n"
+ "whilelt p4.s, x19, x16\n"
+ "incw x19\n"
+ "whilelt p3.s, x19, x16\n"
+ "incw x19\n"
+ "whilelt p2.s, x19, x16\n"
+ "incw x19\n"
+ "whilelt p1.s, x19, x16\n"
+ "cbz x14, 46f\n"
+ "ld1w { z8.s }, p5/Z, [x14]\n"
+ "mov z12.d, z8.d\n"
+ "ld1w { z9.s }, p5/Z, [x14, #1, MUL VL]\n"
+ "mov z16.d, z8.d\n"
+ "ld1w { z10.s }, p5/Z, [x14, #2, MUL VL]\n"
+ "mov z20.d, z8.d\n"
+ "ld1w { z11.s }, p5/Z, [x14, #3, MUL VL]\n"
+ "addvl x14, x14, #4\n"
+ "mov z13.d, z9.d\n"
+ "mov z17.d, z9.d\n"
+ "mov z14.d, z10.d\n"
+ "mov z15.d, z11.d\n"
+ "mov z18.d, z10.d\n"
+ "mov z19.d, z11.d\n"
+ "mov z21.d, z9.d\n"
+ "mov z22.d, z10.d\n"
+ "mov z23.d, z11.d\n"
+ "b 48f\n"
+ "46:" // Height 4: no bias
+ "tbz %x[flags], #0, 47f\n"
+ "ld1w { z8.s }, p4/Z, [x13]\n"
+ "ld1w { z9.s }, p3/Z, [x13, #1, MUL VL]\n"
+ "ld1w { z10.s }, p2/Z, [x13, #2, MUL VL]\n"
+ "ld1w { z11.s }, p1/Z, [x13, #3, MUL VL]\n"
+ "ld1w { z12.s }, p4/Z, [x9]\n"
+ "ld1w { z13.s }, p3/Z, [x9, #1, MUL VL]\n"
+ "ld1w { z14.s }, p2/Z, [x9, #2, MUL VL]\n"
+ "ld1w { z15.s }, p1/Z, [x9, #3, MUL VL]\n"
+ "ld1w { z16.s }, p4/Z, [x27]\n"
+ "ld1w { z17.s }, p3/Z, [x27, #1, MUL VL]\n"
+ "ld1w { z18.s }, p2/Z, [x27, #2, MUL VL]\n"
+ "ld1w { z19.s }, p1/Z, [x27, #3, MUL VL]\n"
+ "ld1w { z20.s }, p4/Z, [x25]\n"
+ "ld1w { z21.s }, p3/Z, [x25, #1, MUL VL]\n"
+ "ld1w { z22.s }, p2/Z, [x25, #2, MUL VL]\n"
+ "ld1w { z23.s }, p1/Z, [x25, #3, MUL VL]\n"
+ "b 48f\n"
+ "47:" // Height 4: no accumulate
+ "mov z8.b, #0x0\n"
+ "mov z9.b, #0x0\n"
+ "mov z10.b, #0x0\n"
+ "mov z11.b, #0x0\n"
+ "mov z12.b, #0x0\n"
+ "mov z13.b, #0x0\n"
+ "mov z14.b, #0x0\n"
+ "mov z15.b, #0x0\n"
+ "mov z16.b, #0x0\n"
+ "mov z17.b, #0x0\n"
+ "mov z18.b, #0x0\n"
+ "mov z19.b, #0x0\n"
+ "mov z20.b, #0x0\n"
+ "mov z21.b, #0x0\n"
+ "mov z22.b, #0x0\n"
+ "mov z23.b, #0x0\n"
+ "48:" // Height 4: setup done
+ "mov x12, #0x0\n"
+ "49:" // Height 4: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w11, [x20, x12, LSL #0x2]\n"
+ "tbz %x[flags], #3, 50f\n"
+ "ldr x20, [%x[input_ptr], x12, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x10, [x20, #0x0]\n"
+ "ldr x28, [x20, #0x8]\n"
+ "ldr x26, [x20, #0x10]\n"
+ "ldr x24, [x20, #0x18]\n"
+ "cbnz x12, 51f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x10, x10, x19, LSL #2\n"
+ "add x28, x28, x19, LSL #2\n"
+ "add x26, x26, x19, LSL #2\n"
+ "add x24, x24, x19, LSL #2\n"
+ "b 51f\n"
+ "50:" // Height 4: setup direct input
+ "mov x10, %x[input_ptr]\n"
+ "add x28, x10, x19, LSL #2\n"
+ "add x26, x28, x19, LSL #2\n"
+ "add x24, x26, x19, LSL #2\n"
+ "51:" // Height 4: input setup done
+ "cmp x11, #0x4\n"
+ "ble 53f\n"
+ "52:" // Height 4: Multiply loop: Main loop head
+ "ld1w { z6.s }, p5/Z, [x15]\n"
+ "whilelt p0.s, XZR, x11\n"
+ "ld1w { z7.s }, p5/Z, [x15, #1, MUL VL]\n"
+ "sub x11, x11, #0x4\n"
+ "ld1rqw { z0.s }, p0/Z, [x10]\n"
+ "fmla z8.s, z6.s, z0.s[0]\n"
+ "ld1rqw { z1.s }, p0/Z, [x28]\n"
+ "add x10, x10, #0x10\n"
+ "fmla z9.s, z7.s, z0.s[0]\n"
+ "ld1rqw { z2.s }, p0/Z, [x26]\n"
+ "add x28, x28, #0x10\n"
+ "fmla z12.s, z6.s, z1.s[0]\n"
+ "ld1rqw { z3.s }, p0/Z, [x24]\n"
+ "add x26, x26, #0x10\n"
+ "fmla z16.s, z6.s, z2.s[0]\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x24, x24, #0x10\n"
+ "fmla z13.s, z7.s, z1.s[0]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "cmp x11, #0x4\n"
+ "fmla z20.s, z6.s, z3.s[0]\n"
+ "ld1w { z6.s }, p5/Z, [x15, #2, MUL VL]\n"
+ "fmla z17.s, z7.s, z2.s[0]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "fmla z21.s, z7.s, z3.s[0]\n"
+ "ld1w { z7.s }, p5/Z, [x15, #3, MUL VL]\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "fmla z10.s, z6.s, z0.s[0]\n"
+ "fmla z14.s, z6.s, z1.s[0]\n"
+ "fmla z18.s, z6.s, z2.s[0]\n"
+ "fmla z22.s, z6.s, z3.s[0]\n"
+ "ld1w { z6.s }, p5/Z, [x15, #4, MUL VL]\n"
+ "fmla z11.s, z7.s, z0.s[0]\n"
+ "fmla z15.s, z7.s, z1.s[0]\n"
+ "fmla z19.s, z7.s, z2.s[0]\n"
+ "fmla z23.s, z7.s, z3.s[0]\n"
+ "ld1w { z7.s }, p5/Z, [x15, #5, MUL VL]\n"
+ "fmla z8.s, z6.s, z0.s[1]\n"
+ "fmla z12.s, z6.s, z1.s[1]\n"
+ "fmla z16.s, z6.s, z2.s[1]\n"
+ "fmla z20.s, z6.s, z3.s[1]\n"
+ "ld1w { z6.s }, p5/Z, [x15, #6, MUL VL]\n"
+ "fmla z9.s, z7.s, z0.s[1]\n"
+ "fmla z13.s, z7.s, z1.s[1]\n"
+ "fmla z17.s, z7.s, z2.s[1]\n"
+ "fmla z21.s, z7.s, z3.s[1]\n"
+ "ld1w { z7.s }, p5/Z, [x15, #7, MUL VL]\n"
+ "addvl x15, x15, #16\n"
+ "fmla z10.s, z6.s, z0.s[1]\n"
+ "fmla z14.s, z6.s, z1.s[1]\n"
+ "fmla z18.s, z6.s, z2.s[1]\n"
+ "fmla z22.s, z6.s, z3.s[1]\n"
+ "ld1w { z6.s }, p5/Z, [x15, #-8, MUL VL]\n"
+ "fmla z11.s, z7.s, z0.s[1]\n"
+ "fmla z15.s, z7.s, z1.s[1]\n"
+ "fmla z19.s, z7.s, z2.s[1]\n"
+ "fmla z23.s, z7.s, z3.s[1]\n"
+ "ld1w { z7.s }, p5/Z, [x15, #-7, MUL VL]\n"
+ "fmla z8.s, z6.s, z0.s[2]\n"
+ "fmla z12.s, z6.s, z1.s[2]\n"
+ "fmla z16.s, z6.s, z2.s[2]\n"
+ "fmla z20.s, z6.s, z3.s[2]\n"
+ "ld1w { z6.s }, p5/Z, [x15, #-6, MUL VL]\n"
+ "fmla z9.s, z7.s, z0.s[2]\n"
+ "fmla z13.s, z7.s, z1.s[2]\n"
+ "fmla z17.s, z7.s, z2.s[2]\n"
+ "fmla z21.s, z7.s, z3.s[2]\n"
+ "ld1w { z7.s }, p5/Z, [x15, #-5, MUL VL]\n"
+ "fmla z10.s, z6.s, z0.s[2]\n"
+ "fmla z14.s, z6.s, z1.s[2]\n"
+ "fmla z18.s, z6.s, z2.s[2]\n"
+ "fmla z22.s, z6.s, z3.s[2]\n"
+ "ld1w { z6.s }, p5/Z, [x15, #-4, MUL VL]\n"
+ "fmla z11.s, z7.s, z0.s[2]\n"
+ "fmla z15.s, z7.s, z1.s[2]\n"
+ "fmla z19.s, z7.s, z2.s[2]\n"
+ "fmla z23.s, z7.s, z3.s[2]\n"
+ "ld1w { z7.s }, p5/Z, [x15, #-3, MUL VL]\n"
+ "fmla z8.s, z6.s, z0.s[3]\n"
+ "fmla z12.s, z6.s, z1.s[3]\n"
+ "fmla z16.s, z6.s, z2.s[3]\n"
+ "fmla z20.s, z6.s, z3.s[3]\n"
+ "ld1w { z6.s }, p5/Z, [x15, #-2, MUL VL]\n"
+ "fmla z9.s, z7.s, z0.s[3]\n"
+ "fmla z13.s, z7.s, z1.s[3]\n"
+ "fmla z17.s, z7.s, z2.s[3]\n"
+ "fmla z21.s, z7.s, z3.s[3]\n"
+ "ld1w { z7.s }, p5/Z, [x15, #-1, MUL VL]\n"
+ "fmla z10.s, z6.s, z0.s[3]\n"
+ "fmla z14.s, z6.s, z1.s[3]\n"
+ "fmla z18.s, z6.s, z2.s[3]\n"
+ "fmla z22.s, z6.s, z3.s[3]\n"
+ "fmla z11.s, z7.s, z0.s[3]\n"
+ "fmla z15.s, z7.s, z1.s[3]\n"
+ "fmla z19.s, z7.s, z2.s[3]\n"
+ "fmla z23.s, z7.s, z3.s[3]\n"
+ "bgt 52b\n"
+ "53:" // Height 4: Multiply loop: Single iteration only
+ "ld1w { z6.s }, p5/Z, [x15]\n"
+ "whilelt p0.s, XZR, x11\n"
+ "ld1w { z7.s }, p5/Z, [x15, #1, MUL VL]\n"
+ "subs x11, x11, #0x1\n"
+ "ld1rqw { z0.s }, p0/Z, [x10]\n"
+ "fmla z8.s, z6.s, z0.s[0]\n"
+ "ld1rqw { z1.s }, p0/Z, [x28]\n"
+ "add x10, x10, #0x10\n"
+ "fmla z9.s, z7.s, z0.s[0]\n"
+ "ld1rqw { z2.s }, p0/Z, [x26]\n"
+ "add x28, x28, #0x10\n"
+ "fmla z12.s, z6.s, z1.s[0]\n"
+ "ld1rqw { z3.s }, p0/Z, [x24]\n"
+ "add x26, x26, #0x10\n"
+ "fmla z16.s, z6.s, z2.s[0]\n"
+ "add x24, x24, #0x10\n"
+ "fmla z13.s, z7.s, z1.s[0]\n"
+ "fmla z17.s, z7.s, z2.s[0]\n"
+ "fmla z20.s, z6.s, z3.s[0]\n"
+ "ld1w { z6.s }, p5/Z, [x15, #2, MUL VL]\n"
+ "fmla z21.s, z7.s, z3.s[0]\n"
+ "ld1w { z7.s }, p5/Z, [x15, #3, MUL VL]\n"
+ "addvl x15, x15, #4\n"
+ "fmla z10.s, z6.s, z0.s[0]\n"
+ "fmla z14.s, z6.s, z1.s[0]\n"
+ "fmla z18.s, z6.s, z2.s[0]\n"
+ "fmla z22.s, z6.s, z3.s[0]\n"
+ "fmla z11.s, z7.s, z0.s[0]\n"
+ "fmla z15.s, z7.s, z1.s[0]\n"
+ "fmla z19.s, z7.s, z2.s[0]\n"
+ "fmla z23.s, z7.s, z3.s[0]\n"
+ "ble 54f\n"
+ "ld1w { z6.s }, p5/Z, [x15]\n"
+ "fmla z8.s, z6.s, z0.s[1]\n"
+ "ld1w { z7.s }, p5/Z, [x15, #1, MUL VL]\n"
+ "subs x11, x11, #0x1\n"
+ "fmla z12.s, z6.s, z1.s[1]\n"
+ "fmla z16.s, z6.s, z2.s[1]\n"
+ "fmla z20.s, z6.s, z3.s[1]\n"
+ "ld1w { z6.s }, p5/Z, [x15, #2, MUL VL]\n"
+ "fmla z9.s, z7.s, z0.s[1]\n"
+ "fmla z13.s, z7.s, z1.s[1]\n"
+ "fmla z17.s, z7.s, z2.s[1]\n"
+ "fmla z21.s, z7.s, z3.s[1]\n"
+ "ld1w { z7.s }, p5/Z, [x15, #3, MUL VL]\n"
+ "addvl x15, x15, #4\n"
+ "fmla z10.s, z6.s, z0.s[1]\n"
+ "fmla z14.s, z6.s, z1.s[1]\n"
+ "fmla z18.s, z6.s, z2.s[1]\n"
+ "fmla z22.s, z6.s, z3.s[1]\n"
+ "fmla z11.s, z7.s, z0.s[1]\n"
+ "fmla z15.s, z7.s, z1.s[1]\n"
+ "fmla z19.s, z7.s, z2.s[1]\n"
+ "fmla z23.s, z7.s, z3.s[1]\n"
+ "ble 54f\n"
+ "ld1w { z6.s }, p5/Z, [x15]\n"
+ "fmla z8.s, z6.s, z0.s[2]\n"
+ "ld1w { z7.s }, p5/Z, [x15, #1, MUL VL]\n"
+ "subs x11, x11, #0x1\n"
+ "fmla z12.s, z6.s, z1.s[2]\n"
+ "fmla z16.s, z6.s, z2.s[2]\n"
+ "fmla z20.s, z6.s, z3.s[2]\n"
+ "ld1w { z6.s }, p5/Z, [x15, #2, MUL VL]\n"
+ "fmla z9.s, z7.s, z0.s[2]\n"
+ "fmla z13.s, z7.s, z1.s[2]\n"
+ "fmla z17.s, z7.s, z2.s[2]\n"
+ "fmla z21.s, z7.s, z3.s[2]\n"
+ "ld1w { z7.s }, p5/Z, [x15, #3, MUL VL]\n"
+ "addvl x15, x15, #4\n"
+ "fmla z10.s, z6.s, z0.s[2]\n"
+ "fmla z14.s, z6.s, z1.s[2]\n"
+ "fmla z18.s, z6.s, z2.s[2]\n"
+ "fmla z22.s, z6.s, z3.s[2]\n"
+ "fmla z11.s, z7.s, z0.s[2]\n"
+ "fmla z15.s, z7.s, z1.s[2]\n"
+ "fmla z19.s, z7.s, z2.s[2]\n"
+ "fmla z23.s, z7.s, z3.s[2]\n"
+ "ble 54f\n"
+ "ld1w { z6.s }, p5/Z, [x15]\n"
+ "fmla z8.s, z6.s, z0.s[3]\n"
+ "ld1w { z7.s }, p5/Z, [x15, #1, MUL VL]\n"
+ "fmla z12.s, z6.s, z1.s[3]\n"
+ "fmla z16.s, z6.s, z2.s[3]\n"
+ "fmla z20.s, z6.s, z3.s[3]\n"
+ "ld1w { z6.s }, p5/Z, [x15, #2, MUL VL]\n"
+ "fmla z9.s, z7.s, z0.s[3]\n"
+ "fmla z13.s, z7.s, z1.s[3]\n"
+ "fmla z17.s, z7.s, z2.s[3]\n"
+ "fmla z21.s, z7.s, z3.s[3]\n"
+ "ld1w { z7.s }, p5/Z, [x15, #3, MUL VL]\n"
+ "addvl x15, x15, #4\n"
+ "fmla z10.s, z6.s, z0.s[3]\n"
+ "fmla z14.s, z6.s, z1.s[3]\n"
+ "fmla z18.s, z6.s, z2.s[3]\n"
+ "fmla z22.s, z6.s, z3.s[3]\n"
+ "fmla z11.s, z7.s, z0.s[3]\n"
+ "fmla z15.s, z7.s, z1.s[3]\n"
+ "fmla z19.s, z7.s, z2.s[3]\n"
+ "fmla z23.s, z7.s, z3.s[3]\n"
+ "54:" // Height 4: Multiply loop: multiply skip
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x12, x12, #0x1\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "cmp x12, x19\n"
+ "bne 49b\n"
+ "prfm pstl1keep, [x13, #0x0]\n"
+ "prfm pstl1keep, [x9, #0x0]\n"
+ "prfm pstl1keep, [x27, #0x0]\n"
+ "prfm pstl1keep, [x25, #0x0]\n"
+ "tbz %x[flags], #1, 55f\n"
+ "add x19, %x[args_ptr], %[offset_min]\n"
+ "ld1rw { z1.s }, p5/Z, [x19]\n"
+ "add x19, %x[args_ptr], %[offset_max]\n"
+ "ld1rw { z0.s }, p5/Z, [x19]\n"
+ "fmin z8.s, p5/M, z8.s, z0.s\n"
+ "fmin z9.s, p5/M, z9.s, z0.s\n"
+ "fmin z10.s, p5/M, z10.s, z0.s\n"
+ "fmin z11.s, p5/M, z11.s, z0.s\n"
+ "fmin z12.s, p5/M, z12.s, z0.s\n"
+ "fmax z8.s, p5/M, z8.s, z1.s\n"
+ "fmax z9.s, p5/M, z9.s, z1.s\n"
+ "fmax z10.s, p5/M, z10.s, z1.s\n"
+ "fmax z11.s, p5/M, z11.s, z1.s\n"
+ "fmax z12.s, p5/M, z12.s, z1.s\n"
+ "fmin z13.s, p5/M, z13.s, z0.s\n"
+ "fmin z14.s, p5/M, z14.s, z0.s\n"
+ "fmin z15.s, p5/M, z15.s, z0.s\n"
+ "fmin z16.s, p5/M, z16.s, z0.s\n"
+ "fmax z13.s, p5/M, z13.s, z1.s\n"
+ "fmax z14.s, p5/M, z14.s, z1.s\n"
+ "fmax z15.s, p5/M, z15.s, z1.s\n"
+ "fmax z16.s, p5/M, z16.s, z1.s\n"
+ "fmin z17.s, p5/M, z17.s, z0.s\n"
+ "fmin z18.s, p5/M, z18.s, z0.s\n"
+ "fmin z19.s, p5/M, z19.s, z0.s\n"
+ "fmin z20.s, p5/M, z20.s, z0.s\n"
+ "fmax z17.s, p5/M, z17.s, z1.s\n"
+ "fmax z18.s, p5/M, z18.s, z1.s\n"
+ "fmax z19.s, p5/M, z19.s, z1.s\n"
+ "fmax z20.s, p5/M, z20.s, z1.s\n"
+ "fmin z21.s, p5/M, z21.s, z0.s\n"
+ "fmin z22.s, p5/M, z22.s, z0.s\n"
+ "fmin z23.s, p5/M, z23.s, z0.s\n"
+ "fmax z21.s, p5/M, z21.s, z1.s\n"
+ "fmax z22.s, p5/M, z22.s, z1.s\n"
+ "fmax z23.s, p5/M, z23.s, z1.s\n"
+ "55:" // Height 4: No activation
+ "st1w { z8.s }, p4, [x13]\n"
+ "st1w { z9.s }, p3, [x13, #1, MUL VL]\n"
+ "st1w { z10.s }, p2, [x13, #2, MUL VL]\n"
+ "st1w { z11.s }, p1, [x13, #3, MUL VL]\n"
+ "addvl x13, x13, #4\n"
+ "st1w { z12.s }, p4, [x9]\n"
+ "st1w { z13.s }, p3, [x9, #1, MUL VL]\n"
+ "st1w { z14.s }, p2, [x9, #2, MUL VL]\n"
+ "st1w { z15.s }, p1, [x9, #3, MUL VL]\n"
+ "addvl x9, x9, #4\n"
+ "st1w { z16.s }, p4, [x27]\n"
+ "st1w { z17.s }, p3, [x27, #1, MUL VL]\n"
+ "st1w { z18.s }, p2, [x27, #2, MUL VL]\n"
+ "st1w { z19.s }, p1, [x27, #3, MUL VL]\n"
+ "addvl x27, x27, #4\n"
+ "st1w { z20.s }, p4, [x25]\n"
+ "st1w { z21.s }, p3, [x25, #1, MUL VL]\n"
+ "st1w { z22.s }, p2, [x25, #2, MUL VL]\n"
+ "st1w { z23.s }, p1, [x25, #3, MUL VL]\n"
+ "addvl x25, x25, #4\n"
+ "56:" // Height 4: Writeback done
+ "mov x19, #0x0\n"
+ "incw x19, ALL, MUL #4\n"
+ "subs x16, x16, x19\n"
+ "bgt 45b\n"
+ "b 86f\n"
+ "57:" // Height 5
+ "ldr x16, [%x[args_ptr], %[offsetof_N]]\n"
+ "mov x14, %x[bias]\n"
+ "ldr x15, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 58f\n"
+ "ldr x13, [%x[output_ptr], #0x0]\n"
+ "add x13, x13, x19, LSL #2\n"
+ "ldr x9, [%x[output_ptr], #0x8]\n"
+ "ldr x27, [%x[output_ptr], #0x10]\n"
+ "add x9, x9, x19, LSL #2\n"
+ "ldr x25, [%x[output_ptr], #0x18]\n"
+ "ldr x23, [%x[output_ptr], #0x20]\n"
+ "add x27, x27, x19, LSL #2\n"
+ "add x25, x25, x19, LSL #2\n"
+ "add x23, x23, x19, LSL #2\n"
+ "b 59f\n"
+ "58:" // Height 5: setup direct output
+ "mov x13, %x[output_ptr]\n"
+ "add x9, x13, x19, LSL #2\n"
+ "add x27, x9, x19, LSL #2\n"
+ "add x25, x27, x19, LSL #2\n"
+ "add x23, x25, x19, LSL #2\n"
+ "59:" // Height 5: Column loop
+ "mov x19, #0x0\n"
+ "whilelt p4.s, x19, x16\n"
+ "incw x19\n"
+ "whilelt p3.s, x19, x16\n"
+ "incw x19\n"
+ "whilelt p2.s, x19, x16\n"
+ "incw x19\n"
+ "whilelt p1.s, x19, x16\n"
+ "cbz x14, 60f\n"
+ "ld1w { z8.s }, p5/Z, [x14]\n"
+ "mov z12.d, z8.d\n"
+ "ld1w { z9.s }, p5/Z, [x14, #1, MUL VL]\n"
+ "mov z16.d, z8.d\n"
+ "ld1w { z10.s }, p5/Z, [x14, #2, MUL VL]\n"
+ "mov z20.d, z8.d\n"
+ "ld1w { z11.s }, p5/Z, [x14, #3, MUL VL]\n"
+ "addvl x14, x14, #4\n"
+ "mov z13.d, z9.d\n"
+ "mov z17.d, z9.d\n"
+ "mov z14.d, z10.d\n"
+ "mov z15.d, z11.d\n"
+ "mov z18.d, z10.d\n"
+ "mov z19.d, z11.d\n"
+ "mov z21.d, z9.d\n"
+ "mov z22.d, z10.d\n"
+ "mov z23.d, z11.d\n"
+ "mov z24.d, z8.d\n"
+ "mov z25.d, z9.d\n"
+ "mov z26.d, z10.d\n"
+ "mov z27.d, z11.d\n"
+ "b 62f\n"
+ "60:" // Height 5: no bias
+ "tbz %x[flags], #0, 61f\n"
+ "ld1w { z8.s }, p4/Z, [x13]\n"
+ "ld1w { z9.s }, p3/Z, [x13, #1, MUL VL]\n"
+ "ld1w { z10.s }, p2/Z, [x13, #2, MUL VL]\n"
+ "ld1w { z11.s }, p1/Z, [x13, #3, MUL VL]\n"
+ "ld1w { z12.s }, p4/Z, [x9]\n"
+ "ld1w { z13.s }, p3/Z, [x9, #1, MUL VL]\n"
+ "ld1w { z14.s }, p2/Z, [x9, #2, MUL VL]\n"
+ "ld1w { z15.s }, p1/Z, [x9, #3, MUL VL]\n"
+ "ld1w { z16.s }, p4/Z, [x27]\n"
+ "ld1w { z17.s }, p3/Z, [x27, #1, MUL VL]\n"
+ "ld1w { z18.s }, p2/Z, [x27, #2, MUL VL]\n"
+ "ld1w { z19.s }, p1/Z, [x27, #3, MUL VL]\n"
+ "ld1w { z20.s }, p4/Z, [x25]\n"
+ "ld1w { z21.s }, p3/Z, [x25, #1, MUL VL]\n"
+ "ld1w { z22.s }, p2/Z, [x25, #2, MUL VL]\n"
+ "ld1w { z23.s }, p1/Z, [x25, #3, MUL VL]\n"
+ "ld1w { z24.s }, p4/Z, [x23]\n"
+ "ld1w { z25.s }, p3/Z, [x23, #1, MUL VL]\n"
+ "ld1w { z26.s }, p2/Z, [x23, #2, MUL VL]\n"
+ "ld1w { z27.s }, p1/Z, [x23, #3, MUL VL]\n"
+ "b 62f\n"
+ "61:" // Height 5: no accumulate
+ "mov z8.b, #0x0\n"
+ "mov z9.b, #0x0\n"
+ "mov z10.b, #0x0\n"
+ "mov z11.b, #0x0\n"
+ "mov z12.b, #0x0\n"
+ "mov z13.b, #0x0\n"
+ "mov z14.b, #0x0\n"
+ "mov z15.b, #0x0\n"
+ "mov z16.b, #0x0\n"
+ "mov z17.b, #0x0\n"
+ "mov z18.b, #0x0\n"
+ "mov z19.b, #0x0\n"
+ "mov z20.b, #0x0\n"
+ "mov z21.b, #0x0\n"
+ "mov z22.b, #0x0\n"
+ "mov z23.b, #0x0\n"
+ "mov z24.b, #0x0\n"
+ "mov z25.b, #0x0\n"
+ "mov z26.b, #0x0\n"
+ "mov z27.b, #0x0\n"
+ "62:" // Height 5: setup done
+ "mov x12, #0x0\n"
+ "63:" // Height 5: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w11, [x20, x12, LSL #0x2]\n"
+ "tbz %x[flags], #3, 64f\n"
+ "ldr x20, [%x[input_ptr], x12, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x10, [x20, #0x0]\n"
+ "ldr x28, [x20, #0x8]\n"
+ "ldr x26, [x20, #0x10]\n"
+ "ldr x24, [x20, #0x18]\n"
+ "ldr x22, [x20, #0x20]\n"
+ "cbnz x12, 65f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x10, x10, x19, LSL #2\n"
+ "add x28, x28, x19, LSL #2\n"
+ "add x26, x26, x19, LSL #2\n"
+ "add x24, x24, x19, LSL #2\n"
+ "add x22, x22, x19, LSL #2\n"
+ "b 65f\n"
+ "64:" // Height 5: setup direct input
+ "mov x10, %x[input_ptr]\n"
+ "add x28, x10, x19, LSL #2\n"
+ "add x26, x28, x19, LSL #2\n"
+ "add x24, x26, x19, LSL #2\n"
+ "add x22, x24, x19, LSL #2\n"
+ "65:" // Height 5: input setup done
+ "cmp x11, #0x4\n"
+ "ble 67f\n"
+ "66:" // Height 5: Multiply loop: Main loop head
+ "ld1w { z6.s }, p5/Z, [x15]\n"
+ "whilelt p0.s, XZR, x11\n"
+ "ld1w { z7.s }, p5/Z, [x15, #1, MUL VL]\n"
+ "sub x11, x11, #0x4\n"
+ "ld1rqw { z0.s }, p0/Z, [x10]\n"
+ "fmla z8.s, z6.s, z0.s[0]\n"
+ "ld1rqw { z1.s }, p0/Z, [x28]\n"
+ "add x10, x10, #0x10\n"
+ "fmla z9.s, z7.s, z0.s[0]\n"
+ "ld1rqw { z2.s }, p0/Z, [x26]\n"
+ "add x28, x28, #0x10\n"
+ "fmla z12.s, z6.s, z1.s[0]\n"
+ "ld1rqw { z3.s }, p0/Z, [x24]\n"
+ "add x26, x26, #0x10\n"
+ "fmla z16.s, z6.s, z2.s[0]\n"
+ "ld1rqw { z4.s }, p0/Z, [x22]\n"
+ "add x24, x24, #0x10\n"
+ "fmla z13.s, z7.s, z1.s[0]\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x22, x22, #0x10\n"
+ "fmla z20.s, z6.s, z3.s[0]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "cmp x11, #0x4\n"
+ "fmla z24.s, z6.s, z4.s[0]\n"
+ "ld1w { z6.s }, p5/Z, [x15, #2, MUL VL]\n"
+ "fmla z17.s, z7.s, z2.s[0]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "fmla z21.s, z7.s, z3.s[0]\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "fmla z25.s, z7.s, z4.s[0]\n"
+ "ld1w { z7.s }, p5/Z, [x15, #3, MUL VL]\n"
+ "fmla z10.s, z6.s, z0.s[0]\n"
+ "prfm pldl1keep, [x22, #0x80]\n"
+ "fmla z14.s, z6.s, z1.s[0]\n"
+ "fmla z18.s, z6.s, z2.s[0]\n"
+ "fmla z22.s, z6.s, z3.s[0]\n"
+ "fmla z26.s, z6.s, z4.s[0]\n"
+ "ld1w { z6.s }, p5/Z, [x15, #4, MUL VL]\n"
+ "fmla z11.s, z7.s, z0.s[0]\n"
+ "fmla z15.s, z7.s, z1.s[0]\n"
+ "fmla z19.s, z7.s, z2.s[0]\n"
+ "fmla z23.s, z7.s, z3.s[0]\n"
+ "fmla z27.s, z7.s, z4.s[0]\n"
+ "ld1w { z7.s }, p5/Z, [x15, #5, MUL VL]\n"
+ "fmla z8.s, z6.s, z0.s[1]\n"
+ "fmla z12.s, z6.s, z1.s[1]\n"
+ "fmla z16.s, z6.s, z2.s[1]\n"
+ "fmla z20.s, z6.s, z3.s[1]\n"
+ "fmla z24.s, z6.s, z4.s[1]\n"
+ "ld1w { z6.s }, p5/Z, [x15, #6, MUL VL]\n"
+ "fmla z9.s, z7.s, z0.s[1]\n"
+ "fmla z13.s, z7.s, z1.s[1]\n"
+ "fmla z17.s, z7.s, z2.s[1]\n"
+ "fmla z21.s, z7.s, z3.s[1]\n"
+ "fmla z25.s, z7.s, z4.s[1]\n"
+ "ld1w { z7.s }, p5/Z, [x15, #7, MUL VL]\n"
+ "addvl x15, x15, #16\n"
+ "fmla z10.s, z6.s, z0.s[1]\n"
+ "fmla z14.s, z6.s, z1.s[1]\n"
+ "fmla z18.s, z6.s, z2.s[1]\n"
+ "fmla z22.s, z6.s, z3.s[1]\n"
+ "fmla z26.s, z6.s, z4.s[1]\n"
+ "ld1w { z6.s }, p5/Z, [x15, #-8, MUL VL]\n"
+ "fmla z11.s, z7.s, z0.s[1]\n"
+ "fmla z15.s, z7.s, z1.s[1]\n"
+ "fmla z19.s, z7.s, z2.s[1]\n"
+ "fmla z23.s, z7.s, z3.s[1]\n"
+ "fmla z27.s, z7.s, z4.s[1]\n"
+ "ld1w { z7.s }, p5/Z, [x15, #-7, MUL VL]\n"
+ "fmla z8.s, z6.s, z0.s[2]\n"
+ "fmla z12.s, z6.s, z1.s[2]\n"
+ "fmla z16.s, z6.s, z2.s[2]\n"
+ "fmla z20.s, z6.s, z3.s[2]\n"
+ "fmla z24.s, z6.s, z4.s[2]\n"
+ "ld1w { z6.s }, p5/Z, [x15, #-6, MUL VL]\n"
+ "fmla z9.s, z7.s, z0.s[2]\n"
+ "fmla z13.s, z7.s, z1.s[2]\n"
+ "fmla z17.s, z7.s, z2.s[2]\n"
+ "fmla z21.s, z7.s, z3.s[2]\n"
+ "fmla z25.s, z7.s, z4.s[2]\n"
+ "ld1w { z7.s }, p5/Z, [x15, #-5, MUL VL]\n"
+ "fmla z10.s, z6.s, z0.s[2]\n"
+ "fmla z14.s, z6.s, z1.s[2]\n"
+ "fmla z18.s, z6.s, z2.s[2]\n"
+ "fmla z22.s, z6.s, z3.s[2]\n"
+ "fmla z26.s, z6.s, z4.s[2]\n"
+ "ld1w { z6.s }, p5/Z, [x15, #-4, MUL VL]\n"
+ "fmla z11.s, z7.s, z0.s[2]\n"
+ "fmla z15.s, z7.s, z1.s[2]\n"
+ "fmla z19.s, z7.s, z2.s[2]\n"
+ "fmla z23.s, z7.s, z3.s[2]\n"
+ "fmla z27.s, z7.s, z4.s[2]\n"
+ "ld1w { z7.s }, p5/Z, [x15, #-3, MUL VL]\n"
+ "fmla z8.s, z6.s, z0.s[3]\n"
+ "fmla z12.s, z6.s, z1.s[3]\n"
+ "fmla z16.s, z6.s, z2.s[3]\n"
+ "fmla z20.s, z6.s, z3.s[3]\n"
+ "fmla z24.s, z6.s, z4.s[3]\n"
+ "ld1w { z6.s }, p5/Z, [x15, #-2, MUL VL]\n"
+ "fmla z9.s, z7.s, z0.s[3]\n"
+ "fmla z13.s, z7.s, z1.s[3]\n"
+ "fmla z17.s, z7.s, z2.s[3]\n"
+ "fmla z21.s, z7.s, z3.s[3]\n"
+ "fmla z25.s, z7.s, z4.s[3]\n"
+ "ld1w { z7.s }, p5/Z, [x15, #-1, MUL VL]\n"
+ "fmla z10.s, z6.s, z0.s[3]\n"
+ "fmla z14.s, z6.s, z1.s[3]\n"
+ "fmla z18.s, z6.s, z2.s[3]\n"
+ "fmla z22.s, z6.s, z3.s[3]\n"
+ "fmla z26.s, z6.s, z4.s[3]\n"
+ "fmla z11.s, z7.s, z0.s[3]\n"
+ "fmla z15.s, z7.s, z1.s[3]\n"
+ "fmla z19.s, z7.s, z2.s[3]\n"
+ "fmla z23.s, z7.s, z3.s[3]\n"
+ "fmla z27.s, z7.s, z4.s[3]\n"
+ "bgt 66b\n"
+ "67:" // Height 5: Multiply loop: Single iteration only
+ "ld1w { z6.s }, p5/Z, [x15]\n"
+ "whilelt p0.s, XZR, x11\n"
+ "ld1w { z7.s }, p5/Z, [x15, #1, MUL VL]\n"
+ "subs x11, x11, #0x1\n"
+ "ld1rqw { z0.s }, p0/Z, [x10]\n"
+ "fmla z8.s, z6.s, z0.s[0]\n"
+ "ld1rqw { z1.s }, p0/Z, [x28]\n"
+ "add x10, x10, #0x10\n"
+ "fmla z9.s, z7.s, z0.s[0]\n"
+ "ld1rqw { z2.s }, p0/Z, [x26]\n"
+ "add x28, x28, #0x10\n"
+ "fmla z12.s, z6.s, z1.s[0]\n"
+ "ld1rqw { z3.s }, p0/Z, [x24]\n"
+ "add x26, x26, #0x10\n"
+ "fmla z16.s, z6.s, z2.s[0]\n"
+ "ld1rqw { z4.s }, p0/Z, [x22]\n"
+ "add x24, x24, #0x10\n"
+ "fmla z13.s, z7.s, z1.s[0]\n"
+ "add x22, x22, #0x10\n"
+ "fmla z17.s, z7.s, z2.s[0]\n"
+ "fmla z20.s, z6.s, z3.s[0]\n"
+ "fmla z24.s, z6.s, z4.s[0]\n"
+ "ld1w { z6.s }, p5/Z, [x15, #2, MUL VL]\n"
+ "fmla z21.s, z7.s, z3.s[0]\n"
+ "fmla z25.s, z7.s, z4.s[0]\n"
+ "ld1w { z7.s }, p5/Z, [x15, #3, MUL VL]\n"
+ "addvl x15, x15, #4\n"
+ "fmla z10.s, z6.s, z0.s[0]\n"
+ "fmla z14.s, z6.s, z1.s[0]\n"
+ "fmla z18.s, z6.s, z2.s[0]\n"
+ "fmla z22.s, z6.s, z3.s[0]\n"
+ "fmla z26.s, z6.s, z4.s[0]\n"
+ "fmla z11.s, z7.s, z0.s[0]\n"
+ "fmla z15.s, z7.s, z1.s[0]\n"
+ "fmla z19.s, z7.s, z2.s[0]\n"
+ "fmla z23.s, z7.s, z3.s[0]\n"
+ "fmla z27.s, z7.s, z4.s[0]\n"
+ "ble 68f\n"
+ "ld1w { z6.s }, p5/Z, [x15]\n"
+ "fmla z8.s, z6.s, z0.s[1]\n"
+ "ld1w { z7.s }, p5/Z, [x15, #1, MUL VL]\n"
+ "subs x11, x11, #0x1\n"
+ "fmla z12.s, z6.s, z1.s[1]\n"
+ "fmla z16.s, z6.s, z2.s[1]\n"
+ "fmla z20.s, z6.s, z3.s[1]\n"
+ "fmla z24.s, z6.s, z4.s[1]\n"
+ "ld1w { z6.s }, p5/Z, [x15, #2, MUL VL]\n"
+ "fmla z9.s, z7.s, z0.s[1]\n"
+ "fmla z13.s, z7.s, z1.s[1]\n"
+ "fmla z17.s, z7.s, z2.s[1]\n"
+ "fmla z21.s, z7.s, z3.s[1]\n"
+ "fmla z25.s, z7.s, z4.s[1]\n"
+ "ld1w { z7.s }, p5/Z, [x15, #3, MUL VL]\n"
+ "addvl x15, x15, #4\n"
+ "fmla z10.s, z6.s, z0.s[1]\n"
+ "fmla z14.s, z6.s, z1.s[1]\n"
+ "fmla z18.s, z6.s, z2.s[1]\n"
+ "fmla z22.s, z6.s, z3.s[1]\n"
+ "fmla z26.s, z6.s, z4.s[1]\n"
+ "fmla z11.s, z7.s, z0.s[1]\n"
+ "fmla z15.s, z7.s, z1.s[1]\n"
+ "fmla z19.s, z7.s, z2.s[1]\n"
+ "fmla z23.s, z7.s, z3.s[1]\n"
+ "fmla z27.s, z7.s, z4.s[1]\n"
+ "ble 68f\n"
+ "ld1w { z6.s }, p5/Z, [x15]\n"
+ "fmla z8.s, z6.s, z0.s[2]\n"
+ "ld1w { z7.s }, p5/Z, [x15, #1, MUL VL]\n"
+ "subs x11, x11, #0x1\n"
+ "fmla z12.s, z6.s, z1.s[2]\n"
+ "fmla z16.s, z6.s, z2.s[2]\n"
+ "fmla z20.s, z6.s, z3.s[2]\n"
+ "fmla z24.s, z6.s, z4.s[2]\n"
+ "ld1w { z6.s }, p5/Z, [x15, #2, MUL VL]\n"
+ "fmla z9.s, z7.s, z0.s[2]\n"
+ "fmla z13.s, z7.s, z1.s[2]\n"
+ "fmla z17.s, z7.s, z2.s[2]\n"
+ "fmla z21.s, z7.s, z3.s[2]\n"
+ "fmla z25.s, z7.s, z4.s[2]\n"
+ "ld1w { z7.s }, p5/Z, [x15, #3, MUL VL]\n"
+ "addvl x15, x15, #4\n"
+ "fmla z10.s, z6.s, z0.s[2]\n"
+ "fmla z14.s, z6.s, z1.s[2]\n"
+ "fmla z18.s, z6.s, z2.s[2]\n"
+ "fmla z22.s, z6.s, z3.s[2]\n"
+ "fmla z26.s, z6.s, z4.s[2]\n"
+ "fmla z11.s, z7.s, z0.s[2]\n"
+ "fmla z15.s, z7.s, z1.s[2]\n"
+ "fmla z19.s, z7.s, z2.s[2]\n"
+ "fmla z23.s, z7.s, z3.s[2]\n"
+ "fmla z27.s, z7.s, z4.s[2]\n"
+ "ble 68f\n"
+ "ld1w { z6.s }, p5/Z, [x15]\n"
+ "fmla z8.s, z6.s, z0.s[3]\n"
+ "ld1w { z7.s }, p5/Z, [x15, #1, MUL VL]\n"
+ "fmla z12.s, z6.s, z1.s[3]\n"
+ "fmla z16.s, z6.s, z2.s[3]\n"
+ "fmla z20.s, z6.s, z3.s[3]\n"
+ "fmla z24.s, z6.s, z4.s[3]\n"
+ "ld1w { z6.s }, p5/Z, [x15, #2, MUL VL]\n"
+ "fmla z9.s, z7.s, z0.s[3]\n"
+ "fmla z13.s, z7.s, z1.s[3]\n"
+ "fmla z17.s, z7.s, z2.s[3]\n"
+ "fmla z21.s, z7.s, z3.s[3]\n"
+ "fmla z25.s, z7.s, z4.s[3]\n"
+ "ld1w { z7.s }, p5/Z, [x15, #3, MUL VL]\n"
+ "addvl x15, x15, #4\n"
+ "fmla z10.s, z6.s, z0.s[3]\n"
+ "fmla z14.s, z6.s, z1.s[3]\n"
+ "fmla z18.s, z6.s, z2.s[3]\n"
+ "fmla z22.s, z6.s, z3.s[3]\n"
+ "fmla z26.s, z6.s, z4.s[3]\n"
+ "fmla z11.s, z7.s, z0.s[3]\n"
+ "fmla z15.s, z7.s, z1.s[3]\n"
+ "fmla z19.s, z7.s, z2.s[3]\n"
+ "fmla z23.s, z7.s, z3.s[3]\n"
+ "fmla z27.s, z7.s, z4.s[3]\n"
+ "68:" // Height 5: Multiply loop: multiply skip
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x12, x12, #0x1\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "prfm pldl1keep, [x22, #0x80]\n"
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "cmp x12, x19\n"
+ "bne 63b\n"
+ "prfm pstl1keep, [x13, #0x0]\n"
+ "prfm pstl1keep, [x9, #0x0]\n"
+ "prfm pstl1keep, [x27, #0x0]\n"
+ "prfm pstl1keep, [x25, #0x0]\n"
+ "prfm pstl1keep, [x23, #0x0]\n"
+ "tbz %x[flags], #1, 69f\n"
+ "add x19, %x[args_ptr], %[offset_min]\n"
+ "ld1rw { z1.s }, p5/Z, [x19]\n"
+ "add x19, %x[args_ptr], %[offset_max]\n"
+ "ld1rw { z0.s }, p5/Z, [x19]\n"
+ "fmin z8.s, p5/M, z8.s, z0.s\n"
+ "fmin z9.s, p5/M, z9.s, z0.s\n"
+ "fmin z10.s, p5/M, z10.s, z0.s\n"
+ "fmin z11.s, p5/M, z11.s, z0.s\n"
+ "fmin z12.s, p5/M, z12.s, z0.s\n"
+ "fmax z8.s, p5/M, z8.s, z1.s\n"
+ "fmax z9.s, p5/M, z9.s, z1.s\n"
+ "fmax z10.s, p5/M, z10.s, z1.s\n"
+ "fmax z11.s, p5/M, z11.s, z1.s\n"
+ "fmax z12.s, p5/M, z12.s, z1.s\n"
+ "fmin z13.s, p5/M, z13.s, z0.s\n"
+ "fmin z14.s, p5/M, z14.s, z0.s\n"
+ "fmin z15.s, p5/M, z15.s, z0.s\n"
+ "fmin z16.s, p5/M, z16.s, z0.s\n"
+ "fmax z13.s, p5/M, z13.s, z1.s\n"
+ "fmax z14.s, p5/M, z14.s, z1.s\n"
+ "fmax z15.s, p5/M, z15.s, z1.s\n"
+ "fmax z16.s, p5/M, z16.s, z1.s\n"
+ "fmin z17.s, p5/M, z17.s, z0.s\n"
+ "fmin z18.s, p5/M, z18.s, z0.s\n"
+ "fmin z19.s, p5/M, z19.s, z0.s\n"
+ "fmin z20.s, p5/M, z20.s, z0.s\n"
+ "fmax z17.s, p5/M, z17.s, z1.s\n"
+ "fmax z18.s, p5/M, z18.s, z1.s\n"
+ "fmax z19.s, p5/M, z19.s, z1.s\n"
+ "fmax z20.s, p5/M, z20.s, z1.s\n"
+ "fmin z21.s, p5/M, z21.s, z0.s\n"
+ "fmin z22.s, p5/M, z22.s, z0.s\n"
+ "fmin z23.s, p5/M, z23.s, z0.s\n"
+ "fmin z24.s, p5/M, z24.s, z0.s\n"
+ "fmax z21.s, p5/M, z21.s, z1.s\n"
+ "fmax z22.s, p5/M, z22.s, z1.s\n"
+ "fmax z23.s, p5/M, z23.s, z1.s\n"
+ "fmax z24.s, p5/M, z24.s, z1.s\n"
+ "fmin z25.s, p5/M, z25.s, z0.s\n"
+ "fmin z26.s, p5/M, z26.s, z0.s\n"
+ "fmin z27.s, p5/M, z27.s, z0.s\n"
+ "fmax z25.s, p5/M, z25.s, z1.s\n"
+ "fmax z26.s, p5/M, z26.s, z1.s\n"
+ "fmax z27.s, p5/M, z27.s, z1.s\n"
+ "69:" // Height 5: No activation
+ "st1w { z8.s }, p4, [x13]\n"
+ "st1w { z9.s }, p3, [x13, #1, MUL VL]\n"
+ "st1w { z10.s }, p2, [x13, #2, MUL VL]\n"
+ "st1w { z11.s }, p1, [x13, #3, MUL VL]\n"
+ "addvl x13, x13, #4\n"
+ "st1w { z12.s }, p4, [x9]\n"
+ "st1w { z13.s }, p3, [x9, #1, MUL VL]\n"
+ "st1w { z14.s }, p2, [x9, #2, MUL VL]\n"
+ "st1w { z15.s }, p1, [x9, #3, MUL VL]\n"
+ "addvl x9, x9, #4\n"
+ "st1w { z16.s }, p4, [x27]\n"
+ "st1w { z17.s }, p3, [x27, #1, MUL VL]\n"
+ "st1w { z18.s }, p2, [x27, #2, MUL VL]\n"
+ "st1w { z19.s }, p1, [x27, #3, MUL VL]\n"
+ "addvl x27, x27, #4\n"
+ "st1w { z20.s }, p4, [x25]\n"
+ "st1w { z21.s }, p3, [x25, #1, MUL VL]\n"
+ "st1w { z22.s }, p2, [x25, #2, MUL VL]\n"
+ "st1w { z23.s }, p1, [x25, #3, MUL VL]\n"
+ "addvl x25, x25, #4\n"
+ "st1w { z24.s }, p4, [x23]\n"
+ "st1w { z25.s }, p3, [x23, #1, MUL VL]\n"
+ "st1w { z26.s }, p2, [x23, #2, MUL VL]\n"
+ "st1w { z27.s }, p1, [x23, #3, MUL VL]\n"
+ "addvl x23, x23, #4\n"
+ "70:" // Height 5: Writeback done
+ "mov x19, #0x0\n"
+ "incw x19, ALL, MUL #4\n"
+ "subs x16, x16, x19\n"
+ "bgt 59b\n"
+ "b 86f\n"
+ "71:" // Height 6
+ "ldr x16, [%x[args_ptr], %[offsetof_N]]\n"
+ "mov x14, %x[bias]\n"
+ "ldr x15, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 72f\n"
+ "ldr x13, [%x[output_ptr], #0x0]\n"
+ "add x13, x13, x19, LSL #2\n"
+ "ldr x9, [%x[output_ptr], #0x8]\n"
+ "ldr x27, [%x[output_ptr], #0x10]\n"
+ "add x9, x9, x19, LSL #2\n"
+ "ldr x25, [%x[output_ptr], #0x18]\n"
+ "ldr x23, [%x[output_ptr], #0x20]\n"
+ "add x27, x27, x19, LSL #2\n"
+ "ldr x21, [%x[output_ptr], #0x28]\n"
+ "add %x[output_ptr], %x[output_ptr], #0x30\n"
+ "add x25, x25, x19, LSL #2\n"
+ "add x23, x23, x19, LSL #2\n"
+ "add x21, x21, x19, LSL #2\n"
+ "b 73f\n"
+ "72:" // Height 6: setup direct output
+ "mov x13, %x[output_ptr]\n"
+ "add x9, x13, x19, LSL #2\n"
+ "add x27, x9, x19, LSL #2\n"
+ "add x25, x27, x19, LSL #2\n"
+ "add x23, x25, x19, LSL #2\n"
+ "add x21, x23, x19, LSL #2\n"
+ "add %x[output_ptr], x21, x19, LSL #2\n"
+ "73:" // Height 6: Column loop
+ "mov x19, #0x0\n"
+ "whilelt p4.s, x19, x16\n"
+ "incw x19\n"
+ "whilelt p3.s, x19, x16\n"
+ "incw x19\n"
+ "whilelt p2.s, x19, x16\n"
+ "incw x19\n"
+ "whilelt p1.s, x19, x16\n"
+ "cbz x14, 74f\n"
+ "ld1w { z8.s }, p5/Z, [x14]\n"
+ "mov z12.d, z8.d\n"
+ "ld1w { z9.s }, p5/Z, [x14, #1, MUL VL]\n"
+ "mov z16.d, z8.d\n"
+ "ld1w { z10.s }, p5/Z, [x14, #2, MUL VL]\n"
+ "mov z20.d, z8.d\n"
+ "ld1w { z11.s }, p5/Z, [x14, #3, MUL VL]\n"
+ "addvl x14, x14, #4\n"
+ "mov z13.d, z9.d\n"
+ "mov z17.d, z9.d\n"
+ "mov z14.d, z10.d\n"
+ "mov z15.d, z11.d\n"
+ "mov z18.d, z10.d\n"
+ "mov z19.d, z11.d\n"
+ "mov z21.d, z9.d\n"
+ "mov z22.d, z10.d\n"
+ "mov z23.d, z11.d\n"
+ "mov z24.d, z8.d\n"
+ "mov z25.d, z9.d\n"
+ "mov z26.d, z10.d\n"
+ "mov z27.d, z11.d\n"
+ "mov z28.d, z8.d\n"
+ "mov z29.d, z9.d\n"
+ "mov z30.d, z10.d\n"
+ "mov z31.d, z11.d\n"
+ "b 76f\n"
+ "74:" // Height 6: no bias
+ "tbz %x[flags], #0, 75f\n"
+ "ld1w { z8.s }, p4/Z, [x13]\n"
+ "ld1w { z9.s }, p3/Z, [x13, #1, MUL VL]\n"
+ "ld1w { z10.s }, p2/Z, [x13, #2, MUL VL]\n"
+ "ld1w { z11.s }, p1/Z, [x13, #3, MUL VL]\n"
+ "ld1w { z12.s }, p4/Z, [x9]\n"
+ "ld1w { z13.s }, p3/Z, [x9, #1, MUL VL]\n"
+ "ld1w { z14.s }, p2/Z, [x9, #2, MUL VL]\n"
+ "ld1w { z15.s }, p1/Z, [x9, #3, MUL VL]\n"
+ "ld1w { z16.s }, p4/Z, [x27]\n"
+ "ld1w { z17.s }, p3/Z, [x27, #1, MUL VL]\n"
+ "ld1w { z18.s }, p2/Z, [x27, #2, MUL VL]\n"
+ "ld1w { z19.s }, p1/Z, [x27, #3, MUL VL]\n"
+ "ld1w { z20.s }, p4/Z, [x25]\n"
+ "ld1w { z21.s }, p3/Z, [x25, #1, MUL VL]\n"
+ "ld1w { z22.s }, p2/Z, [x25, #2, MUL VL]\n"
+ "ld1w { z23.s }, p1/Z, [x25, #3, MUL VL]\n"
+ "ld1w { z24.s }, p4/Z, [x23]\n"
+ "ld1w { z25.s }, p3/Z, [x23, #1, MUL VL]\n"
+ "ld1w { z26.s }, p2/Z, [x23, #2, MUL VL]\n"
+ "ld1w { z27.s }, p1/Z, [x23, #3, MUL VL]\n"
+ "ld1w { z28.s }, p4/Z, [x21]\n"
+ "ld1w { z29.s }, p3/Z, [x21, #1, MUL VL]\n"
+ "ld1w { z30.s }, p2/Z, [x21, #2, MUL VL]\n"
+ "ld1w { z31.s }, p1/Z, [x21, #3, MUL VL]\n"
+ "b 76f\n"
+ "75:" // Height 6: no accumulate
+ "mov z8.b, #0x0\n"
+ "mov z9.b, #0x0\n"
+ "mov z10.b, #0x0\n"
+ "mov z11.b, #0x0\n"
+ "mov z12.b, #0x0\n"
+ "mov z13.b, #0x0\n"
+ "mov z14.b, #0x0\n"
+ "mov z15.b, #0x0\n"
+ "mov z16.b, #0x0\n"
+ "mov z17.b, #0x0\n"
+ "mov z18.b, #0x0\n"
+ "mov z19.b, #0x0\n"
+ "mov z20.b, #0x0\n"
+ "mov z21.b, #0x0\n"
+ "mov z22.b, #0x0\n"
+ "mov z23.b, #0x0\n"
+ "mov z24.b, #0x0\n"
+ "mov z25.b, #0x0\n"
+ "mov z26.b, #0x0\n"
+ "mov z27.b, #0x0\n"
+ "mov z28.b, #0x0\n"
+ "mov z29.b, #0x0\n"
+ "mov z30.b, #0x0\n"
+ "mov z31.b, #0x0\n"
+ "76:" // Height 6: setup done
+ "mov x12, #0x0\n"
+ "77:" // Height 6: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w11, [x20, x12, LSL #0x2]\n"
+ "tbz %x[flags], #3, 78f\n"
+ "ldr x20, [%x[input_ptr], x12, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x10, [x20, #0x0]\n"
+ "ldr x28, [x20, #0x8]\n"
+ "ldr x26, [x20, #0x10]\n"
+ "ldr x24, [x20, #0x18]\n"
+ "ldr x22, [x20, #0x20]\n"
+ "ldr x20, [x20, #0x28]\n"
+ "cbnz x12, 79f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x10, x10, x19, LSL #2\n"
+ "add x28, x28, x19, LSL #2\n"
+ "add x26, x26, x19, LSL #2\n"
+ "add x24, x24, x19, LSL #2\n"
+ "add x22, x22, x19, LSL #2\n"
+ "add x20, x20, x19, LSL #2\n"
+ "b 79f\n"
+ "78:" // Height 6: setup direct input
+ "mov x10, %x[input_ptr]\n"
+ "add x28, x10, x19, LSL #2\n"
+ "add x26, x28, x19, LSL #2\n"
+ "add x24, x26, x19, LSL #2\n"
+ "add x22, x24, x19, LSL #2\n"
+ "add x20, x22, x19, LSL #2\n"
+ "79:" // Height 6: input setup done
+ "cmp x11, #0x4\n"
+ "ble 81f\n"
+ "80:" // Height 6: Multiply loop: Main loop head
+ "ld1w { z6.s }, p5/Z, [x15]\n"
+ "whilelt p0.s, XZR, x11\n"
+ "ld1w { z7.s }, p5/Z, [x15, #1, MUL VL]\n"
+ "sub x11, x11, #0x4\n"
+ "ld1rqw { z0.s }, p0/Z, [x10]\n"
+ "fmla z8.s, z6.s, z0.s[0]\n"
+ "ld1rqw { z1.s }, p0/Z, [x28]\n"
+ "add x10, x10, #0x10\n"
+ "fmla z9.s, z7.s, z0.s[0]\n"
+ "ld1rqw { z2.s }, p0/Z, [x26]\n"
+ "add x28, x28, #0x10\n"
+ "fmla z12.s, z6.s, z1.s[0]\n"
+ "ld1rqw { z3.s }, p0/Z, [x24]\n"
+ "add x26, x26, #0x10\n"
+ "fmla z16.s, z6.s, z2.s[0]\n"
+ "ld1rqw { z4.s }, p0/Z, [x22]\n"
+ "add x24, x24, #0x10\n"
+ "fmla z13.s, z7.s, z1.s[0]\n"
+ "ld1rqw { z5.s }, p0/Z, [x20]\n"
+ "add x22, x22, #0x10\n"
+ "fmla z20.s, z6.s, z3.s[0]\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x20, x20, #0x10\n"
+ "fmla z24.s, z6.s, z4.s[0]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "cmp x11, #0x4\n"
+ "fmla z28.s, z6.s, z5.s[0]\n"
+ "ld1w { z6.s }, p5/Z, [x15, #2, MUL VL]\n"
+ "fmla z17.s, z7.s, z2.s[0]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "fmla z21.s, z7.s, z3.s[0]\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "fmla z25.s, z7.s, z4.s[0]\n"
+ "prfm pldl1keep, [x22, #0x80]\n"
+ "fmla z29.s, z7.s, z5.s[0]\n"
+ "ld1w { z7.s }, p5/Z, [x15, #3, MUL VL]\n"
+ "fmla z10.s, z6.s, z0.s[0]\n"
+ "prfm pldl1keep, [x20, #0x80]\n"
+ "fmla z14.s, z6.s, z1.s[0]\n"
+ "fmla z18.s, z6.s, z2.s[0]\n"
+ "fmla z22.s, z6.s, z3.s[0]\n"
+ "fmla z26.s, z6.s, z4.s[0]\n"
+ "fmla z30.s, z6.s, z5.s[0]\n"
+ "ld1w { z6.s }, p5/Z, [x15, #4, MUL VL]\n"
+ "fmla z11.s, z7.s, z0.s[0]\n"
+ "fmla z15.s, z7.s, z1.s[0]\n"
+ "fmla z19.s, z7.s, z2.s[0]\n"
+ "fmla z23.s, z7.s, z3.s[0]\n"
+ "fmla z27.s, z7.s, z4.s[0]\n"
+ "fmla z31.s, z7.s, z5.s[0]\n"
+ "ld1w { z7.s }, p5/Z, [x15, #5, MUL VL]\n"
+ "fmla z8.s, z6.s, z0.s[1]\n"
+ "fmla z12.s, z6.s, z1.s[1]\n"
+ "fmla z16.s, z6.s, z2.s[1]\n"
+ "fmla z20.s, z6.s, z3.s[1]\n"
+ "fmla z24.s, z6.s, z4.s[1]\n"
+ "fmla z28.s, z6.s, z5.s[1]\n"
+ "ld1w { z6.s }, p5/Z, [x15, #6, MUL VL]\n"
+ "fmla z9.s, z7.s, z0.s[1]\n"
+ "fmla z13.s, z7.s, z1.s[1]\n"
+ "fmla z17.s, z7.s, z2.s[1]\n"
+ "fmla z21.s, z7.s, z3.s[1]\n"
+ "fmla z25.s, z7.s, z4.s[1]\n"
+ "fmla z29.s, z7.s, z5.s[1]\n"
+ "ld1w { z7.s }, p5/Z, [x15, #7, MUL VL]\n"
+ "addvl x15, x15, #16\n"
+ "fmla z10.s, z6.s, z0.s[1]\n"
+ "fmla z14.s, z6.s, z1.s[1]\n"
+ "fmla z18.s, z6.s, z2.s[1]\n"
+ "fmla z22.s, z6.s, z3.s[1]\n"
+ "fmla z26.s, z6.s, z4.s[1]\n"
+ "fmla z30.s, z6.s, z5.s[1]\n"
+ "ld1w { z6.s }, p5/Z, [x15, #-8, MUL VL]\n"
+ "fmla z11.s, z7.s, z0.s[1]\n"
+ "fmla z15.s, z7.s, z1.s[1]\n"
+ "fmla z19.s, z7.s, z2.s[1]\n"
+ "fmla z23.s, z7.s, z3.s[1]\n"
+ "fmla z27.s, z7.s, z4.s[1]\n"
+ "fmla z31.s, z7.s, z5.s[1]\n"
+ "ld1w { z7.s }, p5/Z, [x15, #-7, MUL VL]\n"
+ "fmla z8.s, z6.s, z0.s[2]\n"
+ "fmla z12.s, z6.s, z1.s[2]\n"
+ "fmla z16.s, z6.s, z2.s[2]\n"
+ "fmla z20.s, z6.s, z3.s[2]\n"
+ "fmla z24.s, z6.s, z4.s[2]\n"
+ "fmla z28.s, z6.s, z5.s[2]\n"
+ "ld1w { z6.s }, p5/Z, [x15, #-6, MUL VL]\n"
+ "fmla z9.s, z7.s, z0.s[2]\n"
+ "fmla z13.s, z7.s, z1.s[2]\n"
+ "fmla z17.s, z7.s, z2.s[2]\n"
+ "fmla z21.s, z7.s, z3.s[2]\n"
+ "fmla z25.s, z7.s, z4.s[2]\n"
+ "fmla z29.s, z7.s, z5.s[2]\n"
+ "ld1w { z7.s }, p5/Z, [x15, #-5, MUL VL]\n"
+ "fmla z10.s, z6.s, z0.s[2]\n"
+ "fmla z14.s, z6.s, z1.s[2]\n"
+ "fmla z18.s, z6.s, z2.s[2]\n"
+ "fmla z22.s, z6.s, z3.s[2]\n"
+ "fmla z26.s, z6.s, z4.s[2]\n"
+ "fmla z30.s, z6.s, z5.s[2]\n"
+ "ld1w { z6.s }, p5/Z, [x15, #-4, MUL VL]\n"
+ "fmla z11.s, z7.s, z0.s[2]\n"
+ "fmla z15.s, z7.s, z1.s[2]\n"
+ "fmla z19.s, z7.s, z2.s[2]\n"
+ "fmla z23.s, z7.s, z3.s[2]\n"
+ "fmla z27.s, z7.s, z4.s[2]\n"
+ "fmla z31.s, z7.s, z5.s[2]\n"
+ "ld1w { z7.s }, p5/Z, [x15, #-3, MUL VL]\n"
+ "fmla z8.s, z6.s, z0.s[3]\n"
+ "fmla z12.s, z6.s, z1.s[3]\n"
+ "fmla z16.s, z6.s, z2.s[3]\n"
+ "fmla z20.s, z6.s, z3.s[3]\n"
+ "fmla z24.s, z6.s, z4.s[3]\n"
+ "fmla z28.s, z6.s, z5.s[3]\n"
+ "ld1w { z6.s }, p5/Z, [x15, #-2, MUL VL]\n"
+ "fmla z9.s, z7.s, z0.s[3]\n"
+ "fmla z13.s, z7.s, z1.s[3]\n"
+ "fmla z17.s, z7.s, z2.s[3]\n"
+ "fmla z21.s, z7.s, z3.s[3]\n"
+ "fmla z25.s, z7.s, z4.s[3]\n"
+ "fmla z29.s, z7.s, z5.s[3]\n"
+ "ld1w { z7.s }, p5/Z, [x15, #-1, MUL VL]\n"
+ "fmla z10.s, z6.s, z0.s[3]\n"
+ "fmla z14.s, z6.s, z1.s[3]\n"
+ "fmla z18.s, z6.s, z2.s[3]\n"
+ "fmla z22.s, z6.s, z3.s[3]\n"
+ "fmla z26.s, z6.s, z4.s[3]\n"
+ "fmla z30.s, z6.s, z5.s[3]\n"
+ "fmla z11.s, z7.s, z0.s[3]\n"
+ "fmla z15.s, z7.s, z1.s[3]\n"
+ "fmla z19.s, z7.s, z2.s[3]\n"
+ "fmla z23.s, z7.s, z3.s[3]\n"
+ "fmla z27.s, z7.s, z4.s[3]\n"
+ "fmla z31.s, z7.s, z5.s[3]\n"
+ "bgt 80b\n"
+ "81:" // Height 6: Multiply loop: Single iteration only
+ "ld1w { z6.s }, p5/Z, [x15]\n"
+ "whilelt p0.s, XZR, x11\n"
+ "ld1w { z7.s }, p5/Z, [x15, #1, MUL VL]\n"
+ "subs x11, x11, #0x1\n"
+ "ld1rqw { z0.s }, p0/Z, [x10]\n"
+ "fmla z8.s, z6.s, z0.s[0]\n"
+ "ld1rqw { z1.s }, p0/Z, [x28]\n"
+ "add x10, x10, #0x10\n"
+ "fmla z9.s, z7.s, z0.s[0]\n"
+ "ld1rqw { z2.s }, p0/Z, [x26]\n"
+ "add x28, x28, #0x10\n"
+ "fmla z12.s, z6.s, z1.s[0]\n"
+ "ld1rqw { z3.s }, p0/Z, [x24]\n"
+ "add x26, x26, #0x10\n"
+ "fmla z16.s, z6.s, z2.s[0]\n"
+ "ld1rqw { z4.s }, p0/Z, [x22]\n"
+ "add x24, x24, #0x10\n"
+ "fmla z13.s, z7.s, z1.s[0]\n"
+ "ld1rqw { z5.s }, p0/Z, [x20]\n"
+ "add x22, x22, #0x10\n"
+ "fmla z20.s, z6.s, z3.s[0]\n"
+ "add x20, x20, #0x10\n"
+ "fmla z17.s, z7.s, z2.s[0]\n"
+ "fmla z24.s, z6.s, z4.s[0]\n"
+ "fmla z28.s, z6.s, z5.s[0]\n"
+ "ld1w { z6.s }, p5/Z, [x15, #2, MUL VL]\n"
+ "fmla z21.s, z7.s, z3.s[0]\n"
+ "fmla z25.s, z7.s, z4.s[0]\n"
+ "fmla z29.s, z7.s, z5.s[0]\n"
+ "ld1w { z7.s }, p5/Z, [x15, #3, MUL VL]\n"
+ "addvl x15, x15, #4\n"
+ "fmla z10.s, z6.s, z0.s[0]\n"
+ "fmla z14.s, z6.s, z1.s[0]\n"
+ "fmla z18.s, z6.s, z2.s[0]\n"
+ "fmla z22.s, z6.s, z3.s[0]\n"
+ "fmla z26.s, z6.s, z4.s[0]\n"
+ "fmla z30.s, z6.s, z5.s[0]\n"
+ "fmla z11.s, z7.s, z0.s[0]\n"
+ "fmla z15.s, z7.s, z1.s[0]\n"
+ "fmla z19.s, z7.s, z2.s[0]\n"
+ "fmla z23.s, z7.s, z3.s[0]\n"
+ "fmla z27.s, z7.s, z4.s[0]\n"
+ "fmla z31.s, z7.s, z5.s[0]\n"
+ "ble 82f\n"
+ "ld1w { z6.s }, p5/Z, [x15]\n"
+ "fmla z8.s, z6.s, z0.s[1]\n"
+ "ld1w { z7.s }, p5/Z, [x15, #1, MUL VL]\n"
+ "subs x11, x11, #0x1\n"
+ "fmla z12.s, z6.s, z1.s[1]\n"
+ "fmla z16.s, z6.s, z2.s[1]\n"
+ "fmla z20.s, z6.s, z3.s[1]\n"
+ "fmla z24.s, z6.s, z4.s[1]\n"
+ "fmla z28.s, z6.s, z5.s[1]\n"
+ "ld1w { z6.s }, p5/Z, [x15, #2, MUL VL]\n"
+ "fmla z9.s, z7.s, z0.s[1]\n"
+ "fmla z13.s, z7.s, z1.s[1]\n"
+ "fmla z17.s, z7.s, z2.s[1]\n"
+ "fmla z21.s, z7.s, z3.s[1]\n"
+ "fmla z25.s, z7.s, z4.s[1]\n"
+ "fmla z29.s, z7.s, z5.s[1]\n"
+ "ld1w { z7.s }, p5/Z, [x15, #3, MUL VL]\n"
+ "addvl x15, x15, #4\n"
+ "fmla z10.s, z6.s, z0.s[1]\n"
+ "fmla z14.s, z6.s, z1.s[1]\n"
+ "fmla z18.s, z6.s, z2.s[1]\n"
+ "fmla z22.s, z6.s, z3.s[1]\n"
+ "fmla z26.s, z6.s, z4.s[1]\n"
+ "fmla z30.s, z6.s, z5.s[1]\n"
+ "fmla z11.s, z7.s, z0.s[1]\n"
+ "fmla z15.s, z7.s, z1.s[1]\n"
+ "fmla z19.s, z7.s, z2.s[1]\n"
+ "fmla z23.s, z7.s, z3.s[1]\n"
+ "fmla z27.s, z7.s, z4.s[1]\n"
+ "fmla z31.s, z7.s, z5.s[1]\n"
+ "ble 82f\n"
+ "ld1w { z6.s }, p5/Z, [x15]\n"
+ "fmla z8.s, z6.s, z0.s[2]\n"
+ "ld1w { z7.s }, p5/Z, [x15, #1, MUL VL]\n"
+ "subs x11, x11, #0x1\n"
+ "fmla z12.s, z6.s, z1.s[2]\n"
+ "fmla z16.s, z6.s, z2.s[2]\n"
+ "fmla z20.s, z6.s, z3.s[2]\n"
+ "fmla z24.s, z6.s, z4.s[2]\n"
+ "fmla z28.s, z6.s, z5.s[2]\n"
+ "ld1w { z6.s }, p5/Z, [x15, #2, MUL VL]\n"
+ "fmla z9.s, z7.s, z0.s[2]\n"
+ "fmla z13.s, z7.s, z1.s[2]\n"
+ "fmla z17.s, z7.s, z2.s[2]\n"
+ "fmla z21.s, z7.s, z3.s[2]\n"
+ "fmla z25.s, z7.s, z4.s[2]\n"
+ "fmla z29.s, z7.s, z5.s[2]\n"
+ "ld1w { z7.s }, p5/Z, [x15, #3, MUL VL]\n"
+ "addvl x15, x15, #4\n"
+ "fmla z10.s, z6.s, z0.s[2]\n"
+ "fmla z14.s, z6.s, z1.s[2]\n"
+ "fmla z18.s, z6.s, z2.s[2]\n"
+ "fmla z22.s, z6.s, z3.s[2]\n"
+ "fmla z26.s, z6.s, z4.s[2]\n"
+ "fmla z30.s, z6.s, z5.s[2]\n"
+ "fmla z11.s, z7.s, z0.s[2]\n"
+ "fmla z15.s, z7.s, z1.s[2]\n"
+ "fmla z19.s, z7.s, z2.s[2]\n"
+ "fmla z23.s, z7.s, z3.s[2]\n"
+ "fmla z27.s, z7.s, z4.s[2]\n"
+ "fmla z31.s, z7.s, z5.s[2]\n"
+ "ble 82f\n"
+ "ld1w { z6.s }, p5/Z, [x15]\n"
+ "fmla z8.s, z6.s, z0.s[3]\n"
+ "ld1w { z7.s }, p5/Z, [x15, #1, MUL VL]\n"
+ "fmla z12.s, z6.s, z1.s[3]\n"
+ "fmla z16.s, z6.s, z2.s[3]\n"
+ "fmla z20.s, z6.s, z3.s[3]\n"
+ "fmla z24.s, z6.s, z4.s[3]\n"
+ "fmla z28.s, z6.s, z5.s[3]\n"
+ "ld1w { z6.s }, p5/Z, [x15, #2, MUL VL]\n"
+ "fmla z9.s, z7.s, z0.s[3]\n"
+ "fmla z13.s, z7.s, z1.s[3]\n"
+ "fmla z17.s, z7.s, z2.s[3]\n"
+ "fmla z21.s, z7.s, z3.s[3]\n"
+ "fmla z25.s, z7.s, z4.s[3]\n"
+ "fmla z29.s, z7.s, z5.s[3]\n"
+ "ld1w { z7.s }, p5/Z, [x15, #3, MUL VL]\n"
+ "addvl x15, x15, #4\n"
+ "fmla z10.s, z6.s, z0.s[3]\n"
+ "fmla z14.s, z6.s, z1.s[3]\n"
+ "fmla z18.s, z6.s, z2.s[3]\n"
+ "fmla z22.s, z6.s, z3.s[3]\n"
+ "fmla z26.s, z6.s, z4.s[3]\n"
+ "fmla z30.s, z6.s, z5.s[3]\n"
+ "fmla z11.s, z7.s, z0.s[3]\n"
+ "fmla z15.s, z7.s, z1.s[3]\n"
+ "fmla z19.s, z7.s, z2.s[3]\n"
+ "fmla z23.s, z7.s, z3.s[3]\n"
+ "fmla z27.s, z7.s, z4.s[3]\n"
+ "fmla z31.s, z7.s, z5.s[3]\n"
+ "82:" // Height 6: Multiply loop: multiply skip
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x12, x12, #0x1\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "prfm pldl1keep, [x22, #0x80]\n"
+ "prfm pldl1keep, [x20, #0x80]\n"
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "cmp x12, x19\n"
+ "bne 77b\n"
+ "prfm pstl1keep, [x13, #0x0]\n"
+ "prfm pstl1keep, [x9, #0x0]\n"
+ "prfm pstl1keep, [x27, #0x0]\n"
+ "prfm pstl1keep, [x25, #0x0]\n"
+ "prfm pstl1keep, [x23, #0x0]\n"
+ "prfm pstl1keep, [x21, #0x0]\n"
+ "tbz %x[flags], #1, 83f\n"
+ "add x19, %x[args_ptr], %[offset_min]\n"
+ "ld1rw { z1.s }, p5/Z, [x19]\n"
+ "add x19, %x[args_ptr], %[offset_max]\n"
+ "ld1rw { z0.s }, p5/Z, [x19]\n"
+ "fmin z8.s, p5/M, z8.s, z0.s\n"
+ "fmin z9.s, p5/M, z9.s, z0.s\n"
+ "fmin z10.s, p5/M, z10.s, z0.s\n"
+ "fmin z11.s, p5/M, z11.s, z0.s\n"
+ "fmin z12.s, p5/M, z12.s, z0.s\n"
+ "fmax z8.s, p5/M, z8.s, z1.s\n"
+ "fmax z9.s, p5/M, z9.s, z1.s\n"
+ "fmax z10.s, p5/M, z10.s, z1.s\n"
+ "fmax z11.s, p5/M, z11.s, z1.s\n"
+ "fmax z12.s, p5/M, z12.s, z1.s\n"
+ "fmin z13.s, p5/M, z13.s, z0.s\n"
+ "fmin z14.s, p5/M, z14.s, z0.s\n"
+ "fmin z15.s, p5/M, z15.s, z0.s\n"
+ "fmin z16.s, p5/M, z16.s, z0.s\n"
+ "fmax z13.s, p5/M, z13.s, z1.s\n"
+ "fmax z14.s, p5/M, z14.s, z1.s\n"
+ "fmax z15.s, p5/M, z15.s, z1.s\n"
+ "fmax z16.s, p5/M, z16.s, z1.s\n"
+ "fmin z17.s, p5/M, z17.s, z0.s\n"
+ "fmin z18.s, p5/M, z18.s, z0.s\n"
+ "fmin z19.s, p5/M, z19.s, z0.s\n"
+ "fmin z20.s, p5/M, z20.s, z0.s\n"
+ "fmax z17.s, p5/M, z17.s, z1.s\n"
+ "fmax z18.s, p5/M, z18.s, z1.s\n"
+ "fmax z19.s, p5/M, z19.s, z1.s\n"
+ "fmax z20.s, p5/M, z20.s, z1.s\n"
+ "fmin z21.s, p5/M, z21.s, z0.s\n"
+ "fmin z22.s, p5/M, z22.s, z0.s\n"
+ "fmin z23.s, p5/M, z23.s, z0.s\n"
+ "fmin z24.s, p5/M, z24.s, z0.s\n"
+ "fmax z21.s, p5/M, z21.s, z1.s\n"
+ "fmax z22.s, p5/M, z22.s, z1.s\n"
+ "fmax z23.s, p5/M, z23.s, z1.s\n"
+ "fmax z24.s, p5/M, z24.s, z1.s\n"
+ "fmin z25.s, p5/M, z25.s, z0.s\n"
+ "fmin z26.s, p5/M, z26.s, z0.s\n"
+ "fmin z27.s, p5/M, z27.s, z0.s\n"
+ "fmin z28.s, p5/M, z28.s, z0.s\n"
+ "fmax z25.s, p5/M, z25.s, z1.s\n"
+ "fmax z26.s, p5/M, z26.s, z1.s\n"
+ "fmax z27.s, p5/M, z27.s, z1.s\n"
+ "fmax z28.s, p5/M, z28.s, z1.s\n"
+ "fmin z29.s, p5/M, z29.s, z0.s\n"
+ "fmin z30.s, p5/M, z30.s, z0.s\n"
+ "fmin z31.s, p5/M, z31.s, z0.s\n"
+ "fmax z29.s, p5/M, z29.s, z1.s\n"
+ "fmax z30.s, p5/M, z30.s, z1.s\n"
+ "fmax z31.s, p5/M, z31.s, z1.s\n"
+ "83:" // Height 6: No activation
+ "st1w { z8.s }, p4, [x13]\n"
+ "st1w { z9.s }, p3, [x13, #1, MUL VL]\n"
+ "st1w { z10.s }, p2, [x13, #2, MUL VL]\n"
+ "st1w { z11.s }, p1, [x13, #3, MUL VL]\n"
+ "addvl x13, x13, #4\n"
+ "st1w { z12.s }, p4, [x9]\n"
+ "st1w { z13.s }, p3, [x9, #1, MUL VL]\n"
+ "st1w { z14.s }, p2, [x9, #2, MUL VL]\n"
+ "st1w { z15.s }, p1, [x9, #3, MUL VL]\n"
+ "addvl x9, x9, #4\n"
+ "st1w { z16.s }, p4, [x27]\n"
+ "st1w { z17.s }, p3, [x27, #1, MUL VL]\n"
+ "st1w { z18.s }, p2, [x27, #2, MUL VL]\n"
+ "st1w { z19.s }, p1, [x27, #3, MUL VL]\n"
+ "addvl x27, x27, #4\n"
+ "st1w { z20.s }, p4, [x25]\n"
+ "st1w { z21.s }, p3, [x25, #1, MUL VL]\n"
+ "st1w { z22.s }, p2, [x25, #2, MUL VL]\n"
+ "st1w { z23.s }, p1, [x25, #3, MUL VL]\n"
+ "addvl x25, x25, #4\n"
+ "st1w { z24.s }, p4, [x23]\n"
+ "st1w { z25.s }, p3, [x23, #1, MUL VL]\n"
+ "st1w { z26.s }, p2, [x23, #2, MUL VL]\n"
+ "st1w { z27.s }, p1, [x23, #3, MUL VL]\n"
+ "addvl x23, x23, #4\n"
+ "st1w { z28.s }, p4, [x21]\n"
+ "st1w { z29.s }, p3, [x21, #1, MUL VL]\n"
+ "st1w { z30.s }, p2, [x21, #2, MUL VL]\n"
+ "st1w { z31.s }, p1, [x21, #3, MUL VL]\n"
+ "addvl x21, x21, #4\n"
+ "84:" // Height 6: Writeback done
+ "mov x19, #0x0\n"
+ "incw x19, ALL, MUL #4\n"
+ "subs x16, x16, x19\n"
+ "bgt 73b\n"
+ "subs %x[M], %x[M], #0x6\n"
+ "beq 86f\n"
+ "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "tbz %x[flags], #3, 85f\n"
+ "add x20, x20, #0x6\n"
+ "str x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "b 1b\n"
+ "85:" // Update direct input
+ "mov x19, #0x18\n"
+ "madd %x[input_ptr], x19, x20, %x[input_ptr]\n"
+ "b 1b\n"
+ "86:" // Exit
+
+ : [M] "+r" (M), [input_ptr] "+r" (input_ptr), [output_ptr] "+r" (output_ptr)
+ : [args_ptr] "r" (&ka), [bias] "r" (bias), [flags] "r" (flags), [offset_max] "I" (offsetof(KernelArgs, maxval)), [offset_min] "I" (offsetof(KernelArgs, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths))
+ : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+ );
+}
+
+} // namespace arm_gemm
+#endif // __ARM_FEATURE_SVE
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_fp32_mla_1VLx8.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_8x1VL.hpp
similarity index 69%
copy from src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_fp32_mla_1VLx8.hpp
copy to src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_8x1VL.hpp
index b555066..20d9922 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_fp32_mla_1VLx8.hpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_8x1VL.hpp
@@ -10,36 +10,43 @@
* sell copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
*/
#pragma once
-
#ifdef __ARM_FEATURE_SVE
+#include "../std_transforms_sve.hpp"
+#define ARGLIST \
+ unsigned int, const unsigned int *, \
+ IndirectInputArg<float>, \
+ size_t, size_t, \
+ const float *, \
+ IndirectOutputArg<float>, \
+ const float *, Activation, bool
namespace arm_gemm
{
// Actual kernel implementations
-void sve_smallK_hybrid_fp32_mla_1VLx8(const float *, int, const float *, float *, int, int, int, int, const float *, Activation, bool);
+void sve_hybrid_fp32_mla_8x1VL( ARGLIST );
-class smallK_hybrid_fp32_mla_1VLx8
+class cls_sve_hybrid_fp32_mla_8x1VL
{
public:
typedef float operand_type;
typedef float result_type;
- typedef void (*kern_type)(const float *, int, const float *, float *, int, int, int, int, const float *, Activation, bool);
+ typedef void (*kern_type)( ARGLIST );
/* Kernel blocking parameters */
static constexpr unsigned int out_height()
@@ -59,30 +66,20 @@
static constexpr bool supports_accumulate()
{
- return false;
- }
-
- static constexpr bool supports_bias()
- {
- return true;
- }
-
- static constexpr bool supports_activation()
- {
return true;
}
StdTransformsSVE<operand_type, result_type, 8, 1, 1> transforms = {};
// Default to the generic kernel
- kern_type kernel=sve_smallK_hybrid_fp32_mla_1VLx8;
+ kern_type kernel=sve_hybrid_fp32_mla_8x1VL;
- smallK_hybrid_fp32_mla_1VLx8(const CPUInfo *)
+ cls_sve_hybrid_fp32_mla_8x1VL(const CPUInfo *)
{
-
}
};
} // namespace arm_gemm
+#undef ARGLIST
#endif // __ARM_FEATURE_SVE
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_8x1VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_8x1VL/generic.cpp
new file mode 100644
index 0000000..361e303
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_8x1VL/generic.cpp
@@ -0,0 +1,1751 @@
+/*
+ * Copyright (c) 2019-2020 Arm Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+#ifdef __ARM_FEATURE_SVE
+
+#include "arm_gemm.hpp"
+#include "../../utils.hpp"
+
+#include <cassert>
+
+namespace arm_gemm {
+
+void sve_hybrid_fp32_mla_8x1VL (
+ unsigned int num_strings, const unsigned int *string_lengths, IndirectInputArg<float> A_arg,
+ size_t M, size_t N, const float *B_ptr, IndirectOutputArg<float> output_arg,
+ const float *bias, Activation act, bool accumulate
+)
+{
+ struct KernelArgs {
+ float maxval = static_cast<float>(std::numeric_limits<float>::infinity());
+ float minval = - static_cast<float>(std::numeric_limits<float>::infinity());
+ unsigned int num_strings = {};
+ const unsigned int *string_lengths = {};
+ size_t N = {};
+ const float *B_ptr = {};
+ size_t output_offset = {};
+ size_t input_initial_col = {};
+ size_t input_offset = {};
+ } ka;
+
+ unsigned long flags=0;
+ void *output_ptr;
+ void *input_ptr;
+
+ if (output_arg.is_indirect) {
+ output_ptr=(void *)(output_arg.indirect.ptr);
+ ka.output_offset=output_arg.indirect.offset;
+ flags |= 0x4;
+ } else {
+ output_ptr=(void *)(output_arg.direct.base);
+ ka.output_offset=output_arg.direct.stride;
+ }
+
+ if (A_arg.is_indirect) {
+ input_ptr=(void *)(A_arg.indirect.ptr);
+ ka.input_offset=A_arg.indirect.start_row;
+ ka.input_initial_col=A_arg.indirect.start_col;
+ flags |= 0x8;
+ } else {
+ assert(num_strings==1);
+ input_ptr=(void *)(A_arg.direct.base);
+ ka.input_offset=A_arg.direct.stride;
+ }
+ if (accumulate) {
+ flags |= 0x1;
+ }
+ ka.num_strings = num_strings;
+ ka.string_lengths = string_lengths;
+ ka.N = N;
+ ka.B_ptr = B_ptr;
+ switch(act.type) {
+ default:
+ case Activation::Type::None:
+ break;
+ case Activation::Type::BoundedReLU:
+ ka.maxval = static_cast<float>(act.param1);
+ /* fall through */
+ case Activation::Type::ReLU:
+ ka.minval = 0;
+ flags |= 0x2;
+ break;
+ }
+ __asm__ __volatile__(
+ "ptrue p2.b\n"
+ "1:" // Row loop
+ "cmp %x[M], #0x8\n"
+ "bge 99f\n"
+ "cmp %x[M], #0x6\n"
+ "bgt 85f\n"
+ "beq 71f\n"
+ "cmp %x[M], #0x4\n"
+ "bgt 57f\n"
+ "beq 43f\n"
+ "cmp %x[M], #0x2\n"
+ "bgt 29f\n"
+ "beq 15f\n"
+ "ldr x6, [%x[args_ptr], %[offsetof_N]]\n"
+ "ldr x7, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "mov x8, %x[bias]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 2f\n"
+ "ldr x17, [%x[output_ptr], #0x0]\n"
+ "add x17, x17, x19, LSL #2\n"
+ "b 3f\n"
+ "2:" // Height 1: setup direct output
+ "mov x17, %x[output_ptr]\n"
+ "3:" // Height 1: Column loop
+ "mov x19, #0x0\n"
+ "whilelt p1.s, x19, x6\n"
+ "cbz x8, 4f\n"
+ "ld1w { z24.s }, p2/Z, [x8]\n"
+ "addvl x8, x8, #1\n"
+ "b 6f\n"
+ "4:" // Height 1: no bias
+ "tbz %x[flags], #0, 5f\n"
+ "ld1w { z24.s }, p1/Z, [x17]\n"
+ "b 6f\n"
+ "5:" // Height 1: no accumulate
+ "mov z24.b, #0x0\n"
+ "6:" // Height 1: setup done
+ "mov x16, #0x0\n"
+ "7:" // Height 1: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w15, [x20, x16, LSL #0x2]\n"
+ "tbz %x[flags], #3, 8f\n"
+ "ldr x20, [%x[input_ptr], x16, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x14, [x20, #0x0]\n"
+ "cbnz x16, 9f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x14, x14, x19, LSL #2\n"
+ "b 9f\n"
+ "8:" // Height 1: setup direct input
+ "mov x14, %x[input_ptr]\n"
+ "9:" // Height 1: input setup done
+ "cmp x15, #0x4\n"
+ "ble 11f\n"
+ "10:" // Height 1: Multiply loop: Main loop head
+ "ld1w { z8.s }, p2/Z, [x7]\n"
+ "whilelt p0.s, XZR, x15\n"
+ "ld1w { z9.s }, p2/Z, [x7, #1, MUL VL]\n"
+ "sub x15, x15, #0x4\n"
+ "ld1rqw { z0.s }, p0/Z, [x14]\n"
+ "fmla z24.s, z8.s, z0.s[0]\n"
+ "ld1w { z10.s }, p2/Z, [x7, #2, MUL VL]\n"
+ "add x14, x14, #0x10\n"
+ "fmla z24.s, z9.s, z0.s[1]\n"
+ "ld1w { z11.s }, p2/Z, [x7, #3, MUL VL]\n"
+ "cmp x15, #0x4\n"
+ "fmla z24.s, z10.s, z0.s[2]\n"
+ "prfm pldl1keep, [x14, #0x80]\n"
+ "addvl x7, x7, #4\n"
+ "fmla z24.s, z11.s, z0.s[3]\n"
+ "bgt 10b\n"
+ "11:" // Height 1: Multiply loop: Single iteration only
+ "ld1w { z12.s }, p2/Z, [x7]\n"
+ "whilelt p0.s, XZR, x15\n"
+ "subs x15, x15, #0x1\n"
+ "ld1rqw { z0.s }, p0/Z, [x14]\n"
+ "fmla z24.s, z12.s, z0.s[0]\n"
+ "add x14, x14, #0x10\n"
+ "addvl x7, x7, #1\n"
+ "ble 12f\n"
+ "ld1w { z13.s }, p2/Z, [x7]\n"
+ "fmla z24.s, z13.s, z0.s[1]\n"
+ "subs x15, x15, #0x1\n"
+ "addvl x7, x7, #1\n"
+ "ble 12f\n"
+ "ld1w { z14.s }, p2/Z, [x7]\n"
+ "fmla z24.s, z14.s, z0.s[2]\n"
+ "subs x15, x15, #0x1\n"
+ "addvl x7, x7, #1\n"
+ "ble 12f\n"
+ "ld1w { z15.s }, p2/Z, [x7]\n"
+ "fmla z24.s, z15.s, z0.s[3]\n"
+ "addvl x7, x7, #1\n"
+ "12:" // Height 1: Multiply loop: multiply skip
+ "prfm pldl1keep, [x14, #0x80]\n"
+ "add x16, x16, #0x1\n"
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "cmp x16, x19\n"
+ "bne 7b\n"
+ "prfm pstl1keep, [x17, #0x0]\n"
+ "tbz %x[flags], #1, 13f\n"
+ "add x19, %x[args_ptr], %[offset_min]\n"
+ "ld1rw { z17.s }, p2/Z, [x19]\n"
+ "add x19, %x[args_ptr], %[offset_max]\n"
+ "ld1rw { z16.s }, p2/Z, [x19]\n"
+ "fmin z24.s, p2/M, z24.s, z16.s\n"
+ "fmax z24.s, p2/M, z24.s, z17.s\n"
+ "13:" // Height 1: No activation
+ "st1w { z24.s }, p1, [x17]\n"
+ "addvl x17, x17, #1\n"
+ "14:" // Height 1: Writeback done
+ "mov x19, #0x0\n"
+ "incw x19\n"
+ "subs x6, x6, x19\n"
+ "bgt 3b\n"
+ "b 114f\n"
+ "15:" // Height 2
+ "ldr x6, [%x[args_ptr], %[offsetof_N]]\n"
+ "mov x8, %x[bias]\n"
+ "ldr x7, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 16f\n"
+ "ldr x17, [%x[output_ptr], #0x0]\n"
+ "add x17, x17, x19, LSL #2\n"
+ "ldr x13, [%x[output_ptr], #0x8]\n"
+ "add x13, x13, x19, LSL #2\n"
+ "b 17f\n"
+ "16:" // Height 2: setup direct output
+ "mov x17, %x[output_ptr]\n"
+ "add x13, x17, x19, LSL #2\n"
+ "17:" // Height 2: Column loop
+ "mov x19, #0x0\n"
+ "whilelt p1.s, x19, x6\n"
+ "cbz x8, 18f\n"
+ "ld1w { z24.s }, p2/Z, [x8]\n"
+ "mov z25.d, z24.d\n"
+ "addvl x8, x8, #1\n"
+ "b 20f\n"
+ "18:" // Height 2: no bias
+ "tbz %x[flags], #0, 19f\n"
+ "ld1w { z24.s }, p1/Z, [x17]\n"
+ "ld1w { z25.s }, p1/Z, [x13]\n"
+ "b 20f\n"
+ "19:" // Height 2: no accumulate
+ "mov z24.b, #0x0\n"
+ "mov z25.b, #0x0\n"
+ "20:" // Height 2: setup done
+ "mov x16, #0x0\n"
+ "21:" // Height 2: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w15, [x20, x16, LSL #0x2]\n"
+ "tbz %x[flags], #3, 22f\n"
+ "ldr x20, [%x[input_ptr], x16, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x14, [x20, #0x0]\n"
+ "ldr x12, [x20, #0x8]\n"
+ "cbnz x16, 23f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x14, x14, x19, LSL #2\n"
+ "add x12, x12, x19, LSL #2\n"
+ "b 23f\n"
+ "22:" // Height 2: setup direct input
+ "mov x14, %x[input_ptr]\n"
+ "add x12, x14, x19, LSL #2\n"
+ "23:" // Height 2: input setup done
+ "cmp x15, #0x4\n"
+ "ble 25f\n"
+ "24:" // Height 2: Multiply loop: Main loop head
+ "ld1w { z8.s }, p2/Z, [x7]\n"
+ "whilelt p0.s, XZR, x15\n"
+ "ld1w { z9.s }, p2/Z, [x7, #1, MUL VL]\n"
+ "sub x15, x15, #0x4\n"
+ "ld1rqw { z0.s }, p0/Z, [x14]\n"
+ "fmla z24.s, z8.s, z0.s[0]\n"
+ "ld1rqw { z1.s }, p0/Z, [x12]\n"
+ "add x14, x14, #0x10\n"
+ "fmla z25.s, z8.s, z1.s[0]\n"
+ "ld1w { z10.s }, p2/Z, [x7, #2, MUL VL]\n"
+ "add x12, x12, #0x10\n"
+ "fmla z24.s, z9.s, z0.s[1]\n"
+ "ld1w { z11.s }, p2/Z, [x7, #3, MUL VL]\n"
+ "cmp x15, #0x4\n"
+ "fmla z25.s, z9.s, z1.s[1]\n"
+ "prfm pldl1keep, [x14, #0x80]\n"
+ "addvl x7, x7, #4\n"
+ "fmla z24.s, z10.s, z0.s[2]\n"
+ "prfm pldl1keep, [x12, #0x80]\n"
+ "fmla z25.s, z10.s, z1.s[2]\n"
+ "fmla z24.s, z11.s, z0.s[3]\n"
+ "fmla z25.s, z11.s, z1.s[3]\n"
+ "bgt 24b\n"
+ "25:" // Height 2: Multiply loop: Single iteration only
+ "ld1w { z12.s }, p2/Z, [x7]\n"
+ "whilelt p0.s, XZR, x15\n"
+ "subs x15, x15, #0x1\n"
+ "ld1rqw { z0.s }, p0/Z, [x14]\n"
+ "fmla z24.s, z12.s, z0.s[0]\n"
+ "ld1rqw { z1.s }, p0/Z, [x12]\n"
+ "add x14, x14, #0x10\n"
+ "fmla z25.s, z12.s, z1.s[0]\n"
+ "add x12, x12, #0x10\n"
+ "addvl x7, x7, #1\n"
+ "ble 26f\n"
+ "ld1w { z13.s }, p2/Z, [x7]\n"
+ "fmla z24.s, z13.s, z0.s[1]\n"
+ "subs x15, x15, #0x1\n"
+ "fmla z25.s, z13.s, z1.s[1]\n"
+ "addvl x7, x7, #1\n"
+ "ble 26f\n"
+ "ld1w { z14.s }, p2/Z, [x7]\n"
+ "fmla z24.s, z14.s, z0.s[2]\n"
+ "subs x15, x15, #0x1\n"
+ "fmla z25.s, z14.s, z1.s[2]\n"
+ "addvl x7, x7, #1\n"
+ "ble 26f\n"
+ "ld1w { z15.s }, p2/Z, [x7]\n"
+ "fmla z24.s, z15.s, z0.s[3]\n"
+ "addvl x7, x7, #1\n"
+ "fmla z25.s, z15.s, z1.s[3]\n"
+ "26:" // Height 2: Multiply loop: multiply skip
+ "prfm pldl1keep, [x14, #0x80]\n"
+ "add x16, x16, #0x1\n"
+ "prfm pldl1keep, [x12, #0x80]\n"
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "cmp x16, x19\n"
+ "bne 21b\n"
+ "prfm pstl1keep, [x17, #0x0]\n"
+ "prfm pstl1keep, [x13, #0x0]\n"
+ "tbz %x[flags], #1, 27f\n"
+ "add x19, %x[args_ptr], %[offset_min]\n"
+ "ld1rw { z17.s }, p2/Z, [x19]\n"
+ "add x19, %x[args_ptr], %[offset_max]\n"
+ "ld1rw { z16.s }, p2/Z, [x19]\n"
+ "fmin z24.s, p2/M, z24.s, z16.s\n"
+ "fmin z25.s, p2/M, z25.s, z16.s\n"
+ "fmax z24.s, p2/M, z24.s, z17.s\n"
+ "fmax z25.s, p2/M, z25.s, z17.s\n"
+ "27:" // Height 2: No activation
+ "st1w { z24.s }, p1, [x17]\n"
+ "addvl x17, x17, #1\n"
+ "st1w { z25.s }, p1, [x13]\n"
+ "addvl x13, x13, #1\n"
+ "28:" // Height 2: Writeback done
+ "mov x19, #0x0\n"
+ "incw x19\n"
+ "subs x6, x6, x19\n"
+ "bgt 17b\n"
+ "b 114f\n"
+ "29:" // Height 3
+ "ldr x6, [%x[args_ptr], %[offsetof_N]]\n"
+ "mov x8, %x[bias]\n"
+ "ldr x7, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 30f\n"
+ "ldr x17, [%x[output_ptr], #0x0]\n"
+ "add x17, x17, x19, LSL #2\n"
+ "ldr x13, [%x[output_ptr], #0x8]\n"
+ "ldr x11, [%x[output_ptr], #0x10]\n"
+ "add x13, x13, x19, LSL #2\n"
+ "add x11, x11, x19, LSL #2\n"
+ "b 31f\n"
+ "30:" // Height 3: setup direct output
+ "mov x17, %x[output_ptr]\n"
+ "add x13, x17, x19, LSL #2\n"
+ "add x11, x13, x19, LSL #2\n"
+ "31:" // Height 3: Column loop
+ "mov x19, #0x0\n"
+ "whilelt p1.s, x19, x6\n"
+ "cbz x8, 32f\n"
+ "ld1w { z24.s }, p2/Z, [x8]\n"
+ "mov z25.d, z24.d\n"
+ "addvl x8, x8, #1\n"
+ "mov z26.d, z24.d\n"
+ "b 34f\n"
+ "32:" // Height 3: no bias
+ "tbz %x[flags], #0, 33f\n"
+ "ld1w { z24.s }, p1/Z, [x17]\n"
+ "ld1w { z25.s }, p1/Z, [x13]\n"
+ "ld1w { z26.s }, p1/Z, [x11]\n"
+ "b 34f\n"
+ "33:" // Height 3: no accumulate
+ "mov z24.b, #0x0\n"
+ "mov z25.b, #0x0\n"
+ "mov z26.b, #0x0\n"
+ "34:" // Height 3: setup done
+ "mov x16, #0x0\n"
+ "35:" // Height 3: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w15, [x20, x16, LSL #0x2]\n"
+ "tbz %x[flags], #3, 36f\n"
+ "ldr x20, [%x[input_ptr], x16, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x14, [x20, #0x0]\n"
+ "ldr x12, [x20, #0x8]\n"
+ "ldr x10, [x20, #0x10]\n"
+ "cbnz x16, 37f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x14, x14, x19, LSL #2\n"
+ "add x12, x12, x19, LSL #2\n"
+ "add x10, x10, x19, LSL #2\n"
+ "b 37f\n"
+ "36:" // Height 3: setup direct input
+ "mov x14, %x[input_ptr]\n"
+ "add x12, x14, x19, LSL #2\n"
+ "add x10, x12, x19, LSL #2\n"
+ "37:" // Height 3: input setup done
+ "cmp x15, #0x4\n"
+ "ble 39f\n"
+ "38:" // Height 3: Multiply loop: Main loop head
+ "ld1w { z8.s }, p2/Z, [x7]\n"
+ "whilelt p0.s, XZR, x15\n"
+ "ld1w { z9.s }, p2/Z, [x7, #1, MUL VL]\n"
+ "sub x15, x15, #0x4\n"
+ "ld1rqw { z0.s }, p0/Z, [x14]\n"
+ "fmla z24.s, z8.s, z0.s[0]\n"
+ "ld1rqw { z1.s }, p0/Z, [x12]\n"
+ "add x14, x14, #0x10\n"
+ "fmla z25.s, z8.s, z1.s[0]\n"
+ "ld1rqw { z2.s }, p0/Z, [x10]\n"
+ "add x12, x12, #0x10\n"
+ "fmla z24.s, z9.s, z0.s[1]\n"
+ "ld1w { z10.s }, p2/Z, [x7, #2, MUL VL]\n"
+ "add x10, x10, #0x10\n"
+ "fmla z26.s, z8.s, z2.s[0]\n"
+ "ld1w { z11.s }, p2/Z, [x7, #3, MUL VL]\n"
+ "cmp x15, #0x4\n"
+ "fmla z25.s, z9.s, z1.s[1]\n"
+ "prfm pldl1keep, [x14, #0x80]\n"
+ "addvl x7, x7, #4\n"
+ "fmla z24.s, z10.s, z0.s[2]\n"
+ "prfm pldl1keep, [x12, #0x80]\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "fmla z26.s, z9.s, z2.s[1]\n"
+ "fmla z25.s, z10.s, z1.s[2]\n"
+ "fmla z24.s, z11.s, z0.s[3]\n"
+ "fmla z26.s, z10.s, z2.s[2]\n"
+ "fmla z25.s, z11.s, z1.s[3]\n"
+ "fmla z26.s, z11.s, z2.s[3]\n"
+ "bgt 38b\n"
+ "39:" // Height 3: Multiply loop: Single iteration only
+ "ld1w { z12.s }, p2/Z, [x7]\n"
+ "whilelt p0.s, XZR, x15\n"
+ "subs x15, x15, #0x1\n"
+ "ld1rqw { z0.s }, p0/Z, [x14]\n"
+ "fmla z24.s, z12.s, z0.s[0]\n"
+ "ld1rqw { z1.s }, p0/Z, [x12]\n"
+ "add x14, x14, #0x10\n"
+ "fmla z25.s, z12.s, z1.s[0]\n"
+ "ld1rqw { z2.s }, p0/Z, [x10]\n"
+ "add x12, x12, #0x10\n"
+ "fmla z26.s, z12.s, z2.s[0]\n"
+ "add x10, x10, #0x10\n"
+ "addvl x7, x7, #1\n"
+ "ble 40f\n"
+ "ld1w { z13.s }, p2/Z, [x7]\n"
+ "fmla z24.s, z13.s, z0.s[1]\n"
+ "subs x15, x15, #0x1\n"
+ "fmla z25.s, z13.s, z1.s[1]\n"
+ "addvl x7, x7, #1\n"
+ "fmla z26.s, z13.s, z2.s[1]\n"
+ "ble 40f\n"
+ "ld1w { z14.s }, p2/Z, [x7]\n"
+ "fmla z24.s, z14.s, z0.s[2]\n"
+ "subs x15, x15, #0x1\n"
+ "fmla z25.s, z14.s, z1.s[2]\n"
+ "addvl x7, x7, #1\n"
+ "fmla z26.s, z14.s, z2.s[2]\n"
+ "ble 40f\n"
+ "ld1w { z15.s }, p2/Z, [x7]\n"
+ "fmla z24.s, z15.s, z0.s[3]\n"
+ "addvl x7, x7, #1\n"
+ "fmla z25.s, z15.s, z1.s[3]\n"
+ "fmla z26.s, z15.s, z2.s[3]\n"
+ "40:" // Height 3: Multiply loop: multiply skip
+ "prfm pldl1keep, [x14, #0x80]\n"
+ "add x16, x16, #0x1\n"
+ "prfm pldl1keep, [x12, #0x80]\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "cmp x16, x19\n"
+ "bne 35b\n"
+ "prfm pstl1keep, [x17, #0x0]\n"
+ "prfm pstl1keep, [x13, #0x0]\n"
+ "prfm pstl1keep, [x11, #0x0]\n"
+ "tbz %x[flags], #1, 41f\n"
+ "add x19, %x[args_ptr], %[offset_min]\n"
+ "ld1rw { z17.s }, p2/Z, [x19]\n"
+ "add x19, %x[args_ptr], %[offset_max]\n"
+ "ld1rw { z16.s }, p2/Z, [x19]\n"
+ "fmin z24.s, p2/M, z24.s, z16.s\n"
+ "fmin z25.s, p2/M, z25.s, z16.s\n"
+ "fmin z26.s, p2/M, z26.s, z16.s\n"
+ "fmax z24.s, p2/M, z24.s, z17.s\n"
+ "fmax z25.s, p2/M, z25.s, z17.s\n"
+ "fmax z26.s, p2/M, z26.s, z17.s\n"
+ "41:" // Height 3: No activation
+ "st1w { z24.s }, p1, [x17]\n"
+ "addvl x17, x17, #1\n"
+ "st1w { z25.s }, p1, [x13]\n"
+ "addvl x13, x13, #1\n"
+ "st1w { z26.s }, p1, [x11]\n"
+ "addvl x11, x11, #1\n"
+ "42:" // Height 3: Writeback done
+ "mov x19, #0x0\n"
+ "incw x19\n"
+ "subs x6, x6, x19\n"
+ "bgt 31b\n"
+ "b 114f\n"
+ "43:" // Height 4
+ "ldr x6, [%x[args_ptr], %[offsetof_N]]\n"
+ "mov x8, %x[bias]\n"
+ "ldr x7, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 44f\n"
+ "ldr x17, [%x[output_ptr], #0x0]\n"
+ "add x17, x17, x19, LSL #2\n"
+ "ldr x13, [%x[output_ptr], #0x8]\n"
+ "ldr x11, [%x[output_ptr], #0x10]\n"
+ "add x13, x13, x19, LSL #2\n"
+ "ldr x9, [%x[output_ptr], #0x18]\n"
+ "add x11, x11, x19, LSL #2\n"
+ "add x9, x9, x19, LSL #2\n"
+ "b 45f\n"
+ "44:" // Height 4: setup direct output
+ "mov x17, %x[output_ptr]\n"
+ "add x13, x17, x19, LSL #2\n"
+ "add x11, x13, x19, LSL #2\n"
+ "add x9, x11, x19, LSL #2\n"
+ "45:" // Height 4: Column loop
+ "mov x19, #0x0\n"
+ "whilelt p1.s, x19, x6\n"
+ "cbz x8, 46f\n"
+ "ld1w { z24.s }, p2/Z, [x8]\n"
+ "mov z25.d, z24.d\n"
+ "addvl x8, x8, #1\n"
+ "mov z26.d, z24.d\n"
+ "mov z27.d, z24.d\n"
+ "b 48f\n"
+ "46:" // Height 4: no bias
+ "tbz %x[flags], #0, 47f\n"
+ "ld1w { z24.s }, p1/Z, [x17]\n"
+ "ld1w { z25.s }, p1/Z, [x13]\n"
+ "ld1w { z26.s }, p1/Z, [x11]\n"
+ "ld1w { z27.s }, p1/Z, [x9]\n"
+ "b 48f\n"
+ "47:" // Height 4: no accumulate
+ "mov z24.b, #0x0\n"
+ "mov z25.b, #0x0\n"
+ "mov z26.b, #0x0\n"
+ "mov z27.b, #0x0\n"
+ "48:" // Height 4: setup done
+ "mov x16, #0x0\n"
+ "49:" // Height 4: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w15, [x20, x16, LSL #0x2]\n"
+ "tbz %x[flags], #3, 50f\n"
+ "ldr x20, [%x[input_ptr], x16, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x14, [x20, #0x0]\n"
+ "ldr x12, [x20, #0x8]\n"
+ "ldr x10, [x20, #0x10]\n"
+ "ldr x28, [x20, #0x18]\n"
+ "cbnz x16, 51f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x14, x14, x19, LSL #2\n"
+ "add x12, x12, x19, LSL #2\n"
+ "add x10, x10, x19, LSL #2\n"
+ "add x28, x28, x19, LSL #2\n"
+ "b 51f\n"
+ "50:" // Height 4: setup direct input
+ "mov x14, %x[input_ptr]\n"
+ "add x12, x14, x19, LSL #2\n"
+ "add x10, x12, x19, LSL #2\n"
+ "add x28, x10, x19, LSL #2\n"
+ "51:" // Height 4: input setup done
+ "cmp x15, #0x4\n"
+ "ble 53f\n"
+ "52:" // Height 4: Multiply loop: Main loop head
+ "ld1w { z8.s }, p2/Z, [x7]\n"
+ "whilelt p0.s, XZR, x15\n"
+ "ld1w { z9.s }, p2/Z, [x7, #1, MUL VL]\n"
+ "sub x15, x15, #0x4\n"
+ "ld1rqw { z0.s }, p0/Z, [x14]\n"
+ "fmla z24.s, z8.s, z0.s[0]\n"
+ "ld1rqw { z1.s }, p0/Z, [x12]\n"
+ "add x14, x14, #0x10\n"
+ "fmla z25.s, z8.s, z1.s[0]\n"
+ "ld1rqw { z2.s }, p0/Z, [x10]\n"
+ "add x12, x12, #0x10\n"
+ "fmla z24.s, z9.s, z0.s[1]\n"
+ "ld1rqw { z3.s }, p0/Z, [x28]\n"
+ "add x10, x10, #0x10\n"
+ "fmla z26.s, z8.s, z2.s[0]\n"
+ "ld1w { z10.s }, p2/Z, [x7, #2, MUL VL]\n"
+ "add x28, x28, #0x10\n"
+ "fmla z27.s, z8.s, z3.s[0]\n"
+ "ld1w { z11.s }, p2/Z, [x7, #3, MUL VL]\n"
+ "cmp x15, #0x4\n"
+ "fmla z25.s, z9.s, z1.s[1]\n"
+ "prfm pldl1keep, [x14, #0x80]\n"
+ "addvl x7, x7, #4\n"
+ "fmla z24.s, z10.s, z0.s[2]\n"
+ "prfm pldl1keep, [x12, #0x80]\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "fmla z26.s, z9.s, z2.s[1]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "fmla z27.s, z9.s, z3.s[1]\n"
+ "fmla z25.s, z10.s, z1.s[2]\n"
+ "fmla z24.s, z11.s, z0.s[3]\n"
+ "fmla z26.s, z10.s, z2.s[2]\n"
+ "fmla z27.s, z10.s, z3.s[2]\n"
+ "fmla z25.s, z11.s, z1.s[3]\n"
+ "fmla z26.s, z11.s, z2.s[3]\n"
+ "fmla z27.s, z11.s, z3.s[3]\n"
+ "bgt 52b\n"
+ "53:" // Height 4: Multiply loop: Single iteration only
+ "ld1w { z12.s }, p2/Z, [x7]\n"
+ "whilelt p0.s, XZR, x15\n"
+ "subs x15, x15, #0x1\n"
+ "ld1rqw { z0.s }, p0/Z, [x14]\n"
+ "fmla z24.s, z12.s, z0.s[0]\n"
+ "ld1rqw { z1.s }, p0/Z, [x12]\n"
+ "add x14, x14, #0x10\n"
+ "fmla z25.s, z12.s, z1.s[0]\n"
+ "ld1rqw { z2.s }, p0/Z, [x10]\n"
+ "add x12, x12, #0x10\n"
+ "fmla z26.s, z12.s, z2.s[0]\n"
+ "ld1rqw { z3.s }, p0/Z, [x28]\n"
+ "add x10, x10, #0x10\n"
+ "fmla z27.s, z12.s, z3.s[0]\n"
+ "add x28, x28, #0x10\n"
+ "addvl x7, x7, #1\n"
+ "ble 54f\n"
+ "ld1w { z13.s }, p2/Z, [x7]\n"
+ "fmla z24.s, z13.s, z0.s[1]\n"
+ "subs x15, x15, #0x1\n"
+ "fmla z25.s, z13.s, z1.s[1]\n"
+ "addvl x7, x7, #1\n"
+ "fmla z26.s, z13.s, z2.s[1]\n"
+ "fmla z27.s, z13.s, z3.s[1]\n"
+ "ble 54f\n"
+ "ld1w { z14.s }, p2/Z, [x7]\n"
+ "fmla z24.s, z14.s, z0.s[2]\n"
+ "subs x15, x15, #0x1\n"
+ "fmla z25.s, z14.s, z1.s[2]\n"
+ "addvl x7, x7, #1\n"
+ "fmla z26.s, z14.s, z2.s[2]\n"
+ "fmla z27.s, z14.s, z3.s[2]\n"
+ "ble 54f\n"
+ "ld1w { z15.s }, p2/Z, [x7]\n"
+ "fmla z24.s, z15.s, z0.s[3]\n"
+ "addvl x7, x7, #1\n"
+ "fmla z25.s, z15.s, z1.s[3]\n"
+ "fmla z26.s, z15.s, z2.s[3]\n"
+ "fmla z27.s, z15.s, z3.s[3]\n"
+ "54:" // Height 4: Multiply loop: multiply skip
+ "prfm pldl1keep, [x14, #0x80]\n"
+ "add x16, x16, #0x1\n"
+ "prfm pldl1keep, [x12, #0x80]\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "cmp x16, x19\n"
+ "bne 49b\n"
+ "prfm pstl1keep, [x17, #0x0]\n"
+ "prfm pstl1keep, [x13, #0x0]\n"
+ "prfm pstl1keep, [x11, #0x0]\n"
+ "prfm pstl1keep, [x9, #0x0]\n"
+ "tbz %x[flags], #1, 55f\n"
+ "add x19, %x[args_ptr], %[offset_min]\n"
+ "ld1rw { z17.s }, p2/Z, [x19]\n"
+ "add x19, %x[args_ptr], %[offset_max]\n"
+ "ld1rw { z16.s }, p2/Z, [x19]\n"
+ "fmin z24.s, p2/M, z24.s, z16.s\n"
+ "fmin z25.s, p2/M, z25.s, z16.s\n"
+ "fmin z26.s, p2/M, z26.s, z16.s\n"
+ "fmin z27.s, p2/M, z27.s, z16.s\n"
+ "fmax z24.s, p2/M, z24.s, z17.s\n"
+ "fmax z25.s, p2/M, z25.s, z17.s\n"
+ "fmax z26.s, p2/M, z26.s, z17.s\n"
+ "fmax z27.s, p2/M, z27.s, z17.s\n"
+ "55:" // Height 4: No activation
+ "st1w { z24.s }, p1, [x17]\n"
+ "addvl x17, x17, #1\n"
+ "st1w { z25.s }, p1, [x13]\n"
+ "addvl x13, x13, #1\n"
+ "st1w { z26.s }, p1, [x11]\n"
+ "addvl x11, x11, #1\n"
+ "st1w { z27.s }, p1, [x9]\n"
+ "addvl x9, x9, #1\n"
+ "56:" // Height 4: Writeback done
+ "mov x19, #0x0\n"
+ "incw x19\n"
+ "subs x6, x6, x19\n"
+ "bgt 45b\n"
+ "b 114f\n"
+ "57:" // Height 5
+ "ldr x6, [%x[args_ptr], %[offsetof_N]]\n"
+ "mov x8, %x[bias]\n"
+ "ldr x7, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 58f\n"
+ "ldr x17, [%x[output_ptr], #0x0]\n"
+ "add x17, x17, x19, LSL #2\n"
+ "ldr x13, [%x[output_ptr], #0x8]\n"
+ "ldr x11, [%x[output_ptr], #0x10]\n"
+ "add x13, x13, x19, LSL #2\n"
+ "ldr x9, [%x[output_ptr], #0x18]\n"
+ "ldr x27, [%x[output_ptr], #0x20]\n"
+ "add x11, x11, x19, LSL #2\n"
+ "add x9, x9, x19, LSL #2\n"
+ "add x27, x27, x19, LSL #2\n"
+ "b 59f\n"
+ "58:" // Height 5: setup direct output
+ "mov x17, %x[output_ptr]\n"
+ "add x13, x17, x19, LSL #2\n"
+ "add x11, x13, x19, LSL #2\n"
+ "add x9, x11, x19, LSL #2\n"
+ "add x27, x9, x19, LSL #2\n"
+ "59:" // Height 5: Column loop
+ "mov x19, #0x0\n"
+ "whilelt p1.s, x19, x6\n"
+ "cbz x8, 60f\n"
+ "ld1w { z24.s }, p2/Z, [x8]\n"
+ "mov z25.d, z24.d\n"
+ "addvl x8, x8, #1\n"
+ "mov z26.d, z24.d\n"
+ "mov z27.d, z24.d\n"
+ "mov z28.d, z24.d\n"
+ "b 62f\n"
+ "60:" // Height 5: no bias
+ "tbz %x[flags], #0, 61f\n"
+ "ld1w { z24.s }, p1/Z, [x17]\n"
+ "ld1w { z25.s }, p1/Z, [x13]\n"
+ "ld1w { z26.s }, p1/Z, [x11]\n"
+ "ld1w { z27.s }, p1/Z, [x9]\n"
+ "ld1w { z28.s }, p1/Z, [x27]\n"
+ "b 62f\n"
+ "61:" // Height 5: no accumulate
+ "mov z24.b, #0x0\n"
+ "mov z25.b, #0x0\n"
+ "mov z26.b, #0x0\n"
+ "mov z27.b, #0x0\n"
+ "mov z28.b, #0x0\n"
+ "62:" // Height 5: setup done
+ "mov x16, #0x0\n"
+ "63:" // Height 5: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w15, [x20, x16, LSL #0x2]\n"
+ "tbz %x[flags], #3, 64f\n"
+ "ldr x20, [%x[input_ptr], x16, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x14, [x20, #0x0]\n"
+ "ldr x12, [x20, #0x8]\n"
+ "ldr x10, [x20, #0x10]\n"
+ "ldr x28, [x20, #0x18]\n"
+ "ldr x26, [x20, #0x20]\n"
+ "cbnz x16, 65f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x14, x14, x19, LSL #2\n"
+ "add x12, x12, x19, LSL #2\n"
+ "add x10, x10, x19, LSL #2\n"
+ "add x28, x28, x19, LSL #2\n"
+ "add x26, x26, x19, LSL #2\n"
+ "b 65f\n"
+ "64:" // Height 5: setup direct input
+ "mov x14, %x[input_ptr]\n"
+ "add x12, x14, x19, LSL #2\n"
+ "add x10, x12, x19, LSL #2\n"
+ "add x28, x10, x19, LSL #2\n"
+ "add x26, x28, x19, LSL #2\n"
+ "65:" // Height 5: input setup done
+ "cmp x15, #0x4\n"
+ "ble 67f\n"
+ "66:" // Height 5: Multiply loop: Main loop head
+ "ld1w { z8.s }, p2/Z, [x7]\n"
+ "whilelt p0.s, XZR, x15\n"
+ "ld1w { z9.s }, p2/Z, [x7, #1, MUL VL]\n"
+ "sub x15, x15, #0x4\n"
+ "ld1rqw { z0.s }, p0/Z, [x14]\n"
+ "fmla z24.s, z8.s, z0.s[0]\n"
+ "ld1rqw { z1.s }, p0/Z, [x12]\n"
+ "add x14, x14, #0x10\n"
+ "fmla z25.s, z8.s, z1.s[0]\n"
+ "ld1rqw { z2.s }, p0/Z, [x10]\n"
+ "add x12, x12, #0x10\n"
+ "fmla z24.s, z9.s, z0.s[1]\n"
+ "ld1rqw { z3.s }, p0/Z, [x28]\n"
+ "add x10, x10, #0x10\n"
+ "fmla z26.s, z8.s, z2.s[0]\n"
+ "ld1rqw { z4.s }, p0/Z, [x26]\n"
+ "add x28, x28, #0x10\n"
+ "fmla z27.s, z8.s, z3.s[0]\n"
+ "ld1w { z10.s }, p2/Z, [x7, #2, MUL VL]\n"
+ "add x26, x26, #0x10\n"
+ "fmla z25.s, z9.s, z1.s[1]\n"
+ "ld1w { z11.s }, p2/Z, [x7, #3, MUL VL]\n"
+ "cmp x15, #0x4\n"
+ "fmla z28.s, z8.s, z4.s[0]\n"
+ "prfm pldl1keep, [x14, #0x80]\n"
+ "addvl x7, x7, #4\n"
+ "fmla z26.s, z9.s, z2.s[1]\n"
+ "prfm pldl1keep, [x12, #0x80]\n"
+ "fmla z24.s, z10.s, z0.s[2]\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "fmla z27.s, z9.s, z3.s[1]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "fmla z25.s, z10.s, z1.s[2]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "fmla z28.s, z9.s, z4.s[1]\n"
+ "fmla z26.s, z10.s, z2.s[2]\n"
+ "fmla z27.s, z10.s, z3.s[2]\n"
+ "fmla z24.s, z11.s, z0.s[3]\n"
+ "fmla z28.s, z10.s, z4.s[2]\n"
+ "fmla z25.s, z11.s, z1.s[3]\n"
+ "fmla z26.s, z11.s, z2.s[3]\n"
+ "fmla z27.s, z11.s, z3.s[3]\n"
+ "fmla z28.s, z11.s, z4.s[3]\n"
+ "bgt 66b\n"
+ "67:" // Height 5: Multiply loop: Single iteration only
+ "ld1w { z12.s }, p2/Z, [x7]\n"
+ "whilelt p0.s, XZR, x15\n"
+ "subs x15, x15, #0x1\n"
+ "ld1rqw { z0.s }, p0/Z, [x14]\n"
+ "fmla z24.s, z12.s, z0.s[0]\n"
+ "ld1rqw { z1.s }, p0/Z, [x12]\n"
+ "add x14, x14, #0x10\n"
+ "fmla z25.s, z12.s, z1.s[0]\n"
+ "ld1rqw { z2.s }, p0/Z, [x10]\n"
+ "add x12, x12, #0x10\n"
+ "fmla z26.s, z12.s, z2.s[0]\n"
+ "ld1rqw { z3.s }, p0/Z, [x28]\n"
+ "add x10, x10, #0x10\n"
+ "fmla z27.s, z12.s, z3.s[0]\n"
+ "ld1rqw { z4.s }, p0/Z, [x26]\n"
+ "add x28, x28, #0x10\n"
+ "fmla z28.s, z12.s, z4.s[0]\n"
+ "add x26, x26, #0x10\n"
+ "addvl x7, x7, #1\n"
+ "ble 68f\n"
+ "ld1w { z13.s }, p2/Z, [x7]\n"
+ "fmla z24.s, z13.s, z0.s[1]\n"
+ "subs x15, x15, #0x1\n"
+ "fmla z25.s, z13.s, z1.s[1]\n"
+ "addvl x7, x7, #1\n"
+ "fmla z26.s, z13.s, z2.s[1]\n"
+ "fmla z27.s, z13.s, z3.s[1]\n"
+ "fmla z28.s, z13.s, z4.s[1]\n"
+ "ble 68f\n"
+ "ld1w { z14.s }, p2/Z, [x7]\n"
+ "fmla z24.s, z14.s, z0.s[2]\n"
+ "subs x15, x15, #0x1\n"
+ "fmla z25.s, z14.s, z1.s[2]\n"
+ "addvl x7, x7, #1\n"
+ "fmla z26.s, z14.s, z2.s[2]\n"
+ "fmla z27.s, z14.s, z3.s[2]\n"
+ "fmla z28.s, z14.s, z4.s[2]\n"
+ "ble 68f\n"
+ "ld1w { z15.s }, p2/Z, [x7]\n"
+ "fmla z24.s, z15.s, z0.s[3]\n"
+ "addvl x7, x7, #1\n"
+ "fmla z25.s, z15.s, z1.s[3]\n"
+ "fmla z26.s, z15.s, z2.s[3]\n"
+ "fmla z27.s, z15.s, z3.s[3]\n"
+ "fmla z28.s, z15.s, z4.s[3]\n"
+ "68:" // Height 5: Multiply loop: multiply skip
+ "prfm pldl1keep, [x14, #0x80]\n"
+ "add x16, x16, #0x1\n"
+ "prfm pldl1keep, [x12, #0x80]\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "cmp x16, x19\n"
+ "bne 63b\n"
+ "prfm pstl1keep, [x17, #0x0]\n"
+ "prfm pstl1keep, [x13, #0x0]\n"
+ "prfm pstl1keep, [x11, #0x0]\n"
+ "prfm pstl1keep, [x9, #0x0]\n"
+ "prfm pstl1keep, [x27, #0x0]\n"
+ "tbz %x[flags], #1, 69f\n"
+ "add x19, %x[args_ptr], %[offset_min]\n"
+ "ld1rw { z17.s }, p2/Z, [x19]\n"
+ "add x19, %x[args_ptr], %[offset_max]\n"
+ "ld1rw { z16.s }, p2/Z, [x19]\n"
+ "fmin z24.s, p2/M, z24.s, z16.s\n"
+ "fmin z25.s, p2/M, z25.s, z16.s\n"
+ "fmin z26.s, p2/M, z26.s, z16.s\n"
+ "fmin z27.s, p2/M, z27.s, z16.s\n"
+ "fmin z28.s, p2/M, z28.s, z16.s\n"
+ "fmax z24.s, p2/M, z24.s, z17.s\n"
+ "fmax z25.s, p2/M, z25.s, z17.s\n"
+ "fmax z26.s, p2/M, z26.s, z17.s\n"
+ "fmax z27.s, p2/M, z27.s, z17.s\n"
+ "fmax z28.s, p2/M, z28.s, z17.s\n"
+ "69:" // Height 5: No activation
+ "st1w { z24.s }, p1, [x17]\n"
+ "addvl x17, x17, #1\n"
+ "st1w { z25.s }, p1, [x13]\n"
+ "addvl x13, x13, #1\n"
+ "st1w { z26.s }, p1, [x11]\n"
+ "addvl x11, x11, #1\n"
+ "st1w { z27.s }, p1, [x9]\n"
+ "addvl x9, x9, #1\n"
+ "st1w { z28.s }, p1, [x27]\n"
+ "addvl x27, x27, #1\n"
+ "70:" // Height 5: Writeback done
+ "mov x19, #0x0\n"
+ "incw x19\n"
+ "subs x6, x6, x19\n"
+ "bgt 59b\n"
+ "b 114f\n"
+ "71:" // Height 6
+ "ldr x6, [%x[args_ptr], %[offsetof_N]]\n"
+ "mov x8, %x[bias]\n"
+ "ldr x7, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 72f\n"
+ "ldr x17, [%x[output_ptr], #0x0]\n"
+ "add x17, x17, x19, LSL #2\n"
+ "ldr x13, [%x[output_ptr], #0x8]\n"
+ "ldr x11, [%x[output_ptr], #0x10]\n"
+ "add x13, x13, x19, LSL #2\n"
+ "ldr x9, [%x[output_ptr], #0x18]\n"
+ "ldr x27, [%x[output_ptr], #0x20]\n"
+ "add x11, x11, x19, LSL #2\n"
+ "ldr x25, [%x[output_ptr], #0x28]\n"
+ "add x9, x9, x19, LSL #2\n"
+ "add x27, x27, x19, LSL #2\n"
+ "add x25, x25, x19, LSL #2\n"
+ "b 73f\n"
+ "72:" // Height 6: setup direct output
+ "mov x17, %x[output_ptr]\n"
+ "add x13, x17, x19, LSL #2\n"
+ "add x11, x13, x19, LSL #2\n"
+ "add x9, x11, x19, LSL #2\n"
+ "add x27, x9, x19, LSL #2\n"
+ "add x25, x27, x19, LSL #2\n"
+ "73:" // Height 6: Column loop
+ "mov x19, #0x0\n"
+ "whilelt p1.s, x19, x6\n"
+ "cbz x8, 74f\n"
+ "ld1w { z24.s }, p2/Z, [x8]\n"
+ "mov z25.d, z24.d\n"
+ "addvl x8, x8, #1\n"
+ "mov z26.d, z24.d\n"
+ "mov z27.d, z24.d\n"
+ "mov z28.d, z24.d\n"
+ "mov z29.d, z24.d\n"
+ "b 76f\n"
+ "74:" // Height 6: no bias
+ "tbz %x[flags], #0, 75f\n"
+ "ld1w { z24.s }, p1/Z, [x17]\n"
+ "ld1w { z25.s }, p1/Z, [x13]\n"
+ "ld1w { z26.s }, p1/Z, [x11]\n"
+ "ld1w { z27.s }, p1/Z, [x9]\n"
+ "ld1w { z28.s }, p1/Z, [x27]\n"
+ "ld1w { z29.s }, p1/Z, [x25]\n"
+ "b 76f\n"
+ "75:" // Height 6: no accumulate
+ "mov z24.b, #0x0\n"
+ "mov z25.b, #0x0\n"
+ "mov z26.b, #0x0\n"
+ "mov z27.b, #0x0\n"
+ "mov z28.b, #0x0\n"
+ "mov z29.b, #0x0\n"
+ "76:" // Height 6: setup done
+ "mov x16, #0x0\n"
+ "77:" // Height 6: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w15, [x20, x16, LSL #0x2]\n"
+ "tbz %x[flags], #3, 78f\n"
+ "ldr x20, [%x[input_ptr], x16, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x14, [x20, #0x0]\n"
+ "ldr x12, [x20, #0x8]\n"
+ "ldr x10, [x20, #0x10]\n"
+ "ldr x28, [x20, #0x18]\n"
+ "ldr x26, [x20, #0x20]\n"
+ "ldr x24, [x20, #0x28]\n"
+ "cbnz x16, 79f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x14, x14, x19, LSL #2\n"
+ "add x12, x12, x19, LSL #2\n"
+ "add x10, x10, x19, LSL #2\n"
+ "add x28, x28, x19, LSL #2\n"
+ "add x26, x26, x19, LSL #2\n"
+ "add x24, x24, x19, LSL #2\n"
+ "b 79f\n"
+ "78:" // Height 6: setup direct input
+ "mov x14, %x[input_ptr]\n"
+ "add x12, x14, x19, LSL #2\n"
+ "add x10, x12, x19, LSL #2\n"
+ "add x28, x10, x19, LSL #2\n"
+ "add x26, x28, x19, LSL #2\n"
+ "add x24, x26, x19, LSL #2\n"
+ "79:" // Height 6: input setup done
+ "cmp x15, #0x4\n"
+ "ble 81f\n"
+ "80:" // Height 6: Multiply loop: Main loop head
+ "ld1w { z8.s }, p2/Z, [x7]\n"
+ "whilelt p0.s, XZR, x15\n"
+ "ld1w { z9.s }, p2/Z, [x7, #1, MUL VL]\n"
+ "sub x15, x15, #0x4\n"
+ "ld1rqw { z0.s }, p0/Z, [x14]\n"
+ "fmla z24.s, z8.s, z0.s[0]\n"
+ "ld1rqw { z1.s }, p0/Z, [x12]\n"
+ "add x14, x14, #0x10\n"
+ "fmla z25.s, z8.s, z1.s[0]\n"
+ "ld1rqw { z2.s }, p0/Z, [x10]\n"
+ "add x12, x12, #0x10\n"
+ "fmla z24.s, z9.s, z0.s[1]\n"
+ "ld1rqw { z3.s }, p0/Z, [x28]\n"
+ "add x10, x10, #0x10\n"
+ "fmla z26.s, z8.s, z2.s[0]\n"
+ "ld1rqw { z4.s }, p0/Z, [x26]\n"
+ "add x28, x28, #0x10\n"
+ "fmla z27.s, z8.s, z3.s[0]\n"
+ "ld1rqw { z5.s }, p0/Z, [x24]\n"
+ "add x26, x26, #0x10\n"
+ "fmla z25.s, z9.s, z1.s[1]\n"
+ "ld1w { z10.s }, p2/Z, [x7, #2, MUL VL]\n"
+ "add x24, x24, #0x10\n"
+ "fmla z28.s, z8.s, z4.s[0]\n"
+ "ld1w { z11.s }, p2/Z, [x7, #3, MUL VL]\n"
+ "cmp x15, #0x4\n"
+ "fmla z29.s, z8.s, z5.s[0]\n"
+ "prfm pldl1keep, [x14, #0x80]\n"
+ "addvl x7, x7, #4\n"
+ "fmla z26.s, z9.s, z2.s[1]\n"
+ "prfm pldl1keep, [x12, #0x80]\n"
+ "fmla z27.s, z9.s, z3.s[1]\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "fmla z24.s, z10.s, z0.s[2]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "fmla z28.s, z9.s, z4.s[1]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "fmla z29.s, z9.s, z5.s[1]\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "fmla z25.s, z10.s, z1.s[2]\n"
+ "fmla z26.s, z10.s, z2.s[2]\n"
+ "fmla z27.s, z10.s, z3.s[2]\n"
+ "fmla z28.s, z10.s, z4.s[2]\n"
+ "fmla z29.s, z10.s, z5.s[2]\n"
+ "fmla z24.s, z11.s, z0.s[3]\n"
+ "fmla z25.s, z11.s, z1.s[3]\n"
+ "fmla z26.s, z11.s, z2.s[3]\n"
+ "fmla z27.s, z11.s, z3.s[3]\n"
+ "fmla z28.s, z11.s, z4.s[3]\n"
+ "fmla z29.s, z11.s, z5.s[3]\n"
+ "bgt 80b\n"
+ "81:" // Height 6: Multiply loop: Single iteration only
+ "ld1w { z12.s }, p2/Z, [x7]\n"
+ "whilelt p0.s, XZR, x15\n"
+ "subs x15, x15, #0x1\n"
+ "ld1rqw { z0.s }, p0/Z, [x14]\n"
+ "fmla z24.s, z12.s, z0.s[0]\n"
+ "ld1rqw { z1.s }, p0/Z, [x12]\n"
+ "add x14, x14, #0x10\n"
+ "fmla z25.s, z12.s, z1.s[0]\n"
+ "ld1rqw { z2.s }, p0/Z, [x10]\n"
+ "add x12, x12, #0x10\n"
+ "fmla z26.s, z12.s, z2.s[0]\n"
+ "ld1rqw { z3.s }, p0/Z, [x28]\n"
+ "add x10, x10, #0x10\n"
+ "fmla z27.s, z12.s, z3.s[0]\n"
+ "ld1rqw { z4.s }, p0/Z, [x26]\n"
+ "add x28, x28, #0x10\n"
+ "fmla z28.s, z12.s, z4.s[0]\n"
+ "ld1rqw { z5.s }, p0/Z, [x24]\n"
+ "add x26, x26, #0x10\n"
+ "fmla z29.s, z12.s, z5.s[0]\n"
+ "add x24, x24, #0x10\n"
+ "addvl x7, x7, #1\n"
+ "ble 82f\n"
+ "ld1w { z13.s }, p2/Z, [x7]\n"
+ "fmla z24.s, z13.s, z0.s[1]\n"
+ "subs x15, x15, #0x1\n"
+ "fmla z25.s, z13.s, z1.s[1]\n"
+ "addvl x7, x7, #1\n"
+ "fmla z26.s, z13.s, z2.s[1]\n"
+ "fmla z27.s, z13.s, z3.s[1]\n"
+ "fmla z28.s, z13.s, z4.s[1]\n"
+ "fmla z29.s, z13.s, z5.s[1]\n"
+ "ble 82f\n"
+ "ld1w { z14.s }, p2/Z, [x7]\n"
+ "fmla z24.s, z14.s, z0.s[2]\n"
+ "subs x15, x15, #0x1\n"
+ "fmla z25.s, z14.s, z1.s[2]\n"
+ "addvl x7, x7, #1\n"
+ "fmla z26.s, z14.s, z2.s[2]\n"
+ "fmla z27.s, z14.s, z3.s[2]\n"
+ "fmla z28.s, z14.s, z4.s[2]\n"
+ "fmla z29.s, z14.s, z5.s[2]\n"
+ "ble 82f\n"
+ "ld1w { z15.s }, p2/Z, [x7]\n"
+ "fmla z24.s, z15.s, z0.s[3]\n"
+ "addvl x7, x7, #1\n"
+ "fmla z25.s, z15.s, z1.s[3]\n"
+ "fmla z26.s, z15.s, z2.s[3]\n"
+ "fmla z27.s, z15.s, z3.s[3]\n"
+ "fmla z28.s, z15.s, z4.s[3]\n"
+ "fmla z29.s, z15.s, z5.s[3]\n"
+ "82:" // Height 6: Multiply loop: multiply skip
+ "prfm pldl1keep, [x14, #0x80]\n"
+ "add x16, x16, #0x1\n"
+ "prfm pldl1keep, [x12, #0x80]\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "cmp x16, x19\n"
+ "bne 77b\n"
+ "prfm pstl1keep, [x17, #0x0]\n"
+ "prfm pstl1keep, [x13, #0x0]\n"
+ "prfm pstl1keep, [x11, #0x0]\n"
+ "prfm pstl1keep, [x9, #0x0]\n"
+ "prfm pstl1keep, [x27, #0x0]\n"
+ "prfm pstl1keep, [x25, #0x0]\n"
+ "tbz %x[flags], #1, 83f\n"
+ "add x19, %x[args_ptr], %[offset_min]\n"
+ "ld1rw { z17.s }, p2/Z, [x19]\n"
+ "add x19, %x[args_ptr], %[offset_max]\n"
+ "ld1rw { z16.s }, p2/Z, [x19]\n"
+ "fmin z24.s, p2/M, z24.s, z16.s\n"
+ "fmin z25.s, p2/M, z25.s, z16.s\n"
+ "fmin z26.s, p2/M, z26.s, z16.s\n"
+ "fmin z27.s, p2/M, z27.s, z16.s\n"
+ "fmin z28.s, p2/M, z28.s, z16.s\n"
+ "fmax z24.s, p2/M, z24.s, z17.s\n"
+ "fmax z25.s, p2/M, z25.s, z17.s\n"
+ "fmax z26.s, p2/M, z26.s, z17.s\n"
+ "fmax z27.s, p2/M, z27.s, z17.s\n"
+ "fmax z28.s, p2/M, z28.s, z17.s\n"
+ "fmin z29.s, p2/M, z29.s, z16.s\n"
+ "fmax z29.s, p2/M, z29.s, z17.s\n"
+ "83:" // Height 6: No activation
+ "st1w { z24.s }, p1, [x17]\n"
+ "addvl x17, x17, #1\n"
+ "st1w { z25.s }, p1, [x13]\n"
+ "addvl x13, x13, #1\n"
+ "st1w { z26.s }, p1, [x11]\n"
+ "addvl x11, x11, #1\n"
+ "st1w { z27.s }, p1, [x9]\n"
+ "addvl x9, x9, #1\n"
+ "st1w { z28.s }, p1, [x27]\n"
+ "addvl x27, x27, #1\n"
+ "st1w { z29.s }, p1, [x25]\n"
+ "addvl x25, x25, #1\n"
+ "84:" // Height 6: Writeback done
+ "mov x19, #0x0\n"
+ "incw x19\n"
+ "subs x6, x6, x19\n"
+ "bgt 73b\n"
+ "b 114f\n"
+ "85:" // Height 7
+ "ldr x6, [%x[args_ptr], %[offsetof_N]]\n"
+ "mov x8, %x[bias]\n"
+ "ldr x7, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 86f\n"
+ "ldr x17, [%x[output_ptr], #0x0]\n"
+ "add x17, x17, x19, LSL #2\n"
+ "ldr x13, [%x[output_ptr], #0x8]\n"
+ "ldr x11, [%x[output_ptr], #0x10]\n"
+ "add x13, x13, x19, LSL #2\n"
+ "ldr x9, [%x[output_ptr], #0x18]\n"
+ "ldr x27, [%x[output_ptr], #0x20]\n"
+ "add x11, x11, x19, LSL #2\n"
+ "ldr x25, [%x[output_ptr], #0x28]\n"
+ "add x9, x9, x19, LSL #2\n"
+ "ldr x23, [%x[output_ptr], #0x30]\n"
+ "add x27, x27, x19, LSL #2\n"
+ "add x25, x25, x19, LSL #2\n"
+ "add x23, x23, x19, LSL #2\n"
+ "b 87f\n"
+ "86:" // Height 7: setup direct output
+ "mov x17, %x[output_ptr]\n"
+ "add x13, x17, x19, LSL #2\n"
+ "add x11, x13, x19, LSL #2\n"
+ "add x9, x11, x19, LSL #2\n"
+ "add x27, x9, x19, LSL #2\n"
+ "add x25, x27, x19, LSL #2\n"
+ "add x23, x25, x19, LSL #2\n"
+ "87:" // Height 7: Column loop
+ "mov x19, #0x0\n"
+ "whilelt p1.s, x19, x6\n"
+ "cbz x8, 88f\n"
+ "ld1w { z24.s }, p2/Z, [x8]\n"
+ "mov z25.d, z24.d\n"
+ "addvl x8, x8, #1\n"
+ "mov z26.d, z24.d\n"
+ "mov z27.d, z24.d\n"
+ "mov z28.d, z24.d\n"
+ "mov z29.d, z24.d\n"
+ "mov z30.d, z24.d\n"
+ "b 90f\n"
+ "88:" // Height 7: no bias
+ "tbz %x[flags], #0, 89f\n"
+ "ld1w { z24.s }, p1/Z, [x17]\n"
+ "ld1w { z25.s }, p1/Z, [x13]\n"
+ "ld1w { z26.s }, p1/Z, [x11]\n"
+ "ld1w { z27.s }, p1/Z, [x9]\n"
+ "ld1w { z28.s }, p1/Z, [x27]\n"
+ "ld1w { z29.s }, p1/Z, [x25]\n"
+ "ld1w { z30.s }, p1/Z, [x23]\n"
+ "b 90f\n"
+ "89:" // Height 7: no accumulate
+ "mov z24.b, #0x0\n"
+ "mov z25.b, #0x0\n"
+ "mov z26.b, #0x0\n"
+ "mov z27.b, #0x0\n"
+ "mov z28.b, #0x0\n"
+ "mov z29.b, #0x0\n"
+ "mov z30.b, #0x0\n"
+ "90:" // Height 7: setup done
+ "mov x16, #0x0\n"
+ "91:" // Height 7: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w15, [x20, x16, LSL #0x2]\n"
+ "tbz %x[flags], #3, 92f\n"
+ "ldr x20, [%x[input_ptr], x16, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x14, [x20, #0x0]\n"
+ "ldr x12, [x20, #0x8]\n"
+ "ldr x10, [x20, #0x10]\n"
+ "ldr x28, [x20, #0x18]\n"
+ "ldr x26, [x20, #0x20]\n"
+ "ldr x24, [x20, #0x28]\n"
+ "ldr x22, [x20, #0x30]\n"
+ "cbnz x16, 93f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x14, x14, x19, LSL #2\n"
+ "add x12, x12, x19, LSL #2\n"
+ "add x10, x10, x19, LSL #2\n"
+ "add x28, x28, x19, LSL #2\n"
+ "add x26, x26, x19, LSL #2\n"
+ "add x24, x24, x19, LSL #2\n"
+ "add x22, x22, x19, LSL #2\n"
+ "b 93f\n"
+ "92:" // Height 7: setup direct input
+ "mov x14, %x[input_ptr]\n"
+ "add x12, x14, x19, LSL #2\n"
+ "add x10, x12, x19, LSL #2\n"
+ "add x28, x10, x19, LSL #2\n"
+ "add x26, x28, x19, LSL #2\n"
+ "add x24, x26, x19, LSL #2\n"
+ "add x22, x24, x19, LSL #2\n"
+ "93:" // Height 7: input setup done
+ "cmp x15, #0x4\n"
+ "ble 95f\n"
+ "94:" // Height 7: Multiply loop: Main loop head
+ "ld1w { z8.s }, p2/Z, [x7]\n"
+ "whilelt p0.s, XZR, x15\n"
+ "ld1w { z9.s }, p2/Z, [x7, #1, MUL VL]\n"
+ "sub x15, x15, #0x4\n"
+ "ld1rqw { z0.s }, p0/Z, [x14]\n"
+ "fmla z24.s, z8.s, z0.s[0]\n"
+ "ld1rqw { z1.s }, p0/Z, [x12]\n"
+ "add x14, x14, #0x10\n"
+ "fmla z25.s, z8.s, z1.s[0]\n"
+ "ld1rqw { z2.s }, p0/Z, [x10]\n"
+ "add x12, x12, #0x10\n"
+ "fmla z24.s, z9.s, z0.s[1]\n"
+ "ld1rqw { z3.s }, p0/Z, [x28]\n"
+ "add x10, x10, #0x10\n"
+ "fmla z26.s, z8.s, z2.s[0]\n"
+ "ld1rqw { z4.s }, p0/Z, [x26]\n"
+ "add x28, x28, #0x10\n"
+ "fmla z27.s, z8.s, z3.s[0]\n"
+ "ld1rqw { z5.s }, p0/Z, [x24]\n"
+ "add x26, x26, #0x10\n"
+ "fmla z25.s, z9.s, z1.s[1]\n"
+ "ld1rqw { z6.s }, p0/Z, [x22]\n"
+ "add x24, x24, #0x10\n"
+ "fmla z28.s, z8.s, z4.s[0]\n"
+ "ld1w { z10.s }, p2/Z, [x7, #2, MUL VL]\n"
+ "add x22, x22, #0x10\n"
+ "fmla z29.s, z8.s, z5.s[0]\n"
+ "ld1w { z11.s }, p2/Z, [x7, #3, MUL VL]\n"
+ "cmp x15, #0x4\n"
+ "fmla z30.s, z8.s, z6.s[0]\n"
+ "prfm pldl1keep, [x14, #0x80]\n"
+ "addvl x7, x7, #4\n"
+ "fmla z26.s, z9.s, z2.s[1]\n"
+ "prfm pldl1keep, [x12, #0x80]\n"
+ "fmla z27.s, z9.s, z3.s[1]\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "fmla z28.s, z9.s, z4.s[1]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "fmla z29.s, z9.s, z5.s[1]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "fmla z30.s, z9.s, z6.s[1]\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "fmla z24.s, z10.s, z0.s[2]\n"
+ "prfm pldl1keep, [x22, #0x80]\n"
+ "fmla z25.s, z10.s, z1.s[2]\n"
+ "fmla z26.s, z10.s, z2.s[2]\n"
+ "fmla z27.s, z10.s, z3.s[2]\n"
+ "fmla z28.s, z10.s, z4.s[2]\n"
+ "fmla z29.s, z10.s, z5.s[2]\n"
+ "fmla z30.s, z10.s, z6.s[2]\n"
+ "fmla z24.s, z11.s, z0.s[3]\n"
+ "fmla z25.s, z11.s, z1.s[3]\n"
+ "fmla z26.s, z11.s, z2.s[3]\n"
+ "fmla z27.s, z11.s, z3.s[3]\n"
+ "fmla z28.s, z11.s, z4.s[3]\n"
+ "fmla z29.s, z11.s, z5.s[3]\n"
+ "fmla z30.s, z11.s, z6.s[3]\n"
+ "bgt 94b\n"
+ "95:" // Height 7: Multiply loop: Single iteration only
+ "ld1w { z12.s }, p2/Z, [x7]\n"
+ "whilelt p0.s, XZR, x15\n"
+ "subs x15, x15, #0x1\n"
+ "ld1rqw { z0.s }, p0/Z, [x14]\n"
+ "fmla z24.s, z12.s, z0.s[0]\n"
+ "ld1rqw { z1.s }, p0/Z, [x12]\n"
+ "add x14, x14, #0x10\n"
+ "fmla z25.s, z12.s, z1.s[0]\n"
+ "ld1rqw { z2.s }, p0/Z, [x10]\n"
+ "add x12, x12, #0x10\n"
+ "fmla z26.s, z12.s, z2.s[0]\n"
+ "ld1rqw { z3.s }, p0/Z, [x28]\n"
+ "add x10, x10, #0x10\n"
+ "fmla z27.s, z12.s, z3.s[0]\n"
+ "ld1rqw { z4.s }, p0/Z, [x26]\n"
+ "add x28, x28, #0x10\n"
+ "fmla z28.s, z12.s, z4.s[0]\n"
+ "ld1rqw { z5.s }, p0/Z, [x24]\n"
+ "add x26, x26, #0x10\n"
+ "fmla z29.s, z12.s, z5.s[0]\n"
+ "ld1rqw { z6.s }, p0/Z, [x22]\n"
+ "add x24, x24, #0x10\n"
+ "fmla z30.s, z12.s, z6.s[0]\n"
+ "add x22, x22, #0x10\n"
+ "addvl x7, x7, #1\n"
+ "ble 96f\n"
+ "ld1w { z13.s }, p2/Z, [x7]\n"
+ "fmla z24.s, z13.s, z0.s[1]\n"
+ "subs x15, x15, #0x1\n"
+ "fmla z25.s, z13.s, z1.s[1]\n"
+ "addvl x7, x7, #1\n"
+ "fmla z26.s, z13.s, z2.s[1]\n"
+ "fmla z27.s, z13.s, z3.s[1]\n"
+ "fmla z28.s, z13.s, z4.s[1]\n"
+ "fmla z29.s, z13.s, z5.s[1]\n"
+ "fmla z30.s, z13.s, z6.s[1]\n"
+ "ble 96f\n"
+ "ld1w { z14.s }, p2/Z, [x7]\n"
+ "fmla z24.s, z14.s, z0.s[2]\n"
+ "subs x15, x15, #0x1\n"
+ "fmla z25.s, z14.s, z1.s[2]\n"
+ "addvl x7, x7, #1\n"
+ "fmla z26.s, z14.s, z2.s[2]\n"
+ "fmla z27.s, z14.s, z3.s[2]\n"
+ "fmla z28.s, z14.s, z4.s[2]\n"
+ "fmla z29.s, z14.s, z5.s[2]\n"
+ "fmla z30.s, z14.s, z6.s[2]\n"
+ "ble 96f\n"
+ "ld1w { z15.s }, p2/Z, [x7]\n"
+ "fmla z24.s, z15.s, z0.s[3]\n"
+ "addvl x7, x7, #1\n"
+ "fmla z25.s, z15.s, z1.s[3]\n"
+ "fmla z26.s, z15.s, z2.s[3]\n"
+ "fmla z27.s, z15.s, z3.s[3]\n"
+ "fmla z28.s, z15.s, z4.s[3]\n"
+ "fmla z29.s, z15.s, z5.s[3]\n"
+ "fmla z30.s, z15.s, z6.s[3]\n"
+ "96:" // Height 7: Multiply loop: multiply skip
+ "prfm pldl1keep, [x14, #0x80]\n"
+ "add x16, x16, #0x1\n"
+ "prfm pldl1keep, [x12, #0x80]\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "prfm pldl1keep, [x22, #0x80]\n"
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "cmp x16, x19\n"
+ "bne 91b\n"
+ "prfm pstl1keep, [x17, #0x0]\n"
+ "prfm pstl1keep, [x13, #0x0]\n"
+ "prfm pstl1keep, [x11, #0x0]\n"
+ "prfm pstl1keep, [x9, #0x0]\n"
+ "prfm pstl1keep, [x27, #0x0]\n"
+ "prfm pstl1keep, [x25, #0x0]\n"
+ "prfm pstl1keep, [x23, #0x0]\n"
+ "tbz %x[flags], #1, 97f\n"
+ "add x19, %x[args_ptr], %[offset_min]\n"
+ "ld1rw { z17.s }, p2/Z, [x19]\n"
+ "add x19, %x[args_ptr], %[offset_max]\n"
+ "ld1rw { z16.s }, p2/Z, [x19]\n"
+ "fmin z24.s, p2/M, z24.s, z16.s\n"
+ "fmin z25.s, p2/M, z25.s, z16.s\n"
+ "fmin z26.s, p2/M, z26.s, z16.s\n"
+ "fmin z27.s, p2/M, z27.s, z16.s\n"
+ "fmin z28.s, p2/M, z28.s, z16.s\n"
+ "fmax z24.s, p2/M, z24.s, z17.s\n"
+ "fmax z25.s, p2/M, z25.s, z17.s\n"
+ "fmax z26.s, p2/M, z26.s, z17.s\n"
+ "fmax z27.s, p2/M, z27.s, z17.s\n"
+ "fmax z28.s, p2/M, z28.s, z17.s\n"
+ "fmin z29.s, p2/M, z29.s, z16.s\n"
+ "fmin z30.s, p2/M, z30.s, z16.s\n"
+ "fmax z29.s, p2/M, z29.s, z17.s\n"
+ "fmax z30.s, p2/M, z30.s, z17.s\n"
+ "97:" // Height 7: No activation
+ "st1w { z24.s }, p1, [x17]\n"
+ "addvl x17, x17, #1\n"
+ "st1w { z25.s }, p1, [x13]\n"
+ "addvl x13, x13, #1\n"
+ "st1w { z26.s }, p1, [x11]\n"
+ "addvl x11, x11, #1\n"
+ "st1w { z27.s }, p1, [x9]\n"
+ "addvl x9, x9, #1\n"
+ "st1w { z28.s }, p1, [x27]\n"
+ "addvl x27, x27, #1\n"
+ "st1w { z29.s }, p1, [x25]\n"
+ "addvl x25, x25, #1\n"
+ "st1w { z30.s }, p1, [x23]\n"
+ "addvl x23, x23, #1\n"
+ "98:" // Height 7: Writeback done
+ "mov x19, #0x0\n"
+ "incw x19\n"
+ "subs x6, x6, x19\n"
+ "bgt 87b\n"
+ "b 114f\n"
+ "99:" // Height 8
+ "ldr x6, [%x[args_ptr], %[offsetof_N]]\n"
+ "mov x8, %x[bias]\n"
+ "ldr x7, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 100f\n"
+ "ldr x17, [%x[output_ptr], #0x0]\n"
+ "add x17, x17, x19, LSL #2\n"
+ "ldr x13, [%x[output_ptr], #0x8]\n"
+ "ldr x11, [%x[output_ptr], #0x10]\n"
+ "add x13, x13, x19, LSL #2\n"
+ "ldr x9, [%x[output_ptr], #0x18]\n"
+ "ldr x27, [%x[output_ptr], #0x20]\n"
+ "add x11, x11, x19, LSL #2\n"
+ "ldr x25, [%x[output_ptr], #0x28]\n"
+ "add x9, x9, x19, LSL #2\n"
+ "ldr x23, [%x[output_ptr], #0x30]\n"
+ "ldr x21, [%x[output_ptr], #0x38]\n"
+ "add x27, x27, x19, LSL #2\n"
+ "add x25, x25, x19, LSL #2\n"
+ "add %x[output_ptr], %x[output_ptr], #0x40\n"
+ "add x23, x23, x19, LSL #2\n"
+ "add x21, x21, x19, LSL #2\n"
+ "b 101f\n"
+ "100:" // Height 8: setup direct output
+ "mov x17, %x[output_ptr]\n"
+ "add x13, x17, x19, LSL #2\n"
+ "add x11, x13, x19, LSL #2\n"
+ "add x9, x11, x19, LSL #2\n"
+ "add x27, x9, x19, LSL #2\n"
+ "add x25, x27, x19, LSL #2\n"
+ "add x23, x25, x19, LSL #2\n"
+ "add x21, x23, x19, LSL #2\n"
+ "add %x[output_ptr], x21, x19, LSL #2\n"
+ "101:" // Height 8: Column loop
+ "mov x19, #0x0\n"
+ "whilelt p1.s, x19, x6\n"
+ "cbz x8, 102f\n"
+ "ld1w { z24.s }, p2/Z, [x8]\n"
+ "mov z25.d, z24.d\n"
+ "addvl x8, x8, #1\n"
+ "mov z26.d, z24.d\n"
+ "mov z27.d, z24.d\n"
+ "mov z28.d, z24.d\n"
+ "mov z29.d, z24.d\n"
+ "mov z30.d, z24.d\n"
+ "mov z31.d, z24.d\n"
+ "b 104f\n"
+ "102:" // Height 8: no bias
+ "tbz %x[flags], #0, 103f\n"
+ "ld1w { z24.s }, p1/Z, [x17]\n"
+ "ld1w { z25.s }, p1/Z, [x13]\n"
+ "ld1w { z26.s }, p1/Z, [x11]\n"
+ "ld1w { z27.s }, p1/Z, [x9]\n"
+ "ld1w { z28.s }, p1/Z, [x27]\n"
+ "ld1w { z29.s }, p1/Z, [x25]\n"
+ "ld1w { z30.s }, p1/Z, [x23]\n"
+ "ld1w { z31.s }, p1/Z, [x21]\n"
+ "b 104f\n"
+ "103:" // Height 8: no accumulate
+ "mov z24.b, #0x0\n"
+ "mov z25.b, #0x0\n"
+ "mov z26.b, #0x0\n"
+ "mov z27.b, #0x0\n"
+ "mov z28.b, #0x0\n"
+ "mov z29.b, #0x0\n"
+ "mov z30.b, #0x0\n"
+ "mov z31.b, #0x0\n"
+ "104:" // Height 8: setup done
+ "mov x16, #0x0\n"
+ "105:" // Height 8: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w15, [x20, x16, LSL #0x2]\n"
+ "tbz %x[flags], #3, 106f\n"
+ "ldr x20, [%x[input_ptr], x16, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x14, [x20, #0x0]\n"
+ "ldr x12, [x20, #0x8]\n"
+ "ldr x10, [x20, #0x10]\n"
+ "ldr x28, [x20, #0x18]\n"
+ "ldr x26, [x20, #0x20]\n"
+ "ldr x24, [x20, #0x28]\n"
+ "ldr x22, [x20, #0x30]\n"
+ "ldr x20, [x20, #0x38]\n"
+ "cbnz x16, 107f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x14, x14, x19, LSL #2\n"
+ "add x12, x12, x19, LSL #2\n"
+ "add x10, x10, x19, LSL #2\n"
+ "add x28, x28, x19, LSL #2\n"
+ "add x26, x26, x19, LSL #2\n"
+ "add x24, x24, x19, LSL #2\n"
+ "add x22, x22, x19, LSL #2\n"
+ "add x20, x20, x19, LSL #2\n"
+ "b 107f\n"
+ "106:" // Height 8: setup direct input
+ "mov x14, %x[input_ptr]\n"
+ "add x12, x14, x19, LSL #2\n"
+ "add x10, x12, x19, LSL #2\n"
+ "add x28, x10, x19, LSL #2\n"
+ "add x26, x28, x19, LSL #2\n"
+ "add x24, x26, x19, LSL #2\n"
+ "add x22, x24, x19, LSL #2\n"
+ "add x20, x22, x19, LSL #2\n"
+ "107:" // Height 8: input setup done
+ "cmp x15, #0x4\n"
+ "ble 109f\n"
+ "108:" // Height 8: Multiply loop: Main loop head
+ "ld1w { z8.s }, p2/Z, [x7]\n"
+ "whilelt p0.s, XZR, x15\n"
+ "ld1w { z9.s }, p2/Z, [x7, #1, MUL VL]\n"
+ "sub x15, x15, #0x4\n"
+ "ld1rqw { z0.s }, p0/Z, [x14]\n"
+ "fmla z24.s, z8.s, z0.s[0]\n"
+ "ld1rqw { z1.s }, p0/Z, [x12]\n"
+ "add x14, x14, #0x10\n"
+ "fmla z25.s, z8.s, z1.s[0]\n"
+ "ld1rqw { z2.s }, p0/Z, [x10]\n"
+ "add x12, x12, #0x10\n"
+ "fmla z24.s, z9.s, z0.s[1]\n"
+ "ld1rqw { z3.s }, p0/Z, [x28]\n"
+ "add x10, x10, #0x10\n"
+ "fmla z26.s, z8.s, z2.s[0]\n"
+ "ld1rqw { z4.s }, p0/Z, [x26]\n"
+ "add x28, x28, #0x10\n"
+ "fmla z27.s, z8.s, z3.s[0]\n"
+ "ld1rqw { z5.s }, p0/Z, [x24]\n"
+ "add x26, x26, #0x10\n"
+ "fmla z25.s, z9.s, z1.s[1]\n"
+ "ld1rqw { z6.s }, p0/Z, [x22]\n"
+ "add x24, x24, #0x10\n"
+ "fmla z28.s, z8.s, z4.s[0]\n"
+ "ld1rqw { z7.s }, p0/Z, [x20]\n"
+ "add x22, x22, #0x10\n"
+ "fmla z29.s, z8.s, z5.s[0]\n"
+ "ld1w { z10.s }, p2/Z, [x7, #2, MUL VL]\n"
+ "add x20, x20, #0x10\n"
+ "fmla z30.s, z8.s, z6.s[0]\n"
+ "ld1w { z11.s }, p2/Z, [x7, #3, MUL VL]\n"
+ "cmp x15, #0x4\n"
+ "fmla z31.s, z8.s, z7.s[0]\n"
+ "prfm pldl1keep, [x14, #0x80]\n"
+ "addvl x7, x7, #4\n"
+ "fmla z26.s, z9.s, z2.s[1]\n"
+ "prfm pldl1keep, [x12, #0x80]\n"
+ "fmla z27.s, z9.s, z3.s[1]\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "fmla z28.s, z9.s, z4.s[1]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "fmla z29.s, z9.s, z5.s[1]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "fmla z30.s, z9.s, z6.s[1]\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "fmla z31.s, z9.s, z7.s[1]\n"
+ "prfm pldl1keep, [x22, #0x80]\n"
+ "fmla z24.s, z10.s, z0.s[2]\n"
+ "prfm pldl1keep, [x20, #0x80]\n"
+ "fmla z25.s, z10.s, z1.s[2]\n"
+ "fmla z26.s, z10.s, z2.s[2]\n"
+ "fmla z27.s, z10.s, z3.s[2]\n"
+ "fmla z28.s, z10.s, z4.s[2]\n"
+ "fmla z29.s, z10.s, z5.s[2]\n"
+ "fmla z30.s, z10.s, z6.s[2]\n"
+ "fmla z31.s, z10.s, z7.s[2]\n"
+ "fmla z24.s, z11.s, z0.s[3]\n"
+ "fmla z25.s, z11.s, z1.s[3]\n"
+ "fmla z26.s, z11.s, z2.s[3]\n"
+ "fmla z27.s, z11.s, z3.s[3]\n"
+ "fmla z28.s, z11.s, z4.s[3]\n"
+ "fmla z29.s, z11.s, z5.s[3]\n"
+ "fmla z30.s, z11.s, z6.s[3]\n"
+ "fmla z31.s, z11.s, z7.s[3]\n"
+ "bgt 108b\n"
+ "109:" // Height 8: Multiply loop: Single iteration only
+ "ld1w { z12.s }, p2/Z, [x7]\n"
+ "whilelt p0.s, XZR, x15\n"
+ "subs x15, x15, #0x1\n"
+ "ld1rqw { z0.s }, p0/Z, [x14]\n"
+ "fmla z24.s, z12.s, z0.s[0]\n"
+ "ld1rqw { z1.s }, p0/Z, [x12]\n"
+ "add x14, x14, #0x10\n"
+ "fmla z25.s, z12.s, z1.s[0]\n"
+ "ld1rqw { z2.s }, p0/Z, [x10]\n"
+ "add x12, x12, #0x10\n"
+ "fmla z26.s, z12.s, z2.s[0]\n"
+ "ld1rqw { z3.s }, p0/Z, [x28]\n"
+ "add x10, x10, #0x10\n"
+ "fmla z27.s, z12.s, z3.s[0]\n"
+ "ld1rqw { z4.s }, p0/Z, [x26]\n"
+ "add x28, x28, #0x10\n"
+ "fmla z28.s, z12.s, z4.s[0]\n"
+ "ld1rqw { z5.s }, p0/Z, [x24]\n"
+ "add x26, x26, #0x10\n"
+ "fmla z29.s, z12.s, z5.s[0]\n"
+ "ld1rqw { z6.s }, p0/Z, [x22]\n"
+ "add x24, x24, #0x10\n"
+ "fmla z30.s, z12.s, z6.s[0]\n"
+ "ld1rqw { z7.s }, p0/Z, [x20]\n"
+ "add x22, x22, #0x10\n"
+ "fmla z31.s, z12.s, z7.s[0]\n"
+ "add x20, x20, #0x10\n"
+ "addvl x7, x7, #1\n"
+ "ble 110f\n"
+ "ld1w { z13.s }, p2/Z, [x7]\n"
+ "fmla z24.s, z13.s, z0.s[1]\n"
+ "subs x15, x15, #0x1\n"
+ "fmla z25.s, z13.s, z1.s[1]\n"
+ "addvl x7, x7, #1\n"
+ "fmla z26.s, z13.s, z2.s[1]\n"
+ "fmla z27.s, z13.s, z3.s[1]\n"
+ "fmla z28.s, z13.s, z4.s[1]\n"
+ "fmla z29.s, z13.s, z5.s[1]\n"
+ "fmla z30.s, z13.s, z6.s[1]\n"
+ "fmla z31.s, z13.s, z7.s[1]\n"
+ "ble 110f\n"
+ "ld1w { z14.s }, p2/Z, [x7]\n"
+ "fmla z24.s, z14.s, z0.s[2]\n"
+ "subs x15, x15, #0x1\n"
+ "fmla z25.s, z14.s, z1.s[2]\n"
+ "addvl x7, x7, #1\n"
+ "fmla z26.s, z14.s, z2.s[2]\n"
+ "fmla z27.s, z14.s, z3.s[2]\n"
+ "fmla z28.s, z14.s, z4.s[2]\n"
+ "fmla z29.s, z14.s, z5.s[2]\n"
+ "fmla z30.s, z14.s, z6.s[2]\n"
+ "fmla z31.s, z14.s, z7.s[2]\n"
+ "ble 110f\n"
+ "ld1w { z15.s }, p2/Z, [x7]\n"
+ "fmla z24.s, z15.s, z0.s[3]\n"
+ "addvl x7, x7, #1\n"
+ "fmla z25.s, z15.s, z1.s[3]\n"
+ "fmla z26.s, z15.s, z2.s[3]\n"
+ "fmla z27.s, z15.s, z3.s[3]\n"
+ "fmla z28.s, z15.s, z4.s[3]\n"
+ "fmla z29.s, z15.s, z5.s[3]\n"
+ "fmla z30.s, z15.s, z6.s[3]\n"
+ "fmla z31.s, z15.s, z7.s[3]\n"
+ "110:" // Height 8: Multiply loop: multiply skip
+ "prfm pldl1keep, [x14, #0x80]\n"
+ "add x16, x16, #0x1\n"
+ "prfm pldl1keep, [x12, #0x80]\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "prfm pldl1keep, [x22, #0x80]\n"
+ "prfm pldl1keep, [x20, #0x80]\n"
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "cmp x16, x19\n"
+ "bne 105b\n"
+ "prfm pstl1keep, [x17, #0x0]\n"
+ "prfm pstl1keep, [x13, #0x0]\n"
+ "prfm pstl1keep, [x11, #0x0]\n"
+ "prfm pstl1keep, [x9, #0x0]\n"
+ "prfm pstl1keep, [x27, #0x0]\n"
+ "prfm pstl1keep, [x25, #0x0]\n"
+ "prfm pstl1keep, [x23, #0x0]\n"
+ "prfm pstl1keep, [x21, #0x0]\n"
+ "tbz %x[flags], #1, 111f\n"
+ "add x19, %x[args_ptr], %[offset_min]\n"
+ "ld1rw { z17.s }, p2/Z, [x19]\n"
+ "add x19, %x[args_ptr], %[offset_max]\n"
+ "ld1rw { z16.s }, p2/Z, [x19]\n"
+ "fmin z24.s, p2/M, z24.s, z16.s\n"
+ "fmin z25.s, p2/M, z25.s, z16.s\n"
+ "fmin z26.s, p2/M, z26.s, z16.s\n"
+ "fmin z27.s, p2/M, z27.s, z16.s\n"
+ "fmin z28.s, p2/M, z28.s, z16.s\n"
+ "fmax z24.s, p2/M, z24.s, z17.s\n"
+ "fmax z25.s, p2/M, z25.s, z17.s\n"
+ "fmax z26.s, p2/M, z26.s, z17.s\n"
+ "fmax z27.s, p2/M, z27.s, z17.s\n"
+ "fmax z28.s, p2/M, z28.s, z17.s\n"
+ "fmin z29.s, p2/M, z29.s, z16.s\n"
+ "fmin z30.s, p2/M, z30.s, z16.s\n"
+ "fmin z31.s, p2/M, z31.s, z16.s\n"
+ "fmax z29.s, p2/M, z29.s, z17.s\n"
+ "fmax z30.s, p2/M, z30.s, z17.s\n"
+ "fmax z31.s, p2/M, z31.s, z17.s\n"
+ "111:" // Height 8: No activation
+ "st1w { z24.s }, p1, [x17]\n"
+ "addvl x17, x17, #1\n"
+ "st1w { z25.s }, p1, [x13]\n"
+ "addvl x13, x13, #1\n"
+ "st1w { z26.s }, p1, [x11]\n"
+ "addvl x11, x11, #1\n"
+ "st1w { z27.s }, p1, [x9]\n"
+ "addvl x9, x9, #1\n"
+ "st1w { z28.s }, p1, [x27]\n"
+ "addvl x27, x27, #1\n"
+ "st1w { z29.s }, p1, [x25]\n"
+ "addvl x25, x25, #1\n"
+ "st1w { z30.s }, p1, [x23]\n"
+ "addvl x23, x23, #1\n"
+ "st1w { z31.s }, p1, [x21]\n"
+ "addvl x21, x21, #1\n"
+ "112:" // Height 8: Writeback done
+ "mov x19, #0x0\n"
+ "incw x19\n"
+ "subs x6, x6, x19\n"
+ "bgt 101b\n"
+ "subs %x[M], %x[M], #0x8\n"
+ "beq 114f\n"
+ "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "tbz %x[flags], #3, 113f\n"
+ "add x20, x20, #0x8\n"
+ "str x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "b 1b\n"
+ "113:" // Update direct input
+ "mov x19, #0x20\n"
+ "madd %x[input_ptr], x19, x20, %x[input_ptr]\n"
+ "b 1b\n"
+ "114:" // Exit
+
+ : [M] "+r" (M), [input_ptr] "+r" (input_ptr), [output_ptr] "+r" (output_ptr)
+ : [args_ptr] "r" (&ka), [bias] "r" (bias), [flags] "r" (flags), [offset_max] "I" (offsetof(KernelArgs, maxval)), [offset_min] "I" (offsetof(KernelArgs, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths))
+ : "cc", "memory", "p0", "p1", "p2", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+ );
+}
+
+} // namespace arm_gemm
+#endif // __ARM_FEATURE_SVE
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mmla_4VLx4.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mmla_4VLx4.hpp
deleted file mode 100644
index fd416ed..0000000
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mmla_4VLx4.hpp
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Copyright (c) 2018-2020 Arm Limited.
- *
- * SPDX-License-Identifier: MIT
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to
- * deal in the Software without restriction, including without limitation the
- * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
- * sell copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-#pragma once
-
-#ifdef __ARM_FEATURE_SVE
-
-
-#include "../std_transforms_sve.hpp"
-
-namespace arm_gemm
-{
-
-// Actual kernel implementations
-void sve_hybrid_fp32_mmla_4VLx4(const float *, int, const float *, float *, int, int, int, int, const float *, Activation, bool);
-
-class hybrid_fp32_mmla_4VLx4
-{
-public:
- typedef float operand_type;
- typedef float result_type;
-
- typedef void (*kern_type)(const float *, int, const float *, float *, int, int, int, int, const float *, Activation, bool);
-
- /* Kernel blocking parameters */
- static constexpr unsigned int out_height()
- {
- return 8;
- }
-
- static unsigned int out_width()
- {
- return get_vector_length<float>() * 2;
- }
-
- static constexpr unsigned int k_unroll()
- {
- return 2;
- }
-
- static constexpr bool supports_accumulate()
- {
- return true;
- }
-
- static constexpr bool supports_bias()
- {
- return true;
- }
-
- static constexpr bool supports_activation()
- {
- return true;
- }
-
- StdTransformsSVE<operand_type, result_type, 4, 4, 2> transforms = {};
-
- // Default to the generic kernel
- kern_type kernel=sve_hybrid_fp32_mmla_4VLx4;
-
- hybrid_fp32_mmla_4VLx4(const CPUInfo *)
- {
-
- }
-};
-
-} // namespace arm_gemm
-
-#endif // __ARM_FEATURE_SVE
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mmla_4VLx4/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mmla_4VLx4/generic.cpp
deleted file mode 100644
index 1364585..0000000
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mmla_4VLx4/generic.cpp
+++ /dev/null
@@ -1,3459 +0,0 @@
-/*
- * Copyright (c) 2018-2020 Arm Limited.
- *
- * SPDX-License-Identifier: MIT
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to
- * deal in the Software without restriction, including without limitation the
- * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
- * sell copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-#ifdef __ARM_FEATURE_SVE
-
-#include <algorithm>
-
-#include "arm_gemm.hpp"
-
-#include "../../asmlib.hpp"
-#include "../../utils.hpp"
-
-namespace arm_gemm {
-
-void sve_hybrid_fp32_mmla_4VLx4(const float *A, int lda, const float *B, float *C, int ldc, int M, int N, int K, const float *bias, Activation act, bool accumulate) {
- const int K_stride = ((K + 1) / 2) * 2;
- const long loops_count = ((K + 4) / 8) - 1;
- K -= loops_count * 8;
- const long regs_count = (K / 4) - 1;
- K -= (regs_count + 1) * 4;
- const long leftovers = K;
- const long blocks_count = (K + 1) / 2;
- float nullbias[128];
- if (!accumulate && !bias) {
- memset(nullbias, 0, (2 * get_vector_length<float>() * sizeof(float)));
- }
- float minval = - static_cast<float>(std::numeric_limits<float>::infinity());
- float maxval = static_cast<float>(std::numeric_limits<float>::infinity());
- const float * const minptr = &minval;
- const float * const maxptr = &maxval;
-
- switch(act.type)
- {
- default:
- case Activation::Type::None:
- break;
- case Activation::Type::BoundedReLU:
- maxval = static_cast<float>(act.param1);
- /* fall through */
- case Activation::Type::ReLU:
- minval = 0.0f;
- break;
- }
-
- int rows_to_compute;
-
- for (int y=0; y<M; y+=rows_to_compute) {
- const float * const a_ptr0_base = A + (y * lda);
- const unsigned long ldab = lda * sizeof(float);
-
- float *c_ptr0 = C + (y * ldc);
-
- rows_to_compute = M-y;
- if (rows_to_compute > 8) {
- if (rows_to_compute % 8) {
- rows_to_compute = 8 - 1;
- } else {
- rows_to_compute = 8;
- }
- }
-
- for (int x0=0; x0<N; x0+=(2 * get_vector_length<float>())) {
- const long width = std::min((unsigned long)N-x0, (2 * get_vector_length<float>()));
- long loops = loops_count;
- long regs = regs_count;
- long temp = 0;
- long blocks = blocks_count;
- const float *a_ptr0 = a_ptr0_base;
- const float *b_ptr0 = B + (K_stride * x0);
- const unsigned long ldcb = ldc * sizeof(float);
- const float *biasptr = bias ? bias+x0 : nullbias;
-
- switch(rows_to_compute) {
- case 1:
- __asm __volatile (
- "whilelt p6.s, %[temp], %[leftovers]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "ptrue p7.s\n"
- "whilelt p1.s, %[temp], %[width]\n"
- "cbnz %[accumulate], 1f\n"
- "mov z1.s, #0\n"
- "ld1w z15.s, p0/z, [%[biasptr]]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ld1w z12.s, p7/z, [%[b_ptr0]]\n"
- "zip1 z16.s, z15.s, z15.s\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "zip2 z17.s, z15.s, z15.s\n"
- "ld1w z15.s, p1/z, [%[biasptr], #1, MUL VL]\n"
- "trn1 z8.d, z0.d, z1.d\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "zip1 z18.s, z15.s, z15.s\n"
- "zip2 z19.s, z15.s, z15.s\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #4\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "mov z14.s, #0\n"
- "ld1w z13.s, p0/z, [%[c_ptr0]]\n"
- "mov z1.s, #0\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "ld1w z12.s, p7/z, [%[b_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "zip1 z16.s, z13.s, z14.s\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "zip2 z17.s, z13.s, z14.s\n"
- "ld1w z13.s, p1/z, [%[c_ptr0], #1, MUL VL]\n"
- "trn1 z8.d, z0.d, z1.d\n"
- "mov z14.s, #0\n"
- "zip1 z18.s, z13.s, z14.s\n"
- "zip2 z19.s, z13.s, z14.s\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #4\n"
- "cbz %[loops], 2f\n"
- "3:\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n"
- ".inst 0x64a8e590 // fmmla z16.s, z12.s, z8.s\n"
- "ld1w z12.s, p7/z, [%[b_ptr0]]\n"
- ".inst 0x64a8e5b1 // fmmla z17.s, z13.s, z8.s\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x64a8e5d2 // fmmla z18.s, z14.s, z8.s\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x64a8e5f3 // fmmla z19.s, z15.s, z8.s\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "mov z5.s, #0\n"
- "subs %[loops], %[loops], #0x1\n"
- ".inst 0x64a0e590 // fmmla z16.s, z12.s, z0.s\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x64a0e5b1 // fmmla z17.s, z13.s, z0.s\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "trn2 z8.d, z4.d, z5.d\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
- ".inst 0x64a0e5d2 // fmmla z18.s, z14.s, z0.s\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x64a0e5f3 // fmmla z19.s, z15.s, z0.s\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "trn1 z0.d, z4.d, z5.d\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "mov z1.s, #0\n"
- ".inst 0x64a0e590 // fmmla z16.s, z12.s, z0.s\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- ".inst 0x64a0e5b1 // fmmla z17.s, z13.s, z0.s\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- ".inst 0x64a0e5d2 // fmmla z18.s, z14.s, z0.s\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- ".inst 0x64a0e5f3 // fmmla z19.s, z15.s, z0.s\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- ".inst 0x64a8e590 // fmmla z16.s, z12.s, z8.s\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #-0x10]\n"
- ".inst 0x64a8e5b1 // fmmla z17.s, z13.s, z8.s\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- ".inst 0x64a8e5d2 // fmmla z18.s, z14.s, z8.s\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- ".inst 0x64a8e5f3 // fmmla z19.s, z15.s, z8.s\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "trn1 z8.d, z0.d, z1.d\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "b.ne 3b\n"
- "2:\n"
- "cbz %[regs], 4f\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n"
- ".inst 0x64a8e590 // fmmla z16.s, z12.s, z8.s\n"
- "ld1w z12.s, p7/z, [%[b_ptr0]]\n"
- ".inst 0x64a8e5b1 // fmmla z17.s, z13.s, z8.s\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x64a8e5d2 // fmmla z18.s, z14.s, z8.s\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x64a8e5f3 // fmmla z19.s, z15.s, z8.s\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "mov z5.s, #0\n"
- ".inst 0x64a0e590 // fmmla z16.s, z12.s, z0.s\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x64a0e5b1 // fmmla z17.s, z13.s, z0.s\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x64a0e5d2 // fmmla z18.s, z14.s, z0.s\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "trn2 z8.d, z4.d, z5.d\n"
- ".inst 0x64a0e5f3 // fmmla z19.s, z15.s, z0.s\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "trn1 z0.d, z4.d, z5.d\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "mov z1.s, #0\n"
- ".inst 0x64a0e590 // fmmla z16.s, z12.s, z0.s\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- ".inst 0x64a0e5b1 // fmmla z17.s, z13.s, z0.s\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- ".inst 0x64a0e5d2 // fmmla z18.s, z14.s, z0.s\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- ".inst 0x64a0e5f3 // fmmla z19.s, z15.s, z0.s\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- ".inst 0x64a8e590 // fmmla z16.s, z12.s, z8.s\n"
- "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x10]\n"
- ".inst 0x64a8e5b1 // fmmla z17.s, z13.s, z8.s\n"
- "addvl %[b_ptr0], %[b_ptr0], #-4\n"
- ".inst 0x64a8e5d2 // fmmla z18.s, z14.s, z8.s\n"
- "addvl %[a_ptr0], %[a_ptr0], #2\n"
- ".inst 0x64a8e5f3 // fmmla z19.s, z15.s, z8.s\n"
- "trn1 z8.d, z0.d, z1.d\n"
- "cbz %[blocks], 5f\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "ld1w z12.s, p7/z, [%[b_ptr0]]\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x64a8e590 // fmmla z16.s, z12.s, z8.s\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x64a8e5b1 // fmmla z17.s, z13.s, z8.s\n"
- ".inst 0x64a8e5d2 // fmmla z18.s, z14.s, z8.s\n"
- ".inst 0x64a8e5f3 // fmmla z19.s, z15.s, z8.s\n"
- "b.eq 5f\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x64a0e590 // fmmla z16.s, z12.s, z0.s\n"
- ".inst 0x64a0e5b1 // fmmla z17.s, z13.s, z0.s\n"
- ".inst 0x64a0e5d2 // fmmla z18.s, z14.s, z0.s\n"
- ".inst 0x64a0e5f3 // fmmla z19.s, z15.s, z0.s\n"
- "b 5f\n"
- "4:\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "ld1rqw z4.s, p6/z, [%[a_ptr0]]\n"
- ".inst 0x64a8e590 // fmmla z16.s, z12.s, z8.s\n"
- "ld1w z12.s, p7/z, [%[b_ptr0]]\n"
- ".inst 0x64a8e5b1 // fmmla z17.s, z13.s, z8.s\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x64a8e5d2 // fmmla z18.s, z14.s, z8.s\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x64a8e5f3 // fmmla z19.s, z15.s, z8.s\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "mov z5.s, #0\n"
- "addvl %[b_ptr0], %[b_ptr0], #4\n"
- ".inst 0x64a0e590 // fmmla z16.s, z12.s, z0.s\n"
- "addvl %[a_ptr0], %[a_ptr0], #1\n"
- ".inst 0x64a0e5b1 // fmmla z17.s, z13.s, z0.s\n"
- ".inst 0x64a0e5d2 // fmmla z18.s, z14.s, z0.s\n"
- ".inst 0x64a0e5f3 // fmmla z19.s, z15.s, z0.s\n"
- "trn1 z0.d, z4.d, z5.d\n"
- "cbz %[blocks], 5f\n"
- "trn2 z8.d, z4.d, z5.d\n"
- "ld1w z12.s, p7/z, [%[b_ptr0]]\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x64a0e590 // fmmla z16.s, z12.s, z0.s\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x64a0e5b1 // fmmla z17.s, z13.s, z0.s\n"
- ".inst 0x64a0e5d2 // fmmla z18.s, z14.s, z0.s\n"
- ".inst 0x64a0e5f3 // fmmla z19.s, z15.s, z0.s\n"
- "b.eq 5f\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x64a8e590 // fmmla z16.s, z12.s, z8.s\n"
- ".inst 0x64a8e5b1 // fmmla z17.s, z13.s, z8.s\n"
- ".inst 0x64a8e5d2 // fmmla z18.s, z14.s, z8.s\n"
- ".inst 0x64a8e5f3 // fmmla z19.s, z15.s, z8.s\n"
- "5:\n"
- "ld1rw z14.s, p7/z, [%[minptr]]\n"
- "ld1rw z15.s, p7/z, [%[maxptr]]\n"
- "fmax z16.s, p7/m, z16.s, z14.s\n"
- "fmax z17.s, p7/m, z17.s, z14.s\n"
- "fmax z18.s, p7/m, z18.s, z14.s\n"
- "fmax z19.s, p7/m, z19.s, z14.s\n"
- "fmin z16.s, p7/m, z16.s, z15.s\n"
- "fmin z17.s, p7/m, z17.s, z15.s\n"
- "fmin z18.s, p7/m, z18.s, z15.s\n"
- "fmin z19.s, p7/m, z19.s, z15.s\n"
- "uzp1 z0.s, z16.s, z17.s\n"
- "uzp1 z1.s, z18.s, z19.s\n"
- "st1w z0.s, p0, [%[c_ptr0]]\n"
- "st1w z1.s, p1, [%[c_ptr0], #1, MUL VL]\n"
- "addvl %[c_ptr0], %[c_ptr0], #2\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks)
- : [width] "r" (width), [accumulate] "r" (static_cast<uint64_t>(accumulate)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers)
- : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
- );
- break;
- case 2:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "c_ptr1 .req X1\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "whilelt p6.s, %[temp], %[leftovers]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "ptrue p7.s\n"
- "whilelt p1.s, %[temp], %[width]\n"
- "cbnz %[accumulate], 1f\n"
- "ld1w z15.s, p0/z, [%[biasptr]]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ld1rqw z1.s, p7/z, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "zip1 z16.s, z15.s, z15.s\n"
- "ld1w z12.s, p7/z, [%[b_ptr0]]\n"
- "zip2 z17.s, z15.s, z15.s\n"
- "ld1w z15.s, p1/z, [%[biasptr], #1, MUL VL]\n"
- "trn1 z8.d, z0.d, z1.d\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "zip1 z18.s, z15.s, z15.s\n"
- "zip2 z19.s, z15.s, z15.s\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #4\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "ld1w z13.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z14.s, p0/z, [c_ptr1]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ld1rqw z1.s, p7/z, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "zip1 z16.s, z13.s, z14.s\n"
- "ld1w z12.s, p7/z, [%[b_ptr0]]\n"
- "zip2 z17.s, z13.s, z14.s\n"
- "ld1w z13.s, p1/z, [%[c_ptr0], #1, MUL VL]\n"
- "trn1 z8.d, z0.d, z1.d\n"
- "ld1w z14.s, p1/z, [c_ptr1, #1, MUL VL]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "zip1 z18.s, z13.s, z14.s\n"
- "zip2 z19.s, z13.s, z14.s\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #4\n"
- "cbz %[loops], 2f\n"
- "3:\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n"
- ".inst 0x64a8e590 // fmmla z16.s, z12.s, z8.s\n"
- "ld1w z12.s, p7/z, [%[b_ptr0]]\n"
- ".inst 0x64a8e5b1 // fmmla z17.s, z13.s, z8.s\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x64a8e5d2 // fmmla z18.s, z14.s, z8.s\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x64a8e5f3 // fmmla z19.s, z15.s, z8.s\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x64a0e590 // fmmla z16.s, z12.s, z0.s\n"
- "ld1rqw z5.s, p7/z, [a_ptr1]\n"
- ".inst 0x64a0e5b1 // fmmla z17.s, z13.s, z0.s\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x64a0e5d2 // fmmla z18.s, z14.s, z0.s\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x64a0e5f3 // fmmla z19.s, z15.s, z0.s\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "trn1 z0.d, z4.d, z5.d\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "trn2 z8.d, z4.d, z5.d\n"
- "subs %[loops], %[loops], #0x1\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
- ".inst 0x64a0e590 // fmmla z16.s, z12.s, z0.s\n"
- "add a_ptr1, a_ptr1, #0x20\n"
- ".inst 0x64a0e5b1 // fmmla z17.s, z13.s, z0.s\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #-0x10]\n"
- ".inst 0x64a0e5d2 // fmmla z18.s, z14.s, z0.s\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- ".inst 0x64a0e5f3 // fmmla z19.s, z15.s, z0.s\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #-0x10]\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- ".inst 0x64a8e590 // fmmla z16.s, z12.s, z8.s\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- ".inst 0x64a8e5b1 // fmmla z17.s, z13.s, z8.s\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- ".inst 0x64a8e5d2 // fmmla z18.s, z14.s, z8.s\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- ".inst 0x64a8e5f3 // fmmla z19.s, z15.s, z8.s\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "trn1 z8.d, z0.d, z1.d\n"
- "b.ne 3b\n"
- "2:\n"
- "cbz %[regs], 4f\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n"
- ".inst 0x64a8e590 // fmmla z16.s, z12.s, z8.s\n"
- "ld1w z12.s, p7/z, [%[b_ptr0]]\n"
- ".inst 0x64a8e5b1 // fmmla z17.s, z13.s, z8.s\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x64a8e5d2 // fmmla z18.s, z14.s, z8.s\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x64a8e5f3 // fmmla z19.s, z15.s, z8.s\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x64a0e590 // fmmla z16.s, z12.s, z0.s\n"
- "ld1rqw z5.s, p7/z, [a_ptr1]\n"
- ".inst 0x64a0e5b1 // fmmla z17.s, z13.s, z0.s\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x64a0e5d2 // fmmla z18.s, z14.s, z0.s\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x64a0e5f3 // fmmla z19.s, z15.s, z0.s\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "trn1 z0.d, z4.d, z5.d\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "trn2 z8.d, z4.d, z5.d\n"
- "ld1rqw z1.s, p6/z, [a_ptr1, #0x10]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "addvl a_ptr1, a_ptr1, #2\n"
- ".inst 0x64a0e590 // fmmla z16.s, z12.s, z0.s\n"
- ".inst 0x64a0e5b1 // fmmla z17.s, z13.s, z0.s\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- ".inst 0x64a0e5d2 // fmmla z18.s, z14.s, z0.s\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- ".inst 0x64a0e5f3 // fmmla z19.s, z15.s, z0.s\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #-4\n"
- ".inst 0x64a8e590 // fmmla z16.s, z12.s, z8.s\n"
- "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x10]\n"
- ".inst 0x64a8e5b1 // fmmla z17.s, z13.s, z8.s\n"
- "addvl %[a_ptr0], %[a_ptr0], #2\n"
- ".inst 0x64a8e5d2 // fmmla z18.s, z14.s, z8.s\n"
- ".inst 0x64a8e5f3 // fmmla z19.s, z15.s, z8.s\n"
- "trn1 z8.d, z0.d, z1.d\n"
- "cbz %[blocks], 5f\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "ld1w z12.s, p7/z, [%[b_ptr0]]\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x64a8e590 // fmmla z16.s, z12.s, z8.s\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x64a8e5b1 // fmmla z17.s, z13.s, z8.s\n"
- ".inst 0x64a8e5d2 // fmmla z18.s, z14.s, z8.s\n"
- ".inst 0x64a8e5f3 // fmmla z19.s, z15.s, z8.s\n"
- "b.eq 5f\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x64a0e590 // fmmla z16.s, z12.s, z0.s\n"
- ".inst 0x64a0e5b1 // fmmla z17.s, z13.s, z0.s\n"
- ".inst 0x64a0e5d2 // fmmla z18.s, z14.s, z0.s\n"
- ".inst 0x64a0e5f3 // fmmla z19.s, z15.s, z0.s\n"
- "b 5f\n"
- "4:\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "ld1rqw z4.s, p6/z, [%[a_ptr0]]\n"
- ".inst 0x64a8e590 // fmmla z16.s, z12.s, z8.s\n"
- "ld1w z12.s, p7/z, [%[b_ptr0]]\n"
- ".inst 0x64a8e5b1 // fmmla z17.s, z13.s, z8.s\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x64a8e5d2 // fmmla z18.s, z14.s, z8.s\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x64a8e5f3 // fmmla z19.s, z15.s, z8.s\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x64a0e590 // fmmla z16.s, z12.s, z0.s\n"
- "ld1rqw z5.s, p6/z, [a_ptr1]\n"
- ".inst 0x64a0e5b1 // fmmla z17.s, z13.s, z0.s\n"
- "addvl %[b_ptr0], %[b_ptr0], #4\n"
- ".inst 0x64a0e5d2 // fmmla z18.s, z14.s, z0.s\n"
- "addvl %[a_ptr0], %[a_ptr0], #1\n"
- ".inst 0x64a0e5f3 // fmmla z19.s, z15.s, z0.s\n"
- "addvl a_ptr1, a_ptr1, #1\n"
- "trn1 z0.d, z4.d, z5.d\n"
- "cbz %[blocks], 5f\n"
- "trn2 z8.d, z4.d, z5.d\n"
- "ld1w z12.s, p7/z, [%[b_ptr0]]\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x64a0e590 // fmmla z16.s, z12.s, z0.s\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x64a0e5b1 // fmmla z17.s, z13.s, z0.s\n"
- ".inst 0x64a0e5d2 // fmmla z18.s, z14.s, z0.s\n"
- ".inst 0x64a0e5f3 // fmmla z19.s, z15.s, z0.s\n"
- "b.eq 5f\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x64a8e590 // fmmla z16.s, z12.s, z8.s\n"
- ".inst 0x64a8e5b1 // fmmla z17.s, z13.s, z8.s\n"
- ".inst 0x64a8e5d2 // fmmla z18.s, z14.s, z8.s\n"
- ".inst 0x64a8e5f3 // fmmla z19.s, z15.s, z8.s\n"
- "5:\n"
- "ld1rw z14.s, p7/z, [%[minptr]]\n"
- "ld1rw z15.s, p7/z, [%[maxptr]]\n"
- "fmax z16.s, p7/m, z16.s, z14.s\n"
- "fmax z17.s, p7/m, z17.s, z14.s\n"
- "fmax z18.s, p7/m, z18.s, z14.s\n"
- "fmax z19.s, p7/m, z19.s, z14.s\n"
- "fmin z16.s, p7/m, z16.s, z15.s\n"
- "fmin z17.s, p7/m, z17.s, z15.s\n"
- "fmin z18.s, p7/m, z18.s, z15.s\n"
- "fmin z19.s, p7/m, z19.s, z15.s\n"
- "uzp1 z0.s, z16.s, z17.s\n"
- "uzp2 z1.s, z16.s, z17.s\n"
- "uzp1 z2.s, z18.s, z19.s\n"
- "uzp2 z3.s, z18.s, z19.s\n"
- "st1w z0.s, p0, [%[c_ptr0]]\n"
- "st1w z1.s, p0, [c_ptr1]\n"
- "st1w z2.s, p1, [%[c_ptr0], #1, MUL VL]\n"
- "addvl %[c_ptr0], %[c_ptr0], #2\n"
- "st1w z3.s, p1, [c_ptr1, #1, MUL VL]\n"
- ".unreq a_ptr1\n"
- ".unreq c_ptr1\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks)
- : [width] "r" (width), [accumulate] "r" (static_cast<uint64_t>(accumulate)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers)
- : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "cc", "memory"
- );
- break;
- case 3:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "c_ptr1 .req X2\n"
- "c_ptr2 .req X3\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "whilelt p6.s, %[temp], %[leftovers]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "ptrue p7.s\n"
- "whilelt p1.s, %[temp], %[width]\n"
- "cbnz %[accumulate], 1f\n"
- "mov z3.s, #0\n"
- "ld1w z15.s, p0/z, [%[biasptr]]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ld1rqw z1.s, p7/z, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "zip1 z16.s, z15.s, z15.s\n"
- "ld1rqw z2.s, p7/z, [a_ptr2]\n"
- "zip2 z17.s, z15.s, z15.s\n"
- "ld1w z15.s, p1/z, [%[biasptr], #1, MUL VL]\n"
- "trn1 z8.d, z0.d, z1.d\n"
- "ld1w z12.s, p7/z, [%[b_ptr0]]\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "zip1 z18.s, z15.s, z15.s\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "zip2 z19.s, z15.s, z15.s\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "trn1 z9.d, z2.d, z3.d\n"
- "addvl %[b_ptr0], %[b_ptr0], #4\n"
- "mov z20.d, z16.d\n"
- "mov z21.d, z17.d\n"
- "mov z22.d, z18.d\n"
- "mov z23.d, z19.d\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "mov z3.s, #0\n"
- "ld1w z13.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z14.s, p0/z, [c_ptr1]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ld1rqw z1.s, p7/z, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "zip1 z16.s, z13.s, z14.s\n"
- "ld1rqw z2.s, p7/z, [a_ptr2]\n"
- "zip2 z17.s, z13.s, z14.s\n"
- "ld1w z13.s, p1/z, [%[c_ptr0], #1, MUL VL]\n"
- "trn1 z8.d, z0.d, z1.d\n"
- "ld1w z14.s, p1/z, [c_ptr1, #1, MUL VL]\n"
- "ld1w z12.s, p7/z, [%[b_ptr0]]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "trn1 z9.d, z2.d, z3.d\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "zip1 z18.s, z13.s, z14.s\n"
- "zip2 z19.s, z13.s, z14.s\n"
- "ld1w z13.s, p0/z, [c_ptr2]\n"
- "mov z14.s, #0\n"
- "zip1 z20.s, z13.s, z14.s\n"
- "zip2 z21.s, z13.s, z14.s\n"
- "ld1w z13.s, p1/z, [c_ptr2, #1, MUL VL]\n"
- "mov z14.s, #0\n"
- "zip1 z22.s, z13.s, z14.s\n"
- "zip2 z23.s, z13.s, z14.s\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #4\n"
- "cbz %[loops], 2f\n"
- "3:\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n"
- "trn2 z1.d, z2.d, z3.d\n"
- "ld1rqw z5.s, p7/z, [a_ptr1]\n"
- ".inst 0x64a8e590 // fmmla z16.s, z12.s, z8.s\n"
- "ld1rqw z6.s, p7/z, [a_ptr2]\n"
- ".inst 0x64a8e5b1 // fmmla z17.s, z13.s, z8.s\n"
- "subs %[loops], %[loops], #0x1\n"
- ".inst 0x64a8e5d2 // fmmla z18.s, z14.s, z8.s\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
- ".inst 0x64a8e5f3 // fmmla z19.s, z15.s, z8.s\n"
- "add a_ptr1, a_ptr1, #0x20\n"
- "trn2 z8.d, z4.d, z5.d\n"
- "add a_ptr2, a_ptr2, #0x20\n"
- ".inst 0x64a9e594 // fmmla z20.s, z12.s, z9.s\n"
- "ld1w z12.s, p7/z, [%[b_ptr0]]\n"
- ".inst 0x64a9e5b5 // fmmla z21.s, z13.s, z9.s\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x64a9e5d6 // fmmla z22.s, z14.s, z9.s\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x64a9e5f7 // fmmla z23.s, z15.s, z9.s\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "mov z7.s, #0\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #-0x10]\n"
- ".inst 0x64a0e590 // fmmla z16.s, z12.s, z0.s\n"
- ".inst 0x64a0e5b1 // fmmla z17.s, z13.s, z0.s\n"
- ".inst 0x64a0e5d2 // fmmla z18.s, z14.s, z0.s\n"
- "trn2 z9.d, z6.d, z7.d\n"
- ".inst 0x64a0e5f3 // fmmla z19.s, z15.s, z0.s\n"
- "trn1 z0.d, z4.d, z5.d\n"
- ".inst 0x64a1e594 // fmmla z20.s, z12.s, z1.s\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x64a1e5b5 // fmmla z21.s, z13.s, z1.s\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x64a1e5d6 // fmmla z22.s, z14.s, z1.s\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x64a1e5f7 // fmmla z23.s, z15.s, z1.s\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "trn1 z1.d, z6.d, z7.d\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- ".inst 0x64a0e590 // fmmla z16.s, z12.s, z0.s\n"
- ".inst 0x64a0e5b1 // fmmla z17.s, z13.s, z0.s\n"
- ".inst 0x64a0e5d2 // fmmla z18.s, z14.s, z0.s\n"
- ".inst 0x64a0e5f3 // fmmla z19.s, z15.s, z0.s\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #-0x10]\n"
- ".inst 0x64a1e594 // fmmla z20.s, z12.s, z1.s\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- ".inst 0x64a1e5b5 // fmmla z21.s, z13.s, z1.s\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- ".inst 0x64a1e5d6 // fmmla z22.s, z14.s, z1.s\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- ".inst 0x64a1e5f7 // fmmla z23.s, z15.s, z1.s\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "mov z3.s, #0\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #-0x10]\n"
- ".inst 0x64a8e590 // fmmla z16.s, z12.s, z8.s\n"
- ".inst 0x64a8e5b1 // fmmla z17.s, z13.s, z8.s\n"
- ".inst 0x64a8e5d2 // fmmla z18.s, z14.s, z8.s\n"
- ".inst 0x64a8e5f3 // fmmla z19.s, z15.s, z8.s\n"
- "trn1 z8.d, z0.d, z1.d\n"
- ".inst 0x64a9e594 // fmmla z20.s, z12.s, z9.s\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- ".inst 0x64a9e5b5 // fmmla z21.s, z13.s, z9.s\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- ".inst 0x64a9e5d6 // fmmla z22.s, z14.s, z9.s\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- ".inst 0x64a9e5f7 // fmmla z23.s, z15.s, z9.s\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "trn1 z9.d, z2.d, z3.d\n"
- "b.ne 3b\n"
- "2:\n"
- "cbz %[regs], 4f\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n"
- "trn2 z1.d, z2.d, z3.d\n"
- "ld1rqw z5.s, p7/z, [a_ptr1]\n"
- ".inst 0x64a8e590 // fmmla z16.s, z12.s, z8.s\n"
- "ld1rqw z6.s, p7/z, [a_ptr2]\n"
- ".inst 0x64a8e5b1 // fmmla z17.s, z13.s, z8.s\n"
- "ld1rqw z2.s, p6/z, [a_ptr2, #0x10]\n"
- ".inst 0x64a8e5d2 // fmmla z18.s, z14.s, z8.s\n"
- "addvl a_ptr2, a_ptr2, #2\n"
- ".inst 0x64a8e5f3 // fmmla z19.s, z15.s, z8.s\n"
- "trn2 z8.d, z4.d, z5.d\n"
- ".inst 0x64a9e594 // fmmla z20.s, z12.s, z9.s\n"
- "ld1w z12.s, p7/z, [%[b_ptr0]]\n"
- ".inst 0x64a9e5b5 // fmmla z21.s, z13.s, z9.s\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x64a9e5d6 // fmmla z22.s, z14.s, z9.s\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x64a9e5f7 // fmmla z23.s, z15.s, z9.s\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "mov z7.s, #0\n"
- ".inst 0x64a0e590 // fmmla z16.s, z12.s, z0.s\n"
- ".inst 0x64a0e5b1 // fmmla z17.s, z13.s, z0.s\n"
- ".inst 0x64a0e5d2 // fmmla z18.s, z14.s, z0.s\n"
- "trn2 z9.d, z6.d, z7.d\n"
- ".inst 0x64a0e5f3 // fmmla z19.s, z15.s, z0.s\n"
- "trn1 z0.d, z4.d, z5.d\n"
- ".inst 0x64a1e594 // fmmla z20.s, z12.s, z1.s\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x64a1e5b5 // fmmla z21.s, z13.s, z1.s\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x64a1e5d6 // fmmla z22.s, z14.s, z1.s\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x64a1e5f7 // fmmla z23.s, z15.s, z1.s\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "trn1 z1.d, z6.d, z7.d\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- ".inst 0x64a0e590 // fmmla z16.s, z12.s, z0.s\n"
- ".inst 0x64a0e5b1 // fmmla z17.s, z13.s, z0.s\n"
- ".inst 0x64a0e5d2 // fmmla z18.s, z14.s, z0.s\n"
- ".inst 0x64a0e5f3 // fmmla z19.s, z15.s, z0.s\n"
- "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x10]\n"
- ".inst 0x64a1e594 // fmmla z20.s, z12.s, z1.s\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- ".inst 0x64a1e5b5 // fmmla z21.s, z13.s, z1.s\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- ".inst 0x64a1e5d6 // fmmla z22.s, z14.s, z1.s\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- ".inst 0x64a1e5f7 // fmmla z23.s, z15.s, z1.s\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "mov z3.s, #0\n"
- "ld1rqw z1.s, p6/z, [a_ptr1, #0x10]\n"
- ".inst 0x64a8e590 // fmmla z16.s, z12.s, z8.s\n"
- "addvl %[b_ptr0], %[b_ptr0], #-4\n"
- ".inst 0x64a8e5b1 // fmmla z17.s, z13.s, z8.s\n"
- "addvl %[a_ptr0], %[a_ptr0], #2\n"
- ".inst 0x64a8e5d2 // fmmla z18.s, z14.s, z8.s\n"
- "addvl a_ptr1, a_ptr1, #2\n"
- ".inst 0x64a8e5f3 // fmmla z19.s, z15.s, z8.s\n"
- "trn1 z8.d, z0.d, z1.d\n"
- ".inst 0x64a9e594 // fmmla z20.s, z12.s, z9.s\n"
- ".inst 0x64a9e5b5 // fmmla z21.s, z13.s, z9.s\n"
- ".inst 0x64a9e5d6 // fmmla z22.s, z14.s, z9.s\n"
- ".inst 0x64a9e5f7 // fmmla z23.s, z15.s, z9.s\n"
- "trn1 z9.d, z2.d, z3.d\n"
- "cbz %[blocks], 5f\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "ld1w z12.s, p7/z, [%[b_ptr0]]\n"
- "trn2 z1.d, z2.d, z3.d\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- ".inst 0x64a8e590 // fmmla z16.s, z12.s, z8.s\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x64a8e5b1 // fmmla z17.s, z13.s, z8.s\n"
- ".inst 0x64a8e5d2 // fmmla z18.s, z14.s, z8.s\n"
- ".inst 0x64a9e594 // fmmla z20.s, z12.s, z9.s\n"
- ".inst 0x64a8e5f3 // fmmla z19.s, z15.s, z8.s\n"
- ".inst 0x64a9e5b5 // fmmla z21.s, z13.s, z9.s\n"
- ".inst 0x64a9e5d6 // fmmla z22.s, z14.s, z9.s\n"
- ".inst 0x64a9e5f7 // fmmla z23.s, z15.s, z9.s\n"
- "b.eq 5f\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x64a0e590 // fmmla z16.s, z12.s, z0.s\n"
- ".inst 0x64a0e5b1 // fmmla z17.s, z13.s, z0.s\n"
- ".inst 0x64a0e5d2 // fmmla z18.s, z14.s, z0.s\n"
- ".inst 0x64a0e5f3 // fmmla z19.s, z15.s, z0.s\n"
- ".inst 0x64a1e594 // fmmla z20.s, z12.s, z1.s\n"
- ".inst 0x64a1e5b5 // fmmla z21.s, z13.s, z1.s\n"
- ".inst 0x64a1e5d6 // fmmla z22.s, z14.s, z1.s\n"
- ".inst 0x64a1e5f7 // fmmla z23.s, z15.s, z1.s\n"
- "b 5f\n"
- "4:\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "ld1rqw z4.s, p6/z, [%[a_ptr0]]\n"
- "trn2 z1.d, z2.d, z3.d\n"
- "ld1rqw z5.s, p6/z, [a_ptr1]\n"
- ".inst 0x64a8e590 // fmmla z16.s, z12.s, z8.s\n"
- "ld1rqw z6.s, p6/z, [a_ptr2]\n"
- ".inst 0x64a8e5b1 // fmmla z17.s, z13.s, z8.s\n"
- "addvl %[a_ptr0], %[a_ptr0], #1\n"
- ".inst 0x64a8e5d2 // fmmla z18.s, z14.s, z8.s\n"
- "addvl a_ptr1, a_ptr1, #1\n"
- ".inst 0x64a8e5f3 // fmmla z19.s, z15.s, z8.s\n"
- "addvl a_ptr2, a_ptr2, #1\n"
- ".inst 0x64a9e594 // fmmla z20.s, z12.s, z9.s\n"
- "ld1w z12.s, p7/z, [%[b_ptr0]]\n"
- ".inst 0x64a9e5b5 // fmmla z21.s, z13.s, z9.s\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x64a9e5d6 // fmmla z22.s, z14.s, z9.s\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x64a9e5f7 // fmmla z23.s, z15.s, z9.s\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "mov z7.s, #0\n"
- "addvl %[b_ptr0], %[b_ptr0], #4\n"
- ".inst 0x64a0e590 // fmmla z16.s, z12.s, z0.s\n"
- ".inst 0x64a0e5b1 // fmmla z17.s, z13.s, z0.s\n"
- ".inst 0x64a0e5d2 // fmmla z18.s, z14.s, z0.s\n"
- ".inst 0x64a0e5f3 // fmmla z19.s, z15.s, z0.s\n"
- "trn1 z0.d, z4.d, z5.d\n"
- ".inst 0x64a1e594 // fmmla z20.s, z12.s, z1.s\n"
- ".inst 0x64a1e5b5 // fmmla z21.s, z13.s, z1.s\n"
- ".inst 0x64a1e5d6 // fmmla z22.s, z14.s, z1.s\n"
- ".inst 0x64a1e5f7 // fmmla z23.s, z15.s, z1.s\n"
- "trn1 z1.d, z6.d, z7.d\n"
- "cbz %[blocks], 5f\n"
- "trn2 z9.d, z6.d, z7.d\n"
- "ld1w z12.s, p7/z, [%[b_ptr0]]\n"
- "trn2 z8.d, z4.d, z5.d\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- ".inst 0x64a0e590 // fmmla z16.s, z12.s, z0.s\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x64a0e5b1 // fmmla z17.s, z13.s, z0.s\n"
- ".inst 0x64a0e5d2 // fmmla z18.s, z14.s, z0.s\n"
- ".inst 0x64a1e594 // fmmla z20.s, z12.s, z1.s\n"
- ".inst 0x64a0e5f3 // fmmla z19.s, z15.s, z0.s\n"
- ".inst 0x64a1e5b5 // fmmla z21.s, z13.s, z1.s\n"
- ".inst 0x64a1e5d6 // fmmla z22.s, z14.s, z1.s\n"
- ".inst 0x64a1e5f7 // fmmla z23.s, z15.s, z1.s\n"
- "b.eq 5f\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x64a8e590 // fmmla z16.s, z12.s, z8.s\n"
- ".inst 0x64a8e5b1 // fmmla z17.s, z13.s, z8.s\n"
- ".inst 0x64a8e5d2 // fmmla z18.s, z14.s, z8.s\n"
- ".inst 0x64a8e5f3 // fmmla z19.s, z15.s, z8.s\n"
- ".inst 0x64a9e594 // fmmla z20.s, z12.s, z9.s\n"
- ".inst 0x64a9e5b5 // fmmla z21.s, z13.s, z9.s\n"
- ".inst 0x64a9e5d6 // fmmla z22.s, z14.s, z9.s\n"
- ".inst 0x64a9e5f7 // fmmla z23.s, z15.s, z9.s\n"
- "5:\n"
- "ld1rw z14.s, p7/z, [%[minptr]]\n"
- "ld1rw z15.s, p7/z, [%[maxptr]]\n"
- "fmax z16.s, p7/m, z16.s, z14.s\n"
- "fmax z17.s, p7/m, z17.s, z14.s\n"
- "fmax z18.s, p7/m, z18.s, z14.s\n"
- "fmax z19.s, p7/m, z19.s, z14.s\n"
- "fmin z16.s, p7/m, z16.s, z15.s\n"
- "fmin z17.s, p7/m, z17.s, z15.s\n"
- "fmin z18.s, p7/m, z18.s, z15.s\n"
- "fmin z19.s, p7/m, z19.s, z15.s\n"
- "fmax z20.s, p7/m, z20.s, z14.s\n"
- "uzp1 z0.s, z16.s, z17.s\n"
- "uzp2 z1.s, z16.s, z17.s\n"
- "uzp1 z2.s, z18.s, z19.s\n"
- "uzp2 z3.s, z18.s, z19.s\n"
- "st1w z0.s, p0, [%[c_ptr0]]\n"
- "fmin z20.s, p7/m, z20.s, z15.s\n"
- "fmax z21.s, p7/m, z21.s, z14.s\n"
- "fmax z22.s, p7/m, z22.s, z14.s\n"
- "st1w z1.s, p0, [c_ptr1]\n"
- "fmax z23.s, p7/m, z23.s, z14.s\n"
- "fmin z21.s, p7/m, z21.s, z15.s\n"
- "st1w z2.s, p1, [%[c_ptr0], #1, MUL VL]\n"
- "fmin z22.s, p7/m, z22.s, z15.s\n"
- "addvl %[c_ptr0], %[c_ptr0], #2\n"
- "fmin z23.s, p7/m, z23.s, z15.s\n"
- "st1w z3.s, p1, [c_ptr1, #1, MUL VL]\n"
- "uzp1 z4.s, z20.s, z21.s\n"
- "uzp1 z5.s, z22.s, z23.s\n"
- "st1w z4.s, p0, [c_ptr2]\n"
- "st1w z5.s, p1, [c_ptr2, #1, MUL VL]\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks)
- : [width] "r" (width), [accumulate] "r" (static_cast<uint64_t>(accumulate)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers)
- : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "cc", "memory"
- );
- break;
- case 4:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "a_ptr3 .req X2\n"
- "c_ptr1 .req X3\n"
- "c_ptr2 .req X4\n"
- "c_ptr3 .req X5\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "add a_ptr3, a_ptr2, %[lda]\n"
- "add c_ptr3, c_ptr2, %[ldc]\n"
- "whilelt p6.s, %[temp], %[leftovers]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "ptrue p7.s\n"
- "whilelt p1.s, %[temp], %[width]\n"
- "cbnz %[accumulate], 1f\n"
- "ld1w z15.s, p0/z, [%[biasptr]]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ld1rqw z1.s, p7/z, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "zip1 z16.s, z15.s, z15.s\n"
- "ld1rqw z2.s, p7/z, [a_ptr2]\n"
- "zip2 z17.s, z15.s, z15.s\n"
- "ld1w z15.s, p1/z, [%[biasptr], #1, MUL VL]\n"
- "trn1 z8.d, z0.d, z1.d\n"
- "ld1rqw z3.s, p7/z, [a_ptr3]\n"
- "ld1w z12.s, p7/z, [%[b_ptr0]]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "zip1 z18.s, z15.s, z15.s\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "zip2 z19.s, z15.s, z15.s\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "trn1 z9.d, z2.d, z3.d\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "mov z20.d, z16.d\n"
- "add a_ptr3, a_ptr3, #0x10\n"
- "mov z21.d, z17.d\n"
- "addvl %[b_ptr0], %[b_ptr0], #4\n"
- "mov z22.d, z18.d\n"
- "mov z23.d, z19.d\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "ld1w z13.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z14.s, p0/z, [c_ptr1]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ld1rqw z1.s, p7/z, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "zip1 z16.s, z13.s, z14.s\n"
- "ld1rqw z2.s, p7/z, [a_ptr2]\n"
- "zip2 z17.s, z13.s, z14.s\n"
- "ld1w z13.s, p1/z, [%[c_ptr0], #1, MUL VL]\n"
- "trn1 z8.d, z0.d, z1.d\n"
- "ld1w z14.s, p1/z, [c_ptr1, #1, MUL VL]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "ld1w z12.s, p7/z, [%[b_ptr0]]\n"
- "add a_ptr3, a_ptr3, #0x10\n"
- "zip1 z18.s, z13.s, z14.s\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "zip2 z19.s, z13.s, z14.s\n"
- "ld1w z13.s, p0/z, [c_ptr2]\n"
- "trn1 z9.d, z2.d, z3.d\n"
- "ld1w z14.s, p0/z, [c_ptr3]\n"
- "zip1 z20.s, z13.s, z14.s\n"
- "zip2 z21.s, z13.s, z14.s\n"
- "ld1w z13.s, p1/z, [c_ptr2, #1, MUL VL]\n"
- "ld1w z14.s, p1/z, [c_ptr3, #1, MUL VL]\n"
- "zip1 z22.s, z13.s, z14.s\n"
- "zip2 z23.s, z13.s, z14.s\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #4\n"
- "cbz %[loops], 2f\n"
- "3:\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n"
- "trn2 z1.d, z2.d, z3.d\n"
- "ld1rqw z5.s, p7/z, [a_ptr1]\n"
- ".inst 0x64a8e590 // fmmla z16.s, z12.s, z8.s\n"
- "ld1rqw z6.s, p7/z, [a_ptr2]\n"
- ".inst 0x64a8e5b1 // fmmla z17.s, z13.s, z8.s\n"
- "ld1rqw z7.s, p7/z, [a_ptr3]\n"
- ".inst 0x64a8e5d2 // fmmla z18.s, z14.s, z8.s\n"
- "subs %[loops], %[loops], #0x1\n"
- ".inst 0x64a8e5f3 // fmmla z19.s, z15.s, z8.s\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
- "trn2 z8.d, z4.d, z5.d\n"
- "add a_ptr1, a_ptr1, #0x20\n"
- ".inst 0x64a9e594 // fmmla z20.s, z12.s, z9.s\n"
- "ld1w z12.s, p7/z, [%[b_ptr0]]\n"
- ".inst 0x64a9e5b5 // fmmla z21.s, z13.s, z9.s\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x64a9e5d6 // fmmla z22.s, z14.s, z9.s\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x64a9e5f7 // fmmla z23.s, z15.s, z9.s\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "trn2 z9.d, z6.d, z7.d\n"
- "add a_ptr2, a_ptr2, #0x20\n"
- ".inst 0x64a0e590 // fmmla z16.s, z12.s, z0.s\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #-0x10]\n"
- ".inst 0x64a0e5b1 // fmmla z17.s, z13.s, z0.s\n"
- "add a_ptr3, a_ptr3, #0x20\n"
- ".inst 0x64a0e5d2 // fmmla z18.s, z14.s, z0.s\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #-0x10]\n"
- ".inst 0x64a0e5f3 // fmmla z19.s, z15.s, z0.s\n"
- "trn1 z0.d, z4.d, z5.d\n"
- ".inst 0x64a1e594 // fmmla z20.s, z12.s, z1.s\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x64a1e5b5 // fmmla z21.s, z13.s, z1.s\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x64a1e5d6 // fmmla z22.s, z14.s, z1.s\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x64a1e5f7 // fmmla z23.s, z15.s, z1.s\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "trn1 z1.d, z6.d, z7.d\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- ".inst 0x64a0e590 // fmmla z16.s, z12.s, z0.s\n"
- ".inst 0x64a0e5b1 // fmmla z17.s, z13.s, z0.s\n"
- ".inst 0x64a0e5d2 // fmmla z18.s, z14.s, z0.s\n"
- ".inst 0x64a0e5f3 // fmmla z19.s, z15.s, z0.s\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #-0x10]\n"
- ".inst 0x64a1e594 // fmmla z20.s, z12.s, z1.s\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- ".inst 0x64a1e5b5 // fmmla z21.s, z13.s, z1.s\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- ".inst 0x64a1e5d6 // fmmla z22.s, z14.s, z1.s\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- ".inst 0x64a1e5f7 // fmmla z23.s, z15.s, z1.s\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- ".inst 0x64a8e590 // fmmla z16.s, z12.s, z8.s\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #-0x10]\n"
- ".inst 0x64a8e5b1 // fmmla z17.s, z13.s, z8.s\n"
- ".inst 0x64a8e5d2 // fmmla z18.s, z14.s, z8.s\n"
- ".inst 0x64a8e5f3 // fmmla z19.s, z15.s, z8.s\n"
- "trn1 z8.d, z0.d, z1.d\n"
- ".inst 0x64a9e594 // fmmla z20.s, z12.s, z9.s\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- ".inst 0x64a9e5b5 // fmmla z21.s, z13.s, z9.s\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- ".inst 0x64a9e5d6 // fmmla z22.s, z14.s, z9.s\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- ".inst 0x64a9e5f7 // fmmla z23.s, z15.s, z9.s\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "trn1 z9.d, z2.d, z3.d\n"
- "b.ne 3b\n"
- "2:\n"
- "cbz %[regs], 4f\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n"
- "trn2 z1.d, z2.d, z3.d\n"
- "ld1rqw z5.s, p7/z, [a_ptr1]\n"
- ".inst 0x64a8e590 // fmmla z16.s, z12.s, z8.s\n"
- "ld1rqw z6.s, p7/z, [a_ptr2]\n"
- ".inst 0x64a8e5b1 // fmmla z17.s, z13.s, z8.s\n"
- "ld1rqw z7.s, p7/z, [a_ptr3]\n"
- ".inst 0x64a8e5d2 // fmmla z18.s, z14.s, z8.s\n"
- "ld1rqw z2.s, p6/z, [a_ptr2, #0x10]\n"
- ".inst 0x64a8e5f3 // fmmla z19.s, z15.s, z8.s\n"
- "ld1rqw z3.s, p6/z, [a_ptr3, #0x10]\n"
- "trn2 z8.d, z4.d, z5.d\n"
- "addvl a_ptr2, a_ptr2, #2\n"
- ".inst 0x64a9e594 // fmmla z20.s, z12.s, z9.s\n"
- "ld1w z12.s, p7/z, [%[b_ptr0]]\n"
- ".inst 0x64a9e5b5 // fmmla z21.s, z13.s, z9.s\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x64a9e5d6 // fmmla z22.s, z14.s, z9.s\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x64a9e5f7 // fmmla z23.s, z15.s, z9.s\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "trn2 z9.d, z6.d, z7.d\n"
- "addvl a_ptr3, a_ptr3, #2\n"
- ".inst 0x64a0e590 // fmmla z16.s, z12.s, z0.s\n"
- ".inst 0x64a0e5b1 // fmmla z17.s, z13.s, z0.s\n"
- ".inst 0x64a0e5d2 // fmmla z18.s, z14.s, z0.s\n"
- ".inst 0x64a0e5f3 // fmmla z19.s, z15.s, z0.s\n"
- "trn1 z0.d, z4.d, z5.d\n"
- ".inst 0x64a1e594 // fmmla z20.s, z12.s, z1.s\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x64a1e5b5 // fmmla z21.s, z13.s, z1.s\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x64a1e5d6 // fmmla z22.s, z14.s, z1.s\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x64a1e5f7 // fmmla z23.s, z15.s, z1.s\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "trn1 z1.d, z6.d, z7.d\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- ".inst 0x64a0e590 // fmmla z16.s, z12.s, z0.s\n"
- ".inst 0x64a0e5b1 // fmmla z17.s, z13.s, z0.s\n"
- ".inst 0x64a0e5d2 // fmmla z18.s, z14.s, z0.s\n"
- ".inst 0x64a0e5f3 // fmmla z19.s, z15.s, z0.s\n"
- "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x10]\n"
- ".inst 0x64a1e594 // fmmla z20.s, z12.s, z1.s\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- ".inst 0x64a1e5b5 // fmmla z21.s, z13.s, z1.s\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- ".inst 0x64a1e5d6 // fmmla z22.s, z14.s, z1.s\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- ".inst 0x64a1e5f7 // fmmla z23.s, z15.s, z1.s\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- ".inst 0x64a8e590 // fmmla z16.s, z12.s, z8.s\n"
- "ld1rqw z1.s, p6/z, [a_ptr1, #0x10]\n"
- ".inst 0x64a8e5b1 // fmmla z17.s, z13.s, z8.s\n"
- "addvl %[b_ptr0], %[b_ptr0], #-4\n"
- ".inst 0x64a8e5d2 // fmmla z18.s, z14.s, z8.s\n"
- "addvl %[a_ptr0], %[a_ptr0], #2\n"
- ".inst 0x64a8e5f3 // fmmla z19.s, z15.s, z8.s\n"
- "addvl a_ptr1, a_ptr1, #2\n"
- "trn1 z8.d, z0.d, z1.d\n"
- ".inst 0x64a9e594 // fmmla z20.s, z12.s, z9.s\n"
- ".inst 0x64a9e5b5 // fmmla z21.s, z13.s, z9.s\n"
- ".inst 0x64a9e5d6 // fmmla z22.s, z14.s, z9.s\n"
- ".inst 0x64a9e5f7 // fmmla z23.s, z15.s, z9.s\n"
- "trn1 z9.d, z2.d, z3.d\n"
- "cbz %[blocks], 5f\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "ld1w z12.s, p7/z, [%[b_ptr0]]\n"
- "trn2 z1.d, z2.d, z3.d\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- ".inst 0x64a8e590 // fmmla z16.s, z12.s, z8.s\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x64a8e5b1 // fmmla z17.s, z13.s, z8.s\n"
- ".inst 0x64a8e5d2 // fmmla z18.s, z14.s, z8.s\n"
- ".inst 0x64a9e594 // fmmla z20.s, z12.s, z9.s\n"
- ".inst 0x64a8e5f3 // fmmla z19.s, z15.s, z8.s\n"
- ".inst 0x64a9e5b5 // fmmla z21.s, z13.s, z9.s\n"
- ".inst 0x64a9e5d6 // fmmla z22.s, z14.s, z9.s\n"
- ".inst 0x64a9e5f7 // fmmla z23.s, z15.s, z9.s\n"
- "b.eq 5f\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x64a0e590 // fmmla z16.s, z12.s, z0.s\n"
- ".inst 0x64a0e5b1 // fmmla z17.s, z13.s, z0.s\n"
- ".inst 0x64a0e5d2 // fmmla z18.s, z14.s, z0.s\n"
- ".inst 0x64a0e5f3 // fmmla z19.s, z15.s, z0.s\n"
- ".inst 0x64a1e594 // fmmla z20.s, z12.s, z1.s\n"
- ".inst 0x64a1e5b5 // fmmla z21.s, z13.s, z1.s\n"
- ".inst 0x64a1e5d6 // fmmla z22.s, z14.s, z1.s\n"
- ".inst 0x64a1e5f7 // fmmla z23.s, z15.s, z1.s\n"
- "b 5f\n"
- "4:\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "ld1rqw z4.s, p6/z, [%[a_ptr0]]\n"
- "trn2 z1.d, z2.d, z3.d\n"
- "ld1rqw z5.s, p6/z, [a_ptr1]\n"
- ".inst 0x64a8e590 // fmmla z16.s, z12.s, z8.s\n"
- "ld1rqw z6.s, p6/z, [a_ptr2]\n"
- ".inst 0x64a8e5b1 // fmmla z17.s, z13.s, z8.s\n"
- "ld1rqw z7.s, p6/z, [a_ptr3]\n"
- ".inst 0x64a8e5d2 // fmmla z18.s, z14.s, z8.s\n"
- "addvl %[a_ptr0], %[a_ptr0], #1\n"
- ".inst 0x64a8e5f3 // fmmla z19.s, z15.s, z8.s\n"
- "addvl a_ptr1, a_ptr1, #1\n"
- ".inst 0x64a9e594 // fmmla z20.s, z12.s, z9.s\n"
- "ld1w z12.s, p7/z, [%[b_ptr0]]\n"
- ".inst 0x64a9e5b5 // fmmla z21.s, z13.s, z9.s\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x64a9e5d6 // fmmla z22.s, z14.s, z9.s\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x64a9e5f7 // fmmla z23.s, z15.s, z9.s\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x64a0e590 // fmmla z16.s, z12.s, z0.s\n"
- "addvl %[b_ptr0], %[b_ptr0], #4\n"
- ".inst 0x64a0e5b1 // fmmla z17.s, z13.s, z0.s\n"
- "addvl a_ptr2, a_ptr2, #1\n"
- ".inst 0x64a0e5d2 // fmmla z18.s, z14.s, z0.s\n"
- "addvl a_ptr3, a_ptr3, #1\n"
- ".inst 0x64a0e5f3 // fmmla z19.s, z15.s, z0.s\n"
- "trn1 z0.d, z4.d, z5.d\n"
- ".inst 0x64a1e594 // fmmla z20.s, z12.s, z1.s\n"
- ".inst 0x64a1e5b5 // fmmla z21.s, z13.s, z1.s\n"
- ".inst 0x64a1e5d6 // fmmla z22.s, z14.s, z1.s\n"
- ".inst 0x64a1e5f7 // fmmla z23.s, z15.s, z1.s\n"
- "trn1 z1.d, z6.d, z7.d\n"
- "cbz %[blocks], 5f\n"
- "trn2 z9.d, z6.d, z7.d\n"
- "ld1w z12.s, p7/z, [%[b_ptr0]]\n"
- "trn2 z8.d, z4.d, z5.d\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- ".inst 0x64a0e590 // fmmla z16.s, z12.s, z0.s\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x64a0e5b1 // fmmla z17.s, z13.s, z0.s\n"
- ".inst 0x64a0e5d2 // fmmla z18.s, z14.s, z0.s\n"
- ".inst 0x64a1e594 // fmmla z20.s, z12.s, z1.s\n"
- ".inst 0x64a0e5f3 // fmmla z19.s, z15.s, z0.s\n"
- ".inst 0x64a1e5b5 // fmmla z21.s, z13.s, z1.s\n"
- ".inst 0x64a1e5d6 // fmmla z22.s, z14.s, z1.s\n"
- ".inst 0x64a1e5f7 // fmmla z23.s, z15.s, z1.s\n"
- "b.eq 5f\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x64a8e590 // fmmla z16.s, z12.s, z8.s\n"
- ".inst 0x64a8e5b1 // fmmla z17.s, z13.s, z8.s\n"
- ".inst 0x64a8e5d2 // fmmla z18.s, z14.s, z8.s\n"
- ".inst 0x64a8e5f3 // fmmla z19.s, z15.s, z8.s\n"
- ".inst 0x64a9e594 // fmmla z20.s, z12.s, z9.s\n"
- ".inst 0x64a9e5b5 // fmmla z21.s, z13.s, z9.s\n"
- ".inst 0x64a9e5d6 // fmmla z22.s, z14.s, z9.s\n"
- ".inst 0x64a9e5f7 // fmmla z23.s, z15.s, z9.s\n"
- "5:\n"
- "ld1rw z14.s, p7/z, [%[minptr]]\n"
- "ld1rw z15.s, p7/z, [%[maxptr]]\n"
- "fmax z16.s, p7/m, z16.s, z14.s\n"
- "fmax z17.s, p7/m, z17.s, z14.s\n"
- "fmax z18.s, p7/m, z18.s, z14.s\n"
- "fmax z19.s, p7/m, z19.s, z14.s\n"
- "fmin z16.s, p7/m, z16.s, z15.s\n"
- "fmin z17.s, p7/m, z17.s, z15.s\n"
- "fmin z18.s, p7/m, z18.s, z15.s\n"
- "fmin z19.s, p7/m, z19.s, z15.s\n"
- "fmax z20.s, p7/m, z20.s, z14.s\n"
- "uzp1 z0.s, z16.s, z17.s\n"
- "uzp2 z1.s, z16.s, z17.s\n"
- "uzp1 z2.s, z18.s, z19.s\n"
- "uzp2 z3.s, z18.s, z19.s\n"
- "st1w z0.s, p0, [%[c_ptr0]]\n"
- "fmin z20.s, p7/m, z20.s, z15.s\n"
- "fmax z21.s, p7/m, z21.s, z14.s\n"
- "fmax z22.s, p7/m, z22.s, z14.s\n"
- "st1w z1.s, p0, [c_ptr1]\n"
- "fmax z23.s, p7/m, z23.s, z14.s\n"
- "fmin z21.s, p7/m, z21.s, z15.s\n"
- "st1w z2.s, p1, [%[c_ptr0], #1, MUL VL]\n"
- "fmin z22.s, p7/m, z22.s, z15.s\n"
- "addvl %[c_ptr0], %[c_ptr0], #2\n"
- "fmin z23.s, p7/m, z23.s, z15.s\n"
- "st1w z3.s, p1, [c_ptr1, #1, MUL VL]\n"
- "uzp1 z4.s, z20.s, z21.s\n"
- "uzp2 z5.s, z20.s, z21.s\n"
- "uzp1 z6.s, z22.s, z23.s\n"
- "st1w z4.s, p0, [c_ptr2]\n"
- "uzp2 z7.s, z22.s, z23.s\n"
- "st1w z5.s, p0, [c_ptr3]\n"
- "st1w z6.s, p1, [c_ptr2, #1, MUL VL]\n"
- "st1w z7.s, p1, [c_ptr3, #1, MUL VL]\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq a_ptr3\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- ".unreq c_ptr3\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks)
- : [width] "r" (width), [accumulate] "r" (static_cast<uint64_t>(accumulate)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers)
- : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "x4", "x5", "cc", "memory"
- );
- break;
- case 5:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "a_ptr3 .req X2\n"
- "a_ptr4 .req X3\n"
- "c_ptr1 .req X4\n"
- "c_ptr2 .req X5\n"
- "c_ptr3 .req X6\n"
- "c_ptr4 .req X7\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "add a_ptr3, a_ptr2, %[lda]\n"
- "add c_ptr3, c_ptr2, %[ldc]\n"
- "add a_ptr4, a_ptr3, %[lda]\n"
- "add c_ptr4, c_ptr3, %[ldc]\n"
- "whilelt p6.s, %[temp], %[leftovers]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "ptrue p7.s\n"
- "whilelt p1.s, %[temp], %[width]\n"
- "cbnz %[accumulate], 1f\n"
- "mov z5.s, #0\n"
- "ld1w z15.s, p0/z, [%[biasptr]]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ld1rqw z1.s, p7/z, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "zip1 z16.s, z15.s, z15.s\n"
- "ld1rqw z2.s, p7/z, [a_ptr2]\n"
- "zip2 z17.s, z15.s, z15.s\n"
- "ld1w z15.s, p1/z, [%[biasptr], #1, MUL VL]\n"
- "trn1 z8.d, z0.d, z1.d\n"
- "ld1rqw z3.s, p7/z, [a_ptr3]\n"
- "ld1rqw z4.s, p7/z, [a_ptr4]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "zip1 z18.s, z15.s, z15.s\n"
- "ld1w z12.s, p7/z, [%[b_ptr0]]\n"
- "zip2 z19.s, z15.s, z15.s\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "trn1 z9.d, z2.d, z3.d\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "trn1 z10.d, z4.d, z5.d\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "mov z20.d, z16.d\n"
- "add a_ptr3, a_ptr3, #0x10\n"
- "mov z21.d, z17.d\n"
- "add a_ptr4, a_ptr4, #0x10\n"
- "mov z22.d, z18.d\n"
- "addvl %[b_ptr0], %[b_ptr0], #4\n"
- "mov z23.d, z19.d\n"
- "mov z24.d, z16.d\n"
- "mov z25.d, z17.d\n"
- "mov z26.d, z18.d\n"
- "mov z27.d, z19.d\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "mov z5.s, #0\n"
- "ld1w z13.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z14.s, p0/z, [c_ptr1]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ld1rqw z1.s, p7/z, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "zip1 z16.s, z13.s, z14.s\n"
- "ld1rqw z2.s, p7/z, [a_ptr2]\n"
- "zip2 z17.s, z13.s, z14.s\n"
- "ld1w z13.s, p1/z, [%[c_ptr0], #1, MUL VL]\n"
- "trn1 z8.d, z0.d, z1.d\n"
- "ld1w z14.s, p1/z, [c_ptr1, #1, MUL VL]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "ld1rqw z4.s, p7/z, [a_ptr4]\n"
- "add a_ptr3, a_ptr3, #0x10\n"
- "zip1 z18.s, z13.s, z14.s\n"
- "ld1w z12.s, p7/z, [%[b_ptr0]]\n"
- "zip2 z19.s, z13.s, z14.s\n"
- "ld1w z13.s, p0/z, [c_ptr2]\n"
- "trn1 z9.d, z2.d, z3.d\n"
- "ld1w z14.s, p0/z, [c_ptr3]\n"
- "trn1 z10.d, z4.d, z5.d\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "add a_ptr4, a_ptr4, #0x10\n"
- "zip1 z20.s, z13.s, z14.s\n"
- "zip2 z21.s, z13.s, z14.s\n"
- "ld1w z13.s, p1/z, [c_ptr2, #1, MUL VL]\n"
- "ld1w z14.s, p1/z, [c_ptr3, #1, MUL VL]\n"
- "zip1 z22.s, z13.s, z14.s\n"
- "zip2 z23.s, z13.s, z14.s\n"
- "ld1w z13.s, p0/z, [c_ptr4]\n"
- "mov z14.s, #0\n"
- "zip1 z24.s, z13.s, z14.s\n"
- "zip2 z25.s, z13.s, z14.s\n"
- "ld1w z13.s, p1/z, [c_ptr4, #1, MUL VL]\n"
- "mov z14.s, #0\n"
- "zip1 z26.s, z13.s, z14.s\n"
- "zip2 z27.s, z13.s, z14.s\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #4\n"
- "cbz %[loops], 2f\n"
- "3:\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "ld1rqw z6.s, p7/z, [a_ptr2]\n"
- "trn2 z1.d, z2.d, z3.d\n"
- "ld1rqw z7.s, p7/z, [a_ptr3]\n"
- "trn2 z2.d, z4.d, z5.d\n"
- "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n"
- ".inst 0x64a8e590 // fmmla z16.s, z12.s, z8.s\n"
- "ld1rqw z5.s, p7/z, [a_ptr1]\n"
- ".inst 0x64a8e5b1 // fmmla z17.s, z13.s, z8.s\n"
- "subs %[loops], %[loops], #0x1\n"
- ".inst 0x64a8e5d2 // fmmla z18.s, z14.s, z8.s\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
- ".inst 0x64a8e5f3 // fmmla z19.s, z15.s, z8.s\n"
- "ld1rqw z8.s, p7/z, [a_ptr4]\n"
- ".inst 0x64a9e594 // fmmla z20.s, z12.s, z9.s\n"
- "add a_ptr1, a_ptr1, #0x20\n"
- ".inst 0x64a9e5b5 // fmmla z21.s, z13.s, z9.s\n"
- "add a_ptr2, a_ptr2, #0x20\n"
- ".inst 0x64a9e5d6 // fmmla z22.s, z14.s, z9.s\n"
- "add a_ptr3, a_ptr3, #0x20\n"
- ".inst 0x64a9e5f7 // fmmla z23.s, z15.s, z9.s\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #-0x10]\n"
- ".inst 0x64aae598 // fmmla z24.s, z12.s, z10.s\n"
- "ld1w z12.s, p7/z, [%[b_ptr0]]\n"
- ".inst 0x64aae5b9 // fmmla z25.s, z13.s, z10.s\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x64aae5da // fmmla z26.s, z14.s, z10.s\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x64aae5fb // fmmla z27.s, z15.s, z10.s\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "mov z9.s, #0\n"
- "add a_ptr4, a_ptr4, #0x20\n"
- ".inst 0x64a0e590 // fmmla z16.s, z12.s, z0.s\n"
- ".inst 0x64a0e5b1 // fmmla z17.s, z13.s, z0.s\n"
- "trn2 z10.d, z8.d, z9.d\n"
- ".inst 0x64a0e5d2 // fmmla z18.s, z14.s, z0.s\n"
- ".inst 0x64a0e5f3 // fmmla z19.s, z15.s, z0.s\n"
- "trn1 z0.d, z4.d, z5.d\n"
- ".inst 0x64a1e594 // fmmla z20.s, z12.s, z1.s\n"
- ".inst 0x64a1e5b5 // fmmla z21.s, z13.s, z1.s\n"
- ".inst 0x64a1e5d6 // fmmla z22.s, z14.s, z1.s\n"
- ".inst 0x64a1e5f7 // fmmla z23.s, z15.s, z1.s\n"
- "trn1 z1.d, z6.d, z7.d\n"
- ".inst 0x64a2e598 // fmmla z24.s, z12.s, z2.s\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x64a2e5b9 // fmmla z25.s, z13.s, z2.s\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x64a2e5da // fmmla z26.s, z14.s, z2.s\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x64a2e5fb // fmmla z27.s, z15.s, z2.s\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "trn1 z2.d, z8.d, z9.d\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "trn2 z9.d, z6.d, z7.d\n"
- "trn2 z8.d, z4.d, z5.d\n"
- "ld1rqw z4.s, p7/z, [a_ptr4, #-0x10]\n"
- ".inst 0x64a0e590 // fmmla z16.s, z12.s, z0.s\n"
- ".inst 0x64a0e5b1 // fmmla z17.s, z13.s, z0.s\n"
- ".inst 0x64a0e5d2 // fmmla z18.s, z14.s, z0.s\n"
- ".inst 0x64a0e5f3 // fmmla z19.s, z15.s, z0.s\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #-0x10]\n"
- ".inst 0x64a1e594 // fmmla z20.s, z12.s, z1.s\n"
- ".inst 0x64a1e5b5 // fmmla z21.s, z13.s, z1.s\n"
- ".inst 0x64a1e5d6 // fmmla z22.s, z14.s, z1.s\n"
- ".inst 0x64a1e5f7 // fmmla z23.s, z15.s, z1.s\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #-0x10]\n"
- ".inst 0x64a2e598 // fmmla z24.s, z12.s, z2.s\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- ".inst 0x64a2e5b9 // fmmla z25.s, z13.s, z2.s\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- ".inst 0x64a2e5da // fmmla z26.s, z14.s, z2.s\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- ".inst 0x64a2e5fb // fmmla z27.s, z15.s, z2.s\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "mov z5.s, #0\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #-0x10]\n"
- ".inst 0x64a8e590 // fmmla z16.s, z12.s, z8.s\n"
- ".inst 0x64a8e5b1 // fmmla z17.s, z13.s, z8.s\n"
- ".inst 0x64a8e5d2 // fmmla z18.s, z14.s, z8.s\n"
- ".inst 0x64a8e5f3 // fmmla z19.s, z15.s, z8.s\n"
- "trn1 z8.d, z0.d, z1.d\n"
- ".inst 0x64a9e594 // fmmla z20.s, z12.s, z9.s\n"
- ".inst 0x64a9e5b5 // fmmla z21.s, z13.s, z9.s\n"
- ".inst 0x64a9e5d6 // fmmla z22.s, z14.s, z9.s\n"
- ".inst 0x64a9e5f7 // fmmla z23.s, z15.s, z9.s\n"
- "trn1 z9.d, z2.d, z3.d\n"
- ".inst 0x64aae598 // fmmla z24.s, z12.s, z10.s\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- ".inst 0x64aae5b9 // fmmla z25.s, z13.s, z10.s\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- ".inst 0x64aae5da // fmmla z26.s, z14.s, z10.s\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- ".inst 0x64aae5fb // fmmla z27.s, z15.s, z10.s\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "trn1 z10.d, z4.d, z5.d\n"
- "b.ne 3b\n"
- "2:\n"
- "cbz %[regs], 4f\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "ld1rqw z6.s, p7/z, [a_ptr2]\n"
- "trn2 z1.d, z2.d, z3.d\n"
- "ld1rqw z7.s, p7/z, [a_ptr3]\n"
- "trn2 z2.d, z4.d, z5.d\n"
- "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n"
- ".inst 0x64a8e590 // fmmla z16.s, z12.s, z8.s\n"
- "ld1rqw z5.s, p7/z, [a_ptr1]\n"
- ".inst 0x64a8e5b1 // fmmla z17.s, z13.s, z8.s\n"
- "ld1rqw z3.s, p6/z, [a_ptr3, #0x10]\n"
- ".inst 0x64a8e5d2 // fmmla z18.s, z14.s, z8.s\n"
- "addvl a_ptr3, a_ptr3, #2\n"
- ".inst 0x64a8e5f3 // fmmla z19.s, z15.s, z8.s\n"
- "ld1rqw z8.s, p7/z, [a_ptr4]\n"
- ".inst 0x64a9e594 // fmmla z20.s, z12.s, z9.s\n"
- ".inst 0x64a9e5b5 // fmmla z21.s, z13.s, z9.s\n"
- ".inst 0x64a9e5d6 // fmmla z22.s, z14.s, z9.s\n"
- ".inst 0x64a9e5f7 // fmmla z23.s, z15.s, z9.s\n"
- ".inst 0x64aae598 // fmmla z24.s, z12.s, z10.s\n"
- "ld1w z12.s, p7/z, [%[b_ptr0]]\n"
- ".inst 0x64aae5b9 // fmmla z25.s, z13.s, z10.s\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x64aae5da // fmmla z26.s, z14.s, z10.s\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x64aae5fb // fmmla z27.s, z15.s, z10.s\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "mov z9.s, #0\n"
- ".inst 0x64a0e590 // fmmla z16.s, z12.s, z0.s\n"
- ".inst 0x64a0e5b1 // fmmla z17.s, z13.s, z0.s\n"
- ".inst 0x64a0e5d2 // fmmla z18.s, z14.s, z0.s\n"
- "trn2 z10.d, z8.d, z9.d\n"
- ".inst 0x64a0e5f3 // fmmla z19.s, z15.s, z0.s\n"
- "trn1 z0.d, z4.d, z5.d\n"
- ".inst 0x64a1e594 // fmmla z20.s, z12.s, z1.s\n"
- ".inst 0x64a1e5b5 // fmmla z21.s, z13.s, z1.s\n"
- ".inst 0x64a1e5d6 // fmmla z22.s, z14.s, z1.s\n"
- ".inst 0x64a1e5f7 // fmmla z23.s, z15.s, z1.s\n"
- "trn1 z1.d, z6.d, z7.d\n"
- ".inst 0x64a2e598 // fmmla z24.s, z12.s, z2.s\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x64a2e5b9 // fmmla z25.s, z13.s, z2.s\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x64a2e5da // fmmla z26.s, z14.s, z2.s\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x64a2e5fb // fmmla z27.s, z15.s, z2.s\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "trn1 z2.d, z8.d, z9.d\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "trn2 z9.d, z6.d, z7.d\n"
- "trn2 z8.d, z4.d, z5.d\n"
- "ld1rqw z4.s, p6/z, [a_ptr4, #0x10]\n"
- ".inst 0x64a0e590 // fmmla z16.s, z12.s, z0.s\n"
- "addvl a_ptr4, a_ptr4, #2\n"
- ".inst 0x64a0e5b1 // fmmla z17.s, z13.s, z0.s\n"
- ".inst 0x64a0e5d2 // fmmla z18.s, z14.s, z0.s\n"
- ".inst 0x64a0e5f3 // fmmla z19.s, z15.s, z0.s\n"
- "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x10]\n"
- ".inst 0x64a1e594 // fmmla z20.s, z12.s, z1.s\n"
- "addvl %[a_ptr0], %[a_ptr0], #2\n"
- ".inst 0x64a1e5b5 // fmmla z21.s, z13.s, z1.s\n"
- ".inst 0x64a1e5d6 // fmmla z22.s, z14.s, z1.s\n"
- ".inst 0x64a1e5f7 // fmmla z23.s, z15.s, z1.s\n"
- "ld1rqw z1.s, p6/z, [a_ptr1, #0x10]\n"
- ".inst 0x64a2e598 // fmmla z24.s, z12.s, z2.s\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- ".inst 0x64a2e5b9 // fmmla z25.s, z13.s, z2.s\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- ".inst 0x64a2e5da // fmmla z26.s, z14.s, z2.s\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- ".inst 0x64a2e5fb // fmmla z27.s, z15.s, z2.s\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "mov z5.s, #0\n"
- "ld1rqw z2.s, p6/z, [a_ptr2, #0x10]\n"
- ".inst 0x64a8e590 // fmmla z16.s, z12.s, z8.s\n"
- "addvl %[b_ptr0], %[b_ptr0], #-4\n"
- ".inst 0x64a8e5b1 // fmmla z17.s, z13.s, z8.s\n"
- "addvl a_ptr1, a_ptr1, #2\n"
- ".inst 0x64a8e5d2 // fmmla z18.s, z14.s, z8.s\n"
- "addvl a_ptr2, a_ptr2, #2\n"
- ".inst 0x64a8e5f3 // fmmla z19.s, z15.s, z8.s\n"
- "trn1 z8.d, z0.d, z1.d\n"
- ".inst 0x64a9e594 // fmmla z20.s, z12.s, z9.s\n"
- ".inst 0x64a9e5b5 // fmmla z21.s, z13.s, z9.s\n"
- ".inst 0x64a9e5d6 // fmmla z22.s, z14.s, z9.s\n"
- ".inst 0x64a9e5f7 // fmmla z23.s, z15.s, z9.s\n"
- "trn1 z9.d, z2.d, z3.d\n"
- ".inst 0x64aae598 // fmmla z24.s, z12.s, z10.s\n"
- ".inst 0x64aae5b9 // fmmla z25.s, z13.s, z10.s\n"
- ".inst 0x64aae5da // fmmla z26.s, z14.s, z10.s\n"
- ".inst 0x64aae5fb // fmmla z27.s, z15.s, z10.s\n"
- "trn1 z10.d, z4.d, z5.d\n"
- "cbz %[blocks], 5f\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "ld1w z12.s, p7/z, [%[b_ptr0]]\n"
- "trn2 z1.d, z2.d, z3.d\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "trn2 z2.d, z4.d, z5.d\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- ".inst 0x64a8e590 // fmmla z16.s, z12.s, z8.s\n"
- ".inst 0x64a8e5b1 // fmmla z17.s, z13.s, z8.s\n"
- ".inst 0x64a8e5d2 // fmmla z18.s, z14.s, z8.s\n"
- ".inst 0x64a8e5f3 // fmmla z19.s, z15.s, z8.s\n"
- ".inst 0x64a9e594 // fmmla z20.s, z12.s, z9.s\n"
- ".inst 0x64a9e5b5 // fmmla z21.s, z13.s, z9.s\n"
- ".inst 0x64a9e5d6 // fmmla z22.s, z14.s, z9.s\n"
- ".inst 0x64a9e5f7 // fmmla z23.s, z15.s, z9.s\n"
- ".inst 0x64aae598 // fmmla z24.s, z12.s, z10.s\n"
- ".inst 0x64aae5b9 // fmmla z25.s, z13.s, z10.s\n"
- ".inst 0x64aae5da // fmmla z26.s, z14.s, z10.s\n"
- ".inst 0x64aae5fb // fmmla z27.s, z15.s, z10.s\n"
- "b.eq 5f\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x64a0e590 // fmmla z16.s, z12.s, z0.s\n"
- ".inst 0x64a0e5b1 // fmmla z17.s, z13.s, z0.s\n"
- ".inst 0x64a0e5d2 // fmmla z18.s, z14.s, z0.s\n"
- ".inst 0x64a0e5f3 // fmmla z19.s, z15.s, z0.s\n"
- ".inst 0x64a1e594 // fmmla z20.s, z12.s, z1.s\n"
- ".inst 0x64a1e5b5 // fmmla z21.s, z13.s, z1.s\n"
- ".inst 0x64a1e5d6 // fmmla z22.s, z14.s, z1.s\n"
- ".inst 0x64a1e5f7 // fmmla z23.s, z15.s, z1.s\n"
- ".inst 0x64a2e598 // fmmla z24.s, z12.s, z2.s\n"
- ".inst 0x64a2e5b9 // fmmla z25.s, z13.s, z2.s\n"
- ".inst 0x64a2e5da // fmmla z26.s, z14.s, z2.s\n"
- ".inst 0x64a2e5fb // fmmla z27.s, z15.s, z2.s\n"
- "b 5f\n"
- "4:\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "ld1rqw z6.s, p6/z, [a_ptr2]\n"
- "trn2 z1.d, z2.d, z3.d\n"
- "ld1rqw z7.s, p6/z, [a_ptr3]\n"
- "trn2 z2.d, z4.d, z5.d\n"
- "ld1rqw z4.s, p6/z, [%[a_ptr0]]\n"
- ".inst 0x64a8e590 // fmmla z16.s, z12.s, z8.s\n"
- "ld1rqw z5.s, p6/z, [a_ptr1]\n"
- ".inst 0x64a8e5b1 // fmmla z17.s, z13.s, z8.s\n"
- "addvl %[a_ptr0], %[a_ptr0], #1\n"
- ".inst 0x64a8e5d2 // fmmla z18.s, z14.s, z8.s\n"
- "addvl a_ptr1, a_ptr1, #1\n"
- ".inst 0x64a8e5f3 // fmmla z19.s, z15.s, z8.s\n"
- "ld1rqw z8.s, p6/z, [a_ptr4]\n"
- ".inst 0x64a9e594 // fmmla z20.s, z12.s, z9.s\n"
- "addvl a_ptr2, a_ptr2, #1\n"
- ".inst 0x64a9e5b5 // fmmla z21.s, z13.s, z9.s\n"
- "addvl a_ptr3, a_ptr3, #1\n"
- ".inst 0x64a9e5d6 // fmmla z22.s, z14.s, z9.s\n"
- "addvl a_ptr4, a_ptr4, #1\n"
- ".inst 0x64a9e5f7 // fmmla z23.s, z15.s, z9.s\n"
- ".inst 0x64aae598 // fmmla z24.s, z12.s, z10.s\n"
- "ld1w z12.s, p7/z, [%[b_ptr0]]\n"
- ".inst 0x64aae5b9 // fmmla z25.s, z13.s, z10.s\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x64aae5da // fmmla z26.s, z14.s, z10.s\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x64aae5fb // fmmla z27.s, z15.s, z10.s\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "mov z9.s, #0\n"
- "addvl %[b_ptr0], %[b_ptr0], #4\n"
- ".inst 0x64a0e590 // fmmla z16.s, z12.s, z0.s\n"
- ".inst 0x64a0e5b1 // fmmla z17.s, z13.s, z0.s\n"
- ".inst 0x64a0e5d2 // fmmla z18.s, z14.s, z0.s\n"
- ".inst 0x64a0e5f3 // fmmla z19.s, z15.s, z0.s\n"
- "trn1 z0.d, z4.d, z5.d\n"
- ".inst 0x64a1e594 // fmmla z20.s, z12.s, z1.s\n"
- ".inst 0x64a1e5b5 // fmmla z21.s, z13.s, z1.s\n"
- ".inst 0x64a1e5d6 // fmmla z22.s, z14.s, z1.s\n"
- ".inst 0x64a1e5f7 // fmmla z23.s, z15.s, z1.s\n"
- "trn1 z1.d, z6.d, z7.d\n"
- ".inst 0x64a2e598 // fmmla z24.s, z12.s, z2.s\n"
- ".inst 0x64a2e5b9 // fmmla z25.s, z13.s, z2.s\n"
- ".inst 0x64a2e5da // fmmla z26.s, z14.s, z2.s\n"
- ".inst 0x64a2e5fb // fmmla z27.s, z15.s, z2.s\n"
- "trn1 z2.d, z8.d, z9.d\n"
- "cbz %[blocks], 5f\n"
- "trn2 z10.d, z8.d, z9.d\n"
- "ld1w z12.s, p7/z, [%[b_ptr0]]\n"
- "trn2 z9.d, z6.d, z7.d\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "trn2 z8.d, z4.d, z5.d\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- ".inst 0x64a0e590 // fmmla z16.s, z12.s, z0.s\n"
- ".inst 0x64a0e5b1 // fmmla z17.s, z13.s, z0.s\n"
- ".inst 0x64a0e5d2 // fmmla z18.s, z14.s, z0.s\n"
- ".inst 0x64a0e5f3 // fmmla z19.s, z15.s, z0.s\n"
- ".inst 0x64a1e594 // fmmla z20.s, z12.s, z1.s\n"
- ".inst 0x64a1e5b5 // fmmla z21.s, z13.s, z1.s\n"
- ".inst 0x64a1e5d6 // fmmla z22.s, z14.s, z1.s\n"
- ".inst 0x64a1e5f7 // fmmla z23.s, z15.s, z1.s\n"
- ".inst 0x64a2e598 // fmmla z24.s, z12.s, z2.s\n"
- ".inst 0x64a2e5b9 // fmmla z25.s, z13.s, z2.s\n"
- ".inst 0x64a2e5da // fmmla z26.s, z14.s, z2.s\n"
- ".inst 0x64a2e5fb // fmmla z27.s, z15.s, z2.s\n"
- "b.eq 5f\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x64a8e590 // fmmla z16.s, z12.s, z8.s\n"
- ".inst 0x64a8e5b1 // fmmla z17.s, z13.s, z8.s\n"
- ".inst 0x64a8e5d2 // fmmla z18.s, z14.s, z8.s\n"
- ".inst 0x64a8e5f3 // fmmla z19.s, z15.s, z8.s\n"
- ".inst 0x64a9e594 // fmmla z20.s, z12.s, z9.s\n"
- ".inst 0x64a9e5b5 // fmmla z21.s, z13.s, z9.s\n"
- ".inst 0x64a9e5d6 // fmmla z22.s, z14.s, z9.s\n"
- ".inst 0x64a9e5f7 // fmmla z23.s, z15.s, z9.s\n"
- ".inst 0x64aae598 // fmmla z24.s, z12.s, z10.s\n"
- ".inst 0x64aae5b9 // fmmla z25.s, z13.s, z10.s\n"
- ".inst 0x64aae5da // fmmla z26.s, z14.s, z10.s\n"
- ".inst 0x64aae5fb // fmmla z27.s, z15.s, z10.s\n"
- "5:\n"
- "ld1rw z14.s, p7/z, [%[minptr]]\n"
- "ld1rw z15.s, p7/z, [%[maxptr]]\n"
- "fmax z16.s, p7/m, z16.s, z14.s\n"
- "fmax z17.s, p7/m, z17.s, z14.s\n"
- "fmax z18.s, p7/m, z18.s, z14.s\n"
- "fmax z19.s, p7/m, z19.s, z14.s\n"
- "fmin z16.s, p7/m, z16.s, z15.s\n"
- "fmin z17.s, p7/m, z17.s, z15.s\n"
- "fmin z18.s, p7/m, z18.s, z15.s\n"
- "fmin z19.s, p7/m, z19.s, z15.s\n"
- "fmax z20.s, p7/m, z20.s, z14.s\n"
- "uzp1 z0.s, z16.s, z17.s\n"
- "uzp2 z1.s, z16.s, z17.s\n"
- "uzp1 z2.s, z18.s, z19.s\n"
- "uzp2 z3.s, z18.s, z19.s\n"
- "st1w z0.s, p0, [%[c_ptr0]]\n"
- "fmin z20.s, p7/m, z20.s, z15.s\n"
- "fmax z21.s, p7/m, z21.s, z14.s\n"
- "fmax z22.s, p7/m, z22.s, z14.s\n"
- "st1w z1.s, p0, [c_ptr1]\n"
- "fmax z23.s, p7/m, z23.s, z14.s\n"
- "fmax z24.s, p7/m, z24.s, z14.s\n"
- "fmin z21.s, p7/m, z21.s, z15.s\n"
- "st1w z2.s, p1, [%[c_ptr0], #1, MUL VL]\n"
- "fmin z22.s, p7/m, z22.s, z15.s\n"
- "addvl %[c_ptr0], %[c_ptr0], #2\n"
- "fmin z23.s, p7/m, z23.s, z15.s\n"
- "st1w z3.s, p1, [c_ptr1, #1, MUL VL]\n"
- "uzp1 z4.s, z20.s, z21.s\n"
- "uzp2 z5.s, z20.s, z21.s\n"
- "fmin z24.s, p7/m, z24.s, z15.s\n"
- "uzp1 z6.s, z22.s, z23.s\n"
- "st1w z4.s, p0, [c_ptr2]\n"
- "uzp2 z7.s, z22.s, z23.s\n"
- "fmax z25.s, p7/m, z25.s, z14.s\n"
- "fmax z26.s, p7/m, z26.s, z14.s\n"
- "st1w z5.s, p0, [c_ptr3]\n"
- "fmax z27.s, p7/m, z27.s, z14.s\n"
- "fmin z25.s, p7/m, z25.s, z15.s\n"
- "st1w z6.s, p1, [c_ptr2, #1, MUL VL]\n"
- "fmin z26.s, p7/m, z26.s, z15.s\n"
- "fmin z27.s, p7/m, z27.s, z15.s\n"
- "uzp1 z8.s, z24.s, z25.s\n"
- "st1w z7.s, p1, [c_ptr3, #1, MUL VL]\n"
- "uzp1 z9.s, z26.s, z27.s\n"
- "st1w z8.s, p0, [c_ptr4]\n"
- "st1w z9.s, p1, [c_ptr4, #1, MUL VL]\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq a_ptr3\n"
- ".unreq a_ptr4\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- ".unreq c_ptr3\n"
- ".unreq c_ptr4\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks)
- : [width] "r" (width), [accumulate] "r" (static_cast<uint64_t>(accumulate)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers)
- : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "cc", "memory"
- );
- break;
- case 6:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "a_ptr3 .req X2\n"
- "a_ptr4 .req X3\n"
- "a_ptr5 .req X4\n"
- "c_ptr1 .req X5\n"
- "c_ptr2 .req X6\n"
- "c_ptr3 .req X7\n"
- "c_ptr4 .req X8\n"
- "c_ptr5 .req X9\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "add a_ptr3, a_ptr2, %[lda]\n"
- "add c_ptr3, c_ptr2, %[ldc]\n"
- "add a_ptr4, a_ptr3, %[lda]\n"
- "add c_ptr4, c_ptr3, %[ldc]\n"
- "add a_ptr5, a_ptr4, %[lda]\n"
- "add c_ptr5, c_ptr4, %[ldc]\n"
- "whilelt p6.s, %[temp], %[leftovers]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "ptrue p7.s\n"
- "whilelt p1.s, %[temp], %[width]\n"
- "cbnz %[accumulate], 1f\n"
- "ld1w z15.s, p0/z, [%[biasptr]]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ld1rqw z1.s, p7/z, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "zip1 z16.s, z15.s, z15.s\n"
- "ld1rqw z2.s, p7/z, [a_ptr2]\n"
- "zip2 z17.s, z15.s, z15.s\n"
- "ld1w z15.s, p1/z, [%[biasptr], #1, MUL VL]\n"
- "trn1 z8.d, z0.d, z1.d\n"
- "ld1rqw z3.s, p7/z, [a_ptr3]\n"
- "ld1rqw z4.s, p7/z, [a_ptr4]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "zip1 z18.s, z15.s, z15.s\n"
- "ld1rqw z5.s, p7/z, [a_ptr5]\n"
- "zip2 z19.s, z15.s, z15.s\n"
- "ld1w z12.s, p7/z, [%[b_ptr0]]\n"
- "trn1 z9.d, z2.d, z3.d\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "mov z20.d, z16.d\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "trn1 z10.d, z4.d, z5.d\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "mov z21.d, z17.d\n"
- "add a_ptr3, a_ptr3, #0x10\n"
- "mov z22.d, z18.d\n"
- "add a_ptr4, a_ptr4, #0x10\n"
- "mov z23.d, z19.d\n"
- "add a_ptr5, a_ptr5, #0x10\n"
- "mov z24.d, z16.d\n"
- "addvl %[b_ptr0], %[b_ptr0], #4\n"
- "mov z25.d, z17.d\n"
- "mov z26.d, z18.d\n"
- "mov z27.d, z19.d\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "ld1w z13.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z14.s, p0/z, [c_ptr1]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ld1rqw z1.s, p7/z, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "zip1 z16.s, z13.s, z14.s\n"
- "ld1rqw z2.s, p7/z, [a_ptr2]\n"
- "zip2 z17.s, z13.s, z14.s\n"
- "ld1w z13.s, p1/z, [%[c_ptr0], #1, MUL VL]\n"
- "trn1 z8.d, z0.d, z1.d\n"
- "ld1w z14.s, p1/z, [c_ptr1, #1, MUL VL]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "ld1rqw z4.s, p7/z, [a_ptr4]\n"
- "add a_ptr3, a_ptr3, #0x10\n"
- "zip1 z18.s, z13.s, z14.s\n"
- "ld1rqw z5.s, p7/z, [a_ptr5]\n"
- "zip2 z19.s, z13.s, z14.s\n"
- "ld1w z13.s, p0/z, [c_ptr2]\n"
- "trn1 z9.d, z2.d, z3.d\n"
- "ld1w z14.s, p0/z, [c_ptr3]\n"
- "ld1w z12.s, p7/z, [%[b_ptr0]]\n"
- "add a_ptr4, a_ptr4, #0x10\n"
- "trn1 z10.d, z4.d, z5.d\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "zip1 z20.s, z13.s, z14.s\n"
- "add a_ptr5, a_ptr5, #0x10\n"
- "zip2 z21.s, z13.s, z14.s\n"
- "ld1w z13.s, p1/z, [c_ptr2, #1, MUL VL]\n"
- "ld1w z14.s, p1/z, [c_ptr3, #1, MUL VL]\n"
- "zip1 z22.s, z13.s, z14.s\n"
- "zip2 z23.s, z13.s, z14.s\n"
- "ld1w z13.s, p0/z, [c_ptr4]\n"
- "ld1w z14.s, p0/z, [c_ptr5]\n"
- "zip1 z24.s, z13.s, z14.s\n"
- "zip2 z25.s, z13.s, z14.s\n"
- "ld1w z13.s, p1/z, [c_ptr4, #1, MUL VL]\n"
- "ld1w z14.s, p1/z, [c_ptr5, #1, MUL VL]\n"
- "zip1 z26.s, z13.s, z14.s\n"
- "zip2 z27.s, z13.s, z14.s\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #4\n"
- "cbz %[loops], 2f\n"
- "3:\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "ld1rqw z6.s, p7/z, [a_ptr2]\n"
- "trn2 z1.d, z2.d, z3.d\n"
- "ld1rqw z7.s, p7/z, [a_ptr3]\n"
- "trn2 z2.d, z4.d, z5.d\n"
- "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n"
- ".inst 0x64a8e590 // fmmla z16.s, z12.s, z8.s\n"
- "ld1rqw z5.s, p7/z, [a_ptr1]\n"
- ".inst 0x64a8e5b1 // fmmla z17.s, z13.s, z8.s\n"
- "subs %[loops], %[loops], #0x1\n"
- ".inst 0x64a8e5d2 // fmmla z18.s, z14.s, z8.s\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
- ".inst 0x64a8e5f3 // fmmla z19.s, z15.s, z8.s\n"
- "ld1rqw z8.s, p7/z, [a_ptr4]\n"
- ".inst 0x64a9e594 // fmmla z20.s, z12.s, z9.s\n"
- "add a_ptr1, a_ptr1, #0x20\n"
- ".inst 0x64a9e5b5 // fmmla z21.s, z13.s, z9.s\n"
- "add a_ptr2, a_ptr2, #0x20\n"
- ".inst 0x64a9e5d6 // fmmla z22.s, z14.s, z9.s\n"
- "add a_ptr3, a_ptr3, #0x20\n"
- ".inst 0x64a9e5f7 // fmmla z23.s, z15.s, z9.s\n"
- "ld1rqw z9.s, p7/z, [a_ptr5]\n"
- ".inst 0x64aae598 // fmmla z24.s, z12.s, z10.s\n"
- "ld1w z12.s, p7/z, [%[b_ptr0]]\n"
- ".inst 0x64aae5b9 // fmmla z25.s, z13.s, z10.s\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x64aae5da // fmmla z26.s, z14.s, z10.s\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x64aae5fb // fmmla z27.s, z15.s, z10.s\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "trn2 z10.d, z8.d, z9.d\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #-0x10]\n"
- ".inst 0x64a0e590 // fmmla z16.s, z12.s, z0.s\n"
- "add a_ptr4, a_ptr4, #0x20\n"
- ".inst 0x64a0e5b1 // fmmla z17.s, z13.s, z0.s\n"
- "add a_ptr5, a_ptr5, #0x20\n"
- ".inst 0x64a0e5d2 // fmmla z18.s, z14.s, z0.s\n"
- ".inst 0x64a0e5f3 // fmmla z19.s, z15.s, z0.s\n"
- "trn1 z0.d, z4.d, z5.d\n"
- ".inst 0x64a1e594 // fmmla z20.s, z12.s, z1.s\n"
- ".inst 0x64a1e5b5 // fmmla z21.s, z13.s, z1.s\n"
- ".inst 0x64a1e5d6 // fmmla z22.s, z14.s, z1.s\n"
- ".inst 0x64a1e5f7 // fmmla z23.s, z15.s, z1.s\n"
- "trn1 z1.d, z6.d, z7.d\n"
- ".inst 0x64a2e598 // fmmla z24.s, z12.s, z2.s\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x64a2e5b9 // fmmla z25.s, z13.s, z2.s\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x64a2e5da // fmmla z26.s, z14.s, z2.s\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x64a2e5fb // fmmla z27.s, z15.s, z2.s\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "trn1 z2.d, z8.d, z9.d\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "trn2 z9.d, z6.d, z7.d\n"
- "trn2 z8.d, z4.d, z5.d\n"
- "ld1rqw z4.s, p7/z, [a_ptr4, #-0x10]\n"
- ".inst 0x64a0e590 // fmmla z16.s, z12.s, z0.s\n"
- "ld1rqw z5.s, p7/z, [a_ptr5, #-0x10]\n"
- ".inst 0x64a0e5b1 // fmmla z17.s, z13.s, z0.s\n"
- ".inst 0x64a0e5d2 // fmmla z18.s, z14.s, z0.s\n"
- ".inst 0x64a0e5f3 // fmmla z19.s, z15.s, z0.s\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #-0x10]\n"
- ".inst 0x64a1e594 // fmmla z20.s, z12.s, z1.s\n"
- ".inst 0x64a1e5b5 // fmmla z21.s, z13.s, z1.s\n"
- ".inst 0x64a1e5d6 // fmmla z22.s, z14.s, z1.s\n"
- ".inst 0x64a1e5f7 // fmmla z23.s, z15.s, z1.s\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #-0x10]\n"
- ".inst 0x64a2e598 // fmmla z24.s, z12.s, z2.s\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- ".inst 0x64a2e5b9 // fmmla z25.s, z13.s, z2.s\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- ".inst 0x64a2e5da // fmmla z26.s, z14.s, z2.s\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- ".inst 0x64a2e5fb // fmmla z27.s, z15.s, z2.s\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- ".inst 0x64a8e590 // fmmla z16.s, z12.s, z8.s\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #-0x10]\n"
- ".inst 0x64a8e5b1 // fmmla z17.s, z13.s, z8.s\n"
- ".inst 0x64a8e5d2 // fmmla z18.s, z14.s, z8.s\n"
- ".inst 0x64a8e5f3 // fmmla z19.s, z15.s, z8.s\n"
- "trn1 z8.d, z0.d, z1.d\n"
- ".inst 0x64a9e594 // fmmla z20.s, z12.s, z9.s\n"
- ".inst 0x64a9e5b5 // fmmla z21.s, z13.s, z9.s\n"
- ".inst 0x64a9e5d6 // fmmla z22.s, z14.s, z9.s\n"
- ".inst 0x64a9e5f7 // fmmla z23.s, z15.s, z9.s\n"
- "trn1 z9.d, z2.d, z3.d\n"
- ".inst 0x64aae598 // fmmla z24.s, z12.s, z10.s\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- ".inst 0x64aae5b9 // fmmla z25.s, z13.s, z10.s\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- ".inst 0x64aae5da // fmmla z26.s, z14.s, z10.s\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- ".inst 0x64aae5fb // fmmla z27.s, z15.s, z10.s\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "trn1 z10.d, z4.d, z5.d\n"
- "b.ne 3b\n"
- "2:\n"
- "cbz %[regs], 4f\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "ld1rqw z6.s, p7/z, [a_ptr2]\n"
- "trn2 z1.d, z2.d, z3.d\n"
- "ld1rqw z7.s, p7/z, [a_ptr3]\n"
- "trn2 z2.d, z4.d, z5.d\n"
- "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n"
- ".inst 0x64a8e590 // fmmla z16.s, z12.s, z8.s\n"
- "ld1rqw z5.s, p7/z, [a_ptr1]\n"
- ".inst 0x64a8e5b1 // fmmla z17.s, z13.s, z8.s\n"
- "ld1rqw z3.s, p6/z, [a_ptr3, #0x10]\n"
- ".inst 0x64a8e5d2 // fmmla z18.s, z14.s, z8.s\n"
- "addvl a_ptr3, a_ptr3, #2\n"
- ".inst 0x64a8e5f3 // fmmla z19.s, z15.s, z8.s\n"
- "ld1rqw z8.s, p7/z, [a_ptr4]\n"
- ".inst 0x64a9e594 // fmmla z20.s, z12.s, z9.s\n"
- ".inst 0x64a9e5b5 // fmmla z21.s, z13.s, z9.s\n"
- ".inst 0x64a9e5d6 // fmmla z22.s, z14.s, z9.s\n"
- ".inst 0x64a9e5f7 // fmmla z23.s, z15.s, z9.s\n"
- "ld1rqw z9.s, p7/z, [a_ptr5]\n"
- ".inst 0x64aae598 // fmmla z24.s, z12.s, z10.s\n"
- "ld1w z12.s, p7/z, [%[b_ptr0]]\n"
- ".inst 0x64aae5b9 // fmmla z25.s, z13.s, z10.s\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x64aae5da // fmmla z26.s, z14.s, z10.s\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x64aae5fb // fmmla z27.s, z15.s, z10.s\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "trn2 z10.d, z8.d, z9.d\n"
- ".inst 0x64a0e590 // fmmla z16.s, z12.s, z0.s\n"
- ".inst 0x64a0e5b1 // fmmla z17.s, z13.s, z0.s\n"
- ".inst 0x64a0e5d2 // fmmla z18.s, z14.s, z0.s\n"
- ".inst 0x64a0e5f3 // fmmla z19.s, z15.s, z0.s\n"
- "trn1 z0.d, z4.d, z5.d\n"
- ".inst 0x64a1e594 // fmmla z20.s, z12.s, z1.s\n"
- ".inst 0x64a1e5b5 // fmmla z21.s, z13.s, z1.s\n"
- ".inst 0x64a1e5d6 // fmmla z22.s, z14.s, z1.s\n"
- ".inst 0x64a1e5f7 // fmmla z23.s, z15.s, z1.s\n"
- "trn1 z1.d, z6.d, z7.d\n"
- ".inst 0x64a2e598 // fmmla z24.s, z12.s, z2.s\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x64a2e5b9 // fmmla z25.s, z13.s, z2.s\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x64a2e5da // fmmla z26.s, z14.s, z2.s\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x64a2e5fb // fmmla z27.s, z15.s, z2.s\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "trn1 z2.d, z8.d, z9.d\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "trn2 z9.d, z6.d, z7.d\n"
- "trn2 z8.d, z4.d, z5.d\n"
- "ld1rqw z4.s, p6/z, [a_ptr4, #0x10]\n"
- ".inst 0x64a0e590 // fmmla z16.s, z12.s, z0.s\n"
- "ld1rqw z5.s, p6/z, [a_ptr5, #0x10]\n"
- ".inst 0x64a0e5b1 // fmmla z17.s, z13.s, z0.s\n"
- "addvl a_ptr4, a_ptr4, #2\n"
- ".inst 0x64a0e5d2 // fmmla z18.s, z14.s, z0.s\n"
- "addvl a_ptr5, a_ptr5, #2\n"
- ".inst 0x64a0e5f3 // fmmla z19.s, z15.s, z0.s\n"
- "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x10]\n"
- ".inst 0x64a1e594 // fmmla z20.s, z12.s, z1.s\n"
- "addvl %[a_ptr0], %[a_ptr0], #2\n"
- ".inst 0x64a1e5b5 // fmmla z21.s, z13.s, z1.s\n"
- ".inst 0x64a1e5d6 // fmmla z22.s, z14.s, z1.s\n"
- ".inst 0x64a1e5f7 // fmmla z23.s, z15.s, z1.s\n"
- "ld1rqw z1.s, p6/z, [a_ptr1, #0x10]\n"
- ".inst 0x64a2e598 // fmmla z24.s, z12.s, z2.s\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- ".inst 0x64a2e5b9 // fmmla z25.s, z13.s, z2.s\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- ".inst 0x64a2e5da // fmmla z26.s, z14.s, z2.s\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- ".inst 0x64a2e5fb // fmmla z27.s, z15.s, z2.s\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- ".inst 0x64a8e590 // fmmla z16.s, z12.s, z8.s\n"
- "ld1rqw z2.s, p6/z, [a_ptr2, #0x10]\n"
- ".inst 0x64a8e5b1 // fmmla z17.s, z13.s, z8.s\n"
- "addvl %[b_ptr0], %[b_ptr0], #-4\n"
- ".inst 0x64a8e5d2 // fmmla z18.s, z14.s, z8.s\n"
- "addvl a_ptr1, a_ptr1, #2\n"
- ".inst 0x64a8e5f3 // fmmla z19.s, z15.s, z8.s\n"
- "addvl a_ptr2, a_ptr2, #2\n"
- "trn1 z8.d, z0.d, z1.d\n"
- ".inst 0x64a9e594 // fmmla z20.s, z12.s, z9.s\n"
- ".inst 0x64a9e5b5 // fmmla z21.s, z13.s, z9.s\n"
- ".inst 0x64a9e5d6 // fmmla z22.s, z14.s, z9.s\n"
- ".inst 0x64a9e5f7 // fmmla z23.s, z15.s, z9.s\n"
- "trn1 z9.d, z2.d, z3.d\n"
- ".inst 0x64aae598 // fmmla z24.s, z12.s, z10.s\n"
- ".inst 0x64aae5b9 // fmmla z25.s, z13.s, z10.s\n"
- ".inst 0x64aae5da // fmmla z26.s, z14.s, z10.s\n"
- ".inst 0x64aae5fb // fmmla z27.s, z15.s, z10.s\n"
- "trn1 z10.d, z4.d, z5.d\n"
- "cbz %[blocks], 5f\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "ld1w z12.s, p7/z, [%[b_ptr0]]\n"
- "trn2 z1.d, z2.d, z3.d\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "trn2 z2.d, z4.d, z5.d\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- ".inst 0x64a8e590 // fmmla z16.s, z12.s, z8.s\n"
- ".inst 0x64a8e5b1 // fmmla z17.s, z13.s, z8.s\n"
- ".inst 0x64a8e5d2 // fmmla z18.s, z14.s, z8.s\n"
- ".inst 0x64a8e5f3 // fmmla z19.s, z15.s, z8.s\n"
- ".inst 0x64a9e594 // fmmla z20.s, z12.s, z9.s\n"
- ".inst 0x64a9e5b5 // fmmla z21.s, z13.s, z9.s\n"
- ".inst 0x64a9e5d6 // fmmla z22.s, z14.s, z9.s\n"
- ".inst 0x64a9e5f7 // fmmla z23.s, z15.s, z9.s\n"
- ".inst 0x64aae598 // fmmla z24.s, z12.s, z10.s\n"
- ".inst 0x64aae5b9 // fmmla z25.s, z13.s, z10.s\n"
- ".inst 0x64aae5da // fmmla z26.s, z14.s, z10.s\n"
- ".inst 0x64aae5fb // fmmla z27.s, z15.s, z10.s\n"
- "b.eq 5f\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x64a0e590 // fmmla z16.s, z12.s, z0.s\n"
- ".inst 0x64a0e5b1 // fmmla z17.s, z13.s, z0.s\n"
- ".inst 0x64a0e5d2 // fmmla z18.s, z14.s, z0.s\n"
- ".inst 0x64a0e5f3 // fmmla z19.s, z15.s, z0.s\n"
- ".inst 0x64a1e594 // fmmla z20.s, z12.s, z1.s\n"
- ".inst 0x64a1e5b5 // fmmla z21.s, z13.s, z1.s\n"
- ".inst 0x64a1e5d6 // fmmla z22.s, z14.s, z1.s\n"
- ".inst 0x64a1e5f7 // fmmla z23.s, z15.s, z1.s\n"
- ".inst 0x64a2e598 // fmmla z24.s, z12.s, z2.s\n"
- ".inst 0x64a2e5b9 // fmmla z25.s, z13.s, z2.s\n"
- ".inst 0x64a2e5da // fmmla z26.s, z14.s, z2.s\n"
- ".inst 0x64a2e5fb // fmmla z27.s, z15.s, z2.s\n"
- "b 5f\n"
- "4:\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "ld1rqw z6.s, p6/z, [a_ptr2]\n"
- "trn2 z1.d, z2.d, z3.d\n"
- "ld1rqw z7.s, p6/z, [a_ptr3]\n"
- "trn2 z2.d, z4.d, z5.d\n"
- "ld1rqw z4.s, p6/z, [%[a_ptr0]]\n"
- ".inst 0x64a8e590 // fmmla z16.s, z12.s, z8.s\n"
- "ld1rqw z5.s, p6/z, [a_ptr1]\n"
- ".inst 0x64a8e5b1 // fmmla z17.s, z13.s, z8.s\n"
- "addvl %[a_ptr0], %[a_ptr0], #1\n"
- ".inst 0x64a8e5d2 // fmmla z18.s, z14.s, z8.s\n"
- "addvl a_ptr1, a_ptr1, #1\n"
- ".inst 0x64a8e5f3 // fmmla z19.s, z15.s, z8.s\n"
- "ld1rqw z8.s, p6/z, [a_ptr4]\n"
- ".inst 0x64a9e594 // fmmla z20.s, z12.s, z9.s\n"
- "addvl a_ptr2, a_ptr2, #1\n"
- ".inst 0x64a9e5b5 // fmmla z21.s, z13.s, z9.s\n"
- "addvl a_ptr3, a_ptr3, #1\n"
- ".inst 0x64a9e5d6 // fmmla z22.s, z14.s, z9.s\n"
- "addvl a_ptr4, a_ptr4, #1\n"
- ".inst 0x64a9e5f7 // fmmla z23.s, z15.s, z9.s\n"
- "ld1rqw z9.s, p6/z, [a_ptr5]\n"
- ".inst 0x64aae598 // fmmla z24.s, z12.s, z10.s\n"
- "ld1w z12.s, p7/z, [%[b_ptr0]]\n"
- ".inst 0x64aae5b9 // fmmla z25.s, z13.s, z10.s\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x64aae5da // fmmla z26.s, z14.s, z10.s\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x64aae5fb // fmmla z27.s, z15.s, z10.s\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x64a0e590 // fmmla z16.s, z12.s, z0.s\n"
- "addvl %[b_ptr0], %[b_ptr0], #4\n"
- ".inst 0x64a0e5b1 // fmmla z17.s, z13.s, z0.s\n"
- "addvl a_ptr5, a_ptr5, #1\n"
- ".inst 0x64a0e5d2 // fmmla z18.s, z14.s, z0.s\n"
- ".inst 0x64a0e5f3 // fmmla z19.s, z15.s, z0.s\n"
- "trn1 z0.d, z4.d, z5.d\n"
- ".inst 0x64a1e594 // fmmla z20.s, z12.s, z1.s\n"
- ".inst 0x64a1e5b5 // fmmla z21.s, z13.s, z1.s\n"
- ".inst 0x64a1e5d6 // fmmla z22.s, z14.s, z1.s\n"
- ".inst 0x64a1e5f7 // fmmla z23.s, z15.s, z1.s\n"
- "trn1 z1.d, z6.d, z7.d\n"
- ".inst 0x64a2e598 // fmmla z24.s, z12.s, z2.s\n"
- ".inst 0x64a2e5b9 // fmmla z25.s, z13.s, z2.s\n"
- ".inst 0x64a2e5da // fmmla z26.s, z14.s, z2.s\n"
- ".inst 0x64a2e5fb // fmmla z27.s, z15.s, z2.s\n"
- "trn1 z2.d, z8.d, z9.d\n"
- "cbz %[blocks], 5f\n"
- "trn2 z10.d, z8.d, z9.d\n"
- "ld1w z12.s, p7/z, [%[b_ptr0]]\n"
- "trn2 z9.d, z6.d, z7.d\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "trn2 z8.d, z4.d, z5.d\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- ".inst 0x64a0e590 // fmmla z16.s, z12.s, z0.s\n"
- ".inst 0x64a0e5b1 // fmmla z17.s, z13.s, z0.s\n"
- ".inst 0x64a0e5d2 // fmmla z18.s, z14.s, z0.s\n"
- ".inst 0x64a0e5f3 // fmmla z19.s, z15.s, z0.s\n"
- ".inst 0x64a1e594 // fmmla z20.s, z12.s, z1.s\n"
- ".inst 0x64a1e5b5 // fmmla z21.s, z13.s, z1.s\n"
- ".inst 0x64a1e5d6 // fmmla z22.s, z14.s, z1.s\n"
- ".inst 0x64a1e5f7 // fmmla z23.s, z15.s, z1.s\n"
- ".inst 0x64a2e598 // fmmla z24.s, z12.s, z2.s\n"
- ".inst 0x64a2e5b9 // fmmla z25.s, z13.s, z2.s\n"
- ".inst 0x64a2e5da // fmmla z26.s, z14.s, z2.s\n"
- ".inst 0x64a2e5fb // fmmla z27.s, z15.s, z2.s\n"
- "b.eq 5f\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x64a8e590 // fmmla z16.s, z12.s, z8.s\n"
- ".inst 0x64a8e5b1 // fmmla z17.s, z13.s, z8.s\n"
- ".inst 0x64a8e5d2 // fmmla z18.s, z14.s, z8.s\n"
- ".inst 0x64a8e5f3 // fmmla z19.s, z15.s, z8.s\n"
- ".inst 0x64a9e594 // fmmla z20.s, z12.s, z9.s\n"
- ".inst 0x64a9e5b5 // fmmla z21.s, z13.s, z9.s\n"
- ".inst 0x64a9e5d6 // fmmla z22.s, z14.s, z9.s\n"
- ".inst 0x64a9e5f7 // fmmla z23.s, z15.s, z9.s\n"
- ".inst 0x64aae598 // fmmla z24.s, z12.s, z10.s\n"
- ".inst 0x64aae5b9 // fmmla z25.s, z13.s, z10.s\n"
- ".inst 0x64aae5da // fmmla z26.s, z14.s, z10.s\n"
- ".inst 0x64aae5fb // fmmla z27.s, z15.s, z10.s\n"
- "5:\n"
- "ld1rw z14.s, p7/z, [%[minptr]]\n"
- "ld1rw z15.s, p7/z, [%[maxptr]]\n"
- "fmax z16.s, p7/m, z16.s, z14.s\n"
- "fmax z17.s, p7/m, z17.s, z14.s\n"
- "fmax z18.s, p7/m, z18.s, z14.s\n"
- "fmax z19.s, p7/m, z19.s, z14.s\n"
- "fmin z16.s, p7/m, z16.s, z15.s\n"
- "fmin z17.s, p7/m, z17.s, z15.s\n"
- "fmin z18.s, p7/m, z18.s, z15.s\n"
- "fmin z19.s, p7/m, z19.s, z15.s\n"
- "fmax z20.s, p7/m, z20.s, z14.s\n"
- "uzp1 z0.s, z16.s, z17.s\n"
- "uzp2 z1.s, z16.s, z17.s\n"
- "uzp1 z2.s, z18.s, z19.s\n"
- "uzp2 z3.s, z18.s, z19.s\n"
- "st1w z0.s, p0, [%[c_ptr0]]\n"
- "fmin z20.s, p7/m, z20.s, z15.s\n"
- "fmax z21.s, p7/m, z21.s, z14.s\n"
- "fmax z22.s, p7/m, z22.s, z14.s\n"
- "st1w z1.s, p0, [c_ptr1]\n"
- "fmax z23.s, p7/m, z23.s, z14.s\n"
- "fmax z24.s, p7/m, z24.s, z14.s\n"
- "fmin z21.s, p7/m, z21.s, z15.s\n"
- "st1w z2.s, p1, [%[c_ptr0], #1, MUL VL]\n"
- "fmin z22.s, p7/m, z22.s, z15.s\n"
- "addvl %[c_ptr0], %[c_ptr0], #2\n"
- "fmin z23.s, p7/m, z23.s, z15.s\n"
- "st1w z3.s, p1, [c_ptr1, #1, MUL VL]\n"
- "uzp1 z4.s, z20.s, z21.s\n"
- "uzp2 z5.s, z20.s, z21.s\n"
- "fmin z24.s, p7/m, z24.s, z15.s\n"
- "uzp1 z6.s, z22.s, z23.s\n"
- "st1w z4.s, p0, [c_ptr2]\n"
- "uzp2 z7.s, z22.s, z23.s\n"
- "fmax z25.s, p7/m, z25.s, z14.s\n"
- "fmax z26.s, p7/m, z26.s, z14.s\n"
- "st1w z5.s, p0, [c_ptr3]\n"
- "fmax z27.s, p7/m, z27.s, z14.s\n"
- "fmin z25.s, p7/m, z25.s, z15.s\n"
- "st1w z6.s, p1, [c_ptr2, #1, MUL VL]\n"
- "fmin z26.s, p7/m, z26.s, z15.s\n"
- "fmin z27.s, p7/m, z27.s, z15.s\n"
- "uzp1 z8.s, z24.s, z25.s\n"
- "st1w z7.s, p1, [c_ptr3, #1, MUL VL]\n"
- "uzp2 z9.s, z24.s, z25.s\n"
- "uzp1 z10.s, z26.s, z27.s\n"
- "uzp2 z11.s, z26.s, z27.s\n"
- "st1w z8.s, p0, [c_ptr4]\n"
- "st1w z9.s, p0, [c_ptr5]\n"
- "st1w z10.s, p1, [c_ptr4, #1, MUL VL]\n"
- "st1w z11.s, p1, [c_ptr5, #1, MUL VL]\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq a_ptr3\n"
- ".unreq a_ptr4\n"
- ".unreq a_ptr5\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- ".unreq c_ptr3\n"
- ".unreq c_ptr4\n"
- ".unreq c_ptr5\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks)
- : [width] "r" (width), [accumulate] "r" (static_cast<uint64_t>(accumulate)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers)
- : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "cc", "memory"
- );
- break;
- case 7:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "a_ptr3 .req X2\n"
- "a_ptr4 .req X3\n"
- "a_ptr5 .req X4\n"
- "a_ptr6 .req X5\n"
- "c_ptr1 .req X6\n"
- "c_ptr2 .req X7\n"
- "c_ptr3 .req X8\n"
- "c_ptr4 .req X9\n"
- "c_ptr5 .req X10\n"
- "c_ptr6 .req X11\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "add a_ptr3, a_ptr2, %[lda]\n"
- "add c_ptr3, c_ptr2, %[ldc]\n"
- "add a_ptr4, a_ptr3, %[lda]\n"
- "add c_ptr4, c_ptr3, %[ldc]\n"
- "add a_ptr5, a_ptr4, %[lda]\n"
- "add c_ptr5, c_ptr4, %[ldc]\n"
- "add a_ptr6, a_ptr5, %[lda]\n"
- "add c_ptr6, c_ptr5, %[ldc]\n"
- "whilelt p6.s, %[temp], %[leftovers]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "ptrue p7.s\n"
- "whilelt p1.s, %[temp], %[width]\n"
- "cbnz %[accumulate], 1f\n"
- "mov z7.s, #0\n"
- "ld1w z15.s, p0/z, [%[biasptr]]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ld1rqw z1.s, p7/z, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "zip1 z16.s, z15.s, z15.s\n"
- "ld1rqw z2.s, p7/z, [a_ptr2]\n"
- "zip2 z17.s, z15.s, z15.s\n"
- "ld1w z15.s, p1/z, [%[biasptr], #1, MUL VL]\n"
- "trn1 z8.d, z0.d, z1.d\n"
- "ld1rqw z3.s, p7/z, [a_ptr3]\n"
- "ld1rqw z4.s, p7/z, [a_ptr4]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "zip1 z18.s, z15.s, z15.s\n"
- "ld1rqw z5.s, p7/z, [a_ptr5]\n"
- "zip2 z19.s, z15.s, z15.s\n"
- "ld1rqw z6.s, p7/z, [a_ptr6]\n"
- "trn1 z9.d, z2.d, z3.d\n"
- "ld1w z12.s, p7/z, [%[b_ptr0]]\n"
- "mov z20.d, z16.d\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "trn1 z10.d, z4.d, z5.d\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "trn1 z11.d, z6.d, z7.d\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "mov z21.d, z17.d\n"
- "add a_ptr3, a_ptr3, #0x10\n"
- "mov z22.d, z18.d\n"
- "add a_ptr4, a_ptr4, #0x10\n"
- "mov z23.d, z19.d\n"
- "add a_ptr5, a_ptr5, #0x10\n"
- "mov z24.d, z16.d\n"
- "add a_ptr6, a_ptr6, #0x10\n"
- "mov z25.d, z17.d\n"
- "addvl %[b_ptr0], %[b_ptr0], #4\n"
- "mov z26.d, z18.d\n"
- "mov z27.d, z19.d\n"
- "mov z28.d, z16.d\n"
- "mov z29.d, z17.d\n"
- "mov z30.d, z18.d\n"
- "mov z31.d, z19.d\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "mov z7.s, #0\n"
- "ld1w z13.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z14.s, p0/z, [c_ptr1]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ld1rqw z1.s, p7/z, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "zip1 z16.s, z13.s, z14.s\n"
- "ld1rqw z2.s, p7/z, [a_ptr2]\n"
- "zip2 z17.s, z13.s, z14.s\n"
- "ld1w z13.s, p1/z, [%[c_ptr0], #1, MUL VL]\n"
- "trn1 z8.d, z0.d, z1.d\n"
- "ld1w z14.s, p1/z, [c_ptr1, #1, MUL VL]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "ld1rqw z4.s, p7/z, [a_ptr4]\n"
- "add a_ptr3, a_ptr3, #0x10\n"
- "zip1 z18.s, z13.s, z14.s\n"
- "ld1rqw z5.s, p7/z, [a_ptr5]\n"
- "zip2 z19.s, z13.s, z14.s\n"
- "ld1w z13.s, p0/z, [c_ptr2]\n"
- "trn1 z9.d, z2.d, z3.d\n"
- "ld1w z14.s, p0/z, [c_ptr3]\n"
- "ld1rqw z6.s, p7/z, [a_ptr6]\n"
- "add a_ptr4, a_ptr4, #0x10\n"
- "trn1 z10.d, z4.d, z5.d\n"
- "ld1w z12.s, p7/z, [%[b_ptr0]]\n"
- "zip1 z20.s, z13.s, z14.s\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "zip2 z21.s, z13.s, z14.s\n"
- "ld1w z13.s, p1/z, [c_ptr2, #1, MUL VL]\n"
- "trn1 z11.d, z6.d, z7.d\n"
- "ld1w z14.s, p1/z, [c_ptr3, #1, MUL VL]\n"
- "add a_ptr5, a_ptr5, #0x10\n"
- "add a_ptr6, a_ptr6, #0x10\n"
- "zip1 z22.s, z13.s, z14.s\n"
- "zip2 z23.s, z13.s, z14.s\n"
- "ld1w z13.s, p0/z, [c_ptr4]\n"
- "ld1w z14.s, p0/z, [c_ptr5]\n"
- "zip1 z24.s, z13.s, z14.s\n"
- "zip2 z25.s, z13.s, z14.s\n"
- "ld1w z13.s, p1/z, [c_ptr4, #1, MUL VL]\n"
- "ld1w z14.s, p1/z, [c_ptr5, #1, MUL VL]\n"
- "zip1 z26.s, z13.s, z14.s\n"
- "zip2 z27.s, z13.s, z14.s\n"
- "ld1w z13.s, p0/z, [c_ptr6]\n"
- "mov z14.s, #0\n"
- "zip1 z28.s, z13.s, z14.s\n"
- "zip2 z29.s, z13.s, z14.s\n"
- "ld1w z13.s, p1/z, [c_ptr6, #1, MUL VL]\n"
- "mov z14.s, #0\n"
- "zip1 z30.s, z13.s, z14.s\n"
- "zip2 z31.s, z13.s, z14.s\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #4\n"
- "cbz %[loops], 2f\n"
- "3:\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "subs %[loops], %[loops], #0x1\n"
- "trn2 z1.d, z2.d, z3.d\n"
- "trn2 z2.d, z4.d, z5.d\n"
- "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n"
- "trn2 z3.d, z6.d, z7.d\n"
- "ld1rqw z5.s, p7/z, [a_ptr1]\n"
- ".inst 0x64a8e590 // fmmla z16.s, z12.s, z8.s\n"
- "ld1rqw z6.s, p7/z, [a_ptr2]\n"
- ".inst 0x64a8e5b1 // fmmla z17.s, z13.s, z8.s\n"
- "ld1rqw z7.s, p7/z, [a_ptr3]\n"
- ".inst 0x64a8e5d2 // fmmla z18.s, z14.s, z8.s\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
- ".inst 0x64a8e5f3 // fmmla z19.s, z15.s, z8.s\n"
- "ld1rqw z8.s, p7/z, [a_ptr4]\n"
- ".inst 0x64a9e594 // fmmla z20.s, z12.s, z9.s\n"
- "add a_ptr1, a_ptr1, #0x20\n"
- ".inst 0x64a9e5b5 // fmmla z21.s, z13.s, z9.s\n"
- "add a_ptr2, a_ptr2, #0x20\n"
- ".inst 0x64a9e5d6 // fmmla z22.s, z14.s, z9.s\n"
- "add a_ptr3, a_ptr3, #0x20\n"
- ".inst 0x64a9e5f7 // fmmla z23.s, z15.s, z9.s\n"
- "ld1rqw z9.s, p7/z, [a_ptr5]\n"
- ".inst 0x64aae598 // fmmla z24.s, z12.s, z10.s\n"
- "add a_ptr4, a_ptr4, #0x20\n"
- ".inst 0x64aae5b9 // fmmla z25.s, z13.s, z10.s\n"
- "add a_ptr5, a_ptr5, #0x20\n"
- ".inst 0x64aae5da // fmmla z26.s, z14.s, z10.s\n"
- ".inst 0x64aae5fb // fmmla z27.s, z15.s, z10.s\n"
- "ld1rqw z10.s, p7/z, [a_ptr6]\n"
- ".inst 0x64abe59c // fmmla z28.s, z12.s, z11.s\n"
- "ld1w z12.s, p7/z, [%[b_ptr0]]\n"
- ".inst 0x64abe5bd // fmmla z29.s, z13.s, z11.s\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x64abe5de // fmmla z30.s, z14.s, z11.s\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x64abe5ff // fmmla z31.s, z15.s, z11.s\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "mov z11.s, #0\n"
- "add a_ptr6, a_ptr6, #0x20\n"
- ".inst 0x64a0e590 // fmmla z16.s, z12.s, z0.s\n"
- ".inst 0x64a0e5b1 // fmmla z17.s, z13.s, z0.s\n"
- ".inst 0x64a0e5d2 // fmmla z18.s, z14.s, z0.s\n"
- ".inst 0x64a0e5f3 // fmmla z19.s, z15.s, z0.s\n"
- "trn1 z0.d, z4.d, z5.d\n"
- ".inst 0x64a1e594 // fmmla z20.s, z12.s, z1.s\n"
- ".inst 0x64a1e5b5 // fmmla z21.s, z13.s, z1.s\n"
- ".inst 0x64a1e5d6 // fmmla z22.s, z14.s, z1.s\n"
- ".inst 0x64a1e5f7 // fmmla z23.s, z15.s, z1.s\n"
- "trn1 z1.d, z6.d, z7.d\n"
- ".inst 0x64a2e598 // fmmla z24.s, z12.s, z2.s\n"
- ".inst 0x64a2e5b9 // fmmla z25.s, z13.s, z2.s\n"
- ".inst 0x64a2e5da // fmmla z26.s, z14.s, z2.s\n"
- ".inst 0x64a2e5fb // fmmla z27.s, z15.s, z2.s\n"
- "trn1 z2.d, z8.d, z9.d\n"
- ".inst 0x64a3e59c // fmmla z28.s, z12.s, z3.s\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x64a3e5bd // fmmla z29.s, z13.s, z3.s\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x64a3e5de // fmmla z30.s, z14.s, z3.s\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x64a3e5ff // fmmla z31.s, z15.s, z3.s\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "trn1 z3.d, z10.d, z11.d\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "trn2 z11.d, z10.d, z11.d\n"
- "trn2 z10.d, z8.d, z9.d\n"
- "trn2 z9.d, z6.d, z7.d\n"
- "ld1rqw z6.s, p7/z, [a_ptr6, #-0x10]\n"
- "trn2 z8.d, z4.d, z5.d\n"
- "ld1rqw z4.s, p7/z, [a_ptr4, #-0x10]\n"
- ".inst 0x64a0e590 // fmmla z16.s, z12.s, z0.s\n"
- "ld1rqw z5.s, p7/z, [a_ptr5, #-0x10]\n"
- ".inst 0x64a0e5b1 // fmmla z17.s, z13.s, z0.s\n"
- ".inst 0x64a0e5d2 // fmmla z18.s, z14.s, z0.s\n"
- ".inst 0x64a0e5f3 // fmmla z19.s, z15.s, z0.s\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #-0x10]\n"
- ".inst 0x64a1e594 // fmmla z20.s, z12.s, z1.s\n"
- ".inst 0x64a1e5b5 // fmmla z21.s, z13.s, z1.s\n"
- ".inst 0x64a1e5d6 // fmmla z22.s, z14.s, z1.s\n"
- ".inst 0x64a1e5f7 // fmmla z23.s, z15.s, z1.s\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #-0x10]\n"
- ".inst 0x64a2e598 // fmmla z24.s, z12.s, z2.s\n"
- ".inst 0x64a2e5b9 // fmmla z25.s, z13.s, z2.s\n"
- ".inst 0x64a2e5da // fmmla z26.s, z14.s, z2.s\n"
- ".inst 0x64a2e5fb // fmmla z27.s, z15.s, z2.s\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #-0x10]\n"
- ".inst 0x64a3e59c // fmmla z28.s, z12.s, z3.s\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- ".inst 0x64a3e5bd // fmmla z29.s, z13.s, z3.s\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- ".inst 0x64a3e5de // fmmla z30.s, z14.s, z3.s\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- ".inst 0x64a3e5ff // fmmla z31.s, z15.s, z3.s\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "mov z7.s, #0\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #-0x10]\n"
- ".inst 0x64a8e590 // fmmla z16.s, z12.s, z8.s\n"
- ".inst 0x64a8e5b1 // fmmla z17.s, z13.s, z8.s\n"
- ".inst 0x64a8e5d2 // fmmla z18.s, z14.s, z8.s\n"
- ".inst 0x64a8e5f3 // fmmla z19.s, z15.s, z8.s\n"
- "trn1 z8.d, z0.d, z1.d\n"
- ".inst 0x64a9e594 // fmmla z20.s, z12.s, z9.s\n"
- ".inst 0x64a9e5b5 // fmmla z21.s, z13.s, z9.s\n"
- ".inst 0x64a9e5d6 // fmmla z22.s, z14.s, z9.s\n"
- ".inst 0x64a9e5f7 // fmmla z23.s, z15.s, z9.s\n"
- "trn1 z9.d, z2.d, z3.d\n"
- ".inst 0x64aae598 // fmmla z24.s, z12.s, z10.s\n"
- ".inst 0x64aae5b9 // fmmla z25.s, z13.s, z10.s\n"
- ".inst 0x64aae5da // fmmla z26.s, z14.s, z10.s\n"
- ".inst 0x64aae5fb // fmmla z27.s, z15.s, z10.s\n"
- "trn1 z10.d, z4.d, z5.d\n"
- ".inst 0x64abe59c // fmmla z28.s, z12.s, z11.s\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- ".inst 0x64abe5bd // fmmla z29.s, z13.s, z11.s\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- ".inst 0x64abe5de // fmmla z30.s, z14.s, z11.s\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- ".inst 0x64abe5ff // fmmla z31.s, z15.s, z11.s\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "trn1 z11.d, z6.d, z7.d\n"
- "b.ne 3b\n"
- "2:\n"
- "cbz %[regs], 4f\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "trn2 z1.d, z2.d, z3.d\n"
- "trn2 z2.d, z4.d, z5.d\n"
- "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n"
- "trn2 z3.d, z6.d, z7.d\n"
- "ld1rqw z5.s, p7/z, [a_ptr1]\n"
- ".inst 0x64a8e590 // fmmla z16.s, z12.s, z8.s\n"
- "ld1rqw z6.s, p7/z, [a_ptr2]\n"
- ".inst 0x64a8e5b1 // fmmla z17.s, z13.s, z8.s\n"
- "ld1rqw z7.s, p7/z, [a_ptr3]\n"
- ".inst 0x64a8e5d2 // fmmla z18.s, z14.s, z8.s\n"
- ".inst 0x64a8e5f3 // fmmla z19.s, z15.s, z8.s\n"
- "ld1rqw z8.s, p7/z, [a_ptr4]\n"
- ".inst 0x64a9e594 // fmmla z20.s, z12.s, z9.s\n"
- ".inst 0x64a9e5b5 // fmmla z21.s, z13.s, z9.s\n"
- ".inst 0x64a9e5d6 // fmmla z22.s, z14.s, z9.s\n"
- ".inst 0x64a9e5f7 // fmmla z23.s, z15.s, z9.s\n"
- "ld1rqw z9.s, p7/z, [a_ptr5]\n"
- ".inst 0x64aae598 // fmmla z24.s, z12.s, z10.s\n"
- ".inst 0x64aae5b9 // fmmla z25.s, z13.s, z10.s\n"
- ".inst 0x64aae5da // fmmla z26.s, z14.s, z10.s\n"
- ".inst 0x64aae5fb // fmmla z27.s, z15.s, z10.s\n"
- "ld1rqw z10.s, p7/z, [a_ptr6]\n"
- ".inst 0x64abe59c // fmmla z28.s, z12.s, z11.s\n"
- "ld1w z12.s, p7/z, [%[b_ptr0]]\n"
- ".inst 0x64abe5bd // fmmla z29.s, z13.s, z11.s\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x64abe5de // fmmla z30.s, z14.s, z11.s\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x64abe5ff // fmmla z31.s, z15.s, z11.s\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "mov z11.s, #0\n"
- ".inst 0x64a0e590 // fmmla z16.s, z12.s, z0.s\n"
- ".inst 0x64a0e5b1 // fmmla z17.s, z13.s, z0.s\n"
- ".inst 0x64a0e5d2 // fmmla z18.s, z14.s, z0.s\n"
- ".inst 0x64a0e5f3 // fmmla z19.s, z15.s, z0.s\n"
- "trn1 z0.d, z4.d, z5.d\n"
- ".inst 0x64a1e594 // fmmla z20.s, z12.s, z1.s\n"
- ".inst 0x64a1e5b5 // fmmla z21.s, z13.s, z1.s\n"
- ".inst 0x64a1e5d6 // fmmla z22.s, z14.s, z1.s\n"
- ".inst 0x64a1e5f7 // fmmla z23.s, z15.s, z1.s\n"
- "trn1 z1.d, z6.d, z7.d\n"
- ".inst 0x64a2e598 // fmmla z24.s, z12.s, z2.s\n"
- ".inst 0x64a2e5b9 // fmmla z25.s, z13.s, z2.s\n"
- ".inst 0x64a2e5da // fmmla z26.s, z14.s, z2.s\n"
- ".inst 0x64a2e5fb // fmmla z27.s, z15.s, z2.s\n"
- "trn1 z2.d, z8.d, z9.d\n"
- ".inst 0x64a3e59c // fmmla z28.s, z12.s, z3.s\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x64a3e5bd // fmmla z29.s, z13.s, z3.s\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x64a3e5de // fmmla z30.s, z14.s, z3.s\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x64a3e5ff // fmmla z31.s, z15.s, z3.s\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "trn1 z3.d, z10.d, z11.d\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "trn2 z11.d, z10.d, z11.d\n"
- "trn2 z10.d, z8.d, z9.d\n"
- "trn2 z9.d, z6.d, z7.d\n"
- "ld1rqw z6.s, p6/z, [a_ptr6, #0x10]\n"
- "trn2 z8.d, z4.d, z5.d\n"
- "ld1rqw z4.s, p6/z, [a_ptr4, #0x10]\n"
- ".inst 0x64a0e590 // fmmla z16.s, z12.s, z0.s\n"
- "ld1rqw z5.s, p6/z, [a_ptr5, #0x10]\n"
- ".inst 0x64a0e5b1 // fmmla z17.s, z13.s, z0.s\n"
- "addvl a_ptr4, a_ptr4, #2\n"
- ".inst 0x64a0e5d2 // fmmla z18.s, z14.s, z0.s\n"
- "addvl a_ptr5, a_ptr5, #2\n"
- ".inst 0x64a0e5f3 // fmmla z19.s, z15.s, z0.s\n"
- "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x10]\n"
- ".inst 0x64a1e594 // fmmla z20.s, z12.s, z1.s\n"
- "addvl %[a_ptr0], %[a_ptr0], #2\n"
- ".inst 0x64a1e5b5 // fmmla z21.s, z13.s, z1.s\n"
- "addvl a_ptr6, a_ptr6, #2\n"
- ".inst 0x64a1e5d6 // fmmla z22.s, z14.s, z1.s\n"
- ".inst 0x64a1e5f7 // fmmla z23.s, z15.s, z1.s\n"
- "ld1rqw z1.s, p6/z, [a_ptr1, #0x10]\n"
- ".inst 0x64a2e598 // fmmla z24.s, z12.s, z2.s\n"
- "addvl a_ptr1, a_ptr1, #2\n"
- ".inst 0x64a2e5b9 // fmmla z25.s, z13.s, z2.s\n"
- ".inst 0x64a2e5da // fmmla z26.s, z14.s, z2.s\n"
- ".inst 0x64a2e5fb // fmmla z27.s, z15.s, z2.s\n"
- "ld1rqw z2.s, p6/z, [a_ptr2, #0x10]\n"
- ".inst 0x64a3e59c // fmmla z28.s, z12.s, z3.s\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- ".inst 0x64a3e5bd // fmmla z29.s, z13.s, z3.s\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- ".inst 0x64a3e5de // fmmla z30.s, z14.s, z3.s\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- ".inst 0x64a3e5ff // fmmla z31.s, z15.s, z3.s\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "mov z7.s, #0\n"
- "ld1rqw z3.s, p6/z, [a_ptr3, #0x10]\n"
- ".inst 0x64a8e590 // fmmla z16.s, z12.s, z8.s\n"
- "addvl %[b_ptr0], %[b_ptr0], #-4\n"
- ".inst 0x64a8e5b1 // fmmla z17.s, z13.s, z8.s\n"
- "addvl a_ptr2, a_ptr2, #2\n"
- ".inst 0x64a8e5d2 // fmmla z18.s, z14.s, z8.s\n"
- "addvl a_ptr3, a_ptr3, #2\n"
- ".inst 0x64a8e5f3 // fmmla z19.s, z15.s, z8.s\n"
- "trn1 z8.d, z0.d, z1.d\n"
- ".inst 0x64a9e594 // fmmla z20.s, z12.s, z9.s\n"
- ".inst 0x64a9e5b5 // fmmla z21.s, z13.s, z9.s\n"
- ".inst 0x64a9e5d6 // fmmla z22.s, z14.s, z9.s\n"
- ".inst 0x64a9e5f7 // fmmla z23.s, z15.s, z9.s\n"
- "trn1 z9.d, z2.d, z3.d\n"
- ".inst 0x64aae598 // fmmla z24.s, z12.s, z10.s\n"
- ".inst 0x64aae5b9 // fmmla z25.s, z13.s, z10.s\n"
- ".inst 0x64aae5da // fmmla z26.s, z14.s, z10.s\n"
- ".inst 0x64aae5fb // fmmla z27.s, z15.s, z10.s\n"
- "trn1 z10.d, z4.d, z5.d\n"
- ".inst 0x64abe59c // fmmla z28.s, z12.s, z11.s\n"
- ".inst 0x64abe5bd // fmmla z29.s, z13.s, z11.s\n"
- ".inst 0x64abe5de // fmmla z30.s, z14.s, z11.s\n"
- ".inst 0x64abe5ff // fmmla z31.s, z15.s, z11.s\n"
- "trn1 z11.d, z6.d, z7.d\n"
- "cbz %[blocks], 5f\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "ld1w z12.s, p7/z, [%[b_ptr0]]\n"
- "trn2 z1.d, z2.d, z3.d\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "trn2 z2.d, z4.d, z5.d\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "trn2 z3.d, z6.d, z7.d\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x64a8e590 // fmmla z16.s, z12.s, z8.s\n"
- "subs %[blocks], %[blocks], #0x1\n"
- ".inst 0x64a8e5b1 // fmmla z17.s, z13.s, z8.s\n"
- ".inst 0x64a8e5d2 // fmmla z18.s, z14.s, z8.s\n"
- ".inst 0x64a8e5f3 // fmmla z19.s, z15.s, z8.s\n"
- ".inst 0x64a9e594 // fmmla z20.s, z12.s, z9.s\n"
- ".inst 0x64a9e5b5 // fmmla z21.s, z13.s, z9.s\n"
- ".inst 0x64a9e5d6 // fmmla z22.s, z14.s, z9.s\n"
- ".inst 0x64a9e5f7 // fmmla z23.s, z15.s, z9.s\n"
- ".inst 0x64aae598 // fmmla z24.s, z12.s, z10.s\n"
- ".inst 0x64aae5b9 // fmmla z25.s, z13.s, z10.s\n"
- ".inst 0x64aae5da // fmmla z26.s, z14.s, z10.s\n"
- ".inst 0x64aae5fb // fmmla z27.s, z15.s, z10.s\n"
- ".inst 0x64abe59c // fmmla z28.s, z12.s, z11.s\n"
- ".inst 0x64abe5bd // fmmla z29.s, z13.s, z11.s\n"
- ".inst 0x64abe5de // fmmla z30.s, z14.s, z11.s\n"
- ".inst 0x64abe5ff // fmmla z31.s, z15.s, z11.s\n"
- "b.eq 5f\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x64a0e590 // fmmla z16.s, z12.s, z0.s\n"
- ".inst 0x64a0e5b1 // fmmla z17.s, z13.s, z0.s\n"
- ".inst 0x64a0e5d2 // fmmla z18.s, z14.s, z0.s\n"
- ".inst 0x64a0e5f3 // fmmla z19.s, z15.s, z0.s\n"
- ".inst 0x64a1e594 // fmmla z20.s, z12.s, z1.s\n"
- ".inst 0x64a1e5b5 // fmmla z21.s, z13.s, z1.s\n"
- ".inst 0x64a1e5d6 // fmmla z22.s, z14.s, z1.s\n"
- ".inst 0x64a1e5f7 // fmmla z23.s, z15.s, z1.s\n"
- ".inst 0x64a2e598 // fmmla z24.s, z12.s, z2.s\n"
- ".inst 0x64a2e5b9 // fmmla z25.s, z13.s, z2.s\n"
- ".inst 0x64a2e5da // fmmla z26.s, z14.s, z2.s\n"
- ".inst 0x64a2e5fb // fmmla z27.s, z15.s, z2.s\n"
- ".inst 0x64a3e59c // fmmla z28.s, z12.s, z3.s\n"
- ".inst 0x64a3e5bd // fmmla z29.s, z13.s, z3.s\n"
- ".inst 0x64a3e5de // fmmla z30.s, z14.s, z3.s\n"
- ".inst 0x64a3e5ff // fmmla z31.s, z15.s, z3.s\n"
- "b 5f\n"
- "4:\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "trn2 z1.d, z2.d, z3.d\n"
- "trn2 z2.d, z4.d, z5.d\n"
- "ld1rqw z4.s, p6/z, [%[a_ptr0]]\n"
- "trn2 z3.d, z6.d, z7.d\n"
- "ld1rqw z5.s, p6/z, [a_ptr1]\n"
- ".inst 0x64a8e590 // fmmla z16.s, z12.s, z8.s\n"
- "ld1rqw z6.s, p6/z, [a_ptr2]\n"
- ".inst 0x64a8e5b1 // fmmla z17.s, z13.s, z8.s\n"
- "ld1rqw z7.s, p6/z, [a_ptr3]\n"
- ".inst 0x64a8e5d2 // fmmla z18.s, z14.s, z8.s\n"
- "addvl %[a_ptr0], %[a_ptr0], #1\n"
- ".inst 0x64a8e5f3 // fmmla z19.s, z15.s, z8.s\n"
- "ld1rqw z8.s, p6/z, [a_ptr4]\n"
- ".inst 0x64a9e594 // fmmla z20.s, z12.s, z9.s\n"
- "addvl a_ptr1, a_ptr1, #1\n"
- ".inst 0x64a9e5b5 // fmmla z21.s, z13.s, z9.s\n"
- "addvl a_ptr2, a_ptr2, #1\n"
- ".inst 0x64a9e5d6 // fmmla z22.s, z14.s, z9.s\n"
- "addvl a_ptr3, a_ptr3, #1\n"
- ".inst 0x64a9e5f7 // fmmla z23.s, z15.s, z9.s\n"
- "ld1rqw z9.s, p6/z, [a_ptr5]\n"
- ".inst 0x64aae598 // fmmla z24.s, z12.s, z10.s\n"
- "addvl a_ptr4, a_ptr4, #1\n"
- ".inst 0x64aae5b9 // fmmla z25.s, z13.s, z10.s\n"
- "addvl a_ptr5, a_ptr5, #1\n"
- ".inst 0x64aae5da // fmmla z26.s, z14.s, z10.s\n"
- ".inst 0x64aae5fb // fmmla z27.s, z15.s, z10.s\n"
- "ld1rqw z10.s, p6/z, [a_ptr6]\n"
- ".inst 0x64abe59c // fmmla z28.s, z12.s, z11.s\n"
- "ld1w z12.s, p7/z, [%[b_ptr0]]\n"
- ".inst 0x64abe5bd // fmmla z29.s, z13.s, z11.s\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x64abe5de // fmmla z30.s, z14.s, z11.s\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x64abe5ff // fmmla z31.s, z15.s, z11.s\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "mov z11.s, #0\n"
- "addvl %[b_ptr0], %[b_ptr0], #4\n"
- ".inst 0x64a0e590 // fmmla z16.s, z12.s, z0.s\n"
- "addvl a_ptr6, a_ptr6, #1\n"
- ".inst 0x64a0e5b1 // fmmla z17.s, z13.s, z0.s\n"
- ".inst 0x64a0e5d2 // fmmla z18.s, z14.s, z0.s\n"
- ".inst 0x64a0e5f3 // fmmla z19.s, z15.s, z0.s\n"
- "trn1 z0.d, z4.d, z5.d\n"
- ".inst 0x64a1e594 // fmmla z20.s, z12.s, z1.s\n"
- ".inst 0x64a1e5b5 // fmmla z21.s, z13.s, z1.s\n"
- ".inst 0x64a1e5d6 // fmmla z22.s, z14.s, z1.s\n"
- ".inst 0x64a1e5f7 // fmmla z23.s, z15.s, z1.s\n"
- "trn1 z1.d, z6.d, z7.d\n"
- ".inst 0x64a2e598 // fmmla z24.s, z12.s, z2.s\n"
- ".inst 0x64a2e5b9 // fmmla z25.s, z13.s, z2.s\n"
- ".inst 0x64a2e5da // fmmla z26.s, z14.s, z2.s\n"
- ".inst 0x64a2e5fb // fmmla z27.s, z15.s, z2.s\n"
- "trn1 z2.d, z8.d, z9.d\n"
- ".inst 0x64a3e59c // fmmla z28.s, z12.s, z3.s\n"
- ".inst 0x64a3e5bd // fmmla z29.s, z13.s, z3.s\n"
- ".inst 0x64a3e5de // fmmla z30.s, z14.s, z3.s\n"
- ".inst 0x64a3e5ff // fmmla z31.s, z15.s, z3.s\n"
- "trn1 z3.d, z10.d, z11.d\n"
- "cbz %[blocks], 5f\n"
- "trn2 z11.d, z10.d, z11.d\n"
- "ld1w z12.s, p7/z, [%[b_ptr0]]\n"
- "trn2 z10.d, z8.d, z9.d\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "trn2 z9.d, z6.d, z7.d\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "trn2 z8.d, z4.d, z5.d\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x64a0e590 // fmmla z16.s, z12.s, z0.s\n"
- "subs %[blocks], %[blocks], #0x1\n"
- ".inst 0x64a0e5b1 // fmmla z17.s, z13.s, z0.s\n"
- ".inst 0x64a0e5d2 // fmmla z18.s, z14.s, z0.s\n"
- ".inst 0x64a0e5f3 // fmmla z19.s, z15.s, z0.s\n"
- ".inst 0x64a1e594 // fmmla z20.s, z12.s, z1.s\n"
- ".inst 0x64a1e5b5 // fmmla z21.s, z13.s, z1.s\n"
- ".inst 0x64a1e5d6 // fmmla z22.s, z14.s, z1.s\n"
- ".inst 0x64a1e5f7 // fmmla z23.s, z15.s, z1.s\n"
- ".inst 0x64a2e598 // fmmla z24.s, z12.s, z2.s\n"
- ".inst 0x64a2e5b9 // fmmla z25.s, z13.s, z2.s\n"
- ".inst 0x64a2e5da // fmmla z26.s, z14.s, z2.s\n"
- ".inst 0x64a2e5fb // fmmla z27.s, z15.s, z2.s\n"
- ".inst 0x64a3e59c // fmmla z28.s, z12.s, z3.s\n"
- ".inst 0x64a3e5bd // fmmla z29.s, z13.s, z3.s\n"
- ".inst 0x64a3e5de // fmmla z30.s, z14.s, z3.s\n"
- ".inst 0x64a3e5ff // fmmla z31.s, z15.s, z3.s\n"
- "b.eq 5f\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x64a8e590 // fmmla z16.s, z12.s, z8.s\n"
- ".inst 0x64a8e5b1 // fmmla z17.s, z13.s, z8.s\n"
- ".inst 0x64a8e5d2 // fmmla z18.s, z14.s, z8.s\n"
- ".inst 0x64a8e5f3 // fmmla z19.s, z15.s, z8.s\n"
- ".inst 0x64a9e594 // fmmla z20.s, z12.s, z9.s\n"
- ".inst 0x64a9e5b5 // fmmla z21.s, z13.s, z9.s\n"
- ".inst 0x64a9e5d6 // fmmla z22.s, z14.s, z9.s\n"
- ".inst 0x64a9e5f7 // fmmla z23.s, z15.s, z9.s\n"
- ".inst 0x64aae598 // fmmla z24.s, z12.s, z10.s\n"
- ".inst 0x64aae5b9 // fmmla z25.s, z13.s, z10.s\n"
- ".inst 0x64aae5da // fmmla z26.s, z14.s, z10.s\n"
- ".inst 0x64aae5fb // fmmla z27.s, z15.s, z10.s\n"
- ".inst 0x64abe59c // fmmla z28.s, z12.s, z11.s\n"
- ".inst 0x64abe5bd // fmmla z29.s, z13.s, z11.s\n"
- ".inst 0x64abe5de // fmmla z30.s, z14.s, z11.s\n"
- ".inst 0x64abe5ff // fmmla z31.s, z15.s, z11.s\n"
- "5:\n"
- "ld1rw z14.s, p7/z, [%[minptr]]\n"
- "ld1rw z15.s, p7/z, [%[maxptr]]\n"
- "fmax z16.s, p7/m, z16.s, z14.s\n"
- "fmax z17.s, p7/m, z17.s, z14.s\n"
- "fmax z18.s, p7/m, z18.s, z14.s\n"
- "fmax z19.s, p7/m, z19.s, z14.s\n"
- "fmin z16.s, p7/m, z16.s, z15.s\n"
- "fmin z17.s, p7/m, z17.s, z15.s\n"
- "fmin z18.s, p7/m, z18.s, z15.s\n"
- "fmin z19.s, p7/m, z19.s, z15.s\n"
- "fmax z20.s, p7/m, z20.s, z14.s\n"
- "uzp1 z0.s, z16.s, z17.s\n"
- "uzp2 z1.s, z16.s, z17.s\n"
- "uzp1 z2.s, z18.s, z19.s\n"
- "uzp2 z3.s, z18.s, z19.s\n"
- "st1w z0.s, p0, [%[c_ptr0]]\n"
- "fmin z20.s, p7/m, z20.s, z15.s\n"
- "fmax z21.s, p7/m, z21.s, z14.s\n"
- "fmax z22.s, p7/m, z22.s, z14.s\n"
- "st1w z1.s, p0, [c_ptr1]\n"
- "fmax z23.s, p7/m, z23.s, z14.s\n"
- "fmax z24.s, p7/m, z24.s, z14.s\n"
- "fmin z21.s, p7/m, z21.s, z15.s\n"
- "st1w z2.s, p1, [%[c_ptr0], #1, MUL VL]\n"
- "fmin z22.s, p7/m, z22.s, z15.s\n"
- "addvl %[c_ptr0], %[c_ptr0], #2\n"
- "fmin z23.s, p7/m, z23.s, z15.s\n"
- "st1w z3.s, p1, [c_ptr1, #1, MUL VL]\n"
- "uzp1 z4.s, z20.s, z21.s\n"
- "uzp2 z5.s, z20.s, z21.s\n"
- "fmin z24.s, p7/m, z24.s, z15.s\n"
- "uzp1 z6.s, z22.s, z23.s\n"
- "st1w z4.s, p0, [c_ptr2]\n"
- "uzp2 z7.s, z22.s, z23.s\n"
- "fmax z25.s, p7/m, z25.s, z14.s\n"
- "fmax z26.s, p7/m, z26.s, z14.s\n"
- "st1w z5.s, p0, [c_ptr3]\n"
- "fmax z27.s, p7/m, z27.s, z14.s\n"
- "fmax z28.s, p7/m, z28.s, z14.s\n"
- "fmin z25.s, p7/m, z25.s, z15.s\n"
- "st1w z6.s, p1, [c_ptr2, #1, MUL VL]\n"
- "fmin z26.s, p7/m, z26.s, z15.s\n"
- "fmin z27.s, p7/m, z27.s, z15.s\n"
- "fmin z28.s, p7/m, z28.s, z15.s\n"
- "st1w z7.s, p1, [c_ptr3, #1, MUL VL]\n"
- "uzp1 z8.s, z24.s, z25.s\n"
- "uzp2 z9.s, z24.s, z25.s\n"
- "uzp1 z10.s, z26.s, z27.s\n"
- "uzp2 z11.s, z26.s, z27.s\n"
- "st1w z8.s, p0, [c_ptr4]\n"
- "fmax z29.s, p7/m, z29.s, z14.s\n"
- "fmax z30.s, p7/m, z30.s, z14.s\n"
- "fmax z31.s, p7/m, z31.s, z14.s\n"
- "st1w z9.s, p0, [c_ptr5]\n"
- "fmin z29.s, p7/m, z29.s, z15.s\n"
- "fmin z30.s, p7/m, z30.s, z15.s\n"
- "st1w z10.s, p1, [c_ptr4, #1, MUL VL]\n"
- "fmin z31.s, p7/m, z31.s, z15.s\n"
- "uzp1 z12.s, z28.s, z29.s\n"
- "st1w z11.s, p1, [c_ptr5, #1, MUL VL]\n"
- "uzp1 z13.s, z30.s, z31.s\n"
- "st1w z12.s, p0, [c_ptr6]\n"
- "st1w z13.s, p1, [c_ptr6, #1, MUL VL]\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq a_ptr3\n"
- ".unreq a_ptr4\n"
- ".unreq a_ptr5\n"
- ".unreq a_ptr6\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- ".unreq c_ptr3\n"
- ".unreq c_ptr4\n"
- ".unreq c_ptr5\n"
- ".unreq c_ptr6\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks)
- : [width] "r" (width), [accumulate] "r" (static_cast<uint64_t>(accumulate)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers)
- : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "cc", "memory"
- );
- break;
- default:
- case 8:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "a_ptr3 .req X2\n"
- "a_ptr4 .req X3\n"
- "a_ptr5 .req X4\n"
- "a_ptr6 .req X5\n"
- "a_ptr7 .req X6\n"
- "c_ptr1 .req X7\n"
- "c_ptr2 .req X8\n"
- "c_ptr3 .req X9\n"
- "c_ptr4 .req X10\n"
- "c_ptr5 .req X11\n"
- "c_ptr6 .req X12\n"
- "c_ptr7 .req X13\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "add a_ptr3, a_ptr2, %[lda]\n"
- "add c_ptr3, c_ptr2, %[ldc]\n"
- "add a_ptr4, a_ptr3, %[lda]\n"
- "add c_ptr4, c_ptr3, %[ldc]\n"
- "add a_ptr5, a_ptr4, %[lda]\n"
- "add c_ptr5, c_ptr4, %[ldc]\n"
- "add a_ptr6, a_ptr5, %[lda]\n"
- "add c_ptr6, c_ptr5, %[ldc]\n"
- "add a_ptr7, a_ptr6, %[lda]\n"
- "add c_ptr7, c_ptr6, %[ldc]\n"
- "whilelt p6.s, %[temp], %[leftovers]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "ptrue p7.s\n"
- "whilelt p1.s, %[temp], %[width]\n"
- "cbnz %[accumulate], 1f\n"
- "ld1w z15.s, p0/z, [%[biasptr]]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ld1rqw z1.s, p7/z, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "zip1 z16.s, z15.s, z15.s\n"
- "ld1rqw z2.s, p7/z, [a_ptr2]\n"
- "zip2 z17.s, z15.s, z15.s\n"
- "ld1w z15.s, p1/z, [%[biasptr], #1, MUL VL]\n"
- "trn1 z8.d, z0.d, z1.d\n"
- "ld1rqw z3.s, p7/z, [a_ptr3]\n"
- "ld1rqw z4.s, p7/z, [a_ptr4]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "zip1 z18.s, z15.s, z15.s\n"
- "ld1rqw z5.s, p7/z, [a_ptr5]\n"
- "zip2 z19.s, z15.s, z15.s\n"
- "ld1rqw z6.s, p7/z, [a_ptr6]\n"
- "trn1 z9.d, z2.d, z3.d\n"
- "ld1rqw z7.s, p7/z, [a_ptr7]\n"
- "mov z20.d, z16.d\n"
- "ld1w z12.s, p7/z, [%[b_ptr0]]\n"
- "trn1 z10.d, z4.d, z5.d\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "mov z21.d, z17.d\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "trn1 z11.d, z6.d, z7.d\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "mov z22.d, z18.d\n"
- "add a_ptr3, a_ptr3, #0x10\n"
- "mov z23.d, z19.d\n"
- "add a_ptr4, a_ptr4, #0x10\n"
- "mov z24.d, z16.d\n"
- "add a_ptr5, a_ptr5, #0x10\n"
- "mov z25.d, z17.d\n"
- "add a_ptr6, a_ptr6, #0x10\n"
- "mov z26.d, z18.d\n"
- "add a_ptr7, a_ptr7, #0x10\n"
- "mov z27.d, z19.d\n"
- "addvl %[b_ptr0], %[b_ptr0], #4\n"
- "mov z28.d, z16.d\n"
- "mov z29.d, z17.d\n"
- "mov z30.d, z18.d\n"
- "mov z31.d, z19.d\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "ld1w z13.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z14.s, p0/z, [c_ptr1]\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ld1rqw z1.s, p7/z, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "zip1 z16.s, z13.s, z14.s\n"
- "ld1rqw z2.s, p7/z, [a_ptr2]\n"
- "zip2 z17.s, z13.s, z14.s\n"
- "ld1w z13.s, p1/z, [%[c_ptr0], #1, MUL VL]\n"
- "trn1 z8.d, z0.d, z1.d\n"
- "ld1w z14.s, p1/z, [c_ptr1, #1, MUL VL]\n"
- "ld1rqw z3.s, p7/z, [a_ptr3]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "ld1rqw z4.s, p7/z, [a_ptr4]\n"
- "add a_ptr3, a_ptr3, #0x10\n"
- "zip1 z18.s, z13.s, z14.s\n"
- "ld1rqw z5.s, p7/z, [a_ptr5]\n"
- "zip2 z19.s, z13.s, z14.s\n"
- "ld1w z13.s, p0/z, [c_ptr2]\n"
- "trn1 z9.d, z2.d, z3.d\n"
- "ld1w z14.s, p0/z, [c_ptr3]\n"
- "ld1rqw z6.s, p7/z, [a_ptr6]\n"
- "add a_ptr4, a_ptr4, #0x10\n"
- "trn1 z10.d, z4.d, z5.d\n"
- "ld1rqw z7.s, p7/z, [a_ptr7]\n"
- "zip1 z20.s, z13.s, z14.s\n"
- "ld1w z12.s, p7/z, [%[b_ptr0]]\n"
- "zip2 z21.s, z13.s, z14.s\n"
- "ld1w z13.s, p1/z, [c_ptr2, #1, MUL VL]\n"
- "ld1w z14.s, p1/z, [c_ptr3, #1, MUL VL]\n"
- "add a_ptr5, a_ptr5, #0x10\n"
- "trn1 z11.d, z6.d, z7.d\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "add a_ptr6, a_ptr6, #0x10\n"
- "zip1 z22.s, z13.s, z14.s\n"
- "add a_ptr7, a_ptr7, #0x10\n"
- "zip2 z23.s, z13.s, z14.s\n"
- "ld1w z13.s, p0/z, [c_ptr4]\n"
- "ld1w z14.s, p0/z, [c_ptr5]\n"
- "zip1 z24.s, z13.s, z14.s\n"
- "zip2 z25.s, z13.s, z14.s\n"
- "ld1w z13.s, p1/z, [c_ptr4, #1, MUL VL]\n"
- "ld1w z14.s, p1/z, [c_ptr5, #1, MUL VL]\n"
- "zip1 z26.s, z13.s, z14.s\n"
- "zip2 z27.s, z13.s, z14.s\n"
- "ld1w z13.s, p0/z, [c_ptr6]\n"
- "ld1w z14.s, p0/z, [c_ptr7]\n"
- "zip1 z28.s, z13.s, z14.s\n"
- "zip2 z29.s, z13.s, z14.s\n"
- "ld1w z13.s, p1/z, [c_ptr6, #1, MUL VL]\n"
- "ld1w z14.s, p1/z, [c_ptr7, #1, MUL VL]\n"
- "zip1 z30.s, z13.s, z14.s\n"
- "zip2 z31.s, z13.s, z14.s\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #4\n"
- "cbz %[loops], 2f\n"
- "3:\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "subs %[loops], %[loops], #0x1\n"
- "trn2 z1.d, z2.d, z3.d\n"
- "trn2 z2.d, z4.d, z5.d\n"
- "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n"
- "trn2 z3.d, z6.d, z7.d\n"
- "ld1rqw z5.s, p7/z, [a_ptr1]\n"
- ".inst 0x64a8e590 // fmmla z16.s, z12.s, z8.s\n"
- "ld1rqw z6.s, p7/z, [a_ptr2]\n"
- ".inst 0x64a8e5b1 // fmmla z17.s, z13.s, z8.s\n"
- "ld1rqw z7.s, p7/z, [a_ptr3]\n"
- ".inst 0x64a8e5d2 // fmmla z18.s, z14.s, z8.s\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
- ".inst 0x64a8e5f3 // fmmla z19.s, z15.s, z8.s\n"
- "ld1rqw z8.s, p7/z, [a_ptr4]\n"
- ".inst 0x64a9e594 // fmmla z20.s, z12.s, z9.s\n"
- "add a_ptr1, a_ptr1, #0x20\n"
- ".inst 0x64a9e5b5 // fmmla z21.s, z13.s, z9.s\n"
- "add a_ptr2, a_ptr2, #0x20\n"
- ".inst 0x64a9e5d6 // fmmla z22.s, z14.s, z9.s\n"
- "add a_ptr3, a_ptr3, #0x20\n"
- ".inst 0x64a9e5f7 // fmmla z23.s, z15.s, z9.s\n"
- "ld1rqw z9.s, p7/z, [a_ptr5]\n"
- ".inst 0x64aae598 // fmmla z24.s, z12.s, z10.s\n"
- "add a_ptr4, a_ptr4, #0x20\n"
- ".inst 0x64aae5b9 // fmmla z25.s, z13.s, z10.s\n"
- "add a_ptr5, a_ptr5, #0x20\n"
- ".inst 0x64aae5da // fmmla z26.s, z14.s, z10.s\n"
- ".inst 0x64aae5fb // fmmla z27.s, z15.s, z10.s\n"
- "ld1rqw z10.s, p7/z, [a_ptr6]\n"
- ".inst 0x64abe59c // fmmla z28.s, z12.s, z11.s\n"
- "ld1w z12.s, p7/z, [%[b_ptr0]]\n"
- ".inst 0x64abe5bd // fmmla z29.s, z13.s, z11.s\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x64abe5de // fmmla z30.s, z14.s, z11.s\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x64abe5ff // fmmla z31.s, z15.s, z11.s\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x64a0e590 // fmmla z16.s, z12.s, z0.s\n"
- "ld1rqw z11.s, p7/z, [a_ptr7]\n"
- ".inst 0x64a0e5b1 // fmmla z17.s, z13.s, z0.s\n"
- "add a_ptr6, a_ptr6, #0x20\n"
- ".inst 0x64a0e5d2 // fmmla z18.s, z14.s, z0.s\n"
- "add a_ptr7, a_ptr7, #0x20\n"
- ".inst 0x64a0e5f3 // fmmla z19.s, z15.s, z0.s\n"
- "trn1 z0.d, z4.d, z5.d\n"
- ".inst 0x64a1e594 // fmmla z20.s, z12.s, z1.s\n"
- ".inst 0x64a1e5b5 // fmmla z21.s, z13.s, z1.s\n"
- ".inst 0x64a1e5d6 // fmmla z22.s, z14.s, z1.s\n"
- ".inst 0x64a1e5f7 // fmmla z23.s, z15.s, z1.s\n"
- "trn1 z1.d, z6.d, z7.d\n"
- ".inst 0x64a2e598 // fmmla z24.s, z12.s, z2.s\n"
- ".inst 0x64a2e5b9 // fmmla z25.s, z13.s, z2.s\n"
- ".inst 0x64a2e5da // fmmla z26.s, z14.s, z2.s\n"
- ".inst 0x64a2e5fb // fmmla z27.s, z15.s, z2.s\n"
- "trn1 z2.d, z8.d, z9.d\n"
- ".inst 0x64a3e59c // fmmla z28.s, z12.s, z3.s\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x64a3e5bd // fmmla z29.s, z13.s, z3.s\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x64a3e5de // fmmla z30.s, z14.s, z3.s\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x64a3e5ff // fmmla z31.s, z15.s, z3.s\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "trn1 z3.d, z10.d, z11.d\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "trn2 z11.d, z10.d, z11.d\n"
- "trn2 z10.d, z8.d, z9.d\n"
- "trn2 z9.d, z6.d, z7.d\n"
- "ld1rqw z6.s, p7/z, [a_ptr6, #-0x10]\n"
- "trn2 z8.d, z4.d, z5.d\n"
- "ld1rqw z4.s, p7/z, [a_ptr4, #-0x10]\n"
- ".inst 0x64a0e590 // fmmla z16.s, z12.s, z0.s\n"
- "ld1rqw z5.s, p7/z, [a_ptr5, #-0x10]\n"
- ".inst 0x64a0e5b1 // fmmla z17.s, z13.s, z0.s\n"
- "ld1rqw z7.s, p7/z, [a_ptr7, #-0x10]\n"
- ".inst 0x64a0e5d2 // fmmla z18.s, z14.s, z0.s\n"
- ".inst 0x64a0e5f3 // fmmla z19.s, z15.s, z0.s\n"
- "ld1rqw z0.s, p7/z, [%[a_ptr0], #-0x10]\n"
- ".inst 0x64a1e594 // fmmla z20.s, z12.s, z1.s\n"
- ".inst 0x64a1e5b5 // fmmla z21.s, z13.s, z1.s\n"
- ".inst 0x64a1e5d6 // fmmla z22.s, z14.s, z1.s\n"
- ".inst 0x64a1e5f7 // fmmla z23.s, z15.s, z1.s\n"
- "ld1rqw z1.s, p7/z, [a_ptr1, #-0x10]\n"
- ".inst 0x64a2e598 // fmmla z24.s, z12.s, z2.s\n"
- ".inst 0x64a2e5b9 // fmmla z25.s, z13.s, z2.s\n"
- ".inst 0x64a2e5da // fmmla z26.s, z14.s, z2.s\n"
- ".inst 0x64a2e5fb // fmmla z27.s, z15.s, z2.s\n"
- "ld1rqw z2.s, p7/z, [a_ptr2, #-0x10]\n"
- ".inst 0x64a3e59c // fmmla z28.s, z12.s, z3.s\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- ".inst 0x64a3e5bd // fmmla z29.s, z13.s, z3.s\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- ".inst 0x64a3e5de // fmmla z30.s, z14.s, z3.s\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- ".inst 0x64a3e5ff // fmmla z31.s, z15.s, z3.s\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- ".inst 0x64a8e590 // fmmla z16.s, z12.s, z8.s\n"
- "ld1rqw z3.s, p7/z, [a_ptr3, #-0x10]\n"
- ".inst 0x64a8e5b1 // fmmla z17.s, z13.s, z8.s\n"
- ".inst 0x64a8e5d2 // fmmla z18.s, z14.s, z8.s\n"
- ".inst 0x64a8e5f3 // fmmla z19.s, z15.s, z8.s\n"
- "trn1 z8.d, z0.d, z1.d\n"
- ".inst 0x64a9e594 // fmmla z20.s, z12.s, z9.s\n"
- ".inst 0x64a9e5b5 // fmmla z21.s, z13.s, z9.s\n"
- ".inst 0x64a9e5d6 // fmmla z22.s, z14.s, z9.s\n"
- ".inst 0x64a9e5f7 // fmmla z23.s, z15.s, z9.s\n"
- "trn1 z9.d, z2.d, z3.d\n"
- ".inst 0x64aae598 // fmmla z24.s, z12.s, z10.s\n"
- ".inst 0x64aae5b9 // fmmla z25.s, z13.s, z10.s\n"
- ".inst 0x64aae5da // fmmla z26.s, z14.s, z10.s\n"
- ".inst 0x64aae5fb // fmmla z27.s, z15.s, z10.s\n"
- "trn1 z10.d, z4.d, z5.d\n"
- ".inst 0x64abe59c // fmmla z28.s, z12.s, z11.s\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- ".inst 0x64abe5bd // fmmla z29.s, z13.s, z11.s\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- ".inst 0x64abe5de // fmmla z30.s, z14.s, z11.s\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- ".inst 0x64abe5ff // fmmla z31.s, z15.s, z11.s\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "trn1 z11.d, z6.d, z7.d\n"
- "b.ne 3b\n"
- "2:\n"
- "cbz %[regs], 4f\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "trn2 z1.d, z2.d, z3.d\n"
- "trn2 z2.d, z4.d, z5.d\n"
- "ld1rqw z4.s, p7/z, [%[a_ptr0]]\n"
- "trn2 z3.d, z6.d, z7.d\n"
- "ld1rqw z5.s, p7/z, [a_ptr1]\n"
- ".inst 0x64a8e590 // fmmla z16.s, z12.s, z8.s\n"
- "ld1rqw z6.s, p7/z, [a_ptr2]\n"
- ".inst 0x64a8e5b1 // fmmla z17.s, z13.s, z8.s\n"
- "ld1rqw z7.s, p7/z, [a_ptr3]\n"
- ".inst 0x64a8e5d2 // fmmla z18.s, z14.s, z8.s\n"
- ".inst 0x64a8e5f3 // fmmla z19.s, z15.s, z8.s\n"
- "ld1rqw z8.s, p7/z, [a_ptr4]\n"
- ".inst 0x64a9e594 // fmmla z20.s, z12.s, z9.s\n"
- ".inst 0x64a9e5b5 // fmmla z21.s, z13.s, z9.s\n"
- ".inst 0x64a9e5d6 // fmmla z22.s, z14.s, z9.s\n"
- ".inst 0x64a9e5f7 // fmmla z23.s, z15.s, z9.s\n"
- "ld1rqw z9.s, p7/z, [a_ptr5]\n"
- ".inst 0x64aae598 // fmmla z24.s, z12.s, z10.s\n"
- ".inst 0x64aae5b9 // fmmla z25.s, z13.s, z10.s\n"
- ".inst 0x64aae5da // fmmla z26.s, z14.s, z10.s\n"
- ".inst 0x64aae5fb // fmmla z27.s, z15.s, z10.s\n"
- "ld1rqw z10.s, p7/z, [a_ptr6]\n"
- ".inst 0x64abe59c // fmmla z28.s, z12.s, z11.s\n"
- "ld1w z12.s, p7/z, [%[b_ptr0]]\n"
- ".inst 0x64abe5bd // fmmla z29.s, z13.s, z11.s\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x64abe5de // fmmla z30.s, z14.s, z11.s\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x64abe5ff // fmmla z31.s, z15.s, z11.s\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x64a0e590 // fmmla z16.s, z12.s, z0.s\n"
- "ld1rqw z11.s, p7/z, [a_ptr7]\n"
- ".inst 0x64a0e5b1 // fmmla z17.s, z13.s, z0.s\n"
- ".inst 0x64a0e5d2 // fmmla z18.s, z14.s, z0.s\n"
- ".inst 0x64a0e5f3 // fmmla z19.s, z15.s, z0.s\n"
- "trn1 z0.d, z4.d, z5.d\n"
- ".inst 0x64a1e594 // fmmla z20.s, z12.s, z1.s\n"
- ".inst 0x64a1e5b5 // fmmla z21.s, z13.s, z1.s\n"
- ".inst 0x64a1e5d6 // fmmla z22.s, z14.s, z1.s\n"
- ".inst 0x64a1e5f7 // fmmla z23.s, z15.s, z1.s\n"
- "trn1 z1.d, z6.d, z7.d\n"
- ".inst 0x64a2e598 // fmmla z24.s, z12.s, z2.s\n"
- ".inst 0x64a2e5b9 // fmmla z25.s, z13.s, z2.s\n"
- ".inst 0x64a2e5da // fmmla z26.s, z14.s, z2.s\n"
- ".inst 0x64a2e5fb // fmmla z27.s, z15.s, z2.s\n"
- "trn1 z2.d, z8.d, z9.d\n"
- ".inst 0x64a3e59c // fmmla z28.s, z12.s, z3.s\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- ".inst 0x64a3e5bd // fmmla z29.s, z13.s, z3.s\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- ".inst 0x64a3e5de // fmmla z30.s, z14.s, z3.s\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- ".inst 0x64a3e5ff // fmmla z31.s, z15.s, z3.s\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "trn1 z3.d, z10.d, z11.d\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "trn2 z11.d, z10.d, z11.d\n"
- "trn2 z10.d, z8.d, z9.d\n"
- "trn2 z9.d, z6.d, z7.d\n"
- "ld1rqw z6.s, p6/z, [a_ptr6, #0x10]\n"
- "trn2 z8.d, z4.d, z5.d\n"
- "ld1rqw z4.s, p6/z, [a_ptr4, #0x10]\n"
- ".inst 0x64a0e590 // fmmla z16.s, z12.s, z0.s\n"
- "ld1rqw z5.s, p6/z, [a_ptr5, #0x10]\n"
- ".inst 0x64a0e5b1 // fmmla z17.s, z13.s, z0.s\n"
- "ld1rqw z7.s, p6/z, [a_ptr7, #0x10]\n"
- ".inst 0x64a0e5d2 // fmmla z18.s, z14.s, z0.s\n"
- "addvl a_ptr4, a_ptr4, #2\n"
- ".inst 0x64a0e5f3 // fmmla z19.s, z15.s, z0.s\n"
- "ld1rqw z0.s, p6/z, [%[a_ptr0], #0x10]\n"
- ".inst 0x64a1e594 // fmmla z20.s, z12.s, z1.s\n"
- "addvl %[a_ptr0], %[a_ptr0], #2\n"
- ".inst 0x64a1e5b5 // fmmla z21.s, z13.s, z1.s\n"
- "addvl a_ptr5, a_ptr5, #2\n"
- ".inst 0x64a1e5d6 // fmmla z22.s, z14.s, z1.s\n"
- "addvl a_ptr6, a_ptr6, #2\n"
- ".inst 0x64a1e5f7 // fmmla z23.s, z15.s, z1.s\n"
- "ld1rqw z1.s, p6/z, [a_ptr1, #0x10]\n"
- ".inst 0x64a2e598 // fmmla z24.s, z12.s, z2.s\n"
- "addvl a_ptr1, a_ptr1, #2\n"
- ".inst 0x64a2e5b9 // fmmla z25.s, z13.s, z2.s\n"
- "addvl a_ptr7, a_ptr7, #2\n"
- ".inst 0x64a2e5da // fmmla z26.s, z14.s, z2.s\n"
- ".inst 0x64a2e5fb // fmmla z27.s, z15.s, z2.s\n"
- "ld1rqw z2.s, p6/z, [a_ptr2, #0x10]\n"
- ".inst 0x64a3e59c // fmmla z28.s, z12.s, z3.s\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- ".inst 0x64a3e5bd // fmmla z29.s, z13.s, z3.s\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- ".inst 0x64a3e5de // fmmla z30.s, z14.s, z3.s\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- ".inst 0x64a3e5ff // fmmla z31.s, z15.s, z3.s\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- ".inst 0x64a8e590 // fmmla z16.s, z12.s, z8.s\n"
- "ld1rqw z3.s, p6/z, [a_ptr3, #0x10]\n"
- ".inst 0x64a8e5b1 // fmmla z17.s, z13.s, z8.s\n"
- "addvl %[b_ptr0], %[b_ptr0], #-4\n"
- ".inst 0x64a8e5d2 // fmmla z18.s, z14.s, z8.s\n"
- "addvl a_ptr2, a_ptr2, #2\n"
- ".inst 0x64a8e5f3 // fmmla z19.s, z15.s, z8.s\n"
- "addvl a_ptr3, a_ptr3, #2\n"
- "trn1 z8.d, z0.d, z1.d\n"
- ".inst 0x64a9e594 // fmmla z20.s, z12.s, z9.s\n"
- ".inst 0x64a9e5b5 // fmmla z21.s, z13.s, z9.s\n"
- ".inst 0x64a9e5d6 // fmmla z22.s, z14.s, z9.s\n"
- ".inst 0x64a9e5f7 // fmmla z23.s, z15.s, z9.s\n"
- "trn1 z9.d, z2.d, z3.d\n"
- ".inst 0x64aae598 // fmmla z24.s, z12.s, z10.s\n"
- ".inst 0x64aae5b9 // fmmla z25.s, z13.s, z10.s\n"
- ".inst 0x64aae5da // fmmla z26.s, z14.s, z10.s\n"
- ".inst 0x64aae5fb // fmmla z27.s, z15.s, z10.s\n"
- "trn1 z10.d, z4.d, z5.d\n"
- ".inst 0x64abe59c // fmmla z28.s, z12.s, z11.s\n"
- ".inst 0x64abe5bd // fmmla z29.s, z13.s, z11.s\n"
- ".inst 0x64abe5de // fmmla z30.s, z14.s, z11.s\n"
- ".inst 0x64abe5ff // fmmla z31.s, z15.s, z11.s\n"
- "trn1 z11.d, z6.d, z7.d\n"
- "cbz %[blocks], 5f\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "ld1w z12.s, p7/z, [%[b_ptr0]]\n"
- "trn2 z1.d, z2.d, z3.d\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "trn2 z2.d, z4.d, z5.d\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "trn2 z3.d, z6.d, z7.d\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x64a8e590 // fmmla z16.s, z12.s, z8.s\n"
- "subs %[blocks], %[blocks], #0x1\n"
- ".inst 0x64a8e5b1 // fmmla z17.s, z13.s, z8.s\n"
- ".inst 0x64a8e5d2 // fmmla z18.s, z14.s, z8.s\n"
- ".inst 0x64a8e5f3 // fmmla z19.s, z15.s, z8.s\n"
- ".inst 0x64a9e594 // fmmla z20.s, z12.s, z9.s\n"
- ".inst 0x64a9e5b5 // fmmla z21.s, z13.s, z9.s\n"
- ".inst 0x64a9e5d6 // fmmla z22.s, z14.s, z9.s\n"
- ".inst 0x64a9e5f7 // fmmla z23.s, z15.s, z9.s\n"
- ".inst 0x64aae598 // fmmla z24.s, z12.s, z10.s\n"
- ".inst 0x64aae5b9 // fmmla z25.s, z13.s, z10.s\n"
- ".inst 0x64aae5da // fmmla z26.s, z14.s, z10.s\n"
- ".inst 0x64aae5fb // fmmla z27.s, z15.s, z10.s\n"
- ".inst 0x64abe59c // fmmla z28.s, z12.s, z11.s\n"
- ".inst 0x64abe5bd // fmmla z29.s, z13.s, z11.s\n"
- ".inst 0x64abe5de // fmmla z30.s, z14.s, z11.s\n"
- ".inst 0x64abe5ff // fmmla z31.s, z15.s, z11.s\n"
- "b.eq 5f\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x64a0e590 // fmmla z16.s, z12.s, z0.s\n"
- ".inst 0x64a0e5b1 // fmmla z17.s, z13.s, z0.s\n"
- ".inst 0x64a0e5d2 // fmmla z18.s, z14.s, z0.s\n"
- ".inst 0x64a0e5f3 // fmmla z19.s, z15.s, z0.s\n"
- ".inst 0x64a1e594 // fmmla z20.s, z12.s, z1.s\n"
- ".inst 0x64a1e5b5 // fmmla z21.s, z13.s, z1.s\n"
- ".inst 0x64a1e5d6 // fmmla z22.s, z14.s, z1.s\n"
- ".inst 0x64a1e5f7 // fmmla z23.s, z15.s, z1.s\n"
- ".inst 0x64a2e598 // fmmla z24.s, z12.s, z2.s\n"
- ".inst 0x64a2e5b9 // fmmla z25.s, z13.s, z2.s\n"
- ".inst 0x64a2e5da // fmmla z26.s, z14.s, z2.s\n"
- ".inst 0x64a2e5fb // fmmla z27.s, z15.s, z2.s\n"
- ".inst 0x64a3e59c // fmmla z28.s, z12.s, z3.s\n"
- ".inst 0x64a3e5bd // fmmla z29.s, z13.s, z3.s\n"
- ".inst 0x64a3e5de // fmmla z30.s, z14.s, z3.s\n"
- ".inst 0x64a3e5ff // fmmla z31.s, z15.s, z3.s\n"
- "b 5f\n"
- "4:\n"
- "trn2 z0.d, z0.d, z1.d\n"
- "trn2 z1.d, z2.d, z3.d\n"
- "trn2 z2.d, z4.d, z5.d\n"
- "ld1rqw z4.s, p6/z, [%[a_ptr0]]\n"
- "trn2 z3.d, z6.d, z7.d\n"
- "ld1rqw z5.s, p6/z, [a_ptr1]\n"
- ".inst 0x64a8e590 // fmmla z16.s, z12.s, z8.s\n"
- "ld1rqw z6.s, p6/z, [a_ptr2]\n"
- ".inst 0x64a8e5b1 // fmmla z17.s, z13.s, z8.s\n"
- "ld1rqw z7.s, p6/z, [a_ptr3]\n"
- ".inst 0x64a8e5d2 // fmmla z18.s, z14.s, z8.s\n"
- "addvl %[a_ptr0], %[a_ptr0], #1\n"
- ".inst 0x64a8e5f3 // fmmla z19.s, z15.s, z8.s\n"
- "ld1rqw z8.s, p6/z, [a_ptr4]\n"
- ".inst 0x64a9e594 // fmmla z20.s, z12.s, z9.s\n"
- "addvl a_ptr1, a_ptr1, #1\n"
- ".inst 0x64a9e5b5 // fmmla z21.s, z13.s, z9.s\n"
- "addvl a_ptr2, a_ptr2, #1\n"
- ".inst 0x64a9e5d6 // fmmla z22.s, z14.s, z9.s\n"
- "addvl a_ptr3, a_ptr3, #1\n"
- ".inst 0x64a9e5f7 // fmmla z23.s, z15.s, z9.s\n"
- "ld1rqw z9.s, p6/z, [a_ptr5]\n"
- ".inst 0x64aae598 // fmmla z24.s, z12.s, z10.s\n"
- "addvl a_ptr4, a_ptr4, #1\n"
- ".inst 0x64aae5b9 // fmmla z25.s, z13.s, z10.s\n"
- "addvl a_ptr5, a_ptr5, #1\n"
- ".inst 0x64aae5da // fmmla z26.s, z14.s, z10.s\n"
- ".inst 0x64aae5fb // fmmla z27.s, z15.s, z10.s\n"
- "ld1rqw z10.s, p6/z, [a_ptr6]\n"
- ".inst 0x64abe59c // fmmla z28.s, z12.s, z11.s\n"
- "ld1w z12.s, p7/z, [%[b_ptr0]]\n"
- ".inst 0x64abe5bd // fmmla z29.s, z13.s, z11.s\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- ".inst 0x64abe5de // fmmla z30.s, z14.s, z11.s\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- ".inst 0x64abe5ff // fmmla z31.s, z15.s, z11.s\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x64a0e590 // fmmla z16.s, z12.s, z0.s\n"
- "ld1rqw z11.s, p6/z, [a_ptr7]\n"
- ".inst 0x64a0e5b1 // fmmla z17.s, z13.s, z0.s\n"
- "addvl %[b_ptr0], %[b_ptr0], #4\n"
- ".inst 0x64a0e5d2 // fmmla z18.s, z14.s, z0.s\n"
- "addvl a_ptr6, a_ptr6, #1\n"
- ".inst 0x64a0e5f3 // fmmla z19.s, z15.s, z0.s\n"
- "addvl a_ptr7, a_ptr7, #1\n"
- "trn1 z0.d, z4.d, z5.d\n"
- ".inst 0x64a1e594 // fmmla z20.s, z12.s, z1.s\n"
- ".inst 0x64a1e5b5 // fmmla z21.s, z13.s, z1.s\n"
- ".inst 0x64a1e5d6 // fmmla z22.s, z14.s, z1.s\n"
- ".inst 0x64a1e5f7 // fmmla z23.s, z15.s, z1.s\n"
- "trn1 z1.d, z6.d, z7.d\n"
- ".inst 0x64a2e598 // fmmla z24.s, z12.s, z2.s\n"
- ".inst 0x64a2e5b9 // fmmla z25.s, z13.s, z2.s\n"
- ".inst 0x64a2e5da // fmmla z26.s, z14.s, z2.s\n"
- ".inst 0x64a2e5fb // fmmla z27.s, z15.s, z2.s\n"
- "trn1 z2.d, z8.d, z9.d\n"
- ".inst 0x64a3e59c // fmmla z28.s, z12.s, z3.s\n"
- ".inst 0x64a3e5bd // fmmla z29.s, z13.s, z3.s\n"
- ".inst 0x64a3e5de // fmmla z30.s, z14.s, z3.s\n"
- ".inst 0x64a3e5ff // fmmla z31.s, z15.s, z3.s\n"
- "trn1 z3.d, z10.d, z11.d\n"
- "cbz %[blocks], 5f\n"
- "trn2 z11.d, z10.d, z11.d\n"
- "ld1w z12.s, p7/z, [%[b_ptr0]]\n"
- "trn2 z10.d, z8.d, z9.d\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "trn2 z9.d, z6.d, z7.d\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "trn2 z8.d, z4.d, z5.d\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- ".inst 0x64a0e590 // fmmla z16.s, z12.s, z0.s\n"
- "subs %[blocks], %[blocks], #0x1\n"
- ".inst 0x64a0e5b1 // fmmla z17.s, z13.s, z0.s\n"
- ".inst 0x64a0e5d2 // fmmla z18.s, z14.s, z0.s\n"
- ".inst 0x64a0e5f3 // fmmla z19.s, z15.s, z0.s\n"
- ".inst 0x64a1e594 // fmmla z20.s, z12.s, z1.s\n"
- ".inst 0x64a1e5b5 // fmmla z21.s, z13.s, z1.s\n"
- ".inst 0x64a1e5d6 // fmmla z22.s, z14.s, z1.s\n"
- ".inst 0x64a1e5f7 // fmmla z23.s, z15.s, z1.s\n"
- ".inst 0x64a2e598 // fmmla z24.s, z12.s, z2.s\n"
- ".inst 0x64a2e5b9 // fmmla z25.s, z13.s, z2.s\n"
- ".inst 0x64a2e5da // fmmla z26.s, z14.s, z2.s\n"
- ".inst 0x64a2e5fb // fmmla z27.s, z15.s, z2.s\n"
- ".inst 0x64a3e59c // fmmla z28.s, z12.s, z3.s\n"
- ".inst 0x64a3e5bd // fmmla z29.s, z13.s, z3.s\n"
- ".inst 0x64a3e5de // fmmla z30.s, z14.s, z3.s\n"
- ".inst 0x64a3e5ff // fmmla z31.s, z15.s, z3.s\n"
- "b.eq 5f\n"
- "ld1w z12.s, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1w z13.s, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1w z14.s, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1w z15.s, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- ".inst 0x64a8e590 // fmmla z16.s, z12.s, z8.s\n"
- ".inst 0x64a8e5b1 // fmmla z17.s, z13.s, z8.s\n"
- ".inst 0x64a8e5d2 // fmmla z18.s, z14.s, z8.s\n"
- ".inst 0x64a8e5f3 // fmmla z19.s, z15.s, z8.s\n"
- ".inst 0x64a9e594 // fmmla z20.s, z12.s, z9.s\n"
- ".inst 0x64a9e5b5 // fmmla z21.s, z13.s, z9.s\n"
- ".inst 0x64a9e5d6 // fmmla z22.s, z14.s, z9.s\n"
- ".inst 0x64a9e5f7 // fmmla z23.s, z15.s, z9.s\n"
- ".inst 0x64aae598 // fmmla z24.s, z12.s, z10.s\n"
- ".inst 0x64aae5b9 // fmmla z25.s, z13.s, z10.s\n"
- ".inst 0x64aae5da // fmmla z26.s, z14.s, z10.s\n"
- ".inst 0x64aae5fb // fmmla z27.s, z15.s, z10.s\n"
- ".inst 0x64abe59c // fmmla z28.s, z12.s, z11.s\n"
- ".inst 0x64abe5bd // fmmla z29.s, z13.s, z11.s\n"
- ".inst 0x64abe5de // fmmla z30.s, z14.s, z11.s\n"
- ".inst 0x64abe5ff // fmmla z31.s, z15.s, z11.s\n"
- "5:\n"
- "ld1rw z14.s, p7/z, [%[minptr]]\n"
- "ld1rw z15.s, p7/z, [%[maxptr]]\n"
- "fmax z16.s, p7/m, z16.s, z14.s\n"
- "fmax z17.s, p7/m, z17.s, z14.s\n"
- "fmax z18.s, p7/m, z18.s, z14.s\n"
- "fmax z19.s, p7/m, z19.s, z14.s\n"
- "fmin z16.s, p7/m, z16.s, z15.s\n"
- "fmin z17.s, p7/m, z17.s, z15.s\n"
- "fmin z18.s, p7/m, z18.s, z15.s\n"
- "fmin z19.s, p7/m, z19.s, z15.s\n"
- "fmax z20.s, p7/m, z20.s, z14.s\n"
- "uzp1 z0.s, z16.s, z17.s\n"
- "uzp2 z1.s, z16.s, z17.s\n"
- "uzp1 z2.s, z18.s, z19.s\n"
- "uzp2 z3.s, z18.s, z19.s\n"
- "st1w z0.s, p0, [%[c_ptr0]]\n"
- "fmin z20.s, p7/m, z20.s, z15.s\n"
- "fmax z21.s, p7/m, z21.s, z14.s\n"
- "fmax z22.s, p7/m, z22.s, z14.s\n"
- "st1w z1.s, p0, [c_ptr1]\n"
- "fmax z23.s, p7/m, z23.s, z14.s\n"
- "fmax z24.s, p7/m, z24.s, z14.s\n"
- "fmin z21.s, p7/m, z21.s, z15.s\n"
- "st1w z2.s, p1, [%[c_ptr0], #1, MUL VL]\n"
- "fmin z22.s, p7/m, z22.s, z15.s\n"
- "addvl %[c_ptr0], %[c_ptr0], #2\n"
- "fmin z23.s, p7/m, z23.s, z15.s\n"
- "st1w z3.s, p1, [c_ptr1, #1, MUL VL]\n"
- "uzp1 z4.s, z20.s, z21.s\n"
- "uzp2 z5.s, z20.s, z21.s\n"
- "fmin z24.s, p7/m, z24.s, z15.s\n"
- "uzp1 z6.s, z22.s, z23.s\n"
- "st1w z4.s, p0, [c_ptr2]\n"
- "uzp2 z7.s, z22.s, z23.s\n"
- "fmax z25.s, p7/m, z25.s, z14.s\n"
- "fmax z26.s, p7/m, z26.s, z14.s\n"
- "st1w z5.s, p0, [c_ptr3]\n"
- "fmax z27.s, p7/m, z27.s, z14.s\n"
- "fmax z28.s, p7/m, z28.s, z14.s\n"
- "fmin z25.s, p7/m, z25.s, z15.s\n"
- "st1w z6.s, p1, [c_ptr2, #1, MUL VL]\n"
- "fmin z26.s, p7/m, z26.s, z15.s\n"
- "fmin z27.s, p7/m, z27.s, z15.s\n"
- "fmin z28.s, p7/m, z28.s, z15.s\n"
- "st1w z7.s, p1, [c_ptr3, #1, MUL VL]\n"
- "uzp1 z8.s, z24.s, z25.s\n"
- "uzp2 z9.s, z24.s, z25.s\n"
- "uzp1 z10.s, z26.s, z27.s\n"
- "uzp2 z11.s, z26.s, z27.s\n"
- "st1w z8.s, p0, [c_ptr4]\n"
- "fmax z29.s, p7/m, z29.s, z14.s\n"
- "fmax z30.s, p7/m, z30.s, z14.s\n"
- "fmax z31.s, p7/m, z31.s, z14.s\n"
- "st1w z9.s, p0, [c_ptr5]\n"
- "fmin z29.s, p7/m, z29.s, z15.s\n"
- "fmin z30.s, p7/m, z30.s, z15.s\n"
- "st1w z10.s, p1, [c_ptr4, #1, MUL VL]\n"
- "fmin z31.s, p7/m, z31.s, z15.s\n"
- "uzp1 z12.s, z28.s, z29.s\n"
- "st1w z11.s, p1, [c_ptr5, #1, MUL VL]\n"
- "uzp2 z13.s, z28.s, z29.s\n"
- "uzp1 z14.s, z30.s, z31.s\n"
- "uzp2 z15.s, z30.s, z31.s\n"
- "st1w z12.s, p0, [c_ptr6]\n"
- "st1w z13.s, p0, [c_ptr7]\n"
- "st1w z14.s, p1, [c_ptr6, #1, MUL VL]\n"
- "st1w z15.s, p1, [c_ptr7, #1, MUL VL]\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq a_ptr3\n"
- ".unreq a_ptr4\n"
- ".unreq a_ptr5\n"
- ".unreq a_ptr6\n"
- ".unreq a_ptr7\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- ".unreq c_ptr3\n"
- ".unreq c_ptr4\n"
- ".unreq c_ptr5\n"
- ".unreq c_ptr6\n"
- ".unreq c_ptr7\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks)
- : [width] "r" (width), [accumulate] "r" (static_cast<uint64_t>(accumulate)), [lda] "r" (ldab), [ldc] "r" (ldcb), [biasptr] "r" (biasptr), [minptr] "r" (minptr), [maxptr] "r" (maxptr), [leftovers] "r" (leftovers)
- : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "cc", "memory"
- );
- break;
- }
-
- }
- }
-}
-
-} // namespace arm_gemm
-
-#endif // __ARM_FEATURE_SVE
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8s32_dot_4VLx4.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8qa_dot_4x4VL.hpp
similarity index 66%
rename from src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8s32_dot_4VLx4.hpp
rename to src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8qa_dot_4x4VL.hpp
index c500f43..0150ce8 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8s32_dot_4VLx4.hpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8qa_dot_4x4VL.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2018-2020 Arm Limited.
+ * Copyright (c) 2019-2020 Arm Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -10,37 +10,43 @@
* sell copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
*/
#pragma once
-
#ifdef __ARM_FEATURE_SVE
-#include <cstdint>
#include "../std_transforms_sve.hpp"
+#define ARGLIST \
+ unsigned int, const unsigned int *, \
+ IndirectInputArg<int8_t>, \
+ size_t, size_t, \
+ const int8_t *, \
+ IndirectOutputArg<int8_t>, \
+ const Requantize32 *, const int32_t *, unsigned int
+
namespace arm_gemm
{
// Actual kernel implementations
-void sve_hybrid_s8s32_dot_4VLx4(const int8_t *, int, const int8_t *, int32_t *, int, int, int, int, const int32_t *, Activation, bool);
+void sve_hybrid_s8qa_dot_4x4VL( ARGLIST );
-class hybrid_s8s32_dot_4VLx4
+class cls_sve_hybrid_s8qa_dot_4x4VL
{
public:
typedef int8_t operand_type;
- typedef int32_t result_type;
+ typedef int8_t result_type;
- typedef void (*kern_type)(const int8_t *, int, const int8_t *, int32_t *, int, int, int, int, const int32_t *, Activation, bool);
+ typedef void (*kern_type)( ARGLIST );
/* Kernel blocking parameters */
static constexpr unsigned int out_height()
@@ -60,30 +66,20 @@
static constexpr bool supports_accumulate()
{
- return true;
- }
-
- static constexpr bool supports_bias()
- {
- return false;
- }
-
- static constexpr bool supports_activation()
- {
return false;
}
StdTransformsSVE<operand_type, result_type, 4, 4, 4> transforms = {};
// Default to the generic kernel
- kern_type kernel=sve_hybrid_s8s32_dot_4VLx4;
+ kern_type kernel=sve_hybrid_s8qa_dot_4x4VL;
- hybrid_s8s32_dot_4VLx4(const CPUInfo *)
+ cls_sve_hybrid_s8qa_dot_4x4VL(const CPUInfo *)
{
-
}
};
} // namespace arm_gemm
+#undef ARGLIST
#endif // __ARM_FEATURE_SVE
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8qa_dot_4x4VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8qa_dot_4x4VL/generic.cpp
new file mode 100644
index 0000000..2b1448b
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8qa_dot_4x4VL/generic.cpp
@@ -0,0 +1,1602 @@
+/*
+ * Copyright (c) 2019-2020 Arm Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+#ifdef __ARM_FEATURE_SVE
+
+#include "arm_gemm.hpp"
+#include "../../utils.hpp"
+
+#include <cassert>
+#include <cstdint>
+
+namespace arm_gemm {
+
+void sve_hybrid_s8qa_dot_4x4VL (
+ unsigned int num_strings, const unsigned int *string_lengths, IndirectInputArg<int8_t> A_arg,
+ size_t M, size_t N, const int8_t *B_ptr, IndirectOutputArg<int8_t> output_arg,
+ const Requantize32 *qp, const int32_t *col_bias, unsigned int
+)
+{
+ struct KernelArgs {
+ unsigned int num_strings = {};
+ const unsigned int *string_lengths = {};
+ size_t N = {};
+ const int8_t *B_ptr = {};
+ size_t output_offset = {};
+ size_t input_initial_col = {};
+ size_t input_offset = {};
+ } ka;
+
+ unsigned long flags=0;
+ void *output_ptr;
+ void *input_ptr;
+
+ if (output_arg.is_indirect) {
+ output_ptr=(void *)(output_arg.indirect.ptr);
+ ka.output_offset=output_arg.indirect.offset;
+ flags |= 0x4;
+ } else {
+ output_ptr=(void *)(output_arg.direct.base);
+ ka.output_offset=output_arg.direct.stride;
+ }
+
+ if (A_arg.is_indirect) {
+ input_ptr=(void *)(A_arg.indirect.ptr);
+ ka.input_offset=A_arg.indirect.start_row;
+ ka.input_initial_col=A_arg.indirect.start_col;
+ flags |= 0x8;
+ } else {
+ assert(num_strings==1);
+ input_ptr=(void *)(A_arg.direct.base);
+ ka.input_offset=A_arg.direct.stride;
+ }
+ ka.num_strings = num_strings;
+ ka.string_lengths = string_lengths;
+ ka.N = N;
+ ka.B_ptr = B_ptr;
+ if (qp->c_offset > qp->minval) {
+ flags |= 0x20;
+ }
+ __asm__ __volatile__(
+ "ptrue p2.b\n"
+ "1:" // Row loop
+ "cmp %x[M], #0x4\n"
+ "bge 46f\n"
+ "cmp %x[M], #0x2\n"
+ "bgt 31f\n"
+ "beq 16f\n"
+ "mov z11.s, #0x0\n"
+ "ldr x12, [%x[args_ptr], %[offsetof_N]]\n"
+ "mov z12.s, #0x0\n"
+ "ldr x11, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "mov x10, %x[col_bias]\n"
+ "mov z13.s, #0x0\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "bic %x[flags], %x[flags], #0x80000000\n"
+ "mov z14.s, #0x0\n"
+ "mov z15.b, #0x1\n"
+ "tbz %x[flags], #2, 2f\n"
+ "ldr x9, [%x[output_ptr], #0x0]\n"
+ "add x9, x9, x19\n"
+ "b 3f\n"
+ "2:" // Height 1: setup direct output
+ "mov x9, %x[output_ptr]\n"
+ "3:" // Height 1: Column loop
+ "mov z16.s, #0x0\n"
+ "mov x19, #0x0\n"
+ "mov z17.s, #0x0\n"
+ "whilelt p1.b, x19, x12\n"
+ "mov z18.s, #0x0\n"
+ "mov z19.s, #0x0\n"
+ "4:" // Height 1: setup done
+ "mov x28, #0x0\n"
+ "5:" // Height 1: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w27, [x20, x28, LSL #0x2]\n"
+ "tbz %x[flags], #3, 6f\n"
+ "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x26, [x20, #0x0]\n"
+ "cbnz x28, 7f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x26, x26, x19\n"
+ "b 7f\n"
+ "6:" // Height 1: setup direct input
+ "mov x26, %x[input_ptr]\n"
+ "7:" // Height 1: input setup done
+ "cmp x27, #0x10\n"
+ "ble 10f\n"
+ "8:" // Height 1: Multiply loop: Main loop head
+ "ld1b { z4.b }, p2/Z, [x11]\n"
+ "whilelt p0.b, XZR, x27\n"
+ "ld1b { z5.b }, p2/Z, [x11, #1, MUL VL]\n"
+ "ld1rqb { z0.b }, p0/Z, [x26]\n"
+ "sdot z16.s, z4.b, z0.b[0]\n"
+ "ld1b { z6.b }, p2/Z, [x11, #2, MUL VL]\n"
+ "add x26, x26, #0x10\n"
+ "sdot z17.s, z5.b, z0.b[0]\n"
+ "ld1b { z7.b }, p2/Z, [x11, #3, MUL VL]\n"
+ "ld1b { z8.b }, p2/Z, [x11, #4, MUL VL]\n"
+ "sdot z18.s, z6.b, z0.b[0]\n"
+ "ld1b { z9.b }, p2/Z, [x11, #5, MUL VL]\n"
+ "ld1b { z10.b }, p2/Z, [x11, #6, MUL VL]\n"
+ "sdot z19.s, z7.b, z0.b[0]\n"
+ "sdot z16.s, z8.b, z0.b[1]\n"
+ "ld1b { z4.b }, p2/Z, [x11, #7, MUL VL]\n"
+ "addvl x11, x11, #16\n"
+ "sdot z17.s, z9.b, z0.b[1]\n"
+ "ld1b { z5.b }, p2/Z, [x11, #-8, MUL VL]\n"
+ "ld1b { z6.b }, p2/Z, [x11, #-7, MUL VL]\n"
+ "sdot z18.s, z10.b, z0.b[1]\n"
+ "ld1b { z7.b }, p2/Z, [x11, #-6, MUL VL]\n"
+ "sdot z19.s, z4.b, z0.b[1]\n"
+ "ld1b { z8.b }, p2/Z, [x11, #-5, MUL VL]\n"
+ "sdot z16.s, z5.b, z0.b[2]\n"
+ "ld1b { z9.b }, p2/Z, [x11, #-4, MUL VL]\n"
+ "sdot z17.s, z6.b, z0.b[2]\n"
+ "ld1b { z10.b }, p2/Z, [x11, #-3, MUL VL]\n"
+ "sdot z18.s, z7.b, z0.b[2]\n"
+ "ld1b { z4.b }, p2/Z, [x11, #-2, MUL VL]\n"
+ "sdot z19.s, z8.b, z0.b[2]\n"
+ "ld1b { z5.b }, p2/Z, [x11, #-1, MUL VL]\n"
+ "sdot z16.s, z9.b, z0.b[3]\n"
+ "sdot z17.s, z10.b, z0.b[3]\n"
+ "sdot z18.s, z4.b, z0.b[3]\n"
+ "sdot z19.s, z5.b, z0.b[3]\n"
+ "tbnz %x[flags], #31, 9f\n"
+ "sdot z11.s, z0.b, z15.b\n"
+ "9:" // Height 1: Multiply loop: unique 1: skip row sum
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "sub x27, x27, #0x10\n"
+ "cmp x27, #0x10\n"
+ "bgt 8b\n"
+ "10:" // Height 1: Multiply loop: Single iteration only
+ "ld1b { z6.b }, p2/Z, [x11]\n"
+ "whilelt p0.b, XZR, x27\n"
+ "ld1b { z7.b }, p2/Z, [x11, #1, MUL VL]\n"
+ "subs x27, x27, #0x4\n"
+ "ld1rqb { z0.b }, p0/Z, [x26]\n"
+ "sdot z16.s, z6.b, z0.b[0]\n"
+ "ld1b { z8.b }, p2/Z, [x11, #2, MUL VL]\n"
+ "add x26, x26, #0x10\n"
+ "sdot z17.s, z7.b, z0.b[0]\n"
+ "ld1b { z9.b }, p2/Z, [x11, #3, MUL VL]\n"
+ "addvl x11, x11, #4\n"
+ "sdot z18.s, z8.b, z0.b[0]\n"
+ "sdot z19.s, z9.b, z0.b[0]\n"
+ "ble 11f\n"
+ "ld1b { z10.b }, p2/Z, [x11]\n"
+ "sdot z16.s, z10.b, z0.b[1]\n"
+ "ld1b { z4.b }, p2/Z, [x11, #1, MUL VL]\n"
+ "subs x27, x27, #0x4\n"
+ "sdot z17.s, z4.b, z0.b[1]\n"
+ "ld1b { z5.b }, p2/Z, [x11, #2, MUL VL]\n"
+ "ld1b { z6.b }, p2/Z, [x11, #3, MUL VL]\n"
+ "sdot z18.s, z5.b, z0.b[1]\n"
+ "addvl x11, x11, #4\n"
+ "sdot z19.s, z6.b, z0.b[1]\n"
+ "ble 11f\n"
+ "ld1b { z7.b }, p2/Z, [x11]\n"
+ "sdot z16.s, z7.b, z0.b[2]\n"
+ "ld1b { z8.b }, p2/Z, [x11, #1, MUL VL]\n"
+ "subs x27, x27, #0x4\n"
+ "sdot z17.s, z8.b, z0.b[2]\n"
+ "ld1b { z9.b }, p2/Z, [x11, #2, MUL VL]\n"
+ "ld1b { z10.b }, p2/Z, [x11, #3, MUL VL]\n"
+ "sdot z18.s, z9.b, z0.b[2]\n"
+ "addvl x11, x11, #4\n"
+ "sdot z19.s, z10.b, z0.b[2]\n"
+ "ble 11f\n"
+ "ld1b { z4.b }, p2/Z, [x11]\n"
+ "sdot z16.s, z4.b, z0.b[3]\n"
+ "ld1b { z5.b }, p2/Z, [x11, #1, MUL VL]\n"
+ "ld1b { z6.b }, p2/Z, [x11, #2, MUL VL]\n"
+ "sdot z17.s, z5.b, z0.b[3]\n"
+ "ld1b { z7.b }, p2/Z, [x11, #3, MUL VL]\n"
+ "addvl x11, x11, #4\n"
+ "sdot z18.s, z6.b, z0.b[3]\n"
+ "sdot z19.s, z7.b, z0.b[3]\n"
+ "11:" // Height 1: Multiply loop: multiply skip
+ "tbnz %x[flags], #31, 12f\n"
+ "sdot z11.s, z0.b, z15.b\n"
+ "12:" // Height 1: Multiply loop: unique 2: skip row sum
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "add x28, x28, #0x1\n"
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "cmp x28, x19\n"
+ "bne 5b\n"
+ "prfm pstl1keep, [x9, #0x0]\n"
+ "tbnz %x[flags], #31, 13f\n"
+ "add x19, %x[qp], %[b_offset]\n"
+ "ld1rw { z1.s }, p2/Z, [x19]\n"
+ "neg z1.s, p2/M, z1.s\n"
+ "mov x19, #0x4\n"
+ "whilelt p0.s, XZR, x19\n"
+ "saddv d11, p0, z11.s\n"
+ "mov z11.s, z11.s[0]\n"
+ "mul z11.s, p2/M, z11.s, z1.s\n"
+ "13:" // Height 1: skip row sum fixup
+ "add z16.s, z16.s, z11.s\n"
+ "ld1w { z0.s }, p2/Z, [x10]\n"
+ "orr %x[flags], %x[flags], #0x80000000\n"
+ "add z17.s, z17.s, z11.s\n"
+ "ld1w { z1.s }, p2/Z, [x10, #1, MUL VL]\n"
+ "add x20, %x[qp], %[per_layer_right_shift]\n"
+ "add z18.s, z18.s, z11.s\n"
+ "ld1w { z2.s }, p2/Z, [x10, #2, MUL VL]\n"
+ "add x19, %x[qp], %[per_layer_mul]\n"
+ "add z19.s, z19.s, z11.s\n"
+ "ld1w { z3.s }, p2/Z, [x10, #3, MUL VL]\n"
+ "addvl x10, x10, #4\n"
+ "add z16.s, z16.s, z0.s\n"
+ "ld1rw { z0.s }, p2/Z, [x20]\n"
+ "add z17.s, z17.s, z1.s\n"
+ "ld1rw { z4.s }, p2/Z, [x19]\n"
+ "add z18.s, z18.s, z2.s\n"
+ "add z19.s, z19.s, z3.s\n"
+ ".inst 0x04a47610 // sqrdmulh z16.s, z16.s, z4.s\n"
+ ".inst 0x04a47631 // sqrdmulh z17.s, z17.s, z4.s\n"
+ ".inst 0x04a47652 // sqrdmulh z18.s, z18.s, z4.s\n"
+ ".inst 0x04a47673 // sqrdmulh z19.s, z19.s, z4.s\n"
+ "tbz %x[flags], #5, 14f\n"
+ "and z4.d, z16.d, z0.d\n"
+ "asr z4.s, z4.s, #0x1f\n"
+ "and z5.d, z17.d, z0.d\n"
+ "and z6.d, z18.d, z0.d\n"
+ "asr z5.s, z5.s, #0x1f\n"
+ "and z7.d, z19.d, z0.d\n"
+ "asr z6.s, z6.s, #0x1f\n"
+ "sqadd z16.s, z16.s, z4.s\n"
+ "asr z7.s, z7.s, #0x1f\n"
+ "sqadd z17.s, z17.s, z5.s\n"
+ "sqadd z18.s, z18.s, z6.s\n"
+ "sqadd z19.s, z19.s, z7.s\n"
+ "14:" // Height 1: no shift correction
+ ".inst 0x44828810 // srshl z16.s, p2/M, z16.s, z0.s\n"
+ "add x19, %x[qp], %[c_offset]\n"
+ "ld1rw { z4.s }, p2/Z, [x19]\n"
+ ".inst 0x44828811 // srshl z17.s, p2/M, z17.s, z0.s\n"
+ "add x19, %x[qp], %[minval]\n"
+ ".inst 0x44828812 // srshl z18.s, p2/M, z18.s, z0.s\n"
+ "ld1rw { z5.s }, p2/Z, [x19]\n"
+ "add x19, %x[qp], %[maxval]\n"
+ ".inst 0x44828813 // srshl z19.s, p2/M, z19.s, z0.s\n"
+ "ld1rw { z6.s }, p2/Z, [x19]\n"
+ "add z16.s, z16.s, z4.s\n"
+ "add z17.s, z17.s, z4.s\n"
+ "add z18.s, z18.s, z4.s\n"
+ "add z19.s, z19.s, z4.s\n"
+ "smin z16.s, p2/M, z16.s, z6.s\n"
+ "smin z17.s, p2/M, z17.s, z6.s\n"
+ "smin z18.s, p2/M, z18.s, z6.s\n"
+ "smin z19.s, p2/M, z19.s, z6.s\n"
+ "smax z16.s, p2/M, z16.s, z5.s\n"
+ "smax z17.s, p2/M, z17.s, z5.s\n"
+ "smax z18.s, p2/M, z18.s, z5.s\n"
+ "smax z19.s, p2/M, z19.s, z5.s\n"
+ "uzp1 z16.h, z16.h, z17.h\n"
+ "uzp1 z17.h, z18.h, z19.h\n"
+ "uzp1 z16.b, z16.b, z17.b\n"
+ "st1b { z16.b }, p1, [x9]\n"
+ "addvl x9, x9, #1\n"
+ "15:" // Height 1: Writeback done
+ "mov x19, #0x0\n"
+ "incw x19, ALL, MUL #4\n"
+ "subs x12, x12, x19\n"
+ "bgt 3b\n"
+ "b 62f\n"
+ "16:" // Height 2
+ "mov z11.s, #0x0\n"
+ "ldr x12, [%x[args_ptr], %[offsetof_N]]\n"
+ "mov x10, %x[col_bias]\n"
+ "mov z12.s, #0x0\n"
+ "ldr x11, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "bic %x[flags], %x[flags], #0x80000000\n"
+ "mov z13.s, #0x0\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "mov z14.s, #0x0\n"
+ "mov z15.b, #0x1\n"
+ "tbz %x[flags], #2, 17f\n"
+ "ldr x9, [%x[output_ptr], #0x0]\n"
+ "ldr x25, [%x[output_ptr], #0x8]\n"
+ "add x9, x9, x19\n"
+ "add x25, x25, x19\n"
+ "b 18f\n"
+ "17:" // Height 2: setup direct output
+ "mov x9, %x[output_ptr]\n"
+ "add x25, x9, x19\n"
+ "18:" // Height 2: Column loop
+ "mov z16.s, #0x0\n"
+ "mov x19, #0x0\n"
+ "mov z17.s, #0x0\n"
+ "whilelt p1.b, x19, x12\n"
+ "mov z18.s, #0x0\n"
+ "mov z19.s, #0x0\n"
+ "mov z20.s, #0x0\n"
+ "mov z21.s, #0x0\n"
+ "mov z22.s, #0x0\n"
+ "mov z23.s, #0x0\n"
+ "19:" // Height 2: setup done
+ "mov x28, #0x0\n"
+ "20:" // Height 2: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w27, [x20, x28, LSL #0x2]\n"
+ "tbz %x[flags], #3, 21f\n"
+ "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x26, [x20, #0x0]\n"
+ "ldr x24, [x20, #0x8]\n"
+ "cbnz x28, 22f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x26, x26, x19\n"
+ "add x24, x24, x19\n"
+ "b 22f\n"
+ "21:" // Height 2: setup direct input
+ "mov x26, %x[input_ptr]\n"
+ "add x24, x26, x19\n"
+ "22:" // Height 2: input setup done
+ "cmp x27, #0x10\n"
+ "ble 25f\n"
+ "23:" // Height 2: Multiply loop: Main loop head
+ "ld1b { z4.b }, p2/Z, [x11]\n"
+ "whilelt p0.b, XZR, x27\n"
+ "ld1b { z5.b }, p2/Z, [x11, #1, MUL VL]\n"
+ "ld1rqb { z0.b }, p0/Z, [x26]\n"
+ "sdot z16.s, z4.b, z0.b[0]\n"
+ "ld1rqb { z1.b }, p0/Z, [x24]\n"
+ "add x26, x26, #0x10\n"
+ "sdot z17.s, z5.b, z0.b[0]\n"
+ "ld1b { z6.b }, p2/Z, [x11, #2, MUL VL]\n"
+ "add x24, x24, #0x10\n"
+ "sdot z20.s, z4.b, z1.b[0]\n"
+ "ld1b { z7.b }, p2/Z, [x11, #3, MUL VL]\n"
+ "sdot z21.s, z5.b, z1.b[0]\n"
+ "ld1b { z8.b }, p2/Z, [x11, #4, MUL VL]\n"
+ "sdot z18.s, z6.b, z0.b[0]\n"
+ "ld1b { z9.b }, p2/Z, [x11, #5, MUL VL]\n"
+ "sdot z22.s, z6.b, z1.b[0]\n"
+ "ld1b { z10.b }, p2/Z, [x11, #6, MUL VL]\n"
+ "sdot z19.s, z7.b, z0.b[0]\n"
+ "ld1b { z4.b }, p2/Z, [x11, #7, MUL VL]\n"
+ "addvl x11, x11, #16\n"
+ "sdot z23.s, z7.b, z1.b[0]\n"
+ "ld1b { z5.b }, p2/Z, [x11, #-8, MUL VL]\n"
+ "sdot z16.s, z8.b, z0.b[1]\n"
+ "ld1b { z6.b }, p2/Z, [x11, #-7, MUL VL]\n"
+ "sdot z20.s, z8.b, z1.b[1]\n"
+ "ld1b { z7.b }, p2/Z, [x11, #-6, MUL VL]\n"
+ "sdot z17.s, z9.b, z0.b[1]\n"
+ "ld1b { z8.b }, p2/Z, [x11, #-5, MUL VL]\n"
+ "sdot z21.s, z9.b, z1.b[1]\n"
+ "ld1b { z9.b }, p2/Z, [x11, #-4, MUL VL]\n"
+ "sdot z18.s, z10.b, z0.b[1]\n"
+ "sdot z22.s, z10.b, z1.b[1]\n"
+ "ld1b { z10.b }, p2/Z, [x11, #-3, MUL VL]\n"
+ "sdot z19.s, z4.b, z0.b[1]\n"
+ "sdot z23.s, z4.b, z1.b[1]\n"
+ "ld1b { z4.b }, p2/Z, [x11, #-2, MUL VL]\n"
+ "sdot z16.s, z5.b, z0.b[2]\n"
+ "sdot z20.s, z5.b, z1.b[2]\n"
+ "ld1b { z5.b }, p2/Z, [x11, #-1, MUL VL]\n"
+ "sdot z17.s, z6.b, z0.b[2]\n"
+ "sdot z21.s, z6.b, z1.b[2]\n"
+ "sdot z18.s, z7.b, z0.b[2]\n"
+ "sdot z22.s, z7.b, z1.b[2]\n"
+ "sdot z19.s, z8.b, z0.b[2]\n"
+ "sdot z23.s, z8.b, z1.b[2]\n"
+ "sdot z16.s, z9.b, z0.b[3]\n"
+ "sdot z20.s, z9.b, z1.b[3]\n"
+ "sdot z17.s, z10.b, z0.b[3]\n"
+ "sdot z21.s, z10.b, z1.b[3]\n"
+ "sdot z18.s, z4.b, z0.b[3]\n"
+ "sdot z22.s, z4.b, z1.b[3]\n"
+ "sdot z19.s, z5.b, z0.b[3]\n"
+ "sdot z23.s, z5.b, z1.b[3]\n"
+ "tbnz %x[flags], #31, 24f\n"
+ "sdot z11.s, z0.b, z15.b\n"
+ "sdot z12.s, z1.b, z15.b\n"
+ "24:" // Height 2: Multiply loop: unique 3: skip row sum
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "sub x27, x27, #0x10\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "cmp x27, #0x10\n"
+ "bgt 23b\n"
+ "25:" // Height 2: Multiply loop: Single iteration only
+ "ld1b { z6.b }, p2/Z, [x11]\n"
+ "whilelt p0.b, XZR, x27\n"
+ "ld1b { z7.b }, p2/Z, [x11, #1, MUL VL]\n"
+ "subs x27, x27, #0x4\n"
+ "ld1rqb { z0.b }, p0/Z, [x26]\n"
+ "sdot z16.s, z6.b, z0.b[0]\n"
+ "ld1rqb { z1.b }, p0/Z, [x24]\n"
+ "add x26, x26, #0x10\n"
+ "sdot z17.s, z7.b, z0.b[0]\n"
+ "ld1b { z8.b }, p2/Z, [x11, #2, MUL VL]\n"
+ "add x24, x24, #0x10\n"
+ "sdot z20.s, z6.b, z1.b[0]\n"
+ "ld1b { z9.b }, p2/Z, [x11, #3, MUL VL]\n"
+ "addvl x11, x11, #4\n"
+ "sdot z21.s, z7.b, z1.b[0]\n"
+ "sdot z18.s, z8.b, z0.b[0]\n"
+ "sdot z22.s, z8.b, z1.b[0]\n"
+ "sdot z19.s, z9.b, z0.b[0]\n"
+ "sdot z23.s, z9.b, z1.b[0]\n"
+ "ble 26f\n"
+ "ld1b { z10.b }, p2/Z, [x11]\n"
+ "sdot z16.s, z10.b, z0.b[1]\n"
+ "ld1b { z4.b }, p2/Z, [x11, #1, MUL VL]\n"
+ "subs x27, x27, #0x4\n"
+ "sdot z20.s, z10.b, z1.b[1]\n"
+ "ld1b { z5.b }, p2/Z, [x11, #2, MUL VL]\n"
+ "ld1b { z6.b }, p2/Z, [x11, #3, MUL VL]\n"
+ "sdot z17.s, z4.b, z0.b[1]\n"
+ "addvl x11, x11, #4\n"
+ "sdot z21.s, z4.b, z1.b[1]\n"
+ "sdot z18.s, z5.b, z0.b[1]\n"
+ "sdot z22.s, z5.b, z1.b[1]\n"
+ "sdot z19.s, z6.b, z0.b[1]\n"
+ "sdot z23.s, z6.b, z1.b[1]\n"
+ "ble 26f\n"
+ "ld1b { z7.b }, p2/Z, [x11]\n"
+ "sdot z16.s, z7.b, z0.b[2]\n"
+ "ld1b { z8.b }, p2/Z, [x11, #1, MUL VL]\n"
+ "subs x27, x27, #0x4\n"
+ "sdot z20.s, z7.b, z1.b[2]\n"
+ "ld1b { z9.b }, p2/Z, [x11, #2, MUL VL]\n"
+ "ld1b { z10.b }, p2/Z, [x11, #3, MUL VL]\n"
+ "sdot z17.s, z8.b, z0.b[2]\n"
+ "addvl x11, x11, #4\n"
+ "sdot z21.s, z8.b, z1.b[2]\n"
+ "sdot z18.s, z9.b, z0.b[2]\n"
+ "sdot z22.s, z9.b, z1.b[2]\n"
+ "sdot z19.s, z10.b, z0.b[2]\n"
+ "sdot z23.s, z10.b, z1.b[2]\n"
+ "ble 26f\n"
+ "ld1b { z4.b }, p2/Z, [x11]\n"
+ "sdot z16.s, z4.b, z0.b[3]\n"
+ "ld1b { z5.b }, p2/Z, [x11, #1, MUL VL]\n"
+ "sdot z20.s, z4.b, z1.b[3]\n"
+ "ld1b { z6.b }, p2/Z, [x11, #2, MUL VL]\n"
+ "ld1b { z7.b }, p2/Z, [x11, #3, MUL VL]\n"
+ "sdot z17.s, z5.b, z0.b[3]\n"
+ "addvl x11, x11, #4\n"
+ "sdot z21.s, z5.b, z1.b[3]\n"
+ "sdot z18.s, z6.b, z0.b[3]\n"
+ "sdot z22.s, z6.b, z1.b[3]\n"
+ "sdot z19.s, z7.b, z0.b[3]\n"
+ "sdot z23.s, z7.b, z1.b[3]\n"
+ "26:" // Height 2: Multiply loop: multiply skip
+ "tbnz %x[flags], #31, 27f\n"
+ "sdot z11.s, z0.b, z15.b\n"
+ "sdot z12.s, z1.b, z15.b\n"
+ "27:" // Height 2: Multiply loop: unique 4: skip row sum
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "add x28, x28, #0x1\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "cmp x28, x19\n"
+ "bne 20b\n"
+ "prfm pstl1keep, [x9, #0x0]\n"
+ "prfm pstl1keep, [x25, #0x0]\n"
+ "tbnz %x[flags], #31, 28f\n"
+ "add x19, %x[qp], %[b_offset]\n"
+ "ld1rw { z2.s }, p2/Z, [x19]\n"
+ "neg z2.s, p2/M, z2.s\n"
+ "mov x20, #0x4\n"
+ "mov x19, #0x4\n"
+ "whilelt p0.s, XZR, x20\n"
+ "saddv d11, p0, z11.s\n"
+ "whilelt p0.s, XZR, x19\n"
+ "saddv d12, p0, z12.s\n"
+ "mov z11.s, z11.s[0]\n"
+ "mov z12.s, z12.s[0]\n"
+ "mul z11.s, p2/M, z11.s, z2.s\n"
+ "mul z12.s, p2/M, z12.s, z2.s\n"
+ "28:" // Height 2: skip row sum fixup
+ "add z16.s, z16.s, z11.s\n"
+ "ld1w { z0.s }, p2/Z, [x10]\n"
+ "orr %x[flags], %x[flags], #0x80000000\n"
+ "add z17.s, z17.s, z11.s\n"
+ "ld1w { z1.s }, p2/Z, [x10, #1, MUL VL]\n"
+ "add x20, %x[qp], %[per_layer_right_shift]\n"
+ "add z18.s, z18.s, z11.s\n"
+ "ld1w { z2.s }, p2/Z, [x10, #2, MUL VL]\n"
+ "add x19, %x[qp], %[per_layer_mul]\n"
+ "add z19.s, z19.s, z11.s\n"
+ "ld1w { z3.s }, p2/Z, [x10, #3, MUL VL]\n"
+ "addvl x10, x10, #4\n"
+ "add z20.s, z20.s, z12.s\n"
+ "ld1rw { z4.s }, p2/Z, [x19]\n"
+ "add z21.s, z21.s, z12.s\n"
+ "add z22.s, z22.s, z12.s\n"
+ "add z23.s, z23.s, z12.s\n"
+ "add z16.s, z16.s, z0.s\n"
+ "add z17.s, z17.s, z1.s\n"
+ "add z18.s, z18.s, z2.s\n"
+ "add z19.s, z19.s, z3.s\n"
+ "add z20.s, z20.s, z0.s\n"
+ "ld1rw { z0.s }, p2/Z, [x20]\n"
+ "add z21.s, z21.s, z1.s\n"
+ "add z22.s, z22.s, z2.s\n"
+ "add z23.s, z23.s, z3.s\n"
+ ".inst 0x04a47610 // sqrdmulh z16.s, z16.s, z4.s\n"
+ ".inst 0x04a47631 // sqrdmulh z17.s, z17.s, z4.s\n"
+ ".inst 0x04a47652 // sqrdmulh z18.s, z18.s, z4.s\n"
+ ".inst 0x04a47673 // sqrdmulh z19.s, z19.s, z4.s\n"
+ ".inst 0x04a47694 // sqrdmulh z20.s, z20.s, z4.s\n"
+ ".inst 0x04a476b5 // sqrdmulh z21.s, z21.s, z4.s\n"
+ ".inst 0x04a476d6 // sqrdmulh z22.s, z22.s, z4.s\n"
+ ".inst 0x04a476f7 // sqrdmulh z23.s, z23.s, z4.s\n"
+ "tbz %x[flags], #5, 29f\n"
+ "and z4.d, z16.d, z0.d\n"
+ "asr z4.s, z4.s, #0x1f\n"
+ "and z5.d, z17.d, z0.d\n"
+ "and z6.d, z18.d, z0.d\n"
+ "asr z5.s, z5.s, #0x1f\n"
+ "and z7.d, z19.d, z0.d\n"
+ "and z8.d, z20.d, z0.d\n"
+ "asr z6.s, z6.s, #0x1f\n"
+ "and z9.d, z21.d, z0.d\n"
+ "asr z7.s, z7.s, #0x1f\n"
+ "sqadd z16.s, z16.s, z4.s\n"
+ "and z10.d, z22.d, z0.d\n"
+ "asr z8.s, z8.s, #0x1f\n"
+ "and z4.d, z23.d, z0.d\n"
+ "asr z9.s, z9.s, #0x1f\n"
+ "sqadd z17.s, z17.s, z5.s\n"
+ "asr z10.s, z10.s, #0x1f\n"
+ "sqadd z18.s, z18.s, z6.s\n"
+ "asr z4.s, z4.s, #0x1f\n"
+ "sqadd z19.s, z19.s, z7.s\n"
+ "sqadd z20.s, z20.s, z8.s\n"
+ "sqadd z21.s, z21.s, z9.s\n"
+ "sqadd z22.s, z22.s, z10.s\n"
+ "sqadd z23.s, z23.s, z4.s\n"
+ "29:" // Height 2: no shift correction
+ ".inst 0x44828810 // srshl z16.s, p2/M, z16.s, z0.s\n"
+ "add x19, %x[qp], %[c_offset]\n"
+ "ld1rw { z4.s }, p2/Z, [x19]\n"
+ ".inst 0x44828811 // srshl z17.s, p2/M, z17.s, z0.s\n"
+ "add x19, %x[qp], %[minval]\n"
+ ".inst 0x44828812 // srshl z18.s, p2/M, z18.s, z0.s\n"
+ "ld1rw { z5.s }, p2/Z, [x19]\n"
+ "add x19, %x[qp], %[maxval]\n"
+ ".inst 0x44828813 // srshl z19.s, p2/M, z19.s, z0.s\n"
+ "ld1rw { z6.s }, p2/Z, [x19]\n"
+ ".inst 0x44828814 // srshl z20.s, p2/M, z20.s, z0.s\n"
+ "add z16.s, z16.s, z4.s\n"
+ "add z17.s, z17.s, z4.s\n"
+ "add z18.s, z18.s, z4.s\n"
+ "add z19.s, z19.s, z4.s\n"
+ "add z20.s, z20.s, z4.s\n"
+ "smin z16.s, p2/M, z16.s, z6.s\n"
+ "smin z17.s, p2/M, z17.s, z6.s\n"
+ "smin z18.s, p2/M, z18.s, z6.s\n"
+ "smin z19.s, p2/M, z19.s, z6.s\n"
+ "smax z16.s, p2/M, z16.s, z5.s\n"
+ "smax z17.s, p2/M, z17.s, z5.s\n"
+ "smax z18.s, p2/M, z18.s, z5.s\n"
+ "smax z19.s, p2/M, z19.s, z5.s\n"
+ "smin z20.s, p2/M, z20.s, z6.s\n"
+ "uzp1 z16.h, z16.h, z17.h\n"
+ ".inst 0x44828815 // srshl z21.s, p2/M, z21.s, z0.s\n"
+ "uzp1 z17.h, z18.h, z19.h\n"
+ "smax z20.s, p2/M, z20.s, z5.s\n"
+ "uzp1 z16.b, z16.b, z17.b\n"
+ "st1b { z16.b }, p1, [x9]\n"
+ "add z21.s, z21.s, z4.s\n"
+ "addvl x9, x9, #1\n"
+ ".inst 0x44828816 // srshl z22.s, p2/M, z22.s, z0.s\n"
+ ".inst 0x44828817 // srshl z23.s, p2/M, z23.s, z0.s\n"
+ "smin z21.s, p2/M, z21.s, z6.s\n"
+ "add z22.s, z22.s, z4.s\n"
+ "add z23.s, z23.s, z4.s\n"
+ "smax z21.s, p2/M, z21.s, z5.s\n"
+ "smin z22.s, p2/M, z22.s, z6.s\n"
+ "smin z23.s, p2/M, z23.s, z6.s\n"
+ "uzp1 z20.h, z20.h, z21.h\n"
+ "smax z22.s, p2/M, z22.s, z5.s\n"
+ "smax z23.s, p2/M, z23.s, z5.s\n"
+ "uzp1 z21.h, z22.h, z23.h\n"
+ "uzp1 z20.b, z20.b, z21.b\n"
+ "st1b { z20.b }, p1, [x25]\n"
+ "addvl x25, x25, #1\n"
+ "30:" // Height 2: Writeback done
+ "mov x19, #0x0\n"
+ "incw x19, ALL, MUL #4\n"
+ "subs x12, x12, x19\n"
+ "bgt 18b\n"
+ "b 62f\n"
+ "31:" // Height 3
+ "mov z11.s, #0x0\n"
+ "ldr x12, [%x[args_ptr], %[offsetof_N]]\n"
+ "mov x10, %x[col_bias]\n"
+ "mov z12.s, #0x0\n"
+ "ldr x11, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "bic %x[flags], %x[flags], #0x80000000\n"
+ "mov z13.s, #0x0\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "mov z14.s, #0x0\n"
+ "mov z15.b, #0x1\n"
+ "tbz %x[flags], #2, 32f\n"
+ "ldr x9, [%x[output_ptr], #0x0]\n"
+ "ldr x25, [%x[output_ptr], #0x8]\n"
+ "add x9, x9, x19\n"
+ "ldr x23, [%x[output_ptr], #0x10]\n"
+ "add x25, x25, x19\n"
+ "add x23, x23, x19\n"
+ "b 33f\n"
+ "32:" // Height 3: setup direct output
+ "mov x9, %x[output_ptr]\n"
+ "add x25, x9, x19\n"
+ "add x23, x25, x19\n"
+ "33:" // Height 3: Column loop
+ "mov z16.s, #0x0\n"
+ "mov x19, #0x0\n"
+ "mov z17.s, #0x0\n"
+ "whilelt p1.b, x19, x12\n"
+ "mov z18.s, #0x0\n"
+ "mov z19.s, #0x0\n"
+ "mov z20.s, #0x0\n"
+ "mov z21.s, #0x0\n"
+ "mov z22.s, #0x0\n"
+ "mov z23.s, #0x0\n"
+ "mov z24.s, #0x0\n"
+ "mov z25.s, #0x0\n"
+ "mov z26.s, #0x0\n"
+ "mov z27.s, #0x0\n"
+ "34:" // Height 3: setup done
+ "mov x28, #0x0\n"
+ "35:" // Height 3: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w27, [x20, x28, LSL #0x2]\n"
+ "tbz %x[flags], #3, 36f\n"
+ "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x26, [x20, #0x0]\n"
+ "ldr x24, [x20, #0x8]\n"
+ "ldr x22, [x20, #0x10]\n"
+ "cbnz x28, 37f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x26, x26, x19\n"
+ "add x24, x24, x19\n"
+ "add x22, x22, x19\n"
+ "b 37f\n"
+ "36:" // Height 3: setup direct input
+ "mov x26, %x[input_ptr]\n"
+ "add x24, x26, x19\n"
+ "add x22, x24, x19\n"
+ "37:" // Height 3: input setup done
+ "cmp x27, #0x10\n"
+ "ble 40f\n"
+ "38:" // Height 3: Multiply loop: Main loop head
+ "ld1b { z4.b }, p2/Z, [x11]\n"
+ "whilelt p0.b, XZR, x27\n"
+ "ld1b { z5.b }, p2/Z, [x11, #1, MUL VL]\n"
+ "ld1rqb { z0.b }, p0/Z, [x26]\n"
+ "sdot z16.s, z4.b, z0.b[0]\n"
+ "ld1rqb { z1.b }, p0/Z, [x24]\n"
+ "add x26, x26, #0x10\n"
+ "sdot z17.s, z5.b, z0.b[0]\n"
+ "ld1rqb { z2.b }, p0/Z, [x22]\n"
+ "add x24, x24, #0x10\n"
+ "sdot z20.s, z4.b, z1.b[0]\n"
+ "ld1b { z6.b }, p2/Z, [x11, #2, MUL VL]\n"
+ "add x22, x22, #0x10\n"
+ "sdot z24.s, z4.b, z2.b[0]\n"
+ "ld1b { z7.b }, p2/Z, [x11, #3, MUL VL]\n"
+ "sdot z21.s, z5.b, z1.b[0]\n"
+ "ld1b { z8.b }, p2/Z, [x11, #4, MUL VL]\n"
+ "sdot z25.s, z5.b, z2.b[0]\n"
+ "ld1b { z9.b }, p2/Z, [x11, #5, MUL VL]\n"
+ "sdot z18.s, z6.b, z0.b[0]\n"
+ "ld1b { z10.b }, p2/Z, [x11, #6, MUL VL]\n"
+ "sdot z22.s, z6.b, z1.b[0]\n"
+ "ld1b { z4.b }, p2/Z, [x11, #7, MUL VL]\n"
+ "addvl x11, x11, #16\n"
+ "sdot z26.s, z6.b, z2.b[0]\n"
+ "ld1b { z5.b }, p2/Z, [x11, #-8, MUL VL]\n"
+ "sdot z19.s, z7.b, z0.b[0]\n"
+ "ld1b { z6.b }, p2/Z, [x11, #-7, MUL VL]\n"
+ "sdot z23.s, z7.b, z1.b[0]\n"
+ "sdot z27.s, z7.b, z2.b[0]\n"
+ "ld1b { z7.b }, p2/Z, [x11, #-6, MUL VL]\n"
+ "sdot z16.s, z8.b, z0.b[1]\n"
+ "sdot z20.s, z8.b, z1.b[1]\n"
+ "sdot z24.s, z8.b, z2.b[1]\n"
+ "ld1b { z8.b }, p2/Z, [x11, #-5, MUL VL]\n"
+ "sdot z17.s, z9.b, z0.b[1]\n"
+ "sdot z21.s, z9.b, z1.b[1]\n"
+ "sdot z25.s, z9.b, z2.b[1]\n"
+ "ld1b { z9.b }, p2/Z, [x11, #-4, MUL VL]\n"
+ "sdot z18.s, z10.b, z0.b[1]\n"
+ "sdot z22.s, z10.b, z1.b[1]\n"
+ "sdot z26.s, z10.b, z2.b[1]\n"
+ "ld1b { z10.b }, p2/Z, [x11, #-3, MUL VL]\n"
+ "sdot z19.s, z4.b, z0.b[1]\n"
+ "sdot z23.s, z4.b, z1.b[1]\n"
+ "sdot z27.s, z4.b, z2.b[1]\n"
+ "ld1b { z4.b }, p2/Z, [x11, #-2, MUL VL]\n"
+ "sdot z16.s, z5.b, z0.b[2]\n"
+ "sdot z20.s, z5.b, z1.b[2]\n"
+ "sdot z24.s, z5.b, z2.b[2]\n"
+ "ld1b { z5.b }, p2/Z, [x11, #-1, MUL VL]\n"
+ "sdot z17.s, z6.b, z0.b[2]\n"
+ "sdot z21.s, z6.b, z1.b[2]\n"
+ "sdot z25.s, z6.b, z2.b[2]\n"
+ "sdot z18.s, z7.b, z0.b[2]\n"
+ "sdot z22.s, z7.b, z1.b[2]\n"
+ "sdot z26.s, z7.b, z2.b[2]\n"
+ "sdot z19.s, z8.b, z0.b[2]\n"
+ "sdot z23.s, z8.b, z1.b[2]\n"
+ "sdot z27.s, z8.b, z2.b[2]\n"
+ "sdot z16.s, z9.b, z0.b[3]\n"
+ "sdot z20.s, z9.b, z1.b[3]\n"
+ "sdot z24.s, z9.b, z2.b[3]\n"
+ "sdot z17.s, z10.b, z0.b[3]\n"
+ "sdot z21.s, z10.b, z1.b[3]\n"
+ "sdot z25.s, z10.b, z2.b[3]\n"
+ "sdot z18.s, z4.b, z0.b[3]\n"
+ "sdot z22.s, z4.b, z1.b[3]\n"
+ "sdot z26.s, z4.b, z2.b[3]\n"
+ "sdot z19.s, z5.b, z0.b[3]\n"
+ "sdot z23.s, z5.b, z1.b[3]\n"
+ "sdot z27.s, z5.b, z2.b[3]\n"
+ "tbnz %x[flags], #31, 39f\n"
+ "sdot z11.s, z0.b, z15.b\n"
+ "sdot z12.s, z1.b, z15.b\n"
+ "sdot z13.s, z2.b, z15.b\n"
+ "39:" // Height 3: Multiply loop: unique 5: skip row sum
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "sub x27, x27, #0x10\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "cmp x27, #0x10\n"
+ "prfm pldl1keep, [x22, #0x80]\n"
+ "bgt 38b\n"
+ "40:" // Height 3: Multiply loop: Single iteration only
+ "ld1b { z6.b }, p2/Z, [x11]\n"
+ "whilelt p0.b, XZR, x27\n"
+ "ld1b { z7.b }, p2/Z, [x11, #1, MUL VL]\n"
+ "subs x27, x27, #0x4\n"
+ "ld1rqb { z0.b }, p0/Z, [x26]\n"
+ "sdot z16.s, z6.b, z0.b[0]\n"
+ "ld1rqb { z1.b }, p0/Z, [x24]\n"
+ "add x26, x26, #0x10\n"
+ "sdot z17.s, z7.b, z0.b[0]\n"
+ "ld1rqb { z2.b }, p0/Z, [x22]\n"
+ "add x24, x24, #0x10\n"
+ "sdot z20.s, z6.b, z1.b[0]\n"
+ "ld1b { z8.b }, p2/Z, [x11, #2, MUL VL]\n"
+ "add x22, x22, #0x10\n"
+ "sdot z24.s, z6.b, z2.b[0]\n"
+ "ld1b { z9.b }, p2/Z, [x11, #3, MUL VL]\n"
+ "addvl x11, x11, #4\n"
+ "sdot z21.s, z7.b, z1.b[0]\n"
+ "sdot z25.s, z7.b, z2.b[0]\n"
+ "sdot z18.s, z8.b, z0.b[0]\n"
+ "sdot z22.s, z8.b, z1.b[0]\n"
+ "sdot z26.s, z8.b, z2.b[0]\n"
+ "sdot z19.s, z9.b, z0.b[0]\n"
+ "sdot z23.s, z9.b, z1.b[0]\n"
+ "sdot z27.s, z9.b, z2.b[0]\n"
+ "ble 41f\n"
+ "ld1b { z10.b }, p2/Z, [x11]\n"
+ "sdot z16.s, z10.b, z0.b[1]\n"
+ "ld1b { z4.b }, p2/Z, [x11, #1, MUL VL]\n"
+ "subs x27, x27, #0x4\n"
+ "sdot z20.s, z10.b, z1.b[1]\n"
+ "ld1b { z5.b }, p2/Z, [x11, #2, MUL VL]\n"
+ "sdot z24.s, z10.b, z2.b[1]\n"
+ "ld1b { z6.b }, p2/Z, [x11, #3, MUL VL]\n"
+ "addvl x11, x11, #4\n"
+ "sdot z17.s, z4.b, z0.b[1]\n"
+ "sdot z21.s, z4.b, z1.b[1]\n"
+ "sdot z25.s, z4.b, z2.b[1]\n"
+ "sdot z18.s, z5.b, z0.b[1]\n"
+ "sdot z22.s, z5.b, z1.b[1]\n"
+ "sdot z26.s, z5.b, z2.b[1]\n"
+ "sdot z19.s, z6.b, z0.b[1]\n"
+ "sdot z23.s, z6.b, z1.b[1]\n"
+ "sdot z27.s, z6.b, z2.b[1]\n"
+ "ble 41f\n"
+ "ld1b { z7.b }, p2/Z, [x11]\n"
+ "sdot z16.s, z7.b, z0.b[2]\n"
+ "ld1b { z8.b }, p2/Z, [x11, #1, MUL VL]\n"
+ "subs x27, x27, #0x4\n"
+ "sdot z20.s, z7.b, z1.b[2]\n"
+ "ld1b { z9.b }, p2/Z, [x11, #2, MUL VL]\n"
+ "sdot z24.s, z7.b, z2.b[2]\n"
+ "ld1b { z10.b }, p2/Z, [x11, #3, MUL VL]\n"
+ "addvl x11, x11, #4\n"
+ "sdot z17.s, z8.b, z0.b[2]\n"
+ "sdot z21.s, z8.b, z1.b[2]\n"
+ "sdot z25.s, z8.b, z2.b[2]\n"
+ "sdot z18.s, z9.b, z0.b[2]\n"
+ "sdot z22.s, z9.b, z1.b[2]\n"
+ "sdot z26.s, z9.b, z2.b[2]\n"
+ "sdot z19.s, z10.b, z0.b[2]\n"
+ "sdot z23.s, z10.b, z1.b[2]\n"
+ "sdot z27.s, z10.b, z2.b[2]\n"
+ "ble 41f\n"
+ "ld1b { z4.b }, p2/Z, [x11]\n"
+ "sdot z16.s, z4.b, z0.b[3]\n"
+ "ld1b { z5.b }, p2/Z, [x11, #1, MUL VL]\n"
+ "sdot z20.s, z4.b, z1.b[3]\n"
+ "ld1b { z6.b }, p2/Z, [x11, #2, MUL VL]\n"
+ "sdot z24.s, z4.b, z2.b[3]\n"
+ "ld1b { z7.b }, p2/Z, [x11, #3, MUL VL]\n"
+ "addvl x11, x11, #4\n"
+ "sdot z17.s, z5.b, z0.b[3]\n"
+ "sdot z21.s, z5.b, z1.b[3]\n"
+ "sdot z25.s, z5.b, z2.b[3]\n"
+ "sdot z18.s, z6.b, z0.b[3]\n"
+ "sdot z22.s, z6.b, z1.b[3]\n"
+ "sdot z26.s, z6.b, z2.b[3]\n"
+ "sdot z19.s, z7.b, z0.b[3]\n"
+ "sdot z23.s, z7.b, z1.b[3]\n"
+ "sdot z27.s, z7.b, z2.b[3]\n"
+ "41:" // Height 3: Multiply loop: multiply skip
+ "tbnz %x[flags], #31, 42f\n"
+ "sdot z11.s, z0.b, z15.b\n"
+ "sdot z12.s, z1.b, z15.b\n"
+ "sdot z13.s, z2.b, z15.b\n"
+ "42:" // Height 3: Multiply loop: unique 6: skip row sum
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "add x28, x28, #0x1\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "prfm pldl1keep, [x22, #0x80]\n"
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "cmp x28, x19\n"
+ "bne 35b\n"
+ "prfm pstl1keep, [x9, #0x0]\n"
+ "prfm pstl1keep, [x25, #0x0]\n"
+ "prfm pstl1keep, [x23, #0x0]\n"
+ "tbnz %x[flags], #31, 43f\n"
+ "add x19, %x[qp], %[b_offset]\n"
+ "ld1rw { z3.s }, p2/Z, [x19]\n"
+ "neg z3.s, p2/M, z3.s\n"
+ "mov x20, #0x4\n"
+ "mov x19, #0x4\n"
+ "whilelt p0.s, XZR, x20\n"
+ "saddv d11, p0, z11.s\n"
+ "whilelt p0.s, XZR, x19\n"
+ "saddv d12, p0, z12.s\n"
+ "mov x19, #0x4\n"
+ "mov z11.s, z11.s[0]\n"
+ "whilelt p0.s, XZR, x19\n"
+ "mov z12.s, z12.s[0]\n"
+ "saddv d13, p0, z13.s\n"
+ "mul z11.s, p2/M, z11.s, z3.s\n"
+ "mul z12.s, p2/M, z12.s, z3.s\n"
+ "mov z13.s, z13.s[0]\n"
+ "mul z13.s, p2/M, z13.s, z3.s\n"
+ "43:" // Height 3: skip row sum fixup
+ "add z16.s, z16.s, z11.s\n"
+ "ld1w { z0.s }, p2/Z, [x10]\n"
+ "orr %x[flags], %x[flags], #0x80000000\n"
+ "add z17.s, z17.s, z11.s\n"
+ "ld1w { z1.s }, p2/Z, [x10, #1, MUL VL]\n"
+ "add x20, %x[qp], %[per_layer_right_shift]\n"
+ "add z18.s, z18.s, z11.s\n"
+ "ld1w { z2.s }, p2/Z, [x10, #2, MUL VL]\n"
+ "add x19, %x[qp], %[per_layer_mul]\n"
+ "add z19.s, z19.s, z11.s\n"
+ "ld1w { z3.s }, p2/Z, [x10, #3, MUL VL]\n"
+ "addvl x10, x10, #4\n"
+ "add z20.s, z20.s, z12.s\n"
+ "ld1rw { z4.s }, p2/Z, [x19]\n"
+ "add z21.s, z21.s, z12.s\n"
+ "add z22.s, z22.s, z12.s\n"
+ "add z23.s, z23.s, z12.s\n"
+ "add z24.s, z24.s, z13.s\n"
+ "add z25.s, z25.s, z13.s\n"
+ "add z26.s, z26.s, z13.s\n"
+ "add z27.s, z27.s, z13.s\n"
+ "add z16.s, z16.s, z0.s\n"
+ "add z17.s, z17.s, z1.s\n"
+ "add z18.s, z18.s, z2.s\n"
+ "add z19.s, z19.s, z3.s\n"
+ "add z20.s, z20.s, z0.s\n"
+ "add z21.s, z21.s, z1.s\n"
+ "add z22.s, z22.s, z2.s\n"
+ "add z23.s, z23.s, z3.s\n"
+ "add z24.s, z24.s, z0.s\n"
+ "ld1rw { z0.s }, p2/Z, [x20]\n"
+ "add z25.s, z25.s, z1.s\n"
+ "add z26.s, z26.s, z2.s\n"
+ "add z27.s, z27.s, z3.s\n"
+ ".inst 0x04a47610 // sqrdmulh z16.s, z16.s, z4.s\n"
+ ".inst 0x04a47631 // sqrdmulh z17.s, z17.s, z4.s\n"
+ ".inst 0x04a47652 // sqrdmulh z18.s, z18.s, z4.s\n"
+ ".inst 0x04a47673 // sqrdmulh z19.s, z19.s, z4.s\n"
+ ".inst 0x04a47694 // sqrdmulh z20.s, z20.s, z4.s\n"
+ ".inst 0x04a476b5 // sqrdmulh z21.s, z21.s, z4.s\n"
+ ".inst 0x04a476d6 // sqrdmulh z22.s, z22.s, z4.s\n"
+ ".inst 0x04a476f7 // sqrdmulh z23.s, z23.s, z4.s\n"
+ ".inst 0x04a47718 // sqrdmulh z24.s, z24.s, z4.s\n"
+ ".inst 0x04a47739 // sqrdmulh z25.s, z25.s, z4.s\n"
+ ".inst 0x04a4775a // sqrdmulh z26.s, z26.s, z4.s\n"
+ ".inst 0x04a4777b // sqrdmulh z27.s, z27.s, z4.s\n"
+ "tbz %x[flags], #5, 44f\n"
+ "and z4.d, z16.d, z0.d\n"
+ "asr z4.s, z4.s, #0x1f\n"
+ "and z5.d, z17.d, z0.d\n"
+ "and z6.d, z18.d, z0.d\n"
+ "asr z5.s, z5.s, #0x1f\n"
+ "and z7.d, z19.d, z0.d\n"
+ "and z8.d, z20.d, z0.d\n"
+ "asr z6.s, z6.s, #0x1f\n"
+ "and z9.d, z21.d, z0.d\n"
+ "asr z7.s, z7.s, #0x1f\n"
+ "sqadd z16.s, z16.s, z4.s\n"
+ "and z10.d, z22.d, z0.d\n"
+ "asr z8.s, z8.s, #0x1f\n"
+ "and z4.d, z23.d, z0.d\n"
+ "asr z9.s, z9.s, #0x1f\n"
+ "sqadd z17.s, z17.s, z5.s\n"
+ "asr z10.s, z10.s, #0x1f\n"
+ "sqadd z18.s, z18.s, z6.s\n"
+ "asr z4.s, z4.s, #0x1f\n"
+ "and z5.d, z24.d, z0.d\n"
+ "asr z5.s, z5.s, #0x1f\n"
+ "sqadd z19.s, z19.s, z7.s\n"
+ "sqadd z20.s, z20.s, z8.s\n"
+ "sqadd z21.s, z21.s, z9.s\n"
+ "sqadd z22.s, z22.s, z10.s\n"
+ "sqadd z23.s, z23.s, z4.s\n"
+ "and z6.d, z25.d, z0.d\n"
+ "asr z6.s, z6.s, #0x1f\n"
+ "sqadd z24.s, z24.s, z5.s\n"
+ "and z7.d, z26.d, z0.d\n"
+ "asr z7.s, z7.s, #0x1f\n"
+ "and z8.d, z27.d, z0.d\n"
+ "sqadd z25.s, z25.s, z6.s\n"
+ "asr z8.s, z8.s, #0x1f\n"
+ "sqadd z26.s, z26.s, z7.s\n"
+ "sqadd z27.s, z27.s, z8.s\n"
+ "44:" // Height 3: no shift correction
+ ".inst 0x44828810 // srshl z16.s, p2/M, z16.s, z0.s\n"
+ "add x19, %x[qp], %[c_offset]\n"
+ "ld1rw { z4.s }, p2/Z, [x19]\n"
+ ".inst 0x44828811 // srshl z17.s, p2/M, z17.s, z0.s\n"
+ "add x19, %x[qp], %[minval]\n"
+ ".inst 0x44828812 // srshl z18.s, p2/M, z18.s, z0.s\n"
+ "ld1rw { z5.s }, p2/Z, [x19]\n"
+ "add x19, %x[qp], %[maxval]\n"
+ ".inst 0x44828813 // srshl z19.s, p2/M, z19.s, z0.s\n"
+ "ld1rw { z6.s }, p2/Z, [x19]\n"
+ ".inst 0x44828814 // srshl z20.s, p2/M, z20.s, z0.s\n"
+ "add z16.s, z16.s, z4.s\n"
+ "add z17.s, z17.s, z4.s\n"
+ "add z18.s, z18.s, z4.s\n"
+ "add z19.s, z19.s, z4.s\n"
+ "add z20.s, z20.s, z4.s\n"
+ "smin z16.s, p2/M, z16.s, z6.s\n"
+ "smin z17.s, p2/M, z17.s, z6.s\n"
+ "smin z18.s, p2/M, z18.s, z6.s\n"
+ "smin z19.s, p2/M, z19.s, z6.s\n"
+ "smax z16.s, p2/M, z16.s, z5.s\n"
+ "smax z17.s, p2/M, z17.s, z5.s\n"
+ "smax z18.s, p2/M, z18.s, z5.s\n"
+ "smax z19.s, p2/M, z19.s, z5.s\n"
+ "smin z20.s, p2/M, z20.s, z6.s\n"
+ "uzp1 z16.h, z16.h, z17.h\n"
+ ".inst 0x44828815 // srshl z21.s, p2/M, z21.s, z0.s\n"
+ "uzp1 z17.h, z18.h, z19.h\n"
+ "smax z20.s, p2/M, z20.s, z5.s\n"
+ "uzp1 z16.b, z16.b, z17.b\n"
+ "st1b { z16.b }, p1, [x9]\n"
+ "add z21.s, z21.s, z4.s\n"
+ "addvl x9, x9, #1\n"
+ ".inst 0x44828816 // srshl z22.s, p2/M, z22.s, z0.s\n"
+ ".inst 0x44828817 // srshl z23.s, p2/M, z23.s, z0.s\n"
+ ".inst 0x44828818 // srshl z24.s, p2/M, z24.s, z0.s\n"
+ "smin z21.s, p2/M, z21.s, z6.s\n"
+ ".inst 0x44828819 // srshl z25.s, p2/M, z25.s, z0.s\n"
+ "add z22.s, z22.s, z4.s\n"
+ "add z23.s, z23.s, z4.s\n"
+ "add z24.s, z24.s, z4.s\n"
+ "add z25.s, z25.s, z4.s\n"
+ "smax z21.s, p2/M, z21.s, z5.s\n"
+ "smin z22.s, p2/M, z22.s, z6.s\n"
+ "smin z23.s, p2/M, z23.s, z6.s\n"
+ "smin z24.s, p2/M, z24.s, z6.s\n"
+ "uzp1 z20.h, z20.h, z21.h\n"
+ "smax z22.s, p2/M, z22.s, z5.s\n"
+ "smax z23.s, p2/M, z23.s, z5.s\n"
+ "smax z24.s, p2/M, z24.s, z5.s\n"
+ "smin z25.s, p2/M, z25.s, z6.s\n"
+ ".inst 0x4482881a // srshl z26.s, p2/M, z26.s, z0.s\n"
+ "uzp1 z21.h, z22.h, z23.h\n"
+ ".inst 0x4482881b // srshl z27.s, p2/M, z27.s, z0.s\n"
+ "uzp1 z20.b, z20.b, z21.b\n"
+ "st1b { z20.b }, p1, [x25]\n"
+ "add z26.s, z26.s, z4.s\n"
+ "addvl x25, x25, #1\n"
+ "add z27.s, z27.s, z4.s\n"
+ "smax z25.s, p2/M, z25.s, z5.s\n"
+ "smin z26.s, p2/M, z26.s, z6.s\n"
+ "smin z27.s, p2/M, z27.s, z6.s\n"
+ "uzp1 z24.h, z24.h, z25.h\n"
+ "smax z26.s, p2/M, z26.s, z5.s\n"
+ "smax z27.s, p2/M, z27.s, z5.s\n"
+ "uzp1 z25.h, z26.h, z27.h\n"
+ "uzp1 z24.b, z24.b, z25.b\n"
+ "st1b { z24.b }, p1, [x23]\n"
+ "addvl x23, x23, #1\n"
+ "45:" // Height 3: Writeback done
+ "mov x19, #0x0\n"
+ "incw x19, ALL, MUL #4\n"
+ "subs x12, x12, x19\n"
+ "bgt 33b\n"
+ "b 62f\n"
+ "46:" // Height 4
+ "mov z11.s, #0x0\n"
+ "ldr x12, [%x[args_ptr], %[offsetof_N]]\n"
+ "mov x10, %x[col_bias]\n"
+ "mov z12.s, #0x0\n"
+ "ldr x11, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "bic %x[flags], %x[flags], #0x80000000\n"
+ "mov z13.s, #0x0\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "mov z14.s, #0x0\n"
+ "mov z15.b, #0x1\n"
+ "tbz %x[flags], #2, 47f\n"
+ "ldr x9, [%x[output_ptr], #0x0]\n"
+ "ldr x25, [%x[output_ptr], #0x8]\n"
+ "add x9, x9, x19\n"
+ "ldr x23, [%x[output_ptr], #0x10]\n"
+ "ldr x21, [%x[output_ptr], #0x18]\n"
+ "add x25, x25, x19\n"
+ "add %x[output_ptr], %x[output_ptr], #0x20\n"
+ "add x23, x23, x19\n"
+ "add x21, x21, x19\n"
+ "b 48f\n"
+ "47:" // Height 4: setup direct output
+ "mov x9, %x[output_ptr]\n"
+ "add x25, x9, x19\n"
+ "add x23, x25, x19\n"
+ "add x21, x23, x19\n"
+ "add %x[output_ptr], x21, x19\n"
+ "48:" // Height 4: Column loop
+ "mov z16.s, #0x0\n"
+ "mov x19, #0x0\n"
+ "mov z17.s, #0x0\n"
+ "whilelt p1.b, x19, x12\n"
+ "mov z18.s, #0x0\n"
+ "mov z19.s, #0x0\n"
+ "mov z20.s, #0x0\n"
+ "mov z21.s, #0x0\n"
+ "mov z22.s, #0x0\n"
+ "mov z23.s, #0x0\n"
+ "mov z24.s, #0x0\n"
+ "mov z25.s, #0x0\n"
+ "mov z26.s, #0x0\n"
+ "mov z27.s, #0x0\n"
+ "mov z28.s, #0x0\n"
+ "mov z29.s, #0x0\n"
+ "mov z30.s, #0x0\n"
+ "mov z31.s, #0x0\n"
+ "49:" // Height 4: setup done
+ "mov x28, #0x0\n"
+ "50:" // Height 4: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w27, [x20, x28, LSL #0x2]\n"
+ "tbz %x[flags], #3, 51f\n"
+ "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x26, [x20, #0x0]\n"
+ "ldr x24, [x20, #0x8]\n"
+ "ldr x22, [x20, #0x10]\n"
+ "ldr x20, [x20, #0x18]\n"
+ "cbnz x28, 52f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x26, x26, x19\n"
+ "add x24, x24, x19\n"
+ "add x22, x22, x19\n"
+ "add x20, x20, x19\n"
+ "b 52f\n"
+ "51:" // Height 4: setup direct input
+ "mov x26, %x[input_ptr]\n"
+ "add x24, x26, x19\n"
+ "add x22, x24, x19\n"
+ "add x20, x22, x19\n"
+ "52:" // Height 4: input setup done
+ "cmp x27, #0x10\n"
+ "ble 55f\n"
+ "53:" // Height 4: Multiply loop: Main loop head
+ "ld1b { z4.b }, p2/Z, [x11]\n"
+ "whilelt p0.b, XZR, x27\n"
+ "ld1b { z5.b }, p2/Z, [x11, #1, MUL VL]\n"
+ "ld1rqb { z0.b }, p0/Z, [x26]\n"
+ "sdot z16.s, z4.b, z0.b[0]\n"
+ "ld1rqb { z1.b }, p0/Z, [x24]\n"
+ "add x26, x26, #0x10\n"
+ "sdot z17.s, z5.b, z0.b[0]\n"
+ "ld1rqb { z2.b }, p0/Z, [x22]\n"
+ "add x24, x24, #0x10\n"
+ "sdot z20.s, z4.b, z1.b[0]\n"
+ "ld1rqb { z3.b }, p0/Z, [x20]\n"
+ "add x22, x22, #0x10\n"
+ "sdot z24.s, z4.b, z2.b[0]\n"
+ "ld1b { z6.b }, p2/Z, [x11, #2, MUL VL]\n"
+ "add x20, x20, #0x10\n"
+ "sdot z21.s, z5.b, z1.b[0]\n"
+ "ld1b { z7.b }, p2/Z, [x11, #3, MUL VL]\n"
+ "sdot z25.s, z5.b, z2.b[0]\n"
+ "ld1b { z8.b }, p2/Z, [x11, #4, MUL VL]\n"
+ "sdot z28.s, z4.b, z3.b[0]\n"
+ "ld1b { z9.b }, p2/Z, [x11, #5, MUL VL]\n"
+ "sdot z29.s, z5.b, z3.b[0]\n"
+ "ld1b { z10.b }, p2/Z, [x11, #6, MUL VL]\n"
+ "sdot z18.s, z6.b, z0.b[0]\n"
+ "ld1b { z4.b }, p2/Z, [x11, #7, MUL VL]\n"
+ "addvl x11, x11, #16\n"
+ "sdot z22.s, z6.b, z1.b[0]\n"
+ "ld1b { z5.b }, p2/Z, [x11, #-8, MUL VL]\n"
+ "sdot z26.s, z6.b, z2.b[0]\n"
+ "sdot z30.s, z6.b, z3.b[0]\n"
+ "ld1b { z6.b }, p2/Z, [x11, #-7, MUL VL]\n"
+ "sdot z19.s, z7.b, z0.b[0]\n"
+ "sdot z23.s, z7.b, z1.b[0]\n"
+ "sdot z27.s, z7.b, z2.b[0]\n"
+ "sdot z31.s, z7.b, z3.b[0]\n"
+ "ld1b { z7.b }, p2/Z, [x11, #-6, MUL VL]\n"
+ "sdot z16.s, z8.b, z0.b[1]\n"
+ "sdot z20.s, z8.b, z1.b[1]\n"
+ "sdot z24.s, z8.b, z2.b[1]\n"
+ "sdot z28.s, z8.b, z3.b[1]\n"
+ "ld1b { z8.b }, p2/Z, [x11, #-5, MUL VL]\n"
+ "sdot z17.s, z9.b, z0.b[1]\n"
+ "sdot z21.s, z9.b, z1.b[1]\n"
+ "sdot z25.s, z9.b, z2.b[1]\n"
+ "sdot z29.s, z9.b, z3.b[1]\n"
+ "ld1b { z9.b }, p2/Z, [x11, #-4, MUL VL]\n"
+ "sdot z18.s, z10.b, z0.b[1]\n"
+ "sdot z22.s, z10.b, z1.b[1]\n"
+ "sdot z26.s, z10.b, z2.b[1]\n"
+ "sdot z30.s, z10.b, z3.b[1]\n"
+ "ld1b { z10.b }, p2/Z, [x11, #-3, MUL VL]\n"
+ "sdot z19.s, z4.b, z0.b[1]\n"
+ "sdot z23.s, z4.b, z1.b[1]\n"
+ "sdot z27.s, z4.b, z2.b[1]\n"
+ "sdot z31.s, z4.b, z3.b[1]\n"
+ "ld1b { z4.b }, p2/Z, [x11, #-2, MUL VL]\n"
+ "sdot z16.s, z5.b, z0.b[2]\n"
+ "sdot z20.s, z5.b, z1.b[2]\n"
+ "sdot z24.s, z5.b, z2.b[2]\n"
+ "sdot z28.s, z5.b, z3.b[2]\n"
+ "ld1b { z5.b }, p2/Z, [x11, #-1, MUL VL]\n"
+ "sdot z17.s, z6.b, z0.b[2]\n"
+ "sdot z21.s, z6.b, z1.b[2]\n"
+ "sdot z25.s, z6.b, z2.b[2]\n"
+ "sdot z29.s, z6.b, z3.b[2]\n"
+ "sdot z18.s, z7.b, z0.b[2]\n"
+ "sdot z22.s, z7.b, z1.b[2]\n"
+ "sdot z26.s, z7.b, z2.b[2]\n"
+ "sdot z30.s, z7.b, z3.b[2]\n"
+ "sdot z19.s, z8.b, z0.b[2]\n"
+ "sdot z23.s, z8.b, z1.b[2]\n"
+ "sdot z27.s, z8.b, z2.b[2]\n"
+ "sdot z31.s, z8.b, z3.b[2]\n"
+ "sdot z16.s, z9.b, z0.b[3]\n"
+ "sdot z20.s, z9.b, z1.b[3]\n"
+ "sdot z24.s, z9.b, z2.b[3]\n"
+ "sdot z28.s, z9.b, z3.b[3]\n"
+ "sdot z17.s, z10.b, z0.b[3]\n"
+ "sdot z21.s, z10.b, z1.b[3]\n"
+ "sdot z25.s, z10.b, z2.b[3]\n"
+ "sdot z29.s, z10.b, z3.b[3]\n"
+ "sdot z18.s, z4.b, z0.b[3]\n"
+ "sdot z22.s, z4.b, z1.b[3]\n"
+ "sdot z26.s, z4.b, z2.b[3]\n"
+ "sdot z30.s, z4.b, z3.b[3]\n"
+ "sdot z19.s, z5.b, z0.b[3]\n"
+ "sdot z23.s, z5.b, z1.b[3]\n"
+ "sdot z27.s, z5.b, z2.b[3]\n"
+ "sdot z31.s, z5.b, z3.b[3]\n"
+ "tbnz %x[flags], #31, 54f\n"
+ "sdot z11.s, z0.b, z15.b\n"
+ "sdot z12.s, z1.b, z15.b\n"
+ "sdot z13.s, z2.b, z15.b\n"
+ "sdot z14.s, z3.b, z15.b\n"
+ "54:" // Height 4: Multiply loop: unique 7: skip row sum
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "sub x27, x27, #0x10\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "cmp x27, #0x10\n"
+ "prfm pldl1keep, [x22, #0x80]\n"
+ "prfm pldl1keep, [x20, #0x80]\n"
+ "bgt 53b\n"
+ "55:" // Height 4: Multiply loop: Single iteration only
+ "ld1b { z6.b }, p2/Z, [x11]\n"
+ "whilelt p0.b, XZR, x27\n"
+ "ld1b { z7.b }, p2/Z, [x11, #1, MUL VL]\n"
+ "subs x27, x27, #0x4\n"
+ "ld1rqb { z0.b }, p0/Z, [x26]\n"
+ "sdot z16.s, z6.b, z0.b[0]\n"
+ "ld1rqb { z1.b }, p0/Z, [x24]\n"
+ "add x26, x26, #0x10\n"
+ "sdot z17.s, z7.b, z0.b[0]\n"
+ "ld1rqb { z2.b }, p0/Z, [x22]\n"
+ "add x24, x24, #0x10\n"
+ "sdot z20.s, z6.b, z1.b[0]\n"
+ "ld1rqb { z3.b }, p0/Z, [x20]\n"
+ "add x22, x22, #0x10\n"
+ "sdot z24.s, z6.b, z2.b[0]\n"
+ "ld1b { z8.b }, p2/Z, [x11, #2, MUL VL]\n"
+ "add x20, x20, #0x10\n"
+ "sdot z21.s, z7.b, z1.b[0]\n"
+ "ld1b { z9.b }, p2/Z, [x11, #3, MUL VL]\n"
+ "addvl x11, x11, #4\n"
+ "sdot z28.s, z6.b, z3.b[0]\n"
+ "sdot z25.s, z7.b, z2.b[0]\n"
+ "sdot z29.s, z7.b, z3.b[0]\n"
+ "sdot z18.s, z8.b, z0.b[0]\n"
+ "sdot z22.s, z8.b, z1.b[0]\n"
+ "sdot z26.s, z8.b, z2.b[0]\n"
+ "sdot z30.s, z8.b, z3.b[0]\n"
+ "sdot z19.s, z9.b, z0.b[0]\n"
+ "sdot z23.s, z9.b, z1.b[0]\n"
+ "sdot z27.s, z9.b, z2.b[0]\n"
+ "sdot z31.s, z9.b, z3.b[0]\n"
+ "ble 56f\n"
+ "ld1b { z10.b }, p2/Z, [x11]\n"
+ "sdot z16.s, z10.b, z0.b[1]\n"
+ "ld1b { z4.b }, p2/Z, [x11, #1, MUL VL]\n"
+ "subs x27, x27, #0x4\n"
+ "sdot z20.s, z10.b, z1.b[1]\n"
+ "ld1b { z5.b }, p2/Z, [x11, #2, MUL VL]\n"
+ "sdot z24.s, z10.b, z2.b[1]\n"
+ "ld1b { z6.b }, p2/Z, [x11, #3, MUL VL]\n"
+ "addvl x11, x11, #4\n"
+ "sdot z28.s, z10.b, z3.b[1]\n"
+ "sdot z17.s, z4.b, z0.b[1]\n"
+ "sdot z21.s, z4.b, z1.b[1]\n"
+ "sdot z25.s, z4.b, z2.b[1]\n"
+ "sdot z29.s, z4.b, z3.b[1]\n"
+ "sdot z18.s, z5.b, z0.b[1]\n"
+ "sdot z22.s, z5.b, z1.b[1]\n"
+ "sdot z26.s, z5.b, z2.b[1]\n"
+ "sdot z30.s, z5.b, z3.b[1]\n"
+ "sdot z19.s, z6.b, z0.b[1]\n"
+ "sdot z23.s, z6.b, z1.b[1]\n"
+ "sdot z27.s, z6.b, z2.b[1]\n"
+ "sdot z31.s, z6.b, z3.b[1]\n"
+ "ble 56f\n"
+ "ld1b { z7.b }, p2/Z, [x11]\n"
+ "sdot z16.s, z7.b, z0.b[2]\n"
+ "ld1b { z8.b }, p2/Z, [x11, #1, MUL VL]\n"
+ "subs x27, x27, #0x4\n"
+ "sdot z20.s, z7.b, z1.b[2]\n"
+ "ld1b { z9.b }, p2/Z, [x11, #2, MUL VL]\n"
+ "sdot z24.s, z7.b, z2.b[2]\n"
+ "ld1b { z10.b }, p2/Z, [x11, #3, MUL VL]\n"
+ "addvl x11, x11, #4\n"
+ "sdot z28.s, z7.b, z3.b[2]\n"
+ "sdot z17.s, z8.b, z0.b[2]\n"
+ "sdot z21.s, z8.b, z1.b[2]\n"
+ "sdot z25.s, z8.b, z2.b[2]\n"
+ "sdot z29.s, z8.b, z3.b[2]\n"
+ "sdot z18.s, z9.b, z0.b[2]\n"
+ "sdot z22.s, z9.b, z1.b[2]\n"
+ "sdot z26.s, z9.b, z2.b[2]\n"
+ "sdot z30.s, z9.b, z3.b[2]\n"
+ "sdot z19.s, z10.b, z0.b[2]\n"
+ "sdot z23.s, z10.b, z1.b[2]\n"
+ "sdot z27.s, z10.b, z2.b[2]\n"
+ "sdot z31.s, z10.b, z3.b[2]\n"
+ "ble 56f\n"
+ "ld1b { z4.b }, p2/Z, [x11]\n"
+ "sdot z16.s, z4.b, z0.b[3]\n"
+ "ld1b { z5.b }, p2/Z, [x11, #1, MUL VL]\n"
+ "sdot z20.s, z4.b, z1.b[3]\n"
+ "ld1b { z6.b }, p2/Z, [x11, #2, MUL VL]\n"
+ "sdot z24.s, z4.b, z2.b[3]\n"
+ "ld1b { z7.b }, p2/Z, [x11, #3, MUL VL]\n"
+ "addvl x11, x11, #4\n"
+ "sdot z28.s, z4.b, z3.b[3]\n"
+ "sdot z17.s, z5.b, z0.b[3]\n"
+ "sdot z21.s, z5.b, z1.b[3]\n"
+ "sdot z25.s, z5.b, z2.b[3]\n"
+ "sdot z29.s, z5.b, z3.b[3]\n"
+ "sdot z18.s, z6.b, z0.b[3]\n"
+ "sdot z22.s, z6.b, z1.b[3]\n"
+ "sdot z26.s, z6.b, z2.b[3]\n"
+ "sdot z30.s, z6.b, z3.b[3]\n"
+ "sdot z19.s, z7.b, z0.b[3]\n"
+ "sdot z23.s, z7.b, z1.b[3]\n"
+ "sdot z27.s, z7.b, z2.b[3]\n"
+ "sdot z31.s, z7.b, z3.b[3]\n"
+ "56:" // Height 4: Multiply loop: multiply skip
+ "tbnz %x[flags], #31, 57f\n"
+ "sdot z11.s, z0.b, z15.b\n"
+ "sdot z12.s, z1.b, z15.b\n"
+ "sdot z13.s, z2.b, z15.b\n"
+ "sdot z14.s, z3.b, z15.b\n"
+ "57:" // Height 4: Multiply loop: unique 8: skip row sum
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "add x28, x28, #0x1\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "prfm pldl1keep, [x22, #0x80]\n"
+ "prfm pldl1keep, [x20, #0x80]\n"
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "cmp x28, x19\n"
+ "bne 50b\n"
+ "prfm pstl1keep, [x9, #0x0]\n"
+ "prfm pstl1keep, [x25, #0x0]\n"
+ "prfm pstl1keep, [x23, #0x0]\n"
+ "prfm pstl1keep, [x21, #0x0]\n"
+ "tbnz %x[flags], #31, 58f\n"
+ "add x19, %x[qp], %[b_offset]\n"
+ "ld1rw { z4.s }, p2/Z, [x19]\n"
+ "neg z4.s, p2/M, z4.s\n"
+ "mov x20, #0x4\n"
+ "mov x19, #0x4\n"
+ "whilelt p0.s, XZR, x20\n"
+ "saddv d11, p0, z11.s\n"
+ "whilelt p0.s, XZR, x19\n"
+ "saddv d12, p0, z12.s\n"
+ "mov x19, #0x4\n"
+ "mov z11.s, z11.s[0]\n"
+ "whilelt p0.s, XZR, x19\n"
+ "mov x19, #0x4\n"
+ "mov z12.s, z12.s[0]\n"
+ "saddv d13, p0, z13.s\n"
+ "whilelt p0.s, XZR, x19\n"
+ "mul z11.s, p2/M, z11.s, z4.s\n"
+ "saddv d14, p0, z14.s\n"
+ "mul z12.s, p2/M, z12.s, z4.s\n"
+ "mov z13.s, z13.s[0]\n"
+ "mul z13.s, p2/M, z13.s, z4.s\n"
+ "mov z14.s, z14.s[0]\n"
+ "mul z14.s, p2/M, z14.s, z4.s\n"
+ "58:" // Height 4: skip row sum fixup
+ "add z16.s, z16.s, z11.s\n"
+ "ld1w { z0.s }, p2/Z, [x10]\n"
+ "orr %x[flags], %x[flags], #0x80000000\n"
+ "add z17.s, z17.s, z11.s\n"
+ "ld1w { z1.s }, p2/Z, [x10, #1, MUL VL]\n"
+ "add x20, %x[qp], %[per_layer_right_shift]\n"
+ "add z18.s, z18.s, z11.s\n"
+ "ld1w { z2.s }, p2/Z, [x10, #2, MUL VL]\n"
+ "add x19, %x[qp], %[per_layer_mul]\n"
+ "add z19.s, z19.s, z11.s\n"
+ "ld1w { z3.s }, p2/Z, [x10, #3, MUL VL]\n"
+ "addvl x10, x10, #4\n"
+ "add z20.s, z20.s, z12.s\n"
+ "ld1rw { z4.s }, p2/Z, [x19]\n"
+ "add z21.s, z21.s, z12.s\n"
+ "add z22.s, z22.s, z12.s\n"
+ "add z23.s, z23.s, z12.s\n"
+ "add z24.s, z24.s, z13.s\n"
+ "add z25.s, z25.s, z13.s\n"
+ "add z26.s, z26.s, z13.s\n"
+ "add z27.s, z27.s, z13.s\n"
+ "add z28.s, z28.s, z14.s\n"
+ "add z29.s, z29.s, z14.s\n"
+ "add z30.s, z30.s, z14.s\n"
+ "add z31.s, z31.s, z14.s\n"
+ "add z16.s, z16.s, z0.s\n"
+ "add z17.s, z17.s, z1.s\n"
+ "add z18.s, z18.s, z2.s\n"
+ "add z19.s, z19.s, z3.s\n"
+ "add z20.s, z20.s, z0.s\n"
+ "add z21.s, z21.s, z1.s\n"
+ "add z22.s, z22.s, z2.s\n"
+ "add z23.s, z23.s, z3.s\n"
+ "add z24.s, z24.s, z0.s\n"
+ "add z25.s, z25.s, z1.s\n"
+ "add z26.s, z26.s, z2.s\n"
+ "add z27.s, z27.s, z3.s\n"
+ "add z28.s, z28.s, z0.s\n"
+ "ld1rw { z0.s }, p2/Z, [x20]\n"
+ "add z29.s, z29.s, z1.s\n"
+ "add z30.s, z30.s, z2.s\n"
+ "add z31.s, z31.s, z3.s\n"
+ ".inst 0x04a47610 // sqrdmulh z16.s, z16.s, z4.s\n"
+ ".inst 0x04a47631 // sqrdmulh z17.s, z17.s, z4.s\n"
+ ".inst 0x04a47652 // sqrdmulh z18.s, z18.s, z4.s\n"
+ ".inst 0x04a47673 // sqrdmulh z19.s, z19.s, z4.s\n"
+ ".inst 0x04a47694 // sqrdmulh z20.s, z20.s, z4.s\n"
+ ".inst 0x04a476b5 // sqrdmulh z21.s, z21.s, z4.s\n"
+ ".inst 0x04a476d6 // sqrdmulh z22.s, z22.s, z4.s\n"
+ ".inst 0x04a476f7 // sqrdmulh z23.s, z23.s, z4.s\n"
+ ".inst 0x04a47718 // sqrdmulh z24.s, z24.s, z4.s\n"
+ ".inst 0x04a47739 // sqrdmulh z25.s, z25.s, z4.s\n"
+ ".inst 0x04a4775a // sqrdmulh z26.s, z26.s, z4.s\n"
+ ".inst 0x04a4777b // sqrdmulh z27.s, z27.s, z4.s\n"
+ ".inst 0x04a4779c // sqrdmulh z28.s, z28.s, z4.s\n"
+ ".inst 0x04a477bd // sqrdmulh z29.s, z29.s, z4.s\n"
+ ".inst 0x04a477de // sqrdmulh z30.s, z30.s, z4.s\n"
+ ".inst 0x04a477ff // sqrdmulh z31.s, z31.s, z4.s\n"
+ "tbz %x[flags], #5, 59f\n"
+ "and z4.d, z16.d, z0.d\n"
+ "asr z4.s, z4.s, #0x1f\n"
+ "and z5.d, z17.d, z0.d\n"
+ "and z6.d, z18.d, z0.d\n"
+ "asr z5.s, z5.s, #0x1f\n"
+ "and z7.d, z19.d, z0.d\n"
+ "and z8.d, z20.d, z0.d\n"
+ "asr z6.s, z6.s, #0x1f\n"
+ "and z9.d, z21.d, z0.d\n"
+ "asr z7.s, z7.s, #0x1f\n"
+ "sqadd z16.s, z16.s, z4.s\n"
+ "and z10.d, z22.d, z0.d\n"
+ "asr z8.s, z8.s, #0x1f\n"
+ "and z4.d, z23.d, z0.d\n"
+ "asr z9.s, z9.s, #0x1f\n"
+ "sqadd z17.s, z17.s, z5.s\n"
+ "asr z10.s, z10.s, #0x1f\n"
+ "sqadd z18.s, z18.s, z6.s\n"
+ "asr z4.s, z4.s, #0x1f\n"
+ "and z5.d, z24.d, z0.d\n"
+ "asr z5.s, z5.s, #0x1f\n"
+ "sqadd z19.s, z19.s, z7.s\n"
+ "sqadd z20.s, z20.s, z8.s\n"
+ "sqadd z21.s, z21.s, z9.s\n"
+ "sqadd z22.s, z22.s, z10.s\n"
+ "sqadd z23.s, z23.s, z4.s\n"
+ "and z6.d, z25.d, z0.d\n"
+ "asr z6.s, z6.s, #0x1f\n"
+ "sqadd z24.s, z24.s, z5.s\n"
+ "and z7.d, z26.d, z0.d\n"
+ "asr z7.s, z7.s, #0x1f\n"
+ "and z8.d, z27.d, z0.d\n"
+ "and z9.d, z28.d, z0.d\n"
+ "asr z8.s, z8.s, #0x1f\n"
+ "sqadd z25.s, z25.s, z6.s\n"
+ "and z10.d, z29.d, z0.d\n"
+ "asr z9.s, z9.s, #0x1f\n"
+ "and z4.d, z30.d, z0.d\n"
+ "asr z10.s, z10.s, #0x1f\n"
+ "sqadd z26.s, z26.s, z7.s\n"
+ "and z5.d, z31.d, z0.d\n"
+ "asr z4.s, z4.s, #0x1f\n"
+ "sqadd z27.s, z27.s, z8.s\n"
+ "asr z5.s, z5.s, #0x1f\n"
+ "sqadd z28.s, z28.s, z9.s\n"
+ "sqadd z29.s, z29.s, z10.s\n"
+ "sqadd z30.s, z30.s, z4.s\n"
+ "sqadd z31.s, z31.s, z5.s\n"
+ "59:" // Height 4: no shift correction
+ ".inst 0x44828810 // srshl z16.s, p2/M, z16.s, z0.s\n"
+ "add x19, %x[qp], %[c_offset]\n"
+ "ld1rw { z4.s }, p2/Z, [x19]\n"
+ ".inst 0x44828811 // srshl z17.s, p2/M, z17.s, z0.s\n"
+ "add x19, %x[qp], %[minval]\n"
+ ".inst 0x44828812 // srshl z18.s, p2/M, z18.s, z0.s\n"
+ "ld1rw { z5.s }, p2/Z, [x19]\n"
+ "add x19, %x[qp], %[maxval]\n"
+ ".inst 0x44828813 // srshl z19.s, p2/M, z19.s, z0.s\n"
+ "ld1rw { z6.s }, p2/Z, [x19]\n"
+ ".inst 0x44828814 // srshl z20.s, p2/M, z20.s, z0.s\n"
+ "add z16.s, z16.s, z4.s\n"
+ "add z17.s, z17.s, z4.s\n"
+ "add z18.s, z18.s, z4.s\n"
+ "add z19.s, z19.s, z4.s\n"
+ "add z20.s, z20.s, z4.s\n"
+ "smin z16.s, p2/M, z16.s, z6.s\n"
+ "smin z17.s, p2/M, z17.s, z6.s\n"
+ "smin z18.s, p2/M, z18.s, z6.s\n"
+ "smin z19.s, p2/M, z19.s, z6.s\n"
+ "smax z16.s, p2/M, z16.s, z5.s\n"
+ "smax z17.s, p2/M, z17.s, z5.s\n"
+ "smax z18.s, p2/M, z18.s, z5.s\n"
+ "smax z19.s, p2/M, z19.s, z5.s\n"
+ "smin z20.s, p2/M, z20.s, z6.s\n"
+ "uzp1 z16.h, z16.h, z17.h\n"
+ ".inst 0x44828815 // srshl z21.s, p2/M, z21.s, z0.s\n"
+ "uzp1 z17.h, z18.h, z19.h\n"
+ "smax z20.s, p2/M, z20.s, z5.s\n"
+ "uzp1 z16.b, z16.b, z17.b\n"
+ "st1b { z16.b }, p1, [x9]\n"
+ "add z21.s, z21.s, z4.s\n"
+ "addvl x9, x9, #1\n"
+ ".inst 0x44828816 // srshl z22.s, p2/M, z22.s, z0.s\n"
+ ".inst 0x44828817 // srshl z23.s, p2/M, z23.s, z0.s\n"
+ ".inst 0x44828818 // srshl z24.s, p2/M, z24.s, z0.s\n"
+ "smin z21.s, p2/M, z21.s, z6.s\n"
+ ".inst 0x44828819 // srshl z25.s, p2/M, z25.s, z0.s\n"
+ "add z22.s, z22.s, z4.s\n"
+ "add z23.s, z23.s, z4.s\n"
+ "add z24.s, z24.s, z4.s\n"
+ "add z25.s, z25.s, z4.s\n"
+ "smax z21.s, p2/M, z21.s, z5.s\n"
+ "smin z22.s, p2/M, z22.s, z6.s\n"
+ "smin z23.s, p2/M, z23.s, z6.s\n"
+ "smin z24.s, p2/M, z24.s, z6.s\n"
+ "uzp1 z20.h, z20.h, z21.h\n"
+ "smax z22.s, p2/M, z22.s, z5.s\n"
+ "smax z23.s, p2/M, z23.s, z5.s\n"
+ "smax z24.s, p2/M, z24.s, z5.s\n"
+ "smin z25.s, p2/M, z25.s, z6.s\n"
+ ".inst 0x4482881a // srshl z26.s, p2/M, z26.s, z0.s\n"
+ "uzp1 z21.h, z22.h, z23.h\n"
+ ".inst 0x4482881b // srshl z27.s, p2/M, z27.s, z0.s\n"
+ "uzp1 z20.b, z20.b, z21.b\n"
+ "st1b { z20.b }, p1, [x25]\n"
+ "add z26.s, z26.s, z4.s\n"
+ "addvl x25, x25, #1\n"
+ "add z27.s, z27.s, z4.s\n"
+ "smax z25.s, p2/M, z25.s, z5.s\n"
+ ".inst 0x4482881c // srshl z28.s, p2/M, z28.s, z0.s\n"
+ "smin z26.s, p2/M, z26.s, z6.s\n"
+ "smin z27.s, p2/M, z27.s, z6.s\n"
+ "uzp1 z24.h, z24.h, z25.h\n"
+ "add z28.s, z28.s, z4.s\n"
+ "smax z26.s, p2/M, z26.s, z5.s\n"
+ "smax z27.s, p2/M, z27.s, z5.s\n"
+ "smin z28.s, p2/M, z28.s, z6.s\n"
+ ".inst 0x4482881d // srshl z29.s, p2/M, z29.s, z0.s\n"
+ ".inst 0x4482881e // srshl z30.s, p2/M, z30.s, z0.s\n"
+ "uzp1 z25.h, z26.h, z27.h\n"
+ "smax z28.s, p2/M, z28.s, z5.s\n"
+ "add z29.s, z29.s, z4.s\n"
+ "add z30.s, z30.s, z4.s\n"
+ "uzp1 z24.b, z24.b, z25.b\n"
+ "st1b { z24.b }, p1, [x23]\n"
+ "smin z29.s, p2/M, z29.s, z6.s\n"
+ "addvl x23, x23, #1\n"
+ "smin z30.s, p2/M, z30.s, z6.s\n"
+ ".inst 0x4482881f // srshl z31.s, p2/M, z31.s, z0.s\n"
+ "smax z29.s, p2/M, z29.s, z5.s\n"
+ "add z31.s, z31.s, z4.s\n"
+ "smax z30.s, p2/M, z30.s, z5.s\n"
+ "uzp1 z28.h, z28.h, z29.h\n"
+ "smin z31.s, p2/M, z31.s, z6.s\n"
+ "smax z31.s, p2/M, z31.s, z5.s\n"
+ "uzp1 z29.h, z30.h, z31.h\n"
+ "uzp1 z28.b, z28.b, z29.b\n"
+ "st1b { z28.b }, p1, [x21]\n"
+ "addvl x21, x21, #1\n"
+ "60:" // Height 4: Writeback done
+ "mov x19, #0x0\n"
+ "incw x19, ALL, MUL #4\n"
+ "subs x12, x12, x19\n"
+ "bgt 48b\n"
+ "subs %x[M], %x[M], #0x4\n"
+ "beq 62f\n"
+ "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "tbz %x[flags], #3, 61f\n"
+ "add x20, x20, #0x4\n"
+ "str x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "b 1b\n"
+ "61:" // Update direct input
+ "mov x19, #0x4\n"
+ "madd %x[input_ptr], x19, x20, %x[input_ptr]\n"
+ "b 1b\n"
+ "62:" // Exit
+
+ : [M] "+r" (M), [flags] "+r" (flags), [input_ptr] "+r" (input_ptr), [output_ptr] "+r" (output_ptr)
+ : [args_ptr] "r" (&ka), [b_offset] "I" (offsetof(Requantize32, b_offset)), [c_offset] "I" (offsetof(Requantize32, c_offset)), [col_bias] "r" (col_bias), [maxval] "I" (offsetof(Requantize32, maxval)), [minval] "I" (offsetof(Requantize32, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths)), [per_layer_mul] "I" (offsetof(Requantize32, per_layer_mul)), [per_layer_right_shift] "I" (offsetof(Requantize32, per_layer_right_shift)), [qp] "r" (qp)
+ : "cc", "memory", "p0", "p1", "p2", "x9", "x10", "x11", "x12", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+ );
+}
+
+} // namespace arm_gemm
+#endif // __ARM_FEATURE_SVE
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8s32_dot_4VLx4.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8qs_dot_6x4VL.hpp
similarity index 63%
copy from src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8s32_dot_4VLx4.hpp
copy to src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8qs_dot_6x4VL.hpp
index c500f43..d856289 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8s32_dot_4VLx4.hpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8qs_dot_6x4VL.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2018-2020 Arm Limited.
+ * Copyright (c) 2019-2020 Arm Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -10,42 +10,48 @@
* sell copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
*/
#pragma once
-
#ifdef __ARM_FEATURE_SVE
-#include <cstdint>
#include "../std_transforms_sve.hpp"
+#define ARGLIST \
+ unsigned int, const unsigned int *, \
+ IndirectInputArg<int8_t>, \
+ size_t, size_t, \
+ const int8_t *, \
+ IndirectOutputArg<int8_t>, \
+ const Requantize32 *, const int32_t *, unsigned int
+
namespace arm_gemm
{
// Actual kernel implementations
-void sve_hybrid_s8s32_dot_4VLx4(const int8_t *, int, const int8_t *, int32_t *, int, int, int, int, const int32_t *, Activation, bool);
+void sve_hybrid_s8qs_dot_6x4VL( ARGLIST );
-class hybrid_s8s32_dot_4VLx4
+class cls_sve_hybrid_s8qs_dot_6x4VL
{
public:
typedef int8_t operand_type;
- typedef int32_t result_type;
+ typedef int8_t result_type;
- typedef void (*kern_type)(const int8_t *, int, const int8_t *, int32_t *, int, int, int, int, const int32_t *, Activation, bool);
+ typedef void (*kern_type)( ARGLIST );
/* Kernel blocking parameters */
static constexpr unsigned int out_height()
{
- return 4;
+ return 6;
}
static unsigned int out_width()
@@ -60,30 +66,20 @@
static constexpr bool supports_accumulate()
{
- return true;
- }
-
- static constexpr bool supports_bias()
- {
return false;
}
- static constexpr bool supports_activation()
- {
- return false;
- }
-
- StdTransformsSVE<operand_type, result_type, 4, 4, 4> transforms = {};
+ StdTransformsSVE<operand_type, result_type, 6, 4, 4> transforms = {};
// Default to the generic kernel
- kern_type kernel=sve_hybrid_s8s32_dot_4VLx4;
+ kern_type kernel=sve_hybrid_s8qs_dot_6x4VL;
- hybrid_s8s32_dot_4VLx4(const CPUInfo *)
+ cls_sve_hybrid_s8qs_dot_6x4VL(const CPUInfo *)
{
-
}
};
} // namespace arm_gemm
+#undef ARGLIST
#endif // __ARM_FEATURE_SVE
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8qs_dot_6x4VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8qs_dot_6x4VL/generic.cpp
new file mode 100644
index 0000000..4a4af63
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8qs_dot_6x4VL/generic.cpp
@@ -0,0 +1,2770 @@
+/*
+ * Copyright (c) 2019-2020 Arm Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+#ifdef __ARM_FEATURE_SVE
+
+#include "arm_gemm.hpp"
+#include "../../utils.hpp"
+
+#include <cassert>
+#include <cstdint>
+
+namespace arm_gemm {
+
+void sve_hybrid_s8qs_dot_6x4VL (
+ unsigned int num_strings, const unsigned int *string_lengths, IndirectInputArg<int8_t> A_arg,
+ size_t M, size_t N, const int8_t *B_ptr, IndirectOutputArg<int8_t> output_arg,
+ const Requantize32 *qp, const int32_t *col_bias, unsigned int col_base
+)
+{
+ struct KernelArgs {
+ const int32_t *multiplier_ptr = {};
+ const int32_t *shift_ptr = {};
+ unsigned int num_strings = {};
+ const unsigned int *string_lengths = {};
+ size_t N = {};
+ const int8_t *B_ptr = {};
+ size_t output_offset = {};
+ size_t input_initial_col = {};
+ size_t input_offset = {};
+ } ka;
+
+ unsigned long flags=0;
+ void *output_ptr;
+ void *input_ptr;
+
+ if (output_arg.is_indirect) {
+ output_ptr=(void *)(output_arg.indirect.ptr);
+ ka.output_offset=output_arg.indirect.offset;
+ flags |= 0x4;
+ } else {
+ output_ptr=(void *)(output_arg.direct.base);
+ ka.output_offset=output_arg.direct.stride;
+ }
+
+ if (A_arg.is_indirect) {
+ input_ptr=(void *)(A_arg.indirect.ptr);
+ ka.input_offset=A_arg.indirect.start_row;
+ ka.input_initial_col=A_arg.indirect.start_col;
+ flags |= 0x8;
+ } else {
+ assert(num_strings==1);
+ input_ptr=(void *)(A_arg.direct.base);
+ ka.input_offset=A_arg.direct.stride;
+ }
+ ka.num_strings = num_strings;
+ ka.string_lengths = string_lengths;
+ ka.N = N;
+ ka.B_ptr = B_ptr;
+ if (qp->per_channel_requant) {
+ flags |= 0x10;
+ ka.multiplier_ptr=qp->per_channel_muls + col_base;
+ ka.shift_ptr=qp->per_channel_right_shifts + col_base;
+ }
+ if (qp->c_offset > qp->minval) {
+ flags |= 0x20;
+ }
+ __asm__ __volatile__(
+ "ptrue p2.b\n"
+ "1:" // Row loop
+ "cmp %x[M], #0x6\n"
+ "bge 71f\n"
+ "cmp %x[M], #0x4\n"
+ "bgt 57f\n"
+ "beq 43f\n"
+ "cmp %x[M], #0x2\n"
+ "bgt 29f\n"
+ "beq 15f\n"
+ "ldr x8, [%x[args_ptr], %[offsetof_multiplier_ptr]]\n"
+ "ldr x17, [%x[args_ptr], %[offsetof_shift_ptr]]\n"
+ "mov x16, %x[col_bias]\n"
+ "ldr x15, [%x[args_ptr], %[offsetof_N]]\n"
+ "ldr x14, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 2f\n"
+ "ldr x13, [%x[output_ptr], #0x0]\n"
+ "add x13, x13, x19\n"
+ "b 3f\n"
+ "2:" // Height 1: setup direct output
+ "mov x13, %x[output_ptr]\n"
+ "3:" // Height 1: Column loop
+ "mov z8.s, #0x0\n"
+ "mov x19, #0x0\n"
+ "mov z9.s, #0x0\n"
+ "whilelt p1.b, x19, x15\n"
+ "mov z10.s, #0x0\n"
+ "mov z11.s, #0x0\n"
+ "4:" // Height 1: setup done
+ "mov x12, #0x0\n"
+ "5:" // Height 1: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w11, [x20, x12, LSL #0x2]\n"
+ "tbz %x[flags], #3, 6f\n"
+ "ldr x20, [%x[input_ptr], x12, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x10, [x20, #0x0]\n"
+ "cbnz x12, 7f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x10, x10, x19\n"
+ "b 7f\n"
+ "6:" // Height 1: setup direct input
+ "mov x10, %x[input_ptr]\n"
+ "7:" // Height 1: input setup done
+ "cmp x11, #0x10\n"
+ "ble 9f\n"
+ "8:" // Height 1: Multiply loop: Main loop head
+ "ld1b { z6.b }, p2/Z, [x14]\n"
+ "whilelt p0.b, XZR, x11\n"
+ "ld1b { z7.b }, p2/Z, [x14, #1, MUL VL]\n"
+ "sub x11, x11, #0x10\n"
+ "ld1rqb { z0.b }, p0/Z, [x10]\n"
+ "sdot z8.s, z6.b, z0.b[0]\n"
+ "ld1b { z6.b }, p2/Z, [x14, #2, MUL VL]\n"
+ "add x10, x10, #0x10\n"
+ "sdot z9.s, z7.b, z0.b[0]\n"
+ "ld1b { z7.b }, p2/Z, [x14, #3, MUL VL]\n"
+ "cmp x11, #0x10\n"
+ "sdot z10.s, z6.b, z0.b[0]\n"
+ "ld1b { z6.b }, p2/Z, [x14, #4, MUL VL]\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "sdot z11.s, z7.b, z0.b[0]\n"
+ "ld1b { z7.b }, p2/Z, [x14, #5, MUL VL]\n"
+ "sdot z8.s, z6.b, z0.b[1]\n"
+ "ld1b { z6.b }, p2/Z, [x14, #6, MUL VL]\n"
+ "sdot z9.s, z7.b, z0.b[1]\n"
+ "ld1b { z7.b }, p2/Z, [x14, #7, MUL VL]\n"
+ "addvl x14, x14, #16\n"
+ "sdot z10.s, z6.b, z0.b[1]\n"
+ "ld1b { z6.b }, p2/Z, [x14, #-8, MUL VL]\n"
+ "sdot z11.s, z7.b, z0.b[1]\n"
+ "ld1b { z7.b }, p2/Z, [x14, #-7, MUL VL]\n"
+ "sdot z8.s, z6.b, z0.b[2]\n"
+ "ld1b { z6.b }, p2/Z, [x14, #-6, MUL VL]\n"
+ "sdot z9.s, z7.b, z0.b[2]\n"
+ "ld1b { z7.b }, p2/Z, [x14, #-5, MUL VL]\n"
+ "sdot z10.s, z6.b, z0.b[2]\n"
+ "ld1b { z6.b }, p2/Z, [x14, #-4, MUL VL]\n"
+ "sdot z11.s, z7.b, z0.b[2]\n"
+ "ld1b { z7.b }, p2/Z, [x14, #-3, MUL VL]\n"
+ "sdot z8.s, z6.b, z0.b[3]\n"
+ "ld1b { z6.b }, p2/Z, [x14, #-2, MUL VL]\n"
+ "sdot z9.s, z7.b, z0.b[3]\n"
+ "ld1b { z7.b }, p2/Z, [x14, #-1, MUL VL]\n"
+ "sdot z10.s, z6.b, z0.b[3]\n"
+ "sdot z11.s, z7.b, z0.b[3]\n"
+ "bgt 8b\n"
+ "9:" // Height 1: Multiply loop: Single iteration only
+ "ld1b { z6.b }, p2/Z, [x14]\n"
+ "whilelt p0.b, XZR, x11\n"
+ "ld1b { z7.b }, p2/Z, [x14, #1, MUL VL]\n"
+ "subs x11, x11, #0x4\n"
+ "ld1rqb { z0.b }, p0/Z, [x10]\n"
+ "sdot z8.s, z6.b, z0.b[0]\n"
+ "ld1b { z6.b }, p2/Z, [x14, #2, MUL VL]\n"
+ "add x10, x10, #0x10\n"
+ "sdot z9.s, z7.b, z0.b[0]\n"
+ "ld1b { z7.b }, p2/Z, [x14, #3, MUL VL]\n"
+ "addvl x14, x14, #4\n"
+ "sdot z10.s, z6.b, z0.b[0]\n"
+ "sdot z11.s, z7.b, z0.b[0]\n"
+ "ble 10f\n"
+ "ld1b { z6.b }, p2/Z, [x14]\n"
+ "sdot z8.s, z6.b, z0.b[1]\n"
+ "ld1b { z7.b }, p2/Z, [x14, #1, MUL VL]\n"
+ "subs x11, x11, #0x4\n"
+ "sdot z9.s, z7.b, z0.b[1]\n"
+ "ld1b { z6.b }, p2/Z, [x14, #2, MUL VL]\n"
+ "ld1b { z7.b }, p2/Z, [x14, #3, MUL VL]\n"
+ "sdot z10.s, z6.b, z0.b[1]\n"
+ "addvl x14, x14, #4\n"
+ "sdot z11.s, z7.b, z0.b[1]\n"
+ "ble 10f\n"
+ "ld1b { z6.b }, p2/Z, [x14]\n"
+ "sdot z8.s, z6.b, z0.b[2]\n"
+ "ld1b { z7.b }, p2/Z, [x14, #1, MUL VL]\n"
+ "subs x11, x11, #0x4\n"
+ "sdot z9.s, z7.b, z0.b[2]\n"
+ "ld1b { z6.b }, p2/Z, [x14, #2, MUL VL]\n"
+ "ld1b { z7.b }, p2/Z, [x14, #3, MUL VL]\n"
+ "sdot z10.s, z6.b, z0.b[2]\n"
+ "addvl x14, x14, #4\n"
+ "sdot z11.s, z7.b, z0.b[2]\n"
+ "ble 10f\n"
+ "ld1b { z6.b }, p2/Z, [x14]\n"
+ "sdot z8.s, z6.b, z0.b[3]\n"
+ "ld1b { z7.b }, p2/Z, [x14, #1, MUL VL]\n"
+ "ld1b { z6.b }, p2/Z, [x14, #2, MUL VL]\n"
+ "sdot z9.s, z7.b, z0.b[3]\n"
+ "ld1b { z7.b }, p2/Z, [x14, #3, MUL VL]\n"
+ "addvl x14, x14, #4\n"
+ "sdot z10.s, z6.b, z0.b[3]\n"
+ "sdot z11.s, z7.b, z0.b[3]\n"
+ "10:" // Height 1: Multiply loop: multiply skip
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x12, x12, #0x1\n"
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "cmp x12, x19\n"
+ "bne 5b\n"
+ "prfm pstl1keep, [x13, #0x0]\n"
+ "ld1w { z0.s }, p2/Z, [x16]\n"
+ "add z8.s, z8.s, z0.s\n"
+ "ld1w { z1.s }, p2/Z, [x16, #1, MUL VL]\n"
+ "ld1w { z2.s }, p2/Z, [x16, #2, MUL VL]\n"
+ "add z9.s, z9.s, z1.s\n"
+ "ld1w { z3.s }, p2/Z, [x16, #3, MUL VL]\n"
+ "addvl x16, x16, #4\n"
+ "add z10.s, z10.s, z2.s\n"
+ "add z11.s, z11.s, z3.s\n"
+ "tbz %x[flags], #4, 11f\n"
+ "ld1w { z0.s }, p2/Z, [x17]\n"
+ "ld1w { z4.s }, p2/Z, [x8]\n"
+ "ld1w { z1.s }, p2/Z, [x17, #1, MUL VL]\n"
+ "ld1w { z5.s }, p2/Z, [x8, #1, MUL VL]\n"
+ "ld1w { z2.s }, p2/Z, [x17, #2, MUL VL]\n"
+ "ld1w { z6.s }, p2/Z, [x8, #2, MUL VL]\n"
+ "ld1w { z3.s }, p2/Z, [x17, #3, MUL VL]\n"
+ "addvl x17, x17, #4\n"
+ "ld1w { z7.s }, p2/Z, [x8, #3, MUL VL]\n"
+ "addvl x8, x8, #4\n"
+ "b 12f\n"
+ "11:" // Height 1: per layer parameters
+ "add x19, %x[qp], %[per_layer_right_shift]\n"
+ "ld1rw { z0.s }, p2/Z, [x19]\n"
+ "mov z1.d, z0.d\n"
+ "add x19, %x[qp], %[per_layer_mul]\n"
+ "ld1rw { z4.s }, p2/Z, [x19]\n"
+ "mov z2.d, z0.d\n"
+ "mov z3.d, z0.d\n"
+ "mov z5.d, z4.d\n"
+ "mov z6.d, z4.d\n"
+ "mov z7.d, z4.d\n"
+ "12:" // Height 1: parameters loaded
+ ".inst 0x04a47508 // sqrdmulh z8.s, z8.s, z4.s\n"
+ ".inst 0x04a57529 // sqrdmulh z9.s, z9.s, z5.s\n"
+ ".inst 0x04a6754a // sqrdmulh z10.s, z10.s, z6.s\n"
+ ".inst 0x04a7756b // sqrdmulh z11.s, z11.s, z7.s\n"
+ "tbz %x[flags], #5, 13f\n"
+ "and z4.d, z8.d, z0.d\n"
+ "asr z4.s, z4.s, #0x1f\n"
+ "and z5.d, z9.d, z1.d\n"
+ "and z6.d, z10.d, z2.d\n"
+ "asr z5.s, z5.s, #0x1f\n"
+ "and z7.d, z11.d, z3.d\n"
+ "asr z6.s, z6.s, #0x1f\n"
+ "sqadd z8.s, z8.s, z4.s\n"
+ "asr z7.s, z7.s, #0x1f\n"
+ "sqadd z9.s, z9.s, z5.s\n"
+ "sqadd z10.s, z10.s, z6.s\n"
+ "sqadd z11.s, z11.s, z7.s\n"
+ "13:" // Height 1: no shift correction
+ ".inst 0x44828808 // srshl z8.s, p2/M, z8.s, z0.s\n"
+ "add x19, %x[qp], %[c_offset]\n"
+ "ld1rw { z4.s }, p2/Z, [x19]\n"
+ ".inst 0x44828829 // srshl z9.s, p2/M, z9.s, z1.s\n"
+ "add x19, %x[qp], %[minval]\n"
+ ".inst 0x4482884a // srshl z10.s, p2/M, z10.s, z2.s\n"
+ "ld1rw { z5.s }, p2/Z, [x19]\n"
+ "add x19, %x[qp], %[maxval]\n"
+ ".inst 0x4482886b // srshl z11.s, p2/M, z11.s, z3.s\n"
+ "ld1rw { z6.s }, p2/Z, [x19]\n"
+ "add z8.s, z8.s, z4.s\n"
+ "add z9.s, z9.s, z4.s\n"
+ "add z10.s, z10.s, z4.s\n"
+ "add z11.s, z11.s, z4.s\n"
+ "smin z8.s, p2/M, z8.s, z6.s\n"
+ "smin z9.s, p2/M, z9.s, z6.s\n"
+ "smin z10.s, p2/M, z10.s, z6.s\n"
+ "smin z11.s, p2/M, z11.s, z6.s\n"
+ "smax z8.s, p2/M, z8.s, z5.s\n"
+ "smax z9.s, p2/M, z9.s, z5.s\n"
+ "smax z10.s, p2/M, z10.s, z5.s\n"
+ "smax z11.s, p2/M, z11.s, z5.s\n"
+ "uzp1 z8.h, z8.h, z9.h\n"
+ "uzp1 z9.h, z10.h, z11.h\n"
+ "uzp1 z8.b, z8.b, z9.b\n"
+ "st1b { z8.b }, p1, [x13]\n"
+ "addvl x13, x13, #1\n"
+ "14:" // Height 1: Writeback done
+ "mov x19, #0x0\n"
+ "incw x19, ALL, MUL #4\n"
+ "subs x15, x15, x19\n"
+ "bgt 3b\n"
+ "b 86f\n"
+ "15:" // Height 2
+ "ldr x8, [%x[args_ptr], %[offsetof_multiplier_ptr]]\n"
+ "mov x16, %x[col_bias]\n"
+ "ldr x17, [%x[args_ptr], %[offsetof_shift_ptr]]\n"
+ "ldr x15, [%x[args_ptr], %[offsetof_N]]\n"
+ "ldr x14, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 16f\n"
+ "ldr x13, [%x[output_ptr], #0x0]\n"
+ "add x13, x13, x19\n"
+ "ldr x9, [%x[output_ptr], #0x8]\n"
+ "add x9, x9, x19\n"
+ "b 17f\n"
+ "16:" // Height 2: setup direct output
+ "mov x13, %x[output_ptr]\n"
+ "add x9, x13, x19\n"
+ "17:" // Height 2: Column loop
+ "mov z8.s, #0x0\n"
+ "mov x19, #0x0\n"
+ "mov z9.s, #0x0\n"
+ "whilelt p1.b, x19, x15\n"
+ "mov z10.s, #0x0\n"
+ "mov z11.s, #0x0\n"
+ "mov z12.s, #0x0\n"
+ "mov z13.s, #0x0\n"
+ "mov z14.s, #0x0\n"
+ "mov z15.s, #0x0\n"
+ "18:" // Height 2: setup done
+ "mov x12, #0x0\n"
+ "19:" // Height 2: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w11, [x20, x12, LSL #0x2]\n"
+ "tbz %x[flags], #3, 20f\n"
+ "ldr x20, [%x[input_ptr], x12, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x10, [x20, #0x0]\n"
+ "ldr x28, [x20, #0x8]\n"
+ "cbnz x12, 21f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x10, x10, x19\n"
+ "add x28, x28, x19\n"
+ "b 21f\n"
+ "20:" // Height 2: setup direct input
+ "mov x10, %x[input_ptr]\n"
+ "add x28, x10, x19\n"
+ "21:" // Height 2: input setup done
+ "cmp x11, #0x10\n"
+ "ble 23f\n"
+ "22:" // Height 2: Multiply loop: Main loop head
+ "ld1b { z6.b }, p2/Z, [x14]\n"
+ "whilelt p0.b, XZR, x11\n"
+ "ld1b { z7.b }, p2/Z, [x14, #1, MUL VL]\n"
+ "sub x11, x11, #0x10\n"
+ "ld1rqb { z0.b }, p0/Z, [x10]\n"
+ "sdot z8.s, z6.b, z0.b[0]\n"
+ "ld1rqb { z1.b }, p0/Z, [x28]\n"
+ "add x10, x10, #0x10\n"
+ "sdot z9.s, z7.b, z0.b[0]\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x28, x28, #0x10\n"
+ "sdot z12.s, z6.b, z1.b[0]\n"
+ "ld1b { z6.b }, p2/Z, [x14, #2, MUL VL]\n"
+ "cmp x11, #0x10\n"
+ "sdot z13.s, z7.b, z1.b[0]\n"
+ "ld1b { z7.b }, p2/Z, [x14, #3, MUL VL]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "sdot z10.s, z6.b, z0.b[0]\n"
+ "sdot z14.s, z6.b, z1.b[0]\n"
+ "ld1b { z6.b }, p2/Z, [x14, #4, MUL VL]\n"
+ "sdot z11.s, z7.b, z0.b[0]\n"
+ "sdot z15.s, z7.b, z1.b[0]\n"
+ "ld1b { z7.b }, p2/Z, [x14, #5, MUL VL]\n"
+ "sdot z8.s, z6.b, z0.b[1]\n"
+ "sdot z12.s, z6.b, z1.b[1]\n"
+ "ld1b { z6.b }, p2/Z, [x14, #6, MUL VL]\n"
+ "sdot z9.s, z7.b, z0.b[1]\n"
+ "sdot z13.s, z7.b, z1.b[1]\n"
+ "ld1b { z7.b }, p2/Z, [x14, #7, MUL VL]\n"
+ "addvl x14, x14, #16\n"
+ "sdot z10.s, z6.b, z0.b[1]\n"
+ "sdot z14.s, z6.b, z1.b[1]\n"
+ "ld1b { z6.b }, p2/Z, [x14, #-8, MUL VL]\n"
+ "sdot z11.s, z7.b, z0.b[1]\n"
+ "sdot z15.s, z7.b, z1.b[1]\n"
+ "ld1b { z7.b }, p2/Z, [x14, #-7, MUL VL]\n"
+ "sdot z8.s, z6.b, z0.b[2]\n"
+ "sdot z12.s, z6.b, z1.b[2]\n"
+ "ld1b { z6.b }, p2/Z, [x14, #-6, MUL VL]\n"
+ "sdot z9.s, z7.b, z0.b[2]\n"
+ "sdot z13.s, z7.b, z1.b[2]\n"
+ "ld1b { z7.b }, p2/Z, [x14, #-5, MUL VL]\n"
+ "sdot z10.s, z6.b, z0.b[2]\n"
+ "sdot z14.s, z6.b, z1.b[2]\n"
+ "ld1b { z6.b }, p2/Z, [x14, #-4, MUL VL]\n"
+ "sdot z11.s, z7.b, z0.b[2]\n"
+ "sdot z15.s, z7.b, z1.b[2]\n"
+ "ld1b { z7.b }, p2/Z, [x14, #-3, MUL VL]\n"
+ "sdot z8.s, z6.b, z0.b[3]\n"
+ "sdot z12.s, z6.b, z1.b[3]\n"
+ "ld1b { z6.b }, p2/Z, [x14, #-2, MUL VL]\n"
+ "sdot z9.s, z7.b, z0.b[3]\n"
+ "sdot z13.s, z7.b, z1.b[3]\n"
+ "ld1b { z7.b }, p2/Z, [x14, #-1, MUL VL]\n"
+ "sdot z10.s, z6.b, z0.b[3]\n"
+ "sdot z14.s, z6.b, z1.b[3]\n"
+ "sdot z11.s, z7.b, z0.b[3]\n"
+ "sdot z15.s, z7.b, z1.b[3]\n"
+ "bgt 22b\n"
+ "23:" // Height 2: Multiply loop: Single iteration only
+ "ld1b { z6.b }, p2/Z, [x14]\n"
+ "whilelt p0.b, XZR, x11\n"
+ "ld1b { z7.b }, p2/Z, [x14, #1, MUL VL]\n"
+ "subs x11, x11, #0x4\n"
+ "ld1rqb { z0.b }, p0/Z, [x10]\n"
+ "sdot z8.s, z6.b, z0.b[0]\n"
+ "ld1rqb { z1.b }, p0/Z, [x28]\n"
+ "add x10, x10, #0x10\n"
+ "sdot z9.s, z7.b, z0.b[0]\n"
+ "add x28, x28, #0x10\n"
+ "sdot z12.s, z6.b, z1.b[0]\n"
+ "ld1b { z6.b }, p2/Z, [x14, #2, MUL VL]\n"
+ "sdot z13.s, z7.b, z1.b[0]\n"
+ "ld1b { z7.b }, p2/Z, [x14, #3, MUL VL]\n"
+ "addvl x14, x14, #4\n"
+ "sdot z10.s, z6.b, z0.b[0]\n"
+ "sdot z14.s, z6.b, z1.b[0]\n"
+ "sdot z11.s, z7.b, z0.b[0]\n"
+ "sdot z15.s, z7.b, z1.b[0]\n"
+ "ble 24f\n"
+ "ld1b { z6.b }, p2/Z, [x14]\n"
+ "sdot z8.s, z6.b, z0.b[1]\n"
+ "ld1b { z7.b }, p2/Z, [x14, #1, MUL VL]\n"
+ "subs x11, x11, #0x4\n"
+ "sdot z12.s, z6.b, z1.b[1]\n"
+ "ld1b { z6.b }, p2/Z, [x14, #2, MUL VL]\n"
+ "sdot z9.s, z7.b, z0.b[1]\n"
+ "sdot z13.s, z7.b, z1.b[1]\n"
+ "ld1b { z7.b }, p2/Z, [x14, #3, MUL VL]\n"
+ "addvl x14, x14, #4\n"
+ "sdot z10.s, z6.b, z0.b[1]\n"
+ "sdot z14.s, z6.b, z1.b[1]\n"
+ "sdot z11.s, z7.b, z0.b[1]\n"
+ "sdot z15.s, z7.b, z1.b[1]\n"
+ "ble 24f\n"
+ "ld1b { z6.b }, p2/Z, [x14]\n"
+ "sdot z8.s, z6.b, z0.b[2]\n"
+ "ld1b { z7.b }, p2/Z, [x14, #1, MUL VL]\n"
+ "subs x11, x11, #0x4\n"
+ "sdot z12.s, z6.b, z1.b[2]\n"
+ "ld1b { z6.b }, p2/Z, [x14, #2, MUL VL]\n"
+ "sdot z9.s, z7.b, z0.b[2]\n"
+ "sdot z13.s, z7.b, z1.b[2]\n"
+ "ld1b { z7.b }, p2/Z, [x14, #3, MUL VL]\n"
+ "addvl x14, x14, #4\n"
+ "sdot z10.s, z6.b, z0.b[2]\n"
+ "sdot z14.s, z6.b, z1.b[2]\n"
+ "sdot z11.s, z7.b, z0.b[2]\n"
+ "sdot z15.s, z7.b, z1.b[2]\n"
+ "ble 24f\n"
+ "ld1b { z6.b }, p2/Z, [x14]\n"
+ "sdot z8.s, z6.b, z0.b[3]\n"
+ "ld1b { z7.b }, p2/Z, [x14, #1, MUL VL]\n"
+ "sdot z12.s, z6.b, z1.b[3]\n"
+ "ld1b { z6.b }, p2/Z, [x14, #2, MUL VL]\n"
+ "sdot z9.s, z7.b, z0.b[3]\n"
+ "sdot z13.s, z7.b, z1.b[3]\n"
+ "ld1b { z7.b }, p2/Z, [x14, #3, MUL VL]\n"
+ "addvl x14, x14, #4\n"
+ "sdot z10.s, z6.b, z0.b[3]\n"
+ "sdot z14.s, z6.b, z1.b[3]\n"
+ "sdot z11.s, z7.b, z0.b[3]\n"
+ "sdot z15.s, z7.b, z1.b[3]\n"
+ "24:" // Height 2: Multiply loop: multiply skip
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x12, x12, #0x1\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "cmp x12, x19\n"
+ "bne 19b\n"
+ "prfm pstl1keep, [x13, #0x0]\n"
+ "prfm pstl1keep, [x9, #0x0]\n"
+ "ld1w { z0.s }, p2/Z, [x16]\n"
+ "add z8.s, z8.s, z0.s\n"
+ "ld1w { z1.s }, p2/Z, [x16, #1, MUL VL]\n"
+ "add z12.s, z12.s, z0.s\n"
+ "ld1w { z2.s }, p2/Z, [x16, #2, MUL VL]\n"
+ "ld1w { z3.s }, p2/Z, [x16, #3, MUL VL]\n"
+ "add z9.s, z9.s, z1.s\n"
+ "addvl x16, x16, #4\n"
+ "add z13.s, z13.s, z1.s\n"
+ "add z10.s, z10.s, z2.s\n"
+ "add z11.s, z11.s, z3.s\n"
+ "add z14.s, z14.s, z2.s\n"
+ "add z15.s, z15.s, z3.s\n"
+ "tbz %x[flags], #4, 25f\n"
+ "ld1w { z0.s }, p2/Z, [x17]\n"
+ "ld1w { z4.s }, p2/Z, [x8]\n"
+ "ld1w { z1.s }, p2/Z, [x17, #1, MUL VL]\n"
+ "ld1w { z5.s }, p2/Z, [x8, #1, MUL VL]\n"
+ "ld1w { z2.s }, p2/Z, [x17, #2, MUL VL]\n"
+ "ld1w { z6.s }, p2/Z, [x8, #2, MUL VL]\n"
+ "ld1w { z3.s }, p2/Z, [x17, #3, MUL VL]\n"
+ "addvl x17, x17, #4\n"
+ "ld1w { z7.s }, p2/Z, [x8, #3, MUL VL]\n"
+ "addvl x8, x8, #4\n"
+ "b 26f\n"
+ "25:" // Height 2: per layer parameters
+ "add x19, %x[qp], %[per_layer_right_shift]\n"
+ "ld1rw { z0.s }, p2/Z, [x19]\n"
+ "mov z1.d, z0.d\n"
+ "add x19, %x[qp], %[per_layer_mul]\n"
+ "ld1rw { z4.s }, p2/Z, [x19]\n"
+ "mov z2.d, z0.d\n"
+ "mov z3.d, z0.d\n"
+ "mov z5.d, z4.d\n"
+ "mov z6.d, z4.d\n"
+ "mov z7.d, z4.d\n"
+ "26:" // Height 2: parameters loaded
+ ".inst 0x04a47508 // sqrdmulh z8.s, z8.s, z4.s\n"
+ ".inst 0x04a57529 // sqrdmulh z9.s, z9.s, z5.s\n"
+ ".inst 0x04a6754a // sqrdmulh z10.s, z10.s, z6.s\n"
+ ".inst 0x04a7756b // sqrdmulh z11.s, z11.s, z7.s\n"
+ ".inst 0x04a4758c // sqrdmulh z12.s, z12.s, z4.s\n"
+ ".inst 0x04a575ad // sqrdmulh z13.s, z13.s, z5.s\n"
+ ".inst 0x04a675ce // sqrdmulh z14.s, z14.s, z6.s\n"
+ ".inst 0x04a775ef // sqrdmulh z15.s, z15.s, z7.s\n"
+ "tbz %x[flags], #5, 27f\n"
+ "and z4.d, z8.d, z0.d\n"
+ "asr z4.s, z4.s, #0x1f\n"
+ "and z5.d, z9.d, z1.d\n"
+ "and z6.d, z10.d, z2.d\n"
+ "asr z5.s, z5.s, #0x1f\n"
+ "and z7.d, z11.d, z3.d\n"
+ "asr z6.s, z6.s, #0x1f\n"
+ "sqadd z8.s, z8.s, z4.s\n"
+ "asr z7.s, z7.s, #0x1f\n"
+ "and z4.d, z12.d, z0.d\n"
+ "sqadd z9.s, z9.s, z5.s\n"
+ "asr z4.s, z4.s, #0x1f\n"
+ "sqadd z10.s, z10.s, z6.s\n"
+ "and z5.d, z13.d, z1.d\n"
+ "asr z5.s, z5.s, #0x1f\n"
+ "sqadd z11.s, z11.s, z7.s\n"
+ "and z6.d, z14.d, z2.d\n"
+ "asr z6.s, z6.s, #0x1f\n"
+ "sqadd z12.s, z12.s, z4.s\n"
+ "and z7.d, z15.d, z3.d\n"
+ "asr z7.s, z7.s, #0x1f\n"
+ "sqadd z13.s, z13.s, z5.s\n"
+ "sqadd z14.s, z14.s, z6.s\n"
+ "sqadd z15.s, z15.s, z7.s\n"
+ "27:" // Height 2: no shift correction
+ ".inst 0x44828808 // srshl z8.s, p2/M, z8.s, z0.s\n"
+ "add x19, %x[qp], %[c_offset]\n"
+ "ld1rw { z4.s }, p2/Z, [x19]\n"
+ ".inst 0x44828829 // srshl z9.s, p2/M, z9.s, z1.s\n"
+ "add x19, %x[qp], %[minval]\n"
+ ".inst 0x4482884a // srshl z10.s, p2/M, z10.s, z2.s\n"
+ "ld1rw { z5.s }, p2/Z, [x19]\n"
+ "add x19, %x[qp], %[maxval]\n"
+ ".inst 0x4482886b // srshl z11.s, p2/M, z11.s, z3.s\n"
+ "ld1rw { z6.s }, p2/Z, [x19]\n"
+ ".inst 0x4482880c // srshl z12.s, p2/M, z12.s, z0.s\n"
+ "add z8.s, z8.s, z4.s\n"
+ "add z9.s, z9.s, z4.s\n"
+ "add z10.s, z10.s, z4.s\n"
+ "add z11.s, z11.s, z4.s\n"
+ "add z12.s, z12.s, z4.s\n"
+ "smin z8.s, p2/M, z8.s, z6.s\n"
+ "smin z9.s, p2/M, z9.s, z6.s\n"
+ "smin z10.s, p2/M, z10.s, z6.s\n"
+ "smin z11.s, p2/M, z11.s, z6.s\n"
+ "smax z8.s, p2/M, z8.s, z5.s\n"
+ "smax z9.s, p2/M, z9.s, z5.s\n"
+ "smax z10.s, p2/M, z10.s, z5.s\n"
+ "smax z11.s, p2/M, z11.s, z5.s\n"
+ "smin z12.s, p2/M, z12.s, z6.s\n"
+ "uzp1 z8.h, z8.h, z9.h\n"
+ ".inst 0x4482882d // srshl z13.s, p2/M, z13.s, z1.s\n"
+ "uzp1 z9.h, z10.h, z11.h\n"
+ "smax z12.s, p2/M, z12.s, z5.s\n"
+ "uzp1 z8.b, z8.b, z9.b\n"
+ "st1b { z8.b }, p1, [x13]\n"
+ "add z13.s, z13.s, z4.s\n"
+ "addvl x13, x13, #1\n"
+ ".inst 0x4482884e // srshl z14.s, p2/M, z14.s, z2.s\n"
+ ".inst 0x4482886f // srshl z15.s, p2/M, z15.s, z3.s\n"
+ "smin z13.s, p2/M, z13.s, z6.s\n"
+ "add z14.s, z14.s, z4.s\n"
+ "add z15.s, z15.s, z4.s\n"
+ "smax z13.s, p2/M, z13.s, z5.s\n"
+ "smin z14.s, p2/M, z14.s, z6.s\n"
+ "smin z15.s, p2/M, z15.s, z6.s\n"
+ "uzp1 z12.h, z12.h, z13.h\n"
+ "smax z14.s, p2/M, z14.s, z5.s\n"
+ "smax z15.s, p2/M, z15.s, z5.s\n"
+ "uzp1 z13.h, z14.h, z15.h\n"
+ "uzp1 z12.b, z12.b, z13.b\n"
+ "st1b { z12.b }, p1, [x9]\n"
+ "addvl x9, x9, #1\n"
+ "28:" // Height 2: Writeback done
+ "mov x19, #0x0\n"
+ "incw x19, ALL, MUL #4\n"
+ "subs x15, x15, x19\n"
+ "bgt 17b\n"
+ "b 86f\n"
+ "29:" // Height 3
+ "ldr x8, [%x[args_ptr], %[offsetof_multiplier_ptr]]\n"
+ "mov x16, %x[col_bias]\n"
+ "ldr x17, [%x[args_ptr], %[offsetof_shift_ptr]]\n"
+ "ldr x15, [%x[args_ptr], %[offsetof_N]]\n"
+ "ldr x14, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 30f\n"
+ "ldr x13, [%x[output_ptr], #0x0]\n"
+ "add x13, x13, x19\n"
+ "ldr x9, [%x[output_ptr], #0x8]\n"
+ "ldr x27, [%x[output_ptr], #0x10]\n"
+ "add x9, x9, x19\n"
+ "add x27, x27, x19\n"
+ "b 31f\n"
+ "30:" // Height 3: setup direct output
+ "mov x13, %x[output_ptr]\n"
+ "add x9, x13, x19\n"
+ "add x27, x9, x19\n"
+ "31:" // Height 3: Column loop
+ "mov z8.s, #0x0\n"
+ "mov x19, #0x0\n"
+ "mov z9.s, #0x0\n"
+ "whilelt p1.b, x19, x15\n"
+ "mov z10.s, #0x0\n"
+ "mov z11.s, #0x0\n"
+ "mov z12.s, #0x0\n"
+ "mov z13.s, #0x0\n"
+ "mov z14.s, #0x0\n"
+ "mov z15.s, #0x0\n"
+ "mov z16.s, #0x0\n"
+ "mov z17.s, #0x0\n"
+ "mov z18.s, #0x0\n"
+ "mov z19.s, #0x0\n"
+ "32:" // Height 3: setup done
+ "mov x12, #0x0\n"
+ "33:" // Height 3: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w11, [x20, x12, LSL #0x2]\n"
+ "tbz %x[flags], #3, 34f\n"
+ "ldr x20, [%x[input_ptr], x12, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x10, [x20, #0x0]\n"
+ "ldr x28, [x20, #0x8]\n"
+ "ldr x26, [x20, #0x10]\n"
+ "cbnz x12, 35f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x10, x10, x19\n"
+ "add x28, x28, x19\n"
+ "add x26, x26, x19\n"
+ "b 35f\n"
+ "34:" // Height 3: setup direct input
+ "mov x10, %x[input_ptr]\n"
+ "add x28, x10, x19\n"
+ "add x26, x28, x19\n"
+ "35:" // Height 3: input setup done
+ "cmp x11, #0x10\n"
+ "ble 37f\n"
+ "36:" // Height 3: Multiply loop: Main loop head
+ "ld1b { z6.b }, p2/Z, [x14]\n"
+ "whilelt p0.b, XZR, x11\n"
+ "ld1b { z7.b }, p2/Z, [x14, #1, MUL VL]\n"
+ "sub x11, x11, #0x10\n"
+ "ld1rqb { z0.b }, p0/Z, [x10]\n"
+ "sdot z8.s, z6.b, z0.b[0]\n"
+ "ld1rqb { z1.b }, p0/Z, [x28]\n"
+ "add x10, x10, #0x10\n"
+ "sdot z9.s, z7.b, z0.b[0]\n"
+ "ld1rqb { z2.b }, p0/Z, [x26]\n"
+ "add x28, x28, #0x10\n"
+ "sdot z12.s, z6.b, z1.b[0]\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x26, x26, #0x10\n"
+ "sdot z16.s, z6.b, z2.b[0]\n"
+ "ld1b { z6.b }, p2/Z, [x14, #2, MUL VL]\n"
+ "cmp x11, #0x10\n"
+ "sdot z13.s, z7.b, z1.b[0]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "sdot z17.s, z7.b, z2.b[0]\n"
+ "ld1b { z7.b }, p2/Z, [x14, #3, MUL VL]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "sdot z10.s, z6.b, z0.b[0]\n"
+ "sdot z14.s, z6.b, z1.b[0]\n"
+ "sdot z18.s, z6.b, z2.b[0]\n"
+ "ld1b { z6.b }, p2/Z, [x14, #4, MUL VL]\n"
+ "sdot z11.s, z7.b, z0.b[0]\n"
+ "sdot z15.s, z7.b, z1.b[0]\n"
+ "sdot z19.s, z7.b, z2.b[0]\n"
+ "ld1b { z7.b }, p2/Z, [x14, #5, MUL VL]\n"
+ "sdot z8.s, z6.b, z0.b[1]\n"
+ "sdot z12.s, z6.b, z1.b[1]\n"
+ "sdot z16.s, z6.b, z2.b[1]\n"
+ "ld1b { z6.b }, p2/Z, [x14, #6, MUL VL]\n"
+ "sdot z9.s, z7.b, z0.b[1]\n"
+ "sdot z13.s, z7.b, z1.b[1]\n"
+ "sdot z17.s, z7.b, z2.b[1]\n"
+ "ld1b { z7.b }, p2/Z, [x14, #7, MUL VL]\n"
+ "addvl x14, x14, #16\n"
+ "sdot z10.s, z6.b, z0.b[1]\n"
+ "sdot z14.s, z6.b, z1.b[1]\n"
+ "sdot z18.s, z6.b, z2.b[1]\n"
+ "ld1b { z6.b }, p2/Z, [x14, #-8, MUL VL]\n"
+ "sdot z11.s, z7.b, z0.b[1]\n"
+ "sdot z15.s, z7.b, z1.b[1]\n"
+ "sdot z19.s, z7.b, z2.b[1]\n"
+ "ld1b { z7.b }, p2/Z, [x14, #-7, MUL VL]\n"
+ "sdot z8.s, z6.b, z0.b[2]\n"
+ "sdot z12.s, z6.b, z1.b[2]\n"
+ "sdot z16.s, z6.b, z2.b[2]\n"
+ "ld1b { z6.b }, p2/Z, [x14, #-6, MUL VL]\n"
+ "sdot z9.s, z7.b, z0.b[2]\n"
+ "sdot z13.s, z7.b, z1.b[2]\n"
+ "sdot z17.s, z7.b, z2.b[2]\n"
+ "ld1b { z7.b }, p2/Z, [x14, #-5, MUL VL]\n"
+ "sdot z10.s, z6.b, z0.b[2]\n"
+ "sdot z14.s, z6.b, z1.b[2]\n"
+ "sdot z18.s, z6.b, z2.b[2]\n"
+ "ld1b { z6.b }, p2/Z, [x14, #-4, MUL VL]\n"
+ "sdot z11.s, z7.b, z0.b[2]\n"
+ "sdot z15.s, z7.b, z1.b[2]\n"
+ "sdot z19.s, z7.b, z2.b[2]\n"
+ "ld1b { z7.b }, p2/Z, [x14, #-3, MUL VL]\n"
+ "sdot z8.s, z6.b, z0.b[3]\n"
+ "sdot z12.s, z6.b, z1.b[3]\n"
+ "sdot z16.s, z6.b, z2.b[3]\n"
+ "ld1b { z6.b }, p2/Z, [x14, #-2, MUL VL]\n"
+ "sdot z9.s, z7.b, z0.b[3]\n"
+ "sdot z13.s, z7.b, z1.b[3]\n"
+ "sdot z17.s, z7.b, z2.b[3]\n"
+ "ld1b { z7.b }, p2/Z, [x14, #-1, MUL VL]\n"
+ "sdot z10.s, z6.b, z0.b[3]\n"
+ "sdot z14.s, z6.b, z1.b[3]\n"
+ "sdot z18.s, z6.b, z2.b[3]\n"
+ "sdot z11.s, z7.b, z0.b[3]\n"
+ "sdot z15.s, z7.b, z1.b[3]\n"
+ "sdot z19.s, z7.b, z2.b[3]\n"
+ "bgt 36b\n"
+ "37:" // Height 3: Multiply loop: Single iteration only
+ "ld1b { z6.b }, p2/Z, [x14]\n"
+ "whilelt p0.b, XZR, x11\n"
+ "ld1b { z7.b }, p2/Z, [x14, #1, MUL VL]\n"
+ "subs x11, x11, #0x4\n"
+ "ld1rqb { z0.b }, p0/Z, [x10]\n"
+ "sdot z8.s, z6.b, z0.b[0]\n"
+ "ld1rqb { z1.b }, p0/Z, [x28]\n"
+ "add x10, x10, #0x10\n"
+ "sdot z9.s, z7.b, z0.b[0]\n"
+ "ld1rqb { z2.b }, p0/Z, [x26]\n"
+ "add x28, x28, #0x10\n"
+ "sdot z12.s, z6.b, z1.b[0]\n"
+ "add x26, x26, #0x10\n"
+ "sdot z13.s, z7.b, z1.b[0]\n"
+ "sdot z16.s, z6.b, z2.b[0]\n"
+ "ld1b { z6.b }, p2/Z, [x14, #2, MUL VL]\n"
+ "sdot z17.s, z7.b, z2.b[0]\n"
+ "ld1b { z7.b }, p2/Z, [x14, #3, MUL VL]\n"
+ "addvl x14, x14, #4\n"
+ "sdot z10.s, z6.b, z0.b[0]\n"
+ "sdot z14.s, z6.b, z1.b[0]\n"
+ "sdot z18.s, z6.b, z2.b[0]\n"
+ "sdot z11.s, z7.b, z0.b[0]\n"
+ "sdot z15.s, z7.b, z1.b[0]\n"
+ "sdot z19.s, z7.b, z2.b[0]\n"
+ "ble 38f\n"
+ "ld1b { z6.b }, p2/Z, [x14]\n"
+ "sdot z8.s, z6.b, z0.b[1]\n"
+ "ld1b { z7.b }, p2/Z, [x14, #1, MUL VL]\n"
+ "subs x11, x11, #0x4\n"
+ "sdot z12.s, z6.b, z1.b[1]\n"
+ "sdot z16.s, z6.b, z2.b[1]\n"
+ "ld1b { z6.b }, p2/Z, [x14, #2, MUL VL]\n"
+ "sdot z9.s, z7.b, z0.b[1]\n"
+ "sdot z13.s, z7.b, z1.b[1]\n"
+ "sdot z17.s, z7.b, z2.b[1]\n"
+ "ld1b { z7.b }, p2/Z, [x14, #3, MUL VL]\n"
+ "addvl x14, x14, #4\n"
+ "sdot z10.s, z6.b, z0.b[1]\n"
+ "sdot z14.s, z6.b, z1.b[1]\n"
+ "sdot z18.s, z6.b, z2.b[1]\n"
+ "sdot z11.s, z7.b, z0.b[1]\n"
+ "sdot z15.s, z7.b, z1.b[1]\n"
+ "sdot z19.s, z7.b, z2.b[1]\n"
+ "ble 38f\n"
+ "ld1b { z6.b }, p2/Z, [x14]\n"
+ "sdot z8.s, z6.b, z0.b[2]\n"
+ "ld1b { z7.b }, p2/Z, [x14, #1, MUL VL]\n"
+ "subs x11, x11, #0x4\n"
+ "sdot z12.s, z6.b, z1.b[2]\n"
+ "sdot z16.s, z6.b, z2.b[2]\n"
+ "ld1b { z6.b }, p2/Z, [x14, #2, MUL VL]\n"
+ "sdot z9.s, z7.b, z0.b[2]\n"
+ "sdot z13.s, z7.b, z1.b[2]\n"
+ "sdot z17.s, z7.b, z2.b[2]\n"
+ "ld1b { z7.b }, p2/Z, [x14, #3, MUL VL]\n"
+ "addvl x14, x14, #4\n"
+ "sdot z10.s, z6.b, z0.b[2]\n"
+ "sdot z14.s, z6.b, z1.b[2]\n"
+ "sdot z18.s, z6.b, z2.b[2]\n"
+ "sdot z11.s, z7.b, z0.b[2]\n"
+ "sdot z15.s, z7.b, z1.b[2]\n"
+ "sdot z19.s, z7.b, z2.b[2]\n"
+ "ble 38f\n"
+ "ld1b { z6.b }, p2/Z, [x14]\n"
+ "sdot z8.s, z6.b, z0.b[3]\n"
+ "ld1b { z7.b }, p2/Z, [x14, #1, MUL VL]\n"
+ "sdot z12.s, z6.b, z1.b[3]\n"
+ "sdot z16.s, z6.b, z2.b[3]\n"
+ "ld1b { z6.b }, p2/Z, [x14, #2, MUL VL]\n"
+ "sdot z9.s, z7.b, z0.b[3]\n"
+ "sdot z13.s, z7.b, z1.b[3]\n"
+ "sdot z17.s, z7.b, z2.b[3]\n"
+ "ld1b { z7.b }, p2/Z, [x14, #3, MUL VL]\n"
+ "addvl x14, x14, #4\n"
+ "sdot z10.s, z6.b, z0.b[3]\n"
+ "sdot z14.s, z6.b, z1.b[3]\n"
+ "sdot z18.s, z6.b, z2.b[3]\n"
+ "sdot z11.s, z7.b, z0.b[3]\n"
+ "sdot z15.s, z7.b, z1.b[3]\n"
+ "sdot z19.s, z7.b, z2.b[3]\n"
+ "38:" // Height 3: Multiply loop: multiply skip
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x12, x12, #0x1\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "cmp x12, x19\n"
+ "bne 33b\n"
+ "prfm pstl1keep, [x13, #0x0]\n"
+ "prfm pstl1keep, [x9, #0x0]\n"
+ "prfm pstl1keep, [x27, #0x0]\n"
+ "ld1w { z0.s }, p2/Z, [x16]\n"
+ "add z8.s, z8.s, z0.s\n"
+ "ld1w { z1.s }, p2/Z, [x16, #1, MUL VL]\n"
+ "add z12.s, z12.s, z0.s\n"
+ "ld1w { z2.s }, p2/Z, [x16, #2, MUL VL]\n"
+ "add z16.s, z16.s, z0.s\n"
+ "ld1w { z3.s }, p2/Z, [x16, #3, MUL VL]\n"
+ "addvl x16, x16, #4\n"
+ "add z9.s, z9.s, z1.s\n"
+ "add z13.s, z13.s, z1.s\n"
+ "add z10.s, z10.s, z2.s\n"
+ "add z11.s, z11.s, z3.s\n"
+ "add z14.s, z14.s, z2.s\n"
+ "add z15.s, z15.s, z3.s\n"
+ "add z17.s, z17.s, z1.s\n"
+ "add z18.s, z18.s, z2.s\n"
+ "add z19.s, z19.s, z3.s\n"
+ "tbz %x[flags], #4, 39f\n"
+ "ld1w { z0.s }, p2/Z, [x17]\n"
+ "ld1w { z4.s }, p2/Z, [x8]\n"
+ "ld1w { z1.s }, p2/Z, [x17, #1, MUL VL]\n"
+ "ld1w { z5.s }, p2/Z, [x8, #1, MUL VL]\n"
+ "ld1w { z2.s }, p2/Z, [x17, #2, MUL VL]\n"
+ "ld1w { z6.s }, p2/Z, [x8, #2, MUL VL]\n"
+ "ld1w { z3.s }, p2/Z, [x17, #3, MUL VL]\n"
+ "addvl x17, x17, #4\n"
+ "ld1w { z7.s }, p2/Z, [x8, #3, MUL VL]\n"
+ "addvl x8, x8, #4\n"
+ "b 40f\n"
+ "39:" // Height 3: per layer parameters
+ "add x19, %x[qp], %[per_layer_right_shift]\n"
+ "ld1rw { z0.s }, p2/Z, [x19]\n"
+ "mov z1.d, z0.d\n"
+ "add x19, %x[qp], %[per_layer_mul]\n"
+ "ld1rw { z4.s }, p2/Z, [x19]\n"
+ "mov z2.d, z0.d\n"
+ "mov z3.d, z0.d\n"
+ "mov z5.d, z4.d\n"
+ "mov z6.d, z4.d\n"
+ "mov z7.d, z4.d\n"
+ "40:" // Height 3: parameters loaded
+ ".inst 0x04a47508 // sqrdmulh z8.s, z8.s, z4.s\n"
+ ".inst 0x04a57529 // sqrdmulh z9.s, z9.s, z5.s\n"
+ ".inst 0x04a6754a // sqrdmulh z10.s, z10.s, z6.s\n"
+ ".inst 0x04a7756b // sqrdmulh z11.s, z11.s, z7.s\n"
+ ".inst 0x04a4758c // sqrdmulh z12.s, z12.s, z4.s\n"
+ ".inst 0x04a575ad // sqrdmulh z13.s, z13.s, z5.s\n"
+ ".inst 0x04a675ce // sqrdmulh z14.s, z14.s, z6.s\n"
+ ".inst 0x04a775ef // sqrdmulh z15.s, z15.s, z7.s\n"
+ ".inst 0x04a47610 // sqrdmulh z16.s, z16.s, z4.s\n"
+ ".inst 0x04a57631 // sqrdmulh z17.s, z17.s, z5.s\n"
+ ".inst 0x04a67652 // sqrdmulh z18.s, z18.s, z6.s\n"
+ ".inst 0x04a77673 // sqrdmulh z19.s, z19.s, z7.s\n"
+ "tbz %x[flags], #5, 41f\n"
+ "and z4.d, z8.d, z0.d\n"
+ "asr z4.s, z4.s, #0x1f\n"
+ "and z5.d, z9.d, z1.d\n"
+ "and z6.d, z10.d, z2.d\n"
+ "asr z5.s, z5.s, #0x1f\n"
+ "and z7.d, z11.d, z3.d\n"
+ "asr z6.s, z6.s, #0x1f\n"
+ "sqadd z8.s, z8.s, z4.s\n"
+ "asr z7.s, z7.s, #0x1f\n"
+ "and z4.d, z12.d, z0.d\n"
+ "sqadd z9.s, z9.s, z5.s\n"
+ "asr z4.s, z4.s, #0x1f\n"
+ "sqadd z10.s, z10.s, z6.s\n"
+ "and z5.d, z13.d, z1.d\n"
+ "asr z5.s, z5.s, #0x1f\n"
+ "sqadd z11.s, z11.s, z7.s\n"
+ "and z6.d, z14.d, z2.d\n"
+ "asr z6.s, z6.s, #0x1f\n"
+ "sqadd z12.s, z12.s, z4.s\n"
+ "and z7.d, z15.d, z3.d\n"
+ "asr z7.s, z7.s, #0x1f\n"
+ "sqadd z13.s, z13.s, z5.s\n"
+ "and z4.d, z16.d, z0.d\n"
+ "asr z4.s, z4.s, #0x1f\n"
+ "sqadd z14.s, z14.s, z6.s\n"
+ "and z5.d, z17.d, z1.d\n"
+ "asr z5.s, z5.s, #0x1f\n"
+ "sqadd z15.s, z15.s, z7.s\n"
+ "and z6.d, z18.d, z2.d\n"
+ "asr z6.s, z6.s, #0x1f\n"
+ "sqadd z16.s, z16.s, z4.s\n"
+ "and z7.d, z19.d, z3.d\n"
+ "asr z7.s, z7.s, #0x1f\n"
+ "sqadd z17.s, z17.s, z5.s\n"
+ "sqadd z18.s, z18.s, z6.s\n"
+ "sqadd z19.s, z19.s, z7.s\n"
+ "41:" // Height 3: no shift correction
+ ".inst 0x44828808 // srshl z8.s, p2/M, z8.s, z0.s\n"
+ "add x19, %x[qp], %[c_offset]\n"
+ "ld1rw { z4.s }, p2/Z, [x19]\n"
+ ".inst 0x44828829 // srshl z9.s, p2/M, z9.s, z1.s\n"
+ "add x19, %x[qp], %[minval]\n"
+ ".inst 0x4482884a // srshl z10.s, p2/M, z10.s, z2.s\n"
+ "ld1rw { z5.s }, p2/Z, [x19]\n"
+ "add x19, %x[qp], %[maxval]\n"
+ ".inst 0x4482886b // srshl z11.s, p2/M, z11.s, z3.s\n"
+ "ld1rw { z6.s }, p2/Z, [x19]\n"
+ ".inst 0x4482880c // srshl z12.s, p2/M, z12.s, z0.s\n"
+ "add z8.s, z8.s, z4.s\n"
+ "add z9.s, z9.s, z4.s\n"
+ "add z10.s, z10.s, z4.s\n"
+ "add z11.s, z11.s, z4.s\n"
+ "add z12.s, z12.s, z4.s\n"
+ "smin z8.s, p2/M, z8.s, z6.s\n"
+ "smin z9.s, p2/M, z9.s, z6.s\n"
+ "smin z10.s, p2/M, z10.s, z6.s\n"
+ "smin z11.s, p2/M, z11.s, z6.s\n"
+ "smax z8.s, p2/M, z8.s, z5.s\n"
+ "smax z9.s, p2/M, z9.s, z5.s\n"
+ "smax z10.s, p2/M, z10.s, z5.s\n"
+ "smax z11.s, p2/M, z11.s, z5.s\n"
+ "smin z12.s, p2/M, z12.s, z6.s\n"
+ "uzp1 z8.h, z8.h, z9.h\n"
+ ".inst 0x4482882d // srshl z13.s, p2/M, z13.s, z1.s\n"
+ "uzp1 z9.h, z10.h, z11.h\n"
+ "smax z12.s, p2/M, z12.s, z5.s\n"
+ "uzp1 z8.b, z8.b, z9.b\n"
+ "st1b { z8.b }, p1, [x13]\n"
+ "add z13.s, z13.s, z4.s\n"
+ "addvl x13, x13, #1\n"
+ ".inst 0x4482884e // srshl z14.s, p2/M, z14.s, z2.s\n"
+ ".inst 0x4482886f // srshl z15.s, p2/M, z15.s, z3.s\n"
+ ".inst 0x44828810 // srshl z16.s, p2/M, z16.s, z0.s\n"
+ "smin z13.s, p2/M, z13.s, z6.s\n"
+ ".inst 0x44828831 // srshl z17.s, p2/M, z17.s, z1.s\n"
+ "add z14.s, z14.s, z4.s\n"
+ "add z15.s, z15.s, z4.s\n"
+ "add z16.s, z16.s, z4.s\n"
+ "add z17.s, z17.s, z4.s\n"
+ "smax z13.s, p2/M, z13.s, z5.s\n"
+ "smin z14.s, p2/M, z14.s, z6.s\n"
+ "smin z15.s, p2/M, z15.s, z6.s\n"
+ "smin z16.s, p2/M, z16.s, z6.s\n"
+ "uzp1 z12.h, z12.h, z13.h\n"
+ "smax z14.s, p2/M, z14.s, z5.s\n"
+ "smax z15.s, p2/M, z15.s, z5.s\n"
+ "smax z16.s, p2/M, z16.s, z5.s\n"
+ "smin z17.s, p2/M, z17.s, z6.s\n"
+ ".inst 0x44828852 // srshl z18.s, p2/M, z18.s, z2.s\n"
+ "uzp1 z13.h, z14.h, z15.h\n"
+ ".inst 0x44828873 // srshl z19.s, p2/M, z19.s, z3.s\n"
+ "uzp1 z12.b, z12.b, z13.b\n"
+ "st1b { z12.b }, p1, [x9]\n"
+ "add z18.s, z18.s, z4.s\n"
+ "addvl x9, x9, #1\n"
+ "add z19.s, z19.s, z4.s\n"
+ "smax z17.s, p2/M, z17.s, z5.s\n"
+ "smin z18.s, p2/M, z18.s, z6.s\n"
+ "smin z19.s, p2/M, z19.s, z6.s\n"
+ "uzp1 z16.h, z16.h, z17.h\n"
+ "smax z18.s, p2/M, z18.s, z5.s\n"
+ "smax z19.s, p2/M, z19.s, z5.s\n"
+ "uzp1 z17.h, z18.h, z19.h\n"
+ "uzp1 z16.b, z16.b, z17.b\n"
+ "st1b { z16.b }, p1, [x27]\n"
+ "addvl x27, x27, #1\n"
+ "42:" // Height 3: Writeback done
+ "mov x19, #0x0\n"
+ "incw x19, ALL, MUL #4\n"
+ "subs x15, x15, x19\n"
+ "bgt 31b\n"
+ "b 86f\n"
+ "43:" // Height 4
+ "ldr x8, [%x[args_ptr], %[offsetof_multiplier_ptr]]\n"
+ "mov x16, %x[col_bias]\n"
+ "ldr x17, [%x[args_ptr], %[offsetof_shift_ptr]]\n"
+ "ldr x15, [%x[args_ptr], %[offsetof_N]]\n"
+ "ldr x14, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 44f\n"
+ "ldr x13, [%x[output_ptr], #0x0]\n"
+ "add x13, x13, x19\n"
+ "ldr x9, [%x[output_ptr], #0x8]\n"
+ "ldr x27, [%x[output_ptr], #0x10]\n"
+ "add x9, x9, x19\n"
+ "ldr x25, [%x[output_ptr], #0x18]\n"
+ "add x27, x27, x19\n"
+ "add x25, x25, x19\n"
+ "b 45f\n"
+ "44:" // Height 4: setup direct output
+ "mov x13, %x[output_ptr]\n"
+ "add x9, x13, x19\n"
+ "add x27, x9, x19\n"
+ "add x25, x27, x19\n"
+ "45:" // Height 4: Column loop
+ "mov z8.s, #0x0\n"
+ "mov x19, #0x0\n"
+ "mov z9.s, #0x0\n"
+ "whilelt p1.b, x19, x15\n"
+ "mov z10.s, #0x0\n"
+ "mov z11.s, #0x0\n"
+ "mov z12.s, #0x0\n"
+ "mov z13.s, #0x0\n"
+ "mov z14.s, #0x0\n"
+ "mov z15.s, #0x0\n"
+ "mov z16.s, #0x0\n"
+ "mov z17.s, #0x0\n"
+ "mov z18.s, #0x0\n"
+ "mov z19.s, #0x0\n"
+ "mov z20.s, #0x0\n"
+ "mov z21.s, #0x0\n"
+ "mov z22.s, #0x0\n"
+ "mov z23.s, #0x0\n"
+ "46:" // Height 4: setup done
+ "mov x12, #0x0\n"
+ "47:" // Height 4: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w11, [x20, x12, LSL #0x2]\n"
+ "tbz %x[flags], #3, 48f\n"
+ "ldr x20, [%x[input_ptr], x12, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x10, [x20, #0x0]\n"
+ "ldr x28, [x20, #0x8]\n"
+ "ldr x26, [x20, #0x10]\n"
+ "ldr x24, [x20, #0x18]\n"
+ "cbnz x12, 49f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x10, x10, x19\n"
+ "add x28, x28, x19\n"
+ "add x26, x26, x19\n"
+ "add x24, x24, x19\n"
+ "b 49f\n"
+ "48:" // Height 4: setup direct input
+ "mov x10, %x[input_ptr]\n"
+ "add x28, x10, x19\n"
+ "add x26, x28, x19\n"
+ "add x24, x26, x19\n"
+ "49:" // Height 4: input setup done
+ "cmp x11, #0x10\n"
+ "ble 51f\n"
+ "50:" // Height 4: Multiply loop: Main loop head
+ "ld1b { z6.b }, p2/Z, [x14]\n"
+ "whilelt p0.b, XZR, x11\n"
+ "ld1b { z7.b }, p2/Z, [x14, #1, MUL VL]\n"
+ "sub x11, x11, #0x10\n"
+ "ld1rqb { z0.b }, p0/Z, [x10]\n"
+ "sdot z8.s, z6.b, z0.b[0]\n"
+ "ld1rqb { z1.b }, p0/Z, [x28]\n"
+ "add x10, x10, #0x10\n"
+ "sdot z9.s, z7.b, z0.b[0]\n"
+ "ld1rqb { z2.b }, p0/Z, [x26]\n"
+ "add x28, x28, #0x10\n"
+ "sdot z12.s, z6.b, z1.b[0]\n"
+ "ld1rqb { z3.b }, p0/Z, [x24]\n"
+ "add x26, x26, #0x10\n"
+ "sdot z16.s, z6.b, z2.b[0]\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x24, x24, #0x10\n"
+ "sdot z13.s, z7.b, z1.b[0]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "cmp x11, #0x10\n"
+ "sdot z20.s, z6.b, z3.b[0]\n"
+ "ld1b { z6.b }, p2/Z, [x14, #2, MUL VL]\n"
+ "sdot z17.s, z7.b, z2.b[0]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "sdot z21.s, z7.b, z3.b[0]\n"
+ "ld1b { z7.b }, p2/Z, [x14, #3, MUL VL]\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "sdot z10.s, z6.b, z0.b[0]\n"
+ "sdot z14.s, z6.b, z1.b[0]\n"
+ "sdot z18.s, z6.b, z2.b[0]\n"
+ "sdot z22.s, z6.b, z3.b[0]\n"
+ "ld1b { z6.b }, p2/Z, [x14, #4, MUL VL]\n"
+ "sdot z11.s, z7.b, z0.b[0]\n"
+ "sdot z15.s, z7.b, z1.b[0]\n"
+ "sdot z19.s, z7.b, z2.b[0]\n"
+ "sdot z23.s, z7.b, z3.b[0]\n"
+ "ld1b { z7.b }, p2/Z, [x14, #5, MUL VL]\n"
+ "sdot z8.s, z6.b, z0.b[1]\n"
+ "sdot z12.s, z6.b, z1.b[1]\n"
+ "sdot z16.s, z6.b, z2.b[1]\n"
+ "sdot z20.s, z6.b, z3.b[1]\n"
+ "ld1b { z6.b }, p2/Z, [x14, #6, MUL VL]\n"
+ "sdot z9.s, z7.b, z0.b[1]\n"
+ "sdot z13.s, z7.b, z1.b[1]\n"
+ "sdot z17.s, z7.b, z2.b[1]\n"
+ "sdot z21.s, z7.b, z3.b[1]\n"
+ "ld1b { z7.b }, p2/Z, [x14, #7, MUL VL]\n"
+ "addvl x14, x14, #16\n"
+ "sdot z10.s, z6.b, z0.b[1]\n"
+ "sdot z14.s, z6.b, z1.b[1]\n"
+ "sdot z18.s, z6.b, z2.b[1]\n"
+ "sdot z22.s, z6.b, z3.b[1]\n"
+ "ld1b { z6.b }, p2/Z, [x14, #-8, MUL VL]\n"
+ "sdot z11.s, z7.b, z0.b[1]\n"
+ "sdot z15.s, z7.b, z1.b[1]\n"
+ "sdot z19.s, z7.b, z2.b[1]\n"
+ "sdot z23.s, z7.b, z3.b[1]\n"
+ "ld1b { z7.b }, p2/Z, [x14, #-7, MUL VL]\n"
+ "sdot z8.s, z6.b, z0.b[2]\n"
+ "sdot z12.s, z6.b, z1.b[2]\n"
+ "sdot z16.s, z6.b, z2.b[2]\n"
+ "sdot z20.s, z6.b, z3.b[2]\n"
+ "ld1b { z6.b }, p2/Z, [x14, #-6, MUL VL]\n"
+ "sdot z9.s, z7.b, z0.b[2]\n"
+ "sdot z13.s, z7.b, z1.b[2]\n"
+ "sdot z17.s, z7.b, z2.b[2]\n"
+ "sdot z21.s, z7.b, z3.b[2]\n"
+ "ld1b { z7.b }, p2/Z, [x14, #-5, MUL VL]\n"
+ "sdot z10.s, z6.b, z0.b[2]\n"
+ "sdot z14.s, z6.b, z1.b[2]\n"
+ "sdot z18.s, z6.b, z2.b[2]\n"
+ "sdot z22.s, z6.b, z3.b[2]\n"
+ "ld1b { z6.b }, p2/Z, [x14, #-4, MUL VL]\n"
+ "sdot z11.s, z7.b, z0.b[2]\n"
+ "sdot z15.s, z7.b, z1.b[2]\n"
+ "sdot z19.s, z7.b, z2.b[2]\n"
+ "sdot z23.s, z7.b, z3.b[2]\n"
+ "ld1b { z7.b }, p2/Z, [x14, #-3, MUL VL]\n"
+ "sdot z8.s, z6.b, z0.b[3]\n"
+ "sdot z12.s, z6.b, z1.b[3]\n"
+ "sdot z16.s, z6.b, z2.b[3]\n"
+ "sdot z20.s, z6.b, z3.b[3]\n"
+ "ld1b { z6.b }, p2/Z, [x14, #-2, MUL VL]\n"
+ "sdot z9.s, z7.b, z0.b[3]\n"
+ "sdot z13.s, z7.b, z1.b[3]\n"
+ "sdot z17.s, z7.b, z2.b[3]\n"
+ "sdot z21.s, z7.b, z3.b[3]\n"
+ "ld1b { z7.b }, p2/Z, [x14, #-1, MUL VL]\n"
+ "sdot z10.s, z6.b, z0.b[3]\n"
+ "sdot z14.s, z6.b, z1.b[3]\n"
+ "sdot z18.s, z6.b, z2.b[3]\n"
+ "sdot z22.s, z6.b, z3.b[3]\n"
+ "sdot z11.s, z7.b, z0.b[3]\n"
+ "sdot z15.s, z7.b, z1.b[3]\n"
+ "sdot z19.s, z7.b, z2.b[3]\n"
+ "sdot z23.s, z7.b, z3.b[3]\n"
+ "bgt 50b\n"
+ "51:" // Height 4: Multiply loop: Single iteration only
+ "ld1b { z6.b }, p2/Z, [x14]\n"
+ "whilelt p0.b, XZR, x11\n"
+ "ld1b { z7.b }, p2/Z, [x14, #1, MUL VL]\n"
+ "subs x11, x11, #0x4\n"
+ "ld1rqb { z0.b }, p0/Z, [x10]\n"
+ "sdot z8.s, z6.b, z0.b[0]\n"
+ "ld1rqb { z1.b }, p0/Z, [x28]\n"
+ "add x10, x10, #0x10\n"
+ "sdot z9.s, z7.b, z0.b[0]\n"
+ "ld1rqb { z2.b }, p0/Z, [x26]\n"
+ "add x28, x28, #0x10\n"
+ "sdot z12.s, z6.b, z1.b[0]\n"
+ "ld1rqb { z3.b }, p0/Z, [x24]\n"
+ "add x26, x26, #0x10\n"
+ "sdot z16.s, z6.b, z2.b[0]\n"
+ "add x24, x24, #0x10\n"
+ "sdot z13.s, z7.b, z1.b[0]\n"
+ "sdot z17.s, z7.b, z2.b[0]\n"
+ "sdot z20.s, z6.b, z3.b[0]\n"
+ "ld1b { z6.b }, p2/Z, [x14, #2, MUL VL]\n"
+ "sdot z21.s, z7.b, z3.b[0]\n"
+ "ld1b { z7.b }, p2/Z, [x14, #3, MUL VL]\n"
+ "addvl x14, x14, #4\n"
+ "sdot z10.s, z6.b, z0.b[0]\n"
+ "sdot z14.s, z6.b, z1.b[0]\n"
+ "sdot z18.s, z6.b, z2.b[0]\n"
+ "sdot z22.s, z6.b, z3.b[0]\n"
+ "sdot z11.s, z7.b, z0.b[0]\n"
+ "sdot z15.s, z7.b, z1.b[0]\n"
+ "sdot z19.s, z7.b, z2.b[0]\n"
+ "sdot z23.s, z7.b, z3.b[0]\n"
+ "ble 52f\n"
+ "ld1b { z6.b }, p2/Z, [x14]\n"
+ "sdot z8.s, z6.b, z0.b[1]\n"
+ "ld1b { z7.b }, p2/Z, [x14, #1, MUL VL]\n"
+ "subs x11, x11, #0x4\n"
+ "sdot z12.s, z6.b, z1.b[1]\n"
+ "sdot z16.s, z6.b, z2.b[1]\n"
+ "sdot z20.s, z6.b, z3.b[1]\n"
+ "ld1b { z6.b }, p2/Z, [x14, #2, MUL VL]\n"
+ "sdot z9.s, z7.b, z0.b[1]\n"
+ "sdot z13.s, z7.b, z1.b[1]\n"
+ "sdot z17.s, z7.b, z2.b[1]\n"
+ "sdot z21.s, z7.b, z3.b[1]\n"
+ "ld1b { z7.b }, p2/Z, [x14, #3, MUL VL]\n"
+ "addvl x14, x14, #4\n"
+ "sdot z10.s, z6.b, z0.b[1]\n"
+ "sdot z14.s, z6.b, z1.b[1]\n"
+ "sdot z18.s, z6.b, z2.b[1]\n"
+ "sdot z22.s, z6.b, z3.b[1]\n"
+ "sdot z11.s, z7.b, z0.b[1]\n"
+ "sdot z15.s, z7.b, z1.b[1]\n"
+ "sdot z19.s, z7.b, z2.b[1]\n"
+ "sdot z23.s, z7.b, z3.b[1]\n"
+ "ble 52f\n"
+ "ld1b { z6.b }, p2/Z, [x14]\n"
+ "sdot z8.s, z6.b, z0.b[2]\n"
+ "ld1b { z7.b }, p2/Z, [x14, #1, MUL VL]\n"
+ "subs x11, x11, #0x4\n"
+ "sdot z12.s, z6.b, z1.b[2]\n"
+ "sdot z16.s, z6.b, z2.b[2]\n"
+ "sdot z20.s, z6.b, z3.b[2]\n"
+ "ld1b { z6.b }, p2/Z, [x14, #2, MUL VL]\n"
+ "sdot z9.s, z7.b, z0.b[2]\n"
+ "sdot z13.s, z7.b, z1.b[2]\n"
+ "sdot z17.s, z7.b, z2.b[2]\n"
+ "sdot z21.s, z7.b, z3.b[2]\n"
+ "ld1b { z7.b }, p2/Z, [x14, #3, MUL VL]\n"
+ "addvl x14, x14, #4\n"
+ "sdot z10.s, z6.b, z0.b[2]\n"
+ "sdot z14.s, z6.b, z1.b[2]\n"
+ "sdot z18.s, z6.b, z2.b[2]\n"
+ "sdot z22.s, z6.b, z3.b[2]\n"
+ "sdot z11.s, z7.b, z0.b[2]\n"
+ "sdot z15.s, z7.b, z1.b[2]\n"
+ "sdot z19.s, z7.b, z2.b[2]\n"
+ "sdot z23.s, z7.b, z3.b[2]\n"
+ "ble 52f\n"
+ "ld1b { z6.b }, p2/Z, [x14]\n"
+ "sdot z8.s, z6.b, z0.b[3]\n"
+ "ld1b { z7.b }, p2/Z, [x14, #1, MUL VL]\n"
+ "sdot z12.s, z6.b, z1.b[3]\n"
+ "sdot z16.s, z6.b, z2.b[3]\n"
+ "sdot z20.s, z6.b, z3.b[3]\n"
+ "ld1b { z6.b }, p2/Z, [x14, #2, MUL VL]\n"
+ "sdot z9.s, z7.b, z0.b[3]\n"
+ "sdot z13.s, z7.b, z1.b[3]\n"
+ "sdot z17.s, z7.b, z2.b[3]\n"
+ "sdot z21.s, z7.b, z3.b[3]\n"
+ "ld1b { z7.b }, p2/Z, [x14, #3, MUL VL]\n"
+ "addvl x14, x14, #4\n"
+ "sdot z10.s, z6.b, z0.b[3]\n"
+ "sdot z14.s, z6.b, z1.b[3]\n"
+ "sdot z18.s, z6.b, z2.b[3]\n"
+ "sdot z22.s, z6.b, z3.b[3]\n"
+ "sdot z11.s, z7.b, z0.b[3]\n"
+ "sdot z15.s, z7.b, z1.b[3]\n"
+ "sdot z19.s, z7.b, z2.b[3]\n"
+ "sdot z23.s, z7.b, z3.b[3]\n"
+ "52:" // Height 4: Multiply loop: multiply skip
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x12, x12, #0x1\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "cmp x12, x19\n"
+ "bne 47b\n"
+ "prfm pstl1keep, [x13, #0x0]\n"
+ "prfm pstl1keep, [x9, #0x0]\n"
+ "prfm pstl1keep, [x27, #0x0]\n"
+ "prfm pstl1keep, [x25, #0x0]\n"
+ "ld1w { z0.s }, p2/Z, [x16]\n"
+ "add z8.s, z8.s, z0.s\n"
+ "ld1w { z1.s }, p2/Z, [x16, #1, MUL VL]\n"
+ "add z12.s, z12.s, z0.s\n"
+ "ld1w { z2.s }, p2/Z, [x16, #2, MUL VL]\n"
+ "add z16.s, z16.s, z0.s\n"
+ "ld1w { z3.s }, p2/Z, [x16, #3, MUL VL]\n"
+ "addvl x16, x16, #4\n"
+ "add z9.s, z9.s, z1.s\n"
+ "add z13.s, z13.s, z1.s\n"
+ "add z10.s, z10.s, z2.s\n"
+ "add z11.s, z11.s, z3.s\n"
+ "add z14.s, z14.s, z2.s\n"
+ "add z15.s, z15.s, z3.s\n"
+ "add z17.s, z17.s, z1.s\n"
+ "add z18.s, z18.s, z2.s\n"
+ "add z19.s, z19.s, z3.s\n"
+ "add z20.s, z20.s, z0.s\n"
+ "add z21.s, z21.s, z1.s\n"
+ "add z22.s, z22.s, z2.s\n"
+ "add z23.s, z23.s, z3.s\n"
+ "tbz %x[flags], #4, 53f\n"
+ "ld1w { z0.s }, p2/Z, [x17]\n"
+ "ld1w { z4.s }, p2/Z, [x8]\n"
+ "ld1w { z1.s }, p2/Z, [x17, #1, MUL VL]\n"
+ "ld1w { z5.s }, p2/Z, [x8, #1, MUL VL]\n"
+ "ld1w { z2.s }, p2/Z, [x17, #2, MUL VL]\n"
+ "ld1w { z6.s }, p2/Z, [x8, #2, MUL VL]\n"
+ "ld1w { z3.s }, p2/Z, [x17, #3, MUL VL]\n"
+ "addvl x17, x17, #4\n"
+ "ld1w { z7.s }, p2/Z, [x8, #3, MUL VL]\n"
+ "addvl x8, x8, #4\n"
+ "b 54f\n"
+ "53:" // Height 4: per layer parameters
+ "add x19, %x[qp], %[per_layer_right_shift]\n"
+ "ld1rw { z0.s }, p2/Z, [x19]\n"
+ "mov z1.d, z0.d\n"
+ "add x19, %x[qp], %[per_layer_mul]\n"
+ "ld1rw { z4.s }, p2/Z, [x19]\n"
+ "mov z2.d, z0.d\n"
+ "mov z3.d, z0.d\n"
+ "mov z5.d, z4.d\n"
+ "mov z6.d, z4.d\n"
+ "mov z7.d, z4.d\n"
+ "54:" // Height 4: parameters loaded
+ ".inst 0x04a47508 // sqrdmulh z8.s, z8.s, z4.s\n"
+ ".inst 0x04a57529 // sqrdmulh z9.s, z9.s, z5.s\n"
+ ".inst 0x04a6754a // sqrdmulh z10.s, z10.s, z6.s\n"
+ ".inst 0x04a7756b // sqrdmulh z11.s, z11.s, z7.s\n"
+ ".inst 0x04a4758c // sqrdmulh z12.s, z12.s, z4.s\n"
+ ".inst 0x04a575ad // sqrdmulh z13.s, z13.s, z5.s\n"
+ ".inst 0x04a675ce // sqrdmulh z14.s, z14.s, z6.s\n"
+ ".inst 0x04a775ef // sqrdmulh z15.s, z15.s, z7.s\n"
+ ".inst 0x04a47610 // sqrdmulh z16.s, z16.s, z4.s\n"
+ ".inst 0x04a57631 // sqrdmulh z17.s, z17.s, z5.s\n"
+ ".inst 0x04a67652 // sqrdmulh z18.s, z18.s, z6.s\n"
+ ".inst 0x04a77673 // sqrdmulh z19.s, z19.s, z7.s\n"
+ ".inst 0x04a47694 // sqrdmulh z20.s, z20.s, z4.s\n"
+ ".inst 0x04a576b5 // sqrdmulh z21.s, z21.s, z5.s\n"
+ ".inst 0x04a676d6 // sqrdmulh z22.s, z22.s, z6.s\n"
+ ".inst 0x04a776f7 // sqrdmulh z23.s, z23.s, z7.s\n"
+ "tbz %x[flags], #5, 55f\n"
+ "and z4.d, z8.d, z0.d\n"
+ "asr z4.s, z4.s, #0x1f\n"
+ "and z5.d, z9.d, z1.d\n"
+ "and z6.d, z10.d, z2.d\n"
+ "asr z5.s, z5.s, #0x1f\n"
+ "and z7.d, z11.d, z3.d\n"
+ "asr z6.s, z6.s, #0x1f\n"
+ "sqadd z8.s, z8.s, z4.s\n"
+ "asr z7.s, z7.s, #0x1f\n"
+ "and z4.d, z12.d, z0.d\n"
+ "sqadd z9.s, z9.s, z5.s\n"
+ "asr z4.s, z4.s, #0x1f\n"
+ "sqadd z10.s, z10.s, z6.s\n"
+ "and z5.d, z13.d, z1.d\n"
+ "asr z5.s, z5.s, #0x1f\n"
+ "sqadd z11.s, z11.s, z7.s\n"
+ "and z6.d, z14.d, z2.d\n"
+ "asr z6.s, z6.s, #0x1f\n"
+ "sqadd z12.s, z12.s, z4.s\n"
+ "and z7.d, z15.d, z3.d\n"
+ "asr z7.s, z7.s, #0x1f\n"
+ "sqadd z13.s, z13.s, z5.s\n"
+ "and z4.d, z16.d, z0.d\n"
+ "asr z4.s, z4.s, #0x1f\n"
+ "sqadd z14.s, z14.s, z6.s\n"
+ "and z5.d, z17.d, z1.d\n"
+ "asr z5.s, z5.s, #0x1f\n"
+ "sqadd z15.s, z15.s, z7.s\n"
+ "and z6.d, z18.d, z2.d\n"
+ "asr z6.s, z6.s, #0x1f\n"
+ "sqadd z16.s, z16.s, z4.s\n"
+ "and z7.d, z19.d, z3.d\n"
+ "asr z7.s, z7.s, #0x1f\n"
+ "sqadd z17.s, z17.s, z5.s\n"
+ "and z4.d, z20.d, z0.d\n"
+ "asr z4.s, z4.s, #0x1f\n"
+ "sqadd z18.s, z18.s, z6.s\n"
+ "and z5.d, z21.d, z1.d\n"
+ "asr z5.s, z5.s, #0x1f\n"
+ "sqadd z19.s, z19.s, z7.s\n"
+ "and z6.d, z22.d, z2.d\n"
+ "asr z6.s, z6.s, #0x1f\n"
+ "sqadd z20.s, z20.s, z4.s\n"
+ "and z7.d, z23.d, z3.d\n"
+ "asr z7.s, z7.s, #0x1f\n"
+ "sqadd z21.s, z21.s, z5.s\n"
+ "sqadd z22.s, z22.s, z6.s\n"
+ "sqadd z23.s, z23.s, z7.s\n"
+ "55:" // Height 4: no shift correction
+ ".inst 0x44828808 // srshl z8.s, p2/M, z8.s, z0.s\n"
+ "add x19, %x[qp], %[c_offset]\n"
+ "ld1rw { z4.s }, p2/Z, [x19]\n"
+ ".inst 0x44828829 // srshl z9.s, p2/M, z9.s, z1.s\n"
+ "add x19, %x[qp], %[minval]\n"
+ ".inst 0x4482884a // srshl z10.s, p2/M, z10.s, z2.s\n"
+ "ld1rw { z5.s }, p2/Z, [x19]\n"
+ "add x19, %x[qp], %[maxval]\n"
+ ".inst 0x4482886b // srshl z11.s, p2/M, z11.s, z3.s\n"
+ "ld1rw { z6.s }, p2/Z, [x19]\n"
+ ".inst 0x4482880c // srshl z12.s, p2/M, z12.s, z0.s\n"
+ "add z8.s, z8.s, z4.s\n"
+ "add z9.s, z9.s, z4.s\n"
+ "add z10.s, z10.s, z4.s\n"
+ "add z11.s, z11.s, z4.s\n"
+ "add z12.s, z12.s, z4.s\n"
+ "smin z8.s, p2/M, z8.s, z6.s\n"
+ "smin z9.s, p2/M, z9.s, z6.s\n"
+ "smin z10.s, p2/M, z10.s, z6.s\n"
+ "smin z11.s, p2/M, z11.s, z6.s\n"
+ "smax z8.s, p2/M, z8.s, z5.s\n"
+ "smax z9.s, p2/M, z9.s, z5.s\n"
+ "smax z10.s, p2/M, z10.s, z5.s\n"
+ "smax z11.s, p2/M, z11.s, z5.s\n"
+ "smin z12.s, p2/M, z12.s, z6.s\n"
+ "uzp1 z8.h, z8.h, z9.h\n"
+ ".inst 0x4482882d // srshl z13.s, p2/M, z13.s, z1.s\n"
+ "uzp1 z9.h, z10.h, z11.h\n"
+ "smax z12.s, p2/M, z12.s, z5.s\n"
+ "uzp1 z8.b, z8.b, z9.b\n"
+ "st1b { z8.b }, p1, [x13]\n"
+ "add z13.s, z13.s, z4.s\n"
+ "addvl x13, x13, #1\n"
+ ".inst 0x4482884e // srshl z14.s, p2/M, z14.s, z2.s\n"
+ ".inst 0x4482886f // srshl z15.s, p2/M, z15.s, z3.s\n"
+ ".inst 0x44828810 // srshl z16.s, p2/M, z16.s, z0.s\n"
+ "smin z13.s, p2/M, z13.s, z6.s\n"
+ ".inst 0x44828831 // srshl z17.s, p2/M, z17.s, z1.s\n"
+ "add z14.s, z14.s, z4.s\n"
+ "add z15.s, z15.s, z4.s\n"
+ "add z16.s, z16.s, z4.s\n"
+ "add z17.s, z17.s, z4.s\n"
+ "smax z13.s, p2/M, z13.s, z5.s\n"
+ "smin z14.s, p2/M, z14.s, z6.s\n"
+ "smin z15.s, p2/M, z15.s, z6.s\n"
+ "smin z16.s, p2/M, z16.s, z6.s\n"
+ "uzp1 z12.h, z12.h, z13.h\n"
+ "smax z14.s, p2/M, z14.s, z5.s\n"
+ "smax z15.s, p2/M, z15.s, z5.s\n"
+ "smax z16.s, p2/M, z16.s, z5.s\n"
+ "smin z17.s, p2/M, z17.s, z6.s\n"
+ ".inst 0x44828852 // srshl z18.s, p2/M, z18.s, z2.s\n"
+ "uzp1 z13.h, z14.h, z15.h\n"
+ ".inst 0x44828873 // srshl z19.s, p2/M, z19.s, z3.s\n"
+ "uzp1 z12.b, z12.b, z13.b\n"
+ "st1b { z12.b }, p1, [x9]\n"
+ "add z18.s, z18.s, z4.s\n"
+ "addvl x9, x9, #1\n"
+ "add z19.s, z19.s, z4.s\n"
+ "smax z17.s, p2/M, z17.s, z5.s\n"
+ ".inst 0x44828814 // srshl z20.s, p2/M, z20.s, z0.s\n"
+ "smin z18.s, p2/M, z18.s, z6.s\n"
+ "smin z19.s, p2/M, z19.s, z6.s\n"
+ "uzp1 z16.h, z16.h, z17.h\n"
+ "add z20.s, z20.s, z4.s\n"
+ "smax z18.s, p2/M, z18.s, z5.s\n"
+ "smax z19.s, p2/M, z19.s, z5.s\n"
+ "smin z20.s, p2/M, z20.s, z6.s\n"
+ ".inst 0x44828835 // srshl z21.s, p2/M, z21.s, z1.s\n"
+ ".inst 0x44828856 // srshl z22.s, p2/M, z22.s, z2.s\n"
+ "uzp1 z17.h, z18.h, z19.h\n"
+ "smax z20.s, p2/M, z20.s, z5.s\n"
+ "add z21.s, z21.s, z4.s\n"
+ "add z22.s, z22.s, z4.s\n"
+ "uzp1 z16.b, z16.b, z17.b\n"
+ "st1b { z16.b }, p1, [x27]\n"
+ "smin z21.s, p2/M, z21.s, z6.s\n"
+ "addvl x27, x27, #1\n"
+ "smin z22.s, p2/M, z22.s, z6.s\n"
+ ".inst 0x44828877 // srshl z23.s, p2/M, z23.s, z3.s\n"
+ "smax z21.s, p2/M, z21.s, z5.s\n"
+ "add z23.s, z23.s, z4.s\n"
+ "smax z22.s, p2/M, z22.s, z5.s\n"
+ "uzp1 z20.h, z20.h, z21.h\n"
+ "smin z23.s, p2/M, z23.s, z6.s\n"
+ "smax z23.s, p2/M, z23.s, z5.s\n"
+ "uzp1 z21.h, z22.h, z23.h\n"
+ "uzp1 z20.b, z20.b, z21.b\n"
+ "st1b { z20.b }, p1, [x25]\n"
+ "addvl x25, x25, #1\n"
+ "56:" // Height 4: Writeback done
+ "mov x19, #0x0\n"
+ "incw x19, ALL, MUL #4\n"
+ "subs x15, x15, x19\n"
+ "bgt 45b\n"
+ "b 86f\n"
+ "57:" // Height 5
+ "ldr x8, [%x[args_ptr], %[offsetof_multiplier_ptr]]\n"
+ "mov x16, %x[col_bias]\n"
+ "ldr x17, [%x[args_ptr], %[offsetof_shift_ptr]]\n"
+ "ldr x15, [%x[args_ptr], %[offsetof_N]]\n"
+ "ldr x14, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 58f\n"
+ "ldr x13, [%x[output_ptr], #0x0]\n"
+ "add x13, x13, x19\n"
+ "ldr x9, [%x[output_ptr], #0x8]\n"
+ "ldr x27, [%x[output_ptr], #0x10]\n"
+ "add x9, x9, x19\n"
+ "ldr x25, [%x[output_ptr], #0x18]\n"
+ "ldr x23, [%x[output_ptr], #0x20]\n"
+ "add x27, x27, x19\n"
+ "add x25, x25, x19\n"
+ "add x23, x23, x19\n"
+ "b 59f\n"
+ "58:" // Height 5: setup direct output
+ "mov x13, %x[output_ptr]\n"
+ "add x9, x13, x19\n"
+ "add x27, x9, x19\n"
+ "add x25, x27, x19\n"
+ "add x23, x25, x19\n"
+ "59:" // Height 5: Column loop
+ "mov z8.s, #0x0\n"
+ "mov x19, #0x0\n"
+ "mov z9.s, #0x0\n"
+ "whilelt p1.b, x19, x15\n"
+ "mov z10.s, #0x0\n"
+ "mov z11.s, #0x0\n"
+ "mov z12.s, #0x0\n"
+ "mov z13.s, #0x0\n"
+ "mov z14.s, #0x0\n"
+ "mov z15.s, #0x0\n"
+ "mov z16.s, #0x0\n"
+ "mov z17.s, #0x0\n"
+ "mov z18.s, #0x0\n"
+ "mov z19.s, #0x0\n"
+ "mov z20.s, #0x0\n"
+ "mov z21.s, #0x0\n"
+ "mov z22.s, #0x0\n"
+ "mov z23.s, #0x0\n"
+ "mov z24.s, #0x0\n"
+ "mov z25.s, #0x0\n"
+ "mov z26.s, #0x0\n"
+ "mov z27.s, #0x0\n"
+ "60:" // Height 5: setup done
+ "mov x12, #0x0\n"
+ "61:" // Height 5: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w11, [x20, x12, LSL #0x2]\n"
+ "tbz %x[flags], #3, 62f\n"
+ "ldr x20, [%x[input_ptr], x12, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x10, [x20, #0x0]\n"
+ "ldr x28, [x20, #0x8]\n"
+ "ldr x26, [x20, #0x10]\n"
+ "ldr x24, [x20, #0x18]\n"
+ "ldr x22, [x20, #0x20]\n"
+ "cbnz x12, 63f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x10, x10, x19\n"
+ "add x28, x28, x19\n"
+ "add x26, x26, x19\n"
+ "add x24, x24, x19\n"
+ "add x22, x22, x19\n"
+ "b 63f\n"
+ "62:" // Height 5: setup direct input
+ "mov x10, %x[input_ptr]\n"
+ "add x28, x10, x19\n"
+ "add x26, x28, x19\n"
+ "add x24, x26, x19\n"
+ "add x22, x24, x19\n"
+ "63:" // Height 5: input setup done
+ "cmp x11, #0x10\n"
+ "ble 65f\n"
+ "64:" // Height 5: Multiply loop: Main loop head
+ "ld1b { z6.b }, p2/Z, [x14]\n"
+ "whilelt p0.b, XZR, x11\n"
+ "ld1b { z7.b }, p2/Z, [x14, #1, MUL VL]\n"
+ "sub x11, x11, #0x10\n"
+ "ld1rqb { z0.b }, p0/Z, [x10]\n"
+ "sdot z8.s, z6.b, z0.b[0]\n"
+ "ld1rqb { z1.b }, p0/Z, [x28]\n"
+ "add x10, x10, #0x10\n"
+ "sdot z9.s, z7.b, z0.b[0]\n"
+ "ld1rqb { z2.b }, p0/Z, [x26]\n"
+ "add x28, x28, #0x10\n"
+ "sdot z12.s, z6.b, z1.b[0]\n"
+ "ld1rqb { z3.b }, p0/Z, [x24]\n"
+ "add x26, x26, #0x10\n"
+ "sdot z16.s, z6.b, z2.b[0]\n"
+ "ld1rqb { z4.b }, p0/Z, [x22]\n"
+ "add x24, x24, #0x10\n"
+ "sdot z13.s, z7.b, z1.b[0]\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x22, x22, #0x10\n"
+ "sdot z20.s, z6.b, z3.b[0]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "cmp x11, #0x10\n"
+ "sdot z24.s, z6.b, z4.b[0]\n"
+ "ld1b { z6.b }, p2/Z, [x14, #2, MUL VL]\n"
+ "sdot z17.s, z7.b, z2.b[0]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "sdot z21.s, z7.b, z3.b[0]\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "sdot z25.s, z7.b, z4.b[0]\n"
+ "ld1b { z7.b }, p2/Z, [x14, #3, MUL VL]\n"
+ "sdot z10.s, z6.b, z0.b[0]\n"
+ "prfm pldl1keep, [x22, #0x80]\n"
+ "sdot z14.s, z6.b, z1.b[0]\n"
+ "sdot z18.s, z6.b, z2.b[0]\n"
+ "sdot z22.s, z6.b, z3.b[0]\n"
+ "sdot z26.s, z6.b, z4.b[0]\n"
+ "ld1b { z6.b }, p2/Z, [x14, #4, MUL VL]\n"
+ "sdot z11.s, z7.b, z0.b[0]\n"
+ "sdot z15.s, z7.b, z1.b[0]\n"
+ "sdot z19.s, z7.b, z2.b[0]\n"
+ "sdot z23.s, z7.b, z3.b[0]\n"
+ "sdot z27.s, z7.b, z4.b[0]\n"
+ "ld1b { z7.b }, p2/Z, [x14, #5, MUL VL]\n"
+ "sdot z8.s, z6.b, z0.b[1]\n"
+ "sdot z12.s, z6.b, z1.b[1]\n"
+ "sdot z16.s, z6.b, z2.b[1]\n"
+ "sdot z20.s, z6.b, z3.b[1]\n"
+ "sdot z24.s, z6.b, z4.b[1]\n"
+ "ld1b { z6.b }, p2/Z, [x14, #6, MUL VL]\n"
+ "sdot z9.s, z7.b, z0.b[1]\n"
+ "sdot z13.s, z7.b, z1.b[1]\n"
+ "sdot z17.s, z7.b, z2.b[1]\n"
+ "sdot z21.s, z7.b, z3.b[1]\n"
+ "sdot z25.s, z7.b, z4.b[1]\n"
+ "ld1b { z7.b }, p2/Z, [x14, #7, MUL VL]\n"
+ "addvl x14, x14, #16\n"
+ "sdot z10.s, z6.b, z0.b[1]\n"
+ "sdot z14.s, z6.b, z1.b[1]\n"
+ "sdot z18.s, z6.b, z2.b[1]\n"
+ "sdot z22.s, z6.b, z3.b[1]\n"
+ "sdot z26.s, z6.b, z4.b[1]\n"
+ "ld1b { z6.b }, p2/Z, [x14, #-8, MUL VL]\n"
+ "sdot z11.s, z7.b, z0.b[1]\n"
+ "sdot z15.s, z7.b, z1.b[1]\n"
+ "sdot z19.s, z7.b, z2.b[1]\n"
+ "sdot z23.s, z7.b, z3.b[1]\n"
+ "sdot z27.s, z7.b, z4.b[1]\n"
+ "ld1b { z7.b }, p2/Z, [x14, #-7, MUL VL]\n"
+ "sdot z8.s, z6.b, z0.b[2]\n"
+ "sdot z12.s, z6.b, z1.b[2]\n"
+ "sdot z16.s, z6.b, z2.b[2]\n"
+ "sdot z20.s, z6.b, z3.b[2]\n"
+ "sdot z24.s, z6.b, z4.b[2]\n"
+ "ld1b { z6.b }, p2/Z, [x14, #-6, MUL VL]\n"
+ "sdot z9.s, z7.b, z0.b[2]\n"
+ "sdot z13.s, z7.b, z1.b[2]\n"
+ "sdot z17.s, z7.b, z2.b[2]\n"
+ "sdot z21.s, z7.b, z3.b[2]\n"
+ "sdot z25.s, z7.b, z4.b[2]\n"
+ "ld1b { z7.b }, p2/Z, [x14, #-5, MUL VL]\n"
+ "sdot z10.s, z6.b, z0.b[2]\n"
+ "sdot z14.s, z6.b, z1.b[2]\n"
+ "sdot z18.s, z6.b, z2.b[2]\n"
+ "sdot z22.s, z6.b, z3.b[2]\n"
+ "sdot z26.s, z6.b, z4.b[2]\n"
+ "ld1b { z6.b }, p2/Z, [x14, #-4, MUL VL]\n"
+ "sdot z11.s, z7.b, z0.b[2]\n"
+ "sdot z15.s, z7.b, z1.b[2]\n"
+ "sdot z19.s, z7.b, z2.b[2]\n"
+ "sdot z23.s, z7.b, z3.b[2]\n"
+ "sdot z27.s, z7.b, z4.b[2]\n"
+ "ld1b { z7.b }, p2/Z, [x14, #-3, MUL VL]\n"
+ "sdot z8.s, z6.b, z0.b[3]\n"
+ "sdot z12.s, z6.b, z1.b[3]\n"
+ "sdot z16.s, z6.b, z2.b[3]\n"
+ "sdot z20.s, z6.b, z3.b[3]\n"
+ "sdot z24.s, z6.b, z4.b[3]\n"
+ "ld1b { z6.b }, p2/Z, [x14, #-2, MUL VL]\n"
+ "sdot z9.s, z7.b, z0.b[3]\n"
+ "sdot z13.s, z7.b, z1.b[3]\n"
+ "sdot z17.s, z7.b, z2.b[3]\n"
+ "sdot z21.s, z7.b, z3.b[3]\n"
+ "sdot z25.s, z7.b, z4.b[3]\n"
+ "ld1b { z7.b }, p2/Z, [x14, #-1, MUL VL]\n"
+ "sdot z10.s, z6.b, z0.b[3]\n"
+ "sdot z14.s, z6.b, z1.b[3]\n"
+ "sdot z18.s, z6.b, z2.b[3]\n"
+ "sdot z22.s, z6.b, z3.b[3]\n"
+ "sdot z26.s, z6.b, z4.b[3]\n"
+ "sdot z11.s, z7.b, z0.b[3]\n"
+ "sdot z15.s, z7.b, z1.b[3]\n"
+ "sdot z19.s, z7.b, z2.b[3]\n"
+ "sdot z23.s, z7.b, z3.b[3]\n"
+ "sdot z27.s, z7.b, z4.b[3]\n"
+ "bgt 64b\n"
+ "65:" // Height 5: Multiply loop: Single iteration only
+ "ld1b { z6.b }, p2/Z, [x14]\n"
+ "whilelt p0.b, XZR, x11\n"
+ "ld1b { z7.b }, p2/Z, [x14, #1, MUL VL]\n"
+ "subs x11, x11, #0x4\n"
+ "ld1rqb { z0.b }, p0/Z, [x10]\n"
+ "sdot z8.s, z6.b, z0.b[0]\n"
+ "ld1rqb { z1.b }, p0/Z, [x28]\n"
+ "add x10, x10, #0x10\n"
+ "sdot z9.s, z7.b, z0.b[0]\n"
+ "ld1rqb { z2.b }, p0/Z, [x26]\n"
+ "add x28, x28, #0x10\n"
+ "sdot z12.s, z6.b, z1.b[0]\n"
+ "ld1rqb { z3.b }, p0/Z, [x24]\n"
+ "add x26, x26, #0x10\n"
+ "sdot z16.s, z6.b, z2.b[0]\n"
+ "ld1rqb { z4.b }, p0/Z, [x22]\n"
+ "add x24, x24, #0x10\n"
+ "sdot z13.s, z7.b, z1.b[0]\n"
+ "add x22, x22, #0x10\n"
+ "sdot z17.s, z7.b, z2.b[0]\n"
+ "sdot z20.s, z6.b, z3.b[0]\n"
+ "sdot z24.s, z6.b, z4.b[0]\n"
+ "ld1b { z6.b }, p2/Z, [x14, #2, MUL VL]\n"
+ "sdot z21.s, z7.b, z3.b[0]\n"
+ "sdot z25.s, z7.b, z4.b[0]\n"
+ "ld1b { z7.b }, p2/Z, [x14, #3, MUL VL]\n"
+ "addvl x14, x14, #4\n"
+ "sdot z10.s, z6.b, z0.b[0]\n"
+ "sdot z14.s, z6.b, z1.b[0]\n"
+ "sdot z18.s, z6.b, z2.b[0]\n"
+ "sdot z22.s, z6.b, z3.b[0]\n"
+ "sdot z26.s, z6.b, z4.b[0]\n"
+ "sdot z11.s, z7.b, z0.b[0]\n"
+ "sdot z15.s, z7.b, z1.b[0]\n"
+ "sdot z19.s, z7.b, z2.b[0]\n"
+ "sdot z23.s, z7.b, z3.b[0]\n"
+ "sdot z27.s, z7.b, z4.b[0]\n"
+ "ble 66f\n"
+ "ld1b { z6.b }, p2/Z, [x14]\n"
+ "sdot z8.s, z6.b, z0.b[1]\n"
+ "ld1b { z7.b }, p2/Z, [x14, #1, MUL VL]\n"
+ "subs x11, x11, #0x4\n"
+ "sdot z12.s, z6.b, z1.b[1]\n"
+ "sdot z16.s, z6.b, z2.b[1]\n"
+ "sdot z20.s, z6.b, z3.b[1]\n"
+ "sdot z24.s, z6.b, z4.b[1]\n"
+ "ld1b { z6.b }, p2/Z, [x14, #2, MUL VL]\n"
+ "sdot z9.s, z7.b, z0.b[1]\n"
+ "sdot z13.s, z7.b, z1.b[1]\n"
+ "sdot z17.s, z7.b, z2.b[1]\n"
+ "sdot z21.s, z7.b, z3.b[1]\n"
+ "sdot z25.s, z7.b, z4.b[1]\n"
+ "ld1b { z7.b }, p2/Z, [x14, #3, MUL VL]\n"
+ "addvl x14, x14, #4\n"
+ "sdot z10.s, z6.b, z0.b[1]\n"
+ "sdot z14.s, z6.b, z1.b[1]\n"
+ "sdot z18.s, z6.b, z2.b[1]\n"
+ "sdot z22.s, z6.b, z3.b[1]\n"
+ "sdot z26.s, z6.b, z4.b[1]\n"
+ "sdot z11.s, z7.b, z0.b[1]\n"
+ "sdot z15.s, z7.b, z1.b[1]\n"
+ "sdot z19.s, z7.b, z2.b[1]\n"
+ "sdot z23.s, z7.b, z3.b[1]\n"
+ "sdot z27.s, z7.b, z4.b[1]\n"
+ "ble 66f\n"
+ "ld1b { z6.b }, p2/Z, [x14]\n"
+ "sdot z8.s, z6.b, z0.b[2]\n"
+ "ld1b { z7.b }, p2/Z, [x14, #1, MUL VL]\n"
+ "subs x11, x11, #0x4\n"
+ "sdot z12.s, z6.b, z1.b[2]\n"
+ "sdot z16.s, z6.b, z2.b[2]\n"
+ "sdot z20.s, z6.b, z3.b[2]\n"
+ "sdot z24.s, z6.b, z4.b[2]\n"
+ "ld1b { z6.b }, p2/Z, [x14, #2, MUL VL]\n"
+ "sdot z9.s, z7.b, z0.b[2]\n"
+ "sdot z13.s, z7.b, z1.b[2]\n"
+ "sdot z17.s, z7.b, z2.b[2]\n"
+ "sdot z21.s, z7.b, z3.b[2]\n"
+ "sdot z25.s, z7.b, z4.b[2]\n"
+ "ld1b { z7.b }, p2/Z, [x14, #3, MUL VL]\n"
+ "addvl x14, x14, #4\n"
+ "sdot z10.s, z6.b, z0.b[2]\n"
+ "sdot z14.s, z6.b, z1.b[2]\n"
+ "sdot z18.s, z6.b, z2.b[2]\n"
+ "sdot z22.s, z6.b, z3.b[2]\n"
+ "sdot z26.s, z6.b, z4.b[2]\n"
+ "sdot z11.s, z7.b, z0.b[2]\n"
+ "sdot z15.s, z7.b, z1.b[2]\n"
+ "sdot z19.s, z7.b, z2.b[2]\n"
+ "sdot z23.s, z7.b, z3.b[2]\n"
+ "sdot z27.s, z7.b, z4.b[2]\n"
+ "ble 66f\n"
+ "ld1b { z6.b }, p2/Z, [x14]\n"
+ "sdot z8.s, z6.b, z0.b[3]\n"
+ "ld1b { z7.b }, p2/Z, [x14, #1, MUL VL]\n"
+ "sdot z12.s, z6.b, z1.b[3]\n"
+ "sdot z16.s, z6.b, z2.b[3]\n"
+ "sdot z20.s, z6.b, z3.b[3]\n"
+ "sdot z24.s, z6.b, z4.b[3]\n"
+ "ld1b { z6.b }, p2/Z, [x14, #2, MUL VL]\n"
+ "sdot z9.s, z7.b, z0.b[3]\n"
+ "sdot z13.s, z7.b, z1.b[3]\n"
+ "sdot z17.s, z7.b, z2.b[3]\n"
+ "sdot z21.s, z7.b, z3.b[3]\n"
+ "sdot z25.s, z7.b, z4.b[3]\n"
+ "ld1b { z7.b }, p2/Z, [x14, #3, MUL VL]\n"
+ "addvl x14, x14, #4\n"
+ "sdot z10.s, z6.b, z0.b[3]\n"
+ "sdot z14.s, z6.b, z1.b[3]\n"
+ "sdot z18.s, z6.b, z2.b[3]\n"
+ "sdot z22.s, z6.b, z3.b[3]\n"
+ "sdot z26.s, z6.b, z4.b[3]\n"
+ "sdot z11.s, z7.b, z0.b[3]\n"
+ "sdot z15.s, z7.b, z1.b[3]\n"
+ "sdot z19.s, z7.b, z2.b[3]\n"
+ "sdot z23.s, z7.b, z3.b[3]\n"
+ "sdot z27.s, z7.b, z4.b[3]\n"
+ "66:" // Height 5: Multiply loop: multiply skip
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x12, x12, #0x1\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "prfm pldl1keep, [x22, #0x80]\n"
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "cmp x12, x19\n"
+ "bne 61b\n"
+ "prfm pstl1keep, [x13, #0x0]\n"
+ "prfm pstl1keep, [x9, #0x0]\n"
+ "prfm pstl1keep, [x27, #0x0]\n"
+ "prfm pstl1keep, [x25, #0x0]\n"
+ "prfm pstl1keep, [x23, #0x0]\n"
+ "ld1w { z0.s }, p2/Z, [x16]\n"
+ "add z8.s, z8.s, z0.s\n"
+ "ld1w { z1.s }, p2/Z, [x16, #1, MUL VL]\n"
+ "add z12.s, z12.s, z0.s\n"
+ "ld1w { z2.s }, p2/Z, [x16, #2, MUL VL]\n"
+ "add z16.s, z16.s, z0.s\n"
+ "ld1w { z3.s }, p2/Z, [x16, #3, MUL VL]\n"
+ "addvl x16, x16, #4\n"
+ "add z9.s, z9.s, z1.s\n"
+ "add z13.s, z13.s, z1.s\n"
+ "add z10.s, z10.s, z2.s\n"
+ "add z11.s, z11.s, z3.s\n"
+ "add z14.s, z14.s, z2.s\n"
+ "add z15.s, z15.s, z3.s\n"
+ "add z17.s, z17.s, z1.s\n"
+ "add z18.s, z18.s, z2.s\n"
+ "add z19.s, z19.s, z3.s\n"
+ "add z20.s, z20.s, z0.s\n"
+ "add z21.s, z21.s, z1.s\n"
+ "add z22.s, z22.s, z2.s\n"
+ "add z23.s, z23.s, z3.s\n"
+ "add z24.s, z24.s, z0.s\n"
+ "add z25.s, z25.s, z1.s\n"
+ "add z26.s, z26.s, z2.s\n"
+ "add z27.s, z27.s, z3.s\n"
+ "tbz %x[flags], #4, 67f\n"
+ "ld1w { z0.s }, p2/Z, [x17]\n"
+ "ld1w { z4.s }, p2/Z, [x8]\n"
+ "ld1w { z1.s }, p2/Z, [x17, #1, MUL VL]\n"
+ "ld1w { z5.s }, p2/Z, [x8, #1, MUL VL]\n"
+ "ld1w { z2.s }, p2/Z, [x17, #2, MUL VL]\n"
+ "ld1w { z6.s }, p2/Z, [x8, #2, MUL VL]\n"
+ "ld1w { z3.s }, p2/Z, [x17, #3, MUL VL]\n"
+ "addvl x17, x17, #4\n"
+ "ld1w { z7.s }, p2/Z, [x8, #3, MUL VL]\n"
+ "addvl x8, x8, #4\n"
+ "b 68f\n"
+ "67:" // Height 5: per layer parameters
+ "add x19, %x[qp], %[per_layer_right_shift]\n"
+ "ld1rw { z0.s }, p2/Z, [x19]\n"
+ "mov z1.d, z0.d\n"
+ "add x19, %x[qp], %[per_layer_mul]\n"
+ "ld1rw { z4.s }, p2/Z, [x19]\n"
+ "mov z2.d, z0.d\n"
+ "mov z3.d, z0.d\n"
+ "mov z5.d, z4.d\n"
+ "mov z6.d, z4.d\n"
+ "mov z7.d, z4.d\n"
+ "68:" // Height 5: parameters loaded
+ ".inst 0x04a47508 // sqrdmulh z8.s, z8.s, z4.s\n"
+ ".inst 0x04a57529 // sqrdmulh z9.s, z9.s, z5.s\n"
+ ".inst 0x04a6754a // sqrdmulh z10.s, z10.s, z6.s\n"
+ ".inst 0x04a7756b // sqrdmulh z11.s, z11.s, z7.s\n"
+ ".inst 0x04a4758c // sqrdmulh z12.s, z12.s, z4.s\n"
+ ".inst 0x04a575ad // sqrdmulh z13.s, z13.s, z5.s\n"
+ ".inst 0x04a675ce // sqrdmulh z14.s, z14.s, z6.s\n"
+ ".inst 0x04a775ef // sqrdmulh z15.s, z15.s, z7.s\n"
+ ".inst 0x04a47610 // sqrdmulh z16.s, z16.s, z4.s\n"
+ ".inst 0x04a57631 // sqrdmulh z17.s, z17.s, z5.s\n"
+ ".inst 0x04a67652 // sqrdmulh z18.s, z18.s, z6.s\n"
+ ".inst 0x04a77673 // sqrdmulh z19.s, z19.s, z7.s\n"
+ ".inst 0x04a47694 // sqrdmulh z20.s, z20.s, z4.s\n"
+ ".inst 0x04a576b5 // sqrdmulh z21.s, z21.s, z5.s\n"
+ ".inst 0x04a676d6 // sqrdmulh z22.s, z22.s, z6.s\n"
+ ".inst 0x04a776f7 // sqrdmulh z23.s, z23.s, z7.s\n"
+ ".inst 0x04a47718 // sqrdmulh z24.s, z24.s, z4.s\n"
+ ".inst 0x04a57739 // sqrdmulh z25.s, z25.s, z5.s\n"
+ ".inst 0x04a6775a // sqrdmulh z26.s, z26.s, z6.s\n"
+ ".inst 0x04a7777b // sqrdmulh z27.s, z27.s, z7.s\n"
+ "tbz %x[flags], #5, 69f\n"
+ "and z4.d, z8.d, z0.d\n"
+ "asr z4.s, z4.s, #0x1f\n"
+ "and z5.d, z9.d, z1.d\n"
+ "and z6.d, z10.d, z2.d\n"
+ "asr z5.s, z5.s, #0x1f\n"
+ "and z7.d, z11.d, z3.d\n"
+ "asr z6.s, z6.s, #0x1f\n"
+ "sqadd z8.s, z8.s, z4.s\n"
+ "asr z7.s, z7.s, #0x1f\n"
+ "and z4.d, z12.d, z0.d\n"
+ "sqadd z9.s, z9.s, z5.s\n"
+ "asr z4.s, z4.s, #0x1f\n"
+ "sqadd z10.s, z10.s, z6.s\n"
+ "and z5.d, z13.d, z1.d\n"
+ "asr z5.s, z5.s, #0x1f\n"
+ "sqadd z11.s, z11.s, z7.s\n"
+ "and z6.d, z14.d, z2.d\n"
+ "asr z6.s, z6.s, #0x1f\n"
+ "sqadd z12.s, z12.s, z4.s\n"
+ "and z7.d, z15.d, z3.d\n"
+ "asr z7.s, z7.s, #0x1f\n"
+ "sqadd z13.s, z13.s, z5.s\n"
+ "and z4.d, z16.d, z0.d\n"
+ "asr z4.s, z4.s, #0x1f\n"
+ "sqadd z14.s, z14.s, z6.s\n"
+ "and z5.d, z17.d, z1.d\n"
+ "asr z5.s, z5.s, #0x1f\n"
+ "sqadd z15.s, z15.s, z7.s\n"
+ "and z6.d, z18.d, z2.d\n"
+ "asr z6.s, z6.s, #0x1f\n"
+ "sqadd z16.s, z16.s, z4.s\n"
+ "and z7.d, z19.d, z3.d\n"
+ "asr z7.s, z7.s, #0x1f\n"
+ "sqadd z17.s, z17.s, z5.s\n"
+ "and z4.d, z20.d, z0.d\n"
+ "asr z4.s, z4.s, #0x1f\n"
+ "sqadd z18.s, z18.s, z6.s\n"
+ "and z5.d, z21.d, z1.d\n"
+ "asr z5.s, z5.s, #0x1f\n"
+ "sqadd z19.s, z19.s, z7.s\n"
+ "and z6.d, z22.d, z2.d\n"
+ "asr z6.s, z6.s, #0x1f\n"
+ "sqadd z20.s, z20.s, z4.s\n"
+ "and z7.d, z23.d, z3.d\n"
+ "asr z7.s, z7.s, #0x1f\n"
+ "sqadd z21.s, z21.s, z5.s\n"
+ "and z4.d, z24.d, z0.d\n"
+ "asr z4.s, z4.s, #0x1f\n"
+ "sqadd z22.s, z22.s, z6.s\n"
+ "and z5.d, z25.d, z1.d\n"
+ "asr z5.s, z5.s, #0x1f\n"
+ "sqadd z23.s, z23.s, z7.s\n"
+ "and z6.d, z26.d, z2.d\n"
+ "asr z6.s, z6.s, #0x1f\n"
+ "sqadd z24.s, z24.s, z4.s\n"
+ "and z7.d, z27.d, z3.d\n"
+ "asr z7.s, z7.s, #0x1f\n"
+ "sqadd z25.s, z25.s, z5.s\n"
+ "sqadd z26.s, z26.s, z6.s\n"
+ "sqadd z27.s, z27.s, z7.s\n"
+ "69:" // Height 5: no shift correction
+ ".inst 0x44828808 // srshl z8.s, p2/M, z8.s, z0.s\n"
+ "add x19, %x[qp], %[c_offset]\n"
+ "ld1rw { z4.s }, p2/Z, [x19]\n"
+ ".inst 0x44828829 // srshl z9.s, p2/M, z9.s, z1.s\n"
+ "add x19, %x[qp], %[minval]\n"
+ ".inst 0x4482884a // srshl z10.s, p2/M, z10.s, z2.s\n"
+ "ld1rw { z5.s }, p2/Z, [x19]\n"
+ "add x19, %x[qp], %[maxval]\n"
+ ".inst 0x4482886b // srshl z11.s, p2/M, z11.s, z3.s\n"
+ "ld1rw { z6.s }, p2/Z, [x19]\n"
+ ".inst 0x4482880c // srshl z12.s, p2/M, z12.s, z0.s\n"
+ "add z8.s, z8.s, z4.s\n"
+ "add z9.s, z9.s, z4.s\n"
+ "add z10.s, z10.s, z4.s\n"
+ "add z11.s, z11.s, z4.s\n"
+ "add z12.s, z12.s, z4.s\n"
+ "smin z8.s, p2/M, z8.s, z6.s\n"
+ "smin z9.s, p2/M, z9.s, z6.s\n"
+ "smin z10.s, p2/M, z10.s, z6.s\n"
+ "smin z11.s, p2/M, z11.s, z6.s\n"
+ "smax z8.s, p2/M, z8.s, z5.s\n"
+ "smax z9.s, p2/M, z9.s, z5.s\n"
+ "smax z10.s, p2/M, z10.s, z5.s\n"
+ "smax z11.s, p2/M, z11.s, z5.s\n"
+ "smin z12.s, p2/M, z12.s, z6.s\n"
+ "uzp1 z8.h, z8.h, z9.h\n"
+ ".inst 0x4482882d // srshl z13.s, p2/M, z13.s, z1.s\n"
+ "uzp1 z9.h, z10.h, z11.h\n"
+ "smax z12.s, p2/M, z12.s, z5.s\n"
+ "uzp1 z8.b, z8.b, z9.b\n"
+ "st1b { z8.b }, p1, [x13]\n"
+ "add z13.s, z13.s, z4.s\n"
+ "addvl x13, x13, #1\n"
+ ".inst 0x4482884e // srshl z14.s, p2/M, z14.s, z2.s\n"
+ ".inst 0x4482886f // srshl z15.s, p2/M, z15.s, z3.s\n"
+ ".inst 0x44828810 // srshl z16.s, p2/M, z16.s, z0.s\n"
+ "smin z13.s, p2/M, z13.s, z6.s\n"
+ ".inst 0x44828831 // srshl z17.s, p2/M, z17.s, z1.s\n"
+ "add z14.s, z14.s, z4.s\n"
+ "add z15.s, z15.s, z4.s\n"
+ "add z16.s, z16.s, z4.s\n"
+ "add z17.s, z17.s, z4.s\n"
+ "smax z13.s, p2/M, z13.s, z5.s\n"
+ "smin z14.s, p2/M, z14.s, z6.s\n"
+ "smin z15.s, p2/M, z15.s, z6.s\n"
+ "smin z16.s, p2/M, z16.s, z6.s\n"
+ "uzp1 z12.h, z12.h, z13.h\n"
+ "smax z14.s, p2/M, z14.s, z5.s\n"
+ "smax z15.s, p2/M, z15.s, z5.s\n"
+ "smax z16.s, p2/M, z16.s, z5.s\n"
+ "smin z17.s, p2/M, z17.s, z6.s\n"
+ ".inst 0x44828852 // srshl z18.s, p2/M, z18.s, z2.s\n"
+ "uzp1 z13.h, z14.h, z15.h\n"
+ ".inst 0x44828873 // srshl z19.s, p2/M, z19.s, z3.s\n"
+ "uzp1 z12.b, z12.b, z13.b\n"
+ "st1b { z12.b }, p1, [x9]\n"
+ "add z18.s, z18.s, z4.s\n"
+ "addvl x9, x9, #1\n"
+ "add z19.s, z19.s, z4.s\n"
+ "smax z17.s, p2/M, z17.s, z5.s\n"
+ ".inst 0x44828814 // srshl z20.s, p2/M, z20.s, z0.s\n"
+ "smin z18.s, p2/M, z18.s, z6.s\n"
+ "smin z19.s, p2/M, z19.s, z6.s\n"
+ "uzp1 z16.h, z16.h, z17.h\n"
+ "add z20.s, z20.s, z4.s\n"
+ "smax z18.s, p2/M, z18.s, z5.s\n"
+ "smax z19.s, p2/M, z19.s, z5.s\n"
+ "smin z20.s, p2/M, z20.s, z6.s\n"
+ ".inst 0x44828835 // srshl z21.s, p2/M, z21.s, z1.s\n"
+ ".inst 0x44828856 // srshl z22.s, p2/M, z22.s, z2.s\n"
+ "uzp1 z17.h, z18.h, z19.h\n"
+ "smax z20.s, p2/M, z20.s, z5.s\n"
+ "add z21.s, z21.s, z4.s\n"
+ "add z22.s, z22.s, z4.s\n"
+ "uzp1 z16.b, z16.b, z17.b\n"
+ "st1b { z16.b }, p1, [x27]\n"
+ "smin z21.s, p2/M, z21.s, z6.s\n"
+ "addvl x27, x27, #1\n"
+ "smin z22.s, p2/M, z22.s, z6.s\n"
+ ".inst 0x44828877 // srshl z23.s, p2/M, z23.s, z3.s\n"
+ ".inst 0x44828818 // srshl z24.s, p2/M, z24.s, z0.s\n"
+ ".inst 0x44828839 // srshl z25.s, p2/M, z25.s, z1.s\n"
+ "smax z21.s, p2/M, z21.s, z5.s\n"
+ "add z23.s, z23.s, z4.s\n"
+ "add z24.s, z24.s, z4.s\n"
+ "add z25.s, z25.s, z4.s\n"
+ "uzp1 z20.h, z20.h, z21.h\n"
+ "smax z22.s, p2/M, z22.s, z5.s\n"
+ "smin z23.s, p2/M, z23.s, z6.s\n"
+ "smin z24.s, p2/M, z24.s, z6.s\n"
+ "smin z25.s, p2/M, z25.s, z6.s\n"
+ ".inst 0x4482885a // srshl z26.s, p2/M, z26.s, z2.s\n"
+ "smax z23.s, p2/M, z23.s, z5.s\n"
+ "smax z24.s, p2/M, z24.s, z5.s\n"
+ "smax z25.s, p2/M, z25.s, z5.s\n"
+ "add z26.s, z26.s, z4.s\n"
+ "uzp1 z21.h, z22.h, z23.h\n"
+ ".inst 0x4482887b // srshl z27.s, p2/M, z27.s, z3.s\n"
+ "uzp1 z24.h, z24.h, z25.h\n"
+ "uzp1 z20.b, z20.b, z21.b\n"
+ "st1b { z20.b }, p1, [x25]\n"
+ "add z27.s, z27.s, z4.s\n"
+ "addvl x25, x25, #1\n"
+ "smin z26.s, p2/M, z26.s, z6.s\n"
+ "smin z27.s, p2/M, z27.s, z6.s\n"
+ "smax z26.s, p2/M, z26.s, z5.s\n"
+ "smax z27.s, p2/M, z27.s, z5.s\n"
+ "uzp1 z25.h, z26.h, z27.h\n"
+ "uzp1 z24.b, z24.b, z25.b\n"
+ "st1b { z24.b }, p1, [x23]\n"
+ "addvl x23, x23, #1\n"
+ "70:" // Height 5: Writeback done
+ "mov x19, #0x0\n"
+ "incw x19, ALL, MUL #4\n"
+ "subs x15, x15, x19\n"
+ "bgt 59b\n"
+ "b 86f\n"
+ "71:" // Height 6
+ "ldr x8, [%x[args_ptr], %[offsetof_multiplier_ptr]]\n"
+ "mov x16, %x[col_bias]\n"
+ "ldr x17, [%x[args_ptr], %[offsetof_shift_ptr]]\n"
+ "ldr x15, [%x[args_ptr], %[offsetof_N]]\n"
+ "ldr x14, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 72f\n"
+ "ldr x13, [%x[output_ptr], #0x0]\n"
+ "add x13, x13, x19\n"
+ "ldr x9, [%x[output_ptr], #0x8]\n"
+ "ldr x27, [%x[output_ptr], #0x10]\n"
+ "add x9, x9, x19\n"
+ "ldr x25, [%x[output_ptr], #0x18]\n"
+ "ldr x23, [%x[output_ptr], #0x20]\n"
+ "add x27, x27, x19\n"
+ "ldr x21, [%x[output_ptr], #0x28]\n"
+ "add %x[output_ptr], %x[output_ptr], #0x30\n"
+ "add x25, x25, x19\n"
+ "add x23, x23, x19\n"
+ "add x21, x21, x19\n"
+ "b 73f\n"
+ "72:" // Height 6: setup direct output
+ "mov x13, %x[output_ptr]\n"
+ "add x9, x13, x19\n"
+ "add x27, x9, x19\n"
+ "add x25, x27, x19\n"
+ "add x23, x25, x19\n"
+ "add x21, x23, x19\n"
+ "add %x[output_ptr], x21, x19\n"
+ "73:" // Height 6: Column loop
+ "mov z8.s, #0x0\n"
+ "mov x19, #0x0\n"
+ "mov z9.s, #0x0\n"
+ "whilelt p1.b, x19, x15\n"
+ "mov z10.s, #0x0\n"
+ "mov z11.s, #0x0\n"
+ "mov z12.s, #0x0\n"
+ "mov z13.s, #0x0\n"
+ "mov z14.s, #0x0\n"
+ "mov z15.s, #0x0\n"
+ "mov z16.s, #0x0\n"
+ "mov z17.s, #0x0\n"
+ "mov z18.s, #0x0\n"
+ "mov z19.s, #0x0\n"
+ "mov z20.s, #0x0\n"
+ "mov z21.s, #0x0\n"
+ "mov z22.s, #0x0\n"
+ "mov z23.s, #0x0\n"
+ "mov z24.s, #0x0\n"
+ "mov z25.s, #0x0\n"
+ "mov z26.s, #0x0\n"
+ "mov z27.s, #0x0\n"
+ "mov z28.s, #0x0\n"
+ "mov z29.s, #0x0\n"
+ "mov z30.s, #0x0\n"
+ "mov z31.s, #0x0\n"
+ "74:" // Height 6: setup done
+ "mov x12, #0x0\n"
+ "75:" // Height 6: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w11, [x20, x12, LSL #0x2]\n"
+ "tbz %x[flags], #3, 76f\n"
+ "ldr x20, [%x[input_ptr], x12, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x10, [x20, #0x0]\n"
+ "ldr x28, [x20, #0x8]\n"
+ "ldr x26, [x20, #0x10]\n"
+ "ldr x24, [x20, #0x18]\n"
+ "ldr x22, [x20, #0x20]\n"
+ "ldr x20, [x20, #0x28]\n"
+ "cbnz x12, 77f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x10, x10, x19\n"
+ "add x28, x28, x19\n"
+ "add x26, x26, x19\n"
+ "add x24, x24, x19\n"
+ "add x22, x22, x19\n"
+ "add x20, x20, x19\n"
+ "b 77f\n"
+ "76:" // Height 6: setup direct input
+ "mov x10, %x[input_ptr]\n"
+ "add x28, x10, x19\n"
+ "add x26, x28, x19\n"
+ "add x24, x26, x19\n"
+ "add x22, x24, x19\n"
+ "add x20, x22, x19\n"
+ "77:" // Height 6: input setup done
+ "cmp x11, #0x10\n"
+ "ble 79f\n"
+ "78:" // Height 6: Multiply loop: Main loop head
+ "ld1b { z6.b }, p2/Z, [x14]\n"
+ "whilelt p0.b, XZR, x11\n"
+ "ld1b { z7.b }, p2/Z, [x14, #1, MUL VL]\n"
+ "sub x11, x11, #0x10\n"
+ "ld1rqb { z0.b }, p0/Z, [x10]\n"
+ "sdot z8.s, z6.b, z0.b[0]\n"
+ "ld1rqb { z1.b }, p0/Z, [x28]\n"
+ "add x10, x10, #0x10\n"
+ "sdot z9.s, z7.b, z0.b[0]\n"
+ "ld1rqb { z2.b }, p0/Z, [x26]\n"
+ "add x28, x28, #0x10\n"
+ "sdot z12.s, z6.b, z1.b[0]\n"
+ "ld1rqb { z3.b }, p0/Z, [x24]\n"
+ "add x26, x26, #0x10\n"
+ "sdot z16.s, z6.b, z2.b[0]\n"
+ "ld1rqb { z4.b }, p0/Z, [x22]\n"
+ "add x24, x24, #0x10\n"
+ "sdot z13.s, z7.b, z1.b[0]\n"
+ "ld1rqb { z5.b }, p0/Z, [x20]\n"
+ "add x22, x22, #0x10\n"
+ "sdot z20.s, z6.b, z3.b[0]\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x20, x20, #0x10\n"
+ "sdot z24.s, z6.b, z4.b[0]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "cmp x11, #0x10\n"
+ "sdot z28.s, z6.b, z5.b[0]\n"
+ "ld1b { z6.b }, p2/Z, [x14, #2, MUL VL]\n"
+ "sdot z17.s, z7.b, z2.b[0]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "sdot z21.s, z7.b, z3.b[0]\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "sdot z25.s, z7.b, z4.b[0]\n"
+ "prfm pldl1keep, [x22, #0x80]\n"
+ "sdot z29.s, z7.b, z5.b[0]\n"
+ "ld1b { z7.b }, p2/Z, [x14, #3, MUL VL]\n"
+ "sdot z10.s, z6.b, z0.b[0]\n"
+ "prfm pldl1keep, [x20, #0x80]\n"
+ "sdot z14.s, z6.b, z1.b[0]\n"
+ "sdot z18.s, z6.b, z2.b[0]\n"
+ "sdot z22.s, z6.b, z3.b[0]\n"
+ "sdot z26.s, z6.b, z4.b[0]\n"
+ "sdot z30.s, z6.b, z5.b[0]\n"
+ "ld1b { z6.b }, p2/Z, [x14, #4, MUL VL]\n"
+ "sdot z11.s, z7.b, z0.b[0]\n"
+ "sdot z15.s, z7.b, z1.b[0]\n"
+ "sdot z19.s, z7.b, z2.b[0]\n"
+ "sdot z23.s, z7.b, z3.b[0]\n"
+ "sdot z27.s, z7.b, z4.b[0]\n"
+ "sdot z31.s, z7.b, z5.b[0]\n"
+ "ld1b { z7.b }, p2/Z, [x14, #5, MUL VL]\n"
+ "sdot z8.s, z6.b, z0.b[1]\n"
+ "sdot z12.s, z6.b, z1.b[1]\n"
+ "sdot z16.s, z6.b, z2.b[1]\n"
+ "sdot z20.s, z6.b, z3.b[1]\n"
+ "sdot z24.s, z6.b, z4.b[1]\n"
+ "sdot z28.s, z6.b, z5.b[1]\n"
+ "ld1b { z6.b }, p2/Z, [x14, #6, MUL VL]\n"
+ "sdot z9.s, z7.b, z0.b[1]\n"
+ "sdot z13.s, z7.b, z1.b[1]\n"
+ "sdot z17.s, z7.b, z2.b[1]\n"
+ "sdot z21.s, z7.b, z3.b[1]\n"
+ "sdot z25.s, z7.b, z4.b[1]\n"
+ "sdot z29.s, z7.b, z5.b[1]\n"
+ "ld1b { z7.b }, p2/Z, [x14, #7, MUL VL]\n"
+ "addvl x14, x14, #16\n"
+ "sdot z10.s, z6.b, z0.b[1]\n"
+ "sdot z14.s, z6.b, z1.b[1]\n"
+ "sdot z18.s, z6.b, z2.b[1]\n"
+ "sdot z22.s, z6.b, z3.b[1]\n"
+ "sdot z26.s, z6.b, z4.b[1]\n"
+ "sdot z30.s, z6.b, z5.b[1]\n"
+ "ld1b { z6.b }, p2/Z, [x14, #-8, MUL VL]\n"
+ "sdot z11.s, z7.b, z0.b[1]\n"
+ "sdot z15.s, z7.b, z1.b[1]\n"
+ "sdot z19.s, z7.b, z2.b[1]\n"
+ "sdot z23.s, z7.b, z3.b[1]\n"
+ "sdot z27.s, z7.b, z4.b[1]\n"
+ "sdot z31.s, z7.b, z5.b[1]\n"
+ "ld1b { z7.b }, p2/Z, [x14, #-7, MUL VL]\n"
+ "sdot z8.s, z6.b, z0.b[2]\n"
+ "sdot z12.s, z6.b, z1.b[2]\n"
+ "sdot z16.s, z6.b, z2.b[2]\n"
+ "sdot z20.s, z6.b, z3.b[2]\n"
+ "sdot z24.s, z6.b, z4.b[2]\n"
+ "sdot z28.s, z6.b, z5.b[2]\n"
+ "ld1b { z6.b }, p2/Z, [x14, #-6, MUL VL]\n"
+ "sdot z9.s, z7.b, z0.b[2]\n"
+ "sdot z13.s, z7.b, z1.b[2]\n"
+ "sdot z17.s, z7.b, z2.b[2]\n"
+ "sdot z21.s, z7.b, z3.b[2]\n"
+ "sdot z25.s, z7.b, z4.b[2]\n"
+ "sdot z29.s, z7.b, z5.b[2]\n"
+ "ld1b { z7.b }, p2/Z, [x14, #-5, MUL VL]\n"
+ "sdot z10.s, z6.b, z0.b[2]\n"
+ "sdot z14.s, z6.b, z1.b[2]\n"
+ "sdot z18.s, z6.b, z2.b[2]\n"
+ "sdot z22.s, z6.b, z3.b[2]\n"
+ "sdot z26.s, z6.b, z4.b[2]\n"
+ "sdot z30.s, z6.b, z5.b[2]\n"
+ "ld1b { z6.b }, p2/Z, [x14, #-4, MUL VL]\n"
+ "sdot z11.s, z7.b, z0.b[2]\n"
+ "sdot z15.s, z7.b, z1.b[2]\n"
+ "sdot z19.s, z7.b, z2.b[2]\n"
+ "sdot z23.s, z7.b, z3.b[2]\n"
+ "sdot z27.s, z7.b, z4.b[2]\n"
+ "sdot z31.s, z7.b, z5.b[2]\n"
+ "ld1b { z7.b }, p2/Z, [x14, #-3, MUL VL]\n"
+ "sdot z8.s, z6.b, z0.b[3]\n"
+ "sdot z12.s, z6.b, z1.b[3]\n"
+ "sdot z16.s, z6.b, z2.b[3]\n"
+ "sdot z20.s, z6.b, z3.b[3]\n"
+ "sdot z24.s, z6.b, z4.b[3]\n"
+ "sdot z28.s, z6.b, z5.b[3]\n"
+ "ld1b { z6.b }, p2/Z, [x14, #-2, MUL VL]\n"
+ "sdot z9.s, z7.b, z0.b[3]\n"
+ "sdot z13.s, z7.b, z1.b[3]\n"
+ "sdot z17.s, z7.b, z2.b[3]\n"
+ "sdot z21.s, z7.b, z3.b[3]\n"
+ "sdot z25.s, z7.b, z4.b[3]\n"
+ "sdot z29.s, z7.b, z5.b[3]\n"
+ "ld1b { z7.b }, p2/Z, [x14, #-1, MUL VL]\n"
+ "sdot z10.s, z6.b, z0.b[3]\n"
+ "sdot z14.s, z6.b, z1.b[3]\n"
+ "sdot z18.s, z6.b, z2.b[3]\n"
+ "sdot z22.s, z6.b, z3.b[3]\n"
+ "sdot z26.s, z6.b, z4.b[3]\n"
+ "sdot z30.s, z6.b, z5.b[3]\n"
+ "sdot z11.s, z7.b, z0.b[3]\n"
+ "sdot z15.s, z7.b, z1.b[3]\n"
+ "sdot z19.s, z7.b, z2.b[3]\n"
+ "sdot z23.s, z7.b, z3.b[3]\n"
+ "sdot z27.s, z7.b, z4.b[3]\n"
+ "sdot z31.s, z7.b, z5.b[3]\n"
+ "bgt 78b\n"
+ "79:" // Height 6: Multiply loop: Single iteration only
+ "ld1b { z6.b }, p2/Z, [x14]\n"
+ "whilelt p0.b, XZR, x11\n"
+ "ld1b { z7.b }, p2/Z, [x14, #1, MUL VL]\n"
+ "subs x11, x11, #0x4\n"
+ "ld1rqb { z0.b }, p0/Z, [x10]\n"
+ "sdot z8.s, z6.b, z0.b[0]\n"
+ "ld1rqb { z1.b }, p0/Z, [x28]\n"
+ "add x10, x10, #0x10\n"
+ "sdot z9.s, z7.b, z0.b[0]\n"
+ "ld1rqb { z2.b }, p0/Z, [x26]\n"
+ "add x28, x28, #0x10\n"
+ "sdot z12.s, z6.b, z1.b[0]\n"
+ "ld1rqb { z3.b }, p0/Z, [x24]\n"
+ "add x26, x26, #0x10\n"
+ "sdot z16.s, z6.b, z2.b[0]\n"
+ "ld1rqb { z4.b }, p0/Z, [x22]\n"
+ "add x24, x24, #0x10\n"
+ "sdot z13.s, z7.b, z1.b[0]\n"
+ "ld1rqb { z5.b }, p0/Z, [x20]\n"
+ "add x22, x22, #0x10\n"
+ "sdot z20.s, z6.b, z3.b[0]\n"
+ "add x20, x20, #0x10\n"
+ "sdot z17.s, z7.b, z2.b[0]\n"
+ "sdot z24.s, z6.b, z4.b[0]\n"
+ "sdot z28.s, z6.b, z5.b[0]\n"
+ "ld1b { z6.b }, p2/Z, [x14, #2, MUL VL]\n"
+ "sdot z21.s, z7.b, z3.b[0]\n"
+ "sdot z25.s, z7.b, z4.b[0]\n"
+ "sdot z29.s, z7.b, z5.b[0]\n"
+ "ld1b { z7.b }, p2/Z, [x14, #3, MUL VL]\n"
+ "addvl x14, x14, #4\n"
+ "sdot z10.s, z6.b, z0.b[0]\n"
+ "sdot z14.s, z6.b, z1.b[0]\n"
+ "sdot z18.s, z6.b, z2.b[0]\n"
+ "sdot z22.s, z6.b, z3.b[0]\n"
+ "sdot z26.s, z6.b, z4.b[0]\n"
+ "sdot z30.s, z6.b, z5.b[0]\n"
+ "sdot z11.s, z7.b, z0.b[0]\n"
+ "sdot z15.s, z7.b, z1.b[0]\n"
+ "sdot z19.s, z7.b, z2.b[0]\n"
+ "sdot z23.s, z7.b, z3.b[0]\n"
+ "sdot z27.s, z7.b, z4.b[0]\n"
+ "sdot z31.s, z7.b, z5.b[0]\n"
+ "ble 80f\n"
+ "ld1b { z6.b }, p2/Z, [x14]\n"
+ "sdot z8.s, z6.b, z0.b[1]\n"
+ "ld1b { z7.b }, p2/Z, [x14, #1, MUL VL]\n"
+ "subs x11, x11, #0x4\n"
+ "sdot z12.s, z6.b, z1.b[1]\n"
+ "sdot z16.s, z6.b, z2.b[1]\n"
+ "sdot z20.s, z6.b, z3.b[1]\n"
+ "sdot z24.s, z6.b, z4.b[1]\n"
+ "sdot z28.s, z6.b, z5.b[1]\n"
+ "ld1b { z6.b }, p2/Z, [x14, #2, MUL VL]\n"
+ "sdot z9.s, z7.b, z0.b[1]\n"
+ "sdot z13.s, z7.b, z1.b[1]\n"
+ "sdot z17.s, z7.b, z2.b[1]\n"
+ "sdot z21.s, z7.b, z3.b[1]\n"
+ "sdot z25.s, z7.b, z4.b[1]\n"
+ "sdot z29.s, z7.b, z5.b[1]\n"
+ "ld1b { z7.b }, p2/Z, [x14, #3, MUL VL]\n"
+ "addvl x14, x14, #4\n"
+ "sdot z10.s, z6.b, z0.b[1]\n"
+ "sdot z14.s, z6.b, z1.b[1]\n"
+ "sdot z18.s, z6.b, z2.b[1]\n"
+ "sdot z22.s, z6.b, z3.b[1]\n"
+ "sdot z26.s, z6.b, z4.b[1]\n"
+ "sdot z30.s, z6.b, z5.b[1]\n"
+ "sdot z11.s, z7.b, z0.b[1]\n"
+ "sdot z15.s, z7.b, z1.b[1]\n"
+ "sdot z19.s, z7.b, z2.b[1]\n"
+ "sdot z23.s, z7.b, z3.b[1]\n"
+ "sdot z27.s, z7.b, z4.b[1]\n"
+ "sdot z31.s, z7.b, z5.b[1]\n"
+ "ble 80f\n"
+ "ld1b { z6.b }, p2/Z, [x14]\n"
+ "sdot z8.s, z6.b, z0.b[2]\n"
+ "ld1b { z7.b }, p2/Z, [x14, #1, MUL VL]\n"
+ "subs x11, x11, #0x4\n"
+ "sdot z12.s, z6.b, z1.b[2]\n"
+ "sdot z16.s, z6.b, z2.b[2]\n"
+ "sdot z20.s, z6.b, z3.b[2]\n"
+ "sdot z24.s, z6.b, z4.b[2]\n"
+ "sdot z28.s, z6.b, z5.b[2]\n"
+ "ld1b { z6.b }, p2/Z, [x14, #2, MUL VL]\n"
+ "sdot z9.s, z7.b, z0.b[2]\n"
+ "sdot z13.s, z7.b, z1.b[2]\n"
+ "sdot z17.s, z7.b, z2.b[2]\n"
+ "sdot z21.s, z7.b, z3.b[2]\n"
+ "sdot z25.s, z7.b, z4.b[2]\n"
+ "sdot z29.s, z7.b, z5.b[2]\n"
+ "ld1b { z7.b }, p2/Z, [x14, #3, MUL VL]\n"
+ "addvl x14, x14, #4\n"
+ "sdot z10.s, z6.b, z0.b[2]\n"
+ "sdot z14.s, z6.b, z1.b[2]\n"
+ "sdot z18.s, z6.b, z2.b[2]\n"
+ "sdot z22.s, z6.b, z3.b[2]\n"
+ "sdot z26.s, z6.b, z4.b[2]\n"
+ "sdot z30.s, z6.b, z5.b[2]\n"
+ "sdot z11.s, z7.b, z0.b[2]\n"
+ "sdot z15.s, z7.b, z1.b[2]\n"
+ "sdot z19.s, z7.b, z2.b[2]\n"
+ "sdot z23.s, z7.b, z3.b[2]\n"
+ "sdot z27.s, z7.b, z4.b[2]\n"
+ "sdot z31.s, z7.b, z5.b[2]\n"
+ "ble 80f\n"
+ "ld1b { z6.b }, p2/Z, [x14]\n"
+ "sdot z8.s, z6.b, z0.b[3]\n"
+ "ld1b { z7.b }, p2/Z, [x14, #1, MUL VL]\n"
+ "sdot z12.s, z6.b, z1.b[3]\n"
+ "sdot z16.s, z6.b, z2.b[3]\n"
+ "sdot z20.s, z6.b, z3.b[3]\n"
+ "sdot z24.s, z6.b, z4.b[3]\n"
+ "sdot z28.s, z6.b, z5.b[3]\n"
+ "ld1b { z6.b }, p2/Z, [x14, #2, MUL VL]\n"
+ "sdot z9.s, z7.b, z0.b[3]\n"
+ "sdot z13.s, z7.b, z1.b[3]\n"
+ "sdot z17.s, z7.b, z2.b[3]\n"
+ "sdot z21.s, z7.b, z3.b[3]\n"
+ "sdot z25.s, z7.b, z4.b[3]\n"
+ "sdot z29.s, z7.b, z5.b[3]\n"
+ "ld1b { z7.b }, p2/Z, [x14, #3, MUL VL]\n"
+ "addvl x14, x14, #4\n"
+ "sdot z10.s, z6.b, z0.b[3]\n"
+ "sdot z14.s, z6.b, z1.b[3]\n"
+ "sdot z18.s, z6.b, z2.b[3]\n"
+ "sdot z22.s, z6.b, z3.b[3]\n"
+ "sdot z26.s, z6.b, z4.b[3]\n"
+ "sdot z30.s, z6.b, z5.b[3]\n"
+ "sdot z11.s, z7.b, z0.b[3]\n"
+ "sdot z15.s, z7.b, z1.b[3]\n"
+ "sdot z19.s, z7.b, z2.b[3]\n"
+ "sdot z23.s, z7.b, z3.b[3]\n"
+ "sdot z27.s, z7.b, z4.b[3]\n"
+ "sdot z31.s, z7.b, z5.b[3]\n"
+ "80:" // Height 6: Multiply loop: multiply skip
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x12, x12, #0x1\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "prfm pldl1keep, [x22, #0x80]\n"
+ "prfm pldl1keep, [x20, #0x80]\n"
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "cmp x12, x19\n"
+ "bne 75b\n"
+ "prfm pstl1keep, [x13, #0x0]\n"
+ "prfm pstl1keep, [x9, #0x0]\n"
+ "prfm pstl1keep, [x27, #0x0]\n"
+ "prfm pstl1keep, [x25, #0x0]\n"
+ "prfm pstl1keep, [x23, #0x0]\n"
+ "prfm pstl1keep, [x21, #0x0]\n"
+ "ld1w { z0.s }, p2/Z, [x16]\n"
+ "add z8.s, z8.s, z0.s\n"
+ "ld1w { z1.s }, p2/Z, [x16, #1, MUL VL]\n"
+ "add z12.s, z12.s, z0.s\n"
+ "ld1w { z2.s }, p2/Z, [x16, #2, MUL VL]\n"
+ "add z16.s, z16.s, z0.s\n"
+ "ld1w { z3.s }, p2/Z, [x16, #3, MUL VL]\n"
+ "addvl x16, x16, #4\n"
+ "add z9.s, z9.s, z1.s\n"
+ "add z13.s, z13.s, z1.s\n"
+ "add z10.s, z10.s, z2.s\n"
+ "add z11.s, z11.s, z3.s\n"
+ "add z14.s, z14.s, z2.s\n"
+ "add z15.s, z15.s, z3.s\n"
+ "add z17.s, z17.s, z1.s\n"
+ "add z18.s, z18.s, z2.s\n"
+ "add z19.s, z19.s, z3.s\n"
+ "add z20.s, z20.s, z0.s\n"
+ "add z21.s, z21.s, z1.s\n"
+ "add z22.s, z22.s, z2.s\n"
+ "add z23.s, z23.s, z3.s\n"
+ "add z24.s, z24.s, z0.s\n"
+ "add z25.s, z25.s, z1.s\n"
+ "add z26.s, z26.s, z2.s\n"
+ "add z27.s, z27.s, z3.s\n"
+ "add z28.s, z28.s, z0.s\n"
+ "add z29.s, z29.s, z1.s\n"
+ "add z30.s, z30.s, z2.s\n"
+ "add z31.s, z31.s, z3.s\n"
+ "tbz %x[flags], #4, 81f\n"
+ "ld1w { z0.s }, p2/Z, [x17]\n"
+ "ld1w { z4.s }, p2/Z, [x8]\n"
+ "ld1w { z1.s }, p2/Z, [x17, #1, MUL VL]\n"
+ "ld1w { z5.s }, p2/Z, [x8, #1, MUL VL]\n"
+ "ld1w { z2.s }, p2/Z, [x17, #2, MUL VL]\n"
+ "ld1w { z6.s }, p2/Z, [x8, #2, MUL VL]\n"
+ "ld1w { z3.s }, p2/Z, [x17, #3, MUL VL]\n"
+ "addvl x17, x17, #4\n"
+ "ld1w { z7.s }, p2/Z, [x8, #3, MUL VL]\n"
+ "addvl x8, x8, #4\n"
+ "b 82f\n"
+ "81:" // Height 6: per layer parameters
+ "add x19, %x[qp], %[per_layer_right_shift]\n"
+ "ld1rw { z0.s }, p2/Z, [x19]\n"
+ "mov z1.d, z0.d\n"
+ "add x19, %x[qp], %[per_layer_mul]\n"
+ "ld1rw { z4.s }, p2/Z, [x19]\n"
+ "mov z2.d, z0.d\n"
+ "mov z3.d, z0.d\n"
+ "mov z5.d, z4.d\n"
+ "mov z6.d, z4.d\n"
+ "mov z7.d, z4.d\n"
+ "82:" // Height 6: parameters loaded
+ ".inst 0x04a47508 // sqrdmulh z8.s, z8.s, z4.s\n"
+ ".inst 0x04a57529 // sqrdmulh z9.s, z9.s, z5.s\n"
+ ".inst 0x04a6754a // sqrdmulh z10.s, z10.s, z6.s\n"
+ ".inst 0x04a7756b // sqrdmulh z11.s, z11.s, z7.s\n"
+ ".inst 0x04a4758c // sqrdmulh z12.s, z12.s, z4.s\n"
+ ".inst 0x04a575ad // sqrdmulh z13.s, z13.s, z5.s\n"
+ ".inst 0x04a675ce // sqrdmulh z14.s, z14.s, z6.s\n"
+ ".inst 0x04a775ef // sqrdmulh z15.s, z15.s, z7.s\n"
+ ".inst 0x04a47610 // sqrdmulh z16.s, z16.s, z4.s\n"
+ ".inst 0x04a57631 // sqrdmulh z17.s, z17.s, z5.s\n"
+ ".inst 0x04a67652 // sqrdmulh z18.s, z18.s, z6.s\n"
+ ".inst 0x04a77673 // sqrdmulh z19.s, z19.s, z7.s\n"
+ ".inst 0x04a47694 // sqrdmulh z20.s, z20.s, z4.s\n"
+ ".inst 0x04a576b5 // sqrdmulh z21.s, z21.s, z5.s\n"
+ ".inst 0x04a676d6 // sqrdmulh z22.s, z22.s, z6.s\n"
+ ".inst 0x04a776f7 // sqrdmulh z23.s, z23.s, z7.s\n"
+ ".inst 0x04a47718 // sqrdmulh z24.s, z24.s, z4.s\n"
+ ".inst 0x04a57739 // sqrdmulh z25.s, z25.s, z5.s\n"
+ ".inst 0x04a6775a // sqrdmulh z26.s, z26.s, z6.s\n"
+ ".inst 0x04a7777b // sqrdmulh z27.s, z27.s, z7.s\n"
+ ".inst 0x04a4779c // sqrdmulh z28.s, z28.s, z4.s\n"
+ ".inst 0x04a577bd // sqrdmulh z29.s, z29.s, z5.s\n"
+ ".inst 0x04a677de // sqrdmulh z30.s, z30.s, z6.s\n"
+ ".inst 0x04a777ff // sqrdmulh z31.s, z31.s, z7.s\n"
+ "tbz %x[flags], #5, 83f\n"
+ "and z4.d, z8.d, z0.d\n"
+ "asr z4.s, z4.s, #0x1f\n"
+ "and z5.d, z9.d, z1.d\n"
+ "and z6.d, z10.d, z2.d\n"
+ "asr z5.s, z5.s, #0x1f\n"
+ "and z7.d, z11.d, z3.d\n"
+ "asr z6.s, z6.s, #0x1f\n"
+ "sqadd z8.s, z8.s, z4.s\n"
+ "asr z7.s, z7.s, #0x1f\n"
+ "and z4.d, z12.d, z0.d\n"
+ "sqadd z9.s, z9.s, z5.s\n"
+ "asr z4.s, z4.s, #0x1f\n"
+ "sqadd z10.s, z10.s, z6.s\n"
+ "and z5.d, z13.d, z1.d\n"
+ "asr z5.s, z5.s, #0x1f\n"
+ "sqadd z11.s, z11.s, z7.s\n"
+ "and z6.d, z14.d, z2.d\n"
+ "asr z6.s, z6.s, #0x1f\n"
+ "sqadd z12.s, z12.s, z4.s\n"
+ "and z7.d, z15.d, z3.d\n"
+ "asr z7.s, z7.s, #0x1f\n"
+ "sqadd z13.s, z13.s, z5.s\n"
+ "and z4.d, z16.d, z0.d\n"
+ "asr z4.s, z4.s, #0x1f\n"
+ "sqadd z14.s, z14.s, z6.s\n"
+ "and z5.d, z17.d, z1.d\n"
+ "asr z5.s, z5.s, #0x1f\n"
+ "sqadd z15.s, z15.s, z7.s\n"
+ "and z6.d, z18.d, z2.d\n"
+ "asr z6.s, z6.s, #0x1f\n"
+ "sqadd z16.s, z16.s, z4.s\n"
+ "and z7.d, z19.d, z3.d\n"
+ "asr z7.s, z7.s, #0x1f\n"
+ "sqadd z17.s, z17.s, z5.s\n"
+ "and z4.d, z20.d, z0.d\n"
+ "asr z4.s, z4.s, #0x1f\n"
+ "sqadd z18.s, z18.s, z6.s\n"
+ "and z5.d, z21.d, z1.d\n"
+ "asr z5.s, z5.s, #0x1f\n"
+ "sqadd z19.s, z19.s, z7.s\n"
+ "and z6.d, z22.d, z2.d\n"
+ "asr z6.s, z6.s, #0x1f\n"
+ "sqadd z20.s, z20.s, z4.s\n"
+ "and z7.d, z23.d, z3.d\n"
+ "asr z7.s, z7.s, #0x1f\n"
+ "sqadd z21.s, z21.s, z5.s\n"
+ "and z4.d, z24.d, z0.d\n"
+ "asr z4.s, z4.s, #0x1f\n"
+ "sqadd z22.s, z22.s, z6.s\n"
+ "and z5.d, z25.d, z1.d\n"
+ "asr z5.s, z5.s, #0x1f\n"
+ "sqadd z23.s, z23.s, z7.s\n"
+ "and z6.d, z26.d, z2.d\n"
+ "asr z6.s, z6.s, #0x1f\n"
+ "sqadd z24.s, z24.s, z4.s\n"
+ "and z7.d, z27.d, z3.d\n"
+ "asr z7.s, z7.s, #0x1f\n"
+ "sqadd z25.s, z25.s, z5.s\n"
+ "and z4.d, z28.d, z0.d\n"
+ "asr z4.s, z4.s, #0x1f\n"
+ "sqadd z26.s, z26.s, z6.s\n"
+ "and z5.d, z29.d, z1.d\n"
+ "asr z5.s, z5.s, #0x1f\n"
+ "sqadd z27.s, z27.s, z7.s\n"
+ "and z6.d, z30.d, z2.d\n"
+ "asr z6.s, z6.s, #0x1f\n"
+ "sqadd z28.s, z28.s, z4.s\n"
+ "and z7.d, z31.d, z3.d\n"
+ "asr z7.s, z7.s, #0x1f\n"
+ "sqadd z29.s, z29.s, z5.s\n"
+ "sqadd z30.s, z30.s, z6.s\n"
+ "sqadd z31.s, z31.s, z7.s\n"
+ "83:" // Height 6: no shift correction
+ ".inst 0x44828808 // srshl z8.s, p2/M, z8.s, z0.s\n"
+ "add x19, %x[qp], %[c_offset]\n"
+ "ld1rw { z4.s }, p2/Z, [x19]\n"
+ ".inst 0x44828829 // srshl z9.s, p2/M, z9.s, z1.s\n"
+ "add x19, %x[qp], %[minval]\n"
+ ".inst 0x4482884a // srshl z10.s, p2/M, z10.s, z2.s\n"
+ "ld1rw { z5.s }, p2/Z, [x19]\n"
+ "add x19, %x[qp], %[maxval]\n"
+ ".inst 0x4482886b // srshl z11.s, p2/M, z11.s, z3.s\n"
+ "ld1rw { z6.s }, p2/Z, [x19]\n"
+ ".inst 0x4482880c // srshl z12.s, p2/M, z12.s, z0.s\n"
+ "add z8.s, z8.s, z4.s\n"
+ "add z9.s, z9.s, z4.s\n"
+ "add z10.s, z10.s, z4.s\n"
+ "add z11.s, z11.s, z4.s\n"
+ "add z12.s, z12.s, z4.s\n"
+ "smin z8.s, p2/M, z8.s, z6.s\n"
+ "smin z9.s, p2/M, z9.s, z6.s\n"
+ "smin z10.s, p2/M, z10.s, z6.s\n"
+ "smin z11.s, p2/M, z11.s, z6.s\n"
+ "smax z8.s, p2/M, z8.s, z5.s\n"
+ "smax z9.s, p2/M, z9.s, z5.s\n"
+ "smax z10.s, p2/M, z10.s, z5.s\n"
+ "smax z11.s, p2/M, z11.s, z5.s\n"
+ "smin z12.s, p2/M, z12.s, z6.s\n"
+ "uzp1 z8.h, z8.h, z9.h\n"
+ ".inst 0x4482882d // srshl z13.s, p2/M, z13.s, z1.s\n"
+ "uzp1 z9.h, z10.h, z11.h\n"
+ "smax z12.s, p2/M, z12.s, z5.s\n"
+ "uzp1 z8.b, z8.b, z9.b\n"
+ "st1b { z8.b }, p1, [x13]\n"
+ "add z13.s, z13.s, z4.s\n"
+ "addvl x13, x13, #1\n"
+ ".inst 0x4482884e // srshl z14.s, p2/M, z14.s, z2.s\n"
+ ".inst 0x4482886f // srshl z15.s, p2/M, z15.s, z3.s\n"
+ ".inst 0x44828810 // srshl z16.s, p2/M, z16.s, z0.s\n"
+ "smin z13.s, p2/M, z13.s, z6.s\n"
+ ".inst 0x44828831 // srshl z17.s, p2/M, z17.s, z1.s\n"
+ "add z14.s, z14.s, z4.s\n"
+ "add z15.s, z15.s, z4.s\n"
+ "add z16.s, z16.s, z4.s\n"
+ "add z17.s, z17.s, z4.s\n"
+ "smax z13.s, p2/M, z13.s, z5.s\n"
+ "smin z14.s, p2/M, z14.s, z6.s\n"
+ "smin z15.s, p2/M, z15.s, z6.s\n"
+ "smin z16.s, p2/M, z16.s, z6.s\n"
+ "uzp1 z12.h, z12.h, z13.h\n"
+ "smax z14.s, p2/M, z14.s, z5.s\n"
+ "smax z15.s, p2/M, z15.s, z5.s\n"
+ "smax z16.s, p2/M, z16.s, z5.s\n"
+ "smin z17.s, p2/M, z17.s, z6.s\n"
+ ".inst 0x44828852 // srshl z18.s, p2/M, z18.s, z2.s\n"
+ "uzp1 z13.h, z14.h, z15.h\n"
+ ".inst 0x44828873 // srshl z19.s, p2/M, z19.s, z3.s\n"
+ "uzp1 z12.b, z12.b, z13.b\n"
+ "st1b { z12.b }, p1, [x9]\n"
+ "add z18.s, z18.s, z4.s\n"
+ "addvl x9, x9, #1\n"
+ "add z19.s, z19.s, z4.s\n"
+ "smax z17.s, p2/M, z17.s, z5.s\n"
+ ".inst 0x44828814 // srshl z20.s, p2/M, z20.s, z0.s\n"
+ "smin z18.s, p2/M, z18.s, z6.s\n"
+ "smin z19.s, p2/M, z19.s, z6.s\n"
+ "uzp1 z16.h, z16.h, z17.h\n"
+ "add z20.s, z20.s, z4.s\n"
+ "smax z18.s, p2/M, z18.s, z5.s\n"
+ "smax z19.s, p2/M, z19.s, z5.s\n"
+ "smin z20.s, p2/M, z20.s, z6.s\n"
+ ".inst 0x44828835 // srshl z21.s, p2/M, z21.s, z1.s\n"
+ ".inst 0x44828856 // srshl z22.s, p2/M, z22.s, z2.s\n"
+ "uzp1 z17.h, z18.h, z19.h\n"
+ "smax z20.s, p2/M, z20.s, z5.s\n"
+ "add z21.s, z21.s, z4.s\n"
+ "add z22.s, z22.s, z4.s\n"
+ "uzp1 z16.b, z16.b, z17.b\n"
+ "st1b { z16.b }, p1, [x27]\n"
+ "smin z21.s, p2/M, z21.s, z6.s\n"
+ "addvl x27, x27, #1\n"
+ "smin z22.s, p2/M, z22.s, z6.s\n"
+ ".inst 0x44828877 // srshl z23.s, p2/M, z23.s, z3.s\n"
+ ".inst 0x44828818 // srshl z24.s, p2/M, z24.s, z0.s\n"
+ ".inst 0x44828839 // srshl z25.s, p2/M, z25.s, z1.s\n"
+ "smax z21.s, p2/M, z21.s, z5.s\n"
+ "add z23.s, z23.s, z4.s\n"
+ "add z24.s, z24.s, z4.s\n"
+ "add z25.s, z25.s, z4.s\n"
+ "uzp1 z20.h, z20.h, z21.h\n"
+ "smax z22.s, p2/M, z22.s, z5.s\n"
+ "smin z23.s, p2/M, z23.s, z6.s\n"
+ "smin z24.s, p2/M, z24.s, z6.s\n"
+ "smin z25.s, p2/M, z25.s, z6.s\n"
+ ".inst 0x4482885a // srshl z26.s, p2/M, z26.s, z2.s\n"
+ "smax z23.s, p2/M, z23.s, z5.s\n"
+ "smax z24.s, p2/M, z24.s, z5.s\n"
+ "smax z25.s, p2/M, z25.s, z5.s\n"
+ "add z26.s, z26.s, z4.s\n"
+ "uzp1 z21.h, z22.h, z23.h\n"
+ ".inst 0x4482887b // srshl z27.s, p2/M, z27.s, z3.s\n"
+ "uzp1 z24.h, z24.h, z25.h\n"
+ "uzp1 z20.b, z20.b, z21.b\n"
+ "st1b { z20.b }, p1, [x25]\n"
+ "add z27.s, z27.s, z4.s\n"
+ "addvl x25, x25, #1\n"
+ "smin z26.s, p2/M, z26.s, z6.s\n"
+ ".inst 0x4482881c // srshl z28.s, p2/M, z28.s, z0.s\n"
+ ".inst 0x4482883d // srshl z29.s, p2/M, z29.s, z1.s\n"
+ "smin z27.s, p2/M, z27.s, z6.s\n"
+ ".inst 0x4482885e // srshl z30.s, p2/M, z30.s, z2.s\n"
+ "add z28.s, z28.s, z4.s\n"
+ "add z29.s, z29.s, z4.s\n"
+ "smax z26.s, p2/M, z26.s, z5.s\n"
+ "add z30.s, z30.s, z4.s\n"
+ "smax z27.s, p2/M, z27.s, z5.s\n"
+ "smin z28.s, p2/M, z28.s, z6.s\n"
+ "smin z29.s, p2/M, z29.s, z6.s\n"
+ "smin z30.s, p2/M, z30.s, z6.s\n"
+ "uzp1 z25.h, z26.h, z27.h\n"
+ "smax z28.s, p2/M, z28.s, z5.s\n"
+ "uzp1 z24.b, z24.b, z25.b\n"
+ "st1b { z24.b }, p1, [x23]\n"
+ "smax z29.s, p2/M, z29.s, z5.s\n"
+ "addvl x23, x23, #1\n"
+ "smax z30.s, p2/M, z30.s, z5.s\n"
+ ".inst 0x4482887f // srshl z31.s, p2/M, z31.s, z3.s\n"
+ "uzp1 z28.h, z28.h, z29.h\n"
+ "add z31.s, z31.s, z4.s\n"
+ "smin z31.s, p2/M, z31.s, z6.s\n"
+ "smax z31.s, p2/M, z31.s, z5.s\n"
+ "uzp1 z29.h, z30.h, z31.h\n"
+ "uzp1 z28.b, z28.b, z29.b\n"
+ "st1b { z28.b }, p1, [x21]\n"
+ "addvl x21, x21, #1\n"
+ "84:" // Height 6: Writeback done
+ "mov x19, #0x0\n"
+ "incw x19, ALL, MUL #4\n"
+ "subs x15, x15, x19\n"
+ "bgt 73b\n"
+ "subs %x[M], %x[M], #0x6\n"
+ "beq 86f\n"
+ "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "tbz %x[flags], #3, 85f\n"
+ "add x20, x20, #0x6\n"
+ "str x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "b 1b\n"
+ "85:" // Update direct input
+ "mov x19, #0x6\n"
+ "madd %x[input_ptr], x19, x20, %x[input_ptr]\n"
+ "b 1b\n"
+ "86:" // Exit
+
+ : [M] "+r" (M), [input_ptr] "+r" (input_ptr), [output_ptr] "+r" (output_ptr)
+ : [args_ptr] "r" (&ka), [c_offset] "I" (offsetof(Requantize32, c_offset)), [col_bias] "r" (col_bias), [flags] "r" (flags), [maxval] "I" (offsetof(Requantize32, maxval)), [minval] "I" (offsetof(Requantize32, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_multiplier_ptr] "I" (offsetof(KernelArgs, multiplier_ptr)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_shift_ptr] "I" (offsetof(KernelArgs, shift_ptr)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths)), [per_layer_mul] "I" (offsetof(Requantize32, per_layer_mul)), [per_layer_right_shift] "I" (offsetof(Requantize32, per_layer_right_shift)), [qp] "r" (qp)
+ : "cc", "memory", "p0", "p1", "p2", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+ );
+}
+
+} // namespace arm_gemm
+#endif // __ARM_FEATURE_SVE
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8s32_dot_4VLx4/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8s32_dot_4VLx4/generic.cpp
deleted file mode 100644
index b30b884..0000000
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8s32_dot_4VLx4/generic.cpp
+++ /dev/null
@@ -1,2137 +0,0 @@
-/*
- * Copyright (c) 2018-2020 Arm Limited.
- *
- * SPDX-License-Identifier: MIT
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to
- * deal in the Software without restriction, including without limitation the
- * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
- * sell copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-#ifdef __ARM_FEATURE_SVE
-
-#include <algorithm>
-
-#include "arm_gemm.hpp"
-#include <cstdint>
-#include "../../asmlib.hpp"
-#include "../../utils.hpp"
-
-namespace arm_gemm {
-
-void sve_hybrid_s8s32_dot_4VLx4(const int8_t *A, int lda, const int8_t *B, int32_t *C, int ldc, int M, int N, int K, const int32_t *, Activation , bool accumulate) {
- const int K_stride = ((K + 3) / 4) * 4;
- const long loops_count = ((K + 16) / 32) - 1;
- K -= loops_count * 32;
- const long regs_count = (K / 16) - 1;
- K -= (regs_count + 1) * 16;
- const long leftovers = K;
- const long blocks_count = (K + 3) / 4;
-
- int rows_to_compute;
-
- for (int y=0; y<M; y+=rows_to_compute) {
- const int8_t * const a_ptr0_base = A + (y * lda);
- const unsigned long ldab = lda * sizeof(int8_t);
-
- int32_t *c_ptr0 = C + (y * ldc);
-
- rows_to_compute = M-y;
- if (rows_to_compute > 4) {
- if (rows_to_compute % 4) {
- rows_to_compute = 4 - 1;
- } else {
- rows_to_compute = 4;
- }
- }
-
- for (int x0=0; x0<N; x0+=(4 * get_vector_length<int32_t>())) {
- const long width = std::min((unsigned long)N-x0, (4 * get_vector_length<int32_t>()));
- long loops = loops_count;
- long regs = regs_count;
- long temp = 0;
- long blocks = blocks_count;
- const int8_t *a_ptr0 = a_ptr0_base;
- const int8_t *b_ptr0 = B + (K_stride * x0);
- const unsigned long ldcb = ldc * sizeof(int32_t);
-
- switch(rows_to_compute) {
- case 1:
- __asm __volatile (
- "whilelt p6.b, %[temp], %[leftovers]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "ptrue p7.b\n"
- "whilelt p1.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "whilelt p2.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "whilelt p3.s, %[temp], %[width]\n"
- "cbnz %[accumulate], 1f\n"
- "mov z16.s, #0\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "mov z17.s, #0\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
- "mov z18.s, #0\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "mov z19.s, #0\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "ld1w z16.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n"
- "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n"
- "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "cbz %[loops], 2f\n"
- "3:\n"
- "sdot z16.s, z8.b, z0.b[0]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "sdot z17.s, z9.b, z0.b[0]\n"
- "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n"
- "sdot z18.s, z10.b, z0.b[0]\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
- "sdot z19.s, z11.b, z0.b[0]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "sdot z16.s, z12.b, z0.b[1]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "sdot z17.s, z13.b, z0.b[1]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "sdot z18.s, z14.b, z0.b[1]\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "sdot z19.s, z15.b, z0.b[1]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "sdot z16.s, z8.b, z0.b[2]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "sdot z17.s, z9.b, z0.b[2]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "sdot z18.s, z10.b, z0.b[2]\n"
- "subs %[loops], %[loops], #0x1\n"
- "sdot z19.s, z11.b, z0.b[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "sdot z16.s, z12.b, z0.b[3]\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
- "sdot z17.s, z13.b, z0.b[3]\n"
- "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "sdot z18.s, z14.b, z0.b[3]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "sdot z19.s, z15.b, z0.b[3]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "sdot z16.s, z8.b, z4.b[0]\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "sdot z17.s, z9.b, z4.b[0]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "sdot z18.s, z10.b, z4.b[0]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "sdot z19.s, z11.b, z4.b[0]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "sdot z16.s, z12.b, z4.b[1]\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0], #-0x10]\n"
- "sdot z17.s, z13.b, z4.b[1]\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
- "sdot z18.s, z14.b, z4.b[1]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "sdot z19.s, z15.b, z4.b[1]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "sdot z16.s, z8.b, z4.b[2]\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "sdot z17.s, z9.b, z4.b[2]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "sdot z18.s, z10.b, z4.b[2]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "sdot z19.s, z11.b, z4.b[2]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "sdot z16.s, z12.b, z4.b[3]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "sdot z17.s, z13.b, z4.b[3]\n"
- "sdot z18.s, z14.b, z4.b[3]\n"
- "sdot z19.s, z15.b, z4.b[3]\n"
- "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "b.ne 3b\n"
- "2:\n"
- "cbz %[regs], 4f\n"
- "sdot z16.s, z8.b, z0.b[0]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "sdot z17.s, z9.b, z0.b[0]\n"
- "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n"
- "sdot z18.s, z10.b, z0.b[0]\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
- "sdot z19.s, z11.b, z0.b[0]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "sdot z16.s, z12.b, z0.b[1]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "sdot z17.s, z13.b, z0.b[1]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "sdot z18.s, z14.b, z0.b[1]\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "sdot z19.s, z15.b, z0.b[1]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "sdot z16.s, z8.b, z0.b[2]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "sdot z17.s, z9.b, z0.b[2]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "sdot z18.s, z10.b, z0.b[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "sdot z19.s, z11.b, z0.b[2]\n"
- "sdot z16.s, z12.b, z0.b[3]\n"
- "sdot z17.s, z13.b, z0.b[3]\n"
- "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "sdot z18.s, z14.b, z0.b[3]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "sdot z19.s, z15.b, z0.b[3]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "sdot z16.s, z8.b, z4.b[0]\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "sdot z17.s, z9.b, z4.b[0]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "sdot z18.s, z10.b, z4.b[0]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "sdot z19.s, z11.b, z4.b[0]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "sdot z16.s, z12.b, z4.b[1]\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
- "sdot z17.s, z13.b, z4.b[1]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "sdot z18.s, z14.b, z4.b[1]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "sdot z19.s, z15.b, z4.b[1]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "sdot z16.s, z8.b, z4.b[2]\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "sdot z17.s, z9.b, z4.b[2]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "sdot z18.s, z10.b, z4.b[2]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "sdot z19.s, z11.b, z4.b[2]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "sdot z16.s, z12.b, z4.b[3]\n"
- "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x10]\n"
- "sdot z17.s, z13.b, z4.b[3]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "sdot z18.s, z14.b, z4.b[3]\n"
- "addvl %[a_ptr0], %[a_ptr0], #2\n"
- "sdot z19.s, z15.b, z4.b[3]\n"
- "cbz %[blocks], 5f\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "sdot z16.s, z8.b, z0.b[0]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "sdot z17.s, z9.b, z0.b[0]\n"
- "sdot z18.s, z10.b, z0.b[0]\n"
- "sdot z19.s, z11.b, z0.b[0]\n"
- "b.eq 5f\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "sdot z16.s, z12.b, z0.b[1]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "sdot z17.s, z13.b, z0.b[1]\n"
- "sdot z18.s, z14.b, z0.b[1]\n"
- "sdot z19.s, z15.b, z0.b[1]\n"
- "b.eq 5f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "sdot z16.s, z8.b, z0.b[2]\n"
- "sdot z17.s, z9.b, z0.b[2]\n"
- "sdot z18.s, z10.b, z0.b[2]\n"
- "sdot z19.s, z11.b, z0.b[2]\n"
- "b.eq 5f\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "sdot z16.s, z12.b, z0.b[3]\n"
- "sdot z17.s, z13.b, z0.b[3]\n"
- "sdot z18.s, z14.b, z0.b[3]\n"
- "sdot z19.s, z15.b, z0.b[3]\n"
- "b 5f\n"
- "4:\n"
- "sdot z16.s, z8.b, z0.b[0]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "sdot z17.s, z9.b, z0.b[0]\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
- "sdot z18.s, z10.b, z0.b[0]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "sdot z19.s, z11.b, z0.b[0]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "sdot z16.s, z12.b, z0.b[1]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "sdot z17.s, z13.b, z0.b[1]\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "sdot z18.s, z14.b, z0.b[1]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "sdot z19.s, z15.b, z0.b[1]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "sdot z16.s, z8.b, z0.b[2]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "sdot z17.s, z9.b, z0.b[2]\n"
- "ld1rqb z4.b, p6/z, [%[a_ptr0]]\n"
- "sdot z18.s, z10.b, z0.b[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "sdot z19.s, z11.b, z0.b[2]\n"
- "addvl %[a_ptr0], %[a_ptr0], #1\n"
- "sdot z16.s, z12.b, z0.b[3]\n"
- "sdot z17.s, z13.b, z0.b[3]\n"
- "sdot z18.s, z14.b, z0.b[3]\n"
- "sdot z19.s, z15.b, z0.b[3]\n"
- "cbz %[blocks], 5f\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "sdot z16.s, z8.b, z4.b[0]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "sdot z17.s, z9.b, z4.b[0]\n"
- "sdot z18.s, z10.b, z4.b[0]\n"
- "sdot z19.s, z11.b, z4.b[0]\n"
- "b.eq 5f\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "sdot z16.s, z12.b, z4.b[1]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "sdot z17.s, z13.b, z4.b[1]\n"
- "sdot z18.s, z14.b, z4.b[1]\n"
- "sdot z19.s, z15.b, z4.b[1]\n"
- "b.eq 5f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "sdot z16.s, z8.b, z4.b[2]\n"
- "sdot z17.s, z9.b, z4.b[2]\n"
- "sdot z18.s, z10.b, z4.b[2]\n"
- "sdot z19.s, z11.b, z4.b[2]\n"
- "b.eq 5f\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "sdot z16.s, z12.b, z4.b[3]\n"
- "sdot z17.s, z13.b, z4.b[3]\n"
- "sdot z18.s, z14.b, z4.b[3]\n"
- "sdot z19.s, z15.b, z4.b[3]\n"
- "5:\n"
- "st1w z16.s, p0, [%[c_ptr0]]\n"
- "st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n"
- "st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n"
- "st1w z19.s, p3, [%[c_ptr0], #3, MUL VL]\n"
- "addvl %[c_ptr0], %[c_ptr0], #4\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks)
- : [width] "r" (width), [accumulate] "r" (static_cast<uint64_t>(accumulate)), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers)
- : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
- );
- break;
- case 2:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "c_ptr1 .req X1\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "whilelt p6.b, %[temp], %[leftovers]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "ptrue p7.b\n"
- "whilelt p1.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "whilelt p2.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "whilelt p3.s, %[temp], %[width]\n"
- "cbnz %[accumulate], 1f\n"
- "mov z16.s, #0\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "mov z17.s, #0\n"
- "ld1rqb z1.b, p7/z, [a_ptr1]\n"
- "mov z18.s, #0\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
- "mov z19.s, #0\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "mov z20.s, #0\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "mov z21.s, #0\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "mov z22.s, #0\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "mov z23.s, #0\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "ld1w z16.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n"
- "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n"
- "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n"
- "ld1w z20.s, p0/z, [c_ptr1]\n"
- "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n"
- "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n"
- "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ld1rqb z1.b, p7/z, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "cbz %[loops], 2f\n"
- "3:\n"
- "sdot z16.s, z8.b, z0.b[0]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "sdot z20.s, z8.b, z1.b[0]\n"
- "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n"
- "sdot z17.s, z9.b, z0.b[0]\n"
- "ld1rqb z5.b, p7/z, [a_ptr1]\n"
- "sdot z21.s, z9.b, z1.b[0]\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
- "sdot z18.s, z10.b, z0.b[0]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "sdot z22.s, z10.b, z1.b[0]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "sdot z19.s, z11.b, z0.b[0]\n"
- "subs %[loops], %[loops], #0x1\n"
- "sdot z23.s, z11.b, z1.b[0]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "sdot z16.s, z12.b, z0.b[1]\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
- "sdot z20.s, z12.b, z1.b[1]\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "sdot z17.s, z13.b, z0.b[1]\n"
- "add a_ptr1, a_ptr1, #0x20\n"
- "sdot z21.s, z13.b, z1.b[1]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "sdot z18.s, z14.b, z0.b[1]\n"
- "sdot z22.s, z14.b, z1.b[1]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "sdot z19.s, z15.b, z0.b[1]\n"
- "sdot z23.s, z15.b, z1.b[1]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "sdot z16.s, z8.b, z0.b[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "sdot z20.s, z8.b, z1.b[2]\n"
- "sdot z17.s, z9.b, z0.b[2]\n"
- "sdot z21.s, z9.b, z1.b[2]\n"
- "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "sdot z18.s, z10.b, z0.b[2]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "sdot z22.s, z10.b, z1.b[2]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "sdot z19.s, z11.b, z0.b[2]\n"
- "sdot z23.s, z11.b, z1.b[2]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "sdot z16.s, z12.b, z0.b[3]\n"
- "sdot z20.s, z12.b, z1.b[3]\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "sdot z17.s, z13.b, z0.b[3]\n"
- "sdot z21.s, z13.b, z1.b[3]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "sdot z18.s, z14.b, z0.b[3]\n"
- "sdot z22.s, z14.b, z1.b[3]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "sdot z19.s, z15.b, z0.b[3]\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0], #-0x10]\n"
- "sdot z23.s, z15.b, z1.b[3]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "sdot z16.s, z8.b, z4.b[0]\n"
- "ld1rqb z1.b, p7/z, [a_ptr1, #-0x10]\n"
- "sdot z20.s, z8.b, z5.b[0]\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
- "sdot z17.s, z9.b, z4.b[0]\n"
- "sdot z21.s, z9.b, z5.b[0]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "sdot z18.s, z10.b, z4.b[0]\n"
- "sdot z22.s, z10.b, z5.b[0]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "sdot z19.s, z11.b, z4.b[0]\n"
- "sdot z23.s, z11.b, z5.b[0]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "sdot z16.s, z12.b, z4.b[1]\n"
- "sdot z20.s, z12.b, z5.b[1]\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "sdot z17.s, z13.b, z4.b[1]\n"
- "sdot z21.s, z13.b, z5.b[1]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "sdot z18.s, z14.b, z4.b[1]\n"
- "sdot z22.s, z14.b, z5.b[1]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "sdot z19.s, z15.b, z4.b[1]\n"
- "sdot z23.s, z15.b, z5.b[1]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "sdot z16.s, z8.b, z4.b[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "sdot z20.s, z8.b, z5.b[2]\n"
- "sdot z17.s, z9.b, z4.b[2]\n"
- "sdot z21.s, z9.b, z5.b[2]\n"
- "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "sdot z18.s, z10.b, z4.b[2]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "sdot z22.s, z10.b, z5.b[2]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "sdot z19.s, z11.b, z4.b[2]\n"
- "sdot z23.s, z11.b, z5.b[2]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "sdot z16.s, z12.b, z4.b[3]\n"
- "sdot z20.s, z12.b, z5.b[3]\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "sdot z17.s, z13.b, z4.b[3]\n"
- "sdot z21.s, z13.b, z5.b[3]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "sdot z18.s, z14.b, z4.b[3]\n"
- "sdot z22.s, z14.b, z5.b[3]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "sdot z19.s, z15.b, z4.b[3]\n"
- "sdot z23.s, z15.b, z5.b[3]\n"
- "b.ne 3b\n"
- "2:\n"
- "cbz %[regs], 4f\n"
- "sdot z16.s, z8.b, z0.b[0]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "sdot z20.s, z8.b, z1.b[0]\n"
- "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n"
- "sdot z17.s, z9.b, z0.b[0]\n"
- "ld1rqb z5.b, p7/z, [a_ptr1]\n"
- "sdot z21.s, z9.b, z1.b[0]\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
- "sdot z18.s, z10.b, z0.b[0]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "sdot z22.s, z10.b, z1.b[0]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "sdot z19.s, z11.b, z0.b[0]\n"
- "sdot z23.s, z11.b, z1.b[0]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "sdot z16.s, z12.b, z0.b[1]\n"
- "sdot z20.s, z12.b, z1.b[1]\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "sdot z17.s, z13.b, z0.b[1]\n"
- "sdot z21.s, z13.b, z1.b[1]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "sdot z18.s, z14.b, z0.b[1]\n"
- "sdot z22.s, z14.b, z1.b[1]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "sdot z19.s, z15.b, z0.b[1]\n"
- "sdot z23.s, z15.b, z1.b[1]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "sdot z16.s, z8.b, z0.b[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "sdot z20.s, z8.b, z1.b[2]\n"
- "sdot z17.s, z9.b, z0.b[2]\n"
- "sdot z21.s, z9.b, z1.b[2]\n"
- "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "sdot z18.s, z10.b, z0.b[2]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "sdot z22.s, z10.b, z1.b[2]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "sdot z19.s, z11.b, z0.b[2]\n"
- "sdot z23.s, z11.b, z1.b[2]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "sdot z16.s, z12.b, z0.b[3]\n"
- "sdot z20.s, z12.b, z1.b[3]\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "sdot z17.s, z13.b, z0.b[3]\n"
- "sdot z21.s, z13.b, z1.b[3]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "sdot z18.s, z14.b, z0.b[3]\n"
- "sdot z22.s, z14.b, z1.b[3]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "sdot z19.s, z15.b, z0.b[3]\n"
- "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x10]\n"
- "sdot z23.s, z15.b, z1.b[3]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "sdot z16.s, z8.b, z4.b[0]\n"
- "ld1rqb z1.b, p6/z, [a_ptr1, #0x10]\n"
- "sdot z20.s, z8.b, z5.b[0]\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
- "sdot z17.s, z9.b, z4.b[0]\n"
- "addvl %[a_ptr0], %[a_ptr0], #2\n"
- "sdot z21.s, z9.b, z5.b[0]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "sdot z18.s, z10.b, z4.b[0]\n"
- "addvl a_ptr1, a_ptr1, #2\n"
- "sdot z22.s, z10.b, z5.b[0]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "sdot z19.s, z11.b, z4.b[0]\n"
- "sdot z23.s, z11.b, z5.b[0]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "sdot z16.s, z12.b, z4.b[1]\n"
- "sdot z20.s, z12.b, z5.b[1]\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "sdot z17.s, z13.b, z4.b[1]\n"
- "sdot z21.s, z13.b, z5.b[1]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "sdot z18.s, z14.b, z4.b[1]\n"
- "sdot z22.s, z14.b, z5.b[1]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "sdot z19.s, z15.b, z4.b[1]\n"
- "sdot z23.s, z15.b, z5.b[1]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "sdot z16.s, z8.b, z4.b[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "sdot z20.s, z8.b, z5.b[2]\n"
- "sdot z17.s, z9.b, z4.b[2]\n"
- "sdot z21.s, z9.b, z5.b[2]\n"
- "sdot z18.s, z10.b, z4.b[2]\n"
- "sdot z22.s, z10.b, z5.b[2]\n"
- "sdot z19.s, z11.b, z4.b[2]\n"
- "sdot z23.s, z11.b, z5.b[2]\n"
- "sdot z16.s, z12.b, z4.b[3]\n"
- "sdot z20.s, z12.b, z5.b[3]\n"
- "sdot z17.s, z13.b, z4.b[3]\n"
- "sdot z21.s, z13.b, z5.b[3]\n"
- "sdot z18.s, z14.b, z4.b[3]\n"
- "sdot z22.s, z14.b, z5.b[3]\n"
- "sdot z19.s, z15.b, z4.b[3]\n"
- "sdot z23.s, z15.b, z5.b[3]\n"
- "cbz %[blocks], 5f\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "sdot z16.s, z8.b, z0.b[0]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "sdot z20.s, z8.b, z1.b[0]\n"
- "sdot z17.s, z9.b, z0.b[0]\n"
- "sdot z21.s, z9.b, z1.b[0]\n"
- "sdot z18.s, z10.b, z0.b[0]\n"
- "sdot z22.s, z10.b, z1.b[0]\n"
- "sdot z19.s, z11.b, z0.b[0]\n"
- "sdot z23.s, z11.b, z1.b[0]\n"
- "b.eq 5f\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "sdot z16.s, z12.b, z0.b[1]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "sdot z20.s, z12.b, z1.b[1]\n"
- "sdot z17.s, z13.b, z0.b[1]\n"
- "sdot z21.s, z13.b, z1.b[1]\n"
- "sdot z18.s, z14.b, z0.b[1]\n"
- "sdot z22.s, z14.b, z1.b[1]\n"
- "sdot z19.s, z15.b, z0.b[1]\n"
- "sdot z23.s, z15.b, z1.b[1]\n"
- "b.eq 5f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "sdot z16.s, z8.b, z0.b[2]\n"
- "sdot z20.s, z8.b, z1.b[2]\n"
- "sdot z17.s, z9.b, z0.b[2]\n"
- "sdot z21.s, z9.b, z1.b[2]\n"
- "sdot z18.s, z10.b, z0.b[2]\n"
- "sdot z22.s, z10.b, z1.b[2]\n"
- "sdot z19.s, z11.b, z0.b[2]\n"
- "sdot z23.s, z11.b, z1.b[2]\n"
- "b.eq 5f\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "sdot z16.s, z12.b, z0.b[3]\n"
- "sdot z20.s, z12.b, z1.b[3]\n"
- "sdot z17.s, z13.b, z0.b[3]\n"
- "sdot z21.s, z13.b, z1.b[3]\n"
- "sdot z18.s, z14.b, z0.b[3]\n"
- "sdot z22.s, z14.b, z1.b[3]\n"
- "sdot z19.s, z15.b, z0.b[3]\n"
- "sdot z23.s, z15.b, z1.b[3]\n"
- "b 5f\n"
- "4:\n"
- "sdot z16.s, z8.b, z0.b[0]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "sdot z20.s, z8.b, z1.b[0]\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
- "sdot z17.s, z9.b, z0.b[0]\n"
- "ld1rqb z4.b, p6/z, [%[a_ptr0]]\n"
- "sdot z21.s, z9.b, z1.b[0]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "sdot z18.s, z10.b, z0.b[0]\n"
- "ld1rqb z5.b, p6/z, [a_ptr1]\n"
- "sdot z22.s, z10.b, z1.b[0]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "sdot z19.s, z11.b, z0.b[0]\n"
- "addvl %[a_ptr0], %[a_ptr0], #1\n"
- "sdot z23.s, z11.b, z1.b[0]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "sdot z16.s, z12.b, z0.b[1]\n"
- "addvl a_ptr1, a_ptr1, #1\n"
- "sdot z20.s, z12.b, z1.b[1]\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "sdot z17.s, z13.b, z0.b[1]\n"
- "sdot z21.s, z13.b, z1.b[1]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "sdot z18.s, z14.b, z0.b[1]\n"
- "sdot z22.s, z14.b, z1.b[1]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "sdot z19.s, z15.b, z0.b[1]\n"
- "sdot z23.s, z15.b, z1.b[1]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "sdot z16.s, z8.b, z0.b[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "sdot z20.s, z8.b, z1.b[2]\n"
- "sdot z17.s, z9.b, z0.b[2]\n"
- "sdot z21.s, z9.b, z1.b[2]\n"
- "sdot z18.s, z10.b, z0.b[2]\n"
- "sdot z22.s, z10.b, z1.b[2]\n"
- "sdot z19.s, z11.b, z0.b[2]\n"
- "sdot z23.s, z11.b, z1.b[2]\n"
- "sdot z16.s, z12.b, z0.b[3]\n"
- "sdot z20.s, z12.b, z1.b[3]\n"
- "sdot z17.s, z13.b, z0.b[3]\n"
- "sdot z21.s, z13.b, z1.b[3]\n"
- "sdot z18.s, z14.b, z0.b[3]\n"
- "sdot z22.s, z14.b, z1.b[3]\n"
- "sdot z19.s, z15.b, z0.b[3]\n"
- "sdot z23.s, z15.b, z1.b[3]\n"
- "cbz %[blocks], 5f\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "sdot z16.s, z8.b, z4.b[0]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "sdot z20.s, z8.b, z5.b[0]\n"
- "sdot z17.s, z9.b, z4.b[0]\n"
- "sdot z21.s, z9.b, z5.b[0]\n"
- "sdot z18.s, z10.b, z4.b[0]\n"
- "sdot z22.s, z10.b, z5.b[0]\n"
- "sdot z19.s, z11.b, z4.b[0]\n"
- "sdot z23.s, z11.b, z5.b[0]\n"
- "b.eq 5f\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "sdot z16.s, z12.b, z4.b[1]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "sdot z20.s, z12.b, z5.b[1]\n"
- "sdot z17.s, z13.b, z4.b[1]\n"
- "sdot z21.s, z13.b, z5.b[1]\n"
- "sdot z18.s, z14.b, z4.b[1]\n"
- "sdot z22.s, z14.b, z5.b[1]\n"
- "sdot z19.s, z15.b, z4.b[1]\n"
- "sdot z23.s, z15.b, z5.b[1]\n"
- "b.eq 5f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "sdot z16.s, z8.b, z4.b[2]\n"
- "sdot z20.s, z8.b, z5.b[2]\n"
- "sdot z17.s, z9.b, z4.b[2]\n"
- "sdot z21.s, z9.b, z5.b[2]\n"
- "sdot z18.s, z10.b, z4.b[2]\n"
- "sdot z22.s, z10.b, z5.b[2]\n"
- "sdot z19.s, z11.b, z4.b[2]\n"
- "sdot z23.s, z11.b, z5.b[2]\n"
- "b.eq 5f\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "sdot z16.s, z12.b, z4.b[3]\n"
- "sdot z20.s, z12.b, z5.b[3]\n"
- "sdot z17.s, z13.b, z4.b[3]\n"
- "sdot z21.s, z13.b, z5.b[3]\n"
- "sdot z18.s, z14.b, z4.b[3]\n"
- "sdot z22.s, z14.b, z5.b[3]\n"
- "sdot z19.s, z15.b, z4.b[3]\n"
- "sdot z23.s, z15.b, z5.b[3]\n"
- "5:\n"
- "st1w z16.s, p0, [%[c_ptr0]]\n"
- "st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n"
- "st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n"
- "st1w z19.s, p3, [%[c_ptr0], #3, MUL VL]\n"
- "addvl %[c_ptr0], %[c_ptr0], #4\n"
- "st1w z20.s, p0, [c_ptr1]\n"
- "st1w z21.s, p1, [c_ptr1, #1, MUL VL]\n"
- "st1w z22.s, p2, [c_ptr1, #2, MUL VL]\n"
- "st1w z23.s, p3, [c_ptr1, #3, MUL VL]\n"
- ".unreq a_ptr1\n"
- ".unreq c_ptr1\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks)
- : [width] "r" (width), [accumulate] "r" (static_cast<uint64_t>(accumulate)), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers)
- : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "cc", "memory"
- );
- break;
- case 3:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "c_ptr1 .req X2\n"
- "c_ptr2 .req X3\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "whilelt p6.b, %[temp], %[leftovers]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "ptrue p7.b\n"
- "whilelt p1.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "whilelt p2.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "whilelt p3.s, %[temp], %[width]\n"
- "cbnz %[accumulate], 1f\n"
- "mov z16.s, #0\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "mov z17.s, #0\n"
- "ld1rqb z1.b, p7/z, [a_ptr1]\n"
- "mov z18.s, #0\n"
- "ld1rqb z2.b, p7/z, [a_ptr2]\n"
- "mov z19.s, #0\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
- "mov z20.s, #0\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "mov z21.s, #0\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "mov z22.s, #0\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "mov z23.s, #0\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "mov z24.s, #0\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "mov z25.s, #0\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "mov z26.s, #0\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "mov z27.s, #0\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "ld1w z16.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n"
- "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n"
- "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n"
- "ld1w z20.s, p0/z, [c_ptr1]\n"
- "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n"
- "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n"
- "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n"
- "ld1w z24.s, p0/z, [c_ptr2]\n"
- "ld1w z25.s, p1/z, [c_ptr2, #1, MUL VL]\n"
- "ld1w z26.s, p2/z, [c_ptr2, #2, MUL VL]\n"
- "ld1w z27.s, p3/z, [c_ptr2, #3, MUL VL]\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ld1rqb z1.b, p7/z, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "ld1rqb z2.b, p7/z, [a_ptr2]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "cbz %[loops], 2f\n"
- "3:\n"
- "sdot z16.s, z8.b, z0.b[0]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "sdot z20.s, z8.b, z1.b[0]\n"
- "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n"
- "sdot z24.s, z8.b, z2.b[0]\n"
- "ld1rqb z5.b, p7/z, [a_ptr1]\n"
- "sdot z17.s, z9.b, z0.b[0]\n"
- "ld1rqb z6.b, p7/z, [a_ptr2]\n"
- "sdot z21.s, z9.b, z1.b[0]\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
- "sdot z25.s, z9.b, z2.b[0]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "sdot z18.s, z10.b, z0.b[0]\n"
- "subs %[loops], %[loops], #0x1\n"
- "sdot z22.s, z10.b, z1.b[0]\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
- "sdot z26.s, z10.b, z2.b[0]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "sdot z19.s, z11.b, z0.b[0]\n"
- "add a_ptr1, a_ptr1, #0x20\n"
- "sdot z23.s, z11.b, z1.b[0]\n"
- "add a_ptr2, a_ptr2, #0x20\n"
- "sdot z27.s, z11.b, z2.b[0]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "sdot z16.s, z12.b, z0.b[1]\n"
- "sdot z20.s, z12.b, z1.b[1]\n"
- "sdot z24.s, z12.b, z2.b[1]\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "sdot z17.s, z13.b, z0.b[1]\n"
- "sdot z21.s, z13.b, z1.b[1]\n"
- "sdot z25.s, z13.b, z2.b[1]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "sdot z18.s, z14.b, z0.b[1]\n"
- "sdot z22.s, z14.b, z1.b[1]\n"
- "sdot z26.s, z14.b, z2.b[1]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "sdot z19.s, z15.b, z0.b[1]\n"
- "sdot z23.s, z15.b, z1.b[1]\n"
- "sdot z27.s, z15.b, z2.b[1]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "sdot z16.s, z8.b, z0.b[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "sdot z20.s, z8.b, z1.b[2]\n"
- "sdot z24.s, z8.b, z2.b[2]\n"
- "sdot z17.s, z9.b, z0.b[2]\n"
- "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "sdot z21.s, z9.b, z1.b[2]\n"
- "sdot z25.s, z9.b, z2.b[2]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "sdot z18.s, z10.b, z0.b[2]\n"
- "sdot z22.s, z10.b, z1.b[2]\n"
- "sdot z26.s, z10.b, z2.b[2]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "sdot z19.s, z11.b, z0.b[2]\n"
- "sdot z23.s, z11.b, z1.b[2]\n"
- "sdot z27.s, z11.b, z2.b[2]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "sdot z16.s, z12.b, z0.b[3]\n"
- "sdot z20.s, z12.b, z1.b[3]\n"
- "sdot z24.s, z12.b, z2.b[3]\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "sdot z17.s, z13.b, z0.b[3]\n"
- "sdot z21.s, z13.b, z1.b[3]\n"
- "sdot z25.s, z13.b, z2.b[3]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "sdot z18.s, z14.b, z0.b[3]\n"
- "sdot z22.s, z14.b, z1.b[3]\n"
- "sdot z26.s, z14.b, z2.b[3]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "sdot z19.s, z15.b, z0.b[3]\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0], #-0x10]\n"
- "sdot z23.s, z15.b, z1.b[3]\n"
- "ld1rqb z1.b, p7/z, [a_ptr1, #-0x10]\n"
- "sdot z27.s, z15.b, z2.b[3]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "sdot z16.s, z8.b, z4.b[0]\n"
- "ld1rqb z2.b, p7/z, [a_ptr2, #-0x10]\n"
- "sdot z20.s, z8.b, z5.b[0]\n"
- "sdot z24.s, z8.b, z6.b[0]\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
- "sdot z17.s, z9.b, z4.b[0]\n"
- "sdot z21.s, z9.b, z5.b[0]\n"
- "sdot z25.s, z9.b, z6.b[0]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "sdot z18.s, z10.b, z4.b[0]\n"
- "sdot z22.s, z10.b, z5.b[0]\n"
- "sdot z26.s, z10.b, z6.b[0]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "sdot z19.s, z11.b, z4.b[0]\n"
- "sdot z23.s, z11.b, z5.b[0]\n"
- "sdot z27.s, z11.b, z6.b[0]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "sdot z16.s, z12.b, z4.b[1]\n"
- "sdot z20.s, z12.b, z5.b[1]\n"
- "sdot z24.s, z12.b, z6.b[1]\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "sdot z17.s, z13.b, z4.b[1]\n"
- "sdot z21.s, z13.b, z5.b[1]\n"
- "sdot z25.s, z13.b, z6.b[1]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "sdot z18.s, z14.b, z4.b[1]\n"
- "sdot z22.s, z14.b, z5.b[1]\n"
- "sdot z26.s, z14.b, z6.b[1]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "sdot z19.s, z15.b, z4.b[1]\n"
- "sdot z23.s, z15.b, z5.b[1]\n"
- "sdot z27.s, z15.b, z6.b[1]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "sdot z16.s, z8.b, z4.b[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "sdot z20.s, z8.b, z5.b[2]\n"
- "sdot z24.s, z8.b, z6.b[2]\n"
- "sdot z17.s, z9.b, z4.b[2]\n"
- "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "sdot z21.s, z9.b, z5.b[2]\n"
- "sdot z25.s, z9.b, z6.b[2]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "sdot z18.s, z10.b, z4.b[2]\n"
- "sdot z22.s, z10.b, z5.b[2]\n"
- "sdot z26.s, z10.b, z6.b[2]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "sdot z19.s, z11.b, z4.b[2]\n"
- "sdot z23.s, z11.b, z5.b[2]\n"
- "sdot z27.s, z11.b, z6.b[2]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "sdot z16.s, z12.b, z4.b[3]\n"
- "sdot z20.s, z12.b, z5.b[3]\n"
- "sdot z24.s, z12.b, z6.b[3]\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "sdot z17.s, z13.b, z4.b[3]\n"
- "sdot z21.s, z13.b, z5.b[3]\n"
- "sdot z25.s, z13.b, z6.b[3]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "sdot z18.s, z14.b, z4.b[3]\n"
- "sdot z22.s, z14.b, z5.b[3]\n"
- "sdot z26.s, z14.b, z6.b[3]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "sdot z19.s, z15.b, z4.b[3]\n"
- "sdot z23.s, z15.b, z5.b[3]\n"
- "sdot z27.s, z15.b, z6.b[3]\n"
- "b.ne 3b\n"
- "2:\n"
- "cbz %[regs], 4f\n"
- "sdot z16.s, z8.b, z0.b[0]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "sdot z20.s, z8.b, z1.b[0]\n"
- "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n"
- "sdot z24.s, z8.b, z2.b[0]\n"
- "ld1rqb z5.b, p7/z, [a_ptr1]\n"
- "sdot z17.s, z9.b, z0.b[0]\n"
- "ld1rqb z6.b, p7/z, [a_ptr2]\n"
- "sdot z21.s, z9.b, z1.b[0]\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
- "sdot z25.s, z9.b, z2.b[0]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "sdot z18.s, z10.b, z0.b[0]\n"
- "sdot z22.s, z10.b, z1.b[0]\n"
- "sdot z26.s, z10.b, z2.b[0]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "sdot z19.s, z11.b, z0.b[0]\n"
- "sdot z23.s, z11.b, z1.b[0]\n"
- "sdot z27.s, z11.b, z2.b[0]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "sdot z16.s, z12.b, z0.b[1]\n"
- "sdot z20.s, z12.b, z1.b[1]\n"
- "sdot z24.s, z12.b, z2.b[1]\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "sdot z17.s, z13.b, z0.b[1]\n"
- "sdot z21.s, z13.b, z1.b[1]\n"
- "sdot z25.s, z13.b, z2.b[1]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "sdot z18.s, z14.b, z0.b[1]\n"
- "sdot z22.s, z14.b, z1.b[1]\n"
- "sdot z26.s, z14.b, z2.b[1]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "sdot z19.s, z15.b, z0.b[1]\n"
- "sdot z23.s, z15.b, z1.b[1]\n"
- "sdot z27.s, z15.b, z2.b[1]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "sdot z16.s, z8.b, z0.b[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "sdot z20.s, z8.b, z1.b[2]\n"
- "sdot z24.s, z8.b, z2.b[2]\n"
- "sdot z17.s, z9.b, z0.b[2]\n"
- "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "sdot z21.s, z9.b, z1.b[2]\n"
- "sdot z25.s, z9.b, z2.b[2]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "sdot z18.s, z10.b, z0.b[2]\n"
- "sdot z22.s, z10.b, z1.b[2]\n"
- "sdot z26.s, z10.b, z2.b[2]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "sdot z19.s, z11.b, z0.b[2]\n"
- "sdot z23.s, z11.b, z1.b[2]\n"
- "sdot z27.s, z11.b, z2.b[2]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "sdot z16.s, z12.b, z0.b[3]\n"
- "sdot z20.s, z12.b, z1.b[3]\n"
- "sdot z24.s, z12.b, z2.b[3]\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "sdot z17.s, z13.b, z0.b[3]\n"
- "sdot z21.s, z13.b, z1.b[3]\n"
- "sdot z25.s, z13.b, z2.b[3]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "sdot z18.s, z14.b, z0.b[3]\n"
- "sdot z22.s, z14.b, z1.b[3]\n"
- "sdot z26.s, z14.b, z2.b[3]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "sdot z19.s, z15.b, z0.b[3]\n"
- "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x10]\n"
- "sdot z23.s, z15.b, z1.b[3]\n"
- "ld1rqb z1.b, p6/z, [a_ptr1, #0x10]\n"
- "sdot z27.s, z15.b, z2.b[3]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "sdot z16.s, z8.b, z4.b[0]\n"
- "ld1rqb z2.b, p6/z, [a_ptr2, #0x10]\n"
- "sdot z20.s, z8.b, z5.b[0]\n"
- "addvl %[a_ptr0], %[a_ptr0], #2\n"
- "sdot z24.s, z8.b, z6.b[0]\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
- "sdot z17.s, z9.b, z4.b[0]\n"
- "addvl a_ptr1, a_ptr1, #2\n"
- "sdot z21.s, z9.b, z5.b[0]\n"
- "addvl a_ptr2, a_ptr2, #2\n"
- "sdot z25.s, z9.b, z6.b[0]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "sdot z18.s, z10.b, z4.b[0]\n"
- "sdot z22.s, z10.b, z5.b[0]\n"
- "sdot z26.s, z10.b, z6.b[0]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "sdot z19.s, z11.b, z4.b[0]\n"
- "sdot z23.s, z11.b, z5.b[0]\n"
- "sdot z27.s, z11.b, z6.b[0]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "sdot z16.s, z12.b, z4.b[1]\n"
- "sdot z20.s, z12.b, z5.b[1]\n"
- "sdot z24.s, z12.b, z6.b[1]\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "sdot z17.s, z13.b, z4.b[1]\n"
- "sdot z21.s, z13.b, z5.b[1]\n"
- "sdot z25.s, z13.b, z6.b[1]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "sdot z18.s, z14.b, z4.b[1]\n"
- "sdot z22.s, z14.b, z5.b[1]\n"
- "sdot z26.s, z14.b, z6.b[1]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "sdot z19.s, z15.b, z4.b[1]\n"
- "sdot z23.s, z15.b, z5.b[1]\n"
- "sdot z27.s, z15.b, z6.b[1]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "sdot z16.s, z8.b, z4.b[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "sdot z20.s, z8.b, z5.b[2]\n"
- "sdot z24.s, z8.b, z6.b[2]\n"
- "sdot z17.s, z9.b, z4.b[2]\n"
- "sdot z21.s, z9.b, z5.b[2]\n"
- "sdot z25.s, z9.b, z6.b[2]\n"
- "sdot z18.s, z10.b, z4.b[2]\n"
- "sdot z22.s, z10.b, z5.b[2]\n"
- "sdot z26.s, z10.b, z6.b[2]\n"
- "sdot z19.s, z11.b, z4.b[2]\n"
- "sdot z23.s, z11.b, z5.b[2]\n"
- "sdot z27.s, z11.b, z6.b[2]\n"
- "sdot z16.s, z12.b, z4.b[3]\n"
- "sdot z20.s, z12.b, z5.b[3]\n"
- "sdot z24.s, z12.b, z6.b[3]\n"
- "sdot z17.s, z13.b, z4.b[3]\n"
- "sdot z21.s, z13.b, z5.b[3]\n"
- "sdot z25.s, z13.b, z6.b[3]\n"
- "sdot z18.s, z14.b, z4.b[3]\n"
- "sdot z22.s, z14.b, z5.b[3]\n"
- "sdot z26.s, z14.b, z6.b[3]\n"
- "sdot z19.s, z15.b, z4.b[3]\n"
- "sdot z23.s, z15.b, z5.b[3]\n"
- "sdot z27.s, z15.b, z6.b[3]\n"
- "cbz %[blocks], 5f\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "sdot z16.s, z8.b, z0.b[0]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "sdot z20.s, z8.b, z1.b[0]\n"
- "sdot z24.s, z8.b, z2.b[0]\n"
- "sdot z17.s, z9.b, z0.b[0]\n"
- "sdot z21.s, z9.b, z1.b[0]\n"
- "sdot z25.s, z9.b, z2.b[0]\n"
- "sdot z18.s, z10.b, z0.b[0]\n"
- "sdot z22.s, z10.b, z1.b[0]\n"
- "sdot z26.s, z10.b, z2.b[0]\n"
- "sdot z19.s, z11.b, z0.b[0]\n"
- "sdot z23.s, z11.b, z1.b[0]\n"
- "sdot z27.s, z11.b, z2.b[0]\n"
- "b.eq 5f\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "sdot z16.s, z12.b, z0.b[1]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "sdot z20.s, z12.b, z1.b[1]\n"
- "sdot z24.s, z12.b, z2.b[1]\n"
- "sdot z17.s, z13.b, z0.b[1]\n"
- "sdot z21.s, z13.b, z1.b[1]\n"
- "sdot z25.s, z13.b, z2.b[1]\n"
- "sdot z18.s, z14.b, z0.b[1]\n"
- "sdot z22.s, z14.b, z1.b[1]\n"
- "sdot z26.s, z14.b, z2.b[1]\n"
- "sdot z19.s, z15.b, z0.b[1]\n"
- "sdot z23.s, z15.b, z1.b[1]\n"
- "sdot z27.s, z15.b, z2.b[1]\n"
- "b.eq 5f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "sdot z16.s, z8.b, z0.b[2]\n"
- "sdot z20.s, z8.b, z1.b[2]\n"
- "sdot z24.s, z8.b, z2.b[2]\n"
- "sdot z17.s, z9.b, z0.b[2]\n"
- "sdot z21.s, z9.b, z1.b[2]\n"
- "sdot z25.s, z9.b, z2.b[2]\n"
- "sdot z18.s, z10.b, z0.b[2]\n"
- "sdot z22.s, z10.b, z1.b[2]\n"
- "sdot z26.s, z10.b, z2.b[2]\n"
- "sdot z19.s, z11.b, z0.b[2]\n"
- "sdot z23.s, z11.b, z1.b[2]\n"
- "sdot z27.s, z11.b, z2.b[2]\n"
- "b.eq 5f\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "sdot z16.s, z12.b, z0.b[3]\n"
- "sdot z20.s, z12.b, z1.b[3]\n"
- "sdot z24.s, z12.b, z2.b[3]\n"
- "sdot z17.s, z13.b, z0.b[3]\n"
- "sdot z21.s, z13.b, z1.b[3]\n"
- "sdot z25.s, z13.b, z2.b[3]\n"
- "sdot z18.s, z14.b, z0.b[3]\n"
- "sdot z22.s, z14.b, z1.b[3]\n"
- "sdot z26.s, z14.b, z2.b[3]\n"
- "sdot z19.s, z15.b, z0.b[3]\n"
- "sdot z23.s, z15.b, z1.b[3]\n"
- "sdot z27.s, z15.b, z2.b[3]\n"
- "b 5f\n"
- "4:\n"
- "sdot z16.s, z8.b, z0.b[0]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "sdot z20.s, z8.b, z1.b[0]\n"
- "ld1rqb z4.b, p6/z, [%[a_ptr0]]\n"
- "sdot z24.s, z8.b, z2.b[0]\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
- "sdot z17.s, z9.b, z0.b[0]\n"
- "ld1rqb z5.b, p6/z, [a_ptr1]\n"
- "sdot z21.s, z9.b, z1.b[0]\n"
- "ld1rqb z6.b, p6/z, [a_ptr2]\n"
- "sdot z25.s, z9.b, z2.b[0]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "sdot z18.s, z10.b, z0.b[0]\n"
- "addvl %[a_ptr0], %[a_ptr0], #1\n"
- "sdot z22.s, z10.b, z1.b[0]\n"
- "addvl a_ptr1, a_ptr1, #1\n"
- "sdot z26.s, z10.b, z2.b[0]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "sdot z19.s, z11.b, z0.b[0]\n"
- "addvl a_ptr2, a_ptr2, #1\n"
- "sdot z23.s, z11.b, z1.b[0]\n"
- "sdot z27.s, z11.b, z2.b[0]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "sdot z16.s, z12.b, z0.b[1]\n"
- "sdot z20.s, z12.b, z1.b[1]\n"
- "sdot z24.s, z12.b, z2.b[1]\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "sdot z17.s, z13.b, z0.b[1]\n"
- "sdot z21.s, z13.b, z1.b[1]\n"
- "sdot z25.s, z13.b, z2.b[1]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "sdot z18.s, z14.b, z0.b[1]\n"
- "sdot z22.s, z14.b, z1.b[1]\n"
- "sdot z26.s, z14.b, z2.b[1]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "sdot z19.s, z15.b, z0.b[1]\n"
- "sdot z23.s, z15.b, z1.b[1]\n"
- "sdot z27.s, z15.b, z2.b[1]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "sdot z16.s, z8.b, z0.b[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "sdot z20.s, z8.b, z1.b[2]\n"
- "sdot z24.s, z8.b, z2.b[2]\n"
- "sdot z17.s, z9.b, z0.b[2]\n"
- "sdot z21.s, z9.b, z1.b[2]\n"
- "sdot z25.s, z9.b, z2.b[2]\n"
- "sdot z18.s, z10.b, z0.b[2]\n"
- "sdot z22.s, z10.b, z1.b[2]\n"
- "sdot z26.s, z10.b, z2.b[2]\n"
- "sdot z19.s, z11.b, z0.b[2]\n"
- "sdot z23.s, z11.b, z1.b[2]\n"
- "sdot z27.s, z11.b, z2.b[2]\n"
- "sdot z16.s, z12.b, z0.b[3]\n"
- "sdot z20.s, z12.b, z1.b[3]\n"
- "sdot z24.s, z12.b, z2.b[3]\n"
- "sdot z17.s, z13.b, z0.b[3]\n"
- "sdot z21.s, z13.b, z1.b[3]\n"
- "sdot z25.s, z13.b, z2.b[3]\n"
- "sdot z18.s, z14.b, z0.b[3]\n"
- "sdot z22.s, z14.b, z1.b[3]\n"
- "sdot z26.s, z14.b, z2.b[3]\n"
- "sdot z19.s, z15.b, z0.b[3]\n"
- "sdot z23.s, z15.b, z1.b[3]\n"
- "sdot z27.s, z15.b, z2.b[3]\n"
- "cbz %[blocks], 5f\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "sdot z16.s, z8.b, z4.b[0]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "sdot z20.s, z8.b, z5.b[0]\n"
- "sdot z24.s, z8.b, z6.b[0]\n"
- "sdot z17.s, z9.b, z4.b[0]\n"
- "sdot z21.s, z9.b, z5.b[0]\n"
- "sdot z25.s, z9.b, z6.b[0]\n"
- "sdot z18.s, z10.b, z4.b[0]\n"
- "sdot z22.s, z10.b, z5.b[0]\n"
- "sdot z26.s, z10.b, z6.b[0]\n"
- "sdot z19.s, z11.b, z4.b[0]\n"
- "sdot z23.s, z11.b, z5.b[0]\n"
- "sdot z27.s, z11.b, z6.b[0]\n"
- "b.eq 5f\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "sdot z16.s, z12.b, z4.b[1]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "sdot z20.s, z12.b, z5.b[1]\n"
- "sdot z24.s, z12.b, z6.b[1]\n"
- "sdot z17.s, z13.b, z4.b[1]\n"
- "sdot z21.s, z13.b, z5.b[1]\n"
- "sdot z25.s, z13.b, z6.b[1]\n"
- "sdot z18.s, z14.b, z4.b[1]\n"
- "sdot z22.s, z14.b, z5.b[1]\n"
- "sdot z26.s, z14.b, z6.b[1]\n"
- "sdot z19.s, z15.b, z4.b[1]\n"
- "sdot z23.s, z15.b, z5.b[1]\n"
- "sdot z27.s, z15.b, z6.b[1]\n"
- "b.eq 5f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "sdot z16.s, z8.b, z4.b[2]\n"
- "sdot z20.s, z8.b, z5.b[2]\n"
- "sdot z24.s, z8.b, z6.b[2]\n"
- "sdot z17.s, z9.b, z4.b[2]\n"
- "sdot z21.s, z9.b, z5.b[2]\n"
- "sdot z25.s, z9.b, z6.b[2]\n"
- "sdot z18.s, z10.b, z4.b[2]\n"
- "sdot z22.s, z10.b, z5.b[2]\n"
- "sdot z26.s, z10.b, z6.b[2]\n"
- "sdot z19.s, z11.b, z4.b[2]\n"
- "sdot z23.s, z11.b, z5.b[2]\n"
- "sdot z27.s, z11.b, z6.b[2]\n"
- "b.eq 5f\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "sdot z16.s, z12.b, z4.b[3]\n"
- "sdot z20.s, z12.b, z5.b[3]\n"
- "sdot z24.s, z12.b, z6.b[3]\n"
- "sdot z17.s, z13.b, z4.b[3]\n"
- "sdot z21.s, z13.b, z5.b[3]\n"
- "sdot z25.s, z13.b, z6.b[3]\n"
- "sdot z18.s, z14.b, z4.b[3]\n"
- "sdot z22.s, z14.b, z5.b[3]\n"
- "sdot z26.s, z14.b, z6.b[3]\n"
- "sdot z19.s, z15.b, z4.b[3]\n"
- "sdot z23.s, z15.b, z5.b[3]\n"
- "sdot z27.s, z15.b, z6.b[3]\n"
- "5:\n"
- "st1w z16.s, p0, [%[c_ptr0]]\n"
- "st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n"
- "st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n"
- "st1w z19.s, p3, [%[c_ptr0], #3, MUL VL]\n"
- "addvl %[c_ptr0], %[c_ptr0], #4\n"
- "st1w z20.s, p0, [c_ptr1]\n"
- "st1w z21.s, p1, [c_ptr1, #1, MUL VL]\n"
- "st1w z22.s, p2, [c_ptr1, #2, MUL VL]\n"
- "st1w z23.s, p3, [c_ptr1, #3, MUL VL]\n"
- "st1w z24.s, p0, [c_ptr2]\n"
- "st1w z25.s, p1, [c_ptr2, #1, MUL VL]\n"
- "st1w z26.s, p2, [c_ptr2, #2, MUL VL]\n"
- "st1w z27.s, p3, [c_ptr2, #3, MUL VL]\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks)
- : [width] "r" (width), [accumulate] "r" (static_cast<uint64_t>(accumulate)), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers)
- : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "cc", "memory"
- );
- break;
- default:
- case 4:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "a_ptr3 .req X2\n"
- "c_ptr1 .req X3\n"
- "c_ptr2 .req X4\n"
- "c_ptr3 .req X5\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "add a_ptr3, a_ptr2, %[lda]\n"
- "add c_ptr3, c_ptr2, %[ldc]\n"
- "whilelt p6.b, %[temp], %[leftovers]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "ptrue p7.b\n"
- "whilelt p1.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "whilelt p2.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "whilelt p3.s, %[temp], %[width]\n"
- "cbnz %[accumulate], 1f\n"
- "mov z16.s, #0\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "mov z17.s, #0\n"
- "ld1rqb z1.b, p7/z, [a_ptr1]\n"
- "mov z18.s, #0\n"
- "ld1rqb z2.b, p7/z, [a_ptr2]\n"
- "mov z19.s, #0\n"
- "ld1rqb z3.b, p7/z, [a_ptr3]\n"
- "mov z20.s, #0\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
- "mov z21.s, #0\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "mov z22.s, #0\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "mov z23.s, #0\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "mov z24.s, #0\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "mov z25.s, #0\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "mov z26.s, #0\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "mov z27.s, #0\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "mov z28.s, #0\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "mov z29.s, #0\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "mov z30.s, #0\n"
- "add a_ptr3, a_ptr3, #0x10\n"
- "mov z31.s, #0\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "ld1w z16.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n"
- "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n"
- "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n"
- "ld1w z20.s, p0/z, [c_ptr1]\n"
- "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n"
- "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n"
- "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n"
- "ld1w z24.s, p0/z, [c_ptr2]\n"
- "ld1w z25.s, p1/z, [c_ptr2, #1, MUL VL]\n"
- "ld1w z26.s, p2/z, [c_ptr2, #2, MUL VL]\n"
- "ld1w z27.s, p3/z, [c_ptr2, #3, MUL VL]\n"
- "ld1w z28.s, p0/z, [c_ptr3]\n"
- "ld1w z29.s, p1/z, [c_ptr3, #1, MUL VL]\n"
- "ld1w z30.s, p2/z, [c_ptr3, #2, MUL VL]\n"
- "ld1w z31.s, p3/z, [c_ptr3, #3, MUL VL]\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ld1rqb z1.b, p7/z, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "ld1rqb z2.b, p7/z, [a_ptr2]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "ld1rqb z3.b, p7/z, [a_ptr3]\n"
- "add a_ptr3, a_ptr3, #0x10\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "cbz %[loops], 2f\n"
- "3:\n"
- "sdot z16.s, z8.b, z0.b[0]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "sdot z20.s, z8.b, z1.b[0]\n"
- "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n"
- "sdot z24.s, z8.b, z2.b[0]\n"
- "ld1rqb z5.b, p7/z, [a_ptr1]\n"
- "sdot z28.s, z8.b, z3.b[0]\n"
- "ld1rqb z6.b, p7/z, [a_ptr2]\n"
- "sdot z17.s, z9.b, z0.b[0]\n"
- "ld1rqb z7.b, p7/z, [a_ptr3]\n"
- "sdot z21.s, z9.b, z1.b[0]\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
- "sdot z25.s, z9.b, z2.b[0]\n"
- "subs %[loops], %[loops], #0x1\n"
- "sdot z29.s, z9.b, z3.b[0]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "sdot z18.s, z10.b, z0.b[0]\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
- "sdot z22.s, z10.b, z1.b[0]\n"
- "add a_ptr1, a_ptr1, #0x20\n"
- "sdot z26.s, z10.b, z2.b[0]\n"
- "add a_ptr2, a_ptr2, #0x20\n"
- "sdot z30.s, z10.b, z3.b[0]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "sdot z19.s, z11.b, z0.b[0]\n"
- "add a_ptr3, a_ptr3, #0x20\n"
- "sdot z23.s, z11.b, z1.b[0]\n"
- "sdot z27.s, z11.b, z2.b[0]\n"
- "sdot z31.s, z11.b, z3.b[0]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "sdot z16.s, z12.b, z0.b[1]\n"
- "sdot z20.s, z12.b, z1.b[1]\n"
- "sdot z24.s, z12.b, z2.b[1]\n"
- "sdot z28.s, z12.b, z3.b[1]\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "sdot z17.s, z13.b, z0.b[1]\n"
- "sdot z21.s, z13.b, z1.b[1]\n"
- "sdot z25.s, z13.b, z2.b[1]\n"
- "sdot z29.s, z13.b, z3.b[1]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "sdot z18.s, z14.b, z0.b[1]\n"
- "sdot z22.s, z14.b, z1.b[1]\n"
- "sdot z26.s, z14.b, z2.b[1]\n"
- "sdot z30.s, z14.b, z3.b[1]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "sdot z19.s, z15.b, z0.b[1]\n"
- "sdot z23.s, z15.b, z1.b[1]\n"
- "sdot z27.s, z15.b, z2.b[1]\n"
- "sdot z31.s, z15.b, z3.b[1]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "sdot z16.s, z8.b, z0.b[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "sdot z20.s, z8.b, z1.b[2]\n"
- "sdot z24.s, z8.b, z2.b[2]\n"
- "sdot z28.s, z8.b, z3.b[2]\n"
- "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "sdot z17.s, z9.b, z0.b[2]\n"
- "sdot z21.s, z9.b, z1.b[2]\n"
- "sdot z25.s, z9.b, z2.b[2]\n"
- "sdot z29.s, z9.b, z3.b[2]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "sdot z18.s, z10.b, z0.b[2]\n"
- "sdot z22.s, z10.b, z1.b[2]\n"
- "sdot z26.s, z10.b, z2.b[2]\n"
- "sdot z30.s, z10.b, z3.b[2]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "sdot z19.s, z11.b, z0.b[2]\n"
- "sdot z23.s, z11.b, z1.b[2]\n"
- "sdot z27.s, z11.b, z2.b[2]\n"
- "sdot z31.s, z11.b, z3.b[2]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "sdot z16.s, z12.b, z0.b[3]\n"
- "sdot z20.s, z12.b, z1.b[3]\n"
- "sdot z24.s, z12.b, z2.b[3]\n"
- "sdot z28.s, z12.b, z3.b[3]\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "sdot z17.s, z13.b, z0.b[3]\n"
- "sdot z21.s, z13.b, z1.b[3]\n"
- "sdot z25.s, z13.b, z2.b[3]\n"
- "sdot z29.s, z13.b, z3.b[3]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "sdot z18.s, z14.b, z0.b[3]\n"
- "sdot z22.s, z14.b, z1.b[3]\n"
- "sdot z26.s, z14.b, z2.b[3]\n"
- "sdot z30.s, z14.b, z3.b[3]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "sdot z19.s, z15.b, z0.b[3]\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0], #-0x10]\n"
- "sdot z23.s, z15.b, z1.b[3]\n"
- "ld1rqb z1.b, p7/z, [a_ptr1, #-0x10]\n"
- "sdot z27.s, z15.b, z2.b[3]\n"
- "ld1rqb z2.b, p7/z, [a_ptr2, #-0x10]\n"
- "sdot z31.s, z15.b, z3.b[3]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "sdot z16.s, z8.b, z4.b[0]\n"
- "ld1rqb z3.b, p7/z, [a_ptr3, #-0x10]\n"
- "sdot z20.s, z8.b, z5.b[0]\n"
- "sdot z24.s, z8.b, z6.b[0]\n"
- "sdot z28.s, z8.b, z7.b[0]\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
- "sdot z17.s, z9.b, z4.b[0]\n"
- "sdot z21.s, z9.b, z5.b[0]\n"
- "sdot z25.s, z9.b, z6.b[0]\n"
- "sdot z29.s, z9.b, z7.b[0]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "sdot z18.s, z10.b, z4.b[0]\n"
- "sdot z22.s, z10.b, z5.b[0]\n"
- "sdot z26.s, z10.b, z6.b[0]\n"
- "sdot z30.s, z10.b, z7.b[0]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "sdot z19.s, z11.b, z4.b[0]\n"
- "sdot z23.s, z11.b, z5.b[0]\n"
- "sdot z27.s, z11.b, z6.b[0]\n"
- "sdot z31.s, z11.b, z7.b[0]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "sdot z16.s, z12.b, z4.b[1]\n"
- "sdot z20.s, z12.b, z5.b[1]\n"
- "sdot z24.s, z12.b, z6.b[1]\n"
- "sdot z28.s, z12.b, z7.b[1]\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "sdot z17.s, z13.b, z4.b[1]\n"
- "sdot z21.s, z13.b, z5.b[1]\n"
- "sdot z25.s, z13.b, z6.b[1]\n"
- "sdot z29.s, z13.b, z7.b[1]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "sdot z18.s, z14.b, z4.b[1]\n"
- "sdot z22.s, z14.b, z5.b[1]\n"
- "sdot z26.s, z14.b, z6.b[1]\n"
- "sdot z30.s, z14.b, z7.b[1]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "sdot z19.s, z15.b, z4.b[1]\n"
- "sdot z23.s, z15.b, z5.b[1]\n"
- "sdot z27.s, z15.b, z6.b[1]\n"
- "sdot z31.s, z15.b, z7.b[1]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "sdot z16.s, z8.b, z4.b[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "sdot z20.s, z8.b, z5.b[2]\n"
- "sdot z24.s, z8.b, z6.b[2]\n"
- "sdot z28.s, z8.b, z7.b[2]\n"
- "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "sdot z17.s, z9.b, z4.b[2]\n"
- "sdot z21.s, z9.b, z5.b[2]\n"
- "sdot z25.s, z9.b, z6.b[2]\n"
- "sdot z29.s, z9.b, z7.b[2]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "sdot z18.s, z10.b, z4.b[2]\n"
- "sdot z22.s, z10.b, z5.b[2]\n"
- "sdot z26.s, z10.b, z6.b[2]\n"
- "sdot z30.s, z10.b, z7.b[2]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "sdot z19.s, z11.b, z4.b[2]\n"
- "sdot z23.s, z11.b, z5.b[2]\n"
- "sdot z27.s, z11.b, z6.b[2]\n"
- "sdot z31.s, z11.b, z7.b[2]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "sdot z16.s, z12.b, z4.b[3]\n"
- "sdot z20.s, z12.b, z5.b[3]\n"
- "sdot z24.s, z12.b, z6.b[3]\n"
- "sdot z28.s, z12.b, z7.b[3]\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "sdot z17.s, z13.b, z4.b[3]\n"
- "sdot z21.s, z13.b, z5.b[3]\n"
- "sdot z25.s, z13.b, z6.b[3]\n"
- "sdot z29.s, z13.b, z7.b[3]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "sdot z18.s, z14.b, z4.b[3]\n"
- "sdot z22.s, z14.b, z5.b[3]\n"
- "sdot z26.s, z14.b, z6.b[3]\n"
- "sdot z30.s, z14.b, z7.b[3]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "sdot z19.s, z15.b, z4.b[3]\n"
- "sdot z23.s, z15.b, z5.b[3]\n"
- "sdot z27.s, z15.b, z6.b[3]\n"
- "sdot z31.s, z15.b, z7.b[3]\n"
- "b.ne 3b\n"
- "2:\n"
- "cbz %[regs], 4f\n"
- "sdot z16.s, z8.b, z0.b[0]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "sdot z20.s, z8.b, z1.b[0]\n"
- "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n"
- "sdot z24.s, z8.b, z2.b[0]\n"
- "ld1rqb z5.b, p7/z, [a_ptr1]\n"
- "sdot z28.s, z8.b, z3.b[0]\n"
- "ld1rqb z6.b, p7/z, [a_ptr2]\n"
- "sdot z17.s, z9.b, z0.b[0]\n"
- "ld1rqb z7.b, p7/z, [a_ptr3]\n"
- "sdot z21.s, z9.b, z1.b[0]\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
- "sdot z25.s, z9.b, z2.b[0]\n"
- "sdot z29.s, z9.b, z3.b[0]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "sdot z18.s, z10.b, z0.b[0]\n"
- "sdot z22.s, z10.b, z1.b[0]\n"
- "sdot z26.s, z10.b, z2.b[0]\n"
- "sdot z30.s, z10.b, z3.b[0]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "sdot z19.s, z11.b, z0.b[0]\n"
- "sdot z23.s, z11.b, z1.b[0]\n"
- "sdot z27.s, z11.b, z2.b[0]\n"
- "sdot z31.s, z11.b, z3.b[0]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "sdot z16.s, z12.b, z0.b[1]\n"
- "sdot z20.s, z12.b, z1.b[1]\n"
- "sdot z24.s, z12.b, z2.b[1]\n"
- "sdot z28.s, z12.b, z3.b[1]\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "sdot z17.s, z13.b, z0.b[1]\n"
- "sdot z21.s, z13.b, z1.b[1]\n"
- "sdot z25.s, z13.b, z2.b[1]\n"
- "sdot z29.s, z13.b, z3.b[1]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "sdot z18.s, z14.b, z0.b[1]\n"
- "sdot z22.s, z14.b, z1.b[1]\n"
- "sdot z26.s, z14.b, z2.b[1]\n"
- "sdot z30.s, z14.b, z3.b[1]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "sdot z19.s, z15.b, z0.b[1]\n"
- "sdot z23.s, z15.b, z1.b[1]\n"
- "sdot z27.s, z15.b, z2.b[1]\n"
- "sdot z31.s, z15.b, z3.b[1]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "sdot z16.s, z8.b, z0.b[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "sdot z20.s, z8.b, z1.b[2]\n"
- "sdot z24.s, z8.b, z2.b[2]\n"
- "sdot z28.s, z8.b, z3.b[2]\n"
- "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "sdot z17.s, z9.b, z0.b[2]\n"
- "sdot z21.s, z9.b, z1.b[2]\n"
- "sdot z25.s, z9.b, z2.b[2]\n"
- "sdot z29.s, z9.b, z3.b[2]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "sdot z18.s, z10.b, z0.b[2]\n"
- "sdot z22.s, z10.b, z1.b[2]\n"
- "sdot z26.s, z10.b, z2.b[2]\n"
- "sdot z30.s, z10.b, z3.b[2]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "sdot z19.s, z11.b, z0.b[2]\n"
- "sdot z23.s, z11.b, z1.b[2]\n"
- "sdot z27.s, z11.b, z2.b[2]\n"
- "sdot z31.s, z11.b, z3.b[2]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "sdot z16.s, z12.b, z0.b[3]\n"
- "sdot z20.s, z12.b, z1.b[3]\n"
- "sdot z24.s, z12.b, z2.b[3]\n"
- "sdot z28.s, z12.b, z3.b[3]\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "sdot z17.s, z13.b, z0.b[3]\n"
- "sdot z21.s, z13.b, z1.b[3]\n"
- "sdot z25.s, z13.b, z2.b[3]\n"
- "sdot z29.s, z13.b, z3.b[3]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "sdot z18.s, z14.b, z0.b[3]\n"
- "sdot z22.s, z14.b, z1.b[3]\n"
- "sdot z26.s, z14.b, z2.b[3]\n"
- "sdot z30.s, z14.b, z3.b[3]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "sdot z19.s, z15.b, z0.b[3]\n"
- "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x10]\n"
- "sdot z23.s, z15.b, z1.b[3]\n"
- "ld1rqb z1.b, p6/z, [a_ptr1, #0x10]\n"
- "sdot z27.s, z15.b, z2.b[3]\n"
- "ld1rqb z2.b, p6/z, [a_ptr2, #0x10]\n"
- "sdot z31.s, z15.b, z3.b[3]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "sdot z16.s, z8.b, z4.b[0]\n"
- "ld1rqb z3.b, p6/z, [a_ptr3, #0x10]\n"
- "sdot z20.s, z8.b, z5.b[0]\n"
- "addvl %[a_ptr0], %[a_ptr0], #2\n"
- "sdot z24.s, z8.b, z6.b[0]\n"
- "addvl a_ptr1, a_ptr1, #2\n"
- "sdot z28.s, z8.b, z7.b[0]\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
- "sdot z17.s, z9.b, z4.b[0]\n"
- "addvl a_ptr2, a_ptr2, #2\n"
- "sdot z21.s, z9.b, z5.b[0]\n"
- "addvl a_ptr3, a_ptr3, #2\n"
- "sdot z25.s, z9.b, z6.b[0]\n"
- "sdot z29.s, z9.b, z7.b[0]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "sdot z18.s, z10.b, z4.b[0]\n"
- "sdot z22.s, z10.b, z5.b[0]\n"
- "sdot z26.s, z10.b, z6.b[0]\n"
- "sdot z30.s, z10.b, z7.b[0]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "sdot z19.s, z11.b, z4.b[0]\n"
- "sdot z23.s, z11.b, z5.b[0]\n"
- "sdot z27.s, z11.b, z6.b[0]\n"
- "sdot z31.s, z11.b, z7.b[0]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "sdot z16.s, z12.b, z4.b[1]\n"
- "sdot z20.s, z12.b, z5.b[1]\n"
- "sdot z24.s, z12.b, z6.b[1]\n"
- "sdot z28.s, z12.b, z7.b[1]\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "sdot z17.s, z13.b, z4.b[1]\n"
- "sdot z21.s, z13.b, z5.b[1]\n"
- "sdot z25.s, z13.b, z6.b[1]\n"
- "sdot z29.s, z13.b, z7.b[1]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "sdot z18.s, z14.b, z4.b[1]\n"
- "sdot z22.s, z14.b, z5.b[1]\n"
- "sdot z26.s, z14.b, z6.b[1]\n"
- "sdot z30.s, z14.b, z7.b[1]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "sdot z19.s, z15.b, z4.b[1]\n"
- "sdot z23.s, z15.b, z5.b[1]\n"
- "sdot z27.s, z15.b, z6.b[1]\n"
- "sdot z31.s, z15.b, z7.b[1]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "sdot z16.s, z8.b, z4.b[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "sdot z20.s, z8.b, z5.b[2]\n"
- "sdot z24.s, z8.b, z6.b[2]\n"
- "sdot z28.s, z8.b, z7.b[2]\n"
- "sdot z17.s, z9.b, z4.b[2]\n"
- "sdot z21.s, z9.b, z5.b[2]\n"
- "sdot z25.s, z9.b, z6.b[2]\n"
- "sdot z29.s, z9.b, z7.b[2]\n"
- "sdot z18.s, z10.b, z4.b[2]\n"
- "sdot z22.s, z10.b, z5.b[2]\n"
- "sdot z26.s, z10.b, z6.b[2]\n"
- "sdot z30.s, z10.b, z7.b[2]\n"
- "sdot z19.s, z11.b, z4.b[2]\n"
- "sdot z23.s, z11.b, z5.b[2]\n"
- "sdot z27.s, z11.b, z6.b[2]\n"
- "sdot z31.s, z11.b, z7.b[2]\n"
- "sdot z16.s, z12.b, z4.b[3]\n"
- "sdot z20.s, z12.b, z5.b[3]\n"
- "sdot z24.s, z12.b, z6.b[3]\n"
- "sdot z28.s, z12.b, z7.b[3]\n"
- "sdot z17.s, z13.b, z4.b[3]\n"
- "sdot z21.s, z13.b, z5.b[3]\n"
- "sdot z25.s, z13.b, z6.b[3]\n"
- "sdot z29.s, z13.b, z7.b[3]\n"
- "sdot z18.s, z14.b, z4.b[3]\n"
- "sdot z22.s, z14.b, z5.b[3]\n"
- "sdot z26.s, z14.b, z6.b[3]\n"
- "sdot z30.s, z14.b, z7.b[3]\n"
- "sdot z19.s, z15.b, z4.b[3]\n"
- "sdot z23.s, z15.b, z5.b[3]\n"
- "sdot z27.s, z15.b, z6.b[3]\n"
- "sdot z31.s, z15.b, z7.b[3]\n"
- "cbz %[blocks], 5f\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "sdot z16.s, z8.b, z0.b[0]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "sdot z20.s, z8.b, z1.b[0]\n"
- "sdot z24.s, z8.b, z2.b[0]\n"
- "sdot z28.s, z8.b, z3.b[0]\n"
- "sdot z17.s, z9.b, z0.b[0]\n"
- "sdot z21.s, z9.b, z1.b[0]\n"
- "sdot z25.s, z9.b, z2.b[0]\n"
- "sdot z29.s, z9.b, z3.b[0]\n"
- "sdot z18.s, z10.b, z0.b[0]\n"
- "sdot z22.s, z10.b, z1.b[0]\n"
- "sdot z26.s, z10.b, z2.b[0]\n"
- "sdot z30.s, z10.b, z3.b[0]\n"
- "sdot z19.s, z11.b, z0.b[0]\n"
- "sdot z23.s, z11.b, z1.b[0]\n"
- "sdot z27.s, z11.b, z2.b[0]\n"
- "sdot z31.s, z11.b, z3.b[0]\n"
- "b.eq 5f\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "sdot z16.s, z12.b, z0.b[1]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "sdot z20.s, z12.b, z1.b[1]\n"
- "sdot z24.s, z12.b, z2.b[1]\n"
- "sdot z28.s, z12.b, z3.b[1]\n"
- "sdot z17.s, z13.b, z0.b[1]\n"
- "sdot z21.s, z13.b, z1.b[1]\n"
- "sdot z25.s, z13.b, z2.b[1]\n"
- "sdot z29.s, z13.b, z3.b[1]\n"
- "sdot z18.s, z14.b, z0.b[1]\n"
- "sdot z22.s, z14.b, z1.b[1]\n"
- "sdot z26.s, z14.b, z2.b[1]\n"
- "sdot z30.s, z14.b, z3.b[1]\n"
- "sdot z19.s, z15.b, z0.b[1]\n"
- "sdot z23.s, z15.b, z1.b[1]\n"
- "sdot z27.s, z15.b, z2.b[1]\n"
- "sdot z31.s, z15.b, z3.b[1]\n"
- "b.eq 5f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "sdot z16.s, z8.b, z0.b[2]\n"
- "sdot z20.s, z8.b, z1.b[2]\n"
- "sdot z24.s, z8.b, z2.b[2]\n"
- "sdot z28.s, z8.b, z3.b[2]\n"
- "sdot z17.s, z9.b, z0.b[2]\n"
- "sdot z21.s, z9.b, z1.b[2]\n"
- "sdot z25.s, z9.b, z2.b[2]\n"
- "sdot z29.s, z9.b, z3.b[2]\n"
- "sdot z18.s, z10.b, z0.b[2]\n"
- "sdot z22.s, z10.b, z1.b[2]\n"
- "sdot z26.s, z10.b, z2.b[2]\n"
- "sdot z30.s, z10.b, z3.b[2]\n"
- "sdot z19.s, z11.b, z0.b[2]\n"
- "sdot z23.s, z11.b, z1.b[2]\n"
- "sdot z27.s, z11.b, z2.b[2]\n"
- "sdot z31.s, z11.b, z3.b[2]\n"
- "b.eq 5f\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "sdot z16.s, z12.b, z0.b[3]\n"
- "sdot z20.s, z12.b, z1.b[3]\n"
- "sdot z24.s, z12.b, z2.b[3]\n"
- "sdot z28.s, z12.b, z3.b[3]\n"
- "sdot z17.s, z13.b, z0.b[3]\n"
- "sdot z21.s, z13.b, z1.b[3]\n"
- "sdot z25.s, z13.b, z2.b[3]\n"
- "sdot z29.s, z13.b, z3.b[3]\n"
- "sdot z18.s, z14.b, z0.b[3]\n"
- "sdot z22.s, z14.b, z1.b[3]\n"
- "sdot z26.s, z14.b, z2.b[3]\n"
- "sdot z30.s, z14.b, z3.b[3]\n"
- "sdot z19.s, z15.b, z0.b[3]\n"
- "sdot z23.s, z15.b, z1.b[3]\n"
- "sdot z27.s, z15.b, z2.b[3]\n"
- "sdot z31.s, z15.b, z3.b[3]\n"
- "b 5f\n"
- "4:\n"
- "sdot z16.s, z8.b, z0.b[0]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "sdot z20.s, z8.b, z1.b[0]\n"
- "ld1rqb z4.b, p6/z, [%[a_ptr0]]\n"
- "sdot z24.s, z8.b, z2.b[0]\n"
- "ld1rqb z5.b, p6/z, [a_ptr1]\n"
- "sdot z28.s, z8.b, z3.b[0]\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
- "sdot z17.s, z9.b, z0.b[0]\n"
- "ld1rqb z6.b, p6/z, [a_ptr2]\n"
- "sdot z21.s, z9.b, z1.b[0]\n"
- "ld1rqb z7.b, p6/z, [a_ptr3]\n"
- "sdot z25.s, z9.b, z2.b[0]\n"
- "addvl %[a_ptr0], %[a_ptr0], #1\n"
- "sdot z29.s, z9.b, z3.b[0]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "sdot z18.s, z10.b, z0.b[0]\n"
- "addvl a_ptr1, a_ptr1, #1\n"
- "sdot z22.s, z10.b, z1.b[0]\n"
- "addvl a_ptr2, a_ptr2, #1\n"
- "sdot z26.s, z10.b, z2.b[0]\n"
- "addvl a_ptr3, a_ptr3, #1\n"
- "sdot z30.s, z10.b, z3.b[0]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "sdot z19.s, z11.b, z0.b[0]\n"
- "sdot z23.s, z11.b, z1.b[0]\n"
- "sdot z27.s, z11.b, z2.b[0]\n"
- "sdot z31.s, z11.b, z3.b[0]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "sdot z16.s, z12.b, z0.b[1]\n"
- "sdot z20.s, z12.b, z1.b[1]\n"
- "sdot z24.s, z12.b, z2.b[1]\n"
- "sdot z28.s, z12.b, z3.b[1]\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "sdot z17.s, z13.b, z0.b[1]\n"
- "sdot z21.s, z13.b, z1.b[1]\n"
- "sdot z25.s, z13.b, z2.b[1]\n"
- "sdot z29.s, z13.b, z3.b[1]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "sdot z18.s, z14.b, z0.b[1]\n"
- "sdot z22.s, z14.b, z1.b[1]\n"
- "sdot z26.s, z14.b, z2.b[1]\n"
- "sdot z30.s, z14.b, z3.b[1]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "sdot z19.s, z15.b, z0.b[1]\n"
- "sdot z23.s, z15.b, z1.b[1]\n"
- "sdot z27.s, z15.b, z2.b[1]\n"
- "sdot z31.s, z15.b, z3.b[1]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "sdot z16.s, z8.b, z0.b[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "sdot z20.s, z8.b, z1.b[2]\n"
- "sdot z24.s, z8.b, z2.b[2]\n"
- "sdot z28.s, z8.b, z3.b[2]\n"
- "sdot z17.s, z9.b, z0.b[2]\n"
- "sdot z21.s, z9.b, z1.b[2]\n"
- "sdot z25.s, z9.b, z2.b[2]\n"
- "sdot z29.s, z9.b, z3.b[2]\n"
- "sdot z18.s, z10.b, z0.b[2]\n"
- "sdot z22.s, z10.b, z1.b[2]\n"
- "sdot z26.s, z10.b, z2.b[2]\n"
- "sdot z30.s, z10.b, z3.b[2]\n"
- "sdot z19.s, z11.b, z0.b[2]\n"
- "sdot z23.s, z11.b, z1.b[2]\n"
- "sdot z27.s, z11.b, z2.b[2]\n"
- "sdot z31.s, z11.b, z3.b[2]\n"
- "sdot z16.s, z12.b, z0.b[3]\n"
- "sdot z20.s, z12.b, z1.b[3]\n"
- "sdot z24.s, z12.b, z2.b[3]\n"
- "sdot z28.s, z12.b, z3.b[3]\n"
- "sdot z17.s, z13.b, z0.b[3]\n"
- "sdot z21.s, z13.b, z1.b[3]\n"
- "sdot z25.s, z13.b, z2.b[3]\n"
- "sdot z29.s, z13.b, z3.b[3]\n"
- "sdot z18.s, z14.b, z0.b[3]\n"
- "sdot z22.s, z14.b, z1.b[3]\n"
- "sdot z26.s, z14.b, z2.b[3]\n"
- "sdot z30.s, z14.b, z3.b[3]\n"
- "sdot z19.s, z15.b, z0.b[3]\n"
- "sdot z23.s, z15.b, z1.b[3]\n"
- "sdot z27.s, z15.b, z2.b[3]\n"
- "sdot z31.s, z15.b, z3.b[3]\n"
- "cbz %[blocks], 5f\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "sdot z16.s, z8.b, z4.b[0]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "sdot z20.s, z8.b, z5.b[0]\n"
- "sdot z24.s, z8.b, z6.b[0]\n"
- "sdot z28.s, z8.b, z7.b[0]\n"
- "sdot z17.s, z9.b, z4.b[0]\n"
- "sdot z21.s, z9.b, z5.b[0]\n"
- "sdot z25.s, z9.b, z6.b[0]\n"
- "sdot z29.s, z9.b, z7.b[0]\n"
- "sdot z18.s, z10.b, z4.b[0]\n"
- "sdot z22.s, z10.b, z5.b[0]\n"
- "sdot z26.s, z10.b, z6.b[0]\n"
- "sdot z30.s, z10.b, z7.b[0]\n"
- "sdot z19.s, z11.b, z4.b[0]\n"
- "sdot z23.s, z11.b, z5.b[0]\n"
- "sdot z27.s, z11.b, z6.b[0]\n"
- "sdot z31.s, z11.b, z7.b[0]\n"
- "b.eq 5f\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "sdot z16.s, z12.b, z4.b[1]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "sdot z20.s, z12.b, z5.b[1]\n"
- "sdot z24.s, z12.b, z6.b[1]\n"
- "sdot z28.s, z12.b, z7.b[1]\n"
- "sdot z17.s, z13.b, z4.b[1]\n"
- "sdot z21.s, z13.b, z5.b[1]\n"
- "sdot z25.s, z13.b, z6.b[1]\n"
- "sdot z29.s, z13.b, z7.b[1]\n"
- "sdot z18.s, z14.b, z4.b[1]\n"
- "sdot z22.s, z14.b, z5.b[1]\n"
- "sdot z26.s, z14.b, z6.b[1]\n"
- "sdot z30.s, z14.b, z7.b[1]\n"
- "sdot z19.s, z15.b, z4.b[1]\n"
- "sdot z23.s, z15.b, z5.b[1]\n"
- "sdot z27.s, z15.b, z6.b[1]\n"
- "sdot z31.s, z15.b, z7.b[1]\n"
- "b.eq 5f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "sdot z16.s, z8.b, z4.b[2]\n"
- "sdot z20.s, z8.b, z5.b[2]\n"
- "sdot z24.s, z8.b, z6.b[2]\n"
- "sdot z28.s, z8.b, z7.b[2]\n"
- "sdot z17.s, z9.b, z4.b[2]\n"
- "sdot z21.s, z9.b, z5.b[2]\n"
- "sdot z25.s, z9.b, z6.b[2]\n"
- "sdot z29.s, z9.b, z7.b[2]\n"
- "sdot z18.s, z10.b, z4.b[2]\n"
- "sdot z22.s, z10.b, z5.b[2]\n"
- "sdot z26.s, z10.b, z6.b[2]\n"
- "sdot z30.s, z10.b, z7.b[2]\n"
- "sdot z19.s, z11.b, z4.b[2]\n"
- "sdot z23.s, z11.b, z5.b[2]\n"
- "sdot z27.s, z11.b, z6.b[2]\n"
- "sdot z31.s, z11.b, z7.b[2]\n"
- "b.eq 5f\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "sdot z16.s, z12.b, z4.b[3]\n"
- "sdot z20.s, z12.b, z5.b[3]\n"
- "sdot z24.s, z12.b, z6.b[3]\n"
- "sdot z28.s, z12.b, z7.b[3]\n"
- "sdot z17.s, z13.b, z4.b[3]\n"
- "sdot z21.s, z13.b, z5.b[3]\n"
- "sdot z25.s, z13.b, z6.b[3]\n"
- "sdot z29.s, z13.b, z7.b[3]\n"
- "sdot z18.s, z14.b, z4.b[3]\n"
- "sdot z22.s, z14.b, z5.b[3]\n"
- "sdot z26.s, z14.b, z6.b[3]\n"
- "sdot z30.s, z14.b, z7.b[3]\n"
- "sdot z19.s, z15.b, z4.b[3]\n"
- "sdot z23.s, z15.b, z5.b[3]\n"
- "sdot z27.s, z15.b, z6.b[3]\n"
- "sdot z31.s, z15.b, z7.b[3]\n"
- "5:\n"
- "st1w z16.s, p0, [%[c_ptr0]]\n"
- "st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n"
- "st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n"
- "st1w z19.s, p3, [%[c_ptr0], #3, MUL VL]\n"
- "addvl %[c_ptr0], %[c_ptr0], #4\n"
- "st1w z20.s, p0, [c_ptr1]\n"
- "st1w z21.s, p1, [c_ptr1, #1, MUL VL]\n"
- "st1w z22.s, p2, [c_ptr1, #2, MUL VL]\n"
- "st1w z23.s, p3, [c_ptr1, #3, MUL VL]\n"
- "st1w z24.s, p0, [c_ptr2]\n"
- "st1w z25.s, p1, [c_ptr2, #1, MUL VL]\n"
- "st1w z26.s, p2, [c_ptr2, #2, MUL VL]\n"
- "st1w z27.s, p3, [c_ptr2, #3, MUL VL]\n"
- "st1w z28.s, p0, [c_ptr3]\n"
- "st1w z29.s, p1, [c_ptr3, #1, MUL VL]\n"
- "st1w z30.s, p2, [c_ptr3, #2, MUL VL]\n"
- "st1w z31.s, p3, [c_ptr3, #3, MUL VL]\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq a_ptr3\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- ".unreq c_ptr3\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks)
- : [width] "r" (width), [accumulate] "r" (static_cast<uint64_t>(accumulate)), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers)
- : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "x4", "x5", "cc", "memory"
- );
- break;
- }
-
- }
- }
-}
-
-} // namespace arm_gemm
-
-#endif // __ARM_FEATURE_SVE
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8s32_dot_4VLx4.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8s32_dot_6x4VL.hpp
similarity index 64%
copy from src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8s32_dot_4VLx4.hpp
copy to src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8s32_dot_6x4VL.hpp
index c500f43..1aebedb 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8s32_dot_4VLx4.hpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8s32_dot_6x4VL.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2018-2020 Arm Limited.
+ * Copyright (c) 2019-2020 Arm Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -10,42 +10,48 @@
* sell copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
*/
#pragma once
-
#ifdef __ARM_FEATURE_SVE
-#include <cstdint>
#include "../std_transforms_sve.hpp"
+#define ARGLIST \
+ unsigned int, const unsigned int *, \
+ IndirectInputArg<int8_t>, \
+ size_t, size_t, \
+ const int8_t *, \
+ IndirectOutputArg<int32_t>, \
+ const int32_t *, Activation, bool
+
namespace arm_gemm
{
// Actual kernel implementations
-void sve_hybrid_s8s32_dot_4VLx4(const int8_t *, int, const int8_t *, int32_t *, int, int, int, int, const int32_t *, Activation, bool);
+void sve_hybrid_s8s32_dot_6x4VL( ARGLIST );
-class hybrid_s8s32_dot_4VLx4
+class cls_sve_hybrid_s8s32_dot_6x4VL
{
public:
typedef int8_t operand_type;
typedef int32_t result_type;
- typedef void (*kern_type)(const int8_t *, int, const int8_t *, int32_t *, int, int, int, int, const int32_t *, Activation, bool);
+ typedef void (*kern_type)( ARGLIST );
/* Kernel blocking parameters */
static constexpr unsigned int out_height()
{
- return 4;
+ return 6;
}
static unsigned int out_width()
@@ -63,27 +69,17 @@
return true;
}
- static constexpr bool supports_bias()
- {
- return false;
- }
-
- static constexpr bool supports_activation()
- {
- return false;
- }
-
- StdTransformsSVE<operand_type, result_type, 4, 4, 4> transforms = {};
+ StdTransformsSVE<operand_type, result_type, 6, 4, 4> transforms = {};
// Default to the generic kernel
- kern_type kernel=sve_hybrid_s8s32_dot_4VLx4;
+ kern_type kernel=sve_hybrid_s8s32_dot_6x4VL;
- hybrid_s8s32_dot_4VLx4(const CPUInfo *)
+ cls_sve_hybrid_s8s32_dot_6x4VL(const CPUInfo *)
{
-
}
};
} // namespace arm_gemm
+#undef ARGLIST
#endif // __ARM_FEATURE_SVE
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8s32_dot_6x4VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8s32_dot_6x4VL/generic.cpp
new file mode 100644
index 0000000..cae9bf3
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8s32_dot_6x4VL/generic.cpp
@@ -0,0 +1,1904 @@
+/*
+ * Copyright (c) 2019-2020 Arm Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+#ifdef __ARM_FEATURE_SVE
+
+#include "arm_gemm.hpp"
+#include "../../utils.hpp"
+
+#include <cassert>
+
+namespace arm_gemm {
+
+void sve_hybrid_s8s32_dot_6x4VL (
+ unsigned int num_strings, const unsigned int *string_lengths, IndirectInputArg<int8_t> A_arg,
+ size_t M, size_t N, const int8_t *B_ptr, IndirectOutputArg<int32_t> output_arg,
+ const int32_t *, Activation, bool accumulate
+)
+{
+ struct KernelArgs {
+ unsigned int num_strings = {};
+ const unsigned int *string_lengths = {};
+ size_t N = {};
+ const int8_t *B_ptr = {};
+ size_t output_offset = {};
+ size_t input_initial_col = {};
+ size_t input_offset = {};
+ } ka;
+
+ unsigned long flags=0;
+ void *output_ptr;
+ void *input_ptr;
+
+ if (output_arg.is_indirect) {
+ output_ptr=(void *)(output_arg.indirect.ptr);
+ ka.output_offset=output_arg.indirect.offset;
+ flags |= 0x4;
+ } else {
+ output_ptr=(void *)(output_arg.direct.base);
+ ka.output_offset=output_arg.direct.stride;
+ }
+
+ if (A_arg.is_indirect) {
+ input_ptr=(void *)(A_arg.indirect.ptr);
+ ka.input_offset=A_arg.indirect.start_row;
+ ka.input_initial_col=A_arg.indirect.start_col;
+ flags |= 0x8;
+ } else {
+ assert(num_strings==1);
+ input_ptr=(void *)(A_arg.direct.base);
+ ka.input_offset=A_arg.direct.stride;
+ }
+ if (accumulate) {
+ flags |= 0x1;
+ }
+ ka.num_strings = num_strings;
+ ka.string_lengths = string_lengths;
+ ka.N = N;
+ ka.B_ptr = B_ptr;
+ __asm__ __volatile__(
+ "ptrue p5.b\n"
+ "1:" // Row loop
+ "cmp %x[M], #0x6\n"
+ "bge 61f\n"
+ "cmp %x[M], #0x4\n"
+ "bgt 49f\n"
+ "beq 37f\n"
+ "cmp %x[M], #0x2\n"
+ "bgt 25f\n"
+ "beq 13f\n"
+ "ldr x15, [%x[args_ptr], %[offsetof_N]]\n"
+ "ldr x14, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 2f\n"
+ "ldr x13, [%x[output_ptr], #0x0]\n"
+ "add x13, x13, x19, LSL #2\n"
+ "b 3f\n"
+ "2:" // Height 1: setup direct output
+ "mov x13, %x[output_ptr]\n"
+ "3:" // Height 1: Column loop
+ "mov x19, #0x0\n"
+ "whilelt p4.s, x19, x15\n"
+ "incw x19\n"
+ "whilelt p3.s, x19, x15\n"
+ "incw x19\n"
+ "whilelt p2.s, x19, x15\n"
+ "incw x19\n"
+ "whilelt p1.s, x19, x15\n"
+ "tbz %x[flags], #0, 4f\n"
+ "ld1w { z8.s }, p4/Z, [x13]\n"
+ "ld1w { z9.s }, p3/Z, [x13, #1, MUL VL]\n"
+ "ld1w { z10.s }, p2/Z, [x13, #2, MUL VL]\n"
+ "ld1w { z11.s }, p1/Z, [x13, #3, MUL VL]\n"
+ "b 5f\n"
+ "4:" // Height 1: no accumulate
+ "mov z8.s, #0x0\n"
+ "mov z9.s, #0x0\n"
+ "mov z10.s, #0x0\n"
+ "mov z11.s, #0x0\n"
+ "5:" // Height 1: setup done
+ "mov x12, #0x0\n"
+ "6:" // Height 1: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w11, [x20, x12, LSL #0x2]\n"
+ "tbz %x[flags], #3, 7f\n"
+ "ldr x20, [%x[input_ptr], x12, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x10, [x20, #0x0]\n"
+ "cbnz x12, 8f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x10, x10, x19\n"
+ "b 8f\n"
+ "7:" // Height 1: setup direct input
+ "mov x10, %x[input_ptr]\n"
+ "8:" // Height 1: input setup done
+ "cmp x11, #0x10\n"
+ "ble 10f\n"
+ "9:" // Height 1: Multiply loop: Main loop head
+ "ld1b { z6.b }, p5/Z, [x14]\n"
+ "whilelt p0.b, XZR, x11\n"
+ "ld1b { z7.b }, p5/Z, [x14, #1, MUL VL]\n"
+ "sub x11, x11, #0x10\n"
+ "ld1rqb { z0.b }, p0/Z, [x10]\n"
+ "sdot z8.s, z6.b, z0.b[0]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #2, MUL VL]\n"
+ "add x10, x10, #0x10\n"
+ "sdot z9.s, z7.b, z0.b[0]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #3, MUL VL]\n"
+ "cmp x11, #0x10\n"
+ "sdot z10.s, z6.b, z0.b[0]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #4, MUL VL]\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "sdot z11.s, z7.b, z0.b[0]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #5, MUL VL]\n"
+ "sdot z8.s, z6.b, z0.b[1]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #6, MUL VL]\n"
+ "sdot z9.s, z7.b, z0.b[1]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #7, MUL VL]\n"
+ "addvl x14, x14, #16\n"
+ "sdot z10.s, z6.b, z0.b[1]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #-8, MUL VL]\n"
+ "sdot z11.s, z7.b, z0.b[1]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #-7, MUL VL]\n"
+ "sdot z8.s, z6.b, z0.b[2]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #-6, MUL VL]\n"
+ "sdot z9.s, z7.b, z0.b[2]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #-5, MUL VL]\n"
+ "sdot z10.s, z6.b, z0.b[2]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #-4, MUL VL]\n"
+ "sdot z11.s, z7.b, z0.b[2]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #-3, MUL VL]\n"
+ "sdot z8.s, z6.b, z0.b[3]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #-2, MUL VL]\n"
+ "sdot z9.s, z7.b, z0.b[3]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #-1, MUL VL]\n"
+ "sdot z10.s, z6.b, z0.b[3]\n"
+ "sdot z11.s, z7.b, z0.b[3]\n"
+ "bgt 9b\n"
+ "10:" // Height 1: Multiply loop: Single iteration only
+ "ld1b { z6.b }, p5/Z, [x14]\n"
+ "whilelt p0.b, XZR, x11\n"
+ "ld1b { z7.b }, p5/Z, [x14, #1, MUL VL]\n"
+ "subs x11, x11, #0x4\n"
+ "ld1rqb { z0.b }, p0/Z, [x10]\n"
+ "sdot z8.s, z6.b, z0.b[0]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #2, MUL VL]\n"
+ "add x10, x10, #0x10\n"
+ "sdot z9.s, z7.b, z0.b[0]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #3, MUL VL]\n"
+ "addvl x14, x14, #4\n"
+ "sdot z10.s, z6.b, z0.b[0]\n"
+ "sdot z11.s, z7.b, z0.b[0]\n"
+ "ble 11f\n"
+ "ld1b { z6.b }, p5/Z, [x14]\n"
+ "sdot z8.s, z6.b, z0.b[1]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #1, MUL VL]\n"
+ "subs x11, x11, #0x4\n"
+ "sdot z9.s, z7.b, z0.b[1]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #2, MUL VL]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #3, MUL VL]\n"
+ "sdot z10.s, z6.b, z0.b[1]\n"
+ "addvl x14, x14, #4\n"
+ "sdot z11.s, z7.b, z0.b[1]\n"
+ "ble 11f\n"
+ "ld1b { z6.b }, p5/Z, [x14]\n"
+ "sdot z8.s, z6.b, z0.b[2]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #1, MUL VL]\n"
+ "subs x11, x11, #0x4\n"
+ "sdot z9.s, z7.b, z0.b[2]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #2, MUL VL]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #3, MUL VL]\n"
+ "sdot z10.s, z6.b, z0.b[2]\n"
+ "addvl x14, x14, #4\n"
+ "sdot z11.s, z7.b, z0.b[2]\n"
+ "ble 11f\n"
+ "ld1b { z6.b }, p5/Z, [x14]\n"
+ "sdot z8.s, z6.b, z0.b[3]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #1, MUL VL]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #2, MUL VL]\n"
+ "sdot z9.s, z7.b, z0.b[3]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #3, MUL VL]\n"
+ "addvl x14, x14, #4\n"
+ "sdot z10.s, z6.b, z0.b[3]\n"
+ "sdot z11.s, z7.b, z0.b[3]\n"
+ "11:" // Height 1: Multiply loop: multiply skip
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x12, x12, #0x1\n"
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "cmp x12, x19\n"
+ "bne 6b\n"
+ "prfm pstl1keep, [x13, #0x0]\n"
+ "st1w { z8.s }, p4, [x13]\n"
+ "st1w { z9.s }, p3, [x13, #1, MUL VL]\n"
+ "st1w { z10.s }, p2, [x13, #2, MUL VL]\n"
+ "st1w { z11.s }, p1, [x13, #3, MUL VL]\n"
+ "addvl x13, x13, #4\n"
+ "12:" // Height 1: Writeback done
+ "mov x19, #0x0\n"
+ "incw x19, ALL, MUL #4\n"
+ "subs x15, x15, x19\n"
+ "bgt 3b\n"
+ "b 74f\n"
+ "13:" // Height 2
+ "ldr x15, [%x[args_ptr], %[offsetof_N]]\n"
+ "ldr x14, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 14f\n"
+ "ldr x13, [%x[output_ptr], #0x0]\n"
+ "add x13, x13, x19, LSL #2\n"
+ "ldr x9, [%x[output_ptr], #0x8]\n"
+ "add x9, x9, x19, LSL #2\n"
+ "b 15f\n"
+ "14:" // Height 2: setup direct output
+ "mov x13, %x[output_ptr]\n"
+ "add x9, x13, x19, LSL #2\n"
+ "15:" // Height 2: Column loop
+ "mov x19, #0x0\n"
+ "whilelt p4.s, x19, x15\n"
+ "incw x19\n"
+ "whilelt p3.s, x19, x15\n"
+ "incw x19\n"
+ "whilelt p2.s, x19, x15\n"
+ "incw x19\n"
+ "whilelt p1.s, x19, x15\n"
+ "tbz %x[flags], #0, 16f\n"
+ "ld1w { z8.s }, p4/Z, [x13]\n"
+ "ld1w { z9.s }, p3/Z, [x13, #1, MUL VL]\n"
+ "ld1w { z10.s }, p2/Z, [x13, #2, MUL VL]\n"
+ "ld1w { z11.s }, p1/Z, [x13, #3, MUL VL]\n"
+ "ld1w { z12.s }, p4/Z, [x9]\n"
+ "ld1w { z13.s }, p3/Z, [x9, #1, MUL VL]\n"
+ "ld1w { z14.s }, p2/Z, [x9, #2, MUL VL]\n"
+ "ld1w { z15.s }, p1/Z, [x9, #3, MUL VL]\n"
+ "b 17f\n"
+ "16:" // Height 2: no accumulate
+ "mov z8.s, #0x0\n"
+ "mov z9.s, #0x0\n"
+ "mov z10.s, #0x0\n"
+ "mov z11.s, #0x0\n"
+ "mov z12.s, #0x0\n"
+ "mov z13.s, #0x0\n"
+ "mov z14.s, #0x0\n"
+ "mov z15.s, #0x0\n"
+ "17:" // Height 2: setup done
+ "mov x12, #0x0\n"
+ "18:" // Height 2: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w11, [x20, x12, LSL #0x2]\n"
+ "tbz %x[flags], #3, 19f\n"
+ "ldr x20, [%x[input_ptr], x12, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x10, [x20, #0x0]\n"
+ "ldr x28, [x20, #0x8]\n"
+ "cbnz x12, 20f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x10, x10, x19\n"
+ "add x28, x28, x19\n"
+ "b 20f\n"
+ "19:" // Height 2: setup direct input
+ "mov x10, %x[input_ptr]\n"
+ "add x28, x10, x19\n"
+ "20:" // Height 2: input setup done
+ "cmp x11, #0x10\n"
+ "ble 22f\n"
+ "21:" // Height 2: Multiply loop: Main loop head
+ "ld1b { z6.b }, p5/Z, [x14]\n"
+ "whilelt p0.b, XZR, x11\n"
+ "ld1b { z7.b }, p5/Z, [x14, #1, MUL VL]\n"
+ "sub x11, x11, #0x10\n"
+ "ld1rqb { z0.b }, p0/Z, [x10]\n"
+ "sdot z8.s, z6.b, z0.b[0]\n"
+ "ld1rqb { z1.b }, p0/Z, [x28]\n"
+ "add x10, x10, #0x10\n"
+ "sdot z9.s, z7.b, z0.b[0]\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x28, x28, #0x10\n"
+ "sdot z12.s, z6.b, z1.b[0]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #2, MUL VL]\n"
+ "cmp x11, #0x10\n"
+ "sdot z13.s, z7.b, z1.b[0]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #3, MUL VL]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "sdot z10.s, z6.b, z0.b[0]\n"
+ "sdot z14.s, z6.b, z1.b[0]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #4, MUL VL]\n"
+ "sdot z11.s, z7.b, z0.b[0]\n"
+ "sdot z15.s, z7.b, z1.b[0]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #5, MUL VL]\n"
+ "sdot z8.s, z6.b, z0.b[1]\n"
+ "sdot z12.s, z6.b, z1.b[1]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #6, MUL VL]\n"
+ "sdot z9.s, z7.b, z0.b[1]\n"
+ "sdot z13.s, z7.b, z1.b[1]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #7, MUL VL]\n"
+ "addvl x14, x14, #16\n"
+ "sdot z10.s, z6.b, z0.b[1]\n"
+ "sdot z14.s, z6.b, z1.b[1]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #-8, MUL VL]\n"
+ "sdot z11.s, z7.b, z0.b[1]\n"
+ "sdot z15.s, z7.b, z1.b[1]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #-7, MUL VL]\n"
+ "sdot z8.s, z6.b, z0.b[2]\n"
+ "sdot z12.s, z6.b, z1.b[2]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #-6, MUL VL]\n"
+ "sdot z9.s, z7.b, z0.b[2]\n"
+ "sdot z13.s, z7.b, z1.b[2]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #-5, MUL VL]\n"
+ "sdot z10.s, z6.b, z0.b[2]\n"
+ "sdot z14.s, z6.b, z1.b[2]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #-4, MUL VL]\n"
+ "sdot z11.s, z7.b, z0.b[2]\n"
+ "sdot z15.s, z7.b, z1.b[2]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #-3, MUL VL]\n"
+ "sdot z8.s, z6.b, z0.b[3]\n"
+ "sdot z12.s, z6.b, z1.b[3]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #-2, MUL VL]\n"
+ "sdot z9.s, z7.b, z0.b[3]\n"
+ "sdot z13.s, z7.b, z1.b[3]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #-1, MUL VL]\n"
+ "sdot z10.s, z6.b, z0.b[3]\n"
+ "sdot z14.s, z6.b, z1.b[3]\n"
+ "sdot z11.s, z7.b, z0.b[3]\n"
+ "sdot z15.s, z7.b, z1.b[3]\n"
+ "bgt 21b\n"
+ "22:" // Height 2: Multiply loop: Single iteration only
+ "ld1b { z6.b }, p5/Z, [x14]\n"
+ "whilelt p0.b, XZR, x11\n"
+ "ld1b { z7.b }, p5/Z, [x14, #1, MUL VL]\n"
+ "subs x11, x11, #0x4\n"
+ "ld1rqb { z0.b }, p0/Z, [x10]\n"
+ "sdot z8.s, z6.b, z0.b[0]\n"
+ "ld1rqb { z1.b }, p0/Z, [x28]\n"
+ "add x10, x10, #0x10\n"
+ "sdot z9.s, z7.b, z0.b[0]\n"
+ "add x28, x28, #0x10\n"
+ "sdot z12.s, z6.b, z1.b[0]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #2, MUL VL]\n"
+ "sdot z13.s, z7.b, z1.b[0]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #3, MUL VL]\n"
+ "addvl x14, x14, #4\n"
+ "sdot z10.s, z6.b, z0.b[0]\n"
+ "sdot z14.s, z6.b, z1.b[0]\n"
+ "sdot z11.s, z7.b, z0.b[0]\n"
+ "sdot z15.s, z7.b, z1.b[0]\n"
+ "ble 23f\n"
+ "ld1b { z6.b }, p5/Z, [x14]\n"
+ "sdot z8.s, z6.b, z0.b[1]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #1, MUL VL]\n"
+ "subs x11, x11, #0x4\n"
+ "sdot z12.s, z6.b, z1.b[1]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #2, MUL VL]\n"
+ "sdot z9.s, z7.b, z0.b[1]\n"
+ "sdot z13.s, z7.b, z1.b[1]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #3, MUL VL]\n"
+ "addvl x14, x14, #4\n"
+ "sdot z10.s, z6.b, z0.b[1]\n"
+ "sdot z14.s, z6.b, z1.b[1]\n"
+ "sdot z11.s, z7.b, z0.b[1]\n"
+ "sdot z15.s, z7.b, z1.b[1]\n"
+ "ble 23f\n"
+ "ld1b { z6.b }, p5/Z, [x14]\n"
+ "sdot z8.s, z6.b, z0.b[2]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #1, MUL VL]\n"
+ "subs x11, x11, #0x4\n"
+ "sdot z12.s, z6.b, z1.b[2]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #2, MUL VL]\n"
+ "sdot z9.s, z7.b, z0.b[2]\n"
+ "sdot z13.s, z7.b, z1.b[2]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #3, MUL VL]\n"
+ "addvl x14, x14, #4\n"
+ "sdot z10.s, z6.b, z0.b[2]\n"
+ "sdot z14.s, z6.b, z1.b[2]\n"
+ "sdot z11.s, z7.b, z0.b[2]\n"
+ "sdot z15.s, z7.b, z1.b[2]\n"
+ "ble 23f\n"
+ "ld1b { z6.b }, p5/Z, [x14]\n"
+ "sdot z8.s, z6.b, z0.b[3]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #1, MUL VL]\n"
+ "sdot z12.s, z6.b, z1.b[3]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #2, MUL VL]\n"
+ "sdot z9.s, z7.b, z0.b[3]\n"
+ "sdot z13.s, z7.b, z1.b[3]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #3, MUL VL]\n"
+ "addvl x14, x14, #4\n"
+ "sdot z10.s, z6.b, z0.b[3]\n"
+ "sdot z14.s, z6.b, z1.b[3]\n"
+ "sdot z11.s, z7.b, z0.b[3]\n"
+ "sdot z15.s, z7.b, z1.b[3]\n"
+ "23:" // Height 2: Multiply loop: multiply skip
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x12, x12, #0x1\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "cmp x12, x19\n"
+ "bne 18b\n"
+ "prfm pstl1keep, [x13, #0x0]\n"
+ "prfm pstl1keep, [x9, #0x0]\n"
+ "st1w { z8.s }, p4, [x13]\n"
+ "st1w { z9.s }, p3, [x13, #1, MUL VL]\n"
+ "st1w { z10.s }, p2, [x13, #2, MUL VL]\n"
+ "st1w { z11.s }, p1, [x13, #3, MUL VL]\n"
+ "addvl x13, x13, #4\n"
+ "st1w { z12.s }, p4, [x9]\n"
+ "st1w { z13.s }, p3, [x9, #1, MUL VL]\n"
+ "st1w { z14.s }, p2, [x9, #2, MUL VL]\n"
+ "st1w { z15.s }, p1, [x9, #3, MUL VL]\n"
+ "addvl x9, x9, #4\n"
+ "24:" // Height 2: Writeback done
+ "mov x19, #0x0\n"
+ "incw x19, ALL, MUL #4\n"
+ "subs x15, x15, x19\n"
+ "bgt 15b\n"
+ "b 74f\n"
+ "25:" // Height 3
+ "ldr x15, [%x[args_ptr], %[offsetof_N]]\n"
+ "ldr x14, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 26f\n"
+ "ldr x13, [%x[output_ptr], #0x0]\n"
+ "add x13, x13, x19, LSL #2\n"
+ "ldr x9, [%x[output_ptr], #0x8]\n"
+ "ldr x27, [%x[output_ptr], #0x10]\n"
+ "add x9, x9, x19, LSL #2\n"
+ "add x27, x27, x19, LSL #2\n"
+ "b 27f\n"
+ "26:" // Height 3: setup direct output
+ "mov x13, %x[output_ptr]\n"
+ "add x9, x13, x19, LSL #2\n"
+ "add x27, x9, x19, LSL #2\n"
+ "27:" // Height 3: Column loop
+ "mov x19, #0x0\n"
+ "whilelt p4.s, x19, x15\n"
+ "incw x19\n"
+ "whilelt p3.s, x19, x15\n"
+ "incw x19\n"
+ "whilelt p2.s, x19, x15\n"
+ "incw x19\n"
+ "whilelt p1.s, x19, x15\n"
+ "tbz %x[flags], #0, 28f\n"
+ "ld1w { z8.s }, p4/Z, [x13]\n"
+ "ld1w { z9.s }, p3/Z, [x13, #1, MUL VL]\n"
+ "ld1w { z10.s }, p2/Z, [x13, #2, MUL VL]\n"
+ "ld1w { z11.s }, p1/Z, [x13, #3, MUL VL]\n"
+ "ld1w { z12.s }, p4/Z, [x9]\n"
+ "ld1w { z13.s }, p3/Z, [x9, #1, MUL VL]\n"
+ "ld1w { z14.s }, p2/Z, [x9, #2, MUL VL]\n"
+ "ld1w { z15.s }, p1/Z, [x9, #3, MUL VL]\n"
+ "ld1w { z16.s }, p4/Z, [x27]\n"
+ "ld1w { z17.s }, p3/Z, [x27, #1, MUL VL]\n"
+ "ld1w { z18.s }, p2/Z, [x27, #2, MUL VL]\n"
+ "ld1w { z19.s }, p1/Z, [x27, #3, MUL VL]\n"
+ "b 29f\n"
+ "28:" // Height 3: no accumulate
+ "mov z8.s, #0x0\n"
+ "mov z9.s, #0x0\n"
+ "mov z10.s, #0x0\n"
+ "mov z11.s, #0x0\n"
+ "mov z12.s, #0x0\n"
+ "mov z13.s, #0x0\n"
+ "mov z14.s, #0x0\n"
+ "mov z15.s, #0x0\n"
+ "mov z16.s, #0x0\n"
+ "mov z17.s, #0x0\n"
+ "mov z18.s, #0x0\n"
+ "mov z19.s, #0x0\n"
+ "29:" // Height 3: setup done
+ "mov x12, #0x0\n"
+ "30:" // Height 3: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w11, [x20, x12, LSL #0x2]\n"
+ "tbz %x[flags], #3, 31f\n"
+ "ldr x20, [%x[input_ptr], x12, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x10, [x20, #0x0]\n"
+ "ldr x28, [x20, #0x8]\n"
+ "ldr x26, [x20, #0x10]\n"
+ "cbnz x12, 32f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x10, x10, x19\n"
+ "add x28, x28, x19\n"
+ "add x26, x26, x19\n"
+ "b 32f\n"
+ "31:" // Height 3: setup direct input
+ "mov x10, %x[input_ptr]\n"
+ "add x28, x10, x19\n"
+ "add x26, x28, x19\n"
+ "32:" // Height 3: input setup done
+ "cmp x11, #0x10\n"
+ "ble 34f\n"
+ "33:" // Height 3: Multiply loop: Main loop head
+ "ld1b { z6.b }, p5/Z, [x14]\n"
+ "whilelt p0.b, XZR, x11\n"
+ "ld1b { z7.b }, p5/Z, [x14, #1, MUL VL]\n"
+ "sub x11, x11, #0x10\n"
+ "ld1rqb { z0.b }, p0/Z, [x10]\n"
+ "sdot z8.s, z6.b, z0.b[0]\n"
+ "ld1rqb { z1.b }, p0/Z, [x28]\n"
+ "add x10, x10, #0x10\n"
+ "sdot z9.s, z7.b, z0.b[0]\n"
+ "ld1rqb { z2.b }, p0/Z, [x26]\n"
+ "add x28, x28, #0x10\n"
+ "sdot z12.s, z6.b, z1.b[0]\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x26, x26, #0x10\n"
+ "sdot z16.s, z6.b, z2.b[0]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #2, MUL VL]\n"
+ "cmp x11, #0x10\n"
+ "sdot z13.s, z7.b, z1.b[0]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "sdot z17.s, z7.b, z2.b[0]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #3, MUL VL]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "sdot z10.s, z6.b, z0.b[0]\n"
+ "sdot z14.s, z6.b, z1.b[0]\n"
+ "sdot z18.s, z6.b, z2.b[0]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #4, MUL VL]\n"
+ "sdot z11.s, z7.b, z0.b[0]\n"
+ "sdot z15.s, z7.b, z1.b[0]\n"
+ "sdot z19.s, z7.b, z2.b[0]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #5, MUL VL]\n"
+ "sdot z8.s, z6.b, z0.b[1]\n"
+ "sdot z12.s, z6.b, z1.b[1]\n"
+ "sdot z16.s, z6.b, z2.b[1]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #6, MUL VL]\n"
+ "sdot z9.s, z7.b, z0.b[1]\n"
+ "sdot z13.s, z7.b, z1.b[1]\n"
+ "sdot z17.s, z7.b, z2.b[1]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #7, MUL VL]\n"
+ "addvl x14, x14, #16\n"
+ "sdot z10.s, z6.b, z0.b[1]\n"
+ "sdot z14.s, z6.b, z1.b[1]\n"
+ "sdot z18.s, z6.b, z2.b[1]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #-8, MUL VL]\n"
+ "sdot z11.s, z7.b, z0.b[1]\n"
+ "sdot z15.s, z7.b, z1.b[1]\n"
+ "sdot z19.s, z7.b, z2.b[1]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #-7, MUL VL]\n"
+ "sdot z8.s, z6.b, z0.b[2]\n"
+ "sdot z12.s, z6.b, z1.b[2]\n"
+ "sdot z16.s, z6.b, z2.b[2]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #-6, MUL VL]\n"
+ "sdot z9.s, z7.b, z0.b[2]\n"
+ "sdot z13.s, z7.b, z1.b[2]\n"
+ "sdot z17.s, z7.b, z2.b[2]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #-5, MUL VL]\n"
+ "sdot z10.s, z6.b, z0.b[2]\n"
+ "sdot z14.s, z6.b, z1.b[2]\n"
+ "sdot z18.s, z6.b, z2.b[2]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #-4, MUL VL]\n"
+ "sdot z11.s, z7.b, z0.b[2]\n"
+ "sdot z15.s, z7.b, z1.b[2]\n"
+ "sdot z19.s, z7.b, z2.b[2]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #-3, MUL VL]\n"
+ "sdot z8.s, z6.b, z0.b[3]\n"
+ "sdot z12.s, z6.b, z1.b[3]\n"
+ "sdot z16.s, z6.b, z2.b[3]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #-2, MUL VL]\n"
+ "sdot z9.s, z7.b, z0.b[3]\n"
+ "sdot z13.s, z7.b, z1.b[3]\n"
+ "sdot z17.s, z7.b, z2.b[3]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #-1, MUL VL]\n"
+ "sdot z10.s, z6.b, z0.b[3]\n"
+ "sdot z14.s, z6.b, z1.b[3]\n"
+ "sdot z18.s, z6.b, z2.b[3]\n"
+ "sdot z11.s, z7.b, z0.b[3]\n"
+ "sdot z15.s, z7.b, z1.b[3]\n"
+ "sdot z19.s, z7.b, z2.b[3]\n"
+ "bgt 33b\n"
+ "34:" // Height 3: Multiply loop: Single iteration only
+ "ld1b { z6.b }, p5/Z, [x14]\n"
+ "whilelt p0.b, XZR, x11\n"
+ "ld1b { z7.b }, p5/Z, [x14, #1, MUL VL]\n"
+ "subs x11, x11, #0x4\n"
+ "ld1rqb { z0.b }, p0/Z, [x10]\n"
+ "sdot z8.s, z6.b, z0.b[0]\n"
+ "ld1rqb { z1.b }, p0/Z, [x28]\n"
+ "add x10, x10, #0x10\n"
+ "sdot z9.s, z7.b, z0.b[0]\n"
+ "ld1rqb { z2.b }, p0/Z, [x26]\n"
+ "add x28, x28, #0x10\n"
+ "sdot z12.s, z6.b, z1.b[0]\n"
+ "add x26, x26, #0x10\n"
+ "sdot z13.s, z7.b, z1.b[0]\n"
+ "sdot z16.s, z6.b, z2.b[0]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #2, MUL VL]\n"
+ "sdot z17.s, z7.b, z2.b[0]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #3, MUL VL]\n"
+ "addvl x14, x14, #4\n"
+ "sdot z10.s, z6.b, z0.b[0]\n"
+ "sdot z14.s, z6.b, z1.b[0]\n"
+ "sdot z18.s, z6.b, z2.b[0]\n"
+ "sdot z11.s, z7.b, z0.b[0]\n"
+ "sdot z15.s, z7.b, z1.b[0]\n"
+ "sdot z19.s, z7.b, z2.b[0]\n"
+ "ble 35f\n"
+ "ld1b { z6.b }, p5/Z, [x14]\n"
+ "sdot z8.s, z6.b, z0.b[1]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #1, MUL VL]\n"
+ "subs x11, x11, #0x4\n"
+ "sdot z12.s, z6.b, z1.b[1]\n"
+ "sdot z16.s, z6.b, z2.b[1]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #2, MUL VL]\n"
+ "sdot z9.s, z7.b, z0.b[1]\n"
+ "sdot z13.s, z7.b, z1.b[1]\n"
+ "sdot z17.s, z7.b, z2.b[1]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #3, MUL VL]\n"
+ "addvl x14, x14, #4\n"
+ "sdot z10.s, z6.b, z0.b[1]\n"
+ "sdot z14.s, z6.b, z1.b[1]\n"
+ "sdot z18.s, z6.b, z2.b[1]\n"
+ "sdot z11.s, z7.b, z0.b[1]\n"
+ "sdot z15.s, z7.b, z1.b[1]\n"
+ "sdot z19.s, z7.b, z2.b[1]\n"
+ "ble 35f\n"
+ "ld1b { z6.b }, p5/Z, [x14]\n"
+ "sdot z8.s, z6.b, z0.b[2]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #1, MUL VL]\n"
+ "subs x11, x11, #0x4\n"
+ "sdot z12.s, z6.b, z1.b[2]\n"
+ "sdot z16.s, z6.b, z2.b[2]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #2, MUL VL]\n"
+ "sdot z9.s, z7.b, z0.b[2]\n"
+ "sdot z13.s, z7.b, z1.b[2]\n"
+ "sdot z17.s, z7.b, z2.b[2]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #3, MUL VL]\n"
+ "addvl x14, x14, #4\n"
+ "sdot z10.s, z6.b, z0.b[2]\n"
+ "sdot z14.s, z6.b, z1.b[2]\n"
+ "sdot z18.s, z6.b, z2.b[2]\n"
+ "sdot z11.s, z7.b, z0.b[2]\n"
+ "sdot z15.s, z7.b, z1.b[2]\n"
+ "sdot z19.s, z7.b, z2.b[2]\n"
+ "ble 35f\n"
+ "ld1b { z6.b }, p5/Z, [x14]\n"
+ "sdot z8.s, z6.b, z0.b[3]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #1, MUL VL]\n"
+ "sdot z12.s, z6.b, z1.b[3]\n"
+ "sdot z16.s, z6.b, z2.b[3]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #2, MUL VL]\n"
+ "sdot z9.s, z7.b, z0.b[3]\n"
+ "sdot z13.s, z7.b, z1.b[3]\n"
+ "sdot z17.s, z7.b, z2.b[3]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #3, MUL VL]\n"
+ "addvl x14, x14, #4\n"
+ "sdot z10.s, z6.b, z0.b[3]\n"
+ "sdot z14.s, z6.b, z1.b[3]\n"
+ "sdot z18.s, z6.b, z2.b[3]\n"
+ "sdot z11.s, z7.b, z0.b[3]\n"
+ "sdot z15.s, z7.b, z1.b[3]\n"
+ "sdot z19.s, z7.b, z2.b[3]\n"
+ "35:" // Height 3: Multiply loop: multiply skip
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x12, x12, #0x1\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "cmp x12, x19\n"
+ "bne 30b\n"
+ "prfm pstl1keep, [x13, #0x0]\n"
+ "prfm pstl1keep, [x9, #0x0]\n"
+ "prfm pstl1keep, [x27, #0x0]\n"
+ "st1w { z8.s }, p4, [x13]\n"
+ "st1w { z9.s }, p3, [x13, #1, MUL VL]\n"
+ "st1w { z10.s }, p2, [x13, #2, MUL VL]\n"
+ "st1w { z11.s }, p1, [x13, #3, MUL VL]\n"
+ "addvl x13, x13, #4\n"
+ "st1w { z12.s }, p4, [x9]\n"
+ "st1w { z13.s }, p3, [x9, #1, MUL VL]\n"
+ "st1w { z14.s }, p2, [x9, #2, MUL VL]\n"
+ "st1w { z15.s }, p1, [x9, #3, MUL VL]\n"
+ "addvl x9, x9, #4\n"
+ "st1w { z16.s }, p4, [x27]\n"
+ "st1w { z17.s }, p3, [x27, #1, MUL VL]\n"
+ "st1w { z18.s }, p2, [x27, #2, MUL VL]\n"
+ "st1w { z19.s }, p1, [x27, #3, MUL VL]\n"
+ "addvl x27, x27, #4\n"
+ "36:" // Height 3: Writeback done
+ "mov x19, #0x0\n"
+ "incw x19, ALL, MUL #4\n"
+ "subs x15, x15, x19\n"
+ "bgt 27b\n"
+ "b 74f\n"
+ "37:" // Height 4
+ "ldr x15, [%x[args_ptr], %[offsetof_N]]\n"
+ "ldr x14, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 38f\n"
+ "ldr x13, [%x[output_ptr], #0x0]\n"
+ "add x13, x13, x19, LSL #2\n"
+ "ldr x9, [%x[output_ptr], #0x8]\n"
+ "ldr x27, [%x[output_ptr], #0x10]\n"
+ "add x9, x9, x19, LSL #2\n"
+ "ldr x25, [%x[output_ptr], #0x18]\n"
+ "add x27, x27, x19, LSL #2\n"
+ "add x25, x25, x19, LSL #2\n"
+ "b 39f\n"
+ "38:" // Height 4: setup direct output
+ "mov x13, %x[output_ptr]\n"
+ "add x9, x13, x19, LSL #2\n"
+ "add x27, x9, x19, LSL #2\n"
+ "add x25, x27, x19, LSL #2\n"
+ "39:" // Height 4: Column loop
+ "mov x19, #0x0\n"
+ "whilelt p4.s, x19, x15\n"
+ "incw x19\n"
+ "whilelt p3.s, x19, x15\n"
+ "incw x19\n"
+ "whilelt p2.s, x19, x15\n"
+ "incw x19\n"
+ "whilelt p1.s, x19, x15\n"
+ "tbz %x[flags], #0, 40f\n"
+ "ld1w { z8.s }, p4/Z, [x13]\n"
+ "ld1w { z9.s }, p3/Z, [x13, #1, MUL VL]\n"
+ "ld1w { z10.s }, p2/Z, [x13, #2, MUL VL]\n"
+ "ld1w { z11.s }, p1/Z, [x13, #3, MUL VL]\n"
+ "ld1w { z12.s }, p4/Z, [x9]\n"
+ "ld1w { z13.s }, p3/Z, [x9, #1, MUL VL]\n"
+ "ld1w { z14.s }, p2/Z, [x9, #2, MUL VL]\n"
+ "ld1w { z15.s }, p1/Z, [x9, #3, MUL VL]\n"
+ "ld1w { z16.s }, p4/Z, [x27]\n"
+ "ld1w { z17.s }, p3/Z, [x27, #1, MUL VL]\n"
+ "ld1w { z18.s }, p2/Z, [x27, #2, MUL VL]\n"
+ "ld1w { z19.s }, p1/Z, [x27, #3, MUL VL]\n"
+ "ld1w { z20.s }, p4/Z, [x25]\n"
+ "ld1w { z21.s }, p3/Z, [x25, #1, MUL VL]\n"
+ "ld1w { z22.s }, p2/Z, [x25, #2, MUL VL]\n"
+ "ld1w { z23.s }, p1/Z, [x25, #3, MUL VL]\n"
+ "b 41f\n"
+ "40:" // Height 4: no accumulate
+ "mov z8.s, #0x0\n"
+ "mov z9.s, #0x0\n"
+ "mov z10.s, #0x0\n"
+ "mov z11.s, #0x0\n"
+ "mov z12.s, #0x0\n"
+ "mov z13.s, #0x0\n"
+ "mov z14.s, #0x0\n"
+ "mov z15.s, #0x0\n"
+ "mov z16.s, #0x0\n"
+ "mov z17.s, #0x0\n"
+ "mov z18.s, #0x0\n"
+ "mov z19.s, #0x0\n"
+ "mov z20.s, #0x0\n"
+ "mov z21.s, #0x0\n"
+ "mov z22.s, #0x0\n"
+ "mov z23.s, #0x0\n"
+ "41:" // Height 4: setup done
+ "mov x12, #0x0\n"
+ "42:" // Height 4: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w11, [x20, x12, LSL #0x2]\n"
+ "tbz %x[flags], #3, 43f\n"
+ "ldr x20, [%x[input_ptr], x12, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x10, [x20, #0x0]\n"
+ "ldr x28, [x20, #0x8]\n"
+ "ldr x26, [x20, #0x10]\n"
+ "ldr x24, [x20, #0x18]\n"
+ "cbnz x12, 44f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x10, x10, x19\n"
+ "add x28, x28, x19\n"
+ "add x26, x26, x19\n"
+ "add x24, x24, x19\n"
+ "b 44f\n"
+ "43:" // Height 4: setup direct input
+ "mov x10, %x[input_ptr]\n"
+ "add x28, x10, x19\n"
+ "add x26, x28, x19\n"
+ "add x24, x26, x19\n"
+ "44:" // Height 4: input setup done
+ "cmp x11, #0x10\n"
+ "ble 46f\n"
+ "45:" // Height 4: Multiply loop: Main loop head
+ "ld1b { z6.b }, p5/Z, [x14]\n"
+ "whilelt p0.b, XZR, x11\n"
+ "ld1b { z7.b }, p5/Z, [x14, #1, MUL VL]\n"
+ "sub x11, x11, #0x10\n"
+ "ld1rqb { z0.b }, p0/Z, [x10]\n"
+ "sdot z8.s, z6.b, z0.b[0]\n"
+ "ld1rqb { z1.b }, p0/Z, [x28]\n"
+ "add x10, x10, #0x10\n"
+ "sdot z9.s, z7.b, z0.b[0]\n"
+ "ld1rqb { z2.b }, p0/Z, [x26]\n"
+ "add x28, x28, #0x10\n"
+ "sdot z12.s, z6.b, z1.b[0]\n"
+ "ld1rqb { z3.b }, p0/Z, [x24]\n"
+ "add x26, x26, #0x10\n"
+ "sdot z16.s, z6.b, z2.b[0]\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x24, x24, #0x10\n"
+ "sdot z13.s, z7.b, z1.b[0]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "cmp x11, #0x10\n"
+ "sdot z20.s, z6.b, z3.b[0]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #2, MUL VL]\n"
+ "sdot z17.s, z7.b, z2.b[0]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "sdot z21.s, z7.b, z3.b[0]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #3, MUL VL]\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "sdot z10.s, z6.b, z0.b[0]\n"
+ "sdot z14.s, z6.b, z1.b[0]\n"
+ "sdot z18.s, z6.b, z2.b[0]\n"
+ "sdot z22.s, z6.b, z3.b[0]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #4, MUL VL]\n"
+ "sdot z11.s, z7.b, z0.b[0]\n"
+ "sdot z15.s, z7.b, z1.b[0]\n"
+ "sdot z19.s, z7.b, z2.b[0]\n"
+ "sdot z23.s, z7.b, z3.b[0]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #5, MUL VL]\n"
+ "sdot z8.s, z6.b, z0.b[1]\n"
+ "sdot z12.s, z6.b, z1.b[1]\n"
+ "sdot z16.s, z6.b, z2.b[1]\n"
+ "sdot z20.s, z6.b, z3.b[1]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #6, MUL VL]\n"
+ "sdot z9.s, z7.b, z0.b[1]\n"
+ "sdot z13.s, z7.b, z1.b[1]\n"
+ "sdot z17.s, z7.b, z2.b[1]\n"
+ "sdot z21.s, z7.b, z3.b[1]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #7, MUL VL]\n"
+ "addvl x14, x14, #16\n"
+ "sdot z10.s, z6.b, z0.b[1]\n"
+ "sdot z14.s, z6.b, z1.b[1]\n"
+ "sdot z18.s, z6.b, z2.b[1]\n"
+ "sdot z22.s, z6.b, z3.b[1]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #-8, MUL VL]\n"
+ "sdot z11.s, z7.b, z0.b[1]\n"
+ "sdot z15.s, z7.b, z1.b[1]\n"
+ "sdot z19.s, z7.b, z2.b[1]\n"
+ "sdot z23.s, z7.b, z3.b[1]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #-7, MUL VL]\n"
+ "sdot z8.s, z6.b, z0.b[2]\n"
+ "sdot z12.s, z6.b, z1.b[2]\n"
+ "sdot z16.s, z6.b, z2.b[2]\n"
+ "sdot z20.s, z6.b, z3.b[2]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #-6, MUL VL]\n"
+ "sdot z9.s, z7.b, z0.b[2]\n"
+ "sdot z13.s, z7.b, z1.b[2]\n"
+ "sdot z17.s, z7.b, z2.b[2]\n"
+ "sdot z21.s, z7.b, z3.b[2]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #-5, MUL VL]\n"
+ "sdot z10.s, z6.b, z0.b[2]\n"
+ "sdot z14.s, z6.b, z1.b[2]\n"
+ "sdot z18.s, z6.b, z2.b[2]\n"
+ "sdot z22.s, z6.b, z3.b[2]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #-4, MUL VL]\n"
+ "sdot z11.s, z7.b, z0.b[2]\n"
+ "sdot z15.s, z7.b, z1.b[2]\n"
+ "sdot z19.s, z7.b, z2.b[2]\n"
+ "sdot z23.s, z7.b, z3.b[2]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #-3, MUL VL]\n"
+ "sdot z8.s, z6.b, z0.b[3]\n"
+ "sdot z12.s, z6.b, z1.b[3]\n"
+ "sdot z16.s, z6.b, z2.b[3]\n"
+ "sdot z20.s, z6.b, z3.b[3]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #-2, MUL VL]\n"
+ "sdot z9.s, z7.b, z0.b[3]\n"
+ "sdot z13.s, z7.b, z1.b[3]\n"
+ "sdot z17.s, z7.b, z2.b[3]\n"
+ "sdot z21.s, z7.b, z3.b[3]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #-1, MUL VL]\n"
+ "sdot z10.s, z6.b, z0.b[3]\n"
+ "sdot z14.s, z6.b, z1.b[3]\n"
+ "sdot z18.s, z6.b, z2.b[3]\n"
+ "sdot z22.s, z6.b, z3.b[3]\n"
+ "sdot z11.s, z7.b, z0.b[3]\n"
+ "sdot z15.s, z7.b, z1.b[3]\n"
+ "sdot z19.s, z7.b, z2.b[3]\n"
+ "sdot z23.s, z7.b, z3.b[3]\n"
+ "bgt 45b\n"
+ "46:" // Height 4: Multiply loop: Single iteration only
+ "ld1b { z6.b }, p5/Z, [x14]\n"
+ "whilelt p0.b, XZR, x11\n"
+ "ld1b { z7.b }, p5/Z, [x14, #1, MUL VL]\n"
+ "subs x11, x11, #0x4\n"
+ "ld1rqb { z0.b }, p0/Z, [x10]\n"
+ "sdot z8.s, z6.b, z0.b[0]\n"
+ "ld1rqb { z1.b }, p0/Z, [x28]\n"
+ "add x10, x10, #0x10\n"
+ "sdot z9.s, z7.b, z0.b[0]\n"
+ "ld1rqb { z2.b }, p0/Z, [x26]\n"
+ "add x28, x28, #0x10\n"
+ "sdot z12.s, z6.b, z1.b[0]\n"
+ "ld1rqb { z3.b }, p0/Z, [x24]\n"
+ "add x26, x26, #0x10\n"
+ "sdot z16.s, z6.b, z2.b[0]\n"
+ "add x24, x24, #0x10\n"
+ "sdot z13.s, z7.b, z1.b[0]\n"
+ "sdot z17.s, z7.b, z2.b[0]\n"
+ "sdot z20.s, z6.b, z3.b[0]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #2, MUL VL]\n"
+ "sdot z21.s, z7.b, z3.b[0]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #3, MUL VL]\n"
+ "addvl x14, x14, #4\n"
+ "sdot z10.s, z6.b, z0.b[0]\n"
+ "sdot z14.s, z6.b, z1.b[0]\n"
+ "sdot z18.s, z6.b, z2.b[0]\n"
+ "sdot z22.s, z6.b, z3.b[0]\n"
+ "sdot z11.s, z7.b, z0.b[0]\n"
+ "sdot z15.s, z7.b, z1.b[0]\n"
+ "sdot z19.s, z7.b, z2.b[0]\n"
+ "sdot z23.s, z7.b, z3.b[0]\n"
+ "ble 47f\n"
+ "ld1b { z6.b }, p5/Z, [x14]\n"
+ "sdot z8.s, z6.b, z0.b[1]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #1, MUL VL]\n"
+ "subs x11, x11, #0x4\n"
+ "sdot z12.s, z6.b, z1.b[1]\n"
+ "sdot z16.s, z6.b, z2.b[1]\n"
+ "sdot z20.s, z6.b, z3.b[1]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #2, MUL VL]\n"
+ "sdot z9.s, z7.b, z0.b[1]\n"
+ "sdot z13.s, z7.b, z1.b[1]\n"
+ "sdot z17.s, z7.b, z2.b[1]\n"
+ "sdot z21.s, z7.b, z3.b[1]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #3, MUL VL]\n"
+ "addvl x14, x14, #4\n"
+ "sdot z10.s, z6.b, z0.b[1]\n"
+ "sdot z14.s, z6.b, z1.b[1]\n"
+ "sdot z18.s, z6.b, z2.b[1]\n"
+ "sdot z22.s, z6.b, z3.b[1]\n"
+ "sdot z11.s, z7.b, z0.b[1]\n"
+ "sdot z15.s, z7.b, z1.b[1]\n"
+ "sdot z19.s, z7.b, z2.b[1]\n"
+ "sdot z23.s, z7.b, z3.b[1]\n"
+ "ble 47f\n"
+ "ld1b { z6.b }, p5/Z, [x14]\n"
+ "sdot z8.s, z6.b, z0.b[2]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #1, MUL VL]\n"
+ "subs x11, x11, #0x4\n"
+ "sdot z12.s, z6.b, z1.b[2]\n"
+ "sdot z16.s, z6.b, z2.b[2]\n"
+ "sdot z20.s, z6.b, z3.b[2]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #2, MUL VL]\n"
+ "sdot z9.s, z7.b, z0.b[2]\n"
+ "sdot z13.s, z7.b, z1.b[2]\n"
+ "sdot z17.s, z7.b, z2.b[2]\n"
+ "sdot z21.s, z7.b, z3.b[2]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #3, MUL VL]\n"
+ "addvl x14, x14, #4\n"
+ "sdot z10.s, z6.b, z0.b[2]\n"
+ "sdot z14.s, z6.b, z1.b[2]\n"
+ "sdot z18.s, z6.b, z2.b[2]\n"
+ "sdot z22.s, z6.b, z3.b[2]\n"
+ "sdot z11.s, z7.b, z0.b[2]\n"
+ "sdot z15.s, z7.b, z1.b[2]\n"
+ "sdot z19.s, z7.b, z2.b[2]\n"
+ "sdot z23.s, z7.b, z3.b[2]\n"
+ "ble 47f\n"
+ "ld1b { z6.b }, p5/Z, [x14]\n"
+ "sdot z8.s, z6.b, z0.b[3]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #1, MUL VL]\n"
+ "sdot z12.s, z6.b, z1.b[3]\n"
+ "sdot z16.s, z6.b, z2.b[3]\n"
+ "sdot z20.s, z6.b, z3.b[3]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #2, MUL VL]\n"
+ "sdot z9.s, z7.b, z0.b[3]\n"
+ "sdot z13.s, z7.b, z1.b[3]\n"
+ "sdot z17.s, z7.b, z2.b[3]\n"
+ "sdot z21.s, z7.b, z3.b[3]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #3, MUL VL]\n"
+ "addvl x14, x14, #4\n"
+ "sdot z10.s, z6.b, z0.b[3]\n"
+ "sdot z14.s, z6.b, z1.b[3]\n"
+ "sdot z18.s, z6.b, z2.b[3]\n"
+ "sdot z22.s, z6.b, z3.b[3]\n"
+ "sdot z11.s, z7.b, z0.b[3]\n"
+ "sdot z15.s, z7.b, z1.b[3]\n"
+ "sdot z19.s, z7.b, z2.b[3]\n"
+ "sdot z23.s, z7.b, z3.b[3]\n"
+ "47:" // Height 4: Multiply loop: multiply skip
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x12, x12, #0x1\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "cmp x12, x19\n"
+ "bne 42b\n"
+ "prfm pstl1keep, [x13, #0x0]\n"
+ "prfm pstl1keep, [x9, #0x0]\n"
+ "prfm pstl1keep, [x27, #0x0]\n"
+ "prfm pstl1keep, [x25, #0x0]\n"
+ "st1w { z8.s }, p4, [x13]\n"
+ "st1w { z9.s }, p3, [x13, #1, MUL VL]\n"
+ "st1w { z10.s }, p2, [x13, #2, MUL VL]\n"
+ "st1w { z11.s }, p1, [x13, #3, MUL VL]\n"
+ "addvl x13, x13, #4\n"
+ "st1w { z12.s }, p4, [x9]\n"
+ "st1w { z13.s }, p3, [x9, #1, MUL VL]\n"
+ "st1w { z14.s }, p2, [x9, #2, MUL VL]\n"
+ "st1w { z15.s }, p1, [x9, #3, MUL VL]\n"
+ "addvl x9, x9, #4\n"
+ "st1w { z16.s }, p4, [x27]\n"
+ "st1w { z17.s }, p3, [x27, #1, MUL VL]\n"
+ "st1w { z18.s }, p2, [x27, #2, MUL VL]\n"
+ "st1w { z19.s }, p1, [x27, #3, MUL VL]\n"
+ "addvl x27, x27, #4\n"
+ "st1w { z20.s }, p4, [x25]\n"
+ "st1w { z21.s }, p3, [x25, #1, MUL VL]\n"
+ "st1w { z22.s }, p2, [x25, #2, MUL VL]\n"
+ "st1w { z23.s }, p1, [x25, #3, MUL VL]\n"
+ "addvl x25, x25, #4\n"
+ "48:" // Height 4: Writeback done
+ "mov x19, #0x0\n"
+ "incw x19, ALL, MUL #4\n"
+ "subs x15, x15, x19\n"
+ "bgt 39b\n"
+ "b 74f\n"
+ "49:" // Height 5
+ "ldr x15, [%x[args_ptr], %[offsetof_N]]\n"
+ "ldr x14, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 50f\n"
+ "ldr x13, [%x[output_ptr], #0x0]\n"
+ "add x13, x13, x19, LSL #2\n"
+ "ldr x9, [%x[output_ptr], #0x8]\n"
+ "ldr x27, [%x[output_ptr], #0x10]\n"
+ "add x9, x9, x19, LSL #2\n"
+ "ldr x25, [%x[output_ptr], #0x18]\n"
+ "ldr x23, [%x[output_ptr], #0x20]\n"
+ "add x27, x27, x19, LSL #2\n"
+ "add x25, x25, x19, LSL #2\n"
+ "add x23, x23, x19, LSL #2\n"
+ "b 51f\n"
+ "50:" // Height 5: setup direct output
+ "mov x13, %x[output_ptr]\n"
+ "add x9, x13, x19, LSL #2\n"
+ "add x27, x9, x19, LSL #2\n"
+ "add x25, x27, x19, LSL #2\n"
+ "add x23, x25, x19, LSL #2\n"
+ "51:" // Height 5: Column loop
+ "mov x19, #0x0\n"
+ "whilelt p4.s, x19, x15\n"
+ "incw x19\n"
+ "whilelt p3.s, x19, x15\n"
+ "incw x19\n"
+ "whilelt p2.s, x19, x15\n"
+ "incw x19\n"
+ "whilelt p1.s, x19, x15\n"
+ "tbz %x[flags], #0, 52f\n"
+ "ld1w { z8.s }, p4/Z, [x13]\n"
+ "ld1w { z9.s }, p3/Z, [x13, #1, MUL VL]\n"
+ "ld1w { z10.s }, p2/Z, [x13, #2, MUL VL]\n"
+ "ld1w { z11.s }, p1/Z, [x13, #3, MUL VL]\n"
+ "ld1w { z12.s }, p4/Z, [x9]\n"
+ "ld1w { z13.s }, p3/Z, [x9, #1, MUL VL]\n"
+ "ld1w { z14.s }, p2/Z, [x9, #2, MUL VL]\n"
+ "ld1w { z15.s }, p1/Z, [x9, #3, MUL VL]\n"
+ "ld1w { z16.s }, p4/Z, [x27]\n"
+ "ld1w { z17.s }, p3/Z, [x27, #1, MUL VL]\n"
+ "ld1w { z18.s }, p2/Z, [x27, #2, MUL VL]\n"
+ "ld1w { z19.s }, p1/Z, [x27, #3, MUL VL]\n"
+ "ld1w { z20.s }, p4/Z, [x25]\n"
+ "ld1w { z21.s }, p3/Z, [x25, #1, MUL VL]\n"
+ "ld1w { z22.s }, p2/Z, [x25, #2, MUL VL]\n"
+ "ld1w { z23.s }, p1/Z, [x25, #3, MUL VL]\n"
+ "ld1w { z24.s }, p4/Z, [x23]\n"
+ "ld1w { z25.s }, p3/Z, [x23, #1, MUL VL]\n"
+ "ld1w { z26.s }, p2/Z, [x23, #2, MUL VL]\n"
+ "ld1w { z27.s }, p1/Z, [x23, #3, MUL VL]\n"
+ "b 53f\n"
+ "52:" // Height 5: no accumulate
+ "mov z8.s, #0x0\n"
+ "mov z9.s, #0x0\n"
+ "mov z10.s, #0x0\n"
+ "mov z11.s, #0x0\n"
+ "mov z12.s, #0x0\n"
+ "mov z13.s, #0x0\n"
+ "mov z14.s, #0x0\n"
+ "mov z15.s, #0x0\n"
+ "mov z16.s, #0x0\n"
+ "mov z17.s, #0x0\n"
+ "mov z18.s, #0x0\n"
+ "mov z19.s, #0x0\n"
+ "mov z20.s, #0x0\n"
+ "mov z21.s, #0x0\n"
+ "mov z22.s, #0x0\n"
+ "mov z23.s, #0x0\n"
+ "mov z24.s, #0x0\n"
+ "mov z25.s, #0x0\n"
+ "mov z26.s, #0x0\n"
+ "mov z27.s, #0x0\n"
+ "53:" // Height 5: setup done
+ "mov x12, #0x0\n"
+ "54:" // Height 5: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w11, [x20, x12, LSL #0x2]\n"
+ "tbz %x[flags], #3, 55f\n"
+ "ldr x20, [%x[input_ptr], x12, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x10, [x20, #0x0]\n"
+ "ldr x28, [x20, #0x8]\n"
+ "ldr x26, [x20, #0x10]\n"
+ "ldr x24, [x20, #0x18]\n"
+ "ldr x22, [x20, #0x20]\n"
+ "cbnz x12, 56f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x10, x10, x19\n"
+ "add x28, x28, x19\n"
+ "add x26, x26, x19\n"
+ "add x24, x24, x19\n"
+ "add x22, x22, x19\n"
+ "b 56f\n"
+ "55:" // Height 5: setup direct input
+ "mov x10, %x[input_ptr]\n"
+ "add x28, x10, x19\n"
+ "add x26, x28, x19\n"
+ "add x24, x26, x19\n"
+ "add x22, x24, x19\n"
+ "56:" // Height 5: input setup done
+ "cmp x11, #0x10\n"
+ "ble 58f\n"
+ "57:" // Height 5: Multiply loop: Main loop head
+ "ld1b { z6.b }, p5/Z, [x14]\n"
+ "whilelt p0.b, XZR, x11\n"
+ "ld1b { z7.b }, p5/Z, [x14, #1, MUL VL]\n"
+ "sub x11, x11, #0x10\n"
+ "ld1rqb { z0.b }, p0/Z, [x10]\n"
+ "sdot z8.s, z6.b, z0.b[0]\n"
+ "ld1rqb { z1.b }, p0/Z, [x28]\n"
+ "add x10, x10, #0x10\n"
+ "sdot z9.s, z7.b, z0.b[0]\n"
+ "ld1rqb { z2.b }, p0/Z, [x26]\n"
+ "add x28, x28, #0x10\n"
+ "sdot z12.s, z6.b, z1.b[0]\n"
+ "ld1rqb { z3.b }, p0/Z, [x24]\n"
+ "add x26, x26, #0x10\n"
+ "sdot z16.s, z6.b, z2.b[0]\n"
+ "ld1rqb { z4.b }, p0/Z, [x22]\n"
+ "add x24, x24, #0x10\n"
+ "sdot z13.s, z7.b, z1.b[0]\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x22, x22, #0x10\n"
+ "sdot z20.s, z6.b, z3.b[0]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "cmp x11, #0x10\n"
+ "sdot z24.s, z6.b, z4.b[0]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #2, MUL VL]\n"
+ "sdot z17.s, z7.b, z2.b[0]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "sdot z21.s, z7.b, z3.b[0]\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "sdot z25.s, z7.b, z4.b[0]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #3, MUL VL]\n"
+ "sdot z10.s, z6.b, z0.b[0]\n"
+ "prfm pldl1keep, [x22, #0x80]\n"
+ "sdot z14.s, z6.b, z1.b[0]\n"
+ "sdot z18.s, z6.b, z2.b[0]\n"
+ "sdot z22.s, z6.b, z3.b[0]\n"
+ "sdot z26.s, z6.b, z4.b[0]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #4, MUL VL]\n"
+ "sdot z11.s, z7.b, z0.b[0]\n"
+ "sdot z15.s, z7.b, z1.b[0]\n"
+ "sdot z19.s, z7.b, z2.b[0]\n"
+ "sdot z23.s, z7.b, z3.b[0]\n"
+ "sdot z27.s, z7.b, z4.b[0]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #5, MUL VL]\n"
+ "sdot z8.s, z6.b, z0.b[1]\n"
+ "sdot z12.s, z6.b, z1.b[1]\n"
+ "sdot z16.s, z6.b, z2.b[1]\n"
+ "sdot z20.s, z6.b, z3.b[1]\n"
+ "sdot z24.s, z6.b, z4.b[1]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #6, MUL VL]\n"
+ "sdot z9.s, z7.b, z0.b[1]\n"
+ "sdot z13.s, z7.b, z1.b[1]\n"
+ "sdot z17.s, z7.b, z2.b[1]\n"
+ "sdot z21.s, z7.b, z3.b[1]\n"
+ "sdot z25.s, z7.b, z4.b[1]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #7, MUL VL]\n"
+ "addvl x14, x14, #16\n"
+ "sdot z10.s, z6.b, z0.b[1]\n"
+ "sdot z14.s, z6.b, z1.b[1]\n"
+ "sdot z18.s, z6.b, z2.b[1]\n"
+ "sdot z22.s, z6.b, z3.b[1]\n"
+ "sdot z26.s, z6.b, z4.b[1]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #-8, MUL VL]\n"
+ "sdot z11.s, z7.b, z0.b[1]\n"
+ "sdot z15.s, z7.b, z1.b[1]\n"
+ "sdot z19.s, z7.b, z2.b[1]\n"
+ "sdot z23.s, z7.b, z3.b[1]\n"
+ "sdot z27.s, z7.b, z4.b[1]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #-7, MUL VL]\n"
+ "sdot z8.s, z6.b, z0.b[2]\n"
+ "sdot z12.s, z6.b, z1.b[2]\n"
+ "sdot z16.s, z6.b, z2.b[2]\n"
+ "sdot z20.s, z6.b, z3.b[2]\n"
+ "sdot z24.s, z6.b, z4.b[2]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #-6, MUL VL]\n"
+ "sdot z9.s, z7.b, z0.b[2]\n"
+ "sdot z13.s, z7.b, z1.b[2]\n"
+ "sdot z17.s, z7.b, z2.b[2]\n"
+ "sdot z21.s, z7.b, z3.b[2]\n"
+ "sdot z25.s, z7.b, z4.b[2]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #-5, MUL VL]\n"
+ "sdot z10.s, z6.b, z0.b[2]\n"
+ "sdot z14.s, z6.b, z1.b[2]\n"
+ "sdot z18.s, z6.b, z2.b[2]\n"
+ "sdot z22.s, z6.b, z3.b[2]\n"
+ "sdot z26.s, z6.b, z4.b[2]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #-4, MUL VL]\n"
+ "sdot z11.s, z7.b, z0.b[2]\n"
+ "sdot z15.s, z7.b, z1.b[2]\n"
+ "sdot z19.s, z7.b, z2.b[2]\n"
+ "sdot z23.s, z7.b, z3.b[2]\n"
+ "sdot z27.s, z7.b, z4.b[2]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #-3, MUL VL]\n"
+ "sdot z8.s, z6.b, z0.b[3]\n"
+ "sdot z12.s, z6.b, z1.b[3]\n"
+ "sdot z16.s, z6.b, z2.b[3]\n"
+ "sdot z20.s, z6.b, z3.b[3]\n"
+ "sdot z24.s, z6.b, z4.b[3]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #-2, MUL VL]\n"
+ "sdot z9.s, z7.b, z0.b[3]\n"
+ "sdot z13.s, z7.b, z1.b[3]\n"
+ "sdot z17.s, z7.b, z2.b[3]\n"
+ "sdot z21.s, z7.b, z3.b[3]\n"
+ "sdot z25.s, z7.b, z4.b[3]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #-1, MUL VL]\n"
+ "sdot z10.s, z6.b, z0.b[3]\n"
+ "sdot z14.s, z6.b, z1.b[3]\n"
+ "sdot z18.s, z6.b, z2.b[3]\n"
+ "sdot z22.s, z6.b, z3.b[3]\n"
+ "sdot z26.s, z6.b, z4.b[3]\n"
+ "sdot z11.s, z7.b, z0.b[3]\n"
+ "sdot z15.s, z7.b, z1.b[3]\n"
+ "sdot z19.s, z7.b, z2.b[3]\n"
+ "sdot z23.s, z7.b, z3.b[3]\n"
+ "sdot z27.s, z7.b, z4.b[3]\n"
+ "bgt 57b\n"
+ "58:" // Height 5: Multiply loop: Single iteration only
+ "ld1b { z6.b }, p5/Z, [x14]\n"
+ "whilelt p0.b, XZR, x11\n"
+ "ld1b { z7.b }, p5/Z, [x14, #1, MUL VL]\n"
+ "subs x11, x11, #0x4\n"
+ "ld1rqb { z0.b }, p0/Z, [x10]\n"
+ "sdot z8.s, z6.b, z0.b[0]\n"
+ "ld1rqb { z1.b }, p0/Z, [x28]\n"
+ "add x10, x10, #0x10\n"
+ "sdot z9.s, z7.b, z0.b[0]\n"
+ "ld1rqb { z2.b }, p0/Z, [x26]\n"
+ "add x28, x28, #0x10\n"
+ "sdot z12.s, z6.b, z1.b[0]\n"
+ "ld1rqb { z3.b }, p0/Z, [x24]\n"
+ "add x26, x26, #0x10\n"
+ "sdot z16.s, z6.b, z2.b[0]\n"
+ "ld1rqb { z4.b }, p0/Z, [x22]\n"
+ "add x24, x24, #0x10\n"
+ "sdot z13.s, z7.b, z1.b[0]\n"
+ "add x22, x22, #0x10\n"
+ "sdot z17.s, z7.b, z2.b[0]\n"
+ "sdot z20.s, z6.b, z3.b[0]\n"
+ "sdot z24.s, z6.b, z4.b[0]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #2, MUL VL]\n"
+ "sdot z21.s, z7.b, z3.b[0]\n"
+ "sdot z25.s, z7.b, z4.b[0]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #3, MUL VL]\n"
+ "addvl x14, x14, #4\n"
+ "sdot z10.s, z6.b, z0.b[0]\n"
+ "sdot z14.s, z6.b, z1.b[0]\n"
+ "sdot z18.s, z6.b, z2.b[0]\n"
+ "sdot z22.s, z6.b, z3.b[0]\n"
+ "sdot z26.s, z6.b, z4.b[0]\n"
+ "sdot z11.s, z7.b, z0.b[0]\n"
+ "sdot z15.s, z7.b, z1.b[0]\n"
+ "sdot z19.s, z7.b, z2.b[0]\n"
+ "sdot z23.s, z7.b, z3.b[0]\n"
+ "sdot z27.s, z7.b, z4.b[0]\n"
+ "ble 59f\n"
+ "ld1b { z6.b }, p5/Z, [x14]\n"
+ "sdot z8.s, z6.b, z0.b[1]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #1, MUL VL]\n"
+ "subs x11, x11, #0x4\n"
+ "sdot z12.s, z6.b, z1.b[1]\n"
+ "sdot z16.s, z6.b, z2.b[1]\n"
+ "sdot z20.s, z6.b, z3.b[1]\n"
+ "sdot z24.s, z6.b, z4.b[1]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #2, MUL VL]\n"
+ "sdot z9.s, z7.b, z0.b[1]\n"
+ "sdot z13.s, z7.b, z1.b[1]\n"
+ "sdot z17.s, z7.b, z2.b[1]\n"
+ "sdot z21.s, z7.b, z3.b[1]\n"
+ "sdot z25.s, z7.b, z4.b[1]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #3, MUL VL]\n"
+ "addvl x14, x14, #4\n"
+ "sdot z10.s, z6.b, z0.b[1]\n"
+ "sdot z14.s, z6.b, z1.b[1]\n"
+ "sdot z18.s, z6.b, z2.b[1]\n"
+ "sdot z22.s, z6.b, z3.b[1]\n"
+ "sdot z26.s, z6.b, z4.b[1]\n"
+ "sdot z11.s, z7.b, z0.b[1]\n"
+ "sdot z15.s, z7.b, z1.b[1]\n"
+ "sdot z19.s, z7.b, z2.b[1]\n"
+ "sdot z23.s, z7.b, z3.b[1]\n"
+ "sdot z27.s, z7.b, z4.b[1]\n"
+ "ble 59f\n"
+ "ld1b { z6.b }, p5/Z, [x14]\n"
+ "sdot z8.s, z6.b, z0.b[2]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #1, MUL VL]\n"
+ "subs x11, x11, #0x4\n"
+ "sdot z12.s, z6.b, z1.b[2]\n"
+ "sdot z16.s, z6.b, z2.b[2]\n"
+ "sdot z20.s, z6.b, z3.b[2]\n"
+ "sdot z24.s, z6.b, z4.b[2]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #2, MUL VL]\n"
+ "sdot z9.s, z7.b, z0.b[2]\n"
+ "sdot z13.s, z7.b, z1.b[2]\n"
+ "sdot z17.s, z7.b, z2.b[2]\n"
+ "sdot z21.s, z7.b, z3.b[2]\n"
+ "sdot z25.s, z7.b, z4.b[2]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #3, MUL VL]\n"
+ "addvl x14, x14, #4\n"
+ "sdot z10.s, z6.b, z0.b[2]\n"
+ "sdot z14.s, z6.b, z1.b[2]\n"
+ "sdot z18.s, z6.b, z2.b[2]\n"
+ "sdot z22.s, z6.b, z3.b[2]\n"
+ "sdot z26.s, z6.b, z4.b[2]\n"
+ "sdot z11.s, z7.b, z0.b[2]\n"
+ "sdot z15.s, z7.b, z1.b[2]\n"
+ "sdot z19.s, z7.b, z2.b[2]\n"
+ "sdot z23.s, z7.b, z3.b[2]\n"
+ "sdot z27.s, z7.b, z4.b[2]\n"
+ "ble 59f\n"
+ "ld1b { z6.b }, p5/Z, [x14]\n"
+ "sdot z8.s, z6.b, z0.b[3]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #1, MUL VL]\n"
+ "sdot z12.s, z6.b, z1.b[3]\n"
+ "sdot z16.s, z6.b, z2.b[3]\n"
+ "sdot z20.s, z6.b, z3.b[3]\n"
+ "sdot z24.s, z6.b, z4.b[3]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #2, MUL VL]\n"
+ "sdot z9.s, z7.b, z0.b[3]\n"
+ "sdot z13.s, z7.b, z1.b[3]\n"
+ "sdot z17.s, z7.b, z2.b[3]\n"
+ "sdot z21.s, z7.b, z3.b[3]\n"
+ "sdot z25.s, z7.b, z4.b[3]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #3, MUL VL]\n"
+ "addvl x14, x14, #4\n"
+ "sdot z10.s, z6.b, z0.b[3]\n"
+ "sdot z14.s, z6.b, z1.b[3]\n"
+ "sdot z18.s, z6.b, z2.b[3]\n"
+ "sdot z22.s, z6.b, z3.b[3]\n"
+ "sdot z26.s, z6.b, z4.b[3]\n"
+ "sdot z11.s, z7.b, z0.b[3]\n"
+ "sdot z15.s, z7.b, z1.b[3]\n"
+ "sdot z19.s, z7.b, z2.b[3]\n"
+ "sdot z23.s, z7.b, z3.b[3]\n"
+ "sdot z27.s, z7.b, z4.b[3]\n"
+ "59:" // Height 5: Multiply loop: multiply skip
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x12, x12, #0x1\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "prfm pldl1keep, [x22, #0x80]\n"
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "cmp x12, x19\n"
+ "bne 54b\n"
+ "prfm pstl1keep, [x13, #0x0]\n"
+ "prfm pstl1keep, [x9, #0x0]\n"
+ "prfm pstl1keep, [x27, #0x0]\n"
+ "prfm pstl1keep, [x25, #0x0]\n"
+ "prfm pstl1keep, [x23, #0x0]\n"
+ "st1w { z8.s }, p4, [x13]\n"
+ "st1w { z9.s }, p3, [x13, #1, MUL VL]\n"
+ "st1w { z10.s }, p2, [x13, #2, MUL VL]\n"
+ "st1w { z11.s }, p1, [x13, #3, MUL VL]\n"
+ "addvl x13, x13, #4\n"
+ "st1w { z12.s }, p4, [x9]\n"
+ "st1w { z13.s }, p3, [x9, #1, MUL VL]\n"
+ "st1w { z14.s }, p2, [x9, #2, MUL VL]\n"
+ "st1w { z15.s }, p1, [x9, #3, MUL VL]\n"
+ "addvl x9, x9, #4\n"
+ "st1w { z16.s }, p4, [x27]\n"
+ "st1w { z17.s }, p3, [x27, #1, MUL VL]\n"
+ "st1w { z18.s }, p2, [x27, #2, MUL VL]\n"
+ "st1w { z19.s }, p1, [x27, #3, MUL VL]\n"
+ "addvl x27, x27, #4\n"
+ "st1w { z20.s }, p4, [x25]\n"
+ "st1w { z21.s }, p3, [x25, #1, MUL VL]\n"
+ "st1w { z22.s }, p2, [x25, #2, MUL VL]\n"
+ "st1w { z23.s }, p1, [x25, #3, MUL VL]\n"
+ "addvl x25, x25, #4\n"
+ "st1w { z24.s }, p4, [x23]\n"
+ "st1w { z25.s }, p3, [x23, #1, MUL VL]\n"
+ "st1w { z26.s }, p2, [x23, #2, MUL VL]\n"
+ "st1w { z27.s }, p1, [x23, #3, MUL VL]\n"
+ "addvl x23, x23, #4\n"
+ "60:" // Height 5: Writeback done
+ "mov x19, #0x0\n"
+ "incw x19, ALL, MUL #4\n"
+ "subs x15, x15, x19\n"
+ "bgt 51b\n"
+ "b 74f\n"
+ "61:" // Height 6
+ "ldr x15, [%x[args_ptr], %[offsetof_N]]\n"
+ "ldr x14, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 62f\n"
+ "ldr x13, [%x[output_ptr], #0x0]\n"
+ "add x13, x13, x19, LSL #2\n"
+ "ldr x9, [%x[output_ptr], #0x8]\n"
+ "ldr x27, [%x[output_ptr], #0x10]\n"
+ "add x9, x9, x19, LSL #2\n"
+ "ldr x25, [%x[output_ptr], #0x18]\n"
+ "ldr x23, [%x[output_ptr], #0x20]\n"
+ "add x27, x27, x19, LSL #2\n"
+ "ldr x21, [%x[output_ptr], #0x28]\n"
+ "add %x[output_ptr], %x[output_ptr], #0x30\n"
+ "add x25, x25, x19, LSL #2\n"
+ "add x23, x23, x19, LSL #2\n"
+ "add x21, x21, x19, LSL #2\n"
+ "b 63f\n"
+ "62:" // Height 6: setup direct output
+ "mov x13, %x[output_ptr]\n"
+ "add x9, x13, x19, LSL #2\n"
+ "add x27, x9, x19, LSL #2\n"
+ "add x25, x27, x19, LSL #2\n"
+ "add x23, x25, x19, LSL #2\n"
+ "add x21, x23, x19, LSL #2\n"
+ "add %x[output_ptr], x21, x19, LSL #2\n"
+ "63:" // Height 6: Column loop
+ "mov x19, #0x0\n"
+ "whilelt p4.s, x19, x15\n"
+ "incw x19\n"
+ "whilelt p3.s, x19, x15\n"
+ "incw x19\n"
+ "whilelt p2.s, x19, x15\n"
+ "incw x19\n"
+ "whilelt p1.s, x19, x15\n"
+ "tbz %x[flags], #0, 64f\n"
+ "ld1w { z8.s }, p4/Z, [x13]\n"
+ "ld1w { z9.s }, p3/Z, [x13, #1, MUL VL]\n"
+ "ld1w { z10.s }, p2/Z, [x13, #2, MUL VL]\n"
+ "ld1w { z11.s }, p1/Z, [x13, #3, MUL VL]\n"
+ "ld1w { z12.s }, p4/Z, [x9]\n"
+ "ld1w { z13.s }, p3/Z, [x9, #1, MUL VL]\n"
+ "ld1w { z14.s }, p2/Z, [x9, #2, MUL VL]\n"
+ "ld1w { z15.s }, p1/Z, [x9, #3, MUL VL]\n"
+ "ld1w { z16.s }, p4/Z, [x27]\n"
+ "ld1w { z17.s }, p3/Z, [x27, #1, MUL VL]\n"
+ "ld1w { z18.s }, p2/Z, [x27, #2, MUL VL]\n"
+ "ld1w { z19.s }, p1/Z, [x27, #3, MUL VL]\n"
+ "ld1w { z20.s }, p4/Z, [x25]\n"
+ "ld1w { z21.s }, p3/Z, [x25, #1, MUL VL]\n"
+ "ld1w { z22.s }, p2/Z, [x25, #2, MUL VL]\n"
+ "ld1w { z23.s }, p1/Z, [x25, #3, MUL VL]\n"
+ "ld1w { z24.s }, p4/Z, [x23]\n"
+ "ld1w { z25.s }, p3/Z, [x23, #1, MUL VL]\n"
+ "ld1w { z26.s }, p2/Z, [x23, #2, MUL VL]\n"
+ "ld1w { z27.s }, p1/Z, [x23, #3, MUL VL]\n"
+ "ld1w { z28.s }, p4/Z, [x21]\n"
+ "ld1w { z29.s }, p3/Z, [x21, #1, MUL VL]\n"
+ "ld1w { z30.s }, p2/Z, [x21, #2, MUL VL]\n"
+ "ld1w { z31.s }, p1/Z, [x21, #3, MUL VL]\n"
+ "b 65f\n"
+ "64:" // Height 6: no accumulate
+ "mov z8.s, #0x0\n"
+ "mov z9.s, #0x0\n"
+ "mov z10.s, #0x0\n"
+ "mov z11.s, #0x0\n"
+ "mov z12.s, #0x0\n"
+ "mov z13.s, #0x0\n"
+ "mov z14.s, #0x0\n"
+ "mov z15.s, #0x0\n"
+ "mov z16.s, #0x0\n"
+ "mov z17.s, #0x0\n"
+ "mov z18.s, #0x0\n"
+ "mov z19.s, #0x0\n"
+ "mov z20.s, #0x0\n"
+ "mov z21.s, #0x0\n"
+ "mov z22.s, #0x0\n"
+ "mov z23.s, #0x0\n"
+ "mov z24.s, #0x0\n"
+ "mov z25.s, #0x0\n"
+ "mov z26.s, #0x0\n"
+ "mov z27.s, #0x0\n"
+ "mov z28.s, #0x0\n"
+ "mov z29.s, #0x0\n"
+ "mov z30.s, #0x0\n"
+ "mov z31.s, #0x0\n"
+ "65:" // Height 6: setup done
+ "mov x12, #0x0\n"
+ "66:" // Height 6: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w11, [x20, x12, LSL #0x2]\n"
+ "tbz %x[flags], #3, 67f\n"
+ "ldr x20, [%x[input_ptr], x12, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x10, [x20, #0x0]\n"
+ "ldr x28, [x20, #0x8]\n"
+ "ldr x26, [x20, #0x10]\n"
+ "ldr x24, [x20, #0x18]\n"
+ "ldr x22, [x20, #0x20]\n"
+ "ldr x20, [x20, #0x28]\n"
+ "cbnz x12, 68f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x10, x10, x19\n"
+ "add x28, x28, x19\n"
+ "add x26, x26, x19\n"
+ "add x24, x24, x19\n"
+ "add x22, x22, x19\n"
+ "add x20, x20, x19\n"
+ "b 68f\n"
+ "67:" // Height 6: setup direct input
+ "mov x10, %x[input_ptr]\n"
+ "add x28, x10, x19\n"
+ "add x26, x28, x19\n"
+ "add x24, x26, x19\n"
+ "add x22, x24, x19\n"
+ "add x20, x22, x19\n"
+ "68:" // Height 6: input setup done
+ "cmp x11, #0x10\n"
+ "ble 70f\n"
+ "69:" // Height 6: Multiply loop: Main loop head
+ "ld1b { z6.b }, p5/Z, [x14]\n"
+ "whilelt p0.b, XZR, x11\n"
+ "ld1b { z7.b }, p5/Z, [x14, #1, MUL VL]\n"
+ "sub x11, x11, #0x10\n"
+ "ld1rqb { z0.b }, p0/Z, [x10]\n"
+ "sdot z8.s, z6.b, z0.b[0]\n"
+ "ld1rqb { z1.b }, p0/Z, [x28]\n"
+ "add x10, x10, #0x10\n"
+ "sdot z9.s, z7.b, z0.b[0]\n"
+ "ld1rqb { z2.b }, p0/Z, [x26]\n"
+ "add x28, x28, #0x10\n"
+ "sdot z12.s, z6.b, z1.b[0]\n"
+ "ld1rqb { z3.b }, p0/Z, [x24]\n"
+ "add x26, x26, #0x10\n"
+ "sdot z16.s, z6.b, z2.b[0]\n"
+ "ld1rqb { z4.b }, p0/Z, [x22]\n"
+ "add x24, x24, #0x10\n"
+ "sdot z13.s, z7.b, z1.b[0]\n"
+ "ld1rqb { z5.b }, p0/Z, [x20]\n"
+ "add x22, x22, #0x10\n"
+ "sdot z20.s, z6.b, z3.b[0]\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x20, x20, #0x10\n"
+ "sdot z24.s, z6.b, z4.b[0]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "cmp x11, #0x10\n"
+ "sdot z28.s, z6.b, z5.b[0]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #2, MUL VL]\n"
+ "sdot z17.s, z7.b, z2.b[0]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "sdot z21.s, z7.b, z3.b[0]\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "sdot z25.s, z7.b, z4.b[0]\n"
+ "prfm pldl1keep, [x22, #0x80]\n"
+ "sdot z29.s, z7.b, z5.b[0]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #3, MUL VL]\n"
+ "sdot z10.s, z6.b, z0.b[0]\n"
+ "prfm pldl1keep, [x20, #0x80]\n"
+ "sdot z14.s, z6.b, z1.b[0]\n"
+ "sdot z18.s, z6.b, z2.b[0]\n"
+ "sdot z22.s, z6.b, z3.b[0]\n"
+ "sdot z26.s, z6.b, z4.b[0]\n"
+ "sdot z30.s, z6.b, z5.b[0]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #4, MUL VL]\n"
+ "sdot z11.s, z7.b, z0.b[0]\n"
+ "sdot z15.s, z7.b, z1.b[0]\n"
+ "sdot z19.s, z7.b, z2.b[0]\n"
+ "sdot z23.s, z7.b, z3.b[0]\n"
+ "sdot z27.s, z7.b, z4.b[0]\n"
+ "sdot z31.s, z7.b, z5.b[0]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #5, MUL VL]\n"
+ "sdot z8.s, z6.b, z0.b[1]\n"
+ "sdot z12.s, z6.b, z1.b[1]\n"
+ "sdot z16.s, z6.b, z2.b[1]\n"
+ "sdot z20.s, z6.b, z3.b[1]\n"
+ "sdot z24.s, z6.b, z4.b[1]\n"
+ "sdot z28.s, z6.b, z5.b[1]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #6, MUL VL]\n"
+ "sdot z9.s, z7.b, z0.b[1]\n"
+ "sdot z13.s, z7.b, z1.b[1]\n"
+ "sdot z17.s, z7.b, z2.b[1]\n"
+ "sdot z21.s, z7.b, z3.b[1]\n"
+ "sdot z25.s, z7.b, z4.b[1]\n"
+ "sdot z29.s, z7.b, z5.b[1]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #7, MUL VL]\n"
+ "addvl x14, x14, #16\n"
+ "sdot z10.s, z6.b, z0.b[1]\n"
+ "sdot z14.s, z6.b, z1.b[1]\n"
+ "sdot z18.s, z6.b, z2.b[1]\n"
+ "sdot z22.s, z6.b, z3.b[1]\n"
+ "sdot z26.s, z6.b, z4.b[1]\n"
+ "sdot z30.s, z6.b, z5.b[1]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #-8, MUL VL]\n"
+ "sdot z11.s, z7.b, z0.b[1]\n"
+ "sdot z15.s, z7.b, z1.b[1]\n"
+ "sdot z19.s, z7.b, z2.b[1]\n"
+ "sdot z23.s, z7.b, z3.b[1]\n"
+ "sdot z27.s, z7.b, z4.b[1]\n"
+ "sdot z31.s, z7.b, z5.b[1]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #-7, MUL VL]\n"
+ "sdot z8.s, z6.b, z0.b[2]\n"
+ "sdot z12.s, z6.b, z1.b[2]\n"
+ "sdot z16.s, z6.b, z2.b[2]\n"
+ "sdot z20.s, z6.b, z3.b[2]\n"
+ "sdot z24.s, z6.b, z4.b[2]\n"
+ "sdot z28.s, z6.b, z5.b[2]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #-6, MUL VL]\n"
+ "sdot z9.s, z7.b, z0.b[2]\n"
+ "sdot z13.s, z7.b, z1.b[2]\n"
+ "sdot z17.s, z7.b, z2.b[2]\n"
+ "sdot z21.s, z7.b, z3.b[2]\n"
+ "sdot z25.s, z7.b, z4.b[2]\n"
+ "sdot z29.s, z7.b, z5.b[2]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #-5, MUL VL]\n"
+ "sdot z10.s, z6.b, z0.b[2]\n"
+ "sdot z14.s, z6.b, z1.b[2]\n"
+ "sdot z18.s, z6.b, z2.b[2]\n"
+ "sdot z22.s, z6.b, z3.b[2]\n"
+ "sdot z26.s, z6.b, z4.b[2]\n"
+ "sdot z30.s, z6.b, z5.b[2]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #-4, MUL VL]\n"
+ "sdot z11.s, z7.b, z0.b[2]\n"
+ "sdot z15.s, z7.b, z1.b[2]\n"
+ "sdot z19.s, z7.b, z2.b[2]\n"
+ "sdot z23.s, z7.b, z3.b[2]\n"
+ "sdot z27.s, z7.b, z4.b[2]\n"
+ "sdot z31.s, z7.b, z5.b[2]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #-3, MUL VL]\n"
+ "sdot z8.s, z6.b, z0.b[3]\n"
+ "sdot z12.s, z6.b, z1.b[3]\n"
+ "sdot z16.s, z6.b, z2.b[3]\n"
+ "sdot z20.s, z6.b, z3.b[3]\n"
+ "sdot z24.s, z6.b, z4.b[3]\n"
+ "sdot z28.s, z6.b, z5.b[3]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #-2, MUL VL]\n"
+ "sdot z9.s, z7.b, z0.b[3]\n"
+ "sdot z13.s, z7.b, z1.b[3]\n"
+ "sdot z17.s, z7.b, z2.b[3]\n"
+ "sdot z21.s, z7.b, z3.b[3]\n"
+ "sdot z25.s, z7.b, z4.b[3]\n"
+ "sdot z29.s, z7.b, z5.b[3]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #-1, MUL VL]\n"
+ "sdot z10.s, z6.b, z0.b[3]\n"
+ "sdot z14.s, z6.b, z1.b[3]\n"
+ "sdot z18.s, z6.b, z2.b[3]\n"
+ "sdot z22.s, z6.b, z3.b[3]\n"
+ "sdot z26.s, z6.b, z4.b[3]\n"
+ "sdot z30.s, z6.b, z5.b[3]\n"
+ "sdot z11.s, z7.b, z0.b[3]\n"
+ "sdot z15.s, z7.b, z1.b[3]\n"
+ "sdot z19.s, z7.b, z2.b[3]\n"
+ "sdot z23.s, z7.b, z3.b[3]\n"
+ "sdot z27.s, z7.b, z4.b[3]\n"
+ "sdot z31.s, z7.b, z5.b[3]\n"
+ "bgt 69b\n"
+ "70:" // Height 6: Multiply loop: Single iteration only
+ "ld1b { z6.b }, p5/Z, [x14]\n"
+ "whilelt p0.b, XZR, x11\n"
+ "ld1b { z7.b }, p5/Z, [x14, #1, MUL VL]\n"
+ "subs x11, x11, #0x4\n"
+ "ld1rqb { z0.b }, p0/Z, [x10]\n"
+ "sdot z8.s, z6.b, z0.b[0]\n"
+ "ld1rqb { z1.b }, p0/Z, [x28]\n"
+ "add x10, x10, #0x10\n"
+ "sdot z9.s, z7.b, z0.b[0]\n"
+ "ld1rqb { z2.b }, p0/Z, [x26]\n"
+ "add x28, x28, #0x10\n"
+ "sdot z12.s, z6.b, z1.b[0]\n"
+ "ld1rqb { z3.b }, p0/Z, [x24]\n"
+ "add x26, x26, #0x10\n"
+ "sdot z16.s, z6.b, z2.b[0]\n"
+ "ld1rqb { z4.b }, p0/Z, [x22]\n"
+ "add x24, x24, #0x10\n"
+ "sdot z13.s, z7.b, z1.b[0]\n"
+ "ld1rqb { z5.b }, p0/Z, [x20]\n"
+ "add x22, x22, #0x10\n"
+ "sdot z20.s, z6.b, z3.b[0]\n"
+ "add x20, x20, #0x10\n"
+ "sdot z17.s, z7.b, z2.b[0]\n"
+ "sdot z24.s, z6.b, z4.b[0]\n"
+ "sdot z28.s, z6.b, z5.b[0]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #2, MUL VL]\n"
+ "sdot z21.s, z7.b, z3.b[0]\n"
+ "sdot z25.s, z7.b, z4.b[0]\n"
+ "sdot z29.s, z7.b, z5.b[0]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #3, MUL VL]\n"
+ "addvl x14, x14, #4\n"
+ "sdot z10.s, z6.b, z0.b[0]\n"
+ "sdot z14.s, z6.b, z1.b[0]\n"
+ "sdot z18.s, z6.b, z2.b[0]\n"
+ "sdot z22.s, z6.b, z3.b[0]\n"
+ "sdot z26.s, z6.b, z4.b[0]\n"
+ "sdot z30.s, z6.b, z5.b[0]\n"
+ "sdot z11.s, z7.b, z0.b[0]\n"
+ "sdot z15.s, z7.b, z1.b[0]\n"
+ "sdot z19.s, z7.b, z2.b[0]\n"
+ "sdot z23.s, z7.b, z3.b[0]\n"
+ "sdot z27.s, z7.b, z4.b[0]\n"
+ "sdot z31.s, z7.b, z5.b[0]\n"
+ "ble 71f\n"
+ "ld1b { z6.b }, p5/Z, [x14]\n"
+ "sdot z8.s, z6.b, z0.b[1]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #1, MUL VL]\n"
+ "subs x11, x11, #0x4\n"
+ "sdot z12.s, z6.b, z1.b[1]\n"
+ "sdot z16.s, z6.b, z2.b[1]\n"
+ "sdot z20.s, z6.b, z3.b[1]\n"
+ "sdot z24.s, z6.b, z4.b[1]\n"
+ "sdot z28.s, z6.b, z5.b[1]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #2, MUL VL]\n"
+ "sdot z9.s, z7.b, z0.b[1]\n"
+ "sdot z13.s, z7.b, z1.b[1]\n"
+ "sdot z17.s, z7.b, z2.b[1]\n"
+ "sdot z21.s, z7.b, z3.b[1]\n"
+ "sdot z25.s, z7.b, z4.b[1]\n"
+ "sdot z29.s, z7.b, z5.b[1]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #3, MUL VL]\n"
+ "addvl x14, x14, #4\n"
+ "sdot z10.s, z6.b, z0.b[1]\n"
+ "sdot z14.s, z6.b, z1.b[1]\n"
+ "sdot z18.s, z6.b, z2.b[1]\n"
+ "sdot z22.s, z6.b, z3.b[1]\n"
+ "sdot z26.s, z6.b, z4.b[1]\n"
+ "sdot z30.s, z6.b, z5.b[1]\n"
+ "sdot z11.s, z7.b, z0.b[1]\n"
+ "sdot z15.s, z7.b, z1.b[1]\n"
+ "sdot z19.s, z7.b, z2.b[1]\n"
+ "sdot z23.s, z7.b, z3.b[1]\n"
+ "sdot z27.s, z7.b, z4.b[1]\n"
+ "sdot z31.s, z7.b, z5.b[1]\n"
+ "ble 71f\n"
+ "ld1b { z6.b }, p5/Z, [x14]\n"
+ "sdot z8.s, z6.b, z0.b[2]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #1, MUL VL]\n"
+ "subs x11, x11, #0x4\n"
+ "sdot z12.s, z6.b, z1.b[2]\n"
+ "sdot z16.s, z6.b, z2.b[2]\n"
+ "sdot z20.s, z6.b, z3.b[2]\n"
+ "sdot z24.s, z6.b, z4.b[2]\n"
+ "sdot z28.s, z6.b, z5.b[2]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #2, MUL VL]\n"
+ "sdot z9.s, z7.b, z0.b[2]\n"
+ "sdot z13.s, z7.b, z1.b[2]\n"
+ "sdot z17.s, z7.b, z2.b[2]\n"
+ "sdot z21.s, z7.b, z3.b[2]\n"
+ "sdot z25.s, z7.b, z4.b[2]\n"
+ "sdot z29.s, z7.b, z5.b[2]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #3, MUL VL]\n"
+ "addvl x14, x14, #4\n"
+ "sdot z10.s, z6.b, z0.b[2]\n"
+ "sdot z14.s, z6.b, z1.b[2]\n"
+ "sdot z18.s, z6.b, z2.b[2]\n"
+ "sdot z22.s, z6.b, z3.b[2]\n"
+ "sdot z26.s, z6.b, z4.b[2]\n"
+ "sdot z30.s, z6.b, z5.b[2]\n"
+ "sdot z11.s, z7.b, z0.b[2]\n"
+ "sdot z15.s, z7.b, z1.b[2]\n"
+ "sdot z19.s, z7.b, z2.b[2]\n"
+ "sdot z23.s, z7.b, z3.b[2]\n"
+ "sdot z27.s, z7.b, z4.b[2]\n"
+ "sdot z31.s, z7.b, z5.b[2]\n"
+ "ble 71f\n"
+ "ld1b { z6.b }, p5/Z, [x14]\n"
+ "sdot z8.s, z6.b, z0.b[3]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #1, MUL VL]\n"
+ "sdot z12.s, z6.b, z1.b[3]\n"
+ "sdot z16.s, z6.b, z2.b[3]\n"
+ "sdot z20.s, z6.b, z3.b[3]\n"
+ "sdot z24.s, z6.b, z4.b[3]\n"
+ "sdot z28.s, z6.b, z5.b[3]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #2, MUL VL]\n"
+ "sdot z9.s, z7.b, z0.b[3]\n"
+ "sdot z13.s, z7.b, z1.b[3]\n"
+ "sdot z17.s, z7.b, z2.b[3]\n"
+ "sdot z21.s, z7.b, z3.b[3]\n"
+ "sdot z25.s, z7.b, z4.b[3]\n"
+ "sdot z29.s, z7.b, z5.b[3]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #3, MUL VL]\n"
+ "addvl x14, x14, #4\n"
+ "sdot z10.s, z6.b, z0.b[3]\n"
+ "sdot z14.s, z6.b, z1.b[3]\n"
+ "sdot z18.s, z6.b, z2.b[3]\n"
+ "sdot z22.s, z6.b, z3.b[3]\n"
+ "sdot z26.s, z6.b, z4.b[3]\n"
+ "sdot z30.s, z6.b, z5.b[3]\n"
+ "sdot z11.s, z7.b, z0.b[3]\n"
+ "sdot z15.s, z7.b, z1.b[3]\n"
+ "sdot z19.s, z7.b, z2.b[3]\n"
+ "sdot z23.s, z7.b, z3.b[3]\n"
+ "sdot z27.s, z7.b, z4.b[3]\n"
+ "sdot z31.s, z7.b, z5.b[3]\n"
+ "71:" // Height 6: Multiply loop: multiply skip
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x12, x12, #0x1\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "prfm pldl1keep, [x22, #0x80]\n"
+ "prfm pldl1keep, [x20, #0x80]\n"
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "cmp x12, x19\n"
+ "bne 66b\n"
+ "prfm pstl1keep, [x13, #0x0]\n"
+ "prfm pstl1keep, [x9, #0x0]\n"
+ "prfm pstl1keep, [x27, #0x0]\n"
+ "prfm pstl1keep, [x25, #0x0]\n"
+ "prfm pstl1keep, [x23, #0x0]\n"
+ "prfm pstl1keep, [x21, #0x0]\n"
+ "st1w { z8.s }, p4, [x13]\n"
+ "st1w { z9.s }, p3, [x13, #1, MUL VL]\n"
+ "st1w { z10.s }, p2, [x13, #2, MUL VL]\n"
+ "st1w { z11.s }, p1, [x13, #3, MUL VL]\n"
+ "addvl x13, x13, #4\n"
+ "st1w { z12.s }, p4, [x9]\n"
+ "st1w { z13.s }, p3, [x9, #1, MUL VL]\n"
+ "st1w { z14.s }, p2, [x9, #2, MUL VL]\n"
+ "st1w { z15.s }, p1, [x9, #3, MUL VL]\n"
+ "addvl x9, x9, #4\n"
+ "st1w { z16.s }, p4, [x27]\n"
+ "st1w { z17.s }, p3, [x27, #1, MUL VL]\n"
+ "st1w { z18.s }, p2, [x27, #2, MUL VL]\n"
+ "st1w { z19.s }, p1, [x27, #3, MUL VL]\n"
+ "addvl x27, x27, #4\n"
+ "st1w { z20.s }, p4, [x25]\n"
+ "st1w { z21.s }, p3, [x25, #1, MUL VL]\n"
+ "st1w { z22.s }, p2, [x25, #2, MUL VL]\n"
+ "st1w { z23.s }, p1, [x25, #3, MUL VL]\n"
+ "addvl x25, x25, #4\n"
+ "st1w { z24.s }, p4, [x23]\n"
+ "st1w { z25.s }, p3, [x23, #1, MUL VL]\n"
+ "st1w { z26.s }, p2, [x23, #2, MUL VL]\n"
+ "st1w { z27.s }, p1, [x23, #3, MUL VL]\n"
+ "addvl x23, x23, #4\n"
+ "st1w { z28.s }, p4, [x21]\n"
+ "st1w { z29.s }, p3, [x21, #1, MUL VL]\n"
+ "st1w { z30.s }, p2, [x21, #2, MUL VL]\n"
+ "st1w { z31.s }, p1, [x21, #3, MUL VL]\n"
+ "addvl x21, x21, #4\n"
+ "72:" // Height 6: Writeback done
+ "mov x19, #0x0\n"
+ "incw x19, ALL, MUL #4\n"
+ "subs x15, x15, x19\n"
+ "bgt 63b\n"
+ "subs %x[M], %x[M], #0x6\n"
+ "beq 74f\n"
+ "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "tbz %x[flags], #3, 73f\n"
+ "add x20, x20, #0x6\n"
+ "str x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "b 1b\n"
+ "73:" // Update direct input
+ "mov x19, #0x6\n"
+ "madd %x[input_ptr], x19, x20, %x[input_ptr]\n"
+ "b 1b\n"
+ "74:" // Exit
+
+ : [M] "+r" (M), [input_ptr] "+r" (input_ptr), [output_ptr] "+r" (output_ptr)
+ : [args_ptr] "r" (&ka), [flags] "r" (flags), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths))
+ : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+ );
+}
+
+} // namespace arm_gemm
+#endif // __ARM_FEATURE_SVE
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8u32_dot_4VLx4.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8qa_dot_4x4VL.hpp
similarity index 66%
rename from src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8u32_dot_4VLx4.hpp
rename to src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8qa_dot_4x4VL.hpp
index c325e52..964f7cc 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8u32_dot_4VLx4.hpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8qa_dot_4x4VL.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2018-2020 Arm Limited.
+ * Copyright (c) 2019-2020 Arm Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -10,37 +10,43 @@
* sell copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
*/
#pragma once
-
#ifdef __ARM_FEATURE_SVE
-#include <cstdint>
#include "../std_transforms_sve.hpp"
+#define ARGLIST \
+ unsigned int, const unsigned int *, \
+ IndirectInputArg<uint8_t>, \
+ size_t, size_t, \
+ const uint8_t *, \
+ IndirectOutputArg<uint8_t>, \
+ const Requantize32 *, const int32_t *, unsigned int
+
namespace arm_gemm
{
// Actual kernel implementations
-void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *, int, const uint8_t *, uint32_t *, int, int, int, int, const uint32_t *, Activation, bool);
+void sve_hybrid_u8qa_dot_4x4VL( ARGLIST );
-class hybrid_u8u32_dot_4VLx4
+class cls_sve_hybrid_u8qa_dot_4x4VL
{
public:
typedef uint8_t operand_type;
- typedef uint32_t result_type;
+ typedef uint8_t result_type;
- typedef void (*kern_type)(const uint8_t *, int, const uint8_t *, uint32_t *, int, int, int, int, const uint32_t *, Activation, bool);
+ typedef void (*kern_type)( ARGLIST );
/* Kernel blocking parameters */
static constexpr unsigned int out_height()
@@ -60,30 +66,20 @@
static constexpr bool supports_accumulate()
{
- return true;
- }
-
- static constexpr bool supports_bias()
- {
- return false;
- }
-
- static constexpr bool supports_activation()
- {
return false;
}
StdTransformsSVE<operand_type, result_type, 4, 4, 4> transforms = {};
// Default to the generic kernel
- kern_type kernel=sve_hybrid_u8u32_dot_4VLx4;
+ kern_type kernel=sve_hybrid_u8qa_dot_4x4VL;
- hybrid_u8u32_dot_4VLx4(const CPUInfo *)
+ cls_sve_hybrid_u8qa_dot_4x4VL(const CPUInfo *)
{
-
}
};
} // namespace arm_gemm
+#undef ARGLIST
#endif // __ARM_FEATURE_SVE
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8qa_dot_4x4VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8qa_dot_4x4VL/generic.cpp
new file mode 100644
index 0000000..0a6546b
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8qa_dot_4x4VL/generic.cpp
@@ -0,0 +1,1602 @@
+/*
+ * Copyright (c) 2019-2020 Arm Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+#ifdef __ARM_FEATURE_SVE
+
+#include "arm_gemm.hpp"
+#include "../../utils.hpp"
+
+#include <cassert>
+#include <cstdint>
+
+namespace arm_gemm {
+
+void sve_hybrid_u8qa_dot_4x4VL (
+ unsigned int num_strings, const unsigned int *string_lengths, IndirectInputArg<uint8_t> A_arg,
+ size_t M, size_t N, const uint8_t *B_ptr, IndirectOutputArg<uint8_t> output_arg,
+ const Requantize32 *qp, const int32_t *col_bias, unsigned int
+)
+{
+ struct KernelArgs {
+ unsigned int num_strings = {};
+ const unsigned int *string_lengths = {};
+ size_t N = {};
+ const uint8_t *B_ptr = {};
+ size_t output_offset = {};
+ size_t input_initial_col = {};
+ size_t input_offset = {};
+ } ka;
+
+ unsigned long flags=0;
+ void *output_ptr;
+ void *input_ptr;
+
+ if (output_arg.is_indirect) {
+ output_ptr=(void *)(output_arg.indirect.ptr);
+ ka.output_offset=output_arg.indirect.offset;
+ flags |= 0x4;
+ } else {
+ output_ptr=(void *)(output_arg.direct.base);
+ ka.output_offset=output_arg.direct.stride;
+ }
+
+ if (A_arg.is_indirect) {
+ input_ptr=(void *)(A_arg.indirect.ptr);
+ ka.input_offset=A_arg.indirect.start_row;
+ ka.input_initial_col=A_arg.indirect.start_col;
+ flags |= 0x8;
+ } else {
+ assert(num_strings==1);
+ input_ptr=(void *)(A_arg.direct.base);
+ ka.input_offset=A_arg.direct.stride;
+ }
+ ka.num_strings = num_strings;
+ ka.string_lengths = string_lengths;
+ ka.N = N;
+ ka.B_ptr = B_ptr;
+ if (qp->c_offset > qp->minval) {
+ flags |= 0x20;
+ }
+ __asm__ __volatile__(
+ "ptrue p2.b\n"
+ "1:" // Row loop
+ "cmp %x[M], #0x4\n"
+ "bge 46f\n"
+ "cmp %x[M], #0x2\n"
+ "bgt 31f\n"
+ "beq 16f\n"
+ "mov z11.s, #0x0\n"
+ "ldr x12, [%x[args_ptr], %[offsetof_N]]\n"
+ "mov z12.s, #0x0\n"
+ "ldr x11, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "mov x10, %x[col_bias]\n"
+ "mov z13.s, #0x0\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "bic %x[flags], %x[flags], #0x80000000\n"
+ "mov z14.s, #0x0\n"
+ "mov z15.b, #0x1\n"
+ "tbz %x[flags], #2, 2f\n"
+ "ldr x9, [%x[output_ptr], #0x0]\n"
+ "add x9, x9, x19\n"
+ "b 3f\n"
+ "2:" // Height 1: setup direct output
+ "mov x9, %x[output_ptr]\n"
+ "3:" // Height 1: Column loop
+ "mov z16.s, #0x0\n"
+ "mov x19, #0x0\n"
+ "mov z17.s, #0x0\n"
+ "whilelt p1.b, x19, x12\n"
+ "mov z18.s, #0x0\n"
+ "mov z19.s, #0x0\n"
+ "4:" // Height 1: setup done
+ "mov x28, #0x0\n"
+ "5:" // Height 1: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w27, [x20, x28, LSL #0x2]\n"
+ "tbz %x[flags], #3, 6f\n"
+ "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x26, [x20, #0x0]\n"
+ "cbnz x28, 7f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x26, x26, x19\n"
+ "b 7f\n"
+ "6:" // Height 1: setup direct input
+ "mov x26, %x[input_ptr]\n"
+ "7:" // Height 1: input setup done
+ "cmp x27, #0x10\n"
+ "ble 10f\n"
+ "8:" // Height 1: Multiply loop: Main loop head
+ "ld1b { z4.b }, p2/Z, [x11]\n"
+ "whilelt p0.b, XZR, x27\n"
+ "ld1b { z5.b }, p2/Z, [x11, #1, MUL VL]\n"
+ "ld1rqb { z0.b }, p0/Z, [x26]\n"
+ "udot z16.s, z4.b, z0.b[0]\n"
+ "ld1b { z6.b }, p2/Z, [x11, #2, MUL VL]\n"
+ "add x26, x26, #0x10\n"
+ "udot z17.s, z5.b, z0.b[0]\n"
+ "ld1b { z7.b }, p2/Z, [x11, #3, MUL VL]\n"
+ "ld1b { z8.b }, p2/Z, [x11, #4, MUL VL]\n"
+ "udot z18.s, z6.b, z0.b[0]\n"
+ "ld1b { z9.b }, p2/Z, [x11, #5, MUL VL]\n"
+ "ld1b { z10.b }, p2/Z, [x11, #6, MUL VL]\n"
+ "udot z19.s, z7.b, z0.b[0]\n"
+ "udot z16.s, z8.b, z0.b[1]\n"
+ "ld1b { z4.b }, p2/Z, [x11, #7, MUL VL]\n"
+ "addvl x11, x11, #16\n"
+ "udot z17.s, z9.b, z0.b[1]\n"
+ "ld1b { z5.b }, p2/Z, [x11, #-8, MUL VL]\n"
+ "ld1b { z6.b }, p2/Z, [x11, #-7, MUL VL]\n"
+ "udot z18.s, z10.b, z0.b[1]\n"
+ "ld1b { z7.b }, p2/Z, [x11, #-6, MUL VL]\n"
+ "udot z19.s, z4.b, z0.b[1]\n"
+ "ld1b { z8.b }, p2/Z, [x11, #-5, MUL VL]\n"
+ "udot z16.s, z5.b, z0.b[2]\n"
+ "ld1b { z9.b }, p2/Z, [x11, #-4, MUL VL]\n"
+ "udot z17.s, z6.b, z0.b[2]\n"
+ "ld1b { z10.b }, p2/Z, [x11, #-3, MUL VL]\n"
+ "udot z18.s, z7.b, z0.b[2]\n"
+ "ld1b { z4.b }, p2/Z, [x11, #-2, MUL VL]\n"
+ "udot z19.s, z8.b, z0.b[2]\n"
+ "ld1b { z5.b }, p2/Z, [x11, #-1, MUL VL]\n"
+ "udot z16.s, z9.b, z0.b[3]\n"
+ "udot z17.s, z10.b, z0.b[3]\n"
+ "udot z18.s, z4.b, z0.b[3]\n"
+ "udot z19.s, z5.b, z0.b[3]\n"
+ "tbnz %x[flags], #31, 9f\n"
+ "udot z11.s, z0.b, z15.b\n"
+ "9:" // Height 1: Multiply loop: unique 1: skip row sum
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "sub x27, x27, #0x10\n"
+ "cmp x27, #0x10\n"
+ "bgt 8b\n"
+ "10:" // Height 1: Multiply loop: Single iteration only
+ "ld1b { z6.b }, p2/Z, [x11]\n"
+ "whilelt p0.b, XZR, x27\n"
+ "ld1b { z7.b }, p2/Z, [x11, #1, MUL VL]\n"
+ "subs x27, x27, #0x4\n"
+ "ld1rqb { z0.b }, p0/Z, [x26]\n"
+ "udot z16.s, z6.b, z0.b[0]\n"
+ "ld1b { z8.b }, p2/Z, [x11, #2, MUL VL]\n"
+ "add x26, x26, #0x10\n"
+ "udot z17.s, z7.b, z0.b[0]\n"
+ "ld1b { z9.b }, p2/Z, [x11, #3, MUL VL]\n"
+ "addvl x11, x11, #4\n"
+ "udot z18.s, z8.b, z0.b[0]\n"
+ "udot z19.s, z9.b, z0.b[0]\n"
+ "ble 11f\n"
+ "ld1b { z10.b }, p2/Z, [x11]\n"
+ "udot z16.s, z10.b, z0.b[1]\n"
+ "ld1b { z4.b }, p2/Z, [x11, #1, MUL VL]\n"
+ "subs x27, x27, #0x4\n"
+ "udot z17.s, z4.b, z0.b[1]\n"
+ "ld1b { z5.b }, p2/Z, [x11, #2, MUL VL]\n"
+ "ld1b { z6.b }, p2/Z, [x11, #3, MUL VL]\n"
+ "udot z18.s, z5.b, z0.b[1]\n"
+ "addvl x11, x11, #4\n"
+ "udot z19.s, z6.b, z0.b[1]\n"
+ "ble 11f\n"
+ "ld1b { z7.b }, p2/Z, [x11]\n"
+ "udot z16.s, z7.b, z0.b[2]\n"
+ "ld1b { z8.b }, p2/Z, [x11, #1, MUL VL]\n"
+ "subs x27, x27, #0x4\n"
+ "udot z17.s, z8.b, z0.b[2]\n"
+ "ld1b { z9.b }, p2/Z, [x11, #2, MUL VL]\n"
+ "ld1b { z10.b }, p2/Z, [x11, #3, MUL VL]\n"
+ "udot z18.s, z9.b, z0.b[2]\n"
+ "addvl x11, x11, #4\n"
+ "udot z19.s, z10.b, z0.b[2]\n"
+ "ble 11f\n"
+ "ld1b { z4.b }, p2/Z, [x11]\n"
+ "udot z16.s, z4.b, z0.b[3]\n"
+ "ld1b { z5.b }, p2/Z, [x11, #1, MUL VL]\n"
+ "ld1b { z6.b }, p2/Z, [x11, #2, MUL VL]\n"
+ "udot z17.s, z5.b, z0.b[3]\n"
+ "ld1b { z7.b }, p2/Z, [x11, #3, MUL VL]\n"
+ "addvl x11, x11, #4\n"
+ "udot z18.s, z6.b, z0.b[3]\n"
+ "udot z19.s, z7.b, z0.b[3]\n"
+ "11:" // Height 1: Multiply loop: multiply skip
+ "tbnz %x[flags], #31, 12f\n"
+ "udot z11.s, z0.b, z15.b\n"
+ "12:" // Height 1: Multiply loop: unique 2: skip row sum
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "add x28, x28, #0x1\n"
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "cmp x28, x19\n"
+ "bne 5b\n"
+ "prfm pstl1keep, [x9, #0x0]\n"
+ "tbnz %x[flags], #31, 13f\n"
+ "add x19, %x[qp], %[b_offset]\n"
+ "ld1rw { z1.s }, p2/Z, [x19]\n"
+ "neg z1.s, p2/M, z1.s\n"
+ "mov x19, #0x4\n"
+ "whilelt p0.s, XZR, x19\n"
+ "uaddv d11, p0, z11.s\n"
+ "mov z11.s, z11.s[0]\n"
+ "mul z11.s, p2/M, z11.s, z1.s\n"
+ "13:" // Height 1: skip row sum fixup
+ "add z16.s, z16.s, z11.s\n"
+ "ld1w { z0.s }, p2/Z, [x10]\n"
+ "orr %x[flags], %x[flags], #0x80000000\n"
+ "add z17.s, z17.s, z11.s\n"
+ "ld1w { z1.s }, p2/Z, [x10, #1, MUL VL]\n"
+ "add x20, %x[qp], %[per_layer_right_shift]\n"
+ "add z18.s, z18.s, z11.s\n"
+ "ld1w { z2.s }, p2/Z, [x10, #2, MUL VL]\n"
+ "add x19, %x[qp], %[per_layer_mul]\n"
+ "add z19.s, z19.s, z11.s\n"
+ "ld1w { z3.s }, p2/Z, [x10, #3, MUL VL]\n"
+ "addvl x10, x10, #4\n"
+ "add z16.s, z16.s, z0.s\n"
+ "ld1rw { z0.s }, p2/Z, [x20]\n"
+ "add z17.s, z17.s, z1.s\n"
+ "ld1rw { z4.s }, p2/Z, [x19]\n"
+ "add z18.s, z18.s, z2.s\n"
+ "add z19.s, z19.s, z3.s\n"
+ ".inst 0x04a47610 // sqrdmulh z16.s, z16.s, z4.s\n"
+ ".inst 0x04a47631 // sqrdmulh z17.s, z17.s, z4.s\n"
+ ".inst 0x04a47652 // sqrdmulh z18.s, z18.s, z4.s\n"
+ ".inst 0x04a47673 // sqrdmulh z19.s, z19.s, z4.s\n"
+ "tbz %x[flags], #5, 14f\n"
+ "and z4.d, z16.d, z0.d\n"
+ "asr z4.s, z4.s, #0x1f\n"
+ "and z5.d, z17.d, z0.d\n"
+ "and z6.d, z18.d, z0.d\n"
+ "asr z5.s, z5.s, #0x1f\n"
+ "and z7.d, z19.d, z0.d\n"
+ "asr z6.s, z6.s, #0x1f\n"
+ "sqadd z16.s, z16.s, z4.s\n"
+ "asr z7.s, z7.s, #0x1f\n"
+ "sqadd z17.s, z17.s, z5.s\n"
+ "sqadd z18.s, z18.s, z6.s\n"
+ "sqadd z19.s, z19.s, z7.s\n"
+ "14:" // Height 1: no shift correction
+ ".inst 0x44828810 // srshl z16.s, p2/M, z16.s, z0.s\n"
+ "add x19, %x[qp], %[c_offset]\n"
+ "ld1rw { z4.s }, p2/Z, [x19]\n"
+ ".inst 0x44828811 // srshl z17.s, p2/M, z17.s, z0.s\n"
+ "add x19, %x[qp], %[minval]\n"
+ ".inst 0x44828812 // srshl z18.s, p2/M, z18.s, z0.s\n"
+ "ld1rw { z5.s }, p2/Z, [x19]\n"
+ "add x19, %x[qp], %[maxval]\n"
+ ".inst 0x44828813 // srshl z19.s, p2/M, z19.s, z0.s\n"
+ "ld1rw { z6.s }, p2/Z, [x19]\n"
+ "add z16.s, z16.s, z4.s\n"
+ "add z17.s, z17.s, z4.s\n"
+ "add z18.s, z18.s, z4.s\n"
+ "add z19.s, z19.s, z4.s\n"
+ "smin z16.s, p2/M, z16.s, z6.s\n"
+ "smin z17.s, p2/M, z17.s, z6.s\n"
+ "smin z18.s, p2/M, z18.s, z6.s\n"
+ "smin z19.s, p2/M, z19.s, z6.s\n"
+ "smax z16.s, p2/M, z16.s, z5.s\n"
+ "smax z17.s, p2/M, z17.s, z5.s\n"
+ "smax z18.s, p2/M, z18.s, z5.s\n"
+ "smax z19.s, p2/M, z19.s, z5.s\n"
+ "uzp1 z16.h, z16.h, z17.h\n"
+ "uzp1 z17.h, z18.h, z19.h\n"
+ "uzp1 z16.b, z16.b, z17.b\n"
+ "st1b { z16.b }, p1, [x9]\n"
+ "addvl x9, x9, #1\n"
+ "15:" // Height 1: Writeback done
+ "mov x19, #0x0\n"
+ "incw x19, ALL, MUL #4\n"
+ "subs x12, x12, x19\n"
+ "bgt 3b\n"
+ "b 62f\n"
+ "16:" // Height 2
+ "mov z11.s, #0x0\n"
+ "ldr x12, [%x[args_ptr], %[offsetof_N]]\n"
+ "mov x10, %x[col_bias]\n"
+ "mov z12.s, #0x0\n"
+ "ldr x11, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "bic %x[flags], %x[flags], #0x80000000\n"
+ "mov z13.s, #0x0\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "mov z14.s, #0x0\n"
+ "mov z15.b, #0x1\n"
+ "tbz %x[flags], #2, 17f\n"
+ "ldr x9, [%x[output_ptr], #0x0]\n"
+ "ldr x25, [%x[output_ptr], #0x8]\n"
+ "add x9, x9, x19\n"
+ "add x25, x25, x19\n"
+ "b 18f\n"
+ "17:" // Height 2: setup direct output
+ "mov x9, %x[output_ptr]\n"
+ "add x25, x9, x19\n"
+ "18:" // Height 2: Column loop
+ "mov z16.s, #0x0\n"
+ "mov x19, #0x0\n"
+ "mov z17.s, #0x0\n"
+ "whilelt p1.b, x19, x12\n"
+ "mov z18.s, #0x0\n"
+ "mov z19.s, #0x0\n"
+ "mov z20.s, #0x0\n"
+ "mov z21.s, #0x0\n"
+ "mov z22.s, #0x0\n"
+ "mov z23.s, #0x0\n"
+ "19:" // Height 2: setup done
+ "mov x28, #0x0\n"
+ "20:" // Height 2: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w27, [x20, x28, LSL #0x2]\n"
+ "tbz %x[flags], #3, 21f\n"
+ "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x26, [x20, #0x0]\n"
+ "ldr x24, [x20, #0x8]\n"
+ "cbnz x28, 22f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x26, x26, x19\n"
+ "add x24, x24, x19\n"
+ "b 22f\n"
+ "21:" // Height 2: setup direct input
+ "mov x26, %x[input_ptr]\n"
+ "add x24, x26, x19\n"
+ "22:" // Height 2: input setup done
+ "cmp x27, #0x10\n"
+ "ble 25f\n"
+ "23:" // Height 2: Multiply loop: Main loop head
+ "ld1b { z4.b }, p2/Z, [x11]\n"
+ "whilelt p0.b, XZR, x27\n"
+ "ld1b { z5.b }, p2/Z, [x11, #1, MUL VL]\n"
+ "ld1rqb { z0.b }, p0/Z, [x26]\n"
+ "udot z16.s, z4.b, z0.b[0]\n"
+ "ld1rqb { z1.b }, p0/Z, [x24]\n"
+ "add x26, x26, #0x10\n"
+ "udot z17.s, z5.b, z0.b[0]\n"
+ "ld1b { z6.b }, p2/Z, [x11, #2, MUL VL]\n"
+ "add x24, x24, #0x10\n"
+ "udot z20.s, z4.b, z1.b[0]\n"
+ "ld1b { z7.b }, p2/Z, [x11, #3, MUL VL]\n"
+ "udot z21.s, z5.b, z1.b[0]\n"
+ "ld1b { z8.b }, p2/Z, [x11, #4, MUL VL]\n"
+ "udot z18.s, z6.b, z0.b[0]\n"
+ "ld1b { z9.b }, p2/Z, [x11, #5, MUL VL]\n"
+ "udot z22.s, z6.b, z1.b[0]\n"
+ "ld1b { z10.b }, p2/Z, [x11, #6, MUL VL]\n"
+ "udot z19.s, z7.b, z0.b[0]\n"
+ "ld1b { z4.b }, p2/Z, [x11, #7, MUL VL]\n"
+ "addvl x11, x11, #16\n"
+ "udot z23.s, z7.b, z1.b[0]\n"
+ "ld1b { z5.b }, p2/Z, [x11, #-8, MUL VL]\n"
+ "udot z16.s, z8.b, z0.b[1]\n"
+ "ld1b { z6.b }, p2/Z, [x11, #-7, MUL VL]\n"
+ "udot z20.s, z8.b, z1.b[1]\n"
+ "ld1b { z7.b }, p2/Z, [x11, #-6, MUL VL]\n"
+ "udot z17.s, z9.b, z0.b[1]\n"
+ "ld1b { z8.b }, p2/Z, [x11, #-5, MUL VL]\n"
+ "udot z21.s, z9.b, z1.b[1]\n"
+ "ld1b { z9.b }, p2/Z, [x11, #-4, MUL VL]\n"
+ "udot z18.s, z10.b, z0.b[1]\n"
+ "udot z22.s, z10.b, z1.b[1]\n"
+ "ld1b { z10.b }, p2/Z, [x11, #-3, MUL VL]\n"
+ "udot z19.s, z4.b, z0.b[1]\n"
+ "udot z23.s, z4.b, z1.b[1]\n"
+ "ld1b { z4.b }, p2/Z, [x11, #-2, MUL VL]\n"
+ "udot z16.s, z5.b, z0.b[2]\n"
+ "udot z20.s, z5.b, z1.b[2]\n"
+ "ld1b { z5.b }, p2/Z, [x11, #-1, MUL VL]\n"
+ "udot z17.s, z6.b, z0.b[2]\n"
+ "udot z21.s, z6.b, z1.b[2]\n"
+ "udot z18.s, z7.b, z0.b[2]\n"
+ "udot z22.s, z7.b, z1.b[2]\n"
+ "udot z19.s, z8.b, z0.b[2]\n"
+ "udot z23.s, z8.b, z1.b[2]\n"
+ "udot z16.s, z9.b, z0.b[3]\n"
+ "udot z20.s, z9.b, z1.b[3]\n"
+ "udot z17.s, z10.b, z0.b[3]\n"
+ "udot z21.s, z10.b, z1.b[3]\n"
+ "udot z18.s, z4.b, z0.b[3]\n"
+ "udot z22.s, z4.b, z1.b[3]\n"
+ "udot z19.s, z5.b, z0.b[3]\n"
+ "udot z23.s, z5.b, z1.b[3]\n"
+ "tbnz %x[flags], #31, 24f\n"
+ "udot z11.s, z0.b, z15.b\n"
+ "udot z12.s, z1.b, z15.b\n"
+ "24:" // Height 2: Multiply loop: unique 3: skip row sum
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "sub x27, x27, #0x10\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "cmp x27, #0x10\n"
+ "bgt 23b\n"
+ "25:" // Height 2: Multiply loop: Single iteration only
+ "ld1b { z6.b }, p2/Z, [x11]\n"
+ "whilelt p0.b, XZR, x27\n"
+ "ld1b { z7.b }, p2/Z, [x11, #1, MUL VL]\n"
+ "subs x27, x27, #0x4\n"
+ "ld1rqb { z0.b }, p0/Z, [x26]\n"
+ "udot z16.s, z6.b, z0.b[0]\n"
+ "ld1rqb { z1.b }, p0/Z, [x24]\n"
+ "add x26, x26, #0x10\n"
+ "udot z17.s, z7.b, z0.b[0]\n"
+ "ld1b { z8.b }, p2/Z, [x11, #2, MUL VL]\n"
+ "add x24, x24, #0x10\n"
+ "udot z20.s, z6.b, z1.b[0]\n"
+ "ld1b { z9.b }, p2/Z, [x11, #3, MUL VL]\n"
+ "addvl x11, x11, #4\n"
+ "udot z21.s, z7.b, z1.b[0]\n"
+ "udot z18.s, z8.b, z0.b[0]\n"
+ "udot z22.s, z8.b, z1.b[0]\n"
+ "udot z19.s, z9.b, z0.b[0]\n"
+ "udot z23.s, z9.b, z1.b[0]\n"
+ "ble 26f\n"
+ "ld1b { z10.b }, p2/Z, [x11]\n"
+ "udot z16.s, z10.b, z0.b[1]\n"
+ "ld1b { z4.b }, p2/Z, [x11, #1, MUL VL]\n"
+ "subs x27, x27, #0x4\n"
+ "udot z20.s, z10.b, z1.b[1]\n"
+ "ld1b { z5.b }, p2/Z, [x11, #2, MUL VL]\n"
+ "ld1b { z6.b }, p2/Z, [x11, #3, MUL VL]\n"
+ "udot z17.s, z4.b, z0.b[1]\n"
+ "addvl x11, x11, #4\n"
+ "udot z21.s, z4.b, z1.b[1]\n"
+ "udot z18.s, z5.b, z0.b[1]\n"
+ "udot z22.s, z5.b, z1.b[1]\n"
+ "udot z19.s, z6.b, z0.b[1]\n"
+ "udot z23.s, z6.b, z1.b[1]\n"
+ "ble 26f\n"
+ "ld1b { z7.b }, p2/Z, [x11]\n"
+ "udot z16.s, z7.b, z0.b[2]\n"
+ "ld1b { z8.b }, p2/Z, [x11, #1, MUL VL]\n"
+ "subs x27, x27, #0x4\n"
+ "udot z20.s, z7.b, z1.b[2]\n"
+ "ld1b { z9.b }, p2/Z, [x11, #2, MUL VL]\n"
+ "ld1b { z10.b }, p2/Z, [x11, #3, MUL VL]\n"
+ "udot z17.s, z8.b, z0.b[2]\n"
+ "addvl x11, x11, #4\n"
+ "udot z21.s, z8.b, z1.b[2]\n"
+ "udot z18.s, z9.b, z0.b[2]\n"
+ "udot z22.s, z9.b, z1.b[2]\n"
+ "udot z19.s, z10.b, z0.b[2]\n"
+ "udot z23.s, z10.b, z1.b[2]\n"
+ "ble 26f\n"
+ "ld1b { z4.b }, p2/Z, [x11]\n"
+ "udot z16.s, z4.b, z0.b[3]\n"
+ "ld1b { z5.b }, p2/Z, [x11, #1, MUL VL]\n"
+ "udot z20.s, z4.b, z1.b[3]\n"
+ "ld1b { z6.b }, p2/Z, [x11, #2, MUL VL]\n"
+ "ld1b { z7.b }, p2/Z, [x11, #3, MUL VL]\n"
+ "udot z17.s, z5.b, z0.b[3]\n"
+ "addvl x11, x11, #4\n"
+ "udot z21.s, z5.b, z1.b[3]\n"
+ "udot z18.s, z6.b, z0.b[3]\n"
+ "udot z22.s, z6.b, z1.b[3]\n"
+ "udot z19.s, z7.b, z0.b[3]\n"
+ "udot z23.s, z7.b, z1.b[3]\n"
+ "26:" // Height 2: Multiply loop: multiply skip
+ "tbnz %x[flags], #31, 27f\n"
+ "udot z11.s, z0.b, z15.b\n"
+ "udot z12.s, z1.b, z15.b\n"
+ "27:" // Height 2: Multiply loop: unique 4: skip row sum
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "add x28, x28, #0x1\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "cmp x28, x19\n"
+ "bne 20b\n"
+ "prfm pstl1keep, [x9, #0x0]\n"
+ "prfm pstl1keep, [x25, #0x0]\n"
+ "tbnz %x[flags], #31, 28f\n"
+ "add x19, %x[qp], %[b_offset]\n"
+ "ld1rw { z2.s }, p2/Z, [x19]\n"
+ "neg z2.s, p2/M, z2.s\n"
+ "mov x20, #0x4\n"
+ "mov x19, #0x4\n"
+ "whilelt p0.s, XZR, x20\n"
+ "uaddv d11, p0, z11.s\n"
+ "whilelt p0.s, XZR, x19\n"
+ "uaddv d12, p0, z12.s\n"
+ "mov z11.s, z11.s[0]\n"
+ "mov z12.s, z12.s[0]\n"
+ "mul z11.s, p2/M, z11.s, z2.s\n"
+ "mul z12.s, p2/M, z12.s, z2.s\n"
+ "28:" // Height 2: skip row sum fixup
+ "add z16.s, z16.s, z11.s\n"
+ "ld1w { z0.s }, p2/Z, [x10]\n"
+ "orr %x[flags], %x[flags], #0x80000000\n"
+ "add z17.s, z17.s, z11.s\n"
+ "ld1w { z1.s }, p2/Z, [x10, #1, MUL VL]\n"
+ "add x20, %x[qp], %[per_layer_right_shift]\n"
+ "add z18.s, z18.s, z11.s\n"
+ "ld1w { z2.s }, p2/Z, [x10, #2, MUL VL]\n"
+ "add x19, %x[qp], %[per_layer_mul]\n"
+ "add z19.s, z19.s, z11.s\n"
+ "ld1w { z3.s }, p2/Z, [x10, #3, MUL VL]\n"
+ "addvl x10, x10, #4\n"
+ "add z20.s, z20.s, z12.s\n"
+ "ld1rw { z4.s }, p2/Z, [x19]\n"
+ "add z21.s, z21.s, z12.s\n"
+ "add z22.s, z22.s, z12.s\n"
+ "add z23.s, z23.s, z12.s\n"
+ "add z16.s, z16.s, z0.s\n"
+ "add z17.s, z17.s, z1.s\n"
+ "add z18.s, z18.s, z2.s\n"
+ "add z19.s, z19.s, z3.s\n"
+ "add z20.s, z20.s, z0.s\n"
+ "ld1rw { z0.s }, p2/Z, [x20]\n"
+ "add z21.s, z21.s, z1.s\n"
+ "add z22.s, z22.s, z2.s\n"
+ "add z23.s, z23.s, z3.s\n"
+ ".inst 0x04a47610 // sqrdmulh z16.s, z16.s, z4.s\n"
+ ".inst 0x04a47631 // sqrdmulh z17.s, z17.s, z4.s\n"
+ ".inst 0x04a47652 // sqrdmulh z18.s, z18.s, z4.s\n"
+ ".inst 0x04a47673 // sqrdmulh z19.s, z19.s, z4.s\n"
+ ".inst 0x04a47694 // sqrdmulh z20.s, z20.s, z4.s\n"
+ ".inst 0x04a476b5 // sqrdmulh z21.s, z21.s, z4.s\n"
+ ".inst 0x04a476d6 // sqrdmulh z22.s, z22.s, z4.s\n"
+ ".inst 0x04a476f7 // sqrdmulh z23.s, z23.s, z4.s\n"
+ "tbz %x[flags], #5, 29f\n"
+ "and z4.d, z16.d, z0.d\n"
+ "asr z4.s, z4.s, #0x1f\n"
+ "and z5.d, z17.d, z0.d\n"
+ "and z6.d, z18.d, z0.d\n"
+ "asr z5.s, z5.s, #0x1f\n"
+ "and z7.d, z19.d, z0.d\n"
+ "and z8.d, z20.d, z0.d\n"
+ "asr z6.s, z6.s, #0x1f\n"
+ "and z9.d, z21.d, z0.d\n"
+ "asr z7.s, z7.s, #0x1f\n"
+ "sqadd z16.s, z16.s, z4.s\n"
+ "and z10.d, z22.d, z0.d\n"
+ "asr z8.s, z8.s, #0x1f\n"
+ "and z4.d, z23.d, z0.d\n"
+ "asr z9.s, z9.s, #0x1f\n"
+ "sqadd z17.s, z17.s, z5.s\n"
+ "asr z10.s, z10.s, #0x1f\n"
+ "sqadd z18.s, z18.s, z6.s\n"
+ "asr z4.s, z4.s, #0x1f\n"
+ "sqadd z19.s, z19.s, z7.s\n"
+ "sqadd z20.s, z20.s, z8.s\n"
+ "sqadd z21.s, z21.s, z9.s\n"
+ "sqadd z22.s, z22.s, z10.s\n"
+ "sqadd z23.s, z23.s, z4.s\n"
+ "29:" // Height 2: no shift correction
+ ".inst 0x44828810 // srshl z16.s, p2/M, z16.s, z0.s\n"
+ "add x19, %x[qp], %[c_offset]\n"
+ "ld1rw { z4.s }, p2/Z, [x19]\n"
+ ".inst 0x44828811 // srshl z17.s, p2/M, z17.s, z0.s\n"
+ "add x19, %x[qp], %[minval]\n"
+ ".inst 0x44828812 // srshl z18.s, p2/M, z18.s, z0.s\n"
+ "ld1rw { z5.s }, p2/Z, [x19]\n"
+ "add x19, %x[qp], %[maxval]\n"
+ ".inst 0x44828813 // srshl z19.s, p2/M, z19.s, z0.s\n"
+ "ld1rw { z6.s }, p2/Z, [x19]\n"
+ ".inst 0x44828814 // srshl z20.s, p2/M, z20.s, z0.s\n"
+ "add z16.s, z16.s, z4.s\n"
+ "add z17.s, z17.s, z4.s\n"
+ "add z18.s, z18.s, z4.s\n"
+ "add z19.s, z19.s, z4.s\n"
+ "add z20.s, z20.s, z4.s\n"
+ "smin z16.s, p2/M, z16.s, z6.s\n"
+ "smin z17.s, p2/M, z17.s, z6.s\n"
+ "smin z18.s, p2/M, z18.s, z6.s\n"
+ "smin z19.s, p2/M, z19.s, z6.s\n"
+ "smax z16.s, p2/M, z16.s, z5.s\n"
+ "smax z17.s, p2/M, z17.s, z5.s\n"
+ "smax z18.s, p2/M, z18.s, z5.s\n"
+ "smax z19.s, p2/M, z19.s, z5.s\n"
+ "smin z20.s, p2/M, z20.s, z6.s\n"
+ "uzp1 z16.h, z16.h, z17.h\n"
+ ".inst 0x44828815 // srshl z21.s, p2/M, z21.s, z0.s\n"
+ "uzp1 z17.h, z18.h, z19.h\n"
+ "smax z20.s, p2/M, z20.s, z5.s\n"
+ "uzp1 z16.b, z16.b, z17.b\n"
+ "st1b { z16.b }, p1, [x9]\n"
+ "add z21.s, z21.s, z4.s\n"
+ "addvl x9, x9, #1\n"
+ ".inst 0x44828816 // srshl z22.s, p2/M, z22.s, z0.s\n"
+ ".inst 0x44828817 // srshl z23.s, p2/M, z23.s, z0.s\n"
+ "smin z21.s, p2/M, z21.s, z6.s\n"
+ "add z22.s, z22.s, z4.s\n"
+ "add z23.s, z23.s, z4.s\n"
+ "smax z21.s, p2/M, z21.s, z5.s\n"
+ "smin z22.s, p2/M, z22.s, z6.s\n"
+ "smin z23.s, p2/M, z23.s, z6.s\n"
+ "uzp1 z20.h, z20.h, z21.h\n"
+ "smax z22.s, p2/M, z22.s, z5.s\n"
+ "smax z23.s, p2/M, z23.s, z5.s\n"
+ "uzp1 z21.h, z22.h, z23.h\n"
+ "uzp1 z20.b, z20.b, z21.b\n"
+ "st1b { z20.b }, p1, [x25]\n"
+ "addvl x25, x25, #1\n"
+ "30:" // Height 2: Writeback done
+ "mov x19, #0x0\n"
+ "incw x19, ALL, MUL #4\n"
+ "subs x12, x12, x19\n"
+ "bgt 18b\n"
+ "b 62f\n"
+ "31:" // Height 3
+ "mov z11.s, #0x0\n"
+ "ldr x12, [%x[args_ptr], %[offsetof_N]]\n"
+ "mov x10, %x[col_bias]\n"
+ "mov z12.s, #0x0\n"
+ "ldr x11, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "bic %x[flags], %x[flags], #0x80000000\n"
+ "mov z13.s, #0x0\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "mov z14.s, #0x0\n"
+ "mov z15.b, #0x1\n"
+ "tbz %x[flags], #2, 32f\n"
+ "ldr x9, [%x[output_ptr], #0x0]\n"
+ "ldr x25, [%x[output_ptr], #0x8]\n"
+ "add x9, x9, x19\n"
+ "ldr x23, [%x[output_ptr], #0x10]\n"
+ "add x25, x25, x19\n"
+ "add x23, x23, x19\n"
+ "b 33f\n"
+ "32:" // Height 3: setup direct output
+ "mov x9, %x[output_ptr]\n"
+ "add x25, x9, x19\n"
+ "add x23, x25, x19\n"
+ "33:" // Height 3: Column loop
+ "mov z16.s, #0x0\n"
+ "mov x19, #0x0\n"
+ "mov z17.s, #0x0\n"
+ "whilelt p1.b, x19, x12\n"
+ "mov z18.s, #0x0\n"
+ "mov z19.s, #0x0\n"
+ "mov z20.s, #0x0\n"
+ "mov z21.s, #0x0\n"
+ "mov z22.s, #0x0\n"
+ "mov z23.s, #0x0\n"
+ "mov z24.s, #0x0\n"
+ "mov z25.s, #0x0\n"
+ "mov z26.s, #0x0\n"
+ "mov z27.s, #0x0\n"
+ "34:" // Height 3: setup done
+ "mov x28, #0x0\n"
+ "35:" // Height 3: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w27, [x20, x28, LSL #0x2]\n"
+ "tbz %x[flags], #3, 36f\n"
+ "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x26, [x20, #0x0]\n"
+ "ldr x24, [x20, #0x8]\n"
+ "ldr x22, [x20, #0x10]\n"
+ "cbnz x28, 37f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x26, x26, x19\n"
+ "add x24, x24, x19\n"
+ "add x22, x22, x19\n"
+ "b 37f\n"
+ "36:" // Height 3: setup direct input
+ "mov x26, %x[input_ptr]\n"
+ "add x24, x26, x19\n"
+ "add x22, x24, x19\n"
+ "37:" // Height 3: input setup done
+ "cmp x27, #0x10\n"
+ "ble 40f\n"
+ "38:" // Height 3: Multiply loop: Main loop head
+ "ld1b { z4.b }, p2/Z, [x11]\n"
+ "whilelt p0.b, XZR, x27\n"
+ "ld1b { z5.b }, p2/Z, [x11, #1, MUL VL]\n"
+ "ld1rqb { z0.b }, p0/Z, [x26]\n"
+ "udot z16.s, z4.b, z0.b[0]\n"
+ "ld1rqb { z1.b }, p0/Z, [x24]\n"
+ "add x26, x26, #0x10\n"
+ "udot z17.s, z5.b, z0.b[0]\n"
+ "ld1rqb { z2.b }, p0/Z, [x22]\n"
+ "add x24, x24, #0x10\n"
+ "udot z20.s, z4.b, z1.b[0]\n"
+ "ld1b { z6.b }, p2/Z, [x11, #2, MUL VL]\n"
+ "add x22, x22, #0x10\n"
+ "udot z24.s, z4.b, z2.b[0]\n"
+ "ld1b { z7.b }, p2/Z, [x11, #3, MUL VL]\n"
+ "udot z21.s, z5.b, z1.b[0]\n"
+ "ld1b { z8.b }, p2/Z, [x11, #4, MUL VL]\n"
+ "udot z25.s, z5.b, z2.b[0]\n"
+ "ld1b { z9.b }, p2/Z, [x11, #5, MUL VL]\n"
+ "udot z18.s, z6.b, z0.b[0]\n"
+ "ld1b { z10.b }, p2/Z, [x11, #6, MUL VL]\n"
+ "udot z22.s, z6.b, z1.b[0]\n"
+ "ld1b { z4.b }, p2/Z, [x11, #7, MUL VL]\n"
+ "addvl x11, x11, #16\n"
+ "udot z26.s, z6.b, z2.b[0]\n"
+ "ld1b { z5.b }, p2/Z, [x11, #-8, MUL VL]\n"
+ "udot z19.s, z7.b, z0.b[0]\n"
+ "ld1b { z6.b }, p2/Z, [x11, #-7, MUL VL]\n"
+ "udot z23.s, z7.b, z1.b[0]\n"
+ "udot z27.s, z7.b, z2.b[0]\n"
+ "ld1b { z7.b }, p2/Z, [x11, #-6, MUL VL]\n"
+ "udot z16.s, z8.b, z0.b[1]\n"
+ "udot z20.s, z8.b, z1.b[1]\n"
+ "udot z24.s, z8.b, z2.b[1]\n"
+ "ld1b { z8.b }, p2/Z, [x11, #-5, MUL VL]\n"
+ "udot z17.s, z9.b, z0.b[1]\n"
+ "udot z21.s, z9.b, z1.b[1]\n"
+ "udot z25.s, z9.b, z2.b[1]\n"
+ "ld1b { z9.b }, p2/Z, [x11, #-4, MUL VL]\n"
+ "udot z18.s, z10.b, z0.b[1]\n"
+ "udot z22.s, z10.b, z1.b[1]\n"
+ "udot z26.s, z10.b, z2.b[1]\n"
+ "ld1b { z10.b }, p2/Z, [x11, #-3, MUL VL]\n"
+ "udot z19.s, z4.b, z0.b[1]\n"
+ "udot z23.s, z4.b, z1.b[1]\n"
+ "udot z27.s, z4.b, z2.b[1]\n"
+ "ld1b { z4.b }, p2/Z, [x11, #-2, MUL VL]\n"
+ "udot z16.s, z5.b, z0.b[2]\n"
+ "udot z20.s, z5.b, z1.b[2]\n"
+ "udot z24.s, z5.b, z2.b[2]\n"
+ "ld1b { z5.b }, p2/Z, [x11, #-1, MUL VL]\n"
+ "udot z17.s, z6.b, z0.b[2]\n"
+ "udot z21.s, z6.b, z1.b[2]\n"
+ "udot z25.s, z6.b, z2.b[2]\n"
+ "udot z18.s, z7.b, z0.b[2]\n"
+ "udot z22.s, z7.b, z1.b[2]\n"
+ "udot z26.s, z7.b, z2.b[2]\n"
+ "udot z19.s, z8.b, z0.b[2]\n"
+ "udot z23.s, z8.b, z1.b[2]\n"
+ "udot z27.s, z8.b, z2.b[2]\n"
+ "udot z16.s, z9.b, z0.b[3]\n"
+ "udot z20.s, z9.b, z1.b[3]\n"
+ "udot z24.s, z9.b, z2.b[3]\n"
+ "udot z17.s, z10.b, z0.b[3]\n"
+ "udot z21.s, z10.b, z1.b[3]\n"
+ "udot z25.s, z10.b, z2.b[3]\n"
+ "udot z18.s, z4.b, z0.b[3]\n"
+ "udot z22.s, z4.b, z1.b[3]\n"
+ "udot z26.s, z4.b, z2.b[3]\n"
+ "udot z19.s, z5.b, z0.b[3]\n"
+ "udot z23.s, z5.b, z1.b[3]\n"
+ "udot z27.s, z5.b, z2.b[3]\n"
+ "tbnz %x[flags], #31, 39f\n"
+ "udot z11.s, z0.b, z15.b\n"
+ "udot z12.s, z1.b, z15.b\n"
+ "udot z13.s, z2.b, z15.b\n"
+ "39:" // Height 3: Multiply loop: unique 5: skip row sum
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "sub x27, x27, #0x10\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "cmp x27, #0x10\n"
+ "prfm pldl1keep, [x22, #0x80]\n"
+ "bgt 38b\n"
+ "40:" // Height 3: Multiply loop: Single iteration only
+ "ld1b { z6.b }, p2/Z, [x11]\n"
+ "whilelt p0.b, XZR, x27\n"
+ "ld1b { z7.b }, p2/Z, [x11, #1, MUL VL]\n"
+ "subs x27, x27, #0x4\n"
+ "ld1rqb { z0.b }, p0/Z, [x26]\n"
+ "udot z16.s, z6.b, z0.b[0]\n"
+ "ld1rqb { z1.b }, p0/Z, [x24]\n"
+ "add x26, x26, #0x10\n"
+ "udot z17.s, z7.b, z0.b[0]\n"
+ "ld1rqb { z2.b }, p0/Z, [x22]\n"
+ "add x24, x24, #0x10\n"
+ "udot z20.s, z6.b, z1.b[0]\n"
+ "ld1b { z8.b }, p2/Z, [x11, #2, MUL VL]\n"
+ "add x22, x22, #0x10\n"
+ "udot z24.s, z6.b, z2.b[0]\n"
+ "ld1b { z9.b }, p2/Z, [x11, #3, MUL VL]\n"
+ "addvl x11, x11, #4\n"
+ "udot z21.s, z7.b, z1.b[0]\n"
+ "udot z25.s, z7.b, z2.b[0]\n"
+ "udot z18.s, z8.b, z0.b[0]\n"
+ "udot z22.s, z8.b, z1.b[0]\n"
+ "udot z26.s, z8.b, z2.b[0]\n"
+ "udot z19.s, z9.b, z0.b[0]\n"
+ "udot z23.s, z9.b, z1.b[0]\n"
+ "udot z27.s, z9.b, z2.b[0]\n"
+ "ble 41f\n"
+ "ld1b { z10.b }, p2/Z, [x11]\n"
+ "udot z16.s, z10.b, z0.b[1]\n"
+ "ld1b { z4.b }, p2/Z, [x11, #1, MUL VL]\n"
+ "subs x27, x27, #0x4\n"
+ "udot z20.s, z10.b, z1.b[1]\n"
+ "ld1b { z5.b }, p2/Z, [x11, #2, MUL VL]\n"
+ "udot z24.s, z10.b, z2.b[1]\n"
+ "ld1b { z6.b }, p2/Z, [x11, #3, MUL VL]\n"
+ "addvl x11, x11, #4\n"
+ "udot z17.s, z4.b, z0.b[1]\n"
+ "udot z21.s, z4.b, z1.b[1]\n"
+ "udot z25.s, z4.b, z2.b[1]\n"
+ "udot z18.s, z5.b, z0.b[1]\n"
+ "udot z22.s, z5.b, z1.b[1]\n"
+ "udot z26.s, z5.b, z2.b[1]\n"
+ "udot z19.s, z6.b, z0.b[1]\n"
+ "udot z23.s, z6.b, z1.b[1]\n"
+ "udot z27.s, z6.b, z2.b[1]\n"
+ "ble 41f\n"
+ "ld1b { z7.b }, p2/Z, [x11]\n"
+ "udot z16.s, z7.b, z0.b[2]\n"
+ "ld1b { z8.b }, p2/Z, [x11, #1, MUL VL]\n"
+ "subs x27, x27, #0x4\n"
+ "udot z20.s, z7.b, z1.b[2]\n"
+ "ld1b { z9.b }, p2/Z, [x11, #2, MUL VL]\n"
+ "udot z24.s, z7.b, z2.b[2]\n"
+ "ld1b { z10.b }, p2/Z, [x11, #3, MUL VL]\n"
+ "addvl x11, x11, #4\n"
+ "udot z17.s, z8.b, z0.b[2]\n"
+ "udot z21.s, z8.b, z1.b[2]\n"
+ "udot z25.s, z8.b, z2.b[2]\n"
+ "udot z18.s, z9.b, z0.b[2]\n"
+ "udot z22.s, z9.b, z1.b[2]\n"
+ "udot z26.s, z9.b, z2.b[2]\n"
+ "udot z19.s, z10.b, z0.b[2]\n"
+ "udot z23.s, z10.b, z1.b[2]\n"
+ "udot z27.s, z10.b, z2.b[2]\n"
+ "ble 41f\n"
+ "ld1b { z4.b }, p2/Z, [x11]\n"
+ "udot z16.s, z4.b, z0.b[3]\n"
+ "ld1b { z5.b }, p2/Z, [x11, #1, MUL VL]\n"
+ "udot z20.s, z4.b, z1.b[3]\n"
+ "ld1b { z6.b }, p2/Z, [x11, #2, MUL VL]\n"
+ "udot z24.s, z4.b, z2.b[3]\n"
+ "ld1b { z7.b }, p2/Z, [x11, #3, MUL VL]\n"
+ "addvl x11, x11, #4\n"
+ "udot z17.s, z5.b, z0.b[3]\n"
+ "udot z21.s, z5.b, z1.b[3]\n"
+ "udot z25.s, z5.b, z2.b[3]\n"
+ "udot z18.s, z6.b, z0.b[3]\n"
+ "udot z22.s, z6.b, z1.b[3]\n"
+ "udot z26.s, z6.b, z2.b[3]\n"
+ "udot z19.s, z7.b, z0.b[3]\n"
+ "udot z23.s, z7.b, z1.b[3]\n"
+ "udot z27.s, z7.b, z2.b[3]\n"
+ "41:" // Height 3: Multiply loop: multiply skip
+ "tbnz %x[flags], #31, 42f\n"
+ "udot z11.s, z0.b, z15.b\n"
+ "udot z12.s, z1.b, z15.b\n"
+ "udot z13.s, z2.b, z15.b\n"
+ "42:" // Height 3: Multiply loop: unique 6: skip row sum
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "add x28, x28, #0x1\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "prfm pldl1keep, [x22, #0x80]\n"
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "cmp x28, x19\n"
+ "bne 35b\n"
+ "prfm pstl1keep, [x9, #0x0]\n"
+ "prfm pstl1keep, [x25, #0x0]\n"
+ "prfm pstl1keep, [x23, #0x0]\n"
+ "tbnz %x[flags], #31, 43f\n"
+ "add x19, %x[qp], %[b_offset]\n"
+ "ld1rw { z3.s }, p2/Z, [x19]\n"
+ "neg z3.s, p2/M, z3.s\n"
+ "mov x20, #0x4\n"
+ "mov x19, #0x4\n"
+ "whilelt p0.s, XZR, x20\n"
+ "uaddv d11, p0, z11.s\n"
+ "whilelt p0.s, XZR, x19\n"
+ "uaddv d12, p0, z12.s\n"
+ "mov x19, #0x4\n"
+ "mov z11.s, z11.s[0]\n"
+ "whilelt p0.s, XZR, x19\n"
+ "mov z12.s, z12.s[0]\n"
+ "uaddv d13, p0, z13.s\n"
+ "mul z11.s, p2/M, z11.s, z3.s\n"
+ "mul z12.s, p2/M, z12.s, z3.s\n"
+ "mov z13.s, z13.s[0]\n"
+ "mul z13.s, p2/M, z13.s, z3.s\n"
+ "43:" // Height 3: skip row sum fixup
+ "add z16.s, z16.s, z11.s\n"
+ "ld1w { z0.s }, p2/Z, [x10]\n"
+ "orr %x[flags], %x[flags], #0x80000000\n"
+ "add z17.s, z17.s, z11.s\n"
+ "ld1w { z1.s }, p2/Z, [x10, #1, MUL VL]\n"
+ "add x20, %x[qp], %[per_layer_right_shift]\n"
+ "add z18.s, z18.s, z11.s\n"
+ "ld1w { z2.s }, p2/Z, [x10, #2, MUL VL]\n"
+ "add x19, %x[qp], %[per_layer_mul]\n"
+ "add z19.s, z19.s, z11.s\n"
+ "ld1w { z3.s }, p2/Z, [x10, #3, MUL VL]\n"
+ "addvl x10, x10, #4\n"
+ "add z20.s, z20.s, z12.s\n"
+ "ld1rw { z4.s }, p2/Z, [x19]\n"
+ "add z21.s, z21.s, z12.s\n"
+ "add z22.s, z22.s, z12.s\n"
+ "add z23.s, z23.s, z12.s\n"
+ "add z24.s, z24.s, z13.s\n"
+ "add z25.s, z25.s, z13.s\n"
+ "add z26.s, z26.s, z13.s\n"
+ "add z27.s, z27.s, z13.s\n"
+ "add z16.s, z16.s, z0.s\n"
+ "add z17.s, z17.s, z1.s\n"
+ "add z18.s, z18.s, z2.s\n"
+ "add z19.s, z19.s, z3.s\n"
+ "add z20.s, z20.s, z0.s\n"
+ "add z21.s, z21.s, z1.s\n"
+ "add z22.s, z22.s, z2.s\n"
+ "add z23.s, z23.s, z3.s\n"
+ "add z24.s, z24.s, z0.s\n"
+ "ld1rw { z0.s }, p2/Z, [x20]\n"
+ "add z25.s, z25.s, z1.s\n"
+ "add z26.s, z26.s, z2.s\n"
+ "add z27.s, z27.s, z3.s\n"
+ ".inst 0x04a47610 // sqrdmulh z16.s, z16.s, z4.s\n"
+ ".inst 0x04a47631 // sqrdmulh z17.s, z17.s, z4.s\n"
+ ".inst 0x04a47652 // sqrdmulh z18.s, z18.s, z4.s\n"
+ ".inst 0x04a47673 // sqrdmulh z19.s, z19.s, z4.s\n"
+ ".inst 0x04a47694 // sqrdmulh z20.s, z20.s, z4.s\n"
+ ".inst 0x04a476b5 // sqrdmulh z21.s, z21.s, z4.s\n"
+ ".inst 0x04a476d6 // sqrdmulh z22.s, z22.s, z4.s\n"
+ ".inst 0x04a476f7 // sqrdmulh z23.s, z23.s, z4.s\n"
+ ".inst 0x04a47718 // sqrdmulh z24.s, z24.s, z4.s\n"
+ ".inst 0x04a47739 // sqrdmulh z25.s, z25.s, z4.s\n"
+ ".inst 0x04a4775a // sqrdmulh z26.s, z26.s, z4.s\n"
+ ".inst 0x04a4777b // sqrdmulh z27.s, z27.s, z4.s\n"
+ "tbz %x[flags], #5, 44f\n"
+ "and z4.d, z16.d, z0.d\n"
+ "asr z4.s, z4.s, #0x1f\n"
+ "and z5.d, z17.d, z0.d\n"
+ "and z6.d, z18.d, z0.d\n"
+ "asr z5.s, z5.s, #0x1f\n"
+ "and z7.d, z19.d, z0.d\n"
+ "and z8.d, z20.d, z0.d\n"
+ "asr z6.s, z6.s, #0x1f\n"
+ "and z9.d, z21.d, z0.d\n"
+ "asr z7.s, z7.s, #0x1f\n"
+ "sqadd z16.s, z16.s, z4.s\n"
+ "and z10.d, z22.d, z0.d\n"
+ "asr z8.s, z8.s, #0x1f\n"
+ "and z4.d, z23.d, z0.d\n"
+ "asr z9.s, z9.s, #0x1f\n"
+ "sqadd z17.s, z17.s, z5.s\n"
+ "asr z10.s, z10.s, #0x1f\n"
+ "sqadd z18.s, z18.s, z6.s\n"
+ "asr z4.s, z4.s, #0x1f\n"
+ "and z5.d, z24.d, z0.d\n"
+ "asr z5.s, z5.s, #0x1f\n"
+ "sqadd z19.s, z19.s, z7.s\n"
+ "sqadd z20.s, z20.s, z8.s\n"
+ "sqadd z21.s, z21.s, z9.s\n"
+ "sqadd z22.s, z22.s, z10.s\n"
+ "sqadd z23.s, z23.s, z4.s\n"
+ "and z6.d, z25.d, z0.d\n"
+ "asr z6.s, z6.s, #0x1f\n"
+ "sqadd z24.s, z24.s, z5.s\n"
+ "and z7.d, z26.d, z0.d\n"
+ "asr z7.s, z7.s, #0x1f\n"
+ "and z8.d, z27.d, z0.d\n"
+ "sqadd z25.s, z25.s, z6.s\n"
+ "asr z8.s, z8.s, #0x1f\n"
+ "sqadd z26.s, z26.s, z7.s\n"
+ "sqadd z27.s, z27.s, z8.s\n"
+ "44:" // Height 3: no shift correction
+ ".inst 0x44828810 // srshl z16.s, p2/M, z16.s, z0.s\n"
+ "add x19, %x[qp], %[c_offset]\n"
+ "ld1rw { z4.s }, p2/Z, [x19]\n"
+ ".inst 0x44828811 // srshl z17.s, p2/M, z17.s, z0.s\n"
+ "add x19, %x[qp], %[minval]\n"
+ ".inst 0x44828812 // srshl z18.s, p2/M, z18.s, z0.s\n"
+ "ld1rw { z5.s }, p2/Z, [x19]\n"
+ "add x19, %x[qp], %[maxval]\n"
+ ".inst 0x44828813 // srshl z19.s, p2/M, z19.s, z0.s\n"
+ "ld1rw { z6.s }, p2/Z, [x19]\n"
+ ".inst 0x44828814 // srshl z20.s, p2/M, z20.s, z0.s\n"
+ "add z16.s, z16.s, z4.s\n"
+ "add z17.s, z17.s, z4.s\n"
+ "add z18.s, z18.s, z4.s\n"
+ "add z19.s, z19.s, z4.s\n"
+ "add z20.s, z20.s, z4.s\n"
+ "smin z16.s, p2/M, z16.s, z6.s\n"
+ "smin z17.s, p2/M, z17.s, z6.s\n"
+ "smin z18.s, p2/M, z18.s, z6.s\n"
+ "smin z19.s, p2/M, z19.s, z6.s\n"
+ "smax z16.s, p2/M, z16.s, z5.s\n"
+ "smax z17.s, p2/M, z17.s, z5.s\n"
+ "smax z18.s, p2/M, z18.s, z5.s\n"
+ "smax z19.s, p2/M, z19.s, z5.s\n"
+ "smin z20.s, p2/M, z20.s, z6.s\n"
+ "uzp1 z16.h, z16.h, z17.h\n"
+ ".inst 0x44828815 // srshl z21.s, p2/M, z21.s, z0.s\n"
+ "uzp1 z17.h, z18.h, z19.h\n"
+ "smax z20.s, p2/M, z20.s, z5.s\n"
+ "uzp1 z16.b, z16.b, z17.b\n"
+ "st1b { z16.b }, p1, [x9]\n"
+ "add z21.s, z21.s, z4.s\n"
+ "addvl x9, x9, #1\n"
+ ".inst 0x44828816 // srshl z22.s, p2/M, z22.s, z0.s\n"
+ ".inst 0x44828817 // srshl z23.s, p2/M, z23.s, z0.s\n"
+ ".inst 0x44828818 // srshl z24.s, p2/M, z24.s, z0.s\n"
+ "smin z21.s, p2/M, z21.s, z6.s\n"
+ ".inst 0x44828819 // srshl z25.s, p2/M, z25.s, z0.s\n"
+ "add z22.s, z22.s, z4.s\n"
+ "add z23.s, z23.s, z4.s\n"
+ "add z24.s, z24.s, z4.s\n"
+ "add z25.s, z25.s, z4.s\n"
+ "smax z21.s, p2/M, z21.s, z5.s\n"
+ "smin z22.s, p2/M, z22.s, z6.s\n"
+ "smin z23.s, p2/M, z23.s, z6.s\n"
+ "smin z24.s, p2/M, z24.s, z6.s\n"
+ "uzp1 z20.h, z20.h, z21.h\n"
+ "smax z22.s, p2/M, z22.s, z5.s\n"
+ "smax z23.s, p2/M, z23.s, z5.s\n"
+ "smax z24.s, p2/M, z24.s, z5.s\n"
+ "smin z25.s, p2/M, z25.s, z6.s\n"
+ ".inst 0x4482881a // srshl z26.s, p2/M, z26.s, z0.s\n"
+ "uzp1 z21.h, z22.h, z23.h\n"
+ ".inst 0x4482881b // srshl z27.s, p2/M, z27.s, z0.s\n"
+ "uzp1 z20.b, z20.b, z21.b\n"
+ "st1b { z20.b }, p1, [x25]\n"
+ "add z26.s, z26.s, z4.s\n"
+ "addvl x25, x25, #1\n"
+ "add z27.s, z27.s, z4.s\n"
+ "smax z25.s, p2/M, z25.s, z5.s\n"
+ "smin z26.s, p2/M, z26.s, z6.s\n"
+ "smin z27.s, p2/M, z27.s, z6.s\n"
+ "uzp1 z24.h, z24.h, z25.h\n"
+ "smax z26.s, p2/M, z26.s, z5.s\n"
+ "smax z27.s, p2/M, z27.s, z5.s\n"
+ "uzp1 z25.h, z26.h, z27.h\n"
+ "uzp1 z24.b, z24.b, z25.b\n"
+ "st1b { z24.b }, p1, [x23]\n"
+ "addvl x23, x23, #1\n"
+ "45:" // Height 3: Writeback done
+ "mov x19, #0x0\n"
+ "incw x19, ALL, MUL #4\n"
+ "subs x12, x12, x19\n"
+ "bgt 33b\n"
+ "b 62f\n"
+ "46:" // Height 4
+ "mov z11.s, #0x0\n"
+ "ldr x12, [%x[args_ptr], %[offsetof_N]]\n"
+ "mov x10, %x[col_bias]\n"
+ "mov z12.s, #0x0\n"
+ "ldr x11, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "bic %x[flags], %x[flags], #0x80000000\n"
+ "mov z13.s, #0x0\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "mov z14.s, #0x0\n"
+ "mov z15.b, #0x1\n"
+ "tbz %x[flags], #2, 47f\n"
+ "ldr x9, [%x[output_ptr], #0x0]\n"
+ "ldr x25, [%x[output_ptr], #0x8]\n"
+ "add x9, x9, x19\n"
+ "ldr x23, [%x[output_ptr], #0x10]\n"
+ "ldr x21, [%x[output_ptr], #0x18]\n"
+ "add x25, x25, x19\n"
+ "add %x[output_ptr], %x[output_ptr], #0x20\n"
+ "add x23, x23, x19\n"
+ "add x21, x21, x19\n"
+ "b 48f\n"
+ "47:" // Height 4: setup direct output
+ "mov x9, %x[output_ptr]\n"
+ "add x25, x9, x19\n"
+ "add x23, x25, x19\n"
+ "add x21, x23, x19\n"
+ "add %x[output_ptr], x21, x19\n"
+ "48:" // Height 4: Column loop
+ "mov z16.s, #0x0\n"
+ "mov x19, #0x0\n"
+ "mov z17.s, #0x0\n"
+ "whilelt p1.b, x19, x12\n"
+ "mov z18.s, #0x0\n"
+ "mov z19.s, #0x0\n"
+ "mov z20.s, #0x0\n"
+ "mov z21.s, #0x0\n"
+ "mov z22.s, #0x0\n"
+ "mov z23.s, #0x0\n"
+ "mov z24.s, #0x0\n"
+ "mov z25.s, #0x0\n"
+ "mov z26.s, #0x0\n"
+ "mov z27.s, #0x0\n"
+ "mov z28.s, #0x0\n"
+ "mov z29.s, #0x0\n"
+ "mov z30.s, #0x0\n"
+ "mov z31.s, #0x0\n"
+ "49:" // Height 4: setup done
+ "mov x28, #0x0\n"
+ "50:" // Height 4: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w27, [x20, x28, LSL #0x2]\n"
+ "tbz %x[flags], #3, 51f\n"
+ "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x26, [x20, #0x0]\n"
+ "ldr x24, [x20, #0x8]\n"
+ "ldr x22, [x20, #0x10]\n"
+ "ldr x20, [x20, #0x18]\n"
+ "cbnz x28, 52f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x26, x26, x19\n"
+ "add x24, x24, x19\n"
+ "add x22, x22, x19\n"
+ "add x20, x20, x19\n"
+ "b 52f\n"
+ "51:" // Height 4: setup direct input
+ "mov x26, %x[input_ptr]\n"
+ "add x24, x26, x19\n"
+ "add x22, x24, x19\n"
+ "add x20, x22, x19\n"
+ "52:" // Height 4: input setup done
+ "cmp x27, #0x10\n"
+ "ble 55f\n"
+ "53:" // Height 4: Multiply loop: Main loop head
+ "ld1b { z4.b }, p2/Z, [x11]\n"
+ "whilelt p0.b, XZR, x27\n"
+ "ld1b { z5.b }, p2/Z, [x11, #1, MUL VL]\n"
+ "ld1rqb { z0.b }, p0/Z, [x26]\n"
+ "udot z16.s, z4.b, z0.b[0]\n"
+ "ld1rqb { z1.b }, p0/Z, [x24]\n"
+ "add x26, x26, #0x10\n"
+ "udot z17.s, z5.b, z0.b[0]\n"
+ "ld1rqb { z2.b }, p0/Z, [x22]\n"
+ "add x24, x24, #0x10\n"
+ "udot z20.s, z4.b, z1.b[0]\n"
+ "ld1rqb { z3.b }, p0/Z, [x20]\n"
+ "add x22, x22, #0x10\n"
+ "udot z24.s, z4.b, z2.b[0]\n"
+ "ld1b { z6.b }, p2/Z, [x11, #2, MUL VL]\n"
+ "add x20, x20, #0x10\n"
+ "udot z21.s, z5.b, z1.b[0]\n"
+ "ld1b { z7.b }, p2/Z, [x11, #3, MUL VL]\n"
+ "udot z25.s, z5.b, z2.b[0]\n"
+ "ld1b { z8.b }, p2/Z, [x11, #4, MUL VL]\n"
+ "udot z28.s, z4.b, z3.b[0]\n"
+ "ld1b { z9.b }, p2/Z, [x11, #5, MUL VL]\n"
+ "udot z29.s, z5.b, z3.b[0]\n"
+ "ld1b { z10.b }, p2/Z, [x11, #6, MUL VL]\n"
+ "udot z18.s, z6.b, z0.b[0]\n"
+ "ld1b { z4.b }, p2/Z, [x11, #7, MUL VL]\n"
+ "addvl x11, x11, #16\n"
+ "udot z22.s, z6.b, z1.b[0]\n"
+ "ld1b { z5.b }, p2/Z, [x11, #-8, MUL VL]\n"
+ "udot z26.s, z6.b, z2.b[0]\n"
+ "udot z30.s, z6.b, z3.b[0]\n"
+ "ld1b { z6.b }, p2/Z, [x11, #-7, MUL VL]\n"
+ "udot z19.s, z7.b, z0.b[0]\n"
+ "udot z23.s, z7.b, z1.b[0]\n"
+ "udot z27.s, z7.b, z2.b[0]\n"
+ "udot z31.s, z7.b, z3.b[0]\n"
+ "ld1b { z7.b }, p2/Z, [x11, #-6, MUL VL]\n"
+ "udot z16.s, z8.b, z0.b[1]\n"
+ "udot z20.s, z8.b, z1.b[1]\n"
+ "udot z24.s, z8.b, z2.b[1]\n"
+ "udot z28.s, z8.b, z3.b[1]\n"
+ "ld1b { z8.b }, p2/Z, [x11, #-5, MUL VL]\n"
+ "udot z17.s, z9.b, z0.b[1]\n"
+ "udot z21.s, z9.b, z1.b[1]\n"
+ "udot z25.s, z9.b, z2.b[1]\n"
+ "udot z29.s, z9.b, z3.b[1]\n"
+ "ld1b { z9.b }, p2/Z, [x11, #-4, MUL VL]\n"
+ "udot z18.s, z10.b, z0.b[1]\n"
+ "udot z22.s, z10.b, z1.b[1]\n"
+ "udot z26.s, z10.b, z2.b[1]\n"
+ "udot z30.s, z10.b, z3.b[1]\n"
+ "ld1b { z10.b }, p2/Z, [x11, #-3, MUL VL]\n"
+ "udot z19.s, z4.b, z0.b[1]\n"
+ "udot z23.s, z4.b, z1.b[1]\n"
+ "udot z27.s, z4.b, z2.b[1]\n"
+ "udot z31.s, z4.b, z3.b[1]\n"
+ "ld1b { z4.b }, p2/Z, [x11, #-2, MUL VL]\n"
+ "udot z16.s, z5.b, z0.b[2]\n"
+ "udot z20.s, z5.b, z1.b[2]\n"
+ "udot z24.s, z5.b, z2.b[2]\n"
+ "udot z28.s, z5.b, z3.b[2]\n"
+ "ld1b { z5.b }, p2/Z, [x11, #-1, MUL VL]\n"
+ "udot z17.s, z6.b, z0.b[2]\n"
+ "udot z21.s, z6.b, z1.b[2]\n"
+ "udot z25.s, z6.b, z2.b[2]\n"
+ "udot z29.s, z6.b, z3.b[2]\n"
+ "udot z18.s, z7.b, z0.b[2]\n"
+ "udot z22.s, z7.b, z1.b[2]\n"
+ "udot z26.s, z7.b, z2.b[2]\n"
+ "udot z30.s, z7.b, z3.b[2]\n"
+ "udot z19.s, z8.b, z0.b[2]\n"
+ "udot z23.s, z8.b, z1.b[2]\n"
+ "udot z27.s, z8.b, z2.b[2]\n"
+ "udot z31.s, z8.b, z3.b[2]\n"
+ "udot z16.s, z9.b, z0.b[3]\n"
+ "udot z20.s, z9.b, z1.b[3]\n"
+ "udot z24.s, z9.b, z2.b[3]\n"
+ "udot z28.s, z9.b, z3.b[3]\n"
+ "udot z17.s, z10.b, z0.b[3]\n"
+ "udot z21.s, z10.b, z1.b[3]\n"
+ "udot z25.s, z10.b, z2.b[3]\n"
+ "udot z29.s, z10.b, z3.b[3]\n"
+ "udot z18.s, z4.b, z0.b[3]\n"
+ "udot z22.s, z4.b, z1.b[3]\n"
+ "udot z26.s, z4.b, z2.b[3]\n"
+ "udot z30.s, z4.b, z3.b[3]\n"
+ "udot z19.s, z5.b, z0.b[3]\n"
+ "udot z23.s, z5.b, z1.b[3]\n"
+ "udot z27.s, z5.b, z2.b[3]\n"
+ "udot z31.s, z5.b, z3.b[3]\n"
+ "tbnz %x[flags], #31, 54f\n"
+ "udot z11.s, z0.b, z15.b\n"
+ "udot z12.s, z1.b, z15.b\n"
+ "udot z13.s, z2.b, z15.b\n"
+ "udot z14.s, z3.b, z15.b\n"
+ "54:" // Height 4: Multiply loop: unique 7: skip row sum
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "sub x27, x27, #0x10\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "cmp x27, #0x10\n"
+ "prfm pldl1keep, [x22, #0x80]\n"
+ "prfm pldl1keep, [x20, #0x80]\n"
+ "bgt 53b\n"
+ "55:" // Height 4: Multiply loop: Single iteration only
+ "ld1b { z6.b }, p2/Z, [x11]\n"
+ "whilelt p0.b, XZR, x27\n"
+ "ld1b { z7.b }, p2/Z, [x11, #1, MUL VL]\n"
+ "subs x27, x27, #0x4\n"
+ "ld1rqb { z0.b }, p0/Z, [x26]\n"
+ "udot z16.s, z6.b, z0.b[0]\n"
+ "ld1rqb { z1.b }, p0/Z, [x24]\n"
+ "add x26, x26, #0x10\n"
+ "udot z17.s, z7.b, z0.b[0]\n"
+ "ld1rqb { z2.b }, p0/Z, [x22]\n"
+ "add x24, x24, #0x10\n"
+ "udot z20.s, z6.b, z1.b[0]\n"
+ "ld1rqb { z3.b }, p0/Z, [x20]\n"
+ "add x22, x22, #0x10\n"
+ "udot z24.s, z6.b, z2.b[0]\n"
+ "ld1b { z8.b }, p2/Z, [x11, #2, MUL VL]\n"
+ "add x20, x20, #0x10\n"
+ "udot z21.s, z7.b, z1.b[0]\n"
+ "ld1b { z9.b }, p2/Z, [x11, #3, MUL VL]\n"
+ "addvl x11, x11, #4\n"
+ "udot z28.s, z6.b, z3.b[0]\n"
+ "udot z25.s, z7.b, z2.b[0]\n"
+ "udot z29.s, z7.b, z3.b[0]\n"
+ "udot z18.s, z8.b, z0.b[0]\n"
+ "udot z22.s, z8.b, z1.b[0]\n"
+ "udot z26.s, z8.b, z2.b[0]\n"
+ "udot z30.s, z8.b, z3.b[0]\n"
+ "udot z19.s, z9.b, z0.b[0]\n"
+ "udot z23.s, z9.b, z1.b[0]\n"
+ "udot z27.s, z9.b, z2.b[0]\n"
+ "udot z31.s, z9.b, z3.b[0]\n"
+ "ble 56f\n"
+ "ld1b { z10.b }, p2/Z, [x11]\n"
+ "udot z16.s, z10.b, z0.b[1]\n"
+ "ld1b { z4.b }, p2/Z, [x11, #1, MUL VL]\n"
+ "subs x27, x27, #0x4\n"
+ "udot z20.s, z10.b, z1.b[1]\n"
+ "ld1b { z5.b }, p2/Z, [x11, #2, MUL VL]\n"
+ "udot z24.s, z10.b, z2.b[1]\n"
+ "ld1b { z6.b }, p2/Z, [x11, #3, MUL VL]\n"
+ "addvl x11, x11, #4\n"
+ "udot z28.s, z10.b, z3.b[1]\n"
+ "udot z17.s, z4.b, z0.b[1]\n"
+ "udot z21.s, z4.b, z1.b[1]\n"
+ "udot z25.s, z4.b, z2.b[1]\n"
+ "udot z29.s, z4.b, z3.b[1]\n"
+ "udot z18.s, z5.b, z0.b[1]\n"
+ "udot z22.s, z5.b, z1.b[1]\n"
+ "udot z26.s, z5.b, z2.b[1]\n"
+ "udot z30.s, z5.b, z3.b[1]\n"
+ "udot z19.s, z6.b, z0.b[1]\n"
+ "udot z23.s, z6.b, z1.b[1]\n"
+ "udot z27.s, z6.b, z2.b[1]\n"
+ "udot z31.s, z6.b, z3.b[1]\n"
+ "ble 56f\n"
+ "ld1b { z7.b }, p2/Z, [x11]\n"
+ "udot z16.s, z7.b, z0.b[2]\n"
+ "ld1b { z8.b }, p2/Z, [x11, #1, MUL VL]\n"
+ "subs x27, x27, #0x4\n"
+ "udot z20.s, z7.b, z1.b[2]\n"
+ "ld1b { z9.b }, p2/Z, [x11, #2, MUL VL]\n"
+ "udot z24.s, z7.b, z2.b[2]\n"
+ "ld1b { z10.b }, p2/Z, [x11, #3, MUL VL]\n"
+ "addvl x11, x11, #4\n"
+ "udot z28.s, z7.b, z3.b[2]\n"
+ "udot z17.s, z8.b, z0.b[2]\n"
+ "udot z21.s, z8.b, z1.b[2]\n"
+ "udot z25.s, z8.b, z2.b[2]\n"
+ "udot z29.s, z8.b, z3.b[2]\n"
+ "udot z18.s, z9.b, z0.b[2]\n"
+ "udot z22.s, z9.b, z1.b[2]\n"
+ "udot z26.s, z9.b, z2.b[2]\n"
+ "udot z30.s, z9.b, z3.b[2]\n"
+ "udot z19.s, z10.b, z0.b[2]\n"
+ "udot z23.s, z10.b, z1.b[2]\n"
+ "udot z27.s, z10.b, z2.b[2]\n"
+ "udot z31.s, z10.b, z3.b[2]\n"
+ "ble 56f\n"
+ "ld1b { z4.b }, p2/Z, [x11]\n"
+ "udot z16.s, z4.b, z0.b[3]\n"
+ "ld1b { z5.b }, p2/Z, [x11, #1, MUL VL]\n"
+ "udot z20.s, z4.b, z1.b[3]\n"
+ "ld1b { z6.b }, p2/Z, [x11, #2, MUL VL]\n"
+ "udot z24.s, z4.b, z2.b[3]\n"
+ "ld1b { z7.b }, p2/Z, [x11, #3, MUL VL]\n"
+ "addvl x11, x11, #4\n"
+ "udot z28.s, z4.b, z3.b[3]\n"
+ "udot z17.s, z5.b, z0.b[3]\n"
+ "udot z21.s, z5.b, z1.b[3]\n"
+ "udot z25.s, z5.b, z2.b[3]\n"
+ "udot z29.s, z5.b, z3.b[3]\n"
+ "udot z18.s, z6.b, z0.b[3]\n"
+ "udot z22.s, z6.b, z1.b[3]\n"
+ "udot z26.s, z6.b, z2.b[3]\n"
+ "udot z30.s, z6.b, z3.b[3]\n"
+ "udot z19.s, z7.b, z0.b[3]\n"
+ "udot z23.s, z7.b, z1.b[3]\n"
+ "udot z27.s, z7.b, z2.b[3]\n"
+ "udot z31.s, z7.b, z3.b[3]\n"
+ "56:" // Height 4: Multiply loop: multiply skip
+ "tbnz %x[flags], #31, 57f\n"
+ "udot z11.s, z0.b, z15.b\n"
+ "udot z12.s, z1.b, z15.b\n"
+ "udot z13.s, z2.b, z15.b\n"
+ "udot z14.s, z3.b, z15.b\n"
+ "57:" // Height 4: Multiply loop: unique 8: skip row sum
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "add x28, x28, #0x1\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "prfm pldl1keep, [x22, #0x80]\n"
+ "prfm pldl1keep, [x20, #0x80]\n"
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "cmp x28, x19\n"
+ "bne 50b\n"
+ "prfm pstl1keep, [x9, #0x0]\n"
+ "prfm pstl1keep, [x25, #0x0]\n"
+ "prfm pstl1keep, [x23, #0x0]\n"
+ "prfm pstl1keep, [x21, #0x0]\n"
+ "tbnz %x[flags], #31, 58f\n"
+ "add x19, %x[qp], %[b_offset]\n"
+ "ld1rw { z4.s }, p2/Z, [x19]\n"
+ "neg z4.s, p2/M, z4.s\n"
+ "mov x20, #0x4\n"
+ "mov x19, #0x4\n"
+ "whilelt p0.s, XZR, x20\n"
+ "uaddv d11, p0, z11.s\n"
+ "whilelt p0.s, XZR, x19\n"
+ "uaddv d12, p0, z12.s\n"
+ "mov x19, #0x4\n"
+ "mov z11.s, z11.s[0]\n"
+ "whilelt p0.s, XZR, x19\n"
+ "mov x19, #0x4\n"
+ "mov z12.s, z12.s[0]\n"
+ "uaddv d13, p0, z13.s\n"
+ "whilelt p0.s, XZR, x19\n"
+ "mul z11.s, p2/M, z11.s, z4.s\n"
+ "uaddv d14, p0, z14.s\n"
+ "mul z12.s, p2/M, z12.s, z4.s\n"
+ "mov z13.s, z13.s[0]\n"
+ "mul z13.s, p2/M, z13.s, z4.s\n"
+ "mov z14.s, z14.s[0]\n"
+ "mul z14.s, p2/M, z14.s, z4.s\n"
+ "58:" // Height 4: skip row sum fixup
+ "add z16.s, z16.s, z11.s\n"
+ "ld1w { z0.s }, p2/Z, [x10]\n"
+ "orr %x[flags], %x[flags], #0x80000000\n"
+ "add z17.s, z17.s, z11.s\n"
+ "ld1w { z1.s }, p2/Z, [x10, #1, MUL VL]\n"
+ "add x20, %x[qp], %[per_layer_right_shift]\n"
+ "add z18.s, z18.s, z11.s\n"
+ "ld1w { z2.s }, p2/Z, [x10, #2, MUL VL]\n"
+ "add x19, %x[qp], %[per_layer_mul]\n"
+ "add z19.s, z19.s, z11.s\n"
+ "ld1w { z3.s }, p2/Z, [x10, #3, MUL VL]\n"
+ "addvl x10, x10, #4\n"
+ "add z20.s, z20.s, z12.s\n"
+ "ld1rw { z4.s }, p2/Z, [x19]\n"
+ "add z21.s, z21.s, z12.s\n"
+ "add z22.s, z22.s, z12.s\n"
+ "add z23.s, z23.s, z12.s\n"
+ "add z24.s, z24.s, z13.s\n"
+ "add z25.s, z25.s, z13.s\n"
+ "add z26.s, z26.s, z13.s\n"
+ "add z27.s, z27.s, z13.s\n"
+ "add z28.s, z28.s, z14.s\n"
+ "add z29.s, z29.s, z14.s\n"
+ "add z30.s, z30.s, z14.s\n"
+ "add z31.s, z31.s, z14.s\n"
+ "add z16.s, z16.s, z0.s\n"
+ "add z17.s, z17.s, z1.s\n"
+ "add z18.s, z18.s, z2.s\n"
+ "add z19.s, z19.s, z3.s\n"
+ "add z20.s, z20.s, z0.s\n"
+ "add z21.s, z21.s, z1.s\n"
+ "add z22.s, z22.s, z2.s\n"
+ "add z23.s, z23.s, z3.s\n"
+ "add z24.s, z24.s, z0.s\n"
+ "add z25.s, z25.s, z1.s\n"
+ "add z26.s, z26.s, z2.s\n"
+ "add z27.s, z27.s, z3.s\n"
+ "add z28.s, z28.s, z0.s\n"
+ "ld1rw { z0.s }, p2/Z, [x20]\n"
+ "add z29.s, z29.s, z1.s\n"
+ "add z30.s, z30.s, z2.s\n"
+ "add z31.s, z31.s, z3.s\n"
+ ".inst 0x04a47610 // sqrdmulh z16.s, z16.s, z4.s\n"
+ ".inst 0x04a47631 // sqrdmulh z17.s, z17.s, z4.s\n"
+ ".inst 0x04a47652 // sqrdmulh z18.s, z18.s, z4.s\n"
+ ".inst 0x04a47673 // sqrdmulh z19.s, z19.s, z4.s\n"
+ ".inst 0x04a47694 // sqrdmulh z20.s, z20.s, z4.s\n"
+ ".inst 0x04a476b5 // sqrdmulh z21.s, z21.s, z4.s\n"
+ ".inst 0x04a476d6 // sqrdmulh z22.s, z22.s, z4.s\n"
+ ".inst 0x04a476f7 // sqrdmulh z23.s, z23.s, z4.s\n"
+ ".inst 0x04a47718 // sqrdmulh z24.s, z24.s, z4.s\n"
+ ".inst 0x04a47739 // sqrdmulh z25.s, z25.s, z4.s\n"
+ ".inst 0x04a4775a // sqrdmulh z26.s, z26.s, z4.s\n"
+ ".inst 0x04a4777b // sqrdmulh z27.s, z27.s, z4.s\n"
+ ".inst 0x04a4779c // sqrdmulh z28.s, z28.s, z4.s\n"
+ ".inst 0x04a477bd // sqrdmulh z29.s, z29.s, z4.s\n"
+ ".inst 0x04a477de // sqrdmulh z30.s, z30.s, z4.s\n"
+ ".inst 0x04a477ff // sqrdmulh z31.s, z31.s, z4.s\n"
+ "tbz %x[flags], #5, 59f\n"
+ "and z4.d, z16.d, z0.d\n"
+ "asr z4.s, z4.s, #0x1f\n"
+ "and z5.d, z17.d, z0.d\n"
+ "and z6.d, z18.d, z0.d\n"
+ "asr z5.s, z5.s, #0x1f\n"
+ "and z7.d, z19.d, z0.d\n"
+ "and z8.d, z20.d, z0.d\n"
+ "asr z6.s, z6.s, #0x1f\n"
+ "and z9.d, z21.d, z0.d\n"
+ "asr z7.s, z7.s, #0x1f\n"
+ "sqadd z16.s, z16.s, z4.s\n"
+ "and z10.d, z22.d, z0.d\n"
+ "asr z8.s, z8.s, #0x1f\n"
+ "and z4.d, z23.d, z0.d\n"
+ "asr z9.s, z9.s, #0x1f\n"
+ "sqadd z17.s, z17.s, z5.s\n"
+ "asr z10.s, z10.s, #0x1f\n"
+ "sqadd z18.s, z18.s, z6.s\n"
+ "asr z4.s, z4.s, #0x1f\n"
+ "and z5.d, z24.d, z0.d\n"
+ "asr z5.s, z5.s, #0x1f\n"
+ "sqadd z19.s, z19.s, z7.s\n"
+ "sqadd z20.s, z20.s, z8.s\n"
+ "sqadd z21.s, z21.s, z9.s\n"
+ "sqadd z22.s, z22.s, z10.s\n"
+ "sqadd z23.s, z23.s, z4.s\n"
+ "and z6.d, z25.d, z0.d\n"
+ "asr z6.s, z6.s, #0x1f\n"
+ "sqadd z24.s, z24.s, z5.s\n"
+ "and z7.d, z26.d, z0.d\n"
+ "asr z7.s, z7.s, #0x1f\n"
+ "and z8.d, z27.d, z0.d\n"
+ "and z9.d, z28.d, z0.d\n"
+ "asr z8.s, z8.s, #0x1f\n"
+ "sqadd z25.s, z25.s, z6.s\n"
+ "and z10.d, z29.d, z0.d\n"
+ "asr z9.s, z9.s, #0x1f\n"
+ "and z4.d, z30.d, z0.d\n"
+ "asr z10.s, z10.s, #0x1f\n"
+ "sqadd z26.s, z26.s, z7.s\n"
+ "and z5.d, z31.d, z0.d\n"
+ "asr z4.s, z4.s, #0x1f\n"
+ "sqadd z27.s, z27.s, z8.s\n"
+ "asr z5.s, z5.s, #0x1f\n"
+ "sqadd z28.s, z28.s, z9.s\n"
+ "sqadd z29.s, z29.s, z10.s\n"
+ "sqadd z30.s, z30.s, z4.s\n"
+ "sqadd z31.s, z31.s, z5.s\n"
+ "59:" // Height 4: no shift correction
+ ".inst 0x44828810 // srshl z16.s, p2/M, z16.s, z0.s\n"
+ "add x19, %x[qp], %[c_offset]\n"
+ "ld1rw { z4.s }, p2/Z, [x19]\n"
+ ".inst 0x44828811 // srshl z17.s, p2/M, z17.s, z0.s\n"
+ "add x19, %x[qp], %[minval]\n"
+ ".inst 0x44828812 // srshl z18.s, p2/M, z18.s, z0.s\n"
+ "ld1rw { z5.s }, p2/Z, [x19]\n"
+ "add x19, %x[qp], %[maxval]\n"
+ ".inst 0x44828813 // srshl z19.s, p2/M, z19.s, z0.s\n"
+ "ld1rw { z6.s }, p2/Z, [x19]\n"
+ ".inst 0x44828814 // srshl z20.s, p2/M, z20.s, z0.s\n"
+ "add z16.s, z16.s, z4.s\n"
+ "add z17.s, z17.s, z4.s\n"
+ "add z18.s, z18.s, z4.s\n"
+ "add z19.s, z19.s, z4.s\n"
+ "add z20.s, z20.s, z4.s\n"
+ "smin z16.s, p2/M, z16.s, z6.s\n"
+ "smin z17.s, p2/M, z17.s, z6.s\n"
+ "smin z18.s, p2/M, z18.s, z6.s\n"
+ "smin z19.s, p2/M, z19.s, z6.s\n"
+ "smax z16.s, p2/M, z16.s, z5.s\n"
+ "smax z17.s, p2/M, z17.s, z5.s\n"
+ "smax z18.s, p2/M, z18.s, z5.s\n"
+ "smax z19.s, p2/M, z19.s, z5.s\n"
+ "smin z20.s, p2/M, z20.s, z6.s\n"
+ "uzp1 z16.h, z16.h, z17.h\n"
+ ".inst 0x44828815 // srshl z21.s, p2/M, z21.s, z0.s\n"
+ "uzp1 z17.h, z18.h, z19.h\n"
+ "smax z20.s, p2/M, z20.s, z5.s\n"
+ "uzp1 z16.b, z16.b, z17.b\n"
+ "st1b { z16.b }, p1, [x9]\n"
+ "add z21.s, z21.s, z4.s\n"
+ "addvl x9, x9, #1\n"
+ ".inst 0x44828816 // srshl z22.s, p2/M, z22.s, z0.s\n"
+ ".inst 0x44828817 // srshl z23.s, p2/M, z23.s, z0.s\n"
+ ".inst 0x44828818 // srshl z24.s, p2/M, z24.s, z0.s\n"
+ "smin z21.s, p2/M, z21.s, z6.s\n"
+ ".inst 0x44828819 // srshl z25.s, p2/M, z25.s, z0.s\n"
+ "add z22.s, z22.s, z4.s\n"
+ "add z23.s, z23.s, z4.s\n"
+ "add z24.s, z24.s, z4.s\n"
+ "add z25.s, z25.s, z4.s\n"
+ "smax z21.s, p2/M, z21.s, z5.s\n"
+ "smin z22.s, p2/M, z22.s, z6.s\n"
+ "smin z23.s, p2/M, z23.s, z6.s\n"
+ "smin z24.s, p2/M, z24.s, z6.s\n"
+ "uzp1 z20.h, z20.h, z21.h\n"
+ "smax z22.s, p2/M, z22.s, z5.s\n"
+ "smax z23.s, p2/M, z23.s, z5.s\n"
+ "smax z24.s, p2/M, z24.s, z5.s\n"
+ "smin z25.s, p2/M, z25.s, z6.s\n"
+ ".inst 0x4482881a // srshl z26.s, p2/M, z26.s, z0.s\n"
+ "uzp1 z21.h, z22.h, z23.h\n"
+ ".inst 0x4482881b // srshl z27.s, p2/M, z27.s, z0.s\n"
+ "uzp1 z20.b, z20.b, z21.b\n"
+ "st1b { z20.b }, p1, [x25]\n"
+ "add z26.s, z26.s, z4.s\n"
+ "addvl x25, x25, #1\n"
+ "add z27.s, z27.s, z4.s\n"
+ "smax z25.s, p2/M, z25.s, z5.s\n"
+ ".inst 0x4482881c // srshl z28.s, p2/M, z28.s, z0.s\n"
+ "smin z26.s, p2/M, z26.s, z6.s\n"
+ "smin z27.s, p2/M, z27.s, z6.s\n"
+ "uzp1 z24.h, z24.h, z25.h\n"
+ "add z28.s, z28.s, z4.s\n"
+ "smax z26.s, p2/M, z26.s, z5.s\n"
+ "smax z27.s, p2/M, z27.s, z5.s\n"
+ "smin z28.s, p2/M, z28.s, z6.s\n"
+ ".inst 0x4482881d // srshl z29.s, p2/M, z29.s, z0.s\n"
+ ".inst 0x4482881e // srshl z30.s, p2/M, z30.s, z0.s\n"
+ "uzp1 z25.h, z26.h, z27.h\n"
+ "smax z28.s, p2/M, z28.s, z5.s\n"
+ "add z29.s, z29.s, z4.s\n"
+ "add z30.s, z30.s, z4.s\n"
+ "uzp1 z24.b, z24.b, z25.b\n"
+ "st1b { z24.b }, p1, [x23]\n"
+ "smin z29.s, p2/M, z29.s, z6.s\n"
+ "addvl x23, x23, #1\n"
+ "smin z30.s, p2/M, z30.s, z6.s\n"
+ ".inst 0x4482881f // srshl z31.s, p2/M, z31.s, z0.s\n"
+ "smax z29.s, p2/M, z29.s, z5.s\n"
+ "add z31.s, z31.s, z4.s\n"
+ "smax z30.s, p2/M, z30.s, z5.s\n"
+ "uzp1 z28.h, z28.h, z29.h\n"
+ "smin z31.s, p2/M, z31.s, z6.s\n"
+ "smax z31.s, p2/M, z31.s, z5.s\n"
+ "uzp1 z29.h, z30.h, z31.h\n"
+ "uzp1 z28.b, z28.b, z29.b\n"
+ "st1b { z28.b }, p1, [x21]\n"
+ "addvl x21, x21, #1\n"
+ "60:" // Height 4: Writeback done
+ "mov x19, #0x0\n"
+ "incw x19, ALL, MUL #4\n"
+ "subs x12, x12, x19\n"
+ "bgt 48b\n"
+ "subs %x[M], %x[M], #0x4\n"
+ "beq 62f\n"
+ "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "tbz %x[flags], #3, 61f\n"
+ "add x20, x20, #0x4\n"
+ "str x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "b 1b\n"
+ "61:" // Update direct input
+ "mov x19, #0x4\n"
+ "madd %x[input_ptr], x19, x20, %x[input_ptr]\n"
+ "b 1b\n"
+ "62:" // Exit
+
+ : [M] "+r" (M), [flags] "+r" (flags), [input_ptr] "+r" (input_ptr), [output_ptr] "+r" (output_ptr)
+ : [args_ptr] "r" (&ka), [b_offset] "I" (offsetof(Requantize32, b_offset)), [c_offset] "I" (offsetof(Requantize32, c_offset)), [col_bias] "r" (col_bias), [maxval] "I" (offsetof(Requantize32, maxval)), [minval] "I" (offsetof(Requantize32, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths)), [per_layer_mul] "I" (offsetof(Requantize32, per_layer_mul)), [per_layer_right_shift] "I" (offsetof(Requantize32, per_layer_right_shift)), [qp] "r" (qp)
+ : "cc", "memory", "p0", "p1", "p2", "x9", "x10", "x11", "x12", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+ );
+}
+
+} // namespace arm_gemm
+#endif // __ARM_FEATURE_SVE
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8u32_dot_4VLx4/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8u32_dot_4VLx4/generic.cpp
deleted file mode 100644
index 565832e..0000000
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8u32_dot_4VLx4/generic.cpp
+++ /dev/null
@@ -1,2137 +0,0 @@
-/*
- * Copyright (c) 2018-2020 Arm Limited.
- *
- * SPDX-License-Identifier: MIT
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to
- * deal in the Software without restriction, including without limitation the
- * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
- * sell copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-#ifdef __ARM_FEATURE_SVE
-
-#include <algorithm>
-
-#include "arm_gemm.hpp"
-#include <cstdint>
-#include "../../asmlib.hpp"
-#include "../../utils.hpp"
-
-namespace arm_gemm {
-
-void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *A, int lda, const uint8_t *B, uint32_t *C, int ldc, int M, int N, int K, const uint32_t *, Activation , bool accumulate) {
- const int K_stride = ((K + 3) / 4) * 4;
- const long loops_count = ((K + 16) / 32) - 1;
- K -= loops_count * 32;
- const long regs_count = (K / 16) - 1;
- K -= (regs_count + 1) * 16;
- const long leftovers = K;
- const long blocks_count = (K + 3) / 4;
-
- int rows_to_compute;
-
- for (int y=0; y<M; y+=rows_to_compute) {
- const uint8_t * const a_ptr0_base = A + (y * lda);
- const unsigned long ldab = lda * sizeof(uint8_t);
-
- uint32_t *c_ptr0 = C + (y * ldc);
-
- rows_to_compute = M-y;
- if (rows_to_compute > 4) {
- if (rows_to_compute % 4) {
- rows_to_compute = 4 - 1;
- } else {
- rows_to_compute = 4;
- }
- }
-
- for (int x0=0; x0<N; x0+=(4 * get_vector_length<uint32_t>())) {
- const long width = std::min((unsigned long)N-x0, (4 * get_vector_length<uint32_t>()));
- long loops = loops_count;
- long regs = regs_count;
- long temp = 0;
- long blocks = blocks_count;
- const uint8_t *a_ptr0 = a_ptr0_base;
- const uint8_t *b_ptr0 = B + (K_stride * x0);
- const unsigned long ldcb = ldc * sizeof(uint32_t);
-
- switch(rows_to_compute) {
- case 1:
- __asm __volatile (
- "whilelt p6.b, %[temp], %[leftovers]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "ptrue p7.b\n"
- "whilelt p1.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "whilelt p2.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "whilelt p3.s, %[temp], %[width]\n"
- "cbnz %[accumulate], 1f\n"
- "mov z16.s, #0\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "mov z17.s, #0\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
- "mov z18.s, #0\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "mov z19.s, #0\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "ld1w z16.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n"
- "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n"
- "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "cbz %[loops], 2f\n"
- "3:\n"
- "udot z16.s, z8.b, z0.b[0]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "udot z17.s, z9.b, z0.b[0]\n"
- "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n"
- "udot z18.s, z10.b, z0.b[0]\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
- "udot z19.s, z11.b, z0.b[0]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "udot z16.s, z12.b, z0.b[1]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "udot z17.s, z13.b, z0.b[1]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "udot z18.s, z14.b, z0.b[1]\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "udot z19.s, z15.b, z0.b[1]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "udot z16.s, z8.b, z0.b[2]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "udot z17.s, z9.b, z0.b[2]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "udot z18.s, z10.b, z0.b[2]\n"
- "subs %[loops], %[loops], #0x1\n"
- "udot z19.s, z11.b, z0.b[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "udot z16.s, z12.b, z0.b[3]\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
- "udot z17.s, z13.b, z0.b[3]\n"
- "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "udot z18.s, z14.b, z0.b[3]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "udot z19.s, z15.b, z0.b[3]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "udot z16.s, z8.b, z4.b[0]\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "udot z17.s, z9.b, z4.b[0]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "udot z18.s, z10.b, z4.b[0]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "udot z19.s, z11.b, z4.b[0]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "udot z16.s, z12.b, z4.b[1]\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0], #-0x10]\n"
- "udot z17.s, z13.b, z4.b[1]\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
- "udot z18.s, z14.b, z4.b[1]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "udot z19.s, z15.b, z4.b[1]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "udot z16.s, z8.b, z4.b[2]\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "udot z17.s, z9.b, z4.b[2]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "udot z18.s, z10.b, z4.b[2]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "udot z19.s, z11.b, z4.b[2]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "udot z16.s, z12.b, z4.b[3]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "udot z17.s, z13.b, z4.b[3]\n"
- "udot z18.s, z14.b, z4.b[3]\n"
- "udot z19.s, z15.b, z4.b[3]\n"
- "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "b.ne 3b\n"
- "2:\n"
- "cbz %[regs], 4f\n"
- "udot z16.s, z8.b, z0.b[0]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "udot z17.s, z9.b, z0.b[0]\n"
- "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n"
- "udot z18.s, z10.b, z0.b[0]\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
- "udot z19.s, z11.b, z0.b[0]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "udot z16.s, z12.b, z0.b[1]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "udot z17.s, z13.b, z0.b[1]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "udot z18.s, z14.b, z0.b[1]\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "udot z19.s, z15.b, z0.b[1]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "udot z16.s, z8.b, z0.b[2]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "udot z17.s, z9.b, z0.b[2]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "udot z18.s, z10.b, z0.b[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "udot z19.s, z11.b, z0.b[2]\n"
- "udot z16.s, z12.b, z0.b[3]\n"
- "udot z17.s, z13.b, z0.b[3]\n"
- "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "udot z18.s, z14.b, z0.b[3]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "udot z19.s, z15.b, z0.b[3]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "udot z16.s, z8.b, z4.b[0]\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "udot z17.s, z9.b, z4.b[0]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "udot z18.s, z10.b, z4.b[0]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "udot z19.s, z11.b, z4.b[0]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "udot z16.s, z12.b, z4.b[1]\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
- "udot z17.s, z13.b, z4.b[1]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "udot z18.s, z14.b, z4.b[1]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "udot z19.s, z15.b, z4.b[1]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "udot z16.s, z8.b, z4.b[2]\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "udot z17.s, z9.b, z4.b[2]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "udot z18.s, z10.b, z4.b[2]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "udot z19.s, z11.b, z4.b[2]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "udot z16.s, z12.b, z4.b[3]\n"
- "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x10]\n"
- "udot z17.s, z13.b, z4.b[3]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "udot z18.s, z14.b, z4.b[3]\n"
- "addvl %[a_ptr0], %[a_ptr0], #2\n"
- "udot z19.s, z15.b, z4.b[3]\n"
- "cbz %[blocks], 5f\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "udot z16.s, z8.b, z0.b[0]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "udot z17.s, z9.b, z0.b[0]\n"
- "udot z18.s, z10.b, z0.b[0]\n"
- "udot z19.s, z11.b, z0.b[0]\n"
- "b.eq 5f\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "udot z16.s, z12.b, z0.b[1]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "udot z17.s, z13.b, z0.b[1]\n"
- "udot z18.s, z14.b, z0.b[1]\n"
- "udot z19.s, z15.b, z0.b[1]\n"
- "b.eq 5f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "udot z16.s, z8.b, z0.b[2]\n"
- "udot z17.s, z9.b, z0.b[2]\n"
- "udot z18.s, z10.b, z0.b[2]\n"
- "udot z19.s, z11.b, z0.b[2]\n"
- "b.eq 5f\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "udot z16.s, z12.b, z0.b[3]\n"
- "udot z17.s, z13.b, z0.b[3]\n"
- "udot z18.s, z14.b, z0.b[3]\n"
- "udot z19.s, z15.b, z0.b[3]\n"
- "b 5f\n"
- "4:\n"
- "udot z16.s, z8.b, z0.b[0]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "udot z17.s, z9.b, z0.b[0]\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
- "udot z18.s, z10.b, z0.b[0]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "udot z19.s, z11.b, z0.b[0]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "udot z16.s, z12.b, z0.b[1]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "udot z17.s, z13.b, z0.b[1]\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "udot z18.s, z14.b, z0.b[1]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "udot z19.s, z15.b, z0.b[1]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "udot z16.s, z8.b, z0.b[2]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "udot z17.s, z9.b, z0.b[2]\n"
- "ld1rqb z4.b, p6/z, [%[a_ptr0]]\n"
- "udot z18.s, z10.b, z0.b[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "udot z19.s, z11.b, z0.b[2]\n"
- "addvl %[a_ptr0], %[a_ptr0], #1\n"
- "udot z16.s, z12.b, z0.b[3]\n"
- "udot z17.s, z13.b, z0.b[3]\n"
- "udot z18.s, z14.b, z0.b[3]\n"
- "udot z19.s, z15.b, z0.b[3]\n"
- "cbz %[blocks], 5f\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "udot z16.s, z8.b, z4.b[0]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "udot z17.s, z9.b, z4.b[0]\n"
- "udot z18.s, z10.b, z4.b[0]\n"
- "udot z19.s, z11.b, z4.b[0]\n"
- "b.eq 5f\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "udot z16.s, z12.b, z4.b[1]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "udot z17.s, z13.b, z4.b[1]\n"
- "udot z18.s, z14.b, z4.b[1]\n"
- "udot z19.s, z15.b, z4.b[1]\n"
- "b.eq 5f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "udot z16.s, z8.b, z4.b[2]\n"
- "udot z17.s, z9.b, z4.b[2]\n"
- "udot z18.s, z10.b, z4.b[2]\n"
- "udot z19.s, z11.b, z4.b[2]\n"
- "b.eq 5f\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "udot z16.s, z12.b, z4.b[3]\n"
- "udot z17.s, z13.b, z4.b[3]\n"
- "udot z18.s, z14.b, z4.b[3]\n"
- "udot z19.s, z15.b, z4.b[3]\n"
- "5:\n"
- "st1w z16.s, p0, [%[c_ptr0]]\n"
- "st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n"
- "st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n"
- "st1w z19.s, p3, [%[c_ptr0], #3, MUL VL]\n"
- "addvl %[c_ptr0], %[c_ptr0], #4\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks)
- : [width] "r" (width), [accumulate] "r" (static_cast<uint64_t>(accumulate)), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers)
- : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
- );
- break;
- case 2:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "c_ptr1 .req X1\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "whilelt p6.b, %[temp], %[leftovers]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "ptrue p7.b\n"
- "whilelt p1.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "whilelt p2.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "whilelt p3.s, %[temp], %[width]\n"
- "cbnz %[accumulate], 1f\n"
- "mov z16.s, #0\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "mov z17.s, #0\n"
- "ld1rqb z1.b, p7/z, [a_ptr1]\n"
- "mov z18.s, #0\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
- "mov z19.s, #0\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "mov z20.s, #0\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "mov z21.s, #0\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "mov z22.s, #0\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "mov z23.s, #0\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "ld1w z16.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n"
- "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n"
- "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n"
- "ld1w z20.s, p0/z, [c_ptr1]\n"
- "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n"
- "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n"
- "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ld1rqb z1.b, p7/z, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "cbz %[loops], 2f\n"
- "3:\n"
- "udot z16.s, z8.b, z0.b[0]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "udot z20.s, z8.b, z1.b[0]\n"
- "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n"
- "udot z17.s, z9.b, z0.b[0]\n"
- "ld1rqb z5.b, p7/z, [a_ptr1]\n"
- "udot z21.s, z9.b, z1.b[0]\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
- "udot z18.s, z10.b, z0.b[0]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "udot z22.s, z10.b, z1.b[0]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "udot z19.s, z11.b, z0.b[0]\n"
- "subs %[loops], %[loops], #0x1\n"
- "udot z23.s, z11.b, z1.b[0]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "udot z16.s, z12.b, z0.b[1]\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
- "udot z20.s, z12.b, z1.b[1]\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "udot z17.s, z13.b, z0.b[1]\n"
- "add a_ptr1, a_ptr1, #0x20\n"
- "udot z21.s, z13.b, z1.b[1]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "udot z18.s, z14.b, z0.b[1]\n"
- "udot z22.s, z14.b, z1.b[1]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "udot z19.s, z15.b, z0.b[1]\n"
- "udot z23.s, z15.b, z1.b[1]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "udot z16.s, z8.b, z0.b[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "udot z20.s, z8.b, z1.b[2]\n"
- "udot z17.s, z9.b, z0.b[2]\n"
- "udot z21.s, z9.b, z1.b[2]\n"
- "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "udot z18.s, z10.b, z0.b[2]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "udot z22.s, z10.b, z1.b[2]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "udot z19.s, z11.b, z0.b[2]\n"
- "udot z23.s, z11.b, z1.b[2]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "udot z16.s, z12.b, z0.b[3]\n"
- "udot z20.s, z12.b, z1.b[3]\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "udot z17.s, z13.b, z0.b[3]\n"
- "udot z21.s, z13.b, z1.b[3]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "udot z18.s, z14.b, z0.b[3]\n"
- "udot z22.s, z14.b, z1.b[3]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "udot z19.s, z15.b, z0.b[3]\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0], #-0x10]\n"
- "udot z23.s, z15.b, z1.b[3]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "udot z16.s, z8.b, z4.b[0]\n"
- "ld1rqb z1.b, p7/z, [a_ptr1, #-0x10]\n"
- "udot z20.s, z8.b, z5.b[0]\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
- "udot z17.s, z9.b, z4.b[0]\n"
- "udot z21.s, z9.b, z5.b[0]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "udot z18.s, z10.b, z4.b[0]\n"
- "udot z22.s, z10.b, z5.b[0]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "udot z19.s, z11.b, z4.b[0]\n"
- "udot z23.s, z11.b, z5.b[0]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "udot z16.s, z12.b, z4.b[1]\n"
- "udot z20.s, z12.b, z5.b[1]\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "udot z17.s, z13.b, z4.b[1]\n"
- "udot z21.s, z13.b, z5.b[1]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "udot z18.s, z14.b, z4.b[1]\n"
- "udot z22.s, z14.b, z5.b[1]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "udot z19.s, z15.b, z4.b[1]\n"
- "udot z23.s, z15.b, z5.b[1]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "udot z16.s, z8.b, z4.b[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "udot z20.s, z8.b, z5.b[2]\n"
- "udot z17.s, z9.b, z4.b[2]\n"
- "udot z21.s, z9.b, z5.b[2]\n"
- "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "udot z18.s, z10.b, z4.b[2]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "udot z22.s, z10.b, z5.b[2]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "udot z19.s, z11.b, z4.b[2]\n"
- "udot z23.s, z11.b, z5.b[2]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "udot z16.s, z12.b, z4.b[3]\n"
- "udot z20.s, z12.b, z5.b[3]\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "udot z17.s, z13.b, z4.b[3]\n"
- "udot z21.s, z13.b, z5.b[3]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "udot z18.s, z14.b, z4.b[3]\n"
- "udot z22.s, z14.b, z5.b[3]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "udot z19.s, z15.b, z4.b[3]\n"
- "udot z23.s, z15.b, z5.b[3]\n"
- "b.ne 3b\n"
- "2:\n"
- "cbz %[regs], 4f\n"
- "udot z16.s, z8.b, z0.b[0]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "udot z20.s, z8.b, z1.b[0]\n"
- "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n"
- "udot z17.s, z9.b, z0.b[0]\n"
- "ld1rqb z5.b, p7/z, [a_ptr1]\n"
- "udot z21.s, z9.b, z1.b[0]\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
- "udot z18.s, z10.b, z0.b[0]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "udot z22.s, z10.b, z1.b[0]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "udot z19.s, z11.b, z0.b[0]\n"
- "udot z23.s, z11.b, z1.b[0]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "udot z16.s, z12.b, z0.b[1]\n"
- "udot z20.s, z12.b, z1.b[1]\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "udot z17.s, z13.b, z0.b[1]\n"
- "udot z21.s, z13.b, z1.b[1]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "udot z18.s, z14.b, z0.b[1]\n"
- "udot z22.s, z14.b, z1.b[1]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "udot z19.s, z15.b, z0.b[1]\n"
- "udot z23.s, z15.b, z1.b[1]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "udot z16.s, z8.b, z0.b[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "udot z20.s, z8.b, z1.b[2]\n"
- "udot z17.s, z9.b, z0.b[2]\n"
- "udot z21.s, z9.b, z1.b[2]\n"
- "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "udot z18.s, z10.b, z0.b[2]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "udot z22.s, z10.b, z1.b[2]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "udot z19.s, z11.b, z0.b[2]\n"
- "udot z23.s, z11.b, z1.b[2]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "udot z16.s, z12.b, z0.b[3]\n"
- "udot z20.s, z12.b, z1.b[3]\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "udot z17.s, z13.b, z0.b[3]\n"
- "udot z21.s, z13.b, z1.b[3]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "udot z18.s, z14.b, z0.b[3]\n"
- "udot z22.s, z14.b, z1.b[3]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "udot z19.s, z15.b, z0.b[3]\n"
- "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x10]\n"
- "udot z23.s, z15.b, z1.b[3]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "udot z16.s, z8.b, z4.b[0]\n"
- "ld1rqb z1.b, p6/z, [a_ptr1, #0x10]\n"
- "udot z20.s, z8.b, z5.b[0]\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
- "udot z17.s, z9.b, z4.b[0]\n"
- "addvl %[a_ptr0], %[a_ptr0], #2\n"
- "udot z21.s, z9.b, z5.b[0]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "udot z18.s, z10.b, z4.b[0]\n"
- "addvl a_ptr1, a_ptr1, #2\n"
- "udot z22.s, z10.b, z5.b[0]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "udot z19.s, z11.b, z4.b[0]\n"
- "udot z23.s, z11.b, z5.b[0]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "udot z16.s, z12.b, z4.b[1]\n"
- "udot z20.s, z12.b, z5.b[1]\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "udot z17.s, z13.b, z4.b[1]\n"
- "udot z21.s, z13.b, z5.b[1]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "udot z18.s, z14.b, z4.b[1]\n"
- "udot z22.s, z14.b, z5.b[1]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "udot z19.s, z15.b, z4.b[1]\n"
- "udot z23.s, z15.b, z5.b[1]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "udot z16.s, z8.b, z4.b[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "udot z20.s, z8.b, z5.b[2]\n"
- "udot z17.s, z9.b, z4.b[2]\n"
- "udot z21.s, z9.b, z5.b[2]\n"
- "udot z18.s, z10.b, z4.b[2]\n"
- "udot z22.s, z10.b, z5.b[2]\n"
- "udot z19.s, z11.b, z4.b[2]\n"
- "udot z23.s, z11.b, z5.b[2]\n"
- "udot z16.s, z12.b, z4.b[3]\n"
- "udot z20.s, z12.b, z5.b[3]\n"
- "udot z17.s, z13.b, z4.b[3]\n"
- "udot z21.s, z13.b, z5.b[3]\n"
- "udot z18.s, z14.b, z4.b[3]\n"
- "udot z22.s, z14.b, z5.b[3]\n"
- "udot z19.s, z15.b, z4.b[3]\n"
- "udot z23.s, z15.b, z5.b[3]\n"
- "cbz %[blocks], 5f\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "udot z16.s, z8.b, z0.b[0]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "udot z20.s, z8.b, z1.b[0]\n"
- "udot z17.s, z9.b, z0.b[0]\n"
- "udot z21.s, z9.b, z1.b[0]\n"
- "udot z18.s, z10.b, z0.b[0]\n"
- "udot z22.s, z10.b, z1.b[0]\n"
- "udot z19.s, z11.b, z0.b[0]\n"
- "udot z23.s, z11.b, z1.b[0]\n"
- "b.eq 5f\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "udot z16.s, z12.b, z0.b[1]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "udot z20.s, z12.b, z1.b[1]\n"
- "udot z17.s, z13.b, z0.b[1]\n"
- "udot z21.s, z13.b, z1.b[1]\n"
- "udot z18.s, z14.b, z0.b[1]\n"
- "udot z22.s, z14.b, z1.b[1]\n"
- "udot z19.s, z15.b, z0.b[1]\n"
- "udot z23.s, z15.b, z1.b[1]\n"
- "b.eq 5f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "udot z16.s, z8.b, z0.b[2]\n"
- "udot z20.s, z8.b, z1.b[2]\n"
- "udot z17.s, z9.b, z0.b[2]\n"
- "udot z21.s, z9.b, z1.b[2]\n"
- "udot z18.s, z10.b, z0.b[2]\n"
- "udot z22.s, z10.b, z1.b[2]\n"
- "udot z19.s, z11.b, z0.b[2]\n"
- "udot z23.s, z11.b, z1.b[2]\n"
- "b.eq 5f\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "udot z16.s, z12.b, z0.b[3]\n"
- "udot z20.s, z12.b, z1.b[3]\n"
- "udot z17.s, z13.b, z0.b[3]\n"
- "udot z21.s, z13.b, z1.b[3]\n"
- "udot z18.s, z14.b, z0.b[3]\n"
- "udot z22.s, z14.b, z1.b[3]\n"
- "udot z19.s, z15.b, z0.b[3]\n"
- "udot z23.s, z15.b, z1.b[3]\n"
- "b 5f\n"
- "4:\n"
- "udot z16.s, z8.b, z0.b[0]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "udot z20.s, z8.b, z1.b[0]\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
- "udot z17.s, z9.b, z0.b[0]\n"
- "ld1rqb z4.b, p6/z, [%[a_ptr0]]\n"
- "udot z21.s, z9.b, z1.b[0]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "udot z18.s, z10.b, z0.b[0]\n"
- "ld1rqb z5.b, p6/z, [a_ptr1]\n"
- "udot z22.s, z10.b, z1.b[0]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "udot z19.s, z11.b, z0.b[0]\n"
- "addvl %[a_ptr0], %[a_ptr0], #1\n"
- "udot z23.s, z11.b, z1.b[0]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "udot z16.s, z12.b, z0.b[1]\n"
- "addvl a_ptr1, a_ptr1, #1\n"
- "udot z20.s, z12.b, z1.b[1]\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "udot z17.s, z13.b, z0.b[1]\n"
- "udot z21.s, z13.b, z1.b[1]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "udot z18.s, z14.b, z0.b[1]\n"
- "udot z22.s, z14.b, z1.b[1]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "udot z19.s, z15.b, z0.b[1]\n"
- "udot z23.s, z15.b, z1.b[1]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "udot z16.s, z8.b, z0.b[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "udot z20.s, z8.b, z1.b[2]\n"
- "udot z17.s, z9.b, z0.b[2]\n"
- "udot z21.s, z9.b, z1.b[2]\n"
- "udot z18.s, z10.b, z0.b[2]\n"
- "udot z22.s, z10.b, z1.b[2]\n"
- "udot z19.s, z11.b, z0.b[2]\n"
- "udot z23.s, z11.b, z1.b[2]\n"
- "udot z16.s, z12.b, z0.b[3]\n"
- "udot z20.s, z12.b, z1.b[3]\n"
- "udot z17.s, z13.b, z0.b[3]\n"
- "udot z21.s, z13.b, z1.b[3]\n"
- "udot z18.s, z14.b, z0.b[3]\n"
- "udot z22.s, z14.b, z1.b[3]\n"
- "udot z19.s, z15.b, z0.b[3]\n"
- "udot z23.s, z15.b, z1.b[3]\n"
- "cbz %[blocks], 5f\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "udot z16.s, z8.b, z4.b[0]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "udot z20.s, z8.b, z5.b[0]\n"
- "udot z17.s, z9.b, z4.b[0]\n"
- "udot z21.s, z9.b, z5.b[0]\n"
- "udot z18.s, z10.b, z4.b[0]\n"
- "udot z22.s, z10.b, z5.b[0]\n"
- "udot z19.s, z11.b, z4.b[0]\n"
- "udot z23.s, z11.b, z5.b[0]\n"
- "b.eq 5f\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "udot z16.s, z12.b, z4.b[1]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "udot z20.s, z12.b, z5.b[1]\n"
- "udot z17.s, z13.b, z4.b[1]\n"
- "udot z21.s, z13.b, z5.b[1]\n"
- "udot z18.s, z14.b, z4.b[1]\n"
- "udot z22.s, z14.b, z5.b[1]\n"
- "udot z19.s, z15.b, z4.b[1]\n"
- "udot z23.s, z15.b, z5.b[1]\n"
- "b.eq 5f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "udot z16.s, z8.b, z4.b[2]\n"
- "udot z20.s, z8.b, z5.b[2]\n"
- "udot z17.s, z9.b, z4.b[2]\n"
- "udot z21.s, z9.b, z5.b[2]\n"
- "udot z18.s, z10.b, z4.b[2]\n"
- "udot z22.s, z10.b, z5.b[2]\n"
- "udot z19.s, z11.b, z4.b[2]\n"
- "udot z23.s, z11.b, z5.b[2]\n"
- "b.eq 5f\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "udot z16.s, z12.b, z4.b[3]\n"
- "udot z20.s, z12.b, z5.b[3]\n"
- "udot z17.s, z13.b, z4.b[3]\n"
- "udot z21.s, z13.b, z5.b[3]\n"
- "udot z18.s, z14.b, z4.b[3]\n"
- "udot z22.s, z14.b, z5.b[3]\n"
- "udot z19.s, z15.b, z4.b[3]\n"
- "udot z23.s, z15.b, z5.b[3]\n"
- "5:\n"
- "st1w z16.s, p0, [%[c_ptr0]]\n"
- "st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n"
- "st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n"
- "st1w z19.s, p3, [%[c_ptr0], #3, MUL VL]\n"
- "addvl %[c_ptr0], %[c_ptr0], #4\n"
- "st1w z20.s, p0, [c_ptr1]\n"
- "st1w z21.s, p1, [c_ptr1, #1, MUL VL]\n"
- "st1w z22.s, p2, [c_ptr1, #2, MUL VL]\n"
- "st1w z23.s, p3, [c_ptr1, #3, MUL VL]\n"
- ".unreq a_ptr1\n"
- ".unreq c_ptr1\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks)
- : [width] "r" (width), [accumulate] "r" (static_cast<uint64_t>(accumulate)), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers)
- : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "cc", "memory"
- );
- break;
- case 3:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "c_ptr1 .req X2\n"
- "c_ptr2 .req X3\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "whilelt p6.b, %[temp], %[leftovers]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "ptrue p7.b\n"
- "whilelt p1.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "whilelt p2.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "whilelt p3.s, %[temp], %[width]\n"
- "cbnz %[accumulate], 1f\n"
- "mov z16.s, #0\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "mov z17.s, #0\n"
- "ld1rqb z1.b, p7/z, [a_ptr1]\n"
- "mov z18.s, #0\n"
- "ld1rqb z2.b, p7/z, [a_ptr2]\n"
- "mov z19.s, #0\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
- "mov z20.s, #0\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "mov z21.s, #0\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "mov z22.s, #0\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "mov z23.s, #0\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "mov z24.s, #0\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "mov z25.s, #0\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "mov z26.s, #0\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "mov z27.s, #0\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "ld1w z16.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n"
- "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n"
- "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n"
- "ld1w z20.s, p0/z, [c_ptr1]\n"
- "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n"
- "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n"
- "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n"
- "ld1w z24.s, p0/z, [c_ptr2]\n"
- "ld1w z25.s, p1/z, [c_ptr2, #1, MUL VL]\n"
- "ld1w z26.s, p2/z, [c_ptr2, #2, MUL VL]\n"
- "ld1w z27.s, p3/z, [c_ptr2, #3, MUL VL]\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ld1rqb z1.b, p7/z, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "ld1rqb z2.b, p7/z, [a_ptr2]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "cbz %[loops], 2f\n"
- "3:\n"
- "udot z16.s, z8.b, z0.b[0]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "udot z20.s, z8.b, z1.b[0]\n"
- "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n"
- "udot z24.s, z8.b, z2.b[0]\n"
- "ld1rqb z5.b, p7/z, [a_ptr1]\n"
- "udot z17.s, z9.b, z0.b[0]\n"
- "ld1rqb z6.b, p7/z, [a_ptr2]\n"
- "udot z21.s, z9.b, z1.b[0]\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
- "udot z25.s, z9.b, z2.b[0]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "udot z18.s, z10.b, z0.b[0]\n"
- "subs %[loops], %[loops], #0x1\n"
- "udot z22.s, z10.b, z1.b[0]\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
- "udot z26.s, z10.b, z2.b[0]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "udot z19.s, z11.b, z0.b[0]\n"
- "add a_ptr1, a_ptr1, #0x20\n"
- "udot z23.s, z11.b, z1.b[0]\n"
- "add a_ptr2, a_ptr2, #0x20\n"
- "udot z27.s, z11.b, z2.b[0]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "udot z16.s, z12.b, z0.b[1]\n"
- "udot z20.s, z12.b, z1.b[1]\n"
- "udot z24.s, z12.b, z2.b[1]\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "udot z17.s, z13.b, z0.b[1]\n"
- "udot z21.s, z13.b, z1.b[1]\n"
- "udot z25.s, z13.b, z2.b[1]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "udot z18.s, z14.b, z0.b[1]\n"
- "udot z22.s, z14.b, z1.b[1]\n"
- "udot z26.s, z14.b, z2.b[1]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "udot z19.s, z15.b, z0.b[1]\n"
- "udot z23.s, z15.b, z1.b[1]\n"
- "udot z27.s, z15.b, z2.b[1]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "udot z16.s, z8.b, z0.b[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "udot z20.s, z8.b, z1.b[2]\n"
- "udot z24.s, z8.b, z2.b[2]\n"
- "udot z17.s, z9.b, z0.b[2]\n"
- "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "udot z21.s, z9.b, z1.b[2]\n"
- "udot z25.s, z9.b, z2.b[2]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "udot z18.s, z10.b, z0.b[2]\n"
- "udot z22.s, z10.b, z1.b[2]\n"
- "udot z26.s, z10.b, z2.b[2]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "udot z19.s, z11.b, z0.b[2]\n"
- "udot z23.s, z11.b, z1.b[2]\n"
- "udot z27.s, z11.b, z2.b[2]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "udot z16.s, z12.b, z0.b[3]\n"
- "udot z20.s, z12.b, z1.b[3]\n"
- "udot z24.s, z12.b, z2.b[3]\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "udot z17.s, z13.b, z0.b[3]\n"
- "udot z21.s, z13.b, z1.b[3]\n"
- "udot z25.s, z13.b, z2.b[3]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "udot z18.s, z14.b, z0.b[3]\n"
- "udot z22.s, z14.b, z1.b[3]\n"
- "udot z26.s, z14.b, z2.b[3]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "udot z19.s, z15.b, z0.b[3]\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0], #-0x10]\n"
- "udot z23.s, z15.b, z1.b[3]\n"
- "ld1rqb z1.b, p7/z, [a_ptr1, #-0x10]\n"
- "udot z27.s, z15.b, z2.b[3]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "udot z16.s, z8.b, z4.b[0]\n"
- "ld1rqb z2.b, p7/z, [a_ptr2, #-0x10]\n"
- "udot z20.s, z8.b, z5.b[0]\n"
- "udot z24.s, z8.b, z6.b[0]\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
- "udot z17.s, z9.b, z4.b[0]\n"
- "udot z21.s, z9.b, z5.b[0]\n"
- "udot z25.s, z9.b, z6.b[0]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "udot z18.s, z10.b, z4.b[0]\n"
- "udot z22.s, z10.b, z5.b[0]\n"
- "udot z26.s, z10.b, z6.b[0]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "udot z19.s, z11.b, z4.b[0]\n"
- "udot z23.s, z11.b, z5.b[0]\n"
- "udot z27.s, z11.b, z6.b[0]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "udot z16.s, z12.b, z4.b[1]\n"
- "udot z20.s, z12.b, z5.b[1]\n"
- "udot z24.s, z12.b, z6.b[1]\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "udot z17.s, z13.b, z4.b[1]\n"
- "udot z21.s, z13.b, z5.b[1]\n"
- "udot z25.s, z13.b, z6.b[1]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "udot z18.s, z14.b, z4.b[1]\n"
- "udot z22.s, z14.b, z5.b[1]\n"
- "udot z26.s, z14.b, z6.b[1]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "udot z19.s, z15.b, z4.b[1]\n"
- "udot z23.s, z15.b, z5.b[1]\n"
- "udot z27.s, z15.b, z6.b[1]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "udot z16.s, z8.b, z4.b[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "udot z20.s, z8.b, z5.b[2]\n"
- "udot z24.s, z8.b, z6.b[2]\n"
- "udot z17.s, z9.b, z4.b[2]\n"
- "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "udot z21.s, z9.b, z5.b[2]\n"
- "udot z25.s, z9.b, z6.b[2]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "udot z18.s, z10.b, z4.b[2]\n"
- "udot z22.s, z10.b, z5.b[2]\n"
- "udot z26.s, z10.b, z6.b[2]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "udot z19.s, z11.b, z4.b[2]\n"
- "udot z23.s, z11.b, z5.b[2]\n"
- "udot z27.s, z11.b, z6.b[2]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "udot z16.s, z12.b, z4.b[3]\n"
- "udot z20.s, z12.b, z5.b[3]\n"
- "udot z24.s, z12.b, z6.b[3]\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "udot z17.s, z13.b, z4.b[3]\n"
- "udot z21.s, z13.b, z5.b[3]\n"
- "udot z25.s, z13.b, z6.b[3]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "udot z18.s, z14.b, z4.b[3]\n"
- "udot z22.s, z14.b, z5.b[3]\n"
- "udot z26.s, z14.b, z6.b[3]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "udot z19.s, z15.b, z4.b[3]\n"
- "udot z23.s, z15.b, z5.b[3]\n"
- "udot z27.s, z15.b, z6.b[3]\n"
- "b.ne 3b\n"
- "2:\n"
- "cbz %[regs], 4f\n"
- "udot z16.s, z8.b, z0.b[0]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "udot z20.s, z8.b, z1.b[0]\n"
- "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n"
- "udot z24.s, z8.b, z2.b[0]\n"
- "ld1rqb z5.b, p7/z, [a_ptr1]\n"
- "udot z17.s, z9.b, z0.b[0]\n"
- "ld1rqb z6.b, p7/z, [a_ptr2]\n"
- "udot z21.s, z9.b, z1.b[0]\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
- "udot z25.s, z9.b, z2.b[0]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "udot z18.s, z10.b, z0.b[0]\n"
- "udot z22.s, z10.b, z1.b[0]\n"
- "udot z26.s, z10.b, z2.b[0]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "udot z19.s, z11.b, z0.b[0]\n"
- "udot z23.s, z11.b, z1.b[0]\n"
- "udot z27.s, z11.b, z2.b[0]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "udot z16.s, z12.b, z0.b[1]\n"
- "udot z20.s, z12.b, z1.b[1]\n"
- "udot z24.s, z12.b, z2.b[1]\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "udot z17.s, z13.b, z0.b[1]\n"
- "udot z21.s, z13.b, z1.b[1]\n"
- "udot z25.s, z13.b, z2.b[1]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "udot z18.s, z14.b, z0.b[1]\n"
- "udot z22.s, z14.b, z1.b[1]\n"
- "udot z26.s, z14.b, z2.b[1]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "udot z19.s, z15.b, z0.b[1]\n"
- "udot z23.s, z15.b, z1.b[1]\n"
- "udot z27.s, z15.b, z2.b[1]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "udot z16.s, z8.b, z0.b[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "udot z20.s, z8.b, z1.b[2]\n"
- "udot z24.s, z8.b, z2.b[2]\n"
- "udot z17.s, z9.b, z0.b[2]\n"
- "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "udot z21.s, z9.b, z1.b[2]\n"
- "udot z25.s, z9.b, z2.b[2]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "udot z18.s, z10.b, z0.b[2]\n"
- "udot z22.s, z10.b, z1.b[2]\n"
- "udot z26.s, z10.b, z2.b[2]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "udot z19.s, z11.b, z0.b[2]\n"
- "udot z23.s, z11.b, z1.b[2]\n"
- "udot z27.s, z11.b, z2.b[2]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "udot z16.s, z12.b, z0.b[3]\n"
- "udot z20.s, z12.b, z1.b[3]\n"
- "udot z24.s, z12.b, z2.b[3]\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "udot z17.s, z13.b, z0.b[3]\n"
- "udot z21.s, z13.b, z1.b[3]\n"
- "udot z25.s, z13.b, z2.b[3]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "udot z18.s, z14.b, z0.b[3]\n"
- "udot z22.s, z14.b, z1.b[3]\n"
- "udot z26.s, z14.b, z2.b[3]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "udot z19.s, z15.b, z0.b[3]\n"
- "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x10]\n"
- "udot z23.s, z15.b, z1.b[3]\n"
- "ld1rqb z1.b, p6/z, [a_ptr1, #0x10]\n"
- "udot z27.s, z15.b, z2.b[3]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "udot z16.s, z8.b, z4.b[0]\n"
- "ld1rqb z2.b, p6/z, [a_ptr2, #0x10]\n"
- "udot z20.s, z8.b, z5.b[0]\n"
- "addvl %[a_ptr0], %[a_ptr0], #2\n"
- "udot z24.s, z8.b, z6.b[0]\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
- "udot z17.s, z9.b, z4.b[0]\n"
- "addvl a_ptr1, a_ptr1, #2\n"
- "udot z21.s, z9.b, z5.b[0]\n"
- "addvl a_ptr2, a_ptr2, #2\n"
- "udot z25.s, z9.b, z6.b[0]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "udot z18.s, z10.b, z4.b[0]\n"
- "udot z22.s, z10.b, z5.b[0]\n"
- "udot z26.s, z10.b, z6.b[0]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "udot z19.s, z11.b, z4.b[0]\n"
- "udot z23.s, z11.b, z5.b[0]\n"
- "udot z27.s, z11.b, z6.b[0]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "udot z16.s, z12.b, z4.b[1]\n"
- "udot z20.s, z12.b, z5.b[1]\n"
- "udot z24.s, z12.b, z6.b[1]\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "udot z17.s, z13.b, z4.b[1]\n"
- "udot z21.s, z13.b, z5.b[1]\n"
- "udot z25.s, z13.b, z6.b[1]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "udot z18.s, z14.b, z4.b[1]\n"
- "udot z22.s, z14.b, z5.b[1]\n"
- "udot z26.s, z14.b, z6.b[1]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "udot z19.s, z15.b, z4.b[1]\n"
- "udot z23.s, z15.b, z5.b[1]\n"
- "udot z27.s, z15.b, z6.b[1]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "udot z16.s, z8.b, z4.b[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "udot z20.s, z8.b, z5.b[2]\n"
- "udot z24.s, z8.b, z6.b[2]\n"
- "udot z17.s, z9.b, z4.b[2]\n"
- "udot z21.s, z9.b, z5.b[2]\n"
- "udot z25.s, z9.b, z6.b[2]\n"
- "udot z18.s, z10.b, z4.b[2]\n"
- "udot z22.s, z10.b, z5.b[2]\n"
- "udot z26.s, z10.b, z6.b[2]\n"
- "udot z19.s, z11.b, z4.b[2]\n"
- "udot z23.s, z11.b, z5.b[2]\n"
- "udot z27.s, z11.b, z6.b[2]\n"
- "udot z16.s, z12.b, z4.b[3]\n"
- "udot z20.s, z12.b, z5.b[3]\n"
- "udot z24.s, z12.b, z6.b[3]\n"
- "udot z17.s, z13.b, z4.b[3]\n"
- "udot z21.s, z13.b, z5.b[3]\n"
- "udot z25.s, z13.b, z6.b[3]\n"
- "udot z18.s, z14.b, z4.b[3]\n"
- "udot z22.s, z14.b, z5.b[3]\n"
- "udot z26.s, z14.b, z6.b[3]\n"
- "udot z19.s, z15.b, z4.b[3]\n"
- "udot z23.s, z15.b, z5.b[3]\n"
- "udot z27.s, z15.b, z6.b[3]\n"
- "cbz %[blocks], 5f\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "udot z16.s, z8.b, z0.b[0]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "udot z20.s, z8.b, z1.b[0]\n"
- "udot z24.s, z8.b, z2.b[0]\n"
- "udot z17.s, z9.b, z0.b[0]\n"
- "udot z21.s, z9.b, z1.b[0]\n"
- "udot z25.s, z9.b, z2.b[0]\n"
- "udot z18.s, z10.b, z0.b[0]\n"
- "udot z22.s, z10.b, z1.b[0]\n"
- "udot z26.s, z10.b, z2.b[0]\n"
- "udot z19.s, z11.b, z0.b[0]\n"
- "udot z23.s, z11.b, z1.b[0]\n"
- "udot z27.s, z11.b, z2.b[0]\n"
- "b.eq 5f\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "udot z16.s, z12.b, z0.b[1]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "udot z20.s, z12.b, z1.b[1]\n"
- "udot z24.s, z12.b, z2.b[1]\n"
- "udot z17.s, z13.b, z0.b[1]\n"
- "udot z21.s, z13.b, z1.b[1]\n"
- "udot z25.s, z13.b, z2.b[1]\n"
- "udot z18.s, z14.b, z0.b[1]\n"
- "udot z22.s, z14.b, z1.b[1]\n"
- "udot z26.s, z14.b, z2.b[1]\n"
- "udot z19.s, z15.b, z0.b[1]\n"
- "udot z23.s, z15.b, z1.b[1]\n"
- "udot z27.s, z15.b, z2.b[1]\n"
- "b.eq 5f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "udot z16.s, z8.b, z0.b[2]\n"
- "udot z20.s, z8.b, z1.b[2]\n"
- "udot z24.s, z8.b, z2.b[2]\n"
- "udot z17.s, z9.b, z0.b[2]\n"
- "udot z21.s, z9.b, z1.b[2]\n"
- "udot z25.s, z9.b, z2.b[2]\n"
- "udot z18.s, z10.b, z0.b[2]\n"
- "udot z22.s, z10.b, z1.b[2]\n"
- "udot z26.s, z10.b, z2.b[2]\n"
- "udot z19.s, z11.b, z0.b[2]\n"
- "udot z23.s, z11.b, z1.b[2]\n"
- "udot z27.s, z11.b, z2.b[2]\n"
- "b.eq 5f\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "udot z16.s, z12.b, z0.b[3]\n"
- "udot z20.s, z12.b, z1.b[3]\n"
- "udot z24.s, z12.b, z2.b[3]\n"
- "udot z17.s, z13.b, z0.b[3]\n"
- "udot z21.s, z13.b, z1.b[3]\n"
- "udot z25.s, z13.b, z2.b[3]\n"
- "udot z18.s, z14.b, z0.b[3]\n"
- "udot z22.s, z14.b, z1.b[3]\n"
- "udot z26.s, z14.b, z2.b[3]\n"
- "udot z19.s, z15.b, z0.b[3]\n"
- "udot z23.s, z15.b, z1.b[3]\n"
- "udot z27.s, z15.b, z2.b[3]\n"
- "b 5f\n"
- "4:\n"
- "udot z16.s, z8.b, z0.b[0]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "udot z20.s, z8.b, z1.b[0]\n"
- "ld1rqb z4.b, p6/z, [%[a_ptr0]]\n"
- "udot z24.s, z8.b, z2.b[0]\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
- "udot z17.s, z9.b, z0.b[0]\n"
- "ld1rqb z5.b, p6/z, [a_ptr1]\n"
- "udot z21.s, z9.b, z1.b[0]\n"
- "ld1rqb z6.b, p6/z, [a_ptr2]\n"
- "udot z25.s, z9.b, z2.b[0]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "udot z18.s, z10.b, z0.b[0]\n"
- "addvl %[a_ptr0], %[a_ptr0], #1\n"
- "udot z22.s, z10.b, z1.b[0]\n"
- "addvl a_ptr1, a_ptr1, #1\n"
- "udot z26.s, z10.b, z2.b[0]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "udot z19.s, z11.b, z0.b[0]\n"
- "addvl a_ptr2, a_ptr2, #1\n"
- "udot z23.s, z11.b, z1.b[0]\n"
- "udot z27.s, z11.b, z2.b[0]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "udot z16.s, z12.b, z0.b[1]\n"
- "udot z20.s, z12.b, z1.b[1]\n"
- "udot z24.s, z12.b, z2.b[1]\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "udot z17.s, z13.b, z0.b[1]\n"
- "udot z21.s, z13.b, z1.b[1]\n"
- "udot z25.s, z13.b, z2.b[1]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "udot z18.s, z14.b, z0.b[1]\n"
- "udot z22.s, z14.b, z1.b[1]\n"
- "udot z26.s, z14.b, z2.b[1]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "udot z19.s, z15.b, z0.b[1]\n"
- "udot z23.s, z15.b, z1.b[1]\n"
- "udot z27.s, z15.b, z2.b[1]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "udot z16.s, z8.b, z0.b[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "udot z20.s, z8.b, z1.b[2]\n"
- "udot z24.s, z8.b, z2.b[2]\n"
- "udot z17.s, z9.b, z0.b[2]\n"
- "udot z21.s, z9.b, z1.b[2]\n"
- "udot z25.s, z9.b, z2.b[2]\n"
- "udot z18.s, z10.b, z0.b[2]\n"
- "udot z22.s, z10.b, z1.b[2]\n"
- "udot z26.s, z10.b, z2.b[2]\n"
- "udot z19.s, z11.b, z0.b[2]\n"
- "udot z23.s, z11.b, z1.b[2]\n"
- "udot z27.s, z11.b, z2.b[2]\n"
- "udot z16.s, z12.b, z0.b[3]\n"
- "udot z20.s, z12.b, z1.b[3]\n"
- "udot z24.s, z12.b, z2.b[3]\n"
- "udot z17.s, z13.b, z0.b[3]\n"
- "udot z21.s, z13.b, z1.b[3]\n"
- "udot z25.s, z13.b, z2.b[3]\n"
- "udot z18.s, z14.b, z0.b[3]\n"
- "udot z22.s, z14.b, z1.b[3]\n"
- "udot z26.s, z14.b, z2.b[3]\n"
- "udot z19.s, z15.b, z0.b[3]\n"
- "udot z23.s, z15.b, z1.b[3]\n"
- "udot z27.s, z15.b, z2.b[3]\n"
- "cbz %[blocks], 5f\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "udot z16.s, z8.b, z4.b[0]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "udot z20.s, z8.b, z5.b[0]\n"
- "udot z24.s, z8.b, z6.b[0]\n"
- "udot z17.s, z9.b, z4.b[0]\n"
- "udot z21.s, z9.b, z5.b[0]\n"
- "udot z25.s, z9.b, z6.b[0]\n"
- "udot z18.s, z10.b, z4.b[0]\n"
- "udot z22.s, z10.b, z5.b[0]\n"
- "udot z26.s, z10.b, z6.b[0]\n"
- "udot z19.s, z11.b, z4.b[0]\n"
- "udot z23.s, z11.b, z5.b[0]\n"
- "udot z27.s, z11.b, z6.b[0]\n"
- "b.eq 5f\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "udot z16.s, z12.b, z4.b[1]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "udot z20.s, z12.b, z5.b[1]\n"
- "udot z24.s, z12.b, z6.b[1]\n"
- "udot z17.s, z13.b, z4.b[1]\n"
- "udot z21.s, z13.b, z5.b[1]\n"
- "udot z25.s, z13.b, z6.b[1]\n"
- "udot z18.s, z14.b, z4.b[1]\n"
- "udot z22.s, z14.b, z5.b[1]\n"
- "udot z26.s, z14.b, z6.b[1]\n"
- "udot z19.s, z15.b, z4.b[1]\n"
- "udot z23.s, z15.b, z5.b[1]\n"
- "udot z27.s, z15.b, z6.b[1]\n"
- "b.eq 5f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "udot z16.s, z8.b, z4.b[2]\n"
- "udot z20.s, z8.b, z5.b[2]\n"
- "udot z24.s, z8.b, z6.b[2]\n"
- "udot z17.s, z9.b, z4.b[2]\n"
- "udot z21.s, z9.b, z5.b[2]\n"
- "udot z25.s, z9.b, z6.b[2]\n"
- "udot z18.s, z10.b, z4.b[2]\n"
- "udot z22.s, z10.b, z5.b[2]\n"
- "udot z26.s, z10.b, z6.b[2]\n"
- "udot z19.s, z11.b, z4.b[2]\n"
- "udot z23.s, z11.b, z5.b[2]\n"
- "udot z27.s, z11.b, z6.b[2]\n"
- "b.eq 5f\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "udot z16.s, z12.b, z4.b[3]\n"
- "udot z20.s, z12.b, z5.b[3]\n"
- "udot z24.s, z12.b, z6.b[3]\n"
- "udot z17.s, z13.b, z4.b[3]\n"
- "udot z21.s, z13.b, z5.b[3]\n"
- "udot z25.s, z13.b, z6.b[3]\n"
- "udot z18.s, z14.b, z4.b[3]\n"
- "udot z22.s, z14.b, z5.b[3]\n"
- "udot z26.s, z14.b, z6.b[3]\n"
- "udot z19.s, z15.b, z4.b[3]\n"
- "udot z23.s, z15.b, z5.b[3]\n"
- "udot z27.s, z15.b, z6.b[3]\n"
- "5:\n"
- "st1w z16.s, p0, [%[c_ptr0]]\n"
- "st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n"
- "st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n"
- "st1w z19.s, p3, [%[c_ptr0], #3, MUL VL]\n"
- "addvl %[c_ptr0], %[c_ptr0], #4\n"
- "st1w z20.s, p0, [c_ptr1]\n"
- "st1w z21.s, p1, [c_ptr1, #1, MUL VL]\n"
- "st1w z22.s, p2, [c_ptr1, #2, MUL VL]\n"
- "st1w z23.s, p3, [c_ptr1, #3, MUL VL]\n"
- "st1w z24.s, p0, [c_ptr2]\n"
- "st1w z25.s, p1, [c_ptr2, #1, MUL VL]\n"
- "st1w z26.s, p2, [c_ptr2, #2, MUL VL]\n"
- "st1w z27.s, p3, [c_ptr2, #3, MUL VL]\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks)
- : [width] "r" (width), [accumulate] "r" (static_cast<uint64_t>(accumulate)), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers)
- : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "cc", "memory"
- );
- break;
- default:
- case 4:
- __asm __volatile (
- "a_ptr1 .req X0\n"
- "a_ptr2 .req X1\n"
- "a_ptr3 .req X2\n"
- "c_ptr1 .req X3\n"
- "c_ptr2 .req X4\n"
- "c_ptr3 .req X5\n"
- "add a_ptr1, %[a_ptr0], %[lda]\n"
- "add c_ptr1, %[c_ptr0], %[ldc]\n"
- "add a_ptr2, a_ptr1, %[lda]\n"
- "add c_ptr2, c_ptr1, %[ldc]\n"
- "add a_ptr3, a_ptr2, %[lda]\n"
- "add c_ptr3, c_ptr2, %[ldc]\n"
- "whilelt p6.b, %[temp], %[leftovers]\n"
- "whilelt p0.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "ptrue p7.b\n"
- "whilelt p1.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "whilelt p2.s, %[temp], %[width]\n"
- "incw %[temp], all, mul #1\n"
- "whilelt p3.s, %[temp], %[width]\n"
- "cbnz %[accumulate], 1f\n"
- "mov z16.s, #0\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "mov z17.s, #0\n"
- "ld1rqb z1.b, p7/z, [a_ptr1]\n"
- "mov z18.s, #0\n"
- "ld1rqb z2.b, p7/z, [a_ptr2]\n"
- "mov z19.s, #0\n"
- "ld1rqb z3.b, p7/z, [a_ptr3]\n"
- "mov z20.s, #0\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
- "mov z21.s, #0\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "mov z22.s, #0\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "mov z23.s, #0\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "mov z24.s, #0\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "mov z25.s, #0\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "mov z26.s, #0\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "mov z27.s, #0\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "mov z28.s, #0\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "mov z29.s, #0\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "mov z30.s, #0\n"
- "add a_ptr3, a_ptr3, #0x10\n"
- "mov z31.s, #0\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "cbz %[loops], 2f\n"
- "b 3f\n"
- "1:\n"
- "ld1w z16.s, p0/z, [%[c_ptr0]]\n"
- "ld1w z17.s, p1/z, [%[c_ptr0], #1, MUL VL]\n"
- "ld1w z18.s, p2/z, [%[c_ptr0], #2, MUL VL]\n"
- "ld1w z19.s, p3/z, [%[c_ptr0], #3, MUL VL]\n"
- "ld1w z20.s, p0/z, [c_ptr1]\n"
- "ld1w z21.s, p1/z, [c_ptr1, #1, MUL VL]\n"
- "ld1w z22.s, p2/z, [c_ptr1, #2, MUL VL]\n"
- "ld1w z23.s, p3/z, [c_ptr1, #3, MUL VL]\n"
- "ld1w z24.s, p0/z, [c_ptr2]\n"
- "ld1w z25.s, p1/z, [c_ptr2, #1, MUL VL]\n"
- "ld1w z26.s, p2/z, [c_ptr2, #2, MUL VL]\n"
- "ld1w z27.s, p3/z, [c_ptr2, #3, MUL VL]\n"
- "ld1w z28.s, p0/z, [c_ptr3]\n"
- "ld1w z29.s, p1/z, [c_ptr3, #1, MUL VL]\n"
- "ld1w z30.s, p2/z, [c_ptr3, #2, MUL VL]\n"
- "ld1w z31.s, p3/z, [c_ptr3, #3, MUL VL]\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "add %[a_ptr0], %[a_ptr0], #0x10\n"
- "ld1rqb z1.b, p7/z, [a_ptr1]\n"
- "add a_ptr1, a_ptr1, #0x10\n"
- "ld1rqb z2.b, p7/z, [a_ptr2]\n"
- "add a_ptr2, a_ptr2, #0x10\n"
- "ld1rqb z3.b, p7/z, [a_ptr3]\n"
- "add a_ptr3, a_ptr3, #0x10\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "cbz %[loops], 2f\n"
- "3:\n"
- "udot z16.s, z8.b, z0.b[0]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "udot z20.s, z8.b, z1.b[0]\n"
- "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n"
- "udot z24.s, z8.b, z2.b[0]\n"
- "ld1rqb z5.b, p7/z, [a_ptr1]\n"
- "udot z28.s, z8.b, z3.b[0]\n"
- "ld1rqb z6.b, p7/z, [a_ptr2]\n"
- "udot z17.s, z9.b, z0.b[0]\n"
- "ld1rqb z7.b, p7/z, [a_ptr3]\n"
- "udot z21.s, z9.b, z1.b[0]\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
- "udot z25.s, z9.b, z2.b[0]\n"
- "subs %[loops], %[loops], #0x1\n"
- "udot z29.s, z9.b, z3.b[0]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "udot z18.s, z10.b, z0.b[0]\n"
- "add %[a_ptr0], %[a_ptr0], #0x20\n"
- "udot z22.s, z10.b, z1.b[0]\n"
- "add a_ptr1, a_ptr1, #0x20\n"
- "udot z26.s, z10.b, z2.b[0]\n"
- "add a_ptr2, a_ptr2, #0x20\n"
- "udot z30.s, z10.b, z3.b[0]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "udot z19.s, z11.b, z0.b[0]\n"
- "add a_ptr3, a_ptr3, #0x20\n"
- "udot z23.s, z11.b, z1.b[0]\n"
- "udot z27.s, z11.b, z2.b[0]\n"
- "udot z31.s, z11.b, z3.b[0]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "udot z16.s, z12.b, z0.b[1]\n"
- "udot z20.s, z12.b, z1.b[1]\n"
- "udot z24.s, z12.b, z2.b[1]\n"
- "udot z28.s, z12.b, z3.b[1]\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "udot z17.s, z13.b, z0.b[1]\n"
- "udot z21.s, z13.b, z1.b[1]\n"
- "udot z25.s, z13.b, z2.b[1]\n"
- "udot z29.s, z13.b, z3.b[1]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "udot z18.s, z14.b, z0.b[1]\n"
- "udot z22.s, z14.b, z1.b[1]\n"
- "udot z26.s, z14.b, z2.b[1]\n"
- "udot z30.s, z14.b, z3.b[1]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "udot z19.s, z15.b, z0.b[1]\n"
- "udot z23.s, z15.b, z1.b[1]\n"
- "udot z27.s, z15.b, z2.b[1]\n"
- "udot z31.s, z15.b, z3.b[1]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "udot z16.s, z8.b, z0.b[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "udot z20.s, z8.b, z1.b[2]\n"
- "udot z24.s, z8.b, z2.b[2]\n"
- "udot z28.s, z8.b, z3.b[2]\n"
- "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "udot z17.s, z9.b, z0.b[2]\n"
- "udot z21.s, z9.b, z1.b[2]\n"
- "udot z25.s, z9.b, z2.b[2]\n"
- "udot z29.s, z9.b, z3.b[2]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "udot z18.s, z10.b, z0.b[2]\n"
- "udot z22.s, z10.b, z1.b[2]\n"
- "udot z26.s, z10.b, z2.b[2]\n"
- "udot z30.s, z10.b, z3.b[2]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "udot z19.s, z11.b, z0.b[2]\n"
- "udot z23.s, z11.b, z1.b[2]\n"
- "udot z27.s, z11.b, z2.b[2]\n"
- "udot z31.s, z11.b, z3.b[2]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "udot z16.s, z12.b, z0.b[3]\n"
- "udot z20.s, z12.b, z1.b[3]\n"
- "udot z24.s, z12.b, z2.b[3]\n"
- "udot z28.s, z12.b, z3.b[3]\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "udot z17.s, z13.b, z0.b[3]\n"
- "udot z21.s, z13.b, z1.b[3]\n"
- "udot z25.s, z13.b, z2.b[3]\n"
- "udot z29.s, z13.b, z3.b[3]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "udot z18.s, z14.b, z0.b[3]\n"
- "udot z22.s, z14.b, z1.b[3]\n"
- "udot z26.s, z14.b, z2.b[3]\n"
- "udot z30.s, z14.b, z3.b[3]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "udot z19.s, z15.b, z0.b[3]\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0], #-0x10]\n"
- "udot z23.s, z15.b, z1.b[3]\n"
- "ld1rqb z1.b, p7/z, [a_ptr1, #-0x10]\n"
- "udot z27.s, z15.b, z2.b[3]\n"
- "ld1rqb z2.b, p7/z, [a_ptr2, #-0x10]\n"
- "udot z31.s, z15.b, z3.b[3]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "udot z16.s, z8.b, z4.b[0]\n"
- "ld1rqb z3.b, p7/z, [a_ptr3, #-0x10]\n"
- "udot z20.s, z8.b, z5.b[0]\n"
- "udot z24.s, z8.b, z6.b[0]\n"
- "udot z28.s, z8.b, z7.b[0]\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
- "udot z17.s, z9.b, z4.b[0]\n"
- "udot z21.s, z9.b, z5.b[0]\n"
- "udot z25.s, z9.b, z6.b[0]\n"
- "udot z29.s, z9.b, z7.b[0]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "udot z18.s, z10.b, z4.b[0]\n"
- "udot z22.s, z10.b, z5.b[0]\n"
- "udot z26.s, z10.b, z6.b[0]\n"
- "udot z30.s, z10.b, z7.b[0]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "udot z19.s, z11.b, z4.b[0]\n"
- "udot z23.s, z11.b, z5.b[0]\n"
- "udot z27.s, z11.b, z6.b[0]\n"
- "udot z31.s, z11.b, z7.b[0]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "udot z16.s, z12.b, z4.b[1]\n"
- "udot z20.s, z12.b, z5.b[1]\n"
- "udot z24.s, z12.b, z6.b[1]\n"
- "udot z28.s, z12.b, z7.b[1]\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "udot z17.s, z13.b, z4.b[1]\n"
- "udot z21.s, z13.b, z5.b[1]\n"
- "udot z25.s, z13.b, z6.b[1]\n"
- "udot z29.s, z13.b, z7.b[1]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "udot z18.s, z14.b, z4.b[1]\n"
- "udot z22.s, z14.b, z5.b[1]\n"
- "udot z26.s, z14.b, z6.b[1]\n"
- "udot z30.s, z14.b, z7.b[1]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "udot z19.s, z15.b, z4.b[1]\n"
- "udot z23.s, z15.b, z5.b[1]\n"
- "udot z27.s, z15.b, z6.b[1]\n"
- "udot z31.s, z15.b, z7.b[1]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "udot z16.s, z8.b, z4.b[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "udot z20.s, z8.b, z5.b[2]\n"
- "udot z24.s, z8.b, z6.b[2]\n"
- "udot z28.s, z8.b, z7.b[2]\n"
- "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "udot z17.s, z9.b, z4.b[2]\n"
- "udot z21.s, z9.b, z5.b[2]\n"
- "udot z25.s, z9.b, z6.b[2]\n"
- "udot z29.s, z9.b, z7.b[2]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "udot z18.s, z10.b, z4.b[2]\n"
- "udot z22.s, z10.b, z5.b[2]\n"
- "udot z26.s, z10.b, z6.b[2]\n"
- "udot z30.s, z10.b, z7.b[2]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "udot z19.s, z11.b, z4.b[2]\n"
- "udot z23.s, z11.b, z5.b[2]\n"
- "udot z27.s, z11.b, z6.b[2]\n"
- "udot z31.s, z11.b, z7.b[2]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "udot z16.s, z12.b, z4.b[3]\n"
- "udot z20.s, z12.b, z5.b[3]\n"
- "udot z24.s, z12.b, z6.b[3]\n"
- "udot z28.s, z12.b, z7.b[3]\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "udot z17.s, z13.b, z4.b[3]\n"
- "udot z21.s, z13.b, z5.b[3]\n"
- "udot z25.s, z13.b, z6.b[3]\n"
- "udot z29.s, z13.b, z7.b[3]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "udot z18.s, z14.b, z4.b[3]\n"
- "udot z22.s, z14.b, z5.b[3]\n"
- "udot z26.s, z14.b, z6.b[3]\n"
- "udot z30.s, z14.b, z7.b[3]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "udot z19.s, z15.b, z4.b[3]\n"
- "udot z23.s, z15.b, z5.b[3]\n"
- "udot z27.s, z15.b, z6.b[3]\n"
- "udot z31.s, z15.b, z7.b[3]\n"
- "b.ne 3b\n"
- "2:\n"
- "cbz %[regs], 4f\n"
- "udot z16.s, z8.b, z0.b[0]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "udot z20.s, z8.b, z1.b[0]\n"
- "ld1rqb z4.b, p7/z, [%[a_ptr0]]\n"
- "udot z24.s, z8.b, z2.b[0]\n"
- "ld1rqb z5.b, p7/z, [a_ptr1]\n"
- "udot z28.s, z8.b, z3.b[0]\n"
- "ld1rqb z6.b, p7/z, [a_ptr2]\n"
- "udot z17.s, z9.b, z0.b[0]\n"
- "ld1rqb z7.b, p7/z, [a_ptr3]\n"
- "udot z21.s, z9.b, z1.b[0]\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
- "udot z25.s, z9.b, z2.b[0]\n"
- "udot z29.s, z9.b, z3.b[0]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "udot z18.s, z10.b, z0.b[0]\n"
- "udot z22.s, z10.b, z1.b[0]\n"
- "udot z26.s, z10.b, z2.b[0]\n"
- "udot z30.s, z10.b, z3.b[0]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "udot z19.s, z11.b, z0.b[0]\n"
- "udot z23.s, z11.b, z1.b[0]\n"
- "udot z27.s, z11.b, z2.b[0]\n"
- "udot z31.s, z11.b, z3.b[0]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "udot z16.s, z12.b, z0.b[1]\n"
- "udot z20.s, z12.b, z1.b[1]\n"
- "udot z24.s, z12.b, z2.b[1]\n"
- "udot z28.s, z12.b, z3.b[1]\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "udot z17.s, z13.b, z0.b[1]\n"
- "udot z21.s, z13.b, z1.b[1]\n"
- "udot z25.s, z13.b, z2.b[1]\n"
- "udot z29.s, z13.b, z3.b[1]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "udot z18.s, z14.b, z0.b[1]\n"
- "udot z22.s, z14.b, z1.b[1]\n"
- "udot z26.s, z14.b, z2.b[1]\n"
- "udot z30.s, z14.b, z3.b[1]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "udot z19.s, z15.b, z0.b[1]\n"
- "udot z23.s, z15.b, z1.b[1]\n"
- "udot z27.s, z15.b, z2.b[1]\n"
- "udot z31.s, z15.b, z3.b[1]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "udot z16.s, z8.b, z0.b[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "udot z20.s, z8.b, z1.b[2]\n"
- "udot z24.s, z8.b, z2.b[2]\n"
- "udot z28.s, z8.b, z3.b[2]\n"
- "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "udot z17.s, z9.b, z0.b[2]\n"
- "udot z21.s, z9.b, z1.b[2]\n"
- "udot z25.s, z9.b, z2.b[2]\n"
- "udot z29.s, z9.b, z3.b[2]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "udot z18.s, z10.b, z0.b[2]\n"
- "udot z22.s, z10.b, z1.b[2]\n"
- "udot z26.s, z10.b, z2.b[2]\n"
- "udot z30.s, z10.b, z3.b[2]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "udot z19.s, z11.b, z0.b[2]\n"
- "udot z23.s, z11.b, z1.b[2]\n"
- "udot z27.s, z11.b, z2.b[2]\n"
- "udot z31.s, z11.b, z3.b[2]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "udot z16.s, z12.b, z0.b[3]\n"
- "udot z20.s, z12.b, z1.b[3]\n"
- "udot z24.s, z12.b, z2.b[3]\n"
- "udot z28.s, z12.b, z3.b[3]\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "udot z17.s, z13.b, z0.b[3]\n"
- "udot z21.s, z13.b, z1.b[3]\n"
- "udot z25.s, z13.b, z2.b[3]\n"
- "udot z29.s, z13.b, z3.b[3]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "udot z18.s, z14.b, z0.b[3]\n"
- "udot z22.s, z14.b, z1.b[3]\n"
- "udot z26.s, z14.b, z2.b[3]\n"
- "udot z30.s, z14.b, z3.b[3]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "udot z19.s, z15.b, z0.b[3]\n"
- "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x10]\n"
- "udot z23.s, z15.b, z1.b[3]\n"
- "ld1rqb z1.b, p6/z, [a_ptr1, #0x10]\n"
- "udot z27.s, z15.b, z2.b[3]\n"
- "ld1rqb z2.b, p6/z, [a_ptr2, #0x10]\n"
- "udot z31.s, z15.b, z3.b[3]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "udot z16.s, z8.b, z4.b[0]\n"
- "ld1rqb z3.b, p6/z, [a_ptr3, #0x10]\n"
- "udot z20.s, z8.b, z5.b[0]\n"
- "addvl %[a_ptr0], %[a_ptr0], #2\n"
- "udot z24.s, z8.b, z6.b[0]\n"
- "addvl a_ptr1, a_ptr1, #2\n"
- "udot z28.s, z8.b, z7.b[0]\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
- "udot z17.s, z9.b, z4.b[0]\n"
- "addvl a_ptr2, a_ptr2, #2\n"
- "udot z21.s, z9.b, z5.b[0]\n"
- "addvl a_ptr3, a_ptr3, #2\n"
- "udot z25.s, z9.b, z6.b[0]\n"
- "udot z29.s, z9.b, z7.b[0]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "udot z18.s, z10.b, z4.b[0]\n"
- "udot z22.s, z10.b, z5.b[0]\n"
- "udot z26.s, z10.b, z6.b[0]\n"
- "udot z30.s, z10.b, z7.b[0]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "udot z19.s, z11.b, z4.b[0]\n"
- "udot z23.s, z11.b, z5.b[0]\n"
- "udot z27.s, z11.b, z6.b[0]\n"
- "udot z31.s, z11.b, z7.b[0]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "udot z16.s, z12.b, z4.b[1]\n"
- "udot z20.s, z12.b, z5.b[1]\n"
- "udot z24.s, z12.b, z6.b[1]\n"
- "udot z28.s, z12.b, z7.b[1]\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "udot z17.s, z13.b, z4.b[1]\n"
- "udot z21.s, z13.b, z5.b[1]\n"
- "udot z25.s, z13.b, z6.b[1]\n"
- "udot z29.s, z13.b, z7.b[1]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "udot z18.s, z14.b, z4.b[1]\n"
- "udot z22.s, z14.b, z5.b[1]\n"
- "udot z26.s, z14.b, z6.b[1]\n"
- "udot z30.s, z14.b, z7.b[1]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "udot z19.s, z15.b, z4.b[1]\n"
- "udot z23.s, z15.b, z5.b[1]\n"
- "udot z27.s, z15.b, z6.b[1]\n"
- "udot z31.s, z15.b, z7.b[1]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "udot z16.s, z8.b, z4.b[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "udot z20.s, z8.b, z5.b[2]\n"
- "udot z24.s, z8.b, z6.b[2]\n"
- "udot z28.s, z8.b, z7.b[2]\n"
- "udot z17.s, z9.b, z4.b[2]\n"
- "udot z21.s, z9.b, z5.b[2]\n"
- "udot z25.s, z9.b, z6.b[2]\n"
- "udot z29.s, z9.b, z7.b[2]\n"
- "udot z18.s, z10.b, z4.b[2]\n"
- "udot z22.s, z10.b, z5.b[2]\n"
- "udot z26.s, z10.b, z6.b[2]\n"
- "udot z30.s, z10.b, z7.b[2]\n"
- "udot z19.s, z11.b, z4.b[2]\n"
- "udot z23.s, z11.b, z5.b[2]\n"
- "udot z27.s, z11.b, z6.b[2]\n"
- "udot z31.s, z11.b, z7.b[2]\n"
- "udot z16.s, z12.b, z4.b[3]\n"
- "udot z20.s, z12.b, z5.b[3]\n"
- "udot z24.s, z12.b, z6.b[3]\n"
- "udot z28.s, z12.b, z7.b[3]\n"
- "udot z17.s, z13.b, z4.b[3]\n"
- "udot z21.s, z13.b, z5.b[3]\n"
- "udot z25.s, z13.b, z6.b[3]\n"
- "udot z29.s, z13.b, z7.b[3]\n"
- "udot z18.s, z14.b, z4.b[3]\n"
- "udot z22.s, z14.b, z5.b[3]\n"
- "udot z26.s, z14.b, z6.b[3]\n"
- "udot z30.s, z14.b, z7.b[3]\n"
- "udot z19.s, z15.b, z4.b[3]\n"
- "udot z23.s, z15.b, z5.b[3]\n"
- "udot z27.s, z15.b, z6.b[3]\n"
- "udot z31.s, z15.b, z7.b[3]\n"
- "cbz %[blocks], 5f\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "udot z16.s, z8.b, z0.b[0]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "udot z20.s, z8.b, z1.b[0]\n"
- "udot z24.s, z8.b, z2.b[0]\n"
- "udot z28.s, z8.b, z3.b[0]\n"
- "udot z17.s, z9.b, z0.b[0]\n"
- "udot z21.s, z9.b, z1.b[0]\n"
- "udot z25.s, z9.b, z2.b[0]\n"
- "udot z29.s, z9.b, z3.b[0]\n"
- "udot z18.s, z10.b, z0.b[0]\n"
- "udot z22.s, z10.b, z1.b[0]\n"
- "udot z26.s, z10.b, z2.b[0]\n"
- "udot z30.s, z10.b, z3.b[0]\n"
- "udot z19.s, z11.b, z0.b[0]\n"
- "udot z23.s, z11.b, z1.b[0]\n"
- "udot z27.s, z11.b, z2.b[0]\n"
- "udot z31.s, z11.b, z3.b[0]\n"
- "b.eq 5f\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "udot z16.s, z12.b, z0.b[1]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "udot z20.s, z12.b, z1.b[1]\n"
- "udot z24.s, z12.b, z2.b[1]\n"
- "udot z28.s, z12.b, z3.b[1]\n"
- "udot z17.s, z13.b, z0.b[1]\n"
- "udot z21.s, z13.b, z1.b[1]\n"
- "udot z25.s, z13.b, z2.b[1]\n"
- "udot z29.s, z13.b, z3.b[1]\n"
- "udot z18.s, z14.b, z0.b[1]\n"
- "udot z22.s, z14.b, z1.b[1]\n"
- "udot z26.s, z14.b, z2.b[1]\n"
- "udot z30.s, z14.b, z3.b[1]\n"
- "udot z19.s, z15.b, z0.b[1]\n"
- "udot z23.s, z15.b, z1.b[1]\n"
- "udot z27.s, z15.b, z2.b[1]\n"
- "udot z31.s, z15.b, z3.b[1]\n"
- "b.eq 5f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "udot z16.s, z8.b, z0.b[2]\n"
- "udot z20.s, z8.b, z1.b[2]\n"
- "udot z24.s, z8.b, z2.b[2]\n"
- "udot z28.s, z8.b, z3.b[2]\n"
- "udot z17.s, z9.b, z0.b[2]\n"
- "udot z21.s, z9.b, z1.b[2]\n"
- "udot z25.s, z9.b, z2.b[2]\n"
- "udot z29.s, z9.b, z3.b[2]\n"
- "udot z18.s, z10.b, z0.b[2]\n"
- "udot z22.s, z10.b, z1.b[2]\n"
- "udot z26.s, z10.b, z2.b[2]\n"
- "udot z30.s, z10.b, z3.b[2]\n"
- "udot z19.s, z11.b, z0.b[2]\n"
- "udot z23.s, z11.b, z1.b[2]\n"
- "udot z27.s, z11.b, z2.b[2]\n"
- "udot z31.s, z11.b, z3.b[2]\n"
- "b.eq 5f\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "udot z16.s, z12.b, z0.b[3]\n"
- "udot z20.s, z12.b, z1.b[3]\n"
- "udot z24.s, z12.b, z2.b[3]\n"
- "udot z28.s, z12.b, z3.b[3]\n"
- "udot z17.s, z13.b, z0.b[3]\n"
- "udot z21.s, z13.b, z1.b[3]\n"
- "udot z25.s, z13.b, z2.b[3]\n"
- "udot z29.s, z13.b, z3.b[3]\n"
- "udot z18.s, z14.b, z0.b[3]\n"
- "udot z22.s, z14.b, z1.b[3]\n"
- "udot z26.s, z14.b, z2.b[3]\n"
- "udot z30.s, z14.b, z3.b[3]\n"
- "udot z19.s, z15.b, z0.b[3]\n"
- "udot z23.s, z15.b, z1.b[3]\n"
- "udot z27.s, z15.b, z2.b[3]\n"
- "udot z31.s, z15.b, z3.b[3]\n"
- "b 5f\n"
- "4:\n"
- "udot z16.s, z8.b, z0.b[0]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "udot z20.s, z8.b, z1.b[0]\n"
- "ld1rqb z4.b, p6/z, [%[a_ptr0]]\n"
- "udot z24.s, z8.b, z2.b[0]\n"
- "ld1rqb z5.b, p6/z, [a_ptr1]\n"
- "udot z28.s, z8.b, z3.b[0]\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
- "udot z17.s, z9.b, z0.b[0]\n"
- "ld1rqb z6.b, p6/z, [a_ptr2]\n"
- "udot z21.s, z9.b, z1.b[0]\n"
- "ld1rqb z7.b, p6/z, [a_ptr3]\n"
- "udot z25.s, z9.b, z2.b[0]\n"
- "addvl %[a_ptr0], %[a_ptr0], #1\n"
- "udot z29.s, z9.b, z3.b[0]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "udot z18.s, z10.b, z0.b[0]\n"
- "addvl a_ptr1, a_ptr1, #1\n"
- "udot z22.s, z10.b, z1.b[0]\n"
- "addvl a_ptr2, a_ptr2, #1\n"
- "udot z26.s, z10.b, z2.b[0]\n"
- "addvl a_ptr3, a_ptr3, #1\n"
- "udot z30.s, z10.b, z3.b[0]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "udot z19.s, z11.b, z0.b[0]\n"
- "udot z23.s, z11.b, z1.b[0]\n"
- "udot z27.s, z11.b, z2.b[0]\n"
- "udot z31.s, z11.b, z3.b[0]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "udot z16.s, z12.b, z0.b[1]\n"
- "udot z20.s, z12.b, z1.b[1]\n"
- "udot z24.s, z12.b, z2.b[1]\n"
- "udot z28.s, z12.b, z3.b[1]\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "udot z17.s, z13.b, z0.b[1]\n"
- "udot z21.s, z13.b, z1.b[1]\n"
- "udot z25.s, z13.b, z2.b[1]\n"
- "udot z29.s, z13.b, z3.b[1]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "udot z18.s, z14.b, z0.b[1]\n"
- "udot z22.s, z14.b, z1.b[1]\n"
- "udot z26.s, z14.b, z2.b[1]\n"
- "udot z30.s, z14.b, z3.b[1]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "udot z19.s, z15.b, z0.b[1]\n"
- "udot z23.s, z15.b, z1.b[1]\n"
- "udot z27.s, z15.b, z2.b[1]\n"
- "udot z31.s, z15.b, z3.b[1]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "udot z16.s, z8.b, z0.b[2]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "udot z20.s, z8.b, z1.b[2]\n"
- "udot z24.s, z8.b, z2.b[2]\n"
- "udot z28.s, z8.b, z3.b[2]\n"
- "udot z17.s, z9.b, z0.b[2]\n"
- "udot z21.s, z9.b, z1.b[2]\n"
- "udot z25.s, z9.b, z2.b[2]\n"
- "udot z29.s, z9.b, z3.b[2]\n"
- "udot z18.s, z10.b, z0.b[2]\n"
- "udot z22.s, z10.b, z1.b[2]\n"
- "udot z26.s, z10.b, z2.b[2]\n"
- "udot z30.s, z10.b, z3.b[2]\n"
- "udot z19.s, z11.b, z0.b[2]\n"
- "udot z23.s, z11.b, z1.b[2]\n"
- "udot z27.s, z11.b, z2.b[2]\n"
- "udot z31.s, z11.b, z3.b[2]\n"
- "udot z16.s, z12.b, z0.b[3]\n"
- "udot z20.s, z12.b, z1.b[3]\n"
- "udot z24.s, z12.b, z2.b[3]\n"
- "udot z28.s, z12.b, z3.b[3]\n"
- "udot z17.s, z13.b, z0.b[3]\n"
- "udot z21.s, z13.b, z1.b[3]\n"
- "udot z25.s, z13.b, z2.b[3]\n"
- "udot z29.s, z13.b, z3.b[3]\n"
- "udot z18.s, z14.b, z0.b[3]\n"
- "udot z22.s, z14.b, z1.b[3]\n"
- "udot z26.s, z14.b, z2.b[3]\n"
- "udot z30.s, z14.b, z3.b[3]\n"
- "udot z19.s, z15.b, z0.b[3]\n"
- "udot z23.s, z15.b, z1.b[3]\n"
- "udot z27.s, z15.b, z2.b[3]\n"
- "udot z31.s, z15.b, z3.b[3]\n"
- "cbz %[blocks], 5f\n"
- "ld1b z8.b, p7/z, [%[b_ptr0]]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "udot z16.s, z8.b, z4.b[0]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "udot z20.s, z8.b, z5.b[0]\n"
- "udot z24.s, z8.b, z6.b[0]\n"
- "udot z28.s, z8.b, z7.b[0]\n"
- "udot z17.s, z9.b, z4.b[0]\n"
- "udot z21.s, z9.b, z5.b[0]\n"
- "udot z25.s, z9.b, z6.b[0]\n"
- "udot z29.s, z9.b, z7.b[0]\n"
- "udot z18.s, z10.b, z4.b[0]\n"
- "udot z22.s, z10.b, z5.b[0]\n"
- "udot z26.s, z10.b, z6.b[0]\n"
- "udot z30.s, z10.b, z7.b[0]\n"
- "udot z19.s, z11.b, z4.b[0]\n"
- "udot z23.s, z11.b, z5.b[0]\n"
- "udot z27.s, z11.b, z6.b[0]\n"
- "udot z31.s, z11.b, z7.b[0]\n"
- "b.eq 5f\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "udot z16.s, z12.b, z4.b[1]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "udot z20.s, z12.b, z5.b[1]\n"
- "udot z24.s, z12.b, z6.b[1]\n"
- "udot z28.s, z12.b, z7.b[1]\n"
- "udot z17.s, z13.b, z4.b[1]\n"
- "udot z21.s, z13.b, z5.b[1]\n"
- "udot z25.s, z13.b, z6.b[1]\n"
- "udot z29.s, z13.b, z7.b[1]\n"
- "udot z18.s, z14.b, z4.b[1]\n"
- "udot z22.s, z14.b, z5.b[1]\n"
- "udot z26.s, z14.b, z6.b[1]\n"
- "udot z30.s, z14.b, z7.b[1]\n"
- "udot z19.s, z15.b, z4.b[1]\n"
- "udot z23.s, z15.b, z5.b[1]\n"
- "udot z27.s, z15.b, z6.b[1]\n"
- "udot z31.s, z15.b, z7.b[1]\n"
- "b.eq 5f\n"
- "addvl %[b_ptr0], %[b_ptr0], #16\n"
- "subs %[blocks], %[blocks], #0x1\n"
- "ld1b z8.b, p7/z, [%[b_ptr0], #-8, MUL VL]\n"
- "ld1b z9.b, p7/z, [%[b_ptr0], #-7, MUL VL]\n"
- "ld1b z10.b, p7/z, [%[b_ptr0], #-6, MUL VL]\n"
- "ld1b z11.b, p7/z, [%[b_ptr0], #-5, MUL VL]\n"
- "udot z16.s, z8.b, z4.b[2]\n"
- "udot z20.s, z8.b, z5.b[2]\n"
- "udot z24.s, z8.b, z6.b[2]\n"
- "udot z28.s, z8.b, z7.b[2]\n"
- "udot z17.s, z9.b, z4.b[2]\n"
- "udot z21.s, z9.b, z5.b[2]\n"
- "udot z25.s, z9.b, z6.b[2]\n"
- "udot z29.s, z9.b, z7.b[2]\n"
- "udot z18.s, z10.b, z4.b[2]\n"
- "udot z22.s, z10.b, z5.b[2]\n"
- "udot z26.s, z10.b, z6.b[2]\n"
- "udot z30.s, z10.b, z7.b[2]\n"
- "udot z19.s, z11.b, z4.b[2]\n"
- "udot z23.s, z11.b, z5.b[2]\n"
- "udot z27.s, z11.b, z6.b[2]\n"
- "udot z31.s, z11.b, z7.b[2]\n"
- "b.eq 5f\n"
- "ld1b z12.b, p7/z, [%[b_ptr0], #-4, MUL VL]\n"
- "ld1b z13.b, p7/z, [%[b_ptr0], #-3, MUL VL]\n"
- "ld1b z14.b, p7/z, [%[b_ptr0], #-2, MUL VL]\n"
- "ld1b z15.b, p7/z, [%[b_ptr0], #-1, MUL VL]\n"
- "udot z16.s, z12.b, z4.b[3]\n"
- "udot z20.s, z12.b, z5.b[3]\n"
- "udot z24.s, z12.b, z6.b[3]\n"
- "udot z28.s, z12.b, z7.b[3]\n"
- "udot z17.s, z13.b, z4.b[3]\n"
- "udot z21.s, z13.b, z5.b[3]\n"
- "udot z25.s, z13.b, z6.b[3]\n"
- "udot z29.s, z13.b, z7.b[3]\n"
- "udot z18.s, z14.b, z4.b[3]\n"
- "udot z22.s, z14.b, z5.b[3]\n"
- "udot z26.s, z14.b, z6.b[3]\n"
- "udot z30.s, z14.b, z7.b[3]\n"
- "udot z19.s, z15.b, z4.b[3]\n"
- "udot z23.s, z15.b, z5.b[3]\n"
- "udot z27.s, z15.b, z6.b[3]\n"
- "udot z31.s, z15.b, z7.b[3]\n"
- "5:\n"
- "st1w z16.s, p0, [%[c_ptr0]]\n"
- "st1w z17.s, p1, [%[c_ptr0], #1, MUL VL]\n"
- "st1w z18.s, p2, [%[c_ptr0], #2, MUL VL]\n"
- "st1w z19.s, p3, [%[c_ptr0], #3, MUL VL]\n"
- "addvl %[c_ptr0], %[c_ptr0], #4\n"
- "st1w z20.s, p0, [c_ptr1]\n"
- "st1w z21.s, p1, [c_ptr1, #1, MUL VL]\n"
- "st1w z22.s, p2, [c_ptr1, #2, MUL VL]\n"
- "st1w z23.s, p3, [c_ptr1, #3, MUL VL]\n"
- "st1w z24.s, p0, [c_ptr2]\n"
- "st1w z25.s, p1, [c_ptr2, #1, MUL VL]\n"
- "st1w z26.s, p2, [c_ptr2, #2, MUL VL]\n"
- "st1w z27.s, p3, [c_ptr2, #3, MUL VL]\n"
- "st1w z28.s, p0, [c_ptr3]\n"
- "st1w z29.s, p1, [c_ptr3, #1, MUL VL]\n"
- "st1w z30.s, p2, [c_ptr3, #2, MUL VL]\n"
- "st1w z31.s, p3, [c_ptr3, #3, MUL VL]\n"
- ".unreq a_ptr1\n"
- ".unreq a_ptr2\n"
- ".unreq a_ptr3\n"
- ".unreq c_ptr1\n"
- ".unreq c_ptr2\n"
- ".unreq c_ptr3\n"
- : [a_ptr0] "+r" (a_ptr0), [b_ptr0] "+r" (b_ptr0), [c_ptr0] "+r" (c_ptr0), [loops] "+r" (loops), [regs] "+r" (regs), [temp] "+r" (temp), [blocks] "+r" (blocks)
- : [width] "r" (width), [accumulate] "r" (static_cast<uint64_t>(accumulate)), [lda] "r" (ldab), [ldc] "r" (ldcb), [leftovers] "r" (leftovers)
- : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "x0", "x1", "x2", "x3", "x4", "x5", "cc", "memory"
- );
- break;
- }
-
- }
- }
-}
-
-} // namespace arm_gemm
-
-#endif // __ARM_FEATURE_SVE
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8u32_dot_4VLx4.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8u32_dot_6x4VL.hpp
similarity index 64%
copy from src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8u32_dot_4VLx4.hpp
copy to src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8u32_dot_6x4VL.hpp
index c325e52..af9de4a 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8u32_dot_4VLx4.hpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8u32_dot_6x4VL.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2018-2020 Arm Limited.
+ * Copyright (c) 2019-2020 Arm Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -10,42 +10,48 @@
* sell copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
*/
#pragma once
-
#ifdef __ARM_FEATURE_SVE
-#include <cstdint>
#include "../std_transforms_sve.hpp"
+#define ARGLIST \
+ unsigned int, const unsigned int *, \
+ IndirectInputArg<uint8_t>, \
+ size_t, size_t, \
+ const uint8_t *, \
+ IndirectOutputArg<uint32_t>, \
+ const uint32_t *, Activation, bool
+
namespace arm_gemm
{
// Actual kernel implementations
-void sve_hybrid_u8u32_dot_4VLx4(const uint8_t *, int, const uint8_t *, uint32_t *, int, int, int, int, const uint32_t *, Activation, bool);
+void sve_hybrid_u8u32_dot_6x4VL( ARGLIST );
-class hybrid_u8u32_dot_4VLx4
+class cls_sve_hybrid_u8u32_dot_6x4VL
{
public:
typedef uint8_t operand_type;
typedef uint32_t result_type;
- typedef void (*kern_type)(const uint8_t *, int, const uint8_t *, uint32_t *, int, int, int, int, const uint32_t *, Activation, bool);
+ typedef void (*kern_type)( ARGLIST );
/* Kernel blocking parameters */
static constexpr unsigned int out_height()
{
- return 4;
+ return 6;
}
static unsigned int out_width()
@@ -63,27 +69,17 @@
return true;
}
- static constexpr bool supports_bias()
- {
- return false;
- }
-
- static constexpr bool supports_activation()
- {
- return false;
- }
-
- StdTransformsSVE<operand_type, result_type, 4, 4, 4> transforms = {};
+ StdTransformsSVE<operand_type, result_type, 6, 4, 4> transforms = {};
// Default to the generic kernel
- kern_type kernel=sve_hybrid_u8u32_dot_4VLx4;
+ kern_type kernel=sve_hybrid_u8u32_dot_6x4VL;
- hybrid_u8u32_dot_4VLx4(const CPUInfo *)
+ cls_sve_hybrid_u8u32_dot_6x4VL(const CPUInfo *)
{
-
}
};
} // namespace arm_gemm
+#undef ARGLIST
#endif // __ARM_FEATURE_SVE
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8u32_dot_6x4VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8u32_dot_6x4VL/generic.cpp
new file mode 100644
index 0000000..fc8ce63
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8u32_dot_6x4VL/generic.cpp
@@ -0,0 +1,1904 @@
+/*
+ * Copyright (c) 2019-2020 Arm Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+#ifdef __ARM_FEATURE_SVE
+
+#include "arm_gemm.hpp"
+#include "../../utils.hpp"
+
+#include <cassert>
+
+namespace arm_gemm {
+
+void sve_hybrid_u8u32_dot_6x4VL (
+ unsigned int num_strings, const unsigned int *string_lengths, IndirectInputArg<uint8_t> A_arg,
+ size_t M, size_t N, const uint8_t *B_ptr, IndirectOutputArg<uint32_t> output_arg,
+ const uint32_t *, Activation, bool accumulate
+)
+{
+ struct KernelArgs {
+ unsigned int num_strings = {};
+ const unsigned int *string_lengths = {};
+ size_t N = {};
+ const uint8_t *B_ptr = {};
+ size_t output_offset = {};
+ size_t input_initial_col = {};
+ size_t input_offset = {};
+ } ka;
+
+ unsigned long flags=0;
+ void *output_ptr;
+ void *input_ptr;
+
+ if (output_arg.is_indirect) {
+ output_ptr=(void *)(output_arg.indirect.ptr);
+ ka.output_offset=output_arg.indirect.offset;
+ flags |= 0x4;
+ } else {
+ output_ptr=(void *)(output_arg.direct.base);
+ ka.output_offset=output_arg.direct.stride;
+ }
+
+ if (A_arg.is_indirect) {
+ input_ptr=(void *)(A_arg.indirect.ptr);
+ ka.input_offset=A_arg.indirect.start_row;
+ ka.input_initial_col=A_arg.indirect.start_col;
+ flags |= 0x8;
+ } else {
+ assert(num_strings==1);
+ input_ptr=(void *)(A_arg.direct.base);
+ ka.input_offset=A_arg.direct.stride;
+ }
+ if (accumulate) {
+ flags |= 0x1;
+ }
+ ka.num_strings = num_strings;
+ ka.string_lengths = string_lengths;
+ ka.N = N;
+ ka.B_ptr = B_ptr;
+ __asm__ __volatile__(
+ "ptrue p5.b\n"
+ "1:" // Row loop
+ "cmp %x[M], #0x6\n"
+ "bge 61f\n"
+ "cmp %x[M], #0x4\n"
+ "bgt 49f\n"
+ "beq 37f\n"
+ "cmp %x[M], #0x2\n"
+ "bgt 25f\n"
+ "beq 13f\n"
+ "ldr x15, [%x[args_ptr], %[offsetof_N]]\n"
+ "ldr x14, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 2f\n"
+ "ldr x13, [%x[output_ptr], #0x0]\n"
+ "add x13, x13, x19, LSL #2\n"
+ "b 3f\n"
+ "2:" // Height 1: setup direct output
+ "mov x13, %x[output_ptr]\n"
+ "3:" // Height 1: Column loop
+ "mov x19, #0x0\n"
+ "whilelt p4.s, x19, x15\n"
+ "incw x19\n"
+ "whilelt p3.s, x19, x15\n"
+ "incw x19\n"
+ "whilelt p2.s, x19, x15\n"
+ "incw x19\n"
+ "whilelt p1.s, x19, x15\n"
+ "tbz %x[flags], #0, 4f\n"
+ "ld1w { z8.s }, p4/Z, [x13]\n"
+ "ld1w { z9.s }, p3/Z, [x13, #1, MUL VL]\n"
+ "ld1w { z10.s }, p2/Z, [x13, #2, MUL VL]\n"
+ "ld1w { z11.s }, p1/Z, [x13, #3, MUL VL]\n"
+ "b 5f\n"
+ "4:" // Height 1: no accumulate
+ "mov z8.s, #0x0\n"
+ "mov z9.s, #0x0\n"
+ "mov z10.s, #0x0\n"
+ "mov z11.s, #0x0\n"
+ "5:" // Height 1: setup done
+ "mov x12, #0x0\n"
+ "6:" // Height 1: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w11, [x20, x12, LSL #0x2]\n"
+ "tbz %x[flags], #3, 7f\n"
+ "ldr x20, [%x[input_ptr], x12, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x10, [x20, #0x0]\n"
+ "cbnz x12, 8f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x10, x10, x19\n"
+ "b 8f\n"
+ "7:" // Height 1: setup direct input
+ "mov x10, %x[input_ptr]\n"
+ "8:" // Height 1: input setup done
+ "cmp x11, #0x10\n"
+ "ble 10f\n"
+ "9:" // Height 1: Multiply loop: Main loop head
+ "ld1b { z6.b }, p5/Z, [x14]\n"
+ "whilelt p0.b, XZR, x11\n"
+ "ld1b { z7.b }, p5/Z, [x14, #1, MUL VL]\n"
+ "sub x11, x11, #0x10\n"
+ "ld1rqb { z0.b }, p0/Z, [x10]\n"
+ "udot z8.s, z6.b, z0.b[0]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #2, MUL VL]\n"
+ "add x10, x10, #0x10\n"
+ "udot z9.s, z7.b, z0.b[0]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #3, MUL VL]\n"
+ "cmp x11, #0x10\n"
+ "udot z10.s, z6.b, z0.b[0]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #4, MUL VL]\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "udot z11.s, z7.b, z0.b[0]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #5, MUL VL]\n"
+ "udot z8.s, z6.b, z0.b[1]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #6, MUL VL]\n"
+ "udot z9.s, z7.b, z0.b[1]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #7, MUL VL]\n"
+ "addvl x14, x14, #16\n"
+ "udot z10.s, z6.b, z0.b[1]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #-8, MUL VL]\n"
+ "udot z11.s, z7.b, z0.b[1]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #-7, MUL VL]\n"
+ "udot z8.s, z6.b, z0.b[2]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #-6, MUL VL]\n"
+ "udot z9.s, z7.b, z0.b[2]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #-5, MUL VL]\n"
+ "udot z10.s, z6.b, z0.b[2]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #-4, MUL VL]\n"
+ "udot z11.s, z7.b, z0.b[2]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #-3, MUL VL]\n"
+ "udot z8.s, z6.b, z0.b[3]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #-2, MUL VL]\n"
+ "udot z9.s, z7.b, z0.b[3]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #-1, MUL VL]\n"
+ "udot z10.s, z6.b, z0.b[3]\n"
+ "udot z11.s, z7.b, z0.b[3]\n"
+ "bgt 9b\n"
+ "10:" // Height 1: Multiply loop: Single iteration only
+ "ld1b { z6.b }, p5/Z, [x14]\n"
+ "whilelt p0.b, XZR, x11\n"
+ "ld1b { z7.b }, p5/Z, [x14, #1, MUL VL]\n"
+ "subs x11, x11, #0x4\n"
+ "ld1rqb { z0.b }, p0/Z, [x10]\n"
+ "udot z8.s, z6.b, z0.b[0]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #2, MUL VL]\n"
+ "add x10, x10, #0x10\n"
+ "udot z9.s, z7.b, z0.b[0]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #3, MUL VL]\n"
+ "addvl x14, x14, #4\n"
+ "udot z10.s, z6.b, z0.b[0]\n"
+ "udot z11.s, z7.b, z0.b[0]\n"
+ "ble 11f\n"
+ "ld1b { z6.b }, p5/Z, [x14]\n"
+ "udot z8.s, z6.b, z0.b[1]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #1, MUL VL]\n"
+ "subs x11, x11, #0x4\n"
+ "udot z9.s, z7.b, z0.b[1]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #2, MUL VL]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #3, MUL VL]\n"
+ "udot z10.s, z6.b, z0.b[1]\n"
+ "addvl x14, x14, #4\n"
+ "udot z11.s, z7.b, z0.b[1]\n"
+ "ble 11f\n"
+ "ld1b { z6.b }, p5/Z, [x14]\n"
+ "udot z8.s, z6.b, z0.b[2]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #1, MUL VL]\n"
+ "subs x11, x11, #0x4\n"
+ "udot z9.s, z7.b, z0.b[2]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #2, MUL VL]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #3, MUL VL]\n"
+ "udot z10.s, z6.b, z0.b[2]\n"
+ "addvl x14, x14, #4\n"
+ "udot z11.s, z7.b, z0.b[2]\n"
+ "ble 11f\n"
+ "ld1b { z6.b }, p5/Z, [x14]\n"
+ "udot z8.s, z6.b, z0.b[3]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #1, MUL VL]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #2, MUL VL]\n"
+ "udot z9.s, z7.b, z0.b[3]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #3, MUL VL]\n"
+ "addvl x14, x14, #4\n"
+ "udot z10.s, z6.b, z0.b[3]\n"
+ "udot z11.s, z7.b, z0.b[3]\n"
+ "11:" // Height 1: Multiply loop: multiply skip
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x12, x12, #0x1\n"
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "cmp x12, x19\n"
+ "bne 6b\n"
+ "prfm pstl1keep, [x13, #0x0]\n"
+ "st1w { z8.s }, p4, [x13]\n"
+ "st1w { z9.s }, p3, [x13, #1, MUL VL]\n"
+ "st1w { z10.s }, p2, [x13, #2, MUL VL]\n"
+ "st1w { z11.s }, p1, [x13, #3, MUL VL]\n"
+ "addvl x13, x13, #4\n"
+ "12:" // Height 1: Writeback done
+ "mov x19, #0x0\n"
+ "incw x19, ALL, MUL #4\n"
+ "subs x15, x15, x19\n"
+ "bgt 3b\n"
+ "b 74f\n"
+ "13:" // Height 2
+ "ldr x15, [%x[args_ptr], %[offsetof_N]]\n"
+ "ldr x14, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 14f\n"
+ "ldr x13, [%x[output_ptr], #0x0]\n"
+ "add x13, x13, x19, LSL #2\n"
+ "ldr x9, [%x[output_ptr], #0x8]\n"
+ "add x9, x9, x19, LSL #2\n"
+ "b 15f\n"
+ "14:" // Height 2: setup direct output
+ "mov x13, %x[output_ptr]\n"
+ "add x9, x13, x19, LSL #2\n"
+ "15:" // Height 2: Column loop
+ "mov x19, #0x0\n"
+ "whilelt p4.s, x19, x15\n"
+ "incw x19\n"
+ "whilelt p3.s, x19, x15\n"
+ "incw x19\n"
+ "whilelt p2.s, x19, x15\n"
+ "incw x19\n"
+ "whilelt p1.s, x19, x15\n"
+ "tbz %x[flags], #0, 16f\n"
+ "ld1w { z8.s }, p4/Z, [x13]\n"
+ "ld1w { z9.s }, p3/Z, [x13, #1, MUL VL]\n"
+ "ld1w { z10.s }, p2/Z, [x13, #2, MUL VL]\n"
+ "ld1w { z11.s }, p1/Z, [x13, #3, MUL VL]\n"
+ "ld1w { z12.s }, p4/Z, [x9]\n"
+ "ld1w { z13.s }, p3/Z, [x9, #1, MUL VL]\n"
+ "ld1w { z14.s }, p2/Z, [x9, #2, MUL VL]\n"
+ "ld1w { z15.s }, p1/Z, [x9, #3, MUL VL]\n"
+ "b 17f\n"
+ "16:" // Height 2: no accumulate
+ "mov z8.s, #0x0\n"
+ "mov z9.s, #0x0\n"
+ "mov z10.s, #0x0\n"
+ "mov z11.s, #0x0\n"
+ "mov z12.s, #0x0\n"
+ "mov z13.s, #0x0\n"
+ "mov z14.s, #0x0\n"
+ "mov z15.s, #0x0\n"
+ "17:" // Height 2: setup done
+ "mov x12, #0x0\n"
+ "18:" // Height 2: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w11, [x20, x12, LSL #0x2]\n"
+ "tbz %x[flags], #3, 19f\n"
+ "ldr x20, [%x[input_ptr], x12, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x10, [x20, #0x0]\n"
+ "ldr x28, [x20, #0x8]\n"
+ "cbnz x12, 20f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x10, x10, x19\n"
+ "add x28, x28, x19\n"
+ "b 20f\n"
+ "19:" // Height 2: setup direct input
+ "mov x10, %x[input_ptr]\n"
+ "add x28, x10, x19\n"
+ "20:" // Height 2: input setup done
+ "cmp x11, #0x10\n"
+ "ble 22f\n"
+ "21:" // Height 2: Multiply loop: Main loop head
+ "ld1b { z6.b }, p5/Z, [x14]\n"
+ "whilelt p0.b, XZR, x11\n"
+ "ld1b { z7.b }, p5/Z, [x14, #1, MUL VL]\n"
+ "sub x11, x11, #0x10\n"
+ "ld1rqb { z0.b }, p0/Z, [x10]\n"
+ "udot z8.s, z6.b, z0.b[0]\n"
+ "ld1rqb { z1.b }, p0/Z, [x28]\n"
+ "add x10, x10, #0x10\n"
+ "udot z9.s, z7.b, z0.b[0]\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x28, x28, #0x10\n"
+ "udot z12.s, z6.b, z1.b[0]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #2, MUL VL]\n"
+ "cmp x11, #0x10\n"
+ "udot z13.s, z7.b, z1.b[0]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #3, MUL VL]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "udot z10.s, z6.b, z0.b[0]\n"
+ "udot z14.s, z6.b, z1.b[0]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #4, MUL VL]\n"
+ "udot z11.s, z7.b, z0.b[0]\n"
+ "udot z15.s, z7.b, z1.b[0]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #5, MUL VL]\n"
+ "udot z8.s, z6.b, z0.b[1]\n"
+ "udot z12.s, z6.b, z1.b[1]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #6, MUL VL]\n"
+ "udot z9.s, z7.b, z0.b[1]\n"
+ "udot z13.s, z7.b, z1.b[1]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #7, MUL VL]\n"
+ "addvl x14, x14, #16\n"
+ "udot z10.s, z6.b, z0.b[1]\n"
+ "udot z14.s, z6.b, z1.b[1]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #-8, MUL VL]\n"
+ "udot z11.s, z7.b, z0.b[1]\n"
+ "udot z15.s, z7.b, z1.b[1]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #-7, MUL VL]\n"
+ "udot z8.s, z6.b, z0.b[2]\n"
+ "udot z12.s, z6.b, z1.b[2]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #-6, MUL VL]\n"
+ "udot z9.s, z7.b, z0.b[2]\n"
+ "udot z13.s, z7.b, z1.b[2]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #-5, MUL VL]\n"
+ "udot z10.s, z6.b, z0.b[2]\n"
+ "udot z14.s, z6.b, z1.b[2]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #-4, MUL VL]\n"
+ "udot z11.s, z7.b, z0.b[2]\n"
+ "udot z15.s, z7.b, z1.b[2]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #-3, MUL VL]\n"
+ "udot z8.s, z6.b, z0.b[3]\n"
+ "udot z12.s, z6.b, z1.b[3]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #-2, MUL VL]\n"
+ "udot z9.s, z7.b, z0.b[3]\n"
+ "udot z13.s, z7.b, z1.b[3]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #-1, MUL VL]\n"
+ "udot z10.s, z6.b, z0.b[3]\n"
+ "udot z14.s, z6.b, z1.b[3]\n"
+ "udot z11.s, z7.b, z0.b[3]\n"
+ "udot z15.s, z7.b, z1.b[3]\n"
+ "bgt 21b\n"
+ "22:" // Height 2: Multiply loop: Single iteration only
+ "ld1b { z6.b }, p5/Z, [x14]\n"
+ "whilelt p0.b, XZR, x11\n"
+ "ld1b { z7.b }, p5/Z, [x14, #1, MUL VL]\n"
+ "subs x11, x11, #0x4\n"
+ "ld1rqb { z0.b }, p0/Z, [x10]\n"
+ "udot z8.s, z6.b, z0.b[0]\n"
+ "ld1rqb { z1.b }, p0/Z, [x28]\n"
+ "add x10, x10, #0x10\n"
+ "udot z9.s, z7.b, z0.b[0]\n"
+ "add x28, x28, #0x10\n"
+ "udot z12.s, z6.b, z1.b[0]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #2, MUL VL]\n"
+ "udot z13.s, z7.b, z1.b[0]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #3, MUL VL]\n"
+ "addvl x14, x14, #4\n"
+ "udot z10.s, z6.b, z0.b[0]\n"
+ "udot z14.s, z6.b, z1.b[0]\n"
+ "udot z11.s, z7.b, z0.b[0]\n"
+ "udot z15.s, z7.b, z1.b[0]\n"
+ "ble 23f\n"
+ "ld1b { z6.b }, p5/Z, [x14]\n"
+ "udot z8.s, z6.b, z0.b[1]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #1, MUL VL]\n"
+ "subs x11, x11, #0x4\n"
+ "udot z12.s, z6.b, z1.b[1]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #2, MUL VL]\n"
+ "udot z9.s, z7.b, z0.b[1]\n"
+ "udot z13.s, z7.b, z1.b[1]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #3, MUL VL]\n"
+ "addvl x14, x14, #4\n"
+ "udot z10.s, z6.b, z0.b[1]\n"
+ "udot z14.s, z6.b, z1.b[1]\n"
+ "udot z11.s, z7.b, z0.b[1]\n"
+ "udot z15.s, z7.b, z1.b[1]\n"
+ "ble 23f\n"
+ "ld1b { z6.b }, p5/Z, [x14]\n"
+ "udot z8.s, z6.b, z0.b[2]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #1, MUL VL]\n"
+ "subs x11, x11, #0x4\n"
+ "udot z12.s, z6.b, z1.b[2]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #2, MUL VL]\n"
+ "udot z9.s, z7.b, z0.b[2]\n"
+ "udot z13.s, z7.b, z1.b[2]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #3, MUL VL]\n"
+ "addvl x14, x14, #4\n"
+ "udot z10.s, z6.b, z0.b[2]\n"
+ "udot z14.s, z6.b, z1.b[2]\n"
+ "udot z11.s, z7.b, z0.b[2]\n"
+ "udot z15.s, z7.b, z1.b[2]\n"
+ "ble 23f\n"
+ "ld1b { z6.b }, p5/Z, [x14]\n"
+ "udot z8.s, z6.b, z0.b[3]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #1, MUL VL]\n"
+ "udot z12.s, z6.b, z1.b[3]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #2, MUL VL]\n"
+ "udot z9.s, z7.b, z0.b[3]\n"
+ "udot z13.s, z7.b, z1.b[3]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #3, MUL VL]\n"
+ "addvl x14, x14, #4\n"
+ "udot z10.s, z6.b, z0.b[3]\n"
+ "udot z14.s, z6.b, z1.b[3]\n"
+ "udot z11.s, z7.b, z0.b[3]\n"
+ "udot z15.s, z7.b, z1.b[3]\n"
+ "23:" // Height 2: Multiply loop: multiply skip
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x12, x12, #0x1\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "cmp x12, x19\n"
+ "bne 18b\n"
+ "prfm pstl1keep, [x13, #0x0]\n"
+ "prfm pstl1keep, [x9, #0x0]\n"
+ "st1w { z8.s }, p4, [x13]\n"
+ "st1w { z9.s }, p3, [x13, #1, MUL VL]\n"
+ "st1w { z10.s }, p2, [x13, #2, MUL VL]\n"
+ "st1w { z11.s }, p1, [x13, #3, MUL VL]\n"
+ "addvl x13, x13, #4\n"
+ "st1w { z12.s }, p4, [x9]\n"
+ "st1w { z13.s }, p3, [x9, #1, MUL VL]\n"
+ "st1w { z14.s }, p2, [x9, #2, MUL VL]\n"
+ "st1w { z15.s }, p1, [x9, #3, MUL VL]\n"
+ "addvl x9, x9, #4\n"
+ "24:" // Height 2: Writeback done
+ "mov x19, #0x0\n"
+ "incw x19, ALL, MUL #4\n"
+ "subs x15, x15, x19\n"
+ "bgt 15b\n"
+ "b 74f\n"
+ "25:" // Height 3
+ "ldr x15, [%x[args_ptr], %[offsetof_N]]\n"
+ "ldr x14, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 26f\n"
+ "ldr x13, [%x[output_ptr], #0x0]\n"
+ "add x13, x13, x19, LSL #2\n"
+ "ldr x9, [%x[output_ptr], #0x8]\n"
+ "ldr x27, [%x[output_ptr], #0x10]\n"
+ "add x9, x9, x19, LSL #2\n"
+ "add x27, x27, x19, LSL #2\n"
+ "b 27f\n"
+ "26:" // Height 3: setup direct output
+ "mov x13, %x[output_ptr]\n"
+ "add x9, x13, x19, LSL #2\n"
+ "add x27, x9, x19, LSL #2\n"
+ "27:" // Height 3: Column loop
+ "mov x19, #0x0\n"
+ "whilelt p4.s, x19, x15\n"
+ "incw x19\n"
+ "whilelt p3.s, x19, x15\n"
+ "incw x19\n"
+ "whilelt p2.s, x19, x15\n"
+ "incw x19\n"
+ "whilelt p1.s, x19, x15\n"
+ "tbz %x[flags], #0, 28f\n"
+ "ld1w { z8.s }, p4/Z, [x13]\n"
+ "ld1w { z9.s }, p3/Z, [x13, #1, MUL VL]\n"
+ "ld1w { z10.s }, p2/Z, [x13, #2, MUL VL]\n"
+ "ld1w { z11.s }, p1/Z, [x13, #3, MUL VL]\n"
+ "ld1w { z12.s }, p4/Z, [x9]\n"
+ "ld1w { z13.s }, p3/Z, [x9, #1, MUL VL]\n"
+ "ld1w { z14.s }, p2/Z, [x9, #2, MUL VL]\n"
+ "ld1w { z15.s }, p1/Z, [x9, #3, MUL VL]\n"
+ "ld1w { z16.s }, p4/Z, [x27]\n"
+ "ld1w { z17.s }, p3/Z, [x27, #1, MUL VL]\n"
+ "ld1w { z18.s }, p2/Z, [x27, #2, MUL VL]\n"
+ "ld1w { z19.s }, p1/Z, [x27, #3, MUL VL]\n"
+ "b 29f\n"
+ "28:" // Height 3: no accumulate
+ "mov z8.s, #0x0\n"
+ "mov z9.s, #0x0\n"
+ "mov z10.s, #0x0\n"
+ "mov z11.s, #0x0\n"
+ "mov z12.s, #0x0\n"
+ "mov z13.s, #0x0\n"
+ "mov z14.s, #0x0\n"
+ "mov z15.s, #0x0\n"
+ "mov z16.s, #0x0\n"
+ "mov z17.s, #0x0\n"
+ "mov z18.s, #0x0\n"
+ "mov z19.s, #0x0\n"
+ "29:" // Height 3: setup done
+ "mov x12, #0x0\n"
+ "30:" // Height 3: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w11, [x20, x12, LSL #0x2]\n"
+ "tbz %x[flags], #3, 31f\n"
+ "ldr x20, [%x[input_ptr], x12, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x10, [x20, #0x0]\n"
+ "ldr x28, [x20, #0x8]\n"
+ "ldr x26, [x20, #0x10]\n"
+ "cbnz x12, 32f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x10, x10, x19\n"
+ "add x28, x28, x19\n"
+ "add x26, x26, x19\n"
+ "b 32f\n"
+ "31:" // Height 3: setup direct input
+ "mov x10, %x[input_ptr]\n"
+ "add x28, x10, x19\n"
+ "add x26, x28, x19\n"
+ "32:" // Height 3: input setup done
+ "cmp x11, #0x10\n"
+ "ble 34f\n"
+ "33:" // Height 3: Multiply loop: Main loop head
+ "ld1b { z6.b }, p5/Z, [x14]\n"
+ "whilelt p0.b, XZR, x11\n"
+ "ld1b { z7.b }, p5/Z, [x14, #1, MUL VL]\n"
+ "sub x11, x11, #0x10\n"
+ "ld1rqb { z0.b }, p0/Z, [x10]\n"
+ "udot z8.s, z6.b, z0.b[0]\n"
+ "ld1rqb { z1.b }, p0/Z, [x28]\n"
+ "add x10, x10, #0x10\n"
+ "udot z9.s, z7.b, z0.b[0]\n"
+ "ld1rqb { z2.b }, p0/Z, [x26]\n"
+ "add x28, x28, #0x10\n"
+ "udot z12.s, z6.b, z1.b[0]\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x26, x26, #0x10\n"
+ "udot z16.s, z6.b, z2.b[0]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #2, MUL VL]\n"
+ "cmp x11, #0x10\n"
+ "udot z13.s, z7.b, z1.b[0]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "udot z17.s, z7.b, z2.b[0]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #3, MUL VL]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "udot z10.s, z6.b, z0.b[0]\n"
+ "udot z14.s, z6.b, z1.b[0]\n"
+ "udot z18.s, z6.b, z2.b[0]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #4, MUL VL]\n"
+ "udot z11.s, z7.b, z0.b[0]\n"
+ "udot z15.s, z7.b, z1.b[0]\n"
+ "udot z19.s, z7.b, z2.b[0]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #5, MUL VL]\n"
+ "udot z8.s, z6.b, z0.b[1]\n"
+ "udot z12.s, z6.b, z1.b[1]\n"
+ "udot z16.s, z6.b, z2.b[1]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #6, MUL VL]\n"
+ "udot z9.s, z7.b, z0.b[1]\n"
+ "udot z13.s, z7.b, z1.b[1]\n"
+ "udot z17.s, z7.b, z2.b[1]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #7, MUL VL]\n"
+ "addvl x14, x14, #16\n"
+ "udot z10.s, z6.b, z0.b[1]\n"
+ "udot z14.s, z6.b, z1.b[1]\n"
+ "udot z18.s, z6.b, z2.b[1]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #-8, MUL VL]\n"
+ "udot z11.s, z7.b, z0.b[1]\n"
+ "udot z15.s, z7.b, z1.b[1]\n"
+ "udot z19.s, z7.b, z2.b[1]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #-7, MUL VL]\n"
+ "udot z8.s, z6.b, z0.b[2]\n"
+ "udot z12.s, z6.b, z1.b[2]\n"
+ "udot z16.s, z6.b, z2.b[2]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #-6, MUL VL]\n"
+ "udot z9.s, z7.b, z0.b[2]\n"
+ "udot z13.s, z7.b, z1.b[2]\n"
+ "udot z17.s, z7.b, z2.b[2]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #-5, MUL VL]\n"
+ "udot z10.s, z6.b, z0.b[2]\n"
+ "udot z14.s, z6.b, z1.b[2]\n"
+ "udot z18.s, z6.b, z2.b[2]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #-4, MUL VL]\n"
+ "udot z11.s, z7.b, z0.b[2]\n"
+ "udot z15.s, z7.b, z1.b[2]\n"
+ "udot z19.s, z7.b, z2.b[2]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #-3, MUL VL]\n"
+ "udot z8.s, z6.b, z0.b[3]\n"
+ "udot z12.s, z6.b, z1.b[3]\n"
+ "udot z16.s, z6.b, z2.b[3]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #-2, MUL VL]\n"
+ "udot z9.s, z7.b, z0.b[3]\n"
+ "udot z13.s, z7.b, z1.b[3]\n"
+ "udot z17.s, z7.b, z2.b[3]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #-1, MUL VL]\n"
+ "udot z10.s, z6.b, z0.b[3]\n"
+ "udot z14.s, z6.b, z1.b[3]\n"
+ "udot z18.s, z6.b, z2.b[3]\n"
+ "udot z11.s, z7.b, z0.b[3]\n"
+ "udot z15.s, z7.b, z1.b[3]\n"
+ "udot z19.s, z7.b, z2.b[3]\n"
+ "bgt 33b\n"
+ "34:" // Height 3: Multiply loop: Single iteration only
+ "ld1b { z6.b }, p5/Z, [x14]\n"
+ "whilelt p0.b, XZR, x11\n"
+ "ld1b { z7.b }, p5/Z, [x14, #1, MUL VL]\n"
+ "subs x11, x11, #0x4\n"
+ "ld1rqb { z0.b }, p0/Z, [x10]\n"
+ "udot z8.s, z6.b, z0.b[0]\n"
+ "ld1rqb { z1.b }, p0/Z, [x28]\n"
+ "add x10, x10, #0x10\n"
+ "udot z9.s, z7.b, z0.b[0]\n"
+ "ld1rqb { z2.b }, p0/Z, [x26]\n"
+ "add x28, x28, #0x10\n"
+ "udot z12.s, z6.b, z1.b[0]\n"
+ "add x26, x26, #0x10\n"
+ "udot z13.s, z7.b, z1.b[0]\n"
+ "udot z16.s, z6.b, z2.b[0]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #2, MUL VL]\n"
+ "udot z17.s, z7.b, z2.b[0]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #3, MUL VL]\n"
+ "addvl x14, x14, #4\n"
+ "udot z10.s, z6.b, z0.b[0]\n"
+ "udot z14.s, z6.b, z1.b[0]\n"
+ "udot z18.s, z6.b, z2.b[0]\n"
+ "udot z11.s, z7.b, z0.b[0]\n"
+ "udot z15.s, z7.b, z1.b[0]\n"
+ "udot z19.s, z7.b, z2.b[0]\n"
+ "ble 35f\n"
+ "ld1b { z6.b }, p5/Z, [x14]\n"
+ "udot z8.s, z6.b, z0.b[1]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #1, MUL VL]\n"
+ "subs x11, x11, #0x4\n"
+ "udot z12.s, z6.b, z1.b[1]\n"
+ "udot z16.s, z6.b, z2.b[1]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #2, MUL VL]\n"
+ "udot z9.s, z7.b, z0.b[1]\n"
+ "udot z13.s, z7.b, z1.b[1]\n"
+ "udot z17.s, z7.b, z2.b[1]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #3, MUL VL]\n"
+ "addvl x14, x14, #4\n"
+ "udot z10.s, z6.b, z0.b[1]\n"
+ "udot z14.s, z6.b, z1.b[1]\n"
+ "udot z18.s, z6.b, z2.b[1]\n"
+ "udot z11.s, z7.b, z0.b[1]\n"
+ "udot z15.s, z7.b, z1.b[1]\n"
+ "udot z19.s, z7.b, z2.b[1]\n"
+ "ble 35f\n"
+ "ld1b { z6.b }, p5/Z, [x14]\n"
+ "udot z8.s, z6.b, z0.b[2]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #1, MUL VL]\n"
+ "subs x11, x11, #0x4\n"
+ "udot z12.s, z6.b, z1.b[2]\n"
+ "udot z16.s, z6.b, z2.b[2]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #2, MUL VL]\n"
+ "udot z9.s, z7.b, z0.b[2]\n"
+ "udot z13.s, z7.b, z1.b[2]\n"
+ "udot z17.s, z7.b, z2.b[2]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #3, MUL VL]\n"
+ "addvl x14, x14, #4\n"
+ "udot z10.s, z6.b, z0.b[2]\n"
+ "udot z14.s, z6.b, z1.b[2]\n"
+ "udot z18.s, z6.b, z2.b[2]\n"
+ "udot z11.s, z7.b, z0.b[2]\n"
+ "udot z15.s, z7.b, z1.b[2]\n"
+ "udot z19.s, z7.b, z2.b[2]\n"
+ "ble 35f\n"
+ "ld1b { z6.b }, p5/Z, [x14]\n"
+ "udot z8.s, z6.b, z0.b[3]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #1, MUL VL]\n"
+ "udot z12.s, z6.b, z1.b[3]\n"
+ "udot z16.s, z6.b, z2.b[3]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #2, MUL VL]\n"
+ "udot z9.s, z7.b, z0.b[3]\n"
+ "udot z13.s, z7.b, z1.b[3]\n"
+ "udot z17.s, z7.b, z2.b[3]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #3, MUL VL]\n"
+ "addvl x14, x14, #4\n"
+ "udot z10.s, z6.b, z0.b[3]\n"
+ "udot z14.s, z6.b, z1.b[3]\n"
+ "udot z18.s, z6.b, z2.b[3]\n"
+ "udot z11.s, z7.b, z0.b[3]\n"
+ "udot z15.s, z7.b, z1.b[3]\n"
+ "udot z19.s, z7.b, z2.b[3]\n"
+ "35:" // Height 3: Multiply loop: multiply skip
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x12, x12, #0x1\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "cmp x12, x19\n"
+ "bne 30b\n"
+ "prfm pstl1keep, [x13, #0x0]\n"
+ "prfm pstl1keep, [x9, #0x0]\n"
+ "prfm pstl1keep, [x27, #0x0]\n"
+ "st1w { z8.s }, p4, [x13]\n"
+ "st1w { z9.s }, p3, [x13, #1, MUL VL]\n"
+ "st1w { z10.s }, p2, [x13, #2, MUL VL]\n"
+ "st1w { z11.s }, p1, [x13, #3, MUL VL]\n"
+ "addvl x13, x13, #4\n"
+ "st1w { z12.s }, p4, [x9]\n"
+ "st1w { z13.s }, p3, [x9, #1, MUL VL]\n"
+ "st1w { z14.s }, p2, [x9, #2, MUL VL]\n"
+ "st1w { z15.s }, p1, [x9, #3, MUL VL]\n"
+ "addvl x9, x9, #4\n"
+ "st1w { z16.s }, p4, [x27]\n"
+ "st1w { z17.s }, p3, [x27, #1, MUL VL]\n"
+ "st1w { z18.s }, p2, [x27, #2, MUL VL]\n"
+ "st1w { z19.s }, p1, [x27, #3, MUL VL]\n"
+ "addvl x27, x27, #4\n"
+ "36:" // Height 3: Writeback done
+ "mov x19, #0x0\n"
+ "incw x19, ALL, MUL #4\n"
+ "subs x15, x15, x19\n"
+ "bgt 27b\n"
+ "b 74f\n"
+ "37:" // Height 4
+ "ldr x15, [%x[args_ptr], %[offsetof_N]]\n"
+ "ldr x14, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 38f\n"
+ "ldr x13, [%x[output_ptr], #0x0]\n"
+ "add x13, x13, x19, LSL #2\n"
+ "ldr x9, [%x[output_ptr], #0x8]\n"
+ "ldr x27, [%x[output_ptr], #0x10]\n"
+ "add x9, x9, x19, LSL #2\n"
+ "ldr x25, [%x[output_ptr], #0x18]\n"
+ "add x27, x27, x19, LSL #2\n"
+ "add x25, x25, x19, LSL #2\n"
+ "b 39f\n"
+ "38:" // Height 4: setup direct output
+ "mov x13, %x[output_ptr]\n"
+ "add x9, x13, x19, LSL #2\n"
+ "add x27, x9, x19, LSL #2\n"
+ "add x25, x27, x19, LSL #2\n"
+ "39:" // Height 4: Column loop
+ "mov x19, #0x0\n"
+ "whilelt p4.s, x19, x15\n"
+ "incw x19\n"
+ "whilelt p3.s, x19, x15\n"
+ "incw x19\n"
+ "whilelt p2.s, x19, x15\n"
+ "incw x19\n"
+ "whilelt p1.s, x19, x15\n"
+ "tbz %x[flags], #0, 40f\n"
+ "ld1w { z8.s }, p4/Z, [x13]\n"
+ "ld1w { z9.s }, p3/Z, [x13, #1, MUL VL]\n"
+ "ld1w { z10.s }, p2/Z, [x13, #2, MUL VL]\n"
+ "ld1w { z11.s }, p1/Z, [x13, #3, MUL VL]\n"
+ "ld1w { z12.s }, p4/Z, [x9]\n"
+ "ld1w { z13.s }, p3/Z, [x9, #1, MUL VL]\n"
+ "ld1w { z14.s }, p2/Z, [x9, #2, MUL VL]\n"
+ "ld1w { z15.s }, p1/Z, [x9, #3, MUL VL]\n"
+ "ld1w { z16.s }, p4/Z, [x27]\n"
+ "ld1w { z17.s }, p3/Z, [x27, #1, MUL VL]\n"
+ "ld1w { z18.s }, p2/Z, [x27, #2, MUL VL]\n"
+ "ld1w { z19.s }, p1/Z, [x27, #3, MUL VL]\n"
+ "ld1w { z20.s }, p4/Z, [x25]\n"
+ "ld1w { z21.s }, p3/Z, [x25, #1, MUL VL]\n"
+ "ld1w { z22.s }, p2/Z, [x25, #2, MUL VL]\n"
+ "ld1w { z23.s }, p1/Z, [x25, #3, MUL VL]\n"
+ "b 41f\n"
+ "40:" // Height 4: no accumulate
+ "mov z8.s, #0x0\n"
+ "mov z9.s, #0x0\n"
+ "mov z10.s, #0x0\n"
+ "mov z11.s, #0x0\n"
+ "mov z12.s, #0x0\n"
+ "mov z13.s, #0x0\n"
+ "mov z14.s, #0x0\n"
+ "mov z15.s, #0x0\n"
+ "mov z16.s, #0x0\n"
+ "mov z17.s, #0x0\n"
+ "mov z18.s, #0x0\n"
+ "mov z19.s, #0x0\n"
+ "mov z20.s, #0x0\n"
+ "mov z21.s, #0x0\n"
+ "mov z22.s, #0x0\n"
+ "mov z23.s, #0x0\n"
+ "41:" // Height 4: setup done
+ "mov x12, #0x0\n"
+ "42:" // Height 4: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w11, [x20, x12, LSL #0x2]\n"
+ "tbz %x[flags], #3, 43f\n"
+ "ldr x20, [%x[input_ptr], x12, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x10, [x20, #0x0]\n"
+ "ldr x28, [x20, #0x8]\n"
+ "ldr x26, [x20, #0x10]\n"
+ "ldr x24, [x20, #0x18]\n"
+ "cbnz x12, 44f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x10, x10, x19\n"
+ "add x28, x28, x19\n"
+ "add x26, x26, x19\n"
+ "add x24, x24, x19\n"
+ "b 44f\n"
+ "43:" // Height 4: setup direct input
+ "mov x10, %x[input_ptr]\n"
+ "add x28, x10, x19\n"
+ "add x26, x28, x19\n"
+ "add x24, x26, x19\n"
+ "44:" // Height 4: input setup done
+ "cmp x11, #0x10\n"
+ "ble 46f\n"
+ "45:" // Height 4: Multiply loop: Main loop head
+ "ld1b { z6.b }, p5/Z, [x14]\n"
+ "whilelt p0.b, XZR, x11\n"
+ "ld1b { z7.b }, p5/Z, [x14, #1, MUL VL]\n"
+ "sub x11, x11, #0x10\n"
+ "ld1rqb { z0.b }, p0/Z, [x10]\n"
+ "udot z8.s, z6.b, z0.b[0]\n"
+ "ld1rqb { z1.b }, p0/Z, [x28]\n"
+ "add x10, x10, #0x10\n"
+ "udot z9.s, z7.b, z0.b[0]\n"
+ "ld1rqb { z2.b }, p0/Z, [x26]\n"
+ "add x28, x28, #0x10\n"
+ "udot z12.s, z6.b, z1.b[0]\n"
+ "ld1rqb { z3.b }, p0/Z, [x24]\n"
+ "add x26, x26, #0x10\n"
+ "udot z16.s, z6.b, z2.b[0]\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x24, x24, #0x10\n"
+ "udot z13.s, z7.b, z1.b[0]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "cmp x11, #0x10\n"
+ "udot z20.s, z6.b, z3.b[0]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #2, MUL VL]\n"
+ "udot z17.s, z7.b, z2.b[0]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "udot z21.s, z7.b, z3.b[0]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #3, MUL VL]\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "udot z10.s, z6.b, z0.b[0]\n"
+ "udot z14.s, z6.b, z1.b[0]\n"
+ "udot z18.s, z6.b, z2.b[0]\n"
+ "udot z22.s, z6.b, z3.b[0]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #4, MUL VL]\n"
+ "udot z11.s, z7.b, z0.b[0]\n"
+ "udot z15.s, z7.b, z1.b[0]\n"
+ "udot z19.s, z7.b, z2.b[0]\n"
+ "udot z23.s, z7.b, z3.b[0]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #5, MUL VL]\n"
+ "udot z8.s, z6.b, z0.b[1]\n"
+ "udot z12.s, z6.b, z1.b[1]\n"
+ "udot z16.s, z6.b, z2.b[1]\n"
+ "udot z20.s, z6.b, z3.b[1]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #6, MUL VL]\n"
+ "udot z9.s, z7.b, z0.b[1]\n"
+ "udot z13.s, z7.b, z1.b[1]\n"
+ "udot z17.s, z7.b, z2.b[1]\n"
+ "udot z21.s, z7.b, z3.b[1]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #7, MUL VL]\n"
+ "addvl x14, x14, #16\n"
+ "udot z10.s, z6.b, z0.b[1]\n"
+ "udot z14.s, z6.b, z1.b[1]\n"
+ "udot z18.s, z6.b, z2.b[1]\n"
+ "udot z22.s, z6.b, z3.b[1]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #-8, MUL VL]\n"
+ "udot z11.s, z7.b, z0.b[1]\n"
+ "udot z15.s, z7.b, z1.b[1]\n"
+ "udot z19.s, z7.b, z2.b[1]\n"
+ "udot z23.s, z7.b, z3.b[1]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #-7, MUL VL]\n"
+ "udot z8.s, z6.b, z0.b[2]\n"
+ "udot z12.s, z6.b, z1.b[2]\n"
+ "udot z16.s, z6.b, z2.b[2]\n"
+ "udot z20.s, z6.b, z3.b[2]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #-6, MUL VL]\n"
+ "udot z9.s, z7.b, z0.b[2]\n"
+ "udot z13.s, z7.b, z1.b[2]\n"
+ "udot z17.s, z7.b, z2.b[2]\n"
+ "udot z21.s, z7.b, z3.b[2]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #-5, MUL VL]\n"
+ "udot z10.s, z6.b, z0.b[2]\n"
+ "udot z14.s, z6.b, z1.b[2]\n"
+ "udot z18.s, z6.b, z2.b[2]\n"
+ "udot z22.s, z6.b, z3.b[2]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #-4, MUL VL]\n"
+ "udot z11.s, z7.b, z0.b[2]\n"
+ "udot z15.s, z7.b, z1.b[2]\n"
+ "udot z19.s, z7.b, z2.b[2]\n"
+ "udot z23.s, z7.b, z3.b[2]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #-3, MUL VL]\n"
+ "udot z8.s, z6.b, z0.b[3]\n"
+ "udot z12.s, z6.b, z1.b[3]\n"
+ "udot z16.s, z6.b, z2.b[3]\n"
+ "udot z20.s, z6.b, z3.b[3]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #-2, MUL VL]\n"
+ "udot z9.s, z7.b, z0.b[3]\n"
+ "udot z13.s, z7.b, z1.b[3]\n"
+ "udot z17.s, z7.b, z2.b[3]\n"
+ "udot z21.s, z7.b, z3.b[3]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #-1, MUL VL]\n"
+ "udot z10.s, z6.b, z0.b[3]\n"
+ "udot z14.s, z6.b, z1.b[3]\n"
+ "udot z18.s, z6.b, z2.b[3]\n"
+ "udot z22.s, z6.b, z3.b[3]\n"
+ "udot z11.s, z7.b, z0.b[3]\n"
+ "udot z15.s, z7.b, z1.b[3]\n"
+ "udot z19.s, z7.b, z2.b[3]\n"
+ "udot z23.s, z7.b, z3.b[3]\n"
+ "bgt 45b\n"
+ "46:" // Height 4: Multiply loop: Single iteration only
+ "ld1b { z6.b }, p5/Z, [x14]\n"
+ "whilelt p0.b, XZR, x11\n"
+ "ld1b { z7.b }, p5/Z, [x14, #1, MUL VL]\n"
+ "subs x11, x11, #0x4\n"
+ "ld1rqb { z0.b }, p0/Z, [x10]\n"
+ "udot z8.s, z6.b, z0.b[0]\n"
+ "ld1rqb { z1.b }, p0/Z, [x28]\n"
+ "add x10, x10, #0x10\n"
+ "udot z9.s, z7.b, z0.b[0]\n"
+ "ld1rqb { z2.b }, p0/Z, [x26]\n"
+ "add x28, x28, #0x10\n"
+ "udot z12.s, z6.b, z1.b[0]\n"
+ "ld1rqb { z3.b }, p0/Z, [x24]\n"
+ "add x26, x26, #0x10\n"
+ "udot z16.s, z6.b, z2.b[0]\n"
+ "add x24, x24, #0x10\n"
+ "udot z13.s, z7.b, z1.b[0]\n"
+ "udot z17.s, z7.b, z2.b[0]\n"
+ "udot z20.s, z6.b, z3.b[0]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #2, MUL VL]\n"
+ "udot z21.s, z7.b, z3.b[0]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #3, MUL VL]\n"
+ "addvl x14, x14, #4\n"
+ "udot z10.s, z6.b, z0.b[0]\n"
+ "udot z14.s, z6.b, z1.b[0]\n"
+ "udot z18.s, z6.b, z2.b[0]\n"
+ "udot z22.s, z6.b, z3.b[0]\n"
+ "udot z11.s, z7.b, z0.b[0]\n"
+ "udot z15.s, z7.b, z1.b[0]\n"
+ "udot z19.s, z7.b, z2.b[0]\n"
+ "udot z23.s, z7.b, z3.b[0]\n"
+ "ble 47f\n"
+ "ld1b { z6.b }, p5/Z, [x14]\n"
+ "udot z8.s, z6.b, z0.b[1]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #1, MUL VL]\n"
+ "subs x11, x11, #0x4\n"
+ "udot z12.s, z6.b, z1.b[1]\n"
+ "udot z16.s, z6.b, z2.b[1]\n"
+ "udot z20.s, z6.b, z3.b[1]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #2, MUL VL]\n"
+ "udot z9.s, z7.b, z0.b[1]\n"
+ "udot z13.s, z7.b, z1.b[1]\n"
+ "udot z17.s, z7.b, z2.b[1]\n"
+ "udot z21.s, z7.b, z3.b[1]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #3, MUL VL]\n"
+ "addvl x14, x14, #4\n"
+ "udot z10.s, z6.b, z0.b[1]\n"
+ "udot z14.s, z6.b, z1.b[1]\n"
+ "udot z18.s, z6.b, z2.b[1]\n"
+ "udot z22.s, z6.b, z3.b[1]\n"
+ "udot z11.s, z7.b, z0.b[1]\n"
+ "udot z15.s, z7.b, z1.b[1]\n"
+ "udot z19.s, z7.b, z2.b[1]\n"
+ "udot z23.s, z7.b, z3.b[1]\n"
+ "ble 47f\n"
+ "ld1b { z6.b }, p5/Z, [x14]\n"
+ "udot z8.s, z6.b, z0.b[2]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #1, MUL VL]\n"
+ "subs x11, x11, #0x4\n"
+ "udot z12.s, z6.b, z1.b[2]\n"
+ "udot z16.s, z6.b, z2.b[2]\n"
+ "udot z20.s, z6.b, z3.b[2]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #2, MUL VL]\n"
+ "udot z9.s, z7.b, z0.b[2]\n"
+ "udot z13.s, z7.b, z1.b[2]\n"
+ "udot z17.s, z7.b, z2.b[2]\n"
+ "udot z21.s, z7.b, z3.b[2]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #3, MUL VL]\n"
+ "addvl x14, x14, #4\n"
+ "udot z10.s, z6.b, z0.b[2]\n"
+ "udot z14.s, z6.b, z1.b[2]\n"
+ "udot z18.s, z6.b, z2.b[2]\n"
+ "udot z22.s, z6.b, z3.b[2]\n"
+ "udot z11.s, z7.b, z0.b[2]\n"
+ "udot z15.s, z7.b, z1.b[2]\n"
+ "udot z19.s, z7.b, z2.b[2]\n"
+ "udot z23.s, z7.b, z3.b[2]\n"
+ "ble 47f\n"
+ "ld1b { z6.b }, p5/Z, [x14]\n"
+ "udot z8.s, z6.b, z0.b[3]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #1, MUL VL]\n"
+ "udot z12.s, z6.b, z1.b[3]\n"
+ "udot z16.s, z6.b, z2.b[3]\n"
+ "udot z20.s, z6.b, z3.b[3]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #2, MUL VL]\n"
+ "udot z9.s, z7.b, z0.b[3]\n"
+ "udot z13.s, z7.b, z1.b[3]\n"
+ "udot z17.s, z7.b, z2.b[3]\n"
+ "udot z21.s, z7.b, z3.b[3]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #3, MUL VL]\n"
+ "addvl x14, x14, #4\n"
+ "udot z10.s, z6.b, z0.b[3]\n"
+ "udot z14.s, z6.b, z1.b[3]\n"
+ "udot z18.s, z6.b, z2.b[3]\n"
+ "udot z22.s, z6.b, z3.b[3]\n"
+ "udot z11.s, z7.b, z0.b[3]\n"
+ "udot z15.s, z7.b, z1.b[3]\n"
+ "udot z19.s, z7.b, z2.b[3]\n"
+ "udot z23.s, z7.b, z3.b[3]\n"
+ "47:" // Height 4: Multiply loop: multiply skip
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x12, x12, #0x1\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "cmp x12, x19\n"
+ "bne 42b\n"
+ "prfm pstl1keep, [x13, #0x0]\n"
+ "prfm pstl1keep, [x9, #0x0]\n"
+ "prfm pstl1keep, [x27, #0x0]\n"
+ "prfm pstl1keep, [x25, #0x0]\n"
+ "st1w { z8.s }, p4, [x13]\n"
+ "st1w { z9.s }, p3, [x13, #1, MUL VL]\n"
+ "st1w { z10.s }, p2, [x13, #2, MUL VL]\n"
+ "st1w { z11.s }, p1, [x13, #3, MUL VL]\n"
+ "addvl x13, x13, #4\n"
+ "st1w { z12.s }, p4, [x9]\n"
+ "st1w { z13.s }, p3, [x9, #1, MUL VL]\n"
+ "st1w { z14.s }, p2, [x9, #2, MUL VL]\n"
+ "st1w { z15.s }, p1, [x9, #3, MUL VL]\n"
+ "addvl x9, x9, #4\n"
+ "st1w { z16.s }, p4, [x27]\n"
+ "st1w { z17.s }, p3, [x27, #1, MUL VL]\n"
+ "st1w { z18.s }, p2, [x27, #2, MUL VL]\n"
+ "st1w { z19.s }, p1, [x27, #3, MUL VL]\n"
+ "addvl x27, x27, #4\n"
+ "st1w { z20.s }, p4, [x25]\n"
+ "st1w { z21.s }, p3, [x25, #1, MUL VL]\n"
+ "st1w { z22.s }, p2, [x25, #2, MUL VL]\n"
+ "st1w { z23.s }, p1, [x25, #3, MUL VL]\n"
+ "addvl x25, x25, #4\n"
+ "48:" // Height 4: Writeback done
+ "mov x19, #0x0\n"
+ "incw x19, ALL, MUL #4\n"
+ "subs x15, x15, x19\n"
+ "bgt 39b\n"
+ "b 74f\n"
+ "49:" // Height 5
+ "ldr x15, [%x[args_ptr], %[offsetof_N]]\n"
+ "ldr x14, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 50f\n"
+ "ldr x13, [%x[output_ptr], #0x0]\n"
+ "add x13, x13, x19, LSL #2\n"
+ "ldr x9, [%x[output_ptr], #0x8]\n"
+ "ldr x27, [%x[output_ptr], #0x10]\n"
+ "add x9, x9, x19, LSL #2\n"
+ "ldr x25, [%x[output_ptr], #0x18]\n"
+ "ldr x23, [%x[output_ptr], #0x20]\n"
+ "add x27, x27, x19, LSL #2\n"
+ "add x25, x25, x19, LSL #2\n"
+ "add x23, x23, x19, LSL #2\n"
+ "b 51f\n"
+ "50:" // Height 5: setup direct output
+ "mov x13, %x[output_ptr]\n"
+ "add x9, x13, x19, LSL #2\n"
+ "add x27, x9, x19, LSL #2\n"
+ "add x25, x27, x19, LSL #2\n"
+ "add x23, x25, x19, LSL #2\n"
+ "51:" // Height 5: Column loop
+ "mov x19, #0x0\n"
+ "whilelt p4.s, x19, x15\n"
+ "incw x19\n"
+ "whilelt p3.s, x19, x15\n"
+ "incw x19\n"
+ "whilelt p2.s, x19, x15\n"
+ "incw x19\n"
+ "whilelt p1.s, x19, x15\n"
+ "tbz %x[flags], #0, 52f\n"
+ "ld1w { z8.s }, p4/Z, [x13]\n"
+ "ld1w { z9.s }, p3/Z, [x13, #1, MUL VL]\n"
+ "ld1w { z10.s }, p2/Z, [x13, #2, MUL VL]\n"
+ "ld1w { z11.s }, p1/Z, [x13, #3, MUL VL]\n"
+ "ld1w { z12.s }, p4/Z, [x9]\n"
+ "ld1w { z13.s }, p3/Z, [x9, #1, MUL VL]\n"
+ "ld1w { z14.s }, p2/Z, [x9, #2, MUL VL]\n"
+ "ld1w { z15.s }, p1/Z, [x9, #3, MUL VL]\n"
+ "ld1w { z16.s }, p4/Z, [x27]\n"
+ "ld1w { z17.s }, p3/Z, [x27, #1, MUL VL]\n"
+ "ld1w { z18.s }, p2/Z, [x27, #2, MUL VL]\n"
+ "ld1w { z19.s }, p1/Z, [x27, #3, MUL VL]\n"
+ "ld1w { z20.s }, p4/Z, [x25]\n"
+ "ld1w { z21.s }, p3/Z, [x25, #1, MUL VL]\n"
+ "ld1w { z22.s }, p2/Z, [x25, #2, MUL VL]\n"
+ "ld1w { z23.s }, p1/Z, [x25, #3, MUL VL]\n"
+ "ld1w { z24.s }, p4/Z, [x23]\n"
+ "ld1w { z25.s }, p3/Z, [x23, #1, MUL VL]\n"
+ "ld1w { z26.s }, p2/Z, [x23, #2, MUL VL]\n"
+ "ld1w { z27.s }, p1/Z, [x23, #3, MUL VL]\n"
+ "b 53f\n"
+ "52:" // Height 5: no accumulate
+ "mov z8.s, #0x0\n"
+ "mov z9.s, #0x0\n"
+ "mov z10.s, #0x0\n"
+ "mov z11.s, #0x0\n"
+ "mov z12.s, #0x0\n"
+ "mov z13.s, #0x0\n"
+ "mov z14.s, #0x0\n"
+ "mov z15.s, #0x0\n"
+ "mov z16.s, #0x0\n"
+ "mov z17.s, #0x0\n"
+ "mov z18.s, #0x0\n"
+ "mov z19.s, #0x0\n"
+ "mov z20.s, #0x0\n"
+ "mov z21.s, #0x0\n"
+ "mov z22.s, #0x0\n"
+ "mov z23.s, #0x0\n"
+ "mov z24.s, #0x0\n"
+ "mov z25.s, #0x0\n"
+ "mov z26.s, #0x0\n"
+ "mov z27.s, #0x0\n"
+ "53:" // Height 5: setup done
+ "mov x12, #0x0\n"
+ "54:" // Height 5: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w11, [x20, x12, LSL #0x2]\n"
+ "tbz %x[flags], #3, 55f\n"
+ "ldr x20, [%x[input_ptr], x12, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x10, [x20, #0x0]\n"
+ "ldr x28, [x20, #0x8]\n"
+ "ldr x26, [x20, #0x10]\n"
+ "ldr x24, [x20, #0x18]\n"
+ "ldr x22, [x20, #0x20]\n"
+ "cbnz x12, 56f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x10, x10, x19\n"
+ "add x28, x28, x19\n"
+ "add x26, x26, x19\n"
+ "add x24, x24, x19\n"
+ "add x22, x22, x19\n"
+ "b 56f\n"
+ "55:" // Height 5: setup direct input
+ "mov x10, %x[input_ptr]\n"
+ "add x28, x10, x19\n"
+ "add x26, x28, x19\n"
+ "add x24, x26, x19\n"
+ "add x22, x24, x19\n"
+ "56:" // Height 5: input setup done
+ "cmp x11, #0x10\n"
+ "ble 58f\n"
+ "57:" // Height 5: Multiply loop: Main loop head
+ "ld1b { z6.b }, p5/Z, [x14]\n"
+ "whilelt p0.b, XZR, x11\n"
+ "ld1b { z7.b }, p5/Z, [x14, #1, MUL VL]\n"
+ "sub x11, x11, #0x10\n"
+ "ld1rqb { z0.b }, p0/Z, [x10]\n"
+ "udot z8.s, z6.b, z0.b[0]\n"
+ "ld1rqb { z1.b }, p0/Z, [x28]\n"
+ "add x10, x10, #0x10\n"
+ "udot z9.s, z7.b, z0.b[0]\n"
+ "ld1rqb { z2.b }, p0/Z, [x26]\n"
+ "add x28, x28, #0x10\n"
+ "udot z12.s, z6.b, z1.b[0]\n"
+ "ld1rqb { z3.b }, p0/Z, [x24]\n"
+ "add x26, x26, #0x10\n"
+ "udot z16.s, z6.b, z2.b[0]\n"
+ "ld1rqb { z4.b }, p0/Z, [x22]\n"
+ "add x24, x24, #0x10\n"
+ "udot z13.s, z7.b, z1.b[0]\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x22, x22, #0x10\n"
+ "udot z20.s, z6.b, z3.b[0]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "cmp x11, #0x10\n"
+ "udot z24.s, z6.b, z4.b[0]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #2, MUL VL]\n"
+ "udot z17.s, z7.b, z2.b[0]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "udot z21.s, z7.b, z3.b[0]\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "udot z25.s, z7.b, z4.b[0]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #3, MUL VL]\n"
+ "udot z10.s, z6.b, z0.b[0]\n"
+ "prfm pldl1keep, [x22, #0x80]\n"
+ "udot z14.s, z6.b, z1.b[0]\n"
+ "udot z18.s, z6.b, z2.b[0]\n"
+ "udot z22.s, z6.b, z3.b[0]\n"
+ "udot z26.s, z6.b, z4.b[0]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #4, MUL VL]\n"
+ "udot z11.s, z7.b, z0.b[0]\n"
+ "udot z15.s, z7.b, z1.b[0]\n"
+ "udot z19.s, z7.b, z2.b[0]\n"
+ "udot z23.s, z7.b, z3.b[0]\n"
+ "udot z27.s, z7.b, z4.b[0]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #5, MUL VL]\n"
+ "udot z8.s, z6.b, z0.b[1]\n"
+ "udot z12.s, z6.b, z1.b[1]\n"
+ "udot z16.s, z6.b, z2.b[1]\n"
+ "udot z20.s, z6.b, z3.b[1]\n"
+ "udot z24.s, z6.b, z4.b[1]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #6, MUL VL]\n"
+ "udot z9.s, z7.b, z0.b[1]\n"
+ "udot z13.s, z7.b, z1.b[1]\n"
+ "udot z17.s, z7.b, z2.b[1]\n"
+ "udot z21.s, z7.b, z3.b[1]\n"
+ "udot z25.s, z7.b, z4.b[1]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #7, MUL VL]\n"
+ "addvl x14, x14, #16\n"
+ "udot z10.s, z6.b, z0.b[1]\n"
+ "udot z14.s, z6.b, z1.b[1]\n"
+ "udot z18.s, z6.b, z2.b[1]\n"
+ "udot z22.s, z6.b, z3.b[1]\n"
+ "udot z26.s, z6.b, z4.b[1]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #-8, MUL VL]\n"
+ "udot z11.s, z7.b, z0.b[1]\n"
+ "udot z15.s, z7.b, z1.b[1]\n"
+ "udot z19.s, z7.b, z2.b[1]\n"
+ "udot z23.s, z7.b, z3.b[1]\n"
+ "udot z27.s, z7.b, z4.b[1]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #-7, MUL VL]\n"
+ "udot z8.s, z6.b, z0.b[2]\n"
+ "udot z12.s, z6.b, z1.b[2]\n"
+ "udot z16.s, z6.b, z2.b[2]\n"
+ "udot z20.s, z6.b, z3.b[2]\n"
+ "udot z24.s, z6.b, z4.b[2]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #-6, MUL VL]\n"
+ "udot z9.s, z7.b, z0.b[2]\n"
+ "udot z13.s, z7.b, z1.b[2]\n"
+ "udot z17.s, z7.b, z2.b[2]\n"
+ "udot z21.s, z7.b, z3.b[2]\n"
+ "udot z25.s, z7.b, z4.b[2]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #-5, MUL VL]\n"
+ "udot z10.s, z6.b, z0.b[2]\n"
+ "udot z14.s, z6.b, z1.b[2]\n"
+ "udot z18.s, z6.b, z2.b[2]\n"
+ "udot z22.s, z6.b, z3.b[2]\n"
+ "udot z26.s, z6.b, z4.b[2]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #-4, MUL VL]\n"
+ "udot z11.s, z7.b, z0.b[2]\n"
+ "udot z15.s, z7.b, z1.b[2]\n"
+ "udot z19.s, z7.b, z2.b[2]\n"
+ "udot z23.s, z7.b, z3.b[2]\n"
+ "udot z27.s, z7.b, z4.b[2]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #-3, MUL VL]\n"
+ "udot z8.s, z6.b, z0.b[3]\n"
+ "udot z12.s, z6.b, z1.b[3]\n"
+ "udot z16.s, z6.b, z2.b[3]\n"
+ "udot z20.s, z6.b, z3.b[3]\n"
+ "udot z24.s, z6.b, z4.b[3]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #-2, MUL VL]\n"
+ "udot z9.s, z7.b, z0.b[3]\n"
+ "udot z13.s, z7.b, z1.b[3]\n"
+ "udot z17.s, z7.b, z2.b[3]\n"
+ "udot z21.s, z7.b, z3.b[3]\n"
+ "udot z25.s, z7.b, z4.b[3]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #-1, MUL VL]\n"
+ "udot z10.s, z6.b, z0.b[3]\n"
+ "udot z14.s, z6.b, z1.b[3]\n"
+ "udot z18.s, z6.b, z2.b[3]\n"
+ "udot z22.s, z6.b, z3.b[3]\n"
+ "udot z26.s, z6.b, z4.b[3]\n"
+ "udot z11.s, z7.b, z0.b[3]\n"
+ "udot z15.s, z7.b, z1.b[3]\n"
+ "udot z19.s, z7.b, z2.b[3]\n"
+ "udot z23.s, z7.b, z3.b[3]\n"
+ "udot z27.s, z7.b, z4.b[3]\n"
+ "bgt 57b\n"
+ "58:" // Height 5: Multiply loop: Single iteration only
+ "ld1b { z6.b }, p5/Z, [x14]\n"
+ "whilelt p0.b, XZR, x11\n"
+ "ld1b { z7.b }, p5/Z, [x14, #1, MUL VL]\n"
+ "subs x11, x11, #0x4\n"
+ "ld1rqb { z0.b }, p0/Z, [x10]\n"
+ "udot z8.s, z6.b, z0.b[0]\n"
+ "ld1rqb { z1.b }, p0/Z, [x28]\n"
+ "add x10, x10, #0x10\n"
+ "udot z9.s, z7.b, z0.b[0]\n"
+ "ld1rqb { z2.b }, p0/Z, [x26]\n"
+ "add x28, x28, #0x10\n"
+ "udot z12.s, z6.b, z1.b[0]\n"
+ "ld1rqb { z3.b }, p0/Z, [x24]\n"
+ "add x26, x26, #0x10\n"
+ "udot z16.s, z6.b, z2.b[0]\n"
+ "ld1rqb { z4.b }, p0/Z, [x22]\n"
+ "add x24, x24, #0x10\n"
+ "udot z13.s, z7.b, z1.b[0]\n"
+ "add x22, x22, #0x10\n"
+ "udot z17.s, z7.b, z2.b[0]\n"
+ "udot z20.s, z6.b, z3.b[0]\n"
+ "udot z24.s, z6.b, z4.b[0]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #2, MUL VL]\n"
+ "udot z21.s, z7.b, z3.b[0]\n"
+ "udot z25.s, z7.b, z4.b[0]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #3, MUL VL]\n"
+ "addvl x14, x14, #4\n"
+ "udot z10.s, z6.b, z0.b[0]\n"
+ "udot z14.s, z6.b, z1.b[0]\n"
+ "udot z18.s, z6.b, z2.b[0]\n"
+ "udot z22.s, z6.b, z3.b[0]\n"
+ "udot z26.s, z6.b, z4.b[0]\n"
+ "udot z11.s, z7.b, z0.b[0]\n"
+ "udot z15.s, z7.b, z1.b[0]\n"
+ "udot z19.s, z7.b, z2.b[0]\n"
+ "udot z23.s, z7.b, z3.b[0]\n"
+ "udot z27.s, z7.b, z4.b[0]\n"
+ "ble 59f\n"
+ "ld1b { z6.b }, p5/Z, [x14]\n"
+ "udot z8.s, z6.b, z0.b[1]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #1, MUL VL]\n"
+ "subs x11, x11, #0x4\n"
+ "udot z12.s, z6.b, z1.b[1]\n"
+ "udot z16.s, z6.b, z2.b[1]\n"
+ "udot z20.s, z6.b, z3.b[1]\n"
+ "udot z24.s, z6.b, z4.b[1]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #2, MUL VL]\n"
+ "udot z9.s, z7.b, z0.b[1]\n"
+ "udot z13.s, z7.b, z1.b[1]\n"
+ "udot z17.s, z7.b, z2.b[1]\n"
+ "udot z21.s, z7.b, z3.b[1]\n"
+ "udot z25.s, z7.b, z4.b[1]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #3, MUL VL]\n"
+ "addvl x14, x14, #4\n"
+ "udot z10.s, z6.b, z0.b[1]\n"
+ "udot z14.s, z6.b, z1.b[1]\n"
+ "udot z18.s, z6.b, z2.b[1]\n"
+ "udot z22.s, z6.b, z3.b[1]\n"
+ "udot z26.s, z6.b, z4.b[1]\n"
+ "udot z11.s, z7.b, z0.b[1]\n"
+ "udot z15.s, z7.b, z1.b[1]\n"
+ "udot z19.s, z7.b, z2.b[1]\n"
+ "udot z23.s, z7.b, z3.b[1]\n"
+ "udot z27.s, z7.b, z4.b[1]\n"
+ "ble 59f\n"
+ "ld1b { z6.b }, p5/Z, [x14]\n"
+ "udot z8.s, z6.b, z0.b[2]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #1, MUL VL]\n"
+ "subs x11, x11, #0x4\n"
+ "udot z12.s, z6.b, z1.b[2]\n"
+ "udot z16.s, z6.b, z2.b[2]\n"
+ "udot z20.s, z6.b, z3.b[2]\n"
+ "udot z24.s, z6.b, z4.b[2]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #2, MUL VL]\n"
+ "udot z9.s, z7.b, z0.b[2]\n"
+ "udot z13.s, z7.b, z1.b[2]\n"
+ "udot z17.s, z7.b, z2.b[2]\n"
+ "udot z21.s, z7.b, z3.b[2]\n"
+ "udot z25.s, z7.b, z4.b[2]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #3, MUL VL]\n"
+ "addvl x14, x14, #4\n"
+ "udot z10.s, z6.b, z0.b[2]\n"
+ "udot z14.s, z6.b, z1.b[2]\n"
+ "udot z18.s, z6.b, z2.b[2]\n"
+ "udot z22.s, z6.b, z3.b[2]\n"
+ "udot z26.s, z6.b, z4.b[2]\n"
+ "udot z11.s, z7.b, z0.b[2]\n"
+ "udot z15.s, z7.b, z1.b[2]\n"
+ "udot z19.s, z7.b, z2.b[2]\n"
+ "udot z23.s, z7.b, z3.b[2]\n"
+ "udot z27.s, z7.b, z4.b[2]\n"
+ "ble 59f\n"
+ "ld1b { z6.b }, p5/Z, [x14]\n"
+ "udot z8.s, z6.b, z0.b[3]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #1, MUL VL]\n"
+ "udot z12.s, z6.b, z1.b[3]\n"
+ "udot z16.s, z6.b, z2.b[3]\n"
+ "udot z20.s, z6.b, z3.b[3]\n"
+ "udot z24.s, z6.b, z4.b[3]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #2, MUL VL]\n"
+ "udot z9.s, z7.b, z0.b[3]\n"
+ "udot z13.s, z7.b, z1.b[3]\n"
+ "udot z17.s, z7.b, z2.b[3]\n"
+ "udot z21.s, z7.b, z3.b[3]\n"
+ "udot z25.s, z7.b, z4.b[3]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #3, MUL VL]\n"
+ "addvl x14, x14, #4\n"
+ "udot z10.s, z6.b, z0.b[3]\n"
+ "udot z14.s, z6.b, z1.b[3]\n"
+ "udot z18.s, z6.b, z2.b[3]\n"
+ "udot z22.s, z6.b, z3.b[3]\n"
+ "udot z26.s, z6.b, z4.b[3]\n"
+ "udot z11.s, z7.b, z0.b[3]\n"
+ "udot z15.s, z7.b, z1.b[3]\n"
+ "udot z19.s, z7.b, z2.b[3]\n"
+ "udot z23.s, z7.b, z3.b[3]\n"
+ "udot z27.s, z7.b, z4.b[3]\n"
+ "59:" // Height 5: Multiply loop: multiply skip
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x12, x12, #0x1\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "prfm pldl1keep, [x22, #0x80]\n"
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "cmp x12, x19\n"
+ "bne 54b\n"
+ "prfm pstl1keep, [x13, #0x0]\n"
+ "prfm pstl1keep, [x9, #0x0]\n"
+ "prfm pstl1keep, [x27, #0x0]\n"
+ "prfm pstl1keep, [x25, #0x0]\n"
+ "prfm pstl1keep, [x23, #0x0]\n"
+ "st1w { z8.s }, p4, [x13]\n"
+ "st1w { z9.s }, p3, [x13, #1, MUL VL]\n"
+ "st1w { z10.s }, p2, [x13, #2, MUL VL]\n"
+ "st1w { z11.s }, p1, [x13, #3, MUL VL]\n"
+ "addvl x13, x13, #4\n"
+ "st1w { z12.s }, p4, [x9]\n"
+ "st1w { z13.s }, p3, [x9, #1, MUL VL]\n"
+ "st1w { z14.s }, p2, [x9, #2, MUL VL]\n"
+ "st1w { z15.s }, p1, [x9, #3, MUL VL]\n"
+ "addvl x9, x9, #4\n"
+ "st1w { z16.s }, p4, [x27]\n"
+ "st1w { z17.s }, p3, [x27, #1, MUL VL]\n"
+ "st1w { z18.s }, p2, [x27, #2, MUL VL]\n"
+ "st1w { z19.s }, p1, [x27, #3, MUL VL]\n"
+ "addvl x27, x27, #4\n"
+ "st1w { z20.s }, p4, [x25]\n"
+ "st1w { z21.s }, p3, [x25, #1, MUL VL]\n"
+ "st1w { z22.s }, p2, [x25, #2, MUL VL]\n"
+ "st1w { z23.s }, p1, [x25, #3, MUL VL]\n"
+ "addvl x25, x25, #4\n"
+ "st1w { z24.s }, p4, [x23]\n"
+ "st1w { z25.s }, p3, [x23, #1, MUL VL]\n"
+ "st1w { z26.s }, p2, [x23, #2, MUL VL]\n"
+ "st1w { z27.s }, p1, [x23, #3, MUL VL]\n"
+ "addvl x23, x23, #4\n"
+ "60:" // Height 5: Writeback done
+ "mov x19, #0x0\n"
+ "incw x19, ALL, MUL #4\n"
+ "subs x15, x15, x19\n"
+ "bgt 51b\n"
+ "b 74f\n"
+ "61:" // Height 6
+ "ldr x15, [%x[args_ptr], %[offsetof_N]]\n"
+ "ldr x14, [%x[args_ptr], %[offsetof_B_ptr]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_output_offset]]\n"
+ "tbz %x[flags], #2, 62f\n"
+ "ldr x13, [%x[output_ptr], #0x0]\n"
+ "add x13, x13, x19, LSL #2\n"
+ "ldr x9, [%x[output_ptr], #0x8]\n"
+ "ldr x27, [%x[output_ptr], #0x10]\n"
+ "add x9, x9, x19, LSL #2\n"
+ "ldr x25, [%x[output_ptr], #0x18]\n"
+ "ldr x23, [%x[output_ptr], #0x20]\n"
+ "add x27, x27, x19, LSL #2\n"
+ "ldr x21, [%x[output_ptr], #0x28]\n"
+ "add %x[output_ptr], %x[output_ptr], #0x30\n"
+ "add x25, x25, x19, LSL #2\n"
+ "add x23, x23, x19, LSL #2\n"
+ "add x21, x21, x19, LSL #2\n"
+ "b 63f\n"
+ "62:" // Height 6: setup direct output
+ "mov x13, %x[output_ptr]\n"
+ "add x9, x13, x19, LSL #2\n"
+ "add x27, x9, x19, LSL #2\n"
+ "add x25, x27, x19, LSL #2\n"
+ "add x23, x25, x19, LSL #2\n"
+ "add x21, x23, x19, LSL #2\n"
+ "add %x[output_ptr], x21, x19, LSL #2\n"
+ "63:" // Height 6: Column loop
+ "mov x19, #0x0\n"
+ "whilelt p4.s, x19, x15\n"
+ "incw x19\n"
+ "whilelt p3.s, x19, x15\n"
+ "incw x19\n"
+ "whilelt p2.s, x19, x15\n"
+ "incw x19\n"
+ "whilelt p1.s, x19, x15\n"
+ "tbz %x[flags], #0, 64f\n"
+ "ld1w { z8.s }, p4/Z, [x13]\n"
+ "ld1w { z9.s }, p3/Z, [x13, #1, MUL VL]\n"
+ "ld1w { z10.s }, p2/Z, [x13, #2, MUL VL]\n"
+ "ld1w { z11.s }, p1/Z, [x13, #3, MUL VL]\n"
+ "ld1w { z12.s }, p4/Z, [x9]\n"
+ "ld1w { z13.s }, p3/Z, [x9, #1, MUL VL]\n"
+ "ld1w { z14.s }, p2/Z, [x9, #2, MUL VL]\n"
+ "ld1w { z15.s }, p1/Z, [x9, #3, MUL VL]\n"
+ "ld1w { z16.s }, p4/Z, [x27]\n"
+ "ld1w { z17.s }, p3/Z, [x27, #1, MUL VL]\n"
+ "ld1w { z18.s }, p2/Z, [x27, #2, MUL VL]\n"
+ "ld1w { z19.s }, p1/Z, [x27, #3, MUL VL]\n"
+ "ld1w { z20.s }, p4/Z, [x25]\n"
+ "ld1w { z21.s }, p3/Z, [x25, #1, MUL VL]\n"
+ "ld1w { z22.s }, p2/Z, [x25, #2, MUL VL]\n"
+ "ld1w { z23.s }, p1/Z, [x25, #3, MUL VL]\n"
+ "ld1w { z24.s }, p4/Z, [x23]\n"
+ "ld1w { z25.s }, p3/Z, [x23, #1, MUL VL]\n"
+ "ld1w { z26.s }, p2/Z, [x23, #2, MUL VL]\n"
+ "ld1w { z27.s }, p1/Z, [x23, #3, MUL VL]\n"
+ "ld1w { z28.s }, p4/Z, [x21]\n"
+ "ld1w { z29.s }, p3/Z, [x21, #1, MUL VL]\n"
+ "ld1w { z30.s }, p2/Z, [x21, #2, MUL VL]\n"
+ "ld1w { z31.s }, p1/Z, [x21, #3, MUL VL]\n"
+ "b 65f\n"
+ "64:" // Height 6: no accumulate
+ "mov z8.s, #0x0\n"
+ "mov z9.s, #0x0\n"
+ "mov z10.s, #0x0\n"
+ "mov z11.s, #0x0\n"
+ "mov z12.s, #0x0\n"
+ "mov z13.s, #0x0\n"
+ "mov z14.s, #0x0\n"
+ "mov z15.s, #0x0\n"
+ "mov z16.s, #0x0\n"
+ "mov z17.s, #0x0\n"
+ "mov z18.s, #0x0\n"
+ "mov z19.s, #0x0\n"
+ "mov z20.s, #0x0\n"
+ "mov z21.s, #0x0\n"
+ "mov z22.s, #0x0\n"
+ "mov z23.s, #0x0\n"
+ "mov z24.s, #0x0\n"
+ "mov z25.s, #0x0\n"
+ "mov z26.s, #0x0\n"
+ "mov z27.s, #0x0\n"
+ "mov z28.s, #0x0\n"
+ "mov z29.s, #0x0\n"
+ "mov z30.s, #0x0\n"
+ "mov z31.s, #0x0\n"
+ "65:" // Height 6: setup done
+ "mov x12, #0x0\n"
+ "66:" // Height 6: String loop
+ "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "ldr w11, [x20, x12, LSL #0x2]\n"
+ "tbz %x[flags], #3, 67f\n"
+ "ldr x20, [%x[input_ptr], x12, LSL #0x3]\n"
+ "add x20, x20, x19, LSL #3\n"
+ "ldr x10, [x20, #0x0]\n"
+ "ldr x28, [x20, #0x8]\n"
+ "ldr x26, [x20, #0x10]\n"
+ "ldr x24, [x20, #0x18]\n"
+ "ldr x22, [x20, #0x20]\n"
+ "ldr x20, [x20, #0x28]\n"
+ "cbnz x12, 68f\n"
+ "ldr x19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x10, x10, x19\n"
+ "add x28, x28, x19\n"
+ "add x26, x26, x19\n"
+ "add x24, x24, x19\n"
+ "add x22, x22, x19\n"
+ "add x20, x20, x19\n"
+ "b 68f\n"
+ "67:" // Height 6: setup direct input
+ "mov x10, %x[input_ptr]\n"
+ "add x28, x10, x19\n"
+ "add x26, x28, x19\n"
+ "add x24, x26, x19\n"
+ "add x22, x24, x19\n"
+ "add x20, x22, x19\n"
+ "68:" // Height 6: input setup done
+ "cmp x11, #0x10\n"
+ "ble 70f\n"
+ "69:" // Height 6: Multiply loop: Main loop head
+ "ld1b { z6.b }, p5/Z, [x14]\n"
+ "whilelt p0.b, XZR, x11\n"
+ "ld1b { z7.b }, p5/Z, [x14, #1, MUL VL]\n"
+ "sub x11, x11, #0x10\n"
+ "ld1rqb { z0.b }, p0/Z, [x10]\n"
+ "udot z8.s, z6.b, z0.b[0]\n"
+ "ld1rqb { z1.b }, p0/Z, [x28]\n"
+ "add x10, x10, #0x10\n"
+ "udot z9.s, z7.b, z0.b[0]\n"
+ "ld1rqb { z2.b }, p0/Z, [x26]\n"
+ "add x28, x28, #0x10\n"
+ "udot z12.s, z6.b, z1.b[0]\n"
+ "ld1rqb { z3.b }, p0/Z, [x24]\n"
+ "add x26, x26, #0x10\n"
+ "udot z16.s, z6.b, z2.b[0]\n"
+ "ld1rqb { z4.b }, p0/Z, [x22]\n"
+ "add x24, x24, #0x10\n"
+ "udot z13.s, z7.b, z1.b[0]\n"
+ "ld1rqb { z5.b }, p0/Z, [x20]\n"
+ "add x22, x22, #0x10\n"
+ "udot z20.s, z6.b, z3.b[0]\n"
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x20, x20, #0x10\n"
+ "udot z24.s, z6.b, z4.b[0]\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "cmp x11, #0x10\n"
+ "udot z28.s, z6.b, z5.b[0]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #2, MUL VL]\n"
+ "udot z17.s, z7.b, z2.b[0]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "udot z21.s, z7.b, z3.b[0]\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "udot z25.s, z7.b, z4.b[0]\n"
+ "prfm pldl1keep, [x22, #0x80]\n"
+ "udot z29.s, z7.b, z5.b[0]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #3, MUL VL]\n"
+ "udot z10.s, z6.b, z0.b[0]\n"
+ "prfm pldl1keep, [x20, #0x80]\n"
+ "udot z14.s, z6.b, z1.b[0]\n"
+ "udot z18.s, z6.b, z2.b[0]\n"
+ "udot z22.s, z6.b, z3.b[0]\n"
+ "udot z26.s, z6.b, z4.b[0]\n"
+ "udot z30.s, z6.b, z5.b[0]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #4, MUL VL]\n"
+ "udot z11.s, z7.b, z0.b[0]\n"
+ "udot z15.s, z7.b, z1.b[0]\n"
+ "udot z19.s, z7.b, z2.b[0]\n"
+ "udot z23.s, z7.b, z3.b[0]\n"
+ "udot z27.s, z7.b, z4.b[0]\n"
+ "udot z31.s, z7.b, z5.b[0]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #5, MUL VL]\n"
+ "udot z8.s, z6.b, z0.b[1]\n"
+ "udot z12.s, z6.b, z1.b[1]\n"
+ "udot z16.s, z6.b, z2.b[1]\n"
+ "udot z20.s, z6.b, z3.b[1]\n"
+ "udot z24.s, z6.b, z4.b[1]\n"
+ "udot z28.s, z6.b, z5.b[1]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #6, MUL VL]\n"
+ "udot z9.s, z7.b, z0.b[1]\n"
+ "udot z13.s, z7.b, z1.b[1]\n"
+ "udot z17.s, z7.b, z2.b[1]\n"
+ "udot z21.s, z7.b, z3.b[1]\n"
+ "udot z25.s, z7.b, z4.b[1]\n"
+ "udot z29.s, z7.b, z5.b[1]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #7, MUL VL]\n"
+ "addvl x14, x14, #16\n"
+ "udot z10.s, z6.b, z0.b[1]\n"
+ "udot z14.s, z6.b, z1.b[1]\n"
+ "udot z18.s, z6.b, z2.b[1]\n"
+ "udot z22.s, z6.b, z3.b[1]\n"
+ "udot z26.s, z6.b, z4.b[1]\n"
+ "udot z30.s, z6.b, z5.b[1]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #-8, MUL VL]\n"
+ "udot z11.s, z7.b, z0.b[1]\n"
+ "udot z15.s, z7.b, z1.b[1]\n"
+ "udot z19.s, z7.b, z2.b[1]\n"
+ "udot z23.s, z7.b, z3.b[1]\n"
+ "udot z27.s, z7.b, z4.b[1]\n"
+ "udot z31.s, z7.b, z5.b[1]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #-7, MUL VL]\n"
+ "udot z8.s, z6.b, z0.b[2]\n"
+ "udot z12.s, z6.b, z1.b[2]\n"
+ "udot z16.s, z6.b, z2.b[2]\n"
+ "udot z20.s, z6.b, z3.b[2]\n"
+ "udot z24.s, z6.b, z4.b[2]\n"
+ "udot z28.s, z6.b, z5.b[2]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #-6, MUL VL]\n"
+ "udot z9.s, z7.b, z0.b[2]\n"
+ "udot z13.s, z7.b, z1.b[2]\n"
+ "udot z17.s, z7.b, z2.b[2]\n"
+ "udot z21.s, z7.b, z3.b[2]\n"
+ "udot z25.s, z7.b, z4.b[2]\n"
+ "udot z29.s, z7.b, z5.b[2]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #-5, MUL VL]\n"
+ "udot z10.s, z6.b, z0.b[2]\n"
+ "udot z14.s, z6.b, z1.b[2]\n"
+ "udot z18.s, z6.b, z2.b[2]\n"
+ "udot z22.s, z6.b, z3.b[2]\n"
+ "udot z26.s, z6.b, z4.b[2]\n"
+ "udot z30.s, z6.b, z5.b[2]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #-4, MUL VL]\n"
+ "udot z11.s, z7.b, z0.b[2]\n"
+ "udot z15.s, z7.b, z1.b[2]\n"
+ "udot z19.s, z7.b, z2.b[2]\n"
+ "udot z23.s, z7.b, z3.b[2]\n"
+ "udot z27.s, z7.b, z4.b[2]\n"
+ "udot z31.s, z7.b, z5.b[2]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #-3, MUL VL]\n"
+ "udot z8.s, z6.b, z0.b[3]\n"
+ "udot z12.s, z6.b, z1.b[3]\n"
+ "udot z16.s, z6.b, z2.b[3]\n"
+ "udot z20.s, z6.b, z3.b[3]\n"
+ "udot z24.s, z6.b, z4.b[3]\n"
+ "udot z28.s, z6.b, z5.b[3]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #-2, MUL VL]\n"
+ "udot z9.s, z7.b, z0.b[3]\n"
+ "udot z13.s, z7.b, z1.b[3]\n"
+ "udot z17.s, z7.b, z2.b[3]\n"
+ "udot z21.s, z7.b, z3.b[3]\n"
+ "udot z25.s, z7.b, z4.b[3]\n"
+ "udot z29.s, z7.b, z5.b[3]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #-1, MUL VL]\n"
+ "udot z10.s, z6.b, z0.b[3]\n"
+ "udot z14.s, z6.b, z1.b[3]\n"
+ "udot z18.s, z6.b, z2.b[3]\n"
+ "udot z22.s, z6.b, z3.b[3]\n"
+ "udot z26.s, z6.b, z4.b[3]\n"
+ "udot z30.s, z6.b, z5.b[3]\n"
+ "udot z11.s, z7.b, z0.b[3]\n"
+ "udot z15.s, z7.b, z1.b[3]\n"
+ "udot z19.s, z7.b, z2.b[3]\n"
+ "udot z23.s, z7.b, z3.b[3]\n"
+ "udot z27.s, z7.b, z4.b[3]\n"
+ "udot z31.s, z7.b, z5.b[3]\n"
+ "bgt 69b\n"
+ "70:" // Height 6: Multiply loop: Single iteration only
+ "ld1b { z6.b }, p5/Z, [x14]\n"
+ "whilelt p0.b, XZR, x11\n"
+ "ld1b { z7.b }, p5/Z, [x14, #1, MUL VL]\n"
+ "subs x11, x11, #0x4\n"
+ "ld1rqb { z0.b }, p0/Z, [x10]\n"
+ "udot z8.s, z6.b, z0.b[0]\n"
+ "ld1rqb { z1.b }, p0/Z, [x28]\n"
+ "add x10, x10, #0x10\n"
+ "udot z9.s, z7.b, z0.b[0]\n"
+ "ld1rqb { z2.b }, p0/Z, [x26]\n"
+ "add x28, x28, #0x10\n"
+ "udot z12.s, z6.b, z1.b[0]\n"
+ "ld1rqb { z3.b }, p0/Z, [x24]\n"
+ "add x26, x26, #0x10\n"
+ "udot z16.s, z6.b, z2.b[0]\n"
+ "ld1rqb { z4.b }, p0/Z, [x22]\n"
+ "add x24, x24, #0x10\n"
+ "udot z13.s, z7.b, z1.b[0]\n"
+ "ld1rqb { z5.b }, p0/Z, [x20]\n"
+ "add x22, x22, #0x10\n"
+ "udot z20.s, z6.b, z3.b[0]\n"
+ "add x20, x20, #0x10\n"
+ "udot z17.s, z7.b, z2.b[0]\n"
+ "udot z24.s, z6.b, z4.b[0]\n"
+ "udot z28.s, z6.b, z5.b[0]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #2, MUL VL]\n"
+ "udot z21.s, z7.b, z3.b[0]\n"
+ "udot z25.s, z7.b, z4.b[0]\n"
+ "udot z29.s, z7.b, z5.b[0]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #3, MUL VL]\n"
+ "addvl x14, x14, #4\n"
+ "udot z10.s, z6.b, z0.b[0]\n"
+ "udot z14.s, z6.b, z1.b[0]\n"
+ "udot z18.s, z6.b, z2.b[0]\n"
+ "udot z22.s, z6.b, z3.b[0]\n"
+ "udot z26.s, z6.b, z4.b[0]\n"
+ "udot z30.s, z6.b, z5.b[0]\n"
+ "udot z11.s, z7.b, z0.b[0]\n"
+ "udot z15.s, z7.b, z1.b[0]\n"
+ "udot z19.s, z7.b, z2.b[0]\n"
+ "udot z23.s, z7.b, z3.b[0]\n"
+ "udot z27.s, z7.b, z4.b[0]\n"
+ "udot z31.s, z7.b, z5.b[0]\n"
+ "ble 71f\n"
+ "ld1b { z6.b }, p5/Z, [x14]\n"
+ "udot z8.s, z6.b, z0.b[1]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #1, MUL VL]\n"
+ "subs x11, x11, #0x4\n"
+ "udot z12.s, z6.b, z1.b[1]\n"
+ "udot z16.s, z6.b, z2.b[1]\n"
+ "udot z20.s, z6.b, z3.b[1]\n"
+ "udot z24.s, z6.b, z4.b[1]\n"
+ "udot z28.s, z6.b, z5.b[1]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #2, MUL VL]\n"
+ "udot z9.s, z7.b, z0.b[1]\n"
+ "udot z13.s, z7.b, z1.b[1]\n"
+ "udot z17.s, z7.b, z2.b[1]\n"
+ "udot z21.s, z7.b, z3.b[1]\n"
+ "udot z25.s, z7.b, z4.b[1]\n"
+ "udot z29.s, z7.b, z5.b[1]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #3, MUL VL]\n"
+ "addvl x14, x14, #4\n"
+ "udot z10.s, z6.b, z0.b[1]\n"
+ "udot z14.s, z6.b, z1.b[1]\n"
+ "udot z18.s, z6.b, z2.b[1]\n"
+ "udot z22.s, z6.b, z3.b[1]\n"
+ "udot z26.s, z6.b, z4.b[1]\n"
+ "udot z30.s, z6.b, z5.b[1]\n"
+ "udot z11.s, z7.b, z0.b[1]\n"
+ "udot z15.s, z7.b, z1.b[1]\n"
+ "udot z19.s, z7.b, z2.b[1]\n"
+ "udot z23.s, z7.b, z3.b[1]\n"
+ "udot z27.s, z7.b, z4.b[1]\n"
+ "udot z31.s, z7.b, z5.b[1]\n"
+ "ble 71f\n"
+ "ld1b { z6.b }, p5/Z, [x14]\n"
+ "udot z8.s, z6.b, z0.b[2]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #1, MUL VL]\n"
+ "subs x11, x11, #0x4\n"
+ "udot z12.s, z6.b, z1.b[2]\n"
+ "udot z16.s, z6.b, z2.b[2]\n"
+ "udot z20.s, z6.b, z3.b[2]\n"
+ "udot z24.s, z6.b, z4.b[2]\n"
+ "udot z28.s, z6.b, z5.b[2]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #2, MUL VL]\n"
+ "udot z9.s, z7.b, z0.b[2]\n"
+ "udot z13.s, z7.b, z1.b[2]\n"
+ "udot z17.s, z7.b, z2.b[2]\n"
+ "udot z21.s, z7.b, z3.b[2]\n"
+ "udot z25.s, z7.b, z4.b[2]\n"
+ "udot z29.s, z7.b, z5.b[2]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #3, MUL VL]\n"
+ "addvl x14, x14, #4\n"
+ "udot z10.s, z6.b, z0.b[2]\n"
+ "udot z14.s, z6.b, z1.b[2]\n"
+ "udot z18.s, z6.b, z2.b[2]\n"
+ "udot z22.s, z6.b, z3.b[2]\n"
+ "udot z26.s, z6.b, z4.b[2]\n"
+ "udot z30.s, z6.b, z5.b[2]\n"
+ "udot z11.s, z7.b, z0.b[2]\n"
+ "udot z15.s, z7.b, z1.b[2]\n"
+ "udot z19.s, z7.b, z2.b[2]\n"
+ "udot z23.s, z7.b, z3.b[2]\n"
+ "udot z27.s, z7.b, z4.b[2]\n"
+ "udot z31.s, z7.b, z5.b[2]\n"
+ "ble 71f\n"
+ "ld1b { z6.b }, p5/Z, [x14]\n"
+ "udot z8.s, z6.b, z0.b[3]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #1, MUL VL]\n"
+ "udot z12.s, z6.b, z1.b[3]\n"
+ "udot z16.s, z6.b, z2.b[3]\n"
+ "udot z20.s, z6.b, z3.b[3]\n"
+ "udot z24.s, z6.b, z4.b[3]\n"
+ "udot z28.s, z6.b, z5.b[3]\n"
+ "ld1b { z6.b }, p5/Z, [x14, #2, MUL VL]\n"
+ "udot z9.s, z7.b, z0.b[3]\n"
+ "udot z13.s, z7.b, z1.b[3]\n"
+ "udot z17.s, z7.b, z2.b[3]\n"
+ "udot z21.s, z7.b, z3.b[3]\n"
+ "udot z25.s, z7.b, z4.b[3]\n"
+ "udot z29.s, z7.b, z5.b[3]\n"
+ "ld1b { z7.b }, p5/Z, [x14, #3, MUL VL]\n"
+ "addvl x14, x14, #4\n"
+ "udot z10.s, z6.b, z0.b[3]\n"
+ "udot z14.s, z6.b, z1.b[3]\n"
+ "udot z18.s, z6.b, z2.b[3]\n"
+ "udot z22.s, z6.b, z3.b[3]\n"
+ "udot z26.s, z6.b, z4.b[3]\n"
+ "udot z30.s, z6.b, z5.b[3]\n"
+ "udot z11.s, z7.b, z0.b[3]\n"
+ "udot z15.s, z7.b, z1.b[3]\n"
+ "udot z19.s, z7.b, z2.b[3]\n"
+ "udot z23.s, z7.b, z3.b[3]\n"
+ "udot z27.s, z7.b, z4.b[3]\n"
+ "udot z31.s, z7.b, z5.b[3]\n"
+ "71:" // Height 6: Multiply loop: multiply skip
+ "prfm pldl1keep, [x10, #0x80]\n"
+ "add x12, x12, #0x1\n"
+ "prfm pldl1keep, [x28, #0x80]\n"
+ "prfm pldl1keep, [x26, #0x80]\n"
+ "prfm pldl1keep, [x24, #0x80]\n"
+ "prfm pldl1keep, [x22, #0x80]\n"
+ "prfm pldl1keep, [x20, #0x80]\n"
+ "ldr w19, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "cmp x12, x19\n"
+ "bne 66b\n"
+ "prfm pstl1keep, [x13, #0x0]\n"
+ "prfm pstl1keep, [x9, #0x0]\n"
+ "prfm pstl1keep, [x27, #0x0]\n"
+ "prfm pstl1keep, [x25, #0x0]\n"
+ "prfm pstl1keep, [x23, #0x0]\n"
+ "prfm pstl1keep, [x21, #0x0]\n"
+ "st1w { z8.s }, p4, [x13]\n"
+ "st1w { z9.s }, p3, [x13, #1, MUL VL]\n"
+ "st1w { z10.s }, p2, [x13, #2, MUL VL]\n"
+ "st1w { z11.s }, p1, [x13, #3, MUL VL]\n"
+ "addvl x13, x13, #4\n"
+ "st1w { z12.s }, p4, [x9]\n"
+ "st1w { z13.s }, p3, [x9, #1, MUL VL]\n"
+ "st1w { z14.s }, p2, [x9, #2, MUL VL]\n"
+ "st1w { z15.s }, p1, [x9, #3, MUL VL]\n"
+ "addvl x9, x9, #4\n"
+ "st1w { z16.s }, p4, [x27]\n"
+ "st1w { z17.s }, p3, [x27, #1, MUL VL]\n"
+ "st1w { z18.s }, p2, [x27, #2, MUL VL]\n"
+ "st1w { z19.s }, p1, [x27, #3, MUL VL]\n"
+ "addvl x27, x27, #4\n"
+ "st1w { z20.s }, p4, [x25]\n"
+ "st1w { z21.s }, p3, [x25, #1, MUL VL]\n"
+ "st1w { z22.s }, p2, [x25, #2, MUL VL]\n"
+ "st1w { z23.s }, p1, [x25, #3, MUL VL]\n"
+ "addvl x25, x25, #4\n"
+ "st1w { z24.s }, p4, [x23]\n"
+ "st1w { z25.s }, p3, [x23, #1, MUL VL]\n"
+ "st1w { z26.s }, p2, [x23, #2, MUL VL]\n"
+ "st1w { z27.s }, p1, [x23, #3, MUL VL]\n"
+ "addvl x23, x23, #4\n"
+ "st1w { z28.s }, p4, [x21]\n"
+ "st1w { z29.s }, p3, [x21, #1, MUL VL]\n"
+ "st1w { z30.s }, p2, [x21, #2, MUL VL]\n"
+ "st1w { z31.s }, p1, [x21, #3, MUL VL]\n"
+ "addvl x21, x21, #4\n"
+ "72:" // Height 6: Writeback done
+ "mov x19, #0x0\n"
+ "incw x19, ALL, MUL #4\n"
+ "subs x15, x15, x19\n"
+ "bgt 63b\n"
+ "subs %x[M], %x[M], #0x6\n"
+ "beq 74f\n"
+ "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "tbz %x[flags], #3, 73f\n"
+ "add x20, x20, #0x6\n"
+ "str x20, [%x[args_ptr], %[offsetof_input_offset]]\n"
+ "b 1b\n"
+ "73:" // Update direct input
+ "mov x19, #0x6\n"
+ "madd %x[input_ptr], x19, x20, %x[input_ptr]\n"
+ "b 1b\n"
+ "74:" // Exit
+
+ : [M] "+r" (M), [input_ptr] "+r" (input_ptr), [output_ptr] "+r" (output_ptr)
+ : [args_ptr] "r" (&ka), [flags] "r" (flags), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths))
+ : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+ );
+}
+
+} // namespace arm_gemm
+#endif // __ARM_FEATURE_SVE
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_bf16fp32_dot_3VLx8.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_bf16fp32_dot_8x3VL.hpp
similarity index 89%
rename from src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_bf16fp32_dot_3VLx8.hpp
rename to src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_bf16fp32_dot_8x3VL.hpp
index 43107e4..12bb758 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_bf16fp32_dot_3VLx8.hpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_bf16fp32_dot_8x3VL.hpp
@@ -31,9 +31,9 @@
namespace arm_gemm {
// Actual kernel implementations
-void sve_interleaved_bf16fp32_dot_3VLx8(const bfloat16 *, const bfloat16 *, float *, int, int, int);
+void sve_interleaved_bf16fp32_dot_8x3VL(const bfloat16 *, const bfloat16 *, float *, int, int, int);
-class interleaved_bf16fp32_dot_3VLx8 {
+class cls_sve_interleaved_bf16fp32_dot_8x3VL {
public:
typedef bfloat16 operand_type;
typedef float result_type;
@@ -59,9 +59,9 @@
// Use the standard fixed size transforms.
StdTransformsSVE<operand_type, result_type, 8, 3, 2, 1> transforms = {};
- kern_type kernel=sve_interleaved_bf16fp32_dot_3VLx8;
+ kern_type kernel=sve_interleaved_bf16fp32_dot_8x3VL;
- interleaved_bf16fp32_dot_3VLx8(const CPUInfo *)
+ cls_sve_interleaved_bf16fp32_dot_8x3VL(const CPUInfo *)
{
}
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_bf16fp32_dot_3VLx8/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_bf16fp32_dot_8x3VL/generic.cpp
similarity index 99%
rename from src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_bf16fp32_dot_3VLx8/generic.cpp
rename to src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_bf16fp32_dot_8x3VL/generic.cpp
index 7e20ed0..adee900 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_bf16fp32_dot_3VLx8/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_bf16fp32_dot_8x3VL/generic.cpp
@@ -28,7 +28,7 @@
namespace arm_gemm {
-void sve_interleaved_bf16fp32_dot_3VLx8(const bfloat16 *Apanel, const bfloat16 *Bpanel, float *Cpanel, int ablocks, int bblocks, int K) {
+void sve_interleaved_bf16fp32_dot_8x3VL(const bfloat16 *Apanel, const bfloat16 *Bpanel, float *Cpanel, int ablocks, int bblocks, int K) {
const bfloat16 *a_ptr = Apanel;
float *c_ptr = Cpanel;
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_bf16fp32_mmla_3VLx8.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_bf16fp32_mmla_8x3VL.hpp
similarity index 89%
rename from src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_bf16fp32_mmla_3VLx8.hpp
rename to src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_bf16fp32_mmla_8x3VL.hpp
index f1353e2..2889dd7 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_bf16fp32_mmla_3VLx8.hpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_bf16fp32_mmla_8x3VL.hpp
@@ -31,9 +31,9 @@
namespace arm_gemm {
// Actual kernel implementations
-void sve_interleaved_bf16fp32_mmla_3VLx8(const bfloat16 *, const bfloat16 *, float *, int, int, int);
+void sve_interleaved_bf16fp32_mmla_8x3VL(const bfloat16 *, const bfloat16 *, float *, int, int, int);
-class interleaved_bf16fp32_mmla_3VLx8 {
+class cls_sve_interleaved_bf16fp32_mmla_8x3VL {
public:
typedef bfloat16 operand_type;
typedef float result_type;
@@ -59,9 +59,9 @@
// Use the standard fixed size transforms.
StdTransformsSVE<operand_type, result_type, 8, 6, 4, 2> transforms = {};
- kern_type kernel=sve_interleaved_bf16fp32_mmla_3VLx8;
+ kern_type kernel=sve_interleaved_bf16fp32_mmla_8x3VL;
- interleaved_bf16fp32_mmla_3VLx8(const CPUInfo *)
+ cls_sve_interleaved_bf16fp32_mmla_8x3VL(const CPUInfo *)
{
}
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_bf16fp32_mmla_3VLx8/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_bf16fp32_mmla_8x3VL/generic.cpp
similarity index 99%
rename from src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_bf16fp32_mmla_3VLx8/generic.cpp
rename to src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_bf16fp32_mmla_8x3VL/generic.cpp
index 16cc69b..e43404e 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_bf16fp32_mmla_3VLx8/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_bf16fp32_mmla_8x3VL/generic.cpp
@@ -28,7 +28,7 @@
namespace arm_gemm {
-void sve_interleaved_bf16fp32_mmla_3VLx8(const bfloat16 *Apanel, const bfloat16 *Bpanel, float *Cpanel, int ablocks, int bblocks, int K) {
+void sve_interleaved_bf16fp32_mmla_8x3VL(const bfloat16 *Apanel, const bfloat16 *Bpanel, float *Cpanel, int ablocks, int bblocks, int K) {
const bfloat16 *a_ptr = Apanel;
float *c_ptr = Cpanel;
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp16_mla_3VLx8.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp16_mla_8x3VL.hpp
similarity index 89%
rename from src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp16_mla_3VLx8.hpp
rename to src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp16_mla_8x3VL.hpp
index 816c0cd..eb946d9 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp16_mla_3VLx8.hpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp16_mla_8x3VL.hpp
@@ -31,9 +31,9 @@
namespace arm_gemm {
// Actual kernel implementations
-void sve_interleaved_fp16_mla_3VLx8(const __fp16 *, const __fp16 *, __fp16 *, int, int, int);
+void sve_interleaved_fp16_mla_8x3VL(const __fp16 *, const __fp16 *, __fp16 *, int, int, int);
-class interleaved_fp16_mla_3VLx8 {
+class cls_sve_interleaved_fp16_mla_8x3VL {
public:
typedef __fp16 operand_type;
typedef __fp16 result_type;
@@ -59,9 +59,9 @@
// Use the standard fixed size transforms.
StdTransformsSVE<operand_type, result_type, 8, 3, 1, 1> transforms = {};
- kern_type kernel=sve_interleaved_fp16_mla_3VLx8;
+ kern_type kernel=sve_interleaved_fp16_mla_8x3VL;
- interleaved_fp16_mla_3VLx8(const CPUInfo *)
+ cls_sve_interleaved_fp16_mla_8x3VL(const CPUInfo *)
{
}
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp16_mla_3VLx8/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp16_mla_8x3VL/generic.cpp
similarity index 99%
rename from src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp16_mla_3VLx8/generic.cpp
rename to src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp16_mla_8x3VL/generic.cpp
index f2050cb..46b8770 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp16_mla_3VLx8/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp16_mla_8x3VL/generic.cpp
@@ -28,7 +28,7 @@
namespace arm_gemm {
-void sve_interleaved_fp16_mla_3VLx8(const __fp16 *Apanel, const __fp16 *Bpanel, __fp16 *Cpanel, int ablocks, int bblocks, int K) {
+void sve_interleaved_fp16_mla_8x3VL(const __fp16 *Apanel, const __fp16 *Bpanel, __fp16 *Cpanel, int ablocks, int bblocks, int K) {
const __fp16 *a_ptr = Apanel;
__fp16 *c_ptr = Cpanel;
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp32_mla_3VLx8.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp32_mla_8x3VL.hpp
similarity index 89%
rename from src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp32_mla_3VLx8.hpp
rename to src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp32_mla_8x3VL.hpp
index cce90fb..b84ba83 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp32_mla_3VLx8.hpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp32_mla_8x3VL.hpp
@@ -31,9 +31,9 @@
namespace arm_gemm {
// Actual kernel implementations
-void sve_interleaved_fp32_mla_3VLx8(const float *, const float *, float *, int, int, int);
+void sve_interleaved_fp32_mla_8x3VL(const float *, const float *, float *, int, int, int);
-class interleaved_fp32_mla_3VLx8 {
+class cls_sve_interleaved_fp32_mla_8x3VL {
public:
typedef float operand_type;
typedef float result_type;
@@ -59,9 +59,9 @@
// Use the standard fixed size transforms.
StdTransformsSVE<operand_type, result_type, 8, 3, 1, 1> transforms = {};
- kern_type kernel=sve_interleaved_fp32_mla_3VLx8;
+ kern_type kernel=sve_interleaved_fp32_mla_8x3VL;
- interleaved_fp32_mla_3VLx8(const CPUInfo *)
+ cls_sve_interleaved_fp32_mla_8x3VL(const CPUInfo *)
{
}
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp32_mla_3VLx8/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp32_mla_8x3VL/generic.cpp
similarity index 99%
rename from src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp32_mla_3VLx8/generic.cpp
rename to src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp32_mla_8x3VL/generic.cpp
index cd178c4..1e05a30 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp32_mla_3VLx8/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp32_mla_8x3VL/generic.cpp
@@ -28,7 +28,7 @@
namespace arm_gemm {
-void sve_interleaved_fp32_mla_3VLx8(const float *Apanel, const float *Bpanel, float *Cpanel, int ablocks, int bblocks, int K) {
+void sve_interleaved_fp32_mla_8x3VL(const float *Apanel, const float *Bpanel, float *Cpanel, int ablocks, int bblocks, int K) {
const float *a_ptr = Apanel;
float *c_ptr = Cpanel;
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp32_mmla_3VLx8.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp32_mmla_8x3VL.hpp
similarity index 89%
rename from src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp32_mmla_3VLx8.hpp
rename to src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp32_mmla_8x3VL.hpp
index 4ca43cd..9621696 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp32_mmla_3VLx8.hpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp32_mmla_8x3VL.hpp
@@ -31,9 +31,9 @@
namespace arm_gemm {
// Actual kernel implementations
-void sve_interleaved_fp32_mmla_3VLx8(const float *, const float *, float *, int, int, int);
+void sve_interleaved_fp32_mmla_8x3VL(const float *, const float *, float *, int, int, int);
-class interleaved_fp32_mmla_3VLx8 {
+class cls_sve_interleaved_fp32_mmla_8x3VL {
public:
typedef float operand_type;
typedef float result_type;
@@ -59,9 +59,9 @@
// Use the standard fixed size transforms.
StdTransformsSVE<operand_type, result_type, 8, 6, 2, 2> transforms = {};
- kern_type kernel=sve_interleaved_fp32_mmla_3VLx8;
+ kern_type kernel=sve_interleaved_fp32_mmla_8x3VL;
- interleaved_fp32_mmla_3VLx8(const CPUInfo *)
+ cls_sve_interleaved_fp32_mmla_8x3VL(const CPUInfo *)
{
}
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp32_mmla_3VLx8/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp32_mmla_8x3VL/generic.cpp
similarity index 99%
rename from src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp32_mmla_3VLx8/generic.cpp
rename to src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp32_mmla_8x3VL/generic.cpp
index a404ae9..39daf0f 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp32_mmla_3VLx8/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp32_mmla_8x3VL/generic.cpp
@@ -28,7 +28,7 @@
namespace arm_gemm {
-void sve_interleaved_fp32_mmla_3VLx8(const float *Apanel, const float *Bpanel, float *Cpanel, int ablocks, int bblocks, int K) {
+void sve_interleaved_fp32_mmla_8x3VL(const float *Apanel, const float *Bpanel, float *Cpanel, int ablocks, int bblocks, int K) {
const float *a_ptr = Apanel;
float *c_ptr = Cpanel;
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_dot_3VLx8.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_dot_8x3VL.hpp
similarity index 86%
rename from src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_dot_3VLx8.hpp
rename to src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_dot_8x3VL.hpp
index e40ba21..3e16915 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_dot_3VLx8.hpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_dot_8x3VL.hpp
@@ -31,9 +31,9 @@
namespace arm_gemm {
// Actual kernel implementations
-void sve_interleaved_s8s32_dot_3VLx8(const int8_t *, const int8_t *, int32_t *, int, int, int);
+void sve_interleaved_s8s32_dot_8x3VL(const int8_t *, const int8_t *, int32_t *, int, int, int);
-class interleaved_s8s32_dot_3VLx8 {
+class cls_sve_interleaved_s8s32_dot_8x3VL {
public:
typedef int8_t operand_type;
typedef int32_t result_type;
@@ -58,10 +58,11 @@
// Use the standard fixed size transforms.
StdTransformsSVE<operand_type, result_type, 8, 3, 4, 1> transforms = {};
+ StdTransformsSVE<operand_type, result_type, 8, 3, 4, 1, true> transforms_quantized = {};
- kern_type kernel=sve_interleaved_s8s32_dot_3VLx8;
+ kern_type kernel=sve_interleaved_s8s32_dot_8x3VL;
- interleaved_s8s32_dot_3VLx8(const CPUInfo *)
+ cls_sve_interleaved_s8s32_dot_8x3VL(const CPUInfo *)
{
}
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_dot_3VLx8/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_dot_8x3VL/generic.cpp
similarity index 99%
rename from src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_dot_3VLx8/generic.cpp
rename to src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_dot_8x3VL/generic.cpp
index cdc7070..674c240 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_dot_3VLx8/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_dot_8x3VL/generic.cpp
@@ -28,7 +28,7 @@
namespace arm_gemm {
-void sve_interleaved_s8s32_dot_3VLx8(const int8_t *Apanel, const int8_t *Bpanel, int32_t *Cpanel, int ablocks, int bblocks, int K) {
+void sve_interleaved_s8s32_dot_8x3VL(const int8_t *Apanel, const int8_t *Bpanel, int32_t *Cpanel, int ablocks, int bblocks, int K) {
const int8_t *a_ptr = Apanel;
int32_t *c_ptr = Cpanel;
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_mmla_3VLx8.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_mmla_8x3VL.hpp
similarity index 85%
rename from src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_mmla_3VLx8.hpp
rename to src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_mmla_8x3VL.hpp
index 361598d..02b3451 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_mmla_3VLx8.hpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_mmla_8x3VL.hpp
@@ -31,9 +31,9 @@
namespace arm_gemm {
// Actual kernel implementations
-void sve_interleaved_s8s32_mmla_3VLx8(const int8_t *, const int8_t *, int32_t *, int, int, int);
+void sve_interleaved_s8s32_mmla_8x3VL(const int8_t *, const int8_t *, int32_t *, int, int, int);
-class interleaved_s8s32_mmla_3VLx8 {
+class cls_sve_interleaved_s8s32_mmla_8x3VL {
public:
typedef int8_t operand_type;
typedef int32_t result_type;
@@ -58,10 +58,11 @@
// Use the standard fixed size transforms.
StdTransformsSVE<operand_type, result_type, 8, 6, 8, 2> transforms = {};
+ StdTransformsSVE<operand_type, result_type, 8, 6, 8, 2, true> transforms_quantized = {};
- kern_type kernel=sve_interleaved_s8s32_mmla_3VLx8;
+ kern_type kernel=sve_interleaved_s8s32_mmla_8x3VL;
- interleaved_s8s32_mmla_3VLx8(const CPUInfo *)
+ cls_sve_interleaved_s8s32_mmla_8x3VL(const CPUInfo *)
{
}
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_mmla_3VLx8/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_mmla_8x3VL/generic.cpp
similarity index 99%
rename from src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_mmla_3VLx8/generic.cpp
rename to src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_mmla_8x3VL/generic.cpp
index cde9ec3..578aa01 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_mmla_3VLx8/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_mmla_8x3VL/generic.cpp
@@ -28,7 +28,7 @@
namespace arm_gemm {
-void sve_interleaved_s8s32_mmla_3VLx8(const int8_t *Apanel, const int8_t *Bpanel, int32_t *Cpanel, int ablocks, int bblocks, int K) {
+void sve_interleaved_s8s32_mmla_8x3VL(const int8_t *Apanel, const int8_t *Bpanel, int32_t *Cpanel, int ablocks, int bblocks, int K) {
const int8_t *a_ptr = Apanel;
int32_t *c_ptr = Cpanel;
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_dot_3VLx8.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_dot_8x3VL.hpp
similarity index 86%
rename from src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_dot_3VLx8.hpp
rename to src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_dot_8x3VL.hpp
index 252f38e..832a224 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_dot_3VLx8.hpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_dot_8x3VL.hpp
@@ -31,9 +31,9 @@
namespace arm_gemm {
// Actual kernel implementations
-void sve_interleaved_u8u32_dot_3VLx8(const uint8_t *, const uint8_t *, uint32_t *, int, int, int);
+void sve_interleaved_u8u32_dot_8x3VL(const uint8_t *, const uint8_t *, uint32_t *, int, int, int);
-class interleaved_u8u32_dot_3VLx8 {
+class cls_sve_interleaved_u8u32_dot_8x3VL {
public:
typedef uint8_t operand_type;
typedef uint32_t result_type;
@@ -58,10 +58,11 @@
// Use the standard fixed size transforms.
StdTransformsSVE<operand_type, result_type, 8, 3, 4, 1> transforms = {};
+ StdTransformsSVE<operand_type, result_type, 8, 3, 4, 1, true> transforms_quantized = {};
- kern_type kernel=sve_interleaved_u8u32_dot_3VLx8;
+ kern_type kernel=sve_interleaved_u8u32_dot_8x3VL;
- interleaved_u8u32_dot_3VLx8(const CPUInfo *)
+ cls_sve_interleaved_u8u32_dot_8x3VL(const CPUInfo *)
{
}
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_dot_3VLx8/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_dot_8x3VL/generic.cpp
similarity index 99%
rename from src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_dot_3VLx8/generic.cpp
rename to src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_dot_8x3VL/generic.cpp
index 6626f84..891869c 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_dot_3VLx8/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_dot_8x3VL/generic.cpp
@@ -28,7 +28,7 @@
namespace arm_gemm {
-void sve_interleaved_u8u32_dot_3VLx8(const uint8_t *Apanel, const uint8_t *Bpanel, uint32_t *Cpanel, int ablocks, int bblocks, int K) {
+void sve_interleaved_u8u32_dot_8x3VL(const uint8_t *Apanel, const uint8_t *Bpanel, uint32_t *Cpanel, int ablocks, int bblocks, int K) {
const uint8_t *a_ptr = Apanel;
uint32_t *c_ptr = Cpanel;
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_mmla_3VLx8.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_mmla_8x3VL.hpp
similarity index 85%
rename from src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_mmla_3VLx8.hpp
rename to src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_mmla_8x3VL.hpp
index ed44a9d..4fdaab8 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_mmla_3VLx8.hpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_mmla_8x3VL.hpp
@@ -31,9 +31,9 @@
namespace arm_gemm {
// Actual kernel implementations
-void sve_interleaved_u8u32_mmla_3VLx8(const uint8_t *, const uint8_t *, uint32_t *, int, int, int);
+void sve_interleaved_u8u32_mmla_8x3VL(const uint8_t *, const uint8_t *, uint32_t *, int, int, int);
-class interleaved_u8u32_mmla_3VLx8 {
+class cls_sve_interleaved_u8u32_mmla_8x3VL {
public:
typedef uint8_t operand_type;
typedef uint32_t result_type;
@@ -58,10 +58,11 @@
// Use the standard fixed size transforms.
StdTransformsSVE<operand_type, result_type, 8, 6, 8, 2> transforms = {};
+ StdTransformsSVE<operand_type, result_type, 8, 6, 8, 2, true> transforms_quantized = {};
- kern_type kernel=sve_interleaved_u8u32_mmla_3VLx8;
+ kern_type kernel=sve_interleaved_u8u32_mmla_8x3VL;
- interleaved_u8u32_mmla_3VLx8(const CPUInfo *)
+ cls_sve_interleaved_u8u32_mmla_8x3VL(const CPUInfo *)
{
}
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_mmla_3VLx8/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_mmla_8x3VL/generic.cpp
similarity index 99%
rename from src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_mmla_3VLx8/generic.cpp
rename to src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_mmla_8x3VL/generic.cpp
index 81a1dbc..fa08a9d 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_mmla_3VLx8/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_mmla_8x3VL/generic.cpp
@@ -28,7 +28,7 @@
namespace arm_gemm {
-void sve_interleaved_u8u32_mmla_3VLx8(const uint8_t *Apanel, const uint8_t *Bpanel, uint32_t *Cpanel, int ablocks, int bblocks, int K) {
+void sve_interleaved_u8u32_mmla_8x3VL(const uint8_t *Apanel, const uint8_t *Bpanel, uint32_t *Cpanel, int ablocks, int bblocks, int K) {
const uint8_t *a_ptr = Apanel;
uint32_t *c_ptr = Cpanel;
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_fp32_mla_1VLx8.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_fp32_mla_8x1VL.hpp
similarity index 90%
rename from src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_fp32_mla_1VLx8.hpp
rename to src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_fp32_mla_8x1VL.hpp
index b555066..2097d76a 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_fp32_mla_1VLx8.hpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_fp32_mla_8x1VL.hpp
@@ -31,9 +31,9 @@
{
// Actual kernel implementations
-void sve_smallK_hybrid_fp32_mla_1VLx8(const float *, int, const float *, float *, int, int, int, int, const float *, Activation, bool);
+void sve_smallK_hybrid_fp32_mla_8x1VL(const float *, int, const float *, float *, int, int, int, int, const float *, Activation, bool);
-class smallK_hybrid_fp32_mla_1VLx8
+class cls_sve_smallK_hybrid_fp32_mla_8x1VL
{
public:
typedef float operand_type;
@@ -75,9 +75,9 @@
StdTransformsSVE<operand_type, result_type, 8, 1, 1> transforms = {};
// Default to the generic kernel
- kern_type kernel=sve_smallK_hybrid_fp32_mla_1VLx8;
+ kern_type kernel=sve_smallK_hybrid_fp32_mla_8x1VL;
- smallK_hybrid_fp32_mla_1VLx8(const CPUInfo *)
+ cls_sve_smallK_hybrid_fp32_mla_8x1VL(const CPUInfo *)
{
}
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_fp32_mla_1VLx8/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_fp32_mla_8x1VL/generic.cpp
similarity index 99%
rename from src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_fp32_mla_1VLx8/generic.cpp
rename to src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_fp32_mla_8x1VL/generic.cpp
index 5501688..e07cfa8 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_fp32_mla_1VLx8/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_fp32_mla_8x1VL/generic.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2019 Arm Limited.
+ * Copyright (c) 2019-2020 Arm Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -33,7 +33,7 @@
namespace arm_gemm {
-void sve_smallK_hybrid_fp32_mla_1VLx8(const float *A, int lda, const float *B, float *C, int ldc, int M, int N, int K, const float *bias, Activation act, bool) {
+void sve_smallK_hybrid_fp32_mla_8x1VL(const float *A, int lda, const float *B, float *C, int ldc, int M, int N, int K, const float *bias, Activation act, bool) {
const long loops_count = iceildiv(N, (int)get_vector_length<float>()) - 1;
const long ldab = lda * sizeof(float);
const long ldcb = ldc * sizeof(float);
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_s8s32_dot_1VLx8.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_s8s32_dot_8x1VL.hpp
similarity index 90%
rename from src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_s8s32_dot_1VLx8.hpp
rename to src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_s8s32_dot_8x1VL.hpp
index eef1e4c..e50c05b 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_s8s32_dot_1VLx8.hpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_s8s32_dot_8x1VL.hpp
@@ -31,9 +31,9 @@
{
// Actual kernel implementations
-void sve_smallK_hybrid_s8s32_dot_1VLx8(const int8_t *, int, const int8_t *, int32_t *, int, int, int, int, const int32_t *, Activation, bool);
+void sve_smallK_hybrid_s8s32_dot_8x1VL(const int8_t *, int, const int8_t *, int32_t *, int, int, int, int, const int32_t *, Activation, bool);
-class smallK_hybrid_s8s32_dot_1VLx8
+class cls_sve_smallK_hybrid_s8s32_dot_8x1VL
{
public:
typedef int8_t operand_type;
@@ -75,9 +75,9 @@
StdTransformsSVE<operand_type, result_type, 8, 1, 4> transforms = {};
// Default to the generic kernel
- kern_type kernel=sve_smallK_hybrid_s8s32_dot_1VLx8;
+ kern_type kernel=sve_smallK_hybrid_s8s32_dot_8x1VL;
- smallK_hybrid_s8s32_dot_1VLx8(const CPUInfo *)
+ cls_sve_smallK_hybrid_s8s32_dot_8x1VL(const CPUInfo *)
{
}
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_s8s32_dot_1VLx8/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_s8s32_dot_8x1VL/generic.cpp
similarity index 82%
rename from src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_s8s32_dot_1VLx8/generic.cpp
rename to src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_s8s32_dot_8x1VL/generic.cpp
index e2fbdcb..5770076 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_s8s32_dot_1VLx8/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_s8s32_dot_8x1VL/generic.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2019 Arm Limited.
+ * Copyright (c) 2019-2020 Arm Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -33,7 +33,7 @@
namespace arm_gemm {
-void sve_smallK_hybrid_s8s32_dot_1VLx8(const int8_t *A, int lda, const int8_t *B, int32_t *C, int ldc, int M, int N, int K, const int32_t *, Activation, bool) {
+void sve_smallK_hybrid_s8s32_dot_8x1VL(const int8_t *A, int lda, const int8_t *B, int32_t *C, int ldc, int M, int N, int K, const int32_t *bias, Activation act, bool) {
const long loops_count = iceildiv(N, (int)get_vector_length<int32_t>()) - 1;
const long ldab = lda * sizeof(int8_t);
const long ldcb = ldc * sizeof(int32_t);
@@ -112,55 +112,54 @@
"add c_ptr1, %[c_ptr0], #0x0\n"
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
- "mov z24.s, #0\n"
"ptrue p7.b\n"
- "mov z25.s, #0\n"
"whilelt p6.b, %[temp], %[odd_depth]\n"
- "mov z26.s, #0\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
- "mov z27.s, #0\n"
"whilelt p0.s, %[temp], %[last_width]\n"
- "mov z28.s, #0\n"
- "ld1rqb z0.b, p6/z, [%[a_ptr0]]\n"
- "mov z29.s, #0\n"
- "ld1rqb z1.b, p6/z, [a_ptr1]\n"
- "mov z30.s, #0\n"
- "ld1rqb z2.b, p6/z, [a_ptr2]\n"
- "mov z31.s, #0\n"
- "ld1rqb z3.b, p6/z, [a_ptr3]\n"
- "sdot z24.s, z16.b, z0.b[0]\n"
- "ld1rqb z4.b, p6/z, [a_ptr4]\n"
- "sdot z25.s, z16.b, z1.b[0]\n"
- "ld1rqb z5.b, p6/z, [a_ptr5]\n"
- "sdot z26.s, z16.b, z2.b[0]\n"
- "ld1rqb z6.b, p6/z, [a_ptr6]\n"
- "sdot z27.s, z16.b, z3.b[0]\n"
- "ld1rqb z7.b, p6/z, [a_ptr7]\n"
- "sdot z28.s, z16.b, z4.b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
- "sdot z29.s, z16.b, z5.b[0]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
- "sdot z30.s, z16.b, z6.b[0]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
- "sdot z31.s, z16.b, z7.b[0]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- "addvl %[b_ptr0], %[b_ptr0], #1\n"
- "cbz %[loops], 2f\n"
- "subs %[loops], %[loops], #0x1\n"
"ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
+ "ld1rqb z0.b, p6/z, [%[a_ptr0]]\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
+ "ld1rqb z1.b, p6/z, [a_ptr1]\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
+ "ld1rqb z2.b, p6/z, [a_ptr2]\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
+ "ld1rqb z3.b, p6/z, [a_ptr3]\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
+ "ld1rqb z4.b, p6/z, [a_ptr4]\n"
"addvl %[b_ptr0], %[b_ptr0], #1\n"
+ "ld1rqb z5.b, p6/z, [a_ptr5]\n"
+ "ld1rqb z6.b, p6/z, [a_ptr6]\n"
+ "ld1rqb z7.b, p6/z, [a_ptr7]\n"
+ "cbz %[loops], 2f\n"
+ "mov z24.s, #0\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "mov z25.s, #0\n"
+ "mov z26.s, #0\n"
+ "mov z27.s, #0\n"
+ "mov z28.s, #0\n"
+ "mov z29.s, #0\n"
+ "mov z30.s, #0\n"
+ "mov z31.s, #0\n"
+ "sdot z24.s, z16.b, z0.b[0]\n"
+ "sdot z25.s, z16.b, z1.b[0]\n"
+ "sdot z26.s, z16.b, z2.b[0]\n"
+ "sdot z27.s, z16.b, z3.b[0]\n"
+ "sdot z28.s, z16.b, z4.b[0]\n"
+ "sdot z29.s, z16.b, z5.b[0]\n"
+ "sdot z30.s, z16.b, z6.b[0]\n"
+ "sdot z31.s, z16.b, z7.b[0]\n"
"b.eq 3f\n"
"4:\n"
"st1w z24.s, p7, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"mov z24.s, #0\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
"addvl %[c_ptr0], %[c_ptr0], #1\n"
"st1w z25.s, p7, [c_ptr1]\n"
"addvl c_ptr1, c_ptr1, #1\n"
"mov z25.s, #0\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #1\n"
"sdot z24.s, z16.b, z0.b[0]\n"
"st1w z26.s, p7, [c_ptr2]\n"
"mov z26.s, #0\n"
@@ -186,10 +185,9 @@
"mov z31.s, #0\n"
"addvl c_ptr7, c_ptr7, #1\n"
"sdot z30.s, z16.b, z6.b[0]\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
"sdot z31.s, z16.b, z7.b[0]\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
- "addvl %[b_ptr0], %[b_ptr0], #1\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
"prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
"prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
"prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
@@ -201,6 +199,8 @@
"st1w z24.s, p7, [%[c_ptr0]]\n"
"addvl %[c_ptr0], %[c_ptr0], #1\n"
"mov z24.s, #0\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #1\n"
"st1w z25.s, p7, [c_ptr1]\n"
"addvl c_ptr1, c_ptr1, #1\n"
"mov z25.s, #0\n"
@@ -230,23 +230,34 @@
"addvl c_ptr7, c_ptr7, #1\n"
"sdot z30.s, z16.b, z6.b[0]\n"
"sdot z31.s, z16.b, z7.b[0]\n"
+ "b 5f\n"
"2:\n"
+ "mov z24.s, #0\n"
+ "mov z25.s, #0\n"
+ "mov z26.s, #0\n"
+ "mov z27.s, #0\n"
+ "mov z28.s, #0\n"
+ "mov z29.s, #0\n"
+ "mov z30.s, #0\n"
+ "mov z31.s, #0\n"
+ "sdot z24.s, z16.b, z0.b[0]\n"
+ "sdot z25.s, z16.b, z1.b[0]\n"
+ "sdot z26.s, z16.b, z2.b[0]\n"
+ "sdot z27.s, z16.b, z3.b[0]\n"
+ "sdot z28.s, z16.b, z4.b[0]\n"
+ "sdot z29.s, z16.b, z5.b[0]\n"
+ "sdot z30.s, z16.b, z6.b[0]\n"
+ "sdot z31.s, z16.b, z7.b[0]\n"
+ "5:\n"
"st1w z24.s, p0, [%[c_ptr0]]\n"
"addvl %[c_ptr0], %[c_ptr0], #1\n"
"st1w z25.s, p0, [c_ptr1]\n"
- "addvl c_ptr1, c_ptr1, #1\n"
"st1w z26.s, p0, [c_ptr2]\n"
- "addvl c_ptr2, c_ptr2, #1\n"
"st1w z27.s, p0, [c_ptr3]\n"
- "addvl c_ptr3, c_ptr3, #1\n"
"st1w z28.s, p0, [c_ptr4]\n"
- "addvl c_ptr4, c_ptr4, #1\n"
"st1w z29.s, p0, [c_ptr5]\n"
- "addvl c_ptr5, c_ptr5, #1\n"
"st1w z30.s, p0, [c_ptr6]\n"
- "addvl c_ptr6, c_ptr6, #1\n"
"st1w z31.s, p0, [c_ptr7]\n"
- "addvl c_ptr7, c_ptr7, #1\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -325,112 +336,112 @@
"add c_ptr1, %[c_ptr0], #0x0\n"
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
- "mov z24.s, #0\n"
"ptrue p7.b\n"
- "mov z25.s, #0\n"
"whilelt p6.b, %[temp], %[odd_depth]\n"
- "mov z26.s, #0\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
- "mov z27.s, #0\n"
- "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "mov z28.s, #0\n"
- "ld1rqb z0.b, p6/z, [%[a_ptr0]]\n"
- "mov z29.s, #0\n"
- "ld1rqb z1.b, p6/z, [a_ptr1]\n"
- "mov z30.s, #0\n"
- "ld1rqb z2.b, p6/z, [a_ptr2]\n"
- "mov z31.s, #0\n"
- "ld1rqb z3.b, p6/z, [a_ptr3]\n"
- "sdot z24.s, z16.b, z0.b[0]\n"
- "ld1rqb z4.b, p6/z, [a_ptr4]\n"
- "sdot z25.s, z16.b, z1.b[0]\n"
- "ld1rqb z5.b, p6/z, [a_ptr5]\n"
- "sdot z26.s, z16.b, z2.b[0]\n"
- "ld1rqb z6.b, p6/z, [a_ptr6]\n"
- "sdot z27.s, z16.b, z3.b[0]\n"
- "ld1rqb z7.b, p6/z, [a_ptr7]\n"
- "sdot z28.s, z16.b, z4.b[0]\n"
"whilelt p0.s, %[temp], %[last_width]\n"
- "sdot z29.s, z16.b, z5.b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
- "sdot z30.s, z16.b, z6.b[0]\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
- "sdot z31.s, z16.b, z7.b[0]\n"
+ "ld1rqb z0.b, p6/z, [%[a_ptr0]]\n"
"prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
- "sdot z24.s, z17.b, z0.b[1]\n"
+ "ld1rqb z1.b, p6/z, [a_ptr1]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
- "sdot z25.s, z17.b, z1.b[1]\n"
+ "ld1rqb z2.b, p6/z, [a_ptr2]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- "sdot z26.s, z17.b, z2.b[1]\n"
+ "ld1rqb z3.b, p6/z, [a_ptr3]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- "sdot z27.s, z17.b, z3.b[1]\n"
+ "ld1rqb z4.b, p6/z, [a_ptr4]\n"
+ "ld1rqb z5.b, p6/z, [a_ptr5]\n"
+ "ld1rqb z6.b, p6/z, [a_ptr6]\n"
+ "ld1rqb z7.b, p6/z, [a_ptr7]\n"
+ "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"addvl %[b_ptr0], %[b_ptr0], #2\n"
+ "cbz %[loops], 2f\n"
+ "mov z24.s, #0\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "mov z25.s, #0\n"
+ "mov z26.s, #0\n"
+ "mov z27.s, #0\n"
+ "mov z28.s, #0\n"
+ "mov z29.s, #0\n"
+ "mov z30.s, #0\n"
+ "mov z31.s, #0\n"
+ "sdot z24.s, z16.b, z0.b[0]\n"
+ "sdot z25.s, z16.b, z1.b[0]\n"
+ "sdot z26.s, z16.b, z2.b[0]\n"
+ "sdot z27.s, z16.b, z3.b[0]\n"
+ "sdot z28.s, z16.b, z4.b[0]\n"
+ "sdot z29.s, z16.b, z5.b[0]\n"
+ "sdot z30.s, z16.b, z6.b[0]\n"
+ "sdot z31.s, z16.b, z7.b[0]\n"
+ "sdot z24.s, z17.b, z0.b[1]\n"
+ "sdot z25.s, z17.b, z1.b[1]\n"
+ "sdot z26.s, z17.b, z2.b[1]\n"
+ "sdot z27.s, z17.b, z3.b[1]\n"
"sdot z28.s, z17.b, z4.b[1]\n"
"sdot z29.s, z17.b, z5.b[1]\n"
"sdot z30.s, z17.b, z6.b[1]\n"
"sdot z31.s, z17.b, z7.b[1]\n"
- "cbz %[loops], 2f\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
"b.eq 3f\n"
"4:\n"
"st1w z24.s, p7, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"mov z24.s, #0\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
"ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #2\n"
- "st1w z25.s, p7, [c_ptr1]\n"
"addvl %[c_ptr0], %[c_ptr0], #1\n"
- "mov z25.s, #0\n"
+ "st1w z25.s, p7, [c_ptr1]\n"
"addvl c_ptr1, c_ptr1, #1\n"
+ "mov z25.s, #0\n"
+ "addvl %[b_ptr0], %[b_ptr0], #2\n"
"sdot z24.s, z16.b, z0.b[0]\n"
"st1w z26.s, p7, [c_ptr2]\n"
"mov z26.s, #0\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "addvl c_ptr2, c_ptr2, #1\n"
"sdot z25.s, z16.b, z1.b[0]\n"
"st1w z27.s, p7, [c_ptr3]\n"
"mov z27.s, #0\n"
- "addvl c_ptr2, c_ptr2, #1\n"
+ "addvl c_ptr3, c_ptr3, #1\n"
"sdot z26.s, z16.b, z2.b[0]\n"
"st1w z28.s, p7, [c_ptr4]\n"
"mov z28.s, #0\n"
- "addvl c_ptr3, c_ptr3, #1\n"
+ "addvl c_ptr4, c_ptr4, #1\n"
"sdot z27.s, z16.b, z3.b[0]\n"
"st1w z29.s, p7, [c_ptr5]\n"
"mov z29.s, #0\n"
- "addvl c_ptr4, c_ptr4, #1\n"
+ "addvl c_ptr5, c_ptr5, #1\n"
"sdot z28.s, z16.b, z4.b[0]\n"
"st1w z30.s, p7, [c_ptr6]\n"
"mov z30.s, #0\n"
- "addvl c_ptr5, c_ptr5, #1\n"
+ "addvl c_ptr6, c_ptr6, #1\n"
"sdot z29.s, z16.b, z5.b[0]\n"
"st1w z31.s, p7, [c_ptr7]\n"
"mov z31.s, #0\n"
- "addvl c_ptr6, c_ptr6, #1\n"
- "sdot z30.s, z16.b, z6.b[0]\n"
"addvl c_ptr7, c_ptr7, #1\n"
+ "sdot z30.s, z16.b, z6.b[0]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
"sdot z31.s, z16.b, z7.b[0]\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
- "sdot z24.s, z17.b, z0.b[1]\n"
"prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- "sdot z25.s, z17.b, z1.b[1]\n"
+ "sdot z24.s, z17.b, z0.b[1]\n"
"prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- "sdot z26.s, z17.b, z2.b[1]\n"
+ "sdot z25.s, z17.b, z1.b[1]\n"
"prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- "sdot z27.s, z17.b, z3.b[1]\n"
+ "sdot z26.s, z17.b, z2.b[1]\n"
"prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- "sdot z28.s, z17.b, z4.b[1]\n"
+ "sdot z27.s, z17.b, z3.b[1]\n"
"prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- "sdot z29.s, z17.b, z5.b[1]\n"
+ "sdot z28.s, z17.b, z4.b[1]\n"
"prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
- "sdot z30.s, z17.b, z6.b[1]\n"
+ "sdot z29.s, z17.b, z5.b[1]\n"
"prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
+ "sdot z30.s, z17.b, z6.b[1]\n"
"sdot z31.s, z17.b, z7.b[1]\n"
"b.ne 4b\n"
"3:\n"
"st1w z24.s, p7, [%[c_ptr0]]\n"
"addvl %[c_ptr0], %[c_ptr0], #1\n"
"mov z24.s, #0\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
"ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"addvl %[b_ptr0], %[b_ptr0], #2\n"
"st1w z25.s, p7, [c_ptr1]\n"
@@ -470,23 +481,42 @@
"sdot z29.s, z17.b, z5.b[1]\n"
"sdot z30.s, z17.b, z6.b[1]\n"
"sdot z31.s, z17.b, z7.b[1]\n"
+ "b 5f\n"
"2:\n"
+ "mov z24.s, #0\n"
+ "mov z25.s, #0\n"
+ "mov z26.s, #0\n"
+ "mov z27.s, #0\n"
+ "mov z28.s, #0\n"
+ "mov z29.s, #0\n"
+ "mov z30.s, #0\n"
+ "mov z31.s, #0\n"
+ "sdot z24.s, z16.b, z0.b[0]\n"
+ "sdot z25.s, z16.b, z1.b[0]\n"
+ "sdot z26.s, z16.b, z2.b[0]\n"
+ "sdot z27.s, z16.b, z3.b[0]\n"
+ "sdot z28.s, z16.b, z4.b[0]\n"
+ "sdot z29.s, z16.b, z5.b[0]\n"
+ "sdot z30.s, z16.b, z6.b[0]\n"
+ "sdot z31.s, z16.b, z7.b[0]\n"
+ "sdot z24.s, z17.b, z0.b[1]\n"
+ "sdot z25.s, z17.b, z1.b[1]\n"
+ "sdot z26.s, z17.b, z2.b[1]\n"
+ "sdot z27.s, z17.b, z3.b[1]\n"
+ "sdot z28.s, z17.b, z4.b[1]\n"
+ "sdot z29.s, z17.b, z5.b[1]\n"
+ "sdot z30.s, z17.b, z6.b[1]\n"
+ "sdot z31.s, z17.b, z7.b[1]\n"
+ "5:\n"
"st1w z24.s, p0, [%[c_ptr0]]\n"
"addvl %[c_ptr0], %[c_ptr0], #1\n"
"st1w z25.s, p0, [c_ptr1]\n"
- "addvl c_ptr1, c_ptr1, #1\n"
"st1w z26.s, p0, [c_ptr2]\n"
- "addvl c_ptr2, c_ptr2, #1\n"
"st1w z27.s, p0, [c_ptr3]\n"
- "addvl c_ptr3, c_ptr3, #1\n"
"st1w z28.s, p0, [c_ptr4]\n"
- "addvl c_ptr4, c_ptr4, #1\n"
"st1w z29.s, p0, [c_ptr5]\n"
- "addvl c_ptr5, c_ptr5, #1\n"
"st1w z30.s, p0, [c_ptr6]\n"
- "addvl c_ptr6, c_ptr6, #1\n"
"st1w z31.s, p0, [c_ptr7]\n"
- "addvl c_ptr7, c_ptr7, #1\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -565,48 +595,50 @@
"add c_ptr1, %[c_ptr0], #0x0\n"
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
- "mov z24.s, #0\n"
"ptrue p7.b\n"
- "mov z25.s, #0\n"
"whilelt p6.b, %[temp], %[odd_depth]\n"
- "mov z26.s, #0\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
- "mov z27.s, #0\n"
- "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "mov z28.s, #0\n"
- "ld1rqb z0.b, p6/z, [%[a_ptr0]]\n"
- "mov z29.s, #0\n"
- "ld1rqb z1.b, p6/z, [a_ptr1]\n"
- "mov z30.s, #0\n"
- "ld1rqb z2.b, p6/z, [a_ptr2]\n"
- "mov z31.s, #0\n"
- "ld1rqb z3.b, p6/z, [a_ptr3]\n"
- "sdot z24.s, z16.b, z0.b[0]\n"
- "ld1rqb z4.b, p6/z, [a_ptr4]\n"
- "sdot z25.s, z16.b, z1.b[0]\n"
- "ld1rqb z5.b, p6/z, [a_ptr5]\n"
- "sdot z26.s, z16.b, z2.b[0]\n"
- "ld1rqb z6.b, p6/z, [a_ptr6]\n"
- "sdot z27.s, z16.b, z3.b[0]\n"
- "ld1rqb z7.b, p6/z, [a_ptr7]\n"
- "sdot z28.s, z16.b, z4.b[0]\n"
- "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "sdot z29.s, z16.b, z5.b[0]\n"
"whilelt p0.s, %[temp], %[last_width]\n"
- "sdot z30.s, z16.b, z6.b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
- "sdot z31.s, z16.b, z7.b[0]\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
- "sdot z24.s, z17.b, z0.b[1]\n"
+ "ld1rqb z0.b, p6/z, [%[a_ptr0]]\n"
"prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
- "sdot z25.s, z17.b, z1.b[1]\n"
+ "ld1rqb z1.b, p6/z, [a_ptr1]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
- "sdot z26.s, z17.b, z2.b[1]\n"
+ "ld1rqb z2.b, p6/z, [a_ptr2]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- "sdot z27.s, z17.b, z3.b[1]\n"
+ "ld1rqb z3.b, p6/z, [a_ptr3]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- "sdot z28.s, z17.b, z4.b[1]\n"
+ "ld1rqb z4.b, p6/z, [a_ptr4]\n"
+ "ld1rqb z5.b, p6/z, [a_ptr5]\n"
+ "ld1rqb z6.b, p6/z, [a_ptr6]\n"
+ "ld1rqb z7.b, p6/z, [a_ptr7]\n"
+ "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"addvl %[b_ptr0], %[b_ptr0], #3\n"
+ "cbz %[loops], 2f\n"
+ "mov z24.s, #0\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "mov z25.s, #0\n"
+ "mov z26.s, #0\n"
+ "mov z27.s, #0\n"
+ "mov z28.s, #0\n"
+ "mov z29.s, #0\n"
+ "mov z30.s, #0\n"
+ "mov z31.s, #0\n"
+ "sdot z24.s, z16.b, z0.b[0]\n"
+ "sdot z25.s, z16.b, z1.b[0]\n"
+ "sdot z26.s, z16.b, z2.b[0]\n"
+ "sdot z27.s, z16.b, z3.b[0]\n"
+ "sdot z28.s, z16.b, z4.b[0]\n"
+ "sdot z29.s, z16.b, z5.b[0]\n"
+ "sdot z30.s, z16.b, z6.b[0]\n"
+ "sdot z31.s, z16.b, z7.b[0]\n"
+ "sdot z24.s, z17.b, z0.b[1]\n"
+ "sdot z25.s, z17.b, z1.b[1]\n"
+ "sdot z26.s, z17.b, z2.b[1]\n"
+ "sdot z27.s, z17.b, z3.b[1]\n"
+ "sdot z28.s, z17.b, z4.b[1]\n"
"sdot z29.s, z17.b, z5.b[1]\n"
"sdot z30.s, z17.b, z6.b[1]\n"
"sdot z31.s, z17.b, z7.b[1]\n"
@@ -618,49 +650,46 @@
"sdot z29.s, z18.b, z5.b[2]\n"
"sdot z30.s, z18.b, z6.b[2]\n"
"sdot z31.s, z18.b, z7.b[2]\n"
- "cbz %[loops], 2f\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"b.eq 3f\n"
"4:\n"
"st1w z24.s, p7, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"mov z24.s, #0\n"
- "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #3\n"
- "st1w z25.s, p7, [c_ptr1]\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"addvl %[c_ptr0], %[c_ptr0], #1\n"
- "mov z25.s, #0\n"
+ "st1w z25.s, p7, [c_ptr1]\n"
"addvl c_ptr1, c_ptr1, #1\n"
+ "mov z25.s, #0\n"
+ "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"sdot z24.s, z16.b, z0.b[0]\n"
"st1w z26.s, p7, [c_ptr2]\n"
"mov z26.s, #0\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "addvl c_ptr2, c_ptr2, #1\n"
"sdot z25.s, z16.b, z1.b[0]\n"
"st1w z27.s, p7, [c_ptr3]\n"
"mov z27.s, #0\n"
- "addvl c_ptr2, c_ptr2, #1\n"
+ "addvl c_ptr3, c_ptr3, #1\n"
"sdot z26.s, z16.b, z2.b[0]\n"
"st1w z28.s, p7, [c_ptr4]\n"
"mov z28.s, #0\n"
- "addvl c_ptr3, c_ptr3, #1\n"
+ "addvl c_ptr4, c_ptr4, #1\n"
"sdot z27.s, z16.b, z3.b[0]\n"
"st1w z29.s, p7, [c_ptr5]\n"
"mov z29.s, #0\n"
- "addvl c_ptr4, c_ptr4, #1\n"
+ "addvl c_ptr5, c_ptr5, #1\n"
"sdot z28.s, z16.b, z4.b[0]\n"
"st1w z30.s, p7, [c_ptr6]\n"
"mov z30.s, #0\n"
- "addvl c_ptr5, c_ptr5, #1\n"
+ "addvl c_ptr6, c_ptr6, #1\n"
"sdot z29.s, z16.b, z5.b[0]\n"
"st1w z31.s, p7, [c_ptr7]\n"
"mov z31.s, #0\n"
- "addvl c_ptr6, c_ptr6, #1\n"
- "sdot z30.s, z16.b, z6.b[0]\n"
"addvl c_ptr7, c_ptr7, #1\n"
+ "sdot z30.s, z16.b, z6.b[0]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #3\n"
"sdot z31.s, z16.b, z7.b[0]\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
"sdot z24.s, z17.b, z0.b[1]\n"
"prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
"sdot z25.s, z17.b, z1.b[1]\n"
@@ -676,7 +705,6 @@
"sdot z30.s, z17.b, z6.b[1]\n"
"prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
"sdot z31.s, z17.b, z7.b[1]\n"
- "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"sdot z24.s, z18.b, z0.b[2]\n"
"sdot z25.s, z18.b, z1.b[2]\n"
"sdot z26.s, z18.b, z2.b[2]\n"
@@ -690,11 +718,12 @@
"st1w z24.s, p7, [%[c_ptr0]]\n"
"addvl %[c_ptr0], %[c_ptr0], #1\n"
"mov z24.s, #0\n"
- "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #3\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"st1w z25.s, p7, [c_ptr1]\n"
"addvl c_ptr1, c_ptr1, #1\n"
"mov z25.s, #0\n"
+ "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"sdot z24.s, z16.b, z0.b[0]\n"
"st1w z26.s, p7, [c_ptr2]\n"
"mov z26.s, #0\n"
@@ -720,8 +749,9 @@
"mov z31.s, #0\n"
"addvl c_ptr7, c_ptr7, #1\n"
"sdot z30.s, z16.b, z6.b[0]\n"
- "sdot z24.s, z17.b, z0.b[1]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #3\n"
"sdot z31.s, z16.b, z7.b[0]\n"
+ "sdot z24.s, z17.b, z0.b[1]\n"
"sdot z25.s, z17.b, z1.b[1]\n"
"sdot z26.s, z17.b, z2.b[1]\n"
"sdot z27.s, z17.b, z3.b[1]\n"
@@ -737,23 +767,50 @@
"sdot z29.s, z18.b, z5.b[2]\n"
"sdot z30.s, z18.b, z6.b[2]\n"
"sdot z31.s, z18.b, z7.b[2]\n"
+ "b 5f\n"
"2:\n"
+ "mov z24.s, #0\n"
+ "mov z25.s, #0\n"
+ "mov z26.s, #0\n"
+ "mov z27.s, #0\n"
+ "mov z28.s, #0\n"
+ "mov z29.s, #0\n"
+ "mov z30.s, #0\n"
+ "mov z31.s, #0\n"
+ "sdot z24.s, z16.b, z0.b[0]\n"
+ "sdot z25.s, z16.b, z1.b[0]\n"
+ "sdot z26.s, z16.b, z2.b[0]\n"
+ "sdot z27.s, z16.b, z3.b[0]\n"
+ "sdot z28.s, z16.b, z4.b[0]\n"
+ "sdot z29.s, z16.b, z5.b[0]\n"
+ "sdot z30.s, z16.b, z6.b[0]\n"
+ "sdot z31.s, z16.b, z7.b[0]\n"
+ "sdot z24.s, z17.b, z0.b[1]\n"
+ "sdot z25.s, z17.b, z1.b[1]\n"
+ "sdot z26.s, z17.b, z2.b[1]\n"
+ "sdot z27.s, z17.b, z3.b[1]\n"
+ "sdot z28.s, z17.b, z4.b[1]\n"
+ "sdot z29.s, z17.b, z5.b[1]\n"
+ "sdot z30.s, z17.b, z6.b[1]\n"
+ "sdot z31.s, z17.b, z7.b[1]\n"
+ "sdot z24.s, z18.b, z0.b[2]\n"
+ "sdot z25.s, z18.b, z1.b[2]\n"
+ "sdot z26.s, z18.b, z2.b[2]\n"
+ "sdot z27.s, z18.b, z3.b[2]\n"
+ "sdot z28.s, z18.b, z4.b[2]\n"
+ "sdot z29.s, z18.b, z5.b[2]\n"
+ "sdot z30.s, z18.b, z6.b[2]\n"
+ "sdot z31.s, z18.b, z7.b[2]\n"
+ "5:\n"
"st1w z24.s, p0, [%[c_ptr0]]\n"
"addvl %[c_ptr0], %[c_ptr0], #1\n"
"st1w z25.s, p0, [c_ptr1]\n"
- "addvl c_ptr1, c_ptr1, #1\n"
"st1w z26.s, p0, [c_ptr2]\n"
- "addvl c_ptr2, c_ptr2, #1\n"
"st1w z27.s, p0, [c_ptr3]\n"
- "addvl c_ptr3, c_ptr3, #1\n"
"st1w z28.s, p0, [c_ptr4]\n"
- "addvl c_ptr4, c_ptr4, #1\n"
"st1w z29.s, p0, [c_ptr5]\n"
- "addvl c_ptr5, c_ptr5, #1\n"
"st1w z30.s, p0, [c_ptr6]\n"
- "addvl c_ptr6, c_ptr6, #1\n"
"st1w z31.s, p0, [c_ptr7]\n"
- "addvl c_ptr7, c_ptr7, #1\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -832,50 +889,52 @@
"add c_ptr1, %[c_ptr0], #0x0\n"
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
- "mov z24.s, #0\n"
"ptrue p7.b\n"
- "mov z25.s, #0\n"
"whilelt p6.b, %[temp], %[odd_depth]\n"
- "mov z26.s, #0\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
- "mov z27.s, #0\n"
- "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "mov z28.s, #0\n"
- "ld1rqb z0.b, p6/z, [%[a_ptr0]]\n"
- "mov z29.s, #0\n"
- "ld1rqb z1.b, p6/z, [a_ptr1]\n"
- "mov z30.s, #0\n"
- "ld1rqb z2.b, p6/z, [a_ptr2]\n"
- "mov z31.s, #0\n"
- "ld1rqb z3.b, p6/z, [a_ptr3]\n"
- "sdot z24.s, z16.b, z0.b[0]\n"
- "ld1rqb z4.b, p6/z, [a_ptr4]\n"
- "sdot z25.s, z16.b, z1.b[0]\n"
- "ld1rqb z5.b, p6/z, [a_ptr5]\n"
- "sdot z26.s, z16.b, z2.b[0]\n"
- "ld1rqb z6.b, p6/z, [a_ptr6]\n"
- "sdot z27.s, z16.b, z3.b[0]\n"
- "ld1rqb z7.b, p6/z, [a_ptr7]\n"
- "sdot z28.s, z16.b, z4.b[0]\n"
- "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "sdot z29.s, z16.b, z5.b[0]\n"
- "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "sdot z30.s, z16.b, z6.b[0]\n"
"whilelt p0.s, %[temp], %[last_width]\n"
- "sdot z31.s, z16.b, z7.b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
- "sdot z24.s, z17.b, z0.b[1]\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
- "sdot z25.s, z17.b, z1.b[1]\n"
+ "ld1rqb z0.b, p6/z, [%[a_ptr0]]\n"
"prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
- "sdot z26.s, z17.b, z2.b[1]\n"
+ "ld1rqb z1.b, p6/z, [a_ptr1]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
- "sdot z27.s, z17.b, z3.b[1]\n"
+ "ld1rqb z2.b, p6/z, [a_ptr2]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- "sdot z28.s, z17.b, z4.b[1]\n"
+ "ld1rqb z3.b, p6/z, [a_ptr3]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- "sdot z29.s, z17.b, z5.b[1]\n"
+ "ld1rqb z4.b, p6/z, [a_ptr4]\n"
+ "ld1rqb z5.b, p6/z, [a_ptr5]\n"
+ "ld1rqb z6.b, p6/z, [a_ptr6]\n"
+ "ld1rqb z7.b, p6/z, [a_ptr7]\n"
+ "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"addvl %[b_ptr0], %[b_ptr0], #4\n"
+ "cbz %[loops], 2f\n"
+ "mov z24.s, #0\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "mov z25.s, #0\n"
+ "mov z26.s, #0\n"
+ "mov z27.s, #0\n"
+ "mov z28.s, #0\n"
+ "mov z29.s, #0\n"
+ "mov z30.s, #0\n"
+ "mov z31.s, #0\n"
+ "sdot z24.s, z16.b, z0.b[0]\n"
+ "sdot z25.s, z16.b, z1.b[0]\n"
+ "sdot z26.s, z16.b, z2.b[0]\n"
+ "sdot z27.s, z16.b, z3.b[0]\n"
+ "sdot z28.s, z16.b, z4.b[0]\n"
+ "sdot z29.s, z16.b, z5.b[0]\n"
+ "sdot z30.s, z16.b, z6.b[0]\n"
+ "sdot z31.s, z16.b, z7.b[0]\n"
+ "sdot z24.s, z17.b, z0.b[1]\n"
+ "sdot z25.s, z17.b, z1.b[1]\n"
+ "sdot z26.s, z17.b, z2.b[1]\n"
+ "sdot z27.s, z17.b, z3.b[1]\n"
+ "sdot z28.s, z17.b, z4.b[1]\n"
+ "sdot z29.s, z17.b, z5.b[1]\n"
"sdot z30.s, z17.b, z6.b[1]\n"
"sdot z31.s, z17.b, z7.b[1]\n"
"sdot z24.s, z18.b, z0.b[2]\n"
@@ -894,50 +953,47 @@
"sdot z29.s, z19.b, z5.b[3]\n"
"sdot z30.s, z19.b, z6.b[3]\n"
"sdot z31.s, z19.b, z7.b[3]\n"
- "cbz %[loops], 2f\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"b.eq 3f\n"
"4:\n"
"st1w z24.s, p7, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"mov z24.s, #0\n"
- "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #4\n"
- "st1w z25.s, p7, [c_ptr1]\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"addvl %[c_ptr0], %[c_ptr0], #1\n"
- "mov z25.s, #0\n"
+ "st1w z25.s, p7, [c_ptr1]\n"
"addvl c_ptr1, c_ptr1, #1\n"
+ "mov z25.s, #0\n"
+ "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"sdot z24.s, z16.b, z0.b[0]\n"
"st1w z26.s, p7, [c_ptr2]\n"
"mov z26.s, #0\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "addvl c_ptr2, c_ptr2, #1\n"
"sdot z25.s, z16.b, z1.b[0]\n"
"st1w z27.s, p7, [c_ptr3]\n"
"mov z27.s, #0\n"
- "addvl c_ptr2, c_ptr2, #1\n"
+ "addvl c_ptr3, c_ptr3, #1\n"
"sdot z26.s, z16.b, z2.b[0]\n"
"st1w z28.s, p7, [c_ptr4]\n"
"mov z28.s, #0\n"
- "addvl c_ptr3, c_ptr3, #1\n"
+ "addvl c_ptr4, c_ptr4, #1\n"
"sdot z27.s, z16.b, z3.b[0]\n"
"st1w z29.s, p7, [c_ptr5]\n"
"mov z29.s, #0\n"
- "addvl c_ptr4, c_ptr4, #1\n"
+ "addvl c_ptr5, c_ptr5, #1\n"
"sdot z28.s, z16.b, z4.b[0]\n"
"st1w z30.s, p7, [c_ptr6]\n"
"mov z30.s, #0\n"
- "addvl c_ptr5, c_ptr5, #1\n"
+ "addvl c_ptr6, c_ptr6, #1\n"
"sdot z29.s, z16.b, z5.b[0]\n"
"st1w z31.s, p7, [c_ptr7]\n"
"mov z31.s, #0\n"
- "addvl c_ptr6, c_ptr6, #1\n"
- "sdot z30.s, z16.b, z6.b[0]\n"
"addvl c_ptr7, c_ptr7, #1\n"
+ "sdot z30.s, z16.b, z6.b[0]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #4\n"
"sdot z31.s, z16.b, z7.b[0]\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
"sdot z24.s, z17.b, z0.b[1]\n"
"prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
"sdot z25.s, z17.b, z1.b[1]\n"
@@ -953,7 +1009,6 @@
"sdot z30.s, z17.b, z6.b[1]\n"
"prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
"sdot z31.s, z17.b, z7.b[1]\n"
- "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"sdot z24.s, z18.b, z0.b[2]\n"
"sdot z25.s, z18.b, z1.b[2]\n"
"sdot z26.s, z18.b, z2.b[2]\n"
@@ -962,7 +1017,6 @@
"sdot z29.s, z18.b, z5.b[2]\n"
"sdot z30.s, z18.b, z6.b[2]\n"
"sdot z31.s, z18.b, z7.b[2]\n"
- "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"sdot z24.s, z19.b, z0.b[3]\n"
"sdot z25.s, z19.b, z1.b[3]\n"
"sdot z26.s, z19.b, z2.b[3]\n"
@@ -976,14 +1030,16 @@
"st1w z24.s, p7, [%[c_ptr0]]\n"
"addvl %[c_ptr0], %[c_ptr0], #1\n"
"mov z24.s, #0\n"
- "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #4\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"st1w z25.s, p7, [c_ptr1]\n"
"addvl c_ptr1, c_ptr1, #1\n"
"mov z25.s, #0\n"
+ "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"sdot z24.s, z16.b, z0.b[0]\n"
"st1w z26.s, p7, [c_ptr2]\n"
"mov z26.s, #0\n"
+ "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"addvl c_ptr2, c_ptr2, #1\n"
"sdot z25.s, z16.b, z1.b[0]\n"
"st1w z27.s, p7, [c_ptr3]\n"
@@ -1006,8 +1062,9 @@
"mov z31.s, #0\n"
"addvl c_ptr7, c_ptr7, #1\n"
"sdot z30.s, z16.b, z6.b[0]\n"
- "sdot z24.s, z17.b, z0.b[1]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #4\n"
"sdot z31.s, z16.b, z7.b[0]\n"
+ "sdot z24.s, z17.b, z0.b[1]\n"
"sdot z25.s, z17.b, z1.b[1]\n"
"sdot z26.s, z17.b, z2.b[1]\n"
"sdot z27.s, z17.b, z3.b[1]\n"
@@ -1031,23 +1088,58 @@
"sdot z29.s, z19.b, z5.b[3]\n"
"sdot z30.s, z19.b, z6.b[3]\n"
"sdot z31.s, z19.b, z7.b[3]\n"
+ "b 5f\n"
"2:\n"
+ "mov z24.s, #0\n"
+ "mov z25.s, #0\n"
+ "mov z26.s, #0\n"
+ "mov z27.s, #0\n"
+ "mov z28.s, #0\n"
+ "mov z29.s, #0\n"
+ "mov z30.s, #0\n"
+ "mov z31.s, #0\n"
+ "sdot z24.s, z16.b, z0.b[0]\n"
+ "sdot z25.s, z16.b, z1.b[0]\n"
+ "sdot z26.s, z16.b, z2.b[0]\n"
+ "sdot z27.s, z16.b, z3.b[0]\n"
+ "sdot z28.s, z16.b, z4.b[0]\n"
+ "sdot z29.s, z16.b, z5.b[0]\n"
+ "sdot z30.s, z16.b, z6.b[0]\n"
+ "sdot z31.s, z16.b, z7.b[0]\n"
+ "sdot z24.s, z17.b, z0.b[1]\n"
+ "sdot z25.s, z17.b, z1.b[1]\n"
+ "sdot z26.s, z17.b, z2.b[1]\n"
+ "sdot z27.s, z17.b, z3.b[1]\n"
+ "sdot z28.s, z17.b, z4.b[1]\n"
+ "sdot z29.s, z17.b, z5.b[1]\n"
+ "sdot z30.s, z17.b, z6.b[1]\n"
+ "sdot z31.s, z17.b, z7.b[1]\n"
+ "sdot z24.s, z18.b, z0.b[2]\n"
+ "sdot z25.s, z18.b, z1.b[2]\n"
+ "sdot z26.s, z18.b, z2.b[2]\n"
+ "sdot z27.s, z18.b, z3.b[2]\n"
+ "sdot z28.s, z18.b, z4.b[2]\n"
+ "sdot z29.s, z18.b, z5.b[2]\n"
+ "sdot z30.s, z18.b, z6.b[2]\n"
+ "sdot z31.s, z18.b, z7.b[2]\n"
+ "sdot z24.s, z19.b, z0.b[3]\n"
+ "sdot z25.s, z19.b, z1.b[3]\n"
+ "sdot z26.s, z19.b, z2.b[3]\n"
+ "sdot z27.s, z19.b, z3.b[3]\n"
+ "sdot z28.s, z19.b, z4.b[3]\n"
+ "sdot z29.s, z19.b, z5.b[3]\n"
+ "sdot z30.s, z19.b, z6.b[3]\n"
+ "sdot z31.s, z19.b, z7.b[3]\n"
+ "5:\n"
"st1w z24.s, p0, [%[c_ptr0]]\n"
"addvl %[c_ptr0], %[c_ptr0], #1\n"
"st1w z25.s, p0, [c_ptr1]\n"
- "addvl c_ptr1, c_ptr1, #1\n"
"st1w z26.s, p0, [c_ptr2]\n"
- "addvl c_ptr2, c_ptr2, #1\n"
"st1w z27.s, p0, [c_ptr3]\n"
- "addvl c_ptr3, c_ptr3, #1\n"
"st1w z28.s, p0, [c_ptr4]\n"
- "addvl c_ptr4, c_ptr4, #1\n"
"st1w z29.s, p0, [c_ptr5]\n"
- "addvl c_ptr5, c_ptr5, #1\n"
"st1w z30.s, p0, [c_ptr6]\n"
- "addvl c_ptr6, c_ptr6, #1\n"
"st1w z31.s, p0, [c_ptr7]\n"
- "addvl c_ptr7, c_ptr7, #1\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -1126,46 +1218,48 @@
"add c_ptr1, %[c_ptr0], #0x0\n"
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
- "mov z24.s, #0\n"
"ptrue p7.b\n"
- "mov z25.s, #0\n"
"whilelt p6.b, %[temp], %[odd_depth]\n"
- "mov z26.s, #0\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
- "mov z27.s, #0\n"
- "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "mov z28.s, #0\n"
- "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "mov z29.s, #0\n"
- "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "mov z30.s, #0\n"
- "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "mov z31.s, #0\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "ld1rqb z1.b, p7/z, [a_ptr1]\n"
"whilelt p0.s, %[temp], %[last_width]\n"
- "ld1rqb z2.b, p7/z, [a_ptr2]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
- "sdot z24.s, z16.b, z0.b[0]\n"
- "ld1rqb z3.b, p7/z, [a_ptr3]\n"
- "sdot z25.s, z16.b, z1.b[0]\n"
- "ld1rqb z4.b, p7/z, [a_ptr4]\n"
- "sdot z26.s, z16.b, z2.b[0]\n"
- "ld1rqb z5.b, p7/z, [a_ptr5]\n"
- "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
- "sdot z27.s, z16.b, z3.b[0]\n"
- "ld1rqb z7.b, p7/z, [a_ptr7]\n"
- "sdot z28.s, z16.b, z4.b[0]\n"
+ "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
- "sdot z29.s, z16.b, z5.b[0]\n"
+ "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
- "sdot z30.s, z16.b, z6.b[0]\n"
+ "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- "sdot z31.s, z16.b, z7.b[0]\n"
+ "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- "sdot z24.s, z17.b, z0.b[1]\n"
"addvl %[b_ptr0], %[b_ptr0], #5\n"
+ "cbz %[loops], 2f\n"
+ "mov z24.s, #0\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "mov z25.s, #0\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "mov z26.s, #0\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2]\n"
+ "mov z27.s, #0\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "mov z28.s, #0\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "mov z29.s, #0\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "mov z30.s, #0\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "mov z31.s, #0\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "sdot z24.s, z16.b, z0.b[0]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "sdot z25.s, z16.b, z1.b[0]\n"
+ "sdot z26.s, z16.b, z2.b[0]\n"
+ "sdot z27.s, z16.b, z3.b[0]\n"
+ "sdot z28.s, z16.b, z4.b[0]\n"
+ "sdot z29.s, z16.b, z5.b[0]\n"
+ "sdot z30.s, z16.b, z6.b[0]\n"
+ "sdot z31.s, z16.b, z7.b[0]\n"
+ "sdot z24.s, z17.b, z0.b[1]\n"
"sdot z25.s, z17.b, z1.b[1]\n"
"sdot z26.s, z17.b, z2.b[1]\n"
"sdot z27.s, z17.b, z3.b[1]\n"
@@ -1205,84 +1299,79 @@
"sdot z29.s, z20.b, z5.b[0]\n"
"sdot z30.s, z20.b, z6.b[0]\n"
"sdot z31.s, z20.b, z7.b[0]\n"
- "cbz %[loops], 2f\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"b.eq 3f\n"
"4:\n"
"st1w z24.s, p7, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"mov z24.s, #0\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "addvl %[c_ptr0], %[c_ptr0], #1\n"
+ "st1w z25.s, p7, [c_ptr1]\n"
+ "addvl c_ptr1, c_ptr1, #1\n"
+ "mov z25.s, #0\n"
+ "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "st1w z26.s, p7, [c_ptr2]\n"
+ "addvl c_ptr2, c_ptr2, #1\n"
+ "mov z26.s, #0\n"
"ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
"ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
"addvl %[b_ptr0], %[b_ptr0], #5\n"
- "st1w z25.s, p7, [c_ptr1]\n"
- "addvl %[c_ptr0], %[c_ptr0], #1\n"
- "mov z25.s, #0\n"
+ "st1w z27.s, p7, [c_ptr3]\n"
+ "addvl c_ptr3, c_ptr3, #1\n"
+ "mov z27.s, #0\n"
"ld1rqb z1.b, p7/z, [a_ptr1]\n"
"sdot z24.s, z16.b, z0.b[0]\n"
- "st1w z26.s, p7, [c_ptr2]\n"
- "mov z26.s, #0\n"
+ "st1w z28.s, p7, [c_ptr4]\n"
+ "mov z28.s, #0\n"
"ld1rqb z2.b, p7/z, [a_ptr2]\n"
"ld1rqb z3.b, p7/z, [a_ptr3]\n"
- "addvl c_ptr1, c_ptr1, #1\n"
+ "addvl c_ptr4, c_ptr4, #1\n"
"sdot z25.s, z16.b, z1.b[0]\n"
- "st1w z27.s, p7, [c_ptr3]\n"
- "mov z27.s, #0\n"
+ "st1w z29.s, p7, [c_ptr5]\n"
+ "mov z29.s, #0\n"
"ld1rqb z4.b, p7/z, [a_ptr4]\n"
"sdot z26.s, z16.b, z2.b[0]\n"
"ld1rqb z5.b, p7/z, [a_ptr5]\n"
- "sdot z24.s, z17.b, z0.b[1]\n"
- "st1w z28.s, p7, [c_ptr4]\n"
- "mov z28.s, #0\n"
- "ld1rqb z6.b, p7/z, [a_ptr6]\n"
"sdot z27.s, z16.b, z3.b[0]\n"
- "ld1rqb z7.b, p7/z, [a_ptr7]\n"
- "sdot z25.s, z17.b, z1.b[1]\n"
- "st1w z29.s, p7, [c_ptr5]\n"
- "mov z29.s, #0\n"
- "addvl c_ptr2, c_ptr2, #1\n"
- "sdot z28.s, z16.b, z4.b[0]\n"
"st1w z30.s, p7, [c_ptr6]\n"
"mov z30.s, #0\n"
- "addvl c_ptr3, c_ptr3, #1\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "sdot z28.s, z16.b, z4.b[0]\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
"sdot z29.s, z16.b, z5.b[0]\n"
"st1w z31.s, p7, [c_ptr7]\n"
"mov z31.s, #0\n"
- "addvl c_ptr4, c_ptr4, #1\n"
- "sdot z30.s, z16.b, z6.b[0]\n"
"addvl c_ptr5, c_ptr5, #1\n"
- "sdot z31.s, z16.b, z7.b[0]\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
- "sdot z26.s, z17.b, z2.b[1]\n"
+ "sdot z30.s, z16.b, z6.b[0]\n"
"addvl c_ptr6, c_ptr6, #1\n"
- "sdot z27.s, z17.b, z3.b[1]\n"
+ "sdot z31.s, z16.b, z7.b[0]\n"
"addvl c_ptr7, c_ptr7, #1\n"
- "sdot z28.s, z17.b, z4.b[1]\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
- "sdot z29.s, z17.b, z5.b[1]\n"
+ "sdot z24.s, z17.b, z0.b[1]\n"
"prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- "sdot z30.s, z17.b, z6.b[1]\n"
+ "sdot z25.s, z17.b, z1.b[1]\n"
"prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- "sdot z31.s, z17.b, z7.b[1]\n"
- "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "sdot z24.s, z18.b, z0.b[2]\n"
+ "sdot z26.s, z17.b, z2.b[1]\n"
"prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- "sdot z25.s, z18.b, z1.b[2]\n"
+ "sdot z27.s, z17.b, z3.b[1]\n"
"prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- "sdot z26.s, z18.b, z2.b[2]\n"
+ "sdot z28.s, z17.b, z4.b[1]\n"
"prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- "sdot z27.s, z18.b, z3.b[2]\n"
+ "sdot z29.s, z17.b, z5.b[1]\n"
"prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
- "sdot z28.s, z18.b, z4.b[2]\n"
+ "sdot z30.s, z17.b, z6.b[1]\n"
"prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
+ "sdot z31.s, z17.b, z7.b[1]\n"
+ "sdot z24.s, z18.b, z0.b[2]\n"
+ "sdot z25.s, z18.b, z1.b[2]\n"
+ "sdot z26.s, z18.b, z2.b[2]\n"
+ "sdot z27.s, z18.b, z3.b[2]\n"
+ "sdot z28.s, z18.b, z4.b[2]\n"
"sdot z29.s, z18.b, z5.b[2]\n"
"sdot z30.s, z18.b, z6.b[2]\n"
"sdot z31.s, z18.b, z7.b[2]\n"
- "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"sdot z24.s, z19.b, z0.b[3]\n"
"ld1rqb z0.b, p6/z, [%[a_ptr0], #0x10]\n"
"sdot z25.s, z19.b, z1.b[3]\n"
@@ -1300,7 +1389,6 @@
"sdot z31.s, z19.b, z7.b[3]\n"
"ld1rqb z7.b, p6/z, [a_ptr7, #0x10]\n"
"sdot z24.s, z20.b, z0.b[0]\n"
- "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"sdot z25.s, z20.b, z1.b[0]\n"
"sdot z26.s, z20.b, z2.b[0]\n"
"sdot z27.s, z20.b, z3.b[0]\n"
@@ -1313,39 +1401,41 @@
"st1w z24.s, p7, [%[c_ptr0]]\n"
"addvl %[c_ptr0], %[c_ptr0], #1\n"
"mov z24.s, #0\n"
- "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "addvl %[b_ptr0], %[b_ptr0], #5\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"st1w z25.s, p7, [c_ptr1]\n"
"addvl c_ptr1, c_ptr1, #1\n"
"mov z25.s, #0\n"
+ "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "st1w z26.s, p7, [c_ptr2]\n"
+ "addvl c_ptr2, c_ptr2, #1\n"
+ "mov z26.s, #0\n"
+ "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #5\n"
+ "st1w z27.s, p7, [c_ptr3]\n"
+ "addvl c_ptr3, c_ptr3, #1\n"
+ "mov z27.s, #0\n"
"ld1rqb z1.b, p7/z, [a_ptr1]\n"
"sdot z24.s, z16.b, z0.b[0]\n"
- "st1w z26.s, p7, [c_ptr2]\n"
- "mov z26.s, #0\n"
+ "st1w z28.s, p7, [c_ptr4]\n"
+ "mov z28.s, #0\n"
"ld1rqb z2.b, p7/z, [a_ptr2]\n"
"ld1rqb z3.b, p7/z, [a_ptr3]\n"
- "addvl c_ptr2, c_ptr2, #1\n"
+ "addvl c_ptr4, c_ptr4, #1\n"
"sdot z25.s, z16.b, z1.b[0]\n"
- "st1w z27.s, p7, [c_ptr3]\n"
- "mov z27.s, #0\n"
+ "st1w z29.s, p7, [c_ptr5]\n"
+ "mov z29.s, #0\n"
"ld1rqb z4.b, p7/z, [a_ptr4]\n"
"sdot z26.s, z16.b, z2.b[0]\n"
"ld1rqb z5.b, p7/z, [a_ptr5]\n"
- "sdot z24.s, z17.b, z0.b[1]\n"
- "st1w z28.s, p7, [c_ptr4]\n"
- "mov z28.s, #0\n"
- "ld1rqb z6.b, p7/z, [a_ptr6]\n"
"sdot z27.s, z16.b, z3.b[0]\n"
- "ld1rqb z7.b, p7/z, [a_ptr7]\n"
- "sdot z25.s, z17.b, z1.b[1]\n"
- "st1w z29.s, p7, [c_ptr5]\n"
- "mov z29.s, #0\n"
- "addvl c_ptr3, c_ptr3, #1\n"
- "sdot z28.s, z16.b, z4.b[0]\n"
"st1w z30.s, p7, [c_ptr6]\n"
"mov z30.s, #0\n"
- "addvl c_ptr4, c_ptr4, #1\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "sdot z28.s, z16.b, z4.b[0]\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
"sdot z29.s, z16.b, z5.b[0]\n"
"st1w z31.s, p7, [c_ptr7]\n"
"mov z31.s, #0\n"
@@ -1354,6 +1444,8 @@
"addvl c_ptr6, c_ptr6, #1\n"
"sdot z31.s, z16.b, z7.b[0]\n"
"addvl c_ptr7, c_ptr7, #1\n"
+ "sdot z24.s, z17.b, z0.b[1]\n"
+ "sdot z25.s, z17.b, z1.b[1]\n"
"sdot z26.s, z17.b, z2.b[1]\n"
"sdot z27.s, z17.b, z3.b[1]\n"
"sdot z28.s, z17.b, z4.b[1]\n"
@@ -1392,23 +1484,82 @@
"sdot z29.s, z20.b, z5.b[0]\n"
"sdot z30.s, z20.b, z6.b[0]\n"
"sdot z31.s, z20.b, z7.b[0]\n"
+ "b 5f\n"
"2:\n"
+ "mov z24.s, #0\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "mov z25.s, #0\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "mov z26.s, #0\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2]\n"
+ "mov z27.s, #0\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "mov z28.s, #0\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "mov z29.s, #0\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "mov z30.s, #0\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "mov z31.s, #0\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "sdot z24.s, z16.b, z0.b[0]\n"
+ "sdot z25.s, z16.b, z1.b[0]\n"
+ "sdot z26.s, z16.b, z2.b[0]\n"
+ "sdot z27.s, z16.b, z3.b[0]\n"
+ "sdot z28.s, z16.b, z4.b[0]\n"
+ "sdot z29.s, z16.b, z5.b[0]\n"
+ "sdot z30.s, z16.b, z6.b[0]\n"
+ "sdot z31.s, z16.b, z7.b[0]\n"
+ "sdot z24.s, z17.b, z0.b[1]\n"
+ "sdot z25.s, z17.b, z1.b[1]\n"
+ "sdot z26.s, z17.b, z2.b[1]\n"
+ "sdot z27.s, z17.b, z3.b[1]\n"
+ "sdot z28.s, z17.b, z4.b[1]\n"
+ "sdot z29.s, z17.b, z5.b[1]\n"
+ "sdot z30.s, z17.b, z6.b[1]\n"
+ "sdot z31.s, z17.b, z7.b[1]\n"
+ "sdot z24.s, z18.b, z0.b[2]\n"
+ "sdot z25.s, z18.b, z1.b[2]\n"
+ "sdot z26.s, z18.b, z2.b[2]\n"
+ "sdot z27.s, z18.b, z3.b[2]\n"
+ "sdot z28.s, z18.b, z4.b[2]\n"
+ "sdot z29.s, z18.b, z5.b[2]\n"
+ "sdot z30.s, z18.b, z6.b[2]\n"
+ "sdot z31.s, z18.b, z7.b[2]\n"
+ "sdot z24.s, z19.b, z0.b[3]\n"
+ "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x10]\n"
+ "sdot z25.s, z19.b, z1.b[3]\n"
+ "ld1rqb z1.b, p6/z, [a_ptr1, #0x10]\n"
+ "sdot z26.s, z19.b, z2.b[3]\n"
+ "ld1rqb z2.b, p6/z, [a_ptr2, #0x10]\n"
+ "sdot z27.s, z19.b, z3.b[3]\n"
+ "ld1rqb z3.b, p6/z, [a_ptr3, #0x10]\n"
+ "sdot z28.s, z19.b, z4.b[3]\n"
+ "ld1rqb z4.b, p6/z, [a_ptr4, #0x10]\n"
+ "sdot z29.s, z19.b, z5.b[3]\n"
+ "ld1rqb z5.b, p6/z, [a_ptr5, #0x10]\n"
+ "sdot z30.s, z19.b, z6.b[3]\n"
+ "ld1rqb z6.b, p6/z, [a_ptr6, #0x10]\n"
+ "sdot z31.s, z19.b, z7.b[3]\n"
+ "ld1rqb z7.b, p6/z, [a_ptr7, #0x10]\n"
+ "sdot z24.s, z20.b, z0.b[0]\n"
+ "sdot z25.s, z20.b, z1.b[0]\n"
+ "sdot z26.s, z20.b, z2.b[0]\n"
+ "sdot z27.s, z20.b, z3.b[0]\n"
+ "sdot z28.s, z20.b, z4.b[0]\n"
+ "sdot z29.s, z20.b, z5.b[0]\n"
+ "sdot z30.s, z20.b, z6.b[0]\n"
+ "sdot z31.s, z20.b, z7.b[0]\n"
+ "5:\n"
"st1w z24.s, p0, [%[c_ptr0]]\n"
"addvl %[c_ptr0], %[c_ptr0], #1\n"
"st1w z25.s, p0, [c_ptr1]\n"
- "addvl c_ptr1, c_ptr1, #1\n"
"st1w z26.s, p0, [c_ptr2]\n"
- "addvl c_ptr2, c_ptr2, #1\n"
"st1w z27.s, p0, [c_ptr3]\n"
- "addvl c_ptr3, c_ptr3, #1\n"
"st1w z28.s, p0, [c_ptr4]\n"
- "addvl c_ptr4, c_ptr4, #1\n"
"st1w z29.s, p0, [c_ptr5]\n"
- "addvl c_ptr5, c_ptr5, #1\n"
"st1w z30.s, p0, [c_ptr6]\n"
- "addvl c_ptr6, c_ptr6, #1\n"
"st1w z31.s, p0, [c_ptr7]\n"
- "addvl c_ptr7, c_ptr7, #1\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -1487,48 +1638,50 @@
"add c_ptr1, %[c_ptr0], #0x0\n"
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
- "mov z24.s, #0\n"
"ptrue p7.b\n"
- "mov z25.s, #0\n"
"whilelt p6.b, %[temp], %[odd_depth]\n"
- "mov z26.s, #0\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
- "mov z27.s, #0\n"
- "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "mov z28.s, #0\n"
- "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "mov z29.s, #0\n"
- "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "mov z30.s, #0\n"
- "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "mov z31.s, #0\n"
- "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
"whilelt p0.s, %[temp], %[last_width]\n"
- "ld1rqb z1.b, p7/z, [a_ptr1]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
- "sdot z24.s, z16.b, z0.b[0]\n"
- "ld1rqb z2.b, p7/z, [a_ptr2]\n"
- "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
- "sdot z25.s, z16.b, z1.b[0]\n"
- "ld1rqb z4.b, p7/z, [a_ptr4]\n"
- "sdot z26.s, z16.b, z2.b[0]\n"
- "ld1rqb z5.b, p7/z, [a_ptr5]\n"
- "sdot z27.s, z16.b, z3.b[0]\n"
- "ld1rqb z6.b, p7/z, [a_ptr6]\n"
- "sdot z24.s, z17.b, z0.b[1]\n"
- "ld1rqb z7.b, p7/z, [a_ptr7]\n"
- "sdot z28.s, z16.b, z4.b[0]\n"
+ "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
- "sdot z29.s, z16.b, z5.b[0]\n"
+ "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
- "sdot z30.s, z16.b, z6.b[0]\n"
+ "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- "sdot z31.s, z16.b, z7.b[0]\n"
+ "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- "sdot z25.s, z17.b, z1.b[1]\n"
+ "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
"addvl %[b_ptr0], %[b_ptr0], #6\n"
+ "cbz %[loops], 2f\n"
+ "mov z24.s, #0\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "mov z25.s, #0\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "mov z26.s, #0\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2]\n"
+ "mov z27.s, #0\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "mov z28.s, #0\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "mov z29.s, #0\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "mov z30.s, #0\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "mov z31.s, #0\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "sdot z24.s, z16.b, z0.b[0]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "sdot z25.s, z16.b, z1.b[0]\n"
+ "sdot z26.s, z16.b, z2.b[0]\n"
+ "sdot z27.s, z16.b, z3.b[0]\n"
+ "sdot z28.s, z16.b, z4.b[0]\n"
+ "sdot z29.s, z16.b, z5.b[0]\n"
+ "sdot z30.s, z16.b, z6.b[0]\n"
+ "sdot z31.s, z16.b, z7.b[0]\n"
+ "sdot z24.s, z17.b, z0.b[1]\n"
+ "sdot z25.s, z17.b, z1.b[1]\n"
"sdot z26.s, z17.b, z2.b[1]\n"
"sdot z27.s, z17.b, z3.b[1]\n"
"sdot z28.s, z17.b, z4.b[1]\n"
@@ -1575,85 +1728,80 @@
"sdot z29.s, z21.b, z5.b[1]\n"
"sdot z30.s, z21.b, z6.b[1]\n"
"sdot z31.s, z21.b, z7.b[1]\n"
- "cbz %[loops], 2f\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
"b.eq 3f\n"
"4:\n"
"st1w z24.s, p7, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"mov z24.s, #0\n"
- "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "addvl %[b_ptr0], %[b_ptr0], #6\n"
- "st1w z25.s, p7, [c_ptr1]\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"addvl %[c_ptr0], %[c_ptr0], #1\n"
- "mov z25.s, #0\n"
- "ld1rqb z1.b, p7/z, [a_ptr1]\n"
- "sdot z24.s, z16.b, z0.b[0]\n"
- "st1w z26.s, p7, [c_ptr2]\n"
- "mov z26.s, #0\n"
- "ld1rqb z2.b, p7/z, [a_ptr2]\n"
- "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "st1w z25.s, p7, [c_ptr1]\n"
"addvl c_ptr1, c_ptr1, #1\n"
- "sdot z25.s, z16.b, z1.b[0]\n"
+ "mov z25.s, #0\n"
+ "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "st1w z26.s, p7, [c_ptr2]\n"
+ "addvl c_ptr2, c_ptr2, #1\n"
+ "mov z26.s, #0\n"
+ "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #6\n"
"st1w z27.s, p7, [c_ptr3]\n"
+ "addvl c_ptr3, c_ptr3, #1\n"
"mov z27.s, #0\n"
- "ld1rqb z4.b, p7/z, [a_ptr4]\n"
- "sdot z26.s, z16.b, z2.b[0]\n"
- "ld1rqb z5.b, p7/z, [a_ptr5]\n"
- "sdot z24.s, z17.b, z0.b[1]\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
"st1w z28.s, p7, [c_ptr4]\n"
+ "addvl c_ptr4, c_ptr4, #1\n"
"mov z28.s, #0\n"
- "ld1rqb z6.b, p7/z, [a_ptr6]\n"
- "sdot z27.s, z16.b, z3.b[0]\n"
- "ld1rqb z7.b, p7/z, [a_ptr7]\n"
- "sdot z25.s, z17.b, z1.b[1]\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2]\n"
+ "sdot z24.s, z16.b, z0.b[0]\n"
"st1w z29.s, p7, [c_ptr5]\n"
"mov z29.s, #0\n"
- "addvl c_ptr2, c_ptr2, #1\n"
- "sdot z28.s, z16.b, z4.b[0]\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "sdot z25.s, z16.b, z1.b[0]\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "sdot z26.s, z16.b, z2.b[0]\n"
"st1w z30.s, p7, [c_ptr6]\n"
"mov z30.s, #0\n"
- "addvl c_ptr3, c_ptr3, #1\n"
- "sdot z29.s, z16.b, z5.b[0]\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "sdot z27.s, z16.b, z3.b[0]\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "sdot z28.s, z16.b, z4.b[0]\n"
"st1w z31.s, p7, [c_ptr7]\n"
"mov z31.s, #0\n"
- "addvl c_ptr4, c_ptr4, #1\n"
- "sdot z30.s, z16.b, z6.b[0]\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "sdot z29.s, z16.b, z5.b[0]\n"
"addvl c_ptr5, c_ptr5, #1\n"
- "sdot z31.s, z16.b, z7.b[0]\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
- "sdot z26.s, z17.b, z2.b[1]\n"
+ "sdot z30.s, z16.b, z6.b[0]\n"
"addvl c_ptr6, c_ptr6, #1\n"
- "sdot z27.s, z17.b, z3.b[1]\n"
+ "sdot z31.s, z16.b, z7.b[0]\n"
"addvl c_ptr7, c_ptr7, #1\n"
- "sdot z28.s, z17.b, z4.b[1]\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
- "sdot z29.s, z17.b, z5.b[1]\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- "sdot z30.s, z17.b, z6.b[1]\n"
+ "sdot z24.s, z17.b, z0.b[1]\n"
"prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- "sdot z31.s, z17.b, z7.b[1]\n"
- "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "sdot z24.s, z18.b, z0.b[2]\n"
+ "sdot z25.s, z17.b, z1.b[1]\n"
"prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- "sdot z25.s, z18.b, z1.b[2]\n"
+ "sdot z26.s, z17.b, z2.b[1]\n"
"prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- "sdot z26.s, z18.b, z2.b[2]\n"
+ "sdot z27.s, z17.b, z3.b[1]\n"
"prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- "sdot z27.s, z18.b, z3.b[2]\n"
+ "sdot z28.s, z17.b, z4.b[1]\n"
"prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
- "sdot z28.s, z18.b, z4.b[2]\n"
+ "sdot z29.s, z17.b, z5.b[1]\n"
"prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
+ "sdot z30.s, z17.b, z6.b[1]\n"
+ "sdot z31.s, z17.b, z7.b[1]\n"
+ "sdot z24.s, z18.b, z0.b[2]\n"
+ "sdot z25.s, z18.b, z1.b[2]\n"
+ "sdot z26.s, z18.b, z2.b[2]\n"
+ "sdot z27.s, z18.b, z3.b[2]\n"
+ "sdot z28.s, z18.b, z4.b[2]\n"
"sdot z29.s, z18.b, z5.b[2]\n"
"sdot z30.s, z18.b, z6.b[2]\n"
"sdot z31.s, z18.b, z7.b[2]\n"
- "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"sdot z24.s, z19.b, z0.b[3]\n"
"ld1rqb z0.b, p6/z, [%[a_ptr0], #0x10]\n"
"sdot z25.s, z19.b, z1.b[3]\n"
@@ -1671,7 +1819,6 @@
"sdot z31.s, z19.b, z7.b[3]\n"
"ld1rqb z7.b, p6/z, [a_ptr7, #0x10]\n"
"sdot z24.s, z20.b, z0.b[0]\n"
- "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"sdot z25.s, z20.b, z1.b[0]\n"
"sdot z26.s, z20.b, z2.b[0]\n"
"sdot z27.s, z20.b, z3.b[0]\n"
@@ -1679,7 +1826,6 @@
"sdot z29.s, z20.b, z5.b[0]\n"
"sdot z30.s, z20.b, z6.b[0]\n"
"sdot z31.s, z20.b, z7.b[0]\n"
- "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
"sdot z24.s, z21.b, z0.b[1]\n"
"sdot z25.s, z21.b, z1.b[1]\n"
"sdot z26.s, z21.b, z2.b[1]\n"
@@ -1693,47 +1839,52 @@
"st1w z24.s, p7, [%[c_ptr0]]\n"
"addvl %[c_ptr0], %[c_ptr0], #1\n"
"mov z24.s, #0\n"
- "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "addvl %[b_ptr0], %[b_ptr0], #6\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"st1w z25.s, p7, [c_ptr1]\n"
"addvl c_ptr1, c_ptr1, #1\n"
"mov z25.s, #0\n"
- "ld1rqb z1.b, p7/z, [a_ptr1]\n"
- "sdot z24.s, z16.b, z0.b[0]\n"
+ "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"st1w z26.s, p7, [c_ptr2]\n"
- "mov z26.s, #0\n"
- "ld1rqb z2.b, p7/z, [a_ptr2]\n"
- "ld1rqb z3.b, p7/z, [a_ptr3]\n"
"addvl c_ptr2, c_ptr2, #1\n"
- "sdot z25.s, z16.b, z1.b[0]\n"
+ "mov z26.s, #0\n"
+ "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #6\n"
"st1w z27.s, p7, [c_ptr3]\n"
+ "addvl c_ptr3, c_ptr3, #1\n"
"mov z27.s, #0\n"
- "ld1rqb z4.b, p7/z, [a_ptr4]\n"
- "sdot z26.s, z16.b, z2.b[0]\n"
- "ld1rqb z5.b, p7/z, [a_ptr5]\n"
- "sdot z24.s, z17.b, z0.b[1]\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
"st1w z28.s, p7, [c_ptr4]\n"
+ "addvl c_ptr4, c_ptr4, #1\n"
"mov z28.s, #0\n"
- "ld1rqb z6.b, p7/z, [a_ptr6]\n"
- "sdot z27.s, z16.b, z3.b[0]\n"
- "ld1rqb z7.b, p7/z, [a_ptr7]\n"
- "sdot z25.s, z17.b, z1.b[1]\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2]\n"
+ "sdot z24.s, z16.b, z0.b[0]\n"
"st1w z29.s, p7, [c_ptr5]\n"
"mov z29.s, #0\n"
- "addvl c_ptr3, c_ptr3, #1\n"
- "sdot z28.s, z16.b, z4.b[0]\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "sdot z25.s, z16.b, z1.b[0]\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "sdot z26.s, z16.b, z2.b[0]\n"
"st1w z30.s, p7, [c_ptr6]\n"
"mov z30.s, #0\n"
- "addvl c_ptr4, c_ptr4, #1\n"
- "sdot z29.s, z16.b, z5.b[0]\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "sdot z27.s, z16.b, z3.b[0]\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "sdot z28.s, z16.b, z4.b[0]\n"
"st1w z31.s, p7, [c_ptr7]\n"
"mov z31.s, #0\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "sdot z29.s, z16.b, z5.b[0]\n"
"addvl c_ptr5, c_ptr5, #1\n"
"sdot z30.s, z16.b, z6.b[0]\n"
"addvl c_ptr6, c_ptr6, #1\n"
"sdot z31.s, z16.b, z7.b[0]\n"
"addvl c_ptr7, c_ptr7, #1\n"
+ "sdot z24.s, z17.b, z0.b[1]\n"
+ "sdot z25.s, z17.b, z1.b[1]\n"
"sdot z26.s, z17.b, z2.b[1]\n"
"sdot z27.s, z17.b, z3.b[1]\n"
"sdot z28.s, z17.b, z4.b[1]\n"
@@ -1780,23 +1931,90 @@
"sdot z29.s, z21.b, z5.b[1]\n"
"sdot z30.s, z21.b, z6.b[1]\n"
"sdot z31.s, z21.b, z7.b[1]\n"
+ "b 5f\n"
"2:\n"
+ "mov z24.s, #0\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "mov z25.s, #0\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "mov z26.s, #0\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2]\n"
+ "mov z27.s, #0\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "mov z28.s, #0\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "mov z29.s, #0\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "mov z30.s, #0\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "mov z31.s, #0\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "sdot z24.s, z16.b, z0.b[0]\n"
+ "sdot z25.s, z16.b, z1.b[0]\n"
+ "sdot z26.s, z16.b, z2.b[0]\n"
+ "sdot z27.s, z16.b, z3.b[0]\n"
+ "sdot z28.s, z16.b, z4.b[0]\n"
+ "sdot z29.s, z16.b, z5.b[0]\n"
+ "sdot z30.s, z16.b, z6.b[0]\n"
+ "sdot z31.s, z16.b, z7.b[0]\n"
+ "sdot z24.s, z17.b, z0.b[1]\n"
+ "sdot z25.s, z17.b, z1.b[1]\n"
+ "sdot z26.s, z17.b, z2.b[1]\n"
+ "sdot z27.s, z17.b, z3.b[1]\n"
+ "sdot z28.s, z17.b, z4.b[1]\n"
+ "sdot z29.s, z17.b, z5.b[1]\n"
+ "sdot z30.s, z17.b, z6.b[1]\n"
+ "sdot z31.s, z17.b, z7.b[1]\n"
+ "sdot z24.s, z18.b, z0.b[2]\n"
+ "sdot z25.s, z18.b, z1.b[2]\n"
+ "sdot z26.s, z18.b, z2.b[2]\n"
+ "sdot z27.s, z18.b, z3.b[2]\n"
+ "sdot z28.s, z18.b, z4.b[2]\n"
+ "sdot z29.s, z18.b, z5.b[2]\n"
+ "sdot z30.s, z18.b, z6.b[2]\n"
+ "sdot z31.s, z18.b, z7.b[2]\n"
+ "sdot z24.s, z19.b, z0.b[3]\n"
+ "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x10]\n"
+ "sdot z25.s, z19.b, z1.b[3]\n"
+ "ld1rqb z1.b, p6/z, [a_ptr1, #0x10]\n"
+ "sdot z26.s, z19.b, z2.b[3]\n"
+ "ld1rqb z2.b, p6/z, [a_ptr2, #0x10]\n"
+ "sdot z27.s, z19.b, z3.b[3]\n"
+ "ld1rqb z3.b, p6/z, [a_ptr3, #0x10]\n"
+ "sdot z28.s, z19.b, z4.b[3]\n"
+ "ld1rqb z4.b, p6/z, [a_ptr4, #0x10]\n"
+ "sdot z29.s, z19.b, z5.b[3]\n"
+ "ld1rqb z5.b, p6/z, [a_ptr5, #0x10]\n"
+ "sdot z30.s, z19.b, z6.b[3]\n"
+ "ld1rqb z6.b, p6/z, [a_ptr6, #0x10]\n"
+ "sdot z31.s, z19.b, z7.b[3]\n"
+ "ld1rqb z7.b, p6/z, [a_ptr7, #0x10]\n"
+ "sdot z24.s, z20.b, z0.b[0]\n"
+ "sdot z25.s, z20.b, z1.b[0]\n"
+ "sdot z26.s, z20.b, z2.b[0]\n"
+ "sdot z27.s, z20.b, z3.b[0]\n"
+ "sdot z28.s, z20.b, z4.b[0]\n"
+ "sdot z29.s, z20.b, z5.b[0]\n"
+ "sdot z30.s, z20.b, z6.b[0]\n"
+ "sdot z31.s, z20.b, z7.b[0]\n"
+ "sdot z24.s, z21.b, z0.b[1]\n"
+ "sdot z25.s, z21.b, z1.b[1]\n"
+ "sdot z26.s, z21.b, z2.b[1]\n"
+ "sdot z27.s, z21.b, z3.b[1]\n"
+ "sdot z28.s, z21.b, z4.b[1]\n"
+ "sdot z29.s, z21.b, z5.b[1]\n"
+ "sdot z30.s, z21.b, z6.b[1]\n"
+ "sdot z31.s, z21.b, z7.b[1]\n"
+ "5:\n"
"st1w z24.s, p0, [%[c_ptr0]]\n"
"addvl %[c_ptr0], %[c_ptr0], #1\n"
"st1w z25.s, p0, [c_ptr1]\n"
- "addvl c_ptr1, c_ptr1, #1\n"
"st1w z26.s, p0, [c_ptr2]\n"
- "addvl c_ptr2, c_ptr2, #1\n"
"st1w z27.s, p0, [c_ptr3]\n"
- "addvl c_ptr3, c_ptr3, #1\n"
"st1w z28.s, p0, [c_ptr4]\n"
- "addvl c_ptr4, c_ptr4, #1\n"
"st1w z29.s, p0, [c_ptr5]\n"
- "addvl c_ptr5, c_ptr5, #1\n"
"st1w z30.s, p0, [c_ptr6]\n"
- "addvl c_ptr6, c_ptr6, #1\n"
"st1w z31.s, p0, [c_ptr7]\n"
- "addvl c_ptr7, c_ptr7, #1\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -1875,48 +2093,50 @@
"add c_ptr1, %[c_ptr0], #0x0\n"
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
- "mov z24.s, #0\n"
"ptrue p7.b\n"
- "mov z25.s, #0\n"
"whilelt p6.b, %[temp], %[odd_depth]\n"
- "mov z26.s, #0\n"
+ "whilelt p0.s, %[temp], %[last_width]\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
"ld1b z16.b, p7/z, [%[b_ptr0]]\n"
- "mov z27.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
"ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "mov z28.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
"ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "mov z29.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
"ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "mov z30.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
"ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "mov z31.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
"ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
"ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "whilelt p0.s, %[temp], %[last_width]\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
- "ld1rqb z1.b, p7/z, [a_ptr1]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
- "sdot z24.s, z16.b, z0.b[0]\n"
- "ld1rqb z2.b, p7/z, [a_ptr2]\n"
- "ld1rqb z3.b, p7/z, [a_ptr3]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
- "sdot z25.s, z16.b, z1.b[0]\n"
- "ld1rqb z4.b, p7/z, [a_ptr4]\n"
- "sdot z26.s, z16.b, z2.b[0]\n"
- "ld1rqb z5.b, p7/z, [a_ptr5]\n"
- "sdot z27.s, z16.b, z3.b[0]\n"
- "ld1rqb z6.b, p7/z, [a_ptr6]\n"
- "sdot z24.s, z17.b, z0.b[1]\n"
- "ld1rqb z7.b, p7/z, [a_ptr7]\n"
- "sdot z28.s, z16.b, z4.b[0]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
- "sdot z29.s, z16.b, z5.b[0]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- "sdot z30.s, z16.b, z6.b[0]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- "sdot z31.s, z16.b, z7.b[0]\n"
"addvl %[b_ptr0], %[b_ptr0], #7\n"
+ "cbz %[loops], 2f\n"
+ "mov z24.s, #0\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "mov z25.s, #0\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "mov z26.s, #0\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2]\n"
+ "mov z27.s, #0\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "mov z28.s, #0\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "mov z29.s, #0\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "mov z30.s, #0\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "mov z31.s, #0\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "sdot z24.s, z16.b, z0.b[0]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "sdot z25.s, z16.b, z1.b[0]\n"
+ "sdot z26.s, z16.b, z2.b[0]\n"
+ "sdot z27.s, z16.b, z3.b[0]\n"
+ "sdot z28.s, z16.b, z4.b[0]\n"
+ "sdot z29.s, z16.b, z5.b[0]\n"
+ "sdot z30.s, z16.b, z6.b[0]\n"
+ "sdot z31.s, z16.b, z7.b[0]\n"
+ "sdot z24.s, z17.b, z0.b[1]\n"
"sdot z25.s, z17.b, z1.b[1]\n"
"sdot z26.s, z17.b, z2.b[1]\n"
"sdot z27.s, z17.b, z3.b[1]\n"
@@ -1972,86 +2192,81 @@
"sdot z29.s, z22.b, z5.b[2]\n"
"sdot z30.s, z22.b, z6.b[2]\n"
"sdot z31.s, z22.b, z7.b[2]\n"
- "cbz %[loops], 2f\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
"b.eq 3f\n"
"4:\n"
"st1w z24.s, p7, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"mov z24.s, #0\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "addvl %[c_ptr0], %[c_ptr0], #1\n"
+ "st1w z25.s, p7, [c_ptr1]\n"
+ "addvl c_ptr1, c_ptr1, #1\n"
+ "mov z25.s, #0\n"
+ "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "st1w z26.s, p7, [c_ptr2]\n"
+ "addvl c_ptr2, c_ptr2, #1\n"
+ "mov z26.s, #0\n"
+ "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ "st1w z27.s, p7, [c_ptr3]\n"
+ "addvl c_ptr3, c_ptr3, #1\n"
+ "mov z27.s, #0\n"
"ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
"ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
"addvl %[b_ptr0], %[b_ptr0], #7\n"
- "st1w z25.s, p7, [c_ptr1]\n"
- "addvl %[c_ptr0], %[c_ptr0], #1\n"
- "mov z25.s, #0\n"
+ "st1w z28.s, p7, [c_ptr4]\n"
+ "addvl c_ptr4, c_ptr4, #1\n"
+ "mov z28.s, #0\n"
"ld1rqb z1.b, p7/z, [a_ptr1]\n"
"sdot z24.s, z16.b, z0.b[0]\n"
- "st1w z26.s, p7, [c_ptr2]\n"
- "mov z26.s, #0\n"
+ "st1w z29.s, p7, [c_ptr5]\n"
+ "mov z29.s, #0\n"
"ld1rqb z2.b, p7/z, [a_ptr2]\n"
"ld1rqb z3.b, p7/z, [a_ptr3]\n"
- "addvl c_ptr1, c_ptr1, #1\n"
+ "addvl c_ptr5, c_ptr5, #1\n"
"sdot z25.s, z16.b, z1.b[0]\n"
- "st1w z27.s, p7, [c_ptr3]\n"
- "mov z27.s, #0\n"
+ "st1w z30.s, p7, [c_ptr6]\n"
+ "mov z30.s, #0\n"
"ld1rqb z4.b, p7/z, [a_ptr4]\n"
"sdot z26.s, z16.b, z2.b[0]\n"
"ld1rqb z5.b, p7/z, [a_ptr5]\n"
- "sdot z24.s, z17.b, z0.b[1]\n"
- "st1w z28.s, p7, [c_ptr4]\n"
- "mov z28.s, #0\n"
- "ld1rqb z6.b, p7/z, [a_ptr6]\n"
"sdot z27.s, z16.b, z3.b[0]\n"
- "ld1rqb z7.b, p7/z, [a_ptr7]\n"
- "sdot z25.s, z17.b, z1.b[1]\n"
- "st1w z29.s, p7, [c_ptr5]\n"
- "mov z29.s, #0\n"
- "addvl c_ptr2, c_ptr2, #1\n"
- "sdot z28.s, z16.b, z4.b[0]\n"
- "st1w z30.s, p7, [c_ptr6]\n"
- "mov z30.s, #0\n"
- "addvl c_ptr3, c_ptr3, #1\n"
- "sdot z29.s, z16.b, z5.b[0]\n"
"st1w z31.s, p7, [c_ptr7]\n"
"mov z31.s, #0\n"
- "addvl c_ptr4, c_ptr4, #1\n"
- "sdot z30.s, z16.b, z6.b[0]\n"
- "addvl c_ptr5, c_ptr5, #1\n"
- "sdot z31.s, z16.b, z7.b[0]\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
- "sdot z26.s, z17.b, z2.b[1]\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "sdot z28.s, z16.b, z4.b[0]\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "sdot z29.s, z16.b, z5.b[0]\n"
"addvl c_ptr6, c_ptr6, #1\n"
- "sdot z27.s, z17.b, z3.b[1]\n"
+ "sdot z30.s, z16.b, z6.b[0]\n"
"addvl c_ptr7, c_ptr7, #1\n"
- "sdot z28.s, z17.b, z4.b[1]\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
- "sdot z29.s, z17.b, z5.b[1]\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- "sdot z30.s, z17.b, z6.b[1]\n"
+ "sdot z31.s, z16.b, z7.b[0]\n"
"prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- "sdot z31.s, z17.b, z7.b[1]\n"
- "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "sdot z24.s, z18.b, z0.b[2]\n"
+ "sdot z24.s, z17.b, z0.b[1]\n"
"prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- "sdot z25.s, z18.b, z1.b[2]\n"
+ "sdot z25.s, z17.b, z1.b[1]\n"
"prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- "sdot z26.s, z18.b, z2.b[2]\n"
+ "sdot z26.s, z17.b, z2.b[1]\n"
"prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- "sdot z27.s, z18.b, z3.b[2]\n"
+ "sdot z27.s, z17.b, z3.b[1]\n"
"prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
- "sdot z28.s, z18.b, z4.b[2]\n"
+ "sdot z28.s, z17.b, z4.b[1]\n"
"prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
+ "sdot z29.s, z17.b, z5.b[1]\n"
+ "sdot z30.s, z17.b, z6.b[1]\n"
+ "sdot z31.s, z17.b, z7.b[1]\n"
+ "sdot z24.s, z18.b, z0.b[2]\n"
+ "sdot z25.s, z18.b, z1.b[2]\n"
+ "sdot z26.s, z18.b, z2.b[2]\n"
+ "sdot z27.s, z18.b, z3.b[2]\n"
+ "sdot z28.s, z18.b, z4.b[2]\n"
"sdot z29.s, z18.b, z5.b[2]\n"
"sdot z30.s, z18.b, z6.b[2]\n"
"sdot z31.s, z18.b, z7.b[2]\n"
- "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"sdot z24.s, z19.b, z0.b[3]\n"
"ld1rqb z0.b, p6/z, [%[a_ptr0], #0x10]\n"
"sdot z25.s, z19.b, z1.b[3]\n"
@@ -2069,7 +2284,6 @@
"sdot z31.s, z19.b, z7.b[3]\n"
"ld1rqb z7.b, p6/z, [a_ptr7, #0x10]\n"
"sdot z24.s, z20.b, z0.b[0]\n"
- "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"sdot z25.s, z20.b, z1.b[0]\n"
"sdot z26.s, z20.b, z2.b[0]\n"
"sdot z27.s, z20.b, z3.b[0]\n"
@@ -2077,7 +2291,6 @@
"sdot z29.s, z20.b, z5.b[0]\n"
"sdot z30.s, z20.b, z6.b[0]\n"
"sdot z31.s, z20.b, z7.b[0]\n"
- "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
"sdot z24.s, z21.b, z0.b[1]\n"
"sdot z25.s, z21.b, z1.b[1]\n"
"sdot z26.s, z21.b, z2.b[1]\n"
@@ -2086,7 +2299,6 @@
"sdot z29.s, z21.b, z5.b[1]\n"
"sdot z30.s, z21.b, z6.b[1]\n"
"sdot z31.s, z21.b, z7.b[1]\n"
- "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
"sdot z24.s, z22.b, z0.b[2]\n"
"sdot z25.s, z22.b, z1.b[2]\n"
"sdot z26.s, z22.b, z2.b[2]\n"
@@ -2100,47 +2312,53 @@
"st1w z24.s, p7, [%[c_ptr0]]\n"
"addvl %[c_ptr0], %[c_ptr0], #1\n"
"mov z24.s, #0\n"
- "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "addvl %[b_ptr0], %[b_ptr0], #7\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"st1w z25.s, p7, [c_ptr1]\n"
"addvl c_ptr1, c_ptr1, #1\n"
"mov z25.s, #0\n"
+ "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "st1w z26.s, p7, [c_ptr2]\n"
+ "addvl c_ptr2, c_ptr2, #1\n"
+ "mov z26.s, #0\n"
+ "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "st1w z27.s, p7, [c_ptr3]\n"
+ "addvl c_ptr3, c_ptr3, #1\n"
+ "mov z27.s, #0\n"
+ "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #7\n"
+ "st1w z28.s, p7, [c_ptr4]\n"
+ "addvl c_ptr4, c_ptr4, #1\n"
+ "mov z28.s, #0\n"
"ld1rqb z1.b, p7/z, [a_ptr1]\n"
"sdot z24.s, z16.b, z0.b[0]\n"
- "st1w z26.s, p7, [c_ptr2]\n"
- "mov z26.s, #0\n"
+ "st1w z29.s, p7, [c_ptr5]\n"
+ "mov z29.s, #0\n"
"ld1rqb z2.b, p7/z, [a_ptr2]\n"
"ld1rqb z3.b, p7/z, [a_ptr3]\n"
- "addvl c_ptr2, c_ptr2, #1\n"
+ "addvl c_ptr5, c_ptr5, #1\n"
"sdot z25.s, z16.b, z1.b[0]\n"
- "st1w z27.s, p7, [c_ptr3]\n"
- "mov z27.s, #0\n"
+ "st1w z30.s, p7, [c_ptr6]\n"
+ "mov z30.s, #0\n"
"ld1rqb z4.b, p7/z, [a_ptr4]\n"
"sdot z26.s, z16.b, z2.b[0]\n"
"ld1rqb z5.b, p7/z, [a_ptr5]\n"
- "sdot z24.s, z17.b, z0.b[1]\n"
- "st1w z28.s, p7, [c_ptr4]\n"
- "mov z28.s, #0\n"
- "ld1rqb z6.b, p7/z, [a_ptr6]\n"
"sdot z27.s, z16.b, z3.b[0]\n"
- "ld1rqb z7.b, p7/z, [a_ptr7]\n"
- "sdot z25.s, z17.b, z1.b[1]\n"
- "st1w z29.s, p7, [c_ptr5]\n"
- "mov z29.s, #0\n"
- "addvl c_ptr3, c_ptr3, #1\n"
- "sdot z28.s, z16.b, z4.b[0]\n"
- "st1w z30.s, p7, [c_ptr6]\n"
- "mov z30.s, #0\n"
- "addvl c_ptr4, c_ptr4, #1\n"
- "sdot z29.s, z16.b, z5.b[0]\n"
"st1w z31.s, p7, [c_ptr7]\n"
"mov z31.s, #0\n"
- "addvl c_ptr5, c_ptr5, #1\n"
- "sdot z30.s, z16.b, z6.b[0]\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "sdot z28.s, z16.b, z4.b[0]\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "sdot z29.s, z16.b, z5.b[0]\n"
"addvl c_ptr6, c_ptr6, #1\n"
- "sdot z31.s, z16.b, z7.b[0]\n"
+ "sdot z30.s, z16.b, z6.b[0]\n"
"addvl c_ptr7, c_ptr7, #1\n"
+ "sdot z31.s, z16.b, z7.b[0]\n"
+ "sdot z24.s, z17.b, z0.b[1]\n"
+ "sdot z25.s, z17.b, z1.b[1]\n"
"sdot z26.s, z17.b, z2.b[1]\n"
"sdot z27.s, z17.b, z3.b[1]\n"
"sdot z28.s, z17.b, z4.b[1]\n"
@@ -2195,23 +2413,98 @@
"sdot z29.s, z22.b, z5.b[2]\n"
"sdot z30.s, z22.b, z6.b[2]\n"
"sdot z31.s, z22.b, z7.b[2]\n"
+ "b 5f\n"
"2:\n"
+ "mov z24.s, #0\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "mov z25.s, #0\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "mov z26.s, #0\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2]\n"
+ "mov z27.s, #0\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "mov z28.s, #0\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "mov z29.s, #0\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "mov z30.s, #0\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "mov z31.s, #0\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "sdot z24.s, z16.b, z0.b[0]\n"
+ "sdot z25.s, z16.b, z1.b[0]\n"
+ "sdot z26.s, z16.b, z2.b[0]\n"
+ "sdot z27.s, z16.b, z3.b[0]\n"
+ "sdot z28.s, z16.b, z4.b[0]\n"
+ "sdot z29.s, z16.b, z5.b[0]\n"
+ "sdot z30.s, z16.b, z6.b[0]\n"
+ "sdot z31.s, z16.b, z7.b[0]\n"
+ "sdot z24.s, z17.b, z0.b[1]\n"
+ "sdot z25.s, z17.b, z1.b[1]\n"
+ "sdot z26.s, z17.b, z2.b[1]\n"
+ "sdot z27.s, z17.b, z3.b[1]\n"
+ "sdot z28.s, z17.b, z4.b[1]\n"
+ "sdot z29.s, z17.b, z5.b[1]\n"
+ "sdot z30.s, z17.b, z6.b[1]\n"
+ "sdot z31.s, z17.b, z7.b[1]\n"
+ "sdot z24.s, z18.b, z0.b[2]\n"
+ "sdot z25.s, z18.b, z1.b[2]\n"
+ "sdot z26.s, z18.b, z2.b[2]\n"
+ "sdot z27.s, z18.b, z3.b[2]\n"
+ "sdot z28.s, z18.b, z4.b[2]\n"
+ "sdot z29.s, z18.b, z5.b[2]\n"
+ "sdot z30.s, z18.b, z6.b[2]\n"
+ "sdot z31.s, z18.b, z7.b[2]\n"
+ "sdot z24.s, z19.b, z0.b[3]\n"
+ "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x10]\n"
+ "sdot z25.s, z19.b, z1.b[3]\n"
+ "ld1rqb z1.b, p6/z, [a_ptr1, #0x10]\n"
+ "sdot z26.s, z19.b, z2.b[3]\n"
+ "ld1rqb z2.b, p6/z, [a_ptr2, #0x10]\n"
+ "sdot z27.s, z19.b, z3.b[3]\n"
+ "ld1rqb z3.b, p6/z, [a_ptr3, #0x10]\n"
+ "sdot z28.s, z19.b, z4.b[3]\n"
+ "ld1rqb z4.b, p6/z, [a_ptr4, #0x10]\n"
+ "sdot z29.s, z19.b, z5.b[3]\n"
+ "ld1rqb z5.b, p6/z, [a_ptr5, #0x10]\n"
+ "sdot z30.s, z19.b, z6.b[3]\n"
+ "ld1rqb z6.b, p6/z, [a_ptr6, #0x10]\n"
+ "sdot z31.s, z19.b, z7.b[3]\n"
+ "ld1rqb z7.b, p6/z, [a_ptr7, #0x10]\n"
+ "sdot z24.s, z20.b, z0.b[0]\n"
+ "sdot z25.s, z20.b, z1.b[0]\n"
+ "sdot z26.s, z20.b, z2.b[0]\n"
+ "sdot z27.s, z20.b, z3.b[0]\n"
+ "sdot z28.s, z20.b, z4.b[0]\n"
+ "sdot z29.s, z20.b, z5.b[0]\n"
+ "sdot z30.s, z20.b, z6.b[0]\n"
+ "sdot z31.s, z20.b, z7.b[0]\n"
+ "sdot z24.s, z21.b, z0.b[1]\n"
+ "sdot z25.s, z21.b, z1.b[1]\n"
+ "sdot z26.s, z21.b, z2.b[1]\n"
+ "sdot z27.s, z21.b, z3.b[1]\n"
+ "sdot z28.s, z21.b, z4.b[1]\n"
+ "sdot z29.s, z21.b, z5.b[1]\n"
+ "sdot z30.s, z21.b, z6.b[1]\n"
+ "sdot z31.s, z21.b, z7.b[1]\n"
+ "sdot z24.s, z22.b, z0.b[2]\n"
+ "sdot z25.s, z22.b, z1.b[2]\n"
+ "sdot z26.s, z22.b, z2.b[2]\n"
+ "sdot z27.s, z22.b, z3.b[2]\n"
+ "sdot z28.s, z22.b, z4.b[2]\n"
+ "sdot z29.s, z22.b, z5.b[2]\n"
+ "sdot z30.s, z22.b, z6.b[2]\n"
+ "sdot z31.s, z22.b, z7.b[2]\n"
+ "5:\n"
"st1w z24.s, p0, [%[c_ptr0]]\n"
"addvl %[c_ptr0], %[c_ptr0], #1\n"
"st1w z25.s, p0, [c_ptr1]\n"
- "addvl c_ptr1, c_ptr1, #1\n"
"st1w z26.s, p0, [c_ptr2]\n"
- "addvl c_ptr2, c_ptr2, #1\n"
"st1w z27.s, p0, [c_ptr3]\n"
- "addvl c_ptr3, c_ptr3, #1\n"
"st1w z28.s, p0, [c_ptr4]\n"
- "addvl c_ptr4, c_ptr4, #1\n"
"st1w z29.s, p0, [c_ptr5]\n"
- "addvl c_ptr5, c_ptr5, #1\n"
"st1w z30.s, p0, [c_ptr6]\n"
- "addvl c_ptr6, c_ptr6, #1\n"
"st1w z31.s, p0, [c_ptr7]\n"
- "addvl c_ptr7, c_ptr7, #1\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -2290,49 +2583,51 @@
"add c_ptr1, %[c_ptr0], #0x0\n"
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
- "mov z24.s, #0\n"
"ptrue p7.b\n"
- "mov z25.s, #0\n"
"whilelt p6.b, %[temp], %[odd_depth]\n"
- "mov z26.s, #0\n"
+ "whilelt p0.s, %[temp], %[last_width]\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
"ld1b z16.b, p7/z, [%[b_ptr0]]\n"
- "mov z27.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
"ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "mov z28.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
"ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "mov z29.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
"ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "mov z30.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
"ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "mov z31.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
"ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
"ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "whilelt p0.s, %[temp], %[last_width]\n"
"ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
- "ld1rqb z1.b, p7/z, [a_ptr1]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
- "sdot z24.s, z16.b, z0.b[0]\n"
- "ld1rqb z2.b, p7/z, [a_ptr2]\n"
- "ld1rqb z3.b, p7/z, [a_ptr3]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
- "sdot z25.s, z16.b, z1.b[0]\n"
- "ld1rqb z4.b, p7/z, [a_ptr4]\n"
- "sdot z26.s, z16.b, z2.b[0]\n"
- "ld1rqb z5.b, p7/z, [a_ptr5]\n"
- "sdot z27.s, z16.b, z3.b[0]\n"
- "ld1rqb z6.b, p7/z, [a_ptr6]\n"
- "sdot z24.s, z17.b, z0.b[1]\n"
- "ld1rqb z7.b, p7/z, [a_ptr7]\n"
- "sdot z28.s, z16.b, z4.b[0]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- "sdot z29.s, z16.b, z5.b[0]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- "sdot z30.s, z16.b, z6.b[0]\n"
"addvl %[b_ptr0], %[b_ptr0], #8\n"
+ "cbz %[loops], 2f\n"
+ "mov z24.s, #0\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "mov z25.s, #0\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "mov z26.s, #0\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2]\n"
+ "mov z27.s, #0\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "mov z28.s, #0\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "mov z29.s, #0\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "mov z30.s, #0\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "mov z31.s, #0\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "sdot z24.s, z16.b, z0.b[0]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "sdot z25.s, z16.b, z1.b[0]\n"
+ "sdot z26.s, z16.b, z2.b[0]\n"
+ "sdot z27.s, z16.b, z3.b[0]\n"
+ "sdot z28.s, z16.b, z4.b[0]\n"
+ "sdot z29.s, z16.b, z5.b[0]\n"
+ "sdot z30.s, z16.b, z6.b[0]\n"
"sdot z31.s, z16.b, z7.b[0]\n"
+ "sdot z24.s, z17.b, z0.b[1]\n"
"sdot z25.s, z17.b, z1.b[1]\n"
"sdot z26.s, z17.b, z2.b[1]\n"
"sdot z27.s, z17.b, z3.b[1]\n"
@@ -2396,87 +2691,82 @@
"sdot z29.s, z23.b, z5.b[3]\n"
"sdot z30.s, z23.b, z6.b[3]\n"
"sdot z31.s, z23.b, z7.b[3]\n"
- "cbz %[loops], 2f\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
"b.eq 3f\n"
"4:\n"
"st1w z24.s, p7, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"mov z24.s, #0\n"
- "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "st1w z25.s, p7, [c_ptr1]\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"addvl %[c_ptr0], %[c_ptr0], #1\n"
- "mov z25.s, #0\n"
- "ld1rqb z1.b, p7/z, [a_ptr1]\n"
- "sdot z24.s, z16.b, z0.b[0]\n"
- "st1w z26.s, p7, [c_ptr2]\n"
- "mov z26.s, #0\n"
- "ld1rqb z2.b, p7/z, [a_ptr2]\n"
- "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "st1w z25.s, p7, [c_ptr1]\n"
"addvl c_ptr1, c_ptr1, #1\n"
- "sdot z25.s, z16.b, z1.b[0]\n"
- "st1w z27.s, p7, [c_ptr3]\n"
- "mov z27.s, #0\n"
- "ld1rqb z4.b, p7/z, [a_ptr4]\n"
- "sdot z26.s, z16.b, z2.b[0]\n"
- "ld1rqb z5.b, p7/z, [a_ptr5]\n"
- "sdot z24.s, z17.b, z0.b[1]\n"
- "st1w z28.s, p7, [c_ptr4]\n"
- "mov z28.s, #0\n"
- "ld1rqb z6.b, p7/z, [a_ptr6]\n"
- "sdot z27.s, z16.b, z3.b[0]\n"
- "ld1rqb z7.b, p7/z, [a_ptr7]\n"
- "sdot z25.s, z17.b, z1.b[1]\n"
- "st1w z29.s, p7, [c_ptr5]\n"
- "mov z29.s, #0\n"
+ "mov z25.s, #0\n"
+ "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "st1w z26.s, p7, [c_ptr2]\n"
"addvl c_ptr2, c_ptr2, #1\n"
- "sdot z28.s, z16.b, z4.b[0]\n"
+ "mov z26.s, #0\n"
+ "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ "st1w z27.s, p7, [c_ptr3]\n"
+ "addvl c_ptr3, c_ptr3, #1\n"
+ "mov z27.s, #0\n"
+ "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #8\n"
+ "st1w z28.s, p7, [c_ptr4]\n"
+ "addvl c_ptr4, c_ptr4, #1\n"
+ "mov z28.s, #0\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ "st1w z29.s, p7, [c_ptr5]\n"
+ "addvl c_ptr5, c_ptr5, #1\n"
+ "mov z29.s, #0\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2]\n"
+ "sdot z24.s, z16.b, z0.b[0]\n"
"st1w z30.s, p7, [c_ptr6]\n"
"mov z30.s, #0\n"
- "addvl c_ptr3, c_ptr3, #1\n"
- "sdot z29.s, z16.b, z5.b[0]\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "sdot z25.s, z16.b, z1.b[0]\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "sdot z26.s, z16.b, z2.b[0]\n"
"st1w z31.s, p7, [c_ptr7]\n"
"mov z31.s, #0\n"
- "addvl c_ptr4, c_ptr4, #1\n"
- "sdot z30.s, z16.b, z6.b[0]\n"
- "addvl c_ptr5, c_ptr5, #1\n"
- "sdot z31.s, z16.b, z7.b[0]\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
- "sdot z26.s, z17.b, z2.b[1]\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "sdot z27.s, z16.b, z3.b[0]\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "sdot z28.s, z16.b, z4.b[0]\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "sdot z24.s, z17.b, z0.b[1]\n"
"addvl c_ptr6, c_ptr6, #1\n"
- "sdot z27.s, z17.b, z3.b[1]\n"
+ "sdot z29.s, z16.b, z5.b[0]\n"
"addvl c_ptr7, c_ptr7, #1\n"
- "sdot z28.s, z17.b, z4.b[1]\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
- "sdot z29.s, z17.b, z5.b[1]\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- "sdot z30.s, z17.b, z6.b[1]\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- "sdot z31.s, z17.b, z7.b[1]\n"
- "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "sdot z24.s, z18.b, z0.b[2]\n"
+ "sdot z30.s, z16.b, z6.b[0]\n"
"prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- "sdot z25.s, z18.b, z1.b[2]\n"
+ "sdot z31.s, z16.b, z7.b[0]\n"
"prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- "sdot z26.s, z18.b, z2.b[2]\n"
+ "sdot z25.s, z17.b, z1.b[1]\n"
"prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- "sdot z27.s, z18.b, z3.b[2]\n"
+ "sdot z26.s, z17.b, z2.b[1]\n"
"prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
- "sdot z28.s, z18.b, z4.b[2]\n"
+ "sdot z27.s, z17.b, z3.b[1]\n"
"prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
+ "sdot z28.s, z17.b, z4.b[1]\n"
+ "sdot z29.s, z17.b, z5.b[1]\n"
+ "sdot z30.s, z17.b, z6.b[1]\n"
+ "sdot z31.s, z17.b, z7.b[1]\n"
+ "sdot z24.s, z18.b, z0.b[2]\n"
+ "sdot z25.s, z18.b, z1.b[2]\n"
+ "sdot z26.s, z18.b, z2.b[2]\n"
+ "sdot z27.s, z18.b, z3.b[2]\n"
+ "sdot z28.s, z18.b, z4.b[2]\n"
"sdot z29.s, z18.b, z5.b[2]\n"
"sdot z30.s, z18.b, z6.b[2]\n"
"sdot z31.s, z18.b, z7.b[2]\n"
- "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"sdot z24.s, z19.b, z0.b[3]\n"
"ld1rqb z0.b, p6/z, [%[a_ptr0], #0x10]\n"
"sdot z25.s, z19.b, z1.b[3]\n"
@@ -2494,7 +2784,6 @@
"sdot z31.s, z19.b, z7.b[3]\n"
"ld1rqb z7.b, p6/z, [a_ptr7, #0x10]\n"
"sdot z24.s, z20.b, z0.b[0]\n"
- "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"sdot z25.s, z20.b, z1.b[0]\n"
"sdot z26.s, z20.b, z2.b[0]\n"
"sdot z27.s, z20.b, z3.b[0]\n"
@@ -2502,7 +2791,6 @@
"sdot z29.s, z20.b, z5.b[0]\n"
"sdot z30.s, z20.b, z6.b[0]\n"
"sdot z31.s, z20.b, z7.b[0]\n"
- "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
"sdot z24.s, z21.b, z0.b[1]\n"
"sdot z25.s, z21.b, z1.b[1]\n"
"sdot z26.s, z21.b, z2.b[1]\n"
@@ -2511,7 +2799,6 @@
"sdot z29.s, z21.b, z5.b[1]\n"
"sdot z30.s, z21.b, z6.b[1]\n"
"sdot z31.s, z21.b, z7.b[1]\n"
- "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
"sdot z24.s, z22.b, z0.b[2]\n"
"sdot z25.s, z22.b, z1.b[2]\n"
"sdot z26.s, z22.b, z2.b[2]\n"
@@ -2520,7 +2807,6 @@
"sdot z29.s, z22.b, z5.b[2]\n"
"sdot z30.s, z22.b, z6.b[2]\n"
"sdot z31.s, z22.b, z7.b[2]\n"
- "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
"sdot z24.s, z23.b, z0.b[3]\n"
"sdot z25.s, z23.b, z1.b[3]\n"
"sdot z26.s, z23.b, z2.b[3]\n"
@@ -2534,47 +2820,54 @@
"st1w z24.s, p7, [%[c_ptr0]]\n"
"addvl %[c_ptr0], %[c_ptr0], #1\n"
"mov z24.s, #0\n"
- "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"st1w z25.s, p7, [c_ptr1]\n"
"addvl c_ptr1, c_ptr1, #1\n"
"mov z25.s, #0\n"
- "ld1rqb z1.b, p7/z, [a_ptr1]\n"
- "sdot z24.s, z16.b, z0.b[0]\n"
+ "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"st1w z26.s, p7, [c_ptr2]\n"
- "mov z26.s, #0\n"
- "ld1rqb z2.b, p7/z, [a_ptr2]\n"
- "ld1rqb z3.b, p7/z, [a_ptr3]\n"
"addvl c_ptr2, c_ptr2, #1\n"
- "sdot z25.s, z16.b, z1.b[0]\n"
+ "mov z26.s, #0\n"
+ "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
"st1w z27.s, p7, [c_ptr3]\n"
- "mov z27.s, #0\n"
- "ld1rqb z4.b, p7/z, [a_ptr4]\n"
- "sdot z26.s, z16.b, z2.b[0]\n"
- "ld1rqb z5.b, p7/z, [a_ptr5]\n"
- "sdot z24.s, z17.b, z0.b[1]\n"
- "st1w z28.s, p7, [c_ptr4]\n"
- "mov z28.s, #0\n"
- "ld1rqb z6.b, p7/z, [a_ptr6]\n"
- "sdot z27.s, z16.b, z3.b[0]\n"
- "ld1rqb z7.b, p7/z, [a_ptr7]\n"
- "sdot z25.s, z17.b, z1.b[1]\n"
- "st1w z29.s, p7, [c_ptr5]\n"
- "mov z29.s, #0\n"
"addvl c_ptr3, c_ptr3, #1\n"
- "sdot z28.s, z16.b, z4.b[0]\n"
+ "mov z27.s, #0\n"
+ "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #8\n"
+ "st1w z28.s, p7, [c_ptr4]\n"
+ "addvl c_ptr4, c_ptr4, #1\n"
+ "mov z28.s, #0\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "st1w z29.s, p7, [c_ptr5]\n"
+ "addvl c_ptr5, c_ptr5, #1\n"
+ "mov z29.s, #0\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2]\n"
+ "sdot z24.s, z16.b, z0.b[0]\n"
"st1w z30.s, p7, [c_ptr6]\n"
"mov z30.s, #0\n"
- "addvl c_ptr4, c_ptr4, #1\n"
- "sdot z29.s, z16.b, z5.b[0]\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "sdot z25.s, z16.b, z1.b[0]\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "sdot z26.s, z16.b, z2.b[0]\n"
"st1w z31.s, p7, [c_ptr7]\n"
"mov z31.s, #0\n"
- "addvl c_ptr5, c_ptr5, #1\n"
- "sdot z30.s, z16.b, z6.b[0]\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "sdot z27.s, z16.b, z3.b[0]\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "sdot z28.s, z16.b, z4.b[0]\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "sdot z24.s, z17.b, z0.b[1]\n"
"addvl c_ptr6, c_ptr6, #1\n"
- "sdot z31.s, z16.b, z7.b[0]\n"
+ "sdot z29.s, z16.b, z5.b[0]\n"
"addvl c_ptr7, c_ptr7, #1\n"
+ "sdot z30.s, z16.b, z6.b[0]\n"
+ "sdot z31.s, z16.b, z7.b[0]\n"
+ "sdot z25.s, z17.b, z1.b[1]\n"
"sdot z26.s, z17.b, z2.b[1]\n"
"sdot z27.s, z17.b, z3.b[1]\n"
"sdot z28.s, z17.b, z4.b[1]\n"
@@ -2637,23 +2930,106 @@
"sdot z29.s, z23.b, z5.b[3]\n"
"sdot z30.s, z23.b, z6.b[3]\n"
"sdot z31.s, z23.b, z7.b[3]\n"
+ "b 5f\n"
"2:\n"
+ "mov z24.s, #0\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "mov z25.s, #0\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "mov z26.s, #0\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2]\n"
+ "mov z27.s, #0\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "mov z28.s, #0\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "mov z29.s, #0\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "mov z30.s, #0\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "mov z31.s, #0\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "sdot z24.s, z16.b, z0.b[0]\n"
+ "sdot z25.s, z16.b, z1.b[0]\n"
+ "sdot z26.s, z16.b, z2.b[0]\n"
+ "sdot z27.s, z16.b, z3.b[0]\n"
+ "sdot z28.s, z16.b, z4.b[0]\n"
+ "sdot z29.s, z16.b, z5.b[0]\n"
+ "sdot z30.s, z16.b, z6.b[0]\n"
+ "sdot z31.s, z16.b, z7.b[0]\n"
+ "sdot z24.s, z17.b, z0.b[1]\n"
+ "sdot z25.s, z17.b, z1.b[1]\n"
+ "sdot z26.s, z17.b, z2.b[1]\n"
+ "sdot z27.s, z17.b, z3.b[1]\n"
+ "sdot z28.s, z17.b, z4.b[1]\n"
+ "sdot z29.s, z17.b, z5.b[1]\n"
+ "sdot z30.s, z17.b, z6.b[1]\n"
+ "sdot z31.s, z17.b, z7.b[1]\n"
+ "sdot z24.s, z18.b, z0.b[2]\n"
+ "sdot z25.s, z18.b, z1.b[2]\n"
+ "sdot z26.s, z18.b, z2.b[2]\n"
+ "sdot z27.s, z18.b, z3.b[2]\n"
+ "sdot z28.s, z18.b, z4.b[2]\n"
+ "sdot z29.s, z18.b, z5.b[2]\n"
+ "sdot z30.s, z18.b, z6.b[2]\n"
+ "sdot z31.s, z18.b, z7.b[2]\n"
+ "sdot z24.s, z19.b, z0.b[3]\n"
+ "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x10]\n"
+ "sdot z25.s, z19.b, z1.b[3]\n"
+ "ld1rqb z1.b, p6/z, [a_ptr1, #0x10]\n"
+ "sdot z26.s, z19.b, z2.b[3]\n"
+ "ld1rqb z2.b, p6/z, [a_ptr2, #0x10]\n"
+ "sdot z27.s, z19.b, z3.b[3]\n"
+ "ld1rqb z3.b, p6/z, [a_ptr3, #0x10]\n"
+ "sdot z28.s, z19.b, z4.b[3]\n"
+ "ld1rqb z4.b, p6/z, [a_ptr4, #0x10]\n"
+ "sdot z29.s, z19.b, z5.b[3]\n"
+ "ld1rqb z5.b, p6/z, [a_ptr5, #0x10]\n"
+ "sdot z30.s, z19.b, z6.b[3]\n"
+ "ld1rqb z6.b, p6/z, [a_ptr6, #0x10]\n"
+ "sdot z31.s, z19.b, z7.b[3]\n"
+ "ld1rqb z7.b, p6/z, [a_ptr7, #0x10]\n"
+ "sdot z24.s, z20.b, z0.b[0]\n"
+ "sdot z25.s, z20.b, z1.b[0]\n"
+ "sdot z26.s, z20.b, z2.b[0]\n"
+ "sdot z27.s, z20.b, z3.b[0]\n"
+ "sdot z28.s, z20.b, z4.b[0]\n"
+ "sdot z29.s, z20.b, z5.b[0]\n"
+ "sdot z30.s, z20.b, z6.b[0]\n"
+ "sdot z31.s, z20.b, z7.b[0]\n"
+ "sdot z24.s, z21.b, z0.b[1]\n"
+ "sdot z25.s, z21.b, z1.b[1]\n"
+ "sdot z26.s, z21.b, z2.b[1]\n"
+ "sdot z27.s, z21.b, z3.b[1]\n"
+ "sdot z28.s, z21.b, z4.b[1]\n"
+ "sdot z29.s, z21.b, z5.b[1]\n"
+ "sdot z30.s, z21.b, z6.b[1]\n"
+ "sdot z31.s, z21.b, z7.b[1]\n"
+ "sdot z24.s, z22.b, z0.b[2]\n"
+ "sdot z25.s, z22.b, z1.b[2]\n"
+ "sdot z26.s, z22.b, z2.b[2]\n"
+ "sdot z27.s, z22.b, z3.b[2]\n"
+ "sdot z28.s, z22.b, z4.b[2]\n"
+ "sdot z29.s, z22.b, z5.b[2]\n"
+ "sdot z30.s, z22.b, z6.b[2]\n"
+ "sdot z31.s, z22.b, z7.b[2]\n"
+ "sdot z24.s, z23.b, z0.b[3]\n"
+ "sdot z25.s, z23.b, z1.b[3]\n"
+ "sdot z26.s, z23.b, z2.b[3]\n"
+ "sdot z27.s, z23.b, z3.b[3]\n"
+ "sdot z28.s, z23.b, z4.b[3]\n"
+ "sdot z29.s, z23.b, z5.b[3]\n"
+ "sdot z30.s, z23.b, z6.b[3]\n"
+ "sdot z31.s, z23.b, z7.b[3]\n"
+ "5:\n"
"st1w z24.s, p0, [%[c_ptr0]]\n"
"addvl %[c_ptr0], %[c_ptr0], #1\n"
"st1w z25.s, p0, [c_ptr1]\n"
- "addvl c_ptr1, c_ptr1, #1\n"
"st1w z26.s, p0, [c_ptr2]\n"
- "addvl c_ptr2, c_ptr2, #1\n"
"st1w z27.s, p0, [c_ptr3]\n"
- "addvl c_ptr3, c_ptr3, #1\n"
"st1w z28.s, p0, [c_ptr4]\n"
- "addvl c_ptr4, c_ptr4, #1\n"
"st1w z29.s, p0, [c_ptr5]\n"
- "addvl c_ptr5, c_ptr5, #1\n"
"st1w z30.s, p0, [c_ptr6]\n"
- "addvl c_ptr6, c_ptr6, #1\n"
"st1w z31.s, p0, [c_ptr7]\n"
- "addvl c_ptr7, c_ptr7, #1\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -2732,54 +3108,56 @@
"add c_ptr1, %[c_ptr0], #0x0\n"
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
- "mov z24.s, #0\n"
"ptrue p7.b\n"
- "mov z25.s, #0\n"
"whilelt p6.b, %[temp], %[odd_depth]\n"
- "mov z26.s, #0\n"
+ "whilelt p0.s, %[temp], %[last_width]\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
"ld1b z16.b, p7/z, [%[b_ptr0]]\n"
- "mov z27.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
"ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "mov z28.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
"ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "mov z29.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
"ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "mov z30.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
"ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "mov z31.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
"ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
"ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "whilelt p0.s, %[temp], %[last_width]\n"
"ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
- "ld1rqb z1.b, p7/z, [a_ptr1]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
- "sdot z24.s, z16.b, z0.b[0]\n"
- "ld1rqb z2.b, p7/z, [a_ptr2]\n"
- "ld1rqb z3.b, p7/z, [a_ptr3]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
- "sdot z25.s, z16.b, z1.b[0]\n"
- "ld1rqb z4.b, p7/z, [a_ptr4]\n"
- "sdot z26.s, z16.b, z2.b[0]\n"
- "ld1rqb z5.b, p7/z, [a_ptr5]\n"
- "sdot z27.s, z16.b, z3.b[0]\n"
- "ld1rqb z6.b, p7/z, [a_ptr6]\n"
- "sdot z24.s, z17.b, z0.b[1]\n"
- "ld1rqb z7.b, p7/z, [a_ptr7]\n"
- "sdot z28.s, z16.b, z4.b[0]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- "sdot z29.s, z16.b, z5.b[0]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- "sdot z30.s, z16.b, z6.b[0]\n"
"addvl %[b_ptr0], %[b_ptr0], #8\n"
+ "cbz %[loops], 2f\n"
+ "mov z24.s, #0\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "mov z25.s, #0\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "mov z26.s, #0\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2]\n"
+ "mov z27.s, #0\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "mov z28.s, #0\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "mov z29.s, #0\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "mov z30.s, #0\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "mov z31.s, #0\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "sdot z24.s, z16.b, z0.b[0]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "sdot z25.s, z16.b, z1.b[0]\n"
+ "sdot z26.s, z16.b, z2.b[0]\n"
+ "sdot z27.s, z16.b, z3.b[0]\n"
+ "sdot z28.s, z16.b, z4.b[0]\n"
+ "sdot z29.s, z16.b, z5.b[0]\n"
+ "sdot z30.s, z16.b, z6.b[0]\n"
"sdot z31.s, z16.b, z7.b[0]\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "sdot z24.s, z17.b, z0.b[1]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #1\n"
"sdot z25.s, z17.b, z1.b[1]\n"
"sdot z26.s, z17.b, z2.b[1]\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
"sdot z27.s, z17.b, z3.b[1]\n"
- "addvl %[b_ptr0], %[b_ptr0], #1\n"
"sdot z28.s, z17.b, z4.b[1]\n"
"sdot z29.s, z17.b, z5.b[1]\n"
"sdot z30.s, z17.b, z6.b[1]\n"
@@ -2856,88 +3234,84 @@
"sdot z29.s, z16.b, z5.b[0]\n"
"sdot z30.s, z16.b, z6.b[0]\n"
"sdot z31.s, z16.b, z7.b[0]\n"
- "cbz %[loops], 2f\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"b.eq 3f\n"
"4:\n"
"st1w z24.s, p7, [%[c_ptr0]]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
+ "subs %[loops], %[loops], #0x1\n"
"mov z24.s, #0\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "addvl %[c_ptr0], %[c_ptr0], #1\n"
+ "st1w z25.s, p7, [c_ptr1]\n"
+ "addvl c_ptr1, c_ptr1, #1\n"
+ "mov z25.s, #0\n"
+ "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "st1w z26.s, p7, [c_ptr2]\n"
+ "addvl c_ptr2, c_ptr2, #1\n"
+ "mov z26.s, #0\n"
+ "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ "st1w z27.s, p7, [c_ptr3]\n"
+ "addvl c_ptr3, c_ptr3, #1\n"
+ "mov z27.s, #0\n"
+ "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #8\n"
+ "st1w z28.s, p7, [c_ptr4]\n"
+ "addvl c_ptr4, c_ptr4, #1\n"
+ "mov z28.s, #0\n"
"ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
"ld1rqb z1.b, p7/z, [a_ptr1]\n"
- "subs %[loops], %[loops], #0x1\n"
- "st1w z25.s, p7, [c_ptr1]\n"
- "addvl %[c_ptr0], %[c_ptr0], #1\n"
- "mov z25.s, #0\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ "st1w z29.s, p7, [c_ptr5]\n"
+ "addvl c_ptr5, c_ptr5, #1\n"
+ "mov z29.s, #0\n"
"ld1rqb z2.b, p7/z, [a_ptr2]\n"
"sdot z24.s, z16.b, z0.b[0]\n"
- "st1w z26.s, p7, [c_ptr2]\n"
- "mov z26.s, #0\n"
- "ld1rqb z3.b, p7/z, [a_ptr3]\n"
- "ld1rqb z4.b, p7/z, [a_ptr4]\n"
- "addvl c_ptr1, c_ptr1, #1\n"
- "sdot z25.s, z16.b, z1.b[0]\n"
- "st1w z27.s, p7, [c_ptr3]\n"
- "mov z27.s, #0\n"
- "ld1rqb z5.b, p7/z, [a_ptr5]\n"
- "sdot z26.s, z16.b, z2.b[0]\n"
- "ld1rqb z6.b, p7/z, [a_ptr6]\n"
- "sdot z24.s, z17.b, z0.b[1]\n"
- "st1w z28.s, p7, [c_ptr4]\n"
- "mov z28.s, #0\n"
- "ld1rqb z7.b, p7/z, [a_ptr7]\n"
- "sdot z27.s, z16.b, z3.b[0]\n"
- "addvl c_ptr2, c_ptr2, #1\n"
- "sdot z25.s, z17.b, z1.b[1]\n"
- "st1w z29.s, p7, [c_ptr5]\n"
- "mov z29.s, #0\n"
- "addvl c_ptr3, c_ptr3, #1\n"
- "sdot z28.s, z16.b, z4.b[0]\n"
"st1w z30.s, p7, [c_ptr6]\n"
"mov z30.s, #0\n"
- "addvl c_ptr4, c_ptr4, #1\n"
- "sdot z29.s, z16.b, z5.b[0]\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "sdot z25.s, z16.b, z1.b[0]\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "sdot z26.s, z16.b, z2.b[0]\n"
"st1w z31.s, p7, [c_ptr7]\n"
"mov z31.s, #0\n"
- "addvl c_ptr5, c_ptr5, #1\n"
- "sdot z30.s, z16.b, z6.b[0]\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "sdot z27.s, z16.b, z3.b[0]\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "sdot z28.s, z16.b, z4.b[0]\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "sdot z24.s, z17.b, z0.b[1]\n"
"addvl c_ptr6, c_ptr6, #1\n"
+ "sdot z29.s, z16.b, z5.b[0]\n"
+ "addvl c_ptr7, c_ptr7, #1\n"
+ "sdot z30.s, z16.b, z6.b[0]\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
"sdot z31.s, z16.b, z7.b[0]\n"
"ld1b z16.b, p7/z, [%[b_ptr0]]\n"
- "sdot z26.s, z17.b, z2.b[1]\n"
- "addvl c_ptr7, c_ptr7, #1\n"
- "sdot z27.s, z17.b, z3.b[1]\n"
+ "sdot z25.s, z17.b, z1.b[1]\n"
"addvl %[b_ptr0], %[b_ptr0], #1\n"
- "sdot z28.s, z17.b, z4.b[1]\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
- "sdot z29.s, z17.b, z5.b[1]\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- "sdot z30.s, z17.b, z6.b[1]\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- "sdot z31.s, z17.b, z7.b[1]\n"
- "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "sdot z24.s, z18.b, z0.b[2]\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- "sdot z25.s, z18.b, z1.b[2]\n"
+ "sdot z26.s, z17.b, z2.b[1]\n"
"prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- "sdot z26.s, z18.b, z2.b[2]\n"
+ "sdot z27.s, z17.b, z3.b[1]\n"
"prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- "sdot z27.s, z18.b, z3.b[2]\n"
+ "sdot z28.s, z17.b, z4.b[1]\n"
"prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
- "sdot z28.s, z18.b, z4.b[2]\n"
+ "sdot z29.s, z17.b, z5.b[1]\n"
"prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
+ "sdot z30.s, z17.b, z6.b[1]\n"
+ "sdot z31.s, z17.b, z7.b[1]\n"
+ "sdot z24.s, z18.b, z0.b[2]\n"
+ "sdot z25.s, z18.b, z1.b[2]\n"
+ "sdot z26.s, z18.b, z2.b[2]\n"
+ "sdot z27.s, z18.b, z3.b[2]\n"
+ "sdot z28.s, z18.b, z4.b[2]\n"
"sdot z29.s, z18.b, z5.b[2]\n"
"sdot z30.s, z18.b, z6.b[2]\n"
"sdot z31.s, z18.b, z7.b[2]\n"
- "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"sdot z24.s, z19.b, z0.b[3]\n"
"ld1rqb z0.b, p7/z, [%[a_ptr0], #0x10]\n"
"sdot z25.s, z19.b, z1.b[3]\n"
@@ -2955,7 +3329,6 @@
"sdot z31.s, z19.b, z7.b[3]\n"
"ld1rqb z7.b, p7/z, [a_ptr7, #0x10]\n"
"sdot z24.s, z20.b, z0.b[0]\n"
- "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"sdot z25.s, z20.b, z1.b[0]\n"
"sdot z26.s, z20.b, z2.b[0]\n"
"sdot z27.s, z20.b, z3.b[0]\n"
@@ -2963,7 +3336,6 @@
"sdot z29.s, z20.b, z5.b[0]\n"
"sdot z30.s, z20.b, z6.b[0]\n"
"sdot z31.s, z20.b, z7.b[0]\n"
- "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
"sdot z24.s, z21.b, z0.b[1]\n"
"sdot z25.s, z21.b, z1.b[1]\n"
"sdot z26.s, z21.b, z2.b[1]\n"
@@ -2972,7 +3344,6 @@
"sdot z29.s, z21.b, z5.b[1]\n"
"sdot z30.s, z21.b, z6.b[1]\n"
"sdot z31.s, z21.b, z7.b[1]\n"
- "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
"sdot z24.s, z22.b, z0.b[2]\n"
"sdot z25.s, z22.b, z1.b[2]\n"
"sdot z26.s, z22.b, z2.b[2]\n"
@@ -2981,7 +3352,6 @@
"sdot z29.s, z22.b, z5.b[2]\n"
"sdot z30.s, z22.b, z6.b[2]\n"
"sdot z31.s, z22.b, z7.b[2]\n"
- "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
"sdot z24.s, z23.b, z0.b[3]\n"
"ld1rqb z0.b, p6/z, [%[a_ptr0], #0x20]\n"
"sdot z25.s, z23.b, z1.b[3]\n"
@@ -2999,7 +3369,6 @@
"sdot z31.s, z23.b, z7.b[3]\n"
"ld1rqb z7.b, p6/z, [a_ptr7, #0x20]\n"
"sdot z24.s, z16.b, z0.b[0]\n"
- "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"sdot z25.s, z16.b, z1.b[0]\n"
"sdot z26.s, z16.b, z2.b[0]\n"
"sdot z27.s, z16.b, z3.b[0]\n"
@@ -3007,55 +3376,62 @@
"sdot z29.s, z16.b, z5.b[0]\n"
"sdot z30.s, z16.b, z6.b[0]\n"
"sdot z31.s, z16.b, z7.b[0]\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
"b.ne 4b\n"
"3:\n"
"st1w z24.s, p7, [%[c_ptr0]]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "mov z24.s, #0\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "ld1rqb z1.b, p7/z, [a_ptr1]\n"
"addvl %[c_ptr0], %[c_ptr0], #1\n"
+ "mov z24.s, #0\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"st1w z25.s, p7, [c_ptr1]\n"
"addvl c_ptr1, c_ptr1, #1\n"
"mov z25.s, #0\n"
+ "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "st1w z26.s, p7, [c_ptr2]\n"
+ "addvl c_ptr2, c_ptr2, #1\n"
+ "mov z26.s, #0\n"
+ "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "st1w z27.s, p7, [c_ptr3]\n"
+ "addvl c_ptr3, c_ptr3, #1\n"
+ "mov z27.s, #0\n"
+ "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #8\n"
+ "st1w z28.s, p7, [c_ptr4]\n"
+ "addvl c_ptr4, c_ptr4, #1\n"
+ "mov z28.s, #0\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "st1w z29.s, p7, [c_ptr5]\n"
+ "addvl c_ptr5, c_ptr5, #1\n"
+ "mov z29.s, #0\n"
"ld1rqb z2.b, p7/z, [a_ptr2]\n"
"sdot z24.s, z16.b, z0.b[0]\n"
- "st1w z26.s, p7, [c_ptr2]\n"
- "mov z26.s, #0\n"
- "ld1rqb z3.b, p7/z, [a_ptr3]\n"
- "ld1rqb z4.b, p7/z, [a_ptr4]\n"
- "addvl c_ptr2, c_ptr2, #1\n"
- "sdot z25.s, z16.b, z1.b[0]\n"
- "st1w z27.s, p7, [c_ptr3]\n"
- "mov z27.s, #0\n"
- "ld1rqb z5.b, p7/z, [a_ptr5]\n"
- "sdot z26.s, z16.b, z2.b[0]\n"
- "ld1rqb z6.b, p7/z, [a_ptr6]\n"
- "sdot z24.s, z17.b, z0.b[1]\n"
- "st1w z28.s, p7, [c_ptr4]\n"
- "mov z28.s, #0\n"
- "ld1rqb z7.b, p7/z, [a_ptr7]\n"
- "sdot z27.s, z16.b, z3.b[0]\n"
- "addvl c_ptr3, c_ptr3, #1\n"
- "sdot z25.s, z17.b, z1.b[1]\n"
- "st1w z29.s, p7, [c_ptr5]\n"
- "mov z29.s, #0\n"
- "addvl c_ptr4, c_ptr4, #1\n"
- "sdot z28.s, z16.b, z4.b[0]\n"
"st1w z30.s, p7, [c_ptr6]\n"
"mov z30.s, #0\n"
- "addvl c_ptr5, c_ptr5, #1\n"
- "sdot z29.s, z16.b, z5.b[0]\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "sdot z25.s, z16.b, z1.b[0]\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "sdot z26.s, z16.b, z2.b[0]\n"
"st1w z31.s, p7, [c_ptr7]\n"
"mov z31.s, #0\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "sdot z27.s, z16.b, z3.b[0]\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "sdot z28.s, z16.b, z4.b[0]\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "sdot z24.s, z17.b, z0.b[1]\n"
"addvl c_ptr6, c_ptr6, #1\n"
- "sdot z30.s, z16.b, z6.b[0]\n"
+ "sdot z29.s, z16.b, z5.b[0]\n"
"addvl c_ptr7, c_ptr7, #1\n"
+ "sdot z30.s, z16.b, z6.b[0]\n"
"sdot z31.s, z16.b, z7.b[0]\n"
"ld1b z16.b, p7/z, [%[b_ptr0]]\n"
- "sdot z26.s, z17.b, z2.b[1]\n"
+ "sdot z25.s, z17.b, z1.b[1]\n"
"addvl %[b_ptr0], %[b_ptr0], #1\n"
+ "sdot z26.s, z17.b, z2.b[1]\n"
"sdot z27.s, z17.b, z3.b[1]\n"
"sdot z28.s, z17.b, z4.b[1]\n"
"sdot z29.s, z17.b, z5.b[1]\n"
@@ -3133,23 +3509,124 @@
"sdot z29.s, z16.b, z5.b[0]\n"
"sdot z30.s, z16.b, z6.b[0]\n"
"sdot z31.s, z16.b, z7.b[0]\n"
+ "b 5f\n"
"2:\n"
+ "mov z24.s, #0\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "mov z25.s, #0\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "mov z26.s, #0\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2]\n"
+ "mov z27.s, #0\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "mov z28.s, #0\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "mov z29.s, #0\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "mov z30.s, #0\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "mov z31.s, #0\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "sdot z24.s, z16.b, z0.b[0]\n"
+ "sdot z25.s, z16.b, z1.b[0]\n"
+ "sdot z26.s, z16.b, z2.b[0]\n"
+ "sdot z27.s, z16.b, z3.b[0]\n"
+ "sdot z28.s, z16.b, z4.b[0]\n"
+ "sdot z29.s, z16.b, z5.b[0]\n"
+ "sdot z30.s, z16.b, z6.b[0]\n"
+ "sdot z31.s, z16.b, z7.b[0]\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "sdot z24.s, z17.b, z0.b[1]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #1\n"
+ "sdot z25.s, z17.b, z1.b[1]\n"
+ "sdot z26.s, z17.b, z2.b[1]\n"
+ "sdot z27.s, z17.b, z3.b[1]\n"
+ "sdot z28.s, z17.b, z4.b[1]\n"
+ "sdot z29.s, z17.b, z5.b[1]\n"
+ "sdot z30.s, z17.b, z6.b[1]\n"
+ "sdot z31.s, z17.b, z7.b[1]\n"
+ "sdot z24.s, z18.b, z0.b[2]\n"
+ "sdot z25.s, z18.b, z1.b[2]\n"
+ "sdot z26.s, z18.b, z2.b[2]\n"
+ "sdot z27.s, z18.b, z3.b[2]\n"
+ "sdot z28.s, z18.b, z4.b[2]\n"
+ "sdot z29.s, z18.b, z5.b[2]\n"
+ "sdot z30.s, z18.b, z6.b[2]\n"
+ "sdot z31.s, z18.b, z7.b[2]\n"
+ "sdot z24.s, z19.b, z0.b[3]\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x10]\n"
+ "sdot z25.s, z19.b, z1.b[3]\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1, #0x10]\n"
+ "sdot z26.s, z19.b, z2.b[3]\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2, #0x10]\n"
+ "sdot z27.s, z19.b, z3.b[3]\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3, #0x10]\n"
+ "sdot z28.s, z19.b, z4.b[3]\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4, #0x10]\n"
+ "sdot z29.s, z19.b, z5.b[3]\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5, #0x10]\n"
+ "sdot z30.s, z19.b, z6.b[3]\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6, #0x10]\n"
+ "sdot z31.s, z19.b, z7.b[3]\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7, #0x10]\n"
+ "sdot z24.s, z20.b, z0.b[0]\n"
+ "sdot z25.s, z20.b, z1.b[0]\n"
+ "sdot z26.s, z20.b, z2.b[0]\n"
+ "sdot z27.s, z20.b, z3.b[0]\n"
+ "sdot z28.s, z20.b, z4.b[0]\n"
+ "sdot z29.s, z20.b, z5.b[0]\n"
+ "sdot z30.s, z20.b, z6.b[0]\n"
+ "sdot z31.s, z20.b, z7.b[0]\n"
+ "sdot z24.s, z21.b, z0.b[1]\n"
+ "sdot z25.s, z21.b, z1.b[1]\n"
+ "sdot z26.s, z21.b, z2.b[1]\n"
+ "sdot z27.s, z21.b, z3.b[1]\n"
+ "sdot z28.s, z21.b, z4.b[1]\n"
+ "sdot z29.s, z21.b, z5.b[1]\n"
+ "sdot z30.s, z21.b, z6.b[1]\n"
+ "sdot z31.s, z21.b, z7.b[1]\n"
+ "sdot z24.s, z22.b, z0.b[2]\n"
+ "sdot z25.s, z22.b, z1.b[2]\n"
+ "sdot z26.s, z22.b, z2.b[2]\n"
+ "sdot z27.s, z22.b, z3.b[2]\n"
+ "sdot z28.s, z22.b, z4.b[2]\n"
+ "sdot z29.s, z22.b, z5.b[2]\n"
+ "sdot z30.s, z22.b, z6.b[2]\n"
+ "sdot z31.s, z22.b, z7.b[2]\n"
+ "sdot z24.s, z23.b, z0.b[3]\n"
+ "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x20]\n"
+ "sdot z25.s, z23.b, z1.b[3]\n"
+ "ld1rqb z1.b, p6/z, [a_ptr1, #0x20]\n"
+ "sdot z26.s, z23.b, z2.b[3]\n"
+ "ld1rqb z2.b, p6/z, [a_ptr2, #0x20]\n"
+ "sdot z27.s, z23.b, z3.b[3]\n"
+ "ld1rqb z3.b, p6/z, [a_ptr3, #0x20]\n"
+ "sdot z28.s, z23.b, z4.b[3]\n"
+ "ld1rqb z4.b, p6/z, [a_ptr4, #0x20]\n"
+ "sdot z29.s, z23.b, z5.b[3]\n"
+ "ld1rqb z5.b, p6/z, [a_ptr5, #0x20]\n"
+ "sdot z30.s, z23.b, z6.b[3]\n"
+ "ld1rqb z6.b, p6/z, [a_ptr6, #0x20]\n"
+ "sdot z31.s, z23.b, z7.b[3]\n"
+ "ld1rqb z7.b, p6/z, [a_ptr7, #0x20]\n"
+ "sdot z24.s, z16.b, z0.b[0]\n"
+ "sdot z25.s, z16.b, z1.b[0]\n"
+ "sdot z26.s, z16.b, z2.b[0]\n"
+ "sdot z27.s, z16.b, z3.b[0]\n"
+ "sdot z28.s, z16.b, z4.b[0]\n"
+ "sdot z29.s, z16.b, z5.b[0]\n"
+ "sdot z30.s, z16.b, z6.b[0]\n"
+ "sdot z31.s, z16.b, z7.b[0]\n"
+ "5:\n"
"st1w z24.s, p0, [%[c_ptr0]]\n"
"addvl %[c_ptr0], %[c_ptr0], #1\n"
"st1w z25.s, p0, [c_ptr1]\n"
- "addvl c_ptr1, c_ptr1, #1\n"
"st1w z26.s, p0, [c_ptr2]\n"
- "addvl c_ptr2, c_ptr2, #1\n"
"st1w z27.s, p0, [c_ptr3]\n"
- "addvl c_ptr3, c_ptr3, #1\n"
"st1w z28.s, p0, [c_ptr4]\n"
- "addvl c_ptr4, c_ptr4, #1\n"
"st1w z29.s, p0, [c_ptr5]\n"
- "addvl c_ptr5, c_ptr5, #1\n"
"st1w z30.s, p0, [c_ptr6]\n"
- "addvl c_ptr6, c_ptr6, #1\n"
"st1w z31.s, p0, [c_ptr7]\n"
- "addvl c_ptr7, c_ptr7, #1\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -3228,52 +3705,54 @@
"add c_ptr1, %[c_ptr0], #0x0\n"
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
- "mov z24.s, #0\n"
"ptrue p7.b\n"
- "mov z25.s, #0\n"
"whilelt p6.b, %[temp], %[odd_depth]\n"
- "mov z26.s, #0\n"
+ "whilelt p0.s, %[temp], %[last_width]\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
"ld1b z16.b, p7/z, [%[b_ptr0]]\n"
- "mov z27.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
"ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "mov z28.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
"ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "mov z29.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
"ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "mov z30.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
"ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "mov z31.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
"ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
"ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "whilelt p0.s, %[temp], %[last_width]\n"
"ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
- "ld1rqb z1.b, p7/z, [a_ptr1]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
- "sdot z24.s, z16.b, z0.b[0]\n"
- "ld1rqb z2.b, p7/z, [a_ptr2]\n"
- "ld1rqb z3.b, p7/z, [a_ptr3]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
- "sdot z25.s, z16.b, z1.b[0]\n"
- "ld1rqb z4.b, p7/z, [a_ptr4]\n"
- "sdot z26.s, z16.b, z2.b[0]\n"
- "ld1rqb z5.b, p7/z, [a_ptr5]\n"
- "sdot z27.s, z16.b, z3.b[0]\n"
- "ld1rqb z6.b, p7/z, [a_ptr6]\n"
- "sdot z24.s, z17.b, z0.b[1]\n"
- "ld1rqb z7.b, p7/z, [a_ptr7]\n"
- "sdot z28.s, z16.b, z4.b[0]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- "sdot z29.s, z16.b, z5.b[0]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- "sdot z30.s, z16.b, z6.b[0]\n"
"addvl %[b_ptr0], %[b_ptr0], #8\n"
+ "cbz %[loops], 2f\n"
+ "mov z24.s, #0\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "mov z25.s, #0\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "mov z26.s, #0\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2]\n"
+ "mov z27.s, #0\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "mov z28.s, #0\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "mov z29.s, #0\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "mov z30.s, #0\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "mov z31.s, #0\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "sdot z24.s, z16.b, z0.b[0]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "sdot z25.s, z16.b, z1.b[0]\n"
+ "sdot z26.s, z16.b, z2.b[0]\n"
+ "sdot z27.s, z16.b, z3.b[0]\n"
+ "sdot z28.s, z16.b, z4.b[0]\n"
+ "sdot z29.s, z16.b, z5.b[0]\n"
+ "sdot z30.s, z16.b, z6.b[0]\n"
"sdot z31.s, z16.b, z7.b[0]\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "sdot z24.s, z17.b, z0.b[1]\n"
"sdot z25.s, z17.b, z1.b[1]\n"
"sdot z26.s, z17.b, z2.b[1]\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
"sdot z27.s, z17.b, z3.b[1]\n"
"sdot z28.s, z17.b, z4.b[1]\n"
"sdot z29.s, z17.b, z5.b[1]\n"
@@ -3361,88 +3840,85 @@
"sdot z29.s, z17.b, z5.b[1]\n"
"sdot z30.s, z17.b, z6.b[1]\n"
"sdot z31.s, z17.b, z7.b[1]\n"
- "cbz %[loops], 2f\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"b.eq 3f\n"
"4:\n"
"st1w z24.s, p7, [%[c_ptr0]]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
+ "subs %[loops], %[loops], #0x1\n"
"mov z24.s, #0\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "addvl %[c_ptr0], %[c_ptr0], #1\n"
+ "st1w z25.s, p7, [c_ptr1]\n"
+ "addvl c_ptr1, c_ptr1, #1\n"
+ "mov z25.s, #0\n"
+ "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "st1w z26.s, p7, [c_ptr2]\n"
+ "addvl c_ptr2, c_ptr2, #1\n"
+ "mov z26.s, #0\n"
+ "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ "st1w z27.s, p7, [c_ptr3]\n"
+ "addvl c_ptr3, c_ptr3, #1\n"
+ "mov z27.s, #0\n"
+ "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #8\n"
+ "st1w z28.s, p7, [c_ptr4]\n"
+ "addvl c_ptr4, c_ptr4, #1\n"
+ "mov z28.s, #0\n"
"ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
"ld1rqb z1.b, p7/z, [a_ptr1]\n"
- "subs %[loops], %[loops], #0x1\n"
- "st1w z25.s, p7, [c_ptr1]\n"
- "addvl %[c_ptr0], %[c_ptr0], #1\n"
- "mov z25.s, #0\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ "st1w z29.s, p7, [c_ptr5]\n"
+ "addvl c_ptr5, c_ptr5, #1\n"
+ "mov z29.s, #0\n"
"ld1rqb z2.b, p7/z, [a_ptr2]\n"
"sdot z24.s, z16.b, z0.b[0]\n"
- "st1w z26.s, p7, [c_ptr2]\n"
- "mov z26.s, #0\n"
- "ld1rqb z3.b, p7/z, [a_ptr3]\n"
- "ld1rqb z4.b, p7/z, [a_ptr4]\n"
- "addvl c_ptr1, c_ptr1, #1\n"
- "sdot z25.s, z16.b, z1.b[0]\n"
- "st1w z27.s, p7, [c_ptr3]\n"
- "mov z27.s, #0\n"
- "ld1rqb z5.b, p7/z, [a_ptr5]\n"
- "sdot z26.s, z16.b, z2.b[0]\n"
- "ld1rqb z6.b, p7/z, [a_ptr6]\n"
- "sdot z24.s, z17.b, z0.b[1]\n"
- "st1w z28.s, p7, [c_ptr4]\n"
- "mov z28.s, #0\n"
- "ld1rqb z7.b, p7/z, [a_ptr7]\n"
- "sdot z27.s, z16.b, z3.b[0]\n"
- "addvl c_ptr2, c_ptr2, #1\n"
- "sdot z25.s, z17.b, z1.b[1]\n"
- "st1w z29.s, p7, [c_ptr5]\n"
- "mov z29.s, #0\n"
- "addvl c_ptr3, c_ptr3, #1\n"
- "sdot z28.s, z16.b, z4.b[0]\n"
"st1w z30.s, p7, [c_ptr6]\n"
"mov z30.s, #0\n"
- "addvl c_ptr4, c_ptr4, #1\n"
- "sdot z29.s, z16.b, z5.b[0]\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "sdot z25.s, z16.b, z1.b[0]\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "sdot z26.s, z16.b, z2.b[0]\n"
"st1w z31.s, p7, [c_ptr7]\n"
"mov z31.s, #0\n"
- "addvl c_ptr5, c_ptr5, #1\n"
- "sdot z30.s, z16.b, z6.b[0]\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "sdot z27.s, z16.b, z3.b[0]\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "sdot z28.s, z16.b, z4.b[0]\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "sdot z24.s, z17.b, z0.b[1]\n"
"addvl c_ptr6, c_ptr6, #1\n"
+ "sdot z29.s, z16.b, z5.b[0]\n"
+ "addvl c_ptr7, c_ptr7, #1\n"
+ "sdot z30.s, z16.b, z6.b[0]\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
"sdot z31.s, z16.b, z7.b[0]\n"
"ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "sdot z25.s, z17.b, z1.b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
"sdot z26.s, z17.b, z2.b[1]\n"
- "addvl c_ptr7, c_ptr7, #1\n"
+ "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
"sdot z27.s, z17.b, z3.b[1]\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
"sdot z28.s, z17.b, z4.b[1]\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ "prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
"sdot z29.s, z17.b, z5.b[1]\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
"sdot z30.s, z17.b, z6.b[1]\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
"sdot z31.s, z17.b, z7.b[1]\n"
"ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"sdot z24.s, z18.b, z0.b[2]\n"
- "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- "sdot z25.s, z18.b, z1.b[2]\n"
"addvl %[b_ptr0], %[b_ptr0], #2\n"
+ "sdot z25.s, z18.b, z1.b[2]\n"
"sdot z26.s, z18.b, z2.b[2]\n"
- "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
"sdot z27.s, z18.b, z3.b[2]\n"
- "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
"sdot z28.s, z18.b, z4.b[2]\n"
- "prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
"sdot z29.s, z18.b, z5.b[2]\n"
"sdot z30.s, z18.b, z6.b[2]\n"
"sdot z31.s, z18.b, z7.b[2]\n"
- "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"sdot z24.s, z19.b, z0.b[3]\n"
"ld1rqb z0.b, p7/z, [%[a_ptr0], #0x10]\n"
"sdot z25.s, z19.b, z1.b[3]\n"
@@ -3460,7 +3936,6 @@
"sdot z31.s, z19.b, z7.b[3]\n"
"ld1rqb z7.b, p7/z, [a_ptr7, #0x10]\n"
"sdot z24.s, z20.b, z0.b[0]\n"
- "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"sdot z25.s, z20.b, z1.b[0]\n"
"sdot z26.s, z20.b, z2.b[0]\n"
"sdot z27.s, z20.b, z3.b[0]\n"
@@ -3468,7 +3943,6 @@
"sdot z29.s, z20.b, z5.b[0]\n"
"sdot z30.s, z20.b, z6.b[0]\n"
"sdot z31.s, z20.b, z7.b[0]\n"
- "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
"sdot z24.s, z21.b, z0.b[1]\n"
"sdot z25.s, z21.b, z1.b[1]\n"
"sdot z26.s, z21.b, z2.b[1]\n"
@@ -3477,7 +3951,6 @@
"sdot z29.s, z21.b, z5.b[1]\n"
"sdot z30.s, z21.b, z6.b[1]\n"
"sdot z31.s, z21.b, z7.b[1]\n"
- "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
"sdot z24.s, z22.b, z0.b[2]\n"
"sdot z25.s, z22.b, z1.b[2]\n"
"sdot z26.s, z22.b, z2.b[2]\n"
@@ -3486,7 +3959,6 @@
"sdot z29.s, z22.b, z5.b[2]\n"
"sdot z30.s, z22.b, z6.b[2]\n"
"sdot z31.s, z22.b, z7.b[2]\n"
- "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
"sdot z24.s, z23.b, z0.b[3]\n"
"ld1rqb z0.b, p6/z, [%[a_ptr0], #0x20]\n"
"sdot z25.s, z23.b, z1.b[3]\n"
@@ -3504,7 +3976,6 @@
"sdot z31.s, z23.b, z7.b[3]\n"
"ld1rqb z7.b, p6/z, [a_ptr7, #0x20]\n"
"sdot z24.s, z16.b, z0.b[0]\n"
- "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"sdot z25.s, z16.b, z1.b[0]\n"
"sdot z26.s, z16.b, z2.b[0]\n"
"sdot z27.s, z16.b, z3.b[0]\n"
@@ -3512,7 +3983,6 @@
"sdot z29.s, z16.b, z5.b[0]\n"
"sdot z30.s, z16.b, z6.b[0]\n"
"sdot z31.s, z16.b, z7.b[0]\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
"sdot z24.s, z17.b, z0.b[1]\n"
"sdot z25.s, z17.b, z1.b[1]\n"
"sdot z26.s, z17.b, z2.b[1]\n"
@@ -3521,53 +3991,60 @@
"sdot z29.s, z17.b, z5.b[1]\n"
"sdot z30.s, z17.b, z6.b[1]\n"
"sdot z31.s, z17.b, z7.b[1]\n"
- "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"b.ne 4b\n"
"3:\n"
"st1w z24.s, p7, [%[c_ptr0]]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "mov z24.s, #0\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "ld1rqb z1.b, p7/z, [a_ptr1]\n"
"addvl %[c_ptr0], %[c_ptr0], #1\n"
+ "mov z24.s, #0\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"st1w z25.s, p7, [c_ptr1]\n"
"addvl c_ptr1, c_ptr1, #1\n"
"mov z25.s, #0\n"
+ "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "st1w z26.s, p7, [c_ptr2]\n"
+ "addvl c_ptr2, c_ptr2, #1\n"
+ "mov z26.s, #0\n"
+ "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "st1w z27.s, p7, [c_ptr3]\n"
+ "addvl c_ptr3, c_ptr3, #1\n"
+ "mov z27.s, #0\n"
+ "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #8\n"
+ "st1w z28.s, p7, [c_ptr4]\n"
+ "addvl c_ptr4, c_ptr4, #1\n"
+ "mov z28.s, #0\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "st1w z29.s, p7, [c_ptr5]\n"
+ "addvl c_ptr5, c_ptr5, #1\n"
+ "mov z29.s, #0\n"
"ld1rqb z2.b, p7/z, [a_ptr2]\n"
"sdot z24.s, z16.b, z0.b[0]\n"
- "st1w z26.s, p7, [c_ptr2]\n"
- "mov z26.s, #0\n"
- "ld1rqb z3.b, p7/z, [a_ptr3]\n"
- "ld1rqb z4.b, p7/z, [a_ptr4]\n"
- "addvl c_ptr2, c_ptr2, #1\n"
- "sdot z25.s, z16.b, z1.b[0]\n"
- "st1w z27.s, p7, [c_ptr3]\n"
- "mov z27.s, #0\n"
- "ld1rqb z5.b, p7/z, [a_ptr5]\n"
- "sdot z26.s, z16.b, z2.b[0]\n"
- "ld1rqb z6.b, p7/z, [a_ptr6]\n"
- "sdot z24.s, z17.b, z0.b[1]\n"
- "st1w z28.s, p7, [c_ptr4]\n"
- "mov z28.s, #0\n"
- "ld1rqb z7.b, p7/z, [a_ptr7]\n"
- "sdot z27.s, z16.b, z3.b[0]\n"
- "addvl c_ptr3, c_ptr3, #1\n"
- "sdot z25.s, z17.b, z1.b[1]\n"
- "st1w z29.s, p7, [c_ptr5]\n"
- "mov z29.s, #0\n"
- "addvl c_ptr4, c_ptr4, #1\n"
- "sdot z28.s, z16.b, z4.b[0]\n"
"st1w z30.s, p7, [c_ptr6]\n"
"mov z30.s, #0\n"
- "addvl c_ptr5, c_ptr5, #1\n"
- "sdot z29.s, z16.b, z5.b[0]\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "sdot z25.s, z16.b, z1.b[0]\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "sdot z26.s, z16.b, z2.b[0]\n"
"st1w z31.s, p7, [c_ptr7]\n"
"mov z31.s, #0\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "sdot z27.s, z16.b, z3.b[0]\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "sdot z28.s, z16.b, z4.b[0]\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "sdot z24.s, z17.b, z0.b[1]\n"
"addvl c_ptr6, c_ptr6, #1\n"
- "sdot z30.s, z16.b, z6.b[0]\n"
+ "sdot z29.s, z16.b, z5.b[0]\n"
"addvl c_ptr7, c_ptr7, #1\n"
+ "sdot z30.s, z16.b, z6.b[0]\n"
"sdot z31.s, z16.b, z7.b[0]\n"
"ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "sdot z25.s, z17.b, z1.b[1]\n"
"sdot z26.s, z17.b, z2.b[1]\n"
"sdot z27.s, z17.b, z3.b[1]\n"
"sdot z28.s, z17.b, z4.b[1]\n"
@@ -3656,23 +4133,133 @@
"sdot z29.s, z17.b, z5.b[1]\n"
"sdot z30.s, z17.b, z6.b[1]\n"
"sdot z31.s, z17.b, z7.b[1]\n"
+ "b 5f\n"
"2:\n"
+ "mov z24.s, #0\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "mov z25.s, #0\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "mov z26.s, #0\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2]\n"
+ "mov z27.s, #0\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "mov z28.s, #0\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "mov z29.s, #0\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "mov z30.s, #0\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "mov z31.s, #0\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "sdot z24.s, z16.b, z0.b[0]\n"
+ "sdot z25.s, z16.b, z1.b[0]\n"
+ "sdot z26.s, z16.b, z2.b[0]\n"
+ "sdot z27.s, z16.b, z3.b[0]\n"
+ "sdot z28.s, z16.b, z4.b[0]\n"
+ "sdot z29.s, z16.b, z5.b[0]\n"
+ "sdot z30.s, z16.b, z6.b[0]\n"
+ "sdot z31.s, z16.b, z7.b[0]\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "sdot z24.s, z17.b, z0.b[1]\n"
+ "sdot z25.s, z17.b, z1.b[1]\n"
+ "sdot z26.s, z17.b, z2.b[1]\n"
+ "sdot z27.s, z17.b, z3.b[1]\n"
+ "sdot z28.s, z17.b, z4.b[1]\n"
+ "sdot z29.s, z17.b, z5.b[1]\n"
+ "sdot z30.s, z17.b, z6.b[1]\n"
+ "sdot z31.s, z17.b, z7.b[1]\n"
+ "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "sdot z24.s, z18.b, z0.b[2]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #2\n"
+ "sdot z25.s, z18.b, z1.b[2]\n"
+ "sdot z26.s, z18.b, z2.b[2]\n"
+ "sdot z27.s, z18.b, z3.b[2]\n"
+ "sdot z28.s, z18.b, z4.b[2]\n"
+ "sdot z29.s, z18.b, z5.b[2]\n"
+ "sdot z30.s, z18.b, z6.b[2]\n"
+ "sdot z31.s, z18.b, z7.b[2]\n"
+ "sdot z24.s, z19.b, z0.b[3]\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x10]\n"
+ "sdot z25.s, z19.b, z1.b[3]\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1, #0x10]\n"
+ "sdot z26.s, z19.b, z2.b[3]\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2, #0x10]\n"
+ "sdot z27.s, z19.b, z3.b[3]\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3, #0x10]\n"
+ "sdot z28.s, z19.b, z4.b[3]\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4, #0x10]\n"
+ "sdot z29.s, z19.b, z5.b[3]\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5, #0x10]\n"
+ "sdot z30.s, z19.b, z6.b[3]\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6, #0x10]\n"
+ "sdot z31.s, z19.b, z7.b[3]\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7, #0x10]\n"
+ "sdot z24.s, z20.b, z0.b[0]\n"
+ "sdot z25.s, z20.b, z1.b[0]\n"
+ "sdot z26.s, z20.b, z2.b[0]\n"
+ "sdot z27.s, z20.b, z3.b[0]\n"
+ "sdot z28.s, z20.b, z4.b[0]\n"
+ "sdot z29.s, z20.b, z5.b[0]\n"
+ "sdot z30.s, z20.b, z6.b[0]\n"
+ "sdot z31.s, z20.b, z7.b[0]\n"
+ "sdot z24.s, z21.b, z0.b[1]\n"
+ "sdot z25.s, z21.b, z1.b[1]\n"
+ "sdot z26.s, z21.b, z2.b[1]\n"
+ "sdot z27.s, z21.b, z3.b[1]\n"
+ "sdot z28.s, z21.b, z4.b[1]\n"
+ "sdot z29.s, z21.b, z5.b[1]\n"
+ "sdot z30.s, z21.b, z6.b[1]\n"
+ "sdot z31.s, z21.b, z7.b[1]\n"
+ "sdot z24.s, z22.b, z0.b[2]\n"
+ "sdot z25.s, z22.b, z1.b[2]\n"
+ "sdot z26.s, z22.b, z2.b[2]\n"
+ "sdot z27.s, z22.b, z3.b[2]\n"
+ "sdot z28.s, z22.b, z4.b[2]\n"
+ "sdot z29.s, z22.b, z5.b[2]\n"
+ "sdot z30.s, z22.b, z6.b[2]\n"
+ "sdot z31.s, z22.b, z7.b[2]\n"
+ "sdot z24.s, z23.b, z0.b[3]\n"
+ "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x20]\n"
+ "sdot z25.s, z23.b, z1.b[3]\n"
+ "ld1rqb z1.b, p6/z, [a_ptr1, #0x20]\n"
+ "sdot z26.s, z23.b, z2.b[3]\n"
+ "ld1rqb z2.b, p6/z, [a_ptr2, #0x20]\n"
+ "sdot z27.s, z23.b, z3.b[3]\n"
+ "ld1rqb z3.b, p6/z, [a_ptr3, #0x20]\n"
+ "sdot z28.s, z23.b, z4.b[3]\n"
+ "ld1rqb z4.b, p6/z, [a_ptr4, #0x20]\n"
+ "sdot z29.s, z23.b, z5.b[3]\n"
+ "ld1rqb z5.b, p6/z, [a_ptr5, #0x20]\n"
+ "sdot z30.s, z23.b, z6.b[3]\n"
+ "ld1rqb z6.b, p6/z, [a_ptr6, #0x20]\n"
+ "sdot z31.s, z23.b, z7.b[3]\n"
+ "ld1rqb z7.b, p6/z, [a_ptr7, #0x20]\n"
+ "sdot z24.s, z16.b, z0.b[0]\n"
+ "sdot z25.s, z16.b, z1.b[0]\n"
+ "sdot z26.s, z16.b, z2.b[0]\n"
+ "sdot z27.s, z16.b, z3.b[0]\n"
+ "sdot z28.s, z16.b, z4.b[0]\n"
+ "sdot z29.s, z16.b, z5.b[0]\n"
+ "sdot z30.s, z16.b, z6.b[0]\n"
+ "sdot z31.s, z16.b, z7.b[0]\n"
+ "sdot z24.s, z17.b, z0.b[1]\n"
+ "sdot z25.s, z17.b, z1.b[1]\n"
+ "sdot z26.s, z17.b, z2.b[1]\n"
+ "sdot z27.s, z17.b, z3.b[1]\n"
+ "sdot z28.s, z17.b, z4.b[1]\n"
+ "sdot z29.s, z17.b, z5.b[1]\n"
+ "sdot z30.s, z17.b, z6.b[1]\n"
+ "sdot z31.s, z17.b, z7.b[1]\n"
+ "5:\n"
"st1w z24.s, p0, [%[c_ptr0]]\n"
"addvl %[c_ptr0], %[c_ptr0], #1\n"
"st1w z25.s, p0, [c_ptr1]\n"
- "addvl c_ptr1, c_ptr1, #1\n"
"st1w z26.s, p0, [c_ptr2]\n"
- "addvl c_ptr2, c_ptr2, #1\n"
"st1w z27.s, p0, [c_ptr3]\n"
- "addvl c_ptr3, c_ptr3, #1\n"
"st1w z28.s, p0, [c_ptr4]\n"
- "addvl c_ptr4, c_ptr4, #1\n"
"st1w z29.s, p0, [c_ptr5]\n"
- "addvl c_ptr5, c_ptr5, #1\n"
"st1w z30.s, p0, [c_ptr6]\n"
- "addvl c_ptr6, c_ptr6, #1\n"
"st1w z31.s, p0, [c_ptr7]\n"
- "addvl c_ptr7, c_ptr7, #1\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -3751,52 +4338,54 @@
"add c_ptr1, %[c_ptr0], #0x0\n"
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
- "mov z24.s, #0\n"
"ptrue p7.b\n"
- "mov z25.s, #0\n"
"whilelt p6.b, %[temp], %[odd_depth]\n"
- "mov z26.s, #0\n"
+ "whilelt p0.s, %[temp], %[last_width]\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
"ld1b z16.b, p7/z, [%[b_ptr0]]\n"
- "mov z27.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
"ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "mov z28.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
"ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "mov z29.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
"ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "mov z30.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
"ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "mov z31.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
"ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
"ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "whilelt p0.s, %[temp], %[last_width]\n"
"ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
- "ld1rqb z1.b, p7/z, [a_ptr1]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
- "sdot z24.s, z16.b, z0.b[0]\n"
- "ld1rqb z2.b, p7/z, [a_ptr2]\n"
- "ld1rqb z3.b, p7/z, [a_ptr3]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
- "sdot z25.s, z16.b, z1.b[0]\n"
- "ld1rqb z4.b, p7/z, [a_ptr4]\n"
- "sdot z26.s, z16.b, z2.b[0]\n"
- "ld1rqb z5.b, p7/z, [a_ptr5]\n"
- "sdot z27.s, z16.b, z3.b[0]\n"
- "ld1rqb z6.b, p7/z, [a_ptr6]\n"
- "sdot z24.s, z17.b, z0.b[1]\n"
- "ld1rqb z7.b, p7/z, [a_ptr7]\n"
- "sdot z28.s, z16.b, z4.b[0]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- "sdot z29.s, z16.b, z5.b[0]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- "sdot z30.s, z16.b, z6.b[0]\n"
"addvl %[b_ptr0], %[b_ptr0], #8\n"
+ "cbz %[loops], 2f\n"
+ "mov z24.s, #0\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "mov z25.s, #0\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "mov z26.s, #0\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2]\n"
+ "mov z27.s, #0\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "mov z28.s, #0\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "mov z29.s, #0\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "mov z30.s, #0\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "mov z31.s, #0\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "sdot z24.s, z16.b, z0.b[0]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "sdot z25.s, z16.b, z1.b[0]\n"
+ "sdot z26.s, z16.b, z2.b[0]\n"
+ "sdot z27.s, z16.b, z3.b[0]\n"
+ "sdot z28.s, z16.b, z4.b[0]\n"
+ "sdot z29.s, z16.b, z5.b[0]\n"
+ "sdot z30.s, z16.b, z6.b[0]\n"
"sdot z31.s, z16.b, z7.b[0]\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "sdot z24.s, z17.b, z0.b[1]\n"
"sdot z25.s, z17.b, z1.b[1]\n"
"sdot z26.s, z17.b, z2.b[1]\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
"sdot z27.s, z17.b, z3.b[1]\n"
"sdot z28.s, z17.b, z4.b[1]\n"
"sdot z29.s, z17.b, z5.b[1]\n"
@@ -3893,82 +4482,80 @@
"sdot z29.s, z18.b, z5.b[2]\n"
"sdot z30.s, z18.b, z6.b[2]\n"
"sdot z31.s, z18.b, z7.b[2]\n"
- "cbz %[loops], 2f\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"b.eq 3f\n"
"4:\n"
"st1w z24.s, p7, [%[c_ptr0]]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
+ "subs %[loops], %[loops], #0x1\n"
"mov z24.s, #0\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "addvl %[c_ptr0], %[c_ptr0], #1\n"
+ "st1w z25.s, p7, [c_ptr1]\n"
+ "addvl c_ptr1, c_ptr1, #1\n"
+ "mov z25.s, #0\n"
+ "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "st1w z26.s, p7, [c_ptr2]\n"
+ "addvl c_ptr2, c_ptr2, #1\n"
+ "mov z26.s, #0\n"
+ "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ "st1w z27.s, p7, [c_ptr3]\n"
+ "addvl c_ptr3, c_ptr3, #1\n"
+ "mov z27.s, #0\n"
+ "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #8\n"
+ "st1w z28.s, p7, [c_ptr4]\n"
+ "addvl c_ptr4, c_ptr4, #1\n"
+ "mov z28.s, #0\n"
"ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
"ld1rqb z1.b, p7/z, [a_ptr1]\n"
- "subs %[loops], %[loops], #0x1\n"
- "st1w z25.s, p7, [c_ptr1]\n"
- "addvl %[c_ptr0], %[c_ptr0], #1\n"
- "mov z25.s, #0\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ "st1w z29.s, p7, [c_ptr5]\n"
+ "addvl c_ptr5, c_ptr5, #1\n"
+ "mov z29.s, #0\n"
"ld1rqb z2.b, p7/z, [a_ptr2]\n"
"sdot z24.s, z16.b, z0.b[0]\n"
- "st1w z26.s, p7, [c_ptr2]\n"
- "mov z26.s, #0\n"
- "ld1rqb z3.b, p7/z, [a_ptr3]\n"
- "ld1rqb z4.b, p7/z, [a_ptr4]\n"
- "addvl c_ptr1, c_ptr1, #1\n"
- "sdot z25.s, z16.b, z1.b[0]\n"
- "st1w z27.s, p7, [c_ptr3]\n"
- "mov z27.s, #0\n"
- "ld1rqb z5.b, p7/z, [a_ptr5]\n"
- "sdot z26.s, z16.b, z2.b[0]\n"
- "ld1rqb z6.b, p7/z, [a_ptr6]\n"
- "sdot z24.s, z17.b, z0.b[1]\n"
- "st1w z28.s, p7, [c_ptr4]\n"
- "mov z28.s, #0\n"
- "ld1rqb z7.b, p7/z, [a_ptr7]\n"
- "sdot z27.s, z16.b, z3.b[0]\n"
- "addvl c_ptr2, c_ptr2, #1\n"
- "sdot z25.s, z17.b, z1.b[1]\n"
- "st1w z29.s, p7, [c_ptr5]\n"
- "mov z29.s, #0\n"
- "addvl c_ptr3, c_ptr3, #1\n"
- "sdot z28.s, z16.b, z4.b[0]\n"
"st1w z30.s, p7, [c_ptr6]\n"
"mov z30.s, #0\n"
- "addvl c_ptr4, c_ptr4, #1\n"
- "sdot z29.s, z16.b, z5.b[0]\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "sdot z25.s, z16.b, z1.b[0]\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "sdot z26.s, z16.b, z2.b[0]\n"
"st1w z31.s, p7, [c_ptr7]\n"
"mov z31.s, #0\n"
- "addvl c_ptr5, c_ptr5, #1\n"
- "sdot z30.s, z16.b, z6.b[0]\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "sdot z27.s, z16.b, z3.b[0]\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "sdot z28.s, z16.b, z4.b[0]\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "sdot z24.s, z17.b, z0.b[1]\n"
"addvl c_ptr6, c_ptr6, #1\n"
+ "sdot z29.s, z16.b, z5.b[0]\n"
+ "addvl c_ptr7, c_ptr7, #1\n"
+ "sdot z30.s, z16.b, z6.b[0]\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
"sdot z31.s, z16.b, z7.b[0]\n"
"ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "sdot z25.s, z17.b, z1.b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
"sdot z26.s, z17.b, z2.b[1]\n"
- "addvl c_ptr7, c_ptr7, #1\n"
+ "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
"sdot z27.s, z17.b, z3.b[1]\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
"sdot z28.s, z17.b, z4.b[1]\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ "prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
"sdot z29.s, z17.b, z5.b[1]\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
"sdot z30.s, z17.b, z6.b[1]\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
"sdot z31.s, z17.b, z7.b[1]\n"
"ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"sdot z24.s, z18.b, z0.b[2]\n"
- "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
"sdot z25.s, z18.b, z1.b[2]\n"
- "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
"sdot z26.s, z18.b, z2.b[2]\n"
- "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
"sdot z27.s, z18.b, z3.b[2]\n"
- "prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
"sdot z28.s, z18.b, z4.b[2]\n"
"sdot z29.s, z18.b, z5.b[2]\n"
"sdot z30.s, z18.b, z6.b[2]\n"
@@ -3995,12 +4582,10 @@
"sdot z25.s, z20.b, z1.b[0]\n"
"sdot z26.s, z20.b, z2.b[0]\n"
"sdot z27.s, z20.b, z3.b[0]\n"
- "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"sdot z28.s, z20.b, z4.b[0]\n"
"sdot z29.s, z20.b, z5.b[0]\n"
"sdot z30.s, z20.b, z6.b[0]\n"
"sdot z31.s, z20.b, z7.b[0]\n"
- "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
"sdot z24.s, z21.b, z0.b[1]\n"
"sdot z25.s, z21.b, z1.b[1]\n"
"sdot z26.s, z21.b, z2.b[1]\n"
@@ -4009,7 +4594,6 @@
"sdot z29.s, z21.b, z5.b[1]\n"
"sdot z30.s, z21.b, z6.b[1]\n"
"sdot z31.s, z21.b, z7.b[1]\n"
- "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
"sdot z24.s, z22.b, z0.b[2]\n"
"sdot z25.s, z22.b, z1.b[2]\n"
"sdot z26.s, z22.b, z2.b[2]\n"
@@ -4018,7 +4602,6 @@
"sdot z29.s, z22.b, z5.b[2]\n"
"sdot z30.s, z22.b, z6.b[2]\n"
"sdot z31.s, z22.b, z7.b[2]\n"
- "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
"sdot z24.s, z23.b, z0.b[3]\n"
"ld1rqb z0.b, p6/z, [%[a_ptr0], #0x20]\n"
"sdot z25.s, z23.b, z1.b[3]\n"
@@ -4036,7 +4619,6 @@
"sdot z31.s, z23.b, z7.b[3]\n"
"ld1rqb z7.b, p6/z, [a_ptr7, #0x20]\n"
"sdot z24.s, z16.b, z0.b[0]\n"
- "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"sdot z25.s, z16.b, z1.b[0]\n"
"sdot z26.s, z16.b, z2.b[0]\n"
"sdot z27.s, z16.b, z3.b[0]\n"
@@ -4044,7 +4626,6 @@
"sdot z29.s, z16.b, z5.b[0]\n"
"sdot z30.s, z16.b, z6.b[0]\n"
"sdot z31.s, z16.b, z7.b[0]\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
"sdot z24.s, z17.b, z0.b[1]\n"
"sdot z25.s, z17.b, z1.b[1]\n"
"sdot z26.s, z17.b, z2.b[1]\n"
@@ -4053,7 +4634,6 @@
"sdot z29.s, z17.b, z5.b[1]\n"
"sdot z30.s, z17.b, z6.b[1]\n"
"sdot z31.s, z17.b, z7.b[1]\n"
- "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"sdot z24.s, z18.b, z0.b[2]\n"
"sdot z25.s, z18.b, z1.b[2]\n"
"sdot z26.s, z18.b, z2.b[2]\n"
@@ -4062,53 +4642,60 @@
"sdot z29.s, z18.b, z5.b[2]\n"
"sdot z30.s, z18.b, z6.b[2]\n"
"sdot z31.s, z18.b, z7.b[2]\n"
- "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"b.ne 4b\n"
"3:\n"
"st1w z24.s, p7, [%[c_ptr0]]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "mov z24.s, #0\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "ld1rqb z1.b, p7/z, [a_ptr1]\n"
"addvl %[c_ptr0], %[c_ptr0], #1\n"
+ "mov z24.s, #0\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"st1w z25.s, p7, [c_ptr1]\n"
"addvl c_ptr1, c_ptr1, #1\n"
"mov z25.s, #0\n"
+ "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "st1w z26.s, p7, [c_ptr2]\n"
+ "addvl c_ptr2, c_ptr2, #1\n"
+ "mov z26.s, #0\n"
+ "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "st1w z27.s, p7, [c_ptr3]\n"
+ "addvl c_ptr3, c_ptr3, #1\n"
+ "mov z27.s, #0\n"
+ "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #8\n"
+ "st1w z28.s, p7, [c_ptr4]\n"
+ "addvl c_ptr4, c_ptr4, #1\n"
+ "mov z28.s, #0\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "st1w z29.s, p7, [c_ptr5]\n"
+ "addvl c_ptr5, c_ptr5, #1\n"
+ "mov z29.s, #0\n"
"ld1rqb z2.b, p7/z, [a_ptr2]\n"
"sdot z24.s, z16.b, z0.b[0]\n"
- "st1w z26.s, p7, [c_ptr2]\n"
- "mov z26.s, #0\n"
- "ld1rqb z3.b, p7/z, [a_ptr3]\n"
- "ld1rqb z4.b, p7/z, [a_ptr4]\n"
- "addvl c_ptr2, c_ptr2, #1\n"
- "sdot z25.s, z16.b, z1.b[0]\n"
- "st1w z27.s, p7, [c_ptr3]\n"
- "mov z27.s, #0\n"
- "ld1rqb z5.b, p7/z, [a_ptr5]\n"
- "sdot z26.s, z16.b, z2.b[0]\n"
- "ld1rqb z6.b, p7/z, [a_ptr6]\n"
- "sdot z24.s, z17.b, z0.b[1]\n"
- "st1w z28.s, p7, [c_ptr4]\n"
- "mov z28.s, #0\n"
- "ld1rqb z7.b, p7/z, [a_ptr7]\n"
- "sdot z27.s, z16.b, z3.b[0]\n"
- "addvl c_ptr3, c_ptr3, #1\n"
- "sdot z25.s, z17.b, z1.b[1]\n"
- "st1w z29.s, p7, [c_ptr5]\n"
- "mov z29.s, #0\n"
- "addvl c_ptr4, c_ptr4, #1\n"
- "sdot z28.s, z16.b, z4.b[0]\n"
"st1w z30.s, p7, [c_ptr6]\n"
"mov z30.s, #0\n"
- "addvl c_ptr5, c_ptr5, #1\n"
- "sdot z29.s, z16.b, z5.b[0]\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "sdot z25.s, z16.b, z1.b[0]\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "sdot z26.s, z16.b, z2.b[0]\n"
"st1w z31.s, p7, [c_ptr7]\n"
"mov z31.s, #0\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "sdot z27.s, z16.b, z3.b[0]\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "sdot z28.s, z16.b, z4.b[0]\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "sdot z24.s, z17.b, z0.b[1]\n"
"addvl c_ptr6, c_ptr6, #1\n"
- "sdot z30.s, z16.b, z6.b[0]\n"
+ "sdot z29.s, z16.b, z5.b[0]\n"
"addvl c_ptr7, c_ptr7, #1\n"
+ "sdot z30.s, z16.b, z6.b[0]\n"
"sdot z31.s, z16.b, z7.b[0]\n"
"ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "sdot z25.s, z17.b, z1.b[1]\n"
"sdot z26.s, z17.b, z2.b[1]\n"
"sdot z27.s, z17.b, z3.b[1]\n"
"sdot z28.s, z17.b, z4.b[1]\n"
@@ -4206,23 +4793,142 @@
"sdot z29.s, z18.b, z5.b[2]\n"
"sdot z30.s, z18.b, z6.b[2]\n"
"sdot z31.s, z18.b, z7.b[2]\n"
+ "b 5f\n"
"2:\n"
+ "mov z24.s, #0\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "mov z25.s, #0\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "mov z26.s, #0\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2]\n"
+ "mov z27.s, #0\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "mov z28.s, #0\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "mov z29.s, #0\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "mov z30.s, #0\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "mov z31.s, #0\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "sdot z24.s, z16.b, z0.b[0]\n"
+ "sdot z25.s, z16.b, z1.b[0]\n"
+ "sdot z26.s, z16.b, z2.b[0]\n"
+ "sdot z27.s, z16.b, z3.b[0]\n"
+ "sdot z28.s, z16.b, z4.b[0]\n"
+ "sdot z29.s, z16.b, z5.b[0]\n"
+ "sdot z30.s, z16.b, z6.b[0]\n"
+ "sdot z31.s, z16.b, z7.b[0]\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "sdot z24.s, z17.b, z0.b[1]\n"
+ "sdot z25.s, z17.b, z1.b[1]\n"
+ "sdot z26.s, z17.b, z2.b[1]\n"
+ "sdot z27.s, z17.b, z3.b[1]\n"
+ "sdot z28.s, z17.b, z4.b[1]\n"
+ "sdot z29.s, z17.b, z5.b[1]\n"
+ "sdot z30.s, z17.b, z6.b[1]\n"
+ "sdot z31.s, z17.b, z7.b[1]\n"
+ "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "sdot z24.s, z18.b, z0.b[2]\n"
+ "sdot z25.s, z18.b, z1.b[2]\n"
+ "sdot z26.s, z18.b, z2.b[2]\n"
+ "sdot z27.s, z18.b, z3.b[2]\n"
+ "sdot z28.s, z18.b, z4.b[2]\n"
+ "sdot z29.s, z18.b, z5.b[2]\n"
+ "sdot z30.s, z18.b, z6.b[2]\n"
+ "sdot z31.s, z18.b, z7.b[2]\n"
+ "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "sdot z24.s, z19.b, z0.b[3]\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x10]\n"
+ "sdot z25.s, z19.b, z1.b[3]\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1, #0x10]\n"
+ "sdot z26.s, z19.b, z2.b[3]\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2, #0x10]\n"
+ "sdot z27.s, z19.b, z3.b[3]\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3, #0x10]\n"
+ "sdot z28.s, z19.b, z4.b[3]\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4, #0x10]\n"
+ "sdot z29.s, z19.b, z5.b[3]\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5, #0x10]\n"
+ "sdot z30.s, z19.b, z6.b[3]\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6, #0x10]\n"
+ "sdot z31.s, z19.b, z7.b[3]\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7, #0x10]\n"
+ "sdot z24.s, z20.b, z0.b[0]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #3\n"
+ "sdot z25.s, z20.b, z1.b[0]\n"
+ "sdot z26.s, z20.b, z2.b[0]\n"
+ "sdot z27.s, z20.b, z3.b[0]\n"
+ "sdot z28.s, z20.b, z4.b[0]\n"
+ "sdot z29.s, z20.b, z5.b[0]\n"
+ "sdot z30.s, z20.b, z6.b[0]\n"
+ "sdot z31.s, z20.b, z7.b[0]\n"
+ "sdot z24.s, z21.b, z0.b[1]\n"
+ "sdot z25.s, z21.b, z1.b[1]\n"
+ "sdot z26.s, z21.b, z2.b[1]\n"
+ "sdot z27.s, z21.b, z3.b[1]\n"
+ "sdot z28.s, z21.b, z4.b[1]\n"
+ "sdot z29.s, z21.b, z5.b[1]\n"
+ "sdot z30.s, z21.b, z6.b[1]\n"
+ "sdot z31.s, z21.b, z7.b[1]\n"
+ "sdot z24.s, z22.b, z0.b[2]\n"
+ "sdot z25.s, z22.b, z1.b[2]\n"
+ "sdot z26.s, z22.b, z2.b[2]\n"
+ "sdot z27.s, z22.b, z3.b[2]\n"
+ "sdot z28.s, z22.b, z4.b[2]\n"
+ "sdot z29.s, z22.b, z5.b[2]\n"
+ "sdot z30.s, z22.b, z6.b[2]\n"
+ "sdot z31.s, z22.b, z7.b[2]\n"
+ "sdot z24.s, z23.b, z0.b[3]\n"
+ "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x20]\n"
+ "sdot z25.s, z23.b, z1.b[3]\n"
+ "ld1rqb z1.b, p6/z, [a_ptr1, #0x20]\n"
+ "sdot z26.s, z23.b, z2.b[3]\n"
+ "ld1rqb z2.b, p6/z, [a_ptr2, #0x20]\n"
+ "sdot z27.s, z23.b, z3.b[3]\n"
+ "ld1rqb z3.b, p6/z, [a_ptr3, #0x20]\n"
+ "sdot z28.s, z23.b, z4.b[3]\n"
+ "ld1rqb z4.b, p6/z, [a_ptr4, #0x20]\n"
+ "sdot z29.s, z23.b, z5.b[3]\n"
+ "ld1rqb z5.b, p6/z, [a_ptr5, #0x20]\n"
+ "sdot z30.s, z23.b, z6.b[3]\n"
+ "ld1rqb z6.b, p6/z, [a_ptr6, #0x20]\n"
+ "sdot z31.s, z23.b, z7.b[3]\n"
+ "ld1rqb z7.b, p6/z, [a_ptr7, #0x20]\n"
+ "sdot z24.s, z16.b, z0.b[0]\n"
+ "sdot z25.s, z16.b, z1.b[0]\n"
+ "sdot z26.s, z16.b, z2.b[0]\n"
+ "sdot z27.s, z16.b, z3.b[0]\n"
+ "sdot z28.s, z16.b, z4.b[0]\n"
+ "sdot z29.s, z16.b, z5.b[0]\n"
+ "sdot z30.s, z16.b, z6.b[0]\n"
+ "sdot z31.s, z16.b, z7.b[0]\n"
+ "sdot z24.s, z17.b, z0.b[1]\n"
+ "sdot z25.s, z17.b, z1.b[1]\n"
+ "sdot z26.s, z17.b, z2.b[1]\n"
+ "sdot z27.s, z17.b, z3.b[1]\n"
+ "sdot z28.s, z17.b, z4.b[1]\n"
+ "sdot z29.s, z17.b, z5.b[1]\n"
+ "sdot z30.s, z17.b, z6.b[1]\n"
+ "sdot z31.s, z17.b, z7.b[1]\n"
+ "sdot z24.s, z18.b, z0.b[2]\n"
+ "sdot z25.s, z18.b, z1.b[2]\n"
+ "sdot z26.s, z18.b, z2.b[2]\n"
+ "sdot z27.s, z18.b, z3.b[2]\n"
+ "sdot z28.s, z18.b, z4.b[2]\n"
+ "sdot z29.s, z18.b, z5.b[2]\n"
+ "sdot z30.s, z18.b, z6.b[2]\n"
+ "sdot z31.s, z18.b, z7.b[2]\n"
+ "5:\n"
"st1w z24.s, p0, [%[c_ptr0]]\n"
"addvl %[c_ptr0], %[c_ptr0], #1\n"
"st1w z25.s, p0, [c_ptr1]\n"
- "addvl c_ptr1, c_ptr1, #1\n"
"st1w z26.s, p0, [c_ptr2]\n"
- "addvl c_ptr2, c_ptr2, #1\n"
"st1w z27.s, p0, [c_ptr3]\n"
- "addvl c_ptr3, c_ptr3, #1\n"
"st1w z28.s, p0, [c_ptr4]\n"
- "addvl c_ptr4, c_ptr4, #1\n"
"st1w z29.s, p0, [c_ptr5]\n"
- "addvl c_ptr5, c_ptr5, #1\n"
"st1w z30.s, p0, [c_ptr6]\n"
- "addvl c_ptr6, c_ptr6, #1\n"
"st1w z31.s, p0, [c_ptr7]\n"
- "addvl c_ptr7, c_ptr7, #1\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -4301,52 +5007,54 @@
"add c_ptr1, %[c_ptr0], #0x0\n"
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
- "mov z24.s, #0\n"
"ptrue p7.b\n"
- "mov z25.s, #0\n"
"whilelt p6.b, %[temp], %[odd_depth]\n"
- "mov z26.s, #0\n"
+ "whilelt p0.s, %[temp], %[last_width]\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
"ld1b z16.b, p7/z, [%[b_ptr0]]\n"
- "mov z27.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
"ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "mov z28.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
"ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "mov z29.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
"ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "mov z30.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
"ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "mov z31.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
"ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
"ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "whilelt p0.s, %[temp], %[last_width]\n"
"ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
- "ld1rqb z1.b, p7/z, [a_ptr1]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
- "sdot z24.s, z16.b, z0.b[0]\n"
- "ld1rqb z2.b, p7/z, [a_ptr2]\n"
- "ld1rqb z3.b, p7/z, [a_ptr3]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
- "sdot z25.s, z16.b, z1.b[0]\n"
- "ld1rqb z4.b, p7/z, [a_ptr4]\n"
- "sdot z26.s, z16.b, z2.b[0]\n"
- "ld1rqb z5.b, p7/z, [a_ptr5]\n"
- "sdot z27.s, z16.b, z3.b[0]\n"
- "ld1rqb z6.b, p7/z, [a_ptr6]\n"
- "sdot z24.s, z17.b, z0.b[1]\n"
- "ld1rqb z7.b, p7/z, [a_ptr7]\n"
- "sdot z28.s, z16.b, z4.b[0]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- "sdot z29.s, z16.b, z5.b[0]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- "sdot z30.s, z16.b, z6.b[0]\n"
"addvl %[b_ptr0], %[b_ptr0], #8\n"
+ "cbz %[loops], 2f\n"
+ "mov z24.s, #0\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "mov z25.s, #0\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "mov z26.s, #0\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2]\n"
+ "mov z27.s, #0\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "mov z28.s, #0\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "mov z29.s, #0\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "mov z30.s, #0\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "mov z31.s, #0\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "sdot z24.s, z16.b, z0.b[0]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "sdot z25.s, z16.b, z1.b[0]\n"
+ "sdot z26.s, z16.b, z2.b[0]\n"
+ "sdot z27.s, z16.b, z3.b[0]\n"
+ "sdot z28.s, z16.b, z4.b[0]\n"
+ "sdot z29.s, z16.b, z5.b[0]\n"
+ "sdot z30.s, z16.b, z6.b[0]\n"
"sdot z31.s, z16.b, z7.b[0]\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "sdot z24.s, z17.b, z0.b[1]\n"
"sdot z25.s, z17.b, z1.b[1]\n"
"sdot z26.s, z17.b, z2.b[1]\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
"sdot z27.s, z17.b, z3.b[1]\n"
"sdot z28.s, z17.b, z4.b[1]\n"
"sdot z29.s, z17.b, z5.b[1]\n"
@@ -4452,82 +5160,80 @@
"sdot z29.s, z19.b, z5.b[3]\n"
"sdot z30.s, z19.b, z6.b[3]\n"
"sdot z31.s, z19.b, z7.b[3]\n"
- "cbz %[loops], 2f\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"b.eq 3f\n"
"4:\n"
"st1w z24.s, p7, [%[c_ptr0]]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
+ "subs %[loops], %[loops], #0x1\n"
"mov z24.s, #0\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "addvl %[c_ptr0], %[c_ptr0], #1\n"
+ "st1w z25.s, p7, [c_ptr1]\n"
+ "addvl c_ptr1, c_ptr1, #1\n"
+ "mov z25.s, #0\n"
+ "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "st1w z26.s, p7, [c_ptr2]\n"
+ "addvl c_ptr2, c_ptr2, #1\n"
+ "mov z26.s, #0\n"
+ "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ "st1w z27.s, p7, [c_ptr3]\n"
+ "addvl c_ptr3, c_ptr3, #1\n"
+ "mov z27.s, #0\n"
+ "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #8\n"
+ "st1w z28.s, p7, [c_ptr4]\n"
+ "addvl c_ptr4, c_ptr4, #1\n"
+ "mov z28.s, #0\n"
"ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
"ld1rqb z1.b, p7/z, [a_ptr1]\n"
- "subs %[loops], %[loops], #0x1\n"
- "st1w z25.s, p7, [c_ptr1]\n"
- "addvl %[c_ptr0], %[c_ptr0], #1\n"
- "mov z25.s, #0\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ "st1w z29.s, p7, [c_ptr5]\n"
+ "addvl c_ptr5, c_ptr5, #1\n"
+ "mov z29.s, #0\n"
"ld1rqb z2.b, p7/z, [a_ptr2]\n"
"sdot z24.s, z16.b, z0.b[0]\n"
- "st1w z26.s, p7, [c_ptr2]\n"
- "mov z26.s, #0\n"
- "ld1rqb z3.b, p7/z, [a_ptr3]\n"
- "ld1rqb z4.b, p7/z, [a_ptr4]\n"
- "addvl c_ptr1, c_ptr1, #1\n"
- "sdot z25.s, z16.b, z1.b[0]\n"
- "st1w z27.s, p7, [c_ptr3]\n"
- "mov z27.s, #0\n"
- "ld1rqb z5.b, p7/z, [a_ptr5]\n"
- "sdot z26.s, z16.b, z2.b[0]\n"
- "ld1rqb z6.b, p7/z, [a_ptr6]\n"
- "sdot z24.s, z17.b, z0.b[1]\n"
- "st1w z28.s, p7, [c_ptr4]\n"
- "mov z28.s, #0\n"
- "ld1rqb z7.b, p7/z, [a_ptr7]\n"
- "sdot z27.s, z16.b, z3.b[0]\n"
- "addvl c_ptr2, c_ptr2, #1\n"
- "sdot z25.s, z17.b, z1.b[1]\n"
- "st1w z29.s, p7, [c_ptr5]\n"
- "mov z29.s, #0\n"
- "addvl c_ptr3, c_ptr3, #1\n"
- "sdot z28.s, z16.b, z4.b[0]\n"
"st1w z30.s, p7, [c_ptr6]\n"
"mov z30.s, #0\n"
- "addvl c_ptr4, c_ptr4, #1\n"
- "sdot z29.s, z16.b, z5.b[0]\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "sdot z25.s, z16.b, z1.b[0]\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "sdot z26.s, z16.b, z2.b[0]\n"
"st1w z31.s, p7, [c_ptr7]\n"
"mov z31.s, #0\n"
- "addvl c_ptr5, c_ptr5, #1\n"
- "sdot z30.s, z16.b, z6.b[0]\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "sdot z27.s, z16.b, z3.b[0]\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "sdot z28.s, z16.b, z4.b[0]\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "sdot z24.s, z17.b, z0.b[1]\n"
"addvl c_ptr6, c_ptr6, #1\n"
+ "sdot z29.s, z16.b, z5.b[0]\n"
+ "addvl c_ptr7, c_ptr7, #1\n"
+ "sdot z30.s, z16.b, z6.b[0]\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
"sdot z31.s, z16.b, z7.b[0]\n"
"ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "sdot z25.s, z17.b, z1.b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
"sdot z26.s, z17.b, z2.b[1]\n"
- "addvl c_ptr7, c_ptr7, #1\n"
+ "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
"sdot z27.s, z17.b, z3.b[1]\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
"sdot z28.s, z17.b, z4.b[1]\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ "prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
"sdot z29.s, z17.b, z5.b[1]\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
"sdot z30.s, z17.b, z6.b[1]\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
"sdot z31.s, z17.b, z7.b[1]\n"
"ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"sdot z24.s, z18.b, z0.b[2]\n"
- "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
"sdot z25.s, z18.b, z1.b[2]\n"
- "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
"sdot z26.s, z18.b, z2.b[2]\n"
- "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
"sdot z27.s, z18.b, z3.b[2]\n"
- "prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
"sdot z28.s, z18.b, z4.b[2]\n"
"sdot z29.s, z18.b, z5.b[2]\n"
"sdot z30.s, z18.b, z6.b[2]\n"
@@ -4559,7 +5265,6 @@
"sdot z29.s, z20.b, z5.b[0]\n"
"sdot z30.s, z20.b, z6.b[0]\n"
"sdot z31.s, z20.b, z7.b[0]\n"
- "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
"sdot z24.s, z21.b, z0.b[1]\n"
"sdot z25.s, z21.b, z1.b[1]\n"
"sdot z26.s, z21.b, z2.b[1]\n"
@@ -4568,7 +5273,6 @@
"sdot z29.s, z21.b, z5.b[1]\n"
"sdot z30.s, z21.b, z6.b[1]\n"
"sdot z31.s, z21.b, z7.b[1]\n"
- "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
"sdot z24.s, z22.b, z0.b[2]\n"
"sdot z25.s, z22.b, z1.b[2]\n"
"sdot z26.s, z22.b, z2.b[2]\n"
@@ -4577,7 +5281,6 @@
"sdot z29.s, z22.b, z5.b[2]\n"
"sdot z30.s, z22.b, z6.b[2]\n"
"sdot z31.s, z22.b, z7.b[2]\n"
- "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
"sdot z24.s, z23.b, z0.b[3]\n"
"ld1rqb z0.b, p6/z, [%[a_ptr0], #0x20]\n"
"sdot z25.s, z23.b, z1.b[3]\n"
@@ -4595,7 +5298,6 @@
"sdot z31.s, z23.b, z7.b[3]\n"
"ld1rqb z7.b, p6/z, [a_ptr7, #0x20]\n"
"sdot z24.s, z16.b, z0.b[0]\n"
- "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"sdot z25.s, z16.b, z1.b[0]\n"
"sdot z26.s, z16.b, z2.b[0]\n"
"sdot z27.s, z16.b, z3.b[0]\n"
@@ -4603,7 +5305,6 @@
"sdot z29.s, z16.b, z5.b[0]\n"
"sdot z30.s, z16.b, z6.b[0]\n"
"sdot z31.s, z16.b, z7.b[0]\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
"sdot z24.s, z17.b, z0.b[1]\n"
"sdot z25.s, z17.b, z1.b[1]\n"
"sdot z26.s, z17.b, z2.b[1]\n"
@@ -4612,7 +5313,6 @@
"sdot z29.s, z17.b, z5.b[1]\n"
"sdot z30.s, z17.b, z6.b[1]\n"
"sdot z31.s, z17.b, z7.b[1]\n"
- "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"sdot z24.s, z18.b, z0.b[2]\n"
"sdot z25.s, z18.b, z1.b[2]\n"
"sdot z26.s, z18.b, z2.b[2]\n"
@@ -4621,7 +5321,6 @@
"sdot z29.s, z18.b, z5.b[2]\n"
"sdot z30.s, z18.b, z6.b[2]\n"
"sdot z31.s, z18.b, z7.b[2]\n"
- "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"sdot z24.s, z19.b, z0.b[3]\n"
"sdot z25.s, z19.b, z1.b[3]\n"
"sdot z26.s, z19.b, z2.b[3]\n"
@@ -4630,53 +5329,60 @@
"sdot z29.s, z19.b, z5.b[3]\n"
"sdot z30.s, z19.b, z6.b[3]\n"
"sdot z31.s, z19.b, z7.b[3]\n"
- "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"b.ne 4b\n"
"3:\n"
"st1w z24.s, p7, [%[c_ptr0]]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "mov z24.s, #0\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "ld1rqb z1.b, p7/z, [a_ptr1]\n"
"addvl %[c_ptr0], %[c_ptr0], #1\n"
+ "mov z24.s, #0\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"st1w z25.s, p7, [c_ptr1]\n"
"addvl c_ptr1, c_ptr1, #1\n"
"mov z25.s, #0\n"
+ "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "st1w z26.s, p7, [c_ptr2]\n"
+ "addvl c_ptr2, c_ptr2, #1\n"
+ "mov z26.s, #0\n"
+ "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "st1w z27.s, p7, [c_ptr3]\n"
+ "addvl c_ptr3, c_ptr3, #1\n"
+ "mov z27.s, #0\n"
+ "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #8\n"
+ "st1w z28.s, p7, [c_ptr4]\n"
+ "addvl c_ptr4, c_ptr4, #1\n"
+ "mov z28.s, #0\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "st1w z29.s, p7, [c_ptr5]\n"
+ "addvl c_ptr5, c_ptr5, #1\n"
+ "mov z29.s, #0\n"
"ld1rqb z2.b, p7/z, [a_ptr2]\n"
"sdot z24.s, z16.b, z0.b[0]\n"
- "st1w z26.s, p7, [c_ptr2]\n"
- "mov z26.s, #0\n"
- "ld1rqb z3.b, p7/z, [a_ptr3]\n"
- "ld1rqb z4.b, p7/z, [a_ptr4]\n"
- "addvl c_ptr2, c_ptr2, #1\n"
- "sdot z25.s, z16.b, z1.b[0]\n"
- "st1w z27.s, p7, [c_ptr3]\n"
- "mov z27.s, #0\n"
- "ld1rqb z5.b, p7/z, [a_ptr5]\n"
- "sdot z26.s, z16.b, z2.b[0]\n"
- "ld1rqb z6.b, p7/z, [a_ptr6]\n"
- "sdot z24.s, z17.b, z0.b[1]\n"
- "st1w z28.s, p7, [c_ptr4]\n"
- "mov z28.s, #0\n"
- "ld1rqb z7.b, p7/z, [a_ptr7]\n"
- "sdot z27.s, z16.b, z3.b[0]\n"
- "addvl c_ptr3, c_ptr3, #1\n"
- "sdot z25.s, z17.b, z1.b[1]\n"
- "st1w z29.s, p7, [c_ptr5]\n"
- "mov z29.s, #0\n"
- "addvl c_ptr4, c_ptr4, #1\n"
- "sdot z28.s, z16.b, z4.b[0]\n"
"st1w z30.s, p7, [c_ptr6]\n"
"mov z30.s, #0\n"
- "addvl c_ptr5, c_ptr5, #1\n"
- "sdot z29.s, z16.b, z5.b[0]\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "sdot z25.s, z16.b, z1.b[0]\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "sdot z26.s, z16.b, z2.b[0]\n"
"st1w z31.s, p7, [c_ptr7]\n"
"mov z31.s, #0\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "sdot z27.s, z16.b, z3.b[0]\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "sdot z28.s, z16.b, z4.b[0]\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "sdot z24.s, z17.b, z0.b[1]\n"
"addvl c_ptr6, c_ptr6, #1\n"
- "sdot z30.s, z16.b, z6.b[0]\n"
+ "sdot z29.s, z16.b, z5.b[0]\n"
"addvl c_ptr7, c_ptr7, #1\n"
+ "sdot z30.s, z16.b, z6.b[0]\n"
"sdot z31.s, z16.b, z7.b[0]\n"
"ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "sdot z25.s, z17.b, z1.b[1]\n"
"sdot z26.s, z17.b, z2.b[1]\n"
"sdot z27.s, z17.b, z3.b[1]\n"
"sdot z28.s, z17.b, z4.b[1]\n"
@@ -4783,23 +5489,151 @@
"sdot z29.s, z19.b, z5.b[3]\n"
"sdot z30.s, z19.b, z6.b[3]\n"
"sdot z31.s, z19.b, z7.b[3]\n"
+ "b 5f\n"
"2:\n"
+ "mov z24.s, #0\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "mov z25.s, #0\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "mov z26.s, #0\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2]\n"
+ "mov z27.s, #0\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "mov z28.s, #0\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "mov z29.s, #0\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "mov z30.s, #0\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "mov z31.s, #0\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "sdot z24.s, z16.b, z0.b[0]\n"
+ "sdot z25.s, z16.b, z1.b[0]\n"
+ "sdot z26.s, z16.b, z2.b[0]\n"
+ "sdot z27.s, z16.b, z3.b[0]\n"
+ "sdot z28.s, z16.b, z4.b[0]\n"
+ "sdot z29.s, z16.b, z5.b[0]\n"
+ "sdot z30.s, z16.b, z6.b[0]\n"
+ "sdot z31.s, z16.b, z7.b[0]\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "sdot z24.s, z17.b, z0.b[1]\n"
+ "sdot z25.s, z17.b, z1.b[1]\n"
+ "sdot z26.s, z17.b, z2.b[1]\n"
+ "sdot z27.s, z17.b, z3.b[1]\n"
+ "sdot z28.s, z17.b, z4.b[1]\n"
+ "sdot z29.s, z17.b, z5.b[1]\n"
+ "sdot z30.s, z17.b, z6.b[1]\n"
+ "sdot z31.s, z17.b, z7.b[1]\n"
+ "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "sdot z24.s, z18.b, z0.b[2]\n"
+ "sdot z25.s, z18.b, z1.b[2]\n"
+ "sdot z26.s, z18.b, z2.b[2]\n"
+ "sdot z27.s, z18.b, z3.b[2]\n"
+ "sdot z28.s, z18.b, z4.b[2]\n"
+ "sdot z29.s, z18.b, z5.b[2]\n"
+ "sdot z30.s, z18.b, z6.b[2]\n"
+ "sdot z31.s, z18.b, z7.b[2]\n"
+ "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "sdot z24.s, z19.b, z0.b[3]\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x10]\n"
+ "sdot z25.s, z19.b, z1.b[3]\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1, #0x10]\n"
+ "sdot z26.s, z19.b, z2.b[3]\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2, #0x10]\n"
+ "sdot z27.s, z19.b, z3.b[3]\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3, #0x10]\n"
+ "sdot z28.s, z19.b, z4.b[3]\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4, #0x10]\n"
+ "sdot z29.s, z19.b, z5.b[3]\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5, #0x10]\n"
+ "sdot z30.s, z19.b, z6.b[3]\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6, #0x10]\n"
+ "sdot z31.s, z19.b, z7.b[3]\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7, #0x10]\n"
+ "sdot z24.s, z20.b, z0.b[0]\n"
+ "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "sdot z25.s, z20.b, z1.b[0]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #4\n"
+ "sdot z26.s, z20.b, z2.b[0]\n"
+ "sdot z27.s, z20.b, z3.b[0]\n"
+ "sdot z28.s, z20.b, z4.b[0]\n"
+ "sdot z29.s, z20.b, z5.b[0]\n"
+ "sdot z30.s, z20.b, z6.b[0]\n"
+ "sdot z31.s, z20.b, z7.b[0]\n"
+ "sdot z24.s, z21.b, z0.b[1]\n"
+ "sdot z25.s, z21.b, z1.b[1]\n"
+ "sdot z26.s, z21.b, z2.b[1]\n"
+ "sdot z27.s, z21.b, z3.b[1]\n"
+ "sdot z28.s, z21.b, z4.b[1]\n"
+ "sdot z29.s, z21.b, z5.b[1]\n"
+ "sdot z30.s, z21.b, z6.b[1]\n"
+ "sdot z31.s, z21.b, z7.b[1]\n"
+ "sdot z24.s, z22.b, z0.b[2]\n"
+ "sdot z25.s, z22.b, z1.b[2]\n"
+ "sdot z26.s, z22.b, z2.b[2]\n"
+ "sdot z27.s, z22.b, z3.b[2]\n"
+ "sdot z28.s, z22.b, z4.b[2]\n"
+ "sdot z29.s, z22.b, z5.b[2]\n"
+ "sdot z30.s, z22.b, z6.b[2]\n"
+ "sdot z31.s, z22.b, z7.b[2]\n"
+ "sdot z24.s, z23.b, z0.b[3]\n"
+ "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x20]\n"
+ "sdot z25.s, z23.b, z1.b[3]\n"
+ "ld1rqb z1.b, p6/z, [a_ptr1, #0x20]\n"
+ "sdot z26.s, z23.b, z2.b[3]\n"
+ "ld1rqb z2.b, p6/z, [a_ptr2, #0x20]\n"
+ "sdot z27.s, z23.b, z3.b[3]\n"
+ "ld1rqb z3.b, p6/z, [a_ptr3, #0x20]\n"
+ "sdot z28.s, z23.b, z4.b[3]\n"
+ "ld1rqb z4.b, p6/z, [a_ptr4, #0x20]\n"
+ "sdot z29.s, z23.b, z5.b[3]\n"
+ "ld1rqb z5.b, p6/z, [a_ptr5, #0x20]\n"
+ "sdot z30.s, z23.b, z6.b[3]\n"
+ "ld1rqb z6.b, p6/z, [a_ptr6, #0x20]\n"
+ "sdot z31.s, z23.b, z7.b[3]\n"
+ "ld1rqb z7.b, p6/z, [a_ptr7, #0x20]\n"
+ "sdot z24.s, z16.b, z0.b[0]\n"
+ "sdot z25.s, z16.b, z1.b[0]\n"
+ "sdot z26.s, z16.b, z2.b[0]\n"
+ "sdot z27.s, z16.b, z3.b[0]\n"
+ "sdot z28.s, z16.b, z4.b[0]\n"
+ "sdot z29.s, z16.b, z5.b[0]\n"
+ "sdot z30.s, z16.b, z6.b[0]\n"
+ "sdot z31.s, z16.b, z7.b[0]\n"
+ "sdot z24.s, z17.b, z0.b[1]\n"
+ "sdot z25.s, z17.b, z1.b[1]\n"
+ "sdot z26.s, z17.b, z2.b[1]\n"
+ "sdot z27.s, z17.b, z3.b[1]\n"
+ "sdot z28.s, z17.b, z4.b[1]\n"
+ "sdot z29.s, z17.b, z5.b[1]\n"
+ "sdot z30.s, z17.b, z6.b[1]\n"
+ "sdot z31.s, z17.b, z7.b[1]\n"
+ "sdot z24.s, z18.b, z0.b[2]\n"
+ "sdot z25.s, z18.b, z1.b[2]\n"
+ "sdot z26.s, z18.b, z2.b[2]\n"
+ "sdot z27.s, z18.b, z3.b[2]\n"
+ "sdot z28.s, z18.b, z4.b[2]\n"
+ "sdot z29.s, z18.b, z5.b[2]\n"
+ "sdot z30.s, z18.b, z6.b[2]\n"
+ "sdot z31.s, z18.b, z7.b[2]\n"
+ "sdot z24.s, z19.b, z0.b[3]\n"
+ "sdot z25.s, z19.b, z1.b[3]\n"
+ "sdot z26.s, z19.b, z2.b[3]\n"
+ "sdot z27.s, z19.b, z3.b[3]\n"
+ "sdot z28.s, z19.b, z4.b[3]\n"
+ "sdot z29.s, z19.b, z5.b[3]\n"
+ "sdot z30.s, z19.b, z6.b[3]\n"
+ "sdot z31.s, z19.b, z7.b[3]\n"
+ "5:\n"
"st1w z24.s, p0, [%[c_ptr0]]\n"
"addvl %[c_ptr0], %[c_ptr0], #1\n"
"st1w z25.s, p0, [c_ptr1]\n"
- "addvl c_ptr1, c_ptr1, #1\n"
"st1w z26.s, p0, [c_ptr2]\n"
- "addvl c_ptr2, c_ptr2, #1\n"
"st1w z27.s, p0, [c_ptr3]\n"
- "addvl c_ptr3, c_ptr3, #1\n"
"st1w z28.s, p0, [c_ptr4]\n"
- "addvl c_ptr4, c_ptr4, #1\n"
"st1w z29.s, p0, [c_ptr5]\n"
- "addvl c_ptr5, c_ptr5, #1\n"
"st1w z30.s, p0, [c_ptr6]\n"
- "addvl c_ptr6, c_ptr6, #1\n"
"st1w z31.s, p0, [c_ptr7]\n"
- "addvl c_ptr7, c_ptr7, #1\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -4878,52 +5712,54 @@
"add c_ptr1, %[c_ptr0], #0x0\n"
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
- "mov z24.s, #0\n"
"ptrue p7.b\n"
- "mov z25.s, #0\n"
"whilelt p6.b, %[temp], %[odd_depth]\n"
- "mov z26.s, #0\n"
+ "whilelt p0.s, %[temp], %[last_width]\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
"ld1b z16.b, p7/z, [%[b_ptr0]]\n"
- "mov z27.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
"ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "mov z28.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
"ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "mov z29.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
"ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "mov z30.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
"ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "mov z31.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
"ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
"ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "whilelt p0.s, %[temp], %[last_width]\n"
"ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
- "ld1rqb z1.b, p7/z, [a_ptr1]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
- "sdot z24.s, z16.b, z0.b[0]\n"
- "ld1rqb z2.b, p7/z, [a_ptr2]\n"
- "ld1rqb z3.b, p7/z, [a_ptr3]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
- "sdot z25.s, z16.b, z1.b[0]\n"
- "ld1rqb z4.b, p7/z, [a_ptr4]\n"
- "sdot z26.s, z16.b, z2.b[0]\n"
- "ld1rqb z5.b, p7/z, [a_ptr5]\n"
- "sdot z27.s, z16.b, z3.b[0]\n"
- "ld1rqb z6.b, p7/z, [a_ptr6]\n"
- "sdot z24.s, z17.b, z0.b[1]\n"
- "ld1rqb z7.b, p7/z, [a_ptr7]\n"
- "sdot z28.s, z16.b, z4.b[0]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- "sdot z29.s, z16.b, z5.b[0]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- "sdot z30.s, z16.b, z6.b[0]\n"
"addvl %[b_ptr0], %[b_ptr0], #8\n"
+ "cbz %[loops], 2f\n"
+ "mov z24.s, #0\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "mov z25.s, #0\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "mov z26.s, #0\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2]\n"
+ "mov z27.s, #0\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "mov z28.s, #0\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "mov z29.s, #0\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "mov z30.s, #0\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "mov z31.s, #0\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "sdot z24.s, z16.b, z0.b[0]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "sdot z25.s, z16.b, z1.b[0]\n"
+ "sdot z26.s, z16.b, z2.b[0]\n"
+ "sdot z27.s, z16.b, z3.b[0]\n"
+ "sdot z28.s, z16.b, z4.b[0]\n"
+ "sdot z29.s, z16.b, z5.b[0]\n"
+ "sdot z30.s, z16.b, z6.b[0]\n"
"sdot z31.s, z16.b, z7.b[0]\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "sdot z24.s, z17.b, z0.b[1]\n"
"sdot z25.s, z17.b, z1.b[1]\n"
"sdot z26.s, z17.b, z2.b[1]\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
"sdot z27.s, z17.b, z3.b[1]\n"
"sdot z28.s, z17.b, z4.b[1]\n"
"sdot z29.s, z17.b, z5.b[1]\n"
@@ -5046,82 +5882,80 @@
"sdot z29.s, z20.b, z5.b[0]\n"
"sdot z30.s, z20.b, z6.b[0]\n"
"sdot z31.s, z20.b, z7.b[0]\n"
- "cbz %[loops], 2f\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"b.eq 3f\n"
"4:\n"
"st1w z24.s, p7, [%[c_ptr0]]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
+ "subs %[loops], %[loops], #0x1\n"
"mov z24.s, #0\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "addvl %[c_ptr0], %[c_ptr0], #1\n"
+ "st1w z25.s, p7, [c_ptr1]\n"
+ "addvl c_ptr1, c_ptr1, #1\n"
+ "mov z25.s, #0\n"
+ "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "st1w z26.s, p7, [c_ptr2]\n"
+ "addvl c_ptr2, c_ptr2, #1\n"
+ "mov z26.s, #0\n"
+ "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ "st1w z27.s, p7, [c_ptr3]\n"
+ "addvl c_ptr3, c_ptr3, #1\n"
+ "mov z27.s, #0\n"
+ "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #8\n"
+ "st1w z28.s, p7, [c_ptr4]\n"
+ "addvl c_ptr4, c_ptr4, #1\n"
+ "mov z28.s, #0\n"
"ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
"ld1rqb z1.b, p7/z, [a_ptr1]\n"
- "subs %[loops], %[loops], #0x1\n"
- "st1w z25.s, p7, [c_ptr1]\n"
- "addvl %[c_ptr0], %[c_ptr0], #1\n"
- "mov z25.s, #0\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ "st1w z29.s, p7, [c_ptr5]\n"
+ "addvl c_ptr5, c_ptr5, #1\n"
+ "mov z29.s, #0\n"
"ld1rqb z2.b, p7/z, [a_ptr2]\n"
"sdot z24.s, z16.b, z0.b[0]\n"
- "st1w z26.s, p7, [c_ptr2]\n"
- "mov z26.s, #0\n"
- "ld1rqb z3.b, p7/z, [a_ptr3]\n"
- "ld1rqb z4.b, p7/z, [a_ptr4]\n"
- "addvl c_ptr1, c_ptr1, #1\n"
- "sdot z25.s, z16.b, z1.b[0]\n"
- "st1w z27.s, p7, [c_ptr3]\n"
- "mov z27.s, #0\n"
- "ld1rqb z5.b, p7/z, [a_ptr5]\n"
- "sdot z26.s, z16.b, z2.b[0]\n"
- "ld1rqb z6.b, p7/z, [a_ptr6]\n"
- "sdot z24.s, z17.b, z0.b[1]\n"
- "st1w z28.s, p7, [c_ptr4]\n"
- "mov z28.s, #0\n"
- "ld1rqb z7.b, p7/z, [a_ptr7]\n"
- "sdot z27.s, z16.b, z3.b[0]\n"
- "addvl c_ptr2, c_ptr2, #1\n"
- "sdot z25.s, z17.b, z1.b[1]\n"
- "st1w z29.s, p7, [c_ptr5]\n"
- "mov z29.s, #0\n"
- "addvl c_ptr3, c_ptr3, #1\n"
- "sdot z28.s, z16.b, z4.b[0]\n"
"st1w z30.s, p7, [c_ptr6]\n"
"mov z30.s, #0\n"
- "addvl c_ptr4, c_ptr4, #1\n"
- "sdot z29.s, z16.b, z5.b[0]\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "sdot z25.s, z16.b, z1.b[0]\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "sdot z26.s, z16.b, z2.b[0]\n"
"st1w z31.s, p7, [c_ptr7]\n"
"mov z31.s, #0\n"
- "addvl c_ptr5, c_ptr5, #1\n"
- "sdot z30.s, z16.b, z6.b[0]\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "sdot z27.s, z16.b, z3.b[0]\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "sdot z28.s, z16.b, z4.b[0]\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "sdot z24.s, z17.b, z0.b[1]\n"
"addvl c_ptr6, c_ptr6, #1\n"
+ "sdot z29.s, z16.b, z5.b[0]\n"
+ "addvl c_ptr7, c_ptr7, #1\n"
+ "sdot z30.s, z16.b, z6.b[0]\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
"sdot z31.s, z16.b, z7.b[0]\n"
"ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "sdot z25.s, z17.b, z1.b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
"sdot z26.s, z17.b, z2.b[1]\n"
- "addvl c_ptr7, c_ptr7, #1\n"
+ "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
"sdot z27.s, z17.b, z3.b[1]\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
"sdot z28.s, z17.b, z4.b[1]\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ "prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
"sdot z29.s, z17.b, z5.b[1]\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
"sdot z30.s, z17.b, z6.b[1]\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
"sdot z31.s, z17.b, z7.b[1]\n"
"ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"sdot z24.s, z18.b, z0.b[2]\n"
- "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
"sdot z25.s, z18.b, z1.b[2]\n"
- "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
"sdot z26.s, z18.b, z2.b[2]\n"
- "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
"sdot z27.s, z18.b, z3.b[2]\n"
- "prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
"sdot z28.s, z18.b, z4.b[2]\n"
"sdot z29.s, z18.b, z5.b[2]\n"
"sdot z30.s, z18.b, z6.b[2]\n"
@@ -5162,7 +5996,6 @@
"sdot z29.s, z21.b, z5.b[1]\n"
"sdot z30.s, z21.b, z6.b[1]\n"
"sdot z31.s, z21.b, z7.b[1]\n"
- "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
"sdot z24.s, z22.b, z0.b[2]\n"
"sdot z25.s, z22.b, z1.b[2]\n"
"sdot z26.s, z22.b, z2.b[2]\n"
@@ -5171,7 +6004,6 @@
"sdot z29.s, z22.b, z5.b[2]\n"
"sdot z30.s, z22.b, z6.b[2]\n"
"sdot z31.s, z22.b, z7.b[2]\n"
- "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
"sdot z24.s, z23.b, z0.b[3]\n"
"ld1rqb z0.b, p7/z, [%[a_ptr0], #0x20]\n"
"sdot z25.s, z23.b, z1.b[3]\n"
@@ -5189,7 +6021,6 @@
"sdot z31.s, z23.b, z7.b[3]\n"
"ld1rqb z7.b, p7/z, [a_ptr7, #0x20]\n"
"sdot z24.s, z16.b, z0.b[0]\n"
- "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"sdot z25.s, z16.b, z1.b[0]\n"
"sdot z26.s, z16.b, z2.b[0]\n"
"sdot z27.s, z16.b, z3.b[0]\n"
@@ -5197,7 +6028,6 @@
"sdot z29.s, z16.b, z5.b[0]\n"
"sdot z30.s, z16.b, z6.b[0]\n"
"sdot z31.s, z16.b, z7.b[0]\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
"sdot z24.s, z17.b, z0.b[1]\n"
"sdot z25.s, z17.b, z1.b[1]\n"
"sdot z26.s, z17.b, z2.b[1]\n"
@@ -5206,7 +6036,6 @@
"sdot z29.s, z17.b, z5.b[1]\n"
"sdot z30.s, z17.b, z6.b[1]\n"
"sdot z31.s, z17.b, z7.b[1]\n"
- "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"sdot z24.s, z18.b, z0.b[2]\n"
"sdot z25.s, z18.b, z1.b[2]\n"
"sdot z26.s, z18.b, z2.b[2]\n"
@@ -5215,7 +6044,6 @@
"sdot z29.s, z18.b, z5.b[2]\n"
"sdot z30.s, z18.b, z6.b[2]\n"
"sdot z31.s, z18.b, z7.b[2]\n"
- "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"sdot z24.s, z19.b, z0.b[3]\n"
"ld1rqb z0.b, p6/z, [%[a_ptr0], #0x30]\n"
"sdot z25.s, z19.b, z1.b[3]\n"
@@ -5233,7 +6061,6 @@
"sdot z31.s, z19.b, z7.b[3]\n"
"ld1rqb z7.b, p6/z, [a_ptr7, #0x30]\n"
"sdot z24.s, z20.b, z0.b[0]\n"
- "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"sdot z25.s, z20.b, z1.b[0]\n"
"sdot z26.s, z20.b, z2.b[0]\n"
"sdot z27.s, z20.b, z3.b[0]\n"
@@ -5241,53 +6068,60 @@
"sdot z29.s, z20.b, z5.b[0]\n"
"sdot z30.s, z20.b, z6.b[0]\n"
"sdot z31.s, z20.b, z7.b[0]\n"
- "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
"b.ne 4b\n"
"3:\n"
"st1w z24.s, p7, [%[c_ptr0]]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "mov z24.s, #0\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "ld1rqb z1.b, p7/z, [a_ptr1]\n"
"addvl %[c_ptr0], %[c_ptr0], #1\n"
+ "mov z24.s, #0\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"st1w z25.s, p7, [c_ptr1]\n"
"addvl c_ptr1, c_ptr1, #1\n"
"mov z25.s, #0\n"
+ "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "st1w z26.s, p7, [c_ptr2]\n"
+ "addvl c_ptr2, c_ptr2, #1\n"
+ "mov z26.s, #0\n"
+ "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "st1w z27.s, p7, [c_ptr3]\n"
+ "addvl c_ptr3, c_ptr3, #1\n"
+ "mov z27.s, #0\n"
+ "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #8\n"
+ "st1w z28.s, p7, [c_ptr4]\n"
+ "addvl c_ptr4, c_ptr4, #1\n"
+ "mov z28.s, #0\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "st1w z29.s, p7, [c_ptr5]\n"
+ "addvl c_ptr5, c_ptr5, #1\n"
+ "mov z29.s, #0\n"
"ld1rqb z2.b, p7/z, [a_ptr2]\n"
"sdot z24.s, z16.b, z0.b[0]\n"
- "st1w z26.s, p7, [c_ptr2]\n"
- "mov z26.s, #0\n"
- "ld1rqb z3.b, p7/z, [a_ptr3]\n"
- "ld1rqb z4.b, p7/z, [a_ptr4]\n"
- "addvl c_ptr2, c_ptr2, #1\n"
- "sdot z25.s, z16.b, z1.b[0]\n"
- "st1w z27.s, p7, [c_ptr3]\n"
- "mov z27.s, #0\n"
- "ld1rqb z5.b, p7/z, [a_ptr5]\n"
- "sdot z26.s, z16.b, z2.b[0]\n"
- "ld1rqb z6.b, p7/z, [a_ptr6]\n"
- "sdot z24.s, z17.b, z0.b[1]\n"
- "st1w z28.s, p7, [c_ptr4]\n"
- "mov z28.s, #0\n"
- "ld1rqb z7.b, p7/z, [a_ptr7]\n"
- "sdot z27.s, z16.b, z3.b[0]\n"
- "addvl c_ptr3, c_ptr3, #1\n"
- "sdot z25.s, z17.b, z1.b[1]\n"
- "st1w z29.s, p7, [c_ptr5]\n"
- "mov z29.s, #0\n"
- "addvl c_ptr4, c_ptr4, #1\n"
- "sdot z28.s, z16.b, z4.b[0]\n"
"st1w z30.s, p7, [c_ptr6]\n"
"mov z30.s, #0\n"
- "addvl c_ptr5, c_ptr5, #1\n"
- "sdot z29.s, z16.b, z5.b[0]\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "sdot z25.s, z16.b, z1.b[0]\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "sdot z26.s, z16.b, z2.b[0]\n"
"st1w z31.s, p7, [c_ptr7]\n"
"mov z31.s, #0\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "sdot z27.s, z16.b, z3.b[0]\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "sdot z28.s, z16.b, z4.b[0]\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "sdot z24.s, z17.b, z0.b[1]\n"
"addvl c_ptr6, c_ptr6, #1\n"
- "sdot z30.s, z16.b, z6.b[0]\n"
+ "sdot z29.s, z16.b, z5.b[0]\n"
"addvl c_ptr7, c_ptr7, #1\n"
+ "sdot z30.s, z16.b, z6.b[0]\n"
"sdot z31.s, z16.b, z7.b[0]\n"
"ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "sdot z25.s, z17.b, z1.b[1]\n"
"sdot z26.s, z17.b, z2.b[1]\n"
"sdot z27.s, z17.b, z3.b[1]\n"
"sdot z28.s, z17.b, z4.b[1]\n"
@@ -5411,23 +6245,168 @@
"sdot z29.s, z20.b, z5.b[0]\n"
"sdot z30.s, z20.b, z6.b[0]\n"
"sdot z31.s, z20.b, z7.b[0]\n"
+ "b 5f\n"
"2:\n"
+ "mov z24.s, #0\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "mov z25.s, #0\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "mov z26.s, #0\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2]\n"
+ "mov z27.s, #0\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "mov z28.s, #0\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "mov z29.s, #0\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "mov z30.s, #0\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "mov z31.s, #0\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "sdot z24.s, z16.b, z0.b[0]\n"
+ "sdot z25.s, z16.b, z1.b[0]\n"
+ "sdot z26.s, z16.b, z2.b[0]\n"
+ "sdot z27.s, z16.b, z3.b[0]\n"
+ "sdot z28.s, z16.b, z4.b[0]\n"
+ "sdot z29.s, z16.b, z5.b[0]\n"
+ "sdot z30.s, z16.b, z6.b[0]\n"
+ "sdot z31.s, z16.b, z7.b[0]\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "sdot z24.s, z17.b, z0.b[1]\n"
+ "sdot z25.s, z17.b, z1.b[1]\n"
+ "sdot z26.s, z17.b, z2.b[1]\n"
+ "sdot z27.s, z17.b, z3.b[1]\n"
+ "sdot z28.s, z17.b, z4.b[1]\n"
+ "sdot z29.s, z17.b, z5.b[1]\n"
+ "sdot z30.s, z17.b, z6.b[1]\n"
+ "sdot z31.s, z17.b, z7.b[1]\n"
+ "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "sdot z24.s, z18.b, z0.b[2]\n"
+ "sdot z25.s, z18.b, z1.b[2]\n"
+ "sdot z26.s, z18.b, z2.b[2]\n"
+ "sdot z27.s, z18.b, z3.b[2]\n"
+ "sdot z28.s, z18.b, z4.b[2]\n"
+ "sdot z29.s, z18.b, z5.b[2]\n"
+ "sdot z30.s, z18.b, z6.b[2]\n"
+ "sdot z31.s, z18.b, z7.b[2]\n"
+ "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "sdot z24.s, z19.b, z0.b[3]\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x10]\n"
+ "sdot z25.s, z19.b, z1.b[3]\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1, #0x10]\n"
+ "sdot z26.s, z19.b, z2.b[3]\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2, #0x10]\n"
+ "sdot z27.s, z19.b, z3.b[3]\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3, #0x10]\n"
+ "sdot z28.s, z19.b, z4.b[3]\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4, #0x10]\n"
+ "sdot z29.s, z19.b, z5.b[3]\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5, #0x10]\n"
+ "sdot z30.s, z19.b, z6.b[3]\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6, #0x10]\n"
+ "sdot z31.s, z19.b, z7.b[3]\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7, #0x10]\n"
+ "sdot z24.s, z20.b, z0.b[0]\n"
+ "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "sdot z25.s, z20.b, z1.b[0]\n"
+ "sdot z26.s, z20.b, z2.b[0]\n"
+ "sdot z27.s, z20.b, z3.b[0]\n"
+ "sdot z28.s, z20.b, z4.b[0]\n"
+ "sdot z29.s, z20.b, z5.b[0]\n"
+ "sdot z30.s, z20.b, z6.b[0]\n"
+ "sdot z31.s, z20.b, z7.b[0]\n"
+ "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "sdot z24.s, z21.b, z0.b[1]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #5\n"
+ "sdot z25.s, z21.b, z1.b[1]\n"
+ "sdot z26.s, z21.b, z2.b[1]\n"
+ "sdot z27.s, z21.b, z3.b[1]\n"
+ "sdot z28.s, z21.b, z4.b[1]\n"
+ "sdot z29.s, z21.b, z5.b[1]\n"
+ "sdot z30.s, z21.b, z6.b[1]\n"
+ "sdot z31.s, z21.b, z7.b[1]\n"
+ "sdot z24.s, z22.b, z0.b[2]\n"
+ "sdot z25.s, z22.b, z1.b[2]\n"
+ "sdot z26.s, z22.b, z2.b[2]\n"
+ "sdot z27.s, z22.b, z3.b[2]\n"
+ "sdot z28.s, z22.b, z4.b[2]\n"
+ "sdot z29.s, z22.b, z5.b[2]\n"
+ "sdot z30.s, z22.b, z6.b[2]\n"
+ "sdot z31.s, z22.b, z7.b[2]\n"
+ "sdot z24.s, z23.b, z0.b[3]\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x20]\n"
+ "sdot z25.s, z23.b, z1.b[3]\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1, #0x20]\n"
+ "sdot z26.s, z23.b, z2.b[3]\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2, #0x20]\n"
+ "sdot z27.s, z23.b, z3.b[3]\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3, #0x20]\n"
+ "sdot z28.s, z23.b, z4.b[3]\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4, #0x20]\n"
+ "sdot z29.s, z23.b, z5.b[3]\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5, #0x20]\n"
+ "sdot z30.s, z23.b, z6.b[3]\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6, #0x20]\n"
+ "sdot z31.s, z23.b, z7.b[3]\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7, #0x20]\n"
+ "sdot z24.s, z16.b, z0.b[0]\n"
+ "sdot z25.s, z16.b, z1.b[0]\n"
+ "sdot z26.s, z16.b, z2.b[0]\n"
+ "sdot z27.s, z16.b, z3.b[0]\n"
+ "sdot z28.s, z16.b, z4.b[0]\n"
+ "sdot z29.s, z16.b, z5.b[0]\n"
+ "sdot z30.s, z16.b, z6.b[0]\n"
+ "sdot z31.s, z16.b, z7.b[0]\n"
+ "sdot z24.s, z17.b, z0.b[1]\n"
+ "sdot z25.s, z17.b, z1.b[1]\n"
+ "sdot z26.s, z17.b, z2.b[1]\n"
+ "sdot z27.s, z17.b, z3.b[1]\n"
+ "sdot z28.s, z17.b, z4.b[1]\n"
+ "sdot z29.s, z17.b, z5.b[1]\n"
+ "sdot z30.s, z17.b, z6.b[1]\n"
+ "sdot z31.s, z17.b, z7.b[1]\n"
+ "sdot z24.s, z18.b, z0.b[2]\n"
+ "sdot z25.s, z18.b, z1.b[2]\n"
+ "sdot z26.s, z18.b, z2.b[2]\n"
+ "sdot z27.s, z18.b, z3.b[2]\n"
+ "sdot z28.s, z18.b, z4.b[2]\n"
+ "sdot z29.s, z18.b, z5.b[2]\n"
+ "sdot z30.s, z18.b, z6.b[2]\n"
+ "sdot z31.s, z18.b, z7.b[2]\n"
+ "sdot z24.s, z19.b, z0.b[3]\n"
+ "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x30]\n"
+ "sdot z25.s, z19.b, z1.b[3]\n"
+ "ld1rqb z1.b, p6/z, [a_ptr1, #0x30]\n"
+ "sdot z26.s, z19.b, z2.b[3]\n"
+ "ld1rqb z2.b, p6/z, [a_ptr2, #0x30]\n"
+ "sdot z27.s, z19.b, z3.b[3]\n"
+ "ld1rqb z3.b, p6/z, [a_ptr3, #0x30]\n"
+ "sdot z28.s, z19.b, z4.b[3]\n"
+ "ld1rqb z4.b, p6/z, [a_ptr4, #0x30]\n"
+ "sdot z29.s, z19.b, z5.b[3]\n"
+ "ld1rqb z5.b, p6/z, [a_ptr5, #0x30]\n"
+ "sdot z30.s, z19.b, z6.b[3]\n"
+ "ld1rqb z6.b, p6/z, [a_ptr6, #0x30]\n"
+ "sdot z31.s, z19.b, z7.b[3]\n"
+ "ld1rqb z7.b, p6/z, [a_ptr7, #0x30]\n"
+ "sdot z24.s, z20.b, z0.b[0]\n"
+ "sdot z25.s, z20.b, z1.b[0]\n"
+ "sdot z26.s, z20.b, z2.b[0]\n"
+ "sdot z27.s, z20.b, z3.b[0]\n"
+ "sdot z28.s, z20.b, z4.b[0]\n"
+ "sdot z29.s, z20.b, z5.b[0]\n"
+ "sdot z30.s, z20.b, z6.b[0]\n"
+ "sdot z31.s, z20.b, z7.b[0]\n"
+ "5:\n"
"st1w z24.s, p0, [%[c_ptr0]]\n"
"addvl %[c_ptr0], %[c_ptr0], #1\n"
"st1w z25.s, p0, [c_ptr1]\n"
- "addvl c_ptr1, c_ptr1, #1\n"
"st1w z26.s, p0, [c_ptr2]\n"
- "addvl c_ptr2, c_ptr2, #1\n"
"st1w z27.s, p0, [c_ptr3]\n"
- "addvl c_ptr3, c_ptr3, #1\n"
"st1w z28.s, p0, [c_ptr4]\n"
- "addvl c_ptr4, c_ptr4, #1\n"
"st1w z29.s, p0, [c_ptr5]\n"
- "addvl c_ptr5, c_ptr5, #1\n"
"st1w z30.s, p0, [c_ptr6]\n"
- "addvl c_ptr6, c_ptr6, #1\n"
"st1w z31.s, p0, [c_ptr7]\n"
- "addvl c_ptr7, c_ptr7, #1\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -5506,52 +6485,54 @@
"add c_ptr1, %[c_ptr0], #0x0\n"
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
- "mov z24.s, #0\n"
"ptrue p7.b\n"
- "mov z25.s, #0\n"
"whilelt p6.b, %[temp], %[odd_depth]\n"
- "mov z26.s, #0\n"
+ "whilelt p0.s, %[temp], %[last_width]\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
"ld1b z16.b, p7/z, [%[b_ptr0]]\n"
- "mov z27.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
"ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "mov z28.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
"ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "mov z29.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
"ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "mov z30.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
"ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "mov z31.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
"ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
"ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "whilelt p0.s, %[temp], %[last_width]\n"
"ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
- "ld1rqb z1.b, p7/z, [a_ptr1]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
- "sdot z24.s, z16.b, z0.b[0]\n"
- "ld1rqb z2.b, p7/z, [a_ptr2]\n"
- "ld1rqb z3.b, p7/z, [a_ptr3]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
- "sdot z25.s, z16.b, z1.b[0]\n"
- "ld1rqb z4.b, p7/z, [a_ptr4]\n"
- "sdot z26.s, z16.b, z2.b[0]\n"
- "ld1rqb z5.b, p7/z, [a_ptr5]\n"
- "sdot z27.s, z16.b, z3.b[0]\n"
- "ld1rqb z6.b, p7/z, [a_ptr6]\n"
- "sdot z24.s, z17.b, z0.b[1]\n"
- "ld1rqb z7.b, p7/z, [a_ptr7]\n"
- "sdot z28.s, z16.b, z4.b[0]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- "sdot z29.s, z16.b, z5.b[0]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- "sdot z30.s, z16.b, z6.b[0]\n"
"addvl %[b_ptr0], %[b_ptr0], #8\n"
+ "cbz %[loops], 2f\n"
+ "mov z24.s, #0\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "mov z25.s, #0\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "mov z26.s, #0\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2]\n"
+ "mov z27.s, #0\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "mov z28.s, #0\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "mov z29.s, #0\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "mov z30.s, #0\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "mov z31.s, #0\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "sdot z24.s, z16.b, z0.b[0]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "sdot z25.s, z16.b, z1.b[0]\n"
+ "sdot z26.s, z16.b, z2.b[0]\n"
+ "sdot z27.s, z16.b, z3.b[0]\n"
+ "sdot z28.s, z16.b, z4.b[0]\n"
+ "sdot z29.s, z16.b, z5.b[0]\n"
+ "sdot z30.s, z16.b, z6.b[0]\n"
"sdot z31.s, z16.b, z7.b[0]\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "sdot z24.s, z17.b, z0.b[1]\n"
"sdot z25.s, z17.b, z1.b[1]\n"
"sdot z26.s, z17.b, z2.b[1]\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
"sdot z27.s, z17.b, z3.b[1]\n"
"sdot z28.s, z17.b, z4.b[1]\n"
"sdot z29.s, z17.b, z5.b[1]\n"
@@ -5683,82 +6664,80 @@
"sdot z29.s, z21.b, z5.b[1]\n"
"sdot z30.s, z21.b, z6.b[1]\n"
"sdot z31.s, z21.b, z7.b[1]\n"
- "cbz %[loops], 2f\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"b.eq 3f\n"
"4:\n"
"st1w z24.s, p7, [%[c_ptr0]]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
+ "subs %[loops], %[loops], #0x1\n"
"mov z24.s, #0\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "addvl %[c_ptr0], %[c_ptr0], #1\n"
+ "st1w z25.s, p7, [c_ptr1]\n"
+ "addvl c_ptr1, c_ptr1, #1\n"
+ "mov z25.s, #0\n"
+ "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "st1w z26.s, p7, [c_ptr2]\n"
+ "addvl c_ptr2, c_ptr2, #1\n"
+ "mov z26.s, #0\n"
+ "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ "st1w z27.s, p7, [c_ptr3]\n"
+ "addvl c_ptr3, c_ptr3, #1\n"
+ "mov z27.s, #0\n"
+ "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #8\n"
+ "st1w z28.s, p7, [c_ptr4]\n"
+ "addvl c_ptr4, c_ptr4, #1\n"
+ "mov z28.s, #0\n"
"ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
"ld1rqb z1.b, p7/z, [a_ptr1]\n"
- "subs %[loops], %[loops], #0x1\n"
- "st1w z25.s, p7, [c_ptr1]\n"
- "addvl %[c_ptr0], %[c_ptr0], #1\n"
- "mov z25.s, #0\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ "st1w z29.s, p7, [c_ptr5]\n"
+ "addvl c_ptr5, c_ptr5, #1\n"
+ "mov z29.s, #0\n"
"ld1rqb z2.b, p7/z, [a_ptr2]\n"
"sdot z24.s, z16.b, z0.b[0]\n"
- "st1w z26.s, p7, [c_ptr2]\n"
- "mov z26.s, #0\n"
- "ld1rqb z3.b, p7/z, [a_ptr3]\n"
- "ld1rqb z4.b, p7/z, [a_ptr4]\n"
- "addvl c_ptr1, c_ptr1, #1\n"
- "sdot z25.s, z16.b, z1.b[0]\n"
- "st1w z27.s, p7, [c_ptr3]\n"
- "mov z27.s, #0\n"
- "ld1rqb z5.b, p7/z, [a_ptr5]\n"
- "sdot z26.s, z16.b, z2.b[0]\n"
- "ld1rqb z6.b, p7/z, [a_ptr6]\n"
- "sdot z24.s, z17.b, z0.b[1]\n"
- "st1w z28.s, p7, [c_ptr4]\n"
- "mov z28.s, #0\n"
- "ld1rqb z7.b, p7/z, [a_ptr7]\n"
- "sdot z27.s, z16.b, z3.b[0]\n"
- "addvl c_ptr2, c_ptr2, #1\n"
- "sdot z25.s, z17.b, z1.b[1]\n"
- "st1w z29.s, p7, [c_ptr5]\n"
- "mov z29.s, #0\n"
- "addvl c_ptr3, c_ptr3, #1\n"
- "sdot z28.s, z16.b, z4.b[0]\n"
"st1w z30.s, p7, [c_ptr6]\n"
"mov z30.s, #0\n"
- "addvl c_ptr4, c_ptr4, #1\n"
- "sdot z29.s, z16.b, z5.b[0]\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "sdot z25.s, z16.b, z1.b[0]\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "sdot z26.s, z16.b, z2.b[0]\n"
"st1w z31.s, p7, [c_ptr7]\n"
"mov z31.s, #0\n"
- "addvl c_ptr5, c_ptr5, #1\n"
- "sdot z30.s, z16.b, z6.b[0]\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "sdot z27.s, z16.b, z3.b[0]\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "sdot z28.s, z16.b, z4.b[0]\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "sdot z24.s, z17.b, z0.b[1]\n"
"addvl c_ptr6, c_ptr6, #1\n"
+ "sdot z29.s, z16.b, z5.b[0]\n"
+ "addvl c_ptr7, c_ptr7, #1\n"
+ "sdot z30.s, z16.b, z6.b[0]\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
"sdot z31.s, z16.b, z7.b[0]\n"
"ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "sdot z25.s, z17.b, z1.b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
"sdot z26.s, z17.b, z2.b[1]\n"
- "addvl c_ptr7, c_ptr7, #1\n"
+ "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
"sdot z27.s, z17.b, z3.b[1]\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
"sdot z28.s, z17.b, z4.b[1]\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ "prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
"sdot z29.s, z17.b, z5.b[1]\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
"sdot z30.s, z17.b, z6.b[1]\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
"sdot z31.s, z17.b, z7.b[1]\n"
"ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"sdot z24.s, z18.b, z0.b[2]\n"
- "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
"sdot z25.s, z18.b, z1.b[2]\n"
- "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
"sdot z26.s, z18.b, z2.b[2]\n"
- "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
"sdot z27.s, z18.b, z3.b[2]\n"
- "prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
"sdot z28.s, z18.b, z4.b[2]\n"
"sdot z29.s, z18.b, z5.b[2]\n"
"sdot z30.s, z18.b, z6.b[2]\n"
@@ -5808,7 +6787,6 @@
"sdot z29.s, z22.b, z5.b[2]\n"
"sdot z30.s, z22.b, z6.b[2]\n"
"sdot z31.s, z22.b, z7.b[2]\n"
- "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
"sdot z24.s, z23.b, z0.b[3]\n"
"ld1rqb z0.b, p7/z, [%[a_ptr0], #0x20]\n"
"sdot z25.s, z23.b, z1.b[3]\n"
@@ -5826,7 +6804,6 @@
"sdot z31.s, z23.b, z7.b[3]\n"
"ld1rqb z7.b, p7/z, [a_ptr7, #0x20]\n"
"sdot z24.s, z16.b, z0.b[0]\n"
- "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"sdot z25.s, z16.b, z1.b[0]\n"
"sdot z26.s, z16.b, z2.b[0]\n"
"sdot z27.s, z16.b, z3.b[0]\n"
@@ -5834,7 +6811,6 @@
"sdot z29.s, z16.b, z5.b[0]\n"
"sdot z30.s, z16.b, z6.b[0]\n"
"sdot z31.s, z16.b, z7.b[0]\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
"sdot z24.s, z17.b, z0.b[1]\n"
"sdot z25.s, z17.b, z1.b[1]\n"
"sdot z26.s, z17.b, z2.b[1]\n"
@@ -5843,7 +6819,6 @@
"sdot z29.s, z17.b, z5.b[1]\n"
"sdot z30.s, z17.b, z6.b[1]\n"
"sdot z31.s, z17.b, z7.b[1]\n"
- "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"sdot z24.s, z18.b, z0.b[2]\n"
"sdot z25.s, z18.b, z1.b[2]\n"
"sdot z26.s, z18.b, z2.b[2]\n"
@@ -5852,7 +6827,6 @@
"sdot z29.s, z18.b, z5.b[2]\n"
"sdot z30.s, z18.b, z6.b[2]\n"
"sdot z31.s, z18.b, z7.b[2]\n"
- "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"sdot z24.s, z19.b, z0.b[3]\n"
"ld1rqb z0.b, p6/z, [%[a_ptr0], #0x30]\n"
"sdot z25.s, z19.b, z1.b[3]\n"
@@ -5870,7 +6844,6 @@
"sdot z31.s, z19.b, z7.b[3]\n"
"ld1rqb z7.b, p6/z, [a_ptr7, #0x30]\n"
"sdot z24.s, z20.b, z0.b[0]\n"
- "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"sdot z25.s, z20.b, z1.b[0]\n"
"sdot z26.s, z20.b, z2.b[0]\n"
"sdot z27.s, z20.b, z3.b[0]\n"
@@ -5878,7 +6851,6 @@
"sdot z29.s, z20.b, z5.b[0]\n"
"sdot z30.s, z20.b, z6.b[0]\n"
"sdot z31.s, z20.b, z7.b[0]\n"
- "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
"sdot z24.s, z21.b, z0.b[1]\n"
"sdot z25.s, z21.b, z1.b[1]\n"
"sdot z26.s, z21.b, z2.b[1]\n"
@@ -5887,53 +6859,60 @@
"sdot z29.s, z21.b, z5.b[1]\n"
"sdot z30.s, z21.b, z6.b[1]\n"
"sdot z31.s, z21.b, z7.b[1]\n"
- "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
"b.ne 4b\n"
"3:\n"
"st1w z24.s, p7, [%[c_ptr0]]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "mov z24.s, #0\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "ld1rqb z1.b, p7/z, [a_ptr1]\n"
"addvl %[c_ptr0], %[c_ptr0], #1\n"
+ "mov z24.s, #0\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"st1w z25.s, p7, [c_ptr1]\n"
"addvl c_ptr1, c_ptr1, #1\n"
"mov z25.s, #0\n"
+ "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "st1w z26.s, p7, [c_ptr2]\n"
+ "addvl c_ptr2, c_ptr2, #1\n"
+ "mov z26.s, #0\n"
+ "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "st1w z27.s, p7, [c_ptr3]\n"
+ "addvl c_ptr3, c_ptr3, #1\n"
+ "mov z27.s, #0\n"
+ "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #8\n"
+ "st1w z28.s, p7, [c_ptr4]\n"
+ "addvl c_ptr4, c_ptr4, #1\n"
+ "mov z28.s, #0\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "st1w z29.s, p7, [c_ptr5]\n"
+ "addvl c_ptr5, c_ptr5, #1\n"
+ "mov z29.s, #0\n"
"ld1rqb z2.b, p7/z, [a_ptr2]\n"
"sdot z24.s, z16.b, z0.b[0]\n"
- "st1w z26.s, p7, [c_ptr2]\n"
- "mov z26.s, #0\n"
- "ld1rqb z3.b, p7/z, [a_ptr3]\n"
- "ld1rqb z4.b, p7/z, [a_ptr4]\n"
- "addvl c_ptr2, c_ptr2, #1\n"
- "sdot z25.s, z16.b, z1.b[0]\n"
- "st1w z27.s, p7, [c_ptr3]\n"
- "mov z27.s, #0\n"
- "ld1rqb z5.b, p7/z, [a_ptr5]\n"
- "sdot z26.s, z16.b, z2.b[0]\n"
- "ld1rqb z6.b, p7/z, [a_ptr6]\n"
- "sdot z24.s, z17.b, z0.b[1]\n"
- "st1w z28.s, p7, [c_ptr4]\n"
- "mov z28.s, #0\n"
- "ld1rqb z7.b, p7/z, [a_ptr7]\n"
- "sdot z27.s, z16.b, z3.b[0]\n"
- "addvl c_ptr3, c_ptr3, #1\n"
- "sdot z25.s, z17.b, z1.b[1]\n"
- "st1w z29.s, p7, [c_ptr5]\n"
- "mov z29.s, #0\n"
- "addvl c_ptr4, c_ptr4, #1\n"
- "sdot z28.s, z16.b, z4.b[0]\n"
"st1w z30.s, p7, [c_ptr6]\n"
"mov z30.s, #0\n"
- "addvl c_ptr5, c_ptr5, #1\n"
- "sdot z29.s, z16.b, z5.b[0]\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "sdot z25.s, z16.b, z1.b[0]\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "sdot z26.s, z16.b, z2.b[0]\n"
"st1w z31.s, p7, [c_ptr7]\n"
"mov z31.s, #0\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "sdot z27.s, z16.b, z3.b[0]\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "sdot z28.s, z16.b, z4.b[0]\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "sdot z24.s, z17.b, z0.b[1]\n"
"addvl c_ptr6, c_ptr6, #1\n"
- "sdot z30.s, z16.b, z6.b[0]\n"
+ "sdot z29.s, z16.b, z5.b[0]\n"
"addvl c_ptr7, c_ptr7, #1\n"
+ "sdot z30.s, z16.b, z6.b[0]\n"
"sdot z31.s, z16.b, z7.b[0]\n"
"ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "sdot z25.s, z17.b, z1.b[1]\n"
"sdot z26.s, z17.b, z2.b[1]\n"
"sdot z27.s, z17.b, z3.b[1]\n"
"sdot z28.s, z17.b, z4.b[1]\n"
@@ -6066,23 +7045,177 @@
"sdot z29.s, z21.b, z5.b[1]\n"
"sdot z30.s, z21.b, z6.b[1]\n"
"sdot z31.s, z21.b, z7.b[1]\n"
+ "b 5f\n"
"2:\n"
+ "mov z24.s, #0\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "mov z25.s, #0\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "mov z26.s, #0\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2]\n"
+ "mov z27.s, #0\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "mov z28.s, #0\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "mov z29.s, #0\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "mov z30.s, #0\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "mov z31.s, #0\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "sdot z24.s, z16.b, z0.b[0]\n"
+ "sdot z25.s, z16.b, z1.b[0]\n"
+ "sdot z26.s, z16.b, z2.b[0]\n"
+ "sdot z27.s, z16.b, z3.b[0]\n"
+ "sdot z28.s, z16.b, z4.b[0]\n"
+ "sdot z29.s, z16.b, z5.b[0]\n"
+ "sdot z30.s, z16.b, z6.b[0]\n"
+ "sdot z31.s, z16.b, z7.b[0]\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "sdot z24.s, z17.b, z0.b[1]\n"
+ "sdot z25.s, z17.b, z1.b[1]\n"
+ "sdot z26.s, z17.b, z2.b[1]\n"
+ "sdot z27.s, z17.b, z3.b[1]\n"
+ "sdot z28.s, z17.b, z4.b[1]\n"
+ "sdot z29.s, z17.b, z5.b[1]\n"
+ "sdot z30.s, z17.b, z6.b[1]\n"
+ "sdot z31.s, z17.b, z7.b[1]\n"
+ "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "sdot z24.s, z18.b, z0.b[2]\n"
+ "sdot z25.s, z18.b, z1.b[2]\n"
+ "sdot z26.s, z18.b, z2.b[2]\n"
+ "sdot z27.s, z18.b, z3.b[2]\n"
+ "sdot z28.s, z18.b, z4.b[2]\n"
+ "sdot z29.s, z18.b, z5.b[2]\n"
+ "sdot z30.s, z18.b, z6.b[2]\n"
+ "sdot z31.s, z18.b, z7.b[2]\n"
+ "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "sdot z24.s, z19.b, z0.b[3]\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x10]\n"
+ "sdot z25.s, z19.b, z1.b[3]\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1, #0x10]\n"
+ "sdot z26.s, z19.b, z2.b[3]\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2, #0x10]\n"
+ "sdot z27.s, z19.b, z3.b[3]\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3, #0x10]\n"
+ "sdot z28.s, z19.b, z4.b[3]\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4, #0x10]\n"
+ "sdot z29.s, z19.b, z5.b[3]\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5, #0x10]\n"
+ "sdot z30.s, z19.b, z6.b[3]\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6, #0x10]\n"
+ "sdot z31.s, z19.b, z7.b[3]\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7, #0x10]\n"
+ "sdot z24.s, z20.b, z0.b[0]\n"
+ "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "sdot z25.s, z20.b, z1.b[0]\n"
+ "sdot z26.s, z20.b, z2.b[0]\n"
+ "sdot z27.s, z20.b, z3.b[0]\n"
+ "sdot z28.s, z20.b, z4.b[0]\n"
+ "sdot z29.s, z20.b, z5.b[0]\n"
+ "sdot z30.s, z20.b, z6.b[0]\n"
+ "sdot z31.s, z20.b, z7.b[0]\n"
+ "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "sdot z24.s, z21.b, z0.b[1]\n"
+ "sdot z25.s, z21.b, z1.b[1]\n"
+ "sdot z26.s, z21.b, z2.b[1]\n"
+ "sdot z27.s, z21.b, z3.b[1]\n"
+ "sdot z28.s, z21.b, z4.b[1]\n"
+ "sdot z29.s, z21.b, z5.b[1]\n"
+ "sdot z30.s, z21.b, z6.b[1]\n"
+ "sdot z31.s, z21.b, z7.b[1]\n"
+ "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "sdot z24.s, z22.b, z0.b[2]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #6\n"
+ "sdot z25.s, z22.b, z1.b[2]\n"
+ "sdot z26.s, z22.b, z2.b[2]\n"
+ "sdot z27.s, z22.b, z3.b[2]\n"
+ "sdot z28.s, z22.b, z4.b[2]\n"
+ "sdot z29.s, z22.b, z5.b[2]\n"
+ "sdot z30.s, z22.b, z6.b[2]\n"
+ "sdot z31.s, z22.b, z7.b[2]\n"
+ "sdot z24.s, z23.b, z0.b[3]\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x20]\n"
+ "sdot z25.s, z23.b, z1.b[3]\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1, #0x20]\n"
+ "sdot z26.s, z23.b, z2.b[3]\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2, #0x20]\n"
+ "sdot z27.s, z23.b, z3.b[3]\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3, #0x20]\n"
+ "sdot z28.s, z23.b, z4.b[3]\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4, #0x20]\n"
+ "sdot z29.s, z23.b, z5.b[3]\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5, #0x20]\n"
+ "sdot z30.s, z23.b, z6.b[3]\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6, #0x20]\n"
+ "sdot z31.s, z23.b, z7.b[3]\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7, #0x20]\n"
+ "sdot z24.s, z16.b, z0.b[0]\n"
+ "sdot z25.s, z16.b, z1.b[0]\n"
+ "sdot z26.s, z16.b, z2.b[0]\n"
+ "sdot z27.s, z16.b, z3.b[0]\n"
+ "sdot z28.s, z16.b, z4.b[0]\n"
+ "sdot z29.s, z16.b, z5.b[0]\n"
+ "sdot z30.s, z16.b, z6.b[0]\n"
+ "sdot z31.s, z16.b, z7.b[0]\n"
+ "sdot z24.s, z17.b, z0.b[1]\n"
+ "sdot z25.s, z17.b, z1.b[1]\n"
+ "sdot z26.s, z17.b, z2.b[1]\n"
+ "sdot z27.s, z17.b, z3.b[1]\n"
+ "sdot z28.s, z17.b, z4.b[1]\n"
+ "sdot z29.s, z17.b, z5.b[1]\n"
+ "sdot z30.s, z17.b, z6.b[1]\n"
+ "sdot z31.s, z17.b, z7.b[1]\n"
+ "sdot z24.s, z18.b, z0.b[2]\n"
+ "sdot z25.s, z18.b, z1.b[2]\n"
+ "sdot z26.s, z18.b, z2.b[2]\n"
+ "sdot z27.s, z18.b, z3.b[2]\n"
+ "sdot z28.s, z18.b, z4.b[2]\n"
+ "sdot z29.s, z18.b, z5.b[2]\n"
+ "sdot z30.s, z18.b, z6.b[2]\n"
+ "sdot z31.s, z18.b, z7.b[2]\n"
+ "sdot z24.s, z19.b, z0.b[3]\n"
+ "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x30]\n"
+ "sdot z25.s, z19.b, z1.b[3]\n"
+ "ld1rqb z1.b, p6/z, [a_ptr1, #0x30]\n"
+ "sdot z26.s, z19.b, z2.b[3]\n"
+ "ld1rqb z2.b, p6/z, [a_ptr2, #0x30]\n"
+ "sdot z27.s, z19.b, z3.b[3]\n"
+ "ld1rqb z3.b, p6/z, [a_ptr3, #0x30]\n"
+ "sdot z28.s, z19.b, z4.b[3]\n"
+ "ld1rqb z4.b, p6/z, [a_ptr4, #0x30]\n"
+ "sdot z29.s, z19.b, z5.b[3]\n"
+ "ld1rqb z5.b, p6/z, [a_ptr5, #0x30]\n"
+ "sdot z30.s, z19.b, z6.b[3]\n"
+ "ld1rqb z6.b, p6/z, [a_ptr6, #0x30]\n"
+ "sdot z31.s, z19.b, z7.b[3]\n"
+ "ld1rqb z7.b, p6/z, [a_ptr7, #0x30]\n"
+ "sdot z24.s, z20.b, z0.b[0]\n"
+ "sdot z25.s, z20.b, z1.b[0]\n"
+ "sdot z26.s, z20.b, z2.b[0]\n"
+ "sdot z27.s, z20.b, z3.b[0]\n"
+ "sdot z28.s, z20.b, z4.b[0]\n"
+ "sdot z29.s, z20.b, z5.b[0]\n"
+ "sdot z30.s, z20.b, z6.b[0]\n"
+ "sdot z31.s, z20.b, z7.b[0]\n"
+ "sdot z24.s, z21.b, z0.b[1]\n"
+ "sdot z25.s, z21.b, z1.b[1]\n"
+ "sdot z26.s, z21.b, z2.b[1]\n"
+ "sdot z27.s, z21.b, z3.b[1]\n"
+ "sdot z28.s, z21.b, z4.b[1]\n"
+ "sdot z29.s, z21.b, z5.b[1]\n"
+ "sdot z30.s, z21.b, z6.b[1]\n"
+ "sdot z31.s, z21.b, z7.b[1]\n"
+ "5:\n"
"st1w z24.s, p0, [%[c_ptr0]]\n"
"addvl %[c_ptr0], %[c_ptr0], #1\n"
"st1w z25.s, p0, [c_ptr1]\n"
- "addvl c_ptr1, c_ptr1, #1\n"
"st1w z26.s, p0, [c_ptr2]\n"
- "addvl c_ptr2, c_ptr2, #1\n"
"st1w z27.s, p0, [c_ptr3]\n"
- "addvl c_ptr3, c_ptr3, #1\n"
"st1w z28.s, p0, [c_ptr4]\n"
- "addvl c_ptr4, c_ptr4, #1\n"
"st1w z29.s, p0, [c_ptr5]\n"
- "addvl c_ptr5, c_ptr5, #1\n"
"st1w z30.s, p0, [c_ptr6]\n"
- "addvl c_ptr6, c_ptr6, #1\n"
"st1w z31.s, p0, [c_ptr7]\n"
- "addvl c_ptr7, c_ptr7, #1\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -6161,52 +7294,54 @@
"add c_ptr1, %[c_ptr0], #0x0\n"
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
- "mov z24.s, #0\n"
"ptrue p7.b\n"
- "mov z25.s, #0\n"
"whilelt p6.b, %[temp], %[odd_depth]\n"
- "mov z26.s, #0\n"
+ "whilelt p0.s, %[temp], %[last_width]\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
"ld1b z16.b, p7/z, [%[b_ptr0]]\n"
- "mov z27.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
"ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "mov z28.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
"ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "mov z29.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
"ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "mov z30.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
"ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "mov z31.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
"ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
"ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "whilelt p0.s, %[temp], %[last_width]\n"
"ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
- "ld1rqb z1.b, p7/z, [a_ptr1]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
- "sdot z24.s, z16.b, z0.b[0]\n"
- "ld1rqb z2.b, p7/z, [a_ptr2]\n"
- "ld1rqb z3.b, p7/z, [a_ptr3]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
- "sdot z25.s, z16.b, z1.b[0]\n"
- "ld1rqb z4.b, p7/z, [a_ptr4]\n"
- "sdot z26.s, z16.b, z2.b[0]\n"
- "ld1rqb z5.b, p7/z, [a_ptr5]\n"
- "sdot z27.s, z16.b, z3.b[0]\n"
- "ld1rqb z6.b, p7/z, [a_ptr6]\n"
- "sdot z24.s, z17.b, z0.b[1]\n"
- "ld1rqb z7.b, p7/z, [a_ptr7]\n"
- "sdot z28.s, z16.b, z4.b[0]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- "sdot z29.s, z16.b, z5.b[0]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- "sdot z30.s, z16.b, z6.b[0]\n"
"addvl %[b_ptr0], %[b_ptr0], #8\n"
+ "cbz %[loops], 2f\n"
+ "mov z24.s, #0\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "mov z25.s, #0\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "mov z26.s, #0\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2]\n"
+ "mov z27.s, #0\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "mov z28.s, #0\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "mov z29.s, #0\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "mov z30.s, #0\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "mov z31.s, #0\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "sdot z24.s, z16.b, z0.b[0]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "sdot z25.s, z16.b, z1.b[0]\n"
+ "sdot z26.s, z16.b, z2.b[0]\n"
+ "sdot z27.s, z16.b, z3.b[0]\n"
+ "sdot z28.s, z16.b, z4.b[0]\n"
+ "sdot z29.s, z16.b, z5.b[0]\n"
+ "sdot z30.s, z16.b, z6.b[0]\n"
"sdot z31.s, z16.b, z7.b[0]\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "sdot z24.s, z17.b, z0.b[1]\n"
"sdot z25.s, z17.b, z1.b[1]\n"
"sdot z26.s, z17.b, z2.b[1]\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
"sdot z27.s, z17.b, z3.b[1]\n"
"sdot z28.s, z17.b, z4.b[1]\n"
"sdot z29.s, z17.b, z5.b[1]\n"
@@ -6347,82 +7482,80 @@
"sdot z29.s, z22.b, z5.b[2]\n"
"sdot z30.s, z22.b, z6.b[2]\n"
"sdot z31.s, z22.b, z7.b[2]\n"
- "cbz %[loops], 2f\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"b.eq 3f\n"
"4:\n"
"st1w z24.s, p7, [%[c_ptr0]]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
+ "subs %[loops], %[loops], #0x1\n"
"mov z24.s, #0\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "addvl %[c_ptr0], %[c_ptr0], #1\n"
+ "st1w z25.s, p7, [c_ptr1]\n"
+ "addvl c_ptr1, c_ptr1, #1\n"
+ "mov z25.s, #0\n"
+ "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "st1w z26.s, p7, [c_ptr2]\n"
+ "addvl c_ptr2, c_ptr2, #1\n"
+ "mov z26.s, #0\n"
+ "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ "st1w z27.s, p7, [c_ptr3]\n"
+ "addvl c_ptr3, c_ptr3, #1\n"
+ "mov z27.s, #0\n"
+ "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #8\n"
+ "st1w z28.s, p7, [c_ptr4]\n"
+ "addvl c_ptr4, c_ptr4, #1\n"
+ "mov z28.s, #0\n"
"ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
"ld1rqb z1.b, p7/z, [a_ptr1]\n"
- "subs %[loops], %[loops], #0x1\n"
- "st1w z25.s, p7, [c_ptr1]\n"
- "addvl %[c_ptr0], %[c_ptr0], #1\n"
- "mov z25.s, #0\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ "st1w z29.s, p7, [c_ptr5]\n"
+ "addvl c_ptr5, c_ptr5, #1\n"
+ "mov z29.s, #0\n"
"ld1rqb z2.b, p7/z, [a_ptr2]\n"
"sdot z24.s, z16.b, z0.b[0]\n"
- "st1w z26.s, p7, [c_ptr2]\n"
- "mov z26.s, #0\n"
- "ld1rqb z3.b, p7/z, [a_ptr3]\n"
- "ld1rqb z4.b, p7/z, [a_ptr4]\n"
- "addvl c_ptr1, c_ptr1, #1\n"
- "sdot z25.s, z16.b, z1.b[0]\n"
- "st1w z27.s, p7, [c_ptr3]\n"
- "mov z27.s, #0\n"
- "ld1rqb z5.b, p7/z, [a_ptr5]\n"
- "sdot z26.s, z16.b, z2.b[0]\n"
- "ld1rqb z6.b, p7/z, [a_ptr6]\n"
- "sdot z24.s, z17.b, z0.b[1]\n"
- "st1w z28.s, p7, [c_ptr4]\n"
- "mov z28.s, #0\n"
- "ld1rqb z7.b, p7/z, [a_ptr7]\n"
- "sdot z27.s, z16.b, z3.b[0]\n"
- "addvl c_ptr2, c_ptr2, #1\n"
- "sdot z25.s, z17.b, z1.b[1]\n"
- "st1w z29.s, p7, [c_ptr5]\n"
- "mov z29.s, #0\n"
- "addvl c_ptr3, c_ptr3, #1\n"
- "sdot z28.s, z16.b, z4.b[0]\n"
"st1w z30.s, p7, [c_ptr6]\n"
"mov z30.s, #0\n"
- "addvl c_ptr4, c_ptr4, #1\n"
- "sdot z29.s, z16.b, z5.b[0]\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "sdot z25.s, z16.b, z1.b[0]\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "sdot z26.s, z16.b, z2.b[0]\n"
"st1w z31.s, p7, [c_ptr7]\n"
"mov z31.s, #0\n"
- "addvl c_ptr5, c_ptr5, #1\n"
- "sdot z30.s, z16.b, z6.b[0]\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "sdot z27.s, z16.b, z3.b[0]\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "sdot z28.s, z16.b, z4.b[0]\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "sdot z24.s, z17.b, z0.b[1]\n"
"addvl c_ptr6, c_ptr6, #1\n"
+ "sdot z29.s, z16.b, z5.b[0]\n"
+ "addvl c_ptr7, c_ptr7, #1\n"
+ "sdot z30.s, z16.b, z6.b[0]\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
"sdot z31.s, z16.b, z7.b[0]\n"
"ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "sdot z25.s, z17.b, z1.b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
"sdot z26.s, z17.b, z2.b[1]\n"
- "addvl c_ptr7, c_ptr7, #1\n"
+ "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
"sdot z27.s, z17.b, z3.b[1]\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
"sdot z28.s, z17.b, z4.b[1]\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ "prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
"sdot z29.s, z17.b, z5.b[1]\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
"sdot z30.s, z17.b, z6.b[1]\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
"sdot z31.s, z17.b, z7.b[1]\n"
"ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"sdot z24.s, z18.b, z0.b[2]\n"
- "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
"sdot z25.s, z18.b, z1.b[2]\n"
- "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
"sdot z26.s, z18.b, z2.b[2]\n"
- "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
"sdot z27.s, z18.b, z3.b[2]\n"
- "prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
"sdot z28.s, z18.b, z4.b[2]\n"
"sdot z29.s, z18.b, z5.b[2]\n"
"sdot z30.s, z18.b, z6.b[2]\n"
@@ -6493,12 +7626,10 @@
"sdot z25.s, z16.b, z1.b[0]\n"
"sdot z26.s, z16.b, z2.b[0]\n"
"sdot z27.s, z16.b, z3.b[0]\n"
- "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"sdot z28.s, z16.b, z4.b[0]\n"
"sdot z29.s, z16.b, z5.b[0]\n"
"sdot z30.s, z16.b, z6.b[0]\n"
"sdot z31.s, z16.b, z7.b[0]\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
"sdot z24.s, z17.b, z0.b[1]\n"
"sdot z25.s, z17.b, z1.b[1]\n"
"sdot z26.s, z17.b, z2.b[1]\n"
@@ -6507,7 +7638,6 @@
"sdot z29.s, z17.b, z5.b[1]\n"
"sdot z30.s, z17.b, z6.b[1]\n"
"sdot z31.s, z17.b, z7.b[1]\n"
- "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"sdot z24.s, z18.b, z0.b[2]\n"
"sdot z25.s, z18.b, z1.b[2]\n"
"sdot z26.s, z18.b, z2.b[2]\n"
@@ -6516,7 +7646,6 @@
"sdot z29.s, z18.b, z5.b[2]\n"
"sdot z30.s, z18.b, z6.b[2]\n"
"sdot z31.s, z18.b, z7.b[2]\n"
- "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"sdot z24.s, z19.b, z0.b[3]\n"
"ld1rqb z0.b, p6/z, [%[a_ptr0], #0x30]\n"
"sdot z25.s, z19.b, z1.b[3]\n"
@@ -6534,7 +7663,6 @@
"sdot z31.s, z19.b, z7.b[3]\n"
"ld1rqb z7.b, p6/z, [a_ptr7, #0x30]\n"
"sdot z24.s, z20.b, z0.b[0]\n"
- "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"sdot z25.s, z20.b, z1.b[0]\n"
"sdot z26.s, z20.b, z2.b[0]\n"
"sdot z27.s, z20.b, z3.b[0]\n"
@@ -6542,7 +7670,6 @@
"sdot z29.s, z20.b, z5.b[0]\n"
"sdot z30.s, z20.b, z6.b[0]\n"
"sdot z31.s, z20.b, z7.b[0]\n"
- "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
"sdot z24.s, z21.b, z0.b[1]\n"
"sdot z25.s, z21.b, z1.b[1]\n"
"sdot z26.s, z21.b, z2.b[1]\n"
@@ -6551,7 +7678,6 @@
"sdot z29.s, z21.b, z5.b[1]\n"
"sdot z30.s, z21.b, z6.b[1]\n"
"sdot z31.s, z21.b, z7.b[1]\n"
- "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
"sdot z24.s, z22.b, z0.b[2]\n"
"sdot z25.s, z22.b, z1.b[2]\n"
"sdot z26.s, z22.b, z2.b[2]\n"
@@ -6560,53 +7686,60 @@
"sdot z29.s, z22.b, z5.b[2]\n"
"sdot z30.s, z22.b, z6.b[2]\n"
"sdot z31.s, z22.b, z7.b[2]\n"
- "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
"b.ne 4b\n"
"3:\n"
"st1w z24.s, p7, [%[c_ptr0]]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "mov z24.s, #0\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "ld1rqb z1.b, p7/z, [a_ptr1]\n"
"addvl %[c_ptr0], %[c_ptr0], #1\n"
+ "mov z24.s, #0\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"st1w z25.s, p7, [c_ptr1]\n"
"addvl c_ptr1, c_ptr1, #1\n"
"mov z25.s, #0\n"
+ "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "st1w z26.s, p7, [c_ptr2]\n"
+ "addvl c_ptr2, c_ptr2, #1\n"
+ "mov z26.s, #0\n"
+ "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "st1w z27.s, p7, [c_ptr3]\n"
+ "addvl c_ptr3, c_ptr3, #1\n"
+ "mov z27.s, #0\n"
+ "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #8\n"
+ "st1w z28.s, p7, [c_ptr4]\n"
+ "addvl c_ptr4, c_ptr4, #1\n"
+ "mov z28.s, #0\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "st1w z29.s, p7, [c_ptr5]\n"
+ "addvl c_ptr5, c_ptr5, #1\n"
+ "mov z29.s, #0\n"
"ld1rqb z2.b, p7/z, [a_ptr2]\n"
"sdot z24.s, z16.b, z0.b[0]\n"
- "st1w z26.s, p7, [c_ptr2]\n"
- "mov z26.s, #0\n"
- "ld1rqb z3.b, p7/z, [a_ptr3]\n"
- "ld1rqb z4.b, p7/z, [a_ptr4]\n"
- "addvl c_ptr2, c_ptr2, #1\n"
- "sdot z25.s, z16.b, z1.b[0]\n"
- "st1w z27.s, p7, [c_ptr3]\n"
- "mov z27.s, #0\n"
- "ld1rqb z5.b, p7/z, [a_ptr5]\n"
- "sdot z26.s, z16.b, z2.b[0]\n"
- "ld1rqb z6.b, p7/z, [a_ptr6]\n"
- "sdot z24.s, z17.b, z0.b[1]\n"
- "st1w z28.s, p7, [c_ptr4]\n"
- "mov z28.s, #0\n"
- "ld1rqb z7.b, p7/z, [a_ptr7]\n"
- "sdot z27.s, z16.b, z3.b[0]\n"
- "addvl c_ptr3, c_ptr3, #1\n"
- "sdot z25.s, z17.b, z1.b[1]\n"
- "st1w z29.s, p7, [c_ptr5]\n"
- "mov z29.s, #0\n"
- "addvl c_ptr4, c_ptr4, #1\n"
- "sdot z28.s, z16.b, z4.b[0]\n"
"st1w z30.s, p7, [c_ptr6]\n"
"mov z30.s, #0\n"
- "addvl c_ptr5, c_ptr5, #1\n"
- "sdot z29.s, z16.b, z5.b[0]\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "sdot z25.s, z16.b, z1.b[0]\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "sdot z26.s, z16.b, z2.b[0]\n"
"st1w z31.s, p7, [c_ptr7]\n"
"mov z31.s, #0\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "sdot z27.s, z16.b, z3.b[0]\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "sdot z28.s, z16.b, z4.b[0]\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "sdot z24.s, z17.b, z0.b[1]\n"
"addvl c_ptr6, c_ptr6, #1\n"
- "sdot z30.s, z16.b, z6.b[0]\n"
+ "sdot z29.s, z16.b, z5.b[0]\n"
"addvl c_ptr7, c_ptr7, #1\n"
+ "sdot z30.s, z16.b, z6.b[0]\n"
"sdot z31.s, z16.b, z7.b[0]\n"
"ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "sdot z25.s, z17.b, z1.b[1]\n"
"sdot z26.s, z17.b, z2.b[1]\n"
"sdot z27.s, z17.b, z3.b[1]\n"
"sdot z28.s, z17.b, z4.b[1]\n"
@@ -6748,23 +7881,186 @@
"sdot z29.s, z22.b, z5.b[2]\n"
"sdot z30.s, z22.b, z6.b[2]\n"
"sdot z31.s, z22.b, z7.b[2]\n"
+ "b 5f\n"
"2:\n"
+ "mov z24.s, #0\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "mov z25.s, #0\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "mov z26.s, #0\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2]\n"
+ "mov z27.s, #0\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "mov z28.s, #0\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "mov z29.s, #0\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "mov z30.s, #0\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "mov z31.s, #0\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "sdot z24.s, z16.b, z0.b[0]\n"
+ "sdot z25.s, z16.b, z1.b[0]\n"
+ "sdot z26.s, z16.b, z2.b[0]\n"
+ "sdot z27.s, z16.b, z3.b[0]\n"
+ "sdot z28.s, z16.b, z4.b[0]\n"
+ "sdot z29.s, z16.b, z5.b[0]\n"
+ "sdot z30.s, z16.b, z6.b[0]\n"
+ "sdot z31.s, z16.b, z7.b[0]\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "sdot z24.s, z17.b, z0.b[1]\n"
+ "sdot z25.s, z17.b, z1.b[1]\n"
+ "sdot z26.s, z17.b, z2.b[1]\n"
+ "sdot z27.s, z17.b, z3.b[1]\n"
+ "sdot z28.s, z17.b, z4.b[1]\n"
+ "sdot z29.s, z17.b, z5.b[1]\n"
+ "sdot z30.s, z17.b, z6.b[1]\n"
+ "sdot z31.s, z17.b, z7.b[1]\n"
+ "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "sdot z24.s, z18.b, z0.b[2]\n"
+ "sdot z25.s, z18.b, z1.b[2]\n"
+ "sdot z26.s, z18.b, z2.b[2]\n"
+ "sdot z27.s, z18.b, z3.b[2]\n"
+ "sdot z28.s, z18.b, z4.b[2]\n"
+ "sdot z29.s, z18.b, z5.b[2]\n"
+ "sdot z30.s, z18.b, z6.b[2]\n"
+ "sdot z31.s, z18.b, z7.b[2]\n"
+ "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "sdot z24.s, z19.b, z0.b[3]\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x10]\n"
+ "sdot z25.s, z19.b, z1.b[3]\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1, #0x10]\n"
+ "sdot z26.s, z19.b, z2.b[3]\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2, #0x10]\n"
+ "sdot z27.s, z19.b, z3.b[3]\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3, #0x10]\n"
+ "sdot z28.s, z19.b, z4.b[3]\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4, #0x10]\n"
+ "sdot z29.s, z19.b, z5.b[3]\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5, #0x10]\n"
+ "sdot z30.s, z19.b, z6.b[3]\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6, #0x10]\n"
+ "sdot z31.s, z19.b, z7.b[3]\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7, #0x10]\n"
+ "sdot z24.s, z20.b, z0.b[0]\n"
+ "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "sdot z25.s, z20.b, z1.b[0]\n"
+ "sdot z26.s, z20.b, z2.b[0]\n"
+ "sdot z27.s, z20.b, z3.b[0]\n"
+ "sdot z28.s, z20.b, z4.b[0]\n"
+ "sdot z29.s, z20.b, z5.b[0]\n"
+ "sdot z30.s, z20.b, z6.b[0]\n"
+ "sdot z31.s, z20.b, z7.b[0]\n"
+ "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "sdot z24.s, z21.b, z0.b[1]\n"
+ "sdot z25.s, z21.b, z1.b[1]\n"
+ "sdot z26.s, z21.b, z2.b[1]\n"
+ "sdot z27.s, z21.b, z3.b[1]\n"
+ "sdot z28.s, z21.b, z4.b[1]\n"
+ "sdot z29.s, z21.b, z5.b[1]\n"
+ "sdot z30.s, z21.b, z6.b[1]\n"
+ "sdot z31.s, z21.b, z7.b[1]\n"
+ "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "sdot z24.s, z22.b, z0.b[2]\n"
+ "sdot z25.s, z22.b, z1.b[2]\n"
+ "sdot z26.s, z22.b, z2.b[2]\n"
+ "sdot z27.s, z22.b, z3.b[2]\n"
+ "sdot z28.s, z22.b, z4.b[2]\n"
+ "sdot z29.s, z22.b, z5.b[2]\n"
+ "sdot z30.s, z22.b, z6.b[2]\n"
+ "sdot z31.s, z22.b, z7.b[2]\n"
+ "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "sdot z24.s, z23.b, z0.b[3]\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x20]\n"
+ "sdot z25.s, z23.b, z1.b[3]\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1, #0x20]\n"
+ "sdot z26.s, z23.b, z2.b[3]\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2, #0x20]\n"
+ "sdot z27.s, z23.b, z3.b[3]\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3, #0x20]\n"
+ "sdot z28.s, z23.b, z4.b[3]\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4, #0x20]\n"
+ "sdot z29.s, z23.b, z5.b[3]\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5, #0x20]\n"
+ "sdot z30.s, z23.b, z6.b[3]\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6, #0x20]\n"
+ "sdot z31.s, z23.b, z7.b[3]\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7, #0x20]\n"
+ "sdot z24.s, z16.b, z0.b[0]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #7\n"
+ "sdot z25.s, z16.b, z1.b[0]\n"
+ "sdot z26.s, z16.b, z2.b[0]\n"
+ "sdot z27.s, z16.b, z3.b[0]\n"
+ "sdot z28.s, z16.b, z4.b[0]\n"
+ "sdot z29.s, z16.b, z5.b[0]\n"
+ "sdot z30.s, z16.b, z6.b[0]\n"
+ "sdot z31.s, z16.b, z7.b[0]\n"
+ "sdot z24.s, z17.b, z0.b[1]\n"
+ "sdot z25.s, z17.b, z1.b[1]\n"
+ "sdot z26.s, z17.b, z2.b[1]\n"
+ "sdot z27.s, z17.b, z3.b[1]\n"
+ "sdot z28.s, z17.b, z4.b[1]\n"
+ "sdot z29.s, z17.b, z5.b[1]\n"
+ "sdot z30.s, z17.b, z6.b[1]\n"
+ "sdot z31.s, z17.b, z7.b[1]\n"
+ "sdot z24.s, z18.b, z0.b[2]\n"
+ "sdot z25.s, z18.b, z1.b[2]\n"
+ "sdot z26.s, z18.b, z2.b[2]\n"
+ "sdot z27.s, z18.b, z3.b[2]\n"
+ "sdot z28.s, z18.b, z4.b[2]\n"
+ "sdot z29.s, z18.b, z5.b[2]\n"
+ "sdot z30.s, z18.b, z6.b[2]\n"
+ "sdot z31.s, z18.b, z7.b[2]\n"
+ "sdot z24.s, z19.b, z0.b[3]\n"
+ "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x30]\n"
+ "sdot z25.s, z19.b, z1.b[3]\n"
+ "ld1rqb z1.b, p6/z, [a_ptr1, #0x30]\n"
+ "sdot z26.s, z19.b, z2.b[3]\n"
+ "ld1rqb z2.b, p6/z, [a_ptr2, #0x30]\n"
+ "sdot z27.s, z19.b, z3.b[3]\n"
+ "ld1rqb z3.b, p6/z, [a_ptr3, #0x30]\n"
+ "sdot z28.s, z19.b, z4.b[3]\n"
+ "ld1rqb z4.b, p6/z, [a_ptr4, #0x30]\n"
+ "sdot z29.s, z19.b, z5.b[3]\n"
+ "ld1rqb z5.b, p6/z, [a_ptr5, #0x30]\n"
+ "sdot z30.s, z19.b, z6.b[3]\n"
+ "ld1rqb z6.b, p6/z, [a_ptr6, #0x30]\n"
+ "sdot z31.s, z19.b, z7.b[3]\n"
+ "ld1rqb z7.b, p6/z, [a_ptr7, #0x30]\n"
+ "sdot z24.s, z20.b, z0.b[0]\n"
+ "sdot z25.s, z20.b, z1.b[0]\n"
+ "sdot z26.s, z20.b, z2.b[0]\n"
+ "sdot z27.s, z20.b, z3.b[0]\n"
+ "sdot z28.s, z20.b, z4.b[0]\n"
+ "sdot z29.s, z20.b, z5.b[0]\n"
+ "sdot z30.s, z20.b, z6.b[0]\n"
+ "sdot z31.s, z20.b, z7.b[0]\n"
+ "sdot z24.s, z21.b, z0.b[1]\n"
+ "sdot z25.s, z21.b, z1.b[1]\n"
+ "sdot z26.s, z21.b, z2.b[1]\n"
+ "sdot z27.s, z21.b, z3.b[1]\n"
+ "sdot z28.s, z21.b, z4.b[1]\n"
+ "sdot z29.s, z21.b, z5.b[1]\n"
+ "sdot z30.s, z21.b, z6.b[1]\n"
+ "sdot z31.s, z21.b, z7.b[1]\n"
+ "sdot z24.s, z22.b, z0.b[2]\n"
+ "sdot z25.s, z22.b, z1.b[2]\n"
+ "sdot z26.s, z22.b, z2.b[2]\n"
+ "sdot z27.s, z22.b, z3.b[2]\n"
+ "sdot z28.s, z22.b, z4.b[2]\n"
+ "sdot z29.s, z22.b, z5.b[2]\n"
+ "sdot z30.s, z22.b, z6.b[2]\n"
+ "sdot z31.s, z22.b, z7.b[2]\n"
+ "5:\n"
"st1w z24.s, p0, [%[c_ptr0]]\n"
"addvl %[c_ptr0], %[c_ptr0], #1\n"
"st1w z25.s, p0, [c_ptr1]\n"
- "addvl c_ptr1, c_ptr1, #1\n"
"st1w z26.s, p0, [c_ptr2]\n"
- "addvl c_ptr2, c_ptr2, #1\n"
"st1w z27.s, p0, [c_ptr3]\n"
- "addvl c_ptr3, c_ptr3, #1\n"
"st1w z28.s, p0, [c_ptr4]\n"
- "addvl c_ptr4, c_ptr4, #1\n"
"st1w z29.s, p0, [c_ptr5]\n"
- "addvl c_ptr5, c_ptr5, #1\n"
"st1w z30.s, p0, [c_ptr6]\n"
- "addvl c_ptr6, c_ptr6, #1\n"
"st1w z31.s, p0, [c_ptr7]\n"
- "addvl c_ptr7, c_ptr7, #1\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -6844,52 +8140,54 @@
"add c_ptr1, %[c_ptr0], #0x0\n"
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
- "mov z24.s, #0\n"
"ptrue p7.b\n"
- "mov z25.s, #0\n"
"whilelt p6.b, %[temp], %[odd_depth]\n"
- "mov z26.s, #0\n"
+ "whilelt p0.s, %[temp], %[last_width]\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
"ld1b z16.b, p7/z, [%[b_ptr0]]\n"
- "mov z27.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
"ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "mov z28.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
"ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "mov z29.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
"ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "mov z30.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
"ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "mov z31.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
"ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
"ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "whilelt p0.s, %[temp], %[last_width]\n"
"ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
- "ld1rqb z1.b, p7/z, [a_ptr1]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
- "sdot z24.s, z16.b, z0.b[0]\n"
- "ld1rqb z2.b, p7/z, [a_ptr2]\n"
- "ld1rqb z3.b, p7/z, [a_ptr3]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
- "sdot z25.s, z16.b, z1.b[0]\n"
- "ld1rqb z4.b, p7/z, [a_ptr4]\n"
- "sdot z26.s, z16.b, z2.b[0]\n"
- "ld1rqb z5.b, p7/z, [a_ptr5]\n"
- "sdot z27.s, z16.b, z3.b[0]\n"
- "ld1rqb z6.b, p7/z, [a_ptr6]\n"
- "sdot z24.s, z17.b, z0.b[1]\n"
- "ld1rqb z7.b, p7/z, [a_ptr7]\n"
- "sdot z28.s, z16.b, z4.b[0]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- "sdot z29.s, z16.b, z5.b[0]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- "sdot z30.s, z16.b, z6.b[0]\n"
"addvl %[b_ptr0], %[b_ptr0], #8\n"
+ "cbz %[loops], 2f\n"
+ "mov z24.s, #0\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "mov z25.s, #0\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "mov z26.s, #0\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2]\n"
+ "mov z27.s, #0\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "mov z28.s, #0\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "mov z29.s, #0\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "mov z30.s, #0\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "mov z31.s, #0\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "sdot z24.s, z16.b, z0.b[0]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "sdot z25.s, z16.b, z1.b[0]\n"
+ "sdot z26.s, z16.b, z2.b[0]\n"
+ "sdot z27.s, z16.b, z3.b[0]\n"
+ "sdot z28.s, z16.b, z4.b[0]\n"
+ "sdot z29.s, z16.b, z5.b[0]\n"
+ "sdot z30.s, z16.b, z6.b[0]\n"
"sdot z31.s, z16.b, z7.b[0]\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "sdot z24.s, z17.b, z0.b[1]\n"
"sdot z25.s, z17.b, z1.b[1]\n"
"sdot z26.s, z17.b, z2.b[1]\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
"sdot z27.s, z17.b, z3.b[1]\n"
"sdot z28.s, z17.b, z4.b[1]\n"
"sdot z29.s, z17.b, z5.b[1]\n"
@@ -7039,83 +8337,81 @@
"sdot z29.s, z23.b, z5.b[3]\n"
"sdot z30.s, z23.b, z6.b[3]\n"
"sdot z31.s, z23.b, z7.b[3]\n"
- "cbz %[loops], 2f\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
"b.eq 3f\n"
"4:\n"
"st1w z24.s, p7, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"mov z24.s, #0\n"
- "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "st1w z25.s, p7, [c_ptr1]\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"addvl %[c_ptr0], %[c_ptr0], #1\n"
- "mov z25.s, #0\n"
- "ld1rqb z1.b, p7/z, [a_ptr1]\n"
- "sdot z24.s, z16.b, z0.b[0]\n"
- "st1w z26.s, p7, [c_ptr2]\n"
- "mov z26.s, #0\n"
- "ld1rqb z2.b, p7/z, [a_ptr2]\n"
- "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "st1w z25.s, p7, [c_ptr1]\n"
"addvl c_ptr1, c_ptr1, #1\n"
- "sdot z25.s, z16.b, z1.b[0]\n"
- "st1w z27.s, p7, [c_ptr3]\n"
- "mov z27.s, #0\n"
- "ld1rqb z4.b, p7/z, [a_ptr4]\n"
- "sdot z26.s, z16.b, z2.b[0]\n"
- "ld1rqb z5.b, p7/z, [a_ptr5]\n"
- "sdot z24.s, z17.b, z0.b[1]\n"
- "st1w z28.s, p7, [c_ptr4]\n"
- "mov z28.s, #0\n"
- "ld1rqb z6.b, p7/z, [a_ptr6]\n"
- "sdot z27.s, z16.b, z3.b[0]\n"
- "ld1rqb z7.b, p7/z, [a_ptr7]\n"
- "sdot z25.s, z17.b, z1.b[1]\n"
- "st1w z29.s, p7, [c_ptr5]\n"
- "mov z29.s, #0\n"
+ "mov z25.s, #0\n"
+ "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "st1w z26.s, p7, [c_ptr2]\n"
"addvl c_ptr2, c_ptr2, #1\n"
- "sdot z28.s, z16.b, z4.b[0]\n"
+ "mov z26.s, #0\n"
+ "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ "st1w z27.s, p7, [c_ptr3]\n"
+ "addvl c_ptr3, c_ptr3, #1\n"
+ "mov z27.s, #0\n"
+ "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #8\n"
+ "st1w z28.s, p7, [c_ptr4]\n"
+ "addvl c_ptr4, c_ptr4, #1\n"
+ "mov z28.s, #0\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ "st1w z29.s, p7, [c_ptr5]\n"
+ "addvl c_ptr5, c_ptr5, #1\n"
+ "mov z29.s, #0\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2]\n"
+ "sdot z24.s, z16.b, z0.b[0]\n"
"st1w z30.s, p7, [c_ptr6]\n"
"mov z30.s, #0\n"
- "addvl c_ptr3, c_ptr3, #1\n"
- "sdot z29.s, z16.b, z5.b[0]\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "sdot z25.s, z16.b, z1.b[0]\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "sdot z26.s, z16.b, z2.b[0]\n"
"st1w z31.s, p7, [c_ptr7]\n"
"mov z31.s, #0\n"
- "addvl c_ptr4, c_ptr4, #1\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "sdot z27.s, z16.b, z3.b[0]\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "sdot z28.s, z16.b, z4.b[0]\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "sdot z24.s, z17.b, z0.b[1]\n"
+ "addvl c_ptr6, c_ptr6, #1\n"
+ "sdot z29.s, z16.b, z5.b[0]\n"
+ "addvl c_ptr7, c_ptr7, #1\n"
"sdot z30.s, z16.b, z6.b[0]\n"
- "addvl c_ptr5, c_ptr5, #1\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
"sdot z31.s, z16.b, z7.b[0]\n"
"ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "sdot z25.s, z17.b, z1.b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
"sdot z26.s, z17.b, z2.b[1]\n"
- "addvl c_ptr6, c_ptr6, #1\n"
+ "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
"sdot z27.s, z17.b, z3.b[1]\n"
- "addvl c_ptr7, c_ptr7, #1\n"
+ "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
"sdot z28.s, z17.b, z4.b[1]\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
"sdot z29.s, z17.b, z5.b[1]\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
"sdot z30.s, z17.b, z6.b[1]\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
"sdot z31.s, z17.b, z7.b[1]\n"
"ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"sdot z24.s, z18.b, z0.b[2]\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
"sdot z25.s, z18.b, z1.b[2]\n"
- "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
"sdot z26.s, z18.b, z2.b[2]\n"
- "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
"sdot z27.s, z18.b, z3.b[2]\n"
- "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
"sdot z28.s, z18.b, z4.b[2]\n"
- "prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
"sdot z29.s, z18.b, z5.b[2]\n"
"sdot z30.s, z18.b, z6.b[2]\n"
"sdot z31.s, z18.b, z7.b[2]\n"
@@ -7190,7 +8486,6 @@
"sdot z29.s, z16.b, z5.b[0]\n"
"sdot z30.s, z16.b, z6.b[0]\n"
"sdot z31.s, z16.b, z7.b[0]\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
"sdot z24.s, z17.b, z0.b[1]\n"
"sdot z25.s, z17.b, z1.b[1]\n"
"sdot z26.s, z17.b, z2.b[1]\n"
@@ -7199,7 +8494,6 @@
"sdot z29.s, z17.b, z5.b[1]\n"
"sdot z30.s, z17.b, z6.b[1]\n"
"sdot z31.s, z17.b, z7.b[1]\n"
- "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"sdot z24.s, z18.b, z0.b[2]\n"
"sdot z25.s, z18.b, z1.b[2]\n"
"sdot z26.s, z18.b, z2.b[2]\n"
@@ -7208,7 +8502,6 @@
"sdot z29.s, z18.b, z5.b[2]\n"
"sdot z30.s, z18.b, z6.b[2]\n"
"sdot z31.s, z18.b, z7.b[2]\n"
- "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"sdot z24.s, z19.b, z0.b[3]\n"
"ld1rqb z0.b, p6/z, [%[a_ptr0], #0x30]\n"
"sdot z25.s, z19.b, z1.b[3]\n"
@@ -7226,7 +8519,6 @@
"sdot z31.s, z19.b, z7.b[3]\n"
"ld1rqb z7.b, p6/z, [a_ptr7, #0x30]\n"
"sdot z24.s, z20.b, z0.b[0]\n"
- "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"sdot z25.s, z20.b, z1.b[0]\n"
"sdot z26.s, z20.b, z2.b[0]\n"
"sdot z27.s, z20.b, z3.b[0]\n"
@@ -7234,7 +8526,6 @@
"sdot z29.s, z20.b, z5.b[0]\n"
"sdot z30.s, z20.b, z6.b[0]\n"
"sdot z31.s, z20.b, z7.b[0]\n"
- "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
"sdot z24.s, z21.b, z0.b[1]\n"
"sdot z25.s, z21.b, z1.b[1]\n"
"sdot z26.s, z21.b, z2.b[1]\n"
@@ -7243,7 +8534,6 @@
"sdot z29.s, z21.b, z5.b[1]\n"
"sdot z30.s, z21.b, z6.b[1]\n"
"sdot z31.s, z21.b, z7.b[1]\n"
- "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
"sdot z24.s, z22.b, z0.b[2]\n"
"sdot z25.s, z22.b, z1.b[2]\n"
"sdot z26.s, z22.b, z2.b[2]\n"
@@ -7252,7 +8542,6 @@
"sdot z29.s, z22.b, z5.b[2]\n"
"sdot z30.s, z22.b, z6.b[2]\n"
"sdot z31.s, z22.b, z7.b[2]\n"
- "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
"sdot z24.s, z23.b, z0.b[3]\n"
"sdot z25.s, z23.b, z1.b[3]\n"
"sdot z26.s, z23.b, z2.b[3]\n"
@@ -7266,49 +8555,56 @@
"st1w z24.s, p7, [%[c_ptr0]]\n"
"addvl %[c_ptr0], %[c_ptr0], #1\n"
"mov z24.s, #0\n"
- "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"st1w z25.s, p7, [c_ptr1]\n"
"addvl c_ptr1, c_ptr1, #1\n"
"mov z25.s, #0\n"
- "ld1rqb z1.b, p7/z, [a_ptr1]\n"
- "sdot z24.s, z16.b, z0.b[0]\n"
+ "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"st1w z26.s, p7, [c_ptr2]\n"
- "mov z26.s, #0\n"
- "ld1rqb z2.b, p7/z, [a_ptr2]\n"
- "ld1rqb z3.b, p7/z, [a_ptr3]\n"
"addvl c_ptr2, c_ptr2, #1\n"
- "sdot z25.s, z16.b, z1.b[0]\n"
+ "mov z26.s, #0\n"
+ "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
"st1w z27.s, p7, [c_ptr3]\n"
- "mov z27.s, #0\n"
- "ld1rqb z4.b, p7/z, [a_ptr4]\n"
- "sdot z26.s, z16.b, z2.b[0]\n"
- "ld1rqb z5.b, p7/z, [a_ptr5]\n"
- "sdot z24.s, z17.b, z0.b[1]\n"
- "st1w z28.s, p7, [c_ptr4]\n"
- "mov z28.s, #0\n"
- "ld1rqb z6.b, p7/z, [a_ptr6]\n"
- "sdot z27.s, z16.b, z3.b[0]\n"
- "ld1rqb z7.b, p7/z, [a_ptr7]\n"
- "sdot z25.s, z17.b, z1.b[1]\n"
- "st1w z29.s, p7, [c_ptr5]\n"
- "mov z29.s, #0\n"
"addvl c_ptr3, c_ptr3, #1\n"
- "sdot z28.s, z16.b, z4.b[0]\n"
+ "mov z27.s, #0\n"
+ "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #8\n"
+ "st1w z28.s, p7, [c_ptr4]\n"
+ "addvl c_ptr4, c_ptr4, #1\n"
+ "mov z28.s, #0\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "st1w z29.s, p7, [c_ptr5]\n"
+ "addvl c_ptr5, c_ptr5, #1\n"
+ "mov z29.s, #0\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2]\n"
+ "sdot z24.s, z16.b, z0.b[0]\n"
"st1w z30.s, p7, [c_ptr6]\n"
"mov z30.s, #0\n"
- "addvl c_ptr4, c_ptr4, #1\n"
- "sdot z29.s, z16.b, z5.b[0]\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "sdot z25.s, z16.b, z1.b[0]\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "sdot z26.s, z16.b, z2.b[0]\n"
"st1w z31.s, p7, [c_ptr7]\n"
"mov z31.s, #0\n"
- "addvl c_ptr5, c_ptr5, #1\n"
- "sdot z30.s, z16.b, z6.b[0]\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "sdot z27.s, z16.b, z3.b[0]\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "sdot z28.s, z16.b, z4.b[0]\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "sdot z24.s, z17.b, z0.b[1]\n"
"addvl c_ptr6, c_ptr6, #1\n"
+ "sdot z29.s, z16.b, z5.b[0]\n"
+ "addvl c_ptr7, c_ptr7, #1\n"
+ "sdot z30.s, z16.b, z6.b[0]\n"
"sdot z31.s, z16.b, z7.b[0]\n"
"ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "sdot z25.s, z17.b, z1.b[1]\n"
"sdot z26.s, z17.b, z2.b[1]\n"
- "addvl c_ptr7, c_ptr7, #1\n"
"sdot z27.s, z17.b, z3.b[1]\n"
"sdot z28.s, z17.b, z4.b[1]\n"
"sdot z29.s, z17.b, z5.b[1]\n"
@@ -7458,23 +8754,195 @@
"sdot z29.s, z23.b, z5.b[3]\n"
"sdot z30.s, z23.b, z6.b[3]\n"
"sdot z31.s, z23.b, z7.b[3]\n"
+ "b 5f\n"
"2:\n"
+ "mov z24.s, #0\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "mov z25.s, #0\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "mov z26.s, #0\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2]\n"
+ "mov z27.s, #0\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "mov z28.s, #0\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "mov z29.s, #0\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "mov z30.s, #0\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "mov z31.s, #0\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "sdot z24.s, z16.b, z0.b[0]\n"
+ "sdot z25.s, z16.b, z1.b[0]\n"
+ "sdot z26.s, z16.b, z2.b[0]\n"
+ "sdot z27.s, z16.b, z3.b[0]\n"
+ "sdot z28.s, z16.b, z4.b[0]\n"
+ "sdot z29.s, z16.b, z5.b[0]\n"
+ "sdot z30.s, z16.b, z6.b[0]\n"
+ "sdot z31.s, z16.b, z7.b[0]\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "sdot z24.s, z17.b, z0.b[1]\n"
+ "sdot z25.s, z17.b, z1.b[1]\n"
+ "sdot z26.s, z17.b, z2.b[1]\n"
+ "sdot z27.s, z17.b, z3.b[1]\n"
+ "sdot z28.s, z17.b, z4.b[1]\n"
+ "sdot z29.s, z17.b, z5.b[1]\n"
+ "sdot z30.s, z17.b, z6.b[1]\n"
+ "sdot z31.s, z17.b, z7.b[1]\n"
+ "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "sdot z24.s, z18.b, z0.b[2]\n"
+ "sdot z25.s, z18.b, z1.b[2]\n"
+ "sdot z26.s, z18.b, z2.b[2]\n"
+ "sdot z27.s, z18.b, z3.b[2]\n"
+ "sdot z28.s, z18.b, z4.b[2]\n"
+ "sdot z29.s, z18.b, z5.b[2]\n"
+ "sdot z30.s, z18.b, z6.b[2]\n"
+ "sdot z31.s, z18.b, z7.b[2]\n"
+ "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "sdot z24.s, z19.b, z0.b[3]\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x10]\n"
+ "sdot z25.s, z19.b, z1.b[3]\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1, #0x10]\n"
+ "sdot z26.s, z19.b, z2.b[3]\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2, #0x10]\n"
+ "sdot z27.s, z19.b, z3.b[3]\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3, #0x10]\n"
+ "sdot z28.s, z19.b, z4.b[3]\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4, #0x10]\n"
+ "sdot z29.s, z19.b, z5.b[3]\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5, #0x10]\n"
+ "sdot z30.s, z19.b, z6.b[3]\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6, #0x10]\n"
+ "sdot z31.s, z19.b, z7.b[3]\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7, #0x10]\n"
+ "sdot z24.s, z20.b, z0.b[0]\n"
+ "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "sdot z25.s, z20.b, z1.b[0]\n"
+ "sdot z26.s, z20.b, z2.b[0]\n"
+ "sdot z27.s, z20.b, z3.b[0]\n"
+ "sdot z28.s, z20.b, z4.b[0]\n"
+ "sdot z29.s, z20.b, z5.b[0]\n"
+ "sdot z30.s, z20.b, z6.b[0]\n"
+ "sdot z31.s, z20.b, z7.b[0]\n"
+ "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "sdot z24.s, z21.b, z0.b[1]\n"
+ "sdot z25.s, z21.b, z1.b[1]\n"
+ "sdot z26.s, z21.b, z2.b[1]\n"
+ "sdot z27.s, z21.b, z3.b[1]\n"
+ "sdot z28.s, z21.b, z4.b[1]\n"
+ "sdot z29.s, z21.b, z5.b[1]\n"
+ "sdot z30.s, z21.b, z6.b[1]\n"
+ "sdot z31.s, z21.b, z7.b[1]\n"
+ "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "sdot z24.s, z22.b, z0.b[2]\n"
+ "sdot z25.s, z22.b, z1.b[2]\n"
+ "sdot z26.s, z22.b, z2.b[2]\n"
+ "sdot z27.s, z22.b, z3.b[2]\n"
+ "sdot z28.s, z22.b, z4.b[2]\n"
+ "sdot z29.s, z22.b, z5.b[2]\n"
+ "sdot z30.s, z22.b, z6.b[2]\n"
+ "sdot z31.s, z22.b, z7.b[2]\n"
+ "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "sdot z24.s, z23.b, z0.b[3]\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x20]\n"
+ "sdot z25.s, z23.b, z1.b[3]\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1, #0x20]\n"
+ "sdot z26.s, z23.b, z2.b[3]\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2, #0x20]\n"
+ "sdot z27.s, z23.b, z3.b[3]\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3, #0x20]\n"
+ "sdot z28.s, z23.b, z4.b[3]\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4, #0x20]\n"
+ "sdot z29.s, z23.b, z5.b[3]\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5, #0x20]\n"
+ "sdot z30.s, z23.b, z6.b[3]\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6, #0x20]\n"
+ "sdot z31.s, z23.b, z7.b[3]\n"
+ "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "sdot z24.s, z16.b, z0.b[0]\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7, #0x20]\n"
+ "sdot z25.s, z16.b, z1.b[0]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #8\n"
+ "sdot z26.s, z16.b, z2.b[0]\n"
+ "sdot z27.s, z16.b, z3.b[0]\n"
+ "sdot z28.s, z16.b, z4.b[0]\n"
+ "sdot z29.s, z16.b, z5.b[0]\n"
+ "sdot z30.s, z16.b, z6.b[0]\n"
+ "sdot z31.s, z16.b, z7.b[0]\n"
+ "sdot z24.s, z17.b, z0.b[1]\n"
+ "sdot z25.s, z17.b, z1.b[1]\n"
+ "sdot z26.s, z17.b, z2.b[1]\n"
+ "sdot z27.s, z17.b, z3.b[1]\n"
+ "sdot z28.s, z17.b, z4.b[1]\n"
+ "sdot z29.s, z17.b, z5.b[1]\n"
+ "sdot z30.s, z17.b, z6.b[1]\n"
+ "sdot z31.s, z17.b, z7.b[1]\n"
+ "sdot z24.s, z18.b, z0.b[2]\n"
+ "sdot z25.s, z18.b, z1.b[2]\n"
+ "sdot z26.s, z18.b, z2.b[2]\n"
+ "sdot z27.s, z18.b, z3.b[2]\n"
+ "sdot z28.s, z18.b, z4.b[2]\n"
+ "sdot z29.s, z18.b, z5.b[2]\n"
+ "sdot z30.s, z18.b, z6.b[2]\n"
+ "sdot z31.s, z18.b, z7.b[2]\n"
+ "sdot z24.s, z19.b, z0.b[3]\n"
+ "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x30]\n"
+ "sdot z25.s, z19.b, z1.b[3]\n"
+ "ld1rqb z1.b, p6/z, [a_ptr1, #0x30]\n"
+ "sdot z26.s, z19.b, z2.b[3]\n"
+ "ld1rqb z2.b, p6/z, [a_ptr2, #0x30]\n"
+ "sdot z27.s, z19.b, z3.b[3]\n"
+ "ld1rqb z3.b, p6/z, [a_ptr3, #0x30]\n"
+ "sdot z28.s, z19.b, z4.b[3]\n"
+ "ld1rqb z4.b, p6/z, [a_ptr4, #0x30]\n"
+ "sdot z29.s, z19.b, z5.b[3]\n"
+ "ld1rqb z5.b, p6/z, [a_ptr5, #0x30]\n"
+ "sdot z30.s, z19.b, z6.b[3]\n"
+ "ld1rqb z6.b, p6/z, [a_ptr6, #0x30]\n"
+ "sdot z31.s, z19.b, z7.b[3]\n"
+ "ld1rqb z7.b, p6/z, [a_ptr7, #0x30]\n"
+ "sdot z24.s, z20.b, z0.b[0]\n"
+ "sdot z25.s, z20.b, z1.b[0]\n"
+ "sdot z26.s, z20.b, z2.b[0]\n"
+ "sdot z27.s, z20.b, z3.b[0]\n"
+ "sdot z28.s, z20.b, z4.b[0]\n"
+ "sdot z29.s, z20.b, z5.b[0]\n"
+ "sdot z30.s, z20.b, z6.b[0]\n"
+ "sdot z31.s, z20.b, z7.b[0]\n"
+ "sdot z24.s, z21.b, z0.b[1]\n"
+ "sdot z25.s, z21.b, z1.b[1]\n"
+ "sdot z26.s, z21.b, z2.b[1]\n"
+ "sdot z27.s, z21.b, z3.b[1]\n"
+ "sdot z28.s, z21.b, z4.b[1]\n"
+ "sdot z29.s, z21.b, z5.b[1]\n"
+ "sdot z30.s, z21.b, z6.b[1]\n"
+ "sdot z31.s, z21.b, z7.b[1]\n"
+ "sdot z24.s, z22.b, z0.b[2]\n"
+ "sdot z25.s, z22.b, z1.b[2]\n"
+ "sdot z26.s, z22.b, z2.b[2]\n"
+ "sdot z27.s, z22.b, z3.b[2]\n"
+ "sdot z28.s, z22.b, z4.b[2]\n"
+ "sdot z29.s, z22.b, z5.b[2]\n"
+ "sdot z30.s, z22.b, z6.b[2]\n"
+ "sdot z31.s, z22.b, z7.b[2]\n"
+ "sdot z24.s, z23.b, z0.b[3]\n"
+ "sdot z25.s, z23.b, z1.b[3]\n"
+ "sdot z26.s, z23.b, z2.b[3]\n"
+ "sdot z27.s, z23.b, z3.b[3]\n"
+ "sdot z28.s, z23.b, z4.b[3]\n"
+ "sdot z29.s, z23.b, z5.b[3]\n"
+ "sdot z30.s, z23.b, z6.b[3]\n"
+ "sdot z31.s, z23.b, z7.b[3]\n"
+ "5:\n"
"st1w z24.s, p0, [%[c_ptr0]]\n"
"addvl %[c_ptr0], %[c_ptr0], #1\n"
"st1w z25.s, p0, [c_ptr1]\n"
- "addvl c_ptr1, c_ptr1, #1\n"
"st1w z26.s, p0, [c_ptr2]\n"
- "addvl c_ptr2, c_ptr2, #1\n"
"st1w z27.s, p0, [c_ptr3]\n"
- "addvl c_ptr3, c_ptr3, #1\n"
"st1w z28.s, p0, [c_ptr4]\n"
- "addvl c_ptr4, c_ptr4, #1\n"
"st1w z29.s, p0, [c_ptr5]\n"
- "addvl c_ptr5, c_ptr5, #1\n"
"st1w z30.s, p0, [c_ptr6]\n"
- "addvl c_ptr6, c_ptr6, #1\n"
"st1w z31.s, p0, [c_ptr7]\n"
- "addvl c_ptr7, c_ptr7, #1\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_u8u32_dot_1VLx8.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_u8u32_dot_8x1VL.hpp
similarity index 90%
rename from src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_u8u32_dot_1VLx8.hpp
rename to src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_u8u32_dot_8x1VL.hpp
index 70a0b12..60184be 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_u8u32_dot_1VLx8.hpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_u8u32_dot_8x1VL.hpp
@@ -31,9 +31,9 @@
{
// Actual kernel implementations
-void sve_smallK_hybrid_u8u32_dot_1VLx8(const uint8_t *, int, const uint8_t *, uint32_t *, int, int, int, int, const uint32_t *, Activation, bool);
+void sve_smallK_hybrid_u8u32_dot_8x1VL(const uint8_t *, int, const uint8_t *, uint32_t *, int, int, int, int, const uint32_t *, Activation, bool);
-class smallK_hybrid_u8u32_dot_1VLx8
+class cls_sve_smallK_hybrid_u8u32_dot_8x1VL
{
public:
typedef uint8_t operand_type;
@@ -75,9 +75,9 @@
StdTransformsSVE<operand_type, result_type, 8, 1, 4> transforms = {};
// Default to the generic kernel
- kern_type kernel=sve_smallK_hybrid_u8u32_dot_1VLx8;
+ kern_type kernel=sve_smallK_hybrid_u8u32_dot_8x1VL;
- smallK_hybrid_u8u32_dot_1VLx8(const CPUInfo *)
+ cls_sve_smallK_hybrid_u8u32_dot_8x1VL(const CPUInfo *)
{
}
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_u8u32_dot_1VLx8/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_u8u32_dot_8x1VL/generic.cpp
similarity index 82%
rename from src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_u8u32_dot_1VLx8/generic.cpp
rename to src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_u8u32_dot_8x1VL/generic.cpp
index 1d0b84e..b980d9b 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_u8u32_dot_1VLx8/generic.cpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_smallK_hybrid_u8u32_dot_8x1VL/generic.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2019 Arm Limited.
+ * Copyright (c) 2019-2020 Arm Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -33,7 +33,7 @@
namespace arm_gemm {
-void sve_smallK_hybrid_u8u32_dot_1VLx8(const uint8_t *A, int lda, const uint8_t *B, uint32_t *C, int ldc, int M, int N, int K, const uint32_t *, Activation, bool) {
+void sve_smallK_hybrid_u8u32_dot_8x1VL(const uint8_t *A, int lda, const uint8_t *B, uint32_t *C, int ldc, int M, int N, int K, const uint32_t *bias, Activation act, bool) {
const long loops_count = iceildiv(N, (int)get_vector_length<uint32_t>()) - 1;
const long ldab = lda * sizeof(uint8_t);
const long ldcb = ldc * sizeof(uint32_t);
@@ -112,55 +112,54 @@
"add c_ptr1, %[c_ptr0], #0x0\n"
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
- "mov z24.s, #0\n"
"ptrue p7.b\n"
- "mov z25.s, #0\n"
"whilelt p6.b, %[temp], %[odd_depth]\n"
- "mov z26.s, #0\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
- "mov z27.s, #0\n"
"whilelt p0.s, %[temp], %[last_width]\n"
- "mov z28.s, #0\n"
- "ld1rqb z0.b, p6/z, [%[a_ptr0]]\n"
- "mov z29.s, #0\n"
- "ld1rqb z1.b, p6/z, [a_ptr1]\n"
- "mov z30.s, #0\n"
- "ld1rqb z2.b, p6/z, [a_ptr2]\n"
- "mov z31.s, #0\n"
- "ld1rqb z3.b, p6/z, [a_ptr3]\n"
- "udot z24.s, z16.b, z0.b[0]\n"
- "ld1rqb z4.b, p6/z, [a_ptr4]\n"
- "udot z25.s, z16.b, z1.b[0]\n"
- "ld1rqb z5.b, p6/z, [a_ptr5]\n"
- "udot z26.s, z16.b, z2.b[0]\n"
- "ld1rqb z6.b, p6/z, [a_ptr6]\n"
- "udot z27.s, z16.b, z3.b[0]\n"
- "ld1rqb z7.b, p6/z, [a_ptr7]\n"
- "udot z28.s, z16.b, z4.b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
- "udot z29.s, z16.b, z5.b[0]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
- "udot z30.s, z16.b, z6.b[0]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
- "udot z31.s, z16.b, z7.b[0]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- "addvl %[b_ptr0], %[b_ptr0], #1\n"
- "cbz %[loops], 2f\n"
- "subs %[loops], %[loops], #0x1\n"
"ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
+ "ld1rqb z0.b, p6/z, [%[a_ptr0]]\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
+ "ld1rqb z1.b, p6/z, [a_ptr1]\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
+ "ld1rqb z2.b, p6/z, [a_ptr2]\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
+ "ld1rqb z3.b, p6/z, [a_ptr3]\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
+ "ld1rqb z4.b, p6/z, [a_ptr4]\n"
"addvl %[b_ptr0], %[b_ptr0], #1\n"
+ "ld1rqb z5.b, p6/z, [a_ptr5]\n"
+ "ld1rqb z6.b, p6/z, [a_ptr6]\n"
+ "ld1rqb z7.b, p6/z, [a_ptr7]\n"
+ "cbz %[loops], 2f\n"
+ "mov z24.s, #0\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "mov z25.s, #0\n"
+ "mov z26.s, #0\n"
+ "mov z27.s, #0\n"
+ "mov z28.s, #0\n"
+ "mov z29.s, #0\n"
+ "mov z30.s, #0\n"
+ "mov z31.s, #0\n"
+ "udot z24.s, z16.b, z0.b[0]\n"
+ "udot z25.s, z16.b, z1.b[0]\n"
+ "udot z26.s, z16.b, z2.b[0]\n"
+ "udot z27.s, z16.b, z3.b[0]\n"
+ "udot z28.s, z16.b, z4.b[0]\n"
+ "udot z29.s, z16.b, z5.b[0]\n"
+ "udot z30.s, z16.b, z6.b[0]\n"
+ "udot z31.s, z16.b, z7.b[0]\n"
"b.eq 3f\n"
"4:\n"
"st1w z24.s, p7, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"mov z24.s, #0\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
"addvl %[c_ptr0], %[c_ptr0], #1\n"
"st1w z25.s, p7, [c_ptr1]\n"
"addvl c_ptr1, c_ptr1, #1\n"
"mov z25.s, #0\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #1\n"
"udot z24.s, z16.b, z0.b[0]\n"
"st1w z26.s, p7, [c_ptr2]\n"
"mov z26.s, #0\n"
@@ -186,10 +185,9 @@
"mov z31.s, #0\n"
"addvl c_ptr7, c_ptr7, #1\n"
"udot z30.s, z16.b, z6.b[0]\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
"udot z31.s, z16.b, z7.b[0]\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
- "addvl %[b_ptr0], %[b_ptr0], #1\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
"prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
"prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
"prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
@@ -201,6 +199,8 @@
"st1w z24.s, p7, [%[c_ptr0]]\n"
"addvl %[c_ptr0], %[c_ptr0], #1\n"
"mov z24.s, #0\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #1\n"
"st1w z25.s, p7, [c_ptr1]\n"
"addvl c_ptr1, c_ptr1, #1\n"
"mov z25.s, #0\n"
@@ -230,23 +230,34 @@
"addvl c_ptr7, c_ptr7, #1\n"
"udot z30.s, z16.b, z6.b[0]\n"
"udot z31.s, z16.b, z7.b[0]\n"
+ "b 5f\n"
"2:\n"
+ "mov z24.s, #0\n"
+ "mov z25.s, #0\n"
+ "mov z26.s, #0\n"
+ "mov z27.s, #0\n"
+ "mov z28.s, #0\n"
+ "mov z29.s, #0\n"
+ "mov z30.s, #0\n"
+ "mov z31.s, #0\n"
+ "udot z24.s, z16.b, z0.b[0]\n"
+ "udot z25.s, z16.b, z1.b[0]\n"
+ "udot z26.s, z16.b, z2.b[0]\n"
+ "udot z27.s, z16.b, z3.b[0]\n"
+ "udot z28.s, z16.b, z4.b[0]\n"
+ "udot z29.s, z16.b, z5.b[0]\n"
+ "udot z30.s, z16.b, z6.b[0]\n"
+ "udot z31.s, z16.b, z7.b[0]\n"
+ "5:\n"
"st1w z24.s, p0, [%[c_ptr0]]\n"
"addvl %[c_ptr0], %[c_ptr0], #1\n"
"st1w z25.s, p0, [c_ptr1]\n"
- "addvl c_ptr1, c_ptr1, #1\n"
"st1w z26.s, p0, [c_ptr2]\n"
- "addvl c_ptr2, c_ptr2, #1\n"
"st1w z27.s, p0, [c_ptr3]\n"
- "addvl c_ptr3, c_ptr3, #1\n"
"st1w z28.s, p0, [c_ptr4]\n"
- "addvl c_ptr4, c_ptr4, #1\n"
"st1w z29.s, p0, [c_ptr5]\n"
- "addvl c_ptr5, c_ptr5, #1\n"
"st1w z30.s, p0, [c_ptr6]\n"
- "addvl c_ptr6, c_ptr6, #1\n"
"st1w z31.s, p0, [c_ptr7]\n"
- "addvl c_ptr7, c_ptr7, #1\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -325,112 +336,112 @@
"add c_ptr1, %[c_ptr0], #0x0\n"
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
- "mov z24.s, #0\n"
"ptrue p7.b\n"
- "mov z25.s, #0\n"
"whilelt p6.b, %[temp], %[odd_depth]\n"
- "mov z26.s, #0\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
- "mov z27.s, #0\n"
- "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "mov z28.s, #0\n"
- "ld1rqb z0.b, p6/z, [%[a_ptr0]]\n"
- "mov z29.s, #0\n"
- "ld1rqb z1.b, p6/z, [a_ptr1]\n"
- "mov z30.s, #0\n"
- "ld1rqb z2.b, p6/z, [a_ptr2]\n"
- "mov z31.s, #0\n"
- "ld1rqb z3.b, p6/z, [a_ptr3]\n"
- "udot z24.s, z16.b, z0.b[0]\n"
- "ld1rqb z4.b, p6/z, [a_ptr4]\n"
- "udot z25.s, z16.b, z1.b[0]\n"
- "ld1rqb z5.b, p6/z, [a_ptr5]\n"
- "udot z26.s, z16.b, z2.b[0]\n"
- "ld1rqb z6.b, p6/z, [a_ptr6]\n"
- "udot z27.s, z16.b, z3.b[0]\n"
- "ld1rqb z7.b, p6/z, [a_ptr7]\n"
- "udot z28.s, z16.b, z4.b[0]\n"
"whilelt p0.s, %[temp], %[last_width]\n"
- "udot z29.s, z16.b, z5.b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
- "udot z30.s, z16.b, z6.b[0]\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
- "udot z31.s, z16.b, z7.b[0]\n"
+ "ld1rqb z0.b, p6/z, [%[a_ptr0]]\n"
"prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
- "udot z24.s, z17.b, z0.b[1]\n"
+ "ld1rqb z1.b, p6/z, [a_ptr1]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
- "udot z25.s, z17.b, z1.b[1]\n"
+ "ld1rqb z2.b, p6/z, [a_ptr2]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- "udot z26.s, z17.b, z2.b[1]\n"
+ "ld1rqb z3.b, p6/z, [a_ptr3]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- "udot z27.s, z17.b, z3.b[1]\n"
+ "ld1rqb z4.b, p6/z, [a_ptr4]\n"
+ "ld1rqb z5.b, p6/z, [a_ptr5]\n"
+ "ld1rqb z6.b, p6/z, [a_ptr6]\n"
+ "ld1rqb z7.b, p6/z, [a_ptr7]\n"
+ "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"addvl %[b_ptr0], %[b_ptr0], #2\n"
+ "cbz %[loops], 2f\n"
+ "mov z24.s, #0\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "mov z25.s, #0\n"
+ "mov z26.s, #0\n"
+ "mov z27.s, #0\n"
+ "mov z28.s, #0\n"
+ "mov z29.s, #0\n"
+ "mov z30.s, #0\n"
+ "mov z31.s, #0\n"
+ "udot z24.s, z16.b, z0.b[0]\n"
+ "udot z25.s, z16.b, z1.b[0]\n"
+ "udot z26.s, z16.b, z2.b[0]\n"
+ "udot z27.s, z16.b, z3.b[0]\n"
+ "udot z28.s, z16.b, z4.b[0]\n"
+ "udot z29.s, z16.b, z5.b[0]\n"
+ "udot z30.s, z16.b, z6.b[0]\n"
+ "udot z31.s, z16.b, z7.b[0]\n"
+ "udot z24.s, z17.b, z0.b[1]\n"
+ "udot z25.s, z17.b, z1.b[1]\n"
+ "udot z26.s, z17.b, z2.b[1]\n"
+ "udot z27.s, z17.b, z3.b[1]\n"
"udot z28.s, z17.b, z4.b[1]\n"
"udot z29.s, z17.b, z5.b[1]\n"
"udot z30.s, z17.b, z6.b[1]\n"
"udot z31.s, z17.b, z7.b[1]\n"
- "cbz %[loops], 2f\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
"b.eq 3f\n"
"4:\n"
"st1w z24.s, p7, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"mov z24.s, #0\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
"ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #2\n"
- "st1w z25.s, p7, [c_ptr1]\n"
"addvl %[c_ptr0], %[c_ptr0], #1\n"
- "mov z25.s, #0\n"
+ "st1w z25.s, p7, [c_ptr1]\n"
"addvl c_ptr1, c_ptr1, #1\n"
+ "mov z25.s, #0\n"
+ "addvl %[b_ptr0], %[b_ptr0], #2\n"
"udot z24.s, z16.b, z0.b[0]\n"
"st1w z26.s, p7, [c_ptr2]\n"
"mov z26.s, #0\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "addvl c_ptr2, c_ptr2, #1\n"
"udot z25.s, z16.b, z1.b[0]\n"
"st1w z27.s, p7, [c_ptr3]\n"
"mov z27.s, #0\n"
- "addvl c_ptr2, c_ptr2, #1\n"
+ "addvl c_ptr3, c_ptr3, #1\n"
"udot z26.s, z16.b, z2.b[0]\n"
"st1w z28.s, p7, [c_ptr4]\n"
"mov z28.s, #0\n"
- "addvl c_ptr3, c_ptr3, #1\n"
+ "addvl c_ptr4, c_ptr4, #1\n"
"udot z27.s, z16.b, z3.b[0]\n"
"st1w z29.s, p7, [c_ptr5]\n"
"mov z29.s, #0\n"
- "addvl c_ptr4, c_ptr4, #1\n"
+ "addvl c_ptr5, c_ptr5, #1\n"
"udot z28.s, z16.b, z4.b[0]\n"
"st1w z30.s, p7, [c_ptr6]\n"
"mov z30.s, #0\n"
- "addvl c_ptr5, c_ptr5, #1\n"
+ "addvl c_ptr6, c_ptr6, #1\n"
"udot z29.s, z16.b, z5.b[0]\n"
"st1w z31.s, p7, [c_ptr7]\n"
"mov z31.s, #0\n"
- "addvl c_ptr6, c_ptr6, #1\n"
- "udot z30.s, z16.b, z6.b[0]\n"
"addvl c_ptr7, c_ptr7, #1\n"
+ "udot z30.s, z16.b, z6.b[0]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
"udot z31.s, z16.b, z7.b[0]\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
- "udot z24.s, z17.b, z0.b[1]\n"
"prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- "udot z25.s, z17.b, z1.b[1]\n"
+ "udot z24.s, z17.b, z0.b[1]\n"
"prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- "udot z26.s, z17.b, z2.b[1]\n"
+ "udot z25.s, z17.b, z1.b[1]\n"
"prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- "udot z27.s, z17.b, z3.b[1]\n"
+ "udot z26.s, z17.b, z2.b[1]\n"
"prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- "udot z28.s, z17.b, z4.b[1]\n"
+ "udot z27.s, z17.b, z3.b[1]\n"
"prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- "udot z29.s, z17.b, z5.b[1]\n"
+ "udot z28.s, z17.b, z4.b[1]\n"
"prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
- "udot z30.s, z17.b, z6.b[1]\n"
+ "udot z29.s, z17.b, z5.b[1]\n"
"prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
+ "udot z30.s, z17.b, z6.b[1]\n"
"udot z31.s, z17.b, z7.b[1]\n"
"b.ne 4b\n"
"3:\n"
"st1w z24.s, p7, [%[c_ptr0]]\n"
"addvl %[c_ptr0], %[c_ptr0], #1\n"
"mov z24.s, #0\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
"ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"addvl %[b_ptr0], %[b_ptr0], #2\n"
"st1w z25.s, p7, [c_ptr1]\n"
@@ -470,23 +481,42 @@
"udot z29.s, z17.b, z5.b[1]\n"
"udot z30.s, z17.b, z6.b[1]\n"
"udot z31.s, z17.b, z7.b[1]\n"
+ "b 5f\n"
"2:\n"
+ "mov z24.s, #0\n"
+ "mov z25.s, #0\n"
+ "mov z26.s, #0\n"
+ "mov z27.s, #0\n"
+ "mov z28.s, #0\n"
+ "mov z29.s, #0\n"
+ "mov z30.s, #0\n"
+ "mov z31.s, #0\n"
+ "udot z24.s, z16.b, z0.b[0]\n"
+ "udot z25.s, z16.b, z1.b[0]\n"
+ "udot z26.s, z16.b, z2.b[0]\n"
+ "udot z27.s, z16.b, z3.b[0]\n"
+ "udot z28.s, z16.b, z4.b[0]\n"
+ "udot z29.s, z16.b, z5.b[0]\n"
+ "udot z30.s, z16.b, z6.b[0]\n"
+ "udot z31.s, z16.b, z7.b[0]\n"
+ "udot z24.s, z17.b, z0.b[1]\n"
+ "udot z25.s, z17.b, z1.b[1]\n"
+ "udot z26.s, z17.b, z2.b[1]\n"
+ "udot z27.s, z17.b, z3.b[1]\n"
+ "udot z28.s, z17.b, z4.b[1]\n"
+ "udot z29.s, z17.b, z5.b[1]\n"
+ "udot z30.s, z17.b, z6.b[1]\n"
+ "udot z31.s, z17.b, z7.b[1]\n"
+ "5:\n"
"st1w z24.s, p0, [%[c_ptr0]]\n"
"addvl %[c_ptr0], %[c_ptr0], #1\n"
"st1w z25.s, p0, [c_ptr1]\n"
- "addvl c_ptr1, c_ptr1, #1\n"
"st1w z26.s, p0, [c_ptr2]\n"
- "addvl c_ptr2, c_ptr2, #1\n"
"st1w z27.s, p0, [c_ptr3]\n"
- "addvl c_ptr3, c_ptr3, #1\n"
"st1w z28.s, p0, [c_ptr4]\n"
- "addvl c_ptr4, c_ptr4, #1\n"
"st1w z29.s, p0, [c_ptr5]\n"
- "addvl c_ptr5, c_ptr5, #1\n"
"st1w z30.s, p0, [c_ptr6]\n"
- "addvl c_ptr6, c_ptr6, #1\n"
"st1w z31.s, p0, [c_ptr7]\n"
- "addvl c_ptr7, c_ptr7, #1\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -565,48 +595,50 @@
"add c_ptr1, %[c_ptr0], #0x0\n"
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
- "mov z24.s, #0\n"
"ptrue p7.b\n"
- "mov z25.s, #0\n"
"whilelt p6.b, %[temp], %[odd_depth]\n"
- "mov z26.s, #0\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
- "mov z27.s, #0\n"
- "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "mov z28.s, #0\n"
- "ld1rqb z0.b, p6/z, [%[a_ptr0]]\n"
- "mov z29.s, #0\n"
- "ld1rqb z1.b, p6/z, [a_ptr1]\n"
- "mov z30.s, #0\n"
- "ld1rqb z2.b, p6/z, [a_ptr2]\n"
- "mov z31.s, #0\n"
- "ld1rqb z3.b, p6/z, [a_ptr3]\n"
- "udot z24.s, z16.b, z0.b[0]\n"
- "ld1rqb z4.b, p6/z, [a_ptr4]\n"
- "udot z25.s, z16.b, z1.b[0]\n"
- "ld1rqb z5.b, p6/z, [a_ptr5]\n"
- "udot z26.s, z16.b, z2.b[0]\n"
- "ld1rqb z6.b, p6/z, [a_ptr6]\n"
- "udot z27.s, z16.b, z3.b[0]\n"
- "ld1rqb z7.b, p6/z, [a_ptr7]\n"
- "udot z28.s, z16.b, z4.b[0]\n"
- "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "udot z29.s, z16.b, z5.b[0]\n"
"whilelt p0.s, %[temp], %[last_width]\n"
- "udot z30.s, z16.b, z6.b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
- "udot z31.s, z16.b, z7.b[0]\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
- "udot z24.s, z17.b, z0.b[1]\n"
+ "ld1rqb z0.b, p6/z, [%[a_ptr0]]\n"
"prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
- "udot z25.s, z17.b, z1.b[1]\n"
+ "ld1rqb z1.b, p6/z, [a_ptr1]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
- "udot z26.s, z17.b, z2.b[1]\n"
+ "ld1rqb z2.b, p6/z, [a_ptr2]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- "udot z27.s, z17.b, z3.b[1]\n"
+ "ld1rqb z3.b, p6/z, [a_ptr3]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- "udot z28.s, z17.b, z4.b[1]\n"
+ "ld1rqb z4.b, p6/z, [a_ptr4]\n"
+ "ld1rqb z5.b, p6/z, [a_ptr5]\n"
+ "ld1rqb z6.b, p6/z, [a_ptr6]\n"
+ "ld1rqb z7.b, p6/z, [a_ptr7]\n"
+ "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"addvl %[b_ptr0], %[b_ptr0], #3\n"
+ "cbz %[loops], 2f\n"
+ "mov z24.s, #0\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "mov z25.s, #0\n"
+ "mov z26.s, #0\n"
+ "mov z27.s, #0\n"
+ "mov z28.s, #0\n"
+ "mov z29.s, #0\n"
+ "mov z30.s, #0\n"
+ "mov z31.s, #0\n"
+ "udot z24.s, z16.b, z0.b[0]\n"
+ "udot z25.s, z16.b, z1.b[0]\n"
+ "udot z26.s, z16.b, z2.b[0]\n"
+ "udot z27.s, z16.b, z3.b[0]\n"
+ "udot z28.s, z16.b, z4.b[0]\n"
+ "udot z29.s, z16.b, z5.b[0]\n"
+ "udot z30.s, z16.b, z6.b[0]\n"
+ "udot z31.s, z16.b, z7.b[0]\n"
+ "udot z24.s, z17.b, z0.b[1]\n"
+ "udot z25.s, z17.b, z1.b[1]\n"
+ "udot z26.s, z17.b, z2.b[1]\n"
+ "udot z27.s, z17.b, z3.b[1]\n"
+ "udot z28.s, z17.b, z4.b[1]\n"
"udot z29.s, z17.b, z5.b[1]\n"
"udot z30.s, z17.b, z6.b[1]\n"
"udot z31.s, z17.b, z7.b[1]\n"
@@ -618,49 +650,46 @@
"udot z29.s, z18.b, z5.b[2]\n"
"udot z30.s, z18.b, z6.b[2]\n"
"udot z31.s, z18.b, z7.b[2]\n"
- "cbz %[loops], 2f\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"b.eq 3f\n"
"4:\n"
"st1w z24.s, p7, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"mov z24.s, #0\n"
- "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #3\n"
- "st1w z25.s, p7, [c_ptr1]\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"addvl %[c_ptr0], %[c_ptr0], #1\n"
- "mov z25.s, #0\n"
+ "st1w z25.s, p7, [c_ptr1]\n"
"addvl c_ptr1, c_ptr1, #1\n"
+ "mov z25.s, #0\n"
+ "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"udot z24.s, z16.b, z0.b[0]\n"
"st1w z26.s, p7, [c_ptr2]\n"
"mov z26.s, #0\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "addvl c_ptr2, c_ptr2, #1\n"
"udot z25.s, z16.b, z1.b[0]\n"
"st1w z27.s, p7, [c_ptr3]\n"
"mov z27.s, #0\n"
- "addvl c_ptr2, c_ptr2, #1\n"
+ "addvl c_ptr3, c_ptr3, #1\n"
"udot z26.s, z16.b, z2.b[0]\n"
"st1w z28.s, p7, [c_ptr4]\n"
"mov z28.s, #0\n"
- "addvl c_ptr3, c_ptr3, #1\n"
+ "addvl c_ptr4, c_ptr4, #1\n"
"udot z27.s, z16.b, z3.b[0]\n"
"st1w z29.s, p7, [c_ptr5]\n"
"mov z29.s, #0\n"
- "addvl c_ptr4, c_ptr4, #1\n"
+ "addvl c_ptr5, c_ptr5, #1\n"
"udot z28.s, z16.b, z4.b[0]\n"
"st1w z30.s, p7, [c_ptr6]\n"
"mov z30.s, #0\n"
- "addvl c_ptr5, c_ptr5, #1\n"
+ "addvl c_ptr6, c_ptr6, #1\n"
"udot z29.s, z16.b, z5.b[0]\n"
"st1w z31.s, p7, [c_ptr7]\n"
"mov z31.s, #0\n"
- "addvl c_ptr6, c_ptr6, #1\n"
- "udot z30.s, z16.b, z6.b[0]\n"
"addvl c_ptr7, c_ptr7, #1\n"
+ "udot z30.s, z16.b, z6.b[0]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #3\n"
"udot z31.s, z16.b, z7.b[0]\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
"udot z24.s, z17.b, z0.b[1]\n"
"prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
"udot z25.s, z17.b, z1.b[1]\n"
@@ -676,7 +705,6 @@
"udot z30.s, z17.b, z6.b[1]\n"
"prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
"udot z31.s, z17.b, z7.b[1]\n"
- "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"udot z24.s, z18.b, z0.b[2]\n"
"udot z25.s, z18.b, z1.b[2]\n"
"udot z26.s, z18.b, z2.b[2]\n"
@@ -690,11 +718,12 @@
"st1w z24.s, p7, [%[c_ptr0]]\n"
"addvl %[c_ptr0], %[c_ptr0], #1\n"
"mov z24.s, #0\n"
- "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #3\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"st1w z25.s, p7, [c_ptr1]\n"
"addvl c_ptr1, c_ptr1, #1\n"
"mov z25.s, #0\n"
+ "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"udot z24.s, z16.b, z0.b[0]\n"
"st1w z26.s, p7, [c_ptr2]\n"
"mov z26.s, #0\n"
@@ -720,8 +749,9 @@
"mov z31.s, #0\n"
"addvl c_ptr7, c_ptr7, #1\n"
"udot z30.s, z16.b, z6.b[0]\n"
- "udot z24.s, z17.b, z0.b[1]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #3\n"
"udot z31.s, z16.b, z7.b[0]\n"
+ "udot z24.s, z17.b, z0.b[1]\n"
"udot z25.s, z17.b, z1.b[1]\n"
"udot z26.s, z17.b, z2.b[1]\n"
"udot z27.s, z17.b, z3.b[1]\n"
@@ -737,23 +767,50 @@
"udot z29.s, z18.b, z5.b[2]\n"
"udot z30.s, z18.b, z6.b[2]\n"
"udot z31.s, z18.b, z7.b[2]\n"
+ "b 5f\n"
"2:\n"
+ "mov z24.s, #0\n"
+ "mov z25.s, #0\n"
+ "mov z26.s, #0\n"
+ "mov z27.s, #0\n"
+ "mov z28.s, #0\n"
+ "mov z29.s, #0\n"
+ "mov z30.s, #0\n"
+ "mov z31.s, #0\n"
+ "udot z24.s, z16.b, z0.b[0]\n"
+ "udot z25.s, z16.b, z1.b[0]\n"
+ "udot z26.s, z16.b, z2.b[0]\n"
+ "udot z27.s, z16.b, z3.b[0]\n"
+ "udot z28.s, z16.b, z4.b[0]\n"
+ "udot z29.s, z16.b, z5.b[0]\n"
+ "udot z30.s, z16.b, z6.b[0]\n"
+ "udot z31.s, z16.b, z7.b[0]\n"
+ "udot z24.s, z17.b, z0.b[1]\n"
+ "udot z25.s, z17.b, z1.b[1]\n"
+ "udot z26.s, z17.b, z2.b[1]\n"
+ "udot z27.s, z17.b, z3.b[1]\n"
+ "udot z28.s, z17.b, z4.b[1]\n"
+ "udot z29.s, z17.b, z5.b[1]\n"
+ "udot z30.s, z17.b, z6.b[1]\n"
+ "udot z31.s, z17.b, z7.b[1]\n"
+ "udot z24.s, z18.b, z0.b[2]\n"
+ "udot z25.s, z18.b, z1.b[2]\n"
+ "udot z26.s, z18.b, z2.b[2]\n"
+ "udot z27.s, z18.b, z3.b[2]\n"
+ "udot z28.s, z18.b, z4.b[2]\n"
+ "udot z29.s, z18.b, z5.b[2]\n"
+ "udot z30.s, z18.b, z6.b[2]\n"
+ "udot z31.s, z18.b, z7.b[2]\n"
+ "5:\n"
"st1w z24.s, p0, [%[c_ptr0]]\n"
"addvl %[c_ptr0], %[c_ptr0], #1\n"
"st1w z25.s, p0, [c_ptr1]\n"
- "addvl c_ptr1, c_ptr1, #1\n"
"st1w z26.s, p0, [c_ptr2]\n"
- "addvl c_ptr2, c_ptr2, #1\n"
"st1w z27.s, p0, [c_ptr3]\n"
- "addvl c_ptr3, c_ptr3, #1\n"
"st1w z28.s, p0, [c_ptr4]\n"
- "addvl c_ptr4, c_ptr4, #1\n"
"st1w z29.s, p0, [c_ptr5]\n"
- "addvl c_ptr5, c_ptr5, #1\n"
"st1w z30.s, p0, [c_ptr6]\n"
- "addvl c_ptr6, c_ptr6, #1\n"
"st1w z31.s, p0, [c_ptr7]\n"
- "addvl c_ptr7, c_ptr7, #1\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -832,50 +889,52 @@
"add c_ptr1, %[c_ptr0], #0x0\n"
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
- "mov z24.s, #0\n"
"ptrue p7.b\n"
- "mov z25.s, #0\n"
"whilelt p6.b, %[temp], %[odd_depth]\n"
- "mov z26.s, #0\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
- "mov z27.s, #0\n"
- "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "mov z28.s, #0\n"
- "ld1rqb z0.b, p6/z, [%[a_ptr0]]\n"
- "mov z29.s, #0\n"
- "ld1rqb z1.b, p6/z, [a_ptr1]\n"
- "mov z30.s, #0\n"
- "ld1rqb z2.b, p6/z, [a_ptr2]\n"
- "mov z31.s, #0\n"
- "ld1rqb z3.b, p6/z, [a_ptr3]\n"
- "udot z24.s, z16.b, z0.b[0]\n"
- "ld1rqb z4.b, p6/z, [a_ptr4]\n"
- "udot z25.s, z16.b, z1.b[0]\n"
- "ld1rqb z5.b, p6/z, [a_ptr5]\n"
- "udot z26.s, z16.b, z2.b[0]\n"
- "ld1rqb z6.b, p6/z, [a_ptr6]\n"
- "udot z27.s, z16.b, z3.b[0]\n"
- "ld1rqb z7.b, p6/z, [a_ptr7]\n"
- "udot z28.s, z16.b, z4.b[0]\n"
- "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "udot z29.s, z16.b, z5.b[0]\n"
- "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "udot z30.s, z16.b, z6.b[0]\n"
"whilelt p0.s, %[temp], %[last_width]\n"
- "udot z31.s, z16.b, z7.b[0]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
- "udot z24.s, z17.b, z0.b[1]\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
- "udot z25.s, z17.b, z1.b[1]\n"
+ "ld1rqb z0.b, p6/z, [%[a_ptr0]]\n"
"prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
- "udot z26.s, z17.b, z2.b[1]\n"
+ "ld1rqb z1.b, p6/z, [a_ptr1]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
- "udot z27.s, z17.b, z3.b[1]\n"
+ "ld1rqb z2.b, p6/z, [a_ptr2]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- "udot z28.s, z17.b, z4.b[1]\n"
+ "ld1rqb z3.b, p6/z, [a_ptr3]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- "udot z29.s, z17.b, z5.b[1]\n"
+ "ld1rqb z4.b, p6/z, [a_ptr4]\n"
+ "ld1rqb z5.b, p6/z, [a_ptr5]\n"
+ "ld1rqb z6.b, p6/z, [a_ptr6]\n"
+ "ld1rqb z7.b, p6/z, [a_ptr7]\n"
+ "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"addvl %[b_ptr0], %[b_ptr0], #4\n"
+ "cbz %[loops], 2f\n"
+ "mov z24.s, #0\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "mov z25.s, #0\n"
+ "mov z26.s, #0\n"
+ "mov z27.s, #0\n"
+ "mov z28.s, #0\n"
+ "mov z29.s, #0\n"
+ "mov z30.s, #0\n"
+ "mov z31.s, #0\n"
+ "udot z24.s, z16.b, z0.b[0]\n"
+ "udot z25.s, z16.b, z1.b[0]\n"
+ "udot z26.s, z16.b, z2.b[0]\n"
+ "udot z27.s, z16.b, z3.b[0]\n"
+ "udot z28.s, z16.b, z4.b[0]\n"
+ "udot z29.s, z16.b, z5.b[0]\n"
+ "udot z30.s, z16.b, z6.b[0]\n"
+ "udot z31.s, z16.b, z7.b[0]\n"
+ "udot z24.s, z17.b, z0.b[1]\n"
+ "udot z25.s, z17.b, z1.b[1]\n"
+ "udot z26.s, z17.b, z2.b[1]\n"
+ "udot z27.s, z17.b, z3.b[1]\n"
+ "udot z28.s, z17.b, z4.b[1]\n"
+ "udot z29.s, z17.b, z5.b[1]\n"
"udot z30.s, z17.b, z6.b[1]\n"
"udot z31.s, z17.b, z7.b[1]\n"
"udot z24.s, z18.b, z0.b[2]\n"
@@ -894,50 +953,47 @@
"udot z29.s, z19.b, z5.b[3]\n"
"udot z30.s, z19.b, z6.b[3]\n"
"udot z31.s, z19.b, z7.b[3]\n"
- "cbz %[loops], 2f\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"b.eq 3f\n"
"4:\n"
"st1w z24.s, p7, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"mov z24.s, #0\n"
- "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #4\n"
- "st1w z25.s, p7, [c_ptr1]\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"addvl %[c_ptr0], %[c_ptr0], #1\n"
- "mov z25.s, #0\n"
+ "st1w z25.s, p7, [c_ptr1]\n"
"addvl c_ptr1, c_ptr1, #1\n"
+ "mov z25.s, #0\n"
+ "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"udot z24.s, z16.b, z0.b[0]\n"
"st1w z26.s, p7, [c_ptr2]\n"
"mov z26.s, #0\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "addvl c_ptr2, c_ptr2, #1\n"
"udot z25.s, z16.b, z1.b[0]\n"
"st1w z27.s, p7, [c_ptr3]\n"
"mov z27.s, #0\n"
- "addvl c_ptr2, c_ptr2, #1\n"
+ "addvl c_ptr3, c_ptr3, #1\n"
"udot z26.s, z16.b, z2.b[0]\n"
"st1w z28.s, p7, [c_ptr4]\n"
"mov z28.s, #0\n"
- "addvl c_ptr3, c_ptr3, #1\n"
+ "addvl c_ptr4, c_ptr4, #1\n"
"udot z27.s, z16.b, z3.b[0]\n"
"st1w z29.s, p7, [c_ptr5]\n"
"mov z29.s, #0\n"
- "addvl c_ptr4, c_ptr4, #1\n"
+ "addvl c_ptr5, c_ptr5, #1\n"
"udot z28.s, z16.b, z4.b[0]\n"
"st1w z30.s, p7, [c_ptr6]\n"
"mov z30.s, #0\n"
- "addvl c_ptr5, c_ptr5, #1\n"
+ "addvl c_ptr6, c_ptr6, #1\n"
"udot z29.s, z16.b, z5.b[0]\n"
"st1w z31.s, p7, [c_ptr7]\n"
"mov z31.s, #0\n"
- "addvl c_ptr6, c_ptr6, #1\n"
- "udot z30.s, z16.b, z6.b[0]\n"
"addvl c_ptr7, c_ptr7, #1\n"
+ "udot z30.s, z16.b, z6.b[0]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #4\n"
"udot z31.s, z16.b, z7.b[0]\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
"udot z24.s, z17.b, z0.b[1]\n"
"prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
"udot z25.s, z17.b, z1.b[1]\n"
@@ -953,7 +1009,6 @@
"udot z30.s, z17.b, z6.b[1]\n"
"prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
"udot z31.s, z17.b, z7.b[1]\n"
- "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"udot z24.s, z18.b, z0.b[2]\n"
"udot z25.s, z18.b, z1.b[2]\n"
"udot z26.s, z18.b, z2.b[2]\n"
@@ -962,7 +1017,6 @@
"udot z29.s, z18.b, z5.b[2]\n"
"udot z30.s, z18.b, z6.b[2]\n"
"udot z31.s, z18.b, z7.b[2]\n"
- "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"udot z24.s, z19.b, z0.b[3]\n"
"udot z25.s, z19.b, z1.b[3]\n"
"udot z26.s, z19.b, z2.b[3]\n"
@@ -976,14 +1030,16 @@
"st1w z24.s, p7, [%[c_ptr0]]\n"
"addvl %[c_ptr0], %[c_ptr0], #1\n"
"mov z24.s, #0\n"
- "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "addvl %[b_ptr0], %[b_ptr0], #4\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"st1w z25.s, p7, [c_ptr1]\n"
"addvl c_ptr1, c_ptr1, #1\n"
"mov z25.s, #0\n"
+ "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"udot z24.s, z16.b, z0.b[0]\n"
"st1w z26.s, p7, [c_ptr2]\n"
"mov z26.s, #0\n"
+ "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"addvl c_ptr2, c_ptr2, #1\n"
"udot z25.s, z16.b, z1.b[0]\n"
"st1w z27.s, p7, [c_ptr3]\n"
@@ -1006,8 +1062,9 @@
"mov z31.s, #0\n"
"addvl c_ptr7, c_ptr7, #1\n"
"udot z30.s, z16.b, z6.b[0]\n"
- "udot z24.s, z17.b, z0.b[1]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #4\n"
"udot z31.s, z16.b, z7.b[0]\n"
+ "udot z24.s, z17.b, z0.b[1]\n"
"udot z25.s, z17.b, z1.b[1]\n"
"udot z26.s, z17.b, z2.b[1]\n"
"udot z27.s, z17.b, z3.b[1]\n"
@@ -1031,23 +1088,58 @@
"udot z29.s, z19.b, z5.b[3]\n"
"udot z30.s, z19.b, z6.b[3]\n"
"udot z31.s, z19.b, z7.b[3]\n"
+ "b 5f\n"
"2:\n"
+ "mov z24.s, #0\n"
+ "mov z25.s, #0\n"
+ "mov z26.s, #0\n"
+ "mov z27.s, #0\n"
+ "mov z28.s, #0\n"
+ "mov z29.s, #0\n"
+ "mov z30.s, #0\n"
+ "mov z31.s, #0\n"
+ "udot z24.s, z16.b, z0.b[0]\n"
+ "udot z25.s, z16.b, z1.b[0]\n"
+ "udot z26.s, z16.b, z2.b[0]\n"
+ "udot z27.s, z16.b, z3.b[0]\n"
+ "udot z28.s, z16.b, z4.b[0]\n"
+ "udot z29.s, z16.b, z5.b[0]\n"
+ "udot z30.s, z16.b, z6.b[0]\n"
+ "udot z31.s, z16.b, z7.b[0]\n"
+ "udot z24.s, z17.b, z0.b[1]\n"
+ "udot z25.s, z17.b, z1.b[1]\n"
+ "udot z26.s, z17.b, z2.b[1]\n"
+ "udot z27.s, z17.b, z3.b[1]\n"
+ "udot z28.s, z17.b, z4.b[1]\n"
+ "udot z29.s, z17.b, z5.b[1]\n"
+ "udot z30.s, z17.b, z6.b[1]\n"
+ "udot z31.s, z17.b, z7.b[1]\n"
+ "udot z24.s, z18.b, z0.b[2]\n"
+ "udot z25.s, z18.b, z1.b[2]\n"
+ "udot z26.s, z18.b, z2.b[2]\n"
+ "udot z27.s, z18.b, z3.b[2]\n"
+ "udot z28.s, z18.b, z4.b[2]\n"
+ "udot z29.s, z18.b, z5.b[2]\n"
+ "udot z30.s, z18.b, z6.b[2]\n"
+ "udot z31.s, z18.b, z7.b[2]\n"
+ "udot z24.s, z19.b, z0.b[3]\n"
+ "udot z25.s, z19.b, z1.b[3]\n"
+ "udot z26.s, z19.b, z2.b[3]\n"
+ "udot z27.s, z19.b, z3.b[3]\n"
+ "udot z28.s, z19.b, z4.b[3]\n"
+ "udot z29.s, z19.b, z5.b[3]\n"
+ "udot z30.s, z19.b, z6.b[3]\n"
+ "udot z31.s, z19.b, z7.b[3]\n"
+ "5:\n"
"st1w z24.s, p0, [%[c_ptr0]]\n"
"addvl %[c_ptr0], %[c_ptr0], #1\n"
"st1w z25.s, p0, [c_ptr1]\n"
- "addvl c_ptr1, c_ptr1, #1\n"
"st1w z26.s, p0, [c_ptr2]\n"
- "addvl c_ptr2, c_ptr2, #1\n"
"st1w z27.s, p0, [c_ptr3]\n"
- "addvl c_ptr3, c_ptr3, #1\n"
"st1w z28.s, p0, [c_ptr4]\n"
- "addvl c_ptr4, c_ptr4, #1\n"
"st1w z29.s, p0, [c_ptr5]\n"
- "addvl c_ptr5, c_ptr5, #1\n"
"st1w z30.s, p0, [c_ptr6]\n"
- "addvl c_ptr6, c_ptr6, #1\n"
"st1w z31.s, p0, [c_ptr7]\n"
- "addvl c_ptr7, c_ptr7, #1\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -1126,46 +1218,48 @@
"add c_ptr1, %[c_ptr0], #0x0\n"
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
- "mov z24.s, #0\n"
"ptrue p7.b\n"
- "mov z25.s, #0\n"
"whilelt p6.b, %[temp], %[odd_depth]\n"
- "mov z26.s, #0\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
- "mov z27.s, #0\n"
- "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "mov z28.s, #0\n"
- "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "mov z29.s, #0\n"
- "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "mov z30.s, #0\n"
- "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "mov z31.s, #0\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "ld1rqb z1.b, p7/z, [a_ptr1]\n"
"whilelt p0.s, %[temp], %[last_width]\n"
- "ld1rqb z2.b, p7/z, [a_ptr2]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
- "udot z24.s, z16.b, z0.b[0]\n"
- "ld1rqb z3.b, p7/z, [a_ptr3]\n"
- "udot z25.s, z16.b, z1.b[0]\n"
- "ld1rqb z4.b, p7/z, [a_ptr4]\n"
- "udot z26.s, z16.b, z2.b[0]\n"
- "ld1rqb z5.b, p7/z, [a_ptr5]\n"
- "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
- "udot z27.s, z16.b, z3.b[0]\n"
- "ld1rqb z7.b, p7/z, [a_ptr7]\n"
- "udot z28.s, z16.b, z4.b[0]\n"
+ "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
- "udot z29.s, z16.b, z5.b[0]\n"
+ "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
- "udot z30.s, z16.b, z6.b[0]\n"
+ "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- "udot z31.s, z16.b, z7.b[0]\n"
+ "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- "udot z24.s, z17.b, z0.b[1]\n"
"addvl %[b_ptr0], %[b_ptr0], #5\n"
+ "cbz %[loops], 2f\n"
+ "mov z24.s, #0\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "mov z25.s, #0\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "mov z26.s, #0\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2]\n"
+ "mov z27.s, #0\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "mov z28.s, #0\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "mov z29.s, #0\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "mov z30.s, #0\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "mov z31.s, #0\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "udot z24.s, z16.b, z0.b[0]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "udot z25.s, z16.b, z1.b[0]\n"
+ "udot z26.s, z16.b, z2.b[0]\n"
+ "udot z27.s, z16.b, z3.b[0]\n"
+ "udot z28.s, z16.b, z4.b[0]\n"
+ "udot z29.s, z16.b, z5.b[0]\n"
+ "udot z30.s, z16.b, z6.b[0]\n"
+ "udot z31.s, z16.b, z7.b[0]\n"
+ "udot z24.s, z17.b, z0.b[1]\n"
"udot z25.s, z17.b, z1.b[1]\n"
"udot z26.s, z17.b, z2.b[1]\n"
"udot z27.s, z17.b, z3.b[1]\n"
@@ -1205,84 +1299,79 @@
"udot z29.s, z20.b, z5.b[0]\n"
"udot z30.s, z20.b, z6.b[0]\n"
"udot z31.s, z20.b, z7.b[0]\n"
- "cbz %[loops], 2f\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"b.eq 3f\n"
"4:\n"
"st1w z24.s, p7, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"mov z24.s, #0\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "addvl %[c_ptr0], %[c_ptr0], #1\n"
+ "st1w z25.s, p7, [c_ptr1]\n"
+ "addvl c_ptr1, c_ptr1, #1\n"
+ "mov z25.s, #0\n"
+ "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "st1w z26.s, p7, [c_ptr2]\n"
+ "addvl c_ptr2, c_ptr2, #1\n"
+ "mov z26.s, #0\n"
"ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
"ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
"addvl %[b_ptr0], %[b_ptr0], #5\n"
- "st1w z25.s, p7, [c_ptr1]\n"
- "addvl %[c_ptr0], %[c_ptr0], #1\n"
- "mov z25.s, #0\n"
+ "st1w z27.s, p7, [c_ptr3]\n"
+ "addvl c_ptr3, c_ptr3, #1\n"
+ "mov z27.s, #0\n"
"ld1rqb z1.b, p7/z, [a_ptr1]\n"
"udot z24.s, z16.b, z0.b[0]\n"
- "st1w z26.s, p7, [c_ptr2]\n"
- "mov z26.s, #0\n"
+ "st1w z28.s, p7, [c_ptr4]\n"
+ "mov z28.s, #0\n"
"ld1rqb z2.b, p7/z, [a_ptr2]\n"
"ld1rqb z3.b, p7/z, [a_ptr3]\n"
- "addvl c_ptr1, c_ptr1, #1\n"
+ "addvl c_ptr4, c_ptr4, #1\n"
"udot z25.s, z16.b, z1.b[0]\n"
- "st1w z27.s, p7, [c_ptr3]\n"
- "mov z27.s, #0\n"
+ "st1w z29.s, p7, [c_ptr5]\n"
+ "mov z29.s, #0\n"
"ld1rqb z4.b, p7/z, [a_ptr4]\n"
"udot z26.s, z16.b, z2.b[0]\n"
"ld1rqb z5.b, p7/z, [a_ptr5]\n"
- "udot z24.s, z17.b, z0.b[1]\n"
- "st1w z28.s, p7, [c_ptr4]\n"
- "mov z28.s, #0\n"
- "ld1rqb z6.b, p7/z, [a_ptr6]\n"
"udot z27.s, z16.b, z3.b[0]\n"
- "ld1rqb z7.b, p7/z, [a_ptr7]\n"
- "udot z25.s, z17.b, z1.b[1]\n"
- "st1w z29.s, p7, [c_ptr5]\n"
- "mov z29.s, #0\n"
- "addvl c_ptr2, c_ptr2, #1\n"
- "udot z28.s, z16.b, z4.b[0]\n"
"st1w z30.s, p7, [c_ptr6]\n"
"mov z30.s, #0\n"
- "addvl c_ptr3, c_ptr3, #1\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "udot z28.s, z16.b, z4.b[0]\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
"udot z29.s, z16.b, z5.b[0]\n"
"st1w z31.s, p7, [c_ptr7]\n"
"mov z31.s, #0\n"
- "addvl c_ptr4, c_ptr4, #1\n"
- "udot z30.s, z16.b, z6.b[0]\n"
"addvl c_ptr5, c_ptr5, #1\n"
- "udot z31.s, z16.b, z7.b[0]\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
- "udot z26.s, z17.b, z2.b[1]\n"
+ "udot z30.s, z16.b, z6.b[0]\n"
"addvl c_ptr6, c_ptr6, #1\n"
- "udot z27.s, z17.b, z3.b[1]\n"
+ "udot z31.s, z16.b, z7.b[0]\n"
"addvl c_ptr7, c_ptr7, #1\n"
- "udot z28.s, z17.b, z4.b[1]\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
- "udot z29.s, z17.b, z5.b[1]\n"
+ "udot z24.s, z17.b, z0.b[1]\n"
"prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- "udot z30.s, z17.b, z6.b[1]\n"
+ "udot z25.s, z17.b, z1.b[1]\n"
"prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- "udot z31.s, z17.b, z7.b[1]\n"
- "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "udot z24.s, z18.b, z0.b[2]\n"
+ "udot z26.s, z17.b, z2.b[1]\n"
"prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- "udot z25.s, z18.b, z1.b[2]\n"
+ "udot z27.s, z17.b, z3.b[1]\n"
"prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- "udot z26.s, z18.b, z2.b[2]\n"
+ "udot z28.s, z17.b, z4.b[1]\n"
"prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- "udot z27.s, z18.b, z3.b[2]\n"
+ "udot z29.s, z17.b, z5.b[1]\n"
"prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
- "udot z28.s, z18.b, z4.b[2]\n"
+ "udot z30.s, z17.b, z6.b[1]\n"
"prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
+ "udot z31.s, z17.b, z7.b[1]\n"
+ "udot z24.s, z18.b, z0.b[2]\n"
+ "udot z25.s, z18.b, z1.b[2]\n"
+ "udot z26.s, z18.b, z2.b[2]\n"
+ "udot z27.s, z18.b, z3.b[2]\n"
+ "udot z28.s, z18.b, z4.b[2]\n"
"udot z29.s, z18.b, z5.b[2]\n"
"udot z30.s, z18.b, z6.b[2]\n"
"udot z31.s, z18.b, z7.b[2]\n"
- "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"udot z24.s, z19.b, z0.b[3]\n"
"ld1rqb z0.b, p6/z, [%[a_ptr0], #0x10]\n"
"udot z25.s, z19.b, z1.b[3]\n"
@@ -1300,7 +1389,6 @@
"udot z31.s, z19.b, z7.b[3]\n"
"ld1rqb z7.b, p6/z, [a_ptr7, #0x10]\n"
"udot z24.s, z20.b, z0.b[0]\n"
- "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"udot z25.s, z20.b, z1.b[0]\n"
"udot z26.s, z20.b, z2.b[0]\n"
"udot z27.s, z20.b, z3.b[0]\n"
@@ -1313,39 +1401,41 @@
"st1w z24.s, p7, [%[c_ptr0]]\n"
"addvl %[c_ptr0], %[c_ptr0], #1\n"
"mov z24.s, #0\n"
- "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "addvl %[b_ptr0], %[b_ptr0], #5\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"st1w z25.s, p7, [c_ptr1]\n"
"addvl c_ptr1, c_ptr1, #1\n"
"mov z25.s, #0\n"
+ "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "st1w z26.s, p7, [c_ptr2]\n"
+ "addvl c_ptr2, c_ptr2, #1\n"
+ "mov z26.s, #0\n"
+ "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #5\n"
+ "st1w z27.s, p7, [c_ptr3]\n"
+ "addvl c_ptr3, c_ptr3, #1\n"
+ "mov z27.s, #0\n"
"ld1rqb z1.b, p7/z, [a_ptr1]\n"
"udot z24.s, z16.b, z0.b[0]\n"
- "st1w z26.s, p7, [c_ptr2]\n"
- "mov z26.s, #0\n"
+ "st1w z28.s, p7, [c_ptr4]\n"
+ "mov z28.s, #0\n"
"ld1rqb z2.b, p7/z, [a_ptr2]\n"
"ld1rqb z3.b, p7/z, [a_ptr3]\n"
- "addvl c_ptr2, c_ptr2, #1\n"
+ "addvl c_ptr4, c_ptr4, #1\n"
"udot z25.s, z16.b, z1.b[0]\n"
- "st1w z27.s, p7, [c_ptr3]\n"
- "mov z27.s, #0\n"
+ "st1w z29.s, p7, [c_ptr5]\n"
+ "mov z29.s, #0\n"
"ld1rqb z4.b, p7/z, [a_ptr4]\n"
"udot z26.s, z16.b, z2.b[0]\n"
"ld1rqb z5.b, p7/z, [a_ptr5]\n"
- "udot z24.s, z17.b, z0.b[1]\n"
- "st1w z28.s, p7, [c_ptr4]\n"
- "mov z28.s, #0\n"
- "ld1rqb z6.b, p7/z, [a_ptr6]\n"
"udot z27.s, z16.b, z3.b[0]\n"
- "ld1rqb z7.b, p7/z, [a_ptr7]\n"
- "udot z25.s, z17.b, z1.b[1]\n"
- "st1w z29.s, p7, [c_ptr5]\n"
- "mov z29.s, #0\n"
- "addvl c_ptr3, c_ptr3, #1\n"
- "udot z28.s, z16.b, z4.b[0]\n"
"st1w z30.s, p7, [c_ptr6]\n"
"mov z30.s, #0\n"
- "addvl c_ptr4, c_ptr4, #1\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "udot z28.s, z16.b, z4.b[0]\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
"udot z29.s, z16.b, z5.b[0]\n"
"st1w z31.s, p7, [c_ptr7]\n"
"mov z31.s, #0\n"
@@ -1354,6 +1444,8 @@
"addvl c_ptr6, c_ptr6, #1\n"
"udot z31.s, z16.b, z7.b[0]\n"
"addvl c_ptr7, c_ptr7, #1\n"
+ "udot z24.s, z17.b, z0.b[1]\n"
+ "udot z25.s, z17.b, z1.b[1]\n"
"udot z26.s, z17.b, z2.b[1]\n"
"udot z27.s, z17.b, z3.b[1]\n"
"udot z28.s, z17.b, z4.b[1]\n"
@@ -1392,23 +1484,82 @@
"udot z29.s, z20.b, z5.b[0]\n"
"udot z30.s, z20.b, z6.b[0]\n"
"udot z31.s, z20.b, z7.b[0]\n"
+ "b 5f\n"
"2:\n"
+ "mov z24.s, #0\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "mov z25.s, #0\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "mov z26.s, #0\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2]\n"
+ "mov z27.s, #0\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "mov z28.s, #0\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "mov z29.s, #0\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "mov z30.s, #0\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "mov z31.s, #0\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "udot z24.s, z16.b, z0.b[0]\n"
+ "udot z25.s, z16.b, z1.b[0]\n"
+ "udot z26.s, z16.b, z2.b[0]\n"
+ "udot z27.s, z16.b, z3.b[0]\n"
+ "udot z28.s, z16.b, z4.b[0]\n"
+ "udot z29.s, z16.b, z5.b[0]\n"
+ "udot z30.s, z16.b, z6.b[0]\n"
+ "udot z31.s, z16.b, z7.b[0]\n"
+ "udot z24.s, z17.b, z0.b[1]\n"
+ "udot z25.s, z17.b, z1.b[1]\n"
+ "udot z26.s, z17.b, z2.b[1]\n"
+ "udot z27.s, z17.b, z3.b[1]\n"
+ "udot z28.s, z17.b, z4.b[1]\n"
+ "udot z29.s, z17.b, z5.b[1]\n"
+ "udot z30.s, z17.b, z6.b[1]\n"
+ "udot z31.s, z17.b, z7.b[1]\n"
+ "udot z24.s, z18.b, z0.b[2]\n"
+ "udot z25.s, z18.b, z1.b[2]\n"
+ "udot z26.s, z18.b, z2.b[2]\n"
+ "udot z27.s, z18.b, z3.b[2]\n"
+ "udot z28.s, z18.b, z4.b[2]\n"
+ "udot z29.s, z18.b, z5.b[2]\n"
+ "udot z30.s, z18.b, z6.b[2]\n"
+ "udot z31.s, z18.b, z7.b[2]\n"
+ "udot z24.s, z19.b, z0.b[3]\n"
+ "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x10]\n"
+ "udot z25.s, z19.b, z1.b[3]\n"
+ "ld1rqb z1.b, p6/z, [a_ptr1, #0x10]\n"
+ "udot z26.s, z19.b, z2.b[3]\n"
+ "ld1rqb z2.b, p6/z, [a_ptr2, #0x10]\n"
+ "udot z27.s, z19.b, z3.b[3]\n"
+ "ld1rqb z3.b, p6/z, [a_ptr3, #0x10]\n"
+ "udot z28.s, z19.b, z4.b[3]\n"
+ "ld1rqb z4.b, p6/z, [a_ptr4, #0x10]\n"
+ "udot z29.s, z19.b, z5.b[3]\n"
+ "ld1rqb z5.b, p6/z, [a_ptr5, #0x10]\n"
+ "udot z30.s, z19.b, z6.b[3]\n"
+ "ld1rqb z6.b, p6/z, [a_ptr6, #0x10]\n"
+ "udot z31.s, z19.b, z7.b[3]\n"
+ "ld1rqb z7.b, p6/z, [a_ptr7, #0x10]\n"
+ "udot z24.s, z20.b, z0.b[0]\n"
+ "udot z25.s, z20.b, z1.b[0]\n"
+ "udot z26.s, z20.b, z2.b[0]\n"
+ "udot z27.s, z20.b, z3.b[0]\n"
+ "udot z28.s, z20.b, z4.b[0]\n"
+ "udot z29.s, z20.b, z5.b[0]\n"
+ "udot z30.s, z20.b, z6.b[0]\n"
+ "udot z31.s, z20.b, z7.b[0]\n"
+ "5:\n"
"st1w z24.s, p0, [%[c_ptr0]]\n"
"addvl %[c_ptr0], %[c_ptr0], #1\n"
"st1w z25.s, p0, [c_ptr1]\n"
- "addvl c_ptr1, c_ptr1, #1\n"
"st1w z26.s, p0, [c_ptr2]\n"
- "addvl c_ptr2, c_ptr2, #1\n"
"st1w z27.s, p0, [c_ptr3]\n"
- "addvl c_ptr3, c_ptr3, #1\n"
"st1w z28.s, p0, [c_ptr4]\n"
- "addvl c_ptr4, c_ptr4, #1\n"
"st1w z29.s, p0, [c_ptr5]\n"
- "addvl c_ptr5, c_ptr5, #1\n"
"st1w z30.s, p0, [c_ptr6]\n"
- "addvl c_ptr6, c_ptr6, #1\n"
"st1w z31.s, p0, [c_ptr7]\n"
- "addvl c_ptr7, c_ptr7, #1\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -1487,48 +1638,50 @@
"add c_ptr1, %[c_ptr0], #0x0\n"
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
- "mov z24.s, #0\n"
"ptrue p7.b\n"
- "mov z25.s, #0\n"
"whilelt p6.b, %[temp], %[odd_depth]\n"
- "mov z26.s, #0\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
- "mov z27.s, #0\n"
- "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "mov z28.s, #0\n"
- "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "mov z29.s, #0\n"
- "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "mov z30.s, #0\n"
- "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "mov z31.s, #0\n"
- "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
"whilelt p0.s, %[temp], %[last_width]\n"
- "ld1rqb z1.b, p7/z, [a_ptr1]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
- "udot z24.s, z16.b, z0.b[0]\n"
- "ld1rqb z2.b, p7/z, [a_ptr2]\n"
- "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
- "udot z25.s, z16.b, z1.b[0]\n"
- "ld1rqb z4.b, p7/z, [a_ptr4]\n"
- "udot z26.s, z16.b, z2.b[0]\n"
- "ld1rqb z5.b, p7/z, [a_ptr5]\n"
- "udot z27.s, z16.b, z3.b[0]\n"
- "ld1rqb z6.b, p7/z, [a_ptr6]\n"
- "udot z24.s, z17.b, z0.b[1]\n"
- "ld1rqb z7.b, p7/z, [a_ptr7]\n"
- "udot z28.s, z16.b, z4.b[0]\n"
+ "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
- "udot z29.s, z16.b, z5.b[0]\n"
+ "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
- "udot z30.s, z16.b, z6.b[0]\n"
+ "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- "udot z31.s, z16.b, z7.b[0]\n"
+ "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
"prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- "udot z25.s, z17.b, z1.b[1]\n"
+ "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
"addvl %[b_ptr0], %[b_ptr0], #6\n"
+ "cbz %[loops], 2f\n"
+ "mov z24.s, #0\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "mov z25.s, #0\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "mov z26.s, #0\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2]\n"
+ "mov z27.s, #0\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "mov z28.s, #0\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "mov z29.s, #0\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "mov z30.s, #0\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "mov z31.s, #0\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "udot z24.s, z16.b, z0.b[0]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "udot z25.s, z16.b, z1.b[0]\n"
+ "udot z26.s, z16.b, z2.b[0]\n"
+ "udot z27.s, z16.b, z3.b[0]\n"
+ "udot z28.s, z16.b, z4.b[0]\n"
+ "udot z29.s, z16.b, z5.b[0]\n"
+ "udot z30.s, z16.b, z6.b[0]\n"
+ "udot z31.s, z16.b, z7.b[0]\n"
+ "udot z24.s, z17.b, z0.b[1]\n"
+ "udot z25.s, z17.b, z1.b[1]\n"
"udot z26.s, z17.b, z2.b[1]\n"
"udot z27.s, z17.b, z3.b[1]\n"
"udot z28.s, z17.b, z4.b[1]\n"
@@ -1575,85 +1728,80 @@
"udot z29.s, z21.b, z5.b[1]\n"
"udot z30.s, z21.b, z6.b[1]\n"
"udot z31.s, z21.b, z7.b[1]\n"
- "cbz %[loops], 2f\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
"b.eq 3f\n"
"4:\n"
"st1w z24.s, p7, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"mov z24.s, #0\n"
- "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "addvl %[b_ptr0], %[b_ptr0], #6\n"
- "st1w z25.s, p7, [c_ptr1]\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"addvl %[c_ptr0], %[c_ptr0], #1\n"
- "mov z25.s, #0\n"
- "ld1rqb z1.b, p7/z, [a_ptr1]\n"
- "udot z24.s, z16.b, z0.b[0]\n"
- "st1w z26.s, p7, [c_ptr2]\n"
- "mov z26.s, #0\n"
- "ld1rqb z2.b, p7/z, [a_ptr2]\n"
- "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "st1w z25.s, p7, [c_ptr1]\n"
"addvl c_ptr1, c_ptr1, #1\n"
- "udot z25.s, z16.b, z1.b[0]\n"
+ "mov z25.s, #0\n"
+ "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "st1w z26.s, p7, [c_ptr2]\n"
+ "addvl c_ptr2, c_ptr2, #1\n"
+ "mov z26.s, #0\n"
+ "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #6\n"
"st1w z27.s, p7, [c_ptr3]\n"
+ "addvl c_ptr3, c_ptr3, #1\n"
"mov z27.s, #0\n"
- "ld1rqb z4.b, p7/z, [a_ptr4]\n"
- "udot z26.s, z16.b, z2.b[0]\n"
- "ld1rqb z5.b, p7/z, [a_ptr5]\n"
- "udot z24.s, z17.b, z0.b[1]\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
"st1w z28.s, p7, [c_ptr4]\n"
+ "addvl c_ptr4, c_ptr4, #1\n"
"mov z28.s, #0\n"
- "ld1rqb z6.b, p7/z, [a_ptr6]\n"
- "udot z27.s, z16.b, z3.b[0]\n"
- "ld1rqb z7.b, p7/z, [a_ptr7]\n"
- "udot z25.s, z17.b, z1.b[1]\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2]\n"
+ "udot z24.s, z16.b, z0.b[0]\n"
"st1w z29.s, p7, [c_ptr5]\n"
"mov z29.s, #0\n"
- "addvl c_ptr2, c_ptr2, #1\n"
- "udot z28.s, z16.b, z4.b[0]\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "udot z25.s, z16.b, z1.b[0]\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "udot z26.s, z16.b, z2.b[0]\n"
"st1w z30.s, p7, [c_ptr6]\n"
"mov z30.s, #0\n"
- "addvl c_ptr3, c_ptr3, #1\n"
- "udot z29.s, z16.b, z5.b[0]\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "udot z27.s, z16.b, z3.b[0]\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "udot z28.s, z16.b, z4.b[0]\n"
"st1w z31.s, p7, [c_ptr7]\n"
"mov z31.s, #0\n"
- "addvl c_ptr4, c_ptr4, #1\n"
- "udot z30.s, z16.b, z6.b[0]\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "udot z29.s, z16.b, z5.b[0]\n"
"addvl c_ptr5, c_ptr5, #1\n"
- "udot z31.s, z16.b, z7.b[0]\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
- "udot z26.s, z17.b, z2.b[1]\n"
+ "udot z30.s, z16.b, z6.b[0]\n"
"addvl c_ptr6, c_ptr6, #1\n"
- "udot z27.s, z17.b, z3.b[1]\n"
+ "udot z31.s, z16.b, z7.b[0]\n"
"addvl c_ptr7, c_ptr7, #1\n"
- "udot z28.s, z17.b, z4.b[1]\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
- "udot z29.s, z17.b, z5.b[1]\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- "udot z30.s, z17.b, z6.b[1]\n"
+ "udot z24.s, z17.b, z0.b[1]\n"
"prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- "udot z31.s, z17.b, z7.b[1]\n"
- "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "udot z24.s, z18.b, z0.b[2]\n"
+ "udot z25.s, z17.b, z1.b[1]\n"
"prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- "udot z25.s, z18.b, z1.b[2]\n"
+ "udot z26.s, z17.b, z2.b[1]\n"
"prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- "udot z26.s, z18.b, z2.b[2]\n"
+ "udot z27.s, z17.b, z3.b[1]\n"
"prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- "udot z27.s, z18.b, z3.b[2]\n"
+ "udot z28.s, z17.b, z4.b[1]\n"
"prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
- "udot z28.s, z18.b, z4.b[2]\n"
+ "udot z29.s, z17.b, z5.b[1]\n"
"prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
+ "udot z30.s, z17.b, z6.b[1]\n"
+ "udot z31.s, z17.b, z7.b[1]\n"
+ "udot z24.s, z18.b, z0.b[2]\n"
+ "udot z25.s, z18.b, z1.b[2]\n"
+ "udot z26.s, z18.b, z2.b[2]\n"
+ "udot z27.s, z18.b, z3.b[2]\n"
+ "udot z28.s, z18.b, z4.b[2]\n"
"udot z29.s, z18.b, z5.b[2]\n"
"udot z30.s, z18.b, z6.b[2]\n"
"udot z31.s, z18.b, z7.b[2]\n"
- "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"udot z24.s, z19.b, z0.b[3]\n"
"ld1rqb z0.b, p6/z, [%[a_ptr0], #0x10]\n"
"udot z25.s, z19.b, z1.b[3]\n"
@@ -1671,7 +1819,6 @@
"udot z31.s, z19.b, z7.b[3]\n"
"ld1rqb z7.b, p6/z, [a_ptr7, #0x10]\n"
"udot z24.s, z20.b, z0.b[0]\n"
- "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"udot z25.s, z20.b, z1.b[0]\n"
"udot z26.s, z20.b, z2.b[0]\n"
"udot z27.s, z20.b, z3.b[0]\n"
@@ -1679,7 +1826,6 @@
"udot z29.s, z20.b, z5.b[0]\n"
"udot z30.s, z20.b, z6.b[0]\n"
"udot z31.s, z20.b, z7.b[0]\n"
- "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
"udot z24.s, z21.b, z0.b[1]\n"
"udot z25.s, z21.b, z1.b[1]\n"
"udot z26.s, z21.b, z2.b[1]\n"
@@ -1693,47 +1839,52 @@
"st1w z24.s, p7, [%[c_ptr0]]\n"
"addvl %[c_ptr0], %[c_ptr0], #1\n"
"mov z24.s, #0\n"
- "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "addvl %[b_ptr0], %[b_ptr0], #6\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"st1w z25.s, p7, [c_ptr1]\n"
"addvl c_ptr1, c_ptr1, #1\n"
"mov z25.s, #0\n"
- "ld1rqb z1.b, p7/z, [a_ptr1]\n"
- "udot z24.s, z16.b, z0.b[0]\n"
+ "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"st1w z26.s, p7, [c_ptr2]\n"
- "mov z26.s, #0\n"
- "ld1rqb z2.b, p7/z, [a_ptr2]\n"
- "ld1rqb z3.b, p7/z, [a_ptr3]\n"
"addvl c_ptr2, c_ptr2, #1\n"
- "udot z25.s, z16.b, z1.b[0]\n"
+ "mov z26.s, #0\n"
+ "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #6\n"
"st1w z27.s, p7, [c_ptr3]\n"
+ "addvl c_ptr3, c_ptr3, #1\n"
"mov z27.s, #0\n"
- "ld1rqb z4.b, p7/z, [a_ptr4]\n"
- "udot z26.s, z16.b, z2.b[0]\n"
- "ld1rqb z5.b, p7/z, [a_ptr5]\n"
- "udot z24.s, z17.b, z0.b[1]\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
"st1w z28.s, p7, [c_ptr4]\n"
+ "addvl c_ptr4, c_ptr4, #1\n"
"mov z28.s, #0\n"
- "ld1rqb z6.b, p7/z, [a_ptr6]\n"
- "udot z27.s, z16.b, z3.b[0]\n"
- "ld1rqb z7.b, p7/z, [a_ptr7]\n"
- "udot z25.s, z17.b, z1.b[1]\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2]\n"
+ "udot z24.s, z16.b, z0.b[0]\n"
"st1w z29.s, p7, [c_ptr5]\n"
"mov z29.s, #0\n"
- "addvl c_ptr3, c_ptr3, #1\n"
- "udot z28.s, z16.b, z4.b[0]\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "udot z25.s, z16.b, z1.b[0]\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "udot z26.s, z16.b, z2.b[0]\n"
"st1w z30.s, p7, [c_ptr6]\n"
"mov z30.s, #0\n"
- "addvl c_ptr4, c_ptr4, #1\n"
- "udot z29.s, z16.b, z5.b[0]\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "udot z27.s, z16.b, z3.b[0]\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "udot z28.s, z16.b, z4.b[0]\n"
"st1w z31.s, p7, [c_ptr7]\n"
"mov z31.s, #0\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "udot z29.s, z16.b, z5.b[0]\n"
"addvl c_ptr5, c_ptr5, #1\n"
"udot z30.s, z16.b, z6.b[0]\n"
"addvl c_ptr6, c_ptr6, #1\n"
"udot z31.s, z16.b, z7.b[0]\n"
"addvl c_ptr7, c_ptr7, #1\n"
+ "udot z24.s, z17.b, z0.b[1]\n"
+ "udot z25.s, z17.b, z1.b[1]\n"
"udot z26.s, z17.b, z2.b[1]\n"
"udot z27.s, z17.b, z3.b[1]\n"
"udot z28.s, z17.b, z4.b[1]\n"
@@ -1780,23 +1931,90 @@
"udot z29.s, z21.b, z5.b[1]\n"
"udot z30.s, z21.b, z6.b[1]\n"
"udot z31.s, z21.b, z7.b[1]\n"
+ "b 5f\n"
"2:\n"
+ "mov z24.s, #0\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "mov z25.s, #0\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "mov z26.s, #0\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2]\n"
+ "mov z27.s, #0\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "mov z28.s, #0\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "mov z29.s, #0\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "mov z30.s, #0\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "mov z31.s, #0\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "udot z24.s, z16.b, z0.b[0]\n"
+ "udot z25.s, z16.b, z1.b[0]\n"
+ "udot z26.s, z16.b, z2.b[0]\n"
+ "udot z27.s, z16.b, z3.b[0]\n"
+ "udot z28.s, z16.b, z4.b[0]\n"
+ "udot z29.s, z16.b, z5.b[0]\n"
+ "udot z30.s, z16.b, z6.b[0]\n"
+ "udot z31.s, z16.b, z7.b[0]\n"
+ "udot z24.s, z17.b, z0.b[1]\n"
+ "udot z25.s, z17.b, z1.b[1]\n"
+ "udot z26.s, z17.b, z2.b[1]\n"
+ "udot z27.s, z17.b, z3.b[1]\n"
+ "udot z28.s, z17.b, z4.b[1]\n"
+ "udot z29.s, z17.b, z5.b[1]\n"
+ "udot z30.s, z17.b, z6.b[1]\n"
+ "udot z31.s, z17.b, z7.b[1]\n"
+ "udot z24.s, z18.b, z0.b[2]\n"
+ "udot z25.s, z18.b, z1.b[2]\n"
+ "udot z26.s, z18.b, z2.b[2]\n"
+ "udot z27.s, z18.b, z3.b[2]\n"
+ "udot z28.s, z18.b, z4.b[2]\n"
+ "udot z29.s, z18.b, z5.b[2]\n"
+ "udot z30.s, z18.b, z6.b[2]\n"
+ "udot z31.s, z18.b, z7.b[2]\n"
+ "udot z24.s, z19.b, z0.b[3]\n"
+ "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x10]\n"
+ "udot z25.s, z19.b, z1.b[3]\n"
+ "ld1rqb z1.b, p6/z, [a_ptr1, #0x10]\n"
+ "udot z26.s, z19.b, z2.b[3]\n"
+ "ld1rqb z2.b, p6/z, [a_ptr2, #0x10]\n"
+ "udot z27.s, z19.b, z3.b[3]\n"
+ "ld1rqb z3.b, p6/z, [a_ptr3, #0x10]\n"
+ "udot z28.s, z19.b, z4.b[3]\n"
+ "ld1rqb z4.b, p6/z, [a_ptr4, #0x10]\n"
+ "udot z29.s, z19.b, z5.b[3]\n"
+ "ld1rqb z5.b, p6/z, [a_ptr5, #0x10]\n"
+ "udot z30.s, z19.b, z6.b[3]\n"
+ "ld1rqb z6.b, p6/z, [a_ptr6, #0x10]\n"
+ "udot z31.s, z19.b, z7.b[3]\n"
+ "ld1rqb z7.b, p6/z, [a_ptr7, #0x10]\n"
+ "udot z24.s, z20.b, z0.b[0]\n"
+ "udot z25.s, z20.b, z1.b[0]\n"
+ "udot z26.s, z20.b, z2.b[0]\n"
+ "udot z27.s, z20.b, z3.b[0]\n"
+ "udot z28.s, z20.b, z4.b[0]\n"
+ "udot z29.s, z20.b, z5.b[0]\n"
+ "udot z30.s, z20.b, z6.b[0]\n"
+ "udot z31.s, z20.b, z7.b[0]\n"
+ "udot z24.s, z21.b, z0.b[1]\n"
+ "udot z25.s, z21.b, z1.b[1]\n"
+ "udot z26.s, z21.b, z2.b[1]\n"
+ "udot z27.s, z21.b, z3.b[1]\n"
+ "udot z28.s, z21.b, z4.b[1]\n"
+ "udot z29.s, z21.b, z5.b[1]\n"
+ "udot z30.s, z21.b, z6.b[1]\n"
+ "udot z31.s, z21.b, z7.b[1]\n"
+ "5:\n"
"st1w z24.s, p0, [%[c_ptr0]]\n"
"addvl %[c_ptr0], %[c_ptr0], #1\n"
"st1w z25.s, p0, [c_ptr1]\n"
- "addvl c_ptr1, c_ptr1, #1\n"
"st1w z26.s, p0, [c_ptr2]\n"
- "addvl c_ptr2, c_ptr2, #1\n"
"st1w z27.s, p0, [c_ptr3]\n"
- "addvl c_ptr3, c_ptr3, #1\n"
"st1w z28.s, p0, [c_ptr4]\n"
- "addvl c_ptr4, c_ptr4, #1\n"
"st1w z29.s, p0, [c_ptr5]\n"
- "addvl c_ptr5, c_ptr5, #1\n"
"st1w z30.s, p0, [c_ptr6]\n"
- "addvl c_ptr6, c_ptr6, #1\n"
"st1w z31.s, p0, [c_ptr7]\n"
- "addvl c_ptr7, c_ptr7, #1\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -1875,48 +2093,50 @@
"add c_ptr1, %[c_ptr0], #0x0\n"
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
- "mov z24.s, #0\n"
"ptrue p7.b\n"
- "mov z25.s, #0\n"
"whilelt p6.b, %[temp], %[odd_depth]\n"
- "mov z26.s, #0\n"
+ "whilelt p0.s, %[temp], %[last_width]\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
"ld1b z16.b, p7/z, [%[b_ptr0]]\n"
- "mov z27.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
"ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "mov z28.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
"ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "mov z29.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
"ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "mov z30.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
"ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "mov z31.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
"ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
"ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "whilelt p0.s, %[temp], %[last_width]\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
- "ld1rqb z1.b, p7/z, [a_ptr1]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
- "udot z24.s, z16.b, z0.b[0]\n"
- "ld1rqb z2.b, p7/z, [a_ptr2]\n"
- "ld1rqb z3.b, p7/z, [a_ptr3]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
- "udot z25.s, z16.b, z1.b[0]\n"
- "ld1rqb z4.b, p7/z, [a_ptr4]\n"
- "udot z26.s, z16.b, z2.b[0]\n"
- "ld1rqb z5.b, p7/z, [a_ptr5]\n"
- "udot z27.s, z16.b, z3.b[0]\n"
- "ld1rqb z6.b, p7/z, [a_ptr6]\n"
- "udot z24.s, z17.b, z0.b[1]\n"
- "ld1rqb z7.b, p7/z, [a_ptr7]\n"
- "udot z28.s, z16.b, z4.b[0]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
- "udot z29.s, z16.b, z5.b[0]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- "udot z30.s, z16.b, z6.b[0]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- "udot z31.s, z16.b, z7.b[0]\n"
"addvl %[b_ptr0], %[b_ptr0], #7\n"
+ "cbz %[loops], 2f\n"
+ "mov z24.s, #0\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "mov z25.s, #0\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "mov z26.s, #0\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2]\n"
+ "mov z27.s, #0\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "mov z28.s, #0\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "mov z29.s, #0\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "mov z30.s, #0\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "mov z31.s, #0\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "udot z24.s, z16.b, z0.b[0]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "udot z25.s, z16.b, z1.b[0]\n"
+ "udot z26.s, z16.b, z2.b[0]\n"
+ "udot z27.s, z16.b, z3.b[0]\n"
+ "udot z28.s, z16.b, z4.b[0]\n"
+ "udot z29.s, z16.b, z5.b[0]\n"
+ "udot z30.s, z16.b, z6.b[0]\n"
+ "udot z31.s, z16.b, z7.b[0]\n"
+ "udot z24.s, z17.b, z0.b[1]\n"
"udot z25.s, z17.b, z1.b[1]\n"
"udot z26.s, z17.b, z2.b[1]\n"
"udot z27.s, z17.b, z3.b[1]\n"
@@ -1972,86 +2192,81 @@
"udot z29.s, z22.b, z5.b[2]\n"
"udot z30.s, z22.b, z6.b[2]\n"
"udot z31.s, z22.b, z7.b[2]\n"
- "cbz %[loops], 2f\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
"b.eq 3f\n"
"4:\n"
"st1w z24.s, p7, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"mov z24.s, #0\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "addvl %[c_ptr0], %[c_ptr0], #1\n"
+ "st1w z25.s, p7, [c_ptr1]\n"
+ "addvl c_ptr1, c_ptr1, #1\n"
+ "mov z25.s, #0\n"
+ "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "st1w z26.s, p7, [c_ptr2]\n"
+ "addvl c_ptr2, c_ptr2, #1\n"
+ "mov z26.s, #0\n"
+ "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ "st1w z27.s, p7, [c_ptr3]\n"
+ "addvl c_ptr3, c_ptr3, #1\n"
+ "mov z27.s, #0\n"
"ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
"ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
"addvl %[b_ptr0], %[b_ptr0], #7\n"
- "st1w z25.s, p7, [c_ptr1]\n"
- "addvl %[c_ptr0], %[c_ptr0], #1\n"
- "mov z25.s, #0\n"
+ "st1w z28.s, p7, [c_ptr4]\n"
+ "addvl c_ptr4, c_ptr4, #1\n"
+ "mov z28.s, #0\n"
"ld1rqb z1.b, p7/z, [a_ptr1]\n"
"udot z24.s, z16.b, z0.b[0]\n"
- "st1w z26.s, p7, [c_ptr2]\n"
- "mov z26.s, #0\n"
+ "st1w z29.s, p7, [c_ptr5]\n"
+ "mov z29.s, #0\n"
"ld1rqb z2.b, p7/z, [a_ptr2]\n"
"ld1rqb z3.b, p7/z, [a_ptr3]\n"
- "addvl c_ptr1, c_ptr1, #1\n"
+ "addvl c_ptr5, c_ptr5, #1\n"
"udot z25.s, z16.b, z1.b[0]\n"
- "st1w z27.s, p7, [c_ptr3]\n"
- "mov z27.s, #0\n"
+ "st1w z30.s, p7, [c_ptr6]\n"
+ "mov z30.s, #0\n"
"ld1rqb z4.b, p7/z, [a_ptr4]\n"
"udot z26.s, z16.b, z2.b[0]\n"
"ld1rqb z5.b, p7/z, [a_ptr5]\n"
- "udot z24.s, z17.b, z0.b[1]\n"
- "st1w z28.s, p7, [c_ptr4]\n"
- "mov z28.s, #0\n"
- "ld1rqb z6.b, p7/z, [a_ptr6]\n"
"udot z27.s, z16.b, z3.b[0]\n"
- "ld1rqb z7.b, p7/z, [a_ptr7]\n"
- "udot z25.s, z17.b, z1.b[1]\n"
- "st1w z29.s, p7, [c_ptr5]\n"
- "mov z29.s, #0\n"
- "addvl c_ptr2, c_ptr2, #1\n"
- "udot z28.s, z16.b, z4.b[0]\n"
- "st1w z30.s, p7, [c_ptr6]\n"
- "mov z30.s, #0\n"
- "addvl c_ptr3, c_ptr3, #1\n"
- "udot z29.s, z16.b, z5.b[0]\n"
"st1w z31.s, p7, [c_ptr7]\n"
"mov z31.s, #0\n"
- "addvl c_ptr4, c_ptr4, #1\n"
- "udot z30.s, z16.b, z6.b[0]\n"
- "addvl c_ptr5, c_ptr5, #1\n"
- "udot z31.s, z16.b, z7.b[0]\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
- "udot z26.s, z17.b, z2.b[1]\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "udot z28.s, z16.b, z4.b[0]\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "udot z29.s, z16.b, z5.b[0]\n"
"addvl c_ptr6, c_ptr6, #1\n"
- "udot z27.s, z17.b, z3.b[1]\n"
+ "udot z30.s, z16.b, z6.b[0]\n"
"addvl c_ptr7, c_ptr7, #1\n"
- "udot z28.s, z17.b, z4.b[1]\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
- "udot z29.s, z17.b, z5.b[1]\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- "udot z30.s, z17.b, z6.b[1]\n"
+ "udot z31.s, z16.b, z7.b[0]\n"
"prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- "udot z31.s, z17.b, z7.b[1]\n"
- "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "udot z24.s, z18.b, z0.b[2]\n"
+ "udot z24.s, z17.b, z0.b[1]\n"
"prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- "udot z25.s, z18.b, z1.b[2]\n"
+ "udot z25.s, z17.b, z1.b[1]\n"
"prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- "udot z26.s, z18.b, z2.b[2]\n"
+ "udot z26.s, z17.b, z2.b[1]\n"
"prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- "udot z27.s, z18.b, z3.b[2]\n"
+ "udot z27.s, z17.b, z3.b[1]\n"
"prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
- "udot z28.s, z18.b, z4.b[2]\n"
+ "udot z28.s, z17.b, z4.b[1]\n"
"prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
+ "udot z29.s, z17.b, z5.b[1]\n"
+ "udot z30.s, z17.b, z6.b[1]\n"
+ "udot z31.s, z17.b, z7.b[1]\n"
+ "udot z24.s, z18.b, z0.b[2]\n"
+ "udot z25.s, z18.b, z1.b[2]\n"
+ "udot z26.s, z18.b, z2.b[2]\n"
+ "udot z27.s, z18.b, z3.b[2]\n"
+ "udot z28.s, z18.b, z4.b[2]\n"
"udot z29.s, z18.b, z5.b[2]\n"
"udot z30.s, z18.b, z6.b[2]\n"
"udot z31.s, z18.b, z7.b[2]\n"
- "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"udot z24.s, z19.b, z0.b[3]\n"
"ld1rqb z0.b, p6/z, [%[a_ptr0], #0x10]\n"
"udot z25.s, z19.b, z1.b[3]\n"
@@ -2069,7 +2284,6 @@
"udot z31.s, z19.b, z7.b[3]\n"
"ld1rqb z7.b, p6/z, [a_ptr7, #0x10]\n"
"udot z24.s, z20.b, z0.b[0]\n"
- "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"udot z25.s, z20.b, z1.b[0]\n"
"udot z26.s, z20.b, z2.b[0]\n"
"udot z27.s, z20.b, z3.b[0]\n"
@@ -2077,7 +2291,6 @@
"udot z29.s, z20.b, z5.b[0]\n"
"udot z30.s, z20.b, z6.b[0]\n"
"udot z31.s, z20.b, z7.b[0]\n"
- "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
"udot z24.s, z21.b, z0.b[1]\n"
"udot z25.s, z21.b, z1.b[1]\n"
"udot z26.s, z21.b, z2.b[1]\n"
@@ -2086,7 +2299,6 @@
"udot z29.s, z21.b, z5.b[1]\n"
"udot z30.s, z21.b, z6.b[1]\n"
"udot z31.s, z21.b, z7.b[1]\n"
- "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
"udot z24.s, z22.b, z0.b[2]\n"
"udot z25.s, z22.b, z1.b[2]\n"
"udot z26.s, z22.b, z2.b[2]\n"
@@ -2100,47 +2312,53 @@
"st1w z24.s, p7, [%[c_ptr0]]\n"
"addvl %[c_ptr0], %[c_ptr0], #1\n"
"mov z24.s, #0\n"
- "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "addvl %[b_ptr0], %[b_ptr0], #7\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"st1w z25.s, p7, [c_ptr1]\n"
"addvl c_ptr1, c_ptr1, #1\n"
"mov z25.s, #0\n"
+ "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "st1w z26.s, p7, [c_ptr2]\n"
+ "addvl c_ptr2, c_ptr2, #1\n"
+ "mov z26.s, #0\n"
+ "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "st1w z27.s, p7, [c_ptr3]\n"
+ "addvl c_ptr3, c_ptr3, #1\n"
+ "mov z27.s, #0\n"
+ "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #7\n"
+ "st1w z28.s, p7, [c_ptr4]\n"
+ "addvl c_ptr4, c_ptr4, #1\n"
+ "mov z28.s, #0\n"
"ld1rqb z1.b, p7/z, [a_ptr1]\n"
"udot z24.s, z16.b, z0.b[0]\n"
- "st1w z26.s, p7, [c_ptr2]\n"
- "mov z26.s, #0\n"
+ "st1w z29.s, p7, [c_ptr5]\n"
+ "mov z29.s, #0\n"
"ld1rqb z2.b, p7/z, [a_ptr2]\n"
"ld1rqb z3.b, p7/z, [a_ptr3]\n"
- "addvl c_ptr2, c_ptr2, #1\n"
+ "addvl c_ptr5, c_ptr5, #1\n"
"udot z25.s, z16.b, z1.b[0]\n"
- "st1w z27.s, p7, [c_ptr3]\n"
- "mov z27.s, #0\n"
+ "st1w z30.s, p7, [c_ptr6]\n"
+ "mov z30.s, #0\n"
"ld1rqb z4.b, p7/z, [a_ptr4]\n"
"udot z26.s, z16.b, z2.b[0]\n"
"ld1rqb z5.b, p7/z, [a_ptr5]\n"
- "udot z24.s, z17.b, z0.b[1]\n"
- "st1w z28.s, p7, [c_ptr4]\n"
- "mov z28.s, #0\n"
- "ld1rqb z6.b, p7/z, [a_ptr6]\n"
"udot z27.s, z16.b, z3.b[0]\n"
- "ld1rqb z7.b, p7/z, [a_ptr7]\n"
- "udot z25.s, z17.b, z1.b[1]\n"
- "st1w z29.s, p7, [c_ptr5]\n"
- "mov z29.s, #0\n"
- "addvl c_ptr3, c_ptr3, #1\n"
- "udot z28.s, z16.b, z4.b[0]\n"
- "st1w z30.s, p7, [c_ptr6]\n"
- "mov z30.s, #0\n"
- "addvl c_ptr4, c_ptr4, #1\n"
- "udot z29.s, z16.b, z5.b[0]\n"
"st1w z31.s, p7, [c_ptr7]\n"
"mov z31.s, #0\n"
- "addvl c_ptr5, c_ptr5, #1\n"
- "udot z30.s, z16.b, z6.b[0]\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "udot z28.s, z16.b, z4.b[0]\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "udot z29.s, z16.b, z5.b[0]\n"
"addvl c_ptr6, c_ptr6, #1\n"
- "udot z31.s, z16.b, z7.b[0]\n"
+ "udot z30.s, z16.b, z6.b[0]\n"
"addvl c_ptr7, c_ptr7, #1\n"
+ "udot z31.s, z16.b, z7.b[0]\n"
+ "udot z24.s, z17.b, z0.b[1]\n"
+ "udot z25.s, z17.b, z1.b[1]\n"
"udot z26.s, z17.b, z2.b[1]\n"
"udot z27.s, z17.b, z3.b[1]\n"
"udot z28.s, z17.b, z4.b[1]\n"
@@ -2195,23 +2413,98 @@
"udot z29.s, z22.b, z5.b[2]\n"
"udot z30.s, z22.b, z6.b[2]\n"
"udot z31.s, z22.b, z7.b[2]\n"
+ "b 5f\n"
"2:\n"
+ "mov z24.s, #0\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "mov z25.s, #0\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "mov z26.s, #0\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2]\n"
+ "mov z27.s, #0\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "mov z28.s, #0\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "mov z29.s, #0\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "mov z30.s, #0\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "mov z31.s, #0\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "udot z24.s, z16.b, z0.b[0]\n"
+ "udot z25.s, z16.b, z1.b[0]\n"
+ "udot z26.s, z16.b, z2.b[0]\n"
+ "udot z27.s, z16.b, z3.b[0]\n"
+ "udot z28.s, z16.b, z4.b[0]\n"
+ "udot z29.s, z16.b, z5.b[0]\n"
+ "udot z30.s, z16.b, z6.b[0]\n"
+ "udot z31.s, z16.b, z7.b[0]\n"
+ "udot z24.s, z17.b, z0.b[1]\n"
+ "udot z25.s, z17.b, z1.b[1]\n"
+ "udot z26.s, z17.b, z2.b[1]\n"
+ "udot z27.s, z17.b, z3.b[1]\n"
+ "udot z28.s, z17.b, z4.b[1]\n"
+ "udot z29.s, z17.b, z5.b[1]\n"
+ "udot z30.s, z17.b, z6.b[1]\n"
+ "udot z31.s, z17.b, z7.b[1]\n"
+ "udot z24.s, z18.b, z0.b[2]\n"
+ "udot z25.s, z18.b, z1.b[2]\n"
+ "udot z26.s, z18.b, z2.b[2]\n"
+ "udot z27.s, z18.b, z3.b[2]\n"
+ "udot z28.s, z18.b, z4.b[2]\n"
+ "udot z29.s, z18.b, z5.b[2]\n"
+ "udot z30.s, z18.b, z6.b[2]\n"
+ "udot z31.s, z18.b, z7.b[2]\n"
+ "udot z24.s, z19.b, z0.b[3]\n"
+ "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x10]\n"
+ "udot z25.s, z19.b, z1.b[3]\n"
+ "ld1rqb z1.b, p6/z, [a_ptr1, #0x10]\n"
+ "udot z26.s, z19.b, z2.b[3]\n"
+ "ld1rqb z2.b, p6/z, [a_ptr2, #0x10]\n"
+ "udot z27.s, z19.b, z3.b[3]\n"
+ "ld1rqb z3.b, p6/z, [a_ptr3, #0x10]\n"
+ "udot z28.s, z19.b, z4.b[3]\n"
+ "ld1rqb z4.b, p6/z, [a_ptr4, #0x10]\n"
+ "udot z29.s, z19.b, z5.b[3]\n"
+ "ld1rqb z5.b, p6/z, [a_ptr5, #0x10]\n"
+ "udot z30.s, z19.b, z6.b[3]\n"
+ "ld1rqb z6.b, p6/z, [a_ptr6, #0x10]\n"
+ "udot z31.s, z19.b, z7.b[3]\n"
+ "ld1rqb z7.b, p6/z, [a_ptr7, #0x10]\n"
+ "udot z24.s, z20.b, z0.b[0]\n"
+ "udot z25.s, z20.b, z1.b[0]\n"
+ "udot z26.s, z20.b, z2.b[0]\n"
+ "udot z27.s, z20.b, z3.b[0]\n"
+ "udot z28.s, z20.b, z4.b[0]\n"
+ "udot z29.s, z20.b, z5.b[0]\n"
+ "udot z30.s, z20.b, z6.b[0]\n"
+ "udot z31.s, z20.b, z7.b[0]\n"
+ "udot z24.s, z21.b, z0.b[1]\n"
+ "udot z25.s, z21.b, z1.b[1]\n"
+ "udot z26.s, z21.b, z2.b[1]\n"
+ "udot z27.s, z21.b, z3.b[1]\n"
+ "udot z28.s, z21.b, z4.b[1]\n"
+ "udot z29.s, z21.b, z5.b[1]\n"
+ "udot z30.s, z21.b, z6.b[1]\n"
+ "udot z31.s, z21.b, z7.b[1]\n"
+ "udot z24.s, z22.b, z0.b[2]\n"
+ "udot z25.s, z22.b, z1.b[2]\n"
+ "udot z26.s, z22.b, z2.b[2]\n"
+ "udot z27.s, z22.b, z3.b[2]\n"
+ "udot z28.s, z22.b, z4.b[2]\n"
+ "udot z29.s, z22.b, z5.b[2]\n"
+ "udot z30.s, z22.b, z6.b[2]\n"
+ "udot z31.s, z22.b, z7.b[2]\n"
+ "5:\n"
"st1w z24.s, p0, [%[c_ptr0]]\n"
"addvl %[c_ptr0], %[c_ptr0], #1\n"
"st1w z25.s, p0, [c_ptr1]\n"
- "addvl c_ptr1, c_ptr1, #1\n"
"st1w z26.s, p0, [c_ptr2]\n"
- "addvl c_ptr2, c_ptr2, #1\n"
"st1w z27.s, p0, [c_ptr3]\n"
- "addvl c_ptr3, c_ptr3, #1\n"
"st1w z28.s, p0, [c_ptr4]\n"
- "addvl c_ptr4, c_ptr4, #1\n"
"st1w z29.s, p0, [c_ptr5]\n"
- "addvl c_ptr5, c_ptr5, #1\n"
"st1w z30.s, p0, [c_ptr6]\n"
- "addvl c_ptr6, c_ptr6, #1\n"
"st1w z31.s, p0, [c_ptr7]\n"
- "addvl c_ptr7, c_ptr7, #1\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -2290,49 +2583,51 @@
"add c_ptr1, %[c_ptr0], #0x0\n"
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
- "mov z24.s, #0\n"
"ptrue p7.b\n"
- "mov z25.s, #0\n"
"whilelt p6.b, %[temp], %[odd_depth]\n"
- "mov z26.s, #0\n"
+ "whilelt p0.s, %[temp], %[last_width]\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
"ld1b z16.b, p7/z, [%[b_ptr0]]\n"
- "mov z27.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
"ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "mov z28.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
"ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "mov z29.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
"ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "mov z30.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
"ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "mov z31.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
"ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
"ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "whilelt p0.s, %[temp], %[last_width]\n"
"ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
- "ld1rqb z1.b, p7/z, [a_ptr1]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
- "udot z24.s, z16.b, z0.b[0]\n"
- "ld1rqb z2.b, p7/z, [a_ptr2]\n"
- "ld1rqb z3.b, p7/z, [a_ptr3]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
- "udot z25.s, z16.b, z1.b[0]\n"
- "ld1rqb z4.b, p7/z, [a_ptr4]\n"
- "udot z26.s, z16.b, z2.b[0]\n"
- "ld1rqb z5.b, p7/z, [a_ptr5]\n"
- "udot z27.s, z16.b, z3.b[0]\n"
- "ld1rqb z6.b, p7/z, [a_ptr6]\n"
- "udot z24.s, z17.b, z0.b[1]\n"
- "ld1rqb z7.b, p7/z, [a_ptr7]\n"
- "udot z28.s, z16.b, z4.b[0]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- "udot z29.s, z16.b, z5.b[0]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- "udot z30.s, z16.b, z6.b[0]\n"
"addvl %[b_ptr0], %[b_ptr0], #8\n"
+ "cbz %[loops], 2f\n"
+ "mov z24.s, #0\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "mov z25.s, #0\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "mov z26.s, #0\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2]\n"
+ "mov z27.s, #0\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "mov z28.s, #0\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "mov z29.s, #0\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "mov z30.s, #0\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "mov z31.s, #0\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "udot z24.s, z16.b, z0.b[0]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "udot z25.s, z16.b, z1.b[0]\n"
+ "udot z26.s, z16.b, z2.b[0]\n"
+ "udot z27.s, z16.b, z3.b[0]\n"
+ "udot z28.s, z16.b, z4.b[0]\n"
+ "udot z29.s, z16.b, z5.b[0]\n"
+ "udot z30.s, z16.b, z6.b[0]\n"
"udot z31.s, z16.b, z7.b[0]\n"
+ "udot z24.s, z17.b, z0.b[1]\n"
"udot z25.s, z17.b, z1.b[1]\n"
"udot z26.s, z17.b, z2.b[1]\n"
"udot z27.s, z17.b, z3.b[1]\n"
@@ -2396,87 +2691,82 @@
"udot z29.s, z23.b, z5.b[3]\n"
"udot z30.s, z23.b, z6.b[3]\n"
"udot z31.s, z23.b, z7.b[3]\n"
- "cbz %[loops], 2f\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
"b.eq 3f\n"
"4:\n"
"st1w z24.s, p7, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"mov z24.s, #0\n"
- "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "st1w z25.s, p7, [c_ptr1]\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"addvl %[c_ptr0], %[c_ptr0], #1\n"
- "mov z25.s, #0\n"
- "ld1rqb z1.b, p7/z, [a_ptr1]\n"
- "udot z24.s, z16.b, z0.b[0]\n"
- "st1w z26.s, p7, [c_ptr2]\n"
- "mov z26.s, #0\n"
- "ld1rqb z2.b, p7/z, [a_ptr2]\n"
- "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "st1w z25.s, p7, [c_ptr1]\n"
"addvl c_ptr1, c_ptr1, #1\n"
- "udot z25.s, z16.b, z1.b[0]\n"
- "st1w z27.s, p7, [c_ptr3]\n"
- "mov z27.s, #0\n"
- "ld1rqb z4.b, p7/z, [a_ptr4]\n"
- "udot z26.s, z16.b, z2.b[0]\n"
- "ld1rqb z5.b, p7/z, [a_ptr5]\n"
- "udot z24.s, z17.b, z0.b[1]\n"
- "st1w z28.s, p7, [c_ptr4]\n"
- "mov z28.s, #0\n"
- "ld1rqb z6.b, p7/z, [a_ptr6]\n"
- "udot z27.s, z16.b, z3.b[0]\n"
- "ld1rqb z7.b, p7/z, [a_ptr7]\n"
- "udot z25.s, z17.b, z1.b[1]\n"
- "st1w z29.s, p7, [c_ptr5]\n"
- "mov z29.s, #0\n"
+ "mov z25.s, #0\n"
+ "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "st1w z26.s, p7, [c_ptr2]\n"
"addvl c_ptr2, c_ptr2, #1\n"
- "udot z28.s, z16.b, z4.b[0]\n"
+ "mov z26.s, #0\n"
+ "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ "st1w z27.s, p7, [c_ptr3]\n"
+ "addvl c_ptr3, c_ptr3, #1\n"
+ "mov z27.s, #0\n"
+ "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #8\n"
+ "st1w z28.s, p7, [c_ptr4]\n"
+ "addvl c_ptr4, c_ptr4, #1\n"
+ "mov z28.s, #0\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ "st1w z29.s, p7, [c_ptr5]\n"
+ "addvl c_ptr5, c_ptr5, #1\n"
+ "mov z29.s, #0\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2]\n"
+ "udot z24.s, z16.b, z0.b[0]\n"
"st1w z30.s, p7, [c_ptr6]\n"
"mov z30.s, #0\n"
- "addvl c_ptr3, c_ptr3, #1\n"
- "udot z29.s, z16.b, z5.b[0]\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "udot z25.s, z16.b, z1.b[0]\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "udot z26.s, z16.b, z2.b[0]\n"
"st1w z31.s, p7, [c_ptr7]\n"
"mov z31.s, #0\n"
- "addvl c_ptr4, c_ptr4, #1\n"
- "udot z30.s, z16.b, z6.b[0]\n"
- "addvl c_ptr5, c_ptr5, #1\n"
- "udot z31.s, z16.b, z7.b[0]\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
- "udot z26.s, z17.b, z2.b[1]\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "udot z27.s, z16.b, z3.b[0]\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "udot z28.s, z16.b, z4.b[0]\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "udot z24.s, z17.b, z0.b[1]\n"
"addvl c_ptr6, c_ptr6, #1\n"
- "udot z27.s, z17.b, z3.b[1]\n"
+ "udot z29.s, z16.b, z5.b[0]\n"
"addvl c_ptr7, c_ptr7, #1\n"
- "udot z28.s, z17.b, z4.b[1]\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
- "udot z29.s, z17.b, z5.b[1]\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- "udot z30.s, z17.b, z6.b[1]\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- "udot z31.s, z17.b, z7.b[1]\n"
- "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "udot z24.s, z18.b, z0.b[2]\n"
+ "udot z30.s, z16.b, z6.b[0]\n"
"prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- "udot z25.s, z18.b, z1.b[2]\n"
+ "udot z31.s, z16.b, z7.b[0]\n"
"prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- "udot z26.s, z18.b, z2.b[2]\n"
+ "udot z25.s, z17.b, z1.b[1]\n"
"prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- "udot z27.s, z18.b, z3.b[2]\n"
+ "udot z26.s, z17.b, z2.b[1]\n"
"prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
- "udot z28.s, z18.b, z4.b[2]\n"
+ "udot z27.s, z17.b, z3.b[1]\n"
"prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
+ "udot z28.s, z17.b, z4.b[1]\n"
+ "udot z29.s, z17.b, z5.b[1]\n"
+ "udot z30.s, z17.b, z6.b[1]\n"
+ "udot z31.s, z17.b, z7.b[1]\n"
+ "udot z24.s, z18.b, z0.b[2]\n"
+ "udot z25.s, z18.b, z1.b[2]\n"
+ "udot z26.s, z18.b, z2.b[2]\n"
+ "udot z27.s, z18.b, z3.b[2]\n"
+ "udot z28.s, z18.b, z4.b[2]\n"
"udot z29.s, z18.b, z5.b[2]\n"
"udot z30.s, z18.b, z6.b[2]\n"
"udot z31.s, z18.b, z7.b[2]\n"
- "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"udot z24.s, z19.b, z0.b[3]\n"
"ld1rqb z0.b, p6/z, [%[a_ptr0], #0x10]\n"
"udot z25.s, z19.b, z1.b[3]\n"
@@ -2494,7 +2784,6 @@
"udot z31.s, z19.b, z7.b[3]\n"
"ld1rqb z7.b, p6/z, [a_ptr7, #0x10]\n"
"udot z24.s, z20.b, z0.b[0]\n"
- "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"udot z25.s, z20.b, z1.b[0]\n"
"udot z26.s, z20.b, z2.b[0]\n"
"udot z27.s, z20.b, z3.b[0]\n"
@@ -2502,7 +2791,6 @@
"udot z29.s, z20.b, z5.b[0]\n"
"udot z30.s, z20.b, z6.b[0]\n"
"udot z31.s, z20.b, z7.b[0]\n"
- "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
"udot z24.s, z21.b, z0.b[1]\n"
"udot z25.s, z21.b, z1.b[1]\n"
"udot z26.s, z21.b, z2.b[1]\n"
@@ -2511,7 +2799,6 @@
"udot z29.s, z21.b, z5.b[1]\n"
"udot z30.s, z21.b, z6.b[1]\n"
"udot z31.s, z21.b, z7.b[1]\n"
- "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
"udot z24.s, z22.b, z0.b[2]\n"
"udot z25.s, z22.b, z1.b[2]\n"
"udot z26.s, z22.b, z2.b[2]\n"
@@ -2520,7 +2807,6 @@
"udot z29.s, z22.b, z5.b[2]\n"
"udot z30.s, z22.b, z6.b[2]\n"
"udot z31.s, z22.b, z7.b[2]\n"
- "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
"udot z24.s, z23.b, z0.b[3]\n"
"udot z25.s, z23.b, z1.b[3]\n"
"udot z26.s, z23.b, z2.b[3]\n"
@@ -2534,47 +2820,54 @@
"st1w z24.s, p7, [%[c_ptr0]]\n"
"addvl %[c_ptr0], %[c_ptr0], #1\n"
"mov z24.s, #0\n"
- "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"st1w z25.s, p7, [c_ptr1]\n"
"addvl c_ptr1, c_ptr1, #1\n"
"mov z25.s, #0\n"
- "ld1rqb z1.b, p7/z, [a_ptr1]\n"
- "udot z24.s, z16.b, z0.b[0]\n"
+ "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"st1w z26.s, p7, [c_ptr2]\n"
- "mov z26.s, #0\n"
- "ld1rqb z2.b, p7/z, [a_ptr2]\n"
- "ld1rqb z3.b, p7/z, [a_ptr3]\n"
"addvl c_ptr2, c_ptr2, #1\n"
- "udot z25.s, z16.b, z1.b[0]\n"
+ "mov z26.s, #0\n"
+ "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
"st1w z27.s, p7, [c_ptr3]\n"
- "mov z27.s, #0\n"
- "ld1rqb z4.b, p7/z, [a_ptr4]\n"
- "udot z26.s, z16.b, z2.b[0]\n"
- "ld1rqb z5.b, p7/z, [a_ptr5]\n"
- "udot z24.s, z17.b, z0.b[1]\n"
- "st1w z28.s, p7, [c_ptr4]\n"
- "mov z28.s, #0\n"
- "ld1rqb z6.b, p7/z, [a_ptr6]\n"
- "udot z27.s, z16.b, z3.b[0]\n"
- "ld1rqb z7.b, p7/z, [a_ptr7]\n"
- "udot z25.s, z17.b, z1.b[1]\n"
- "st1w z29.s, p7, [c_ptr5]\n"
- "mov z29.s, #0\n"
"addvl c_ptr3, c_ptr3, #1\n"
- "udot z28.s, z16.b, z4.b[0]\n"
+ "mov z27.s, #0\n"
+ "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #8\n"
+ "st1w z28.s, p7, [c_ptr4]\n"
+ "addvl c_ptr4, c_ptr4, #1\n"
+ "mov z28.s, #0\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "st1w z29.s, p7, [c_ptr5]\n"
+ "addvl c_ptr5, c_ptr5, #1\n"
+ "mov z29.s, #0\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2]\n"
+ "udot z24.s, z16.b, z0.b[0]\n"
"st1w z30.s, p7, [c_ptr6]\n"
"mov z30.s, #0\n"
- "addvl c_ptr4, c_ptr4, #1\n"
- "udot z29.s, z16.b, z5.b[0]\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "udot z25.s, z16.b, z1.b[0]\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "udot z26.s, z16.b, z2.b[0]\n"
"st1w z31.s, p7, [c_ptr7]\n"
"mov z31.s, #0\n"
- "addvl c_ptr5, c_ptr5, #1\n"
- "udot z30.s, z16.b, z6.b[0]\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "udot z27.s, z16.b, z3.b[0]\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "udot z28.s, z16.b, z4.b[0]\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "udot z24.s, z17.b, z0.b[1]\n"
"addvl c_ptr6, c_ptr6, #1\n"
- "udot z31.s, z16.b, z7.b[0]\n"
+ "udot z29.s, z16.b, z5.b[0]\n"
"addvl c_ptr7, c_ptr7, #1\n"
+ "udot z30.s, z16.b, z6.b[0]\n"
+ "udot z31.s, z16.b, z7.b[0]\n"
+ "udot z25.s, z17.b, z1.b[1]\n"
"udot z26.s, z17.b, z2.b[1]\n"
"udot z27.s, z17.b, z3.b[1]\n"
"udot z28.s, z17.b, z4.b[1]\n"
@@ -2637,23 +2930,106 @@
"udot z29.s, z23.b, z5.b[3]\n"
"udot z30.s, z23.b, z6.b[3]\n"
"udot z31.s, z23.b, z7.b[3]\n"
+ "b 5f\n"
"2:\n"
+ "mov z24.s, #0\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "mov z25.s, #0\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "mov z26.s, #0\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2]\n"
+ "mov z27.s, #0\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "mov z28.s, #0\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "mov z29.s, #0\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "mov z30.s, #0\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "mov z31.s, #0\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "udot z24.s, z16.b, z0.b[0]\n"
+ "udot z25.s, z16.b, z1.b[0]\n"
+ "udot z26.s, z16.b, z2.b[0]\n"
+ "udot z27.s, z16.b, z3.b[0]\n"
+ "udot z28.s, z16.b, z4.b[0]\n"
+ "udot z29.s, z16.b, z5.b[0]\n"
+ "udot z30.s, z16.b, z6.b[0]\n"
+ "udot z31.s, z16.b, z7.b[0]\n"
+ "udot z24.s, z17.b, z0.b[1]\n"
+ "udot z25.s, z17.b, z1.b[1]\n"
+ "udot z26.s, z17.b, z2.b[1]\n"
+ "udot z27.s, z17.b, z3.b[1]\n"
+ "udot z28.s, z17.b, z4.b[1]\n"
+ "udot z29.s, z17.b, z5.b[1]\n"
+ "udot z30.s, z17.b, z6.b[1]\n"
+ "udot z31.s, z17.b, z7.b[1]\n"
+ "udot z24.s, z18.b, z0.b[2]\n"
+ "udot z25.s, z18.b, z1.b[2]\n"
+ "udot z26.s, z18.b, z2.b[2]\n"
+ "udot z27.s, z18.b, z3.b[2]\n"
+ "udot z28.s, z18.b, z4.b[2]\n"
+ "udot z29.s, z18.b, z5.b[2]\n"
+ "udot z30.s, z18.b, z6.b[2]\n"
+ "udot z31.s, z18.b, z7.b[2]\n"
+ "udot z24.s, z19.b, z0.b[3]\n"
+ "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x10]\n"
+ "udot z25.s, z19.b, z1.b[3]\n"
+ "ld1rqb z1.b, p6/z, [a_ptr1, #0x10]\n"
+ "udot z26.s, z19.b, z2.b[3]\n"
+ "ld1rqb z2.b, p6/z, [a_ptr2, #0x10]\n"
+ "udot z27.s, z19.b, z3.b[3]\n"
+ "ld1rqb z3.b, p6/z, [a_ptr3, #0x10]\n"
+ "udot z28.s, z19.b, z4.b[3]\n"
+ "ld1rqb z4.b, p6/z, [a_ptr4, #0x10]\n"
+ "udot z29.s, z19.b, z5.b[3]\n"
+ "ld1rqb z5.b, p6/z, [a_ptr5, #0x10]\n"
+ "udot z30.s, z19.b, z6.b[3]\n"
+ "ld1rqb z6.b, p6/z, [a_ptr6, #0x10]\n"
+ "udot z31.s, z19.b, z7.b[3]\n"
+ "ld1rqb z7.b, p6/z, [a_ptr7, #0x10]\n"
+ "udot z24.s, z20.b, z0.b[0]\n"
+ "udot z25.s, z20.b, z1.b[0]\n"
+ "udot z26.s, z20.b, z2.b[0]\n"
+ "udot z27.s, z20.b, z3.b[0]\n"
+ "udot z28.s, z20.b, z4.b[0]\n"
+ "udot z29.s, z20.b, z5.b[0]\n"
+ "udot z30.s, z20.b, z6.b[0]\n"
+ "udot z31.s, z20.b, z7.b[0]\n"
+ "udot z24.s, z21.b, z0.b[1]\n"
+ "udot z25.s, z21.b, z1.b[1]\n"
+ "udot z26.s, z21.b, z2.b[1]\n"
+ "udot z27.s, z21.b, z3.b[1]\n"
+ "udot z28.s, z21.b, z4.b[1]\n"
+ "udot z29.s, z21.b, z5.b[1]\n"
+ "udot z30.s, z21.b, z6.b[1]\n"
+ "udot z31.s, z21.b, z7.b[1]\n"
+ "udot z24.s, z22.b, z0.b[2]\n"
+ "udot z25.s, z22.b, z1.b[2]\n"
+ "udot z26.s, z22.b, z2.b[2]\n"
+ "udot z27.s, z22.b, z3.b[2]\n"
+ "udot z28.s, z22.b, z4.b[2]\n"
+ "udot z29.s, z22.b, z5.b[2]\n"
+ "udot z30.s, z22.b, z6.b[2]\n"
+ "udot z31.s, z22.b, z7.b[2]\n"
+ "udot z24.s, z23.b, z0.b[3]\n"
+ "udot z25.s, z23.b, z1.b[3]\n"
+ "udot z26.s, z23.b, z2.b[3]\n"
+ "udot z27.s, z23.b, z3.b[3]\n"
+ "udot z28.s, z23.b, z4.b[3]\n"
+ "udot z29.s, z23.b, z5.b[3]\n"
+ "udot z30.s, z23.b, z6.b[3]\n"
+ "udot z31.s, z23.b, z7.b[3]\n"
+ "5:\n"
"st1w z24.s, p0, [%[c_ptr0]]\n"
"addvl %[c_ptr0], %[c_ptr0], #1\n"
"st1w z25.s, p0, [c_ptr1]\n"
- "addvl c_ptr1, c_ptr1, #1\n"
"st1w z26.s, p0, [c_ptr2]\n"
- "addvl c_ptr2, c_ptr2, #1\n"
"st1w z27.s, p0, [c_ptr3]\n"
- "addvl c_ptr3, c_ptr3, #1\n"
"st1w z28.s, p0, [c_ptr4]\n"
- "addvl c_ptr4, c_ptr4, #1\n"
"st1w z29.s, p0, [c_ptr5]\n"
- "addvl c_ptr5, c_ptr5, #1\n"
"st1w z30.s, p0, [c_ptr6]\n"
- "addvl c_ptr6, c_ptr6, #1\n"
"st1w z31.s, p0, [c_ptr7]\n"
- "addvl c_ptr7, c_ptr7, #1\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -2732,54 +3108,56 @@
"add c_ptr1, %[c_ptr0], #0x0\n"
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
- "mov z24.s, #0\n"
"ptrue p7.b\n"
- "mov z25.s, #0\n"
"whilelt p6.b, %[temp], %[odd_depth]\n"
- "mov z26.s, #0\n"
+ "whilelt p0.s, %[temp], %[last_width]\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
"ld1b z16.b, p7/z, [%[b_ptr0]]\n"
- "mov z27.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
"ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "mov z28.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
"ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "mov z29.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
"ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "mov z30.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
"ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "mov z31.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
"ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
"ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "whilelt p0.s, %[temp], %[last_width]\n"
"ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
- "ld1rqb z1.b, p7/z, [a_ptr1]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
- "udot z24.s, z16.b, z0.b[0]\n"
- "ld1rqb z2.b, p7/z, [a_ptr2]\n"
- "ld1rqb z3.b, p7/z, [a_ptr3]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
- "udot z25.s, z16.b, z1.b[0]\n"
- "ld1rqb z4.b, p7/z, [a_ptr4]\n"
- "udot z26.s, z16.b, z2.b[0]\n"
- "ld1rqb z5.b, p7/z, [a_ptr5]\n"
- "udot z27.s, z16.b, z3.b[0]\n"
- "ld1rqb z6.b, p7/z, [a_ptr6]\n"
- "udot z24.s, z17.b, z0.b[1]\n"
- "ld1rqb z7.b, p7/z, [a_ptr7]\n"
- "udot z28.s, z16.b, z4.b[0]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- "udot z29.s, z16.b, z5.b[0]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- "udot z30.s, z16.b, z6.b[0]\n"
"addvl %[b_ptr0], %[b_ptr0], #8\n"
+ "cbz %[loops], 2f\n"
+ "mov z24.s, #0\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "mov z25.s, #0\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "mov z26.s, #0\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2]\n"
+ "mov z27.s, #0\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "mov z28.s, #0\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "mov z29.s, #0\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "mov z30.s, #0\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "mov z31.s, #0\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "udot z24.s, z16.b, z0.b[0]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "udot z25.s, z16.b, z1.b[0]\n"
+ "udot z26.s, z16.b, z2.b[0]\n"
+ "udot z27.s, z16.b, z3.b[0]\n"
+ "udot z28.s, z16.b, z4.b[0]\n"
+ "udot z29.s, z16.b, z5.b[0]\n"
+ "udot z30.s, z16.b, z6.b[0]\n"
"udot z31.s, z16.b, z7.b[0]\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "udot z24.s, z17.b, z0.b[1]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #1\n"
"udot z25.s, z17.b, z1.b[1]\n"
"udot z26.s, z17.b, z2.b[1]\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
"udot z27.s, z17.b, z3.b[1]\n"
- "addvl %[b_ptr0], %[b_ptr0], #1\n"
"udot z28.s, z17.b, z4.b[1]\n"
"udot z29.s, z17.b, z5.b[1]\n"
"udot z30.s, z17.b, z6.b[1]\n"
@@ -2856,88 +3234,84 @@
"udot z29.s, z16.b, z5.b[0]\n"
"udot z30.s, z16.b, z6.b[0]\n"
"udot z31.s, z16.b, z7.b[0]\n"
- "cbz %[loops], 2f\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"b.eq 3f\n"
"4:\n"
"st1w z24.s, p7, [%[c_ptr0]]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
+ "subs %[loops], %[loops], #0x1\n"
"mov z24.s, #0\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "addvl %[c_ptr0], %[c_ptr0], #1\n"
+ "st1w z25.s, p7, [c_ptr1]\n"
+ "addvl c_ptr1, c_ptr1, #1\n"
+ "mov z25.s, #0\n"
+ "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "st1w z26.s, p7, [c_ptr2]\n"
+ "addvl c_ptr2, c_ptr2, #1\n"
+ "mov z26.s, #0\n"
+ "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ "st1w z27.s, p7, [c_ptr3]\n"
+ "addvl c_ptr3, c_ptr3, #1\n"
+ "mov z27.s, #0\n"
+ "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #8\n"
+ "st1w z28.s, p7, [c_ptr4]\n"
+ "addvl c_ptr4, c_ptr4, #1\n"
+ "mov z28.s, #0\n"
"ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
"ld1rqb z1.b, p7/z, [a_ptr1]\n"
- "subs %[loops], %[loops], #0x1\n"
- "st1w z25.s, p7, [c_ptr1]\n"
- "addvl %[c_ptr0], %[c_ptr0], #1\n"
- "mov z25.s, #0\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ "st1w z29.s, p7, [c_ptr5]\n"
+ "addvl c_ptr5, c_ptr5, #1\n"
+ "mov z29.s, #0\n"
"ld1rqb z2.b, p7/z, [a_ptr2]\n"
"udot z24.s, z16.b, z0.b[0]\n"
- "st1w z26.s, p7, [c_ptr2]\n"
- "mov z26.s, #0\n"
- "ld1rqb z3.b, p7/z, [a_ptr3]\n"
- "ld1rqb z4.b, p7/z, [a_ptr4]\n"
- "addvl c_ptr1, c_ptr1, #1\n"
- "udot z25.s, z16.b, z1.b[0]\n"
- "st1w z27.s, p7, [c_ptr3]\n"
- "mov z27.s, #0\n"
- "ld1rqb z5.b, p7/z, [a_ptr5]\n"
- "udot z26.s, z16.b, z2.b[0]\n"
- "ld1rqb z6.b, p7/z, [a_ptr6]\n"
- "udot z24.s, z17.b, z0.b[1]\n"
- "st1w z28.s, p7, [c_ptr4]\n"
- "mov z28.s, #0\n"
- "ld1rqb z7.b, p7/z, [a_ptr7]\n"
- "udot z27.s, z16.b, z3.b[0]\n"
- "addvl c_ptr2, c_ptr2, #1\n"
- "udot z25.s, z17.b, z1.b[1]\n"
- "st1w z29.s, p7, [c_ptr5]\n"
- "mov z29.s, #0\n"
- "addvl c_ptr3, c_ptr3, #1\n"
- "udot z28.s, z16.b, z4.b[0]\n"
"st1w z30.s, p7, [c_ptr6]\n"
"mov z30.s, #0\n"
- "addvl c_ptr4, c_ptr4, #1\n"
- "udot z29.s, z16.b, z5.b[0]\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "udot z25.s, z16.b, z1.b[0]\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "udot z26.s, z16.b, z2.b[0]\n"
"st1w z31.s, p7, [c_ptr7]\n"
"mov z31.s, #0\n"
- "addvl c_ptr5, c_ptr5, #1\n"
- "udot z30.s, z16.b, z6.b[0]\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "udot z27.s, z16.b, z3.b[0]\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "udot z28.s, z16.b, z4.b[0]\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "udot z24.s, z17.b, z0.b[1]\n"
"addvl c_ptr6, c_ptr6, #1\n"
+ "udot z29.s, z16.b, z5.b[0]\n"
+ "addvl c_ptr7, c_ptr7, #1\n"
+ "udot z30.s, z16.b, z6.b[0]\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
"udot z31.s, z16.b, z7.b[0]\n"
"ld1b z16.b, p7/z, [%[b_ptr0]]\n"
- "udot z26.s, z17.b, z2.b[1]\n"
- "addvl c_ptr7, c_ptr7, #1\n"
- "udot z27.s, z17.b, z3.b[1]\n"
+ "udot z25.s, z17.b, z1.b[1]\n"
"addvl %[b_ptr0], %[b_ptr0], #1\n"
- "udot z28.s, z17.b, z4.b[1]\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
- "udot z29.s, z17.b, z5.b[1]\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
- "udot z30.s, z17.b, z6.b[1]\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
- "udot z31.s, z17.b, z7.b[1]\n"
- "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "udot z24.s, z18.b, z0.b[2]\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
- "udot z25.s, z18.b, z1.b[2]\n"
+ "udot z26.s, z17.b, z2.b[1]\n"
"prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- "udot z26.s, z18.b, z2.b[2]\n"
+ "udot z27.s, z17.b, z3.b[1]\n"
"prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
- "udot z27.s, z18.b, z3.b[2]\n"
+ "udot z28.s, z17.b, z4.b[1]\n"
"prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
- "udot z28.s, z18.b, z4.b[2]\n"
+ "udot z29.s, z17.b, z5.b[1]\n"
"prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
+ "udot z30.s, z17.b, z6.b[1]\n"
+ "udot z31.s, z17.b, z7.b[1]\n"
+ "udot z24.s, z18.b, z0.b[2]\n"
+ "udot z25.s, z18.b, z1.b[2]\n"
+ "udot z26.s, z18.b, z2.b[2]\n"
+ "udot z27.s, z18.b, z3.b[2]\n"
+ "udot z28.s, z18.b, z4.b[2]\n"
"udot z29.s, z18.b, z5.b[2]\n"
"udot z30.s, z18.b, z6.b[2]\n"
"udot z31.s, z18.b, z7.b[2]\n"
- "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"udot z24.s, z19.b, z0.b[3]\n"
"ld1rqb z0.b, p7/z, [%[a_ptr0], #0x10]\n"
"udot z25.s, z19.b, z1.b[3]\n"
@@ -2955,7 +3329,6 @@
"udot z31.s, z19.b, z7.b[3]\n"
"ld1rqb z7.b, p7/z, [a_ptr7, #0x10]\n"
"udot z24.s, z20.b, z0.b[0]\n"
- "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"udot z25.s, z20.b, z1.b[0]\n"
"udot z26.s, z20.b, z2.b[0]\n"
"udot z27.s, z20.b, z3.b[0]\n"
@@ -2963,7 +3336,6 @@
"udot z29.s, z20.b, z5.b[0]\n"
"udot z30.s, z20.b, z6.b[0]\n"
"udot z31.s, z20.b, z7.b[0]\n"
- "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
"udot z24.s, z21.b, z0.b[1]\n"
"udot z25.s, z21.b, z1.b[1]\n"
"udot z26.s, z21.b, z2.b[1]\n"
@@ -2972,7 +3344,6 @@
"udot z29.s, z21.b, z5.b[1]\n"
"udot z30.s, z21.b, z6.b[1]\n"
"udot z31.s, z21.b, z7.b[1]\n"
- "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
"udot z24.s, z22.b, z0.b[2]\n"
"udot z25.s, z22.b, z1.b[2]\n"
"udot z26.s, z22.b, z2.b[2]\n"
@@ -2981,7 +3352,6 @@
"udot z29.s, z22.b, z5.b[2]\n"
"udot z30.s, z22.b, z6.b[2]\n"
"udot z31.s, z22.b, z7.b[2]\n"
- "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
"udot z24.s, z23.b, z0.b[3]\n"
"ld1rqb z0.b, p6/z, [%[a_ptr0], #0x20]\n"
"udot z25.s, z23.b, z1.b[3]\n"
@@ -2999,7 +3369,6 @@
"udot z31.s, z23.b, z7.b[3]\n"
"ld1rqb z7.b, p6/z, [a_ptr7, #0x20]\n"
"udot z24.s, z16.b, z0.b[0]\n"
- "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"udot z25.s, z16.b, z1.b[0]\n"
"udot z26.s, z16.b, z2.b[0]\n"
"udot z27.s, z16.b, z3.b[0]\n"
@@ -3007,55 +3376,62 @@
"udot z29.s, z16.b, z5.b[0]\n"
"udot z30.s, z16.b, z6.b[0]\n"
"udot z31.s, z16.b, z7.b[0]\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
"b.ne 4b\n"
"3:\n"
"st1w z24.s, p7, [%[c_ptr0]]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "mov z24.s, #0\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "ld1rqb z1.b, p7/z, [a_ptr1]\n"
"addvl %[c_ptr0], %[c_ptr0], #1\n"
+ "mov z24.s, #0\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"st1w z25.s, p7, [c_ptr1]\n"
"addvl c_ptr1, c_ptr1, #1\n"
"mov z25.s, #0\n"
+ "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "st1w z26.s, p7, [c_ptr2]\n"
+ "addvl c_ptr2, c_ptr2, #1\n"
+ "mov z26.s, #0\n"
+ "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "st1w z27.s, p7, [c_ptr3]\n"
+ "addvl c_ptr3, c_ptr3, #1\n"
+ "mov z27.s, #0\n"
+ "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #8\n"
+ "st1w z28.s, p7, [c_ptr4]\n"
+ "addvl c_ptr4, c_ptr4, #1\n"
+ "mov z28.s, #0\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "st1w z29.s, p7, [c_ptr5]\n"
+ "addvl c_ptr5, c_ptr5, #1\n"
+ "mov z29.s, #0\n"
"ld1rqb z2.b, p7/z, [a_ptr2]\n"
"udot z24.s, z16.b, z0.b[0]\n"
- "st1w z26.s, p7, [c_ptr2]\n"
- "mov z26.s, #0\n"
- "ld1rqb z3.b, p7/z, [a_ptr3]\n"
- "ld1rqb z4.b, p7/z, [a_ptr4]\n"
- "addvl c_ptr2, c_ptr2, #1\n"
- "udot z25.s, z16.b, z1.b[0]\n"
- "st1w z27.s, p7, [c_ptr3]\n"
- "mov z27.s, #0\n"
- "ld1rqb z5.b, p7/z, [a_ptr5]\n"
- "udot z26.s, z16.b, z2.b[0]\n"
- "ld1rqb z6.b, p7/z, [a_ptr6]\n"
- "udot z24.s, z17.b, z0.b[1]\n"
- "st1w z28.s, p7, [c_ptr4]\n"
- "mov z28.s, #0\n"
- "ld1rqb z7.b, p7/z, [a_ptr7]\n"
- "udot z27.s, z16.b, z3.b[0]\n"
- "addvl c_ptr3, c_ptr3, #1\n"
- "udot z25.s, z17.b, z1.b[1]\n"
- "st1w z29.s, p7, [c_ptr5]\n"
- "mov z29.s, #0\n"
- "addvl c_ptr4, c_ptr4, #1\n"
- "udot z28.s, z16.b, z4.b[0]\n"
"st1w z30.s, p7, [c_ptr6]\n"
"mov z30.s, #0\n"
- "addvl c_ptr5, c_ptr5, #1\n"
- "udot z29.s, z16.b, z5.b[0]\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "udot z25.s, z16.b, z1.b[0]\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "udot z26.s, z16.b, z2.b[0]\n"
"st1w z31.s, p7, [c_ptr7]\n"
"mov z31.s, #0\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "udot z27.s, z16.b, z3.b[0]\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "udot z28.s, z16.b, z4.b[0]\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "udot z24.s, z17.b, z0.b[1]\n"
"addvl c_ptr6, c_ptr6, #1\n"
- "udot z30.s, z16.b, z6.b[0]\n"
+ "udot z29.s, z16.b, z5.b[0]\n"
"addvl c_ptr7, c_ptr7, #1\n"
+ "udot z30.s, z16.b, z6.b[0]\n"
"udot z31.s, z16.b, z7.b[0]\n"
"ld1b z16.b, p7/z, [%[b_ptr0]]\n"
- "udot z26.s, z17.b, z2.b[1]\n"
+ "udot z25.s, z17.b, z1.b[1]\n"
"addvl %[b_ptr0], %[b_ptr0], #1\n"
+ "udot z26.s, z17.b, z2.b[1]\n"
"udot z27.s, z17.b, z3.b[1]\n"
"udot z28.s, z17.b, z4.b[1]\n"
"udot z29.s, z17.b, z5.b[1]\n"
@@ -3133,23 +3509,124 @@
"udot z29.s, z16.b, z5.b[0]\n"
"udot z30.s, z16.b, z6.b[0]\n"
"udot z31.s, z16.b, z7.b[0]\n"
+ "b 5f\n"
"2:\n"
+ "mov z24.s, #0\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "mov z25.s, #0\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "mov z26.s, #0\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2]\n"
+ "mov z27.s, #0\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "mov z28.s, #0\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "mov z29.s, #0\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "mov z30.s, #0\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "mov z31.s, #0\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "udot z24.s, z16.b, z0.b[0]\n"
+ "udot z25.s, z16.b, z1.b[0]\n"
+ "udot z26.s, z16.b, z2.b[0]\n"
+ "udot z27.s, z16.b, z3.b[0]\n"
+ "udot z28.s, z16.b, z4.b[0]\n"
+ "udot z29.s, z16.b, z5.b[0]\n"
+ "udot z30.s, z16.b, z6.b[0]\n"
+ "udot z31.s, z16.b, z7.b[0]\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "udot z24.s, z17.b, z0.b[1]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #1\n"
+ "udot z25.s, z17.b, z1.b[1]\n"
+ "udot z26.s, z17.b, z2.b[1]\n"
+ "udot z27.s, z17.b, z3.b[1]\n"
+ "udot z28.s, z17.b, z4.b[1]\n"
+ "udot z29.s, z17.b, z5.b[1]\n"
+ "udot z30.s, z17.b, z6.b[1]\n"
+ "udot z31.s, z17.b, z7.b[1]\n"
+ "udot z24.s, z18.b, z0.b[2]\n"
+ "udot z25.s, z18.b, z1.b[2]\n"
+ "udot z26.s, z18.b, z2.b[2]\n"
+ "udot z27.s, z18.b, z3.b[2]\n"
+ "udot z28.s, z18.b, z4.b[2]\n"
+ "udot z29.s, z18.b, z5.b[2]\n"
+ "udot z30.s, z18.b, z6.b[2]\n"
+ "udot z31.s, z18.b, z7.b[2]\n"
+ "udot z24.s, z19.b, z0.b[3]\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x10]\n"
+ "udot z25.s, z19.b, z1.b[3]\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1, #0x10]\n"
+ "udot z26.s, z19.b, z2.b[3]\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2, #0x10]\n"
+ "udot z27.s, z19.b, z3.b[3]\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3, #0x10]\n"
+ "udot z28.s, z19.b, z4.b[3]\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4, #0x10]\n"
+ "udot z29.s, z19.b, z5.b[3]\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5, #0x10]\n"
+ "udot z30.s, z19.b, z6.b[3]\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6, #0x10]\n"
+ "udot z31.s, z19.b, z7.b[3]\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7, #0x10]\n"
+ "udot z24.s, z20.b, z0.b[0]\n"
+ "udot z25.s, z20.b, z1.b[0]\n"
+ "udot z26.s, z20.b, z2.b[0]\n"
+ "udot z27.s, z20.b, z3.b[0]\n"
+ "udot z28.s, z20.b, z4.b[0]\n"
+ "udot z29.s, z20.b, z5.b[0]\n"
+ "udot z30.s, z20.b, z6.b[0]\n"
+ "udot z31.s, z20.b, z7.b[0]\n"
+ "udot z24.s, z21.b, z0.b[1]\n"
+ "udot z25.s, z21.b, z1.b[1]\n"
+ "udot z26.s, z21.b, z2.b[1]\n"
+ "udot z27.s, z21.b, z3.b[1]\n"
+ "udot z28.s, z21.b, z4.b[1]\n"
+ "udot z29.s, z21.b, z5.b[1]\n"
+ "udot z30.s, z21.b, z6.b[1]\n"
+ "udot z31.s, z21.b, z7.b[1]\n"
+ "udot z24.s, z22.b, z0.b[2]\n"
+ "udot z25.s, z22.b, z1.b[2]\n"
+ "udot z26.s, z22.b, z2.b[2]\n"
+ "udot z27.s, z22.b, z3.b[2]\n"
+ "udot z28.s, z22.b, z4.b[2]\n"
+ "udot z29.s, z22.b, z5.b[2]\n"
+ "udot z30.s, z22.b, z6.b[2]\n"
+ "udot z31.s, z22.b, z7.b[2]\n"
+ "udot z24.s, z23.b, z0.b[3]\n"
+ "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x20]\n"
+ "udot z25.s, z23.b, z1.b[3]\n"
+ "ld1rqb z1.b, p6/z, [a_ptr1, #0x20]\n"
+ "udot z26.s, z23.b, z2.b[3]\n"
+ "ld1rqb z2.b, p6/z, [a_ptr2, #0x20]\n"
+ "udot z27.s, z23.b, z3.b[3]\n"
+ "ld1rqb z3.b, p6/z, [a_ptr3, #0x20]\n"
+ "udot z28.s, z23.b, z4.b[3]\n"
+ "ld1rqb z4.b, p6/z, [a_ptr4, #0x20]\n"
+ "udot z29.s, z23.b, z5.b[3]\n"
+ "ld1rqb z5.b, p6/z, [a_ptr5, #0x20]\n"
+ "udot z30.s, z23.b, z6.b[3]\n"
+ "ld1rqb z6.b, p6/z, [a_ptr6, #0x20]\n"
+ "udot z31.s, z23.b, z7.b[3]\n"
+ "ld1rqb z7.b, p6/z, [a_ptr7, #0x20]\n"
+ "udot z24.s, z16.b, z0.b[0]\n"
+ "udot z25.s, z16.b, z1.b[0]\n"
+ "udot z26.s, z16.b, z2.b[0]\n"
+ "udot z27.s, z16.b, z3.b[0]\n"
+ "udot z28.s, z16.b, z4.b[0]\n"
+ "udot z29.s, z16.b, z5.b[0]\n"
+ "udot z30.s, z16.b, z6.b[0]\n"
+ "udot z31.s, z16.b, z7.b[0]\n"
+ "5:\n"
"st1w z24.s, p0, [%[c_ptr0]]\n"
"addvl %[c_ptr0], %[c_ptr0], #1\n"
"st1w z25.s, p0, [c_ptr1]\n"
- "addvl c_ptr1, c_ptr1, #1\n"
"st1w z26.s, p0, [c_ptr2]\n"
- "addvl c_ptr2, c_ptr2, #1\n"
"st1w z27.s, p0, [c_ptr3]\n"
- "addvl c_ptr3, c_ptr3, #1\n"
"st1w z28.s, p0, [c_ptr4]\n"
- "addvl c_ptr4, c_ptr4, #1\n"
"st1w z29.s, p0, [c_ptr5]\n"
- "addvl c_ptr5, c_ptr5, #1\n"
"st1w z30.s, p0, [c_ptr6]\n"
- "addvl c_ptr6, c_ptr6, #1\n"
"st1w z31.s, p0, [c_ptr7]\n"
- "addvl c_ptr7, c_ptr7, #1\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -3228,52 +3705,54 @@
"add c_ptr1, %[c_ptr0], #0x0\n"
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
- "mov z24.s, #0\n"
"ptrue p7.b\n"
- "mov z25.s, #0\n"
"whilelt p6.b, %[temp], %[odd_depth]\n"
- "mov z26.s, #0\n"
+ "whilelt p0.s, %[temp], %[last_width]\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
"ld1b z16.b, p7/z, [%[b_ptr0]]\n"
- "mov z27.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
"ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "mov z28.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
"ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "mov z29.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
"ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "mov z30.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
"ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "mov z31.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
"ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
"ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "whilelt p0.s, %[temp], %[last_width]\n"
"ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
- "ld1rqb z1.b, p7/z, [a_ptr1]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
- "udot z24.s, z16.b, z0.b[0]\n"
- "ld1rqb z2.b, p7/z, [a_ptr2]\n"
- "ld1rqb z3.b, p7/z, [a_ptr3]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
- "udot z25.s, z16.b, z1.b[0]\n"
- "ld1rqb z4.b, p7/z, [a_ptr4]\n"
- "udot z26.s, z16.b, z2.b[0]\n"
- "ld1rqb z5.b, p7/z, [a_ptr5]\n"
- "udot z27.s, z16.b, z3.b[0]\n"
- "ld1rqb z6.b, p7/z, [a_ptr6]\n"
- "udot z24.s, z17.b, z0.b[1]\n"
- "ld1rqb z7.b, p7/z, [a_ptr7]\n"
- "udot z28.s, z16.b, z4.b[0]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- "udot z29.s, z16.b, z5.b[0]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- "udot z30.s, z16.b, z6.b[0]\n"
"addvl %[b_ptr0], %[b_ptr0], #8\n"
+ "cbz %[loops], 2f\n"
+ "mov z24.s, #0\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "mov z25.s, #0\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "mov z26.s, #0\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2]\n"
+ "mov z27.s, #0\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "mov z28.s, #0\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "mov z29.s, #0\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "mov z30.s, #0\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "mov z31.s, #0\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "udot z24.s, z16.b, z0.b[0]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "udot z25.s, z16.b, z1.b[0]\n"
+ "udot z26.s, z16.b, z2.b[0]\n"
+ "udot z27.s, z16.b, z3.b[0]\n"
+ "udot z28.s, z16.b, z4.b[0]\n"
+ "udot z29.s, z16.b, z5.b[0]\n"
+ "udot z30.s, z16.b, z6.b[0]\n"
"udot z31.s, z16.b, z7.b[0]\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "udot z24.s, z17.b, z0.b[1]\n"
"udot z25.s, z17.b, z1.b[1]\n"
"udot z26.s, z17.b, z2.b[1]\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
"udot z27.s, z17.b, z3.b[1]\n"
"udot z28.s, z17.b, z4.b[1]\n"
"udot z29.s, z17.b, z5.b[1]\n"
@@ -3361,88 +3840,85 @@
"udot z29.s, z17.b, z5.b[1]\n"
"udot z30.s, z17.b, z6.b[1]\n"
"udot z31.s, z17.b, z7.b[1]\n"
- "cbz %[loops], 2f\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"b.eq 3f\n"
"4:\n"
"st1w z24.s, p7, [%[c_ptr0]]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
+ "subs %[loops], %[loops], #0x1\n"
"mov z24.s, #0\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "addvl %[c_ptr0], %[c_ptr0], #1\n"
+ "st1w z25.s, p7, [c_ptr1]\n"
+ "addvl c_ptr1, c_ptr1, #1\n"
+ "mov z25.s, #0\n"
+ "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "st1w z26.s, p7, [c_ptr2]\n"
+ "addvl c_ptr2, c_ptr2, #1\n"
+ "mov z26.s, #0\n"
+ "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ "st1w z27.s, p7, [c_ptr3]\n"
+ "addvl c_ptr3, c_ptr3, #1\n"
+ "mov z27.s, #0\n"
+ "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #8\n"
+ "st1w z28.s, p7, [c_ptr4]\n"
+ "addvl c_ptr4, c_ptr4, #1\n"
+ "mov z28.s, #0\n"
"ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
"ld1rqb z1.b, p7/z, [a_ptr1]\n"
- "subs %[loops], %[loops], #0x1\n"
- "st1w z25.s, p7, [c_ptr1]\n"
- "addvl %[c_ptr0], %[c_ptr0], #1\n"
- "mov z25.s, #0\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ "st1w z29.s, p7, [c_ptr5]\n"
+ "addvl c_ptr5, c_ptr5, #1\n"
+ "mov z29.s, #0\n"
"ld1rqb z2.b, p7/z, [a_ptr2]\n"
"udot z24.s, z16.b, z0.b[0]\n"
- "st1w z26.s, p7, [c_ptr2]\n"
- "mov z26.s, #0\n"
- "ld1rqb z3.b, p7/z, [a_ptr3]\n"
- "ld1rqb z4.b, p7/z, [a_ptr4]\n"
- "addvl c_ptr1, c_ptr1, #1\n"
- "udot z25.s, z16.b, z1.b[0]\n"
- "st1w z27.s, p7, [c_ptr3]\n"
- "mov z27.s, #0\n"
- "ld1rqb z5.b, p7/z, [a_ptr5]\n"
- "udot z26.s, z16.b, z2.b[0]\n"
- "ld1rqb z6.b, p7/z, [a_ptr6]\n"
- "udot z24.s, z17.b, z0.b[1]\n"
- "st1w z28.s, p7, [c_ptr4]\n"
- "mov z28.s, #0\n"
- "ld1rqb z7.b, p7/z, [a_ptr7]\n"
- "udot z27.s, z16.b, z3.b[0]\n"
- "addvl c_ptr2, c_ptr2, #1\n"
- "udot z25.s, z17.b, z1.b[1]\n"
- "st1w z29.s, p7, [c_ptr5]\n"
- "mov z29.s, #0\n"
- "addvl c_ptr3, c_ptr3, #1\n"
- "udot z28.s, z16.b, z4.b[0]\n"
"st1w z30.s, p7, [c_ptr6]\n"
"mov z30.s, #0\n"
- "addvl c_ptr4, c_ptr4, #1\n"
- "udot z29.s, z16.b, z5.b[0]\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "udot z25.s, z16.b, z1.b[0]\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "udot z26.s, z16.b, z2.b[0]\n"
"st1w z31.s, p7, [c_ptr7]\n"
"mov z31.s, #0\n"
- "addvl c_ptr5, c_ptr5, #1\n"
- "udot z30.s, z16.b, z6.b[0]\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "udot z27.s, z16.b, z3.b[0]\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "udot z28.s, z16.b, z4.b[0]\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "udot z24.s, z17.b, z0.b[1]\n"
"addvl c_ptr6, c_ptr6, #1\n"
+ "udot z29.s, z16.b, z5.b[0]\n"
+ "addvl c_ptr7, c_ptr7, #1\n"
+ "udot z30.s, z16.b, z6.b[0]\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
"udot z31.s, z16.b, z7.b[0]\n"
"ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "udot z25.s, z17.b, z1.b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
"udot z26.s, z17.b, z2.b[1]\n"
- "addvl c_ptr7, c_ptr7, #1\n"
+ "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
"udot z27.s, z17.b, z3.b[1]\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
"udot z28.s, z17.b, z4.b[1]\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ "prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
"udot z29.s, z17.b, z5.b[1]\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
"udot z30.s, z17.b, z6.b[1]\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
"udot z31.s, z17.b, z7.b[1]\n"
"ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"udot z24.s, z18.b, z0.b[2]\n"
- "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
- "udot z25.s, z18.b, z1.b[2]\n"
"addvl %[b_ptr0], %[b_ptr0], #2\n"
+ "udot z25.s, z18.b, z1.b[2]\n"
"udot z26.s, z18.b, z2.b[2]\n"
- "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
"udot z27.s, z18.b, z3.b[2]\n"
- "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
"udot z28.s, z18.b, z4.b[2]\n"
- "prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
"udot z29.s, z18.b, z5.b[2]\n"
"udot z30.s, z18.b, z6.b[2]\n"
"udot z31.s, z18.b, z7.b[2]\n"
- "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"udot z24.s, z19.b, z0.b[3]\n"
"ld1rqb z0.b, p7/z, [%[a_ptr0], #0x10]\n"
"udot z25.s, z19.b, z1.b[3]\n"
@@ -3460,7 +3936,6 @@
"udot z31.s, z19.b, z7.b[3]\n"
"ld1rqb z7.b, p7/z, [a_ptr7, #0x10]\n"
"udot z24.s, z20.b, z0.b[0]\n"
- "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"udot z25.s, z20.b, z1.b[0]\n"
"udot z26.s, z20.b, z2.b[0]\n"
"udot z27.s, z20.b, z3.b[0]\n"
@@ -3468,7 +3943,6 @@
"udot z29.s, z20.b, z5.b[0]\n"
"udot z30.s, z20.b, z6.b[0]\n"
"udot z31.s, z20.b, z7.b[0]\n"
- "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
"udot z24.s, z21.b, z0.b[1]\n"
"udot z25.s, z21.b, z1.b[1]\n"
"udot z26.s, z21.b, z2.b[1]\n"
@@ -3477,7 +3951,6 @@
"udot z29.s, z21.b, z5.b[1]\n"
"udot z30.s, z21.b, z6.b[1]\n"
"udot z31.s, z21.b, z7.b[1]\n"
- "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
"udot z24.s, z22.b, z0.b[2]\n"
"udot z25.s, z22.b, z1.b[2]\n"
"udot z26.s, z22.b, z2.b[2]\n"
@@ -3486,7 +3959,6 @@
"udot z29.s, z22.b, z5.b[2]\n"
"udot z30.s, z22.b, z6.b[2]\n"
"udot z31.s, z22.b, z7.b[2]\n"
- "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
"udot z24.s, z23.b, z0.b[3]\n"
"ld1rqb z0.b, p6/z, [%[a_ptr0], #0x20]\n"
"udot z25.s, z23.b, z1.b[3]\n"
@@ -3504,7 +3976,6 @@
"udot z31.s, z23.b, z7.b[3]\n"
"ld1rqb z7.b, p6/z, [a_ptr7, #0x20]\n"
"udot z24.s, z16.b, z0.b[0]\n"
- "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"udot z25.s, z16.b, z1.b[0]\n"
"udot z26.s, z16.b, z2.b[0]\n"
"udot z27.s, z16.b, z3.b[0]\n"
@@ -3512,7 +3983,6 @@
"udot z29.s, z16.b, z5.b[0]\n"
"udot z30.s, z16.b, z6.b[0]\n"
"udot z31.s, z16.b, z7.b[0]\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
"udot z24.s, z17.b, z0.b[1]\n"
"udot z25.s, z17.b, z1.b[1]\n"
"udot z26.s, z17.b, z2.b[1]\n"
@@ -3521,53 +3991,60 @@
"udot z29.s, z17.b, z5.b[1]\n"
"udot z30.s, z17.b, z6.b[1]\n"
"udot z31.s, z17.b, z7.b[1]\n"
- "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"b.ne 4b\n"
"3:\n"
"st1w z24.s, p7, [%[c_ptr0]]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "mov z24.s, #0\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "ld1rqb z1.b, p7/z, [a_ptr1]\n"
"addvl %[c_ptr0], %[c_ptr0], #1\n"
+ "mov z24.s, #0\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"st1w z25.s, p7, [c_ptr1]\n"
"addvl c_ptr1, c_ptr1, #1\n"
"mov z25.s, #0\n"
+ "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "st1w z26.s, p7, [c_ptr2]\n"
+ "addvl c_ptr2, c_ptr2, #1\n"
+ "mov z26.s, #0\n"
+ "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "st1w z27.s, p7, [c_ptr3]\n"
+ "addvl c_ptr3, c_ptr3, #1\n"
+ "mov z27.s, #0\n"
+ "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #8\n"
+ "st1w z28.s, p7, [c_ptr4]\n"
+ "addvl c_ptr4, c_ptr4, #1\n"
+ "mov z28.s, #0\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "st1w z29.s, p7, [c_ptr5]\n"
+ "addvl c_ptr5, c_ptr5, #1\n"
+ "mov z29.s, #0\n"
"ld1rqb z2.b, p7/z, [a_ptr2]\n"
"udot z24.s, z16.b, z0.b[0]\n"
- "st1w z26.s, p7, [c_ptr2]\n"
- "mov z26.s, #0\n"
- "ld1rqb z3.b, p7/z, [a_ptr3]\n"
- "ld1rqb z4.b, p7/z, [a_ptr4]\n"
- "addvl c_ptr2, c_ptr2, #1\n"
- "udot z25.s, z16.b, z1.b[0]\n"
- "st1w z27.s, p7, [c_ptr3]\n"
- "mov z27.s, #0\n"
- "ld1rqb z5.b, p7/z, [a_ptr5]\n"
- "udot z26.s, z16.b, z2.b[0]\n"
- "ld1rqb z6.b, p7/z, [a_ptr6]\n"
- "udot z24.s, z17.b, z0.b[1]\n"
- "st1w z28.s, p7, [c_ptr4]\n"
- "mov z28.s, #0\n"
- "ld1rqb z7.b, p7/z, [a_ptr7]\n"
- "udot z27.s, z16.b, z3.b[0]\n"
- "addvl c_ptr3, c_ptr3, #1\n"
- "udot z25.s, z17.b, z1.b[1]\n"
- "st1w z29.s, p7, [c_ptr5]\n"
- "mov z29.s, #0\n"
- "addvl c_ptr4, c_ptr4, #1\n"
- "udot z28.s, z16.b, z4.b[0]\n"
"st1w z30.s, p7, [c_ptr6]\n"
"mov z30.s, #0\n"
- "addvl c_ptr5, c_ptr5, #1\n"
- "udot z29.s, z16.b, z5.b[0]\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "udot z25.s, z16.b, z1.b[0]\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "udot z26.s, z16.b, z2.b[0]\n"
"st1w z31.s, p7, [c_ptr7]\n"
"mov z31.s, #0\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "udot z27.s, z16.b, z3.b[0]\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "udot z28.s, z16.b, z4.b[0]\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "udot z24.s, z17.b, z0.b[1]\n"
"addvl c_ptr6, c_ptr6, #1\n"
- "udot z30.s, z16.b, z6.b[0]\n"
+ "udot z29.s, z16.b, z5.b[0]\n"
"addvl c_ptr7, c_ptr7, #1\n"
+ "udot z30.s, z16.b, z6.b[0]\n"
"udot z31.s, z16.b, z7.b[0]\n"
"ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "udot z25.s, z17.b, z1.b[1]\n"
"udot z26.s, z17.b, z2.b[1]\n"
"udot z27.s, z17.b, z3.b[1]\n"
"udot z28.s, z17.b, z4.b[1]\n"
@@ -3656,23 +4133,133 @@
"udot z29.s, z17.b, z5.b[1]\n"
"udot z30.s, z17.b, z6.b[1]\n"
"udot z31.s, z17.b, z7.b[1]\n"
+ "b 5f\n"
"2:\n"
+ "mov z24.s, #0\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "mov z25.s, #0\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "mov z26.s, #0\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2]\n"
+ "mov z27.s, #0\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "mov z28.s, #0\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "mov z29.s, #0\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "mov z30.s, #0\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "mov z31.s, #0\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "udot z24.s, z16.b, z0.b[0]\n"
+ "udot z25.s, z16.b, z1.b[0]\n"
+ "udot z26.s, z16.b, z2.b[0]\n"
+ "udot z27.s, z16.b, z3.b[0]\n"
+ "udot z28.s, z16.b, z4.b[0]\n"
+ "udot z29.s, z16.b, z5.b[0]\n"
+ "udot z30.s, z16.b, z6.b[0]\n"
+ "udot z31.s, z16.b, z7.b[0]\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "udot z24.s, z17.b, z0.b[1]\n"
+ "udot z25.s, z17.b, z1.b[1]\n"
+ "udot z26.s, z17.b, z2.b[1]\n"
+ "udot z27.s, z17.b, z3.b[1]\n"
+ "udot z28.s, z17.b, z4.b[1]\n"
+ "udot z29.s, z17.b, z5.b[1]\n"
+ "udot z30.s, z17.b, z6.b[1]\n"
+ "udot z31.s, z17.b, z7.b[1]\n"
+ "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "udot z24.s, z18.b, z0.b[2]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #2\n"
+ "udot z25.s, z18.b, z1.b[2]\n"
+ "udot z26.s, z18.b, z2.b[2]\n"
+ "udot z27.s, z18.b, z3.b[2]\n"
+ "udot z28.s, z18.b, z4.b[2]\n"
+ "udot z29.s, z18.b, z5.b[2]\n"
+ "udot z30.s, z18.b, z6.b[2]\n"
+ "udot z31.s, z18.b, z7.b[2]\n"
+ "udot z24.s, z19.b, z0.b[3]\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x10]\n"
+ "udot z25.s, z19.b, z1.b[3]\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1, #0x10]\n"
+ "udot z26.s, z19.b, z2.b[3]\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2, #0x10]\n"
+ "udot z27.s, z19.b, z3.b[3]\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3, #0x10]\n"
+ "udot z28.s, z19.b, z4.b[3]\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4, #0x10]\n"
+ "udot z29.s, z19.b, z5.b[3]\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5, #0x10]\n"
+ "udot z30.s, z19.b, z6.b[3]\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6, #0x10]\n"
+ "udot z31.s, z19.b, z7.b[3]\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7, #0x10]\n"
+ "udot z24.s, z20.b, z0.b[0]\n"
+ "udot z25.s, z20.b, z1.b[0]\n"
+ "udot z26.s, z20.b, z2.b[0]\n"
+ "udot z27.s, z20.b, z3.b[0]\n"
+ "udot z28.s, z20.b, z4.b[0]\n"
+ "udot z29.s, z20.b, z5.b[0]\n"
+ "udot z30.s, z20.b, z6.b[0]\n"
+ "udot z31.s, z20.b, z7.b[0]\n"
+ "udot z24.s, z21.b, z0.b[1]\n"
+ "udot z25.s, z21.b, z1.b[1]\n"
+ "udot z26.s, z21.b, z2.b[1]\n"
+ "udot z27.s, z21.b, z3.b[1]\n"
+ "udot z28.s, z21.b, z4.b[1]\n"
+ "udot z29.s, z21.b, z5.b[1]\n"
+ "udot z30.s, z21.b, z6.b[1]\n"
+ "udot z31.s, z21.b, z7.b[1]\n"
+ "udot z24.s, z22.b, z0.b[2]\n"
+ "udot z25.s, z22.b, z1.b[2]\n"
+ "udot z26.s, z22.b, z2.b[2]\n"
+ "udot z27.s, z22.b, z3.b[2]\n"
+ "udot z28.s, z22.b, z4.b[2]\n"
+ "udot z29.s, z22.b, z5.b[2]\n"
+ "udot z30.s, z22.b, z6.b[2]\n"
+ "udot z31.s, z22.b, z7.b[2]\n"
+ "udot z24.s, z23.b, z0.b[3]\n"
+ "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x20]\n"
+ "udot z25.s, z23.b, z1.b[3]\n"
+ "ld1rqb z1.b, p6/z, [a_ptr1, #0x20]\n"
+ "udot z26.s, z23.b, z2.b[3]\n"
+ "ld1rqb z2.b, p6/z, [a_ptr2, #0x20]\n"
+ "udot z27.s, z23.b, z3.b[3]\n"
+ "ld1rqb z3.b, p6/z, [a_ptr3, #0x20]\n"
+ "udot z28.s, z23.b, z4.b[3]\n"
+ "ld1rqb z4.b, p6/z, [a_ptr4, #0x20]\n"
+ "udot z29.s, z23.b, z5.b[3]\n"
+ "ld1rqb z5.b, p6/z, [a_ptr5, #0x20]\n"
+ "udot z30.s, z23.b, z6.b[3]\n"
+ "ld1rqb z6.b, p6/z, [a_ptr6, #0x20]\n"
+ "udot z31.s, z23.b, z7.b[3]\n"
+ "ld1rqb z7.b, p6/z, [a_ptr7, #0x20]\n"
+ "udot z24.s, z16.b, z0.b[0]\n"
+ "udot z25.s, z16.b, z1.b[0]\n"
+ "udot z26.s, z16.b, z2.b[0]\n"
+ "udot z27.s, z16.b, z3.b[0]\n"
+ "udot z28.s, z16.b, z4.b[0]\n"
+ "udot z29.s, z16.b, z5.b[0]\n"
+ "udot z30.s, z16.b, z6.b[0]\n"
+ "udot z31.s, z16.b, z7.b[0]\n"
+ "udot z24.s, z17.b, z0.b[1]\n"
+ "udot z25.s, z17.b, z1.b[1]\n"
+ "udot z26.s, z17.b, z2.b[1]\n"
+ "udot z27.s, z17.b, z3.b[1]\n"
+ "udot z28.s, z17.b, z4.b[1]\n"
+ "udot z29.s, z17.b, z5.b[1]\n"
+ "udot z30.s, z17.b, z6.b[1]\n"
+ "udot z31.s, z17.b, z7.b[1]\n"
+ "5:\n"
"st1w z24.s, p0, [%[c_ptr0]]\n"
"addvl %[c_ptr0], %[c_ptr0], #1\n"
"st1w z25.s, p0, [c_ptr1]\n"
- "addvl c_ptr1, c_ptr1, #1\n"
"st1w z26.s, p0, [c_ptr2]\n"
- "addvl c_ptr2, c_ptr2, #1\n"
"st1w z27.s, p0, [c_ptr3]\n"
- "addvl c_ptr3, c_ptr3, #1\n"
"st1w z28.s, p0, [c_ptr4]\n"
- "addvl c_ptr4, c_ptr4, #1\n"
"st1w z29.s, p0, [c_ptr5]\n"
- "addvl c_ptr5, c_ptr5, #1\n"
"st1w z30.s, p0, [c_ptr6]\n"
- "addvl c_ptr6, c_ptr6, #1\n"
"st1w z31.s, p0, [c_ptr7]\n"
- "addvl c_ptr7, c_ptr7, #1\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -3751,52 +4338,54 @@
"add c_ptr1, %[c_ptr0], #0x0\n"
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
- "mov z24.s, #0\n"
"ptrue p7.b\n"
- "mov z25.s, #0\n"
"whilelt p6.b, %[temp], %[odd_depth]\n"
- "mov z26.s, #0\n"
+ "whilelt p0.s, %[temp], %[last_width]\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
"ld1b z16.b, p7/z, [%[b_ptr0]]\n"
- "mov z27.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
"ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "mov z28.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
"ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "mov z29.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
"ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "mov z30.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
"ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "mov z31.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
"ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
"ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "whilelt p0.s, %[temp], %[last_width]\n"
"ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
- "ld1rqb z1.b, p7/z, [a_ptr1]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
- "udot z24.s, z16.b, z0.b[0]\n"
- "ld1rqb z2.b, p7/z, [a_ptr2]\n"
- "ld1rqb z3.b, p7/z, [a_ptr3]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
- "udot z25.s, z16.b, z1.b[0]\n"
- "ld1rqb z4.b, p7/z, [a_ptr4]\n"
- "udot z26.s, z16.b, z2.b[0]\n"
- "ld1rqb z5.b, p7/z, [a_ptr5]\n"
- "udot z27.s, z16.b, z3.b[0]\n"
- "ld1rqb z6.b, p7/z, [a_ptr6]\n"
- "udot z24.s, z17.b, z0.b[1]\n"
- "ld1rqb z7.b, p7/z, [a_ptr7]\n"
- "udot z28.s, z16.b, z4.b[0]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- "udot z29.s, z16.b, z5.b[0]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- "udot z30.s, z16.b, z6.b[0]\n"
"addvl %[b_ptr0], %[b_ptr0], #8\n"
+ "cbz %[loops], 2f\n"
+ "mov z24.s, #0\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "mov z25.s, #0\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "mov z26.s, #0\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2]\n"
+ "mov z27.s, #0\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "mov z28.s, #0\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "mov z29.s, #0\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "mov z30.s, #0\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "mov z31.s, #0\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "udot z24.s, z16.b, z0.b[0]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "udot z25.s, z16.b, z1.b[0]\n"
+ "udot z26.s, z16.b, z2.b[0]\n"
+ "udot z27.s, z16.b, z3.b[0]\n"
+ "udot z28.s, z16.b, z4.b[0]\n"
+ "udot z29.s, z16.b, z5.b[0]\n"
+ "udot z30.s, z16.b, z6.b[0]\n"
"udot z31.s, z16.b, z7.b[0]\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "udot z24.s, z17.b, z0.b[1]\n"
"udot z25.s, z17.b, z1.b[1]\n"
"udot z26.s, z17.b, z2.b[1]\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
"udot z27.s, z17.b, z3.b[1]\n"
"udot z28.s, z17.b, z4.b[1]\n"
"udot z29.s, z17.b, z5.b[1]\n"
@@ -3893,82 +4482,80 @@
"udot z29.s, z18.b, z5.b[2]\n"
"udot z30.s, z18.b, z6.b[2]\n"
"udot z31.s, z18.b, z7.b[2]\n"
- "cbz %[loops], 2f\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"b.eq 3f\n"
"4:\n"
"st1w z24.s, p7, [%[c_ptr0]]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
+ "subs %[loops], %[loops], #0x1\n"
"mov z24.s, #0\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "addvl %[c_ptr0], %[c_ptr0], #1\n"
+ "st1w z25.s, p7, [c_ptr1]\n"
+ "addvl c_ptr1, c_ptr1, #1\n"
+ "mov z25.s, #0\n"
+ "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "st1w z26.s, p7, [c_ptr2]\n"
+ "addvl c_ptr2, c_ptr2, #1\n"
+ "mov z26.s, #0\n"
+ "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ "st1w z27.s, p7, [c_ptr3]\n"
+ "addvl c_ptr3, c_ptr3, #1\n"
+ "mov z27.s, #0\n"
+ "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #8\n"
+ "st1w z28.s, p7, [c_ptr4]\n"
+ "addvl c_ptr4, c_ptr4, #1\n"
+ "mov z28.s, #0\n"
"ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
"ld1rqb z1.b, p7/z, [a_ptr1]\n"
- "subs %[loops], %[loops], #0x1\n"
- "st1w z25.s, p7, [c_ptr1]\n"
- "addvl %[c_ptr0], %[c_ptr0], #1\n"
- "mov z25.s, #0\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ "st1w z29.s, p7, [c_ptr5]\n"
+ "addvl c_ptr5, c_ptr5, #1\n"
+ "mov z29.s, #0\n"
"ld1rqb z2.b, p7/z, [a_ptr2]\n"
"udot z24.s, z16.b, z0.b[0]\n"
- "st1w z26.s, p7, [c_ptr2]\n"
- "mov z26.s, #0\n"
- "ld1rqb z3.b, p7/z, [a_ptr3]\n"
- "ld1rqb z4.b, p7/z, [a_ptr4]\n"
- "addvl c_ptr1, c_ptr1, #1\n"
- "udot z25.s, z16.b, z1.b[0]\n"
- "st1w z27.s, p7, [c_ptr3]\n"
- "mov z27.s, #0\n"
- "ld1rqb z5.b, p7/z, [a_ptr5]\n"
- "udot z26.s, z16.b, z2.b[0]\n"
- "ld1rqb z6.b, p7/z, [a_ptr6]\n"
- "udot z24.s, z17.b, z0.b[1]\n"
- "st1w z28.s, p7, [c_ptr4]\n"
- "mov z28.s, #0\n"
- "ld1rqb z7.b, p7/z, [a_ptr7]\n"
- "udot z27.s, z16.b, z3.b[0]\n"
- "addvl c_ptr2, c_ptr2, #1\n"
- "udot z25.s, z17.b, z1.b[1]\n"
- "st1w z29.s, p7, [c_ptr5]\n"
- "mov z29.s, #0\n"
- "addvl c_ptr3, c_ptr3, #1\n"
- "udot z28.s, z16.b, z4.b[0]\n"
"st1w z30.s, p7, [c_ptr6]\n"
"mov z30.s, #0\n"
- "addvl c_ptr4, c_ptr4, #1\n"
- "udot z29.s, z16.b, z5.b[0]\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "udot z25.s, z16.b, z1.b[0]\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "udot z26.s, z16.b, z2.b[0]\n"
"st1w z31.s, p7, [c_ptr7]\n"
"mov z31.s, #0\n"
- "addvl c_ptr5, c_ptr5, #1\n"
- "udot z30.s, z16.b, z6.b[0]\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "udot z27.s, z16.b, z3.b[0]\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "udot z28.s, z16.b, z4.b[0]\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "udot z24.s, z17.b, z0.b[1]\n"
"addvl c_ptr6, c_ptr6, #1\n"
+ "udot z29.s, z16.b, z5.b[0]\n"
+ "addvl c_ptr7, c_ptr7, #1\n"
+ "udot z30.s, z16.b, z6.b[0]\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
"udot z31.s, z16.b, z7.b[0]\n"
"ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "udot z25.s, z17.b, z1.b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
"udot z26.s, z17.b, z2.b[1]\n"
- "addvl c_ptr7, c_ptr7, #1\n"
+ "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
"udot z27.s, z17.b, z3.b[1]\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
"udot z28.s, z17.b, z4.b[1]\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ "prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
"udot z29.s, z17.b, z5.b[1]\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
"udot z30.s, z17.b, z6.b[1]\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
"udot z31.s, z17.b, z7.b[1]\n"
"ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"udot z24.s, z18.b, z0.b[2]\n"
- "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
"udot z25.s, z18.b, z1.b[2]\n"
- "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
"udot z26.s, z18.b, z2.b[2]\n"
- "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
"udot z27.s, z18.b, z3.b[2]\n"
- "prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
"udot z28.s, z18.b, z4.b[2]\n"
"udot z29.s, z18.b, z5.b[2]\n"
"udot z30.s, z18.b, z6.b[2]\n"
@@ -3995,12 +4582,10 @@
"udot z25.s, z20.b, z1.b[0]\n"
"udot z26.s, z20.b, z2.b[0]\n"
"udot z27.s, z20.b, z3.b[0]\n"
- "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"udot z28.s, z20.b, z4.b[0]\n"
"udot z29.s, z20.b, z5.b[0]\n"
"udot z30.s, z20.b, z6.b[0]\n"
"udot z31.s, z20.b, z7.b[0]\n"
- "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
"udot z24.s, z21.b, z0.b[1]\n"
"udot z25.s, z21.b, z1.b[1]\n"
"udot z26.s, z21.b, z2.b[1]\n"
@@ -4009,7 +4594,6 @@
"udot z29.s, z21.b, z5.b[1]\n"
"udot z30.s, z21.b, z6.b[1]\n"
"udot z31.s, z21.b, z7.b[1]\n"
- "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
"udot z24.s, z22.b, z0.b[2]\n"
"udot z25.s, z22.b, z1.b[2]\n"
"udot z26.s, z22.b, z2.b[2]\n"
@@ -4018,7 +4602,6 @@
"udot z29.s, z22.b, z5.b[2]\n"
"udot z30.s, z22.b, z6.b[2]\n"
"udot z31.s, z22.b, z7.b[2]\n"
- "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
"udot z24.s, z23.b, z0.b[3]\n"
"ld1rqb z0.b, p6/z, [%[a_ptr0], #0x20]\n"
"udot z25.s, z23.b, z1.b[3]\n"
@@ -4036,7 +4619,6 @@
"udot z31.s, z23.b, z7.b[3]\n"
"ld1rqb z7.b, p6/z, [a_ptr7, #0x20]\n"
"udot z24.s, z16.b, z0.b[0]\n"
- "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"udot z25.s, z16.b, z1.b[0]\n"
"udot z26.s, z16.b, z2.b[0]\n"
"udot z27.s, z16.b, z3.b[0]\n"
@@ -4044,7 +4626,6 @@
"udot z29.s, z16.b, z5.b[0]\n"
"udot z30.s, z16.b, z6.b[0]\n"
"udot z31.s, z16.b, z7.b[0]\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
"udot z24.s, z17.b, z0.b[1]\n"
"udot z25.s, z17.b, z1.b[1]\n"
"udot z26.s, z17.b, z2.b[1]\n"
@@ -4053,7 +4634,6 @@
"udot z29.s, z17.b, z5.b[1]\n"
"udot z30.s, z17.b, z6.b[1]\n"
"udot z31.s, z17.b, z7.b[1]\n"
- "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"udot z24.s, z18.b, z0.b[2]\n"
"udot z25.s, z18.b, z1.b[2]\n"
"udot z26.s, z18.b, z2.b[2]\n"
@@ -4062,53 +4642,60 @@
"udot z29.s, z18.b, z5.b[2]\n"
"udot z30.s, z18.b, z6.b[2]\n"
"udot z31.s, z18.b, z7.b[2]\n"
- "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"b.ne 4b\n"
"3:\n"
"st1w z24.s, p7, [%[c_ptr0]]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "mov z24.s, #0\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "ld1rqb z1.b, p7/z, [a_ptr1]\n"
"addvl %[c_ptr0], %[c_ptr0], #1\n"
+ "mov z24.s, #0\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"st1w z25.s, p7, [c_ptr1]\n"
"addvl c_ptr1, c_ptr1, #1\n"
"mov z25.s, #0\n"
+ "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "st1w z26.s, p7, [c_ptr2]\n"
+ "addvl c_ptr2, c_ptr2, #1\n"
+ "mov z26.s, #0\n"
+ "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "st1w z27.s, p7, [c_ptr3]\n"
+ "addvl c_ptr3, c_ptr3, #1\n"
+ "mov z27.s, #0\n"
+ "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #8\n"
+ "st1w z28.s, p7, [c_ptr4]\n"
+ "addvl c_ptr4, c_ptr4, #1\n"
+ "mov z28.s, #0\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "st1w z29.s, p7, [c_ptr5]\n"
+ "addvl c_ptr5, c_ptr5, #1\n"
+ "mov z29.s, #0\n"
"ld1rqb z2.b, p7/z, [a_ptr2]\n"
"udot z24.s, z16.b, z0.b[0]\n"
- "st1w z26.s, p7, [c_ptr2]\n"
- "mov z26.s, #0\n"
- "ld1rqb z3.b, p7/z, [a_ptr3]\n"
- "ld1rqb z4.b, p7/z, [a_ptr4]\n"
- "addvl c_ptr2, c_ptr2, #1\n"
- "udot z25.s, z16.b, z1.b[0]\n"
- "st1w z27.s, p7, [c_ptr3]\n"
- "mov z27.s, #0\n"
- "ld1rqb z5.b, p7/z, [a_ptr5]\n"
- "udot z26.s, z16.b, z2.b[0]\n"
- "ld1rqb z6.b, p7/z, [a_ptr6]\n"
- "udot z24.s, z17.b, z0.b[1]\n"
- "st1w z28.s, p7, [c_ptr4]\n"
- "mov z28.s, #0\n"
- "ld1rqb z7.b, p7/z, [a_ptr7]\n"
- "udot z27.s, z16.b, z3.b[0]\n"
- "addvl c_ptr3, c_ptr3, #1\n"
- "udot z25.s, z17.b, z1.b[1]\n"
- "st1w z29.s, p7, [c_ptr5]\n"
- "mov z29.s, #0\n"
- "addvl c_ptr4, c_ptr4, #1\n"
- "udot z28.s, z16.b, z4.b[0]\n"
"st1w z30.s, p7, [c_ptr6]\n"
"mov z30.s, #0\n"
- "addvl c_ptr5, c_ptr5, #1\n"
- "udot z29.s, z16.b, z5.b[0]\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "udot z25.s, z16.b, z1.b[0]\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "udot z26.s, z16.b, z2.b[0]\n"
"st1w z31.s, p7, [c_ptr7]\n"
"mov z31.s, #0\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "udot z27.s, z16.b, z3.b[0]\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "udot z28.s, z16.b, z4.b[0]\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "udot z24.s, z17.b, z0.b[1]\n"
"addvl c_ptr6, c_ptr6, #1\n"
- "udot z30.s, z16.b, z6.b[0]\n"
+ "udot z29.s, z16.b, z5.b[0]\n"
"addvl c_ptr7, c_ptr7, #1\n"
+ "udot z30.s, z16.b, z6.b[0]\n"
"udot z31.s, z16.b, z7.b[0]\n"
"ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "udot z25.s, z17.b, z1.b[1]\n"
"udot z26.s, z17.b, z2.b[1]\n"
"udot z27.s, z17.b, z3.b[1]\n"
"udot z28.s, z17.b, z4.b[1]\n"
@@ -4206,23 +4793,142 @@
"udot z29.s, z18.b, z5.b[2]\n"
"udot z30.s, z18.b, z6.b[2]\n"
"udot z31.s, z18.b, z7.b[2]\n"
+ "b 5f\n"
"2:\n"
+ "mov z24.s, #0\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "mov z25.s, #0\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "mov z26.s, #0\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2]\n"
+ "mov z27.s, #0\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "mov z28.s, #0\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "mov z29.s, #0\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "mov z30.s, #0\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "mov z31.s, #0\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "udot z24.s, z16.b, z0.b[0]\n"
+ "udot z25.s, z16.b, z1.b[0]\n"
+ "udot z26.s, z16.b, z2.b[0]\n"
+ "udot z27.s, z16.b, z3.b[0]\n"
+ "udot z28.s, z16.b, z4.b[0]\n"
+ "udot z29.s, z16.b, z5.b[0]\n"
+ "udot z30.s, z16.b, z6.b[0]\n"
+ "udot z31.s, z16.b, z7.b[0]\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "udot z24.s, z17.b, z0.b[1]\n"
+ "udot z25.s, z17.b, z1.b[1]\n"
+ "udot z26.s, z17.b, z2.b[1]\n"
+ "udot z27.s, z17.b, z3.b[1]\n"
+ "udot z28.s, z17.b, z4.b[1]\n"
+ "udot z29.s, z17.b, z5.b[1]\n"
+ "udot z30.s, z17.b, z6.b[1]\n"
+ "udot z31.s, z17.b, z7.b[1]\n"
+ "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "udot z24.s, z18.b, z0.b[2]\n"
+ "udot z25.s, z18.b, z1.b[2]\n"
+ "udot z26.s, z18.b, z2.b[2]\n"
+ "udot z27.s, z18.b, z3.b[2]\n"
+ "udot z28.s, z18.b, z4.b[2]\n"
+ "udot z29.s, z18.b, z5.b[2]\n"
+ "udot z30.s, z18.b, z6.b[2]\n"
+ "udot z31.s, z18.b, z7.b[2]\n"
+ "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "udot z24.s, z19.b, z0.b[3]\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x10]\n"
+ "udot z25.s, z19.b, z1.b[3]\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1, #0x10]\n"
+ "udot z26.s, z19.b, z2.b[3]\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2, #0x10]\n"
+ "udot z27.s, z19.b, z3.b[3]\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3, #0x10]\n"
+ "udot z28.s, z19.b, z4.b[3]\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4, #0x10]\n"
+ "udot z29.s, z19.b, z5.b[3]\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5, #0x10]\n"
+ "udot z30.s, z19.b, z6.b[3]\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6, #0x10]\n"
+ "udot z31.s, z19.b, z7.b[3]\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7, #0x10]\n"
+ "udot z24.s, z20.b, z0.b[0]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #3\n"
+ "udot z25.s, z20.b, z1.b[0]\n"
+ "udot z26.s, z20.b, z2.b[0]\n"
+ "udot z27.s, z20.b, z3.b[0]\n"
+ "udot z28.s, z20.b, z4.b[0]\n"
+ "udot z29.s, z20.b, z5.b[0]\n"
+ "udot z30.s, z20.b, z6.b[0]\n"
+ "udot z31.s, z20.b, z7.b[0]\n"
+ "udot z24.s, z21.b, z0.b[1]\n"
+ "udot z25.s, z21.b, z1.b[1]\n"
+ "udot z26.s, z21.b, z2.b[1]\n"
+ "udot z27.s, z21.b, z3.b[1]\n"
+ "udot z28.s, z21.b, z4.b[1]\n"
+ "udot z29.s, z21.b, z5.b[1]\n"
+ "udot z30.s, z21.b, z6.b[1]\n"
+ "udot z31.s, z21.b, z7.b[1]\n"
+ "udot z24.s, z22.b, z0.b[2]\n"
+ "udot z25.s, z22.b, z1.b[2]\n"
+ "udot z26.s, z22.b, z2.b[2]\n"
+ "udot z27.s, z22.b, z3.b[2]\n"
+ "udot z28.s, z22.b, z4.b[2]\n"
+ "udot z29.s, z22.b, z5.b[2]\n"
+ "udot z30.s, z22.b, z6.b[2]\n"
+ "udot z31.s, z22.b, z7.b[2]\n"
+ "udot z24.s, z23.b, z0.b[3]\n"
+ "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x20]\n"
+ "udot z25.s, z23.b, z1.b[3]\n"
+ "ld1rqb z1.b, p6/z, [a_ptr1, #0x20]\n"
+ "udot z26.s, z23.b, z2.b[3]\n"
+ "ld1rqb z2.b, p6/z, [a_ptr2, #0x20]\n"
+ "udot z27.s, z23.b, z3.b[3]\n"
+ "ld1rqb z3.b, p6/z, [a_ptr3, #0x20]\n"
+ "udot z28.s, z23.b, z4.b[3]\n"
+ "ld1rqb z4.b, p6/z, [a_ptr4, #0x20]\n"
+ "udot z29.s, z23.b, z5.b[3]\n"
+ "ld1rqb z5.b, p6/z, [a_ptr5, #0x20]\n"
+ "udot z30.s, z23.b, z6.b[3]\n"
+ "ld1rqb z6.b, p6/z, [a_ptr6, #0x20]\n"
+ "udot z31.s, z23.b, z7.b[3]\n"
+ "ld1rqb z7.b, p6/z, [a_ptr7, #0x20]\n"
+ "udot z24.s, z16.b, z0.b[0]\n"
+ "udot z25.s, z16.b, z1.b[0]\n"
+ "udot z26.s, z16.b, z2.b[0]\n"
+ "udot z27.s, z16.b, z3.b[0]\n"
+ "udot z28.s, z16.b, z4.b[0]\n"
+ "udot z29.s, z16.b, z5.b[0]\n"
+ "udot z30.s, z16.b, z6.b[0]\n"
+ "udot z31.s, z16.b, z7.b[0]\n"
+ "udot z24.s, z17.b, z0.b[1]\n"
+ "udot z25.s, z17.b, z1.b[1]\n"
+ "udot z26.s, z17.b, z2.b[1]\n"
+ "udot z27.s, z17.b, z3.b[1]\n"
+ "udot z28.s, z17.b, z4.b[1]\n"
+ "udot z29.s, z17.b, z5.b[1]\n"
+ "udot z30.s, z17.b, z6.b[1]\n"
+ "udot z31.s, z17.b, z7.b[1]\n"
+ "udot z24.s, z18.b, z0.b[2]\n"
+ "udot z25.s, z18.b, z1.b[2]\n"
+ "udot z26.s, z18.b, z2.b[2]\n"
+ "udot z27.s, z18.b, z3.b[2]\n"
+ "udot z28.s, z18.b, z4.b[2]\n"
+ "udot z29.s, z18.b, z5.b[2]\n"
+ "udot z30.s, z18.b, z6.b[2]\n"
+ "udot z31.s, z18.b, z7.b[2]\n"
+ "5:\n"
"st1w z24.s, p0, [%[c_ptr0]]\n"
"addvl %[c_ptr0], %[c_ptr0], #1\n"
"st1w z25.s, p0, [c_ptr1]\n"
- "addvl c_ptr1, c_ptr1, #1\n"
"st1w z26.s, p0, [c_ptr2]\n"
- "addvl c_ptr2, c_ptr2, #1\n"
"st1w z27.s, p0, [c_ptr3]\n"
- "addvl c_ptr3, c_ptr3, #1\n"
"st1w z28.s, p0, [c_ptr4]\n"
- "addvl c_ptr4, c_ptr4, #1\n"
"st1w z29.s, p0, [c_ptr5]\n"
- "addvl c_ptr5, c_ptr5, #1\n"
"st1w z30.s, p0, [c_ptr6]\n"
- "addvl c_ptr6, c_ptr6, #1\n"
"st1w z31.s, p0, [c_ptr7]\n"
- "addvl c_ptr7, c_ptr7, #1\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -4301,52 +5007,54 @@
"add c_ptr1, %[c_ptr0], #0x0\n"
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
- "mov z24.s, #0\n"
"ptrue p7.b\n"
- "mov z25.s, #0\n"
"whilelt p6.b, %[temp], %[odd_depth]\n"
- "mov z26.s, #0\n"
+ "whilelt p0.s, %[temp], %[last_width]\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
"ld1b z16.b, p7/z, [%[b_ptr0]]\n"
- "mov z27.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
"ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "mov z28.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
"ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "mov z29.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
"ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "mov z30.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
"ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "mov z31.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
"ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
"ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "whilelt p0.s, %[temp], %[last_width]\n"
"ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
- "ld1rqb z1.b, p7/z, [a_ptr1]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
- "udot z24.s, z16.b, z0.b[0]\n"
- "ld1rqb z2.b, p7/z, [a_ptr2]\n"
- "ld1rqb z3.b, p7/z, [a_ptr3]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
- "udot z25.s, z16.b, z1.b[0]\n"
- "ld1rqb z4.b, p7/z, [a_ptr4]\n"
- "udot z26.s, z16.b, z2.b[0]\n"
- "ld1rqb z5.b, p7/z, [a_ptr5]\n"
- "udot z27.s, z16.b, z3.b[0]\n"
- "ld1rqb z6.b, p7/z, [a_ptr6]\n"
- "udot z24.s, z17.b, z0.b[1]\n"
- "ld1rqb z7.b, p7/z, [a_ptr7]\n"
- "udot z28.s, z16.b, z4.b[0]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- "udot z29.s, z16.b, z5.b[0]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- "udot z30.s, z16.b, z6.b[0]\n"
"addvl %[b_ptr0], %[b_ptr0], #8\n"
+ "cbz %[loops], 2f\n"
+ "mov z24.s, #0\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "mov z25.s, #0\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "mov z26.s, #0\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2]\n"
+ "mov z27.s, #0\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "mov z28.s, #0\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "mov z29.s, #0\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "mov z30.s, #0\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "mov z31.s, #0\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "udot z24.s, z16.b, z0.b[0]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "udot z25.s, z16.b, z1.b[0]\n"
+ "udot z26.s, z16.b, z2.b[0]\n"
+ "udot z27.s, z16.b, z3.b[0]\n"
+ "udot z28.s, z16.b, z4.b[0]\n"
+ "udot z29.s, z16.b, z5.b[0]\n"
+ "udot z30.s, z16.b, z6.b[0]\n"
"udot z31.s, z16.b, z7.b[0]\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "udot z24.s, z17.b, z0.b[1]\n"
"udot z25.s, z17.b, z1.b[1]\n"
"udot z26.s, z17.b, z2.b[1]\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
"udot z27.s, z17.b, z3.b[1]\n"
"udot z28.s, z17.b, z4.b[1]\n"
"udot z29.s, z17.b, z5.b[1]\n"
@@ -4452,82 +5160,80 @@
"udot z29.s, z19.b, z5.b[3]\n"
"udot z30.s, z19.b, z6.b[3]\n"
"udot z31.s, z19.b, z7.b[3]\n"
- "cbz %[loops], 2f\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"b.eq 3f\n"
"4:\n"
"st1w z24.s, p7, [%[c_ptr0]]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
+ "subs %[loops], %[loops], #0x1\n"
"mov z24.s, #0\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "addvl %[c_ptr0], %[c_ptr0], #1\n"
+ "st1w z25.s, p7, [c_ptr1]\n"
+ "addvl c_ptr1, c_ptr1, #1\n"
+ "mov z25.s, #0\n"
+ "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "st1w z26.s, p7, [c_ptr2]\n"
+ "addvl c_ptr2, c_ptr2, #1\n"
+ "mov z26.s, #0\n"
+ "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ "st1w z27.s, p7, [c_ptr3]\n"
+ "addvl c_ptr3, c_ptr3, #1\n"
+ "mov z27.s, #0\n"
+ "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #8\n"
+ "st1w z28.s, p7, [c_ptr4]\n"
+ "addvl c_ptr4, c_ptr4, #1\n"
+ "mov z28.s, #0\n"
"ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
"ld1rqb z1.b, p7/z, [a_ptr1]\n"
- "subs %[loops], %[loops], #0x1\n"
- "st1w z25.s, p7, [c_ptr1]\n"
- "addvl %[c_ptr0], %[c_ptr0], #1\n"
- "mov z25.s, #0\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ "st1w z29.s, p7, [c_ptr5]\n"
+ "addvl c_ptr5, c_ptr5, #1\n"
+ "mov z29.s, #0\n"
"ld1rqb z2.b, p7/z, [a_ptr2]\n"
"udot z24.s, z16.b, z0.b[0]\n"
- "st1w z26.s, p7, [c_ptr2]\n"
- "mov z26.s, #0\n"
- "ld1rqb z3.b, p7/z, [a_ptr3]\n"
- "ld1rqb z4.b, p7/z, [a_ptr4]\n"
- "addvl c_ptr1, c_ptr1, #1\n"
- "udot z25.s, z16.b, z1.b[0]\n"
- "st1w z27.s, p7, [c_ptr3]\n"
- "mov z27.s, #0\n"
- "ld1rqb z5.b, p7/z, [a_ptr5]\n"
- "udot z26.s, z16.b, z2.b[0]\n"
- "ld1rqb z6.b, p7/z, [a_ptr6]\n"
- "udot z24.s, z17.b, z0.b[1]\n"
- "st1w z28.s, p7, [c_ptr4]\n"
- "mov z28.s, #0\n"
- "ld1rqb z7.b, p7/z, [a_ptr7]\n"
- "udot z27.s, z16.b, z3.b[0]\n"
- "addvl c_ptr2, c_ptr2, #1\n"
- "udot z25.s, z17.b, z1.b[1]\n"
- "st1w z29.s, p7, [c_ptr5]\n"
- "mov z29.s, #0\n"
- "addvl c_ptr3, c_ptr3, #1\n"
- "udot z28.s, z16.b, z4.b[0]\n"
"st1w z30.s, p7, [c_ptr6]\n"
"mov z30.s, #0\n"
- "addvl c_ptr4, c_ptr4, #1\n"
- "udot z29.s, z16.b, z5.b[0]\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "udot z25.s, z16.b, z1.b[0]\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "udot z26.s, z16.b, z2.b[0]\n"
"st1w z31.s, p7, [c_ptr7]\n"
"mov z31.s, #0\n"
- "addvl c_ptr5, c_ptr5, #1\n"
- "udot z30.s, z16.b, z6.b[0]\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "udot z27.s, z16.b, z3.b[0]\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "udot z28.s, z16.b, z4.b[0]\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "udot z24.s, z17.b, z0.b[1]\n"
"addvl c_ptr6, c_ptr6, #1\n"
+ "udot z29.s, z16.b, z5.b[0]\n"
+ "addvl c_ptr7, c_ptr7, #1\n"
+ "udot z30.s, z16.b, z6.b[0]\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
"udot z31.s, z16.b, z7.b[0]\n"
"ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "udot z25.s, z17.b, z1.b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
"udot z26.s, z17.b, z2.b[1]\n"
- "addvl c_ptr7, c_ptr7, #1\n"
+ "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
"udot z27.s, z17.b, z3.b[1]\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
"udot z28.s, z17.b, z4.b[1]\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ "prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
"udot z29.s, z17.b, z5.b[1]\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
"udot z30.s, z17.b, z6.b[1]\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
"udot z31.s, z17.b, z7.b[1]\n"
"ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"udot z24.s, z18.b, z0.b[2]\n"
- "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
"udot z25.s, z18.b, z1.b[2]\n"
- "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
"udot z26.s, z18.b, z2.b[2]\n"
- "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
"udot z27.s, z18.b, z3.b[2]\n"
- "prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
"udot z28.s, z18.b, z4.b[2]\n"
"udot z29.s, z18.b, z5.b[2]\n"
"udot z30.s, z18.b, z6.b[2]\n"
@@ -4559,7 +5265,6 @@
"udot z29.s, z20.b, z5.b[0]\n"
"udot z30.s, z20.b, z6.b[0]\n"
"udot z31.s, z20.b, z7.b[0]\n"
- "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
"udot z24.s, z21.b, z0.b[1]\n"
"udot z25.s, z21.b, z1.b[1]\n"
"udot z26.s, z21.b, z2.b[1]\n"
@@ -4568,7 +5273,6 @@
"udot z29.s, z21.b, z5.b[1]\n"
"udot z30.s, z21.b, z6.b[1]\n"
"udot z31.s, z21.b, z7.b[1]\n"
- "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
"udot z24.s, z22.b, z0.b[2]\n"
"udot z25.s, z22.b, z1.b[2]\n"
"udot z26.s, z22.b, z2.b[2]\n"
@@ -4577,7 +5281,6 @@
"udot z29.s, z22.b, z5.b[2]\n"
"udot z30.s, z22.b, z6.b[2]\n"
"udot z31.s, z22.b, z7.b[2]\n"
- "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
"udot z24.s, z23.b, z0.b[3]\n"
"ld1rqb z0.b, p6/z, [%[a_ptr0], #0x20]\n"
"udot z25.s, z23.b, z1.b[3]\n"
@@ -4595,7 +5298,6 @@
"udot z31.s, z23.b, z7.b[3]\n"
"ld1rqb z7.b, p6/z, [a_ptr7, #0x20]\n"
"udot z24.s, z16.b, z0.b[0]\n"
- "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"udot z25.s, z16.b, z1.b[0]\n"
"udot z26.s, z16.b, z2.b[0]\n"
"udot z27.s, z16.b, z3.b[0]\n"
@@ -4603,7 +5305,6 @@
"udot z29.s, z16.b, z5.b[0]\n"
"udot z30.s, z16.b, z6.b[0]\n"
"udot z31.s, z16.b, z7.b[0]\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
"udot z24.s, z17.b, z0.b[1]\n"
"udot z25.s, z17.b, z1.b[1]\n"
"udot z26.s, z17.b, z2.b[1]\n"
@@ -4612,7 +5313,6 @@
"udot z29.s, z17.b, z5.b[1]\n"
"udot z30.s, z17.b, z6.b[1]\n"
"udot z31.s, z17.b, z7.b[1]\n"
- "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"udot z24.s, z18.b, z0.b[2]\n"
"udot z25.s, z18.b, z1.b[2]\n"
"udot z26.s, z18.b, z2.b[2]\n"
@@ -4621,7 +5321,6 @@
"udot z29.s, z18.b, z5.b[2]\n"
"udot z30.s, z18.b, z6.b[2]\n"
"udot z31.s, z18.b, z7.b[2]\n"
- "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"udot z24.s, z19.b, z0.b[3]\n"
"udot z25.s, z19.b, z1.b[3]\n"
"udot z26.s, z19.b, z2.b[3]\n"
@@ -4630,53 +5329,60 @@
"udot z29.s, z19.b, z5.b[3]\n"
"udot z30.s, z19.b, z6.b[3]\n"
"udot z31.s, z19.b, z7.b[3]\n"
- "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"b.ne 4b\n"
"3:\n"
"st1w z24.s, p7, [%[c_ptr0]]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "mov z24.s, #0\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "ld1rqb z1.b, p7/z, [a_ptr1]\n"
"addvl %[c_ptr0], %[c_ptr0], #1\n"
+ "mov z24.s, #0\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"st1w z25.s, p7, [c_ptr1]\n"
"addvl c_ptr1, c_ptr1, #1\n"
"mov z25.s, #0\n"
+ "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "st1w z26.s, p7, [c_ptr2]\n"
+ "addvl c_ptr2, c_ptr2, #1\n"
+ "mov z26.s, #0\n"
+ "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "st1w z27.s, p7, [c_ptr3]\n"
+ "addvl c_ptr3, c_ptr3, #1\n"
+ "mov z27.s, #0\n"
+ "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #8\n"
+ "st1w z28.s, p7, [c_ptr4]\n"
+ "addvl c_ptr4, c_ptr4, #1\n"
+ "mov z28.s, #0\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "st1w z29.s, p7, [c_ptr5]\n"
+ "addvl c_ptr5, c_ptr5, #1\n"
+ "mov z29.s, #0\n"
"ld1rqb z2.b, p7/z, [a_ptr2]\n"
"udot z24.s, z16.b, z0.b[0]\n"
- "st1w z26.s, p7, [c_ptr2]\n"
- "mov z26.s, #0\n"
- "ld1rqb z3.b, p7/z, [a_ptr3]\n"
- "ld1rqb z4.b, p7/z, [a_ptr4]\n"
- "addvl c_ptr2, c_ptr2, #1\n"
- "udot z25.s, z16.b, z1.b[0]\n"
- "st1w z27.s, p7, [c_ptr3]\n"
- "mov z27.s, #0\n"
- "ld1rqb z5.b, p7/z, [a_ptr5]\n"
- "udot z26.s, z16.b, z2.b[0]\n"
- "ld1rqb z6.b, p7/z, [a_ptr6]\n"
- "udot z24.s, z17.b, z0.b[1]\n"
- "st1w z28.s, p7, [c_ptr4]\n"
- "mov z28.s, #0\n"
- "ld1rqb z7.b, p7/z, [a_ptr7]\n"
- "udot z27.s, z16.b, z3.b[0]\n"
- "addvl c_ptr3, c_ptr3, #1\n"
- "udot z25.s, z17.b, z1.b[1]\n"
- "st1w z29.s, p7, [c_ptr5]\n"
- "mov z29.s, #0\n"
- "addvl c_ptr4, c_ptr4, #1\n"
- "udot z28.s, z16.b, z4.b[0]\n"
"st1w z30.s, p7, [c_ptr6]\n"
"mov z30.s, #0\n"
- "addvl c_ptr5, c_ptr5, #1\n"
- "udot z29.s, z16.b, z5.b[0]\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "udot z25.s, z16.b, z1.b[0]\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "udot z26.s, z16.b, z2.b[0]\n"
"st1w z31.s, p7, [c_ptr7]\n"
"mov z31.s, #0\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "udot z27.s, z16.b, z3.b[0]\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "udot z28.s, z16.b, z4.b[0]\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "udot z24.s, z17.b, z0.b[1]\n"
"addvl c_ptr6, c_ptr6, #1\n"
- "udot z30.s, z16.b, z6.b[0]\n"
+ "udot z29.s, z16.b, z5.b[0]\n"
"addvl c_ptr7, c_ptr7, #1\n"
+ "udot z30.s, z16.b, z6.b[0]\n"
"udot z31.s, z16.b, z7.b[0]\n"
"ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "udot z25.s, z17.b, z1.b[1]\n"
"udot z26.s, z17.b, z2.b[1]\n"
"udot z27.s, z17.b, z3.b[1]\n"
"udot z28.s, z17.b, z4.b[1]\n"
@@ -4783,23 +5489,151 @@
"udot z29.s, z19.b, z5.b[3]\n"
"udot z30.s, z19.b, z6.b[3]\n"
"udot z31.s, z19.b, z7.b[3]\n"
+ "b 5f\n"
"2:\n"
+ "mov z24.s, #0\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "mov z25.s, #0\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "mov z26.s, #0\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2]\n"
+ "mov z27.s, #0\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "mov z28.s, #0\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "mov z29.s, #0\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "mov z30.s, #0\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "mov z31.s, #0\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "udot z24.s, z16.b, z0.b[0]\n"
+ "udot z25.s, z16.b, z1.b[0]\n"
+ "udot z26.s, z16.b, z2.b[0]\n"
+ "udot z27.s, z16.b, z3.b[0]\n"
+ "udot z28.s, z16.b, z4.b[0]\n"
+ "udot z29.s, z16.b, z5.b[0]\n"
+ "udot z30.s, z16.b, z6.b[0]\n"
+ "udot z31.s, z16.b, z7.b[0]\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "udot z24.s, z17.b, z0.b[1]\n"
+ "udot z25.s, z17.b, z1.b[1]\n"
+ "udot z26.s, z17.b, z2.b[1]\n"
+ "udot z27.s, z17.b, z3.b[1]\n"
+ "udot z28.s, z17.b, z4.b[1]\n"
+ "udot z29.s, z17.b, z5.b[1]\n"
+ "udot z30.s, z17.b, z6.b[1]\n"
+ "udot z31.s, z17.b, z7.b[1]\n"
+ "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "udot z24.s, z18.b, z0.b[2]\n"
+ "udot z25.s, z18.b, z1.b[2]\n"
+ "udot z26.s, z18.b, z2.b[2]\n"
+ "udot z27.s, z18.b, z3.b[2]\n"
+ "udot z28.s, z18.b, z4.b[2]\n"
+ "udot z29.s, z18.b, z5.b[2]\n"
+ "udot z30.s, z18.b, z6.b[2]\n"
+ "udot z31.s, z18.b, z7.b[2]\n"
+ "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "udot z24.s, z19.b, z0.b[3]\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x10]\n"
+ "udot z25.s, z19.b, z1.b[3]\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1, #0x10]\n"
+ "udot z26.s, z19.b, z2.b[3]\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2, #0x10]\n"
+ "udot z27.s, z19.b, z3.b[3]\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3, #0x10]\n"
+ "udot z28.s, z19.b, z4.b[3]\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4, #0x10]\n"
+ "udot z29.s, z19.b, z5.b[3]\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5, #0x10]\n"
+ "udot z30.s, z19.b, z6.b[3]\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6, #0x10]\n"
+ "udot z31.s, z19.b, z7.b[3]\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7, #0x10]\n"
+ "udot z24.s, z20.b, z0.b[0]\n"
+ "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "udot z25.s, z20.b, z1.b[0]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #4\n"
+ "udot z26.s, z20.b, z2.b[0]\n"
+ "udot z27.s, z20.b, z3.b[0]\n"
+ "udot z28.s, z20.b, z4.b[0]\n"
+ "udot z29.s, z20.b, z5.b[0]\n"
+ "udot z30.s, z20.b, z6.b[0]\n"
+ "udot z31.s, z20.b, z7.b[0]\n"
+ "udot z24.s, z21.b, z0.b[1]\n"
+ "udot z25.s, z21.b, z1.b[1]\n"
+ "udot z26.s, z21.b, z2.b[1]\n"
+ "udot z27.s, z21.b, z3.b[1]\n"
+ "udot z28.s, z21.b, z4.b[1]\n"
+ "udot z29.s, z21.b, z5.b[1]\n"
+ "udot z30.s, z21.b, z6.b[1]\n"
+ "udot z31.s, z21.b, z7.b[1]\n"
+ "udot z24.s, z22.b, z0.b[2]\n"
+ "udot z25.s, z22.b, z1.b[2]\n"
+ "udot z26.s, z22.b, z2.b[2]\n"
+ "udot z27.s, z22.b, z3.b[2]\n"
+ "udot z28.s, z22.b, z4.b[2]\n"
+ "udot z29.s, z22.b, z5.b[2]\n"
+ "udot z30.s, z22.b, z6.b[2]\n"
+ "udot z31.s, z22.b, z7.b[2]\n"
+ "udot z24.s, z23.b, z0.b[3]\n"
+ "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x20]\n"
+ "udot z25.s, z23.b, z1.b[3]\n"
+ "ld1rqb z1.b, p6/z, [a_ptr1, #0x20]\n"
+ "udot z26.s, z23.b, z2.b[3]\n"
+ "ld1rqb z2.b, p6/z, [a_ptr2, #0x20]\n"
+ "udot z27.s, z23.b, z3.b[3]\n"
+ "ld1rqb z3.b, p6/z, [a_ptr3, #0x20]\n"
+ "udot z28.s, z23.b, z4.b[3]\n"
+ "ld1rqb z4.b, p6/z, [a_ptr4, #0x20]\n"
+ "udot z29.s, z23.b, z5.b[3]\n"
+ "ld1rqb z5.b, p6/z, [a_ptr5, #0x20]\n"
+ "udot z30.s, z23.b, z6.b[3]\n"
+ "ld1rqb z6.b, p6/z, [a_ptr6, #0x20]\n"
+ "udot z31.s, z23.b, z7.b[3]\n"
+ "ld1rqb z7.b, p6/z, [a_ptr7, #0x20]\n"
+ "udot z24.s, z16.b, z0.b[0]\n"
+ "udot z25.s, z16.b, z1.b[0]\n"
+ "udot z26.s, z16.b, z2.b[0]\n"
+ "udot z27.s, z16.b, z3.b[0]\n"
+ "udot z28.s, z16.b, z4.b[0]\n"
+ "udot z29.s, z16.b, z5.b[0]\n"
+ "udot z30.s, z16.b, z6.b[0]\n"
+ "udot z31.s, z16.b, z7.b[0]\n"
+ "udot z24.s, z17.b, z0.b[1]\n"
+ "udot z25.s, z17.b, z1.b[1]\n"
+ "udot z26.s, z17.b, z2.b[1]\n"
+ "udot z27.s, z17.b, z3.b[1]\n"
+ "udot z28.s, z17.b, z4.b[1]\n"
+ "udot z29.s, z17.b, z5.b[1]\n"
+ "udot z30.s, z17.b, z6.b[1]\n"
+ "udot z31.s, z17.b, z7.b[1]\n"
+ "udot z24.s, z18.b, z0.b[2]\n"
+ "udot z25.s, z18.b, z1.b[2]\n"
+ "udot z26.s, z18.b, z2.b[2]\n"
+ "udot z27.s, z18.b, z3.b[2]\n"
+ "udot z28.s, z18.b, z4.b[2]\n"
+ "udot z29.s, z18.b, z5.b[2]\n"
+ "udot z30.s, z18.b, z6.b[2]\n"
+ "udot z31.s, z18.b, z7.b[2]\n"
+ "udot z24.s, z19.b, z0.b[3]\n"
+ "udot z25.s, z19.b, z1.b[3]\n"
+ "udot z26.s, z19.b, z2.b[3]\n"
+ "udot z27.s, z19.b, z3.b[3]\n"
+ "udot z28.s, z19.b, z4.b[3]\n"
+ "udot z29.s, z19.b, z5.b[3]\n"
+ "udot z30.s, z19.b, z6.b[3]\n"
+ "udot z31.s, z19.b, z7.b[3]\n"
+ "5:\n"
"st1w z24.s, p0, [%[c_ptr0]]\n"
"addvl %[c_ptr0], %[c_ptr0], #1\n"
"st1w z25.s, p0, [c_ptr1]\n"
- "addvl c_ptr1, c_ptr1, #1\n"
"st1w z26.s, p0, [c_ptr2]\n"
- "addvl c_ptr2, c_ptr2, #1\n"
"st1w z27.s, p0, [c_ptr3]\n"
- "addvl c_ptr3, c_ptr3, #1\n"
"st1w z28.s, p0, [c_ptr4]\n"
- "addvl c_ptr4, c_ptr4, #1\n"
"st1w z29.s, p0, [c_ptr5]\n"
- "addvl c_ptr5, c_ptr5, #1\n"
"st1w z30.s, p0, [c_ptr6]\n"
- "addvl c_ptr6, c_ptr6, #1\n"
"st1w z31.s, p0, [c_ptr7]\n"
- "addvl c_ptr7, c_ptr7, #1\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -4878,52 +5712,54 @@
"add c_ptr1, %[c_ptr0], #0x0\n"
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
- "mov z24.s, #0\n"
"ptrue p7.b\n"
- "mov z25.s, #0\n"
"whilelt p6.b, %[temp], %[odd_depth]\n"
- "mov z26.s, #0\n"
+ "whilelt p0.s, %[temp], %[last_width]\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
"ld1b z16.b, p7/z, [%[b_ptr0]]\n"
- "mov z27.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
"ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "mov z28.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
"ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "mov z29.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
"ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "mov z30.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
"ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "mov z31.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
"ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
"ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "whilelt p0.s, %[temp], %[last_width]\n"
"ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
- "ld1rqb z1.b, p7/z, [a_ptr1]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
- "udot z24.s, z16.b, z0.b[0]\n"
- "ld1rqb z2.b, p7/z, [a_ptr2]\n"
- "ld1rqb z3.b, p7/z, [a_ptr3]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
- "udot z25.s, z16.b, z1.b[0]\n"
- "ld1rqb z4.b, p7/z, [a_ptr4]\n"
- "udot z26.s, z16.b, z2.b[0]\n"
- "ld1rqb z5.b, p7/z, [a_ptr5]\n"
- "udot z27.s, z16.b, z3.b[0]\n"
- "ld1rqb z6.b, p7/z, [a_ptr6]\n"
- "udot z24.s, z17.b, z0.b[1]\n"
- "ld1rqb z7.b, p7/z, [a_ptr7]\n"
- "udot z28.s, z16.b, z4.b[0]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- "udot z29.s, z16.b, z5.b[0]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- "udot z30.s, z16.b, z6.b[0]\n"
"addvl %[b_ptr0], %[b_ptr0], #8\n"
+ "cbz %[loops], 2f\n"
+ "mov z24.s, #0\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "mov z25.s, #0\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "mov z26.s, #0\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2]\n"
+ "mov z27.s, #0\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "mov z28.s, #0\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "mov z29.s, #0\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "mov z30.s, #0\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "mov z31.s, #0\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "udot z24.s, z16.b, z0.b[0]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "udot z25.s, z16.b, z1.b[0]\n"
+ "udot z26.s, z16.b, z2.b[0]\n"
+ "udot z27.s, z16.b, z3.b[0]\n"
+ "udot z28.s, z16.b, z4.b[0]\n"
+ "udot z29.s, z16.b, z5.b[0]\n"
+ "udot z30.s, z16.b, z6.b[0]\n"
"udot z31.s, z16.b, z7.b[0]\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "udot z24.s, z17.b, z0.b[1]\n"
"udot z25.s, z17.b, z1.b[1]\n"
"udot z26.s, z17.b, z2.b[1]\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
"udot z27.s, z17.b, z3.b[1]\n"
"udot z28.s, z17.b, z4.b[1]\n"
"udot z29.s, z17.b, z5.b[1]\n"
@@ -5046,82 +5882,80 @@
"udot z29.s, z20.b, z5.b[0]\n"
"udot z30.s, z20.b, z6.b[0]\n"
"udot z31.s, z20.b, z7.b[0]\n"
- "cbz %[loops], 2f\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"b.eq 3f\n"
"4:\n"
"st1w z24.s, p7, [%[c_ptr0]]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
+ "subs %[loops], %[loops], #0x1\n"
"mov z24.s, #0\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "addvl %[c_ptr0], %[c_ptr0], #1\n"
+ "st1w z25.s, p7, [c_ptr1]\n"
+ "addvl c_ptr1, c_ptr1, #1\n"
+ "mov z25.s, #0\n"
+ "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "st1w z26.s, p7, [c_ptr2]\n"
+ "addvl c_ptr2, c_ptr2, #1\n"
+ "mov z26.s, #0\n"
+ "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ "st1w z27.s, p7, [c_ptr3]\n"
+ "addvl c_ptr3, c_ptr3, #1\n"
+ "mov z27.s, #0\n"
+ "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #8\n"
+ "st1w z28.s, p7, [c_ptr4]\n"
+ "addvl c_ptr4, c_ptr4, #1\n"
+ "mov z28.s, #0\n"
"ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
"ld1rqb z1.b, p7/z, [a_ptr1]\n"
- "subs %[loops], %[loops], #0x1\n"
- "st1w z25.s, p7, [c_ptr1]\n"
- "addvl %[c_ptr0], %[c_ptr0], #1\n"
- "mov z25.s, #0\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ "st1w z29.s, p7, [c_ptr5]\n"
+ "addvl c_ptr5, c_ptr5, #1\n"
+ "mov z29.s, #0\n"
"ld1rqb z2.b, p7/z, [a_ptr2]\n"
"udot z24.s, z16.b, z0.b[0]\n"
- "st1w z26.s, p7, [c_ptr2]\n"
- "mov z26.s, #0\n"
- "ld1rqb z3.b, p7/z, [a_ptr3]\n"
- "ld1rqb z4.b, p7/z, [a_ptr4]\n"
- "addvl c_ptr1, c_ptr1, #1\n"
- "udot z25.s, z16.b, z1.b[0]\n"
- "st1w z27.s, p7, [c_ptr3]\n"
- "mov z27.s, #0\n"
- "ld1rqb z5.b, p7/z, [a_ptr5]\n"
- "udot z26.s, z16.b, z2.b[0]\n"
- "ld1rqb z6.b, p7/z, [a_ptr6]\n"
- "udot z24.s, z17.b, z0.b[1]\n"
- "st1w z28.s, p7, [c_ptr4]\n"
- "mov z28.s, #0\n"
- "ld1rqb z7.b, p7/z, [a_ptr7]\n"
- "udot z27.s, z16.b, z3.b[0]\n"
- "addvl c_ptr2, c_ptr2, #1\n"
- "udot z25.s, z17.b, z1.b[1]\n"
- "st1w z29.s, p7, [c_ptr5]\n"
- "mov z29.s, #0\n"
- "addvl c_ptr3, c_ptr3, #1\n"
- "udot z28.s, z16.b, z4.b[0]\n"
"st1w z30.s, p7, [c_ptr6]\n"
"mov z30.s, #0\n"
- "addvl c_ptr4, c_ptr4, #1\n"
- "udot z29.s, z16.b, z5.b[0]\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "udot z25.s, z16.b, z1.b[0]\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "udot z26.s, z16.b, z2.b[0]\n"
"st1w z31.s, p7, [c_ptr7]\n"
"mov z31.s, #0\n"
- "addvl c_ptr5, c_ptr5, #1\n"
- "udot z30.s, z16.b, z6.b[0]\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "udot z27.s, z16.b, z3.b[0]\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "udot z28.s, z16.b, z4.b[0]\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "udot z24.s, z17.b, z0.b[1]\n"
"addvl c_ptr6, c_ptr6, #1\n"
+ "udot z29.s, z16.b, z5.b[0]\n"
+ "addvl c_ptr7, c_ptr7, #1\n"
+ "udot z30.s, z16.b, z6.b[0]\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
"udot z31.s, z16.b, z7.b[0]\n"
"ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "udot z25.s, z17.b, z1.b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
"udot z26.s, z17.b, z2.b[1]\n"
- "addvl c_ptr7, c_ptr7, #1\n"
+ "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
"udot z27.s, z17.b, z3.b[1]\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
"udot z28.s, z17.b, z4.b[1]\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ "prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
"udot z29.s, z17.b, z5.b[1]\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
"udot z30.s, z17.b, z6.b[1]\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
"udot z31.s, z17.b, z7.b[1]\n"
"ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"udot z24.s, z18.b, z0.b[2]\n"
- "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
"udot z25.s, z18.b, z1.b[2]\n"
- "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
"udot z26.s, z18.b, z2.b[2]\n"
- "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
"udot z27.s, z18.b, z3.b[2]\n"
- "prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
"udot z28.s, z18.b, z4.b[2]\n"
"udot z29.s, z18.b, z5.b[2]\n"
"udot z30.s, z18.b, z6.b[2]\n"
@@ -5162,7 +5996,6 @@
"udot z29.s, z21.b, z5.b[1]\n"
"udot z30.s, z21.b, z6.b[1]\n"
"udot z31.s, z21.b, z7.b[1]\n"
- "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
"udot z24.s, z22.b, z0.b[2]\n"
"udot z25.s, z22.b, z1.b[2]\n"
"udot z26.s, z22.b, z2.b[2]\n"
@@ -5171,7 +6004,6 @@
"udot z29.s, z22.b, z5.b[2]\n"
"udot z30.s, z22.b, z6.b[2]\n"
"udot z31.s, z22.b, z7.b[2]\n"
- "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
"udot z24.s, z23.b, z0.b[3]\n"
"ld1rqb z0.b, p7/z, [%[a_ptr0], #0x20]\n"
"udot z25.s, z23.b, z1.b[3]\n"
@@ -5189,7 +6021,6 @@
"udot z31.s, z23.b, z7.b[3]\n"
"ld1rqb z7.b, p7/z, [a_ptr7, #0x20]\n"
"udot z24.s, z16.b, z0.b[0]\n"
- "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"udot z25.s, z16.b, z1.b[0]\n"
"udot z26.s, z16.b, z2.b[0]\n"
"udot z27.s, z16.b, z3.b[0]\n"
@@ -5197,7 +6028,6 @@
"udot z29.s, z16.b, z5.b[0]\n"
"udot z30.s, z16.b, z6.b[0]\n"
"udot z31.s, z16.b, z7.b[0]\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
"udot z24.s, z17.b, z0.b[1]\n"
"udot z25.s, z17.b, z1.b[1]\n"
"udot z26.s, z17.b, z2.b[1]\n"
@@ -5206,7 +6036,6 @@
"udot z29.s, z17.b, z5.b[1]\n"
"udot z30.s, z17.b, z6.b[1]\n"
"udot z31.s, z17.b, z7.b[1]\n"
- "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"udot z24.s, z18.b, z0.b[2]\n"
"udot z25.s, z18.b, z1.b[2]\n"
"udot z26.s, z18.b, z2.b[2]\n"
@@ -5215,7 +6044,6 @@
"udot z29.s, z18.b, z5.b[2]\n"
"udot z30.s, z18.b, z6.b[2]\n"
"udot z31.s, z18.b, z7.b[2]\n"
- "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"udot z24.s, z19.b, z0.b[3]\n"
"ld1rqb z0.b, p6/z, [%[a_ptr0], #0x30]\n"
"udot z25.s, z19.b, z1.b[3]\n"
@@ -5233,7 +6061,6 @@
"udot z31.s, z19.b, z7.b[3]\n"
"ld1rqb z7.b, p6/z, [a_ptr7, #0x30]\n"
"udot z24.s, z20.b, z0.b[0]\n"
- "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"udot z25.s, z20.b, z1.b[0]\n"
"udot z26.s, z20.b, z2.b[0]\n"
"udot z27.s, z20.b, z3.b[0]\n"
@@ -5241,53 +6068,60 @@
"udot z29.s, z20.b, z5.b[0]\n"
"udot z30.s, z20.b, z6.b[0]\n"
"udot z31.s, z20.b, z7.b[0]\n"
- "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
"b.ne 4b\n"
"3:\n"
"st1w z24.s, p7, [%[c_ptr0]]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "mov z24.s, #0\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "ld1rqb z1.b, p7/z, [a_ptr1]\n"
"addvl %[c_ptr0], %[c_ptr0], #1\n"
+ "mov z24.s, #0\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"st1w z25.s, p7, [c_ptr1]\n"
"addvl c_ptr1, c_ptr1, #1\n"
"mov z25.s, #0\n"
+ "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "st1w z26.s, p7, [c_ptr2]\n"
+ "addvl c_ptr2, c_ptr2, #1\n"
+ "mov z26.s, #0\n"
+ "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "st1w z27.s, p7, [c_ptr3]\n"
+ "addvl c_ptr3, c_ptr3, #1\n"
+ "mov z27.s, #0\n"
+ "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #8\n"
+ "st1w z28.s, p7, [c_ptr4]\n"
+ "addvl c_ptr4, c_ptr4, #1\n"
+ "mov z28.s, #0\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "st1w z29.s, p7, [c_ptr5]\n"
+ "addvl c_ptr5, c_ptr5, #1\n"
+ "mov z29.s, #0\n"
"ld1rqb z2.b, p7/z, [a_ptr2]\n"
"udot z24.s, z16.b, z0.b[0]\n"
- "st1w z26.s, p7, [c_ptr2]\n"
- "mov z26.s, #0\n"
- "ld1rqb z3.b, p7/z, [a_ptr3]\n"
- "ld1rqb z4.b, p7/z, [a_ptr4]\n"
- "addvl c_ptr2, c_ptr2, #1\n"
- "udot z25.s, z16.b, z1.b[0]\n"
- "st1w z27.s, p7, [c_ptr3]\n"
- "mov z27.s, #0\n"
- "ld1rqb z5.b, p7/z, [a_ptr5]\n"
- "udot z26.s, z16.b, z2.b[0]\n"
- "ld1rqb z6.b, p7/z, [a_ptr6]\n"
- "udot z24.s, z17.b, z0.b[1]\n"
- "st1w z28.s, p7, [c_ptr4]\n"
- "mov z28.s, #0\n"
- "ld1rqb z7.b, p7/z, [a_ptr7]\n"
- "udot z27.s, z16.b, z3.b[0]\n"
- "addvl c_ptr3, c_ptr3, #1\n"
- "udot z25.s, z17.b, z1.b[1]\n"
- "st1w z29.s, p7, [c_ptr5]\n"
- "mov z29.s, #0\n"
- "addvl c_ptr4, c_ptr4, #1\n"
- "udot z28.s, z16.b, z4.b[0]\n"
"st1w z30.s, p7, [c_ptr6]\n"
"mov z30.s, #0\n"
- "addvl c_ptr5, c_ptr5, #1\n"
- "udot z29.s, z16.b, z5.b[0]\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "udot z25.s, z16.b, z1.b[0]\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "udot z26.s, z16.b, z2.b[0]\n"
"st1w z31.s, p7, [c_ptr7]\n"
"mov z31.s, #0\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "udot z27.s, z16.b, z3.b[0]\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "udot z28.s, z16.b, z4.b[0]\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "udot z24.s, z17.b, z0.b[1]\n"
"addvl c_ptr6, c_ptr6, #1\n"
- "udot z30.s, z16.b, z6.b[0]\n"
+ "udot z29.s, z16.b, z5.b[0]\n"
"addvl c_ptr7, c_ptr7, #1\n"
+ "udot z30.s, z16.b, z6.b[0]\n"
"udot z31.s, z16.b, z7.b[0]\n"
"ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "udot z25.s, z17.b, z1.b[1]\n"
"udot z26.s, z17.b, z2.b[1]\n"
"udot z27.s, z17.b, z3.b[1]\n"
"udot z28.s, z17.b, z4.b[1]\n"
@@ -5411,23 +6245,168 @@
"udot z29.s, z20.b, z5.b[0]\n"
"udot z30.s, z20.b, z6.b[0]\n"
"udot z31.s, z20.b, z7.b[0]\n"
+ "b 5f\n"
"2:\n"
+ "mov z24.s, #0\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "mov z25.s, #0\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "mov z26.s, #0\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2]\n"
+ "mov z27.s, #0\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "mov z28.s, #0\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "mov z29.s, #0\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "mov z30.s, #0\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "mov z31.s, #0\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "udot z24.s, z16.b, z0.b[0]\n"
+ "udot z25.s, z16.b, z1.b[0]\n"
+ "udot z26.s, z16.b, z2.b[0]\n"
+ "udot z27.s, z16.b, z3.b[0]\n"
+ "udot z28.s, z16.b, z4.b[0]\n"
+ "udot z29.s, z16.b, z5.b[0]\n"
+ "udot z30.s, z16.b, z6.b[0]\n"
+ "udot z31.s, z16.b, z7.b[0]\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "udot z24.s, z17.b, z0.b[1]\n"
+ "udot z25.s, z17.b, z1.b[1]\n"
+ "udot z26.s, z17.b, z2.b[1]\n"
+ "udot z27.s, z17.b, z3.b[1]\n"
+ "udot z28.s, z17.b, z4.b[1]\n"
+ "udot z29.s, z17.b, z5.b[1]\n"
+ "udot z30.s, z17.b, z6.b[1]\n"
+ "udot z31.s, z17.b, z7.b[1]\n"
+ "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "udot z24.s, z18.b, z0.b[2]\n"
+ "udot z25.s, z18.b, z1.b[2]\n"
+ "udot z26.s, z18.b, z2.b[2]\n"
+ "udot z27.s, z18.b, z3.b[2]\n"
+ "udot z28.s, z18.b, z4.b[2]\n"
+ "udot z29.s, z18.b, z5.b[2]\n"
+ "udot z30.s, z18.b, z6.b[2]\n"
+ "udot z31.s, z18.b, z7.b[2]\n"
+ "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "udot z24.s, z19.b, z0.b[3]\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x10]\n"
+ "udot z25.s, z19.b, z1.b[3]\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1, #0x10]\n"
+ "udot z26.s, z19.b, z2.b[3]\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2, #0x10]\n"
+ "udot z27.s, z19.b, z3.b[3]\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3, #0x10]\n"
+ "udot z28.s, z19.b, z4.b[3]\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4, #0x10]\n"
+ "udot z29.s, z19.b, z5.b[3]\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5, #0x10]\n"
+ "udot z30.s, z19.b, z6.b[3]\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6, #0x10]\n"
+ "udot z31.s, z19.b, z7.b[3]\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7, #0x10]\n"
+ "udot z24.s, z20.b, z0.b[0]\n"
+ "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "udot z25.s, z20.b, z1.b[0]\n"
+ "udot z26.s, z20.b, z2.b[0]\n"
+ "udot z27.s, z20.b, z3.b[0]\n"
+ "udot z28.s, z20.b, z4.b[0]\n"
+ "udot z29.s, z20.b, z5.b[0]\n"
+ "udot z30.s, z20.b, z6.b[0]\n"
+ "udot z31.s, z20.b, z7.b[0]\n"
+ "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "udot z24.s, z21.b, z0.b[1]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #5\n"
+ "udot z25.s, z21.b, z1.b[1]\n"
+ "udot z26.s, z21.b, z2.b[1]\n"
+ "udot z27.s, z21.b, z3.b[1]\n"
+ "udot z28.s, z21.b, z4.b[1]\n"
+ "udot z29.s, z21.b, z5.b[1]\n"
+ "udot z30.s, z21.b, z6.b[1]\n"
+ "udot z31.s, z21.b, z7.b[1]\n"
+ "udot z24.s, z22.b, z0.b[2]\n"
+ "udot z25.s, z22.b, z1.b[2]\n"
+ "udot z26.s, z22.b, z2.b[2]\n"
+ "udot z27.s, z22.b, z3.b[2]\n"
+ "udot z28.s, z22.b, z4.b[2]\n"
+ "udot z29.s, z22.b, z5.b[2]\n"
+ "udot z30.s, z22.b, z6.b[2]\n"
+ "udot z31.s, z22.b, z7.b[2]\n"
+ "udot z24.s, z23.b, z0.b[3]\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x20]\n"
+ "udot z25.s, z23.b, z1.b[3]\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1, #0x20]\n"
+ "udot z26.s, z23.b, z2.b[3]\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2, #0x20]\n"
+ "udot z27.s, z23.b, z3.b[3]\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3, #0x20]\n"
+ "udot z28.s, z23.b, z4.b[3]\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4, #0x20]\n"
+ "udot z29.s, z23.b, z5.b[3]\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5, #0x20]\n"
+ "udot z30.s, z23.b, z6.b[3]\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6, #0x20]\n"
+ "udot z31.s, z23.b, z7.b[3]\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7, #0x20]\n"
+ "udot z24.s, z16.b, z0.b[0]\n"
+ "udot z25.s, z16.b, z1.b[0]\n"
+ "udot z26.s, z16.b, z2.b[0]\n"
+ "udot z27.s, z16.b, z3.b[0]\n"
+ "udot z28.s, z16.b, z4.b[0]\n"
+ "udot z29.s, z16.b, z5.b[0]\n"
+ "udot z30.s, z16.b, z6.b[0]\n"
+ "udot z31.s, z16.b, z7.b[0]\n"
+ "udot z24.s, z17.b, z0.b[1]\n"
+ "udot z25.s, z17.b, z1.b[1]\n"
+ "udot z26.s, z17.b, z2.b[1]\n"
+ "udot z27.s, z17.b, z3.b[1]\n"
+ "udot z28.s, z17.b, z4.b[1]\n"
+ "udot z29.s, z17.b, z5.b[1]\n"
+ "udot z30.s, z17.b, z6.b[1]\n"
+ "udot z31.s, z17.b, z7.b[1]\n"
+ "udot z24.s, z18.b, z0.b[2]\n"
+ "udot z25.s, z18.b, z1.b[2]\n"
+ "udot z26.s, z18.b, z2.b[2]\n"
+ "udot z27.s, z18.b, z3.b[2]\n"
+ "udot z28.s, z18.b, z4.b[2]\n"
+ "udot z29.s, z18.b, z5.b[2]\n"
+ "udot z30.s, z18.b, z6.b[2]\n"
+ "udot z31.s, z18.b, z7.b[2]\n"
+ "udot z24.s, z19.b, z0.b[3]\n"
+ "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x30]\n"
+ "udot z25.s, z19.b, z1.b[3]\n"
+ "ld1rqb z1.b, p6/z, [a_ptr1, #0x30]\n"
+ "udot z26.s, z19.b, z2.b[3]\n"
+ "ld1rqb z2.b, p6/z, [a_ptr2, #0x30]\n"
+ "udot z27.s, z19.b, z3.b[3]\n"
+ "ld1rqb z3.b, p6/z, [a_ptr3, #0x30]\n"
+ "udot z28.s, z19.b, z4.b[3]\n"
+ "ld1rqb z4.b, p6/z, [a_ptr4, #0x30]\n"
+ "udot z29.s, z19.b, z5.b[3]\n"
+ "ld1rqb z5.b, p6/z, [a_ptr5, #0x30]\n"
+ "udot z30.s, z19.b, z6.b[3]\n"
+ "ld1rqb z6.b, p6/z, [a_ptr6, #0x30]\n"
+ "udot z31.s, z19.b, z7.b[3]\n"
+ "ld1rqb z7.b, p6/z, [a_ptr7, #0x30]\n"
+ "udot z24.s, z20.b, z0.b[0]\n"
+ "udot z25.s, z20.b, z1.b[0]\n"
+ "udot z26.s, z20.b, z2.b[0]\n"
+ "udot z27.s, z20.b, z3.b[0]\n"
+ "udot z28.s, z20.b, z4.b[0]\n"
+ "udot z29.s, z20.b, z5.b[0]\n"
+ "udot z30.s, z20.b, z6.b[0]\n"
+ "udot z31.s, z20.b, z7.b[0]\n"
+ "5:\n"
"st1w z24.s, p0, [%[c_ptr0]]\n"
"addvl %[c_ptr0], %[c_ptr0], #1\n"
"st1w z25.s, p0, [c_ptr1]\n"
- "addvl c_ptr1, c_ptr1, #1\n"
"st1w z26.s, p0, [c_ptr2]\n"
- "addvl c_ptr2, c_ptr2, #1\n"
"st1w z27.s, p0, [c_ptr3]\n"
- "addvl c_ptr3, c_ptr3, #1\n"
"st1w z28.s, p0, [c_ptr4]\n"
- "addvl c_ptr4, c_ptr4, #1\n"
"st1w z29.s, p0, [c_ptr5]\n"
- "addvl c_ptr5, c_ptr5, #1\n"
"st1w z30.s, p0, [c_ptr6]\n"
- "addvl c_ptr6, c_ptr6, #1\n"
"st1w z31.s, p0, [c_ptr7]\n"
- "addvl c_ptr7, c_ptr7, #1\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -5506,52 +6485,54 @@
"add c_ptr1, %[c_ptr0], #0x0\n"
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
- "mov z24.s, #0\n"
"ptrue p7.b\n"
- "mov z25.s, #0\n"
"whilelt p6.b, %[temp], %[odd_depth]\n"
- "mov z26.s, #0\n"
+ "whilelt p0.s, %[temp], %[last_width]\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
"ld1b z16.b, p7/z, [%[b_ptr0]]\n"
- "mov z27.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
"ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "mov z28.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
"ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "mov z29.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
"ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "mov z30.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
"ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "mov z31.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
"ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
"ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "whilelt p0.s, %[temp], %[last_width]\n"
"ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
- "ld1rqb z1.b, p7/z, [a_ptr1]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
- "udot z24.s, z16.b, z0.b[0]\n"
- "ld1rqb z2.b, p7/z, [a_ptr2]\n"
- "ld1rqb z3.b, p7/z, [a_ptr3]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
- "udot z25.s, z16.b, z1.b[0]\n"
- "ld1rqb z4.b, p7/z, [a_ptr4]\n"
- "udot z26.s, z16.b, z2.b[0]\n"
- "ld1rqb z5.b, p7/z, [a_ptr5]\n"
- "udot z27.s, z16.b, z3.b[0]\n"
- "ld1rqb z6.b, p7/z, [a_ptr6]\n"
- "udot z24.s, z17.b, z0.b[1]\n"
- "ld1rqb z7.b, p7/z, [a_ptr7]\n"
- "udot z28.s, z16.b, z4.b[0]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- "udot z29.s, z16.b, z5.b[0]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- "udot z30.s, z16.b, z6.b[0]\n"
"addvl %[b_ptr0], %[b_ptr0], #8\n"
+ "cbz %[loops], 2f\n"
+ "mov z24.s, #0\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "mov z25.s, #0\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "mov z26.s, #0\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2]\n"
+ "mov z27.s, #0\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "mov z28.s, #0\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "mov z29.s, #0\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "mov z30.s, #0\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "mov z31.s, #0\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "udot z24.s, z16.b, z0.b[0]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "udot z25.s, z16.b, z1.b[0]\n"
+ "udot z26.s, z16.b, z2.b[0]\n"
+ "udot z27.s, z16.b, z3.b[0]\n"
+ "udot z28.s, z16.b, z4.b[0]\n"
+ "udot z29.s, z16.b, z5.b[0]\n"
+ "udot z30.s, z16.b, z6.b[0]\n"
"udot z31.s, z16.b, z7.b[0]\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "udot z24.s, z17.b, z0.b[1]\n"
"udot z25.s, z17.b, z1.b[1]\n"
"udot z26.s, z17.b, z2.b[1]\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
"udot z27.s, z17.b, z3.b[1]\n"
"udot z28.s, z17.b, z4.b[1]\n"
"udot z29.s, z17.b, z5.b[1]\n"
@@ -5683,82 +6664,80 @@
"udot z29.s, z21.b, z5.b[1]\n"
"udot z30.s, z21.b, z6.b[1]\n"
"udot z31.s, z21.b, z7.b[1]\n"
- "cbz %[loops], 2f\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"b.eq 3f\n"
"4:\n"
"st1w z24.s, p7, [%[c_ptr0]]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
+ "subs %[loops], %[loops], #0x1\n"
"mov z24.s, #0\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "addvl %[c_ptr0], %[c_ptr0], #1\n"
+ "st1w z25.s, p7, [c_ptr1]\n"
+ "addvl c_ptr1, c_ptr1, #1\n"
+ "mov z25.s, #0\n"
+ "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "st1w z26.s, p7, [c_ptr2]\n"
+ "addvl c_ptr2, c_ptr2, #1\n"
+ "mov z26.s, #0\n"
+ "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ "st1w z27.s, p7, [c_ptr3]\n"
+ "addvl c_ptr3, c_ptr3, #1\n"
+ "mov z27.s, #0\n"
+ "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #8\n"
+ "st1w z28.s, p7, [c_ptr4]\n"
+ "addvl c_ptr4, c_ptr4, #1\n"
+ "mov z28.s, #0\n"
"ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
"ld1rqb z1.b, p7/z, [a_ptr1]\n"
- "subs %[loops], %[loops], #0x1\n"
- "st1w z25.s, p7, [c_ptr1]\n"
- "addvl %[c_ptr0], %[c_ptr0], #1\n"
- "mov z25.s, #0\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ "st1w z29.s, p7, [c_ptr5]\n"
+ "addvl c_ptr5, c_ptr5, #1\n"
+ "mov z29.s, #0\n"
"ld1rqb z2.b, p7/z, [a_ptr2]\n"
"udot z24.s, z16.b, z0.b[0]\n"
- "st1w z26.s, p7, [c_ptr2]\n"
- "mov z26.s, #0\n"
- "ld1rqb z3.b, p7/z, [a_ptr3]\n"
- "ld1rqb z4.b, p7/z, [a_ptr4]\n"
- "addvl c_ptr1, c_ptr1, #1\n"
- "udot z25.s, z16.b, z1.b[0]\n"
- "st1w z27.s, p7, [c_ptr3]\n"
- "mov z27.s, #0\n"
- "ld1rqb z5.b, p7/z, [a_ptr5]\n"
- "udot z26.s, z16.b, z2.b[0]\n"
- "ld1rqb z6.b, p7/z, [a_ptr6]\n"
- "udot z24.s, z17.b, z0.b[1]\n"
- "st1w z28.s, p7, [c_ptr4]\n"
- "mov z28.s, #0\n"
- "ld1rqb z7.b, p7/z, [a_ptr7]\n"
- "udot z27.s, z16.b, z3.b[0]\n"
- "addvl c_ptr2, c_ptr2, #1\n"
- "udot z25.s, z17.b, z1.b[1]\n"
- "st1w z29.s, p7, [c_ptr5]\n"
- "mov z29.s, #0\n"
- "addvl c_ptr3, c_ptr3, #1\n"
- "udot z28.s, z16.b, z4.b[0]\n"
"st1w z30.s, p7, [c_ptr6]\n"
"mov z30.s, #0\n"
- "addvl c_ptr4, c_ptr4, #1\n"
- "udot z29.s, z16.b, z5.b[0]\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "udot z25.s, z16.b, z1.b[0]\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "udot z26.s, z16.b, z2.b[0]\n"
"st1w z31.s, p7, [c_ptr7]\n"
"mov z31.s, #0\n"
- "addvl c_ptr5, c_ptr5, #1\n"
- "udot z30.s, z16.b, z6.b[0]\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "udot z27.s, z16.b, z3.b[0]\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "udot z28.s, z16.b, z4.b[0]\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "udot z24.s, z17.b, z0.b[1]\n"
"addvl c_ptr6, c_ptr6, #1\n"
+ "udot z29.s, z16.b, z5.b[0]\n"
+ "addvl c_ptr7, c_ptr7, #1\n"
+ "udot z30.s, z16.b, z6.b[0]\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
"udot z31.s, z16.b, z7.b[0]\n"
"ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "udot z25.s, z17.b, z1.b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
"udot z26.s, z17.b, z2.b[1]\n"
- "addvl c_ptr7, c_ptr7, #1\n"
+ "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
"udot z27.s, z17.b, z3.b[1]\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
"udot z28.s, z17.b, z4.b[1]\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ "prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
"udot z29.s, z17.b, z5.b[1]\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
"udot z30.s, z17.b, z6.b[1]\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
"udot z31.s, z17.b, z7.b[1]\n"
"ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"udot z24.s, z18.b, z0.b[2]\n"
- "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
"udot z25.s, z18.b, z1.b[2]\n"
- "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
"udot z26.s, z18.b, z2.b[2]\n"
- "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
"udot z27.s, z18.b, z3.b[2]\n"
- "prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
"udot z28.s, z18.b, z4.b[2]\n"
"udot z29.s, z18.b, z5.b[2]\n"
"udot z30.s, z18.b, z6.b[2]\n"
@@ -5808,7 +6787,6 @@
"udot z29.s, z22.b, z5.b[2]\n"
"udot z30.s, z22.b, z6.b[2]\n"
"udot z31.s, z22.b, z7.b[2]\n"
- "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
"udot z24.s, z23.b, z0.b[3]\n"
"ld1rqb z0.b, p7/z, [%[a_ptr0], #0x20]\n"
"udot z25.s, z23.b, z1.b[3]\n"
@@ -5826,7 +6804,6 @@
"udot z31.s, z23.b, z7.b[3]\n"
"ld1rqb z7.b, p7/z, [a_ptr7, #0x20]\n"
"udot z24.s, z16.b, z0.b[0]\n"
- "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"udot z25.s, z16.b, z1.b[0]\n"
"udot z26.s, z16.b, z2.b[0]\n"
"udot z27.s, z16.b, z3.b[0]\n"
@@ -5834,7 +6811,6 @@
"udot z29.s, z16.b, z5.b[0]\n"
"udot z30.s, z16.b, z6.b[0]\n"
"udot z31.s, z16.b, z7.b[0]\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
"udot z24.s, z17.b, z0.b[1]\n"
"udot z25.s, z17.b, z1.b[1]\n"
"udot z26.s, z17.b, z2.b[1]\n"
@@ -5843,7 +6819,6 @@
"udot z29.s, z17.b, z5.b[1]\n"
"udot z30.s, z17.b, z6.b[1]\n"
"udot z31.s, z17.b, z7.b[1]\n"
- "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"udot z24.s, z18.b, z0.b[2]\n"
"udot z25.s, z18.b, z1.b[2]\n"
"udot z26.s, z18.b, z2.b[2]\n"
@@ -5852,7 +6827,6 @@
"udot z29.s, z18.b, z5.b[2]\n"
"udot z30.s, z18.b, z6.b[2]\n"
"udot z31.s, z18.b, z7.b[2]\n"
- "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"udot z24.s, z19.b, z0.b[3]\n"
"ld1rqb z0.b, p6/z, [%[a_ptr0], #0x30]\n"
"udot z25.s, z19.b, z1.b[3]\n"
@@ -5870,7 +6844,6 @@
"udot z31.s, z19.b, z7.b[3]\n"
"ld1rqb z7.b, p6/z, [a_ptr7, #0x30]\n"
"udot z24.s, z20.b, z0.b[0]\n"
- "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"udot z25.s, z20.b, z1.b[0]\n"
"udot z26.s, z20.b, z2.b[0]\n"
"udot z27.s, z20.b, z3.b[0]\n"
@@ -5878,7 +6851,6 @@
"udot z29.s, z20.b, z5.b[0]\n"
"udot z30.s, z20.b, z6.b[0]\n"
"udot z31.s, z20.b, z7.b[0]\n"
- "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
"udot z24.s, z21.b, z0.b[1]\n"
"udot z25.s, z21.b, z1.b[1]\n"
"udot z26.s, z21.b, z2.b[1]\n"
@@ -5887,53 +6859,60 @@
"udot z29.s, z21.b, z5.b[1]\n"
"udot z30.s, z21.b, z6.b[1]\n"
"udot z31.s, z21.b, z7.b[1]\n"
- "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
"b.ne 4b\n"
"3:\n"
"st1w z24.s, p7, [%[c_ptr0]]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "mov z24.s, #0\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "ld1rqb z1.b, p7/z, [a_ptr1]\n"
"addvl %[c_ptr0], %[c_ptr0], #1\n"
+ "mov z24.s, #0\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"st1w z25.s, p7, [c_ptr1]\n"
"addvl c_ptr1, c_ptr1, #1\n"
"mov z25.s, #0\n"
+ "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "st1w z26.s, p7, [c_ptr2]\n"
+ "addvl c_ptr2, c_ptr2, #1\n"
+ "mov z26.s, #0\n"
+ "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "st1w z27.s, p7, [c_ptr3]\n"
+ "addvl c_ptr3, c_ptr3, #1\n"
+ "mov z27.s, #0\n"
+ "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #8\n"
+ "st1w z28.s, p7, [c_ptr4]\n"
+ "addvl c_ptr4, c_ptr4, #1\n"
+ "mov z28.s, #0\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "st1w z29.s, p7, [c_ptr5]\n"
+ "addvl c_ptr5, c_ptr5, #1\n"
+ "mov z29.s, #0\n"
"ld1rqb z2.b, p7/z, [a_ptr2]\n"
"udot z24.s, z16.b, z0.b[0]\n"
- "st1w z26.s, p7, [c_ptr2]\n"
- "mov z26.s, #0\n"
- "ld1rqb z3.b, p7/z, [a_ptr3]\n"
- "ld1rqb z4.b, p7/z, [a_ptr4]\n"
- "addvl c_ptr2, c_ptr2, #1\n"
- "udot z25.s, z16.b, z1.b[0]\n"
- "st1w z27.s, p7, [c_ptr3]\n"
- "mov z27.s, #0\n"
- "ld1rqb z5.b, p7/z, [a_ptr5]\n"
- "udot z26.s, z16.b, z2.b[0]\n"
- "ld1rqb z6.b, p7/z, [a_ptr6]\n"
- "udot z24.s, z17.b, z0.b[1]\n"
- "st1w z28.s, p7, [c_ptr4]\n"
- "mov z28.s, #0\n"
- "ld1rqb z7.b, p7/z, [a_ptr7]\n"
- "udot z27.s, z16.b, z3.b[0]\n"
- "addvl c_ptr3, c_ptr3, #1\n"
- "udot z25.s, z17.b, z1.b[1]\n"
- "st1w z29.s, p7, [c_ptr5]\n"
- "mov z29.s, #0\n"
- "addvl c_ptr4, c_ptr4, #1\n"
- "udot z28.s, z16.b, z4.b[0]\n"
"st1w z30.s, p7, [c_ptr6]\n"
"mov z30.s, #0\n"
- "addvl c_ptr5, c_ptr5, #1\n"
- "udot z29.s, z16.b, z5.b[0]\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "udot z25.s, z16.b, z1.b[0]\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "udot z26.s, z16.b, z2.b[0]\n"
"st1w z31.s, p7, [c_ptr7]\n"
"mov z31.s, #0\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "udot z27.s, z16.b, z3.b[0]\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "udot z28.s, z16.b, z4.b[0]\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "udot z24.s, z17.b, z0.b[1]\n"
"addvl c_ptr6, c_ptr6, #1\n"
- "udot z30.s, z16.b, z6.b[0]\n"
+ "udot z29.s, z16.b, z5.b[0]\n"
"addvl c_ptr7, c_ptr7, #1\n"
+ "udot z30.s, z16.b, z6.b[0]\n"
"udot z31.s, z16.b, z7.b[0]\n"
"ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "udot z25.s, z17.b, z1.b[1]\n"
"udot z26.s, z17.b, z2.b[1]\n"
"udot z27.s, z17.b, z3.b[1]\n"
"udot z28.s, z17.b, z4.b[1]\n"
@@ -6066,23 +7045,177 @@
"udot z29.s, z21.b, z5.b[1]\n"
"udot z30.s, z21.b, z6.b[1]\n"
"udot z31.s, z21.b, z7.b[1]\n"
+ "b 5f\n"
"2:\n"
+ "mov z24.s, #0\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "mov z25.s, #0\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "mov z26.s, #0\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2]\n"
+ "mov z27.s, #0\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "mov z28.s, #0\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "mov z29.s, #0\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "mov z30.s, #0\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "mov z31.s, #0\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "udot z24.s, z16.b, z0.b[0]\n"
+ "udot z25.s, z16.b, z1.b[0]\n"
+ "udot z26.s, z16.b, z2.b[0]\n"
+ "udot z27.s, z16.b, z3.b[0]\n"
+ "udot z28.s, z16.b, z4.b[0]\n"
+ "udot z29.s, z16.b, z5.b[0]\n"
+ "udot z30.s, z16.b, z6.b[0]\n"
+ "udot z31.s, z16.b, z7.b[0]\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "udot z24.s, z17.b, z0.b[1]\n"
+ "udot z25.s, z17.b, z1.b[1]\n"
+ "udot z26.s, z17.b, z2.b[1]\n"
+ "udot z27.s, z17.b, z3.b[1]\n"
+ "udot z28.s, z17.b, z4.b[1]\n"
+ "udot z29.s, z17.b, z5.b[1]\n"
+ "udot z30.s, z17.b, z6.b[1]\n"
+ "udot z31.s, z17.b, z7.b[1]\n"
+ "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "udot z24.s, z18.b, z0.b[2]\n"
+ "udot z25.s, z18.b, z1.b[2]\n"
+ "udot z26.s, z18.b, z2.b[2]\n"
+ "udot z27.s, z18.b, z3.b[2]\n"
+ "udot z28.s, z18.b, z4.b[2]\n"
+ "udot z29.s, z18.b, z5.b[2]\n"
+ "udot z30.s, z18.b, z6.b[2]\n"
+ "udot z31.s, z18.b, z7.b[2]\n"
+ "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "udot z24.s, z19.b, z0.b[3]\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x10]\n"
+ "udot z25.s, z19.b, z1.b[3]\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1, #0x10]\n"
+ "udot z26.s, z19.b, z2.b[3]\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2, #0x10]\n"
+ "udot z27.s, z19.b, z3.b[3]\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3, #0x10]\n"
+ "udot z28.s, z19.b, z4.b[3]\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4, #0x10]\n"
+ "udot z29.s, z19.b, z5.b[3]\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5, #0x10]\n"
+ "udot z30.s, z19.b, z6.b[3]\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6, #0x10]\n"
+ "udot z31.s, z19.b, z7.b[3]\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7, #0x10]\n"
+ "udot z24.s, z20.b, z0.b[0]\n"
+ "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "udot z25.s, z20.b, z1.b[0]\n"
+ "udot z26.s, z20.b, z2.b[0]\n"
+ "udot z27.s, z20.b, z3.b[0]\n"
+ "udot z28.s, z20.b, z4.b[0]\n"
+ "udot z29.s, z20.b, z5.b[0]\n"
+ "udot z30.s, z20.b, z6.b[0]\n"
+ "udot z31.s, z20.b, z7.b[0]\n"
+ "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "udot z24.s, z21.b, z0.b[1]\n"
+ "udot z25.s, z21.b, z1.b[1]\n"
+ "udot z26.s, z21.b, z2.b[1]\n"
+ "udot z27.s, z21.b, z3.b[1]\n"
+ "udot z28.s, z21.b, z4.b[1]\n"
+ "udot z29.s, z21.b, z5.b[1]\n"
+ "udot z30.s, z21.b, z6.b[1]\n"
+ "udot z31.s, z21.b, z7.b[1]\n"
+ "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "udot z24.s, z22.b, z0.b[2]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #6\n"
+ "udot z25.s, z22.b, z1.b[2]\n"
+ "udot z26.s, z22.b, z2.b[2]\n"
+ "udot z27.s, z22.b, z3.b[2]\n"
+ "udot z28.s, z22.b, z4.b[2]\n"
+ "udot z29.s, z22.b, z5.b[2]\n"
+ "udot z30.s, z22.b, z6.b[2]\n"
+ "udot z31.s, z22.b, z7.b[2]\n"
+ "udot z24.s, z23.b, z0.b[3]\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x20]\n"
+ "udot z25.s, z23.b, z1.b[3]\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1, #0x20]\n"
+ "udot z26.s, z23.b, z2.b[3]\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2, #0x20]\n"
+ "udot z27.s, z23.b, z3.b[3]\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3, #0x20]\n"
+ "udot z28.s, z23.b, z4.b[3]\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4, #0x20]\n"
+ "udot z29.s, z23.b, z5.b[3]\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5, #0x20]\n"
+ "udot z30.s, z23.b, z6.b[3]\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6, #0x20]\n"
+ "udot z31.s, z23.b, z7.b[3]\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7, #0x20]\n"
+ "udot z24.s, z16.b, z0.b[0]\n"
+ "udot z25.s, z16.b, z1.b[0]\n"
+ "udot z26.s, z16.b, z2.b[0]\n"
+ "udot z27.s, z16.b, z3.b[0]\n"
+ "udot z28.s, z16.b, z4.b[0]\n"
+ "udot z29.s, z16.b, z5.b[0]\n"
+ "udot z30.s, z16.b, z6.b[0]\n"
+ "udot z31.s, z16.b, z7.b[0]\n"
+ "udot z24.s, z17.b, z0.b[1]\n"
+ "udot z25.s, z17.b, z1.b[1]\n"
+ "udot z26.s, z17.b, z2.b[1]\n"
+ "udot z27.s, z17.b, z3.b[1]\n"
+ "udot z28.s, z17.b, z4.b[1]\n"
+ "udot z29.s, z17.b, z5.b[1]\n"
+ "udot z30.s, z17.b, z6.b[1]\n"
+ "udot z31.s, z17.b, z7.b[1]\n"
+ "udot z24.s, z18.b, z0.b[2]\n"
+ "udot z25.s, z18.b, z1.b[2]\n"
+ "udot z26.s, z18.b, z2.b[2]\n"
+ "udot z27.s, z18.b, z3.b[2]\n"
+ "udot z28.s, z18.b, z4.b[2]\n"
+ "udot z29.s, z18.b, z5.b[2]\n"
+ "udot z30.s, z18.b, z6.b[2]\n"
+ "udot z31.s, z18.b, z7.b[2]\n"
+ "udot z24.s, z19.b, z0.b[3]\n"
+ "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x30]\n"
+ "udot z25.s, z19.b, z1.b[3]\n"
+ "ld1rqb z1.b, p6/z, [a_ptr1, #0x30]\n"
+ "udot z26.s, z19.b, z2.b[3]\n"
+ "ld1rqb z2.b, p6/z, [a_ptr2, #0x30]\n"
+ "udot z27.s, z19.b, z3.b[3]\n"
+ "ld1rqb z3.b, p6/z, [a_ptr3, #0x30]\n"
+ "udot z28.s, z19.b, z4.b[3]\n"
+ "ld1rqb z4.b, p6/z, [a_ptr4, #0x30]\n"
+ "udot z29.s, z19.b, z5.b[3]\n"
+ "ld1rqb z5.b, p6/z, [a_ptr5, #0x30]\n"
+ "udot z30.s, z19.b, z6.b[3]\n"
+ "ld1rqb z6.b, p6/z, [a_ptr6, #0x30]\n"
+ "udot z31.s, z19.b, z7.b[3]\n"
+ "ld1rqb z7.b, p6/z, [a_ptr7, #0x30]\n"
+ "udot z24.s, z20.b, z0.b[0]\n"
+ "udot z25.s, z20.b, z1.b[0]\n"
+ "udot z26.s, z20.b, z2.b[0]\n"
+ "udot z27.s, z20.b, z3.b[0]\n"
+ "udot z28.s, z20.b, z4.b[0]\n"
+ "udot z29.s, z20.b, z5.b[0]\n"
+ "udot z30.s, z20.b, z6.b[0]\n"
+ "udot z31.s, z20.b, z7.b[0]\n"
+ "udot z24.s, z21.b, z0.b[1]\n"
+ "udot z25.s, z21.b, z1.b[1]\n"
+ "udot z26.s, z21.b, z2.b[1]\n"
+ "udot z27.s, z21.b, z3.b[1]\n"
+ "udot z28.s, z21.b, z4.b[1]\n"
+ "udot z29.s, z21.b, z5.b[1]\n"
+ "udot z30.s, z21.b, z6.b[1]\n"
+ "udot z31.s, z21.b, z7.b[1]\n"
+ "5:\n"
"st1w z24.s, p0, [%[c_ptr0]]\n"
"addvl %[c_ptr0], %[c_ptr0], #1\n"
"st1w z25.s, p0, [c_ptr1]\n"
- "addvl c_ptr1, c_ptr1, #1\n"
"st1w z26.s, p0, [c_ptr2]\n"
- "addvl c_ptr2, c_ptr2, #1\n"
"st1w z27.s, p0, [c_ptr3]\n"
- "addvl c_ptr3, c_ptr3, #1\n"
"st1w z28.s, p0, [c_ptr4]\n"
- "addvl c_ptr4, c_ptr4, #1\n"
"st1w z29.s, p0, [c_ptr5]\n"
- "addvl c_ptr5, c_ptr5, #1\n"
"st1w z30.s, p0, [c_ptr6]\n"
- "addvl c_ptr6, c_ptr6, #1\n"
"st1w z31.s, p0, [c_ptr7]\n"
- "addvl c_ptr7, c_ptr7, #1\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -6161,52 +7294,54 @@
"add c_ptr1, %[c_ptr0], #0x0\n"
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
- "mov z24.s, #0\n"
"ptrue p7.b\n"
- "mov z25.s, #0\n"
"whilelt p6.b, %[temp], %[odd_depth]\n"
- "mov z26.s, #0\n"
+ "whilelt p0.s, %[temp], %[last_width]\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
"ld1b z16.b, p7/z, [%[b_ptr0]]\n"
- "mov z27.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
"ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "mov z28.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
"ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "mov z29.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
"ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "mov z30.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
"ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "mov z31.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
"ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
"ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "whilelt p0.s, %[temp], %[last_width]\n"
"ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
- "ld1rqb z1.b, p7/z, [a_ptr1]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
- "udot z24.s, z16.b, z0.b[0]\n"
- "ld1rqb z2.b, p7/z, [a_ptr2]\n"
- "ld1rqb z3.b, p7/z, [a_ptr3]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
- "udot z25.s, z16.b, z1.b[0]\n"
- "ld1rqb z4.b, p7/z, [a_ptr4]\n"
- "udot z26.s, z16.b, z2.b[0]\n"
- "ld1rqb z5.b, p7/z, [a_ptr5]\n"
- "udot z27.s, z16.b, z3.b[0]\n"
- "ld1rqb z6.b, p7/z, [a_ptr6]\n"
- "udot z24.s, z17.b, z0.b[1]\n"
- "ld1rqb z7.b, p7/z, [a_ptr7]\n"
- "udot z28.s, z16.b, z4.b[0]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- "udot z29.s, z16.b, z5.b[0]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- "udot z30.s, z16.b, z6.b[0]\n"
"addvl %[b_ptr0], %[b_ptr0], #8\n"
+ "cbz %[loops], 2f\n"
+ "mov z24.s, #0\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "mov z25.s, #0\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "mov z26.s, #0\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2]\n"
+ "mov z27.s, #0\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "mov z28.s, #0\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "mov z29.s, #0\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "mov z30.s, #0\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "mov z31.s, #0\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "udot z24.s, z16.b, z0.b[0]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "udot z25.s, z16.b, z1.b[0]\n"
+ "udot z26.s, z16.b, z2.b[0]\n"
+ "udot z27.s, z16.b, z3.b[0]\n"
+ "udot z28.s, z16.b, z4.b[0]\n"
+ "udot z29.s, z16.b, z5.b[0]\n"
+ "udot z30.s, z16.b, z6.b[0]\n"
"udot z31.s, z16.b, z7.b[0]\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "udot z24.s, z17.b, z0.b[1]\n"
"udot z25.s, z17.b, z1.b[1]\n"
"udot z26.s, z17.b, z2.b[1]\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
"udot z27.s, z17.b, z3.b[1]\n"
"udot z28.s, z17.b, z4.b[1]\n"
"udot z29.s, z17.b, z5.b[1]\n"
@@ -6347,82 +7482,80 @@
"udot z29.s, z22.b, z5.b[2]\n"
"udot z30.s, z22.b, z6.b[2]\n"
"udot z31.s, z22.b, z7.b[2]\n"
- "cbz %[loops], 2f\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"b.eq 3f\n"
"4:\n"
"st1w z24.s, p7, [%[c_ptr0]]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
+ "subs %[loops], %[loops], #0x1\n"
"mov z24.s, #0\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "addvl %[c_ptr0], %[c_ptr0], #1\n"
+ "st1w z25.s, p7, [c_ptr1]\n"
+ "addvl c_ptr1, c_ptr1, #1\n"
+ "mov z25.s, #0\n"
+ "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "st1w z26.s, p7, [c_ptr2]\n"
+ "addvl c_ptr2, c_ptr2, #1\n"
+ "mov z26.s, #0\n"
+ "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ "st1w z27.s, p7, [c_ptr3]\n"
+ "addvl c_ptr3, c_ptr3, #1\n"
+ "mov z27.s, #0\n"
+ "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #8\n"
+ "st1w z28.s, p7, [c_ptr4]\n"
+ "addvl c_ptr4, c_ptr4, #1\n"
+ "mov z28.s, #0\n"
"ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
"ld1rqb z1.b, p7/z, [a_ptr1]\n"
- "subs %[loops], %[loops], #0x1\n"
- "st1w z25.s, p7, [c_ptr1]\n"
- "addvl %[c_ptr0], %[c_ptr0], #1\n"
- "mov z25.s, #0\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ "st1w z29.s, p7, [c_ptr5]\n"
+ "addvl c_ptr5, c_ptr5, #1\n"
+ "mov z29.s, #0\n"
"ld1rqb z2.b, p7/z, [a_ptr2]\n"
"udot z24.s, z16.b, z0.b[0]\n"
- "st1w z26.s, p7, [c_ptr2]\n"
- "mov z26.s, #0\n"
- "ld1rqb z3.b, p7/z, [a_ptr3]\n"
- "ld1rqb z4.b, p7/z, [a_ptr4]\n"
- "addvl c_ptr1, c_ptr1, #1\n"
- "udot z25.s, z16.b, z1.b[0]\n"
- "st1w z27.s, p7, [c_ptr3]\n"
- "mov z27.s, #0\n"
- "ld1rqb z5.b, p7/z, [a_ptr5]\n"
- "udot z26.s, z16.b, z2.b[0]\n"
- "ld1rqb z6.b, p7/z, [a_ptr6]\n"
- "udot z24.s, z17.b, z0.b[1]\n"
- "st1w z28.s, p7, [c_ptr4]\n"
- "mov z28.s, #0\n"
- "ld1rqb z7.b, p7/z, [a_ptr7]\n"
- "udot z27.s, z16.b, z3.b[0]\n"
- "addvl c_ptr2, c_ptr2, #1\n"
- "udot z25.s, z17.b, z1.b[1]\n"
- "st1w z29.s, p7, [c_ptr5]\n"
- "mov z29.s, #0\n"
- "addvl c_ptr3, c_ptr3, #1\n"
- "udot z28.s, z16.b, z4.b[0]\n"
"st1w z30.s, p7, [c_ptr6]\n"
"mov z30.s, #0\n"
- "addvl c_ptr4, c_ptr4, #1\n"
- "udot z29.s, z16.b, z5.b[0]\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "udot z25.s, z16.b, z1.b[0]\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "udot z26.s, z16.b, z2.b[0]\n"
"st1w z31.s, p7, [c_ptr7]\n"
"mov z31.s, #0\n"
- "addvl c_ptr5, c_ptr5, #1\n"
- "udot z30.s, z16.b, z6.b[0]\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "udot z27.s, z16.b, z3.b[0]\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "udot z28.s, z16.b, z4.b[0]\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "udot z24.s, z17.b, z0.b[1]\n"
"addvl c_ptr6, c_ptr6, #1\n"
+ "udot z29.s, z16.b, z5.b[0]\n"
+ "addvl c_ptr7, c_ptr7, #1\n"
+ "udot z30.s, z16.b, z6.b[0]\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
"udot z31.s, z16.b, z7.b[0]\n"
"ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "udot z25.s, z17.b, z1.b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
"udot z26.s, z17.b, z2.b[1]\n"
- "addvl c_ptr7, c_ptr7, #1\n"
+ "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
"udot z27.s, z17.b, z3.b[1]\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
"udot z28.s, z17.b, z4.b[1]\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ "prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
"udot z29.s, z17.b, z5.b[1]\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
"udot z30.s, z17.b, z6.b[1]\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
"udot z31.s, z17.b, z7.b[1]\n"
"ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"udot z24.s, z18.b, z0.b[2]\n"
- "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
"udot z25.s, z18.b, z1.b[2]\n"
- "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
"udot z26.s, z18.b, z2.b[2]\n"
- "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
"udot z27.s, z18.b, z3.b[2]\n"
- "prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
"udot z28.s, z18.b, z4.b[2]\n"
"udot z29.s, z18.b, z5.b[2]\n"
"udot z30.s, z18.b, z6.b[2]\n"
@@ -6493,12 +7626,10 @@
"udot z25.s, z16.b, z1.b[0]\n"
"udot z26.s, z16.b, z2.b[0]\n"
"udot z27.s, z16.b, z3.b[0]\n"
- "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
"udot z28.s, z16.b, z4.b[0]\n"
"udot z29.s, z16.b, z5.b[0]\n"
"udot z30.s, z16.b, z6.b[0]\n"
"udot z31.s, z16.b, z7.b[0]\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
"udot z24.s, z17.b, z0.b[1]\n"
"udot z25.s, z17.b, z1.b[1]\n"
"udot z26.s, z17.b, z2.b[1]\n"
@@ -6507,7 +7638,6 @@
"udot z29.s, z17.b, z5.b[1]\n"
"udot z30.s, z17.b, z6.b[1]\n"
"udot z31.s, z17.b, z7.b[1]\n"
- "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"udot z24.s, z18.b, z0.b[2]\n"
"udot z25.s, z18.b, z1.b[2]\n"
"udot z26.s, z18.b, z2.b[2]\n"
@@ -6516,7 +7646,6 @@
"udot z29.s, z18.b, z5.b[2]\n"
"udot z30.s, z18.b, z6.b[2]\n"
"udot z31.s, z18.b, z7.b[2]\n"
- "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"udot z24.s, z19.b, z0.b[3]\n"
"ld1rqb z0.b, p6/z, [%[a_ptr0], #0x30]\n"
"udot z25.s, z19.b, z1.b[3]\n"
@@ -6534,7 +7663,6 @@
"udot z31.s, z19.b, z7.b[3]\n"
"ld1rqb z7.b, p6/z, [a_ptr7, #0x30]\n"
"udot z24.s, z20.b, z0.b[0]\n"
- "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"udot z25.s, z20.b, z1.b[0]\n"
"udot z26.s, z20.b, z2.b[0]\n"
"udot z27.s, z20.b, z3.b[0]\n"
@@ -6542,7 +7670,6 @@
"udot z29.s, z20.b, z5.b[0]\n"
"udot z30.s, z20.b, z6.b[0]\n"
"udot z31.s, z20.b, z7.b[0]\n"
- "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
"udot z24.s, z21.b, z0.b[1]\n"
"udot z25.s, z21.b, z1.b[1]\n"
"udot z26.s, z21.b, z2.b[1]\n"
@@ -6551,7 +7678,6 @@
"udot z29.s, z21.b, z5.b[1]\n"
"udot z30.s, z21.b, z6.b[1]\n"
"udot z31.s, z21.b, z7.b[1]\n"
- "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
"udot z24.s, z22.b, z0.b[2]\n"
"udot z25.s, z22.b, z1.b[2]\n"
"udot z26.s, z22.b, z2.b[2]\n"
@@ -6560,53 +7686,60 @@
"udot z29.s, z22.b, z5.b[2]\n"
"udot z30.s, z22.b, z6.b[2]\n"
"udot z31.s, z22.b, z7.b[2]\n"
- "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
"b.ne 4b\n"
"3:\n"
"st1w z24.s, p7, [%[c_ptr0]]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "mov z24.s, #0\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "ld1rqb z1.b, p7/z, [a_ptr1]\n"
"addvl %[c_ptr0], %[c_ptr0], #1\n"
+ "mov z24.s, #0\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"st1w z25.s, p7, [c_ptr1]\n"
"addvl c_ptr1, c_ptr1, #1\n"
"mov z25.s, #0\n"
+ "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "st1w z26.s, p7, [c_ptr2]\n"
+ "addvl c_ptr2, c_ptr2, #1\n"
+ "mov z26.s, #0\n"
+ "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "st1w z27.s, p7, [c_ptr3]\n"
+ "addvl c_ptr3, c_ptr3, #1\n"
+ "mov z27.s, #0\n"
+ "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #8\n"
+ "st1w z28.s, p7, [c_ptr4]\n"
+ "addvl c_ptr4, c_ptr4, #1\n"
+ "mov z28.s, #0\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "st1w z29.s, p7, [c_ptr5]\n"
+ "addvl c_ptr5, c_ptr5, #1\n"
+ "mov z29.s, #0\n"
"ld1rqb z2.b, p7/z, [a_ptr2]\n"
"udot z24.s, z16.b, z0.b[0]\n"
- "st1w z26.s, p7, [c_ptr2]\n"
- "mov z26.s, #0\n"
- "ld1rqb z3.b, p7/z, [a_ptr3]\n"
- "ld1rqb z4.b, p7/z, [a_ptr4]\n"
- "addvl c_ptr2, c_ptr2, #1\n"
- "udot z25.s, z16.b, z1.b[0]\n"
- "st1w z27.s, p7, [c_ptr3]\n"
- "mov z27.s, #0\n"
- "ld1rqb z5.b, p7/z, [a_ptr5]\n"
- "udot z26.s, z16.b, z2.b[0]\n"
- "ld1rqb z6.b, p7/z, [a_ptr6]\n"
- "udot z24.s, z17.b, z0.b[1]\n"
- "st1w z28.s, p7, [c_ptr4]\n"
- "mov z28.s, #0\n"
- "ld1rqb z7.b, p7/z, [a_ptr7]\n"
- "udot z27.s, z16.b, z3.b[0]\n"
- "addvl c_ptr3, c_ptr3, #1\n"
- "udot z25.s, z17.b, z1.b[1]\n"
- "st1w z29.s, p7, [c_ptr5]\n"
- "mov z29.s, #0\n"
- "addvl c_ptr4, c_ptr4, #1\n"
- "udot z28.s, z16.b, z4.b[0]\n"
"st1w z30.s, p7, [c_ptr6]\n"
"mov z30.s, #0\n"
- "addvl c_ptr5, c_ptr5, #1\n"
- "udot z29.s, z16.b, z5.b[0]\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "udot z25.s, z16.b, z1.b[0]\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "udot z26.s, z16.b, z2.b[0]\n"
"st1w z31.s, p7, [c_ptr7]\n"
"mov z31.s, #0\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "udot z27.s, z16.b, z3.b[0]\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "udot z28.s, z16.b, z4.b[0]\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "udot z24.s, z17.b, z0.b[1]\n"
"addvl c_ptr6, c_ptr6, #1\n"
- "udot z30.s, z16.b, z6.b[0]\n"
+ "udot z29.s, z16.b, z5.b[0]\n"
"addvl c_ptr7, c_ptr7, #1\n"
+ "udot z30.s, z16.b, z6.b[0]\n"
"udot z31.s, z16.b, z7.b[0]\n"
"ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "udot z25.s, z17.b, z1.b[1]\n"
"udot z26.s, z17.b, z2.b[1]\n"
"udot z27.s, z17.b, z3.b[1]\n"
"udot z28.s, z17.b, z4.b[1]\n"
@@ -6748,23 +7881,186 @@
"udot z29.s, z22.b, z5.b[2]\n"
"udot z30.s, z22.b, z6.b[2]\n"
"udot z31.s, z22.b, z7.b[2]\n"
+ "b 5f\n"
"2:\n"
+ "mov z24.s, #0\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "mov z25.s, #0\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "mov z26.s, #0\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2]\n"
+ "mov z27.s, #0\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "mov z28.s, #0\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "mov z29.s, #0\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "mov z30.s, #0\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "mov z31.s, #0\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "udot z24.s, z16.b, z0.b[0]\n"
+ "udot z25.s, z16.b, z1.b[0]\n"
+ "udot z26.s, z16.b, z2.b[0]\n"
+ "udot z27.s, z16.b, z3.b[0]\n"
+ "udot z28.s, z16.b, z4.b[0]\n"
+ "udot z29.s, z16.b, z5.b[0]\n"
+ "udot z30.s, z16.b, z6.b[0]\n"
+ "udot z31.s, z16.b, z7.b[0]\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "udot z24.s, z17.b, z0.b[1]\n"
+ "udot z25.s, z17.b, z1.b[1]\n"
+ "udot z26.s, z17.b, z2.b[1]\n"
+ "udot z27.s, z17.b, z3.b[1]\n"
+ "udot z28.s, z17.b, z4.b[1]\n"
+ "udot z29.s, z17.b, z5.b[1]\n"
+ "udot z30.s, z17.b, z6.b[1]\n"
+ "udot z31.s, z17.b, z7.b[1]\n"
+ "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "udot z24.s, z18.b, z0.b[2]\n"
+ "udot z25.s, z18.b, z1.b[2]\n"
+ "udot z26.s, z18.b, z2.b[2]\n"
+ "udot z27.s, z18.b, z3.b[2]\n"
+ "udot z28.s, z18.b, z4.b[2]\n"
+ "udot z29.s, z18.b, z5.b[2]\n"
+ "udot z30.s, z18.b, z6.b[2]\n"
+ "udot z31.s, z18.b, z7.b[2]\n"
+ "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "udot z24.s, z19.b, z0.b[3]\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x10]\n"
+ "udot z25.s, z19.b, z1.b[3]\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1, #0x10]\n"
+ "udot z26.s, z19.b, z2.b[3]\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2, #0x10]\n"
+ "udot z27.s, z19.b, z3.b[3]\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3, #0x10]\n"
+ "udot z28.s, z19.b, z4.b[3]\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4, #0x10]\n"
+ "udot z29.s, z19.b, z5.b[3]\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5, #0x10]\n"
+ "udot z30.s, z19.b, z6.b[3]\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6, #0x10]\n"
+ "udot z31.s, z19.b, z7.b[3]\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7, #0x10]\n"
+ "udot z24.s, z20.b, z0.b[0]\n"
+ "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "udot z25.s, z20.b, z1.b[0]\n"
+ "udot z26.s, z20.b, z2.b[0]\n"
+ "udot z27.s, z20.b, z3.b[0]\n"
+ "udot z28.s, z20.b, z4.b[0]\n"
+ "udot z29.s, z20.b, z5.b[0]\n"
+ "udot z30.s, z20.b, z6.b[0]\n"
+ "udot z31.s, z20.b, z7.b[0]\n"
+ "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "udot z24.s, z21.b, z0.b[1]\n"
+ "udot z25.s, z21.b, z1.b[1]\n"
+ "udot z26.s, z21.b, z2.b[1]\n"
+ "udot z27.s, z21.b, z3.b[1]\n"
+ "udot z28.s, z21.b, z4.b[1]\n"
+ "udot z29.s, z21.b, z5.b[1]\n"
+ "udot z30.s, z21.b, z6.b[1]\n"
+ "udot z31.s, z21.b, z7.b[1]\n"
+ "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "udot z24.s, z22.b, z0.b[2]\n"
+ "udot z25.s, z22.b, z1.b[2]\n"
+ "udot z26.s, z22.b, z2.b[2]\n"
+ "udot z27.s, z22.b, z3.b[2]\n"
+ "udot z28.s, z22.b, z4.b[2]\n"
+ "udot z29.s, z22.b, z5.b[2]\n"
+ "udot z30.s, z22.b, z6.b[2]\n"
+ "udot z31.s, z22.b, z7.b[2]\n"
+ "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "udot z24.s, z23.b, z0.b[3]\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x20]\n"
+ "udot z25.s, z23.b, z1.b[3]\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1, #0x20]\n"
+ "udot z26.s, z23.b, z2.b[3]\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2, #0x20]\n"
+ "udot z27.s, z23.b, z3.b[3]\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3, #0x20]\n"
+ "udot z28.s, z23.b, z4.b[3]\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4, #0x20]\n"
+ "udot z29.s, z23.b, z5.b[3]\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5, #0x20]\n"
+ "udot z30.s, z23.b, z6.b[3]\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6, #0x20]\n"
+ "udot z31.s, z23.b, z7.b[3]\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7, #0x20]\n"
+ "udot z24.s, z16.b, z0.b[0]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #7\n"
+ "udot z25.s, z16.b, z1.b[0]\n"
+ "udot z26.s, z16.b, z2.b[0]\n"
+ "udot z27.s, z16.b, z3.b[0]\n"
+ "udot z28.s, z16.b, z4.b[0]\n"
+ "udot z29.s, z16.b, z5.b[0]\n"
+ "udot z30.s, z16.b, z6.b[0]\n"
+ "udot z31.s, z16.b, z7.b[0]\n"
+ "udot z24.s, z17.b, z0.b[1]\n"
+ "udot z25.s, z17.b, z1.b[1]\n"
+ "udot z26.s, z17.b, z2.b[1]\n"
+ "udot z27.s, z17.b, z3.b[1]\n"
+ "udot z28.s, z17.b, z4.b[1]\n"
+ "udot z29.s, z17.b, z5.b[1]\n"
+ "udot z30.s, z17.b, z6.b[1]\n"
+ "udot z31.s, z17.b, z7.b[1]\n"
+ "udot z24.s, z18.b, z0.b[2]\n"
+ "udot z25.s, z18.b, z1.b[2]\n"
+ "udot z26.s, z18.b, z2.b[2]\n"
+ "udot z27.s, z18.b, z3.b[2]\n"
+ "udot z28.s, z18.b, z4.b[2]\n"
+ "udot z29.s, z18.b, z5.b[2]\n"
+ "udot z30.s, z18.b, z6.b[2]\n"
+ "udot z31.s, z18.b, z7.b[2]\n"
+ "udot z24.s, z19.b, z0.b[3]\n"
+ "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x30]\n"
+ "udot z25.s, z19.b, z1.b[3]\n"
+ "ld1rqb z1.b, p6/z, [a_ptr1, #0x30]\n"
+ "udot z26.s, z19.b, z2.b[3]\n"
+ "ld1rqb z2.b, p6/z, [a_ptr2, #0x30]\n"
+ "udot z27.s, z19.b, z3.b[3]\n"
+ "ld1rqb z3.b, p6/z, [a_ptr3, #0x30]\n"
+ "udot z28.s, z19.b, z4.b[3]\n"
+ "ld1rqb z4.b, p6/z, [a_ptr4, #0x30]\n"
+ "udot z29.s, z19.b, z5.b[3]\n"
+ "ld1rqb z5.b, p6/z, [a_ptr5, #0x30]\n"
+ "udot z30.s, z19.b, z6.b[3]\n"
+ "ld1rqb z6.b, p6/z, [a_ptr6, #0x30]\n"
+ "udot z31.s, z19.b, z7.b[3]\n"
+ "ld1rqb z7.b, p6/z, [a_ptr7, #0x30]\n"
+ "udot z24.s, z20.b, z0.b[0]\n"
+ "udot z25.s, z20.b, z1.b[0]\n"
+ "udot z26.s, z20.b, z2.b[0]\n"
+ "udot z27.s, z20.b, z3.b[0]\n"
+ "udot z28.s, z20.b, z4.b[0]\n"
+ "udot z29.s, z20.b, z5.b[0]\n"
+ "udot z30.s, z20.b, z6.b[0]\n"
+ "udot z31.s, z20.b, z7.b[0]\n"
+ "udot z24.s, z21.b, z0.b[1]\n"
+ "udot z25.s, z21.b, z1.b[1]\n"
+ "udot z26.s, z21.b, z2.b[1]\n"
+ "udot z27.s, z21.b, z3.b[1]\n"
+ "udot z28.s, z21.b, z4.b[1]\n"
+ "udot z29.s, z21.b, z5.b[1]\n"
+ "udot z30.s, z21.b, z6.b[1]\n"
+ "udot z31.s, z21.b, z7.b[1]\n"
+ "udot z24.s, z22.b, z0.b[2]\n"
+ "udot z25.s, z22.b, z1.b[2]\n"
+ "udot z26.s, z22.b, z2.b[2]\n"
+ "udot z27.s, z22.b, z3.b[2]\n"
+ "udot z28.s, z22.b, z4.b[2]\n"
+ "udot z29.s, z22.b, z5.b[2]\n"
+ "udot z30.s, z22.b, z6.b[2]\n"
+ "udot z31.s, z22.b, z7.b[2]\n"
+ "5:\n"
"st1w z24.s, p0, [%[c_ptr0]]\n"
"addvl %[c_ptr0], %[c_ptr0], #1\n"
"st1w z25.s, p0, [c_ptr1]\n"
- "addvl c_ptr1, c_ptr1, #1\n"
"st1w z26.s, p0, [c_ptr2]\n"
- "addvl c_ptr2, c_ptr2, #1\n"
"st1w z27.s, p0, [c_ptr3]\n"
- "addvl c_ptr3, c_ptr3, #1\n"
"st1w z28.s, p0, [c_ptr4]\n"
- "addvl c_ptr4, c_ptr4, #1\n"
"st1w z29.s, p0, [c_ptr5]\n"
- "addvl c_ptr5, c_ptr5, #1\n"
"st1w z30.s, p0, [c_ptr6]\n"
- "addvl c_ptr6, c_ptr6, #1\n"
"st1w z31.s, p0, [c_ptr7]\n"
- "addvl c_ptr7, c_ptr7, #1\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
@@ -6844,52 +8140,54 @@
"add c_ptr1, %[c_ptr0], #0x0\n"
"add a_ptr1, %[a_ptr0], #0x0\n"
"1:\n"
- "mov z24.s, #0\n"
"ptrue p7.b\n"
- "mov z25.s, #0\n"
"whilelt p6.b, %[temp], %[odd_depth]\n"
- "mov z26.s, #0\n"
+ "whilelt p0.s, %[temp], %[last_width]\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
"ld1b z16.b, p7/z, [%[b_ptr0]]\n"
- "mov z27.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
"ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "mov z28.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
"ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "mov z29.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
"ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "mov z30.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
"ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "mov z31.s, #0\n"
+ "prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
"ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
"ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
- "whilelt p0.s, %[temp], %[last_width]\n"
"ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x40]\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x80]\n"
- "ld1rqb z1.b, p7/z, [a_ptr1]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0xc0]\n"
- "udot z24.s, z16.b, z0.b[0]\n"
- "ld1rqb z2.b, p7/z, [a_ptr2]\n"
- "ld1rqb z3.b, p7/z, [a_ptr3]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x100]\n"
- "udot z25.s, z16.b, z1.b[0]\n"
- "ld1rqb z4.b, p7/z, [a_ptr4]\n"
- "udot z26.s, z16.b, z2.b[0]\n"
- "ld1rqb z5.b, p7/z, [a_ptr5]\n"
- "udot z27.s, z16.b, z3.b[0]\n"
- "ld1rqb z6.b, p7/z, [a_ptr6]\n"
- "udot z24.s, z17.b, z0.b[1]\n"
- "ld1rqb z7.b, p7/z, [a_ptr7]\n"
- "udot z28.s, z16.b, z4.b[0]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x140]\n"
- "udot z29.s, z16.b, z5.b[0]\n"
- "prfm PLDL1KEEP, [a_ptr7, #0x180]\n"
- "udot z30.s, z16.b, z6.b[0]\n"
"addvl %[b_ptr0], %[b_ptr0], #8\n"
+ "cbz %[loops], 2f\n"
+ "mov z24.s, #0\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "mov z25.s, #0\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "mov z26.s, #0\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2]\n"
+ "mov z27.s, #0\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "mov z28.s, #0\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "mov z29.s, #0\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "mov z30.s, #0\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "mov z31.s, #0\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "udot z24.s, z16.b, z0.b[0]\n"
+ "subs %[loops], %[loops], #0x1\n"
+ "udot z25.s, z16.b, z1.b[0]\n"
+ "udot z26.s, z16.b, z2.b[0]\n"
+ "udot z27.s, z16.b, z3.b[0]\n"
+ "udot z28.s, z16.b, z4.b[0]\n"
+ "udot z29.s, z16.b, z5.b[0]\n"
+ "udot z30.s, z16.b, z6.b[0]\n"
"udot z31.s, z16.b, z7.b[0]\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "udot z24.s, z17.b, z0.b[1]\n"
"udot z25.s, z17.b, z1.b[1]\n"
"udot z26.s, z17.b, z2.b[1]\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
"udot z27.s, z17.b, z3.b[1]\n"
"udot z28.s, z17.b, z4.b[1]\n"
"udot z29.s, z17.b, z5.b[1]\n"
@@ -7039,83 +8337,81 @@
"udot z29.s, z23.b, z5.b[3]\n"
"udot z30.s, z23.b, z6.b[3]\n"
"udot z31.s, z23.b, z7.b[3]\n"
- "cbz %[loops], 2f\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
- "subs %[loops], %[loops], #0x1\n"
- "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
- "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
- "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
- "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
- "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
- "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
"b.eq 3f\n"
"4:\n"
"st1w z24.s, p7, [%[c_ptr0]]\n"
"subs %[loops], %[loops], #0x1\n"
"mov z24.s, #0\n"
- "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
- "st1w z25.s, p7, [c_ptr1]\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"addvl %[c_ptr0], %[c_ptr0], #1\n"
- "mov z25.s, #0\n"
- "ld1rqb z1.b, p7/z, [a_ptr1]\n"
- "udot z24.s, z16.b, z0.b[0]\n"
- "st1w z26.s, p7, [c_ptr2]\n"
- "mov z26.s, #0\n"
- "ld1rqb z2.b, p7/z, [a_ptr2]\n"
- "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "st1w z25.s, p7, [c_ptr1]\n"
"addvl c_ptr1, c_ptr1, #1\n"
- "udot z25.s, z16.b, z1.b[0]\n"
- "st1w z27.s, p7, [c_ptr3]\n"
- "mov z27.s, #0\n"
- "ld1rqb z4.b, p7/z, [a_ptr4]\n"
- "udot z26.s, z16.b, z2.b[0]\n"
- "ld1rqb z5.b, p7/z, [a_ptr5]\n"
- "udot z24.s, z17.b, z0.b[1]\n"
- "st1w z28.s, p7, [c_ptr4]\n"
- "mov z28.s, #0\n"
- "ld1rqb z6.b, p7/z, [a_ptr6]\n"
- "udot z27.s, z16.b, z3.b[0]\n"
- "ld1rqb z7.b, p7/z, [a_ptr7]\n"
- "udot z25.s, z17.b, z1.b[1]\n"
- "st1w z29.s, p7, [c_ptr5]\n"
- "mov z29.s, #0\n"
+ "mov z25.s, #0\n"
+ "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "st1w z26.s, p7, [c_ptr2]\n"
"addvl c_ptr2, c_ptr2, #1\n"
- "udot z28.s, z16.b, z4.b[0]\n"
+ "mov z26.s, #0\n"
+ "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
+ "st1w z27.s, p7, [c_ptr3]\n"
+ "addvl c_ptr3, c_ptr3, #1\n"
+ "mov z27.s, #0\n"
+ "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #8\n"
+ "st1w z28.s, p7, [c_ptr4]\n"
+ "addvl c_ptr4, c_ptr4, #1\n"
+ "mov z28.s, #0\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
+ "st1w z29.s, p7, [c_ptr5]\n"
+ "addvl c_ptr5, c_ptr5, #1\n"
+ "mov z29.s, #0\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2]\n"
+ "udot z24.s, z16.b, z0.b[0]\n"
"st1w z30.s, p7, [c_ptr6]\n"
"mov z30.s, #0\n"
- "addvl c_ptr3, c_ptr3, #1\n"
- "udot z29.s, z16.b, z5.b[0]\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "udot z25.s, z16.b, z1.b[0]\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "udot z26.s, z16.b, z2.b[0]\n"
"st1w z31.s, p7, [c_ptr7]\n"
"mov z31.s, #0\n"
- "addvl c_ptr4, c_ptr4, #1\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "udot z27.s, z16.b, z3.b[0]\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "udot z28.s, z16.b, z4.b[0]\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "udot z24.s, z17.b, z0.b[1]\n"
+ "addvl c_ptr6, c_ptr6, #1\n"
+ "udot z29.s, z16.b, z5.b[0]\n"
+ "addvl c_ptr7, c_ptr7, #1\n"
"udot z30.s, z16.b, z6.b[0]\n"
- "addvl c_ptr5, c_ptr5, #1\n"
+ "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
"udot z31.s, z16.b, z7.b[0]\n"
"ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "udot z25.s, z17.b, z1.b[1]\n"
+ "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
"udot z26.s, z17.b, z2.b[1]\n"
- "addvl c_ptr6, c_ptr6, #1\n"
+ "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
"udot z27.s, z17.b, z3.b[1]\n"
- "addvl c_ptr7, c_ptr7, #1\n"
+ "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
"udot z28.s, z17.b, z4.b[1]\n"
- "prfm PSTL1KEEP, [%[c_ptr0], #0x40]\n"
+ "prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
"udot z29.s, z17.b, z5.b[1]\n"
- "prfm PSTL1KEEP, [c_ptr1, #0x40]\n"
"udot z30.s, z17.b, z6.b[1]\n"
- "prfm PSTL1KEEP, [c_ptr2, #0x40]\n"
"udot z31.s, z17.b, z7.b[1]\n"
"ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"udot z24.s, z18.b, z0.b[2]\n"
- "prfm PSTL1KEEP, [c_ptr3, #0x40]\n"
"udot z25.s, z18.b, z1.b[2]\n"
- "prfm PSTL1KEEP, [c_ptr4, #0x40]\n"
"udot z26.s, z18.b, z2.b[2]\n"
- "prfm PSTL1KEEP, [c_ptr5, #0x40]\n"
"udot z27.s, z18.b, z3.b[2]\n"
- "prfm PSTL1KEEP, [c_ptr6, #0x40]\n"
"udot z28.s, z18.b, z4.b[2]\n"
- "prfm PSTL1KEEP, [c_ptr7, #0x40]\n"
"udot z29.s, z18.b, z5.b[2]\n"
"udot z30.s, z18.b, z6.b[2]\n"
"udot z31.s, z18.b, z7.b[2]\n"
@@ -7190,7 +8486,6 @@
"udot z29.s, z16.b, z5.b[0]\n"
"udot z30.s, z16.b, z6.b[0]\n"
"udot z31.s, z16.b, z7.b[0]\n"
- "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
"udot z24.s, z17.b, z0.b[1]\n"
"udot z25.s, z17.b, z1.b[1]\n"
"udot z26.s, z17.b, z2.b[1]\n"
@@ -7199,7 +8494,6 @@
"udot z29.s, z17.b, z5.b[1]\n"
"udot z30.s, z17.b, z6.b[1]\n"
"udot z31.s, z17.b, z7.b[1]\n"
- "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"udot z24.s, z18.b, z0.b[2]\n"
"udot z25.s, z18.b, z1.b[2]\n"
"udot z26.s, z18.b, z2.b[2]\n"
@@ -7208,7 +8502,6 @@
"udot z29.s, z18.b, z5.b[2]\n"
"udot z30.s, z18.b, z6.b[2]\n"
"udot z31.s, z18.b, z7.b[2]\n"
- "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
"udot z24.s, z19.b, z0.b[3]\n"
"ld1rqb z0.b, p6/z, [%[a_ptr0], #0x30]\n"
"udot z25.s, z19.b, z1.b[3]\n"
@@ -7226,7 +8519,6 @@
"udot z31.s, z19.b, z7.b[3]\n"
"ld1rqb z7.b, p6/z, [a_ptr7, #0x30]\n"
"udot z24.s, z20.b, z0.b[0]\n"
- "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"udot z25.s, z20.b, z1.b[0]\n"
"udot z26.s, z20.b, z2.b[0]\n"
"udot z27.s, z20.b, z3.b[0]\n"
@@ -7234,7 +8526,6 @@
"udot z29.s, z20.b, z5.b[0]\n"
"udot z30.s, z20.b, z6.b[0]\n"
"udot z31.s, z20.b, z7.b[0]\n"
- "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
"udot z24.s, z21.b, z0.b[1]\n"
"udot z25.s, z21.b, z1.b[1]\n"
"udot z26.s, z21.b, z2.b[1]\n"
@@ -7243,7 +8534,6 @@
"udot z29.s, z21.b, z5.b[1]\n"
"udot z30.s, z21.b, z6.b[1]\n"
"udot z31.s, z21.b, z7.b[1]\n"
- "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
"udot z24.s, z22.b, z0.b[2]\n"
"udot z25.s, z22.b, z1.b[2]\n"
"udot z26.s, z22.b, z2.b[2]\n"
@@ -7252,7 +8542,6 @@
"udot z29.s, z22.b, z5.b[2]\n"
"udot z30.s, z22.b, z6.b[2]\n"
"udot z31.s, z22.b, z7.b[2]\n"
- "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
"udot z24.s, z23.b, z0.b[3]\n"
"udot z25.s, z23.b, z1.b[3]\n"
"udot z26.s, z23.b, z2.b[3]\n"
@@ -7266,49 +8555,56 @@
"st1w z24.s, p7, [%[c_ptr0]]\n"
"addvl %[c_ptr0], %[c_ptr0], #1\n"
"mov z24.s, #0\n"
- "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
- "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
- "addvl %[b_ptr0], %[b_ptr0], #8\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
"st1w z25.s, p7, [c_ptr1]\n"
"addvl c_ptr1, c_ptr1, #1\n"
"mov z25.s, #0\n"
- "ld1rqb z1.b, p7/z, [a_ptr1]\n"
- "udot z24.s, z16.b, z0.b[0]\n"
+ "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
"st1w z26.s, p7, [c_ptr2]\n"
- "mov z26.s, #0\n"
- "ld1rqb z2.b, p7/z, [a_ptr2]\n"
- "ld1rqb z3.b, p7/z, [a_ptr3]\n"
"addvl c_ptr2, c_ptr2, #1\n"
- "udot z25.s, z16.b, z1.b[0]\n"
+ "mov z26.s, #0\n"
+ "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
"st1w z27.s, p7, [c_ptr3]\n"
- "mov z27.s, #0\n"
- "ld1rqb z4.b, p7/z, [a_ptr4]\n"
- "udot z26.s, z16.b, z2.b[0]\n"
- "ld1rqb z5.b, p7/z, [a_ptr5]\n"
- "udot z24.s, z17.b, z0.b[1]\n"
- "st1w z28.s, p7, [c_ptr4]\n"
- "mov z28.s, #0\n"
- "ld1rqb z6.b, p7/z, [a_ptr6]\n"
- "udot z27.s, z16.b, z3.b[0]\n"
- "ld1rqb z7.b, p7/z, [a_ptr7]\n"
- "udot z25.s, z17.b, z1.b[1]\n"
- "st1w z29.s, p7, [c_ptr5]\n"
- "mov z29.s, #0\n"
"addvl c_ptr3, c_ptr3, #1\n"
- "udot z28.s, z16.b, z4.b[0]\n"
+ "mov z27.s, #0\n"
+ "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #8\n"
+ "st1w z28.s, p7, [c_ptr4]\n"
+ "addvl c_ptr4, c_ptr4, #1\n"
+ "mov z28.s, #0\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "st1w z29.s, p7, [c_ptr5]\n"
+ "addvl c_ptr5, c_ptr5, #1\n"
+ "mov z29.s, #0\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2]\n"
+ "udot z24.s, z16.b, z0.b[0]\n"
"st1w z30.s, p7, [c_ptr6]\n"
"mov z30.s, #0\n"
- "addvl c_ptr4, c_ptr4, #1\n"
- "udot z29.s, z16.b, z5.b[0]\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "udot z25.s, z16.b, z1.b[0]\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "udot z26.s, z16.b, z2.b[0]\n"
"st1w z31.s, p7, [c_ptr7]\n"
"mov z31.s, #0\n"
- "addvl c_ptr5, c_ptr5, #1\n"
- "udot z30.s, z16.b, z6.b[0]\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "udot z27.s, z16.b, z3.b[0]\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "udot z28.s, z16.b, z4.b[0]\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "udot z24.s, z17.b, z0.b[1]\n"
"addvl c_ptr6, c_ptr6, #1\n"
+ "udot z29.s, z16.b, z5.b[0]\n"
+ "addvl c_ptr7, c_ptr7, #1\n"
+ "udot z30.s, z16.b, z6.b[0]\n"
"udot z31.s, z16.b, z7.b[0]\n"
"ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "udot z25.s, z17.b, z1.b[1]\n"
"udot z26.s, z17.b, z2.b[1]\n"
- "addvl c_ptr7, c_ptr7, #1\n"
"udot z27.s, z17.b, z3.b[1]\n"
"udot z28.s, z17.b, z4.b[1]\n"
"udot z29.s, z17.b, z5.b[1]\n"
@@ -7458,23 +8754,195 @@
"udot z29.s, z23.b, z5.b[3]\n"
"udot z30.s, z23.b, z6.b[3]\n"
"udot z31.s, z23.b, z7.b[3]\n"
+ "b 5f\n"
"2:\n"
+ "mov z24.s, #0\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0]]\n"
+ "mov z25.s, #0\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1]\n"
+ "mov z26.s, #0\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2]\n"
+ "mov z27.s, #0\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3]\n"
+ "mov z28.s, #0\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4]\n"
+ "mov z29.s, #0\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5]\n"
+ "mov z30.s, #0\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6]\n"
+ "mov z31.s, #0\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7]\n"
+ "udot z24.s, z16.b, z0.b[0]\n"
+ "udot z25.s, z16.b, z1.b[0]\n"
+ "udot z26.s, z16.b, z2.b[0]\n"
+ "udot z27.s, z16.b, z3.b[0]\n"
+ "udot z28.s, z16.b, z4.b[0]\n"
+ "udot z29.s, z16.b, z5.b[0]\n"
+ "udot z30.s, z16.b, z6.b[0]\n"
+ "udot z31.s, z16.b, z7.b[0]\n"
+ "ld1b z16.b, p7/z, [%[b_ptr0]]\n"
+ "udot z24.s, z17.b, z0.b[1]\n"
+ "udot z25.s, z17.b, z1.b[1]\n"
+ "udot z26.s, z17.b, z2.b[1]\n"
+ "udot z27.s, z17.b, z3.b[1]\n"
+ "udot z28.s, z17.b, z4.b[1]\n"
+ "udot z29.s, z17.b, z5.b[1]\n"
+ "udot z30.s, z17.b, z6.b[1]\n"
+ "udot z31.s, z17.b, z7.b[1]\n"
+ "ld1b z17.b, p7/z, [%[b_ptr0], #1, MUL VL]\n"
+ "udot z24.s, z18.b, z0.b[2]\n"
+ "udot z25.s, z18.b, z1.b[2]\n"
+ "udot z26.s, z18.b, z2.b[2]\n"
+ "udot z27.s, z18.b, z3.b[2]\n"
+ "udot z28.s, z18.b, z4.b[2]\n"
+ "udot z29.s, z18.b, z5.b[2]\n"
+ "udot z30.s, z18.b, z6.b[2]\n"
+ "udot z31.s, z18.b, z7.b[2]\n"
+ "ld1b z18.b, p7/z, [%[b_ptr0], #2, MUL VL]\n"
+ "udot z24.s, z19.b, z0.b[3]\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x10]\n"
+ "udot z25.s, z19.b, z1.b[3]\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1, #0x10]\n"
+ "udot z26.s, z19.b, z2.b[3]\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2, #0x10]\n"
+ "udot z27.s, z19.b, z3.b[3]\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3, #0x10]\n"
+ "udot z28.s, z19.b, z4.b[3]\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4, #0x10]\n"
+ "udot z29.s, z19.b, z5.b[3]\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5, #0x10]\n"
+ "udot z30.s, z19.b, z6.b[3]\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6, #0x10]\n"
+ "udot z31.s, z19.b, z7.b[3]\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7, #0x10]\n"
+ "udot z24.s, z20.b, z0.b[0]\n"
+ "ld1b z19.b, p7/z, [%[b_ptr0], #3, MUL VL]\n"
+ "udot z25.s, z20.b, z1.b[0]\n"
+ "udot z26.s, z20.b, z2.b[0]\n"
+ "udot z27.s, z20.b, z3.b[0]\n"
+ "udot z28.s, z20.b, z4.b[0]\n"
+ "udot z29.s, z20.b, z5.b[0]\n"
+ "udot z30.s, z20.b, z6.b[0]\n"
+ "udot z31.s, z20.b, z7.b[0]\n"
+ "ld1b z20.b, p7/z, [%[b_ptr0], #4, MUL VL]\n"
+ "udot z24.s, z21.b, z0.b[1]\n"
+ "udot z25.s, z21.b, z1.b[1]\n"
+ "udot z26.s, z21.b, z2.b[1]\n"
+ "udot z27.s, z21.b, z3.b[1]\n"
+ "udot z28.s, z21.b, z4.b[1]\n"
+ "udot z29.s, z21.b, z5.b[1]\n"
+ "udot z30.s, z21.b, z6.b[1]\n"
+ "udot z31.s, z21.b, z7.b[1]\n"
+ "ld1b z21.b, p7/z, [%[b_ptr0], #5, MUL VL]\n"
+ "udot z24.s, z22.b, z0.b[2]\n"
+ "udot z25.s, z22.b, z1.b[2]\n"
+ "udot z26.s, z22.b, z2.b[2]\n"
+ "udot z27.s, z22.b, z3.b[2]\n"
+ "udot z28.s, z22.b, z4.b[2]\n"
+ "udot z29.s, z22.b, z5.b[2]\n"
+ "udot z30.s, z22.b, z6.b[2]\n"
+ "udot z31.s, z22.b, z7.b[2]\n"
+ "ld1b z22.b, p7/z, [%[b_ptr0], #6, MUL VL]\n"
+ "udot z24.s, z23.b, z0.b[3]\n"
+ "ld1rqb z0.b, p7/z, [%[a_ptr0], #0x20]\n"
+ "udot z25.s, z23.b, z1.b[3]\n"
+ "ld1rqb z1.b, p7/z, [a_ptr1, #0x20]\n"
+ "udot z26.s, z23.b, z2.b[3]\n"
+ "ld1rqb z2.b, p7/z, [a_ptr2, #0x20]\n"
+ "udot z27.s, z23.b, z3.b[3]\n"
+ "ld1rqb z3.b, p7/z, [a_ptr3, #0x20]\n"
+ "udot z28.s, z23.b, z4.b[3]\n"
+ "ld1rqb z4.b, p7/z, [a_ptr4, #0x20]\n"
+ "udot z29.s, z23.b, z5.b[3]\n"
+ "ld1rqb z5.b, p7/z, [a_ptr5, #0x20]\n"
+ "udot z30.s, z23.b, z6.b[3]\n"
+ "ld1rqb z6.b, p7/z, [a_ptr6, #0x20]\n"
+ "udot z31.s, z23.b, z7.b[3]\n"
+ "ld1b z23.b, p7/z, [%[b_ptr0], #7, MUL VL]\n"
+ "udot z24.s, z16.b, z0.b[0]\n"
+ "ld1rqb z7.b, p7/z, [a_ptr7, #0x20]\n"
+ "udot z25.s, z16.b, z1.b[0]\n"
+ "addvl %[b_ptr0], %[b_ptr0], #8\n"
+ "udot z26.s, z16.b, z2.b[0]\n"
+ "udot z27.s, z16.b, z3.b[0]\n"
+ "udot z28.s, z16.b, z4.b[0]\n"
+ "udot z29.s, z16.b, z5.b[0]\n"
+ "udot z30.s, z16.b, z6.b[0]\n"
+ "udot z31.s, z16.b, z7.b[0]\n"
+ "udot z24.s, z17.b, z0.b[1]\n"
+ "udot z25.s, z17.b, z1.b[1]\n"
+ "udot z26.s, z17.b, z2.b[1]\n"
+ "udot z27.s, z17.b, z3.b[1]\n"
+ "udot z28.s, z17.b, z4.b[1]\n"
+ "udot z29.s, z17.b, z5.b[1]\n"
+ "udot z30.s, z17.b, z6.b[1]\n"
+ "udot z31.s, z17.b, z7.b[1]\n"
+ "udot z24.s, z18.b, z0.b[2]\n"
+ "udot z25.s, z18.b, z1.b[2]\n"
+ "udot z26.s, z18.b, z2.b[2]\n"
+ "udot z27.s, z18.b, z3.b[2]\n"
+ "udot z28.s, z18.b, z4.b[2]\n"
+ "udot z29.s, z18.b, z5.b[2]\n"
+ "udot z30.s, z18.b, z6.b[2]\n"
+ "udot z31.s, z18.b, z7.b[2]\n"
+ "udot z24.s, z19.b, z0.b[3]\n"
+ "ld1rqb z0.b, p6/z, [%[a_ptr0], #0x30]\n"
+ "udot z25.s, z19.b, z1.b[3]\n"
+ "ld1rqb z1.b, p6/z, [a_ptr1, #0x30]\n"
+ "udot z26.s, z19.b, z2.b[3]\n"
+ "ld1rqb z2.b, p6/z, [a_ptr2, #0x30]\n"
+ "udot z27.s, z19.b, z3.b[3]\n"
+ "ld1rqb z3.b, p6/z, [a_ptr3, #0x30]\n"
+ "udot z28.s, z19.b, z4.b[3]\n"
+ "ld1rqb z4.b, p6/z, [a_ptr4, #0x30]\n"
+ "udot z29.s, z19.b, z5.b[3]\n"
+ "ld1rqb z5.b, p6/z, [a_ptr5, #0x30]\n"
+ "udot z30.s, z19.b, z6.b[3]\n"
+ "ld1rqb z6.b, p6/z, [a_ptr6, #0x30]\n"
+ "udot z31.s, z19.b, z7.b[3]\n"
+ "ld1rqb z7.b, p6/z, [a_ptr7, #0x30]\n"
+ "udot z24.s, z20.b, z0.b[0]\n"
+ "udot z25.s, z20.b, z1.b[0]\n"
+ "udot z26.s, z20.b, z2.b[0]\n"
+ "udot z27.s, z20.b, z3.b[0]\n"
+ "udot z28.s, z20.b, z4.b[0]\n"
+ "udot z29.s, z20.b, z5.b[0]\n"
+ "udot z30.s, z20.b, z6.b[0]\n"
+ "udot z31.s, z20.b, z7.b[0]\n"
+ "udot z24.s, z21.b, z0.b[1]\n"
+ "udot z25.s, z21.b, z1.b[1]\n"
+ "udot z26.s, z21.b, z2.b[1]\n"
+ "udot z27.s, z21.b, z3.b[1]\n"
+ "udot z28.s, z21.b, z4.b[1]\n"
+ "udot z29.s, z21.b, z5.b[1]\n"
+ "udot z30.s, z21.b, z6.b[1]\n"
+ "udot z31.s, z21.b, z7.b[1]\n"
+ "udot z24.s, z22.b, z0.b[2]\n"
+ "udot z25.s, z22.b, z1.b[2]\n"
+ "udot z26.s, z22.b, z2.b[2]\n"
+ "udot z27.s, z22.b, z3.b[2]\n"
+ "udot z28.s, z22.b, z4.b[2]\n"
+ "udot z29.s, z22.b, z5.b[2]\n"
+ "udot z30.s, z22.b, z6.b[2]\n"
+ "udot z31.s, z22.b, z7.b[2]\n"
+ "udot z24.s, z23.b, z0.b[3]\n"
+ "udot z25.s, z23.b, z1.b[3]\n"
+ "udot z26.s, z23.b, z2.b[3]\n"
+ "udot z27.s, z23.b, z3.b[3]\n"
+ "udot z28.s, z23.b, z4.b[3]\n"
+ "udot z29.s, z23.b, z5.b[3]\n"
+ "udot z30.s, z23.b, z6.b[3]\n"
+ "udot z31.s, z23.b, z7.b[3]\n"
+ "5:\n"
"st1w z24.s, p0, [%[c_ptr0]]\n"
"addvl %[c_ptr0], %[c_ptr0], #1\n"
"st1w z25.s, p0, [c_ptr1]\n"
- "addvl c_ptr1, c_ptr1, #1\n"
"st1w z26.s, p0, [c_ptr2]\n"
- "addvl c_ptr2, c_ptr2, #1\n"
"st1w z27.s, p0, [c_ptr3]\n"
- "addvl c_ptr3, c_ptr3, #1\n"
"st1w z28.s, p0, [c_ptr4]\n"
- "addvl c_ptr4, c_ptr4, #1\n"
"st1w z29.s, p0, [c_ptr5]\n"
- "addvl c_ptr5, c_ptr5, #1\n"
"st1w z30.s, p0, [c_ptr6]\n"
- "addvl c_ptr6, c_ptr6, #1\n"
"st1w z31.s, p0, [c_ptr7]\n"
- "addvl c_ptr7, c_ptr7, #1\n"
".unreq a_ptr1\n"
".unreq a_ptr2\n"
".unreq a_ptr3\n"
diff --git a/src/core/NEON/kernels/arm_gemm/quantize_wrapper.hpp b/src/core/NEON/kernels/arm_gemm/quantize_wrapper.hpp
index eec842d..fdb4f58 100644
--- a/src/core/NEON/kernels/arm_gemm/quantize_wrapper.hpp
+++ b/src/core/NEON/kernels/arm_gemm/quantize_wrapper.hpp
@@ -110,7 +110,7 @@
QuantizeWrapper operator=(const QuantizeWrapper &) = delete;
QuantizeWrapper(const GemmArgs &args, const Requantize32 &qp) : _params(qp), _args(args), _barrier(args._maxthreads) {
- GemmArgs newargs = GemmArgs(args._ci, args._Msize, args._Nsize, args._Ksize, args._nbatches, args._nmulti, Activation(), args._maxthreads, nullptr);
+ GemmArgs newargs = GemmArgs(args._ci, args._Msize, args._Nsize, args._Ksize, args._Ksections, args._nbatches, args._nmulti, args._indirect_input, Activation(), args._maxthreads);
_subgemm = gemm<To, Tgemm>(newargs);
if (_subgemm == nullptr) {
diff --git a/src/core/NEON/kernels/arm_gemm/quantized.cpp b/src/core/NEON/kernels/arm_gemm/quantized.cpp
index cac02cf..111d01e 100644
--- a/src/core/NEON/kernels/arm_gemm/quantized.cpp
+++ b/src/core/NEON/kernels/arm_gemm/quantized.cpp
@@ -301,6 +301,179 @@
out_ptr1 += 16;
}
+ // We are often quantizing one block of interleaved kernel output at a time - these are three registers
+ // wide. Special case that here.
+ if (regs==3) {
+ regs -= 3;
+
+ int32x4_t v_mul0;
+ int32x4_t v_mul1;
+ int32x4_t v_mul2;
+
+ int32x4_t v_shf0;
+ int32x4_t v_shf1;
+ int32x4_t v_shf2;
+
+ int32x4_t v_shf0l;
+ int32x4_t v_shf1l;
+ int32x4_t v_shf2l;
+
+ if (per_channel) {
+ v_mul0 = vld1q_s32(perch_mul_ptr);
+ v_mul1 = vld1q_s32(perch_mul_ptr + 4);
+ v_mul2 = vld1q_s32(perch_mul_ptr + 8);
+ perch_mul_ptr += 12;
+
+ v_shf0 = vld1q_s32(perch_shift_ptr);
+ v_shf1 = vld1q_s32(perch_shift_ptr + 4);
+ v_shf2 = vld1q_s32(perch_shift_ptr + 8);
+ perch_shift_ptr += 12;
+
+ if (do_left_shift) {
+ v_shf0l = vld1q_s32(perch_shiftl_ptr);
+ v_shf1l = vld1q_s32(perch_shiftl_ptr + 4);
+ v_shf2l = vld1q_s32(perch_shiftl_ptr + 8);
+ perch_shiftl_ptr += 12;
+ }
+ } else {
+ v_mul0=v_mul1=v_mul2=v_mul;
+ v_shf0=v_shf1=v_shf2=v_right_shift;
+ v_shf0l=v_shf1l=v_shf2l=v_left_shift;
+ }
+
+ // Load column pointers
+ int32x4_t v_col0 = vld1q_s32(colptr);
+ int32x4_t v_col1 = vld1q_s32(colptr + 4);
+ int32x4_t v_col2 = vld1q_s32(colptr + 8);
+ colptr += 12;
+
+ // Load input data (row 0);
+ int32x4_t v_in00 = vld1q_s32(in_ptr);
+ int32x4_t v_in01 = vld1q_s32(in_ptr + 4);
+ int32x4_t v_in02 = vld1q_s32(in_ptr + 8);
+ in_ptr += 12;
+
+ // Load input data (row 1);
+ int32x4_t v_in10 = vld1q_s32(in_ptr1);
+ int32x4_t v_in11 = vld1q_s32(in_ptr1 + 4);
+ int32x4_t v_in12 = vld1q_s32(in_ptr1 + 8);
+ in_ptr1 += 12;
+
+ // Add on row bias and column bias
+ v_in00 = vaddq_s32(v_in00, v_row_sum);
+ v_in01 = vaddq_s32(v_in01, v_row_sum);
+ v_in02 = vaddq_s32(v_in02, v_row_sum);
+
+ v_in10 = vaddq_s32(v_in10, v_row_sum1);
+ v_in11 = vaddq_s32(v_in11, v_row_sum1);
+ v_in12 = vaddq_s32(v_in12, v_row_sum1);
+
+ v_in00 = vaddq_s32(v_in00, v_col0);
+ v_in01 = vaddq_s32(v_in01, v_col1);
+ v_in02 = vaddq_s32(v_in02, v_col2);
+
+ v_in10 = vaddq_s32(v_in10, v_col0);
+ v_in11 = vaddq_s32(v_in11, v_col1);
+ v_in12 = vaddq_s32(v_in12, v_col2);
+
+ // Quantize
+
+ // If a left shift is needed it needs to happen first.
+ if (do_left_shift) {
+ v_in00 = vrshlq_s32(v_in00, v_shf0l);
+ v_in01 = vrshlq_s32(v_in01, v_shf1l);
+ v_in02 = vrshlq_s32(v_in02, v_shf2l);
+
+ v_in10 = vrshlq_s32(v_in10, v_shf0l);
+ v_in11 = vrshlq_s32(v_in11, v_shf1l);
+ v_in12 = vrshlq_s32(v_in12, v_shf2l);
+ }
+
+ // Multiply
+ v_in00 = vqrdmulhq_s32(v_in00, v_mul0);
+ v_in01 = vqrdmulhq_s32(v_in01, v_mul1);
+ v_in02 = vqrdmulhq_s32(v_in02, v_mul2);
+
+ v_in10 = vqrdmulhq_s32(v_in10, v_mul0);
+ v_in11 = vqrdmulhq_s32(v_in11, v_mul1);
+ v_in12 = vqrdmulhq_s32(v_in12, v_mul2);
+
+ // Compute and add on corrective offset
+ if (do_shift_correction) {
+ int32x4_t v_temp00 = vandq_s32(v_in00, v_shf0);
+ int32x4_t v_temp01 = vandq_s32(v_in01, v_shf1);
+ int32x4_t v_temp02 = vandq_s32(v_in02, v_shf2);
+
+ int32x4_t v_temp10 = vandq_s32(v_in10, v_shf0);
+ int32x4_t v_temp11 = vandq_s32(v_in11, v_shf1);
+ int32x4_t v_temp12 = vandq_s32(v_in12, v_shf2);
+
+ v_temp00 = vshrq_n_s32(v_temp00, 31);
+ v_temp01 = vshrq_n_s32(v_temp01, 31);
+ v_temp02 = vshrq_n_s32(v_temp02, 31);
+
+ v_temp10 = vshrq_n_s32(v_temp10, 31);
+ v_temp11 = vshrq_n_s32(v_temp11, 31);
+ v_temp12 = vshrq_n_s32(v_temp12, 31);
+
+ v_in00 = vqaddq_s32(v_in00, v_temp00);
+ v_in01 = vqaddq_s32(v_in01, v_temp01);
+ v_in02 = vqaddq_s32(v_in02, v_temp02);
+
+ v_in10 = vqaddq_s32(v_in10, v_temp10);
+ v_in11 = vqaddq_s32(v_in11, v_temp11);
+ v_in12 = vqaddq_s32(v_in12, v_temp12);
+ }
+
+ v_in00 = vrshlq_s32(v_in00, v_shf0);
+ v_in01 = vrshlq_s32(v_in01, v_shf1);
+ v_in02 = vrshlq_s32(v_in02, v_shf2);
+
+ v_in10 = vrshlq_s32(v_in10, v_shf0);
+ v_in11 = vrshlq_s32(v_in11, v_shf1);
+ v_in12 = vrshlq_s32(v_in12, v_shf2);
+
+ v_in00 = vaddq_s32(v_in00, v_c_offset);
+ v_in01 = vaddq_s32(v_in01, v_c_offset);
+ v_in02 = vaddq_s32(v_in02, v_c_offset);
+
+ v_in10 = vaddq_s32(v_in10, v_c_offset);
+ v_in11 = vaddq_s32(v_in11, v_c_offset);
+ v_in12 = vaddq_s32(v_in12, v_c_offset);
+
+ v_in00 = vmaxq_s32(v_in00, v_minval);
+ v_in01 = vmaxq_s32(v_in01, v_minval);
+ v_in02 = vmaxq_s32(v_in02, v_minval);
+
+ v_in10 = vmaxq_s32(v_in10, v_minval);
+ v_in11 = vmaxq_s32(v_in11, v_minval);
+ v_in12 = vmaxq_s32(v_in12, v_minval);
+
+ v_in00 = vminq_s32(v_in00, v_maxval);
+ v_in01 = vminq_s32(v_in01, v_maxval);
+ v_in02 = vminq_s32(v_in02, v_maxval);
+
+ v_in10 = vminq_s32(v_in10, v_maxval);
+ v_in11 = vminq_s32(v_in11, v_maxval);
+ v_in12 = vminq_s32(v_in12, v_maxval);
+
+ int16x8_t v_uz00 = vuzp1q_s16(vreinterpretq_s16_s32(v_in00), vreinterpretq_s16_s32(v_in01));
+ int16x8_t v_uz01 = vuzp1q_s16(vreinterpretq_s16_s32(v_in02), vreinterpretq_s16_s32(v_in02));
+
+ int16x8_t v_uz10 = vuzp1q_s16(vreinterpretq_s16_s32(v_in10), vreinterpretq_s16_s32(v_in11));
+ int16x8_t v_uz11 = vuzp1q_s16(vreinterpretq_s16_s32(v_in12), vreinterpretq_s16_s32(v_in12));
+
+ int8x16_t v_uz0 = vuzp1q_s8(vreinterpretq_s8_s16(v_uz00), vreinterpretq_s8_s16(v_uz01));
+ int8x16_t v_uz1 = vuzp1q_s8(vreinterpretq_s8_s16(v_uz10), vreinterpretq_s8_s16(v_uz11));
+
+ vst1q_lane_s64(reinterpret_cast<int64_t *>(out_ptr), vreinterpretq_s64_s8(v_uz0), 0);
+ vst1q_lane_s32(reinterpret_cast<int32_t *>(out_ptr + 8), vreinterpretq_s32_s8(v_uz0), 2);
+ out_ptr += 12;
+ vst1q_lane_s64(reinterpret_cast<int64_t *>(out_ptr1), vreinterpretq_s64_s8(v_uz1), 0);
+ vst1q_lane_s32(reinterpret_cast<int32_t *>(out_ptr1 + 8), vreinterpretq_s32_s8(v_uz1), 2);
+ out_ptr1 += 12;
+ }
+
while (regs--) {
int32x4_t v_mul0;
int32x4_t v_shf0;
diff --git a/src/core/NEON/kernels/arm_gemm/quantized.hpp b/src/core/NEON/kernels/arm_gemm/quantized.hpp
index b0e0c3b..3f34430 100644
--- a/src/core/NEON/kernels/arm_gemm/quantized.hpp
+++ b/src/core/NEON/kernels/arm_gemm/quantized.hpp
@@ -23,6 +23,8 @@
*/
#pragma once
+#include "utils.hpp" // IndirectInputArg
+
namespace arm_gemm {
template<typename Tin, typename Tout>
@@ -39,4 +41,8 @@
const T *input, unsigned int in_stride, int32_t *col_bias, unsigned int depth,
unsigned int multi, unsigned int first_col);
+template<typename T>
+void row_sums_indirect(unsigned int num_strings, const unsigned int *string_lengths, IndirectInputArg<T> A_arg,
+ size_t M, int32_t *output_ptr, const Requantize32 *qp);
+
} // namespace arm_gemm
diff --git a/src/core/NEON/kernels/arm_gemm/rowsum_indirect_s8.cpp b/src/core/NEON/kernels/arm_gemm/rowsum_indirect_s8.cpp
new file mode 100644
index 0000000..5433676
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/rowsum_indirect_s8.cpp
@@ -0,0 +1,1160 @@
+/*
+ * Copyright (c) 2019-2020 Arm Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+#ifdef __aarch64__
+
+#include "arm_gemm.hpp"
+#include "quantized.hpp"
+#include "utils.hpp"
+
+#include <cassert>
+
+namespace arm_gemm {
+
+template<>
+void row_sums_indirect(
+ unsigned int num_strings, const unsigned int *string_lengths, IndirectInputArg<int8_t> A_arg,
+ size_t M, int32_t *out_ptr, const Requantize32 *qp
+)
+{
+ struct KernelArgs {
+ unsigned int num_strings;
+ const unsigned int *string_lengths;
+ unsigned int input_initial_col;
+ } ka;
+
+ unsigned long flags=0;
+ void *input_ptr;
+ size_t input_offset;
+
+ if (A_arg.is_indirect) {
+ input_ptr=(void *)(A_arg.indirect.ptr);
+ input_offset=A_arg.indirect.start_row;
+ ka.input_initial_col=A_arg.indirect.start_col;
+ flags |= 0x8;
+ } else {
+ assert(num_strings==1);
+ input_ptr=(void *)(A_arg.direct.base);
+ input_offset=A_arg.direct.stride;
+ }
+
+ ka.num_strings = num_strings;
+ ka.string_lengths = string_lengths;
+
+ __asm__ __volatile__(
+ "add x19, %x[qp], %[b_offset]\n"
+ "ld1r { v2.4s }, [x19]\n"
+ "neg v2.4s, v2.4s\n"
+ "1:" // Row loop
+ "cmp %x[M], #0x6\n"
+ "bge 86f\n"
+ "cmp %x[M], #0x4\n"
+ "bgt 69f\n"
+ "beq 52f\n"
+ "cmp %x[M], #0x2\n"
+ "bgt 35f\n"
+ "beq 18f\n"
+ "movi v1.8h, #0x0\n"
+ "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "movi v0.4s, #0x0\n"
+ "mov x9, #0x0\n"
+ "mov x28, #0x0\n"
+ "2:" // Height 1: String loop
+ "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr w27, [x19, x28, LSL #0x2]\n"
+ "tbz %x[flags], #3, 3f\n"
+ "ldr x19, [%x[input_ptr], x28, LSL #0x3]\n"
+ "add x19, x19, %x[input_offset], LSL #3\n"
+ "ldr x26, [x19, #0x0]\n"
+ "cbnz x28, 4f\n"
+ "ldr w19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x26, x26, x19\n"
+ "b 4f\n"
+ "3:" // Height 1: setup direct input
+ "mov x26, %x[input_ptr]\n"
+ "4:" // Height 1: input setup done
+ "cmp x27, #0x10\n"
+ "blt 8f\n"
+ "cmp x27, #0x20\n"
+ "blt 7f\n"
+ "5:" // Height 1: Multiply loop: Main loop head
+ "ldr q31, [x26, #0x0]\n"
+ "cmp x9, #0x7e\n"
+ "add x26, x26, #0x10\n"
+ "blt 6f\n"
+ "sadalp v0.4s, v1.8h\n"
+ "movi v1.8h, #0x0\n"
+ "mov x9, #0x0\n"
+ "6:" // Height 1: Multiply loop: unique 1: no collapse
+ "sadalp v1.8h, v31.16b\n"
+ "add x9, x9, #0x1\n"
+ "sub x27, x27, #0x10\n"
+ "cmp x27, #0x20\n"
+ "bge 5b\n"
+ "7:" // Height 1: Multiply loop: Single iteration only
+ "sub x27, x27, #0x10\n"
+ "ldr q31, [x26, #0x0]\n"
+ "add x26, x26, #0x10\n"
+ "sadalp v1.8h, v31.16b\n"
+ "8:" // Height 1: Multiply loop: Main loop skip
+ "cbz x27, 17f\n"
+ "tbz x27, #3, 12f\n"
+ "ldr d31, [x26], #0x8\n"
+ "tbz x27, #2, 10f\n"
+ "ld1 { v31.s }[2], [x26], #0x4\n"
+ "tbz x27, #1, 9f\n"
+ "ld1 { v31.h }[6], [x26], #0x2\n"
+ "tbz x27, #0, 16f\n"
+ "ld1 { v31.b }[14], [x26]\n"
+ "b 16f\n"
+ "9:" // Height 1: Multiply loop: Ragged operand read: partial_1_12
+ "tbz x27, #0, 16f\n"
+ "ld1 { v31.b }[12], [x26]\n"
+ "b 16f\n"
+ "10:" // Height 1: Multiply loop: Ragged operand read: partial_2_8
+ "tbz x27, #1, 11f\n"
+ "ld1 { v31.h }[4], [x26], #0x2\n"
+ "tbz x27, #0, 16f\n"
+ "ld1 { v31.b }[10], [x26]\n"
+ "b 16f\n"
+ "11:" // Height 1: Multiply loop: Ragged operand read: partial_1_8
+ "tbz x27, #0, 16f\n"
+ "ld1 { v31.b }[8], [x26]\n"
+ "b 16f\n"
+ "12:" // Height 1: Multiply loop: Ragged operand read: partial_4_0
+ "tbz x27, #2, 14f\n"
+ "ldr s31, [x26], #0x4\n"
+ "tbz x27, #1, 13f\n"
+ "ld1 { v31.h }[2], [x26], #0x2\n"
+ "tbz x27, #0, 16f\n"
+ "ld1 { v31.b }[6], [x26]\n"
+ "b 16f\n"
+ "13:" // Height 1: Multiply loop: Ragged operand read: partial_1_4
+ "tbz x27, #0, 16f\n"
+ "ld1 { v31.b }[4], [x26]\n"
+ "b 16f\n"
+ "14:" // Height 1: Multiply loop: Ragged operand read: partial_2_0
+ "tbz x27, #1, 15f\n"
+ "ldr h31, [x26], #0x2\n"
+ "tbz x27, #0, 16f\n"
+ "ld1 { v31.b }[2], [x26]\n"
+ "b 16f\n"
+ "15:" // Height 1: Multiply loop: Ragged operand read: partial_1_0
+ "ldr b31, [x26, #0x0]\n"
+ "16:" // Height 1: Multiply loop: Ragged operand read: Done
+ "sadalp v1.8h, v31.16b\n"
+ "17:" // Height 1: Multiply loop: No odd multiplies
+ "add x28, x28, #0x1\n"
+ "cmp x28, x20\n"
+ "bne 2b\n"
+ "sadalp v0.4s, v1.8h\n"
+ "addp v0.4s, v0.4s, v0.4s\n"
+ "addp v0.4s, v0.4s, v0.4s\n"
+ "mul v0.4s, v0.4s, v2.4s\n"
+ "str s0, [%x[out_ptr]], #0x4\n"
+ "b 104f\n"
+ "18:" // Height 2
+ "movi v1.8h, #0x0\n"
+ "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "mov x9, #0x0\n"
+ "movi v0.4s, #0x0\n"
+ "mov x28, #0x0\n"
+ "movi v30.8h, #0x0\n"
+ "movi v29.4s, #0x0\n"
+ "19:" // Height 2: String loop
+ "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr w27, [x19, x28, LSL #0x2]\n"
+ "tbz %x[flags], #3, 20f\n"
+ "ldr x19, [%x[input_ptr], x28, LSL #0x3]\n"
+ "add x19, x19, %x[input_offset], LSL #3\n"
+ "ldr x26, [x19, #0x0]\n"
+ "ldr x25, [x19, #0x8]\n"
+ "cbnz x28, 21f\n"
+ "ldr w19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x26, x26, x19\n"
+ "add x25, x25, x19\n"
+ "b 21f\n"
+ "20:" // Height 2: setup direct input
+ "mov x26, %x[input_ptr]\n"
+ "add x25, x26, %x[input_offset]\n"
+ "21:" // Height 2: input setup done
+ "cmp x27, #0x10\n"
+ "blt 25f\n"
+ "cmp x27, #0x20\n"
+ "blt 24f\n"
+ "22:" // Height 2: Multiply loop: Main loop head
+ "ldr q31, [x26, #0x0]\n"
+ "ldr q28, [x25, #0x0]\n"
+ "cmp x9, #0x7e\n"
+ "add x26, x26, #0x10\n"
+ "add x25, x25, #0x10\n"
+ "blt 23f\n"
+ "sadalp v0.4s, v1.8h\n"
+ "movi v1.8h, #0x0\n"
+ "sadalp v29.4s, v30.8h\n"
+ "movi v30.8h, #0x0\n"
+ "mov x9, #0x0\n"
+ "23:" // Height 2: Multiply loop: unique 2: no collapse
+ "sadalp v1.8h, v31.16b\n"
+ "sadalp v30.8h, v28.16b\n"
+ "add x9, x9, #0x1\n"
+ "sub x27, x27, #0x10\n"
+ "cmp x27, #0x20\n"
+ "bge 22b\n"
+ "24:" // Height 2: Multiply loop: Single iteration only
+ "sub x27, x27, #0x10\n"
+ "ldr q31, [x26, #0x0]\n"
+ "ldr q28, [x25, #0x0]\n"
+ "add x26, x26, #0x10\n"
+ "add x25, x25, #0x10\n"
+ "sadalp v1.8h, v31.16b\n"
+ "sadalp v30.8h, v28.16b\n"
+ "25:" // Height 2: Multiply loop: Main loop skip
+ "cbz x27, 34f\n"
+ "tbz x27, #3, 29f\n"
+ "ldr d31, [x26], #0x8\n"
+ "ldr d28, [x25], #0x8\n"
+ "tbz x27, #2, 27f\n"
+ "ld1 { v31.s }[2], [x26], #0x4\n"
+ "ld1 { v28.s }[2], [x25], #0x4\n"
+ "tbz x27, #1, 26f\n"
+ "ld1 { v31.h }[6], [x26], #0x2\n"
+ "ld1 { v28.h }[6], [x25], #0x2\n"
+ "tbz x27, #0, 33f\n"
+ "ld1 { v31.b }[14], [x26]\n"
+ "ld1 { v28.b }[14], [x25]\n"
+ "b 33f\n"
+ "26:" // Height 2: Multiply loop: Ragged operand read: partial_1_12
+ "tbz x27, #0, 33f\n"
+ "ld1 { v31.b }[12], [x26]\n"
+ "ld1 { v28.b }[12], [x25]\n"
+ "b 33f\n"
+ "27:" // Height 2: Multiply loop: Ragged operand read: partial_2_8
+ "tbz x27, #1, 28f\n"
+ "ld1 { v31.h }[4], [x26], #0x2\n"
+ "ld1 { v28.h }[4], [x25], #0x2\n"
+ "tbz x27, #0, 33f\n"
+ "ld1 { v31.b }[10], [x26]\n"
+ "ld1 { v28.b }[10], [x25]\n"
+ "b 33f\n"
+ "28:" // Height 2: Multiply loop: Ragged operand read: partial_1_8
+ "tbz x27, #0, 33f\n"
+ "ld1 { v31.b }[8], [x26]\n"
+ "ld1 { v28.b }[8], [x25]\n"
+ "b 33f\n"
+ "29:" // Height 2: Multiply loop: Ragged operand read: partial_4_0
+ "tbz x27, #2, 31f\n"
+ "ldr s31, [x26], #0x4\n"
+ "ldr s28, [x25], #0x4\n"
+ "tbz x27, #1, 30f\n"
+ "ld1 { v31.h }[2], [x26], #0x2\n"
+ "ld1 { v28.h }[2], [x25], #0x2\n"
+ "tbz x27, #0, 33f\n"
+ "ld1 { v31.b }[6], [x26]\n"
+ "ld1 { v28.b }[6], [x25]\n"
+ "b 33f\n"
+ "30:" // Height 2: Multiply loop: Ragged operand read: partial_1_4
+ "tbz x27, #0, 33f\n"
+ "ld1 { v31.b }[4], [x26]\n"
+ "ld1 { v28.b }[4], [x25]\n"
+ "b 33f\n"
+ "31:" // Height 2: Multiply loop: Ragged operand read: partial_2_0
+ "tbz x27, #1, 32f\n"
+ "ldr h31, [x26], #0x2\n"
+ "ldr h28, [x25], #0x2\n"
+ "tbz x27, #0, 33f\n"
+ "ld1 { v31.b }[2], [x26]\n"
+ "ld1 { v28.b }[2], [x25]\n"
+ "b 33f\n"
+ "32:" // Height 2: Multiply loop: Ragged operand read: partial_1_0
+ "ldr b31, [x26, #0x0]\n"
+ "ldr b28, [x25, #0x0]\n"
+ "33:" // Height 2: Multiply loop: Ragged operand read: Done
+ "sadalp v1.8h, v31.16b\n"
+ "sadalp v30.8h, v28.16b\n"
+ "34:" // Height 2: Multiply loop: No odd multiplies
+ "add x28, x28, #0x1\n"
+ "cmp x28, x20\n"
+ "bne 19b\n"
+ "sadalp v0.4s, v1.8h\n"
+ "sadalp v29.4s, v30.8h\n"
+ "addp v0.4s, v0.4s, v29.4s\n"
+ "addp v0.4s, v0.4s, v0.4s\n"
+ "mul v0.4s, v0.4s, v2.4s\n"
+ "str d0, [%x[out_ptr]], #0x8\n"
+ "b 104f\n"
+ "35:" // Height 3
+ "movi v1.8h, #0x0\n"
+ "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "mov x9, #0x0\n"
+ "movi v0.4s, #0x0\n"
+ "mov x28, #0x0\n"
+ "movi v30.8h, #0x0\n"
+ "movi v29.4s, #0x0\n"
+ "movi v27.8h, #0x0\n"
+ "movi v26.4s, #0x0\n"
+ "36:" // Height 3: String loop
+ "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr w27, [x19, x28, LSL #0x2]\n"
+ "tbz %x[flags], #3, 37f\n"
+ "ldr x19, [%x[input_ptr], x28, LSL #0x3]\n"
+ "add x19, x19, %x[input_offset], LSL #3\n"
+ "ldr x26, [x19, #0x0]\n"
+ "ldr x25, [x19, #0x8]\n"
+ "ldr x24, [x19, #0x10]\n"
+ "cbnz x28, 38f\n"
+ "ldr w19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x26, x26, x19\n"
+ "add x25, x25, x19\n"
+ "add x24, x24, x19\n"
+ "b 38f\n"
+ "37:" // Height 3: setup direct input
+ "mov x26, %x[input_ptr]\n"
+ "add x25, x26, %x[input_offset]\n"
+ "add x24, x25, %x[input_offset]\n"
+ "38:" // Height 3: input setup done
+ "cmp x27, #0x10\n"
+ "blt 42f\n"
+ "cmp x27, #0x20\n"
+ "blt 41f\n"
+ "39:" // Height 3: Multiply loop: Main loop head
+ "ldr q31, [x26, #0x0]\n"
+ "ldr q28, [x25, #0x0]\n"
+ "ldr q25, [x24, #0x0]\n"
+ "cmp x9, #0x7e\n"
+ "add x26, x26, #0x10\n"
+ "add x25, x25, #0x10\n"
+ "add x24, x24, #0x10\n"
+ "blt 40f\n"
+ "sadalp v0.4s, v1.8h\n"
+ "movi v1.8h, #0x0\n"
+ "sadalp v29.4s, v30.8h\n"
+ "movi v30.8h, #0x0\n"
+ "sadalp v26.4s, v27.8h\n"
+ "movi v27.8h, #0x0\n"
+ "mov x9, #0x0\n"
+ "40:" // Height 3: Multiply loop: unique 3: no collapse
+ "sadalp v1.8h, v31.16b\n"
+ "sadalp v30.8h, v28.16b\n"
+ "sadalp v27.8h, v25.16b\n"
+ "add x9, x9, #0x1\n"
+ "sub x27, x27, #0x10\n"
+ "cmp x27, #0x20\n"
+ "bge 39b\n"
+ "41:" // Height 3: Multiply loop: Single iteration only
+ "sub x27, x27, #0x10\n"
+ "ldr q31, [x26, #0x0]\n"
+ "ldr q28, [x25, #0x0]\n"
+ "ldr q25, [x24, #0x0]\n"
+ "add x26, x26, #0x10\n"
+ "add x25, x25, #0x10\n"
+ "sadalp v1.8h, v31.16b\n"
+ "sadalp v30.8h, v28.16b\n"
+ "sadalp v27.8h, v25.16b\n"
+ "add x24, x24, #0x10\n"
+ "42:" // Height 3: Multiply loop: Main loop skip
+ "cbz x27, 51f\n"
+ "tbz x27, #3, 46f\n"
+ "ldr d31, [x26], #0x8\n"
+ "ldr d28, [x25], #0x8\n"
+ "ldr d25, [x24], #0x8\n"
+ "tbz x27, #2, 44f\n"
+ "ld1 { v31.s }[2], [x26], #0x4\n"
+ "ld1 { v28.s }[2], [x25], #0x4\n"
+ "ld1 { v25.s }[2], [x24], #0x4\n"
+ "tbz x27, #1, 43f\n"
+ "ld1 { v31.h }[6], [x26], #0x2\n"
+ "ld1 { v28.h }[6], [x25], #0x2\n"
+ "ld1 { v25.h }[6], [x24], #0x2\n"
+ "tbz x27, #0, 50f\n"
+ "ld1 { v31.b }[14], [x26]\n"
+ "ld1 { v28.b }[14], [x25]\n"
+ "ld1 { v25.b }[14], [x24]\n"
+ "b 50f\n"
+ "43:" // Height 3: Multiply loop: Ragged operand read: partial_1_12
+ "tbz x27, #0, 50f\n"
+ "ld1 { v31.b }[12], [x26]\n"
+ "ld1 { v28.b }[12], [x25]\n"
+ "ld1 { v25.b }[12], [x24]\n"
+ "b 50f\n"
+ "44:" // Height 3: Multiply loop: Ragged operand read: partial_2_8
+ "tbz x27, #1, 45f\n"
+ "ld1 { v31.h }[4], [x26], #0x2\n"
+ "ld1 { v28.h }[4], [x25], #0x2\n"
+ "ld1 { v25.h }[4], [x24], #0x2\n"
+ "tbz x27, #0, 50f\n"
+ "ld1 { v31.b }[10], [x26]\n"
+ "ld1 { v28.b }[10], [x25]\n"
+ "ld1 { v25.b }[10], [x24]\n"
+ "b 50f\n"
+ "45:" // Height 3: Multiply loop: Ragged operand read: partial_1_8
+ "tbz x27, #0, 50f\n"
+ "ld1 { v31.b }[8], [x26]\n"
+ "ld1 { v28.b }[8], [x25]\n"
+ "ld1 { v25.b }[8], [x24]\n"
+ "b 50f\n"
+ "46:" // Height 3: Multiply loop: Ragged operand read: partial_4_0
+ "tbz x27, #2, 48f\n"
+ "ldr s31, [x26], #0x4\n"
+ "ldr s28, [x25], #0x4\n"
+ "ldr s25, [x24], #0x4\n"
+ "tbz x27, #1, 47f\n"
+ "ld1 { v31.h }[2], [x26], #0x2\n"
+ "ld1 { v28.h }[2], [x25], #0x2\n"
+ "ld1 { v25.h }[2], [x24], #0x2\n"
+ "tbz x27, #0, 50f\n"
+ "ld1 { v31.b }[6], [x26]\n"
+ "ld1 { v28.b }[6], [x25]\n"
+ "ld1 { v25.b }[6], [x24]\n"
+ "b 50f\n"
+ "47:" // Height 3: Multiply loop: Ragged operand read: partial_1_4
+ "tbz x27, #0, 50f\n"
+ "ld1 { v31.b }[4], [x26]\n"
+ "ld1 { v28.b }[4], [x25]\n"
+ "ld1 { v25.b }[4], [x24]\n"
+ "b 50f\n"
+ "48:" // Height 3: Multiply loop: Ragged operand read: partial_2_0
+ "tbz x27, #1, 49f\n"
+ "ldr h31, [x26], #0x2\n"
+ "ldr h28, [x25], #0x2\n"
+ "ldr h25, [x24], #0x2\n"
+ "tbz x27, #0, 50f\n"
+ "ld1 { v31.b }[2], [x26]\n"
+ "ld1 { v28.b }[2], [x25]\n"
+ "ld1 { v25.b }[2], [x24]\n"
+ "b 50f\n"
+ "49:" // Height 3: Multiply loop: Ragged operand read: partial_1_0
+ "ldr b31, [x26, #0x0]\n"
+ "ldr b28, [x25, #0x0]\n"
+ "ldr b25, [x24, #0x0]\n"
+ "50:" // Height 3: Multiply loop: Ragged operand read: Done
+ "sadalp v1.8h, v31.16b\n"
+ "sadalp v30.8h, v28.16b\n"
+ "sadalp v27.8h, v25.16b\n"
+ "51:" // Height 3: Multiply loop: No odd multiplies
+ "add x28, x28, #0x1\n"
+ "cmp x28, x20\n"
+ "bne 36b\n"
+ "sadalp v0.4s, v1.8h\n"
+ "sadalp v29.4s, v30.8h\n"
+ "addp v0.4s, v0.4s, v29.4s\n"
+ "sadalp v26.4s, v27.8h\n"
+ "addp v0.4s, v0.4s, v0.4s\n"
+ "addp v26.4s, v26.4s, v26.4s\n"
+ "mul v0.4s, v0.4s, v2.4s\n"
+ "str d0, [%x[out_ptr]], #0x8\n"
+ "addp v26.4s, v26.4s, v26.4s\n"
+ "mul v26.4s, v26.4s, v2.4s\n"
+ "str s26, [%x[out_ptr]], #0x4\n"
+ "b 104f\n"
+ "52:" // Height 4
+ "movi v1.8h, #0x0\n"
+ "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "mov x9, #0x0\n"
+ "movi v0.4s, #0x0\n"
+ "mov x28, #0x0\n"
+ "movi v30.8h, #0x0\n"
+ "movi v29.4s, #0x0\n"
+ "movi v27.8h, #0x0\n"
+ "movi v26.4s, #0x0\n"
+ "movi v24.8h, #0x0\n"
+ "movi v23.4s, #0x0\n"
+ "53:" // Height 4: String loop
+ "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr w27, [x19, x28, LSL #0x2]\n"
+ "tbz %x[flags], #3, 54f\n"
+ "ldr x19, [%x[input_ptr], x28, LSL #0x3]\n"
+ "add x19, x19, %x[input_offset], LSL #3\n"
+ "ldr x26, [x19, #0x0]\n"
+ "ldr x25, [x19, #0x8]\n"
+ "ldr x24, [x19, #0x10]\n"
+ "ldr x23, [x19, #0x18]\n"
+ "cbnz x28, 55f\n"
+ "ldr w19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x26, x26, x19\n"
+ "add x25, x25, x19\n"
+ "add x24, x24, x19\n"
+ "add x23, x23, x19\n"
+ "b 55f\n"
+ "54:" // Height 4: setup direct input
+ "mov x26, %x[input_ptr]\n"
+ "add x25, x26, %x[input_offset]\n"
+ "add x24, x25, %x[input_offset]\n"
+ "add x23, x24, %x[input_offset]\n"
+ "55:" // Height 4: input setup done
+ "cmp x27, #0x10\n"
+ "blt 59f\n"
+ "cmp x27, #0x20\n"
+ "blt 58f\n"
+ "56:" // Height 4: Multiply loop: Main loop head
+ "ldr q31, [x26, #0x0]\n"
+ "ldr q28, [x25, #0x0]\n"
+ "ldr q25, [x24, #0x0]\n"
+ "ldr q22, [x23, #0x0]\n"
+ "cmp x9, #0x7e\n"
+ "add x26, x26, #0x10\n"
+ "add x25, x25, #0x10\n"
+ "add x24, x24, #0x10\n"
+ "add x23, x23, #0x10\n"
+ "blt 57f\n"
+ "sadalp v0.4s, v1.8h\n"
+ "movi v1.8h, #0x0\n"
+ "sadalp v29.4s, v30.8h\n"
+ "movi v30.8h, #0x0\n"
+ "sadalp v26.4s, v27.8h\n"
+ "movi v27.8h, #0x0\n"
+ "sadalp v23.4s, v24.8h\n"
+ "movi v24.8h, #0x0\n"
+ "mov x9, #0x0\n"
+ "57:" // Height 4: Multiply loop: unique 4: no collapse
+ "sadalp v1.8h, v31.16b\n"
+ "sadalp v30.8h, v28.16b\n"
+ "sadalp v27.8h, v25.16b\n"
+ "sadalp v24.8h, v22.16b\n"
+ "add x9, x9, #0x1\n"
+ "sub x27, x27, #0x10\n"
+ "cmp x27, #0x20\n"
+ "bge 56b\n"
+ "58:" // Height 4: Multiply loop: Single iteration only
+ "sub x27, x27, #0x10\n"
+ "ldr q31, [x26, #0x0]\n"
+ "ldr q28, [x25, #0x0]\n"
+ "ldr q25, [x24, #0x0]\n"
+ "ldr q22, [x23, #0x0]\n"
+ "add x26, x26, #0x10\n"
+ "sadalp v1.8h, v31.16b\n"
+ "sadalp v30.8h, v28.16b\n"
+ "sadalp v27.8h, v25.16b\n"
+ "sadalp v24.8h, v22.16b\n"
+ "add x25, x25, #0x10\n"
+ "add x24, x24, #0x10\n"
+ "add x23, x23, #0x10\n"
+ "59:" // Height 4: Multiply loop: Main loop skip
+ "cbz x27, 68f\n"
+ "tbz x27, #3, 63f\n"
+ "ldr d31, [x26], #0x8\n"
+ "ldr d28, [x25], #0x8\n"
+ "ldr d25, [x24], #0x8\n"
+ "ldr d22, [x23], #0x8\n"
+ "tbz x27, #2, 61f\n"
+ "ld1 { v31.s }[2], [x26], #0x4\n"
+ "ld1 { v28.s }[2], [x25], #0x4\n"
+ "ld1 { v25.s }[2], [x24], #0x4\n"
+ "ld1 { v22.s }[2], [x23], #0x4\n"
+ "tbz x27, #1, 60f\n"
+ "ld1 { v31.h }[6], [x26], #0x2\n"
+ "ld1 { v28.h }[6], [x25], #0x2\n"
+ "ld1 { v25.h }[6], [x24], #0x2\n"
+ "ld1 { v22.h }[6], [x23], #0x2\n"
+ "tbz x27, #0, 67f\n"
+ "ld1 { v31.b }[14], [x26]\n"
+ "ld1 { v28.b }[14], [x25]\n"
+ "ld1 { v25.b }[14], [x24]\n"
+ "ld1 { v22.b }[14], [x23]\n"
+ "b 67f\n"
+ "60:" // Height 4: Multiply loop: Ragged operand read: partial_1_12
+ "tbz x27, #0, 67f\n"
+ "ld1 { v31.b }[12], [x26]\n"
+ "ld1 { v28.b }[12], [x25]\n"
+ "ld1 { v25.b }[12], [x24]\n"
+ "ld1 { v22.b }[12], [x23]\n"
+ "b 67f\n"
+ "61:" // Height 4: Multiply loop: Ragged operand read: partial_2_8
+ "tbz x27, #1, 62f\n"
+ "ld1 { v31.h }[4], [x26], #0x2\n"
+ "ld1 { v28.h }[4], [x25], #0x2\n"
+ "ld1 { v25.h }[4], [x24], #0x2\n"
+ "ld1 { v22.h }[4], [x23], #0x2\n"
+ "tbz x27, #0, 67f\n"
+ "ld1 { v31.b }[10], [x26]\n"
+ "ld1 { v28.b }[10], [x25]\n"
+ "ld1 { v25.b }[10], [x24]\n"
+ "ld1 { v22.b }[10], [x23]\n"
+ "b 67f\n"
+ "62:" // Height 4: Multiply loop: Ragged operand read: partial_1_8
+ "tbz x27, #0, 67f\n"
+ "ld1 { v31.b }[8], [x26]\n"
+ "ld1 { v28.b }[8], [x25]\n"
+ "ld1 { v25.b }[8], [x24]\n"
+ "ld1 { v22.b }[8], [x23]\n"
+ "b 67f\n"
+ "63:" // Height 4: Multiply loop: Ragged operand read: partial_4_0
+ "tbz x27, #2, 65f\n"
+ "ldr s31, [x26], #0x4\n"
+ "ldr s28, [x25], #0x4\n"
+ "ldr s25, [x24], #0x4\n"
+ "ldr s22, [x23], #0x4\n"
+ "tbz x27, #1, 64f\n"
+ "ld1 { v31.h }[2], [x26], #0x2\n"
+ "ld1 { v28.h }[2], [x25], #0x2\n"
+ "ld1 { v25.h }[2], [x24], #0x2\n"
+ "ld1 { v22.h }[2], [x23], #0x2\n"
+ "tbz x27, #0, 67f\n"
+ "ld1 { v31.b }[6], [x26]\n"
+ "ld1 { v28.b }[6], [x25]\n"
+ "ld1 { v25.b }[6], [x24]\n"
+ "ld1 { v22.b }[6], [x23]\n"
+ "b 67f\n"
+ "64:" // Height 4: Multiply loop: Ragged operand read: partial_1_4
+ "tbz x27, #0, 67f\n"
+ "ld1 { v31.b }[4], [x26]\n"
+ "ld1 { v28.b }[4], [x25]\n"
+ "ld1 { v25.b }[4], [x24]\n"
+ "ld1 { v22.b }[4], [x23]\n"
+ "b 67f\n"
+ "65:" // Height 4: Multiply loop: Ragged operand read: partial_2_0
+ "tbz x27, #1, 66f\n"
+ "ldr h31, [x26], #0x2\n"
+ "ldr h28, [x25], #0x2\n"
+ "ldr h25, [x24], #0x2\n"
+ "ldr h22, [x23], #0x2\n"
+ "tbz x27, #0, 67f\n"
+ "ld1 { v31.b }[2], [x26]\n"
+ "ld1 { v28.b }[2], [x25]\n"
+ "ld1 { v25.b }[2], [x24]\n"
+ "ld1 { v22.b }[2], [x23]\n"
+ "b 67f\n"
+ "66:" // Height 4: Multiply loop: Ragged operand read: partial_1_0
+ "ldr b31, [x26, #0x0]\n"
+ "ldr b28, [x25, #0x0]\n"
+ "ldr b25, [x24, #0x0]\n"
+ "ldr b22, [x23, #0x0]\n"
+ "67:" // Height 4: Multiply loop: Ragged operand read: Done
+ "sadalp v1.8h, v31.16b\n"
+ "sadalp v30.8h, v28.16b\n"
+ "sadalp v27.8h, v25.16b\n"
+ "sadalp v24.8h, v22.16b\n"
+ "68:" // Height 4: Multiply loop: No odd multiplies
+ "add x28, x28, #0x1\n"
+ "cmp x28, x20\n"
+ "bne 53b\n"
+ "sadalp v0.4s, v1.8h\n"
+ "sadalp v29.4s, v30.8h\n"
+ "addp v0.4s, v0.4s, v29.4s\n"
+ "sadalp v26.4s, v27.8h\n"
+ "sadalp v23.4s, v24.8h\n"
+ "addp v29.4s, v26.4s, v23.4s\n"
+ "addp v0.4s, v0.4s, v29.4s\n"
+ "mul v0.4s, v0.4s, v2.4s\n"
+ "st1 { v0.4s }, [%x[out_ptr]], #0x10\n"
+ "b 104f\n"
+ "69:" // Height 5
+ "movi v1.8h, #0x0\n"
+ "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "mov x9, #0x0\n"
+ "movi v0.4s, #0x0\n"
+ "mov x28, #0x0\n"
+ "movi v30.8h, #0x0\n"
+ "movi v29.4s, #0x0\n"
+ "movi v27.8h, #0x0\n"
+ "movi v26.4s, #0x0\n"
+ "movi v24.8h, #0x0\n"
+ "movi v23.4s, #0x0\n"
+ "movi v21.8h, #0x0\n"
+ "movi v20.4s, #0x0\n"
+ "70:" // Height 5: String loop
+ "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr w27, [x19, x28, LSL #0x2]\n"
+ "tbz %x[flags], #3, 71f\n"
+ "ldr x19, [%x[input_ptr], x28, LSL #0x3]\n"
+ "add x19, x19, %x[input_offset], LSL #3\n"
+ "ldr x26, [x19, #0x0]\n"
+ "ldr x25, [x19, #0x8]\n"
+ "ldr x24, [x19, #0x10]\n"
+ "ldr x23, [x19, #0x18]\n"
+ "ldr x22, [x19, #0x20]\n"
+ "cbnz x28, 72f\n"
+ "ldr w19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x26, x26, x19\n"
+ "add x25, x25, x19\n"
+ "add x24, x24, x19\n"
+ "add x23, x23, x19\n"
+ "add x22, x22, x19\n"
+ "b 72f\n"
+ "71:" // Height 5: setup direct input
+ "mov x26, %x[input_ptr]\n"
+ "add x25, x26, %x[input_offset]\n"
+ "add x24, x25, %x[input_offset]\n"
+ "add x23, x24, %x[input_offset]\n"
+ "add x22, x23, %x[input_offset]\n"
+ "72:" // Height 5: input setup done
+ "cmp x27, #0x10\n"
+ "blt 76f\n"
+ "cmp x27, #0x20\n"
+ "blt 75f\n"
+ "73:" // Height 5: Multiply loop: Main loop head
+ "ldr q31, [x26, #0x0]\n"
+ "ldr q28, [x25, #0x0]\n"
+ "ldr q25, [x24, #0x0]\n"
+ "ldr q22, [x23, #0x0]\n"
+ "ldr q19, [x22, #0x0]\n"
+ "cmp x9, #0x7e\n"
+ "add x26, x26, #0x10\n"
+ "add x25, x25, #0x10\n"
+ "add x24, x24, #0x10\n"
+ "add x23, x23, #0x10\n"
+ "add x22, x22, #0x10\n"
+ "blt 74f\n"
+ "sadalp v0.4s, v1.8h\n"
+ "movi v1.8h, #0x0\n"
+ "sadalp v29.4s, v30.8h\n"
+ "movi v30.8h, #0x0\n"
+ "sadalp v26.4s, v27.8h\n"
+ "movi v27.8h, #0x0\n"
+ "sadalp v23.4s, v24.8h\n"
+ "movi v24.8h, #0x0\n"
+ "sadalp v20.4s, v21.8h\n"
+ "movi v21.8h, #0x0\n"
+ "mov x9, #0x0\n"
+ "74:" // Height 5: Multiply loop: unique 5: no collapse
+ "sadalp v1.8h, v31.16b\n"
+ "sadalp v30.8h, v28.16b\n"
+ "sadalp v27.8h, v25.16b\n"
+ "sadalp v24.8h, v22.16b\n"
+ "sadalp v21.8h, v19.16b\n"
+ "add x9, x9, #0x1\n"
+ "sub x27, x27, #0x10\n"
+ "cmp x27, #0x20\n"
+ "bge 73b\n"
+ "75:" // Height 5: Multiply loop: Single iteration only
+ "sub x27, x27, #0x10\n"
+ "ldr q31, [x26, #0x0]\n"
+ "ldr q28, [x25, #0x0]\n"
+ "ldr q25, [x24, #0x0]\n"
+ "ldr q22, [x23, #0x0]\n"
+ "ldr q19, [x22, #0x0]\n"
+ "sadalp v1.8h, v31.16b\n"
+ "sadalp v30.8h, v28.16b\n"
+ "sadalp v27.8h, v25.16b\n"
+ "sadalp v24.8h, v22.16b\n"
+ "sadalp v21.8h, v19.16b\n"
+ "add x26, x26, #0x10\n"
+ "add x25, x25, #0x10\n"
+ "add x24, x24, #0x10\n"
+ "add x23, x23, #0x10\n"
+ "add x22, x22, #0x10\n"
+ "76:" // Height 5: Multiply loop: Main loop skip
+ "cbz x27, 85f\n"
+ "tbz x27, #3, 80f\n"
+ "ldr d31, [x26], #0x8\n"
+ "ldr d28, [x25], #0x8\n"
+ "ldr d25, [x24], #0x8\n"
+ "ldr d22, [x23], #0x8\n"
+ "ldr d19, [x22], #0x8\n"
+ "tbz x27, #2, 78f\n"
+ "ld1 { v31.s }[2], [x26], #0x4\n"
+ "ld1 { v28.s }[2], [x25], #0x4\n"
+ "ld1 { v25.s }[2], [x24], #0x4\n"
+ "ld1 { v22.s }[2], [x23], #0x4\n"
+ "ld1 { v19.s }[2], [x22], #0x4\n"
+ "tbz x27, #1, 77f\n"
+ "ld1 { v31.h }[6], [x26], #0x2\n"
+ "ld1 { v28.h }[6], [x25], #0x2\n"
+ "ld1 { v25.h }[6], [x24], #0x2\n"
+ "ld1 { v22.h }[6], [x23], #0x2\n"
+ "ld1 { v19.h }[6], [x22], #0x2\n"
+ "tbz x27, #0, 84f\n"
+ "ld1 { v31.b }[14], [x26]\n"
+ "ld1 { v28.b }[14], [x25]\n"
+ "ld1 { v25.b }[14], [x24]\n"
+ "ld1 { v22.b }[14], [x23]\n"
+ "ld1 { v19.b }[14], [x22]\n"
+ "b 84f\n"
+ "77:" // Height 5: Multiply loop: Ragged operand read: partial_1_12
+ "tbz x27, #0, 84f\n"
+ "ld1 { v31.b }[12], [x26]\n"
+ "ld1 { v28.b }[12], [x25]\n"
+ "ld1 { v25.b }[12], [x24]\n"
+ "ld1 { v22.b }[12], [x23]\n"
+ "ld1 { v19.b }[12], [x22]\n"
+ "b 84f\n"
+ "78:" // Height 5: Multiply loop: Ragged operand read: partial_2_8
+ "tbz x27, #1, 79f\n"
+ "ld1 { v31.h }[4], [x26], #0x2\n"
+ "ld1 { v28.h }[4], [x25], #0x2\n"
+ "ld1 { v25.h }[4], [x24], #0x2\n"
+ "ld1 { v22.h }[4], [x23], #0x2\n"
+ "ld1 { v19.h }[4], [x22], #0x2\n"
+ "tbz x27, #0, 84f\n"
+ "ld1 { v31.b }[10], [x26]\n"
+ "ld1 { v28.b }[10], [x25]\n"
+ "ld1 { v25.b }[10], [x24]\n"
+ "ld1 { v22.b }[10], [x23]\n"
+ "ld1 { v19.b }[10], [x22]\n"
+ "b 84f\n"
+ "79:" // Height 5: Multiply loop: Ragged operand read: partial_1_8
+ "tbz x27, #0, 84f\n"
+ "ld1 { v31.b }[8], [x26]\n"
+ "ld1 { v28.b }[8], [x25]\n"
+ "ld1 { v25.b }[8], [x24]\n"
+ "ld1 { v22.b }[8], [x23]\n"
+ "ld1 { v19.b }[8], [x22]\n"
+ "b 84f\n"
+ "80:" // Height 5: Multiply loop: Ragged operand read: partial_4_0
+ "tbz x27, #2, 82f\n"
+ "ldr s31, [x26], #0x4\n"
+ "ldr s28, [x25], #0x4\n"
+ "ldr s25, [x24], #0x4\n"
+ "ldr s22, [x23], #0x4\n"
+ "ldr s19, [x22], #0x4\n"
+ "tbz x27, #1, 81f\n"
+ "ld1 { v31.h }[2], [x26], #0x2\n"
+ "ld1 { v28.h }[2], [x25], #0x2\n"
+ "ld1 { v25.h }[2], [x24], #0x2\n"
+ "ld1 { v22.h }[2], [x23], #0x2\n"
+ "ld1 { v19.h }[2], [x22], #0x2\n"
+ "tbz x27, #0, 84f\n"
+ "ld1 { v31.b }[6], [x26]\n"
+ "ld1 { v28.b }[6], [x25]\n"
+ "ld1 { v25.b }[6], [x24]\n"
+ "ld1 { v22.b }[6], [x23]\n"
+ "ld1 { v19.b }[6], [x22]\n"
+ "b 84f\n"
+ "81:" // Height 5: Multiply loop: Ragged operand read: partial_1_4
+ "tbz x27, #0, 84f\n"
+ "ld1 { v31.b }[4], [x26]\n"
+ "ld1 { v28.b }[4], [x25]\n"
+ "ld1 { v25.b }[4], [x24]\n"
+ "ld1 { v22.b }[4], [x23]\n"
+ "ld1 { v19.b }[4], [x22]\n"
+ "b 84f\n"
+ "82:" // Height 5: Multiply loop: Ragged operand read: partial_2_0
+ "tbz x27, #1, 83f\n"
+ "ldr h31, [x26], #0x2\n"
+ "ldr h28, [x25], #0x2\n"
+ "ldr h25, [x24], #0x2\n"
+ "ldr h22, [x23], #0x2\n"
+ "ldr h19, [x22], #0x2\n"
+ "tbz x27, #0, 84f\n"
+ "ld1 { v31.b }[2], [x26]\n"
+ "ld1 { v28.b }[2], [x25]\n"
+ "ld1 { v25.b }[2], [x24]\n"
+ "ld1 { v22.b }[2], [x23]\n"
+ "ld1 { v19.b }[2], [x22]\n"
+ "b 84f\n"
+ "83:" // Height 5: Multiply loop: Ragged operand read: partial_1_0
+ "ldr b31, [x26, #0x0]\n"
+ "ldr b28, [x25, #0x0]\n"
+ "ldr b25, [x24, #0x0]\n"
+ "ldr b22, [x23, #0x0]\n"
+ "ldr b19, [x22, #0x0]\n"
+ "84:" // Height 5: Multiply loop: Ragged operand read: Done
+ "sadalp v1.8h, v31.16b\n"
+ "sadalp v30.8h, v28.16b\n"
+ "sadalp v27.8h, v25.16b\n"
+ "sadalp v24.8h, v22.16b\n"
+ "sadalp v21.8h, v19.16b\n"
+ "85:" // Height 5: Multiply loop: No odd multiplies
+ "add x28, x28, #0x1\n"
+ "cmp x28, x20\n"
+ "bne 70b\n"
+ "sadalp v0.4s, v1.8h\n"
+ "sadalp v29.4s, v30.8h\n"
+ "addp v0.4s, v0.4s, v29.4s\n"
+ "sadalp v26.4s, v27.8h\n"
+ "sadalp v23.4s, v24.8h\n"
+ "addp v29.4s, v26.4s, v23.4s\n"
+ "sadalp v20.4s, v21.8h\n"
+ "addp v0.4s, v0.4s, v29.4s\n"
+ "addp v20.4s, v20.4s, v20.4s\n"
+ "mul v0.4s, v0.4s, v2.4s\n"
+ "st1 { v0.4s }, [%x[out_ptr]], #0x10\n"
+ "addp v20.4s, v20.4s, v20.4s\n"
+ "mul v20.4s, v20.4s, v2.4s\n"
+ "str s20, [%x[out_ptr]], #0x4\n"
+ "b 104f\n"
+ "86:" // Height 6
+ "movi v1.8h, #0x0\n"
+ "ldr w21, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "mov x9, #0x0\n"
+ "movi v0.4s, #0x0\n"
+ "mov x28, #0x0\n"
+ "movi v30.8h, #0x0\n"
+ "movi v29.4s, #0x0\n"
+ "movi v27.8h, #0x0\n"
+ "movi v26.4s, #0x0\n"
+ "movi v24.8h, #0x0\n"
+ "movi v23.4s, #0x0\n"
+ "movi v21.8h, #0x0\n"
+ "movi v20.4s, #0x0\n"
+ "movi v18.8h, #0x0\n"
+ "movi v17.4s, #0x0\n"
+ "87:" // Height 6: String loop
+ "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr w27, [x19, x28, LSL #0x2]\n"
+ "tbz %x[flags], #3, 88f\n"
+ "ldr x19, [%x[input_ptr], x28, LSL #0x3]\n"
+ "add x19, x19, %x[input_offset], LSL #3\n"
+ "ldr x26, [x19, #0x0]\n"
+ "ldr x25, [x19, #0x8]\n"
+ "ldr x24, [x19, #0x10]\n"
+ "ldr x23, [x19, #0x18]\n"
+ "ldr x22, [x19, #0x20]\n"
+ "ldr x20, [x19, #0x28]\n"
+ "cbnz x28, 89f\n"
+ "ldr w19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x26, x26, x19\n"
+ "add x25, x25, x19\n"
+ "add x24, x24, x19\n"
+ "add x23, x23, x19\n"
+ "add x22, x22, x19\n"
+ "add x20, x20, x19\n"
+ "b 89f\n"
+ "88:" // Height 6: setup direct input
+ "mov x26, %x[input_ptr]\n"
+ "add x25, x26, %x[input_offset]\n"
+ "add x24, x25, %x[input_offset]\n"
+ "add x23, x24, %x[input_offset]\n"
+ "add x22, x23, %x[input_offset]\n"
+ "add x20, x22, %x[input_offset]\n"
+ "89:" // Height 6: input setup done
+ "cmp x27, #0x10\n"
+ "blt 93f\n"
+ "cmp x27, #0x20\n"
+ "blt 92f\n"
+ "90:" // Height 6: Multiply loop: Main loop head
+ "ldr q31, [x26, #0x0]\n"
+ "ldr q28, [x25, #0x0]\n"
+ "ldr q25, [x24, #0x0]\n"
+ "ldr q22, [x23, #0x0]\n"
+ "ldr q19, [x22, #0x0]\n"
+ "ldr q16, [x20, #0x0]\n"
+ "cmp x9, #0x7e\n"
+ "add x26, x26, #0x10\n"
+ "add x25, x25, #0x10\n"
+ "add x24, x24, #0x10\n"
+ "add x23, x23, #0x10\n"
+ "add x22, x22, #0x10\n"
+ "add x20, x20, #0x10\n"
+ "blt 91f\n"
+ "sadalp v0.4s, v1.8h\n"
+ "movi v1.8h, #0x0\n"
+ "sadalp v29.4s, v30.8h\n"
+ "movi v30.8h, #0x0\n"
+ "sadalp v26.4s, v27.8h\n"
+ "movi v27.8h, #0x0\n"
+ "sadalp v23.4s, v24.8h\n"
+ "movi v24.8h, #0x0\n"
+ "sadalp v20.4s, v21.8h\n"
+ "movi v21.8h, #0x0\n"
+ "sadalp v17.4s, v18.8h\n"
+ "movi v18.8h, #0x0\n"
+ "mov x9, #0x0\n"
+ "91:" // Height 6: Multiply loop: unique 6: no collapse
+ "sadalp v1.8h, v31.16b\n"
+ "sadalp v30.8h, v28.16b\n"
+ "sadalp v27.8h, v25.16b\n"
+ "sadalp v24.8h, v22.16b\n"
+ "sadalp v21.8h, v19.16b\n"
+ "sadalp v18.8h, v16.16b\n"
+ "add x9, x9, #0x1\n"
+ "sub x27, x27, #0x10\n"
+ "cmp x27, #0x20\n"
+ "bge 90b\n"
+ "92:" // Height 6: Multiply loop: Single iteration only
+ "sub x27, x27, #0x10\n"
+ "ldr q31, [x26, #0x0]\n"
+ "ldr q28, [x25, #0x0]\n"
+ "ldr q25, [x24, #0x0]\n"
+ "ldr q22, [x23, #0x0]\n"
+ "ldr q19, [x22, #0x0]\n"
+ "ldr q16, [x20, #0x0]\n"
+ "sadalp v1.8h, v31.16b\n"
+ "sadalp v30.8h, v28.16b\n"
+ "sadalp v27.8h, v25.16b\n"
+ "sadalp v24.8h, v22.16b\n"
+ "sadalp v21.8h, v19.16b\n"
+ "sadalp v18.8h, v16.16b\n"
+ "add x26, x26, #0x10\n"
+ "add x25, x25, #0x10\n"
+ "add x24, x24, #0x10\n"
+ "add x23, x23, #0x10\n"
+ "add x22, x22, #0x10\n"
+ "add x20, x20, #0x10\n"
+ "93:" // Height 6: Multiply loop: Main loop skip
+ "cbz x27, 102f\n"
+ "tbz x27, #3, 97f\n"
+ "ldr d31, [x26], #0x8\n"
+ "ldr d28, [x25], #0x8\n"
+ "ldr d25, [x24], #0x8\n"
+ "ldr d22, [x23], #0x8\n"
+ "ldr d19, [x22], #0x8\n"
+ "ldr d16, [x20], #0x8\n"
+ "tbz x27, #2, 95f\n"
+ "ld1 { v31.s }[2], [x26], #0x4\n"
+ "ld1 { v28.s }[2], [x25], #0x4\n"
+ "ld1 { v25.s }[2], [x24], #0x4\n"
+ "ld1 { v22.s }[2], [x23], #0x4\n"
+ "ld1 { v19.s }[2], [x22], #0x4\n"
+ "ld1 { v16.s }[2], [x20], #0x4\n"
+ "tbz x27, #1, 94f\n"
+ "ld1 { v31.h }[6], [x26], #0x2\n"
+ "ld1 { v28.h }[6], [x25], #0x2\n"
+ "ld1 { v25.h }[6], [x24], #0x2\n"
+ "ld1 { v22.h }[6], [x23], #0x2\n"
+ "ld1 { v19.h }[6], [x22], #0x2\n"
+ "ld1 { v16.h }[6], [x20], #0x2\n"
+ "tbz x27, #0, 101f\n"
+ "ld1 { v31.b }[14], [x26]\n"
+ "ld1 { v28.b }[14], [x25]\n"
+ "ld1 { v25.b }[14], [x24]\n"
+ "ld1 { v22.b }[14], [x23]\n"
+ "ld1 { v19.b }[14], [x22]\n"
+ "ld1 { v16.b }[14], [x20]\n"
+ "b 101f\n"
+ "94:" // Height 6: Multiply loop: Ragged operand read: partial_1_12
+ "tbz x27, #0, 101f\n"
+ "ld1 { v31.b }[12], [x26]\n"
+ "ld1 { v28.b }[12], [x25]\n"
+ "ld1 { v25.b }[12], [x24]\n"
+ "ld1 { v22.b }[12], [x23]\n"
+ "ld1 { v19.b }[12], [x22]\n"
+ "ld1 { v16.b }[12], [x20]\n"
+ "b 101f\n"
+ "95:" // Height 6: Multiply loop: Ragged operand read: partial_2_8
+ "tbz x27, #1, 96f\n"
+ "ld1 { v31.h }[4], [x26], #0x2\n"
+ "ld1 { v28.h }[4], [x25], #0x2\n"
+ "ld1 { v25.h }[4], [x24], #0x2\n"
+ "ld1 { v22.h }[4], [x23], #0x2\n"
+ "ld1 { v19.h }[4], [x22], #0x2\n"
+ "ld1 { v16.h }[4], [x20], #0x2\n"
+ "tbz x27, #0, 101f\n"
+ "ld1 { v31.b }[10], [x26]\n"
+ "ld1 { v28.b }[10], [x25]\n"
+ "ld1 { v25.b }[10], [x24]\n"
+ "ld1 { v22.b }[10], [x23]\n"
+ "ld1 { v19.b }[10], [x22]\n"
+ "ld1 { v16.b }[10], [x20]\n"
+ "b 101f\n"
+ "96:" // Height 6: Multiply loop: Ragged operand read: partial_1_8
+ "tbz x27, #0, 101f\n"
+ "ld1 { v31.b }[8], [x26]\n"
+ "ld1 { v28.b }[8], [x25]\n"
+ "ld1 { v25.b }[8], [x24]\n"
+ "ld1 { v22.b }[8], [x23]\n"
+ "ld1 { v19.b }[8], [x22]\n"
+ "ld1 { v16.b }[8], [x20]\n"
+ "b 101f\n"
+ "97:" // Height 6: Multiply loop: Ragged operand read: partial_4_0
+ "tbz x27, #2, 99f\n"
+ "ldr s31, [x26], #0x4\n"
+ "ldr s28, [x25], #0x4\n"
+ "ldr s25, [x24], #0x4\n"
+ "ldr s22, [x23], #0x4\n"
+ "ldr s19, [x22], #0x4\n"
+ "ldr s16, [x20], #0x4\n"
+ "tbz x27, #1, 98f\n"
+ "ld1 { v31.h }[2], [x26], #0x2\n"
+ "ld1 { v28.h }[2], [x25], #0x2\n"
+ "ld1 { v25.h }[2], [x24], #0x2\n"
+ "ld1 { v22.h }[2], [x23], #0x2\n"
+ "ld1 { v19.h }[2], [x22], #0x2\n"
+ "ld1 { v16.h }[2], [x20], #0x2\n"
+ "tbz x27, #0, 101f\n"
+ "ld1 { v31.b }[6], [x26]\n"
+ "ld1 { v28.b }[6], [x25]\n"
+ "ld1 { v25.b }[6], [x24]\n"
+ "ld1 { v22.b }[6], [x23]\n"
+ "ld1 { v19.b }[6], [x22]\n"
+ "ld1 { v16.b }[6], [x20]\n"
+ "b 101f\n"
+ "98:" // Height 6: Multiply loop: Ragged operand read: partial_1_4
+ "tbz x27, #0, 101f\n"
+ "ld1 { v31.b }[4], [x26]\n"
+ "ld1 { v28.b }[4], [x25]\n"
+ "ld1 { v25.b }[4], [x24]\n"
+ "ld1 { v22.b }[4], [x23]\n"
+ "ld1 { v19.b }[4], [x22]\n"
+ "ld1 { v16.b }[4], [x20]\n"
+ "b 101f\n"
+ "99:" // Height 6: Multiply loop: Ragged operand read: partial_2_0
+ "tbz x27, #1, 100f\n"
+ "ldr h31, [x26], #0x2\n"
+ "ldr h28, [x25], #0x2\n"
+ "ldr h25, [x24], #0x2\n"
+ "ldr h22, [x23], #0x2\n"
+ "ldr h19, [x22], #0x2\n"
+ "ldr h16, [x20], #0x2\n"
+ "tbz x27, #0, 101f\n"
+ "ld1 { v31.b }[2], [x26]\n"
+ "ld1 { v28.b }[2], [x25]\n"
+ "ld1 { v25.b }[2], [x24]\n"
+ "ld1 { v22.b }[2], [x23]\n"
+ "ld1 { v19.b }[2], [x22]\n"
+ "ld1 { v16.b }[2], [x20]\n"
+ "b 101f\n"
+ "100:" // Height 6: Multiply loop: Ragged operand read: partial_1_0
+ "ldr b31, [x26, #0x0]\n"
+ "ldr b28, [x25, #0x0]\n"
+ "ldr b25, [x24, #0x0]\n"
+ "ldr b22, [x23, #0x0]\n"
+ "ldr b19, [x22, #0x0]\n"
+ "ldr b16, [x20, #0x0]\n"
+ "101:" // Height 6: Multiply loop: Ragged operand read: Done
+ "sadalp v1.8h, v31.16b\n"
+ "sadalp v30.8h, v28.16b\n"
+ "sadalp v27.8h, v25.16b\n"
+ "sadalp v24.8h, v22.16b\n"
+ "sadalp v21.8h, v19.16b\n"
+ "sadalp v18.8h, v16.16b\n"
+ "102:" // Height 6: Multiply loop: No odd multiplies
+ "add x28, x28, #0x1\n"
+ "cmp x28, x21\n"
+ "bne 87b\n"
+ "sadalp v0.4s, v1.8h\n"
+ "sadalp v29.4s, v30.8h\n"
+ "addp v0.4s, v0.4s, v29.4s\n"
+ "sadalp v26.4s, v27.8h\n"
+ "sadalp v23.4s, v24.8h\n"
+ "addp v29.4s, v26.4s, v23.4s\n"
+ "sadalp v20.4s, v21.8h\n"
+ "sadalp v17.4s, v18.8h\n"
+ "addp v0.4s, v0.4s, v29.4s\n"
+ "subs %x[M], %x[M], #0x6\n"
+ "addp v20.4s, v20.4s, v17.4s\n"
+ "mul v0.4s, v0.4s, v2.4s\n"
+ "st1 { v0.4s }, [%x[out_ptr]], #0x10\n"
+ "addp v20.4s, v20.4s, v20.4s\n"
+ "mul v20.4s, v20.4s, v2.4s\n"
+ "str d20, [%x[out_ptr]], #0x8\n"
+ "beq 104f\n"
+ "tbz %x[flags], #3, 103f\n"
+ "add %x[input_offset], %x[input_offset], #0x6\n"
+ "b 1b\n"
+ "103:" // Update direct input
+ "mov x19, #0x6\n"
+ "madd %x[input_ptr], x19, %x[input_offset], %x[input_ptr]\n"
+ "b 1b\n"
+ "104:" // Exit
+
+ : [M] "+r" (M), [input_offset] "+r" (input_offset), [input_ptr] "+r" (input_ptr), [out_ptr] "+r" (out_ptr)
+ : [args_ptr] "r" (&ka), [b_offset] "I" (offsetof(Requantize32, b_offset)), [flags] "r" (flags), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths)), [qp] "r" (qp)
+ : "cc", "memory", "v0", "v1", "v2", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
+ );
+}
+
+} // namespace arm_gemm
+
+#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_gemm/rowsum_indirect_u8.cpp b/src/core/NEON/kernels/arm_gemm/rowsum_indirect_u8.cpp
new file mode 100644
index 0000000..f5709d9
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/rowsum_indirect_u8.cpp
@@ -0,0 +1,1160 @@
+/*
+ * Copyright (c) 2019-2020 Arm Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+#ifdef __aarch64__
+
+#include "arm_gemm.hpp"
+#include "quantized.hpp"
+#include "utils.hpp"
+
+#include <cassert>
+
+namespace arm_gemm {
+
+template<>
+void row_sums_indirect(
+ unsigned int num_strings, const unsigned int *string_lengths, IndirectInputArg<uint8_t> A_arg,
+ size_t M, int32_t *out_ptr, const Requantize32 *qp
+)
+{
+ struct KernelArgs {
+ unsigned int num_strings;
+ const unsigned int *string_lengths;
+ unsigned int input_initial_col;
+ } ka;
+
+ unsigned long flags=0;
+ void *input_ptr;
+ size_t input_offset;
+
+ if (A_arg.is_indirect) {
+ input_ptr=(void *)(A_arg.indirect.ptr);
+ input_offset=A_arg.indirect.start_row;
+ ka.input_initial_col=A_arg.indirect.start_col;
+ flags |= 0x8;
+ } else {
+ assert(num_strings==1);
+ input_ptr=(void *)(A_arg.direct.base);
+ input_offset=A_arg.direct.stride;
+ }
+
+ ka.num_strings = num_strings;
+ ka.string_lengths = string_lengths;
+
+ __asm__ __volatile__(
+ "add x19, %x[qp], %[b_offset]\n"
+ "ld1r { v2.4s }, [x19]\n"
+ "neg v2.4s, v2.4s\n"
+ "1:" // Row loop
+ "cmp %x[M], #0x6\n"
+ "bge 86f\n"
+ "cmp %x[M], #0x4\n"
+ "bgt 69f\n"
+ "beq 52f\n"
+ "cmp %x[M], #0x2\n"
+ "bgt 35f\n"
+ "beq 18f\n"
+ "movi v1.8h, #0x0\n"
+ "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "movi v0.4s, #0x0\n"
+ "mov x9, #0x0\n"
+ "mov x28, #0x0\n"
+ "2:" // Height 1: String loop
+ "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr w27, [x19, x28, LSL #0x2]\n"
+ "tbz %x[flags], #3, 3f\n"
+ "ldr x19, [%x[input_ptr], x28, LSL #0x3]\n"
+ "add x19, x19, %x[input_offset], LSL #3\n"
+ "ldr x26, [x19, #0x0]\n"
+ "cbnz x28, 4f\n"
+ "ldr w19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x26, x26, x19\n"
+ "b 4f\n"
+ "3:" // Height 1: setup direct input
+ "mov x26, %x[input_ptr]\n"
+ "4:" // Height 1: input setup done
+ "cmp x27, #0x10\n"
+ "blt 8f\n"
+ "cmp x27, #0x20\n"
+ "blt 7f\n"
+ "5:" // Height 1: Multiply loop: Main loop head
+ "ldr q31, [x26, #0x0]\n"
+ "cmp x9, #0x7e\n"
+ "add x26, x26, #0x10\n"
+ "blt 6f\n"
+ "uadalp v0.4s, v1.8h\n"
+ "movi v1.8h, #0x0\n"
+ "mov x9, #0x0\n"
+ "6:" // Height 1: Multiply loop: unique 1: no collapse
+ "uadalp v1.8h, v31.16b\n"
+ "add x9, x9, #0x1\n"
+ "sub x27, x27, #0x10\n"
+ "cmp x27, #0x20\n"
+ "bge 5b\n"
+ "7:" // Height 1: Multiply loop: Single iteration only
+ "sub x27, x27, #0x10\n"
+ "ldr q31, [x26, #0x0]\n"
+ "add x26, x26, #0x10\n"
+ "uadalp v1.8h, v31.16b\n"
+ "8:" // Height 1: Multiply loop: Main loop skip
+ "cbz x27, 17f\n"
+ "tbz x27, #3, 12f\n"
+ "ldr d31, [x26], #0x8\n"
+ "tbz x27, #2, 10f\n"
+ "ld1 { v31.s }[2], [x26], #0x4\n"
+ "tbz x27, #1, 9f\n"
+ "ld1 { v31.h }[6], [x26], #0x2\n"
+ "tbz x27, #0, 16f\n"
+ "ld1 { v31.b }[14], [x26]\n"
+ "b 16f\n"
+ "9:" // Height 1: Multiply loop: Ragged operand read: partial_1_12
+ "tbz x27, #0, 16f\n"
+ "ld1 { v31.b }[12], [x26]\n"
+ "b 16f\n"
+ "10:" // Height 1: Multiply loop: Ragged operand read: partial_2_8
+ "tbz x27, #1, 11f\n"
+ "ld1 { v31.h }[4], [x26], #0x2\n"
+ "tbz x27, #0, 16f\n"
+ "ld1 { v31.b }[10], [x26]\n"
+ "b 16f\n"
+ "11:" // Height 1: Multiply loop: Ragged operand read: partial_1_8
+ "tbz x27, #0, 16f\n"
+ "ld1 { v31.b }[8], [x26]\n"
+ "b 16f\n"
+ "12:" // Height 1: Multiply loop: Ragged operand read: partial_4_0
+ "tbz x27, #2, 14f\n"
+ "ldr s31, [x26], #0x4\n"
+ "tbz x27, #1, 13f\n"
+ "ld1 { v31.h }[2], [x26], #0x2\n"
+ "tbz x27, #0, 16f\n"
+ "ld1 { v31.b }[6], [x26]\n"
+ "b 16f\n"
+ "13:" // Height 1: Multiply loop: Ragged operand read: partial_1_4
+ "tbz x27, #0, 16f\n"
+ "ld1 { v31.b }[4], [x26]\n"
+ "b 16f\n"
+ "14:" // Height 1: Multiply loop: Ragged operand read: partial_2_0
+ "tbz x27, #1, 15f\n"
+ "ldr h31, [x26], #0x2\n"
+ "tbz x27, #0, 16f\n"
+ "ld1 { v31.b }[2], [x26]\n"
+ "b 16f\n"
+ "15:" // Height 1: Multiply loop: Ragged operand read: partial_1_0
+ "ldr b31, [x26, #0x0]\n"
+ "16:" // Height 1: Multiply loop: Ragged operand read: Done
+ "uadalp v1.8h, v31.16b\n"
+ "17:" // Height 1: Multiply loop: No odd multiplies
+ "add x28, x28, #0x1\n"
+ "cmp x28, x20\n"
+ "bne 2b\n"
+ "uadalp v0.4s, v1.8h\n"
+ "addp v0.4s, v0.4s, v0.4s\n"
+ "addp v0.4s, v0.4s, v0.4s\n"
+ "mul v0.4s, v0.4s, v2.4s\n"
+ "str s0, [%x[out_ptr]], #0x4\n"
+ "b 104f\n"
+ "18:" // Height 2
+ "movi v1.8h, #0x0\n"
+ "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "mov x9, #0x0\n"
+ "movi v0.4s, #0x0\n"
+ "mov x28, #0x0\n"
+ "movi v30.8h, #0x0\n"
+ "movi v29.4s, #0x0\n"
+ "19:" // Height 2: String loop
+ "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr w27, [x19, x28, LSL #0x2]\n"
+ "tbz %x[flags], #3, 20f\n"
+ "ldr x19, [%x[input_ptr], x28, LSL #0x3]\n"
+ "add x19, x19, %x[input_offset], LSL #3\n"
+ "ldr x26, [x19, #0x0]\n"
+ "ldr x25, [x19, #0x8]\n"
+ "cbnz x28, 21f\n"
+ "ldr w19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x26, x26, x19\n"
+ "add x25, x25, x19\n"
+ "b 21f\n"
+ "20:" // Height 2: setup direct input
+ "mov x26, %x[input_ptr]\n"
+ "add x25, x26, %x[input_offset]\n"
+ "21:" // Height 2: input setup done
+ "cmp x27, #0x10\n"
+ "blt 25f\n"
+ "cmp x27, #0x20\n"
+ "blt 24f\n"
+ "22:" // Height 2: Multiply loop: Main loop head
+ "ldr q31, [x26, #0x0]\n"
+ "ldr q28, [x25, #0x0]\n"
+ "cmp x9, #0x7e\n"
+ "add x26, x26, #0x10\n"
+ "add x25, x25, #0x10\n"
+ "blt 23f\n"
+ "uadalp v0.4s, v1.8h\n"
+ "movi v1.8h, #0x0\n"
+ "uadalp v29.4s, v30.8h\n"
+ "movi v30.8h, #0x0\n"
+ "mov x9, #0x0\n"
+ "23:" // Height 2: Multiply loop: unique 2: no collapse
+ "uadalp v1.8h, v31.16b\n"
+ "uadalp v30.8h, v28.16b\n"
+ "add x9, x9, #0x1\n"
+ "sub x27, x27, #0x10\n"
+ "cmp x27, #0x20\n"
+ "bge 22b\n"
+ "24:" // Height 2: Multiply loop: Single iteration only
+ "sub x27, x27, #0x10\n"
+ "ldr q31, [x26, #0x0]\n"
+ "ldr q28, [x25, #0x0]\n"
+ "add x26, x26, #0x10\n"
+ "add x25, x25, #0x10\n"
+ "uadalp v1.8h, v31.16b\n"
+ "uadalp v30.8h, v28.16b\n"
+ "25:" // Height 2: Multiply loop: Main loop skip
+ "cbz x27, 34f\n"
+ "tbz x27, #3, 29f\n"
+ "ldr d31, [x26], #0x8\n"
+ "ldr d28, [x25], #0x8\n"
+ "tbz x27, #2, 27f\n"
+ "ld1 { v31.s }[2], [x26], #0x4\n"
+ "ld1 { v28.s }[2], [x25], #0x4\n"
+ "tbz x27, #1, 26f\n"
+ "ld1 { v31.h }[6], [x26], #0x2\n"
+ "ld1 { v28.h }[6], [x25], #0x2\n"
+ "tbz x27, #0, 33f\n"
+ "ld1 { v31.b }[14], [x26]\n"
+ "ld1 { v28.b }[14], [x25]\n"
+ "b 33f\n"
+ "26:" // Height 2: Multiply loop: Ragged operand read: partial_1_12
+ "tbz x27, #0, 33f\n"
+ "ld1 { v31.b }[12], [x26]\n"
+ "ld1 { v28.b }[12], [x25]\n"
+ "b 33f\n"
+ "27:" // Height 2: Multiply loop: Ragged operand read: partial_2_8
+ "tbz x27, #1, 28f\n"
+ "ld1 { v31.h }[4], [x26], #0x2\n"
+ "ld1 { v28.h }[4], [x25], #0x2\n"
+ "tbz x27, #0, 33f\n"
+ "ld1 { v31.b }[10], [x26]\n"
+ "ld1 { v28.b }[10], [x25]\n"
+ "b 33f\n"
+ "28:" // Height 2: Multiply loop: Ragged operand read: partial_1_8
+ "tbz x27, #0, 33f\n"
+ "ld1 { v31.b }[8], [x26]\n"
+ "ld1 { v28.b }[8], [x25]\n"
+ "b 33f\n"
+ "29:" // Height 2: Multiply loop: Ragged operand read: partial_4_0
+ "tbz x27, #2, 31f\n"
+ "ldr s31, [x26], #0x4\n"
+ "ldr s28, [x25], #0x4\n"
+ "tbz x27, #1, 30f\n"
+ "ld1 { v31.h }[2], [x26], #0x2\n"
+ "ld1 { v28.h }[2], [x25], #0x2\n"
+ "tbz x27, #0, 33f\n"
+ "ld1 { v31.b }[6], [x26]\n"
+ "ld1 { v28.b }[6], [x25]\n"
+ "b 33f\n"
+ "30:" // Height 2: Multiply loop: Ragged operand read: partial_1_4
+ "tbz x27, #0, 33f\n"
+ "ld1 { v31.b }[4], [x26]\n"
+ "ld1 { v28.b }[4], [x25]\n"
+ "b 33f\n"
+ "31:" // Height 2: Multiply loop: Ragged operand read: partial_2_0
+ "tbz x27, #1, 32f\n"
+ "ldr h31, [x26], #0x2\n"
+ "ldr h28, [x25], #0x2\n"
+ "tbz x27, #0, 33f\n"
+ "ld1 { v31.b }[2], [x26]\n"
+ "ld1 { v28.b }[2], [x25]\n"
+ "b 33f\n"
+ "32:" // Height 2: Multiply loop: Ragged operand read: partial_1_0
+ "ldr b31, [x26, #0x0]\n"
+ "ldr b28, [x25, #0x0]\n"
+ "33:" // Height 2: Multiply loop: Ragged operand read: Done
+ "uadalp v1.8h, v31.16b\n"
+ "uadalp v30.8h, v28.16b\n"
+ "34:" // Height 2: Multiply loop: No odd multiplies
+ "add x28, x28, #0x1\n"
+ "cmp x28, x20\n"
+ "bne 19b\n"
+ "uadalp v0.4s, v1.8h\n"
+ "uadalp v29.4s, v30.8h\n"
+ "addp v0.4s, v0.4s, v29.4s\n"
+ "addp v0.4s, v0.4s, v0.4s\n"
+ "mul v0.4s, v0.4s, v2.4s\n"
+ "str d0, [%x[out_ptr]], #0x8\n"
+ "b 104f\n"
+ "35:" // Height 3
+ "movi v1.8h, #0x0\n"
+ "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "mov x9, #0x0\n"
+ "movi v0.4s, #0x0\n"
+ "mov x28, #0x0\n"
+ "movi v30.8h, #0x0\n"
+ "movi v29.4s, #0x0\n"
+ "movi v27.8h, #0x0\n"
+ "movi v26.4s, #0x0\n"
+ "36:" // Height 3: String loop
+ "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr w27, [x19, x28, LSL #0x2]\n"
+ "tbz %x[flags], #3, 37f\n"
+ "ldr x19, [%x[input_ptr], x28, LSL #0x3]\n"
+ "add x19, x19, %x[input_offset], LSL #3\n"
+ "ldr x26, [x19, #0x0]\n"
+ "ldr x25, [x19, #0x8]\n"
+ "ldr x24, [x19, #0x10]\n"
+ "cbnz x28, 38f\n"
+ "ldr w19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x26, x26, x19\n"
+ "add x25, x25, x19\n"
+ "add x24, x24, x19\n"
+ "b 38f\n"
+ "37:" // Height 3: setup direct input
+ "mov x26, %x[input_ptr]\n"
+ "add x25, x26, %x[input_offset]\n"
+ "add x24, x25, %x[input_offset]\n"
+ "38:" // Height 3: input setup done
+ "cmp x27, #0x10\n"
+ "blt 42f\n"
+ "cmp x27, #0x20\n"
+ "blt 41f\n"
+ "39:" // Height 3: Multiply loop: Main loop head
+ "ldr q31, [x26, #0x0]\n"
+ "ldr q28, [x25, #0x0]\n"
+ "ldr q25, [x24, #0x0]\n"
+ "cmp x9, #0x7e\n"
+ "add x26, x26, #0x10\n"
+ "add x25, x25, #0x10\n"
+ "add x24, x24, #0x10\n"
+ "blt 40f\n"
+ "uadalp v0.4s, v1.8h\n"
+ "movi v1.8h, #0x0\n"
+ "uadalp v29.4s, v30.8h\n"
+ "movi v30.8h, #0x0\n"
+ "uadalp v26.4s, v27.8h\n"
+ "movi v27.8h, #0x0\n"
+ "mov x9, #0x0\n"
+ "40:" // Height 3: Multiply loop: unique 3: no collapse
+ "uadalp v1.8h, v31.16b\n"
+ "uadalp v30.8h, v28.16b\n"
+ "uadalp v27.8h, v25.16b\n"
+ "add x9, x9, #0x1\n"
+ "sub x27, x27, #0x10\n"
+ "cmp x27, #0x20\n"
+ "bge 39b\n"
+ "41:" // Height 3: Multiply loop: Single iteration only
+ "sub x27, x27, #0x10\n"
+ "ldr q31, [x26, #0x0]\n"
+ "ldr q28, [x25, #0x0]\n"
+ "ldr q25, [x24, #0x0]\n"
+ "add x26, x26, #0x10\n"
+ "add x25, x25, #0x10\n"
+ "uadalp v1.8h, v31.16b\n"
+ "uadalp v30.8h, v28.16b\n"
+ "uadalp v27.8h, v25.16b\n"
+ "add x24, x24, #0x10\n"
+ "42:" // Height 3: Multiply loop: Main loop skip
+ "cbz x27, 51f\n"
+ "tbz x27, #3, 46f\n"
+ "ldr d31, [x26], #0x8\n"
+ "ldr d28, [x25], #0x8\n"
+ "ldr d25, [x24], #0x8\n"
+ "tbz x27, #2, 44f\n"
+ "ld1 { v31.s }[2], [x26], #0x4\n"
+ "ld1 { v28.s }[2], [x25], #0x4\n"
+ "ld1 { v25.s }[2], [x24], #0x4\n"
+ "tbz x27, #1, 43f\n"
+ "ld1 { v31.h }[6], [x26], #0x2\n"
+ "ld1 { v28.h }[6], [x25], #0x2\n"
+ "ld1 { v25.h }[6], [x24], #0x2\n"
+ "tbz x27, #0, 50f\n"
+ "ld1 { v31.b }[14], [x26]\n"
+ "ld1 { v28.b }[14], [x25]\n"
+ "ld1 { v25.b }[14], [x24]\n"
+ "b 50f\n"
+ "43:" // Height 3: Multiply loop: Ragged operand read: partial_1_12
+ "tbz x27, #0, 50f\n"
+ "ld1 { v31.b }[12], [x26]\n"
+ "ld1 { v28.b }[12], [x25]\n"
+ "ld1 { v25.b }[12], [x24]\n"
+ "b 50f\n"
+ "44:" // Height 3: Multiply loop: Ragged operand read: partial_2_8
+ "tbz x27, #1, 45f\n"
+ "ld1 { v31.h }[4], [x26], #0x2\n"
+ "ld1 { v28.h }[4], [x25], #0x2\n"
+ "ld1 { v25.h }[4], [x24], #0x2\n"
+ "tbz x27, #0, 50f\n"
+ "ld1 { v31.b }[10], [x26]\n"
+ "ld1 { v28.b }[10], [x25]\n"
+ "ld1 { v25.b }[10], [x24]\n"
+ "b 50f\n"
+ "45:" // Height 3: Multiply loop: Ragged operand read: partial_1_8
+ "tbz x27, #0, 50f\n"
+ "ld1 { v31.b }[8], [x26]\n"
+ "ld1 { v28.b }[8], [x25]\n"
+ "ld1 { v25.b }[8], [x24]\n"
+ "b 50f\n"
+ "46:" // Height 3: Multiply loop: Ragged operand read: partial_4_0
+ "tbz x27, #2, 48f\n"
+ "ldr s31, [x26], #0x4\n"
+ "ldr s28, [x25], #0x4\n"
+ "ldr s25, [x24], #0x4\n"
+ "tbz x27, #1, 47f\n"
+ "ld1 { v31.h }[2], [x26], #0x2\n"
+ "ld1 { v28.h }[2], [x25], #0x2\n"
+ "ld1 { v25.h }[2], [x24], #0x2\n"
+ "tbz x27, #0, 50f\n"
+ "ld1 { v31.b }[6], [x26]\n"
+ "ld1 { v28.b }[6], [x25]\n"
+ "ld1 { v25.b }[6], [x24]\n"
+ "b 50f\n"
+ "47:" // Height 3: Multiply loop: Ragged operand read: partial_1_4
+ "tbz x27, #0, 50f\n"
+ "ld1 { v31.b }[4], [x26]\n"
+ "ld1 { v28.b }[4], [x25]\n"
+ "ld1 { v25.b }[4], [x24]\n"
+ "b 50f\n"
+ "48:" // Height 3: Multiply loop: Ragged operand read: partial_2_0
+ "tbz x27, #1, 49f\n"
+ "ldr h31, [x26], #0x2\n"
+ "ldr h28, [x25], #0x2\n"
+ "ldr h25, [x24], #0x2\n"
+ "tbz x27, #0, 50f\n"
+ "ld1 { v31.b }[2], [x26]\n"
+ "ld1 { v28.b }[2], [x25]\n"
+ "ld1 { v25.b }[2], [x24]\n"
+ "b 50f\n"
+ "49:" // Height 3: Multiply loop: Ragged operand read: partial_1_0
+ "ldr b31, [x26, #0x0]\n"
+ "ldr b28, [x25, #0x0]\n"
+ "ldr b25, [x24, #0x0]\n"
+ "50:" // Height 3: Multiply loop: Ragged operand read: Done
+ "uadalp v1.8h, v31.16b\n"
+ "uadalp v30.8h, v28.16b\n"
+ "uadalp v27.8h, v25.16b\n"
+ "51:" // Height 3: Multiply loop: No odd multiplies
+ "add x28, x28, #0x1\n"
+ "cmp x28, x20\n"
+ "bne 36b\n"
+ "uadalp v0.4s, v1.8h\n"
+ "uadalp v29.4s, v30.8h\n"
+ "addp v0.4s, v0.4s, v29.4s\n"
+ "uadalp v26.4s, v27.8h\n"
+ "addp v0.4s, v0.4s, v0.4s\n"
+ "addp v26.4s, v26.4s, v26.4s\n"
+ "mul v0.4s, v0.4s, v2.4s\n"
+ "str d0, [%x[out_ptr]], #0x8\n"
+ "addp v26.4s, v26.4s, v26.4s\n"
+ "mul v26.4s, v26.4s, v2.4s\n"
+ "str s26, [%x[out_ptr]], #0x4\n"
+ "b 104f\n"
+ "52:" // Height 4
+ "movi v1.8h, #0x0\n"
+ "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "mov x9, #0x0\n"
+ "movi v0.4s, #0x0\n"
+ "mov x28, #0x0\n"
+ "movi v30.8h, #0x0\n"
+ "movi v29.4s, #0x0\n"
+ "movi v27.8h, #0x0\n"
+ "movi v26.4s, #0x0\n"
+ "movi v24.8h, #0x0\n"
+ "movi v23.4s, #0x0\n"
+ "53:" // Height 4: String loop
+ "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr w27, [x19, x28, LSL #0x2]\n"
+ "tbz %x[flags], #3, 54f\n"
+ "ldr x19, [%x[input_ptr], x28, LSL #0x3]\n"
+ "add x19, x19, %x[input_offset], LSL #3\n"
+ "ldr x26, [x19, #0x0]\n"
+ "ldr x25, [x19, #0x8]\n"
+ "ldr x24, [x19, #0x10]\n"
+ "ldr x23, [x19, #0x18]\n"
+ "cbnz x28, 55f\n"
+ "ldr w19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x26, x26, x19\n"
+ "add x25, x25, x19\n"
+ "add x24, x24, x19\n"
+ "add x23, x23, x19\n"
+ "b 55f\n"
+ "54:" // Height 4: setup direct input
+ "mov x26, %x[input_ptr]\n"
+ "add x25, x26, %x[input_offset]\n"
+ "add x24, x25, %x[input_offset]\n"
+ "add x23, x24, %x[input_offset]\n"
+ "55:" // Height 4: input setup done
+ "cmp x27, #0x10\n"
+ "blt 59f\n"
+ "cmp x27, #0x20\n"
+ "blt 58f\n"
+ "56:" // Height 4: Multiply loop: Main loop head
+ "ldr q31, [x26, #0x0]\n"
+ "ldr q28, [x25, #0x0]\n"
+ "ldr q25, [x24, #0x0]\n"
+ "ldr q22, [x23, #0x0]\n"
+ "cmp x9, #0x7e\n"
+ "add x26, x26, #0x10\n"
+ "add x25, x25, #0x10\n"
+ "add x24, x24, #0x10\n"
+ "add x23, x23, #0x10\n"
+ "blt 57f\n"
+ "uadalp v0.4s, v1.8h\n"
+ "movi v1.8h, #0x0\n"
+ "uadalp v29.4s, v30.8h\n"
+ "movi v30.8h, #0x0\n"
+ "uadalp v26.4s, v27.8h\n"
+ "movi v27.8h, #0x0\n"
+ "uadalp v23.4s, v24.8h\n"
+ "movi v24.8h, #0x0\n"
+ "mov x9, #0x0\n"
+ "57:" // Height 4: Multiply loop: unique 4: no collapse
+ "uadalp v1.8h, v31.16b\n"
+ "uadalp v30.8h, v28.16b\n"
+ "uadalp v27.8h, v25.16b\n"
+ "uadalp v24.8h, v22.16b\n"
+ "add x9, x9, #0x1\n"
+ "sub x27, x27, #0x10\n"
+ "cmp x27, #0x20\n"
+ "bge 56b\n"
+ "58:" // Height 4: Multiply loop: Single iteration only
+ "sub x27, x27, #0x10\n"
+ "ldr q31, [x26, #0x0]\n"
+ "ldr q28, [x25, #0x0]\n"
+ "ldr q25, [x24, #0x0]\n"
+ "ldr q22, [x23, #0x0]\n"
+ "add x26, x26, #0x10\n"
+ "uadalp v1.8h, v31.16b\n"
+ "uadalp v30.8h, v28.16b\n"
+ "uadalp v27.8h, v25.16b\n"
+ "uadalp v24.8h, v22.16b\n"
+ "add x25, x25, #0x10\n"
+ "add x24, x24, #0x10\n"
+ "add x23, x23, #0x10\n"
+ "59:" // Height 4: Multiply loop: Main loop skip
+ "cbz x27, 68f\n"
+ "tbz x27, #3, 63f\n"
+ "ldr d31, [x26], #0x8\n"
+ "ldr d28, [x25], #0x8\n"
+ "ldr d25, [x24], #0x8\n"
+ "ldr d22, [x23], #0x8\n"
+ "tbz x27, #2, 61f\n"
+ "ld1 { v31.s }[2], [x26], #0x4\n"
+ "ld1 { v28.s }[2], [x25], #0x4\n"
+ "ld1 { v25.s }[2], [x24], #0x4\n"
+ "ld1 { v22.s }[2], [x23], #0x4\n"
+ "tbz x27, #1, 60f\n"
+ "ld1 { v31.h }[6], [x26], #0x2\n"
+ "ld1 { v28.h }[6], [x25], #0x2\n"
+ "ld1 { v25.h }[6], [x24], #0x2\n"
+ "ld1 { v22.h }[6], [x23], #0x2\n"
+ "tbz x27, #0, 67f\n"
+ "ld1 { v31.b }[14], [x26]\n"
+ "ld1 { v28.b }[14], [x25]\n"
+ "ld1 { v25.b }[14], [x24]\n"
+ "ld1 { v22.b }[14], [x23]\n"
+ "b 67f\n"
+ "60:" // Height 4: Multiply loop: Ragged operand read: partial_1_12
+ "tbz x27, #0, 67f\n"
+ "ld1 { v31.b }[12], [x26]\n"
+ "ld1 { v28.b }[12], [x25]\n"
+ "ld1 { v25.b }[12], [x24]\n"
+ "ld1 { v22.b }[12], [x23]\n"
+ "b 67f\n"
+ "61:" // Height 4: Multiply loop: Ragged operand read: partial_2_8
+ "tbz x27, #1, 62f\n"
+ "ld1 { v31.h }[4], [x26], #0x2\n"
+ "ld1 { v28.h }[4], [x25], #0x2\n"
+ "ld1 { v25.h }[4], [x24], #0x2\n"
+ "ld1 { v22.h }[4], [x23], #0x2\n"
+ "tbz x27, #0, 67f\n"
+ "ld1 { v31.b }[10], [x26]\n"
+ "ld1 { v28.b }[10], [x25]\n"
+ "ld1 { v25.b }[10], [x24]\n"
+ "ld1 { v22.b }[10], [x23]\n"
+ "b 67f\n"
+ "62:" // Height 4: Multiply loop: Ragged operand read: partial_1_8
+ "tbz x27, #0, 67f\n"
+ "ld1 { v31.b }[8], [x26]\n"
+ "ld1 { v28.b }[8], [x25]\n"
+ "ld1 { v25.b }[8], [x24]\n"
+ "ld1 { v22.b }[8], [x23]\n"
+ "b 67f\n"
+ "63:" // Height 4: Multiply loop: Ragged operand read: partial_4_0
+ "tbz x27, #2, 65f\n"
+ "ldr s31, [x26], #0x4\n"
+ "ldr s28, [x25], #0x4\n"
+ "ldr s25, [x24], #0x4\n"
+ "ldr s22, [x23], #0x4\n"
+ "tbz x27, #1, 64f\n"
+ "ld1 { v31.h }[2], [x26], #0x2\n"
+ "ld1 { v28.h }[2], [x25], #0x2\n"
+ "ld1 { v25.h }[2], [x24], #0x2\n"
+ "ld1 { v22.h }[2], [x23], #0x2\n"
+ "tbz x27, #0, 67f\n"
+ "ld1 { v31.b }[6], [x26]\n"
+ "ld1 { v28.b }[6], [x25]\n"
+ "ld1 { v25.b }[6], [x24]\n"
+ "ld1 { v22.b }[6], [x23]\n"
+ "b 67f\n"
+ "64:" // Height 4: Multiply loop: Ragged operand read: partial_1_4
+ "tbz x27, #0, 67f\n"
+ "ld1 { v31.b }[4], [x26]\n"
+ "ld1 { v28.b }[4], [x25]\n"
+ "ld1 { v25.b }[4], [x24]\n"
+ "ld1 { v22.b }[4], [x23]\n"
+ "b 67f\n"
+ "65:" // Height 4: Multiply loop: Ragged operand read: partial_2_0
+ "tbz x27, #1, 66f\n"
+ "ldr h31, [x26], #0x2\n"
+ "ldr h28, [x25], #0x2\n"
+ "ldr h25, [x24], #0x2\n"
+ "ldr h22, [x23], #0x2\n"
+ "tbz x27, #0, 67f\n"
+ "ld1 { v31.b }[2], [x26]\n"
+ "ld1 { v28.b }[2], [x25]\n"
+ "ld1 { v25.b }[2], [x24]\n"
+ "ld1 { v22.b }[2], [x23]\n"
+ "b 67f\n"
+ "66:" // Height 4: Multiply loop: Ragged operand read: partial_1_0
+ "ldr b31, [x26, #0x0]\n"
+ "ldr b28, [x25, #0x0]\n"
+ "ldr b25, [x24, #0x0]\n"
+ "ldr b22, [x23, #0x0]\n"
+ "67:" // Height 4: Multiply loop: Ragged operand read: Done
+ "uadalp v1.8h, v31.16b\n"
+ "uadalp v30.8h, v28.16b\n"
+ "uadalp v27.8h, v25.16b\n"
+ "uadalp v24.8h, v22.16b\n"
+ "68:" // Height 4: Multiply loop: No odd multiplies
+ "add x28, x28, #0x1\n"
+ "cmp x28, x20\n"
+ "bne 53b\n"
+ "uadalp v0.4s, v1.8h\n"
+ "uadalp v29.4s, v30.8h\n"
+ "addp v0.4s, v0.4s, v29.4s\n"
+ "uadalp v26.4s, v27.8h\n"
+ "uadalp v23.4s, v24.8h\n"
+ "addp v29.4s, v26.4s, v23.4s\n"
+ "addp v0.4s, v0.4s, v29.4s\n"
+ "mul v0.4s, v0.4s, v2.4s\n"
+ "st1 { v0.4s }, [%x[out_ptr]], #0x10\n"
+ "b 104f\n"
+ "69:" // Height 5
+ "movi v1.8h, #0x0\n"
+ "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "mov x9, #0x0\n"
+ "movi v0.4s, #0x0\n"
+ "mov x28, #0x0\n"
+ "movi v30.8h, #0x0\n"
+ "movi v29.4s, #0x0\n"
+ "movi v27.8h, #0x0\n"
+ "movi v26.4s, #0x0\n"
+ "movi v24.8h, #0x0\n"
+ "movi v23.4s, #0x0\n"
+ "movi v21.8h, #0x0\n"
+ "movi v20.4s, #0x0\n"
+ "70:" // Height 5: String loop
+ "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr w27, [x19, x28, LSL #0x2]\n"
+ "tbz %x[flags], #3, 71f\n"
+ "ldr x19, [%x[input_ptr], x28, LSL #0x3]\n"
+ "add x19, x19, %x[input_offset], LSL #3\n"
+ "ldr x26, [x19, #0x0]\n"
+ "ldr x25, [x19, #0x8]\n"
+ "ldr x24, [x19, #0x10]\n"
+ "ldr x23, [x19, #0x18]\n"
+ "ldr x22, [x19, #0x20]\n"
+ "cbnz x28, 72f\n"
+ "ldr w19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x26, x26, x19\n"
+ "add x25, x25, x19\n"
+ "add x24, x24, x19\n"
+ "add x23, x23, x19\n"
+ "add x22, x22, x19\n"
+ "b 72f\n"
+ "71:" // Height 5: setup direct input
+ "mov x26, %x[input_ptr]\n"
+ "add x25, x26, %x[input_offset]\n"
+ "add x24, x25, %x[input_offset]\n"
+ "add x23, x24, %x[input_offset]\n"
+ "add x22, x23, %x[input_offset]\n"
+ "72:" // Height 5: input setup done
+ "cmp x27, #0x10\n"
+ "blt 76f\n"
+ "cmp x27, #0x20\n"
+ "blt 75f\n"
+ "73:" // Height 5: Multiply loop: Main loop head
+ "ldr q31, [x26, #0x0]\n"
+ "ldr q28, [x25, #0x0]\n"
+ "ldr q25, [x24, #0x0]\n"
+ "ldr q22, [x23, #0x0]\n"
+ "ldr q19, [x22, #0x0]\n"
+ "cmp x9, #0x7e\n"
+ "add x26, x26, #0x10\n"
+ "add x25, x25, #0x10\n"
+ "add x24, x24, #0x10\n"
+ "add x23, x23, #0x10\n"
+ "add x22, x22, #0x10\n"
+ "blt 74f\n"
+ "uadalp v0.4s, v1.8h\n"
+ "movi v1.8h, #0x0\n"
+ "uadalp v29.4s, v30.8h\n"
+ "movi v30.8h, #0x0\n"
+ "uadalp v26.4s, v27.8h\n"
+ "movi v27.8h, #0x0\n"
+ "uadalp v23.4s, v24.8h\n"
+ "movi v24.8h, #0x0\n"
+ "uadalp v20.4s, v21.8h\n"
+ "movi v21.8h, #0x0\n"
+ "mov x9, #0x0\n"
+ "74:" // Height 5: Multiply loop: unique 5: no collapse
+ "uadalp v1.8h, v31.16b\n"
+ "uadalp v30.8h, v28.16b\n"
+ "uadalp v27.8h, v25.16b\n"
+ "uadalp v24.8h, v22.16b\n"
+ "uadalp v21.8h, v19.16b\n"
+ "add x9, x9, #0x1\n"
+ "sub x27, x27, #0x10\n"
+ "cmp x27, #0x20\n"
+ "bge 73b\n"
+ "75:" // Height 5: Multiply loop: Single iteration only
+ "sub x27, x27, #0x10\n"
+ "ldr q31, [x26, #0x0]\n"
+ "ldr q28, [x25, #0x0]\n"
+ "ldr q25, [x24, #0x0]\n"
+ "ldr q22, [x23, #0x0]\n"
+ "ldr q19, [x22, #0x0]\n"
+ "uadalp v1.8h, v31.16b\n"
+ "uadalp v30.8h, v28.16b\n"
+ "uadalp v27.8h, v25.16b\n"
+ "uadalp v24.8h, v22.16b\n"
+ "uadalp v21.8h, v19.16b\n"
+ "add x26, x26, #0x10\n"
+ "add x25, x25, #0x10\n"
+ "add x24, x24, #0x10\n"
+ "add x23, x23, #0x10\n"
+ "add x22, x22, #0x10\n"
+ "76:" // Height 5: Multiply loop: Main loop skip
+ "cbz x27, 85f\n"
+ "tbz x27, #3, 80f\n"
+ "ldr d31, [x26], #0x8\n"
+ "ldr d28, [x25], #0x8\n"
+ "ldr d25, [x24], #0x8\n"
+ "ldr d22, [x23], #0x8\n"
+ "ldr d19, [x22], #0x8\n"
+ "tbz x27, #2, 78f\n"
+ "ld1 { v31.s }[2], [x26], #0x4\n"
+ "ld1 { v28.s }[2], [x25], #0x4\n"
+ "ld1 { v25.s }[2], [x24], #0x4\n"
+ "ld1 { v22.s }[2], [x23], #0x4\n"
+ "ld1 { v19.s }[2], [x22], #0x4\n"
+ "tbz x27, #1, 77f\n"
+ "ld1 { v31.h }[6], [x26], #0x2\n"
+ "ld1 { v28.h }[6], [x25], #0x2\n"
+ "ld1 { v25.h }[6], [x24], #0x2\n"
+ "ld1 { v22.h }[6], [x23], #0x2\n"
+ "ld1 { v19.h }[6], [x22], #0x2\n"
+ "tbz x27, #0, 84f\n"
+ "ld1 { v31.b }[14], [x26]\n"
+ "ld1 { v28.b }[14], [x25]\n"
+ "ld1 { v25.b }[14], [x24]\n"
+ "ld1 { v22.b }[14], [x23]\n"
+ "ld1 { v19.b }[14], [x22]\n"
+ "b 84f\n"
+ "77:" // Height 5: Multiply loop: Ragged operand read: partial_1_12
+ "tbz x27, #0, 84f\n"
+ "ld1 { v31.b }[12], [x26]\n"
+ "ld1 { v28.b }[12], [x25]\n"
+ "ld1 { v25.b }[12], [x24]\n"
+ "ld1 { v22.b }[12], [x23]\n"
+ "ld1 { v19.b }[12], [x22]\n"
+ "b 84f\n"
+ "78:" // Height 5: Multiply loop: Ragged operand read: partial_2_8
+ "tbz x27, #1, 79f\n"
+ "ld1 { v31.h }[4], [x26], #0x2\n"
+ "ld1 { v28.h }[4], [x25], #0x2\n"
+ "ld1 { v25.h }[4], [x24], #0x2\n"
+ "ld1 { v22.h }[4], [x23], #0x2\n"
+ "ld1 { v19.h }[4], [x22], #0x2\n"
+ "tbz x27, #0, 84f\n"
+ "ld1 { v31.b }[10], [x26]\n"
+ "ld1 { v28.b }[10], [x25]\n"
+ "ld1 { v25.b }[10], [x24]\n"
+ "ld1 { v22.b }[10], [x23]\n"
+ "ld1 { v19.b }[10], [x22]\n"
+ "b 84f\n"
+ "79:" // Height 5: Multiply loop: Ragged operand read: partial_1_8
+ "tbz x27, #0, 84f\n"
+ "ld1 { v31.b }[8], [x26]\n"
+ "ld1 { v28.b }[8], [x25]\n"
+ "ld1 { v25.b }[8], [x24]\n"
+ "ld1 { v22.b }[8], [x23]\n"
+ "ld1 { v19.b }[8], [x22]\n"
+ "b 84f\n"
+ "80:" // Height 5: Multiply loop: Ragged operand read: partial_4_0
+ "tbz x27, #2, 82f\n"
+ "ldr s31, [x26], #0x4\n"
+ "ldr s28, [x25], #0x4\n"
+ "ldr s25, [x24], #0x4\n"
+ "ldr s22, [x23], #0x4\n"
+ "ldr s19, [x22], #0x4\n"
+ "tbz x27, #1, 81f\n"
+ "ld1 { v31.h }[2], [x26], #0x2\n"
+ "ld1 { v28.h }[2], [x25], #0x2\n"
+ "ld1 { v25.h }[2], [x24], #0x2\n"
+ "ld1 { v22.h }[2], [x23], #0x2\n"
+ "ld1 { v19.h }[2], [x22], #0x2\n"
+ "tbz x27, #0, 84f\n"
+ "ld1 { v31.b }[6], [x26]\n"
+ "ld1 { v28.b }[6], [x25]\n"
+ "ld1 { v25.b }[6], [x24]\n"
+ "ld1 { v22.b }[6], [x23]\n"
+ "ld1 { v19.b }[6], [x22]\n"
+ "b 84f\n"
+ "81:" // Height 5: Multiply loop: Ragged operand read: partial_1_4
+ "tbz x27, #0, 84f\n"
+ "ld1 { v31.b }[4], [x26]\n"
+ "ld1 { v28.b }[4], [x25]\n"
+ "ld1 { v25.b }[4], [x24]\n"
+ "ld1 { v22.b }[4], [x23]\n"
+ "ld1 { v19.b }[4], [x22]\n"
+ "b 84f\n"
+ "82:" // Height 5: Multiply loop: Ragged operand read: partial_2_0
+ "tbz x27, #1, 83f\n"
+ "ldr h31, [x26], #0x2\n"
+ "ldr h28, [x25], #0x2\n"
+ "ldr h25, [x24], #0x2\n"
+ "ldr h22, [x23], #0x2\n"
+ "ldr h19, [x22], #0x2\n"
+ "tbz x27, #0, 84f\n"
+ "ld1 { v31.b }[2], [x26]\n"
+ "ld1 { v28.b }[2], [x25]\n"
+ "ld1 { v25.b }[2], [x24]\n"
+ "ld1 { v22.b }[2], [x23]\n"
+ "ld1 { v19.b }[2], [x22]\n"
+ "b 84f\n"
+ "83:" // Height 5: Multiply loop: Ragged operand read: partial_1_0
+ "ldr b31, [x26, #0x0]\n"
+ "ldr b28, [x25, #0x0]\n"
+ "ldr b25, [x24, #0x0]\n"
+ "ldr b22, [x23, #0x0]\n"
+ "ldr b19, [x22, #0x0]\n"
+ "84:" // Height 5: Multiply loop: Ragged operand read: Done
+ "uadalp v1.8h, v31.16b\n"
+ "uadalp v30.8h, v28.16b\n"
+ "uadalp v27.8h, v25.16b\n"
+ "uadalp v24.8h, v22.16b\n"
+ "uadalp v21.8h, v19.16b\n"
+ "85:" // Height 5: Multiply loop: No odd multiplies
+ "add x28, x28, #0x1\n"
+ "cmp x28, x20\n"
+ "bne 70b\n"
+ "uadalp v0.4s, v1.8h\n"
+ "uadalp v29.4s, v30.8h\n"
+ "addp v0.4s, v0.4s, v29.4s\n"
+ "uadalp v26.4s, v27.8h\n"
+ "uadalp v23.4s, v24.8h\n"
+ "addp v29.4s, v26.4s, v23.4s\n"
+ "uadalp v20.4s, v21.8h\n"
+ "addp v0.4s, v0.4s, v29.4s\n"
+ "addp v20.4s, v20.4s, v20.4s\n"
+ "mul v0.4s, v0.4s, v2.4s\n"
+ "st1 { v0.4s }, [%x[out_ptr]], #0x10\n"
+ "addp v20.4s, v20.4s, v20.4s\n"
+ "mul v20.4s, v20.4s, v2.4s\n"
+ "str s20, [%x[out_ptr]], #0x4\n"
+ "b 104f\n"
+ "86:" // Height 6
+ "movi v1.8h, #0x0\n"
+ "ldr w21, [%x[args_ptr], %[offsetof_num_strings]]\n"
+ "mov x9, #0x0\n"
+ "movi v0.4s, #0x0\n"
+ "mov x28, #0x0\n"
+ "movi v30.8h, #0x0\n"
+ "movi v29.4s, #0x0\n"
+ "movi v27.8h, #0x0\n"
+ "movi v26.4s, #0x0\n"
+ "movi v24.8h, #0x0\n"
+ "movi v23.4s, #0x0\n"
+ "movi v21.8h, #0x0\n"
+ "movi v20.4s, #0x0\n"
+ "movi v18.8h, #0x0\n"
+ "movi v17.4s, #0x0\n"
+ "87:" // Height 6: String loop
+ "ldr x19, [%x[args_ptr], %[offsetof_string_lengths]]\n"
+ "ldr w27, [x19, x28, LSL #0x2]\n"
+ "tbz %x[flags], #3, 88f\n"
+ "ldr x19, [%x[input_ptr], x28, LSL #0x3]\n"
+ "add x19, x19, %x[input_offset], LSL #3\n"
+ "ldr x26, [x19, #0x0]\n"
+ "ldr x25, [x19, #0x8]\n"
+ "ldr x24, [x19, #0x10]\n"
+ "ldr x23, [x19, #0x18]\n"
+ "ldr x22, [x19, #0x20]\n"
+ "ldr x20, [x19, #0x28]\n"
+ "cbnz x28, 89f\n"
+ "ldr w19, [%x[args_ptr], %[offsetof_input_initial_col]]\n"
+ "add x26, x26, x19\n"
+ "add x25, x25, x19\n"
+ "add x24, x24, x19\n"
+ "add x23, x23, x19\n"
+ "add x22, x22, x19\n"
+ "add x20, x20, x19\n"
+ "b 89f\n"
+ "88:" // Height 6: setup direct input
+ "mov x26, %x[input_ptr]\n"
+ "add x25, x26, %x[input_offset]\n"
+ "add x24, x25, %x[input_offset]\n"
+ "add x23, x24, %x[input_offset]\n"
+ "add x22, x23, %x[input_offset]\n"
+ "add x20, x22, %x[input_offset]\n"
+ "89:" // Height 6: input setup done
+ "cmp x27, #0x10\n"
+ "blt 93f\n"
+ "cmp x27, #0x20\n"
+ "blt 92f\n"
+ "90:" // Height 6: Multiply loop: Main loop head
+ "ldr q31, [x26, #0x0]\n"
+ "ldr q28, [x25, #0x0]\n"
+ "ldr q25, [x24, #0x0]\n"
+ "ldr q22, [x23, #0x0]\n"
+ "ldr q19, [x22, #0x0]\n"
+ "ldr q16, [x20, #0x0]\n"
+ "cmp x9, #0x7e\n"
+ "add x26, x26, #0x10\n"
+ "add x25, x25, #0x10\n"
+ "add x24, x24, #0x10\n"
+ "add x23, x23, #0x10\n"
+ "add x22, x22, #0x10\n"
+ "add x20, x20, #0x10\n"
+ "blt 91f\n"
+ "uadalp v0.4s, v1.8h\n"
+ "movi v1.8h, #0x0\n"
+ "uadalp v29.4s, v30.8h\n"
+ "movi v30.8h, #0x0\n"
+ "uadalp v26.4s, v27.8h\n"
+ "movi v27.8h, #0x0\n"
+ "uadalp v23.4s, v24.8h\n"
+ "movi v24.8h, #0x0\n"
+ "uadalp v20.4s, v21.8h\n"
+ "movi v21.8h, #0x0\n"
+ "uadalp v17.4s, v18.8h\n"
+ "movi v18.8h, #0x0\n"
+ "mov x9, #0x0\n"
+ "91:" // Height 6: Multiply loop: unique 6: no collapse
+ "uadalp v1.8h, v31.16b\n"
+ "uadalp v30.8h, v28.16b\n"
+ "uadalp v27.8h, v25.16b\n"
+ "uadalp v24.8h, v22.16b\n"
+ "uadalp v21.8h, v19.16b\n"
+ "uadalp v18.8h, v16.16b\n"
+ "add x9, x9, #0x1\n"
+ "sub x27, x27, #0x10\n"
+ "cmp x27, #0x20\n"
+ "bge 90b\n"
+ "92:" // Height 6: Multiply loop: Single iteration only
+ "sub x27, x27, #0x10\n"
+ "ldr q31, [x26, #0x0]\n"
+ "ldr q28, [x25, #0x0]\n"
+ "ldr q25, [x24, #0x0]\n"
+ "ldr q22, [x23, #0x0]\n"
+ "ldr q19, [x22, #0x0]\n"
+ "ldr q16, [x20, #0x0]\n"
+ "uadalp v1.8h, v31.16b\n"
+ "uadalp v30.8h, v28.16b\n"
+ "uadalp v27.8h, v25.16b\n"
+ "uadalp v24.8h, v22.16b\n"
+ "uadalp v21.8h, v19.16b\n"
+ "uadalp v18.8h, v16.16b\n"
+ "add x26, x26, #0x10\n"
+ "add x25, x25, #0x10\n"
+ "add x24, x24, #0x10\n"
+ "add x23, x23, #0x10\n"
+ "add x22, x22, #0x10\n"
+ "add x20, x20, #0x10\n"
+ "93:" // Height 6: Multiply loop: Main loop skip
+ "cbz x27, 102f\n"
+ "tbz x27, #3, 97f\n"
+ "ldr d31, [x26], #0x8\n"
+ "ldr d28, [x25], #0x8\n"
+ "ldr d25, [x24], #0x8\n"
+ "ldr d22, [x23], #0x8\n"
+ "ldr d19, [x22], #0x8\n"
+ "ldr d16, [x20], #0x8\n"
+ "tbz x27, #2, 95f\n"
+ "ld1 { v31.s }[2], [x26], #0x4\n"
+ "ld1 { v28.s }[2], [x25], #0x4\n"
+ "ld1 { v25.s }[2], [x24], #0x4\n"
+ "ld1 { v22.s }[2], [x23], #0x4\n"
+ "ld1 { v19.s }[2], [x22], #0x4\n"
+ "ld1 { v16.s }[2], [x20], #0x4\n"
+ "tbz x27, #1, 94f\n"
+ "ld1 { v31.h }[6], [x26], #0x2\n"
+ "ld1 { v28.h }[6], [x25], #0x2\n"
+ "ld1 { v25.h }[6], [x24], #0x2\n"
+ "ld1 { v22.h }[6], [x23], #0x2\n"
+ "ld1 { v19.h }[6], [x22], #0x2\n"
+ "ld1 { v16.h }[6], [x20], #0x2\n"
+ "tbz x27, #0, 101f\n"
+ "ld1 { v31.b }[14], [x26]\n"
+ "ld1 { v28.b }[14], [x25]\n"
+ "ld1 { v25.b }[14], [x24]\n"
+ "ld1 { v22.b }[14], [x23]\n"
+ "ld1 { v19.b }[14], [x22]\n"
+ "ld1 { v16.b }[14], [x20]\n"
+ "b 101f\n"
+ "94:" // Height 6: Multiply loop: Ragged operand read: partial_1_12
+ "tbz x27, #0, 101f\n"
+ "ld1 { v31.b }[12], [x26]\n"
+ "ld1 { v28.b }[12], [x25]\n"
+ "ld1 { v25.b }[12], [x24]\n"
+ "ld1 { v22.b }[12], [x23]\n"
+ "ld1 { v19.b }[12], [x22]\n"
+ "ld1 { v16.b }[12], [x20]\n"
+ "b 101f\n"
+ "95:" // Height 6: Multiply loop: Ragged operand read: partial_2_8
+ "tbz x27, #1, 96f\n"
+ "ld1 { v31.h }[4], [x26], #0x2\n"
+ "ld1 { v28.h }[4], [x25], #0x2\n"
+ "ld1 { v25.h }[4], [x24], #0x2\n"
+ "ld1 { v22.h }[4], [x23], #0x2\n"
+ "ld1 { v19.h }[4], [x22], #0x2\n"
+ "ld1 { v16.h }[4], [x20], #0x2\n"
+ "tbz x27, #0, 101f\n"
+ "ld1 { v31.b }[10], [x26]\n"
+ "ld1 { v28.b }[10], [x25]\n"
+ "ld1 { v25.b }[10], [x24]\n"
+ "ld1 { v22.b }[10], [x23]\n"
+ "ld1 { v19.b }[10], [x22]\n"
+ "ld1 { v16.b }[10], [x20]\n"
+ "b 101f\n"
+ "96:" // Height 6: Multiply loop: Ragged operand read: partial_1_8
+ "tbz x27, #0, 101f\n"
+ "ld1 { v31.b }[8], [x26]\n"
+ "ld1 { v28.b }[8], [x25]\n"
+ "ld1 { v25.b }[8], [x24]\n"
+ "ld1 { v22.b }[8], [x23]\n"
+ "ld1 { v19.b }[8], [x22]\n"
+ "ld1 { v16.b }[8], [x20]\n"
+ "b 101f\n"
+ "97:" // Height 6: Multiply loop: Ragged operand read: partial_4_0
+ "tbz x27, #2, 99f\n"
+ "ldr s31, [x26], #0x4\n"
+ "ldr s28, [x25], #0x4\n"
+ "ldr s25, [x24], #0x4\n"
+ "ldr s22, [x23], #0x4\n"
+ "ldr s19, [x22], #0x4\n"
+ "ldr s16, [x20], #0x4\n"
+ "tbz x27, #1, 98f\n"
+ "ld1 { v31.h }[2], [x26], #0x2\n"
+ "ld1 { v28.h }[2], [x25], #0x2\n"
+ "ld1 { v25.h }[2], [x24], #0x2\n"
+ "ld1 { v22.h }[2], [x23], #0x2\n"
+ "ld1 { v19.h }[2], [x22], #0x2\n"
+ "ld1 { v16.h }[2], [x20], #0x2\n"
+ "tbz x27, #0, 101f\n"
+ "ld1 { v31.b }[6], [x26]\n"
+ "ld1 { v28.b }[6], [x25]\n"
+ "ld1 { v25.b }[6], [x24]\n"
+ "ld1 { v22.b }[6], [x23]\n"
+ "ld1 { v19.b }[6], [x22]\n"
+ "ld1 { v16.b }[6], [x20]\n"
+ "b 101f\n"
+ "98:" // Height 6: Multiply loop: Ragged operand read: partial_1_4
+ "tbz x27, #0, 101f\n"
+ "ld1 { v31.b }[4], [x26]\n"
+ "ld1 { v28.b }[4], [x25]\n"
+ "ld1 { v25.b }[4], [x24]\n"
+ "ld1 { v22.b }[4], [x23]\n"
+ "ld1 { v19.b }[4], [x22]\n"
+ "ld1 { v16.b }[4], [x20]\n"
+ "b 101f\n"
+ "99:" // Height 6: Multiply loop: Ragged operand read: partial_2_0
+ "tbz x27, #1, 100f\n"
+ "ldr h31, [x26], #0x2\n"
+ "ldr h28, [x25], #0x2\n"
+ "ldr h25, [x24], #0x2\n"
+ "ldr h22, [x23], #0x2\n"
+ "ldr h19, [x22], #0x2\n"
+ "ldr h16, [x20], #0x2\n"
+ "tbz x27, #0, 101f\n"
+ "ld1 { v31.b }[2], [x26]\n"
+ "ld1 { v28.b }[2], [x25]\n"
+ "ld1 { v25.b }[2], [x24]\n"
+ "ld1 { v22.b }[2], [x23]\n"
+ "ld1 { v19.b }[2], [x22]\n"
+ "ld1 { v16.b }[2], [x20]\n"
+ "b 101f\n"
+ "100:" // Height 6: Multiply loop: Ragged operand read: partial_1_0
+ "ldr b31, [x26, #0x0]\n"
+ "ldr b28, [x25, #0x0]\n"
+ "ldr b25, [x24, #0x0]\n"
+ "ldr b22, [x23, #0x0]\n"
+ "ldr b19, [x22, #0x0]\n"
+ "ldr b16, [x20, #0x0]\n"
+ "101:" // Height 6: Multiply loop: Ragged operand read: Done
+ "uadalp v1.8h, v31.16b\n"
+ "uadalp v30.8h, v28.16b\n"
+ "uadalp v27.8h, v25.16b\n"
+ "uadalp v24.8h, v22.16b\n"
+ "uadalp v21.8h, v19.16b\n"
+ "uadalp v18.8h, v16.16b\n"
+ "102:" // Height 6: Multiply loop: No odd multiplies
+ "add x28, x28, #0x1\n"
+ "cmp x28, x21\n"
+ "bne 87b\n"
+ "uadalp v0.4s, v1.8h\n"
+ "uadalp v29.4s, v30.8h\n"
+ "addp v0.4s, v0.4s, v29.4s\n"
+ "uadalp v26.4s, v27.8h\n"
+ "uadalp v23.4s, v24.8h\n"
+ "addp v29.4s, v26.4s, v23.4s\n"
+ "uadalp v20.4s, v21.8h\n"
+ "uadalp v17.4s, v18.8h\n"
+ "addp v0.4s, v0.4s, v29.4s\n"
+ "subs %x[M], %x[M], #0x6\n"
+ "addp v20.4s, v20.4s, v17.4s\n"
+ "mul v0.4s, v0.4s, v2.4s\n"
+ "st1 { v0.4s }, [%x[out_ptr]], #0x10\n"
+ "addp v20.4s, v20.4s, v20.4s\n"
+ "mul v20.4s, v20.4s, v2.4s\n"
+ "str d20, [%x[out_ptr]], #0x8\n"
+ "beq 104f\n"
+ "tbz %x[flags], #3, 103f\n"
+ "add %x[input_offset], %x[input_offset], #0x6\n"
+ "b 1b\n"
+ "103:" // Update direct input
+ "mov x19, #0x6\n"
+ "madd %x[input_ptr], x19, %x[input_offset], %x[input_ptr]\n"
+ "b 1b\n"
+ "104:" // Exit
+
+ : [M] "+r" (M), [input_offset] "+r" (input_offset), [input_ptr] "+r" (input_ptr), [out_ptr] "+r" (out_ptr)
+ : [args_ptr] "r" (&ka), [b_offset] "I" (offsetof(Requantize32, b_offset)), [flags] "r" (flags), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths)), [qp] "r" (qp)
+ : "cc", "memory", "v0", "v1", "v2", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
+ );
+}
+
+} // namespace arm_gemm
+
+#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_gemm/std_transforms_fixed.hpp b/src/core/NEON/kernels/arm_gemm/std_transforms_fixed.hpp
index 1d3aee7..4669be9 100644
--- a/src/core/NEON/kernels/arm_gemm/std_transforms_fixed.hpp
+++ b/src/core/NEON/kernels/arm_gemm/std_transforms_fixed.hpp
@@ -23,8 +23,10 @@
*/
#pragma once
+#include "convolver.hpp"
#include "mergeresults.hpp"
#include "transform.hpp"
+#include "interleave_indirect.hpp"
namespace arm_gemm {
@@ -39,14 +41,26 @@
* The optional 'block' parameter is for kernels using dot-product type
* instructions like UDOT and SDOT.
*/
-template<typename TOperand, typename TResult, unsigned int height, unsigned int width, unsigned int block=1>
+template<typename TOperand, typename TResult, unsigned int height, unsigned int width, unsigned int block=1, bool integrate_sums=false>
class StdTransformsFixed
{
public:
template<typename TIn>
void PrepareA(TOperand *out, const TIn *in, const int stride, const int y0,
- const int ymax, const int k0, const int kmax) const {
- Transform<height, block, false>(out, in, stride, y0, ymax, k0, kmax);
+ const int ymax, const int k0, const int kmax, int32_t row_sum_multiplier) const {
+ Interleave<height, block, VLType::None>(out, in, stride, y0, ymax, k0, kmax, integrate_sums, row_sum_multiplier);
+ }
+
+ template<typename TIn>
+ void PrepareA_indirect(TOperand *out, const TIn * const * const *ptr, size_t stringlen, size_t rounded_stringlen, const int y0,
+ const int ymax, const int k0, const int kmax, int32_t row_sum_multiplier) {
+ IndirectInterleave<height, block, VLType::None>(out, ptr, stringlen, rounded_stringlen, y0, ymax, k0, kmax, integrate_sums, row_sum_multiplier);
+ }
+
+ template<typename TIn>
+ void PrepareA_convolution(TOperand *out, const TIn *ptr, size_t stride, const convolver<TIn> &conv, size_t rounded_stringlen,
+ const int y0, const int ymax, const int k0, const int kmax, int32_t row_sum_multiplier) {
+ ConvolutionInterleave<height, block, VLType::None>(out, ptr, stride, conv, rounded_stringlen, y0, ymax, k0, kmax, integrate_sums, row_sum_multiplier);
}
template<typename TIn>
diff --git a/src/core/NEON/kernels/arm_gemm/std_transforms_sve.hpp b/src/core/NEON/kernels/arm_gemm/std_transforms_sve.hpp
index 13c4c47..3256d91 100644
--- a/src/core/NEON/kernels/arm_gemm/std_transforms_sve.hpp
+++ b/src/core/NEON/kernels/arm_gemm/std_transforms_sve.hpp
@@ -23,6 +23,7 @@
*/
#pragma once
+#include "convolver.hpp"
#include "mergeresults.hpp"
#include "transform.hpp"
@@ -38,20 +39,32 @@
* The optional 'block' parameter is for kernels using dot-product type
* instructions like UDOT and SDOT.
*/
-template<typename TOperand, typename TResult, unsigned int height, unsigned int width_vectors, unsigned int block=1, unsigned int mmla=1>
+template<typename TOperand, typename TResult, unsigned int height, unsigned int width_vectors, unsigned int block=1, unsigned int mmla=1, bool integrate_sums=false>
class StdTransformsSVE
{
public:
template<typename TIn>
void PrepareA(TOperand *out, const TIn *in, const int stride, const int y0,
- const int ymax, const int k0, const int kmax) {
- Transform<height, block, false>(out, in, stride, y0, ymax, k0, kmax);
+ const int ymax, const int k0, const int kmax, int32_t row_sum_multiplier) {
+ Interleave<height, block, VLType::None>(out, in, stride, y0, ymax, k0, kmax, integrate_sums, row_sum_multiplier);
+ }
+
+ template<typename TIn>
+ void PrepareA_indirect(TOperand *out, const TIn * const * const *ptr, size_t stringlen, size_t rounded_stringlen, const int y0,
+ const int ymax, const int k0, const int kmax, int32_t row_sum_multiplier) {
+ IndirectInterleave<height, block, VLType::None>(out, ptr, stringlen, rounded_stringlen, y0, ymax, k0, kmax, integrate_sums, row_sum_multiplier);
+ }
+
+ template<typename TIn>
+ void PrepareA_convolution(TOperand *out, const TIn *ptr, size_t stride, const convolver<TIn> &conv, size_t rounded_stringlen,
+ const int y0, const int ymax, const int k0, const int kmax, int32_t row_sum_multiplier) {
+ ConvolutionInterleave<height, block, VLType::None>(out, ptr, stride, conv, rounded_stringlen, y0, ymax, k0, kmax, integrate_sums, row_sum_multiplier);
}
template<typename TIn>
void PrepareB(TOperand *out, const TIn *in, const int stride, const int x0,
const int xmax, const int k0, const int kmax) {
- Transform<width_vectors, block, true, true>(out, in, stride, x0, xmax, k0, kmax);
+ Transform<width_vectors, block, true, VLType::SVE>(out, in, stride, x0, xmax, k0, kmax);
}
template<typename TOut>
diff --git a/src/core/NEON/kernels/arm_gemm/transform.hpp b/src/core/NEON/kernels/arm_gemm/transform.hpp
index c6ea079..5efeee5 100644
--- a/src/core/NEON/kernels/arm_gemm/transform.hpp
+++ b/src/core/NEON/kernels/arm_gemm/transform.hpp
@@ -38,13 +38,13 @@
* Need to cope with the work requested in either dimension not actually
* being a multiple of the block sizes.
*/
-template <unsigned int tIntBy, unsigned int BlockBy, bool Transposed, size_t TOutSize, size_t TInSize, bool sve>
+template <unsigned int tIntBy, unsigned int BlockBy, bool Transposed, size_t TOutSize, size_t TInSize, VLType vlt>
struct TransformImpl {
template <typename TOut, typename TIn>
static void Transform(TOut* out, const TIn* const in, const int stride,
const int y0, const int ymax, const int x0, const int xmax) {
// For SVE cases we multiply the interleave factor by the vector length.
- const unsigned int IntBy = tIntBy * (sve ? get_vector_length<TOut>() / BlockBy : 1);
+ const unsigned int IntBy = tIntBy * (vlt == VLType::SVE ? get_vector_length<TOut>() / BlockBy : 1);
const int n_whole_y_blocks = (ymax - y0) / IntBy;
const int y_remainders = (ymax - y0) % IntBy;
@@ -105,13 +105,13 @@
};
/*****************************************************************************/
-template <unsigned int IntBy, unsigned int BlockBy, bool Transposed, bool sve=false, typename TOut, typename TIn>
+template <unsigned int IntBy, unsigned int BlockBy, bool Transposed, VLType vlt=VLType::None, typename TOut, typename TIn>
void Transform(
TOut* out, const TIn* const in, const int stride,
const int k0, const int kmax, const int x0, const int xmax
) {
// Redirect to a specialised implementation predicated on argument size.
- TransformImpl<IntBy, BlockBy, Transposed, sizeof(TOut), sizeof(TIn), sve>::Transform(
+ TransformImpl<IntBy, BlockBy, Transposed, sizeof(TOut), sizeof(TIn), vlt>::Transform(
out, in, stride, k0, kmax, x0, xmax
);
}
diff --git a/src/core/NEON/kernels/arm_gemm/transforms/a32_interleave_6way_32bit.hpp b/src/core/NEON/kernels/arm_gemm/transforms/a32_interleave_6way_32bit.hpp
deleted file mode 100644
index 2df5d1b..0000000
--- a/src/core/NEON/kernels/arm_gemm/transforms/a32_interleave_6way_32bit.hpp
+++ /dev/null
@@ -1,167 +0,0 @@
-/*
- * Copyright (c) 2017-2018 Arm Limited.
- *
- * SPDX-License-Identifier: MIT
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to
- * deal in the Software without restriction, including without limitation the
- * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
- * sell copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-#pragma once
-
-#ifdef __arm__
-
-#include <arm_neon.h>
-
-#include "../asmlib.hpp"
-
-template<>
-template<typename T>
-inline void TransformImpl<6, 1, false, 4, 4, false>::Transform(T *out, const T *in, int ldin, int y0, int ymax, int k0, int kmax) {
- uint32_t *outptr = reinterpret_cast<uint32_t *>(out);
- const uint32_t *inptr = reinterpret_cast<const uint32_t *>(in);
- bool first = true;
-
- uint32_t zerobuff[16] = { 0 }; // 8 for asm loop plus up to 7 for overflow loop
-
- for (int y=y0; y<ymax; y+=6) {
- const uint32_t *inptr0 = inptr + y * ldin + k0;
- const uint32_t *inptr1 = inptr0 + ldin;
- const uint32_t *inptr2 = inptr1 + ldin;
- const uint32_t *inptr3 = inptr2 + ldin;
- const uint32_t *inptr4 = inptr3 + ldin;
- const uint32_t *inptr5 = inptr4 + ldin;
-
- //prefetch_2x(inptr0);
- //prefetch_2x(inptr1);
- //prefetch_2x(inptr2);
- //prefetch_2x(inptr3);
- //prefetch_2x(inptr4);
- //prefetch_2x(inptr5);
-
- int x=(kmax-k0);
- for (;(x>7) || first;x-=8) {
- /* Cope with ragged cases by copying from a buffer of zeroes instead */
- /* 'first' forces this to always run at least once, needed if the total size is <=7. */
- if ((y + 5) >= ymax) {
- switch ((y + 5) - ymax) {
- case 4:
- inptr1 = zerobuff;
- // fall through
- case 3:
- inptr2 = zerobuff;
- // fall through
- case 2:
- inptr3 = zerobuff;
- // fall through
- case 1:
- inptr4 = zerobuff;
- // fall through
- case 0:
- inptr5 = zerobuff;
- break;
-
- default:
- UNREACHABLE("Impossible.");
- }
- }
-
- if (first) {
- if (x<=7) {
- break;
- }
-
- first = false;
- }
-
- __asm __volatile (
- // Load up 8 elements (2 vectors) from each of 8 sources.
- "VLD1.32 {d0-d3}, [%[inptr0]]!\n" // q0=A0A1A2A3
- "VLD1.32 {d4-d7}, [%[inptr1]]!\n" // q2=B0B1B2B3
- "VLD1.32 {d8-d11}, [%[inptr2]]!\n" // q4=C0C1C2C3
- "VZIP.32 q0, q4\n" // q0=A0C0A1C1, q4 = A2C2A3C3
- "VLD1.32 {d12-d15}, [%[inptr3]]!\n" // q6=D0D1D2D3
- "VZIP.32 q2, q6\n" // q2=B0D0B1D1, q6 = B2D2B3D3
- "VLD1.32 {d16-d19}, [%[inptr4]]!\n"
- "VLD1.32 {d20-d23}, [%[inptr5]]!\n"
- "VZIP.32 q8, q10\n" // q8=E0F0E1F1, q10 = E2F2E3F3
- ASM_PREFETCH("[%[inptr0], #128]")
- "VZIP.32 q0, q2\n" // q0 = A0B0C0D0, q2 = A1B1C1D1
-
- // Store first elements
- "VST1.32 {d0-d1}, [%[outptr]]!\n"
- "VST1.32 {d16}, [%[outptr]]!\n"
-
- "VZIP.32 q4, q6\n" // q4 = A2B2C2D2, q6 = A3B3C3D3
-
- // Store second elements
- "VST1.32 {d4-d5}, [%[outptr]]!\n"
- "VZIP.32 q1, q5\n"
- ASM_PREFETCH("[%[inptr1], #128]")
- "VST1.32 {d17}, [%[outptr]]!\n"
- "VZIP.32 q3, q7\n"
-
- // Store third elements
- "VZIP.32 q9, q11\n"
- "VST1.32 {d8-d9}, [%[outptr]]!\n"
- "VZIP.32 q1, q3\n"
- ASM_PREFETCH("[%[inptr2], #128]")
- "VST1.32 {d20}, [%[outptr]]!\n"
-
- // Store fourth elements
- "VZIP.32 q5, q7\n"
- "VST1.32 {d12-d13}, [%[outptr]]!\n"
- ASM_PREFETCH("[%[inptr3], #128]")
- "VST1.32 {d21}, [%[outptr]]!\n"
-
- // Fifth
- "VST1.32 {d2-d3}, [%[outptr]]!\n"
- ASM_PREFETCH("[%[inptr4], #128]")
- "VST1.32 {d18}, [%[outptr]]!\n"
-
- // Sixth
- "VST1.32 {d6-d7}, [%[outptr]]!\n"
- ASM_PREFETCH("[%[inptr5], #128]")
- "VST1.32 {d19}, [%[outptr]]!\n"
-
- // Seventh
- "VST1.32 {d10-d11}, [%[outptr]]!\n"
- "VST1.32 {d22}, [%[outptr]]!\n"
-
- // Eighth
- "VST1.32 {d14-d15}, [%[outptr]]!\n"
- "VST1.32 {d23}, [%[outptr]]!\n"
-
- : [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3),
- [inptr4] "+r" (inptr4), [inptr5] "+r" (inptr5), [outptr] "+r" (outptr)
- :
- : "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q8", "q9", "q10", "q11", "q12", "memory"
- );
- }
-
- for (;x>0;x--) {
- *outptr++ = *inptr0++;
- *outptr++ = *inptr1++;
- *outptr++ = *inptr2++;
- *outptr++ = *inptr3++;
- *outptr++ = *inptr4++;
- *outptr++ = *inptr5++;
- }
- }
-}
-
-#endif // __arm__
diff --git a/src/core/NEON/kernels/arm_gemm/transforms/a32_transpose_interleave_8way_32bit.hpp b/src/core/NEON/kernels/arm_gemm/transforms/a32_transpose_interleave_8way_32bit.hpp
index 8f0b8ae..3ce1d32 100644
--- a/src/core/NEON/kernels/arm_gemm/transforms/a32_transpose_interleave_8way_32bit.hpp
+++ b/src/core/NEON/kernels/arm_gemm/transforms/a32_transpose_interleave_8way_32bit.hpp
@@ -30,22 +30,22 @@
// Generic unblocked transposed 8x32-bit sized specialisation
template <>
template <typename T>
-inline void TransformImpl<8, 1, true, 4, 4, false>::Transform(
+inline void TransformImpl<8, 1, true, 4, 4, VLType::None>::Transform(
T* out, const T* const in, const int stride,
const int x0, const int xmax, const int k0, const int kmax
) {
// Redirect to a 16x uint16_t specialisation
- TransformImpl<16, 1, true, 2, 2, false>::Transform(
+ TransformImpl<16, 1, true, 2, 2, VLType::None>::Transform(
reinterpret_cast<uint16_t *>(out),
reinterpret_cast<const uint16_t *>(in),
stride*2, x0*2, xmax*2, k0, kmax
);
}
-// Generic 12x16-bit sized specialisation
+// Generic 16x16-bit sized specialisation
template <>
template <typename T>
-inline void TransformImpl<16, 1, true, 2, 2, false>::Transform(
+inline void TransformImpl<16, 1, true, 2, 2, VLType::None>::Transform(
T* out, const T* const in, const int stride,
const int x0, const int xmax, const int k0, const int kmax
) {
@@ -117,7 +117,7 @@
template <>
template <>
-inline void TransformImpl<16, 1, true, 2, 2, false>::Transform(
+inline void TransformImpl<16, 1, true, 2, 2, VLType::None>::Transform(
uint16_t* out, const uint16_t* const in, const int stride,
const int x0, const int xmax, const int k0, const int kmax
) {
diff --git a/src/core/NEON/kernels/arm_gemm/transforms/a64_block16_interleave4_8bit.hpp b/src/core/NEON/kernels/arm_gemm/transforms/a64_block16_interleave4_8bit.hpp
deleted file mode 100644
index 9b6f4de..0000000
--- a/src/core/NEON/kernels/arm_gemm/transforms/a64_block16_interleave4_8bit.hpp
+++ /dev/null
@@ -1,128 +0,0 @@
-/*
- * Copyright (c) 2017-2020 Arm Limited.
- *
- * SPDX-License-Identifier: MIT
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to
- * deal in the Software without restriction, including without limitation the
- * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
- * sell copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-#pragma once
-
-#ifdef __aarch64__
-
-#include <arm_neon.h>
-
-#include "../asmlib.hpp"
-#include "../utils.hpp"
-
-template<>
-template<typename T>
-void TransformImpl<4, 16, false, 1, 1, false>::Transform(T *out, const T *in, int ldin, int y0, int ymax, int k0, int kmax) {
- uint8_t *outptr = (uint8_t *)out;
- const uint8_t *inptr = (uint8_t *)in;
-
- uint8_t zerobuff[16] = { 0 };
-
- for (int y=y0; y<ymax; y+=4) {
- const uint8_t *inptr0 = inptr + static_cast<intptr_t>(y) * ldin + k0;
- const uint8_t *inptr1 = inptr0 + ldin;
- const uint8_t *inptr2 = inptr1 + ldin;
- const uint8_t *inptr3 = inptr2 + ldin;
-
- prefetch_2x(inptr0);
- prefetch_2x(inptr1);
- prefetch_2x(inptr2);
- prefetch_2x(inptr3);
-
- int x=(kmax-k0);
- for (;x>15;x-=16) {
- /* Cope with ragged cases by copying from a buffer of zeroes instead */
- if ((y + 3) >= ymax) {
- switch ((y + 3) - ymax) {
- case 2:
- inptr1 = zerobuff;
- // fall through
- case 1:
- inptr2 = zerobuff;
- // fall through
- case 0:
- inptr3 = zerobuff;
- break;
-
- default:
- UNREACHABLE("Impossible.");
- }
- }
-
- __asm __volatile (
- "LDR q0, [%[inptr0]], #16\n"
- ASM_PREFETCH("[%[inptr0], #176]")
- "LDR q1, [%[inptr1]], #16\n"
- ASM_PREFETCH("[%[inptr1], #176]")
- "STP q0, q1, [%[outptr]], #32\n"
- "LDR q0, [%[inptr2]], #16\n"
- ASM_PREFETCH("[%[inptr2], #176]")
- "LDR q1, [%[inptr3]], #16\n"
- ASM_PREFETCH("[%[inptr3], #176]")
- "STP q0, q1, [%[outptr]], #32\n"
- : [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3),
- [outptr] "+r" (outptr)
- :
- : "v0", "v1"
- );
- }
-
- if (x>0) {
- /* Need to duplicate this here, in case we didn't run the main loop. */
- if ((y + 3) >= ymax) {
- switch ((y + 3) - ymax) {
- case 2:
- inptr1 = zerobuff;
- // fall through
- case 1:
- inptr2 = zerobuff;
- // fall through
- case 0:
- inptr3 = zerobuff;
- break;
-
- default:
- UNREACHABLE("Impossible.");
- }
- }
-
- /* We have to write out 16 values, copy as many legal values as there are and pad with 0 */
- auto f = [&outptr, x](const uint8_t *&p) {
- for (int i=0; i<16; i++) {
- if (i < x) {
- *outptr++ = *p++;
- } else {
- *outptr++ = 0;
- }
- }
- };
-
- f(inptr0);
- f(inptr1);
- f(inptr2);
- f(inptr3);
- }
- }
-}
-
-#endif // __aarch64__
\ No newline at end of file
diff --git a/src/core/NEON/kernels/arm_gemm/transforms/a64_interleave_8way_16bit.hpp b/src/core/NEON/kernels/arm_gemm/transforms/a64_interleave_8way_16bit.hpp
deleted file mode 100644
index 3d912c4..0000000
--- a/src/core/NEON/kernels/arm_gemm/transforms/a64_interleave_8way_16bit.hpp
+++ /dev/null
@@ -1,182 +0,0 @@
-/*
- * Copyright (c) 2017-2020 Arm Limited.
- *
- * SPDX-License-Identifier: MIT
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to
- * deal in the Software without restriction, including without limitation the
- * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
- * sell copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-#pragma once
-
-#ifdef __aarch64__
-
-#include <arm_neon.h>
-
-#include "../asmlib.hpp"
-
-template<>
-template<typename T>
-void TransformImpl<8, 1, false, 2, 2, false>::Transform(T *out, const T *in, int ldin, int y0, int ymax, int k0, int kmax) {
- uint16_t *outptr = (uint16_t *)out;
- const uint16_t *inptr = (const uint16_t *)in;
- bool first=true;
-
- uint16_t zerobuff[16] = { 0 }; // 8 for asm loop plus up to 7 for overflow loop
-
- for (int y=y0; y<ymax; y+=8) {
- const uint16_t *inptr0 = inptr + y * ldin + k0;
- const uint16_t *inptr1 = inptr0 + ldin;
- const uint16_t *inptr2 = inptr1 + ldin;
- const uint16_t *inptr3 = inptr2 + ldin;
- const uint16_t *inptr4 = inptr3 + ldin;
- const uint16_t *inptr5 = inptr4 + ldin;
- const uint16_t *inptr6 = inptr5 + ldin;
- const uint16_t *inptr7 = inptr6 + ldin;
-
- prefetch_2x(inptr0);
- prefetch_2x(inptr1);
- prefetch_2x(inptr2);
- prefetch_2x(inptr3);
- prefetch_2x(inptr4);
- prefetch_2x(inptr5);
- prefetch_2x(inptr6);
- prefetch_2x(inptr7);
-
- int x=(kmax-k0);
- for (;(x>7) || first;x-=8) {
- /* Cope with ragged cases by copying from a buffer of zeroes instead */
- /* 'first' forces this to always run at least once, needed if the total size is <=7. */
- if ((y + 7) >= ymax) {
- switch ((y + 7) - ymax) {
- case 6:
- inptr1 = zerobuff;
- // fall through
- case 5:
- inptr2 = zerobuff;
- // fall through
- case 4:
- inptr3 = zerobuff;
- // fall through
- case 3:
- inptr4 = zerobuff;
- // fall through
- case 2:
- inptr5 = zerobuff;
- // fall through
- case 1:
- inptr6 = zerobuff;
- // fall through
- case 0:
- inptr7 = zerobuff;
- break;
-
- default:
- UNREACHABLE("Impossible.");
- }
- }
-
- if (first) {
- if (x <= 7) {
- break;
- }
-
- first = false;
- }
-
- int skippf = (x & 31);
- __asm __volatile (
- // Load up 8 elements (1 vector) from each of 8 sources.
- "CBNZ %w[skippf], 1f\n"
- ASM_PREFETCH("[%[inptr0], #128]")
- ASM_PREFETCH("[%[inptr1], #128]")
- ASM_PREFETCH("[%[inptr2], #128]")
- ASM_PREFETCH("[%[inptr3], #128]")
- "1:\n"
-
- "LDR q0, [%[inptr0]], #16\n" // q0=A0A1A2A3A4A5A6A7
- "LDR q4, [%[inptr4]], #16\n" // q8=E0E1E2E3E4E5E6E7
- "LDR q2, [%[inptr2]], #16\n" // q4=C0C1C2C3...
- "LDR q6, [%[inptr6]], #16\n"
- "ZIP1 v8.8h, v0.8h, v4.8h\n" // q8=A0E0A1E1A2E2A3E3
- "ZIP2 v16.8h, v0.8h, v4.8h\n" // q16=A4E4A5E5A6E6A7E7
- "ZIP1 v9.8h, v2.8h, v6.8h\n" // q9=C0G0C1G1C2G2C3G3
- "ZIP2 v17.8h, v2.8h, v6.8h\n" // q17=C4G4C5G5C6G6C7G7
- "LDR q1, [%[inptr1]], #16\n" // q1=B0B1B2B3B4B5B6B7
- "LDR q5, [%[inptr5]], #16\n"
- "LDR q3, [%[inptr3]], #16\n" // q3=D0D1D2D3....
- "LDR q7, [%[inptr7]], #16\n"
- "ZIP1 v10.8h, v1.8h, v5.8h\n" // q18=B0F0B1F1B2F2B3F3
- "ZIP2 v18.8h, v1.8h, v5.8h\n" // q18=B4F4B5F5B6F6B7F7
- "ZIP1 v11.8h, v3.8h, v7.8h\n" // q19=D0H0D1H1D2H2D3H3
- "ZIP2 v19.8h, v3.8h, v7.8h\n" // q19=D4H4D5H5D6H6D7H7
-
- "ZIP1 v12.8h, v8.8h, v9.8h\n" // q20=A0C0E0G0A1C1E1G1
- "ZIP2 v20.8h, v8.8h, v9.8h\n"
- "ZIP1 v13.8h, v10.8h, v11.8h\n" // q21=B0D0F0H0B1I1F1H1
- "ZIP2 v21.8h, v10.8h, v11.8h\n"
-
- "CBNZ %w[skippf], 2f\n"
- ASM_PREFETCH("[%[inptr4], #112]")
- ASM_PREFETCH("[%[inptr5], #112]")
- ASM_PREFETCH("[%[inptr6], #112]")
- ASM_PREFETCH("[%[inptr7], #112]")
- "2:\n"
-
- "ZIP1 v22.8h, v16.8h, v17.8h\n"
- "ZIP2 v30.8h, v16.8h, v17.8h\n"
- "ZIP1 v23.8h, v18.8h, v19.8h\n"
- "ZIP2 v31.8h, v18.8h, v19.8h\n"
-
- "ZIP1 v14.8h, v12.8h, v13.8h\n" // q22=A0B0C0D0E0F0G0H0
- "ZIP2 v15.8h, v12.8h, v13.8h\n" // q23=A1B1C1D1E1F1G1H1
- "STP q14, q15, [%[outptr]], #32\n" // Write back first two elements
-
- "ZIP1 v0.8h, v20.8h, v21.8h\n"
- "ZIP2 v1.8h, v20.8h, v21.8h\n"
- "STP q0, q1, [%[outptr]], #32\n" // Write back next two elements
-
- "ZIP1 v2.8h, v22.8h, v23.8h\n"
- "ZIP2 v3.8h, v22.8h, v23.8h\n"
- "STP q2, q3, [%[outptr]], #32\n" // Write back next two elements
-
- "ZIP1 v4.8h, v30.8h, v31.8h\n"
- "ZIP2 v5.8h, v30.8h, v31.8h\n"
- "STP q4, q5, [%[outptr]], #32\n" // Write back last two elements
- : [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3),
- [inptr4] "+r" (inptr4), [inptr5] "+r" (inptr5), [inptr6] "+r" (inptr6), [inptr7] "+r" (inptr7), [outptr] "+r" (outptr)
- : [skippf] "r" (skippf)
- : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12",
- "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24",
- "v25", "v26", "v27", "v28", "v29", "v30", "v31", "memory"
- );
- }
-
- for (;x>0;x--) {
- *outptr++ = *inptr0++;
- *outptr++ = *inptr1++;
- *outptr++ = *inptr2++;
- *outptr++ = *inptr3++;
- *outptr++ = *inptr4++;
- *outptr++ = *inptr5++;
- *outptr++ = *inptr6++;
- *outptr++ = *inptr7++;
- }
- }
-}
-
-#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_gemm/transforms/a64_interleave_8way_32bit.hpp b/src/core/NEON/kernels/arm_gemm/transforms/a64_interleave_8way_32bit.hpp
deleted file mode 100644
index 701d688..0000000
--- a/src/core/NEON/kernels/arm_gemm/transforms/a64_interleave_8way_32bit.hpp
+++ /dev/null
@@ -1,191 +0,0 @@
-/*
- * Copyright (c) 2017-2020 Arm Limited.
- *
- * SPDX-License-Identifier: MIT
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to
- * deal in the Software without restriction, including without limitation the
- * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
- * sell copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-#pragma once
-
-#if defined(__aarch64__) && !defined(__ARM_FEATURE_SVE)
-
-#include <arm_neon.h>
-
-#include "../asmlib.hpp"
-
-template<>
-template<typename T>
-inline void TransformImpl<8, 1, false, 4, 4, false>::Transform(T *out, const T *in, int ldin, int y0, int ymax, int k0, int kmax) {
- uint32_t *outptr = (uint32_t *)out;
- const uint32_t *inptr = (uint32_t *)in;
- bool first = true;
-
- uint32_t zerobuff[16] = { 0 }; // 8 for asm loop plus up to 7 for overflow loop
-
- for (int y=y0; y<ymax; y+=8) {
- const uint32_t *inptr0 = inptr + y * ldin + k0;
- const uint32_t *inptr1 = inptr0 + ldin;
- const uint32_t *inptr2 = inptr1 + ldin;
- const uint32_t *inptr3 = inptr2 + ldin;
- const uint32_t *inptr4 = inptr3 + ldin;
- const uint32_t *inptr5 = inptr4 + ldin;
- const uint32_t *inptr6 = inptr5 + ldin;
- const uint32_t *inptr7 = inptr6 + ldin;
-
- prefetch_2x(inptr0);
- prefetch_2x(inptr1);
- prefetch_2x(inptr2);
- prefetch_2x(inptr3);
- prefetch_2x(inptr4);
- prefetch_2x(inptr5);
- prefetch_2x(inptr6);
- prefetch_2x(inptr7);
-
- int x=(kmax-k0);
- for (;(x>7) || first;x-=8) {
- /* Cope with ragged cases by copying from a buffer of zeroes instead */
- /* 'first' forces this to always run at least once, needed if the total size is <=7. */
- if ((y + 7) >= ymax) {
- switch ((y + 7) - ymax) {
- case 6:
- inptr1 = zerobuff;
- // fall through
- case 5:
- inptr2 = zerobuff;
- // fall through
- case 4:
- inptr3 = zerobuff;
- // fall through
- case 3:
- inptr4 = zerobuff;
- // fall through
- case 2:
- inptr5 = zerobuff;
- // fall through
- case 1:
- inptr6 = zerobuff;
- // fall through
- case 0:
- inptr7 = zerobuff;
- break;
-
- default:
- UNREACHABLE("Impossible.");
- }
- }
-
- if (first) {
- if (x<=7) {
- break;
- }
-
- first = false;
- }
-
- __asm __volatile (
- // Load up 8 elements (2 vectors) from each of 8 sources.
- "LDP q0, q1, [%[inptr0]], #32\n" // q0=A0A1A2A3
- "LDP q2, q3, [%[inptr1]], #32\n" // q2=B0B1B2B3
- "LDP q4, q5, [%[inptr2]], #32\n" // q4=C0C1C2C3
- "ZIP1 v16.4s, v0.4s, v4.4s\n" // q16=A0C0A1C1
- ASM_PREFETCH("[%[inptr0], #128]")
- "LDP q6, q7, [%[inptr3]], #32\n" // q6=D0D1D2D3
- "ZIP1 v17.4s, v2.4s, v6.4s\n" // q17=B0D0B1D1
- "LDP q8, q9, [%[inptr4]], #32\n"
- "LDP q10, q11, [%[inptr5]], #32\n"
- "LDP q12, q13, [%[inptr6]], #32\n"
- "ZIP1 v18.4s, v8.4s, v12.4s\n"
- ASM_PREFETCH("[%[inptr1], #128]")
- "LDP q14, q15, [%[inptr7]], #32\n"
- "ZIP1 v19.4s, v10.4s, v14.4s\n"
-
- "ZIP1 v20.4s, v16.4s, v17.4s\n" // q20=A0B0C0D0
- ASM_PREFETCH("[%[inptr2], #128]")
- "ZIP1 v21.4s, v18.4s, v19.4s\n"
- "ZIP2 v22.4s, v16.4s, v17.4s\n"
- "ZIP2 v23.4s, v18.4s, v19.4s\n"
-
- "ZIP2 v16.4s, v0.4s, v4.4s\n"
- ASM_PREFETCH("[%[inptr3], #128]")
- "ZIP2 v17.4s, v2.4s, v6.4s\n"
- "STP q20, q21, [%[outptr]], #32\n" // Write back the first element of each source
-
- "ZIP2 v18.4s, v8.4s, v12.4s\n"
- "ZIP2 v19.4s, v10.4s, v14.4s\n"
- "STP q22, q23, [%[outptr]], #32\n" // Write back the second element of each source
-
- "ZIP1 v20.4s, v16.4s, v17.4s\n"
- ASM_PREFETCH("[%[inptr4], #128]")
- "ZIP1 v21.4s, v18.4s, v19.4s\n"
- "ZIP2 v22.4s, v16.4s, v17.4s\n"
- "ZIP2 v23.4s, v18.4s, v19.4s\n"
-
- "ZIP1 v16.4s, v1.4s, v5.4s\n"
- ASM_PREFETCH("[%[inptr5], #128]")
- "ZIP1 v17.4s, v3.4s, v7.4s\n"
- "STP q20, q21, [%[outptr]], #32\n" // Third element
-
- "ZIP1 v18.4s, v9.4s, v13.4s\n"
- "ZIP1 v19.4s, v11.4s, v15.4s\n"
- "STP q22, q23, [%[outptr]], #32\n" // Fourth element
-
- "ZIP1 v20.4s, v16.4s, v17.4s\n"
- "ZIP1 v21.4s, v18.4s, v19.4s\n"
- "ZIP2 v22.4s, v16.4s, v17.4s\n"
- ASM_PREFETCH("[%[inptr6], #128]")
- "ZIP2 v23.4s, v18.4s, v19.4s\n"
-
- "ZIP2 v16.4s, v1.4s, v5.4s\n"
- "ZIP2 v17.4s, v3.4s, v7.4s\n"
- "STP q20, q21, [%[outptr]], #32\n" // Fifth element
-
- "ZIP2 v18.4s, v9.4s, v13.4s\n"
- ASM_PREFETCH("[%[inptr7], #128]")
- "ZIP2 v19.4s, v11.4s, v15.4s\n"
- "STP q22, q23, [%[outptr]], #32\n" // Sixth element
-
- "ZIP1 v20.4s, v16.4s, v17.4s\n"
- "ZIP1 v21.4s, v18.4s, v19.4s\n"
- "STP q20, q21, [%[outptr]], #32\n" // Seventh element
-
- "ZIP2 v22.4s, v16.4s, v17.4s\n"
- "ZIP2 v23.4s, v18.4s, v19.4s\n"
- "STP q22, q23, [%[outptr]], #32\n" // Eighth element
- : [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3),
- [inptr4] "+r" (inptr4), [inptr5] "+r" (inptr5), [inptr6] "+r" (inptr6), [inptr7] "+r" (inptr7), [outptr] "+r" (outptr)
- :
- : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12",
- "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "memory"
- );
- }
-
- for (;x>0;x--) {
- *outptr++ = *inptr0++;
- *outptr++ = *inptr1++;
- *outptr++ = *inptr2++;
- *outptr++ = *inptr3++;
- *outptr++ = *inptr4++;
- *outptr++ = *inptr5++;
- *outptr++ = *inptr6++;
- *outptr++ = *inptr7++;
- }
- }
-}
-
-#endif // __aarch64__ && !__ARM_FEATURE_SVE
diff --git a/src/core/NEON/kernels/arm_gemm/transforms/a64_interleave_8way_block4_8bit.hpp b/src/core/NEON/kernels/arm_gemm/transforms/a64_interleave_8way_block4_8bit.hpp
deleted file mode 100644
index 2546cc5..0000000
--- a/src/core/NEON/kernels/arm_gemm/transforms/a64_interleave_8way_block4_8bit.hpp
+++ /dev/null
@@ -1,228 +0,0 @@
-/*
- * Copyright (c) 2017-2020 Arm Limited.
- *
- * SPDX-License-Identifier: MIT
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to
- * deal in the Software without restriction, including without limitation the
- * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
- * sell copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-#pragma once
-
-#if defined(__aarch64__) && !defined(__ARM_FEATURE_SVE)
-
-#include <arm_neon.h>
-
-#include "../asmlib.hpp"
-
-template<>
-template<typename T>
-inline void TransformImpl<8, 4, false, 1, 1, false>::Transform(T *out, const T *in, int ldin, int y0, int ymax, int k0, int kmax) {
- uint8_t *outptr = reinterpret_cast<uint8_t *>(out);
- const uint8_t *inptr = reinterpret_cast<const uint8_t *>(in);
- bool first = true;
-
- /* Helper functions to copy blocks about used for odd case. */
- class t {
- public:
- static inline void copy_4_inc(uint8_t *&out, const uint8_t *&in) {
- uint32_t *out_word = reinterpret_cast<uint32_t *>(out);
- const uint32_t *in_word = reinterpret_cast<const uint32_t *>(in);
-
- *out_word++ = *in_word++;
-
- out = reinterpret_cast<uint8_t *>(out_word);
- in = reinterpret_cast<const uint8_t *>(in_word);
- }
-
- static inline void copy_pad(uint8_t *&out, const uint8_t *&in, size_t count) {
- for (unsigned int i=0; i<4; i++) {
- if (i < count) {
- *out++ = *in++;
- } else {
- *out++ = 0;
- }
- }
- }
- };
-
- uint8_t zerobuff[64] = { 0 }; // 32 for asm loop plus up to 31 for overflow loop
-
- for (int y=y0; y<ymax; y+=8) {
- const uint8_t *inptr0 = inptr + y * ldin + k0;
- const uint8_t *inptr1 = inptr0 + ldin;
- const uint8_t *inptr2 = inptr1 + ldin;
- const uint8_t *inptr3 = inptr2 + ldin;
- const uint8_t *inptr4 = inptr3 + ldin;
- const uint8_t *inptr5 = inptr4 + ldin;
- const uint8_t *inptr6 = inptr5 + ldin;
- const uint8_t *inptr7 = inptr6 + ldin;
-
- prefetch_2x(inptr0);
- prefetch_2x(inptr1);
- prefetch_2x(inptr2);
- prefetch_2x(inptr3);
- prefetch_2x(inptr4);
- prefetch_2x(inptr5);
- prefetch_2x(inptr6);
- prefetch_2x(inptr7);
-
- int x=(kmax-k0);
- for (;(x>31) || first;x-=32) {
- /* Cope with ragged cases by copying from a buffer of zeroes instead */
- /* 'first' forces this to always run at least once, needed if the total size is <=32. */
- if ((y + 7) >= ymax) {
- switch ((y + 7) - ymax) {
- case 6:
- inptr1 = zerobuff;
- // fall through
- case 5:
- inptr2 = zerobuff;
- // fall through
- case 4:
- inptr3 = zerobuff;
- // fall through
- case 3:
- inptr4 = zerobuff;
- // fall through
- case 2:
- inptr5 = zerobuff;
- // fall through
- case 1:
- inptr6 = zerobuff;
- // fall through
- case 0:
- inptr7 = zerobuff;
- break;
-
- default:
- UNREACHABLE("Impossible.");
- }
- }
-
- if (first) {
- if (x<=31) {
- break;
- }
-
- first = false;
- }
-
- __asm __volatile (
- // Load up 8 elements (2 vectors) from each of 8 sources.
- "LDP q0, q1, [%[inptr0]], #32\n" // q0=A0A1A2A3
- "LDP q2, q3, [%[inptr1]], #32\n" // q2=B0B1B2B3
- "LDP q4, q5, [%[inptr2]], #32\n" // q4=C0C1C2C3
- "ZIP1 v16.4s, v0.4s, v4.4s\n" // q16=A0C0A1C1
- ASM_PREFETCH("[%[inptr0], #128]")
- "LDP q6, q7, [%[inptr3]], #32\n" // q6=D0D1D2D3
- "ZIP1 v17.4s, v2.4s, v6.4s\n" // q17=B0D0B1D1
- "LDP q8, q9, [%[inptr4]], #32\n"
- "LDP q10, q11, [%[inptr5]], #32\n"
- "LDP q12, q13, [%[inptr6]], #32\n"
- "ZIP1 v18.4s, v8.4s, v12.4s\n"
- ASM_PREFETCH("[%[inptr1], #128]")
- "LDP q14, q15, [%[inptr7]], #32\n"
- "ZIP1 v19.4s, v10.4s, v14.4s\n"
-
- "ZIP1 v20.4s, v16.4s, v17.4s\n" // q20=A0B0C0D0
- ASM_PREFETCH("[%[inptr2], #128]")
- "ZIP1 v21.4s, v18.4s, v19.4s\n"
- "ZIP2 v22.4s, v16.4s, v17.4s\n"
- "ZIP2 v23.4s, v18.4s, v19.4s\n"
-
- "ZIP2 v16.4s, v0.4s, v4.4s\n"
- ASM_PREFETCH("[%[inptr3], #128]")
- "ZIP2 v17.4s, v2.4s, v6.4s\n"
- "STP q20, q21, [%[outptr]], #32\n" // Write back the first element of each source
-
- "ZIP2 v18.4s, v8.4s, v12.4s\n"
- "ZIP2 v19.4s, v10.4s, v14.4s\n"
- "STP q22, q23, [%[outptr]], #32\n" // Write back the second element of each source
-
- "ZIP1 v20.4s, v16.4s, v17.4s\n"
- ASM_PREFETCH("[%[inptr4], #128]")
- "ZIP1 v21.4s, v18.4s, v19.4s\n"
- "ZIP2 v22.4s, v16.4s, v17.4s\n"
- "ZIP2 v23.4s, v18.4s, v19.4s\n"
-
- "ZIP1 v16.4s, v1.4s, v5.4s\n"
- ASM_PREFETCH("[%[inptr5], #128]")
- "ZIP1 v17.4s, v3.4s, v7.4s\n"
- "STP q20, q21, [%[outptr]], #32\n" // Third element
-
- "ZIP1 v18.4s, v9.4s, v13.4s\n"
- "ZIP1 v19.4s, v11.4s, v15.4s\n"
- "STP q22, q23, [%[outptr]], #32\n" // Fourth element
-
- "ZIP1 v20.4s, v16.4s, v17.4s\n"
- "ZIP1 v21.4s, v18.4s, v19.4s\n"
- "ZIP2 v22.4s, v16.4s, v17.4s\n"
- ASM_PREFETCH("[%[inptr6], #128]")
- "ZIP2 v23.4s, v18.4s, v19.4s\n"
-
- "ZIP2 v16.4s, v1.4s, v5.4s\n"
- "ZIP2 v17.4s, v3.4s, v7.4s\n"
- "STP q20, q21, [%[outptr]], #32\n" // Fifth element
-
- "ZIP2 v18.4s, v9.4s, v13.4s\n"
- ASM_PREFETCH("[%[inptr7], #128]")
- "ZIP2 v19.4s, v11.4s, v15.4s\n"
- "STP q22, q23, [%[outptr]], #32\n" // Sixth element
-
- "ZIP1 v20.4s, v16.4s, v17.4s\n"
- "ZIP1 v21.4s, v18.4s, v19.4s\n"
- "STP q20, q21, [%[outptr]], #32\n" // Seventh element
-
- "ZIP2 v22.4s, v16.4s, v17.4s\n"
- "ZIP2 v23.4s, v18.4s, v19.4s\n"
- "STP q22, q23, [%[outptr]], #32\n" // Eighth element
- : [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3),
- [inptr4] "+r" (inptr4), [inptr5] "+r" (inptr5), [inptr6] "+r" (inptr6), [inptr7] "+r" (inptr7), [outptr] "+r" (outptr)
- :
- : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12",
- "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "memory"
- );
- }
-
- // Copy any leftover blocks of 4 a complete block at a time.
- for (;x>4;x-=4) {
- t::copy_4_inc(outptr, inptr0);
- t::copy_4_inc(outptr, inptr1);
- t::copy_4_inc(outptr, inptr2);
- t::copy_4_inc(outptr, inptr3);
- t::copy_4_inc(outptr, inptr4);
- t::copy_4_inc(outptr, inptr5);
- t::copy_4_inc(outptr, inptr6);
- t::copy_4_inc(outptr, inptr7);
- }
-
- // Final block with padding, if any.
- if (x > 0) {
- t::copy_pad(outptr, inptr0, x);
- t::copy_pad(outptr, inptr1, x);
- t::copy_pad(outptr, inptr2, x);
- t::copy_pad(outptr, inptr3, x);
- t::copy_pad(outptr, inptr4, x);
- t::copy_pad(outptr, inptr5, x);
- t::copy_pad(outptr, inptr6, x);
- t::copy_pad(outptr, inptr7, x);
- }
- }
-}
-
-#endif // __aarch64__ && !__ARM_FEATURE_SVE
diff --git a/src/core/NEON/kernels/arm_gemm/transforms/a64_interleave_8way_half_to_float.hpp b/src/core/NEON/kernels/arm_gemm/transforms/a64_interleave_8way_half_to_float.hpp
deleted file mode 100644
index a342d6c..0000000
--- a/src/core/NEON/kernels/arm_gemm/transforms/a64_interleave_8way_half_to_float.hpp
+++ /dev/null
@@ -1,207 +0,0 @@
-/*
- * Copyright (c) 2017-2020 Arm Limited.
- *
- * SPDX-License-Identifier: MIT
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to
- * deal in the Software without restriction, including without limitation the
- * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
- * sell copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-#pragma once
-
-#if defined(__aarch64__) && defined(__ARM_FP16_ARGS)
-
-#include <arm_neon.h>
-
-#include "../asmlib.hpp"
-
-template<>
-template<>
-inline void TransformImpl<8, 1, false, 4, 2, false>::Transform(float *out, const __fp16 *in, int ldin, int y0, int ymax, int k0, int kmax) {
- float *outptr = out;
- const __fp16 *inptr = in;
- bool first = true;
-
- __fp16 zerobuff[16] = { 0 }; // 8 for asm loop plus up to 7 for overflow loop
-
- for (int y=y0; y<ymax; y+=8) {
- const __fp16 *inptr0 = inptr + y * ldin + k0;
- const __fp16 *inptr1 = inptr0 + ldin;
- const __fp16 *inptr2 = inptr1 + ldin;
- const __fp16 *inptr3 = inptr2 + ldin;
- const __fp16 *inptr4 = inptr3 + ldin;
- const __fp16 *inptr5 = inptr4 + ldin;
- const __fp16 *inptr6 = inptr5 + ldin;
- const __fp16 *inptr7 = inptr6 + ldin;
-
- prefetch_2x(inptr0);
- prefetch_2x(inptr1);
- prefetch_2x(inptr2);
- prefetch_2x(inptr3);
- prefetch_2x(inptr4);
- prefetch_2x(inptr5);
- prefetch_2x(inptr6);
- prefetch_2x(inptr7);
-
- int x=(kmax-k0);
- for (;(x>7) || first;x-=8) {
- /* Cope with ragged cases by copying from a buffer of zeroes instead */
- /* 'first' forces this to always run at least once, needed if the total size is <=7. */
- if ((y + 7) >= ymax) {
- switch ((y + 7) - ymax) {
- case 6:
- inptr1 = zerobuff;
- // fall through
- case 5:
- inptr2 = zerobuff;
- // fall through
- case 4:
- inptr3 = zerobuff;
- // fall through
- case 3:
- inptr4 = zerobuff;
- // fall through
- case 2:
- inptr5 = zerobuff;
- // fall through
- case 1:
- inptr6 = zerobuff;
- // fall through
- case 0:
- inptr7 = zerobuff;
- break;
-
- default:
- UNREACHABLE("Impossible.");
- }
- }
-
- if (first) {
- if (x<=7) {
- break;
- }
-
- first = false;
- }
-
- __asm __volatile (
- // Load up 8 elements (2 vectors) from each of 8 sources.
- "LDR q0, [%[inptr0]], #16\n"
- "LDR q2, [%[inptr1]], #16\n"
- "FCVTL2 v1.4s, v0.8h\n"
- "FCVTL v0.4s, v0.4h\n"
- "LDR q4, [%[inptr2]], #16\n" // q4=C0C1C2C3
- "FCVTL2 v3.4s, v2.8h\n"
- "FCVTL v2.4s, v2.4h\n"
- "FCVTL2 v5.4s, v4.8h\n"
- "FCVTL v4.4s, v4.4h\n"
- "ZIP1 v16.4s, v0.4s, v4.4s\n" // q16=A0C0A1C1
- ASM_PREFETCH("[%[inptr0], #128]")
- "LDR q6, [%[inptr3]], #16\n" // q6=D0D1D2D3
- "FCVTL2 v7.4s, v6.8h\n"
- "FCVTL v6.4s, v6.4h\n"
- "ZIP1 v17.4s, v2.4s, v6.4s\n" // q17=B0D0B1D1
- "LDR q8, [%[inptr4]], #16\n"
- "LDR q10, [%[inptr5]], #16\n"
- "FCVTL2 v9.4s, v8.8h\n"
- "FCVTL v8.4s, v8.4h\n"
- ASM_PREFETCH("[%[inptr1], #128]")
- "LDR q12, [%[inptr6]], #16\n"
- "FCVTL2 v11.4s, v10.8h\n"
- "FCVTL v10.4s, v10.4h\n"
- "FCVTL2 v13.4s, v12.8h\n"
- "FCVTL v12.4s, v12.4h\n"
- "ZIP1 v18.4s, v8.4s, v12.4s\n"
- "LDR q14, [%[inptr7]], #16\n"
- "FCVTL2 v15.4s, v14.8h\n"
- "FCVTL v14.4s, v14.4h\n"
- "ZIP1 v19.4s, v10.4s, v14.4s\n"
-
- ASM_PREFETCH("[%[inptr2], #128]")
- "ZIP1 v20.4s, v16.4s, v17.4s\n" // q20=A0B0C0D0
- "ZIP1 v21.4s, v18.4s, v19.4s\n"
- "ZIP2 v22.4s, v16.4s, v17.4s\n"
- "ZIP2 v23.4s, v18.4s, v19.4s\n"
- ASM_PREFETCH("[%[inptr3], #128]")
-
- "ZIP2 v16.4s, v0.4s, v4.4s\n"
- "ZIP2 v17.4s, v2.4s, v6.4s\n"
- "STP q20, q21, [%[outptr]], #32\n" // Write back the first element of each source
-
- "ZIP2 v18.4s, v8.4s, v12.4s\n"
- ASM_PREFETCH("[%[inptr4], #128]")
- "ZIP2 v19.4s, v10.4s, v14.4s\n"
- "STP q22, q23, [%[outptr]], #32\n" // Write back the second element of each source
-
- "ZIP1 v20.4s, v16.4s, v17.4s\n"
- "ZIP1 v21.4s, v18.4s, v19.4s\n"
- ASM_PREFETCH("[%[inptr5], #128]")
- "ZIP2 v22.4s, v16.4s, v17.4s\n"
- "ZIP2 v23.4s, v18.4s, v19.4s\n"
-
- "ZIP1 v16.4s, v1.4s, v5.4s\n"
- "ZIP1 v17.4s, v3.4s, v7.4s\n"
- ASM_PREFETCH("[%[inptr6], #128]")
- "STP q20, q21, [%[outptr]], #32\n" // Third element
-
- "ZIP1 v18.4s, v9.4s, v13.4s\n"
- "ZIP1 v19.4s, v11.4s, v15.4s\n"
- "STP q22, q23, [%[outptr]], #32\n" // Fourth element
- ASM_PREFETCH("[%[inptr7], #128]")
-
- "ZIP1 v20.4s, v16.4s, v17.4s\n"
- "ZIP1 v21.4s, v18.4s, v19.4s\n"
- "ZIP2 v22.4s, v16.4s, v17.4s\n"
- "ZIP2 v23.4s, v18.4s, v19.4s\n"
-
- "ZIP2 v16.4s, v1.4s, v5.4s\n"
- "ZIP2 v17.4s, v3.4s, v7.4s\n"
- "STP q20, q21, [%[outptr]], #32\n" // Fifth element
-
- "ZIP2 v18.4s, v9.4s, v13.4s\n"
- "ZIP2 v19.4s, v11.4s, v15.4s\n"
- "STP q22, q23, [%[outptr]], #32\n" // Sixth element
-
- "ZIP1 v20.4s, v16.4s, v17.4s\n"
- "ZIP1 v21.4s, v18.4s, v19.4s\n"
- "STP q20, q21, [%[outptr]], #32\n" // Seventh element
-
- "ZIP2 v22.4s, v16.4s, v17.4s\n"
- "ZIP2 v23.4s, v18.4s, v19.4s\n"
- "STP q22, q23, [%[outptr]], #32\n" // Eighth element
- : [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3),
- [inptr4] "+r" (inptr4), [inptr5] "+r" (inptr5), [inptr6] "+r" (inptr6), [inptr7] "+r" (inptr7), [outptr] "+r" (outptr)
- :
- : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12",
- "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "memory"
- );
- }
-
- for (;x>0;x--) {
- *outptr++ = *inptr0++;
- *outptr++ = *inptr1++;
- *outptr++ = *inptr2++;
- *outptr++ = *inptr3++;
- *outptr++ = *inptr4++;
- *outptr++ = *inptr5++;
- *outptr++ = *inptr6++;
- *outptr++ = *inptr7++;
- }
- }
-}
-
-#endif // __aarch64__ && __ARM_FP16_ARGS
diff --git a/src/core/NEON/kernels/arm_gemm/transforms/a64_interleave_8way_s8_to_s16.hpp b/src/core/NEON/kernels/arm_gemm/transforms/a64_interleave_8way_s8_to_s16.hpp
deleted file mode 100644
index 37344a8..0000000
--- a/src/core/NEON/kernels/arm_gemm/transforms/a64_interleave_8way_s8_to_s16.hpp
+++ /dev/null
@@ -1,224 +0,0 @@
-/*
- * Copyright (c) 2017-2020 Arm Limited.
- *
- * SPDX-License-Identifier: MIT
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to
- * deal in the Software without restriction, including without limitation the
- * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
- * sell copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-#pragma once
-
-#if defined(__aarch64__) && defined(__ARM_FP16_ARGS)
-
-#include <arm_neon.h>
-#include <cstdint>
-
-#include "../asmlib.hpp"
-
-template<>
-template<>
-inline void TransformImpl<8, 1, false, 2, 1, false>::Transform(int16_t *out, const int8_t *in, int ldin, int y0, int ymax, int k0, int kmax) {
- int16_t *outptr = out;
- const int8_t *inptr = in;
- bool first = true;
-
- int8_t zerobuff[32] = { 0 }; // 16 for asm loop plus up to 15 for overflow loop
-
- for (int y=y0; y<ymax; y+=8) {
- const int8_t *inptr0 = inptr + y * ldin + k0;
- const int8_t *inptr1 = inptr0 + ldin;
- const int8_t *inptr2 = inptr1 + ldin;
- const int8_t *inptr3 = inptr2 + ldin;
- const int8_t *inptr4 = inptr3 + ldin;
- const int8_t *inptr5 = inptr4 + ldin;
- const int8_t *inptr6 = inptr5 + ldin;
- const int8_t *inptr7 = inptr6 + ldin;
-
- prefetch_2x(inptr0);
- prefetch_2x(inptr1);
- prefetch_2x(inptr2);
- prefetch_2x(inptr3);
- prefetch_2x(inptr4);
- prefetch_2x(inptr5);
- prefetch_2x(inptr6);
- prefetch_2x(inptr7);
-
- int x=(kmax-k0);
- for (;(x>15) || first;x-=16) {
- /* Cope with ragged cases by copying from a buffer of zeroes instead */
- /* 'first' forces this to always run at least once, needed if the total size is <=7. */
- if ((y + 7) >= ymax) {
- switch ((y + 7) - ymax) {
- case 6:
- inptr1 = zerobuff;
- // fall through
- case 5:
- inptr2 = zerobuff;
- // fall through
- case 4:
- inptr3 = zerobuff;
- // fall through
- case 3:
- inptr4 = zerobuff;
- // fall through
- case 2:
- inptr5 = zerobuff;
- // fall through
- case 1:
- inptr6 = zerobuff;
- // fall through
- case 0:
- inptr7 = zerobuff;
- break;
-
- default:
- UNREACHABLE("Impossible.");
- }
- }
-
- if (first) {
- if (x<=15) {
- break;
- }
-
- first = false;
- }
-
- __asm __volatile (
- // Load up 16 elements (1 source vector, 2 destination vectors) from each of 8 sources.
- "LDR q0, [%[inptr0]], #16\n"
- "LDR q2, [%[inptr1]], #16\n"
- "SSHLL2 v1.8h, v0.16b, #0\n"
- "SSHLL v0.8h, v0.8b, #0\n"
- "LDR q4, [%[inptr2]], #16\n" // q4=C0C1C2C3
- "SSHLL2 v3.8h, v2.16b, #0\n"
- "SSHLL v2.8h, v2.8b, #0\n"
- "SSHLL2 v5.8h, v4.16b, #0\n"
- "SSHLL v4.8h, v4.8b, #0\n"
- "ZIP1 v16.8h, v0.8h, v4.8h\n" // q16=A0C0A1C1
- ASM_PREFETCH("[%[inptr0], #128]")
- "LDR q6, [%[inptr3]], #16\n" // q6=D0D1D2D3
- "SSHLL2 v7.8h, v6.16b, #0\n"
- "SSHLL v6.8h, v6.8b, #0\n"
- "ZIP1 v17.8h, v2.8h, v6.8h\n" // q17=B0D0B1D1
- "LDR q8, [%[inptr4]], #16\n"
- "LDR q10, [%[inptr5]], #16\n"
- "SSHLL2 v9.8h, v8.16b, #0\n"
- "SSHLL v8.8h, v8.8b, #0\n"
- ASM_PREFETCH("[%[inptr1], #128]")
- "LDR q12, [%[inptr6]], #16\n"
- "SSHLL2 v11.8h, v10.16b, #0\n"
- "SSHLL v10.8h, v10.8b, #0\n"
- "SSHLL2 v13.8h, v12.16b, #0\n"
- "SSHLL v12.8h, v12.8b, #0\n"
- "ZIP1 v18.8h, v8.8h, v12.8h\n"
- "LDR q14, [%[inptr7]], #16\n"
- "SSHLL2 v15.8h, v14.16b, #0\n"
- "SSHLL v14.8h, v14.8b, #0\n"
- "ZIP1 v19.8h, v10.8h, v14.8h\n"
-
- ASM_PREFETCH("[%[inptr2], #128]")
- "ZIP1 v20.8h, v16.8h, v17.8h\n" // q20=A0B0C0D0A1B1C1D1
- "ZIP1 v21.8h, v18.8h, v19.8h\n" // q21=E0F0G0H0E1F1G1H1
- "ZIP2 v22.8h, v16.8h, v17.8h\n" // q22=A2B2C2D2A3B3C3D3
- "ZIP2 v23.8h, v18.8h, v19.8h\n" // q23=E2F2G2H1E3F3G3H3
- ASM_PREFETCH("[%[inptr3], #128]")
-
- "ZIP2 v16.8h, v0.8h, v4.8h\n"
- "ZIP2 v17.8h, v2.8h, v6.8h\n"
- "TRN1 v24.2d, v20.2d, v21.2d\n"
- "TRN2 v25.2d, v20.2d, v21.2d\n"
-
- "ZIP2 v18.8h, v8.8h, v12.8h\n"
- ASM_PREFETCH("[%[inptr4], #128]")
- "ZIP2 v19.8h, v10.8h, v14.8h\n"
- "STP q24, q25, [%[outptr]], #32\n" // Write back the first element of each source
- "TRN1 v24.2d, v22.2d, v23.2d\n"
- "TRN2 v25.2d, v22.2d, v23.2d\n"
-
- "ZIP1 v20.8h, v16.8h, v17.8h\n"
- "ZIP1 v21.8h, v18.8h, v19.8h\n"
- ASM_PREFETCH("[%[inptr5], #128]")
- "ZIP2 v22.8h, v16.8h, v17.8h\n"
- "ZIP2 v23.8h, v18.8h, v19.8h\n"
- "STP q24, q25, [%[outptr]], #32\n" // Write back the second element of each source
-
- "ZIP1 v16.8h, v1.8h, v5.8h\n"
- "ZIP1 v17.8h, v3.8h, v7.8h\n"
- ASM_PREFETCH("[%[inptr6], #128]")
- "TRN1 v24.2d, v20.2d, v21.2d\n"
- "TRN2 v25.2d, v20.2d, v21.2d\n"
-
- "ZIP1 v18.8h, v9.8h, v13.8h\n"
- "ZIP1 v19.8h, v11.8h, v15.8h\n"
- "STP q24, q25, [%[outptr]], #32\n" // Third element
- "TRN1 v24.2d, v22.2d, v23.2d\n"
- "TRN2 v25.2d, v22.2d, v23.2d\n"
- ASM_PREFETCH("[%[inptr7], #128]")
-
- "ZIP1 v20.8h, v16.8h, v17.8h\n"
- "ZIP1 v21.8h, v18.8h, v19.8h\n"
- "STP q24, q25, [%[outptr]], #32\n" // Fourth element
- "ZIP2 v22.8h, v16.8h, v17.8h\n"
- "ZIP2 v23.8h, v18.8h, v19.8h\n"
-
- "ZIP2 v16.8h, v1.8h, v5.8h\n"
- "ZIP2 v17.8h, v3.8h, v7.8h\n"
- "TRN1 v24.2d, v20.2d, v21.2d\n"
- "TRN2 v25.2d, v20.2d, v21.2d\n"
-
- "ZIP2 v18.8h, v9.8h, v13.8h\n"
- "ZIP2 v19.8h, v11.8h, v15.8h\n"
- "STP q24, q25, [%[outptr]], #32\n" // Fifth element
- "TRN1 v24.2d, v22.2d, v23.2d\n"
- "TRN2 v25.2d, v22.2d, v23.2d\n"
-
- "ZIP1 v20.8h, v16.8h, v17.8h\n"
- "ZIP1 v21.8h, v18.8h, v19.8h\n"
- "STP q24, q25, [%[outptr]], #32\n" // Sixth element
- "TRN1 v24.2d, v20.2d, v21.2d\n"
- "TRN2 v25.2d, v20.2d, v21.2d\n"
-
- "ZIP2 v22.8h, v16.8h, v17.8h\n"
- "ZIP2 v23.8h, v18.8h, v19.8h\n"
- "STP q24, q25, [%[outptr]], #32\n" // Seventh element
- "TRN1 v24.2d, v22.2d, v23.2d\n"
- "TRN2 v25.2d, v22.2d, v23.2d\n"
- "STP q24, q25, [%[outptr]], #32\n" // Eighth element
- : [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3),
- [inptr4] "+r" (inptr4), [inptr5] "+r" (inptr5), [inptr6] "+r" (inptr6), [inptr7] "+r" (inptr7), [outptr] "+r" (outptr)
- :
- : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12",
- "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "memory"
- );
- }
-
- for (;x>0;x--) {
- *outptr++ = *inptr0++;
- *outptr++ = *inptr1++;
- *outptr++ = *inptr2++;
- *outptr++ = *inptr3++;
- *outptr++ = *inptr4++;
- *outptr++ = *inptr5++;
- *outptr++ = *inptr6++;
- *outptr++ = *inptr7++;
- }
- }
-}
-
-#endif // __aarch64__ && __ARM_FP16_ARGS
diff --git a/src/core/NEON/kernels/arm_gemm/transforms/a64_interleave_8way_u8_to_u16.hpp b/src/core/NEON/kernels/arm_gemm/transforms/a64_interleave_8way_u8_to_u16.hpp
deleted file mode 100644
index a3a269c..0000000
--- a/src/core/NEON/kernels/arm_gemm/transforms/a64_interleave_8way_u8_to_u16.hpp
+++ /dev/null
@@ -1,224 +0,0 @@
-/*
- * Copyright (c) 2017-2020 Arm Limited.
- *
- * SPDX-License-Identifier: MIT
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to
- * deal in the Software without restriction, including without limitation the
- * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
- * sell copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-#pragma once
-
-#if defined(__aarch64__) && defined(__ARM_FP16_ARGS)
-
-#include <arm_neon.h>
-#include <cstdint>
-
-#include "../asmlib.hpp"
-
-template<>
-template<>
-inline void TransformImpl<8, 1, false, 2, 1, false>::Transform(uint16_t *out, const uint8_t *in, int ldin, int y0, int ymax, int k0, int kmax) {
- uint16_t *outptr = out;
- const uint8_t *inptr = in;
- bool first = true;
-
- uint8_t zerobuff[32] = { 0 }; // 16 for asm loop plus up to 15 for overflow loop
-
- for (int y=y0; y<ymax; y+=8) {
- const uint8_t *inptr0 = inptr + y * ldin + k0;
- const uint8_t *inptr1 = inptr0 + ldin;
- const uint8_t *inptr2 = inptr1 + ldin;
- const uint8_t *inptr3 = inptr2 + ldin;
- const uint8_t *inptr4 = inptr3 + ldin;
- const uint8_t *inptr5 = inptr4 + ldin;
- const uint8_t *inptr6 = inptr5 + ldin;
- const uint8_t *inptr7 = inptr6 + ldin;
-
- prefetch_2x(inptr0);
- prefetch_2x(inptr1);
- prefetch_2x(inptr2);
- prefetch_2x(inptr3);
- prefetch_2x(inptr4);
- prefetch_2x(inptr5);
- prefetch_2x(inptr6);
- prefetch_2x(inptr7);
-
- int x=(kmax-k0);
- for (;(x>15) || first;x-=16) {
- /* Cope with ragged cases by copying from a buffer of zeroes instead */
- /* 'first' forces this to always run at least once, needed if the total size is <=7. */
- if ((y + 7) >= ymax) {
- switch ((y + 7) - ymax) {
- case 6:
- inptr1 = zerobuff;
- // fall through
- case 5:
- inptr2 = zerobuff;
- // fall through
- case 4:
- inptr3 = zerobuff;
- // fall through
- case 3:
- inptr4 = zerobuff;
- // fall through
- case 2:
- inptr5 = zerobuff;
- // fall through
- case 1:
- inptr6 = zerobuff;
- // fall through
- case 0:
- inptr7 = zerobuff;
- break;
-
- default:
- UNREACHABLE("Impossible.");
- }
- }
-
- if (first) {
- if (x<=15) {
- break;
- }
-
- first = false;
- }
-
- __asm __volatile (
- // Load up 16 elements (1 source vector, 2 destination vectors) from each of 8 sources.
- "LDR q0, [%[inptr0]], #16\n"
- "LDR q2, [%[inptr1]], #16\n"
- "USHLL2 v1.8h, v0.16b, #0\n"
- "USHLL v0.8h, v0.8b, #0\n"
- "LDR q4, [%[inptr2]], #16\n" // q4=C0C1C2C3
- "USHLL2 v3.8h, v2.16b, #0\n"
- "USHLL v2.8h, v2.8b, #0\n"
- "USHLL2 v5.8h, v4.16b, #0\n"
- "USHLL v4.8h, v4.8b, #0\n"
- "ZIP1 v16.8h, v0.8h, v4.8h\n" // q16=A0C0A1C1
- ASM_PREFETCH("[%[inptr0], #128]")
- "LDR q6, [%[inptr3]], #16\n" // q6=D0D1D2D3
- "USHLL2 v7.8h, v6.16b, #0\n"
- "USHLL v6.8h, v6.8b, #0\n"
- "ZIP1 v17.8h, v2.8h, v6.8h\n" // q17=B0D0B1D1
- "LDR q8, [%[inptr4]], #16\n"
- "LDR q10, [%[inptr5]], #16\n"
- "USHLL2 v9.8h, v8.16b, #0\n"
- "USHLL v8.8h, v8.8b, #0\n"
- ASM_PREFETCH("[%[inptr1], #128]")
- "LDR q12, [%[inptr6]], #16\n"
- "USHLL2 v11.8h, v10.16b, #0\n"
- "USHLL v10.8h, v10.8b, #0\n"
- "USHLL2 v13.8h, v12.16b, #0\n"
- "USHLL v12.8h, v12.8b, #0\n"
- "ZIP1 v18.8h, v8.8h, v12.8h\n"
- "LDR q14, [%[inptr7]], #16\n"
- "USHLL2 v15.8h, v14.16b, #0\n"
- "USHLL v14.8h, v14.8b, #0\n"
- "ZIP1 v19.8h, v10.8h, v14.8h\n"
-
- ASM_PREFETCH("[%[inptr2], #128]")
- "ZIP1 v20.8h, v16.8h, v17.8h\n" // q20=A0B0C0D0A1B1C1D1
- "ZIP1 v21.8h, v18.8h, v19.8h\n" // q21=E0F0G0H0E1F1G1H1
- "ZIP2 v22.8h, v16.8h, v17.8h\n" // q22=A2B2C2D2A3B3C3D3
- "ZIP2 v23.8h, v18.8h, v19.8h\n" // q23=E2F2G2H1E3F3G3H3
- ASM_PREFETCH("[%[inptr3], #128]")
-
- "ZIP2 v16.8h, v0.8h, v4.8h\n"
- "ZIP2 v17.8h, v2.8h, v6.8h\n"
- "TRN1 v24.2d, v20.2d, v21.2d\n"
- "TRN2 v25.2d, v20.2d, v21.2d\n"
-
- "ZIP2 v18.8h, v8.8h, v12.8h\n"
- ASM_PREFETCH("[%[inptr4], #128]")
- "ZIP2 v19.8h, v10.8h, v14.8h\n"
- "STP q24, q25, [%[outptr]], #32\n" // Write back the first element of each source
- "TRN1 v24.2d, v22.2d, v23.2d\n"
- "TRN2 v25.2d, v22.2d, v23.2d\n"
-
- "ZIP1 v20.8h, v16.8h, v17.8h\n"
- "ZIP1 v21.8h, v18.8h, v19.8h\n"
- ASM_PREFETCH("[%[inptr5], #128]")
- "ZIP2 v22.8h, v16.8h, v17.8h\n"
- "ZIP2 v23.8h, v18.8h, v19.8h\n"
- "STP q24, q25, [%[outptr]], #32\n" // Write back the second element of each source
-
- "ZIP1 v16.8h, v1.8h, v5.8h\n"
- "ZIP1 v17.8h, v3.8h, v7.8h\n"
- ASM_PREFETCH("[%[inptr6], #128]")
- "TRN1 v24.2d, v20.2d, v21.2d\n"
- "TRN2 v25.2d, v20.2d, v21.2d\n"
-
- "ZIP1 v18.8h, v9.8h, v13.8h\n"
- "ZIP1 v19.8h, v11.8h, v15.8h\n"
- "STP q24, q25, [%[outptr]], #32\n" // Third element
- "TRN1 v24.2d, v22.2d, v23.2d\n"
- "TRN2 v25.2d, v22.2d, v23.2d\n"
- ASM_PREFETCH("[%[inptr7], #128]")
-
- "ZIP1 v20.8h, v16.8h, v17.8h\n"
- "ZIP1 v21.8h, v18.8h, v19.8h\n"
- "STP q24, q25, [%[outptr]], #32\n" // Fourth element
- "ZIP2 v22.8h, v16.8h, v17.8h\n"
- "ZIP2 v23.8h, v18.8h, v19.8h\n"
-
- "ZIP2 v16.8h, v1.8h, v5.8h\n"
- "ZIP2 v17.8h, v3.8h, v7.8h\n"
- "TRN1 v24.2d, v20.2d, v21.2d\n"
- "TRN2 v25.2d, v20.2d, v21.2d\n"
-
- "ZIP2 v18.8h, v9.8h, v13.8h\n"
- "ZIP2 v19.8h, v11.8h, v15.8h\n"
- "STP q24, q25, [%[outptr]], #32\n" // Fifth element
- "TRN1 v24.2d, v22.2d, v23.2d\n"
- "TRN2 v25.2d, v22.2d, v23.2d\n"
-
- "ZIP1 v20.8h, v16.8h, v17.8h\n"
- "ZIP1 v21.8h, v18.8h, v19.8h\n"
- "STP q24, q25, [%[outptr]], #32\n" // Sixth element
- "TRN1 v24.2d, v20.2d, v21.2d\n"
- "TRN2 v25.2d, v20.2d, v21.2d\n"
-
- "ZIP2 v22.8h, v16.8h, v17.8h\n"
- "ZIP2 v23.8h, v18.8h, v19.8h\n"
- "STP q24, q25, [%[outptr]], #32\n" // Seventh element
- "TRN1 v24.2d, v22.2d, v23.2d\n"
- "TRN2 v25.2d, v22.2d, v23.2d\n"
- "STP q24, q25, [%[outptr]], #32\n" // Eighth element
- : [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3),
- [inptr4] "+r" (inptr4), [inptr5] "+r" (inptr5), [inptr6] "+r" (inptr6), [inptr7] "+r" (inptr7), [outptr] "+r" (outptr)
- :
- : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12",
- "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "memory"
- );
- }
-
- for (;x>0;x--) {
- *outptr++ = *inptr0++;
- *outptr++ = *inptr1++;
- *outptr++ = *inptr2++;
- *outptr++ = *inptr3++;
- *outptr++ = *inptr4++;
- *outptr++ = *inptr5++;
- *outptr++ = *inptr6++;
- *outptr++ = *inptr7++;
- }
- }
-}
-
-#endif // __aarch64__ && __ARM_FP16_ARGS
diff --git a/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_12way_16bit.hpp b/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_12way_16bit.hpp
index 5ab5774..f6233ef 100644
--- a/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_12way_16bit.hpp
+++ b/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_12way_16bit.hpp
@@ -30,12 +30,12 @@
// Generic unblocked transposed 6x32-bit sized specialisation
template <>
template <typename T>
-inline void TransformImpl<6, 1, true, 4, 4, false>::Transform(
+inline void TransformImpl<6, 1, true, 4, 4, VLType::None>::Transform(
T* out, const T* const in, const int stride,
const int x0, const int xmax, const int k0, const int kmax
) {
// Redirect to a 12 x uint16_t specialisation
- TransformImpl<12, 1, true, 2, 2, false>::Transform(
+ TransformImpl<12, 1, true, 2, 2, VLType::None>::Transform(
reinterpret_cast<uint16_t *>(out),
reinterpret_cast<const uint16_t *>(in),
stride*2, x0*2, xmax*2, k0, kmax
@@ -45,7 +45,7 @@
// Generic 12x16-bit sized specialisation
template <>
template <typename T>
-inline void TransformImpl<12, 1, true, 2, 2, false>::Transform(
+inline void TransformImpl<12, 1, true, 2, 2, VLType::None>::Transform(
T* out, const T* const in, const int stride,
const int x0, const int xmax, const int k0, const int kmax
) {
@@ -135,7 +135,7 @@
template <>
template <>
-inline void TransformImpl<12, 1, true, 2, 2, false>::Transform(
+inline void TransformImpl<12, 1, true, 2, 2, VLType::None>::Transform(
uint16_t* out, const uint16_t* const in, const int stride,
const int x0, const int xmax, const int k0, const int kmax
) {
diff --git a/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_12way_half_to_float.hpp b/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_12way_half_to_float.hpp
index d7de9ff..c0f3e17 100644
--- a/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_12way_half_to_float.hpp
+++ b/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_12way_half_to_float.hpp
@@ -110,7 +110,7 @@
template <>
template <>
-inline void TransformImpl<12, 1, true, 4, 2, false>::Transform(
+inline void TransformImpl<12, 1, true, 4, 2, VLType::None>::Transform(
float* out, const __fp16* const in, const int stride,
const int x0, const int xmax, const int k0, const int kmax
) {
diff --git a/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_24way_16bit.hpp b/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_24way_16bit.hpp
index a137f93..bcbe2b8 100644
--- a/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_24way_16bit.hpp
+++ b/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_24way_16bit.hpp
@@ -30,12 +30,12 @@
// Generic unblocked transposed 12x32-bit sized specialisation
template <>
template <typename T>
-inline void TransformImpl<12, 1, true, 4, 4, false>::Transform(
+inline void TransformImpl<12, 1, true, 4, 4, VLType::None>::Transform(
T* out, const T* const in, const int stride,
const int x0, const int xmax, const int k0, const int kmax
) {
// Redirect to a 24 x uint16_t specialisation
- TransformImpl<24, 1, true, 2, 2, false>::Transform(
+ TransformImpl<24, 1, true, 2, 2, VLType::None>::Transform(
reinterpret_cast<uint16_t *>(out),
reinterpret_cast<const uint16_t * const>(in),
stride*2, x0*2, xmax*2, k0, kmax
@@ -45,7 +45,7 @@
// Generic 24x16-bit sized specialisation
template <>
template <typename T>
-inline void TransformImpl<24, 1, true, 2, 2, false>::Transform(
+inline void TransformImpl<24, 1, true, 2, 2, VLType::None>::Transform(
T* out, const T* const in, const int stride,
const int x0, const int xmax, const int k0, const int kmax
) {
@@ -120,7 +120,7 @@
template <>
template <>
-inline void TransformImpl<24, 1, true, 2, 2, false>::Transform(
+inline void TransformImpl<24, 1, true, 2, 2, VLType::None>::Transform(
uint16_t* out, const uint16_t* const in, const int stride,
const int x0, const int xmax, const int k0, const int kmax
) {
diff --git a/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_8way_32bit.hpp b/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_8way_32bit.hpp
index 974be48..df68740 100644
--- a/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_8way_32bit.hpp
+++ b/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_8way_32bit.hpp
@@ -30,12 +30,12 @@
// Generic unblocked transposed 8x32-bit sized specialisation
template <>
template <typename T>
-inline void TransformImpl<8, 1, true, 4, 4, false>::Transform(
+inline void TransformImpl<8, 1, true, 4, 4, VLType::None>::Transform(
T* out, const T* const in, const int stride,
const int x0, const int xmax, const int k0, const int kmax
) {
// Redirect to a 16 x uint16_t specialisation
- TransformImpl<16, 1, true, 2, 2, false>::Transform(
+ TransformImpl<16, 1, true, 2, 2, VLType::None>::Transform(
reinterpret_cast<uint16_t *>(out),
reinterpret_cast<const uint16_t *>(in),
stride*2, x0*2, xmax*2, k0, kmax
@@ -45,7 +45,7 @@
// Generic 16x16-bit sized specialisation
template <>
template <typename T>
-inline void TransformImpl<16, 1, true, 2, 2, false>::Transform(
+inline void TransformImpl<16, 1, true, 2, 2, VLType::None>::Transform(
T* out, const T* const in, const int stride,
const int x0, const int xmax, const int k0, const int kmax
) {
@@ -137,7 +137,7 @@
template <>
template <>
-inline void TransformImpl<16, 1, true, 2, 2, false>::Transform(
+inline void TransformImpl<16, 1, true, 2, 2, VLType::None>::Transform(
uint16_t* out, const uint16_t* const in, const int stride,
const int x0, const int xmax, const int k0, const int kmax
) {
diff --git a/src/core/NEON/kernels/arm_gemm/transforms/list.hpp b/src/core/NEON/kernels/arm_gemm/transforms/list.hpp
index b825e1c..e092c72 100644
--- a/src/core/NEON/kernels/arm_gemm/transforms/list.hpp
+++ b/src/core/NEON/kernels/arm_gemm/transforms/list.hpp
@@ -21,22 +21,8 @@
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
-#include "a32_interleave_6way_32bit.hpp"
#include "a32_transpose_interleave_8way_32bit.hpp"
-#include "a64_block16_interleave4_8bit.hpp"
-#include "a64_interleave_8way_16bit.hpp"
-#include "a64_interleave_8way_32bit.hpp"
-#include "a64_interleave_8way_block4_8bit.hpp"
-#include "a64_interleave_8way_half_to_float.hpp"
-#include "a64_interleave_8way_s8_to_s16.hpp"
-#include "a64_interleave_8way_u8_to_u16.hpp"
#include "a64_transpose_interleave_12way_16bit.hpp"
#include "a64_transpose_interleave_12way_half_to_float.hpp"
#include "a64_transpose_interleave_24way_16bit.hpp"
#include "a64_transpose_interleave_8way_32bit.hpp"
-#include "sve_interleave_8way_32bit.hpp"
-#include "sve_interleave_8way_block2_16bit.hpp"
-#include "sve_interleave_8way_block2_32bit.hpp"
-#include "sve_interleave_8way_block4_16bit.hpp"
-#include "sve_interleave_8way_block4_8bit.hpp"
-#include "sve_interleave_8way_block8_8bit.hpp"
diff --git a/src/core/NEON/kernels/arm_gemm/transforms/sve_interleave_8way_32bit.hpp b/src/core/NEON/kernels/arm_gemm/transforms/sve_interleave_8way_32bit.hpp
deleted file mode 100644
index 348d78e..0000000
--- a/src/core/NEON/kernels/arm_gemm/transforms/sve_interleave_8way_32bit.hpp
+++ /dev/null
@@ -1,596 +0,0 @@
-/*
- * Copyright (c) 2019 Arm Limited.
- *
- * SPDX-License-Identifier: MIT
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to
- * deal in the Software without restriction, including without limitation the
- * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
- * sell copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-#pragma once
-
-#ifdef __ARM_FEATURE_SVE
-
-template<>
-template<typename T>
-inline void TransformImpl<8, 1, false, 4, 4, false>::Transform(T *out, const T *in, int ldin, int y0, int ymax, int k0, int kmax)
-{
- uint32_t *master_outptr = reinterpret_cast<uint32_t *>(out);
- const uint32_t *inptr = reinterpret_cast<const uint32_t *>(in);
-
- for (int y=y0; y<ymax; y+=8)
- {
- const int height = ymax-y;
- const long inwidth = (kmax - k0);
- const long outwidth = inwidth * 8;
- long inpos = 0;
- long outpos = 0;
-
- uint32_t *outptr = master_outptr;
- master_outptr += outwidth;
-
- const uint32_t *inptr0 = inptr + y * ldin + k0;
- const uint32_t *inptr1 = inptr0 + ldin;
- const uint32_t *inptr2 = inptr1 + ldin;
- const uint32_t *inptr3 = inptr2 + ldin;
- const uint32_t *inptr4 = inptr3 + ldin;
- const uint32_t *inptr5 = inptr4 + ldin;
- const uint32_t *inptr6 = inptr5 + ldin;
- const uint32_t *inptr7 = inptr6 + ldin;
-
- switch(height)
- {
- case 1:
- __asm __volatile(
- "1:\n"
- "whilelt p0.s, %[inpos], %[inwidth]\n"
- "b.none 2f\n"
- "mov z4.s, #0\n"
- "ld1w z0.s, p0/z, [%[inptr0], %[inpos], LSL #2]\n"
- "incw %[inpos], all, mul #1\n"
- "whilelt p0.s, %[outpos], %[outwidth]\n"
- "incw %[outpos], all, mul #1\n"
- "zip1 z8.s, z0.s, z4.s\n"
- "zip2 z9.s, z0.s, z4.s\n"
- "whilelt p1.s, %[outpos], %[outwidth]\n"
- "zip1 z0.s, z8.s, z4.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip2 z1.s, z8.s, z4.s\n"
- "zip1 z2.s, z9.s, z4.s\n"
- "zip2 z3.s, z9.s, z4.s\n"
- "whilelt p2.s, %[outpos], %[outwidth]\n"
- "zip1 z8.s, z0.s, z4.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip2 z9.s, z0.s, z4.s\n"
- "zip1 z10.s, z1.s, z4.s\n"
- "st1w z8.s, p0, [%[outptr]]\n"
- "zip2 z11.s, z1.s, z4.s\n"
- "whilelt p3.s, %[outpos], %[outwidth]\n"
- "zip1 z12.s, z2.s, z4.s\n"
- "st1w z9.s, p1, [%[outptr], #1, MUL VL]\n"
- "zip2 z13.s, z2.s, z4.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip1 z14.s, z3.s, z4.s\n"
- "st1w z10.s, p2, [%[outptr], #2, MUL VL]\n"
- "zip2 z15.s, z3.s, z4.s\n"
- "whilelt p4.s, %[outpos], %[outwidth]\n"
- "st1w z11.s, p3, [%[outptr], #3, MUL VL]\n"
- "incw %[outpos], all, mul #1\n"
- "st1w z12.s, p4, [%[outptr], #4, MUL VL]\n"
- "whilelt p5.s, %[outpos], %[outwidth]\n"
- "incw %[outpos], all, mul #1\n"
- "st1w z13.s, p5, [%[outptr], #5, MUL VL]\n"
- "whilelt p6.s, %[outpos], %[outwidth]\n"
- "incw %[outpos], all, mul #1\n"
- "st1w z14.s, p6, [%[outptr], #6, MUL VL]\n"
- "whilelt p7.s, %[outpos], %[outwidth]\n"
- "incw %[outpos], all, mul #1\n"
- "st1w z15.s, p7, [%[outptr], #7, MUL VL]\n"
- "addvl %[outptr], %[outptr], #8\n"
- "b 1b\n"
- "2:\n"
- : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0)
- : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
- : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
- );
- break;
-
- case 2:
- __asm __volatile(
- "1:\n"
- "whilelt p0.s, %[inpos], %[inwidth]\n"
- "b.none 2f\n"
- "mov z4.s, #0\n"
- "mov z14.s, #0\n"
- "ld1w z0.s, p0/z, [%[inptr0], %[inpos], LSL #2]\n"
- "ld1w z1.s, p0/z, [%[inptr1], %[inpos], LSL #2]\n"
- "incw %[inpos], all, mul #1\n"
- "whilelt p0.s, %[outpos], %[outwidth]\n"
- "zip1 z8.s, z0.s, z4.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip2 z9.s, z0.s, z4.s\n"
- "zip1 z10.s, z1.s, z4.s\n"
- "zip2 z11.s, z1.s, z4.s\n"
- "whilelt p1.s, %[outpos], %[outwidth]\n"
- "zip1 z0.s, z8.s, z4.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip2 z1.s, z8.s, z4.s\n"
- "zip1 z2.s, z9.s, z4.s\n"
- "zip2 z3.s, z9.s, z4.s\n"
- "whilelt p2.s, %[outpos], %[outwidth]\n"
- "zip1 z4.s, z10.s, z14.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip2 z5.s, z10.s, z14.s\n"
- "zip1 z6.s, z11.s, z14.s\n"
- "zip2 z7.s, z11.s, z14.s\n"
- "whilelt p3.s, %[outpos], %[outwidth]\n"
- "zip1 z8.s, z0.s, z4.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip2 z9.s, z0.s, z4.s\n"
- "zip1 z10.s, z1.s, z5.s\n"
- "st1w z8.s, p0, [%[outptr]]\n"
- "zip2 z11.s, z1.s, z5.s\n"
- "whilelt p4.s, %[outpos], %[outwidth]\n"
- "zip1 z12.s, z2.s, z6.s\n"
- "st1w z9.s, p1, [%[outptr], #1, MUL VL]\n"
- "zip2 z13.s, z2.s, z6.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip1 z14.s, z3.s, z7.s\n"
- "st1w z10.s, p2, [%[outptr], #2, MUL VL]\n"
- "zip2 z15.s, z3.s, z7.s\n"
- "whilelt p5.s, %[outpos], %[outwidth]\n"
- "st1w z11.s, p3, [%[outptr], #3, MUL VL]\n"
- "incw %[outpos], all, mul #1\n"
- "st1w z12.s, p4, [%[outptr], #4, MUL VL]\n"
- "whilelt p6.s, %[outpos], %[outwidth]\n"
- "incw %[outpos], all, mul #1\n"
- "st1w z13.s, p5, [%[outptr], #5, MUL VL]\n"
- "whilelt p7.s, %[outpos], %[outwidth]\n"
- "incw %[outpos], all, mul #1\n"
- "st1w z14.s, p6, [%[outptr], #6, MUL VL]\n"
- "st1w z15.s, p7, [%[outptr], #7, MUL VL]\n"
- "addvl %[outptr], %[outptr], #8\n"
- "b 1b\n"
- "2:\n"
- : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1)
- : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
- : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
- );
- break;
-
- case 3:
- __asm __volatile(
- "1:\n"
- "whilelt p0.s, %[inpos], %[inwidth]\n"
- "b.none 2f\n"
- "mov z4.s, #0\n"
- "mov z14.s, #0\n"
- "ld1w z0.s, p0/z, [%[inptr0], %[inpos], LSL #2]\n"
- "ld1w z1.s, p0/z, [%[inptr1], %[inpos], LSL #2]\n"
- "ld1w z2.s, p0/z, [%[inptr2], %[inpos], LSL #2]\n"
- "incw %[inpos], all, mul #1\n"
- "zip1 z8.s, z0.s, z4.s\n"
- "whilelt p0.s, %[outpos], %[outwidth]\n"
- "zip2 z9.s, z0.s, z4.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip1 z10.s, z1.s, z4.s\n"
- "zip2 z11.s, z1.s, z4.s\n"
- "zip1 z12.s, z2.s, z4.s\n"
- "whilelt p1.s, %[outpos], %[outwidth]\n"
- "zip2 z13.s, z2.s, z4.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip1 z0.s, z8.s, z12.s\n"
- "zip2 z1.s, z8.s, z12.s\n"
- "zip1 z2.s, z9.s, z13.s\n"
- "whilelt p2.s, %[outpos], %[outwidth]\n"
- "zip2 z3.s, z9.s, z13.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip1 z4.s, z10.s, z14.s\n"
- "zip2 z5.s, z10.s, z14.s\n"
- "zip1 z6.s, z11.s, z14.s\n"
- "whilelt p3.s, %[outpos], %[outwidth]\n"
- "zip2 z7.s, z11.s, z14.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip1 z8.s, z0.s, z4.s\n"
- "zip2 z9.s, z0.s, z4.s\n"
- "zip1 z10.s, z1.s, z5.s\n"
- "whilelt p4.s, %[outpos], %[outwidth]\n"
- "zip2 z11.s, z1.s, z5.s\n"
- "st1w z8.s, p0, [%[outptr]]\n"
- "zip1 z12.s, z2.s, z6.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip2 z13.s, z2.s, z6.s\n"
- "st1w z9.s, p1, [%[outptr], #1, MUL VL]\n"
- "zip1 z14.s, z3.s, z7.s\n"
- "zip2 z15.s, z3.s, z7.s\n"
- "whilelt p5.s, %[outpos], %[outwidth]\n"
- "st1w z10.s, p2, [%[outptr], #2, MUL VL]\n"
- "incw %[outpos], all, mul #1\n"
- "st1w z11.s, p3, [%[outptr], #3, MUL VL]\n"
- "whilelt p6.s, %[outpos], %[outwidth]\n"
- "incw %[outpos], all, mul #1\n"
- "st1w z12.s, p4, [%[outptr], #4, MUL VL]\n"
- "whilelt p7.s, %[outpos], %[outwidth]\n"
- "incw %[outpos], all, mul #1\n"
- "st1w z13.s, p5, [%[outptr], #5, MUL VL]\n"
- "st1w z14.s, p6, [%[outptr], #6, MUL VL]\n"
- "st1w z15.s, p7, [%[outptr], #7, MUL VL]\n"
- "addvl %[outptr], %[outptr], #8\n"
- "b 1b\n"
- "2:\n"
- : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2)
- : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
- : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
- );
- break;
-
- case 4:
- __asm __volatile(
- "1:\n"
- "whilelt p0.s, %[inpos], %[inwidth]\n"
- "b.none 2f\n"
- "mov z4.s, #0\n"
- "ld1w z0.s, p0/z, [%[inptr0], %[inpos], LSL #2]\n"
- "ld1w z1.s, p0/z, [%[inptr1], %[inpos], LSL #2]\n"
- "ld1w z2.s, p0/z, [%[inptr2], %[inpos], LSL #2]\n"
- "ld1w z3.s, p0/z, [%[inptr3], %[inpos], LSL #2]\n"
- "incw %[inpos], all, mul #1\n"
- "zip1 z8.s, z0.s, z4.s\n"
- "whilelt p0.s, %[outpos], %[outwidth]\n"
- "zip2 z9.s, z0.s, z4.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip1 z10.s, z1.s, z4.s\n"
- "zip2 z11.s, z1.s, z4.s\n"
- "zip1 z12.s, z2.s, z4.s\n"
- "whilelt p1.s, %[outpos], %[outwidth]\n"
- "zip2 z13.s, z2.s, z4.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip1 z14.s, z3.s, z4.s\n"
- "zip2 z15.s, z3.s, z4.s\n"
- "zip1 z0.s, z8.s, z12.s\n"
- "whilelt p2.s, %[outpos], %[outwidth]\n"
- "zip2 z1.s, z8.s, z12.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip1 z2.s, z9.s, z13.s\n"
- "zip2 z3.s, z9.s, z13.s\n"
- "zip1 z4.s, z10.s, z14.s\n"
- "whilelt p3.s, %[outpos], %[outwidth]\n"
- "zip2 z5.s, z10.s, z14.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip1 z6.s, z11.s, z15.s\n"
- "zip2 z7.s, z11.s, z15.s\n"
- "zip1 z8.s, z0.s, z4.s\n"
- "whilelt p4.s, %[outpos], %[outwidth]\n"
- "zip2 z9.s, z0.s, z4.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip1 z10.s, z1.s, z5.s\n"
- "st1w z8.s, p0, [%[outptr]]\n"
- "zip2 z11.s, z1.s, z5.s\n"
- "zip1 z12.s, z2.s, z6.s\n"
- "whilelt p5.s, %[outpos], %[outwidth]\n"
- "zip2 z13.s, z2.s, z6.s\n"
- "st1w z9.s, p1, [%[outptr], #1, MUL VL]\n"
- "zip1 z14.s, z3.s, z7.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip2 z15.s, z3.s, z7.s\n"
- "st1w z10.s, p2, [%[outptr], #2, MUL VL]\n"
- "whilelt p6.s, %[outpos], %[outwidth]\n"
- "st1w z11.s, p3, [%[outptr], #3, MUL VL]\n"
- "incw %[outpos], all, mul #1\n"
- "st1w z12.s, p4, [%[outptr], #4, MUL VL]\n"
- "whilelt p7.s, %[outpos], %[outwidth]\n"
- "incw %[outpos], all, mul #1\n"
- "st1w z13.s, p5, [%[outptr], #5, MUL VL]\n"
- "st1w z14.s, p6, [%[outptr], #6, MUL VL]\n"
- "st1w z15.s, p7, [%[outptr], #7, MUL VL]\n"
- "addvl %[outptr], %[outptr], #8\n"
- "b 1b\n"
- "2:\n"
- : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3)
- : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
- : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
- );
- break;
-
- case 5:
- __asm __volatile(
- "1:\n"
- "whilelt p0.s, %[inpos], %[inwidth]\n"
- "b.none 2f\n"
- "mov z5.s, #0\n"
- "ld1w z0.s, p0/z, [%[inptr0], %[inpos], LSL #2]\n"
- "ld1w z1.s, p0/z, [%[inptr1], %[inpos], LSL #2]\n"
- "ld1w z2.s, p0/z, [%[inptr2], %[inpos], LSL #2]\n"
- "ld1w z3.s, p0/z, [%[inptr3], %[inpos], LSL #2]\n"
- "ld1w z4.s, p0/z, [%[inptr4], %[inpos], LSL #2]\n"
- "incw %[inpos], all, mul #1\n"
- "zip1 z10.s, z1.s, z5.s\n"
- "whilelt p0.s, %[outpos], %[outwidth]\n"
- "zip1 z8.s, z0.s, z4.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip2 z9.s, z0.s, z4.s\n"
- "zip2 z11.s, z1.s, z5.s\n"
- "zip1 z12.s, z2.s, z5.s\n"
- "whilelt p1.s, %[outpos], %[outwidth]\n"
- "zip2 z13.s, z2.s, z5.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip1 z14.s, z3.s, z5.s\n"
- "zip2 z15.s, z3.s, z5.s\n"
- "zip1 z0.s, z8.s, z12.s\n"
- "whilelt p2.s, %[outpos], %[outwidth]\n"
- "zip2 z1.s, z8.s, z12.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip1 z2.s, z9.s, z13.s\n"
- "zip2 z3.s, z9.s, z13.s\n"
- "zip1 z4.s, z10.s, z14.s\n"
- "whilelt p3.s, %[outpos], %[outwidth]\n"
- "zip2 z5.s, z10.s, z14.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip1 z6.s, z11.s, z15.s\n"
- "zip2 z7.s, z11.s, z15.s\n"
- "zip1 z8.s, z0.s, z4.s\n"
- "whilelt p4.s, %[outpos], %[outwidth]\n"
- "zip2 z9.s, z0.s, z4.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip1 z10.s, z1.s, z5.s\n"
- "st1w z8.s, p0, [%[outptr]]\n"
- "zip2 z11.s, z1.s, z5.s\n"
- "zip1 z12.s, z2.s, z6.s\n"
- "whilelt p5.s, %[outpos], %[outwidth]\n"
- "zip2 z13.s, z2.s, z6.s\n"
- "st1w z9.s, p1, [%[outptr], #1, MUL VL]\n"
- "zip1 z14.s, z3.s, z7.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip2 z15.s, z3.s, z7.s\n"
- "st1w z10.s, p2, [%[outptr], #2, MUL VL]\n"
- "whilelt p6.s, %[outpos], %[outwidth]\n"
- "st1w z11.s, p3, [%[outptr], #3, MUL VL]\n"
- "incw %[outpos], all, mul #1\n"
- "st1w z12.s, p4, [%[outptr], #4, MUL VL]\n"
- "whilelt p7.s, %[outpos], %[outwidth]\n"
- "incw %[outpos], all, mul #1\n"
- "st1w z13.s, p5, [%[outptr], #5, MUL VL]\n"
- "st1w z14.s, p6, [%[outptr], #6, MUL VL]\n"
- "st1w z15.s, p7, [%[outptr], #7, MUL VL]\n"
- "addvl %[outptr], %[outptr], #8\n"
- "b 1b\n"
- "2:\n"
- : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3), [inptr4] "+r" (inptr4)
- : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
- : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
- );
- break;
-
- case 6:
- __asm __volatile(
- "1:\n"
- "whilelt p0.s, %[inpos], %[inwidth]\n"
- "b.none 2f\n"
- "mov z6.s, #0\n"
- "ld1w z0.s, p0/z, [%[inptr0], %[inpos], LSL #2]\n"
- "ld1w z1.s, p0/z, [%[inptr1], %[inpos], LSL #2]\n"
- "ld1w z2.s, p0/z, [%[inptr2], %[inpos], LSL #2]\n"
- "ld1w z3.s, p0/z, [%[inptr3], %[inpos], LSL #2]\n"
- "ld1w z4.s, p0/z, [%[inptr4], %[inpos], LSL #2]\n"
- "ld1w z5.s, p0/z, [%[inptr5], %[inpos], LSL #2]\n"
- "incw %[inpos], all, mul #1\n"
- "zip1 z12.s, z2.s, z6.s\n"
- "whilelt p0.s, %[outpos], %[outwidth]\n"
- "zip1 z8.s, z0.s, z4.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip2 z9.s, z0.s, z4.s\n"
- "zip1 z10.s, z1.s, z5.s\n"
- "zip2 z11.s, z1.s, z5.s\n"
- "whilelt p1.s, %[outpos], %[outwidth]\n"
- "zip2 z13.s, z2.s, z6.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip1 z14.s, z3.s, z6.s\n"
- "zip2 z15.s, z3.s, z6.s\n"
- "zip1 z0.s, z8.s, z12.s\n"
- "whilelt p2.s, %[outpos], %[outwidth]\n"
- "zip2 z1.s, z8.s, z12.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip1 z2.s, z9.s, z13.s\n"
- "zip2 z3.s, z9.s, z13.s\n"
- "zip1 z4.s, z10.s, z14.s\n"
- "whilelt p3.s, %[outpos], %[outwidth]\n"
- "zip2 z5.s, z10.s, z14.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip1 z6.s, z11.s, z15.s\n"
- "zip2 z7.s, z11.s, z15.s\n"
- "zip1 z8.s, z0.s, z4.s\n"
- "whilelt p4.s, %[outpos], %[outwidth]\n"
- "zip2 z9.s, z0.s, z4.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip1 z10.s, z1.s, z5.s\n"
- "st1w z8.s, p0, [%[outptr]]\n"
- "zip2 z11.s, z1.s, z5.s\n"
- "zip1 z12.s, z2.s, z6.s\n"
- "whilelt p5.s, %[outpos], %[outwidth]\n"
- "zip2 z13.s, z2.s, z6.s\n"
- "st1w z9.s, p1, [%[outptr], #1, MUL VL]\n"
- "zip1 z14.s, z3.s, z7.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip2 z15.s, z3.s, z7.s\n"
- "st1w z10.s, p2, [%[outptr], #2, MUL VL]\n"
- "whilelt p6.s, %[outpos], %[outwidth]\n"
- "st1w z11.s, p3, [%[outptr], #3, MUL VL]\n"
- "incw %[outpos], all, mul #1\n"
- "st1w z12.s, p4, [%[outptr], #4, MUL VL]\n"
- "whilelt p7.s, %[outpos], %[outwidth]\n"
- "incw %[outpos], all, mul #1\n"
- "st1w z13.s, p5, [%[outptr], #5, MUL VL]\n"
- "st1w z14.s, p6, [%[outptr], #6, MUL VL]\n"
- "st1w z15.s, p7, [%[outptr], #7, MUL VL]\n"
- "addvl %[outptr], %[outptr], #8\n"
- "b 1b\n"
- "2:\n"
- : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3), [inptr4] "+r" (inptr4), [inptr5] "+r" (inptr5)
- : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
- : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
- );
- break;
-
- case 7:
- __asm __volatile(
- "1:\n"
- "whilelt p0.s, %[inpos], %[inwidth]\n"
- "b.none 2f\n"
- "mov z7.s, #0\n"
- "ld1w z0.s, p0/z, [%[inptr0], %[inpos], LSL #2]\n"
- "ld1w z1.s, p0/z, [%[inptr1], %[inpos], LSL #2]\n"
- "ld1w z2.s, p0/z, [%[inptr2], %[inpos], LSL #2]\n"
- "ld1w z3.s, p0/z, [%[inptr3], %[inpos], LSL #2]\n"
- "ld1w z4.s, p0/z, [%[inptr4], %[inpos], LSL #2]\n"
- "ld1w z5.s, p0/z, [%[inptr5], %[inpos], LSL #2]\n"
- "ld1w z6.s, p0/z, [%[inptr6], %[inpos], LSL #2]\n"
- "incw %[inpos], all, mul #1\n"
- "zip1 z8.s, z0.s, z4.s\n"
- "whilelt p0.s, %[outpos], %[outwidth]\n"
- "zip2 z9.s, z0.s, z4.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip1 z10.s, z1.s, z5.s\n"
- "zip2 z11.s, z1.s, z5.s\n"
- "zip1 z12.s, z2.s, z6.s\n"
- "whilelt p1.s, %[outpos], %[outwidth]\n"
- "zip2 z13.s, z2.s, z6.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip1 z14.s, z3.s, z7.s\n"
- "zip2 z15.s, z3.s, z7.s\n"
- "zip1 z0.s, z8.s, z12.s\n"
- "whilelt p2.s, %[outpos], %[outwidth]\n"
- "zip2 z1.s, z8.s, z12.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip1 z2.s, z9.s, z13.s\n"
- "zip2 z3.s, z9.s, z13.s\n"
- "zip1 z4.s, z10.s, z14.s\n"
- "whilelt p3.s, %[outpos], %[outwidth]\n"
- "zip2 z5.s, z10.s, z14.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip1 z6.s, z11.s, z15.s\n"
- "zip2 z7.s, z11.s, z15.s\n"
- "zip1 z8.s, z0.s, z4.s\n"
- "whilelt p4.s, %[outpos], %[outwidth]\n"
- "zip2 z9.s, z0.s, z4.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip1 z10.s, z1.s, z5.s\n"
- "st1w z8.s, p0, [%[outptr]]\n"
- "zip2 z11.s, z1.s, z5.s\n"
- "zip1 z12.s, z2.s, z6.s\n"
- "whilelt p5.s, %[outpos], %[outwidth]\n"
- "zip2 z13.s, z2.s, z6.s\n"
- "st1w z9.s, p1, [%[outptr], #1, MUL VL]\n"
- "zip1 z14.s, z3.s, z7.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip2 z15.s, z3.s, z7.s\n"
- "st1w z10.s, p2, [%[outptr], #2, MUL VL]\n"
- "whilelt p6.s, %[outpos], %[outwidth]\n"
- "st1w z11.s, p3, [%[outptr], #3, MUL VL]\n"
- "incw %[outpos], all, mul #1\n"
- "st1w z12.s, p4, [%[outptr], #4, MUL VL]\n"
- "whilelt p7.s, %[outpos], %[outwidth]\n"
- "incw %[outpos], all, mul #1\n"
- "st1w z13.s, p5, [%[outptr], #5, MUL VL]\n"
- "st1w z14.s, p6, [%[outptr], #6, MUL VL]\n"
- "st1w z15.s, p7, [%[outptr], #7, MUL VL]\n"
- "addvl %[outptr], %[outptr], #8\n"
- "b 1b\n"
- "2:\n"
- : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3), [inptr4] "+r" (inptr4), [inptr5] "+r" (inptr5), [inptr6] "+r" (inptr6)
- : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
- : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
- );
- break;
-
- default:
- case 8:
- __asm __volatile(
- "1:\n"
- "whilelt p0.s, %[inpos], %[inwidth]\n"
- "b.none 2f\n"
- "ld1w z0.s, p0/z, [%[inptr0], %[inpos], LSL #2]\n"
- "ld1w z1.s, p0/z, [%[inptr1], %[inpos], LSL #2]\n"
- "ld1w z2.s, p0/z, [%[inptr2], %[inpos], LSL #2]\n"
- "ld1w z3.s, p0/z, [%[inptr3], %[inpos], LSL #2]\n"
- "ld1w z4.s, p0/z, [%[inptr4], %[inpos], LSL #2]\n"
- "ld1w z5.s, p0/z, [%[inptr5], %[inpos], LSL #2]\n"
- "ld1w z6.s, p0/z, [%[inptr6], %[inpos], LSL #2]\n"
- "ld1w z7.s, p0/z, [%[inptr7], %[inpos], LSL #2]\n"
- "incw %[inpos], all, mul #1\n"
- "zip1 z8.s, z0.s, z4.s\n"
- "whilelt p0.s, %[outpos], %[outwidth]\n"
- "zip2 z9.s, z0.s, z4.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip1 z10.s, z1.s, z5.s\n"
- "zip2 z11.s, z1.s, z5.s\n"
- "zip1 z12.s, z2.s, z6.s\n"
- "whilelt p1.s, %[outpos], %[outwidth]\n"
- "zip2 z13.s, z2.s, z6.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip1 z14.s, z3.s, z7.s\n"
- "zip2 z15.s, z3.s, z7.s\n"
- "zip1 z0.s, z8.s, z12.s\n"
- "whilelt p2.s, %[outpos], %[outwidth]\n"
- "zip2 z1.s, z8.s, z12.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip1 z2.s, z9.s, z13.s\n"
- "zip2 z3.s, z9.s, z13.s\n"
- "zip1 z4.s, z10.s, z14.s\n"
- "whilelt p3.s, %[outpos], %[outwidth]\n"
- "zip2 z5.s, z10.s, z14.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip1 z6.s, z11.s, z15.s\n"
- "zip2 z7.s, z11.s, z15.s\n"
- "zip1 z8.s, z0.s, z4.s\n"
- "whilelt p4.s, %[outpos], %[outwidth]\n"
- "zip2 z9.s, z0.s, z4.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip1 z10.s, z1.s, z5.s\n"
- "st1w z8.s, p0, [%[outptr]]\n"
- "zip2 z11.s, z1.s, z5.s\n"
- "zip1 z12.s, z2.s, z6.s\n"
- "whilelt p5.s, %[outpos], %[outwidth]\n"
- "zip2 z13.s, z2.s, z6.s\n"
- "st1w z9.s, p1, [%[outptr], #1, MUL VL]\n"
- "zip1 z14.s, z3.s, z7.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip2 z15.s, z3.s, z7.s\n"
- "st1w z10.s, p2, [%[outptr], #2, MUL VL]\n"
- "whilelt p6.s, %[outpos], %[outwidth]\n"
- "st1w z11.s, p3, [%[outptr], #3, MUL VL]\n"
- "incw %[outpos], all, mul #1\n"
- "st1w z12.s, p4, [%[outptr], #4, MUL VL]\n"
- "whilelt p7.s, %[outpos], %[outwidth]\n"
- "incw %[outpos], all, mul #1\n"
- "st1w z13.s, p5, [%[outptr], #5, MUL VL]\n"
- "st1w z14.s, p6, [%[outptr], #6, MUL VL]\n"
- "st1w z15.s, p7, [%[outptr], #7, MUL VL]\n"
- "addvl %[outptr], %[outptr], #8\n"
- "b 1b\n"
- "2:\n"
- : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3), [inptr4] "+r" (inptr4), [inptr5] "+r" (inptr5), [inptr6] "+r" (inptr6), [inptr7] "+r" (inptr7)
- : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
- : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
- );
- break;
-
-
- }
- }
-}
-
-#endif // __ARM_FEATURE_SVE
diff --git a/src/core/NEON/kernels/arm_gemm/transforms/sve_interleave_8way_block2_16bit.hpp b/src/core/NEON/kernels/arm_gemm/transforms/sve_interleave_8way_block2_16bit.hpp
deleted file mode 100644
index 234433a..0000000
--- a/src/core/NEON/kernels/arm_gemm/transforms/sve_interleave_8way_block2_16bit.hpp
+++ /dev/null
@@ -1,596 +0,0 @@
-/*
- * Copyright (c) 2019 Arm Limited.
- *
- * SPDX-License-Identifier: MIT
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to
- * deal in the Software without restriction, including without limitation the
- * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
- * sell copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-#pragma once
-
-#ifdef __ARM_FEATURE_SVE
-
-template<>
-template<typename T>
-inline void TransformImpl<8, 2, false, 2, 2, false>::Transform(T *out, const T *in, int ldin, int y0, int ymax, int k0, int kmax)
-{
- uint16_t *master_outptr = reinterpret_cast<uint16_t *>(out);
- const uint16_t *inptr = reinterpret_cast<const uint16_t *>(in);
-
- for (int y=y0; y<ymax; y+=8)
- {
- const int height = ymax-y;
- const long inwidth = (kmax - k0);
- const long outwidth = ((inwidth + 1) / 2) * 16;
- long inpos = 0;
- long outpos = 0;
-
- uint16_t *outptr = master_outptr;
- master_outptr += outwidth;
-
- const uint16_t *inptr0 = inptr + y * ldin + k0;
- const uint16_t *inptr1 = inptr0 + ldin;
- const uint16_t *inptr2 = inptr1 + ldin;
- const uint16_t *inptr3 = inptr2 + ldin;
- const uint16_t *inptr4 = inptr3 + ldin;
- const uint16_t *inptr5 = inptr4 + ldin;
- const uint16_t *inptr6 = inptr5 + ldin;
- const uint16_t *inptr7 = inptr6 + ldin;
-
- switch(height)
- {
- case 1:
- __asm __volatile(
- "1:\n"
- "whilelt p0.h, %[inpos], %[inwidth]\n"
- "b.none 2f\n"
- "mov z4.h, #0\n"
- "ld1h z0.h, p0/z, [%[inptr0], %[inpos], LSL #1]\n"
- "inch %[inpos], all, mul #1\n"
- "whilelt p0.h, %[outpos], %[outwidth]\n"
- "inch %[outpos], all, mul #1\n"
- "zip1 z8.s, z0.s, z4.s\n"
- "zip2 z9.s, z0.s, z4.s\n"
- "whilelt p1.h, %[outpos], %[outwidth]\n"
- "zip1 z0.s, z8.s, z4.s\n"
- "inch %[outpos], all, mul #1\n"
- "zip2 z1.s, z8.s, z4.s\n"
- "zip1 z2.s, z9.s, z4.s\n"
- "zip2 z3.s, z9.s, z4.s\n"
- "whilelt p2.h, %[outpos], %[outwidth]\n"
- "zip1 z8.s, z0.s, z4.s\n"
- "inch %[outpos], all, mul #1\n"
- "zip2 z9.s, z0.s, z4.s\n"
- "zip1 z10.s, z1.s, z4.s\n"
- "st1h z8.h, p0, [%[outptr]]\n"
- "zip2 z11.s, z1.s, z4.s\n"
- "whilelt p3.h, %[outpos], %[outwidth]\n"
- "zip1 z12.s, z2.s, z4.s\n"
- "st1h z9.h, p1, [%[outptr], #1, MUL VL]\n"
- "zip2 z13.s, z2.s, z4.s\n"
- "inch %[outpos], all, mul #1\n"
- "zip1 z14.s, z3.s, z4.s\n"
- "st1h z10.h, p2, [%[outptr], #2, MUL VL]\n"
- "zip2 z15.s, z3.s, z4.s\n"
- "whilelt p4.h, %[outpos], %[outwidth]\n"
- "st1h z11.h, p3, [%[outptr], #3, MUL VL]\n"
- "inch %[outpos], all, mul #1\n"
- "st1h z12.h, p4, [%[outptr], #4, MUL VL]\n"
- "whilelt p5.h, %[outpos], %[outwidth]\n"
- "inch %[outpos], all, mul #1\n"
- "st1h z13.h, p5, [%[outptr], #5, MUL VL]\n"
- "whilelt p6.h, %[outpos], %[outwidth]\n"
- "inch %[outpos], all, mul #1\n"
- "st1h z14.h, p6, [%[outptr], #6, MUL VL]\n"
- "whilelt p7.h, %[outpos], %[outwidth]\n"
- "inch %[outpos], all, mul #1\n"
- "st1h z15.h, p7, [%[outptr], #7, MUL VL]\n"
- "addvl %[outptr], %[outptr], #8\n"
- "b 1b\n"
- "2:\n"
- : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0)
- : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
- : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
- );
- break;
-
- case 2:
- __asm __volatile(
- "1:\n"
- "whilelt p0.h, %[inpos], %[inwidth]\n"
- "b.none 2f\n"
- "mov z4.h, #0\n"
- "mov z14.h, #0\n"
- "ld1h z0.h, p0/z, [%[inptr0], %[inpos], LSL #1]\n"
- "ld1h z1.h, p0/z, [%[inptr1], %[inpos], LSL #1]\n"
- "inch %[inpos], all, mul #1\n"
- "whilelt p0.h, %[outpos], %[outwidth]\n"
- "zip1 z8.s, z0.s, z4.s\n"
- "inch %[outpos], all, mul #1\n"
- "zip2 z9.s, z0.s, z4.s\n"
- "zip1 z10.s, z1.s, z4.s\n"
- "zip2 z11.s, z1.s, z4.s\n"
- "whilelt p1.h, %[outpos], %[outwidth]\n"
- "zip1 z0.s, z8.s, z4.s\n"
- "inch %[outpos], all, mul #1\n"
- "zip2 z1.s, z8.s, z4.s\n"
- "zip1 z2.s, z9.s, z4.s\n"
- "zip2 z3.s, z9.s, z4.s\n"
- "whilelt p2.h, %[outpos], %[outwidth]\n"
- "zip1 z4.s, z10.s, z14.s\n"
- "inch %[outpos], all, mul #1\n"
- "zip2 z5.s, z10.s, z14.s\n"
- "zip1 z6.s, z11.s, z14.s\n"
- "zip2 z7.s, z11.s, z14.s\n"
- "whilelt p3.h, %[outpos], %[outwidth]\n"
- "zip1 z8.s, z0.s, z4.s\n"
- "inch %[outpos], all, mul #1\n"
- "zip2 z9.s, z0.s, z4.s\n"
- "zip1 z10.s, z1.s, z5.s\n"
- "st1h z8.h, p0, [%[outptr]]\n"
- "zip2 z11.s, z1.s, z5.s\n"
- "whilelt p4.h, %[outpos], %[outwidth]\n"
- "zip1 z12.s, z2.s, z6.s\n"
- "st1h z9.h, p1, [%[outptr], #1, MUL VL]\n"
- "zip2 z13.s, z2.s, z6.s\n"
- "inch %[outpos], all, mul #1\n"
- "zip1 z14.s, z3.s, z7.s\n"
- "st1h z10.h, p2, [%[outptr], #2, MUL VL]\n"
- "zip2 z15.s, z3.s, z7.s\n"
- "whilelt p5.h, %[outpos], %[outwidth]\n"
- "st1h z11.h, p3, [%[outptr], #3, MUL VL]\n"
- "inch %[outpos], all, mul #1\n"
- "st1h z12.h, p4, [%[outptr], #4, MUL VL]\n"
- "whilelt p6.h, %[outpos], %[outwidth]\n"
- "inch %[outpos], all, mul #1\n"
- "st1h z13.h, p5, [%[outptr], #5, MUL VL]\n"
- "whilelt p7.h, %[outpos], %[outwidth]\n"
- "inch %[outpos], all, mul #1\n"
- "st1h z14.h, p6, [%[outptr], #6, MUL VL]\n"
- "st1h z15.h, p7, [%[outptr], #7, MUL VL]\n"
- "addvl %[outptr], %[outptr], #8\n"
- "b 1b\n"
- "2:\n"
- : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1)
- : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
- : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
- );
- break;
-
- case 3:
- __asm __volatile(
- "1:\n"
- "whilelt p0.h, %[inpos], %[inwidth]\n"
- "b.none 2f\n"
- "mov z4.h, #0\n"
- "mov z14.h, #0\n"
- "ld1h z0.h, p0/z, [%[inptr0], %[inpos], LSL #1]\n"
- "ld1h z1.h, p0/z, [%[inptr1], %[inpos], LSL #1]\n"
- "ld1h z2.h, p0/z, [%[inptr2], %[inpos], LSL #1]\n"
- "inch %[inpos], all, mul #1\n"
- "zip1 z8.s, z0.s, z4.s\n"
- "whilelt p0.h, %[outpos], %[outwidth]\n"
- "zip2 z9.s, z0.s, z4.s\n"
- "inch %[outpos], all, mul #1\n"
- "zip1 z10.s, z1.s, z4.s\n"
- "zip2 z11.s, z1.s, z4.s\n"
- "zip1 z12.s, z2.s, z4.s\n"
- "whilelt p1.h, %[outpos], %[outwidth]\n"
- "zip2 z13.s, z2.s, z4.s\n"
- "inch %[outpos], all, mul #1\n"
- "zip1 z0.s, z8.s, z12.s\n"
- "zip2 z1.s, z8.s, z12.s\n"
- "zip1 z2.s, z9.s, z13.s\n"
- "whilelt p2.h, %[outpos], %[outwidth]\n"
- "zip2 z3.s, z9.s, z13.s\n"
- "inch %[outpos], all, mul #1\n"
- "zip1 z4.s, z10.s, z14.s\n"
- "zip2 z5.s, z10.s, z14.s\n"
- "zip1 z6.s, z11.s, z14.s\n"
- "whilelt p3.h, %[outpos], %[outwidth]\n"
- "zip2 z7.s, z11.s, z14.s\n"
- "inch %[outpos], all, mul #1\n"
- "zip1 z8.s, z0.s, z4.s\n"
- "zip2 z9.s, z0.s, z4.s\n"
- "zip1 z10.s, z1.s, z5.s\n"
- "whilelt p4.h, %[outpos], %[outwidth]\n"
- "zip2 z11.s, z1.s, z5.s\n"
- "st1h z8.h, p0, [%[outptr]]\n"
- "zip1 z12.s, z2.s, z6.s\n"
- "inch %[outpos], all, mul #1\n"
- "zip2 z13.s, z2.s, z6.s\n"
- "st1h z9.h, p1, [%[outptr], #1, MUL VL]\n"
- "zip1 z14.s, z3.s, z7.s\n"
- "zip2 z15.s, z3.s, z7.s\n"
- "whilelt p5.h, %[outpos], %[outwidth]\n"
- "st1h z10.h, p2, [%[outptr], #2, MUL VL]\n"
- "inch %[outpos], all, mul #1\n"
- "st1h z11.h, p3, [%[outptr], #3, MUL VL]\n"
- "whilelt p6.h, %[outpos], %[outwidth]\n"
- "inch %[outpos], all, mul #1\n"
- "st1h z12.h, p4, [%[outptr], #4, MUL VL]\n"
- "whilelt p7.h, %[outpos], %[outwidth]\n"
- "inch %[outpos], all, mul #1\n"
- "st1h z13.h, p5, [%[outptr], #5, MUL VL]\n"
- "st1h z14.h, p6, [%[outptr], #6, MUL VL]\n"
- "st1h z15.h, p7, [%[outptr], #7, MUL VL]\n"
- "addvl %[outptr], %[outptr], #8\n"
- "b 1b\n"
- "2:\n"
- : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2)
- : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
- : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
- );
- break;
-
- case 4:
- __asm __volatile(
- "1:\n"
- "whilelt p0.h, %[inpos], %[inwidth]\n"
- "b.none 2f\n"
- "mov z4.h, #0\n"
- "ld1h z0.h, p0/z, [%[inptr0], %[inpos], LSL #1]\n"
- "ld1h z1.h, p0/z, [%[inptr1], %[inpos], LSL #1]\n"
- "ld1h z2.h, p0/z, [%[inptr2], %[inpos], LSL #1]\n"
- "ld1h z3.h, p0/z, [%[inptr3], %[inpos], LSL #1]\n"
- "inch %[inpos], all, mul #1\n"
- "zip1 z8.s, z0.s, z4.s\n"
- "whilelt p0.h, %[outpos], %[outwidth]\n"
- "zip2 z9.s, z0.s, z4.s\n"
- "inch %[outpos], all, mul #1\n"
- "zip1 z10.s, z1.s, z4.s\n"
- "zip2 z11.s, z1.s, z4.s\n"
- "zip1 z12.s, z2.s, z4.s\n"
- "whilelt p1.h, %[outpos], %[outwidth]\n"
- "zip2 z13.s, z2.s, z4.s\n"
- "inch %[outpos], all, mul #1\n"
- "zip1 z14.s, z3.s, z4.s\n"
- "zip2 z15.s, z3.s, z4.s\n"
- "zip1 z0.s, z8.s, z12.s\n"
- "whilelt p2.h, %[outpos], %[outwidth]\n"
- "zip2 z1.s, z8.s, z12.s\n"
- "inch %[outpos], all, mul #1\n"
- "zip1 z2.s, z9.s, z13.s\n"
- "zip2 z3.s, z9.s, z13.s\n"
- "zip1 z4.s, z10.s, z14.s\n"
- "whilelt p3.h, %[outpos], %[outwidth]\n"
- "zip2 z5.s, z10.s, z14.s\n"
- "inch %[outpos], all, mul #1\n"
- "zip1 z6.s, z11.s, z15.s\n"
- "zip2 z7.s, z11.s, z15.s\n"
- "zip1 z8.s, z0.s, z4.s\n"
- "whilelt p4.h, %[outpos], %[outwidth]\n"
- "zip2 z9.s, z0.s, z4.s\n"
- "inch %[outpos], all, mul #1\n"
- "zip1 z10.s, z1.s, z5.s\n"
- "st1h z8.h, p0, [%[outptr]]\n"
- "zip2 z11.s, z1.s, z5.s\n"
- "zip1 z12.s, z2.s, z6.s\n"
- "whilelt p5.h, %[outpos], %[outwidth]\n"
- "zip2 z13.s, z2.s, z6.s\n"
- "st1h z9.h, p1, [%[outptr], #1, MUL VL]\n"
- "zip1 z14.s, z3.s, z7.s\n"
- "inch %[outpos], all, mul #1\n"
- "zip2 z15.s, z3.s, z7.s\n"
- "st1h z10.h, p2, [%[outptr], #2, MUL VL]\n"
- "whilelt p6.h, %[outpos], %[outwidth]\n"
- "st1h z11.h, p3, [%[outptr], #3, MUL VL]\n"
- "inch %[outpos], all, mul #1\n"
- "st1h z12.h, p4, [%[outptr], #4, MUL VL]\n"
- "whilelt p7.h, %[outpos], %[outwidth]\n"
- "inch %[outpos], all, mul #1\n"
- "st1h z13.h, p5, [%[outptr], #5, MUL VL]\n"
- "st1h z14.h, p6, [%[outptr], #6, MUL VL]\n"
- "st1h z15.h, p7, [%[outptr], #7, MUL VL]\n"
- "addvl %[outptr], %[outptr], #8\n"
- "b 1b\n"
- "2:\n"
- : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3)
- : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
- : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
- );
- break;
-
- case 5:
- __asm __volatile(
- "1:\n"
- "whilelt p0.h, %[inpos], %[inwidth]\n"
- "b.none 2f\n"
- "mov z5.h, #0\n"
- "ld1h z0.h, p0/z, [%[inptr0], %[inpos], LSL #1]\n"
- "ld1h z1.h, p0/z, [%[inptr1], %[inpos], LSL #1]\n"
- "ld1h z2.h, p0/z, [%[inptr2], %[inpos], LSL #1]\n"
- "ld1h z3.h, p0/z, [%[inptr3], %[inpos], LSL #1]\n"
- "ld1h z4.h, p0/z, [%[inptr4], %[inpos], LSL #1]\n"
- "inch %[inpos], all, mul #1\n"
- "zip1 z10.s, z1.s, z5.s\n"
- "whilelt p0.h, %[outpos], %[outwidth]\n"
- "zip1 z8.s, z0.s, z4.s\n"
- "inch %[outpos], all, mul #1\n"
- "zip2 z9.s, z0.s, z4.s\n"
- "zip2 z11.s, z1.s, z5.s\n"
- "zip1 z12.s, z2.s, z5.s\n"
- "whilelt p1.h, %[outpos], %[outwidth]\n"
- "zip2 z13.s, z2.s, z5.s\n"
- "inch %[outpos], all, mul #1\n"
- "zip1 z14.s, z3.s, z5.s\n"
- "zip2 z15.s, z3.s, z5.s\n"
- "zip1 z0.s, z8.s, z12.s\n"
- "whilelt p2.h, %[outpos], %[outwidth]\n"
- "zip2 z1.s, z8.s, z12.s\n"
- "inch %[outpos], all, mul #1\n"
- "zip1 z2.s, z9.s, z13.s\n"
- "zip2 z3.s, z9.s, z13.s\n"
- "zip1 z4.s, z10.s, z14.s\n"
- "whilelt p3.h, %[outpos], %[outwidth]\n"
- "zip2 z5.s, z10.s, z14.s\n"
- "inch %[outpos], all, mul #1\n"
- "zip1 z6.s, z11.s, z15.s\n"
- "zip2 z7.s, z11.s, z15.s\n"
- "zip1 z8.s, z0.s, z4.s\n"
- "whilelt p4.h, %[outpos], %[outwidth]\n"
- "zip2 z9.s, z0.s, z4.s\n"
- "inch %[outpos], all, mul #1\n"
- "zip1 z10.s, z1.s, z5.s\n"
- "st1h z8.h, p0, [%[outptr]]\n"
- "zip2 z11.s, z1.s, z5.s\n"
- "zip1 z12.s, z2.s, z6.s\n"
- "whilelt p5.h, %[outpos], %[outwidth]\n"
- "zip2 z13.s, z2.s, z6.s\n"
- "st1h z9.h, p1, [%[outptr], #1, MUL VL]\n"
- "zip1 z14.s, z3.s, z7.s\n"
- "inch %[outpos], all, mul #1\n"
- "zip2 z15.s, z3.s, z7.s\n"
- "st1h z10.h, p2, [%[outptr], #2, MUL VL]\n"
- "whilelt p6.h, %[outpos], %[outwidth]\n"
- "st1h z11.h, p3, [%[outptr], #3, MUL VL]\n"
- "inch %[outpos], all, mul #1\n"
- "st1h z12.h, p4, [%[outptr], #4, MUL VL]\n"
- "whilelt p7.h, %[outpos], %[outwidth]\n"
- "inch %[outpos], all, mul #1\n"
- "st1h z13.h, p5, [%[outptr], #5, MUL VL]\n"
- "st1h z14.h, p6, [%[outptr], #6, MUL VL]\n"
- "st1h z15.h, p7, [%[outptr], #7, MUL VL]\n"
- "addvl %[outptr], %[outptr], #8\n"
- "b 1b\n"
- "2:\n"
- : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3), [inptr4] "+r" (inptr4)
- : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
- : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
- );
- break;
-
- case 6:
- __asm __volatile(
- "1:\n"
- "whilelt p0.h, %[inpos], %[inwidth]\n"
- "b.none 2f\n"
- "mov z6.h, #0\n"
- "ld1h z0.h, p0/z, [%[inptr0], %[inpos], LSL #1]\n"
- "ld1h z1.h, p0/z, [%[inptr1], %[inpos], LSL #1]\n"
- "ld1h z2.h, p0/z, [%[inptr2], %[inpos], LSL #1]\n"
- "ld1h z3.h, p0/z, [%[inptr3], %[inpos], LSL #1]\n"
- "ld1h z4.h, p0/z, [%[inptr4], %[inpos], LSL #1]\n"
- "ld1h z5.h, p0/z, [%[inptr5], %[inpos], LSL #1]\n"
- "inch %[inpos], all, mul #1\n"
- "zip1 z12.s, z2.s, z6.s\n"
- "whilelt p0.h, %[outpos], %[outwidth]\n"
- "zip1 z8.s, z0.s, z4.s\n"
- "inch %[outpos], all, mul #1\n"
- "zip2 z9.s, z0.s, z4.s\n"
- "zip1 z10.s, z1.s, z5.s\n"
- "zip2 z11.s, z1.s, z5.s\n"
- "whilelt p1.h, %[outpos], %[outwidth]\n"
- "zip2 z13.s, z2.s, z6.s\n"
- "inch %[outpos], all, mul #1\n"
- "zip1 z14.s, z3.s, z6.s\n"
- "zip2 z15.s, z3.s, z6.s\n"
- "zip1 z0.s, z8.s, z12.s\n"
- "whilelt p2.h, %[outpos], %[outwidth]\n"
- "zip2 z1.s, z8.s, z12.s\n"
- "inch %[outpos], all, mul #1\n"
- "zip1 z2.s, z9.s, z13.s\n"
- "zip2 z3.s, z9.s, z13.s\n"
- "zip1 z4.s, z10.s, z14.s\n"
- "whilelt p3.h, %[outpos], %[outwidth]\n"
- "zip2 z5.s, z10.s, z14.s\n"
- "inch %[outpos], all, mul #1\n"
- "zip1 z6.s, z11.s, z15.s\n"
- "zip2 z7.s, z11.s, z15.s\n"
- "zip1 z8.s, z0.s, z4.s\n"
- "whilelt p4.h, %[outpos], %[outwidth]\n"
- "zip2 z9.s, z0.s, z4.s\n"
- "inch %[outpos], all, mul #1\n"
- "zip1 z10.s, z1.s, z5.s\n"
- "st1h z8.h, p0, [%[outptr]]\n"
- "zip2 z11.s, z1.s, z5.s\n"
- "zip1 z12.s, z2.s, z6.s\n"
- "whilelt p5.h, %[outpos], %[outwidth]\n"
- "zip2 z13.s, z2.s, z6.s\n"
- "st1h z9.h, p1, [%[outptr], #1, MUL VL]\n"
- "zip1 z14.s, z3.s, z7.s\n"
- "inch %[outpos], all, mul #1\n"
- "zip2 z15.s, z3.s, z7.s\n"
- "st1h z10.h, p2, [%[outptr], #2, MUL VL]\n"
- "whilelt p6.h, %[outpos], %[outwidth]\n"
- "st1h z11.h, p3, [%[outptr], #3, MUL VL]\n"
- "inch %[outpos], all, mul #1\n"
- "st1h z12.h, p4, [%[outptr], #4, MUL VL]\n"
- "whilelt p7.h, %[outpos], %[outwidth]\n"
- "inch %[outpos], all, mul #1\n"
- "st1h z13.h, p5, [%[outptr], #5, MUL VL]\n"
- "st1h z14.h, p6, [%[outptr], #6, MUL VL]\n"
- "st1h z15.h, p7, [%[outptr], #7, MUL VL]\n"
- "addvl %[outptr], %[outptr], #8\n"
- "b 1b\n"
- "2:\n"
- : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3), [inptr4] "+r" (inptr4), [inptr5] "+r" (inptr5)
- : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
- : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
- );
- break;
-
- case 7:
- __asm __volatile(
- "1:\n"
- "whilelt p0.h, %[inpos], %[inwidth]\n"
- "b.none 2f\n"
- "mov z7.h, #0\n"
- "ld1h z0.h, p0/z, [%[inptr0], %[inpos], LSL #1]\n"
- "ld1h z1.h, p0/z, [%[inptr1], %[inpos], LSL #1]\n"
- "ld1h z2.h, p0/z, [%[inptr2], %[inpos], LSL #1]\n"
- "ld1h z3.h, p0/z, [%[inptr3], %[inpos], LSL #1]\n"
- "ld1h z4.h, p0/z, [%[inptr4], %[inpos], LSL #1]\n"
- "ld1h z5.h, p0/z, [%[inptr5], %[inpos], LSL #1]\n"
- "ld1h z6.h, p0/z, [%[inptr6], %[inpos], LSL #1]\n"
- "inch %[inpos], all, mul #1\n"
- "zip1 z8.s, z0.s, z4.s\n"
- "whilelt p0.h, %[outpos], %[outwidth]\n"
- "zip2 z9.s, z0.s, z4.s\n"
- "inch %[outpos], all, mul #1\n"
- "zip1 z10.s, z1.s, z5.s\n"
- "zip2 z11.s, z1.s, z5.s\n"
- "zip1 z12.s, z2.s, z6.s\n"
- "whilelt p1.h, %[outpos], %[outwidth]\n"
- "zip2 z13.s, z2.s, z6.s\n"
- "inch %[outpos], all, mul #1\n"
- "zip1 z14.s, z3.s, z7.s\n"
- "zip2 z15.s, z3.s, z7.s\n"
- "zip1 z0.s, z8.s, z12.s\n"
- "whilelt p2.h, %[outpos], %[outwidth]\n"
- "zip2 z1.s, z8.s, z12.s\n"
- "inch %[outpos], all, mul #1\n"
- "zip1 z2.s, z9.s, z13.s\n"
- "zip2 z3.s, z9.s, z13.s\n"
- "zip1 z4.s, z10.s, z14.s\n"
- "whilelt p3.h, %[outpos], %[outwidth]\n"
- "zip2 z5.s, z10.s, z14.s\n"
- "inch %[outpos], all, mul #1\n"
- "zip1 z6.s, z11.s, z15.s\n"
- "zip2 z7.s, z11.s, z15.s\n"
- "zip1 z8.s, z0.s, z4.s\n"
- "whilelt p4.h, %[outpos], %[outwidth]\n"
- "zip2 z9.s, z0.s, z4.s\n"
- "inch %[outpos], all, mul #1\n"
- "zip1 z10.s, z1.s, z5.s\n"
- "st1h z8.h, p0, [%[outptr]]\n"
- "zip2 z11.s, z1.s, z5.s\n"
- "zip1 z12.s, z2.s, z6.s\n"
- "whilelt p5.h, %[outpos], %[outwidth]\n"
- "zip2 z13.s, z2.s, z6.s\n"
- "st1h z9.h, p1, [%[outptr], #1, MUL VL]\n"
- "zip1 z14.s, z3.s, z7.s\n"
- "inch %[outpos], all, mul #1\n"
- "zip2 z15.s, z3.s, z7.s\n"
- "st1h z10.h, p2, [%[outptr], #2, MUL VL]\n"
- "whilelt p6.h, %[outpos], %[outwidth]\n"
- "st1h z11.h, p3, [%[outptr], #3, MUL VL]\n"
- "inch %[outpos], all, mul #1\n"
- "st1h z12.h, p4, [%[outptr], #4, MUL VL]\n"
- "whilelt p7.h, %[outpos], %[outwidth]\n"
- "inch %[outpos], all, mul #1\n"
- "st1h z13.h, p5, [%[outptr], #5, MUL VL]\n"
- "st1h z14.h, p6, [%[outptr], #6, MUL VL]\n"
- "st1h z15.h, p7, [%[outptr], #7, MUL VL]\n"
- "addvl %[outptr], %[outptr], #8\n"
- "b 1b\n"
- "2:\n"
- : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3), [inptr4] "+r" (inptr4), [inptr5] "+r" (inptr5), [inptr6] "+r" (inptr6)
- : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
- : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
- );
- break;
-
- default:
- case 8:
- __asm __volatile(
- "1:\n"
- "whilelt p0.h, %[inpos], %[inwidth]\n"
- "b.none 2f\n"
- "ld1h z0.h, p0/z, [%[inptr0], %[inpos], LSL #1]\n"
- "ld1h z1.h, p0/z, [%[inptr1], %[inpos], LSL #1]\n"
- "ld1h z2.h, p0/z, [%[inptr2], %[inpos], LSL #1]\n"
- "ld1h z3.h, p0/z, [%[inptr3], %[inpos], LSL #1]\n"
- "ld1h z4.h, p0/z, [%[inptr4], %[inpos], LSL #1]\n"
- "ld1h z5.h, p0/z, [%[inptr5], %[inpos], LSL #1]\n"
- "ld1h z6.h, p0/z, [%[inptr6], %[inpos], LSL #1]\n"
- "ld1h z7.h, p0/z, [%[inptr7], %[inpos], LSL #1]\n"
- "inch %[inpos], all, mul #1\n"
- "zip1 z8.s, z0.s, z4.s\n"
- "whilelt p0.h, %[outpos], %[outwidth]\n"
- "zip2 z9.s, z0.s, z4.s\n"
- "inch %[outpos], all, mul #1\n"
- "zip1 z10.s, z1.s, z5.s\n"
- "zip2 z11.s, z1.s, z5.s\n"
- "zip1 z12.s, z2.s, z6.s\n"
- "whilelt p1.h, %[outpos], %[outwidth]\n"
- "zip2 z13.s, z2.s, z6.s\n"
- "inch %[outpos], all, mul #1\n"
- "zip1 z14.s, z3.s, z7.s\n"
- "zip2 z15.s, z3.s, z7.s\n"
- "zip1 z0.s, z8.s, z12.s\n"
- "whilelt p2.h, %[outpos], %[outwidth]\n"
- "zip2 z1.s, z8.s, z12.s\n"
- "inch %[outpos], all, mul #1\n"
- "zip1 z2.s, z9.s, z13.s\n"
- "zip2 z3.s, z9.s, z13.s\n"
- "zip1 z4.s, z10.s, z14.s\n"
- "whilelt p3.h, %[outpos], %[outwidth]\n"
- "zip2 z5.s, z10.s, z14.s\n"
- "inch %[outpos], all, mul #1\n"
- "zip1 z6.s, z11.s, z15.s\n"
- "zip2 z7.s, z11.s, z15.s\n"
- "zip1 z8.s, z0.s, z4.s\n"
- "whilelt p4.h, %[outpos], %[outwidth]\n"
- "zip2 z9.s, z0.s, z4.s\n"
- "inch %[outpos], all, mul #1\n"
- "zip1 z10.s, z1.s, z5.s\n"
- "st1h z8.h, p0, [%[outptr]]\n"
- "zip2 z11.s, z1.s, z5.s\n"
- "zip1 z12.s, z2.s, z6.s\n"
- "whilelt p5.h, %[outpos], %[outwidth]\n"
- "zip2 z13.s, z2.s, z6.s\n"
- "st1h z9.h, p1, [%[outptr], #1, MUL VL]\n"
- "zip1 z14.s, z3.s, z7.s\n"
- "inch %[outpos], all, mul #1\n"
- "zip2 z15.s, z3.s, z7.s\n"
- "st1h z10.h, p2, [%[outptr], #2, MUL VL]\n"
- "whilelt p6.h, %[outpos], %[outwidth]\n"
- "st1h z11.h, p3, [%[outptr], #3, MUL VL]\n"
- "inch %[outpos], all, mul #1\n"
- "st1h z12.h, p4, [%[outptr], #4, MUL VL]\n"
- "whilelt p7.h, %[outpos], %[outwidth]\n"
- "inch %[outpos], all, mul #1\n"
- "st1h z13.h, p5, [%[outptr], #5, MUL VL]\n"
- "st1h z14.h, p6, [%[outptr], #6, MUL VL]\n"
- "st1h z15.h, p7, [%[outptr], #7, MUL VL]\n"
- "addvl %[outptr], %[outptr], #8\n"
- "b 1b\n"
- "2:\n"
- : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3), [inptr4] "+r" (inptr4), [inptr5] "+r" (inptr5), [inptr6] "+r" (inptr6), [inptr7] "+r" (inptr7)
- : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
- : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
- );
- break;
-
-
- }
- }
-}
-
-#endif // __ARM_FEATURE_SVE
diff --git a/src/core/NEON/kernels/arm_gemm/transforms/sve_interleave_8way_block2_32bit.hpp b/src/core/NEON/kernels/arm_gemm/transforms/sve_interleave_8way_block2_32bit.hpp
deleted file mode 100644
index f21933b..0000000
--- a/src/core/NEON/kernels/arm_gemm/transforms/sve_interleave_8way_block2_32bit.hpp
+++ /dev/null
@@ -1,596 +0,0 @@
-/*
- * Copyright (c) 2019 Arm Limited.
- *
- * SPDX-License-Identifier: MIT
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to
- * deal in the Software without restriction, including without limitation the
- * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
- * sell copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-#pragma once
-
-#ifdef __ARM_FEATURE_SVE
-
-template<>
-template<typename T>
-inline void TransformImpl<8, 2, false, 4, 4, false>::Transform(T *out, const T *in, int ldin, int y0, int ymax, int k0, int kmax)
-{
- uint32_t *master_outptr = reinterpret_cast<uint32_t *>(out);
- const uint32_t *inptr = reinterpret_cast<const uint32_t *>(in);
-
- for (int y=y0; y<ymax; y+=8)
- {
- const int height = ymax-y;
- const long inwidth = (kmax - k0);
- const long outwidth = ((inwidth + 1) / 2) * 16;
- long inpos = 0;
- long outpos = 0;
-
- uint32_t *outptr = master_outptr;
- master_outptr += outwidth;
-
- const uint32_t *inptr0 = inptr + y * ldin + k0;
- const uint32_t *inptr1 = inptr0 + ldin;
- const uint32_t *inptr2 = inptr1 + ldin;
- const uint32_t *inptr3 = inptr2 + ldin;
- const uint32_t *inptr4 = inptr3 + ldin;
- const uint32_t *inptr5 = inptr4 + ldin;
- const uint32_t *inptr6 = inptr5 + ldin;
- const uint32_t *inptr7 = inptr6 + ldin;
-
- switch(height)
- {
- case 1:
- __asm __volatile(
- "1:\n"
- "whilelt p0.s, %[inpos], %[inwidth]\n"
- "b.none 2f\n"
- "mov z4.s, #0\n"
- "ld1w z0.s, p0/z, [%[inptr0], %[inpos], LSL #2]\n"
- "incw %[inpos], all, mul #1\n"
- "whilelt p0.s, %[outpos], %[outwidth]\n"
- "incw %[outpos], all, mul #1\n"
- "zip1 z8.d, z0.d, z4.d\n"
- "zip2 z9.d, z0.d, z4.d\n"
- "whilelt p1.s, %[outpos], %[outwidth]\n"
- "zip1 z0.d, z8.d, z4.d\n"
- "incw %[outpos], all, mul #1\n"
- "zip2 z1.d, z8.d, z4.d\n"
- "zip1 z2.d, z9.d, z4.d\n"
- "zip2 z3.d, z9.d, z4.d\n"
- "whilelt p2.s, %[outpos], %[outwidth]\n"
- "zip1 z8.d, z0.d, z4.d\n"
- "incw %[outpos], all, mul #1\n"
- "zip2 z9.d, z0.d, z4.d\n"
- "zip1 z10.d, z1.d, z4.d\n"
- "st1w z8.s, p0, [%[outptr]]\n"
- "zip2 z11.d, z1.d, z4.d\n"
- "whilelt p3.s, %[outpos], %[outwidth]\n"
- "zip1 z12.d, z2.d, z4.d\n"
- "st1w z9.s, p1, [%[outptr], #1, MUL VL]\n"
- "zip2 z13.d, z2.d, z4.d\n"
- "incw %[outpos], all, mul #1\n"
- "zip1 z14.d, z3.d, z4.d\n"
- "st1w z10.s, p2, [%[outptr], #2, MUL VL]\n"
- "zip2 z15.d, z3.d, z4.d\n"
- "whilelt p4.s, %[outpos], %[outwidth]\n"
- "st1w z11.s, p3, [%[outptr], #3, MUL VL]\n"
- "incw %[outpos], all, mul #1\n"
- "st1w z12.s, p4, [%[outptr], #4, MUL VL]\n"
- "whilelt p5.s, %[outpos], %[outwidth]\n"
- "incw %[outpos], all, mul #1\n"
- "st1w z13.s, p5, [%[outptr], #5, MUL VL]\n"
- "whilelt p6.s, %[outpos], %[outwidth]\n"
- "incw %[outpos], all, mul #1\n"
- "st1w z14.s, p6, [%[outptr], #6, MUL VL]\n"
- "whilelt p7.s, %[outpos], %[outwidth]\n"
- "incw %[outpos], all, mul #1\n"
- "st1w z15.s, p7, [%[outptr], #7, MUL VL]\n"
- "addvl %[outptr], %[outptr], #8\n"
- "b 1b\n"
- "2:\n"
- : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0)
- : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
- : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
- );
- break;
-
- case 2:
- __asm __volatile(
- "1:\n"
- "whilelt p0.s, %[inpos], %[inwidth]\n"
- "b.none 2f\n"
- "mov z4.s, #0\n"
- "mov z14.s, #0\n"
- "ld1w z0.s, p0/z, [%[inptr0], %[inpos], LSL #2]\n"
- "ld1w z1.s, p0/z, [%[inptr1], %[inpos], LSL #2]\n"
- "incw %[inpos], all, mul #1\n"
- "whilelt p0.s, %[outpos], %[outwidth]\n"
- "zip1 z8.d, z0.d, z4.d\n"
- "incw %[outpos], all, mul #1\n"
- "zip2 z9.d, z0.d, z4.d\n"
- "zip1 z10.d, z1.d, z4.d\n"
- "zip2 z11.d, z1.d, z4.d\n"
- "whilelt p1.s, %[outpos], %[outwidth]\n"
- "zip1 z0.d, z8.d, z4.d\n"
- "incw %[outpos], all, mul #1\n"
- "zip2 z1.d, z8.d, z4.d\n"
- "zip1 z2.d, z9.d, z4.d\n"
- "zip2 z3.d, z9.d, z4.d\n"
- "whilelt p2.s, %[outpos], %[outwidth]\n"
- "zip1 z4.d, z10.d, z14.d\n"
- "incw %[outpos], all, mul #1\n"
- "zip2 z5.d, z10.d, z14.d\n"
- "zip1 z6.d, z11.d, z14.d\n"
- "zip2 z7.d, z11.d, z14.d\n"
- "whilelt p3.s, %[outpos], %[outwidth]\n"
- "zip1 z8.d, z0.d, z4.d\n"
- "incw %[outpos], all, mul #1\n"
- "zip2 z9.d, z0.d, z4.d\n"
- "zip1 z10.d, z1.d, z5.d\n"
- "st1w z8.s, p0, [%[outptr]]\n"
- "zip2 z11.d, z1.d, z5.d\n"
- "whilelt p4.s, %[outpos], %[outwidth]\n"
- "zip1 z12.d, z2.d, z6.d\n"
- "st1w z9.s, p1, [%[outptr], #1, MUL VL]\n"
- "zip2 z13.d, z2.d, z6.d\n"
- "incw %[outpos], all, mul #1\n"
- "zip1 z14.d, z3.d, z7.d\n"
- "st1w z10.s, p2, [%[outptr], #2, MUL VL]\n"
- "zip2 z15.d, z3.d, z7.d\n"
- "whilelt p5.s, %[outpos], %[outwidth]\n"
- "st1w z11.s, p3, [%[outptr], #3, MUL VL]\n"
- "incw %[outpos], all, mul #1\n"
- "st1w z12.s, p4, [%[outptr], #4, MUL VL]\n"
- "whilelt p6.s, %[outpos], %[outwidth]\n"
- "incw %[outpos], all, mul #1\n"
- "st1w z13.s, p5, [%[outptr], #5, MUL VL]\n"
- "whilelt p7.s, %[outpos], %[outwidth]\n"
- "incw %[outpos], all, mul #1\n"
- "st1w z14.s, p6, [%[outptr], #6, MUL VL]\n"
- "st1w z15.s, p7, [%[outptr], #7, MUL VL]\n"
- "addvl %[outptr], %[outptr], #8\n"
- "b 1b\n"
- "2:\n"
- : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1)
- : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
- : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
- );
- break;
-
- case 3:
- __asm __volatile(
- "1:\n"
- "whilelt p0.s, %[inpos], %[inwidth]\n"
- "b.none 2f\n"
- "mov z4.s, #0\n"
- "mov z14.s, #0\n"
- "ld1w z0.s, p0/z, [%[inptr0], %[inpos], LSL #2]\n"
- "ld1w z1.s, p0/z, [%[inptr1], %[inpos], LSL #2]\n"
- "ld1w z2.s, p0/z, [%[inptr2], %[inpos], LSL #2]\n"
- "incw %[inpos], all, mul #1\n"
- "zip1 z8.d, z0.d, z4.d\n"
- "whilelt p0.s, %[outpos], %[outwidth]\n"
- "zip2 z9.d, z0.d, z4.d\n"
- "incw %[outpos], all, mul #1\n"
- "zip1 z10.d, z1.d, z4.d\n"
- "zip2 z11.d, z1.d, z4.d\n"
- "zip1 z12.d, z2.d, z4.d\n"
- "whilelt p1.s, %[outpos], %[outwidth]\n"
- "zip2 z13.d, z2.d, z4.d\n"
- "incw %[outpos], all, mul #1\n"
- "zip1 z0.d, z8.d, z12.d\n"
- "zip2 z1.d, z8.d, z12.d\n"
- "zip1 z2.d, z9.d, z13.d\n"
- "whilelt p2.s, %[outpos], %[outwidth]\n"
- "zip2 z3.d, z9.d, z13.d\n"
- "incw %[outpos], all, mul #1\n"
- "zip1 z4.d, z10.d, z14.d\n"
- "zip2 z5.d, z10.d, z14.d\n"
- "zip1 z6.d, z11.d, z14.d\n"
- "whilelt p3.s, %[outpos], %[outwidth]\n"
- "zip2 z7.d, z11.d, z14.d\n"
- "incw %[outpos], all, mul #1\n"
- "zip1 z8.d, z0.d, z4.d\n"
- "zip2 z9.d, z0.d, z4.d\n"
- "zip1 z10.d, z1.d, z5.d\n"
- "whilelt p4.s, %[outpos], %[outwidth]\n"
- "zip2 z11.d, z1.d, z5.d\n"
- "st1w z8.s, p0, [%[outptr]]\n"
- "zip1 z12.d, z2.d, z6.d\n"
- "incw %[outpos], all, mul #1\n"
- "zip2 z13.d, z2.d, z6.d\n"
- "st1w z9.s, p1, [%[outptr], #1, MUL VL]\n"
- "zip1 z14.d, z3.d, z7.d\n"
- "zip2 z15.d, z3.d, z7.d\n"
- "whilelt p5.s, %[outpos], %[outwidth]\n"
- "st1w z10.s, p2, [%[outptr], #2, MUL VL]\n"
- "incw %[outpos], all, mul #1\n"
- "st1w z11.s, p3, [%[outptr], #3, MUL VL]\n"
- "whilelt p6.s, %[outpos], %[outwidth]\n"
- "incw %[outpos], all, mul #1\n"
- "st1w z12.s, p4, [%[outptr], #4, MUL VL]\n"
- "whilelt p7.s, %[outpos], %[outwidth]\n"
- "incw %[outpos], all, mul #1\n"
- "st1w z13.s, p5, [%[outptr], #5, MUL VL]\n"
- "st1w z14.s, p6, [%[outptr], #6, MUL VL]\n"
- "st1w z15.s, p7, [%[outptr], #7, MUL VL]\n"
- "addvl %[outptr], %[outptr], #8\n"
- "b 1b\n"
- "2:\n"
- : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2)
- : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
- : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
- );
- break;
-
- case 4:
- __asm __volatile(
- "1:\n"
- "whilelt p0.s, %[inpos], %[inwidth]\n"
- "b.none 2f\n"
- "mov z4.s, #0\n"
- "ld1w z0.s, p0/z, [%[inptr0], %[inpos], LSL #2]\n"
- "ld1w z1.s, p0/z, [%[inptr1], %[inpos], LSL #2]\n"
- "ld1w z2.s, p0/z, [%[inptr2], %[inpos], LSL #2]\n"
- "ld1w z3.s, p0/z, [%[inptr3], %[inpos], LSL #2]\n"
- "incw %[inpos], all, mul #1\n"
- "zip1 z8.d, z0.d, z4.d\n"
- "whilelt p0.s, %[outpos], %[outwidth]\n"
- "zip2 z9.d, z0.d, z4.d\n"
- "incw %[outpos], all, mul #1\n"
- "zip1 z10.d, z1.d, z4.d\n"
- "zip2 z11.d, z1.d, z4.d\n"
- "zip1 z12.d, z2.d, z4.d\n"
- "whilelt p1.s, %[outpos], %[outwidth]\n"
- "zip2 z13.d, z2.d, z4.d\n"
- "incw %[outpos], all, mul #1\n"
- "zip1 z14.d, z3.d, z4.d\n"
- "zip2 z15.d, z3.d, z4.d\n"
- "zip1 z0.d, z8.d, z12.d\n"
- "whilelt p2.s, %[outpos], %[outwidth]\n"
- "zip2 z1.d, z8.d, z12.d\n"
- "incw %[outpos], all, mul #1\n"
- "zip1 z2.d, z9.d, z13.d\n"
- "zip2 z3.d, z9.d, z13.d\n"
- "zip1 z4.d, z10.d, z14.d\n"
- "whilelt p3.s, %[outpos], %[outwidth]\n"
- "zip2 z5.d, z10.d, z14.d\n"
- "incw %[outpos], all, mul #1\n"
- "zip1 z6.d, z11.d, z15.d\n"
- "zip2 z7.d, z11.d, z15.d\n"
- "zip1 z8.d, z0.d, z4.d\n"
- "whilelt p4.s, %[outpos], %[outwidth]\n"
- "zip2 z9.d, z0.d, z4.d\n"
- "incw %[outpos], all, mul #1\n"
- "zip1 z10.d, z1.d, z5.d\n"
- "st1w z8.s, p0, [%[outptr]]\n"
- "zip2 z11.d, z1.d, z5.d\n"
- "zip1 z12.d, z2.d, z6.d\n"
- "whilelt p5.s, %[outpos], %[outwidth]\n"
- "zip2 z13.d, z2.d, z6.d\n"
- "st1w z9.s, p1, [%[outptr], #1, MUL VL]\n"
- "zip1 z14.d, z3.d, z7.d\n"
- "incw %[outpos], all, mul #1\n"
- "zip2 z15.d, z3.d, z7.d\n"
- "st1w z10.s, p2, [%[outptr], #2, MUL VL]\n"
- "whilelt p6.s, %[outpos], %[outwidth]\n"
- "st1w z11.s, p3, [%[outptr], #3, MUL VL]\n"
- "incw %[outpos], all, mul #1\n"
- "st1w z12.s, p4, [%[outptr], #4, MUL VL]\n"
- "whilelt p7.s, %[outpos], %[outwidth]\n"
- "incw %[outpos], all, mul #1\n"
- "st1w z13.s, p5, [%[outptr], #5, MUL VL]\n"
- "st1w z14.s, p6, [%[outptr], #6, MUL VL]\n"
- "st1w z15.s, p7, [%[outptr], #7, MUL VL]\n"
- "addvl %[outptr], %[outptr], #8\n"
- "b 1b\n"
- "2:\n"
- : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3)
- : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
- : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
- );
- break;
-
- case 5:
- __asm __volatile(
- "1:\n"
- "whilelt p0.s, %[inpos], %[inwidth]\n"
- "b.none 2f\n"
- "mov z5.s, #0\n"
- "ld1w z0.s, p0/z, [%[inptr0], %[inpos], LSL #2]\n"
- "ld1w z1.s, p0/z, [%[inptr1], %[inpos], LSL #2]\n"
- "ld1w z2.s, p0/z, [%[inptr2], %[inpos], LSL #2]\n"
- "ld1w z3.s, p0/z, [%[inptr3], %[inpos], LSL #2]\n"
- "ld1w z4.s, p0/z, [%[inptr4], %[inpos], LSL #2]\n"
- "incw %[inpos], all, mul #1\n"
- "zip1 z10.d, z1.d, z5.d\n"
- "whilelt p0.s, %[outpos], %[outwidth]\n"
- "zip1 z8.d, z0.d, z4.d\n"
- "incw %[outpos], all, mul #1\n"
- "zip2 z9.d, z0.d, z4.d\n"
- "zip2 z11.d, z1.d, z5.d\n"
- "zip1 z12.d, z2.d, z5.d\n"
- "whilelt p1.s, %[outpos], %[outwidth]\n"
- "zip2 z13.d, z2.d, z5.d\n"
- "incw %[outpos], all, mul #1\n"
- "zip1 z14.d, z3.d, z5.d\n"
- "zip2 z15.d, z3.d, z5.d\n"
- "zip1 z0.d, z8.d, z12.d\n"
- "whilelt p2.s, %[outpos], %[outwidth]\n"
- "zip2 z1.d, z8.d, z12.d\n"
- "incw %[outpos], all, mul #1\n"
- "zip1 z2.d, z9.d, z13.d\n"
- "zip2 z3.d, z9.d, z13.d\n"
- "zip1 z4.d, z10.d, z14.d\n"
- "whilelt p3.s, %[outpos], %[outwidth]\n"
- "zip2 z5.d, z10.d, z14.d\n"
- "incw %[outpos], all, mul #1\n"
- "zip1 z6.d, z11.d, z15.d\n"
- "zip2 z7.d, z11.d, z15.d\n"
- "zip1 z8.d, z0.d, z4.d\n"
- "whilelt p4.s, %[outpos], %[outwidth]\n"
- "zip2 z9.d, z0.d, z4.d\n"
- "incw %[outpos], all, mul #1\n"
- "zip1 z10.d, z1.d, z5.d\n"
- "st1w z8.s, p0, [%[outptr]]\n"
- "zip2 z11.d, z1.d, z5.d\n"
- "zip1 z12.d, z2.d, z6.d\n"
- "whilelt p5.s, %[outpos], %[outwidth]\n"
- "zip2 z13.d, z2.d, z6.d\n"
- "st1w z9.s, p1, [%[outptr], #1, MUL VL]\n"
- "zip1 z14.d, z3.d, z7.d\n"
- "incw %[outpos], all, mul #1\n"
- "zip2 z15.d, z3.d, z7.d\n"
- "st1w z10.s, p2, [%[outptr], #2, MUL VL]\n"
- "whilelt p6.s, %[outpos], %[outwidth]\n"
- "st1w z11.s, p3, [%[outptr], #3, MUL VL]\n"
- "incw %[outpos], all, mul #1\n"
- "st1w z12.s, p4, [%[outptr], #4, MUL VL]\n"
- "whilelt p7.s, %[outpos], %[outwidth]\n"
- "incw %[outpos], all, mul #1\n"
- "st1w z13.s, p5, [%[outptr], #5, MUL VL]\n"
- "st1w z14.s, p6, [%[outptr], #6, MUL VL]\n"
- "st1w z15.s, p7, [%[outptr], #7, MUL VL]\n"
- "addvl %[outptr], %[outptr], #8\n"
- "b 1b\n"
- "2:\n"
- : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3), [inptr4] "+r" (inptr4)
- : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
- : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
- );
- break;
-
- case 6:
- __asm __volatile(
- "1:\n"
- "whilelt p0.s, %[inpos], %[inwidth]\n"
- "b.none 2f\n"
- "mov z6.s, #0\n"
- "ld1w z0.s, p0/z, [%[inptr0], %[inpos], LSL #2]\n"
- "ld1w z1.s, p0/z, [%[inptr1], %[inpos], LSL #2]\n"
- "ld1w z2.s, p0/z, [%[inptr2], %[inpos], LSL #2]\n"
- "ld1w z3.s, p0/z, [%[inptr3], %[inpos], LSL #2]\n"
- "ld1w z4.s, p0/z, [%[inptr4], %[inpos], LSL #2]\n"
- "ld1w z5.s, p0/z, [%[inptr5], %[inpos], LSL #2]\n"
- "incw %[inpos], all, mul #1\n"
- "zip1 z12.d, z2.d, z6.d\n"
- "whilelt p0.s, %[outpos], %[outwidth]\n"
- "zip1 z8.d, z0.d, z4.d\n"
- "incw %[outpos], all, mul #1\n"
- "zip2 z9.d, z0.d, z4.d\n"
- "zip1 z10.d, z1.d, z5.d\n"
- "zip2 z11.d, z1.d, z5.d\n"
- "whilelt p1.s, %[outpos], %[outwidth]\n"
- "zip2 z13.d, z2.d, z6.d\n"
- "incw %[outpos], all, mul #1\n"
- "zip1 z14.d, z3.d, z6.d\n"
- "zip2 z15.d, z3.d, z6.d\n"
- "zip1 z0.d, z8.d, z12.d\n"
- "whilelt p2.s, %[outpos], %[outwidth]\n"
- "zip2 z1.d, z8.d, z12.d\n"
- "incw %[outpos], all, mul #1\n"
- "zip1 z2.d, z9.d, z13.d\n"
- "zip2 z3.d, z9.d, z13.d\n"
- "zip1 z4.d, z10.d, z14.d\n"
- "whilelt p3.s, %[outpos], %[outwidth]\n"
- "zip2 z5.d, z10.d, z14.d\n"
- "incw %[outpos], all, mul #1\n"
- "zip1 z6.d, z11.d, z15.d\n"
- "zip2 z7.d, z11.d, z15.d\n"
- "zip1 z8.d, z0.d, z4.d\n"
- "whilelt p4.s, %[outpos], %[outwidth]\n"
- "zip2 z9.d, z0.d, z4.d\n"
- "incw %[outpos], all, mul #1\n"
- "zip1 z10.d, z1.d, z5.d\n"
- "st1w z8.s, p0, [%[outptr]]\n"
- "zip2 z11.d, z1.d, z5.d\n"
- "zip1 z12.d, z2.d, z6.d\n"
- "whilelt p5.s, %[outpos], %[outwidth]\n"
- "zip2 z13.d, z2.d, z6.d\n"
- "st1w z9.s, p1, [%[outptr], #1, MUL VL]\n"
- "zip1 z14.d, z3.d, z7.d\n"
- "incw %[outpos], all, mul #1\n"
- "zip2 z15.d, z3.d, z7.d\n"
- "st1w z10.s, p2, [%[outptr], #2, MUL VL]\n"
- "whilelt p6.s, %[outpos], %[outwidth]\n"
- "st1w z11.s, p3, [%[outptr], #3, MUL VL]\n"
- "incw %[outpos], all, mul #1\n"
- "st1w z12.s, p4, [%[outptr], #4, MUL VL]\n"
- "whilelt p7.s, %[outpos], %[outwidth]\n"
- "incw %[outpos], all, mul #1\n"
- "st1w z13.s, p5, [%[outptr], #5, MUL VL]\n"
- "st1w z14.s, p6, [%[outptr], #6, MUL VL]\n"
- "st1w z15.s, p7, [%[outptr], #7, MUL VL]\n"
- "addvl %[outptr], %[outptr], #8\n"
- "b 1b\n"
- "2:\n"
- : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3), [inptr4] "+r" (inptr4), [inptr5] "+r" (inptr5)
- : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
- : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
- );
- break;
-
- case 7:
- __asm __volatile(
- "1:\n"
- "whilelt p0.s, %[inpos], %[inwidth]\n"
- "b.none 2f\n"
- "mov z7.s, #0\n"
- "ld1w z0.s, p0/z, [%[inptr0], %[inpos], LSL #2]\n"
- "ld1w z1.s, p0/z, [%[inptr1], %[inpos], LSL #2]\n"
- "ld1w z2.s, p0/z, [%[inptr2], %[inpos], LSL #2]\n"
- "ld1w z3.s, p0/z, [%[inptr3], %[inpos], LSL #2]\n"
- "ld1w z4.s, p0/z, [%[inptr4], %[inpos], LSL #2]\n"
- "ld1w z5.s, p0/z, [%[inptr5], %[inpos], LSL #2]\n"
- "ld1w z6.s, p0/z, [%[inptr6], %[inpos], LSL #2]\n"
- "incw %[inpos], all, mul #1\n"
- "zip1 z8.d, z0.d, z4.d\n"
- "whilelt p0.s, %[outpos], %[outwidth]\n"
- "zip2 z9.d, z0.d, z4.d\n"
- "incw %[outpos], all, mul #1\n"
- "zip1 z10.d, z1.d, z5.d\n"
- "zip2 z11.d, z1.d, z5.d\n"
- "zip1 z12.d, z2.d, z6.d\n"
- "whilelt p1.s, %[outpos], %[outwidth]\n"
- "zip2 z13.d, z2.d, z6.d\n"
- "incw %[outpos], all, mul #1\n"
- "zip1 z14.d, z3.d, z7.d\n"
- "zip2 z15.d, z3.d, z7.d\n"
- "zip1 z0.d, z8.d, z12.d\n"
- "whilelt p2.s, %[outpos], %[outwidth]\n"
- "zip2 z1.d, z8.d, z12.d\n"
- "incw %[outpos], all, mul #1\n"
- "zip1 z2.d, z9.d, z13.d\n"
- "zip2 z3.d, z9.d, z13.d\n"
- "zip1 z4.d, z10.d, z14.d\n"
- "whilelt p3.s, %[outpos], %[outwidth]\n"
- "zip2 z5.d, z10.d, z14.d\n"
- "incw %[outpos], all, mul #1\n"
- "zip1 z6.d, z11.d, z15.d\n"
- "zip2 z7.d, z11.d, z15.d\n"
- "zip1 z8.d, z0.d, z4.d\n"
- "whilelt p4.s, %[outpos], %[outwidth]\n"
- "zip2 z9.d, z0.d, z4.d\n"
- "incw %[outpos], all, mul #1\n"
- "zip1 z10.d, z1.d, z5.d\n"
- "st1w z8.s, p0, [%[outptr]]\n"
- "zip2 z11.d, z1.d, z5.d\n"
- "zip1 z12.d, z2.d, z6.d\n"
- "whilelt p5.s, %[outpos], %[outwidth]\n"
- "zip2 z13.d, z2.d, z6.d\n"
- "st1w z9.s, p1, [%[outptr], #1, MUL VL]\n"
- "zip1 z14.d, z3.d, z7.d\n"
- "incw %[outpos], all, mul #1\n"
- "zip2 z15.d, z3.d, z7.d\n"
- "st1w z10.s, p2, [%[outptr], #2, MUL VL]\n"
- "whilelt p6.s, %[outpos], %[outwidth]\n"
- "st1w z11.s, p3, [%[outptr], #3, MUL VL]\n"
- "incw %[outpos], all, mul #1\n"
- "st1w z12.s, p4, [%[outptr], #4, MUL VL]\n"
- "whilelt p7.s, %[outpos], %[outwidth]\n"
- "incw %[outpos], all, mul #1\n"
- "st1w z13.s, p5, [%[outptr], #5, MUL VL]\n"
- "st1w z14.s, p6, [%[outptr], #6, MUL VL]\n"
- "st1w z15.s, p7, [%[outptr], #7, MUL VL]\n"
- "addvl %[outptr], %[outptr], #8\n"
- "b 1b\n"
- "2:\n"
- : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3), [inptr4] "+r" (inptr4), [inptr5] "+r" (inptr5), [inptr6] "+r" (inptr6)
- : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
- : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
- );
- break;
-
- default:
- case 8:
- __asm __volatile(
- "1:\n"
- "whilelt p0.s, %[inpos], %[inwidth]\n"
- "b.none 2f\n"
- "ld1w z0.s, p0/z, [%[inptr0], %[inpos], LSL #2]\n"
- "ld1w z1.s, p0/z, [%[inptr1], %[inpos], LSL #2]\n"
- "ld1w z2.s, p0/z, [%[inptr2], %[inpos], LSL #2]\n"
- "ld1w z3.s, p0/z, [%[inptr3], %[inpos], LSL #2]\n"
- "ld1w z4.s, p0/z, [%[inptr4], %[inpos], LSL #2]\n"
- "ld1w z5.s, p0/z, [%[inptr5], %[inpos], LSL #2]\n"
- "ld1w z6.s, p0/z, [%[inptr6], %[inpos], LSL #2]\n"
- "ld1w z7.s, p0/z, [%[inptr7], %[inpos], LSL #2]\n"
- "incw %[inpos], all, mul #1\n"
- "zip1 z8.d, z0.d, z4.d\n"
- "whilelt p0.s, %[outpos], %[outwidth]\n"
- "zip2 z9.d, z0.d, z4.d\n"
- "incw %[outpos], all, mul #1\n"
- "zip1 z10.d, z1.d, z5.d\n"
- "zip2 z11.d, z1.d, z5.d\n"
- "zip1 z12.d, z2.d, z6.d\n"
- "whilelt p1.s, %[outpos], %[outwidth]\n"
- "zip2 z13.d, z2.d, z6.d\n"
- "incw %[outpos], all, mul #1\n"
- "zip1 z14.d, z3.d, z7.d\n"
- "zip2 z15.d, z3.d, z7.d\n"
- "zip1 z0.d, z8.d, z12.d\n"
- "whilelt p2.s, %[outpos], %[outwidth]\n"
- "zip2 z1.d, z8.d, z12.d\n"
- "incw %[outpos], all, mul #1\n"
- "zip1 z2.d, z9.d, z13.d\n"
- "zip2 z3.d, z9.d, z13.d\n"
- "zip1 z4.d, z10.d, z14.d\n"
- "whilelt p3.s, %[outpos], %[outwidth]\n"
- "zip2 z5.d, z10.d, z14.d\n"
- "incw %[outpos], all, mul #1\n"
- "zip1 z6.d, z11.d, z15.d\n"
- "zip2 z7.d, z11.d, z15.d\n"
- "zip1 z8.d, z0.d, z4.d\n"
- "whilelt p4.s, %[outpos], %[outwidth]\n"
- "zip2 z9.d, z0.d, z4.d\n"
- "incw %[outpos], all, mul #1\n"
- "zip1 z10.d, z1.d, z5.d\n"
- "st1w z8.s, p0, [%[outptr]]\n"
- "zip2 z11.d, z1.d, z5.d\n"
- "zip1 z12.d, z2.d, z6.d\n"
- "whilelt p5.s, %[outpos], %[outwidth]\n"
- "zip2 z13.d, z2.d, z6.d\n"
- "st1w z9.s, p1, [%[outptr], #1, MUL VL]\n"
- "zip1 z14.d, z3.d, z7.d\n"
- "incw %[outpos], all, mul #1\n"
- "zip2 z15.d, z3.d, z7.d\n"
- "st1w z10.s, p2, [%[outptr], #2, MUL VL]\n"
- "whilelt p6.s, %[outpos], %[outwidth]\n"
- "st1w z11.s, p3, [%[outptr], #3, MUL VL]\n"
- "incw %[outpos], all, mul #1\n"
- "st1w z12.s, p4, [%[outptr], #4, MUL VL]\n"
- "whilelt p7.s, %[outpos], %[outwidth]\n"
- "incw %[outpos], all, mul #1\n"
- "st1w z13.s, p5, [%[outptr], #5, MUL VL]\n"
- "st1w z14.s, p6, [%[outptr], #6, MUL VL]\n"
- "st1w z15.s, p7, [%[outptr], #7, MUL VL]\n"
- "addvl %[outptr], %[outptr], #8\n"
- "b 1b\n"
- "2:\n"
- : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3), [inptr4] "+r" (inptr4), [inptr5] "+r" (inptr5), [inptr6] "+r" (inptr6), [inptr7] "+r" (inptr7)
- : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
- : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
- );
- break;
-
-
- }
- }
-}
-
-#endif // __ARM_FEATURE_SVE
diff --git a/src/core/NEON/kernels/arm_gemm/transforms/sve_interleave_8way_block4_16bit.hpp b/src/core/NEON/kernels/arm_gemm/transforms/sve_interleave_8way_block4_16bit.hpp
deleted file mode 100644
index 26e1051..0000000
--- a/src/core/NEON/kernels/arm_gemm/transforms/sve_interleave_8way_block4_16bit.hpp
+++ /dev/null
@@ -1,596 +0,0 @@
-/*
- * Copyright (c) 2019 Arm Limited.
- *
- * SPDX-License-Identifier: MIT
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to
- * deal in the Software without restriction, including without limitation the
- * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
- * sell copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-#pragma once
-
-#ifdef __ARM_FEATURE_SVE
-
-template<>
-template<typename T>
-inline void TransformImpl<8, 4, false, 2, 2, false>::Transform(T *out, const T *in, int ldin, int y0, int ymax, int k0, int kmax)
-{
- uint16_t *master_outptr = reinterpret_cast<uint16_t *>(out);
- const uint16_t *inptr = reinterpret_cast<const uint16_t *>(in);
-
- for (int y=y0; y<ymax; y+=8)
- {
- const int height = ymax-y;
- const long inwidth = (kmax - k0);
- const long outwidth = ((inwidth + 3) / 4) * 32;
- long inpos = 0;
- long outpos = 0;
-
- uint16_t *outptr = master_outptr;
- master_outptr += outwidth;
-
- const uint16_t *inptr0 = inptr + y * ldin + k0;
- const uint16_t *inptr1 = inptr0 + ldin;
- const uint16_t *inptr2 = inptr1 + ldin;
- const uint16_t *inptr3 = inptr2 + ldin;
- const uint16_t *inptr4 = inptr3 + ldin;
- const uint16_t *inptr5 = inptr4 + ldin;
- const uint16_t *inptr6 = inptr5 + ldin;
- const uint16_t *inptr7 = inptr6 + ldin;
-
- switch(height)
- {
- case 1:
- __asm __volatile(
- "1:\n"
- "whilelt p0.h, %[inpos], %[inwidth]\n"
- "b.none 2f\n"
- "mov z4.h, #0\n"
- "ld1h z0.h, p0/z, [%[inptr0], %[inpos], LSL #1]\n"
- "inch %[inpos], all, mul #1\n"
- "whilelt p0.h, %[outpos], %[outwidth]\n"
- "inch %[outpos], all, mul #1\n"
- "zip1 z8.d, z0.d, z4.d\n"
- "zip2 z9.d, z0.d, z4.d\n"
- "whilelt p1.h, %[outpos], %[outwidth]\n"
- "zip1 z0.d, z8.d, z4.d\n"
- "inch %[outpos], all, mul #1\n"
- "zip2 z1.d, z8.d, z4.d\n"
- "zip1 z2.d, z9.d, z4.d\n"
- "zip2 z3.d, z9.d, z4.d\n"
- "whilelt p2.h, %[outpos], %[outwidth]\n"
- "zip1 z8.d, z0.d, z4.d\n"
- "inch %[outpos], all, mul #1\n"
- "zip2 z9.d, z0.d, z4.d\n"
- "zip1 z10.d, z1.d, z4.d\n"
- "st1h z8.h, p0, [%[outptr]]\n"
- "zip2 z11.d, z1.d, z4.d\n"
- "whilelt p3.h, %[outpos], %[outwidth]\n"
- "zip1 z12.d, z2.d, z4.d\n"
- "st1h z9.h, p1, [%[outptr], #1, MUL VL]\n"
- "zip2 z13.d, z2.d, z4.d\n"
- "inch %[outpos], all, mul #1\n"
- "zip1 z14.d, z3.d, z4.d\n"
- "st1h z10.h, p2, [%[outptr], #2, MUL VL]\n"
- "zip2 z15.d, z3.d, z4.d\n"
- "whilelt p4.h, %[outpos], %[outwidth]\n"
- "st1h z11.h, p3, [%[outptr], #3, MUL VL]\n"
- "inch %[outpos], all, mul #1\n"
- "st1h z12.h, p4, [%[outptr], #4, MUL VL]\n"
- "whilelt p5.h, %[outpos], %[outwidth]\n"
- "inch %[outpos], all, mul #1\n"
- "st1h z13.h, p5, [%[outptr], #5, MUL VL]\n"
- "whilelt p6.h, %[outpos], %[outwidth]\n"
- "inch %[outpos], all, mul #1\n"
- "st1h z14.h, p6, [%[outptr], #6, MUL VL]\n"
- "whilelt p7.h, %[outpos], %[outwidth]\n"
- "inch %[outpos], all, mul #1\n"
- "st1h z15.h, p7, [%[outptr], #7, MUL VL]\n"
- "addvl %[outptr], %[outptr], #8\n"
- "b 1b\n"
- "2:\n"
- : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0)
- : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
- : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
- );
- break;
-
- case 2:
- __asm __volatile(
- "1:\n"
- "whilelt p0.h, %[inpos], %[inwidth]\n"
- "b.none 2f\n"
- "mov z4.h, #0\n"
- "mov z14.h, #0\n"
- "ld1h z0.h, p0/z, [%[inptr0], %[inpos], LSL #1]\n"
- "ld1h z1.h, p0/z, [%[inptr1], %[inpos], LSL #1]\n"
- "inch %[inpos], all, mul #1\n"
- "whilelt p0.h, %[outpos], %[outwidth]\n"
- "zip1 z8.d, z0.d, z4.d\n"
- "inch %[outpos], all, mul #1\n"
- "zip2 z9.d, z0.d, z4.d\n"
- "zip1 z10.d, z1.d, z4.d\n"
- "zip2 z11.d, z1.d, z4.d\n"
- "whilelt p1.h, %[outpos], %[outwidth]\n"
- "zip1 z0.d, z8.d, z4.d\n"
- "inch %[outpos], all, mul #1\n"
- "zip2 z1.d, z8.d, z4.d\n"
- "zip1 z2.d, z9.d, z4.d\n"
- "zip2 z3.d, z9.d, z4.d\n"
- "whilelt p2.h, %[outpos], %[outwidth]\n"
- "zip1 z4.d, z10.d, z14.d\n"
- "inch %[outpos], all, mul #1\n"
- "zip2 z5.d, z10.d, z14.d\n"
- "zip1 z6.d, z11.d, z14.d\n"
- "zip2 z7.d, z11.d, z14.d\n"
- "whilelt p3.h, %[outpos], %[outwidth]\n"
- "zip1 z8.d, z0.d, z4.d\n"
- "inch %[outpos], all, mul #1\n"
- "zip2 z9.d, z0.d, z4.d\n"
- "zip1 z10.d, z1.d, z5.d\n"
- "st1h z8.h, p0, [%[outptr]]\n"
- "zip2 z11.d, z1.d, z5.d\n"
- "whilelt p4.h, %[outpos], %[outwidth]\n"
- "zip1 z12.d, z2.d, z6.d\n"
- "st1h z9.h, p1, [%[outptr], #1, MUL VL]\n"
- "zip2 z13.d, z2.d, z6.d\n"
- "inch %[outpos], all, mul #1\n"
- "zip1 z14.d, z3.d, z7.d\n"
- "st1h z10.h, p2, [%[outptr], #2, MUL VL]\n"
- "zip2 z15.d, z3.d, z7.d\n"
- "whilelt p5.h, %[outpos], %[outwidth]\n"
- "st1h z11.h, p3, [%[outptr], #3, MUL VL]\n"
- "inch %[outpos], all, mul #1\n"
- "st1h z12.h, p4, [%[outptr], #4, MUL VL]\n"
- "whilelt p6.h, %[outpos], %[outwidth]\n"
- "inch %[outpos], all, mul #1\n"
- "st1h z13.h, p5, [%[outptr], #5, MUL VL]\n"
- "whilelt p7.h, %[outpos], %[outwidth]\n"
- "inch %[outpos], all, mul #1\n"
- "st1h z14.h, p6, [%[outptr], #6, MUL VL]\n"
- "st1h z15.h, p7, [%[outptr], #7, MUL VL]\n"
- "addvl %[outptr], %[outptr], #8\n"
- "b 1b\n"
- "2:\n"
- : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1)
- : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
- : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
- );
- break;
-
- case 3:
- __asm __volatile(
- "1:\n"
- "whilelt p0.h, %[inpos], %[inwidth]\n"
- "b.none 2f\n"
- "mov z4.h, #0\n"
- "mov z14.h, #0\n"
- "ld1h z0.h, p0/z, [%[inptr0], %[inpos], LSL #1]\n"
- "ld1h z1.h, p0/z, [%[inptr1], %[inpos], LSL #1]\n"
- "ld1h z2.h, p0/z, [%[inptr2], %[inpos], LSL #1]\n"
- "inch %[inpos], all, mul #1\n"
- "zip1 z8.d, z0.d, z4.d\n"
- "whilelt p0.h, %[outpos], %[outwidth]\n"
- "zip2 z9.d, z0.d, z4.d\n"
- "inch %[outpos], all, mul #1\n"
- "zip1 z10.d, z1.d, z4.d\n"
- "zip2 z11.d, z1.d, z4.d\n"
- "zip1 z12.d, z2.d, z4.d\n"
- "whilelt p1.h, %[outpos], %[outwidth]\n"
- "zip2 z13.d, z2.d, z4.d\n"
- "inch %[outpos], all, mul #1\n"
- "zip1 z0.d, z8.d, z12.d\n"
- "zip2 z1.d, z8.d, z12.d\n"
- "zip1 z2.d, z9.d, z13.d\n"
- "whilelt p2.h, %[outpos], %[outwidth]\n"
- "zip2 z3.d, z9.d, z13.d\n"
- "inch %[outpos], all, mul #1\n"
- "zip1 z4.d, z10.d, z14.d\n"
- "zip2 z5.d, z10.d, z14.d\n"
- "zip1 z6.d, z11.d, z14.d\n"
- "whilelt p3.h, %[outpos], %[outwidth]\n"
- "zip2 z7.d, z11.d, z14.d\n"
- "inch %[outpos], all, mul #1\n"
- "zip1 z8.d, z0.d, z4.d\n"
- "zip2 z9.d, z0.d, z4.d\n"
- "zip1 z10.d, z1.d, z5.d\n"
- "whilelt p4.h, %[outpos], %[outwidth]\n"
- "zip2 z11.d, z1.d, z5.d\n"
- "st1h z8.h, p0, [%[outptr]]\n"
- "zip1 z12.d, z2.d, z6.d\n"
- "inch %[outpos], all, mul #1\n"
- "zip2 z13.d, z2.d, z6.d\n"
- "st1h z9.h, p1, [%[outptr], #1, MUL VL]\n"
- "zip1 z14.d, z3.d, z7.d\n"
- "zip2 z15.d, z3.d, z7.d\n"
- "whilelt p5.h, %[outpos], %[outwidth]\n"
- "st1h z10.h, p2, [%[outptr], #2, MUL VL]\n"
- "inch %[outpos], all, mul #1\n"
- "st1h z11.h, p3, [%[outptr], #3, MUL VL]\n"
- "whilelt p6.h, %[outpos], %[outwidth]\n"
- "inch %[outpos], all, mul #1\n"
- "st1h z12.h, p4, [%[outptr], #4, MUL VL]\n"
- "whilelt p7.h, %[outpos], %[outwidth]\n"
- "inch %[outpos], all, mul #1\n"
- "st1h z13.h, p5, [%[outptr], #5, MUL VL]\n"
- "st1h z14.h, p6, [%[outptr], #6, MUL VL]\n"
- "st1h z15.h, p7, [%[outptr], #7, MUL VL]\n"
- "addvl %[outptr], %[outptr], #8\n"
- "b 1b\n"
- "2:\n"
- : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2)
- : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
- : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
- );
- break;
-
- case 4:
- __asm __volatile(
- "1:\n"
- "whilelt p0.h, %[inpos], %[inwidth]\n"
- "b.none 2f\n"
- "mov z4.h, #0\n"
- "ld1h z0.h, p0/z, [%[inptr0], %[inpos], LSL #1]\n"
- "ld1h z1.h, p0/z, [%[inptr1], %[inpos], LSL #1]\n"
- "ld1h z2.h, p0/z, [%[inptr2], %[inpos], LSL #1]\n"
- "ld1h z3.h, p0/z, [%[inptr3], %[inpos], LSL #1]\n"
- "inch %[inpos], all, mul #1\n"
- "zip1 z8.d, z0.d, z4.d\n"
- "whilelt p0.h, %[outpos], %[outwidth]\n"
- "zip2 z9.d, z0.d, z4.d\n"
- "inch %[outpos], all, mul #1\n"
- "zip1 z10.d, z1.d, z4.d\n"
- "zip2 z11.d, z1.d, z4.d\n"
- "zip1 z12.d, z2.d, z4.d\n"
- "whilelt p1.h, %[outpos], %[outwidth]\n"
- "zip2 z13.d, z2.d, z4.d\n"
- "inch %[outpos], all, mul #1\n"
- "zip1 z14.d, z3.d, z4.d\n"
- "zip2 z15.d, z3.d, z4.d\n"
- "zip1 z0.d, z8.d, z12.d\n"
- "whilelt p2.h, %[outpos], %[outwidth]\n"
- "zip2 z1.d, z8.d, z12.d\n"
- "inch %[outpos], all, mul #1\n"
- "zip1 z2.d, z9.d, z13.d\n"
- "zip2 z3.d, z9.d, z13.d\n"
- "zip1 z4.d, z10.d, z14.d\n"
- "whilelt p3.h, %[outpos], %[outwidth]\n"
- "zip2 z5.d, z10.d, z14.d\n"
- "inch %[outpos], all, mul #1\n"
- "zip1 z6.d, z11.d, z15.d\n"
- "zip2 z7.d, z11.d, z15.d\n"
- "zip1 z8.d, z0.d, z4.d\n"
- "whilelt p4.h, %[outpos], %[outwidth]\n"
- "zip2 z9.d, z0.d, z4.d\n"
- "inch %[outpos], all, mul #1\n"
- "zip1 z10.d, z1.d, z5.d\n"
- "st1h z8.h, p0, [%[outptr]]\n"
- "zip2 z11.d, z1.d, z5.d\n"
- "zip1 z12.d, z2.d, z6.d\n"
- "whilelt p5.h, %[outpos], %[outwidth]\n"
- "zip2 z13.d, z2.d, z6.d\n"
- "st1h z9.h, p1, [%[outptr], #1, MUL VL]\n"
- "zip1 z14.d, z3.d, z7.d\n"
- "inch %[outpos], all, mul #1\n"
- "zip2 z15.d, z3.d, z7.d\n"
- "st1h z10.h, p2, [%[outptr], #2, MUL VL]\n"
- "whilelt p6.h, %[outpos], %[outwidth]\n"
- "st1h z11.h, p3, [%[outptr], #3, MUL VL]\n"
- "inch %[outpos], all, mul #1\n"
- "st1h z12.h, p4, [%[outptr], #4, MUL VL]\n"
- "whilelt p7.h, %[outpos], %[outwidth]\n"
- "inch %[outpos], all, mul #1\n"
- "st1h z13.h, p5, [%[outptr], #5, MUL VL]\n"
- "st1h z14.h, p6, [%[outptr], #6, MUL VL]\n"
- "st1h z15.h, p7, [%[outptr], #7, MUL VL]\n"
- "addvl %[outptr], %[outptr], #8\n"
- "b 1b\n"
- "2:\n"
- : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3)
- : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
- : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
- );
- break;
-
- case 5:
- __asm __volatile(
- "1:\n"
- "whilelt p0.h, %[inpos], %[inwidth]\n"
- "b.none 2f\n"
- "mov z5.h, #0\n"
- "ld1h z0.h, p0/z, [%[inptr0], %[inpos], LSL #1]\n"
- "ld1h z1.h, p0/z, [%[inptr1], %[inpos], LSL #1]\n"
- "ld1h z2.h, p0/z, [%[inptr2], %[inpos], LSL #1]\n"
- "ld1h z3.h, p0/z, [%[inptr3], %[inpos], LSL #1]\n"
- "ld1h z4.h, p0/z, [%[inptr4], %[inpos], LSL #1]\n"
- "inch %[inpos], all, mul #1\n"
- "zip1 z10.d, z1.d, z5.d\n"
- "whilelt p0.h, %[outpos], %[outwidth]\n"
- "zip1 z8.d, z0.d, z4.d\n"
- "inch %[outpos], all, mul #1\n"
- "zip2 z9.d, z0.d, z4.d\n"
- "zip2 z11.d, z1.d, z5.d\n"
- "zip1 z12.d, z2.d, z5.d\n"
- "whilelt p1.h, %[outpos], %[outwidth]\n"
- "zip2 z13.d, z2.d, z5.d\n"
- "inch %[outpos], all, mul #1\n"
- "zip1 z14.d, z3.d, z5.d\n"
- "zip2 z15.d, z3.d, z5.d\n"
- "zip1 z0.d, z8.d, z12.d\n"
- "whilelt p2.h, %[outpos], %[outwidth]\n"
- "zip2 z1.d, z8.d, z12.d\n"
- "inch %[outpos], all, mul #1\n"
- "zip1 z2.d, z9.d, z13.d\n"
- "zip2 z3.d, z9.d, z13.d\n"
- "zip1 z4.d, z10.d, z14.d\n"
- "whilelt p3.h, %[outpos], %[outwidth]\n"
- "zip2 z5.d, z10.d, z14.d\n"
- "inch %[outpos], all, mul #1\n"
- "zip1 z6.d, z11.d, z15.d\n"
- "zip2 z7.d, z11.d, z15.d\n"
- "zip1 z8.d, z0.d, z4.d\n"
- "whilelt p4.h, %[outpos], %[outwidth]\n"
- "zip2 z9.d, z0.d, z4.d\n"
- "inch %[outpos], all, mul #1\n"
- "zip1 z10.d, z1.d, z5.d\n"
- "st1h z8.h, p0, [%[outptr]]\n"
- "zip2 z11.d, z1.d, z5.d\n"
- "zip1 z12.d, z2.d, z6.d\n"
- "whilelt p5.h, %[outpos], %[outwidth]\n"
- "zip2 z13.d, z2.d, z6.d\n"
- "st1h z9.h, p1, [%[outptr], #1, MUL VL]\n"
- "zip1 z14.d, z3.d, z7.d\n"
- "inch %[outpos], all, mul #1\n"
- "zip2 z15.d, z3.d, z7.d\n"
- "st1h z10.h, p2, [%[outptr], #2, MUL VL]\n"
- "whilelt p6.h, %[outpos], %[outwidth]\n"
- "st1h z11.h, p3, [%[outptr], #3, MUL VL]\n"
- "inch %[outpos], all, mul #1\n"
- "st1h z12.h, p4, [%[outptr], #4, MUL VL]\n"
- "whilelt p7.h, %[outpos], %[outwidth]\n"
- "inch %[outpos], all, mul #1\n"
- "st1h z13.h, p5, [%[outptr], #5, MUL VL]\n"
- "st1h z14.h, p6, [%[outptr], #6, MUL VL]\n"
- "st1h z15.h, p7, [%[outptr], #7, MUL VL]\n"
- "addvl %[outptr], %[outptr], #8\n"
- "b 1b\n"
- "2:\n"
- : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3), [inptr4] "+r" (inptr4)
- : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
- : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
- );
- break;
-
- case 6:
- __asm __volatile(
- "1:\n"
- "whilelt p0.h, %[inpos], %[inwidth]\n"
- "b.none 2f\n"
- "mov z6.h, #0\n"
- "ld1h z0.h, p0/z, [%[inptr0], %[inpos], LSL #1]\n"
- "ld1h z1.h, p0/z, [%[inptr1], %[inpos], LSL #1]\n"
- "ld1h z2.h, p0/z, [%[inptr2], %[inpos], LSL #1]\n"
- "ld1h z3.h, p0/z, [%[inptr3], %[inpos], LSL #1]\n"
- "ld1h z4.h, p0/z, [%[inptr4], %[inpos], LSL #1]\n"
- "ld1h z5.h, p0/z, [%[inptr5], %[inpos], LSL #1]\n"
- "inch %[inpos], all, mul #1\n"
- "zip1 z12.d, z2.d, z6.d\n"
- "whilelt p0.h, %[outpos], %[outwidth]\n"
- "zip1 z8.d, z0.d, z4.d\n"
- "inch %[outpos], all, mul #1\n"
- "zip2 z9.d, z0.d, z4.d\n"
- "zip1 z10.d, z1.d, z5.d\n"
- "zip2 z11.d, z1.d, z5.d\n"
- "whilelt p1.h, %[outpos], %[outwidth]\n"
- "zip2 z13.d, z2.d, z6.d\n"
- "inch %[outpos], all, mul #1\n"
- "zip1 z14.d, z3.d, z6.d\n"
- "zip2 z15.d, z3.d, z6.d\n"
- "zip1 z0.d, z8.d, z12.d\n"
- "whilelt p2.h, %[outpos], %[outwidth]\n"
- "zip2 z1.d, z8.d, z12.d\n"
- "inch %[outpos], all, mul #1\n"
- "zip1 z2.d, z9.d, z13.d\n"
- "zip2 z3.d, z9.d, z13.d\n"
- "zip1 z4.d, z10.d, z14.d\n"
- "whilelt p3.h, %[outpos], %[outwidth]\n"
- "zip2 z5.d, z10.d, z14.d\n"
- "inch %[outpos], all, mul #1\n"
- "zip1 z6.d, z11.d, z15.d\n"
- "zip2 z7.d, z11.d, z15.d\n"
- "zip1 z8.d, z0.d, z4.d\n"
- "whilelt p4.h, %[outpos], %[outwidth]\n"
- "zip2 z9.d, z0.d, z4.d\n"
- "inch %[outpos], all, mul #1\n"
- "zip1 z10.d, z1.d, z5.d\n"
- "st1h z8.h, p0, [%[outptr]]\n"
- "zip2 z11.d, z1.d, z5.d\n"
- "zip1 z12.d, z2.d, z6.d\n"
- "whilelt p5.h, %[outpos], %[outwidth]\n"
- "zip2 z13.d, z2.d, z6.d\n"
- "st1h z9.h, p1, [%[outptr], #1, MUL VL]\n"
- "zip1 z14.d, z3.d, z7.d\n"
- "inch %[outpos], all, mul #1\n"
- "zip2 z15.d, z3.d, z7.d\n"
- "st1h z10.h, p2, [%[outptr], #2, MUL VL]\n"
- "whilelt p6.h, %[outpos], %[outwidth]\n"
- "st1h z11.h, p3, [%[outptr], #3, MUL VL]\n"
- "inch %[outpos], all, mul #1\n"
- "st1h z12.h, p4, [%[outptr], #4, MUL VL]\n"
- "whilelt p7.h, %[outpos], %[outwidth]\n"
- "inch %[outpos], all, mul #1\n"
- "st1h z13.h, p5, [%[outptr], #5, MUL VL]\n"
- "st1h z14.h, p6, [%[outptr], #6, MUL VL]\n"
- "st1h z15.h, p7, [%[outptr], #7, MUL VL]\n"
- "addvl %[outptr], %[outptr], #8\n"
- "b 1b\n"
- "2:\n"
- : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3), [inptr4] "+r" (inptr4), [inptr5] "+r" (inptr5)
- : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
- : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
- );
- break;
-
- case 7:
- __asm __volatile(
- "1:\n"
- "whilelt p0.h, %[inpos], %[inwidth]\n"
- "b.none 2f\n"
- "mov z7.h, #0\n"
- "ld1h z0.h, p0/z, [%[inptr0], %[inpos], LSL #1]\n"
- "ld1h z1.h, p0/z, [%[inptr1], %[inpos], LSL #1]\n"
- "ld1h z2.h, p0/z, [%[inptr2], %[inpos], LSL #1]\n"
- "ld1h z3.h, p0/z, [%[inptr3], %[inpos], LSL #1]\n"
- "ld1h z4.h, p0/z, [%[inptr4], %[inpos], LSL #1]\n"
- "ld1h z5.h, p0/z, [%[inptr5], %[inpos], LSL #1]\n"
- "ld1h z6.h, p0/z, [%[inptr6], %[inpos], LSL #1]\n"
- "inch %[inpos], all, mul #1\n"
- "zip1 z8.d, z0.d, z4.d\n"
- "whilelt p0.h, %[outpos], %[outwidth]\n"
- "zip2 z9.d, z0.d, z4.d\n"
- "inch %[outpos], all, mul #1\n"
- "zip1 z10.d, z1.d, z5.d\n"
- "zip2 z11.d, z1.d, z5.d\n"
- "zip1 z12.d, z2.d, z6.d\n"
- "whilelt p1.h, %[outpos], %[outwidth]\n"
- "zip2 z13.d, z2.d, z6.d\n"
- "inch %[outpos], all, mul #1\n"
- "zip1 z14.d, z3.d, z7.d\n"
- "zip2 z15.d, z3.d, z7.d\n"
- "zip1 z0.d, z8.d, z12.d\n"
- "whilelt p2.h, %[outpos], %[outwidth]\n"
- "zip2 z1.d, z8.d, z12.d\n"
- "inch %[outpos], all, mul #1\n"
- "zip1 z2.d, z9.d, z13.d\n"
- "zip2 z3.d, z9.d, z13.d\n"
- "zip1 z4.d, z10.d, z14.d\n"
- "whilelt p3.h, %[outpos], %[outwidth]\n"
- "zip2 z5.d, z10.d, z14.d\n"
- "inch %[outpos], all, mul #1\n"
- "zip1 z6.d, z11.d, z15.d\n"
- "zip2 z7.d, z11.d, z15.d\n"
- "zip1 z8.d, z0.d, z4.d\n"
- "whilelt p4.h, %[outpos], %[outwidth]\n"
- "zip2 z9.d, z0.d, z4.d\n"
- "inch %[outpos], all, mul #1\n"
- "zip1 z10.d, z1.d, z5.d\n"
- "st1h z8.h, p0, [%[outptr]]\n"
- "zip2 z11.d, z1.d, z5.d\n"
- "zip1 z12.d, z2.d, z6.d\n"
- "whilelt p5.h, %[outpos], %[outwidth]\n"
- "zip2 z13.d, z2.d, z6.d\n"
- "st1h z9.h, p1, [%[outptr], #1, MUL VL]\n"
- "zip1 z14.d, z3.d, z7.d\n"
- "inch %[outpos], all, mul #1\n"
- "zip2 z15.d, z3.d, z7.d\n"
- "st1h z10.h, p2, [%[outptr], #2, MUL VL]\n"
- "whilelt p6.h, %[outpos], %[outwidth]\n"
- "st1h z11.h, p3, [%[outptr], #3, MUL VL]\n"
- "inch %[outpos], all, mul #1\n"
- "st1h z12.h, p4, [%[outptr], #4, MUL VL]\n"
- "whilelt p7.h, %[outpos], %[outwidth]\n"
- "inch %[outpos], all, mul #1\n"
- "st1h z13.h, p5, [%[outptr], #5, MUL VL]\n"
- "st1h z14.h, p6, [%[outptr], #6, MUL VL]\n"
- "st1h z15.h, p7, [%[outptr], #7, MUL VL]\n"
- "addvl %[outptr], %[outptr], #8\n"
- "b 1b\n"
- "2:\n"
- : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3), [inptr4] "+r" (inptr4), [inptr5] "+r" (inptr5), [inptr6] "+r" (inptr6)
- : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
- : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
- );
- break;
-
- default:
- case 8:
- __asm __volatile(
- "1:\n"
- "whilelt p0.h, %[inpos], %[inwidth]\n"
- "b.none 2f\n"
- "ld1h z0.h, p0/z, [%[inptr0], %[inpos], LSL #1]\n"
- "ld1h z1.h, p0/z, [%[inptr1], %[inpos], LSL #1]\n"
- "ld1h z2.h, p0/z, [%[inptr2], %[inpos], LSL #1]\n"
- "ld1h z3.h, p0/z, [%[inptr3], %[inpos], LSL #1]\n"
- "ld1h z4.h, p0/z, [%[inptr4], %[inpos], LSL #1]\n"
- "ld1h z5.h, p0/z, [%[inptr5], %[inpos], LSL #1]\n"
- "ld1h z6.h, p0/z, [%[inptr6], %[inpos], LSL #1]\n"
- "ld1h z7.h, p0/z, [%[inptr7], %[inpos], LSL #1]\n"
- "inch %[inpos], all, mul #1\n"
- "zip1 z8.d, z0.d, z4.d\n"
- "whilelt p0.h, %[outpos], %[outwidth]\n"
- "zip2 z9.d, z0.d, z4.d\n"
- "inch %[outpos], all, mul #1\n"
- "zip1 z10.d, z1.d, z5.d\n"
- "zip2 z11.d, z1.d, z5.d\n"
- "zip1 z12.d, z2.d, z6.d\n"
- "whilelt p1.h, %[outpos], %[outwidth]\n"
- "zip2 z13.d, z2.d, z6.d\n"
- "inch %[outpos], all, mul #1\n"
- "zip1 z14.d, z3.d, z7.d\n"
- "zip2 z15.d, z3.d, z7.d\n"
- "zip1 z0.d, z8.d, z12.d\n"
- "whilelt p2.h, %[outpos], %[outwidth]\n"
- "zip2 z1.d, z8.d, z12.d\n"
- "inch %[outpos], all, mul #1\n"
- "zip1 z2.d, z9.d, z13.d\n"
- "zip2 z3.d, z9.d, z13.d\n"
- "zip1 z4.d, z10.d, z14.d\n"
- "whilelt p3.h, %[outpos], %[outwidth]\n"
- "zip2 z5.d, z10.d, z14.d\n"
- "inch %[outpos], all, mul #1\n"
- "zip1 z6.d, z11.d, z15.d\n"
- "zip2 z7.d, z11.d, z15.d\n"
- "zip1 z8.d, z0.d, z4.d\n"
- "whilelt p4.h, %[outpos], %[outwidth]\n"
- "zip2 z9.d, z0.d, z4.d\n"
- "inch %[outpos], all, mul #1\n"
- "zip1 z10.d, z1.d, z5.d\n"
- "st1h z8.h, p0, [%[outptr]]\n"
- "zip2 z11.d, z1.d, z5.d\n"
- "zip1 z12.d, z2.d, z6.d\n"
- "whilelt p5.h, %[outpos], %[outwidth]\n"
- "zip2 z13.d, z2.d, z6.d\n"
- "st1h z9.h, p1, [%[outptr], #1, MUL VL]\n"
- "zip1 z14.d, z3.d, z7.d\n"
- "inch %[outpos], all, mul #1\n"
- "zip2 z15.d, z3.d, z7.d\n"
- "st1h z10.h, p2, [%[outptr], #2, MUL VL]\n"
- "whilelt p6.h, %[outpos], %[outwidth]\n"
- "st1h z11.h, p3, [%[outptr], #3, MUL VL]\n"
- "inch %[outpos], all, mul #1\n"
- "st1h z12.h, p4, [%[outptr], #4, MUL VL]\n"
- "whilelt p7.h, %[outpos], %[outwidth]\n"
- "inch %[outpos], all, mul #1\n"
- "st1h z13.h, p5, [%[outptr], #5, MUL VL]\n"
- "st1h z14.h, p6, [%[outptr], #6, MUL VL]\n"
- "st1h z15.h, p7, [%[outptr], #7, MUL VL]\n"
- "addvl %[outptr], %[outptr], #8\n"
- "b 1b\n"
- "2:\n"
- : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3), [inptr4] "+r" (inptr4), [inptr5] "+r" (inptr5), [inptr6] "+r" (inptr6), [inptr7] "+r" (inptr7)
- : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
- : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
- );
- break;
-
-
- }
- }
-}
-
-#endif // __ARM_FEATURE_SVE
diff --git a/src/core/NEON/kernels/arm_gemm/transforms/sve_interleave_8way_block4_8bit.hpp b/src/core/NEON/kernels/arm_gemm/transforms/sve_interleave_8way_block4_8bit.hpp
deleted file mode 100644
index ed0d58a..0000000
--- a/src/core/NEON/kernels/arm_gemm/transforms/sve_interleave_8way_block4_8bit.hpp
+++ /dev/null
@@ -1,596 +0,0 @@
-/*
- * Copyright (c) 2018-2020 Arm Limited.
- *
- * SPDX-License-Identifier: MIT
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to
- * deal in the Software without restriction, including without limitation the
- * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
- * sell copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-#pragma once
-
-#ifdef __ARM_FEATURE_SVE
-
-template<>
-template<typename T>
-inline void TransformImpl<8, 4, false, 1, 1, false>::Transform(T *out, const T *in, int ldin, int y0, int ymax, int k0, int kmax)
-{
- uint8_t *master_outptr = reinterpret_cast<uint8_t *>(out);
- const uint8_t *inptr = reinterpret_cast<const uint8_t *>(in);
-
- for (int y=y0; y<ymax; y+=8)
- {
- const int height = ymax-y;
- const long inwidth = (kmax - k0);
- const long outwidth = ((inwidth + 3) / 4) * 32;
- long inpos = 0;
- long outpos = 0;
-
- uint8_t *outptr = master_outptr;
- master_outptr += outwidth;
-
- const uint8_t *inptr0 = inptr + y * ldin + k0;
- const uint8_t *inptr1 = inptr0 + ldin;
- const uint8_t *inptr2 = inptr1 + ldin;
- const uint8_t *inptr3 = inptr2 + ldin;
- const uint8_t *inptr4 = inptr3 + ldin;
- const uint8_t *inptr5 = inptr4 + ldin;
- const uint8_t *inptr6 = inptr5 + ldin;
- const uint8_t *inptr7 = inptr6 + ldin;
-
- switch(height)
- {
- case 1:
- __asm __volatile(
- "1:\n"
- "whilelt p0.b, %[inpos], %[inwidth]\n"
- "b.none 2f\n"
- "mov z4.b, #0\n"
- "ld1b z0.b, p0/z, [%[inptr0], %[inpos]]\n"
- "incb %[inpos], all, mul #1\n"
- "whilelt p0.b, %[outpos], %[outwidth]\n"
- "incb %[outpos], all, mul #1\n"
- "zip1 z8.s, z0.s, z4.s\n"
- "zip2 z9.s, z0.s, z4.s\n"
- "whilelt p1.b, %[outpos], %[outwidth]\n"
- "zip1 z0.s, z8.s, z4.s\n"
- "incb %[outpos], all, mul #1\n"
- "zip2 z1.s, z8.s, z4.s\n"
- "zip1 z2.s, z9.s, z4.s\n"
- "zip2 z3.s, z9.s, z4.s\n"
- "whilelt p2.b, %[outpos], %[outwidth]\n"
- "zip1 z8.s, z0.s, z4.s\n"
- "incb %[outpos], all, mul #1\n"
- "zip2 z9.s, z0.s, z4.s\n"
- "zip1 z10.s, z1.s, z4.s\n"
- "st1b z8.b, p0, [%[outptr]]\n"
- "zip2 z11.s, z1.s, z4.s\n"
- "whilelt p3.b, %[outpos], %[outwidth]\n"
- "zip1 z12.s, z2.s, z4.s\n"
- "st1b z9.b, p1, [%[outptr], #1, MUL VL]\n"
- "zip2 z13.s, z2.s, z4.s\n"
- "incb %[outpos], all, mul #1\n"
- "zip1 z14.s, z3.s, z4.s\n"
- "st1b z10.b, p2, [%[outptr], #2, MUL VL]\n"
- "zip2 z15.s, z3.s, z4.s\n"
- "whilelt p4.b, %[outpos], %[outwidth]\n"
- "st1b z11.b, p3, [%[outptr], #3, MUL VL]\n"
- "incb %[outpos], all, mul #1\n"
- "st1b z12.b, p4, [%[outptr], #4, MUL VL]\n"
- "whilelt p5.b, %[outpos], %[outwidth]\n"
- "incb %[outpos], all, mul #1\n"
- "st1b z13.b, p5, [%[outptr], #5, MUL VL]\n"
- "whilelt p6.b, %[outpos], %[outwidth]\n"
- "incb %[outpos], all, mul #1\n"
- "st1b z14.b, p6, [%[outptr], #6, MUL VL]\n"
- "whilelt p7.b, %[outpos], %[outwidth]\n"
- "incb %[outpos], all, mul #1\n"
- "st1b z15.b, p7, [%[outptr], #7, MUL VL]\n"
- "addvl %[outptr], %[outptr], #8\n"
- "b 1b\n"
- "2:\n"
- : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0)
- : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
- : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
- );
- break;
-
- case 2:
- __asm __volatile(
- "1:\n"
- "whilelt p0.b, %[inpos], %[inwidth]\n"
- "b.none 2f\n"
- "mov z4.b, #0\n"
- "mov z14.b, #0\n"
- "ld1b z0.b, p0/z, [%[inptr0], %[inpos]]\n"
- "ld1b z1.b, p0/z, [%[inptr1], %[inpos]]\n"
- "incb %[inpos], all, mul #1\n"
- "whilelt p0.b, %[outpos], %[outwidth]\n"
- "zip1 z8.s, z0.s, z4.s\n"
- "incb %[outpos], all, mul #1\n"
- "zip2 z9.s, z0.s, z4.s\n"
- "zip1 z10.s, z1.s, z4.s\n"
- "zip2 z11.s, z1.s, z4.s\n"
- "whilelt p1.b, %[outpos], %[outwidth]\n"
- "zip1 z0.s, z8.s, z4.s\n"
- "incb %[outpos], all, mul #1\n"
- "zip2 z1.s, z8.s, z4.s\n"
- "zip1 z2.s, z9.s, z4.s\n"
- "zip2 z3.s, z9.s, z4.s\n"
- "whilelt p2.b, %[outpos], %[outwidth]\n"
- "zip1 z4.s, z10.s, z14.s\n"
- "incb %[outpos], all, mul #1\n"
- "zip2 z5.s, z10.s, z14.s\n"
- "zip1 z6.s, z11.s, z14.s\n"
- "zip2 z7.s, z11.s, z14.s\n"
- "whilelt p3.b, %[outpos], %[outwidth]\n"
- "zip1 z8.s, z0.s, z4.s\n"
- "incb %[outpos], all, mul #1\n"
- "zip2 z9.s, z0.s, z4.s\n"
- "zip1 z10.s, z1.s, z5.s\n"
- "st1b z8.b, p0, [%[outptr]]\n"
- "zip2 z11.s, z1.s, z5.s\n"
- "whilelt p4.b, %[outpos], %[outwidth]\n"
- "zip1 z12.s, z2.s, z6.s\n"
- "st1b z9.b, p1, [%[outptr], #1, MUL VL]\n"
- "zip2 z13.s, z2.s, z6.s\n"
- "incb %[outpos], all, mul #1\n"
- "zip1 z14.s, z3.s, z7.s\n"
- "st1b z10.b, p2, [%[outptr], #2, MUL VL]\n"
- "zip2 z15.s, z3.s, z7.s\n"
- "whilelt p5.b, %[outpos], %[outwidth]\n"
- "st1b z11.b, p3, [%[outptr], #3, MUL VL]\n"
- "incb %[outpos], all, mul #1\n"
- "st1b z12.b, p4, [%[outptr], #4, MUL VL]\n"
- "whilelt p6.b, %[outpos], %[outwidth]\n"
- "incb %[outpos], all, mul #1\n"
- "st1b z13.b, p5, [%[outptr], #5, MUL VL]\n"
- "whilelt p7.b, %[outpos], %[outwidth]\n"
- "incb %[outpos], all, mul #1\n"
- "st1b z14.b, p6, [%[outptr], #6, MUL VL]\n"
- "st1b z15.b, p7, [%[outptr], #7, MUL VL]\n"
- "addvl %[outptr], %[outptr], #8\n"
- "b 1b\n"
- "2:\n"
- : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1)
- : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
- : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
- );
- break;
-
- case 3:
- __asm __volatile(
- "1:\n"
- "whilelt p0.b, %[inpos], %[inwidth]\n"
- "b.none 2f\n"
- "mov z4.b, #0\n"
- "mov z14.b, #0\n"
- "ld1b z0.b, p0/z, [%[inptr0], %[inpos]]\n"
- "ld1b z1.b, p0/z, [%[inptr1], %[inpos]]\n"
- "ld1b z2.b, p0/z, [%[inptr2], %[inpos]]\n"
- "incb %[inpos], all, mul #1\n"
- "zip1 z8.s, z0.s, z4.s\n"
- "whilelt p0.b, %[outpos], %[outwidth]\n"
- "zip2 z9.s, z0.s, z4.s\n"
- "incb %[outpos], all, mul #1\n"
- "zip1 z10.s, z1.s, z4.s\n"
- "zip2 z11.s, z1.s, z4.s\n"
- "zip1 z12.s, z2.s, z4.s\n"
- "whilelt p1.b, %[outpos], %[outwidth]\n"
- "zip2 z13.s, z2.s, z4.s\n"
- "incb %[outpos], all, mul #1\n"
- "zip1 z0.s, z8.s, z12.s\n"
- "zip2 z1.s, z8.s, z12.s\n"
- "zip1 z2.s, z9.s, z13.s\n"
- "whilelt p2.b, %[outpos], %[outwidth]\n"
- "zip2 z3.s, z9.s, z13.s\n"
- "incb %[outpos], all, mul #1\n"
- "zip1 z4.s, z10.s, z14.s\n"
- "zip2 z5.s, z10.s, z14.s\n"
- "zip1 z6.s, z11.s, z14.s\n"
- "whilelt p3.b, %[outpos], %[outwidth]\n"
- "zip2 z7.s, z11.s, z14.s\n"
- "incb %[outpos], all, mul #1\n"
- "zip1 z8.s, z0.s, z4.s\n"
- "zip2 z9.s, z0.s, z4.s\n"
- "zip1 z10.s, z1.s, z5.s\n"
- "whilelt p4.b, %[outpos], %[outwidth]\n"
- "zip2 z11.s, z1.s, z5.s\n"
- "st1b z8.b, p0, [%[outptr]]\n"
- "zip1 z12.s, z2.s, z6.s\n"
- "incb %[outpos], all, mul #1\n"
- "zip2 z13.s, z2.s, z6.s\n"
- "st1b z9.b, p1, [%[outptr], #1, MUL VL]\n"
- "zip1 z14.s, z3.s, z7.s\n"
- "zip2 z15.s, z3.s, z7.s\n"
- "whilelt p5.b, %[outpos], %[outwidth]\n"
- "st1b z10.b, p2, [%[outptr], #2, MUL VL]\n"
- "incb %[outpos], all, mul #1\n"
- "st1b z11.b, p3, [%[outptr], #3, MUL VL]\n"
- "whilelt p6.b, %[outpos], %[outwidth]\n"
- "incb %[outpos], all, mul #1\n"
- "st1b z12.b, p4, [%[outptr], #4, MUL VL]\n"
- "whilelt p7.b, %[outpos], %[outwidth]\n"
- "incb %[outpos], all, mul #1\n"
- "st1b z13.b, p5, [%[outptr], #5, MUL VL]\n"
- "st1b z14.b, p6, [%[outptr], #6, MUL VL]\n"
- "st1b z15.b, p7, [%[outptr], #7, MUL VL]\n"
- "addvl %[outptr], %[outptr], #8\n"
- "b 1b\n"
- "2:\n"
- : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2)
- : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
- : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
- );
- break;
-
- case 4:
- __asm __volatile(
- "1:\n"
- "whilelt p0.b, %[inpos], %[inwidth]\n"
- "b.none 2f\n"
- "mov z4.b, #0\n"
- "ld1b z0.b, p0/z, [%[inptr0], %[inpos]]\n"
- "ld1b z1.b, p0/z, [%[inptr1], %[inpos]]\n"
- "ld1b z2.b, p0/z, [%[inptr2], %[inpos]]\n"
- "ld1b z3.b, p0/z, [%[inptr3], %[inpos]]\n"
- "incb %[inpos], all, mul #1\n"
- "zip1 z8.s, z0.s, z4.s\n"
- "whilelt p0.b, %[outpos], %[outwidth]\n"
- "zip2 z9.s, z0.s, z4.s\n"
- "incb %[outpos], all, mul #1\n"
- "zip1 z10.s, z1.s, z4.s\n"
- "zip2 z11.s, z1.s, z4.s\n"
- "zip1 z12.s, z2.s, z4.s\n"
- "whilelt p1.b, %[outpos], %[outwidth]\n"
- "zip2 z13.s, z2.s, z4.s\n"
- "incb %[outpos], all, mul #1\n"
- "zip1 z14.s, z3.s, z4.s\n"
- "zip2 z15.s, z3.s, z4.s\n"
- "zip1 z0.s, z8.s, z12.s\n"
- "whilelt p2.b, %[outpos], %[outwidth]\n"
- "zip2 z1.s, z8.s, z12.s\n"
- "incb %[outpos], all, mul #1\n"
- "zip1 z2.s, z9.s, z13.s\n"
- "zip2 z3.s, z9.s, z13.s\n"
- "zip1 z4.s, z10.s, z14.s\n"
- "whilelt p3.b, %[outpos], %[outwidth]\n"
- "zip2 z5.s, z10.s, z14.s\n"
- "incb %[outpos], all, mul #1\n"
- "zip1 z6.s, z11.s, z15.s\n"
- "zip2 z7.s, z11.s, z15.s\n"
- "zip1 z8.s, z0.s, z4.s\n"
- "whilelt p4.b, %[outpos], %[outwidth]\n"
- "zip2 z9.s, z0.s, z4.s\n"
- "incb %[outpos], all, mul #1\n"
- "zip1 z10.s, z1.s, z5.s\n"
- "st1b z8.b, p0, [%[outptr]]\n"
- "zip2 z11.s, z1.s, z5.s\n"
- "zip1 z12.s, z2.s, z6.s\n"
- "whilelt p5.b, %[outpos], %[outwidth]\n"
- "zip2 z13.s, z2.s, z6.s\n"
- "st1b z9.b, p1, [%[outptr], #1, MUL VL]\n"
- "zip1 z14.s, z3.s, z7.s\n"
- "incb %[outpos], all, mul #1\n"
- "zip2 z15.s, z3.s, z7.s\n"
- "st1b z10.b, p2, [%[outptr], #2, MUL VL]\n"
- "whilelt p6.b, %[outpos], %[outwidth]\n"
- "st1b z11.b, p3, [%[outptr], #3, MUL VL]\n"
- "incb %[outpos], all, mul #1\n"
- "st1b z12.b, p4, [%[outptr], #4, MUL VL]\n"
- "whilelt p7.b, %[outpos], %[outwidth]\n"
- "incb %[outpos], all, mul #1\n"
- "st1b z13.b, p5, [%[outptr], #5, MUL VL]\n"
- "st1b z14.b, p6, [%[outptr], #6, MUL VL]\n"
- "st1b z15.b, p7, [%[outptr], #7, MUL VL]\n"
- "addvl %[outptr], %[outptr], #8\n"
- "b 1b\n"
- "2:\n"
- : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3)
- : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
- : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
- );
- break;
-
- case 5:
- __asm __volatile(
- "1:\n"
- "whilelt p0.b, %[inpos], %[inwidth]\n"
- "b.none 2f\n"
- "mov z5.b, #0\n"
- "ld1b z0.b, p0/z, [%[inptr0], %[inpos]]\n"
- "ld1b z1.b, p0/z, [%[inptr1], %[inpos]]\n"
- "ld1b z2.b, p0/z, [%[inptr2], %[inpos]]\n"
- "ld1b z3.b, p0/z, [%[inptr3], %[inpos]]\n"
- "ld1b z4.b, p0/z, [%[inptr4], %[inpos]]\n"
- "incb %[inpos], all, mul #1\n"
- "zip1 z10.s, z1.s, z5.s\n"
- "whilelt p0.b, %[outpos], %[outwidth]\n"
- "zip1 z8.s, z0.s, z4.s\n"
- "incb %[outpos], all, mul #1\n"
- "zip2 z9.s, z0.s, z4.s\n"
- "zip2 z11.s, z1.s, z5.s\n"
- "zip1 z12.s, z2.s, z5.s\n"
- "whilelt p1.b, %[outpos], %[outwidth]\n"
- "zip2 z13.s, z2.s, z5.s\n"
- "incb %[outpos], all, mul #1\n"
- "zip1 z14.s, z3.s, z5.s\n"
- "zip2 z15.s, z3.s, z5.s\n"
- "zip1 z0.s, z8.s, z12.s\n"
- "whilelt p2.b, %[outpos], %[outwidth]\n"
- "zip2 z1.s, z8.s, z12.s\n"
- "incb %[outpos], all, mul #1\n"
- "zip1 z2.s, z9.s, z13.s\n"
- "zip2 z3.s, z9.s, z13.s\n"
- "zip1 z4.s, z10.s, z14.s\n"
- "whilelt p3.b, %[outpos], %[outwidth]\n"
- "zip2 z5.s, z10.s, z14.s\n"
- "incb %[outpos], all, mul #1\n"
- "zip1 z6.s, z11.s, z15.s\n"
- "zip2 z7.s, z11.s, z15.s\n"
- "zip1 z8.s, z0.s, z4.s\n"
- "whilelt p4.b, %[outpos], %[outwidth]\n"
- "zip2 z9.s, z0.s, z4.s\n"
- "incb %[outpos], all, mul #1\n"
- "zip1 z10.s, z1.s, z5.s\n"
- "st1b z8.b, p0, [%[outptr]]\n"
- "zip2 z11.s, z1.s, z5.s\n"
- "zip1 z12.s, z2.s, z6.s\n"
- "whilelt p5.b, %[outpos], %[outwidth]\n"
- "zip2 z13.s, z2.s, z6.s\n"
- "st1b z9.b, p1, [%[outptr], #1, MUL VL]\n"
- "zip1 z14.s, z3.s, z7.s\n"
- "incb %[outpos], all, mul #1\n"
- "zip2 z15.s, z3.s, z7.s\n"
- "st1b z10.b, p2, [%[outptr], #2, MUL VL]\n"
- "whilelt p6.b, %[outpos], %[outwidth]\n"
- "st1b z11.b, p3, [%[outptr], #3, MUL VL]\n"
- "incb %[outpos], all, mul #1\n"
- "st1b z12.b, p4, [%[outptr], #4, MUL VL]\n"
- "whilelt p7.b, %[outpos], %[outwidth]\n"
- "incb %[outpos], all, mul #1\n"
- "st1b z13.b, p5, [%[outptr], #5, MUL VL]\n"
- "st1b z14.b, p6, [%[outptr], #6, MUL VL]\n"
- "st1b z15.b, p7, [%[outptr], #7, MUL VL]\n"
- "addvl %[outptr], %[outptr], #8\n"
- "b 1b\n"
- "2:\n"
- : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3), [inptr4] "+r" (inptr4)
- : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
- : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
- );
- break;
-
- case 6:
- __asm __volatile(
- "1:\n"
- "whilelt p0.b, %[inpos], %[inwidth]\n"
- "b.none 2f\n"
- "mov z6.b, #0\n"
- "ld1b z0.b, p0/z, [%[inptr0], %[inpos]]\n"
- "ld1b z1.b, p0/z, [%[inptr1], %[inpos]]\n"
- "ld1b z2.b, p0/z, [%[inptr2], %[inpos]]\n"
- "ld1b z3.b, p0/z, [%[inptr3], %[inpos]]\n"
- "ld1b z4.b, p0/z, [%[inptr4], %[inpos]]\n"
- "ld1b z5.b, p0/z, [%[inptr5], %[inpos]]\n"
- "incb %[inpos], all, mul #1\n"
- "zip1 z12.s, z2.s, z6.s\n"
- "whilelt p0.b, %[outpos], %[outwidth]\n"
- "zip1 z8.s, z0.s, z4.s\n"
- "incb %[outpos], all, mul #1\n"
- "zip2 z9.s, z0.s, z4.s\n"
- "zip1 z10.s, z1.s, z5.s\n"
- "zip2 z11.s, z1.s, z5.s\n"
- "whilelt p1.b, %[outpos], %[outwidth]\n"
- "zip2 z13.s, z2.s, z6.s\n"
- "incb %[outpos], all, mul #1\n"
- "zip1 z14.s, z3.s, z6.s\n"
- "zip2 z15.s, z3.s, z6.s\n"
- "zip1 z0.s, z8.s, z12.s\n"
- "whilelt p2.b, %[outpos], %[outwidth]\n"
- "zip2 z1.s, z8.s, z12.s\n"
- "incb %[outpos], all, mul #1\n"
- "zip1 z2.s, z9.s, z13.s\n"
- "zip2 z3.s, z9.s, z13.s\n"
- "zip1 z4.s, z10.s, z14.s\n"
- "whilelt p3.b, %[outpos], %[outwidth]\n"
- "zip2 z5.s, z10.s, z14.s\n"
- "incb %[outpos], all, mul #1\n"
- "zip1 z6.s, z11.s, z15.s\n"
- "zip2 z7.s, z11.s, z15.s\n"
- "zip1 z8.s, z0.s, z4.s\n"
- "whilelt p4.b, %[outpos], %[outwidth]\n"
- "zip2 z9.s, z0.s, z4.s\n"
- "incb %[outpos], all, mul #1\n"
- "zip1 z10.s, z1.s, z5.s\n"
- "st1b z8.b, p0, [%[outptr]]\n"
- "zip2 z11.s, z1.s, z5.s\n"
- "zip1 z12.s, z2.s, z6.s\n"
- "whilelt p5.b, %[outpos], %[outwidth]\n"
- "zip2 z13.s, z2.s, z6.s\n"
- "st1b z9.b, p1, [%[outptr], #1, MUL VL]\n"
- "zip1 z14.s, z3.s, z7.s\n"
- "incb %[outpos], all, mul #1\n"
- "zip2 z15.s, z3.s, z7.s\n"
- "st1b z10.b, p2, [%[outptr], #2, MUL VL]\n"
- "whilelt p6.b, %[outpos], %[outwidth]\n"
- "st1b z11.b, p3, [%[outptr], #3, MUL VL]\n"
- "incb %[outpos], all, mul #1\n"
- "st1b z12.b, p4, [%[outptr], #4, MUL VL]\n"
- "whilelt p7.b, %[outpos], %[outwidth]\n"
- "incb %[outpos], all, mul #1\n"
- "st1b z13.b, p5, [%[outptr], #5, MUL VL]\n"
- "st1b z14.b, p6, [%[outptr], #6, MUL VL]\n"
- "st1b z15.b, p7, [%[outptr], #7, MUL VL]\n"
- "addvl %[outptr], %[outptr], #8\n"
- "b 1b\n"
- "2:\n"
- : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3), [inptr4] "+r" (inptr4), [inptr5] "+r" (inptr5)
- : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
- : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
- );
- break;
-
- case 7:
- __asm __volatile(
- "1:\n"
- "whilelt p0.b, %[inpos], %[inwidth]\n"
- "b.none 2f\n"
- "mov z7.b, #0\n"
- "ld1b z0.b, p0/z, [%[inptr0], %[inpos]]\n"
- "ld1b z1.b, p0/z, [%[inptr1], %[inpos]]\n"
- "ld1b z2.b, p0/z, [%[inptr2], %[inpos]]\n"
- "ld1b z3.b, p0/z, [%[inptr3], %[inpos]]\n"
- "ld1b z4.b, p0/z, [%[inptr4], %[inpos]]\n"
- "ld1b z5.b, p0/z, [%[inptr5], %[inpos]]\n"
- "ld1b z6.b, p0/z, [%[inptr6], %[inpos]]\n"
- "incb %[inpos], all, mul #1\n"
- "zip1 z8.s, z0.s, z4.s\n"
- "whilelt p0.b, %[outpos], %[outwidth]\n"
- "zip2 z9.s, z0.s, z4.s\n"
- "incb %[outpos], all, mul #1\n"
- "zip1 z10.s, z1.s, z5.s\n"
- "zip2 z11.s, z1.s, z5.s\n"
- "zip1 z12.s, z2.s, z6.s\n"
- "whilelt p1.b, %[outpos], %[outwidth]\n"
- "zip2 z13.s, z2.s, z6.s\n"
- "incb %[outpos], all, mul #1\n"
- "zip1 z14.s, z3.s, z7.s\n"
- "zip2 z15.s, z3.s, z7.s\n"
- "zip1 z0.s, z8.s, z12.s\n"
- "whilelt p2.b, %[outpos], %[outwidth]\n"
- "zip2 z1.s, z8.s, z12.s\n"
- "incb %[outpos], all, mul #1\n"
- "zip1 z2.s, z9.s, z13.s\n"
- "zip2 z3.s, z9.s, z13.s\n"
- "zip1 z4.s, z10.s, z14.s\n"
- "whilelt p3.b, %[outpos], %[outwidth]\n"
- "zip2 z5.s, z10.s, z14.s\n"
- "incb %[outpos], all, mul #1\n"
- "zip1 z6.s, z11.s, z15.s\n"
- "zip2 z7.s, z11.s, z15.s\n"
- "zip1 z8.s, z0.s, z4.s\n"
- "whilelt p4.b, %[outpos], %[outwidth]\n"
- "zip2 z9.s, z0.s, z4.s\n"
- "incb %[outpos], all, mul #1\n"
- "zip1 z10.s, z1.s, z5.s\n"
- "st1b z8.b, p0, [%[outptr]]\n"
- "zip2 z11.s, z1.s, z5.s\n"
- "zip1 z12.s, z2.s, z6.s\n"
- "whilelt p5.b, %[outpos], %[outwidth]\n"
- "zip2 z13.s, z2.s, z6.s\n"
- "st1b z9.b, p1, [%[outptr], #1, MUL VL]\n"
- "zip1 z14.s, z3.s, z7.s\n"
- "incb %[outpos], all, mul #1\n"
- "zip2 z15.s, z3.s, z7.s\n"
- "st1b z10.b, p2, [%[outptr], #2, MUL VL]\n"
- "whilelt p6.b, %[outpos], %[outwidth]\n"
- "st1b z11.b, p3, [%[outptr], #3, MUL VL]\n"
- "incb %[outpos], all, mul #1\n"
- "st1b z12.b, p4, [%[outptr], #4, MUL VL]\n"
- "whilelt p7.b, %[outpos], %[outwidth]\n"
- "incb %[outpos], all, mul #1\n"
- "st1b z13.b, p5, [%[outptr], #5, MUL VL]\n"
- "st1b z14.b, p6, [%[outptr], #6, MUL VL]\n"
- "st1b z15.b, p7, [%[outptr], #7, MUL VL]\n"
- "addvl %[outptr], %[outptr], #8\n"
- "b 1b\n"
- "2:\n"
- : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3), [inptr4] "+r" (inptr4), [inptr5] "+r" (inptr5), [inptr6] "+r" (inptr6)
- : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
- : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
- );
- break;
-
- default:
- case 8:
- __asm __volatile(
- "1:\n"
- "whilelt p0.b, %[inpos], %[inwidth]\n"
- "b.none 2f\n"
- "ld1b z0.b, p0/z, [%[inptr0], %[inpos]]\n"
- "ld1b z1.b, p0/z, [%[inptr1], %[inpos]]\n"
- "ld1b z2.b, p0/z, [%[inptr2], %[inpos]]\n"
- "ld1b z3.b, p0/z, [%[inptr3], %[inpos]]\n"
- "ld1b z4.b, p0/z, [%[inptr4], %[inpos]]\n"
- "ld1b z5.b, p0/z, [%[inptr5], %[inpos]]\n"
- "ld1b z6.b, p0/z, [%[inptr6], %[inpos]]\n"
- "ld1b z7.b, p0/z, [%[inptr7], %[inpos]]\n"
- "incb %[inpos], all, mul #1\n"
- "zip1 z8.s, z0.s, z4.s\n"
- "whilelt p0.b, %[outpos], %[outwidth]\n"
- "zip2 z9.s, z0.s, z4.s\n"
- "incb %[outpos], all, mul #1\n"
- "zip1 z10.s, z1.s, z5.s\n"
- "zip2 z11.s, z1.s, z5.s\n"
- "zip1 z12.s, z2.s, z6.s\n"
- "whilelt p1.b, %[outpos], %[outwidth]\n"
- "zip2 z13.s, z2.s, z6.s\n"
- "incb %[outpos], all, mul #1\n"
- "zip1 z14.s, z3.s, z7.s\n"
- "zip2 z15.s, z3.s, z7.s\n"
- "zip1 z0.s, z8.s, z12.s\n"
- "whilelt p2.b, %[outpos], %[outwidth]\n"
- "zip2 z1.s, z8.s, z12.s\n"
- "incb %[outpos], all, mul #1\n"
- "zip1 z2.s, z9.s, z13.s\n"
- "zip2 z3.s, z9.s, z13.s\n"
- "zip1 z4.s, z10.s, z14.s\n"
- "whilelt p3.b, %[outpos], %[outwidth]\n"
- "zip2 z5.s, z10.s, z14.s\n"
- "incb %[outpos], all, mul #1\n"
- "zip1 z6.s, z11.s, z15.s\n"
- "zip2 z7.s, z11.s, z15.s\n"
- "zip1 z8.s, z0.s, z4.s\n"
- "whilelt p4.b, %[outpos], %[outwidth]\n"
- "zip2 z9.s, z0.s, z4.s\n"
- "incb %[outpos], all, mul #1\n"
- "zip1 z10.s, z1.s, z5.s\n"
- "st1b z8.b, p0, [%[outptr]]\n"
- "zip2 z11.s, z1.s, z5.s\n"
- "zip1 z12.s, z2.s, z6.s\n"
- "whilelt p5.b, %[outpos], %[outwidth]\n"
- "zip2 z13.s, z2.s, z6.s\n"
- "st1b z9.b, p1, [%[outptr], #1, MUL VL]\n"
- "zip1 z14.s, z3.s, z7.s\n"
- "incb %[outpos], all, mul #1\n"
- "zip2 z15.s, z3.s, z7.s\n"
- "st1b z10.b, p2, [%[outptr], #2, MUL VL]\n"
- "whilelt p6.b, %[outpos], %[outwidth]\n"
- "st1b z11.b, p3, [%[outptr], #3, MUL VL]\n"
- "incb %[outpos], all, mul #1\n"
- "st1b z12.b, p4, [%[outptr], #4, MUL VL]\n"
- "whilelt p7.b, %[outpos], %[outwidth]\n"
- "incb %[outpos], all, mul #1\n"
- "st1b z13.b, p5, [%[outptr], #5, MUL VL]\n"
- "st1b z14.b, p6, [%[outptr], #6, MUL VL]\n"
- "st1b z15.b, p7, [%[outptr], #7, MUL VL]\n"
- "addvl %[outptr], %[outptr], #8\n"
- "b 1b\n"
- "2:\n"
- : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3), [inptr4] "+r" (inptr4), [inptr5] "+r" (inptr5), [inptr6] "+r" (inptr6), [inptr7] "+r" (inptr7)
- : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
- : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
- );
- break;
-
-
- }
- }
-}
-
-#endif // __ARM_FEATURE_SVE
diff --git a/src/core/NEON/kernels/arm_gemm/transforms/sve_interleave_8way_block8_8bit.hpp b/src/core/NEON/kernels/arm_gemm/transforms/sve_interleave_8way_block8_8bit.hpp
deleted file mode 100644
index b4935e6..0000000
--- a/src/core/NEON/kernels/arm_gemm/transforms/sve_interleave_8way_block8_8bit.hpp
+++ /dev/null
@@ -1,596 +0,0 @@
-/*
- * Copyright (c) 2019 Arm Limited.
- *
- * SPDX-License-Identifier: MIT
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to
- * deal in the Software without restriction, including without limitation the
- * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
- * sell copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-#pragma once
-
-#ifdef __ARM_FEATURE_SVE
-
-template<>
-template<typename T>
-inline void TransformImpl<8, 8, false, 1, 1, false>::Transform(T *out, const T *in, int ldin, int y0, int ymax, int k0, int kmax)
-{
- uint8_t *master_outptr = reinterpret_cast<uint8_t *>(out);
- const uint8_t *inptr = reinterpret_cast<const uint8_t *>(in);
-
- for (int y=y0; y<ymax; y+=8)
- {
- const int height = ymax-y;
- const long inwidth = (kmax - k0);
- const long outwidth = ((inwidth + 7) / 8) * 64;
- long inpos = 0;
- long outpos = 0;
-
- uint8_t *outptr = master_outptr;
- master_outptr += outwidth;
-
- const uint8_t *inptr0 = inptr + y * ldin + k0;
- const uint8_t *inptr1 = inptr0 + ldin;
- const uint8_t *inptr2 = inptr1 + ldin;
- const uint8_t *inptr3 = inptr2 + ldin;
- const uint8_t *inptr4 = inptr3 + ldin;
- const uint8_t *inptr5 = inptr4 + ldin;
- const uint8_t *inptr6 = inptr5 + ldin;
- const uint8_t *inptr7 = inptr6 + ldin;
-
- switch(height)
- {
- case 1:
- __asm __volatile(
- "1:\n"
- "whilelt p0.b, %[inpos], %[inwidth]\n"
- "b.none 2f\n"
- "mov z4.b, #0\n"
- "ld1b z0.b, p0/z, [%[inptr0], %[inpos]]\n"
- "incb %[inpos], all, mul #1\n"
- "whilelt p0.b, %[outpos], %[outwidth]\n"
- "incb %[outpos], all, mul #1\n"
- "zip1 z8.d, z0.d, z4.d\n"
- "zip2 z9.d, z0.d, z4.d\n"
- "whilelt p1.b, %[outpos], %[outwidth]\n"
- "zip1 z0.d, z8.d, z4.d\n"
- "incb %[outpos], all, mul #1\n"
- "zip2 z1.d, z8.d, z4.d\n"
- "zip1 z2.d, z9.d, z4.d\n"
- "zip2 z3.d, z9.d, z4.d\n"
- "whilelt p2.b, %[outpos], %[outwidth]\n"
- "zip1 z8.d, z0.d, z4.d\n"
- "incb %[outpos], all, mul #1\n"
- "zip2 z9.d, z0.d, z4.d\n"
- "zip1 z10.d, z1.d, z4.d\n"
- "st1b z8.b, p0, [%[outptr]]\n"
- "zip2 z11.d, z1.d, z4.d\n"
- "whilelt p3.b, %[outpos], %[outwidth]\n"
- "zip1 z12.d, z2.d, z4.d\n"
- "st1b z9.b, p1, [%[outptr], #1, MUL VL]\n"
- "zip2 z13.d, z2.d, z4.d\n"
- "incb %[outpos], all, mul #1\n"
- "zip1 z14.d, z3.d, z4.d\n"
- "st1b z10.b, p2, [%[outptr], #2, MUL VL]\n"
- "zip2 z15.d, z3.d, z4.d\n"
- "whilelt p4.b, %[outpos], %[outwidth]\n"
- "st1b z11.b, p3, [%[outptr], #3, MUL VL]\n"
- "incb %[outpos], all, mul #1\n"
- "st1b z12.b, p4, [%[outptr], #4, MUL VL]\n"
- "whilelt p5.b, %[outpos], %[outwidth]\n"
- "incb %[outpos], all, mul #1\n"
- "st1b z13.b, p5, [%[outptr], #5, MUL VL]\n"
- "whilelt p6.b, %[outpos], %[outwidth]\n"
- "incb %[outpos], all, mul #1\n"
- "st1b z14.b, p6, [%[outptr], #6, MUL VL]\n"
- "whilelt p7.b, %[outpos], %[outwidth]\n"
- "incb %[outpos], all, mul #1\n"
- "st1b z15.b, p7, [%[outptr], #7, MUL VL]\n"
- "addvl %[outptr], %[outptr], #8\n"
- "b 1b\n"
- "2:\n"
- : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0)
- : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
- : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
- );
- break;
-
- case 2:
- __asm __volatile(
- "1:\n"
- "whilelt p0.b, %[inpos], %[inwidth]\n"
- "b.none 2f\n"
- "mov z4.b, #0\n"
- "mov z14.b, #0\n"
- "ld1b z0.b, p0/z, [%[inptr0], %[inpos]]\n"
- "ld1b z1.b, p0/z, [%[inptr1], %[inpos]]\n"
- "incb %[inpos], all, mul #1\n"
- "whilelt p0.b, %[outpos], %[outwidth]\n"
- "zip1 z8.d, z0.d, z4.d\n"
- "incb %[outpos], all, mul #1\n"
- "zip2 z9.d, z0.d, z4.d\n"
- "zip1 z10.d, z1.d, z4.d\n"
- "zip2 z11.d, z1.d, z4.d\n"
- "whilelt p1.b, %[outpos], %[outwidth]\n"
- "zip1 z0.d, z8.d, z4.d\n"
- "incb %[outpos], all, mul #1\n"
- "zip2 z1.d, z8.d, z4.d\n"
- "zip1 z2.d, z9.d, z4.d\n"
- "zip2 z3.d, z9.d, z4.d\n"
- "whilelt p2.b, %[outpos], %[outwidth]\n"
- "zip1 z4.d, z10.d, z14.d\n"
- "incb %[outpos], all, mul #1\n"
- "zip2 z5.d, z10.d, z14.d\n"
- "zip1 z6.d, z11.d, z14.d\n"
- "zip2 z7.d, z11.d, z14.d\n"
- "whilelt p3.b, %[outpos], %[outwidth]\n"
- "zip1 z8.d, z0.d, z4.d\n"
- "incb %[outpos], all, mul #1\n"
- "zip2 z9.d, z0.d, z4.d\n"
- "zip1 z10.d, z1.d, z5.d\n"
- "st1b z8.b, p0, [%[outptr]]\n"
- "zip2 z11.d, z1.d, z5.d\n"
- "whilelt p4.b, %[outpos], %[outwidth]\n"
- "zip1 z12.d, z2.d, z6.d\n"
- "st1b z9.b, p1, [%[outptr], #1, MUL VL]\n"
- "zip2 z13.d, z2.d, z6.d\n"
- "incb %[outpos], all, mul #1\n"
- "zip1 z14.d, z3.d, z7.d\n"
- "st1b z10.b, p2, [%[outptr], #2, MUL VL]\n"
- "zip2 z15.d, z3.d, z7.d\n"
- "whilelt p5.b, %[outpos], %[outwidth]\n"
- "st1b z11.b, p3, [%[outptr], #3, MUL VL]\n"
- "incb %[outpos], all, mul #1\n"
- "st1b z12.b, p4, [%[outptr], #4, MUL VL]\n"
- "whilelt p6.b, %[outpos], %[outwidth]\n"
- "incb %[outpos], all, mul #1\n"
- "st1b z13.b, p5, [%[outptr], #5, MUL VL]\n"
- "whilelt p7.b, %[outpos], %[outwidth]\n"
- "incb %[outpos], all, mul #1\n"
- "st1b z14.b, p6, [%[outptr], #6, MUL VL]\n"
- "st1b z15.b, p7, [%[outptr], #7, MUL VL]\n"
- "addvl %[outptr], %[outptr], #8\n"
- "b 1b\n"
- "2:\n"
- : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1)
- : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
- : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
- );
- break;
-
- case 3:
- __asm __volatile(
- "1:\n"
- "whilelt p0.b, %[inpos], %[inwidth]\n"
- "b.none 2f\n"
- "mov z4.b, #0\n"
- "mov z14.b, #0\n"
- "ld1b z0.b, p0/z, [%[inptr0], %[inpos]]\n"
- "ld1b z1.b, p0/z, [%[inptr1], %[inpos]]\n"
- "ld1b z2.b, p0/z, [%[inptr2], %[inpos]]\n"
- "incb %[inpos], all, mul #1\n"
- "zip1 z8.d, z0.d, z4.d\n"
- "whilelt p0.b, %[outpos], %[outwidth]\n"
- "zip2 z9.d, z0.d, z4.d\n"
- "incb %[outpos], all, mul #1\n"
- "zip1 z10.d, z1.d, z4.d\n"
- "zip2 z11.d, z1.d, z4.d\n"
- "zip1 z12.d, z2.d, z4.d\n"
- "whilelt p1.b, %[outpos], %[outwidth]\n"
- "zip2 z13.d, z2.d, z4.d\n"
- "incb %[outpos], all, mul #1\n"
- "zip1 z0.d, z8.d, z12.d\n"
- "zip2 z1.d, z8.d, z12.d\n"
- "zip1 z2.d, z9.d, z13.d\n"
- "whilelt p2.b, %[outpos], %[outwidth]\n"
- "zip2 z3.d, z9.d, z13.d\n"
- "incb %[outpos], all, mul #1\n"
- "zip1 z4.d, z10.d, z14.d\n"
- "zip2 z5.d, z10.d, z14.d\n"
- "zip1 z6.d, z11.d, z14.d\n"
- "whilelt p3.b, %[outpos], %[outwidth]\n"
- "zip2 z7.d, z11.d, z14.d\n"
- "incb %[outpos], all, mul #1\n"
- "zip1 z8.d, z0.d, z4.d\n"
- "zip2 z9.d, z0.d, z4.d\n"
- "zip1 z10.d, z1.d, z5.d\n"
- "whilelt p4.b, %[outpos], %[outwidth]\n"
- "zip2 z11.d, z1.d, z5.d\n"
- "st1b z8.b, p0, [%[outptr]]\n"
- "zip1 z12.d, z2.d, z6.d\n"
- "incb %[outpos], all, mul #1\n"
- "zip2 z13.d, z2.d, z6.d\n"
- "st1b z9.b, p1, [%[outptr], #1, MUL VL]\n"
- "zip1 z14.d, z3.d, z7.d\n"
- "zip2 z15.d, z3.d, z7.d\n"
- "whilelt p5.b, %[outpos], %[outwidth]\n"
- "st1b z10.b, p2, [%[outptr], #2, MUL VL]\n"
- "incb %[outpos], all, mul #1\n"
- "st1b z11.b, p3, [%[outptr], #3, MUL VL]\n"
- "whilelt p6.b, %[outpos], %[outwidth]\n"
- "incb %[outpos], all, mul #1\n"
- "st1b z12.b, p4, [%[outptr], #4, MUL VL]\n"
- "whilelt p7.b, %[outpos], %[outwidth]\n"
- "incb %[outpos], all, mul #1\n"
- "st1b z13.b, p5, [%[outptr], #5, MUL VL]\n"
- "st1b z14.b, p6, [%[outptr], #6, MUL VL]\n"
- "st1b z15.b, p7, [%[outptr], #7, MUL VL]\n"
- "addvl %[outptr], %[outptr], #8\n"
- "b 1b\n"
- "2:\n"
- : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2)
- : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
- : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
- );
- break;
-
- case 4:
- __asm __volatile(
- "1:\n"
- "whilelt p0.b, %[inpos], %[inwidth]\n"
- "b.none 2f\n"
- "mov z4.b, #0\n"
- "ld1b z0.b, p0/z, [%[inptr0], %[inpos]]\n"
- "ld1b z1.b, p0/z, [%[inptr1], %[inpos]]\n"
- "ld1b z2.b, p0/z, [%[inptr2], %[inpos]]\n"
- "ld1b z3.b, p0/z, [%[inptr3], %[inpos]]\n"
- "incb %[inpos], all, mul #1\n"
- "zip1 z8.d, z0.d, z4.d\n"
- "whilelt p0.b, %[outpos], %[outwidth]\n"
- "zip2 z9.d, z0.d, z4.d\n"
- "incb %[outpos], all, mul #1\n"
- "zip1 z10.d, z1.d, z4.d\n"
- "zip2 z11.d, z1.d, z4.d\n"
- "zip1 z12.d, z2.d, z4.d\n"
- "whilelt p1.b, %[outpos], %[outwidth]\n"
- "zip2 z13.d, z2.d, z4.d\n"
- "incb %[outpos], all, mul #1\n"
- "zip1 z14.d, z3.d, z4.d\n"
- "zip2 z15.d, z3.d, z4.d\n"
- "zip1 z0.d, z8.d, z12.d\n"
- "whilelt p2.b, %[outpos], %[outwidth]\n"
- "zip2 z1.d, z8.d, z12.d\n"
- "incb %[outpos], all, mul #1\n"
- "zip1 z2.d, z9.d, z13.d\n"
- "zip2 z3.d, z9.d, z13.d\n"
- "zip1 z4.d, z10.d, z14.d\n"
- "whilelt p3.b, %[outpos], %[outwidth]\n"
- "zip2 z5.d, z10.d, z14.d\n"
- "incb %[outpos], all, mul #1\n"
- "zip1 z6.d, z11.d, z15.d\n"
- "zip2 z7.d, z11.d, z15.d\n"
- "zip1 z8.d, z0.d, z4.d\n"
- "whilelt p4.b, %[outpos], %[outwidth]\n"
- "zip2 z9.d, z0.d, z4.d\n"
- "incb %[outpos], all, mul #1\n"
- "zip1 z10.d, z1.d, z5.d\n"
- "st1b z8.b, p0, [%[outptr]]\n"
- "zip2 z11.d, z1.d, z5.d\n"
- "zip1 z12.d, z2.d, z6.d\n"
- "whilelt p5.b, %[outpos], %[outwidth]\n"
- "zip2 z13.d, z2.d, z6.d\n"
- "st1b z9.b, p1, [%[outptr], #1, MUL VL]\n"
- "zip1 z14.d, z3.d, z7.d\n"
- "incb %[outpos], all, mul #1\n"
- "zip2 z15.d, z3.d, z7.d\n"
- "st1b z10.b, p2, [%[outptr], #2, MUL VL]\n"
- "whilelt p6.b, %[outpos], %[outwidth]\n"
- "st1b z11.b, p3, [%[outptr], #3, MUL VL]\n"
- "incb %[outpos], all, mul #1\n"
- "st1b z12.b, p4, [%[outptr], #4, MUL VL]\n"
- "whilelt p7.b, %[outpos], %[outwidth]\n"
- "incb %[outpos], all, mul #1\n"
- "st1b z13.b, p5, [%[outptr], #5, MUL VL]\n"
- "st1b z14.b, p6, [%[outptr], #6, MUL VL]\n"
- "st1b z15.b, p7, [%[outptr], #7, MUL VL]\n"
- "addvl %[outptr], %[outptr], #8\n"
- "b 1b\n"
- "2:\n"
- : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3)
- : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
- : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
- );
- break;
-
- case 5:
- __asm __volatile(
- "1:\n"
- "whilelt p0.b, %[inpos], %[inwidth]\n"
- "b.none 2f\n"
- "mov z5.b, #0\n"
- "ld1b z0.b, p0/z, [%[inptr0], %[inpos]]\n"
- "ld1b z1.b, p0/z, [%[inptr1], %[inpos]]\n"
- "ld1b z2.b, p0/z, [%[inptr2], %[inpos]]\n"
- "ld1b z3.b, p0/z, [%[inptr3], %[inpos]]\n"
- "ld1b z4.b, p0/z, [%[inptr4], %[inpos]]\n"
- "incb %[inpos], all, mul #1\n"
- "zip1 z10.d, z1.d, z5.d\n"
- "whilelt p0.b, %[outpos], %[outwidth]\n"
- "zip1 z8.d, z0.d, z4.d\n"
- "incb %[outpos], all, mul #1\n"
- "zip2 z9.d, z0.d, z4.d\n"
- "zip2 z11.d, z1.d, z5.d\n"
- "zip1 z12.d, z2.d, z5.d\n"
- "whilelt p1.b, %[outpos], %[outwidth]\n"
- "zip2 z13.d, z2.d, z5.d\n"
- "incb %[outpos], all, mul #1\n"
- "zip1 z14.d, z3.d, z5.d\n"
- "zip2 z15.d, z3.d, z5.d\n"
- "zip1 z0.d, z8.d, z12.d\n"
- "whilelt p2.b, %[outpos], %[outwidth]\n"
- "zip2 z1.d, z8.d, z12.d\n"
- "incb %[outpos], all, mul #1\n"
- "zip1 z2.d, z9.d, z13.d\n"
- "zip2 z3.d, z9.d, z13.d\n"
- "zip1 z4.d, z10.d, z14.d\n"
- "whilelt p3.b, %[outpos], %[outwidth]\n"
- "zip2 z5.d, z10.d, z14.d\n"
- "incb %[outpos], all, mul #1\n"
- "zip1 z6.d, z11.d, z15.d\n"
- "zip2 z7.d, z11.d, z15.d\n"
- "zip1 z8.d, z0.d, z4.d\n"
- "whilelt p4.b, %[outpos], %[outwidth]\n"
- "zip2 z9.d, z0.d, z4.d\n"
- "incb %[outpos], all, mul #1\n"
- "zip1 z10.d, z1.d, z5.d\n"
- "st1b z8.b, p0, [%[outptr]]\n"
- "zip2 z11.d, z1.d, z5.d\n"
- "zip1 z12.d, z2.d, z6.d\n"
- "whilelt p5.b, %[outpos], %[outwidth]\n"
- "zip2 z13.d, z2.d, z6.d\n"
- "st1b z9.b, p1, [%[outptr], #1, MUL VL]\n"
- "zip1 z14.d, z3.d, z7.d\n"
- "incb %[outpos], all, mul #1\n"
- "zip2 z15.d, z3.d, z7.d\n"
- "st1b z10.b, p2, [%[outptr], #2, MUL VL]\n"
- "whilelt p6.b, %[outpos], %[outwidth]\n"
- "st1b z11.b, p3, [%[outptr], #3, MUL VL]\n"
- "incb %[outpos], all, mul #1\n"
- "st1b z12.b, p4, [%[outptr], #4, MUL VL]\n"
- "whilelt p7.b, %[outpos], %[outwidth]\n"
- "incb %[outpos], all, mul #1\n"
- "st1b z13.b, p5, [%[outptr], #5, MUL VL]\n"
- "st1b z14.b, p6, [%[outptr], #6, MUL VL]\n"
- "st1b z15.b, p7, [%[outptr], #7, MUL VL]\n"
- "addvl %[outptr], %[outptr], #8\n"
- "b 1b\n"
- "2:\n"
- : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3), [inptr4] "+r" (inptr4)
- : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
- : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
- );
- break;
-
- case 6:
- __asm __volatile(
- "1:\n"
- "whilelt p0.b, %[inpos], %[inwidth]\n"
- "b.none 2f\n"
- "mov z6.b, #0\n"
- "ld1b z0.b, p0/z, [%[inptr0], %[inpos]]\n"
- "ld1b z1.b, p0/z, [%[inptr1], %[inpos]]\n"
- "ld1b z2.b, p0/z, [%[inptr2], %[inpos]]\n"
- "ld1b z3.b, p0/z, [%[inptr3], %[inpos]]\n"
- "ld1b z4.b, p0/z, [%[inptr4], %[inpos]]\n"
- "ld1b z5.b, p0/z, [%[inptr5], %[inpos]]\n"
- "incb %[inpos], all, mul #1\n"
- "zip1 z12.d, z2.d, z6.d\n"
- "whilelt p0.b, %[outpos], %[outwidth]\n"
- "zip1 z8.d, z0.d, z4.d\n"
- "incb %[outpos], all, mul #1\n"
- "zip2 z9.d, z0.d, z4.d\n"
- "zip1 z10.d, z1.d, z5.d\n"
- "zip2 z11.d, z1.d, z5.d\n"
- "whilelt p1.b, %[outpos], %[outwidth]\n"
- "zip2 z13.d, z2.d, z6.d\n"
- "incb %[outpos], all, mul #1\n"
- "zip1 z14.d, z3.d, z6.d\n"
- "zip2 z15.d, z3.d, z6.d\n"
- "zip1 z0.d, z8.d, z12.d\n"
- "whilelt p2.b, %[outpos], %[outwidth]\n"
- "zip2 z1.d, z8.d, z12.d\n"
- "incb %[outpos], all, mul #1\n"
- "zip1 z2.d, z9.d, z13.d\n"
- "zip2 z3.d, z9.d, z13.d\n"
- "zip1 z4.d, z10.d, z14.d\n"
- "whilelt p3.b, %[outpos], %[outwidth]\n"
- "zip2 z5.d, z10.d, z14.d\n"
- "incb %[outpos], all, mul #1\n"
- "zip1 z6.d, z11.d, z15.d\n"
- "zip2 z7.d, z11.d, z15.d\n"
- "zip1 z8.d, z0.d, z4.d\n"
- "whilelt p4.b, %[outpos], %[outwidth]\n"
- "zip2 z9.d, z0.d, z4.d\n"
- "incb %[outpos], all, mul #1\n"
- "zip1 z10.d, z1.d, z5.d\n"
- "st1b z8.b, p0, [%[outptr]]\n"
- "zip2 z11.d, z1.d, z5.d\n"
- "zip1 z12.d, z2.d, z6.d\n"
- "whilelt p5.b, %[outpos], %[outwidth]\n"
- "zip2 z13.d, z2.d, z6.d\n"
- "st1b z9.b, p1, [%[outptr], #1, MUL VL]\n"
- "zip1 z14.d, z3.d, z7.d\n"
- "incb %[outpos], all, mul #1\n"
- "zip2 z15.d, z3.d, z7.d\n"
- "st1b z10.b, p2, [%[outptr], #2, MUL VL]\n"
- "whilelt p6.b, %[outpos], %[outwidth]\n"
- "st1b z11.b, p3, [%[outptr], #3, MUL VL]\n"
- "incb %[outpos], all, mul #1\n"
- "st1b z12.b, p4, [%[outptr], #4, MUL VL]\n"
- "whilelt p7.b, %[outpos], %[outwidth]\n"
- "incb %[outpos], all, mul #1\n"
- "st1b z13.b, p5, [%[outptr], #5, MUL VL]\n"
- "st1b z14.b, p6, [%[outptr], #6, MUL VL]\n"
- "st1b z15.b, p7, [%[outptr], #7, MUL VL]\n"
- "addvl %[outptr], %[outptr], #8\n"
- "b 1b\n"
- "2:\n"
- : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3), [inptr4] "+r" (inptr4), [inptr5] "+r" (inptr5)
- : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
- : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
- );
- break;
-
- case 7:
- __asm __volatile(
- "1:\n"
- "whilelt p0.b, %[inpos], %[inwidth]\n"
- "b.none 2f\n"
- "mov z7.b, #0\n"
- "ld1b z0.b, p0/z, [%[inptr0], %[inpos]]\n"
- "ld1b z1.b, p0/z, [%[inptr1], %[inpos]]\n"
- "ld1b z2.b, p0/z, [%[inptr2], %[inpos]]\n"
- "ld1b z3.b, p0/z, [%[inptr3], %[inpos]]\n"
- "ld1b z4.b, p0/z, [%[inptr4], %[inpos]]\n"
- "ld1b z5.b, p0/z, [%[inptr5], %[inpos]]\n"
- "ld1b z6.b, p0/z, [%[inptr6], %[inpos]]\n"
- "incb %[inpos], all, mul #1\n"
- "zip1 z8.d, z0.d, z4.d\n"
- "whilelt p0.b, %[outpos], %[outwidth]\n"
- "zip2 z9.d, z0.d, z4.d\n"
- "incb %[outpos], all, mul #1\n"
- "zip1 z10.d, z1.d, z5.d\n"
- "zip2 z11.d, z1.d, z5.d\n"
- "zip1 z12.d, z2.d, z6.d\n"
- "whilelt p1.b, %[outpos], %[outwidth]\n"
- "zip2 z13.d, z2.d, z6.d\n"
- "incb %[outpos], all, mul #1\n"
- "zip1 z14.d, z3.d, z7.d\n"
- "zip2 z15.d, z3.d, z7.d\n"
- "zip1 z0.d, z8.d, z12.d\n"
- "whilelt p2.b, %[outpos], %[outwidth]\n"
- "zip2 z1.d, z8.d, z12.d\n"
- "incb %[outpos], all, mul #1\n"
- "zip1 z2.d, z9.d, z13.d\n"
- "zip2 z3.d, z9.d, z13.d\n"
- "zip1 z4.d, z10.d, z14.d\n"
- "whilelt p3.b, %[outpos], %[outwidth]\n"
- "zip2 z5.d, z10.d, z14.d\n"
- "incb %[outpos], all, mul #1\n"
- "zip1 z6.d, z11.d, z15.d\n"
- "zip2 z7.d, z11.d, z15.d\n"
- "zip1 z8.d, z0.d, z4.d\n"
- "whilelt p4.b, %[outpos], %[outwidth]\n"
- "zip2 z9.d, z0.d, z4.d\n"
- "incb %[outpos], all, mul #1\n"
- "zip1 z10.d, z1.d, z5.d\n"
- "st1b z8.b, p0, [%[outptr]]\n"
- "zip2 z11.d, z1.d, z5.d\n"
- "zip1 z12.d, z2.d, z6.d\n"
- "whilelt p5.b, %[outpos], %[outwidth]\n"
- "zip2 z13.d, z2.d, z6.d\n"
- "st1b z9.b, p1, [%[outptr], #1, MUL VL]\n"
- "zip1 z14.d, z3.d, z7.d\n"
- "incb %[outpos], all, mul #1\n"
- "zip2 z15.d, z3.d, z7.d\n"
- "st1b z10.b, p2, [%[outptr], #2, MUL VL]\n"
- "whilelt p6.b, %[outpos], %[outwidth]\n"
- "st1b z11.b, p3, [%[outptr], #3, MUL VL]\n"
- "incb %[outpos], all, mul #1\n"
- "st1b z12.b, p4, [%[outptr], #4, MUL VL]\n"
- "whilelt p7.b, %[outpos], %[outwidth]\n"
- "incb %[outpos], all, mul #1\n"
- "st1b z13.b, p5, [%[outptr], #5, MUL VL]\n"
- "st1b z14.b, p6, [%[outptr], #6, MUL VL]\n"
- "st1b z15.b, p7, [%[outptr], #7, MUL VL]\n"
- "addvl %[outptr], %[outptr], #8\n"
- "b 1b\n"
- "2:\n"
- : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3), [inptr4] "+r" (inptr4), [inptr5] "+r" (inptr5), [inptr6] "+r" (inptr6)
- : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
- : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
- );
- break;
-
- default:
- case 8:
- __asm __volatile(
- "1:\n"
- "whilelt p0.b, %[inpos], %[inwidth]\n"
- "b.none 2f\n"
- "ld1b z0.b, p0/z, [%[inptr0], %[inpos]]\n"
- "ld1b z1.b, p0/z, [%[inptr1], %[inpos]]\n"
- "ld1b z2.b, p0/z, [%[inptr2], %[inpos]]\n"
- "ld1b z3.b, p0/z, [%[inptr3], %[inpos]]\n"
- "ld1b z4.b, p0/z, [%[inptr4], %[inpos]]\n"
- "ld1b z5.b, p0/z, [%[inptr5], %[inpos]]\n"
- "ld1b z6.b, p0/z, [%[inptr6], %[inpos]]\n"
- "ld1b z7.b, p0/z, [%[inptr7], %[inpos]]\n"
- "incb %[inpos], all, mul #1\n"
- "zip1 z8.d, z0.d, z4.d\n"
- "whilelt p0.b, %[outpos], %[outwidth]\n"
- "zip2 z9.d, z0.d, z4.d\n"
- "incb %[outpos], all, mul #1\n"
- "zip1 z10.d, z1.d, z5.d\n"
- "zip2 z11.d, z1.d, z5.d\n"
- "zip1 z12.d, z2.d, z6.d\n"
- "whilelt p1.b, %[outpos], %[outwidth]\n"
- "zip2 z13.d, z2.d, z6.d\n"
- "incb %[outpos], all, mul #1\n"
- "zip1 z14.d, z3.d, z7.d\n"
- "zip2 z15.d, z3.d, z7.d\n"
- "zip1 z0.d, z8.d, z12.d\n"
- "whilelt p2.b, %[outpos], %[outwidth]\n"
- "zip2 z1.d, z8.d, z12.d\n"
- "incb %[outpos], all, mul #1\n"
- "zip1 z2.d, z9.d, z13.d\n"
- "zip2 z3.d, z9.d, z13.d\n"
- "zip1 z4.d, z10.d, z14.d\n"
- "whilelt p3.b, %[outpos], %[outwidth]\n"
- "zip2 z5.d, z10.d, z14.d\n"
- "incb %[outpos], all, mul #1\n"
- "zip1 z6.d, z11.d, z15.d\n"
- "zip2 z7.d, z11.d, z15.d\n"
- "zip1 z8.d, z0.d, z4.d\n"
- "whilelt p4.b, %[outpos], %[outwidth]\n"
- "zip2 z9.d, z0.d, z4.d\n"
- "incb %[outpos], all, mul #1\n"
- "zip1 z10.d, z1.d, z5.d\n"
- "st1b z8.b, p0, [%[outptr]]\n"
- "zip2 z11.d, z1.d, z5.d\n"
- "zip1 z12.d, z2.d, z6.d\n"
- "whilelt p5.b, %[outpos], %[outwidth]\n"
- "zip2 z13.d, z2.d, z6.d\n"
- "st1b z9.b, p1, [%[outptr], #1, MUL VL]\n"
- "zip1 z14.d, z3.d, z7.d\n"
- "incb %[outpos], all, mul #1\n"
- "zip2 z15.d, z3.d, z7.d\n"
- "st1b z10.b, p2, [%[outptr], #2, MUL VL]\n"
- "whilelt p6.b, %[outpos], %[outwidth]\n"
- "st1b z11.b, p3, [%[outptr], #3, MUL VL]\n"
- "incb %[outpos], all, mul #1\n"
- "st1b z12.b, p4, [%[outptr], #4, MUL VL]\n"
- "whilelt p7.b, %[outpos], %[outwidth]\n"
- "incb %[outpos], all, mul #1\n"
- "st1b z13.b, p5, [%[outptr], #5, MUL VL]\n"
- "st1b z14.b, p6, [%[outptr], #6, MUL VL]\n"
- "st1b z15.b, p7, [%[outptr], #7, MUL VL]\n"
- "addvl %[outptr], %[outptr], #8\n"
- "b 1b\n"
- "2:\n"
- : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3), [inptr4] "+r" (inptr4), [inptr5] "+r" (inptr5), [inptr6] "+r" (inptr6), [inptr7] "+r" (inptr7)
- : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
- : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
- );
- break;
-
-
- }
- }
-}
-
-#endif // __ARM_FEATURE_SVE
diff --git a/src/core/NEON/kernels/arm_gemm/transforms/transpose_interleave_common.hpp b/src/core/NEON/kernels/arm_gemm/transforms/transpose_interleave_common.hpp
index aac5e19..a3216c4 100644
--- a/src/core/NEON/kernels/arm_gemm/transforms/transpose_interleave_common.hpp
+++ b/src/core/NEON/kernels/arm_gemm/transforms/transpose_interleave_common.hpp
@@ -23,6 +23,8 @@
*/
#pragma once
+#include "../asmlib.hpp"
+
template <unsigned int IntBy, typename TIn, typename TOut>
struct TransposeInterleaveCommon {
// Override the moveblock_1xY methods to improve performance
diff --git a/src/core/NEON/kernels/arm_gemm/utils.hpp b/src/core/NEON/kernels/arm_gemm/utils.hpp
index 6e47a97..6d483a3 100644
--- a/src/core/NEON/kernels/arm_gemm/utils.hpp
+++ b/src/core/NEON/kernels/arm_gemm/utils.hpp
@@ -24,6 +24,8 @@
#pragma once
+#include "arm_gemm.hpp"
+
#include <cstddef>
// Macro for unreachable code (e.g. impossible default cases on switch)
@@ -32,6 +34,8 @@
// Paranoid option for the above with assert
// #define UNREACHABLE(why) assert(0 && why)
+namespace arm_gemm {
+
template<typename T>
inline T iceildiv(const T a, const T b) {
return (a + b - 1) / b;
@@ -48,7 +52,94 @@
}
}
-namespace arm_gemm {
+enum class VLType {
+ None,
+ SVE,
+};
+
+template<typename T>
+struct IndirectOutputArg {
+ struct {
+ T *base;
+ size_t stride;
+ } direct = {};
+ struct {
+ T * const *ptr;
+ size_t offset;
+ } indirect = {};
+ bool is_indirect;
+
+ // Direct
+ IndirectOutputArg(T *base, size_t stride) : is_indirect(false) {
+ direct.base = base;
+ direct.stride = stride;
+ }
+
+ // Indirect
+ IndirectOutputArg(T * const * ptr, size_t offset) : is_indirect(true) {
+ indirect.ptr = ptr;
+ indirect.offset = offset;
+ }
+
+ IndirectOutputArg() : is_indirect(false) {
+ direct.base = nullptr;
+ direct.stride = 0;
+ }
+};
+
+// Check that the provided Requantize32 doesn't have a left shift.
+inline bool quant_no_left_shift(const Requantize32 &qp) {
+ if (qp.per_channel_requant) {
+ return (qp.per_channel_left_shifts == nullptr);
+ } else {
+ return (qp.per_layer_left_shift == 0);
+ }
+}
+
+// Check that the provided Requantize32 is compatible with the "symmetric" hybrid kernels. These don't include row
+// sums, so the 'b_offset' has to be zero.
+inline bool quant_hybrid_symmetric(const Requantize32 &qp) {
+ return quant_no_left_shift(qp) && qp.b_offset == 0;
+}
+
+// Check that the provided Requantize32 is compatible with the "asymmetric" hybrid kernels. These don't support per
+// channel quantization. Technically b_offset==0 cases would work, but it is a waste to sum and then multiply by 0...
+inline bool quant_hybrid_asymmetric(const Requantize32 &qp) {
+ return quant_no_left_shift(qp) /* && qp.b_offset != 0 */ && qp.per_channel_requant==false;
+}
+
+template<typename T>
+struct IndirectInputArg {
+ struct {
+ const T *base;
+ size_t stride;
+ } direct = {};
+ struct {
+ const T * const * const * ptr;
+ unsigned int start_row;
+ unsigned int start_col;
+ } indirect = {};
+ bool is_indirect;
+
+ // Direct
+ IndirectInputArg(const T *base, size_t stride) : is_indirect(false) {
+ direct.base = base;
+ direct.stride = stride;
+ }
+
+ // Indirect
+ IndirectInputArg(const T * const * const *ptr, unsigned int start_row, unsigned int start_col) : is_indirect(true) {
+ indirect.ptr = ptr;
+ indirect.start_row = start_row;
+ indirect.start_col = start_col;
+ }
+
+ IndirectInputArg() : is_indirect(false) {
+ direct.base = nullptr;
+ direct.stride = 0;
+ }
+};
+
namespace utils {
namespace {
diff --git a/src/core/NEON/kernels/assembly/INEGEMMWrapperKernel.cpp b/src/core/NEON/kernels/assembly/INEGEMMWrapperKernel.cpp
deleted file mode 100644
index 760274d..0000000
--- a/src/core/NEON/kernels/assembly/INEGEMMWrapperKernel.cpp
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Copyright (c) 2018-2020 Arm Limited.
- *
- * SPDX-License-Identifier: MIT
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to
- * deal in the Software without restriction, including without limitation the
- * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
- * sell copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-#include "src/core/NEON/kernels/assembly/INEGEMMWrapperKernel.h"
-
-#include "arm_compute/core/Error.h"
-#include "arm_compute/core/Helpers.h"
-#include "arm_compute/core/ITensor.h"
-#include "arm_compute/core/Validate.h"
-#include "arm_compute/core/WindowIterator.h"
-
-using namespace arm_compute;
-
-INEGEMMWrapperKernel::INEGEMMWrapperKernel()
- : _a(nullptr), _b(nullptr), _c(nullptr), _params(), _gemm_info(), _window3d(), _window_shape()
-{
-}
-
-INEGEMMWrapperKernel::Params INEGEMMWrapperKernel::extract_parameters(const ITensor *a, const ITensor *b, const ITensor *c, const GEMMInfo &gemm_info)
-{
- Params p;
-
- ARM_COMPUTE_ERROR_ON_NULLPTR(a);
- ARM_COMPUTE_ERROR_ON_NULLPTR(b);
- ARM_COMPUTE_ERROR_ON_NULLPTR(c);
-
- // Initalize params
- p.M = c->info()->tensor_shape().y();
- p.N = c->info()->tensor_shape().x();
- p.K = a->info()->tensor_shape().x();
- p.multis = b->info()->tensor_shape().z();
- p.batches = c->info()->tensor_shape().total_size_upper(2) / p.multis; //COMPMID-1423: Agree on and document the layout of gemm inputs/outputs
-
- // Update M in case of GEMM3D for output
- if(gemm_info.depth_output_gemm3d() != 0)
- {
- p.M = c->info()->tensor_shape().y() * c->info()->tensor_shape().z();
- p.batches = c->info()->tensor_shape().total_size_upper(3) / p.multis;
- }
-
- return p;
-}
-
-void INEGEMMWrapperKernel::configure(const ITensor *a, const ITensor *b, ITensor *c, float alpha, float beta, const GEMMInfo &gemm_info)
-{
- _gemm_info = gemm_info;
- _params = extract_parameters(a, b, c, gemm_info);
- _a = a;
- _b = b;
- _c = c;
-
- _window3d = configure_internal(alpha, beta);
- _window_shape = _window3d.shape();
-
- // Convert the 3D window into a 1D window in order to allow the scheduler to arbitrary split it.
- Window collapsed;
- collapsed.set(0, Window::Dimension(0, _window3d.num_iterations_total()));
-
- INEKernel::configure(collapsed);
-}
-
-void INEGEMMWrapperKernel::run(const Window &window, const ThreadInfo &info)
-{
- const Coordinates start_offset = index2coords(_window_shape, window.x().start());
- const Coordinates end_offset = index2coords(_window_shape, window.x().end() - 1);
-
- run_internal(_window3d, start_offset, end_offset, info);
-}
diff --git a/src/core/NEON/kernels/assembly/INEGEMMWrapperKernel.h b/src/core/NEON/kernels/assembly/INEGEMMWrapperKernel.h
deleted file mode 100644
index 92c0132..0000000
--- a/src/core/NEON/kernels/assembly/INEGEMMWrapperKernel.h
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
- * Copyright (c) 2018-2020 Arm Limited.
- *
- * SPDX-License-Identifier: MIT
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to
- * deal in the Software without restriction, including without limitation the
- * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
- * sell copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-#ifndef SRC_INEGEMMWRAPPERKERNEL_H
-#define SRC_INEGEMMWRAPPERKERNEL_H
-
-#include "src/core/NEON/INEKernel.h"
-
-namespace arm_compute
-{
-class ITensor;
-
-/** Common interface for all the arm_gemm Gemms
- */
-class INEGEMMWrapperKernel : public INEKernel
-{
-public:
- /** Parameters defining the dimensions of the matrices being multiplied */
- struct Params
- {
- unsigned int M{ 0 }; /**< Rows in output matrix C (and input matrix A). */
- unsigned int N{ 0 }; /**< Columns in output matrix C (and input matrix B). */
- unsigned int K{ 0 }; /**< Columns of input matrix A (= rows of input matrix B). */
- unsigned int batches{ 0 }; /**< Number of "batched" GEMMs (unique A and C, shared B). */
- unsigned int multis{ 0 }; /**< Number of "multi" GEMMs (unique A, B and C). */
- };
-
- static Params extract_parameters(const ITensor *a, const ITensor *b, const ITensor *c, const GEMMInfo &gemm_info);
-
- /** Constructor */
- INEGEMMWrapperKernel();
- /** Prevent instances of this class from being copied */
- INEGEMMWrapperKernel(const INEGEMMWrapperKernel &) = delete;
- /** Prevent instances of this class from being copied */
- INEGEMMWrapperKernel &operator=(const INEGEMMWrapperKernel &) = delete;
- /** Allow instances of this class to be moved */
- INEGEMMWrapperKernel(INEGEMMWrapperKernel &&) = default;
- /** Allow instances of this class to be moved */
- INEGEMMWrapperKernel &operator=(INEGEMMWrapperKernel &&) = default;
- /** Initialise the kernel's input and output.
- *
- * @note The input and output tensor must have the same dimensions
- *
- * @param[in] a Input tensor (Matrix A)
- * @param[in] b Input tensor (Matrix B)
- * @param[out] c Output tensor to store the result of matrix multiplication. Data type supported: same as @p input0.
- * @param[in] alpha Scalar multiplier to apply to AB matrix product.
- * @param[in] beta Scalar multiplier to apply to input C matrix before adding product.
- * @param[in] gemm_info GEMM meta-data
- */
- void configure(const ITensor *a, const ITensor *b, ITensor *c, float alpha, float beta, const GEMMInfo &gemm_info);
-
- // Inherited methods overridden:
- void run(const Window &window, const ThreadInfo &info) override;
-
-protected:
- /** Called as part of configure() after _a, _b, _c and _params have been set.
- *
- * @param[in] alpha Scalar multiplier to apply to AB matrix product.
- * @param[in] beta Scalar multiplier to apply to input C matrix before adding product.
- *
- * @return A 3D execution window.
- */
- virtual Window configure_internal(float alpha, float beta) = 0;
-
- /** Run the kernel from the start to the end offset in window.
- *
- * @param[in] window Window to use for the iteration
- * @param[in] start_offset Where to start iterating from (In Window coordinates)
- * @param[in] end_offset Where to stop iterating (In Window coordinates).
- * @param[in] info Info about executing thread and CPU.
- */
- virtual void run_internal(const Window &window, const Coordinates &start_offset, const Coordinates &end_offset, const ThreadInfo &info) = 0;
-
- const ITensor *_a;
- const ITensor *_b;
- ITensor *_c;
- Params _params;
- GEMMInfo _gemm_info;
-
-private:
- Window _window3d;
- TensorShape _window_shape;
-};
-
-} // namespace arm_compute
-
-#endif /* SRC_INEGEMMRAPPERKERNEL_H */
diff --git a/src/core/NEON/kernels/assembly/arm_gemm.hpp b/src/core/NEON/kernels/assembly/arm_gemm.hpp
index f6421c1..3088b08 100644
--- a/src/core/NEON/kernels/assembly/arm_gemm.hpp
+++ b/src/core/NEON/kernels/assembly/arm_gemm.hpp
@@ -43,7 +43,9 @@
GEMM_INTERLEAVED_2D,
QUANTIZE_WRAPPER,
QUANTIZE_WRAPPER_2D,
- GEMM_HYBRID_QUANTIZED
+ GEMM_HYBRID_QUANTIZED,
+ INDIRECT_GEMM,
+ CONVOLUTION_GEMM
};
struct KernelDescription
@@ -104,17 +106,19 @@
unsigned int _Msize;
unsigned int _Nsize;
unsigned int _Ksize;
+ unsigned int _Ksections;
unsigned int _nbatches;
unsigned int _nmulti;
+ bool _indirect_input;
Activation _act;
int _maxthreads;
const GemmConfig *_cfg;
- GemmArgs(const CPUInfo *ci, const unsigned int M, const unsigned int N,
- const unsigned int K, const unsigned int nbatches,
- const unsigned int nmulti, Activation act, const int maxthreads,
+ GemmArgs(const CPUInfo *ci, unsigned int M, unsigned int N,
+ unsigned int K, unsigned int Ksections, unsigned int nbatches,
+ unsigned int nmulti, bool indirect_input, Activation act, const int maxthreads,
const GemmConfig *cfg = nullptr)
- : _ci(ci), _Msize(M), _Nsize(N), _Ksize(K), _nbatches(nbatches), _nmulti(nmulti), _act(act), _maxthreads(maxthreads), _cfg(cfg)
+ : _ci(ci), _Msize(M), _Nsize(N), _Ksize(K), _Ksections(Ksections), _nbatches(nbatches), _nmulti(nmulti), _indirect_input(indirect_input), _act(act), _maxthreads(maxthreads), _cfg(cfg)
{
}
};
@@ -143,8 +147,8 @@
Requantize32(const int32_t *bias, size_t bias_multi_stride,
int32_t a_offset, int32_t b_offset, int32_t c_offset,
int32_t requant_shift, int32_t requant_mul, int32_t minv, int32_t maxv)
- : bias(bias), bias_multi_stride(bias_multi_stride), a_offset(a_offset), b_offset(b_offset), c_offset(c_offset), per_channel_requant(false), per_layer_left_shift(std::max(requant_shift, int32_t(0))),
- per_layer_right_shift(std::min(requant_shift, int32_t(0))), per_layer_mul(requant_mul), minval(minv), maxval(maxv)
+ : bias(bias), bias_multi_stride(bias_multi_stride), a_offset(a_offset), b_offset(b_offset), c_offset(c_offset), per_channel_requant(false), per_layer_left_shift(std::max<int32_t>(requant_shift, 0)),
+ per_layer_right_shift(std::min<int32_t>(requant_shift, 0)), per_layer_mul(requant_mul), minval(minv), maxval(maxv)
{
}
diff --git a/src/core/NEON/kernels/assembly/convolution_parameters.hpp b/src/core/NEON/kernels/assembly/convolution_parameters.hpp
new file mode 100644
index 0000000..d0ef5b5
--- /dev/null
+++ b/src/core/NEON/kernels/assembly/convolution_parameters.hpp
@@ -0,0 +1,65 @@
+/*
+ * Copyright (c) 2018-2020 Arm Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#pragma once
+
+#include <cstdint>
+
+namespace arm_gemm
+{
+/*
+ * Parameter set for "convolution" type GEMM.
+ *
+ * For a "convolution" GEMM, the GEMM parameters (M, K) are specified as if
+ * an im2row had been performed on the input tensor to generate the operand
+ * matrix, but instead this structure describes the convolution parameters
+ * such that this can be done on the fly.
+ *
+ * The parameters describe the convolution details - the notional shape of
+ * the input and output tensors, whether padding is to be applied, the size
+ * of the kernel and a constant value to be used for padding (needed for
+ * quantized tensors).
+ *
+ * The second part describes the layout of the input tensor in memory, which
+ * is assumed to be in NHWC format. This consists of a base pointer and
+ * strides for columns, rows and batches. 'multis' are not supported for
+ * convolution type GEMMs.
+ */
+struct ConvolutionParameters
+{
+ int64_t input_width;
+ int64_t input_height;
+ int64_t input_channels;
+ int64_t kernel_width;
+ int64_t kernel_height;
+ int64_t output_width;
+ int64_t output_height;
+ int64_t output_stride_w;
+ int64_t output_stride_h;
+ // output_channels not included as they do not affect the input.
+ int64_t padding_top;
+ int64_t padding_left;
+ float padding_value;
+};
+
+} // namespace arm_gemm
diff --git a/src/core/NEON/kernels/assembly/gemm_common.hpp b/src/core/NEON/kernels/assembly/gemm_common.hpp
index e9e5684..e1fb7a4 100644
--- a/src/core/NEON/kernels/assembly/gemm_common.hpp
+++ b/src/core/NEON/kernels/assembly/gemm_common.hpp
@@ -23,6 +23,7 @@
*/
#pragma once
+#include "convolution_parameters.hpp"
#include "ndrange.hpp"
#include <cstddef>
@@ -77,7 +78,7 @@
return false;
}
- /** Main execute member function
+ /** Main execute member fucntion
* @param [in] work_range specifies the range of work we want to be computed, total range defined by get_window_size()
* @param [in] thread_locator where are we inside of the thread space
* @naram [in] threadid a unique threadid
@@ -123,6 +124,19 @@
{
}
+ /*** Indirect interface (optional) ***/
+ /* Set the indirect table. This comprises a number of values per kernel point, and a densely packed array of pointers,
+ * multis * batches * kernel_points */
+ virtual void set_indirect_parameters_generic(size_t, const void *const *const *)
+ {
+ }
+
+ /*** Convolution interface (optional) ***/
+ /* Set the convolution parameters. */
+ virtual void set_convolution_parameters(ConvolutionParameters)
+ {
+ }
+
// Destructor
virtual ~IGemmCommon()
{
@@ -200,6 +214,16 @@
{
pretranspose_B_array(out, static_cast<const To *>(in), row_stride, multi_stride);
}
+
+ /*** Indirect interface ***/
+ virtual void set_indirect_parameters(size_t, const To *const *const *)
+ {
+ }
+
+ void set_indirect_parameters_generic(size_t sz, const void *const *const *ptr) override
+ {
+ set_indirect_parameters(sz, reinterpret_cast<const To *const *const *>(ptr));
+ }
};
} // namespace arm_gemm