COMPMID-1867: Add NEON/SVE GEMM Hybrid kernels.
Change-Id: Ib40a9921e7f9a6a8be6c38872d6b3a0f24ed0cd3
Reviewed-on: https://review.mlplatform.org/515
Reviewed-by: Anthony Barbier <Anthony.barbier@arm.com>
Tested-by: Arm Jenkins <bsgcomp@arm.com>
diff --git a/src/core/NEON/kernels/arm_gemm/transforms/a64_interleave_8way_32bit.hpp b/src/core/NEON/kernels/arm_gemm/transforms/a64_interleave_8way_32bit.hpp
index 347eafb..0648ff6 100644
--- a/src/core/NEON/kernels/arm_gemm/transforms/a64_interleave_8way_32bit.hpp
+++ b/src/core/NEON/kernels/arm_gemm/transforms/a64_interleave_8way_32bit.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2017-2018 ARM Limited.
+ * Copyright (c) 2017-2019 ARM Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -23,7 +23,7 @@
*/
#pragma once
-#ifdef __aarch64__
+#if defined(__aarch64__) && !defined(__ARM_FEATURE_SVE)
#include <arm_neon.h>
@@ -173,4 +173,4 @@
}
}
-#endif // __aarch64__
+#endif // __aarch64__ && !__ARM_FEATURE_SVE
diff --git a/src/core/NEON/kernels/arm_gemm/transforms/list.hpp b/src/core/NEON/kernels/arm_gemm/transforms/list.hpp
index fc1f2c2..e1ebba0 100644
--- a/src/core/NEON/kernels/arm_gemm/transforms/list.hpp
+++ b/src/core/NEON/kernels/arm_gemm/transforms/list.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2017-2018 ARM Limited.
+ * Copyright (c) 2017-2019 ARM Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -23,17 +23,14 @@
*/
#include "a32_interleave_6way_32bit.hpp"
#include "a32_transpose_interleave_8way_32bit.hpp"
-#ifdef __ARM_FEATURE_SVE
-#include "sve_interleave_8way_32bit.hpp"
-#include "sve_interleave_8way_block2_32bit.hpp"
-#include "sve_interleave_8way_block4_8bit.hpp"
-#else
-#include "a64_interleave_8way_32bit.hpp"
-#endif
#include "a64_block16_interleave4_8bit.hpp"
#include "a64_interleave_8way_16bit.hpp"
+#include "a64_interleave_8way_32bit.hpp"
#include "a64_interleave_8way_half_to_float.hpp"
#include "a64_transpose_interleave_12way_16bit.hpp"
#include "a64_transpose_interleave_12way_half_to_float.hpp"
#include "a64_transpose_interleave_24way_16bit.hpp"
-#include "transpose_interleave_common.hpp"
+#include "sve_interleave_8way_32bit.hpp"
+#include "sve_interleave_8way_block2_32bit.hpp"
+#include "sve_interleave_8way_block4_8bit.hpp"
+#include "transpose_interleave_common.hpp"
\ No newline at end of file
diff --git a/src/core/NEON/kernels/arm_gemm/transforms/sve_interleave_8way_32bit.hpp b/src/core/NEON/kernels/arm_gemm/transforms/sve_interleave_8way_32bit.hpp
index 752e837..07c8219 100644
--- a/src/core/NEON/kernels/arm_gemm/transforms/sve_interleave_8way_32bit.hpp
+++ b/src/core/NEON/kernels/arm_gemm/transforms/sve_interleave_8way_32bit.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2018 Arm Limited.
+ * Copyright (c) 2018-2019 Arm Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -41,7 +41,7 @@
long outpos = 0;
uint32_t *outptr = master_outptr;
- master_outptr += outwidth;
+ master_outptr += (outwidth * 1);
const uint32_t *inptr0 = inptr + y * ldin + k0;
const uint32_t *inptr1 = inptr0 + ldin;
@@ -60,52 +60,53 @@
"whilelt p0.s, %[inpos], %[inwidth]\n"
"b.none 2f\n"
"mov z4.s, #0\n"
- "ld1w z0.s, p0/z, [%[inptr0], %[inpos], LSL #2]\n"
+ "ld1w z0.s, p0/z, [%[inptr0]]\n"
+ "zip1 z8.s, z0.s, z4.s\n"
"incw %[inpos], all, mul #1\n"
- "whilelt p0.s, %[outpos], %[outwidth]\n"
- "incw %[outpos], all, mul #1\n"
- "zip1 z8.s, z0.s, z4.s\n"
"zip2 z9.s, z0.s, z4.s\n"
- "whilelt p1.s, %[outpos], %[outwidth]\n"
- "incw %[outpos], all, mul #1\n"
+ "addvl %[inptr0], %[inptr0], #1\n"
"zip1 z0.s, z8.s, z4.s\n"
+ "whilelt p0.s, %[outpos], %[outwidth]\n"
"zip2 z1.s, z8.s, z4.s\n"
+ "incw %[outpos], all, mul #1\n"
"zip1 z2.s, z9.s, z4.s\n"
+ "whilelt p1.s, %[outpos], %[outwidth]\n"
"zip2 z3.s, z9.s, z4.s\n"
- "whilelt p2.s, %[outpos], %[outwidth]\n"
+ "incw %[outpos], all, mul #1\n"
"zip1 z8.s, z0.s, z4.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip2 z9.s, z0.s, z4.s\n"
- "zip1 z10.s, z1.s, z4.s\n"
- "zip2 z11.s, z1.s, z4.s\n"
"st1w z8.s, p0, [%[outptr]]\n"
- "zip1 z12.s, z2.s, z4.s\n"
- "whilelt p3.s, %[outpos], %[outwidth]\n"
- "zip2 z13.s, z2.s, z4.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip1 z14.s, z3.s, z4.s\n"
+ "zip2 z9.s, z0.s, z4.s\n"
"st1w z9.s, p1, [%[outptr], #1, MUL VL]\n"
- "zip2 z15.s, z3.s, z4.s\n"
- "whilelt p4.s, %[outpos], %[outwidth]\n"
+ "zip1 z10.s, z1.s, z4.s\n"
+ "whilelt p2.s, %[outpos], %[outwidth]\n"
+ "zip2 z11.s, z1.s, z4.s\n"
"st1w z10.s, p2, [%[outptr], #2, MUL VL]\n"
+ "zip1 z12.s, z2.s, z4.s\n"
"incw %[outpos], all, mul #1\n"
+ "zip2 z13.s, z2.s, z4.s\n"
+ "whilelt p3.s, %[outpos], %[outwidth]\n"
+ "zip1 z14.s, z3.s, z4.s\n"
"st1w z11.s, p3, [%[outptr], #3, MUL VL]\n"
- "whilelt p5.s, %[outpos], %[outwidth]\n"
+ "zip2 z15.s, z3.s, z4.s\n"
"incw %[outpos], all, mul #1\n"
- "st1w z12.s, p4, [%[outptr], #4, MUL VL]\n"
- "whilelt p6.s, %[outpos], %[outwidth]\n"
+ "whilelt p0.s, %[outpos], %[outwidth]\n"
+ "st1w z12.s, p0, [%[outptr], #4, MUL VL]\n"
"incw %[outpos], all, mul #1\n"
- "st1w z13.s, p5, [%[outptr], #5, MUL VL]\n"
- "whilelt p7.s, %[outpos], %[outwidth]\n"
+ "whilelt p1.s, %[outpos], %[outwidth]\n"
+ "st1w z13.s, p1, [%[outptr], #5, MUL VL]\n"
"incw %[outpos], all, mul #1\n"
- "st1w z14.s, p6, [%[outptr], #6, MUL VL]\n"
- "st1w z15.s, p7, [%[outptr], #7, MUL VL]\n"
+ "whilelt p2.s, %[outpos], %[outwidth]\n"
+ "st1w z14.s, p2, [%[outptr], #6, MUL VL]\n"
+ "incw %[outpos], all, mul #1\n"
+ "whilelt p3.s, %[outpos], %[outwidth]\n"
+ "st1w z15.s, p3, [%[outptr], #7, MUL VL]\n"
+ "incw %[outpos], all, mul #1\n"
"addvl %[outptr], %[outptr], #8\n"
"b 1b\n"
"2:\n"
: [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0)
: [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
- : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
+ : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
);
break;
@@ -115,60 +116,62 @@
"whilelt p0.s, %[inpos], %[inwidth]\n"
"b.none 2f\n"
"mov z4.s, #0\n"
- "mov z14.s, #0\n"
- "ld1w z0.s, p0/z, [%[inptr0], %[inpos], LSL #2]\n"
- "ld1w z1.s, p0/z, [%[inptr1], %[inpos], LSL #2]\n"
- "incw %[inpos], all, mul #1\n"
- "whilelt p0.s, %[outpos], %[outwidth]\n"
- "incw %[outpos], all, mul #1\n"
+ "ld1w z0.s, p0/z, [%[inptr0]]\n"
"zip1 z8.s, z0.s, z4.s\n"
+ "ld1w z1.s, p0/z, [%[inptr1]]\n"
"zip2 z9.s, z0.s, z4.s\n"
+ "incw %[inpos], all, mul #1\n"
"zip1 z10.s, z1.s, z4.s\n"
+ "addvl %[inptr0], %[inptr0], #1\n"
"zip2 z11.s, z1.s, z4.s\n"
- "whilelt p1.s, %[outpos], %[outwidth]\n"
+ "addvl %[inptr1], %[inptr1], #1\n"
"zip1 z0.s, z8.s, z4.s\n"
- "incw %[outpos], all, mul #1\n"
+ "whilelt p0.s, %[outpos], %[outwidth]\n"
"zip2 z1.s, z8.s, z4.s\n"
- "zip1 z2.s, z9.s, z4.s\n"
- "zip2 z3.s, z9.s, z4.s\n"
- "zip1 z4.s, z10.s, z14.s\n"
- "whilelt p2.s, %[outpos], %[outwidth]\n"
- "zip2 z5.s, z10.s, z14.s\n"
"incw %[outpos], all, mul #1\n"
+ "zip1 z2.s, z9.s, z4.s\n"
+ "whilelt p1.s, %[outpos], %[outwidth]\n"
+ "zip2 z3.s, z9.s, z4.s\n"
+ "incw %[outpos], all, mul #1\n"
+ "mov z14.s, #0\n"
+ "whilelt p2.s, %[outpos], %[outwidth]\n"
+ "zip1 z4.s, z10.s, z14.s\n"
+ "incw %[outpos], all, mul #1\n"
+ "zip2 z5.s, z10.s, z14.s\n"
+ "whilelt p3.s, %[outpos], %[outwidth]\n"
"zip1 z6.s, z11.s, z14.s\n"
+ "incw %[outpos], all, mul #1\n"
"zip2 z7.s, z11.s, z14.s\n"
"zip1 z8.s, z0.s, z4.s\n"
- "zip2 z9.s, z0.s, z4.s\n"
- "whilelt p3.s, %[outpos], %[outwidth]\n"
- "zip1 z10.s, z1.s, z5.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip2 z11.s, z1.s, z5.s\n"
"st1w z8.s, p0, [%[outptr]]\n"
- "zip1 z12.s, z2.s, z6.s\n"
- "zip2 z13.s, z2.s, z6.s\n"
- "zip1 z14.s, z3.s, z7.s\n"
- "whilelt p4.s, %[outpos], %[outwidth]\n"
- "zip2 z15.s, z3.s, z7.s\n"
+ "zip2 z9.s, z0.s, z4.s\n"
"st1w z9.s, p1, [%[outptr], #1, MUL VL]\n"
- "incw %[outpos], all, mul #1\n"
+ "zip1 z10.s, z1.s, z5.s\n"
"st1w z10.s, p2, [%[outptr], #2, MUL VL]\n"
- "whilelt p5.s, %[outpos], %[outwidth]\n"
- "incw %[outpos], all, mul #1\n"
+ "zip2 z11.s, z1.s, z5.s\n"
"st1w z11.s, p3, [%[outptr], #3, MUL VL]\n"
- "whilelt p6.s, %[outpos], %[outwidth]\n"
+ "zip1 z12.s, z2.s, z6.s\n"
+ "whilelt p0.s, %[outpos], %[outwidth]\n"
+ "zip2 z13.s, z2.s, z6.s\n"
+ "st1w z12.s, p0, [%[outptr], #4, MUL VL]\n"
+ "zip1 z14.s, z3.s, z7.s\n"
"incw %[outpos], all, mul #1\n"
- "st1w z12.s, p4, [%[outptr], #4, MUL VL]\n"
- "whilelt p7.s, %[outpos], %[outwidth]\n"
+ "zip2 z15.s, z3.s, z7.s\n"
+ "whilelt p1.s, %[outpos], %[outwidth]\n"
+ "st1w z13.s, p1, [%[outptr], #5, MUL VL]\n"
"incw %[outpos], all, mul #1\n"
- "st1w z13.s, p5, [%[outptr], #5, MUL VL]\n"
- "st1w z14.s, p6, [%[outptr], #6, MUL VL]\n"
- "st1w z15.s, p7, [%[outptr], #7, MUL VL]\n"
+ "whilelt p2.s, %[outpos], %[outwidth]\n"
+ "st1w z14.s, p2, [%[outptr], #6, MUL VL]\n"
+ "incw %[outpos], all, mul #1\n"
+ "whilelt p3.s, %[outpos], %[outwidth]\n"
+ "st1w z15.s, p3, [%[outptr], #7, MUL VL]\n"
+ "incw %[outpos], all, mul #1\n"
"addvl %[outptr], %[outptr], #8\n"
"b 1b\n"
"2:\n"
: [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1)
: [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
- : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
+ : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
);
break;
@@ -178,63 +181,66 @@
"whilelt p0.s, %[inpos], %[inwidth]\n"
"b.none 2f\n"
"mov z4.s, #0\n"
- "mov z14.s, #0\n"
- "ld1w z0.s, p0/z, [%[inptr0], %[inpos], LSL #2]\n"
- "ld1w z1.s, p0/z, [%[inptr1], %[inpos], LSL #2]\n"
- "ld1w z2.s, p0/z, [%[inptr2], %[inpos], LSL #2]\n"
- "incw %[inpos], all, mul #1\n"
- "whilelt p0.s, %[outpos], %[outwidth]\n"
+ "ld1w z0.s, p0/z, [%[inptr0]]\n"
"zip1 z8.s, z0.s, z4.s\n"
- "incw %[outpos], all, mul #1\n"
+ "ld1w z1.s, p0/z, [%[inptr1]]\n"
"zip2 z9.s, z0.s, z4.s\n"
+ "ld1w z2.s, p0/z, [%[inptr2]]\n"
"zip1 z10.s, z1.s, z4.s\n"
+ "incw %[inpos], all, mul #1\n"
"zip2 z11.s, z1.s, z4.s\n"
+ "addvl %[inptr0], %[inptr0], #1\n"
"zip1 z12.s, z2.s, z4.s\n"
- "whilelt p1.s, %[outpos], %[outwidth]\n"
+ "addvl %[inptr1], %[inptr1], #1\n"
"zip2 z13.s, z2.s, z4.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip1 z4.s, z10.s, z14.s\n"
+ "addvl %[inptr2], %[inptr2], #1\n"
"zip1 z0.s, z8.s, z12.s\n"
+ "whilelt p0.s, %[outpos], %[outwidth]\n"
"zip2 z1.s, z8.s, z12.s\n"
+ "incw %[outpos], all, mul #1\n"
"zip1 z2.s, z9.s, z13.s\n"
- "whilelt p2.s, %[outpos], %[outwidth]\n"
+ "whilelt p1.s, %[outpos], %[outwidth]\n"
"zip2 z3.s, z9.s, z13.s\n"
"incw %[outpos], all, mul #1\n"
+ "mov z14.s, #0\n"
+ "whilelt p2.s, %[outpos], %[outwidth]\n"
+ "zip1 z4.s, z10.s, z14.s\n"
+ "incw %[outpos], all, mul #1\n"
"zip2 z5.s, z10.s, z14.s\n"
+ "whilelt p3.s, %[outpos], %[outwidth]\n"
"zip1 z6.s, z11.s, z14.s\n"
+ "incw %[outpos], all, mul #1\n"
"zip2 z7.s, z11.s, z14.s\n"
"zip1 z8.s, z0.s, z4.s\n"
- "whilelt p3.s, %[outpos], %[outwidth]\n"
- "zip2 z9.s, z0.s, z4.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip1 z10.s, z1.s, z5.s\n"
"st1w z8.s, p0, [%[outptr]]\n"
- "zip2 z11.s, z1.s, z5.s\n"
- "zip1 z12.s, z2.s, z6.s\n"
- "zip2 z13.s, z2.s, z6.s\n"
- "whilelt p4.s, %[outpos], %[outwidth]\n"
- "zip1 z14.s, z3.s, z7.s\n"
+ "zip2 z9.s, z0.s, z4.s\n"
"st1w z9.s, p1, [%[outptr], #1, MUL VL]\n"
- "zip2 z15.s, z3.s, z7.s\n"
- "incw %[outpos], all, mul #1\n"
+ "zip1 z10.s, z1.s, z5.s\n"
"st1w z10.s, p2, [%[outptr], #2, MUL VL]\n"
- "whilelt p5.s, %[outpos], %[outwidth]\n"
- "incw %[outpos], all, mul #1\n"
+ "zip2 z11.s, z1.s, z5.s\n"
"st1w z11.s, p3, [%[outptr], #3, MUL VL]\n"
- "whilelt p6.s, %[outpos], %[outwidth]\n"
+ "zip1 z12.s, z2.s, z6.s\n"
+ "whilelt p0.s, %[outpos], %[outwidth]\n"
+ "zip2 z13.s, z2.s, z6.s\n"
+ "st1w z12.s, p0, [%[outptr], #4, MUL VL]\n"
+ "zip1 z14.s, z3.s, z7.s\n"
"incw %[outpos], all, mul #1\n"
- "st1w z12.s, p4, [%[outptr], #4, MUL VL]\n"
- "whilelt p7.s, %[outpos], %[outwidth]\n"
- "st1w z13.s, p5, [%[outptr], #5, MUL VL]\n"
+ "zip2 z15.s, z3.s, z7.s\n"
+ "whilelt p1.s, %[outpos], %[outwidth]\n"
+ "st1w z13.s, p1, [%[outptr], #5, MUL VL]\n"
"incw %[outpos], all, mul #1\n"
- "st1w z14.s, p6, [%[outptr], #6, MUL VL]\n"
- "st1w z15.s, p7, [%[outptr], #7, MUL VL]\n"
+ "whilelt p2.s, %[outpos], %[outwidth]\n"
+ "st1w z14.s, p2, [%[outptr], #6, MUL VL]\n"
+ "incw %[outpos], all, mul #1\n"
+ "whilelt p3.s, %[outpos], %[outwidth]\n"
+ "st1w z15.s, p3, [%[outptr], #7, MUL VL]\n"
+ "incw %[outpos], all, mul #1\n"
"addvl %[outptr], %[outptr], #8\n"
"b 1b\n"
"2:\n"
: [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2)
: [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
- : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
+ : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
);
break;
@@ -244,65 +250,69 @@
"whilelt p0.s, %[inpos], %[inwidth]\n"
"b.none 2f\n"
"mov z4.s, #0\n"
- "ld1w z0.s, p0/z, [%[inptr0], %[inpos], LSL #2]\n"
- "ld1w z1.s, p0/z, [%[inptr1], %[inpos], LSL #2]\n"
- "ld1w z2.s, p0/z, [%[inptr2], %[inpos], LSL #2]\n"
- "ld1w z3.s, p0/z, [%[inptr3], %[inpos], LSL #2]\n"
- "incw %[inpos], all, mul #1\n"
+ "ld1w z0.s, p0/z, [%[inptr0]]\n"
"zip1 z8.s, z0.s, z4.s\n"
- "whilelt p0.s, %[outpos], %[outwidth]\n"
+ "ld1w z1.s, p0/z, [%[inptr1]]\n"
"zip2 z9.s, z0.s, z4.s\n"
- "incw %[outpos], all, mul #1\n"
+ "ld1w z2.s, p0/z, [%[inptr2]]\n"
"zip1 z10.s, z1.s, z4.s\n"
+ "ld1w z3.s, p0/z, [%[inptr3]]\n"
"zip2 z11.s, z1.s, z4.s\n"
+ "incw %[inpos], all, mul #1\n"
"zip1 z12.s, z2.s, z4.s\n"
+ "addvl %[inptr0], %[inptr0], #1\n"
"zip2 z13.s, z2.s, z4.s\n"
- "whilelt p1.s, %[outpos], %[outwidth]\n"
+ "addvl %[inptr1], %[inptr1], #1\n"
"zip1 z14.s, z3.s, z4.s\n"
- "incw %[outpos], all, mul #1\n"
+ "addvl %[inptr2], %[inptr2], #1\n"
"zip2 z15.s, z3.s, z4.s\n"
+ "addvl %[inptr3], %[inptr3], #1\n"
"zip1 z0.s, z8.s, z12.s\n"
+ "whilelt p0.s, %[outpos], %[outwidth]\n"
"zip2 z1.s, z8.s, z12.s\n"
+ "incw %[outpos], all, mul #1\n"
"zip1 z2.s, z9.s, z13.s\n"
- "whilelt p2.s, %[outpos], %[outwidth]\n"
+ "whilelt p1.s, %[outpos], %[outwidth]\n"
"zip2 z3.s, z9.s, z13.s\n"
"incw %[outpos], all, mul #1\n"
"zip1 z4.s, z10.s, z14.s\n"
+ "whilelt p2.s, %[outpos], %[outwidth]\n"
"zip2 z5.s, z10.s, z14.s\n"
+ "incw %[outpos], all, mul #1\n"
"zip1 z6.s, z11.s, z15.s\n"
- "zip2 z7.s, z11.s, z15.s\n"
"whilelt p3.s, %[outpos], %[outwidth]\n"
+ "zip2 z7.s, z11.s, z15.s\n"
+ "incw %[outpos], all, mul #1\n"
"zip1 z8.s, z0.s, z4.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip2 z9.s, z0.s, z4.s\n"
- "zip1 z10.s, z1.s, z5.s\n"
- "zip2 z11.s, z1.s, z5.s\n"
"st1w z8.s, p0, [%[outptr]]\n"
- "zip1 z12.s, z2.s, z6.s\n"
- "whilelt p4.s, %[outpos], %[outwidth]\n"
- "zip2 z13.s, z2.s, z6.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip1 z14.s, z3.s, z7.s\n"
+ "zip2 z9.s, z0.s, z4.s\n"
"st1w z9.s, p1, [%[outptr], #1, MUL VL]\n"
- "zip2 z15.s, z3.s, z7.s\n"
- "whilelt p5.s, %[outpos], %[outwidth]\n"
+ "zip1 z10.s, z1.s, z5.s\n"
"st1w z10.s, p2, [%[outptr], #2, MUL VL]\n"
- "incw %[outpos], all, mul #1\n"
+ "zip2 z11.s, z1.s, z5.s\n"
"st1w z11.s, p3, [%[outptr], #3, MUL VL]\n"
- "whilelt p6.s, %[outpos], %[outwidth]\n"
+ "zip1 z12.s, z2.s, z6.s\n"
+ "whilelt p0.s, %[outpos], %[outwidth]\n"
+ "zip2 z13.s, z2.s, z6.s\n"
+ "st1w z12.s, p0, [%[outptr], #4, MUL VL]\n"
+ "zip1 z14.s, z3.s, z7.s\n"
"incw %[outpos], all, mul #1\n"
- "st1w z12.s, p4, [%[outptr], #4, MUL VL]\n"
- "whilelt p7.s, %[outpos], %[outwidth]\n"
+ "zip2 z15.s, z3.s, z7.s\n"
+ "whilelt p1.s, %[outpos], %[outwidth]\n"
+ "st1w z13.s, p1, [%[outptr], #5, MUL VL]\n"
"incw %[outpos], all, mul #1\n"
- "st1w z13.s, p5, [%[outptr], #5, MUL VL]\n"
- "st1w z14.s, p6, [%[outptr], #6, MUL VL]\n"
- "st1w z15.s, p7, [%[outptr], #7, MUL VL]\n"
+ "whilelt p2.s, %[outpos], %[outwidth]\n"
+ "st1w z14.s, p2, [%[outptr], #6, MUL VL]\n"
+ "incw %[outpos], all, mul #1\n"
+ "whilelt p3.s, %[outpos], %[outwidth]\n"
+ "st1w z15.s, p3, [%[outptr], #7, MUL VL]\n"
+ "incw %[outpos], all, mul #1\n"
"addvl %[outptr], %[outptr], #8\n"
"b 1b\n"
"2:\n"
: [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3)
: [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
- : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
+ : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
);
break;
@@ -312,66 +322,71 @@
"whilelt p0.s, %[inpos], %[inwidth]\n"
"b.none 2f\n"
"mov z5.s, #0\n"
- "ld1w z0.s, p0/z, [%[inptr0], %[inpos], LSL #2]\n"
- "ld1w z1.s, p0/z, [%[inptr1], %[inpos], LSL #2]\n"
- "ld1w z2.s, p0/z, [%[inptr2], %[inpos], LSL #2]\n"
- "ld1w z3.s, p0/z, [%[inptr3], %[inpos], LSL #2]\n"
- "ld1w z4.s, p0/z, [%[inptr4], %[inpos], LSL #2]\n"
+ "ld1w z0.s, p0/z, [%[inptr0]]\n"
+ "ld1w z1.s, p0/z, [%[inptr1]]\n"
"incw %[inpos], all, mul #1\n"
"zip1 z10.s, z1.s, z5.s\n"
- "whilelt p0.s, %[outpos], %[outwidth]\n"
+ "ld1w z2.s, p0/z, [%[inptr2]]\n"
"zip2 z11.s, z1.s, z5.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip1 z8.s, z0.s, z4.s\n"
- "zip2 z9.s, z0.s, z4.s\n"
+ "ld1w z3.s, p0/z, [%[inptr3]]\n"
"zip1 z12.s, z2.s, z5.s\n"
+ "ld1w z4.s, p0/z, [%[inptr4]]\n"
+ "zip1 z8.s, z0.s, z4.s\n"
+ "addvl %[inptr0], %[inptr0], #1\n"
+ "zip2 z9.s, z0.s, z4.s\n"
+ "addvl %[inptr1], %[inptr1], #1\n"
"zip2 z13.s, z2.s, z5.s\n"
- "whilelt p1.s, %[outpos], %[outwidth]\n"
+ "addvl %[inptr2], %[inptr2], #1\n"
"zip1 z14.s, z3.s, z5.s\n"
- "incw %[outpos], all, mul #1\n"
+ "addvl %[inptr3], %[inptr3], #1\n"
"zip2 z15.s, z3.s, z5.s\n"
+ "addvl %[inptr4], %[inptr4], #1\n"
"zip1 z0.s, z8.s, z12.s\n"
+ "whilelt p0.s, %[outpos], %[outwidth]\n"
"zip2 z1.s, z8.s, z12.s\n"
+ "incw %[outpos], all, mul #1\n"
"zip1 z2.s, z9.s, z13.s\n"
- "whilelt p2.s, %[outpos], %[outwidth]\n"
+ "whilelt p1.s, %[outpos], %[outwidth]\n"
"zip2 z3.s, z9.s, z13.s\n"
"incw %[outpos], all, mul #1\n"
"zip1 z4.s, z10.s, z14.s\n"
+ "whilelt p2.s, %[outpos], %[outwidth]\n"
"zip2 z5.s, z10.s, z14.s\n"
+ "incw %[outpos], all, mul #1\n"
"zip1 z6.s, z11.s, z15.s\n"
- "zip2 z7.s, z11.s, z15.s\n"
"whilelt p3.s, %[outpos], %[outwidth]\n"
+ "zip2 z7.s, z11.s, z15.s\n"
+ "incw %[outpos], all, mul #1\n"
"zip1 z8.s, z0.s, z4.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip2 z9.s, z0.s, z4.s\n"
- "zip1 z10.s, z1.s, z5.s\n"
- "zip2 z11.s, z1.s, z5.s\n"
"st1w z8.s, p0, [%[outptr]]\n"
- "zip1 z12.s, z2.s, z6.s\n"
- "whilelt p4.s, %[outpos], %[outwidth]\n"
- "zip2 z13.s, z2.s, z6.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip1 z14.s, z3.s, z7.s\n"
+ "zip2 z9.s, z0.s, z4.s\n"
"st1w z9.s, p1, [%[outptr], #1, MUL VL]\n"
- "zip2 z15.s, z3.s, z7.s\n"
- "whilelt p5.s, %[outpos], %[outwidth]\n"
+ "zip1 z10.s, z1.s, z5.s\n"
"st1w z10.s, p2, [%[outptr], #2, MUL VL]\n"
- "incw %[outpos], all, mul #1\n"
+ "zip2 z11.s, z1.s, z5.s\n"
"st1w z11.s, p3, [%[outptr], #3, MUL VL]\n"
- "whilelt p6.s, %[outpos], %[outwidth]\n"
+ "zip1 z12.s, z2.s, z6.s\n"
+ "whilelt p0.s, %[outpos], %[outwidth]\n"
+ "zip2 z13.s, z2.s, z6.s\n"
+ "st1w z12.s, p0, [%[outptr], #4, MUL VL]\n"
+ "zip1 z14.s, z3.s, z7.s\n"
"incw %[outpos], all, mul #1\n"
- "st1w z12.s, p4, [%[outptr], #4, MUL VL]\n"
- "whilelt p7.s, %[outpos], %[outwidth]\n"
+ "zip2 z15.s, z3.s, z7.s\n"
+ "whilelt p1.s, %[outpos], %[outwidth]\n"
+ "st1w z13.s, p1, [%[outptr], #5, MUL VL]\n"
"incw %[outpos], all, mul #1\n"
- "st1w z13.s, p5, [%[outptr], #5, MUL VL]\n"
- "st1w z14.s, p6, [%[outptr], #6, MUL VL]\n"
- "st1w z15.s, p7, [%[outptr], #7, MUL VL]\n"
+ "whilelt p2.s, %[outpos], %[outwidth]\n"
+ "st1w z14.s, p2, [%[outptr], #6, MUL VL]\n"
+ "incw %[outpos], all, mul #1\n"
+ "whilelt p3.s, %[outpos], %[outwidth]\n"
+ "st1w z15.s, p3, [%[outptr], #7, MUL VL]\n"
+ "incw %[outpos], all, mul #1\n"
"addvl %[outptr], %[outptr], #8\n"
"b 1b\n"
"2:\n"
: [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3), [inptr4] "+r" (inptr4)
: [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
- : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
+ : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
);
break;
@@ -381,67 +396,73 @@
"whilelt p0.s, %[inpos], %[inwidth]\n"
"b.none 2f\n"
"mov z6.s, #0\n"
- "ld1w z0.s, p0/z, [%[inptr0], %[inpos], LSL #2]\n"
- "ld1w z1.s, p0/z, [%[inptr1], %[inpos], LSL #2]\n"
- "ld1w z2.s, p0/z, [%[inptr2], %[inpos], LSL #2]\n"
- "ld1w z3.s, p0/z, [%[inptr3], %[inpos], LSL #2]\n"
- "ld1w z4.s, p0/z, [%[inptr4], %[inpos], LSL #2]\n"
- "ld1w z5.s, p0/z, [%[inptr5], %[inpos], LSL #2]\n"
+ "ld1w z0.s, p0/z, [%[inptr0]]\n"
+ "ld1w z1.s, p0/z, [%[inptr1]]\n"
"incw %[inpos], all, mul #1\n"
+ "ld1w z2.s, p0/z, [%[inptr2]]\n"
+ "addvl %[inptr0], %[inptr0], #1\n"
"zip1 z12.s, z2.s, z6.s\n"
- "whilelt p0.s, %[outpos], %[outwidth]\n"
- "zip1 z8.s, z0.s, z4.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip2 z9.s, z0.s, z4.s\n"
- "zip1 z10.s, z1.s, z5.s\n"
- "zip2 z11.s, z1.s, z5.s\n"
+ "ld1w z3.s, p0/z, [%[inptr3]]\n"
"zip2 z13.s, z2.s, z6.s\n"
- "whilelt p1.s, %[outpos], %[outwidth]\n"
+ "ld1w z4.s, p0/z, [%[inptr4]]\n"
+ "zip1 z8.s, z0.s, z4.s\n"
+ "ld1w z5.s, p0/z, [%[inptr5]]\n"
+ "zip2 z9.s, z0.s, z4.s\n"
+ "addvl %[inptr1], %[inptr1], #1\n"
+ "zip1 z10.s, z1.s, z5.s\n"
+ "addvl %[inptr2], %[inptr2], #1\n"
+ "zip2 z11.s, z1.s, z5.s\n"
+ "addvl %[inptr3], %[inptr3], #1\n"
"zip1 z14.s, z3.s, z6.s\n"
- "incw %[outpos], all, mul #1\n"
+ "addvl %[inptr4], %[inptr4], #1\n"
"zip2 z15.s, z3.s, z6.s\n"
+ "addvl %[inptr5], %[inptr5], #1\n"
"zip1 z0.s, z8.s, z12.s\n"
+ "whilelt p0.s, %[outpos], %[outwidth]\n"
"zip2 z1.s, z8.s, z12.s\n"
+ "incw %[outpos], all, mul #1\n"
"zip1 z2.s, z9.s, z13.s\n"
- "whilelt p2.s, %[outpos], %[outwidth]\n"
+ "whilelt p1.s, %[outpos], %[outwidth]\n"
"zip2 z3.s, z9.s, z13.s\n"
"incw %[outpos], all, mul #1\n"
"zip1 z4.s, z10.s, z14.s\n"
+ "whilelt p2.s, %[outpos], %[outwidth]\n"
"zip2 z5.s, z10.s, z14.s\n"
+ "incw %[outpos], all, mul #1\n"
"zip1 z6.s, z11.s, z15.s\n"
- "zip2 z7.s, z11.s, z15.s\n"
"whilelt p3.s, %[outpos], %[outwidth]\n"
+ "zip2 z7.s, z11.s, z15.s\n"
+ "incw %[outpos], all, mul #1\n"
"zip1 z8.s, z0.s, z4.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip2 z9.s, z0.s, z4.s\n"
- "zip1 z10.s, z1.s, z5.s\n"
- "zip2 z11.s, z1.s, z5.s\n"
"st1w z8.s, p0, [%[outptr]]\n"
- "zip1 z12.s, z2.s, z6.s\n"
- "whilelt p4.s, %[outpos], %[outwidth]\n"
- "zip2 z13.s, z2.s, z6.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip1 z14.s, z3.s, z7.s\n"
+ "zip2 z9.s, z0.s, z4.s\n"
"st1w z9.s, p1, [%[outptr], #1, MUL VL]\n"
- "zip2 z15.s, z3.s, z7.s\n"
- "whilelt p5.s, %[outpos], %[outwidth]\n"
+ "zip1 z10.s, z1.s, z5.s\n"
"st1w z10.s, p2, [%[outptr], #2, MUL VL]\n"
- "incw %[outpos], all, mul #1\n"
+ "zip2 z11.s, z1.s, z5.s\n"
"st1w z11.s, p3, [%[outptr], #3, MUL VL]\n"
- "whilelt p6.s, %[outpos], %[outwidth]\n"
+ "zip1 z12.s, z2.s, z6.s\n"
+ "whilelt p0.s, %[outpos], %[outwidth]\n"
+ "zip2 z13.s, z2.s, z6.s\n"
+ "st1w z12.s, p0, [%[outptr], #4, MUL VL]\n"
+ "zip1 z14.s, z3.s, z7.s\n"
"incw %[outpos], all, mul #1\n"
- "st1w z12.s, p4, [%[outptr], #4, MUL VL]\n"
- "whilelt p7.s, %[outpos], %[outwidth]\n"
+ "zip2 z15.s, z3.s, z7.s\n"
+ "whilelt p1.s, %[outpos], %[outwidth]\n"
+ "st1w z13.s, p1, [%[outptr], #5, MUL VL]\n"
"incw %[outpos], all, mul #1\n"
- "st1w z13.s, p5, [%[outptr], #5, MUL VL]\n"
- "st1w z14.s, p6, [%[outptr], #6, MUL VL]\n"
- "st1w z15.s, p7, [%[outptr], #7, MUL VL]\n"
+ "whilelt p2.s, %[outpos], %[outwidth]\n"
+ "st1w z14.s, p2, [%[outptr], #6, MUL VL]\n"
+ "incw %[outpos], all, mul #1\n"
+ "whilelt p3.s, %[outpos], %[outwidth]\n"
+ "st1w z15.s, p3, [%[outptr], #7, MUL VL]\n"
+ "incw %[outpos], all, mul #1\n"
"addvl %[outptr], %[outptr], #8\n"
"b 1b\n"
"2:\n"
: [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3), [inptr4] "+r" (inptr4), [inptr5] "+r" (inptr5)
: [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
- : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
+ : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
);
break;
@@ -451,68 +472,75 @@
"whilelt p0.s, %[inpos], %[inwidth]\n"
"b.none 2f\n"
"mov z7.s, #0\n"
- "ld1w z0.s, p0/z, [%[inptr0], %[inpos], LSL #2]\n"
- "ld1w z1.s, p0/z, [%[inptr1], %[inpos], LSL #2]\n"
- "ld1w z2.s, p0/z, [%[inptr2], %[inpos], LSL #2]\n"
- "ld1w z3.s, p0/z, [%[inptr3], %[inpos], LSL #2]\n"
- "ld1w z4.s, p0/z, [%[inptr4], %[inpos], LSL #2]\n"
- "ld1w z5.s, p0/z, [%[inptr5], %[inpos], LSL #2]\n"
- "ld1w z6.s, p0/z, [%[inptr6], %[inpos], LSL #2]\n"
+ "ld1w z0.s, p0/z, [%[inptr0]]\n"
+ "ld1w z1.s, p0/z, [%[inptr1]]\n"
"incw %[inpos], all, mul #1\n"
+ "ld1w z2.s, p0/z, [%[inptr2]]\n"
+ "addvl %[inptr0], %[inptr0], #1\n"
+ "ld1w z3.s, p0/z, [%[inptr3]]\n"
+ "addvl %[inptr1], %[inptr1], #1\n"
"zip1 z14.s, z3.s, z7.s\n"
- "whilelt p0.s, %[outpos], %[outwidth]\n"
+ "ld1w z4.s, p0/z, [%[inptr4]]\n"
"zip1 z8.s, z0.s, z4.s\n"
- "incw %[outpos], all, mul #1\n"
+ "ld1w z5.s, p0/z, [%[inptr5]]\n"
"zip2 z9.s, z0.s, z4.s\n"
+ "ld1w z6.s, p0/z, [%[inptr6]]\n"
"zip1 z10.s, z1.s, z5.s\n"
+ "addvl %[inptr2], %[inptr2], #1\n"
"zip2 z11.s, z1.s, z5.s\n"
+ "addvl %[inptr3], %[inptr3], #1\n"
"zip1 z12.s, z2.s, z6.s\n"
- "whilelt p1.s, %[outpos], %[outwidth]\n"
+ "addvl %[inptr4], %[inptr4], #1\n"
"zip2 z13.s, z2.s, z6.s\n"
- "incw %[outpos], all, mul #1\n"
+ "addvl %[inptr5], %[inptr5], #1\n"
"zip2 z15.s, z3.s, z7.s\n"
+ "addvl %[inptr6], %[inptr6], #1\n"
"zip1 z0.s, z8.s, z12.s\n"
+ "whilelt p0.s, %[outpos], %[outwidth]\n"
"zip2 z1.s, z8.s, z12.s\n"
+ "incw %[outpos], all, mul #1\n"
"zip1 z2.s, z9.s, z13.s\n"
- "whilelt p2.s, %[outpos], %[outwidth]\n"
+ "whilelt p1.s, %[outpos], %[outwidth]\n"
"zip2 z3.s, z9.s, z13.s\n"
"incw %[outpos], all, mul #1\n"
"zip1 z4.s, z10.s, z14.s\n"
+ "whilelt p2.s, %[outpos], %[outwidth]\n"
"zip2 z5.s, z10.s, z14.s\n"
+ "incw %[outpos], all, mul #1\n"
"zip1 z6.s, z11.s, z15.s\n"
- "zip2 z7.s, z11.s, z15.s\n"
"whilelt p3.s, %[outpos], %[outwidth]\n"
+ "zip2 z7.s, z11.s, z15.s\n"
+ "incw %[outpos], all, mul #1\n"
"zip1 z8.s, z0.s, z4.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip2 z9.s, z0.s, z4.s\n"
- "zip1 z10.s, z1.s, z5.s\n"
- "zip2 z11.s, z1.s, z5.s\n"
"st1w z8.s, p0, [%[outptr]]\n"
- "zip1 z12.s, z2.s, z6.s\n"
- "whilelt p4.s, %[outpos], %[outwidth]\n"
- "zip2 z13.s, z2.s, z6.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip1 z14.s, z3.s, z7.s\n"
+ "zip2 z9.s, z0.s, z4.s\n"
"st1w z9.s, p1, [%[outptr], #1, MUL VL]\n"
- "zip2 z15.s, z3.s, z7.s\n"
- "whilelt p5.s, %[outpos], %[outwidth]\n"
+ "zip1 z10.s, z1.s, z5.s\n"
"st1w z10.s, p2, [%[outptr], #2, MUL VL]\n"
- "incw %[outpos], all, mul #1\n"
+ "zip2 z11.s, z1.s, z5.s\n"
"st1w z11.s, p3, [%[outptr], #3, MUL VL]\n"
- "whilelt p6.s, %[outpos], %[outwidth]\n"
+ "zip1 z12.s, z2.s, z6.s\n"
+ "whilelt p0.s, %[outpos], %[outwidth]\n"
+ "zip2 z13.s, z2.s, z6.s\n"
+ "st1w z12.s, p0, [%[outptr], #4, MUL VL]\n"
+ "zip1 z14.s, z3.s, z7.s\n"
"incw %[outpos], all, mul #1\n"
- "st1w z12.s, p4, [%[outptr], #4, MUL VL]\n"
- "whilelt p7.s, %[outpos], %[outwidth]\n"
+ "zip2 z15.s, z3.s, z7.s\n"
+ "whilelt p1.s, %[outpos], %[outwidth]\n"
+ "st1w z13.s, p1, [%[outptr], #5, MUL VL]\n"
"incw %[outpos], all, mul #1\n"
- "st1w z13.s, p5, [%[outptr], #5, MUL VL]\n"
- "st1w z14.s, p6, [%[outptr], #6, MUL VL]\n"
- "st1w z15.s, p7, [%[outptr], #7, MUL VL]\n"
+ "whilelt p2.s, %[outpos], %[outwidth]\n"
+ "st1w z14.s, p2, [%[outptr], #6, MUL VL]\n"
+ "incw %[outpos], all, mul #1\n"
+ "whilelt p3.s, %[outpos], %[outwidth]\n"
+ "st1w z15.s, p3, [%[outptr], #7, MUL VL]\n"
+ "incw %[outpos], all, mul #1\n"
"addvl %[outptr], %[outptr], #8\n"
"b 1b\n"
"2:\n"
: [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3), [inptr4] "+r" (inptr4), [inptr5] "+r" (inptr5), [inptr6] "+r" (inptr6)
: [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
- : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
+ : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
);
break;
@@ -522,69 +550,77 @@
"1:\n"
"whilelt p0.s, %[inpos], %[inwidth]\n"
"b.none 2f\n"
- "ld1w z0.s, p0/z, [%[inptr0], %[inpos], LSL #2]\n"
- "ld1w z1.s, p0/z, [%[inptr1], %[inpos], LSL #2]\n"
- "ld1w z2.s, p0/z, [%[inptr2], %[inpos], LSL #2]\n"
- "ld1w z3.s, p0/z, [%[inptr3], %[inpos], LSL #2]\n"
- "ld1w z4.s, p0/z, [%[inptr4], %[inpos], LSL #2]\n"
- "ld1w z5.s, p0/z, [%[inptr5], %[inpos], LSL #2]\n"
- "ld1w z6.s, p0/z, [%[inptr6], %[inpos], LSL #2]\n"
- "ld1w z7.s, p0/z, [%[inptr7], %[inpos], LSL #2]\n"
+ "ld1w z0.s, p0/z, [%[inptr0]]\n"
"incw %[inpos], all, mul #1\n"
+ "ld1w z1.s, p0/z, [%[inptr1]]\n"
+ "addvl %[inptr0], %[inptr0], #1\n"
+ "ld1w z2.s, p0/z, [%[inptr2]]\n"
+ "addvl %[inptr1], %[inptr1], #1\n"
+ "ld1w z3.s, p0/z, [%[inptr3]]\n"
+ "addvl %[inptr2], %[inptr2], #1\n"
+ "ld1w z4.s, p0/z, [%[inptr4]]\n"
+ "addvl %[inptr3], %[inptr3], #1\n"
"zip1 z8.s, z0.s, z4.s\n"
+ "ld1w z5.s, p0/z, [%[inptr5]]\n"
+ "zip2 z9.s, z0.s, z4.s\n"
+ "ld1w z6.s, p0/z, [%[inptr6]]\n"
+ "zip1 z10.s, z1.s, z5.s\n"
+ "ld1w z7.s, p0/z, [%[inptr7]]\n"
+ "zip2 z11.s, z1.s, z5.s\n"
+ "addvl %[inptr4], %[inptr4], #1\n"
+ "zip1 z12.s, z2.s, z6.s\n"
+ "addvl %[inptr5], %[inptr5], #1\n"
+ "zip2 z13.s, z2.s, z6.s\n"
+ "addvl %[inptr6], %[inptr6], #1\n"
+ "zip1 z14.s, z3.s, z7.s\n"
+ "addvl %[inptr7], %[inptr7], #1\n"
+ "zip2 z15.s, z3.s, z7.s\n"
"whilelt p0.s, %[outpos], %[outwidth]\n"
- "zip2 z9.s, z0.s, z4.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip1 z10.s, z1.s, z5.s\n"
- "zip2 z11.s, z1.s, z5.s\n"
- "zip1 z12.s, z2.s, z6.s\n"
- "zip2 z13.s, z2.s, z6.s\n"
- "whilelt p1.s, %[outpos], %[outwidth]\n"
- "zip1 z14.s, z3.s, z7.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip2 z15.s, z3.s, z7.s\n"
"zip1 z0.s, z8.s, z12.s\n"
+ "incw %[outpos], all, mul #1\n"
"zip2 z1.s, z8.s, z12.s\n"
+ "whilelt p1.s, %[outpos], %[outwidth]\n"
"zip1 z2.s, z9.s, z13.s\n"
- "whilelt p2.s, %[outpos], %[outwidth]\n"
+ "incw %[outpos], all, mul #1\n"
"zip2 z3.s, z9.s, z13.s\n"
- "incw %[outpos], all, mul #1\n"
+ "whilelt p2.s, %[outpos], %[outwidth]\n"
"zip1 z4.s, z10.s, z14.s\n"
+ "incw %[outpos], all, mul #1\n"
"zip2 z5.s, z10.s, z14.s\n"
- "zip1 z6.s, z11.s, z15.s\n"
- "zip2 z7.s, z11.s, z15.s\n"
"whilelt p3.s, %[outpos], %[outwidth]\n"
+ "zip1 z6.s, z11.s, z15.s\n"
+ "incw %[outpos], all, mul #1\n"
+ "zip2 z7.s, z11.s, z15.s\n"
"zip1 z8.s, z0.s, z4.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip2 z9.s, z0.s, z4.s\n"
- "zip1 z10.s, z1.s, z5.s\n"
- "zip2 z11.s, z1.s, z5.s\n"
"st1w z8.s, p0, [%[outptr]]\n"
- "zip1 z12.s, z2.s, z6.s\n"
- "whilelt p4.s, %[outpos], %[outwidth]\n"
- "zip2 z13.s, z2.s, z6.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip1 z14.s, z3.s, z7.s\n"
+ "zip2 z9.s, z0.s, z4.s\n"
"st1w z9.s, p1, [%[outptr], #1, MUL VL]\n"
- "zip2 z15.s, z3.s, z7.s\n"
- "whilelt p5.s, %[outpos], %[outwidth]\n"
+ "zip1 z10.s, z1.s, z5.s\n"
"st1w z10.s, p2, [%[outptr], #2, MUL VL]\n"
- "incw %[outpos], all, mul #1\n"
+ "zip2 z11.s, z1.s, z5.s\n"
"st1w z11.s, p3, [%[outptr], #3, MUL VL]\n"
- "whilelt p6.s, %[outpos], %[outwidth]\n"
+ "zip1 z12.s, z2.s, z6.s\n"
+ "whilelt p0.s, %[outpos], %[outwidth]\n"
+ "zip2 z13.s, z2.s, z6.s\n"
+ "st1w z12.s, p0, [%[outptr], #4, MUL VL]\n"
+ "zip1 z14.s, z3.s, z7.s\n"
"incw %[outpos], all, mul #1\n"
- "st1w z12.s, p4, [%[outptr], #4, MUL VL]\n"
- "whilelt p7.s, %[outpos], %[outwidth]\n"
+ "zip2 z15.s, z3.s, z7.s\n"
+ "whilelt p1.s, %[outpos], %[outwidth]\n"
+ "st1w z13.s, p1, [%[outptr], #5, MUL VL]\n"
"incw %[outpos], all, mul #1\n"
- "st1w z13.s, p5, [%[outptr], #5, MUL VL]\n"
- "st1w z14.s, p6, [%[outptr], #6, MUL VL]\n"
- "st1w z15.s, p7, [%[outptr], #7, MUL VL]\n"
+ "whilelt p2.s, %[outpos], %[outwidth]\n"
+ "st1w z14.s, p2, [%[outptr], #6, MUL VL]\n"
+ "incw %[outpos], all, mul #1\n"
+ "whilelt p3.s, %[outpos], %[outwidth]\n"
+ "st1w z15.s, p3, [%[outptr], #7, MUL VL]\n"
+ "incw %[outpos], all, mul #1\n"
"addvl %[outptr], %[outptr], #8\n"
"b 1b\n"
"2:\n"
: [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3), [inptr4] "+r" (inptr4), [inptr5] "+r" (inptr5), [inptr6] "+r" (inptr6), [inptr7] "+r" (inptr7)
: [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
- : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
+ : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
);
break;