COMPMID-2178: Update GEMM assembly code.
Perform offset reduction and requantization within the assembly wrapper.
Change-Id: I5d5b3e1f6f9ef4c71805362c57f88ff199c027a3
Signed-off-by: Georgios Pinitas <georgios.pinitas@arm.com>
Reviewed-on: https://review.mlplatform.org/c/1541
Comments-Addressed: Pablo Marquez <pablo.tello@arm.com>
Reviewed-by: Gian Marco Iodice <gianmarco.iodice@arm.com>
Comments-Addressed: Arm Jenkins <bsgcomp@arm.com>
Tested-by: Arm Jenkins <bsgcomp@arm.com>
diff --git a/src/core/NEON/kernels/arm_gemm/transforms/sve_interleave_8way_32bit.hpp b/src/core/NEON/kernels/arm_gemm/transforms/sve_interleave_8way_32bit.hpp
index 07c8219..881dc7b 100644
--- a/src/core/NEON/kernels/arm_gemm/transforms/sve_interleave_8way_32bit.hpp
+++ b/src/core/NEON/kernels/arm_gemm/transforms/sve_interleave_8way_32bit.hpp
@@ -41,7 +41,7 @@
long outpos = 0;
uint32_t *outptr = master_outptr;
- master_outptr += (outwidth * 1);
+ master_outptr += outwidth;
const uint32_t *inptr0 = inptr + y * ldin + k0;
const uint32_t *inptr1 = inptr0 + ldin;
@@ -60,53 +60,52 @@
"whilelt p0.s, %[inpos], %[inwidth]\n"
"b.none 2f\n"
"mov z4.s, #0\n"
- "ld1w z0.s, p0/z, [%[inptr0]]\n"
- "zip1 z8.s, z0.s, z4.s\n"
+ "ld1w z0.s, p0/z, [%[inptr0], %[inpos], LSL #2]\n"
"incw %[inpos], all, mul #1\n"
- "zip2 z9.s, z0.s, z4.s\n"
- "addvl %[inptr0], %[inptr0], #1\n"
- "zip1 z0.s, z8.s, z4.s\n"
"whilelt p0.s, %[outpos], %[outwidth]\n"
- "zip2 z1.s, z8.s, z4.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip1 z2.s, z9.s, z4.s\n"
- "whilelt p1.s, %[outpos], %[outwidth]\n"
- "zip2 z3.s, z9.s, z4.s\n"
"incw %[outpos], all, mul #1\n"
"zip1 z8.s, z0.s, z4.s\n"
- "st1w z8.s, p0, [%[outptr]]\n"
"zip2 z9.s, z0.s, z4.s\n"
- "st1w z9.s, p1, [%[outptr], #1, MUL VL]\n"
- "zip1 z10.s, z1.s, z4.s\n"
- "whilelt p2.s, %[outpos], %[outwidth]\n"
- "zip2 z11.s, z1.s, z4.s\n"
- "st1w z10.s, p2, [%[outptr], #2, MUL VL]\n"
- "zip1 z12.s, z2.s, z4.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip2 z13.s, z2.s, z4.s\n"
- "whilelt p3.s, %[outpos], %[outwidth]\n"
- "zip1 z14.s, z3.s, z4.s\n"
- "st1w z11.s, p3, [%[outptr], #3, MUL VL]\n"
- "zip2 z15.s, z3.s, z4.s\n"
- "incw %[outpos], all, mul #1\n"
- "whilelt p0.s, %[outpos], %[outwidth]\n"
- "st1w z12.s, p0, [%[outptr], #4, MUL VL]\n"
- "incw %[outpos], all, mul #1\n"
"whilelt p1.s, %[outpos], %[outwidth]\n"
- "st1w z13.s, p1, [%[outptr], #5, MUL VL]\n"
+ "zip1 z0.s, z8.s, z4.s\n"
"incw %[outpos], all, mul #1\n"
+ "zip2 z1.s, z8.s, z4.s\n"
+ "zip1 z2.s, z9.s, z4.s\n"
+ "zip2 z3.s, z9.s, z4.s\n"
"whilelt p2.s, %[outpos], %[outwidth]\n"
- "st1w z14.s, p2, [%[outptr], #6, MUL VL]\n"
+ "zip1 z8.s, z0.s, z4.s\n"
"incw %[outpos], all, mul #1\n"
+ "zip2 z9.s, z0.s, z4.s\n"
+ "zip1 z10.s, z1.s, z4.s\n"
+ "st1w z8.s, p0, [%[outptr]]\n"
+ "zip2 z11.s, z1.s, z4.s\n"
"whilelt p3.s, %[outpos], %[outwidth]\n"
- "st1w z15.s, p3, [%[outptr], #7, MUL VL]\n"
+ "zip1 z12.s, z2.s, z4.s\n"
+ "st1w z9.s, p1, [%[outptr], #1, MUL VL]\n"
+ "zip2 z13.s, z2.s, z4.s\n"
"incw %[outpos], all, mul #1\n"
+ "zip1 z14.s, z3.s, z4.s\n"
+ "st1w z10.s, p2, [%[outptr], #2, MUL VL]\n"
+ "zip2 z15.s, z3.s, z4.s\n"
+ "whilelt p4.s, %[outpos], %[outwidth]\n"
+ "st1w z11.s, p3, [%[outptr], #3, MUL VL]\n"
+ "incw %[outpos], all, mul #1\n"
+ "st1w z12.s, p4, [%[outptr], #4, MUL VL]\n"
+ "whilelt p5.s, %[outpos], %[outwidth]\n"
+ "incw %[outpos], all, mul #1\n"
+ "st1w z13.s, p5, [%[outptr], #5, MUL VL]\n"
+ "whilelt p6.s, %[outpos], %[outwidth]\n"
+ "incw %[outpos], all, mul #1\n"
+ "st1w z14.s, p6, [%[outptr], #6, MUL VL]\n"
+ "whilelt p7.s, %[outpos], %[outwidth]\n"
+ "incw %[outpos], all, mul #1\n"
+ "st1w z15.s, p7, [%[outptr], #7, MUL VL]\n"
"addvl %[outptr], %[outptr], #8\n"
"b 1b\n"
"2:\n"
: [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0)
: [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
- : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
+ : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
);
break;
@@ -116,62 +115,60 @@
"whilelt p0.s, %[inpos], %[inwidth]\n"
"b.none 2f\n"
"mov z4.s, #0\n"
- "ld1w z0.s, p0/z, [%[inptr0]]\n"
- "zip1 z8.s, z0.s, z4.s\n"
- "ld1w z1.s, p0/z, [%[inptr1]]\n"
- "zip2 z9.s, z0.s, z4.s\n"
- "incw %[inpos], all, mul #1\n"
- "zip1 z10.s, z1.s, z4.s\n"
- "addvl %[inptr0], %[inptr0], #1\n"
- "zip2 z11.s, z1.s, z4.s\n"
- "addvl %[inptr1], %[inptr1], #1\n"
- "zip1 z0.s, z8.s, z4.s\n"
- "whilelt p0.s, %[outpos], %[outwidth]\n"
- "zip2 z1.s, z8.s, z4.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip1 z2.s, z9.s, z4.s\n"
- "whilelt p1.s, %[outpos], %[outwidth]\n"
- "zip2 z3.s, z9.s, z4.s\n"
- "incw %[outpos], all, mul #1\n"
"mov z14.s, #0\n"
+ "ld1w z0.s, p0/z, [%[inptr0], %[inpos], LSL #2]\n"
+ "ld1w z1.s, p0/z, [%[inptr1], %[inpos], LSL #2]\n"
+ "incw %[inpos], all, mul #1\n"
+ "whilelt p0.s, %[outpos], %[outwidth]\n"
+ "zip1 z8.s, z0.s, z4.s\n"
+ "incw %[outpos], all, mul #1\n"
+ "zip2 z9.s, z0.s, z4.s\n"
+ "zip1 z10.s, z1.s, z4.s\n"
+ "zip2 z11.s, z1.s, z4.s\n"
+ "whilelt p1.s, %[outpos], %[outwidth]\n"
+ "zip1 z0.s, z8.s, z4.s\n"
+ "incw %[outpos], all, mul #1\n"
+ "zip2 z1.s, z8.s, z4.s\n"
+ "zip1 z2.s, z9.s, z4.s\n"
+ "zip2 z3.s, z9.s, z4.s\n"
"whilelt p2.s, %[outpos], %[outwidth]\n"
"zip1 z4.s, z10.s, z14.s\n"
"incw %[outpos], all, mul #1\n"
"zip2 z5.s, z10.s, z14.s\n"
- "whilelt p3.s, %[outpos], %[outwidth]\n"
"zip1 z6.s, z11.s, z14.s\n"
- "incw %[outpos], all, mul #1\n"
"zip2 z7.s, z11.s, z14.s\n"
- "zip1 z8.s, z0.s, z4.s\n"
- "st1w z8.s, p0, [%[outptr]]\n"
- "zip2 z9.s, z0.s, z4.s\n"
- "st1w z9.s, p1, [%[outptr], #1, MUL VL]\n"
- "zip1 z10.s, z1.s, z5.s\n"
- "st1w z10.s, p2, [%[outptr], #2, MUL VL]\n"
- "zip2 z11.s, z1.s, z5.s\n"
- "st1w z11.s, p3, [%[outptr], #3, MUL VL]\n"
- "zip1 z12.s, z2.s, z6.s\n"
- "whilelt p0.s, %[outpos], %[outwidth]\n"
- "zip2 z13.s, z2.s, z6.s\n"
- "st1w z12.s, p0, [%[outptr], #4, MUL VL]\n"
- "zip1 z14.s, z3.s, z7.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip2 z15.s, z3.s, z7.s\n"
- "whilelt p1.s, %[outpos], %[outwidth]\n"
- "st1w z13.s, p1, [%[outptr], #5, MUL VL]\n"
- "incw %[outpos], all, mul #1\n"
- "whilelt p2.s, %[outpos], %[outwidth]\n"
- "st1w z14.s, p2, [%[outptr], #6, MUL VL]\n"
- "incw %[outpos], all, mul #1\n"
"whilelt p3.s, %[outpos], %[outwidth]\n"
- "st1w z15.s, p3, [%[outptr], #7, MUL VL]\n"
+ "zip1 z8.s, z0.s, z4.s\n"
"incw %[outpos], all, mul #1\n"
+ "zip2 z9.s, z0.s, z4.s\n"
+ "zip1 z10.s, z1.s, z5.s\n"
+ "st1w z8.s, p0, [%[outptr]]\n"
+ "zip2 z11.s, z1.s, z5.s\n"
+ "whilelt p4.s, %[outpos], %[outwidth]\n"
+ "zip1 z12.s, z2.s, z6.s\n"
+ "st1w z9.s, p1, [%[outptr], #1, MUL VL]\n"
+ "zip2 z13.s, z2.s, z6.s\n"
+ "incw %[outpos], all, mul #1\n"
+ "zip1 z14.s, z3.s, z7.s\n"
+ "st1w z10.s, p2, [%[outptr], #2, MUL VL]\n"
+ "zip2 z15.s, z3.s, z7.s\n"
+ "whilelt p5.s, %[outpos], %[outwidth]\n"
+ "st1w z11.s, p3, [%[outptr], #3, MUL VL]\n"
+ "incw %[outpos], all, mul #1\n"
+ "st1w z12.s, p4, [%[outptr], #4, MUL VL]\n"
+ "whilelt p6.s, %[outpos], %[outwidth]\n"
+ "incw %[outpos], all, mul #1\n"
+ "st1w z13.s, p5, [%[outptr], #5, MUL VL]\n"
+ "whilelt p7.s, %[outpos], %[outwidth]\n"
+ "incw %[outpos], all, mul #1\n"
+ "st1w z14.s, p6, [%[outptr], #6, MUL VL]\n"
+ "st1w z15.s, p7, [%[outptr], #7, MUL VL]\n"
"addvl %[outptr], %[outptr], #8\n"
"b 1b\n"
"2:\n"
: [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1)
: [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
- : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
+ : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
);
break;
@@ -181,66 +178,63 @@
"whilelt p0.s, %[inpos], %[inwidth]\n"
"b.none 2f\n"
"mov z4.s, #0\n"
- "ld1w z0.s, p0/z, [%[inptr0]]\n"
- "zip1 z8.s, z0.s, z4.s\n"
- "ld1w z1.s, p0/z, [%[inptr1]]\n"
- "zip2 z9.s, z0.s, z4.s\n"
- "ld1w z2.s, p0/z, [%[inptr2]]\n"
- "zip1 z10.s, z1.s, z4.s\n"
+ "mov z14.s, #0\n"
+ "ld1w z0.s, p0/z, [%[inptr0], %[inpos], LSL #2]\n"
+ "ld1w z1.s, p0/z, [%[inptr1], %[inpos], LSL #2]\n"
+ "ld1w z2.s, p0/z, [%[inptr2], %[inpos], LSL #2]\n"
"incw %[inpos], all, mul #1\n"
- "zip2 z11.s, z1.s, z4.s\n"
- "addvl %[inptr0], %[inptr0], #1\n"
- "zip1 z12.s, z2.s, z4.s\n"
- "addvl %[inptr1], %[inptr1], #1\n"
- "zip2 z13.s, z2.s, z4.s\n"
- "addvl %[inptr2], %[inptr2], #1\n"
- "zip1 z0.s, z8.s, z12.s\n"
+ "zip1 z8.s, z0.s, z4.s\n"
"whilelt p0.s, %[outpos], %[outwidth]\n"
- "zip2 z1.s, z8.s, z12.s\n"
+ "zip2 z9.s, z0.s, z4.s\n"
"incw %[outpos], all, mul #1\n"
- "zip1 z2.s, z9.s, z13.s\n"
+ "zip1 z10.s, z1.s, z4.s\n"
+ "zip2 z11.s, z1.s, z4.s\n"
+ "zip1 z12.s, z2.s, z4.s\n"
"whilelt p1.s, %[outpos], %[outwidth]\n"
+ "zip2 z13.s, z2.s, z4.s\n"
+ "incw %[outpos], all, mul #1\n"
+ "zip1 z0.s, z8.s, z12.s\n"
+ "zip2 z1.s, z8.s, z12.s\n"
+ "zip1 z2.s, z9.s, z13.s\n"
+ "whilelt p2.s, %[outpos], %[outwidth]\n"
"zip2 z3.s, z9.s, z13.s\n"
"incw %[outpos], all, mul #1\n"
- "mov z14.s, #0\n"
- "whilelt p2.s, %[outpos], %[outwidth]\n"
"zip1 z4.s, z10.s, z14.s\n"
- "incw %[outpos], all, mul #1\n"
"zip2 z5.s, z10.s, z14.s\n"
- "whilelt p3.s, %[outpos], %[outwidth]\n"
"zip1 z6.s, z11.s, z14.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip2 z7.s, z11.s, z14.s\n"
- "zip1 z8.s, z0.s, z4.s\n"
- "st1w z8.s, p0, [%[outptr]]\n"
- "zip2 z9.s, z0.s, z4.s\n"
- "st1w z9.s, p1, [%[outptr], #1, MUL VL]\n"
- "zip1 z10.s, z1.s, z5.s\n"
- "st1w z10.s, p2, [%[outptr], #2, MUL VL]\n"
- "zip2 z11.s, z1.s, z5.s\n"
- "st1w z11.s, p3, [%[outptr], #3, MUL VL]\n"
- "zip1 z12.s, z2.s, z6.s\n"
- "whilelt p0.s, %[outpos], %[outwidth]\n"
- "zip2 z13.s, z2.s, z6.s\n"
- "st1w z12.s, p0, [%[outptr], #4, MUL VL]\n"
- "zip1 z14.s, z3.s, z7.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip2 z15.s, z3.s, z7.s\n"
- "whilelt p1.s, %[outpos], %[outwidth]\n"
- "st1w z13.s, p1, [%[outptr], #5, MUL VL]\n"
- "incw %[outpos], all, mul #1\n"
- "whilelt p2.s, %[outpos], %[outwidth]\n"
- "st1w z14.s, p2, [%[outptr], #6, MUL VL]\n"
- "incw %[outpos], all, mul #1\n"
"whilelt p3.s, %[outpos], %[outwidth]\n"
- "st1w z15.s, p3, [%[outptr], #7, MUL VL]\n"
+ "zip2 z7.s, z11.s, z14.s\n"
"incw %[outpos], all, mul #1\n"
+ "zip1 z8.s, z0.s, z4.s\n"
+ "zip2 z9.s, z0.s, z4.s\n"
+ "zip1 z10.s, z1.s, z5.s\n"
+ "whilelt p4.s, %[outpos], %[outwidth]\n"
+ "zip2 z11.s, z1.s, z5.s\n"
+ "st1w z8.s, p0, [%[outptr]]\n"
+ "zip1 z12.s, z2.s, z6.s\n"
+ "incw %[outpos], all, mul #1\n"
+ "zip2 z13.s, z2.s, z6.s\n"
+ "st1w z9.s, p1, [%[outptr], #1, MUL VL]\n"
+ "zip1 z14.s, z3.s, z7.s\n"
+ "zip2 z15.s, z3.s, z7.s\n"
+ "whilelt p5.s, %[outpos], %[outwidth]\n"
+ "st1w z10.s, p2, [%[outptr], #2, MUL VL]\n"
+ "incw %[outpos], all, mul #1\n"
+ "st1w z11.s, p3, [%[outptr], #3, MUL VL]\n"
+ "whilelt p6.s, %[outpos], %[outwidth]\n"
+ "incw %[outpos], all, mul #1\n"
+ "st1w z12.s, p4, [%[outptr], #4, MUL VL]\n"
+ "whilelt p7.s, %[outpos], %[outwidth]\n"
+ "incw %[outpos], all, mul #1\n"
+ "st1w z13.s, p5, [%[outptr], #5, MUL VL]\n"
+ "st1w z14.s, p6, [%[outptr], #6, MUL VL]\n"
+ "st1w z15.s, p7, [%[outptr], #7, MUL VL]\n"
"addvl %[outptr], %[outptr], #8\n"
"b 1b\n"
"2:\n"
: [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2)
: [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
- : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
+ : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
);
break;
@@ -250,69 +244,65 @@
"whilelt p0.s, %[inpos], %[inwidth]\n"
"b.none 2f\n"
"mov z4.s, #0\n"
- "ld1w z0.s, p0/z, [%[inptr0]]\n"
- "zip1 z8.s, z0.s, z4.s\n"
- "ld1w z1.s, p0/z, [%[inptr1]]\n"
- "zip2 z9.s, z0.s, z4.s\n"
- "ld1w z2.s, p0/z, [%[inptr2]]\n"
- "zip1 z10.s, z1.s, z4.s\n"
- "ld1w z3.s, p0/z, [%[inptr3]]\n"
- "zip2 z11.s, z1.s, z4.s\n"
+ "ld1w z0.s, p0/z, [%[inptr0], %[inpos], LSL #2]\n"
+ "ld1w z1.s, p0/z, [%[inptr1], %[inpos], LSL #2]\n"
+ "ld1w z2.s, p0/z, [%[inptr2], %[inpos], LSL #2]\n"
+ "ld1w z3.s, p0/z, [%[inptr3], %[inpos], LSL #2]\n"
"incw %[inpos], all, mul #1\n"
- "zip1 z12.s, z2.s, z4.s\n"
- "addvl %[inptr0], %[inptr0], #1\n"
- "zip2 z13.s, z2.s, z4.s\n"
- "addvl %[inptr1], %[inptr1], #1\n"
- "zip1 z14.s, z3.s, z4.s\n"
- "addvl %[inptr2], %[inptr2], #1\n"
- "zip2 z15.s, z3.s, z4.s\n"
- "addvl %[inptr3], %[inptr3], #1\n"
- "zip1 z0.s, z8.s, z12.s\n"
+ "zip1 z8.s, z0.s, z4.s\n"
"whilelt p0.s, %[outpos], %[outwidth]\n"
+ "zip2 z9.s, z0.s, z4.s\n"
+ "incw %[outpos], all, mul #1\n"
+ "zip1 z10.s, z1.s, z4.s\n"
+ "zip2 z11.s, z1.s, z4.s\n"
+ "zip1 z12.s, z2.s, z4.s\n"
+ "whilelt p1.s, %[outpos], %[outwidth]\n"
+ "zip2 z13.s, z2.s, z4.s\n"
+ "incw %[outpos], all, mul #1\n"
+ "zip1 z14.s, z3.s, z4.s\n"
+ "zip2 z15.s, z3.s, z4.s\n"
+ "zip1 z0.s, z8.s, z12.s\n"
+ "whilelt p2.s, %[outpos], %[outwidth]\n"
"zip2 z1.s, z8.s, z12.s\n"
"incw %[outpos], all, mul #1\n"
"zip1 z2.s, z9.s, z13.s\n"
- "whilelt p1.s, %[outpos], %[outwidth]\n"
"zip2 z3.s, z9.s, z13.s\n"
- "incw %[outpos], all, mul #1\n"
"zip1 z4.s, z10.s, z14.s\n"
- "whilelt p2.s, %[outpos], %[outwidth]\n"
+ "whilelt p3.s, %[outpos], %[outwidth]\n"
"zip2 z5.s, z10.s, z14.s\n"
"incw %[outpos], all, mul #1\n"
"zip1 z6.s, z11.s, z15.s\n"
- "whilelt p3.s, %[outpos], %[outwidth]\n"
"zip2 z7.s, z11.s, z15.s\n"
- "incw %[outpos], all, mul #1\n"
"zip1 z8.s, z0.s, z4.s\n"
- "st1w z8.s, p0, [%[outptr]]\n"
+ "whilelt p4.s, %[outpos], %[outwidth]\n"
"zip2 z9.s, z0.s, z4.s\n"
- "st1w z9.s, p1, [%[outptr], #1, MUL VL]\n"
+ "incw %[outpos], all, mul #1\n"
"zip1 z10.s, z1.s, z5.s\n"
- "st1w z10.s, p2, [%[outptr], #2, MUL VL]\n"
+ "st1w z8.s, p0, [%[outptr]]\n"
"zip2 z11.s, z1.s, z5.s\n"
- "st1w z11.s, p3, [%[outptr], #3, MUL VL]\n"
"zip1 z12.s, z2.s, z6.s\n"
- "whilelt p0.s, %[outpos], %[outwidth]\n"
+ "whilelt p5.s, %[outpos], %[outwidth]\n"
"zip2 z13.s, z2.s, z6.s\n"
- "st1w z12.s, p0, [%[outptr], #4, MUL VL]\n"
+ "st1w z9.s, p1, [%[outptr], #1, MUL VL]\n"
"zip1 z14.s, z3.s, z7.s\n"
"incw %[outpos], all, mul #1\n"
"zip2 z15.s, z3.s, z7.s\n"
- "whilelt p1.s, %[outpos], %[outwidth]\n"
- "st1w z13.s, p1, [%[outptr], #5, MUL VL]\n"
+ "st1w z10.s, p2, [%[outptr], #2, MUL VL]\n"
+ "whilelt p6.s, %[outpos], %[outwidth]\n"
+ "st1w z11.s, p3, [%[outptr], #3, MUL VL]\n"
"incw %[outpos], all, mul #1\n"
- "whilelt p2.s, %[outpos], %[outwidth]\n"
- "st1w z14.s, p2, [%[outptr], #6, MUL VL]\n"
+ "st1w z12.s, p4, [%[outptr], #4, MUL VL]\n"
+ "whilelt p7.s, %[outpos], %[outwidth]\n"
"incw %[outpos], all, mul #1\n"
- "whilelt p3.s, %[outpos], %[outwidth]\n"
- "st1w z15.s, p3, [%[outptr], #7, MUL VL]\n"
- "incw %[outpos], all, mul #1\n"
+ "st1w z13.s, p5, [%[outptr], #5, MUL VL]\n"
+ "st1w z14.s, p6, [%[outptr], #6, MUL VL]\n"
+ "st1w z15.s, p7, [%[outptr], #7, MUL VL]\n"
"addvl %[outptr], %[outptr], #8\n"
"b 1b\n"
"2:\n"
: [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3)
: [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
- : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
+ : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
);
break;
@@ -322,71 +312,66 @@
"whilelt p0.s, %[inpos], %[inwidth]\n"
"b.none 2f\n"
"mov z5.s, #0\n"
- "ld1w z0.s, p0/z, [%[inptr0]]\n"
- "ld1w z1.s, p0/z, [%[inptr1]]\n"
+ "ld1w z0.s, p0/z, [%[inptr0], %[inpos], LSL #2]\n"
+ "ld1w z1.s, p0/z, [%[inptr1], %[inpos], LSL #2]\n"
+ "ld1w z2.s, p0/z, [%[inptr2], %[inpos], LSL #2]\n"
+ "ld1w z3.s, p0/z, [%[inptr3], %[inpos], LSL #2]\n"
+ "ld1w z4.s, p0/z, [%[inptr4], %[inpos], LSL #2]\n"
"incw %[inpos], all, mul #1\n"
"zip1 z10.s, z1.s, z5.s\n"
- "ld1w z2.s, p0/z, [%[inptr2]]\n"
- "zip2 z11.s, z1.s, z5.s\n"
- "ld1w z3.s, p0/z, [%[inptr3]]\n"
- "zip1 z12.s, z2.s, z5.s\n"
- "ld1w z4.s, p0/z, [%[inptr4]]\n"
- "zip1 z8.s, z0.s, z4.s\n"
- "addvl %[inptr0], %[inptr0], #1\n"
- "zip2 z9.s, z0.s, z4.s\n"
- "addvl %[inptr1], %[inptr1], #1\n"
- "zip2 z13.s, z2.s, z5.s\n"
- "addvl %[inptr2], %[inptr2], #1\n"
- "zip1 z14.s, z3.s, z5.s\n"
- "addvl %[inptr3], %[inptr3], #1\n"
- "zip2 z15.s, z3.s, z5.s\n"
- "addvl %[inptr4], %[inptr4], #1\n"
- "zip1 z0.s, z8.s, z12.s\n"
"whilelt p0.s, %[outpos], %[outwidth]\n"
+ "zip1 z8.s, z0.s, z4.s\n"
+ "incw %[outpos], all, mul #1\n"
+ "zip2 z9.s, z0.s, z4.s\n"
+ "zip2 z11.s, z1.s, z5.s\n"
+ "zip1 z12.s, z2.s, z5.s\n"
+ "whilelt p1.s, %[outpos], %[outwidth]\n"
+ "zip2 z13.s, z2.s, z5.s\n"
+ "incw %[outpos], all, mul #1\n"
+ "zip1 z14.s, z3.s, z5.s\n"
+ "zip2 z15.s, z3.s, z5.s\n"
+ "zip1 z0.s, z8.s, z12.s\n"
+ "whilelt p2.s, %[outpos], %[outwidth]\n"
"zip2 z1.s, z8.s, z12.s\n"
"incw %[outpos], all, mul #1\n"
"zip1 z2.s, z9.s, z13.s\n"
- "whilelt p1.s, %[outpos], %[outwidth]\n"
"zip2 z3.s, z9.s, z13.s\n"
- "incw %[outpos], all, mul #1\n"
"zip1 z4.s, z10.s, z14.s\n"
- "whilelt p2.s, %[outpos], %[outwidth]\n"
+ "whilelt p3.s, %[outpos], %[outwidth]\n"
"zip2 z5.s, z10.s, z14.s\n"
"incw %[outpos], all, mul #1\n"
"zip1 z6.s, z11.s, z15.s\n"
- "whilelt p3.s, %[outpos], %[outwidth]\n"
"zip2 z7.s, z11.s, z15.s\n"
- "incw %[outpos], all, mul #1\n"
"zip1 z8.s, z0.s, z4.s\n"
- "st1w z8.s, p0, [%[outptr]]\n"
+ "whilelt p4.s, %[outpos], %[outwidth]\n"
"zip2 z9.s, z0.s, z4.s\n"
- "st1w z9.s, p1, [%[outptr], #1, MUL VL]\n"
+ "incw %[outpos], all, mul #1\n"
"zip1 z10.s, z1.s, z5.s\n"
- "st1w z10.s, p2, [%[outptr], #2, MUL VL]\n"
+ "st1w z8.s, p0, [%[outptr]]\n"
"zip2 z11.s, z1.s, z5.s\n"
- "st1w z11.s, p3, [%[outptr], #3, MUL VL]\n"
"zip1 z12.s, z2.s, z6.s\n"
- "whilelt p0.s, %[outpos], %[outwidth]\n"
+ "whilelt p5.s, %[outpos], %[outwidth]\n"
"zip2 z13.s, z2.s, z6.s\n"
- "st1w z12.s, p0, [%[outptr], #4, MUL VL]\n"
+ "st1w z9.s, p1, [%[outptr], #1, MUL VL]\n"
"zip1 z14.s, z3.s, z7.s\n"
"incw %[outpos], all, mul #1\n"
"zip2 z15.s, z3.s, z7.s\n"
- "whilelt p1.s, %[outpos], %[outwidth]\n"
- "st1w z13.s, p1, [%[outptr], #5, MUL VL]\n"
+ "st1w z10.s, p2, [%[outptr], #2, MUL VL]\n"
+ "whilelt p6.s, %[outpos], %[outwidth]\n"
+ "st1w z11.s, p3, [%[outptr], #3, MUL VL]\n"
"incw %[outpos], all, mul #1\n"
- "whilelt p2.s, %[outpos], %[outwidth]\n"
- "st1w z14.s, p2, [%[outptr], #6, MUL VL]\n"
+ "st1w z12.s, p4, [%[outptr], #4, MUL VL]\n"
+ "whilelt p7.s, %[outpos], %[outwidth]\n"
"incw %[outpos], all, mul #1\n"
- "whilelt p3.s, %[outpos], %[outwidth]\n"
- "st1w z15.s, p3, [%[outptr], #7, MUL VL]\n"
- "incw %[outpos], all, mul #1\n"
+ "st1w z13.s, p5, [%[outptr], #5, MUL VL]\n"
+ "st1w z14.s, p6, [%[outptr], #6, MUL VL]\n"
+ "st1w z15.s, p7, [%[outptr], #7, MUL VL]\n"
"addvl %[outptr], %[outptr], #8\n"
"b 1b\n"
"2:\n"
: [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3), [inptr4] "+r" (inptr4)
: [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
- : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
+ : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
);
break;
@@ -396,73 +381,67 @@
"whilelt p0.s, %[inpos], %[inwidth]\n"
"b.none 2f\n"
"mov z6.s, #0\n"
- "ld1w z0.s, p0/z, [%[inptr0]]\n"
- "ld1w z1.s, p0/z, [%[inptr1]]\n"
+ "ld1w z0.s, p0/z, [%[inptr0], %[inpos], LSL #2]\n"
+ "ld1w z1.s, p0/z, [%[inptr1], %[inpos], LSL #2]\n"
+ "ld1w z2.s, p0/z, [%[inptr2], %[inpos], LSL #2]\n"
+ "ld1w z3.s, p0/z, [%[inptr3], %[inpos], LSL #2]\n"
+ "ld1w z4.s, p0/z, [%[inptr4], %[inpos], LSL #2]\n"
+ "ld1w z5.s, p0/z, [%[inptr5], %[inpos], LSL #2]\n"
"incw %[inpos], all, mul #1\n"
- "ld1w z2.s, p0/z, [%[inptr2]]\n"
- "addvl %[inptr0], %[inptr0], #1\n"
"zip1 z12.s, z2.s, z6.s\n"
- "ld1w z3.s, p0/z, [%[inptr3]]\n"
- "zip2 z13.s, z2.s, z6.s\n"
- "ld1w z4.s, p0/z, [%[inptr4]]\n"
- "zip1 z8.s, z0.s, z4.s\n"
- "ld1w z5.s, p0/z, [%[inptr5]]\n"
- "zip2 z9.s, z0.s, z4.s\n"
- "addvl %[inptr1], %[inptr1], #1\n"
- "zip1 z10.s, z1.s, z5.s\n"
- "addvl %[inptr2], %[inptr2], #1\n"
- "zip2 z11.s, z1.s, z5.s\n"
- "addvl %[inptr3], %[inptr3], #1\n"
- "zip1 z14.s, z3.s, z6.s\n"
- "addvl %[inptr4], %[inptr4], #1\n"
- "zip2 z15.s, z3.s, z6.s\n"
- "addvl %[inptr5], %[inptr5], #1\n"
- "zip1 z0.s, z8.s, z12.s\n"
"whilelt p0.s, %[outpos], %[outwidth]\n"
+ "zip1 z8.s, z0.s, z4.s\n"
+ "incw %[outpos], all, mul #1\n"
+ "zip2 z9.s, z0.s, z4.s\n"
+ "zip1 z10.s, z1.s, z5.s\n"
+ "zip2 z11.s, z1.s, z5.s\n"
+ "whilelt p1.s, %[outpos], %[outwidth]\n"
+ "zip2 z13.s, z2.s, z6.s\n"
+ "incw %[outpos], all, mul #1\n"
+ "zip1 z14.s, z3.s, z6.s\n"
+ "zip2 z15.s, z3.s, z6.s\n"
+ "zip1 z0.s, z8.s, z12.s\n"
+ "whilelt p2.s, %[outpos], %[outwidth]\n"
"zip2 z1.s, z8.s, z12.s\n"
"incw %[outpos], all, mul #1\n"
"zip1 z2.s, z9.s, z13.s\n"
- "whilelt p1.s, %[outpos], %[outwidth]\n"
"zip2 z3.s, z9.s, z13.s\n"
- "incw %[outpos], all, mul #1\n"
"zip1 z4.s, z10.s, z14.s\n"
- "whilelt p2.s, %[outpos], %[outwidth]\n"
+ "whilelt p3.s, %[outpos], %[outwidth]\n"
"zip2 z5.s, z10.s, z14.s\n"
"incw %[outpos], all, mul #1\n"
"zip1 z6.s, z11.s, z15.s\n"
- "whilelt p3.s, %[outpos], %[outwidth]\n"
"zip2 z7.s, z11.s, z15.s\n"
- "incw %[outpos], all, mul #1\n"
"zip1 z8.s, z0.s, z4.s\n"
- "st1w z8.s, p0, [%[outptr]]\n"
+ "whilelt p4.s, %[outpos], %[outwidth]\n"
"zip2 z9.s, z0.s, z4.s\n"
- "st1w z9.s, p1, [%[outptr], #1, MUL VL]\n"
+ "incw %[outpos], all, mul #1\n"
"zip1 z10.s, z1.s, z5.s\n"
- "st1w z10.s, p2, [%[outptr], #2, MUL VL]\n"
+ "st1w z8.s, p0, [%[outptr]]\n"
"zip2 z11.s, z1.s, z5.s\n"
- "st1w z11.s, p3, [%[outptr], #3, MUL VL]\n"
"zip1 z12.s, z2.s, z6.s\n"
- "whilelt p0.s, %[outpos], %[outwidth]\n"
+ "whilelt p5.s, %[outpos], %[outwidth]\n"
"zip2 z13.s, z2.s, z6.s\n"
- "st1w z12.s, p0, [%[outptr], #4, MUL VL]\n"
+ "st1w z9.s, p1, [%[outptr], #1, MUL VL]\n"
"zip1 z14.s, z3.s, z7.s\n"
"incw %[outpos], all, mul #1\n"
"zip2 z15.s, z3.s, z7.s\n"
- "whilelt p1.s, %[outpos], %[outwidth]\n"
- "st1w z13.s, p1, [%[outptr], #5, MUL VL]\n"
+ "st1w z10.s, p2, [%[outptr], #2, MUL VL]\n"
+ "whilelt p6.s, %[outpos], %[outwidth]\n"
+ "st1w z11.s, p3, [%[outptr], #3, MUL VL]\n"
"incw %[outpos], all, mul #1\n"
- "whilelt p2.s, %[outpos], %[outwidth]\n"
- "st1w z14.s, p2, [%[outptr], #6, MUL VL]\n"
+ "st1w z12.s, p4, [%[outptr], #4, MUL VL]\n"
+ "whilelt p7.s, %[outpos], %[outwidth]\n"
"incw %[outpos], all, mul #1\n"
- "whilelt p3.s, %[outpos], %[outwidth]\n"
- "st1w z15.s, p3, [%[outptr], #7, MUL VL]\n"
- "incw %[outpos], all, mul #1\n"
+ "st1w z13.s, p5, [%[outptr], #5, MUL VL]\n"
+ "st1w z14.s, p6, [%[outptr], #6, MUL VL]\n"
+ "st1w z15.s, p7, [%[outptr], #7, MUL VL]\n"
"addvl %[outptr], %[outptr], #8\n"
"b 1b\n"
"2:\n"
: [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3), [inptr4] "+r" (inptr4), [inptr5] "+r" (inptr5)
: [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
- : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
+ : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
);
break;
@@ -472,75 +451,68 @@
"whilelt p0.s, %[inpos], %[inwidth]\n"
"b.none 2f\n"
"mov z7.s, #0\n"
- "ld1w z0.s, p0/z, [%[inptr0]]\n"
- "ld1w z1.s, p0/z, [%[inptr1]]\n"
+ "ld1w z0.s, p0/z, [%[inptr0], %[inpos], LSL #2]\n"
+ "ld1w z1.s, p0/z, [%[inptr1], %[inpos], LSL #2]\n"
+ "ld1w z2.s, p0/z, [%[inptr2], %[inpos], LSL #2]\n"
+ "ld1w z3.s, p0/z, [%[inptr3], %[inpos], LSL #2]\n"
+ "ld1w z4.s, p0/z, [%[inptr4], %[inpos], LSL #2]\n"
+ "ld1w z5.s, p0/z, [%[inptr5], %[inpos], LSL #2]\n"
+ "ld1w z6.s, p0/z, [%[inptr6], %[inpos], LSL #2]\n"
"incw %[inpos], all, mul #1\n"
- "ld1w z2.s, p0/z, [%[inptr2]]\n"
- "addvl %[inptr0], %[inptr0], #1\n"
- "ld1w z3.s, p0/z, [%[inptr3]]\n"
- "addvl %[inptr1], %[inptr1], #1\n"
- "zip1 z14.s, z3.s, z7.s\n"
- "ld1w z4.s, p0/z, [%[inptr4]]\n"
"zip1 z8.s, z0.s, z4.s\n"
- "ld1w z5.s, p0/z, [%[inptr5]]\n"
- "zip2 z9.s, z0.s, z4.s\n"
- "ld1w z6.s, p0/z, [%[inptr6]]\n"
- "zip1 z10.s, z1.s, z5.s\n"
- "addvl %[inptr2], %[inptr2], #1\n"
- "zip2 z11.s, z1.s, z5.s\n"
- "addvl %[inptr3], %[inptr3], #1\n"
- "zip1 z12.s, z2.s, z6.s\n"
- "addvl %[inptr4], %[inptr4], #1\n"
- "zip2 z13.s, z2.s, z6.s\n"
- "addvl %[inptr5], %[inptr5], #1\n"
- "zip2 z15.s, z3.s, z7.s\n"
- "addvl %[inptr6], %[inptr6], #1\n"
- "zip1 z0.s, z8.s, z12.s\n"
"whilelt p0.s, %[outpos], %[outwidth]\n"
+ "zip2 z9.s, z0.s, z4.s\n"
+ "incw %[outpos], all, mul #1\n"
+ "zip1 z10.s, z1.s, z5.s\n"
+ "zip2 z11.s, z1.s, z5.s\n"
+ "zip1 z12.s, z2.s, z6.s\n"
+ "whilelt p1.s, %[outpos], %[outwidth]\n"
+ "zip2 z13.s, z2.s, z6.s\n"
+ "incw %[outpos], all, mul #1\n"
+ "zip1 z14.s, z3.s, z7.s\n"
+ "zip2 z15.s, z3.s, z7.s\n"
+ "zip1 z0.s, z8.s, z12.s\n"
+ "whilelt p2.s, %[outpos], %[outwidth]\n"
"zip2 z1.s, z8.s, z12.s\n"
"incw %[outpos], all, mul #1\n"
"zip1 z2.s, z9.s, z13.s\n"
- "whilelt p1.s, %[outpos], %[outwidth]\n"
"zip2 z3.s, z9.s, z13.s\n"
- "incw %[outpos], all, mul #1\n"
"zip1 z4.s, z10.s, z14.s\n"
- "whilelt p2.s, %[outpos], %[outwidth]\n"
+ "whilelt p3.s, %[outpos], %[outwidth]\n"
"zip2 z5.s, z10.s, z14.s\n"
"incw %[outpos], all, mul #1\n"
"zip1 z6.s, z11.s, z15.s\n"
- "whilelt p3.s, %[outpos], %[outwidth]\n"
"zip2 z7.s, z11.s, z15.s\n"
- "incw %[outpos], all, mul #1\n"
"zip1 z8.s, z0.s, z4.s\n"
- "st1w z8.s, p0, [%[outptr]]\n"
+ "whilelt p4.s, %[outpos], %[outwidth]\n"
"zip2 z9.s, z0.s, z4.s\n"
- "st1w z9.s, p1, [%[outptr], #1, MUL VL]\n"
+ "incw %[outpos], all, mul #1\n"
"zip1 z10.s, z1.s, z5.s\n"
- "st1w z10.s, p2, [%[outptr], #2, MUL VL]\n"
+ "st1w z8.s, p0, [%[outptr]]\n"
"zip2 z11.s, z1.s, z5.s\n"
- "st1w z11.s, p3, [%[outptr], #3, MUL VL]\n"
"zip1 z12.s, z2.s, z6.s\n"
- "whilelt p0.s, %[outpos], %[outwidth]\n"
+ "whilelt p5.s, %[outpos], %[outwidth]\n"
"zip2 z13.s, z2.s, z6.s\n"
- "st1w z12.s, p0, [%[outptr], #4, MUL VL]\n"
+ "st1w z9.s, p1, [%[outptr], #1, MUL VL]\n"
"zip1 z14.s, z3.s, z7.s\n"
"incw %[outpos], all, mul #1\n"
"zip2 z15.s, z3.s, z7.s\n"
- "whilelt p1.s, %[outpos], %[outwidth]\n"
- "st1w z13.s, p1, [%[outptr], #5, MUL VL]\n"
+ "st1w z10.s, p2, [%[outptr], #2, MUL VL]\n"
+ "whilelt p6.s, %[outpos], %[outwidth]\n"
+ "st1w z11.s, p3, [%[outptr], #3, MUL VL]\n"
"incw %[outpos], all, mul #1\n"
- "whilelt p2.s, %[outpos], %[outwidth]\n"
- "st1w z14.s, p2, [%[outptr], #6, MUL VL]\n"
+ "st1w z12.s, p4, [%[outptr], #4, MUL VL]\n"
+ "whilelt p7.s, %[outpos], %[outwidth]\n"
"incw %[outpos], all, mul #1\n"
- "whilelt p3.s, %[outpos], %[outwidth]\n"
- "st1w z15.s, p3, [%[outptr], #7, MUL VL]\n"
- "incw %[outpos], all, mul #1\n"
+ "st1w z13.s, p5, [%[outptr], #5, MUL VL]\n"
+ "st1w z14.s, p6, [%[outptr], #6, MUL VL]\n"
+ "st1w z15.s, p7, [%[outptr], #7, MUL VL]\n"
"addvl %[outptr], %[outptr], #8\n"
"b 1b\n"
"2:\n"
: [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3), [inptr4] "+r" (inptr4), [inptr5] "+r" (inptr5), [inptr6] "+r" (inptr6)
: [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
- : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
+ : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
);
break;
@@ -550,77 +522,69 @@
"1:\n"
"whilelt p0.s, %[inpos], %[inwidth]\n"
"b.none 2f\n"
- "ld1w z0.s, p0/z, [%[inptr0]]\n"
+ "ld1w z0.s, p0/z, [%[inptr0], %[inpos], LSL #2]\n"
+ "ld1w z1.s, p0/z, [%[inptr1], %[inpos], LSL #2]\n"
+ "ld1w z2.s, p0/z, [%[inptr2], %[inpos], LSL #2]\n"
+ "ld1w z3.s, p0/z, [%[inptr3], %[inpos], LSL #2]\n"
+ "ld1w z4.s, p0/z, [%[inptr4], %[inpos], LSL #2]\n"
+ "ld1w z5.s, p0/z, [%[inptr5], %[inpos], LSL #2]\n"
+ "ld1w z6.s, p0/z, [%[inptr6], %[inpos], LSL #2]\n"
+ "ld1w z7.s, p0/z, [%[inptr7], %[inpos], LSL #2]\n"
"incw %[inpos], all, mul #1\n"
- "ld1w z1.s, p0/z, [%[inptr1]]\n"
- "addvl %[inptr0], %[inptr0], #1\n"
- "ld1w z2.s, p0/z, [%[inptr2]]\n"
- "addvl %[inptr1], %[inptr1], #1\n"
- "ld1w z3.s, p0/z, [%[inptr3]]\n"
- "addvl %[inptr2], %[inptr2], #1\n"
- "ld1w z4.s, p0/z, [%[inptr4]]\n"
- "addvl %[inptr3], %[inptr3], #1\n"
"zip1 z8.s, z0.s, z4.s\n"
- "ld1w z5.s, p0/z, [%[inptr5]]\n"
- "zip2 z9.s, z0.s, z4.s\n"
- "ld1w z6.s, p0/z, [%[inptr6]]\n"
- "zip1 z10.s, z1.s, z5.s\n"
- "ld1w z7.s, p0/z, [%[inptr7]]\n"
- "zip2 z11.s, z1.s, z5.s\n"
- "addvl %[inptr4], %[inptr4], #1\n"
- "zip1 z12.s, z2.s, z6.s\n"
- "addvl %[inptr5], %[inptr5], #1\n"
- "zip2 z13.s, z2.s, z6.s\n"
- "addvl %[inptr6], %[inptr6], #1\n"
- "zip1 z14.s, z3.s, z7.s\n"
- "addvl %[inptr7], %[inptr7], #1\n"
- "zip2 z15.s, z3.s, z7.s\n"
"whilelt p0.s, %[outpos], %[outwidth]\n"
- "zip1 z0.s, z8.s, z12.s\n"
+ "zip2 z9.s, z0.s, z4.s\n"
"incw %[outpos], all, mul #1\n"
- "zip2 z1.s, z8.s, z12.s\n"
+ "zip1 z10.s, z1.s, z5.s\n"
+ "zip2 z11.s, z1.s, z5.s\n"
+ "zip1 z12.s, z2.s, z6.s\n"
"whilelt p1.s, %[outpos], %[outwidth]\n"
- "zip1 z2.s, z9.s, z13.s\n"
+ "zip2 z13.s, z2.s, z6.s\n"
"incw %[outpos], all, mul #1\n"
- "zip2 z3.s, z9.s, z13.s\n"
+ "zip1 z14.s, z3.s, z7.s\n"
+ "zip2 z15.s, z3.s, z7.s\n"
+ "zip1 z0.s, z8.s, z12.s\n"
"whilelt p2.s, %[outpos], %[outwidth]\n"
+ "zip2 z1.s, z8.s, z12.s\n"
+ "incw %[outpos], all, mul #1\n"
+ "zip1 z2.s, z9.s, z13.s\n"
+ "zip2 z3.s, z9.s, z13.s\n"
"zip1 z4.s, z10.s, z14.s\n"
- "incw %[outpos], all, mul #1\n"
- "zip2 z5.s, z10.s, z14.s\n"
"whilelt p3.s, %[outpos], %[outwidth]\n"
- "zip1 z6.s, z11.s, z15.s\n"
+ "zip2 z5.s, z10.s, z14.s\n"
"incw %[outpos], all, mul #1\n"
+ "zip1 z6.s, z11.s, z15.s\n"
"zip2 z7.s, z11.s, z15.s\n"
"zip1 z8.s, z0.s, z4.s\n"
- "st1w z8.s, p0, [%[outptr]]\n"
+ "whilelt p4.s, %[outpos], %[outwidth]\n"
"zip2 z9.s, z0.s, z4.s\n"
- "st1w z9.s, p1, [%[outptr], #1, MUL VL]\n"
+ "incw %[outpos], all, mul #1\n"
"zip1 z10.s, z1.s, z5.s\n"
- "st1w z10.s, p2, [%[outptr], #2, MUL VL]\n"
+ "st1w z8.s, p0, [%[outptr]]\n"
"zip2 z11.s, z1.s, z5.s\n"
- "st1w z11.s, p3, [%[outptr], #3, MUL VL]\n"
"zip1 z12.s, z2.s, z6.s\n"
- "whilelt p0.s, %[outpos], %[outwidth]\n"
+ "whilelt p5.s, %[outpos], %[outwidth]\n"
"zip2 z13.s, z2.s, z6.s\n"
- "st1w z12.s, p0, [%[outptr], #4, MUL VL]\n"
+ "st1w z9.s, p1, [%[outptr], #1, MUL VL]\n"
"zip1 z14.s, z3.s, z7.s\n"
"incw %[outpos], all, mul #1\n"
"zip2 z15.s, z3.s, z7.s\n"
- "whilelt p1.s, %[outpos], %[outwidth]\n"
- "st1w z13.s, p1, [%[outptr], #5, MUL VL]\n"
+ "st1w z10.s, p2, [%[outptr], #2, MUL VL]\n"
+ "whilelt p6.s, %[outpos], %[outwidth]\n"
+ "st1w z11.s, p3, [%[outptr], #3, MUL VL]\n"
"incw %[outpos], all, mul #1\n"
- "whilelt p2.s, %[outpos], %[outwidth]\n"
- "st1w z14.s, p2, [%[outptr], #6, MUL VL]\n"
+ "st1w z12.s, p4, [%[outptr], #4, MUL VL]\n"
+ "whilelt p7.s, %[outpos], %[outwidth]\n"
"incw %[outpos], all, mul #1\n"
- "whilelt p3.s, %[outpos], %[outwidth]\n"
- "st1w z15.s, p3, [%[outptr], #7, MUL VL]\n"
- "incw %[outpos], all, mul #1\n"
+ "st1w z13.s, p5, [%[outptr], #5, MUL VL]\n"
+ "st1w z14.s, p6, [%[outptr], #6, MUL VL]\n"
+ "st1w z15.s, p7, [%[outptr], #7, MUL VL]\n"
"addvl %[outptr], %[outptr], #8\n"
"b 1b\n"
"2:\n"
: [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3), [inptr4] "+r" (inptr4), [inptr5] "+r" (inptr5), [inptr6] "+r" (inptr6), [inptr7] "+r" (inptr7)
: [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
- : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
+ : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
);
break;