Gitiles
Code Review Sign In
review.mlplatform.org / ml / ComputeLibrary / aaa9da1efa83911c7a67d50811ad669a92a7d12f / . / src / core / NEON / kernels / arm_gemm / merges / sve_merge_s32_3VLx8.hpp
blob: 115ba59459558964125884c3647f4e43184a7909 [file] [log] [blame]
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]1/*
David Mansellce79ac62022-09-23 09:57:43 +0100[diff] [blame]2 * Copyright (c) 2019-2020,2022 Arm Limited.
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]3 *
4 * SPDX-License-Identifier: MIT
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to
8 * deal in the Software without restriction, including without limitation the
9 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
10 * sell copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in all
14 * copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22 * SOFTWARE.
23 */
24#pragma once
25
Michalis Spyrou20fca522021-06-07 14:23:57 +0100[diff] [blame]26#ifdef ARM_COMPUTE_ENABLE_SVE
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]27
28template<>
Georgios Pinitas5aa1a0b2020-07-02 20:02:20 +0100[diff] [blame]29void MergeResults<3, 8, true>(int32_t *out, const int32_t *in, const int ldout, const int y0, const int ymax, const int x0, const int xmax, const int32_t *bias, Activation , bool append)
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]30{
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]31 const int32_t *inptr = in;
Georgios Pinitasc7b183a2020-03-06 18:12:09 +0000[diff] [blame]32 int32_t nullbias[192];
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]33
Georgios Pinitas5aa1a0b2020-07-02 20:02:20 +0100[diff] [blame]34
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]35 if (!append && !bias)
36 {
37 memset(nullbias, 0, (3 * get_vector_length<int32_t>() * sizeof(int32_t)));
38 }
39
40 for (int y=y0; y<ymax; y+=8)
41 {
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]42 int32_t *outptr0 = out + (y * ldout) + x0;
43 int32_t *outptr1 = outptr0 + ldout;
44 int32_t *outptr2 = outptr1 + ldout;
45 int32_t *outptr3 = outptr2 + ldout;
46 int32_t *outptr4 = outptr3 + ldout;
47 int32_t *outptr5 = outptr4 + ldout;
48 int32_t *outptr6 = outptr5 + ldout;
49 int32_t *outptr7 = outptr6 + ldout;
50
51 const int height = ymax - y;
52
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]53 for (int i=x0; i<xmax; i+=(3 * get_vector_length<int32_t>()))
54 {
55 if (append)
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]56 {
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]57 switch(height)
58 {
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]59 case 1:
60 {
61 long w = xmax - i;
62 long p = 0;
63 /* Optimized routine to copy an entire block */
64 __asm __volatile (
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]65 "addvl x8, %[inptr], #16\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]66 "whilelt p0.s, %[p], %[w]\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]67 "incw %[p], all, mul #1\n"
68 "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]69 "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
70 "ld1w z2.s, p0/z, [%[outptr0]]\n"
71 "whilelt p1.s, %[p], %[w]\n"
72 "ld1w z10.s, p0/z, [%[inptr]]\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]73 "incw %[p], all, mul #1\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]74 "ld1w z3.s, p1/z, [%[outptr0], #1, MUL VL]\n"
75 "add z10.s, z10.s, z2.s\n"
76 "ld1w z11.s, p1/z, [%[inptr], #1, MUL VL]\n"
77 "whilelt p2.s, %[p], %[w]\n"
78 "add z11.s, z11.s, z3.s\n"
79 "st1w z10.s, p0, [%[outptr0]]\n"
80 "ld1w z4.s, p2/z, [%[outptr0], #2, MUL VL]\n"
81 "ld1w z12.s, p2/z, [%[inptr], #2, MUL VL]\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]82 "addvl %[inptr], %[inptr], #24\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]83 "st1w z11.s, p1, [%[outptr0], #1, MUL VL]\n"
84 "add z12.s, z12.s, z4.s\n"
85 "st1w z12.s, p2, [%[outptr0], #2, MUL VL]\n"
86 "addvl %[outptr0], %[outptr0], #3\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]87 : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
88 [inptr] "+r" (inptr), [p] "+r" (p)
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]89 : [w] "r" (w)
90 : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]91 );
92 }
93 break;
94
95 case 2:
96 {
97 long w = xmax - i;
98 long p = 0;
99 /* Optimized routine to copy an entire block */
100 __asm __volatile (
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]101 "addvl x8, %[inptr], #16\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]102 "whilelt p0.s, %[p], %[w]\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]103 "incw %[p], all, mul #1\n"
104 "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]105 "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
106 "ld1w z2.s, p0/z, [%[outptr0]]\n"
107 "whilelt p1.s, %[p], %[w]\n"
108 "ld1w z10.s, p0/z, [%[inptr]]\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]109 "incw %[p], all, mul #1\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]110 "ld1w z5.s, p0/z, [%[outptr1]]\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]111 "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]112 "add z10.s, z10.s, z2.s\n"
113 "ld1w z3.s, p1/z, [%[outptr0], #1, MUL VL]\n"
114 "ld1w z11.s, p1/z, [%[inptr], #1, MUL VL]\n"
115 "whilelt p2.s, %[p], %[w]\n"
116 "ld1w z13.s, p0/z, [%[inptr], #3, MUL VL]\n"
117 "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
118 "add z11.s, z11.s, z3.s\n"
119 "st1w z10.s, p0, [%[outptr0]]\n"
120 "ld1w z4.s, p2/z, [%[outptr0], #2, MUL VL]\n"
121 "add z13.s, z13.s, z5.s\n"
122 "ld1w z12.s, p2/z, [%[inptr], #2, MUL VL]\n"
123 "ld1w z6.s, p1/z, [%[outptr1], #1, MUL VL]\n"
124 "st1w z11.s, p1, [%[outptr0], #1, MUL VL]\n"
125 "ld1w z14.s, p1/z, [%[inptr], #4, MUL VL]\n"
126 "add z12.s, z12.s, z4.s\n"
127 "ld1w z7.s, p2/z, [%[outptr1], #2, MUL VL]\n"
128 "ld1w z15.s, p2/z, [%[inptr], #5, MUL VL]\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]129 "addvl %[inptr], %[inptr], #24\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]130 "add z14.s, z14.s, z6.s\n"
131 "st1w z12.s, p2, [%[outptr0], #2, MUL VL]\n"
132 "addvl %[outptr0], %[outptr0], #3\n"
133 "add z15.s, z15.s, z7.s\n"
134 "st1w z13.s, p0, [%[outptr1]]\n"
135 "st1w z14.s, p1, [%[outptr1], #1, MUL VL]\n"
136 "st1w z15.s, p2, [%[outptr1], #2, MUL VL]\n"
137 "addvl %[outptr1], %[outptr1], #3\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]138 : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
139 [inptr] "+r" (inptr), [p] "+r" (p)
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]140 : [w] "r" (w)
141 : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]142 );
143 }
144 break;
145
146 case 3:
147 {
148 long w = xmax - i;
149 long p = 0;
150 /* Optimized routine to copy an entire block */
151 __asm __volatile (
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]152 "addvl x8, %[inptr], #16\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]153 "whilelt p0.s, %[p], %[w]\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]154 "incw %[p], all, mul #1\n"
155 "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]156 "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
157 "ld1w z2.s, p0/z, [%[outptr0]]\n"
158 "whilelt p1.s, %[p], %[w]\n"
159 "ld1w z10.s, p0/z, [%[inptr]]\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]160 "incw %[p], all, mul #1\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]161 "ld1w z5.s, p0/z, [%[outptr1]]\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]162 "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]163 "add z10.s, z10.s, z2.s\n"
164 "ld1w z3.s, p1/z, [%[outptr0], #1, MUL VL]\n"
165 "ld1w z11.s, p1/z, [%[inptr], #1, MUL VL]\n"
166 "whilelt p2.s, %[p], %[w]\n"
167 "ld1w z13.s, p0/z, [%[inptr], #3, MUL VL]\n"
168 "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
169 "add z11.s, z11.s, z3.s\n"
170 "st1w z10.s, p0, [%[outptr0]]\n"
171 "ld1w z4.s, p2/z, [%[outptr0], #2, MUL VL]\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]172 "prfm PLDL1KEEP, [%[inptr], #0x200]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]173 "add z13.s, z13.s, z5.s\n"
174 "st1w z11.s, p1, [%[outptr0], #1, MUL VL]\n"
175 "ld1w z12.s, p2/z, [%[inptr], #2, MUL VL]\n"
176 "prfm PLDL1KEEP, [%[outptr2], #0x60]\n"
177 "ld1w z6.s, p1/z, [%[outptr1], #1, MUL VL]\n"
178 "ld1w z14.s, p1/z, [%[inptr], #4, MUL VL]\n"
179 "add z12.s, z12.s, z4.s\n"
180 "ld1w z7.s, p2/z, [%[outptr1], #2, MUL VL]\n"
181 "ld1w z15.s, p2/z, [%[inptr], #5, MUL VL]\n"
182 "ld1w z8.s, p0/z, [%[outptr2]]\n"
183 "add z14.s, z14.s, z6.s\n"
184 "st1w z12.s, p2, [%[outptr0], #2, MUL VL]\n"
185 "ld1w z16.s, p0/z, [%[inptr], #6, MUL VL]\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]186 "addvl %[outptr0], %[outptr0], #3\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]187 "add z15.s, z15.s, z7.s\n"
188 "st1w z13.s, p0, [%[outptr1]]\n"
189 "ld1w z9.s, p1/z, [%[outptr2], #1, MUL VL]\n"
190 "add z16.s, z16.s, z8.s\n"
191 "ld1w z17.s, p1/z, [%[inptr], #7, MUL VL]\n"
192 "ld1w z2.s, p2/z, [%[outptr2], #2, MUL VL]\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]193 "addvl %[inptr], %[inptr], #24\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]194 "st1w z14.s, p1, [%[outptr1], #1, MUL VL]\n"
195 "add z17.s, z17.s, z9.s\n"
196 "ld1w z10.s, p2/z, [x8, #-8, MUL VL]\n"
197 "st1w z15.s, p2, [%[outptr1], #2, MUL VL]\n"
198 "addvl %[outptr1], %[outptr1], #3\n"
199 "add z10.s, z10.s, z2.s\n"
200 "st1w z16.s, p0, [%[outptr2]]\n"
201 "st1w z17.s, p1, [%[outptr2], #1, MUL VL]\n"
202 "st1w z10.s, p2, [%[outptr2], #2, MUL VL]\n"
203 "addvl %[outptr2], %[outptr2], #3\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]204 : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
205 [inptr] "+r" (inptr), [p] "+r" (p)
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]206 : [w] "r" (w)
207 : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]208 );
209 }
210 break;
211
212 case 4:
213 {
214 long w = xmax - i;
215 long p = 0;
216 /* Optimized routine to copy an entire block */
217 __asm __volatile (
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]218 "addvl x8, %[inptr], #16\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]219 "whilelt p0.s, %[p], %[w]\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]220 "incw %[p], all, mul #1\n"
221 "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]222 "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
223 "ld1w z2.s, p0/z, [%[outptr0]]\n"
224 "whilelt p1.s, %[p], %[w]\n"
225 "ld1w z10.s, p0/z, [%[inptr]]\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]226 "incw %[p], all, mul #1\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]227 "ld1w z5.s, p0/z, [%[outptr1]]\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]228 "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]229 "add z10.s, z10.s, z2.s\n"
230 "ld1w z3.s, p1/z, [%[outptr0], #1, MUL VL]\n"
231 "ld1w z11.s, p1/z, [%[inptr], #1, MUL VL]\n"
232 "whilelt p2.s, %[p], %[w]\n"
233 "ld1w z13.s, p0/z, [%[inptr], #3, MUL VL]\n"
234 "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
235 "add z11.s, z11.s, z3.s\n"
236 "st1w z10.s, p0, [%[outptr0]]\n"
237 "ld1w z4.s, p2/z, [%[outptr0], #2, MUL VL]\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]238 "prfm PLDL1KEEP, [%[inptr], #0x200]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]239 "add z13.s, z13.s, z5.s\n"
240 "st1w z11.s, p1, [%[outptr0], #1, MUL VL]\n"
241 "ld1w z12.s, p2/z, [%[inptr], #2, MUL VL]\n"
242 "prfm PLDL1KEEP, [%[outptr2], #0x60]\n"
243 "ld1w z6.s, p1/z, [%[outptr1], #1, MUL VL]\n"
244 "prfm PLDL1KEEP, [%[outptr3], #0x60]\n"
245 "add z12.s, z12.s, z4.s\n"
246 "ld1w z14.s, p1/z, [%[inptr], #4, MUL VL]\n"
247 "ld1w z7.s, p2/z, [%[outptr1], #2, MUL VL]\n"
248 "ld1w z15.s, p2/z, [%[inptr], #5, MUL VL]\n"
249 "ld1w z8.s, p0/z, [%[outptr2]]\n"
250 "add z14.s, z14.s, z6.s\n"
251 "st1w z12.s, p2, [%[outptr0], #2, MUL VL]\n"
252 "ld1w z16.s, p0/z, [%[inptr], #6, MUL VL]\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]253 "addvl %[outptr0], %[outptr0], #3\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]254 "add z15.s, z15.s, z7.s\n"
255 "st1w z13.s, p0, [%[outptr1]]\n"
256 "ld1w z9.s, p1/z, [%[outptr2], #1, MUL VL]\n"
257 "add z16.s, z16.s, z8.s\n"
258 "ld1w z17.s, p1/z, [%[inptr], #7, MUL VL]\n"
259 "ld1w z2.s, p2/z, [%[outptr2], #2, MUL VL]\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]260 "addvl %[inptr], %[inptr], #24\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]261 "st1w z14.s, p1, [%[outptr1], #1, MUL VL]\n"
262 "add z17.s, z17.s, z9.s\n"
263 "ld1w z10.s, p2/z, [x8, #-8, MUL VL]\n"
264 "ld1w z3.s, p0/z, [%[outptr3]]\n"
265 "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n"
266 "st1w z15.s, p2, [%[outptr1], #2, MUL VL]\n"
267 "addvl %[outptr1], %[outptr1], #3\n"
268 "add z10.s, z10.s, z2.s\n"
269 "ld1w z4.s, p1/z, [%[outptr3], #1, MUL VL]\n"
270 "add z11.s, z11.s, z3.s\n"
271 "st1w z16.s, p0, [%[outptr2]]\n"
272 "ld1w z12.s, p1/z, [x8, #-6, MUL VL]\n"
273 "ld1w z5.s, p2/z, [%[outptr3], #2, MUL VL]\n"
274 "ld1w z13.s, p2/z, [x8, #-5, MUL VL]\n"
275 "st1w z17.s, p1, [%[outptr2], #1, MUL VL]\n"
276 "add z12.s, z12.s, z4.s\n"
277 "add z13.s, z13.s, z5.s\n"
278 "st1w z10.s, p2, [%[outptr2], #2, MUL VL]\n"
279 "addvl %[outptr2], %[outptr2], #3\n"
280 "st1w z11.s, p0, [%[outptr3]]\n"
281 "st1w z12.s, p1, [%[outptr3], #1, MUL VL]\n"
282 "st1w z13.s, p2, [%[outptr3], #2, MUL VL]\n"
283 "addvl %[outptr3], %[outptr3], #3\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]284 : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
285 [inptr] "+r" (inptr), [p] "+r" (p)
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]286 : [w] "r" (w)
287 : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]288 );
289 }
290 break;
291
292 case 5:
293 {
294 long w = xmax - i;
295 long p = 0;
296 /* Optimized routine to copy an entire block */
297 __asm __volatile (
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]298 "addvl x8, %[inptr], #16\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]299 "whilelt p0.s, %[p], %[w]\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]300 "incw %[p], all, mul #1\n"
301 "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]302 "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
303 "ld1w z2.s, p0/z, [%[outptr0]]\n"
304 "whilelt p1.s, %[p], %[w]\n"
305 "ld1w z10.s, p0/z, [%[inptr]]\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]306 "incw %[p], all, mul #1\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]307 "ld1w z5.s, p0/z, [%[outptr1]]\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]308 "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]309 "add z10.s, z10.s, z2.s\n"
310 "ld1w z3.s, p1/z, [%[outptr0], #1, MUL VL]\n"
311 "ld1w z11.s, p1/z, [%[inptr], #1, MUL VL]\n"
312 "whilelt p2.s, %[p], %[w]\n"
313 "ld1w z13.s, p0/z, [%[inptr], #3, MUL VL]\n"
314 "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
315 "add z11.s, z11.s, z3.s\n"
316 "st1w z10.s, p0, [%[outptr0]]\n"
317 "ld1w z4.s, p2/z, [%[outptr0], #2, MUL VL]\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]318 "prfm PLDL1KEEP, [%[inptr], #0x200]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]319 "add z13.s, z13.s, z5.s\n"
320 "st1w z11.s, p1, [%[outptr0], #1, MUL VL]\n"
321 "ld1w z12.s, p2/z, [%[inptr], #2, MUL VL]\n"
322 "prfm PLDL1KEEP, [%[outptr2], #0x60]\n"
323 "ld1w z6.s, p1/z, [%[outptr1], #1, MUL VL]\n"
324 "prfm PLDL1KEEP, [%[outptr3], #0x60]\n"
325 "add z12.s, z12.s, z4.s\n"
326 "ld1w z14.s, p1/z, [%[inptr], #4, MUL VL]\n"
327 "ld1w z7.s, p2/z, [%[outptr1], #2, MUL VL]\n"
328 "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
329 "ld1w z15.s, p2/z, [%[inptr], #5, MUL VL]\n"
330 "prfm PLDL1KEEP, [%[outptr4], #0x60]\n"
331 "add z14.s, z14.s, z6.s\n"
332 "st1w z12.s, p2, [%[outptr0], #2, MUL VL]\n"
333 "ld1w z8.s, p0/z, [%[outptr2]]\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]334 "addvl %[outptr0], %[outptr0], #3\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]335 "add z15.s, z15.s, z7.s\n"
336 "st1w z13.s, p0, [%[outptr1]]\n"
337 "ld1w z16.s, p0/z, [%[inptr], #6, MUL VL]\n"
338 "ld1w z9.s, p1/z, [%[outptr2], #1, MUL VL]\n"
339 "ld1w z17.s, p1/z, [%[inptr], #7, MUL VL]\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]340 "addvl %[inptr], %[inptr], #24\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]341 "add z16.s, z16.s, z8.s\n"
342 "st1w z14.s, p1, [%[outptr1], #1, MUL VL]\n"
343 "ld1w z2.s, p2/z, [%[outptr2], #2, MUL VL]\n"
344 "add z17.s, z17.s, z9.s\n"
345 "ld1w z10.s, p2/z, [x8, #-8, MUL VL]\n"
346 "ld1w z3.s, p0/z, [%[outptr3]]\n"
347 "st1w z15.s, p2, [%[outptr1], #2, MUL VL]\n"
348 "addvl %[outptr1], %[outptr1], #3\n"
349 "add z10.s, z10.s, z2.s\n"
350 "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n"
351 "ld1w z4.s, p1/z, [%[outptr3], #1, MUL VL]\n"
352 "st1w z16.s, p0, [%[outptr2]]\n"
353 "ld1w z12.s, p1/z, [x8, #-6, MUL VL]\n"
354 "add z11.s, z11.s, z3.s\n"
355 "ld1w z5.s, p2/z, [%[outptr3], #2, MUL VL]\n"
356 "ld1w z13.s, p2/z, [x8, #-5, MUL VL]\n"
357 "st1w z17.s, p1, [%[outptr2], #1, MUL VL]\n"
358 "add z12.s, z12.s, z4.s\n"
359 "ld1w z6.s, p0/z, [%[outptr4]]\n"
360 "ld1w z14.s, p0/z, [x8, #-4, MUL VL]\n"
361 "add z13.s, z13.s, z5.s\n"
362 "st1w z10.s, p2, [%[outptr2], #2, MUL VL]\n"
363 "ld1w z7.s, p1/z, [%[outptr4], #1, MUL VL]\n"
364 "addvl %[outptr2], %[outptr2], #3\n"
365 "add z14.s, z14.s, z6.s\n"
366 "st1w z11.s, p0, [%[outptr3]]\n"
367 "ld1w z15.s, p1/z, [x8, #-3, MUL VL]\n"
368 "ld1w z8.s, p2/z, [%[outptr4], #2, MUL VL]\n"
369 "ld1w z16.s, p2/z, [x8, #-2, MUL VL]\n"
370 "st1w z12.s, p1, [%[outptr3], #1, MUL VL]\n"
371 "add z15.s, z15.s, z7.s\n"
372 "add z16.s, z16.s, z8.s\n"
373 "st1w z13.s, p2, [%[outptr3], #2, MUL VL]\n"
374 "addvl %[outptr3], %[outptr3], #3\n"
375 "st1w z14.s, p0, [%[outptr4]]\n"
376 "st1w z15.s, p1, [%[outptr4], #1, MUL VL]\n"
377 "st1w z16.s, p2, [%[outptr4], #2, MUL VL]\n"
378 "addvl %[outptr4], %[outptr4], #3\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]379 : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
380 [inptr] "+r" (inptr), [p] "+r" (p)
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]381 : [w] "r" (w)
382 : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]383 );
384 }
385 break;
386
387 case 6:
388 {
389 long w = xmax - i;
390 long p = 0;
391 /* Optimized routine to copy an entire block */
392 __asm __volatile (
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]393 "addvl x8, %[inptr], #16\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]394 "whilelt p0.s, %[p], %[w]\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]395 "incw %[p], all, mul #1\n"
396 "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]397 "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
398 "ld1w z2.s, p0/z, [%[outptr0]]\n"
399 "whilelt p1.s, %[p], %[w]\n"
400 "ld1w z10.s, p0/z, [%[inptr]]\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]401 "incw %[p], all, mul #1\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]402 "ld1w z5.s, p0/z, [%[outptr1]]\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]403 "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]404 "add z10.s, z10.s, z2.s\n"
405 "ld1w z3.s, p1/z, [%[outptr0], #1, MUL VL]\n"
406 "ld1w z11.s, p1/z, [%[inptr], #1, MUL VL]\n"
407 "whilelt p2.s, %[p], %[w]\n"
408 "ld1w z13.s, p0/z, [%[inptr], #3, MUL VL]\n"
409 "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
410 "add z11.s, z11.s, z3.s\n"
411 "st1w z10.s, p0, [%[outptr0]]\n"
412 "ld1w z4.s, p2/z, [%[outptr0], #2, MUL VL]\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]413 "prfm PLDL1KEEP, [%[inptr], #0x200]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]414 "add z13.s, z13.s, z5.s\n"
415 "st1w z11.s, p1, [%[outptr0], #1, MUL VL]\n"
416 "ld1w z12.s, p2/z, [%[inptr], #2, MUL VL]\n"
417 "prfm PLDL1KEEP, [%[outptr2], #0x60]\n"
418 "ld1w z6.s, p1/z, [%[outptr1], #1, MUL VL]\n"
419 "prfm PLDL1KEEP, [%[outptr3], #0x60]\n"
420 "add z12.s, z12.s, z4.s\n"
421 "ld1w z14.s, p1/z, [%[inptr], #4, MUL VL]\n"
422 "ld1w z7.s, p2/z, [%[outptr1], #2, MUL VL]\n"
423 "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
424 "ld1w z15.s, p2/z, [%[inptr], #5, MUL VL]\n"
425 "prfm PLDL1KEEP, [%[outptr4], #0x60]\n"
426 "add z14.s, z14.s, z6.s\n"
427 "st1w z12.s, p2, [%[outptr0], #2, MUL VL]\n"
428 "ld1w z8.s, p0/z, [%[outptr2]]\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]429 "addvl %[outptr0], %[outptr0], #3\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]430 "add z15.s, z15.s, z7.s\n"
431 "st1w z13.s, p0, [%[outptr1]]\n"
432 "ld1w z16.s, p0/z, [%[inptr], #6, MUL VL]\n"
433 "prfm PLDL1KEEP, [%[inptr], #0x280]\n"
434 "ld1w z9.s, p1/z, [%[outptr2], #1, MUL VL]\n"
435 "prfm PLDL1KEEP, [%[outptr5], #0x60]\n"
436 "add z16.s, z16.s, z8.s\n"
437 "st1w z14.s, p1, [%[outptr1], #1, MUL VL]\n"
438 "ld1w z17.s, p1/z, [%[inptr], #7, MUL VL]\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]439 "addvl %[inptr], %[inptr], #24\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]440 "ld1w z2.s, p2/z, [%[outptr2], #2, MUL VL]\n"
441 "st1w z15.s, p2, [%[outptr1], #2, MUL VL]\n"
442 "addvl %[outptr1], %[outptr1], #3\n"
443 "add z17.s, z17.s, z9.s\n"
444 "ld1w z10.s, p2/z, [x8, #-8, MUL VL]\n"
445 "ld1w z3.s, p0/z, [%[outptr3]]\n"
446 "st1w z16.s, p0, [%[outptr2]]\n"
447 "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n"
448 "add z10.s, z10.s, z2.s\n"
449 "ld1w z4.s, p1/z, [%[outptr3], #1, MUL VL]\n"
450 "ld1w z12.s, p1/z, [x8, #-6, MUL VL]\n"
451 "st1w z17.s, p1, [%[outptr2], #1, MUL VL]\n"
452 "add z11.s, z11.s, z3.s\n"
453 "ld1w z5.s, p2/z, [%[outptr3], #2, MUL VL]\n"
454 "ld1w z13.s, p2/z, [x8, #-5, MUL VL]\n"
455 "add z12.s, z12.s, z4.s\n"
456 "st1w z10.s, p2, [%[outptr2], #2, MUL VL]\n"
457 "ld1w z6.s, p0/z, [%[outptr4]]\n"
458 "addvl %[outptr2], %[outptr2], #3\n"
459 "add z13.s, z13.s, z5.s\n"
460 "st1w z11.s, p0, [%[outptr3]]\n"
461 "ld1w z14.s, p0/z, [x8, #-4, MUL VL]\n"
462 "ld1w z7.s, p1/z, [%[outptr4], #1, MUL VL]\n"
463 "ld1w z15.s, p1/z, [x8, #-3, MUL VL]\n"
464 "st1w z12.s, p1, [%[outptr3], #1, MUL VL]\n"
465 "add z14.s, z14.s, z6.s\n"
466 "ld1w z8.s, p2/z, [%[outptr4], #2, MUL VL]\n"
467 "ld1w z16.s, p2/z, [x8, #-2, MUL VL]\n"
468 "add z15.s, z15.s, z7.s\n"
469 "st1w z13.s, p2, [%[outptr3], #2, MUL VL]\n"
470 "ld1w z9.s, p0/z, [%[outptr5]]\n"
471 "addvl %[outptr3], %[outptr3], #3\n"
472 "add z16.s, z16.s, z8.s\n"
473 "st1w z14.s, p0, [%[outptr4]]\n"
474 "ld1w z17.s, p0/z, [x8, #-1, MUL VL]\n"
475 "ld1w z2.s, p1/z, [%[outptr5], #1, MUL VL]\n"
476 "ld1w z10.s, p1/z, [x8]\n"
477 "st1w z15.s, p1, [%[outptr4], #1, MUL VL]\n"
478 "add z17.s, z17.s, z9.s\n"
479 "ld1w z3.s, p2/z, [%[outptr5], #2, MUL VL]\n"
480 "ld1w z11.s, p2/z, [x8, #1, MUL VL]\n"
481 "add z10.s, z10.s, z2.s\n"
482 "st1w z16.s, p2, [%[outptr4], #2, MUL VL]\n"
483 "addvl %[outptr4], %[outptr4], #3\n"
484 "add z11.s, z11.s, z3.s\n"
485 "st1w z17.s, p0, [%[outptr5]]\n"
486 "st1w z10.s, p1, [%[outptr5], #1, MUL VL]\n"
487 "st1w z11.s, p2, [%[outptr5], #2, MUL VL]\n"
488 "addvl %[outptr5], %[outptr5], #3\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]489 : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
490 [inptr] "+r" (inptr), [p] "+r" (p)
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]491 : [w] "r" (w)
492 : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]493 );
494 }
495 break;
496
497 case 7:
498 {
499 long w = xmax - i;
500 long p = 0;
501 /* Optimized routine to copy an entire block */
502 __asm __volatile (
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]503 "addvl x8, %[inptr], #16\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]504 "whilelt p0.s, %[p], %[w]\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]505 "incw %[p], all, mul #1\n"
506 "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]507 "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
508 "ld1w z2.s, p0/z, [%[outptr0]]\n"
509 "whilelt p1.s, %[p], %[w]\n"
510 "ld1w z10.s, p0/z, [%[inptr]]\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]511 "incw %[p], all, mul #1\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]512 "ld1w z5.s, p0/z, [%[outptr1]]\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]513 "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]514 "add z10.s, z10.s, z2.s\n"
515 "ld1w z3.s, p1/z, [%[outptr0], #1, MUL VL]\n"
516 "ld1w z11.s, p1/z, [%[inptr], #1, MUL VL]\n"
517 "whilelt p2.s, %[p], %[w]\n"
518 "ld1w z13.s, p0/z, [%[inptr], #3, MUL VL]\n"
519 "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
520 "add z11.s, z11.s, z3.s\n"
521 "st1w z10.s, p0, [%[outptr0]]\n"
522 "ld1w z4.s, p2/z, [%[outptr0], #2, MUL VL]\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]523 "prfm PLDL1KEEP, [%[inptr], #0x200]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]524 "add z13.s, z13.s, z5.s\n"
525 "st1w z11.s, p1, [%[outptr0], #1, MUL VL]\n"
526 "ld1w z12.s, p2/z, [%[inptr], #2, MUL VL]\n"
527 "prfm PLDL1KEEP, [%[outptr2], #0x60]\n"
528 "ld1w z6.s, p1/z, [%[outptr1], #1, MUL VL]\n"
529 "prfm PLDL1KEEP, [%[outptr3], #0x60]\n"
530 "add z12.s, z12.s, z4.s\n"
531 "ld1w z14.s, p1/z, [%[inptr], #4, MUL VL]\n"
532 "ld1w z7.s, p2/z, [%[outptr1], #2, MUL VL]\n"
533 "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
534 "ld1w z15.s, p2/z, [%[inptr], #5, MUL VL]\n"
535 "prfm PLDL1KEEP, [%[outptr4], #0x60]\n"
536 "add z14.s, z14.s, z6.s\n"
537 "st1w z12.s, p2, [%[outptr0], #2, MUL VL]\n"
538 "ld1w z8.s, p0/z, [%[outptr2]]\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]539 "addvl %[outptr0], %[outptr0], #3\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]540 "add z15.s, z15.s, z7.s\n"
541 "st1w z13.s, p0, [%[outptr1]]\n"
542 "ld1w z16.s, p0/z, [%[inptr], #6, MUL VL]\n"
543 "prfm PLDL1KEEP, [%[inptr], #0x280]\n"
544 "ld1w z9.s, p1/z, [%[outptr2], #1, MUL VL]\n"
545 "prfm PLDL1KEEP, [%[outptr5], #0x60]\n"
546 "add z16.s, z16.s, z8.s\n"
547 "st1w z14.s, p1, [%[outptr1], #1, MUL VL]\n"
548 "ld1w z17.s, p1/z, [%[inptr], #7, MUL VL]\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]549 "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]550 "ld1w z2.s, p2/z, [%[outptr2], #2, MUL VL]\n"
551 "prfm PLDL1KEEP, [%[outptr6], #0x60]\n"
552 "add z17.s, z17.s, z9.s\n"
553 "st1w z15.s, p2, [%[outptr1], #2, MUL VL]\n"
554 "ld1w z10.s, p2/z, [x8, #-8, MUL VL]\n"
555 "addvl %[outptr1], %[outptr1], #3\n"
556 "ld1w z3.s, p0/z, [%[outptr3]]\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]557 "addvl %[inptr], %[inptr], #24\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]558 "add z10.s, z10.s, z2.s\n"
559 "st1w z16.s, p0, [%[outptr2]]\n"
560 "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n"
561 "ld1w z4.s, p1/z, [%[outptr3], #1, MUL VL]\n"
562 "ld1w z12.s, p1/z, [x8, #-6, MUL VL]\n"
563 "st1w z17.s, p1, [%[outptr2], #1, MUL VL]\n"
564 "add z11.s, z11.s, z3.s\n"
565 "ld1w z5.s, p2/z, [%[outptr3], #2, MUL VL]\n"
566 "ld1w z13.s, p2/z, [x8, #-5, MUL VL]\n"
567 "add z12.s, z12.s, z4.s\n"
568 "st1w z10.s, p2, [%[outptr2], #2, MUL VL]\n"
569 "ld1w z6.s, p0/z, [%[outptr4]]\n"
570 "addvl %[outptr2], %[outptr2], #3\n"
571 "add z13.s, z13.s, z5.s\n"
572 "st1w z11.s, p0, [%[outptr3]]\n"
573 "ld1w z14.s, p0/z, [x8, #-4, MUL VL]\n"
574 "ld1w z7.s, p1/z, [%[outptr4], #1, MUL VL]\n"
575 "ld1w z15.s, p1/z, [x8, #-3, MUL VL]\n"
576 "st1w z12.s, p1, [%[outptr3], #1, MUL VL]\n"
577 "add z14.s, z14.s, z6.s\n"
578 "ld1w z8.s, p2/z, [%[outptr4], #2, MUL VL]\n"
579 "ld1w z16.s, p2/z, [x8, #-2, MUL VL]\n"
580 "add z15.s, z15.s, z7.s\n"
581 "st1w z13.s, p2, [%[outptr3], #2, MUL VL]\n"
582 "ld1w z9.s, p0/z, [%[outptr5]]\n"
583 "addvl %[outptr3], %[outptr3], #3\n"
584 "add z16.s, z16.s, z8.s\n"
585 "st1w z14.s, p0, [%[outptr4]]\n"
586 "ld1w z17.s, p0/z, [x8, #-1, MUL VL]\n"
587 "ld1w z2.s, p1/z, [%[outptr5], #1, MUL VL]\n"
588 "ld1w z10.s, p1/z, [x8]\n"
589 "st1w z15.s, p1, [%[outptr4], #1, MUL VL]\n"
590 "add z17.s, z17.s, z9.s\n"
591 "ld1w z3.s, p2/z, [%[outptr5], #2, MUL VL]\n"
592 "ld1w z11.s, p2/z, [x8, #1, MUL VL]\n"
593 "add z10.s, z10.s, z2.s\n"
594 "st1w z16.s, p2, [%[outptr4], #2, MUL VL]\n"
595 "ld1w z4.s, p0/z, [%[outptr6]]\n"
596 "addvl %[outptr4], %[outptr4], #3\n"
597 "add z11.s, z11.s, z3.s\n"
598 "st1w z17.s, p0, [%[outptr5]]\n"
599 "ld1w z12.s, p0/z, [x8, #2, MUL VL]\n"
600 "ld1w z5.s, p1/z, [%[outptr6], #1, MUL VL]\n"
601 "ld1w z13.s, p1/z, [x8, #3, MUL VL]\n"
602 "st1w z10.s, p1, [%[outptr5], #1, MUL VL]\n"
603 "add z12.s, z12.s, z4.s\n"
604 "ld1w z6.s, p2/z, [%[outptr6], #2, MUL VL]\n"
605 "ld1w z14.s, p2/z, [x8, #4, MUL VL]\n"
606 "add z13.s, z13.s, z5.s\n"
607 "st1w z11.s, p2, [%[outptr5], #2, MUL VL]\n"
608 "addvl %[outptr5], %[outptr5], #3\n"
609 "add z14.s, z14.s, z6.s\n"
610 "st1w z12.s, p0, [%[outptr6]]\n"
611 "st1w z13.s, p1, [%[outptr6], #1, MUL VL]\n"
612 "st1w z14.s, p2, [%[outptr6], #2, MUL VL]\n"
613 "addvl %[outptr6], %[outptr6], #3\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]614 : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
615 [inptr] "+r" (inptr), [p] "+r" (p)
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]616 : [w] "r" (w)
617 : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]618 );
619 }
620 break;
621
622 default:
623 case 8:
624 {
625 long w = xmax - i;
626 long p = 0;
627 /* Optimized routine to copy an entire block */
628 __asm __volatile (
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]629 "addvl x8, %[inptr], #16\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]630 "whilelt p0.s, %[p], %[w]\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]631 "incw %[p], all, mul #1\n"
632 "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]633 "prfm PLDL1KEEP, [%[outptr0], #0x60]\n"
634 "ld1w z2.s, p0/z, [%[outptr0]]\n"
635 "whilelt p1.s, %[p], %[w]\n"
636 "ld1w z10.s, p0/z, [%[inptr]]\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]637 "incw %[p], all, mul #1\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]638 "ld1w z5.s, p0/z, [%[outptr1]]\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]639 "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]640 "add z10.s, z10.s, z2.s\n"
641 "ld1w z3.s, p1/z, [%[outptr0], #1, MUL VL]\n"
642 "ld1w z11.s, p1/z, [%[inptr], #1, MUL VL]\n"
643 "whilelt p2.s, %[p], %[w]\n"
644 "ld1w z13.s, p0/z, [%[inptr], #3, MUL VL]\n"
645 "prfm PLDL1KEEP, [%[outptr1], #0x60]\n"
646 "add z11.s, z11.s, z3.s\n"
647 "st1w z10.s, p0, [%[outptr0]]\n"
648 "ld1w z4.s, p2/z, [%[outptr0], #2, MUL VL]\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]649 "prfm PLDL1KEEP, [%[inptr], #0x200]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]650 "add z13.s, z13.s, z5.s\n"
651 "st1w z11.s, p1, [%[outptr0], #1, MUL VL]\n"
652 "ld1w z12.s, p2/z, [%[inptr], #2, MUL VL]\n"
653 "prfm PLDL1KEEP, [%[outptr2], #0x60]\n"
654 "ld1w z6.s, p1/z, [%[outptr1], #1, MUL VL]\n"
655 "prfm PLDL1KEEP, [%[outptr3], #0x60]\n"
656 "add z12.s, z12.s, z4.s\n"
657 "ld1w z14.s, p1/z, [%[inptr], #4, MUL VL]\n"
658 "ld1w z7.s, p2/z, [%[outptr1], #2, MUL VL]\n"
659 "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
660 "ld1w z15.s, p2/z, [%[inptr], #5, MUL VL]\n"
661 "prfm PLDL1KEEP, [%[outptr4], #0x60]\n"
662 "add z14.s, z14.s, z6.s\n"
663 "st1w z12.s, p2, [%[outptr0], #2, MUL VL]\n"
664 "ld1w z8.s, p0/z, [%[outptr2]]\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]665 "addvl %[outptr0], %[outptr0], #3\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]666 "add z15.s, z15.s, z7.s\n"
667 "st1w z13.s, p0, [%[outptr1]]\n"
668 "ld1w z16.s, p0/z, [%[inptr], #6, MUL VL]\n"
669 "prfm PLDL1KEEP, [%[inptr], #0x280]\n"
670 "ld1w z9.s, p1/z, [%[outptr2], #1, MUL VL]\n"
671 "prfm PLDL1KEEP, [%[outptr5], #0x60]\n"
672 "add z16.s, z16.s, z8.s\n"
673 "st1w z14.s, p1, [%[outptr1], #1, MUL VL]\n"
674 "ld1w z17.s, p1/z, [%[inptr], #7, MUL VL]\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]675 "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]676 "ld1w z2.s, p2/z, [%[outptr2], #2, MUL VL]\n"
677 "prfm PLDL1KEEP, [%[outptr6], #0x60]\n"
678 "add z17.s, z17.s, z9.s\n"
679 "st1w z15.s, p2, [%[outptr1], #2, MUL VL]\n"
680 "ld1w z10.s, p2/z, [x8, #-8, MUL VL]\n"
681 "addvl %[outptr1], %[outptr1], #3\n"
682 "ld1w z3.s, p0/z, [%[outptr3]]\n"
683 "prfm PLDL1KEEP, [%[outptr7], #0x60]\n"
684 "add z10.s, z10.s, z2.s\n"
685 "st1w z16.s, p0, [%[outptr2]]\n"
686 "ld1w z11.s, p0/z, [x8, #-7, MUL VL]\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]687 "addvl %[inptr], %[inptr], #24\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]688 "ld1w z4.s, p1/z, [%[outptr3], #1, MUL VL]\n"
689 "st1w z17.s, p1, [%[outptr2], #1, MUL VL]\n"
690 "add z11.s, z11.s, z3.s\n"
691 "ld1w z12.s, p1/z, [x8, #-6, MUL VL]\n"
692 "ld1w z5.s, p2/z, [%[outptr3], #2, MUL VL]\n"
693 "ld1w z13.s, p2/z, [x8, #-5, MUL VL]\n"
694 "st1w z10.s, p2, [%[outptr2], #2, MUL VL]\n"
695 "addvl %[outptr2], %[outptr2], #3\n"
696 "add z12.s, z12.s, z4.s\n"
697 "ld1w z6.s, p0/z, [%[outptr4]]\n"
698 "add z13.s, z13.s, z5.s\n"
699 "st1w z11.s, p0, [%[outptr3]]\n"
700 "ld1w z14.s, p0/z, [x8, #-4, MUL VL]\n"
701 "ld1w z7.s, p1/z, [%[outptr4], #1, MUL VL]\n"
702 "ld1w z15.s, p1/z, [x8, #-3, MUL VL]\n"
703 "st1w z12.s, p1, [%[outptr3], #1, MUL VL]\n"
704 "add z14.s, z14.s, z6.s\n"
705 "ld1w z8.s, p2/z, [%[outptr4], #2, MUL VL]\n"
706 "ld1w z16.s, p2/z, [x8, #-2, MUL VL]\n"
707 "add z15.s, z15.s, z7.s\n"
708 "st1w z13.s, p2, [%[outptr3], #2, MUL VL]\n"
709 "ld1w z9.s, p0/z, [%[outptr5]]\n"
710 "addvl %[outptr3], %[outptr3], #3\n"
711 "add z16.s, z16.s, z8.s\n"
712 "st1w z14.s, p0, [%[outptr4]]\n"
713 "ld1w z17.s, p0/z, [x8, #-1, MUL VL]\n"
714 "ld1w z2.s, p1/z, [%[outptr5], #1, MUL VL]\n"
715 "ld1w z10.s, p1/z, [x8]\n"
716 "st1w z15.s, p1, [%[outptr4], #1, MUL VL]\n"
717 "add z17.s, z17.s, z9.s\n"
718 "ld1w z3.s, p2/z, [%[outptr5], #2, MUL VL]\n"
719 "ld1w z11.s, p2/z, [x8, #1, MUL VL]\n"
720 "add z10.s, z10.s, z2.s\n"
721 "st1w z16.s, p2, [%[outptr4], #2, MUL VL]\n"
722 "ld1w z4.s, p0/z, [%[outptr6]]\n"
723 "addvl %[outptr4], %[outptr4], #3\n"
724 "add z11.s, z11.s, z3.s\n"
725 "st1w z17.s, p0, [%[outptr5]]\n"
726 "ld1w z12.s, p0/z, [x8, #2, MUL VL]\n"
727 "ld1w z5.s, p1/z, [%[outptr6], #1, MUL VL]\n"
728 "ld1w z13.s, p1/z, [x8, #3, MUL VL]\n"
729 "st1w z10.s, p1, [%[outptr5], #1, MUL VL]\n"
730 "add z12.s, z12.s, z4.s\n"
731 "ld1w z6.s, p2/z, [%[outptr6], #2, MUL VL]\n"
732 "ld1w z14.s, p2/z, [x8, #4, MUL VL]\n"
733 "add z13.s, z13.s, z5.s\n"
734 "st1w z11.s, p2, [%[outptr5], #2, MUL VL]\n"
735 "ld1w z7.s, p0/z, [%[outptr7]]\n"
736 "addvl %[outptr5], %[outptr5], #3\n"
737 "add z14.s, z14.s, z6.s\n"
738 "st1w z12.s, p0, [%[outptr6]]\n"
739 "ld1w z15.s, p0/z, [x8, #5, MUL VL]\n"
740 "ld1w z8.s, p1/z, [%[outptr7], #1, MUL VL]\n"
741 "ld1w z16.s, p1/z, [x8, #6, MUL VL]\n"
742 "st1w z13.s, p1, [%[outptr6], #1, MUL VL]\n"
743 "add z15.s, z15.s, z7.s\n"
744 "ld1w z9.s, p2/z, [%[outptr7], #2, MUL VL]\n"
745 "ld1w z17.s, p2/z, [x8, #7, MUL VL]\n"
746 "add z16.s, z16.s, z8.s\n"
747 "st1w z14.s, p2, [%[outptr6], #2, MUL VL]\n"
748 "addvl %[outptr6], %[outptr6], #3\n"
749 "add z17.s, z17.s, z9.s\n"
750 "st1w z15.s, p0, [%[outptr7]]\n"
751 "st1w z16.s, p1, [%[outptr7], #1, MUL VL]\n"
752 "st1w z17.s, p2, [%[outptr7], #2, MUL VL]\n"
753 "addvl %[outptr7], %[outptr7], #3\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]754 : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
755 [inptr] "+r" (inptr), [p] "+r" (p)
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]756 : [w] "r" (w)
757 : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]758 );
759 }
760 break;
761
762
763 }
764 }
765 else
766 {
Georgios Pinitas5aa1a0b2020-07-02 20:02:20 +0100[diff] [blame]767 const int32_t *biasptr = bias ? bias + i : nullbias;
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]768
769 switch(height)
770 {
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]771 case 1:
772 {
773 long w = xmax - i;
774 long p = 0;
775 /* Optimized routine to copy an entire block */
776 __asm __volatile (
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]777 "addvl x8, %[inptr], #16\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]778 "whilelt p0.s, %[p], %[w]\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]779 "incw %[p], all, mul #1\n"
780 "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]781 "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
782 "ld1w z2.s, p0/z, [%[biasptr]]\n"
783 "whilelt p1.s, %[p], %[w]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]784 "ld1w z13.s, p0/z, [%[inptr]]\n"
David Mansellce79ac62022-09-23 09:57:43 +0100[diff] [blame]785 "incw %[p], all, mul #1\n"
786 "ld1w z3.s, p1/z, [%[biasptr], #1, MUL VL]\n"
787 "add z13.s, z13.s, z2.s\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]788 "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n"
789 "whilelt p2.s, %[p], %[w]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]790 "add z14.s, z14.s, z3.s\n"
David Mansellce79ac62022-09-23 09:57:43 +0100[diff] [blame]791 "st1w z13.s, p0, [%[outptr0]]\n"
792 "ld1w z4.s, p2/z, [%[biasptr], #2, MUL VL]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]793 "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]794 "addvl %[inptr], %[inptr], #24\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]795 "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n"
David Mansellce79ac62022-09-23 09:57:43 +0100[diff] [blame]796 "add z15.s, z15.s, z4.s\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]797 "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n"
798 "addvl %[outptr0], %[outptr0], #3\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]799 : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
800 [inptr] "+r" (inptr), [p] "+r" (p)
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]801 : [w] "r" (w), [biasptr] "r" (biasptr)
802 : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]803 );
804 }
805 break;
806
807 case 2:
808 {
809 long w = xmax - i;
810 long p = 0;
811 /* Optimized routine to copy an entire block */
812 __asm __volatile (
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]813 "addvl x8, %[inptr], #16\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]814 "whilelt p0.s, %[p], %[w]\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]815 "incw %[p], all, mul #1\n"
816 "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]817 "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
818 "ld1w z2.s, p0/z, [%[biasptr]]\n"
819 "whilelt p1.s, %[p], %[w]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]820 "ld1w z13.s, p0/z, [%[inptr]]\n"
David Mansellce79ac62022-09-23 09:57:43 +0100[diff] [blame]821 "incw %[p], all, mul #1\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]822 "ld1w z16.s, p0/z, [%[inptr], #3, MUL VL]\n"
David Mansellce79ac62022-09-23 09:57:43 +0100[diff] [blame]823 "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
824 "add z13.s, z13.s, z2.s\n"
825 "ld1w z3.s, p1/z, [%[biasptr], #1, MUL VL]\n"
826 "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n"
827 "whilelt p2.s, %[p], %[w]\n"
828 "add z16.s, z16.s, z2.s\n"
829 "st1w z13.s, p0, [%[outptr0]]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]830 "ld1w z17.s, p1/z, [%[inptr], #4, MUL VL]\n"
David Mansellce79ac62022-09-23 09:57:43 +0100[diff] [blame]831 "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
832 "add z14.s, z14.s, z3.s\n"
833 "ld1w z4.s, p2/z, [%[biasptr], #2, MUL VL]\n"
834 "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n"
835 "add z17.s, z17.s, z3.s\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]836 "ld1w z18.s, p2/z, [%[inptr], #5, MUL VL]\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]837 "addvl %[inptr], %[inptr], #24\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]838 "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n"
David Mansellce79ac62022-09-23 09:57:43 +0100[diff] [blame]839 "add z15.s, z15.s, z4.s\n"
840 "add z18.s, z18.s, z4.s\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]841 "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n"
842 "addvl %[outptr0], %[outptr0], #3\n"
843 "st1w z16.s, p0, [%[outptr1]]\n"
844 "st1w z17.s, p1, [%[outptr1], #1, MUL VL]\n"
845 "st1w z18.s, p2, [%[outptr1], #2, MUL VL]\n"
846 "addvl %[outptr1], %[outptr1], #3\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]847 : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
848 [inptr] "+r" (inptr), [p] "+r" (p)
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]849 : [w] "r" (w), [biasptr] "r" (biasptr)
850 : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]851 );
852 }
853 break;
854
855 case 3:
856 {
857 long w = xmax - i;
858 long p = 0;
859 /* Optimized routine to copy an entire block */
860 __asm __volatile (
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]861 "addvl x8, %[inptr], #16\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]862 "whilelt p0.s, %[p], %[w]\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]863 "incw %[p], all, mul #1\n"
864 "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]865 "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
866 "ld1w z2.s, p0/z, [%[biasptr]]\n"
867 "whilelt p1.s, %[p], %[w]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]868 "ld1w z13.s, p0/z, [%[inptr]]\n"
David Mansellce79ac62022-09-23 09:57:43 +0100[diff] [blame]869 "incw %[p], all, mul #1\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]870 "ld1w z16.s, p0/z, [%[inptr], #3, MUL VL]\n"
David Mansellce79ac62022-09-23 09:57:43 +0100[diff] [blame]871 "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
872 "add z13.s, z13.s, z2.s\n"
873 "ld1w z3.s, p1/z, [%[biasptr], #1, MUL VL]\n"
874 "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n"
875 "whilelt p2.s, %[p], %[w]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]876 "add z16.s, z16.s, z2.s\n"
David Mansellce79ac62022-09-23 09:57:43 +0100[diff] [blame]877 "st1w z13.s, p0, [%[outptr0]]\n"
878 "ld1w z17.s, p1/z, [%[inptr], #4, MUL VL]\n"
879 "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
880 "add z14.s, z14.s, z3.s\n"
881 "ld1w z4.s, p2/z, [%[biasptr], #2, MUL VL]\n"
882 "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n"
883 "prfm PLDL1KEEP, [%[inptr], #0x200]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]884 "add z17.s, z17.s, z3.s\n"
885 "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n"
David Mansellce79ac62022-09-23 09:57:43 +0100[diff] [blame]886 "ld1w z18.s, p2/z, [%[inptr], #5, MUL VL]\n"
887 "prfm PSTL1KEEP, [%[outptr2], #0x60]\n"
888 "add z15.s, z15.s, z4.s\n"
889 "ld1w z19.s, p0/z, [%[inptr], #6, MUL VL]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]890 "ld1w z20.s, p1/z, [%[inptr], #7, MUL VL]\n"
David Mansellce79ac62022-09-23 09:57:43 +0100[diff] [blame]891 "addvl %[inptr], %[inptr], #24\n"
892 "add z18.s, z18.s, z4.s\n"
893 "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]894 "add z19.s, z19.s, z2.s\n"
895 "ld1w z13.s, p2/z, [x8, #-8, MUL VL]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]896 "add z20.s, z20.s, z3.s\n"
David Mansellce79ac62022-09-23 09:57:43 +0100[diff] [blame]897 "addvl %[outptr0], %[outptr0], #3\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]898 "st1w z16.s, p0, [%[outptr1]]\n"
David Mansellce79ac62022-09-23 09:57:43 +0100[diff] [blame]899 "add z13.s, z13.s, z4.s\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]900 "st1w z17.s, p1, [%[outptr1], #1, MUL VL]\n"
901 "st1w z18.s, p2, [%[outptr1], #2, MUL VL]\n"
902 "addvl %[outptr1], %[outptr1], #3\n"
903 "st1w z19.s, p0, [%[outptr2]]\n"
904 "st1w z20.s, p1, [%[outptr2], #1, MUL VL]\n"
905 "st1w z13.s, p2, [%[outptr2], #2, MUL VL]\n"
906 "addvl %[outptr2], %[outptr2], #3\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]907 : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
908 [inptr] "+r" (inptr), [p] "+r" (p)
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]909 : [w] "r" (w), [biasptr] "r" (biasptr)
910 : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]911 );
912 }
913 break;
914
915 case 4:
916 {
917 long w = xmax - i;
918 long p = 0;
919 /* Optimized routine to copy an entire block */
920 __asm __volatile (
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]921 "addvl x8, %[inptr], #16\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]922 "whilelt p0.s, %[p], %[w]\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]923 "incw %[p], all, mul #1\n"
924 "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]925 "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
926 "ld1w z2.s, p0/z, [%[biasptr]]\n"
927 "whilelt p1.s, %[p], %[w]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]928 "ld1w z13.s, p0/z, [%[inptr]]\n"
David Mansellce79ac62022-09-23 09:57:43 +0100[diff] [blame]929 "incw %[p], all, mul #1\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]930 "ld1w z16.s, p0/z, [%[inptr], #3, MUL VL]\n"
David Mansellce79ac62022-09-23 09:57:43 +0100[diff] [blame]931 "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
932 "add z13.s, z13.s, z2.s\n"
933 "ld1w z3.s, p1/z, [%[biasptr], #1, MUL VL]\n"
934 "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n"
935 "whilelt p2.s, %[p], %[w]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]936 "add z16.s, z16.s, z2.s\n"
David Mansellce79ac62022-09-23 09:57:43 +0100[diff] [blame]937 "st1w z13.s, p0, [%[outptr0]]\n"
938 "ld1w z17.s, p1/z, [%[inptr], #4, MUL VL]\n"
939 "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
940 "add z14.s, z14.s, z3.s\n"
941 "ld1w z4.s, p2/z, [%[biasptr], #2, MUL VL]\n"
942 "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n"
943 "prfm PLDL1KEEP, [%[inptr], #0x200]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]944 "add z17.s, z17.s, z3.s\n"
945 "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n"
David Mansellce79ac62022-09-23 09:57:43 +0100[diff] [blame]946 "ld1w z18.s, p2/z, [%[inptr], #5, MUL VL]\n"
947 "prfm PSTL1KEEP, [%[outptr2], #0x60]\n"
948 "add z15.s, z15.s, z4.s\n"
949 "ld1w z19.s, p0/z, [%[inptr], #6, MUL VL]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]950 "ld1w z20.s, p1/z, [%[inptr], #7, MUL VL]\n"
David Mansellce79ac62022-09-23 09:57:43 +0100[diff] [blame]951 "prfm PSTL1KEEP, [%[outptr3], #0x60]\n"
952 "add z18.s, z18.s, z4.s\n"
953 "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]954 "add z19.s, z19.s, z2.s\n"
955 "ld1w z13.s, p2/z, [x8, #-8, MUL VL]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]956 "add z20.s, z20.s, z3.s\n"
David Mansellce79ac62022-09-23 09:57:43 +0100[diff] [blame]957 "ld1w z14.s, p0/z, [x8, #-7, MUL VL]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]958 "ld1w z15.s, p1/z, [x8, #-6, MUL VL]\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]959 "addvl %[outptr0], %[outptr0], #3\n"
David Mansellce79ac62022-09-23 09:57:43 +0100[diff] [blame]960 "add z13.s, z13.s, z4.s\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]961 "st1w z16.s, p0, [%[outptr1]]\n"
David Mansellce79ac62022-09-23 09:57:43 +0100[diff] [blame]962 "add z14.s, z14.s, z2.s\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]963 "ld1w z16.s, p2/z, [x8, #-5, MUL VL]\n"
David Mansellce79ac62022-09-23 09:57:43 +0100[diff] [blame]964 "add z15.s, z15.s, z3.s\n"
965 "addvl %[inptr], %[inptr], #24\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]966 "st1w z17.s, p1, [%[outptr1], #1, MUL VL]\n"
967 "add z16.s, z16.s, z4.s\n"
968 "st1w z18.s, p2, [%[outptr1], #2, MUL VL]\n"
969 "addvl %[outptr1], %[outptr1], #3\n"
970 "st1w z19.s, p0, [%[outptr2]]\n"
971 "st1w z20.s, p1, [%[outptr2], #1, MUL VL]\n"
972 "st1w z13.s, p2, [%[outptr2], #2, MUL VL]\n"
973 "addvl %[outptr2], %[outptr2], #3\n"
974 "st1w z14.s, p0, [%[outptr3]]\n"
975 "st1w z15.s, p1, [%[outptr3], #1, MUL VL]\n"
976 "st1w z16.s, p2, [%[outptr3], #2, MUL VL]\n"
977 "addvl %[outptr3], %[outptr3], #3\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]978 : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
979 [inptr] "+r" (inptr), [p] "+r" (p)
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]980 : [w] "r" (w), [biasptr] "r" (biasptr)
981 : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]982 );
983 }
984 break;
985
986 case 5:
987 {
988 long w = xmax - i;
989 long p = 0;
990 /* Optimized routine to copy an entire block */
991 __asm __volatile (
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]992 "addvl x8, %[inptr], #16\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]993 "whilelt p0.s, %[p], %[w]\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]994 "incw %[p], all, mul #1\n"
995 "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]996 "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
997 "ld1w z2.s, p0/z, [%[biasptr]]\n"
998 "whilelt p1.s, %[p], %[w]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]999 "ld1w z13.s, p0/z, [%[inptr]]\n"
David Mansellce79ac62022-09-23 09:57:43 +0100[diff] [blame]1000 "incw %[p], all, mul #1\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]1001 "ld1w z16.s, p0/z, [%[inptr], #3, MUL VL]\n"
David Mansellce79ac62022-09-23 09:57:43 +0100[diff] [blame]1002 "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
1003 "add z13.s, z13.s, z2.s\n"
1004 "ld1w z3.s, p1/z, [%[biasptr], #1, MUL VL]\n"
1005 "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n"
1006 "whilelt p2.s, %[p], %[w]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]1007 "add z16.s, z16.s, z2.s\n"
David Mansellce79ac62022-09-23 09:57:43 +0100[diff] [blame]1008 "st1w z13.s, p0, [%[outptr0]]\n"
1009 "ld1w z17.s, p1/z, [%[inptr], #4, MUL VL]\n"
1010 "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
1011 "add z14.s, z14.s, z3.s\n"
1012 "ld1w z4.s, p2/z, [%[biasptr], #2, MUL VL]\n"
1013 "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n"
1014 "prfm PLDL1KEEP, [%[inptr], #0x200]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]1015 "add z17.s, z17.s, z3.s\n"
1016 "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n"
David Mansellce79ac62022-09-23 09:57:43 +0100[diff] [blame]1017 "ld1w z18.s, p2/z, [%[inptr], #5, MUL VL]\n"
1018 "prfm PSTL1KEEP, [%[outptr2], #0x60]\n"
1019 "add z15.s, z15.s, z4.s\n"
1020 "ld1w z19.s, p0/z, [%[inptr], #6, MUL VL]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]1021 "ld1w z20.s, p1/z, [%[inptr], #7, MUL VL]\n"
David Mansellce79ac62022-09-23 09:57:43 +0100[diff] [blame]1022 "prfm PSTL1KEEP, [%[outptr3], #0x60]\n"
1023 "add z18.s, z18.s, z4.s\n"
1024 "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]1025 "add z19.s, z19.s, z2.s\n"
1026 "ld1w z13.s, p2/z, [x8, #-8, MUL VL]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]1027 "add z20.s, z20.s, z3.s\n"
David Mansellce79ac62022-09-23 09:57:43 +0100[diff] [blame]1028 "ld1w z14.s, p0/z, [x8, #-7, MUL VL]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]1029 "ld1w z15.s, p1/z, [x8, #-6, MUL VL]\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]1030 "addvl %[outptr0], %[outptr0], #3\n"
David Mansellce79ac62022-09-23 09:57:43 +0100[diff] [blame]1031 "add z13.s, z13.s, z4.s\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]1032 "st1w z16.s, p0, [%[outptr1]]\n"
David Mansellce79ac62022-09-23 09:57:43 +0100[diff] [blame]1033 "add z14.s, z14.s, z2.s\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]1034 "ld1w z16.s, p2/z, [x8, #-5, MUL VL]\n"
David Mansellce79ac62022-09-23 09:57:43 +0100[diff] [blame]1035 "add z15.s, z15.s, z3.s\n"
1036 "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]1037 "st1w z17.s, p1, [%[outptr1], #1, MUL VL]\n"
David Mansellce79ac62022-09-23 09:57:43 +0100[diff] [blame]1038 "prfm PSTL1KEEP, [%[outptr4], #0x60]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]1039 "add z16.s, z16.s, z4.s\n"
1040 "ld1w z17.s, p0/z, [x8, #-4, MUL VL]\n"
David Mansellce79ac62022-09-23 09:57:43 +0100[diff] [blame]1041 "addvl %[inptr], %[inptr], #24\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]1042 "st1w z18.s, p2, [%[outptr1], #2, MUL VL]\n"
1043 "addvl %[outptr1], %[outptr1], #3\n"
1044 "add z17.s, z17.s, z2.s\n"
1045 "ld1w z18.s, p1/z, [x8, #-3, MUL VL]\n"
1046 "st1w z19.s, p0, [%[outptr2]]\n"
1047 "ld1w z19.s, p2/z, [x8, #-2, MUL VL]\n"
1048 "add z18.s, z18.s, z3.s\n"
1049 "st1w z20.s, p1, [%[outptr2], #1, MUL VL]\n"
1050 "add z19.s, z19.s, z4.s\n"
1051 "st1w z13.s, p2, [%[outptr2], #2, MUL VL]\n"
1052 "addvl %[outptr2], %[outptr2], #3\n"
1053 "st1w z14.s, p0, [%[outptr3]]\n"
1054 "st1w z15.s, p1, [%[outptr3], #1, MUL VL]\n"
1055 "st1w z16.s, p2, [%[outptr3], #2, MUL VL]\n"
1056 "addvl %[outptr3], %[outptr3], #3\n"
1057 "st1w z17.s, p0, [%[outptr4]]\n"
1058 "st1w z18.s, p1, [%[outptr4], #1, MUL VL]\n"
1059 "st1w z19.s, p2, [%[outptr4], #2, MUL VL]\n"
1060 "addvl %[outptr4], %[outptr4], #3\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]1061 : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
1062 [inptr] "+r" (inptr), [p] "+r" (p)
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]1063 : [w] "r" (w), [biasptr] "r" (biasptr)
1064 : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]1065 );
1066 }
1067 break;
1068
1069 case 6:
1070 {
1071 long w = xmax - i;
1072 long p = 0;
1073 /* Optimized routine to copy an entire block */
1074 __asm __volatile (
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]1075 "addvl x8, %[inptr], #16\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]1076 "whilelt p0.s, %[p], %[w]\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]1077 "incw %[p], all, mul #1\n"
1078 "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]1079 "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
1080 "ld1w z2.s, p0/z, [%[biasptr]]\n"
1081 "whilelt p1.s, %[p], %[w]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]1082 "ld1w z13.s, p0/z, [%[inptr]]\n"
David Mansellce79ac62022-09-23 09:57:43 +0100[diff] [blame]1083 "incw %[p], all, mul #1\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]1084 "ld1w z16.s, p0/z, [%[inptr], #3, MUL VL]\n"
David Mansellce79ac62022-09-23 09:57:43 +0100[diff] [blame]1085 "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
1086 "add z13.s, z13.s, z2.s\n"
1087 "ld1w z3.s, p1/z, [%[biasptr], #1, MUL VL]\n"
1088 "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n"
1089 "whilelt p2.s, %[p], %[w]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]1090 "add z16.s, z16.s, z2.s\n"
David Mansellce79ac62022-09-23 09:57:43 +0100[diff] [blame]1091 "st1w z13.s, p0, [%[outptr0]]\n"
1092 "ld1w z17.s, p1/z, [%[inptr], #4, MUL VL]\n"
1093 "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
1094 "add z14.s, z14.s, z3.s\n"
1095 "ld1w z4.s, p2/z, [%[biasptr], #2, MUL VL]\n"
1096 "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n"
1097 "prfm PLDL1KEEP, [%[inptr], #0x200]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]1098 "add z17.s, z17.s, z3.s\n"
1099 "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n"
David Mansellce79ac62022-09-23 09:57:43 +0100[diff] [blame]1100 "ld1w z18.s, p2/z, [%[inptr], #5, MUL VL]\n"
1101 "prfm PSTL1KEEP, [%[outptr2], #0x60]\n"
1102 "add z15.s, z15.s, z4.s\n"
1103 "ld1w z19.s, p0/z, [%[inptr], #6, MUL VL]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]1104 "ld1w z20.s, p1/z, [%[inptr], #7, MUL VL]\n"
David Mansellce79ac62022-09-23 09:57:43 +0100[diff] [blame]1105 "prfm PSTL1KEEP, [%[outptr3], #0x60]\n"
1106 "add z18.s, z18.s, z4.s\n"
1107 "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]1108 "add z19.s, z19.s, z2.s\n"
1109 "ld1w z13.s, p2/z, [x8, #-8, MUL VL]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]1110 "add z20.s, z20.s, z3.s\n"
David Mansellce79ac62022-09-23 09:57:43 +0100[diff] [blame]1111 "ld1w z14.s, p0/z, [x8, #-7, MUL VL]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]1112 "ld1w z15.s, p1/z, [x8, #-6, MUL VL]\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]1113 "addvl %[outptr0], %[outptr0], #3\n"
David Mansellce79ac62022-09-23 09:57:43 +0100[diff] [blame]1114 "add z13.s, z13.s, z4.s\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]1115 "st1w z16.s, p0, [%[outptr1]]\n"
David Mansellce79ac62022-09-23 09:57:43 +0100[diff] [blame]1116 "add z14.s, z14.s, z2.s\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]1117 "ld1w z16.s, p2/z, [x8, #-5, MUL VL]\n"
David Mansellce79ac62022-09-23 09:57:43 +0100[diff] [blame]1118 "add z15.s, z15.s, z3.s\n"
1119 "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]1120 "st1w z17.s, p1, [%[outptr1], #1, MUL VL]\n"
David Mansellce79ac62022-09-23 09:57:43 +0100[diff] [blame]1121 "prfm PSTL1KEEP, [%[outptr4], #0x60]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]1122 "add z16.s, z16.s, z4.s\n"
1123 "ld1w z17.s, p0/z, [x8, #-4, MUL VL]\n"
David Mansellce79ac62022-09-23 09:57:43 +0100[diff] [blame]1124 "prfm PLDL1KEEP, [%[inptr], #0x280]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]1125 "st1w z18.s, p2, [%[outptr1], #2, MUL VL]\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]1126 "addvl %[outptr1], %[outptr1], #3\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]1127 "add z17.s, z17.s, z2.s\n"
1128 "ld1w z18.s, p1/z, [x8, #-3, MUL VL]\n"
David Mansellce79ac62022-09-23 09:57:43 +0100[diff] [blame]1129 "prfm PSTL1KEEP, [%[outptr5], #0x60]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]1130 "st1w z19.s, p0, [%[outptr2]]\n"
David Mansellce79ac62022-09-23 09:57:43 +0100[diff] [blame]1131 "addvl %[inptr], %[inptr], #24\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]1132 "add z18.s, z18.s, z3.s\n"
David Mansellce79ac62022-09-23 09:57:43 +0100[diff] [blame]1133 "ld1w z19.s, p2/z, [x8, #-2, MUL VL]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]1134 "st1w z20.s, p1, [%[outptr2], #1, MUL VL]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]1135 "ld1w z20.s, p0/z, [x8, #-1, MUL VL]\n"
David Mansellce79ac62022-09-23 09:57:43 +0100[diff] [blame]1136 "add z19.s, z19.s, z4.s\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]1137 "st1w z13.s, p2, [%[outptr2], #2, MUL VL]\n"
1138 "addvl %[outptr2], %[outptr2], #3\n"
1139 "add z20.s, z20.s, z2.s\n"
1140 "ld1w z13.s, p1/z, [x8]\n"
1141 "st1w z14.s, p0, [%[outptr3]]\n"
1142 "ld1w z14.s, p2/z, [x8, #1, MUL VL]\n"
1143 "add z13.s, z13.s, z3.s\n"
1144 "st1w z15.s, p1, [%[outptr3], #1, MUL VL]\n"
1145 "add z14.s, z14.s, z4.s\n"
1146 "st1w z16.s, p2, [%[outptr3], #2, MUL VL]\n"
1147 "addvl %[outptr3], %[outptr3], #3\n"
1148 "st1w z17.s, p0, [%[outptr4]]\n"
1149 "st1w z18.s, p1, [%[outptr4], #1, MUL VL]\n"
1150 "st1w z19.s, p2, [%[outptr4], #2, MUL VL]\n"
1151 "addvl %[outptr4], %[outptr4], #3\n"
1152 "st1w z20.s, p0, [%[outptr5]]\n"
1153 "st1w z13.s, p1, [%[outptr5], #1, MUL VL]\n"
1154 "st1w z14.s, p2, [%[outptr5], #2, MUL VL]\n"
1155 "addvl %[outptr5], %[outptr5], #3\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]1156 : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
1157 [inptr] "+r" (inptr), [p] "+r" (p)
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]1158 : [w] "r" (w), [biasptr] "r" (biasptr)
1159 : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]1160 );
1161 }
1162 break;
1163
1164 case 7:
1165 {
1166 long w = xmax - i;
1167 long p = 0;
1168 /* Optimized routine to copy an entire block */
1169 __asm __volatile (
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]1170 "addvl x8, %[inptr], #16\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]1171 "whilelt p0.s, %[p], %[w]\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]1172 "incw %[p], all, mul #1\n"
1173 "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]1174 "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
1175 "ld1w z2.s, p0/z, [%[biasptr]]\n"
1176 "whilelt p1.s, %[p], %[w]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]1177 "ld1w z13.s, p0/z, [%[inptr]]\n"
David Mansellce79ac62022-09-23 09:57:43 +0100[diff] [blame]1178 "incw %[p], all, mul #1\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]1179 "ld1w z16.s, p0/z, [%[inptr], #3, MUL VL]\n"
David Mansellce79ac62022-09-23 09:57:43 +0100[diff] [blame]1180 "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
1181 "add z13.s, z13.s, z2.s\n"
1182 "ld1w z3.s, p1/z, [%[biasptr], #1, MUL VL]\n"
1183 "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n"
1184 "whilelt p2.s, %[p], %[w]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]1185 "add z16.s, z16.s, z2.s\n"
David Mansellce79ac62022-09-23 09:57:43 +0100[diff] [blame]1186 "st1w z13.s, p0, [%[outptr0]]\n"
1187 "ld1w z17.s, p1/z, [%[inptr], #4, MUL VL]\n"
1188 "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
1189 "add z14.s, z14.s, z3.s\n"
1190 "ld1w z4.s, p2/z, [%[biasptr], #2, MUL VL]\n"
1191 "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n"
1192 "prfm PLDL1KEEP, [%[inptr], #0x200]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]1193 "add z17.s, z17.s, z3.s\n"
1194 "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n"
David Mansellce79ac62022-09-23 09:57:43 +0100[diff] [blame]1195 "ld1w z18.s, p2/z, [%[inptr], #5, MUL VL]\n"
1196 "prfm PSTL1KEEP, [%[outptr2], #0x60]\n"
1197 "add z15.s, z15.s, z4.s\n"
1198 "ld1w z19.s, p0/z, [%[inptr], #6, MUL VL]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]1199 "ld1w z20.s, p1/z, [%[inptr], #7, MUL VL]\n"
David Mansellce79ac62022-09-23 09:57:43 +0100[diff] [blame]1200 "prfm PSTL1KEEP, [%[outptr3], #0x60]\n"
1201 "add z18.s, z18.s, z4.s\n"
1202 "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]1203 "add z19.s, z19.s, z2.s\n"
1204 "ld1w z13.s, p2/z, [x8, #-8, MUL VL]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]1205 "add z20.s, z20.s, z3.s\n"
David Mansellce79ac62022-09-23 09:57:43 +0100[diff] [blame]1206 "ld1w z14.s, p0/z, [x8, #-7, MUL VL]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]1207 "ld1w z15.s, p1/z, [x8, #-6, MUL VL]\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]1208 "addvl %[outptr0], %[outptr0], #3\n"
David Mansellce79ac62022-09-23 09:57:43 +0100[diff] [blame]1209 "add z13.s, z13.s, z4.s\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]1210 "st1w z16.s, p0, [%[outptr1]]\n"
David Mansellce79ac62022-09-23 09:57:43 +0100[diff] [blame]1211 "add z14.s, z14.s, z2.s\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]1212 "ld1w z16.s, p2/z, [x8, #-5, MUL VL]\n"
David Mansellce79ac62022-09-23 09:57:43 +0100[diff] [blame]1213 "add z15.s, z15.s, z3.s\n"
1214 "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]1215 "st1w z17.s, p1, [%[outptr1], #1, MUL VL]\n"
David Mansellce79ac62022-09-23 09:57:43 +0100[diff] [blame]1216 "prfm PSTL1KEEP, [%[outptr4], #0x60]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]1217 "add z16.s, z16.s, z4.s\n"
1218 "ld1w z17.s, p0/z, [x8, #-4, MUL VL]\n"
David Mansellce79ac62022-09-23 09:57:43 +0100[diff] [blame]1219 "prfm PLDL1KEEP, [%[inptr], #0x280]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]1220 "st1w z18.s, p2, [%[outptr1], #2, MUL VL]\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]1221 "addvl %[outptr1], %[outptr1], #3\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]1222 "add z17.s, z17.s, z2.s\n"
1223 "ld1w z18.s, p1/z, [x8, #-3, MUL VL]\n"
David Mansellce79ac62022-09-23 09:57:43 +0100[diff] [blame]1224 "prfm PSTL1KEEP, [%[outptr5], #0x60]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]1225 "st1w z19.s, p0, [%[outptr2]]\n"
David Mansellce79ac62022-09-23 09:57:43 +0100[diff] [blame]1226 "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]1227 "add z18.s, z18.s, z3.s\n"
1228 "ld1w z19.s, p2/z, [x8, #-2, MUL VL]\n"
David Mansellce79ac62022-09-23 09:57:43 +0100[diff] [blame]1229 "prfm PSTL1KEEP, [%[outptr6], #0x60]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]1230 "st1w z20.s, p1, [%[outptr2], #1, MUL VL]\n"
David Mansellce79ac62022-09-23 09:57:43 +0100[diff] [blame]1231 "addvl %[inptr], %[inptr], #24\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]1232 "add z19.s, z19.s, z4.s\n"
David Mansellce79ac62022-09-23 09:57:43 +0100[diff] [blame]1233 "ld1w z20.s, p0/z, [x8, #-1, MUL VL]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]1234 "st1w z13.s, p2, [%[outptr2], #2, MUL VL]\n"
1235 "addvl %[outptr2], %[outptr2], #3\n"
1236 "add z20.s, z20.s, z2.s\n"
1237 "ld1w z13.s, p1/z, [x8]\n"
1238 "st1w z14.s, p0, [%[outptr3]]\n"
1239 "ld1w z14.s, p2/z, [x8, #1, MUL VL]\n"
1240 "add z13.s, z13.s, z3.s\n"
1241 "st1w z15.s, p1, [%[outptr3], #1, MUL VL]\n"
1242 "add z14.s, z14.s, z4.s\n"
1243 "ld1w z15.s, p0/z, [x8, #2, MUL VL]\n"
1244 "st1w z16.s, p2, [%[outptr3], #2, MUL VL]\n"
1245 "addvl %[outptr3], %[outptr3], #3\n"
1246 "add z15.s, z15.s, z2.s\n"
1247 "ld1w z16.s, p1/z, [x8, #3, MUL VL]\n"
1248 "st1w z17.s, p0, [%[outptr4]]\n"
1249 "ld1w z17.s, p2/z, [x8, #4, MUL VL]\n"
1250 "add z16.s, z16.s, z3.s\n"
1251 "st1w z18.s, p1, [%[outptr4], #1, MUL VL]\n"
1252 "add z17.s, z17.s, z4.s\n"
1253 "st1w z19.s, p2, [%[outptr4], #2, MUL VL]\n"
1254 "addvl %[outptr4], %[outptr4], #3\n"
1255 "st1w z20.s, p0, [%[outptr5]]\n"
1256 "st1w z13.s, p1, [%[outptr5], #1, MUL VL]\n"
1257 "st1w z14.s, p2, [%[outptr5], #2, MUL VL]\n"
1258 "addvl %[outptr5], %[outptr5], #3\n"
1259 "st1w z15.s, p0, [%[outptr6]]\n"
1260 "st1w z16.s, p1, [%[outptr6], #1, MUL VL]\n"
1261 "st1w z17.s, p2, [%[outptr6], #2, MUL VL]\n"
1262 "addvl %[outptr6], %[outptr6], #3\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]1263 : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
1264 [inptr] "+r" (inptr), [p] "+r" (p)
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]1265 : [w] "r" (w), [biasptr] "r" (biasptr)
1266 : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]1267 );
1268 }
1269 break;
1270
1271 default:
1272 case 8:
1273 {
1274 long w = xmax - i;
1275 long p = 0;
1276 /* Optimized routine to copy an entire block */
1277 __asm __volatile (
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]1278 "addvl x8, %[inptr], #16\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]1279 "whilelt p0.s, %[p], %[w]\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]1280 "incw %[p], all, mul #1\n"
1281 "prfm PLDL1KEEP, [%[inptr], #0x180]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]1282 "prfm PSTL1KEEP, [%[outptr0], #0x60]\n"
1283 "ld1w z2.s, p0/z, [%[biasptr]]\n"
1284 "whilelt p1.s, %[p], %[w]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]1285 "ld1w z13.s, p0/z, [%[inptr]]\n"
David Mansellce79ac62022-09-23 09:57:43 +0100[diff] [blame]1286 "incw %[p], all, mul #1\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]1287 "ld1w z16.s, p0/z, [%[inptr], #3, MUL VL]\n"
David Mansellce79ac62022-09-23 09:57:43 +0100[diff] [blame]1288 "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n"
1289 "add z13.s, z13.s, z2.s\n"
1290 "ld1w z3.s, p1/z, [%[biasptr], #1, MUL VL]\n"
1291 "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n"
1292 "whilelt p2.s, %[p], %[w]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]1293 "add z16.s, z16.s, z2.s\n"
David Mansellce79ac62022-09-23 09:57:43 +0100[diff] [blame]1294 "st1w z13.s, p0, [%[outptr0]]\n"
1295 "ld1w z17.s, p1/z, [%[inptr], #4, MUL VL]\n"
1296 "prfm PSTL1KEEP, [%[outptr1], #0x60]\n"
1297 "add z14.s, z14.s, z3.s\n"
1298 "ld1w z4.s, p2/z, [%[biasptr], #2, MUL VL]\n"
1299 "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n"
1300 "prfm PLDL1KEEP, [%[inptr], #0x200]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]1301 "add z17.s, z17.s, z3.s\n"
1302 "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n"
David Mansellce79ac62022-09-23 09:57:43 +0100[diff] [blame]1303 "ld1w z18.s, p2/z, [%[inptr], #5, MUL VL]\n"
1304 "prfm PSTL1KEEP, [%[outptr2], #0x60]\n"
1305 "add z15.s, z15.s, z4.s\n"
1306 "ld1w z19.s, p0/z, [%[inptr], #6, MUL VL]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]1307 "ld1w z20.s, p1/z, [%[inptr], #7, MUL VL]\n"
David Mansellce79ac62022-09-23 09:57:43 +0100[diff] [blame]1308 "prfm PSTL1KEEP, [%[outptr3], #0x60]\n"
1309 "add z18.s, z18.s, z4.s\n"
1310 "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]1311 "add z19.s, z19.s, z2.s\n"
1312 "ld1w z13.s, p2/z, [x8, #-8, MUL VL]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]1313 "add z20.s, z20.s, z3.s\n"
David Mansellce79ac62022-09-23 09:57:43 +0100[diff] [blame]1314 "ld1w z14.s, p0/z, [x8, #-7, MUL VL]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]1315 "ld1w z15.s, p1/z, [x8, #-6, MUL VL]\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]1316 "addvl %[outptr0], %[outptr0], #3\n"
David Mansellce79ac62022-09-23 09:57:43 +0100[diff] [blame]1317 "add z13.s, z13.s, z4.s\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]1318 "st1w z16.s, p0, [%[outptr1]]\n"
David Mansellce79ac62022-09-23 09:57:43 +0100[diff] [blame]1319 "add z14.s, z14.s, z2.s\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]1320 "ld1w z16.s, p2/z, [x8, #-5, MUL VL]\n"
David Mansellce79ac62022-09-23 09:57:43 +0100[diff] [blame]1321 "add z15.s, z15.s, z3.s\n"
1322 "prfm PLDL1KEEP, [%[inptr], #0x240]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]1323 "st1w z17.s, p1, [%[outptr1], #1, MUL VL]\n"
David Mansellce79ac62022-09-23 09:57:43 +0100[diff] [blame]1324 "prfm PSTL1KEEP, [%[outptr4], #0x60]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]1325 "add z16.s, z16.s, z4.s\n"
1326 "ld1w z17.s, p0/z, [x8, #-4, MUL VL]\n"
David Mansellce79ac62022-09-23 09:57:43 +0100[diff] [blame]1327 "prfm PLDL1KEEP, [%[inptr], #0x280]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]1328 "st1w z18.s, p2, [%[outptr1], #2, MUL VL]\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]1329 "addvl %[outptr1], %[outptr1], #3\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]1330 "add z17.s, z17.s, z2.s\n"
1331 "ld1w z18.s, p1/z, [x8, #-3, MUL VL]\n"
David Mansellce79ac62022-09-23 09:57:43 +0100[diff] [blame]1332 "prfm PSTL1KEEP, [%[outptr5], #0x60]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]1333 "st1w z19.s, p0, [%[outptr2]]\n"
David Mansellce79ac62022-09-23 09:57:43 +0100[diff] [blame]1334 "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]1335 "add z18.s, z18.s, z3.s\n"
1336 "ld1w z19.s, p2/z, [x8, #-2, MUL VL]\n"
David Mansellce79ac62022-09-23 09:57:43 +0100[diff] [blame]1337 "prfm PSTL1KEEP, [%[outptr6], #0x60]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]1338 "st1w z20.s, p1, [%[outptr2], #1, MUL VL]\n"
David Mansellce79ac62022-09-23 09:57:43 +0100[diff] [blame]1339 "prfm PSTL1KEEP, [%[outptr7], #0x60]\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]1340 "add z19.s, z19.s, z4.s\n"
1341 "ld1w z20.s, p0/z, [x8, #-1, MUL VL]\n"
David Mansellce79ac62022-09-23 09:57:43 +0100[diff] [blame]1342 "addvl %[inptr], %[inptr], #24\n"
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]1343 "st1w z13.s, p2, [%[outptr2], #2, MUL VL]\n"
1344 "addvl %[outptr2], %[outptr2], #3\n"
1345 "add z20.s, z20.s, z2.s\n"
1346 "ld1w z13.s, p1/z, [x8]\n"
1347 "st1w z14.s, p0, [%[outptr3]]\n"
1348 "ld1w z14.s, p2/z, [x8, #1, MUL VL]\n"
1349 "add z13.s, z13.s, z3.s\n"
1350 "st1w z15.s, p1, [%[outptr3], #1, MUL VL]\n"
1351 "add z14.s, z14.s, z4.s\n"
1352 "ld1w z15.s, p0/z, [x8, #2, MUL VL]\n"
1353 "st1w z16.s, p2, [%[outptr3], #2, MUL VL]\n"
1354 "addvl %[outptr3], %[outptr3], #3\n"
1355 "add z15.s, z15.s, z2.s\n"
1356 "ld1w z16.s, p1/z, [x8, #3, MUL VL]\n"
1357 "st1w z17.s, p0, [%[outptr4]]\n"
1358 "ld1w z17.s, p2/z, [x8, #4, MUL VL]\n"
1359 "add z16.s, z16.s, z3.s\n"
1360 "st1w z18.s, p1, [%[outptr4], #1, MUL VL]\n"
1361 "add z17.s, z17.s, z4.s\n"
1362 "ld1w z18.s, p0/z, [x8, #5, MUL VL]\n"
1363 "st1w z19.s, p2, [%[outptr4], #2, MUL VL]\n"
1364 "addvl %[outptr4], %[outptr4], #3\n"
1365 "add z18.s, z18.s, z2.s\n"
1366 "ld1w z19.s, p1/z, [x8, #6, MUL VL]\n"
1367 "st1w z20.s, p0, [%[outptr5]]\n"
1368 "ld1w z20.s, p2/z, [x8, #7, MUL VL]\n"
1369 "add z19.s, z19.s, z3.s\n"
1370 "st1w z13.s, p1, [%[outptr5], #1, MUL VL]\n"
1371 "add z20.s, z20.s, z4.s\n"
1372 "st1w z14.s, p2, [%[outptr5], #2, MUL VL]\n"
1373 "addvl %[outptr5], %[outptr5], #3\n"
1374 "st1w z15.s, p0, [%[outptr6]]\n"
1375 "st1w z16.s, p1, [%[outptr6], #1, MUL VL]\n"
1376 "st1w z17.s, p2, [%[outptr6], #2, MUL VL]\n"
1377 "addvl %[outptr6], %[outptr6], #3\n"
1378 "st1w z18.s, p0, [%[outptr7]]\n"
1379 "st1w z19.s, p1, [%[outptr7], #1, MUL VL]\n"
1380 "st1w z20.s, p2, [%[outptr7], #2, MUL VL]\n"
1381 "addvl %[outptr7], %[outptr7], #3\n"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]1382 : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7),
1383 [inptr] "+r" (inptr), [p] "+r" (p)
Georgios Pinitas48b3ef82019-10-14 19:03:09 +0100[diff] [blame]1384 : [w] "r" (w), [biasptr] "r" (biasptr)
1385 : "x8", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "memory", "cc"
Georgios Pinitascfa2bba2019-06-27 17:00:52 +0100[diff] [blame]1386 );
1387 }
1388 break;
1389
1390
1391 }
1392 }
1393 }
1394 }
1395}
1396
Michalis Spyrou20fca522021-06-07 14:23:57 +0100[diff] [blame]1397#endif // ARM_COMPUTE_ENABLE_SVE