Gitiles
Code Review Sign In
review.mlplatform.org / ml / ComputeLibrary / 7b7858df42fccefbe6eb086ad516d5c011becd07 / . / src / core / CL / cl_kernels / transpose.cl
blob: c30158f280346d043552c3c73b65eaa83da892cd [file] [log] [blame]
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100[diff] [blame]1/*
2 * Copyright (c) 2017 ARM Limited.
3 *
4 * SPDX-License-Identifier: MIT
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to
8 * deal in the Software without restriction, including without limitation the
9 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
10 * sell copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in all
14 * copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22 * SOFTWARE.
23 */
24#include "helpers.h"
25
26#define SWAP_ROW(u0, l0) \
27 ({ \
28 tmp_swap = u0; \
29 u0 = l0; \
30 l0 = tmp_swap; \
31 })
32
33#define SWAP_4x4(u0, u1, u2, u3, l0, l1, l2, l3) \
34 ({ \
35 VEC_DATA_TYPE(DATA_TYPE, 4) \
36 tmp_swap; \
37 SWAP_ROW(u0, l0); \
38 SWAP_ROW(u1, l1); \
39 SWAP_ROW(u2, l2); \
40 SWAP_ROW(u3, l3); \
41 })
42
43#define SWAP_8x8(u0, u1, u2, u3, u4, u5, u6, u7, l0, l1, l2, l3, l4, l5, l6, l7) \
44 ({ \
45 VEC_DATA_TYPE(DATA_TYPE, 8) \
46 tmp_swap; \
47 SWAP_ROW(u0, l0); \
48 SWAP_ROW(u1, l1); \
49 SWAP_ROW(u2, l2); \
50 SWAP_ROW(u3, l3); \
51 SWAP_ROW(u4, l4); \
52 SWAP_ROW(u5, l5); \
53 SWAP_ROW(u6, l6); \
54 SWAP_ROW(u7, l7); \
55 })
56
57#define TRANSPOSE_4x4(u0, u1, u2, u3) \
58 ({ \
59 VEC_DATA_TYPE(DATA_TYPE, 4) \
60 tmp; \
61 tmp.s012 = u0.s123; \
62 u0.s1 = u1.s0; \
63 u0.s2 = u2.s0; \
64 u0.s3 = u3.s0; \
65 u1.s0 = tmp.s0; \
66 u2.s0 = tmp.s1; \
67 u3.s0 = tmp.s2; \
68 \
69 tmp.s01 = u1.s23; \
70 u1.s2 = u2.s1; \
71 u1.s3 = u3.s1; \
72 u2.s1 = tmp.s0; \
73 u3.s1 = tmp.s1; \
74 \
75 tmp.s0 = u2.s3; \
76 u2.s3 = u3.s2; \
77 u3.s2 = tmp.s0; \
78 })
79
80#define TRANSPOSE_8x8(u0, u1, u2, u3, u4, u5, u6, u7) \
81 ({ \
82 TRANSPOSE_4x4(u0.s0123, u1.s0123, u2.s0123, u3.s0123); \
83 TRANSPOSE_4x4(u0.s4567, u1.s4567, u2.s4567, u3.s4567); \
84 TRANSPOSE_4x4(u4.s0123, u5.s0123, u6.s0123, u7.s0123); \
85 TRANSPOSE_4x4(u4.s4567, u5.s4567, u6.s4567, u7.s4567); \
86 SWAP_4x4(u0.s4567, u1.s4567, u2.s4567, u3.s4567, u4.s0123, u5.s0123, u6.s0123, u7.s0123); \
87 })
88
89#define TRANSPOSE_16x16(u0, u1, u2, u3, u4, u5, u6, u7, u8, u9, u10, u11, u12, u13, u14, u15) \
90 ({ \
91 TRANSPOSE_8x8(u0.s01234567, u1.s01234567, u2.s01234567, u3.s01234567, u4.s01234567, u5.s01234567, u6.s01234567, u7.s01234567); \
92 TRANSPOSE_8x8(u0.s89ABCDEF, u1.s89ABCDEF, u2.s89ABCDEF, u3.s89ABCDEF, u4.s89ABCDEF, u5.s89ABCDEF, u6.s89ABCDEF, u7.s89ABCDEF); \
93 TRANSPOSE_8x8(u8.s01234567, u9.s01234567, u10.s01234567, u11.s01234567, u12.s01234567, u13.s01234567, u14.s01234567, u15.s01234567); \
94 TRANSPOSE_8x8(u8.s89ABCDEF, u9.s89ABCDEF, u10.s89ABCDEF, u11.s89ABCDEF, u12.s89ABCDEF, u13.s89ABCDEF, u14.s89ABCDEF, u15.s89ABCDEF); \
95 SWAP_8x8(u0.s89ABCDEF, u1.s89ABCDEF, u2.s89ABCDEF, u3.s89ABCDEF, u4.s89ABCDEF, u5.s89ABCDEF, u6.s89ABCDEF, u7.s89ABCDEF, \
96 u8.s01234567, u9.s01234567, u10.s01234567, u11.s01234567, u12.s01234567, u13.s01234567, u14.s01234567, u15.s01234567); \
97 })
98
99#ifndef DATA_TYPE_IN_BYTES
100#error DATA_TYPE_IN_BYTES not set for the transpose OpenCL kernel
101#endif
102
103#if DATA_TYPE_IN_BYTES == 4
104#define DATA_TYPE uint
105#define TRANSPOSE() TRANSPOSE_4x4(u0, u1, u2, u3)
106#define VLOAD(x, y) vload4(x, y)
107#define VSTORE(x, y, z) vstore4(x, y, z)
108#define BLOCK_SIZE 4
109#elif DATA_TYPE_IN_BYTES == 2
110#define DATA_TYPE ushort
111#define TRANSPOSE() TRANSPOSE_8x8(u0, u1, u2, u3, u4, u5, u6, u7)
112#define VLOAD(x, y) vload8(x, y)
113#define VSTORE(x, y, z) vstore8(x, y, z)
114#define BLOCK_SIZE 8
115#elif DATA_TYPE_IN_BYTES == 1
116#define DATA_TYPE uchar
117#define TRANSPOSE() TRANSPOSE_16x16(u0, u1, u2, u3, u4, u5, u6, u7, u8, u9, u10, u11, u12, u13, u14, u15)
118#define VLOAD(x, y) vload16(x, y)
119#define VSTORE(x, y, z) vstore16(x, y, z)
120#define BLOCK_SIZE 16
121#else
122#error DATA_TYPE_IN_BYTES not supported for transpose
123#endif
124
125/** This OpenCL kernel computes the matrix transposition of input matrix
126 *
127 * @attention The number of bytes of the data type need to be passed at compile time using -DDATA_TYPE_IN_BYTES. DATA_TYPE_IN_BYTES can be:
128 * -# -DDATA_TYPE_IN_BYTES=1 for transposing U8 or S8 matrices
129 * -# -DDATA_TYPE_IN_BYTES=2 for transposing U16, S16 or FP16 matrices
130 * -# -DDATA_TYPE_IN_BYTES=4 for transposing U32, S32 or FP32 matrices
131 *
132 * @param[in] src_ptr Pointer to the source matrix. Supported data types: U8/S8/U16/S16/F16/U32/S32/F32
133 * @param[in] src_stride_x Stride of the source matrix in X dimension (in bytes)
134 * @param[in] src_step_x src_stride_x * number of elements along X processed per workitem(in bytes)
135 * @param[in] src_stride_y Stride of the source matrix in Y dimension (in bytes)
136 * @param[in] src_step_y src_stride_y * number of elements along Y processed per workitem(in bytes)
137 * @param[in] src_offset_first_element_in_bytes The offset of the first element in the source matrix
138 * @param[out] dst_ptr Pointer to the destination matrix Supported data type: same as src_ptr
139 * @param[in] dst_stride_x Stride of the destination matrix in X dimension (in bytes)
140 * @param[in] dst_step_x dst_gx_stride_x * number of elements along X processed per workitem(in bytes)
141 * @param[in] dst_stride_y Stride of the destination matrix in Y dimension (in bytes)
142 * @param[in] dst_step_y dst_gx_stride_y * number of elements along Y processed per workitem(in bytes)
143 * @param[in] dst_offset_first_element_in_bytes The offset of the first element in the destination matrix
144 */
145__kernel void transpose(IMAGE_DECLARATION(src),
146 IMAGE_DECLARATION(dst))
147{
148 uint x = get_global_id(0) * BLOCK_SIZE;
149 uint y = get_global_id(1) * BLOCK_SIZE;
150
151 // Compute source address
152 Image src = CONVERT_TO_IMAGE_STRUCT(src);
153
154 // Load the NxN block at (x, y)
155 VEC_DATA_TYPE(DATA_TYPE, BLOCK_SIZE)
156 u0 = VLOAD(0, (__global DATA_TYPE *)(offset(&src, 0, 0)));
157 VEC_DATA_TYPE(DATA_TYPE, BLOCK_SIZE)
158 u1 = VLOAD(0, (__global DATA_TYPE *)(offset(&src, 0, 1)));
159 VEC_DATA_TYPE(DATA_TYPE, BLOCK_SIZE)
160 u2 = VLOAD(0, (__global DATA_TYPE *)(offset(&src, 0, 2)));
161 VEC_DATA_TYPE(DATA_TYPE, BLOCK_SIZE)
162 u3 = VLOAD(0, (__global DATA_TYPE *)(offset(&src, 0, 3)));
163#if BLOCK_SIZE > 4
164 VEC_DATA_TYPE(DATA_TYPE, BLOCK_SIZE)
165 u4 = VLOAD(0, (__global DATA_TYPE *)(offset(&src, 0, 4)));
166 VEC_DATA_TYPE(DATA_TYPE, BLOCK_SIZE)
167 u5 = VLOAD(0, (__global DATA_TYPE *)(offset(&src, 0, 5)));
168 VEC_DATA_TYPE(DATA_TYPE, BLOCK_SIZE)
169 u6 = VLOAD(0, (__global DATA_TYPE *)(offset(&src, 0, 6)));
170 VEC_DATA_TYPE(DATA_TYPE, BLOCK_SIZE)
171 u7 = VLOAD(0, (__global DATA_TYPE *)(offset(&src, 0, 7)));
172#if BLOCK_SIZE == 16
173 VEC_DATA_TYPE(DATA_TYPE, BLOCK_SIZE)
174 u8 = VLOAD(0, (__global DATA_TYPE *)(offset(&src, 0, 8)));
175 VEC_DATA_TYPE(DATA_TYPE, BLOCK_SIZE)
176 u9 = VLOAD(0, (__global DATA_TYPE *)(offset(&src, 0, 9)));
177 VEC_DATA_TYPE(DATA_TYPE, BLOCK_SIZE)
178 u10 = VLOAD(0, (__global DATA_TYPE *)(offset(&src, 0, 10)));
179 VEC_DATA_TYPE(DATA_TYPE, BLOCK_SIZE)
180 u11 = VLOAD(0, (__global DATA_TYPE *)(offset(&src, 0, 11)));
181 VEC_DATA_TYPE(DATA_TYPE, BLOCK_SIZE)
182 u12 = VLOAD(0, (__global DATA_TYPE *)(offset(&src, 0, 12)));
183 VEC_DATA_TYPE(DATA_TYPE, BLOCK_SIZE)
184 u13 = VLOAD(0, (__global DATA_TYPE *)(offset(&src, 0, 13)));
185 VEC_DATA_TYPE(DATA_TYPE, BLOCK_SIZE)
186 u14 = VLOAD(0, (__global DATA_TYPE *)(offset(&src, 0, 14)));
187 VEC_DATA_TYPE(DATA_TYPE, BLOCK_SIZE)
188 u15 = VLOAD(0, (__global DATA_TYPE *)(offset(&src, 0, 15)));
189#endif /* BLOCK_SIZE == 16 */
190#endif /* BLOCK_SIZE > 4 */
191
192 // Transpose the block
193 TRANSPOSE();
194
195 // Store the block at (y, x)
196 uint dst_offset_in_bytes = y * DATA_TYPE_IN_BYTES + x * dst_stride_y + dst_offset_first_element_in_bytes;
197 VSTORE(u0, 0, (__global DATA_TYPE *)(dst_ptr + dst_offset_in_bytes + 0 * dst_stride_y));
198 VSTORE(u1, 0, (__global DATA_TYPE *)(dst_ptr + dst_offset_in_bytes + 1 * dst_stride_y));
199 VSTORE(u2, 0, (__global DATA_TYPE *)(dst_ptr + dst_offset_in_bytes + 2 * dst_stride_y));
200 VSTORE(u3, 0, (__global DATA_TYPE *)(dst_ptr + dst_offset_in_bytes + 3 * dst_stride_y));
201#if BLOCK_SIZE > 4
202 VSTORE(u4, 0, (__global DATA_TYPE *)(dst_ptr + dst_offset_in_bytes + 4 * dst_stride_y));
203 VSTORE(u5, 0, (__global DATA_TYPE *)(dst_ptr + dst_offset_in_bytes + 5 * dst_stride_y));
204 VSTORE(u6, 0, (__global DATA_TYPE *)(dst_ptr + dst_offset_in_bytes + 6 * dst_stride_y));
205 VSTORE(u7, 0, (__global DATA_TYPE *)(dst_ptr + dst_offset_in_bytes + 7 * dst_stride_y));
206#if BLOCK_SIZE == 16
207 VSTORE(u8, 0, (__global DATA_TYPE *)(dst_ptr + dst_offset_in_bytes + 8 * dst_stride_y));
208 VSTORE(u9, 0, (__global DATA_TYPE *)(dst_ptr + dst_offset_in_bytes + 9 * dst_stride_y));
209 VSTORE(u10, 0, (__global DATA_TYPE *)(dst_ptr + dst_offset_in_bytes + 10 * dst_stride_y));
210 VSTORE(u11, 0, (__global DATA_TYPE *)(dst_ptr + dst_offset_in_bytes + 11 * dst_stride_y));
211 VSTORE(u12, 0, (__global DATA_TYPE *)(dst_ptr + dst_offset_in_bytes + 12 * dst_stride_y));
212 VSTORE(u13, 0, (__global DATA_TYPE *)(dst_ptr + dst_offset_in_bytes + 13 * dst_stride_y));
213 VSTORE(u14, 0, (__global DATA_TYPE *)(dst_ptr + dst_offset_in_bytes + 14 * dst_stride_y));
214 VSTORE(u15, 0, (__global DATA_TYPE *)(dst_ptr + dst_offset_in_bytes + 15 * dst_stride_y));
215#endif /* BLOCK_SIZE == 16 */
216#endif /* BLOCK_SIZE > 4 */
217}