| /* |
| * Copyright (c) 2017 ARM Limited. |
| * |
| * SPDX-License-Identifier: MIT |
| * |
| * Permission is hereby granted, free of charge, to any person obtaining a copy |
| * of this software and associated documentation files (the "Software"), to |
| * deal in the Software without restriction, including without limitation the |
| * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or |
| * sell copies of the Software, and to permit persons to whom the Software is |
| * furnished to do so, subject to the following conditions: |
| * |
| * The above copyright notice and this permission notice shall be included in all |
| * copies or substantial portions of the Software. |
| * |
| * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
| * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
| * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE |
| * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
| * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
| * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
| * SOFTWARE. |
| */ |
| #include "helpers.h" |
| |
| #define SWAP_ROW(u0, l0) \ |
| ({ \ |
| tmp_swap = u0; \ |
| u0 = l0; \ |
| l0 = tmp_swap; \ |
| }) |
| |
| #define SWAP_4x4(u0, u1, u2, u3, l0, l1, l2, l3) \ |
| ({ \ |
| VEC_DATA_TYPE(DATA_TYPE, 4) \ |
| tmp_swap; \ |
| SWAP_ROW(u0, l0); \ |
| SWAP_ROW(u1, l1); \ |
| SWAP_ROW(u2, l2); \ |
| SWAP_ROW(u3, l3); \ |
| }) |
| |
| #define SWAP_8x8(u0, u1, u2, u3, u4, u5, u6, u7, l0, l1, l2, l3, l4, l5, l6, l7) \ |
| ({ \ |
| VEC_DATA_TYPE(DATA_TYPE, 8) \ |
| tmp_swap; \ |
| SWAP_ROW(u0, l0); \ |
| SWAP_ROW(u1, l1); \ |
| SWAP_ROW(u2, l2); \ |
| SWAP_ROW(u3, l3); \ |
| SWAP_ROW(u4, l4); \ |
| SWAP_ROW(u5, l5); \ |
| SWAP_ROW(u6, l6); \ |
| SWAP_ROW(u7, l7); \ |
| }) |
| |
| #define TRANSPOSE_4x4(u0, u1, u2, u3) \ |
| ({ \ |
| VEC_DATA_TYPE(DATA_TYPE, 4) \ |
| tmp; \ |
| tmp.s012 = u0.s123; \ |
| u0.s1 = u1.s0; \ |
| u0.s2 = u2.s0; \ |
| u0.s3 = u3.s0; \ |
| u1.s0 = tmp.s0; \ |
| u2.s0 = tmp.s1; \ |
| u3.s0 = tmp.s2; \ |
| \ |
| tmp.s01 = u1.s23; \ |
| u1.s2 = u2.s1; \ |
| u1.s3 = u3.s1; \ |
| u2.s1 = tmp.s0; \ |
| u3.s1 = tmp.s1; \ |
| \ |
| tmp.s0 = u2.s3; \ |
| u2.s3 = u3.s2; \ |
| u3.s2 = tmp.s0; \ |
| }) |
| |
| #define TRANSPOSE_8x8(u0, u1, u2, u3, u4, u5, u6, u7) \ |
| ({ \ |
| TRANSPOSE_4x4(u0.s0123, u1.s0123, u2.s0123, u3.s0123); \ |
| TRANSPOSE_4x4(u0.s4567, u1.s4567, u2.s4567, u3.s4567); \ |
| TRANSPOSE_4x4(u4.s0123, u5.s0123, u6.s0123, u7.s0123); \ |
| TRANSPOSE_4x4(u4.s4567, u5.s4567, u6.s4567, u7.s4567); \ |
| SWAP_4x4(u0.s4567, u1.s4567, u2.s4567, u3.s4567, u4.s0123, u5.s0123, u6.s0123, u7.s0123); \ |
| }) |
| |
| #define TRANSPOSE_16x16(u0, u1, u2, u3, u4, u5, u6, u7, u8, u9, u10, u11, u12, u13, u14, u15) \ |
| ({ \ |
| TRANSPOSE_8x8(u0.s01234567, u1.s01234567, u2.s01234567, u3.s01234567, u4.s01234567, u5.s01234567, u6.s01234567, u7.s01234567); \ |
| TRANSPOSE_8x8(u0.s89ABCDEF, u1.s89ABCDEF, u2.s89ABCDEF, u3.s89ABCDEF, u4.s89ABCDEF, u5.s89ABCDEF, u6.s89ABCDEF, u7.s89ABCDEF); \ |
| TRANSPOSE_8x8(u8.s01234567, u9.s01234567, u10.s01234567, u11.s01234567, u12.s01234567, u13.s01234567, u14.s01234567, u15.s01234567); \ |
| TRANSPOSE_8x8(u8.s89ABCDEF, u9.s89ABCDEF, u10.s89ABCDEF, u11.s89ABCDEF, u12.s89ABCDEF, u13.s89ABCDEF, u14.s89ABCDEF, u15.s89ABCDEF); \ |
| SWAP_8x8(u0.s89ABCDEF, u1.s89ABCDEF, u2.s89ABCDEF, u3.s89ABCDEF, u4.s89ABCDEF, u5.s89ABCDEF, u6.s89ABCDEF, u7.s89ABCDEF, \ |
| u8.s01234567, u9.s01234567, u10.s01234567, u11.s01234567, u12.s01234567, u13.s01234567, u14.s01234567, u15.s01234567); \ |
| }) |
| |
| #ifndef DATA_TYPE_IN_BYTES |
| #error DATA_TYPE_IN_BYTES not set for the transpose OpenCL kernel |
| #endif |
| |
| #if DATA_TYPE_IN_BYTES == 4 |
| #define DATA_TYPE uint |
| #define TRANSPOSE() TRANSPOSE_4x4(u0, u1, u2, u3) |
| #define VLOAD(x, y) vload4(x, y) |
| #define VSTORE(x, y, z) vstore4(x, y, z) |
| #define BLOCK_SIZE 4 |
| #elif DATA_TYPE_IN_BYTES == 2 |
| #define DATA_TYPE ushort |
| #define TRANSPOSE() TRANSPOSE_8x8(u0, u1, u2, u3, u4, u5, u6, u7) |
| #define VLOAD(x, y) vload8(x, y) |
| #define VSTORE(x, y, z) vstore8(x, y, z) |
| #define BLOCK_SIZE 8 |
| #elif DATA_TYPE_IN_BYTES == 1 |
| #define DATA_TYPE uchar |
| #define TRANSPOSE() TRANSPOSE_16x16(u0, u1, u2, u3, u4, u5, u6, u7, u8, u9, u10, u11, u12, u13, u14, u15) |
| #define VLOAD(x, y) vload16(x, y) |
| #define VSTORE(x, y, z) vstore16(x, y, z) |
| #define BLOCK_SIZE 16 |
| #else |
| #error DATA_TYPE_IN_BYTES not supported for transpose |
| #endif |
| |
| /** This OpenCL kernel computes the matrix transposition of input matrix |
| * |
| * @attention The number of bytes of the data type need to be passed at compile time using -DDATA_TYPE_IN_BYTES. DATA_TYPE_IN_BYTES can be: |
| * -# -DDATA_TYPE_IN_BYTES=1 for transposing U8 or S8 matrices |
| * -# -DDATA_TYPE_IN_BYTES=2 for transposing U16, S16 or FP16 matrices |
| * -# -DDATA_TYPE_IN_BYTES=4 for transposing U32, S32 or FP32 matrices |
| * |
| * @param[in] src_ptr Pointer to the source matrix. Supported data types: U8/S8/U16/S16/F16/U32/S32/F32 |
| * @param[in] src_stride_x Stride of the source matrix in X dimension (in bytes) |
| * @param[in] src_step_x src_stride_x * number of elements along X processed per workitem(in bytes) |
| * @param[in] src_stride_y Stride of the source matrix in Y dimension (in bytes) |
| * @param[in] src_step_y src_stride_y * number of elements along Y processed per workitem(in bytes) |
| * @param[in] src_offset_first_element_in_bytes The offset of the first element in the source matrix |
| * @param[out] dst_ptr Pointer to the destination matrix Supported data type: same as src_ptr |
| * @param[in] dst_stride_x Stride of the destination matrix in X dimension (in bytes) |
| * @param[in] dst_step_x dst_gx_stride_x * number of elements along X processed per workitem(in bytes) |
| * @param[in] dst_stride_y Stride of the destination matrix in Y dimension (in bytes) |
| * @param[in] dst_step_y dst_gx_stride_y * number of elements along Y processed per workitem(in bytes) |
| * @param[in] dst_offset_first_element_in_bytes The offset of the first element in the destination matrix |
| */ |
| __kernel void transpose(IMAGE_DECLARATION(src), |
| IMAGE_DECLARATION(dst)) |
| { |
| uint x = get_global_id(0) * BLOCK_SIZE; |
| uint y = get_global_id(1) * BLOCK_SIZE; |
| |
| // Compute source address |
| Image src = CONVERT_TO_IMAGE_STRUCT(src); |
| |
| // Load the NxN block at (x, y) |
| VEC_DATA_TYPE(DATA_TYPE, BLOCK_SIZE) |
| u0 = VLOAD(0, (__global DATA_TYPE *)(offset(&src, 0, 0))); |
| VEC_DATA_TYPE(DATA_TYPE, BLOCK_SIZE) |
| u1 = VLOAD(0, (__global DATA_TYPE *)(offset(&src, 0, 1))); |
| VEC_DATA_TYPE(DATA_TYPE, BLOCK_SIZE) |
| u2 = VLOAD(0, (__global DATA_TYPE *)(offset(&src, 0, 2))); |
| VEC_DATA_TYPE(DATA_TYPE, BLOCK_SIZE) |
| u3 = VLOAD(0, (__global DATA_TYPE *)(offset(&src, 0, 3))); |
| #if BLOCK_SIZE > 4 |
| VEC_DATA_TYPE(DATA_TYPE, BLOCK_SIZE) |
| u4 = VLOAD(0, (__global DATA_TYPE *)(offset(&src, 0, 4))); |
| VEC_DATA_TYPE(DATA_TYPE, BLOCK_SIZE) |
| u5 = VLOAD(0, (__global DATA_TYPE *)(offset(&src, 0, 5))); |
| VEC_DATA_TYPE(DATA_TYPE, BLOCK_SIZE) |
| u6 = VLOAD(0, (__global DATA_TYPE *)(offset(&src, 0, 6))); |
| VEC_DATA_TYPE(DATA_TYPE, BLOCK_SIZE) |
| u7 = VLOAD(0, (__global DATA_TYPE *)(offset(&src, 0, 7))); |
| #if BLOCK_SIZE == 16 |
| VEC_DATA_TYPE(DATA_TYPE, BLOCK_SIZE) |
| u8 = VLOAD(0, (__global DATA_TYPE *)(offset(&src, 0, 8))); |
| VEC_DATA_TYPE(DATA_TYPE, BLOCK_SIZE) |
| u9 = VLOAD(0, (__global DATA_TYPE *)(offset(&src, 0, 9))); |
| VEC_DATA_TYPE(DATA_TYPE, BLOCK_SIZE) |
| u10 = VLOAD(0, (__global DATA_TYPE *)(offset(&src, 0, 10))); |
| VEC_DATA_TYPE(DATA_TYPE, BLOCK_SIZE) |
| u11 = VLOAD(0, (__global DATA_TYPE *)(offset(&src, 0, 11))); |
| VEC_DATA_TYPE(DATA_TYPE, BLOCK_SIZE) |
| u12 = VLOAD(0, (__global DATA_TYPE *)(offset(&src, 0, 12))); |
| VEC_DATA_TYPE(DATA_TYPE, BLOCK_SIZE) |
| u13 = VLOAD(0, (__global DATA_TYPE *)(offset(&src, 0, 13))); |
| VEC_DATA_TYPE(DATA_TYPE, BLOCK_SIZE) |
| u14 = VLOAD(0, (__global DATA_TYPE *)(offset(&src, 0, 14))); |
| VEC_DATA_TYPE(DATA_TYPE, BLOCK_SIZE) |
| u15 = VLOAD(0, (__global DATA_TYPE *)(offset(&src, 0, 15))); |
| #endif /* BLOCK_SIZE == 16 */ |
| #endif /* BLOCK_SIZE > 4 */ |
| |
| // Transpose the block |
| TRANSPOSE(); |
| |
| // Store the block at (y, x) |
| uint dst_offset_in_bytes = y * DATA_TYPE_IN_BYTES + x * dst_stride_y + dst_offset_first_element_in_bytes; |
| VSTORE(u0, 0, (__global DATA_TYPE *)(dst_ptr + dst_offset_in_bytes + 0 * dst_stride_y)); |
| VSTORE(u1, 0, (__global DATA_TYPE *)(dst_ptr + dst_offset_in_bytes + 1 * dst_stride_y)); |
| VSTORE(u2, 0, (__global DATA_TYPE *)(dst_ptr + dst_offset_in_bytes + 2 * dst_stride_y)); |
| VSTORE(u3, 0, (__global DATA_TYPE *)(dst_ptr + dst_offset_in_bytes + 3 * dst_stride_y)); |
| #if BLOCK_SIZE > 4 |
| VSTORE(u4, 0, (__global DATA_TYPE *)(dst_ptr + dst_offset_in_bytes + 4 * dst_stride_y)); |
| VSTORE(u5, 0, (__global DATA_TYPE *)(dst_ptr + dst_offset_in_bytes + 5 * dst_stride_y)); |
| VSTORE(u6, 0, (__global DATA_TYPE *)(dst_ptr + dst_offset_in_bytes + 6 * dst_stride_y)); |
| VSTORE(u7, 0, (__global DATA_TYPE *)(dst_ptr + dst_offset_in_bytes + 7 * dst_stride_y)); |
| #if BLOCK_SIZE == 16 |
| VSTORE(u8, 0, (__global DATA_TYPE *)(dst_ptr + dst_offset_in_bytes + 8 * dst_stride_y)); |
| VSTORE(u9, 0, (__global DATA_TYPE *)(dst_ptr + dst_offset_in_bytes + 9 * dst_stride_y)); |
| VSTORE(u10, 0, (__global DATA_TYPE *)(dst_ptr + dst_offset_in_bytes + 10 * dst_stride_y)); |
| VSTORE(u11, 0, (__global DATA_TYPE *)(dst_ptr + dst_offset_in_bytes + 11 * dst_stride_y)); |
| VSTORE(u12, 0, (__global DATA_TYPE *)(dst_ptr + dst_offset_in_bytes + 12 * dst_stride_y)); |
| VSTORE(u13, 0, (__global DATA_TYPE *)(dst_ptr + dst_offset_in_bytes + 13 * dst_stride_y)); |
| VSTORE(u14, 0, (__global DATA_TYPE *)(dst_ptr + dst_offset_in_bytes + 14 * dst_stride_y)); |
| VSTORE(u15, 0, (__global DATA_TYPE *)(dst_ptr + dst_offset_in_bytes + 15 * dst_stride_y)); |
| #endif /* BLOCK_SIZE == 16 */ |
| #endif /* BLOCK_SIZE > 4 */ |
| } |