| /* |
| * Copyright (c) 2017-2021 Arm Limited. |
| * |
| * SPDX-License-Identifier: MIT |
| * |
| * Permission is hereby granted, free of charge, to any person obtaining a copy |
| * of this software and associated documentation files (the "Software"), to |
| * deal in the Software without restriction, including without limitation the |
| * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or |
| * sell copies of the Software, and to permit persons to whom the Software is |
| * furnished to do so, subject to the following conditions: |
| * |
| * The above copyright notice and this permission notice shall be included in all |
| * copies or substantial portions of the Software. |
| * |
| * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
| * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
| * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE |
| * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
| * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
| * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
| * SOFTWARE. |
| */ |
| #include "gemm_helpers.h" |
| #include "repeat.h" |
| |
| #if defined(M0) && defined(K0) && defined(V0) && defined(DATA_TYPE) && defined(SRC_WIDTH) && defined(SRC_HEIGHT) && defined(PARTIAL_LOAD_M0) && defined(PARTIAL_LOAD_K0) |
| #define INC2 (VEC_DATA_TYPE(uint, 2))(0, 1) |
| #define INC3 (VEC_DATA_TYPE(uint, 3))(0, 1, 2) |
| #define INC4 (VEC_DATA_TYPE(uint, 4))(0, 1, 2, 3) |
| #define INC8 (VEC_DATA_TYPE(uint, 8))(0, 1, 2, 3, 4, 5, 6, 7) |
| #define INC16 (VEC_DATA_TYPE(uint, 16))(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15) |
| #define CONCAT_INC(K0) INC##K0 |
| #define INC(K0) CONCAT_INC(K0) |
| |
| #if(SRC_WIDTH % K0) |
| #define BOUNDARY_CONDITION_X(x, a) \ |
| ({ \ |
| a = select(0, a, CONVERT(((x * (VEC_DATA_TYPE(uint, K0))K0 + INC(K0)) < (VEC_DATA_TYPE(uint, K0))SRC_WIDTH), VEC_DATA_TYPE(DATA_TYPE, K0))); \ |
| }) |
| #else // (SRC_WIDTH % K0) |
| #define BOUNDARY_CONDITION_X(x, a) \ |
| ({}) |
| #endif // (SRC_WIDTH % K0) |
| |
| #define LOAD_TENSOR_BOUNDARY_AWARE_M0XK0(M0, K0, DATA_TYPE, a, input_ptr, src_stride_y, zin) \ |
| ({ \ |
| if(y * M0 + M0 >= SRC_HEIGHT && PARTIAL_LOAD_M0 != 0) \ |
| { \ |
| if(x * K0 + K0 >= SRC_WIDTH && (PARTIAL_LOAD_K0 != 0)) \ |
| { \ |
| LOAD_TENSOR_M0XN0(PARTIAL_LOAD_M0, PARTIAL_LOAD_K0, DATA_TYPE, a, input_ptr, src_stride_y, zin); \ |
| } \ |
| else \ |
| { \ |
| LOAD_TENSOR_M0XN0(PARTIAL_LOAD_M0, K0, DATA_TYPE, a, input_ptr, src_stride_y, zin); \ |
| } \ |
| } \ |
| else \ |
| { \ |
| if(x * K0 + K0 >= SRC_WIDTH && (PARTIAL_LOAD_K0 != 0)) \ |
| { \ |
| LOAD_TENSOR_M0XN0(M0, PARTIAL_LOAD_K0, DATA_TYPE, a, input_ptr, src_stride_y, zin); \ |
| } \ |
| else \ |
| { \ |
| LOAD_TENSOR_M0XN0(M0, K0, DATA_TYPE, a, input_ptr, src_stride_y, zin); \ |
| } \ |
| } \ |
| }) |
| |
| /** This OpenCL kernel reshapes the lhs input matrix. The kernel splits the input matrix in blocks of size M0xK0 and stores each one (not transposed) in |
| * the output matrix unrolling the values. |
| * |
| * @note The data type must be passed at compile time using -DDATA_TYPE (e.g. -DDATA_TYPE=float) |
| * @note The width of the input tensor must be passed at compile time using -DSRC_WIDTH (e.g. -DSRC_WIDTH=16) |
| * @note The height of the input tensor must be passed at compile time using -DSRC_HEIGHT (e.g. -DSRC_HEIGHT=16) |
| * @note The block's dimensions (M0 and K0) must be passed at compile time using -DM0 and -DK0 (e.g. -DM0=2, -DK0=2). |
| * @note The number of M0xK0 vertical blocks to store on the same output row must be passed at compile time using -DV0 (e.g. -DV0=2) |
| * @note The size of the partial load block in y must be passed at compile time using -DPARTIAL_LOAD_M0 (e.g. -DPARTIAL_LOAD_M0=1) |
| * @note The size of the partial load block in x must be passed at compile time using -DPARTIAL_LOAD_K0 (e.g. -DPARTIAL_LOAD_K0=1) |
| * @note Only the following values for M0, K0 and V0 are supported: |
| * M0: 2,3,4,5,6,7,8 |
| * K0: 2,3,4,8,16 |
| * V0: greater than 0 |
| * @note In case the input has to be reinterpreted as a 3D tensor (e.g. input of convolution layer 1x1), the following information must be passed at compile time: |
| * -# REINTERPRET_INPUT_AS_3D: To reinterpret the input as 3D |
| * -# HEIGHT_GEMM3D: The height of the input in case it has to be reinterpreted as a 3D tensor. |
| * -# DEPTH_GEMM3D: The depth of the input in case it has to be reinterpreted as a 3D tensor |
| * (HEIGHT_GEMM3D * DEPTH_GEMM3D) = columns matrix A NOT reshaped |
| * @note If the M0xK0 blocks have to be interleaved, the option -DINTERLEAVE must passed at compile time. |
| * |
| * @param[in] src_ptr Pointer to the source LHS tensor. Supported data types: All |
| * @param[in] src_stride_x Stride of the source LHS tensor in X dimension (in bytes) |
| * @param[in] src_step_x src_stride_x * number of elements along X processed per workitem(in bytes) |
| * @param[in] src_stride_y Stride of the source LHS tensor in Y dimension (in bytes) |
| * @param[in] src_step_y src_stride_y * number of elements along Y processed per workitem(in bytes) |
| * @param[in] src_stride_z Stride of the source LHS tensor in Z dimension (in bytes) |
| * @param[in] src_step_z src_stride_z * number of elements along Z processed per workitem(in bytes) |
| * @param[in] src_offset_first_element_in_bytes The offset of the first element in the source LHS tensor |
| * @param[out] dst_ptr Pointer to the destination matrix Supported data types: same as @p src_ptr |
| * @param[in] dst_stride_x Stride of the destination matrix in X dimension (in bytes) |
| * @param[in] dst_step_x dst_stride_x * number of elements along X processed per workitem(in bytes) |
| * @param[in] dst_stride_y Stride of the destination matrix in Y dimension (in bytes) |
| * @param[in] dst_step_y dst_stride_y * number of elements along Y processed per workitem(in bytes) |
| * @param[in] dst_stride_z Stride of the destination tensor in Z dimension (in bytes) |
| * @param[in] dst_step_z dst_stride_z * number of elements along Z processed per workitem(in bytes) |
| * @param[in] dst_offset_first_element_in_bytes The offset of the first element in the destination matrix |
| * @param[in] cross_plane_pad (Optional) Bottom paddings in unit of elements (only if defined REINTERPRET_INPUT_AS_3D) |
| */ |
| __kernel void gemm_reshape_lhs_matrix_nt(TENSOR3D_DECLARATION(src), |
| TENSOR3D_DECLARATION(dst) |
| #if defined(REINTERPRET_INPUT_AS_3D) |
| , |
| uint cross_plane_pad |
| #endif // REINTERPRET_INPUT_AS_3D |
| ) |
| { |
| // Block size |
| #define BLOCK_SIZE ((M0) * (K0)) |
| |
| // Output offset X |
| #if defined(INTERLEAVE) |
| #define OUTPUT_OFFSET_X (K0) |
| #else // defined(INTERLEAVE) |
| #define OUTPUT_OFFSET_X (BLOCK_SIZE) |
| #endif // defined(INTERLEAVE) |
| |
| // Output step X |
| #if defined(INTERLEAVE) |
| #define OUTPUT_STEP_X (K0) * (V0) |
| #else // Do not interleave |
| #define OUTPUT_STEP_X (K0) |
| #endif // defined(INTERLEAVE) |
| |
| // Compute source and destination addresses |
| uint x = get_global_id(0); |
| uint y = get_global_id(1); |
| uint z = get_global_id(2); |
| |
| // ------------------ Compute input/output addresses --------------------------- |
| |
| // Compute the input address |
| __global uchar *input_ptr = src_ptr + src_offset_first_element_in_bytes + x * (uint)K0 * sizeof(DATA_TYPE) + y * (uint)M0 * src_stride_y; |
| |
| // Compute the output address |
| __global uchar *output_ptr = dst_ptr + dst_offset_first_element_in_bytes + (x * (uint)BLOCK_SIZE * (uint)V0 * sizeof(DATA_TYPE)) + ((y / (uint)V0) * (uint)dst_stride_y) + ((y % V0) * |
| (uint)OUTPUT_OFFSET_X * sizeof(DATA_TYPE)); |
| |
| // Create variables: uint zin0=0, zin1=0, zin2=0...zin(M0-1)=0; |
| REPEAT_VAR_INIT_TO_CONST(M0, uint, zin, 0); |
| |
| #if defined(REINTERPRET_INPUT_AS_3D) |
| // Add offset for batched GEMM. The batches will be in the fourth dimension and for this reason we |
| // multiply src_stride_z by DEPTH_GEMM3D |
| |
| input_ptr += z * (uint)src_stride_z * DEPTH_GEMM3D; |
| |
| // The plane (zin) is calculated dividing M (y * M0) by HEIGHT_GEMM3D |
| CALCULATE_Z_OFFSET(M0, uint, zin, y, HEIGHT_GEMM3D, DEPTH_GEMM3D, cross_plane_pad, src_stride_y); |
| |
| #else // defined(REINTERPRET_INPUT_AS_3D) |
| |
| input_ptr += z * (uint)src_stride_z; |
| |
| #endif // defined(REINTERPRET_INPUT_AS_3D) |
| |
| // Add offset for batched GEMM |
| output_ptr += z * (uint)dst_stride_z; |
| |
| // ---------------------------Load input values -------------------------------- |
| // Load values from the LHS matrix |
| REPEAT_VAR_INIT_TO_CONST(M0, VEC_DATA_TYPE(DATA_TYPE, K0), a, 0); |
| |
| LOAD_TENSOR_BOUNDARY_AWARE_M0XK0(M0, K0, DATA_TYPE, a, input_ptr, src_stride_y, zin); |
| |
| // ---------------------------Store output values ------------------------------ |
| REPEAT_VAR_INIT_TO_CONST(16, uint, zout, 0); |
| STORE_BLOCK(M0, K0, DATA_TYPE, a, output_ptr, OUTPUT_STEP_X * sizeof(DATA_TYPE), zout); |
| |
| #undef BLOCK_SIZE |
| #undef OUTPUT_OFFSET_X |
| #undef OUTPUT_STEP_X |
| } |
| |
| #if M0 == 2 |
| #define TRANSPOSE_COLUMN_AND_STORE(output_ptr, output_step_x, i) \ |
| ({ \ |
| VEC_DATA_TYPE(DATA_TYPE, M0) \ |
| res = (VEC_DATA_TYPE(DATA_TYPE, M0))(a0.s##i, a1.s##i); \ |
| VSTORE(M0) \ |
| (res, 0, (__global DATA_TYPE *)(output_ptr + 0x##i * output_step_x * sizeof(DATA_TYPE))); \ |
| }) |
| #elif M0 == 3 // M0 == 3 |
| #define TRANSPOSE_COLUMN_AND_STORE(output_ptr, output_step_x, i) \ |
| ({ \ |
| VEC_DATA_TYPE(DATA_TYPE, M0) \ |
| res = (VEC_DATA_TYPE(DATA_TYPE, M0))(a0.s##i, a1.s##i, a2.s##i); \ |
| VSTORE(M0) \ |
| (res, 0, (__global DATA_TYPE *)(output_ptr + 0x##i * output_step_x * sizeof(DATA_TYPE))); \ |
| }) |
| #elif M0 == 4 // M0 == 4 |
| #define TRANSPOSE_COLUMN_AND_STORE(output_ptr, output_step_x, i) \ |
| ({ \ |
| VEC_DATA_TYPE(DATA_TYPE, M0) \ |
| res = (VEC_DATA_TYPE(DATA_TYPE, M0))(a0.s##i, a1.s##i, a2.s##i, a3.s##i); \ |
| VSTORE(M0) \ |
| (res, 0, (__global DATA_TYPE *)(output_ptr + 0x##i * output_step_x * sizeof(DATA_TYPE))); \ |
| }) |
| #elif M0 == 5 // M0 == 5 |
| #define TRANSPOSE_COLUMN_AND_STORE(output_ptr, output_step_x, i) \ |
| ({ \ |
| VEC_DATA_TYPE(DATA_TYPE, 4) \ |
| res0 = (VEC_DATA_TYPE(DATA_TYPE, 4))(a0.s##i, a1.s##i, a2.s##i, a3.s##i); \ |
| DATA_TYPE res1 = a4.s##i; \ |
| VSTORE(4) \ |
| (res0, 0, (__global DATA_TYPE *)(output_ptr + 0x##i * output_step_x * sizeof(DATA_TYPE))); \ |
| *((__global DATA_TYPE *)(output_ptr + 0x##i * output_step_x * sizeof(DATA_TYPE)) + 4) = res1; \ |
| }) |
| #elif M0 == 6 // M0 == 6 |
| #define TRANSPOSE_COLUMN_AND_STORE(output_ptr, output_step_x, i) \ |
| ({ \ |
| VEC_DATA_TYPE(DATA_TYPE, 4) \ |
| res0 = (VEC_DATA_TYPE(DATA_TYPE, 4))(a0.s##i, a1.s##i, a2.s##i, a3.s##i); \ |
| VEC_DATA_TYPE(DATA_TYPE, 2) \ |
| res1 = (VEC_DATA_TYPE(DATA_TYPE, 2))(a4.s##i, a5.s##i); \ |
| VSTORE(4) \ |
| (res0, 0, (__global DATA_TYPE *)(output_ptr + 0x##i * output_step_x * sizeof(DATA_TYPE))); \ |
| VSTORE(2) \ |
| (res1, 0, (__global DATA_TYPE *)(output_ptr + 0x##i * output_step_x * sizeof(DATA_TYPE)) + 4); \ |
| }) |
| #elif M0 == 7 // M0 == 7 |
| #define TRANSPOSE_COLUMN_AND_STORE(output_ptr, output_step_x, i) \ |
| ({ \ |
| VEC_DATA_TYPE(DATA_TYPE, 4) \ |
| res0 = (VEC_DATA_TYPE(DATA_TYPE, 4))(a0.s##i, a1.s##i, a2.s##i, a3.s##i); \ |
| VEC_DATA_TYPE(DATA_TYPE, 3) \ |
| res1 = (VEC_DATA_TYPE(DATA_TYPE, 3))(a4.s##i, a5.s##i, a6.s##i); \ |
| VSTORE(4) \ |
| (res0, 0, (__global DATA_TYPE *)(output_ptr + 0x##i * output_step_x * sizeof(DATA_TYPE))); \ |
| VSTORE(3) \ |
| (res1, 0, (__global DATA_TYPE *)(output_ptr + 0x##i * output_step_x * sizeof(DATA_TYPE)) + 4); \ |
| }) |
| #elif M0 == 8 // M0 == 8 |
| #define TRANSPOSE_COLUMN_AND_STORE(output_ptr, output_step_x, i) \ |
| ({ \ |
| VEC_DATA_TYPE(DATA_TYPE, M0) \ |
| res = (VEC_DATA_TYPE(DATA_TYPE, M0))(a0.s##i, a1.s##i, a2.s##i, a3.s##i, a4.s##i, a5.s##i, a6.s##i, a7.s##i); \ |
| VSTORE(M0) \ |
| (res, 0, (__global DATA_TYPE *)(output_ptr + 0x##i * output_step_x * sizeof(DATA_TYPE))); \ |
| }) |
| #else // M0 not supported |
| #error "M0 value not supported" |
| #endif // N0 conditions |
| |
| /** This OpenCL kernel reshapes the lhs input matrix. The kernel splits the input matrix in blocks of size M0xK0 and stores each one (transposed) in |
| * the output matrix unrolling the values. |
| * |
| * @note The data type must be passed at compile time using -DDATA_TYPE (e.g. -DDATA_TYPE=float) |
| * @note The width of the input tensor must be passed at compile time using -DSRC_WIDTH (e.g. -DSRC_WIDTH=16) |
| * @note The height of the input tensor must be passed at compile time using -DSRC_HEIGHT (e.g. -DSRC_HEIGHT=16) |
| * @note The block's dimensions (M0 and K0) must be passed at compile time using -DM0 and -DK0 (e.g. -DM0=2, -DK0=2). |
| * @note The number of M0xK0 vertical blocks to store on the same output row must be passed at compile time using -DV0 (e.g. -DV0=2) |
| * @note The size of the partial load block in y must be passed at compile time using -DPARTIAL_LOAD_M0 (e.g. -DPARTIAL_LOAD_M0=1) |
| * @note The size of the partial load block in x must be passed at compile time using -DPARTIAL_LOAD_K0 (e.g. -DPARTIAL_LOAD_K0=1) |
| * @note Only the following values for M0, K0 and V0 are supported: |
| * M0: 2,3,4,5,6,7,8 |
| * K0: 2,3,4,8,16 |
| * V0: greater than 0 |
| * @note In case the input has to be reinterpreted as a 3D tensor (e.g. input of convolution layer 1x1), the following information must be passed at compile time: |
| * -# REINTERPRET_INPUT_AS_3D: To reinterpret the input as 3D |
| * -# HEIGHT_GEMM3D: The height of the input in case it has to be reinterpreted as a 3D tensor. |
| * -# DEPTH_GEMM3D: The depth of the input in case it has to be reinterpreted as a 3D tensor |
| * (HEIGHT_GEMM3D * DEPTH_GEMM3D) = columns matrix A NOT reshaped |
| * @note If the M0xK0 blocks have to be interleaved, the option -DINTERLEAVE must passed at compile time. |
| * |
| * @param[in] src_ptr Pointer to the source LHS tensor. Supported data types: All |
| * @param[in] src_stride_x Stride of the source LHS tensor in X dimension (in bytes) |
| * @param[in] src_step_x src_stride_x * number of elements along X processed per workitem(in bytes) |
| * @param[in] src_stride_y Stride of the source LHS tensor in Y dimension (in bytes) |
| * @param[in] src_step_y src_stride_y * number of elements along Y processed per workitem(in bytes) |
| * @param[in] src_stride_z Stride of the source LHS tensor in Z dimension (in bytes) |
| * @param[in] src_step_z src_stride_z * number of elements along Z processed per workitem(in bytes) |
| * @param[in] src_offset_first_element_in_bytes The offset of the first element in the source LHS tensor |
| * @param[out] dst_ptr Pointer to the destination matrix Supported data types: same as @p src_ptr |
| * @param[in] dst_stride_x Stride of the destination matrix in X dimension (in bytes) |
| * @param[in] dst_step_x dst_stride_x * number of elements along X processed per workitem(in bytes) |
| * @param[in] dst_stride_y Stride of the destination matrix in Y dimension (in bytes) |
| * @param[in] dst_step_y dst_stride_y * number of elements along Y processed per workitem(in bytes) |
| * @param[in] dst_stride_z Stride of the destination tensor in Z dimension (in bytes) |
| * @param[in] dst_step_z dst_stride_z * number of elements along Z processed per workitem(in bytes) |
| * @param[in] dst_offset_first_element_in_bytes The offset of the first element in the destination matrix |
| * @param[in] cross_plane_pad (Optional) Bottom paddings in unit of elements (only if defined REINTERPRET_INPUT_AS_3D) |
| */ |
| __kernel void gemm_reshape_lhs_matrix_t(TENSOR3D_DECLARATION(src), |
| TENSOR3D_DECLARATION(dst) |
| #if defined(REINTERPRET_INPUT_AS_3D) |
| , |
| uint cross_plane_pad |
| #endif // REINTERPRET_INPUT_AS_3D |
| ) |
| { |
| // Block size |
| #define BLOCK_SIZE ((M0) * (K0)) |
| |
| // Output offset X |
| #if defined(INTERLEAVE) |
| #define OUTPUT_OFFSET_X (M0) |
| #else // defined(INTERLEAVE) |
| #define OUTPUT_OFFSET_X (BLOCK_SIZE) |
| #endif // defined(INTERLEAVE) |
| |
| // Output step X |
| #if defined(INTERLEAVE) |
| #define OUTPUT_STEP_X (M0) * (V0) |
| #else // Do not interleave |
| #define OUTPUT_STEP_X (M0) |
| #endif // defined(INTERLEAVE) |
| |
| // Compute source and destination addresses |
| uint x = get_global_id(0); |
| uint y = get_global_id(1); |
| uint z = get_global_id(2); |
| |
| // ------------------ Compute input/output addresses --------------------------- |
| |
| // Compute the input address |
| __global uchar *input_ptr = src_ptr + src_offset_first_element_in_bytes + x * (uint)K0 * sizeof(DATA_TYPE) + y * (uint)M0 * src_stride_y; |
| |
| // Compute the output address |
| __global uchar *output_ptr = dst_ptr + dst_offset_first_element_in_bytes + (x * (uint)BLOCK_SIZE * (uint)V0 * sizeof(DATA_TYPE)) + ((y / (uint)V0) * (uint)dst_stride_y) + ((y % V0) * |
| (uint)OUTPUT_OFFSET_X * sizeof(DATA_TYPE)); |
| |
| // Create variables: uint zin0=0, zin1=0, zin2=0...zin(M0-1)=0; |
| REPEAT_VAR_INIT_TO_CONST(M0, uint, zin, 0); |
| |
| #if defined(REINTERPRET_INPUT_AS_3D) |
| // Add offset for batched GEMM. The batches will be in the fourth dimension and for this reason we |
| // multiply src_stride_z by DEPTH_GEMM3D |
| |
| input_ptr += z * (uint)src_stride_z * DEPTH_GEMM3D; |
| |
| // The plane (zin) is calculated dividing M (y * M0) by HEIGHT_GEMM3D |
| CALCULATE_Z_OFFSET(M0, uint, zin, y, HEIGHT_GEMM3D, DEPTH_GEMM3D, cross_plane_pad, src_stride_y); |
| |
| #else // defined(REINTERPRET_INPUT_AS_3D) |
| |
| input_ptr += z * (uint)src_stride_z; |
| |
| #endif // defined(REINTERPRET_INPUT_AS_3D) |
| |
| // Add offset for batched GEMM |
| output_ptr += z * (uint)dst_stride_z; |
| |
| // ---------------------------Load input values -------------------------------- |
| REPEAT_VAR_INIT_TO_CONST(M0, VEC_DATA_TYPE(DATA_TYPE, K0), a, 0); |
| |
| LOAD_TENSOR_BOUNDARY_AWARE_M0XK0(M0, K0, DATA_TYPE, a, input_ptr, src_stride_y, zin); |
| |
| // ---------------------------Transpose and store block ----------------------- |
| |
| TRANSPOSE_COLUMN_AND_STORE(output_ptr, OUTPUT_STEP_X, 0); |
| TRANSPOSE_COLUMN_AND_STORE(output_ptr, OUTPUT_STEP_X, 1); |
| #if K0 > 2 |
| TRANSPOSE_COLUMN_AND_STORE(output_ptr, OUTPUT_STEP_X, 2); |
| #endif // K0 > 2 |
| #if K0 > 3 |
| TRANSPOSE_COLUMN_AND_STORE(output_ptr, OUTPUT_STEP_X, 3); |
| #endif // K0 > 3 |
| #if K0 > 4 |
| TRANSPOSE_COLUMN_AND_STORE(output_ptr, OUTPUT_STEP_X, 4); |
| TRANSPOSE_COLUMN_AND_STORE(output_ptr, OUTPUT_STEP_X, 5); |
| TRANSPOSE_COLUMN_AND_STORE(output_ptr, OUTPUT_STEP_X, 6); |
| TRANSPOSE_COLUMN_AND_STORE(output_ptr, OUTPUT_STEP_X, 7); |
| #endif // K0 > 4 |
| #if K0 > 8 |
| TRANSPOSE_COLUMN_AND_STORE(output_ptr, OUTPUT_STEP_X, 8); |
| TRANSPOSE_COLUMN_AND_STORE(output_ptr, OUTPUT_STEP_X, 9); |
| TRANSPOSE_COLUMN_AND_STORE(output_ptr, OUTPUT_STEP_X, A); |
| TRANSPOSE_COLUMN_AND_STORE(output_ptr, OUTPUT_STEP_X, B); |
| TRANSPOSE_COLUMN_AND_STORE(output_ptr, OUTPUT_STEP_X, C); |
| TRANSPOSE_COLUMN_AND_STORE(output_ptr, OUTPUT_STEP_X, D); |
| TRANSPOSE_COLUMN_AND_STORE(output_ptr, OUTPUT_STEP_X, E); |
| TRANSPOSE_COLUMN_AND_STORE(output_ptr, OUTPUT_STEP_X, F); |
| #endif // K0 > 8 |
| |
| #undef BLOCK_SIZE |
| #undef OUTPUT_OFFSET_X |
| #undef OUTPUT_STEP_X |
| } |
| #endif // defined(M0) && defined(K0) && defined(V0) && defined(DATA_TYPE) && defined(SRC_WIDTH) && defined(SRC_HEIGHT) && defined(PARTIAL_LOAD_M0) && defined(PARTIAL_LOAD_K0) |
| |
| #if defined(K0) && defined(N0) && defined(H0) && defined(DATA_TYPE) && defined(SRC_HEIGHT) |
| /** This OpenCL kernel reshapes the rhs input matrix. The kernel splits the input matrix in blocks of size K0xN0 and stores each one (not transposed) in |
| * the output matrix unrolling the values. |
| * |
| * @note The data type must be passed at compile time using -DDATA_TYPE (e.g. -DDATA_TYPE=float) |
| * @note The height of the input tensor must be passed at compile time using -DSRC_HEIGHT (e.g. -DSRC_HEIGHT=16) |
| * @note The block's dimensions (K0 and N0) must be passed at compile time using -DK0 and -DN0 (e.g. -DK0=2, -DN0=2). |
| * @note The number of K0xN0 vertical blocks to store on the same output row must be passed at compile time using -DH0 (e.g. -DH0=2) |
| * @note If the K0xN0 blocks have to be interleaved, the option -DINTERLEAVE must passed at compile time. |
| * @note Only the following values for K0, N0 and H0 are supported: |
| * N0: 2,3,4,8,16 |
| * K0: 1,2,3,4,8,16 |
| * H0: greater than 0 |
| * |
| * @param[in] src_ptr Pointer to the source RHS tensor. Supported data types: All |
| * @param[in] src_stride_x Stride of the source RHS tensor in X dimension (in bytes) |
| * @param[in] src_step_x src_stride_x * number of elements along X processed per workitem(in bytes) |
| * @param[in] src_stride_y Stride of the source RHS tensor in Y dimension (in bytes) |
| * @param[in] src_step_y src_stride_y * number of elements along Y processed per workitem(in bytes) |
| * @param[in] src_stride_z Stride of the source RHS tensor in Z dimension (in bytes) |
| * @param[in] src_step_z src_stride_z * number of elements along Z processed per workitem(in bytes) |
| * @param[in] src_offset_first_element_in_bytes The offset of the first element in the source RHS tensor |
| * @param[out] dst_ptr Pointer to the destination matrix Supported data types: same as @p src_ptr |
| * @param[in] dst_stride_x Stride of the destination matrix in X dimension (in bytes) |
| * @param[in] dst_step_x dst_stride_x * number of elements along X processed per workitem(in bytes) |
| * @param[in] dst_stride_y Stride of the destination matrix in Y dimension (in bytes) |
| * @param[in] dst_step_y dst_stride_y * number of elements along Y processed per workitem(in bytes) |
| * @param[in] dst_stride_z Stride of the destination tensor in Z dimension (in bytes) |
| * @param[in] dst_step_z dst_stride_z * number of elements along Z processed per workitem(in bytes) |
| * @param[in] dst_offset_first_element_in_bytes The offset of the first element in the destination matrix |
| */ |
| __kernel void gemm_reshape_rhs_matrix_nt(TENSOR3D_DECLARATION(src), |
| TENSOR3D_DECLARATION(dst)) |
| { |
| // Block size |
| #define BLOCK_SIZE ((K0) * (N0)) |
| |
| // Output offset X |
| #if defined(INTERLEAVE) |
| #define OUTPUT_OFFSET_X (N0) |
| #else // defined(INTERLEAVE) |
| #define OUTPUT_OFFSET_X (BLOCK_SIZE) |
| #endif // defined(INTERLEAVE) |
| |
| // Output step X |
| #if defined(INTERLEAVE) |
| #define OUTPUT_STEP_X (N0) * (H0) |
| #else // Do not interleave |
| #define OUTPUT_STEP_X (N0) |
| #endif // defined(INTERLEAVE) |
| |
| // Compute source and destination addresses |
| uint x = get_global_id(0); |
| uint y = get_global_id(1); |
| uint z = get_global_id(2); |
| |
| // ------------------ Compute input/output addresses --------------------------- |
| |
| // Compute the input address |
| __global uchar *input_ptr = src_ptr + src_offset_first_element_in_bytes + x * (uint)N0 * sizeof(DATA_TYPE) + y * (uint)K0 * src_stride_y + z * (uint)src_stride_z; |
| |
| // Compute the output address |
| __global uchar *output_ptr = dst_ptr + dst_offset_first_element_in_bytes + (y * (uint)BLOCK_SIZE * (uint)H0 * sizeof(DATA_TYPE)) + ((x % (uint)H0) * (uint)OUTPUT_OFFSET_X * sizeof(DATA_TYPE)) + (( |
| x / (uint)H0) |
| * (uint)dst_stride_y) |
| + z * (uint)dst_stride_z; |
| |
| // ---------------------------Load input values -------------------------------- |
| |
| REPEAT_VAR_INIT_TO_CONST(K0, VEC_DATA_TYPE(DATA_TYPE, N0), a, 0); ////uint a0=0, a1=0, a2=0...a(M0-1)=0; |
| |
| // Load values from the RHS matrix |
| a0 = VLOAD(N0)(0, (__global DATA_TYPE *)(input_ptr + 0 * src_stride_y)); |
| #if K0 > 1 |
| if(y * (uint)K0 + 1 < SRC_HEIGHT) |
| { |
| a1 = VLOAD(N0)(0, (__global DATA_TYPE *)(input_ptr + 1 * src_stride_y)); |
| } |
| #endif // K0 > 1 |
| #if K0 > 2 |
| if(y * (uint)K0 + 2 < SRC_HEIGHT) |
| { |
| a2 = VLOAD(N0)(0, (__global DATA_TYPE *)(input_ptr + 2 * src_stride_y)); |
| } |
| #endif // K0 > 2 |
| #if K0 > 3 |
| if(y * (uint)K0 + 3 < SRC_HEIGHT) |
| { |
| a3 = VLOAD(N0)(0, (__global DATA_TYPE *)(input_ptr + 3 * src_stride_y)); |
| } |
| #endif // K0 > 3 |
| #if K0 > 4 |
| if(y * (uint)K0 + 4 < SRC_HEIGHT) |
| { |
| a4 = VLOAD(N0)(0, (__global DATA_TYPE *)(input_ptr + 4 * src_stride_y)); |
| } |
| if(y * (uint)K0 + 5 < SRC_HEIGHT) |
| { |
| a5 = VLOAD(N0)(0, (__global DATA_TYPE *)(input_ptr + 5 * src_stride_y)); |
| } |
| if(y * (uint)K0 + 6 < SRC_HEIGHT) |
| { |
| a6 = VLOAD(N0)(0, (__global DATA_TYPE *)(input_ptr + 6 * src_stride_y)); |
| } |
| if(y * (uint)K0 + 7 < SRC_HEIGHT) |
| { |
| a7 = VLOAD(N0)(0, (__global DATA_TYPE *)(input_ptr + 7 * src_stride_y)); |
| } |
| #endif // K0 > 4 |
| #if K0 > 8 |
| if(y * (uint)K0 + 8 < SRC_HEIGHT) |
| { |
| a8 = VLOAD(N0)(0, (__global DATA_TYPE *)(input_ptr + 8 * src_stride_y)); |
| } |
| if(y * (uint)K0 + 9 < SRC_HEIGHT) |
| { |
| a9 = VLOAD(N0)(0, (__global DATA_TYPE *)(input_ptr + 9 * src_stride_y)); |
| } |
| if(y * (uint)K0 + 10 < SRC_HEIGHT) |
| { |
| aA = VLOAD(N0)(0, (__global DATA_TYPE *)(input_ptr + 10 * src_stride_y)); |
| } |
| if(y * (uint)K0 + 11 < SRC_HEIGHT) |
| { |
| aB = VLOAD(N0)(0, (__global DATA_TYPE *)(input_ptr + 11 * src_stride_y)); |
| } |
| if(y * (uint)K0 + 12 < SRC_HEIGHT) |
| { |
| aC = VLOAD(N0)(0, (__global DATA_TYPE *)(input_ptr + 12 * src_stride_y)); |
| } |
| if(y * (uint)K0 + 13 < SRC_HEIGHT) |
| { |
| aD = VLOAD(N0)(0, (__global DATA_TYPE *)(input_ptr + 13 * src_stride_y)); |
| } |
| if(y * (uint)K0 + 14 < SRC_HEIGHT) |
| { |
| aE = VLOAD(N0)(0, (__global DATA_TYPE *)(input_ptr + 14 * src_stride_y)); |
| } |
| if(y * (uint)K0 + 15 < SRC_HEIGHT) |
| { |
| aF = VLOAD(N0)(0, (__global DATA_TYPE *)(input_ptr + 15 * src_stride_y)); |
| } |
| #endif // K0 > 8 |
| |
| // ---------------------------Store output values ------------------------------ |
| REPEAT_VAR_INIT_TO_CONST(16, uint, zout, 0); |
| STORE_BLOCK(K0, N0, DATA_TYPE, a, output_ptr, OUTPUT_STEP_X * sizeof(DATA_TYPE), zout); |
| |
| #undef BLOCK_SIZE |
| #undef OUTPUT_OFFSET_X |
| #undef OUTPUT_STEP_X |
| } |
| |
| #if defined(TRANSPOSE) |
| /** This OpenCL kernel reshapes the rhs input matrix. The kernel splits the input matrix in blocks of size K0xN0 and stores each one (transposed) in |
| * the output matrix unrolling the values. |
| * |
| * @note The data type must be passed at compile time using -DDATA_TYPE (e.g. -DDATA_TYPE=float) |
| * @note The height of the input tensor must be passed at compile time using -DSRC_HEIGHT (e.g. -DSRC_HEIGHT=16) |
| * @note The block's dimensions (K0 and N0) must be passed at compile time using -DK0 and -DN0 (e.g. -DK0=2, -DN0=2). |
| * @note The number of K0xN0 vertical blocks to store on the same output row must be passed at compile time using -DH0 (e.g. -DH0=2) |
| * @note If the K0xN0 blocks have to be interleaved, the option -DINTERLEAVE must passed at compile time. |
| * @note The option -DTRANSPOSE must passed at compile time. |
| * @note Only the following values for K0, N0 and H0 are supported: |
| * N0: 2,3,4,8,16 |
| * K0: 2,3,4,8,16 |
| * H0: greater than 0 |
| * |
| * @param[in] src_ptr Pointer to the source RHS tensor. Supported data types: All |
| * @param[in] src_stride_x Stride of the source RHS tensor in X dimension (in bytes) |
| * @param[in] src_step_x src_stride_x * number of elements along X processed per workitem(in bytes) |
| * @param[in] src_stride_y Stride of the source RHS tensor in Y dimension (in bytes) |
| * @param[in] src_step_y src_stride_y * number of elements along Y processed per workitem(in bytes) |
| * @param[in] src_stride_z Stride of the source RHS tensor in Z dimension (in bytes) |
| * @param[in] src_step_z src_stride_z * number of elements along Z processed per workitem(in bytes) |
| * @param[in] src_offset_first_element_in_bytes The offset of the first element in the source RHS tensor |
| * @param[out] dst_ptr Pointer to the destination matrix Supported data types: same as @p src_ptr |
| * @param[in] dst_stride_x Stride of the destination matrix in X dimension (in bytes) |
| * @param[in] dst_step_x dst_stride_x * number of elements along X processed per workitem(in bytes) |
| * @param[in] dst_stride_y Stride of the destination matrix in Y dimension (in bytes) |
| * @param[in] dst_step_y dst_stride_y * number of elements along Y processed per workitem(in bytes) |
| * @param[in] dst_stride_z Stride of the destination tensor in Z dimension (in bytes) |
| * @param[in] dst_step_z dst_stride_z * number of elements along Z processed per workitem(in bytes) |
| * @param[in] dst_offset_first_element_in_bytes The offset of the first element in the destination matrix |
| */ |
| __kernel void gemm_reshape_rhs_matrix_t(TENSOR3D_DECLARATION(src), |
| TENSOR3D_DECLARATION(dst)) |
| { |
| // Block size |
| #define BLOCK_SIZE ((K0) * (N0)) |
| |
| // Output offset X |
| #if defined(INTERLEAVE) |
| #define OUTPUT_OFFSET_X (K0) |
| #else // defined(INTERLEAVE) |
| #define OUTPUT_OFFSET_X (BLOCK_SIZE) |
| #endif // defined(INTERLEAVE) |
| |
| // Output step X |
| #if defined(INTERLEAVE) |
| #define OUTPUT_STEP_X (K0) * (H0) |
| #else // Do not interleave |
| #define OUTPUT_STEP_X (K0) |
| #endif // defined(INTERLEAVE) |
| |
| // Compute source and destination addresses |
| uint x = get_global_id(0); |
| uint y = get_global_id(1); |
| uint z = get_global_id(2); |
| |
| // ------------------ Compute input/output addresses --------------------------- |
| |
| // Compute the input address |
| __global uchar *input_ptr = src_ptr + src_offset_first_element_in_bytes + x * (uint)N0 * sizeof(DATA_TYPE) + y * (uint)K0 * src_stride_y + z * (uint)src_stride_z; |
| |
| // Compute the output address |
| __global uchar *output_ptr = dst_ptr + dst_offset_first_element_in_bytes + (y * (uint)BLOCK_SIZE * (uint)H0 * sizeof(DATA_TYPE)) + ((x % H0) * (uint)OUTPUT_OFFSET_X * sizeof(DATA_TYPE)) + ((x / |
| (uint)H0) * (uint)dst_stride_y) + z * (uint)dst_stride_z; |
| |
| // ---------------------------Load input values -------------------------------- |
| REPEAT_VAR_INIT_TO_CONST(K0, VEC_DATA_TYPE(DATA_TYPE, N0), a, 0); //VEC_DATA_TYPE(DATA_TYPE, N0) a0=0, a1=0, ... a(K0-1)=0; |
| |
| // Load values from the RHS matrix |
| a0 = VLOAD(N0)(0, (__global DATA_TYPE *)(input_ptr + 0 * src_stride_y)); |
| if(y * (uint)K0 + 1 < SRC_HEIGHT) |
| { |
| a1 = VLOAD(N0)(0, (__global DATA_TYPE *)(input_ptr + 1 * src_stride_y)); |
| } |
| #if K0 > 2 |
| if(y * (uint)K0 + 2 < SRC_HEIGHT) |
| { |
| a2 = VLOAD(N0)(0, (__global DATA_TYPE *)(input_ptr + 2 * src_stride_y)); |
| } |
| #endif // K0 > 2 |
| #if K0 > 3 |
| if(y * (uint)K0 + 3 < SRC_HEIGHT) |
| { |
| a3 = VLOAD(N0)(0, (__global DATA_TYPE *)(input_ptr + 3 * src_stride_y)); |
| } |
| #endif // K0 > 3 |
| #if K0 > 4 |
| if(y * (uint)K0 + 4 < SRC_HEIGHT) |
| { |
| a4 = VLOAD(N0)(0, (__global DATA_TYPE *)(input_ptr + 4 * src_stride_y)); |
| } |
| if(y * (uint)K0 + 5 < SRC_HEIGHT) |
| { |
| a5 = VLOAD(N0)(0, (__global DATA_TYPE *)(input_ptr + 5 * src_stride_y)); |
| } |
| if(y * (uint)K0 + 6 < SRC_HEIGHT) |
| { |
| a6 = VLOAD(N0)(0, (__global DATA_TYPE *)(input_ptr + 6 * src_stride_y)); |
| } |
| if(y * (uint)K0 + 7 < SRC_HEIGHT) |
| { |
| a7 = VLOAD(N0)(0, (__global DATA_TYPE *)(input_ptr + 7 * src_stride_y)); |
| } |
| #endif // K0 > 4 |
| #if K0 > 8 |
| if(y * (uint)K0 + 8 < SRC_HEIGHT) |
| { |
| a8 = VLOAD(N0)(0, (__global DATA_TYPE *)(input_ptr + 8 * src_stride_y)); |
| } |
| if(y * (uint)K0 + 9 < SRC_HEIGHT) |
| { |
| a9 = VLOAD(N0)(0, (__global DATA_TYPE *)(input_ptr + 9 * src_stride_y)); |
| } |
| if(y * (uint)K0 + 10 < SRC_HEIGHT) |
| { |
| aA = VLOAD(N0)(0, (__global DATA_TYPE *)(input_ptr + 10 * src_stride_y)); |
| } |
| if(y * (uint)K0 + 11 < SRC_HEIGHT) |
| { |
| aB = VLOAD(N0)(0, (__global DATA_TYPE *)(input_ptr + 11 * src_stride_y)); |
| } |
| if(y * (uint)K0 + 12 < SRC_HEIGHT) |
| { |
| aC = VLOAD(N0)(0, (__global DATA_TYPE *)(input_ptr + 12 * src_stride_y)); |
| } |
| if(y * (uint)K0 + 13 < SRC_HEIGHT) |
| { |
| aD = VLOAD(N0)(0, (__global DATA_TYPE *)(input_ptr + 13 * src_stride_y)); |
| } |
| if(y * (uint)K0 + 14 < SRC_HEIGHT) |
| { |
| aE = VLOAD(N0)(0, (__global DATA_TYPE *)(input_ptr + 14 * src_stride_y)); |
| } |
| if(y * (uint)K0 + 15 < SRC_HEIGHT) |
| { |
| aF = VLOAD(N0)(0, (__global DATA_TYPE *)(input_ptr + 15 * src_stride_y)); |
| } |
| #endif // K0 > 8 |
| |
| // ---------------------------Transpose the block ------------------------------ |
| REPEAT_VAR_INIT_TO_CONST(N0, VEC_DATA_TYPE(DATA_TYPE, K0), res, 0); //VEC_DATA_TYPE(DATA_TYPE, K0) res0=0, res1=0, res2=0,... res(N0-1)=0; |
| |
| #if K0 == 2 |
| // This part computes the following transpositions: |
| // 2x2 -> 2x2 |
| // 2x4 -> 4x2 |
| // 2x8 -> 8x2 |
| // 2x16 -> 16x2 |
| res0 = (VEC_DATA_TYPE(DATA_TYPE, K0))(a0.s0, a1.s0); |
| res1 = (VEC_DATA_TYPE(DATA_TYPE, K0))(a0.s1, a1.s1); |
| #if N0 > 2 |
| res2 = (VEC_DATA_TYPE(DATA_TYPE, K0))(a0.s2, a1.s2); |
| #endif // N0 > 2 |
| #if N0 > 3 |
| res3 = (VEC_DATA_TYPE(DATA_TYPE, K0))(a0.s3, a1.s3); |
| #endif // N0 > 3 |
| #if N0 > 4 |
| res4 = (VEC_DATA_TYPE(DATA_TYPE, K0))(a0.s4, a1.s4); |
| res5 = (VEC_DATA_TYPE(DATA_TYPE, K0))(a0.s5, a1.s5); |
| res6 = (VEC_DATA_TYPE(DATA_TYPE, K0))(a0.s6, a1.s6); |
| res7 = (VEC_DATA_TYPE(DATA_TYPE, K0))(a0.s7, a1.s7); |
| #endif // N0 > 4 |
| #if N0 > 8 |
| res8 = (VEC_DATA_TYPE(DATA_TYPE, K0))(a0.s8, a1.s8); |
| res9 = (VEC_DATA_TYPE(DATA_TYPE, K0))(a0.s9, a1.s9); |
| resA = (VEC_DATA_TYPE(DATA_TYPE, K0))(a0.sA, a1.sA); |
| resB = (VEC_DATA_TYPE(DATA_TYPE, K0))(a0.sB, a1.sB); |
| resC = (VEC_DATA_TYPE(DATA_TYPE, K0))(a0.sC, a1.sC); |
| resD = (VEC_DATA_TYPE(DATA_TYPE, K0))(a0.sD, a1.sD); |
| resE = (VEC_DATA_TYPE(DATA_TYPE, K0))(a0.sE, a1.sE); |
| resF = (VEC_DATA_TYPE(DATA_TYPE, K0))(a0.sF, a1.sF); |
| #endif // N0 > 8 |
| |
| #elif K0 == 3 // K0 == 2 |
| // This part computes the following transpositions: |
| // 3x2 -> 2x3 |
| // 3x4 -> 4x3 |
| // 3x8 -> 8x3 |
| // 3x16 -> 16x3 |
| res0 = (VEC_DATA_TYPE(DATA_TYPE, K0))(a0.s0, a1.s0, a2.s0); |
| res1 = (VEC_DATA_TYPE(DATA_TYPE, K0))(a0.s1, a1.s1, a2.s1); |
| #if N0 > 2 |
| res2 = (VEC_DATA_TYPE(DATA_TYPE, K0))(a0.s2, a1.s2, a2.s2); |
| #endif // N0 > 2 |
| #if N0 > 3 |
| res3 = (VEC_DATA_TYPE(DATA_TYPE, K0))(a0.s3, a1.s3, a2.s3); |
| #endif // N0 > 3 |
| #if N0 > 4 |
| res4 = (VEC_DATA_TYPE(DATA_TYPE, K0))(a0.s4, a1.s4, a2.s4); |
| res5 = (VEC_DATA_TYPE(DATA_TYPE, K0))(a0.s5, a1.s5, a2.s5); |
| res6 = (VEC_DATA_TYPE(DATA_TYPE, K0))(a0.s6, a1.s6, a2.s6); |
| res7 = (VEC_DATA_TYPE(DATA_TYPE, K0))(a0.s7, a1.s7, a2.s7); |
| #endif // N0 > 4 |
| #if N0 > 8 |
| res8 = (VEC_DATA_TYPE(DATA_TYPE, K0))(a0.s8, a1.s8, a2.s8); |
| res9 = (VEC_DATA_TYPE(DATA_TYPE, K0))(a0.s9, a1.s9, a2.s9); |
| resA = (VEC_DATA_TYPE(DATA_TYPE, K0))(a0.sA, a1.sA, a2.sA); |
| resB = (VEC_DATA_TYPE(DATA_TYPE, K0))(a0.sB, a1.sB, a2.sB); |
| resC = (VEC_DATA_TYPE(DATA_TYPE, K0))(a0.sC, a1.sC, a2.sC); |
| resD = (VEC_DATA_TYPE(DATA_TYPE, K0))(a0.sD, a1.sD, a2.sD); |
| resE = (VEC_DATA_TYPE(DATA_TYPE, K0))(a0.sE, a1.sE, a2.sE); |
| resF = (VEC_DATA_TYPE(DATA_TYPE, K0))(a0.sF, a1.sF, a2.sF); |
| #endif // N0 > 8 |
| |
| #elif K0 == 4 // K0 == 4 |
| // This part computes the following transpositions: |
| // 4x2 -> 2x4 |
| // 4x4 -> 4x4 |
| // 4x8 -> 8x4 |
| // 4x16 -> 16x4 |
| res0 = (VEC_DATA_TYPE(DATA_TYPE, K0))(a0.s0, a1.s0, a2.s0, a3.s0); |
| res1 = (VEC_DATA_TYPE(DATA_TYPE, K0))(a0.s1, a1.s1, a2.s1, a3.s1); |
| #if N0 > 2 |
| res2 = (VEC_DATA_TYPE(DATA_TYPE, K0))(a0.s2, a1.s2, a2.s2, a3.s2); |
| #endif // N0 > 2 |
| #if N0 > 3 |
| res3 = (VEC_DATA_TYPE(DATA_TYPE, K0))(a0.s3, a1.s3, a2.s3, a3.s3); |
| #endif // N0 > 3 |
| #if N0 > 4 |
| res4 = (VEC_DATA_TYPE(DATA_TYPE, K0))(a0.s4, a1.s4, a2.s4, a3.s4); |
| res5 = (VEC_DATA_TYPE(DATA_TYPE, K0))(a0.s5, a1.s5, a2.s5, a3.s5); |
| res6 = (VEC_DATA_TYPE(DATA_TYPE, K0))(a0.s6, a1.s6, a2.s6, a3.s6); |
| res7 = (VEC_DATA_TYPE(DATA_TYPE, K0))(a0.s7, a1.s7, a2.s7, a3.s7); |
| #endif // N0 > 4 |
| #if N0 > 8 |
| res8 = (VEC_DATA_TYPE(DATA_TYPE, K0))(a0.s8, a1.s8, a2.s8, a3.s8); |
| res9 = (VEC_DATA_TYPE(DATA_TYPE, K0))(a0.s9, a1.s9, a2.s9, a3.s9); |
| resA = (VEC_DATA_TYPE(DATA_TYPE, K0))(a0.sA, a1.sA, a2.sA, a3.sA); |
| resB = (VEC_DATA_TYPE(DATA_TYPE, K0))(a0.sB, a1.sB, a2.sB, a3.sB); |
| resC = (VEC_DATA_TYPE(DATA_TYPE, K0))(a0.sC, a1.sC, a2.sC, a3.sC); |
| resD = (VEC_DATA_TYPE(DATA_TYPE, K0))(a0.sD, a1.sD, a2.sD, a3.sD); |
| resE = (VEC_DATA_TYPE(DATA_TYPE, K0))(a0.sE, a1.sE, a2.sE, a3.sE); |
| resF = (VEC_DATA_TYPE(DATA_TYPE, K0))(a0.sF, a1.sF, a2.sF, a3.sF); |
| #endif // N0 > 8 |
| |
| #elif K0 == 8 // K0 == 8 |
| // This part computes the following transpositions: |
| // 8x2 -> 2x8 |
| // 8x4 -> 4x8 |
| // 8x8 -> 8x8 |
| // 8x16 -> 16x8 |
| res0 = (VEC_DATA_TYPE(DATA_TYPE, K0))(a0.s0, a1.s0, a2.s0, a3.s0, a4.s0, a5.s0, a6.s0, a7.s0); |
| res1 = (VEC_DATA_TYPE(DATA_TYPE, K0))(a0.s1, a1.s1, a2.s1, a3.s1, a4.s1, a5.s1, a6.s1, a7.s1); |
| #if N0 > 2 |
| res2 = (VEC_DATA_TYPE(DATA_TYPE, K0))(a0.s2, a1.s2, a2.s2, a3.s2, a4.s2, a5.s2, a6.s2, a7.s2); |
| #endif // N0 > 2 |
| #if N0 > 3 |
| res3 = (VEC_DATA_TYPE(DATA_TYPE, K0))(a0.s3, a1.s3, a2.s3, a3.s3, a4.s3, a5.s3, a6.s3, a7.s3); |
| #endif // N0 > 3 |
| #if N0 > 4 |
| res4 = (VEC_DATA_TYPE(DATA_TYPE, K0))(a0.s4, a1.s4, a2.s4, a3.s4, a4.s4, a5.s4, a6.s4, a7.s4); |
| res5 = (VEC_DATA_TYPE(DATA_TYPE, K0))(a0.s5, a1.s5, a2.s5, a3.s5, a4.s5, a5.s5, a6.s5, a7.s5); |
| res6 = (VEC_DATA_TYPE(DATA_TYPE, K0))(a0.s6, a1.s6, a2.s6, a3.s6, a4.s6, a5.s6, a6.s6, a7.s6); |
| res7 = (VEC_DATA_TYPE(DATA_TYPE, K0))(a0.s7, a1.s7, a2.s7, a3.s7, a4.s7, a5.s7, a6.s7, a7.s7); |
| #endif // N0 > 4 |
| #if N0 > 8 |
| res8 = (VEC_DATA_TYPE(DATA_TYPE, K0))(a0.s8, a1.s8, a2.s8, a3.s8, a4.s8, a5.s8, a6.s8, a7.s8); |
| res9 = (VEC_DATA_TYPE(DATA_TYPE, K0))(a0.s9, a1.s9, a2.s9, a3.s9, a4.s9, a5.s9, a6.s9, a7.s9); |
| resA = (VEC_DATA_TYPE(DATA_TYPE, K0))(a0.sA, a1.sA, a2.sA, a3.sA, a4.sA, a5.sA, a6.sA, a7.sA); |
| resB = (VEC_DATA_TYPE(DATA_TYPE, K0))(a0.sB, a1.sB, a2.sB, a3.sB, a4.sB, a5.sB, a6.sB, a7.sB); |
| resC = (VEC_DATA_TYPE(DATA_TYPE, K0))(a0.sC, a1.sC, a2.sC, a3.sC, a4.sC, a5.sC, a6.sC, a7.sC); |
| resD = (VEC_DATA_TYPE(DATA_TYPE, K0))(a0.sD, a1.sD, a2.sD, a3.sD, a4.sD, a5.sD, a6.sD, a7.sD); |
| resE = (VEC_DATA_TYPE(DATA_TYPE, K0))(a0.sE, a1.sE, a2.sE, a3.sE, a4.sE, a5.sE, a6.sE, a7.sE); |
| resF = (VEC_DATA_TYPE(DATA_TYPE, K0))(a0.sF, a1.sF, a2.sF, a3.sF, a4.sF, a5.sF, a6.sF, a7.sF); |
| #endif // N0 > 8 |
| |
| #elif K0 == 16 // K0 == 16 |
| |
| // This part computes the following transpositions: |
| // 16x2 -> 2x16 |
| // 16x4 -> 4x16 |
| // 16x8 -> 8x16 |
| // 16x16 -> 16x16 |
| res0 = (VEC_DATA_TYPE(DATA_TYPE, K0))(a0.s0, a1.s0, a2.s0, a3.s0, a4.s0, a5.s0, a6.s0, a7.s0, |
| a8.s0, a9.s0, aA.s0, aB.s0, aC.s0, aD.s0, aE.s0, aF.s0); |
| res1 = (VEC_DATA_TYPE(DATA_TYPE, K0))(a0.s1, a1.s1, a2.s1, a3.s1, a4.s1, a5.s1, a6.s1, a7.s1, |
| a8.s1, a9.s1, aA.s1, aB.s1, aC.s1, aD.s1, aE.s1, aF.s1); |
| #if N0 > 2 |
| res2 = (VEC_DATA_TYPE(DATA_TYPE, K0))(a0.s2, a1.s2, a2.s2, a3.s2, a4.s2, a5.s2, a6.s2, a7.s2, |
| a8.s2, a9.s2, aA.s2, aB.s2, aC.s2, aD.s2, aE.s2, aF.s2); |
| #endif // N0 > 2 |
| #if N0 > 3 |
| res3 = (VEC_DATA_TYPE(DATA_TYPE, K0))(a0.s3, a1.s3, a2.s3, a3.s3, a4.s3, a5.s3, a6.s3, a7.s3, |
| a8.s3, a9.s3, aA.s3, aB.s3, aC.s3, aD.s3, aE.s3, aF.s3); |
| #endif // N0 > 3 |
| #if N0 > 4 |
| res4 = (VEC_DATA_TYPE(DATA_TYPE, K0))(a0.s4, a1.s4, a2.s4, a3.s4, a4.s4, a5.s4, a6.s4, a7.s4, |
| a8.s4, a9.s4, aA.s4, aB.s4, aC.s4, aD.s4, aE.s4, aF.s4); |
| res5 = (VEC_DATA_TYPE(DATA_TYPE, K0))(a0.s5, a1.s5, a2.s5, a3.s5, a4.s5, a5.s5, a6.s5, a7.s5, |
| a8.s5, a9.s5, aA.s5, aB.s5, aC.s5, aD.s5, aE.s5, aF.s5); |
| res6 = (VEC_DATA_TYPE(DATA_TYPE, K0))(a0.s6, a1.s6, a2.s6, a3.s6, a4.s6, a5.s6, a6.s6, a7.s6, |
| a8.s6, a9.s6, aA.s6, aB.s6, aC.s6, aD.s6, aE.s6, aF.s6); |
| res7 = (VEC_DATA_TYPE(DATA_TYPE, K0))(a0.s7, a1.s7, a2.s7, a3.s7, a4.s7, a5.s7, a6.s7, a7.s7, |
| a8.s7, a9.s7, aA.s7, aB.s7, aC.s7, aD.s7, aE.s7, aF.s7); |
| #endif // N0 > 4 |
| #if N0 > 8 |
| res8 = (VEC_DATA_TYPE(DATA_TYPE, K0))(a0.s8, a1.s8, a2.s8, a3.s8, a4.s8, a5.s8, a6.s8, a7.s8, |
| a8.s8, a9.s8, aA.s8, aB.s8, aC.s8, aD.s8, aE.s8, aF.s8); |
| res9 = (VEC_DATA_TYPE(DATA_TYPE, K0))(a0.s9, a1.s9, a2.s9, a3.s9, a4.s9, a5.s9, a6.s9, a7.s9, |
| a8.s9, a9.s9, aA.s9, aB.s9, aC.s9, aD.s9, aE.s9, aF.s9); |
| resA = (VEC_DATA_TYPE(DATA_TYPE, K0))(a0.sA, a1.sA, a2.sA, a3.sA, a4.sA, a5.sA, a6.sA, a7.sA, |
| a8.sA, a9.sA, aA.sA, aB.sA, aC.sA, aD.sA, aE.sA, aF.sA); |
| resB = (VEC_DATA_TYPE(DATA_TYPE, K0))(a0.sB, a1.sB, a2.sB, a3.sB, a4.sB, a5.sB, a6.sB, a7.sB, |
| a8.sB, a9.sB, aA.sB, aB.sB, aC.sB, aD.sB, aE.sB, aF.sB); |
| resC = (VEC_DATA_TYPE(DATA_TYPE, K0))(a0.sC, a1.sC, a2.sC, a3.sC, a4.sC, a5.sC, a6.sC, a7.sC, |
| a8.sC, a9.sC, aA.sC, aB.sC, aC.sC, aD.sC, aE.sC, aF.sC); |
| resD = (VEC_DATA_TYPE(DATA_TYPE, K0))(a0.sD, a1.sD, a2.sD, a3.sD, a4.sD, a5.sD, a6.sD, a7.sD, |
| a8.sD, a9.sD, aA.sD, aB.sD, aC.sD, aD.sD, aE.sD, aF.sD); |
| resE = (VEC_DATA_TYPE(DATA_TYPE, K0))(a0.sE, a1.sE, a2.sE, a3.sE, a4.sE, a5.sE, a6.sE, a7.sE, |
| a8.sE, a9.sE, aA.sE, aB.sE, aC.sE, aD.sE, aE.sE, aF.sE); |
| resF = (VEC_DATA_TYPE(DATA_TYPE, K0))(a0.sF, a1.sF, a2.sF, a3.sF, a4.sF, a5.sF, a6.sF, a7.sF, |
| a8.sF, a9.sF, aA.sF, aB.sF, aC.sF, aD.sF, aE.sF, aF.sF); |
| #endif // N0 > 8 |
| |
| #else // N0 == 16 |
| #error "Not supported N0 value" |
| #endif // N0 > 2 |
| |
| // ---------------------------Store the output values ------------------------------ |
| REPEAT_VAR_INIT_TO_CONST(16, uint, zout, 0); |
| STORE_BLOCK(N0, K0, DATA_TYPE, res, output_ptr, OUTPUT_STEP_X * sizeof(DATA_TYPE), zout); |
| |
| #undef BLOCK_SIZE |
| #undef OUTPUT_OFFSET_X |
| #undef OUTPUT_STEP_X |
| } |
| #endif // defined(TRANSPOSE) |
| #endif // defined(K0) && defined(N0) && defined(H0) && defined(DATA_TYPE) && defined(SRC_HEIGHT) |