| /* |
| * Copyright (c) 2023 Arm Limited. |
| * |
| * SPDX-License-Identifier: MIT |
| * |
| * Permission is hereby granted, free of charge, to any person obtaining a copy |
| * of this software and associated documentation files (the "Software"), to |
| * deal in the Software without restriction, including without limitation the |
| * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or |
| * sell copies of the Software, and to permit persons to whom the Software is |
| * furnished to do so, subject to the following conditions: |
| * |
| * The above copyright notice and this permission notice shall be included in all |
| * copies or substantial portions of the Software. |
| * |
| * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
| * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
| * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE |
| * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
| * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
| * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
| * SOFTWARE. |
| */ |
| #include "activation_float_helpers.h" |
| #include "helpers.h" |
| #include "tile_helpers.h" |
| |
| #ifdef BIAS |
| // This function performs in-place bias addition for integer datatype when bias is enabled. |
| // Note The tile's dimensions used for the LHS and RHS matrices (M0, N0) must be passed at compile time using -DN0, -DM0 (e.g. -DN0=8, -DM0=4). |
| inline void perform_bias_addition(uchar *bias_ptr, uint bias_offset_first_element_in_bytes, TILE(int, M0, N0, acc), uint x) |
| { |
| TILE(int, 1, N0, bias_tile); |
| |
| // below expands to use bias_ptr and bias_offset_first_element_in_bytes |
| T_LOAD(int, 1, N0, BUFFER, bias, x, 0, 1, 0, bias_tile); |
| |
| // c = c + bias[broadcasted] |
| T_ELTWISE_BROADCAST_ADD_X(int, M0, N0, acc, bias_tile, acc); |
| } |
| #endif // defined(BIAS) |
| |
| #define MMUL_BLOCK_SIZE (MMUL_M0 * MMUL_N0) // MMUL block size for the output matrix |
| |
| #if defined(MAT_MUL_NATIVE_QUANTIZED_MMUL_NT_NT) |
| /** This OpenCL kernel performs the batch matrix multiplication (BatchMatMul): LHS non-transposed, RHS non-transposed - buffer only |
| * |
| * @note the "batch" here expresses the number of matrix multiplications to run in parallel. However, it |
| * should NOT be confused with the batch size of the model. For NHWC the "batch" is the "H" dimension |
| * @note The data type must be passed at compile time using -DDATA_TYPE (e.g. -DDATA_TYPE=uchar) |
| * @note The block's dimensions used for the LHS and RHS matrices (M0, N0 and K0) must be passed at |
| * compile time using -DN0, -DM0 and -DK0 (e.g. -DN0=8, -DM0=4, -DK0=4). |
| * @note The number of leftover outputs rows/columns must be passed using -DN0_LEFTOVER and -DM0_LEFTOVER |
| * (e.g. -DN0_LEFTOVER=2, -DM0_LEFTOVER=3) |
| * @note The dimensions M, N, K must be passed at compile time using -DK (e.g. -DM=5, -DN=8, -DK=6). |
| * K must be a multiple of 16. |
| * @note MMUL block sizes must be passed at compile time using -DMMUL_K0, -DMMUL_M0, -DMMUL_N0 |
| * (e.g. -DMMUL_K0=16, -DMMUL_M0=4, -DMMUL_N0=4) |
| * @note If there is bias -DBIAS option must be passed at compile time |
| * @note Quantization offsets of lhs, rhs and dst tensors must be passed at compile time using -DLHS_OFFSET, |
| * -DRHS_OFFSET, -DDST_OFFSET (e.g. -DLHS_OFFSET=10, -DRHS_OFFSET=0, -DDST_OFFSET=-6) |
| * @note Effective quantization multiplier and shift for the destination tensor must be passed at compile time using |
| * -DDST_MULTIPLIER and -DDST_SHIFT (e.g. -DDST_MULTIPLIER=2091, -DST_SHIFT=8) |
| * @note The kernel name in uppercase must be passed at compile time (e.g. -DMAT_MUL_NATIVE_QUANTIZED_MMUL_NT_NT) |
| * @note Only the following configurations of M0, N0 and K0 are currently supported: |
| * - M0 > 0 |
| * - N0 = 1, 2, 3, 4, 8, 16 |
| * - K0 = 4 |
| * @note For a generic view on how the MMUL works, see mat_mul_mmul.cl |
| * |
| * @param[in] lhs_ptr Pointer to the lhs matrix. Supported data types: QASYMM8_SIGNED/QASYMM8 |
| * @param[in] lhs_stride_y Stride of the lhs matrix in Y (2nd) dimension (in bytes) |
| * @param[in] lhs_stride_z Stride of the lhs tensor in Z (3rd) dimension (in bytes) |
| * @param[in] lhs_w The width of the lhs tensor |
| * @param[in] lhs_h The height of the lhs tensor |
| * @param[in] lhs_n Number of the matrices (buffers) in the batch |
| * @param[in] lhs_offset_first_element_in_bytes The offset of the first element in the lhs matrix |
| * @param[in] rhs_ptr Pointer to the rhs matrix. Supported data types: same as @p lhs_ptr |
| * @param[in] rhs_stride_y Stride of the rhs matrix in Y (2nd) dimension (in bytes) |
| * @param[in] rhs_stride_z Stride of the rhs tensor in Z (3rd) dimension (in bytes) |
| * @param[in] rhs_w The width of the rhs tensor |
| * @param[in] rhs_h The height of the rhs tensor |
| * @param[in] rhs_n Number of the matrices (buffers) in the batch |
| * @param[in] rhs_offset_first_element_in_bytes The offset of the first element in the rhs matrix |
| * @param[in] bias_ptr (Optional) Pointer to the bias tensor. Supported data type: S32 |
| * @param[in] bias_stride_y (Optional) Stride of the bias tensor in Y dimension (in bytes) |
| * @param[in] bias_stride_z (Optional) Stride of the bias tensor in Z dimension (in bytes) |
| * @param[in] bias_w (Optional) The size of the width dimension of the bias tensor |
| * @param[in] bias_h (Optional) The size of the height dimension of the bias tensor |
| * @param[in] bias_n (Optional) The size of the depth dimension of the bias tensor |
| * @param[in] bias_offset_first_element_in_bytes (Optional) The offset of the first element in the bias tensor |
| * @param[out] dst_ptr Pointer to the dst matrix. Supported data types: same as @p lhs_ptr |
| * @param[in] dst_stride_y Stride of the dst matrix in Y (2nd) dimension (in bytes) |
| * @param[in] dst_stride_z Stride of the dst tensor in Z (3rd) dimension (in bytes) |
| * @param[in] dst_w The width of the dst tensor |
| * @param[in] dst_h The height of the dst tensor |
| * @param[in] dst_n Number of the matrices (buffers) in the batch |
| * @param[in] dst_offset_first_element_in_bytes The offset of the first element in the dst matrix |
| */ |
| __kernel void mat_mul_native_quantized_mmul_nt_nt( |
| TENSOR3D_T(lhs, BUFFER), |
| TENSOR3D_T(rhs, BUFFER), |
| #ifdef BIAS |
| TENSOR3D_T(bias, BUFFER), |
| #endif // defined(BIAS) |
| TENSOR3D_T(dst, BUFFER)) |
| { |
| // The explanation of how this kernel works is very similar to the explanation given in |
| // mat_mul_mmul.cl. The MMUL logic, and terminology is the same. The only difference is |
| // in quantization multiplication, the MMUL block sizes are (4 x 16) for Lhs matrix and |
| // (16 x 4) for Rhs matrix, resulting in (4 x 4) MMUL block size for the destination. |
| // |
| // Figures 1, 2 and 3 in the previous explanation works the same. Since the Lhs and Rhs |
| // MMUL block sizes are different in quantized extension, the thread access pattern is |
| // slightly different. We can redraw Figure 4 (Thread access pattern) as follows: |
| // |
| // (Modified Figure 4 from mat_mul_mmul.cl) |
| // Thread Access Layouts in LHS & RHS matrices |
| // |
| // LHS matrix |
| // 4 times 4 times 4 times 4 times |
| // _______________________________________________________________ |
| // |T0_|T0_|T0_|T0_|T1_|T1_|T1_|T1_|T2_|T2_|T2_|T2_|T3_|T3_|T3_|T3_| |
| // |T0_| ... | |
| // M0 | . . | |
| // Times | . . | |
| // | . . | |
| // |T0_|T0_|T0_|T0_|T1_|T1_|T1_|T1_|T2_|T2_|T2_|T2_|T3_|T3_|T3_|T3_| |
| // |T4_|T4_|T4_|T4_|T5_|T5_|T5_|T5_|T6_|T6_|T6_|T6_|T7_|T7_|T7_|T7_| |
| // |T4_|T4_|T4_|T4_|T5_|T5_|T5_|T5_|T6_|T6_|T6_|T6_|T7_|T7_|T7_|T7_| |
| // M0 | . . | |
| // Times | . . | |
| // | . . | |
| // |T4_|T4_|T4_|T4_|T5_|T5_|T5_|T5_|T6_|T6_|T6_|T6_|T7_|T7_|T7_|T7_| |
| // |T8_|T8_|T8_|T8_|T9_|T9_|T9_|T9_|T10|T10|T10|T10|T11|T11|T11|T11| |
| // M0 | . | |
| // Times | . | |
| // | . | |
| // |T8_|T8_|T8_|T8_|T9_|T9_|T9_|T9_|T10|T10|T10|T10|T11|T11|T11|T11| |
| // M0 | . | |
| // Times | . | |
| // | . | |
| // |T12|T12|T12|T12|T13|T13|T13|T13|T14|T14|T14|T14|T15|T15|T15|T15| |
| // |
| // |
| // RHS Matrix |
| // |
| // __________N0 times______N0 times____________________N0 times_______ |
| // |__T0__| ... |__T0__|__T1__| ... |__T1__| ... |__T3__| ... |__T3__| |
| // 4 times |__T0__| ... |__T0__|__T1__| ... |__T1__| ... |__T3__| ... |__T3__| |
| // |__T0__| ... |__T0__|__T1__| ... |__T1__| ... |__T3__| ... |__T3__| |
| // |__T0__| ... |__T0__|__T1__| ... |__T1__| ... |__T3__| ... |__T3__| |
| // |__T4__| ... |__T4__|__T5__| ... |__T5__| ... |__T7__| ... |__T7__| |
| // 4 times |__T4__| ... |__T4__|__T5__| ... |__T5__| ... |__T7__| ... |__T7__| |
| // |__T4__| ... |__T4__|__T5__| ... |__T5__| ... |__T7__| ... |__T7__| |
| // X |__T4__| ... |__T4__|__T5__| ... |__T5__| ... |__T7__| ... |__T7__| |
| // |__T8__| ... |__T8__|__T9__| ... |__T9__| ... |__T11_| ... |__T11_| |
| // |__T8__| ... |__T8__|__T9__| ... |__T9__| ... |__T11_| ... |__T11_| |
| // 4 times |__T8__| ... |__T8__|__T9__| ... |__T9__| ... |__T11_| ... |__T11_| |
| // |__T8__| ... |__T8__|__T9__| ... |__T9__| ... |__T11_| ... |__T11_| |
| // |__T12_| ... |__T12_|__T13_| ... |__T13_| ... |__T15_| ... |__T15_| |
| // 4 times |__T12_| ... |__T12_|__T13_| ... |__T13_| ... |__T15_| ... |__T15_| |
| // |__T12_| ... |__T12_|__T13_| ... |__T13_| ... |__T15_| ... |__T15_| |
| // |__T12_|_____|__T12_|__T13_|______|__T13_|_____|__T15_|_____|__T15_| |
| // |
| // |
| // The logic behind this thread access pattern is already descried in the explanation |
| // in mat_mul_mmul.cl. The only change is threads accesses are extended to 4 elements |
| // from 1, in rightward direction in Lhs, and in downward direction in Rhs, because they |
| // are now operating on 4 char/uchar's (again 32-bit data), instead of one 32-bit floating point. |
| // |
| // The mathematical view of the matrix multiplication explained in Figure 5 also holds for this, |
| // except the dimension 4 is 16 instead, but the vector notations do not change, i.e. it's as follows: |
| // |
| // Settings: |
| // - a 8 x 16 LHS section |
| // - 16 x 8 RHS section |
| // - Each vector variable ai, bj represent a 16x1 vector |
| // - ^T (superscript T) denotes transpose |
| // - M0 = N0 = 2 |
| // - MMUL_N0 = MMUL_M0 = 4, MMUL_K0 = 16 |
| // |
| // |
| // (Modified Figure 5) |
| // Mathematical view of the Matrix Multiplication |
| // |
| // LHS RHS DST |
| // [ a1^T ] [ b1 b2 b3 b4 b5 b6 b7 ] [ a1^Tb1 a1^Tb2 a1^Tb3 ... a1^Tb7 ] |
| // [ a2^T ] 16 x 8 [ a2^Tb1 a2^Tb2 a2^Tb3 ... a2^Tb7 ] |
| // [ a3^T ] [ ] |
| // [ a4^T ] = [ . . ] |
| // [ a5^T ] X [ . . ] |
| // [ a6^T ] [ . . ] |
| // [ a7^T ] [ ] |
| // [ a8^T ] [ a7^Tb1 a7^Tb2 a7^Tb3 ... a7^Tb7 ] |
| // 8 x 16 8 x 8 |
| // |
| // |
| // For the first iteration, i.e. (m0, n0) = (0, 0), the arm_matrix_multiply would multiply the following matrices: |
| // |
| // [ a1^T ] [ b1 b3 b5 b7 ] [ a1^Tb1 a1^Tb3 a1^Tb5 a1^Tb7 ] |
| // [ a3^T ] x 4 x 4 = [ a3^Tb1 a1^Tb3 a1^Tb5 a1^Tb7 ] |
| // [ a5^T ] [ a5^Tb1 a1^Tb3 a1^Tb5 a1^Tb7 ] |
| // [ a7^T ] [ a7^Tb1 a7^Tb3 a7^Tb5 a7^Tb7 ] |
| // 4 x 4 4 x 4 |
| // The elements calculated in the 4x4 output block are the "interleaved" elements in the DST above. |
| // When we follow for each combination of (m0, n0), every element of the DST matrix "section" is filled. |
| // |
| // Please refer to mat_mul_mmul.cl for more details. |
| |
| const uint x0 = get_global_id(0); // [0, (N / N0) * MMUL_M0) |
| // The upper limit is a simplified version of (N / N0) / MMUL_N0) * MMUL_BLOCK_SIZE) |
| const uint y0 = get_global_id(1); // [0, (M / M0) / MMUL_M0) |
| const uint z = get_global_id(2); // Batch |
| |
| // Get section coordinates |
| const uint section_x = (x0 / MMUL_BLOCK_SIZE); |
| const uint section_y = y0; |
| |
| // Get thread coordinates within an mmul block |
| const uint thread_id = (x0 % MMUL_BLOCK_SIZE); |
| const uint thread_x = thread_id % MMUL_N0; |
| const uint thread_y = (thread_id / MMUL_N0); |
| |
| // Calculate dst coordinates |
| const uint dst_x_unclamped = thread_x * N0 + section_x * N0 * MMUL_N0; |
| const uint dst_y_unclamped = thread_y * M0 + section_y * M0 * MMUL_M0; |
| const uint dst_x = min(dst_x_unclamped, (uint)(N - N0)); |
| const uint dst_y = min(dst_y_unclamped, (uint)(M - M0)); |
| |
| // Starting LHS coordinates |
| const uint lhs_x = K0 * thread_x; |
| const uint lhs_y = dst_y; |
| |
| // Starting RHS coordinates |
| const uint rhs_x = dst_x; |
| const uint rhs_y = K0 * thread_y; |
| |
| // Compute LHS/RHS/DST matrix address |
| lhs_offset_first_element_in_bytes += lhs_x * sizeof(DATA_TYPE) + lhs_y * lhs_stride_y + z * lhs_stride_z; |
| rhs_offset_first_element_in_bytes += rhs_x * sizeof(DATA_TYPE) + rhs_y * rhs_stride_y + z * rhs_stride_z; |
| dst_offset_first_element_in_bytes += dst_x * sizeof(DATA_TYPE) + dst_y * dst_stride_y + z * dst_stride_z; |
| |
| // Initialize the accumulators |
| TILE(int, M0, N0, c); |
| LOOP_UNROLLING(int, i, 0, 1, M0, |
| { |
| c[i].v = K * ((int)LHS_OFFSET) * ((int)RHS_OFFSET); |
| }) |
| |
| // Calculate row and column sums |
| TILE(int, 1, N0, b_sum); |
| b_sum[0].v = 0; |
| |
| TILE(int, 1, M0, a_sum); |
| a_sum[0].v = 0; |
| |
| VEC_DATA_TYPE(DATA_TYPE, K0) |
| vec_1 = (VEC_DATA_TYPE(DATA_TYPE, K0))(1, 1, 1, 1); |
| |
| for(int k = 0; k < lhs_w; k += MMUL_K0) |
| { |
| // A tile of M0xK0 but K0 must be set to K0 |
| TILE(DATA_TYPE, M0, K0, a); |
| // A tile of K0xN0 but K0 must be set to K0 |
| TILE(DATA_TYPE, K0, N0, b); |
| |
| // Load tile from the lhs/rhs tensors |
| T_LOAD(DATA_TYPE, M0, K0, BUFFER, lhs, 0, 0, 1, lhs_stride_y, a); |
| T_LOAD(DATA_TYPE, K0, N0, BUFFER, rhs, 0, 0, 1, rhs_stride_y, b); |
| |
| LOOP_UNROLLING(int, n0, 0, 1, N0, |
| { |
| VEC_DATA_TYPE(DATA_TYPE, K0) |
| vec_b = (VEC_DATA_TYPE(DATA_TYPE, K0))(b[0].s[n0], b[1].s[n0], b[2].s[n0], b[3].s[n0]); |
| |
| LOOP_UNROLLING(int, m0, 0, 1, M0, |
| { |
| c[m0].s[n0] = arm_matrix_multiply(a[m0].v, vec_b, c[m0].s[n0]); |
| }) |
| |
| #if LHS_OFFSET != 0 |
| // Column Sum of B: Calculate the sum of columns by multiplying B |
| // with a matrix of 1's from Left |
| b_sum[0].s[n0] = arm_matrix_multiply(vec_1, vec_b, b_sum[0].s[n0]); |
| #endif // LHS_OFFSET != 0s |
| }) |
| |
| #if RHS_OFFSET != 0 |
| // Row Sum of A: Calculate the sum of rows by multiplying A with |
| // a matrix of 1's from Right |
| LOOP_UNROLLING(int, m0, 0, 1, M0, |
| { |
| a_sum[0].s[m0] = arm_matrix_multiply(a[m0].v, vec_1, a_sum[0].s[m0]); |
| }) |
| #endif // RHS_OFFSET != 0 |
| |
| lhs_offset_first_element_in_bytes += MMUL_K0 * sizeof(DATA_TYPE); |
| rhs_offset_first_element_in_bytes += MMUL_K0 * rhs_stride_y; |
| } |
| |
| // Do not write if the coordinates are out of bound |
| // But, read has to happen as arm_matrix_multiply() expects certain number of calls |
| if(dst_x_unclamped >= N || dst_y_unclamped >= M) |
| { |
| return; |
| } |
| |
| #if RHS_OFFSET != 0 || LHS_OFFSET != 0 |
| LOOP_UNROLLING(int, i, 0, 1, M0, |
| { |
| const int A = ((int)RHS_OFFSET) * a_sum[0].s[i]; |
| LOOP_UNROLLING(int, j, 0, 1, N0, |
| { |
| c[i].s[j] -= A + ((int)(LHS_OFFSET)) * b_sum[0].s[j]; |
| }) |
| }) |
| #endif // RHS_OFFSET != 0 || LHS_OFFSET != 0 |
| |
| #ifdef BIAS |
| perform_bias_addition(bias_ptr, bias_offset_first_element_in_bytes, c, dst_x); |
| #endif // defined(BIAS) |
| |
| // Quantize the tile |
| TILE(DATA_TYPE, M0, N0, cq); |
| T_QUANTIZE8_ASYMMETRIC(int, DATA_TYPE, M0, N0, DST_OFFSET, DST_SHIFT, DST_MULTIPLIER, c, cq); |
| |
| if(dst_x + N0 <= N || N0_LEFTOVER == 0) |
| { |
| LOOP_UNROLLING(int, m0, 0, 1, M0, |
| { |
| if(dst_y + m0 < M || M0_LEFTOVER == 0) |
| { |
| VSTORE(N0) |
| (cq[m0].v, 0, (__global DATA_TYPE *)(dst_ptr + dst_offset_first_element_in_bytes + m0 * dst_stride_y)); |
| } |
| }) |
| } |
| else |
| { |
| LOOP_UNROLLING(int, m0, 0, 1, M0, |
| { |
| if(dst_y + m0 < M || M0_LEFTOVER == 0) |
| { |
| VSTORE_PARTIAL(N0, N0_LEFTOVER) |
| (cq[m0].v, 0, (__global DATA_TYPE *)(dst_ptr + dst_offset_first_element_in_bytes + m0 * dst_stride_y)); |
| } |
| }) |
| } |
| } |
| #endif // defined(MAT_MUL_NATIVE_QUANTIZED_MMUL_NT_NT) |
| |
| #if defined(MAT_MUL_NATIVE_QUANTIZED_MMUL_NT_T) |
| /** This OpenCL kernel performs the batch matrix multiplication (BatchMatMul): LHS non-transposed, RHS transposed - buffer only |
| * |
| * Supported block configurations: |
| * - M0 > 0 |
| * - N0 = 1, 2, 3, 4, 8, 16 |
| * - K0 = 4 |
| * |
| * Similar to mat_mul_native_quantized_mmul_nt_nt() |
| */ |
| __kernel void mat_mul_native_quantized_mmul_nt_t( |
| TENSOR3D_T(lhs, BUFFER), |
| TENSOR3D_T(rhs, BUFFER), |
| #ifdef BIAS |
| TENSOR3D_T(bias, BUFFER), |
| #endif // defined(BIAS) |
| TENSOR3D_T(dst, BUFFER)) |
| { |
| const uint x0 = get_global_id(0); // [0, (N / N0) * MMUL_M0) |
| // The upper limit is a simplified version of (N / N0) / MMUL_N0) * MMUL_BLOCK_SIZE) |
| const uint y0 = get_global_id(1); // [0, (M / M0) / MMUL_M0) |
| const uint z = get_global_id(2); // Batch |
| |
| // Get section coordinates |
| const uint section_x = (x0 / MMUL_BLOCK_SIZE); |
| const uint section_y = y0; |
| |
| // Get thread coordinates within an mmul block |
| const uint thread_id = (x0 % MMUL_BLOCK_SIZE); |
| const uint thread_x = thread_id % MMUL_N0; |
| const uint thread_y = (thread_id / MMUL_N0); |
| |
| // Calculate dst coordinates |
| const uint dst_x_unclamped = thread_x * N0 + section_x * N0 * MMUL_N0; |
| const uint dst_y_unclamped = thread_y * M0 + section_y * M0 * MMUL_M0; |
| const uint dst_x = min(dst_x_unclamped, (uint)(N - N0)); |
| const uint dst_y = min(dst_y_unclamped, (uint)(M - M0)); |
| |
| // Starting LHS coordinates |
| const uint lhs_x = K0 * thread_x; |
| const uint lhs_y = dst_y; |
| |
| // Starting RHS coordinates |
| const uint rhs_x = K0 * thread_y; |
| const uint rhs_y = dst_x; |
| |
| // Compute LHS/RHS/DST matrix address |
| lhs_offset_first_element_in_bytes += lhs_x * sizeof(DATA_TYPE) + lhs_y * lhs_stride_y + z * lhs_stride_z; |
| rhs_offset_first_element_in_bytes += rhs_x * sizeof(DATA_TYPE) + rhs_y * rhs_stride_y + z * rhs_stride_z; |
| dst_offset_first_element_in_bytes += dst_x * sizeof(DATA_TYPE) + dst_y * dst_stride_y + z * dst_stride_z; |
| |
| // Initialize the accumulators |
| TILE(int, M0, N0, c); |
| LOOP_UNROLLING(int, i, 0, 1, M0, |
| { |
| c[i].v = K * ((int)LHS_OFFSET) * ((int)RHS_OFFSET); |
| }) |
| |
| // Calculate row and column sums |
| TILE(int, 1, N0, b_sum); |
| b_sum[0].v = 0; |
| |
| TILE(int, 1, M0, a_sum); |
| a_sum[0].v = 0; |
| |
| VEC_DATA_TYPE(DATA_TYPE, K0) |
| vec_1 = (VEC_DATA_TYPE(DATA_TYPE, K0))(1, 1, 1, 1); |
| |
| for(int k = 0; k < lhs_w; k += MMUL_K0) |
| { |
| // A tile of M0xK0 but K0 must be set to K0 |
| TILE(DATA_TYPE, M0, K0, a); |
| // A tile of K0xN0 but K0 must be set to K0 |
| TILE(DATA_TYPE, N0, K0, b); |
| |
| // Load tile from the lhs/rhs tensors |
| T_LOAD(DATA_TYPE, M0, K0, BUFFER, lhs, 0, 0, 1, lhs_stride_y, a); |
| T_LOAD(DATA_TYPE, N0, K0, BUFFER, rhs, 0, 0, 1, rhs_stride_y, b); |
| |
| LOOP_UNROLLING(int, m0, 0, 1, M0, |
| { |
| LOOP_UNROLLING(int, n0, 0, 1, N0, |
| { |
| c[m0].s[n0] = arm_matrix_multiply(a[m0].v, b[n0].v, c[m0].s[n0]); |
| }) |
| }) |
| |
| #if RHS_OFFSET != 0 |
| // Row Sum of A: Calculate the sum of rows by multiplying A with |
| // a matrix of 1's from Right |
| LOOP_UNROLLING(int, m0, 0, 1, M0, |
| { |
| a_sum[0].s[m0] = arm_matrix_multiply(a[m0].v, vec_1, a_sum[0].s[m0]); |
| }) |
| #endif // RHS_OFFSET != 0 |
| |
| #if LHS_OFFSET != 0 |
| // Column Sum of B: Calculate the sum of columns by multiplying B |
| // with a matrix of 1's from Left |
| LOOP_UNROLLING(int, n0, 0, 1, N0, |
| { |
| b_sum[0].s[n0] = arm_matrix_multiply(vec_1, b[n0].v, b_sum[0].s[n0]); |
| }) |
| #endif // LHS_OFFSET != 0 |
| |
| lhs_offset_first_element_in_bytes += MMUL_K0 * sizeof(DATA_TYPE); |
| rhs_offset_first_element_in_bytes += MMUL_K0 * sizeof(DATA_TYPE); |
| } |
| |
| // Do not write if the coordinates are out of bound |
| // But, read has to happen as arm_matrix_multiply() expects certain number of calls |
| if(dst_x_unclamped >= N || dst_y_unclamped >= M) |
| { |
| return; |
| } |
| |
| #if RHS_OFFSET != 0 || LHS_OFFSET != 0 |
| LOOP_UNROLLING(int, i, 0, 1, M0, |
| { |
| const int A = ((int)RHS_OFFSET) * a_sum[0].s[i]; |
| LOOP_UNROLLING(int, j, 0, 1, N0, |
| { |
| c[i].s[j] -= A + ((int)(LHS_OFFSET)) * b_sum[0].s[j]; |
| }) |
| }) |
| #endif // RHS_OFFSET != 0 || LHS_OFFSET != 0 |
| |
| #ifdef BIAS |
| perform_bias_addition(bias_ptr, bias_offset_first_element_in_bytes, c, dst_x); |
| #endif // defined(BIAS) |
| |
| // Quantize the tile |
| TILE(DATA_TYPE, M0, N0, cq); |
| T_QUANTIZE8_ASYMMETRIC(int, DATA_TYPE, M0, N0, DST_OFFSET, DST_SHIFT, DST_MULTIPLIER, c, cq); |
| |
| if(dst_x + N0 <= N || N0_LEFTOVER == 0) |
| { |
| LOOP_UNROLLING(int, m0, 0, 1, M0, |
| { |
| if(dst_y + m0 < M || M0_LEFTOVER == 0) |
| { |
| VSTORE(N0) |
| (cq[m0].v, 0, (__global DATA_TYPE *)(dst_ptr + dst_offset_first_element_in_bytes + m0 * dst_stride_y)); |
| } |
| }) |
| } |
| else |
| { |
| LOOP_UNROLLING(int, m0, 0, 1, M0, |
| { |
| if(dst_y + m0 < M || M0_LEFTOVER == 0) |
| { |
| VSTORE_PARTIAL(N0, N0_LEFTOVER) |
| (cq[m0].v, 0, (__global DATA_TYPE *)(dst_ptr + dst_offset_first_element_in_bytes + m0 * dst_stride_y)); |
| } |
| }) |
| } |
| } |
| #endif // defined(MAT_MUL_NATIVE_QUANTIZED_MMUL_NT_T) |
| |
| #if defined(MAT_MUL_NATIVE_QUANTIZED_MMUL_T_NT) |
| /** This OpenCL kernel performs the batch matrix multiplication (BatchMatMul): LHS transposed, RHS non-transposed |
| * |
| * Supported block configurations: |
| * - M0 = 1, 2, 3, 4, 8, 16 |
| * - N0 = 1, 2, 3, 4, 8, 16 |
| * - K0 = 4 |
| * |
| * Similar to mat_mul_native_quantized_mmul_nt_nt() |
| */ |
| __kernel void mat_mul_native_quantized_mmul_t_nt( |
| TENSOR3D_T(lhs, BUFFER), |
| TENSOR3D_T(rhs, BUFFER), |
| #ifdef BIAS |
| TENSOR3D_T(bias, BUFFER), |
| #endif // defined(BIAS) |
| TENSOR3D_T(dst, BUFFER)) |
| { |
| const uint x0 = get_global_id(0); // [0, (N / N0) * MMUL_M0) |
| // The upper limit is a simplified version of (N / N0) / MMUL_N0) * MMUL_BLOCK_SIZE) |
| const uint y0 = get_global_id(1); // [0, (M / M0) / MMUL_M0) |
| const uint z = get_global_id(2); // Batch |
| |
| // Get section coordinates |
| const uint section_x = (x0 / MMUL_BLOCK_SIZE); |
| const uint section_y = y0; |
| |
| // Get thread coordinates within an mmul block |
| const uint thread_id = (x0 % MMUL_BLOCK_SIZE); |
| const uint thread_x = thread_id % MMUL_N0; |
| const uint thread_y = (thread_id / MMUL_N0); |
| |
| // Calculate dst coordinates |
| const uint dst_x_unclamped = thread_x * N0 + section_x * N0 * MMUL_N0; |
| const uint dst_y_unclamped = thread_y * M0 + section_y * M0 * MMUL_M0; |
| const uint dst_x = min(dst_x_unclamped, (uint)(N - N0)); |
| const uint dst_y = min(dst_y_unclamped, (uint)(M - M0)); |
| |
| // Starting LHS coordinates |
| const uint lhs_x = dst_y; |
| const uint lhs_y = K0 * thread_x; |
| |
| // Starting RHS coordinates |
| const uint rhs_x = dst_x; |
| const uint rhs_y = K0 * thread_y; |
| |
| // Compute LHS/RHS/DST matrix address |
| lhs_offset_first_element_in_bytes += lhs_x * sizeof(DATA_TYPE) + lhs_y * lhs_stride_y + z * lhs_stride_z; |
| rhs_offset_first_element_in_bytes += rhs_x * sizeof(DATA_TYPE) + rhs_y * rhs_stride_y + z * rhs_stride_z; |
| dst_offset_first_element_in_bytes += dst_x * sizeof(DATA_TYPE) + dst_y * dst_stride_y + z * dst_stride_z; |
| |
| // Initialize the accumulators |
| TILE(int, M0, N0, c); |
| LOOP_UNROLLING(int, i, 0, 1, M0, |
| { |
| c[i].v = K * ((int)LHS_OFFSET) * ((int)RHS_OFFSET); |
| }) |
| |
| // Calculate row and column sums |
| TILE(int, 1, N0, b_sum); |
| b_sum[0].v = 0; |
| |
| TILE(int, 1, M0, a_sum); |
| a_sum[0].v = 0; |
| |
| VEC_DATA_TYPE(DATA_TYPE, K0) |
| vec_1 = (VEC_DATA_TYPE(DATA_TYPE, K0))(1, 1, 1, 1); |
| |
| for(int k = 0; k < lhs_h; k += MMUL_K0) |
| { |
| TILE(DATA_TYPE, K0, M0, a); |
| TILE(DATA_TYPE, K0, N0, b); |
| |
| // Load tile from the lhs/rhs tensors |
| T_LOAD(DATA_TYPE, K0, M0, BUFFER, lhs, 0, 0, 1, lhs_stride_y, a); |
| T_LOAD(DATA_TYPE, K0, N0, BUFFER, rhs, 0, 0, 1, rhs_stride_y, b); |
| |
| LOOP_UNROLLING(int, m0, 0, 1, M0, |
| { |
| VEC_DATA_TYPE(DATA_TYPE, K0) |
| vec_a = (VEC_DATA_TYPE(DATA_TYPE, K0))(a[0].s[m0], a[1].s[m0], a[2].s[m0], a[3].s[m0]); |
| |
| LOOP_UNROLLING(int, n0, 0, 1, N0, |
| { |
| VEC_DATA_TYPE(DATA_TYPE, K0) |
| vec_b = (VEC_DATA_TYPE(DATA_TYPE, K0))(b[0].s[n0], b[1].s[n0], b[2].s[n0], b[3].s[n0]); |
| |
| c[m0].s[n0] = arm_matrix_multiply(vec_a, vec_b, c[m0].s[n0]); |
| }) |
| |
| #if RHS_OFFSET != 0 |
| // Row Sum of A: Calculate the sum of rows by multiplying A with |
| // a matrix of 1's from Right |
| a_sum[0].s[m0] = arm_matrix_multiply(vec_a, vec_1, a_sum[0].s[m0]); |
| #endif // RHS_OFFSET != 0 |
| }) |
| |
| #if LHS_OFFSET != 0 |
| // Column Sum of B: Calculate the sum of columns by multiplying B |
| // with a matrix of 1's from Left |
| LOOP_UNROLLING(int, n0, 0, 1, N0, |
| { |
| VEC_DATA_TYPE(DATA_TYPE, K0) |
| vec_b = (VEC_DATA_TYPE(DATA_TYPE, K0))(b[0].s[n0], b[1].s[n0], b[2].s[n0], b[3].s[n0]); |
| |
| b_sum[0].s[n0] = arm_matrix_multiply(vec_1, vec_b, b_sum[0].s[n0]); |
| }) |
| #endif // LHS_OFFSET != 0 |
| |
| lhs_offset_first_element_in_bytes += MMUL_K0 * lhs_stride_y; |
| rhs_offset_first_element_in_bytes += MMUL_K0 * rhs_stride_y; |
| } |
| |
| // Do not write if the coordinates are out of bound |
| // But, read has to happen as arm_matrix_multiply() expects certain number of calls |
| if(dst_x_unclamped >= N || dst_y_unclamped >= M) |
| { |
| return; |
| } |
| |
| #if RHS_OFFSET != 0 || LHS_OFFSET != 0 |
| LOOP_UNROLLING(int, i, 0, 1, M0, |
| { |
| const int A = ((int)RHS_OFFSET) * a_sum[0].s[i]; |
| LOOP_UNROLLING(int, j, 0, 1, N0, |
| { |
| c[i].s[j] -= A + ((int)(LHS_OFFSET)) * b_sum[0].s[j]; |
| }) |
| }) |
| #endif // RHS_OFFSET != 0 || LHS_OFFSET != 0 |
| |
| #ifdef BIAS |
| perform_bias_addition(bias_ptr, bias_offset_first_element_in_bytes, c, dst_x); |
| #endif // defined(BIAS) |
| |
| // Quantize the tile |
| TILE(DATA_TYPE, M0, N0, cq); |
| T_QUANTIZE8_ASYMMETRIC(int, DATA_TYPE, M0, N0, DST_OFFSET, DST_SHIFT, DST_MULTIPLIER, c, cq); |
| |
| if(dst_x + N0 <= N || N0_LEFTOVER == 0) |
| { |
| LOOP_UNROLLING(int, m0, 0, 1, M0, |
| { |
| if(dst_y + m0 < M || M0_LEFTOVER == 0) |
| { |
| VSTORE(N0) |
| (cq[m0].v, 0, (__global DATA_TYPE *)(dst_ptr + dst_offset_first_element_in_bytes + m0 * dst_stride_y)); |
| } |
| }) |
| } |
| else |
| { |
| LOOP_UNROLLING(int, m0, 0, 1, M0, |
| { |
| if(dst_y + m0 < M || M0_LEFTOVER == 0) |
| { |
| VSTORE_PARTIAL(N0, N0_LEFTOVER) |
| (cq[m0].v, 0, (__global DATA_TYPE *)(dst_ptr + dst_offset_first_element_in_bytes + m0 * dst_stride_y)); |
| } |
| }) |
| } |
| } |
| #endif // defined(MAT_MUL_NATIVE_QUANTIZED_MMUL_T_NT) |
| |
| #if defined(MAT_MUL_NATIVE_QUANTIZED_MMUL_T_T) |
| /** This OpenCL kernel performs the batch matrix multiplication (BatchMatMul): LHS transposed, RHS transposed |
| * |
| * Supported block configurations: |
| * - M0 = 1, 2, 3, 4, 8, 16 |
| * - N0 = 1, 2, 3, 4, 8, 16 |
| * - K0 = 4 |
| * |
| * Similar to mat_mul_native_quantized_mmul_nt_nt() |
| */ |
| __kernel void mat_mul_native_quantized_mmul_t_t( |
| TENSOR3D_T(lhs, BUFFER), |
| TENSOR3D_T(rhs, BUFFER), |
| #ifdef BIAS |
| TENSOR3D_T(bias, BUFFER), |
| #endif // defined(BIAS) |
| TENSOR3D_T(dst, BUFFER)) |
| { |
| const uint x0 = get_global_id(0); // [0, (N / N0) * MMUL_M0) |
| // The upper limit is a simplified version of (N / N0) / MMUL_N0) * MMUL_BLOCK_SIZE) |
| const uint y0 = get_global_id(1); // [0, (M / M0) / MMUL_M0) |
| const uint z = get_global_id(2); // Batch |
| |
| // Get section coordinates |
| const uint section_x = (x0 / MMUL_BLOCK_SIZE); |
| const uint section_y = y0; |
| |
| // Get thread coordinates within an mmul block |
| const uint thread_id = (x0 % MMUL_BLOCK_SIZE); |
| const uint thread_x = thread_id % MMUL_N0; |
| const uint thread_y = (thread_id / MMUL_N0); |
| |
| // Calculate dst coordinates |
| const uint dst_x_unclamped = thread_x * N0 + section_x * N0 * MMUL_N0; |
| const uint dst_y_unclamped = thread_y * M0 + section_y * M0 * MMUL_M0; |
| const uint dst_x = min(dst_x_unclamped, (uint)(N - N0)); |
| const uint dst_y = min(dst_y_unclamped, (uint)(M - M0)); |
| |
| // Starting LHS coordinates |
| const uint lhs_x = dst_y; |
| const uint lhs_y = K0 * thread_x; |
| |
| // Starting RHS coordinates |
| const uint rhs_x = K0 * thread_y; |
| const uint rhs_y = dst_x; |
| |
| // Compute LHS/RHS/DST matrix address |
| lhs_offset_first_element_in_bytes += lhs_x * sizeof(DATA_TYPE) + lhs_y * lhs_stride_y + z * lhs_stride_z; |
| rhs_offset_first_element_in_bytes += rhs_x * sizeof(DATA_TYPE) + rhs_y * rhs_stride_y + z * rhs_stride_z; |
| dst_offset_first_element_in_bytes += dst_x * sizeof(DATA_TYPE) + dst_y * dst_stride_y + z * dst_stride_z; |
| |
| // Initialize the accumulators |
| TILE(int, M0, N0, c); |
| LOOP_UNROLLING(int, i, 0, 1, M0, |
| { |
| c[i].v = K * ((int)LHS_OFFSET) * ((int)RHS_OFFSET); |
| }) |
| |
| // Calculate row and column sums |
| TILE(int, 1, N0, b_sum); |
| b_sum[0].v = 0; |
| |
| TILE(int, 1, M0, a_sum); |
| a_sum[0].v = 0; |
| |
| VEC_DATA_TYPE(DATA_TYPE, K0) |
| vec_1 = (VEC_DATA_TYPE(DATA_TYPE, K0))(1, 1, 1, 1); |
| |
| for(int k = 0; k < lhs_h; k += MMUL_K0) |
| { |
| TILE(DATA_TYPE, K0, M0, a); |
| TILE(DATA_TYPE, N0, K0, b); |
| |
| // Load tile from the lhs/rhs tensors |
| T_LOAD(DATA_TYPE, K0, M0, BUFFER, lhs, 0, 0, 1, lhs_stride_y, a); |
| T_LOAD(DATA_TYPE, N0, K0, BUFFER, rhs, 0, 0, 1, rhs_stride_y, b); |
| |
| LOOP_UNROLLING(int, m0, 0, 1, M0, |
| { |
| VEC_DATA_TYPE(DATA_TYPE, K0) |
| vec_a = (VEC_DATA_TYPE(DATA_TYPE, K0))(a[0].s[m0], a[1].s[m0], a[2].s[m0], a[3].s[m0]); |
| |
| LOOP_UNROLLING(int, n0, 0, 1, N0, |
| { |
| c[m0].s[n0] = arm_matrix_multiply(vec_a, b[n0].v, c[m0].s[n0]); |
| }) |
| #if RHS_OFFSET != 0 |
| // Row Sum of A: Calculate the sum of rows by multiplying A with |
| // a matrix of 1's from Right |
| a_sum[0].s[m0] = arm_matrix_multiply(vec_a, vec_1, a_sum[0].s[m0]); |
| #endif // RHS_OFFSET != 0 |
| }) |
| |
| #if LHS_OFFSET != 0 |
| // Column Sum of B: Calculate the sum of columns by multiplying B |
| // with a matrix of 1's from Left |
| LOOP_UNROLLING(int, n0, 0, 1, N0, |
| { |
| b_sum[0].s[n0] = arm_matrix_multiply(vec_1, b[n0].v, b_sum[0].s[n0]); |
| }) |
| #endif // LHS_OFFSET != 0 |
| |
| lhs_offset_first_element_in_bytes += MMUL_K0 * lhs_stride_y; |
| rhs_offset_first_element_in_bytes += MMUL_K0 * sizeof(DATA_TYPE); |
| } |
| |
| // Do not write if the coordinates are out of bound |
| // But, read has to happen as arm_matrix_multiply() expects certain number of calls |
| if(dst_x_unclamped >= N || dst_y_unclamped >= M) |
| { |
| return; |
| } |
| |
| #if RHS_OFFSET != 0 || LHS_OFFSET != 0 |
| LOOP_UNROLLING(int, i, 0, 1, M0, |
| { |
| const int A = ((int)RHS_OFFSET) * a_sum[0].s[i]; |
| LOOP_UNROLLING(int, j, 0, 1, N0, |
| { |
| c[i].s[j] -= A + ((int)(LHS_OFFSET)) * b_sum[0].s[j]; |
| }) |
| }) |
| #endif // RHS_OFFSET != 0 || LHS_OFFSET != 0 |
| |
| #ifdef BIAS |
| perform_bias_addition(bias_ptr, bias_offset_first_element_in_bytes, c, dst_x); |
| #endif // defined(BIAS) |
| |
| // Quantize the tile |
| TILE(DATA_TYPE, M0, N0, cq); |
| T_QUANTIZE8_ASYMMETRIC(int, DATA_TYPE, M0, N0, DST_OFFSET, DST_SHIFT, DST_MULTIPLIER, c, cq); |
| |
| if(dst_x + N0 <= N || N0_LEFTOVER == 0) |
| { |
| LOOP_UNROLLING(int, m0, 0, 1, M0, |
| { |
| if(dst_y + m0 < M || M0_LEFTOVER == 0) |
| { |
| VSTORE(N0) |
| (cq[m0].v, 0, (__global DATA_TYPE *)(dst_ptr + dst_offset_first_element_in_bytes + m0 * dst_stride_y)); |
| } |
| }) |
| } |
| else |
| { |
| LOOP_UNROLLING(int, m0, 0, 1, M0, |
| { |
| if(dst_y + m0 < M || M0_LEFTOVER == 0) |
| { |
| VSTORE_PARTIAL(N0, N0_LEFTOVER) |
| (cq[m0].v, 0, (__global DATA_TYPE *)(dst_ptr + dst_offset_first_element_in_bytes + m0 * dst_stride_y)); |
| } |
| }) |
| } |
| } |
| #endif // defined(MAT_MUL_NATIVE_QUANTIZED_MMUL_T_T) |