Yair Schwarzbaum | 298b2c0 | 2022-02-01 08:55:56 +0200 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (c) 2016-2022 Arm Limited. |
| 3 | * |
| 4 | * SPDX-License-Identifier: MIT |
| 5 | * |
| 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy |
| 7 | * of this software and associated documentation files (the "Software"), to |
| 8 | * deal in the Software without restriction, including without limitation the |
| 9 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or |
| 10 | * sell copies of the Software, and to permit persons to whom the Software is |
| 11 | * furnished to do so, subject to the following conditions: |
| 12 | * |
| 13 | * The above copyright notice and this permission notice shall be included in all |
| 14 | * copies or substantial portions of the Software. |
| 15 | * |
| 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
| 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
| 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE |
| 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
| 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
| 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
| 22 | * SOFTWARE. |
| 23 | */ |
Pablo Marquez Tello | d208f4f | 2022-07-19 12:19:46 +0100 | [diff] [blame] | 24 | #if defined(ARM_COMPUTE_ENABLE_BF16) |
Yair Schwarzbaum | 298b2c0 | 2022-02-01 08:55:56 +0200 | [diff] [blame] | 25 | |
| 26 | #include "arm_compute/core/TensorInfo.h" |
Michalis Spyrou | 168d6a8 | 2022-05-03 17:15:42 +0100 | [diff] [blame] | 27 | #include "src/core/NEON/wrapper/wrapper.h" |
Yair Schwarzbaum | 298b2c0 | 2022-02-01 08:55:56 +0200 | [diff] [blame] | 28 | #include "src/cpu/kernels/CpuCastKernel.h" |
| 29 | #include "src/cpu/kernels/cast/list.h" |
| 30 | #include "support/SaturateCast.h" |
| 31 | |
| 32 | namespace arm_compute |
| 33 | { |
| 34 | namespace cpu |
| 35 | { |
| 36 | void neon_fp32_to_bfloat16_cast(const ITensor *_src, ITensor *_dst, const ThreadInfo &info, ConvertPolicy _policy, const Window &window) |
| 37 | { |
| 38 | ARM_COMPUTE_UNUSED(info); |
| 39 | ARM_COMPUTE_UNUSED(_policy); |
| 40 | |
| 41 | const auto window_start_x = static_cast<int>(window.x().start()); |
| 42 | const auto window_end_x = static_cast<int>(window.x().end()); |
| 43 | const int window_step_x = 16; |
| 44 | |
| 45 | ARM_COMPUTE_ERROR_ON_NULLPTR(_src, _dst); |
| 46 | ARM_COMPUTE_ERROR_ON(_src == _dst); |
| 47 | |
| 48 | ARM_COMPUTE_ERROR_ON_NULLPTR(_src, _dst); |
| 49 | |
| 50 | Window win{ window }; |
| 51 | win.set(Window::DimX, Window::Dimension(0, 1, 1)); |
| 52 | |
| 53 | Iterator src(_src, win); |
| 54 | Iterator dst(_dst, win); |
| 55 | |
| 56 | /* Down-conversion F32 -> BFLOAT16 */ |
| 57 | execute_window_loop(win, [&](const Coordinates &) |
| 58 | { |
| 59 | const auto src_ptr = reinterpret_cast<const float *>(src.ptr()); |
| 60 | const auto dst_ptr = reinterpret_cast<bfloat16 *>(dst.ptr()); |
| 61 | |
| 62 | int x = window_start_x; |
| 63 | for(; x <= (window_end_x - window_step_x); x += window_step_x) |
| 64 | { |
| 65 | wrapper::vcvt_bf16_f32(reinterpret_cast<float *>(src.ptr()), |
| 66 | reinterpret_cast<uint16_t *>(dst.ptr())); |
| 67 | wrapper::vcvt_bf16_f32(reinterpret_cast<float *>(src.ptr()) + 8, |
| 68 | reinterpret_cast<uint16_t *>(dst.ptr()) + 8); |
| 69 | } |
| 70 | |
| 71 | for(; x < window_end_x; ++x) |
| 72 | { |
| 73 | *(dst_ptr + x) = *(src_ptr + x); |
| 74 | } |
| 75 | }, |
| 76 | src, dst); |
| 77 | } |
| 78 | |
| 79 | void neon_bfloat16_to_fp32_cast(const ITensor *_src, ITensor *_dst, const ThreadInfo &info, ConvertPolicy _policy, const Window &window) |
| 80 | { |
| 81 | ARM_COMPUTE_UNUSED(info); |
| 82 | ARM_COMPUTE_UNUSED(_policy); |
| 83 | |
| 84 | const auto window_start_x = static_cast<int>(window.x().start()); |
| 85 | const auto window_end_x = static_cast<int>(window.x().end()); |
| 86 | const int window_step_x = 16; |
| 87 | |
| 88 | ARM_COMPUTE_ERROR_ON_NULLPTR(_src, _dst); |
| 89 | ARM_COMPUTE_ERROR_ON(_src == _dst); |
| 90 | |
| 91 | ARM_COMPUTE_ERROR_ON_NULLPTR(_src, _dst); |
| 92 | |
| 93 | Window win{ window }; |
| 94 | win.set(Window::DimX, Window::Dimension(0, 1, 1)); |
| 95 | |
| 96 | Iterator src(_src, win); |
| 97 | Iterator dst(_dst, win); |
| 98 | switch(_dst->info()->data_type()) |
| 99 | { |
| 100 | case DataType::F32: |
| 101 | { |
| 102 | /* Up-conversion BFLOAT16 -> F32 */ |
| 103 | execute_window_loop(win, [&](const Coordinates &) |
| 104 | { |
| 105 | const auto src_ptr = reinterpret_cast<const bfloat16 *>(src.ptr()); |
| 106 | const auto dst_ptr = reinterpret_cast<float *>(dst.ptr()); |
| 107 | |
| 108 | int x = window_start_x; |
| 109 | for(; x <= (window_end_x - window_step_x); x += window_step_x) |
| 110 | { |
| 111 | const uint16x8x2_t texels = |
| 112 | { |
| 113 | { |
| 114 | vld1q_u16(reinterpret_cast<uint16_t *>(src.ptr())), |
| 115 | vld1q_u16(reinterpret_cast<uint16_t *>(src.ptr()) + 8) |
| 116 | } |
| 117 | }; |
| 118 | |
| 119 | vst1q_f32(reinterpret_cast<float *>(dst.ptr()), |
| 120 | vreinterpretq_f32_u32(vshlq_n_u32(vmovl_u16(vget_low_u16(texels.val[0])), 16))); |
| 121 | vst1q_f32(reinterpret_cast<float *>(dst.ptr()) + 4, |
| 122 | vreinterpretq_f32_u32(vshlq_n_u32(vmovl_u16(vget_high_u16(texels.val[0])), 16))); |
| 123 | vst1q_f32(reinterpret_cast<float *>(dst.ptr()) + 8, |
| 124 | vreinterpretq_f32_u32(vshlq_n_u32(vmovl_u16(vget_low_u16(texels.val[1])), 16))); |
| 125 | vst1q_f32(reinterpret_cast<float *>(dst.ptr()) + 12, |
| 126 | vreinterpretq_f32_u32(vshlq_n_u32(vmovl_u16(vget_high_u16(texels.val[1])), 16))); |
| 127 | } |
| 128 | |
| 129 | for(; x < window_end_x; ++x) |
| 130 | { |
| 131 | *(dst_ptr + x) = float(*(src_ptr + x)); |
| 132 | } |
| 133 | }, |
| 134 | src, dst); |
| 135 | break; |
| 136 | } |
| 137 | default: |
| 138 | ARM_COMPUTE_ERROR("dst data type unsupported"); |
| 139 | } |
| 140 | } |
| 141 | |
| 142 | } // namespace cpu |
| 143 | } // namespace arm_compute |
| 144 | |
Pablo Marquez Tello | d208f4f | 2022-07-19 12:19:46 +0100 | [diff] [blame] | 145 | #endif /* defined(ARM_COMPUTE_ENABLE_BF16) */ |