Anthony Barbier | 71d9b57 | 2018-07-06 17:05:59 +0100 | [diff] [blame] | 1 | /* |
Michalis Spyrou | 71ac903 | 2019-11-14 14:31:44 +0000 | [diff] [blame] | 2 | * Copyright (c) 2018-2020 ARM Limited. |
Anthony Barbier | 71d9b57 | 2018-07-06 17:05:59 +0100 | [diff] [blame] | 3 | * |
| 4 | * SPDX-License-Identifier: MIT |
| 5 | * |
| 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy |
| 7 | * of this software and associated documentation files (the "Software"), to |
| 8 | * deal in the Software without restriction, including without limitation the |
| 9 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or |
| 10 | * sell copies of the Software, and to permit persons to whom the Software is |
| 11 | * furnished to do so, subject to the following conditions: |
| 12 | * |
| 13 | * The above copyright notice and this permission notice shall be included in all |
| 14 | * copies or substantial portions of the Software. |
| 15 | * |
| 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
| 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
| 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE |
| 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
| 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
| 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
| 22 | * SOFTWARE. |
| 23 | */ |
| 24 | #include "arm_compute/runtime/NEON/functions/NEGEMMAssemblyDispatch.h" |
| 25 | |
Anthony Barbier | eaefd00 | 2018-07-20 17:49:35 +0100 | [diff] [blame] | 26 | #include "arm_compute/core/CPP/Validate.h" |
Anthony Barbier | 71d9b57 | 2018-07-06 17:05:59 +0100 | [diff] [blame] | 27 | #include "arm_compute/runtime/NEON/NEScheduler.h" |
Anthony Barbier | c8e84b5 | 2018-07-17 16:48:42 +0100 | [diff] [blame] | 28 | #include "arm_compute/runtime/NEON/functions/NESimpleAssemblyFunction.h" |
Anthony Barbier | 71d9b57 | 2018-07-06 17:05:59 +0100 | [diff] [blame] | 29 | |
Anthony Barbier | eaefd00 | 2018-07-20 17:49:35 +0100 | [diff] [blame] | 30 | #include <arm_neon.h> |
| 31 | |
Anthony Barbier | c8e84b5 | 2018-07-17 16:48:42 +0100 | [diff] [blame] | 32 | namespace arm_compute |
| 33 | { |
Anthony Barbier | eaefd00 | 2018-07-20 17:49:35 +0100 | [diff] [blame] | 34 | namespace |
Anthony Barbier | 71d9b57 | 2018-07-06 17:05:59 +0100 | [diff] [blame] | 35 | { |
Georgios Pinitas | 48b3ef8 | 2019-10-14 19:03:09 +0100 | [diff] [blame] | 36 | arm_gemm::Activation map_to_arm_gemm_activation(const ActivationLayerInfo &act) |
Anthony Barbier | eaefd00 | 2018-07-20 17:49:35 +0100 | [diff] [blame] | 37 | { |
Georgios Pinitas | 48b3ef8 | 2019-10-14 19:03:09 +0100 | [diff] [blame] | 38 | arm_gemm::Activation gemm_act; |
| 39 | |
| 40 | // Early exit in case lower bound is other than 0, as it's not yet supported |
| 41 | if(act.b() != 0.f) |
Anthony Barbier | c8e84b5 | 2018-07-17 16:48:42 +0100 | [diff] [blame] | 42 | { |
Georgios Pinitas | 48b3ef8 | 2019-10-14 19:03:09 +0100 | [diff] [blame] | 43 | return gemm_act; |
Anthony Barbier | c8e84b5 | 2018-07-17 16:48:42 +0100 | [diff] [blame] | 44 | } |
Georgios Pinitas | 48b3ef8 | 2019-10-14 19:03:09 +0100 | [diff] [blame] | 45 | |
| 46 | switch(act.activation()) |
| 47 | { |
| 48 | case ActivationLayerInfo::ActivationFunction::RELU: |
| 49 | gemm_act.type = arm_gemm::Activation::Type::ReLU; |
| 50 | break; |
| 51 | case ActivationLayerInfo::ActivationFunction::BOUNDED_RELU: |
| 52 | gemm_act.type = arm_gemm::Activation::Type::BoundedReLU; |
| 53 | gemm_act.param1 = act.a(); |
| 54 | gemm_act.param2 = 0.f; |
| 55 | break; |
| 56 | case ActivationLayerInfo::ActivationFunction::LU_BOUNDED_RELU: |
| 57 | gemm_act.type = arm_gemm::Activation::Type::BoundedReLU; |
| 58 | gemm_act.param1 = act.a(); |
| 59 | gemm_act.param2 = act.b(); |
| 60 | break; |
| 61 | default: |
| 62 | gemm_act.type = arm_gemm::Activation::Type::None; |
| 63 | } |
| 64 | |
| 65 | return gemm_act; |
Anthony Barbier | c8e84b5 | 2018-07-17 16:48:42 +0100 | [diff] [blame] | 66 | } |
| 67 | |
Michalis Spyrou | 1a569a3 | 2019-09-10 17:20:34 +0100 | [diff] [blame] | 68 | template <typename TypeInput, typename TypeOutput> |
| 69 | class FallbackTransform : public ITransformWeights |
| 70 | { |
| 71 | public: |
Michalis Spyrou | 5cb49dc | 2019-12-03 13:42:25 +0000 | [diff] [blame] | 72 | FallbackTransform() noexcept {}; |
| 73 | /** Prevent instances of this class from being copied (As this class contains pointers) */ |
| 74 | FallbackTransform(const FallbackTransform &) = delete; |
| 75 | /** Default move constructor */ |
| 76 | FallbackTransform(FallbackTransform &&) = default; |
| 77 | /** Prevent instances of this class from being copied (As this class contains pointers) */ |
| 78 | FallbackTransform &operator=(const FallbackTransform &) = delete; |
| 79 | /** Default move assignment operator */ |
| 80 | FallbackTransform &operator=(FallbackTransform &&) = default; |
| 81 | void run() override |
Michalis Spyrou | 1a569a3 | 2019-09-10 17:20:34 +0100 | [diff] [blame] | 82 | { |
| 83 | _output.allocator()->allocate(); |
| 84 | ARM_COMPUTE_ERROR_ON(_output.buffer() == nullptr); |
| 85 | _gemm_kernel_asm->pretranspose_B_array(_output.buffer(), _in1_ptr, _ldb, _multi_stride_b); |
| 86 | _reshape_run = true; |
| 87 | } |
| 88 | |
| 89 | void release() override |
| 90 | { |
| 91 | _output.allocator()->free(); |
| 92 | } |
| 93 | |
| 94 | ITensor *get_weights() override |
| 95 | { |
| 96 | return &_output; |
| 97 | } |
| 98 | |
| 99 | uint32_t uid() override |
| 100 | { |
| 101 | uint32_t id = (_B_pretranspose_size | 0x80000000); |
| 102 | return id; |
| 103 | } |
| 104 | |
| 105 | void configure(size_t B_pretranspose_size, unsigned int alignment) |
| 106 | { |
| 107 | _output.allocator()->init(TensorInfo(TensorShape{ (B_pretranspose_size + alignment /* FIXME: remove alignment after COMPMID-1088 */) }, 1, DataType::S8), alignment); |
| 108 | _B_pretranspose_size = B_pretranspose_size; |
| 109 | } |
| 110 | |
| 111 | void set_pretranspose(ITensor *tensor) |
| 112 | { |
| 113 | if(!_reshape_run) |
| 114 | { |
| 115 | _gemm_kernel_asm->set_pretransposed_B_data(tensor->buffer()); |
| 116 | } |
| 117 | } |
| 118 | |
| 119 | void set_args(const int ldb, const TypeInput *in1_ptr, const int multi_stride_b, std::shared_ptr<arm_gemm::GemmCommon<TypeInput, TypeOutput>> gemm_kernel_asm) |
| 120 | { |
| 121 | _ldb = ldb; |
| 122 | _in1_ptr = in1_ptr; |
| 123 | _multi_stride_b = multi_stride_b; |
| 124 | _gemm_kernel_asm = gemm_kernel_asm; |
| 125 | } |
| 126 | |
| 127 | private: |
| 128 | Tensor _output{}; |
| 129 | int _ldb{}; |
| 130 | const TypeInput *_in1_ptr{}; |
| 131 | int _multi_stride_b{}; |
| 132 | size_t _B_pretranspose_size{}; |
| 133 | std::shared_ptr<arm_gemm::GemmCommon<TypeInput, TypeOutput>> _gemm_kernel_asm{ nullptr }; |
| 134 | }; |
| 135 | |
Anthony Barbier | eaefd00 | 2018-07-20 17:49:35 +0100 | [diff] [blame] | 136 | /** Fallback in case ACL doesn't have a function */ |
Georgios Pinitas | cfa2bba | 2019-06-27 17:00:52 +0100 | [diff] [blame] | 137 | template <typename TypeInput, typename TypeOutput, class OutputStage = arm_gemm::Nothing> |
Anthony Barbier | eaefd00 | 2018-07-20 17:49:35 +0100 | [diff] [blame] | 138 | class Fallback : public NEGEMMAssemblyDispatch::IFallback |
Anthony Barbier | c8e84b5 | 2018-07-17 16:48:42 +0100 | [diff] [blame] | 139 | { |
Anthony Barbier | eaefd00 | 2018-07-20 17:49:35 +0100 | [diff] [blame] | 140 | public: |
Michalis Spyrou | 1a569a3 | 2019-09-10 17:20:34 +0100 | [diff] [blame] | 141 | /** Destructor */ |
| 142 | ~Fallback() |
| 143 | { |
| 144 | // Release memory if we have allocated the memory ourselves |
| 145 | if(_pretranspose && !(_weights_manager && _weights_manager->are_weights_managed(_b))) |
| 146 | { |
| 147 | delete _pretranspose; |
| 148 | } |
| 149 | } |
| 150 | |
Georgios Pinitas | 3dbfd23 | 2019-01-30 17:17:16 +0000 | [diff] [blame] | 151 | /** Initialise the functions's input and output. |
| 152 | * |
Michalis Spyrou | 1a569a3 | 2019-09-10 17:20:34 +0100 | [diff] [blame] | 153 | * @param[in] a Input tensor containing the Matrix A. |
| 154 | * @param[in] b Input tensor containing the Matrix B. |
| 155 | * @param[in] c Input tensor containing the Matrix C. |
| 156 | * @param[out] d Output tensor to store the result of matrix multiplication. |
| 157 | * @param[in] args Matrix multiplication information. |
| 158 | * @param[in] gemm_info GEMM meta-data |
| 159 | * @param[in] memory_group Memory group to be used by the function. |
| 160 | * @param[in] weights_manager Weights manager to be used by the function. |
| 161 | * @param[in] os Output stage meta-data. |
Georgios Pinitas | 3dbfd23 | 2019-01-30 17:17:16 +0000 | [diff] [blame] | 162 | */ |
Georgios Pinitas | cfa2bba | 2019-06-27 17:00:52 +0100 | [diff] [blame] | 163 | void configure(const ITensor *a, const ITensor *b, const ITensor *c, ITensor *d, |
Georgios Pinitas | 48b3ef8 | 2019-10-14 19:03:09 +0100 | [diff] [blame] | 164 | arm_gemm::GemmArgs args, const GEMMInfo &gemm_info, |
Michalis Spyrou | 1a569a3 | 2019-09-10 17:20:34 +0100 | [diff] [blame] | 165 | MemoryGroup &memory_group, IWeightsManager *weights_manager, const OutputStage &os = {}); |
Georgios Pinitas | 3dbfd23 | 2019-01-30 17:17:16 +0000 | [diff] [blame] | 166 | |
Michalis Spyrou | 71ac903 | 2019-11-14 14:31:44 +0000 | [diff] [blame] | 167 | /** Set requantization shifts to be used |
| 168 | * |
| 169 | * @param[in] shifts Requantization shifts |
| 170 | * |
| 171 | * @return Pointer to the shift data |
| 172 | */ |
| 173 | /** Set requantization data to be used |
| 174 | * |
| 175 | * |
| 176 | * @param shifts Requantization shifts |
| 177 | * @param multipliers Requantization multipliers |
| 178 | * |
| 179 | * @return A tuple with the pointers to the shift and multiplier data respectively |
| 180 | */ |
| 181 | std::tuple<const int32_t *, const int32_t *> set_requantize_data(const std::vector<int32_t> &shifts, |
| 182 | const std::vector<int32_t> &multipliers); |
| 183 | |
Georgios Pinitas | 3dbfd23 | 2019-01-30 17:17:16 +0000 | [diff] [blame] | 184 | // Inherited methods overridden: |
Anthony Barbier | eaefd00 | 2018-07-20 17:49:35 +0100 | [diff] [blame] | 185 | void run() override; |
| 186 | void prepare() override; |
| 187 | bool is_configured() const override; |
Anthony Barbier | c8e84b5 | 2018-07-17 16:48:42 +0100 | [diff] [blame] | 188 | |
Anthony Barbier | eaefd00 | 2018-07-20 17:49:35 +0100 | [diff] [blame] | 189 | private: |
| 190 | /** Allocate a workspace tensor. |
| 191 | * |
| 192 | * @param[in] workspace_size Size to allocate. |
| 193 | * @param[in] memory_group Tensor memory group. |
| 194 | * @param[in] alignment Workspace memory alignment. |
| 195 | */ |
Anthony Barbier | 20394d5 | 2018-08-02 11:29:09 +0100 | [diff] [blame] | 196 | void allocate_workspace(size_t workspace_size, MemoryGroup &memory_group, size_t alignment); |
Anthony Barbier | c8e84b5 | 2018-07-17 16:48:42 +0100 | [diff] [blame] | 197 | |
Anthony Barbier | eaefd00 | 2018-07-20 17:49:35 +0100 | [diff] [blame] | 198 | /** Assembly Gemm kernel */ |
Michalis Spyrou | 1a569a3 | 2019-09-10 17:20:34 +0100 | [diff] [blame] | 199 | std::shared_ptr<arm_gemm::GemmCommon<TypeInput, TypeOutput>> _gemm_kernel_asm{ nullptr }; |
Anthony Barbier | eaefd00 | 2018-07-20 17:49:35 +0100 | [diff] [blame] | 200 | /** Optimised NEON kernel */ |
| 201 | std::unique_ptr<INEKernel> _optimised_kernel{ nullptr }; |
| 202 | /** Input A */ |
| 203 | const ITensor *_a |
Anthony Barbier | c8e84b5 | 2018-07-17 16:48:42 +0100 | [diff] [blame] | 204 | { |
Anthony Barbier | eaefd00 | 2018-07-20 17:49:35 +0100 | [diff] [blame] | 205 | nullptr |
| 206 | }; |
| 207 | /** Input B */ |
| 208 | const ITensor *_b |
| 209 | { |
| 210 | nullptr |
| 211 | }; |
Georgios Pinitas | cfa2bba | 2019-06-27 17:00:52 +0100 | [diff] [blame] | 212 | const ITensor *_c |
| 213 | { |
| 214 | nullptr |
| 215 | }; |
Anthony Barbier | eaefd00 | 2018-07-20 17:49:35 +0100 | [diff] [blame] | 216 | /** Output */ |
| 217 | ITensor *_d{ nullptr }; |
| 218 | /** GEMM workspace */ |
| 219 | Tensor _workspace{}; |
| 220 | /** Pre-transpose tensor */ |
Michalis Spyrou | 1a569a3 | 2019-09-10 17:20:34 +0100 | [diff] [blame] | 221 | ITensor *_pretranspose{ nullptr }; |
Anthony Barbier | eaefd00 | 2018-07-20 17:49:35 +0100 | [diff] [blame] | 222 | /** Prepared flag */ |
| 223 | bool _is_prepared{ false }; |
Georgios Pinitas | 37d080f | 2019-06-21 18:43:12 +0100 | [diff] [blame] | 224 | /** GEMM meta-data */ |
| 225 | GEMMInfo _gemm_info{}; |
Michalis Spyrou | 1a569a3 | 2019-09-10 17:20:34 +0100 | [diff] [blame] | 226 | /** Weights manager */ |
| 227 | IWeightsManager *_weights_manager{ nullptr }; |
| 228 | /** Weights transform object */ |
| 229 | FallbackTransform<TypeInput, TypeOutput> _weights_transform{}; |
Georgios Pinitas | 77d4252 | 2019-11-05 13:35:47 +0000 | [diff] [blame] | 230 | /** GEMM kernel description */ |
| 231 | arm_gemm::KernelDescription _kernel_info{}; |
Michalis Spyrou | 71ac903 | 2019-11-14 14:31:44 +0000 | [diff] [blame] | 232 | /** Per channel quantization shifts */ |
| 233 | std::vector<int32_t> _shifts{}; |
| 234 | /** Per channel quantization multipliers */ |
| 235 | std::vector<int32_t> _multipliers{}; |
Anthony Barbier | eaefd00 | 2018-07-20 17:49:35 +0100 | [diff] [blame] | 236 | }; |
Anthony Barbier | 71d9b57 | 2018-07-06 17:05:59 +0100 | [diff] [blame] | 237 | |
Georgios Pinitas | cfa2bba | 2019-06-27 17:00:52 +0100 | [diff] [blame] | 238 | template <typename TypeInput, typename TypeOutput, class OutputStage> |
Michalis Spyrou | 71ac903 | 2019-11-14 14:31:44 +0000 | [diff] [blame] | 239 | std::tuple<const int32_t *, const int32_t *> Fallback<TypeInput, TypeOutput, OutputStage>::set_requantize_data(const std::vector<int32_t> &shifts, |
| 240 | const std::vector<int32_t> &multipliers) |
| 241 | { |
| 242 | _multipliers = multipliers; |
| 243 | _shifts = shifts; |
Georgios Pinitas | 47fd61f | 2020-01-29 12:02:20 +0000 | [diff] [blame] | 244 | std::transform(_shifts.begin(), _shifts.end(), _shifts.begin(), std::negate<int32_t>()); |
Michalis Spyrou | 71ac903 | 2019-11-14 14:31:44 +0000 | [diff] [blame] | 245 | return std::make_tuple(_shifts.data(), _multipliers.data()); |
| 246 | } |
| 247 | |
| 248 | template <typename TypeInput, typename TypeOutput, class OutputStage> |
Georgios Pinitas | cfa2bba | 2019-06-27 17:00:52 +0100 | [diff] [blame] | 249 | void Fallback<TypeInput, TypeOutput, OutputStage>::configure(const ITensor *a, const ITensor *b, const ITensor *c, ITensor *d, |
Georgios Pinitas | 48b3ef8 | 2019-10-14 19:03:09 +0100 | [diff] [blame] | 250 | arm_gemm::GemmArgs args, const GEMMInfo &gemm_info, |
Michalis Spyrou | 1a569a3 | 2019-09-10 17:20:34 +0100 | [diff] [blame] | 251 | MemoryGroup &memory_group, IWeightsManager *weights_manager, const OutputStage &os) |
Anthony Barbier | 71d9b57 | 2018-07-06 17:05:59 +0100 | [diff] [blame] | 252 | { |
Georgios Pinitas | 77d4252 | 2019-11-05 13:35:47 +0000 | [diff] [blame] | 253 | arm_gemm::GemmConfig gemm_cfg; |
| 254 | _kernel_info = arm_gemm::get_gemm_method<TypeInput, TypeOutput, OutputStage>(args, os); |
| 255 | _weights_manager = weights_manager; |
| 256 | if(_kernel_info.method != arm_gemm::GemmMethod::GEMV_BATCHED) |
Georgios Pinitas | 3dbfd23 | 2019-01-30 17:17:16 +0000 | [diff] [blame] | 257 | { |
Georgios Pinitas | 77d4252 | 2019-11-05 13:35:47 +0000 | [diff] [blame] | 258 | gemm_cfg.filter = _kernel_info.name; |
Georgios Pinitas | 3dbfd23 | 2019-01-30 17:17:16 +0000 | [diff] [blame] | 259 | args._cfg = &gemm_cfg; |
| 260 | } |
Georgios Pinitas | cfa2bba | 2019-06-27 17:00:52 +0100 | [diff] [blame] | 261 | _gemm_kernel_asm = arm_gemm::gemm<TypeInput, TypeOutput, OutputStage>(args, os); |
Anthony Barbier | 71d9b57 | 2018-07-06 17:05:59 +0100 | [diff] [blame] | 262 | if(_gemm_kernel_asm == nullptr) |
| 263 | { |
| 264 | //configuration not supported: Leave function unconfigured: |
| 265 | return; |
| 266 | } |
| 267 | |
| 268 | // arm_compute wrapper for the Gemm object (see above) |
| 269 | std::unique_ptr<NEGEMMAssemblyWrapperKernel<TypeInput, TypeOutput>> acl_gemm_wrapper = support::cpp14::make_unique<NEGEMMAssemblyWrapperKernel<TypeInput, TypeOutput>>(); |
| 270 | ARM_COMPUTE_ERROR_ON(acl_gemm_wrapper == nullptr); |
Georgios Pinitas | 3dbfd23 | 2019-01-30 17:17:16 +0000 | [diff] [blame] | 271 | acl_gemm_wrapper->configure(_gemm_kernel_asm.get(), gemm_cfg.filter); |
Anthony Barbier | 71d9b57 | 2018-07-06 17:05:59 +0100 | [diff] [blame] | 272 | const size_t workspace_size = _gemm_kernel_asm->get_working_size(); |
| 273 | if(workspace_size > 0) |
| 274 | { |
| 275 | // Allocate workspace |
| 276 | const unsigned int alignment = 4096; |
Anthony Barbier | 20394d5 | 2018-08-02 11:29:09 +0100 | [diff] [blame] | 277 | allocate_workspace(workspace_size, memory_group, alignment); |
Anthony Barbier | 71d9b57 | 2018-07-06 17:05:59 +0100 | [diff] [blame] | 278 | } |
| 279 | |
| 280 | //if we disable this code below in brackets then ConvLayer deadlocks when threads > 1 and |
| 281 | //the shapes are In=1x1x1024 Weights=1x1x1024x1001 Biases=1001 Out=1x1x1001 |
| 282 | { |
Anthony Barbier | c8e84b5 | 2018-07-17 16:48:42 +0100 | [diff] [blame] | 283 | const int window_size = _gemm_kernel_asm->get_window_size(); |
| 284 | if(window_size < args._maxthreads) |
Anthony Barbier | 71d9b57 | 2018-07-06 17:05:59 +0100 | [diff] [blame] | 285 | { |
Anthony Barbier | c8e84b5 | 2018-07-17 16:48:42 +0100 | [diff] [blame] | 286 | _gemm_kernel_asm->set_nthreads(window_size); |
Anthony Barbier | 71d9b57 | 2018-07-06 17:05:59 +0100 | [diff] [blame] | 287 | } |
| 288 | } |
| 289 | |
| 290 | _optimised_kernel = std::move(acl_gemm_wrapper); |
| 291 | _a = a; |
| 292 | _b = b; |
Georgios Pinitas | cfa2bba | 2019-06-27 17:00:52 +0100 | [diff] [blame] | 293 | _c = c; |
Anthony Barbier | 71d9b57 | 2018-07-06 17:05:59 +0100 | [diff] [blame] | 294 | _d = d; |
Georgios Pinitas | 37d080f | 2019-06-21 18:43:12 +0100 | [diff] [blame] | 295 | _gemm_info = gemm_info; |
Anthony Barbier | 71d9b57 | 2018-07-06 17:05:59 +0100 | [diff] [blame] | 296 | // Check for pre-transposed support |
| 297 | if(_gemm_kernel_asm->B_pretranspose_required()) |
| 298 | { |
| 299 | // Forcing 128-byte alignment (required by 32-bit kernels) |
| 300 | const unsigned int alignment = 128; |
| 301 | const size_t B_pretranspose_size = _gemm_kernel_asm->get_B_pretransposed_array_size(); |
Michalis Spyrou | 1a569a3 | 2019-09-10 17:20:34 +0100 | [diff] [blame] | 302 | if(weights_manager && _weights_manager->are_weights_managed(b)) |
| 303 | { |
| 304 | _weights_transform.configure(B_pretranspose_size, alignment); |
| 305 | _pretranspose = _weights_manager->acquire(b, &_weights_transform); |
| 306 | } |
| 307 | else |
| 308 | { |
| 309 | _pretranspose = new Tensor(); |
| 310 | static_cast<Tensor *>(_pretranspose)->allocator()->init(TensorInfo(TensorShape{ (B_pretranspose_size + alignment /* FIXME: remove alignment after COMPMID-1088 */) }, 1, DataType::S8), alignment); |
| 311 | } |
Anthony Barbier | 71d9b57 | 2018-07-06 17:05:59 +0100 | [diff] [blame] | 312 | } |
| 313 | } |
| 314 | |
Georgios Pinitas | cfa2bba | 2019-06-27 17:00:52 +0100 | [diff] [blame] | 315 | template <typename TypeInput, typename TypeOutput, class OutputStage> |
| 316 | void Fallback<TypeInput, TypeOutput, OutputStage>::prepare() |
Anthony Barbier | 71d9b57 | 2018-07-06 17:05:59 +0100 | [diff] [blame] | 317 | { |
| 318 | if(!_is_prepared) |
| 319 | { |
Georgios Pinitas | cfa2bba | 2019-06-27 17:00:52 +0100 | [diff] [blame] | 320 | // Setup up matrix bias in the assembly kernel, it's just a pointer to matrix C. |
| 321 | if(_c && _c->info()->data_type() == DataType::S32) |
| 322 | { |
Georgios Pinitas | 48b3ef8 | 2019-10-14 19:03:09 +0100 | [diff] [blame] | 323 | _gemm_kernel_asm->set_quantized_bias(reinterpret_cast<const int32_t *>(_c->buffer() + _c->info()->offset_first_element_in_bytes()), 0); |
Georgios Pinitas | cfa2bba | 2019-06-27 17:00:52 +0100 | [diff] [blame] | 324 | } |
| 325 | |
Anthony Barbier | 71d9b57 | 2018-07-06 17:05:59 +0100 | [diff] [blame] | 326 | // Pretranspose B if required |
| 327 | if(_gemm_kernel_asm->B_pretranspose_required()) |
| 328 | { |
| 329 | const int ldb = _b->info()->strides_in_bytes().y() / sizeof(TypeInput); |
Georgios Pinitas | eb84d6b | 2018-07-27 18:28:10 +0100 | [diff] [blame] | 330 | const auto in1_ptr = reinterpret_cast<const TypeInput *>(_b->buffer() + _b->info()->offset_first_element_in_bytes()); |
Anthony Barbier | 71d9b57 | 2018-07-06 17:05:59 +0100 | [diff] [blame] | 331 | const int multi_stride_b = _b->info()->strides_in_bytes().z() / sizeof(TypeInput); |
| 332 | |
Michalis Spyrou | 1a569a3 | 2019-09-10 17:20:34 +0100 | [diff] [blame] | 333 | if(_weights_manager && _weights_manager->are_weights_managed(_b)) |
| 334 | { |
| 335 | _weights_transform.set_args(ldb, in1_ptr, multi_stride_b, _gemm_kernel_asm); |
| 336 | _weights_manager->run(_b, &_weights_transform); |
| 337 | |
| 338 | // If we didn't run the reshape function, set the pretransposed buffer |
| 339 | if(!_weights_transform.is_reshape_run()) |
| 340 | { |
| 341 | _weights_transform.set_pretranspose(_pretranspose); |
| 342 | } |
| 343 | } |
| 344 | else |
| 345 | { |
| 346 | static_cast<Tensor *>(_pretranspose)->allocator()->allocate(); |
| 347 | ARM_COMPUTE_ERROR_ON(_pretranspose->buffer() == nullptr); |
| 348 | _gemm_kernel_asm->pretranspose_B_array(_pretranspose->buffer(), in1_ptr, ldb, multi_stride_b); |
| 349 | _b->mark_as_unused(); |
| 350 | } |
Anthony Barbier | 71d9b57 | 2018-07-06 17:05:59 +0100 | [diff] [blame] | 351 | } |
| 352 | |
| 353 | _is_prepared = true; |
| 354 | } |
| 355 | } |
| 356 | |
Georgios Pinitas | cfa2bba | 2019-06-27 17:00:52 +0100 | [diff] [blame] | 357 | template <typename TypeInput, typename TypeOutput, class OutputStage> |
| 358 | void Fallback<TypeInput, TypeOutput, OutputStage>::allocate_workspace(size_t workspace_size, MemoryGroup &memory_group, size_t alignment) |
Anthony Barbier | 71d9b57 | 2018-07-06 17:05:59 +0100 | [diff] [blame] | 359 | { |
| 360 | ARM_COMPUTE_ERROR_ON_MSG(workspace_size == 0, "size cannot be 0"); |
| 361 | _workspace.allocator()->init(TensorInfo(TensorShape{ (workspace_size + alignment /* FIXME: remove alignment after COMPMID-1088 */) }, 1, DataType::S8), alignment); |
Anthony Barbier | 20394d5 | 2018-08-02 11:29:09 +0100 | [diff] [blame] | 362 | memory_group.manage(&_workspace); |
Anthony Barbier | 71d9b57 | 2018-07-06 17:05:59 +0100 | [diff] [blame] | 363 | _workspace.allocator()->allocate(); |
| 364 | } |
| 365 | |
Georgios Pinitas | cfa2bba | 2019-06-27 17:00:52 +0100 | [diff] [blame] | 366 | template <typename TypeInput, typename TypeOutput, class OutputStage> |
| 367 | bool Fallback<TypeInput, TypeOutput, OutputStage>::is_configured() const |
Anthony Barbier | 71d9b57 | 2018-07-06 17:05:59 +0100 | [diff] [blame] | 368 | { |
| 369 | return _optimised_kernel != nullptr; |
| 370 | } |
| 371 | |
Georgios Pinitas | cfa2bba | 2019-06-27 17:00:52 +0100 | [diff] [blame] | 372 | template <typename TypeInput, typename TypeOutput, class OutputStage> |
| 373 | void Fallback<TypeInput, TypeOutput, OutputStage>::run() |
Anthony Barbier | 71d9b57 | 2018-07-06 17:05:59 +0100 | [diff] [blame] | 374 | { |
| 375 | const int lda = _a->info()->strides_in_bytes().y() / sizeof(TypeInput); |
Georgios Pinitas | 40ed6d8 | 2018-07-31 17:22:11 +0100 | [diff] [blame] | 376 | int ldb = 0; |
Anthony Barbier | 71d9b57 | 2018-07-06 17:05:59 +0100 | [diff] [blame] | 377 | const int ldd = _d->info()->strides_in_bytes().y() / sizeof(TypeOutput); |
| 378 | |
Georgios Pinitas | 37d080f | 2019-06-21 18:43:12 +0100 | [diff] [blame] | 379 | const size_t a_batch_idx = _gemm_info.reinterpret_input_as_3d() != 0 ? 3 : 2; |
| 380 | const size_t a_multi_idx = a_batch_idx + 1; |
| 381 | const size_t d_batch_idx = _gemm_info.depth_output_gemm3d() != 0 ? 3 : 2; |
| 382 | const size_t d_multi_idx = d_batch_idx + 1; |
Anthony Barbier | 71d9b57 | 2018-07-06 17:05:59 +0100 | [diff] [blame] | 383 | |
Georgios Pinitas | 37d080f | 2019-06-21 18:43:12 +0100 | [diff] [blame] | 384 | const int batch_stride_a = _a->info()->strides_in_bytes()[a_batch_idx] / sizeof(TypeInput); |
| 385 | const int batch_stride_d = _d->info()->strides_in_bytes()[d_batch_idx] / sizeof(TypeOutput); |
Anthony Barbier | 71d9b57 | 2018-07-06 17:05:59 +0100 | [diff] [blame] | 386 | |
Georgios Pinitas | 37d080f | 2019-06-21 18:43:12 +0100 | [diff] [blame] | 387 | const int multi_stride_a = _a->info()->strides_in_bytes()[a_multi_idx] / sizeof(TypeInput); |
Georgios Pinitas | 40ed6d8 | 2018-07-31 17:22:11 +0100 | [diff] [blame] | 388 | int multi_stride_b = 0; |
Georgios Pinitas | 37d080f | 2019-06-21 18:43:12 +0100 | [diff] [blame] | 389 | const int multi_stride_d = _d->info()->strides_in_bytes()[d_multi_idx] / sizeof(TypeOutput); |
Anthony Barbier | 71d9b57 | 2018-07-06 17:05:59 +0100 | [diff] [blame] | 390 | |
Georgios Pinitas | 40ed6d8 | 2018-07-31 17:22:11 +0100 | [diff] [blame] | 391 | const auto in0_ptr = reinterpret_cast<const TypeInput *>(_a->buffer() + _a->info()->offset_first_element_in_bytes()); |
| 392 | const TypeInput *in1_ptr = nullptr; |
| 393 | auto out_ptr = reinterpret_cast<TypeOutput *>(_d->buffer() + _d->info()->offset_first_element_in_bytes()); |
| 394 | |
| 395 | // Check if B is pre-tranposed and de-reference if not |
| 396 | if(!_gemm_kernel_asm->B_is_pretransposed()) |
| 397 | { |
| 398 | ldb = _b->info()->strides_in_bytes().y() / sizeof(TypeInput); |
| 399 | multi_stride_b = _b->info()->strides_in_bytes().z() / sizeof(TypeInput); |
| 400 | in1_ptr = reinterpret_cast<const TypeInput *>(_b->buffer() + _b->info()->offset_first_element_in_bytes()); |
| 401 | } |
Anthony Barbier | 71d9b57 | 2018-07-06 17:05:59 +0100 | [diff] [blame] | 402 | |
| 403 | // Set workspace if needed and reset number of threads as buffer manager gets re-created with max_threads |
| 404 | if(_workspace.buffer() != nullptr) |
| 405 | { |
| 406 | _gemm_kernel_asm->set_working_space(reinterpret_cast<void *>(_workspace.buffer())); |
| 407 | const unsigned int window_size = _gemm_kernel_asm->get_window_size(); |
| 408 | unsigned int num_threads = NEScheduler::get().num_threads(); |
| 409 | if(window_size < num_threads) |
| 410 | { |
| 411 | num_threads = window_size; |
| 412 | _gemm_kernel_asm->set_nthreads(num_threads); |
| 413 | } |
| 414 | } |
| 415 | |
| 416 | // Prepare assembly kernel |
| 417 | prepare(); |
| 418 | |
Georgios Pinitas | 48b3ef8 | 2019-10-14 19:03:09 +0100 | [diff] [blame] | 419 | TypeOutput *bias = nullptr; |
| 420 | // Setup up matrix bias in the assembly kernel, it's just a pointer to matrix C. |
| 421 | if(_c && _c->info()->data_type() != DataType::S32) |
| 422 | { |
| 423 | bias = reinterpret_cast<TypeOutput *>(_c->buffer() + _c->info()->offset_first_element_in_bytes()); |
| 424 | } |
Anthony Barbier | 71d9b57 | 2018-07-06 17:05:59 +0100 | [diff] [blame] | 425 | // Set gemm parameters |
Georgios Pinitas | 48b3ef8 | 2019-10-14 19:03:09 +0100 | [diff] [blame] | 426 | _gemm_kernel_asm->set_arrays(in0_ptr, lda, batch_stride_a, multi_stride_a, |
| 427 | in1_ptr, ldb, multi_stride_b, |
| 428 | out_ptr, ldd, batch_stride_d, multi_stride_d, |
| 429 | bias, 0); |
Anthony Barbier | 71d9b57 | 2018-07-06 17:05:59 +0100 | [diff] [blame] | 430 | |
| 431 | // Schedule assembly kernel |
Georgios Pinitas | 77d4252 | 2019-11-05 13:35:47 +0000 | [diff] [blame] | 432 | IScheduler::Hints scheduling_hint = IScheduler::Hints(Window::DimX); |
Georgios Pinitas | 6011f24 | 2019-11-15 14:26:44 +0000 | [diff] [blame] | 433 | if(_kernel_info.method == arm_gemm::GemmMethod::GEMM_INTERLEAVED && _d->info()->data_type() == DataType::F32) |
Georgios Pinitas | 77d4252 | 2019-11-05 13:35:47 +0000 | [diff] [blame] | 434 | { |
Georgios Pinitas | 6011f24 | 2019-11-15 14:26:44 +0000 | [diff] [blame] | 435 | const int granule_threshold = 200; |
| 436 | scheduling_hint = IScheduler::Hints(Window::DimX, IScheduler::StrategyHint::DYNAMIC, granule_threshold); |
Georgios Pinitas | 77d4252 | 2019-11-05 13:35:47 +0000 | [diff] [blame] | 437 | } |
| 438 | NEScheduler::get().schedule(_optimised_kernel.get(), scheduling_hint); |
Anthony Barbier | 71d9b57 | 2018-07-06 17:05:59 +0100 | [diff] [blame] | 439 | } |
| 440 | |
Anthony Barbier | eaefd00 | 2018-07-20 17:49:35 +0100 | [diff] [blame] | 441 | template <typename TypeInput, typename TypeOutput> |
Georgios Pinitas | 48b3ef8 | 2019-10-14 19:03:09 +0100 | [diff] [blame] | 442 | void create_arm_gemm(std::unique_ptr<NEGEMMAssemblyDispatch::IFallback> &arm_gemm, MemoryGroup &memory_group, |
| 443 | const ITensor *a, const ITensor *b, const ITensor *c, ITensor *d, arm_gemm::Activation activation, const GEMMInfo &gemm_info, |
| 444 | IWeightsManager *weights_manager) |
Anthony Barbier | eaefd00 | 2018-07-20 17:49:35 +0100 | [diff] [blame] | 445 | { |
Georgios Pinitas | 37d080f | 2019-06-21 18:43:12 +0100 | [diff] [blame] | 446 | INEGEMMWrapperKernel::Params p = INEGEMMWrapperKernel::extract_parameters(a, b, d, gemm_info); |
Anthony Barbier | eaefd00 | 2018-07-20 17:49:35 +0100 | [diff] [blame] | 447 | const CPUInfo &ci = NEScheduler::get().cpu_info(); |
| 448 | unsigned int num_threads = NEScheduler::get().num_threads(); |
| 449 | |
Georgios Pinitas | 48b3ef8 | 2019-10-14 19:03:09 +0100 | [diff] [blame] | 450 | arm_gemm::GemmArgs args(&ci, p.M, p.N, p.K, p.batches, p.multis, false, false, activation, num_threads, gemm_info.pretranpose_B()); |
Anthony Barbier | eaefd00 | 2018-07-20 17:49:35 +0100 | [diff] [blame] | 451 | |
Georgios Pinitas | 48b3ef8 | 2019-10-14 19:03:09 +0100 | [diff] [blame] | 452 | // Create arm_gemm fallback |
| 453 | auto fallback = support::cpp14::make_unique<Fallback<TypeInput, TypeOutput>>(); |
| 454 | fallback->configure(a, b, c, d, args, gemm_info, memory_group, weights_manager); |
| 455 | arm_gemm = std::move(fallback); |
Georgios Pinitas | cfa2bba | 2019-06-27 17:00:52 +0100 | [diff] [blame] | 456 | } |
| 457 | |
| 458 | template <typename TypeInput, typename TypeOutput> |
Georgios Pinitas | 48b3ef8 | 2019-10-14 19:03:09 +0100 | [diff] [blame] | 459 | void create_arm_gemm_quant(std::unique_ptr<NEGEMMAssemblyDispatch::IFallback> &arm_gemm, MemoryGroup &memory_group, |
| 460 | const ITensor *a, const ITensor *b, const ITensor *c, ITensor *d, arm_gemm::Activation activation, const GEMMInfo &gemm_info, |
| 461 | IWeightsManager *weights_manager) |
Georgios Pinitas | cfa2bba | 2019-06-27 17:00:52 +0100 | [diff] [blame] | 462 | { |
| 463 | INEGEMMWrapperKernel::Params p = INEGEMMWrapperKernel::extract_parameters(a, b, d, gemm_info); |
| 464 | const CPUInfo &ci = NEScheduler::get().cpu_info(); |
| 465 | unsigned int num_threads = NEScheduler::get().num_threads(); |
| 466 | |
Georgios Pinitas | 48b3ef8 | 2019-10-14 19:03:09 +0100 | [diff] [blame] | 467 | arm_gemm::GemmArgs args(&ci, p.M, p.N, p.K, p.batches, p.multis, false, false, activation, num_threads, gemm_info.pretranpose_B()); |
Georgios Pinitas | cfa2bba | 2019-06-27 17:00:52 +0100 | [diff] [blame] | 468 | |
Michalis Spyrou | 71ac903 | 2019-11-14 14:31:44 +0000 | [diff] [blame] | 469 | // Create arm_gemm fallback |
| 470 | auto fallback = support::cpp14::make_unique<Fallback<TypeInput, TypeOutput, arm_gemm::Requantize32>>(); |
| 471 | |
Georgios Pinitas | cfa2bba | 2019-06-27 17:00:52 +0100 | [diff] [blame] | 472 | // Configure requantization info |
| 473 | const int32_t a_offset = -a->info()->quantization_info().uniform().offset; |
| 474 | const int32_t b_offset = -b->info()->quantization_info().uniform().offset; |
| 475 | const GEMMLowpOutputStageInfo os_info = gemm_info.gemmlowp_output_stage(); |
| 476 | |
Michalis Spyrou | 71ac903 | 2019-11-14 14:31:44 +0000 | [diff] [blame] | 477 | arm_gemm::Requantize32 gemm_requant_info{}; |
| 478 | if(os_info.gemmlowp_shifts.size() > 1) |
| 479 | { |
| 480 | const auto requantize_data = fallback->set_requantize_data(os_info.gemmlowp_shifts, os_info.gemmlowp_multipliers); |
| 481 | gemm_requant_info = arm_gemm::Requantize32(nullptr, 0, |
| 482 | a_offset, b_offset, os_info.gemmlowp_offset, |
| 483 | std::get<0>(requantize_data), std::get<1>(requantize_data), |
| 484 | os_info.gemmlowp_min_bound, os_info.gemmlowp_max_bound); |
| 485 | } |
| 486 | else |
| 487 | { |
| 488 | gemm_requant_info = arm_gemm::Requantize32(nullptr, 0, |
| 489 | a_offset, b_offset, os_info.gemmlowp_offset, |
| 490 | -os_info.gemmlowp_shift, os_info.gemmlowp_multiplier, |
| 491 | os_info.gemmlowp_min_bound, os_info.gemmlowp_max_bound); |
| 492 | } |
Georgios Pinitas | cfa2bba | 2019-06-27 17:00:52 +0100 | [diff] [blame] | 493 | |
Michalis Spyrou | 71ac903 | 2019-11-14 14:31:44 +0000 | [diff] [blame] | 494 | // Configure fallback |
Georgios Pinitas | 48b3ef8 | 2019-10-14 19:03:09 +0100 | [diff] [blame] | 495 | fallback->configure(a, b, c, d, args, gemm_info, memory_group, weights_manager, gemm_requant_info); |
| 496 | arm_gemm = std::move(fallback); |
Anthony Barbier | eaefd00 | 2018-07-20 17:49:35 +0100 | [diff] [blame] | 497 | } |
| 498 | |
| 499 | } //namespace |
| 500 | |
Michalis Spyrou | 1a569a3 | 2019-09-10 17:20:34 +0100 | [diff] [blame] | 501 | NEGEMMAssemblyDispatch::NEGEMMAssemblyDispatch(std::shared_ptr<IMemoryManager> memory_manager, IWeightsManager *weights_manager) |
Georgios Pinitas | 48b3ef8 | 2019-10-14 19:03:09 +0100 | [diff] [blame] | 502 | : _arm_gemm(nullptr), _memory_group(std::move(memory_manager)), _weights_manager(weights_manager) |
Anthony Barbier | eaefd00 | 2018-07-20 17:49:35 +0100 | [diff] [blame] | 503 | { |
| 504 | } |
| 505 | |
Georgios Pinitas | 48b3ef8 | 2019-10-14 19:03:09 +0100 | [diff] [blame] | 506 | Status NEGEMMAssemblyDispatch::validate(const ITensorInfo *a, const ITensorInfo *b, const ITensorInfo *c, const ITensorInfo *d, const GEMMInfo &gemm_info) |
Anthony Barbier | eaefd00 | 2018-07-20 17:49:35 +0100 | [diff] [blame] | 507 | { |
Georgios Pinitas | c7b183a | 2020-03-06 18:12:09 +0000 | [diff] [blame^] | 508 | ARM_COMPUTE_UNUSED(gemm_info, c); |
Anthony Barbier | eaefd00 | 2018-07-20 17:49:35 +0100 | [diff] [blame] | 509 | ARM_COMPUTE_RETURN_ERROR_ON_NULLPTR(a, b, d); |
| 510 | ARM_COMPUTE_RETURN_ERROR_ON_CPU_F16_UNSUPPORTED(a); |
Georgios Pinitas | c7b183a | 2020-03-06 18:12:09 +0000 | [diff] [blame^] | 511 | ARM_COMPUTE_RETURN_ERROR_ON_CPU_BF16_UNSUPPORTED(a); |
Anthony Barbier | eaefd00 | 2018-07-20 17:49:35 +0100 | [diff] [blame] | 512 | #ifndef __aarch64__ |
Michele Di Giorgio | 5255672 | 2019-12-23 16:35:12 +0000 | [diff] [blame] | 513 | ARM_COMPUTE_RETURN_ERROR_ON_MSG(a->element_size() == 1, "8bit integer types only supported for aarch64"); |
Anthony Barbier | eaefd00 | 2018-07-20 17:49:35 +0100 | [diff] [blame] | 514 | #endif /* __aarch64__ */ |
Georgios Pinitas | dbdea0d | 2019-10-16 19:21:40 +0100 | [diff] [blame] | 515 | ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(a, 1, DataType::U8, DataType::QASYMM8, DataType::QASYMM8_SIGNED, DataType::S8, |
Georgios Pinitas | c7b183a | 2020-03-06 18:12:09 +0000 | [diff] [blame^] | 516 | DataType::BFLOAT16, DataType::F16, DataType::F32); |
Georgios Pinitas | dbdea0d | 2019-10-16 19:21:40 +0100 | [diff] [blame] | 517 | ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(b, 1, DataType::U8, DataType::QASYMM8, DataType::QASYMM8_SIGNED, DataType::QSYMM8_PER_CHANNEL, DataType::S8, |
Georgios Pinitas | c7b183a | 2020-03-06 18:12:09 +0000 | [diff] [blame^] | 518 | DataType::BFLOAT16, DataType::F16, DataType::F32); |
Georgios Pinitas | dbdea0d | 2019-10-16 19:21:40 +0100 | [diff] [blame] | 519 | if(is_data_type_quantized_per_channel(b->data_type())) |
| 520 | { |
| 521 | ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(a, 1, DataType::QASYMM8_SIGNED, DataType::S8); |
| 522 | } |
| 523 | else |
| 524 | { |
| 525 | ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(a, b); |
| 526 | } |
Anthony Barbier | eaefd00 | 2018-07-20 17:49:35 +0100 | [diff] [blame] | 527 | ARM_COMPUTE_RETURN_ERROR_ON_MSG(a->data_type() == DataType::F32 && d->data_type() != DataType::F32, "Only F32 output supported for F32 input"); |
| 528 | ARM_COMPUTE_RETURN_ERROR_ON_MSG(a->data_type() == DataType::F16 && d->data_type() != DataType::F16, "Only F16 output supported for F16 input"); |
Georgios Pinitas | c7b183a | 2020-03-06 18:12:09 +0000 | [diff] [blame^] | 529 | ARM_COMPUTE_RETURN_ERROR_ON_MSG(a->data_type() == DataType::BFLOAT16 && d->data_type() != DataType::F32, "Only F32 output supported for BFLOAT16 input"); |
Anthony Barbier | 9036749 | 2018-08-01 13:56:08 +0100 | [diff] [blame] | 530 | ARM_COMPUTE_RETURN_ERROR_ON_MSG(a->data_type() == DataType::U8 && d->data_type() != DataType::U32, "Only U32 output supported for U8 input"); |
Anthony Barbier | eaefd00 | 2018-07-20 17:49:35 +0100 | [diff] [blame] | 531 | ARM_COMPUTE_RETURN_ERROR_ON_MSG(a->data_type() == DataType::S8 && d->data_type() != DataType::S32, "Only S32 output supported for S8 input"); |
Georgios Pinitas | cfa2bba | 2019-06-27 17:00:52 +0100 | [diff] [blame] | 532 | ARM_COMPUTE_RETURN_ERROR_ON_MSG(a->data_type() == DataType::QASYMM8 && d->data_type() != DataType::QASYMM8, "Only QASYMM8 output supported for QASYMM8 input"); |
Anthony Barbier | eaefd00 | 2018-07-20 17:49:35 +0100 | [diff] [blame] | 533 | return Status{}; |
| 534 | } |
| 535 | |
Georgios Pinitas | 48b3ef8 | 2019-10-14 19:03:09 +0100 | [diff] [blame] | 536 | bool NEGEMMAssemblyDispatch::is_activation_supported(const ActivationLayerInfo &activation) |
| 537 | { |
| 538 | arm_gemm::Activation act = map_to_arm_gemm_activation(activation); |
| 539 | return act.type != arm_gemm::Activation::Type::None; |
| 540 | } |
| 541 | |
| 542 | void NEGEMMAssemblyDispatch::configure(const ITensor *a, const ITensor *b, const ITensor *c, ITensor *d, const GEMMInfo &gemm_info) |
Anthony Barbier | eaefd00 | 2018-07-20 17:49:35 +0100 | [diff] [blame] | 543 | { |
Georgios Pinitas | cfa2bba | 2019-06-27 17:00:52 +0100 | [diff] [blame] | 544 | ARM_COMPUTE_ERROR_ON_NULLPTR(a, b, d); |
Georgios Pinitas | 48b3ef8 | 2019-10-14 19:03:09 +0100 | [diff] [blame] | 545 | arm_gemm::Activation act = map_to_arm_gemm_activation(gemm_info.activation_info()); |
Anthony Barbier | eaefd00 | 2018-07-20 17:49:35 +0100 | [diff] [blame] | 546 | |
| 547 | //If we don't support a combination of data types, silently return: it is the caller's responsibility to check if configure() was successful via is_configured() |
Georgios Pinitas | 48b3ef8 | 2019-10-14 19:03:09 +0100 | [diff] [blame] | 548 | if(!NEGEMMAssemblyDispatch::validate(a->info(), b->info(), c != nullptr ? c->info() : nullptr, d->info(), gemm_info)) |
Anthony Barbier | eaefd00 | 2018-07-20 17:49:35 +0100 | [diff] [blame] | 549 | { |
| 550 | return; |
| 551 | } |
| 552 | |
| 553 | switch(a->info()->data_type()) |
| 554 | { |
| 555 | case DataType::F32: |
Georgios Pinitas | 48b3ef8 | 2019-10-14 19:03:09 +0100 | [diff] [blame] | 556 | create_arm_gemm<float, float>(_arm_gemm, _memory_group, a, b, c, d, act, gemm_info, _weights_manager); |
Anthony Barbier | eaefd00 | 2018-07-20 17:49:35 +0100 | [diff] [blame] | 557 | break; |
| 558 | #ifdef __aarch64__ |
| 559 | case DataType::U8: |
| 560 | case DataType::QASYMM8: |
Georgios Pinitas | cfa2bba | 2019-06-27 17:00:52 +0100 | [diff] [blame] | 561 | if(d->info()->data_type() == DataType::S32) |
| 562 | { |
Georgios Pinitas | 48b3ef8 | 2019-10-14 19:03:09 +0100 | [diff] [blame] | 563 | create_arm_gemm<uint8_t, uint32_t>(_arm_gemm, _memory_group, a, b, c, d, act, gemm_info, _weights_manager); |
Georgios Pinitas | cfa2bba | 2019-06-27 17:00:52 +0100 | [diff] [blame] | 564 | } |
| 565 | else |
| 566 | { |
Georgios Pinitas | 48b3ef8 | 2019-10-14 19:03:09 +0100 | [diff] [blame] | 567 | create_arm_gemm_quant<uint8_t, uint8_t>(_arm_gemm, _memory_group, a, b, c, d, act, gemm_info, _weights_manager); |
Georgios Pinitas | cfa2bba | 2019-06-27 17:00:52 +0100 | [diff] [blame] | 568 | } |
Anthony Barbier | eaefd00 | 2018-07-20 17:49:35 +0100 | [diff] [blame] | 569 | break; |
| 570 | case DataType::S8: |
Georgios Pinitas | dbdea0d | 2019-10-16 19:21:40 +0100 | [diff] [blame] | 571 | case DataType::QASYMM8_SIGNED: |
Michalis Spyrou | 71ac903 | 2019-11-14 14:31:44 +0000 | [diff] [blame] | 572 | if(d->info()->data_type() == DataType::S32) |
| 573 | { |
| 574 | create_arm_gemm<int8_t, int32_t>(_arm_gemm, _memory_group, a, b, c, d, act, gemm_info, _weights_manager); |
| 575 | } |
| 576 | else |
| 577 | { |
| 578 | create_arm_gemm_quant<int8_t, int8_t>(_arm_gemm, _memory_group, a, b, c, d, act, gemm_info, _weights_manager); |
| 579 | } |
Anthony Barbier | eaefd00 | 2018-07-20 17:49:35 +0100 | [diff] [blame] | 580 | break; |
| 581 | #endif /* __aarch64__ */ |
Georgios Pinitas | c7b183a | 2020-03-06 18:12:09 +0000 | [diff] [blame^] | 582 | #if defined(__ARM_FEATURE_BF16_VECTOR_ARITHMETIC) || defined(ARM_COMPUTE_FORCE_BF16) |
| 583 | case DataType::BFLOAT16: |
| 584 | create_arm_gemm<bfloat16, float>(_arm_gemm, _memory_group, a, b, c, d, act, gemm_info, _weights_manager); |
| 585 | break; |
| 586 | #endif /* defined(__ARM_FEATURE_BF16_VECTOR_ARITHMETIC) || defined(ARM_COMPUTE_FORCE_BF16) */ |
Anthony Barbier | eaefd00 | 2018-07-20 17:49:35 +0100 | [diff] [blame] | 587 | #ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC |
| 588 | case DataType::F16: |
Georgios Pinitas | 48b3ef8 | 2019-10-14 19:03:09 +0100 | [diff] [blame] | 589 | create_arm_gemm<float16_t, float16_t>(_arm_gemm, _memory_group, a, b, c, d, act, gemm_info, _weights_manager); |
Anthony Barbier | eaefd00 | 2018-07-20 17:49:35 +0100 | [diff] [blame] | 590 | break; |
| 591 | #endif /* __ARM_FEATURE_FP16_VECTOR_ARITHMETIC */ |
| 592 | default: |
| 593 | break; |
| 594 | } |
| 595 | } |
| 596 | |
| 597 | void NEGEMMAssemblyDispatch::prepare() |
| 598 | { |
Georgios Pinitas | 48b3ef8 | 2019-10-14 19:03:09 +0100 | [diff] [blame] | 599 | ARM_COMPUTE_ERROR_ON(_arm_gemm == nullptr); |
| 600 | _arm_gemm->prepare(); |
Anthony Barbier | eaefd00 | 2018-07-20 17:49:35 +0100 | [diff] [blame] | 601 | } |
| 602 | |
| 603 | bool NEGEMMAssemblyDispatch::is_configured() const |
| 604 | { |
Georgios Pinitas | 48b3ef8 | 2019-10-14 19:03:09 +0100 | [diff] [blame] | 605 | return _arm_gemm != nullptr && _arm_gemm->is_configured(); |
Anthony Barbier | eaefd00 | 2018-07-20 17:49:35 +0100 | [diff] [blame] | 606 | } |
| 607 | |
| 608 | void NEGEMMAssemblyDispatch::run() |
| 609 | { |
Georgios Pinitas | da953f2 | 2019-04-02 17:27:03 +0100 | [diff] [blame] | 610 | MemoryGroupResourceScope scope_mg(_memory_group); |
Georgios Pinitas | 48b3ef8 | 2019-10-14 19:03:09 +0100 | [diff] [blame] | 611 | |
| 612 | ARM_COMPUTE_ERROR_ON(_arm_gemm == nullptr); |
| 613 | _arm_gemm->run(); |
Anthony Barbier | eaefd00 | 2018-07-20 17:49:35 +0100 | [diff] [blame] | 614 | } |
Anthony Barbier | 71d9b57 | 2018-07-06 17:05:59 +0100 | [diff] [blame] | 615 | } //namespace arm_compute |