Anthony Barbier | 71d9b57 | 2018-07-06 17:05:59 +0100 | [diff] [blame] | 1 | /* |
Mohammed Suhail Munshi | a1b1e41 | 2023-03-23 22:21:31 +0000 | [diff] [blame] | 2 | * Copyright (c) 2018-2023 Arm Limited. |
Anthony Barbier | 71d9b57 | 2018-07-06 17:05:59 +0100 | [diff] [blame] | 3 | * |
| 4 | * SPDX-License-Identifier: MIT |
| 5 | * |
| 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy |
| 7 | * of this software and associated documentation files (the "Software"), to |
| 8 | * deal in the Software without restriction, including without limitation the |
| 9 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or |
| 10 | * sell copies of the Software, and to permit persons to whom the Software is |
| 11 | * furnished to do so, subject to the following conditions: |
| 12 | * |
| 13 | * The above copyright notice and this permission notice shall be included in all |
| 14 | * copies or substantial portions of the Software. |
| 15 | * |
| 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
| 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
| 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE |
| 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
| 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
| 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
| 22 | * SOFTWARE. |
| 23 | */ |
Sang-Hoon Park | d89e2fa | 2021-05-17 17:04:50 +0100 | [diff] [blame] | 24 | #ifndef ARM_COMPUTE_CPU_INTERNAL_CPU_GEMM_ASSEMBLY_DISPATCH_H |
| 25 | #define ARM_COMPUTE_CPU_INTERNAL_CPU_GEMM_ASSEMBLY_DISPATCH_H |
Anthony Barbier | 71d9b57 | 2018-07-06 17:05:59 +0100 | [diff] [blame] | 26 | |
Matthew Bentham | f1aeab9 | 2023-05-30 13:35:34 +0000 | [diff] [blame^] | 27 | #include "arm_compute/core/ActivationLayerInfo.h" |
Sang-Hoon Park | d89e2fa | 2021-05-17 17:04:50 +0100 | [diff] [blame] | 28 | #include "src/core/common/Macros.h" |
Georgios Pinitas | 7891a73 | 2021-08-20 21:39:25 +0100 | [diff] [blame] | 29 | #include "src/cpu/ICpuOperator.h" |
Anthony Barbier | 71d9b57 | 2018-07-06 17:05:59 +0100 | [diff] [blame] | 30 | |
Anthony Barbier | 71d9b57 | 2018-07-06 17:05:59 +0100 | [diff] [blame] | 31 | namespace arm_compute |
| 32 | { |
Sang-Hoon Park | 4f7693d | 2021-05-12 13:59:10 +0100 | [diff] [blame] | 33 | namespace cpu |
| 34 | { |
Georgios Pinitas | c0b6f76 | 2020-11-02 01:37:17 +0000 | [diff] [blame] | 35 | /* Convolution method supported by the assembly gemm interface */ |
| 36 | enum class AsmConvMethod |
| 37 | { |
| 38 | Im2Col, |
| 39 | Indirect, |
| 40 | Conv |
| 41 | }; |
| 42 | |
| 43 | struct AsmGemmInfo |
| 44 | { |
Ramy Elgammal | 9178002 | 2022-07-20 14:57:37 +0100 | [diff] [blame] | 45 | AsmConvMethod method{ AsmConvMethod::Im2Col }; |
| 46 | PadStrideInfo ps_info{}; |
| 47 | ActivationLayerInfo activation_info{}; |
| 48 | GEMMLowpOutputStageInfo output_stage{}; |
| 49 | bool negated_offsets{ true }; |
| 50 | bool reinterpret_input_as_3d{ false }; |
| 51 | bool depth_output_gemm3d{ false }; |
| 52 | int64_t padding_top{ 0 }; |
| 53 | int64_t padding_left{ 0 }; |
| 54 | float padding_value{ 0.f }; |
| 55 | bool fast_mode{ false }; |
| 56 | bool fixed_format{ false }; |
| 57 | arm_compute::WeightFormat weight_format{ arm_compute::WeightFormat::UNSPECIFIED }; |
Mohammed Suhail Munshi | 4b5f6ef | 2022-10-21 11:15:54 +0100 | [diff] [blame] | 58 | bool reshape_b_only_on_first_run{ true }; |
Georgios Pinitas | c0b6f76 | 2020-11-02 01:37:17 +0000 | [diff] [blame] | 59 | }; |
| 60 | |
Anthony Barbier | 71d9b57 | 2018-07-06 17:05:59 +0100 | [diff] [blame] | 61 | /** Assembly kernel glue */ |
Sang-Hoon Park | d89e2fa | 2021-05-17 17:04:50 +0100 | [diff] [blame] | 62 | class CpuGemmAssemblyDispatch : public ICpuOperator |
Anthony Barbier | 71d9b57 | 2018-07-06 17:05:59 +0100 | [diff] [blame] | 63 | { |
| 64 | public: |
Michalis Spyrou | 1a569a3 | 2019-09-10 17:20:34 +0100 | [diff] [blame] | 65 | /** Constructor */ |
Michele Di Giorgio | d7316eb | 2021-06-16 11:14:41 +0100 | [diff] [blame] | 66 | CpuGemmAssemblyDispatch(); |
Sang-Hoon Park | 4f7693d | 2021-05-12 13:59:10 +0100 | [diff] [blame] | 67 | /** Defautl destructor */ |
| 68 | ~CpuGemmAssemblyDispatch() = default; |
Anthony Barbier | eaefd00 | 2018-07-20 17:49:35 +0100 | [diff] [blame] | 69 | |
Sang-Hoon Park | d89e2fa | 2021-05-17 17:04:50 +0100 | [diff] [blame] | 70 | ARM_COMPUTE_DISALLOW_COPY_ALLOW_MOVE(CpuGemmAssemblyDispatch); |
| 71 | |
Anthony Barbier | eaefd00 | 2018-07-20 17:49:35 +0100 | [diff] [blame] | 72 | class IFallback |
| 73 | { |
| 74 | public: |
Ramy Elgammal | 9178002 | 2022-07-20 14:57:37 +0100 | [diff] [blame] | 75 | virtual void run(ITensorPack &tensors) = 0; |
| 76 | virtual void prepare(ITensorPack &tensors) = 0; |
| 77 | virtual experimental::MemoryRequirements workspace() const = 0; |
| 78 | virtual bool is_configured() const = 0; |
| 79 | virtual bool isVarWeightsKernel() const = 0; |
| 80 | virtual ~IFallback() = default; |
Anthony Barbier | eaefd00 | 2018-07-20 17:49:35 +0100 | [diff] [blame] | 81 | }; |
Anthony Barbier | 71d9b57 | 2018-07-06 17:05:59 +0100 | [diff] [blame] | 82 | |
Anthony Barbier | 71d9b57 | 2018-07-06 17:05:59 +0100 | [diff] [blame] | 83 | public: |
Michele Di Giorgio | 57f30a9 | 2020-09-08 14:03:51 +0100 | [diff] [blame] | 84 | /** If supported create a Compute Library function else fallback to the arm_gemm function. |
Anthony Barbier | c8e84b5 | 2018-07-17 16:48:42 +0100 | [diff] [blame] | 85 | * |
Mohammed Suhail Munshi | a1b1e41 | 2023-03-23 22:21:31 +0000 | [diff] [blame] | 86 | * @note Configuring "batches" |
| 87 | * The shapes of @p a @p b and @p d are arranged as follows: |
| 88 | * Lowest dimension <-> Highest dimension |
| 89 | * a: [K, M, Batch, Multi] |
| 90 | * b: [N, K, Multi] |
| 91 | * d: [N, M, Batch, Multi] |
| 92 | * |
| 93 | * The "Batch" refers to where "Batch" number of MxK slices of tensor a multiplies with a single KxN slice of b |
| 94 | * The "Multi" refers to where "Multi" number of individual multiplication of a with b |
| 95 | * |
| 96 | * E.g. the following are some example input shape configurations |
| 97 | * |
| 98 | * (1) Normal 2D gemm |
| 99 | * a: [K=3, M=4] |
| 100 | * b: [N=5, K=3] |
| 101 | * d: [N=5, M=4] |
| 102 | * |
| 103 | * (2) Batches of a sharing b (e.g. gemm-based batched convolution where b is the shared ) |
| 104 | * a: [K=3, M=4, Batch=9] |
| 105 | * b: [N=5, K=3] |
| 106 | * d: [N=5, M=4, Batch=9] |
| 107 | * |
| 108 | * (3) "Batches" of independent gemm (e.g. batched matmul) |
| 109 | * a: [K=3, M=4, Batch=1, Multi=7] |
| 110 | * b: [N=5, K=3, Multi=7] |
| 111 | * d: [N=5, M=4, Batch=1, Multi=7] |
| 112 | * |
| 113 | * (4) "Batches" of independent gemm where b is also shared |
| 114 | * a: [K=3, M=4, Batch=4, Multi=7] |
| 115 | * b: [N=5, K=3, Multi=7] |
| 116 | * d: [N=5, M=4, Batch=4, Multi=7] |
| 117 | * |
Georgios Pinitas | c0b6f76 | 2020-11-02 01:37:17 +0000 | [diff] [blame] | 118 | * @param[in] a Input tensor (Matrix A) |
| 119 | * @param[in] b Input tensor (Matrix B) |
| 120 | * @param[in] c Input tensor (Matrix C) used to pass the bias for quantized calculations |
| 121 | * @param[out] d Output tensor to store the result of matrix multiplication. Data type supported: same as @p input0. |
| 122 | * @param[in] info GEMM meta-data |
Anthony Barbier | c8e84b5 | 2018-07-17 16:48:42 +0100 | [diff] [blame] | 123 | */ |
Sang-Hoon Park | d89e2fa | 2021-05-17 17:04:50 +0100 | [diff] [blame] | 124 | void configure(const ITensorInfo *a, const ITensorInfo *b, const ITensorInfo *c, ITensorInfo *d, const AsmGemmInfo &info); |
Anthony Barbier | eaefd00 | 2018-07-20 17:49:35 +0100 | [diff] [blame] | 125 | |
| 126 | /** Indicates whether or not this function can be used to process the given parameters. |
| 127 | * |
Georgios Pinitas | c0b6f76 | 2020-11-02 01:37:17 +0000 | [diff] [blame] | 128 | * @param[in] a Input tensor info (Matrix A) |
| 129 | * @param[in] b Input tensor info (Matrix B) |
| 130 | * @param[in] c Input tensor info (Matrix C) used to pass the bias for quantized calculations |
| 131 | * @param[in] d Output tensor to store the result of matrix multiplication. Data type supported: same as @p input0. |
| 132 | * @param[in] info GEMM meta-data |
Anthony Barbier | eaefd00 | 2018-07-20 17:49:35 +0100 | [diff] [blame] | 133 | * |
| 134 | * @return a status. |
| 135 | */ |
Georgios Pinitas | c0b6f76 | 2020-11-02 01:37:17 +0000 | [diff] [blame] | 136 | static Status validate(const ITensorInfo *a, const ITensorInfo *b, const ITensorInfo *c, const ITensorInfo *d, const AsmGemmInfo &info); |
Francesco.Petrogalli@arm.com | e33c556 | 2022-03-31 17:55:35 +0000 | [diff] [blame] | 137 | |
| 138 | /** Indicates whether or not there is an optimal assembly implementation that can be used to process the given parameters. |
| 139 | * |
Francesco Petrogalli | 553f695 | 2022-06-30 10:22:01 +0000 | [diff] [blame] | 140 | * This method has the same use of @ref |
| 141 | * NEGEMMConvolutionLayer::has_opt_impl, with the only caveat that |
Ramy Elgammal | 9178002 | 2022-07-20 14:57:37 +0100 | [diff] [blame] | 142 | * the value of arm_compute::WeightFormat need to be passed via the |
Francesco Petrogalli | 553f695 | 2022-06-30 10:22:01 +0000 | [diff] [blame] | 143 | * parameter info. |
Francesco.Petrogalli@arm.com | e33c556 | 2022-03-31 17:55:35 +0000 | [diff] [blame] | 144 | * |
| 145 | * @return a status. |
| 146 | */ |
Ramy Elgammal | 9178002 | 2022-07-20 14:57:37 +0100 | [diff] [blame] | 147 | static Status has_opt_impl(arm_compute::WeightFormat &weight_format, const ITensorInfo *a, const ITensorInfo *b, const ITensorInfo *c, const ITensorInfo *d, const AsmGemmInfo &info); |
Georgios Pinitas | 48b3ef8 | 2019-10-14 19:03:09 +0100 | [diff] [blame] | 148 | /** Checks if activation is supported by the gemm assembly dispatcher |
| 149 | * |
| 150 | * @param[in] activation Activation to check |
| 151 | * |
| 152 | * @return True if activation is supported else false |
| 153 | */ |
| 154 | static bool is_activation_supported(const ActivationLayerInfo &activation); |
Anthony Barbier | c8e84b5 | 2018-07-17 16:48:42 +0100 | [diff] [blame] | 155 | /** Was the function successfully configured ? |
| 156 | * |
| 157 | * @return True if the function is configured and ready to run |
| 158 | */ |
Anthony Barbier | 71d9b57 | 2018-07-06 17:05:59 +0100 | [diff] [blame] | 159 | bool is_configured() const; |
Francesco Petrogalli | 553f695 | 2022-06-30 10:22:01 +0000 | [diff] [blame] | 160 | /** Indicates if the convolution executes in variable weights mode. |
| 161 | * |
| 162 | * Similar to @ref CpuGemm::isVarWeightsKernel |
| 163 | */ |
| 164 | bool isVarWeightsKernel() const |
| 165 | { |
| 166 | return _arm_gemm && _arm_gemm->isVarWeightsKernel(); |
| 167 | } |
Georgios Pinitas | c0b6f76 | 2020-11-02 01:37:17 +0000 | [diff] [blame] | 168 | |
Anthony Barbier | 71d9b57 | 2018-07-06 17:05:59 +0100 | [diff] [blame] | 169 | // Inherited methods overridden: |
Ramy Elgammal | 9178002 | 2022-07-20 14:57:37 +0100 | [diff] [blame] | 170 | void prepare(ITensorPack &tensors) override; |
| 171 | void run(ITensorPack &tensors) override; |
Michele Di Giorgio | d7316eb | 2021-06-16 11:14:41 +0100 | [diff] [blame] | 172 | experimental::MemoryRequirements workspace() const override; |
Georgios Pinitas | c0b6f76 | 2020-11-02 01:37:17 +0000 | [diff] [blame] | 173 | |
| 174 | private: |
Michele Di Giorgio | d7316eb | 2021-06-16 11:14:41 +0100 | [diff] [blame] | 175 | std::unique_ptr<IFallback> _arm_gemm; /**< Interface for the arm_gemm fallback */ |
Anthony Barbier | 71d9b57 | 2018-07-06 17:05:59 +0100 | [diff] [blame] | 176 | }; |
Sang-Hoon Park | 4f7693d | 2021-05-12 13:59:10 +0100 | [diff] [blame] | 177 | } // namespace cpu |
Anthony Barbier | c8e84b5 | 2018-07-17 16:48:42 +0100 | [diff] [blame] | 178 | } // namespace arm_compute |
Sang-Hoon Park | d89e2fa | 2021-05-17 17:04:50 +0100 | [diff] [blame] | 179 | #endif /* ARM_COMPUTE_CPU_INTERNAL_CPU_GEMM_ASSEMBLY_DISPATCH_H */ |