blob: 7f37520f1059de304945fb7781e5da4a82444836 [file] [log] [blame]
Anthony Barbier6ff3b192017-09-04 18:44:23 +01001/*
Gian Marco20d78482018-01-11 15:10:58 +00002 * Copyright (c) 2017-2018 ARM Limited.
Anthony Barbier6ff3b192017-09-04 18:44:23 +01003 *
4 * SPDX-License-Identifier: MIT
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to
8 * deal in the Software without restriction, including without limitation the
9 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
10 * sell copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in all
14 * copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22 * SOFTWARE.
23 */
24#include "arm_compute/runtime/CL/functions/CLGEMM.h"
25
26#include "arm_compute/core/CL/ICLTensor.h"
27#include "arm_compute/core/CL/kernels/CLGEMMInterleave4x4Kernel.h"
28#include "arm_compute/core/CL/kernels/CLGEMMMatrixAdditionKernel.h"
29#include "arm_compute/core/CL/kernels/CLGEMMMatrixMultiplyKernel.h"
30#include "arm_compute/core/CL/kernels/CLGEMMTranspose1xWKernel.h"
31#include "arm_compute/core/Error.h"
Gian Marco Iodice750641d2018-05-08 12:01:57 +010032#include "arm_compute/core/GPUTarget.h"
Anthony Barbier6ff3b192017-09-04 18:44:23 +010033#include "arm_compute/core/Helpers.h"
34#include "arm_compute/core/TensorInfo.h"
35#include "arm_compute/core/Types.h"
Gian Marco Iodicebb36a8e2018-04-19 12:05:08 +010036#include "arm_compute/core/Utils.h"
Anthony Barbier6ff3b192017-09-04 18:44:23 +010037#include "arm_compute/core/Validate.h"
Gian Marco Iodice750641d2018-05-08 12:01:57 +010038#include "arm_compute/core/utils/misc/ShapeCalculator.h"
Anthony Barbier6ff3b192017-09-04 18:44:23 +010039#include "arm_compute/runtime/CL/CLScheduler.h"
40#include "arm_compute/runtime/ITensorAllocator.h"
41
42using namespace arm_compute;
Gian Marco Iodice750641d2018-05-08 12:01:57 +010043using namespace arm_compute::misc::shape_calculator;
Anthony Barbier6ff3b192017-09-04 18:44:23 +010044
Gian Marco36a0a462018-01-12 10:21:40 +000045namespace
46{
47inline bool is_interleaved_transposed(int m, int n, int k, DataType data_type, bool reshape_b_only_on_first_run, GPUTarget gpu_target)
48{
49 bool flag = true;
50
Gian Marco Iodice513fe2e2018-06-04 18:08:48 +010051 if(gpu_target_is_in(gpu_target, GPUTarget::G71, GPUTarget::G72))
Gian Marco36a0a462018-01-12 10:21:40 +000052 {
53 // COMPMID-852
Gian Marco Iodicebb36a8e2018-04-19 12:05:08 +010054 if(k > 256 && m > 4 && is_data_type_float(data_type) && reshape_b_only_on_first_run)
Gian Marco36a0a462018-01-12 10:21:40 +000055 {
Gian Marco Iodice513fe2e2018-06-04 18:08:48 +010056 constexpr float alpha = 3.2f;
57 constexpr float fact0 = 1.51f;
58 constexpr float fact1 = 1.66f;
59 constexpr float ops = 12.0f;
60 const float scale = k > 1024 ? 1.07f : 1.0f;
61 flag = alpha + ((n * fact0) / ops) < ((fact1 * n * scale) / ops);
Gian Marco36a0a462018-01-12 10:21:40 +000062 }
63 else
64 {
65 flag = false;
66 }
67 }
Gian Marco Iodicecda0c382018-04-23 16:16:22 +010068 else
69 {
70 // We reshape the matrices only if we do not have the vector-by-matrix case and we reshape the matrix B only once
71 flag = m != 1 && reshape_b_only_on_first_run;
72 }
Gian Marco36a0a462018-01-12 10:21:40 +000073
74 return flag;
75}
76} // namespace
77
Georgios Pinitas8a94e7c2017-09-15 19:06:47 +010078CLGEMM::CLGEMM(std::shared_ptr<IMemoryManager> memory_manager)
Georgios Pinitas82b51482018-04-24 15:14:12 +010079 : _memory_group(std::move(memory_manager)), _interleave_kernel(), _transpose_kernel(), _mm_kernel(), _ma_kernel(), _tmp_a(), _tmp_b(), _original_b(nullptr), _is_interleaved_transposed(false),
Georgios Pinitase0437672018-05-02 14:07:55 +010080 _run_addition(false), _reshape_b_only_on_first_run(false), _is_prepared(false)
Anthony Barbier6ff3b192017-09-04 18:44:23 +010081{
82}
83
Gian Marco1d25ed52017-12-16 19:33:50 +000084void CLGEMM::configure(const ICLTensor *a, const ICLTensor *b, const ICLTensor *c, ICLTensor *output, float alpha, float beta, const GEMMInfo &gemm_info)
Anthony Barbier6ff3b192017-09-04 18:44:23 +010085{
Georgios Pinitas78c00902018-01-09 17:33:11 +000086 ARM_COMPUTE_ERROR_ON_NULLPTR(a, b, output);
Anthony Barbier6ff3b192017-09-04 18:44:23 +010087
Georgios Pinitas78c00902018-01-09 17:33:11 +000088 // Perform validation step
Gian Marco Iodice750641d2018-05-08 12:01:57 +010089 ARM_COMPUTE_ERROR_THROW_ON(validate(a->info(), b->info(), c != nullptr ? c->info() : nullptr, output->info(), alpha, beta, gemm_info));
Anthony Barbier6ff3b192017-09-04 18:44:23 +010090
Georgios Pinitas82b51482018-04-24 15:14:12 +010091 // Store original b matrix
92 _original_b = b;
93
Gian Marco1d25ed52017-12-16 19:33:50 +000094 // Check if we need to reshape the matrix B only on the first run
95 _reshape_b_only_on_first_run = gemm_info.reshape_b_only_on_first_run();
Georgios Pinitase0437672018-05-02 14:07:55 +010096 _is_prepared = false;
Gian Marco Iodice1246b632017-08-16 18:38:32 +010097
Gian Marco Iodiceedfa9f42017-08-15 11:45:22 +010098 const ICLTensor *matrix_a = a;
99 const ICLTensor *matrix_b = b;
100
Gian Marco36a0a462018-01-12 10:21:40 +0000101 // Get the GPU target
102 const GPUTarget gpu_target = CLScheduler::get().target();
103
104 // Set the target for the kernels
105 _interleave_kernel.set_target(gpu_target);
106 _mm_kernel.set_target(gpu_target);
107
108 // Arguments used by GEMMReshapeInfo
109 // If we pass the matrix A and matrix B reshaped to CLGEMMMatrixMultiplyKernel, we need to pass m, n, k, mult_transpose1xW_width and mult_interleave4x4_height to CLGEMMReshapeInfo
110 // in order to know how the matrices have been reshaped
111 const int m = a->info()->dimension(1);
112 const int n = b->info()->dimension(0);
113 const int k = a->info()->dimension(0);
114 int mult_transpose1xW_width = 1;
115 int mult_interleave4x4_height = 1;
116
Gian Marco Iodice750641d2018-05-08 12:01:57 +0100117 if(get_arch_from_target(gpu_target) == GPUTarget::BIFROST)
Gian Marco36a0a462018-01-12 10:21:40 +0000118 {
119 mult_transpose1xW_width = 4;
120 mult_interleave4x4_height = 2;
121 }
122
123 // Check if we need to reshape the matrix A and matrix B
124 _is_interleaved_transposed = is_interleaved_transposed(m, n, k, a->info()->data_type(), _reshape_b_only_on_first_run, gpu_target);
Gian Marcob5311a62017-12-13 12:48:03 +0000125
Gian Marco Iodiceedfa9f42017-08-15 11:45:22 +0100126 if(_is_interleaved_transposed)
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100127 {
Gian Marco Iodiceedfa9f42017-08-15 11:45:22 +0100128 matrix_a = &_tmp_a;
129 matrix_b = &_tmp_b;
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100130
Gian Marco19835e52018-01-30 13:35:54 +0000131 // Manage intermediate buffers
132 _memory_group.manage(&_tmp_a);
Georgios Pinitasae4ce7b2018-03-19 17:50:45 +0000133 if(!_reshape_b_only_on_first_run)
Georgios Pinitasd8734b52017-12-22 15:27:52 +0000134 {
135 _memory_group.manage(&_tmp_b);
136 }
Gian Marco20d78482018-01-11 15:10:58 +0000137 // _tmp_a and _tmp_b will be auto configured in _interleave_kernel and in _transpose_kernel
Georgios Pinitas8a94e7c2017-09-15 19:06:47 +0100138
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100139 // Configure interleave kernel
Gian Marco36a0a462018-01-12 10:21:40 +0000140 _interleave_kernel.configure(a, &_tmp_a, mult_interleave4x4_height);
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100141
142 // Configure transpose kernel
Gian Marco36a0a462018-01-12 10:21:40 +0000143 _transpose_kernel.configure(b, &_tmp_b, mult_transpose1xW_width);
Gian Marco Iodiceedfa9f42017-08-15 11:45:22 +0100144 }
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100145
Gian Marco36a0a462018-01-12 10:21:40 +0000146 _mm_kernel.configure(matrix_a, matrix_b, output, alpha, _is_interleaved_transposed, GEMMReshapeInfo(m, n, k, mult_transpose1xW_width, mult_interleave4x4_height));
Gian Marco Iodiceedfa9f42017-08-15 11:45:22 +0100147
148 if(_is_interleaved_transposed)
149 {
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100150 // Allocate intermediate tensors
151 _tmp_a.allocator()->allocate();
Georgios Pinitase0437672018-05-02 14:07:55 +0100152 if(!_reshape_b_only_on_first_run)
153 {
154 _tmp_b.allocator()->allocate();
155 }
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100156 }
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100157
158 // Configure matrix addition kernel
159 if(beta != 0 && c != nullptr)
160 {
161 _ma_kernel.configure(c, output, beta);
162 _run_addition = true;
163 }
164}
165
Gian Marco Iodice750641d2018-05-08 12:01:57 +0100166Status CLGEMM::validate(const ITensorInfo *a, const ITensorInfo *b, const ITensorInfo *c, const ITensorInfo *output, float alpha, float beta, const GEMMInfo &gemm_info)
Georgios Pinitas78c00902018-01-09 17:33:11 +0000167{
Gian Marco Iodice750641d2018-05-08 12:01:57 +0100168 ARM_COMPUTE_UNUSED(alpha);
169
170 // Check if we need to reshape the matrix B only on the first run
171 const bool reshape_b_only_on_first_run = gemm_info.reshape_b_only_on_first_run();
172
173 const ITensorInfo *matrix_a_info = a;
174 const ITensorInfo *matrix_b_info = b;
175
176 TensorInfo tmp_a_info{};
177 TensorInfo tmp_b_info{};
178 TensorInfo tmp_output_info = *output->clone();
179
180 // Get the GPU target
181 const GPUTarget gpu_target = CLScheduler::get().target();
182
183 // Arguments used by GEMMReshapeInfo
184 // If we pass the matrix A and matrix B reshaped to CLGEMMMatrixMultiplyKernel, we need to pass m, n, k, mult_transpose1xW_width and mult_interleave4x4_height to CLGEMMReshapeInfo
185 // in order to know how the matrices have been reshaped
186 const int m = a->dimension(1);
187 const int n = b->dimension(0);
188 const int k = a->dimension(0);
189 int mult_transpose1xW_width = 1;
190 int mult_interleave4x4_height = 1;
191
192 if(get_arch_from_target(gpu_target) == GPUTarget::BIFROST)
193 {
194 mult_transpose1xW_width = 4;
195 mult_interleave4x4_height = 2;
196 }
197
198 const GEMMReshapeInfo reshape_info = GEMMReshapeInfo(m, n, k, mult_transpose1xW_width, mult_interleave4x4_height);
199
200 // Check if we need to reshape the matrix A and matrix B
201 const bool run_interleave_transpose = is_interleaved_transposed(m, n, k, a->data_type(), reshape_b_only_on_first_run, gpu_target);
202
203 if(run_interleave_transpose)
204 {
205 matrix_a_info = &tmp_a_info;
206 matrix_b_info = &tmp_b_info;
207
208 // Validate interleave kernel
209 auto_init_if_empty(tmp_a_info, a->clone()->set_tensor_shape(compute_interleaved_shape(*a, mult_interleave4x4_height)));
210 ARM_COMPUTE_RETURN_ON_ERROR(CLGEMMInterleave4x4Kernel::validate(a, &tmp_a_info, mult_interleave4x4_height));
211
212 // Validate transpose kernel
213 auto_init_if_empty(tmp_b_info, b->clone()->set_tensor_shape(compute_transpose1xW_with_element_size_shape(*b, mult_transpose1xW_width)));
214 ARM_COMPUTE_RETURN_ON_ERROR(CLGEMMTranspose1xWKernel::validate(b, &tmp_b_info, mult_transpose1xW_width));
215 }
216
217 // Validate matrix multiply
218 auto_init_if_empty(tmp_output_info, matrix_a_info->clone()->set_tensor_shape(compute_mm_shape(*matrix_a_info, *matrix_b_info, run_interleave_transpose, reshape_info)));
219 ARM_COMPUTE_RETURN_ON_ERROR(CLGEMMMatrixMultiplyKernel::validate(matrix_a_info, matrix_b_info, &tmp_output_info, alpha, run_interleave_transpose, reshape_info, gpu_target));
220
221 if(beta != 0 && c != nullptr)
222 {
223 // Validate matrix addition kernel
224 ARM_COMPUTE_RETURN_ON_ERROR(CLGEMMMatrixAdditionKernel::validate(c, &tmp_output_info, beta));
225 }
226
Georgios Pinitas78c00902018-01-09 17:33:11 +0000227 return Status{};
228}
229
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100230void CLGEMM::run()
231{
Georgios Pinitase0437672018-05-02 14:07:55 +0100232 prepare();
233
Georgios Pinitas8a94e7c2017-09-15 19:06:47 +0100234 _memory_group.acquire();
235
Gian Marco Iodiceedfa9f42017-08-15 11:45:22 +0100236 if(_is_interleaved_transposed)
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100237 {
238 // Run interleave kernel
239 CLScheduler::get().enqueue(_interleave_kernel, false);
240
Georgios Pinitase0437672018-05-02 14:07:55 +0100241 if(!_reshape_b_only_on_first_run)
Gian Marco1d25ed52017-12-16 19:33:50 +0000242 {
243 // Run transpose kernel
244 CLScheduler::get().enqueue(_transpose_kernel, false);
245 }
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100246 }
247
248 // Run matrix multiply kernel
249 CLScheduler::get().enqueue(_mm_kernel, !_run_addition);
250
251 // Run matrix addition kernel
252 if(_run_addition)
253 {
254 CLScheduler::get().enqueue(_ma_kernel);
255 }
Georgios Pinitas8a94e7c2017-09-15 19:06:47 +0100256
257 _memory_group.release();
Georgios Pinitase0437672018-05-02 14:07:55 +0100258}
Georgios Pinitas82b51482018-04-24 15:14:12 +0100259
Georgios Pinitase0437672018-05-02 14:07:55 +0100260void CLGEMM::prepare()
261{
262 if(!_is_prepared)
263 {
264 if(_is_interleaved_transposed && _reshape_b_only_on_first_run)
265 {
266 // Run transpose kernel
267 _tmp_b.allocator()->allocate();
268 CLScheduler::get().enqueue(_transpose_kernel, false);
269 _original_b->mark_as_unused();
270 }
271 CLScheduler::get().queue().finish();
272 _is_prepared = true;
273 }
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100274}