blob: e5e97910d8f2c06990afe4f6374b62e176e881a0 [file] [log] [blame]
Pablo Telloeb82fd22018-02-23 13:43:50 +00001/* Copyright (c) 2017-2018 ARM Limited.
Pablo Tello181e6512017-11-15 13:28:27 +00002 *
3 * SPDX-License-Identifier: MIT
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a copy
6 * of this software and associated documentation files (the "Software"), to
7 * deal in the Software without restriction, including without limitation the
8 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
9 * sell copies of the Software, and to permit persons to whom the Software is
10 * furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be included in all
13 * copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 * SOFTWARE.
22 */
23#include "arm_compute/runtime/NEON/functions/NEGEMMLowpAssemblyMatrixMultiplyCore.h"
24
25#include "arm_compute/core/Error.h"
26#include "arm_compute/core/Helpers.h"
27#include "arm_compute/core/ITensor.h"
Pablo Tello181e6512017-11-15 13:28:27 +000028#include "arm_compute/core/NEON/kernels/NEGEMMInterleave4x4Kernel.h"
29#include "arm_compute/core/NEON/kernels/NEGEMMLowpMatrixMultiplyKernel.h"
30#include "arm_compute/core/NEON/kernels/NEGEMMTranspose1xWKernel.h"
Pablo Tello181e6512017-11-15 13:28:27 +000031#include "arm_compute/core/TensorInfo.h"
32#include "arm_compute/core/Types.h"
33#include "arm_compute/core/Validate.h"
34#include "arm_compute/runtime/NEON/NEScheduler.h"
35#include "arm_compute/runtime/TensorAllocator.h"
36#include "support/ToolchainSupport.h"
37
Pablo Tello181e6512017-11-15 13:28:27 +000038using namespace arm_compute;
39
40NEGEMMLowpAssemblyMatrixMultiplyCore::NEGEMMLowpAssemblyMatrixMultiplyCore(std::shared_ptr<IMemoryManager> memory_manager)
Pablo Telloeb82fd22018-02-23 13:43:50 +000041 : _memory_group(std::move(memory_manager)), _asm_glue_unsigned(), _asm_glue_signed(), _mm_kernel(nullptr), _mtx_a_reshape_kernel(nullptr), _mtx_b_reshape_kernel(nullptr), _tmp_a(), _tmp_b(),
42 _workspace()
Pablo Tello181e6512017-11-15 13:28:27 +000043{
44}
45
46void NEGEMMLowpAssemblyMatrixMultiplyCore::configure(const ITensor *a, const ITensor *b, ITensor *output)
47{
Michalis Spyrouf3dfa272017-11-21 17:52:12 +000048 ARM_COMPUTE_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(a, 1, DataType::U8, DataType::S8);
49 ARM_COMPUTE_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(output, 1, DataType::U32, DataType::S32);
Pablo Tello181e6512017-11-15 13:28:27 +000050 ARM_COMPUTE_ERROR_ON_MISMATCHING_DATA_TYPES(a, b);
51 ARM_COMPUTE_ERROR_ON_MSG((a)->info()->dimension(0) != (b)->info()->dimension(1), "The product AB is defined only if the number of columns in A is equal to the number of rows in B");
52 ARM_COMPUTE_ERROR_ON_MSG((a)->info()->dimension(1) != (output)->info()->dimension(1), "The output matrix must have the same number of rows as the matrix A");
53 ARM_COMPUTE_ERROR_ON_MSG((b)->info()->dimension(0) != (output)->info()->dimension(0), "The output matrix must have the same number of columns as the matrix B");
54
Pablo Telloeb82fd22018-02-23 13:43:50 +000055 bool run_optimised = false;
Pablo Tello181e6512017-11-15 13:28:27 +000056#ifdef __aarch64__
Pablo Telloeb82fd22018-02-23 13:43:50 +000057 switch(a->info()->data_type())
58 {
59 case DataType::S8:
60 {
61 run_optimised = setup_assembly_kernel(a, b, nullptr, output, 1.f, 1.f, _workspace, _memory_group, _asm_glue_signed);
62 break;
63 }
64 case DataType::U8:
65 {
66 run_optimised = setup_assembly_kernel(a, b, nullptr, output, 1.f, 1.f, _workspace, _memory_group, _asm_glue_unsigned);
67 break;
68 }
69 default:
70 {
71 ARM_COMPUTE_ERROR("Datatype not supported");
72 break;
73 }
74 }
Pablo Tello181e6512017-11-15 13:28:27 +000075#endif /* __aarch64__ */
Pablo Telloeb82fd22018-02-23 13:43:50 +000076 if(!run_optimised)
Pablo Tello181e6512017-11-15 13:28:27 +000077 {
78 // The interleaved output matrix will have the following shape: [ a_height * 4, ceil(a_width / 4.0f) ]
79 TensorShape shape_tmp_a = a->info()->tensor_shape();
80 shape_tmp_a.set(0, a->info()->dimension(0) * 4);
81 shape_tmp_a.set(1, std::ceil(a->info()->dimension(1) / 4.f));
82
83 // The transpose1xW output matrix will have the following shape: [ b_height * 16, ceil(b_width / 16.0f) ]
84 TensorShape shape_tmp_b = b->info()->tensor_shape();
85 shape_tmp_b.set(0, b->info()->dimension(1) * 16);
86 shape_tmp_b.set(1, std::ceil(b->info()->dimension(0) / 16.f));
87
88 TensorInfo info_a(shape_tmp_a, 1, a->info()->data_type());
89 TensorInfo info_b(shape_tmp_b, 1, b->info()->data_type());
90 _tmp_a.allocator()->init(info_a);
91 _tmp_b.allocator()->init(info_b);
92 _memory_group.manage(&_tmp_a);
93 _memory_group.manage(&_tmp_b);
94
95 // Configure interleave kernel
96 {
97 auto k = arm_compute::support::cpp14::make_unique<NEGEMMInterleave4x4Kernel>();
98 k->configure(a, &_tmp_a);
99 _mtx_a_reshape_kernel = std::move(k);
100 }
101
102 // Configure transpose kernel
103 {
104 auto k = arm_compute::support::cpp14::make_unique<NEGEMMTranspose1xWKernel>();
105 k->configure(b, &_tmp_b);
106 _mtx_b_reshape_kernel = std::move(k);
107 }
108
109 // Configure matrix multiply kernel
110 {
111 auto k = arm_compute::support::cpp14::make_unique<NEGEMMLowpMatrixMultiplyKernel>();
112 k->configure(&_tmp_a, &_tmp_b, output);
113 _mm_kernel = std::move(k);
114 }
115
116 // Allocate tensors
117 _tmp_a.allocator()->allocate();
118 _tmp_b.allocator()->allocate();
119 }
120}
121
122void NEGEMMLowpAssemblyMatrixMultiplyCore::run()
123{
124 _memory_group.acquire();
125 if(_mtx_a_reshape_kernel)
126 {
127 NEScheduler::get().schedule(_mtx_a_reshape_kernel.get(), Window::DimY);
128 }
129
130 if(_mtx_b_reshape_kernel)
131 {
132 NEScheduler::get().schedule(_mtx_b_reshape_kernel.get(), Window::DimY);
133 }
134
Pablo Telloeb82fd22018-02-23 13:43:50 +0000135 if(_asm_glue_unsigned._optimised_kernel != nullptr)
136 {
137 _asm_glue_unsigned.run();
138 }
139 else if(_asm_glue_signed._optimised_kernel != nullptr)
140 {
141 _asm_glue_signed.run();
142 }
143 else
144 {
145 NEScheduler::get().schedule(_mm_kernel.get(), Window::DimY);
146 }
Pablo Tello181e6512017-11-15 13:28:27 +0000147
148 _memory_group.release();
149}