blob: bd931469a3c25d02a476bc5a917585ead63c439d [file] [log] [blame]
Anthony Barbier6ff3b192017-09-04 18:44:23 +01001/*
Georgios Pinitasddb93bb2020-10-02 16:38:59 +01002 * Copyright (c) 2016-2020 Arm Limited.
Anthony Barbier6ff3b192017-09-04 18:44:23 +01003 *
4 * SPDX-License-Identifier: MIT
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to
8 * deal in the Software without restriction, including without limitation the
9 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
10 * sell copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in all
14 * copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22 * SOFTWARE.
23 */
24#include "arm_compute/core/NEON/kernels/NEGEMMMatrixAdditionKernel.h"
25
Anthony Barbiereaefd002018-07-20 17:49:35 +010026#include "arm_compute/core/CPP/Validate.h"
Anthony Barbier6ff3b192017-09-04 18:44:23 +010027#include "arm_compute/core/Error.h"
28#include "arm_compute/core/Helpers.h"
Anthony Barbier6ff3b192017-09-04 18:44:23 +010029#include "arm_compute/core/Types.h"
30#include "arm_compute/core/Validate.h"
Georgios Pinitasddb93bb2020-10-02 16:38:59 +010031#include "src/core/NEON/NEFixedPoint.h"
Anthony Barbier6ff3b192017-09-04 18:44:23 +010032
33#include <arm_neon.h>
34
Anthony Barbier6ff3b192017-09-04 18:44:23 +010035namespace arm_compute
36{
Anthony Barbier6ff3b192017-09-04 18:44:23 +010037namespace
38{
Michele Di Giorgio4646d2e2019-06-19 12:28:47 +010039constexpr unsigned int num_elems_processed_per_iteration = 16;
40
Georgios Pinitasea9e0dc2018-08-28 16:24:56 +010041Status validate_arguments(const ITensorInfo *input, const ITensorInfo *output, float beta)
42{
43 ARM_COMPUTE_RETURN_ERROR_ON_NULLPTR(input, output);
44 ARM_COMPUTE_UNUSED(beta);
45
46 ARM_COMPUTE_RETURN_ERROR_ON_CPU_F16_UNSUPPORTED(input);
47 ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(input, 1, DataType::F16, DataType::F32);
48
49 if(output->total_size() > 0)
50 {
51 ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(input, output);
52 ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_SHAPES(input, output);
53 }
54
55 return Status{};
56}
57
Anthony Barbier6ff3b192017-09-04 18:44:23 +010058void matrix_addition_f32(const ITensor *input, ITensor *output, const Window &window, float beta)
59{
60 const float32x4_t beta_f32 = vdupq_n_f32(beta);
61
62 Iterator in(input, window);
63 Iterator out(output, window);
64
Michalis Spyroua4f378d2019-04-26 14:54:54 +010065 execute_window_loop(window, [&](const Coordinates &)
Anthony Barbier6ff3b192017-09-04 18:44:23 +010066 {
67 const auto in_ptr = reinterpret_cast<const float *>(in.ptr());
68 const auto out_ptr = reinterpret_cast<float *>(out.ptr());
69
Pablo Tello221f3812017-06-28 17:27:56 +010070 float32x4x4_t alpha_ab = vld4q_f32(out_ptr);
71 const float32x4x4_t c = vld4q_f32(in_ptr);
Anthony Barbier6ff3b192017-09-04 18:44:23 +010072
73 // Multiply matrix C by its weight and accumulate
74 alpha_ab.val[0] = vmlaq_f32(alpha_ab.val[0], c.val[0], beta_f32);
75 alpha_ab.val[1] = vmlaq_f32(alpha_ab.val[1], c.val[1], beta_f32);
76 alpha_ab.val[2] = vmlaq_f32(alpha_ab.val[2], c.val[2], beta_f32);
77 alpha_ab.val[3] = vmlaq_f32(alpha_ab.val[3], c.val[3], beta_f32);
78
Pablo Tello221f3812017-06-28 17:27:56 +010079 vst4q_f32(out_ptr, alpha_ab);
Anthony Barbier6ff3b192017-09-04 18:44:23 +010080 },
81 in, out);
82}
83
Ioan-Cristian Szabo5edbd1c2017-11-13 13:34:08 +000084#ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
Anthony Barbier6ff3b192017-09-04 18:44:23 +010085void matrix_addition_f16(const ITensor *input, ITensor *output, const Window &window, float beta)
86{
87 const float16x8_t beta_f16 = vdupq_n_f16(beta);
88
89 Iterator in(input, window);
90 Iterator out(output, window);
91
Michalis Spyroua4f378d2019-04-26 14:54:54 +010092 execute_window_loop(window, [&](const Coordinates &)
Anthony Barbier6ff3b192017-09-04 18:44:23 +010093 {
94 const auto in_ptr = reinterpret_cast<const float16_t *>(in.ptr());
95 const auto out_ptr = reinterpret_cast<float16_t *>(out.ptr());
96
Pablo Tello221f3812017-06-28 17:27:56 +010097 float16x8x2_t alpha_ab = vld2q_f16(out_ptr);
98 const float16x8x2_t c = vld2q_f16(in_ptr);
Anthony Barbier6ff3b192017-09-04 18:44:23 +010099 // Multiply matrix C by its weight and accumulate
100 alpha_ab.val[0] = vaddq_f16(alpha_ab.val[0], vmulq_f16(c.val[0], beta_f16));
101 alpha_ab.val[1] = vaddq_f16(alpha_ab.val[1], vmulq_f16(c.val[1], beta_f16));
102
Pablo Tello221f3812017-06-28 17:27:56 +0100103 vst2q_f16(out_ptr + 0, alpha_ab);
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100104 },
105 in, out);
106}
Ioan-Cristian Szabo5edbd1c2017-11-13 13:34:08 +0000107#endif /* __ARM_FEATURE_FP16_VECTOR_ARITHMETIC */
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100108
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100109} // namespace
110
111NEGEMMMatrixAdditionKernel::NEGEMMMatrixAdditionKernel()
112 : INESimpleKernel(), _func(nullptr), _beta(0.0f)
113{
114}
115
116void NEGEMMMatrixAdditionKernel::configure(const ITensor *input, ITensor *output, float beta)
117{
Georgios Pinitasea9e0dc2018-08-28 16:24:56 +0100118 ARM_COMPUTE_ERROR_ON_NULLPTR(input, output);
119
120 // Perform validation step
121 ARM_COMPUTE_ERROR_THROW_ON(validate_arguments(input->info(), output->info(), beta));
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100122
123 switch(input->info()->data_type())
124 {
125 case DataType::F32:
126 _func = &matrix_addition_f32;
127 break;
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100128 case DataType::F16:
Ioan-Cristian Szabo5edbd1c2017-11-13 13:34:08 +0000129#ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100130 _func = &matrix_addition_f16;
131 break;
Ioan-Cristian Szabo5edbd1c2017-11-13 13:34:08 +0000132#endif /* __ARM_FEATURE_FP16_VECTOR_ARITHMETIC */
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100133 default:
134 ARM_COMPUTE_ERROR("Data type not supported");
135 break;
136 }
137
Georgios Pinitasea9e0dc2018-08-28 16:24:56 +0100138 // Configure kernel window
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100139 INESimpleKernel::configure(input, output, num_elems_processed_per_iteration);
140
141 _beta = beta;
142}
143
Georgios Pinitasea9e0dc2018-08-28 16:24:56 +0100144Status NEGEMMMatrixAdditionKernel::validate(const ITensorInfo *input, const ITensorInfo *output, float beta)
145{
Georgios Pinitasea9e0dc2018-08-28 16:24:56 +0100146 ARM_COMPUTE_RETURN_ON_ERROR(validate_arguments(input, output, beta));
147 ARM_COMPUTE_RETURN_ON_ERROR(INESimpleKernel::validate(input->clone().get(), output->clone().get(), num_elems_processed_per_iteration));
148 return Status{};
149}
150
Moritz Pflanzerc186b572017-09-07 09:48:04 +0100151void NEGEMMMatrixAdditionKernel::run(const Window &window, const ThreadInfo &info)
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100152{
Moritz Pflanzerc186b572017-09-07 09:48:04 +0100153 ARM_COMPUTE_UNUSED(info);
Anthony Barbier6ff3b192017-09-04 18:44:23 +0100154 ARM_COMPUTE_ERROR_ON_UNCONFIGURED_KERNEL(this);
155 ARM_COMPUTE_ERROR_ON_INVALID_SUBWINDOW(INESimpleKernel::window(), window);
156
157 if(_beta != 0.0f)
158 {
159 (*_func)(_input, _output, window, _beta);
160 }
161}
Georgios Pinitasea9e0dc2018-08-28 16:24:56 +0100162} // namespace arm_compute