src/core/NEON/kernels/NEAccumulateKernel.cpp

/*
 * Copyright (c) 2016, 2017 ARM Limited.
 *
 * SPDX-License-Identifier: MIT
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to
 * deal in the Software without restriction, including without limitation the
 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
 * sell copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */
#include "arm_compute/core/NEON/kernels/NEAccumulateKernel.h"

#include "arm_compute/core/Error.h"
#include "arm_compute/core/Helpers.h"
#include "arm_compute/core/IAccessWindow.h"
#include "arm_compute/core/Types.h"
#include "arm_compute/core/Validate.h"

#include <arm_neon.h>

using namespace arm_compute;

namespace arm_compute
{
class Coordinates;
} // namespace arm_compute

/* Max S16 value used for saturation purposes. */
const static uint16x8_t max_int_u16 = vdupq_n_u16(static_cast<uint16_t>(INT16_MAX));

#ifdef ARM_COMPUTE_ENABLE_FP16
namespace fp16
{
inline float16x8x2_t convert_u8x16_to_f16x8x2(uint8x16_t input)
{
    const float16x8x2_t out =
    {
        {
            vcvtq_f16_u16(vmovl_u8(vget_low_u8(input))),
            vcvtq_f16_u16(vmovl_u8(vget_high_u8(input)))
        }
    };

    return out;
}

inline uint8x16_t convert_f16x8x2_to_u8x16(const float16x8x2_t &input)
{
    return vcombine_u8(vmovn_u16(vcvtq_u16_f16(input.val[0])),
                       vmovn_u16(vcvtq_u16_f16(input.val[1])));
}

inline float16x8x2_t vector_accumulate_weighted(const float16x8x2_t &vec0, const float16x8x2_t &vec1, float16x8_t scale_val, float16x8_t scale_val2)
{
    const float16x8x2_t res =
    {
        {
            vfmaq_f16(vmulq_f16(vec1.val[0], scale_val), vec0.val[0], scale_val2),
            vfmaq_f16(vmulq_f16(vec1.val[1], scale_val), vec0.val[1], scale_val2)
        }
    };

    return res;
}

void acc_we_v16_u8(const void *__restrict input, void *__restrict accum, float16x8_t scale_val, float16x8_t scale_val2)
{
    ARM_COMPUTE_ERROR_ON(nullptr == input);
    ARM_COMPUTE_ERROR_ON(nullptr == accum);

    const auto input_ptr = static_cast<const uint8_t *__restrict>(input);
    const auto accum_ptr = static_cast<uint8_t *__restrict>(accum);

    const uint8x16x4_t input_buffer = vld4q_u8(input_ptr);
    uint8x16x4_t       accum_buffer = vld4q_u8(accum_ptr);

    const float16x8x2_t f16_input_0 = convert_u8x16_to_f16x8x2(input_buffer.val[0]);
    const float16x8x2_t f16_input_1 = convert_u8x16_to_f16x8x2(input_buffer.val[1]);
    const float16x8x2_t f16_input_2 = convert_u8x16_to_f16x8x2(input_buffer.val[2]);
    const float16x8x2_t f16_input_3 = convert_u8x16_to_f16x8x2(input_buffer.val[3]);

    float16x8x2_t f16_accum_0 = convert_u8x16_to_f16x8x2(accum_buffer.val[0]);
    float16x8x2_t f16_accum_1 = convert_u8x16_to_f16x8x2(accum_buffer.val[1]);
    float16x8x2_t f16_accum_2 = convert_u8x16_to_f16x8x2(accum_buffer.val[2]);
    float16x8x2_t f16_accum_3 = convert_u8x16_to_f16x8x2(accum_buffer.val[3]);

    f16_accum_0 = vector_accumulate_weighted(f16_input_0, f16_accum_0, scale_val, scale_val2);
    f16_accum_1 = vector_accumulate_weighted(f16_input_1, f16_accum_1, scale_val, scale_val2);
    f16_accum_2 = vector_accumulate_weighted(f16_input_2, f16_accum_2, scale_val, scale_val2);
    f16_accum_3 = vector_accumulate_weighted(f16_input_3, f16_accum_3, scale_val, scale_val2);

    accum_buffer = { {
            convert_f16x8x2_to_u8x16(f16_accum_0),
            convert_f16x8x2_to_u8x16(f16_accum_1),
            convert_f16x8x2_to_u8x16(f16_accum_2),
            convert_f16x8x2_to_u8x16(f16_accum_3)
        }
    };

    vst4q_u8(accum_ptr, accum_buffer);
}
} // namespace fp16

void NEAccumulateWeightedFP16Kernel::run(const Window &window)
{
    ARM_COMPUTE_ERROR_ON_UNCONFIGURED_KERNEL(this);
    ARM_COMPUTE_ERROR_ON_INVALID_SUBWINDOW(INESimpleKernel::window(), window);

    Iterator input(_input, window);
    Iterator accum(_output, window);

    const float16x8_t scale_val  = vdupq_n_f16(1.f - _alpha);
    const float16x8_t scale_val2 = vdupq_n_f16(_alpha);

    execute_window_loop(window, [&](const Coordinates & id)
    {
        fp16::acc_we_v16_u8(input.ptr(), accum.ptr(), scale_val, scale_val2);
    },
    input, accum);
}
#endif /* ARM_COMPUTE_ENABLE_FP16 */

namespace
{
inline void acc_v16_u8(const void *__restrict input, void *__restrict accum)
{
    ARM_COMPUTE_ERROR_ON(nullptr == input);
    ARM_COMPUTE_ERROR_ON(nullptr == accum);

    const auto in  = static_cast<const uint8_t *__restrict>(input);
    const auto out = static_cast<int16_t *__restrict>(accum);

    uint8x16_t ta1 = vld1q_u8(in);
    int16x8_t  ta2 = vld1q_s16(out);
    int16x8_t  ta3 = vld1q_s16(out + 8);

    ta2 = vqaddq_s16(ta2, vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(ta1))));
    ta3 = vqaddq_s16(ta3, vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(ta1))));

    vst1q_s16(out, ta2);
    vst1q_s16(out + 8, ta3);
}

inline float32x4x4_t convert_u8x16_to_f32x4x4(uint8x16_t input)
{
    const uint16x8_t u16_output_low = vmovl_u8(vget_low_u8(input));
    const uint16x8_t u16_output_hi  = vmovl_u8(vget_high_u8(input));

    const float32x4x4_t res =
    {
        {
            vcvtq_f32_u32(vmovl_u16(vget_low_u16(u16_output_low))),
            vcvtq_f32_u32(vmovl_u16(vget_high_u16(u16_output_low))),
            vcvtq_f32_u32(vmovl_u16(vget_low_u16(u16_output_hi))),
            vcvtq_f32_u32(vmovl_u16(vget_high_u16(u16_output_hi)))
        }
    };

    return res;
}

inline uint8x16_t convert_f32x4x4_to_u8x16(const float32x4x4_t &input)
{
    return vcombine_u8(vmovn_u16(vcombine_u16(vmovn_u32(vcvtq_u32_f32(input.val[0])),
                                              vmovn_u32(vcvtq_u32_f32(input.val[1])))),
                       vmovn_u16(vcombine_u16(vmovn_u32(vcvtq_u32_f32(input.val[2])),
                                              vmovn_u32(vcvtq_u32_f32(input.val[3])))));
}

inline float32x4x4_t vector_accumulate_weighted(const float32x4x4_t &vector_input, float32x4x4_t vector_output, float32x4_t scale_val, float32x4_t scale_val2)
{
    vector_output.val[0] = vmulq_f32(vector_output.val[0], scale_val);
    vector_output.val[1] = vmulq_f32(vector_output.val[1], scale_val);
    vector_output.val[2] = vmulq_f32(vector_output.val[2], scale_val);
    vector_output.val[3] = vmulq_f32(vector_output.val[3], scale_val);

    vector_output.val[0] = vmlaq_f32(vector_output.val[0], vector_input.val[0], scale_val2);
    vector_output.val[1] = vmlaq_f32(vector_output.val[1], vector_input.val[1], scale_val2);
    vector_output.val[2] = vmlaq_f32(vector_output.val[2], vector_input.val[2], scale_val2);
    vector_output.val[3] = vmlaq_f32(vector_output.val[3], vector_input.val[3], scale_val2);

    return vector_output;
}

inline void acc_we_v16_u8(const void *__restrict input, void *__restrict accum, const float32x4_t scale_val, const float32x4_t scale_val2)
{
    ARM_COMPUTE_ERROR_ON(nullptr == input);
    ARM_COMPUTE_ERROR_ON(nullptr == accum);

    const auto input_ptr = static_cast<const uint8_t *__restrict>(input);
    const auto accum_ptr = static_cast<uint8_t *__restrict>(accum);

    const uint8x16_t input_buffer = vld1q_u8(input_ptr);
    const uint8x16_t accum_buffer = vld1q_u8(accum_ptr);

    const float32x4x4_t f32_input_0  = convert_u8x16_to_f32x4x4(input_buffer);
    const float32x4x4_t f32_output_0 = convert_u8x16_to_f32x4x4(accum_buffer);

    const float32x4x4_t f32_res_0 = vector_accumulate_weighted(f32_input_0, f32_output_0, scale_val, scale_val2);

    vst1q_u8(accum_ptr, convert_f32x4x4_to_u8x16(f32_res_0));
}

void acc_sq_v16_u8(const void *__restrict input, uint32_t shift, void *__restrict accum)
{
    ARM_COMPUTE_ERROR_ON(nullptr == input);
    ARM_COMPUTE_ERROR_ON(nullptr == accum);
    ARM_COMPUTE_ERROR_ON(shift > 15);

    const auto input_buffer = static_cast<const uint8_t *__restrict>(input);
    const auto accum_buffer = static_cast<int16_t *__restrict>(accum);

    const uint8x16_t ta1 = vld1q_u8(input_buffer);
    uint16x8_t       ta2 = vreinterpretq_u16_s16(vld1q_s16(accum_buffer));
    uint16x8_t       ta3 = vreinterpretq_u16_s16(vld1q_s16(accum_buffer + 8));

    const int16x8_t vector_shift = vdupq_n_s16(-static_cast<int16_t>(shift));

    uint16x8_t linput = vmovl_u8(vget_low_u8(ta1));
    uint16x8_t hinput = vmovl_u8(vget_high_u8(ta1));

    linput = vmulq_u16(linput, linput);
    hinput = vmulq_u16(hinput, hinput);

    linput = vqshlq_u16(linput, vector_shift);
    hinput = vqshlq_u16(hinput, vector_shift);

    ta2 = vqaddq_u16(ta2, linput);
    ta3 = vqaddq_u16(ta3, hinput);

    vst1q_s16(accum_buffer, vreinterpretq_s16_u16(vminq_u16(max_int_u16, ta2)));
    vst1q_s16(accum_buffer + 8, vreinterpretq_s16_u16(vminq_u16(max_int_u16, ta3)));
}
} // namespace

void NEAccumulateKernel::configure(const ITensor *input, ITensor *accum)
{
    ARM_COMPUTE_ERROR_ON_NULLPTR(input, accum);

    set_shape_if_empty(*accum->info(), input->info()->tensor_shape());

    set_format_if_unknown(*accum->info(), Format::S16);

    ARM_COMPUTE_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(input, 1, DataType::U8);
    ARM_COMPUTE_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(accum, 1, DataType::S16);
    ARM_COMPUTE_ERROR_ON_MISMATCHING_SHAPES(input, accum);

    constexpr unsigned int num_elems_processed_per_iteration = 16;
    INESimpleKernel::configure(input, accum, num_elems_processed_per_iteration);
}

void NEAccumulateKernel::run(const Window &window)
{
    ARM_COMPUTE_ERROR_ON_UNCONFIGURED_KERNEL(this);
    ARM_COMPUTE_ERROR_ON_INVALID_SUBWINDOW(INESimpleKernel::window(), window);
    Iterator input(_input, window);
    Iterator accum(_output, window);

    execute_window_loop(window, [&](const Coordinates & id)
    {
        acc_v16_u8(input.ptr(), accum.ptr());
    },
    input, accum);
}

NEAccumulateWeightedKernel::NEAccumulateWeightedKernel()
    : _alpha(0.0f)
{
}

void NEAccumulateWeightedKernel::configure(const ITensor *input, float alpha, ITensor *accum)
{
    ARM_COMPUTE_ERROR_ON_NULLPTR(input, accum);

    set_shape_if_empty(*accum->info(), input->info()->tensor_shape());

    set_format_if_unknown(*accum->info(), Format::U8);

    ARM_COMPUTE_ERROR_ON_MISMATCHING_SHAPES(input, accum);
    ARM_COMPUTE_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(input, 1, DataType::U8);
    ARM_COMPUTE_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(accum, 1, DataType::U8);
    ARM_COMPUTE_ERROR_ON(alpha < 0.0 || alpha > 1.0);

    _alpha = alpha;

    constexpr unsigned int num_elems_processed_per_iteration = 16;
    INESimpleKernel::configure(input, accum, num_elems_processed_per_iteration);
}

void NEAccumulateWeightedKernel::run(const Window &window)
{
    ARM_COMPUTE_ERROR_ON_UNCONFIGURED_KERNEL(this);
    ARM_COMPUTE_ERROR_ON_INVALID_SUBWINDOW(INESimpleKernel::window(), window);

    Iterator input(_input, window);
    Iterator accum(_output, window);

    const float32x4_t scale_val  = vdupq_n_f32(1.f - _alpha);
    const float32x4_t scale_val2 = vdupq_n_f32(_alpha);

    execute_window_loop(window, [&](const Coordinates & id)
    {
        acc_we_v16_u8(input.ptr(), accum.ptr(), scale_val, scale_val2);
    },
    input, accum);
}

NEAccumulateSquaredKernel::NEAccumulateSquaredKernel()
    : _shift(0)
{
}

void NEAccumulateSquaredKernel::configure(const ITensor *input, uint32_t shift, ITensor *accum)
{
    ARM_COMPUTE_ERROR_ON_NULLPTR(input, accum);

    set_shape_if_empty(*accum->info(), input->info()->tensor_shape());

    set_format_if_unknown(*accum->info(), Format::S16);

    ARM_COMPUTE_ERROR_ON_MISMATCHING_SHAPES(input, accum);
    ARM_COMPUTE_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(input, 1, DataType::U8);
    ARM_COMPUTE_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(accum, 1, DataType::S16);
    ARM_COMPUTE_ERROR_ON(shift > 15);

    _shift = shift;

    constexpr unsigned int num_elems_processed_per_iteration = 16;
    INESimpleKernel::configure(input, accum, num_elems_processed_per_iteration);
}

void NEAccumulateSquaredKernel::run(const Window &window)
{
    ARM_COMPUTE_ERROR_ON_UNCONFIGURED_KERNEL(this);
    ARM_COMPUTE_ERROR_ON_INVALID_SUBWINDOW(INESimpleKernel::window(), window);
    Iterator input(_input, window);
    Iterator accum(_output, window);

    execute_window_loop(window, [&](const Coordinates & id)
    {
        acc_sq_v16_u8(input.ptr(), _shift, accum.ptr());
    },
    input, accum);
}