src/core/NEON/SVEAsymm.h

/*
 * Copyright (c) 2020 Arm Limited.
 *
 * SPDX-License-Identifier: MIT
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to
 * deal in the Software without restriction, including without limitation the
 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
 * sell copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */
#ifndef ARM_COMPUTE_SVEASYMM_H
#define ARM_COMPUTE_SVEASYMM_H

#if defined(__ARM_FEATURE_SVE2)
#include "src/core/NEON/SVEMath.h"
#include <arm_sve.h>

namespace arm_compute
{
/** Perform a multiply-accumulate on all components of a QASYMM8 vector
 *
 * vd*vs + vo
 *
 * @param[in] pg Predicate value.
 * @param[in] vd Input vector value in QASYMM8 format
 * @param[in] vs Vector multiplier in F32 format. The multiplier value must be duplicated across all four lanes.
 * @param[in] vo Vector addend in F32 format. The addend value must be duplicated across all four lanes.
 *
 * @return A vector in QASYMM8 format, saturated to fit
 */
svuint8_t svmla_qasymm8_z(svbool_t pg, svuint8_t vd, svfloat32_t vs, svfloat32_t vo);

/** Perform a multiply-accumulate on all components of a QASYMM8_SIGNED vector
 *
 * vd*vs + vo
 *
 * @param[in] pg Predicate value.
 * @param[in] vd Input vector value in QASYMM8_SIGNED format
 * @param[in] vs Vector multiplier in F32 format. The multiplier value must be duplicated across all four lanes.
 * @param[in] vo Vector addend in F32 format. The addend value must be duplicated across all four lanes.
 *
 * @return A vector in QASYMM8_SIGNED format, saturated to fit
 */
svint8_t svmla_qasymm8_signed_z(svbool_t pg, svint8_t vd, svfloat32_t vs, svfloat32_t vo);

/** Dequantize following an asymmetric quantization scheme a sve vector.
 *
 * @param[in] pg     Predicate value.
 * @param[in] qv     Input values to be dequantized.
 * @param[in] scale  Quantization scaling factor.
 * @param[in] offset Zero quantization offset.
 *
 * @return Dequantized values in an sve vector
 */
inline svfloat32x4_t svdequantize_z(svbool_t pg, const svuint8_t &qv, float scale, int32_t offset)
{
    const auto          voffset = svdup_n_s32(offset);
    const auto          vscale  = svdup_n_f32(scale);
    const svfloat32x4_t vdequantized_input =
    {
        { {
                svmul_f32_z(pg, svcvt_f32_s32_z(pg, svsub_s32_z(pg, svreinterpret_s32_u32(svmovlb_u32(svmovlb_u16(qv))), voffset)), vscale),
                svmul_f32_z(pg, svcvt_f32_s32_z(pg, svsub_s32_z(pg, svreinterpret_s32_u32(svmovlt_u32(svmovlb_u16(qv))), voffset)), vscale),
                svmul_f32_z(pg, svcvt_f32_s32_z(pg, svsub_s32_z(pg, svreinterpret_s32_u32(svmovlb_u32(svmovlt_u16(qv))), voffset)), vscale),
                svmul_f32_z(pg, svcvt_f32_s32_z(pg, svsub_s32_z(pg, svreinterpret_s32_u32(svmovlt_u32(svmovlt_u16(qv))), voffset)), vscale),
            }
        }
    };
    return vdequantized_input;
}

/** Dequantize an sve vector
 *
 * @param[in] pg Predicate value.
 * @param[in] qv Input values to be dequantized.
 * @param[in] qi Quantization information to be used in the computation.
 *
 * @return Dequantized values in an sve vector
 */
inline svfloat32x4_t svdequantize_z(svbool_t pg, const svuint8_t &qv, const UniformQuantizationInfo &qi)
{
    return svdequantize_z(pg, qv, qi.scale, qi.offset);
}

/** Dequantize an sve vector stored as signed asymmetric.
 *
 * @param[in] pg     Predicate value.
 * @param[in] qv     Input values to be dequantized.
 * @param[in] scale  Quantization scaling factor.
 * @param[in] offset Zero quantization offset.
 *
 * @return Dequantized values in a sve vector
 */
inline svfloat32x4_t svdequantize_z(svbool_t pg, const svint8_t &qv, float scale, int32_t offset)
{
    const auto          voffset = svdup_n_s32(offset);
    const auto          vscale  = svdup_n_f32(scale);
    const svfloat32x4_t vdequantized_input =
    {
        { {
                svmul_f32_z(pg, svcvt_f32_s32_z(pg, svsub_s32_z(pg, svmovlb_s32(svmovlb_s16(qv)), voffset)), vscale),
                svmul_f32_z(pg, svcvt_f32_s32_z(pg, svsub_s32_z(pg, svmovlt_s32(svmovlb_s16(qv)), voffset)), vscale),
                svmul_f32_z(pg, svcvt_f32_s32_z(pg, svsub_s32_z(pg, svmovlb_s32(svmovlt_s16(qv)), voffset)), vscale),
                svmul_f32_z(pg, svcvt_f32_s32_z(pg, svsub_s32_z(pg, svmovlt_s32(svmovlt_s16(qv)), voffset)), vscale),
            }
        }
    };
    return vdequantized_input;
}

/** Dequantize an sve vector.
 *
 * @param[in] pg Predicate value.
 * @param[in] qv Input values to be dequantized.
 * @param[in] qi Quantization information to be used in the computation.
 *
 * @return Dequantized values in an sve vector
 */
inline svfloat32x4_t svdequantize_z(svbool_t pg, const svint8_t &qv, const UniformQuantizationInfo &qi)
{
    return svdequantize_z(pg, qv, qi.scale, qi.offset);
}

/** Dequantize following symmetric quantization scheme on an sve vector.
 *
 * @param[in] pg     Predicate value.
 * @param[in] qv     Input values to be dequantized.
 * @param[in] vscale Vector containing quantization scaling factors.
 *
 * @return Dequantized values in a sve vector
 */
inline svfloat32x4_t svdequantize_z(svbool_t pg, const svint8_t &qv, const svfloat32x4_t vscale)
{
    const svfloat32x4_t vdequantized_input =
    {
        { {
                svmul_f32_z(pg, svcvt_f32_s32_z(pg, svmovlb_s32(svmovlb_s16(qv))), svget4_f32(vscale, 0)),
                svmul_f32_z(pg, svcvt_f32_s32_z(pg, svmovlt_s32(svmovlb_s16(qv))), svget4_f32(vscale, 1)),
                svmul_f32_z(pg, svcvt_f32_s32_z(pg, svmovlb_s32(svmovlt_s16(qv))), svget4_f32(vscale, 2)),
                svmul_f32_z(pg, svcvt_f32_s32_z(pg, svmovlt_s32(svmovlt_s16(qv))), svget4_f32(vscale, 3)),
            }
        }
    };
    return vdequantized_input;
}

/** Dequantize following a symmetric quantization scheme an sve vector.
 *
 * @param[in] qv    Input values to be dequantized.
 * @param[in] scale Quantization scaling factor.
 *
 * @return Dequantized values in a sve vector
 */
inline svfloat32x4_t svdequantize_z(svbool_t pg, const svint8_t &qv, float scale)
{
    const auto          vscale = svdup_n_f32(scale);
    const svfloat32x4_t vdequantized_input =
    {
        { {
                svmul_f32_z(pg, svcvt_f32_s32_z(pg, svmovlb_s32(svmovlb_s16(qv))), vscale),
                svmul_f32_z(pg, svcvt_f32_s32_z(pg, svmovlt_s32(svmovlb_s16(qv))), vscale),
                svmul_f32_z(pg, svcvt_f32_s32_z(pg, svmovlb_s32(svmovlt_s16(qv))), vscale),
                svmul_f32_z(pg, svcvt_f32_s32_z(pg, svmovlt_s32(svmovlt_s16(qv))), vscale),
            }
        }
    };
    return vdequantized_input;
}

/** Quantize an sve vector holding floating point values.
 *
 * @param[in] pg Predicate value.
 * @param[in] qv Input values to be quantized.
 * @param[in] qi Quantization information to be used in the computation.
 *
 * @return An sve vector holding the quantized values
 */
inline svuint8_t svquantize_z(svbool_t pg, const svfloat32x4_t qv, const UniformQuantizationInfo &qi)
{
    const float scale     = qi.scale;
    const int   offset    = qi.offset;
    const auto  voffset   = svdup_n_f32(offset);
    const auto  vinvscale = svdup_n_f32(1.f / scale);

    const auto rf_0 = svcvt_u32_f32_z(pg, svmla_f32_z(pg, voffset, svget4_f32(qv, 0), vinvscale));
    const auto rf_1 = svcvt_u32_f32_z(pg, svmla_f32_z(pg, voffset, svget4_f32(qv, 1), vinvscale));
    const auto rf_2 = svcvt_u32_f32_z(pg, svmla_f32_z(pg, voffset, svget4_f32(qv, 2), vinvscale));
    const auto rf_3 = svcvt_u32_f32_z(pg, svmla_f32_z(pg, voffset, svget4_f32(qv, 3), vinvscale));

    const auto pa = svqxtnt_u32(svqxtnb_u32(rf_0), rf_1);
    const auto pb = svqxtnt_u32(svqxtnb_u32(rf_2), rf_3);

    return svqxtnt_u16(svqxtnb_u16(pa), pb);
}

/** Signed quantize an sve vector holding floating point values.
 *
 * @param[in] pg Predicate value.
 * @param[in] qv Input values to be quantized.
 * @param[in] qi Quantization information to be used in the computation.
 *
 * @return An sve vector holding the quantized values
 */
inline svint8_t svquantize_signed_z(svbool_t pg, const svfloat32x4_t qv, const UniformQuantizationInfo &qi)
{
    const float scale     = qi.scale;
    const int   offset    = qi.offset;
    const auto  voffset   = svdup_n_f32(offset);
    const auto  vinvscale = svdup_n_f32(1.f / scale);
    const auto  rf_0      = svcvt_s32_f32_z(pg, svmla_f32_z(pg, voffset, svget4_f32(qv, 0), vinvscale));
    const auto  rf_1      = svcvt_s32_f32_z(pg, svmla_f32_z(pg, voffset, svget4_f32(qv, 1), vinvscale));
    const auto  rf_2      = svcvt_s32_f32_z(pg, svmla_f32_z(pg, voffset, svget4_f32(qv, 2), vinvscale));
    const auto  rf_3      = svcvt_s32_f32_z(pg, svmla_f32_z(pg, voffset, svget4_f32(qv, 3), vinvscale));

    const auto pa = svqxtnt_s32(svqxtnb_s32(rf_0), rf_1);
    const auto pb = svqxtnt_s32(svqxtnb_s32(rf_2), rf_3);

    return svqxtnt_s16(svqxtnb_s16(pa), pb);
}

/** Quantize to QASYMM16 an sve vector holding 16 floating point values.
 *
 * @param[in] pg Predicate value.
 * @param[in] qv Input values to be quantized.
 * @param[in] qi Quantization information to be used in the computation.
 *
 * @return An sve vector holding the quantized values
 */
inline svuint16x2_t svquantize_qasymm16_z(svbool_t pg, const svfloat32x4_t qv, const UniformQuantizationInfo &qi)
{
    const float scale     = qi.scale;
    const int   offset    = qi.offset;
    const auto  voffset   = svdup_n_f32(offset);
    const auto  vinvscale = svdup_n_f32(1.f / scale);

    const auto rf_0 = svcvt_u32_f32_z(pg, svmla_f32_z(pg, voffset, svget4_f32(qv, 0), vinvscale));
    const auto rf_1 = svcvt_u32_f32_z(pg, svmla_f32_z(pg, voffset, svget4_f32(qv, 1), vinvscale));
    const auto rf_2 = svcvt_u32_f32_z(pg, svmla_f32_z(pg, voffset, svget4_f32(qv, 2), vinvscale));
    const auto rf_3 = svcvt_u32_f32_z(pg, svmla_f32_z(pg, voffset, svget4_f32(qv, 3), vinvscale));

    const auto pa = svqxtnt_u32(svqxtnb_u32(rf_0), rf_1);
    const auto pb = svqxtnt_u32(svqxtnb_u32(rf_2), rf_3);

    return svcreate2_u16(pa, pb);
}
} // namespace arm_compute
#include "src/core/NEON/SVEAsymm.inl"
#endif /* defined(__ARM_FEATURE_SVE2) */
#endif // ARM_COMPUTE_NEASYMM_H