Update cpu depthwise kernels
Resolves: COMPMID-4688
Signed-off-by: Freddie Liardet <frederick.liardet@arm.com>
Change-Id: I9e22f967f5b7ccaebff2fc49f0253f621d62d820
Reviewed-on: https://review.mlplatform.org/c/ml/ComputeLibrary/+/6030
Tested-by: Arm Jenkins <bsgcomp@arm.com>
Reviewed-by: Pablo Marquez Tello <pablo.tello@arm.com>
Reviewed-by: Georgios Pinitas <georgios.pinitas@arm.com>
Comments-Addressed: Arm Jenkins <bsgcomp@arm.com>
diff --git a/Android.bp b/Android.bp
index 20bca36..92248fd 100644
--- a/Android.bp
+++ b/Android.bp
@@ -712,6 +712,7 @@
srcs: [
"src/core/NEON/kernels/arm_conv/depthwise/interleaves/a64_s8q_3x3_dot.cpp",
"src/core/NEON/kernels/arm_conv/depthwise/interleaves/a64_u8q_3x3_dot.cpp",
+ "src/core/NEON/kernels/arm_conv/depthwise/interleaves/sve_8b_mla.cpp",
"src/core/NEON/kernels/arm_conv/depthwise/interleaves/sve_s8q_3x3_dot.cpp",
"src/core/NEON/kernels/arm_conv/depthwise/interleaves/sve_u8q_3x3_dot.cpp",
"src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp",
diff --git a/filelist.json b/filelist.json
index 8aa869a..5171f39 100644
--- a/filelist.json
+++ b/filelist.json
@@ -950,6 +950,7 @@
],
"sve": {
"all": [
+ "src/core/NEON/kernels/arm_conv/depthwise/interleaves/sve_8b_mla.cpp",
"src/core/NEON/kernels/arm_conv/depthwise/interleaves/sve_s8q_3x3_dot.cpp",
"src/core/NEON/kernels/arm_conv/depthwise/interleaves/sve_u8q_3x3_dot.cpp",
"src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp",
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/depthwise_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/depthwise_depthfirst.hpp
index 53ad5b5..8d1a869 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/depthwise_depthfirst.hpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/depthwise_depthfirst.hpp
@@ -35,19 +35,61 @@
namespace arm_conv {
namespace depthwise {
-template <class strategy>
-class DepthwiseDepthfirst : public DepthwiseCommon<typename strategy::input_type,
- typename strategy::weight_type,
- typename strategy::return_type>
+struct IDepthwiseDepthfirstStrategy
{
- using TInput = typename strategy::input_type;
- using TWeight = typename strategy::weight_type;
- using TOutput = typename strategy::return_type;
- using TAccum = typename strategy::bias_type;
+ virtual arm_gemm::VLType get_vl_type() const = 0;
+
+ virtual unsigned int get_input_rows() const = 0;
+ virtual unsigned int get_input_cols() const = 0;
+
+ virtual unsigned int get_output_rows() const = 0;
+ virtual unsigned int get_output_cols() const = 0;
+
+ virtual unsigned int get_kernel_rows() const = 0;
+ virtual unsigned int get_kernel_cols() const = 0;
+
+ virtual unsigned int get_stride_rows() const = 0;
+ virtual unsigned int get_stride_cols() const = 0;
+
+ virtual void indirect_kernel(
+ const void *const *const input_ptrs,
+ void *const *const output_ptrs,
+ const void *params,
+ unsigned int n_channels,
+ const void *activation_min,
+ const void *activation_max
+ ) const = 0;
+
+ virtual void direct_kernel(
+ const unsigned int n_tile_rows, const unsigned int n_tile_cols,
+ const void *inptr, int64_t ld_input_row, int64_t ld_input_col,
+ void *outptr, int64_t ld_output_row, int64_t ld_output_col,
+ const void *params, unsigned int n_channels,
+ const void *activation_min,
+ const void *activation_max
+ ) const = 0;
+
+ virtual ~IDepthwiseDepthfirstStrategy() {}
+};
+
+template <typename TInput, typename TWeight, typename TOutput, typename TAccum>
+class DepthwiseDepthfirst : public DepthwiseCommon<TInput, TWeight, TOutput>
+{
+ const std::unique_ptr<IDepthwiseDepthfirstStrategy> m_strat;
+
+ size_t sizeof_inptr_array(void) const
+ {
+ return sizeof(TInput *) * m_strat->get_input_rows() * m_strat->get_input_cols();
+ }
size_t sizeof_input_buffer(unsigned int n_input_channels) const
{
- return sizeof(TInput) * n_input_channels;
+ return sizeof(TInput) * n_input_channels;
+ }
+
+ size_t sizeof_outptr_array(void) const
+ {
+ return sizeof(TInput *) * m_strat->get_output_rows() * m_strat->get_output_cols();
}
size_t sizeof_output_buffer(unsigned int n_output_channels) const
@@ -56,8 +98,10 @@
}
public:
-
- DepthwiseDepthfirst(const DepthwiseArgs &args) : DepthwiseCommon<TInput, TWeight, TOutput>(args)
+ DepthwiseDepthfirst(
+ IDepthwiseDepthfirstStrategy *const strat,
+ const DepthwiseArgs &args
+ ) : DepthwiseCommon<TInput, TWeight, TOutput>(args), m_strat(strat)
{
}
@@ -67,7 +111,7 @@
size_t get_storage_size(void) const override
{
// TODO What if we insert extra padding? Biases are a different size to the inputs, ...
- const unsigned int vl = arm_gemm::utils::get_vector_length<TInput>(strategy::vl_type);
+ const unsigned int vl = arm_gemm::utils::get_vector_length<TInput>(m_strat->get_vl_type());
const auto rounded_channels = arm_gemm::roundup(this->m_args.input_channels, vl);
return (1 + this->m_args.kernel_rows * this->m_args.kernel_cols) * rounded_channels * sizeof(TWeight);
}
@@ -81,7 +125,7 @@
const TAccum *biases = static_cast<const TAccum *>(_biases);
const TWeight *const weights = static_cast<const TWeight *>(_weights);
- const unsigned int vl = arm_gemm::utils::get_vector_length<TAccum>(strategy::vl_type);
+ const unsigned int vl = arm_gemm::utils::get_vector_length<TAccum>(m_strat->get_vl_type());
ld_weight_col = (ld_weight_col == 0) ? this->m_args.input_channels : ld_weight_col;
ld_weight_row = (ld_weight_row == 0) ? this->m_args.kernel_cols * ld_weight_col : ld_weight_row;
@@ -121,10 +165,12 @@
size_t get_working_size(const unsigned int n_threads, const unsigned int n_channels) const override
{
const unsigned int n_output_channels = n_channels * this->m_args.channel_multiplier;
- return n_threads * (sizeof_output_buffer(n_output_channels) + sizeof_input_buffer(n_channels));
+ return n_threads * (sizeof_inptr_array() + sizeof_outptr_array() +
+ sizeof_output_buffer(n_output_channels) +
+ sizeof_input_buffer(n_channels));
}
- using DepthwiseCommon<typename strategy::input_type, typename strategy::weight_type, typename strategy::return_type>::execute;
+ using DepthwiseCommon<TInput, TWeight, TOutput>::execute;
void execute(
const unsigned int batches,
const unsigned int input_height,
@@ -147,7 +193,6 @@
const unsigned int n_threads
) const override
{
- strategy strat(this->m_args.cpu_info);
#ifdef CYCLE_PROFILING
arm_gemm::profiler prof;
#endif
@@ -177,18 +222,19 @@
const TInput *const inptr = static_cast<const TInput *>(_input);
TOutput *const outptr = static_cast<TOutput *>(_output);
- // Create an array for the input pointers
- const TInput * _inptr_array[strategy::input_rows * strategy::input_cols];
- const TInput **const inptr_array = _inptr_array;
-
- // Create an array for the output pointers
- TOutput * _outptr_array[strategy::output_rows * strategy::output_cols];
- TOutput **const outptr_array = _outptr_array;
-
// Allocate portions of the working space
- uint8_t *const working_space = static_cast<uint8_t *>(_working_space) + get_working_size(thread_id, input_channels);
+ uint8_t *working_space = static_cast<uint8_t *>(_working_space) + get_working_size(thread_id, input_channels);
+
+ const void **const inptr_array = reinterpret_cast<const void **>(working_space);
+ working_space += sizeof_inptr_array();
+
+ void **const outptr_array = reinterpret_cast<void **>(working_space);
+ working_space += sizeof_outptr_array();
+
TOutput *const output_buffer = reinterpret_cast<TOutput *>(working_space);
- TInput *const input_buffer = reinterpret_cast<TInput *>(working_space + sizeof_output_buffer(input_channels * this->m_args.channel_multiplier));
+ working_space += sizeof_output_buffer(input_channels * this->m_args.channel_multiplier);
+
+ TInput *const input_buffer = reinterpret_cast<TInput *>(working_space);
// Initialise the input buffer
for (unsigned int c = 0; c < input_channels; c++)
@@ -206,11 +252,11 @@
for (int start_out_i = start_out_height;
start_out_i < end_out_height;
- start_out_i += static_cast<int>(strategy::output_rows))
+ start_out_i += static_cast<int>(m_strat->get_output_rows()))
{
- const int end_out_i = start_out_i + strategy::output_rows;
- const int start_in_i = start_out_i * strategy::stride_rows - padding.top;
- const int end_in_i = start_in_i + strategy::input_rows;
+ const int end_out_i = start_out_i + m_strat->get_output_rows();
+ const int start_in_i = start_out_i * m_strat->get_stride_rows() - padding.top;
+ const int end_in_i = start_in_i + m_strat->get_input_rows();
// Compute top/bottom padding
const auto pad_top = static_cast<unsigned int>(-std::min(start_in_i, 0));
@@ -221,14 +267,14 @@
);
// Fill the input pointer array with padding values
- for (auto index = 0u; index < strategy::input_rows * strategy::input_cols; index++)
+ for (auto index = 0u; index < m_strat->get_input_rows() * m_strat->get_input_cols(); index++)
{
inptr_array[index] = input_buffer;
}
for (int start_out_j = 0; start_out_j < static_cast<int>(output_width);)
{
- const int start_in_j = start_out_j * strategy::stride_cols - this->m_args.padding.left;
+ const int start_in_j = start_out_j * m_strat->get_stride_cols() - this->m_args.padding.left;
const int pad_left = -std::min(0, start_in_j);
// Compute how many output tiles we can compute with the direct kernel.
@@ -236,17 +282,17 @@
if (!pad_top && !pad_bottom && !pad_left)
{
// Determine the maximum number of tiles we could handle.
- n_direct_tiles = (output_width - start_out_j) / strategy::output_cols;
+ n_direct_tiles = (output_width - start_out_j) / m_strat->get_output_cols();
// Continue to reduce this number as required to avoid reading
// padding on the right edge.
- int end_in_j = start_in_j + n_direct_tiles * strategy::input_cols;
+ int end_in_j = start_in_j + n_direct_tiles * m_strat->get_input_cols();
int pad_right = std::max(0, end_in_j - static_cast<int>(input_width));
while (pad_right && n_direct_tiles)
{
n_direct_tiles--;
- end_in_j -= strategy::input_cols;
+ end_in_j -= m_strat->get_input_cols();
pad_right = std::max(0, end_in_j - static_cast<int>(input_width));
}
}
@@ -256,21 +302,21 @@
{
auto inptr = inptr_batch + start_in_i*ld_input_row + start_in_j*ld_input_col;
auto outptr = outptr_batch + start_out_i*ld_output_row + start_out_j*ld_output_col;
- start_out_j += n_direct_tiles*strategy::output_cols;
+ start_out_j += n_direct_tiles*m_strat->get_output_cols();
#ifdef CYCLE_PROFILING
auto p = prof.ScopedProfiler(PROFILE_KERNEL, 0);
#endif
- strat.direct_kernel(1, n_direct_tiles,
- inptr, ld_input_row, ld_input_col,
- outptr, ld_output_row, ld_output_col,
- parameters, this->m_args.input_channels,
- activation_min, activation_max);
+ m_strat->direct_kernel(1, n_direct_tiles,
+ inptr, ld_input_row, ld_input_col,
+ outptr, ld_output_row, ld_output_col,
+ parameters, this->m_args.input_channels,
+ &activation_min, &activation_max);
continue;
}
- const int end_out_j = start_out_j + strategy::output_cols;
- const int end_in_j = start_in_j + strategy::input_cols;
+ const int end_out_j = start_out_j + m_strat->get_output_cols();
+ const int end_in_j = start_in_j + m_strat->get_input_cols();
const auto pad_right = static_cast<unsigned int>(-std::min(static_cast<int>(input_width) - end_in_j, 0));
const unsigned int valid_output_cols = std::min(
@@ -280,26 +326,26 @@
// Construct the input pointer array - fill the array with pointers to
// the input buffer and then fill in the required values.
- for (auto i = pad_top; i < strategy::input_rows - pad_bottom; i++)
+ for (auto i = pad_top; i < m_strat->get_input_rows() - pad_bottom; i++)
{
// Can skip over the left padding because we will have either the
// same or less than the previous tile.
unsigned int j = pad_left;
const TInput *colptr = inptr_batch + (start_in_i + i) * ld_input_row + (start_in_j + j) * ld_input_col;
- const TInput **ptrs = inptr_array + i * strategy::input_cols + j;
- for (; j < strategy::input_cols - pad_right; j++)
+ const void **ptrs = inptr_array + i * m_strat->get_input_cols() + j;
+ for (; j < m_strat->get_input_cols() - pad_right; j++)
{
*(ptrs++) = colptr;
colptr += ld_input_col;
}
- for (; j < strategy::input_cols; j++)
+ for (; j < m_strat->get_input_cols(); j++)
{
*(ptrs++) = input_buffer;
}
}
// Construct the output pointer array.
- TOutput **outptr_pos = outptr_array;
+ void **outptr_pos = outptr_array;
for (auto i = 0u; i < valid_output_rows; i++)
{
unsigned int j = 0u;
@@ -309,27 +355,28 @@
*(outptr_pos++) = colptr;
colptr += ld_output_col;
}
- for (; j < strategy::output_cols; j++)
+ for (; j < m_strat->get_output_cols(); j++)
{
*(outptr_pos++) = output_buffer;
}
}
- for (auto i = valid_output_rows; i < strategy::output_rows; i++)
+ for (auto i = valid_output_rows; i < m_strat->get_output_rows(); i++)
{
- for (auto j = 0u; j < strategy::output_cols; j++)
+ for (auto j = 0u; j < m_strat->get_output_cols(); j++)
{
*(outptr_pos++) = output_buffer;
}
}
- start_out_j += strategy::output_cols;
+ start_out_j += m_strat->get_output_cols();
#ifdef CYCLE_PROFILING
// TODO Work number
auto p = prof.ScopedProfiler(PROFILE_KERNEL, (unsigned long)(0));
#endif
- strat.indirect_kernel(inptr_array, outptr_array, parameters,
- this->m_args.input_channels, activation_min, activation_max);
+ m_strat->indirect_kernel(inptr_array, outptr_array, parameters,
+ this->m_args.input_channels,
+ &activation_min, &activation_max);
}
}
}
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/depthwise_fp16.cpp b/src/core/NEON/kernels/arm_conv/depthwise/depthwise_fp16.cpp
index 6ba7c78..934272a 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/depthwise_fp16.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/depthwise_fp16.cpp
@@ -32,6 +32,9 @@
#include "depthwise_implementation_constraints.hpp"
+// This can only be built if the target/compiler supports FP16 arguments.
+#if defined(__ARM_FP16_ARGS)
+
#if defined(__aarch64__)
#if defined(ARM_COMPUTE_ENABLE_SVE)
#include "kernels/sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp"
@@ -76,11 +79,9 @@
{
return args.channel_multiplier > 1 ? 0 : std::numeric_limits<unsigned int>::max();
}
-#endif // defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
+#endif // defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
}
-#if defined(__ARM_FP16_ARGS)
-
static const DepthwiseImplementation<__fp16, __fp16> depthwise_fp16_methods[] = {
#if defined(__aarch64__)
#if defined(ARM_COMPUTE_ENABLE_SVE)
@@ -92,7 +93,8 @@
cpu_has_sve),
cycle_estimate<sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst>,
[] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon<__fp16, __fp16, __fp16> * {
- return new DepthwiseDepthfirst<sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst>(args);
+ auto strat = new sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst(args.cpu_info);
+ return new DepthwiseDepthfirst<__fp16, __fp16, __fp16, __fp16>(strat, args);
},
},
{
@@ -103,7 +105,8 @@
cpu_has_sve),
cycle_estimate<sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst>,
[] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon<__fp16, __fp16, __fp16> * {
- return new DepthwiseDepthfirst<sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst>(args);
+ auto strat = new sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst(args.cpu_info);
+ return new DepthwiseDepthfirst<__fp16, __fp16, __fp16, __fp16>(strat, args);
},
},
{
@@ -114,7 +117,8 @@
cpu_has_sve),
cycle_estimate<sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst>,
[] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon<__fp16, __fp16, __fp16> * {
- return new DepthwiseDepthfirst<sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst>(args);
+ auto strat = new sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst(args.cpu_info);
+ return new DepthwiseDepthfirst<__fp16, __fp16, __fp16, __fp16>(strat, args);
},
},
{
@@ -125,7 +129,8 @@
cpu_has_sve),
cycle_estimate<sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst>,
[] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon<__fp16, __fp16, __fp16> * {
- return new DepthwiseDepthfirst<sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst>(args);
+ auto strat = new sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst(args.cpu_info);
+ return new DepthwiseDepthfirst<__fp16, __fp16, __fp16, __fp16>(strat, args);
},
},
{
@@ -136,7 +141,8 @@
cpu_has_sve),
cycle_estimate<sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst>,
[] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon<__fp16, __fp16, __fp16> * {
- return new DepthwiseDepthfirst<sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst>(args);
+ auto strat = new sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst(args.cpu_info);
+ return new DepthwiseDepthfirst<__fp16, __fp16, __fp16, __fp16>(strat, args);
},
},
#endif // defined(ARM_COMPUTE_ENABLE_SVE)
@@ -145,56 +151,66 @@
DepthwiseMethod::DEPTHFIRST,
"a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst",
constraint(is_supported<a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst>,
- has_no_channel_multiplier),
+ has_no_channel_multiplier,
+ cpu_has_fp16),
cycle_estimate<a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst>,
[] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon<__fp16, __fp16, __fp16> * {
- return new DepthwiseDepthfirst<a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst>(args);
+ auto strat = new a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst(args.cpu_info);
+ return new DepthwiseDepthfirst<__fp16, __fp16, __fp16, __fp16>(strat, args);
},
},
{
DepthwiseMethod::DEPTHFIRST,
"a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst",
constraint(is_supported<a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst>,
- has_no_channel_multiplier),
+ has_no_channel_multiplier,
+ cpu_has_fp16),
cycle_estimate<a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst>,
[] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon<__fp16, __fp16, __fp16> * {
- return new DepthwiseDepthfirst<a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst>(args);
+ auto strat = new a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst(args.cpu_info);
+ return new DepthwiseDepthfirst<__fp16, __fp16, __fp16, __fp16>(strat, args);
},
},
{
DepthwiseMethod::DEPTHFIRST,
"a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst",
constraint(is_supported<a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst>,
- has_no_channel_multiplier),
+ has_no_channel_multiplier,
+ cpu_has_fp16),
cycle_estimate<a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst>,
[] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon<__fp16, __fp16, __fp16> * {
- return new DepthwiseDepthfirst<a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst>(args);
+ auto strat = new a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst(args.cpu_info);
+ return new DepthwiseDepthfirst<__fp16, __fp16, __fp16, __fp16>(strat, args);
},
},
{
DepthwiseMethod::DEPTHFIRST,
"a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst",
constraint(is_supported<a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst>,
- has_no_channel_multiplier),
+ has_no_channel_multiplier,
+ cpu_has_fp16),
cycle_estimate<a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst>,
[] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon<__fp16, __fp16, __fp16> * {
- return new DepthwiseDepthfirst<a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst>(args);
+ auto strat = new a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst(args.cpu_info);
+ return new DepthwiseDepthfirst<__fp16, __fp16, __fp16, __fp16>(strat, args);
},
},
{
DepthwiseMethod::DEPTHFIRST,
"a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst",
constraint(is_supported<a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst>,
- has_no_channel_multiplier),
+ has_no_channel_multiplier,
+ cpu_has_fp16),
cycle_estimate<a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst>,
[] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon<__fp16, __fp16, __fp16> * {
- return new DepthwiseDepthfirst<a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst>(args);
+ auto strat = new a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst(args.cpu_info);
+ return new DepthwiseDepthfirst<__fp16, __fp16, __fp16, __fp16>(strat, args);
},
},
{
DepthwiseMethod::DEPTHFIRST,
"a64_fp16_nhwc_generic_output3x3_mla_depthfirst",
- constraint(has_no_channel_multiplier),
+ constraint(has_no_channel_multiplier, cpu_has_fp16),
not_preferred,
[] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon<__fp16, __fp16, __fp16> * {
return new DepthwiseDepthfirstGeneric<a64_fp16_nhwc_generic_output9_mla_depthfirst, 3, 3>(args);
@@ -203,7 +219,7 @@
{
DepthwiseMethod::DEPTHFIRST,
"a64_fp16_nhwc_generic_with_multiplier_output2x8_mla_depthfirst",
- nullptr,
+ constraint(cpu_has_fp16),
not_preferred_if_no_multiplier,
[] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon<__fp16, __fp16, __fp16> * {
return new DepthwiseDepthfirstGenericWithMultiplier<a64_fp16_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst>(args);
@@ -223,7 +239,7 @@
template UniqueDepthwiseCommon<__fp16> depthwise(const DepthwiseArgs &, const Nothing &);
template std::vector<KernelDescription> get_compatible_kernels<__fp16>(const DepthwiseArgs &, const Nothing &);
-#endif // defined(__ARM_FP16_ARGS)
-
} // namespace depthwise
} // namespace arm_conv
+
+#endif // defined(__ARM_FP16_ARGS)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/depthwise_fp32.cpp b/src/core/NEON/kernels/arm_conv/depthwise/depthwise_fp32.cpp
index ac43df9..7a26ba4 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/depthwise_fp32.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/depthwise_fp32.cpp
@@ -94,7 +94,8 @@
cpu_has_sve),
cycle_estimate<sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst>,
[] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon<float, float, float> * {
- return new DepthwiseDepthfirst<sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst>(args);
+ auto strat = new sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst(args.cpu_info);
+ return new DepthwiseDepthfirst<float, float, float, float>(strat, args);
},
},
{
@@ -105,7 +106,8 @@
cpu_has_sve),
cycle_estimate<sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst>,
[] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon<float, float, float> * {
- return new DepthwiseDepthfirst<sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst>(args);
+ auto strat = new sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst(args.cpu_info);
+ return new DepthwiseDepthfirst<float, float, float, float>(strat, args);
},
},
{
@@ -116,7 +118,8 @@
cpu_has_sve),
cycle_estimate<sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst>,
[] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon<float, float, float> * {
- return new DepthwiseDepthfirst<sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst>(args);
+ auto strat = new sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst(args.cpu_info);
+ return new DepthwiseDepthfirst<float, float, float, float>(strat, args);
},
},
{
@@ -127,7 +130,8 @@
cpu_has_sve),
cycle_estimate<sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst>,
[] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon<float, float, float> * {
- return new DepthwiseDepthfirst<sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst>(args);
+ auto strat = new sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst(args.cpu_info);
+ return new DepthwiseDepthfirst<float, float, float, float>(strat, args);
},
},
{
@@ -138,7 +142,8 @@
cpu_has_sve),
cycle_estimate<sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst>,
[] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon<float, float, float> * {
- return new DepthwiseDepthfirst<sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst>(args);
+ auto strat = new sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst(args.cpu_info);
+ return new DepthwiseDepthfirst<float, float, float, float>(strat, args);
},
},
{
@@ -153,7 +158,8 @@
{
DepthwiseMethod::DEPTHFIRST,
"sve_fp32_nhwc_3x3_s2_with_multiplier_output3x3_mla_depthfirst",
- constraint(is_supported<sve_fp32_packed_to_nhwc_3x3_s2_with_multiplier_output3x3_mla_depthfirst>, cpu_has_sve),
+ constraint(is_supported<sve_fp32_packed_to_nhwc_3x3_s2_with_multiplier_output3x3_mla_depthfirst>,
+ cpu_has_sve),
not_preferred_if_no_multiplier,
[] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon<float, float, float> * {
return new DepthwiseDepthfirstWithMultiplier<sve_fp32_packed_to_nhwc_3x3_s2_with_multiplier_output3x3_mla_depthfirst>(args);
@@ -162,7 +168,8 @@
{
DepthwiseMethod::DEPTHFIRST,
"sve_fp32_nhwc_5x5_s1_with_multiplier_output2x4_mla_depthfirst",
- constraint(is_supported<sve_fp32_packed_to_nhwc_5x5_s1_with_multiplier_output2x4_mla_depthfirst>, cpu_has_sve),
+ constraint(is_supported<sve_fp32_packed_to_nhwc_5x5_s1_with_multiplier_output2x4_mla_depthfirst>,
+ cpu_has_sve),
not_preferred_if_no_multiplier,
[] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon<float, float, float> * {
return new DepthwiseDepthfirstWithMultiplier<sve_fp32_packed_to_nhwc_5x5_s1_with_multiplier_output2x4_mla_depthfirst>(args);
@@ -185,7 +192,8 @@
has_no_channel_multiplier),
cycle_estimate<a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst>,
[] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon<float, float, float> * {
- return new DepthwiseDepthfirst<a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst>(args);
+ auto strat = new a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst(args.cpu_info);
+ return new DepthwiseDepthfirst<float, float, float, float>(strat, args);
},
},
{
@@ -195,17 +203,19 @@
has_no_channel_multiplier),
cycle_estimate<a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst>,
[] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon<float, float, float> * {
- return new DepthwiseDepthfirst<a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst>(args);
+ auto strat = new a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst(args.cpu_info);
+ return new DepthwiseDepthfirst<float, float, float, float>(strat, args);
},
},
{
DepthwiseMethod::DEPTHFIRST,
"a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst",
constraint(is_supported<a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst>,
- has_no_channel_multiplier),
+ has_no_channel_multiplier),
cycle_estimate<a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst>,
[] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon<float, float, float> * {
- return new DepthwiseDepthfirst<a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst>(args);
+ auto strat = new a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst(args.cpu_info);
+ return new DepthwiseDepthfirst<float, float, float, float>(strat, args);
},
},
{
@@ -215,7 +225,8 @@
has_no_channel_multiplier),
cycle_estimate<a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst>,
[] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon<float, float, float> * {
- return new DepthwiseDepthfirst<a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst>(args);
+ auto strat = new a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst(args.cpu_info);
+ return new DepthwiseDepthfirst<float, float, float, float>(strat, args);
},
},
{
@@ -225,7 +236,8 @@
has_no_channel_multiplier),
cycle_estimate<a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst>,
[] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon<float, float, float> * {
- return new DepthwiseDepthfirst<a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst>(args);
+ auto strat = new a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst(args.cpu_info);
+ return new DepthwiseDepthfirst<float, float, float, float>(strat, args);
},
},
{
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/depthwise_implementation_constraints.hpp b/src/core/NEON/kernels/arm_conv/depthwise/depthwise_implementation_constraints.hpp
index 6526d00..4198727 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/depthwise_implementation_constraints.hpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/depthwise_implementation_constraints.hpp
@@ -33,7 +33,7 @@
#pragma once
#include "arm_gemm.hpp"
-#include "depthwise.hpp"
+#include "src/core/NEON/kernels/assembly/depthwise.hpp"
namespace arm_conv
{
@@ -97,6 +97,12 @@
return args.cpu_info->has_sve2();
}
+bool cpu_has_fp16(const DepthwiseArgs &args, const void *) __attribute__ ((unused));
+bool cpu_has_fp16(const DepthwiseArgs &args, const void *)
+{
+ return args.cpu_info->has_fp16();
+}
+
bool has_no_channel_multiplier(const DepthwiseArgs &args, const void *) __attribute__ ((unused));
bool has_no_channel_multiplier(const DepthwiseArgs &args, const void *)
{
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/interleaves/8b_mla.cpp b/src/core/NEON/kernels/arm_conv/depthwise/interleaves/8b_mla.cpp
index 04b9042..d59d6b7 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/interleaves/8b_mla.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/interleaves/8b_mla.cpp
@@ -22,13 +22,7 @@
* SOFTWARE.
*/
-#include "arm_gemm.hpp"
-#include "src/core/NEON/kernels/arm_gemm/utils.hpp"
-#include "src/core/NEON/kernels/assembly/depthwise.hpp"
-#include <cstdint>
-#include <cstring>
-
-using namespace arm_gemm;
+#include "8b_mla.hpp"
size_t generic_get_packed_size(
const VLType vec_type,
@@ -84,41 +78,9 @@
}
}
-
-#define ADD_IMPLEMENTATION(ARCH, TYPENAME, TYPE, VEC_TYPE, ACC_DEPTH, KERN_ROWS, KERN_COLS) \
-struct interleave_ ## ARCH ## _ ## TYPENAME ## _ ## KERN_ROWS ## x ## KERN_COLS ## _mla \
-{ \
- static size_t get_packed_size(const DepthwiseArgs &args); \
- static void pack_parameters( \
- unsigned int n_channels, void *outptr, \
- const TYPE *weights, size_t ld_weight_col, size_t ld_weight_row \
- ); \
-}; \
-\
-size_t interleave_ ## ARCH ## _ ## TYPENAME ## _ ## KERN_ROWS ## x ## KERN_COLS ## _mla::get_packed_size(const DepthwiseArgs &args) \
-{ \
- return generic_get_packed_size(VLType::VEC_TYPE, ACC_DEPTH, KERN_ROWS, KERN_COLS, args.input_channels); \
-} \
-\
-void interleave_ ## ARCH ## _ ## TYPENAME ## _ ## KERN_ROWS ## x ## KERN_COLS ## _mla::pack_parameters(unsigned int n_channels, void *outptr, \
- const TYPE *weights, size_t ld_weight_col, size_t ld_weight_row) \
-{ \
- generic_pack(VLType::VEC_TYPE, ACC_DEPTH, KERN_ROWS, KERN_COLS, n_channels, outptr, weights, ld_weight_col, ld_weight_row); \
-}
-
-
namespace arm_conv {
namespace depthwise {
-#if defined(ARM_COMPUTE_ENABLE_SVE)
-
-ADD_IMPLEMENTATION(sve, s8q, int8_t, SVE, 2, 3, 3)
-ADD_IMPLEMENTATION(sve, s8q, int8_t, SVE, 2, 5, 5)
-ADD_IMPLEMENTATION(sve, u8q, uint8_t, SVE, 2, 3, 3)
-ADD_IMPLEMENTATION(sve, u8q, uint8_t, SVE, 2, 5, 5)
-
-#endif // defined(ARM_COMPUTE_ENABLE_SVE)
-
ADD_IMPLEMENTATION(a64, s8q, int8_t, None, 2, 3, 3)
ADD_IMPLEMENTATION(a64, s8q, int8_t, None, 2, 5, 5)
ADD_IMPLEMENTATION(a64, u8q, uint8_t, None, 2, 3, 3)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/interleaves/8b_mla.hpp b/src/core/NEON/kernels/arm_conv/depthwise/interleaves/8b_mla.hpp
new file mode 100644
index 0000000..3176d1d
--- /dev/null
+++ b/src/core/NEON/kernels/arm_conv/depthwise/interleaves/8b_mla.hpp
@@ -0,0 +1,72 @@
+/*
+ * Copyright (c) 2021 Arm Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "arm_gemm.hpp"
+#include "src/core/NEON/kernels/arm_gemm/utils.hpp"
+#include "src/core/NEON/kernels/assembly/depthwise.hpp"
+#include <cstdint>
+#include <cstring>
+
+using namespace arm_gemm;
+
+size_t generic_get_packed_size(
+ const VLType vec_type,
+ const unsigned int acc_depth,
+ const unsigned int kernel_rows,
+ const unsigned int kernel_cols,
+ const unsigned int n_input_channels
+);
+
+void generic_pack(
+ const VLType vec_type,
+ const unsigned int acc_depth,
+ const unsigned int kernel_rows,
+ const unsigned int kernel_cols,
+ const unsigned int n_channels,
+ void *_outptr,
+ const void *_weights,
+ size_t ld_weight_col,
+ size_t ld_weight_row
+);
+
+#define ADD_IMPLEMENTATION(ARCH, TYPENAME, TYPE, VEC_TYPE, ACC_DEPTH, KERN_ROWS, KERN_COLS) \
+struct interleave_ ## ARCH ## _ ## TYPENAME ## _ ## KERN_ROWS ## x ## KERN_COLS ## _mla \
+{ \
+ static size_t get_packed_size(const DepthwiseArgs &args); \
+ static void pack_parameters( \
+ unsigned int n_channels, void *outptr, \
+ const TYPE *weights, size_t ld_weight_col, size_t ld_weight_row \
+ ); \
+}; \
+\
+size_t interleave_ ## ARCH ## _ ## TYPENAME ## _ ## KERN_ROWS ## x ## KERN_COLS ## _mla::get_packed_size(const DepthwiseArgs &args) \
+{ \
+ return generic_get_packed_size(VLType::VEC_TYPE, ACC_DEPTH, KERN_ROWS, KERN_COLS, args.input_channels); \
+} \
+\
+void interleave_ ## ARCH ## _ ## TYPENAME ## _ ## KERN_ROWS ## x ## KERN_COLS ## _mla::pack_parameters(unsigned int n_channels, void *outptr, \
+ const TYPE *weights, size_t ld_weight_col, size_t ld_weight_row) \
+{ \
+ generic_pack(VLType::VEC_TYPE, ACC_DEPTH, KERN_ROWS, KERN_COLS, n_channels, outptr, weights, ld_weight_col, ld_weight_row); \
+}
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/interleaves/sve_8b_mla.cpp b/src/core/NEON/kernels/arm_conv/depthwise/interleaves/sve_8b_mla.cpp
new file mode 100644
index 0000000..de74ca5
--- /dev/null
+++ b/src/core/NEON/kernels/arm_conv/depthwise/interleaves/sve_8b_mla.cpp
@@ -0,0 +1,40 @@
+/*
+ * Copyright (c) 2021 Arm Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "8b_mla.hpp"
+
+namespace arm_conv {
+namespace depthwise {
+
+#if defined(__ARM_FEATURE_SVE)
+
+ADD_IMPLEMENTATION(sve, s8q, int8_t, SVE, 2, 3, 3)
+ADD_IMPLEMENTATION(sve, s8q, int8_t, SVE, 2, 5, 5)
+ADD_IMPLEMENTATION(sve, u8q, uint8_t, SVE, 2, 3, 3)
+ADD_IMPLEMENTATION(sve, u8q, uint8_t, SVE, 2, 5, 5)
+
+#endif // defined(__ARM_FEATURE_SVE)
+
+} // namespace depthwise
+} // namespace arm_conv
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp
index bb43d57..be50d1c 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp
@@ -28,7 +28,7 @@
#pragma once
-#if defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
+#if defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
namespace arm_conv {
namespace depthwise {
@@ -36,15 +36,17 @@
void a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16);
void a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16);
-struct a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst
+class a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst : public IDepthwiseDepthfirstStrategy
{
- typedef __fp16 bias_type;
- typedef __fp16 input_type;
- typedef __fp16 weight_type;
- typedef __fp16 return_type;
-
+ private:
typedef void (*indirect_kern_type)(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16);
+ indirect_kern_type m_indirect_kernel = a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl;
+
typedef void (*direct_kern_type)(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16);
+ direct_kern_type m_direct_kernel = a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl;
+
+ public:
+ typedef __fp16 return_type;
constexpr static arm_gemm::VLType vl_type = arm_gemm::VLType::None;
@@ -60,13 +62,60 @@
constexpr static unsigned int input_rows = 4;
constexpr static unsigned int input_cols = 4;
- indirect_kern_type indirect_kernel = a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl;
- direct_kern_type direct_kernel = a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl;
-
a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst(const CPUInfo *) {}
+
+ arm_gemm::VLType get_vl_type(void) const override { return vl_type; }
+
+ unsigned int get_kernel_rows(void) const override { return kernel_rows; }
+ unsigned int get_kernel_cols(void) const override { return kernel_cols; }
+
+ unsigned int get_stride_rows(void) const override { return stride_rows; }
+ unsigned int get_stride_cols(void) const override { return stride_cols; }
+
+ unsigned int get_output_rows(void) const override { return output_rows; }
+ unsigned int get_output_cols(void) const override { return output_cols; }
+
+ unsigned int get_input_rows(void) const override { return input_rows; }
+ unsigned int get_input_cols(void) const override { return input_cols; }
+
+ void indirect_kernel(
+ const void *const *const input_ptrs,
+ void *const *const outptrs,
+ const void *params,
+ unsigned int n_channels,
+ const void *activation_min,
+ const void *activation_max
+ ) const override
+ {
+ m_indirect_kernel(
+ reinterpret_cast<const __fp16 *const *>(input_ptrs),
+ reinterpret_cast<__fp16 *const *>(outptrs),
+ params, n_channels,
+ *static_cast<const __fp16 *>(activation_min),
+ *static_cast<const __fp16 *>(activation_max)
+ );
+ }
+
+ void direct_kernel(
+ const unsigned int n_tile_rows, const unsigned int n_tile_cols,
+ const void *inptr, int64_t ld_input_row, int64_t ld_input_col,
+ void *outptr, int64_t ld_output_row, int64_t ld_output_col,
+ const void *params, unsigned int n_channels,
+ const void *activation_min, const void *activation_max
+ ) const override
+ {
+ m_direct_kernel(
+ n_tile_rows, n_tile_cols,
+ static_cast<const __fp16 *>(inptr), ld_input_row, ld_input_col,
+ static_cast<__fp16 *>(outptr), ld_output_row, ld_output_col,
+ params, n_channels,
+ *static_cast<const __fp16 *>(activation_min),
+ *static_cast<const __fp16 *>(activation_max)
+ );
+ }
};
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
+#endif // defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp
index 143070c..a85e443 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp
@@ -25,7 +25,7 @@
#include <cstddef>
#include <cstdint>
-#if defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
+#if defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
namespace arm_conv {
namespace depthwise {
@@ -721,4 +721,4 @@
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
+#endif // defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp
index 35a9714..a0a4499 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp
@@ -25,7 +25,7 @@
#include <cstddef>
#include <cstdint>
-#if defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
+#if defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
namespace arm_conv {
namespace depthwise {
@@ -696,4 +696,4 @@
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
+#endif // defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp
index 90db870..39fa7f6 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp
@@ -28,7 +28,7 @@
#pragma once
-#if defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
+#if defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
namespace arm_conv {
namespace depthwise {
@@ -36,15 +36,17 @@
void a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16);
void a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16);
-struct a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst
+class a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst : public IDepthwiseDepthfirstStrategy
{
- typedef __fp16 bias_type;
- typedef __fp16 input_type;
- typedef __fp16 weight_type;
- typedef __fp16 return_type;
-
+ private:
typedef void (*indirect_kern_type)(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16);
+ indirect_kern_type m_indirect_kernel = a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl;
+
typedef void (*direct_kern_type)(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16);
+ direct_kern_type m_direct_kernel = a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl;
+
+ public:
+ typedef __fp16 return_type;
constexpr static arm_gemm::VLType vl_type = arm_gemm::VLType::None;
@@ -60,13 +62,60 @@
constexpr static unsigned int input_rows = 5;
constexpr static unsigned int input_cols = 5;
- indirect_kern_type indirect_kernel = a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl;
- direct_kern_type direct_kernel = a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl;
-
a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst(const CPUInfo *) {}
+
+ arm_gemm::VLType get_vl_type(void) const override { return vl_type; }
+
+ unsigned int get_kernel_rows(void) const override { return kernel_rows; }
+ unsigned int get_kernel_cols(void) const override { return kernel_cols; }
+
+ unsigned int get_stride_rows(void) const override { return stride_rows; }
+ unsigned int get_stride_cols(void) const override { return stride_cols; }
+
+ unsigned int get_output_rows(void) const override { return output_rows; }
+ unsigned int get_output_cols(void) const override { return output_cols; }
+
+ unsigned int get_input_rows(void) const override { return input_rows; }
+ unsigned int get_input_cols(void) const override { return input_cols; }
+
+ void indirect_kernel(
+ const void *const *const input_ptrs,
+ void *const *const outptrs,
+ const void *params,
+ unsigned int n_channels,
+ const void *activation_min,
+ const void *activation_max
+ ) const override
+ {
+ m_indirect_kernel(
+ reinterpret_cast<const __fp16 *const *>(input_ptrs),
+ reinterpret_cast<__fp16 *const *>(outptrs),
+ params, n_channels,
+ *static_cast<const __fp16 *>(activation_min),
+ *static_cast<const __fp16 *>(activation_max)
+ );
+ }
+
+ void direct_kernel(
+ const unsigned int n_tile_rows, const unsigned int n_tile_cols,
+ const void *inptr, int64_t ld_input_row, int64_t ld_input_col,
+ void *outptr, int64_t ld_output_row, int64_t ld_output_col,
+ const void *params, unsigned int n_channels,
+ const void *activation_min, const void *activation_max
+ ) const override
+ {
+ m_direct_kernel(
+ n_tile_rows, n_tile_cols,
+ static_cast<const __fp16 *>(inptr), ld_input_row, ld_input_col,
+ static_cast<__fp16 *>(outptr), ld_output_row, ld_output_col,
+ params, n_channels,
+ *static_cast<const __fp16 *>(activation_min),
+ *static_cast<const __fp16 *>(activation_max)
+ );
+ }
};
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
+#endif // defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp
index c2d6766..9b4b3ee 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp
@@ -25,7 +25,7 @@
#include <cstddef>
#include <cstdint>
-#if defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
+#if defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
namespace arm_conv {
namespace depthwise {
@@ -1156,4 +1156,4 @@
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
+#endif // defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp
index fc98891..e0abca9 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp
@@ -25,7 +25,7 @@
#include <cstddef>
#include <cstdint>
-#if defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
+#if defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
namespace arm_conv {
namespace depthwise {
@@ -1289,4 +1289,4 @@
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
+#endif // defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp
index df53287..1e0d922 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp
@@ -28,7 +28,7 @@
#pragma once
-#if defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
+#if defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
namespace arm_conv {
namespace depthwise {
@@ -36,15 +36,17 @@
void a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16);
void a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16);
-struct a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst
+class a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst : public IDepthwiseDepthfirstStrategy
{
- typedef __fp16 bias_type;
- typedef __fp16 input_type;
- typedef __fp16 weight_type;
- typedef __fp16 return_type;
-
+ private:
typedef void (*indirect_kern_type)(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16);
+ indirect_kern_type m_indirect_kernel = a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl;
+
typedef void (*direct_kern_type)(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16);
+ direct_kern_type m_direct_kernel = a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl;
+
+ public:
+ typedef __fp16 return_type;
constexpr static arm_gemm::VLType vl_type = arm_gemm::VLType::None;
@@ -60,13 +62,60 @@
constexpr static unsigned int input_rows = 6;
constexpr static unsigned int input_cols = 6;
- indirect_kern_type indirect_kernel = a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl;
- direct_kern_type direct_kernel = a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl;
-
a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst(const CPUInfo *) {}
+
+ arm_gemm::VLType get_vl_type(void) const override { return vl_type; }
+
+ unsigned int get_kernel_rows(void) const override { return kernel_rows; }
+ unsigned int get_kernel_cols(void) const override { return kernel_cols; }
+
+ unsigned int get_stride_rows(void) const override { return stride_rows; }
+ unsigned int get_stride_cols(void) const override { return stride_cols; }
+
+ unsigned int get_output_rows(void) const override { return output_rows; }
+ unsigned int get_output_cols(void) const override { return output_cols; }
+
+ unsigned int get_input_rows(void) const override { return input_rows; }
+ unsigned int get_input_cols(void) const override { return input_cols; }
+
+ void indirect_kernel(
+ const void *const *const input_ptrs,
+ void *const *const outptrs,
+ const void *params,
+ unsigned int n_channels,
+ const void *activation_min,
+ const void *activation_max
+ ) const override
+ {
+ m_indirect_kernel(
+ reinterpret_cast<const __fp16 *const *>(input_ptrs),
+ reinterpret_cast<__fp16 *const *>(outptrs),
+ params, n_channels,
+ *static_cast<const __fp16 *>(activation_min),
+ *static_cast<const __fp16 *>(activation_max)
+ );
+ }
+
+ void direct_kernel(
+ const unsigned int n_tile_rows, const unsigned int n_tile_cols,
+ const void *inptr, int64_t ld_input_row, int64_t ld_input_col,
+ void *outptr, int64_t ld_output_row, int64_t ld_output_col,
+ const void *params, unsigned int n_channels,
+ const void *activation_min, const void *activation_max
+ ) const override
+ {
+ m_direct_kernel(
+ n_tile_rows, n_tile_cols,
+ static_cast<const __fp16 *>(inptr), ld_input_row, ld_input_col,
+ static_cast<__fp16 *>(outptr), ld_output_row, ld_output_col,
+ params, n_channels,
+ *static_cast<const __fp16 *>(activation_min),
+ *static_cast<const __fp16 *>(activation_max)
+ );
+ }
};
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
+#endif // defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp
index c403576..b5bee7a 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp
@@ -25,7 +25,7 @@
#include <cstddef>
#include <cstdint>
-#if defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
+#if defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
namespace arm_conv {
namespace depthwise {
@@ -1734,4 +1734,4 @@
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
+#endif // defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp
index 2e87728..e493104 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp
@@ -25,7 +25,7 @@
#include <cstddef>
#include <cstdint>
-#if defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
+#if defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
namespace arm_conv {
namespace depthwise {
@@ -2006,4 +2006,4 @@
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
+#endif // defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp
index ca367cc..d89ae0c 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp
@@ -28,7 +28,7 @@
#pragma once
-#if defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
+#if defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
namespace arm_conv {
namespace depthwise {
@@ -36,15 +36,17 @@
void a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16);
void a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16);
-struct a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst
+class a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst : public IDepthwiseDepthfirstStrategy
{
- typedef __fp16 bias_type;
- typedef __fp16 input_type;
- typedef __fp16 weight_type;
- typedef __fp16 return_type;
-
+ private:
typedef void (*indirect_kern_type)(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16);
+ indirect_kern_type m_indirect_kernel = a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl;
+
typedef void (*direct_kern_type)(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16);
+ direct_kern_type m_direct_kernel = a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl;
+
+ public:
+ typedef __fp16 return_type;
constexpr static arm_gemm::VLType vl_type = arm_gemm::VLType::None;
@@ -60,13 +62,60 @@
constexpr static unsigned int input_rows = 5;
constexpr static unsigned int input_cols = 5;
- indirect_kern_type indirect_kernel = a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl;
- direct_kern_type direct_kernel = a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl;
-
a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst(const CPUInfo *) {}
+
+ arm_gemm::VLType get_vl_type(void) const override { return vl_type; }
+
+ unsigned int get_kernel_rows(void) const override { return kernel_rows; }
+ unsigned int get_kernel_cols(void) const override { return kernel_cols; }
+
+ unsigned int get_stride_rows(void) const override { return stride_rows; }
+ unsigned int get_stride_cols(void) const override { return stride_cols; }
+
+ unsigned int get_output_rows(void) const override { return output_rows; }
+ unsigned int get_output_cols(void) const override { return output_cols; }
+
+ unsigned int get_input_rows(void) const override { return input_rows; }
+ unsigned int get_input_cols(void) const override { return input_cols; }
+
+ void indirect_kernel(
+ const void *const *const input_ptrs,
+ void *const *const outptrs,
+ const void *params,
+ unsigned int n_channels,
+ const void *activation_min,
+ const void *activation_max
+ ) const override
+ {
+ m_indirect_kernel(
+ reinterpret_cast<const __fp16 *const *>(input_ptrs),
+ reinterpret_cast<__fp16 *const *>(outptrs),
+ params, n_channels,
+ *static_cast<const __fp16 *>(activation_min),
+ *static_cast<const __fp16 *>(activation_max)
+ );
+ }
+
+ void direct_kernel(
+ const unsigned int n_tile_rows, const unsigned int n_tile_cols,
+ const void *inptr, int64_t ld_input_row, int64_t ld_input_col,
+ void *outptr, int64_t ld_output_row, int64_t ld_output_col,
+ const void *params, unsigned int n_channels,
+ const void *activation_min, const void *activation_max
+ ) const override
+ {
+ m_direct_kernel(
+ n_tile_rows, n_tile_cols,
+ static_cast<const __fp16 *>(inptr), ld_input_row, ld_input_col,
+ static_cast<__fp16 *>(outptr), ld_output_row, ld_output_col,
+ params, n_channels,
+ *static_cast<const __fp16 *>(activation_min),
+ *static_cast<const __fp16 *>(activation_max)
+ );
+ }
};
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
+#endif // defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp
index 9b2d77f..a5df51c 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp
@@ -25,7 +25,7 @@
#include <cstddef>
#include <cstdint>
-#if defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
+#if defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
namespace arm_conv {
namespace depthwise {
@@ -893,4 +893,4 @@
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
+#endif // defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp
index 4857077..61c5818 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp
@@ -25,7 +25,7 @@
#include <cstddef>
#include <cstdint>
-#if defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
+#if defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
namespace arm_conv {
namespace depthwise {
@@ -896,4 +896,4 @@
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
+#endif // defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp
index 53d2a3a..6b5f91f 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp
@@ -28,7 +28,7 @@
#pragma once
-#if defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
+#if defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
namespace arm_conv {
namespace depthwise {
@@ -36,15 +36,17 @@
void a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_indirect_impl(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16);
void a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16);
-struct a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst
+class a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst : public IDepthwiseDepthfirstStrategy
{
- typedef __fp16 bias_type;
- typedef __fp16 input_type;
- typedef __fp16 weight_type;
- typedef __fp16 return_type;
-
+ private:
typedef void (*indirect_kern_type)(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16);
+ indirect_kern_type m_indirect_kernel = a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_indirect_impl;
+
typedef void (*direct_kern_type)(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16);
+ direct_kern_type m_direct_kernel = a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl;
+
+ public:
+ typedef __fp16 return_type;
constexpr static arm_gemm::VLType vl_type = arm_gemm::VLType::None;
@@ -60,13 +62,60 @@
constexpr static unsigned int input_rows = 6;
constexpr static unsigned int input_cols = 6;
- indirect_kern_type indirect_kernel = a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_indirect_impl;
- direct_kern_type direct_kernel = a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl;
-
a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst(const CPUInfo *) {}
+
+ arm_gemm::VLType get_vl_type(void) const override { return vl_type; }
+
+ unsigned int get_kernel_rows(void) const override { return kernel_rows; }
+ unsigned int get_kernel_cols(void) const override { return kernel_cols; }
+
+ unsigned int get_stride_rows(void) const override { return stride_rows; }
+ unsigned int get_stride_cols(void) const override { return stride_cols; }
+
+ unsigned int get_output_rows(void) const override { return output_rows; }
+ unsigned int get_output_cols(void) const override { return output_cols; }
+
+ unsigned int get_input_rows(void) const override { return input_rows; }
+ unsigned int get_input_cols(void) const override { return input_cols; }
+
+ void indirect_kernel(
+ const void *const *const input_ptrs,
+ void *const *const outptrs,
+ const void *params,
+ unsigned int n_channels,
+ const void *activation_min,
+ const void *activation_max
+ ) const override
+ {
+ m_indirect_kernel(
+ reinterpret_cast<const __fp16 *const *>(input_ptrs),
+ reinterpret_cast<__fp16 *const *>(outptrs),
+ params, n_channels,
+ *static_cast<const __fp16 *>(activation_min),
+ *static_cast<const __fp16 *>(activation_max)
+ );
+ }
+
+ void direct_kernel(
+ const unsigned int n_tile_rows, const unsigned int n_tile_cols,
+ const void *inptr, int64_t ld_input_row, int64_t ld_input_col,
+ void *outptr, int64_t ld_output_row, int64_t ld_output_col,
+ const void *params, unsigned int n_channels,
+ const void *activation_min, const void *activation_max
+ ) const override
+ {
+ m_direct_kernel(
+ n_tile_rows, n_tile_cols,
+ static_cast<const __fp16 *>(inptr), ld_input_row, ld_input_col,
+ static_cast<__fp16 *>(outptr), ld_output_row, ld_output_col,
+ params, n_channels,
+ *static_cast<const __fp16 *>(activation_min),
+ *static_cast<const __fp16 *>(activation_max)
+ );
+ }
};
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
+#endif // defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_direct.cpp
index 027fd7f..b08059d 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_direct.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_direct.cpp
@@ -25,7 +25,7 @@
#include <cstddef>
#include <cstdint>
-#if defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
+#if defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
namespace arm_conv {
namespace depthwise {
@@ -1385,4 +1385,4 @@
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
+#endif // defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_indirect.cpp
index 43eac0e..5b086ec 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_indirect.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_indirect.cpp
@@ -25,7 +25,7 @@
#include <cstddef>
#include <cstdint>
-#if defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
+#if defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
namespace arm_conv {
namespace depthwise {
@@ -1426,4 +1426,4 @@
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
+#endif // defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp
index 88f20bb..a888eb5 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp
@@ -28,21 +28,25 @@
#pragma once
+#if __aarch64__
+
namespace arm_conv {
namespace depthwise {
void a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl(const float *const *const, float *const *const, const void *, unsigned int, const float, const float);
void a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float);
-struct a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst
+class a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst : public IDepthwiseDepthfirstStrategy
{
- typedef float bias_type;
- typedef float input_type;
- typedef float weight_type;
- typedef float return_type;
-
+ private:
typedef void (*indirect_kern_type)(const float *const *const, float *const *const, const void *, unsigned int, const float, const float);
+ indirect_kern_type m_indirect_kernel = a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl;
+
typedef void (*direct_kern_type)(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float);
+ direct_kern_type m_direct_kernel = a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl;
+
+ public:
+ typedef float return_type;
constexpr static arm_gemm::VLType vl_type = arm_gemm::VLType::None;
@@ -58,11 +62,60 @@
constexpr static unsigned int input_rows = 4;
constexpr static unsigned int input_cols = 4;
- indirect_kern_type indirect_kernel = a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl;
- direct_kern_type direct_kernel = a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl;
-
a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst(const CPUInfo *) {}
+
+ arm_gemm::VLType get_vl_type(void) const override { return vl_type; }
+
+ unsigned int get_kernel_rows(void) const override { return kernel_rows; }
+ unsigned int get_kernel_cols(void) const override { return kernel_cols; }
+
+ unsigned int get_stride_rows(void) const override { return stride_rows; }
+ unsigned int get_stride_cols(void) const override { return stride_cols; }
+
+ unsigned int get_output_rows(void) const override { return output_rows; }
+ unsigned int get_output_cols(void) const override { return output_cols; }
+
+ unsigned int get_input_rows(void) const override { return input_rows; }
+ unsigned int get_input_cols(void) const override { return input_cols; }
+
+ void indirect_kernel(
+ const void *const *const input_ptrs,
+ void *const *const outptrs,
+ const void *params,
+ unsigned int n_channels,
+ const void *activation_min,
+ const void *activation_max
+ ) const override
+ {
+ m_indirect_kernel(
+ reinterpret_cast<const float *const *>(input_ptrs),
+ reinterpret_cast<float *const *>(outptrs),
+ params, n_channels,
+ *static_cast<const float *>(activation_min),
+ *static_cast<const float *>(activation_max)
+ );
+ }
+
+ void direct_kernel(
+ const unsigned int n_tile_rows, const unsigned int n_tile_cols,
+ const void *inptr, int64_t ld_input_row, int64_t ld_input_col,
+ void *outptr, int64_t ld_output_row, int64_t ld_output_col,
+ const void *params, unsigned int n_channels,
+ const void *activation_min, const void *activation_max
+ ) const override
+ {
+ m_direct_kernel(
+ n_tile_rows, n_tile_cols,
+ static_cast<const float *>(inptr), ld_input_row, ld_input_col,
+ static_cast<float *>(outptr), ld_output_row, ld_output_col,
+ params, n_channels,
+ *static_cast<const float *>(activation_min),
+ *static_cast<const float *>(activation_max)
+ );
+ }
};
} // namespace depthwise
} // namespace arm_conv
+
+#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp
index fae208f..73c1e07 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp
@@ -25,6 +25,8 @@
#include <cstddef>
#include <cstdint>
+#if __aarch64__
+
namespace arm_conv {
namespace depthwise {
@@ -522,3 +524,5 @@
} // namespace depthwise
} // namespace arm_conv
+
+#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp
index 2f93a68..42931fb 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp
@@ -25,6 +25,8 @@
#include <cstddef>
#include <cstdint>
+#if __aarch64__
+
namespace arm_conv {
namespace depthwise {
@@ -509,3 +511,5 @@
} // namespace depthwise
} // namespace arm_conv
+
+#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp
index 6a882ec..01bb06a 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp
@@ -28,21 +28,25 @@
#pragma once
+#if __aarch64__
+
namespace arm_conv {
namespace depthwise {
void a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl(const float *const *const, float *const *const, const void *, unsigned int, const float, const float);
void a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float);
-struct a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst
+class a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst : public IDepthwiseDepthfirstStrategy
{
- typedef float bias_type;
- typedef float input_type;
- typedef float weight_type;
- typedef float return_type;
-
+ private:
typedef void (*indirect_kern_type)(const float *const *const, float *const *const, const void *, unsigned int, const float, const float);
+ indirect_kern_type m_indirect_kernel = a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl;
+
typedef void (*direct_kern_type)(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float);
+ direct_kern_type m_direct_kernel = a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl;
+
+ public:
+ typedef float return_type;
constexpr static arm_gemm::VLType vl_type = arm_gemm::VLType::None;
@@ -58,11 +62,60 @@
constexpr static unsigned int input_rows = 5;
constexpr static unsigned int input_cols = 5;
- indirect_kern_type indirect_kernel = a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl;
- direct_kern_type direct_kernel = a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl;
-
a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst(const CPUInfo *) {}
+
+ arm_gemm::VLType get_vl_type(void) const override { return vl_type; }
+
+ unsigned int get_kernel_rows(void) const override { return kernel_rows; }
+ unsigned int get_kernel_cols(void) const override { return kernel_cols; }
+
+ unsigned int get_stride_rows(void) const override { return stride_rows; }
+ unsigned int get_stride_cols(void) const override { return stride_cols; }
+
+ unsigned int get_output_rows(void) const override { return output_rows; }
+ unsigned int get_output_cols(void) const override { return output_cols; }
+
+ unsigned int get_input_rows(void) const override { return input_rows; }
+ unsigned int get_input_cols(void) const override { return input_cols; }
+
+ void indirect_kernel(
+ const void *const *const input_ptrs,
+ void *const *const outptrs,
+ const void *params,
+ unsigned int n_channels,
+ const void *activation_min,
+ const void *activation_max
+ ) const override
+ {
+ m_indirect_kernel(
+ reinterpret_cast<const float *const *>(input_ptrs),
+ reinterpret_cast<float *const *>(outptrs),
+ params, n_channels,
+ *static_cast<const float *>(activation_min),
+ *static_cast<const float *>(activation_max)
+ );
+ }
+
+ void direct_kernel(
+ const unsigned int n_tile_rows, const unsigned int n_tile_cols,
+ const void *inptr, int64_t ld_input_row, int64_t ld_input_col,
+ void *outptr, int64_t ld_output_row, int64_t ld_output_col,
+ const void *params, unsigned int n_channels,
+ const void *activation_min, const void *activation_max
+ ) const override
+ {
+ m_direct_kernel(
+ n_tile_rows, n_tile_cols,
+ static_cast<const float *>(inptr), ld_input_row, ld_input_col,
+ static_cast<float *>(outptr), ld_output_row, ld_output_col,
+ params, n_channels,
+ *static_cast<const float *>(activation_min),
+ *static_cast<const float *>(activation_max)
+ );
+ }
};
} // namespace depthwise
} // namespace arm_conv
+
+#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp
index 401528a..6d185e7 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp
@@ -25,6 +25,8 @@
#include <cstddef>
#include <cstdint>
+#if __aarch64__
+
namespace arm_conv {
namespace depthwise {
@@ -823,3 +825,5 @@
} // namespace depthwise
} // namespace arm_conv
+
+#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp
index 39ec001..aa7d35e 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp
@@ -25,6 +25,8 @@
#include <cstddef>
#include <cstdint>
+#if __aarch64__
+
namespace arm_conv {
namespace depthwise {
@@ -901,3 +903,5 @@
} // namespace depthwise
} // namespace arm_conv
+
+#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp
index 84bac12..17084b5 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp
@@ -28,21 +28,25 @@
#pragma once
+#if __aarch64__
+
namespace arm_conv {
namespace depthwise {
void a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl(const float *const *const, float *const *const, const void *, unsigned int, const float, const float);
void a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float);
-struct a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst
+class a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst : public IDepthwiseDepthfirstStrategy
{
- typedef float bias_type;
- typedef float input_type;
- typedef float weight_type;
- typedef float return_type;
-
+ private:
typedef void (*indirect_kern_type)(const float *const *const, float *const *const, const void *, unsigned int, const float, const float);
+ indirect_kern_type m_indirect_kernel = a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl;
+
typedef void (*direct_kern_type)(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float);
+ direct_kern_type m_direct_kernel = a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl;
+
+ public:
+ typedef float return_type;
constexpr static arm_gemm::VLType vl_type = arm_gemm::VLType::None;
@@ -58,11 +62,60 @@
constexpr static unsigned int input_rows = 6;
constexpr static unsigned int input_cols = 6;
- indirect_kern_type indirect_kernel = a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl;
- direct_kern_type direct_kernel = a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl;
-
a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst(const CPUInfo *) {}
+
+ arm_gemm::VLType get_vl_type(void) const override { return vl_type; }
+
+ unsigned int get_kernel_rows(void) const override { return kernel_rows; }
+ unsigned int get_kernel_cols(void) const override { return kernel_cols; }
+
+ unsigned int get_stride_rows(void) const override { return stride_rows; }
+ unsigned int get_stride_cols(void) const override { return stride_cols; }
+
+ unsigned int get_output_rows(void) const override { return output_rows; }
+ unsigned int get_output_cols(void) const override { return output_cols; }
+
+ unsigned int get_input_rows(void) const override { return input_rows; }
+ unsigned int get_input_cols(void) const override { return input_cols; }
+
+ void indirect_kernel(
+ const void *const *const input_ptrs,
+ void *const *const outptrs,
+ const void *params,
+ unsigned int n_channels,
+ const void *activation_min,
+ const void *activation_max
+ ) const override
+ {
+ m_indirect_kernel(
+ reinterpret_cast<const float *const *>(input_ptrs),
+ reinterpret_cast<float *const *>(outptrs),
+ params, n_channels,
+ *static_cast<const float *>(activation_min),
+ *static_cast<const float *>(activation_max)
+ );
+ }
+
+ void direct_kernel(
+ const unsigned int n_tile_rows, const unsigned int n_tile_cols,
+ const void *inptr, int64_t ld_input_row, int64_t ld_input_col,
+ void *outptr, int64_t ld_output_row, int64_t ld_output_col,
+ const void *params, unsigned int n_channels,
+ const void *activation_min, const void *activation_max
+ ) const override
+ {
+ m_direct_kernel(
+ n_tile_rows, n_tile_cols,
+ static_cast<const float *>(inptr), ld_input_row, ld_input_col,
+ static_cast<float *>(outptr), ld_output_row, ld_output_col,
+ params, n_channels,
+ *static_cast<const float *>(activation_min),
+ *static_cast<const float *>(activation_max)
+ );
+ }
};
} // namespace depthwise
} // namespace arm_conv
+
+#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp
index 616fd0d..6faacf1 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp
@@ -25,6 +25,8 @@
#include <cstddef>
#include <cstdint>
+#if __aarch64__
+
namespace arm_conv {
namespace depthwise {
@@ -1227,3 +1229,5 @@
} // namespace depthwise
} // namespace arm_conv
+
+#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp
index 51a5679..aeaf104 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp
@@ -25,6 +25,8 @@
#include <cstddef>
#include <cstdint>
+#if __aarch64__
+
namespace arm_conv {
namespace depthwise {
@@ -1393,3 +1395,5 @@
} // namespace depthwise
} // namespace arm_conv
+
+#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp
index 8eb5605..f23862b 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp
@@ -28,21 +28,25 @@
#pragma once
+#if __aarch64__
+
namespace arm_conv {
namespace depthwise {
void a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl(const float *const *const, float *const *const, const void *, unsigned int, const float, const float);
void a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float);
-struct a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst
+class a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst : public IDepthwiseDepthfirstStrategy
{
- typedef float bias_type;
- typedef float input_type;
- typedef float weight_type;
- typedef float return_type;
-
+ private:
typedef void (*indirect_kern_type)(const float *const *const, float *const *const, const void *, unsigned int, const float, const float);
+ indirect_kern_type m_indirect_kernel = a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl;
+
typedef void (*direct_kern_type)(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float);
+ direct_kern_type m_direct_kernel = a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl;
+
+ public:
+ typedef float return_type;
constexpr static arm_gemm::VLType vl_type = arm_gemm::VLType::None;
@@ -58,11 +62,60 @@
constexpr static unsigned int input_rows = 5;
constexpr static unsigned int input_cols = 5;
- indirect_kern_type indirect_kernel = a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl;
- direct_kern_type direct_kernel = a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl;
-
a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst(const CPUInfo *) {}
+
+ arm_gemm::VLType get_vl_type(void) const override { return vl_type; }
+
+ unsigned int get_kernel_rows(void) const override { return kernel_rows; }
+ unsigned int get_kernel_cols(void) const override { return kernel_cols; }
+
+ unsigned int get_stride_rows(void) const override { return stride_rows; }
+ unsigned int get_stride_cols(void) const override { return stride_cols; }
+
+ unsigned int get_output_rows(void) const override { return output_rows; }
+ unsigned int get_output_cols(void) const override { return output_cols; }
+
+ unsigned int get_input_rows(void) const override { return input_rows; }
+ unsigned int get_input_cols(void) const override { return input_cols; }
+
+ void indirect_kernel(
+ const void *const *const input_ptrs,
+ void *const *const outptrs,
+ const void *params,
+ unsigned int n_channels,
+ const void *activation_min,
+ const void *activation_max
+ ) const override
+ {
+ m_indirect_kernel(
+ reinterpret_cast<const float *const *>(input_ptrs),
+ reinterpret_cast<float *const *>(outptrs),
+ params, n_channels,
+ *static_cast<const float *>(activation_min),
+ *static_cast<const float *>(activation_max)
+ );
+ }
+
+ void direct_kernel(
+ const unsigned int n_tile_rows, const unsigned int n_tile_cols,
+ const void *inptr, int64_t ld_input_row, int64_t ld_input_col,
+ void *outptr, int64_t ld_output_row, int64_t ld_output_col,
+ const void *params, unsigned int n_channels,
+ const void *activation_min, const void *activation_max
+ ) const override
+ {
+ m_direct_kernel(
+ n_tile_rows, n_tile_cols,
+ static_cast<const float *>(inptr), ld_input_row, ld_input_col,
+ static_cast<float *>(outptr), ld_output_row, ld_output_col,
+ params, n_channels,
+ *static_cast<const float *>(activation_min),
+ *static_cast<const float *>(activation_max)
+ );
+ }
};
} // namespace depthwise
} // namespace arm_conv
+
+#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp
index 4466ec1..65e487d 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp
@@ -25,6 +25,8 @@
#include <cstddef>
#include <cstdint>
+#if __aarch64__
+
namespace arm_conv {
namespace depthwise {
@@ -610,3 +612,5 @@
} // namespace depthwise
} // namespace arm_conv
+
+#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp
index a515301..4b24862 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp
@@ -25,6 +25,8 @@
#include <cstddef>
#include <cstdint>
+#if __aarch64__
+
namespace arm_conv {
namespace depthwise {
@@ -625,3 +627,5 @@
} // namespace depthwise
} // namespace arm_conv
+
+#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp
index 314fe76..e4bfbe6 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp
@@ -28,21 +28,25 @@
#pragma once
+#if defined(__aarch64__)
+
namespace arm_conv {
namespace depthwise {
void a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_indirect_impl(const float *const *const, float *const *const, const void *, unsigned int, const float, const float);
void a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float);
-struct a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst
+class a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst : public IDepthwiseDepthfirstStrategy
{
- typedef float bias_type;
- typedef float input_type;
- typedef float weight_type;
- typedef float return_type;
-
+ private:
typedef void (*indirect_kern_type)(const float *const *const, float *const *const, const void *, unsigned int, const float, const float);
+ indirect_kern_type m_indirect_kernel = a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_indirect_impl;
+
typedef void (*direct_kern_type)(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float);
+ direct_kern_type m_direct_kernel = a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl;
+
+ public:
+ typedef float return_type;
constexpr static arm_gemm::VLType vl_type = arm_gemm::VLType::None;
@@ -58,11 +62,60 @@
constexpr static unsigned int input_rows = 6;
constexpr static unsigned int input_cols = 6;
- indirect_kern_type indirect_kernel = a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_indirect_impl;
- direct_kern_type direct_kernel = a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl;
-
a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst(const CPUInfo *) {}
+
+ arm_gemm::VLType get_vl_type(void) const override { return vl_type; }
+
+ unsigned int get_kernel_rows(void) const override { return kernel_rows; }
+ unsigned int get_kernel_cols(void) const override { return kernel_cols; }
+
+ unsigned int get_stride_rows(void) const override { return stride_rows; }
+ unsigned int get_stride_cols(void) const override { return stride_cols; }
+
+ unsigned int get_output_rows(void) const override { return output_rows; }
+ unsigned int get_output_cols(void) const override { return output_cols; }
+
+ unsigned int get_input_rows(void) const override { return input_rows; }
+ unsigned int get_input_cols(void) const override { return input_cols; }
+
+ void indirect_kernel(
+ const void *const *const input_ptrs,
+ void *const *const outptrs,
+ const void *params,
+ unsigned int n_channels,
+ const void *activation_min,
+ const void *activation_max
+ ) const override
+ {
+ m_indirect_kernel(
+ reinterpret_cast<const float *const *>(input_ptrs),
+ reinterpret_cast<float *const *>(outptrs),
+ params, n_channels,
+ *static_cast<const float *>(activation_min),
+ *static_cast<const float *>(activation_max)
+ );
+ }
+
+ void direct_kernel(
+ const unsigned int n_tile_rows, const unsigned int n_tile_cols,
+ const void *inptr, int64_t ld_input_row, int64_t ld_input_col,
+ void *outptr, int64_t ld_output_row, int64_t ld_output_col,
+ const void *params, unsigned int n_channels,
+ const void *activation_min, const void *activation_max
+ ) const override
+ {
+ m_direct_kernel(
+ n_tile_rows, n_tile_cols,
+ static_cast<const float *>(inptr), ld_input_row, ld_input_col,
+ static_cast<float *>(outptr), ld_output_row, ld_output_col,
+ params, n_channels,
+ *static_cast<const float *>(activation_min),
+ *static_cast<const float *>(activation_max)
+ );
+ }
};
} // namespace depthwise
} // namespace arm_conv
+
+#endif // defined(__aarch64__)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_direct.cpp
index b74a3c9..8b030ec 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_direct.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_direct.cpp
@@ -25,6 +25,8 @@
#include <cstddef>
#include <cstdint>
+#if defined(__aarch64__)
+
namespace arm_conv {
namespace depthwise {
@@ -986,3 +988,5 @@
} // namespace depthwise
} // namespace arm_conv
+
+#endif // defined(__aarch64__)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_indirect.cpp
index adc5170..4754a6f 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_indirect.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_indirect.cpp
@@ -25,6 +25,8 @@
#include <cstddef>
#include <cstdint>
+#if defined(__aarch64__)
+
namespace arm_conv {
namespace depthwise {
@@ -1039,3 +1041,5 @@
} // namespace depthwise
} // namespace arm_conv
+
+#endif // defined(__aarch64__)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst/generic.cpp
index 14e113b..b62ebb1 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst/generic.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst/generic.cpp
@@ -22,7 +22,7 @@
* SOFTWARE.
*/
-#include "src/core/NEON/kernels/arm_gemm/utils.hpp"
+#include "arm_gemm.hpp"
#include <cstddef>
#include <cstdint>
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_3x3_s2_output2x2_mla_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_3x3_s2_output2x2_mla_depthfirst/generic.cpp
index ccdde41..8d22836 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_3x3_s2_output2x2_mla_depthfirst/generic.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_3x3_s2_output2x2_mla_depthfirst/generic.cpp
@@ -22,7 +22,7 @@
* SOFTWARE.
*/
-#include "src/core/NEON/kernels/arm_gemm/utils.hpp"
+#include "arm_gemm.hpp"
#include <cstddef>
#include <cstdint>
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_5x5_s1_output2x2_mla_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_5x5_s1_output2x2_mla_depthfirst/generic.cpp
index 699cc6c..b42f29a 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_5x5_s1_output2x2_mla_depthfirst/generic.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_5x5_s1_output2x2_mla_depthfirst/generic.cpp
@@ -22,7 +22,7 @@
* SOFTWARE.
*/
-#include "src/core/NEON/kernels/arm_gemm/utils.hpp"
+#include "arm_gemm.hpp"
#include <cstddef>
#include <cstdint>
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp
index b8e5930..1cfea9d 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp
@@ -28,7 +28,7 @@
#pragma once
-#if defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
+#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
namespace arm_conv {
namespace depthwise {
@@ -36,15 +36,17 @@
void sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16);
void sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16);
-struct sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst
+class sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst : public IDepthwiseDepthfirstStrategy
{
- typedef __fp16 bias_type;
- typedef __fp16 input_type;
- typedef __fp16 weight_type;
- typedef __fp16 return_type;
-
+ private:
typedef void (*indirect_kern_type)(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16);
+ indirect_kern_type m_indirect_kernel = sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl;
+
typedef void (*direct_kern_type)(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16);
+ direct_kern_type m_direct_kernel = sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl;
+
+ public:
+ typedef __fp16 return_type;
constexpr static arm_gemm::VLType vl_type = arm_gemm::VLType::SVE;
@@ -60,13 +62,60 @@
constexpr static unsigned int input_rows = 4;
constexpr static unsigned int input_cols = 4;
- indirect_kern_type indirect_kernel = sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl;
- direct_kern_type direct_kernel = sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl;
-
sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst(const CPUInfo *) {}
+
+ arm_gemm::VLType get_vl_type(void) const override { return vl_type; }
+
+ unsigned int get_kernel_rows(void) const override { return kernel_rows; }
+ unsigned int get_kernel_cols(void) const override { return kernel_cols; }
+
+ unsigned int get_stride_rows(void) const override { return stride_rows; }
+ unsigned int get_stride_cols(void) const override { return stride_cols; }
+
+ unsigned int get_output_rows(void) const override { return output_rows; }
+ unsigned int get_output_cols(void) const override { return output_cols; }
+
+ unsigned int get_input_rows(void) const override { return input_rows; }
+ unsigned int get_input_cols(void) const override { return input_cols; }
+
+ void indirect_kernel(
+ const void *const *const input_ptrs,
+ void *const *const outptrs,
+ const void *params,
+ unsigned int n_channels,
+ const void *activation_min,
+ const void *activation_max
+ ) const override
+ {
+ m_indirect_kernel(
+ reinterpret_cast<const __fp16 *const *>(input_ptrs),
+ reinterpret_cast<__fp16 *const *>(outptrs),
+ params, n_channels,
+ *static_cast<const __fp16 *>(activation_min),
+ *static_cast<const __fp16 *>(activation_max)
+ );
+ }
+
+ void direct_kernel(
+ const unsigned int n_tile_rows, const unsigned int n_tile_cols,
+ const void *inptr, int64_t ld_input_row, int64_t ld_input_col,
+ void *outptr, int64_t ld_output_row, int64_t ld_output_col,
+ const void *params, unsigned int n_channels,
+ const void *activation_min, const void *activation_max
+ ) const override
+ {
+ m_direct_kernel(
+ n_tile_rows, n_tile_cols,
+ static_cast<const __fp16 *>(inptr), ld_input_row, ld_input_col,
+ static_cast<__fp16 *>(outptr), ld_output_row, ld_output_col,
+ params, n_channels,
+ *static_cast<const __fp16 *>(activation_min),
+ *static_cast<const __fp16 *>(activation_max)
+ );
+ }
};
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
+#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp
index a4c1a40..955a02d 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp
@@ -25,7 +25,7 @@
#include <cstddef>
#include <cstdint>
-#if defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
+#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
namespace arm_conv {
namespace depthwise {
@@ -113,149 +113,141 @@
"add x27, x10, x22, LSL #1\n"
"ld1rh { z17.h }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n"
"add x26, x27, x22, LSL #1\n"
- "ld1h { z16.h }, p3/Z, [x14]\n" // Load from weights and bias
- "mov z31.d, z16.d\n"
- "ld1h { z0.h }, p3/Z, [x14, #1, MUL VL]\n" // Load from weights and bias
+ "ld1h { z16.h }, p3/Z, [x14]\n"
"add x25, x26, x22, LSL #1\n"
- "mov z30.d, z16.d\n"
- "ld1h { z1.h }, p3/Z, [x14, #2, MUL VL]\n" // Load from weights and bias
+ "ld1h { z0.h }, p3/Z, [x14, #1, MUL VL]\n"
"add x24, x11, x11\n"
- "mov z29.d, z16.d\n"
- "ld1h { z2.h }, p3/Z, [x14, #3, MUL VL]\n" // Load from weights and bias
+ "ld1h { z1.h }, p3/Z, [x14, #2, MUL VL]\n"
"add x23, x24, x11\n"
- "mov z28.d, z16.d\n"
- "ld1h { z3.h }, p3/Z, [x14, #4, MUL VL]\n" // Load from weights and bias
+ "ld1h { z2.h }, p3/Z, [x14, #3, MUL VL]\n"
"mul x19, x17, x20\n" // offset = tile_i * ld_output_row
- "ld1h { z4.h }, p3/Z, [x14, #5, MUL VL]\n" // Load from weights and bias
+ "ld1h { z3.h }, p3/Z, [x14, #4, MUL VL]\n"
"madd x19, x16, x9, x19\n" // offset += tile_j * ld_output_col
- "ld1h { z5.h }, p3/Z, [x14, #6, MUL VL]\n" // Load from weights and bias
+ "ld1h { z4.h }, p3/Z, [x14, #5, MUL VL]\n"
"mul x19, x19, x15\n" // offset *= output_tile_size
- "ld1h { z6.h }, p3/Z, [x14, #7, MUL VL]\n" // Load from weights and bias
+ "ld1h { z5.h }, p3/Z, [x14, #6, MUL VL]\n"
"add x28, x28, x19, LSL #1\n" // outptrs[0] += offset * sizeof(__fp16)
- "whilelt p2.h, XZR, %x[n_channels]\n"
- "ld1h { z9.h }, p2/Z, [x27, x11, LSL #1]\n" // Load input point (1, 1)
- "ld1h { z10.h }, p2/Z, [x10]\n" // Load input point (0, 0)
+ "ld1h { z6.h }, p3/Z, [x14, #7, MUL VL]\n"
"add x22, x28, x20, LSL #1\n"
- "ld1h { z11.h }, p2/Z, [x10, x23, LSL #1]\n" // Load input point (0, 3)
+ "whilelt p2.h, XZR, %x[n_channels]\n"
+ "ld1h { z9.h }, p2/Z, [x27, x11, LSL #1]\n"
+ "ld1h { z10.h }, p2/Z, [x10]\n"
"addvl x14, x14, #16\n"
- "ld1h { z12.h }, p2/Z, [x27, x24, LSL #1]\n" // Load input point (1, 2)
+ "ld1h { z11.h }, p2/Z, [x10, x23, LSL #1]\n"
"cmp x12, %x[n_channels]\n"
- "ld1h { z7.h }, p3/Z, [x14, #-8, MUL VL]\n" // Load from weights and bias
- "ld1h { z8.h }, p3/Z, [x14, #-7, MUL VL]\n" // Load from weights and bias
+ "ld1h { z7.h }, p3/Z, [x14, #-8, MUL VL]\n"
+ "ld1h { z8.h }, p3/Z, [x14, #-7, MUL VL]\n"
"addvl x14, x14, #-6\n"
- "ld1h { z13.h }, p2/Z, [x26, x11, LSL #1]\n" // Load input point (2, 1)
+ "ld1h { z12.h }, p2/Z, [x27, x24, LSL #1]\n"
+ "ld1h { z13.h }, p2/Z, [x26, x11, LSL #1]\n"
"bge 3f\n"
"2:" // Tile loop: Channel loop
- "fmla z31.h, p3/M, z4.h, z9.h\n"
- "ld1h { z16.h }, p3/Z, [x14]\n" // Load from weights and bias
+ "movprfx z31, z16\n fmla z31.h, p3/M, z4.h, z9.h\n"
"whilelt p1.h, x12, %x[n_channels]\n"
- "fmla z30.h, p3/M, z3.h, z9.h\n"
+ "movprfx z30, z16\n fmla z30.h, p3/M, z3.h, z9.h\n"
"inch x21\n"
- "fmla z29.h, p3/M, z1.h, z9.h\n"
+ "movprfx z29, z16\n fmla z29.h, p3/M, z1.h, z9.h\n"
"mov p0.b, p2.b\n"
- "fmla z28.h, p3/M, z0.h, z9.h\n"
- "ld1h { z9.h }, p2/Z, [x25]\n" // Load input point (3, 0)
+ "movprfx z28, z16\n fmla z28.h, p3/M, z0.h, z9.h\n"
+ "ld1h { z9.h }, p2/Z, [x25]\n"
"inch x13\n"
"fmla z31.h, p3/M, z0.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x26, x24, LSL #1]\n" // Load input point (2, 2)
+ "ld1h { z10.h }, p2/Z, [x26, x24, LSL #1]\n"
"inch x12\n"
"fmla z30.h, p3/M, z2.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x25, x23, LSL #1]\n" // Load input point (3, 3)
+ "ld1h { z11.h }, p2/Z, [x25, x23, LSL #1]\n"
"fmla z29.h, p3/M, z2.h, z12.h\n"
+ "ld1h { z16.h }, p3/Z, [x14]\n"
"fmla z28.h, p3/M, z1.h, z12.h\n"
"fmla z31.h, p3/M, z5.h, z12.h\n"
"fmla z30.h, p3/M, z4.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x10, x11, LSL #1]\n" // Load input point (0, 1)
+ "ld1h { z12.h }, p2/Z, [x10, x11, LSL #1]\n"
"fmla z29.h, p3/M, z6.h, z9.h\n"
- "ld1h { z9.h }, p2/Z, [x10, x24, LSL #1]\n" // Load input point (0, 2)
+ "ld1h { z9.h }, p2/Z, [x10, x24, LSL #1]\n"
"addvl x10, x10, #1\n"
"fmla z28.h, p3/M, z3.h, z13.h\n"
"fmla z31.h, p3/M, z7.h, z13.h\n"
"fmla z30.h, p3/M, z6.h, z13.h\n"
"fmla z29.h, p3/M, z4.h, z13.h\n"
"fmla z28.h, p3/M, z8.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x27]\n" // Load input point (1, 0)
+ "ld1h { z11.h }, p2/Z, [x27]\n"
"fmla z31.h, p3/M, z1.h, z12.h\n"
"fmla z30.h, p3/M, z0.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x27, x23, LSL #1]\n" // Load input point (1, 3)
+ "ld1h { z12.h }, p2/Z, [x27, x23, LSL #1]\n"
"addvl x27, x27, #1\n"
"fmla z29.h, p3/M, z5.h, z10.h\n"
"fmla z28.h, p3/M, z4.h, z10.h\n"
- "ld1h { z4.h }, p3/Z, [x14, #5, MUL VL]\n" // Load from weights and bias
+ "ld1h { z4.h }, p3/Z, [x14, #5, MUL VL]\n"
"fmla z31.h, p3/M, z2.h, z9.h\n"
"fmla z30.h, p3/M, z1.h, z9.h\n"
- "ld1h { z9.h }, p2/Z, [x26]\n" // Load input point (2, 0)
- "ld1h { z1.h }, p3/Z, [x14, #2, MUL VL]\n" // Load from weights and bias
+ "ld1h { z9.h }, p2/Z, [x26]\n"
+ "ld1h { z1.h }, p3/Z, [x14, #2, MUL VL]\n"
"fmla z29.h, p3/M, z0.h, z11.h\n"
- "ld1h { z0.h }, p3/Z, [x14, #1, MUL VL]\n" // Load from weights and bias
+ "ld1h { z0.h }, p3/Z, [x14, #1, MUL VL]\n"
"fmla z28.h, p3/M, z2.h, z12.h\n"
- "ld1h { z2.h }, p3/Z, [x14, #3, MUL VL]\n" // Load from weights and bias
+ "ld1h { z2.h }, p3/Z, [x14, #3, MUL VL]\n"
"fmla z31.h, p3/M, z8.h, z10.h\n"
"fmla z30.h, p3/M, z7.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x26, x23, LSL #1]\n" // Load input point (2, 3)
+ "ld1h { z10.h }, p2/Z, [x26, x23, LSL #1]\n"
"addvl x26, x26, #1\n"
"fmla z29.h, p3/M, z3.h, z9.h\n"
- "ld1h { z13.h }, p1/Z, [x26, x11, LSL #1]\n" // Load input point (2, 1)
+ "ld1h { z13.h }, p1/Z, [x26, x11, LSL #1]\n"
"fmla z31.h, p3/M, z3.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x25, x11, LSL #1]\n" // Load input point (3, 1)
+ "ld1h { z11.h }, p2/Z, [x25, x11, LSL #1]\n"
"fmla z28.h, p3/M, z5.h, z10.h\n"
- "ld1h { z3.h }, p3/Z, [x14, #4, MUL VL]\n" // Load from weights and bias
+ "ld1h { z3.h }, p3/Z, [x14, #4, MUL VL]\n"
"fmla z30.h, p3/M, z5.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x25, x24, LSL #1]\n" // Load input point (3, 2)
+ "ld1h { z12.h }, p2/Z, [x25, x24, LSL #1]\n"
"whilelt p2.h, x13, %x[n_channels]\n"
"fmla z29.h, p3/M, z7.h, z11.h\n"
- "ld1h { z5.h }, p3/Z, [x14, #6, MUL VL]\n" // Load from weights and bias
+ "ld1h { z5.h }, p3/Z, [x14, #6, MUL VL]\n"
"addvl x25, x25, #1\n"
"fmla z31.h, p3/M, z6.h, z9.h\n"
- "ld1h { z9.h }, p1/Z, [x27, x11, LSL #1]\n" // Load input point (1, 1)
+ "ld1h { z9.h }, p1/Z, [x27, x11, LSL #1]\n"
"cmp x12, %x[n_channels]\n"
"fmla z30.h, p3/M, z8.h, z10.h\n"
- "ld1h { z10.h }, p1/Z, [x10]\n" // Load input point (0, 0)
+ "ld1h { z10.h }, p1/Z, [x10]\n"
"fmla z28.h, p3/M, z6.h, z11.h\n"
- "ld1h { z11.h }, p1/Z, [x10, x23, LSL #1]\n" // Load input point (0, 3)
- "ld1h { z6.h }, p3/Z, [x14, #7, MUL VL]\n" // Load from weights and bias
+ "ld1h { z11.h }, p1/Z, [x10, x23, LSL #1]\n"
+ "ld1h { z6.h }, p3/Z, [x14, #7, MUL VL]\n"
"fmla z29.h, p3/M, z8.h, z12.h\n"
"addvl x14, x14, #16\n"
"fmax z31.h, p3/M, z31.h, z18.h\n"
- "ld1h { z8.h }, p3/Z, [x14, #-7, MUL VL]\n" // Load from weights and bias
+ "ld1h { z8.h }, p3/Z, [x14, #-7, MUL VL]\n"
"fmla z28.h, p3/M, z7.h, z12.h\n"
- "ld1h { z12.h }, p1/Z, [x27, x24, LSL #1]\n" // Load input point (1, 2)
+ "ld1h { z12.h }, p1/Z, [x27, x24, LSL #1]\n"
"fmax z30.h, p3/M, z30.h, z18.h\n"
- "ld1h { z7.h }, p3/Z, [x14, #-8, MUL VL]\n" // Load from weights and bias
+ "ld1h { z7.h }, p3/Z, [x14, #-8, MUL VL]\n"
"addvl x14, x14, #-6\n"
"fmax z29.h, p3/M, z29.h, z18.h\n"
"fmin z31.h, p3/M, z31.h, z17.h\n"
- "st1h { z31.h }, p0, [x28]\n" // Store output point (0, 0)
- "mov z31.d, z16.d\n"
+ "st1h { z31.h }, p0, [x28]\n"
"fmin z30.h, p3/M, z30.h, z17.h\n"
- "st1h { z30.h }, p0, [x28, x9, LSL #1]\n" // Store output point (0, 1)
- "mov z30.d, z16.d\n"
- "addvl x28, x28, #1\n"
"fmin z29.h, p3/M, z29.h, z17.h\n"
- "st1h { z29.h }, p0, [x22]\n" // Store output point (1, 0)
- "mov z29.d, z16.d\n"
+ "st1h { z30.h }, p0, [x28, x9, LSL #1]\n"
"fmax z28.h, p3/M, z28.h, z18.h\n"
+ "addvl x28, x28, #1\n"
"fmin z28.h, p3/M, z28.h, z17.h\n"
- "st1h { z28.h }, p0, [x22, x9, LSL #1]\n" // Store output point (1, 1)
- "mov z28.d, z16.d\n"
+ "st1h { z29.h }, p0, [x22]\n"
+ "st1h { z28.h }, p0, [x22, x9, LSL #1]\n"
"addvl x22, x22, #1\n"
"blt 2b\n"
"3:" // Tile loop: Channel tail
- "fmla z31.h, p3/M, z4.h, z9.h\n"
+ "movprfx z31, z16\n fmla z31.h, p3/M, z4.h, z9.h\n"
"ldr x17, [%x[params_struct], %[offsetof_args_tile_i]]\n"
"mov p0.b, p2.b\n"
- "fmla z30.h, p3/M, z3.h, z9.h\n"
+ "movprfx z30, z16\n fmla z30.h, p3/M, z3.h, z9.h\n"
"ldr x16, [%x[params_struct], %[offsetof_args_tile_j]]\n"
"add x21, x17, #0x1\n"
- "fmla z29.h, p3/M, z1.h, z9.h\n"
+ "movprfx z29, z16\n fmla z29.h, p3/M, z1.h, z9.h\n"
"ldr x20, [%x[params_struct], %[offsetof_args_n_tile_rows]]\n"
- "fmla z28.h, p3/M, z0.h, z9.h\n"
- "ld1h { z9.h }, p2/Z, [x25]\n" // Load input point (3, 0)
+ "movprfx z28, z16\n fmla z28.h, p3/M, z0.h, z9.h\n"
+ "ld1h { z9.h }, p2/Z, [x25]\n"
"add x16, x16, #0x1\n"
"fmla z31.h, p3/M, z0.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x26, x24, LSL #1]\n" // Load input point (2, 2)
+ "ld1h { z10.h }, p2/Z, [x26, x24, LSL #1]\n"
"fmla z30.h, p3/M, z2.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x25, x23, LSL #1]\n" // Load input point (3, 3)
+ "ld1h { z11.h }, p2/Z, [x25, x23, LSL #1]\n"
"ldr x19, [%x[params_struct], %[offsetof_args_n_tile_cols]]\n"
"fmla z29.h, p3/M, z2.h, z12.h\n"
"cmp x16, x19\n"
@@ -263,10 +255,10 @@
"fmla z30.h, p3/M, z4.h, z12.h\n"
"csel x16, x16, XZR, LT\n"
"fmla z28.h, p3/M, z1.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x10, x11, LSL #1]\n" // Load input point (0, 1)
+ "ld1h { z12.h }, p2/Z, [x10, x11, LSL #1]\n"
"csel x17, x17, x21, LT\n"
"fmla z29.h, p3/M, z6.h, z9.h\n"
- "ld1h { z9.h }, p2/Z, [x10, x24, LSL #1]\n" // Load input point (0, 2)
+ "ld1h { z9.h }, p2/Z, [x10, x24, LSL #1]\n"
"cmp x17, x20\n"
"fmla z31.h, p3/M, z7.h, z13.h\n"
"fmla z30.h, p3/M, z6.h, z13.h\n"
@@ -274,24 +266,24 @@
"fmla z29.h, p3/M, z4.h, z13.h\n"
"fmla z31.h, p3/M, z1.h, z12.h\n"
"fmla z30.h, p3/M, z0.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x27, x23, LSL #1]\n" // Load input point (1, 3)
+ "ld1h { z12.h }, p2/Z, [x27, x23, LSL #1]\n"
"fmla z28.h, p3/M, z8.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x27]\n" // Load input point (1, 0)
+ "ld1h { z11.h }, p2/Z, [x27]\n"
"fmla z29.h, p3/M, z5.h, z10.h\n"
"fmla z31.h, p3/M, z2.h, z9.h\n"
"fmla z30.h, p3/M, z1.h, z9.h\n"
- "ld1h { z9.h }, p2/Z, [x26]\n" // Load input point (2, 0)
+ "ld1h { z9.h }, p2/Z, [x26]\n"
"fmla z28.h, p3/M, z4.h, z10.h\n"
"fmla z29.h, p3/M, z0.h, z11.h\n"
"fmla z31.h, p3/M, z8.h, z10.h\n"
"fmla z30.h, p3/M, z7.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x26, x23, LSL #1]\n" // Load input point (2, 3)
+ "ld1h { z10.h }, p2/Z, [x26, x23, LSL #1]\n"
"fmla z28.h, p3/M, z2.h, z12.h\n"
"fmla z29.h, p3/M, z3.h, z9.h\n"
"fmla z31.h, p3/M, z3.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x25, x11, LSL #1]\n" // Load input point (3, 1)
+ "ld1h { z11.h }, p2/Z, [x25, x11, LSL #1]\n"
"fmla z30.h, p3/M, z5.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x25, x24, LSL #1]\n" // Load input point (3, 2)
+ "ld1h { z12.h }, p2/Z, [x25, x24, LSL #1]\n"
"fmla z28.h, p3/M, z5.h, z10.h\n"
"fmla z29.h, p3/M, z7.h, z11.h\n"
"fmla z31.h, p3/M, z6.h, z9.h\n"
@@ -303,14 +295,14 @@
"fmla z28.h, p3/M, z7.h, z12.h\n"
"fmax z29.h, p3/M, z29.h, z18.h\n"
"fmin z31.h, p3/M, z31.h, z17.h\n"
- "st1h { z31.h }, p0, [x28]\n" // Store output point (0, 0)
+ "st1h { z31.h }, p0, [x28]\n"
"fmin z30.h, p3/M, z30.h, z17.h\n"
"fmin z29.h, p3/M, z29.h, z17.h\n"
- "st1h { z30.h }, p0, [x28, x9, LSL #1]\n" // Store output point (0, 1)
+ "st1h { z30.h }, p0, [x28, x9, LSL #1]\n"
"fmax z28.h, p3/M, z28.h, z18.h\n"
- "st1h { z29.h }, p0, [x22]\n" // Store output point (1, 0)
+ "st1h { z29.h }, p0, [x22]\n"
"fmin z28.h, p3/M, z28.h, z17.h\n"
- "st1h { z28.h }, p0, [x22, x9, LSL #1]\n" // Store output point (1, 1)
+ "st1h { z28.h }, p0, [x22, x9, LSL #1]\n"
"blt 1b\n"
:
: [n_channels] "r" ((unsigned long) n_channels), [offsetof_args_inptr] "I" (offsetof(Args, inptr)), [offsetof_args_ld_input_col] "I" (offsetof(Args, ld_input_col)), [offsetof_args_ld_input_row] "I" (offsetof(Args, ld_input_row)), [offsetof_args_ld_output_col] "I" (offsetof(Args, ld_output_col)), [offsetof_args_ld_output_row] "I" (offsetof(Args, ld_output_row)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_n_tile_cols] "I" (offsetof(Args, n_tile_cols)), [offsetof_args_n_tile_rows] "I" (offsetof(Args, n_tile_rows)), [offsetof_args_outptr] "I" (offsetof(Args, outptr)), [offsetof_args_params] "I" (offsetof(Args, params)), [offsetof_args_tile_i] "I" (offsetof(Args, tile_i)), [offsetof_args_tile_j] "I" (offsetof(Args, tile_j)), [params_struct] "r" (¶ms_struct)
@@ -321,4 +313,4 @@
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
+#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp
index a845e7c..7cca6fb 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp
@@ -25,7 +25,7 @@
#include <cstddef>
#include <cstdint>
-#if defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
+#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
namespace arm_conv {
namespace depthwise {
@@ -54,22 +54,22 @@
const __fp16 max
) : outptrs(outptrs), params(params), min(min), max(max)
{
- inptrs[0] = input_ptrs[0];
- inptrs[1] = input_ptrs[1];
- inptrs[2] = input_ptrs[2];
- inptrs[3] = input_ptrs[3];
- inptrs[4] = input_ptrs[4];
- inptrs[5] = input_ptrs[5];
- inptrs[6] = input_ptrs[6];
- inptrs[7] = input_ptrs[7];
- inptrs[8] = input_ptrs[8];
- inptrs[9] = input_ptrs[9];
- inptrs[10] = input_ptrs[10];
- inptrs[11] = input_ptrs[11];
- inptrs[12] = input_ptrs[12];
- inptrs[13] = input_ptrs[13];
- inptrs[14] = input_ptrs[14];
- inptrs[15] = input_ptrs[15];
+ inptrs[0] = input_ptrs[5];
+ inptrs[1] = input_ptrs[0];
+ inptrs[2] = input_ptrs[3];
+ inptrs[3] = input_ptrs[6];
+ inptrs[4] = input_ptrs[9];
+ inptrs[5] = input_ptrs[12];
+ inptrs[6] = input_ptrs[15];
+ inptrs[7] = input_ptrs[1];
+ inptrs[8] = input_ptrs[2];
+ inptrs[9] = input_ptrs[10];
+ inptrs[10] = input_ptrs[4];
+ inptrs[11] = input_ptrs[7];
+ inptrs[12] = input_ptrs[8];
+ inptrs[13] = input_ptrs[11];
+ inptrs[14] = input_ptrs[13];
+ inptrs[15] = input_ptrs[14];
}
};
@@ -78,207 +78,219 @@
activation_min, activation_max);
__asm__ __volatile__(
- "ldr x3, [%x[params_struct], %[offsetof_args_outptrs]]\n"
+ "ldr x19, [%x[params_struct], %[offsetof_args_outptrs]]\n"
"ptrue p3.b\n"
- "ldr x4, [%x[params_struct], %[offsetof_args_params]]\n"
- "add x19, %x[params_struct], %[offsetof_Args_inptrs]\n"
+ "ldr x15, [%x[params_struct], %[offsetof_args_params]]\n"
+ "add x14, %x[params_struct], %[offsetof_Args_inptrs]\n"
"ld1rh { z18.h }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n"
- "mov x5, #0x0\n"
- "ldp x6, x7, [x19, #0x0]\n"
- "cnth x8\n"
- "ldp x17, x16, [x19, #0x10]\n"
- "sub x15, XZR, x8\n"
- "ldp x14, x13, [x19, #0x20]\n"
- "whilelt p2.h, XZR, %x[n_channels]\n"
- "ldp x12, x11, [x19, #0x30]\n"
- "cmp x8, %x[n_channels]\n"
- "ldp x10, x9, [x19, #0x40]\n"
- "ldp x28, x27, [x19, #0x50]\n"
- "ldp x26, x25, [x19, #0x60]\n"
- "ldp x24, x23, [x19, #0x70]\n"
- "ldp x22, x21, [x3, #0x0]\n"
- "ldp x20, x19, [x3, #0x10]\n"
+ "mov x13, #0x0\n"
"ld1rh { z17.h }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n"
- "ld1h { z16.h }, p3/Z, [x4]\n" // Load from weights and bias
- "mov z31.d, z16.d\n"
- "ld1h { z0.h }, p3/Z, [x4, #1, MUL VL]\n" // Load from weights and bias
- "mov z30.d, z16.d\n"
- "ld1h { z1.h }, p3/Z, [x4, #2, MUL VL]\n" // Load from weights and bias
- "mov z29.d, z16.d\n"
- "ld1h { z2.h }, p3/Z, [x4, #3, MUL VL]\n" // Load from weights and bias
- "mov z28.d, z16.d\n"
- "ld1h { z3.h }, p3/Z, [x4, #4, MUL VL]\n" // Load from weights and bias
- "ld1h { z4.h }, p3/Z, [x4, #5, MUL VL]\n" // Load from weights and bias
- "ld1h { z5.h }, p3/Z, [x4, #6, MUL VL]\n" // Load from weights and bias
- "ld1h { z6.h }, p3/Z, [x4, #7, MUL VL]\n" // Load from weights and bias
- "addvl x4, x4, #16\n"
- "ld1h { z9.h }, p2/Z, [x13, x5, LSL #1]\n"
- "ld1h { z7.h }, p3/Z, [x4, #-8, MUL VL]\n" // Load from weights and bias
- "ld1h { z8.h }, p3/Z, [x4, #-7, MUL VL]\n" // Load from weights and bias
- "addvl x4, x4, #-6\n"
- "ld1h { z10.h }, p2/Z, [x6, x5, LSL #1]\n"
- "ld1h { z11.h }, p2/Z, [x16, x5, LSL #1]\n"
- "ld1h { z12.h }, p2/Z, [x12, x5, LSL #1]\n"
- "ld1h { z13.h }, p2/Z, [x9, x5, LSL #1]\n"
+ "cnth x12\n"
+ "ldp x11, x10, [x19, #0x0]\n"
+ "sub x9, XZR, x12\n"
+ "ldp x28, x27, [x19, #0x10]\n"
+ "whilelt p2.h, XZR, %x[n_channels]\n"
+ "ld1h { z16.h }, p3/Z, [x15]\n"
+ "cmp x12, %x[n_channels]\n"
+ "ld1h { z0.h }, p3/Z, [x15, #1, MUL VL]\n"
+ "ld1h { z1.h }, p3/Z, [x15, #2, MUL VL]\n"
+ "ld1h { z2.h }, p3/Z, [x15, #3, MUL VL]\n"
+ "ld1h { z3.h }, p3/Z, [x15, #4, MUL VL]\n"
+ "ld1h { z4.h }, p3/Z, [x15, #5, MUL VL]\n"
+ "ld1h { z5.h }, p3/Z, [x15, #6, MUL VL]\n"
+ "ld1h { z6.h }, p3/Z, [x15, #7, MUL VL]\n"
+ "addvl x15, x15, #16\n"
+ "ldp x26, x25, [x14, #0x0]\n"
+ "ld1h { z7.h }, p3/Z, [x15, #-8, MUL VL]\n"
+ "ld1h { z8.h }, p3/Z, [x15, #-7, MUL VL]\n"
+ "addvl x15, x15, #-6\n"
+ "ld1h { z9.h }, p2/Z, [x26, x13, LSL #1]\n"
+ "ld1h { z10.h }, p2/Z, [x25, x13, LSL #1]\n"
+ "ldp x24, x23, [x14, #0x10]\n"
+ "ldr x22, [x14, #0x20]\n"
+ "ld1h { z11.h }, p2/Z, [x24, x13, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x23, x13, LSL #1]\n"
+ "ld1h { z13.h }, p2/Z, [x22, x13, LSL #1]\n"
"bge 2f\n"
"1:" // Channel loop
- "fmla z31.h, p3/M, z4.h, z9.h\n"
- "ld1h { z16.h }, p3/Z, [x4]\n" // Load from weights and bias
- "whilelt p1.h, x8, %x[n_channels]\n"
- "fmla z30.h, p3/M, z3.h, z9.h\n"
- "inch x15\n"
- "fmla z29.h, p3/M, z1.h, z9.h\n"
+ "movprfx z31, z16\n fmla z31.h, p3/M, z4.h, z9.h\n"
+ "ldr x21, [x14, #0x28]\n"
+ "whilelt p1.h, x12, %x[n_channels]\n"
+ "movprfx z30, z16\n fmla z30.h, p3/M, z3.h, z9.h\n"
+ "ldr x20, [x14, #0x30]\n"
+ "inch x9\n"
+ "movprfx z29, z16\n fmla z29.h, p3/M, z1.h, z9.h\n"
+ "ldr x19, [x14, #0x38]\n"
"mov p0.b, p2.b\n"
- "fmla z28.h, p3/M, z0.h, z9.h\n"
- "ld1h { z9.h }, p2/Z, [x26, x5, LSL #1]\n"
+ "movprfx z28, z16\n fmla z28.h, p3/M, z0.h, z9.h\n"
+ "ld1h { z9.h }, p2/Z, [x21, x13, LSL #1]\n"
+ "ldr x26, [x14, #0x40]\n"
"fmla z31.h, p3/M, z0.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x28, x5, LSL #1]\n"
+ "ldr x25, [x14, #0x48]\n"
"fmla z30.h, p3/M, z2.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x23, x5, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x20, x13, LSL #1]\n"
"fmla z29.h, p3/M, z2.h, z12.h\n"
+ "ldr x24, [x14, #0x50]\n"
"fmla z28.h, p3/M, z1.h, z12.h\n"
+ "ld1h { z10.h }, p2/Z, [x25, x13, LSL #1]\n"
"fmla z31.h, p3/M, z5.h, z12.h\n"
+ "ldr x23, [x14, #0x58]\n"
"fmla z30.h, p3/M, z4.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x7, x5, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x19, x13, LSL #1]\n"
"fmla z29.h, p3/M, z6.h, z9.h\n"
- "ld1h { z9.h }, p2/Z, [x17, x5, LSL #1]\n"
+ "ld1h { z9.h }, p2/Z, [x26, x13, LSL #1]\n"
"fmla z28.h, p3/M, z3.h, z13.h\n"
+ "ldr x22, [x14, #0x60]\n"
"fmla z31.h, p3/M, z7.h, z13.h\n"
+ "ldr x21, [x14, #0x68]\n"
"fmla z30.h, p3/M, z6.h, z13.h\n"
+ "ldr x20, [x14, #0x70]\n"
"fmla z29.h, p3/M, z4.h, z13.h\n"
- "ld1h { z13.h }, p1/Z, [x9, x8, LSL #1]\n"
+ "ldr x19, [x14, #0x78]\n"
"fmla z28.h, p3/M, z8.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x14, x5, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x24, x13, LSL #1]\n"
"fmla z31.h, p3/M, z1.h, z12.h\n"
+ "ldp x26, x25, [x14, #0x0]\n"
"fmla z30.h, p3/M, z0.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x11, x5, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x23, x13, LSL #1]\n"
"fmla z29.h, p3/M, z5.h, z10.h\n"
+ "ldp x24, x23, [x14, #0x10]\n"
"fmla z28.h, p3/M, z4.h, z10.h\n"
- "ld1h { z4.h }, p3/Z, [x4, #5, MUL VL]\n" // Load from weights and bias
+ "ld1h { z16.h }, p3/Z, [x15]\n"
"fmla z31.h, p3/M, z2.h, z9.h\n"
+ "ld1h { z4.h }, p3/Z, [x15, #5, MUL VL]\n"
"fmla z30.h, p3/M, z1.h, z9.h\n"
- "ld1h { z9.h }, p2/Z, [x10, x5, LSL #1]\n"
+ "ld1h { z9.h }, p2/Z, [x22, x13, LSL #1]\n"
"fmla z29.h, p3/M, z0.h, z11.h\n"
- "ld1h { z0.h }, p3/Z, [x4, #1, MUL VL]\n" // Load from weights and bias
+ "ldr x22, [x14, #0x20]\n"
"fmla z28.h, p3/M, z2.h, z12.h\n"
- "ld1h { z1.h }, p3/Z, [x4, #2, MUL VL]\n" // Load from weights and bias
- "ld1h { z2.h }, p3/Z, [x4, #3, MUL VL]\n" // Load from weights and bias
+ "ld1h { z0.h }, p3/Z, [x15, #1, MUL VL]\n"
"fmla z31.h, p3/M, z8.h, z10.h\n"
+ "ld1h { z1.h }, p3/Z, [x15, #2, MUL VL]\n"
"fmla z30.h, p3/M, z7.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x27, x5, LSL #1]\n"
+ "ld1h { z10.h }, p2/Z, [x21, x13, LSL #1]\n"
"fmla z29.h, p3/M, z3.h, z9.h\n"
+ "ld1h { z13.h }, p1/Z, [x22, x12, LSL #1]\n"
"fmla z31.h, p3/M, z3.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x25, x5, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x20, x13, LSL #1]\n"
"fmla z28.h, p3/M, z5.h, z10.h\n"
- "ld1h { z3.h }, p3/Z, [x4, #4, MUL VL]\n" // Load from weights and bias
+ "ld1h { z2.h }, p3/Z, [x15, #3, MUL VL]\n"
"fmla z30.h, p3/M, z5.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x24, x5, LSL #1]\n"
- "inch x5\n"
+ "ld1h { z12.h }, p2/Z, [x19, x13, LSL #1]\n"
+ "inch x13\n"
"fmla z29.h, p3/M, z7.h, z11.h\n"
- "ld1h { z5.h }, p3/Z, [x4, #6, MUL VL]\n" // Load from weights and bias
- "whilelt p2.h, x5, %x[n_channels]\n"
+ "ld1h { z3.h }, p3/Z, [x15, #4, MUL VL]\n"
+ "whilelt p2.h, x13, %x[n_channels]\n"
"fmla z31.h, p3/M, z6.h, z9.h\n"
- "ld1h { z9.h }, p1/Z, [x13, x8, LSL #1]\n"
+ "ld1h { z9.h }, p1/Z, [x26, x12, LSL #1]\n"
"fmla z28.h, p3/M, z6.h, z11.h\n"
- "ld1h { z11.h }, p1/Z, [x16, x8, LSL #1]\n"
+ "ld1h { z11.h }, p1/Z, [x24, x12, LSL #1]\n"
"fmla z30.h, p3/M, z8.h, z10.h\n"
- "ld1h { z10.h }, p1/Z, [x6, x8, LSL #1]\n"
- "ld1h { z6.h }, p3/Z, [x4, #7, MUL VL]\n" // Load from weights and bias
+ "ld1h { z10.h }, p1/Z, [x25, x12, LSL #1]\n"
+ "ld1h { z5.h }, p3/Z, [x15, #6, MUL VL]\n"
"fmla z29.h, p3/M, z8.h, z12.h\n"
- "addvl x4, x4, #16\n"
+ "ld1h { z6.h }, p3/Z, [x15, #7, MUL VL]\n"
"fmla z28.h, p3/M, z7.h, z12.h\n"
- "ld1h { z12.h }, p1/Z, [x12, x8, LSL #1]\n"
- "inch x8\n"
+ "addvl x15, x15, #16\n"
"fmax z31.h, p3/M, z31.h, z18.h\n"
- "ld1h { z7.h }, p3/Z, [x4, #-8, MUL VL]\n" // Load from weights and bias
- "cmp x8, %x[n_channels]\n"
+ "ld1h { z12.h }, p1/Z, [x23, x12, LSL #1]\n"
+ "inch x12\n"
"fmax z30.h, p3/M, z30.h, z18.h\n"
- "ld1h { z8.h }, p3/Z, [x4, #-7, MUL VL]\n" // Load from weights and bias
- "addvl x4, x4, #-6\n"
+ "ld1h { z7.h }, p3/Z, [x15, #-8, MUL VL]\n"
+ "cmp x12, %x[n_channels]\n"
"fmax z29.h, p3/M, z29.h, z18.h\n"
+ "ld1h { z8.h }, p3/Z, [x15, #-7, MUL VL]\n"
+ "addvl x15, x15, #-6\n"
"fmax z28.h, p3/M, z28.h, z18.h\n"
"fmin z31.h, p3/M, z31.h, z17.h\n"
- "st1h { z31.h }, p0, [x22, x15, LSL #1]\n"
- "mov z31.d, z16.d\n"
+ "st1h { z31.h }, p0, [x11, x9, LSL #1]\n"
"fmin z30.h, p3/M, z30.h, z17.h\n"
- "st1h { z30.h }, p0, [x21, x15, LSL #1]\n"
- "mov z30.d, z16.d\n"
"fmin z29.h, p3/M, z29.h, z17.h\n"
- "st1h { z29.h }, p0, [x20, x15, LSL #1]\n"
- "mov z29.d, z16.d\n"
+ "st1h { z30.h }, p0, [x10, x9, LSL #1]\n"
"fmin z28.h, p3/M, z28.h, z17.h\n"
- "st1h { z28.h }, p0, [x19, x15, LSL #1]\n"
- "mov z28.d, z16.d\n"
+ "st1h { z29.h }, p0, [x28, x9, LSL #1]\n"
+ "st1h { z28.h }, p0, [x27, x9, LSL #1]\n"
"blt 1b\n"
"2:" // Channel tail
- "fmla z31.h, p3/M, z4.h, z9.h\n"
- "inch x15\n"
- "fmla z30.h, p3/M, z3.h, z9.h\n"
+ "movprfx z31, z16\n fmla z31.h, p3/M, z4.h, z9.h\n"
+ "ldr x21, [x14, #0x28]\n"
+ "inch x9\n"
+ "movprfx z30, z16\n fmla z30.h, p3/M, z3.h, z9.h\n"
+ "ldr x20, [x14, #0x30]\n"
"mov p0.b, p2.b\n"
- "fmla z29.h, p3/M, z1.h, z9.h\n"
- "fmla z28.h, p3/M, z0.h, z9.h\n"
- "ld1h { z9.h }, p2/Z, [x26, x5, LSL #1]\n"
+ "movprfx z29, z16\n fmla z29.h, p3/M, z1.h, z9.h\n"
+ "ldr x19, [x14, #0x38]\n"
+ "movprfx z28, z16\n fmla z28.h, p3/M, z0.h, z9.h\n"
+ "ld1h { z9.h }, p2/Z, [x21, x13, LSL #1]\n"
+ "ldr x26, [x14, #0x40]\n"
"fmla z31.h, p3/M, z0.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x28, x5, LSL #1]\n"
+ "ldr x25, [x14, #0x48]\n"
"fmla z30.h, p3/M, z2.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x23, x5, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x20, x13, LSL #1]\n"
"fmla z29.h, p3/M, z2.h, z12.h\n"
"fmla z28.h, p3/M, z1.h, z12.h\n"
+ "ld1h { z10.h }, p2/Z, [x25, x13, LSL #1]\n"
+ "ldr x24, [x14, #0x50]\n"
"fmla z31.h, p3/M, z5.h, z12.h\n"
+ "ldr x23, [x14, #0x58]\n"
"fmla z30.h, p3/M, z4.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x7, x5, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x19, x13, LSL #1]\n"
"fmla z29.h, p3/M, z6.h, z9.h\n"
- "ld1h { z9.h }, p2/Z, [x17, x5, LSL #1]\n"
"fmla z28.h, p3/M, z3.h, z13.h\n"
+ "ld1h { z9.h }, p2/Z, [x26, x13, LSL #1]\n"
+ "ldr x22, [x14, #0x60]\n"
"fmla z31.h, p3/M, z7.h, z13.h\n"
+ "ldr x21, [x14, #0x68]\n"
"fmla z30.h, p3/M, z6.h, z13.h\n"
+ "ldr x20, [x14, #0x70]\n"
"fmla z29.h, p3/M, z4.h, z13.h\n"
"fmla z28.h, p3/M, z8.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x14, x5, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x24, x13, LSL #1]\n"
+ "ldr x19, [x14, #0x78]\n"
"fmla z31.h, p3/M, z1.h, z12.h\n"
"fmla z30.h, p3/M, z0.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x11, x5, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x23, x13, LSL #1]\n"
"fmla z29.h, p3/M, z5.h, z10.h\n"
"fmla z28.h, p3/M, z4.h, z10.h\n"
"fmla z31.h, p3/M, z2.h, z9.h\n"
"fmla z30.h, p3/M, z1.h, z9.h\n"
- "ld1h { z9.h }, p2/Z, [x10, x5, LSL #1]\n"
+ "ld1h { z9.h }, p2/Z, [x22, x13, LSL #1]\n"
"fmla z29.h, p3/M, z0.h, z11.h\n"
"fmla z28.h, p3/M, z2.h, z12.h\n"
"fmla z31.h, p3/M, z8.h, z10.h\n"
"fmla z30.h, p3/M, z7.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x27, x5, LSL #1]\n"
+ "ld1h { z10.h }, p2/Z, [x21, x13, LSL #1]\n"
"fmla z29.h, p3/M, z3.h, z9.h\n"
"fmla z31.h, p3/M, z3.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x25, x5, LSL #1]\n"
- "fmla z30.h, p3/M, z5.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x24, x5, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x20, x13, LSL #1]\n"
"fmla z28.h, p3/M, z5.h, z10.h\n"
+ "fmla z30.h, p3/M, z5.h, z12.h\n"
+ "ld1h { z12.h }, p2/Z, [x19, x13, LSL #1]\n"
"fmla z29.h, p3/M, z7.h, z11.h\n"
"fmla z31.h, p3/M, z6.h, z9.h\n"
- "fmla z30.h, p3/M, z8.h, z10.h\n"
"fmla z28.h, p3/M, z6.h, z11.h\n"
+ "fmla z30.h, p3/M, z8.h, z10.h\n"
"fmla z29.h, p3/M, z8.h, z12.h\n"
+ "fmla z28.h, p3/M, z7.h, z12.h\n"
"fmax z31.h, p3/M, z31.h, z18.h\n"
"fmax z30.h, p3/M, z30.h, z18.h\n"
- "fmla z28.h, p3/M, z7.h, z12.h\n"
"fmax z29.h, p3/M, z29.h, z18.h\n"
"fmin z31.h, p3/M, z31.h, z17.h\n"
- "st1h { z31.h }, p0, [x22, x15, LSL #1]\n"
+ "st1h { z31.h }, p0, [x11, x9, LSL #1]\n"
"fmin z30.h, p3/M, z30.h, z17.h\n"
"fmin z29.h, p3/M, z29.h, z17.h\n"
- "st1h { z30.h }, p0, [x21, x15, LSL #1]\n"
+ "st1h { z30.h }, p0, [x10, x9, LSL #1]\n"
"fmax z28.h, p3/M, z28.h, z18.h\n"
- "st1h { z29.h }, p0, [x20, x15, LSL #1]\n"
+ "st1h { z29.h }, p0, [x28, x9, LSL #1]\n"
"fmin z28.h, p3/M, z28.h, z17.h\n"
- "st1h { z28.h }, p0, [x19, x15, LSL #1]\n"
+ "st1h { z28.h }, p0, [x27, x9, LSL #1]\n"
:
: [n_channels] "r" ((unsigned long) n_channels), [offsetof_Args_inptrs] "I" (offsetof(Args, inptrs)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_outptrs] "I" (offsetof(Args, outptrs)), [offsetof_args_params] "I" (offsetof(Args, params)), [params_struct] "r" (¶ms_struct)
- : "cc", "memory", "p0", "p1", "p2", "p3", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z16", "z17", "z18", "z28", "z29", "z30", "z31"
+ : "cc", "memory", "p0", "p1", "p2", "p3", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z16", "z17", "z18", "z28", "z29", "z30", "z31"
);
}
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
+#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp
index e1f23aa..af8af18 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp
@@ -28,7 +28,7 @@
#pragma once
-#if defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
+#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
namespace arm_conv {
namespace depthwise {
@@ -36,15 +36,17 @@
void sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16);
void sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16);
-struct sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst
+class sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst : public IDepthwiseDepthfirstStrategy
{
- typedef __fp16 bias_type;
- typedef __fp16 input_type;
- typedef __fp16 weight_type;
- typedef __fp16 return_type;
-
+ private:
typedef void (*indirect_kern_type)(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16);
+ indirect_kern_type m_indirect_kernel = sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl;
+
typedef void (*direct_kern_type)(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16);
+ direct_kern_type m_direct_kernel = sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl;
+
+ public:
+ typedef __fp16 return_type;
constexpr static arm_gemm::VLType vl_type = arm_gemm::VLType::SVE;
@@ -60,13 +62,60 @@
constexpr static unsigned int input_rows = 5;
constexpr static unsigned int input_cols = 5;
- indirect_kern_type indirect_kernel = sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl;
- direct_kern_type direct_kernel = sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl;
-
sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst(const CPUInfo *) {}
+
+ arm_gemm::VLType get_vl_type(void) const override { return vl_type; }
+
+ unsigned int get_kernel_rows(void) const override { return kernel_rows; }
+ unsigned int get_kernel_cols(void) const override { return kernel_cols; }
+
+ unsigned int get_stride_rows(void) const override { return stride_rows; }
+ unsigned int get_stride_cols(void) const override { return stride_cols; }
+
+ unsigned int get_output_rows(void) const override { return output_rows; }
+ unsigned int get_output_cols(void) const override { return output_cols; }
+
+ unsigned int get_input_rows(void) const override { return input_rows; }
+ unsigned int get_input_cols(void) const override { return input_cols; }
+
+ void indirect_kernel(
+ const void *const *const input_ptrs,
+ void *const *const outptrs,
+ const void *params,
+ unsigned int n_channels,
+ const void *activation_min,
+ const void *activation_max
+ ) const override
+ {
+ m_indirect_kernel(
+ reinterpret_cast<const __fp16 *const *>(input_ptrs),
+ reinterpret_cast<__fp16 *const *>(outptrs),
+ params, n_channels,
+ *static_cast<const __fp16 *>(activation_min),
+ *static_cast<const __fp16 *>(activation_max)
+ );
+ }
+
+ void direct_kernel(
+ const unsigned int n_tile_rows, const unsigned int n_tile_cols,
+ const void *inptr, int64_t ld_input_row, int64_t ld_input_col,
+ void *outptr, int64_t ld_output_row, int64_t ld_output_col,
+ const void *params, unsigned int n_channels,
+ const void *activation_min, const void *activation_max
+ ) const override
+ {
+ m_direct_kernel(
+ n_tile_rows, n_tile_cols,
+ static_cast<const __fp16 *>(inptr), ld_input_row, ld_input_col,
+ static_cast<__fp16 *>(outptr), ld_output_row, ld_output_col,
+ params, n_channels,
+ *static_cast<const __fp16 *>(activation_min),
+ *static_cast<const __fp16 *>(activation_max)
+ );
+ }
};
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
+#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp
index 0708f57..4126cef 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp
@@ -25,7 +25,7 @@
#include <cstddef>
#include <cstdint>
-#if defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
+#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
namespace arm_conv {
namespace depthwise {
@@ -113,107 +113,98 @@
"add x11, x14, x22, LSL #1\n"
"ld1rh { z17.h }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n"
"add x10, x11, x22, LSL #1\n"
- "ld1h { z16.h }, p3/Z, [x8]\n" // Load from weights and bias
- "mov z31.d, z16.d\n"
- "ld1h { z0.h }, p3/Z, [x8, #1, MUL VL]\n" // Load from weights and bias
+ "ld1h { z16.h }, p3/Z, [x8]\n"
"add x9, x10, x22, LSL #1\n"
- "mov z30.d, z16.d\n"
- "ld1h { z1.h }, p3/Z, [x8, #2, MUL VL]\n" // Load from weights and bias
+ "ld1h { z0.h }, p3/Z, [x8, #1, MUL VL]\n"
"add x28, x9, x22, LSL #1\n"
- "mov z29.d, z16.d\n"
- "ld1h { z2.h }, p3/Z, [x8, #3, MUL VL]\n" // Load from weights and bias
+ "ld1h { z1.h }, p3/Z, [x8, #2, MUL VL]\n"
"add x27, x15, x15\n"
- "mov z28.d, z16.d\n"
- "ld1h { z3.h }, p3/Z, [x8, #4, MUL VL]\n" // Load from weights and bias
+ "ld1h { z2.h }, p3/Z, [x8, #3, MUL VL]\n"
"add x26, x27, x15\n"
- "mov z27.d, z16.d\n"
- "ld1h { z4.h }, p3/Z, [x8, #5, MUL VL]\n" // Load from weights and bias
+ "ld1h { z3.h }, p3/Z, [x8, #4, MUL VL]\n"
"add x25, x26, x15\n"
- "mov z26.d, z16.d\n"
- "ld1h { z5.h }, p3/Z, [x8, #6, MUL VL]\n" // Load from weights and bias
+ "ld1h { z4.h }, p3/Z, [x8, #5, MUL VL]\n"
"mul x19, x6, x20\n" // offset = tile_i * ld_output_row
- "mov z25.d, z16.d\n"
- "ld1h { z6.h }, p3/Z, [x8, #7, MUL VL]\n" // Load from weights and bias
+ "ld1h { z5.h }, p3/Z, [x8, #6, MUL VL]\n"
"madd x19, x7, x13, x19\n" // offset += tile_j * ld_output_col
- "mov z24.d, z16.d\n"
+ "ld1h { z6.h }, p3/Z, [x8, #7, MUL VL]\n"
"mul x19, x19, x23\n" // offset *= output_tile_size
- "mov z23.d, z16.d\n"
- "add x12, x12, x19, LSL #1\n" // outptrs[0] += offset * sizeof(__fp16)
"add x24, x13, x13\n"
+ "add x12, x12, x19, LSL #1\n" // outptrs[0] += offset * sizeof(__fp16)
"add x23, x12, x20, LSL #1\n"
"add x22, x23, x20, LSL #1\n"
"whilelt p2.h, XZR, %x[n_channels]\n"
- "ld1h { z9.h }, p2/Z, [x10, x27, LSL #1]\n" // Load input point (2, 2)
- "ld1h { z10.h }, p2/Z, [x14]\n" // Load input point (0, 0)
+ "ld1h { z9.h }, p2/Z, [x10, x27, LSL #1]\n"
+ "ld1h { z10.h }, p2/Z, [x14]\n"
"addvl x8, x8, #16\n"
- "ld1h { z11.h }, p2/Z, [x14, x25, LSL #1]\n" // Load input point (0, 4)
+ "ld1h { z11.h }, p2/Z, [x14, x25, LSL #1]\n"
"cmp x16, %x[n_channels]\n"
- "ld1h { z7.h }, p3/Z, [x8, #-8, MUL VL]\n" // Load from weights and bias
- "ld1h { z8.h }, p3/Z, [x8, #-7, MUL VL]\n" // Load from weights and bias
+ "ld1h { z7.h }, p3/Z, [x8, #-8, MUL VL]\n"
+ "ld1h { z8.h }, p3/Z, [x8, #-7, MUL VL]\n"
"addvl x8, x8, #-6\n"
- "ld1h { z12.h }, p2/Z, [x28]\n" // Load input point (4, 0)
- "ld1h { z13.h }, p2/Z, [x11, x27, LSL #1]\n" // Load input point (1, 2)
+ "ld1h { z12.h }, p2/Z, [x28]\n"
+ "ld1h { z13.h }, p2/Z, [x11, x27, LSL #1]\n"
"bge 3f\n"
"2:" // Tile loop: Channel loop
- "fmla z31.h, p3/M, z8.h, z9.h\n"
- "ld1h { z16.h }, p3/Z, [x8]\n" // Load from weights and bias
+ "movprfx z31, z16\n fmla z31.h, p3/M, z8.h, z9.h\n"
"whilelt p1.h, x16, %x[n_channels]\n"
- "fmla z30.h, p3/M, z7.h, z9.h\n"
+ "movprfx z30, z16\n fmla z30.h, p3/M, z7.h, z9.h\n"
"inch x21\n"
- "fmla z29.h, p3/M, z6.h, z9.h\n"
+ "movprfx z29, z16\n fmla z29.h, p3/M, z6.h, z9.h\n"
"mov p0.b, p2.b\n"
- "fmla z28.h, p3/M, z5.h, z9.h\n"
+ "movprfx z28, z16\n fmla z28.h, p3/M, z5.h, z9.h\n"
"inch x17\n"
- "fmla z27.h, p3/M, z4.h, z9.h\n"
+ "movprfx z27, z16\n fmla z27.h, p3/M, z4.h, z9.h\n"
"inch x16\n"
- "fmla z26.h, p3/M, z3.h, z9.h\n"
- "fmla z25.h, p3/M, z2.h, z9.h\n"
- "fmla z24.h, p3/M, z1.h, z9.h\n"
- "fmla z23.h, p3/M, z0.h, z9.h\n"
+ "movprfx z26, z16\n fmla z26.h, p3/M, z3.h, z9.h\n"
+ "movprfx z25, z16\n fmla z25.h, p3/M, z2.h, z9.h\n"
+ "movprfx z24, z16\n fmla z24.h, p3/M, z1.h, z9.h\n"
+ "movprfx z23, z16\n fmla z23.h, p3/M, z0.h, z9.h\n"
+ "ld1h { z16.h }, p3/Z, [x8]\n"
"fmla z31.h, p3/M, z0.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x10, x26, LSL #1]\n" // Load input point (2, 3)
+ "ld1h { z10.h }, p2/Z, [x10, x26, LSL #1]\n"
"fmla z29.h, p3/M, z2.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x10, x15, LSL #1]\n" // Load input point (2, 1)
+ "ld1h { z11.h }, p2/Z, [x10, x15, LSL #1]\n"
"fmla z25.h, p3/M, z6.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x28, x25, LSL #1]\n" // Load input point (4, 4)
+ "ld1h { z12.h }, p2/Z, [x28, x25, LSL #1]\n"
"fmla z30.h, p3/M, z4.h, z13.h\n"
"fmla z31.h, p3/M, z5.h, z13.h\n"
"fmla z29.h, p3/M, z3.h, z13.h\n"
"fmla z28.h, p3/M, z2.h, z13.h\n"
"fmla z27.h, p3/M, z1.h, z13.h\n"
"fmla z26.h, p3/M, z0.h, z13.h\n"
- "ld1h { z13.h }, p2/Z, [x14, x15, LSL #1]\n" // Load input point (0, 1)
+ "ld1h { z13.h }, p2/Z, [x14, x15, LSL #1]\n"
"fmla z23.h, p3/M, z8.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x14, x26, LSL #1]\n" // Load input point (0, 3)
+ "ld1h { z12.h }, p2/Z, [x14, x26, LSL #1]\n"
"fmla z31.h, p3/M, z7.h, z11.h\n"
"fmla z30.h, p3/M, z6.h, z11.h\n"
"fmla z28.h, p3/M, z4.h, z11.h\n"
"fmla z27.h, p3/M, z3.h, z11.h\n"
"fmla z25.h, p3/M, z1.h, z11.h\n"
"fmla z24.h, p3/M, z0.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x11]\n" // Load input point (1, 0)
+ "ld1h { z11.h }, p2/Z, [x11]\n"
"fmla z31.h, p3/M, z1.h, z13.h\n"
"fmla z30.h, p3/M, z0.h, z13.h\n"
- "ld1h { z13.h }, p2/Z, [x11, x25, LSL #1]\n" // Load input point (1, 4)
+ "ld1h { z13.h }, p2/Z, [x11, x25, LSL #1]\n"
"fmla z29.h, p3/M, z1.h, z12.h\n"
"fmla z27.h, p3/M, z5.h, z10.h\n"
"fmla z26.h, p3/M, z4.h, z10.h\n"
"fmla z30.h, p3/M, z2.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x9]\n" // Load input point (3, 0)
+ "ld1h { z12.h }, p2/Z, [x9]\n"
"fmla z29.h, p3/M, z7.h, z10.h\n"
"fmla z24.h, p3/M, z2.h, z10.h\n"
"fmla z23.h, p3/M, z1.h, z10.h\n"
"fmla z30.h, p3/M, z8.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x9, x27, LSL #1]\n" // Load input point (3, 2)
+ "ld1h { z10.h }, p2/Z, [x9, x27, LSL #1]\n"
"fmla z31.h, p3/M, z3.h, z11.h\n"
"fmla z28.h, p3/M, z0.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x9, x25, LSL #1]\n" // Load input point (3, 4)
+ "ld1h { z11.h }, p2/Z, [x9, x25, LSL #1]\n"
"fmla z29.h, p3/M, z5.h, z13.h\n"
"fmla z26.h, p3/M, z2.h, z13.h\n"
- "ld1h { z13.h }, p2/Z, [x28, x15, LSL #1]\n" // Load input point (4, 1)
+ "ld1h { z13.h }, p2/Z, [x28, x15, LSL #1]\n"
"fmla z25.h, p3/M, z3.h, z12.h\n"
"fmla z28.h, p3/M, z6.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x11, x15, LSL #1]\n" // Load input point (1, 1)
+ "ld1h { z12.h }, p2/Z, [x11, x15, LSL #1]\n"
"fmla z27.h, p3/M, z7.h, z10.h\n"
"fmla z26.h, p3/M, z6.h, z10.h\n"
"fmla z25.h, p3/M, z5.h, z10.h\n"
@@ -223,173 +214,164 @@
"fmla z26.h, p3/M, z8.h, z11.h\n"
"fmla z25.h, p3/M, z7.h, z13.h\n"
"fmla z24.h, p3/M, z6.h, z13.h\n"
- "ld1h { z13.h }, p2/Z, [x28, x26, LSL #1]\n" // Load input point (4, 3)
+ "ld1h { z13.h }, p2/Z, [x28, x26, LSL #1]\n"
"fmla z23.h, p3/M, z5.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x11, x26, LSL #1]\n" // Load input point (1, 3)
+ "ld1h { z11.h }, p2/Z, [x11, x26, LSL #1]\n"
"addvl x11, x11, #1\n"
"fmla z31.h, p3/M, z4.h, z12.h\n"
"fmla z30.h, p3/M, z3.h, z12.h\n"
"fmla z28.h, p3/M, z1.h, z12.h\n"
"fmla z27.h, p3/M, z0.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x9, x15, LSL #1]\n" // Load input point (3, 1)
+ "ld1h { z12.h }, p2/Z, [x9, x15, LSL #1]\n"
"fmla z29.h, p3/M, z4.h, z11.h\n"
"fmla z30.h, p3/M, z5.h, z11.h\n"
"fmla z26.h, p3/M, z1.h, z11.h\n"
"fmla z27.h, p3/M, z2.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x14, x27, LSL #1]\n" // Load input point (0, 2)
+ "ld1h { z11.h }, p2/Z, [x14, x27, LSL #1]\n"
"addvl x14, x14, #1\n"
"fmla z24.h, p3/M, z8.h, z13.h\n"
- "ld1h { z10.h }, p1/Z, [x14]\n" // Load input point (0, 0)
+ "ld1h { z10.h }, p1/Z, [x14]\n"
"fmla z23.h, p3/M, z7.h, z13.h\n"
- "ld1h { z13.h }, p2/Z, [x9, x26, LSL #1]\n" // Load input point (3, 3)
+ "ld1h { z13.h }, p2/Z, [x9, x26, LSL #1]\n"
"addvl x9, x9, #1\n"
"fmla z28.h, p3/M, z7.h, z12.h\n"
"fmla z27.h, p3/M, z6.h, z12.h\n"
"fmla z25.h, p3/M, z4.h, z12.h\n"
"fmla z24.h, p3/M, z3.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x10]\n" // Load input point (2, 0)
+ "ld1h { z12.h }, p2/Z, [x10]\n"
"fmla z31.h, p3/M, z2.h, z11.h\n"
"fmla z30.h, p3/M, z1.h, z11.h\n"
- "ld1h { z1.h }, p3/Z, [x8, #2, MUL VL]\n" // Load from weights and bias
+ "ld1h { z1.h }, p3/Z, [x8, #2, MUL VL]\n"
"fmla z29.h, p3/M, z0.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x10, x25, LSL #1]\n" // Load input point (2, 4)
+ "ld1h { z11.h }, p2/Z, [x10, x25, LSL #1]\n"
"addvl x10, x10, #1\n"
"fmla z27.h, p3/M, z8.h, z13.h\n"
- "ld1h { z9.h }, p1/Z, [x10, x27, LSL #1]\n" // Load input point (2, 2)
+ "ld1h { z9.h }, p1/Z, [x10, x27, LSL #1]\n"
"fmla z26.h, p3/M, z7.h, z13.h\n"
"fmla z24.h, p3/M, z5.h, z13.h\n"
"fmla z23.h, p3/M, z4.h, z13.h\n"
- "ld1h { z13.h }, p2/Z, [x28, x27, LSL #1]\n" // Load input point (4, 2)
+ "ld1h { z13.h }, p2/Z, [x28, x27, LSL #1]\n"
"whilelt p2.h, x17, %x[n_channels]\n"
"fmla z31.h, p3/M, z6.h, z12.h\n"
- "ld1h { z4.h }, p3/Z, [x8, #5, MUL VL]\n" // Load from weights and bias
+ "ld1h { z4.h }, p3/Z, [x8, #5, MUL VL]\n"
"addvl x28, x28, #1\n"
"fmla z28.h, p3/M, z3.h, z12.h\n"
- "ld1h { z3.h }, p3/Z, [x8, #4, MUL VL]\n" // Load from weights and bias
+ "ld1h { z3.h }, p3/Z, [x8, #4, MUL VL]\n"
"cmp x16, %x[n_channels]\n"
"fmla z25.h, p3/M, z0.h, z12.h\n"
- "ld1h { z12.h }, p1/Z, [x28]\n" // Load input point (4, 0)
+ "ld1h { z12.h }, p1/Z, [x28]\n"
"fmla z29.h, p3/M, z8.h, z11.h\n"
- "ld1h { z0.h }, p3/Z, [x8, #1, MUL VL]\n" // Load from weights and bias
+ "ld1h { z0.h }, p3/Z, [x8, #1, MUL VL]\n"
"fmla z26.h, p3/M, z5.h, z11.h\n"
- "ld1h { z5.h }, p3/Z, [x8, #6, MUL VL]\n" // Load from weights and bias
+ "ld1h { z5.h }, p3/Z, [x8, #6, MUL VL]\n"
"fmla z23.h, p3/M, z2.h, z11.h\n"
- "ld1h { z11.h }, p1/Z, [x14, x25, LSL #1]\n" // Load input point (0, 4)
+ "ld1h { z11.h }, p1/Z, [x14, x25, LSL #1]\n"
"fmla z24.h, p3/M, z7.h, z13.h\n"
- "ld1h { z2.h }, p3/Z, [x8, #3, MUL VL]\n" // Load from weights and bias
+ "ld1h { z2.h }, p3/Z, [x8, #3, MUL VL]\n"
"fmla z25.h, p3/M, z8.h, z13.h\n"
"fmax z31.h, p3/M, z31.h, z18.h\n"
"fmla z23.h, p3/M, z6.h, z13.h\n"
- "ld1h { z13.h }, p1/Z, [x11, x27, LSL #1]\n" // Load input point (1, 2)
+ "ld1h { z13.h }, p1/Z, [x11, x27, LSL #1]\n"
"fmax z30.h, p3/M, z30.h, z18.h\n"
- "ld1h { z6.h }, p3/Z, [x8, #7, MUL VL]\n" // Load from weights and bias
+ "ld1h { z6.h }, p3/Z, [x8, #7, MUL VL]\n"
"addvl x8, x8, #16\n"
"fmin z31.h, p3/M, z31.h, z17.h\n"
- "ld1h { z7.h }, p3/Z, [x8, #-8, MUL VL]\n" // Load from weights and bias
+ "ld1h { z7.h }, p3/Z, [x8, #-8, MUL VL]\n"
"fmax z29.h, p3/M, z29.h, z18.h\n"
- "ld1h { z8.h }, p3/Z, [x8, #-7, MUL VL]\n" // Load from weights and bias
+ "ld1h { z8.h }, p3/Z, [x8, #-7, MUL VL]\n"
"addvl x8, x8, #-6\n"
"fmin z30.h, p3/M, z30.h, z17.h\n"
- "st1h { z31.h }, p0, [x12]\n" // Store output point (0, 0)
- "mov z31.d, z16.d\n"
+ "st1h { z31.h }, p0, [x12]\n"
"fmin z29.h, p3/M, z29.h, z17.h\n"
- "st1h { z30.h }, p0, [x12, x13, LSL #1]\n" // Store output point (0, 1)
- "mov z30.d, z16.d\n"
"fmax z28.h, p3/M, z28.h, z18.h\n"
- "st1h { z29.h }, p0, [x12, x24, LSL #1]\n" // Store output point (0, 2)
- "mov z29.d, z16.d\n"
- "addvl x12, x12, #1\n"
+ "st1h { z30.h }, p0, [x12, x13, LSL #1]\n"
"fmax z27.h, p3/M, z27.h, z18.h\n"
"fmax z26.h, p3/M, z26.h, z18.h\n"
+ "st1h { z29.h }, p0, [x12, x24, LSL #1]\n"
"fmin z28.h, p3/M, z28.h, z17.h\n"
- "st1h { z28.h }, p0, [x23]\n" // Store output point (1, 0)
- "mov z28.d, z16.d\n"
- "fmin z27.h, p3/M, z27.h, z17.h\n"
- "st1h { z27.h }, p0, [x23, x13, LSL #1]\n" // Store output point (1, 1)
- "mov z27.d, z16.d\n"
- "fmin z26.h, p3/M, z26.h, z17.h\n"
- "st1h { z26.h }, p0, [x23, x24, LSL #1]\n" // Store output point (1, 2)
- "mov z26.d, z16.d\n"
- "addvl x23, x23, #1\n"
+ "addvl x12, x12, #1\n"
"fmax z25.h, p3/M, z25.h, z18.h\n"
- "fmax z24.h, p3/M, z24.h, z18.h\n"
- "fmax z23.h, p3/M, z23.h, z18.h\n"
+ "st1h { z28.h }, p0, [x23]\n"
+ "fmin z27.h, p3/M, z27.h, z17.h\n"
+ "fmin z26.h, p3/M, z26.h, z17.h\n"
+ "st1h { z27.h }, p0, [x23, x13, LSL #1]\n"
"fmin z25.h, p3/M, z25.h, z17.h\n"
- "st1h { z25.h }, p0, [x22]\n" // Store output point (2, 0)
- "mov z25.d, z16.d\n"
+ "fmax z24.h, p3/M, z24.h, z18.h\n"
+ "st1h { z26.h }, p0, [x23, x24, LSL #1]\n"
+ "addvl x23, x23, #1\n"
+ "fmax z23.h, p3/M, z23.h, z18.h\n"
+ "st1h { z25.h }, p0, [x22]\n"
"fmin z24.h, p3/M, z24.h, z17.h\n"
- "st1h { z24.h }, p0, [x22, x13, LSL #1]\n" // Store output point (2, 1)
- "mov z24.d, z16.d\n"
"fmin z23.h, p3/M, z23.h, z17.h\n"
- "st1h { z23.h }, p0, [x22, x24, LSL #1]\n" // Store output point (2, 2)
- "mov z23.d, z16.d\n"
+ "st1h { z24.h }, p0, [x22, x13, LSL #1]\n"
+ "st1h { z23.h }, p0, [x22, x24, LSL #1]\n"
"addvl x22, x22, #1\n"
"blt 2b\n"
"3:" // Tile loop: Channel tail
- "fmla z31.h, p3/M, z8.h, z9.h\n"
+ "movprfx z31, z16\n fmla z31.h, p3/M, z8.h, z9.h\n"
"ldr x6, [%x[params_struct], %[offsetof_args_tile_i]]\n"
"mov p0.b, p2.b\n"
- "fmla z30.h, p3/M, z7.h, z9.h\n"
+ "movprfx z30, z16\n fmla z30.h, p3/M, z7.h, z9.h\n"
"ldr x7, [%x[params_struct], %[offsetof_args_tile_j]]\n"
"add x21, x6, #0x1\n"
- "fmla z29.h, p3/M, z6.h, z9.h\n"
+ "movprfx z29, z16\n fmla z29.h, p3/M, z6.h, z9.h\n"
"ldr x20, [%x[params_struct], %[offsetof_args_n_tile_rows]]\n"
- "fmla z28.h, p3/M, z5.h, z9.h\n"
+ "movprfx z28, z16\n fmla z28.h, p3/M, z5.h, z9.h\n"
"ldr x19, [%x[params_struct], %[offsetof_args_n_tile_cols]]\n"
"add x7, x7, #0x1\n"
- "fmla z27.h, p3/M, z4.h, z9.h\n"
+ "movprfx z27, z16\n fmla z27.h, p3/M, z4.h, z9.h\n"
"cmp x7, x19\n"
- "fmla z26.h, p3/M, z3.h, z9.h\n"
- "fmla z25.h, p3/M, z2.h, z9.h\n"
+ "movprfx z26, z16\n fmla z26.h, p3/M, z3.h, z9.h\n"
+ "movprfx z25, z16\n fmla z25.h, p3/M, z2.h, z9.h\n"
"csel x7, x7, XZR, LT\n"
- "fmla z24.h, p3/M, z1.h, z9.h\n"
+ "movprfx z24, z16\n fmla z24.h, p3/M, z1.h, z9.h\n"
"csel x6, x6, x21, LT\n"
- "fmla z23.h, p3/M, z0.h, z9.h\n"
+ "movprfx z23, z16\n fmla z23.h, p3/M, z0.h, z9.h\n"
"cmp x6, x20\n"
"fmla z31.h, p3/M, z0.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x10, x26, LSL #1]\n" // Load input point (2, 3)
+ "ld1h { z10.h }, p2/Z, [x10, x26, LSL #1]\n"
"fmla z29.h, p3/M, z2.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x10, x15, LSL #1]\n" // Load input point (2, 1)
+ "ld1h { z11.h }, p2/Z, [x10, x15, LSL #1]\n"
"fmla z25.h, p3/M, z6.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x28, x25, LSL #1]\n" // Load input point (4, 4)
+ "ld1h { z12.h }, p2/Z, [x28, x25, LSL #1]\n"
"fmla z30.h, p3/M, z4.h, z13.h\n"
"fmla z31.h, p3/M, z5.h, z13.h\n"
"fmla z29.h, p3/M, z3.h, z13.h\n"
"fmla z28.h, p3/M, z2.h, z13.h\n"
"fmla z27.h, p3/M, z1.h, z13.h\n"
"fmla z26.h, p3/M, z0.h, z13.h\n"
- "ld1h { z13.h }, p2/Z, [x14, x15, LSL #1]\n" // Load input point (0, 1)
+ "ld1h { z13.h }, p2/Z, [x14, x15, LSL #1]\n"
"fmla z23.h, p3/M, z8.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x14, x26, LSL #1]\n" // Load input point (0, 3)
+ "ld1h { z12.h }, p2/Z, [x14, x26, LSL #1]\n"
"fmla z31.h, p3/M, z7.h, z11.h\n"
"fmla z30.h, p3/M, z6.h, z11.h\n"
"fmla z28.h, p3/M, z4.h, z11.h\n"
"fmla z27.h, p3/M, z3.h, z11.h\n"
"fmla z25.h, p3/M, z1.h, z11.h\n"
"fmla z24.h, p3/M, z0.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x11]\n" // Load input point (1, 0)
+ "ld1h { z11.h }, p2/Z, [x11]\n"
"fmla z31.h, p3/M, z1.h, z13.h\n"
"fmla z30.h, p3/M, z0.h, z13.h\n"
- "ld1h { z13.h }, p2/Z, [x11, x25, LSL #1]\n" // Load input point (1, 4)
+ "ld1h { z13.h }, p2/Z, [x11, x25, LSL #1]\n"
"fmla z29.h, p3/M, z1.h, z12.h\n"
"fmla z27.h, p3/M, z5.h, z10.h\n"
"fmla z26.h, p3/M, z4.h, z10.h\n"
"fmla z30.h, p3/M, z2.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x9]\n" // Load input point (3, 0)
+ "ld1h { z12.h }, p2/Z, [x9]\n"
"fmla z29.h, p3/M, z7.h, z10.h\n"
"fmla z24.h, p3/M, z2.h, z10.h\n"
"fmla z23.h, p3/M, z1.h, z10.h\n"
"fmla z30.h, p3/M, z8.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x9, x27, LSL #1]\n" // Load input point (3, 2)
+ "ld1h { z10.h }, p2/Z, [x9, x27, LSL #1]\n"
"fmla z31.h, p3/M, z3.h, z11.h\n"
"fmla z28.h, p3/M, z0.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x9, x25, LSL #1]\n" // Load input point (3, 4)
+ "ld1h { z11.h }, p2/Z, [x9, x25, LSL #1]\n"
"fmla z29.h, p3/M, z5.h, z13.h\n"
"fmla z26.h, p3/M, z2.h, z13.h\n"
- "ld1h { z13.h }, p2/Z, [x28, x15, LSL #1]\n" // Load input point (4, 1)
+ "ld1h { z13.h }, p2/Z, [x28, x15, LSL #1]\n"
"fmla z25.h, p3/M, z3.h, z12.h\n"
"fmla z28.h, p3/M, z6.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x11, x15, LSL #1]\n" // Load input point (1, 1)
+ "ld1h { z12.h }, p2/Z, [x11, x15, LSL #1]\n"
"fmla z27.h, p3/M, z7.h, z10.h\n"
"fmla z26.h, p3/M, z6.h, z10.h\n"
"fmla z25.h, p3/M, z5.h, z10.h\n"
@@ -399,36 +381,36 @@
"fmla z26.h, p3/M, z8.h, z11.h\n"
"fmla z25.h, p3/M, z7.h, z13.h\n"
"fmla z24.h, p3/M, z6.h, z13.h\n"
- "ld1h { z13.h }, p2/Z, [x28, x26, LSL #1]\n" // Load input point (4, 3)
+ "ld1h { z13.h }, p2/Z, [x28, x26, LSL #1]\n"
"fmla z23.h, p3/M, z5.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x11, x26, LSL #1]\n" // Load input point (1, 3)
+ "ld1h { z11.h }, p2/Z, [x11, x26, LSL #1]\n"
"fmla z31.h, p3/M, z4.h, z12.h\n"
"fmla z30.h, p3/M, z3.h, z12.h\n"
"fmla z28.h, p3/M, z1.h, z12.h\n"
"fmla z27.h, p3/M, z0.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x9, x15, LSL #1]\n" // Load input point (3, 1)
+ "ld1h { z12.h }, p2/Z, [x9, x15, LSL #1]\n"
"fmla z29.h, p3/M, z4.h, z11.h\n"
"fmla z30.h, p3/M, z5.h, z11.h\n"
"fmla z26.h, p3/M, z1.h, z11.h\n"
"fmla z27.h, p3/M, z2.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x14, x27, LSL #1]\n" // Load input point (0, 2)
+ "ld1h { z11.h }, p2/Z, [x14, x27, LSL #1]\n"
"fmla z24.h, p3/M, z8.h, z13.h\n"
"fmla z23.h, p3/M, z7.h, z13.h\n"
- "ld1h { z13.h }, p2/Z, [x9, x26, LSL #1]\n" // Load input point (3, 3)
+ "ld1h { z13.h }, p2/Z, [x9, x26, LSL #1]\n"
"fmla z28.h, p3/M, z7.h, z12.h\n"
"fmla z27.h, p3/M, z6.h, z12.h\n"
"fmla z25.h, p3/M, z4.h, z12.h\n"
"fmla z24.h, p3/M, z3.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x10]\n" // Load input point (2, 0)
+ "ld1h { z12.h }, p2/Z, [x10]\n"
"fmla z31.h, p3/M, z2.h, z11.h\n"
"fmla z30.h, p3/M, z1.h, z11.h\n"
"fmla z29.h, p3/M, z0.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x10, x25, LSL #1]\n" // Load input point (2, 4)
+ "ld1h { z11.h }, p2/Z, [x10, x25, LSL #1]\n"
"fmla z27.h, p3/M, z8.h, z13.h\n"
"fmla z26.h, p3/M, z7.h, z13.h\n"
"fmla z24.h, p3/M, z5.h, z13.h\n"
"fmla z23.h, p3/M, z4.h, z13.h\n"
- "ld1h { z13.h }, p2/Z, [x28, x27, LSL #1]\n" // Load input point (4, 2)
+ "ld1h { z13.h }, p2/Z, [x28, x27, LSL #1]\n"
"fmla z31.h, p3/M, z6.h, z12.h\n"
"fmla z28.h, p3/M, z3.h, z12.h\n"
"fmla z25.h, p3/M, z0.h, z12.h\n"
@@ -442,29 +424,29 @@
"fmax z30.h, p3/M, z30.h, z18.h\n"
"fmax z29.h, p3/M, z29.h, z18.h\n"
"fmin z31.h, p3/M, z31.h, z17.h\n"
- "st1h { z31.h }, p0, [x12]\n" // Store output point (0, 0)
+ "st1h { z31.h }, p0, [x12]\n"
"fmin z30.h, p3/M, z30.h, z17.h\n"
"fmin z29.h, p3/M, z29.h, z17.h\n"
- "st1h { z30.h }, p0, [x12, x13, LSL #1]\n" // Store output point (0, 1)
+ "st1h { z30.h }, p0, [x12, x13, LSL #1]\n"
"fmax z28.h, p3/M, z28.h, z18.h\n"
"fmax z27.h, p3/M, z27.h, z18.h\n"
- "st1h { z29.h }, p0, [x12, x24, LSL #1]\n" // Store output point (0, 2)
+ "st1h { z29.h }, p0, [x12, x24, LSL #1]\n"
"fmax z26.h, p3/M, z26.h, z18.h\n"
"fmax z25.h, p3/M, z25.h, z18.h\n"
"fmax z24.h, p3/M, z24.h, z18.h\n"
"fmin z28.h, p3/M, z28.h, z17.h\n"
- "st1h { z28.h }, p0, [x23]\n" // Store output point (1, 0)
+ "st1h { z28.h }, p0, [x23]\n"
"fmin z27.h, p3/M, z27.h, z17.h\n"
"fmin z26.h, p3/M, z26.h, z17.h\n"
- "st1h { z27.h }, p0, [x23, x13, LSL #1]\n" // Store output point (1, 1)
+ "st1h { z27.h }, p0, [x23, x13, LSL #1]\n"
"fmin z25.h, p3/M, z25.h, z17.h\n"
"fmin z24.h, p3/M, z24.h, z17.h\n"
- "st1h { z26.h }, p0, [x23, x24, LSL #1]\n" // Store output point (1, 2)
+ "st1h { z26.h }, p0, [x23, x24, LSL #1]\n"
"fmax z23.h, p3/M, z23.h, z18.h\n"
- "st1h { z25.h }, p0, [x22]\n" // Store output point (2, 0)
+ "st1h { z25.h }, p0, [x22]\n"
"fmin z23.h, p3/M, z23.h, z17.h\n"
- "st1h { z24.h }, p0, [x22, x13, LSL #1]\n" // Store output point (2, 1)
- "st1h { z23.h }, p0, [x22, x24, LSL #1]\n" // Store output point (2, 2)
+ "st1h { z24.h }, p0, [x22, x13, LSL #1]\n"
+ "st1h { z23.h }, p0, [x22, x24, LSL #1]\n"
"blt 1b\n"
:
: [n_channels] "r" ((unsigned long) n_channels), [offsetof_args_inptr] "I" (offsetof(Args, inptr)), [offsetof_args_ld_input_col] "I" (offsetof(Args, ld_input_col)), [offsetof_args_ld_input_row] "I" (offsetof(Args, ld_input_row)), [offsetof_args_ld_output_col] "I" (offsetof(Args, ld_output_col)), [offsetof_args_ld_output_row] "I" (offsetof(Args, ld_output_row)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_n_tile_cols] "I" (offsetof(Args, n_tile_cols)), [offsetof_args_n_tile_rows] "I" (offsetof(Args, n_tile_rows)), [offsetof_args_outptr] "I" (offsetof(Args, outptr)), [offsetof_args_params] "I" (offsetof(Args, params)), [offsetof_args_tile_i] "I" (offsetof(Args, tile_i)), [offsetof_args_tile_j] "I" (offsetof(Args, tile_j)), [params_struct] "r" (¶ms_struct)
@@ -475,4 +457,4 @@
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
+#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp
index 770576c..f79a36b 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp
@@ -25,7 +25,7 @@
#include <cstddef>
#include <cstdint>
-#if defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
+#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
namespace arm_conv {
namespace depthwise {
@@ -87,130 +87,121 @@
activation_min, activation_max);
__asm__ __volatile__(
- "ldr x6, [%x[params_struct], %[offsetof_args_outptrs]]\n"
+ "ldr x16, [%x[params_struct], %[offsetof_args_outptrs]]\n"
"ptrue p3.b\n"
- "ldr x7, [%x[params_struct], %[offsetof_args_params]]\n"
- "add x8, %x[params_struct], %[offsetof_Args_inptrs]\n"
+ "ldr x15, [%x[params_struct], %[offsetof_args_params]]\n"
+ "add x14, %x[params_struct], %[offsetof_Args_inptrs]\n"
"ld1rh { z18.h }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n"
- "mov x17, #0x0\n"
+ "mov x13, #0x0\n"
"ld1rh { z17.h }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n"
- "cnth x16\n"
- "ld1h { z16.h }, p3/Z, [x7]\n" // Load from weights and bias
- "mov z31.d, z16.d\n"
- "ld1h { z0.h }, p3/Z, [x7, #1, MUL VL]\n" // Load from weights and bias
- "sub x15, XZR, x16\n"
- "mov z30.d, z16.d\n"
- "ld1h { z1.h }, p3/Z, [x7, #2, MUL VL]\n" // Load from weights and bias
+ "cnth x12\n"
+ "ld1h { z16.h }, p3/Z, [x15]\n"
+ "sub x11, XZR, x12\n"
+ "ld1h { z0.h }, p3/Z, [x15, #1, MUL VL]\n"
"whilelt p2.h, XZR, %x[n_channels]\n"
- "mov z29.d, z16.d\n"
- "ld1h { z2.h }, p3/Z, [x7, #3, MUL VL]\n" // Load from weights and bias
- "cmp x16, %x[n_channels]\n"
- "mov z28.d, z16.d\n"
- "ld1h { z3.h }, p3/Z, [x7, #4, MUL VL]\n" // Load from weights and bias
- "mov z27.d, z16.d\n"
- "ld1h { z4.h }, p3/Z, [x7, #5, MUL VL]\n" // Load from weights and bias
- "mov z26.d, z16.d\n"
- "ld1h { z5.h }, p3/Z, [x7, #6, MUL VL]\n" // Load from weights and bias
- "mov z25.d, z16.d\n"
- "ld1h { z6.h }, p3/Z, [x7, #7, MUL VL]\n" // Load from weights and bias
- "addvl x7, x7, #16\n"
- "mov z24.d, z16.d\n"
- "ld1h { z7.h }, p3/Z, [x7, #-8, MUL VL]\n" // Load from weights and bias
- "mov z23.d, z16.d\n"
- "ld1h { z8.h }, p3/Z, [x7, #-7, MUL VL]\n" // Load from weights and bias
- "addvl x7, x7, #-6\n"
- "ldp x14, x13, [x8, #0x0]\n"
- "ldp x12, x11, [x8, #0x10]\n"
- "ldr x10, [x8, #0x20]\n"
- "ld1h { z9.h }, p2/Z, [x14, x17, LSL #1]\n"
- "ld1h { z10.h }, p2/Z, [x13, x17, LSL #1]\n"
- "ld1h { z11.h }, p2/Z, [x12, x17, LSL #1]\n"
- "ld1h { z12.h }, p2/Z, [x11, x17, LSL #1]\n"
- "ld1h { z13.h }, p2/Z, [x10, x17, LSL #1]\n"
+ "ld1h { z1.h }, p3/Z, [x15, #2, MUL VL]\n"
+ "cmp x12, %x[n_channels]\n"
+ "ld1h { z2.h }, p3/Z, [x15, #3, MUL VL]\n"
+ "ld1h { z3.h }, p3/Z, [x15, #4, MUL VL]\n"
+ "ld1h { z4.h }, p3/Z, [x15, #5, MUL VL]\n"
+ "ld1h { z5.h }, p3/Z, [x15, #6, MUL VL]\n"
+ "ld1h { z6.h }, p3/Z, [x15, #7, MUL VL]\n"
+ "addvl x15, x15, #16\n"
+ "ldp x10, x9, [x14, #0x0]\n"
+ "ld1h { z7.h }, p3/Z, [x15, #-8, MUL VL]\n"
+ "ld1h { z8.h }, p3/Z, [x15, #-7, MUL VL]\n"
+ "addvl x15, x15, #-6\n"
+ "ld1h { z9.h }, p2/Z, [x10, x13, LSL #1]\n"
+ "ld1h { z10.h }, p2/Z, [x9, x13, LSL #1]\n"
+ "ldp x28, x27, [x14, #0x10]\n"
+ "ldr x26, [x14, #0x20]\n"
+ "ld1h { z11.h }, p2/Z, [x28, x13, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x27, x13, LSL #1]\n"
+ "ld1h { z13.h }, p2/Z, [x26, x13, LSL #1]\n"
"bge 2f\n"
"1:" // Channel loop
- "fmla z31.h, p3/M, z8.h, z9.h\n"
- "ldr x9, [x8, #0x28]\n"
- "whilelt p1.h, x16, %x[n_channels]\n"
- "fmla z30.h, p3/M, z7.h, z9.h\n"
- "ldr x28, [x8, #0x30]\n"
- "inch x15\n"
- "fmla z29.h, p3/M, z6.h, z9.h\n"
- "ldr x27, [x8, #0x38]\n"
+ "movprfx z31, z16\n fmla z31.h, p3/M, z8.h, z9.h\n"
+ "ldr x25, [x14, #0x28]\n"
+ "whilelt p1.h, x12, %x[n_channels]\n"
+ "movprfx z30, z16\n fmla z30.h, p3/M, z7.h, z9.h\n"
+ "ldr x24, [x14, #0x30]\n"
+ "inch x11\n"
+ "movprfx z29, z16\n fmla z29.h, p3/M, z6.h, z9.h\n"
+ "ldr x23, [x14, #0x38]\n"
"mov p0.b, p2.b\n"
- "fmla z28.h, p3/M, z5.h, z9.h\n"
- "ldr x26, [x8, #0x40]\n"
- "fmla z27.h, p3/M, z4.h, z9.h\n"
- "ldr x22, [x8, #0x48]\n"
- "fmla z26.h, p3/M, z3.h, z9.h\n"
- "ldr x21, [x8, #0x50]\n"
- "fmla z25.h, p3/M, z2.h, z9.h\n"
- "ldr x20, [x8, #0x58]\n"
- "fmla z24.h, p3/M, z1.h, z9.h\n"
- "ldr x19, [x8, #0x60]\n"
- "fmla z23.h, p3/M, z0.h, z9.h\n"
- "ldr x25, [x8, #0x68]\n"
+ "movprfx z28, z16\n fmla z28.h, p3/M, z5.h, z9.h\n"
+ "ldr x10, [x14, #0x40]\n"
+ "movprfx z27, z16\n fmla z27.h, p3/M, z4.h, z9.h\n"
+ "ldr x9, [x14, #0x48]\n"
+ "movprfx z26, z16\n fmla z26.h, p3/M, z3.h, z9.h\n"
+ "ldr x28, [x14, #0x50]\n"
+ "movprfx z25, z16\n fmla z25.h, p3/M, z2.h, z9.h\n"
+ "ldr x27, [x14, #0x58]\n"
+ "movprfx z24, z16\n fmla z24.h, p3/M, z1.h, z9.h\n"
+ "ldr x26, [x14, #0x60]\n"
+ "movprfx z23, z16\n fmla z23.h, p3/M, z0.h, z9.h\n"
+ "ldr x22, [x16, #0x0]\n"
"fmla z31.h, p3/M, z0.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x22, x17, LSL #1]\n"
+ "ld1h { z10.h }, p2/Z, [x9, x13, LSL #1]\n"
"fmla z29.h, p3/M, z2.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x28, x17, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x24, x13, LSL #1]\n"
"fmla z25.h, p3/M, z6.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x9, x17, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x25, x13, LSL #1]\n"
"fmla z30.h, p3/M, z4.h, z13.h\n"
- "ldr x24, [x8, #0x70]\n"
+ "ldr x25, [x14, #0x68]\n"
"fmla z31.h, p3/M, z5.h, z13.h\n"
- "ldr x23, [x8, #0x78]\n"
+ "ldr x24, [x14, #0x70]\n"
"fmla z29.h, p3/M, z3.h, z13.h\n"
- "ldr x14, [x8, #0x80]\n"
+ "ldr x9, [x14, #0x88]\n"
"fmla z28.h, p3/M, z2.h, z13.h\n"
- "ldr x13, [x8, #0x88]\n"
+ "ldr x21, [x16, #0x8]\n"
"fmla z27.h, p3/M, z1.h, z13.h\n"
- "ldr x12, [x8, #0x90]\n"
+ "ldr x20, [x16, #0x10]\n"
"fmla z26.h, p3/M, z0.h, z13.h\n"
- "ld1h { z13.h }, p2/Z, [x27, x17, LSL #1]\n"
+ "ld1h { z13.h }, p2/Z, [x23, x13, LSL #1]\n"
"fmla z23.h, p3/M, z8.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x26, x17, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x10, x13, LSL #1]\n"
"fmla z31.h, p3/M, z7.h, z11.h\n"
- "ldr x11, [x8, #0x98]\n"
+ "ldr x23, [x14, #0x78]\n"
"fmla z30.h, p3/M, z6.h, z11.h\n"
- "ldr x10, [x8, #0xa0]\n"
+ "ldr x10, [x14, #0x80]\n"
"fmla z28.h, p3/M, z4.h, z11.h\n"
- "ldr x9, [x8, #0xa8]\n"
+ "ldr x19, [x16, #0x18]\n"
"fmla z27.h, p3/M, z3.h, z11.h\n"
- "ldr x28, [x8, #0xb0]\n"
+ "ld1h { z16.h }, p3/Z, [x15]\n"
"fmla z25.h, p3/M, z1.h, z11.h\n"
- "ldr x27, [x8, #0xb8]\n"
"fmla z24.h, p3/M, z0.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x21, x17, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x28, x13, LSL #1]\n"
"fmla z31.h, p3/M, z1.h, z13.h\n"
- "ldr x26, [x8, #0xc0]\n"
+ "ldr x28, [x14, #0x90]\n"
"fmla z30.h, p3/M, z0.h, z13.h\n"
- "ld1h { z13.h }, p2/Z, [x20, x17, LSL #1]\n"
+ "ld1h { z13.h }, p2/Z, [x27, x13, LSL #1]\n"
"fmla z29.h, p3/M, z1.h, z12.h\n"
- "ldr x22, [x6, #0x0]\n"
+ "ldr x27, [x14, #0x98]\n"
"fmla z27.h, p3/M, z5.h, z10.h\n"
- "ldr x21, [x6, #0x8]\n"
"fmla z26.h, p3/M, z4.h, z10.h\n"
- "ldr x20, [x6, #0x10]\n"
"fmla z30.h, p3/M, z2.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x19, x17, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x26, x13, LSL #1]\n"
"fmla z29.h, p3/M, z7.h, z10.h\n"
- "ldr x19, [x6, #0x18]\n"
+ "ldr x26, [x14, #0xa0]\n"
"fmla z24.h, p3/M, z2.h, z10.h\n"
- "ld1h { z16.h }, p3/Z, [x7]\n" // Load from weights and bias
"fmla z23.h, p3/M, z1.h, z10.h\n"
"fmla z30.h, p3/M, z8.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x25, x17, LSL #1]\n"
+ "ld1h { z10.h }, p2/Z, [x25, x13, LSL #1]\n"
"fmla z31.h, p3/M, z3.h, z11.h\n"
+ "ldr x25, [x14, #0xa8]\n"
"fmla z28.h, p3/M, z0.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x24, x17, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x24, x13, LSL #1]\n"
"fmla z29.h, p3/M, z5.h, z13.h\n"
+ "ldr x24, [x14, #0xb0]\n"
"fmla z26.h, p3/M, z2.h, z13.h\n"
- "ld1h { z13.h }, p2/Z, [x23, x17, LSL #1]\n"
+ "ld1h { z13.h }, p2/Z, [x23, x13, LSL #1]\n"
"fmla z25.h, p3/M, z3.h, z12.h\n"
+ "ldr x23, [x14, #0xb8]\n"
"fmla z28.h, p3/M, z6.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x14, x17, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x10, x13, LSL #1]\n"
"fmla z27.h, p3/M, z7.h, z10.h\n"
+ "ldr x10, [x14, #0xc0]\n"
"fmla z26.h, p3/M, z6.h, z10.h\n"
"fmla z25.h, p3/M, z5.h, z10.h\n"
"fmla z28.h, p3/M, z8.h, z10.h\n"
@@ -219,191 +210,182 @@
"fmla z26.h, p3/M, z8.h, z11.h\n"
"fmla z25.h, p3/M, z7.h, z13.h\n"
"fmla z24.h, p3/M, z6.h, z13.h\n"
- "ld1h { z13.h }, p2/Z, [x12, x17, LSL #1]\n"
+ "ld1h { z13.h }, p2/Z, [x28, x13, LSL #1]\n"
"fmla z23.h, p3/M, z5.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x13, x17, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x9, x13, LSL #1]\n"
"fmla z31.h, p3/M, z4.h, z12.h\n"
- "ldp x14, x13, [x8, #0x0]\n"
"fmla z30.h, p3/M, z3.h, z12.h\n"
- "ld1h { z9.h }, p1/Z, [x14, x16, LSL #1]\n"
"fmla z28.h, p3/M, z1.h, z12.h\n"
"fmla z27.h, p3/M, z0.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x11, x17, LSL #1]\n"
- "fmla z30.h, p3/M, z5.h, z11.h\n"
- "ld1h { z10.h }, p1/Z, [x13, x16, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x27, x13, LSL #1]\n"
"fmla z29.h, p3/M, z4.h, z11.h\n"
- "ldp x12, x11, [x8, #0x10]\n"
+ "fmla z30.h, p3/M, z5.h, z11.h\n"
"fmla z26.h, p3/M, z1.h, z11.h\n"
"fmla z27.h, p3/M, z2.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x10, x17, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x26, x13, LSL #1]\n"
"fmla z24.h, p3/M, z8.h, z13.h\n"
- "ldr x10, [x8, #0x20]\n"
+ "ldr x26, [x14, #0x20]\n"
"fmla z23.h, p3/M, z7.h, z13.h\n"
- "ld1h { z13.h }, p2/Z, [x9, x17, LSL #1]\n"
+ "ld1h { z13.h }, p2/Z, [x25, x13, LSL #1]\n"
"fmla z28.h, p3/M, z7.h, z12.h\n"
"fmla z27.h, p3/M, z6.h, z12.h\n"
"fmla z25.h, p3/M, z4.h, z12.h\n"
"fmla z24.h, p3/M, z3.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x28, x17, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x24, x13, LSL #1]\n"
"fmla z31.h, p3/M, z2.h, z11.h\n"
"fmla z30.h, p3/M, z1.h, z11.h\n"
- "ld1h { z1.h }, p3/Z, [x7, #2, MUL VL]\n" // Load from weights and bias
+ "ld1h { z1.h }, p3/Z, [x15, #2, MUL VL]\n"
"fmla z29.h, p3/M, z0.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x27, x17, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x23, x13, LSL #1]\n"
"fmla z27.h, p3/M, z8.h, z13.h\n"
"fmla z26.h, p3/M, z7.h, z13.h\n"
"fmla z24.h, p3/M, z5.h, z13.h\n"
"fmla z23.h, p3/M, z4.h, z13.h\n"
- "ld1h { z13.h }, p2/Z, [x26, x17, LSL #1]\n"
- "inch x17\n"
+ "ld1h { z13.h }, p2/Z, [x10, x13, LSL #1]\n"
+ "inch x13\n"
"fmla z31.h, p3/M, z6.h, z12.h\n"
- "ld1h { z4.h }, p3/Z, [x7, #5, MUL VL]\n" // Load from weights and bias
- "whilelt p2.h, x17, %x[n_channels]\n"
+ "ldp x10, x9, [x14, #0x0]\n"
+ "whilelt p2.h, x13, %x[n_channels]\n"
"fmla z28.h, p3/M, z3.h, z12.h\n"
- "ld1h { z3.h }, p3/Z, [x7, #4, MUL VL]\n" // Load from weights and bias
+ "ldp x28, x27, [x14, #0x10]\n"
"fmla z25.h, p3/M, z0.h, z12.h\n"
- "ld1h { z12.h }, p1/Z, [x11, x16, LSL #1]\n"
+ "ld1h { z0.h }, p3/Z, [x15, #1, MUL VL]\n"
"fmla z29.h, p3/M, z8.h, z11.h\n"
- "ld1h { z0.h }, p3/Z, [x7, #1, MUL VL]\n" // Load from weights and bias
+ "ld1h { z9.h }, p1/Z, [x10, x12, LSL #1]\n"
"fmla z26.h, p3/M, z5.h, z11.h\n"
- "ld1h { z5.h }, p3/Z, [x7, #6, MUL VL]\n" // Load from weights and bias
+ "ld1h { z10.h }, p1/Z, [x9, x12, LSL #1]\n"
"fmla z23.h, p3/M, z2.h, z11.h\n"
- "ld1h { z11.h }, p1/Z, [x12, x16, LSL #1]\n"
+ "ld1h { z11.h }, p1/Z, [x28, x12, LSL #1]\n"
"fmla z25.h, p3/M, z8.h, z13.h\n"
- "ld1h { z2.h }, p3/Z, [x7, #3, MUL VL]\n" // Load from weights and bias
+ "ld1h { z12.h }, p1/Z, [x27, x12, LSL #1]\n"
"fmla z24.h, p3/M, z7.h, z13.h\n"
+ "ld1h { z2.h }, p3/Z, [x15, #3, MUL VL]\n"
"fmax z31.h, p3/M, z31.h, z18.h\n"
+ "ld1h { z3.h }, p3/Z, [x15, #4, MUL VL]\n"
"fmla z23.h, p3/M, z6.h, z13.h\n"
- "ld1h { z13.h }, p1/Z, [x10, x16, LSL #1]\n"
- "inch x16\n"
+ "ld1h { z13.h }, p1/Z, [x26, x12, LSL #1]\n"
+ "inch x12\n"
"fmax z30.h, p3/M, z30.h, z18.h\n"
- "ld1h { z6.h }, p3/Z, [x7, #7, MUL VL]\n" // Load from weights and bias
- "addvl x7, x7, #16\n"
+ "ld1h { z4.h }, p3/Z, [x15, #5, MUL VL]\n"
+ "cmp x12, %x[n_channels]\n"
"fmin z31.h, p3/M, z31.h, z17.h\n"
- "ld1h { z7.h }, p3/Z, [x7, #-8, MUL VL]\n" // Load from weights and bias
- "cmp x16, %x[n_channels]\n"
+ "ld1h { z5.h }, p3/Z, [x15, #6, MUL VL]\n"
"fmax z29.h, p3/M, z29.h, z18.h\n"
- "ld1h { z8.h }, p3/Z, [x7, #-7, MUL VL]\n" // Load from weights and bias
- "addvl x7, x7, #-6\n"
+ "ld1h { z6.h }, p3/Z, [x15, #7, MUL VL]\n"
+ "addvl x15, x15, #16\n"
"fmax z28.h, p3/M, z28.h, z18.h\n"
- "st1h { z31.h }, p0, [x22, x15, LSL #1]\n"
- "mov z31.d, z16.d\n"
- "fmin z30.h, p3/M, z30.h, z17.h\n"
- "ldr x22, [x6, #0x20]\n"
- "fmin z29.h, p3/M, z29.h, z17.h\n"
- "st1h { z30.h }, p0, [x21, x15, LSL #1]\n"
- "mov z30.d, z16.d\n"
- "fmin z28.h, p3/M, z28.h, z17.h\n"
- "st1h { z29.h }, p0, [x20, x15, LSL #1]\n"
- "mov z29.d, z16.d\n"
- "ldr x21, [x6, #0x28]\n"
+ "ld1h { z7.h }, p3/Z, [x15, #-8, MUL VL]\n"
"fmax z27.h, p3/M, z27.h, z18.h\n"
- "ldr x20, [x6, #0x30]\n"
+ "ld1h { z8.h }, p3/Z, [x15, #-7, MUL VL]\n"
+ "addvl x15, x15, #-6\n"
+ "fmin z30.h, p3/M, z30.h, z17.h\n"
+ "st1h { z31.h }, p0, [x22, x11, LSL #1]\n"
+ "fmin z29.h, p3/M, z29.h, z17.h\n"
+ "ldr x22, [x16, #0x20]\n"
"fmax z26.h, p3/M, z26.h, z18.h\n"
- "st1h { z28.h }, p0, [x19, x15, LSL #1]\n"
- "mov z28.d, z16.d\n"
- "ldr x19, [x6, #0x38]\n"
- "fmax z25.h, p3/M, z25.h, z18.h\n"
+ "st1h { z30.h }, p0, [x21, x11, LSL #1]\n"
+ "fmin z28.h, p3/M, z28.h, z17.h\n"
"fmin z27.h, p3/M, z27.h, z17.h\n"
- "st1h { z27.h }, p0, [x22, x15, LSL #1]\n"
- "mov z27.d, z16.d\n"
+ "st1h { z29.h }, p0, [x20, x11, LSL #1]\n"
"fmin z26.h, p3/M, z26.h, z17.h\n"
- "ldr x22, [x6, #0x40]\n"
- "fmin z25.h, p3/M, z25.h, z17.h\n"
- "st1h { z26.h }, p0, [x21, x15, LSL #1]\n"
- "mov z26.d, z16.d\n"
+ "ldr x21, [x16, #0x28]\n"
+ "fmax z25.h, p3/M, z25.h, z18.h\n"
+ "ldr x20, [x16, #0x30]\n"
"fmax z24.h, p3/M, z24.h, z18.h\n"
- "st1h { z25.h }, p0, [x20, x15, LSL #1]\n"
- "mov z25.d, z16.d\n"
+ "st1h { z28.h }, p0, [x19, x11, LSL #1]\n"
"fmax z23.h, p3/M, z23.h, z18.h\n"
+ "st1h { z27.h }, p0, [x22, x11, LSL #1]\n"
+ "st1h { z26.h }, p0, [x21, x11, LSL #1]\n"
+ "fmin z25.h, p3/M, z25.h, z17.h\n"
+ "ldr x19, [x16, #0x38]\n"
"fmin z24.h, p3/M, z24.h, z17.h\n"
- "st1h { z24.h }, p0, [x19, x15, LSL #1]\n"
- "mov z24.d, z16.d\n"
+ "ldr x22, [x16, #0x40]\n"
"fmin z23.h, p3/M, z23.h, z17.h\n"
- "st1h { z23.h }, p0, [x22, x15, LSL #1]\n"
- "mov z23.d, z16.d\n"
+ "st1h { z25.h }, p0, [x20, x11, LSL #1]\n"
+ "st1h { z24.h }, p0, [x19, x11, LSL #1]\n"
+ "st1h { z23.h }, p0, [x22, x11, LSL #1]\n"
"blt 1b\n"
"2:" // Channel tail
- "fmla z31.h, p3/M, z8.h, z9.h\n"
- "ldr x9, [x8, #0x28]\n"
- "inch x15\n"
- "fmla z30.h, p3/M, z7.h, z9.h\n"
- "ldr x28, [x8, #0x30]\n"
+ "movprfx z31, z16\n fmla z31.h, p3/M, z8.h, z9.h\n"
+ "ldr x25, [x14, #0x28]\n"
+ "inch x11\n"
+ "movprfx z30, z16\n fmla z30.h, p3/M, z7.h, z9.h\n"
+ "ldr x24, [x14, #0x30]\n"
"mov p0.b, p2.b\n"
- "fmla z29.h, p3/M, z6.h, z9.h\n"
- "ldr x27, [x8, #0x38]\n"
- "fmla z28.h, p3/M, z5.h, z9.h\n"
- "ldr x26, [x8, #0x40]\n"
- "fmla z27.h, p3/M, z4.h, z9.h\n"
- "ldr x22, [x8, #0x48]\n"
- "fmla z26.h, p3/M, z3.h, z9.h\n"
- "ldr x21, [x8, #0x50]\n"
- "fmla z25.h, p3/M, z2.h, z9.h\n"
- "ldr x20, [x8, #0x58]\n"
- "fmla z24.h, p3/M, z1.h, z9.h\n"
- "ldr x19, [x8, #0x60]\n"
- "fmla z23.h, p3/M, z0.h, z9.h\n"
- "ldr x25, [x8, #0x68]\n"
+ "movprfx z29, z16\n fmla z29.h, p3/M, z6.h, z9.h\n"
+ "ldr x23, [x14, #0x38]\n"
+ "movprfx z28, z16\n fmla z28.h, p3/M, z5.h, z9.h\n"
+ "ldr x10, [x14, #0x40]\n"
+ "movprfx z27, z16\n fmla z27.h, p3/M, z4.h, z9.h\n"
+ "ldr x9, [x14, #0x48]\n"
+ "movprfx z26, z16\n fmla z26.h, p3/M, z3.h, z9.h\n"
+ "ldr x28, [x14, #0x50]\n"
+ "movprfx z25, z16\n fmla z25.h, p3/M, z2.h, z9.h\n"
+ "ldr x27, [x14, #0x58]\n"
+ "movprfx z24, z16\n fmla z24.h, p3/M, z1.h, z9.h\n"
+ "ldr x26, [x14, #0x60]\n"
+ "movprfx z23, z16\n fmla z23.h, p3/M, z0.h, z9.h\n"
+ "ldr x22, [x16, #0x0]\n"
"fmla z31.h, p3/M, z0.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x22, x17, LSL #1]\n"
+ "ld1h { z10.h }, p2/Z, [x9, x13, LSL #1]\n"
"fmla z29.h, p3/M, z2.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x28, x17, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x24, x13, LSL #1]\n"
"fmla z25.h, p3/M, z6.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x9, x17, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x25, x13, LSL #1]\n"
"fmla z30.h, p3/M, z4.h, z13.h\n"
- "ldr x24, [x8, #0x70]\n"
+ "ldr x25, [x14, #0x68]\n"
"fmla z31.h, p3/M, z5.h, z13.h\n"
- "ldr x23, [x8, #0x78]\n"
+ "ldr x24, [x14, #0x70]\n"
"fmla z29.h, p3/M, z3.h, z13.h\n"
- "ldr x14, [x8, #0x80]\n"
+ "ldr x9, [x14, #0x88]\n"
"fmla z28.h, p3/M, z2.h, z13.h\n"
- "ldr x13, [x8, #0x88]\n"
+ "ldr x21, [x16, #0x8]\n"
"fmla z27.h, p3/M, z1.h, z13.h\n"
- "ldr x12, [x8, #0x90]\n"
+ "ldr x20, [x16, #0x10]\n"
"fmla z26.h, p3/M, z0.h, z13.h\n"
- "ld1h { z13.h }, p2/Z, [x27, x17, LSL #1]\n"
+ "ld1h { z13.h }, p2/Z, [x23, x13, LSL #1]\n"
"fmla z23.h, p3/M, z8.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x26, x17, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x10, x13, LSL #1]\n"
"fmla z31.h, p3/M, z7.h, z11.h\n"
- "ldr x11, [x8, #0x98]\n"
+ "ldr x23, [x14, #0x78]\n"
"fmla z30.h, p3/M, z6.h, z11.h\n"
- "ldr x10, [x8, #0xa0]\n"
+ "ldr x10, [x14, #0x80]\n"
"fmla z28.h, p3/M, z4.h, z11.h\n"
- "ldr x9, [x8, #0xa8]\n"
+ "ldr x19, [x16, #0x18]\n"
"fmla z27.h, p3/M, z3.h, z11.h\n"
- "ldr x28, [x8, #0xb0]\n"
"fmla z25.h, p3/M, z1.h, z11.h\n"
- "ldr x27, [x8, #0xb8]\n"
"fmla z24.h, p3/M, z0.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x21, x17, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x28, x13, LSL #1]\n"
"fmla z31.h, p3/M, z1.h, z13.h\n"
- "ldr x26, [x8, #0xc0]\n"
+ "ldr x28, [x14, #0x90]\n"
"fmla z30.h, p3/M, z0.h, z13.h\n"
- "ld1h { z13.h }, p2/Z, [x20, x17, LSL #1]\n"
+ "ld1h { z13.h }, p2/Z, [x27, x13, LSL #1]\n"
"fmla z29.h, p3/M, z1.h, z12.h\n"
- "ldr x22, [x6, #0x0]\n"
+ "ldr x27, [x14, #0x98]\n"
"fmla z27.h, p3/M, z5.h, z10.h\n"
- "ldr x21, [x6, #0x8]\n"
"fmla z26.h, p3/M, z4.h, z10.h\n"
- "ldr x20, [x6, #0x10]\n"
"fmla z30.h, p3/M, z2.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x19, x17, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x26, x13, LSL #1]\n"
"fmla z29.h, p3/M, z7.h, z10.h\n"
- "ldr x19, [x6, #0x18]\n"
+ "ldr x26, [x14, #0xa0]\n"
"fmla z24.h, p3/M, z2.h, z10.h\n"
"fmla z23.h, p3/M, z1.h, z10.h\n"
"fmla z30.h, p3/M, z8.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x25, x17, LSL #1]\n"
+ "ld1h { z10.h }, p2/Z, [x25, x13, LSL #1]\n"
"fmla z31.h, p3/M, z3.h, z11.h\n"
+ "ldr x25, [x14, #0xa8]\n"
"fmla z28.h, p3/M, z0.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x24, x17, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x24, x13, LSL #1]\n"
"fmla z29.h, p3/M, z5.h, z13.h\n"
+ "ldr x24, [x14, #0xb0]\n"
"fmla z26.h, p3/M, z2.h, z13.h\n"
- "ld1h { z13.h }, p2/Z, [x23, x17, LSL #1]\n"
+ "ld1h { z13.h }, p2/Z, [x23, x13, LSL #1]\n"
"fmla z25.h, p3/M, z3.h, z12.h\n"
+ "ldr x23, [x14, #0xb8]\n"
"fmla z28.h, p3/M, z6.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x14, x17, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x10, x13, LSL #1]\n"
"fmla z27.h, p3/M, z7.h, z10.h\n"
+ "ldr x10, [x14, #0xc0]\n"
"fmla z26.h, p3/M, z6.h, z10.h\n"
"fmla z25.h, p3/M, z5.h, z10.h\n"
"fmla z28.h, p3/M, z8.h, z10.h\n"
@@ -412,36 +394,36 @@
"fmla z26.h, p3/M, z8.h, z11.h\n"
"fmla z25.h, p3/M, z7.h, z13.h\n"
"fmla z24.h, p3/M, z6.h, z13.h\n"
- "ld1h { z13.h }, p2/Z, [x12, x17, LSL #1]\n"
+ "ld1h { z13.h }, p2/Z, [x28, x13, LSL #1]\n"
"fmla z23.h, p3/M, z5.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x13, x17, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x9, x13, LSL #1]\n"
"fmla z31.h, p3/M, z4.h, z12.h\n"
"fmla z30.h, p3/M, z3.h, z12.h\n"
"fmla z28.h, p3/M, z1.h, z12.h\n"
"fmla z27.h, p3/M, z0.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x11, x17, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x27, x13, LSL #1]\n"
"fmla z29.h, p3/M, z4.h, z11.h\n"
"fmla z30.h, p3/M, z5.h, z11.h\n"
"fmla z26.h, p3/M, z1.h, z11.h\n"
"fmla z27.h, p3/M, z2.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x10, x17, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x26, x13, LSL #1]\n"
"fmla z24.h, p3/M, z8.h, z13.h\n"
"fmla z23.h, p3/M, z7.h, z13.h\n"
- "ld1h { z13.h }, p2/Z, [x9, x17, LSL #1]\n"
+ "ld1h { z13.h }, p2/Z, [x25, x13, LSL #1]\n"
"fmla z28.h, p3/M, z7.h, z12.h\n"
"fmla z27.h, p3/M, z6.h, z12.h\n"
"fmla z25.h, p3/M, z4.h, z12.h\n"
"fmla z24.h, p3/M, z3.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x28, x17, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x24, x13, LSL #1]\n"
"fmla z31.h, p3/M, z2.h, z11.h\n"
"fmla z30.h, p3/M, z1.h, z11.h\n"
"fmla z29.h, p3/M, z0.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x27, x17, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x23, x13, LSL #1]\n"
"fmla z27.h, p3/M, z8.h, z13.h\n"
"fmla z26.h, p3/M, z7.h, z13.h\n"
"fmla z24.h, p3/M, z5.h, z13.h\n"
"fmla z23.h, p3/M, z4.h, z13.h\n"
- "ld1h { z13.h }, p2/Z, [x26, x17, LSL #1]\n"
+ "ld1h { z13.h }, p2/Z, [x10, x13, LSL #1]\n"
"fmla z31.h, p3/M, z6.h, z12.h\n"
"fmla z28.h, p3/M, z3.h, z12.h\n"
"fmla z25.h, p3/M, z0.h, z12.h\n"
@@ -455,41 +437,41 @@
"fmax z30.h, p3/M, z30.h, z18.h\n"
"fmax z29.h, p3/M, z29.h, z18.h\n"
"fmin z31.h, p3/M, z31.h, z17.h\n"
- "st1h { z31.h }, p0, [x22, x15, LSL #1]\n"
+ "st1h { z31.h }, p0, [x22, x11, LSL #1]\n"
"fmin z30.h, p3/M, z30.h, z17.h\n"
"fmin z29.h, p3/M, z29.h, z17.h\n"
- "ldr x22, [x6, #0x20]\n"
+ "ldr x22, [x16, #0x20]\n"
"fmax z28.h, p3/M, z28.h, z18.h\n"
- "st1h { z30.h }, p0, [x21, x15, LSL #1]\n"
+ "st1h { z30.h }, p0, [x21, x11, LSL #1]\n"
"fmax z27.h, p3/M, z27.h, z18.h\n"
"fmax z26.h, p3/M, z26.h, z18.h\n"
- "st1h { z29.h }, p0, [x20, x15, LSL #1]\n"
+ "st1h { z29.h }, p0, [x20, x11, LSL #1]\n"
"fmin z28.h, p3/M, z28.h, z17.h\n"
- "ldr x21, [x6, #0x28]\n"
+ "ldr x21, [x16, #0x28]\n"
"fmax z25.h, p3/M, z25.h, z18.h\n"
- "ldr x20, [x6, #0x30]\n"
+ "ldr x20, [x16, #0x30]\n"
"fmax z24.h, p3/M, z24.h, z18.h\n"
- "st1h { z28.h }, p0, [x19, x15, LSL #1]\n"
+ "st1h { z28.h }, p0, [x19, x11, LSL #1]\n"
"fmin z27.h, p3/M, z27.h, z17.h\n"
"fmin z26.h, p3/M, z26.h, z17.h\n"
- "ldr x19, [x6, #0x38]\n"
+ "ldr x19, [x16, #0x38]\n"
"fmin z25.h, p3/M, z25.h, z17.h\n"
- "st1h { z27.h }, p0, [x22, x15, LSL #1]\n"
+ "st1h { z27.h }, p0, [x22, x11, LSL #1]\n"
"fmin z24.h, p3/M, z24.h, z17.h\n"
"fmax z23.h, p3/M, z23.h, z18.h\n"
- "st1h { z26.h }, p0, [x21, x15, LSL #1]\n"
- "st1h { z25.h }, p0, [x20, x15, LSL #1]\n"
+ "st1h { z26.h }, p0, [x21, x11, LSL #1]\n"
+ "st1h { z25.h }, p0, [x20, x11, LSL #1]\n"
"fmin z23.h, p3/M, z23.h, z17.h\n"
- "st1h { z24.h }, p0, [x19, x15, LSL #1]\n"
- "ldr x22, [x6, #0x40]\n"
- "st1h { z23.h }, p0, [x22, x15, LSL #1]\n"
+ "st1h { z24.h }, p0, [x19, x11, LSL #1]\n"
+ "ldr x22, [x16, #0x40]\n"
+ "st1h { z23.h }, p0, [x22, x11, LSL #1]\n"
:
: [n_channels] "r" ((unsigned long) n_channels), [offsetof_Args_inptrs] "I" (offsetof(Args, inptrs)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_outptrs] "I" (offsetof(Args, outptrs)), [offsetof_args_params] "I" (offsetof(Args, params)), [params_struct] "r" (¶ms_struct)
- : "cc", "memory", "p0", "p1", "p2", "p3", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z16", "z17", "z18", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+ : "cc", "memory", "p0", "p1", "p2", "p3", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z16", "z17", "z18", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
);
}
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
+#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp
index 7d035f0..60234c8 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp
@@ -28,7 +28,7 @@
#pragma once
-#if defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
+#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
namespace arm_conv {
namespace depthwise {
@@ -36,15 +36,17 @@
void sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16);
void sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16);
-struct sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst
+class sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst : public IDepthwiseDepthfirstStrategy
{
- typedef __fp16 bias_type;
- typedef __fp16 input_type;
- typedef __fp16 weight_type;
- typedef __fp16 return_type;
-
+ private:
typedef void (*indirect_kern_type)(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16);
+ indirect_kern_type m_indirect_kernel = sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl;
+
typedef void (*direct_kern_type)(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16);
+ direct_kern_type m_direct_kernel = sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl;
+
+ public:
+ typedef __fp16 return_type;
constexpr static arm_gemm::VLType vl_type = arm_gemm::VLType::SVE;
@@ -60,13 +62,60 @@
constexpr static unsigned int input_rows = 6;
constexpr static unsigned int input_cols = 6;
- indirect_kern_type indirect_kernel = sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl;
- direct_kern_type direct_kernel = sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl;
-
sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst(const CPUInfo *) {}
+
+ arm_gemm::VLType get_vl_type(void) const override { return vl_type; }
+
+ unsigned int get_kernel_rows(void) const override { return kernel_rows; }
+ unsigned int get_kernel_cols(void) const override { return kernel_cols; }
+
+ unsigned int get_stride_rows(void) const override { return stride_rows; }
+ unsigned int get_stride_cols(void) const override { return stride_cols; }
+
+ unsigned int get_output_rows(void) const override { return output_rows; }
+ unsigned int get_output_cols(void) const override { return output_cols; }
+
+ unsigned int get_input_rows(void) const override { return input_rows; }
+ unsigned int get_input_cols(void) const override { return input_cols; }
+
+ void indirect_kernel(
+ const void *const *const input_ptrs,
+ void *const *const outptrs,
+ const void *params,
+ unsigned int n_channels,
+ const void *activation_min,
+ const void *activation_max
+ ) const override
+ {
+ m_indirect_kernel(
+ reinterpret_cast<const __fp16 *const *>(input_ptrs),
+ reinterpret_cast<__fp16 *const *>(outptrs),
+ params, n_channels,
+ *static_cast<const __fp16 *>(activation_min),
+ *static_cast<const __fp16 *>(activation_max)
+ );
+ }
+
+ void direct_kernel(
+ const unsigned int n_tile_rows, const unsigned int n_tile_cols,
+ const void *inptr, int64_t ld_input_row, int64_t ld_input_col,
+ void *outptr, int64_t ld_output_row, int64_t ld_output_col,
+ const void *params, unsigned int n_channels,
+ const void *activation_min, const void *activation_max
+ ) const override
+ {
+ m_direct_kernel(
+ n_tile_rows, n_tile_cols,
+ static_cast<const __fp16 *>(inptr), ld_input_row, ld_input_col,
+ static_cast<__fp16 *>(outptr), ld_output_row, ld_output_col,
+ params, n_channels,
+ *static_cast<const __fp16 *>(activation_min),
+ *static_cast<const __fp16 *>(activation_max)
+ );
+ }
};
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
+#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp
index 93e1908..3cfac06 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp
@@ -25,7 +25,7 @@
#include <cstddef>
#include <cstdint>
-#if defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
+#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
namespace arm_conv {
namespace depthwise {
@@ -113,92 +113,75 @@
"add x15, x8, x22, LSL #1\n"
"ld1rh { z14.h }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n"
"add x14, x15, x22, LSL #1\n"
- "ld1h { z13.h }, p3/Z, [x4]\n" // Load from weights and bias
- "mov z31.d, z13.d\n"
- "ld1h { z0.h }, p3/Z, [x4, #1, MUL VL]\n" // Load from weights and bias
+ "ld1h { z13.h }, p3/Z, [x4]\n"
"add x13, x14, x22, LSL #1\n"
- "mov z30.d, z13.d\n"
- "ld1h { z1.h }, p3/Z, [x4, #2, MUL VL]\n" // Load from weights and bias
+ "ld1h { z0.h }, p3/Z, [x4, #1, MUL VL]\n"
"add x12, x13, x22, LSL #1\n"
- "mov z29.d, z13.d\n"
- "ld1h { z2.h }, p3/Z, [x4, #3, MUL VL]\n" // Load from weights and bias
+ "ld1h { z1.h }, p3/Z, [x4, #2, MUL VL]\n"
"add x11, x12, x22, LSL #1\n"
- "mov z28.d, z13.d\n"
- "ld1h { z3.h }, p3/Z, [x4, #4, MUL VL]\n" // Load from weights and bias
+ "ld1h { z2.h }, p3/Z, [x4, #3, MUL VL]\n"
"add x10, x7, x7\n"
- "mov z27.d, z13.d\n"
- "ld1h { z4.h }, p3/Z, [x4, #5, MUL VL]\n" // Load from weights and bias
+ "ld1h { z3.h }, p3/Z, [x4, #4, MUL VL]\n"
"add x9, x10, x7\n"
- "mov z26.d, z13.d\n"
- "ld1h { z5.h }, p3/Z, [x4, #6, MUL VL]\n" // Load from weights and bias
+ "ld1h { z4.h }, p3/Z, [x4, #5, MUL VL]\n"
"add x28, x9, x7\n"
- "mov z25.d, z13.d\n"
- "ld1h { z6.h }, p3/Z, [x4, #7, MUL VL]\n" // Load from weights and bias
+ "ld1h { z5.h }, p3/Z, [x4, #6, MUL VL]\n"
"add x27, x28, x7\n"
- "mov z24.d, z13.d\n"
+ "ld1h { z6.h }, p3/Z, [x4, #7, MUL VL]\n"
"mul x19, x2, x20\n" // offset = tile_i * ld_output_row
- "mov z23.d, z13.d\n"
+ "add x26, x17, x17\n"
"madd x19, x3, x17, x19\n" // offset += tile_j * ld_output_col
- "mov z22.d, z13.d\n"
"mul x19, x19, x23\n" // offset *= output_tile_size
- "mov z21.d, z13.d\n"
"add x16, x16, x19, LSL #1\n" // outptrs[0] += offset * sizeof(__fp16)
- "mov z20.d, z13.d\n"
- "add x26, x16, x20, LSL #1\n"
- "mov z19.d, z13.d\n"
- "add x25, x26, x20, LSL #1\n"
- "mov z18.d, z13.d\n"
+ "add x25, x16, x20, LSL #1\n"
"add x24, x25, x20, LSL #1\n"
- "mov z17.d, z13.d\n"
- "add x23, x17, x17\n"
- "mov z16.d, z13.d\n"
- "add x22, x23, x17\n"
+ "add x23, x24, x20, LSL #1\n"
+ "add x22, x26, x17\n"
"whilelt p2.h, XZR, %x[n_channels]\n"
- "ld1h { z9.h }, p2/Z, [x14, x10, LSL #1]\n" // Load input point (2, 2)
- "ld1h { z10.h }, p2/Z, [x8]\n" // Load input point (0, 0)
+ "ld1h { z9.h }, p2/Z, [x14, x10, LSL #1]\n"
+ "ld1h { z10.h }, p2/Z, [x8]\n"
"addvl x4, x4, #16\n"
- "ld1h { z11.h }, p2/Z, [x8, x27, LSL #1]\n" // Load input point (0, 5)
+ "ld1h { z11.h }, p2/Z, [x8, x27, LSL #1]\n"
"cmp x6, %x[n_channels]\n"
- "ld1h { z7.h }, p3/Z, [x4, #-8, MUL VL]\n" // Load from weights and bias
- "ld1h { z8.h }, p3/Z, [x4, #-7, MUL VL]\n" // Load from weights and bias
+ "ld1h { z7.h }, p3/Z, [x4, #-8, MUL VL]\n"
+ "ld1h { z8.h }, p3/Z, [x4, #-7, MUL VL]\n"
"addvl x4, x4, #-6\n"
- "ld1h { z12.h }, p2/Z, [x14, x9, LSL #1]\n" // Load input point (2, 3)
+ "ld1h { z12.h }, p2/Z, [x14, x9, LSL #1]\n"
"bge 3f\n"
"2:" // Tile loop: Channel loop
- "fmla z31.h, p3/M, z8.h, z9.h\n"
- "ld1h { z13.h }, p3/Z, [x4]\n" // Load from weights and bias
+ "movprfx z31, z13\n fmla z31.h, p3/M, z8.h, z9.h\n"
"whilelt p1.h, x6, %x[n_channels]\n"
- "fmla z30.h, p3/M, z7.h, z9.h\n"
+ "movprfx z30, z13\n fmla z30.h, p3/M, z7.h, z9.h\n"
"inch x21\n"
- "fmla z29.h, p3/M, z6.h, z9.h\n"
+ "movprfx z29, z13\n fmla z29.h, p3/M, z6.h, z9.h\n"
"mov p0.b, p2.b\n"
- "fmla z27.h, p3/M, z5.h, z9.h\n"
+ "movprfx z27, z13\n fmla z27.h, p3/M, z5.h, z9.h\n"
"inch x5\n"
- "fmla z26.h, p3/M, z4.h, z9.h\n"
+ "movprfx z26, z13\n fmla z26.h, p3/M, z4.h, z9.h\n"
"inch x6\n"
- "fmla z25.h, p3/M, z3.h, z9.h\n"
- "fmla z23.h, p3/M, z2.h, z9.h\n"
- "fmla z22.h, p3/M, z1.h, z9.h\n"
- "fmla z21.h, p3/M, z0.h, z9.h\n"
- "ld1h { z9.h }, p2/Z, [x13, x10, LSL #1]\n" // Load input point (3, 2)
+ "movprfx z25, z13\n fmla z25.h, p3/M, z3.h, z9.h\n"
+ "movprfx z23, z13\n fmla z23.h, p3/M, z2.h, z9.h\n"
+ "movprfx z22, z13\n fmla z22.h, p3/M, z1.h, z9.h\n"
+ "movprfx z21, z13\n fmla z21.h, p3/M, z0.h, z9.h\n"
+ "ld1h { z9.h }, p2/Z, [x13, x10, LSL #1]\n"
"fmla z31.h, p3/M, z0.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x11]\n" // Load input point (5, 0)
- "fmla z28.h, p3/M, z2.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x11, x27, LSL #1]\n" // Load input point (5, 5)
+ "ld1h { z10.h }, p2/Z, [x11]\n"
+ "movprfx z28, z13\n fmla z28.h, p3/M, z2.h, z11.h\n"
+ "ld1h { z11.h }, p2/Z, [x11, x27, LSL #1]\n"
"fmla z30.h, p3/M, z8.h, z12.h\n"
"fmla z29.h, p3/M, z7.h, z12.h\n"
"fmla z26.h, p3/M, z5.h, z12.h\n"
"fmla z28.h, p3/M, z6.h, z12.h\n"
"fmla z25.h, p3/M, z4.h, z12.h\n"
- "fmla z24.h, p3/M, z3.h, z12.h\n"
+ "movprfx z24, z13\n fmla z24.h, p3/M, z3.h, z12.h\n"
"fmla z22.h, p3/M, z2.h, z12.h\n"
"fmla z21.h, p3/M, z1.h, z12.h\n"
- "fmla z20.h, p3/M, z0.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x8, x7, LSL #1]\n" // Load input point (0, 1)
- "fmla z19.h, p3/M, z6.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x13, x9, LSL #1]\n" // Load input point (3, 3)
- "fmla z16.h, p3/M, z8.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x8, x28, LSL #1]\n" // Load input point (0, 4)
+ "movprfx z20, z13\n fmla z20.h, p3/M, z0.h, z12.h\n"
+ "ld1h { z12.h }, p2/Z, [x8, x7, LSL #1]\n"
+ "movprfx z19, z13\n fmla z19.h, p3/M, z6.h, z10.h\n"
+ "ld1h { z10.h }, p2/Z, [x13, x9, LSL #1]\n"
+ "movprfx z16, z13\n fmla z16.h, p3/M, z8.h, z11.h\n"
+ "ld1h { z11.h }, p2/Z, [x8, x28, LSL #1]\n"
"fmla z27.h, p3/M, z8.h, z9.h\n"
"fmla z26.h, p3/M, z7.h, z9.h\n"
"fmla z25.h, p3/M, z6.h, z9.h\n"
@@ -206,15 +189,16 @@
"fmla z22.h, p3/M, z4.h, z9.h\n"
"fmla z21.h, p3/M, z3.h, z9.h\n"
"fmla z19.h, p3/M, z2.h, z9.h\n"
- "fmla z18.h, p3/M, z1.h, z9.h\n"
- "fmla z17.h, p3/M, z0.h, z9.h\n"
- "ld1h { z9.h }, p2/Z, [x15]\n" // Load input point (1, 0)
+ "movprfx z18, z13\n fmla z18.h, p3/M, z1.h, z9.h\n"
+ "movprfx z17, z13\n fmla z17.h, p3/M, z0.h, z9.h\n"
+ "ld1h { z9.h }, p2/Z, [x15]\n"
"fmla z31.h, p3/M, z1.h, z12.h\n"
+ "ld1h { z13.h }, p3/Z, [x4]\n"
"fmla z30.h, p3/M, z0.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x15, x27, LSL #1]\n" // Load input point (1, 5)
+ "ld1h { z12.h }, p2/Z, [x15, x27, LSL #1]\n"
"fmla z29.h, p3/M, z2.h, z11.h\n"
"fmla z28.h, p3/M, z1.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x12]\n" // Load input point (4, 0)
+ "ld1h { z11.h }, p2/Z, [x12]\n"
"fmla z26.h, p3/M, z8.h, z10.h\n"
"fmla z25.h, p3/M, z7.h, z10.h\n"
"fmla z24.h, p3/M, z6.h, z10.h\n"
@@ -224,261 +208,245 @@
"fmla z18.h, p3/M, z2.h, z10.h\n"
"fmla z17.h, p3/M, z1.h, z10.h\n"
"fmla z16.h, p3/M, z0.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x15, x10, LSL #1]\n" // Load input point (1, 2)
+ "ld1h { z10.h }, p2/Z, [x15, x10, LSL #1]\n"
"fmla z31.h, p3/M, z3.h, z9.h\n"
"fmla z27.h, p3/M, z0.h, z9.h\n"
"fmla z28.h, p3/M, z5.h, z12.h\n"
"fmla z24.h, p3/M, z2.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x15, x9, LSL #1]\n" // Load input point (1, 3)
+ "ld1h { z12.h }, p2/Z, [x15, x9, LSL #1]\n"
"fmla z23.h, p3/M, z6.h, z11.h\n"
"fmla z19.h, p3/M, z3.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x12, x27, LSL #1]\n" // Load input point (4, 5)
+ "ld1h { z11.h }, p2/Z, [x12, x27, LSL #1]\n"
"fmla z31.h, p3/M, z5.h, z10.h\n"
"fmla z30.h, p3/M, z4.h, z10.h\n"
"fmla z29.h, p3/M, z3.h, z10.h\n"
"fmla z27.h, p3/M, z2.h, z10.h\n"
"fmla z26.h, p3/M, z1.h, z10.h\n"
"fmla z25.h, p3/M, z0.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x14, x7, LSL #1]\n" // Load input point (2, 1)
+ "ld1h { z10.h }, p2/Z, [x14, x7, LSL #1]\n"
"fmla z20.h, p3/M, z8.h, z11.h\n"
"fmla z16.h, p3/M, z5.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x11, x7, LSL #1]\n" // Load input point (5, 1)
+ "ld1h { z11.h }, p2/Z, [x11, x7, LSL #1]\n"
"fmla z30.h, p3/M, z5.h, z12.h\n"
"fmla z29.h, p3/M, z4.h, z12.h\n"
"fmla z28.h, p3/M, z3.h, z12.h\n"
"fmla z26.h, p3/M, z2.h, z12.h\n"
"fmla z25.h, p3/M, z1.h, z12.h\n"
"fmla z24.h, p3/M, z0.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x14, x28, LSL #1]\n" // Load input point (2, 4)
+ "ld1h { z12.h }, p2/Z, [x14, x28, LSL #1]\n"
"fmla z19.h, p3/M, z7.h, z11.h\n"
"fmla z18.h, p3/M, z6.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x11, x28, LSL #1]\n" // Load input point (5, 4)
+ "ld1h { z11.h }, p2/Z, [x11, x28, LSL #1]\n"
"fmla z31.h, p3/M, z7.h, z10.h\n"
"fmla z30.h, p3/M, z6.h, z10.h\n"
"fmla z27.h, p3/M, z4.h, z10.h\n"
"fmla z26.h, p3/M, z3.h, z10.h\n"
"fmla z23.h, p3/M, z1.h, z10.h\n"
"fmla z22.h, p3/M, z0.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x8, x10, LSL #1]\n" // Load input point (0, 2)
+ "ld1h { z10.h }, p2/Z, [x8, x10, LSL #1]\n"
"fmla z17.h, p3/M, z8.h, z11.h\n"
"fmla z16.h, p3/M, z7.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x13, x7, LSL #1]\n" // Load input point (3, 1)
+ "ld1h { z11.h }, p2/Z, [x13, x7, LSL #1]\n"
"fmla z29.h, p3/M, z8.h, z12.h\n"
"fmla z28.h, p3/M, z7.h, z12.h\n"
"fmla z25.h, p3/M, z5.h, z12.h\n"
"fmla z24.h, p3/M, z4.h, z12.h\n"
"fmla z21.h, p3/M, z2.h, z12.h\n"
"fmla z20.h, p3/M, z1.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x8, x9, LSL #1]\n" // Load input point (0, 3)
+ "ld1h { z12.h }, p2/Z, [x8, x9, LSL #1]\n"
"addvl x8, x8, #1\n"
"fmla z31.h, p3/M, z2.h, z10.h\n"
"fmla z30.h, p3/M, z1.h, z10.h\n"
"fmla z29.h, p3/M, z0.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x14]\n" // Load input point (2, 0)
+ "ld1h { z10.h }, p2/Z, [x14]\n"
"fmla z27.h, p3/M, z7.h, z11.h\n"
"fmla z26.h, p3/M, z6.h, z11.h\n"
"fmla z23.h, p3/M, z4.h, z11.h\n"
"fmla z22.h, p3/M, z3.h, z11.h\n"
"fmla z19.h, p3/M, z1.h, z11.h\n"
"fmla z18.h, p3/M, z0.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x13, x28, LSL #1]\n" // Load input point (3, 4)
+ "ld1h { z11.h }, p2/Z, [x13, x28, LSL #1]\n"
"fmla z30.h, p3/M, z2.h, z12.h\n"
"fmla z29.h, p3/M, z1.h, z12.h\n"
"fmla z28.h, p3/M, z0.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x14, x27, LSL #1]\n" // Load input point (2, 5)
+ "ld1h { z12.h }, p2/Z, [x14, x27, LSL #1]\n"
"addvl x14, x14, #1\n"
"fmla z31.h, p3/M, z6.h, z10.h\n"
- "ld1h { z9.h }, p1/Z, [x14, x10, LSL #1]\n" // Load input point (2, 2)
+ "ld1h { z9.h }, p1/Z, [x14, x10, LSL #1]\n"
"fmla z27.h, p3/M, z3.h, z10.h\n"
"fmla z23.h, p3/M, z0.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x13]\n" // Load input point (3, 0)
+ "ld1h { z10.h }, p2/Z, [x13]\n"
"fmla z25.h, p3/M, z8.h, z11.h\n"
"fmla z24.h, p3/M, z7.h, z11.h\n"
"fmla z21.h, p3/M, z5.h, z11.h\n"
"fmla z20.h, p3/M, z4.h, z11.h\n"
"fmla z17.h, p3/M, z2.h, z11.h\n"
"fmla z16.h, p3/M, z1.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x12, x10, LSL #1]\n" // Load input point (4, 2)
+ "ld1h { z11.h }, p2/Z, [x12, x10, LSL #1]\n"
"fmla z28.h, p3/M, z8.h, z12.h\n"
"fmla z24.h, p3/M, z5.h, z12.h\n"
"fmla z20.h, p3/M, z2.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x13, x27, LSL #1]\n" // Load input point (3, 5)
+ "ld1h { z12.h }, p2/Z, [x13, x27, LSL #1]\n"
"addvl x13, x13, #1\n"
"fmla z27.h, p3/M, z6.h, z10.h\n"
"fmla z23.h, p3/M, z3.h, z10.h\n"
"fmla z19.h, p3/M, z0.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x11, x10, LSL #1]\n" // Load input point (5, 2)
+ "ld1h { z10.h }, p2/Z, [x11, x10, LSL #1]\n"
"fmla z22.h, p3/M, z7.h, z11.h\n"
"fmla z21.h, p3/M, z6.h, z11.h\n"
"fmla z23.h, p3/M, z8.h, z11.h\n"
"fmla z19.h, p3/M, z5.h, z11.h\n"
"fmla z18.h, p3/M, z4.h, z11.h\n"
"fmla z17.h, p3/M, z3.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x12, x9, LSL #1]\n" // Load input point (4, 3)
+ "ld1h { z11.h }, p2/Z, [x12, x9, LSL #1]\n"
"fmla z24.h, p3/M, z8.h, z12.h\n"
"fmla z20.h, p3/M, z5.h, z12.h\n"
"fmla z16.h, p3/M, z2.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x11, x9, LSL #1]\n" // Load input point (5, 3)
+ "ld1h { z12.h }, p2/Z, [x11, x9, LSL #1]\n"
"addvl x11, x11, #1\n"
"fmla z19.h, p3/M, z8.h, z10.h\n"
"fmla z18.h, p3/M, z7.h, z10.h\n"
"fmla z17.h, p3/M, z6.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x15, x7, LSL #1]\n" // Load input point (1, 1)
+ "ld1h { z10.h }, p2/Z, [x15, x7, LSL #1]\n"
"fmla z22.h, p3/M, z8.h, z11.h\n"
"fmla z21.h, p3/M, z7.h, z11.h\n"
"fmla z20.h, p3/M, z6.h, z11.h\n"
"fmla z18.h, p3/M, z5.h, z11.h\n"
"fmla z17.h, p3/M, z4.h, z11.h\n"
"fmla z16.h, p3/M, z3.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x15, x28, LSL #1]\n" // Load input point (1, 4)
+ "ld1h { z11.h }, p2/Z, [x15, x28, LSL #1]\n"
"addvl x15, x15, #1\n"
"fmla z18.h, p3/M, z8.h, z12.h\n"
"fmla z31.h, p3/M, z4.h, z10.h\n"
"fmla z17.h, p3/M, z7.h, z12.h\n"
"fmla z16.h, p3/M, z6.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x12, x7, LSL #1]\n" // Load input point (4, 1)
+ "ld1h { z12.h }, p2/Z, [x12, x7, LSL #1]\n"
"fmla z30.h, p3/M, z3.h, z10.h\n"
"fmla z27.h, p3/M, z1.h, z10.h\n"
"fmla z26.h, p3/M, z0.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x12, x28, LSL #1]\n" // Load input point (4, 4)
+ "ld1h { z10.h }, p2/Z, [x12, x28, LSL #1]\n"
"whilelt p2.h, x5, %x[n_channels]\n"
"fmla z29.h, p3/M, z5.h, z11.h\n"
- "ld1h { z0.h }, p3/Z, [x4, #1, MUL VL]\n" // Load from weights and bias
+ "ld1h { z0.h }, p3/Z, [x4, #1, MUL VL]\n"
"addvl x12, x12, #1\n"
"fmla z28.h, p3/M, z4.h, z11.h\n"
"cmp x6, %x[n_channels]\n"
"fmla z25.h, p3/M, z2.h, z11.h\n"
- "ld1h { z2.h }, p3/Z, [x4, #3, MUL VL]\n" // Load from weights and bias
+ "ld1h { z2.h }, p3/Z, [x4, #3, MUL VL]\n"
"fmla z24.h, p3/M, z1.h, z11.h\n"
- "ld1h { z11.h }, p1/Z, [x8, x27, LSL #1]\n" // Load input point (0, 5)
+ "ld1h { z11.h }, p1/Z, [x8, x27, LSL #1]\n"
"fmla z23.h, p3/M, z7.h, z12.h\n"
- "ld1h { z1.h }, p3/Z, [x4, #2, MUL VL]\n" // Load from weights and bias
+ "ld1h { z1.h }, p3/Z, [x4, #2, MUL VL]\n"
"fmla z22.h, p3/M, z6.h, z12.h\n"
- "ld1h { z6.h }, p3/Z, [x4, #7, MUL VL]\n" // Load from weights and bias
+ "ld1h { z6.h }, p3/Z, [x4, #7, MUL VL]\n"
"fmla z19.h, p3/M, z4.h, z12.h\n"
"fmla z18.h, p3/M, z3.h, z12.h\n"
- "ld1h { z12.h }, p1/Z, [x14, x9, LSL #1]\n" // Load input point (2, 3)
+ "ld1h { z12.h }, p1/Z, [x14, x9, LSL #1]\n"
"fmla z21.h, p3/M, z8.h, z10.h\n"
- "ld1h { z3.h }, p3/Z, [x4, #4, MUL VL]\n" // Load from weights and bias
+ "ld1h { z3.h }, p3/Z, [x4, #4, MUL VL]\n"
"fmla z20.h, p3/M, z7.h, z10.h\n"
"fmla z17.h, p3/M, z5.h, z10.h\n"
- "ld1h { z5.h }, p3/Z, [x4, #6, MUL VL]\n" // Load from weights and bias
+ "ld1h { z5.h }, p3/Z, [x4, #6, MUL VL]\n"
"fmla z16.h, p3/M, z4.h, z10.h\n"
- "ld1h { z10.h }, p1/Z, [x8]\n" // Load input point (0, 0)
+ "ld1h { z10.h }, p1/Z, [x8]\n"
"fmax z31.h, p3/M, z31.h, z15.h\n"
- "ld1h { z4.h }, p3/Z, [x4, #5, MUL VL]\n" // Load from weights and bias
+ "ld1h { z4.h }, p3/Z, [x4, #5, MUL VL]\n"
"addvl x4, x4, #16\n"
"fmax z30.h, p3/M, z30.h, z15.h\n"
- "ld1h { z7.h }, p3/Z, [x4, #-8, MUL VL]\n" // Load from weights and bias
+ "ld1h { z7.h }, p3/Z, [x4, #-8, MUL VL]\n"
"fmax z29.h, p3/M, z29.h, z15.h\n"
- "ld1h { z8.h }, p3/Z, [x4, #-7, MUL VL]\n" // Load from weights and bias
+ "ld1h { z8.h }, p3/Z, [x4, #-7, MUL VL]\n"
"addvl x4, x4, #-6\n"
"fmin z31.h, p3/M, z31.h, z14.h\n"
- "st1h { z31.h }, p0, [x16]\n" // Store output point (0, 0)
- "mov z31.d, z13.d\n"
+ "st1h { z31.h }, p0, [x16]\n"
"fmin z30.h, p3/M, z30.h, z14.h\n"
- "st1h { z30.h }, p0, [x16, x17, LSL #1]\n" // Store output point (0, 1)
- "mov z30.d, z13.d\n"
"fmin z29.h, p3/M, z29.h, z14.h\n"
- "st1h { z29.h }, p0, [x16, x23, LSL #1]\n" // Store output point (0, 2)
- "mov z29.d, z13.d\n"
+ "st1h { z30.h }, p0, [x16, x17, LSL #1]\n"
"fmax z28.h, p3/M, z28.h, z15.h\n"
"fmax z27.h, p3/M, z27.h, z15.h\n"
+ "st1h { z29.h }, p0, [x16, x26, LSL #1]\n"
"fmax z26.h, p3/M, z26.h, z15.h\n"
"fmax z25.h, p3/M, z25.h, z15.h\n"
- "fmin z28.h, p3/M, z28.h, z14.h\n"
- "st1h { z28.h }, p0, [x16, x22, LSL #1]\n" // Store output point (0, 3)
- "mov z28.d, z13.d\n"
- "addvl x16, x16, #1\n"
- "fmin z27.h, p3/M, z27.h, z14.h\n"
- "st1h { z27.h }, p0, [x26]\n" // Store output point (1, 0)
- "mov z27.d, z13.d\n"
- "fmin z26.h, p3/M, z26.h, z14.h\n"
- "st1h { z26.h }, p0, [x26, x17, LSL #1]\n" // Store output point (1, 1)
- "mov z26.d, z13.d\n"
- "fmin z25.h, p3/M, z25.h, z14.h\n"
- "st1h { z25.h }, p0, [x26, x23, LSL #1]\n" // Store output point (1, 2)
- "mov z25.d, z13.d\n"
"fmax z24.h, p3/M, z24.h, z15.h\n"
+ "fmin z28.h, p3/M, z28.h, z14.h\n"
+ "st1h { z28.h }, p0, [x16, x22, LSL #1]\n"
+ "fmin z27.h, p3/M, z27.h, z14.h\n"
+ "addvl x16, x16, #1\n"
+ "fmin z26.h, p3/M, z26.h, z14.h\n"
+ "st1h { z27.h }, p0, [x25]\n"
+ "fmin z25.h, p3/M, z25.h, z14.h\n"
+ "fmin z24.h, p3/M, z24.h, z14.h\n"
+ "st1h { z26.h }, p0, [x25, x17, LSL #1]\n"
"fmax z23.h, p3/M, z23.h, z15.h\n"
+ "st1h { z25.h }, p0, [x25, x26, LSL #1]\n"
"fmax z22.h, p3/M, z22.h, z15.h\n"
"fmax z21.h, p3/M, z21.h, z15.h\n"
- "fmin z24.h, p3/M, z24.h, z14.h\n"
- "st1h { z24.h }, p0, [x26, x22, LSL #1]\n" // Store output point (1, 3)
- "mov z24.d, z13.d\n"
- "addvl x26, x26, #1\n"
+ "st1h { z24.h }, p0, [x25, x22, LSL #1]\n"
+ "addvl x25, x25, #1\n"
"fmin z23.h, p3/M, z23.h, z14.h\n"
- "st1h { z23.h }, p0, [x25]\n" // Store output point (2, 0)
- "mov z23.d, z13.d\n"
+ "st1h { z23.h }, p0, [x24]\n"
"fmin z22.h, p3/M, z22.h, z14.h\n"
- "st1h { z22.h }, p0, [x25, x17, LSL #1]\n" // Store output point (2, 1)
- "mov z22.d, z13.d\n"
"fmin z21.h, p3/M, z21.h, z14.h\n"
- "st1h { z21.h }, p0, [x25, x23, LSL #1]\n" // Store output point (2, 2)
- "mov z21.d, z13.d\n"
+ "st1h { z22.h }, p0, [x24, x17, LSL #1]\n"
"fmax z20.h, p3/M, z20.h, z15.h\n"
"fmax z19.h, p3/M, z19.h, z15.h\n"
+ "st1h { z21.h }, p0, [x24, x26, LSL #1]\n"
"fmax z18.h, p3/M, z18.h, z15.h\n"
"fmax z17.h, p3/M, z17.h, z15.h\n"
- "fmin z20.h, p3/M, z20.h, z14.h\n"
- "st1h { z20.h }, p0, [x25, x22, LSL #1]\n" // Store output point (2, 3)
- "mov z20.d, z13.d\n"
- "addvl x25, x25, #1\n"
- "fmin z19.h, p3/M, z19.h, z14.h\n"
- "st1h { z19.h }, p0, [x24]\n" // Store output point (3, 0)
- "mov z19.d, z13.d\n"
- "fmin z18.h, p3/M, z18.h, z14.h\n"
- "st1h { z18.h }, p0, [x24, x17, LSL #1]\n" // Store output point (3, 1)
- "mov z18.d, z13.d\n"
- "fmin z17.h, p3/M, z17.h, z14.h\n"
- "st1h { z17.h }, p0, [x24, x23, LSL #1]\n" // Store output point (3, 2)
- "mov z17.d, z13.d\n"
"fmax z16.h, p3/M, z16.h, z15.h\n"
- "fmin z16.h, p3/M, z16.h, z14.h\n"
- "st1h { z16.h }, p0, [x24, x22, LSL #1]\n" // Store output point (3, 3)
- "mov z16.d, z13.d\n"
+ "fmin z20.h, p3/M, z20.h, z14.h\n"
+ "st1h { z20.h }, p0, [x24, x22, LSL #1]\n"
+ "fmin z19.h, p3/M, z19.h, z14.h\n"
"addvl x24, x24, #1\n"
+ "fmin z18.h, p3/M, z18.h, z14.h\n"
+ "st1h { z19.h }, p0, [x23]\n"
+ "fmin z17.h, p3/M, z17.h, z14.h\n"
+ "fmin z16.h, p3/M, z16.h, z14.h\n"
+ "st1h { z18.h }, p0, [x23, x17, LSL #1]\n"
+ "st1h { z17.h }, p0, [x23, x26, LSL #1]\n"
+ "st1h { z16.h }, p0, [x23, x22, LSL #1]\n"
+ "addvl x23, x23, #1\n"
"blt 2b\n"
"3:" // Tile loop: Channel tail
- "fmla z31.h, p3/M, z8.h, z9.h\n"
+ "movprfx z31, z13\n fmla z31.h, p3/M, z8.h, z9.h\n"
"ldr x2, [%x[params_struct], %[offsetof_args_tile_i]]\n"
"mov p0.b, p2.b\n"
- "fmla z30.h, p3/M, z7.h, z9.h\n"
+ "movprfx z30, z13\n fmla z30.h, p3/M, z7.h, z9.h\n"
"ldr x3, [%x[params_struct], %[offsetof_args_tile_j]]\n"
"add x21, x2, #0x1\n"
- "fmla z29.h, p3/M, z6.h, z9.h\n"
+ "movprfx z29, z13\n fmla z29.h, p3/M, z6.h, z9.h\n"
"ldr x20, [%x[params_struct], %[offsetof_args_n_tile_rows]]\n"
- "fmla z27.h, p3/M, z5.h, z9.h\n"
+ "movprfx z27, z13\n fmla z27.h, p3/M, z5.h, z9.h\n"
"ldr x19, [%x[params_struct], %[offsetof_args_n_tile_cols]]\n"
"add x3, x3, #0x1\n"
- "fmla z26.h, p3/M, z4.h, z9.h\n"
+ "movprfx z26, z13\n fmla z26.h, p3/M, z4.h, z9.h\n"
"cmp x3, x19\n"
- "fmla z25.h, p3/M, z3.h, z9.h\n"
- "fmla z23.h, p3/M, z2.h, z9.h\n"
+ "movprfx z25, z13\n fmla z25.h, p3/M, z3.h, z9.h\n"
+ "movprfx z23, z13\n fmla z23.h, p3/M, z2.h, z9.h\n"
"csel x3, x3, XZR, LT\n"
- "fmla z22.h, p3/M, z1.h, z9.h\n"
+ "movprfx z22, z13\n fmla z22.h, p3/M, z1.h, z9.h\n"
"csel x2, x2, x21, LT\n"
- "fmla z21.h, p3/M, z0.h, z9.h\n"
- "ld1h { z9.h }, p2/Z, [x13, x10, LSL #1]\n" // Load input point (3, 2)
+ "movprfx z21, z13\n fmla z21.h, p3/M, z0.h, z9.h\n"
+ "ld1h { z9.h }, p2/Z, [x13, x10, LSL #1]\n"
"cmp x2, x20\n"
"fmla z31.h, p3/M, z0.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x11]\n" // Load input point (5, 0)
- "fmla z28.h, p3/M, z2.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x11, x27, LSL #1]\n" // Load input point (5, 5)
+ "ld1h { z10.h }, p2/Z, [x11]\n"
+ "movprfx z28, z13\n fmla z28.h, p3/M, z2.h, z11.h\n"
+ "ld1h { z11.h }, p2/Z, [x11, x27, LSL #1]\n"
"fmla z30.h, p3/M, z8.h, z12.h\n"
"fmla z29.h, p3/M, z7.h, z12.h\n"
"fmla z26.h, p3/M, z5.h, z12.h\n"
"fmla z28.h, p3/M, z6.h, z12.h\n"
"fmla z25.h, p3/M, z4.h, z12.h\n"
- "fmla z24.h, p3/M, z3.h, z12.h\n"
+ "movprfx z24, z13\n fmla z24.h, p3/M, z3.h, z12.h\n"
"fmla z22.h, p3/M, z2.h, z12.h\n"
"fmla z21.h, p3/M, z1.h, z12.h\n"
- "fmla z20.h, p3/M, z0.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x8, x7, LSL #1]\n" // Load input point (0, 1)
- "fmla z19.h, p3/M, z6.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x13, x9, LSL #1]\n" // Load input point (3, 3)
- "fmla z16.h, p3/M, z8.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x8, x28, LSL #1]\n" // Load input point (0, 4)
+ "movprfx z20, z13\n fmla z20.h, p3/M, z0.h, z12.h\n"
+ "ld1h { z12.h }, p2/Z, [x8, x7, LSL #1]\n"
+ "movprfx z19, z13\n fmla z19.h, p3/M, z6.h, z10.h\n"
+ "ld1h { z10.h }, p2/Z, [x13, x9, LSL #1]\n"
+ "movprfx z16, z13\n fmla z16.h, p3/M, z8.h, z11.h\n"
+ "ld1h { z11.h }, p2/Z, [x8, x28, LSL #1]\n"
"fmla z27.h, p3/M, z8.h, z9.h\n"
"fmla z26.h, p3/M, z7.h, z9.h\n"
"fmla z25.h, p3/M, z6.h, z9.h\n"
@@ -486,15 +454,15 @@
"fmla z22.h, p3/M, z4.h, z9.h\n"
"fmla z21.h, p3/M, z3.h, z9.h\n"
"fmla z19.h, p3/M, z2.h, z9.h\n"
- "fmla z18.h, p3/M, z1.h, z9.h\n"
- "fmla z17.h, p3/M, z0.h, z9.h\n"
- "ld1h { z9.h }, p2/Z, [x15]\n" // Load input point (1, 0)
+ "movprfx z18, z13\n fmla z18.h, p3/M, z1.h, z9.h\n"
+ "movprfx z17, z13\n fmla z17.h, p3/M, z0.h, z9.h\n"
+ "ld1h { z9.h }, p2/Z, [x15]\n"
"fmla z31.h, p3/M, z1.h, z12.h\n"
"fmla z30.h, p3/M, z0.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x15, x27, LSL #1]\n" // Load input point (1, 5)
+ "ld1h { z12.h }, p2/Z, [x15, x27, LSL #1]\n"
"fmla z29.h, p3/M, z2.h, z11.h\n"
"fmla z28.h, p3/M, z1.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x12]\n" // Load input point (4, 0)
+ "ld1h { z11.h }, p2/Z, [x12]\n"
"fmla z26.h, p3/M, z8.h, z10.h\n"
"fmla z25.h, p3/M, z7.h, z10.h\n"
"fmla z24.h, p3/M, z6.h, z10.h\n"
@@ -504,117 +472,117 @@
"fmla z18.h, p3/M, z2.h, z10.h\n"
"fmla z17.h, p3/M, z1.h, z10.h\n"
"fmla z16.h, p3/M, z0.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x15, x10, LSL #1]\n" // Load input point (1, 2)
+ "ld1h { z10.h }, p2/Z, [x15, x10, LSL #1]\n"
"fmla z31.h, p3/M, z3.h, z9.h\n"
"fmla z27.h, p3/M, z0.h, z9.h\n"
"fmla z28.h, p3/M, z5.h, z12.h\n"
"fmla z24.h, p3/M, z2.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x15, x9, LSL #1]\n" // Load input point (1, 3)
+ "ld1h { z12.h }, p2/Z, [x15, x9, LSL #1]\n"
"fmla z23.h, p3/M, z6.h, z11.h\n"
"fmla z19.h, p3/M, z3.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x12, x27, LSL #1]\n" // Load input point (4, 5)
+ "ld1h { z11.h }, p2/Z, [x12, x27, LSL #1]\n"
"fmla z31.h, p3/M, z5.h, z10.h\n"
"fmla z30.h, p3/M, z4.h, z10.h\n"
"fmla z29.h, p3/M, z3.h, z10.h\n"
"fmla z27.h, p3/M, z2.h, z10.h\n"
"fmla z26.h, p3/M, z1.h, z10.h\n"
"fmla z25.h, p3/M, z0.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x14, x7, LSL #1]\n" // Load input point (2, 1)
+ "ld1h { z10.h }, p2/Z, [x14, x7, LSL #1]\n"
"fmla z20.h, p3/M, z8.h, z11.h\n"
"fmla z16.h, p3/M, z5.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x11, x7, LSL #1]\n" // Load input point (5, 1)
+ "ld1h { z11.h }, p2/Z, [x11, x7, LSL #1]\n"
"fmla z30.h, p3/M, z5.h, z12.h\n"
"fmla z29.h, p3/M, z4.h, z12.h\n"
"fmla z28.h, p3/M, z3.h, z12.h\n"
"fmla z26.h, p3/M, z2.h, z12.h\n"
"fmla z25.h, p3/M, z1.h, z12.h\n"
"fmla z24.h, p3/M, z0.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x14, x28, LSL #1]\n" // Load input point (2, 4)
+ "ld1h { z12.h }, p2/Z, [x14, x28, LSL #1]\n"
"fmla z19.h, p3/M, z7.h, z11.h\n"
"fmla z18.h, p3/M, z6.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x11, x28, LSL #1]\n" // Load input point (5, 4)
+ "ld1h { z11.h }, p2/Z, [x11, x28, LSL #1]\n"
"fmla z31.h, p3/M, z7.h, z10.h\n"
"fmla z30.h, p3/M, z6.h, z10.h\n"
"fmla z27.h, p3/M, z4.h, z10.h\n"
"fmla z26.h, p3/M, z3.h, z10.h\n"
"fmla z23.h, p3/M, z1.h, z10.h\n"
"fmla z22.h, p3/M, z0.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x8, x10, LSL #1]\n" // Load input point (0, 2)
+ "ld1h { z10.h }, p2/Z, [x8, x10, LSL #1]\n"
"fmla z17.h, p3/M, z8.h, z11.h\n"
"fmla z16.h, p3/M, z7.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x13, x7, LSL #1]\n" // Load input point (3, 1)
+ "ld1h { z11.h }, p2/Z, [x13, x7, LSL #1]\n"
"fmla z29.h, p3/M, z8.h, z12.h\n"
"fmla z28.h, p3/M, z7.h, z12.h\n"
"fmla z25.h, p3/M, z5.h, z12.h\n"
"fmla z24.h, p3/M, z4.h, z12.h\n"
"fmla z21.h, p3/M, z2.h, z12.h\n"
"fmla z20.h, p3/M, z1.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x8, x9, LSL #1]\n" // Load input point (0, 3)
+ "ld1h { z12.h }, p2/Z, [x8, x9, LSL #1]\n"
"fmla z31.h, p3/M, z2.h, z10.h\n"
"fmla z30.h, p3/M, z1.h, z10.h\n"
"fmla z29.h, p3/M, z0.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x14]\n" // Load input point (2, 0)
+ "ld1h { z10.h }, p2/Z, [x14]\n"
"fmla z27.h, p3/M, z7.h, z11.h\n"
"fmla z26.h, p3/M, z6.h, z11.h\n"
"fmla z23.h, p3/M, z4.h, z11.h\n"
"fmla z22.h, p3/M, z3.h, z11.h\n"
"fmla z19.h, p3/M, z1.h, z11.h\n"
"fmla z18.h, p3/M, z0.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x13, x28, LSL #1]\n" // Load input point (3, 4)
+ "ld1h { z11.h }, p2/Z, [x13, x28, LSL #1]\n"
"fmla z30.h, p3/M, z2.h, z12.h\n"
"fmla z29.h, p3/M, z1.h, z12.h\n"
"fmla z28.h, p3/M, z0.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x14, x27, LSL #1]\n" // Load input point (2, 5)
+ "ld1h { z12.h }, p2/Z, [x14, x27, LSL #1]\n"
"fmla z31.h, p3/M, z6.h, z10.h\n"
"fmla z27.h, p3/M, z3.h, z10.h\n"
"fmla z23.h, p3/M, z0.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x13]\n" // Load input point (3, 0)
+ "ld1h { z10.h }, p2/Z, [x13]\n"
"fmla z25.h, p3/M, z8.h, z11.h\n"
"fmla z24.h, p3/M, z7.h, z11.h\n"
"fmla z21.h, p3/M, z5.h, z11.h\n"
"fmla z20.h, p3/M, z4.h, z11.h\n"
"fmla z17.h, p3/M, z2.h, z11.h\n"
"fmla z16.h, p3/M, z1.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x12, x10, LSL #1]\n" // Load input point (4, 2)
+ "ld1h { z11.h }, p2/Z, [x12, x10, LSL #1]\n"
"fmla z28.h, p3/M, z8.h, z12.h\n"
"fmla z24.h, p3/M, z5.h, z12.h\n"
"fmla z20.h, p3/M, z2.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x13, x27, LSL #1]\n" // Load input point (3, 5)
+ "ld1h { z12.h }, p2/Z, [x13, x27, LSL #1]\n"
"fmla z27.h, p3/M, z6.h, z10.h\n"
"fmla z23.h, p3/M, z3.h, z10.h\n"
"fmla z19.h, p3/M, z0.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x11, x10, LSL #1]\n" // Load input point (5, 2)
+ "ld1h { z10.h }, p2/Z, [x11, x10, LSL #1]\n"
"fmla z22.h, p3/M, z7.h, z11.h\n"
"fmla z21.h, p3/M, z6.h, z11.h\n"
"fmla z23.h, p3/M, z8.h, z11.h\n"
"fmla z19.h, p3/M, z5.h, z11.h\n"
"fmla z18.h, p3/M, z4.h, z11.h\n"
"fmla z17.h, p3/M, z3.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x12, x9, LSL #1]\n" // Load input point (4, 3)
+ "ld1h { z11.h }, p2/Z, [x12, x9, LSL #1]\n"
"fmla z24.h, p3/M, z8.h, z12.h\n"
"fmla z20.h, p3/M, z5.h, z12.h\n"
"fmla z16.h, p3/M, z2.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x11, x9, LSL #1]\n" // Load input point (5, 3)
+ "ld1h { z12.h }, p2/Z, [x11, x9, LSL #1]\n"
"fmla z19.h, p3/M, z8.h, z10.h\n"
"fmla z18.h, p3/M, z7.h, z10.h\n"
"fmla z17.h, p3/M, z6.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x15, x7, LSL #1]\n" // Load input point (1, 1)
+ "ld1h { z10.h }, p2/Z, [x15, x7, LSL #1]\n"
"fmla z22.h, p3/M, z8.h, z11.h\n"
"fmla z21.h, p3/M, z7.h, z11.h\n"
"fmla z20.h, p3/M, z6.h, z11.h\n"
"fmla z18.h, p3/M, z5.h, z11.h\n"
"fmla z17.h, p3/M, z4.h, z11.h\n"
"fmla z16.h, p3/M, z3.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x15, x28, LSL #1]\n" // Load input point (1, 4)
+ "ld1h { z11.h }, p2/Z, [x15, x28, LSL #1]\n"
"fmla z31.h, p3/M, z4.h, z10.h\n"
"fmla z18.h, p3/M, z8.h, z12.h\n"
"fmla z17.h, p3/M, z7.h, z12.h\n"
"fmla z16.h, p3/M, z6.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x12, x7, LSL #1]\n" // Load input point (4, 1)
+ "ld1h { z12.h }, p2/Z, [x12, x7, LSL #1]\n"
"fmla z30.h, p3/M, z3.h, z10.h\n"
"fmla z27.h, p3/M, z1.h, z10.h\n"
"fmla z26.h, p3/M, z0.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x12, x28, LSL #1]\n" // Load input point (4, 4)
+ "ld1h { z10.h }, p2/Z, [x12, x28, LSL #1]\n"
"fmla z29.h, p3/M, z5.h, z11.h\n"
"fmla z28.h, p3/M, z4.h, z11.h\n"
"fmla z25.h, p3/M, z2.h, z11.h\n"
@@ -632,49 +600,49 @@
"fmax z29.h, p3/M, z29.h, z15.h\n"
"fmax z28.h, p3/M, z28.h, z15.h\n"
"fmin z31.h, p3/M, z31.h, z14.h\n"
- "st1h { z31.h }, p0, [x16]\n" // Store output point (0, 0)
+ "st1h { z31.h }, p0, [x16]\n"
"fmin z30.h, p3/M, z30.h, z14.h\n"
"fmin z29.h, p3/M, z29.h, z14.h\n"
- "st1h { z30.h }, p0, [x16, x17, LSL #1]\n" // Store output point (0, 1)
+ "st1h { z30.h }, p0, [x16, x17, LSL #1]\n"
"fmin z28.h, p3/M, z28.h, z14.h\n"
"fmax z27.h, p3/M, z27.h, z15.h\n"
- "st1h { z29.h }, p0, [x16, x23, LSL #1]\n" // Store output point (0, 2)
+ "st1h { z29.h }, p0, [x16, x26, LSL #1]\n"
"fmax z26.h, p3/M, z26.h, z15.h\n"
- "st1h { z28.h }, p0, [x16, x22, LSL #1]\n" // Store output point (0, 3)
+ "st1h { z28.h }, p0, [x16, x22, LSL #1]\n"
"fmin z27.h, p3/M, z27.h, z14.h\n"
"fmax z25.h, p3/M, z25.h, z15.h\n"
- "st1h { z27.h }, p0, [x26]\n" // Store output point (1, 0)
+ "st1h { z27.h }, p0, [x25]\n"
"fmin z26.h, p3/M, z26.h, z14.h\n"
"fmin z25.h, p3/M, z25.h, z14.h\n"
- "st1h { z26.h }, p0, [x26, x17, LSL #1]\n" // Store output point (1, 1)
+ "st1h { z26.h }, p0, [x25, x17, LSL #1]\n"
"fmax z24.h, p3/M, z24.h, z15.h\n"
"fmax z23.h, p3/M, z23.h, z15.h\n"
- "st1h { z25.h }, p0, [x26, x23, LSL #1]\n" // Store output point (1, 2)
+ "st1h { z25.h }, p0, [x25, x26, LSL #1]\n"
"fmax z22.h, p3/M, z22.h, z15.h\n"
"fmax z21.h, p3/M, z21.h, z15.h\n"
"fmax z20.h, p3/M, z20.h, z15.h\n"
"fmin z24.h, p3/M, z24.h, z14.h\n"
- "st1h { z24.h }, p0, [x26, x22, LSL #1]\n" // Store output point (1, 3)
+ "st1h { z24.h }, p0, [x25, x22, LSL #1]\n"
"fmin z23.h, p3/M, z23.h, z14.h\n"
"fmin z22.h, p3/M, z22.h, z14.h\n"
- "st1h { z23.h }, p0, [x25]\n" // Store output point (2, 0)
+ "st1h { z23.h }, p0, [x24]\n"
"fmin z21.h, p3/M, z21.h, z14.h\n"
"fmin z20.h, p3/M, z20.h, z14.h\n"
- "st1h { z22.h }, p0, [x25, x17, LSL #1]\n" // Store output point (2, 1)
+ "st1h { z22.h }, p0, [x24, x17, LSL #1]\n"
"fmax z19.h, p3/M, z19.h, z15.h\n"
- "st1h { z21.h }, p0, [x25, x23, LSL #1]\n" // Store output point (2, 2)
+ "st1h { z21.h }, p0, [x24, x26, LSL #1]\n"
"fmax z18.h, p3/M, z18.h, z15.h\n"
"fmax z17.h, p3/M, z17.h, z15.h\n"
- "st1h { z20.h }, p0, [x25, x22, LSL #1]\n" // Store output point (2, 3)
+ "st1h { z20.h }, p0, [x24, x22, LSL #1]\n"
"fmin z19.h, p3/M, z19.h, z14.h\n"
- "st1h { z19.h }, p0, [x24]\n" // Store output point (3, 0)
+ "st1h { z19.h }, p0, [x23]\n"
"fmin z18.h, p3/M, z18.h, z14.h\n"
"fmin z17.h, p3/M, z17.h, z14.h\n"
- "st1h { z18.h }, p0, [x24, x17, LSL #1]\n" // Store output point (3, 1)
+ "st1h { z18.h }, p0, [x23, x17, LSL #1]\n"
"fmax z16.h, p3/M, z16.h, z15.h\n"
- "st1h { z17.h }, p0, [x24, x23, LSL #1]\n" // Store output point (3, 2)
+ "st1h { z17.h }, p0, [x23, x26, LSL #1]\n"
"fmin z16.h, p3/M, z16.h, z14.h\n"
- "st1h { z16.h }, p0, [x24, x22, LSL #1]\n" // Store output point (3, 3)
+ "st1h { z16.h }, p0, [x23, x22, LSL #1]\n"
"blt 1b\n"
:
: [n_channels] "r" ((unsigned long) n_channels), [offsetof_args_inptr] "I" (offsetof(Args, inptr)), [offsetof_args_ld_input_col] "I" (offsetof(Args, ld_input_col)), [offsetof_args_ld_input_row] "I" (offsetof(Args, ld_input_row)), [offsetof_args_ld_output_col] "I" (offsetof(Args, ld_output_col)), [offsetof_args_ld_output_row] "I" (offsetof(Args, ld_output_row)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_n_tile_cols] "I" (offsetof(Args, n_tile_cols)), [offsetof_args_n_tile_rows] "I" (offsetof(Args, n_tile_rows)), [offsetof_args_outptr] "I" (offsetof(Args, outptr)), [offsetof_args_params] "I" (offsetof(Args, params)), [offsetof_args_tile_i] "I" (offsetof(Args, tile_i)), [offsetof_args_tile_j] "I" (offsetof(Args, tile_j)), [params_struct] "r" (¶ms_struct)
@@ -685,4 +653,4 @@
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
+#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp
index 8eaf0a4..66f6c3b 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp
@@ -25,7 +25,7 @@
#include <cstddef>
#include <cstdint>
-#if defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
+#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
namespace arm_conv {
namespace depthwise {
@@ -98,570 +98,538 @@
activation_min, activation_max);
__asm__ __volatile__(
- "ldr x2, [%x[params_struct], %[offsetof_args_outptrs]]\n"
+ "ldr x16, [%x[params_struct], %[offsetof_args_outptrs]]\n"
"ptrue p3.b\n"
- "ldr x3, [%x[params_struct], %[offsetof_args_params]]\n"
- "add x4, %x[params_struct], %[offsetof_Args_inptrs]\n"
+ "ldr x15, [%x[params_struct], %[offsetof_args_params]]\n"
+ "add x14, %x[params_struct], %[offsetof_Args_inptrs]\n"
"ld1rh { z15.h }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n"
- "mov x5, #0x0\n"
+ "mov x13, #0x0\n"
"ld1rh { z14.h }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n"
- "cnth x6\n"
- "ld1h { z13.h }, p3/Z, [x3]\n" // Load from weights and bias
- "mov z31.d, z13.d\n"
- "ld1h { z0.h }, p3/Z, [x3, #1, MUL VL]\n" // Load from weights and bias
- "sub x7, XZR, x6\n"
- "mov z30.d, z13.d\n"
- "ld1h { z1.h }, p3/Z, [x3, #2, MUL VL]\n" // Load from weights and bias
+ "cnth x12\n"
+ "ld1h { z13.h }, p3/Z, [x15]\n"
+ "sub x11, XZR, x12\n"
+ "ld1h { z0.h }, p3/Z, [x15, #1, MUL VL]\n"
"whilelt p2.h, XZR, %x[n_channels]\n"
- "mov z29.d, z13.d\n"
- "ld1h { z2.h }, p3/Z, [x3, #3, MUL VL]\n" // Load from weights and bias
- "cmp x6, %x[n_channels]\n"
- "mov z28.d, z13.d\n"
- "ld1h { z3.h }, p3/Z, [x3, #4, MUL VL]\n" // Load from weights and bias
- "mov z27.d, z13.d\n"
- "ld1h { z4.h }, p3/Z, [x3, #5, MUL VL]\n" // Load from weights and bias
- "mov z26.d, z13.d\n"
- "ld1h { z5.h }, p3/Z, [x3, #6, MUL VL]\n" // Load from weights and bias
- "mov z25.d, z13.d\n"
- "ld1h { z6.h }, p3/Z, [x3, #7, MUL VL]\n" // Load from weights and bias
- "addvl x3, x3, #16\n"
- "mov z24.d, z13.d\n"
- "ld1h { z7.h }, p3/Z, [x3, #-8, MUL VL]\n" // Load from weights and bias
- "mov z23.d, z13.d\n"
- "ld1h { z8.h }, p3/Z, [x3, #-7, MUL VL]\n" // Load from weights and bias
- "addvl x3, x3, #-6\n"
- "mov z22.d, z13.d\n"
- "ldp x8, x17, [x4, #0x0]\n"
- "mov z21.d, z13.d\n"
- "ldp x16, x15, [x4, #0x10]\n"
- "mov z20.d, z13.d\n"
- "ld1h { z9.h }, p2/Z, [x8, x5, LSL #1]\n"
- "mov z19.d, z13.d\n"
- "mov z18.d, z13.d\n"
- "ld1h { z10.h }, p2/Z, [x17, x5, LSL #1]\n"
- "mov z17.d, z13.d\n"
- "ld1h { z11.h }, p2/Z, [x16, x5, LSL #1]\n"
- "mov z16.d, z13.d\n"
- "ld1h { z12.h }, p2/Z, [x15, x5, LSL #1]\n"
+ "ld1h { z1.h }, p3/Z, [x15, #2, MUL VL]\n"
+ "cmp x12, %x[n_channels]\n"
+ "ld1h { z2.h }, p3/Z, [x15, #3, MUL VL]\n"
+ "ld1h { z3.h }, p3/Z, [x15, #4, MUL VL]\n"
+ "ld1h { z4.h }, p3/Z, [x15, #5, MUL VL]\n"
+ "ld1h { z5.h }, p3/Z, [x15, #6, MUL VL]\n"
+ "ld1h { z6.h }, p3/Z, [x15, #7, MUL VL]\n"
+ "addvl x15, x15, #16\n"
+ "ldp x10, x9, [x14, #0x0]\n"
+ "ld1h { z7.h }, p3/Z, [x15, #-8, MUL VL]\n"
+ "ld1h { z8.h }, p3/Z, [x15, #-7, MUL VL]\n"
+ "addvl x15, x15, #-6\n"
+ "ld1h { z9.h }, p2/Z, [x10, x13, LSL #1]\n"
+ "ld1h { z10.h }, p2/Z, [x9, x13, LSL #1]\n"
+ "ldp x28, x27, [x14, #0x10]\n"
+ "ld1h { z11.h }, p2/Z, [x28, x13, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x27, x13, LSL #1]\n"
"bge 2f\n"
"1:" // Channel loop
- "fmla z31.h, p3/M, z8.h, z9.h\n"
- "ldr x14, [x4, #0x20]\n"
- "whilelt p1.h, x6, %x[n_channels]\n"
- "fmla z30.h, p3/M, z7.h, z9.h\n"
- "ldr x13, [x4, #0x28]\n"
- "inch x7\n"
- "fmla z29.h, p3/M, z6.h, z9.h\n"
- "ldr x12, [x4, #0x30]\n"
+ "movprfx z31, z13\n fmla z31.h, p3/M, z8.h, z9.h\n"
+ "ldr x26, [x14, #0x20]\n"
+ "whilelt p1.h, x12, %x[n_channels]\n"
+ "movprfx z30, z13\n fmla z30.h, p3/M, z7.h, z9.h\n"
+ "ldr x25, [x14, #0x28]\n"
+ "inch x11\n"
+ "movprfx z29, z13\n fmla z29.h, p3/M, z6.h, z9.h\n"
+ "ldr x24, [x14, #0x30]\n"
"mov p0.b, p2.b\n"
- "fmla z27.h, p3/M, z5.h, z9.h\n"
- "ldr x11, [x4, #0x38]\n"
- "fmla z26.h, p3/M, z4.h, z9.h\n"
- "ldr x10, [x4, #0x40]\n"
- "fmla z25.h, p3/M, z3.h, z9.h\n"
- "ldr x9, [x4, #0x48]\n"
- "fmla z23.h, p3/M, z2.h, z9.h\n"
- "ldr x28, [x4, #0x50]\n"
- "fmla z22.h, p3/M, z1.h, z9.h\n"
- "ldr x27, [x4, #0x58]\n"
- "fmla z21.h, p3/M, z0.h, z9.h\n"
- "ld1h { z9.h }, p2/Z, [x12, x5, LSL #1]\n"
+ "movprfx z27, z13\n fmla z27.h, p3/M, z5.h, z9.h\n"
+ "ldr x23, [x14, #0x38]\n"
+ "movprfx z26, z13\n fmla z26.h, p3/M, z4.h, z9.h\n"
+ "ldr x10, [x14, #0x40]\n"
+ "movprfx z25, z13\n fmla z25.h, p3/M, z3.h, z9.h\n"
+ "ldr x9, [x14, #0x48]\n"
+ "movprfx z23, z13\n fmla z23.h, p3/M, z2.h, z9.h\n"
+ "ldr x28, [x14, #0x50]\n"
+ "movprfx z22, z13\n fmla z22.h, p3/M, z1.h, z9.h\n"
+ "ldr x27, [x14, #0x58]\n"
+ "movprfx z21, z13\n fmla z21.h, p3/M, z0.h, z9.h\n"
+ "ld1h { z9.h }, p2/Z, [x24, x13, LSL #1]\n"
"fmla z31.h, p3/M, z0.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x14, x5, LSL #1]\n"
- "fmla z28.h, p3/M, z2.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x13, x5, LSL #1]\n"
+ "ld1h { z10.h }, p2/Z, [x26, x13, LSL #1]\n"
+ "movprfx z28, z13\n fmla z28.h, p3/M, z2.h, z11.h\n"
+ "ld1h { z11.h }, p2/Z, [x25, x13, LSL #1]\n"
"fmla z30.h, p3/M, z8.h, z12.h\n"
- "ldr x26, [x4, #0x60]\n"
+ "ldr x26, [x14, #0x60]\n"
"fmla z29.h, p3/M, z7.h, z12.h\n"
- "ldr x25, [x4, #0x68]\n"
+ "ldr x25, [x14, #0x68]\n"
"fmla z26.h, p3/M, z5.h, z12.h\n"
- "ldr x24, [x4, #0x70]\n"
+ "ldr x24, [x14, #0x70]\n"
"fmla z28.h, p3/M, z6.h, z12.h\n"
- "ldr x23, [x4, #0x78]\n"
+ "ldr x22, [x16, #0x0]\n"
"fmla z25.h, p3/M, z4.h, z12.h\n"
- "ldr x8, [x4, #0x80]\n"
- "fmla z24.h, p3/M, z3.h, z12.h\n"
- "ldr x17, [x4, #0x88]\n"
+ "ldr x21, [x16, #0x8]\n"
+ "movprfx z24, z13\n fmla z24.h, p3/M, z3.h, z12.h\n"
+ "ldr x20, [x16, #0x10]\n"
"fmla z22.h, p3/M, z2.h, z12.h\n"
- "ldr x16, [x4, #0x90]\n"
+ "ldr x19, [x16, #0x18]\n"
"fmla z21.h, p3/M, z1.h, z12.h\n"
- "ldr x15, [x4, #0x98]\n"
- "fmla z20.h, p3/M, z0.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x11, x5, LSL #1]\n"
- "fmla z19.h, p3/M, z6.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x9, x5, LSL #1]\n"
- "fmla z16.h, p3/M, z8.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x10, x5, LSL #1]\n"
+ "movprfx z20, z13\n fmla z20.h, p3/M, z0.h, z12.h\n"
+ "ld1h { z12.h }, p2/Z, [x23, x13, LSL #1]\n"
+ "movprfx z19, z13\n fmla z19.h, p3/M, z6.h, z10.h\n"
+ "ld1h { z10.h }, p2/Z, [x9, x13, LSL #1]\n"
+ "movprfx z16, z13\n fmla z16.h, p3/M, z8.h, z11.h\n"
+ "ld1h { z11.h }, p2/Z, [x10, x13, LSL #1]\n"
"fmla z27.h, p3/M, z8.h, z9.h\n"
- "ldr x14, [x4, #0xa0]\n"
+ "ldr x23, [x14, #0x78]\n"
"fmla z26.h, p3/M, z7.h, z9.h\n"
- "ldr x13, [x4, #0xa8]\n"
+ "ldr x10, [x14, #0x80]\n"
"fmla z25.h, p3/M, z6.h, z9.h\n"
- "ldr x12, [x4, #0xb0]\n"
+ "ldr x9, [x14, #0x88]\n"
"fmla z23.h, p3/M, z5.h, z9.h\n"
- "ldr x11, [x4, #0xb8]\n"
"fmla z22.h, p3/M, z4.h, z9.h\n"
- "ldr x10, [x4, #0xc0]\n"
"fmla z21.h, p3/M, z3.h, z9.h\n"
- "ldr x9, [x4, #0xc8]\n"
"fmla z19.h, p3/M, z2.h, z9.h\n"
- "ldr x22, [x2, #0x0]\n"
- "fmla z18.h, p3/M, z1.h, z9.h\n"
- "ldr x21, [x2, #0x8]\n"
- "fmla z17.h, p3/M, z0.h, z9.h\n"
- "ld1h { z9.h }, p2/Z, [x28, x5, LSL #1]\n"
+ "movprfx z18, z13\n fmla z18.h, p3/M, z1.h, z9.h\n"
+ "movprfx z17, z13\n fmla z17.h, p3/M, z0.h, z9.h\n"
+ "ld1h { z9.h }, p2/Z, [x28, x13, LSL #1]\n"
"fmla z31.h, p3/M, z1.h, z12.h\n"
- "ldr x28, [x4, #0xd0]\n"
+ "ldr x28, [x14, #0x90]\n"
"fmla z30.h, p3/M, z0.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x27, x5, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x27, x13, LSL #1]\n"
"fmla z29.h, p3/M, z2.h, z11.h\n"
- "ldr x27, [x4, #0xd8]\n"
+ "ldr x27, [x14, #0x98]\n"
"fmla z28.h, p3/M, z1.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x26, x5, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x26, x13, LSL #1]\n"
"fmla z26.h, p3/M, z8.h, z10.h\n"
- "ldr x26, [x4, #0xe0]\n"
+ "ldr x26, [x14, #0xa0]\n"
"fmla z25.h, p3/M, z7.h, z10.h\n"
- "ldr x20, [x2, #0x10]\n"
+ "ld1h { z13.h }, p3/Z, [x15]\n"
"fmla z24.h, p3/M, z6.h, z10.h\n"
- "ldr x19, [x2, #0x18]\n"
"fmla z22.h, p3/M, z5.h, z10.h\n"
- "ld1h { z13.h }, p3/Z, [x3]\n" // Load from weights and bias
"fmla z21.h, p3/M, z4.h, z10.h\n"
"fmla z20.h, p3/M, z3.h, z10.h\n"
"fmla z18.h, p3/M, z2.h, z10.h\n"
"fmla z17.h, p3/M, z1.h, z10.h\n"
"fmla z16.h, p3/M, z0.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x25, x5, LSL #1]\n"
+ "ld1h { z10.h }, p2/Z, [x25, x13, LSL #1]\n"
"fmla z31.h, p3/M, z3.h, z9.h\n"
- "ldr x25, [x4, #0xe8]\n"
+ "ldr x25, [x14, #0xa8]\n"
"fmla z27.h, p3/M, z0.h, z9.h\n"
"fmla z28.h, p3/M, z5.h, z12.h\n"
"fmla z24.h, p3/M, z2.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x23, x5, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x23, x13, LSL #1]\n"
"fmla z23.h, p3/M, z6.h, z11.h\n"
- "ldr x23, [x4, #0xf8]\n"
+ "ldr x23, [x14, #0xb8]\n"
"fmla z19.h, p3/M, z3.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x24, x5, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x24, x13, LSL #1]\n"
"fmla z31.h, p3/M, z5.h, z10.h\n"
- "ldr x24, [x4, #0xf0]\n"
+ "ldr x24, [x14, #0xb0]\n"
"fmla z30.h, p3/M, z4.h, z10.h\n"
"fmla z29.h, p3/M, z3.h, z10.h\n"
"fmla z27.h, p3/M, z2.h, z10.h\n"
"fmla z26.h, p3/M, z1.h, z10.h\n"
"fmla z25.h, p3/M, z0.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x17, x5, LSL #1]\n"
+ "ld1h { z10.h }, p2/Z, [x9, x13, LSL #1]\n"
"fmla z20.h, p3/M, z8.h, z11.h\n"
- "ldr x17, [x4, #0x108]\n"
+ "ldr x9, [x14, #0xc8]\n"
"fmla z16.h, p3/M, z5.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x8, x5, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x10, x13, LSL #1]\n"
"fmla z30.h, p3/M, z5.h, z12.h\n"
- "ldr x8, [x4, #0x100]\n"
+ "ldr x10, [x14, #0xc0]\n"
"fmla z29.h, p3/M, z4.h, z12.h\n"
"fmla z28.h, p3/M, z3.h, z12.h\n"
"fmla z26.h, p3/M, z2.h, z12.h\n"
"fmla z25.h, p3/M, z1.h, z12.h\n"
"fmla z24.h, p3/M, z0.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x15, x5, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x27, x13, LSL #1]\n"
"fmla z19.h, p3/M, z7.h, z11.h\n"
- "ldr x15, [x4, #0x118]\n"
+ "ldr x27, [x14, #0xd8]\n"
"fmla z18.h, p3/M, z6.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x16, x5, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x28, x13, LSL #1]\n"
"fmla z31.h, p3/M, z7.h, z10.h\n"
- "ldr x16, [x4, #0x110]\n"
+ "ldr x28, [x14, #0xd0]\n"
"fmla z30.h, p3/M, z6.h, z10.h\n"
"fmla z27.h, p3/M, z4.h, z10.h\n"
"fmla z26.h, p3/M, z3.h, z10.h\n"
"fmla z23.h, p3/M, z1.h, z10.h\n"
"fmla z22.h, p3/M, z0.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x14, x5, LSL #1]\n"
+ "ld1h { z10.h }, p2/Z, [x26, x13, LSL #1]\n"
"fmla z17.h, p3/M, z8.h, z11.h\n"
+ "ldr x26, [x14, #0xe0]\n"
"fmla z16.h, p3/M, z7.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x13, x5, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x25, x13, LSL #1]\n"
"fmla z29.h, p3/M, z8.h, z12.h\n"
+ "ldr x25, [x14, #0xe8]\n"
"fmla z28.h, p3/M, z7.h, z12.h\n"
"fmla z25.h, p3/M, z5.h, z12.h\n"
"fmla z24.h, p3/M, z4.h, z12.h\n"
"fmla z21.h, p3/M, z2.h, z12.h\n"
"fmla z20.h, p3/M, z1.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x12, x5, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x24, x13, LSL #1]\n"
"fmla z31.h, p3/M, z2.h, z10.h\n"
+ "ldr x24, [x14, #0xf0]\n"
"fmla z30.h, p3/M, z1.h, z10.h\n"
"fmla z29.h, p3/M, z0.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x11, x5, LSL #1]\n"
+ "ld1h { z10.h }, p2/Z, [x23, x13, LSL #1]\n"
"fmla z27.h, p3/M, z7.h, z11.h\n"
+ "ldr x23, [x14, #0xf8]\n"
"fmla z26.h, p3/M, z6.h, z11.h\n"
"fmla z23.h, p3/M, z4.h, z11.h\n"
"fmla z22.h, p3/M, z3.h, z11.h\n"
"fmla z19.h, p3/M, z1.h, z11.h\n"
"fmla z18.h, p3/M, z0.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x10, x5, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x10, x13, LSL #1]\n"
"fmla z30.h, p3/M, z2.h, z12.h\n"
+ "ldr x10, [x14, #0x100]\n"
"fmla z29.h, p3/M, z1.h, z12.h\n"
"fmla z28.h, p3/M, z0.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x9, x5, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x9, x13, LSL #1]\n"
"fmla z31.h, p3/M, z6.h, z10.h\n"
+ "ldr x9, [x14, #0x108]\n"
"fmla z27.h, p3/M, z3.h, z10.h\n"
"fmla z23.h, p3/M, z0.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x28, x5, LSL #1]\n"
+ "ld1h { z10.h }, p2/Z, [x28, x13, LSL #1]\n"
"fmla z25.h, p3/M, z8.h, z11.h\n"
+ "ldr x28, [x14, #0x110]\n"
"fmla z24.h, p3/M, z7.h, z11.h\n"
"fmla z21.h, p3/M, z5.h, z11.h\n"
"fmla z20.h, p3/M, z4.h, z11.h\n"
"fmla z17.h, p3/M, z2.h, z11.h\n"
"fmla z16.h, p3/M, z1.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x27, x5, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x27, x13, LSL #1]\n"
"fmla z28.h, p3/M, z8.h, z12.h\n"
+ "ldr x27, [x14, #0x118]\n"
"fmla z24.h, p3/M, z5.h, z12.h\n"
"fmla z20.h, p3/M, z2.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x26, x5, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x26, x13, LSL #1]\n"
"fmla z27.h, p3/M, z6.h, z10.h\n"
"fmla z23.h, p3/M, z3.h, z10.h\n"
"fmla z19.h, p3/M, z0.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x25, x5, LSL #1]\n"
+ "ld1h { z10.h }, p2/Z, [x25, x13, LSL #1]\n"
"fmla z22.h, p3/M, z7.h, z11.h\n"
"fmla z21.h, p3/M, z6.h, z11.h\n"
"fmla z23.h, p3/M, z8.h, z11.h\n"
"fmla z19.h, p3/M, z5.h, z11.h\n"
"fmla z18.h, p3/M, z4.h, z11.h\n"
"fmla z17.h, p3/M, z3.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x24, x5, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x24, x13, LSL #1]\n"
"fmla z24.h, p3/M, z8.h, z12.h\n"
"fmla z20.h, p3/M, z5.h, z12.h\n"
"fmla z16.h, p3/M, z2.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x23, x5, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x23, x13, LSL #1]\n"
"fmla z19.h, p3/M, z8.h, z10.h\n"
"fmla z18.h, p3/M, z7.h, z10.h\n"
"fmla z17.h, p3/M, z6.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x8, x5, LSL #1]\n"
+ "ld1h { z10.h }, p2/Z, [x10, x13, LSL #1]\n"
"fmla z22.h, p3/M, z8.h, z11.h\n"
"fmla z21.h, p3/M, z7.h, z11.h\n"
"fmla z20.h, p3/M, z6.h, z11.h\n"
"fmla z18.h, p3/M, z5.h, z11.h\n"
"fmla z17.h, p3/M, z4.h, z11.h\n"
"fmla z16.h, p3/M, z3.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x17, x5, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x9, x13, LSL #1]\n"
"fmla z31.h, p3/M, z4.h, z10.h\n"
- "ldp x8, x17, [x4, #0x0]\n"
+ "ldp x10, x9, [x14, #0x0]\n"
"fmla z18.h, p3/M, z8.h, z12.h\n"
- "ld1h { z9.h }, p1/Z, [x8, x6, LSL #1]\n"
+ "ld1h { z9.h }, p1/Z, [x10, x12, LSL #1]\n"
"fmla z17.h, p3/M, z7.h, z12.h\n"
"fmla z16.h, p3/M, z6.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x16, x5, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x28, x13, LSL #1]\n"
"fmla z30.h, p3/M, z3.h, z10.h\n"
"fmla z27.h, p3/M, z1.h, z10.h\n"
"fmla z26.h, p3/M, z0.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x15, x5, LSL #1]\n"
- "inch x5\n"
+ "ld1h { z10.h }, p2/Z, [x27, x13, LSL #1]\n"
+ "inch x13\n"
"fmla z29.h, p3/M, z5.h, z11.h\n"
- "ldp x16, x15, [x4, #0x10]\n"
- "whilelt p2.h, x5, %x[n_channels]\n"
+ "ldp x28, x27, [x14, #0x10]\n"
+ "whilelt p2.h, x13, %x[n_channels]\n"
"fmla z28.h, p3/M, z4.h, z11.h\n"
- "ld1h { z0.h }, p3/Z, [x3, #1, MUL VL]\n" // Load from weights and bias
+ "ld1h { z0.h }, p3/Z, [x15, #1, MUL VL]\n"
"fmla z25.h, p3/M, z2.h, z11.h\n"
- "ld1h { z2.h }, p3/Z, [x3, #3, MUL VL]\n" // Load from weights and bias
+ "ld1h { z2.h }, p3/Z, [x15, #3, MUL VL]\n"
"fmla z24.h, p3/M, z1.h, z11.h\n"
- "ld1h { z11.h }, p1/Z, [x16, x6, LSL #1]\n"
+ "ld1h { z11.h }, p1/Z, [x28, x12, LSL #1]\n"
"fmla z23.h, p3/M, z7.h, z12.h\n"
- "ld1h { z1.h }, p3/Z, [x3, #2, MUL VL]\n" // Load from weights and bias
+ "ld1h { z1.h }, p3/Z, [x15, #2, MUL VL]\n"
"fmla z22.h, p3/M, z6.h, z12.h\n"
- "ld1h { z6.h }, p3/Z, [x3, #7, MUL VL]\n" // Load from weights and bias
+ "ld1h { z6.h }, p3/Z, [x15, #7, MUL VL]\n"
"fmla z19.h, p3/M, z4.h, z12.h\n"
"fmla z18.h, p3/M, z3.h, z12.h\n"
- "ld1h { z12.h }, p1/Z, [x15, x6, LSL #1]\n"
+ "ld1h { z12.h }, p1/Z, [x27, x12, LSL #1]\n"
"fmla z21.h, p3/M, z8.h, z10.h\n"
- "ld1h { z3.h }, p3/Z, [x3, #4, MUL VL]\n" // Load from weights and bias
+ "ld1h { z3.h }, p3/Z, [x15, #4, MUL VL]\n"
"fmla z20.h, p3/M, z7.h, z10.h\n"
"fmla z17.h, p3/M, z5.h, z10.h\n"
- "ld1h { z5.h }, p3/Z, [x3, #6, MUL VL]\n" // Load from weights and bias
+ "ld1h { z5.h }, p3/Z, [x15, #6, MUL VL]\n"
"fmla z16.h, p3/M, z4.h, z10.h\n"
- "ld1h { z10.h }, p1/Z, [x17, x6, LSL #1]\n"
- "inch x6\n"
+ "ld1h { z10.h }, p1/Z, [x9, x12, LSL #1]\n"
+ "inch x12\n"
"fmax z31.h, p3/M, z31.h, z15.h\n"
- "ld1h { z4.h }, p3/Z, [x3, #5, MUL VL]\n" // Load from weights and bias
- "addvl x3, x3, #16\n"
+ "ld1h { z4.h }, p3/Z, [x15, #5, MUL VL]\n"
+ "addvl x15, x15, #16\n"
"fmax z30.h, p3/M, z30.h, z15.h\n"
- "ld1h { z7.h }, p3/Z, [x3, #-8, MUL VL]\n" // Load from weights and bias
- "cmp x6, %x[n_channels]\n"
+ "ld1h { z7.h }, p3/Z, [x15, #-8, MUL VL]\n"
+ "cmp x12, %x[n_channels]\n"
"fmax z29.h, p3/M, z29.h, z15.h\n"
- "ld1h { z8.h }, p3/Z, [x3, #-7, MUL VL]\n" // Load from weights and bias
- "addvl x3, x3, #-6\n"
+ "ld1h { z8.h }, p3/Z, [x15, #-7, MUL VL]\n"
+ "addvl x15, x15, #-6\n"
"fmax z28.h, p3/M, z28.h, z15.h\n"
"fmax z27.h, p3/M, z27.h, z15.h\n"
"fmin z31.h, p3/M, z31.h, z14.h\n"
- "st1h { z31.h }, p0, [x22, x7, LSL #1]\n"
- "mov z31.d, z13.d\n"
+ "st1h { z31.h }, p0, [x22, x11, LSL #1]\n"
"fmin z30.h, p3/M, z30.h, z14.h\n"
- "ldr x22, [x2, #0x20]\n"
"fmin z29.h, p3/M, z29.h, z14.h\n"
- "st1h { z30.h }, p0, [x21, x7, LSL #1]\n"
- "mov z30.d, z13.d\n"
+ "ldr x22, [x16, #0x20]\n"
"fmin z28.h, p3/M, z28.h, z14.h\n"
- "st1h { z29.h }, p0, [x20, x7, LSL #1]\n"
- "mov z29.d, z13.d\n"
- "ldr x21, [x2, #0x28]\n"
+ "st1h { z30.h }, p0, [x21, x11, LSL #1]\n"
"fmin z27.h, p3/M, z27.h, z14.h\n"
- "ldr x20, [x2, #0x30]\n"
"fmax z26.h, p3/M, z26.h, z15.h\n"
- "st1h { z28.h }, p0, [x19, x7, LSL #1]\n"
- "mov z28.d, z13.d\n"
- "ldr x19, [x2, #0x38]\n"
+ "st1h { z29.h }, p0, [x20, x11, LSL #1]\n"
"fmax z25.h, p3/M, z25.h, z15.h\n"
- "st1h { z27.h }, p0, [x22, x7, LSL #1]\n"
- "mov z27.d, z13.d\n"
- "ldr x22, [x2, #0x40]\n"
- "fmin z26.h, p3/M, z26.h, z14.h\n"
- "st1h { z26.h }, p0, [x21, x7, LSL #1]\n"
- "mov z26.d, z13.d\n"
- "fmin z25.h, p3/M, z25.h, z14.h\n"
- "ldr x21, [x2, #0x48]\n"
+ "st1h { z28.h }, p0, [x19, x11, LSL #1]\n"
"fmax z24.h, p3/M, z24.h, z15.h\n"
- "st1h { z25.h }, p0, [x20, x7, LSL #1]\n"
- "mov z25.d, z13.d\n"
+ "ldr x21, [x16, #0x28]\n"
"fmax z23.h, p3/M, z23.h, z15.h\n"
- "ldr x20, [x2, #0x50]\n"
+ "st1h { z27.h }, p0, [x22, x11, LSL #1]\n"
+ "fmin z26.h, p3/M, z26.h, z14.h\n"
+ "ldr x20, [x16, #0x30]\n"
+ "fmin z25.h, p3/M, z25.h, z14.h\n"
+ "ldr x19, [x16, #0x38]\n"
"fmin z24.h, p3/M, z24.h, z14.h\n"
- "st1h { z24.h }, p0, [x19, x7, LSL #1]\n"
- "mov z24.d, z13.d\n"
+ "ldr x22, [x16, #0x40]\n"
"fmin z23.h, p3/M, z23.h, z14.h\n"
- "ldr x19, [x2, #0x58]\n"
+ "st1h { z26.h }, p0, [x21, x11, LSL #1]\n"
"fmax z22.h, p3/M, z22.h, z15.h\n"
- "st1h { z23.h }, p0, [x22, x7, LSL #1]\n"
- "mov z23.d, z13.d\n"
+ "st1h { z25.h }, p0, [x20, x11, LSL #1]\n"
"fmax z21.h, p3/M, z21.h, z15.h\n"
- "ldr x22, [x2, #0x60]\n"
- "fmin z22.h, p3/M, z22.h, z14.h\n"
- "st1h { z22.h }, p0, [x21, x7, LSL #1]\n"
- "mov z22.d, z13.d\n"
- "fmin z21.h, p3/M, z21.h, z14.h\n"
- "ldr x21, [x2, #0x68]\n"
+ "st1h { z24.h }, p0, [x19, x11, LSL #1]\n"
"fmax z20.h, p3/M, z20.h, z15.h\n"
- "st1h { z21.h }, p0, [x20, x7, LSL #1]\n"
- "mov z21.d, z13.d\n"
+ "st1h { z23.h }, p0, [x22, x11, LSL #1]\n"
"fmax z19.h, p3/M, z19.h, z15.h\n"
- "ldr x20, [x2, #0x70]\n"
+ "ldr x21, [x16, #0x48]\n"
+ "fmin z22.h, p3/M, z22.h, z14.h\n"
+ "ldr x20, [x16, #0x50]\n"
+ "fmin z21.h, p3/M, z21.h, z14.h\n"
+ "ldr x19, [x16, #0x58]\n"
"fmin z20.h, p3/M, z20.h, z14.h\n"
- "st1h { z20.h }, p0, [x19, x7, LSL #1]\n"
- "mov z20.d, z13.d\n"
+ "ldr x22, [x16, #0x60]\n"
"fmin z19.h, p3/M, z19.h, z14.h\n"
- "ldr x19, [x2, #0x78]\n"
+ "st1h { z22.h }, p0, [x21, x11, LSL #1]\n"
"fmax z18.h, p3/M, z18.h, z15.h\n"
- "st1h { z19.h }, p0, [x22, x7, LSL #1]\n"
- "mov z19.d, z13.d\n"
+ "st1h { z21.h }, p0, [x20, x11, LSL #1]\n"
"fmax z17.h, p3/M, z17.h, z15.h\n"
- "fmin z18.h, p3/M, z18.h, z14.h\n"
- "st1h { z18.h }, p0, [x21, x7, LSL #1]\n"
- "mov z18.d, z13.d\n"
- "fmin z17.h, p3/M, z17.h, z14.h\n"
- "st1h { z17.h }, p0, [x20, x7, LSL #1]\n"
- "mov z17.d, z13.d\n"
+ "st1h { z20.h }, p0, [x19, x11, LSL #1]\n"
"fmax z16.h, p3/M, z16.h, z15.h\n"
+ "st1h { z19.h }, p0, [x22, x11, LSL #1]\n"
+ "ldr x21, [x16, #0x68]\n"
+ "fmin z18.h, p3/M, z18.h, z14.h\n"
+ "ldr x20, [x16, #0x70]\n"
+ "fmin z17.h, p3/M, z17.h, z14.h\n"
+ "ldr x19, [x16, #0x78]\n"
"fmin z16.h, p3/M, z16.h, z14.h\n"
- "st1h { z16.h }, p0, [x19, x7, LSL #1]\n"
- "mov z16.d, z13.d\n"
+ "st1h { z18.h }, p0, [x21, x11, LSL #1]\n"
+ "st1h { z17.h }, p0, [x20, x11, LSL #1]\n"
+ "st1h { z16.h }, p0, [x19, x11, LSL #1]\n"
"blt 1b\n"
"2:" // Channel tail
- "fmla z31.h, p3/M, z8.h, z9.h\n"
- "ldr x14, [x4, #0x20]\n"
- "inch x7\n"
- "fmla z30.h, p3/M, z7.h, z9.h\n"
- "ldr x13, [x4, #0x28]\n"
+ "movprfx z31, z13\n fmla z31.h, p3/M, z8.h, z9.h\n"
+ "ldr x26, [x14, #0x20]\n"
+ "inch x11\n"
+ "movprfx z30, z13\n fmla z30.h, p3/M, z7.h, z9.h\n"
+ "ldr x25, [x14, #0x28]\n"
"mov p0.b, p2.b\n"
- "fmla z29.h, p3/M, z6.h, z9.h\n"
- "ldr x12, [x4, #0x30]\n"
- "fmla z27.h, p3/M, z5.h, z9.h\n"
- "ldr x11, [x4, #0x38]\n"
- "fmla z26.h, p3/M, z4.h, z9.h\n"
- "ldr x10, [x4, #0x40]\n"
- "fmla z25.h, p3/M, z3.h, z9.h\n"
- "ldr x9, [x4, #0x48]\n"
- "fmla z23.h, p3/M, z2.h, z9.h\n"
- "ldr x28, [x4, #0x50]\n"
- "fmla z22.h, p3/M, z1.h, z9.h\n"
- "ldr x27, [x4, #0x58]\n"
- "fmla z21.h, p3/M, z0.h, z9.h\n"
- "ld1h { z9.h }, p2/Z, [x12, x5, LSL #1]\n"
+ "movprfx z29, z13\n fmla z29.h, p3/M, z6.h, z9.h\n"
+ "ldr x24, [x14, #0x30]\n"
+ "movprfx z27, z13\n fmla z27.h, p3/M, z5.h, z9.h\n"
+ "ldr x23, [x14, #0x38]\n"
+ "movprfx z26, z13\n fmla z26.h, p3/M, z4.h, z9.h\n"
+ "ldr x10, [x14, #0x40]\n"
+ "movprfx z25, z13\n fmla z25.h, p3/M, z3.h, z9.h\n"
+ "ldr x9, [x14, #0x48]\n"
+ "movprfx z23, z13\n fmla z23.h, p3/M, z2.h, z9.h\n"
+ "ldr x28, [x14, #0x50]\n"
+ "movprfx z22, z13\n fmla z22.h, p3/M, z1.h, z9.h\n"
+ "ldr x27, [x14, #0x58]\n"
+ "movprfx z21, z13\n fmla z21.h, p3/M, z0.h, z9.h\n"
+ "ld1h { z9.h }, p2/Z, [x24, x13, LSL #1]\n"
"fmla z31.h, p3/M, z0.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x14, x5, LSL #1]\n"
- "fmla z28.h, p3/M, z2.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x13, x5, LSL #1]\n"
+ "ld1h { z10.h }, p2/Z, [x26, x13, LSL #1]\n"
+ "movprfx z28, z13\n fmla z28.h, p3/M, z2.h, z11.h\n"
+ "ld1h { z11.h }, p2/Z, [x25, x13, LSL #1]\n"
"fmla z30.h, p3/M, z8.h, z12.h\n"
- "ldr x26, [x4, #0x60]\n"
+ "ldr x26, [x14, #0x60]\n"
"fmla z29.h, p3/M, z7.h, z12.h\n"
- "ldr x25, [x4, #0x68]\n"
+ "ldr x25, [x14, #0x68]\n"
"fmla z26.h, p3/M, z5.h, z12.h\n"
- "ldr x24, [x4, #0x70]\n"
+ "ldr x24, [x14, #0x70]\n"
"fmla z28.h, p3/M, z6.h, z12.h\n"
- "ldr x23, [x4, #0x78]\n"
+ "ldr x22, [x16, #0x0]\n"
"fmla z25.h, p3/M, z4.h, z12.h\n"
- "ldr x8, [x4, #0x80]\n"
- "fmla z24.h, p3/M, z3.h, z12.h\n"
- "ldr x17, [x4, #0x88]\n"
+ "ldr x21, [x16, #0x8]\n"
+ "movprfx z24, z13\n fmla z24.h, p3/M, z3.h, z12.h\n"
+ "ldr x20, [x16, #0x10]\n"
"fmla z22.h, p3/M, z2.h, z12.h\n"
- "ldr x16, [x4, #0x90]\n"
+ "ldr x19, [x16, #0x18]\n"
"fmla z21.h, p3/M, z1.h, z12.h\n"
- "ldr x15, [x4, #0x98]\n"
- "fmla z20.h, p3/M, z0.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x11, x5, LSL #1]\n"
- "fmla z19.h, p3/M, z6.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x9, x5, LSL #1]\n"
- "fmla z16.h, p3/M, z8.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x10, x5, LSL #1]\n"
+ "movprfx z20, z13\n fmla z20.h, p3/M, z0.h, z12.h\n"
+ "ld1h { z12.h }, p2/Z, [x23, x13, LSL #1]\n"
+ "movprfx z19, z13\n fmla z19.h, p3/M, z6.h, z10.h\n"
+ "ld1h { z10.h }, p2/Z, [x9, x13, LSL #1]\n"
+ "movprfx z16, z13\n fmla z16.h, p3/M, z8.h, z11.h\n"
+ "ld1h { z11.h }, p2/Z, [x10, x13, LSL #1]\n"
"fmla z27.h, p3/M, z8.h, z9.h\n"
- "ldr x14, [x4, #0xa0]\n"
+ "ldr x23, [x14, #0x78]\n"
"fmla z26.h, p3/M, z7.h, z9.h\n"
- "ldr x13, [x4, #0xa8]\n"
+ "ldr x10, [x14, #0x80]\n"
"fmla z25.h, p3/M, z6.h, z9.h\n"
- "ldr x12, [x4, #0xb0]\n"
+ "ldr x9, [x14, #0x88]\n"
"fmla z23.h, p3/M, z5.h, z9.h\n"
- "ldr x11, [x4, #0xb8]\n"
"fmla z22.h, p3/M, z4.h, z9.h\n"
- "ldr x10, [x4, #0xc0]\n"
"fmla z21.h, p3/M, z3.h, z9.h\n"
- "ldr x9, [x4, #0xc8]\n"
"fmla z19.h, p3/M, z2.h, z9.h\n"
- "ldr x22, [x2, #0x0]\n"
- "fmla z18.h, p3/M, z1.h, z9.h\n"
- "ldr x21, [x2, #0x8]\n"
- "fmla z17.h, p3/M, z0.h, z9.h\n"
- "ld1h { z9.h }, p2/Z, [x28, x5, LSL #1]\n"
+ "movprfx z18, z13\n fmla z18.h, p3/M, z1.h, z9.h\n"
+ "movprfx z17, z13\n fmla z17.h, p3/M, z0.h, z9.h\n"
+ "ld1h { z9.h }, p2/Z, [x28, x13, LSL #1]\n"
"fmla z31.h, p3/M, z1.h, z12.h\n"
- "ldr x28, [x4, #0xd0]\n"
+ "ldr x28, [x14, #0x90]\n"
"fmla z30.h, p3/M, z0.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x27, x5, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x27, x13, LSL #1]\n"
"fmla z29.h, p3/M, z2.h, z11.h\n"
- "ldr x27, [x4, #0xd8]\n"
+ "ldr x27, [x14, #0x98]\n"
"fmla z28.h, p3/M, z1.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x26, x5, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x26, x13, LSL #1]\n"
"fmla z26.h, p3/M, z8.h, z10.h\n"
- "ldr x26, [x4, #0xe0]\n"
+ "ldr x26, [x14, #0xa0]\n"
"fmla z25.h, p3/M, z7.h, z10.h\n"
- "ldr x20, [x2, #0x10]\n"
"fmla z24.h, p3/M, z6.h, z10.h\n"
- "ldr x19, [x2, #0x18]\n"
"fmla z22.h, p3/M, z5.h, z10.h\n"
"fmla z21.h, p3/M, z4.h, z10.h\n"
"fmla z20.h, p3/M, z3.h, z10.h\n"
"fmla z18.h, p3/M, z2.h, z10.h\n"
"fmla z17.h, p3/M, z1.h, z10.h\n"
"fmla z16.h, p3/M, z0.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x25, x5, LSL #1]\n"
+ "ld1h { z10.h }, p2/Z, [x25, x13, LSL #1]\n"
"fmla z31.h, p3/M, z3.h, z9.h\n"
- "ldr x25, [x4, #0xe8]\n"
+ "ldr x25, [x14, #0xa8]\n"
"fmla z27.h, p3/M, z0.h, z9.h\n"
"fmla z28.h, p3/M, z5.h, z12.h\n"
"fmla z24.h, p3/M, z2.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x23, x5, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x23, x13, LSL #1]\n"
"fmla z23.h, p3/M, z6.h, z11.h\n"
- "ldr x23, [x4, #0xf8]\n"
+ "ldr x23, [x14, #0xb8]\n"
"fmla z19.h, p3/M, z3.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x24, x5, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x24, x13, LSL #1]\n"
"fmla z31.h, p3/M, z5.h, z10.h\n"
- "ldr x24, [x4, #0xf0]\n"
+ "ldr x24, [x14, #0xb0]\n"
"fmla z30.h, p3/M, z4.h, z10.h\n"
"fmla z29.h, p3/M, z3.h, z10.h\n"
"fmla z27.h, p3/M, z2.h, z10.h\n"
"fmla z26.h, p3/M, z1.h, z10.h\n"
"fmla z25.h, p3/M, z0.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x17, x5, LSL #1]\n"
+ "ld1h { z10.h }, p2/Z, [x9, x13, LSL #1]\n"
"fmla z20.h, p3/M, z8.h, z11.h\n"
- "ldr x17, [x4, #0x108]\n"
+ "ldr x9, [x14, #0xc8]\n"
"fmla z16.h, p3/M, z5.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x8, x5, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x10, x13, LSL #1]\n"
"fmla z30.h, p3/M, z5.h, z12.h\n"
- "ldr x8, [x4, #0x100]\n"
+ "ldr x10, [x14, #0xc0]\n"
"fmla z29.h, p3/M, z4.h, z12.h\n"
"fmla z28.h, p3/M, z3.h, z12.h\n"
"fmla z26.h, p3/M, z2.h, z12.h\n"
"fmla z25.h, p3/M, z1.h, z12.h\n"
"fmla z24.h, p3/M, z0.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x15, x5, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x27, x13, LSL #1]\n"
"fmla z19.h, p3/M, z7.h, z11.h\n"
- "ldr x15, [x4, #0x118]\n"
+ "ldr x27, [x14, #0xd8]\n"
"fmla z18.h, p3/M, z6.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x16, x5, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x28, x13, LSL #1]\n"
"fmla z31.h, p3/M, z7.h, z10.h\n"
- "ldr x16, [x4, #0x110]\n"
+ "ldr x28, [x14, #0xd0]\n"
"fmla z30.h, p3/M, z6.h, z10.h\n"
"fmla z27.h, p3/M, z4.h, z10.h\n"
"fmla z26.h, p3/M, z3.h, z10.h\n"
"fmla z23.h, p3/M, z1.h, z10.h\n"
"fmla z22.h, p3/M, z0.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x14, x5, LSL #1]\n"
+ "ld1h { z10.h }, p2/Z, [x26, x13, LSL #1]\n"
"fmla z17.h, p3/M, z8.h, z11.h\n"
+ "ldr x26, [x14, #0xe0]\n"
"fmla z16.h, p3/M, z7.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x13, x5, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x25, x13, LSL #1]\n"
"fmla z29.h, p3/M, z8.h, z12.h\n"
+ "ldr x25, [x14, #0xe8]\n"
"fmla z28.h, p3/M, z7.h, z12.h\n"
"fmla z25.h, p3/M, z5.h, z12.h\n"
"fmla z24.h, p3/M, z4.h, z12.h\n"
"fmla z21.h, p3/M, z2.h, z12.h\n"
"fmla z20.h, p3/M, z1.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x12, x5, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x24, x13, LSL #1]\n"
"fmla z31.h, p3/M, z2.h, z10.h\n"
+ "ldr x24, [x14, #0xf0]\n"
"fmla z30.h, p3/M, z1.h, z10.h\n"
"fmla z29.h, p3/M, z0.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x11, x5, LSL #1]\n"
+ "ld1h { z10.h }, p2/Z, [x23, x13, LSL #1]\n"
"fmla z27.h, p3/M, z7.h, z11.h\n"
+ "ldr x23, [x14, #0xf8]\n"
"fmla z26.h, p3/M, z6.h, z11.h\n"
"fmla z23.h, p3/M, z4.h, z11.h\n"
"fmla z22.h, p3/M, z3.h, z11.h\n"
"fmla z19.h, p3/M, z1.h, z11.h\n"
"fmla z18.h, p3/M, z0.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x10, x5, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x10, x13, LSL #1]\n"
"fmla z30.h, p3/M, z2.h, z12.h\n"
+ "ldr x10, [x14, #0x100]\n"
"fmla z29.h, p3/M, z1.h, z12.h\n"
"fmla z28.h, p3/M, z0.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x9, x5, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x9, x13, LSL #1]\n"
"fmla z31.h, p3/M, z6.h, z10.h\n"
+ "ldr x9, [x14, #0x108]\n"
"fmla z27.h, p3/M, z3.h, z10.h\n"
"fmla z23.h, p3/M, z0.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x28, x5, LSL #1]\n"
+ "ld1h { z10.h }, p2/Z, [x28, x13, LSL #1]\n"
"fmla z25.h, p3/M, z8.h, z11.h\n"
+ "ldr x28, [x14, #0x110]\n"
"fmla z24.h, p3/M, z7.h, z11.h\n"
"fmla z21.h, p3/M, z5.h, z11.h\n"
"fmla z20.h, p3/M, z4.h, z11.h\n"
"fmla z17.h, p3/M, z2.h, z11.h\n"
"fmla z16.h, p3/M, z1.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x27, x5, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x27, x13, LSL #1]\n"
"fmla z28.h, p3/M, z8.h, z12.h\n"
+ "ldr x27, [x14, #0x118]\n"
"fmla z24.h, p3/M, z5.h, z12.h\n"
"fmla z20.h, p3/M, z2.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x26, x5, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x26, x13, LSL #1]\n"
"fmla z27.h, p3/M, z6.h, z10.h\n"
"fmla z23.h, p3/M, z3.h, z10.h\n"
"fmla z19.h, p3/M, z0.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x25, x5, LSL #1]\n"
+ "ld1h { z10.h }, p2/Z, [x25, x13, LSL #1]\n"
"fmla z22.h, p3/M, z7.h, z11.h\n"
"fmla z21.h, p3/M, z6.h, z11.h\n"
"fmla z23.h, p3/M, z8.h, z11.h\n"
"fmla z19.h, p3/M, z5.h, z11.h\n"
"fmla z18.h, p3/M, z4.h, z11.h\n"
"fmla z17.h, p3/M, z3.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x24, x5, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x24, x13, LSL #1]\n"
"fmla z24.h, p3/M, z8.h, z12.h\n"
"fmla z20.h, p3/M, z5.h, z12.h\n"
"fmla z16.h, p3/M, z2.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x23, x5, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x23, x13, LSL #1]\n"
"fmla z19.h, p3/M, z8.h, z10.h\n"
"fmla z18.h, p3/M, z7.h, z10.h\n"
"fmla z17.h, p3/M, z6.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x8, x5, LSL #1]\n"
+ "ld1h { z10.h }, p2/Z, [x10, x13, LSL #1]\n"
"fmla z22.h, p3/M, z8.h, z11.h\n"
"fmla z21.h, p3/M, z7.h, z11.h\n"
"fmla z20.h, p3/M, z6.h, z11.h\n"
"fmla z18.h, p3/M, z5.h, z11.h\n"
"fmla z17.h, p3/M, z4.h, z11.h\n"
"fmla z16.h, p3/M, z3.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x17, x5, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x9, x13, LSL #1]\n"
"fmla z31.h, p3/M, z4.h, z10.h\n"
"fmla z18.h, p3/M, z8.h, z12.h\n"
"fmla z17.h, p3/M, z7.h, z12.h\n"
"fmla z16.h, p3/M, z6.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x16, x5, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x28, x13, LSL #1]\n"
"fmla z30.h, p3/M, z3.h, z10.h\n"
"fmla z27.h, p3/M, z1.h, z10.h\n"
"fmla z26.h, p3/M, z0.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x15, x5, LSL #1]\n"
+ "ld1h { z10.h }, p2/Z, [x27, x13, LSL #1]\n"
"fmla z29.h, p3/M, z5.h, z11.h\n"
"fmla z28.h, p3/M, z4.h, z11.h\n"
"fmla z25.h, p3/M, z2.h, z11.h\n"
@@ -679,68 +647,68 @@
"fmax z29.h, p3/M, z29.h, z15.h\n"
"fmax z28.h, p3/M, z28.h, z15.h\n"
"fmin z31.h, p3/M, z31.h, z14.h\n"
- "st1h { z31.h }, p0, [x22, x7, LSL #1]\n"
+ "st1h { z31.h }, p0, [x22, x11, LSL #1]\n"
"fmin z30.h, p3/M, z30.h, z14.h\n"
"fmin z29.h, p3/M, z29.h, z14.h\n"
- "ldr x22, [x2, #0x20]\n"
+ "ldr x22, [x16, #0x20]\n"
"fmin z28.h, p3/M, z28.h, z14.h\n"
- "st1h { z30.h }, p0, [x21, x7, LSL #1]\n"
+ "st1h { z30.h }, p0, [x21, x11, LSL #1]\n"
"fmax z27.h, p3/M, z27.h, z15.h\n"
"fmax z26.h, p3/M, z26.h, z15.h\n"
- "st1h { z29.h }, p0, [x20, x7, LSL #1]\n"
+ "st1h { z29.h }, p0, [x20, x11, LSL #1]\n"
"fmax z25.h, p3/M, z25.h, z15.h\n"
- "st1h { z28.h }, p0, [x19, x7, LSL #1]\n"
+ "st1h { z28.h }, p0, [x19, x11, LSL #1]\n"
"fmax z24.h, p3/M, z24.h, z15.h\n"
- "ldr x21, [x2, #0x28]\n"
+ "ldr x21, [x16, #0x28]\n"
"fmax z23.h, p3/M, z23.h, z15.h\n"
- "ldr x20, [x2, #0x30]\n"
+ "ldr x20, [x16, #0x30]\n"
"fmin z27.h, p3/M, z27.h, z14.h\n"
- "ldr x19, [x2, #0x38]\n"
+ "ldr x19, [x16, #0x38]\n"
"fmin z26.h, p3/M, z26.h, z14.h\n"
- "st1h { z27.h }, p0, [x22, x7, LSL #1]\n"
+ "st1h { z27.h }, p0, [x22, x11, LSL #1]\n"
"fmin z25.h, p3/M, z25.h, z14.h\n"
"fmin z24.h, p3/M, z24.h, z14.h\n"
- "st1h { z26.h }, p0, [x21, x7, LSL #1]\n"
+ "st1h { z26.h }, p0, [x21, x11, LSL #1]\n"
"fmin z23.h, p3/M, z23.h, z14.h\n"
- "ldr x22, [x2, #0x40]\n"
+ "ldr x22, [x16, #0x40]\n"
"fmax z22.h, p3/M, z22.h, z15.h\n"
- "ldr x21, [x2, #0x48]\n"
+ "ldr x21, [x16, #0x48]\n"
"fmax z21.h, p3/M, z21.h, z15.h\n"
- "st1h { z25.h }, p0, [x20, x7, LSL #1]\n"
+ "st1h { z25.h }, p0, [x20, x11, LSL #1]\n"
"fmax z20.h, p3/M, z20.h, z15.h\n"
- "st1h { z24.h }, p0, [x19, x7, LSL #1]\n"
+ "st1h { z24.h }, p0, [x19, x11, LSL #1]\n"
"fmax z19.h, p3/M, z19.h, z15.h\n"
- "st1h { z23.h }, p0, [x22, x7, LSL #1]\n"
+ "st1h { z23.h }, p0, [x22, x11, LSL #1]\n"
"fmin z22.h, p3/M, z22.h, z14.h\n"
- "ldr x20, [x2, #0x50]\n"
+ "ldr x20, [x16, #0x50]\n"
"fmin z21.h, p3/M, z21.h, z14.h\n"
- "ldr x19, [x2, #0x58]\n"
+ "ldr x19, [x16, #0x58]\n"
"fmin z20.h, p3/M, z20.h, z14.h\n"
- "ldr x22, [x2, #0x60]\n"
+ "ldr x22, [x16, #0x60]\n"
"fmin z19.h, p3/M, z19.h, z14.h\n"
- "st1h { z22.h }, p0, [x21, x7, LSL #1]\n"
+ "st1h { z22.h }, p0, [x21, x11, LSL #1]\n"
"fmax z18.h, p3/M, z18.h, z15.h\n"
- "st1h { z21.h }, p0, [x20, x7, LSL #1]\n"
+ "st1h { z21.h }, p0, [x20, x11, LSL #1]\n"
"fmax z17.h, p3/M, z17.h, z15.h\n"
- "st1h { z20.h }, p0, [x19, x7, LSL #1]\n"
+ "st1h { z20.h }, p0, [x19, x11, LSL #1]\n"
"fmax z16.h, p3/M, z16.h, z15.h\n"
- "st1h { z19.h }, p0, [x22, x7, LSL #1]\n"
- "ldr x21, [x2, #0x68]\n"
+ "st1h { z19.h }, p0, [x22, x11, LSL #1]\n"
+ "ldr x21, [x16, #0x68]\n"
"fmin z18.h, p3/M, z18.h, z14.h\n"
- "ldr x20, [x2, #0x70]\n"
+ "ldr x20, [x16, #0x70]\n"
"fmin z17.h, p3/M, z17.h, z14.h\n"
- "ldr x19, [x2, #0x78]\n"
+ "ldr x19, [x16, #0x78]\n"
"fmin z16.h, p3/M, z16.h, z14.h\n"
- "st1h { z18.h }, p0, [x21, x7, LSL #1]\n"
- "st1h { z17.h }, p0, [x20, x7, LSL #1]\n"
- "st1h { z16.h }, p0, [x19, x7, LSL #1]\n"
+ "st1h { z18.h }, p0, [x21, x11, LSL #1]\n"
+ "st1h { z17.h }, p0, [x20, x11, LSL #1]\n"
+ "st1h { z16.h }, p0, [x19, x11, LSL #1]\n"
:
: [n_channels] "r" ((unsigned long) n_channels), [offsetof_Args_inptrs] "I" (offsetof(Args, inptrs)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_outptrs] "I" (offsetof(Args, outptrs)), [offsetof_args_params] "I" (offsetof(Args, params)), [params_struct] "r" (¶ms_struct)
- : "cc", "memory", "p0", "p1", "p2", "p3", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+ : "cc", "memory", "p0", "p1", "p2", "p3", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
);
}
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
+#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp
index a6a4afb..5968309 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp
@@ -28,7 +28,7 @@
#pragma once
-#if defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
+#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
namespace arm_conv {
namespace depthwise {
@@ -36,15 +36,17 @@
void sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16);
void sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16);
-struct sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst
+class sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst : public IDepthwiseDepthfirstStrategy
{
- typedef __fp16 bias_type;
- typedef __fp16 input_type;
- typedef __fp16 weight_type;
- typedef __fp16 return_type;
-
+ private:
typedef void (*indirect_kern_type)(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16);
+ indirect_kern_type m_indirect_kernel = sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl;
+
typedef void (*direct_kern_type)(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16);
+ direct_kern_type m_direct_kernel = sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl;
+
+ public:
+ typedef __fp16 return_type;
constexpr static arm_gemm::VLType vl_type = arm_gemm::VLType::SVE;
@@ -60,13 +62,60 @@
constexpr static unsigned int input_rows = 5;
constexpr static unsigned int input_cols = 5;
- indirect_kern_type indirect_kernel = sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl;
- direct_kern_type direct_kernel = sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl;
-
sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst(const CPUInfo *) {}
+
+ arm_gemm::VLType get_vl_type(void) const override { return vl_type; }
+
+ unsigned int get_kernel_rows(void) const override { return kernel_rows; }
+ unsigned int get_kernel_cols(void) const override { return kernel_cols; }
+
+ unsigned int get_stride_rows(void) const override { return stride_rows; }
+ unsigned int get_stride_cols(void) const override { return stride_cols; }
+
+ unsigned int get_output_rows(void) const override { return output_rows; }
+ unsigned int get_output_cols(void) const override { return output_cols; }
+
+ unsigned int get_input_rows(void) const override { return input_rows; }
+ unsigned int get_input_cols(void) const override { return input_cols; }
+
+ void indirect_kernel(
+ const void *const *const input_ptrs,
+ void *const *const outptrs,
+ const void *params,
+ unsigned int n_channels,
+ const void *activation_min,
+ const void *activation_max
+ ) const override
+ {
+ m_indirect_kernel(
+ reinterpret_cast<const __fp16 *const *>(input_ptrs),
+ reinterpret_cast<__fp16 *const *>(outptrs),
+ params, n_channels,
+ *static_cast<const __fp16 *>(activation_min),
+ *static_cast<const __fp16 *>(activation_max)
+ );
+ }
+
+ void direct_kernel(
+ const unsigned int n_tile_rows, const unsigned int n_tile_cols,
+ const void *inptr, int64_t ld_input_row, int64_t ld_input_col,
+ void *outptr, int64_t ld_output_row, int64_t ld_output_col,
+ const void *params, unsigned int n_channels,
+ const void *activation_min, const void *activation_max
+ ) const override
+ {
+ m_direct_kernel(
+ n_tile_rows, n_tile_cols,
+ static_cast<const __fp16 *>(inptr), ld_input_row, ld_input_col,
+ static_cast<__fp16 *>(outptr), ld_output_row, ld_output_col,
+ params, n_channels,
+ *static_cast<const __fp16 *>(activation_min),
+ *static_cast<const __fp16 *>(activation_max)
+ );
+ }
};
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
+#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp
index 2238bf0..d20e991 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp
@@ -25,7 +25,7 @@
#include <cstddef>
#include <cstdint>
-#if defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
+#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
namespace arm_conv {
namespace depthwise {
@@ -113,225 +113,217 @@
"add x9, x21, x22, LSL #1\n"
"ld1rh { z18.h }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n"
"add x28, x9, x22, LSL #1\n"
- "ld1h { z17.h }, p3/Z, [x16]\n" // Load from weights and bias
- "mov z31.d, z17.d\n"
- "ld1h { z0.h }, p3/Z, [x16, #1, MUL VL]\n" // Load from weights and bias
+ "ld1h { z17.h }, p3/Z, [x16]\n"
"add x27, x28, x22, LSL #1\n"
- "mov z30.d, z17.d\n"
- "ld1h { z1.h }, p3/Z, [x16, #2, MUL VL]\n" // Load from weights and bias
+ "ld1h { z0.h }, p3/Z, [x16, #1, MUL VL]\n"
"add x26, x27, x22, LSL #1\n"
- "mov z29.d, z17.d\n"
- "ld1h { z2.h }, p3/Z, [x16, #3, MUL VL]\n" // Load from weights and bias
+ "ld1h { z1.h }, p3/Z, [x16, #2, MUL VL]\n"
"add x25, x13, x13\n"
- "mov z28.d, z17.d\n"
- "ld1h { z3.h }, p3/Z, [x16, #4, MUL VL]\n" // Load from weights and bias
+ "ld1h { z2.h }, p3/Z, [x16, #3, MUL VL]\n"
"add x24, x25, x13\n"
- "ld1h { z4.h }, p3/Z, [x16, #5, MUL VL]\n" // Load from weights and bias
+ "ld1h { z3.h }, p3/Z, [x16, #4, MUL VL]\n"
"add x23, x24, x13\n"
- "ld1h { z5.h }, p3/Z, [x16, #6, MUL VL]\n" // Load from weights and bias
+ "ld1h { z4.h }, p3/Z, [x16, #5, MUL VL]\n"
"mul x19, x7, x20\n" // offset = tile_i * ld_output_row
- "ld1h { z6.h }, p3/Z, [x16, #7, MUL VL]\n" // Load from weights and bias
+ "ld1h { z5.h }, p3/Z, [x16, #6, MUL VL]\n"
"madd x19, x8, x11, x19\n" // offset += tile_j * ld_output_col
- "whilelt p2.h, XZR, %x[n_channels]\n"
- "ld1h { z9.h }, p2/Z, [x28, x25, LSL #1]\n" // Load input point (2, 2)
- "ld1h { z10.h }, p2/Z, [x21]\n" // Load input point (0, 0)
+ "ld1h { z6.h }, p3/Z, [x16, #7, MUL VL]\n"
"mul x19, x19, x17\n" // offset *= output_tile_size
- "ld1h { z11.h }, p2/Z, [x21, x13, LSL #1]\n" // Load input point (0, 1)
+ "whilelt p2.h, XZR, %x[n_channels]\n"
+ "ld1h { z9.h }, p2/Z, [x28, x25, LSL #1]\n"
+ "ld1h { z10.h }, p2/Z, [x21]\n"
"add x10, x10, x19, LSL #1\n" // outptrs[0] += offset * sizeof(__fp16)
- "ld1h { z12.h }, p2/Z, [x21, x24, LSL #1]\n" // Load input point (0, 3)
+ "ld1h { z11.h }, p2/Z, [x21, x13, LSL #1]\n"
"add x22, x10, x20, LSL #1\n"
- "ld1h { z13.h }, p2/Z, [x21, x23, LSL #1]\n" // Load input point (0, 4)
+ "ld1h { z12.h }, p2/Z, [x21, x24, LSL #1]\n"
"addvl x16, x16, #16\n"
- "ld1h { z14.h }, p2/Z, [x9]\n" // Load input point (1, 0)
+ "ld1h { z13.h }, p2/Z, [x21, x23, LSL #1]\n"
"cmp x14, %x[n_channels]\n"
- "ld1h { z7.h }, p3/Z, [x16, #-8, MUL VL]\n" // Load from weights and bias
- "ld1h { z8.h }, p3/Z, [x16, #-7, MUL VL]\n" // Load from weights and bias
+ "ld1h { z7.h }, p3/Z, [x16, #-8, MUL VL]\n"
+ "ld1h { z8.h }, p3/Z, [x16, #-7, MUL VL]\n"
"addvl x16, x16, #-6\n"
- "ld1h { z15.h }, p2/Z, [x9, x13, LSL #1]\n" // Load input point (1, 1)
- "ld1h { z16.h }, p2/Z, [x21, x25, LSL #1]\n" // Load input point (0, 2)
+ "ld1h { z14.h }, p2/Z, [x9]\n"
+ "ld1h { z15.h }, p2/Z, [x9, x13, LSL #1]\n"
+ "ld1h { z16.h }, p2/Z, [x21, x25, LSL #1]\n"
"bge 3f\n"
"2:" // Tile loop: Channel loop
- "fmla z31.h, p3/M, z8.h, z9.h\n"
- "ld1h { z17.h }, p3/Z, [x16]\n" // Load from weights and bias
+ "movprfx z31, z17\n fmla z31.h, p3/M, z8.h, z9.h\n"
"whilelt p1.h, x14, %x[n_channels]\n"
- "fmla z30.h, p3/M, z6.h, z9.h\n"
+ "movprfx z30, z17\n fmla z30.h, p3/M, z6.h, z9.h\n"
"inch x12\n"
- "fmla z29.h, p3/M, z2.h, z9.h\n"
+ "movprfx z29, z17\n fmla z29.h, p3/M, z2.h, z9.h\n"
"mov p0.b, p2.b\n"
- "fmla z28.h, p3/M, z0.h, z9.h\n"
+ "movprfx z28, z17\n fmla z28.h, p3/M, z0.h, z9.h\n"
+ "ld1h { z17.h }, p3/Z, [x16]\n"
"inch x15\n"
- "addvl x21, x21, #1\n"
"fmla z31.h, p3/M, z0.h, z10.h\n"
- "ld1h { z10.h }, p1/Z, [x21]\n" // Load input point (0, 0)
+ "addvl x21, x21, #1\n"
+ "ld1h { z10.h }, p1/Z, [x21]\n"
"fmla z30.h, p3/M, z1.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x9, x23, LSL #1]\n" // Load input point (1, 4)
"inch x14\n"
"fmla z31.h, p3/M, z1.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x9, x24, LSL #1]\n" // Load input point (1, 3)
+ "ld1h { z11.h }, p2/Z, [x9, x24, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x9, x23, LSL #1]\n"
"fmla z30.h, p3/M, z2.h, z13.h\n"
- "ld1h { z13.h }, p2/Z, [x9, x25, LSL #1]\n" // Load input point (1, 2)
- "addvl x9, x9, #1\n"
+ "ld1h { z13.h }, p2/Z, [x9, x25, LSL #1]\n"
"fmla z31.h, p3/M, z3.h, z14.h\n"
- "ld1h { z14.h }, p2/Z, [x27]\n" // Load input point (3, 0)
+ "ld1h { z14.h }, p2/Z, [x27]\n"
+ "addvl x9, x9, #1\n"
"fmla z30.h, p3/M, z0.h, z16.h\n"
"fmla z29.h, p3/M, z3.h, z14.h\n"
- "ld1h { z14.h }, p2/Z, [x27, x23, LSL #1]\n" // Load input point (3, 4)
+ "ld1h { z14.h }, p2/Z, [x27, x23, LSL #1]\n"
"fmla z31.h, p3/M, z4.h, z15.h\n"
- "ld1h { z15.h }, p2/Z, [x28]\n" // Load input point (2, 0)
+ "ld1h { z15.h }, p2/Z, [x28]\n"
"fmla z30.h, p3/M, z4.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x27, x13, LSL #1]\n" // Load input point (3, 1)
+ "ld1h { z11.h }, p2/Z, [x27, x13, LSL #1]\n"
"fmla z29.h, p3/M, z0.h, z15.h\n"
- "ld1h { z0.h }, p3/Z, [x16, #1, MUL VL]\n" // Load from weights and bias
+ "ld1h { z0.h }, p3/Z, [x16, #1, MUL VL]\n"
"fmla z31.h, p3/M, z2.h, z16.h\n"
- "ld1h { z16.h }, p2/Z, [x28, x13, LSL #1]\n" // Load input point (2, 1)
+ "ld1h { z16.h }, p2/Z, [x28, x13, LSL #1]\n"
"fmla z30.h, p3/M, z5.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x28, x24, LSL #1]\n" // Load input point (2, 3)
+ "ld1h { z12.h }, p2/Z, [x28, x24, LSL #1]\n"
"fmla z29.h, p3/M, z4.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x28, x23, LSL #1]\n" // Load input point (2, 4)
+ "ld1h { z11.h }, p2/Z, [x28, x23, LSL #1]\n"
"addvl x28, x28, #1\n"
"fmla z31.h, p3/M, z5.h, z13.h\n"
- "ld1h { z9.h }, p1/Z, [x28, x25, LSL #1]\n" // Load input point (2, 2)
+ "ld1h { z9.h }, p1/Z, [x28, x25, LSL #1]\n"
"fmla z30.h, p3/M, z3.h, z13.h\n"
- "ld1h { z13.h }, p2/Z, [x27, x24, LSL #1]\n" // Load input point (3, 3)
+ "ld1h { z13.h }, p2/Z, [x27, x24, LSL #1]\n"
"fmla z29.h, p3/M, z1.h, z16.h\n"
"fmla z31.h, p3/M, z6.h, z15.h\n"
- "ld1h { z15.h }, p2/Z, [x26]\n" // Load input point (4, 0)
+ "ld1h { z15.h }, p2/Z, [x26]\n"
"fmla z28.h, p3/M, z4.h, z13.h\n"
- "ld1h { z13.h }, p2/Z, [x26, x13, LSL #1]\n" // Load input point (4, 1)
+ "ld1h { z13.h }, p2/Z, [x26, x13, LSL #1]\n"
"fmla z30.h, p3/M, z7.h, z12.h\n"
- "ld1h { z4.h }, p3/Z, [x16, #5, MUL VL]\n" // Load from weights and bias
+ "ld1h { z4.h }, p3/Z, [x16, #5, MUL VL]\n"
"fmla z29.h, p3/M, z6.h, z15.h\n"
- "ld1h { z15.h }, p2/Z, [x26, x25, LSL #1]\n" // Load input point (4, 2)
+ "ld1h { z15.h }, p2/Z, [x26, x25, LSL #1]\n"
"fmla z31.h, p3/M, z7.h, z16.h\n"
- "ld1h { z16.h }, p2/Z, [x27, x25, LSL #1]\n" // Load input point (3, 2)
+ "ld1h { z16.h }, p2/Z, [x27, x25, LSL #1]\n"
"addvl x27, x27, #1\n"
"fmla z28.h, p3/M, z1.h, z12.h\n"
- "ld1h { z12.h }, p1/Z, [x21, x24, LSL #1]\n" // Load input point (0, 3)
+ "ld1h { z12.h }, p1/Z, [x21, x24, LSL #1]\n"
"fmla z30.h, p3/M, z8.h, z11.h\n"
- "ld1h { z1.h }, p3/Z, [x16, #2, MUL VL]\n" // Load from weights and bias
+ "ld1h { z1.h }, p3/Z, [x16, #2, MUL VL]\n"
"fmla z29.h, p3/M, z7.h, z13.h\n"
- "ld1h { z13.h }, p1/Z, [x21, x23, LSL #1]\n" // Load input point (0, 4)
+ "ld1h { z13.h }, p1/Z, [x21, x23, LSL #1]\n"
"fmax z31.h, p3/M, z31.h, z19.h\n"
"fmla z28.h, p3/M, z5.h, z14.h\n"
- "ld1h { z14.h }, p2/Z, [x26, x24, LSL #1]\n" // Load input point (4, 3)
+ "ld1h { z14.h }, p2/Z, [x26, x24, LSL #1]\n"
"fmax z30.h, p3/M, z30.h, z19.h\n"
"fmla z29.h, p3/M, z5.h, z16.h\n"
- "ld1h { z5.h }, p3/Z, [x16, #6, MUL VL]\n" // Load from weights and bias
+ "ld1h { z5.h }, p3/Z, [x16, #6, MUL VL]\n"
"fmin z31.h, p3/M, z31.h, z18.h\n"
- "st1h { z31.h }, p0, [x10]\n" // Store output point (0, 0)
- "mov z31.d, z17.d\n"
+ "st1h { z31.h }, p0, [x10]\n"
"fmla z28.h, p3/M, z2.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x26, x23, LSL #1]\n" // Load input point (4, 4)
- "whilelt p2.h, x15, %x[n_channels]\n"
"fmla z29.h, p3/M, z8.h, z15.h\n"
- "ld1h { z2.h }, p3/Z, [x16, #3, MUL VL]\n" // Load from weights and bias
- "addvl x26, x26, #1\n"
+ "ld1h { z11.h }, p2/Z, [x26, x23, LSL #1]\n"
+ "whilelt p2.h, x15, %x[n_channels]\n"
"fmin z30.h, p3/M, z30.h, z18.h\n"
- "st1h { z30.h }, p0, [x10, x11, LSL #1]\n" // Store output point (0, 1)
- "mov z30.d, z17.d\n"
- "addvl x10, x10, #1\n"
+ "ld1h { z2.h }, p3/Z, [x16, #3, MUL VL]\n"
+ "addvl x26, x26, #1\n"
"fmla z28.h, p3/M, z3.h, z16.h\n"
- "ld1h { z16.h }, p1/Z, [x21, x25, LSL #1]\n" // Load input point (0, 2)
+ "ld1h { z16.h }, p1/Z, [x21, x25, LSL #1]\n"
"cmp x14, %x[n_channels]\n"
"fmax z29.h, p3/M, z29.h, z19.h\n"
- "ld1h { z3.h }, p3/Z, [x16, #4, MUL VL]\n" // Load from weights and bias
+ "ld1h { z3.h }, p3/Z, [x16, #4, MUL VL]\n"
+ "st1h { z30.h }, p0, [x10, x11, LSL #1]\n"
"fmla z28.h, p3/M, z7.h, z14.h\n"
- "ld1h { z14.h }, p1/Z, [x9]\n" // Load input point (1, 0)
+ "ld1h { z14.h }, p1/Z, [x9]\n"
+ "addvl x10, x10, #1\n"
"fmin z29.h, p3/M, z29.h, z18.h\n"
- "st1h { z29.h }, p0, [x22]\n" // Store output point (1, 0)
- "mov z29.d, z17.d\n"
+ "st1h { z29.h }, p0, [x22]\n"
"fmla z28.h, p3/M, z6.h, z15.h\n"
- "ld1h { z15.h }, p1/Z, [x9, x13, LSL #1]\n" // Load input point (1, 1)
- "ld1h { z6.h }, p3/Z, [x16, #7, MUL VL]\n" // Load from weights and bias
+ "ld1h { z15.h }, p1/Z, [x9, x13, LSL #1]\n"
"fmla z28.h, p3/M, z8.h, z11.h\n"
- "ld1h { z11.h }, p1/Z, [x21, x13, LSL #1]\n" // Load input point (0, 1)
- "addvl x16, x16, #16\n"
+ "ld1h { z11.h }, p1/Z, [x21, x13, LSL #1]\n"
+ "ld1h { z6.h }, p3/Z, [x16, #7, MUL VL]\n"
"fmax z28.h, p3/M, z28.h, z19.h\n"
- "ld1h { z7.h }, p3/Z, [x16, #-8, MUL VL]\n" // Load from weights and bias
- "ld1h { z8.h }, p3/Z, [x16, #-7, MUL VL]\n" // Load from weights and bias
+ "addvl x16, x16, #16\n"
+ "ld1h { z7.h }, p3/Z, [x16, #-8, MUL VL]\n"
"fmin z28.h, p3/M, z28.h, z18.h\n"
- "st1h { z28.h }, p0, [x22, x11, LSL #1]\n" // Store output point (1, 1)
- "mov z28.d, z17.d\n"
- "addvl x22, x22, #1\n"
+ "ld1h { z8.h }, p3/Z, [x16, #-7, MUL VL]\n"
"addvl x16, x16, #-6\n"
+ "st1h { z28.h }, p0, [x22, x11, LSL #1]\n"
+ "addvl x22, x22, #1\n"
"blt 2b\n"
"3:" // Tile loop: Channel tail
- "fmla z31.h, p3/M, z8.h, z9.h\n"
+ "movprfx z31, z17\n fmla z31.h, p3/M, z8.h, z9.h\n"
"ldr x7, [%x[params_struct], %[offsetof_args_tile_i]]\n"
"mov p0.b, p2.b\n"
- "fmla z30.h, p3/M, z6.h, z9.h\n"
+ "movprfx z30, z17\n fmla z30.h, p3/M, z6.h, z9.h\n"
"ldr x8, [%x[params_struct], %[offsetof_args_tile_j]]\n"
"add x21, x7, #0x1\n"
- "fmla z29.h, p3/M, z2.h, z9.h\n"
+ "movprfx z29, z17\n fmla z29.h, p3/M, z2.h, z9.h\n"
"ldr x20, [%x[params_struct], %[offsetof_args_n_tile_rows]]\n"
- "fmla z28.h, p3/M, z0.h, z9.h\n"
+ "movprfx z28, z17\n fmla z28.h, p3/M, z0.h, z9.h\n"
"ldr x19, [%x[params_struct], %[offsetof_args_n_tile_cols]]\n"
"add x8, x8, #0x1\n"
"fmla z31.h, p3/M, z0.h, z10.h\n"
"cmp x8, x19\n"
"fmla z30.h, p3/M, z1.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x9, x23, LSL #1]\n" // Load input point (1, 4)
+ "ld1h { z12.h }, p2/Z, [x9, x23, LSL #1]\n"
"fmla z31.h, p3/M, z1.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x9, x24, LSL #1]\n" // Load input point (1, 3)
+ "ld1h { z11.h }, p2/Z, [x9, x24, LSL #1]\n"
"csel x8, x8, XZR, LT\n"
"fmla z30.h, p3/M, z2.h, z13.h\n"
- "ld1h { z13.h }, p2/Z, [x9, x25, LSL #1]\n" // Load input point (1, 2)
+ "ld1h { z13.h }, p2/Z, [x9, x25, LSL #1]\n"
"csel x7, x7, x21, LT\n"
"fmla z31.h, p3/M, z3.h, z14.h\n"
- "ld1h { z14.h }, p2/Z, [x27]\n" // Load input point (3, 0)
+ "ld1h { z14.h }, p2/Z, [x27]\n"
"cmp x7, x20\n"
"fmla z30.h, p3/M, z0.h, z16.h\n"
"fmla z29.h, p3/M, z3.h, z14.h\n"
- "ld1h { z14.h }, p2/Z, [x27, x23, LSL #1]\n" // Load input point (3, 4)
+ "ld1h { z14.h }, p2/Z, [x27, x23, LSL #1]\n"
"fmla z31.h, p3/M, z4.h, z15.h\n"
- "ld1h { z15.h }, p2/Z, [x28]\n" // Load input point (2, 0)
+ "ld1h { z15.h }, p2/Z, [x28]\n"
"fmla z30.h, p3/M, z4.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x27, x13, LSL #1]\n" // Load input point (3, 1)
+ "ld1h { z11.h }, p2/Z, [x27, x13, LSL #1]\n"
"fmla z29.h, p3/M, z0.h, z15.h\n"
"fmla z31.h, p3/M, z2.h, z16.h\n"
- "ld1h { z16.h }, p2/Z, [x28, x13, LSL #1]\n" // Load input point (2, 1)
+ "ld1h { z16.h }, p2/Z, [x28, x13, LSL #1]\n"
"fmla z30.h, p3/M, z5.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x28, x24, LSL #1]\n" // Load input point (2, 3)
+ "ld1h { z12.h }, p2/Z, [x28, x24, LSL #1]\n"
"fmla z29.h, p3/M, z4.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x28, x23, LSL #1]\n" // Load input point (2, 4)
+ "ld1h { z11.h }, p2/Z, [x28, x23, LSL #1]\n"
"fmla z31.h, p3/M, z5.h, z13.h\n"
"fmla z30.h, p3/M, z3.h, z13.h\n"
- "ld1h { z13.h }, p2/Z, [x27, x24, LSL #1]\n" // Load input point (3, 3)
+ "ld1h { z13.h }, p2/Z, [x27, x24, LSL #1]\n"
"fmla z29.h, p3/M, z1.h, z16.h\n"
"fmla z31.h, p3/M, z6.h, z15.h\n"
- "ld1h { z15.h }, p2/Z, [x26]\n" // Load input point (4, 0)
+ "ld1h { z15.h }, p2/Z, [x26]\n"
"fmla z28.h, p3/M, z4.h, z13.h\n"
- "ld1h { z13.h }, p2/Z, [x26, x13, LSL #1]\n" // Load input point (4, 1)
+ "ld1h { z13.h }, p2/Z, [x26, x13, LSL #1]\n"
"fmla z30.h, p3/M, z7.h, z12.h\n"
"fmla z29.h, p3/M, z6.h, z15.h\n"
- "ld1h { z15.h }, p2/Z, [x26, x25, LSL #1]\n" // Load input point (4, 2)
+ "ld1h { z15.h }, p2/Z, [x26, x25, LSL #1]\n"
"fmla z31.h, p3/M, z7.h, z16.h\n"
- "ld1h { z16.h }, p2/Z, [x27, x25, LSL #1]\n" // Load input point (3, 2)
+ "ld1h { z16.h }, p2/Z, [x27, x25, LSL #1]\n"
"fmla z28.h, p3/M, z1.h, z12.h\n"
"fmla z30.h, p3/M, z8.h, z11.h\n"
"fmla z29.h, p3/M, z7.h, z13.h\n"
"fmax z31.h, p3/M, z31.h, z19.h\n"
"fmla z28.h, p3/M, z5.h, z14.h\n"
- "ld1h { z14.h }, p2/Z, [x26, x24, LSL #1]\n" // Load input point (4, 3)
+ "ld1h { z14.h }, p2/Z, [x26, x24, LSL #1]\n"
"fmax z30.h, p3/M, z30.h, z19.h\n"
"fmla z29.h, p3/M, z5.h, z16.h\n"
"fmin z31.h, p3/M, z31.h, z18.h\n"
- "st1h { z31.h }, p0, [x10]\n" // Store output point (0, 0)
+ "st1h { z31.h }, p0, [x10]\n"
"fmla z28.h, p3/M, z2.h, z11.h\n"
"fmla z29.h, p3/M, z8.h, z15.h\n"
- "ld1h { z11.h }, p2/Z, [x26, x23, LSL #1]\n" // Load input point (4, 4)
+ "ld1h { z11.h }, p2/Z, [x26, x23, LSL #1]\n"
"fmin z30.h, p3/M, z30.h, z18.h\n"
- "st1h { z30.h }, p0, [x10, x11, LSL #1]\n" // Store output point (0, 1)
+ "st1h { z30.h }, p0, [x10, x11, LSL #1]\n"
"fmla z28.h, p3/M, z3.h, z16.h\n"
"fmax z29.h, p3/M, z29.h, z19.h\n"
"fmla z28.h, p3/M, z7.h, z14.h\n"
"fmin z29.h, p3/M, z29.h, z18.h\n"
- "st1h { z29.h }, p0, [x22]\n" // Store output point (1, 0)
+ "st1h { z29.h }, p0, [x22]\n"
"fmla z28.h, p3/M, z6.h, z15.h\n"
"fmla z28.h, p3/M, z8.h, z11.h\n"
"fmax z28.h, p3/M, z28.h, z19.h\n"
"fmin z28.h, p3/M, z28.h, z18.h\n"
- "st1h { z28.h }, p0, [x22, x11, LSL #1]\n" // Store output point (1, 1)
+ "st1h { z28.h }, p0, [x22, x11, LSL #1]\n"
"blt 1b\n"
:
: [n_channels] "r" ((unsigned long) n_channels), [offsetof_args_inptr] "I" (offsetof(Args, inptr)), [offsetof_args_ld_input_col] "I" (offsetof(Args, ld_input_col)), [offsetof_args_ld_input_row] "I" (offsetof(Args, ld_input_row)), [offsetof_args_ld_output_col] "I" (offsetof(Args, ld_output_col)), [offsetof_args_ld_output_row] "I" (offsetof(Args, ld_output_row)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_n_tile_cols] "I" (offsetof(Args, n_tile_cols)), [offsetof_args_n_tile_rows] "I" (offsetof(Args, n_tile_rows)), [offsetof_args_outptr] "I" (offsetof(Args, outptr)), [offsetof_args_params] "I" (offsetof(Args, params)), [offsetof_args_tile_i] "I" (offsetof(Args, tile_i)), [offsetof_args_tile_j] "I" (offsetof(Args, tile_j)), [params_struct] "r" (¶ms_struct)
@@ -342,4 +334,4 @@
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
+#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp
index 0d5d417..ceba36d 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp
@@ -25,7 +25,7 @@
#include <cstddef>
#include <cstdint>
-#if defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
+#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
namespace arm_conv {
namespace depthwise {
@@ -89,257 +89,249 @@
__asm__ __volatile__(
"ldr x19, [%x[params_struct], %[offsetof_args_outptrs]]\n"
"ptrue p3.b\n"
- "ldr x16, [%x[params_struct], %[offsetof_args_params]]\n"
- "add x15, %x[params_struct], %[offsetof_Args_inptrs]\n"
+ "ldr x15, [%x[params_struct], %[offsetof_args_params]]\n"
+ "add x14, %x[params_struct], %[offsetof_Args_inptrs]\n"
"ld1rh { z19.h }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n"
- "mov x14, #0x0\n"
+ "mov x13, #0x0\n"
"ld1rh { z18.h }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n"
- "cnth x13\n"
- "ldp x12, x11, [x19, #0x0]\n"
- "sub x10, XZR, x13\n"
- "ldp x9, x28, [x19, #0x10]\n"
+ "cnth x12\n"
+ "ldp x11, x10, [x19, #0x0]\n"
+ "sub x9, XZR, x12\n"
+ "ldp x28, x27, [x19, #0x10]\n"
"whilelt p2.h, XZR, %x[n_channels]\n"
- "ld1h { z17.h }, p3/Z, [x16]\n" // Load from weights and bias
- "mov z31.d, z17.d\n"
- "ld1h { z0.h }, p3/Z, [x16, #1, MUL VL]\n" // Load from weights and bias
- "cmp x13, %x[n_channels]\n"
- "mov z30.d, z17.d\n"
- "ld1h { z1.h }, p3/Z, [x16, #2, MUL VL]\n" // Load from weights and bias
- "mov z29.d, z17.d\n"
- "ld1h { z2.h }, p3/Z, [x16, #3, MUL VL]\n" // Load from weights and bias
- "mov z28.d, z17.d\n"
- "ld1h { z3.h }, p3/Z, [x16, #4, MUL VL]\n" // Load from weights and bias
- "ld1h { z4.h }, p3/Z, [x16, #5, MUL VL]\n" // Load from weights and bias
- "ld1h { z5.h }, p3/Z, [x16, #6, MUL VL]\n" // Load from weights and bias
- "ld1h { z6.h }, p3/Z, [x16, #7, MUL VL]\n" // Load from weights and bias
- "addvl x16, x16, #16\n"
- "ldp x27, x26, [x15, #0x0]\n"
- "ld1h { z7.h }, p3/Z, [x16, #-8, MUL VL]\n" // Load from weights and bias
- "ld1h { z8.h }, p3/Z, [x16, #-7, MUL VL]\n" // Load from weights and bias
- "addvl x16, x16, #-6\n"
- "ld1h { z9.h }, p2/Z, [x27, x14, LSL #1]\n"
- "ld1h { z10.h }, p2/Z, [x26, x14, LSL #1]\n"
- "ldp x25, x23, [x15, #0x10]\n"
- "ldp x22, x21, [x15, #0x20]\n"
- "ldp x20, x19, [x15, #0x30]\n"
- "ld1h { z11.h }, p2/Z, [x25, x14, LSL #1]\n"
- "ld1h { z12.h }, p2/Z, [x23, x14, LSL #1]\n"
- "ld1h { z13.h }, p2/Z, [x22, x14, LSL #1]\n"
- "ld1h { z14.h }, p2/Z, [x21, x14, LSL #1]\n"
- "ld1h { z15.h }, p2/Z, [x20, x14, LSL #1]\n"
- "ld1h { z16.h }, p2/Z, [x19, x14, LSL #1]\n"
+ "ld1h { z17.h }, p3/Z, [x15]\n"
+ "cmp x12, %x[n_channels]\n"
+ "ld1h { z0.h }, p3/Z, [x15, #1, MUL VL]\n"
+ "ld1h { z1.h }, p3/Z, [x15, #2, MUL VL]\n"
+ "ld1h { z2.h }, p3/Z, [x15, #3, MUL VL]\n"
+ "ld1h { z3.h }, p3/Z, [x15, #4, MUL VL]\n"
+ "ld1h { z4.h }, p3/Z, [x15, #5, MUL VL]\n"
+ "ld1h { z5.h }, p3/Z, [x15, #6, MUL VL]\n"
+ "ld1h { z6.h }, p3/Z, [x15, #7, MUL VL]\n"
+ "addvl x15, x15, #16\n"
+ "ldp x26, x25, [x14, #0x0]\n"
+ "ld1h { z7.h }, p3/Z, [x15, #-8, MUL VL]\n"
+ "ld1h { z8.h }, p3/Z, [x15, #-7, MUL VL]\n"
+ "addvl x15, x15, #-6\n"
+ "ld1h { z9.h }, p2/Z, [x26, x13, LSL #1]\n"
+ "ld1h { z10.h }, p2/Z, [x25, x13, LSL #1]\n"
+ "ldp x24, x23, [x14, #0x10]\n"
+ "ldp x22, x21, [x14, #0x20]\n"
+ "ldp x20, x19, [x14, #0x30]\n"
+ "ld1h { z11.h }, p2/Z, [x24, x13, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x23, x13, LSL #1]\n"
+ "ld1h { z13.h }, p2/Z, [x22, x13, LSL #1]\n"
+ "ld1h { z14.h }, p2/Z, [x21, x13, LSL #1]\n"
+ "ld1h { z15.h }, p2/Z, [x20, x13, LSL #1]\n"
+ "ld1h { z16.h }, p2/Z, [x19, x13, LSL #1]\n"
"bge 2f\n"
"1:" // Channel loop
- "fmla z31.h, p3/M, z8.h, z9.h\n"
- "ldr x24, [x15, #0x40]\n"
- "whilelt p1.h, x13, %x[n_channels]\n"
- "fmla z30.h, p3/M, z6.h, z9.h\n"
- "ldr x20, [x15, #0x48]\n"
- "inch x10\n"
- "fmla z29.h, p3/M, z2.h, z9.h\n"
- "ldr x23, [x15, #0x50]\n"
+ "movprfx z31, z17\n fmla z31.h, p3/M, z8.h, z9.h\n"
+ "ldr x26, [x14, #0x40]\n"
+ "whilelt p1.h, x12, %x[n_channels]\n"
+ "movprfx z30, z17\n fmla z30.h, p3/M, z6.h, z9.h\n"
+ "ldr x25, [x14, #0x48]\n"
+ "inch x9\n"
+ "movprfx z29, z17\n fmla z29.h, p3/M, z2.h, z9.h\n"
+ "ldr x24, [x14, #0x50]\n"
"mov p0.b, p2.b\n"
- "fmla z28.h, p3/M, z0.h, z9.h\n"
- "ldr x19, [x15, #0x58]\n"
- "ldr x22, [x15, #0x60]\n"
+ "movprfx z28, z17\n fmla z28.h, p3/M, z0.h, z9.h\n"
+ "ldr x23, [x14, #0x58]\n"
+ "ldr x22, [x14, #0x60]\n"
"fmla z31.h, p3/M, z0.h, z10.h\n"
- "ldr x21, [x15, #0x68]\n"
+ "ldr x21, [x14, #0x68]\n"
"fmla z30.h, p3/M, z1.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x20, x14, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x25, x13, LSL #1]\n"
"fmla z31.h, p3/M, z1.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x24, x14, LSL #1]\n"
- "ldr x20, [x15, #0x70]\n"
+ "ld1h { z11.h }, p2/Z, [x26, x13, LSL #1]\n"
+ "ldr x20, [x14, #0x70]\n"
"fmla z30.h, p3/M, z2.h, z13.h\n"
- "ld1h { z13.h }, p2/Z, [x23, x14, LSL #1]\n"
+ "ld1h { z13.h }, p2/Z, [x24, x13, LSL #1]\n"
"fmla z31.h, p3/M, z3.h, z14.h\n"
- "ld1h { z14.h }, p2/Z, [x19, x14, LSL #1]\n"
- "ldr x19, [x15, #0x78]\n"
+ "ld1h { z14.h }, p2/Z, [x23, x13, LSL #1]\n"
+ "ldr x19, [x14, #0x78]\n"
"fmla z30.h, p3/M, z0.h, z16.h\n"
- "ldr x27, [x15, #0x80]\n"
+ "ldr x26, [x14, #0x80]\n"
"fmla z29.h, p3/M, z3.h, z14.h\n"
- "ldr x26, [x15, #0x88]\n"
- "ldr x25, [x15, #0x90]\n"
+ "ldr x25, [x14, #0x88]\n"
+ "ldr x24, [x14, #0x90]\n"
"fmla z31.h, p3/M, z4.h, z15.h\n"
- "ld1h { z15.h }, p2/Z, [x22, x14, LSL #1]\n"
+ "ld1h { z15.h }, p2/Z, [x22, x13, LSL #1]\n"
"fmla z30.h, p3/M, z4.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x21, x14, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x21, x13, LSL #1]\n"
"fmla z29.h, p3/M, z0.h, z15.h\n"
- "ld1h { z14.h }, p2/Z, [x26, x14, LSL #1]\n"
- "ldr x23, [x15, #0x98]\n"
+ "ld1h { z14.h }, p2/Z, [x25, x13, LSL #1]\n"
+ "ldr x23, [x14, #0x98]\n"
"fmla z31.h, p3/M, z2.h, z16.h\n"
- "ld1h { z16.h }, p2/Z, [x20, x14, LSL #1]\n"
+ "ld1h { z16.h }, p2/Z, [x20, x13, LSL #1]\n"
"fmla z30.h, p3/M, z5.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x27, x14, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x26, x13, LSL #1]\n"
"fmla z29.h, p3/M, z4.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x23, x14, LSL #1]\n"
- "ldr x22, [x15, #0xa0]\n"
+ "ld1h { z11.h }, p2/Z, [x23, x13, LSL #1]\n"
+ "ldr x22, [x14, #0xa0]\n"
"fmla z31.h, p3/M, z5.h, z13.h\n"
- "ldr x21, [x15, #0xa8]\n"
+ "ldr x21, [x14, #0xa8]\n"
"fmla z30.h, p3/M, z3.h, z13.h\n"
- "ld1h { z13.h }, p2/Z, [x19, x14, LSL #1]\n"
+ "ld1h { z13.h }, p2/Z, [x19, x13, LSL #1]\n"
"fmla z29.h, p3/M, z1.h, z16.h\n"
- "ldr x20, [x15, #0xb0]\n"
- "ldr x19, [x15, #0xb8]\n"
+ "ldr x20, [x14, #0xb0]\n"
+ "ldr x19, [x14, #0xb8]\n"
"fmla z31.h, p3/M, z6.h, z15.h\n"
"fmla z28.h, p3/M, z4.h, z13.h\n"
- "ld1h { z15.h }, p2/Z, [x25, x14, LSL #1]\n"
+ "ld1h { z15.h }, p2/Z, [x24, x13, LSL #1]\n"
"fmla z30.h, p3/M, z7.h, z12.h\n"
- "ld1h { z13.h }, p2/Z, [x22, x14, LSL #1]\n"
- "ldr x24, [x15, #0xc0]\n"
+ "ld1h { z13.h }, p2/Z, [x22, x13, LSL #1]\n"
+ "ldr x26, [x14, #0xc0]\n"
"fmla z31.h, p3/M, z7.h, z16.h\n"
- "ld1h { z16.h }, p2/Z, [x21, x14, LSL #1]\n"
+ "ld1h { z16.h }, p2/Z, [x21, x13, LSL #1]\n"
"fmla z28.h, p3/M, z1.h, z12.h\n"
- "ldp x27, x26, [x15, #0x0]\n"
+ "ld1h { z17.h }, p3/Z, [x15]\n"
"fmla z29.h, p3/M, z6.h, z15.h\n"
- "ld1h { z15.h }, p2/Z, [x19, x14, LSL #1]\n"
+ "ld1h { z15.h }, p2/Z, [x19, x13, LSL #1]\n"
"fmla z30.h, p3/M, z8.h, z11.h\n"
- "ldp x25, x23, [x15, #0x10]\n"
- "ldp x22, x21, [x15, #0x20]\n"
+ "ld1h { z0.h }, p3/Z, [x15, #1, MUL VL]\n"
+ "ld1h { z1.h }, p3/Z, [x15, #2, MUL VL]\n"
"fmla z28.h, p3/M, z5.h, z14.h\n"
"fmax z31.h, p3/M, z31.h, z19.h\n"
- "ld1h { z14.h }, p2/Z, [x20, x14, LSL #1]\n"
+ "ld1h { z14.h }, p2/Z, [x20, x13, LSL #1]\n"
"fmla z29.h, p3/M, z7.h, z13.h\n"
- "ld1h { z9.h }, p1/Z, [x27, x13, LSL #1]\n"
+ "ld1h { z4.h }, p3/Z, [x15, #5, MUL VL]\n"
"fmax z30.h, p3/M, z30.h, z19.h\n"
- "ld1h { z10.h }, p1/Z, [x26, x13, LSL #1]\n"
- "ld1h { z12.h }, p1/Z, [x23, x13, LSL #1]\n"
"fmla z28.h, p3/M, z2.h, z11.h\n"
- "fmin z31.h, p3/M, z31.h, z18.h\n"
- "ld1h { z11.h }, p2/Z, [x24, x14, LSL #1]\n"
- "inch x14\n"
- "fmla z29.h, p3/M, z5.h, z16.h\n"
- "ld1h { z13.h }, p1/Z, [x22, x13, LSL #1]\n"
- "whilelt p2.h, x14, %x[n_channels]\n"
- "fmin z30.h, p3/M, z30.h, z18.h\n"
- "ldp x20, x19, [x15, #0x30]\n"
- "ld1h { z17.h }, p3/Z, [x16]\n" // Load from weights and bias
- "fmla z28.h, p3/M, z3.h, z16.h\n"
- "st1h { z31.h }, p0, [x12, x10, LSL #1]\n"
- "mov z31.d, z17.d\n"
- "ld1h { z16.h }, p1/Z, [x19, x13, LSL #1]\n"
- "fmla z29.h, p3/M, z8.h, z15.h\n"
- "st1h { z30.h }, p0, [x11, x10, LSL #1]\n"
- "mov z30.d, z17.d\n"
- "ld1h { z0.h }, p3/Z, [x16, #1, MUL VL]\n" // Load from weights and bias
- "fmla z28.h, p3/M, z7.h, z14.h\n"
- "ld1h { z14.h }, p1/Z, [x21, x13, LSL #1]\n"
- "ld1h { z1.h }, p3/Z, [x16, #2, MUL VL]\n" // Load from weights and bias
- "fmax z29.h, p3/M, z29.h, z19.h\n"
- "ld1h { z2.h }, p3/Z, [x16, #3, MUL VL]\n" // Load from weights and bias
- "fmla z28.h, p3/M, z6.h, z15.h\n"
- "ld1h { z15.h }, p1/Z, [x20, x13, LSL #1]\n"
- "ld1h { z3.h }, p3/Z, [x16, #4, MUL VL]\n" // Load from weights and bias
- "fmin z29.h, p3/M, z29.h, z18.h\n"
- "ld1h { z4.h }, p3/Z, [x16, #5, MUL VL]\n" // Load from weights and bias
- "ld1h { z5.h }, p3/Z, [x16, #6, MUL VL]\n" // Load from weights and bias
- "fmla z28.h, p3/M, z8.h, z11.h\n"
- "ld1h { z11.h }, p1/Z, [x25, x13, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x26, x13, LSL #1]\n"
"inch x13\n"
+ "fmla z29.h, p3/M, z5.h, z16.h\n"
+ "ldp x26, x25, [x14, #0x0]\n"
+ "whilelt p2.h, x13, %x[n_channels]\n"
+ "fmin z31.h, p3/M, z31.h, z18.h\n"
+ "ldp x24, x23, [x14, #0x10]\n"
+ "fmin z30.h, p3/M, z30.h, z18.h\n"
+ "ldp x22, x21, [x14, #0x20]\n"
+ "ldp x20, x19, [x14, #0x30]\n"
+ "fmla z28.h, p3/M, z3.h, z16.h\n"
+ "ld1h { z9.h }, p1/Z, [x26, x12, LSL #1]\n"
+ "fmla z29.h, p3/M, z8.h, z15.h\n"
+ "ld1h { z10.h }, p1/Z, [x25, x12, LSL #1]\n"
+ "fmla z28.h, p3/M, z7.h, z14.h\n"
+ "ld1h { z12.h }, p1/Z, [x23, x12, LSL #1]\n"
+ "ld1h { z13.h }, p1/Z, [x22, x12, LSL #1]\n"
+ "fmax z29.h, p3/M, z29.h, z19.h\n"
+ "ld1h { z14.h }, p1/Z, [x21, x12, LSL #1]\n"
+ "fmla z28.h, p3/M, z6.h, z15.h\n"
+ "ld1h { z15.h }, p1/Z, [x20, x12, LSL #1]\n"
+ "ld1h { z16.h }, p1/Z, [x19, x12, LSL #1]\n"
+ "fmin z29.h, p3/M, z29.h, z18.h\n"
+ "st1h { z31.h }, p0, [x11, x9, LSL #1]\n"
+ "fmla z28.h, p3/M, z8.h, z11.h\n"
+ "ld1h { z11.h }, p1/Z, [x24, x12, LSL #1]\n"
+ "inch x12\n"
"fmax z28.h, p3/M, z28.h, z19.h\n"
- "st1h { z29.h }, p0, [x9, x10, LSL #1]\n"
- "cmp x13, %x[n_channels]\n"
- "mov z29.d, z17.d\n"
- "ld1h { z6.h }, p3/Z, [x16, #7, MUL VL]\n" // Load from weights and bias
- "addvl x16, x16, #16\n"
+ "st1h { z30.h }, p0, [x10, x9, LSL #1]\n"
+ "cmp x12, %x[n_channels]\n"
"fmin z28.h, p3/M, z28.h, z18.h\n"
- "ld1h { z7.h }, p3/Z, [x16, #-8, MUL VL]\n" // Load from weights and bias
- "ld1h { z8.h }, p3/Z, [x16, #-7, MUL VL]\n" // Load from weights and bias
- "addvl x16, x16, #-6\n"
- "st1h { z28.h }, p0, [x28, x10, LSL #1]\n"
- "mov z28.d, z17.d\n"
+ "st1h { z29.h }, p0, [x28, x9, LSL #1]\n"
+ "ld1h { z2.h }, p3/Z, [x15, #3, MUL VL]\n"
+ "ld1h { z3.h }, p3/Z, [x15, #4, MUL VL]\n"
+ "ld1h { z5.h }, p3/Z, [x15, #6, MUL VL]\n"
+ "ld1h { z6.h }, p3/Z, [x15, #7, MUL VL]\n"
+ "addvl x15, x15, #16\n"
+ "st1h { z28.h }, p0, [x27, x9, LSL #1]\n"
+ "ld1h { z7.h }, p3/Z, [x15, #-8, MUL VL]\n"
+ "ld1h { z8.h }, p3/Z, [x15, #-7, MUL VL]\n"
+ "addvl x15, x15, #-6\n"
"blt 1b\n"
"2:" // Channel tail
- "fmla z31.h, p3/M, z8.h, z9.h\n"
- "ldr x24, [x15, #0x40]\n"
- "inch x10\n"
- "fmla z30.h, p3/M, z6.h, z9.h\n"
- "ldr x20, [x15, #0x48]\n"
+ "movprfx z31, z17\n fmla z31.h, p3/M, z8.h, z9.h\n"
+ "ldr x26, [x14, #0x40]\n"
+ "inch x9\n"
+ "movprfx z30, z17\n fmla z30.h, p3/M, z6.h, z9.h\n"
+ "ldr x25, [x14, #0x48]\n"
"mov p0.b, p2.b\n"
- "fmla z29.h, p3/M, z2.h, z9.h\n"
- "ldr x23, [x15, #0x50]\n"
- "fmla z28.h, p3/M, z0.h, z9.h\n"
- "ldr x19, [x15, #0x58]\n"
- "ldr x22, [x15, #0x60]\n"
+ "movprfx z29, z17\n fmla z29.h, p3/M, z2.h, z9.h\n"
+ "ldr x24, [x14, #0x50]\n"
+ "movprfx z28, z17\n fmla z28.h, p3/M, z0.h, z9.h\n"
+ "ldr x23, [x14, #0x58]\n"
+ "ldr x22, [x14, #0x60]\n"
"fmla z31.h, p3/M, z0.h, z10.h\n"
- "ldr x21, [x15, #0x68]\n"
+ "ldr x21, [x14, #0x68]\n"
"fmla z30.h, p3/M, z1.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x20, x14, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x25, x13, LSL #1]\n"
"fmla z31.h, p3/M, z1.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x24, x14, LSL #1]\n"
- "ldr x20, [x15, #0x70]\n"
+ "ld1h { z11.h }, p2/Z, [x26, x13, LSL #1]\n"
+ "ldr x20, [x14, #0x70]\n"
"fmla z30.h, p3/M, z2.h, z13.h\n"
- "ld1h { z13.h }, p2/Z, [x23, x14, LSL #1]\n"
+ "ld1h { z13.h }, p2/Z, [x24, x13, LSL #1]\n"
"fmla z31.h, p3/M, z3.h, z14.h\n"
- "ld1h { z14.h }, p2/Z, [x19, x14, LSL #1]\n"
- "ldr x19, [x15, #0x78]\n"
+ "ld1h { z14.h }, p2/Z, [x23, x13, LSL #1]\n"
+ "ldr x19, [x14, #0x78]\n"
"fmla z30.h, p3/M, z0.h, z16.h\n"
- "ldr x27, [x15, #0x80]\n"
+ "ldr x26, [x14, #0x80]\n"
"fmla z29.h, p3/M, z3.h, z14.h\n"
- "ldr x26, [x15, #0x88]\n"
- "ldr x25, [x15, #0x90]\n"
+ "ldr x25, [x14, #0x88]\n"
+ "ldr x24, [x14, #0x90]\n"
"fmla z31.h, p3/M, z4.h, z15.h\n"
- "ld1h { z15.h }, p2/Z, [x22, x14, LSL #1]\n"
+ "ld1h { z15.h }, p2/Z, [x22, x13, LSL #1]\n"
"fmla z30.h, p3/M, z4.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x21, x14, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x21, x13, LSL #1]\n"
"fmla z29.h, p3/M, z0.h, z15.h\n"
- "ld1h { z14.h }, p2/Z, [x26, x14, LSL #1]\n"
- "ldr x23, [x15, #0x98]\n"
+ "ld1h { z14.h }, p2/Z, [x25, x13, LSL #1]\n"
+ "ldr x23, [x14, #0x98]\n"
"fmla z31.h, p3/M, z2.h, z16.h\n"
- "ld1h { z16.h }, p2/Z, [x20, x14, LSL #1]\n"
+ "ld1h { z16.h }, p2/Z, [x20, x13, LSL #1]\n"
"fmla z30.h, p3/M, z5.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x27, x14, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x26, x13, LSL #1]\n"
"fmla z29.h, p3/M, z4.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x23, x14, LSL #1]\n"
- "ldr x22, [x15, #0xa0]\n"
+ "ld1h { z11.h }, p2/Z, [x23, x13, LSL #1]\n"
+ "ldr x22, [x14, #0xa0]\n"
"fmla z31.h, p3/M, z5.h, z13.h\n"
- "ldr x21, [x15, #0xa8]\n"
+ "ldr x21, [x14, #0xa8]\n"
"fmla z30.h, p3/M, z3.h, z13.h\n"
- "ld1h { z13.h }, p2/Z, [x19, x14, LSL #1]\n"
+ "ld1h { z13.h }, p2/Z, [x19, x13, LSL #1]\n"
"fmla z29.h, p3/M, z1.h, z16.h\n"
- "ldr x20, [x15, #0xb0]\n"
- "ldr x19, [x15, #0xb8]\n"
+ "ldr x20, [x14, #0xb0]\n"
+ "ldr x19, [x14, #0xb8]\n"
"fmla z31.h, p3/M, z6.h, z15.h\n"
"fmla z28.h, p3/M, z4.h, z13.h\n"
- "ld1h { z15.h }, p2/Z, [x25, x14, LSL #1]\n"
+ "ld1h { z15.h }, p2/Z, [x24, x13, LSL #1]\n"
"fmla z30.h, p3/M, z7.h, z12.h\n"
- "ld1h { z13.h }, p2/Z, [x22, x14, LSL #1]\n"
- "ldr x24, [x15, #0xc0]\n"
+ "ld1h { z13.h }, p2/Z, [x22, x13, LSL #1]\n"
+ "ldr x26, [x14, #0xc0]\n"
"fmla z31.h, p3/M, z7.h, z16.h\n"
- "ld1h { z16.h }, p2/Z, [x21, x14, LSL #1]\n"
+ "ld1h { z16.h }, p2/Z, [x21, x13, LSL #1]\n"
"fmla z28.h, p3/M, z1.h, z12.h\n"
"fmla z29.h, p3/M, z6.h, z15.h\n"
- "ld1h { z15.h }, p2/Z, [x19, x14, LSL #1]\n"
+ "ld1h { z15.h }, p2/Z, [x19, x13, LSL #1]\n"
"fmla z30.h, p3/M, z8.h, z11.h\n"
"fmla z28.h, p3/M, z5.h, z14.h\n"
- "ld1h { z14.h }, p2/Z, [x20, x14, LSL #1]\n"
+ "ld1h { z14.h }, p2/Z, [x20, x13, LSL #1]\n"
"fmax z31.h, p3/M, z31.h, z19.h\n"
"fmla z29.h, p3/M, z7.h, z13.h\n"
"fmax z30.h, p3/M, z30.h, z19.h\n"
"fmla z28.h, p3/M, z2.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x24, x14, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x26, x13, LSL #1]\n"
"fmin z31.h, p3/M, z31.h, z18.h\n"
- "st1h { z31.h }, p0, [x12, x10, LSL #1]\n"
+ "st1h { z31.h }, p0, [x11, x9, LSL #1]\n"
"fmla z29.h, p3/M, z5.h, z16.h\n"
"fmla z28.h, p3/M, z3.h, z16.h\n"
"fmin z30.h, p3/M, z30.h, z18.h\n"
- "st1h { z30.h }, p0, [x11, x10, LSL #1]\n"
+ "st1h { z30.h }, p0, [x10, x9, LSL #1]\n"
"fmla z28.h, p3/M, z7.h, z14.h\n"
"fmla z29.h, p3/M, z8.h, z15.h\n"
"fmla z28.h, p3/M, z6.h, z15.h\n"
"fmax z29.h, p3/M, z29.h, z19.h\n"
"fmla z28.h, p3/M, z8.h, z11.h\n"
"fmin z29.h, p3/M, z29.h, z18.h\n"
- "st1h { z29.h }, p0, [x9, x10, LSL #1]\n"
+ "st1h { z29.h }, p0, [x28, x9, LSL #1]\n"
"fmax z28.h, p3/M, z28.h, z19.h\n"
"fmin z28.h, p3/M, z28.h, z18.h\n"
- "st1h { z28.h }, p0, [x28, x10, LSL #1]\n"
+ "st1h { z28.h }, p0, [x27, x9, LSL #1]\n"
:
: [n_channels] "r" ((unsigned long) n_channels), [offsetof_Args_inptrs] "I" (offsetof(Args, inptrs)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_outptrs] "I" (offsetof(Args, outptrs)), [offsetof_args_params] "I" (offsetof(Args, params)), [params_struct] "r" (¶ms_struct)
- : "cc", "memory", "p0", "p1", "p2", "p3", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z28", "z29", "z30", "z31"
+ : "cc", "memory", "p0", "p1", "p2", "p3", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z28", "z29", "z30", "z31"
);
}
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
+#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp
index 236f9bf..4a9bd33 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp
@@ -28,7 +28,7 @@
#pragma once
-#if defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
+#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
namespace arm_conv {
namespace depthwise {
@@ -36,15 +36,17 @@
void sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_indirect_impl(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16);
void sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16);
-struct sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst
+class sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst : public IDepthwiseDepthfirstStrategy
{
- typedef __fp16 bias_type;
- typedef __fp16 input_type;
- typedef __fp16 weight_type;
- typedef __fp16 return_type;
-
+ private:
typedef void (*indirect_kern_type)(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16);
+ indirect_kern_type m_indirect_kernel = sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_indirect_impl;
+
typedef void (*direct_kern_type)(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16);
+ direct_kern_type m_direct_kernel = sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl;
+
+ public:
+ typedef __fp16 return_type;
constexpr static arm_gemm::VLType vl_type = arm_gemm::VLType::SVE;
@@ -60,13 +62,60 @@
constexpr static unsigned int input_rows = 6;
constexpr static unsigned int input_cols = 6;
- indirect_kern_type indirect_kernel = sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_indirect_impl;
- direct_kern_type direct_kernel = sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl;
-
sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst(const CPUInfo *) {}
+
+ arm_gemm::VLType get_vl_type(void) const override { return vl_type; }
+
+ unsigned int get_kernel_rows(void) const override { return kernel_rows; }
+ unsigned int get_kernel_cols(void) const override { return kernel_cols; }
+
+ unsigned int get_stride_rows(void) const override { return stride_rows; }
+ unsigned int get_stride_cols(void) const override { return stride_cols; }
+
+ unsigned int get_output_rows(void) const override { return output_rows; }
+ unsigned int get_output_cols(void) const override { return output_cols; }
+
+ unsigned int get_input_rows(void) const override { return input_rows; }
+ unsigned int get_input_cols(void) const override { return input_cols; }
+
+ void indirect_kernel(
+ const void *const *const input_ptrs,
+ void *const *const outptrs,
+ const void *params,
+ unsigned int n_channels,
+ const void *activation_min,
+ const void *activation_max
+ ) const override
+ {
+ m_indirect_kernel(
+ reinterpret_cast<const __fp16 *const *>(input_ptrs),
+ reinterpret_cast<__fp16 *const *>(outptrs),
+ params, n_channels,
+ *static_cast<const __fp16 *>(activation_min),
+ *static_cast<const __fp16 *>(activation_max)
+ );
+ }
+
+ void direct_kernel(
+ const unsigned int n_tile_rows, const unsigned int n_tile_cols,
+ const void *inptr, int64_t ld_input_row, int64_t ld_input_col,
+ void *outptr, int64_t ld_output_row, int64_t ld_output_col,
+ const void *params, unsigned int n_channels,
+ const void *activation_min, const void *activation_max
+ ) const override
+ {
+ m_direct_kernel(
+ n_tile_rows, n_tile_cols,
+ static_cast<const __fp16 *>(inptr), ld_input_row, ld_input_col,
+ static_cast<__fp16 *>(outptr), ld_output_row, ld_output_col,
+ params, n_channels,
+ *static_cast<const __fp16 *>(activation_min),
+ *static_cast<const __fp16 *>(activation_max)
+ );
+ }
};
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
+#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_direct.cpp
index 6b1564e..1c2e1e2 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_direct.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_direct.cpp
@@ -25,7 +25,7 @@
#include <cstddef>
#include <cstdint>
-#if defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
+#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
namespace arm_conv {
namespace depthwise {
@@ -113,390 +113,382 @@
"add x20, x13, x22, LSL #1\n"
"ld1rh { z17.h }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n"
"add x10, x20, x22, LSL #1\n"
- "ld1h { z16.h }, p3/Z, [x8]\n" // Load from weights and bias
- "mov z31.d, z16.d\n"
- "ld1h { z0.h }, p3/Z, [x8, #1, MUL VL]\n" // Load from weights and bias
+ "ld1h { z16.h }, p3/Z, [x8]\n"
"add x9, x10, x22, LSL #1\n"
- "mov z30.d, z16.d\n"
- "ld1h { z1.h }, p3/Z, [x8, #2, MUL VL]\n" // Load from weights and bias
+ "ld1h { z0.h }, p3/Z, [x8, #1, MUL VL]\n"
"add x28, x9, x22, LSL #1\n"
- "mov z29.d, z16.d\n"
- "ld1h { z2.h }, p3/Z, [x8, #3, MUL VL]\n" // Load from weights and bias
+ "ld1h { z1.h }, p3/Z, [x8, #2, MUL VL]\n"
"add x27, x28, x22, LSL #1\n"
- "mov z28.d, z16.d\n"
- "ld1h { z3.h }, p3/Z, [x8, #4, MUL VL]\n" // Load from weights and bias
+ "ld1h { z2.h }, p3/Z, [x8, #3, MUL VL]\n"
"add x26, x15, x15\n"
- "ld1h { z4.h }, p3/Z, [x8, #5, MUL VL]\n" // Load from weights and bias
+ "ld1h { z3.h }, p3/Z, [x8, #4, MUL VL]\n"
"add x25, x26, x15\n"
- "mul x19, x5, x21\n" // offset = tile_i * ld_output_row
+ "ld1h { z4.h }, p3/Z, [x8, #5, MUL VL]\n"
"add x24, x25, x15\n"
+ "mul x19, x5, x21\n" // offset = tile_i * ld_output_row
"add x23, x24, x15\n"
"madd x19, x6, x12, x19\n" // offset += tile_j * ld_output_col
"mul x19, x19, x7\n" // offset *= output_tile_size
"add x11, x11, x19, LSL #1\n" // outptrs[0] += offset * sizeof(__fp16)
"add x22, x11, x21, LSL #1\n"
"whilelt p2.h, XZR, %x[n_channels]\n"
- "ld1h { z5.h }, p2/Z, [x13]\n" // Load input point (0, 0)
- "ld1h { z6.h }, p2/Z, [x13, x15, LSL #1]\n" // Load input point (0, 1)
+ "ld1h { z5.h }, p2/Z, [x13]\n"
+ "ld1h { z6.h }, p2/Z, [x13, x15, LSL #1]\n"
"cmp x16, %x[n_channels]\n"
- "ld1h { z7.h }, p2/Z, [x20]\n" // Load input point (1, 0)
+ "ld1h { z7.h }, p2/Z, [x20]\n"
"addvl x8, x8, #6\n"
- "ld1h { z8.h }, p2/Z, [x20, x15, LSL #1]\n" // Load input point (1, 1)
- "ld1h { z9.h }, p2/Z, [x13, x26, LSL #1]\n" // Load input point (0, 2)
- "ld1h { z13.h }, p2/Z, [x20, x26, LSL #1]\n" // Load input point (1, 2)
- "ld1h { z11.h }, p2/Z, [x13, x25, LSL #1]\n" // Load input point (0, 3)
- "ld1h { z12.h }, p2/Z, [x13, x24, LSL #1]\n" // Load input point (0, 4)
- "ld1h { z10.h }, p2/Z, [x20, x23, LSL #1]\n" // Load input point (1, 5)
- "ld1h { z14.h }, p2/Z, [x10]\n" // Load input point (2, 0)
+ "ld1h { z8.h }, p2/Z, [x20, x15, LSL #1]\n"
+ "ld1h { z9.h }, p2/Z, [x13, x26, LSL #1]\n"
+ "ld1h { z13.h }, p2/Z, [x20, x26, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x13, x25, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x13, x24, LSL #1]\n"
+ "ld1h { z10.h }, p2/Z, [x20, x23, LSL #1]\n"
+ "ld1h { z14.h }, p2/Z, [x10]\n"
"bge 3f\n"
"2:" // Tile loop: Channel loop
- "fmla z31.h, p3/M, z0.h, z5.h\n"
- "ld1h { z5.h }, p2/Z, [x20, x25, LSL #1]\n" // Load input point (1, 3)
+ "movprfx z31, z16\n fmla z31.h, p3/M, z0.h, z5.h\n"
+ "ld1h { z5.h }, p2/Z, [x20, x25, LSL #1]\n"
"whilelt p1.h, x16, %x[n_channels]\n"
- "fmla z30.h, p3/M, z0.h, z6.h\n"
+ "movprfx z30, z16\n fmla z30.h, p3/M, z0.h, z6.h\n"
"inch x14\n"
- "fmla z29.h, p3/M, z0.h, z7.h\n"
+ "movprfx z29, z16\n fmla z29.h, p3/M, z0.h, z7.h\n"
"mov p0.b, p2.b\n"
- "fmla z28.h, p3/M, z0.h, z8.h\n"
- "ld1h { z0.h }, p3/Z, [x8]\n" // Load from weights and bias
+ "movprfx z28, z16\n fmla z28.h, p3/M, z0.h, z8.h\n"
+ "ld1h { z0.h }, p3/Z, [x8]\n"
"inch x17\n"
"fmla z31.h, p3/M, z1.h, z6.h\n"
- "ld1h { z6.h }, p2/Z, [x20, x24, LSL #1]\n" // Load input point (1, 4)
+ "ld1h { z6.h }, p2/Z, [x20, x24, LSL #1]\n"
"addvl x20, x20, #1\n"
"fmla z30.h, p3/M, z1.h, z9.h\n"
"inch x16\n"
"fmla z29.h, p3/M, z1.h, z8.h\n"
"fmla z28.h, p3/M, z1.h, z13.h\n"
- "ld1h { z1.h }, p3/Z, [x8, #1, MUL VL]\n" // Load from weights and bias
+ "ld1h { z1.h }, p3/Z, [x8, #1, MUL VL]\n"
"fmla z31.h, p3/M, z2.h, z9.h\n"
- "ld1h { z9.h }, p2/Z, [x13, x23, LSL #1]\n" // Load input point (0, 5)
+ "ld1h { z9.h }, p2/Z, [x13, x23, LSL #1]\n"
"addvl x13, x13, #1\n"
"fmla z30.h, p3/M, z2.h, z11.h\n"
"fmla z29.h, p3/M, z2.h, z13.h\n"
"fmla z28.h, p3/M, z2.h, z5.h\n"
- "ld1h { z2.h }, p3/Z, [x8, #2, MUL VL]\n" // Load from weights and bias
+ "ld1h { z2.h }, p3/Z, [x8, #2, MUL VL]\n"
"fmla z31.h, p3/M, z3.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x10, x15, LSL #1]\n" // Load input point (2, 1)
+ "ld1h { z11.h }, p2/Z, [x10, x15, LSL #1]\n"
"fmla z30.h, p3/M, z3.h, z12.h\n"
"fmla z29.h, p3/M, z3.h, z5.h\n"
"fmla z28.h, p3/M, z3.h, z6.h\n"
- "ld1h { z3.h }, p3/Z, [x8, #3, MUL VL]\n" // Load from weights and bias
+ "ld1h { z3.h }, p3/Z, [x8, #3, MUL VL]\n"
"fmla z31.h, p3/M, z4.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x10, x26, LSL #1]\n" // Load input point (2, 2)
+ "ld1h { z12.h }, p2/Z, [x10, x26, LSL #1]\n"
"fmla z30.h, p3/M, z4.h, z9.h\n"
- "ld1h { z9.h }, p2/Z, [x10, x25, LSL #1]\n" // Load input point (2, 3)
+ "ld1h { z9.h }, p2/Z, [x10, x25, LSL #1]\n"
"fmla z29.h, p3/M, z4.h, z6.h\n"
"fmla z28.h, p3/M, z4.h, z10.h\n"
- "ld1h { z4.h }, p3/Z, [x8, #4, MUL VL]\n" // Load from weights and bias
+ "ld1h { z4.h }, p3/Z, [x8, #4, MUL VL]\n"
"fmla z31.h, p3/M, z0.h, z7.h\n"
- "ld1h { z7.h }, p1/Z, [x20]\n" // Load input point (1, 0)
+ "ld1h { z7.h }, p1/Z, [x20]\n"
"fmla z30.h, p3/M, z0.h, z8.h\n"
"fmla z29.h, p3/M, z0.h, z14.h\n"
"fmla z28.h, p3/M, z0.h, z11.h\n"
- "ld1h { z0.h }, p3/Z, [x8, #5, MUL VL]\n" // Load from weights and bias
+ "ld1h { z0.h }, p3/Z, [x8, #5, MUL VL]\n"
"fmla z31.h, p3/M, z1.h, z8.h\n"
- "ld1h { z8.h }, p2/Z, [x10, x23, LSL #1]\n" // Load input point (2, 5)
+ "ld1h { z8.h }, p2/Z, [x10, x23, LSL #1]\n"
"fmla z30.h, p3/M, z1.h, z13.h\n"
"fmla z29.h, p3/M, z1.h, z11.h\n"
"fmla z28.h, p3/M, z1.h, z12.h\n"
- "ld1h { z1.h }, p3/Z, [x8, #6, MUL VL]\n" // Load from weights and bias
+ "ld1h { z1.h }, p3/Z, [x8, #6, MUL VL]\n"
"fmla z31.h, p3/M, z2.h, z13.h\n"
- "ld1h { z13.h }, p2/Z, [x10, x24, LSL #1]\n" // Load input point (2, 4)
+ "ld1h { z13.h }, p2/Z, [x10, x24, LSL #1]\n"
"addvl x10, x10, #1\n"
"fmla z30.h, p3/M, z2.h, z5.h\n"
"fmla z29.h, p3/M, z2.h, z12.h\n"
"fmla z28.h, p3/M, z2.h, z9.h\n"
- "ld1h { z2.h }, p3/Z, [x8, #7, MUL VL]\n" // Load from weights and bias
+ "ld1h { z2.h }, p3/Z, [x8, #7, MUL VL]\n"
"addvl x8, x8, #16\n"
"fmla z31.h, p3/M, z3.h, z5.h\n"
- "ld1h { z5.h }, p2/Z, [x9]\n" // Load input point (3, 0)
- "ld1h { z16.h }, p3/Z, [x8, #4, MUL VL]\n" // Load from weights and bias
+ "ld1h { z5.h }, p2/Z, [x9]\n"
+ "ld1h { z16.h }, p3/Z, [x8, #4, MUL VL]\n"
"fmla z30.h, p3/M, z3.h, z6.h\n"
"fmla z29.h, p3/M, z3.h, z9.h\n"
"fmla z28.h, p3/M, z3.h, z13.h\n"
- "ld1h { z3.h }, p3/Z, [x8, #-8, MUL VL]\n" // Load from weights and bias
+ "ld1h { z3.h }, p3/Z, [x8, #-8, MUL VL]\n"
"fmla z31.h, p3/M, z4.h, z6.h\n"
- "ld1h { z6.h }, p2/Z, [x9, x15, LSL #1]\n" // Load input point (3, 1)
+ "ld1h { z6.h }, p2/Z, [x9, x15, LSL #1]\n"
"fmla z30.h, p3/M, z4.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x9, x26, LSL #1]\n" // Load input point (3, 2)
+ "ld1h { z10.h }, p2/Z, [x9, x26, LSL #1]\n"
"fmla z29.h, p3/M, z4.h, z13.h\n"
"fmla z28.h, p3/M, z4.h, z8.h\n"
- "ld1h { z4.h }, p3/Z, [x8, #-7, MUL VL]\n" // Load from weights and bias
+ "ld1h { z4.h }, p3/Z, [x8, #-7, MUL VL]\n"
"fmla z31.h, p3/M, z0.h, z14.h\n"
- "ld1h { z14.h }, p2/Z, [x9, x23, LSL #1]\n" // Load input point (3, 5)
+ "ld1h { z14.h }, p2/Z, [x9, x23, LSL #1]\n"
"fmla z30.h, p3/M, z0.h, z11.h\n"
"fmla z29.h, p3/M, z0.h, z5.h\n"
"fmla z28.h, p3/M, z0.h, z6.h\n"
- "ld1h { z0.h }, p3/Z, [x8, #-6, MUL VL]\n" // Load from weights and bias
+ "ld1h { z0.h }, p3/Z, [x8, #-6, MUL VL]\n"
"fmla z31.h, p3/M, z1.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x9, x25, LSL #1]\n" // Load input point (3, 3)
+ "ld1h { z11.h }, p2/Z, [x9, x25, LSL #1]\n"
"fmla z30.h, p3/M, z1.h, z12.h\n"
"fmla z29.h, p3/M, z1.h, z6.h\n"
"fmla z28.h, p3/M, z1.h, z10.h\n"
- "ld1h { z1.h }, p3/Z, [x8, #-5, MUL VL]\n" // Load from weights and bias
+ "ld1h { z1.h }, p3/Z, [x8, #-5, MUL VL]\n"
"fmla z31.h, p3/M, z2.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x9, x24, LSL #1]\n" // Load input point (3, 4)
+ "ld1h { z12.h }, p2/Z, [x9, x24, LSL #1]\n"
"addvl x9, x9, #1\n"
"fmla z30.h, p3/M, z2.h, z9.h\n"
"fmla z29.h, p3/M, z2.h, z10.h\n"
"fmla z28.h, p3/M, z2.h, z11.h\n"
- "ld1h { z2.h }, p3/Z, [x8, #-4, MUL VL]\n" // Load from weights and bias
+ "ld1h { z2.h }, p3/Z, [x8, #-4, MUL VL]\n"
"fmla z31.h, p3/M, z3.h, z9.h\n"
- "ld1h { z9.h }, p2/Z, [x28]\n" // Load input point (4, 0)
+ "ld1h { z9.h }, p2/Z, [x28]\n"
"fmla z30.h, p3/M, z3.h, z13.h\n"
"fmla z29.h, p3/M, z3.h, z11.h\n"
"fmla z28.h, p3/M, z3.h, z12.h\n"
- "ld1h { z3.h }, p3/Z, [x8, #-3, MUL VL]\n" // Load from weights and bias
+ "ld1h { z3.h }, p3/Z, [x8, #-3, MUL VL]\n"
"fmla z31.h, p3/M, z4.h, z13.h\n"
- "ld1h { z13.h }, p2/Z, [x28, x15, LSL #1]\n" // Load input point (4, 1)
+ "ld1h { z13.h }, p2/Z, [x28, x15, LSL #1]\n"
"fmla z30.h, p3/M, z4.h, z8.h\n"
- "ld1h { z8.h }, p2/Z, [x28, x24, LSL #1]\n" // Load input point (4, 4)
+ "ld1h { z8.h }, p2/Z, [x28, x24, LSL #1]\n"
"fmla z29.h, p3/M, z4.h, z12.h\n"
"fmla z28.h, p3/M, z4.h, z14.h\n"
- "ld1h { z4.h }, p3/Z, [x8, #-2, MUL VL]\n" // Load from weights and bias
+ "ld1h { z4.h }, p3/Z, [x8, #-2, MUL VL]\n"
"fmla z31.h, p3/M, z0.h, z5.h\n"
- "ld1h { z5.h }, p2/Z, [x28, x26, LSL #1]\n" // Load input point (4, 2)
+ "ld1h { z5.h }, p2/Z, [x28, x26, LSL #1]\n"
"fmla z30.h, p3/M, z0.h, z6.h\n"
"fmla z29.h, p3/M, z0.h, z9.h\n"
"fmla z28.h, p3/M, z0.h, z13.h\n"
- "ld1h { z0.h }, p3/Z, [x8, #-1, MUL VL]\n" // Load from weights and bias
+ "ld1h { z0.h }, p3/Z, [x8, #-1, MUL VL]\n"
"fmla z31.h, p3/M, z1.h, z6.h\n"
- "ld1h { z6.h }, p2/Z, [x28, x25, LSL #1]\n" // Load input point (4, 3)
+ "ld1h { z6.h }, p2/Z, [x28, x25, LSL #1]\n"
"fmla z30.h, p3/M, z1.h, z10.h\n"
"fmla z29.h, p3/M, z1.h, z13.h\n"
"fmla z28.h, p3/M, z1.h, z5.h\n"
- "ld1h { z1.h }, p3/Z, [x8]\n" // Load from weights and bias
+ "ld1h { z1.h }, p3/Z, [x8]\n"
"fmla z31.h, p3/M, z2.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x28, x23, LSL #1]\n" // Load input point (4, 5)
+ "ld1h { z10.h }, p2/Z, [x28, x23, LSL #1]\n"
"addvl x28, x28, #1\n"
"fmla z30.h, p3/M, z2.h, z11.h\n"
"fmla z29.h, p3/M, z2.h, z5.h\n"
"fmla z28.h, p3/M, z2.h, z6.h\n"
- "ld1h { z2.h }, p3/Z, [x8, #1, MUL VL]\n" // Load from weights and bias
+ "ld1h { z2.h }, p3/Z, [x8, #1, MUL VL]\n"
"fmla z31.h, p3/M, z3.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x27]\n" // Load input point (5, 0)
+ "ld1h { z11.h }, p2/Z, [x27]\n"
"fmla z30.h, p3/M, z3.h, z12.h\n"
"fmla z29.h, p3/M, z3.h, z6.h\n"
"fmla z28.h, p3/M, z3.h, z8.h\n"
- "ld1h { z3.h }, p3/Z, [x8, #2, MUL VL]\n" // Load from weights and bias
+ "ld1h { z3.h }, p3/Z, [x8, #2, MUL VL]\n"
"fmla z31.h, p3/M, z4.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x27, x15, LSL #1]\n" // Load input point (5, 1)
+ "ld1h { z12.h }, p2/Z, [x27, x15, LSL #1]\n"
"fmla z30.h, p3/M, z4.h, z14.h\n"
- "ld1h { z14.h }, p1/Z, [x10]\n" // Load input point (2, 0)
+ "ld1h { z14.h }, p1/Z, [x10]\n"
"fmla z29.h, p3/M, z4.h, z8.h\n"
"fmla z28.h, p3/M, z4.h, z10.h\n"
- "ld1h { z4.h }, p3/Z, [x8, #3, MUL VL]\n" // Load from weights and bias
+ "ld1h { z4.h }, p3/Z, [x8, #3, MUL VL]\n"
"fmla z31.h, p3/M, z0.h, z9.h\n"
- "ld1h { z9.h }, p2/Z, [x27, x26, LSL #1]\n" // Load input point (5, 2)
+ "ld1h { z9.h }, p2/Z, [x27, x26, LSL #1]\n"
"fmla z30.h, p3/M, z0.h, z13.h\n"
"fmla z29.h, p3/M, z0.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x27, x25, LSL #1]\n" // Load input point (5, 3)
+ "ld1h { z11.h }, p2/Z, [x27, x25, LSL #1]\n"
"fmla z28.h, p3/M, z0.h, z12.h\n"
- "ld1h { z0.h }, p3/Z, [x8, #5, MUL VL]\n" // Load from weights and bias
+ "ld1h { z0.h }, p3/Z, [x8, #5, MUL VL]\n"
"fmla z31.h, p3/M, z1.h, z13.h\n"
- "ld1h { z13.h }, p1/Z, [x20, x26, LSL #1]\n" // Load input point (1, 2)
+ "ld1h { z13.h }, p1/Z, [x20, x26, LSL #1]\n"
"fmla z30.h, p3/M, z1.h, z5.h\n"
"fmla z29.h, p3/M, z1.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x27, x24, LSL #1]\n" // Load input point (5, 4)
+ "ld1h { z12.h }, p2/Z, [x27, x24, LSL #1]\n"
"fmla z28.h, p3/M, z1.h, z9.h\n"
- "ld1h { z1.h }, p3/Z, [x8, #6, MUL VL]\n" // Load from weights and bias
+ "ld1h { z1.h }, p3/Z, [x8, #6, MUL VL]\n"
"fmla z31.h, p3/M, z2.h, z5.h\n"
- "ld1h { z5.h }, p1/Z, [x13]\n" // Load input point (0, 0)
+ "ld1h { z5.h }, p1/Z, [x13]\n"
"fmla z30.h, p3/M, z2.h, z6.h\n"
"fmla z29.h, p3/M, z2.h, z9.h\n"
- "ld1h { z9.h }, p2/Z, [x27, x23, LSL #1]\n" // Load input point (5, 5)
+ "ld1h { z9.h }, p2/Z, [x27, x23, LSL #1]\n"
"whilelt p2.h, x17, %x[n_channels]\n"
"fmla z28.h, p3/M, z2.h, z11.h\n"
- "ld1h { z2.h }, p3/Z, [x8, #7, MUL VL]\n" // Load from weights and bias
+ "ld1h { z2.h }, p3/Z, [x8, #7, MUL VL]\n"
"addvl x27, x27, #1\n"
"fmla z31.h, p3/M, z3.h, z6.h\n"
- "ld1h { z6.h }, p1/Z, [x13, x15, LSL #1]\n" // Load input point (0, 1)
+ "ld1h { z6.h }, p1/Z, [x13, x15, LSL #1]\n"
"addvl x8, x8, #16\n"
"fmla z30.h, p3/M, z3.h, z8.h\n"
"cmp x16, %x[n_channels]\n"
"fmla z29.h, p3/M, z3.h, z11.h\n"
- "ld1h { z11.h }, p1/Z, [x13, x25, LSL #1]\n" // Load input point (0, 3)
+ "ld1h { z11.h }, p1/Z, [x13, x25, LSL #1]\n"
"fmla z28.h, p3/M, z3.h, z12.h\n"
- "ld1h { z3.h }, p3/Z, [x8, #-8, MUL VL]\n" // Load from weights and bias
+ "ld1h { z3.h }, p3/Z, [x8, #-8, MUL VL]\n"
"fmla z31.h, p3/M, z4.h, z8.h\n"
- "ld1h { z8.h }, p1/Z, [x20, x15, LSL #1]\n" // Load input point (1, 1)
+ "ld1h { z8.h }, p1/Z, [x20, x15, LSL #1]\n"
"fmla z30.h, p3/M, z4.h, z10.h\n"
- "ld1h { z10.h }, p1/Z, [x20, x23, LSL #1]\n" // Load input point (1, 5)
+ "ld1h { z10.h }, p1/Z, [x20, x23, LSL #1]\n"
"fmla z29.h, p3/M, z4.h, z12.h\n"
- "ld1h { z12.h }, p1/Z, [x13, x24, LSL #1]\n" // Load input point (0, 4)
+ "ld1h { z12.h }, p1/Z, [x13, x24, LSL #1]\n"
"fmla z28.h, p3/M, z4.h, z9.h\n"
- "ld1h { z9.h }, p1/Z, [x13, x26, LSL #1]\n" // Load input point (0, 2)
- "ld1h { z4.h }, p3/Z, [x8, #-7, MUL VL]\n" // Load from weights and bias
+ "ld1h { z9.h }, p1/Z, [x13, x26, LSL #1]\n"
+ "ld1h { z4.h }, p3/Z, [x8, #-7, MUL VL]\n"
"fmax z31.h, p3/M, z31.h, z18.h\n"
"addvl x8, x8, #-6\n"
"fmax z30.h, p3/M, z30.h, z18.h\n"
"fmax z29.h, p3/M, z29.h, z18.h\n"
"fmax z28.h, p3/M, z28.h, z18.h\n"
"fmin z31.h, p3/M, z31.h, z17.h\n"
- "st1h { z31.h }, p0, [x11]\n" // Store output point (0, 0)
- "mov z31.d, z16.d\n"
+ "st1h { z31.h }, p0, [x11]\n"
"fmin z30.h, p3/M, z30.h, z17.h\n"
- "st1h { z30.h }, p0, [x11, x12, LSL #1]\n" // Store output point (0, 1)
- "mov z30.d, z16.d\n"
- "addvl x11, x11, #1\n"
"fmin z29.h, p3/M, z29.h, z17.h\n"
- "st1h { z29.h }, p0, [x22]\n" // Store output point (1, 0)
- "mov z29.d, z16.d\n"
+ "st1h { z30.h }, p0, [x11, x12, LSL #1]\n"
"fmin z28.h, p3/M, z28.h, z17.h\n"
- "st1h { z28.h }, p0, [x22, x12, LSL #1]\n" // Store output point (1, 1)
- "mov z28.d, z16.d\n"
+ "addvl x11, x11, #1\n"
+ "st1h { z29.h }, p0, [x22]\n"
+ "st1h { z28.h }, p0, [x22, x12, LSL #1]\n"
"addvl x22, x22, #1\n"
"blt 2b\n"
"3:" // Tile loop: Channel tail
- "fmla z31.h, p3/M, z0.h, z5.h\n"
- "ld1h { z5.h }, p2/Z, [x20, x25, LSL #1]\n" // Load input point (1, 3)
+ "movprfx z31, z16\n fmla z31.h, p3/M, z0.h, z5.h\n"
+ "ld1h { z5.h }, p2/Z, [x20, x25, LSL #1]\n"
"mov p0.b, p2.b\n"
- "fmla z30.h, p3/M, z0.h, z6.h\n"
+ "movprfx z30, z16\n fmla z30.h, p3/M, z0.h, z6.h\n"
"ldr x5, [%x[params_struct], %[offsetof_args_tile_i]]\n"
"add x21, x5, #0x1\n"
- "fmla z29.h, p3/M, z0.h, z7.h\n"
+ "movprfx z29, z16\n fmla z29.h, p3/M, z0.h, z7.h\n"
"ldr x6, [%x[params_struct], %[offsetof_args_tile_j]]\n"
- "fmla z28.h, p3/M, z0.h, z8.h\n"
- "ld1h { z0.h }, p3/Z, [x8]\n" // Load from weights and bias
+ "movprfx z28, z16\n fmla z28.h, p3/M, z0.h, z8.h\n"
+ "ld1h { z0.h }, p3/Z, [x8]\n"
"add x6, x6, #0x1\n"
"fmla z31.h, p3/M, z1.h, z6.h\n"
- "ld1h { z6.h }, p2/Z, [x20, x24, LSL #1]\n" // Load input point (1, 4)
+ "ld1h { z6.h }, p2/Z, [x20, x24, LSL #1]\n"
"fmla z30.h, p3/M, z1.h, z9.h\n"
"ldr x20, [%x[params_struct], %[offsetof_args_n_tile_rows]]\n"
"fmla z29.h, p3/M, z1.h, z8.h\n"
"ldr x19, [%x[params_struct], %[offsetof_args_n_tile_cols]]\n"
"cmp x6, x19\n"
"fmla z28.h, p3/M, z1.h, z13.h\n"
- "ld1h { z1.h }, p3/Z, [x8, #1, MUL VL]\n" // Load from weights and bias
+ "ld1h { z1.h }, p3/Z, [x8, #1, MUL VL]\n"
"fmla z31.h, p3/M, z2.h, z9.h\n"
- "ld1h { z9.h }, p2/Z, [x13, x23, LSL #1]\n" // Load input point (0, 5)
+ "ld1h { z9.h }, p2/Z, [x13, x23, LSL #1]\n"
"csel x6, x6, XZR, LT\n"
"fmla z30.h, p3/M, z2.h, z11.h\n"
"csel x5, x5, x21, LT\n"
"fmla z29.h, p3/M, z2.h, z13.h\n"
"cmp x5, x20\n"
"fmla z28.h, p3/M, z2.h, z5.h\n"
- "ld1h { z2.h }, p3/Z, [x8, #2, MUL VL]\n" // Load from weights and bias
+ "ld1h { z2.h }, p3/Z, [x8, #2, MUL VL]\n"
"fmla z31.h, p3/M, z3.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x10, x15, LSL #1]\n" // Load input point (2, 1)
+ "ld1h { z11.h }, p2/Z, [x10, x15, LSL #1]\n"
"fmla z30.h, p3/M, z3.h, z12.h\n"
"fmla z29.h, p3/M, z3.h, z5.h\n"
"fmla z28.h, p3/M, z3.h, z6.h\n"
- "ld1h { z3.h }, p3/Z, [x8, #3, MUL VL]\n" // Load from weights and bias
+ "ld1h { z3.h }, p3/Z, [x8, #3, MUL VL]\n"
"fmla z31.h, p3/M, z4.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x10, x26, LSL #1]\n" // Load input point (2, 2)
+ "ld1h { z12.h }, p2/Z, [x10, x26, LSL #1]\n"
"fmla z30.h, p3/M, z4.h, z9.h\n"
- "ld1h { z9.h }, p2/Z, [x10, x25, LSL #1]\n" // Load input point (2, 3)
+ "ld1h { z9.h }, p2/Z, [x10, x25, LSL #1]\n"
"fmla z29.h, p3/M, z4.h, z6.h\n"
"fmla z28.h, p3/M, z4.h, z10.h\n"
- "ld1h { z4.h }, p3/Z, [x8, #4, MUL VL]\n" // Load from weights and bias
+ "ld1h { z4.h }, p3/Z, [x8, #4, MUL VL]\n"
"fmla z31.h, p3/M, z0.h, z7.h\n"
"fmla z30.h, p3/M, z0.h, z8.h\n"
"fmla z29.h, p3/M, z0.h, z14.h\n"
"fmla z28.h, p3/M, z0.h, z11.h\n"
- "ld1h { z0.h }, p3/Z, [x8, #5, MUL VL]\n" // Load from weights and bias
+ "ld1h { z0.h }, p3/Z, [x8, #5, MUL VL]\n"
"fmla z31.h, p3/M, z1.h, z8.h\n"
- "ld1h { z8.h }, p2/Z, [x10, x23, LSL #1]\n" // Load input point (2, 5)
+ "ld1h { z8.h }, p2/Z, [x10, x23, LSL #1]\n"
"fmla z30.h, p3/M, z1.h, z13.h\n"
"fmla z29.h, p3/M, z1.h, z11.h\n"
"fmla z28.h, p3/M, z1.h, z12.h\n"
- "ld1h { z1.h }, p3/Z, [x8, #6, MUL VL]\n" // Load from weights and bias
+ "ld1h { z1.h }, p3/Z, [x8, #6, MUL VL]\n"
"fmla z31.h, p3/M, z2.h, z13.h\n"
- "ld1h { z13.h }, p2/Z, [x10, x24, LSL #1]\n" // Load input point (2, 4)
+ "ld1h { z13.h }, p2/Z, [x10, x24, LSL #1]\n"
"fmla z30.h, p3/M, z2.h, z5.h\n"
"fmla z29.h, p3/M, z2.h, z12.h\n"
"fmla z28.h, p3/M, z2.h, z9.h\n"
- "ld1h { z2.h }, p3/Z, [x8, #7, MUL VL]\n" // Load from weights and bias
+ "ld1h { z2.h }, p3/Z, [x8, #7, MUL VL]\n"
"addvl x8, x8, #16\n"
"fmla z31.h, p3/M, z3.h, z5.h\n"
- "ld1h { z5.h }, p2/Z, [x9]\n" // Load input point (3, 0)
+ "ld1h { z5.h }, p2/Z, [x9]\n"
"fmla z30.h, p3/M, z3.h, z6.h\n"
"fmla z29.h, p3/M, z3.h, z9.h\n"
"fmla z28.h, p3/M, z3.h, z13.h\n"
- "ld1h { z3.h }, p3/Z, [x8, #-8, MUL VL]\n" // Load from weights and bias
+ "ld1h { z3.h }, p3/Z, [x8, #-8, MUL VL]\n"
"fmla z31.h, p3/M, z4.h, z6.h\n"
- "ld1h { z6.h }, p2/Z, [x9, x15, LSL #1]\n" // Load input point (3, 1)
+ "ld1h { z6.h }, p2/Z, [x9, x15, LSL #1]\n"
"fmla z30.h, p3/M, z4.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x9, x26, LSL #1]\n" // Load input point (3, 2)
+ "ld1h { z10.h }, p2/Z, [x9, x26, LSL #1]\n"
"fmla z29.h, p3/M, z4.h, z13.h\n"
"fmla z28.h, p3/M, z4.h, z8.h\n"
- "ld1h { z4.h }, p3/Z, [x8, #-7, MUL VL]\n" // Load from weights and bias
+ "ld1h { z4.h }, p3/Z, [x8, #-7, MUL VL]\n"
"fmla z31.h, p3/M, z0.h, z14.h\n"
- "ld1h { z14.h }, p2/Z, [x9, x23, LSL #1]\n" // Load input point (3, 5)
+ "ld1h { z14.h }, p2/Z, [x9, x23, LSL #1]\n"
"fmla z30.h, p3/M, z0.h, z11.h\n"
"fmla z29.h, p3/M, z0.h, z5.h\n"
"fmla z28.h, p3/M, z0.h, z6.h\n"
- "ld1h { z0.h }, p3/Z, [x8, #-6, MUL VL]\n" // Load from weights and bias
+ "ld1h { z0.h }, p3/Z, [x8, #-6, MUL VL]\n"
"fmla z31.h, p3/M, z1.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x9, x25, LSL #1]\n" // Load input point (3, 3)
+ "ld1h { z11.h }, p2/Z, [x9, x25, LSL #1]\n"
"fmla z30.h, p3/M, z1.h, z12.h\n"
"fmla z29.h, p3/M, z1.h, z6.h\n"
"fmla z28.h, p3/M, z1.h, z10.h\n"
- "ld1h { z1.h }, p3/Z, [x8, #-5, MUL VL]\n" // Load from weights and bias
+ "ld1h { z1.h }, p3/Z, [x8, #-5, MUL VL]\n"
"fmla z31.h, p3/M, z2.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x9, x24, LSL #1]\n" // Load input point (3, 4)
+ "ld1h { z12.h }, p2/Z, [x9, x24, LSL #1]\n"
"fmla z30.h, p3/M, z2.h, z9.h\n"
"fmla z29.h, p3/M, z2.h, z10.h\n"
"fmla z28.h, p3/M, z2.h, z11.h\n"
- "ld1h { z2.h }, p3/Z, [x8, #-4, MUL VL]\n" // Load from weights and bias
+ "ld1h { z2.h }, p3/Z, [x8, #-4, MUL VL]\n"
"fmla z31.h, p3/M, z3.h, z9.h\n"
- "ld1h { z9.h }, p2/Z, [x28]\n" // Load input point (4, 0)
+ "ld1h { z9.h }, p2/Z, [x28]\n"
"fmla z30.h, p3/M, z3.h, z13.h\n"
"fmla z29.h, p3/M, z3.h, z11.h\n"
"fmla z28.h, p3/M, z3.h, z12.h\n"
- "ld1h { z3.h }, p3/Z, [x8, #-3, MUL VL]\n" // Load from weights and bias
+ "ld1h { z3.h }, p3/Z, [x8, #-3, MUL VL]\n"
"fmla z31.h, p3/M, z4.h, z13.h\n"
- "ld1h { z13.h }, p2/Z, [x28, x15, LSL #1]\n" // Load input point (4, 1)
+ "ld1h { z13.h }, p2/Z, [x28, x15, LSL #1]\n"
"fmla z30.h, p3/M, z4.h, z8.h\n"
- "ld1h { z8.h }, p2/Z, [x28, x24, LSL #1]\n" // Load input point (4, 4)
+ "ld1h { z8.h }, p2/Z, [x28, x24, LSL #1]\n"
"fmla z29.h, p3/M, z4.h, z12.h\n"
"fmla z28.h, p3/M, z4.h, z14.h\n"
- "ld1h { z4.h }, p3/Z, [x8, #-2, MUL VL]\n" // Load from weights and bias
+ "ld1h { z4.h }, p3/Z, [x8, #-2, MUL VL]\n"
"fmla z31.h, p3/M, z0.h, z5.h\n"
- "ld1h { z5.h }, p2/Z, [x28, x26, LSL #1]\n" // Load input point (4, 2)
+ "ld1h { z5.h }, p2/Z, [x28, x26, LSL #1]\n"
"fmla z30.h, p3/M, z0.h, z6.h\n"
"fmla z29.h, p3/M, z0.h, z9.h\n"
"fmla z28.h, p3/M, z0.h, z13.h\n"
- "ld1h { z0.h }, p3/Z, [x8, #-1, MUL VL]\n" // Load from weights and bias
+ "ld1h { z0.h }, p3/Z, [x8, #-1, MUL VL]\n"
"fmla z31.h, p3/M, z1.h, z6.h\n"
- "ld1h { z6.h }, p2/Z, [x28, x25, LSL #1]\n" // Load input point (4, 3)
+ "ld1h { z6.h }, p2/Z, [x28, x25, LSL #1]\n"
"fmla z30.h, p3/M, z1.h, z10.h\n"
"fmla z29.h, p3/M, z1.h, z13.h\n"
"fmla z28.h, p3/M, z1.h, z5.h\n"
- "ld1h { z1.h }, p3/Z, [x8]\n" // Load from weights and bias
+ "ld1h { z1.h }, p3/Z, [x8]\n"
"fmla z31.h, p3/M, z2.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x28, x23, LSL #1]\n" // Load input point (4, 5)
+ "ld1h { z10.h }, p2/Z, [x28, x23, LSL #1]\n"
"fmla z30.h, p3/M, z2.h, z11.h\n"
"fmla z29.h, p3/M, z2.h, z5.h\n"
"fmla z28.h, p3/M, z2.h, z6.h\n"
- "ld1h { z2.h }, p3/Z, [x8, #1, MUL VL]\n" // Load from weights and bias
+ "ld1h { z2.h }, p3/Z, [x8, #1, MUL VL]\n"
"fmla z31.h, p3/M, z3.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x27]\n" // Load input point (5, 0)
+ "ld1h { z11.h }, p2/Z, [x27]\n"
"fmla z30.h, p3/M, z3.h, z12.h\n"
"fmla z29.h, p3/M, z3.h, z6.h\n"
"fmla z28.h, p3/M, z3.h, z8.h\n"
- "ld1h { z3.h }, p3/Z, [x8, #2, MUL VL]\n" // Load from weights and bias
+ "ld1h { z3.h }, p3/Z, [x8, #2, MUL VL]\n"
"fmla z31.h, p3/M, z4.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x27, x15, LSL #1]\n" // Load input point (5, 1)
+ "ld1h { z12.h }, p2/Z, [x27, x15, LSL #1]\n"
"fmla z30.h, p3/M, z4.h, z14.h\n"
"fmla z29.h, p3/M, z4.h, z8.h\n"
"fmla z28.h, p3/M, z4.h, z10.h\n"
- "ld1h { z4.h }, p3/Z, [x8, #3, MUL VL]\n" // Load from weights and bias
+ "ld1h { z4.h }, p3/Z, [x8, #3, MUL VL]\n"
"fmla z31.h, p3/M, z0.h, z9.h\n"
- "ld1h { z9.h }, p2/Z, [x27, x26, LSL #1]\n" // Load input point (5, 2)
+ "ld1h { z9.h }, p2/Z, [x27, x26, LSL #1]\n"
"fmla z30.h, p3/M, z0.h, z13.h\n"
"fmla z29.h, p3/M, z0.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x27, x25, LSL #1]\n" // Load input point (5, 3)
+ "ld1h { z11.h }, p2/Z, [x27, x25, LSL #1]\n"
"fmla z28.h, p3/M, z0.h, z12.h\n"
"fmla z31.h, p3/M, z1.h, z13.h\n"
"fmla z30.h, p3/M, z1.h, z5.h\n"
"fmla z29.h, p3/M, z1.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x27, x24, LSL #1]\n" // Load input point (5, 4)
+ "ld1h { z12.h }, p2/Z, [x27, x24, LSL #1]\n"
"fmla z28.h, p3/M, z1.h, z9.h\n"
"fmla z31.h, p3/M, z2.h, z5.h\n"
"fmla z30.h, p3/M, z2.h, z6.h\n"
"fmla z29.h, p3/M, z2.h, z9.h\n"
- "ld1h { z9.h }, p2/Z, [x27, x23, LSL #1]\n" // Load input point (5, 5)
+ "ld1h { z9.h }, p2/Z, [x27, x23, LSL #1]\n"
"fmla z28.h, p3/M, z2.h, z11.h\n"
"fmla z31.h, p3/M, z3.h, z6.h\n"
"fmla z30.h, p3/M, z3.h, z8.h\n"
@@ -511,13 +503,13 @@
"fmax z29.h, p3/M, z29.h, z18.h\n"
"fmax z28.h, p3/M, z28.h, z18.h\n"
"fmin z31.h, p3/M, z31.h, z17.h\n"
- "st1h { z31.h }, p0, [x11]\n" // Store output point (0, 0)
+ "st1h { z31.h }, p0, [x11]\n"
"fmin z30.h, p3/M, z30.h, z17.h\n"
"fmin z29.h, p3/M, z29.h, z17.h\n"
- "st1h { z30.h }, p0, [x11, x12, LSL #1]\n" // Store output point (0, 1)
+ "st1h { z30.h }, p0, [x11, x12, LSL #1]\n"
"fmin z28.h, p3/M, z28.h, z17.h\n"
- "st1h { z29.h }, p0, [x22]\n" // Store output point (1, 0)
- "st1h { z28.h }, p0, [x22, x12, LSL #1]\n" // Store output point (1, 1)
+ "st1h { z29.h }, p0, [x22]\n"
+ "st1h { z28.h }, p0, [x22, x12, LSL #1]\n"
"blt 1b\n"
:
: [n_channels] "r" ((unsigned long) n_channels), [offsetof_args_inptr] "I" (offsetof(Args, inptr)), [offsetof_args_ld_input_col] "I" (offsetof(Args, ld_input_col)), [offsetof_args_ld_input_row] "I" (offsetof(Args, ld_input_row)), [offsetof_args_ld_output_col] "I" (offsetof(Args, ld_output_col)), [offsetof_args_ld_output_row] "I" (offsetof(Args, ld_output_row)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_n_tile_cols] "I" (offsetof(Args, n_tile_cols)), [offsetof_args_n_tile_rows] "I" (offsetof(Args, n_tile_rows)), [offsetof_args_outptr] "I" (offsetof(Args, outptr)), [offsetof_args_params] "I" (offsetof(Args, params)), [offsetof_args_tile_i] "I" (offsetof(Args, tile_i)), [offsetof_args_tile_j] "I" (offsetof(Args, tile_j)), [params_struct] "r" (¶ms_struct)
@@ -528,4 +520,4 @@
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
+#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_indirect.cpp
index be128b4..a0640da 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_indirect.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_indirect.cpp
@@ -25,7 +25,7 @@
#include <cstddef>
#include <cstdint>
-#if defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
+#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
namespace arm_conv {
namespace depthwise {
@@ -100,432 +100,424 @@
__asm__ __volatile__(
"ldr x19, [%x[params_struct], %[offsetof_args_outptrs]]\n"
"ptrue p3.b\n"
- "ldr x5, [%x[params_struct], %[offsetof_args_params]]\n"
- "add x6, %x[params_struct], %[offsetof_Args_inptrs]\n"
+ "ldr x15, [%x[params_struct], %[offsetof_args_params]]\n"
+ "add x14, %x[params_struct], %[offsetof_Args_inptrs]\n"
"ld1rh { z18.h }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n"
- "mov x7, #0x0\n"
+ "mov x13, #0x0\n"
"ld1rh { z17.h }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n"
- "cnth x8\n"
- "ldp x17, x16, [x19, #0x0]\n"
- "sub x15, XZR, x8\n"
- "ldp x14, x13, [x19, #0x10]\n"
+ "cnth x12\n"
+ "ldp x11, x10, [x19, #0x0]\n"
+ "sub x9, XZR, x12\n"
+ "ldp x28, x27, [x19, #0x10]\n"
"whilelt p2.h, XZR, %x[n_channels]\n"
- "ld1h { z16.h }, p3/Z, [x5]\n" // Load from weights and bias
- "mov z31.d, z16.d\n"
- "ld1h { z0.h }, p3/Z, [x5, #1, MUL VL]\n" // Load from weights and bias
- "cmp x8, %x[n_channels]\n"
- "mov z30.d, z16.d\n"
- "ld1h { z1.h }, p3/Z, [x5, #2, MUL VL]\n" // Load from weights and bias
- "mov z29.d, z16.d\n"
- "ld1h { z2.h }, p3/Z, [x5, #3, MUL VL]\n" // Load from weights and bias
- "mov z28.d, z16.d\n"
- "ld1h { z3.h }, p3/Z, [x5, #4, MUL VL]\n" // Load from weights and bias
- "ld1h { z4.h }, p3/Z, [x5, #5, MUL VL]\n" // Load from weights and bias
- "addvl x5, x5, #6\n"
- "ldp x12, x11, [x6, #0x0]\n"
- "ldp x10, x9, [x6, #0x10]\n"
- "ldp x20, x28, [x6, #0x20]\n"
- "ld1h { z5.h }, p2/Z, [x12, x7, LSL #1]\n"
- "ld1h { z6.h }, p2/Z, [x11, x7, LSL #1]\n"
- "ld1h { z7.h }, p2/Z, [x10, x7, LSL #1]\n"
- "ld1h { z8.h }, p2/Z, [x9, x7, LSL #1]\n"
- "ld1h { z9.h }, p2/Z, [x20, x7, LSL #1]\n"
- "ld1h { z13.h }, p2/Z, [x28, x7, LSL #1]\n"
- "ldp x27, x19, [x6, #0x30]\n"
- "ldp x26, x25, [x6, #0x40]\n"
- "ld1h { z11.h }, p2/Z, [x27, x7, LSL #1]\n"
- "ld1h { z12.h }, p2/Z, [x19, x7, LSL #1]\n"
- "ld1h { z10.h }, p2/Z, [x26, x7, LSL #1]\n"
- "ld1h { z14.h }, p2/Z, [x25, x7, LSL #1]\n"
+ "ld1h { z16.h }, p3/Z, [x15]\n"
+ "cmp x12, %x[n_channels]\n"
+ "ld1h { z0.h }, p3/Z, [x15, #1, MUL VL]\n"
+ "ld1h { z1.h }, p3/Z, [x15, #2, MUL VL]\n"
+ "ld1h { z2.h }, p3/Z, [x15, #3, MUL VL]\n"
+ "ld1h { z3.h }, p3/Z, [x15, #4, MUL VL]\n"
+ "ld1h { z4.h }, p3/Z, [x15, #5, MUL VL]\n"
+ "addvl x15, x15, #6\n"
+ "ldp x26, x25, [x14, #0x0]\n"
+ "ldp x24, x23, [x14, #0x10]\n"
+ "ldp x22, x21, [x14, #0x20]\n"
+ "ld1h { z5.h }, p2/Z, [x26, x13, LSL #1]\n"
+ "ld1h { z6.h }, p2/Z, [x25, x13, LSL #1]\n"
+ "ld1h { z7.h }, p2/Z, [x24, x13, LSL #1]\n"
+ "ld1h { z8.h }, p2/Z, [x23, x13, LSL #1]\n"
+ "ld1h { z9.h }, p2/Z, [x22, x13, LSL #1]\n"
+ "ld1h { z13.h }, p2/Z, [x21, x13, LSL #1]\n"
+ "ldp x20, x19, [x14, #0x30]\n"
+ "ldp x26, x25, [x14, #0x40]\n"
+ "ld1h { z11.h }, p2/Z, [x20, x13, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x19, x13, LSL #1]\n"
+ "ld1h { z10.h }, p2/Z, [x26, x13, LSL #1]\n"
+ "ld1h { z14.h }, p2/Z, [x25, x13, LSL #1]\n"
"bge 2f\n"
"1:" // Channel loop
- "fmla z31.h, p3/M, z0.h, z5.h\n"
- "ldr x24, [x6, #0x50]\n"
- "whilelt p1.h, x8, %x[n_channels]\n"
- "fmla z30.h, p3/M, z0.h, z6.h\n"
- "ldr x23, [x6, #0x58]\n"
- "inch x15\n"
- "fmla z29.h, p3/M, z0.h, z7.h\n"
- "ldr x22, [x6, #0x60]\n"
+ "movprfx z31, z16\n fmla z31.h, p3/M, z0.h, z5.h\n"
+ "ldr x24, [x14, #0x50]\n"
+ "whilelt p1.h, x12, %x[n_channels]\n"
+ "movprfx z30, z16\n fmla z30.h, p3/M, z0.h, z6.h\n"
+ "ldr x23, [x14, #0x58]\n"
+ "inch x9\n"
+ "movprfx z29, z16\n fmla z29.h, p3/M, z0.h, z7.h\n"
+ "ldr x22, [x14, #0x60]\n"
"mov p0.b, p2.b\n"
- "fmla z28.h, p3/M, z0.h, z8.h\n"
- "ld1h { z5.h }, p2/Z, [x24, x7, LSL #1]\n"
- "ld1h { z0.h }, p3/Z, [x5]\n" // Load from weights and bias
+ "movprfx z28, z16\n fmla z28.h, p3/M, z0.h, z8.h\n"
+ "ld1h { z5.h }, p2/Z, [x24, x13, LSL #1]\n"
+ "ld1h { z0.h }, p3/Z, [x15]\n"
"fmla z31.h, p3/M, z1.h, z6.h\n"
- "ld1h { z6.h }, p2/Z, [x23, x7, LSL #1]\n"
+ "ld1h { z6.h }, p2/Z, [x23, x13, LSL #1]\n"
"fmla z30.h, p3/M, z1.h, z9.h\n"
- "ldr x21, [x6, #0x68]\n"
+ "ldr x21, [x14, #0x68]\n"
"fmla z29.h, p3/M, z1.h, z8.h\n"
- "ldr x20, [x6, #0x70]\n"
+ "ldr x20, [x14, #0x70]\n"
"fmla z28.h, p3/M, z1.h, z13.h\n"
- "ld1h { z1.h }, p3/Z, [x5, #1, MUL VL]\n" // Load from weights and bias
+ "ld1h { z1.h }, p3/Z, [x15, #1, MUL VL]\n"
"fmla z31.h, p3/M, z2.h, z9.h\n"
- "ld1h { z9.h }, p2/Z, [x22, x7, LSL #1]\n"
+ "ld1h { z9.h }, p2/Z, [x22, x13, LSL #1]\n"
"fmla z30.h, p3/M, z2.h, z11.h\n"
- "ldr x19, [x6, #0x78]\n"
+ "ldr x19, [x14, #0x78]\n"
"fmla z29.h, p3/M, z2.h, z13.h\n"
- "ldr x12, [x6, #0x80]\n"
+ "ldr x26, [x14, #0x80]\n"
"fmla z28.h, p3/M, z2.h, z5.h\n"
- "ld1h { z2.h }, p3/Z, [x5, #2, MUL VL]\n" // Load from weights and bias
+ "ld1h { z2.h }, p3/Z, [x15, #2, MUL VL]\n"
"fmla z31.h, p3/M, z3.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x21, x7, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x21, x13, LSL #1]\n"
"fmla z30.h, p3/M, z3.h, z12.h\n"
- "ldr x11, [x6, #0x88]\n"
+ "ldr x25, [x14, #0x88]\n"
"fmla z29.h, p3/M, z3.h, z5.h\n"
- "ldr x10, [x6, #0x90]\n"
+ "ldr x24, [x14, #0x90]\n"
"fmla z28.h, p3/M, z3.h, z6.h\n"
- "ld1h { z3.h }, p3/Z, [x5, #3, MUL VL]\n" // Load from weights and bias
+ "ld1h { z3.h }, p3/Z, [x15, #3, MUL VL]\n"
"fmla z31.h, p3/M, z4.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x20, x7, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x20, x13, LSL #1]\n"
"fmla z30.h, p3/M, z4.h, z9.h\n"
- "ld1h { z9.h }, p2/Z, [x19, x7, LSL #1]\n"
+ "ld1h { z9.h }, p2/Z, [x19, x13, LSL #1]\n"
"fmla z29.h, p3/M, z4.h, z6.h\n"
- "ldr x9, [x6, #0x98]\n"
+ "ldr x23, [x14, #0x98]\n"
"fmla z28.h, p3/M, z4.h, z10.h\n"
- "ld1h { z4.h }, p3/Z, [x5, #4, MUL VL]\n" // Load from weights and bias
+ "ld1h { z4.h }, p3/Z, [x15, #4, MUL VL]\n"
"fmla z31.h, p3/M, z0.h, z7.h\n"
- "ldr x20, [x6, #0xa0]\n"
+ "ldr x22, [x14, #0xa0]\n"
"fmla z30.h, p3/M, z0.h, z8.h\n"
- "ldr x28, [x6, #0xa8]\n"
+ "ldr x21, [x14, #0xa8]\n"
"fmla z29.h, p3/M, z0.h, z14.h\n"
- "ldr x27, [x6, #0xb0]\n"
+ "ldr x20, [x14, #0xb0]\n"
"fmla z28.h, p3/M, z0.h, z11.h\n"
- "ld1h { z0.h }, p3/Z, [x5, #5, MUL VL]\n" // Load from weights and bias
+ "ld1h { z0.h }, p3/Z, [x15, #5, MUL VL]\n"
"fmla z31.h, p3/M, z1.h, z8.h\n"
- "ld1h { z8.h }, p2/Z, [x11, x7, LSL #1]\n"
+ "ld1h { z8.h }, p2/Z, [x25, x13, LSL #1]\n"
"fmla z30.h, p3/M, z1.h, z13.h\n"
- "ldr x19, [x6, #0xb8]\n"
+ "ldr x19, [x14, #0xb8]\n"
"fmla z29.h, p3/M, z1.h, z11.h\n"
- "ldr x26, [x6, #0xc0]\n"
+ "ldr x25, [x14, #0xc8]\n"
"fmla z28.h, p3/M, z1.h, z12.h\n"
- "ld1h { z1.h }, p3/Z, [x5, #6, MUL VL]\n" // Load from weights and bias
+ "ld1h { z1.h }, p3/Z, [x15, #6, MUL VL]\n"
"fmla z31.h, p3/M, z2.h, z13.h\n"
- "ld1h { z13.h }, p2/Z, [x12, x7, LSL #1]\n"
+ "ld1h { z13.h }, p2/Z, [x26, x13, LSL #1]\n"
"fmla z30.h, p3/M, z2.h, z5.h\n"
- "ldr x25, [x6, #0xc8]\n"
+ "ldr x26, [x14, #0xc0]\n"
"fmla z29.h, p3/M, z2.h, z12.h\n"
- "ldr x24, [x6, #0xd0]\n"
"fmla z28.h, p3/M, z2.h, z9.h\n"
- "ld1h { z2.h }, p3/Z, [x5, #7, MUL VL]\n" // Load from weights and bias
+ "ld1h { z2.h }, p3/Z, [x15, #7, MUL VL]\n"
+ "addvl x15, x15, #16\n"
"fmla z31.h, p3/M, z3.h, z5.h\n"
- "addvl x5, x5, #16\n"
+ "ld1h { z5.h }, p2/Z, [x24, x13, LSL #1]\n"
+ "ldr x24, [x14, #0xd0]\n"
"fmla z30.h, p3/M, z3.h, z6.h\n"
- "ld1h { z5.h }, p2/Z, [x10, x7, LSL #1]\n"
- "ldr x23, [x6, #0xd8]\n"
+ "ld1h { z16.h }, p3/Z, [x15, #4, MUL VL]\n"
"fmla z29.h, p3/M, z3.h, z9.h\n"
- "ldr x22, [x6, #0xe0]\n"
"fmla z28.h, p3/M, z3.h, z13.h\n"
- "ld1h { z3.h }, p3/Z, [x5, #-8, MUL VL]\n" // Load from weights and bias
+ "ld1h { z3.h }, p3/Z, [x15, #-8, MUL VL]\n"
"fmla z31.h, p3/M, z4.h, z6.h\n"
- "ld1h { z6.h }, p2/Z, [x9, x7, LSL #1]\n"
+ "ld1h { z6.h }, p2/Z, [x23, x13, LSL #1]\n"
+ "ldr x23, [x14, #0xd8]\n"
"fmla z30.h, p3/M, z4.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x20, x7, LSL #1]\n"
+ "ld1h { z10.h }, p2/Z, [x22, x13, LSL #1]\n"
"fmla z29.h, p3/M, z4.h, z13.h\n"
- "ldr x21, [x6, #0xe8]\n"
+ "ldr x22, [x14, #0xe0]\n"
"fmla z28.h, p3/M, z4.h, z8.h\n"
- "ld1h { z4.h }, p3/Z, [x5, #-7, MUL VL]\n" // Load from weights and bias
+ "ld1h { z4.h }, p3/Z, [x15, #-7, MUL VL]\n"
"fmla z31.h, p3/M, z0.h, z14.h\n"
- "ld1h { z14.h }, p2/Z, [x19, x7, LSL #1]\n"
+ "ld1h { z14.h }, p2/Z, [x19, x13, LSL #1]\n"
"fmla z30.h, p3/M, z0.h, z11.h\n"
- "ldr x20, [x6, #0xf0]\n"
+ "ldr x19, [x14, #0xf8]\n"
"fmla z29.h, p3/M, z0.h, z5.h\n"
- "ldr x19, [x6, #0xf8]\n"
"fmla z28.h, p3/M, z0.h, z6.h\n"
- "ld1h { z0.h }, p3/Z, [x5, #-6, MUL VL]\n" // Load from weights and bias
+ "ld1h { z0.h }, p3/Z, [x15, #-6, MUL VL]\n"
"fmla z31.h, p3/M, z1.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x28, x7, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x21, x13, LSL #1]\n"
+ "ldr x21, [x14, #0xe8]\n"
"fmla z30.h, p3/M, z1.h, z12.h\n"
- "ldr x12, [x6, #0x100]\n"
"fmla z29.h, p3/M, z1.h, z6.h\n"
- "ldr x11, [x6, #0x108]\n"
"fmla z28.h, p3/M, z1.h, z10.h\n"
- "ld1h { z1.h }, p3/Z, [x5, #-5, MUL VL]\n" // Load from weights and bias
+ "ld1h { z1.h }, p3/Z, [x15, #-5, MUL VL]\n"
"fmla z31.h, p3/M, z2.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x27, x7, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x20, x13, LSL #1]\n"
+ "ldr x20, [x14, #0xf0]\n"
"fmla z30.h, p3/M, z2.h, z9.h\n"
- "ldr x10, [x6, #0x110]\n"
"fmla z29.h, p3/M, z2.h, z10.h\n"
- "ldr x9, [x6, #0x118]\n"
"fmla z28.h, p3/M, z2.h, z11.h\n"
- "ld1h { z2.h }, p3/Z, [x5, #-4, MUL VL]\n" // Load from weights and bias
+ "ld1h { z2.h }, p3/Z, [x15, #-4, MUL VL]\n"
"fmla z31.h, p3/M, z3.h, z9.h\n"
- "ld1h { z9.h }, p2/Z, [x26, x7, LSL #1]\n"
+ "ld1h { z9.h }, p2/Z, [x26, x13, LSL #1]\n"
+ "ldr x26, [x14, #0x100]\n"
"fmla z30.h, p3/M, z3.h, z13.h\n"
- "ld1h { z16.h }, p3/Z, [x5, #4, MUL VL]\n" // Load from weights and bias
"fmla z29.h, p3/M, z3.h, z11.h\n"
"fmla z28.h, p3/M, z3.h, z12.h\n"
- "ld1h { z3.h }, p3/Z, [x5, #-3, MUL VL]\n" // Load from weights and bias
+ "ld1h { z3.h }, p3/Z, [x15, #-3, MUL VL]\n"
"fmla z31.h, p3/M, z4.h, z13.h\n"
- "ld1h { z13.h }, p2/Z, [x25, x7, LSL #1]\n"
+ "ld1h { z13.h }, p2/Z, [x25, x13, LSL #1]\n"
+ "ldr x25, [x14, #0x108]\n"
"fmla z30.h, p3/M, z4.h, z8.h\n"
- "ld1h { z8.h }, p2/Z, [x22, x7, LSL #1]\n"
+ "ld1h { z8.h }, p2/Z, [x22, x13, LSL #1]\n"
"fmla z29.h, p3/M, z4.h, z12.h\n"
"fmla z28.h, p3/M, z4.h, z14.h\n"
- "ld1h { z4.h }, p3/Z, [x5, #-2, MUL VL]\n" // Load from weights and bias
+ "ld1h { z4.h }, p3/Z, [x15, #-2, MUL VL]\n"
"fmla z31.h, p3/M, z0.h, z5.h\n"
- "ld1h { z5.h }, p2/Z, [x24, x7, LSL #1]\n"
+ "ld1h { z5.h }, p2/Z, [x24, x13, LSL #1]\n"
+ "ldr x24, [x14, #0x110]\n"
"fmla z30.h, p3/M, z0.h, z6.h\n"
"fmla z29.h, p3/M, z0.h, z9.h\n"
"fmla z28.h, p3/M, z0.h, z13.h\n"
- "ld1h { z0.h }, p3/Z, [x5, #-1, MUL VL]\n" // Load from weights and bias
+ "ld1h { z0.h }, p3/Z, [x15, #-1, MUL VL]\n"
"fmla z31.h, p3/M, z1.h, z6.h\n"
- "ld1h { z6.h }, p2/Z, [x23, x7, LSL #1]\n"
+ "ld1h { z6.h }, p2/Z, [x23, x13, LSL #1]\n"
+ "ldr x23, [x14, #0x118]\n"
"fmla z30.h, p3/M, z1.h, z10.h\n"
"fmla z29.h, p3/M, z1.h, z13.h\n"
"fmla z28.h, p3/M, z1.h, z5.h\n"
- "ld1h { z1.h }, p3/Z, [x5]\n" // Load from weights and bias
+ "ld1h { z1.h }, p3/Z, [x15]\n"
"fmla z31.h, p3/M, z2.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x21, x7, LSL #1]\n"
+ "ld1h { z10.h }, p2/Z, [x21, x13, LSL #1]\n"
"fmla z30.h, p3/M, z2.h, z11.h\n"
"fmla z29.h, p3/M, z2.h, z5.h\n"
"fmla z28.h, p3/M, z2.h, z6.h\n"
- "ld1h { z2.h }, p3/Z, [x5, #1, MUL VL]\n" // Load from weights and bias
+ "ld1h { z2.h }, p3/Z, [x15, #1, MUL VL]\n"
"fmla z31.h, p3/M, z3.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x20, x7, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x20, x13, LSL #1]\n"
"fmla z30.h, p3/M, z3.h, z12.h\n"
"fmla z29.h, p3/M, z3.h, z6.h\n"
"fmla z28.h, p3/M, z3.h, z8.h\n"
- "ld1h { z3.h }, p3/Z, [x5, #2, MUL VL]\n" // Load from weights and bias
+ "ld1h { z3.h }, p3/Z, [x15, #2, MUL VL]\n"
"fmla z31.h, p3/M, z4.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x19, x7, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x19, x13, LSL #1]\n"
"fmla z30.h, p3/M, z4.h, z14.h\n"
"fmla z29.h, p3/M, z4.h, z8.h\n"
"fmla z28.h, p3/M, z4.h, z10.h\n"
- "ld1h { z4.h }, p3/Z, [x5, #3, MUL VL]\n" // Load from weights and bias
+ "ld1h { z4.h }, p3/Z, [x15, #3, MUL VL]\n"
"fmla z31.h, p3/M, z0.h, z9.h\n"
- "ld1h { z9.h }, p2/Z, [x12, x7, LSL #1]\n"
+ "ld1h { z9.h }, p2/Z, [x26, x13, LSL #1]\n"
"fmla z30.h, p3/M, z0.h, z13.h\n"
"fmla z29.h, p3/M, z0.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x11, x7, LSL #1]\n"
- "ldp x12, x11, [x6, #0x0]\n"
+ "ld1h { z11.h }, p2/Z, [x25, x13, LSL #1]\n"
+ "ldp x26, x25, [x14, #0x0]\n"
"fmla z28.h, p3/M, z0.h, z12.h\n"
- "ld1h { z0.h }, p3/Z, [x5, #5, MUL VL]\n" // Load from weights and bias
+ "ld1h { z0.h }, p3/Z, [x15, #5, MUL VL]\n"
"fmla z31.h, p3/M, z1.h, z13.h\n"
"fmla z30.h, p3/M, z1.h, z5.h\n"
"fmla z29.h, p3/M, z1.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x10, x7, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x24, x13, LSL #1]\n"
"fmla z28.h, p3/M, z1.h, z9.h\n"
- "ld1h { z1.h }, p3/Z, [x5, #6, MUL VL]\n" // Load from weights and bias
+ "ld1h { z1.h }, p3/Z, [x15, #6, MUL VL]\n"
"fmla z31.h, p3/M, z2.h, z5.h\n"
- "ld1h { z5.h }, p1/Z, [x12, x8, LSL #1]\n"
+ "ld1h { z5.h }, p1/Z, [x26, x12, LSL #1]\n"
"fmla z30.h, p3/M, z2.h, z6.h\n"
"fmla z29.h, p3/M, z2.h, z9.h\n"
- "ld1h { z9.h }, p2/Z, [x9, x7, LSL #1]\n"
- "inch x7\n"
+ "ld1h { z9.h }, p2/Z, [x23, x13, LSL #1]\n"
+ "inch x13\n"
"fmla z28.h, p3/M, z2.h, z11.h\n"
- "ldp x10, x9, [x6, #0x10]\n"
- "whilelt p2.h, x7, %x[n_channels]\n"
+ "ldp x24, x23, [x14, #0x10]\n"
+ "whilelt p2.h, x13, %x[n_channels]\n"
"fmla z31.h, p3/M, z3.h, z6.h\n"
- "ld1h { z6.h }, p1/Z, [x11, x8, LSL #1]\n"
- "ldp x20, x28, [x6, #0x20]\n"
+ "ld1h { z6.h }, p1/Z, [x25, x12, LSL #1]\n"
+ "ldp x22, x21, [x14, #0x20]\n"
"fmla z30.h, p3/M, z3.h, z8.h\n"
- "ldp x27, x19, [x6, #0x30]\n"
+ "ldp x20, x19, [x14, #0x30]\n"
"fmla z29.h, p3/M, z3.h, z11.h\n"
- "ld1h { z7.h }, p1/Z, [x10, x8, LSL #1]\n"
+ "ld1h { z7.h }, p1/Z, [x24, x12, LSL #1]\n"
"fmla z28.h, p3/M, z3.h, z12.h\n"
- "ld1h { z13.h }, p1/Z, [x28, x8, LSL #1]\n"
+ "ld1h { z13.h }, p1/Z, [x21, x12, LSL #1]\n"
"fmla z31.h, p3/M, z4.h, z8.h\n"
- "ld1h { z8.h }, p1/Z, [x9, x8, LSL #1]\n"
+ "ld1h { z8.h }, p1/Z, [x23, x12, LSL #1]\n"
"fmla z30.h, p3/M, z4.h, z10.h\n"
- "ld1h { z11.h }, p1/Z, [x27, x8, LSL #1]\n"
+ "ld1h { z11.h }, p1/Z, [x20, x12, LSL #1]\n"
"fmla z29.h, p3/M, z4.h, z12.h\n"
- "ld1h { z12.h }, p1/Z, [x19, x8, LSL #1]\n"
+ "ld1h { z12.h }, p1/Z, [x19, x12, LSL #1]\n"
"fmla z28.h, p3/M, z4.h, z9.h\n"
- "ld1h { z9.h }, p1/Z, [x20, x8, LSL #1]\n"
+ "ld1h { z9.h }, p1/Z, [x22, x12, LSL #1]\n"
"fmax z31.h, p3/M, z31.h, z18.h\n"
- "ldp x26, x25, [x6, #0x40]\n"
+ "ldp x26, x25, [x14, #0x40]\n"
"fmax z30.h, p3/M, z30.h, z18.h\n"
- "ld1h { z2.h }, p3/Z, [x5, #7, MUL VL]\n" // Load from weights and bias
+ "ld1h { z2.h }, p3/Z, [x15, #7, MUL VL]\n"
"fmax z29.h, p3/M, z29.h, z18.h\n"
- "addvl x5, x5, #16\n"
+ "addvl x15, x15, #16\n"
"fmax z28.h, p3/M, z28.h, z18.h\n"
- "ld1h { z10.h }, p1/Z, [x26, x8, LSL #1]\n"
- "ld1h { z14.h }, p1/Z, [x25, x8, LSL #1]\n"
+ "ld1h { z10.h }, p1/Z, [x26, x12, LSL #1]\n"
+ "ld1h { z14.h }, p1/Z, [x25, x12, LSL #1]\n"
"fmin z31.h, p3/M, z31.h, z17.h\n"
- "inch x8\n"
+ "inch x12\n"
"fmin z30.h, p3/M, z30.h, z17.h\n"
- "ld1h { z3.h }, p3/Z, [x5, #-8, MUL VL]\n" // Load from weights and bias
- "cmp x8, %x[n_channels]\n"
+ "ld1h { z3.h }, p3/Z, [x15, #-8, MUL VL]\n"
+ "cmp x12, %x[n_channels]\n"
"fmin z29.h, p3/M, z29.h, z17.h\n"
- "ld1h { z4.h }, p3/Z, [x5, #-7, MUL VL]\n" // Load from weights and bias
- "addvl x5, x5, #-6\n"
+ "ld1h { z4.h }, p3/Z, [x15, #-7, MUL VL]\n"
+ "addvl x15, x15, #-6\n"
"fmin z28.h, p3/M, z28.h, z17.h\n"
- "st1h { z31.h }, p0, [x17, x15, LSL #1]\n"
- "mov z31.d, z16.d\n"
- "st1h { z30.h }, p0, [x16, x15, LSL #1]\n"
- "mov z30.d, z16.d\n"
- "st1h { z29.h }, p0, [x14, x15, LSL #1]\n"
- "mov z29.d, z16.d\n"
- "st1h { z28.h }, p0, [x13, x15, LSL #1]\n"
- "mov z28.d, z16.d\n"
+ "st1h { z31.h }, p0, [x11, x9, LSL #1]\n"
+ "st1h { z30.h }, p0, [x10, x9, LSL #1]\n"
+ "st1h { z29.h }, p0, [x28, x9, LSL #1]\n"
+ "st1h { z28.h }, p0, [x27, x9, LSL #1]\n"
"blt 1b\n"
"2:" // Channel tail
- "fmla z31.h, p3/M, z0.h, z5.h\n"
- "ldr x24, [x6, #0x50]\n"
- "inch x15\n"
- "fmla z30.h, p3/M, z0.h, z6.h\n"
- "ldr x23, [x6, #0x58]\n"
+ "movprfx z31, z16\n fmla z31.h, p3/M, z0.h, z5.h\n"
+ "ldr x24, [x14, #0x50]\n"
+ "inch x9\n"
+ "movprfx z30, z16\n fmla z30.h, p3/M, z0.h, z6.h\n"
+ "ldr x23, [x14, #0x58]\n"
"mov p0.b, p2.b\n"
- "fmla z29.h, p3/M, z0.h, z7.h\n"
- "ldr x22, [x6, #0x60]\n"
- "fmla z28.h, p3/M, z0.h, z8.h\n"
- "ld1h { z5.h }, p2/Z, [x24, x7, LSL #1]\n"
- "ld1h { z0.h }, p3/Z, [x5]\n" // Load from weights and bias
+ "movprfx z29, z16\n fmla z29.h, p3/M, z0.h, z7.h\n"
+ "ldr x22, [x14, #0x60]\n"
+ "movprfx z28, z16\n fmla z28.h, p3/M, z0.h, z8.h\n"
+ "ld1h { z5.h }, p2/Z, [x24, x13, LSL #1]\n"
+ "ld1h { z0.h }, p3/Z, [x15]\n"
"fmla z31.h, p3/M, z1.h, z6.h\n"
- "ld1h { z6.h }, p2/Z, [x23, x7, LSL #1]\n"
+ "ld1h { z6.h }, p2/Z, [x23, x13, LSL #1]\n"
"fmla z30.h, p3/M, z1.h, z9.h\n"
- "ldr x21, [x6, #0x68]\n"
+ "ldr x21, [x14, #0x68]\n"
"fmla z29.h, p3/M, z1.h, z8.h\n"
"fmla z28.h, p3/M, z1.h, z13.h\n"
- "ld1h { z1.h }, p3/Z, [x5, #1, MUL VL]\n" // Load from weights and bias
- "ldr x20, [x6, #0x70]\n"
+ "ld1h { z1.h }, p3/Z, [x15, #1, MUL VL]\n"
+ "ldr x20, [x14, #0x70]\n"
"fmla z31.h, p3/M, z2.h, z9.h\n"
- "ld1h { z9.h }, p2/Z, [x22, x7, LSL #1]\n"
+ "ld1h { z9.h }, p2/Z, [x22, x13, LSL #1]\n"
"fmla z30.h, p3/M, z2.h, z11.h\n"
- "ldr x19, [x6, #0x78]\n"
+ "ldr x19, [x14, #0x78]\n"
"fmla z29.h, p3/M, z2.h, z13.h\n"
"fmla z28.h, p3/M, z2.h, z5.h\n"
- "ld1h { z2.h }, p3/Z, [x5, #2, MUL VL]\n" // Load from weights and bias
- "ldr x12, [x6, #0x80]\n"
+ "ld1h { z2.h }, p3/Z, [x15, #2, MUL VL]\n"
+ "ldr x26, [x14, #0x80]\n"
"fmla z31.h, p3/M, z3.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x21, x7, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x21, x13, LSL #1]\n"
"fmla z30.h, p3/M, z3.h, z12.h\n"
- "ldr x11, [x6, #0x88]\n"
+ "ldr x25, [x14, #0x88]\n"
"fmla z29.h, p3/M, z3.h, z5.h\n"
"fmla z28.h, p3/M, z3.h, z6.h\n"
- "ld1h { z3.h }, p3/Z, [x5, #3, MUL VL]\n" // Load from weights and bias
- "ldr x10, [x6, #0x90]\n"
+ "ld1h { z3.h }, p3/Z, [x15, #3, MUL VL]\n"
+ "ldr x24, [x14, #0x90]\n"
"fmla z31.h, p3/M, z4.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x20, x7, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x20, x13, LSL #1]\n"
"fmla z30.h, p3/M, z4.h, z9.h\n"
- "ld1h { z9.h }, p2/Z, [x19, x7, LSL #1]\n"
+ "ld1h { z9.h }, p2/Z, [x19, x13, LSL #1]\n"
"fmla z29.h, p3/M, z4.h, z6.h\n"
"fmla z28.h, p3/M, z4.h, z10.h\n"
- "ld1h { z4.h }, p3/Z, [x5, #4, MUL VL]\n" // Load from weights and bias
- "ldr x9, [x6, #0x98]\n"
+ "ld1h { z4.h }, p3/Z, [x15, #4, MUL VL]\n"
+ "ldr x23, [x14, #0x98]\n"
"fmla z31.h, p3/M, z0.h, z7.h\n"
- "ldr x20, [x6, #0xa0]\n"
+ "ldr x22, [x14, #0xa0]\n"
"fmla z30.h, p3/M, z0.h, z8.h\n"
- "ldr x28, [x6, #0xa8]\n"
+ "ldr x21, [x14, #0xa8]\n"
"fmla z29.h, p3/M, z0.h, z14.h\n"
"fmla z28.h, p3/M, z0.h, z11.h\n"
- "ld1h { z0.h }, p3/Z, [x5, #5, MUL VL]\n" // Load from weights and bias
- "ldr x27, [x6, #0xb0]\n"
+ "ld1h { z0.h }, p3/Z, [x15, #5, MUL VL]\n"
+ "ldr x20, [x14, #0xb0]\n"
"fmla z31.h, p3/M, z1.h, z8.h\n"
- "ld1h { z8.h }, p2/Z, [x11, x7, LSL #1]\n"
+ "ld1h { z8.h }, p2/Z, [x25, x13, LSL #1]\n"
"fmla z30.h, p3/M, z1.h, z13.h\n"
- "ldr x19, [x6, #0xb8]\n"
+ "ldr x19, [x14, #0xb8]\n"
"fmla z29.h, p3/M, z1.h, z11.h\n"
"fmla z28.h, p3/M, z1.h, z12.h\n"
- "ld1h { z1.h }, p3/Z, [x5, #6, MUL VL]\n" // Load from weights and bias
- "ldr x26, [x6, #0xc0]\n"
+ "ld1h { z1.h }, p3/Z, [x15, #6, MUL VL]\n"
+ "ldr x25, [x14, #0xc8]\n"
"fmla z31.h, p3/M, z2.h, z13.h\n"
- "ld1h { z13.h }, p2/Z, [x12, x7, LSL #1]\n"
+ "ld1h { z13.h }, p2/Z, [x26, x13, LSL #1]\n"
"fmla z30.h, p3/M, z2.h, z5.h\n"
- "ldr x25, [x6, #0xc8]\n"
+ "ldr x26, [x14, #0xc0]\n"
"fmla z29.h, p3/M, z2.h, z12.h\n"
"fmla z28.h, p3/M, z2.h, z9.h\n"
- "ld1h { z2.h }, p3/Z, [x5, #7, MUL VL]\n" // Load from weights and bias
- "addvl x5, x5, #16\n"
+ "ld1h { z2.h }, p3/Z, [x15, #7, MUL VL]\n"
+ "addvl x15, x15, #16\n"
"fmla z31.h, p3/M, z3.h, z5.h\n"
- "ld1h { z5.h }, p2/Z, [x10, x7, LSL #1]\n"
- "ldr x24, [x6, #0xd0]\n"
+ "ld1h { z5.h }, p2/Z, [x24, x13, LSL #1]\n"
+ "ldr x24, [x14, #0xd0]\n"
"fmla z30.h, p3/M, z3.h, z6.h\n"
- "ldr x23, [x6, #0xd8]\n"
"fmla z29.h, p3/M, z3.h, z9.h\n"
"fmla z28.h, p3/M, z3.h, z13.h\n"
- "ld1h { z3.h }, p3/Z, [x5, #-8, MUL VL]\n" // Load from weights and bias
- "ldr x22, [x6, #0xe0]\n"
+ "ld1h { z3.h }, p3/Z, [x15, #-8, MUL VL]\n"
"fmla z31.h, p3/M, z4.h, z6.h\n"
- "ld1h { z6.h }, p2/Z, [x9, x7, LSL #1]\n"
+ "ld1h { z6.h }, p2/Z, [x23, x13, LSL #1]\n"
+ "ldr x23, [x14, #0xd8]\n"
"fmla z30.h, p3/M, z4.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x20, x7, LSL #1]\n"
+ "ld1h { z10.h }, p2/Z, [x22, x13, LSL #1]\n"
"fmla z29.h, p3/M, z4.h, z13.h\n"
"fmla z28.h, p3/M, z4.h, z8.h\n"
- "ld1h { z4.h }, p3/Z, [x5, #-7, MUL VL]\n" // Load from weights and bias
- "ldr x21, [x6, #0xe8]\n"
+ "ld1h { z4.h }, p3/Z, [x15, #-7, MUL VL]\n"
+ "ldr x22, [x14, #0xe0]\n"
"fmla z31.h, p3/M, z0.h, z14.h\n"
- "ld1h { z14.h }, p2/Z, [x19, x7, LSL #1]\n"
+ "ld1h { z14.h }, p2/Z, [x19, x13, LSL #1]\n"
"fmla z30.h, p3/M, z0.h, z11.h\n"
- "ldr x20, [x6, #0xf0]\n"
+ "ldr x19, [x14, #0xf8]\n"
"fmla z29.h, p3/M, z0.h, z5.h\n"
"fmla z28.h, p3/M, z0.h, z6.h\n"
- "ld1h { z0.h }, p3/Z, [x5, #-6, MUL VL]\n" // Load from weights and bias
- "ldr x19, [x6, #0xf8]\n"
+ "ld1h { z0.h }, p3/Z, [x15, #-6, MUL VL]\n"
"fmla z31.h, p3/M, z1.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x28, x7, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x21, x13, LSL #1]\n"
+ "ldr x21, [x14, #0xe8]\n"
"fmla z30.h, p3/M, z1.h, z12.h\n"
- "ldr x12, [x6, #0x100]\n"
"fmla z29.h, p3/M, z1.h, z6.h\n"
"fmla z28.h, p3/M, z1.h, z10.h\n"
- "ld1h { z1.h }, p3/Z, [x5, #-5, MUL VL]\n" // Load from weights and bias
- "ldr x11, [x6, #0x108]\n"
+ "ld1h { z1.h }, p3/Z, [x15, #-5, MUL VL]\n"
"fmla z31.h, p3/M, z2.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x27, x7, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x20, x13, LSL #1]\n"
+ "ldr x20, [x14, #0xf0]\n"
"fmla z30.h, p3/M, z2.h, z9.h\n"
- "ldr x10, [x6, #0x110]\n"
"fmla z29.h, p3/M, z2.h, z10.h\n"
"fmla z28.h, p3/M, z2.h, z11.h\n"
- "ld1h { z2.h }, p3/Z, [x5, #-4, MUL VL]\n" // Load from weights and bias
- "ldr x9, [x6, #0x118]\n"
+ "ld1h { z2.h }, p3/Z, [x15, #-4, MUL VL]\n"
"fmla z31.h, p3/M, z3.h, z9.h\n"
- "ld1h { z9.h }, p2/Z, [x26, x7, LSL #1]\n"
+ "ld1h { z9.h }, p2/Z, [x26, x13, LSL #1]\n"
+ "ldr x26, [x14, #0x100]\n"
"fmla z30.h, p3/M, z3.h, z13.h\n"
"fmla z29.h, p3/M, z3.h, z11.h\n"
"fmla z28.h, p3/M, z3.h, z12.h\n"
- "ld1h { z3.h }, p3/Z, [x5, #-3, MUL VL]\n" // Load from weights and bias
+ "ld1h { z3.h }, p3/Z, [x15, #-3, MUL VL]\n"
"fmla z31.h, p3/M, z4.h, z13.h\n"
- "ld1h { z13.h }, p2/Z, [x25, x7, LSL #1]\n"
+ "ld1h { z13.h }, p2/Z, [x25, x13, LSL #1]\n"
+ "ldr x25, [x14, #0x108]\n"
"fmla z30.h, p3/M, z4.h, z8.h\n"
- "ld1h { z8.h }, p2/Z, [x22, x7, LSL #1]\n"
+ "ld1h { z8.h }, p2/Z, [x22, x13, LSL #1]\n"
"fmla z29.h, p3/M, z4.h, z12.h\n"
"fmla z28.h, p3/M, z4.h, z14.h\n"
- "ld1h { z4.h }, p3/Z, [x5, #-2, MUL VL]\n" // Load from weights and bias
+ "ld1h { z4.h }, p3/Z, [x15, #-2, MUL VL]\n"
"fmla z31.h, p3/M, z0.h, z5.h\n"
- "ld1h { z5.h }, p2/Z, [x24, x7, LSL #1]\n"
+ "ld1h { z5.h }, p2/Z, [x24, x13, LSL #1]\n"
+ "ldr x24, [x14, #0x110]\n"
"fmla z30.h, p3/M, z0.h, z6.h\n"
"fmla z29.h, p3/M, z0.h, z9.h\n"
"fmla z28.h, p3/M, z0.h, z13.h\n"
- "ld1h { z0.h }, p3/Z, [x5, #-1, MUL VL]\n" // Load from weights and bias
+ "ld1h { z0.h }, p3/Z, [x15, #-1, MUL VL]\n"
"fmla z31.h, p3/M, z1.h, z6.h\n"
- "ld1h { z6.h }, p2/Z, [x23, x7, LSL #1]\n"
+ "ld1h { z6.h }, p2/Z, [x23, x13, LSL #1]\n"
+ "ldr x23, [x14, #0x118]\n"
"fmla z30.h, p3/M, z1.h, z10.h\n"
"fmla z29.h, p3/M, z1.h, z13.h\n"
"fmla z28.h, p3/M, z1.h, z5.h\n"
- "ld1h { z1.h }, p3/Z, [x5]\n" // Load from weights and bias
+ "ld1h { z1.h }, p3/Z, [x15]\n"
"fmla z31.h, p3/M, z2.h, z10.h\n"
- "ld1h { z10.h }, p2/Z, [x21, x7, LSL #1]\n"
+ "ld1h { z10.h }, p2/Z, [x21, x13, LSL #1]\n"
"fmla z30.h, p3/M, z2.h, z11.h\n"
"fmla z29.h, p3/M, z2.h, z5.h\n"
"fmla z28.h, p3/M, z2.h, z6.h\n"
- "ld1h { z2.h }, p3/Z, [x5, #1, MUL VL]\n" // Load from weights and bias
+ "ld1h { z2.h }, p3/Z, [x15, #1, MUL VL]\n"
"fmla z31.h, p3/M, z3.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x20, x7, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x20, x13, LSL #1]\n"
"fmla z30.h, p3/M, z3.h, z12.h\n"
"fmla z29.h, p3/M, z3.h, z6.h\n"
"fmla z28.h, p3/M, z3.h, z8.h\n"
- "ld1h { z3.h }, p3/Z, [x5, #2, MUL VL]\n" // Load from weights and bias
+ "ld1h { z3.h }, p3/Z, [x15, #2, MUL VL]\n"
"fmla z31.h, p3/M, z4.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x19, x7, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x19, x13, LSL #1]\n"
"fmla z30.h, p3/M, z4.h, z14.h\n"
"fmla z29.h, p3/M, z4.h, z8.h\n"
"fmla z28.h, p3/M, z4.h, z10.h\n"
- "ld1h { z4.h }, p3/Z, [x5, #3, MUL VL]\n" // Load from weights and bias
+ "ld1h { z4.h }, p3/Z, [x15, #3, MUL VL]\n"
"fmla z31.h, p3/M, z0.h, z9.h\n"
- "ld1h { z9.h }, p2/Z, [x12, x7, LSL #1]\n"
+ "ld1h { z9.h }, p2/Z, [x26, x13, LSL #1]\n"
"fmla z30.h, p3/M, z0.h, z13.h\n"
"fmla z29.h, p3/M, z0.h, z11.h\n"
- "ld1h { z11.h }, p2/Z, [x11, x7, LSL #1]\n"
+ "ld1h { z11.h }, p2/Z, [x25, x13, LSL #1]\n"
"fmla z28.h, p3/M, z0.h, z12.h\n"
"fmla z31.h, p3/M, z1.h, z13.h\n"
"fmla z30.h, p3/M, z1.h, z5.h\n"
"fmla z29.h, p3/M, z1.h, z12.h\n"
- "ld1h { z12.h }, p2/Z, [x10, x7, LSL #1]\n"
+ "ld1h { z12.h }, p2/Z, [x24, x13, LSL #1]\n"
"fmla z28.h, p3/M, z1.h, z9.h\n"
"fmla z31.h, p3/M, z2.h, z5.h\n"
"fmla z30.h, p3/M, z2.h, z6.h\n"
"fmla z29.h, p3/M, z2.h, z9.h\n"
- "ld1h { z9.h }, p2/Z, [x9, x7, LSL #1]\n"
+ "ld1h { z9.h }, p2/Z, [x23, x13, LSL #1]\n"
"fmla z28.h, p3/M, z2.h, z11.h\n"
"fmla z31.h, p3/M, z3.h, z6.h\n"
"fmla z30.h, p3/M, z3.h, z8.h\n"
@@ -540,20 +532,20 @@
"fmax z29.h, p3/M, z29.h, z18.h\n"
"fmax z28.h, p3/M, z28.h, z18.h\n"
"fmin z31.h, p3/M, z31.h, z17.h\n"
- "st1h { z31.h }, p0, [x17, x15, LSL #1]\n"
+ "st1h { z31.h }, p0, [x11, x9, LSL #1]\n"
"fmin z30.h, p3/M, z30.h, z17.h\n"
"fmin z29.h, p3/M, z29.h, z17.h\n"
- "st1h { z30.h }, p0, [x16, x15, LSL #1]\n"
+ "st1h { z30.h }, p0, [x10, x9, LSL #1]\n"
"fmin z28.h, p3/M, z28.h, z17.h\n"
- "st1h { z29.h }, p0, [x14, x15, LSL #1]\n"
- "st1h { z28.h }, p0, [x13, x15, LSL #1]\n"
+ "st1h { z29.h }, p0, [x28, x9, LSL #1]\n"
+ "st1h { z28.h }, p0, [x27, x9, LSL #1]\n"
:
: [n_channels] "r" ((unsigned long) n_channels), [offsetof_Args_inptrs] "I" (offsetof(Args, inptrs)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_outptrs] "I" (offsetof(Args, outptrs)), [offsetof_args_params] "I" (offsetof(Args, params)), [params_struct] "r" (¶ms_struct)
- : "cc", "memory", "p0", "p1", "p2", "p3", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z16", "z17", "z18", "z28", "z29", "z30", "z31"
+ : "cc", "memory", "p0", "p1", "p2", "p3", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z16", "z17", "z18", "z28", "z29", "z30", "z31"
);
}
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
+#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp
index 05e82d4..e07e631 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp
@@ -28,7 +28,7 @@
#pragma once
-#if defined(ARM_COMPUTE_ENABLE_SVE)
+#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE)
namespace arm_conv {
namespace depthwise {
@@ -36,15 +36,17 @@
void sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl(const float *const *const, float *const *const, const void *, unsigned int, const float, const float);
void sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float);
-struct sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst
+class sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst : public IDepthwiseDepthfirstStrategy
{
- typedef float bias_type;
- typedef float input_type;
- typedef float weight_type;
- typedef float return_type;
-
+ private:
typedef void (*indirect_kern_type)(const float *const *const, float *const *const, const void *, unsigned int, const float, const float);
+ indirect_kern_type m_indirect_kernel = sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl;
+
typedef void (*direct_kern_type)(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float);
+ direct_kern_type m_direct_kernel = sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl;
+
+ public:
+ typedef float return_type;
constexpr static arm_gemm::VLType vl_type = arm_gemm::VLType::SVE;
@@ -60,13 +62,60 @@
constexpr static unsigned int input_rows = 4;
constexpr static unsigned int input_cols = 4;
- indirect_kern_type indirect_kernel = sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl;
- direct_kern_type direct_kernel = sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl;
-
sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst(const CPUInfo *) {}
+
+ arm_gemm::VLType get_vl_type(void) const override { return vl_type; }
+
+ unsigned int get_kernel_rows(void) const override { return kernel_rows; }
+ unsigned int get_kernel_cols(void) const override { return kernel_cols; }
+
+ unsigned int get_stride_rows(void) const override { return stride_rows; }
+ unsigned int get_stride_cols(void) const override { return stride_cols; }
+
+ unsigned int get_output_rows(void) const override { return output_rows; }
+ unsigned int get_output_cols(void) const override { return output_cols; }
+
+ unsigned int get_input_rows(void) const override { return input_rows; }
+ unsigned int get_input_cols(void) const override { return input_cols; }
+
+ void indirect_kernel(
+ const void *const *const input_ptrs,
+ void *const *const outptrs,
+ const void *params,
+ unsigned int n_channels,
+ const void *activation_min,
+ const void *activation_max
+ ) const override
+ {
+ m_indirect_kernel(
+ reinterpret_cast<const float *const *>(input_ptrs),
+ reinterpret_cast<float *const *>(outptrs),
+ params, n_channels,
+ *static_cast<const float *>(activation_min),
+ *static_cast<const float *>(activation_max)
+ );
+ }
+
+ void direct_kernel(
+ const unsigned int n_tile_rows, const unsigned int n_tile_cols,
+ const void *inptr, int64_t ld_input_row, int64_t ld_input_col,
+ void *outptr, int64_t ld_output_row, int64_t ld_output_col,
+ const void *params, unsigned int n_channels,
+ const void *activation_min, const void *activation_max
+ ) const override
+ {
+ m_direct_kernel(
+ n_tile_rows, n_tile_cols,
+ static_cast<const float *>(inptr), ld_input_row, ld_input_col,
+ static_cast<float *>(outptr), ld_output_row, ld_output_col,
+ params, n_channels,
+ *static_cast<const float *>(activation_min),
+ *static_cast<const float *>(activation_max)
+ );
+ }
};
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(ARM_COMPUTE_ENABLE_SVE)
+#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp
index eb632eb..571246b 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp
@@ -25,7 +25,7 @@
#include <cstddef>
#include <cstdint>
-#if defined(ARM_COMPUTE_ENABLE_SVE)
+#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE)
namespace arm_conv {
namespace depthwise {
@@ -88,277 +88,229 @@
__asm__ __volatile__(
"ptrue p3.b\n"
- "mov x8, #0x0\n"
"mov x17, #0x0\n"
+ "mov x16, #0x0\n"
"1:" // Tile loop
- "str x8, [%x[params_struct], %[offsetof_args_tile_i]]\n"
- "mov x21, #0x2\n"
- "str x17, [%x[params_struct], %[offsetof_args_tile_j]]\n"
- "cntb x16\n"
- "ldr x15, [%x[params_struct], %[offsetof_args_params]]\n"
- "add x16, x16, XZR, LSL #4\n"
- "ldr x20, [%x[params_struct], %[offsetof_args_ld_input_row]]\n"
- "cntb x14\n"
- "ldr x13, [%x[params_struct], %[offsetof_args_ld_input_col]]\n"
- "cntb x12\n"
- "ldr x11, [%x[params_struct], %[offsetof_args_inptr]]\n"
- "mul x19, x8, x20\n" // offset = tile_i * ld_input_row
- "ldr x22, [%x[params_struct], %[offsetof_args_ld_output_row]]\n"
- "madd x19, x17, x13, x19\n" // offset += tile_j * ld_input_col
- "ldr x10, [%x[params_struct], %[offsetof_args_ld_output_col]]\n"
- "mul x19, x19, x21\n" // offset *= kernel_stride * output_size
- "ldr x9, [%x[params_struct], %[offsetof_args_outptr]]\n"
- "add x11, x11, x19, LSL #2\n" // inptr[0] += offset * sizeof(float)
+ "str x17, [%x[params_struct], %[offsetof_args_tile_i]]\n"
+ "mov x23, #0x2\n"
+ "str x16, [%x[params_struct], %[offsetof_args_tile_j]]\n"
+ "mov x15, #0x2\n"
+ "ldr x14, [%x[params_struct], %[offsetof_args_params]]\n"
+ "mov x13, #0x0\n"
+ "ldr x22, [%x[params_struct], %[offsetof_args_ld_input_row]]\n"
+ "cntw x12\n"
+ "ldr x11, [%x[params_struct], %[offsetof_args_ld_input_col]]\n"
+ "sub x21, XZR, x12\n"
+ "ldr x10, [%x[params_struct], %[offsetof_args_inptr]]\n"
+ "mul x19, x17, x22\n" // offset = tile_i * ld_input_row
+ "ldr x20, [%x[params_struct], %[offsetof_args_ld_output_row]]\n"
+ "madd x19, x16, x11, x19\n" // offset += tile_j * ld_input_col
+ "ldr x9, [%x[params_struct], %[offsetof_args_ld_output_col]]\n"
+ "mul x19, x19, x23\n" // offset *= kernel_stride * output_size
+ "ldr x28, [%x[params_struct], %[offsetof_args_outptr]]\n"
+ "add x10, x10, x19, LSL #2\n" // inptr[0] += offset * sizeof(float)
"ld1rw { z18.s }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n"
- "add x28, x11, x20, LSL #2\n"
+ "add x27, x10, x22, LSL #2\n"
"ld1rw { z17.s }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n"
- "add x27, x28, x20, LSL #2\n"
- "ld1w { z16.s }, p3/Z, [x15]\n"
- "mov z31.d, z16.d\n"
- "ld1w { z0.s }, p3/Z, [x15, #1, MUL VL]\n"
- "add x26, x27, x20, LSL #2\n"
- "mov z30.d, z16.d\n"
- "ld1w { z1.s }, p3/Z, [x15, #2, MUL VL]\n"
- "add x25, x13, x13\n"
- "mov z29.d, z16.d\n"
- "ld1w { z2.s }, p3/Z, [x15, #3, MUL VL]\n"
- "add x24, x25, x13\n"
- "mov z28.d, z16.d\n"
- "ld1w { z3.s }, p3/Z, [x15, #4, MUL VL]\n"
- "add x14, x14, x13, LSL #4\n"
- "ld1w { z4.s }, p3/Z, [x15, #5, MUL VL]\n"
- "add x12, x12, x25, LSL #4\n"
- "ld1w { z5.s }, p3/Z, [x15, #6, MUL VL]\n"
- "cntb x23\n"
- "ld1w { z6.s }, p3/Z, [x15, #7, MUL VL]\n"
- "add x23, x23, x24, LSL #4\n"
- "prfm pldl1keep, [x28, x14]\n"
- "mov x20, #0x2\n"
- "prfm pldl1keep, [x11, x16]\n"
- "mul x19, x8, x22\n" // offset = tile_i * ld_output_row
- "prfm pldl1keep, [x11, x23]\n"
- "madd x19, x17, x10, x19\n" // offset += tile_j * ld_output_col
- "prfm pldl1keep, [x28, x12]\n"
- "mul x19, x19, x20\n" // offset *= output_tile_size
- "prfm pldl1keep, [x27, x14]\n"
- "add x9, x9, x19, LSL #2\n" // outptrs[0] += offset * sizeof(float)
- "mov x21, #0x0\n"
- "add x22, x9, x22, LSL #2\n"
- "cntw x20\n"
- "sub x19, XZR, x20\n"
+ "add x26, x27, x22, LSL #2\n"
+ "ld1w { z16.s }, p3/Z, [x14]\n"
+ "add x25, x26, x22, LSL #2\n"
+ "ld1w { z0.s }, p3/Z, [x14, #1, MUL VL]\n"
+ "add x24, x11, x11\n"
+ "ld1w { z1.s }, p3/Z, [x14, #2, MUL VL]\n"
+ "add x23, x24, x11\n"
+ "ld1w { z2.s }, p3/Z, [x14, #3, MUL VL]\n"
+ "mul x19, x17, x20\n" // offset = tile_i * ld_output_row
+ "ld1w { z3.s }, p3/Z, [x14, #4, MUL VL]\n"
+ "madd x19, x16, x9, x19\n" // offset += tile_j * ld_output_col
+ "ld1w { z4.s }, p3/Z, [x14, #5, MUL VL]\n"
+ "mul x19, x19, x15\n" // offset *= output_tile_size
+ "ld1w { z5.s }, p3/Z, [x14, #6, MUL VL]\n"
+ "add x28, x28, x19, LSL #2\n" // outptrs[0] += offset * sizeof(float)
+ "ld1w { z6.s }, p3/Z, [x14, #7, MUL VL]\n"
+ "add x22, x28, x20, LSL #2\n"
"whilelt p2.s, XZR, %x[n_channels]\n"
- "ld1w { z9.s }, p2/Z, [x28, x13, LSL #2]\n"
- "ld1w { z10.s }, p2/Z, [x11]\n"
- "addvl x15, x15, #16\n"
- "ld1w { z11.s }, p2/Z, [x11, x24, LSL #2]\n"
- "cmp x20, %x[n_channels]\n"
- "ld1w { z7.s }, p3/Z, [x15, #-8, MUL VL]\n"
- "ld1w { z8.s }, p3/Z, [x15, #-7, MUL VL]\n"
- "addvl x15, x15, #-6\n"
- "ld1w { z12.s }, p2/Z, [x28, x25, LSL #2]\n"
- "ld1w { z13.s }, p2/Z, [x27, x13, LSL #2]\n"
+ "ld1w { z9.s }, p2/Z, [x27, x11, LSL #2]\n"
+ "ld1w { z10.s }, p2/Z, [x10]\n"
+ "addvl x14, x14, #16\n"
+ "ld1w { z11.s }, p2/Z, [x10, x23, LSL #2]\n"
+ "cmp x12, %x[n_channels]\n"
+ "ld1w { z7.s }, p3/Z, [x14, #-8, MUL VL]\n"
+ "ld1w { z8.s }, p3/Z, [x14, #-7, MUL VL]\n"
+ "addvl x14, x14, #-6\n"
+ "ld1w { z12.s }, p2/Z, [x27, x24, LSL #2]\n"
+ "ld1w { z13.s }, p2/Z, [x26, x11, LSL #2]\n"
"bge 3f\n"
"2:" // Tile loop: Channel loop
- "fmla z31.s, p3/M, z4.s, z9.s\n"
- "prfm pldl1keep, [x26, x16]\n"
- "whilelt p1.s, x20, %x[n_channels]\n"
- "fmla z30.s, p3/M, z3.s, z9.s\n"
- "prfm pldl1keep, [x26, x23]\n"
- "incw x19\n"
- "fmla z29.s, p3/M, z1.s, z9.s\n"
- "prfm pldl1keep, [x11, x14]\n"
- "mov p0.b, p2.b\n"
- "fmla z28.s, p3/M, z0.s, z9.s\n"
- "ld1w { z9.s }, p2/Z, [x26]\n"
+ "movprfx z31, z16\n fmla z31.s, p3/M, z4.s, z9.s\n"
+ "whilelt p1.s, x12, %x[n_channels]\n"
+ "movprfx z30, z16\n fmla z30.s, p3/M, z3.s, z9.s\n"
"incw x21\n"
+ "movprfx z29, z16\n fmla z29.s, p3/M, z1.s, z9.s\n"
+ "mov p0.b, p2.b\n"
+ "movprfx z28, z16\n fmla z28.s, p3/M, z0.s, z9.s\n"
+ "ld1w { z9.s }, p2/Z, [x25]\n"
+ "incw x13\n"
"fmla z31.s, p3/M, z0.s, z10.s\n"
- "prfm pldl1keep, [x11, x12]\n"
- "incw x20\n"
+ "ld1w { z10.s }, p2/Z, [x26, x24, LSL #2]\n"
+ "incw x12\n"
"fmla z30.s, p3/M, z2.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x26, x24, LSL #2]\n"
- "ld1w { z10.s }, p2/Z, [x27, x25, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x25, x23, LSL #2]\n"
"fmla z29.s, p3/M, z2.s, z12.s\n"
- "prfm pldl1keep, [x27, x12]\n"
+ "ld1w { z16.s }, p3/Z, [x14]\n"
"fmla z28.s, p3/M, z1.s, z12.s\n"
- "prfm pldl1keep, [x28, x16]\n"
"fmla z31.s, p3/M, z5.s, z12.s\n"
- "prfm pldl1keep, [x28, x23]\n"
"fmla z30.s, p3/M, z4.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x11, x13, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x10, x11, LSL #2]\n"
"fmla z29.s, p3/M, z6.s, z9.s\n"
- "ld1w { z9.s }, p2/Z, [x11, x25, LSL #2]\n"
+ "ld1w { z9.s }, p2/Z, [x10, x24, LSL #2]\n"
+ "addvl x10, x10, #1\n"
"fmla z28.s, p3/M, z3.s, z13.s\n"
- "addvl x11, x11, #1\n"
"fmla z31.s, p3/M, z7.s, z13.s\n"
- "prfm pldl1keep, [x27, x16]\n"
- "prfm pldl1keep, [x27, x23]\n"
"fmla z30.s, p3/M, z6.s, z13.s\n"
- "prfm pldl1keep, [x26, x14]\n"
"fmla z29.s, p3/M, z4.s, z13.s\n"
- "prfm pldl1keep, [x26, x12]\n"
"fmla z28.s, p3/M, z8.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x28]\n"
+ "ld1w { z11.s }, p2/Z, [x27]\n"
"fmla z31.s, p3/M, z1.s, z12.s\n"
- "ld1w { z16.s }, p3/Z, [x15]\n"
"fmla z30.s, p3/M, z0.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x28, x24, LSL #2]\n"
- "fmla z29.s, p3/M, z5.s, z10.s\n"
- "addvl x28, x28, #1\n"
- "fmla z28.s, p3/M, z4.s, z10.s\n"
- "prfm pldl1keep, [x28, x14]\n"
- "prfm pldl1keep, [x11, x16]\n"
- "fmla z31.s, p3/M, z2.s, z9.s\n"
- "prfm pldl1keep, [x11, x23]\n"
- "fmla z30.s, p3/M, z1.s, z9.s\n"
- "ld1w { z9.s }, p2/Z, [x27]\n"
- "fmla z29.s, p3/M, z0.s, z11.s\n"
- "prfm pldl1keep, [x28, x12]\n"
- "fmla z28.s, p3/M, z2.s, z12.s\n"
- "ld1w { z0.s }, p3/Z, [x15, #1, MUL VL]\n"
- "fmla z31.s, p3/M, z8.s, z10.s\n"
- "ld1w { z1.s }, p3/Z, [x15, #2, MUL VL]\n"
- "fmla z30.s, p3/M, z7.s, z10.s\n"
- "ld1w { z10.s }, p2/Z, [x27, x24, LSL #2]\n"
- "fmla z29.s, p3/M, z3.s, z9.s\n"
+ "ld1w { z12.s }, p2/Z, [x27, x23, LSL #2]\n"
"addvl x27, x27, #1\n"
- "fmla z31.s, p3/M, z3.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x26, x13, LSL #2]\n"
- "fmla z28.s, p3/M, z5.s, z10.s\n"
- "ld1w { z13.s }, p1/Z, [x27, x13, LSL #2]\n"
- "fmla z30.s, p3/M, z5.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x26, x25, LSL #2]\n"
- "whilelt p2.s, x21, %x[n_channels]\n"
- "fmla z29.s, p3/M, z7.s, z11.s\n"
- "prfm pldl1keep, [x27, x14]\n"
+ "fmla z29.s, p3/M, z5.s, z10.s\n"
+ "fmla z28.s, p3/M, z4.s, z10.s\n"
+ "ld1w { z4.s }, p3/Z, [x14, #5, MUL VL]\n"
+ "fmla z31.s, p3/M, z2.s, z9.s\n"
+ "fmla z30.s, p3/M, z1.s, z9.s\n"
+ "ld1w { z9.s }, p2/Z, [x26]\n"
+ "ld1w { z1.s }, p3/Z, [x14, #2, MUL VL]\n"
+ "fmla z29.s, p3/M, z0.s, z11.s\n"
+ "ld1w { z0.s }, p3/Z, [x14, #1, MUL VL]\n"
+ "fmla z28.s, p3/M, z2.s, z12.s\n"
+ "ld1w { z2.s }, p3/Z, [x14, #3, MUL VL]\n"
+ "fmla z31.s, p3/M, z8.s, z10.s\n"
+ "fmla z30.s, p3/M, z7.s, z10.s\n"
+ "ld1w { z10.s }, p2/Z, [x26, x23, LSL #2]\n"
"addvl x26, x26, #1\n"
+ "fmla z29.s, p3/M, z3.s, z9.s\n"
+ "ld1w { z13.s }, p1/Z, [x26, x11, LSL #2]\n"
+ "fmla z31.s, p3/M, z3.s, z11.s\n"
+ "ld1w { z11.s }, p2/Z, [x25, x11, LSL #2]\n"
+ "fmla z28.s, p3/M, z5.s, z10.s\n"
+ "ld1w { z3.s }, p3/Z, [x14, #4, MUL VL]\n"
+ "fmla z30.s, p3/M, z5.s, z12.s\n"
+ "ld1w { z12.s }, p2/Z, [x25, x24, LSL #2]\n"
+ "whilelt p2.s, x13, %x[n_channels]\n"
+ "fmla z29.s, p3/M, z7.s, z11.s\n"
+ "ld1w { z5.s }, p3/Z, [x14, #6, MUL VL]\n"
+ "addvl x25, x25, #1\n"
"fmla z31.s, p3/M, z6.s, z9.s\n"
- "ld1w { z9.s }, p1/Z, [x28, x13, LSL #2]\n"
- "cmp x20, %x[n_channels]\n"
+ "ld1w { z9.s }, p1/Z, [x27, x11, LSL #2]\n"
+ "cmp x12, %x[n_channels]\n"
"fmla z30.s, p3/M, z8.s, z10.s\n"
- "ld1w { z10.s }, p1/Z, [x11]\n"
+ "ld1w { z10.s }, p1/Z, [x10]\n"
"fmla z28.s, p3/M, z6.s, z11.s\n"
- "ld1w { z11.s }, p1/Z, [x11, x24, LSL #2]\n"
- "ld1w { z2.s }, p3/Z, [x15, #3, MUL VL]\n"
+ "ld1w { z11.s }, p1/Z, [x10, x23, LSL #2]\n"
+ "ld1w { z6.s }, p3/Z, [x14, #7, MUL VL]\n"
"fmla z29.s, p3/M, z8.s, z12.s\n"
- "ld1w { z3.s }, p3/Z, [x15, #4, MUL VL]\n"
+ "addvl x14, x14, #16\n"
"fmax z31.s, p3/M, z31.s, z18.s\n"
- "ld1w { z4.s }, p3/Z, [x15, #5, MUL VL]\n"
+ "ld1w { z8.s }, p3/Z, [x14, #-7, MUL VL]\n"
"fmla z28.s, p3/M, z7.s, z12.s\n"
+ "ld1w { z12.s }, p1/Z, [x27, x24, LSL #2]\n"
"fmax z30.s, p3/M, z30.s, z18.s\n"
- "ld1w { z12.s }, p1/Z, [x28, x25, LSL #2]\n"
- "ld1w { z5.s }, p3/Z, [x15, #6, MUL VL]\n"
+ "ld1w { z7.s }, p3/Z, [x14, #-8, MUL VL]\n"
+ "addvl x14, x14, #-6\n"
"fmax z29.s, p3/M, z29.s, z18.s\n"
- "ld1w { z6.s }, p3/Z, [x15, #7, MUL VL]\n"
"fmin z31.s, p3/M, z31.s, z17.s\n"
- "addvl x15, x15, #16\n"
+ "st1w { z31.s }, p0, [x28]\n"
"fmin z30.s, p3/M, z30.s, z17.s\n"
- "ld1w { z7.s }, p3/Z, [x15, #-8, MUL VL]\n"
- "fmax z28.s, p3/M, z28.s, z18.s\n"
- "ld1w { z8.s }, p3/Z, [x15, #-7, MUL VL]\n"
- "addvl x15, x15, #-6\n"
"fmin z29.s, p3/M, z29.s, z17.s\n"
- "st1w { z31.s }, p0, [x9]\n"
- "mov z31.d, z16.d\n"
+ "st1w { z30.s }, p0, [x28, x9, LSL #2]\n"
+ "fmax z28.s, p3/M, z28.s, z18.s\n"
+ "addvl x28, x28, #1\n"
"fmin z28.s, p3/M, z28.s, z17.s\n"
- "st1w { z30.s }, p0, [x9, x10, LSL #2]\n"
- "addvl x9, x9, #1\n"
- "mov z30.d, z16.d\n"
"st1w { z29.s }, p0, [x22]\n"
- "mov z29.d, z16.d\n"
- "st1w { z28.s }, p0, [x22, x10, LSL #2]\n"
- "mov z28.d, z16.d\n"
+ "st1w { z28.s }, p0, [x22, x9, LSL #2]\n"
"addvl x22, x22, #1\n"
"blt 2b\n"
"3:" // Tile loop: Channel tail
- "fmla z31.s, p3/M, z4.s, z9.s\n"
- "prfm pldl1keep, [x26, x16]\n"
+ "movprfx z31, z16\n fmla z31.s, p3/M, z4.s, z9.s\n"
+ "ldr x17, [%x[params_struct], %[offsetof_args_tile_i]]\n"
"mov p0.b, p2.b\n"
- "fmla z30.s, p3/M, z3.s, z9.s\n"
- "prfm pldl1keep, [x26, x23]\n"
- "fmla z29.s, p3/M, z1.s, z9.s\n"
- "prfm pldl1keep, [x11, x14]\n"
- "fmla z28.s, p3/M, z0.s, z9.s\n"
- "ld1w { z9.s }, p2/Z, [x26]\n"
- "prfm pldl1keep, [x11, x12]\n"
- "fmla z31.s, p3/M, z0.s, z10.s\n"
- "ld1w { z10.s }, p2/Z, [x27, x25, LSL #2]\n"
- "fmla z30.s, p3/M, z2.s, z11.s\n"
- "fmla z29.s, p3/M, z2.s, z12.s\n"
- "ld1w { z11.s }, p2/Z, [x26, x24, LSL #2]\n"
- "fmla z28.s, p3/M, z1.s, z12.s\n"
- "prfm pldl1keep, [x27, x12]\n"
- "prfm pldl1keep, [x28, x16]\n"
- "fmla z31.s, p3/M, z5.s, z12.s\n"
- "prfm pldl1keep, [x28, x23]\n"
- "fmla z30.s, p3/M, z4.s, z12.s\n"
- "fmla z29.s, p3/M, z6.s, z9.s\n"
- "ld1w { z12.s }, p2/Z, [x11, x13, LSL #2]\n"
- "fmla z28.s, p3/M, z3.s, z13.s\n"
- "ld1w { z9.s }, p2/Z, [x11, x25, LSL #2]\n"
- "prfm pldl1keep, [x27, x16]\n"
- "fmla z31.s, p3/M, z7.s, z13.s\n"
- "prfm pldl1keep, [x27, x23]\n"
- "fmla z30.s, p3/M, z6.s, z13.s\n"
- "fmla z29.s, p3/M, z4.s, z13.s\n"
- "prfm pldl1keep, [x26, x14]\n"
- "fmla z28.s, p3/M, z8.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x28]\n"
- "prfm pldl1keep, [x26, x12]\n"
- "fmla z31.s, p3/M, z1.s, z12.s\n"
- "ldr x8, [%x[params_struct], %[offsetof_args_tile_i]]\n"
- "fmla z30.s, p3/M, z0.s, z12.s\n"
- "add x21, x8, #0x1\n"
- "fmla z29.s, p3/M, z5.s, z10.s\n"
- "ld1w { z12.s }, p2/Z, [x28, x24, LSL #2]\n"
- "fmla z31.s, p3/M, z2.s, z9.s\n"
- "ldr x17, [%x[params_struct], %[offsetof_args_tile_j]]\n"
- "add x17, x17, #0x1\n"
- "fmla z30.s, p3/M, z1.s, z9.s\n"
- "ld1w { z9.s }, p2/Z, [x27]\n"
- "fmla z31.s, p3/M, z8.s, z10.s\n"
+ "movprfx z30, z16\n fmla z30.s, p3/M, z3.s, z9.s\n"
+ "ldr x16, [%x[params_struct], %[offsetof_args_tile_j]]\n"
+ "add x21, x17, #0x1\n"
+ "movprfx z29, z16\n fmla z29.s, p3/M, z1.s, z9.s\n"
"ldr x20, [%x[params_struct], %[offsetof_args_n_tile_rows]]\n"
- "fmla z28.s, p3/M, z4.s, z10.s\n"
+ "movprfx z28, z16\n fmla z28.s, p3/M, z0.s, z9.s\n"
+ "ld1w { z9.s }, p2/Z, [x25]\n"
+ "add x16, x16, #0x1\n"
+ "fmla z31.s, p3/M, z0.s, z10.s\n"
+ "ld1w { z10.s }, p2/Z, [x26, x24, LSL #2]\n"
+ "fmla z30.s, p3/M, z2.s, z11.s\n"
+ "ld1w { z11.s }, p2/Z, [x25, x23, LSL #2]\n"
"ldr x19, [%x[params_struct], %[offsetof_args_n_tile_cols]]\n"
- "cmp x17, x19\n"
- "fmla z30.s, p3/M, z7.s, z10.s\n"
- "ld1w { z10.s }, p2/Z, [x27, x24, LSL #2]\n"
- "fmla z31.s, p3/M, z3.s, z11.s\n"
- "csel x17, x17, XZR, LT\n"
+ "fmla z29.s, p3/M, z2.s, z12.s\n"
+ "cmp x16, x19\n"
+ "fmla z31.s, p3/M, z5.s, z12.s\n"
+ "fmla z30.s, p3/M, z4.s, z12.s\n"
+ "csel x16, x16, XZR, LT\n"
+ "fmla z28.s, p3/M, z1.s, z12.s\n"
+ "ld1w { z12.s }, p2/Z, [x10, x11, LSL #2]\n"
+ "csel x17, x17, x21, LT\n"
+ "fmla z29.s, p3/M, z6.s, z9.s\n"
+ "ld1w { z9.s }, p2/Z, [x10, x24, LSL #2]\n"
+ "cmp x17, x20\n"
+ "fmla z31.s, p3/M, z7.s, z13.s\n"
+ "fmla z30.s, p3/M, z6.s, z13.s\n"
+ "fmla z28.s, p3/M, z3.s, z13.s\n"
+ "fmla z29.s, p3/M, z4.s, z13.s\n"
+ "fmla z31.s, p3/M, z1.s, z12.s\n"
+ "fmla z30.s, p3/M, z0.s, z12.s\n"
+ "ld1w { z12.s }, p2/Z, [x27, x23, LSL #2]\n"
+ "fmla z28.s, p3/M, z8.s, z11.s\n"
+ "ld1w { z11.s }, p2/Z, [x27]\n"
+ "fmla z29.s, p3/M, z5.s, z10.s\n"
+ "fmla z31.s, p3/M, z2.s, z9.s\n"
+ "fmla z30.s, p3/M, z1.s, z9.s\n"
+ "ld1w { z9.s }, p2/Z, [x26]\n"
+ "fmla z28.s, p3/M, z4.s, z10.s\n"
"fmla z29.s, p3/M, z0.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x26, x13, LSL #2]\n"
- "csel x8, x8, x21, LT\n"
+ "fmla z31.s, p3/M, z8.s, z10.s\n"
+ "fmla z30.s, p3/M, z7.s, z10.s\n"
+ "ld1w { z10.s }, p2/Z, [x26, x23, LSL #2]\n"
"fmla z28.s, p3/M, z2.s, z12.s\n"
- "cmp x8, x20\n"
- "fmla z30.s, p3/M, z5.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x26, x25, LSL #2]\n"
- "fmla z31.s, p3/M, z6.s, z9.s\n"
"fmla z29.s, p3/M, z3.s, z9.s\n"
+ "fmla z31.s, p3/M, z3.s, z11.s\n"
+ "ld1w { z11.s }, p2/Z, [x25, x11, LSL #2]\n"
+ "fmla z30.s, p3/M, z5.s, z12.s\n"
+ "ld1w { z12.s }, p2/Z, [x25, x24, LSL #2]\n"
"fmla z28.s, p3/M, z5.s, z10.s\n"
- "fmla z30.s, p3/M, z8.s, z10.s\n"
"fmla z29.s, p3/M, z7.s, z11.s\n"
- "fmax z31.s, p3/M, z31.s, z18.s\n"
+ "fmla z31.s, p3/M, z6.s, z9.s\n"
+ "fmla z30.s, p3/M, z8.s, z10.s\n"
"fmla z28.s, p3/M, z6.s, z11.s\n"
- "fmax z30.s, p3/M, z30.s, z18.s\n"
"fmla z29.s, p3/M, z8.s, z12.s\n"
- "fmin z31.s, p3/M, z31.s, z17.s\n"
- "st1w { z31.s }, p0, [x9]\n"
+ "fmax z31.s, p3/M, z31.s, z18.s\n"
+ "fmax z30.s, p3/M, z30.s, z18.s\n"
"fmla z28.s, p3/M, z7.s, z12.s\n"
- "fmin z30.s, p3/M, z30.s, z17.s\n"
- "st1w { z30.s }, p0, [x9, x10, LSL #2]\n"
"fmax z29.s, p3/M, z29.s, z18.s\n"
- "fmax z28.s, p3/M, z28.s, z18.s\n"
+ "fmin z31.s, p3/M, z31.s, z17.s\n"
+ "st1w { z31.s }, p0, [x28]\n"
+ "fmin z30.s, p3/M, z30.s, z17.s\n"
"fmin z29.s, p3/M, z29.s, z17.s\n"
+ "st1w { z30.s }, p0, [x28, x9, LSL #2]\n"
+ "fmax z28.s, p3/M, z28.s, z18.s\n"
"st1w { z29.s }, p0, [x22]\n"
"fmin z28.s, p3/M, z28.s, z17.s\n"
- "st1w { z28.s }, p0, [x22, x10, LSL #2]\n"
+ "st1w { z28.s }, p0, [x22, x9, LSL #2]\n"
"blt 1b\n"
:
: [n_channels] "r" ((unsigned long) n_channels), [offsetof_args_inptr] "I" (offsetof(Args, inptr)), [offsetof_args_ld_input_col] "I" (offsetof(Args, ld_input_col)), [offsetof_args_ld_input_row] "I" (offsetof(Args, ld_input_row)), [offsetof_args_ld_output_col] "I" (offsetof(Args, ld_output_col)), [offsetof_args_ld_output_row] "I" (offsetof(Args, ld_output_row)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_n_tile_cols] "I" (offsetof(Args, n_tile_cols)), [offsetof_args_n_tile_rows] "I" (offsetof(Args, n_tile_rows)), [offsetof_args_outptr] "I" (offsetof(Args, outptr)), [offsetof_args_params] "I" (offsetof(Args, params)), [offsetof_args_tile_i] "I" (offsetof(Args, tile_i)), [offsetof_args_tile_j] "I" (offsetof(Args, tile_j)), [params_struct] "r" (¶ms_struct)
- : "cc", "memory", "p0", "p1", "p2", "p3", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z16", "z17", "z18", "z28", "z29", "z30", "z31"
+ : "cc", "memory", "p0", "p1", "p2", "p3", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z16", "z17", "z18", "z28", "z29", "z30", "z31"
);
}
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(ARM_COMPUTE_ENABLE_SVE)
+#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp
index fb41ca0..77a6c68 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp
@@ -25,7 +25,7 @@
#include <cstddef>
#include <cstdint>
-#if defined(ARM_COMPUTE_ENABLE_SVE)
+#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE)
namespace arm_conv {
namespace depthwise {
@@ -54,22 +54,22 @@
const float max
) : outptrs(outptrs), params(params), min(min), max(max)
{
- inptrs[0] = input_ptrs[0];
- inptrs[1] = input_ptrs[1];
- inptrs[2] = input_ptrs[2];
- inptrs[3] = input_ptrs[3];
- inptrs[4] = input_ptrs[4];
- inptrs[5] = input_ptrs[5];
- inptrs[6] = input_ptrs[6];
- inptrs[7] = input_ptrs[7];
- inptrs[8] = input_ptrs[8];
- inptrs[9] = input_ptrs[9];
- inptrs[10] = input_ptrs[10];
- inptrs[11] = input_ptrs[11];
- inptrs[12] = input_ptrs[12];
- inptrs[13] = input_ptrs[13];
- inptrs[14] = input_ptrs[14];
- inptrs[15] = input_ptrs[15];
+ inptrs[0] = input_ptrs[5];
+ inptrs[1] = input_ptrs[0];
+ inptrs[2] = input_ptrs[3];
+ inptrs[3] = input_ptrs[6];
+ inptrs[4] = input_ptrs[9];
+ inptrs[5] = input_ptrs[12];
+ inptrs[6] = input_ptrs[15];
+ inptrs[7] = input_ptrs[1];
+ inptrs[8] = input_ptrs[2];
+ inptrs[9] = input_ptrs[10];
+ inptrs[10] = input_ptrs[4];
+ inptrs[11] = input_ptrs[7];
+ inptrs[12] = input_ptrs[8];
+ inptrs[13] = input_ptrs[11];
+ inptrs[14] = input_ptrs[13];
+ inptrs[15] = input_ptrs[14];
}
};
@@ -78,216 +78,194 @@
activation_min, activation_max);
__asm__ __volatile__(
- "ldr x2, [%x[params_struct], %[offsetof_args_outptrs]]\n"
+ "ldr x19, [%x[params_struct], %[offsetof_args_outptrs]]\n"
"ptrue p3.b\n"
- "ldr x3, [%x[params_struct], %[offsetof_args_params]]\n"
- "add x19, %x[params_struct], %[offsetof_Args_inptrs]\n"
+ "ldr x15, [%x[params_struct], %[offsetof_args_params]]\n"
+ "add x14, %x[params_struct], %[offsetof_Args_inptrs]\n"
"ld1rw { z18.s }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n"
- "cntb x4, ALL, MUL #2\n"
- "ldp x5, x6, [x19, #0x0]\n"
- "mov x7, #0x0\n"
- "ldp x8, x17, [x19, #0x10]\n"
- "cntw x16\n"
- "ldp x15, x14, [x19, #0x20]\n"
- "sub x13, XZR, x16\n"
- "ldp x12, x11, [x19, #0x30]\n"
- "whilelt p2.s, XZR, %x[n_channels]\n"
- "ldp x10, x9, [x19, #0x40]\n"
- "cmp x16, %x[n_channels]\n"
- "ldp x28, x27, [x19, #0x50]\n"
- "ldp x26, x25, [x19, #0x60]\n"
- "ldp x24, x23, [x19, #0x70]\n"
- "ldp x22, x21, [x2, #0x0]\n"
- "ldp x20, x19, [x2, #0x10]\n"
+ "mov x13, #0x0\n"
"ld1rw { z17.s }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n"
- "ld1w { z16.s }, p3/Z, [x3]\n"
- "mov z31.d, z16.d\n"
- "ld1w { z0.s }, p3/Z, [x3, #1, MUL VL]\n"
- "mov z30.d, z16.d\n"
- "ld1w { z1.s }, p3/Z, [x3, #2, MUL VL]\n"
- "mov z29.d, z16.d\n"
- "ld1w { z2.s }, p3/Z, [x3, #3, MUL VL]\n"
- "mov z28.d, z16.d\n"
- "ld1w { z3.s }, p3/Z, [x3, #4, MUL VL]\n"
- "ld1w { z4.s }, p3/Z, [x3, #5, MUL VL]\n"
- "ld1w { z5.s }, p3/Z, [x3, #6, MUL VL]\n"
- "ld1w { z6.s }, p3/Z, [x3, #7, MUL VL]\n"
- "addvl x3, x3, #16\n"
- "ld1w { z9.s }, p2/Z, [x14, x7, LSL #2]\n"
- "ld1w { z7.s }, p3/Z, [x3, #-8, MUL VL]\n"
- "ld1w { z8.s }, p3/Z, [x3, #-7, MUL VL]\n"
- "addvl x3, x3, #-6\n"
- "prfm pldl1keep, [x14, x4]\n"
- "ld1w { z10.s }, p2/Z, [x5, x7, LSL #2]\n"
- "prfm pldl1keep, [x5, x4]\n"
- "ld1w { z11.s }, p2/Z, [x17, x7, LSL #2]\n"
- "prfm pldl1keep, [x17, x4]\n"
- "ld1w { z12.s }, p2/Z, [x12, x7, LSL #2]\n"
- "prfm pldl1keep, [x12, x4]\n"
- "ld1w { z13.s }, p2/Z, [x9, x7, LSL #2]\n"
- "prfm pldl1keep, [x9, x4]\n"
+ "cntw x12\n"
+ "ldp x11, x10, [x19, #0x0]\n"
+ "sub x9, XZR, x12\n"
+ "ldp x28, x27, [x19, #0x10]\n"
+ "whilelt p2.s, XZR, %x[n_channels]\n"
+ "ld1w { z16.s }, p3/Z, [x15]\n"
+ "cmp x12, %x[n_channels]\n"
+ "ld1w { z0.s }, p3/Z, [x15, #1, MUL VL]\n"
+ "ld1w { z1.s }, p3/Z, [x15, #2, MUL VL]\n"
+ "ld1w { z2.s }, p3/Z, [x15, #3, MUL VL]\n"
+ "ld1w { z3.s }, p3/Z, [x15, #4, MUL VL]\n"
+ "ld1w { z4.s }, p3/Z, [x15, #5, MUL VL]\n"
+ "ld1w { z5.s }, p3/Z, [x15, #6, MUL VL]\n"
+ "ld1w { z6.s }, p3/Z, [x15, #7, MUL VL]\n"
+ "addvl x15, x15, #16\n"
+ "ldp x26, x25, [x14, #0x0]\n"
+ "ld1w { z7.s }, p3/Z, [x15, #-8, MUL VL]\n"
+ "ld1w { z8.s }, p3/Z, [x15, #-7, MUL VL]\n"
+ "addvl x15, x15, #-6\n"
+ "ld1w { z9.s }, p2/Z, [x26, x13, LSL #2]\n"
+ "ld1w { z10.s }, p2/Z, [x25, x13, LSL #2]\n"
+ "ldp x24, x23, [x14, #0x10]\n"
+ "ldr x22, [x14, #0x20]\n"
+ "ld1w { z11.s }, p2/Z, [x24, x13, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x23, x13, LSL #2]\n"
+ "ld1w { z13.s }, p2/Z, [x22, x13, LSL #2]\n"
"bge 2f\n"
"1:" // Channel loop
- "fmla z31.s, p3/M, z4.s, z9.s\n"
- "prfm pldl1keep, [x26, x4]\n"
- "whilelt p1.s, x16, %x[n_channels]\n"
- "fmla z30.s, p3/M, z3.s, z9.s\n"
- "prfm pldl1keep, [x23, x4]\n"
- "incw x13\n"
- "fmla z29.s, p3/M, z1.s, z9.s\n"
- "prfm pldl1keep, [x6, x4]\n"
+ "movprfx z31, z16\n fmla z31.s, p3/M, z4.s, z9.s\n"
+ "ldr x21, [x14, #0x28]\n"
+ "whilelt p1.s, x12, %x[n_channels]\n"
+ "movprfx z30, z16\n fmla z30.s, p3/M, z3.s, z9.s\n"
+ "ldr x20, [x14, #0x30]\n"
+ "incw x9\n"
+ "movprfx z29, z16\n fmla z29.s, p3/M, z1.s, z9.s\n"
+ "ldr x19, [x14, #0x38]\n"
"mov p0.b, p2.b\n"
- "fmla z28.s, p3/M, z0.s, z9.s\n"
- "ld1w { z9.s }, p2/Z, [x26, x7, LSL #2]\n"
- "prfm pldl1keep, [x8, x4]\n"
+ "movprfx z28, z16\n fmla z28.s, p3/M, z0.s, z9.s\n"
+ "ld1w { z9.s }, p2/Z, [x21, x13, LSL #2]\n"
+ "ldr x26, [x14, #0x40]\n"
"fmla z31.s, p3/M, z0.s, z10.s\n"
- "ld1w { z10.s }, p2/Z, [x28, x7, LSL #2]\n"
+ "ldr x25, [x14, #0x48]\n"
"fmla z30.s, p3/M, z2.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x23, x7, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x20, x13, LSL #2]\n"
"fmla z29.s, p3/M, z2.s, z12.s\n"
- "prfm pldl1keep, [x28, x4]\n"
+ "ldr x24, [x14, #0x50]\n"
"fmla z28.s, p3/M, z1.s, z12.s\n"
- "prfm pldl1keep, [x15, x4]\n"
+ "ld1w { z10.s }, p2/Z, [x25, x13, LSL #2]\n"
"fmla z31.s, p3/M, z5.s, z12.s\n"
- "prfm pldl1keep, [x11, x4]\n"
+ "ldr x23, [x14, #0x58]\n"
"fmla z30.s, p3/M, z4.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x6, x7, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x19, x13, LSL #2]\n"
"fmla z29.s, p3/M, z6.s, z9.s\n"
- "ld1w { z9.s }, p2/Z, [x8, x7, LSL #2]\n"
+ "ld1w { z9.s }, p2/Z, [x26, x13, LSL #2]\n"
"fmla z28.s, p3/M, z3.s, z13.s\n"
- "prfm pldl1keep, [x10, x4]\n"
+ "ldr x22, [x14, #0x60]\n"
"fmla z31.s, p3/M, z7.s, z13.s\n"
- "prfm pldl1keep, [x27, x4]\n"
+ "ldr x21, [x14, #0x68]\n"
"fmla z30.s, p3/M, z6.s, z13.s\n"
- "prfm pldl1keep, [x25, x4]\n"
+ "ldr x20, [x14, #0x70]\n"
"fmla z29.s, p3/M, z4.s, z13.s\n"
- "prfm pldl1keep, [x24, x4]\n"
+ "ldr x19, [x14, #0x78]\n"
"fmla z28.s, p3/M, z8.s, z11.s\n"
- "addvl x4, x4, #1\n"
+ "ld1w { z11.s }, p2/Z, [x24, x13, LSL #2]\n"
"fmla z31.s, p3/M, z1.s, z12.s\n"
- "ld1w { z11.s }, p2/Z, [x15, x7, LSL #2]\n"
- "prfm pldl1keep, [x14, x4]\n"
+ "ldp x26, x25, [x14, #0x0]\n"
"fmla z30.s, p3/M, z0.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x11, x7, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x23, x13, LSL #2]\n"
"fmla z29.s, p3/M, z5.s, z10.s\n"
- "prfm pldl1keep, [x5, x4]\n"
+ "ldp x24, x23, [x14, #0x10]\n"
"fmla z28.s, p3/M, z4.s, z10.s\n"
- "prfm pldl1keep, [x17, x4]\n"
+ "ld1w { z16.s }, p3/Z, [x15]\n"
"fmla z31.s, p3/M, z2.s, z9.s\n"
- "prfm pldl1keep, [x12, x4]\n"
+ "ld1w { z4.s }, p3/Z, [x15, #5, MUL VL]\n"
"fmla z30.s, p3/M, z1.s, z9.s\n"
- "ld1w { z9.s }, p2/Z, [x10, x7, LSL #2]\n"
+ "ld1w { z9.s }, p2/Z, [x22, x13, LSL #2]\n"
"fmla z29.s, p3/M, z0.s, z11.s\n"
- "ld1w { z13.s }, p1/Z, [x9, x16, LSL #2]\n"
+ "ldr x22, [x14, #0x20]\n"
"fmla z28.s, p3/M, z2.s, z12.s\n"
- "prfm pldl1keep, [x9, x4]\n"
+ "ld1w { z0.s }, p3/Z, [x15, #1, MUL VL]\n"
"fmla z31.s, p3/M, z8.s, z10.s\n"
- "ld1w { z16.s }, p3/Z, [x3]\n"
+ "ld1w { z1.s }, p3/Z, [x15, #2, MUL VL]\n"
"fmla z30.s, p3/M, z7.s, z10.s\n"
- "ld1w { z10.s }, p2/Z, [x27, x7, LSL #2]\n"
+ "ld1w { z10.s }, p2/Z, [x21, x13, LSL #2]\n"
"fmla z29.s, p3/M, z3.s, z9.s\n"
- "ld1w { z0.s }, p3/Z, [x3, #1, MUL VL]\n"
+ "ld1w { z13.s }, p1/Z, [x22, x12, LSL #2]\n"
"fmla z31.s, p3/M, z3.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x25, x7, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x20, x13, LSL #2]\n"
"fmla z28.s, p3/M, z5.s, z10.s\n"
- "ld1w { z1.s }, p3/Z, [x3, #2, MUL VL]\n"
+ "ld1w { z2.s }, p3/Z, [x15, #3, MUL VL]\n"
"fmla z30.s, p3/M, z5.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x24, x7, LSL #2]\n"
- "incw x7\n"
+ "ld1w { z12.s }, p2/Z, [x19, x13, LSL #2]\n"
+ "incw x13\n"
"fmla z29.s, p3/M, z7.s, z11.s\n"
- "ld1w { z2.s }, p3/Z, [x3, #3, MUL VL]\n"
- "whilelt p2.s, x7, %x[n_channels]\n"
+ "ld1w { z3.s }, p3/Z, [x15, #4, MUL VL]\n"
+ "whilelt p2.s, x13, %x[n_channels]\n"
"fmla z31.s, p3/M, z6.s, z9.s\n"
- "ld1w { z9.s }, p1/Z, [x14, x16, LSL #2]\n"
+ "ld1w { z9.s }, p1/Z, [x26, x12, LSL #2]\n"
"fmla z28.s, p3/M, z6.s, z11.s\n"
- "ld1w { z11.s }, p1/Z, [x17, x16, LSL #2]\n"
+ "ld1w { z11.s }, p1/Z, [x24, x12, LSL #2]\n"
"fmla z30.s, p3/M, z8.s, z10.s\n"
- "ld1w { z10.s }, p1/Z, [x5, x16, LSL #2]\n"
- "ld1w { z3.s }, p3/Z, [x3, #4, MUL VL]\n"
+ "ld1w { z10.s }, p1/Z, [x25, x12, LSL #2]\n"
+ "ld1w { z5.s }, p3/Z, [x15, #6, MUL VL]\n"
"fmla z29.s, p3/M, z8.s, z12.s\n"
- "ld1w { z4.s }, p3/Z, [x3, #5, MUL VL]\n"
+ "ld1w { z6.s }, p3/Z, [x15, #7, MUL VL]\n"
"fmla z28.s, p3/M, z7.s, z12.s\n"
+ "addvl x15, x15, #16\n"
"fmax z31.s, p3/M, z31.s, z18.s\n"
- "ld1w { z12.s }, p1/Z, [x12, x16, LSL #2]\n"
- "incw x16\n"
+ "ld1w { z12.s }, p1/Z, [x23, x12, LSL #2]\n"
+ "incw x12\n"
"fmax z30.s, p3/M, z30.s, z18.s\n"
- "ld1w { z5.s }, p3/Z, [x3, #6, MUL VL]\n"
- "cmp x16, %x[n_channels]\n"
+ "ld1w { z7.s }, p3/Z, [x15, #-8, MUL VL]\n"
+ "cmp x12, %x[n_channels]\n"
"fmax z29.s, p3/M, z29.s, z18.s\n"
- "ld1w { z6.s }, p3/Z, [x3, #7, MUL VL]\n"
- "addvl x3, x3, #16\n"
- "fmin z31.s, p3/M, z31.s, z17.s\n"
- "ld1w { z7.s }, p3/Z, [x3, #-8, MUL VL]\n"
+ "ld1w { z8.s }, p3/Z, [x15, #-7, MUL VL]\n"
+ "addvl x15, x15, #-6\n"
"fmax z28.s, p3/M, z28.s, z18.s\n"
- "ld1w { z8.s }, p3/Z, [x3, #-7, MUL VL]\n"
- "addvl x3, x3, #-6\n"
+ "fmin z31.s, p3/M, z31.s, z17.s\n"
+ "st1w { z31.s }, p0, [x11, x9, LSL #2]\n"
"fmin z30.s, p3/M, z30.s, z17.s\n"
- "st1w { z31.s }, p0, [x22, x13, LSL #2]\n"
- "mov z31.d, z16.d\n"
"fmin z29.s, p3/M, z29.s, z17.s\n"
- "st1w { z30.s }, p0, [x21, x13, LSL #2]\n"
- "mov z30.d, z16.d\n"
+ "st1w { z30.s }, p0, [x10, x9, LSL #2]\n"
"fmin z28.s, p3/M, z28.s, z17.s\n"
- "st1w { z29.s }, p0, [x20, x13, LSL #2]\n"
- "mov z29.d, z16.d\n"
- "st1w { z28.s }, p0, [x19, x13, LSL #2]\n"
- "mov z28.d, z16.d\n"
+ "st1w { z29.s }, p0, [x28, x9, LSL #2]\n"
+ "st1w { z28.s }, p0, [x27, x9, LSL #2]\n"
"blt 1b\n"
"2:" // Channel tail
- "fmla z31.s, p3/M, z4.s, z9.s\n"
- "prfm pldl1keep, [x26, x4]\n"
- "incw x13\n"
- "fmla z30.s, p3/M, z3.s, z9.s\n"
- "prfm pldl1keep, [x23, x4]\n"
+ "movprfx z31, z16\n fmla z31.s, p3/M, z4.s, z9.s\n"
+ "ldr x21, [x14, #0x28]\n"
+ "incw x9\n"
+ "movprfx z30, z16\n fmla z30.s, p3/M, z3.s, z9.s\n"
+ "ldr x20, [x14, #0x30]\n"
"mov p0.b, p2.b\n"
- "fmla z29.s, p3/M, z1.s, z9.s\n"
- "prfm pldl1keep, [x6, x4]\n"
- "fmla z28.s, p3/M, z0.s, z9.s\n"
- "ld1w { z9.s }, p2/Z, [x26, x7, LSL #2]\n"
- "prfm pldl1keep, [x8, x4]\n"
+ "movprfx z29, z16\n fmla z29.s, p3/M, z1.s, z9.s\n"
+ "ldr x19, [x14, #0x38]\n"
+ "movprfx z28, z16\n fmla z28.s, p3/M, z0.s, z9.s\n"
+ "ld1w { z9.s }, p2/Z, [x21, x13, LSL #2]\n"
+ "ldr x26, [x14, #0x40]\n"
"fmla z31.s, p3/M, z0.s, z10.s\n"
- "ld1w { z10.s }, p2/Z, [x28, x7, LSL #2]\n"
+ "ldr x25, [x14, #0x48]\n"
"fmla z30.s, p3/M, z2.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x23, x7, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x20, x13, LSL #2]\n"
"fmla z29.s, p3/M, z2.s, z12.s\n"
"fmla z28.s, p3/M, z1.s, z12.s\n"
- "prfm pldl1keep, [x28, x4]\n"
- "prfm pldl1keep, [x15, x4]\n"
+ "ld1w { z10.s }, p2/Z, [x25, x13, LSL #2]\n"
+ "ldr x24, [x14, #0x50]\n"
"fmla z31.s, p3/M, z5.s, z12.s\n"
- "prfm pldl1keep, [x11, x4]\n"
+ "ldr x23, [x14, #0x58]\n"
"fmla z30.s, p3/M, z4.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x6, x7, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x19, x13, LSL #2]\n"
"fmla z29.s, p3/M, z6.s, z9.s\n"
"fmla z28.s, p3/M, z3.s, z13.s\n"
- "ld1w { z9.s }, p2/Z, [x8, x7, LSL #2]\n"
- "prfm pldl1keep, [x10, x4]\n"
+ "ld1w { z9.s }, p2/Z, [x26, x13, LSL #2]\n"
+ "ldr x22, [x14, #0x60]\n"
"fmla z31.s, p3/M, z7.s, z13.s\n"
- "prfm pldl1keep, [x27, x4]\n"
+ "ldr x21, [x14, #0x68]\n"
"fmla z30.s, p3/M, z6.s, z13.s\n"
- "prfm pldl1keep, [x25, x4]\n"
+ "ldr x20, [x14, #0x70]\n"
"fmla z29.s, p3/M, z4.s, z13.s\n"
"fmla z28.s, p3/M, z8.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x15, x7, LSL #2]\n"
- "prfm pldl1keep, [x24, x4]\n"
+ "ld1w { z11.s }, p2/Z, [x24, x13, LSL #2]\n"
+ "ldr x19, [x14, #0x78]\n"
"fmla z31.s, p3/M, z1.s, z12.s\n"
"fmla z30.s, p3/M, z0.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x11, x7, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x23, x13, LSL #2]\n"
"fmla z29.s, p3/M, z5.s, z10.s\n"
"fmla z28.s, p3/M, z4.s, z10.s\n"
"fmla z31.s, p3/M, z2.s, z9.s\n"
"fmla z30.s, p3/M, z1.s, z9.s\n"
- "ld1w { z9.s }, p2/Z, [x10, x7, LSL #2]\n"
+ "ld1w { z9.s }, p2/Z, [x22, x13, LSL #2]\n"
"fmla z29.s, p3/M, z0.s, z11.s\n"
"fmla z28.s, p3/M, z2.s, z12.s\n"
"fmla z31.s, p3/M, z8.s, z10.s\n"
"fmla z30.s, p3/M, z7.s, z10.s\n"
- "ld1w { z10.s }, p2/Z, [x27, x7, LSL #2]\n"
+ "ld1w { z10.s }, p2/Z, [x21, x13, LSL #2]\n"
"fmla z29.s, p3/M, z3.s, z9.s\n"
"fmla z31.s, p3/M, z3.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x25, x7, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x20, x13, LSL #2]\n"
"fmla z28.s, p3/M, z5.s, z10.s\n"
"fmla z30.s, p3/M, z5.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x24, x7, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x19, x13, LSL #2]\n"
"fmla z29.s, p3/M, z7.s, z11.s\n"
"fmla z31.s, p3/M, z6.s, z9.s\n"
"fmla z28.s, p3/M, z6.s, z11.s\n"
@@ -298,21 +276,21 @@
"fmax z30.s, p3/M, z30.s, z18.s\n"
"fmax z29.s, p3/M, z29.s, z18.s\n"
"fmin z31.s, p3/M, z31.s, z17.s\n"
- "st1w { z31.s }, p0, [x22, x13, LSL #2]\n"
+ "st1w { z31.s }, p0, [x11, x9, LSL #2]\n"
"fmin z30.s, p3/M, z30.s, z17.s\n"
"fmin z29.s, p3/M, z29.s, z17.s\n"
- "st1w { z30.s }, p0, [x21, x13, LSL #2]\n"
+ "st1w { z30.s }, p0, [x10, x9, LSL #2]\n"
"fmax z28.s, p3/M, z28.s, z18.s\n"
- "st1w { z29.s }, p0, [x20, x13, LSL #2]\n"
+ "st1w { z29.s }, p0, [x28, x9, LSL #2]\n"
"fmin z28.s, p3/M, z28.s, z17.s\n"
- "st1w { z28.s }, p0, [x19, x13, LSL #2]\n"
+ "st1w { z28.s }, p0, [x27, x9, LSL #2]\n"
:
: [n_channels] "r" ((unsigned long) n_channels), [offsetof_Args_inptrs] "I" (offsetof(Args, inptrs)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_outptrs] "I" (offsetof(Args, outptrs)), [offsetof_args_params] "I" (offsetof(Args, params)), [params_struct] "r" (¶ms_struct)
- : "cc", "memory", "p0", "p1", "p2", "p3", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z16", "z17", "z18", "z28", "z29", "z30", "z31"
+ : "cc", "memory", "p0", "p1", "p2", "p3", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z16", "z17", "z18", "z28", "z29", "z30", "z31"
);
}
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(ARM_COMPUTE_ENABLE_SVE)
+#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp
index ef5f418..eb9de9f 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp
@@ -28,7 +28,7 @@
#pragma once
-#if defined(ARM_COMPUTE_ENABLE_SVE)
+#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE)
namespace arm_conv {
namespace depthwise {
@@ -36,15 +36,17 @@
void sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl(const float *const *const, float *const *const, const void *, unsigned int, const float, const float);
void sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float);
-struct sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst
+class sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst : public IDepthwiseDepthfirstStrategy
{
- typedef float bias_type;
- typedef float input_type;
- typedef float weight_type;
- typedef float return_type;
-
+ private:
typedef void (*indirect_kern_type)(const float *const *const, float *const *const, const void *, unsigned int, const float, const float);
+ indirect_kern_type m_indirect_kernel = sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl;
+
typedef void (*direct_kern_type)(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float);
+ direct_kern_type m_direct_kernel = sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl;
+
+ public:
+ typedef float return_type;
constexpr static arm_gemm::VLType vl_type = arm_gemm::VLType::SVE;
@@ -60,13 +62,60 @@
constexpr static unsigned int input_rows = 5;
constexpr static unsigned int input_cols = 5;
- indirect_kern_type indirect_kernel = sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl;
- direct_kern_type direct_kernel = sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl;
-
sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst(const CPUInfo *) {}
+
+ arm_gemm::VLType get_vl_type(void) const override { return vl_type; }
+
+ unsigned int get_kernel_rows(void) const override { return kernel_rows; }
+ unsigned int get_kernel_cols(void) const override { return kernel_cols; }
+
+ unsigned int get_stride_rows(void) const override { return stride_rows; }
+ unsigned int get_stride_cols(void) const override { return stride_cols; }
+
+ unsigned int get_output_rows(void) const override { return output_rows; }
+ unsigned int get_output_cols(void) const override { return output_cols; }
+
+ unsigned int get_input_rows(void) const override { return input_rows; }
+ unsigned int get_input_cols(void) const override { return input_cols; }
+
+ void indirect_kernel(
+ const void *const *const input_ptrs,
+ void *const *const outptrs,
+ const void *params,
+ unsigned int n_channels,
+ const void *activation_min,
+ const void *activation_max
+ ) const override
+ {
+ m_indirect_kernel(
+ reinterpret_cast<const float *const *>(input_ptrs),
+ reinterpret_cast<float *const *>(outptrs),
+ params, n_channels,
+ *static_cast<const float *>(activation_min),
+ *static_cast<const float *>(activation_max)
+ );
+ }
+
+ void direct_kernel(
+ const unsigned int n_tile_rows, const unsigned int n_tile_cols,
+ const void *inptr, int64_t ld_input_row, int64_t ld_input_col,
+ void *outptr, int64_t ld_output_row, int64_t ld_output_col,
+ const void *params, unsigned int n_channels,
+ const void *activation_min, const void *activation_max
+ ) const override
+ {
+ m_direct_kernel(
+ n_tile_rows, n_tile_cols,
+ static_cast<const float *>(inptr), ld_input_row, ld_input_col,
+ static_cast<float *>(outptr), ld_output_row, ld_output_col,
+ params, n_channels,
+ *static_cast<const float *>(activation_min),
+ *static_cast<const float *>(activation_max)
+ );
+ }
};
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(ARM_COMPUTE_ENABLE_SVE)
+#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp
index 6bc333b..c485b7d 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp
@@ -25,7 +25,7 @@
#include <cstddef>
#include <cstdint>
-#if defined(ARM_COMPUTE_ENABLE_SVE)
+#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE)
namespace arm_conv {
namespace depthwise {
@@ -88,167 +88,123 @@
__asm__ __volatile__(
"ptrue p3.b\n"
- "mov x3, #0x0\n"
- "mov x4, #0x0\n"
+ "mov x6, #0x0\n"
+ "mov x7, #0x0\n"
"1:" // Tile loop
- "str x3, [%x[params_struct], %[offsetof_args_tile_i]]\n"
- "mov x22, #0x3\n"
- "str x4, [%x[params_struct], %[offsetof_args_tile_j]]\n"
- "cntb x5\n"
- "ldr x6, [%x[params_struct], %[offsetof_args_params]]\n"
- "add x5, x5, XZR, LSL #4\n"
- "ldr x20, [%x[params_struct], %[offsetof_args_ld_input_row]]\n"
- "cntb x7\n"
- "ldr x8, [%x[params_struct], %[offsetof_args_ld_input_col]]\n"
- "cntb x17\n"
- "ldr x16, [%x[params_struct], %[offsetof_args_inptr]]\n"
- "mul x19, x3, x20\n" // offset = tile_i * ld_input_row
- "ldr x21, [%x[params_struct], %[offsetof_args_ld_output_row]]\n"
- "madd x19, x4, x8, x19\n" // offset += tile_j * ld_input_col
- "ldr x15, [%x[params_struct], %[offsetof_args_ld_output_col]]\n"
- "mul x19, x19, x22\n" // offset *= kernel_stride * output_size
- "ldr x14, [%x[params_struct], %[offsetof_args_outptr]]\n"
- "add x16, x16, x19, LSL #2\n" // inptr[0] += offset * sizeof(float)
+ "str x6, [%x[params_struct], %[offsetof_args_tile_i]]\n"
+ "mov x24, #0x3\n"
+ "str x7, [%x[params_struct], %[offsetof_args_tile_j]]\n"
+ "mov x23, #0x3\n"
+ "ldr x8, [%x[params_struct], %[offsetof_args_params]]\n"
+ "mov x17, #0x0\n"
+ "ldr x22, [%x[params_struct], %[offsetof_args_ld_input_row]]\n"
+ "cntw x16\n"
+ "ldr x15, [%x[params_struct], %[offsetof_args_ld_input_col]]\n"
+ "sub x21, XZR, x16\n"
+ "ldr x14, [%x[params_struct], %[offsetof_args_inptr]]\n"
+ "mul x19, x6, x22\n" // offset = tile_i * ld_input_row
+ "ldr x20, [%x[params_struct], %[offsetof_args_ld_output_row]]\n"
+ "madd x19, x7, x15, x19\n" // offset += tile_j * ld_input_col
+ "ldr x13, [%x[params_struct], %[offsetof_args_ld_output_col]]\n"
+ "mul x19, x19, x24\n" // offset *= kernel_stride * output_size
+ "ldr x12, [%x[params_struct], %[offsetof_args_outptr]]\n"
+ "add x14, x14, x19, LSL #2\n" // inptr[0] += offset * sizeof(float)
"ld1rw { z18.s }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n"
- "add x13, x16, x20, LSL #2\n"
+ "add x11, x14, x22, LSL #2\n"
"ld1rw { z17.s }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n"
- "add x12, x13, x20, LSL #2\n"
- "ld1w { z16.s }, p3/Z, [x6]\n"
- "mov z31.d, z16.d\n"
- "ld1w { z0.s }, p3/Z, [x6, #1, MUL VL]\n"
- "add x11, x12, x20, LSL #2\n"
- "mov z30.d, z16.d\n"
- "ld1w { z1.s }, p3/Z, [x6, #2, MUL VL]\n"
- "add x10, x11, x20, LSL #2\n"
- "mov z29.d, z16.d\n"
- "ld1w { z2.s }, p3/Z, [x6, #3, MUL VL]\n"
- "add x9, x8, x8\n"
- "mov z28.d, z16.d\n"
- "ld1w { z3.s }, p3/Z, [x6, #4, MUL VL]\n"
- "add x28, x9, x8\n"
- "mov z27.d, z16.d\n"
- "ld1w { z4.s }, p3/Z, [x6, #5, MUL VL]\n"
- "add x27, x28, x8\n"
- "mov z26.d, z16.d\n"
- "ld1w { z5.s }, p3/Z, [x6, #6, MUL VL]\n"
- "add x7, x7, x8, LSL #4\n"
- "mov z25.d, z16.d\n"
- "ld1w { z6.s }, p3/Z, [x6, #7, MUL VL]\n"
- "add x17, x17, x9, LSL #4\n"
- "mov z24.d, z16.d\n"
- "prfm pldl1keep, [x12, x17]\n"
- "cntb x26\n"
- "mov z23.d, z16.d\n"
- "prfm pldl1keep, [x16, x5]\n"
- "add x26, x26, x28, LSL #4\n"
- "cntb x25\n"
- "mov x20, #0x3\n"
- "add x25, x25, x27, LSL #4\n"
- "prfm pldl1keep, [x16, x25]\n"
- "prfm pldl1keep, [x10, x5]\n"
- "mul x19, x3, x21\n" // offset = tile_i * ld_output_row
- "prfm pldl1keep, [x13, x17]\n"
- "madd x19, x4, x15, x19\n" // offset += tile_j * ld_output_col
- "add x24, x15, x15\n"
- "mul x19, x19, x20\n" // offset *= output_tile_size
- "add x14, x14, x19, LSL #2\n" // outptrs[0] += offset * sizeof(float)
- "add x23, x14, x21, LSL #2\n"
- "add x22, x23, x21, LSL #2\n"
- "mov x21, #0x0\n"
- "cntw x20\n"
- "sub x19, XZR, x20\n"
+ "add x10, x11, x22, LSL #2\n"
+ "ld1w { z16.s }, p3/Z, [x8]\n"
+ "add x9, x10, x22, LSL #2\n"
+ "ld1w { z0.s }, p3/Z, [x8, #1, MUL VL]\n"
+ "add x28, x9, x22, LSL #2\n"
+ "ld1w { z1.s }, p3/Z, [x8, #2, MUL VL]\n"
+ "add x27, x15, x15\n"
+ "ld1w { z2.s }, p3/Z, [x8, #3, MUL VL]\n"
+ "add x26, x27, x15\n"
+ "ld1w { z3.s }, p3/Z, [x8, #4, MUL VL]\n"
+ "add x25, x26, x15\n"
+ "ld1w { z4.s }, p3/Z, [x8, #5, MUL VL]\n"
+ "mul x19, x6, x20\n" // offset = tile_i * ld_output_row
+ "ld1w { z5.s }, p3/Z, [x8, #6, MUL VL]\n"
+ "madd x19, x7, x13, x19\n" // offset += tile_j * ld_output_col
+ "ld1w { z6.s }, p3/Z, [x8, #7, MUL VL]\n"
+ "mul x19, x19, x23\n" // offset *= output_tile_size
+ "add x24, x13, x13\n"
+ "add x12, x12, x19, LSL #2\n" // outptrs[0] += offset * sizeof(float)
+ "add x23, x12, x20, LSL #2\n"
+ "add x22, x23, x20, LSL #2\n"
"whilelt p2.s, XZR, %x[n_channels]\n"
- "ld1w { z9.s }, p2/Z, [x12, x9, LSL #2]\n"
- "ld1w { z10.s }, p2/Z, [x16]\n"
- "addvl x6, x6, #16\n"
- "ld1w { z11.s }, p2/Z, [x16, x27, LSL #2]\n"
- "cmp x20, %x[n_channels]\n"
- "ld1w { z7.s }, p3/Z, [x6, #-8, MUL VL]\n"
- "ld1w { z8.s }, p3/Z, [x6, #-7, MUL VL]\n"
- "addvl x6, x6, #-6\n"
- "ld1w { z12.s }, p2/Z, [x10]\n"
- "ld1w { z13.s }, p2/Z, [x13, x9, LSL #2]\n"
+ "ld1w { z9.s }, p2/Z, [x10, x27, LSL #2]\n"
+ "ld1w { z10.s }, p2/Z, [x14]\n"
+ "addvl x8, x8, #16\n"
+ "ld1w { z11.s }, p2/Z, [x14, x25, LSL #2]\n"
+ "cmp x16, %x[n_channels]\n"
+ "ld1w { z7.s }, p3/Z, [x8, #-8, MUL VL]\n"
+ "ld1w { z8.s }, p3/Z, [x8, #-7, MUL VL]\n"
+ "addvl x8, x8, #-6\n"
+ "ld1w { z12.s }, p2/Z, [x28]\n"
+ "ld1w { z13.s }, p2/Z, [x11, x27, LSL #2]\n"
"bge 3f\n"
"2:" // Tile loop: Channel loop
- "fmla z31.s, p3/M, z8.s, z9.s\n"
- "prfm pldl1keep, [x10, x25]\n"
- "whilelt p1.s, x20, %x[n_channels]\n"
- "fmla z30.s, p3/M, z7.s, z9.s\n"
- "prfm pldl1keep, [x12, x7]\n"
- "incw x19\n"
- "fmla z29.s, p3/M, z6.s, z9.s\n"
- "prfm pldl1keep, [x16, x7]\n"
- "mov p0.b, p2.b\n"
- "fmla z28.s, p3/M, z5.s, z9.s\n"
- "prfm pldl1keep, [x16, x26]\n"
+ "movprfx z31, z16\n fmla z31.s, p3/M, z8.s, z9.s\n"
+ "whilelt p1.s, x16, %x[n_channels]\n"
+ "movprfx z30, z16\n fmla z30.s, p3/M, z7.s, z9.s\n"
"incw x21\n"
- "fmla z27.s, p3/M, z4.s, z9.s\n"
- "prfm pldl1keep, [x12, x26]\n"
- "incw x20\n"
- "fmla z26.s, p3/M, z3.s, z9.s\n"
- "prfm pldl1keep, [x13, x5]\n"
- "fmla z25.s, p3/M, z2.s, z9.s\n"
- "prfm pldl1keep, [x13, x25]\n"
- "fmla z24.s, p3/M, z1.s, z9.s\n"
- "prfm pldl1keep, [x11, x5]\n"
- "fmla z23.s, p3/M, z0.s, z9.s\n"
- "prfm pldl1keep, [x11, x17]\n"
+ "movprfx z29, z16\n fmla z29.s, p3/M, z6.s, z9.s\n"
+ "mov p0.b, p2.b\n"
+ "movprfx z28, z16\n fmla z28.s, p3/M, z5.s, z9.s\n"
+ "incw x17\n"
+ "movprfx z27, z16\n fmla z27.s, p3/M, z4.s, z9.s\n"
+ "incw x16\n"
+ "movprfx z26, z16\n fmla z26.s, p3/M, z3.s, z9.s\n"
+ "movprfx z25, z16\n fmla z25.s, p3/M, z2.s, z9.s\n"
+ "movprfx z24, z16\n fmla z24.s, p3/M, z1.s, z9.s\n"
+ "movprfx z23, z16\n fmla z23.s, p3/M, z0.s, z9.s\n"
+ "ld1w { z16.s }, p3/Z, [x8]\n"
"fmla z31.s, p3/M, z0.s, z10.s\n"
- "ld1w { z10.s }, p2/Z, [x12, x28, LSL #2]\n"
+ "ld1w { z10.s }, p2/Z, [x10, x26, LSL #2]\n"
"fmla z29.s, p3/M, z2.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x12, x8, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x10, x15, LSL #2]\n"
"fmla z25.s, p3/M, z6.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x10, x27, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x28, x25, LSL #2]\n"
"fmla z30.s, p3/M, z4.s, z13.s\n"
- "prfm pldl1keep, [x11, x25]\n"
"fmla z31.s, p3/M, z5.s, z13.s\n"
- "prfm pldl1keep, [x10, x7]\n"
"fmla z29.s, p3/M, z3.s, z13.s\n"
- "prfm pldl1keep, [x13, x7]\n"
"fmla z28.s, p3/M, z2.s, z13.s\n"
- "prfm pldl1keep, [x13, x26]\n"
"fmla z27.s, p3/M, z1.s, z13.s\n"
- "prfm pldl1keep, [x10, x26]\n"
"fmla z26.s, p3/M, z0.s, z13.s\n"
- "ld1w { z13.s }, p2/Z, [x16, x8, LSL #2]\n"
+ "ld1w { z13.s }, p2/Z, [x14, x15, LSL #2]\n"
"fmla z23.s, p3/M, z8.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x16, x28, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x14, x26, LSL #2]\n"
"fmla z31.s, p3/M, z7.s, z11.s\n"
- "prfm pldl1keep, [x11, x7]\n"
"fmla z30.s, p3/M, z6.s, z11.s\n"
- "prfm pldl1keep, [x16, x17]\n"
"fmla z28.s, p3/M, z4.s, z11.s\n"
- "prfm pldl1keep, [x11, x26]\n"
"fmla z27.s, p3/M, z3.s, z11.s\n"
- "prfm pldl1keep, [x12, x5]\n"
"fmla z25.s, p3/M, z1.s, z11.s\n"
- "prfm pldl1keep, [x12, x25]\n"
"fmla z24.s, p3/M, z0.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x13]\n"
+ "ld1w { z11.s }, p2/Z, [x11]\n"
"fmla z31.s, p3/M, z1.s, z13.s\n"
- "prfm pldl1keep, [x10, x17]\n"
"fmla z30.s, p3/M, z0.s, z13.s\n"
- "ld1w { z13.s }, p2/Z, [x13, x27, LSL #2]\n"
+ "ld1w { z13.s }, p2/Z, [x11, x25, LSL #2]\n"
"fmla z29.s, p3/M, z1.s, z12.s\n"
- "ld1w { z16.s }, p3/Z, [x6]\n"
"fmla z27.s, p3/M, z5.s, z10.s\n"
"fmla z26.s, p3/M, z4.s, z10.s\n"
"fmla z30.s, p3/M, z2.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x11]\n"
+ "ld1w { z12.s }, p2/Z, [x9]\n"
"fmla z29.s, p3/M, z7.s, z10.s\n"
"fmla z24.s, p3/M, z2.s, z10.s\n"
"fmla z23.s, p3/M, z1.s, z10.s\n"
"fmla z30.s, p3/M, z8.s, z10.s\n"
- "ld1w { z10.s }, p2/Z, [x11, x9, LSL #2]\n"
+ "ld1w { z10.s }, p2/Z, [x9, x27, LSL #2]\n"
"fmla z31.s, p3/M, z3.s, z11.s\n"
"fmla z28.s, p3/M, z0.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x11, x27, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x9, x25, LSL #2]\n"
"fmla z29.s, p3/M, z5.s, z13.s\n"
"fmla z26.s, p3/M, z2.s, z13.s\n"
- "ld1w { z13.s }, p2/Z, [x10, x8, LSL #2]\n"
+ "ld1w { z13.s }, p2/Z, [x28, x15, LSL #2]\n"
"fmla z25.s, p3/M, z3.s, z12.s\n"
"fmla z28.s, p3/M, z6.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x13, x8, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x11, x15, LSL #2]\n"
"fmla z27.s, p3/M, z7.s, z10.s\n"
"fmla z26.s, p3/M, z6.s, z10.s\n"
"fmla z25.s, p3/M, z5.s, z10.s\n"
@@ -258,198 +214,164 @@
"fmla z26.s, p3/M, z8.s, z11.s\n"
"fmla z25.s, p3/M, z7.s, z13.s\n"
"fmla z24.s, p3/M, z6.s, z13.s\n"
- "ld1w { z13.s }, p2/Z, [x10, x28, LSL #2]\n"
+ "ld1w { z13.s }, p2/Z, [x28, x26, LSL #2]\n"
"fmla z23.s, p3/M, z5.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x13, x28, LSL #2]\n"
- "addvl x13, x13, #1\n"
+ "ld1w { z11.s }, p2/Z, [x11, x26, LSL #2]\n"
+ "addvl x11, x11, #1\n"
"fmla z31.s, p3/M, z4.s, z12.s\n"
"fmla z30.s, p3/M, z3.s, z12.s\n"
"fmla z28.s, p3/M, z1.s, z12.s\n"
"fmla z27.s, p3/M, z0.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x11, x8, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x9, x15, LSL #2]\n"
"fmla z29.s, p3/M, z4.s, z11.s\n"
"fmla z30.s, p3/M, z5.s, z11.s\n"
"fmla z26.s, p3/M, z1.s, z11.s\n"
"fmla z27.s, p3/M, z2.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x16, x9, LSL #2]\n"
- "addvl x16, x16, #1\n"
+ "ld1w { z11.s }, p2/Z, [x14, x27, LSL #2]\n"
+ "addvl x14, x14, #1\n"
"fmla z24.s, p3/M, z8.s, z13.s\n"
- "ld1w { z10.s }, p1/Z, [x16]\n"
+ "ld1w { z10.s }, p1/Z, [x14]\n"
"fmla z23.s, p3/M, z7.s, z13.s\n"
- "ld1w { z13.s }, p2/Z, [x11, x28, LSL #2]\n"
- "addvl x11, x11, #1\n"
+ "ld1w { z13.s }, p2/Z, [x9, x26, LSL #2]\n"
+ "addvl x9, x9, #1\n"
"fmla z28.s, p3/M, z7.s, z12.s\n"
"fmla z27.s, p3/M, z6.s, z12.s\n"
"fmla z25.s, p3/M, z4.s, z12.s\n"
"fmla z24.s, p3/M, z3.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x12]\n"
+ "ld1w { z12.s }, p2/Z, [x10]\n"
"fmla z31.s, p3/M, z2.s, z11.s\n"
"fmla z30.s, p3/M, z1.s, z11.s\n"
- "ld1w { z1.s }, p3/Z, [x6, #2, MUL VL]\n"
+ "ld1w { z1.s }, p3/Z, [x8, #2, MUL VL]\n"
"fmla z29.s, p3/M, z0.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x12, x27, LSL #2]\n"
- "addvl x12, x12, #1\n"
- "fmla z27.s, p3/M, z8.s, z13.s\n"
- "ld1w { z9.s }, p1/Z, [x12, x9, LSL #2]\n"
- "fmla z26.s, p3/M, z7.s, z13.s\n"
- "prfm pldl1keep, [x12, x17]\n"
- "fmla z24.s, p3/M, z5.s, z13.s\n"
- "prfm pldl1keep, [x16, x5]\n"
- "fmla z23.s, p3/M, z4.s, z13.s\n"
- "ld1w { z13.s }, p2/Z, [x10, x9, LSL #2]\n"
- "whilelt p2.s, x21, %x[n_channels]\n"
- "fmla z31.s, p3/M, z6.s, z12.s\n"
- "prfm pldl1keep, [x16, x25]\n"
+ "ld1w { z11.s }, p2/Z, [x10, x25, LSL #2]\n"
"addvl x10, x10, #1\n"
+ "fmla z27.s, p3/M, z8.s, z13.s\n"
+ "ld1w { z9.s }, p1/Z, [x10, x27, LSL #2]\n"
+ "fmla z26.s, p3/M, z7.s, z13.s\n"
+ "fmla z24.s, p3/M, z5.s, z13.s\n"
+ "fmla z23.s, p3/M, z4.s, z13.s\n"
+ "ld1w { z13.s }, p2/Z, [x28, x27, LSL #2]\n"
+ "whilelt p2.s, x17, %x[n_channels]\n"
+ "fmla z31.s, p3/M, z6.s, z12.s\n"
+ "ld1w { z4.s }, p3/Z, [x8, #5, MUL VL]\n"
+ "addvl x28, x28, #1\n"
"fmla z28.s, p3/M, z3.s, z12.s\n"
- "prfm pldl1keep, [x10, x5]\n"
- "cmp x20, %x[n_channels]\n"
+ "ld1w { z3.s }, p3/Z, [x8, #4, MUL VL]\n"
+ "cmp x16, %x[n_channels]\n"
"fmla z25.s, p3/M, z0.s, z12.s\n"
- "ld1w { z12.s }, p1/Z, [x10]\n"
+ "ld1w { z12.s }, p1/Z, [x28]\n"
"fmla z29.s, p3/M, z8.s, z11.s\n"
- "prfm pldl1keep, [x13, x17]\n"
+ "ld1w { z0.s }, p3/Z, [x8, #1, MUL VL]\n"
"fmla z26.s, p3/M, z5.s, z11.s\n"
- "ld1w { z0.s }, p3/Z, [x6, #1, MUL VL]\n"
+ "ld1w { z5.s }, p3/Z, [x8, #6, MUL VL]\n"
"fmla z23.s, p3/M, z2.s, z11.s\n"
- "ld1w { z11.s }, p1/Z, [x16, x27, LSL #2]\n"
+ "ld1w { z11.s }, p1/Z, [x14, x25, LSL #2]\n"
"fmla z24.s, p3/M, z7.s, z13.s\n"
- "ld1w { z2.s }, p3/Z, [x6, #3, MUL VL]\n"
+ "ld1w { z2.s }, p3/Z, [x8, #3, MUL VL]\n"
"fmla z25.s, p3/M, z8.s, z13.s\n"
- "ld1w { z3.s }, p3/Z, [x6, #4, MUL VL]\n"
"fmax z31.s, p3/M, z31.s, z18.s\n"
- "ld1w { z4.s }, p3/Z, [x6, #5, MUL VL]\n"
"fmla z23.s, p3/M, z6.s, z13.s\n"
- "ld1w { z13.s }, p1/Z, [x13, x9, LSL #2]\n"
+ "ld1w { z13.s }, p1/Z, [x11, x27, LSL #2]\n"
"fmax z30.s, p3/M, z30.s, z18.s\n"
- "ld1w { z5.s }, p3/Z, [x6, #6, MUL VL]\n"
- "fmax z29.s, p3/M, z29.s, z18.s\n"
- "ld1w { z6.s }, p3/Z, [x6, #7, MUL VL]\n"
- "addvl x6, x6, #16\n"
+ "ld1w { z6.s }, p3/Z, [x8, #7, MUL VL]\n"
+ "addvl x8, x8, #16\n"
"fmin z31.s, p3/M, z31.s, z17.s\n"
- "ld1w { z7.s }, p3/Z, [x6, #-8, MUL VL]\n"
- "fmax z28.s, p3/M, z28.s, z18.s\n"
- "ld1w { z8.s }, p3/Z, [x6, #-7, MUL VL]\n"
- "addvl x6, x6, #-6\n"
+ "ld1w { z7.s }, p3/Z, [x8, #-8, MUL VL]\n"
+ "fmax z29.s, p3/M, z29.s, z18.s\n"
+ "ld1w { z8.s }, p3/Z, [x8, #-7, MUL VL]\n"
+ "addvl x8, x8, #-6\n"
"fmin z30.s, p3/M, z30.s, z17.s\n"
- "st1w { z31.s }, p0, [x14]\n"
- "mov z31.d, z16.d\n"
+ "st1w { z31.s }, p0, [x12]\n"
"fmin z29.s, p3/M, z29.s, z17.s\n"
- "st1w { z30.s }, p0, [x14, x15, LSL #2]\n"
- "mov z30.d, z16.d\n"
- "fmin z28.s, p3/M, z28.s, z17.s\n"
- "st1w { z29.s }, p0, [x14, x24, LSL #2]\n"
- "mov z29.d, z16.d\n"
- "addvl x14, x14, #1\n"
+ "fmax z28.s, p3/M, z28.s, z18.s\n"
+ "st1w { z30.s }, p0, [x12, x13, LSL #2]\n"
"fmax z27.s, p3/M, z27.s, z18.s\n"
- "st1w { z28.s }, p0, [x23]\n"
- "mov z28.d, z16.d\n"
"fmax z26.s, p3/M, z26.s, z18.s\n"
+ "st1w { z29.s }, p0, [x12, x24, LSL #2]\n"
+ "fmin z28.s, p3/M, z28.s, z17.s\n"
+ "addvl x12, x12, #1\n"
"fmax z25.s, p3/M, z25.s, z18.s\n"
- "fmax z24.s, p3/M, z24.s, z18.s\n"
+ "st1w { z28.s }, p0, [x23]\n"
"fmin z27.s, p3/M, z27.s, z17.s\n"
- "st1w { z27.s }, p0, [x23, x15, LSL #2]\n"
- "mov z27.d, z16.d\n"
"fmin z26.s, p3/M, z26.s, z17.s\n"
- "st1w { z26.s }, p0, [x23, x24, LSL #2]\n"
- "mov z26.d, z16.d\n"
- "addvl x23, x23, #1\n"
+ "st1w { z27.s }, p0, [x23, x13, LSL #2]\n"
"fmin z25.s, p3/M, z25.s, z17.s\n"
- "st1w { z25.s }, p0, [x22]\n"
- "mov z25.d, z16.d\n"
- "fmin z24.s, p3/M, z24.s, z17.s\n"
- "st1w { z24.s }, p0, [x22, x15, LSL #2]\n"
- "mov z24.d, z16.d\n"
+ "fmax z24.s, p3/M, z24.s, z18.s\n"
+ "st1w { z26.s }, p0, [x23, x24, LSL #2]\n"
+ "addvl x23, x23, #1\n"
"fmax z23.s, p3/M, z23.s, z18.s\n"
+ "st1w { z25.s }, p0, [x22]\n"
+ "fmin z24.s, p3/M, z24.s, z17.s\n"
"fmin z23.s, p3/M, z23.s, z17.s\n"
+ "st1w { z24.s }, p0, [x22, x13, LSL #2]\n"
"st1w { z23.s }, p0, [x22, x24, LSL #2]\n"
- "mov z23.d, z16.d\n"
"addvl x22, x22, #1\n"
"blt 2b\n"
"3:" // Tile loop: Channel tail
- "fmla z31.s, p3/M, z8.s, z9.s\n"
- "prfm pldl1keep, [x10, x25]\n"
+ "movprfx z31, z16\n fmla z31.s, p3/M, z8.s, z9.s\n"
+ "ldr x6, [%x[params_struct], %[offsetof_args_tile_i]]\n"
"mov p0.b, p2.b\n"
- "fmla z30.s, p3/M, z7.s, z9.s\n"
- "prfm pldl1keep, [x12, x7]\n"
- "fmla z29.s, p3/M, z6.s, z9.s\n"
- "prfm pldl1keep, [x16, x7]\n"
- "fmla z28.s, p3/M, z5.s, z9.s\n"
- "prfm pldl1keep, [x16, x26]\n"
- "fmla z27.s, p3/M, z4.s, z9.s\n"
- "prfm pldl1keep, [x12, x26]\n"
- "fmla z26.s, p3/M, z3.s, z9.s\n"
- "prfm pldl1keep, [x13, x5]\n"
- "fmla z25.s, p3/M, z2.s, z9.s\n"
- "prfm pldl1keep, [x13, x25]\n"
- "fmla z24.s, p3/M, z1.s, z9.s\n"
- "prfm pldl1keep, [x11, x5]\n"
- "fmla z23.s, p3/M, z0.s, z9.s\n"
- "prfm pldl1keep, [x11, x17]\n"
- "fmla z31.s, p3/M, z0.s, z10.s\n"
- "ld1w { z10.s }, p2/Z, [x12, x28, LSL #2]\n"
- "fmla z29.s, p3/M, z2.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x12, x8, LSL #2]\n"
- "fmla z25.s, p3/M, z6.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x10, x27, LSL #2]\n"
- "fmla z30.s, p3/M, z4.s, z13.s\n"
- "prfm pldl1keep, [x11, x25]\n"
- "fmla z31.s, p3/M, z5.s, z13.s\n"
- "prfm pldl1keep, [x10, x7]\n"
- "fmla z29.s, p3/M, z3.s, z13.s\n"
- "prfm pldl1keep, [x13, x7]\n"
- "fmla z28.s, p3/M, z2.s, z13.s\n"
- "prfm pldl1keep, [x13, x26]\n"
- "fmla z27.s, p3/M, z1.s, z13.s\n"
- "prfm pldl1keep, [x10, x26]\n"
- "fmla z26.s, p3/M, z0.s, z13.s\n"
- "ld1w { z13.s }, p2/Z, [x16, x8, LSL #2]\n"
- "fmla z23.s, p3/M, z8.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x16, x28, LSL #2]\n"
- "fmla z31.s, p3/M, z7.s, z11.s\n"
- "prfm pldl1keep, [x11, x7]\n"
- "fmla z30.s, p3/M, z6.s, z11.s\n"
- "prfm pldl1keep, [x16, x17]\n"
- "fmla z28.s, p3/M, z4.s, z11.s\n"
- "prfm pldl1keep, [x11, x26]\n"
- "fmla z27.s, p3/M, z3.s, z11.s\n"
- "prfm pldl1keep, [x12, x5]\n"
- "fmla z25.s, p3/M, z1.s, z11.s\n"
- "prfm pldl1keep, [x12, x25]\n"
- "fmla z24.s, p3/M, z0.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x13]\n"
- "fmla z31.s, p3/M, z1.s, z13.s\n"
- "prfm pldl1keep, [x10, x17]\n"
- "fmla z30.s, p3/M, z0.s, z13.s\n"
- "ld1w { z13.s }, p2/Z, [x13, x27, LSL #2]\n"
- "fmla z29.s, p3/M, z1.s, z12.s\n"
- "ldr x3, [%x[params_struct], %[offsetof_args_tile_i]]\n"
- "add x21, x3, #0x1\n"
- "fmla z30.s, p3/M, z2.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x11]\n"
- "fmla z29.s, p3/M, z7.s, z10.s\n"
- "ldr x4, [%x[params_struct], %[offsetof_args_tile_j]]\n"
- "add x4, x4, #0x1\n"
- "fmla z30.s, p3/M, z8.s, z10.s\n"
+ "movprfx z30, z16\n fmla z30.s, p3/M, z7.s, z9.s\n"
+ "ldr x7, [%x[params_struct], %[offsetof_args_tile_j]]\n"
+ "add x21, x6, #0x1\n"
+ "movprfx z29, z16\n fmla z29.s, p3/M, z6.s, z9.s\n"
"ldr x20, [%x[params_struct], %[offsetof_args_n_tile_rows]]\n"
- "fmla z27.s, p3/M, z5.s, z10.s\n"
+ "movprfx z28, z16\n fmla z28.s, p3/M, z5.s, z9.s\n"
"ldr x19, [%x[params_struct], %[offsetof_args_n_tile_cols]]\n"
- "cmp x4, x19\n"
+ "add x7, x7, #0x1\n"
+ "movprfx z27, z16\n fmla z27.s, p3/M, z4.s, z9.s\n"
+ "cmp x7, x19\n"
+ "movprfx z26, z16\n fmla z26.s, p3/M, z3.s, z9.s\n"
+ "movprfx z25, z16\n fmla z25.s, p3/M, z2.s, z9.s\n"
+ "csel x7, x7, XZR, LT\n"
+ "movprfx z24, z16\n fmla z24.s, p3/M, z1.s, z9.s\n"
+ "csel x6, x6, x21, LT\n"
+ "movprfx z23, z16\n fmla z23.s, p3/M, z0.s, z9.s\n"
+ "cmp x6, x20\n"
+ "fmla z31.s, p3/M, z0.s, z10.s\n"
+ "ld1w { z10.s }, p2/Z, [x10, x26, LSL #2]\n"
+ "fmla z29.s, p3/M, z2.s, z11.s\n"
+ "ld1w { z11.s }, p2/Z, [x10, x15, LSL #2]\n"
+ "fmla z25.s, p3/M, z6.s, z12.s\n"
+ "ld1w { z12.s }, p2/Z, [x28, x25, LSL #2]\n"
+ "fmla z30.s, p3/M, z4.s, z13.s\n"
+ "fmla z31.s, p3/M, z5.s, z13.s\n"
+ "fmla z29.s, p3/M, z3.s, z13.s\n"
+ "fmla z28.s, p3/M, z2.s, z13.s\n"
+ "fmla z27.s, p3/M, z1.s, z13.s\n"
+ "fmla z26.s, p3/M, z0.s, z13.s\n"
+ "ld1w { z13.s }, p2/Z, [x14, x15, LSL #2]\n"
+ "fmla z23.s, p3/M, z8.s, z12.s\n"
+ "ld1w { z12.s }, p2/Z, [x14, x26, LSL #2]\n"
+ "fmla z31.s, p3/M, z7.s, z11.s\n"
+ "fmla z30.s, p3/M, z6.s, z11.s\n"
+ "fmla z28.s, p3/M, z4.s, z11.s\n"
+ "fmla z27.s, p3/M, z3.s, z11.s\n"
+ "fmla z25.s, p3/M, z1.s, z11.s\n"
+ "fmla z24.s, p3/M, z0.s, z11.s\n"
+ "ld1w { z11.s }, p2/Z, [x11]\n"
+ "fmla z31.s, p3/M, z1.s, z13.s\n"
+ "fmla z30.s, p3/M, z0.s, z13.s\n"
+ "ld1w { z13.s }, p2/Z, [x11, x25, LSL #2]\n"
+ "fmla z29.s, p3/M, z1.s, z12.s\n"
+ "fmla z27.s, p3/M, z5.s, z10.s\n"
"fmla z26.s, p3/M, z4.s, z10.s\n"
+ "fmla z30.s, p3/M, z2.s, z12.s\n"
+ "ld1w { z12.s }, p2/Z, [x9]\n"
+ "fmla z29.s, p3/M, z7.s, z10.s\n"
"fmla z24.s, p3/M, z2.s, z10.s\n"
- "csel x4, x4, XZR, LT\n"
"fmla z23.s, p3/M, z1.s, z10.s\n"
- "ld1w { z10.s }, p2/Z, [x11, x9, LSL #2]\n"
- "csel x3, x3, x21, LT\n"
+ "fmla z30.s, p3/M, z8.s, z10.s\n"
+ "ld1w { z10.s }, p2/Z, [x9, x27, LSL #2]\n"
"fmla z31.s, p3/M, z3.s, z11.s\n"
- "cmp x3, x20\n"
"fmla z28.s, p3/M, z0.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x11, x27, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x9, x25, LSL #2]\n"
"fmla z29.s, p3/M, z5.s, z13.s\n"
"fmla z26.s, p3/M, z2.s, z13.s\n"
- "ld1w { z13.s }, p2/Z, [x10, x8, LSL #2]\n"
+ "ld1w { z13.s }, p2/Z, [x28, x15, LSL #2]\n"
"fmla z25.s, p3/M, z3.s, z12.s\n"
"fmla z28.s, p3/M, z6.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x13, x8, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x11, x15, LSL #2]\n"
"fmla z27.s, p3/M, z7.s, z10.s\n"
"fmla z26.s, p3/M, z6.s, z10.s\n"
"fmla z25.s, p3/M, z5.s, z10.s\n"
@@ -459,36 +381,36 @@
"fmla z26.s, p3/M, z8.s, z11.s\n"
"fmla z25.s, p3/M, z7.s, z13.s\n"
"fmla z24.s, p3/M, z6.s, z13.s\n"
- "ld1w { z13.s }, p2/Z, [x10, x28, LSL #2]\n"
+ "ld1w { z13.s }, p2/Z, [x28, x26, LSL #2]\n"
"fmla z23.s, p3/M, z5.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x13, x28, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x11, x26, LSL #2]\n"
"fmla z31.s, p3/M, z4.s, z12.s\n"
"fmla z30.s, p3/M, z3.s, z12.s\n"
"fmla z28.s, p3/M, z1.s, z12.s\n"
"fmla z27.s, p3/M, z0.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x11, x8, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x9, x15, LSL #2]\n"
"fmla z29.s, p3/M, z4.s, z11.s\n"
"fmla z30.s, p3/M, z5.s, z11.s\n"
"fmla z26.s, p3/M, z1.s, z11.s\n"
"fmla z27.s, p3/M, z2.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x16, x9, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x14, x27, LSL #2]\n"
"fmla z24.s, p3/M, z8.s, z13.s\n"
"fmla z23.s, p3/M, z7.s, z13.s\n"
- "ld1w { z13.s }, p2/Z, [x11, x28, LSL #2]\n"
+ "ld1w { z13.s }, p2/Z, [x9, x26, LSL #2]\n"
"fmla z28.s, p3/M, z7.s, z12.s\n"
"fmla z27.s, p3/M, z6.s, z12.s\n"
"fmla z25.s, p3/M, z4.s, z12.s\n"
"fmla z24.s, p3/M, z3.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x12]\n"
+ "ld1w { z12.s }, p2/Z, [x10]\n"
"fmla z31.s, p3/M, z2.s, z11.s\n"
"fmla z30.s, p3/M, z1.s, z11.s\n"
"fmla z29.s, p3/M, z0.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x12, x27, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x10, x25, LSL #2]\n"
"fmla z27.s, p3/M, z8.s, z13.s\n"
"fmla z26.s, p3/M, z7.s, z13.s\n"
"fmla z24.s, p3/M, z5.s, z13.s\n"
"fmla z23.s, p3/M, z4.s, z13.s\n"
- "ld1w { z13.s }, p2/Z, [x10, x9, LSL #2]\n"
+ "ld1w { z13.s }, p2/Z, [x28, x27, LSL #2]\n"
"fmla z31.s, p3/M, z6.s, z12.s\n"
"fmla z28.s, p3/M, z3.s, z12.s\n"
"fmla z25.s, p3/M, z0.s, z12.s\n"
@@ -502,13 +424,13 @@
"fmax z30.s, p3/M, z30.s, z18.s\n"
"fmax z29.s, p3/M, z29.s, z18.s\n"
"fmin z31.s, p3/M, z31.s, z17.s\n"
- "st1w { z31.s }, p0, [x14]\n"
+ "st1w { z31.s }, p0, [x12]\n"
"fmin z30.s, p3/M, z30.s, z17.s\n"
"fmin z29.s, p3/M, z29.s, z17.s\n"
- "st1w { z30.s }, p0, [x14, x15, LSL #2]\n"
+ "st1w { z30.s }, p0, [x12, x13, LSL #2]\n"
"fmax z28.s, p3/M, z28.s, z18.s\n"
"fmax z27.s, p3/M, z27.s, z18.s\n"
- "st1w { z29.s }, p0, [x14, x24, LSL #2]\n"
+ "st1w { z29.s }, p0, [x12, x24, LSL #2]\n"
"fmax z26.s, p3/M, z26.s, z18.s\n"
"fmax z25.s, p3/M, z25.s, z18.s\n"
"fmax z24.s, p3/M, z24.s, z18.s\n"
@@ -516,23 +438,23 @@
"st1w { z28.s }, p0, [x23]\n"
"fmin z27.s, p3/M, z27.s, z17.s\n"
"fmin z26.s, p3/M, z26.s, z17.s\n"
- "st1w { z27.s }, p0, [x23, x15, LSL #2]\n"
+ "st1w { z27.s }, p0, [x23, x13, LSL #2]\n"
"fmin z25.s, p3/M, z25.s, z17.s\n"
"fmin z24.s, p3/M, z24.s, z17.s\n"
"st1w { z26.s }, p0, [x23, x24, LSL #2]\n"
"fmax z23.s, p3/M, z23.s, z18.s\n"
"st1w { z25.s }, p0, [x22]\n"
"fmin z23.s, p3/M, z23.s, z17.s\n"
- "st1w { z24.s }, p0, [x22, x15, LSL #2]\n"
+ "st1w { z24.s }, p0, [x22, x13, LSL #2]\n"
"st1w { z23.s }, p0, [x22, x24, LSL #2]\n"
"blt 1b\n"
:
: [n_channels] "r" ((unsigned long) n_channels), [offsetof_args_inptr] "I" (offsetof(Args, inptr)), [offsetof_args_ld_input_col] "I" (offsetof(Args, ld_input_col)), [offsetof_args_ld_input_row] "I" (offsetof(Args, ld_input_row)), [offsetof_args_ld_output_col] "I" (offsetof(Args, ld_output_col)), [offsetof_args_ld_output_row] "I" (offsetof(Args, ld_output_row)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_n_tile_cols] "I" (offsetof(Args, n_tile_cols)), [offsetof_args_n_tile_rows] "I" (offsetof(Args, n_tile_rows)), [offsetof_args_outptr] "I" (offsetof(Args, outptr)), [offsetof_args_params] "I" (offsetof(Args, params)), [offsetof_args_tile_i] "I" (offsetof(Args, tile_i)), [offsetof_args_tile_j] "I" (offsetof(Args, tile_j)), [params_struct] "r" (¶ms_struct)
- : "cc", "memory", "p0", "p1", "p2", "p3", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z16", "z17", "z18", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+ : "cc", "memory", "p0", "p1", "p2", "p3", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z16", "z17", "z18", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
);
}
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(ARM_COMPUTE_ENABLE_SVE)
+#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp
index 3877ae2..72b1826 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp
@@ -25,7 +25,7 @@
#include <cstddef>
#include <cstdint>
-#if defined(ARM_COMPUTE_ENABLE_SVE)
+#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE)
namespace arm_conv {
namespace depthwise {
@@ -87,413 +87,343 @@
activation_min, activation_max);
__asm__ __volatile__(
- "ldr x17, [%x[params_struct], %[offsetof_args_outptrs]]\n"
+ "ldr x16, [%x[params_struct], %[offsetof_args_outptrs]]\n"
"ptrue p3.b\n"
- "ldr x16, [%x[params_struct], %[offsetof_args_params]]\n"
- "add x15, %x[params_struct], %[offsetof_Args_inptrs]\n"
+ "ldr x15, [%x[params_struct], %[offsetof_args_params]]\n"
+ "add x14, %x[params_struct], %[offsetof_Args_inptrs]\n"
"ld1rw { z18.s }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n"
- "cntb x14, ALL, MUL #2\n"
- "ld1rw { z17.s }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n"
"mov x13, #0x0\n"
- "ld1w { z16.s }, p3/Z, [x16]\n"
- "mov z31.d, z16.d\n"
- "ld1w { z0.s }, p3/Z, [x16, #1, MUL VL]\n"
+ "ld1rw { z17.s }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n"
"cntw x12\n"
- "mov z30.d, z16.d\n"
- "ld1w { z1.s }, p3/Z, [x16, #2, MUL VL]\n"
+ "ld1w { z16.s }, p3/Z, [x15]\n"
"sub x11, XZR, x12\n"
- "mov z29.d, z16.d\n"
- "ld1w { z2.s }, p3/Z, [x16, #3, MUL VL]\n"
+ "ld1w { z0.s }, p3/Z, [x15, #1, MUL VL]\n"
"whilelt p2.s, XZR, %x[n_channels]\n"
- "mov z28.d, z16.d\n"
- "ld1w { z3.s }, p3/Z, [x16, #4, MUL VL]\n"
+ "ld1w { z1.s }, p3/Z, [x15, #2, MUL VL]\n"
"cmp x12, %x[n_channels]\n"
- "mov z27.d, z16.d\n"
- "ld1w { z4.s }, p3/Z, [x16, #5, MUL VL]\n"
- "mov z26.d, z16.d\n"
- "ld1w { z5.s }, p3/Z, [x16, #6, MUL VL]\n"
- "mov z25.d, z16.d\n"
- "ld1w { z6.s }, p3/Z, [x16, #7, MUL VL]\n"
- "addvl x16, x16, #16\n"
- "mov z24.d, z16.d\n"
- "ld1w { z7.s }, p3/Z, [x16, #-8, MUL VL]\n"
- "mov z23.d, z16.d\n"
- "ld1w { z8.s }, p3/Z, [x16, #-7, MUL VL]\n"
- "addvl x16, x16, #-6\n"
- "ldp x10, x22, [x15, #0x0]\n"
- "ldp x9, x28, [x15, #0x10]\n"
- "ldr x24, [x15, #0x20]\n"
+ "ld1w { z2.s }, p3/Z, [x15, #3, MUL VL]\n"
+ "ld1w { z3.s }, p3/Z, [x15, #4, MUL VL]\n"
+ "ld1w { z4.s }, p3/Z, [x15, #5, MUL VL]\n"
+ "ld1w { z5.s }, p3/Z, [x15, #6, MUL VL]\n"
+ "ld1w { z6.s }, p3/Z, [x15, #7, MUL VL]\n"
+ "addvl x15, x15, #16\n"
+ "ldp x10, x9, [x14, #0x0]\n"
+ "ld1w { z7.s }, p3/Z, [x15, #-8, MUL VL]\n"
+ "ld1w { z8.s }, p3/Z, [x15, #-7, MUL VL]\n"
+ "addvl x15, x15, #-6\n"
"ld1w { z9.s }, p2/Z, [x10, x13, LSL #2]\n"
- "prfm pldl1keep, [x10, x14]\n"
- "ld1w { z10.s }, p2/Z, [x22, x13, LSL #2]\n"
- "prfm pldl1keep, [x22, x14]\n"
- "ld1w { z11.s }, p2/Z, [x9, x13, LSL #2]\n"
- "prfm pldl1keep, [x9, x14]\n"
- "ld1w { z12.s }, p2/Z, [x28, x13, LSL #2]\n"
- "prfm pldl1keep, [x28, x14]\n"
- "ld1w { z13.s }, p2/Z, [x24, x13, LSL #2]\n"
- "prfm pldl1keep, [x24, x14]\n"
+ "ld1w { z10.s }, p2/Z, [x9, x13, LSL #2]\n"
+ "ldp x28, x27, [x14, #0x10]\n"
+ "ldr x26, [x14, #0x20]\n"
+ "ld1w { z11.s }, p2/Z, [x28, x13, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x27, x13, LSL #2]\n"
+ "ld1w { z13.s }, p2/Z, [x26, x13, LSL #2]\n"
"bge 2f\n"
"1:" // Channel loop
- "fmla z31.s, p3/M, z8.s, z9.s\n"
- "ldr x27, [x15, #0x28]\n"
+ "movprfx z31, z16\n fmla z31.s, p3/M, z8.s, z9.s\n"
+ "ldr x25, [x14, #0x28]\n"
"whilelt p1.s, x12, %x[n_channels]\n"
- "fmla z30.s, p3/M, z7.s, z9.s\n"
- "ldr x23, [x15, #0x30]\n"
+ "movprfx z30, z16\n fmla z30.s, p3/M, z7.s, z9.s\n"
+ "ldr x24, [x14, #0x30]\n"
"incw x11\n"
- "fmla z29.s, p3/M, z6.s, z9.s\n"
- "ldr x26, [x15, #0x38]\n"
+ "movprfx z29, z16\n fmla z29.s, p3/M, z6.s, z9.s\n"
+ "ldr x23, [x14, #0x38]\n"
"mov p0.b, p2.b\n"
- "fmla z28.s, p3/M, z5.s, z9.s\n"
- "prfm pldl1keep, [x27, x14]\n"
- "fmla z27.s, p3/M, z4.s, z9.s\n"
- "prfm pldl1keep, [x23, x14]\n"
- "fmla z26.s, p3/M, z3.s, z9.s\n"
- "prfm pldl1keep, [x26, x14]\n"
- "fmla z25.s, p3/M, z2.s, z9.s\n"
- "ldr x25, [x15, #0x40]\n"
- "fmla z24.s, p3/M, z1.s, z9.s\n"
- "ldr x19, [x15, #0x48]\n"
- "fmla z23.s, p3/M, z0.s, z9.s\n"
- "ldr x24, [x15, #0x50]\n"
+ "movprfx z28, z16\n fmla z28.s, p3/M, z5.s, z9.s\n"
+ "ldr x10, [x14, #0x40]\n"
+ "movprfx z27, z16\n fmla z27.s, p3/M, z4.s, z9.s\n"
+ "ldr x9, [x14, #0x48]\n"
+ "movprfx z26, z16\n fmla z26.s, p3/M, z3.s, z9.s\n"
+ "ldr x28, [x14, #0x50]\n"
+ "movprfx z25, z16\n fmla z25.s, p3/M, z2.s, z9.s\n"
+ "ldr x27, [x14, #0x58]\n"
+ "movprfx z24, z16\n fmla z24.s, p3/M, z1.s, z9.s\n"
+ "ldr x26, [x14, #0x60]\n"
+ "movprfx z23, z16\n fmla z23.s, p3/M, z0.s, z9.s\n"
+ "ldr x22, [x16, #0x0]\n"
"fmla z31.s, p3/M, z0.s, z10.s\n"
- "prfm pldl1keep, [x25, x14]\n"
+ "ld1w { z10.s }, p2/Z, [x9, x13, LSL #2]\n"
"fmla z29.s, p3/M, z2.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x23, x13, LSL #2]\n"
- "fmla z25.s, p3/M, z6.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x27, x13, LSL #2]\n"
- "fmla z30.s, p3/M, z4.s, z13.s\n"
- "ld1w { z10.s }, p2/Z, [x19, x13, LSL #2]\n"
- "fmla z31.s, p3/M, z5.s, z13.s\n"
- "prfm pldl1keep, [x19, x14]\n"
- "fmla z29.s, p3/M, z3.s, z13.s\n"
- "prfm pldl1keep, [x24, x14]\n"
- "fmla z28.s, p3/M, z2.s, z13.s\n"
- "ldr x23, [x15, #0x58]\n"
- "fmla z27.s, p3/M, z1.s, z13.s\n"
- "ldr x22, [x15, #0x60]\n"
- "fmla z26.s, p3/M, z0.s, z13.s\n"
- "ld1w { z13.s }, p2/Z, [x26, x13, LSL #2]\n"
- "fmla z23.s, p3/M, z8.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x25, x13, LSL #2]\n"
- "fmla z31.s, p3/M, z7.s, z11.s\n"
- "prfm pldl1keep, [x23, x14]\n"
- "fmla z30.s, p3/M, z6.s, z11.s\n"
- "prfm pldl1keep, [x22, x14]\n"
- "fmla z28.s, p3/M, z4.s, z11.s\n"
- "ldr x21, [x15, #0x68]\n"
- "fmla z27.s, p3/M, z3.s, z11.s\n"
- "ldr x20, [x15, #0x70]\n"
- "fmla z25.s, p3/M, z1.s, z11.s\n"
- "ldr x19, [x15, #0x78]\n"
- "fmla z24.s, p3/M, z0.s, z11.s\n"
"ld1w { z11.s }, p2/Z, [x24, x13, LSL #2]\n"
- "fmla z31.s, p3/M, z1.s, z13.s\n"
- "prfm pldl1keep, [x21, x14]\n"
- "fmla z30.s, p3/M, z0.s, z13.s\n"
+ "fmla z25.s, p3/M, z6.s, z12.s\n"
+ "ld1w { z12.s }, p2/Z, [x25, x13, LSL #2]\n"
+ "fmla z30.s, p3/M, z4.s, z13.s\n"
+ "ldr x25, [x14, #0x68]\n"
+ "fmla z31.s, p3/M, z5.s, z13.s\n"
+ "ldr x24, [x14, #0x70]\n"
+ "fmla z29.s, p3/M, z3.s, z13.s\n"
+ "ldr x9, [x14, #0x88]\n"
+ "fmla z28.s, p3/M, z2.s, z13.s\n"
+ "ldr x21, [x16, #0x8]\n"
+ "fmla z27.s, p3/M, z1.s, z13.s\n"
+ "ldr x20, [x16, #0x10]\n"
+ "fmla z26.s, p3/M, z0.s, z13.s\n"
"ld1w { z13.s }, p2/Z, [x23, x13, LSL #2]\n"
+ "fmla z23.s, p3/M, z8.s, z12.s\n"
+ "ld1w { z12.s }, p2/Z, [x10, x13, LSL #2]\n"
+ "fmla z31.s, p3/M, z7.s, z11.s\n"
+ "ldr x23, [x14, #0x78]\n"
+ "fmla z30.s, p3/M, z6.s, z11.s\n"
+ "ldr x10, [x14, #0x80]\n"
+ "fmla z28.s, p3/M, z4.s, z11.s\n"
+ "ldr x19, [x16, #0x18]\n"
+ "fmla z27.s, p3/M, z3.s, z11.s\n"
+ "ld1w { z16.s }, p3/Z, [x15]\n"
+ "fmla z25.s, p3/M, z1.s, z11.s\n"
+ "fmla z24.s, p3/M, z0.s, z11.s\n"
+ "ld1w { z11.s }, p2/Z, [x28, x13, LSL #2]\n"
+ "fmla z31.s, p3/M, z1.s, z13.s\n"
+ "ldr x28, [x14, #0x90]\n"
+ "fmla z30.s, p3/M, z0.s, z13.s\n"
+ "ld1w { z13.s }, p2/Z, [x27, x13, LSL #2]\n"
"fmla z29.s, p3/M, z1.s, z12.s\n"
- "prfm pldl1keep, [x20, x14]\n"
+ "ldr x27, [x14, #0x98]\n"
"fmla z27.s, p3/M, z5.s, z10.s\n"
- "prfm pldl1keep, [x19, x14]\n"
"fmla z26.s, p3/M, z4.s, z10.s\n"
- "ldr x10, [x15, #0x80]\n"
"fmla z30.s, p3/M, z2.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x22, x13, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x26, x13, LSL #2]\n"
"fmla z29.s, p3/M, z7.s, z10.s\n"
- "ldr x22, [x15, #0x88]\n"
+ "ldr x26, [x14, #0xa0]\n"
"fmla z24.s, p3/M, z2.s, z10.s\n"
- "prfm pldl1keep, [x10, x14]\n"
"fmla z23.s, p3/M, z1.s, z10.s\n"
- "ldr x9, [x15, #0x90]\n"
"fmla z30.s, p3/M, z8.s, z10.s\n"
- "ld1w { z10.s }, p2/Z, [x21, x13, LSL #2]\n"
+ "ld1w { z10.s }, p2/Z, [x25, x13, LSL #2]\n"
"fmla z31.s, p3/M, z3.s, z11.s\n"
- "prfm pldl1keep, [x22, x14]\n"
+ "ldr x25, [x14, #0xa8]\n"
"fmla z28.s, p3/M, z0.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x20, x13, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x24, x13, LSL #2]\n"
"fmla z29.s, p3/M, z5.s, z13.s\n"
- "prfm pldl1keep, [x9, x14]\n"
+ "ldr x24, [x14, #0xb0]\n"
"fmla z26.s, p3/M, z2.s, z13.s\n"
- "ld1w { z13.s }, p2/Z, [x19, x13, LSL #2]\n"
+ "ld1w { z13.s }, p2/Z, [x23, x13, LSL #2]\n"
"fmla z25.s, p3/M, z3.s, z12.s\n"
- "ldr x28, [x15, #0x98]\n"
+ "ldr x23, [x14, #0xb8]\n"
"fmla z28.s, p3/M, z6.s, z12.s\n"
"ld1w { z12.s }, p2/Z, [x10, x13, LSL #2]\n"
"fmla z27.s, p3/M, z7.s, z10.s\n"
- "ldr x24, [x15, #0xa0]\n"
+ "ldr x10, [x14, #0xc0]\n"
"fmla z26.s, p3/M, z6.s, z10.s\n"
- "prfm pldl1keep, [x28, x14]\n"
"fmla z25.s, p3/M, z5.s, z10.s\n"
- "ldr x27, [x15, #0xa8]\n"
"fmla z28.s, p3/M, z8.s, z10.s\n"
- "prfm pldl1keep, [x24, x14]\n"
"fmla z24.s, p3/M, z4.s, z10.s\n"
- "ldr x23, [x15, #0xb0]\n"
"fmla z23.s, p3/M, z3.s, z10.s\n"
- "prfm pldl1keep, [x27, x14]\n"
"fmla z26.s, p3/M, z8.s, z11.s\n"
- "ldr x26, [x15, #0xb8]\n"
"fmla z25.s, p3/M, z7.s, z13.s\n"
- "prfm pldl1keep, [x23, x14]\n"
"fmla z24.s, p3/M, z6.s, z13.s\n"
- "ld1w { z13.s }, p2/Z, [x9, x13, LSL #2]\n"
+ "ld1w { z13.s }, p2/Z, [x28, x13, LSL #2]\n"
"fmla z23.s, p3/M, z5.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x22, x13, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x9, x13, LSL #2]\n"
"fmla z31.s, p3/M, z4.s, z12.s\n"
- "prfm pldl1keep, [x26, x14]\n"
"fmla z30.s, p3/M, z3.s, z12.s\n"
- "ldr x25, [x15, #0xc0]\n"
"fmla z28.s, p3/M, z1.s, z12.s\n"
- "ldp x10, x22, [x15, #0x0]\n"
"fmla z27.s, p3/M, z0.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x28, x13, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x27, x13, LSL #2]\n"
"fmla z29.s, p3/M, z4.s, z11.s\n"
- "prfm pldl1keep, [x25, x14]\n"
- "addvl x14, x14, #1\n"
"fmla z30.s, p3/M, z5.s, z11.s\n"
- "ld1w { z9.s }, p1/Z, [x10, x12, LSL #2]\n"
"fmla z26.s, p3/M, z1.s, z11.s\n"
- "prfm pldl1keep, [x10, x14]\n"
"fmla z27.s, p3/M, z2.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x24, x13, LSL #2]\n"
- "fmla z24.s, p3/M, z8.s, z13.s\n"
- "ld1w { z10.s }, p1/Z, [x22, x12, LSL #2]\n"
- "fmla z23.s, p3/M, z7.s, z13.s\n"
- "ld1w { z13.s }, p2/Z, [x27, x13, LSL #2]\n"
- "fmla z28.s, p3/M, z7.s, z12.s\n"
- "prfm pldl1keep, [x22, x14]\n"
- "fmla z27.s, p3/M, z6.s, z12.s\n"
- "ldp x9, x28, [x15, #0x10]\n"
- "fmla z25.s, p3/M, z4.s, z12.s\n"
- "ldr x24, [x15, #0x20]\n"
- "fmla z24.s, p3/M, z3.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x23, x13, LSL #2]\n"
- "fmla z31.s, p3/M, z2.s, z11.s\n"
- "prfm pldl1keep, [x9, x14]\n"
- "fmla z30.s, p3/M, z1.s, z11.s\n"
- "prfm pldl1keep, [x28, x14]\n"
- "fmla z29.s, p3/M, z0.s, z11.s\n"
"ld1w { z11.s }, p2/Z, [x26, x13, LSL #2]\n"
- "fmla z27.s, p3/M, z8.s, z13.s\n"
- "prfm pldl1keep, [x24, x14]\n"
- "fmla z26.s, p3/M, z7.s, z13.s\n"
- "ldr x22, [x17, #0x0]\n"
- "fmla z24.s, p3/M, z5.s, z13.s\n"
- "ldr x21, [x17, #0x8]\n"
- "fmla z23.s, p3/M, z4.s, z13.s\n"
+ "fmla z24.s, p3/M, z8.s, z13.s\n"
+ "ldr x26, [x14, #0x20]\n"
+ "fmla z23.s, p3/M, z7.s, z13.s\n"
"ld1w { z13.s }, p2/Z, [x25, x13, LSL #2]\n"
+ "fmla z28.s, p3/M, z7.s, z12.s\n"
+ "fmla z27.s, p3/M, z6.s, z12.s\n"
+ "fmla z25.s, p3/M, z4.s, z12.s\n"
+ "fmla z24.s, p3/M, z3.s, z12.s\n"
+ "ld1w { z12.s }, p2/Z, [x24, x13, LSL #2]\n"
+ "fmla z31.s, p3/M, z2.s, z11.s\n"
+ "fmla z30.s, p3/M, z1.s, z11.s\n"
+ "ld1w { z1.s }, p3/Z, [x15, #2, MUL VL]\n"
+ "fmla z29.s, p3/M, z0.s, z11.s\n"
+ "ld1w { z11.s }, p2/Z, [x23, x13, LSL #2]\n"
+ "fmla z27.s, p3/M, z8.s, z13.s\n"
+ "fmla z26.s, p3/M, z7.s, z13.s\n"
+ "fmla z24.s, p3/M, z5.s, z13.s\n"
+ "fmla z23.s, p3/M, z4.s, z13.s\n"
+ "ld1w { z13.s }, p2/Z, [x10, x13, LSL #2]\n"
"incw x13\n"
"fmla z31.s, p3/M, z6.s, z12.s\n"
- "ldr x20, [x17, #0x10]\n"
+ "ldp x10, x9, [x14, #0x0]\n"
"whilelt p2.s, x13, %x[n_channels]\n"
"fmla z28.s, p3/M, z3.s, z12.s\n"
- "ldr x19, [x17, #0x18]\n"
+ "ldp x28, x27, [x14, #0x10]\n"
"fmla z25.s, p3/M, z0.s, z12.s\n"
- "ld1w { z12.s }, p1/Z, [x28, x12, LSL #2]\n"
+ "ld1w { z0.s }, p3/Z, [x15, #1, MUL VL]\n"
"fmla z29.s, p3/M, z8.s, z11.s\n"
- "ld1w { z16.s }, p3/Z, [x16]\n"
+ "ld1w { z9.s }, p1/Z, [x10, x12, LSL #2]\n"
"fmla z26.s, p3/M, z5.s, z11.s\n"
- "ld1w { z0.s }, p3/Z, [x16, #1, MUL VL]\n"
+ "ld1w { z10.s }, p1/Z, [x9, x12, LSL #2]\n"
"fmla z23.s, p3/M, z2.s, z11.s\n"
- "ld1w { z11.s }, p1/Z, [x9, x12, LSL #2]\n"
+ "ld1w { z11.s }, p1/Z, [x28, x12, LSL #2]\n"
"fmla z25.s, p3/M, z8.s, z13.s\n"
- "ld1w { z1.s }, p3/Z, [x16, #2, MUL VL]\n"
+ "ld1w { z12.s }, p1/Z, [x27, x12, LSL #2]\n"
"fmla z24.s, p3/M, z7.s, z13.s\n"
- "ld1w { z2.s }, p3/Z, [x16, #3, MUL VL]\n"
+ "ld1w { z2.s }, p3/Z, [x15, #3, MUL VL]\n"
"fmax z31.s, p3/M, z31.s, z18.s\n"
- "ld1w { z3.s }, p3/Z, [x16, #4, MUL VL]\n"
+ "ld1w { z3.s }, p3/Z, [x15, #4, MUL VL]\n"
"fmla z23.s, p3/M, z6.s, z13.s\n"
- "ld1w { z13.s }, p1/Z, [x24, x12, LSL #2]\n"
+ "ld1w { z13.s }, p1/Z, [x26, x12, LSL #2]\n"
"incw x12\n"
"fmax z30.s, p3/M, z30.s, z18.s\n"
- "ld1w { z4.s }, p3/Z, [x16, #5, MUL VL]\n"
+ "ld1w { z4.s }, p3/Z, [x15, #5, MUL VL]\n"
"cmp x12, %x[n_channels]\n"
"fmin z31.s, p3/M, z31.s, z17.s\n"
- "ld1w { z5.s }, p3/Z, [x16, #6, MUL VL]\n"
+ "ld1w { z5.s }, p3/Z, [x15, #6, MUL VL]\n"
"fmax z29.s, p3/M, z29.s, z18.s\n"
- "ld1w { z6.s }, p3/Z, [x16, #7, MUL VL]\n"
- "addvl x16, x16, #16\n"
+ "ld1w { z6.s }, p3/Z, [x15, #7, MUL VL]\n"
+ "addvl x15, x15, #16\n"
"fmax z28.s, p3/M, z28.s, z18.s\n"
- "ld1w { z7.s }, p3/Z, [x16, #-8, MUL VL]\n"
+ "ld1w { z7.s }, p3/Z, [x15, #-8, MUL VL]\n"
"fmax z27.s, p3/M, z27.s, z18.s\n"
- "ld1w { z8.s }, p3/Z, [x16, #-7, MUL VL]\n"
- "addvl x16, x16, #-6\n"
+ "ld1w { z8.s }, p3/Z, [x15, #-7, MUL VL]\n"
+ "addvl x15, x15, #-6\n"
"fmin z30.s, p3/M, z30.s, z17.s\n"
"st1w { z31.s }, p0, [x22, x11, LSL #2]\n"
- "mov z31.d, z16.d\n"
- "ldr x22, [x17, #0x20]\n"
"fmin z29.s, p3/M, z29.s, z17.s\n"
- "st1w { z30.s }, p0, [x21, x11, LSL #2]\n"
- "mov z30.d, z16.d\n"
- "fmin z28.s, p3/M, z28.s, z17.s\n"
- "st1w { z29.s }, p0, [x20, x11, LSL #2]\n"
- "mov z29.d, z16.d\n"
- "ldr x21, [x17, #0x28]\n"
- "fmin z27.s, p3/M, z27.s, z17.s\n"
- "ldr x20, [x17, #0x30]\n"
+ "ldr x22, [x16, #0x20]\n"
"fmax z26.s, p3/M, z26.s, z18.s\n"
- "st1w { z28.s }, p0, [x19, x11, LSL #2]\n"
- "mov z28.d, z16.d\n"
- "ldr x19, [x17, #0x38]\n"
- "fmax z25.s, p3/M, z25.s, z18.s\n"
- "st1w { z27.s }, p0, [x22, x11, LSL #2]\n"
- "mov z27.d, z16.d\n"
- "ldr x22, [x17, #0x40]\n"
+ "st1w { z30.s }, p0, [x21, x11, LSL #2]\n"
+ "fmin z28.s, p3/M, z28.s, z17.s\n"
+ "fmin z27.s, p3/M, z27.s, z17.s\n"
+ "st1w { z29.s }, p0, [x20, x11, LSL #2]\n"
"fmin z26.s, p3/M, z26.s, z17.s\n"
- "st1w { z26.s }, p0, [x21, x11, LSL #2]\n"
- "mov z26.d, z16.d\n"
- "fmin z25.s, p3/M, z25.s, z17.s\n"
- "st1w { z25.s }, p0, [x20, x11, LSL #2]\n"
- "mov z25.d, z16.d\n"
+ "ldr x21, [x16, #0x28]\n"
+ "fmax z25.s, p3/M, z25.s, z18.s\n"
+ "ldr x20, [x16, #0x30]\n"
"fmax z24.s, p3/M, z24.s, z18.s\n"
+ "st1w { z28.s }, p0, [x19, x11, LSL #2]\n"
"fmax z23.s, p3/M, z23.s, z18.s\n"
+ "st1w { z27.s }, p0, [x22, x11, LSL #2]\n"
+ "st1w { z26.s }, p0, [x21, x11, LSL #2]\n"
+ "fmin z25.s, p3/M, z25.s, z17.s\n"
+ "ldr x19, [x16, #0x38]\n"
"fmin z24.s, p3/M, z24.s, z17.s\n"
- "st1w { z24.s }, p0, [x19, x11, LSL #2]\n"
- "mov z24.d, z16.d\n"
+ "ldr x22, [x16, #0x40]\n"
"fmin z23.s, p3/M, z23.s, z17.s\n"
+ "st1w { z25.s }, p0, [x20, x11, LSL #2]\n"
+ "st1w { z24.s }, p0, [x19, x11, LSL #2]\n"
"st1w { z23.s }, p0, [x22, x11, LSL #2]\n"
- "mov z23.d, z16.d\n"
"blt 1b\n"
"2:" // Channel tail
- "fmla z31.s, p3/M, z8.s, z9.s\n"
- "ldr x27, [x15, #0x28]\n"
+ "movprfx z31, z16\n fmla z31.s, p3/M, z8.s, z9.s\n"
+ "ldr x25, [x14, #0x28]\n"
"incw x11\n"
- "fmla z30.s, p3/M, z7.s, z9.s\n"
- "ldr x23, [x15, #0x30]\n"
+ "movprfx z30, z16\n fmla z30.s, p3/M, z7.s, z9.s\n"
+ "ldr x24, [x14, #0x30]\n"
"mov p0.b, p2.b\n"
- "fmla z29.s, p3/M, z6.s, z9.s\n"
- "ldr x26, [x15, #0x38]\n"
- "fmla z28.s, p3/M, z5.s, z9.s\n"
- "prfm pldl1keep, [x27, x14]\n"
- "fmla z27.s, p3/M, z4.s, z9.s\n"
- "prfm pldl1keep, [x23, x14]\n"
- "fmla z26.s, p3/M, z3.s, z9.s\n"
- "prfm pldl1keep, [x26, x14]\n"
- "fmla z25.s, p3/M, z2.s, z9.s\n"
- "ldr x25, [x15, #0x40]\n"
- "fmla z24.s, p3/M, z1.s, z9.s\n"
- "ldr x19, [x15, #0x48]\n"
- "fmla z23.s, p3/M, z0.s, z9.s\n"
- "ldr x24, [x15, #0x50]\n"
+ "movprfx z29, z16\n fmla z29.s, p3/M, z6.s, z9.s\n"
+ "ldr x23, [x14, #0x38]\n"
+ "movprfx z28, z16\n fmla z28.s, p3/M, z5.s, z9.s\n"
+ "ldr x10, [x14, #0x40]\n"
+ "movprfx z27, z16\n fmla z27.s, p3/M, z4.s, z9.s\n"
+ "ldr x9, [x14, #0x48]\n"
+ "movprfx z26, z16\n fmla z26.s, p3/M, z3.s, z9.s\n"
+ "ldr x28, [x14, #0x50]\n"
+ "movprfx z25, z16\n fmla z25.s, p3/M, z2.s, z9.s\n"
+ "ldr x27, [x14, #0x58]\n"
+ "movprfx z24, z16\n fmla z24.s, p3/M, z1.s, z9.s\n"
+ "ldr x26, [x14, #0x60]\n"
+ "movprfx z23, z16\n fmla z23.s, p3/M, z0.s, z9.s\n"
+ "ldr x22, [x16, #0x0]\n"
"fmla z31.s, p3/M, z0.s, z10.s\n"
- "prfm pldl1keep, [x25, x14]\n"
+ "ld1w { z10.s }, p2/Z, [x9, x13, LSL #2]\n"
"fmla z29.s, p3/M, z2.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x23, x13, LSL #2]\n"
- "fmla z25.s, p3/M, z6.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x27, x13, LSL #2]\n"
- "fmla z30.s, p3/M, z4.s, z13.s\n"
- "ld1w { z10.s }, p2/Z, [x19, x13, LSL #2]\n"
- "fmla z31.s, p3/M, z5.s, z13.s\n"
- "prfm pldl1keep, [x19, x14]\n"
- "fmla z29.s, p3/M, z3.s, z13.s\n"
- "prfm pldl1keep, [x24, x14]\n"
- "fmla z28.s, p3/M, z2.s, z13.s\n"
- "ldr x23, [x15, #0x58]\n"
- "fmla z27.s, p3/M, z1.s, z13.s\n"
- "ldr x22, [x15, #0x60]\n"
- "fmla z26.s, p3/M, z0.s, z13.s\n"
- "ld1w { z13.s }, p2/Z, [x26, x13, LSL #2]\n"
- "fmla z23.s, p3/M, z8.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x25, x13, LSL #2]\n"
- "fmla z31.s, p3/M, z7.s, z11.s\n"
- "prfm pldl1keep, [x23, x14]\n"
- "fmla z30.s, p3/M, z6.s, z11.s\n"
- "prfm pldl1keep, [x22, x14]\n"
- "fmla z28.s, p3/M, z4.s, z11.s\n"
- "ldr x21, [x15, #0x68]\n"
- "fmla z27.s, p3/M, z3.s, z11.s\n"
- "ldr x20, [x15, #0x70]\n"
- "fmla z25.s, p3/M, z1.s, z11.s\n"
- "ldr x19, [x15, #0x78]\n"
- "fmla z24.s, p3/M, z0.s, z11.s\n"
"ld1w { z11.s }, p2/Z, [x24, x13, LSL #2]\n"
- "fmla z31.s, p3/M, z1.s, z13.s\n"
- "prfm pldl1keep, [x21, x14]\n"
- "fmla z30.s, p3/M, z0.s, z13.s\n"
+ "fmla z25.s, p3/M, z6.s, z12.s\n"
+ "ld1w { z12.s }, p2/Z, [x25, x13, LSL #2]\n"
+ "fmla z30.s, p3/M, z4.s, z13.s\n"
+ "ldr x25, [x14, #0x68]\n"
+ "fmla z31.s, p3/M, z5.s, z13.s\n"
+ "ldr x24, [x14, #0x70]\n"
+ "fmla z29.s, p3/M, z3.s, z13.s\n"
+ "ldr x9, [x14, #0x88]\n"
+ "fmla z28.s, p3/M, z2.s, z13.s\n"
+ "ldr x21, [x16, #0x8]\n"
+ "fmla z27.s, p3/M, z1.s, z13.s\n"
+ "ldr x20, [x16, #0x10]\n"
+ "fmla z26.s, p3/M, z0.s, z13.s\n"
"ld1w { z13.s }, p2/Z, [x23, x13, LSL #2]\n"
+ "fmla z23.s, p3/M, z8.s, z12.s\n"
+ "ld1w { z12.s }, p2/Z, [x10, x13, LSL #2]\n"
+ "fmla z31.s, p3/M, z7.s, z11.s\n"
+ "ldr x23, [x14, #0x78]\n"
+ "fmla z30.s, p3/M, z6.s, z11.s\n"
+ "ldr x10, [x14, #0x80]\n"
+ "fmla z28.s, p3/M, z4.s, z11.s\n"
+ "ldr x19, [x16, #0x18]\n"
+ "fmla z27.s, p3/M, z3.s, z11.s\n"
+ "fmla z25.s, p3/M, z1.s, z11.s\n"
+ "fmla z24.s, p3/M, z0.s, z11.s\n"
+ "ld1w { z11.s }, p2/Z, [x28, x13, LSL #2]\n"
+ "fmla z31.s, p3/M, z1.s, z13.s\n"
+ "ldr x28, [x14, #0x90]\n"
+ "fmla z30.s, p3/M, z0.s, z13.s\n"
+ "ld1w { z13.s }, p2/Z, [x27, x13, LSL #2]\n"
"fmla z29.s, p3/M, z1.s, z12.s\n"
- "prfm pldl1keep, [x20, x14]\n"
+ "ldr x27, [x14, #0x98]\n"
"fmla z27.s, p3/M, z5.s, z10.s\n"
- "prfm pldl1keep, [x19, x14]\n"
"fmla z26.s, p3/M, z4.s, z10.s\n"
- "ldr x10, [x15, #0x80]\n"
"fmla z30.s, p3/M, z2.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x22, x13, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x26, x13, LSL #2]\n"
"fmla z29.s, p3/M, z7.s, z10.s\n"
- "ldr x22, [x15, #0x88]\n"
+ "ldr x26, [x14, #0xa0]\n"
"fmla z24.s, p3/M, z2.s, z10.s\n"
- "prfm pldl1keep, [x10, x14]\n"
"fmla z23.s, p3/M, z1.s, z10.s\n"
- "ldr x9, [x15, #0x90]\n"
"fmla z30.s, p3/M, z8.s, z10.s\n"
- "ld1w { z10.s }, p2/Z, [x21, x13, LSL #2]\n"
+ "ld1w { z10.s }, p2/Z, [x25, x13, LSL #2]\n"
"fmla z31.s, p3/M, z3.s, z11.s\n"
- "prfm pldl1keep, [x22, x14]\n"
+ "ldr x25, [x14, #0xa8]\n"
"fmla z28.s, p3/M, z0.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x20, x13, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x24, x13, LSL #2]\n"
"fmla z29.s, p3/M, z5.s, z13.s\n"
- "prfm pldl1keep, [x9, x14]\n"
+ "ldr x24, [x14, #0xb0]\n"
"fmla z26.s, p3/M, z2.s, z13.s\n"
- "ld1w { z13.s }, p2/Z, [x19, x13, LSL #2]\n"
+ "ld1w { z13.s }, p2/Z, [x23, x13, LSL #2]\n"
"fmla z25.s, p3/M, z3.s, z12.s\n"
- "ldr x28, [x15, #0x98]\n"
+ "ldr x23, [x14, #0xb8]\n"
"fmla z28.s, p3/M, z6.s, z12.s\n"
"ld1w { z12.s }, p2/Z, [x10, x13, LSL #2]\n"
"fmla z27.s, p3/M, z7.s, z10.s\n"
- "ldr x24, [x15, #0xa0]\n"
+ "ldr x10, [x14, #0xc0]\n"
"fmla z26.s, p3/M, z6.s, z10.s\n"
- "prfm pldl1keep, [x28, x14]\n"
"fmla z25.s, p3/M, z5.s, z10.s\n"
- "ldr x27, [x15, #0xa8]\n"
"fmla z28.s, p3/M, z8.s, z10.s\n"
- "prfm pldl1keep, [x24, x14]\n"
"fmla z24.s, p3/M, z4.s, z10.s\n"
- "ldr x23, [x15, #0xb0]\n"
"fmla z23.s, p3/M, z3.s, z10.s\n"
- "prfm pldl1keep, [x27, x14]\n"
"fmla z26.s, p3/M, z8.s, z11.s\n"
- "ldr x26, [x15, #0xb8]\n"
"fmla z25.s, p3/M, z7.s, z13.s\n"
- "prfm pldl1keep, [x23, x14]\n"
"fmla z24.s, p3/M, z6.s, z13.s\n"
- "ld1w { z13.s }, p2/Z, [x9, x13, LSL #2]\n"
+ "ld1w { z13.s }, p2/Z, [x28, x13, LSL #2]\n"
"fmla z23.s, p3/M, z5.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x22, x13, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x9, x13, LSL #2]\n"
"fmla z31.s, p3/M, z4.s, z12.s\n"
- "prfm pldl1keep, [x26, x14]\n"
"fmla z30.s, p3/M, z3.s, z12.s\n"
- "ldr x25, [x15, #0xc0]\n"
"fmla z28.s, p3/M, z1.s, z12.s\n"
- "ldr x22, [x17, #0x0]\n"
"fmla z27.s, p3/M, z0.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x28, x13, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x27, x13, LSL #2]\n"
"fmla z29.s, p3/M, z4.s, z11.s\n"
- "prfm pldl1keep, [x25, x14]\n"
"fmla z30.s, p3/M, z5.s, z11.s\n"
- "ldr x21, [x17, #0x8]\n"
"fmla z26.s, p3/M, z1.s, z11.s\n"
- "ldr x20, [x17, #0x10]\n"
"fmla z27.s, p3/M, z2.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x24, x13, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x26, x13, LSL #2]\n"
"fmla z24.s, p3/M, z8.s, z13.s\n"
- "ldr x19, [x17, #0x18]\n"
"fmla z23.s, p3/M, z7.s, z13.s\n"
- "ld1w { z13.s }, p2/Z, [x27, x13, LSL #2]\n"
+ "ld1w { z13.s }, p2/Z, [x25, x13, LSL #2]\n"
"fmla z28.s, p3/M, z7.s, z12.s\n"
"fmla z27.s, p3/M, z6.s, z12.s\n"
"fmla z25.s, p3/M, z4.s, z12.s\n"
"fmla z24.s, p3/M, z3.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x23, x13, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x24, x13, LSL #2]\n"
"fmla z31.s, p3/M, z2.s, z11.s\n"
"fmla z30.s, p3/M, z1.s, z11.s\n"
"fmla z29.s, p3/M, z0.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x26, x13, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x23, x13, LSL #2]\n"
"fmla z27.s, p3/M, z8.s, z13.s\n"
"fmla z26.s, p3/M, z7.s, z13.s\n"
"fmla z24.s, p3/M, z5.s, z13.s\n"
"fmla z23.s, p3/M, z4.s, z13.s\n"
- "ld1w { z13.s }, p2/Z, [x25, x13, LSL #2]\n"
+ "ld1w { z13.s }, p2/Z, [x10, x13, LSL #2]\n"
"fmla z31.s, p3/M, z6.s, z12.s\n"
"fmla z28.s, p3/M, z3.s, z12.s\n"
"fmla z25.s, p3/M, z0.s, z12.s\n"
@@ -510,21 +440,21 @@
"st1w { z31.s }, p0, [x22, x11, LSL #2]\n"
"fmin z30.s, p3/M, z30.s, z17.s\n"
"fmin z29.s, p3/M, z29.s, z17.s\n"
- "ldr x22, [x17, #0x20]\n"
+ "ldr x22, [x16, #0x20]\n"
"fmax z28.s, p3/M, z28.s, z18.s\n"
"st1w { z30.s }, p0, [x21, x11, LSL #2]\n"
"fmax z27.s, p3/M, z27.s, z18.s\n"
"fmax z26.s, p3/M, z26.s, z18.s\n"
"st1w { z29.s }, p0, [x20, x11, LSL #2]\n"
"fmin z28.s, p3/M, z28.s, z17.s\n"
- "ldr x21, [x17, #0x28]\n"
+ "ldr x21, [x16, #0x28]\n"
"fmax z25.s, p3/M, z25.s, z18.s\n"
- "ldr x20, [x17, #0x30]\n"
+ "ldr x20, [x16, #0x30]\n"
"fmax z24.s, p3/M, z24.s, z18.s\n"
"st1w { z28.s }, p0, [x19, x11, LSL #2]\n"
"fmin z27.s, p3/M, z27.s, z17.s\n"
"fmin z26.s, p3/M, z26.s, z17.s\n"
- "ldr x19, [x17, #0x38]\n"
+ "ldr x19, [x16, #0x38]\n"
"fmin z25.s, p3/M, z25.s, z17.s\n"
"st1w { z27.s }, p0, [x22, x11, LSL #2]\n"
"fmin z24.s, p3/M, z24.s, z17.s\n"
@@ -533,15 +463,15 @@
"st1w { z25.s }, p0, [x20, x11, LSL #2]\n"
"fmin z23.s, p3/M, z23.s, z17.s\n"
"st1w { z24.s }, p0, [x19, x11, LSL #2]\n"
- "ldr x22, [x17, #0x40]\n"
+ "ldr x22, [x16, #0x40]\n"
"st1w { z23.s }, p0, [x22, x11, LSL #2]\n"
:
: [n_channels] "r" ((unsigned long) n_channels), [offsetof_Args_inptrs] "I" (offsetof(Args, inptrs)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_outptrs] "I" (offsetof(Args, outptrs)), [offsetof_args_params] "I" (offsetof(Args, params)), [params_struct] "r" (¶ms_struct)
- : "cc", "memory", "p0", "p1", "p2", "p3", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z16", "z17", "z18", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+ : "cc", "memory", "p0", "p1", "p2", "p3", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z16", "z17", "z18", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
);
}
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(ARM_COMPUTE_ENABLE_SVE)
+#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp
index fc9588c..d7be9b1 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp
@@ -28,7 +28,7 @@
#pragma once
-#if defined(ARM_COMPUTE_ENABLE_SVE)
+#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE)
namespace arm_conv {
namespace depthwise {
@@ -36,15 +36,17 @@
void sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl(const float *const *const, float *const *const, const void *, unsigned int, const float, const float);
void sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float);
-struct sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst
+class sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst : public IDepthwiseDepthfirstStrategy
{
- typedef float bias_type;
- typedef float input_type;
- typedef float weight_type;
- typedef float return_type;
-
+ private:
typedef void (*indirect_kern_type)(const float *const *const, float *const *const, const void *, unsigned int, const float, const float);
+ indirect_kern_type m_indirect_kernel = sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl;
+
typedef void (*direct_kern_type)(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float);
+ direct_kern_type m_direct_kernel = sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl;
+
+ public:
+ typedef float return_type;
constexpr static arm_gemm::VLType vl_type = arm_gemm::VLType::SVE;
@@ -60,13 +62,60 @@
constexpr static unsigned int input_rows = 6;
constexpr static unsigned int input_cols = 6;
- indirect_kern_type indirect_kernel = sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl;
- direct_kern_type direct_kernel = sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl;
-
sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst(const CPUInfo *) {}
+
+ arm_gemm::VLType get_vl_type(void) const override { return vl_type; }
+
+ unsigned int get_kernel_rows(void) const override { return kernel_rows; }
+ unsigned int get_kernel_cols(void) const override { return kernel_cols; }
+
+ unsigned int get_stride_rows(void) const override { return stride_rows; }
+ unsigned int get_stride_cols(void) const override { return stride_cols; }
+
+ unsigned int get_output_rows(void) const override { return output_rows; }
+ unsigned int get_output_cols(void) const override { return output_cols; }
+
+ unsigned int get_input_rows(void) const override { return input_rows; }
+ unsigned int get_input_cols(void) const override { return input_cols; }
+
+ void indirect_kernel(
+ const void *const *const input_ptrs,
+ void *const *const outptrs,
+ const void *params,
+ unsigned int n_channels,
+ const void *activation_min,
+ const void *activation_max
+ ) const override
+ {
+ m_indirect_kernel(
+ reinterpret_cast<const float *const *>(input_ptrs),
+ reinterpret_cast<float *const *>(outptrs),
+ params, n_channels,
+ *static_cast<const float *>(activation_min),
+ *static_cast<const float *>(activation_max)
+ );
+ }
+
+ void direct_kernel(
+ const unsigned int n_tile_rows, const unsigned int n_tile_cols,
+ const void *inptr, int64_t ld_input_row, int64_t ld_input_col,
+ void *outptr, int64_t ld_output_row, int64_t ld_output_col,
+ const void *params, unsigned int n_channels,
+ const void *activation_min, const void *activation_max
+ ) const override
+ {
+ m_direct_kernel(
+ n_tile_rows, n_tile_cols,
+ static_cast<const float *>(inptr), ld_input_row, ld_input_col,
+ static_cast<float *>(outptr), ld_output_row, ld_output_col,
+ params, n_channels,
+ *static_cast<const float *>(activation_min),
+ *static_cast<const float *>(activation_max)
+ );
+ }
};
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(ARM_COMPUTE_ENABLE_SVE)
+#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp
index 7df8e48..84b4b3b 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp
@@ -25,7 +25,7 @@
#include <cstddef>
#include <cstdint>
-#if defined(ARM_COMPUTE_ENABLE_SVE)
+#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE)
namespace arm_conv {
namespace depthwise {
@@ -114,45 +114,29 @@
"ld1rw { z14.s }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n"
"add x14, x15, x22, LSL #2\n"
"ld1w { z13.s }, p3/Z, [x4]\n"
- "mov z31.d, z13.d\n"
- "ld1w { z0.s }, p3/Z, [x4, #1, MUL VL]\n"
"add x13, x14, x22, LSL #2\n"
- "mov z30.d, z13.d\n"
- "ld1w { z1.s }, p3/Z, [x4, #2, MUL VL]\n"
+ "ld1w { z0.s }, p3/Z, [x4, #1, MUL VL]\n"
"add x12, x13, x22, LSL #2\n"
- "mov z29.d, z13.d\n"
- "ld1w { z2.s }, p3/Z, [x4, #3, MUL VL]\n"
+ "ld1w { z1.s }, p3/Z, [x4, #2, MUL VL]\n"
"add x11, x12, x22, LSL #2\n"
- "mov z28.d, z13.d\n"
- "ld1w { z3.s }, p3/Z, [x4, #4, MUL VL]\n"
+ "ld1w { z2.s }, p3/Z, [x4, #3, MUL VL]\n"
"add x10, x7, x7\n"
- "mov z27.d, z13.d\n"
- "ld1w { z4.s }, p3/Z, [x4, #5, MUL VL]\n"
+ "ld1w { z3.s }, p3/Z, [x4, #4, MUL VL]\n"
"add x9, x10, x7\n"
- "mov z26.d, z13.d\n"
- "ld1w { z5.s }, p3/Z, [x4, #6, MUL VL]\n"
+ "ld1w { z4.s }, p3/Z, [x4, #5, MUL VL]\n"
"add x28, x9, x7\n"
- "mov z25.d, z13.d\n"
- "ld1w { z6.s }, p3/Z, [x4, #7, MUL VL]\n"
+ "ld1w { z5.s }, p3/Z, [x4, #6, MUL VL]\n"
"add x27, x28, x7\n"
- "mov z24.d, z13.d\n"
+ "ld1w { z6.s }, p3/Z, [x4, #7, MUL VL]\n"
"mul x19, x2, x20\n" // offset = tile_i * ld_output_row
- "mov z23.d, z13.d\n"
+ "add x26, x17, x17\n"
"madd x19, x3, x17, x19\n" // offset += tile_j * ld_output_col
- "mov z22.d, z13.d\n"
"mul x19, x19, x23\n" // offset *= output_tile_size
- "mov z21.d, z13.d\n"
"add x16, x16, x19, LSL #2\n" // outptrs[0] += offset * sizeof(float)
- "mov z20.d, z13.d\n"
- "add x26, x16, x20, LSL #2\n"
- "mov z19.d, z13.d\n"
- "add x25, x26, x20, LSL #2\n"
- "mov z18.d, z13.d\n"
+ "add x25, x16, x20, LSL #2\n"
"add x24, x25, x20, LSL #2\n"
- "mov z17.d, z13.d\n"
- "add x23, x17, x17\n"
- "mov z16.d, z13.d\n"
- "add x22, x23, x17\n"
+ "add x23, x24, x20, LSL #2\n"
+ "add x22, x26, x17\n"
"whilelt p2.s, XZR, %x[n_channels]\n"
"ld1w { z9.s }, p2/Z, [x14, x10, LSL #2]\n"
"ld1w { z10.s }, p2/Z, [x8]\n"
@@ -165,39 +149,38 @@
"ld1w { z12.s }, p2/Z, [x14, x9, LSL #2]\n"
"bge 3f\n"
"2:" // Tile loop: Channel loop
- "fmla z31.s, p3/M, z8.s, z9.s\n"
- "ld1w { z13.s }, p3/Z, [x4]\n"
+ "movprfx z31, z13\n fmla z31.s, p3/M, z8.s, z9.s\n"
"whilelt p1.s, x6, %x[n_channels]\n"
- "fmla z30.s, p3/M, z7.s, z9.s\n"
+ "movprfx z30, z13\n fmla z30.s, p3/M, z7.s, z9.s\n"
"incw x21\n"
- "fmla z29.s, p3/M, z6.s, z9.s\n"
+ "movprfx z29, z13\n fmla z29.s, p3/M, z6.s, z9.s\n"
"mov p0.b, p2.b\n"
- "fmla z27.s, p3/M, z5.s, z9.s\n"
+ "movprfx z27, z13\n fmla z27.s, p3/M, z5.s, z9.s\n"
"incw x5\n"
- "fmla z26.s, p3/M, z4.s, z9.s\n"
+ "movprfx z26, z13\n fmla z26.s, p3/M, z4.s, z9.s\n"
"incw x6\n"
- "fmla z25.s, p3/M, z3.s, z9.s\n"
- "fmla z23.s, p3/M, z2.s, z9.s\n"
- "fmla z22.s, p3/M, z1.s, z9.s\n"
- "fmla z21.s, p3/M, z0.s, z9.s\n"
+ "movprfx z25, z13\n fmla z25.s, p3/M, z3.s, z9.s\n"
+ "movprfx z23, z13\n fmla z23.s, p3/M, z2.s, z9.s\n"
+ "movprfx z22, z13\n fmla z22.s, p3/M, z1.s, z9.s\n"
+ "movprfx z21, z13\n fmla z21.s, p3/M, z0.s, z9.s\n"
"ld1w { z9.s }, p2/Z, [x13, x10, LSL #2]\n"
"fmla z31.s, p3/M, z0.s, z10.s\n"
"ld1w { z10.s }, p2/Z, [x11]\n"
- "fmla z28.s, p3/M, z2.s, z11.s\n"
+ "movprfx z28, z13\n fmla z28.s, p3/M, z2.s, z11.s\n"
"ld1w { z11.s }, p2/Z, [x11, x27, LSL #2]\n"
"fmla z30.s, p3/M, z8.s, z12.s\n"
"fmla z29.s, p3/M, z7.s, z12.s\n"
"fmla z26.s, p3/M, z5.s, z12.s\n"
"fmla z28.s, p3/M, z6.s, z12.s\n"
"fmla z25.s, p3/M, z4.s, z12.s\n"
- "fmla z24.s, p3/M, z3.s, z12.s\n"
+ "movprfx z24, z13\n fmla z24.s, p3/M, z3.s, z12.s\n"
"fmla z22.s, p3/M, z2.s, z12.s\n"
"fmla z21.s, p3/M, z1.s, z12.s\n"
- "fmla z20.s, p3/M, z0.s, z12.s\n"
+ "movprfx z20, z13\n fmla z20.s, p3/M, z0.s, z12.s\n"
"ld1w { z12.s }, p2/Z, [x8, x7, LSL #2]\n"
- "fmla z19.s, p3/M, z6.s, z10.s\n"
+ "movprfx z19, z13\n fmla z19.s, p3/M, z6.s, z10.s\n"
"ld1w { z10.s }, p2/Z, [x13, x9, LSL #2]\n"
- "fmla z16.s, p3/M, z8.s, z11.s\n"
+ "movprfx z16, z13\n fmla z16.s, p3/M, z8.s, z11.s\n"
"ld1w { z11.s }, p2/Z, [x8, x28, LSL #2]\n"
"fmla z27.s, p3/M, z8.s, z9.s\n"
"fmla z26.s, p3/M, z7.s, z9.s\n"
@@ -206,10 +189,11 @@
"fmla z22.s, p3/M, z4.s, z9.s\n"
"fmla z21.s, p3/M, z3.s, z9.s\n"
"fmla z19.s, p3/M, z2.s, z9.s\n"
- "fmla z18.s, p3/M, z1.s, z9.s\n"
- "fmla z17.s, p3/M, z0.s, z9.s\n"
+ "movprfx z18, z13\n fmla z18.s, p3/M, z1.s, z9.s\n"
+ "movprfx z17, z13\n fmla z17.s, p3/M, z0.s, z9.s\n"
"ld1w { z9.s }, p2/Z, [x15]\n"
"fmla z31.s, p3/M, z1.s, z12.s\n"
+ "ld1w { z13.s }, p3/Z, [x4]\n"
"fmla z30.s, p3/M, z0.s, z12.s\n"
"ld1w { z12.s }, p2/Z, [x15, x27, LSL #2]\n"
"fmla z29.s, p3/M, z2.s, z11.s\n"
@@ -375,109 +359,93 @@
"addvl x4, x4, #-6\n"
"fmin z31.s, p3/M, z31.s, z14.s\n"
"st1w { z31.s }, p0, [x16]\n"
- "mov z31.d, z13.d\n"
"fmin z30.s, p3/M, z30.s, z14.s\n"
- "st1w { z30.s }, p0, [x16, x17, LSL #2]\n"
- "mov z30.d, z13.d\n"
"fmin z29.s, p3/M, z29.s, z14.s\n"
- "st1w { z29.s }, p0, [x16, x23, LSL #2]\n"
- "mov z29.d, z13.d\n"
+ "st1w { z30.s }, p0, [x16, x17, LSL #2]\n"
"fmax z28.s, p3/M, z28.s, z15.s\n"
"fmax z27.s, p3/M, z27.s, z15.s\n"
+ "st1w { z29.s }, p0, [x16, x26, LSL #2]\n"
"fmax z26.s, p3/M, z26.s, z15.s\n"
"fmax z25.s, p3/M, z25.s, z15.s\n"
+ "fmax z24.s, p3/M, z24.s, z15.s\n"
"fmin z28.s, p3/M, z28.s, z14.s\n"
"st1w { z28.s }, p0, [x16, x22, LSL #2]\n"
- "mov z28.d, z13.d\n"
- "addvl x16, x16, #1\n"
"fmin z27.s, p3/M, z27.s, z14.s\n"
- "st1w { z27.s }, p0, [x26]\n"
- "mov z27.d, z13.d\n"
+ "addvl x16, x16, #1\n"
"fmin z26.s, p3/M, z26.s, z14.s\n"
- "st1w { z26.s }, p0, [x26, x17, LSL #2]\n"
- "mov z26.d, z13.d\n"
+ "st1w { z27.s }, p0, [x25]\n"
"fmin z25.s, p3/M, z25.s, z14.s\n"
- "st1w { z25.s }, p0, [x26, x23, LSL #2]\n"
- "mov z25.d, z13.d\n"
- "fmax z24.s, p3/M, z24.s, z15.s\n"
+ "fmin z24.s, p3/M, z24.s, z14.s\n"
+ "st1w { z26.s }, p0, [x25, x17, LSL #2]\n"
"fmax z23.s, p3/M, z23.s, z15.s\n"
+ "st1w { z25.s }, p0, [x25, x26, LSL #2]\n"
"fmax z22.s, p3/M, z22.s, z15.s\n"
"fmax z21.s, p3/M, z21.s, z15.s\n"
- "fmin z24.s, p3/M, z24.s, z14.s\n"
- "st1w { z24.s }, p0, [x26, x22, LSL #2]\n"
- "mov z24.d, z13.d\n"
- "addvl x26, x26, #1\n"
+ "st1w { z24.s }, p0, [x25, x22, LSL #2]\n"
+ "addvl x25, x25, #1\n"
"fmin z23.s, p3/M, z23.s, z14.s\n"
- "st1w { z23.s }, p0, [x25]\n"
- "mov z23.d, z13.d\n"
+ "st1w { z23.s }, p0, [x24]\n"
"fmin z22.s, p3/M, z22.s, z14.s\n"
- "st1w { z22.s }, p0, [x25, x17, LSL #2]\n"
- "mov z22.d, z13.d\n"
"fmin z21.s, p3/M, z21.s, z14.s\n"
- "st1w { z21.s }, p0, [x25, x23, LSL #2]\n"
- "mov z21.d, z13.d\n"
+ "st1w { z22.s }, p0, [x24, x17, LSL #2]\n"
"fmax z20.s, p3/M, z20.s, z15.s\n"
"fmax z19.s, p3/M, z19.s, z15.s\n"
+ "st1w { z21.s }, p0, [x24, x26, LSL #2]\n"
"fmax z18.s, p3/M, z18.s, z15.s\n"
"fmax z17.s, p3/M, z17.s, z15.s\n"
- "fmin z20.s, p3/M, z20.s, z14.s\n"
- "st1w { z20.s }, p0, [x25, x22, LSL #2]\n"
- "mov z20.d, z13.d\n"
- "addvl x25, x25, #1\n"
- "fmin z19.s, p3/M, z19.s, z14.s\n"
- "st1w { z19.s }, p0, [x24]\n"
- "mov z19.d, z13.d\n"
- "fmin z18.s, p3/M, z18.s, z14.s\n"
- "st1w { z18.s }, p0, [x24, x17, LSL #2]\n"
- "mov z18.d, z13.d\n"
- "fmin z17.s, p3/M, z17.s, z14.s\n"
- "st1w { z17.s }, p0, [x24, x23, LSL #2]\n"
- "mov z17.d, z13.d\n"
"fmax z16.s, p3/M, z16.s, z15.s\n"
- "fmin z16.s, p3/M, z16.s, z14.s\n"
- "st1w { z16.s }, p0, [x24, x22, LSL #2]\n"
- "mov z16.d, z13.d\n"
+ "fmin z20.s, p3/M, z20.s, z14.s\n"
+ "st1w { z20.s }, p0, [x24, x22, LSL #2]\n"
+ "fmin z19.s, p3/M, z19.s, z14.s\n"
"addvl x24, x24, #1\n"
+ "fmin z18.s, p3/M, z18.s, z14.s\n"
+ "st1w { z19.s }, p0, [x23]\n"
+ "fmin z17.s, p3/M, z17.s, z14.s\n"
+ "fmin z16.s, p3/M, z16.s, z14.s\n"
+ "st1w { z18.s }, p0, [x23, x17, LSL #2]\n"
+ "st1w { z17.s }, p0, [x23, x26, LSL #2]\n"
+ "st1w { z16.s }, p0, [x23, x22, LSL #2]\n"
+ "addvl x23, x23, #1\n"
"blt 2b\n"
"3:" // Tile loop: Channel tail
- "fmla z31.s, p3/M, z8.s, z9.s\n"
+ "movprfx z31, z13\n fmla z31.s, p3/M, z8.s, z9.s\n"
"ldr x2, [%x[params_struct], %[offsetof_args_tile_i]]\n"
"mov p0.b, p2.b\n"
- "fmla z30.s, p3/M, z7.s, z9.s\n"
+ "movprfx z30, z13\n fmla z30.s, p3/M, z7.s, z9.s\n"
"ldr x3, [%x[params_struct], %[offsetof_args_tile_j]]\n"
"add x21, x2, #0x1\n"
- "fmla z29.s, p3/M, z6.s, z9.s\n"
+ "movprfx z29, z13\n fmla z29.s, p3/M, z6.s, z9.s\n"
"ldr x20, [%x[params_struct], %[offsetof_args_n_tile_rows]]\n"
- "fmla z27.s, p3/M, z5.s, z9.s\n"
+ "movprfx z27, z13\n fmla z27.s, p3/M, z5.s, z9.s\n"
"ldr x19, [%x[params_struct], %[offsetof_args_n_tile_cols]]\n"
"add x3, x3, #0x1\n"
- "fmla z26.s, p3/M, z4.s, z9.s\n"
+ "movprfx z26, z13\n fmla z26.s, p3/M, z4.s, z9.s\n"
"cmp x3, x19\n"
- "fmla z25.s, p3/M, z3.s, z9.s\n"
- "fmla z23.s, p3/M, z2.s, z9.s\n"
+ "movprfx z25, z13\n fmla z25.s, p3/M, z3.s, z9.s\n"
+ "movprfx z23, z13\n fmla z23.s, p3/M, z2.s, z9.s\n"
"csel x3, x3, XZR, LT\n"
- "fmla z22.s, p3/M, z1.s, z9.s\n"
+ "movprfx z22, z13\n fmla z22.s, p3/M, z1.s, z9.s\n"
"csel x2, x2, x21, LT\n"
- "fmla z21.s, p3/M, z0.s, z9.s\n"
+ "movprfx z21, z13\n fmla z21.s, p3/M, z0.s, z9.s\n"
"ld1w { z9.s }, p2/Z, [x13, x10, LSL #2]\n"
"cmp x2, x20\n"
"fmla z31.s, p3/M, z0.s, z10.s\n"
"ld1w { z10.s }, p2/Z, [x11]\n"
- "fmla z28.s, p3/M, z2.s, z11.s\n"
+ "movprfx z28, z13\n fmla z28.s, p3/M, z2.s, z11.s\n"
"ld1w { z11.s }, p2/Z, [x11, x27, LSL #2]\n"
"fmla z30.s, p3/M, z8.s, z12.s\n"
"fmla z29.s, p3/M, z7.s, z12.s\n"
"fmla z26.s, p3/M, z5.s, z12.s\n"
"fmla z28.s, p3/M, z6.s, z12.s\n"
"fmla z25.s, p3/M, z4.s, z12.s\n"
- "fmla z24.s, p3/M, z3.s, z12.s\n"
+ "movprfx z24, z13\n fmla z24.s, p3/M, z3.s, z12.s\n"
"fmla z22.s, p3/M, z2.s, z12.s\n"
"fmla z21.s, p3/M, z1.s, z12.s\n"
- "fmla z20.s, p3/M, z0.s, z12.s\n"
+ "movprfx z20, z13\n fmla z20.s, p3/M, z0.s, z12.s\n"
"ld1w { z12.s }, p2/Z, [x8, x7, LSL #2]\n"
- "fmla z19.s, p3/M, z6.s, z10.s\n"
+ "movprfx z19, z13\n fmla z19.s, p3/M, z6.s, z10.s\n"
"ld1w { z10.s }, p2/Z, [x13, x9, LSL #2]\n"
- "fmla z16.s, p3/M, z8.s, z11.s\n"
+ "movprfx z16, z13\n fmla z16.s, p3/M, z8.s, z11.s\n"
"ld1w { z11.s }, p2/Z, [x8, x28, LSL #2]\n"
"fmla z27.s, p3/M, z8.s, z9.s\n"
"fmla z26.s, p3/M, z7.s, z9.s\n"
@@ -486,8 +454,8 @@
"fmla z22.s, p3/M, z4.s, z9.s\n"
"fmla z21.s, p3/M, z3.s, z9.s\n"
"fmla z19.s, p3/M, z2.s, z9.s\n"
- "fmla z18.s, p3/M, z1.s, z9.s\n"
- "fmla z17.s, p3/M, z0.s, z9.s\n"
+ "movprfx z18, z13\n fmla z18.s, p3/M, z1.s, z9.s\n"
+ "movprfx z17, z13\n fmla z17.s, p3/M, z0.s, z9.s\n"
"ld1w { z9.s }, p2/Z, [x15]\n"
"fmla z31.s, p3/M, z1.s, z12.s\n"
"fmla z30.s, p3/M, z0.s, z12.s\n"
@@ -638,43 +606,43 @@
"st1w { z30.s }, p0, [x16, x17, LSL #2]\n"
"fmin z28.s, p3/M, z28.s, z14.s\n"
"fmax z27.s, p3/M, z27.s, z15.s\n"
- "st1w { z29.s }, p0, [x16, x23, LSL #2]\n"
+ "st1w { z29.s }, p0, [x16, x26, LSL #2]\n"
"fmax z26.s, p3/M, z26.s, z15.s\n"
"st1w { z28.s }, p0, [x16, x22, LSL #2]\n"
"fmin z27.s, p3/M, z27.s, z14.s\n"
"fmax z25.s, p3/M, z25.s, z15.s\n"
- "st1w { z27.s }, p0, [x26]\n"
+ "st1w { z27.s }, p0, [x25]\n"
"fmin z26.s, p3/M, z26.s, z14.s\n"
"fmin z25.s, p3/M, z25.s, z14.s\n"
- "st1w { z26.s }, p0, [x26, x17, LSL #2]\n"
+ "st1w { z26.s }, p0, [x25, x17, LSL #2]\n"
"fmax z24.s, p3/M, z24.s, z15.s\n"
"fmax z23.s, p3/M, z23.s, z15.s\n"
- "st1w { z25.s }, p0, [x26, x23, LSL #2]\n"
+ "st1w { z25.s }, p0, [x25, x26, LSL #2]\n"
"fmax z22.s, p3/M, z22.s, z15.s\n"
"fmax z21.s, p3/M, z21.s, z15.s\n"
"fmax z20.s, p3/M, z20.s, z15.s\n"
"fmin z24.s, p3/M, z24.s, z14.s\n"
- "st1w { z24.s }, p0, [x26, x22, LSL #2]\n"
+ "st1w { z24.s }, p0, [x25, x22, LSL #2]\n"
"fmin z23.s, p3/M, z23.s, z14.s\n"
"fmin z22.s, p3/M, z22.s, z14.s\n"
- "st1w { z23.s }, p0, [x25]\n"
+ "st1w { z23.s }, p0, [x24]\n"
"fmin z21.s, p3/M, z21.s, z14.s\n"
"fmin z20.s, p3/M, z20.s, z14.s\n"
- "st1w { z22.s }, p0, [x25, x17, LSL #2]\n"
+ "st1w { z22.s }, p0, [x24, x17, LSL #2]\n"
"fmax z19.s, p3/M, z19.s, z15.s\n"
- "st1w { z21.s }, p0, [x25, x23, LSL #2]\n"
+ "st1w { z21.s }, p0, [x24, x26, LSL #2]\n"
"fmax z18.s, p3/M, z18.s, z15.s\n"
"fmax z17.s, p3/M, z17.s, z15.s\n"
- "st1w { z20.s }, p0, [x25, x22, LSL #2]\n"
+ "st1w { z20.s }, p0, [x24, x22, LSL #2]\n"
"fmin z19.s, p3/M, z19.s, z14.s\n"
- "st1w { z19.s }, p0, [x24]\n"
+ "st1w { z19.s }, p0, [x23]\n"
"fmin z18.s, p3/M, z18.s, z14.s\n"
"fmin z17.s, p3/M, z17.s, z14.s\n"
- "st1w { z18.s }, p0, [x24, x17, LSL #2]\n"
+ "st1w { z18.s }, p0, [x23, x17, LSL #2]\n"
"fmax z16.s, p3/M, z16.s, z15.s\n"
- "st1w { z17.s }, p0, [x24, x23, LSL #2]\n"
+ "st1w { z17.s }, p0, [x23, x26, LSL #2]\n"
"fmin z16.s, p3/M, z16.s, z14.s\n"
- "st1w { z16.s }, p0, [x24, x22, LSL #2]\n"
+ "st1w { z16.s }, p0, [x23, x22, LSL #2]\n"
"blt 1b\n"
:
: [n_channels] "r" ((unsigned long) n_channels), [offsetof_args_inptr] "I" (offsetof(Args, inptr)), [offsetof_args_ld_input_col] "I" (offsetof(Args, ld_input_col)), [offsetof_args_ld_input_row] "I" (offsetof(Args, ld_input_row)), [offsetof_args_ld_output_col] "I" (offsetof(Args, ld_output_col)), [offsetof_args_ld_output_row] "I" (offsetof(Args, ld_output_row)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_n_tile_cols] "I" (offsetof(Args, n_tile_cols)), [offsetof_args_n_tile_rows] "I" (offsetof(Args, n_tile_rows)), [offsetof_args_outptr] "I" (offsetof(Args, outptr)), [offsetof_args_params] "I" (offsetof(Args, params)), [offsetof_args_tile_i] "I" (offsetof(Args, tile_i)), [offsetof_args_tile_j] "I" (offsetof(Args, tile_j)), [params_struct] "r" (¶ms_struct)
@@ -685,4 +653,4 @@
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(ARM_COMPUTE_ENABLE_SVE)
+#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp
index 22e12a7..158d440 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp
@@ -25,7 +25,7 @@
#include <cstddef>
#include <cstdint>
-#if defined(ARM_COMPUTE_ENABLE_SVE)
+#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE)
namespace arm_conv {
namespace depthwise {
@@ -98,644 +98,538 @@
activation_min, activation_max);
__asm__ __volatile__(
- "ldr x5, [%x[params_struct], %[offsetof_args_outptrs]]\n"
+ "ldr x16, [%x[params_struct], %[offsetof_args_outptrs]]\n"
"ptrue p3.b\n"
- "ldr x6, [%x[params_struct], %[offsetof_args_params]]\n"
- "add x7, %x[params_struct], %[offsetof_Args_inptrs]\n"
+ "ldr x15, [%x[params_struct], %[offsetof_args_params]]\n"
+ "add x14, %x[params_struct], %[offsetof_Args_inptrs]\n"
"ld1rw { z15.s }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n"
- "cntb x8, ALL, MUL #2\n"
+ "mov x13, #0x0\n"
"ld1rw { z14.s }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n"
- "mov x17, #0x0\n"
- "ld1w { z13.s }, p3/Z, [x6]\n"
- "mov z31.d, z13.d\n"
- "ld1w { z0.s }, p3/Z, [x6, #1, MUL VL]\n"
- "cntw x16\n"
- "mov z30.d, z13.d\n"
- "ld1w { z1.s }, p3/Z, [x6, #2, MUL VL]\n"
- "sub x15, XZR, x16\n"
- "mov z29.d, z13.d\n"
- "ld1w { z2.s }, p3/Z, [x6, #3, MUL VL]\n"
+ "cntw x12\n"
+ "ld1w { z13.s }, p3/Z, [x15]\n"
+ "sub x11, XZR, x12\n"
+ "ld1w { z0.s }, p3/Z, [x15, #1, MUL VL]\n"
"whilelt p2.s, XZR, %x[n_channels]\n"
- "mov z28.d, z13.d\n"
- "ld1w { z3.s }, p3/Z, [x6, #4, MUL VL]\n"
- "cmp x16, %x[n_channels]\n"
- "mov z27.d, z13.d\n"
- "ld1w { z4.s }, p3/Z, [x6, #5, MUL VL]\n"
- "mov z26.d, z13.d\n"
- "ld1w { z5.s }, p3/Z, [x6, #6, MUL VL]\n"
- "mov z25.d, z13.d\n"
- "ld1w { z6.s }, p3/Z, [x6, #7, MUL VL]\n"
- "addvl x6, x6, #16\n"
- "mov z24.d, z13.d\n"
- "ld1w { z7.s }, p3/Z, [x6, #-8, MUL VL]\n"
- "mov z23.d, z13.d\n"
- "ld1w { z8.s }, p3/Z, [x6, #-7, MUL VL]\n"
- "addvl x6, x6, #-6\n"
- "mov z22.d, z13.d\n"
- "ldp x14, x13, [x7, #0x0]\n"
- "mov z21.d, z13.d\n"
- "ldp x12, x11, [x7, #0x10]\n"
- "mov z20.d, z13.d\n"
- "ld1w { z9.s }, p2/Z, [x14, x17, LSL #2]\n"
- "mov z19.d, z13.d\n"
- "mov z18.d, z13.d\n"
- "prfm pldl1keep, [x14, x8]\n"
- "mov z17.d, z13.d\n"
- "ld1w { z10.s }, p2/Z, [x13, x17, LSL #2]\n"
- "mov z16.d, z13.d\n"
- "prfm pldl1keep, [x13, x8]\n"
- "ld1w { z11.s }, p2/Z, [x12, x17, LSL #2]\n"
- "prfm pldl1keep, [x12, x8]\n"
- "ld1w { z12.s }, p2/Z, [x11, x17, LSL #2]\n"
- "prfm pldl1keep, [x11, x8]\n"
+ "ld1w { z1.s }, p3/Z, [x15, #2, MUL VL]\n"
+ "cmp x12, %x[n_channels]\n"
+ "ld1w { z2.s }, p3/Z, [x15, #3, MUL VL]\n"
+ "ld1w { z3.s }, p3/Z, [x15, #4, MUL VL]\n"
+ "ld1w { z4.s }, p3/Z, [x15, #5, MUL VL]\n"
+ "ld1w { z5.s }, p3/Z, [x15, #6, MUL VL]\n"
+ "ld1w { z6.s }, p3/Z, [x15, #7, MUL VL]\n"
+ "addvl x15, x15, #16\n"
+ "ldp x10, x9, [x14, #0x0]\n"
+ "ld1w { z7.s }, p3/Z, [x15, #-8, MUL VL]\n"
+ "ld1w { z8.s }, p3/Z, [x15, #-7, MUL VL]\n"
+ "addvl x15, x15, #-6\n"
+ "ld1w { z9.s }, p2/Z, [x10, x13, LSL #2]\n"
+ "ld1w { z10.s }, p2/Z, [x9, x13, LSL #2]\n"
+ "ldp x28, x27, [x14, #0x10]\n"
+ "ld1w { z11.s }, p2/Z, [x28, x13, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x27, x13, LSL #2]\n"
"bge 2f\n"
"1:" // Channel loop
- "fmla z31.s, p3/M, z8.s, z9.s\n"
- "ldr x22, [x7, #0x20]\n"
- "whilelt p1.s, x16, %x[n_channels]\n"
- "fmla z30.s, p3/M, z7.s, z9.s\n"
- "ldr x21, [x7, #0x28]\n"
- "incw x15\n"
- "fmla z29.s, p3/M, z6.s, z9.s\n"
- "ldr x20, [x7, #0x30]\n"
+ "movprfx z31, z13\n fmla z31.s, p3/M, z8.s, z9.s\n"
+ "ldr x26, [x14, #0x20]\n"
+ "whilelt p1.s, x12, %x[n_channels]\n"
+ "movprfx z30, z13\n fmla z30.s, p3/M, z7.s, z9.s\n"
+ "ldr x25, [x14, #0x28]\n"
+ "incw x11\n"
+ "movprfx z29, z13\n fmla z29.s, p3/M, z6.s, z9.s\n"
+ "ldr x24, [x14, #0x30]\n"
"mov p0.b, p2.b\n"
- "fmla z27.s, p3/M, z5.s, z9.s\n"
- "prfm pldl1keep, [x22, x8]\n"
- "fmla z26.s, p3/M, z4.s, z9.s\n"
- "prfm pldl1keep, [x21, x8]\n"
- "fmla z25.s, p3/M, z3.s, z9.s\n"
- "prfm pldl1keep, [x20, x8]\n"
- "fmla z23.s, p3/M, z2.s, z9.s\n"
- "ldr x19, [x7, #0x38]\n"
- "fmla z22.s, p3/M, z1.s, z9.s\n"
- "ldr x10, [x7, #0x40]\n"
- "fmla z21.s, p3/M, z0.s, z9.s\n"
- "ld1w { z9.s }, p2/Z, [x20, x17, LSL #2]\n"
+ "movprfx z27, z13\n fmla z27.s, p3/M, z5.s, z9.s\n"
+ "ldr x23, [x14, #0x38]\n"
+ "movprfx z26, z13\n fmla z26.s, p3/M, z4.s, z9.s\n"
+ "ldr x10, [x14, #0x40]\n"
+ "movprfx z25, z13\n fmla z25.s, p3/M, z3.s, z9.s\n"
+ "ldr x9, [x14, #0x48]\n"
+ "movprfx z23, z13\n fmla z23.s, p3/M, z2.s, z9.s\n"
+ "ldr x28, [x14, #0x50]\n"
+ "movprfx z22, z13\n fmla z22.s, p3/M, z1.s, z9.s\n"
+ "ldr x27, [x14, #0x58]\n"
+ "movprfx z21, z13\n fmla z21.s, p3/M, z0.s, z9.s\n"
+ "ld1w { z9.s }, p2/Z, [x24, x13, LSL #2]\n"
"fmla z31.s, p3/M, z0.s, z10.s\n"
- "ld1w { z10.s }, p2/Z, [x22, x17, LSL #2]\n"
- "fmla z28.s, p3/M, z2.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x21, x17, LSL #2]\n"
+ "ld1w { z10.s }, p2/Z, [x26, x13, LSL #2]\n"
+ "movprfx z28, z13\n fmla z28.s, p3/M, z2.s, z11.s\n"
+ "ld1w { z11.s }, p2/Z, [x25, x13, LSL #2]\n"
"fmla z30.s, p3/M, z8.s, z12.s\n"
- "prfm pldl1keep, [x19, x8]\n"
+ "ldr x26, [x14, #0x60]\n"
"fmla z29.s, p3/M, z7.s, z12.s\n"
- "prfm pldl1keep, [x10, x8]\n"
+ "ldr x25, [x14, #0x68]\n"
"fmla z26.s, p3/M, z5.s, z12.s\n"
- "ldr x9, [x7, #0x48]\n"
+ "ldr x24, [x14, #0x70]\n"
"fmla z28.s, p3/M, z6.s, z12.s\n"
- "ldr x28, [x7, #0x50]\n"
+ "ldr x22, [x16, #0x0]\n"
"fmla z25.s, p3/M, z4.s, z12.s\n"
- "ldr x27, [x7, #0x58]\n"
- "fmla z24.s, p3/M, z3.s, z12.s\n"
- "prfm pldl1keep, [x9, x8]\n"
+ "ldr x21, [x16, #0x8]\n"
+ "movprfx z24, z13\n fmla z24.s, p3/M, z3.s, z12.s\n"
+ "ldr x20, [x16, #0x10]\n"
"fmla z22.s, p3/M, z2.s, z12.s\n"
- "prfm pldl1keep, [x28, x8]\n"
+ "ldr x19, [x16, #0x18]\n"
"fmla z21.s, p3/M, z1.s, z12.s\n"
- "prfm pldl1keep, [x27, x8]\n"
- "fmla z20.s, p3/M, z0.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x19, x17, LSL #2]\n"
- "fmla z19.s, p3/M, z6.s, z10.s\n"
- "ld1w { z10.s }, p2/Z, [x9, x17, LSL #2]\n"
- "fmla z16.s, p3/M, z8.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x10, x17, LSL #2]\n"
+ "movprfx z20, z13\n fmla z20.s, p3/M, z0.s, z12.s\n"
+ "ld1w { z12.s }, p2/Z, [x23, x13, LSL #2]\n"
+ "movprfx z19, z13\n fmla z19.s, p3/M, z6.s, z10.s\n"
+ "ld1w { z10.s }, p2/Z, [x9, x13, LSL #2]\n"
+ "movprfx z16, z13\n fmla z16.s, p3/M, z8.s, z11.s\n"
+ "ld1w { z11.s }, p2/Z, [x10, x13, LSL #2]\n"
"fmla z27.s, p3/M, z8.s, z9.s\n"
- "ldr x26, [x7, #0x60]\n"
+ "ldr x23, [x14, #0x78]\n"
"fmla z26.s, p3/M, z7.s, z9.s\n"
- "ldr x25, [x7, #0x68]\n"
+ "ldr x10, [x14, #0x80]\n"
"fmla z25.s, p3/M, z6.s, z9.s\n"
- "ldr x24, [x7, #0x70]\n"
+ "ldr x9, [x14, #0x88]\n"
"fmla z23.s, p3/M, z5.s, z9.s\n"
- "prfm pldl1keep, [x26, x8]\n"
"fmla z22.s, p3/M, z4.s, z9.s\n"
- "prfm pldl1keep, [x25, x8]\n"
"fmla z21.s, p3/M, z3.s, z9.s\n"
- "prfm pldl1keep, [x24, x8]\n"
"fmla z19.s, p3/M, z2.s, z9.s\n"
- "ldr x23, [x7, #0x78]\n"
- "fmla z18.s, p3/M, z1.s, z9.s\n"
- "ldr x14, [x7, #0x80]\n"
- "fmla z17.s, p3/M, z0.s, z9.s\n"
- "ld1w { z9.s }, p2/Z, [x28, x17, LSL #2]\n"
+ "movprfx z18, z13\n fmla z18.s, p3/M, z1.s, z9.s\n"
+ "movprfx z17, z13\n fmla z17.s, p3/M, z0.s, z9.s\n"
+ "ld1w { z9.s }, p2/Z, [x28, x13, LSL #2]\n"
"fmla z31.s, p3/M, z1.s, z12.s\n"
- "prfm pldl1keep, [x23, x8]\n"
+ "ldr x28, [x14, #0x90]\n"
"fmla z30.s, p3/M, z0.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x27, x17, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x27, x13, LSL #2]\n"
"fmla z29.s, p3/M, z2.s, z11.s\n"
- "prfm pldl1keep, [x14, x8]\n"
+ "ldr x27, [x14, #0x98]\n"
"fmla z28.s, p3/M, z1.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x26, x17, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x26, x13, LSL #2]\n"
"fmla z26.s, p3/M, z8.s, z10.s\n"
- "ldr x13, [x7, #0x88]\n"
+ "ldr x26, [x14, #0xa0]\n"
"fmla z25.s, p3/M, z7.s, z10.s\n"
- "ldr x12, [x7, #0x90]\n"
+ "ld1w { z13.s }, p3/Z, [x15]\n"
"fmla z24.s, p3/M, z6.s, z10.s\n"
- "ldr x11, [x7, #0x98]\n"
"fmla z22.s, p3/M, z5.s, z10.s\n"
- "prfm pldl1keep, [x13, x8]\n"
"fmla z21.s, p3/M, z4.s, z10.s\n"
- "prfm pldl1keep, [x12, x8]\n"
"fmla z20.s, p3/M, z3.s, z10.s\n"
- "prfm pldl1keep, [x11, x8]\n"
"fmla z18.s, p3/M, z2.s, z10.s\n"
- "ldr x22, [x7, #0xa0]\n"
"fmla z17.s, p3/M, z1.s, z10.s\n"
- "ldr x21, [x7, #0xa8]\n"
"fmla z16.s, p3/M, z0.s, z10.s\n"
- "ld1w { z10.s }, p2/Z, [x25, x17, LSL #2]\n"
+ "ld1w { z10.s }, p2/Z, [x25, x13, LSL #2]\n"
"fmla z31.s, p3/M, z3.s, z9.s\n"
- "prfm pldl1keep, [x22, x8]\n"
+ "ldr x25, [x14, #0xa8]\n"
"fmla z27.s, p3/M, z0.s, z9.s\n"
- "prfm pldl1keep, [x21, x8]\n"
"fmla z28.s, p3/M, z5.s, z12.s\n"
- "ldr x20, [x7, #0xb0]\n"
"fmla z24.s, p3/M, z2.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x23, x17, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x23, x13, LSL #2]\n"
"fmla z23.s, p3/M, z6.s, z11.s\n"
- "ldr x19, [x7, #0xb8]\n"
+ "ldr x23, [x14, #0xb8]\n"
"fmla z19.s, p3/M, z3.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x24, x17, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x24, x13, LSL #2]\n"
"fmla z31.s, p3/M, z5.s, z10.s\n"
- "prfm pldl1keep, [x20, x8]\n"
+ "ldr x24, [x14, #0xb0]\n"
"fmla z30.s, p3/M, z4.s, z10.s\n"
- "prfm pldl1keep, [x19, x8]\n"
"fmla z29.s, p3/M, z3.s, z10.s\n"
- "ldr x10, [x7, #0xc0]\n"
"fmla z27.s, p3/M, z2.s, z10.s\n"
- "ldr x9, [x7, #0xc8]\n"
"fmla z26.s, p3/M, z1.s, z10.s\n"
- "ldr x28, [x7, #0xd0]\n"
"fmla z25.s, p3/M, z0.s, z10.s\n"
- "ld1w { z10.s }, p2/Z, [x13, x17, LSL #2]\n"
+ "ld1w { z10.s }, p2/Z, [x9, x13, LSL #2]\n"
"fmla z20.s, p3/M, z8.s, z11.s\n"
- "prfm pldl1keep, [x10, x8]\n"
+ "ldr x9, [x14, #0xc8]\n"
"fmla z16.s, p3/M, z5.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x14, x17, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x10, x13, LSL #2]\n"
"fmla z30.s, p3/M, z5.s, z12.s\n"
- "prfm pldl1keep, [x9, x8]\n"
+ "ldr x10, [x14, #0xc0]\n"
"fmla z29.s, p3/M, z4.s, z12.s\n"
- "prfm pldl1keep, [x28, x8]\n"
"fmla z28.s, p3/M, z3.s, z12.s\n"
- "ldr x27, [x7, #0xd8]\n"
"fmla z26.s, p3/M, z2.s, z12.s\n"
- "ldr x26, [x7, #0xe0]\n"
"fmla z25.s, p3/M, z1.s, z12.s\n"
- "ldr x25, [x7, #0xe8]\n"
"fmla z24.s, p3/M, z0.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x11, x17, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x27, x13, LSL #2]\n"
"fmla z19.s, p3/M, z7.s, z11.s\n"
- "prfm pldl1keep, [x27, x8]\n"
+ "ldr x27, [x14, #0xd8]\n"
"fmla z18.s, p3/M, z6.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x12, x17, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x28, x13, LSL #2]\n"
"fmla z31.s, p3/M, z7.s, z10.s\n"
- "prfm pldl1keep, [x26, x8]\n"
+ "ldr x28, [x14, #0xd0]\n"
"fmla z30.s, p3/M, z6.s, z10.s\n"
- "prfm pldl1keep, [x25, x8]\n"
"fmla z27.s, p3/M, z4.s, z10.s\n"
- "ldr x24, [x7, #0xf0]\n"
"fmla z26.s, p3/M, z3.s, z10.s\n"
- "ldr x23, [x7, #0xf8]\n"
"fmla z23.s, p3/M, z1.s, z10.s\n"
- "ldr x14, [x7, #0x100]\n"
"fmla z22.s, p3/M, z0.s, z10.s\n"
- "ld1w { z10.s }, p2/Z, [x22, x17, LSL #2]\n"
+ "ld1w { z10.s }, p2/Z, [x26, x13, LSL #2]\n"
"fmla z17.s, p3/M, z8.s, z11.s\n"
- "prfm pldl1keep, [x24, x8]\n"
+ "ldr x26, [x14, #0xe0]\n"
"fmla z16.s, p3/M, z7.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x21, x17, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x25, x13, LSL #2]\n"
"fmla z29.s, p3/M, z8.s, z12.s\n"
- "prfm pldl1keep, [x23, x8]\n"
+ "ldr x25, [x14, #0xe8]\n"
"fmla z28.s, p3/M, z7.s, z12.s\n"
- "prfm pldl1keep, [x14, x8]\n"
"fmla z25.s, p3/M, z5.s, z12.s\n"
- "ldr x13, [x7, #0x108]\n"
"fmla z24.s, p3/M, z4.s, z12.s\n"
- "ldr x12, [x7, #0x110]\n"
"fmla z21.s, p3/M, z2.s, z12.s\n"
- "ldr x11, [x7, #0x118]\n"
"fmla z20.s, p3/M, z1.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x20, x17, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x24, x13, LSL #2]\n"
"fmla z31.s, p3/M, z2.s, z10.s\n"
- "prfm pldl1keep, [x13, x8]\n"
+ "ldr x24, [x14, #0xf0]\n"
"fmla z30.s, p3/M, z1.s, z10.s\n"
- "prfm pldl1keep, [x12, x8]\n"
"fmla z29.s, p3/M, z0.s, z10.s\n"
- "ld1w { z10.s }, p2/Z, [x19, x17, LSL #2]\n"
+ "ld1w { z10.s }, p2/Z, [x23, x13, LSL #2]\n"
"fmla z27.s, p3/M, z7.s, z11.s\n"
- "prfm pldl1keep, [x11, x8]\n"
- "addvl x8, x8, #1\n"
+ "ldr x23, [x14, #0xf8]\n"
"fmla z26.s, p3/M, z6.s, z11.s\n"
- "ldr x22, [x5, #0x0]\n"
"fmla z23.s, p3/M, z4.s, z11.s\n"
- "ldr x21, [x5, #0x8]\n"
"fmla z22.s, p3/M, z3.s, z11.s\n"
- "ldr x20, [x5, #0x10]\n"
"fmla z19.s, p3/M, z1.s, z11.s\n"
- "ldr x19, [x5, #0x18]\n"
"fmla z18.s, p3/M, z0.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x10, x17, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x10, x13, LSL #2]\n"
"fmla z30.s, p3/M, z2.s, z12.s\n"
- "ld1w { z13.s }, p3/Z, [x6]\n"
+ "ldr x10, [x14, #0x100]\n"
"fmla z29.s, p3/M, z1.s, z12.s\n"
"fmla z28.s, p3/M, z0.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x9, x17, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x9, x13, LSL #2]\n"
"fmla z31.s, p3/M, z6.s, z10.s\n"
+ "ldr x9, [x14, #0x108]\n"
"fmla z27.s, p3/M, z3.s, z10.s\n"
"fmla z23.s, p3/M, z0.s, z10.s\n"
- "ld1w { z10.s }, p2/Z, [x28, x17, LSL #2]\n"
+ "ld1w { z10.s }, p2/Z, [x28, x13, LSL #2]\n"
"fmla z25.s, p3/M, z8.s, z11.s\n"
+ "ldr x28, [x14, #0x110]\n"
"fmla z24.s, p3/M, z7.s, z11.s\n"
"fmla z21.s, p3/M, z5.s, z11.s\n"
"fmla z20.s, p3/M, z4.s, z11.s\n"
"fmla z17.s, p3/M, z2.s, z11.s\n"
"fmla z16.s, p3/M, z1.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x27, x17, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x27, x13, LSL #2]\n"
"fmla z28.s, p3/M, z8.s, z12.s\n"
+ "ldr x27, [x14, #0x118]\n"
"fmla z24.s, p3/M, z5.s, z12.s\n"
"fmla z20.s, p3/M, z2.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x26, x17, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x26, x13, LSL #2]\n"
"fmla z27.s, p3/M, z6.s, z10.s\n"
"fmla z23.s, p3/M, z3.s, z10.s\n"
"fmla z19.s, p3/M, z0.s, z10.s\n"
- "ld1w { z10.s }, p2/Z, [x25, x17, LSL #2]\n"
+ "ld1w { z10.s }, p2/Z, [x25, x13, LSL #2]\n"
"fmla z22.s, p3/M, z7.s, z11.s\n"
"fmla z21.s, p3/M, z6.s, z11.s\n"
"fmla z23.s, p3/M, z8.s, z11.s\n"
"fmla z19.s, p3/M, z5.s, z11.s\n"
"fmla z18.s, p3/M, z4.s, z11.s\n"
"fmla z17.s, p3/M, z3.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x24, x17, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x24, x13, LSL #2]\n"
"fmla z24.s, p3/M, z8.s, z12.s\n"
"fmla z20.s, p3/M, z5.s, z12.s\n"
"fmla z16.s, p3/M, z2.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x23, x17, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x23, x13, LSL #2]\n"
"fmla z19.s, p3/M, z8.s, z10.s\n"
"fmla z18.s, p3/M, z7.s, z10.s\n"
"fmla z17.s, p3/M, z6.s, z10.s\n"
- "ld1w { z10.s }, p2/Z, [x14, x17, LSL #2]\n"
+ "ld1w { z10.s }, p2/Z, [x10, x13, LSL #2]\n"
"fmla z22.s, p3/M, z8.s, z11.s\n"
"fmla z21.s, p3/M, z7.s, z11.s\n"
"fmla z20.s, p3/M, z6.s, z11.s\n"
"fmla z18.s, p3/M, z5.s, z11.s\n"
"fmla z17.s, p3/M, z4.s, z11.s\n"
"fmla z16.s, p3/M, z3.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x13, x17, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x9, x13, LSL #2]\n"
"fmla z31.s, p3/M, z4.s, z10.s\n"
- "ldp x14, x13, [x7, #0x0]\n"
+ "ldp x10, x9, [x14, #0x0]\n"
"fmla z18.s, p3/M, z8.s, z12.s\n"
- "ld1w { z9.s }, p1/Z, [x14, x16, LSL #2]\n"
+ "ld1w { z9.s }, p1/Z, [x10, x12, LSL #2]\n"
"fmla z17.s, p3/M, z7.s, z12.s\n"
"fmla z16.s, p3/M, z6.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x12, x17, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x28, x13, LSL #2]\n"
"fmla z30.s, p3/M, z3.s, z10.s\n"
- "prfm pldl1keep, [x14, x8]\n"
"fmla z27.s, p3/M, z1.s, z10.s\n"
- "prfm pldl1keep, [x13, x8]\n"
"fmla z26.s, p3/M, z0.s, z10.s\n"
- "ld1w { z10.s }, p2/Z, [x11, x17, LSL #2]\n"
- "incw x17\n"
+ "ld1w { z10.s }, p2/Z, [x27, x13, LSL #2]\n"
+ "incw x13\n"
"fmla z29.s, p3/M, z5.s, z11.s\n"
- "ldp x12, x11, [x7, #0x10]\n"
- "whilelt p2.s, x17, %x[n_channels]\n"
+ "ldp x28, x27, [x14, #0x10]\n"
+ "whilelt p2.s, x13, %x[n_channels]\n"
"fmla z28.s, p3/M, z4.s, z11.s\n"
- "ld1w { z0.s }, p3/Z, [x6, #1, MUL VL]\n"
+ "ld1w { z0.s }, p3/Z, [x15, #1, MUL VL]\n"
"fmla z25.s, p3/M, z2.s, z11.s\n"
- "ld1w { z2.s }, p3/Z, [x6, #3, MUL VL]\n"
+ "ld1w { z2.s }, p3/Z, [x15, #3, MUL VL]\n"
"fmla z24.s, p3/M, z1.s, z11.s\n"
- "ld1w { z11.s }, p1/Z, [x12, x16, LSL #2]\n"
+ "ld1w { z11.s }, p1/Z, [x28, x12, LSL #2]\n"
"fmla z23.s, p3/M, z7.s, z12.s\n"
- "prfm pldl1keep, [x12, x8]\n"
+ "ld1w { z1.s }, p3/Z, [x15, #2, MUL VL]\n"
"fmla z22.s, p3/M, z6.s, z12.s\n"
- "prfm pldl1keep, [x11, x8]\n"
+ "ld1w { z6.s }, p3/Z, [x15, #7, MUL VL]\n"
"fmla z19.s, p3/M, z4.s, z12.s\n"
- "ld1w { z1.s }, p3/Z, [x6, #2, MUL VL]\n"
"fmla z18.s, p3/M, z3.s, z12.s\n"
- "ld1w { z12.s }, p1/Z, [x11, x16, LSL #2]\n"
+ "ld1w { z12.s }, p1/Z, [x27, x12, LSL #2]\n"
"fmla z21.s, p3/M, z8.s, z10.s\n"
- "ld1w { z3.s }, p3/Z, [x6, #4, MUL VL]\n"
+ "ld1w { z3.s }, p3/Z, [x15, #4, MUL VL]\n"
"fmla z20.s, p3/M, z7.s, z10.s\n"
- "ld1w { z6.s }, p3/Z, [x6, #7, MUL VL]\n"
"fmla z17.s, p3/M, z5.s, z10.s\n"
- "ld1w { z5.s }, p3/Z, [x6, #6, MUL VL]\n"
+ "ld1w { z5.s }, p3/Z, [x15, #6, MUL VL]\n"
"fmla z16.s, p3/M, z4.s, z10.s\n"
- "ld1w { z10.s }, p1/Z, [x13, x16, LSL #2]\n"
- "incw x16\n"
+ "ld1w { z10.s }, p1/Z, [x9, x12, LSL #2]\n"
+ "incw x12\n"
"fmax z31.s, p3/M, z31.s, z15.s\n"
- "ld1w { z4.s }, p3/Z, [x6, #5, MUL VL]\n"
- "addvl x6, x6, #16\n"
+ "ld1w { z4.s }, p3/Z, [x15, #5, MUL VL]\n"
+ "addvl x15, x15, #16\n"
"fmax z30.s, p3/M, z30.s, z15.s\n"
- "ld1w { z7.s }, p3/Z, [x6, #-8, MUL VL]\n"
- "cmp x16, %x[n_channels]\n"
+ "ld1w { z7.s }, p3/Z, [x15, #-8, MUL VL]\n"
+ "cmp x12, %x[n_channels]\n"
"fmax z29.s, p3/M, z29.s, z15.s\n"
- "ld1w { z8.s }, p3/Z, [x6, #-7, MUL VL]\n"
- "addvl x6, x6, #-6\n"
+ "ld1w { z8.s }, p3/Z, [x15, #-7, MUL VL]\n"
+ "addvl x15, x15, #-6\n"
"fmax z28.s, p3/M, z28.s, z15.s\n"
"fmax z27.s, p3/M, z27.s, z15.s\n"
"fmin z31.s, p3/M, z31.s, z14.s\n"
- "st1w { z31.s }, p0, [x22, x15, LSL #2]\n"
- "mov z31.d, z13.d\n"
+ "st1w { z31.s }, p0, [x22, x11, LSL #2]\n"
"fmin z30.s, p3/M, z30.s, z14.s\n"
- "ldr x22, [x5, #0x20]\n"
"fmin z29.s, p3/M, z29.s, z14.s\n"
- "st1w { z30.s }, p0, [x21, x15, LSL #2]\n"
- "mov z30.d, z13.d\n"
+ "ldr x22, [x16, #0x20]\n"
"fmin z28.s, p3/M, z28.s, z14.s\n"
- "st1w { z29.s }, p0, [x20, x15, LSL #2]\n"
- "mov z29.d, z13.d\n"
- "ldr x21, [x5, #0x28]\n"
+ "st1w { z30.s }, p0, [x21, x11, LSL #2]\n"
"fmin z27.s, p3/M, z27.s, z14.s\n"
- "ldr x20, [x5, #0x30]\n"
"fmax z26.s, p3/M, z26.s, z15.s\n"
- "st1w { z28.s }, p0, [x19, x15, LSL #2]\n"
- "mov z28.d, z13.d\n"
- "ldr x19, [x5, #0x38]\n"
+ "st1w { z29.s }, p0, [x20, x11, LSL #2]\n"
"fmax z25.s, p3/M, z25.s, z15.s\n"
- "st1w { z27.s }, p0, [x22, x15, LSL #2]\n"
- "mov z27.d, z13.d\n"
- "ldr x22, [x5, #0x40]\n"
- "fmin z26.s, p3/M, z26.s, z14.s\n"
- "st1w { z26.s }, p0, [x21, x15, LSL #2]\n"
- "mov z26.d, z13.d\n"
- "fmin z25.s, p3/M, z25.s, z14.s\n"
- "ldr x21, [x5, #0x48]\n"
+ "st1w { z28.s }, p0, [x19, x11, LSL #2]\n"
"fmax z24.s, p3/M, z24.s, z15.s\n"
- "st1w { z25.s }, p0, [x20, x15, LSL #2]\n"
- "mov z25.d, z13.d\n"
+ "ldr x21, [x16, #0x28]\n"
"fmax z23.s, p3/M, z23.s, z15.s\n"
- "ldr x20, [x5, #0x50]\n"
+ "st1w { z27.s }, p0, [x22, x11, LSL #2]\n"
+ "fmin z26.s, p3/M, z26.s, z14.s\n"
+ "ldr x20, [x16, #0x30]\n"
+ "fmin z25.s, p3/M, z25.s, z14.s\n"
+ "ldr x19, [x16, #0x38]\n"
"fmin z24.s, p3/M, z24.s, z14.s\n"
- "st1w { z24.s }, p0, [x19, x15, LSL #2]\n"
- "mov z24.d, z13.d\n"
+ "ldr x22, [x16, #0x40]\n"
"fmin z23.s, p3/M, z23.s, z14.s\n"
- "ldr x19, [x5, #0x58]\n"
+ "st1w { z26.s }, p0, [x21, x11, LSL #2]\n"
"fmax z22.s, p3/M, z22.s, z15.s\n"
- "st1w { z23.s }, p0, [x22, x15, LSL #2]\n"
- "mov z23.d, z13.d\n"
+ "st1w { z25.s }, p0, [x20, x11, LSL #2]\n"
"fmax z21.s, p3/M, z21.s, z15.s\n"
- "ldr x22, [x5, #0x60]\n"
- "fmin z22.s, p3/M, z22.s, z14.s\n"
- "st1w { z22.s }, p0, [x21, x15, LSL #2]\n"
- "mov z22.d, z13.d\n"
- "fmin z21.s, p3/M, z21.s, z14.s\n"
- "ldr x21, [x5, #0x68]\n"
+ "st1w { z24.s }, p0, [x19, x11, LSL #2]\n"
"fmax z20.s, p3/M, z20.s, z15.s\n"
- "st1w { z21.s }, p0, [x20, x15, LSL #2]\n"
- "mov z21.d, z13.d\n"
+ "st1w { z23.s }, p0, [x22, x11, LSL #2]\n"
"fmax z19.s, p3/M, z19.s, z15.s\n"
- "ldr x20, [x5, #0x70]\n"
+ "ldr x21, [x16, #0x48]\n"
+ "fmin z22.s, p3/M, z22.s, z14.s\n"
+ "ldr x20, [x16, #0x50]\n"
+ "fmin z21.s, p3/M, z21.s, z14.s\n"
+ "ldr x19, [x16, #0x58]\n"
"fmin z20.s, p3/M, z20.s, z14.s\n"
- "st1w { z20.s }, p0, [x19, x15, LSL #2]\n"
- "mov z20.d, z13.d\n"
+ "ldr x22, [x16, #0x60]\n"
"fmin z19.s, p3/M, z19.s, z14.s\n"
- "ldr x19, [x5, #0x78]\n"
+ "st1w { z22.s }, p0, [x21, x11, LSL #2]\n"
"fmax z18.s, p3/M, z18.s, z15.s\n"
- "st1w { z19.s }, p0, [x22, x15, LSL #2]\n"
- "mov z19.d, z13.d\n"
+ "st1w { z21.s }, p0, [x20, x11, LSL #2]\n"
"fmax z17.s, p3/M, z17.s, z15.s\n"
- "fmin z18.s, p3/M, z18.s, z14.s\n"
- "st1w { z18.s }, p0, [x21, x15, LSL #2]\n"
- "mov z18.d, z13.d\n"
- "fmin z17.s, p3/M, z17.s, z14.s\n"
- "st1w { z17.s }, p0, [x20, x15, LSL #2]\n"
- "mov z17.d, z13.d\n"
+ "st1w { z20.s }, p0, [x19, x11, LSL #2]\n"
"fmax z16.s, p3/M, z16.s, z15.s\n"
+ "st1w { z19.s }, p0, [x22, x11, LSL #2]\n"
+ "ldr x21, [x16, #0x68]\n"
+ "fmin z18.s, p3/M, z18.s, z14.s\n"
+ "ldr x20, [x16, #0x70]\n"
+ "fmin z17.s, p3/M, z17.s, z14.s\n"
+ "ldr x19, [x16, #0x78]\n"
"fmin z16.s, p3/M, z16.s, z14.s\n"
- "st1w { z16.s }, p0, [x19, x15, LSL #2]\n"
- "mov z16.d, z13.d\n"
+ "st1w { z18.s }, p0, [x21, x11, LSL #2]\n"
+ "st1w { z17.s }, p0, [x20, x11, LSL #2]\n"
+ "st1w { z16.s }, p0, [x19, x11, LSL #2]\n"
"blt 1b\n"
"2:" // Channel tail
- "fmla z31.s, p3/M, z8.s, z9.s\n"
- "ldr x22, [x7, #0x20]\n"
- "incw x15\n"
- "fmla z30.s, p3/M, z7.s, z9.s\n"
- "ldr x21, [x7, #0x28]\n"
+ "movprfx z31, z13\n fmla z31.s, p3/M, z8.s, z9.s\n"
+ "ldr x26, [x14, #0x20]\n"
+ "incw x11\n"
+ "movprfx z30, z13\n fmla z30.s, p3/M, z7.s, z9.s\n"
+ "ldr x25, [x14, #0x28]\n"
"mov p0.b, p2.b\n"
- "fmla z29.s, p3/M, z6.s, z9.s\n"
- "ldr x20, [x7, #0x30]\n"
- "fmla z27.s, p3/M, z5.s, z9.s\n"
- "prfm pldl1keep, [x22, x8]\n"
- "fmla z26.s, p3/M, z4.s, z9.s\n"
- "prfm pldl1keep, [x21, x8]\n"
- "fmla z25.s, p3/M, z3.s, z9.s\n"
- "prfm pldl1keep, [x20, x8]\n"
- "fmla z23.s, p3/M, z2.s, z9.s\n"
- "ldr x19, [x7, #0x38]\n"
- "fmla z22.s, p3/M, z1.s, z9.s\n"
- "ldr x10, [x7, #0x40]\n"
- "fmla z21.s, p3/M, z0.s, z9.s\n"
- "ld1w { z9.s }, p2/Z, [x20, x17, LSL #2]\n"
+ "movprfx z29, z13\n fmla z29.s, p3/M, z6.s, z9.s\n"
+ "ldr x24, [x14, #0x30]\n"
+ "movprfx z27, z13\n fmla z27.s, p3/M, z5.s, z9.s\n"
+ "ldr x23, [x14, #0x38]\n"
+ "movprfx z26, z13\n fmla z26.s, p3/M, z4.s, z9.s\n"
+ "ldr x10, [x14, #0x40]\n"
+ "movprfx z25, z13\n fmla z25.s, p3/M, z3.s, z9.s\n"
+ "ldr x9, [x14, #0x48]\n"
+ "movprfx z23, z13\n fmla z23.s, p3/M, z2.s, z9.s\n"
+ "ldr x28, [x14, #0x50]\n"
+ "movprfx z22, z13\n fmla z22.s, p3/M, z1.s, z9.s\n"
+ "ldr x27, [x14, #0x58]\n"
+ "movprfx z21, z13\n fmla z21.s, p3/M, z0.s, z9.s\n"
+ "ld1w { z9.s }, p2/Z, [x24, x13, LSL #2]\n"
"fmla z31.s, p3/M, z0.s, z10.s\n"
- "ld1w { z10.s }, p2/Z, [x22, x17, LSL #2]\n"
- "fmla z28.s, p3/M, z2.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x21, x17, LSL #2]\n"
+ "ld1w { z10.s }, p2/Z, [x26, x13, LSL #2]\n"
+ "movprfx z28, z13\n fmla z28.s, p3/M, z2.s, z11.s\n"
+ "ld1w { z11.s }, p2/Z, [x25, x13, LSL #2]\n"
"fmla z30.s, p3/M, z8.s, z12.s\n"
- "prfm pldl1keep, [x19, x8]\n"
+ "ldr x26, [x14, #0x60]\n"
"fmla z29.s, p3/M, z7.s, z12.s\n"
- "prfm pldl1keep, [x10, x8]\n"
+ "ldr x25, [x14, #0x68]\n"
"fmla z26.s, p3/M, z5.s, z12.s\n"
- "ldr x9, [x7, #0x48]\n"
+ "ldr x24, [x14, #0x70]\n"
"fmla z28.s, p3/M, z6.s, z12.s\n"
- "ldr x28, [x7, #0x50]\n"
+ "ldr x22, [x16, #0x0]\n"
"fmla z25.s, p3/M, z4.s, z12.s\n"
- "ldr x27, [x7, #0x58]\n"
- "fmla z24.s, p3/M, z3.s, z12.s\n"
- "prfm pldl1keep, [x9, x8]\n"
+ "ldr x21, [x16, #0x8]\n"
+ "movprfx z24, z13\n fmla z24.s, p3/M, z3.s, z12.s\n"
+ "ldr x20, [x16, #0x10]\n"
"fmla z22.s, p3/M, z2.s, z12.s\n"
- "prfm pldl1keep, [x28, x8]\n"
+ "ldr x19, [x16, #0x18]\n"
"fmla z21.s, p3/M, z1.s, z12.s\n"
- "prfm pldl1keep, [x27, x8]\n"
- "fmla z20.s, p3/M, z0.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x19, x17, LSL #2]\n"
- "fmla z19.s, p3/M, z6.s, z10.s\n"
- "ld1w { z10.s }, p2/Z, [x9, x17, LSL #2]\n"
- "fmla z16.s, p3/M, z8.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x10, x17, LSL #2]\n"
+ "movprfx z20, z13\n fmla z20.s, p3/M, z0.s, z12.s\n"
+ "ld1w { z12.s }, p2/Z, [x23, x13, LSL #2]\n"
+ "movprfx z19, z13\n fmla z19.s, p3/M, z6.s, z10.s\n"
+ "ld1w { z10.s }, p2/Z, [x9, x13, LSL #2]\n"
+ "movprfx z16, z13\n fmla z16.s, p3/M, z8.s, z11.s\n"
+ "ld1w { z11.s }, p2/Z, [x10, x13, LSL #2]\n"
"fmla z27.s, p3/M, z8.s, z9.s\n"
- "ldr x26, [x7, #0x60]\n"
+ "ldr x23, [x14, #0x78]\n"
"fmla z26.s, p3/M, z7.s, z9.s\n"
- "ldr x25, [x7, #0x68]\n"
+ "ldr x10, [x14, #0x80]\n"
"fmla z25.s, p3/M, z6.s, z9.s\n"
- "ldr x24, [x7, #0x70]\n"
+ "ldr x9, [x14, #0x88]\n"
"fmla z23.s, p3/M, z5.s, z9.s\n"
- "prfm pldl1keep, [x26, x8]\n"
"fmla z22.s, p3/M, z4.s, z9.s\n"
- "prfm pldl1keep, [x25, x8]\n"
"fmla z21.s, p3/M, z3.s, z9.s\n"
- "prfm pldl1keep, [x24, x8]\n"
"fmla z19.s, p3/M, z2.s, z9.s\n"
- "ldr x23, [x7, #0x78]\n"
- "fmla z18.s, p3/M, z1.s, z9.s\n"
- "ldr x14, [x7, #0x80]\n"
- "fmla z17.s, p3/M, z0.s, z9.s\n"
- "ld1w { z9.s }, p2/Z, [x28, x17, LSL #2]\n"
+ "movprfx z18, z13\n fmla z18.s, p3/M, z1.s, z9.s\n"
+ "movprfx z17, z13\n fmla z17.s, p3/M, z0.s, z9.s\n"
+ "ld1w { z9.s }, p2/Z, [x28, x13, LSL #2]\n"
"fmla z31.s, p3/M, z1.s, z12.s\n"
- "prfm pldl1keep, [x23, x8]\n"
+ "ldr x28, [x14, #0x90]\n"
"fmla z30.s, p3/M, z0.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x27, x17, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x27, x13, LSL #2]\n"
"fmla z29.s, p3/M, z2.s, z11.s\n"
- "prfm pldl1keep, [x14, x8]\n"
+ "ldr x27, [x14, #0x98]\n"
"fmla z28.s, p3/M, z1.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x26, x17, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x26, x13, LSL #2]\n"
"fmla z26.s, p3/M, z8.s, z10.s\n"
- "ldr x13, [x7, #0x88]\n"
+ "ldr x26, [x14, #0xa0]\n"
"fmla z25.s, p3/M, z7.s, z10.s\n"
- "ldr x12, [x7, #0x90]\n"
"fmla z24.s, p3/M, z6.s, z10.s\n"
- "ldr x11, [x7, #0x98]\n"
"fmla z22.s, p3/M, z5.s, z10.s\n"
- "prfm pldl1keep, [x13, x8]\n"
"fmla z21.s, p3/M, z4.s, z10.s\n"
- "prfm pldl1keep, [x12, x8]\n"
"fmla z20.s, p3/M, z3.s, z10.s\n"
- "prfm pldl1keep, [x11, x8]\n"
"fmla z18.s, p3/M, z2.s, z10.s\n"
- "ldr x22, [x7, #0xa0]\n"
"fmla z17.s, p3/M, z1.s, z10.s\n"
- "ldr x21, [x7, #0xa8]\n"
"fmla z16.s, p3/M, z0.s, z10.s\n"
- "ld1w { z10.s }, p2/Z, [x25, x17, LSL #2]\n"
+ "ld1w { z10.s }, p2/Z, [x25, x13, LSL #2]\n"
"fmla z31.s, p3/M, z3.s, z9.s\n"
- "prfm pldl1keep, [x22, x8]\n"
+ "ldr x25, [x14, #0xa8]\n"
"fmla z27.s, p3/M, z0.s, z9.s\n"
- "prfm pldl1keep, [x21, x8]\n"
"fmla z28.s, p3/M, z5.s, z12.s\n"
- "ldr x20, [x7, #0xb0]\n"
"fmla z24.s, p3/M, z2.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x23, x17, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x23, x13, LSL #2]\n"
"fmla z23.s, p3/M, z6.s, z11.s\n"
- "ldr x19, [x7, #0xb8]\n"
+ "ldr x23, [x14, #0xb8]\n"
"fmla z19.s, p3/M, z3.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x24, x17, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x24, x13, LSL #2]\n"
"fmla z31.s, p3/M, z5.s, z10.s\n"
- "prfm pldl1keep, [x20, x8]\n"
+ "ldr x24, [x14, #0xb0]\n"
"fmla z30.s, p3/M, z4.s, z10.s\n"
- "prfm pldl1keep, [x19, x8]\n"
"fmla z29.s, p3/M, z3.s, z10.s\n"
- "ldr x10, [x7, #0xc0]\n"
"fmla z27.s, p3/M, z2.s, z10.s\n"
- "ldr x9, [x7, #0xc8]\n"
"fmla z26.s, p3/M, z1.s, z10.s\n"
- "ldr x28, [x7, #0xd0]\n"
"fmla z25.s, p3/M, z0.s, z10.s\n"
- "ld1w { z10.s }, p2/Z, [x13, x17, LSL #2]\n"
+ "ld1w { z10.s }, p2/Z, [x9, x13, LSL #2]\n"
"fmla z20.s, p3/M, z8.s, z11.s\n"
- "prfm pldl1keep, [x10, x8]\n"
+ "ldr x9, [x14, #0xc8]\n"
"fmla z16.s, p3/M, z5.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x14, x17, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x10, x13, LSL #2]\n"
"fmla z30.s, p3/M, z5.s, z12.s\n"
- "prfm pldl1keep, [x9, x8]\n"
+ "ldr x10, [x14, #0xc0]\n"
"fmla z29.s, p3/M, z4.s, z12.s\n"
- "prfm pldl1keep, [x28, x8]\n"
"fmla z28.s, p3/M, z3.s, z12.s\n"
- "ldr x27, [x7, #0xd8]\n"
"fmla z26.s, p3/M, z2.s, z12.s\n"
- "ldr x26, [x7, #0xe0]\n"
"fmla z25.s, p3/M, z1.s, z12.s\n"
- "ldr x25, [x7, #0xe8]\n"
"fmla z24.s, p3/M, z0.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x11, x17, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x27, x13, LSL #2]\n"
"fmla z19.s, p3/M, z7.s, z11.s\n"
- "prfm pldl1keep, [x27, x8]\n"
+ "ldr x27, [x14, #0xd8]\n"
"fmla z18.s, p3/M, z6.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x12, x17, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x28, x13, LSL #2]\n"
"fmla z31.s, p3/M, z7.s, z10.s\n"
- "prfm pldl1keep, [x26, x8]\n"
+ "ldr x28, [x14, #0xd0]\n"
"fmla z30.s, p3/M, z6.s, z10.s\n"
- "prfm pldl1keep, [x25, x8]\n"
"fmla z27.s, p3/M, z4.s, z10.s\n"
- "ldr x24, [x7, #0xf0]\n"
"fmla z26.s, p3/M, z3.s, z10.s\n"
- "ldr x23, [x7, #0xf8]\n"
"fmla z23.s, p3/M, z1.s, z10.s\n"
- "ldr x14, [x7, #0x100]\n"
"fmla z22.s, p3/M, z0.s, z10.s\n"
- "ld1w { z10.s }, p2/Z, [x22, x17, LSL #2]\n"
+ "ld1w { z10.s }, p2/Z, [x26, x13, LSL #2]\n"
"fmla z17.s, p3/M, z8.s, z11.s\n"
- "prfm pldl1keep, [x24, x8]\n"
+ "ldr x26, [x14, #0xe0]\n"
"fmla z16.s, p3/M, z7.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x21, x17, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x25, x13, LSL #2]\n"
"fmla z29.s, p3/M, z8.s, z12.s\n"
- "prfm pldl1keep, [x23, x8]\n"
+ "ldr x25, [x14, #0xe8]\n"
"fmla z28.s, p3/M, z7.s, z12.s\n"
- "prfm pldl1keep, [x14, x8]\n"
"fmla z25.s, p3/M, z5.s, z12.s\n"
- "ldr x13, [x7, #0x108]\n"
"fmla z24.s, p3/M, z4.s, z12.s\n"
- "ldr x12, [x7, #0x110]\n"
"fmla z21.s, p3/M, z2.s, z12.s\n"
- "ldr x11, [x7, #0x118]\n"
"fmla z20.s, p3/M, z1.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x20, x17, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x24, x13, LSL #2]\n"
"fmla z31.s, p3/M, z2.s, z10.s\n"
- "prfm pldl1keep, [x13, x8]\n"
+ "ldr x24, [x14, #0xf0]\n"
"fmla z30.s, p3/M, z1.s, z10.s\n"
- "prfm pldl1keep, [x12, x8]\n"
"fmla z29.s, p3/M, z0.s, z10.s\n"
- "ld1w { z10.s }, p2/Z, [x19, x17, LSL #2]\n"
+ "ld1w { z10.s }, p2/Z, [x23, x13, LSL #2]\n"
"fmla z27.s, p3/M, z7.s, z11.s\n"
- "prfm pldl1keep, [x11, x8]\n"
+ "ldr x23, [x14, #0xf8]\n"
"fmla z26.s, p3/M, z6.s, z11.s\n"
- "ldr x22, [x5, #0x0]\n"
"fmla z23.s, p3/M, z4.s, z11.s\n"
- "ldr x21, [x5, #0x8]\n"
"fmla z22.s, p3/M, z3.s, z11.s\n"
- "ldr x20, [x5, #0x10]\n"
"fmla z19.s, p3/M, z1.s, z11.s\n"
- "ldr x19, [x5, #0x18]\n"
"fmla z18.s, p3/M, z0.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x10, x17, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x10, x13, LSL #2]\n"
"fmla z30.s, p3/M, z2.s, z12.s\n"
+ "ldr x10, [x14, #0x100]\n"
"fmla z29.s, p3/M, z1.s, z12.s\n"
"fmla z28.s, p3/M, z0.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x9, x17, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x9, x13, LSL #2]\n"
"fmla z31.s, p3/M, z6.s, z10.s\n"
+ "ldr x9, [x14, #0x108]\n"
"fmla z27.s, p3/M, z3.s, z10.s\n"
"fmla z23.s, p3/M, z0.s, z10.s\n"
- "ld1w { z10.s }, p2/Z, [x28, x17, LSL #2]\n"
+ "ld1w { z10.s }, p2/Z, [x28, x13, LSL #2]\n"
"fmla z25.s, p3/M, z8.s, z11.s\n"
+ "ldr x28, [x14, #0x110]\n"
"fmla z24.s, p3/M, z7.s, z11.s\n"
"fmla z21.s, p3/M, z5.s, z11.s\n"
"fmla z20.s, p3/M, z4.s, z11.s\n"
"fmla z17.s, p3/M, z2.s, z11.s\n"
"fmla z16.s, p3/M, z1.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x27, x17, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x27, x13, LSL #2]\n"
"fmla z28.s, p3/M, z8.s, z12.s\n"
+ "ldr x27, [x14, #0x118]\n"
"fmla z24.s, p3/M, z5.s, z12.s\n"
"fmla z20.s, p3/M, z2.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x26, x17, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x26, x13, LSL #2]\n"
"fmla z27.s, p3/M, z6.s, z10.s\n"
"fmla z23.s, p3/M, z3.s, z10.s\n"
"fmla z19.s, p3/M, z0.s, z10.s\n"
- "ld1w { z10.s }, p2/Z, [x25, x17, LSL #2]\n"
+ "ld1w { z10.s }, p2/Z, [x25, x13, LSL #2]\n"
"fmla z22.s, p3/M, z7.s, z11.s\n"
"fmla z21.s, p3/M, z6.s, z11.s\n"
"fmla z23.s, p3/M, z8.s, z11.s\n"
"fmla z19.s, p3/M, z5.s, z11.s\n"
"fmla z18.s, p3/M, z4.s, z11.s\n"
"fmla z17.s, p3/M, z3.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x24, x17, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x24, x13, LSL #2]\n"
"fmla z24.s, p3/M, z8.s, z12.s\n"
"fmla z20.s, p3/M, z5.s, z12.s\n"
"fmla z16.s, p3/M, z2.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x23, x17, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x23, x13, LSL #2]\n"
"fmla z19.s, p3/M, z8.s, z10.s\n"
"fmla z18.s, p3/M, z7.s, z10.s\n"
"fmla z17.s, p3/M, z6.s, z10.s\n"
- "ld1w { z10.s }, p2/Z, [x14, x17, LSL #2]\n"
+ "ld1w { z10.s }, p2/Z, [x10, x13, LSL #2]\n"
"fmla z22.s, p3/M, z8.s, z11.s\n"
"fmla z21.s, p3/M, z7.s, z11.s\n"
"fmla z20.s, p3/M, z6.s, z11.s\n"
"fmla z18.s, p3/M, z5.s, z11.s\n"
"fmla z17.s, p3/M, z4.s, z11.s\n"
"fmla z16.s, p3/M, z3.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x13, x17, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x9, x13, LSL #2]\n"
"fmla z31.s, p3/M, z4.s, z10.s\n"
"fmla z18.s, p3/M, z8.s, z12.s\n"
"fmla z17.s, p3/M, z7.s, z12.s\n"
"fmla z16.s, p3/M, z6.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x12, x17, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x28, x13, LSL #2]\n"
"fmla z30.s, p3/M, z3.s, z10.s\n"
"fmla z27.s, p3/M, z1.s, z10.s\n"
"fmla z26.s, p3/M, z0.s, z10.s\n"
- "ld1w { z10.s }, p2/Z, [x11, x17, LSL #2]\n"
+ "ld1w { z10.s }, p2/Z, [x27, x13, LSL #2]\n"
"fmla z29.s, p3/M, z5.s, z11.s\n"
"fmla z28.s, p3/M, z4.s, z11.s\n"
"fmla z25.s, p3/M, z2.s, z11.s\n"
@@ -753,68 +647,68 @@
"fmax z29.s, p3/M, z29.s, z15.s\n"
"fmax z28.s, p3/M, z28.s, z15.s\n"
"fmin z31.s, p3/M, z31.s, z14.s\n"
- "st1w { z31.s }, p0, [x22, x15, LSL #2]\n"
+ "st1w { z31.s }, p0, [x22, x11, LSL #2]\n"
"fmin z30.s, p3/M, z30.s, z14.s\n"
"fmin z29.s, p3/M, z29.s, z14.s\n"
- "ldr x22, [x5, #0x20]\n"
+ "ldr x22, [x16, #0x20]\n"
"fmin z28.s, p3/M, z28.s, z14.s\n"
- "st1w { z30.s }, p0, [x21, x15, LSL #2]\n"
+ "st1w { z30.s }, p0, [x21, x11, LSL #2]\n"
"fmax z27.s, p3/M, z27.s, z15.s\n"
"fmax z26.s, p3/M, z26.s, z15.s\n"
- "st1w { z29.s }, p0, [x20, x15, LSL #2]\n"
+ "st1w { z29.s }, p0, [x20, x11, LSL #2]\n"
"fmax z25.s, p3/M, z25.s, z15.s\n"
- "st1w { z28.s }, p0, [x19, x15, LSL #2]\n"
+ "st1w { z28.s }, p0, [x19, x11, LSL #2]\n"
"fmax z24.s, p3/M, z24.s, z15.s\n"
- "ldr x21, [x5, #0x28]\n"
+ "ldr x21, [x16, #0x28]\n"
"fmax z23.s, p3/M, z23.s, z15.s\n"
- "ldr x20, [x5, #0x30]\n"
+ "ldr x20, [x16, #0x30]\n"
"fmin z27.s, p3/M, z27.s, z14.s\n"
- "ldr x19, [x5, #0x38]\n"
+ "ldr x19, [x16, #0x38]\n"
"fmin z26.s, p3/M, z26.s, z14.s\n"
- "st1w { z27.s }, p0, [x22, x15, LSL #2]\n"
+ "st1w { z27.s }, p0, [x22, x11, LSL #2]\n"
"fmin z25.s, p3/M, z25.s, z14.s\n"
"fmin z24.s, p3/M, z24.s, z14.s\n"
- "st1w { z26.s }, p0, [x21, x15, LSL #2]\n"
+ "st1w { z26.s }, p0, [x21, x11, LSL #2]\n"
"fmin z23.s, p3/M, z23.s, z14.s\n"
- "ldr x22, [x5, #0x40]\n"
+ "ldr x22, [x16, #0x40]\n"
"fmax z22.s, p3/M, z22.s, z15.s\n"
- "ldr x21, [x5, #0x48]\n"
+ "ldr x21, [x16, #0x48]\n"
"fmax z21.s, p3/M, z21.s, z15.s\n"
- "st1w { z25.s }, p0, [x20, x15, LSL #2]\n"
+ "st1w { z25.s }, p0, [x20, x11, LSL #2]\n"
"fmax z20.s, p3/M, z20.s, z15.s\n"
- "st1w { z24.s }, p0, [x19, x15, LSL #2]\n"
+ "st1w { z24.s }, p0, [x19, x11, LSL #2]\n"
"fmax z19.s, p3/M, z19.s, z15.s\n"
- "st1w { z23.s }, p0, [x22, x15, LSL #2]\n"
+ "st1w { z23.s }, p0, [x22, x11, LSL #2]\n"
"fmin z22.s, p3/M, z22.s, z14.s\n"
- "ldr x20, [x5, #0x50]\n"
+ "ldr x20, [x16, #0x50]\n"
"fmin z21.s, p3/M, z21.s, z14.s\n"
- "ldr x19, [x5, #0x58]\n"
+ "ldr x19, [x16, #0x58]\n"
"fmin z20.s, p3/M, z20.s, z14.s\n"
- "ldr x22, [x5, #0x60]\n"
+ "ldr x22, [x16, #0x60]\n"
"fmin z19.s, p3/M, z19.s, z14.s\n"
- "st1w { z22.s }, p0, [x21, x15, LSL #2]\n"
+ "st1w { z22.s }, p0, [x21, x11, LSL #2]\n"
"fmax z18.s, p3/M, z18.s, z15.s\n"
- "st1w { z21.s }, p0, [x20, x15, LSL #2]\n"
+ "st1w { z21.s }, p0, [x20, x11, LSL #2]\n"
"fmax z17.s, p3/M, z17.s, z15.s\n"
- "st1w { z20.s }, p0, [x19, x15, LSL #2]\n"
+ "st1w { z20.s }, p0, [x19, x11, LSL #2]\n"
"fmax z16.s, p3/M, z16.s, z15.s\n"
- "st1w { z19.s }, p0, [x22, x15, LSL #2]\n"
- "ldr x21, [x5, #0x68]\n"
+ "st1w { z19.s }, p0, [x22, x11, LSL #2]\n"
+ "ldr x21, [x16, #0x68]\n"
"fmin z18.s, p3/M, z18.s, z14.s\n"
- "ldr x20, [x5, #0x70]\n"
+ "ldr x20, [x16, #0x70]\n"
"fmin z17.s, p3/M, z17.s, z14.s\n"
- "ldr x19, [x5, #0x78]\n"
+ "ldr x19, [x16, #0x78]\n"
"fmin z16.s, p3/M, z16.s, z14.s\n"
- "st1w { z18.s }, p0, [x21, x15, LSL #2]\n"
- "st1w { z17.s }, p0, [x20, x15, LSL #2]\n"
- "st1w { z16.s }, p0, [x19, x15, LSL #2]\n"
+ "st1w { z18.s }, p0, [x21, x11, LSL #2]\n"
+ "st1w { z17.s }, p0, [x20, x11, LSL #2]\n"
+ "st1w { z16.s }, p0, [x19, x11, LSL #2]\n"
:
: [n_channels] "r" ((unsigned long) n_channels), [offsetof_Args_inptrs] "I" (offsetof(Args, inptrs)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_outptrs] "I" (offsetof(Args, outptrs)), [offsetof_args_params] "I" (offsetof(Args, params)), [params_struct] "r" (¶ms_struct)
- : "cc", "memory", "p0", "p1", "p2", "p3", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+ : "cc", "memory", "p0", "p1", "p2", "p3", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
);
}
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(ARM_COMPUTE_ENABLE_SVE)
+#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp
index 2119c06..28d44d0 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp
@@ -28,7 +28,7 @@
#pragma once
-#if defined(ARM_COMPUTE_ENABLE_SVE)
+#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE)
namespace arm_conv {
namespace depthwise {
@@ -36,15 +36,17 @@
void sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl(const float *const *const, float *const *const, const void *, unsigned int, const float, const float);
void sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float);
-struct sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst
+class sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst : public IDepthwiseDepthfirstStrategy
{
- typedef float bias_type;
- typedef float input_type;
- typedef float weight_type;
- typedef float return_type;
-
+ private:
typedef void (*indirect_kern_type)(const float *const *const, float *const *const, const void *, unsigned int, const float, const float);
+ indirect_kern_type m_indirect_kernel = sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl;
+
typedef void (*direct_kern_type)(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float);
+ direct_kern_type m_direct_kernel = sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl;
+
+ public:
+ typedef float return_type;
constexpr static arm_gemm::VLType vl_type = arm_gemm::VLType::SVE;
@@ -60,13 +62,60 @@
constexpr static unsigned int input_rows = 5;
constexpr static unsigned int input_cols = 5;
- indirect_kern_type indirect_kernel = sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl;
- direct_kern_type direct_kernel = sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl;
-
sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst(const CPUInfo *) {}
+
+ arm_gemm::VLType get_vl_type(void) const override { return vl_type; }
+
+ unsigned int get_kernel_rows(void) const override { return kernel_rows; }
+ unsigned int get_kernel_cols(void) const override { return kernel_cols; }
+
+ unsigned int get_stride_rows(void) const override { return stride_rows; }
+ unsigned int get_stride_cols(void) const override { return stride_cols; }
+
+ unsigned int get_output_rows(void) const override { return output_rows; }
+ unsigned int get_output_cols(void) const override { return output_cols; }
+
+ unsigned int get_input_rows(void) const override { return input_rows; }
+ unsigned int get_input_cols(void) const override { return input_cols; }
+
+ void indirect_kernel(
+ const void *const *const input_ptrs,
+ void *const *const outptrs,
+ const void *params,
+ unsigned int n_channels,
+ const void *activation_min,
+ const void *activation_max
+ ) const override
+ {
+ m_indirect_kernel(
+ reinterpret_cast<const float *const *>(input_ptrs),
+ reinterpret_cast<float *const *>(outptrs),
+ params, n_channels,
+ *static_cast<const float *>(activation_min),
+ *static_cast<const float *>(activation_max)
+ );
+ }
+
+ void direct_kernel(
+ const unsigned int n_tile_rows, const unsigned int n_tile_cols,
+ const void *inptr, int64_t ld_input_row, int64_t ld_input_col,
+ void *outptr, int64_t ld_output_row, int64_t ld_output_col,
+ const void *params, unsigned int n_channels,
+ const void *activation_min, const void *activation_max
+ ) const override
+ {
+ m_direct_kernel(
+ n_tile_rows, n_tile_cols,
+ static_cast<const float *>(inptr), ld_input_row, ld_input_col,
+ static_cast<float *>(outptr), ld_output_row, ld_output_col,
+ params, n_channels,
+ *static_cast<const float *>(activation_min),
+ *static_cast<const float *>(activation_max)
+ );
+ }
};
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(ARM_COMPUTE_ENABLE_SVE)
+#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp
index 78e67e1..ac33dcb 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp
@@ -25,7 +25,7 @@
#include <cstddef>
#include <cstdint>
-#if defined(ARM_COMPUTE_ENABLE_SVE)
+#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE)
namespace arm_conv {
namespace depthwise {
@@ -88,318 +88,250 @@
__asm__ __volatile__(
"ptrue p3.b\n"
- "mov x5, #0x0\n"
- "mov x6, #0x0\n"
+ "mov x7, #0x0\n"
+ "mov x8, #0x0\n"
"1:" // Tile loop
- "str x5, [%x[params_struct], %[offsetof_args_tile_i]]\n"
- "mov x21, #0x4\n"
- "str x6, [%x[params_struct], %[offsetof_args_tile_j]]\n"
- "cntb x7\n"
- "ldr x8, [%x[params_struct], %[offsetof_args_params]]\n"
- "add x7, x7, XZR, LSL #4\n"
- "ldr x20, [%x[params_struct], %[offsetof_args_ld_input_row]]\n"
- "cntb x17\n"
- "ldr x16, [%x[params_struct], %[offsetof_args_ld_input_col]]\n"
- "cntb x15\n"
- "ldr x14, [%x[params_struct], %[offsetof_args_inptr]]\n"
- "mul x19, x5, x20\n" // offset = tile_i * ld_input_row
- "ldr x22, [%x[params_struct], %[offsetof_args_ld_output_row]]\n"
- "madd x19, x6, x16, x19\n" // offset += tile_j * ld_input_col
- "ldr x13, [%x[params_struct], %[offsetof_args_ld_output_col]]\n"
- "mul x19, x19, x21\n" // offset *= kernel_stride * output_size
- "ldr x12, [%x[params_struct], %[offsetof_args_outptr]]\n"
- "add x14, x14, x19, LSL #2\n" // inptr[0] += offset * sizeof(float)
+ "str x7, [%x[params_struct], %[offsetof_args_tile_i]]\n"
+ "mov x23, #0x4\n"
+ "str x8, [%x[params_struct], %[offsetof_args_tile_j]]\n"
+ "mov x17, #0x2\n"
+ "ldr x16, [%x[params_struct], %[offsetof_args_params]]\n"
+ "mov x15, #0x0\n"
+ "ldr x22, [%x[params_struct], %[offsetof_args_ld_input_row]]\n"
+ "cntw x14\n"
+ "ldr x13, [%x[params_struct], %[offsetof_args_ld_input_col]]\n"
+ "sub x12, XZR, x14\n"
+ "ldr x21, [%x[params_struct], %[offsetof_args_inptr]]\n"
+ "mul x19, x7, x22\n" // offset = tile_i * ld_input_row
+ "ldr x20, [%x[params_struct], %[offsetof_args_ld_output_row]]\n"
+ "madd x19, x8, x13, x19\n" // offset += tile_j * ld_input_col
+ "ldr x11, [%x[params_struct], %[offsetof_args_ld_output_col]]\n"
+ "mul x19, x19, x23\n" // offset *= kernel_stride * output_size
+ "ldr x10, [%x[params_struct], %[offsetof_args_outptr]]\n"
+ "add x21, x21, x19, LSL #2\n" // inptr[0] += offset * sizeof(float)
"ld1rw { z19.s }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n"
- "add x11, x14, x20, LSL #2\n"
+ "add x9, x21, x22, LSL #2\n"
"ld1rw { z18.s }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n"
- "add x10, x11, x20, LSL #2\n"
- "ld1w { z17.s }, p3/Z, [x8]\n"
- "mov z31.d, z17.d\n"
- "ld1w { z0.s }, p3/Z, [x8, #1, MUL VL]\n"
- "add x9, x10, x20, LSL #2\n"
- "mov z30.d, z17.d\n"
- "ld1w { z1.s }, p3/Z, [x8, #2, MUL VL]\n"
- "add x28, x9, x20, LSL #2\n"
- "mov z29.d, z17.d\n"
- "ld1w { z2.s }, p3/Z, [x8, #3, MUL VL]\n"
- "add x27, x16, x16\n"
- "mov z28.d, z17.d\n"
- "ld1w { z3.s }, p3/Z, [x8, #4, MUL VL]\n"
- "add x26, x27, x16\n"
- "ld1w { z4.s }, p3/Z, [x8, #5, MUL VL]\n"
- "add x25, x26, x16\n"
- "ld1w { z5.s }, p3/Z, [x8, #6, MUL VL]\n"
- "add x17, x17, x16, LSL #4\n"
- "ld1w { z6.s }, p3/Z, [x8, #7, MUL VL]\n"
- "add x15, x15, x27, LSL #4\n"
- "cntb x24\n"
- "prfm pldl1keep, [x10, x15]\n"
- "prfm pldl1keep, [x14, x7]\n"
- "add x24, x24, x26, LSL #4\n"
- "prfm pldl1keep, [x14, x17]\n"
- "cntb x23\n"
- "prfm pldl1keep, [x14, x24]\n"
- "add x23, x23, x25, LSL #4\n"
- "mov x20, #0x2\n"
- "prfm pldl1keep, [x14, x23]\n"
- "prfm pldl1keep, [x11, x7]\n"
- "mul x19, x5, x22\n" // offset = tile_i * ld_output_row
- "prfm pldl1keep, [x11, x17]\n"
- "madd x19, x6, x13, x19\n" // offset += tile_j * ld_output_col
- "prfm pldl1keep, [x14, x15]\n"
- "mul x19, x19, x20\n" // offset *= output_tile_size
- "mov x21, #0x0\n"
- "add x12, x12, x19, LSL #2\n" // outptrs[0] += offset * sizeof(float)
- "add x22, x12, x22, LSL #2\n"
- "cntw x20\n"
- "sub x19, XZR, x20\n"
+ "add x28, x9, x22, LSL #2\n"
+ "ld1w { z17.s }, p3/Z, [x16]\n"
+ "add x27, x28, x22, LSL #2\n"
+ "ld1w { z0.s }, p3/Z, [x16, #1, MUL VL]\n"
+ "add x26, x27, x22, LSL #2\n"
+ "ld1w { z1.s }, p3/Z, [x16, #2, MUL VL]\n"
+ "add x25, x13, x13\n"
+ "ld1w { z2.s }, p3/Z, [x16, #3, MUL VL]\n"
+ "add x24, x25, x13\n"
+ "ld1w { z3.s }, p3/Z, [x16, #4, MUL VL]\n"
+ "add x23, x24, x13\n"
+ "ld1w { z4.s }, p3/Z, [x16, #5, MUL VL]\n"
+ "mul x19, x7, x20\n" // offset = tile_i * ld_output_row
+ "ld1w { z5.s }, p3/Z, [x16, #6, MUL VL]\n"
+ "madd x19, x8, x11, x19\n" // offset += tile_j * ld_output_col
+ "ld1w { z6.s }, p3/Z, [x16, #7, MUL VL]\n"
+ "mul x19, x19, x17\n" // offset *= output_tile_size
"whilelt p2.s, XZR, %x[n_channels]\n"
- "ld1w { z9.s }, p2/Z, [x10, x27, LSL #2]\n"
- "ld1w { z10.s }, p2/Z, [x14]\n"
- "addvl x8, x8, #16\n"
- "ld1w { z11.s }, p2/Z, [x14, x16, LSL #2]\n"
- "cmp x20, %x[n_channels]\n"
- "ld1w { z7.s }, p3/Z, [x8, #-8, MUL VL]\n"
- "ld1w { z8.s }, p3/Z, [x8, #-7, MUL VL]\n"
- "addvl x8, x8, #-6\n"
- "ld1w { z12.s }, p2/Z, [x14, x26, LSL #2]\n"
- "ld1w { z13.s }, p2/Z, [x14, x25, LSL #2]\n"
- "ld1w { z14.s }, p2/Z, [x11]\n"
- "ld1w { z15.s }, p2/Z, [x11, x16, LSL #2]\n"
- "ld1w { z16.s }, p2/Z, [x14, x27, LSL #2]\n"
+ "ld1w { z9.s }, p2/Z, [x28, x25, LSL #2]\n"
+ "ld1w { z10.s }, p2/Z, [x21]\n"
+ "add x10, x10, x19, LSL #2\n" // outptrs[0] += offset * sizeof(float)
+ "ld1w { z11.s }, p2/Z, [x21, x13, LSL #2]\n"
+ "add x22, x10, x20, LSL #2\n"
+ "ld1w { z12.s }, p2/Z, [x21, x24, LSL #2]\n"
+ "addvl x16, x16, #16\n"
+ "ld1w { z13.s }, p2/Z, [x21, x23, LSL #2]\n"
+ "cmp x14, %x[n_channels]\n"
+ "ld1w { z7.s }, p3/Z, [x16, #-8, MUL VL]\n"
+ "ld1w { z8.s }, p3/Z, [x16, #-7, MUL VL]\n"
+ "addvl x16, x16, #-6\n"
+ "ld1w { z14.s }, p2/Z, [x9]\n"
+ "ld1w { z15.s }, p2/Z, [x9, x13, LSL #2]\n"
+ "ld1w { z16.s }, p2/Z, [x21, x25, LSL #2]\n"
"bge 3f\n"
"2:" // Tile loop: Channel loop
- "fmla z31.s, p3/M, z8.s, z9.s\n"
- "prfm pldl1keep, [x11, x24]\n"
- "whilelt p1.s, x20, %x[n_channels]\n"
- "fmla z30.s, p3/M, z6.s, z9.s\n"
- "prfm pldl1keep, [x11, x23]\n"
- "incw x19\n"
- "fmla z29.s, p3/M, z2.s, z9.s\n"
- "prfm pldl1keep, [x11, x15]\n"
+ "movprfx z31, z17\n fmla z31.s, p3/M, z8.s, z9.s\n"
+ "whilelt p1.s, x14, %x[n_channels]\n"
+ "movprfx z30, z17\n fmla z30.s, p3/M, z6.s, z9.s\n"
+ "incw x12\n"
+ "movprfx z29, z17\n fmla z29.s, p3/M, z2.s, z9.s\n"
"mov p0.b, p2.b\n"
- "fmla z28.s, p3/M, z0.s, z9.s\n"
- "prfm pldl1keep, [x9, x7]\n"
- "incw x21\n"
+ "movprfx z28, z17\n fmla z28.s, p3/M, z0.s, z9.s\n"
+ "ld1w { z17.s }, p3/Z, [x16]\n"
+ "incw x15\n"
"fmla z31.s, p3/M, z0.s, z10.s\n"
- "prfm pldl1keep, [x10, x7]\n"
- "addvl x14, x14, #1\n"
+ "addvl x21, x21, #1\n"
+ "ld1w { z10.s }, p1/Z, [x21]\n"
"fmla z30.s, p3/M, z1.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x11, x25, LSL #2]\n"
- "incw x20\n"
+ "incw x14\n"
"fmla z31.s, p3/M, z1.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x11, x26, LSL #2]\n"
- "prfm pldl1keep, [x9, x17]\n"
+ "ld1w { z11.s }, p2/Z, [x9, x24, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x9, x23, LSL #2]\n"
"fmla z30.s, p3/M, z2.s, z13.s\n"
- "ld1w { z13.s }, p2/Z, [x11, x27, LSL #2]\n"
+ "ld1w { z13.s }, p2/Z, [x9, x25, LSL #2]\n"
"fmla z31.s, p3/M, z3.s, z14.s\n"
- "ld1w { z14.s }, p2/Z, [x9]\n"
- "addvl x11, x11, #1\n"
- "fmla z30.s, p3/M, z0.s, z16.s\n"
- "prfm pldl1keep, [x10, x17]\n"
- "prfm pldl1keep, [x9, x24]\n"
- "fmla z29.s, p3/M, z3.s, z14.s\n"
- "prfm pldl1keep, [x10, x24]\n"
- "ld1w { z14.s }, p2/Z, [x9, x25, LSL #2]\n"
- "fmla z31.s, p3/M, z4.s, z15.s\n"
- "ld1w { z15.s }, p2/Z, [x10]\n"
- "fmla z30.s, p3/M, z4.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x9, x16, LSL #2]\n"
- "fmla z29.s, p3/M, z0.s, z15.s\n"
- "prfm pldl1keep, [x9, x23]\n"
- "prfm pldl1keep, [x28, x7]\n"
- "fmla z31.s, p3/M, z2.s, z16.s\n"
- "ld1w { z16.s }, p2/Z, [x10, x16, LSL #2]\n"
- "fmla z30.s, p3/M, z5.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x10, x26, LSL #2]\n"
- "fmla z29.s, p3/M, z4.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x10, x25, LSL #2]\n"
- "prfm pldl1keep, [x10, x23]\n"
- "fmla z31.s, p3/M, z5.s, z13.s\n"
- "addvl x10, x10, #1\n"
- "fmla z30.s, p3/M, z3.s, z13.s\n"
- "ld1w { z13.s }, p2/Z, [x9, x26, LSL #2]\n"
- "prfm pldl1keep, [x28, x17]\n"
- "fmla z29.s, p3/M, z1.s, z16.s\n"
- "prfm pldl1keep, [x9, x15]\n"
- "prfm pldl1keep, [x28, x24]\n"
- "fmla z31.s, p3/M, z6.s, z15.s\n"
- "fmla z28.s, p3/M, z4.s, z13.s\n"
- "ld1w { z15.s }, p2/Z, [x28]\n"
- "fmla z30.s, p3/M, z7.s, z12.s\n"
- "ld1w { z13.s }, p2/Z, [x28, x16, LSL #2]\n"
- "prfm pldl1keep, [x28, x15]\n"
- "fmla z31.s, p3/M, z7.s, z16.s\n"
- "ld1w { z16.s }, p2/Z, [x9, x27, LSL #2]\n"
+ "ld1w { z14.s }, p2/Z, [x27]\n"
"addvl x9, x9, #1\n"
- "fmla z28.s, p3/M, z1.s, z12.s\n"
- "prfm pldl1keep, [x28, x23]\n"
- "fmla z29.s, p3/M, z6.s, z15.s\n"
- "ld1w { z15.s }, p2/Z, [x28, x27, LSL #2]\n"
- "fmla z30.s, p3/M, z8.s, z11.s\n"
- "ld1w { z9.s }, p1/Z, [x10, x27, LSL #2]\n"
- "prfm pldl1keep, [x10, x15]\n"
- "fmax z31.s, p3/M, z31.s, z19.s\n"
- "ld1w { z10.s }, p1/Z, [x14]\n"
- "fmla z28.s, p3/M, z5.s, z14.s\n"
- "fmla z29.s, p3/M, z7.s, z13.s\n"
- "ld1w { z14.s }, p2/Z, [x28, x26, LSL #2]\n"
- "fmax z30.s, p3/M, z30.s, z19.s\n"
- "prfm pldl1keep, [x14, x7]\n"
- "prfm pldl1keep, [x14, x17]\n"
- "fmin z31.s, p3/M, z31.s, z18.s\n"
- "ld1w { z12.s }, p1/Z, [x14, x26, LSL #2]\n"
- "fmla z28.s, p3/M, z2.s, z11.s\n"
- "fmla z29.s, p3/M, z5.s, z16.s\n"
- "ld1w { z11.s }, p2/Z, [x28, x25, LSL #2]\n"
- "whilelt p2.s, x21, %x[n_channels]\n"
- "fmin z30.s, p3/M, z30.s, z18.s\n"
- "prfm pldl1keep, [x14, x24]\n"
+ "fmla z30.s, p3/M, z0.s, z16.s\n"
+ "fmla z29.s, p3/M, z3.s, z14.s\n"
+ "ld1w { z14.s }, p2/Z, [x27, x23, LSL #2]\n"
+ "fmla z31.s, p3/M, z4.s, z15.s\n"
+ "ld1w { z15.s }, p2/Z, [x28]\n"
+ "fmla z30.s, p3/M, z4.s, z11.s\n"
+ "ld1w { z11.s }, p2/Z, [x27, x13, LSL #2]\n"
+ "fmla z29.s, p3/M, z0.s, z15.s\n"
+ "ld1w { z0.s }, p3/Z, [x16, #1, MUL VL]\n"
+ "fmla z31.s, p3/M, z2.s, z16.s\n"
+ "ld1w { z16.s }, p2/Z, [x28, x13, LSL #2]\n"
+ "fmla z30.s, p3/M, z5.s, z12.s\n"
+ "ld1w { z12.s }, p2/Z, [x28, x24, LSL #2]\n"
+ "fmla z29.s, p3/M, z4.s, z11.s\n"
+ "ld1w { z11.s }, p2/Z, [x28, x23, LSL #2]\n"
"addvl x28, x28, #1\n"
- "fmla z28.s, p3/M, z3.s, z16.s\n"
- "ld1w { z13.s }, p1/Z, [x14, x25, LSL #2]\n"
- "cmp x20, %x[n_channels]\n"
+ "fmla z31.s, p3/M, z5.s, z13.s\n"
+ "ld1w { z9.s }, p1/Z, [x28, x25, LSL #2]\n"
+ "fmla z30.s, p3/M, z3.s, z13.s\n"
+ "ld1w { z13.s }, p2/Z, [x27, x24, LSL #2]\n"
+ "fmla z29.s, p3/M, z1.s, z16.s\n"
+ "fmla z31.s, p3/M, z6.s, z15.s\n"
+ "ld1w { z15.s }, p2/Z, [x26]\n"
+ "fmla z28.s, p3/M, z4.s, z13.s\n"
+ "ld1w { z13.s }, p2/Z, [x26, x13, LSL #2]\n"
+ "fmla z30.s, p3/M, z7.s, z12.s\n"
+ "ld1w { z4.s }, p3/Z, [x16, #5, MUL VL]\n"
+ "fmla z29.s, p3/M, z6.s, z15.s\n"
+ "ld1w { z15.s }, p2/Z, [x26, x25, LSL #2]\n"
+ "fmla z31.s, p3/M, z7.s, z16.s\n"
+ "ld1w { z16.s }, p2/Z, [x27, x25, LSL #2]\n"
+ "addvl x27, x27, #1\n"
+ "fmla z28.s, p3/M, z1.s, z12.s\n"
+ "ld1w { z12.s }, p1/Z, [x21, x24, LSL #2]\n"
+ "fmla z30.s, p3/M, z8.s, z11.s\n"
+ "ld1w { z1.s }, p3/Z, [x16, #2, MUL VL]\n"
+ "fmla z29.s, p3/M, z7.s, z13.s\n"
+ "ld1w { z13.s }, p1/Z, [x21, x23, LSL #2]\n"
+ "fmax z31.s, p3/M, z31.s, z19.s\n"
+ "fmla z28.s, p3/M, z5.s, z14.s\n"
+ "ld1w { z14.s }, p2/Z, [x26, x24, LSL #2]\n"
+ "fmax z30.s, p3/M, z30.s, z19.s\n"
+ "fmla z29.s, p3/M, z5.s, z16.s\n"
+ "ld1w { z5.s }, p3/Z, [x16, #6, MUL VL]\n"
+ "fmin z31.s, p3/M, z31.s, z18.s\n"
+ "st1w { z31.s }, p0, [x10]\n"
+ "fmla z28.s, p3/M, z2.s, z11.s\n"
"fmla z29.s, p3/M, z8.s, z15.s\n"
- "prfm pldl1keep, [x14, x23]\n"
- "prfm pldl1keep, [x11, x7]\n"
- "fmla z28.s, p3/M, z7.s, z14.s\n"
- "ld1w { z14.s }, p1/Z, [x11]\n"
- "prfm pldl1keep, [x11, x17]\n"
+ "ld1w { z11.s }, p2/Z, [x26, x23, LSL #2]\n"
+ "whilelt p2.s, x15, %x[n_channels]\n"
+ "fmin z30.s, p3/M, z30.s, z18.s\n"
+ "ld1w { z2.s }, p3/Z, [x16, #3, MUL VL]\n"
+ "addvl x26, x26, #1\n"
+ "fmla z28.s, p3/M, z3.s, z16.s\n"
+ "ld1w { z16.s }, p1/Z, [x21, x25, LSL #2]\n"
+ "cmp x14, %x[n_channels]\n"
"fmax z29.s, p3/M, z29.s, z19.s\n"
- "ld1w { z16.s }, p1/Z, [x14, x27, LSL #2]\n"
- "fmla z28.s, p3/M, z6.s, z15.s\n"
- "ld1w { z15.s }, p1/Z, [x11, x16, LSL #2]\n"
- "prfm pldl1keep, [x14, x15]\n"
+ "ld1w { z3.s }, p3/Z, [x16, #4, MUL VL]\n"
+ "st1w { z30.s }, p0, [x10, x11, LSL #2]\n"
+ "fmla z28.s, p3/M, z7.s, z14.s\n"
+ "ld1w { z14.s }, p1/Z, [x9]\n"
+ "addvl x10, x10, #1\n"
"fmin z29.s, p3/M, z29.s, z18.s\n"
- "st1w { z31.s }, p0, [x12]\n"
- "fmla z28.s, p3/M, z8.s, z11.s\n"
- "ld1w { z11.s }, p1/Z, [x14, x16, LSL #2]\n"
- "st1w { z30.s }, p0, [x12, x13, LSL #2]\n"
- "fmax z28.s, p3/M, z28.s, z19.s\n"
"st1w { z29.s }, p0, [x22]\n"
- "addvl x12, x12, #1\n"
+ "fmla z28.s, p3/M, z6.s, z15.s\n"
+ "ld1w { z15.s }, p1/Z, [x9, x13, LSL #2]\n"
+ "fmla z28.s, p3/M, z8.s, z11.s\n"
+ "ld1w { z11.s }, p1/Z, [x21, x13, LSL #2]\n"
+ "ld1w { z6.s }, p3/Z, [x16, #7, MUL VL]\n"
+ "fmax z28.s, p3/M, z28.s, z19.s\n"
+ "addvl x16, x16, #16\n"
+ "ld1w { z7.s }, p3/Z, [x16, #-8, MUL VL]\n"
"fmin z28.s, p3/M, z28.s, z18.s\n"
- "ld1w { z17.s }, p3/Z, [x8]\n"
- "ld1w { z0.s }, p3/Z, [x8, #1, MUL VL]\n"
- "mov z31.d, z17.d\n"
- "ld1w { z1.s }, p3/Z, [x8, #2, MUL VL]\n"
- "mov z30.d, z17.d\n"
- "st1w { z28.s }, p0, [x22, x13, LSL #2]\n"
+ "ld1w { z8.s }, p3/Z, [x16, #-7, MUL VL]\n"
+ "addvl x16, x16, #-6\n"
+ "st1w { z28.s }, p0, [x22, x11, LSL #2]\n"
"addvl x22, x22, #1\n"
- "mov z29.d, z17.d\n"
- "ld1w { z2.s }, p3/Z, [x8, #3, MUL VL]\n"
- "mov z28.d, z17.d\n"
- "ld1w { z3.s }, p3/Z, [x8, #4, MUL VL]\n"
- "ld1w { z4.s }, p3/Z, [x8, #5, MUL VL]\n"
- "ld1w { z5.s }, p3/Z, [x8, #6, MUL VL]\n"
- "ld1w { z6.s }, p3/Z, [x8, #7, MUL VL]\n"
- "addvl x8, x8, #16\n"
- "ld1w { z7.s }, p3/Z, [x8, #-8, MUL VL]\n"
- "ld1w { z8.s }, p3/Z, [x8, #-7, MUL VL]\n"
- "addvl x8, x8, #-6\n"
"blt 2b\n"
"3:" // Tile loop: Channel tail
- "fmla z31.s, p3/M, z8.s, z9.s\n"
- "prfm pldl1keep, [x11, x24]\n"
+ "movprfx z31, z17\n fmla z31.s, p3/M, z8.s, z9.s\n"
+ "ldr x7, [%x[params_struct], %[offsetof_args_tile_i]]\n"
"mov p0.b, p2.b\n"
- "fmla z30.s, p3/M, z6.s, z9.s\n"
- "prfm pldl1keep, [x11, x23]\n"
- "fmla z29.s, p3/M, z2.s, z9.s\n"
- "prfm pldl1keep, [x11, x15]\n"
- "fmla z28.s, p3/M, z0.s, z9.s\n"
- "prfm pldl1keep, [x9, x7]\n"
- "prfm pldl1keep, [x10, x7]\n"
- "fmla z31.s, p3/M, z0.s, z10.s\n"
- "prfm pldl1keep, [x9, x17]\n"
- "fmla z30.s, p3/M, z1.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x11, x25, LSL #2]\n"
- "fmla z31.s, p3/M, z1.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x11, x26, LSL #2]\n"
- "prfm pldl1keep, [x10, x17]\n"
- "fmla z30.s, p3/M, z2.s, z13.s\n"
- "ld1w { z13.s }, p2/Z, [x11, x27, LSL #2]\n"
- "fmla z31.s, p3/M, z3.s, z14.s\n"
- "ld1w { z14.s }, p2/Z, [x9]\n"
- "prfm pldl1keep, [x9, x24]\n"
- "fmla z30.s, p3/M, z0.s, z16.s\n"
- "prfm pldl1keep, [x10, x24]\n"
- "fmla z29.s, p3/M, z3.s, z14.s\n"
- "ld1w { z14.s }, p2/Z, [x9, x25, LSL #2]\n"
- "prfm pldl1keep, [x9, x23]\n"
- "fmla z31.s, p3/M, z4.s, z15.s\n"
- "ld1w { z15.s }, p2/Z, [x10]\n"
- "fmla z30.s, p3/M, z4.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x9, x16, LSL #2]\n"
- "fmla z29.s, p3/M, z0.s, z15.s\n"
- "prfm pldl1keep, [x28, x7]\n"
- "prfm pldl1keep, [x10, x23]\n"
- "fmla z31.s, p3/M, z2.s, z16.s\n"
- "ld1w { z16.s }, p2/Z, [x10, x16, LSL #2]\n"
- "fmla z30.s, p3/M, z5.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x10, x26, LSL #2]\n"
- "fmla z29.s, p3/M, z4.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x10, x25, LSL #2]\n"
- "prfm pldl1keep, [x28, x17]\n"
- "fmla z31.s, p3/M, z5.s, z13.s\n"
- "prfm pldl1keep, [x9, x15]\n"
- "fmla z30.s, p3/M, z3.s, z13.s\n"
- "ld1w { z13.s }, p2/Z, [x9, x26, LSL #2]\n"
- "fmla z29.s, p3/M, z1.s, z16.s\n"
- "prfm pldl1keep, [x28, x24]\n"
- "prfm pldl1keep, [x28, x15]\n"
- "fmla z31.s, p3/M, z6.s, z15.s\n"
- "fmla z28.s, p3/M, z4.s, z13.s\n"
- "ld1w { z15.s }, p2/Z, [x28]\n"
- "fmla z30.s, p3/M, z7.s, z12.s\n"
- "ld1w { z13.s }, p2/Z, [x28, x16, LSL #2]\n"
- "prfm pldl1keep, [x28, x23]\n"
- "fmla z31.s, p3/M, z7.s, z16.s\n"
- "ld1w { z16.s }, p2/Z, [x9, x27, LSL #2]\n"
- "fmla z28.s, p3/M, z1.s, z12.s\n"
- "ldr x5, [%x[params_struct], %[offsetof_args_tile_i]]\n"
- "add x21, x5, #0x1\n"
- "fmla z28.s, p3/M, z5.s, z14.s\n"
- "ld1w { z14.s }, p2/Z, [x28, x26, LSL #2]\n"
- "fmla z29.s, p3/M, z6.s, z15.s\n"
- "ld1w { z15.s }, p2/Z, [x28, x27, LSL #2]\n"
- "fmla z30.s, p3/M, z8.s, z11.s\n"
- "ldr x6, [%x[params_struct], %[offsetof_args_tile_j]]\n"
- "add x6, x6, #0x1\n"
- "fmla z28.s, p3/M, z2.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x28, x25, LSL #2]\n"
- "fmla z29.s, p3/M, z7.s, z13.s\n"
+ "movprfx z30, z17\n fmla z30.s, p3/M, z6.s, z9.s\n"
+ "ldr x8, [%x[params_struct], %[offsetof_args_tile_j]]\n"
+ "add x21, x7, #0x1\n"
+ "movprfx z29, z17\n fmla z29.s, p3/M, z2.s, z9.s\n"
"ldr x20, [%x[params_struct], %[offsetof_args_n_tile_rows]]\n"
- "fmax z31.s, p3/M, z31.s, z19.s\n"
+ "movprfx z28, z17\n fmla z28.s, p3/M, z0.s, z9.s\n"
"ldr x19, [%x[params_struct], %[offsetof_args_n_tile_cols]]\n"
- "cmp x6, x19\n"
- "fmla z29.s, p3/M, z5.s, z16.s\n"
- "fmla z28.s, p3/M, z3.s, z16.s\n"
- "csel x6, x6, XZR, LT\n"
- "fmin z31.s, p3/M, z31.s, z18.s\n"
- "st1w { z31.s }, p0, [x12]\n"
- "fmla z28.s, p3/M, z7.s, z14.s\n"
- "csel x5, x5, x21, LT\n"
- "fmla z29.s, p3/M, z8.s, z15.s\n"
- "cmp x5, x20\n"
+ "add x8, x8, #0x1\n"
+ "fmla z31.s, p3/M, z0.s, z10.s\n"
+ "cmp x8, x19\n"
+ "fmla z30.s, p3/M, z1.s, z12.s\n"
+ "ld1w { z12.s }, p2/Z, [x9, x23, LSL #2]\n"
+ "fmla z31.s, p3/M, z1.s, z11.s\n"
+ "ld1w { z11.s }, p2/Z, [x9, x24, LSL #2]\n"
+ "csel x8, x8, XZR, LT\n"
+ "fmla z30.s, p3/M, z2.s, z13.s\n"
+ "ld1w { z13.s }, p2/Z, [x9, x25, LSL #2]\n"
+ "csel x7, x7, x21, LT\n"
+ "fmla z31.s, p3/M, z3.s, z14.s\n"
+ "ld1w { z14.s }, p2/Z, [x27]\n"
+ "cmp x7, x20\n"
+ "fmla z30.s, p3/M, z0.s, z16.s\n"
+ "fmla z29.s, p3/M, z3.s, z14.s\n"
+ "ld1w { z14.s }, p2/Z, [x27, x23, LSL #2]\n"
+ "fmla z31.s, p3/M, z4.s, z15.s\n"
+ "ld1w { z15.s }, p2/Z, [x28]\n"
+ "fmla z30.s, p3/M, z4.s, z11.s\n"
+ "ld1w { z11.s }, p2/Z, [x27, x13, LSL #2]\n"
+ "fmla z29.s, p3/M, z0.s, z15.s\n"
+ "fmla z31.s, p3/M, z2.s, z16.s\n"
+ "ld1w { z16.s }, p2/Z, [x28, x13, LSL #2]\n"
+ "fmla z30.s, p3/M, z5.s, z12.s\n"
+ "ld1w { z12.s }, p2/Z, [x28, x24, LSL #2]\n"
+ "fmla z29.s, p3/M, z4.s, z11.s\n"
+ "ld1w { z11.s }, p2/Z, [x28, x23, LSL #2]\n"
+ "fmla z31.s, p3/M, z5.s, z13.s\n"
+ "fmla z30.s, p3/M, z3.s, z13.s\n"
+ "ld1w { z13.s }, p2/Z, [x27, x24, LSL #2]\n"
+ "fmla z29.s, p3/M, z1.s, z16.s\n"
+ "fmla z31.s, p3/M, z6.s, z15.s\n"
+ "ld1w { z15.s }, p2/Z, [x26]\n"
+ "fmla z28.s, p3/M, z4.s, z13.s\n"
+ "ld1w { z13.s }, p2/Z, [x26, x13, LSL #2]\n"
+ "fmla z30.s, p3/M, z7.s, z12.s\n"
+ "fmla z29.s, p3/M, z6.s, z15.s\n"
+ "ld1w { z15.s }, p2/Z, [x26, x25, LSL #2]\n"
+ "fmla z31.s, p3/M, z7.s, z16.s\n"
+ "ld1w { z16.s }, p2/Z, [x27, x25, LSL #2]\n"
+ "fmla z28.s, p3/M, z1.s, z12.s\n"
+ "fmla z30.s, p3/M, z8.s, z11.s\n"
+ "fmla z29.s, p3/M, z7.s, z13.s\n"
+ "fmax z31.s, p3/M, z31.s, z19.s\n"
+ "fmla z28.s, p3/M, z5.s, z14.s\n"
+ "ld1w { z14.s }, p2/Z, [x26, x24, LSL #2]\n"
"fmax z30.s, p3/M, z30.s, z19.s\n"
- "fmla z28.s, p3/M, z6.s, z15.s\n"
+ "fmla z29.s, p3/M, z5.s, z16.s\n"
+ "fmin z31.s, p3/M, z31.s, z18.s\n"
+ "st1w { z31.s }, p0, [x10]\n"
+ "fmla z28.s, p3/M, z2.s, z11.s\n"
+ "fmla z29.s, p3/M, z8.s, z15.s\n"
+ "ld1w { z11.s }, p2/Z, [x26, x23, LSL #2]\n"
"fmin z30.s, p3/M, z30.s, z18.s\n"
- "st1w { z30.s }, p0, [x12, x13, LSL #2]\n"
- "fmla z28.s, p3/M, z8.s, z11.s\n"
+ "st1w { z30.s }, p0, [x10, x11, LSL #2]\n"
+ "fmla z28.s, p3/M, z3.s, z16.s\n"
"fmax z29.s, p3/M, z29.s, z19.s\n"
+ "fmla z28.s, p3/M, z7.s, z14.s\n"
"fmin z29.s, p3/M, z29.s, z18.s\n"
"st1w { z29.s }, p0, [x22]\n"
+ "fmla z28.s, p3/M, z6.s, z15.s\n"
+ "fmla z28.s, p3/M, z8.s, z11.s\n"
"fmax z28.s, p3/M, z28.s, z19.s\n"
"fmin z28.s, p3/M, z28.s, z18.s\n"
- "st1w { z28.s }, p0, [x22, x13, LSL #2]\n"
+ "st1w { z28.s }, p0, [x22, x11, LSL #2]\n"
"blt 1b\n"
:
: [n_channels] "r" ((unsigned long) n_channels), [offsetof_args_inptr] "I" (offsetof(Args, inptr)), [offsetof_args_ld_input_col] "I" (offsetof(Args, ld_input_col)), [offsetof_args_ld_input_row] "I" (offsetof(Args, ld_input_row)), [offsetof_args_ld_output_col] "I" (offsetof(Args, ld_output_col)), [offsetof_args_ld_output_row] "I" (offsetof(Args, ld_output_row)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_n_tile_cols] "I" (offsetof(Args, n_tile_cols)), [offsetof_args_n_tile_rows] "I" (offsetof(Args, n_tile_rows)), [offsetof_args_outptr] "I" (offsetof(Args, outptr)), [offsetof_args_params] "I" (offsetof(Args, params)), [offsetof_args_tile_i] "I" (offsetof(Args, tile_i)), [offsetof_args_tile_j] "I" (offsetof(Args, tile_j)), [params_struct] "r" (¶ms_struct)
- : "cc", "memory", "p0", "p1", "p2", "p3", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z28", "z29", "z30", "z31"
+ : "cc", "memory", "p0", "p1", "p2", "p3", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z28", "z29", "z30", "z31"
);
}
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(ARM_COMPUTE_ENABLE_SVE)
+#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp
index 8555cfe..829b0ff 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp
@@ -25,7 +25,7 @@
#include <cstddef>
#include <cstdint>
-#if defined(ARM_COMPUTE_ENABLE_SVE)
+#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE)
namespace arm_conv {
namespace depthwise {
@@ -89,309 +89,249 @@
__asm__ __volatile__(
"ldr x19, [%x[params_struct], %[offsetof_args_outptrs]]\n"
"ptrue p3.b\n"
- "ldr x14, [%x[params_struct], %[offsetof_args_params]]\n"
- "add x13, %x[params_struct], %[offsetof_Args_inptrs]\n"
+ "ldr x15, [%x[params_struct], %[offsetof_args_params]]\n"
+ "add x14, %x[params_struct], %[offsetof_Args_inptrs]\n"
"ld1rw { z19.s }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n"
- "cntb x12, ALL, MUL #2\n"
+ "mov x13, #0x0\n"
"ld1rw { z18.s }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n"
- "mov x11, #0x0\n"
- "ldp x10, x9, [x19, #0x0]\n"
- "cntw x28\n"
- "ldp x27, x26, [x19, #0x10]\n"
- "sub x25, XZR, x28\n"
- "ld1w { z17.s }, p3/Z, [x14]\n"
- "mov z31.d, z17.d\n"
- "ld1w { z0.s }, p3/Z, [x14, #1, MUL VL]\n"
+ "cntw x12\n"
+ "ldp x11, x10, [x19, #0x0]\n"
+ "sub x9, XZR, x12\n"
+ "ldp x28, x27, [x19, #0x10]\n"
"whilelt p2.s, XZR, %x[n_channels]\n"
- "mov z30.d, z17.d\n"
- "ld1w { z1.s }, p3/Z, [x14, #2, MUL VL]\n"
- "cmp x28, %x[n_channels]\n"
- "mov z29.d, z17.d\n"
- "ld1w { z2.s }, p3/Z, [x14, #3, MUL VL]\n"
- "mov z28.d, z17.d\n"
- "ld1w { z3.s }, p3/Z, [x14, #4, MUL VL]\n"
- "ld1w { z4.s }, p3/Z, [x14, #5, MUL VL]\n"
- "ld1w { z5.s }, p3/Z, [x14, #6, MUL VL]\n"
- "ld1w { z6.s }, p3/Z, [x14, #7, MUL VL]\n"
- "addvl x14, x14, #16\n"
- "ldp x21, x20, [x13, #0x0]\n"
- "ld1w { z7.s }, p3/Z, [x14, #-8, MUL VL]\n"
- "ld1w { z8.s }, p3/Z, [x14, #-7, MUL VL]\n"
- "addvl x14, x14, #-6\n"
- "ld1w { z9.s }, p2/Z, [x21, x11, LSL #2]\n"
- "prfm pldl1keep, [x21, x12]\n"
- "ld1w { z10.s }, p2/Z, [x20, x11, LSL #2]\n"
- "prfm pldl1keep, [x20, x12]\n"
- "ldp x24, x23, [x13, #0x10]\n"
- "ldp x22, x21, [x13, #0x20]\n"
- "ldp x20, x19, [x13, #0x30]\n"
- "ld1w { z11.s }, p2/Z, [x24, x11, LSL #2]\n"
- "prfm pldl1keep, [x24, x12]\n"
- "ld1w { z12.s }, p2/Z, [x23, x11, LSL #2]\n"
- "prfm pldl1keep, [x23, x12]\n"
- "ld1w { z13.s }, p2/Z, [x22, x11, LSL #2]\n"
- "prfm pldl1keep, [x22, x12]\n"
- "ld1w { z14.s }, p2/Z, [x21, x11, LSL #2]\n"
- "prfm pldl1keep, [x21, x12]\n"
- "ld1w { z15.s }, p2/Z, [x20, x11, LSL #2]\n"
- "prfm pldl1keep, [x20, x12]\n"
- "ld1w { z16.s }, p2/Z, [x19, x11, LSL #2]\n"
- "prfm pldl1keep, [x19, x12]\n"
+ "ld1w { z17.s }, p3/Z, [x15]\n"
+ "cmp x12, %x[n_channels]\n"
+ "ld1w { z0.s }, p3/Z, [x15, #1, MUL VL]\n"
+ "ld1w { z1.s }, p3/Z, [x15, #2, MUL VL]\n"
+ "ld1w { z2.s }, p3/Z, [x15, #3, MUL VL]\n"
+ "ld1w { z3.s }, p3/Z, [x15, #4, MUL VL]\n"
+ "ld1w { z4.s }, p3/Z, [x15, #5, MUL VL]\n"
+ "ld1w { z5.s }, p3/Z, [x15, #6, MUL VL]\n"
+ "ld1w { z6.s }, p3/Z, [x15, #7, MUL VL]\n"
+ "addvl x15, x15, #16\n"
+ "ldp x26, x25, [x14, #0x0]\n"
+ "ld1w { z7.s }, p3/Z, [x15, #-8, MUL VL]\n"
+ "ld1w { z8.s }, p3/Z, [x15, #-7, MUL VL]\n"
+ "addvl x15, x15, #-6\n"
+ "ld1w { z9.s }, p2/Z, [x26, x13, LSL #2]\n"
+ "ld1w { z10.s }, p2/Z, [x25, x13, LSL #2]\n"
+ "ldp x24, x23, [x14, #0x10]\n"
+ "ldp x22, x21, [x14, #0x20]\n"
+ "ldp x20, x19, [x14, #0x30]\n"
+ "ld1w { z11.s }, p2/Z, [x24, x13, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x23, x13, LSL #2]\n"
+ "ld1w { z13.s }, p2/Z, [x22, x13, LSL #2]\n"
+ "ld1w { z14.s }, p2/Z, [x21, x13, LSL #2]\n"
+ "ld1w { z15.s }, p2/Z, [x20, x13, LSL #2]\n"
+ "ld1w { z16.s }, p2/Z, [x19, x13, LSL #2]\n"
"bge 2f\n"
"1:" // Channel loop
- "fmla z31.s, p3/M, z8.s, z9.s\n"
- "ldr x22, [x13, #0x40]\n"
- "whilelt p1.s, x28, %x[n_channels]\n"
- "fmla z30.s, p3/M, z6.s, z9.s\n"
- "ldr x21, [x13, #0x48]\n"
- "incw x25\n"
- "fmla z29.s, p3/M, z2.s, z9.s\n"
- "ldr x20, [x13, #0x50]\n"
+ "movprfx z31, z17\n fmla z31.s, p3/M, z8.s, z9.s\n"
+ "ldr x26, [x14, #0x40]\n"
+ "whilelt p1.s, x12, %x[n_channels]\n"
+ "movprfx z30, z17\n fmla z30.s, p3/M, z6.s, z9.s\n"
+ "ldr x25, [x14, #0x48]\n"
+ "incw x9\n"
+ "movprfx z29, z17\n fmla z29.s, p3/M, z2.s, z9.s\n"
+ "ldr x24, [x14, #0x50]\n"
"mov p0.b, p2.b\n"
- "fmla z28.s, p3/M, z0.s, z9.s\n"
- "prfm pldl1keep, [x22, x12]\n"
- "ldr x19, [x13, #0x58]\n"
+ "movprfx z28, z17\n fmla z28.s, p3/M, z0.s, z9.s\n"
+ "ldr x23, [x14, #0x58]\n"
+ "ldr x22, [x14, #0x60]\n"
"fmla z31.s, p3/M, z0.s, z10.s\n"
- "prfm pldl1keep, [x21, x12]\n"
+ "ldr x21, [x14, #0x68]\n"
"fmla z30.s, p3/M, z1.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x21, x11, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x25, x13, LSL #2]\n"
"fmla z31.s, p3/M, z1.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x22, x11, LSL #2]\n"
- "prfm pldl1keep, [x20, x12]\n"
+ "ld1w { z11.s }, p2/Z, [x26, x13, LSL #2]\n"
+ "ldr x20, [x14, #0x70]\n"
"fmla z30.s, p3/M, z2.s, z13.s\n"
- "ld1w { z13.s }, p2/Z, [x20, x11, LSL #2]\n"
+ "ld1w { z13.s }, p2/Z, [x24, x13, LSL #2]\n"
"fmla z31.s, p3/M, z3.s, z14.s\n"
- "ld1w { z14.s }, p2/Z, [x19, x11, LSL #2]\n"
- "prfm pldl1keep, [x19, x12]\n"
+ "ld1w { z14.s }, p2/Z, [x23, x13, LSL #2]\n"
+ "ldr x19, [x14, #0x78]\n"
"fmla z30.s, p3/M, z0.s, z16.s\n"
- "ldr x21, [x13, #0x60]\n"
+ "ldr x26, [x14, #0x80]\n"
"fmla z29.s, p3/M, z3.s, z14.s\n"
- "ldr x20, [x13, #0x68]\n"
- "ldr x19, [x13, #0x70]\n"
+ "ldr x25, [x14, #0x88]\n"
+ "ldr x24, [x14, #0x90]\n"
"fmla z31.s, p3/M, z4.s, z15.s\n"
- "ld1w { z15.s }, p2/Z, [x21, x11, LSL #2]\n"
+ "ld1w { z15.s }, p2/Z, [x22, x13, LSL #2]\n"
"fmla z30.s, p3/M, z4.s, z11.s\n"
- "prfm pldl1keep, [x21, x12]\n"
+ "ld1w { z11.s }, p2/Z, [x21, x13, LSL #2]\n"
"fmla z29.s, p3/M, z0.s, z15.s\n"
- "ld1w { z11.s }, p2/Z, [x20, x11, LSL #2]\n"
- "prfm pldl1keep, [x20, x12]\n"
+ "ld1w { z14.s }, p2/Z, [x25, x13, LSL #2]\n"
+ "ldr x23, [x14, #0x98]\n"
"fmla z31.s, p3/M, z2.s, z16.s\n"
- "ld1w { z16.s }, p2/Z, [x19, x11, LSL #2]\n"
+ "ld1w { z16.s }, p2/Z, [x20, x13, LSL #2]\n"
"fmla z30.s, p3/M, z5.s, z12.s\n"
- "prfm pldl1keep, [x19, x12]\n"
+ "ld1w { z12.s }, p2/Z, [x26, x13, LSL #2]\n"
"fmla z29.s, p3/M, z4.s, z11.s\n"
- "ldr x19, [x13, #0x78]\n"
- "ldr x21, [x13, #0x80]\n"
+ "ld1w { z11.s }, p2/Z, [x23, x13, LSL #2]\n"
+ "ldr x22, [x14, #0xa0]\n"
"fmla z31.s, p3/M, z5.s, z13.s\n"
- "ldr x20, [x13, #0x88]\n"
+ "ldr x21, [x14, #0xa8]\n"
"fmla z30.s, p3/M, z3.s, z13.s\n"
- "ldr x24, [x13, #0x90]\n"
+ "ld1w { z13.s }, p2/Z, [x19, x13, LSL #2]\n"
"fmla z29.s, p3/M, z1.s, z16.s\n"
- "ld1w { z13.s }, p2/Z, [x19, x11, LSL #2]\n"
- "prfm pldl1keep, [x19, x12]\n"
+ "ldr x20, [x14, #0xb0]\n"
+ "ldr x19, [x14, #0xb8]\n"
"fmla z31.s, p3/M, z6.s, z15.s\n"
- "ld1w { z12.s }, p2/Z, [x21, x11, LSL #2]\n"
"fmla z28.s, p3/M, z4.s, z13.s\n"
- "prfm pldl1keep, [x21, x12]\n"
- "ld1w { z14.s }, p2/Z, [x20, x11, LSL #2]\n"
+ "ld1w { z15.s }, p2/Z, [x24, x13, LSL #2]\n"
"fmla z30.s, p3/M, z7.s, z12.s\n"
- "prfm pldl1keep, [x20, x12]\n"
+ "ld1w { z13.s }, p2/Z, [x22, x13, LSL #2]\n"
+ "ldr x26, [x14, #0xc0]\n"
"fmla z31.s, p3/M, z7.s, z16.s\n"
- "ld1w { z15.s }, p2/Z, [x24, x11, LSL #2]\n"
+ "ld1w { z16.s }, p2/Z, [x21, x13, LSL #2]\n"
"fmla z28.s, p3/M, z1.s, z12.s\n"
- "prfm pldl1keep, [x24, x12]\n"
- "ldr x23, [x13, #0x98]\n"
+ "ld1w { z17.s }, p3/Z, [x15]\n"
"fmla z29.s, p3/M, z6.s, z15.s\n"
- "ldr x22, [x13, #0xa0]\n"
- "fmax z31.s, p3/M, z31.s, z19.s\n"
- "ldr x21, [x13, #0xa8]\n"
- "fmla z28.s, p3/M, z5.s, z14.s\n"
- "ld1w { z11.s }, p2/Z, [x23, x11, LSL #2]\n"
- "prfm pldl1keep, [x23, x12]\n"
- "fmin z31.s, p3/M, z31.s, z18.s\n"
- "ld1w { z13.s }, p2/Z, [x22, x11, LSL #2]\n"
- "prfm pldl1keep, [x22, x12]\n"
+ "ld1w { z15.s }, p2/Z, [x19, x13, LSL #2]\n"
"fmla z30.s, p3/M, z8.s, z11.s\n"
- "ld1w { z16.s }, p2/Z, [x21, x11, LSL #2]\n"
- "fmla z28.s, p3/M, z2.s, z11.s\n"
- "prfm pldl1keep, [x21, x12]\n"
+ "ld1w { z0.s }, p3/Z, [x15, #1, MUL VL]\n"
+ "ld1w { z1.s }, p3/Z, [x15, #2, MUL VL]\n"
+ "fmla z28.s, p3/M, z5.s, z14.s\n"
+ "fmax z31.s, p3/M, z31.s, z19.s\n"
+ "ld1w { z14.s }, p2/Z, [x20, x13, LSL #2]\n"
"fmla z29.s, p3/M, z7.s, z13.s\n"
- "ldr x20, [x13, #0xb0]\n"
+ "ld1w { z4.s }, p3/Z, [x15, #5, MUL VL]\n"
"fmax z30.s, p3/M, z30.s, z19.s\n"
- "ldr x19, [x13, #0xb8]\n"
- "ldr x22, [x13, #0xc0]\n"
- "fmla z28.s, p3/M, z3.s, z16.s\n"
- "ld1w { z14.s }, p2/Z, [x20, x11, LSL #2]\n"
+ "fmla z28.s, p3/M, z2.s, z11.s\n"
+ "ld1w { z11.s }, p2/Z, [x26, x13, LSL #2]\n"
+ "incw x13\n"
"fmla z29.s, p3/M, z5.s, z16.s\n"
- "prfm pldl1keep, [x20, x12]\n"
+ "ldp x26, x25, [x14, #0x0]\n"
+ "whilelt p2.s, x13, %x[n_channels]\n"
+ "fmin z31.s, p3/M, z31.s, z18.s\n"
+ "ldp x24, x23, [x14, #0x10]\n"
"fmin z30.s, p3/M, z30.s, z18.s\n"
- "ld1w { z15.s }, p2/Z, [x19, x11, LSL #2]\n"
- "prfm pldl1keep, [x19, x12]\n"
- "fmla z28.s, p3/M, z7.s, z14.s\n"
- "ld1w { z11.s }, p2/Z, [x22, x11, LSL #2]\n"
+ "ldp x22, x21, [x14, #0x20]\n"
+ "ldp x20, x19, [x14, #0x30]\n"
+ "fmla z28.s, p3/M, z3.s, z16.s\n"
+ "ld1w { z9.s }, p1/Z, [x26, x12, LSL #2]\n"
"fmla z29.s, p3/M, z8.s, z15.s\n"
- "prfm pldl1keep, [x22, x12]\n"
- "incw x11\n"
- "fmla z28.s, p3/M, z6.s, z15.s\n"
- "ldp x21, x20, [x13, #0x0]\n"
- "whilelt p2.s, x11, %x[n_channels]\n"
+ "ld1w { z10.s }, p1/Z, [x25, x12, LSL #2]\n"
+ "fmla z28.s, p3/M, z7.s, z14.s\n"
+ "ld1w { z12.s }, p1/Z, [x23, x12, LSL #2]\n"
+ "ld1w { z13.s }, p1/Z, [x22, x12, LSL #2]\n"
"fmax z29.s, p3/M, z29.s, z19.s\n"
- "ldp x24, x23, [x13, #0x10]\n"
- "addvl x12, x12, #1\n"
- "fmla z28.s, p3/M, z8.s, z11.s\n"
- "ld1w { z9.s }, p1/Z, [x21, x28, LSL #2]\n"
- "prfm pldl1keep, [x21, x12]\n"
+ "ld1w { z14.s }, p1/Z, [x21, x12, LSL #2]\n"
+ "fmla z28.s, p3/M, z6.s, z15.s\n"
+ "ld1w { z15.s }, p1/Z, [x20, x12, LSL #2]\n"
+ "ld1w { z16.s }, p1/Z, [x19, x12, LSL #2]\n"
"fmin z29.s, p3/M, z29.s, z18.s\n"
- "ld1w { z10.s }, p1/Z, [x20, x28, LSL #2]\n"
- "prfm pldl1keep, [x20, x12]\n"
+ "st1w { z31.s }, p0, [x11, x9, LSL #2]\n"
+ "fmla z28.s, p3/M, z8.s, z11.s\n"
+ "ld1w { z11.s }, p1/Z, [x24, x12, LSL #2]\n"
+ "incw x12\n"
"fmax z28.s, p3/M, z28.s, z19.s\n"
- "ld1w { z11.s }, p1/Z, [x24, x28, LSL #2]\n"
- "prfm pldl1keep, [x24, x12]\n"
+ "st1w { z30.s }, p0, [x10, x9, LSL #2]\n"
+ "cmp x12, %x[n_channels]\n"
"fmin z28.s, p3/M, z28.s, z18.s\n"
- "ld1w { z12.s }, p1/Z, [x23, x28, LSL #2]\n"
- "prfm pldl1keep, [x23, x12]\n"
- "ldp x22, x21, [x13, #0x20]\n"
- "ldp x20, x19, [x13, #0x30]\n"
- "st1w { z31.s }, p0, [x10, x25, LSL #2]\n"
- "ld1w { z13.s }, p1/Z, [x22, x28, LSL #2]\n"
- "prfm pldl1keep, [x22, x12]\n"
- "ld1w { z14.s }, p1/Z, [x21, x28, LSL #2]\n"
- "prfm pldl1keep, [x21, x12]\n"
- "ld1w { z15.s }, p1/Z, [x20, x28, LSL #2]\n"
- "prfm pldl1keep, [x20, x12]\n"
- "ld1w { z16.s }, p1/Z, [x19, x28, LSL #2]\n"
- "incw x28\n"
- "prfm pldl1keep, [x19, x12]\n"
- "cmp x28, %x[n_channels]\n"
- "st1w { z30.s }, p0, [x9, x25, LSL #2]\n"
- "st1w { z29.s }, p0, [x27, x25, LSL #2]\n"
- "st1w { z28.s }, p0, [x26, x25, LSL #2]\n"
- "ld1w { z17.s }, p3/Z, [x14]\n"
- "mov z31.d, z17.d\n"
- "ld1w { z0.s }, p3/Z, [x14, #1, MUL VL]\n"
- "mov z30.d, z17.d\n"
- "ld1w { z1.s }, p3/Z, [x14, #2, MUL VL]\n"
- "mov z29.d, z17.d\n"
- "ld1w { z2.s }, p3/Z, [x14, #3, MUL VL]\n"
- "mov z28.d, z17.d\n"
- "ld1w { z3.s }, p3/Z, [x14, #4, MUL VL]\n"
- "ld1w { z4.s }, p3/Z, [x14, #5, MUL VL]\n"
- "ld1w { z5.s }, p3/Z, [x14, #6, MUL VL]\n"
- "ld1w { z6.s }, p3/Z, [x14, #7, MUL VL]\n"
- "addvl x14, x14, #16\n"
- "ld1w { z7.s }, p3/Z, [x14, #-8, MUL VL]\n"
- "ld1w { z8.s }, p3/Z, [x14, #-7, MUL VL]\n"
- "addvl x14, x14, #-6\n"
+ "st1w { z29.s }, p0, [x28, x9, LSL #2]\n"
+ "ld1w { z2.s }, p3/Z, [x15, #3, MUL VL]\n"
+ "ld1w { z3.s }, p3/Z, [x15, #4, MUL VL]\n"
+ "ld1w { z5.s }, p3/Z, [x15, #6, MUL VL]\n"
+ "ld1w { z6.s }, p3/Z, [x15, #7, MUL VL]\n"
+ "addvl x15, x15, #16\n"
+ "st1w { z28.s }, p0, [x27, x9, LSL #2]\n"
+ "ld1w { z7.s }, p3/Z, [x15, #-8, MUL VL]\n"
+ "ld1w { z8.s }, p3/Z, [x15, #-7, MUL VL]\n"
+ "addvl x15, x15, #-6\n"
"blt 1b\n"
"2:" // Channel tail
- "fmla z31.s, p3/M, z8.s, z9.s\n"
- "ldr x22, [x13, #0x40]\n"
- "incw x25\n"
- "fmla z30.s, p3/M, z6.s, z9.s\n"
- "ldr x21, [x13, #0x48]\n"
+ "movprfx z31, z17\n fmla z31.s, p3/M, z8.s, z9.s\n"
+ "ldr x26, [x14, #0x40]\n"
+ "incw x9\n"
+ "movprfx z30, z17\n fmla z30.s, p3/M, z6.s, z9.s\n"
+ "ldr x25, [x14, #0x48]\n"
"mov p0.b, p2.b\n"
- "fmla z29.s, p3/M, z2.s, z9.s\n"
- "ldr x20, [x13, #0x50]\n"
- "fmla z28.s, p3/M, z0.s, z9.s\n"
- "prfm pldl1keep, [x22, x12]\n"
- "ldr x19, [x13, #0x58]\n"
+ "movprfx z29, z17\n fmla z29.s, p3/M, z2.s, z9.s\n"
+ "ldr x24, [x14, #0x50]\n"
+ "movprfx z28, z17\n fmla z28.s, p3/M, z0.s, z9.s\n"
+ "ldr x23, [x14, #0x58]\n"
+ "ldr x22, [x14, #0x60]\n"
"fmla z31.s, p3/M, z0.s, z10.s\n"
- "prfm pldl1keep, [x21, x12]\n"
+ "ldr x21, [x14, #0x68]\n"
"fmla z30.s, p3/M, z1.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x21, x11, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x25, x13, LSL #2]\n"
"fmla z31.s, p3/M, z1.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x22, x11, LSL #2]\n"
- "prfm pldl1keep, [x20, x12]\n"
+ "ld1w { z11.s }, p2/Z, [x26, x13, LSL #2]\n"
+ "ldr x20, [x14, #0x70]\n"
"fmla z30.s, p3/M, z2.s, z13.s\n"
- "ld1w { z13.s }, p2/Z, [x20, x11, LSL #2]\n"
+ "ld1w { z13.s }, p2/Z, [x24, x13, LSL #2]\n"
"fmla z31.s, p3/M, z3.s, z14.s\n"
- "ld1w { z14.s }, p2/Z, [x19, x11, LSL #2]\n"
- "prfm pldl1keep, [x19, x12]\n"
+ "ld1w { z14.s }, p2/Z, [x23, x13, LSL #2]\n"
+ "ldr x19, [x14, #0x78]\n"
"fmla z30.s, p3/M, z0.s, z16.s\n"
- "ldr x21, [x13, #0x60]\n"
+ "ldr x26, [x14, #0x80]\n"
"fmla z29.s, p3/M, z3.s, z14.s\n"
- "ldr x20, [x13, #0x68]\n"
- "ldr x19, [x13, #0x70]\n"
+ "ldr x25, [x14, #0x88]\n"
+ "ldr x24, [x14, #0x90]\n"
"fmla z31.s, p3/M, z4.s, z15.s\n"
- "ld1w { z15.s }, p2/Z, [x21, x11, LSL #2]\n"
+ "ld1w { z15.s }, p2/Z, [x22, x13, LSL #2]\n"
"fmla z30.s, p3/M, z4.s, z11.s\n"
- "prfm pldl1keep, [x21, x12]\n"
+ "ld1w { z11.s }, p2/Z, [x21, x13, LSL #2]\n"
"fmla z29.s, p3/M, z0.s, z15.s\n"
- "ld1w { z11.s }, p2/Z, [x20, x11, LSL #2]\n"
- "prfm pldl1keep, [x20, x12]\n"
+ "ld1w { z14.s }, p2/Z, [x25, x13, LSL #2]\n"
+ "ldr x23, [x14, #0x98]\n"
"fmla z31.s, p3/M, z2.s, z16.s\n"
- "ld1w { z16.s }, p2/Z, [x19, x11, LSL #2]\n"
+ "ld1w { z16.s }, p2/Z, [x20, x13, LSL #2]\n"
"fmla z30.s, p3/M, z5.s, z12.s\n"
- "prfm pldl1keep, [x19, x12]\n"
+ "ld1w { z12.s }, p2/Z, [x26, x13, LSL #2]\n"
"fmla z29.s, p3/M, z4.s, z11.s\n"
- "ldr x19, [x13, #0x78]\n"
- "ldr x21, [x13, #0x80]\n"
+ "ld1w { z11.s }, p2/Z, [x23, x13, LSL #2]\n"
+ "ldr x22, [x14, #0xa0]\n"
"fmla z31.s, p3/M, z5.s, z13.s\n"
- "ldr x20, [x13, #0x88]\n"
+ "ldr x21, [x14, #0xa8]\n"
"fmla z30.s, p3/M, z3.s, z13.s\n"
- "ldr x24, [x13, #0x90]\n"
+ "ld1w { z13.s }, p2/Z, [x19, x13, LSL #2]\n"
"fmla z29.s, p3/M, z1.s, z16.s\n"
- "ld1w { z13.s }, p2/Z, [x19, x11, LSL #2]\n"
- "prfm pldl1keep, [x19, x12]\n"
+ "ldr x20, [x14, #0xb0]\n"
+ "ldr x19, [x14, #0xb8]\n"
"fmla z31.s, p3/M, z6.s, z15.s\n"
- "ld1w { z12.s }, p2/Z, [x21, x11, LSL #2]\n"
"fmla z28.s, p3/M, z4.s, z13.s\n"
- "prfm pldl1keep, [x21, x12]\n"
- "ld1w { z14.s }, p2/Z, [x20, x11, LSL #2]\n"
+ "ld1w { z15.s }, p2/Z, [x24, x13, LSL #2]\n"
"fmla z30.s, p3/M, z7.s, z12.s\n"
- "prfm pldl1keep, [x20, x12]\n"
+ "ld1w { z13.s }, p2/Z, [x22, x13, LSL #2]\n"
+ "ldr x26, [x14, #0xc0]\n"
"fmla z31.s, p3/M, z7.s, z16.s\n"
- "ld1w { z15.s }, p2/Z, [x24, x11, LSL #2]\n"
+ "ld1w { z16.s }, p2/Z, [x21, x13, LSL #2]\n"
"fmla z28.s, p3/M, z1.s, z12.s\n"
- "prfm pldl1keep, [x24, x12]\n"
- "ldr x23, [x13, #0x98]\n"
"fmla z29.s, p3/M, z6.s, z15.s\n"
- "ldr x22, [x13, #0xa0]\n"
- "fmax z31.s, p3/M, z31.s, z19.s\n"
- "ldr x21, [x13, #0xa8]\n"
- "fmla z28.s, p3/M, z5.s, z14.s\n"
- "ld1w { z11.s }, p2/Z, [x23, x11, LSL #2]\n"
- "prfm pldl1keep, [x23, x12]\n"
- "fmin z31.s, p3/M, z31.s, z18.s\n"
- "ld1w { z13.s }, p2/Z, [x22, x11, LSL #2]\n"
- "prfm pldl1keep, [x22, x12]\n"
+ "ld1w { z15.s }, p2/Z, [x19, x13, LSL #2]\n"
"fmla z30.s, p3/M, z8.s, z11.s\n"
- "ld1w { z16.s }, p2/Z, [x21, x11, LSL #2]\n"
- "fmla z28.s, p3/M, z2.s, z11.s\n"
- "prfm pldl1keep, [x21, x12]\n"
+ "fmla z28.s, p3/M, z5.s, z14.s\n"
+ "ld1w { z14.s }, p2/Z, [x20, x13, LSL #2]\n"
+ "fmax z31.s, p3/M, z31.s, z19.s\n"
"fmla z29.s, p3/M, z7.s, z13.s\n"
- "ldr x20, [x13, #0xb0]\n"
"fmax z30.s, p3/M, z30.s, z19.s\n"
- "ldr x19, [x13, #0xb8]\n"
- "ldr x22, [x13, #0xc0]\n"
- "fmla z28.s, p3/M, z3.s, z16.s\n"
- "ld1w { z14.s }, p2/Z, [x20, x11, LSL #2]\n"
+ "fmla z28.s, p3/M, z2.s, z11.s\n"
+ "ld1w { z11.s }, p2/Z, [x26, x13, LSL #2]\n"
+ "fmin z31.s, p3/M, z31.s, z18.s\n"
+ "st1w { z31.s }, p0, [x11, x9, LSL #2]\n"
"fmla z29.s, p3/M, z5.s, z16.s\n"
- "prfm pldl1keep, [x20, x12]\n"
+ "fmla z28.s, p3/M, z3.s, z16.s\n"
"fmin z30.s, p3/M, z30.s, z18.s\n"
- "ld1w { z15.s }, p2/Z, [x19, x11, LSL #2]\n"
- "prfm pldl1keep, [x19, x12]\n"
+ "st1w { z30.s }, p0, [x10, x9, LSL #2]\n"
"fmla z28.s, p3/M, z7.s, z14.s\n"
- "ld1w { z11.s }, p2/Z, [x22, x11, LSL #2]\n"
"fmla z29.s, p3/M, z8.s, z15.s\n"
- "prfm pldl1keep, [x22, x12]\n"
- "st1w { z31.s }, p0, [x10, x25, LSL #2]\n"
"fmla z28.s, p3/M, z6.s, z15.s\n"
- "st1w { z30.s }, p0, [x9, x25, LSL #2]\n"
"fmax z29.s, p3/M, z29.s, z19.s\n"
"fmla z28.s, p3/M, z8.s, z11.s\n"
"fmin z29.s, p3/M, z29.s, z18.s\n"
- "st1w { z29.s }, p0, [x27, x25, LSL #2]\n"
+ "st1w { z29.s }, p0, [x28, x9, LSL #2]\n"
"fmax z28.s, p3/M, z28.s, z19.s\n"
"fmin z28.s, p3/M, z28.s, z18.s\n"
- "st1w { z28.s }, p0, [x26, x25, LSL #2]\n"
+ "st1w { z28.s }, p0, [x27, x9, LSL #2]\n"
:
: [n_channels] "r" ((unsigned long) n_channels), [offsetof_Args_inptrs] "I" (offsetof(Args, inptrs)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_outptrs] "I" (offsetof(Args, outptrs)), [offsetof_args_params] "I" (offsetof(Args, params)), [params_struct] "r" (¶ms_struct)
- : "cc", "memory", "p0", "p1", "p2", "p3", "x9", "x10", "x11", "x12", "x13", "x14", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z28", "z29", "z30", "z31"
+ : "cc", "memory", "p0", "p1", "p2", "p3", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z28", "z29", "z30", "z31"
);
}
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(ARM_COMPUTE_ENABLE_SVE)
+#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp
index 6f1f187..751874f 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp
@@ -28,7 +28,7 @@
#pragma once
-#if defined(ARM_COMPUTE_ENABLE_SVE)
+#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE)
namespace arm_conv {
namespace depthwise {
@@ -36,15 +36,17 @@
void sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_indirect_impl(const float *const *const, float *const *const, const void *, unsigned int, const float, const float);
void sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float);
-struct sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst
+class sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst : public IDepthwiseDepthfirstStrategy
{
- typedef float bias_type;
- typedef float input_type;
- typedef float weight_type;
- typedef float return_type;
-
+ private:
typedef void (*indirect_kern_type)(const float *const *const, float *const *const, const void *, unsigned int, const float, const float);
+ indirect_kern_type m_indirect_kernel = sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_indirect_impl;
+
typedef void (*direct_kern_type)(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float);
+ direct_kern_type m_direct_kernel = sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl;
+
+ public:
+ typedef float return_type;
constexpr static arm_gemm::VLType vl_type = arm_gemm::VLType::SVE;
@@ -60,13 +62,60 @@
constexpr static unsigned int input_rows = 6;
constexpr static unsigned int input_cols = 6;
- indirect_kern_type indirect_kernel = sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_indirect_impl;
- direct_kern_type direct_kernel = sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl;
-
sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst(const CPUInfo *) {}
+
+ arm_gemm::VLType get_vl_type(void) const override { return vl_type; }
+
+ unsigned int get_kernel_rows(void) const override { return kernel_rows; }
+ unsigned int get_kernel_cols(void) const override { return kernel_cols; }
+
+ unsigned int get_stride_rows(void) const override { return stride_rows; }
+ unsigned int get_stride_cols(void) const override { return stride_cols; }
+
+ unsigned int get_output_rows(void) const override { return output_rows; }
+ unsigned int get_output_cols(void) const override { return output_cols; }
+
+ unsigned int get_input_rows(void) const override { return input_rows; }
+ unsigned int get_input_cols(void) const override { return input_cols; }
+
+ void indirect_kernel(
+ const void *const *const input_ptrs,
+ void *const *const outptrs,
+ const void *params,
+ unsigned int n_channels,
+ const void *activation_min,
+ const void *activation_max
+ ) const override
+ {
+ m_indirect_kernel(
+ reinterpret_cast<const float *const *>(input_ptrs),
+ reinterpret_cast<float *const *>(outptrs),
+ params, n_channels,
+ *static_cast<const float *>(activation_min),
+ *static_cast<const float *>(activation_max)
+ );
+ }
+
+ void direct_kernel(
+ const unsigned int n_tile_rows, const unsigned int n_tile_cols,
+ const void *inptr, int64_t ld_input_row, int64_t ld_input_col,
+ void *outptr, int64_t ld_output_row, int64_t ld_output_col,
+ const void *params, unsigned int n_channels,
+ const void *activation_min, const void *activation_max
+ ) const override
+ {
+ m_direct_kernel(
+ n_tile_rows, n_tile_cols,
+ static_cast<const float *>(inptr), ld_input_row, ld_input_col,
+ static_cast<float *>(outptr), ld_output_row, ld_output_col,
+ params, n_channels,
+ *static_cast<const float *>(activation_min),
+ *static_cast<const float *>(activation_max)
+ );
+ }
};
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(ARM_COMPUTE_ENABLE_SVE)
+#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_direct.cpp
index edafe82..ea8bbbd 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_direct.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_direct.cpp
@@ -25,7 +25,7 @@
#include <cstddef>
#include <cstdint>
-#if defined(ARM_COMPUTE_ENABLE_SVE)
+#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE)
namespace arm_conv {
namespace depthwise {
@@ -114,22 +114,18 @@
"ld1rw { z17.s }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n"
"add x10, x20, x22, LSL #2\n"
"ld1w { z16.s }, p3/Z, [x8]\n"
- "mov z31.d, z16.d\n"
- "ld1w { z0.s }, p3/Z, [x8, #1, MUL VL]\n"
"add x9, x10, x22, LSL #2\n"
- "mov z30.d, z16.d\n"
- "ld1w { z1.s }, p3/Z, [x8, #2, MUL VL]\n"
+ "ld1w { z0.s }, p3/Z, [x8, #1, MUL VL]\n"
"add x28, x9, x22, LSL #2\n"
- "mov z29.d, z16.d\n"
- "ld1w { z2.s }, p3/Z, [x8, #3, MUL VL]\n"
+ "ld1w { z1.s }, p3/Z, [x8, #2, MUL VL]\n"
"add x27, x28, x22, LSL #2\n"
- "mov z28.d, z16.d\n"
- "ld1w { z3.s }, p3/Z, [x8, #4, MUL VL]\n"
+ "ld1w { z2.s }, p3/Z, [x8, #3, MUL VL]\n"
"add x26, x15, x15\n"
- "ld1w { z4.s }, p3/Z, [x8, #5, MUL VL]\n"
+ "ld1w { z3.s }, p3/Z, [x8, #4, MUL VL]\n"
"add x25, x26, x15\n"
- "mul x19, x5, x21\n" // offset = tile_i * ld_output_row
+ "ld1w { z4.s }, p3/Z, [x8, #5, MUL VL]\n"
"add x24, x25, x15\n"
+ "mul x19, x5, x21\n" // offset = tile_i * ld_output_row
"add x23, x24, x15\n"
"madd x19, x6, x12, x19\n" // offset += tile_j * ld_output_col
"mul x19, x19, x7\n" // offset *= output_tile_size
@@ -150,14 +146,14 @@
"ld1w { z14.s }, p2/Z, [x10]\n"
"bge 3f\n"
"2:" // Tile loop: Channel loop
- "fmla z31.s, p3/M, z0.s, z5.s\n"
+ "movprfx z31, z16\n fmla z31.s, p3/M, z0.s, z5.s\n"
"ld1w { z5.s }, p2/Z, [x20, x25, LSL #2]\n"
"whilelt p1.s, x16, %x[n_channels]\n"
- "fmla z30.s, p3/M, z0.s, z6.s\n"
+ "movprfx z30, z16\n fmla z30.s, p3/M, z0.s, z6.s\n"
"incw x14\n"
- "fmla z29.s, p3/M, z0.s, z7.s\n"
+ "movprfx z29, z16\n fmla z29.s, p3/M, z0.s, z7.s\n"
"mov p0.b, p2.b\n"
- "fmla z28.s, p3/M, z0.s, z8.s\n"
+ "movprfx z28, z16\n fmla z28.s, p3/M, z0.s, z8.s\n"
"ld1w { z0.s }, p3/Z, [x8]\n"
"incw x17\n"
"fmla z31.s, p3/M, z1.s, z6.s\n"
@@ -334,29 +330,25 @@
"fmax z28.s, p3/M, z28.s, z18.s\n"
"fmin z31.s, p3/M, z31.s, z17.s\n"
"st1w { z31.s }, p0, [x11]\n"
- "mov z31.d, z16.d\n"
"fmin z30.s, p3/M, z30.s, z17.s\n"
- "st1w { z30.s }, p0, [x11, x12, LSL #2]\n"
- "mov z30.d, z16.d\n"
- "addvl x11, x11, #1\n"
"fmin z29.s, p3/M, z29.s, z17.s\n"
- "st1w { z29.s }, p0, [x22]\n"
- "mov z29.d, z16.d\n"
+ "st1w { z30.s }, p0, [x11, x12, LSL #2]\n"
"fmin z28.s, p3/M, z28.s, z17.s\n"
+ "addvl x11, x11, #1\n"
+ "st1w { z29.s }, p0, [x22]\n"
"st1w { z28.s }, p0, [x22, x12, LSL #2]\n"
- "mov z28.d, z16.d\n"
"addvl x22, x22, #1\n"
"blt 2b\n"
"3:" // Tile loop: Channel tail
- "fmla z31.s, p3/M, z0.s, z5.s\n"
+ "movprfx z31, z16\n fmla z31.s, p3/M, z0.s, z5.s\n"
"ld1w { z5.s }, p2/Z, [x20, x25, LSL #2]\n"
"mov p0.b, p2.b\n"
- "fmla z30.s, p3/M, z0.s, z6.s\n"
+ "movprfx z30, z16\n fmla z30.s, p3/M, z0.s, z6.s\n"
"ldr x5, [%x[params_struct], %[offsetof_args_tile_i]]\n"
"add x21, x5, #0x1\n"
- "fmla z29.s, p3/M, z0.s, z7.s\n"
+ "movprfx z29, z16\n fmla z29.s, p3/M, z0.s, z7.s\n"
"ldr x6, [%x[params_struct], %[offsetof_args_tile_j]]\n"
- "fmla z28.s, p3/M, z0.s, z8.s\n"
+ "movprfx z28, z16\n fmla z28.s, p3/M, z0.s, z8.s\n"
"ld1w { z0.s }, p3/Z, [x8]\n"
"add x6, x6, #0x1\n"
"fmla z31.s, p3/M, z1.s, z6.s\n"
@@ -528,4 +520,4 @@
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(ARM_COMPUTE_ENABLE_SVE)
+#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_indirect.cpp
index 1bfe7eb..0702707 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_indirect.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_indirect.cpp
@@ -25,7 +25,7 @@
#include <cstddef>
#include <cstdint>
-#if defined(ARM_COMPUTE_ENABLE_SVE)
+#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE)
namespace arm_conv {
namespace depthwise {
@@ -100,511 +100,429 @@
__asm__ __volatile__(
"ldr x19, [%x[params_struct], %[offsetof_args_outptrs]]\n"
"ptrue p3.b\n"
- "ldr x16, [%x[params_struct], %[offsetof_args_params]]\n"
- "add x15, %x[params_struct], %[offsetof_Args_inptrs]\n"
+ "ldr x15, [%x[params_struct], %[offsetof_args_params]]\n"
+ "add x14, %x[params_struct], %[offsetof_Args_inptrs]\n"
"ld1rw { z18.s }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n"
- "cntb x14, ALL, MUL #2\n"
- "ld1rw { z17.s }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n"
"mov x13, #0x0\n"
- "ldp x12, x11, [x19, #0x0]\n"
- "cntw x10\n"
- "ldp x9, x28, [x19, #0x10]\n"
- "sub x27, XZR, x10\n"
- "ld1w { z16.s }, p3/Z, [x16]\n"
- "mov z31.d, z16.d\n"
- "ld1w { z0.s }, p3/Z, [x16, #1, MUL VL]\n"
+ "ld1rw { z17.s }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n"
+ "cntw x12\n"
+ "ldp x11, x10, [x19, #0x0]\n"
+ "sub x9, XZR, x12\n"
+ "ldp x28, x27, [x19, #0x10]\n"
"whilelt p2.s, XZR, %x[n_channels]\n"
- "mov z30.d, z16.d\n"
- "ld1w { z1.s }, p3/Z, [x16, #2, MUL VL]\n"
- "cmp x10, %x[n_channels]\n"
- "mov z29.d, z16.d\n"
- "ld1w { z2.s }, p3/Z, [x16, #3, MUL VL]\n"
- "mov z28.d, z16.d\n"
- "ld1w { z3.s }, p3/Z, [x16, #4, MUL VL]\n"
- "ld1w { z4.s }, p3/Z, [x16, #5, MUL VL]\n"
- "addvl x16, x16, #6\n"
- "ldp x26, x25, [x15, #0x0]\n"
- "ldp x24, x23, [x15, #0x10]\n"
- "ldp x20, x19, [x15, #0x20]\n"
+ "ld1w { z16.s }, p3/Z, [x15]\n"
+ "cmp x12, %x[n_channels]\n"
+ "ld1w { z0.s }, p3/Z, [x15, #1, MUL VL]\n"
+ "ld1w { z1.s }, p3/Z, [x15, #2, MUL VL]\n"
+ "ld1w { z2.s }, p3/Z, [x15, #3, MUL VL]\n"
+ "ld1w { z3.s }, p3/Z, [x15, #4, MUL VL]\n"
+ "ld1w { z4.s }, p3/Z, [x15, #5, MUL VL]\n"
+ "addvl x15, x15, #6\n"
+ "ldp x26, x25, [x14, #0x0]\n"
+ "ldp x24, x23, [x14, #0x10]\n"
+ "ldp x22, x21, [x14, #0x20]\n"
"ld1w { z5.s }, p2/Z, [x26, x13, LSL #2]\n"
- "prfm pldl1keep, [x26, x14]\n"
"ld1w { z6.s }, p2/Z, [x25, x13, LSL #2]\n"
- "prfm pldl1keep, [x25, x14]\n"
"ld1w { z7.s }, p2/Z, [x24, x13, LSL #2]\n"
- "prfm pldl1keep, [x24, x14]\n"
"ld1w { z8.s }, p2/Z, [x23, x13, LSL #2]\n"
- "prfm pldl1keep, [x23, x14]\n"
- "ld1w { z9.s }, p2/Z, [x20, x13, LSL #2]\n"
- "prfm pldl1keep, [x20, x14]\n"
- "ld1w { z13.s }, p2/Z, [x19, x13, LSL #2]\n"
- "prfm pldl1keep, [x19, x14]\n"
- "ldp x22, x21, [x15, #0x30]\n"
- "ldp x20, x19, [x15, #0x40]\n"
- "ld1w { z11.s }, p2/Z, [x22, x13, LSL #2]\n"
- "prfm pldl1keep, [x22, x14]\n"
- "ld1w { z12.s }, p2/Z, [x21, x13, LSL #2]\n"
- "prfm pldl1keep, [x21, x14]\n"
- "ld1w { z10.s }, p2/Z, [x20, x13, LSL #2]\n"
- "prfm pldl1keep, [x20, x14]\n"
- "ld1w { z14.s }, p2/Z, [x19, x13, LSL #2]\n"
- "prfm pldl1keep, [x19, x14]\n"
+ "ld1w { z9.s }, p2/Z, [x22, x13, LSL #2]\n"
+ "ld1w { z13.s }, p2/Z, [x21, x13, LSL #2]\n"
+ "ldp x20, x19, [x14, #0x30]\n"
+ "ldp x26, x25, [x14, #0x40]\n"
+ "ld1w { z11.s }, p2/Z, [x20, x13, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x19, x13, LSL #2]\n"
+ "ld1w { z10.s }, p2/Z, [x26, x13, LSL #2]\n"
+ "ld1w { z14.s }, p2/Z, [x25, x13, LSL #2]\n"
"bge 2f\n"
"1:" // Channel loop
- "fmla z31.s, p3/M, z0.s, z5.s\n"
- "ldr x21, [x15, #0x50]\n"
- "whilelt p1.s, x10, %x[n_channels]\n"
- "fmla z30.s, p3/M, z0.s, z6.s\n"
- "ldr x19, [x15, #0x58]\n"
- "incw x27\n"
- "fmla z29.s, p3/M, z0.s, z7.s\n"
- "ldr x20, [x15, #0x60]\n"
+ "movprfx z31, z16\n fmla z31.s, p3/M, z0.s, z5.s\n"
+ "ldr x24, [x14, #0x50]\n"
+ "whilelt p1.s, x12, %x[n_channels]\n"
+ "movprfx z30, z16\n fmla z30.s, p3/M, z0.s, z6.s\n"
+ "ldr x23, [x14, #0x58]\n"
+ "incw x9\n"
+ "movprfx z29, z16\n fmla z29.s, p3/M, z0.s, z7.s\n"
+ "ldr x22, [x14, #0x60]\n"
"mov p0.b, p2.b\n"
- "fmla z28.s, p3/M, z0.s, z8.s\n"
- "ld1w { z5.s }, p2/Z, [x21, x13, LSL #2]\n"
- "prfm pldl1keep, [x21, x14]\n"
+ "movprfx z28, z16\n fmla z28.s, p3/M, z0.s, z8.s\n"
+ "ld1w { z5.s }, p2/Z, [x24, x13, LSL #2]\n"
+ "ld1w { z0.s }, p3/Z, [x15]\n"
"fmla z31.s, p3/M, z1.s, z6.s\n"
- "ld1w { z6.s }, p2/Z, [x19, x13, LSL #2]\n"
+ "ld1w { z6.s }, p2/Z, [x23, x13, LSL #2]\n"
"fmla z30.s, p3/M, z1.s, z9.s\n"
- "prfm pldl1keep, [x19, x14]\n"
+ "ldr x21, [x14, #0x68]\n"
"fmla z29.s, p3/M, z1.s, z8.s\n"
- "prfm pldl1keep, [x20, x14]\n"
+ "ldr x20, [x14, #0x70]\n"
"fmla z28.s, p3/M, z1.s, z13.s\n"
- "ld1w { z0.s }, p3/Z, [x16]\n"
+ "ld1w { z1.s }, p3/Z, [x15, #1, MUL VL]\n"
"fmla z31.s, p3/M, z2.s, z9.s\n"
- "ld1w { z9.s }, p2/Z, [x20, x13, LSL #2]\n"
+ "ld1w { z9.s }, p2/Z, [x22, x13, LSL #2]\n"
"fmla z30.s, p3/M, z2.s, z11.s\n"
- "ldr x19, [x15, #0x68]\n"
+ "ldr x19, [x14, #0x78]\n"
"fmla z29.s, p3/M, z2.s, z13.s\n"
- "ld1w { z1.s }, p3/Z, [x16, #1, MUL VL]\n"
+ "ldr x26, [x14, #0x80]\n"
"fmla z28.s, p3/M, z2.s, z5.s\n"
- "ldr x20, [x15, #0x70]\n"
+ "ld1w { z2.s }, p3/Z, [x15, #2, MUL VL]\n"
"fmla z31.s, p3/M, z3.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x19, x13, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x21, x13, LSL #2]\n"
"fmla z30.s, p3/M, z3.s, z12.s\n"
- "prfm pldl1keep, [x19, x14]\n"
+ "ldr x25, [x14, #0x88]\n"
"fmla z29.s, p3/M, z3.s, z5.s\n"
- "prfm pldl1keep, [x20, x14]\n"
+ "ldr x24, [x14, #0x90]\n"
"fmla z28.s, p3/M, z3.s, z6.s\n"
- "ld1w { z2.s }, p3/Z, [x16, #2, MUL VL]\n"
+ "ld1w { z3.s }, p3/Z, [x15, #3, MUL VL]\n"
"fmla z31.s, p3/M, z4.s, z12.s\n"
"ld1w { z12.s }, p2/Z, [x20, x13, LSL #2]\n"
"fmla z30.s, p3/M, z4.s, z9.s\n"
- "ldr x19, [x15, #0x78]\n"
- "fmla z29.s, p3/M, z4.s, z6.s\n"
- "ld1w { z3.s }, p3/Z, [x16, #3, MUL VL]\n"
- "fmla z28.s, p3/M, z4.s, z10.s\n"
- "ldr x26, [x15, #0x80]\n"
- "fmla z31.s, p3/M, z0.s, z7.s\n"
"ld1w { z9.s }, p2/Z, [x19, x13, LSL #2]\n"
+ "fmla z29.s, p3/M, z4.s, z6.s\n"
+ "ldr x23, [x14, #0x98]\n"
+ "fmla z28.s, p3/M, z4.s, z10.s\n"
+ "ld1w { z4.s }, p3/Z, [x15, #4, MUL VL]\n"
+ "fmla z31.s, p3/M, z0.s, z7.s\n"
+ "ldr x22, [x14, #0xa0]\n"
"fmla z30.s, p3/M, z0.s, z8.s\n"
- "prfm pldl1keep, [x19, x14]\n"
+ "ldr x21, [x14, #0xa8]\n"
"fmla z29.s, p3/M, z0.s, z14.s\n"
- "prfm pldl1keep, [x26, x14]\n"
+ "ldr x20, [x14, #0xb0]\n"
"fmla z28.s, p3/M, z0.s, z11.s\n"
- "ld1w { z4.s }, p3/Z, [x16, #4, MUL VL]\n"
+ "ld1w { z0.s }, p3/Z, [x15, #5, MUL VL]\n"
"fmla z31.s, p3/M, z1.s, z8.s\n"
- "ldr x25, [x15, #0x88]\n"
- "fmla z30.s, p3/M, z1.s, z13.s\n"
- "ld1w { z0.s }, p3/Z, [x16, #5, MUL VL]\n"
- "fmla z29.s, p3/M, z1.s, z11.s\n"
- "ldr x24, [x15, #0x90]\n"
- "fmla z28.s, p3/M, z1.s, z12.s\n"
"ld1w { z8.s }, p2/Z, [x25, x13, LSL #2]\n"
+ "fmla z30.s, p3/M, z1.s, z13.s\n"
+ "ldr x19, [x14, #0xb8]\n"
+ "fmla z29.s, p3/M, z1.s, z11.s\n"
+ "ldr x25, [x14, #0xc8]\n"
+ "fmla z28.s, p3/M, z1.s, z12.s\n"
+ "ld1w { z1.s }, p3/Z, [x15, #6, MUL VL]\n"
"fmla z31.s, p3/M, z2.s, z13.s\n"
"ld1w { z13.s }, p2/Z, [x26, x13, LSL #2]\n"
"fmla z30.s, p3/M, z2.s, z5.s\n"
- "prfm pldl1keep, [x25, x14]\n"
+ "ldr x26, [x14, #0xc0]\n"
"fmla z29.s, p3/M, z2.s, z12.s\n"
- "prfm pldl1keep, [x24, x14]\n"
"fmla z28.s, p3/M, z2.s, z9.s\n"
- "ldr x23, [x15, #0x98]\n"
+ "ld1w { z2.s }, p3/Z, [x15, #7, MUL VL]\n"
+ "addvl x15, x15, #16\n"
"fmla z31.s, p3/M, z3.s, z5.s\n"
"ld1w { z5.s }, p2/Z, [x24, x13, LSL #2]\n"
+ "ldr x24, [x14, #0xd0]\n"
"fmla z30.s, p3/M, z3.s, z6.s\n"
- "ld1w { z1.s }, p3/Z, [x16, #6, MUL VL]\n"
+ "ld1w { z16.s }, p3/Z, [x15, #4, MUL VL]\n"
"fmla z29.s, p3/M, z3.s, z9.s\n"
- "prfm pldl1keep, [x23, x14]\n"
"fmla z28.s, p3/M, z3.s, z13.s\n"
- "ldr x20, [x15, #0xa0]\n"
+ "ld1w { z3.s }, p3/Z, [x15, #-8, MUL VL]\n"
"fmla z31.s, p3/M, z4.s, z6.s\n"
"ld1w { z6.s }, p2/Z, [x23, x13, LSL #2]\n"
+ "ldr x23, [x14, #0xd8]\n"
"fmla z30.s, p3/M, z4.s, z10.s\n"
- "ld1w { z2.s }, p3/Z, [x16, #7, MUL VL]\n"
+ "ld1w { z10.s }, p2/Z, [x22, x13, LSL #2]\n"
"fmla z29.s, p3/M, z4.s, z13.s\n"
- "addvl x16, x16, #16\n"
+ "ldr x22, [x14, #0xe0]\n"
"fmla z28.s, p3/M, z4.s, z8.s\n"
- "ld1w { z10.s }, p2/Z, [x20, x13, LSL #2]\n"
- "prfm pldl1keep, [x20, x14]\n"
+ "ld1w { z4.s }, p3/Z, [x15, #-7, MUL VL]\n"
"fmla z31.s, p3/M, z0.s, z14.s\n"
- "ldr x19, [x15, #0xa8]\n"
+ "ld1w { z14.s }, p2/Z, [x19, x13, LSL #2]\n"
"fmla z30.s, p3/M, z0.s, z11.s\n"
- "ld1w { z3.s }, p3/Z, [x16, #-8, MUL VL]\n"
+ "ldr x19, [x14, #0xf8]\n"
"fmla z29.s, p3/M, z0.s, z5.s\n"
- "ldr x22, [x15, #0xb0]\n"
"fmla z28.s, p3/M, z0.s, z6.s\n"
- "prfm pldl1keep, [x19, x14]\n"
+ "ld1w { z0.s }, p3/Z, [x15, #-6, MUL VL]\n"
"fmla z31.s, p3/M, z1.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x19, x13, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x21, x13, LSL #2]\n"
+ "ldr x21, [x14, #0xe8]\n"
"fmla z30.s, p3/M, z1.s, z12.s\n"
- "prfm pldl1keep, [x22, x14]\n"
"fmla z29.s, p3/M, z1.s, z6.s\n"
- "ld1w { z4.s }, p3/Z, [x16, #-7, MUL VL]\n"
"fmla z28.s, p3/M, z1.s, z10.s\n"
- "ldr x21, [x15, #0xb8]\n"
+ "ld1w { z1.s }, p3/Z, [x15, #-5, MUL VL]\n"
"fmla z31.s, p3/M, z2.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x22, x13, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x20, x13, LSL #2]\n"
+ "ldr x20, [x14, #0xf0]\n"
"fmla z30.s, p3/M, z2.s, z9.s\n"
- "ld1w { z0.s }, p3/Z, [x16, #-6, MUL VL]\n"
"fmla z29.s, p3/M, z2.s, z10.s\n"
- "ld1w { z14.s }, p2/Z, [x21, x13, LSL #2]\n"
"fmla z28.s, p3/M, z2.s, z11.s\n"
- "prfm pldl1keep, [x21, x14]\n"
+ "ld1w { z2.s }, p3/Z, [x15, #-4, MUL VL]\n"
"fmla z31.s, p3/M, z3.s, z9.s\n"
- "ldr x20, [x15, #0xc0]\n"
- "fmla z30.s, p3/M, z3.s, z13.s\n"
- "ldr x19, [x15, #0xc8]\n"
- "fmla z29.s, p3/M, z3.s, z11.s\n"
- "ld1w { z1.s }, p3/Z, [x16, #-5, MUL VL]\n"
- "fmla z28.s, p3/M, z3.s, z12.s\n"
- "ld1w { z9.s }, p2/Z, [x20, x13, LSL #2]\n"
- "fmla z31.s, p3/M, z4.s, z13.s\n"
- "prfm pldl1keep, [x20, x14]\n"
- "fmla z30.s, p3/M, z4.s, z8.s\n"
- "ld1w { z13.s }, p2/Z, [x19, x13, LSL #2]\n"
- "fmla z29.s, p3/M, z4.s, z12.s\n"
- "prfm pldl1keep, [x19, x14]\n"
- "fmla z28.s, p3/M, z4.s, z14.s\n"
- "ldr x21, [x15, #0xd0]\n"
- "fmla z31.s, p3/M, z0.s, z5.s\n"
- "ld1w { z2.s }, p3/Z, [x16, #-4, MUL VL]\n"
- "fmla z30.s, p3/M, z0.s, z6.s\n"
- "ldr x19, [x15, #0xd8]\n"
- "fmla z29.s, p3/M, z0.s, z9.s\n"
- "ld1w { z5.s }, p2/Z, [x21, x13, LSL #2]\n"
- "fmla z28.s, p3/M, z0.s, z13.s\n"
- "prfm pldl1keep, [x21, x14]\n"
- "fmla z31.s, p3/M, z1.s, z6.s\n"
- "ld1w { z6.s }, p2/Z, [x19, x13, LSL #2]\n"
- "fmla z30.s, p3/M, z1.s, z10.s\n"
- "prfm pldl1keep, [x19, x14]\n"
- "fmla z29.s, p3/M, z1.s, z13.s\n"
- "ld1w { z3.s }, p3/Z, [x16, #-3, MUL VL]\n"
- "fmla z28.s, p3/M, z1.s, z5.s\n"
- "ldr x20, [x15, #0xe0]\n"
- "fmla z31.s, p3/M, z2.s, z10.s\n"
- "ld1w { z4.s }, p3/Z, [x16, #-2, MUL VL]\n"
- "fmla z30.s, p3/M, z2.s, z11.s\n"
- "ldr x19, [x15, #0xe8]\n"
- "fmla z29.s, p3/M, z2.s, z5.s\n"
- "ld1w { z8.s }, p2/Z, [x20, x13, LSL #2]\n"
- "fmla z28.s, p3/M, z2.s, z6.s\n"
- "prfm pldl1keep, [x20, x14]\n"
- "fmla z31.s, p3/M, z3.s, z11.s\n"
- "ld1w { z10.s }, p2/Z, [x19, x13, LSL #2]\n"
- "fmla z30.s, p3/M, z3.s, z12.s\n"
- "prfm pldl1keep, [x19, x14]\n"
- "fmla z29.s, p3/M, z3.s, z6.s\n"
- "ld1w { z0.s }, p3/Z, [x16, #-1, MUL VL]\n"
- "fmla z28.s, p3/M, z3.s, z8.s\n"
- "ldr x20, [x15, #0xf0]\n"
- "fmla z31.s, p3/M, z4.s, z12.s\n"
- "ldr x19, [x15, #0xf8]\n"
- "fmla z30.s, p3/M, z4.s, z14.s\n"
- "ld1w { z1.s }, p3/Z, [x16]\n"
- "fmla z29.s, p3/M, z4.s, z8.s\n"
- "ld1w { z11.s }, p2/Z, [x20, x13, LSL #2]\n"
- "fmla z28.s, p3/M, z4.s, z10.s\n"
- "prfm pldl1keep, [x20, x14]\n"
- "fmla z31.s, p3/M, z0.s, z9.s\n"
- "ld1w { z12.s }, p2/Z, [x19, x13, LSL #2]\n"
- "fmla z30.s, p3/M, z0.s, z13.s\n"
- "prfm pldl1keep, [x19, x14]\n"
- "fmla z29.s, p3/M, z0.s, z11.s\n"
- "ldr x26, [x15, #0x100]\n"
- "fmla z28.s, p3/M, z0.s, z12.s\n"
- "ld1w { z2.s }, p3/Z, [x16, #1, MUL VL]\n"
- "fmla z31.s, p3/M, z1.s, z13.s\n"
- "ldr x25, [x15, #0x108]\n"
- "fmla z30.s, p3/M, z1.s, z5.s\n"
"ld1w { z9.s }, p2/Z, [x26, x13, LSL #2]\n"
- "fmla z29.s, p3/M, z1.s, z12.s\n"
- "prfm pldl1keep, [x26, x14]\n"
- "fmla z28.s, p3/M, z1.s, z9.s\n"
- "ld1w { z11.s }, p2/Z, [x25, x13, LSL #2]\n"
- "fmla z31.s, p3/M, z2.s, z5.s\n"
- "prfm pldl1keep, [x25, x14]\n"
- "fmla z30.s, p3/M, z2.s, z6.s\n"
- "ld1w { z3.s }, p3/Z, [x16, #2, MUL VL]\n"
- "fmla z29.s, p3/M, z2.s, z9.s\n"
- "ldr x24, [x15, #0x110]\n"
- "ld1w { z4.s }, p3/Z, [x16, #3, MUL VL]\n"
- "fmla z28.s, p3/M, z2.s, z11.s\n"
- "ldr x23, [x15, #0x118]\n"
- "fmla z31.s, p3/M, z3.s, z6.s\n"
- "fmla z30.s, p3/M, z3.s, z8.s\n"
- "ld1w { z12.s }, p2/Z, [x24, x13, LSL #2]\n"
+ "ldr x26, [x14, #0x100]\n"
+ "fmla z30.s, p3/M, z3.s, z13.s\n"
"fmla z29.s, p3/M, z3.s, z11.s\n"
- "prfm pldl1keep, [x24, x14]\n"
- "ld1w { z9.s }, p2/Z, [x23, x13, LSL #2]\n"
"fmla z28.s, p3/M, z3.s, z12.s\n"
- "prfm pldl1keep, [x23, x14]\n"
- "incw x13\n"
- "fmla z31.s, p3/M, z4.s, z8.s\n"
- "ldp x26, x25, [x15, #0x0]\n"
- "whilelt p2.s, x13, %x[n_channels]\n"
- "fmla z30.s, p3/M, z4.s, z10.s\n"
- "ldp x24, x23, [x15, #0x10]\n"
- "addvl x14, x14, #1\n"
+ "ld1w { z3.s }, p3/Z, [x15, #-3, MUL VL]\n"
+ "fmla z31.s, p3/M, z4.s, z13.s\n"
+ "ld1w { z13.s }, p2/Z, [x25, x13, LSL #2]\n"
+ "ldr x25, [x14, #0x108]\n"
+ "fmla z30.s, p3/M, z4.s, z8.s\n"
+ "ld1w { z8.s }, p2/Z, [x22, x13, LSL #2]\n"
"fmla z29.s, p3/M, z4.s, z12.s\n"
- "ldp x20, x19, [x15, #0x20]\n"
- "ldp x22, x21, [x15, #0x30]\n"
+ "fmla z28.s, p3/M, z4.s, z14.s\n"
+ "ld1w { z4.s }, p3/Z, [x15, #-2, MUL VL]\n"
+ "fmla z31.s, p3/M, z0.s, z5.s\n"
+ "ld1w { z5.s }, p2/Z, [x24, x13, LSL #2]\n"
+ "ldr x24, [x14, #0x110]\n"
+ "fmla z30.s, p3/M, z0.s, z6.s\n"
+ "fmla z29.s, p3/M, z0.s, z9.s\n"
+ "fmla z28.s, p3/M, z0.s, z13.s\n"
+ "ld1w { z0.s }, p3/Z, [x15, #-1, MUL VL]\n"
+ "fmla z31.s, p3/M, z1.s, z6.s\n"
+ "ld1w { z6.s }, p2/Z, [x23, x13, LSL #2]\n"
+ "ldr x23, [x14, #0x118]\n"
+ "fmla z30.s, p3/M, z1.s, z10.s\n"
+ "fmla z29.s, p3/M, z1.s, z13.s\n"
+ "fmla z28.s, p3/M, z1.s, z5.s\n"
+ "ld1w { z1.s }, p3/Z, [x15]\n"
+ "fmla z31.s, p3/M, z2.s, z10.s\n"
+ "ld1w { z10.s }, p2/Z, [x21, x13, LSL #2]\n"
+ "fmla z30.s, p3/M, z2.s, z11.s\n"
+ "fmla z29.s, p3/M, z2.s, z5.s\n"
+ "fmla z28.s, p3/M, z2.s, z6.s\n"
+ "ld1w { z2.s }, p3/Z, [x15, #1, MUL VL]\n"
+ "fmla z31.s, p3/M, z3.s, z11.s\n"
+ "ld1w { z11.s }, p2/Z, [x20, x13, LSL #2]\n"
+ "fmla z30.s, p3/M, z3.s, z12.s\n"
+ "fmla z29.s, p3/M, z3.s, z6.s\n"
+ "fmla z28.s, p3/M, z3.s, z8.s\n"
+ "ld1w { z3.s }, p3/Z, [x15, #2, MUL VL]\n"
+ "fmla z31.s, p3/M, z4.s, z12.s\n"
+ "ld1w { z12.s }, p2/Z, [x19, x13, LSL #2]\n"
+ "fmla z30.s, p3/M, z4.s, z14.s\n"
+ "fmla z29.s, p3/M, z4.s, z8.s\n"
+ "fmla z28.s, p3/M, z4.s, z10.s\n"
+ "ld1w { z4.s }, p3/Z, [x15, #3, MUL VL]\n"
+ "fmla z31.s, p3/M, z0.s, z9.s\n"
+ "ld1w { z9.s }, p2/Z, [x26, x13, LSL #2]\n"
+ "fmla z30.s, p3/M, z0.s, z13.s\n"
+ "fmla z29.s, p3/M, z0.s, z11.s\n"
+ "ld1w { z11.s }, p2/Z, [x25, x13, LSL #2]\n"
+ "ldp x26, x25, [x14, #0x0]\n"
+ "fmla z28.s, p3/M, z0.s, z12.s\n"
+ "ld1w { z0.s }, p3/Z, [x15, #5, MUL VL]\n"
+ "fmla z31.s, p3/M, z1.s, z13.s\n"
+ "fmla z30.s, p3/M, z1.s, z5.s\n"
+ "fmla z29.s, p3/M, z1.s, z12.s\n"
+ "ld1w { z12.s }, p2/Z, [x24, x13, LSL #2]\n"
+ "fmla z28.s, p3/M, z1.s, z9.s\n"
+ "ld1w { z1.s }, p3/Z, [x15, #6, MUL VL]\n"
+ "fmla z31.s, p3/M, z2.s, z5.s\n"
+ "ld1w { z5.s }, p1/Z, [x26, x12, LSL #2]\n"
+ "fmla z30.s, p3/M, z2.s, z6.s\n"
+ "fmla z29.s, p3/M, z2.s, z9.s\n"
+ "ld1w { z9.s }, p2/Z, [x23, x13, LSL #2]\n"
+ "incw x13\n"
+ "fmla z28.s, p3/M, z2.s, z11.s\n"
+ "ldp x24, x23, [x14, #0x10]\n"
+ "whilelt p2.s, x13, %x[n_channels]\n"
+ "fmla z31.s, p3/M, z3.s, z6.s\n"
+ "ld1w { z6.s }, p1/Z, [x25, x12, LSL #2]\n"
+ "ldp x22, x21, [x14, #0x20]\n"
+ "fmla z30.s, p3/M, z3.s, z8.s\n"
+ "ldp x20, x19, [x14, #0x30]\n"
+ "fmla z29.s, p3/M, z3.s, z11.s\n"
+ "ld1w { z7.s }, p1/Z, [x24, x12, LSL #2]\n"
+ "fmla z28.s, p3/M, z3.s, z12.s\n"
+ "ld1w { z13.s }, p1/Z, [x21, x12, LSL #2]\n"
+ "fmla z31.s, p3/M, z4.s, z8.s\n"
+ "ld1w { z8.s }, p1/Z, [x23, x12, LSL #2]\n"
+ "fmla z30.s, p3/M, z4.s, z10.s\n"
+ "ld1w { z11.s }, p1/Z, [x20, x12, LSL #2]\n"
+ "fmla z29.s, p3/M, z4.s, z12.s\n"
+ "ld1w { z12.s }, p1/Z, [x19, x12, LSL #2]\n"
"fmla z28.s, p3/M, z4.s, z9.s\n"
- "ld1w { z5.s }, p1/Z, [x26, x10, LSL #2]\n"
+ "ld1w { z9.s }, p1/Z, [x22, x12, LSL #2]\n"
"fmax z31.s, p3/M, z31.s, z18.s\n"
- "prfm pldl1keep, [x26, x14]\n"
+ "ldp x26, x25, [x14, #0x40]\n"
"fmax z30.s, p3/M, z30.s, z18.s\n"
- "ld1w { z6.s }, p1/Z, [x25, x10, LSL #2]\n"
+ "ld1w { z2.s }, p3/Z, [x15, #7, MUL VL]\n"
"fmax z29.s, p3/M, z29.s, z18.s\n"
- "prfm pldl1keep, [x25, x14]\n"
+ "addvl x15, x15, #16\n"
"fmax z28.s, p3/M, z28.s, z18.s\n"
- "ld1w { z7.s }, p1/Z, [x24, x10, LSL #2]\n"
+ "ld1w { z10.s }, p1/Z, [x26, x12, LSL #2]\n"
+ "ld1w { z14.s }, p1/Z, [x25, x12, LSL #2]\n"
"fmin z31.s, p3/M, z31.s, z17.s\n"
- "prfm pldl1keep, [x24, x14]\n"
+ "incw x12\n"
"fmin z30.s, p3/M, z30.s, z17.s\n"
- "ld1w { z8.s }, p1/Z, [x23, x10, LSL #2]\n"
+ "ld1w { z3.s }, p3/Z, [x15, #-8, MUL VL]\n"
+ "cmp x12, %x[n_channels]\n"
"fmin z29.s, p3/M, z29.s, z17.s\n"
- "prfm pldl1keep, [x23, x14]\n"
+ "ld1w { z4.s }, p3/Z, [x15, #-7, MUL VL]\n"
+ "addvl x15, x15, #-6\n"
"fmin z28.s, p3/M, z28.s, z17.s\n"
- "ld1w { z9.s }, p1/Z, [x20, x10, LSL #2]\n"
- "prfm pldl1keep, [x20, x14]\n"
- "ld1w { z13.s }, p1/Z, [x19, x10, LSL #2]\n"
- "prfm pldl1keep, [x19, x14]\n"
- "ld1w { z11.s }, p1/Z, [x22, x10, LSL #2]\n"
- "prfm pldl1keep, [x22, x14]\n"
- "ld1w { z12.s }, p1/Z, [x21, x10, LSL #2]\n"
- "prfm pldl1keep, [x21, x14]\n"
- "ldp x20, x19, [x15, #0x40]\n"
- "st1w { z31.s }, p0, [x12, x27, LSL #2]\n"
- "st1w { z30.s }, p0, [x11, x27, LSL #2]\n"
- "ld1w { z10.s }, p1/Z, [x20, x10, LSL #2]\n"
- "prfm pldl1keep, [x20, x14]\n"
- "ld1w { z14.s }, p1/Z, [x19, x10, LSL #2]\n"
- "incw x10\n"
- "prfm pldl1keep, [x19, x14]\n"
- "cmp x10, %x[n_channels]\n"
- "st1w { z29.s }, p0, [x9, x27, LSL #2]\n"
- "st1w { z28.s }, p0, [x28, x27, LSL #2]\n"
- "ld1w { z16.s }, p3/Z, [x16, #4, MUL VL]\n"
- "mov z31.d, z16.d\n"
- "ld1w { z0.s }, p3/Z, [x16, #5, MUL VL]\n"
- "mov z30.d, z16.d\n"
- "ld1w { z1.s }, p3/Z, [x16, #6, MUL VL]\n"
- "mov z29.d, z16.d\n"
- "ld1w { z2.s }, p3/Z, [x16, #7, MUL VL]\n"
- "addvl x16, x16, #16\n"
- "mov z28.d, z16.d\n"
- "ld1w { z3.s }, p3/Z, [x16, #-8, MUL VL]\n"
- "ld1w { z4.s }, p3/Z, [x16, #-7, MUL VL]\n"
- "addvl x16, x16, #-6\n"
+ "st1w { z31.s }, p0, [x11, x9, LSL #2]\n"
+ "st1w { z30.s }, p0, [x10, x9, LSL #2]\n"
+ "st1w { z29.s }, p0, [x28, x9, LSL #2]\n"
+ "st1w { z28.s }, p0, [x27, x9, LSL #2]\n"
"blt 1b\n"
"2:" // Channel tail
- "fmla z31.s, p3/M, z0.s, z5.s\n"
- "ldr x21, [x15, #0x50]\n"
- "incw x27\n"
- "fmla z30.s, p3/M, z0.s, z6.s\n"
- "ldr x19, [x15, #0x58]\n"
+ "movprfx z31, z16\n fmla z31.s, p3/M, z0.s, z5.s\n"
+ "ldr x24, [x14, #0x50]\n"
+ "incw x9\n"
+ "movprfx z30, z16\n fmla z30.s, p3/M, z0.s, z6.s\n"
+ "ldr x23, [x14, #0x58]\n"
"mov p0.b, p2.b\n"
- "fmla z29.s, p3/M, z0.s, z7.s\n"
- "ldr x20, [x15, #0x60]\n"
- "fmla z28.s, p3/M, z0.s, z8.s\n"
- "ld1w { z5.s }, p2/Z, [x21, x13, LSL #2]\n"
- "prfm pldl1keep, [x21, x14]\n"
+ "movprfx z29, z16\n fmla z29.s, p3/M, z0.s, z7.s\n"
+ "ldr x22, [x14, #0x60]\n"
+ "movprfx z28, z16\n fmla z28.s, p3/M, z0.s, z8.s\n"
+ "ld1w { z5.s }, p2/Z, [x24, x13, LSL #2]\n"
+ "ld1w { z0.s }, p3/Z, [x15]\n"
"fmla z31.s, p3/M, z1.s, z6.s\n"
- "ld1w { z6.s }, p2/Z, [x19, x13, LSL #2]\n"
+ "ld1w { z6.s }, p2/Z, [x23, x13, LSL #2]\n"
"fmla z30.s, p3/M, z1.s, z9.s\n"
- "prfm pldl1keep, [x19, x14]\n"
+ "ldr x21, [x14, #0x68]\n"
"fmla z29.s, p3/M, z1.s, z8.s\n"
"fmla z28.s, p3/M, z1.s, z13.s\n"
- "prfm pldl1keep, [x20, x14]\n"
- "ld1w { z0.s }, p3/Z, [x16]\n"
+ "ld1w { z1.s }, p3/Z, [x15, #1, MUL VL]\n"
+ "ldr x20, [x14, #0x70]\n"
"fmla z31.s, p3/M, z2.s, z9.s\n"
- "ld1w { z9.s }, p2/Z, [x20, x13, LSL #2]\n"
+ "ld1w { z9.s }, p2/Z, [x22, x13, LSL #2]\n"
"fmla z30.s, p3/M, z2.s, z11.s\n"
- "ldr x19, [x15, #0x68]\n"
+ "ldr x19, [x14, #0x78]\n"
"fmla z29.s, p3/M, z2.s, z13.s\n"
"fmla z28.s, p3/M, z2.s, z5.s\n"
- "ld1w { z1.s }, p3/Z, [x16, #1, MUL VL]\n"
- "ldr x20, [x15, #0x70]\n"
+ "ld1w { z2.s }, p3/Z, [x15, #2, MUL VL]\n"
+ "ldr x26, [x14, #0x80]\n"
"fmla z31.s, p3/M, z3.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x19, x13, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x21, x13, LSL #2]\n"
"fmla z30.s, p3/M, z3.s, z12.s\n"
- "prfm pldl1keep, [x19, x14]\n"
+ "ldr x25, [x14, #0x88]\n"
"fmla z29.s, p3/M, z3.s, z5.s\n"
"fmla z28.s, p3/M, z3.s, z6.s\n"
- "prfm pldl1keep, [x20, x14]\n"
- "ld1w { z2.s }, p3/Z, [x16, #2, MUL VL]\n"
+ "ld1w { z3.s }, p3/Z, [x15, #3, MUL VL]\n"
+ "ldr x24, [x14, #0x90]\n"
"fmla z31.s, p3/M, z4.s, z12.s\n"
"ld1w { z12.s }, p2/Z, [x20, x13, LSL #2]\n"
"fmla z30.s, p3/M, z4.s, z9.s\n"
- "ldr x19, [x15, #0x78]\n"
+ "ld1w { z9.s }, p2/Z, [x19, x13, LSL #2]\n"
"fmla z29.s, p3/M, z4.s, z6.s\n"
"fmla z28.s, p3/M, z4.s, z10.s\n"
- "ld1w { z3.s }, p3/Z, [x16, #3, MUL VL]\n"
- "ldr x26, [x15, #0x80]\n"
+ "ld1w { z4.s }, p3/Z, [x15, #4, MUL VL]\n"
+ "ldr x23, [x14, #0x98]\n"
"fmla z31.s, p3/M, z0.s, z7.s\n"
- "ld1w { z9.s }, p2/Z, [x19, x13, LSL #2]\n"
+ "ldr x22, [x14, #0xa0]\n"
"fmla z30.s, p3/M, z0.s, z8.s\n"
- "prfm pldl1keep, [x19, x14]\n"
+ "ldr x21, [x14, #0xa8]\n"
"fmla z29.s, p3/M, z0.s, z14.s\n"
"fmla z28.s, p3/M, z0.s, z11.s\n"
- "prfm pldl1keep, [x26, x14]\n"
- "ld1w { z4.s }, p3/Z, [x16, #4, MUL VL]\n"
+ "ld1w { z0.s }, p3/Z, [x15, #5, MUL VL]\n"
+ "ldr x20, [x14, #0xb0]\n"
"fmla z31.s, p3/M, z1.s, z8.s\n"
- "ldr x25, [x15, #0x88]\n"
+ "ld1w { z8.s }, p2/Z, [x25, x13, LSL #2]\n"
"fmla z30.s, p3/M, z1.s, z13.s\n"
- "ld1w { z0.s }, p3/Z, [x16, #5, MUL VL]\n"
+ "ldr x19, [x14, #0xb8]\n"
"fmla z29.s, p3/M, z1.s, z11.s\n"
"fmla z28.s, p3/M, z1.s, z12.s\n"
- "ld1w { z8.s }, p2/Z, [x25, x13, LSL #2]\n"
- "prfm pldl1keep, [x25, x14]\n"
+ "ld1w { z1.s }, p3/Z, [x15, #6, MUL VL]\n"
+ "ldr x25, [x14, #0xc8]\n"
"fmla z31.s, p3/M, z2.s, z13.s\n"
"ld1w { z13.s }, p2/Z, [x26, x13, LSL #2]\n"
"fmla z30.s, p3/M, z2.s, z5.s\n"
- "ldr x24, [x15, #0x90]\n"
+ "ldr x26, [x14, #0xc0]\n"
"fmla z29.s, p3/M, z2.s, z12.s\n"
"fmla z28.s, p3/M, z2.s, z9.s\n"
- "ldr x23, [x15, #0x98]\n"
- "ld1w { z1.s }, p3/Z, [x16, #6, MUL VL]\n"
+ "ld1w { z2.s }, p3/Z, [x15, #7, MUL VL]\n"
+ "addvl x15, x15, #16\n"
"fmla z31.s, p3/M, z3.s, z5.s\n"
"ld1w { z5.s }, p2/Z, [x24, x13, LSL #2]\n"
+ "ldr x24, [x14, #0xd0]\n"
"fmla z30.s, p3/M, z3.s, z6.s\n"
- "prfm pldl1keep, [x24, x14]\n"
"fmla z29.s, p3/M, z3.s, z9.s\n"
"fmla z28.s, p3/M, z3.s, z13.s\n"
- "prfm pldl1keep, [x23, x14]\n"
- "ldr x20, [x15, #0xa0]\n"
+ "ld1w { z3.s }, p3/Z, [x15, #-8, MUL VL]\n"
"fmla z31.s, p3/M, z4.s, z6.s\n"
"ld1w { z6.s }, p2/Z, [x23, x13, LSL #2]\n"
+ "ldr x23, [x14, #0xd8]\n"
"fmla z30.s, p3/M, z4.s, z10.s\n"
- "ld1w { z2.s }, p3/Z, [x16, #7, MUL VL]\n"
+ "ld1w { z10.s }, p2/Z, [x22, x13, LSL #2]\n"
"fmla z29.s, p3/M, z4.s, z13.s\n"
- "addvl x16, x16, #16\n"
"fmla z28.s, p3/M, z4.s, z8.s\n"
- "ld1w { z10.s }, p2/Z, [x20, x13, LSL #2]\n"
- "prfm pldl1keep, [x20, x14]\n"
+ "ld1w { z4.s }, p3/Z, [x15, #-7, MUL VL]\n"
+ "ldr x22, [x14, #0xe0]\n"
"fmla z31.s, p3/M, z0.s, z14.s\n"
- "ldr x19, [x15, #0xa8]\n"
+ "ld1w { z14.s }, p2/Z, [x19, x13, LSL #2]\n"
"fmla z30.s, p3/M, z0.s, z11.s\n"
- "ld1w { z3.s }, p3/Z, [x16, #-8, MUL VL]\n"
+ "ldr x19, [x14, #0xf8]\n"
"fmla z29.s, p3/M, z0.s, z5.s\n"
- "ldr x22, [x15, #0xb0]\n"
"fmla z28.s, p3/M, z0.s, z6.s\n"
- "prfm pldl1keep, [x19, x14]\n"
+ "ld1w { z0.s }, p3/Z, [x15, #-6, MUL VL]\n"
"fmla z31.s, p3/M, z1.s, z11.s\n"
- "ld1w { z11.s }, p2/Z, [x19, x13, LSL #2]\n"
+ "ld1w { z11.s }, p2/Z, [x21, x13, LSL #2]\n"
+ "ldr x21, [x14, #0xe8]\n"
"fmla z30.s, p3/M, z1.s, z12.s\n"
- "prfm pldl1keep, [x22, x14]\n"
"fmla z29.s, p3/M, z1.s, z6.s\n"
- "ld1w { z4.s }, p3/Z, [x16, #-7, MUL VL]\n"
"fmla z28.s, p3/M, z1.s, z10.s\n"
- "ldr x21, [x15, #0xb8]\n"
+ "ld1w { z1.s }, p3/Z, [x15, #-5, MUL VL]\n"
"fmla z31.s, p3/M, z2.s, z12.s\n"
- "ld1w { z12.s }, p2/Z, [x22, x13, LSL #2]\n"
+ "ld1w { z12.s }, p2/Z, [x20, x13, LSL #2]\n"
+ "ldr x20, [x14, #0xf0]\n"
"fmla z30.s, p3/M, z2.s, z9.s\n"
- "ld1w { z0.s }, p3/Z, [x16, #-6, MUL VL]\n"
"fmla z29.s, p3/M, z2.s, z10.s\n"
- "ld1w { z14.s }, p2/Z, [x21, x13, LSL #2]\n"
"fmla z28.s, p3/M, z2.s, z11.s\n"
- "prfm pldl1keep, [x21, x14]\n"
+ "ld1w { z2.s }, p3/Z, [x15, #-4, MUL VL]\n"
"fmla z31.s, p3/M, z3.s, z9.s\n"
- "ldr x20, [x15, #0xc0]\n"
- "fmla z30.s, p3/M, z3.s, z13.s\n"
- "ldr x19, [x15, #0xc8]\n"
- "fmla z29.s, p3/M, z3.s, z11.s\n"
- "ld1w { z1.s }, p3/Z, [x16, #-5, MUL VL]\n"
- "fmla z28.s, p3/M, z3.s, z12.s\n"
- "ld1w { z9.s }, p2/Z, [x20, x13, LSL #2]\n"
- "fmla z31.s, p3/M, z4.s, z13.s\n"
- "prfm pldl1keep, [x20, x14]\n"
- "fmla z30.s, p3/M, z4.s, z8.s\n"
- "ld1w { z13.s }, p2/Z, [x19, x13, LSL #2]\n"
- "fmla z29.s, p3/M, z4.s, z12.s\n"
- "prfm pldl1keep, [x19, x14]\n"
- "fmla z28.s, p3/M, z4.s, z14.s\n"
- "ldr x21, [x15, #0xd0]\n"
- "fmla z31.s, p3/M, z0.s, z5.s\n"
- "ld1w { z2.s }, p3/Z, [x16, #-4, MUL VL]\n"
- "fmla z30.s, p3/M, z0.s, z6.s\n"
- "ldr x19, [x15, #0xd8]\n"
- "fmla z29.s, p3/M, z0.s, z9.s\n"
- "ld1w { z5.s }, p2/Z, [x21, x13, LSL #2]\n"
- "fmla z28.s, p3/M, z0.s, z13.s\n"
- "prfm pldl1keep, [x21, x14]\n"
- "fmla z31.s, p3/M, z1.s, z6.s\n"
- "ld1w { z6.s }, p2/Z, [x19, x13, LSL #2]\n"
- "fmla z30.s, p3/M, z1.s, z10.s\n"
- "prfm pldl1keep, [x19, x14]\n"
- "fmla z29.s, p3/M, z1.s, z13.s\n"
- "ld1w { z3.s }, p3/Z, [x16, #-3, MUL VL]\n"
- "fmla z28.s, p3/M, z1.s, z5.s\n"
- "ldr x20, [x15, #0xe0]\n"
- "fmla z31.s, p3/M, z2.s, z10.s\n"
- "ld1w { z4.s }, p3/Z, [x16, #-2, MUL VL]\n"
- "fmla z30.s, p3/M, z2.s, z11.s\n"
- "ldr x19, [x15, #0xe8]\n"
- "fmla z29.s, p3/M, z2.s, z5.s\n"
- "ld1w { z8.s }, p2/Z, [x20, x13, LSL #2]\n"
- "fmla z28.s, p3/M, z2.s, z6.s\n"
- "prfm pldl1keep, [x20, x14]\n"
- "fmla z31.s, p3/M, z3.s, z11.s\n"
- "ld1w { z10.s }, p2/Z, [x19, x13, LSL #2]\n"
- "fmla z30.s, p3/M, z3.s, z12.s\n"
- "prfm pldl1keep, [x19, x14]\n"
- "fmla z29.s, p3/M, z3.s, z6.s\n"
- "ld1w { z0.s }, p3/Z, [x16, #-1, MUL VL]\n"
- "fmla z28.s, p3/M, z3.s, z8.s\n"
- "ldr x20, [x15, #0xf0]\n"
- "fmla z31.s, p3/M, z4.s, z12.s\n"
- "ldr x19, [x15, #0xf8]\n"
- "fmla z30.s, p3/M, z4.s, z14.s\n"
- "ld1w { z1.s }, p3/Z, [x16]\n"
- "fmla z29.s, p3/M, z4.s, z8.s\n"
- "ld1w { z11.s }, p2/Z, [x20, x13, LSL #2]\n"
- "fmla z28.s, p3/M, z4.s, z10.s\n"
- "prfm pldl1keep, [x20, x14]\n"
- "fmla z31.s, p3/M, z0.s, z9.s\n"
- "ld1w { z12.s }, p2/Z, [x19, x13, LSL #2]\n"
- "fmla z30.s, p3/M, z0.s, z13.s\n"
- "prfm pldl1keep, [x19, x14]\n"
- "fmla z29.s, p3/M, z0.s, z11.s\n"
- "ldr x26, [x15, #0x100]\n"
- "fmla z28.s, p3/M, z0.s, z12.s\n"
- "ld1w { z2.s }, p3/Z, [x16, #1, MUL VL]\n"
- "fmla z31.s, p3/M, z1.s, z13.s\n"
- "ldr x25, [x15, #0x108]\n"
- "fmla z30.s, p3/M, z1.s, z5.s\n"
"ld1w { z9.s }, p2/Z, [x26, x13, LSL #2]\n"
- "fmla z29.s, p3/M, z1.s, z12.s\n"
- "prfm pldl1keep, [x26, x14]\n"
- "fmla z28.s, p3/M, z1.s, z9.s\n"
+ "ldr x26, [x14, #0x100]\n"
+ "fmla z30.s, p3/M, z3.s, z13.s\n"
+ "fmla z29.s, p3/M, z3.s, z11.s\n"
+ "fmla z28.s, p3/M, z3.s, z12.s\n"
+ "ld1w { z3.s }, p3/Z, [x15, #-3, MUL VL]\n"
+ "fmla z31.s, p3/M, z4.s, z13.s\n"
+ "ld1w { z13.s }, p2/Z, [x25, x13, LSL #2]\n"
+ "ldr x25, [x14, #0x108]\n"
+ "fmla z30.s, p3/M, z4.s, z8.s\n"
+ "ld1w { z8.s }, p2/Z, [x22, x13, LSL #2]\n"
+ "fmla z29.s, p3/M, z4.s, z12.s\n"
+ "fmla z28.s, p3/M, z4.s, z14.s\n"
+ "ld1w { z4.s }, p3/Z, [x15, #-2, MUL VL]\n"
+ "fmla z31.s, p3/M, z0.s, z5.s\n"
+ "ld1w { z5.s }, p2/Z, [x24, x13, LSL #2]\n"
+ "ldr x24, [x14, #0x110]\n"
+ "fmla z30.s, p3/M, z0.s, z6.s\n"
+ "fmla z29.s, p3/M, z0.s, z9.s\n"
+ "fmla z28.s, p3/M, z0.s, z13.s\n"
+ "ld1w { z0.s }, p3/Z, [x15, #-1, MUL VL]\n"
+ "fmla z31.s, p3/M, z1.s, z6.s\n"
+ "ld1w { z6.s }, p2/Z, [x23, x13, LSL #2]\n"
+ "ldr x23, [x14, #0x118]\n"
+ "fmla z30.s, p3/M, z1.s, z10.s\n"
+ "fmla z29.s, p3/M, z1.s, z13.s\n"
+ "fmla z28.s, p3/M, z1.s, z5.s\n"
+ "ld1w { z1.s }, p3/Z, [x15]\n"
+ "fmla z31.s, p3/M, z2.s, z10.s\n"
+ "ld1w { z10.s }, p2/Z, [x21, x13, LSL #2]\n"
+ "fmla z30.s, p3/M, z2.s, z11.s\n"
+ "fmla z29.s, p3/M, z2.s, z5.s\n"
+ "fmla z28.s, p3/M, z2.s, z6.s\n"
+ "ld1w { z2.s }, p3/Z, [x15, #1, MUL VL]\n"
+ "fmla z31.s, p3/M, z3.s, z11.s\n"
+ "ld1w { z11.s }, p2/Z, [x20, x13, LSL #2]\n"
+ "fmla z30.s, p3/M, z3.s, z12.s\n"
+ "fmla z29.s, p3/M, z3.s, z6.s\n"
+ "fmla z28.s, p3/M, z3.s, z8.s\n"
+ "ld1w { z3.s }, p3/Z, [x15, #2, MUL VL]\n"
+ "fmla z31.s, p3/M, z4.s, z12.s\n"
+ "ld1w { z12.s }, p2/Z, [x19, x13, LSL #2]\n"
+ "fmla z30.s, p3/M, z4.s, z14.s\n"
+ "fmla z29.s, p3/M, z4.s, z8.s\n"
+ "fmla z28.s, p3/M, z4.s, z10.s\n"
+ "ld1w { z4.s }, p3/Z, [x15, #3, MUL VL]\n"
+ "fmla z31.s, p3/M, z0.s, z9.s\n"
+ "ld1w { z9.s }, p2/Z, [x26, x13, LSL #2]\n"
+ "fmla z30.s, p3/M, z0.s, z13.s\n"
+ "fmla z29.s, p3/M, z0.s, z11.s\n"
"ld1w { z11.s }, p2/Z, [x25, x13, LSL #2]\n"
+ "fmla z28.s, p3/M, z0.s, z12.s\n"
+ "fmla z31.s, p3/M, z1.s, z13.s\n"
+ "fmla z30.s, p3/M, z1.s, z5.s\n"
+ "fmla z29.s, p3/M, z1.s, z12.s\n"
+ "ld1w { z12.s }, p2/Z, [x24, x13, LSL #2]\n"
+ "fmla z28.s, p3/M, z1.s, z9.s\n"
"fmla z31.s, p3/M, z2.s, z5.s\n"
- "prfm pldl1keep, [x25, x14]\n"
"fmla z30.s, p3/M, z2.s, z6.s\n"
- "ld1w { z3.s }, p3/Z, [x16, #2, MUL VL]\n"
"fmla z29.s, p3/M, z2.s, z9.s\n"
- "ldr x24, [x15, #0x110]\n"
- "ld1w { z4.s }, p3/Z, [x16, #3, MUL VL]\n"
+ "ld1w { z9.s }, p2/Z, [x23, x13, LSL #2]\n"
"fmla z28.s, p3/M, z2.s, z11.s\n"
- "ldr x23, [x15, #0x118]\n"
"fmla z31.s, p3/M, z3.s, z6.s\n"
"fmla z30.s, p3/M, z3.s, z8.s\n"
- "ld1w { z12.s }, p2/Z, [x24, x13, LSL #2]\n"
"fmla z29.s, p3/M, z3.s, z11.s\n"
- "prfm pldl1keep, [x24, x14]\n"
- "ld1w { z9.s }, p2/Z, [x23, x13, LSL #2]\n"
"fmla z28.s, p3/M, z3.s, z12.s\n"
- "prfm pldl1keep, [x23, x14]\n"
"fmla z31.s, p3/M, z4.s, z8.s\n"
"fmla z30.s, p3/M, z4.s, z10.s\n"
"fmla z29.s, p3/M, z4.s, z12.s\n"
@@ -612,22 +530,22 @@
"fmax z31.s, p3/M, z31.s, z18.s\n"
"fmax z30.s, p3/M, z30.s, z18.s\n"
"fmax z29.s, p3/M, z29.s, z18.s\n"
+ "fmax z28.s, p3/M, z28.s, z18.s\n"
"fmin z31.s, p3/M, z31.s, z17.s\n"
- "st1w { z31.s }, p0, [x12, x27, LSL #2]\n"
+ "st1w { z31.s }, p0, [x11, x9, LSL #2]\n"
"fmin z30.s, p3/M, z30.s, z17.s\n"
"fmin z29.s, p3/M, z29.s, z17.s\n"
- "st1w { z30.s }, p0, [x11, x27, LSL #2]\n"
- "fmax z28.s, p3/M, z28.s, z18.s\n"
- "st1w { z29.s }, p0, [x9, x27, LSL #2]\n"
+ "st1w { z30.s }, p0, [x10, x9, LSL #2]\n"
"fmin z28.s, p3/M, z28.s, z17.s\n"
- "st1w { z28.s }, p0, [x28, x27, LSL #2]\n"
+ "st1w { z29.s }, p0, [x28, x9, LSL #2]\n"
+ "st1w { z28.s }, p0, [x27, x9, LSL #2]\n"
:
: [n_channels] "r" ((unsigned long) n_channels), [offsetof_Args_inptrs] "I" (offsetof(Args, inptrs)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_outptrs] "I" (offsetof(Args, outptrs)), [offsetof_args_params] "I" (offsetof(Args, params)), [params_struct] "r" (¶ms_struct)
- : "cc", "memory", "p0", "p1", "p2", "p3", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z16", "z17", "z18", "z28", "z29", "z30", "z31"
+ : "cc", "memory", "p0", "p1", "p2", "p3", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z16", "z17", "z18", "z28", "z29", "z30", "z31"
);
}
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(ARM_COMPUTE_ENABLE_SVE)
+#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp
index c501c67..b524fd7 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp
@@ -29,7 +29,7 @@
#pragma once
-#if defined(__aarch64__) && defined(ARM_COMPUTE_ENABLE_SVE) && defined(ARM_COMPUTE_ENABLE_SVE2)
+#if defined(__aarch64__) && defined(ARM_COMPUTE_ENABLE_SVE)
namespace arm_conv {
namespace depthwise {
@@ -72,4 +72,4 @@
} // namespace depthwise
} // namespace arm_conv
-#endif // defined(__aarch64__) && defined(ARM_COMPUTE_ENABLE_SVE) && defined(ARM_COMPUTE_ENABLE_SVE2)
+#endif // defined(__aarch64__) && defined(ARM_COMPUTE_ENABLE_SVE)
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst/generic.cpp
index 40220ad..52dc468 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst/generic.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst/generic.cpp
@@ -22,7 +22,7 @@
* SOFTWARE.
*/
-#include "src/core/NEON/kernels/arm_gemm/utils.hpp"
+#include "arm_gemm.hpp"
#include <cstddef>
#include <cstdint>
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_3x3_s2_output2x2_mla_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_3x3_s2_output2x2_mla_depthfirst/generic.cpp
index 39ab353..34ba8ec 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_3x3_s2_output2x2_mla_depthfirst/generic.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_3x3_s2_output2x2_mla_depthfirst/generic.cpp
@@ -22,7 +22,7 @@
* SOFTWARE.
*/
-#include "src/core/NEON/kernels/arm_gemm/utils.hpp"
+#include "arm_gemm.hpp"
#include <cstddef>
#include <cstdint>
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_5x5_s1_output2x2_mla_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_5x5_s1_output2x2_mla_depthfirst/generic.cpp
index 7f42726..441da6d 100644
--- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_5x5_s1_output2x2_mla_depthfirst/generic.cpp
+++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_5x5_s1_output2x2_mla_depthfirst/generic.cpp
@@ -22,7 +22,7 @@
* SOFTWARE.
*/
-#include "src/core/NEON/kernels/arm_gemm/utils.hpp"
+#include "arm_gemm.hpp"
#include <cstddef>
#include <cstdint>
diff --git a/src/core/NEON/kernels/arm_gemm/utils.hpp b/src/core/NEON/kernels/arm_gemm/utils.hpp
index 0aca0e6..18e124b 100644
--- a/src/core/NEON/kernels/arm_gemm/utils.hpp
+++ b/src/core/NEON/kernels/arm_gemm/utils.hpp
@@ -27,8 +27,8 @@
#include "arm_gemm.hpp"
#include <cstddef>
-#include <tuple>
#include <limits>
+#include <tuple>
// Macro for unreachable code (e.g. impossible default cases on switch)
#define UNREACHABLE(why) __builtin_unreachable()