src/core/NEON/kernels/arm_conv/depthwise/depthwise_depthfirst_generic.hpp

/*
 * Copyright (c) 2021 Arm Limited.
 *
 * SPDX-License-Identifier: MIT
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to
 * deal in the Software without restriction, including without limitation the
 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
 * sell copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

#pragma once

#include "src/core/NEON/kernels/arm_gemm/utils.hpp"

#ifdef CYCLE_PROFILING
#include "profiler.hpp"
#endif

#include <limits>

namespace arm_conv {
namespace depthwise {

template <class Strategy, unsigned OutputRows, unsigned int OutputCols>
class DepthwiseDepthfirstGenericBase :
  public DepthwiseCommon<typename Strategy::input_type,
                         typename Strategy::weight_type,
                         typename Strategy::return_type>
{
  protected:

  using TInput = typename Strategy::input_type;
  using TWeight = typename Strategy::weight_type;
  using TOutput = typename Strategy::return_type;
  using TAccum = typename Strategy::bias_type;

  size_t sizeof_input_ptr_array(void) const
  {
    return sizeof(TInput *) * this->m_args.kernel_rows * this->m_args.kernel_cols * Strategy::n_output_points;
  }

  size_t sizeof_input_buffer(unsigned int n_channels) const
  {
    const unsigned int vl = arm_gemm::utils::get_vector_length<TInput>(Strategy::vl_type);
    const auto rounded_channels = arm_gemm::roundup(n_channels, vl);
    return sizeof(TInput) * rounded_channels;
  }

  size_t sizeof_output_buffer(unsigned int n_channels) const
  {
    const unsigned int vl = arm_gemm::utils::get_vector_length<TOutput>(Strategy::vl_type);
    const auto rounded_channels = arm_gemm::roundup(n_channels, vl);
    return sizeof(TOutput) * rounded_channels;
  }

  unsigned int input_rows(void) const
  {
    return this->m_args.kernel_rows + (OutputRows - 1)*this->m_args.stride_rows;
  }

  unsigned int input_cols(void) const
  {
    return this->m_args.kernel_cols + (OutputCols - 1)*this->m_args.stride_cols;
  }

  void execute_tiles(
    std::function<void(const TInput *const *, TOutput *const *)> tile_fn,
    std::function<void(TInput *, unsigned int)> initialise_input_buffer,
    const unsigned int batches,
    const unsigned int input_height,
    const unsigned int input_width,
    const unsigned int input_channels,
    const PaddingValues &padding,
    const void *const _input,
    const size_t ld_input_col,
    const size_t ld_input_row,
    const size_t ld_input_batch,
    const unsigned int output_height,
    const unsigned int output_width,
    void *const _output,
    const size_t ld_output_col,
    const size_t ld_output_row,
    const size_t ld_output_batch,
    void *const _working_space,
    const unsigned int thread_id,
    const unsigned int n_threads
  ) const
  {
    static_assert(OutputRows * OutputCols <= Strategy::n_output_points,
                  "Too many output points for kernel.");

    // Determine what portion of the work to do.
    const unsigned int n_rows_per_thread = arm_gemm::iceildiv(output_height, n_threads);
    const int start_out_height = std::min(thread_id * n_rows_per_thread, output_height);
    const int end_out_height = std::min(start_out_height + n_rows_per_thread, output_height);

    // Cast input and output pointers into the right types
    const TInput *const inptr = static_cast<const TInput *>(_input);
    TOutput *const outptr = static_cast<TOutput *>(_output);

    // Allocate portions of the working space
    uint8_t *const working_space = static_cast<uint8_t *>(_working_space) + this->get_working_size(thread_id, input_channels);
    const TInput **const inptr_array = reinterpret_cast<const TInput **>(working_space);
    TOutput *const output_buffer = reinterpret_cast<TOutput *>(working_space + this->sizeof_input_ptr_array());
    TInput *const input_buffer = reinterpret_cast<TInput *>(working_space + this->sizeof_input_ptr_array() + this->sizeof_output_buffer(input_channels * this->m_args.channel_multiplier));

    // Create an array for the output pointers
    TOutput * _outptr_array[Strategy::n_output_points];
    TOutput **const outptr_array = _outptr_array;

    // Initialise the input buffer
    initialise_input_buffer(input_buffer, input_channels);

    // For each output tile, construct the requisite set of pointers and call
    // into the kernel.
    for (unsigned int batch = 0; batch < batches; batch++)
    {
      // Get batch pointers
      const auto inptr_batch = inptr + batch * ld_input_batch;
      const auto outptr_batch = outptr + batch * ld_output_batch;

      for (int start_out_i = start_out_height;
           start_out_i < end_out_height;
           start_out_i += static_cast<int>(OutputRows))
      {
        const int end_out_i = std::min(start_out_i + OutputRows,
                                       output_height);

        for (int start_out_j = 0;
             start_out_j < static_cast<int>(output_width);
             start_out_j += static_cast<int>(OutputCols))
        {
          const int end_out_j = std::min(start_out_j + OutputCols,
                                         output_width);

          // Fill the pointer arrays with pointers to the input/output buffers.
          for (auto index = 0u;
               index < (Strategy::n_output_points * this->m_args.kernel_rows * this->m_args.kernel_cols);
               index++)
          {
            inptr_array[index] = input_buffer;
          }
          for (auto index = 0u; index < Strategy::n_output_points; index++)
          {
            outptr_array[index] = output_buffer;
          }

          // Construct the pointer arrays together. Note that the input pointer
          // array is striped. Since the array has already been filled with
          // pointers to the padding array we merely fill in the valid points
          // as we get to them.
          unsigned int output_index = 0;
          auto outptr_row = outptr_batch + start_out_i * ld_output_row + start_out_j * ld_output_col;
          for (auto out_i = start_out_i; out_i < end_out_i; out_i++)
          {
            auto outptr_col = outptr_row;

            // Compute the padding for this row of tiles.
            const int start_in_i = out_i * this->m_args.stride_rows - padding.top;
            const int end_in_i = start_in_i + this->m_args.kernel_rows;
            const auto pad_top = static_cast<unsigned int>(std::max<int>(0, 0 - start_in_i));
            const auto pad_bottom = static_cast<unsigned int>(std::max<int>(0, end_in_i - input_height));
            const unsigned int valid_rows = this->m_args.kernel_rows - pad_top - pad_bottom;

            for (auto out_j = start_out_j; out_j < end_out_j; out_j++, output_index++)
            {
              // Compute the output pointer.
              outptr_array[output_index] = outptr_col;
              outptr_col += ld_output_col;

              // Compute the padding for this tile.
              const int start_in_j = out_j * this->m_args.stride_cols - padding.left;
              const int end_in_j = start_in_j + this->m_args.kernel_cols;
              const auto pad_left = static_cast<unsigned int>(std::max<int>(0, 0 - start_in_j));
              const auto pad_right = static_cast<unsigned int>(std::max<int>(0, end_in_j - input_width));
              const unsigned int valid_cols = this->m_args.kernel_cols - pad_left - pad_right;

              // Hence compute the input pointers.
              auto input_index = output_index + Strategy::n_output_points * (pad_top * this->m_args.kernel_cols + pad_left);
              auto inptr_row = inptr_batch + (start_in_i + pad_top) * ld_input_row + (start_in_j + pad_left) * ld_input_col;
              for (auto in_i = 0u; in_i < valid_rows; in_i++)
              {
                auto inptr_col = inptr_row;
                auto input_index_col = input_index;

                for (auto in_j = 0u; in_j < valid_cols; in_j++)
                {
                  inptr_array[input_index_col] = inptr_col;
                  inptr_col += ld_input_col;
                  input_index_col += Strategy::n_output_points;
                }

                inptr_row += ld_input_row;
                input_index += Strategy::n_output_points * this->m_args.kernel_cols;
              }
            }

            outptr_row += ld_output_row;
          }

          tile_fn(inptr_array, outptr_array);
        }
      }
    }
  }

  public:
  DepthwiseDepthfirstGenericBase(const DepthwiseArgs &args) : DepthwiseCommon<TInput, TWeight, TOutput>(args)
  {
  }

  DepthwiseDepthfirstGenericBase(DepthwiseDepthfirstGenericBase &) = delete;
  DepthwiseDepthfirstGenericBase &operator=(DepthwiseDepthfirstGenericBase &) = delete;

  size_t get_storage_size(void) const override
  {
    const unsigned int vl = arm_gemm::utils::get_vector_length<TAccum>(Strategy::vl_type);
    const auto rounded_channels = arm_gemm::roundup(this->m_args.input_channels, vl);
    return (this->m_args.kernel_rows * this->m_args.kernel_cols) * rounded_channels * sizeof(TWeight);
  }

  void pack_parameters(void *_buffer, const void *, const void *_weights, size_t ld_weight_col, size_t ld_weight_row) override
  {
    // Cast the pointers
    TWeight *buffer = static_cast<TWeight *>(_buffer);
    const TWeight *const weights = static_cast<const TWeight *>(_weights);

    const unsigned int vl = arm_gemm::utils::get_vector_length<TAccum>(Strategy::vl_type);
    ld_weight_col = (ld_weight_col == 0) ? this->m_args.input_channels : ld_weight_col;
    ld_weight_row = (ld_weight_row == 0) ? this->m_args.kernel_cols * ld_weight_col : ld_weight_row;

    for (unsigned int n = 0; n < this->m_args.input_channels; n += vl)
    {
      const unsigned int todo = std::min(vl, this->m_args.input_channels - n);

      // Copy each of the weights in turn
      auto weights_row = weights + n;
      for (unsigned int i = 0; i < this->m_args.kernel_rows; i++)
      {
        auto weights_col = weights_row;

        for (unsigned int j = 0; j < this->m_args.kernel_cols; j++)
        {
          for (unsigned int m = 0; m < todo; m++)
          {
            buffer[m] = weights_col[m];
          }
          buffer += vl;

          weights_col += ld_weight_col;
        }

        weights_row += ld_weight_row;
      }
    }
  }

  size_t get_working_size(const unsigned int n_threads, const unsigned int n_channels) const override
  {
    const unsigned int n_output_channels = n_channels * this->m_args.channel_multiplier;
    return n_threads * (sizeof_input_ptr_array() +
                        sizeof_output_buffer(n_output_channels) +
                        sizeof_input_buffer(n_channels));
  }
};

template <class Strategy, unsigned OutputRows, unsigned int OutputCols>
class DepthwiseDepthfirstGeneric : public DepthwiseDepthfirstGenericBase<Strategy, OutputRows, OutputCols>
{
  using Parent = DepthwiseDepthfirstGenericBase<Strategy, OutputRows, OutputCols>;
  using TInput = typename Parent::TInput;
  using TWeight = typename Parent::TWeight;
  using TAccum = typename Parent::TAccum;
  using TOutput = typename Parent::TOutput;

  const TAccum *m_bias = nullptr;

  public:
  DepthwiseDepthfirstGeneric(const DepthwiseArgs &args) : Parent(args)
  {
  }

  DepthwiseDepthfirstGeneric(DepthwiseDepthfirstGeneric &) = delete;
  DepthwiseDepthfirstGeneric &operator=(DepthwiseDepthfirstGeneric &) = delete;

  void pack_parameters(void *buffer, const void *bias, const void *weights, size_t ld_weight_col, size_t ld_weight_row) override
  {
    m_bias = static_cast<const TAccum *>(bias);
    Parent::pack_parameters(buffer, bias, weights, ld_weight_col, ld_weight_row);
  }

  using DepthwiseDepthfirstGenericBase<Strategy, OutputRows, OutputCols>::execute;
  void execute(
    const unsigned int batches,
    const unsigned int input_height,
    const unsigned int input_width,
    const unsigned int input_channels,
    const PaddingValues &padding,
    const void *const _input,
    const size_t ld_input_col,
    const size_t ld_input_row,
    const size_t ld_input_batch,
    const void *const parameters,
    const unsigned int output_height,
    const unsigned int output_width,
    void *const _output,
    const size_t ld_output_col,
    const size_t ld_output_row,
    const size_t ld_output_batch,
    void *const _working_space,
    const unsigned int thread_id,
    const unsigned int n_threads
  ) const override
  {
    Strategy strat(this->m_args.cpu_info);
#ifdef CYCLE_PROFILING
    arm_gemm::profiler prof;
#endif

    // Compute activation values
    TAccum activation_min, activation_max;
    if (std::numeric_limits<TAccum>::is_integer)
    {
      activation_min = std::numeric_limits<TAccum>::min();
      activation_max = std::numeric_limits<TAccum>::max();
    }
    else
    {
      activation_min = static_cast<TAccum>(-std::numeric_limits<float>::infinity());
      activation_max = static_cast<TAccum>(std::numeric_limits<float>::infinity());
    }

    switch (this->m_args.activation.type)
    {
      case arm_gemm::Activation::Type::BoundedReLU:
        activation_max = static_cast<TAccum>(this->m_args.activation.param1);
        // Fall through
      case arm_gemm::Activation::Type::ReLU:
        activation_min = static_cast<TAccum>(0);
        break;
      default:
        break;
    }

    // Create a function to initialise the input buffer
    const auto initialise_input_buffer = [] (TInput *const buffer, const unsigned int n) {
      std::memset(buffer, 0, n * sizeof(TInput));
    };

    // Create a function to execute a tile of work
    const auto tile_fn = [&] (const TInput *const *const inptrs, TOutput *const * const outptrs) {
#ifdef CYCLE_PROFILING
      auto p = prof.ScopedProfiler(
        PROFILE_KERNEL,
        (unsigned long) (OutputRows * OutputCols * this->m_args.kernel_rows* this->m_args.kernel_cols)
      );
#endif
      strat.kernel(inptrs, outptrs, parameters, m_bias,
                   this->m_args.kernel_rows * this->m_args.kernel_cols,
                   this->m_args.input_channels, activation_min, activation_max);
    };

    // Call into a parent utility function to do the actual work.
    Parent::execute_tiles(
      tile_fn, initialise_input_buffer,
      batches, input_height, input_width, input_channels, padding,
      _input, ld_input_col, ld_input_row, ld_input_batch,
      output_height, output_width,
      _output, ld_output_col, ld_output_row, ld_output_batch,
      _working_space, thread_id, n_threads
    );
  }
};

}  // namespace depthwise
}  // namespace arm_conv