Georgios Pinitas | 30271c7 | 2019-06-24 14:56:34 +0100 | [diff] [blame] | 1 | /* |
Michele Di Giorgio | d9eaf61 | 2020-07-08 11:12:57 +0100 | [diff] [blame^] | 2 | * Copyright (c) 2019 Arm Limited. |
Georgios Pinitas | 30271c7 | 2019-06-24 14:56:34 +0100 | [diff] [blame] | 3 | * |
| 4 | * SPDX-License-Identifier: MIT |
| 5 | * |
| 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy |
| 7 | * of this software and associated documentation files (the "Software"), to |
| 8 | * deal in the Software without restriction, including without limitation the |
| 9 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or |
| 10 | * sell copies of the Software, and to permit persons to whom the Software is |
| 11 | * furnished to do so, subject to the following conditions: |
| 12 | * |
| 13 | * The above copyright notice and this permission notice shall be included in |
| 14 | * all |
| 15 | * copies or substantial portions of the Software. |
| 16 | * |
| 17 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
| 18 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
| 19 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE |
| 20 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
| 21 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
| 22 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
| 23 | * SOFTWARE. |
| 24 | */ |
| 25 | |
| 26 | #include "depthwise_dilated.hpp" |
| 27 | #include "utils.hpp" |
| 28 | |
| 29 | #define MEMBERFN(TOUT) \ |
| 30 | template <unsigned int OutputTileRows, unsigned int OutputTileColumns, \ |
| 31 | unsigned int KernelRows, unsigned int KernelColumns, \ |
| 32 | unsigned int StrideRows, unsigned int StrideColumns, typename TIn, \ |
| 33 | typename TBias, typename TOut> \ |
| 34 | TOUT DilatedDepthwiseConvolution<OutputTileRows, OutputTileColumns, \ |
| 35 | KernelRows, KernelColumns, StrideRows, \ |
| 36 | StrideColumns, TIn, TBias, TOut> |
| 37 | |
| 38 | namespace depthwise { |
| 39 | |
| 40 | MEMBERFN() |
| 41 | ::DilatedDepthwiseConvolution(const int n_batches, const int n_input_rows, |
| 42 | const int n_input_cols, const int n_channels, |
| 43 | const int dilation_factor, |
| 44 | nck::ActivationFunction activation, |
| 45 | const unsigned int padding_top, |
| 46 | const unsigned int padding_left, |
| 47 | const unsigned int padding_bottom, |
| 48 | const unsigned int padding_right) |
| 49 | : DilatedDepthwiseConvolution( |
| 50 | n_batches, n_input_rows, n_input_cols, n_channels, dilation_factor, |
| 51 | DilatedDepthwiseConvolution::get_output_size( |
| 52 | n_input_rows, padding_top, padding_bottom, dilation_factor), |
| 53 | DilatedDepthwiseConvolution::get_output_size( |
| 54 | n_input_cols, padding_left, padding_right, dilation_factor), |
| 55 | activation, padding_top, padding_left, padding_bottom, |
| 56 | padding_right) {} |
| 57 | |
| 58 | MEMBERFN() |
| 59 | ::DilatedDepthwiseConvolution(const int n_batches, const int n_input_rows, |
| 60 | const int n_input_cols, const int n_channels, |
| 61 | const int dilation_factor, |
| 62 | const int n_output_rows, const int n_output_cols, |
| 63 | nck::ActivationFunction activation, |
| 64 | const unsigned int padding_top, |
| 65 | const unsigned int padding_left, |
| 66 | const unsigned int, // padding_bottom |
| 67 | const unsigned int // padding_right |
| 68 | ) |
| 69 | : DilatedDepthwiseConvolution( |
| 70 | n_batches, n_input_rows, n_input_cols, n_channels, dilation_factor, |
| 71 | n_output_rows, n_output_cols, activation, padding_top, padding_left, |
| 72 | 0, 0, |
| 73 | // Function which creates a new (standard) depthwise convolution |
| 74 | [](const int n_batches, const int n_input_rows, |
| 75 | const int n_input_cols, const int n_channels, |
| 76 | const int n_output_rows, const int n_output_cols, |
| 77 | const nck::ActivationFunction activation, |
| 78 | const unsigned int padding_top, const unsigned int padding_left, |
| 79 | const unsigned int padding_bottom, |
| 80 | const unsigned int padding_right) -> IDepthwiseConvolution * { |
| 81 | return new DepthwiseConvolution< |
| 82 | OutputTileRows, OutputTileColumns, KernelRows, KernelColumns, |
| 83 | StrideRows, StrideColumns, TIn, TBias, TOut>( |
| 84 | n_batches, n_input_rows, n_input_cols, n_channels, |
| 85 | n_output_rows, n_output_cols, activation, padding_top, |
| 86 | padding_left, padding_bottom, padding_right); |
| 87 | }) {} |
| 88 | |
| 89 | MEMBERFN() |
| 90 | ::DilatedDepthwiseConvolution( |
| 91 | const int n_batches, const int n_input_rows, const int n_input_cols, |
| 92 | const int n_channels, const int dilation_factor, const int n_output_rows, |
| 93 | const int n_output_cols, nck::ActivationFunction activation, |
| 94 | const unsigned int padding_top, const unsigned int padding_left, |
| 95 | const unsigned int, // padding_bottom |
| 96 | const unsigned int, // padding_right |
| 97 | std::function<IDepthwiseConvolution *( |
| 98 | int, int, int, int, int, int, nck::ActivationFunction, unsigned int, |
| 99 | unsigned int, unsigned int, unsigned int)> |
| 100 | subconvfn // Function to create a new convolution |
| 101 | ) |
| 102 | : _dilation_factor(dilation_factor), _n_input_rows(n_input_rows), |
| 103 | _n_input_cols(n_input_cols), _n_channels(n_channels), |
| 104 | _padding_top(static_cast<int>(padding_top)), |
| 105 | _padding_left(static_cast<int>(padding_left)), |
| 106 | _n_output_rows(n_output_rows), _n_output_cols(n_output_cols), |
| 107 | _convs(_dilation_factor) { |
| 108 | // Instantiate the base convolutions |
Michalis Spyrou | 84dca2d | 2019-10-18 13:34:54 +0100 | [diff] [blame] | 109 | for (uint32_t i = 0; i < static_cast<uint32_t>(_dilation_factor); i++) { |
Georgios Pinitas | 30271c7 | 2019-06-24 14:56:34 +0100 | [diff] [blame] | 110 | // Compute properties of this row of base convolutions |
| 111 | const int row_top = |
| 112 | i * StrideRows - _padding_top; // -ve values are in the padding |
| 113 | const int row_pad_top = |
| 114 | row_top < 0 ? iceildiv(-row_top, dilation_factor) : 0; |
| 115 | |
| 116 | const int _n_input_rows = iceildiv(n_input_rows - i, dilation_factor); |
| 117 | const int _n_output_rows = iceildiv(n_output_rows - i, dilation_factor); |
| 118 | |
Michalis Spyrou | 84dca2d | 2019-10-18 13:34:54 +0100 | [diff] [blame] | 119 | for (uint32_t j = 0; j < static_cast<uint32_t>(_dilation_factor); j++) { |
Georgios Pinitas | 30271c7 | 2019-06-24 14:56:34 +0100 | [diff] [blame] | 120 | // Compute properties of the base convolution |
| 121 | const int col_left = |
| 122 | j * StrideColumns - padding_left; // -ve values are in the padding |
| 123 | const int col_pad_left = |
| 124 | col_left < 0 ? iceildiv(-col_left, dilation_factor) : 0; |
| 125 | |
| 126 | const int _n_input_cols = iceildiv(n_input_cols - j, dilation_factor); |
| 127 | const int _n_output_cols = iceildiv(n_output_cols - j, dilation_factor); |
| 128 | |
| 129 | // Create new depthwise convolution engine and include it in the vector |
| 130 | // of engines. The new depthwise convolution engine is created by calling |
| 131 | // the delegate function we received as an argument. |
| 132 | _convs[i].emplace_back(subconvfn( |
| 133 | n_batches, _n_input_rows, _n_input_cols, n_channels, _n_output_rows, |
| 134 | _n_output_cols, activation, |
| 135 | // Note: since we have computed the output tensor size we don't need |
| 136 | // to explicitly provide bottom and right padding values to the |
| 137 | // depthwise convolution. |
| 138 | row_pad_top, col_pad_left, 0, 0)); |
| 139 | } |
| 140 | } |
| 141 | } |
| 142 | |
| 143 | MEMBERFN(void)::set_input(const void *const inptr) { |
| 144 | set_input(inptr, _n_channels); |
| 145 | } |
| 146 | |
| 147 | MEMBERFN(void)::set_input(const void *const inptr, const int ldcol) { |
| 148 | set_input(inptr, _n_input_cols * ldcol, ldcol); |
| 149 | } |
| 150 | |
| 151 | MEMBERFN(void) |
| 152 | ::set_input(const void *const inptr, const int ldrow, const int ldcol) { |
| 153 | set_input(inptr, _n_input_rows * ldrow, ldrow, ldcol); |
| 154 | } |
| 155 | |
| 156 | MEMBERFN(void) |
| 157 | ::set_input(const void *const inptr, const int ldbatch, const int ldrow, |
| 158 | const int ldcol) { |
| 159 | // Compute dilated strides |
| 160 | const int ldrow_dilated = ldrow * _dilation_factor; |
| 161 | const int ldcol_dilated = ldcol * _dilation_factor; |
| 162 | |
| 163 | // Pass input parameters on to base convolutions |
Michalis Spyrou | 84dca2d | 2019-10-18 13:34:54 +0100 | [diff] [blame] | 164 | for (uint32_t i = 0; i < static_cast<uint32_t>(_dilation_factor); i++) { |
Georgios Pinitas | 30271c7 | 2019-06-24 14:56:34 +0100 | [diff] [blame] | 165 | const int top_pos = |
| 166 | i * StrideRows - _padding_top + |
| 167 | ((static_cast<int>(i * StrideRows) < _padding_top) |
| 168 | ? iceildiv(_padding_top - i * StrideRows, _dilation_factor) * |
| 169 | _dilation_factor |
| 170 | : 0); |
| 171 | const TIn *const inptr_i = |
| 172 | static_cast<const TIn *>(inptr) + top_pos * ldrow; |
| 173 | |
Michalis Spyrou | 84dca2d | 2019-10-18 13:34:54 +0100 | [diff] [blame] | 174 | for (uint32_t j = 0; j < static_cast<uint32_t>(_dilation_factor); j++) { |
Georgios Pinitas | 30271c7 | 2019-06-24 14:56:34 +0100 | [diff] [blame] | 175 | int left_pos = j * StrideColumns - _padding_left; |
| 176 | while (left_pos < 0) |
| 177 | left_pos += _dilation_factor; |
| 178 | |
| 179 | // Modify the pointer to point to the first element of the dilated input |
| 180 | // tensor, then set the input for this convolution engine. |
| 181 | const void *const inptr_ij = inptr_i + left_pos * ldcol; |
| 182 | _convs[i][j]->set_input(inptr_ij, ldbatch, ldrow_dilated, ldcol_dilated); |
| 183 | } |
| 184 | } |
| 185 | } |
| 186 | |
| 187 | MEMBERFN(void)::set_output(void *const outptr) { |
| 188 | set_output(outptr, _n_channels); |
| 189 | } |
| 190 | |
| 191 | MEMBERFN(void)::set_output(void *const outptr, const int ldcol) { |
| 192 | set_output(outptr, _n_output_cols * ldcol, ldcol); |
| 193 | } |
| 194 | |
| 195 | MEMBERFN(void) |
| 196 | ::set_output(void *const outptr, const int ldrow, const int ldcol) { |
| 197 | set_output(outptr, _n_output_rows * ldrow, ldrow, ldcol); |
| 198 | } |
| 199 | |
| 200 | MEMBERFN(void) |
| 201 | ::set_output(void *const outptr, const int ldbatch, const int ldrow, |
| 202 | const int ldcol) { |
| 203 | // Compute dilated strides |
| 204 | const int ldrow_dilated = ldrow * _dilation_factor; |
| 205 | const int ldcol_dilated = ldcol * _dilation_factor; |
| 206 | |
| 207 | // Pass input parameters on to base convolutions |
Michalis Spyrou | 84dca2d | 2019-10-18 13:34:54 +0100 | [diff] [blame] | 208 | for (uint32_t i = 0; i < static_cast<uint32_t>(_dilation_factor); i++) { |
| 209 | for (uint32_t j = 0; j < static_cast<uint32_t>(_dilation_factor); j++) { |
Georgios Pinitas | 30271c7 | 2019-06-24 14:56:34 +0100 | [diff] [blame] | 210 | // Modify the pointer to point to the first element of the dilated input |
| 211 | // tensor, then set the input for this convolution engine. |
| 212 | void *const outptr_ij = |
| 213 | static_cast<TOut *>(outptr) + i * ldrow + j * ldcol; |
| 214 | _convs[i][j]->set_output(outptr_ij, ldbatch, ldrow_dilated, |
| 215 | ldcol_dilated); |
| 216 | } |
| 217 | } |
| 218 | } |
| 219 | |
| 220 | MEMBERFN(int) |
| 221 | ::get_output_size(const int dim_size, const unsigned int padding_before, |
| 222 | const unsigned int padding_after, const int dilation_factor) { |
| 223 | const int input_size = |
| 224 | dim_size + static_cast<int>(padding_before + padding_after); |
| 225 | const int window_size = (KernelRows - 1) * dilation_factor + 1; |
| 226 | return iceildiv(input_size - window_size + 1, StrideRows); |
| 227 | } |
| 228 | |
| 229 | MEMBERFN(int) |
| 230 | ::output_size(const int dim_size, const unsigned int padding_before, |
| 231 | const unsigned int padding_after) const { |
| 232 | return get_output_size(dim_size, padding_before, padding_after, |
| 233 | _dilation_factor); |
| 234 | } |
| 235 | |
| 236 | MEMBERFN(size_t)::get_packed_params_size(void) const { |
| 237 | return _convs[0][0]->get_packed_params_size(); |
| 238 | } |
| 239 | |
| 240 | MEMBERFN(void)::set_packed_params_buffer(void *buffer) { |
| 241 | // Set the buffer for all convolution engines |
| 242 | for (auto &&row : _convs) { |
| 243 | for (auto &&conv : row) { |
| 244 | conv->set_packed_params_buffer(buffer); |
| 245 | } |
| 246 | } |
| 247 | } |
| 248 | |
| 249 | MEMBERFN(void) |
| 250 | ::pack_params(const void *const weights, const void *const biases) const { |
| 251 | _convs[0][0]->pack_params(weights, biases); |
| 252 | } |
| 253 | |
| 254 | MEMBERFN(void) |
| 255 | ::pack_params(void *const buffer, const void *const weights, |
| 256 | const void *const biases) const { |
| 257 | _convs[0][0]->pack_params(buffer, weights, biases); |
| 258 | } |
| 259 | |
| 260 | MEMBERFN(void) |
| 261 | ::pack_params(void *const buffer, const void *const weights, |
| 262 | const unsigned int ldrow, const unsigned int ldcol, |
| 263 | const void *const biases) const { |
| 264 | _convs[0][0]->pack_params(buffer, weights, ldrow, ldcol, biases); |
| 265 | } |
| 266 | |
| 267 | MEMBERFN(size_t)::get_working_space_size(unsigned int nthreads) const { |
| 268 | return _convs[0][0]->get_working_space_size(nthreads); |
| 269 | } |
| 270 | |
| 271 | MEMBERFN(void)::set_working_space(void *const ws) { |
| 272 | // Use the same working space set for all contained depthwise engines. |
| 273 | for (auto &&row : _convs) { |
| 274 | for (auto &&conv : row) { |
| 275 | conv->set_working_space(ws); |
| 276 | } |
| 277 | } |
| 278 | } |
| 279 | |
| 280 | MEMBERFN(unsigned int)::get_window(void) const { |
| 281 | return _convs[0][0]->get_window(); |
| 282 | } |
| 283 | |
| 284 | MEMBERFN(void) |
| 285 | ::run(const unsigned int start, const unsigned int stop, |
| 286 | const unsigned int threadid) { |
| 287 | // Run each contained convolution in turn |
| 288 | for (auto &&row : _convs) { |
| 289 | for (auto &&conv : row) { |
| 290 | conv->run(start, stop, threadid); |
| 291 | } |
| 292 | } |
| 293 | } |
| 294 | |
| 295 | } // namespace depthwise |