Gunes Bayir | 038fe91 | 2023-08-11 12:50:31 +0100 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (c) 2023 Arm Limited. |
| 3 | * |
| 4 | * SPDX-License-Identifier: MIT |
| 5 | * |
| 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy |
| 7 | * of this software and associated documentation files (the "Software"), to |
| 8 | * deal in the Software without restriction, including without limitation the |
| 9 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or |
| 10 | * sell copies of the Software, and to permit persons to whom the Software is |
| 11 | * furnished to do so, subject to the following conditions: |
| 12 | * |
| 13 | * The above copyright notice and this permission notice shall be included in all |
| 14 | * copies or substantial portions of the Software. |
| 15 | * |
| 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
| 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
| 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE |
| 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
| 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
| 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
| 22 | * SOFTWARE. |
| 23 | */ |
| 24 | #include "src/cl/helpers/CLMemoryOpImage2dHelper.h" |
| 25 | |
| 26 | #include "ckw/Error.h" |
| 27 | #include "ckw/TensorSampler.h" |
| 28 | #include "ckw/types/MemoryOperation.h" |
| 29 | #include "ckw/types/TensorStorageType.h" |
| 30 | |
Gunes Bayir | 038fe91 | 2023-08-11 12:50:31 +0100 | [diff] [blame] | 31 | #include "src/cl/CLKernelWriter.h" |
| 32 | #include "src/cl/CLTensorArgument.h" |
| 33 | #include "src/cl/CLTile.h" |
Felix Thomasmathibalan | afd38f0 | 2023-09-27 17:46:17 +0100 | [diff] [blame] | 34 | #include "src/ITensor.h" |
| 35 | #include "src/Tensor3dMapper.h" |
Gunes Bayir | 2b9fa59 | 2024-01-17 16:07:03 +0000 | [diff] [blame] | 36 | #include "src/TileView.h" |
Gunes Bayir | 038fe91 | 2023-08-11 12:50:31 +0100 | [diff] [blame] | 37 | |
| 38 | namespace ckw |
| 39 | { |
Gunes Bayir | 2b9fa59 | 2024-01-17 16:07:03 +0000 | [diff] [blame] | 40 | void CLMemoryOpImage2dHelper::initialize(const CLTile *x, const CLTile *z, const CLTile *b) |
Gunes Bayir | 038fe91 | 2023-08-11 12:50:31 +0100 | [diff] [blame] | 41 | { |
Gunes Bayir | 2b9fa59 | 2024-01-17 16:07:03 +0000 | [diff] [blame] | 42 | _coord_x = x->scalar(0, 0).str; |
| 43 | _coord_z = z->scalar(0, 0).str; |
| 44 | _coord_b = b->scalar(0, 0).str; |
Gunes Bayir | 038fe91 | 2023-08-11 12:50:31 +0100 | [diff] [blame] | 45 | } |
| 46 | |
| 47 | void CLMemoryOpImage2dHelper::write_row(int32_t row_id, const std::string &coord_y) |
| 48 | { |
| 49 | // The only check required is on Y. |
| 50 | out_of_bound_initialize_y(coord_y); |
| 51 | |
Gunes Bayir | 2b9fa59 | 2024-01-17 16:07:03 +0000 | [diff] [blame] | 52 | const std::string dst = _dst.vector(row_id).str; |
Gunes Bayir | 038fe91 | 2023-08-11 12:50:31 +0100 | [diff] [blame] | 53 | const std::string sampler = to_ls_image2d_sampler(); |
| 54 | const std::string coord = to_ls_image2d_address(_coord_x, coord_y, _coord_z, _coord_b); |
| 55 | const std::string ls_buf = to_ls_image2d(_op, _ls_width_full, dst, sampler, coord); |
| 56 | |
| 57 | _writer->op_write_raw_code(ls_buf + ";\n"); |
| 58 | |
| 59 | out_of_bound_finalize_y(); |
| 60 | } |
| 61 | |
| 62 | void CLMemoryOpImage2dHelper::finalize() |
| 63 | { |
| 64 | } |
| 65 | |
Gunes Bayir | 2b9fa59 | 2024-01-17 16:07:03 +0000 | [diff] [blame] | 66 | bool CLMemoryOpImage2dHelper::validate(const CLKernelWriter *writer, |
| 67 | const ITensor *tensor, |
| 68 | const TensorSampler *sampler, |
| 69 | const Tensor3dMapper *mapper, |
| 70 | MemoryOperation op, |
| 71 | const TileView<CLTile> &dst) |
Gunes Bayir | 038fe91 | 2023-08-11 12:50:31 +0100 | [diff] [blame] | 72 | { |
| 73 | CKW_UNUSED(writer, tensor, mapper); |
| 74 | |
Gunes Bayir | 2b9fa59 | 2024-01-17 16:07:03 +0000 | [diff] [blame] | 75 | if (dst.width() != 4) |
Gunes Bayir | 038fe91 | 2023-08-11 12:50:31 +0100 | [diff] [blame] | 76 | { |
| 77 | return false; |
| 78 | } |
Felix Thomasmathibalan | afd38f0 | 2023-09-27 17:46:17 +0100 | [diff] [blame] | 79 | if (sampler->address_mode_x() != TensorSamplerAddressModeX::None) |
Gunes Bayir | 038fe91 | 2023-08-11 12:50:31 +0100 | [diff] [blame] | 80 | { |
| 81 | return false; |
| 82 | } |
Felix Thomasmathibalan | afd38f0 | 2023-09-27 17:46:17 +0100 | [diff] [blame] | 83 | if (sampler->address_mode_z() != TensorSamplerAddressModeZ::None) |
Gunes Bayir | 038fe91 | 2023-08-11 12:50:31 +0100 | [diff] [blame] | 84 | { |
| 85 | return false; |
| 86 | } |
Felix Thomasmathibalan | afd38f0 | 2023-09-27 17:46:17 +0100 | [diff] [blame] | 87 | if (sampler->storage() != TensorStorageType::Texture2dReadOnly && op == MemoryOperation::Load) |
Gunes Bayir | 038fe91 | 2023-08-11 12:50:31 +0100 | [diff] [blame] | 88 | { |
| 89 | return false; |
| 90 | } |
Felix Thomasmathibalan | afd38f0 | 2023-09-27 17:46:17 +0100 | [diff] [blame] | 91 | if (sampler->storage() != TensorStorageType::Texture2dWriteOnly && op == MemoryOperation::Store) |
Gunes Bayir | 038fe91 | 2023-08-11 12:50:31 +0100 | [diff] [blame] | 92 | { |
| 93 | return false; |
| 94 | } |
Gunes Bayir | 2b9fa59 | 2024-01-17 16:07:03 +0000 | [diff] [blame] | 95 | if ((dst.data_type() != DataType::Fp32) && (dst.data_type() != DataType::Fp16)) |
Gunes Bayir | 038fe91 | 2023-08-11 12:50:31 +0100 | [diff] [blame] | 96 | { |
| 97 | return false; |
| 98 | } |
| 99 | return true; |
| 100 | } |
| 101 | |
| 102 | void CLMemoryOpImage2dHelper::out_of_bound_initialize_y(const std::string &coord) |
| 103 | { |
| 104 | CKW_UNUSED(coord); |
| 105 | |
| 106 | const TensorSamplerAddressModeY address_mode_y = _sampler->address_mode_y(); |
Felix Thomasmathibalan | afd38f0 | 2023-09-27 17:46:17 +0100 | [diff] [blame] | 107 | switch (address_mode_y) |
Gunes Bayir | 038fe91 | 2023-08-11 12:50:31 +0100 | [diff] [blame] | 108 | { |
Gunes Bayir | d5f9a1c | 2023-08-17 11:04:02 +0100 | [diff] [blame] | 109 | case TensorSamplerAddressModeY::SkipLessThanZero: |
| 110 | _writer->op_write_raw_code("if(" + coord + " >= 0)\n{\n"); |
| 111 | break; |
Gunes Bayir | 038fe91 | 2023-08-11 12:50:31 +0100 | [diff] [blame] | 112 | case TensorSamplerAddressModeY::ClampToBorderMaxOnly: |
| 113 | case TensorSamplerAddressModeY::None: |
| 114 | break; |
| 115 | default: |
| 116 | CKW_THROW_MSG("Unsupported address mode for Y dimension"); |
| 117 | } |
| 118 | } |
| 119 | |
| 120 | void CLMemoryOpImage2dHelper::out_of_bound_finalize_y() |
| 121 | { |
| 122 | const TensorSamplerAddressModeY address_mode_y = _sampler->address_mode_y(); |
Felix Thomasmathibalan | afd38f0 | 2023-09-27 17:46:17 +0100 | [diff] [blame] | 123 | switch (address_mode_y) |
Gunes Bayir | 038fe91 | 2023-08-11 12:50:31 +0100 | [diff] [blame] | 124 | { |
Gunes Bayir | d5f9a1c | 2023-08-17 11:04:02 +0100 | [diff] [blame] | 125 | case TensorSamplerAddressModeY::SkipLessThanZero: |
| 126 | _writer->op_write_raw_code("}\n"); |
| 127 | break; |
Gunes Bayir | 038fe91 | 2023-08-11 12:50:31 +0100 | [diff] [blame] | 128 | case TensorSamplerAddressModeY::ClampToBorderMaxOnly: |
| 129 | case TensorSamplerAddressModeY::None: |
| 130 | break; |
| 131 | default: |
| 132 | CKW_THROW_MSG("Unsupported address mode for Y dimension"); |
| 133 | } |
| 134 | } |
| 135 | |
Felix Thomasmathibalan | afd38f0 | 2023-09-27 17:46:17 +0100 | [diff] [blame] | 136 | std::string CLMemoryOpImage2dHelper::to_ls_image2d(MemoryOperation op, |
| 137 | int32_t vector_width, |
| 138 | const std::string &data, |
| 139 | const std::string &sampler, |
| 140 | const std::string &address) const |
Gunes Bayir | 038fe91 | 2023-08-11 12:50:31 +0100 | [diff] [blame] | 141 | { |
| 142 | CKW_UNUSED(vector_width); |
Gunes Bayir | 2b9fa59 | 2024-01-17 16:07:03 +0000 | [diff] [blame] | 143 | CKW_ASSERT_MSG(_dst.data_type() == DataType::Fp32 || _dst.data_type() == DataType::Fp16, |
| 144 | "Image2d only supports floating-point data type"); |
Gunes Bayir | 038fe91 | 2023-08-11 12:50:31 +0100 | [diff] [blame] | 145 | |
| 146 | const TensorStorageType tensor_storage = _sampler->storage(); |
Felix Thomasmathibalan | afd38f0 | 2023-09-27 17:46:17 +0100 | [diff] [blame] | 147 | const std::string image2d_obj = _tensor->storage(tensor_storage).val; |
Gunes Bayir | 2b9fa59 | 2024-01-17 16:07:03 +0000 | [diff] [blame] | 148 | const std::string post_fix = _dst.data_type() == DataType::Fp32 ? "f" : "h"; |
Gunes Bayir | 038fe91 | 2023-08-11 12:50:31 +0100 | [diff] [blame] | 149 | |
Felix Thomasmathibalan | afd38f0 | 2023-09-27 17:46:17 +0100 | [diff] [blame] | 150 | switch (op) |
Gunes Bayir | 038fe91 | 2023-08-11 12:50:31 +0100 | [diff] [blame] | 151 | { |
| 152 | case MemoryOperation::Load: |
| 153 | return data + " = read_image" + post_fix + "(" + image2d_obj + ", " + sampler + ", " + address + ")"; |
| 154 | break; |
| 155 | case MemoryOperation::Store: |
| 156 | return "write_image" + post_fix + "(" + image2d_obj + ", " + address + ", " + data + ")"; |
| 157 | default: |
| 158 | CKW_THROW_MSG("Unsupported MemoryOperation"); |
| 159 | } |
| 160 | } |
| 161 | |
| 162 | std::string CLMemoryOpImage2dHelper::to_ls_image2d_sampler() const |
| 163 | { |
| 164 | const auto address_mode_y = _sampler->address_mode_y(); |
| 165 | |
Felix Thomasmathibalan | afd38f0 | 2023-09-27 17:46:17 +0100 | [diff] [blame] | 166 | switch (address_mode_y) |
Gunes Bayir | 038fe91 | 2023-08-11 12:50:31 +0100 | [diff] [blame] | 167 | { |
| 168 | case TensorSamplerAddressModeY::None: |
| 169 | return "CLK_NORMALIZED_COORDS_FALSE | CLK_ADDRESS_NONE | CLK_FILTER_NEAREST"; |
Gunes Bayir | d5f9a1c | 2023-08-17 11:04:02 +0100 | [diff] [blame] | 170 | case TensorSamplerAddressModeY::SkipLessThanZero: |
Gunes Bayir | 038fe91 | 2023-08-11 12:50:31 +0100 | [diff] [blame] | 171 | case TensorSamplerAddressModeY::ClampToBorderMaxOnly: |
| 172 | return "CLK_NORMALIZED_COORDS_FALSE | CLK_ADDRESS_CLAMP | CLK_FILTER_NEAREST"; |
| 173 | default: |
| 174 | CKW_THROW_MSG("Unsupported address_mode_coord"); |
| 175 | } |
| 176 | } |
| 177 | |
Felix Thomasmathibalan | afd38f0 | 2023-09-27 17:46:17 +0100 | [diff] [blame] | 178 | std::string CLMemoryOpImage2dHelper::to_ls_image2d_address(const std::string &x, |
| 179 | const std::string &y, |
| 180 | const std::string &z, |
Gunes Bayir | 038fe91 | 2023-08-11 12:50:31 +0100 | [diff] [blame] | 181 | const std::string &b) const |
| 182 | { |
| 183 | std::string coord_x = "(" + x + ") >> 2"; |
| 184 | std::string coord_y = "("; |
| 185 | |
Felix Thomasmathibalan | afd38f0 | 2023-09-27 17:46:17 +0100 | [diff] [blame] | 186 | if (y != "0") |
Gunes Bayir | 038fe91 | 2023-08-11 12:50:31 +0100 | [diff] [blame] | 187 | { |
| 188 | coord_y += y; |
| 189 | } |
Felix Thomasmathibalan | afd38f0 | 2023-09-27 17:46:17 +0100 | [diff] [blame] | 190 | if (z != "0" && (_mapper->dim_z().str != "1")) |
Gunes Bayir | 038fe91 | 2023-08-11 12:50:31 +0100 | [diff] [blame] | 191 | { |
| 192 | const std::string dim = _mapper->dim_y().str; |
| 193 | coord_y += " + ("; |
| 194 | coord_y += z + ")"; |
| 195 | coord_y += " * "; |
| 196 | coord_y += dim; |
| 197 | } |
Felix Thomasmathibalan | afd38f0 | 2023-09-27 17:46:17 +0100 | [diff] [blame] | 198 | if (b != "0" && (_mapper->dim_batch().str != "1")) |
Gunes Bayir | 038fe91 | 2023-08-11 12:50:31 +0100 | [diff] [blame] | 199 | { |
| 200 | const std::string dim0 = _mapper->dim_y().str; |
| 201 | const std::string dim1 = _mapper->dim_z().str; |
| 202 | coord_y += " + ("; |
| 203 | coord_y += b + ")"; |
| 204 | coord_y += " * "; |
| 205 | coord_y += dim0; |
| 206 | coord_y += " * "; |
| 207 | coord_y += dim1; |
| 208 | } |
| 209 | coord_y += ")"; |
| 210 | return "(int2)(" + coord_x + ", " + coord_y + ")"; |
| 211 | } |
| 212 | |
| 213 | } // namespace ckw |