Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (c) 2022 Arm Limited. |
| 3 | * |
| 4 | * SPDX-License-Identifier: MIT |
| 5 | * |
| 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy |
| 7 | * of this software and associated documentation files (the "Software"), to |
| 8 | * deal in the Software without restriction, including without limitation the |
| 9 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or |
| 10 | * sell copies of the Software, and to permit persons to whom the Software is |
| 11 | * furnished to do so, subject to the following conditions: |
| 12 | * |
| 13 | * The above copyright notice and this permission notice shall be included in all |
| 14 | * copies or substantial portions of the Software. |
| 15 | * |
| 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
| 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
| 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE |
| 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
| 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
| 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
| 22 | * SOFTWARE. |
| 23 | */ |
SiCong Li | b63b119 | 2022-01-28 18:24:39 +0000 | [diff] [blame^] | 24 | #ifndef ENABLE_EXPERIMENTAL_DYNAMIC_FUSION |
| 25 | #error "This experimental feature must be enabled with -DENABLE_EXPERIMENTAL_DYNAMIC_FUSION" |
| 26 | #endif /* ENABLE_EXPERIMENTAL_DYNAMIC_FUSION */ |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 27 | |
| 28 | #ifndef ARM_COMPUTE_EXPERIMENTAL_DYNAMICFUSION_IMPL_COMMON_H |
| 29 | #define ARM_COMPUTE_EXPERIMENTAL_DYNAMICFUSION_IMPL_COMMON_H |
| 30 | |
| 31 | #include "arm_compute/core/CL/CLCompileContext.h" |
Giorgio Arena | 892b70a | 2022-03-30 12:23:10 +0100 | [diff] [blame] | 32 | #include "arm_compute/core/CL/CLKernelLibrary.h" |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 33 | #include "arm_compute/core/Error.h" |
| 34 | #include "arm_compute/core/GPUTarget.h" |
Gunes Bayir | 8a87983 | 2022-03-10 21:21:01 +0000 | [diff] [blame] | 35 | #include "src/core/common/Macros.h" |
Gunes Bayir | 16c5697 | 2022-03-28 21:32:33 +0100 | [diff] [blame] | 36 | #include "support/Requires.h" |
Giorgio Arena | bd44caa | 2022-03-15 13:45:15 +0000 | [diff] [blame] | 37 | #include "support/StringSupport.h" |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 38 | |
| 39 | #include "src/core/experimental/dynamic_fusion/ClKernelBuildingAPI.h" |
| 40 | |
SiCong Li | b63b119 | 2022-01-28 18:24:39 +0000 | [diff] [blame^] | 41 | #include <iostream> |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 42 | #include <queue> |
| 43 | #include <stack> |
| 44 | #include <string> |
| 45 | #include <unordered_set> |
| 46 | |
| 47 | namespace arm_compute |
| 48 | { |
| 49 | namespace experimental |
| 50 | { |
| 51 | namespace dynamic_fusion |
| 52 | { |
| 53 | /** We introduce the concept of *Shared Variables* in the context of kernel building. |
| 54 | * They are variables that can be accessed / shared among all the kernel components within a single kernel. |
| 55 | * For now we consider 2 groups of shared variables: |
| 56 | * Argument: The argument variables (parameters) of a kernel |
| 57 | * Automatic: The automatic variables declared inside a kernel |
| 58 | * All Shared Variables have the same kernel scope, and are thus visible to all kernel components |
| 59 | */ |
| 60 | |
| 61 | enum class SharedVarIO |
| 62 | { |
| 63 | Input, |
| 64 | Output |
| 65 | }; |
| 66 | |
| 67 | enum class SharedVarGroup |
| 68 | { |
SiCong Li | b63b119 | 2022-01-28 18:24:39 +0000 | [diff] [blame^] | 69 | Argument, // Parameters to a kernel function == dst or src tensors of the whole blueprint graph |
| 70 | Automatic // Automatic variables declared within the kernel body == intermediate tensors of the whole blueprint graph |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 71 | }; |
| 72 | |
Gunes Bayir | 8a87983 | 2022-03-10 21:21:01 +0000 | [diff] [blame] | 73 | /** Specifies a shared variable link for a component. |
| 74 | * It describes all the information that's available when a component is constructed / added: |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 75 | * e.g. its linkage (via ArgumentID and io) and its group |
| 76 | * This is not shared variable on its own, but is used for instantiating a SharedVar when building the code |
| 77 | */ |
| 78 | struct SharedVarLink |
| 79 | { |
SiCong Li | b63b119 | 2022-01-28 18:24:39 +0000 | [diff] [blame^] | 80 | ArgumentID arg_id{ g_arg_placeholder }; |
| 81 | SharedVarIO io{ SharedVarIO::Input }; |
| 82 | bool is_empty() const |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 83 | { |
| 84 | return arg_id == g_arg_placeholder; |
| 85 | } |
| 86 | }; |
| 87 | |
| 88 | /** A table of all the variables used in the kernel / blueprint |
SiCong Li | b63b119 | 2022-01-28 18:24:39 +0000 | [diff] [blame^] | 89 | * Because we limit the DependencyGraph in the blueprint to a Linear Sequence for now, we only allow ** a single global variable (the accumulator) ** |
| 90 | * |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 91 | * NOTE: the order they appear in the table is the order of their "declaration" in the component code, and is also their ID |
| 92 | * NOTE: the variables all have the scope of the full kernel function |
| 93 | */ |
| 94 | class SharedVarTable |
| 95 | { |
| 96 | public: |
SiCong Li | b63b119 | 2022-01-28 18:24:39 +0000 | [diff] [blame^] | 97 | /** A fully realized SharedVarLink |
| 98 | */ |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 99 | struct SharedVar |
| 100 | { |
SiCong Li | b63b119 | 2022-01-28 18:24:39 +0000 | [diff] [blame^] | 101 | ArgumentID arg_id{ g_arg_placeholder }; |
| 102 | SharedVarIO io{ SharedVarIO::Input }; |
| 103 | SharedVarGroup group{ SharedVarGroup::Argument }; |
| 104 | std::string uniq_name{}; // Unique name, also the final variable name used in the built code |
| 105 | ClKernelArgDescriptor desc{}; // Automatic variables can and should still be described using this struct |
| 106 | bool is_empty() const |
| 107 | { |
| 108 | return arg_id == g_arg_placeholder; |
| 109 | } |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 110 | }; |
| 111 | |
SiCong Li | b63b119 | 2022-01-28 18:24:39 +0000 | [diff] [blame^] | 112 | class Arguments |
| 113 | { |
| 114 | public: |
| 115 | Arguments() = default; |
| 116 | void add_var(const SharedVar &var) |
| 117 | { |
| 118 | ARM_COMPUTE_ERROR_ON(var.group != SharedVarGroup::Argument); |
| 119 | _vars.push_back(var); |
| 120 | } |
| 121 | std::vector<SharedVar> get_all_vars() const |
| 122 | { |
| 123 | return _vars; |
| 124 | } |
| 125 | std::vector<SharedVar> get_src_vars() const |
| 126 | { |
| 127 | std::vector<SharedVar> src_vars; |
| 128 | std::copy_if(_vars.begin(), _vars.end(), std::back_inserter(src_vars), [](const SharedVar & var) |
| 129 | { |
| 130 | return var.io == SharedVarIO::Input; |
| 131 | }); |
| 132 | return src_vars; |
| 133 | } |
| 134 | SharedVar get_dst_var() const |
| 135 | { |
| 136 | std::vector<SharedVar> dst_vars; |
| 137 | std::copy_if(_vars.begin(), _vars.end(), std::back_inserter(dst_vars), [](const SharedVar & var) |
| 138 | { |
| 139 | return var.io == SharedVarIO::Output; |
| 140 | }); |
| 141 | ARM_COMPUTE_ERROR_ON(dst_vars.size() != 1); |
| 142 | return dst_vars.at(0); |
| 143 | } |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 144 | |
SiCong Li | b63b119 | 2022-01-28 18:24:39 +0000 | [diff] [blame^] | 145 | private: |
| 146 | std::vector<SharedVar> _vars{}; |
| 147 | }; |
| 148 | |
| 149 | /** Create a SharedVar for a corresponding SharedVarLink (contains ArgumentID). If one has already been created for the SharedVarLink, simply return it instead of creating a new one |
| 150 | * |
| 151 | * @note: The order of insertion is important. There is one precondition: |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 152 | * PRECOND: The components have been sorted topologically / is being traversed in topological order |
| 153 | * This ensures that all the consumer var links (Output, Automatic Links) can consume (return) the producer var links when they're referred |
| 154 | */ |
SiCong Li | b63b119 | 2022-01-28 18:24:39 +0000 | [diff] [blame^] | 155 | void add(SharedVarLink var_link, SharedVarGroup group, ClKernelArgDescriptor runtime_desc, const std::string &name = "unnamed") |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 156 | { |
| 157 | ARM_COMPUTE_ERROR_ON_MSG(var_link.is_empty(), "Non-empty SharedVarLink expected"); |
SiCong Li | b63b119 | 2022-01-28 18:24:39 +0000 | [diff] [blame^] | 158 | if(!get(var_link).is_empty()) |
| 159 | { |
| 160 | return; |
| 161 | } |
| 162 | |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 163 | auto var_id = _num_var; |
| 164 | std::stringstream ss; |
| 165 | ss << name << "_" << var_id; |
| 166 | const auto uniq_name = ss.str(); |
SiCong Li | b63b119 | 2022-01-28 18:24:39 +0000 | [diff] [blame^] | 167 | SharedVar var{ var_link.arg_id, var_link.io, group, uniq_name, runtime_desc }; |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 168 | |
SiCong Li | b63b119 | 2022-01-28 18:24:39 +0000 | [diff] [blame^] | 169 | if(group == SharedVarGroup::Argument) |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 170 | { |
| 171 | _arguments.emplace(var_id, var); |
SiCong Li | b63b119 | 2022-01-28 18:24:39 +0000 | [diff] [blame^] | 172 | _arg_id_map.emplace(var_link.arg_id, var_id); |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 173 | _num_var++; |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 174 | } |
SiCong Li | b63b119 | 2022-01-28 18:24:39 +0000 | [diff] [blame^] | 175 | else if(group == SharedVarGroup::Automatic) |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 176 | { |
SiCong Li | b63b119 | 2022-01-28 18:24:39 +0000 | [diff] [blame^] | 177 | if(_global_vars.empty()) |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 178 | { |
SiCong Li | b63b119 | 2022-01-28 18:24:39 +0000 | [diff] [blame^] | 179 | if(var_link.io == SharedVarIO::Output) |
| 180 | { |
| 181 | _global_vars.emplace(var_id, var); |
| 182 | _arg_id_map.emplace(var_link.arg_id, var_id); |
| 183 | _num_var++; |
| 184 | } |
| 185 | else |
| 186 | { |
| 187 | ARM_COMPUTE_ERROR("Component likely not traversed in topological order"); |
| 188 | } |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 189 | } |
| 190 | else |
| 191 | { |
SiCong Li | b63b119 | 2022-01-28 18:24:39 +0000 | [diff] [blame^] | 192 | // Associate additional SharedVarLinks with the single global shared variable |
| 193 | const auto global_var_id = _global_vars.begin()->first; |
| 194 | _arg_id_map[var_link.arg_id] = global_var_id; |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 195 | } |
| 196 | } |
| 197 | else |
| 198 | { |
| 199 | ARM_COMPUTE_ERROR("Unrecognised SharedVarGroup"); |
| 200 | } |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 201 | } |
| 202 | |
SiCong Li | b63b119 | 2022-01-28 18:24:39 +0000 | [diff] [blame^] | 203 | /** Get the SharedVar associated with @p var_link |
| 204 | * |
| 205 | * @param var_link |
| 206 | * @return SharedVar |
| 207 | */ |
| 208 | SharedVar get(const SharedVarLink &var_link) const |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 209 | { |
SiCong Li | b63b119 | 2022-01-28 18:24:39 +0000 | [diff] [blame^] | 210 | const SharedVar empty_var{}; |
| 211 | if(_arg_id_map.find(var_link.arg_id) != _arg_id_map.end()) |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 212 | { |
SiCong Li | b63b119 | 2022-01-28 18:24:39 +0000 | [diff] [blame^] | 213 | const auto var_id = _arg_id_map.at(var_link.arg_id); |
| 214 | const auto arg_var = _arguments.find(var_id); |
| 215 | if(arg_var != _arguments.end()) |
| 216 | { |
| 217 | return arg_var->second; |
| 218 | } |
| 219 | else |
| 220 | { |
| 221 | return _global_vars.at(var_id); |
| 222 | } |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 223 | } |
SiCong Li | b63b119 | 2022-01-28 18:24:39 +0000 | [diff] [blame^] | 224 | return empty_var; |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 225 | } |
| 226 | |
| 227 | /** @note The arguments are returned in the order they are added |
| 228 | */ |
| 229 | Arguments get_kernel_arguments() const |
| 230 | { |
| 231 | Arguments args{}; |
| 232 | for(const auto &a : _arguments) |
| 233 | { |
SiCong Li | b63b119 | 2022-01-28 18:24:39 +0000 | [diff] [blame^] | 234 | args.add_var(a.second); |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 235 | } |
| 236 | return args; |
| 237 | } |
| 238 | |
| 239 | private: |
| 240 | using VarID = int32_t; |
| 241 | |
| 242 | private: |
SiCong Li | b63b119 | 2022-01-28 18:24:39 +0000 | [diff] [blame^] | 243 | std::map<VarID, SharedVar> _global_vars{}; // Shared, global variable |
| 244 | std::map<VarID, SharedVar> _arguments{}; |
| 245 | std::map<ArgumentID, VarID> _arg_id_map{}; // Track ArgumentIDs that have already been added |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 246 | VarID _num_var{ 0 }; |
| 247 | }; |
| 248 | |
| 249 | enum class ComponentType |
| 250 | { |
| 251 | Simple, |
| 252 | Complex, |
| 253 | Store |
| 254 | }; |
| 255 | |
SiCong Li | b63b119 | 2022-01-28 18:24:39 +0000 | [diff] [blame^] | 256 | using ComponentID = DependencyGraph::Id; |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 257 | using ComponentList = std::vector<ComponentID>; |
| 258 | class IClKernelComponent |
| 259 | { |
| 260 | public: |
| 261 | using Link = SharedVarLink; |
| 262 | using Tag = std::string; |
| 263 | struct TagVal |
| 264 | { |
| 265 | TagVal() = default; |
Giorgio Arena | bd44caa | 2022-03-15 13:45:15 +0000 | [diff] [blame] | 266 | TagVal(const SharedVarTable::SharedVar &var) |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 267 | : value{ var.uniq_name } |
| 268 | { |
| 269 | } |
| 270 | |
Gunes Bayir | 16c5697 | 2022-03-28 21:32:33 +0100 | [diff] [blame] | 271 | template <typename T, ARM_COMPUTE_REQUIRES_TA(std::is_integral<T>::value)> |
| 272 | TagVal(T val) |
| 273 | : value{ support::cpp11::to_string(val) } |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 274 | { |
| 275 | } |
| 276 | |
Giorgio Arena | bd44caa | 2022-03-15 13:45:15 +0000 | [diff] [blame] | 277 | TagVal(const std::string &val) |
| 278 | : value{ val } |
| 279 | { |
| 280 | } |
| 281 | |
Gunes Bayir | 16c5697 | 2022-03-28 21:32:33 +0100 | [diff] [blame] | 282 | TagVal(const char *val) |
| 283 | : value{ std::string(val) } |
| 284 | { |
| 285 | } |
| 286 | |
| 287 | TagVal(const DataType &data_type) |
| 288 | : value{ get_cl_type_from_data_type(data_type) } |
| 289 | { |
| 290 | } |
| 291 | |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 292 | std::string value{}; |
| 293 | }; |
| 294 | using TagLUT = std::unordered_map<Tag, TagVal>; // Used to instantiating a code template / replacing tags |
| 295 | public: |
SiCong Li | b63b119 | 2022-01-28 18:24:39 +0000 | [diff] [blame^] | 296 | IClKernelComponent(ClKernelBlueprint *blueprint) |
Gunes Bayir | 8a87983 | 2022-03-10 21:21:01 +0000 | [diff] [blame] | 297 | : _blueprint(blueprint) |
| 298 | { |
| 299 | } |
| 300 | |
| 301 | ARM_COMPUTE_DISALLOW_COPY_ALLOW_MOVE(IClKernelComponent); |
| 302 | |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 303 | virtual ~IClKernelComponent() = default; |
| 304 | virtual ComponentType get_component_type() const = 0; |
| 305 | virtual std::vector<Link> get_links() const = 0; |
| 306 | virtual std::string name() const = 0; |
| 307 | |
Giorgio Arena | bd44caa | 2022-03-15 13:45:15 +0000 | [diff] [blame] | 308 | // @note: some tags can be unused since they could be used only for the macros, or only for the component code |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 309 | static std::string replace_tags(const std::string &code_template, const TagLUT &tags) |
| 310 | { |
Giorgio Arena | bd44caa | 2022-03-15 13:45:15 +0000 | [diff] [blame] | 311 | std::string replaced_code = ""; |
| 312 | bool scanning_pattern = false; |
| 313 | std::string pattern_found = ""; |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 314 | for(size_t i = 0; i < code_template.size() - 1; ++i) |
| 315 | { |
| 316 | if(!scanning_pattern) |
| 317 | { |
| 318 | if(code_template[i] == '{' && code_template[i + 1] == '{') |
| 319 | { |
| 320 | i += 1; |
| 321 | scanning_pattern = true; |
| 322 | pattern_found = ""; |
| 323 | } |
| 324 | else |
| 325 | { |
| 326 | replaced_code += code_template[i]; |
| 327 | } |
| 328 | } |
| 329 | else |
| 330 | { |
| 331 | if(code_template[i] == '}' && code_template[i + 1] == '}') |
| 332 | { |
| 333 | i += 1; |
| 334 | scanning_pattern = false; |
| 335 | std::string err = "Pattern " + pattern_found + " not found in tags"; |
| 336 | ARM_COMPUTE_ERROR_ON_MSG(tags.find(pattern_found) == tags.end(), err.c_str()); |
| 337 | replaced_code += tags.find(pattern_found)->second.value; |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 338 | } |
| 339 | else |
| 340 | { |
| 341 | pattern_found += code_template[i]; |
| 342 | } |
| 343 | } |
| 344 | } |
Giorgio Arena | bd44caa | 2022-03-15 13:45:15 +0000 | [diff] [blame] | 345 | |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 346 | return replaced_code; |
| 347 | } |
| 348 | ComponentID id() const |
| 349 | { |
| 350 | return _id; |
| 351 | } |
| 352 | void set_id(ComponentID id) |
| 353 | { |
| 354 | _id = id; |
| 355 | } |
| 356 | |
| 357 | virtual std::set<std::string> get_headers_list() const |
| 358 | { |
| 359 | return std::set<std::string> {}; |
| 360 | } |
| 361 | |
| 362 | virtual std::string get_additional_macros() const |
| 363 | { |
| 364 | return ""; |
| 365 | } |
| 366 | |
| 367 | virtual std::string get_component_code() const |
| 368 | { |
| 369 | return ""; |
| 370 | } |
Gunes Bayir | 8a87983 | 2022-03-10 21:21:01 +0000 | [diff] [blame] | 371 | |
| 372 | virtual Window get_window() const |
| 373 | { |
| 374 | return Window{}; |
| 375 | } |
SiCong Li | b63b119 | 2022-01-28 18:24:39 +0000 | [diff] [blame^] | 376 | /** Get the tag look-up table used to instantiate the component code. |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 377 | * |
| 378 | * @param vtable |
| 379 | * @return TagLUT |
| 380 | */ |
SiCong Li | b63b119 | 2022-01-28 18:24:39 +0000 | [diff] [blame^] | 381 | virtual TagLUT get_tag_lut(const SharedVarTable &vtable) const = 0; |
| 382 | |
| 383 | /** Allocate all shared variables used by the component in the @p vtable |
| 384 | * |
| 385 | * @param vtable |
| 386 | */ |
| 387 | virtual void allocate_shared_vars(SharedVarTable &vtable) const = 0; |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 388 | |
| 389 | virtual std::string get_dst_addr_calculation() const |
| 390 | { |
| 391 | return ""; |
| 392 | } |
| 393 | |
SiCong Li | 0a486cf | 2022-04-07 17:41:51 +0100 | [diff] [blame] | 394 | /** Generate config id of the component |
| 395 | * |
| 396 | * @return std::string |
| 397 | */ |
| 398 | virtual std::string generate_config_id() const |
| 399 | { |
| 400 | return ""; |
| 401 | } |
| 402 | |
Giorgio Arena | bd44caa | 2022-03-15 13:45:15 +0000 | [diff] [blame] | 403 | virtual CLBuildOptions generate_build_options() const |
| 404 | { |
| 405 | return CLBuildOptions{}; |
| 406 | } |
| 407 | |
Gunes Bayir | 8a87983 | 2022-03-10 21:21:01 +0000 | [diff] [blame] | 408 | protected: |
SiCong Li | b63b119 | 2022-01-28 18:24:39 +0000 | [diff] [blame^] | 409 | ClKernelBlueprint *_blueprint; |
Gunes Bayir | 8a87983 | 2022-03-10 21:21:01 +0000 | [diff] [blame] | 410 | |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 411 | private: |
| 412 | ComponentID _id{}; |
| 413 | }; |
| 414 | |
| 415 | using ComponentUniquePtr = std::unique_ptr<IClKernelComponent>; |
| 416 | |
| 417 | /** Intermediate representation of the final, complete kernel source. |
| 418 | */ |
| 419 | struct ClKernelBlueprint::Implementation |
| 420 | { |
| 421 | public: |
| 422 | Implementation() = default; |
| 423 | ~Implementation() = default; |
| 424 | |
| 425 | public: |
SiCong Li | b63b119 | 2022-01-28 18:24:39 +0000 | [diff] [blame^] | 426 | Status update_merge_point(ArgumentID t_id, ArgumentID merge_point) |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 427 | { |
SiCong Li | b63b119 | 2022-01-28 18:24:39 +0000 | [diff] [blame^] | 428 | return _graph.update_merge_point(t_id, merge_point); |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 429 | } |
| 430 | |
SiCong Li | b63b119 | 2022-01-28 18:24:39 +0000 | [diff] [blame^] | 431 | ArgumentID add_kernel_tensor(ITensorInfo *tensor_info, ArgumentID merge_point = DependencyGraph::empty_id()) |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 432 | { |
SiCong Li | b63b119 | 2022-01-28 18:24:39 +0000 | [diff] [blame^] | 433 | const auto id = _graph.add_tensor(merge_point); |
| 434 | if(_kernel_tensors.find(id) == _kernel_tensors.end()) |
| 435 | { |
| 436 | _kernel_tensors.insert(std::make_pair(id, tensor_info)); |
| 437 | } |
| 438 | return id; |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 439 | } |
| 440 | |
| 441 | void set_tile_info(const TileDescriptor &tile_info) |
| 442 | { |
| 443 | _tile_info = tile_info; |
| 444 | } |
| 445 | |
| 446 | SharedVarGroup group(ArgumentID arg_id) const |
| 447 | { |
| 448 | if(arg_id == g_arg_placeholder) |
| 449 | { |
| 450 | // In case of placeholder, don't care what we return; |
| 451 | return SharedVarGroup::Argument; |
| 452 | } |
| 453 | return _shared_var_group_lut.at(arg_id); |
| 454 | } |
| 455 | |
| 456 | void validate_arg_ids(std::initializer_list<ArgumentID> args) const |
| 457 | { |
| 458 | for(const auto arg_id : args) |
| 459 | { |
| 460 | ARM_COMPUTE_UNUSED(arg_id); |
SiCong Li | b63b119 | 2022-01-28 18:24:39 +0000 | [diff] [blame^] | 461 | ARM_COMPUTE_ERROR_ON_MSG(_kernel_tensors.find(arg_id) == _kernel_tensors.end() && arg_id != g_arg_placeholder, |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 462 | "Trying to use an argument that hasn't been added to the blueprint"); |
| 463 | } |
| 464 | } |
| 465 | |
| 466 | void add_component(ComponentUniquePtr component) |
| 467 | { |
| 468 | if(component->get_component_type() == ComponentType::Complex) |
| 469 | { |
| 470 | ++_num_complex_components; |
| 471 | ARM_COMPUTE_ERROR_ON_MSG(_num_complex_components > 1, "Only one complex component per blueprint is supported."); |
| 472 | } |
| 473 | |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 474 | // Get an unique ID for the component that's being added |
SiCong Li | b63b119 | 2022-01-28 18:24:39 +0000 | [diff] [blame^] | 475 | std::vector<ArgumentID> src_tensors; |
| 476 | std::vector<ArgumentID> dst_tensors; |
| 477 | for(const auto &link : component->get_links()) |
| 478 | { |
| 479 | if(link.is_empty()) |
| 480 | { |
| 481 | continue; |
| 482 | } |
| 483 | if(link.io == SharedVarIO::Input) |
| 484 | { |
| 485 | src_tensors.push_back(link.arg_id); |
| 486 | } |
| 487 | else |
| 488 | { |
| 489 | dst_tensors.push_back(link.arg_id); |
| 490 | } |
| 491 | } |
| 492 | const ComponentID component_id = _graph.add_operator(src_tensors, dst_tensors).second; |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 493 | component->set_id(component_id); |
| 494 | |
| 495 | // Add this component to the component graph. Don't connect it to anything yet |
| 496 | _component_graph.emplace(component_id, ComponentList{}); |
| 497 | |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 498 | // For every { arg_id, arg_io } passed along with this component... |
| 499 | for(const auto &link : component->get_links()) |
| 500 | { |
| 501 | const ArgumentID &arg_id = link.arg_id; |
| 502 | const SharedVarIO &arg_io = link.io; |
| 503 | |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 504 | // Add the arg_id to the map describing the input/output relationship between an argument and the components that use it, if it doesn't yet exist there |
| 505 | if(_outgoing_components.find(arg_id) == _outgoing_components.end()) |
| 506 | { |
| 507 | _outgoing_components.emplace(arg_id, ComponentList{}); |
| 508 | _incoming_components.emplace(arg_id, ComponentList{}); |
| 509 | } |
| 510 | |
| 511 | // If it's an input argument, connect any other component that has it as output with this component |
| 512 | // Additionally, set this component as one that treats this argument as "Input" (append to index 0) |
| 513 | // This is used so that we keep track of whether two components use the same argument, one as input and one as output |
| 514 | if(arg_io == SharedVarIO::Input) |
| 515 | { |
| 516 | for(const auto &prev_component : _incoming_components[arg_id]) |
| 517 | { |
| 518 | _component_graph[prev_component].push_back(component_id); |
| 519 | } |
| 520 | |
| 521 | _outgoing_components[arg_id].push_back(component_id); |
| 522 | } |
| 523 | // If it's an output argument, connect this component with any other component that has it as input |
| 524 | // Additionally, set this component as one that treats this argument as "Output" (append to index 1) |
| 525 | else |
| 526 | { |
Gunes Bayir | 8a87983 | 2022-03-10 21:21:01 +0000 | [diff] [blame] | 527 | if(component->get_component_type() == ComponentType::Store) |
| 528 | { |
| 529 | ARM_COMPUTE_ERROR_ON_MSG(_dst_id >= 0, "Trying to add more than one dst argument to the graph"); |
| 530 | _dst_id = arg_id; |
| 531 | } |
| 532 | |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 533 | for(const auto &subseq_component : _outgoing_components[arg_id]) |
| 534 | { |
| 535 | _component_graph[component_id].push_back(subseq_component); |
| 536 | } |
| 537 | |
| 538 | _incoming_components[arg_id].push_back(component_id); |
| 539 | } |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 540 | } |
| 541 | |
SiCong Li | b63b119 | 2022-01-28 18:24:39 +0000 | [diff] [blame^] | 542 | ARM_COMPUTE_ERROR_ON_MSG(_graph.get_root_ops().size() != 1, "Trying to add more than one root to the graph"); |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 543 | |
| 544 | // Finally, add this component to the dictionary of components |
| 545 | _components.insert(std::make_pair(component_id, std::move(component))); |
| 546 | } |
| 547 | |
| 548 | std::string build_kernel_name() const |
| 549 | { |
| 550 | std::string name = ""; |
| 551 | |
Giorgio Arena | bd44caa | 2022-03-15 13:45:15 +0000 | [diff] [blame] | 552 | traverse([&](std::stack<ComponentID> stack) |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 553 | { |
| 554 | name += _components.find(stack.top())->second->name() + (stack.size() > 2 ? "___" : ""); |
Giorgio Arena | bd44caa | 2022-03-15 13:45:15 +0000 | [diff] [blame] | 555 | }); |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 556 | |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 557 | return name; |
| 558 | } |
| 559 | |
| 560 | std::string build_code() |
| 561 | { |
| 562 | ARM_COMPUTE_ERROR_ON_MSG(_graph_root < 0, "No root found in the component graph"); |
| 563 | |
| 564 | // These data structures will hold the data from all the components in the blueprint |
| 565 | std::set<std::string> headers_list{}; |
| 566 | std::set<std::string> additional_macros{}; |
| 567 | std::vector<std::string> component_codes{}; // vector because order matters |
| 568 | |
SiCong Li | b63b119 | 2022-01-28 18:24:39 +0000 | [diff] [blame^] | 569 | // Step 1: Allocate all kernel argument shared variables before generating the component code |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 570 | auto stack = topological_sort(); |
| 571 | while(!stack.empty()) |
| 572 | { |
| 573 | auto curr_component_id = stack.top(); |
| 574 | auto &curr_component = _components.find(curr_component_id)->second; |
| 575 | |
SiCong Li | b63b119 | 2022-01-28 18:24:39 +0000 | [diff] [blame^] | 576 | curr_component->allocate_shared_vars(_vtable); |
| 577 | |
| 578 | stack.pop(); |
| 579 | } |
| 580 | // Step 2: Generate component codes |
| 581 | stack = topological_sort(); |
| 582 | while(!stack.empty()) |
| 583 | { |
| 584 | auto curr_component_id = stack.top(); |
| 585 | auto &curr_component = _components.find(curr_component_id)->second; |
| 586 | |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 587 | auto curr_headers_list = curr_component->get_headers_list(); |
| 588 | auto curr_additional_macros = curr_component->get_additional_macros(); |
| 589 | auto curr_component_code = curr_component->get_component_code(); |
SiCong Li | b63b119 | 2022-01-28 18:24:39 +0000 | [diff] [blame^] | 590 | const auto var_lut = curr_component->get_tag_lut(_vtable); // Ideally can be merged with get_component_code once we have finer-grained code generation technique |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 591 | component_codes.push_back(IClKernelComponent::replace_tags(curr_component_code, var_lut)); |
| 592 | |
| 593 | headers_list.insert(curr_headers_list.begin(), curr_headers_list.end()); |
| 594 | if(!curr_additional_macros.empty()) // Some components might not have any |
| 595 | { |
Giorgio Arena | bd44caa | 2022-03-15 13:45:15 +0000 | [diff] [blame] | 596 | additional_macros.insert(IClKernelComponent::replace_tags(curr_additional_macros, var_lut)); |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 597 | } |
| 598 | |
| 599 | stack.pop(); |
| 600 | } |
| 601 | |
SiCong Li | b63b119 | 2022-01-28 18:24:39 +0000 | [diff] [blame^] | 602 | // Step 3: Assemble the data gathered by traversing the graph into the string "code" |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 603 | std::string code = ""; |
| 604 | |
| 605 | for(auto &header : headers_list) |
| 606 | { |
Giorgio Arena | 892b70a | 2022-03-30 12:23:10 +0100 | [diff] [blame] | 607 | #if defined(EMBEDDED_KERNELS) |
| 608 | code += CLKernelLibrary::get().get_program(header).first; |
| 609 | #else // defined(EMBEDDED_KERNELS) |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 610 | code += "#include \"" + header + "\"\n"; |
Giorgio Arena | 892b70a | 2022-03-30 12:23:10 +0100 | [diff] [blame] | 611 | #endif // defined(EMBEDDED_KERNELS) |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 612 | } |
| 613 | |
| 614 | for(auto ¯os : additional_macros) |
| 615 | { |
| 616 | code += macros; |
| 617 | } |
| 618 | |
| 619 | code += generate_kernel_signature(_vtable.get_kernel_arguments()); |
| 620 | |
| 621 | code += "\n{\n\n"; |
| 622 | |
| 623 | code += " //------------------ START KERNEL_BUILDER_COORDINATE ---------------------\n\n"; |
| 624 | code += generate_global_section(); |
| 625 | code += " //------------------ END KERNEL_BUILDER_COORDINATE ---------------------\n"; |
| 626 | |
| 627 | for(auto &component_code : component_codes) |
| 628 | { |
| 629 | code += component_code; |
| 630 | } |
| 631 | |
| 632 | code += "}\n"; |
| 633 | |
| 634 | return code; |
| 635 | } |
| 636 | |
SiCong Li | 0a486cf | 2022-04-07 17:41:51 +0100 | [diff] [blame] | 637 | /** Generate config id of the entire kernel |
| 638 | * |
| 639 | * Format: kernel_name--comp0_config_id--comp1_config_id--... |
| 640 | * |
| 641 | * @return std::string |
| 642 | */ |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 643 | std::string build_config_id() const |
| 644 | { |
SiCong Li | 0a486cf | 2022-04-07 17:41:51 +0100 | [diff] [blame] | 645 | std::string config_id = build_kernel_name(); |
| 646 | traverse([&](std::stack<ComponentID> stack) |
| 647 | { |
| 648 | config_id += "--" + _components.find(stack.top())->second->generate_config_id() + "--"; |
| 649 | }); |
| 650 | |
| 651 | return config_id; |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 652 | } |
| 653 | |
| 654 | CLBuildOptions build_options() const |
| 655 | { |
Giorgio Arena | bd44caa | 2022-03-15 13:45:15 +0000 | [diff] [blame] | 656 | CLBuildOptions build_opts{}; |
| 657 | |
| 658 | traverse([&](std::stack<ComponentID> stack) |
| 659 | { |
| 660 | build_opts.add_options(_components.find(stack.top())->second->generate_build_options().options()); |
| 661 | }); |
| 662 | |
| 663 | return build_opts; |
| 664 | } |
| 665 | |
| 666 | TileDescriptor get_tile_info() const |
| 667 | { |
| 668 | return _tile_info; |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 669 | } |
| 670 | |
| 671 | Window get_execution_window() const |
| 672 | { |
Gunes Bayir | 8a87983 | 2022-03-10 21:21:01 +0000 | [diff] [blame] | 673 | ARM_COMPUTE_ERROR_ON_MSG(_graph_root < 0, "No root found in the component graph"); |
| 674 | ARM_COMPUTE_ERROR_ON_MSG(_dst_id == -1, "Destination Tensor Id should be ready before calling get_execution_window()"); |
| 675 | |
| 676 | return _components.find(_graph_root)->second->get_window(); |
| 677 | } |
| 678 | |
| 679 | ArgumentID get_dst_id() const |
| 680 | { |
| 681 | return _dst_id; |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 682 | } |
| 683 | |
| 684 | ClKernelArgList get_arguments() const |
| 685 | { |
| 686 | ClKernelArgList arg_list{}; |
SiCong Li | b63b119 | 2022-01-28 18:24:39 +0000 | [diff] [blame^] | 687 | for(const auto &arg_var : _vtable.get_kernel_arguments().get_all_vars()) |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 688 | { |
SiCong Li | b63b119 | 2022-01-28 18:24:39 +0000 | [diff] [blame^] | 689 | arg_list[arg_var.desc.arg_id] = arg_var.desc; |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 690 | } |
| 691 | return arg_list; |
| 692 | } |
| 693 | |
SiCong Li | b63b119 | 2022-01-28 18:24:39 +0000 | [diff] [blame^] | 694 | /** Get the arguments as shared vars from the vtable |
| 695 | * |
| 696 | * @return SharedVarTable::Arguments |
| 697 | */ |
| 698 | SharedVarTable::Arguments get_argument_shared_vars() const |
Gunes Bayir | 8a87983 | 2022-03-10 21:21:01 +0000 | [diff] [blame] | 699 | { |
SiCong Li | b63b119 | 2022-01-28 18:24:39 +0000 | [diff] [blame^] | 700 | return _vtable.get_kernel_arguments(); |
| 701 | } |
| 702 | |
| 703 | const ITensorInfo *get_kernel_argument_info(const ArgumentID id) const |
| 704 | { |
| 705 | auto it = _kernel_tensors.find(id); |
| 706 | if(it != _kernel_tensors.end()) |
Gunes Bayir | 8a87983 | 2022-03-10 21:21:01 +0000 | [diff] [blame] | 707 | { |
SiCong Li | b63b119 | 2022-01-28 18:24:39 +0000 | [diff] [blame^] | 708 | return it->second; |
Gunes Bayir | 8a87983 | 2022-03-10 21:21:01 +0000 | [diff] [blame] | 709 | } |
| 710 | return nullptr; |
| 711 | } |
| 712 | |
SiCong Li | b63b119 | 2022-01-28 18:24:39 +0000 | [diff] [blame^] | 713 | ITensorInfo *get_kernel_argument_info(const ArgumentID id) |
Gunes Bayir | 8a87983 | 2022-03-10 21:21:01 +0000 | [diff] [blame] | 714 | { |
SiCong Li | b63b119 | 2022-01-28 18:24:39 +0000 | [diff] [blame^] | 715 | auto it = _kernel_tensors.find(id); |
| 716 | if(it != _kernel_tensors.end()) |
Gunes Bayir | 8a87983 | 2022-03-10 21:21:01 +0000 | [diff] [blame] | 717 | { |
SiCong Li | b63b119 | 2022-01-28 18:24:39 +0000 | [diff] [blame^] | 718 | return it->second; |
Gunes Bayir | 8a87983 | 2022-03-10 21:21:01 +0000 | [diff] [blame] | 719 | } |
| 720 | return nullptr; |
| 721 | } |
SiCong Li | b63b119 | 2022-01-28 18:24:39 +0000 | [diff] [blame^] | 722 | /** Finalize graph construction. Graph is expected to not mutate after being finalized |
| 723 | */ |
| 724 | void finalize() |
| 725 | { |
| 726 | cache_root_component(); |
| 727 | assign_shared_var_group(); |
| 728 | } |
| 729 | |
| 730 | DependencyGraph get_graph() const |
| 731 | { |
| 732 | return _graph; |
| 733 | } |
Gunes Bayir | 8a87983 | 2022-03-10 21:21:01 +0000 | [diff] [blame] | 734 | |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 735 | private: |
SiCong Li | b63b119 | 2022-01-28 18:24:39 +0000 | [diff] [blame^] | 736 | void cache_root_component() |
| 737 | { |
| 738 | const auto roots = _graph.get_root_ops(); |
| 739 | ARM_COMPUTE_ERROR_ON_MSG(roots.size() != 1, "Trying to add more than one root to the graph"); |
| 740 | _graph_root = roots.at(0); |
| 741 | } |
| 742 | /** Assign the group for each shared var. Can only be performed at the end of the graph construction, before building |
| 743 | */ |
| 744 | void assign_shared_var_group() |
| 745 | { |
| 746 | for(const auto &tensor : _kernel_tensors) |
| 747 | { |
| 748 | const auto tensor_id = tensor.first; |
| 749 | if(_graph.is_src_tensor(tensor_id) || _graph.is_dst_tensor(tensor_id)) |
| 750 | { |
| 751 | _shared_var_group_lut[tensor_id] = SharedVarGroup::Argument; |
| 752 | } |
| 753 | else |
| 754 | { |
| 755 | _shared_var_group_lut[tensor_id] = SharedVarGroup::Automatic; |
| 756 | } |
| 757 | } |
| 758 | } |
| 759 | |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 760 | void topological_sort_utility(ComponentID component_id, std::unordered_set<ComponentID> &visited, std::stack<ComponentID> &stack) const |
| 761 | { |
| 762 | visited.insert(component_id); |
| 763 | |
| 764 | for(auto connected_component : _component_graph.find(component_id)->second) |
| 765 | { |
| 766 | if(visited.find(connected_component) == visited.end()) |
| 767 | { |
| 768 | topological_sort_utility(connected_component, visited, stack); |
| 769 | } |
| 770 | } |
| 771 | |
| 772 | stack.push(component_id); |
| 773 | } |
| 774 | |
| 775 | std::stack<ComponentID> topological_sort() const |
| 776 | { |
| 777 | std::stack<ComponentID> stack{}; |
| 778 | std::unordered_set<ComponentID> visited{}; |
| 779 | |
| 780 | topological_sort_utility(_graph_root, visited, stack); |
| 781 | |
| 782 | return stack; |
| 783 | } |
| 784 | |
Giorgio Arena | bd44caa | 2022-03-15 13:45:15 +0000 | [diff] [blame] | 785 | void traverse(const std::function<void(std::stack<ComponentID>)> &func) const |
| 786 | { |
| 787 | std::stack<ComponentID> stack = topological_sort(); |
| 788 | |
| 789 | while(!stack.empty()) |
| 790 | { |
| 791 | func(stack); |
| 792 | stack.pop(); |
| 793 | } |
| 794 | } |
| 795 | |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 796 | std::string generate_argument_declaration(const SharedVarTable::SharedVar &var) const |
| 797 | { |
| 798 | ARM_COMPUTE_ERROR_ON_MSG(var.group != SharedVarGroup::Argument, "An argument declaration can only be generated from a kernel argument"); |
| 799 | std::string code; |
| 800 | switch(var.desc.tensor_arg_type) |
| 801 | { |
SiCong Li | b63b119 | 2022-01-28 18:24:39 +0000 | [diff] [blame^] | 802 | case ClKernelTensorArgType::Vector: |
Gunes Bayir | 16c5697 | 2022-03-28 21:32:33 +0100 | [diff] [blame] | 803 | { |
| 804 | code += "\n VECTOR_DECLARATION(" + var.uniq_name + ")"; |
| 805 | break; |
| 806 | } |
SiCong Li | b63b119 | 2022-01-28 18:24:39 +0000 | [diff] [blame^] | 807 | case ClKernelTensorArgType::Image: |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 808 | { |
Gunes Bayir | 16c5697 | 2022-03-28 21:32:33 +0100 | [diff] [blame] | 809 | code += "\n IMAGE_DECLARATION(" + var.uniq_name + ")"; |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 810 | break; |
| 811 | } |
SiCong Li | b63b119 | 2022-01-28 18:24:39 +0000 | [diff] [blame^] | 812 | case ClKernelTensorArgType::Image_3D: |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 813 | { |
Gunes Bayir | 16c5697 | 2022-03-28 21:32:33 +0100 | [diff] [blame] | 814 | code += "\n IMAGE_DECLARATION(" + var.uniq_name + "),"; |
| 815 | code += "\n uint " + var.uniq_name + "_stride_z"; |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 816 | break; |
| 817 | } |
SiCong Li | b63b119 | 2022-01-28 18:24:39 +0000 | [diff] [blame^] | 818 | case ClKernelTensorArgType::Image_3D_Export_To_ClImage2D: |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 819 | { |
Gunes Bayir | 16c5697 | 2022-03-28 21:32:33 +0100 | [diff] [blame] | 820 | code += "\n __read_only image2d_t " + var.uniq_name + "_img,"; |
| 821 | code += "\n uint " + var.uniq_name + "_stride_z"; |
| 822 | break; |
| 823 | } |
SiCong Li | b63b119 | 2022-01-28 18:24:39 +0000 | [diff] [blame^] | 824 | case ClKernelTensorArgType::Tensor_4D_t_Buffer: |
Gunes Bayir | 16c5697 | 2022-03-28 21:32:33 +0100 | [diff] [blame] | 825 | { |
| 826 | code += "\n TENSOR4D_T(" + var.uniq_name + ", BUFFER)"; |
| 827 | break; |
| 828 | } |
SiCong Li | b63b119 | 2022-01-28 18:24:39 +0000 | [diff] [blame^] | 829 | case ClKernelTensorArgType::Tensor_4D_t_Image: |
Gunes Bayir | 16c5697 | 2022-03-28 21:32:33 +0100 | [diff] [blame] | 830 | { |
| 831 | code += "\n TENSOR4D_T(" + var.uniq_name + ", IMAGE)"; |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 832 | break; |
| 833 | } |
| 834 | default: |
| 835 | { |
SiCong Li | b63b119 | 2022-01-28 18:24:39 +0000 | [diff] [blame^] | 836 | ARM_COMPUTE_ERROR("Unsupported declaration generation for ClKernelTensorArgType"); |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 837 | } |
| 838 | } |
| 839 | return code; |
| 840 | } |
| 841 | |
| 842 | std::string generate_kernel_signature(const SharedVarTable::Arguments &argument_list) const |
| 843 | { |
| 844 | std::string code = "\n__kernel void " + build_kernel_name() + "("; |
| 845 | |
SiCong Li | b63b119 | 2022-01-28 18:24:39 +0000 | [diff] [blame^] | 846 | for(const auto &arg : argument_list.get_all_vars()) |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 847 | { |
Gunes Bayir | 16c5697 | 2022-03-28 21:32:33 +0100 | [diff] [blame] | 848 | code += generate_argument_declaration(arg) + ","; |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 849 | } |
| 850 | |
| 851 | code[code.length() - 1] = ')'; |
| 852 | |
| 853 | return code; |
| 854 | } |
| 855 | |
| 856 | std::string generate_global_section() const |
| 857 | { |
SiCong Li | b63b119 | 2022-01-28 18:24:39 +0000 | [diff] [blame^] | 858 | auto dst_info = get_kernel_argument_info(_dst_id); |
| 859 | auto dst_w = dst_info->dimension(0); |
| 860 | auto dst_h = dst_info->dimension(1); |
| 861 | const auto tile_w = std::max(1, get_execution_window().x().step()); |
| 862 | const auto tile_h = std::max(1, get_execution_window().y().step()); |
| 863 | auto leftover_w = dst_w % tile_w; |
| 864 | auto leftover_h = dst_h % tile_h; |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 865 | |
SiCong Li | b63b119 | 2022-01-28 18:24:39 +0000 | [diff] [blame^] | 866 | std::string code = ""; |
| 867 | code += std::string(" int cout = GET_SPATIAL_IDX(0, ") + std::to_string(tile_w) + ", " + std::to_string(leftover_w) + ");\n"; |
| 868 | code += std::string(" int mout = GET_SPATIAL_IDX(1, ") + std::to_string(tile_h) + ", " + std::to_string(leftover_h) + ");\n"; |
| 869 | code += std::string(" int bout = GET_SPATIAL_IDX(2, 1, 0);\n\n"); |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 870 | |
| 871 | switch(_tile_info.clipping) |
| 872 | { |
| 873 | case ClippingStrategy::TOP_LEFT: |
SiCong Li | b63b119 | 2022-01-28 18:24:39 +0000 | [diff] [blame^] | 874 | code += " const bool g_cond_x = (cout == 0);\n"; |
| 875 | code += " const bool g_cond_y = (mout == 0);\n"; |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 876 | break; |
| 877 | case ClippingStrategy::TOP_RIGHT: |
SiCong Li | b63b119 | 2022-01-28 18:24:39 +0000 | [diff] [blame^] | 878 | code += " const bool g_cond_x = ((cout + 1) * " + std::to_string(tile_w) + " >= " + std::to_string(_tile_info.boundaries.x()) + ");\n"; |
| 879 | code += " const bool g_cond_y = (mout == 0);\n"; |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 880 | break; |
| 881 | case ClippingStrategy::BOTTOM_LEFT: |
SiCong Li | b63b119 | 2022-01-28 18:24:39 +0000 | [diff] [blame^] | 882 | code += " const bool g_cond_x = (cout == 0);\n"; |
| 883 | code += " const bool g_cond_y = ((mout + 1) * " + std::to_string(tile_h) + " >= " + std::to_string(_tile_info.boundaries.y()) + ");\n"; |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 884 | break; |
| 885 | case ClippingStrategy::BOTTOM_RIGHT: |
SiCong Li | b63b119 | 2022-01-28 18:24:39 +0000 | [diff] [blame^] | 886 | code += " const bool g_cond_x = ((cout + 1) * " + std::to_string(tile_w) + " >= " + std::to_string(_tile_info.boundaries.x()) + ");\n"; |
| 887 | code += " const bool g_cond_y = ((mout + 1) * " + std::to_string(tile_h) + " >= " + std::to_string(_tile_info.boundaries.y()) + ");\n"; |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 888 | break; |
| 889 | default: |
| 890 | ARM_COMPUTE_ERROR("Unsupported clipping strategy"); |
| 891 | } |
| 892 | |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 893 | return code; |
| 894 | } |
| 895 | |
| 896 | TileDescriptor _tile_info{}; |
| 897 | |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 898 | int32_t _num_complex_components{}; |
| 899 | |
Giorgio Arena | bd44caa | 2022-03-15 13:45:15 +0000 | [diff] [blame] | 900 | ArgumentID _dst_id{ -1 }; // Initially set to -1, which means the graph has no dst yet, since node IDs are positive numbers |
Gunes Bayir | 8a87983 | 2022-03-10 21:21:01 +0000 | [diff] [blame] | 901 | |
SiCong Li | b63b119 | 2022-01-28 18:24:39 +0000 | [diff] [blame^] | 902 | DependencyGraph _graph{}; |
| 903 | |
| 904 | // Tensors, components and IDs with corresponding ptrs (except intermediate) |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 905 | std::unordered_map<ComponentID, ComponentUniquePtr> _components{}; |
SiCong Li | b63b119 | 2022-01-28 18:24:39 +0000 | [diff] [blame^] | 906 | std::unordered_map<ArgumentID, ITensorInfo *> _kernel_tensors{}; |
Giorgio Arena | 232c452 | 2022-03-03 10:09:01 +0000 | [diff] [blame] | 907 | // Argument group lookup. Can be replaced by extending the ArgumentID type to include group info |
| 908 | std::unordered_map<ArgumentID, SharedVarGroup> _shared_var_group_lut{}; |
| 909 | |
| 910 | // Tracks all variables (e.g.: kernel arguments, kernel "global variables") |
| 911 | SharedVarTable _vtable{}; |
| 912 | |
| 913 | // Component directed graph (represented by an adjecency list of Component IDs) |
| 914 | // This is used to understand the ordering and bindings between components when generating the kernel |
| 915 | // It's initially set to -1 which means the graph has no root yet, since node IDs are positive numbers |
| 916 | ComponentID _graph_root{ -1 }; |
| 917 | std::unordered_map<ComponentID, ComponentList> _component_graph{}; |
| 918 | |
| 919 | // Additional data structures used to define the relationships between components and arguments |
| 920 | // For each argument, it contains the list of components that consider it as an incoming or an outgoing argument |
| 921 | // E.g. tensor0 -> component0 -> tensor1 |
| 922 | // _outgoing_components[tensor0] == {component0} (component0 is the outgoing component of tensor0. Component0 treats tensor0 as an input tensor) |
| 923 | // _incoming_components[tensor1] == {component0} (component0 is the incoming component of tensor1. Component1 treats tensor1 as an output tensor) |
| 924 | std::unordered_map<ArgumentID, ComponentList> _outgoing_components{}; |
| 925 | std::unordered_map<ArgumentID, ComponentList> _incoming_components{}; |
| 926 | }; |
| 927 | |
| 928 | } // namespace dynamic_fusion |
| 929 | } // namespace experimental |
| 930 | } // namespace arm_compute |
SiCong Li | b63b119 | 2022-01-28 18:24:39 +0000 | [diff] [blame^] | 931 | #endif //ARM_COMPUTE_EXPERIMENTAL_DYNAMICFUSION_IMPL_COMMON_H |