Anthony Barbier | 6ff3b19 | 2017-09-04 18:44:23 +0100 | [diff] [blame] | 1 | /* |
Matthew Bentham | 0a59e69 | 2023-07-19 15:01:00 +0000 | [diff] [blame] | 2 | * Copyright (c) 2016-2023 Arm Limited. |
Anthony Barbier | 6ff3b19 | 2017-09-04 18:44:23 +0100 | [diff] [blame] | 3 | * |
| 4 | * SPDX-License-Identifier: MIT |
| 5 | * |
| 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy |
| 7 | * of this software and associated documentation files (the "Software"), to |
| 8 | * deal in the Software without restriction, including without limitation the |
| 9 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or |
| 10 | * sell copies of the Software, and to permit persons to whom the Software is |
| 11 | * furnished to do so, subject to the following conditions: |
| 12 | * |
| 13 | * The above copyright notice and this permission notice shall be included in all |
| 14 | * copies or substantial portions of the Software. |
| 15 | * |
| 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
| 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
| 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE |
| 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
| 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
| 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
| 22 | * SOFTWARE. |
| 23 | */ |
| 24 | #include "arm_compute/runtime/CPP/CPPScheduler.h" |
| 25 | |
| 26 | #include "arm_compute/core/CPP/ICPPKernel.h" |
| 27 | #include "arm_compute/core/Error.h" |
| 28 | #include "arm_compute/core/Helpers.h" |
SiCongLi | 3b5981c | 2021-03-12 12:31:17 +0000 | [diff] [blame] | 29 | #include "arm_compute/core/Log.h" |
Anthony Barbier | 6ff3b19 | 2017-09-04 18:44:23 +0100 | [diff] [blame] | 30 | #include "arm_compute/core/Utils.h" |
SiCongLi | 3b5981c | 2021-03-12 12:31:17 +0000 | [diff] [blame] | 31 | #include "arm_compute/core/utils/misc/Utility.h" |
Pablo Tello | 2725197 | 2019-09-19 16:39:04 +0100 | [diff] [blame] | 32 | #include "support/Mutex.h" |
Anthony Barbier | 6ff3b19 | 2017-09-04 18:44:23 +0100 | [diff] [blame] | 33 | |
Anthony Barbier | d89940e | 2018-06-28 13:39:35 +0100 | [diff] [blame] | 34 | #include <atomic> |
Moritz Pflanzer | ff06f20 | 2017-09-08 13:48:23 +0100 | [diff] [blame] | 35 | #include <condition_variable> |
Anthony Barbier | 6ff3b19 | 2017-09-04 18:44:23 +0100 | [diff] [blame] | 36 | #include <iostream> |
Georgios Pinitas | 12833d0 | 2019-07-25 13:31:10 +0100 | [diff] [blame] | 37 | #include <list> |
Georgios Pinitas | 40f51a6 | 2020-11-21 03:04:18 +0000 | [diff] [blame] | 38 | #include <memory> |
Moritz Pflanzer | ff06f20 | 2017-09-08 13:48:23 +0100 | [diff] [blame] | 39 | #include <mutex> |
Anthony Barbier | 6ff3b19 | 2017-09-04 18:44:23 +0100 | [diff] [blame] | 40 | #include <system_error> |
| 41 | #include <thread> |
SiCongLi | 3b5981c | 2021-03-12 12:31:17 +0000 | [diff] [blame] | 42 | #include <vector> |
Anthony Barbier | 6ff3b19 | 2017-09-04 18:44:23 +0100 | [diff] [blame] | 43 | |
Moritz Pflanzer | ff06f20 | 2017-09-08 13:48:23 +0100 | [diff] [blame] | 44 | namespace arm_compute |
| 45 | { |
Anthony Barbier | 52ecb06 | 2018-05-25 13:32:10 +0100 | [diff] [blame] | 46 | namespace |
| 47 | { |
| 48 | class ThreadFeeder |
| 49 | { |
| 50 | public: |
| 51 | /** Constructor |
| 52 | * |
| 53 | * @param[in] start First value that will be returned by the feeder |
| 54 | * @param[in] end End condition (The last value returned by get_next() will be end - 1) |
| 55 | */ |
| 56 | explicit ThreadFeeder(unsigned int start = 0, unsigned int end = 0) |
Anthony Barbier | d89940e | 2018-06-28 13:39:35 +0100 | [diff] [blame] | 57 | : _atomic_counter(start), _end(end) |
Anthony Barbier | 52ecb06 | 2018-05-25 13:32:10 +0100 | [diff] [blame] | 58 | { |
| 59 | } |
| 60 | /** Return the next element in the range if there is one. |
| 61 | * |
| 62 | * @param[out] next Will contain the next element if there is one. |
| 63 | * |
| 64 | * @return False if the end of the range has been reached and next wasn't set. |
| 65 | */ |
| 66 | bool get_next(unsigned int &next) |
| 67 | { |
Anthony Barbier | d89940e | 2018-06-28 13:39:35 +0100 | [diff] [blame] | 68 | next = atomic_fetch_add_explicit(&_atomic_counter, 1u, std::memory_order_relaxed); |
| 69 | return next < _end; |
Anthony Barbier | 52ecb06 | 2018-05-25 13:32:10 +0100 | [diff] [blame] | 70 | } |
| 71 | |
| 72 | private: |
Anthony Barbier | d89940e | 2018-06-28 13:39:35 +0100 | [diff] [blame] | 73 | std::atomic_uint _atomic_counter; |
Anthony Barbier | 52ecb06 | 2018-05-25 13:32:10 +0100 | [diff] [blame] | 74 | const unsigned int _end; |
Anthony Barbier | 52ecb06 | 2018-05-25 13:32:10 +0100 | [diff] [blame] | 75 | }; |
| 76 | |
| 77 | /** Execute workloads[info.thread_id] first, then call the feeder to get the index of the next workload to run. |
| 78 | * |
| 79 | * Will run workloads until the feeder reaches the end of its range. |
| 80 | * |
| 81 | * @param[in] workloads The array of workloads |
| 82 | * @param[in,out] feeder The feeder indicating which workload to execute next. |
| 83 | * @param[in] info Threading and CPU info. |
| 84 | */ |
| 85 | void process_workloads(std::vector<IScheduler::Workload> &workloads, ThreadFeeder &feeder, const ThreadInfo &info) |
| 86 | { |
| 87 | unsigned int workload_index = info.thread_id; |
| 88 | do |
| 89 | { |
| 90 | ARM_COMPUTE_ERROR_ON(workload_index >= workloads.size()); |
| 91 | workloads[workload_index](info); |
| 92 | } |
| 93 | while(feeder.get_next(workload_index)); |
| 94 | } |
Anthony Barbier | 52ecb06 | 2018-05-25 13:32:10 +0100 | [diff] [blame] | 95 | |
SiCongLi | 3b5981c | 2021-03-12 12:31:17 +0000 | [diff] [blame] | 96 | /** Set thread affinity. Pin current thread to a particular core |
| 97 | * |
| 98 | * @param[in] core_id ID of the core to which the current thread is pinned |
| 99 | */ |
Georgios Pinitas | 06e890b | 2020-07-09 18:38:34 +0100 | [diff] [blame] | 100 | void set_thread_affinity(int core_id) |
Georgios Pinitas | 12833d0 | 2019-07-25 13:31:10 +0100 | [diff] [blame] | 101 | { |
Georgios Pinitas | 06e890b | 2020-07-09 18:38:34 +0100 | [diff] [blame] | 102 | if(core_id < 0) |
Georgios Pinitas | 12833d0 | 2019-07-25 13:31:10 +0100 | [diff] [blame] | 103 | { |
Georgios Pinitas | 06e890b | 2020-07-09 18:38:34 +0100 | [diff] [blame] | 104 | return; |
Georgios Pinitas | 12833d0 | 2019-07-25 13:31:10 +0100 | [diff] [blame] | 105 | } |
| 106 | |
Pablo Tello | 4e66d70 | 2022-03-07 18:20:12 +0000 | [diff] [blame] | 107 | #if !defined(_WIN64) && !defined(__APPLE__) && !defined(__OpenBSD__) |
Georgios Pinitas | 06e890b | 2020-07-09 18:38:34 +0100 | [diff] [blame] | 108 | cpu_set_t set; |
| 109 | CPU_ZERO(&set); |
| 110 | CPU_SET(core_id, &set); |
Georgios Pinitas | 4551403 | 2020-12-30 00:03:09 +0000 | [diff] [blame] | 111 | ARM_COMPUTE_EXIT_ON_MSG(sched_setaffinity(0, sizeof(set), &set), "Error setting thread affinity"); |
Kevin Lo | 7195f71 | 2022-01-07 15:46:02 +0800 | [diff] [blame] | 112 | #endif /* !defined(__APPLE__) && !defined(__OpenBSD__) */ |
Georgios Pinitas | 06e890b | 2020-07-09 18:38:34 +0100 | [diff] [blame] | 113 | } |
Georgios Pinitas | 12833d0 | 2019-07-25 13:31:10 +0100 | [diff] [blame] | 114 | |
SiCongLi | 3b5981c | 2021-03-12 12:31:17 +0000 | [diff] [blame] | 115 | /** There are currently 2 scheduling modes supported by CPPScheduler |
| 116 | * |
| 117 | * Linear: |
| 118 | * The default mode where all the scheduling is carried out by the main thread linearly (in a loop). |
| 119 | * E.G. If there are 8 threads in total, there will be 1 main thread + 7 threads in the thread pool, and it is main |
| 120 | * thread's responsibility to start all the other threads in the thread pool. |
| 121 | * |
| 122 | * Fanout: |
| 123 | * In fanout mode, the scheduling (starting other threads) task is distributed across many threads instead of just |
| 124 | * the main thread. |
| 125 | * |
| 126 | * The scheduler has a fixed parameter: wake_fanout, and the scheduling sequence goes like this: |
| 127 | * 1. Main thread wakes the first wake_fanout - 1 number of FanoutThreads from the thread pool |
| 128 | * From thread: 0 |
| 129 | * To thread (non-inclusive): Wake_fanout - 1 |
| 130 | * 2. Each FanoutThread then wakes wake_fanout number of FanoutThreads from the thread pool: |
| 131 | * From thread: (i + 1) * wake_fanout - 1 |
| 132 | * To thread (non-inclusive): (i + 2) * wake_fanout - 1 |
| 133 | * where i is the current thread's thread id |
| 134 | * The end is clamped at the size of the thread pool / the number of threads in use - 1 |
| 135 | * |
| 136 | * E.G. for a total number of 8 threads (1 main thread, 7 FanoutThreads in thread pool) with a fanout of 3 |
| 137 | * 1. Main thread wakes FanoutThread 0, 1 |
| 138 | * 2. FanoutThread 0 wakes FanoutThread 2, 3, 4 |
| 139 | * 3. FanoutThread 1 wakes FanoutThread 5, 6 |
| 140 | */ |
| 141 | |
Georgios Pinitas | 06e890b | 2020-07-09 18:38:34 +0100 | [diff] [blame] | 142 | class Thread final |
Anthony Barbier | 6ff3b19 | 2017-09-04 18:44:23 +0100 | [diff] [blame] | 143 | { |
| 144 | public: |
Georgios Pinitas | 06e890b | 2020-07-09 18:38:34 +0100 | [diff] [blame] | 145 | /** Start a new thread |
| 146 | * |
| 147 | * Thread will be pinned to a given core id if value is non-negative |
| 148 | * |
| 149 | * @param[in] core_pin Core id to pin the thread on. If negative no thread pinning will take place |
| 150 | */ |
| 151 | explicit Thread(int core_pin = -1); |
Moritz Pflanzer | ff06f20 | 2017-09-08 13:48:23 +0100 | [diff] [blame] | 152 | |
Anthony Barbier | 6ff3b19 | 2017-09-04 18:44:23 +0100 | [diff] [blame] | 153 | Thread(const Thread &) = delete; |
| 154 | Thread &operator=(const Thread &) = delete; |
| 155 | Thread(Thread &&) = delete; |
| 156 | Thread &operator=(Thread &&) = delete; |
Moritz Pflanzer | ff06f20 | 2017-09-08 13:48:23 +0100 | [diff] [blame] | 157 | |
| 158 | /** Destructor. Make the thread join. */ |
Anthony Barbier | 6ff3b19 | 2017-09-04 18:44:23 +0100 | [diff] [blame] | 159 | ~Thread(); |
Moritz Pflanzer | ff06f20 | 2017-09-08 13:48:23 +0100 | [diff] [blame] | 160 | |
SiCongLi | 3b5981c | 2021-03-12 12:31:17 +0000 | [diff] [blame] | 161 | /** Set workloads */ |
| 162 | void set_workload(std::vector<IScheduler::Workload> *workloads, ThreadFeeder &feeder, const ThreadInfo &info); |
| 163 | |
Anthony Barbier | 52ecb06 | 2018-05-25 13:32:10 +0100 | [diff] [blame] | 164 | /** Request the worker thread to start executing workloads. |
| 165 | * |
| 166 | * The thread will start by executing workloads[info.thread_id] and will then call the feeder to |
| 167 | * get the index of the following workload to run. |
| 168 | * |
| 169 | * @note This function will return as soon as the workloads have been sent to the worker thread. |
Anthony Barbier | 6ff3b19 | 2017-09-04 18:44:23 +0100 | [diff] [blame] | 170 | * wait() needs to be called to ensure the execution is complete. |
| 171 | */ |
SiCongLi | 3b5981c | 2021-03-12 12:31:17 +0000 | [diff] [blame] | 172 | void start(); |
Moritz Pflanzer | ff06f20 | 2017-09-08 13:48:23 +0100 | [diff] [blame] | 173 | |
| 174 | /** Wait for the current kernel execution to complete. */ |
Matthew Bentham | 0a59e69 | 2023-07-19 15:01:00 +0000 | [diff] [blame] | 175 | std::exception_ptr wait(); |
Moritz Pflanzer | ff06f20 | 2017-09-08 13:48:23 +0100 | [diff] [blame] | 176 | |
| 177 | /** Function ran by the worker thread. */ |
Anthony Barbier | 6ff3b19 | 2017-09-04 18:44:23 +0100 | [diff] [blame] | 178 | void worker_thread(); |
| 179 | |
SiCongLi | 3b5981c | 2021-03-12 12:31:17 +0000 | [diff] [blame] | 180 | /** Set the scheduling strategy to be linear */ |
| 181 | void set_linear_mode() |
| 182 | { |
| 183 | _thread_pool = nullptr; |
| 184 | _wake_beg = 0; |
| 185 | _wake_end = 0; |
| 186 | } |
| 187 | |
| 188 | /** Set the scheduling strategy to be fanout */ |
| 189 | void set_fanout_mode(std::list<Thread> *thread_pool, unsigned int wake_beg, unsigned int wake_end) |
| 190 | { |
| 191 | _thread_pool = thread_pool; |
| 192 | _wake_beg = wake_beg; |
| 193 | _wake_end = wake_end; |
| 194 | } |
| 195 | |
Anthony Barbier | 6ff3b19 | 2017-09-04 18:44:23 +0100 | [diff] [blame] | 196 | private: |
Anthony Barbier | 52ecb06 | 2018-05-25 13:32:10 +0100 | [diff] [blame] | 197 | std::thread _thread{}; |
| 198 | ThreadInfo _info{}; |
| 199 | std::vector<IScheduler::Workload> *_workloads{ nullptr }; |
| 200 | ThreadFeeder *_feeder{ nullptr }; |
| 201 | std::mutex _m{}; |
| 202 | std::condition_variable _cv{}; |
| 203 | bool _wait_for_work{ false }; |
| 204 | bool _job_complete{ true }; |
| 205 | std::exception_ptr _current_exception{ nullptr }; |
Georgios Pinitas | 06e890b | 2020-07-09 18:38:34 +0100 | [diff] [blame] | 206 | int _core_pin{ -1 }; |
SiCongLi | 3b5981c | 2021-03-12 12:31:17 +0000 | [diff] [blame] | 207 | std::list<Thread> *_thread_pool{ nullptr }; |
| 208 | unsigned int _wake_beg{ 0 }; |
| 209 | unsigned int _wake_end{ 0 }; |
Anthony Barbier | 6ff3b19 | 2017-09-04 18:44:23 +0100 | [diff] [blame] | 210 | }; |
| 211 | |
Georgios Pinitas | 06e890b | 2020-07-09 18:38:34 +0100 | [diff] [blame] | 212 | Thread::Thread(int core_pin) |
| 213 | : _core_pin(core_pin) |
Anthony Barbier | 6ff3b19 | 2017-09-04 18:44:23 +0100 | [diff] [blame] | 214 | { |
Anthony Barbier | 6ff3b19 | 2017-09-04 18:44:23 +0100 | [diff] [blame] | 215 | _thread = std::thread(&Thread::worker_thread, this); |
| 216 | } |
| 217 | |
Georgios Pinitas | 06e890b | 2020-07-09 18:38:34 +0100 | [diff] [blame] | 218 | Thread::~Thread() |
Anthony Barbier | 6ff3b19 | 2017-09-04 18:44:23 +0100 | [diff] [blame] | 219 | { |
Moritz Pflanzer | ff06f20 | 2017-09-08 13:48:23 +0100 | [diff] [blame] | 220 | // Make sure worker thread has ended |
| 221 | if(_thread.joinable()) |
| 222 | { |
Anthony Barbier | 52ecb06 | 2018-05-25 13:32:10 +0100 | [diff] [blame] | 223 | ThreadFeeder feeder; |
SiCongLi | 3b5981c | 2021-03-12 12:31:17 +0000 | [diff] [blame] | 224 | set_workload(nullptr, feeder, ThreadInfo()); |
| 225 | start(); |
Moritz Pflanzer | ff06f20 | 2017-09-08 13:48:23 +0100 | [diff] [blame] | 226 | _thread.join(); |
| 227 | } |
Anthony Barbier | 6ff3b19 | 2017-09-04 18:44:23 +0100 | [diff] [blame] | 228 | } |
| 229 | |
SiCongLi | 3b5981c | 2021-03-12 12:31:17 +0000 | [diff] [blame] | 230 | void Thread::set_workload(std::vector<IScheduler::Workload> *workloads, ThreadFeeder &feeder, const ThreadInfo &info) |
Anthony Barbier | 6ff3b19 | 2017-09-04 18:44:23 +0100 | [diff] [blame] | 231 | { |
Anthony Barbier | 52ecb06 | 2018-05-25 13:32:10 +0100 | [diff] [blame] | 232 | _workloads = workloads; |
| 233 | _feeder = &feeder; |
| 234 | _info = info; |
SiCongLi | 3b5981c | 2021-03-12 12:31:17 +0000 | [diff] [blame] | 235 | } |
| 236 | |
| 237 | void Thread::start() |
| 238 | { |
Moritz Pflanzer | ff06f20 | 2017-09-08 13:48:23 +0100 | [diff] [blame] | 239 | { |
| 240 | std::lock_guard<std::mutex> lock(_m); |
| 241 | _wait_for_work = true; |
| 242 | _job_complete = false; |
| 243 | } |
| 244 | _cv.notify_one(); |
Anthony Barbier | 6ff3b19 | 2017-09-04 18:44:23 +0100 | [diff] [blame] | 245 | } |
| 246 | |
Matthew Bentham | 0a59e69 | 2023-07-19 15:01:00 +0000 | [diff] [blame] | 247 | std::exception_ptr Thread::wait() |
Anthony Barbier | 6ff3b19 | 2017-09-04 18:44:23 +0100 | [diff] [blame] | 248 | { |
Moritz Pflanzer | ff06f20 | 2017-09-08 13:48:23 +0100 | [diff] [blame] | 249 | { |
| 250 | std::unique_lock<std::mutex> lock(_m); |
| 251 | _cv.wait(lock, [&] { return _job_complete; }); |
| 252 | } |
Matthew Bentham | 0a59e69 | 2023-07-19 15:01:00 +0000 | [diff] [blame] | 253 | return _current_exception; |
Anthony Barbier | 6ff3b19 | 2017-09-04 18:44:23 +0100 | [diff] [blame] | 254 | } |
| 255 | |
Georgios Pinitas | 06e890b | 2020-07-09 18:38:34 +0100 | [diff] [blame] | 256 | void Thread::worker_thread() |
Anthony Barbier | 6ff3b19 | 2017-09-04 18:44:23 +0100 | [diff] [blame] | 257 | { |
Georgios Pinitas | 06e890b | 2020-07-09 18:38:34 +0100 | [diff] [blame] | 258 | set_thread_affinity(_core_pin); |
| 259 | |
Moritz Pflanzer | ff06f20 | 2017-09-08 13:48:23 +0100 | [diff] [blame] | 260 | while(true) |
Anthony Barbier | 6ff3b19 | 2017-09-04 18:44:23 +0100 | [diff] [blame] | 261 | { |
Moritz Pflanzer | ff06f20 | 2017-09-08 13:48:23 +0100 | [diff] [blame] | 262 | std::unique_lock<std::mutex> lock(_m); |
| 263 | _cv.wait(lock, [&] { return _wait_for_work; }); |
| 264 | _wait_for_work = false; |
| 265 | |
Anthony Barbier | 6ff3b19 | 2017-09-04 18:44:23 +0100 | [diff] [blame] | 266 | _current_exception = nullptr; |
Moritz Pflanzer | ff06f20 | 2017-09-08 13:48:23 +0100 | [diff] [blame] | 267 | |
SiCongLi | 3b5981c | 2021-03-12 12:31:17 +0000 | [diff] [blame] | 268 | // Exit if the worker thread has not been fed with workloads |
| 269 | if(_workloads == nullptr || _feeder == nullptr) |
Anthony Barbier | 6ff3b19 | 2017-09-04 18:44:23 +0100 | [diff] [blame] | 270 | { |
| 271 | return; |
| 272 | } |
| 273 | |
SiCongLi | 3b5981c | 2021-03-12 12:31:17 +0000 | [diff] [blame] | 274 | // Wake up more peer threads from thread pool if this job has been delegated to the current thread |
| 275 | if(_thread_pool != nullptr) |
| 276 | { |
| 277 | auto thread_it = _thread_pool->begin(); |
| 278 | std::advance(thread_it, std::min(static_cast<unsigned int>(_thread_pool->size()), _wake_beg)); |
| 279 | auto wake_end = std::min(_wake_end, static_cast<unsigned int>(_info.num_threads - 1)); |
| 280 | for(unsigned int t = _wake_beg; t < wake_end; ++t, ++thread_it) |
| 281 | { |
| 282 | thread_it->start(); |
| 283 | } |
| 284 | } |
| 285 | |
Michalis Spyrou | 323ce0f | 2018-11-30 16:30:43 +0000 | [diff] [blame] | 286 | #ifndef ARM_COMPUTE_EXCEPTIONS_DISABLED |
Anthony Barbier | 6ff3b19 | 2017-09-04 18:44:23 +0100 | [diff] [blame] | 287 | try |
| 288 | { |
Michalis Spyrou | 323ce0f | 2018-11-30 16:30:43 +0000 | [diff] [blame] | 289 | #endif /* ARM_COMPUTE_EXCEPTIONS_ENABLED */ |
Anthony Barbier | 52ecb06 | 2018-05-25 13:32:10 +0100 | [diff] [blame] | 290 | process_workloads(*_workloads, *_feeder, _info); |
Michalis Spyrou | 323ce0f | 2018-11-30 16:30:43 +0000 | [diff] [blame] | 291 | |
| 292 | #ifndef ARM_COMPUTE_EXCEPTIONS_DISABLED |
Anthony Barbier | 6ff3b19 | 2017-09-04 18:44:23 +0100 | [diff] [blame] | 293 | } |
| 294 | catch(...) |
| 295 | { |
| 296 | _current_exception = std::current_exception(); |
| 297 | } |
Michalis Spyrou | 323ce0f | 2018-11-30 16:30:43 +0000 | [diff] [blame] | 298 | #endif /* ARM_COMPUTE_EXCEPTIONS_DISABLED */ |
SiCongLi | 3b5981c | 2021-03-12 12:31:17 +0000 | [diff] [blame] | 299 | _workloads = nullptr; |
Moritz Pflanzer | ff06f20 | 2017-09-08 13:48:23 +0100 | [diff] [blame] | 300 | _job_complete = true; |
| 301 | lock.unlock(); |
| 302 | _cv.notify_one(); |
Anthony Barbier | 6ff3b19 | 2017-09-04 18:44:23 +0100 | [diff] [blame] | 303 | } |
Anthony Barbier | 6ff3b19 | 2017-09-04 18:44:23 +0100 | [diff] [blame] | 304 | } |
Georgios Pinitas | 06e890b | 2020-07-09 18:38:34 +0100 | [diff] [blame] | 305 | } //namespace |
| 306 | |
| 307 | struct CPPScheduler::Impl final |
| 308 | { |
SiCongLi | 3b5981c | 2021-03-12 12:31:17 +0000 | [diff] [blame] | 309 | constexpr static unsigned int m_default_wake_fanout = 4; |
| 310 | enum class Mode |
Georgios Pinitas | 06e890b | 2020-07-09 18:38:34 +0100 | [diff] [blame] | 311 | { |
SiCongLi | 3b5981c | 2021-03-12 12:31:17 +0000 | [diff] [blame] | 312 | Linear, |
| 313 | Fanout |
| 314 | }; |
| 315 | enum class ModeToggle |
| 316 | { |
| 317 | None, |
| 318 | Linear, |
| 319 | Fanout |
| 320 | }; |
| 321 | explicit Impl(unsigned int thread_hint) |
| 322 | : _num_threads(thread_hint), _threads(_num_threads - 1), _mode(Mode::Linear), _wake_fanout(0U) |
| 323 | { |
| 324 | const auto mode_env_v = utility::tolower(utility::getenv("ARM_COMPUTE_CPP_SCHEDULER_MODE")); |
| 325 | if(mode_env_v == "linear") |
| 326 | { |
| 327 | _forced_mode = ModeToggle::Linear; |
| 328 | } |
| 329 | else if(mode_env_v == "fanout") |
| 330 | { |
| 331 | _forced_mode = ModeToggle::Fanout; |
| 332 | } |
| 333 | else |
| 334 | { |
| 335 | _forced_mode = ModeToggle::None; |
| 336 | } |
Georgios Pinitas | 06e890b | 2020-07-09 18:38:34 +0100 | [diff] [blame] | 337 | } |
| 338 | void set_num_threads(unsigned int num_threads, unsigned int thread_hint) |
| 339 | { |
| 340 | _num_threads = num_threads == 0 ? thread_hint : num_threads; |
| 341 | _threads.resize(_num_threads - 1); |
SiCongLi | 3b5981c | 2021-03-12 12:31:17 +0000 | [diff] [blame] | 342 | auto_switch_mode(_num_threads); |
Georgios Pinitas | 06e890b | 2020-07-09 18:38:34 +0100 | [diff] [blame] | 343 | } |
| 344 | void set_num_threads_with_affinity(unsigned int num_threads, unsigned int thread_hint, BindFunc func) |
| 345 | { |
| 346 | _num_threads = num_threads == 0 ? thread_hint : num_threads; |
| 347 | |
| 348 | // Set affinity on main thread |
| 349 | set_thread_affinity(func(0, thread_hint)); |
| 350 | |
| 351 | // Set affinity on worked threads |
| 352 | _threads.clear(); |
| 353 | for(auto i = 1U; i < _num_threads; ++i) |
| 354 | { |
| 355 | _threads.emplace_back(func(i, thread_hint)); |
| 356 | } |
SiCongLi | 3b5981c | 2021-03-12 12:31:17 +0000 | [diff] [blame] | 357 | auto_switch_mode(_num_threads); |
| 358 | } |
| 359 | void auto_switch_mode(unsigned int num_threads_to_use) |
| 360 | { |
| 361 | // If the environment variable is set to any of the modes, it overwrites the mode selected over num_threads_to_use |
| 362 | if(_forced_mode == ModeToggle::Fanout || (_forced_mode == ModeToggle::None && num_threads_to_use > 8)) |
| 363 | { |
| 364 | set_fanout_mode(m_default_wake_fanout, num_threads_to_use); |
| 365 | ARM_COMPUTE_LOG_INFO_MSG_WITH_FORMAT_CORE("Set CPPScheduler to Fanout mode, with wake up fanout : %d and %d threads to use\n", this->wake_fanout(), num_threads_to_use); |
| 366 | } |
| 367 | else // Equivalent to (_forced_mode == ModeToggle::Linear || (_forced_mode == ModeToggle::None && num_threads_to_use <= 8)) |
| 368 | { |
| 369 | set_linear_mode(); |
| 370 | ARM_COMPUTE_LOG_INFO_MSG_WITH_FORMAT_CORE("Set CPPScheduler to Linear mode, with %d threads to use\n", num_threads_to_use); |
| 371 | } |
| 372 | } |
| 373 | void set_linear_mode() |
| 374 | { |
| 375 | for(auto &thread : _threads) |
| 376 | { |
| 377 | thread.set_linear_mode(); |
| 378 | } |
| 379 | _mode = Mode::Linear; |
| 380 | _wake_fanout = 0U; |
| 381 | } |
| 382 | void set_fanout_mode(unsigned int wake_fanout, unsigned int num_threads_to_use) |
| 383 | { |
| 384 | ARM_COMPUTE_ERROR_ON(num_threads_to_use > _threads.size() + 1); |
| 385 | const auto actual_wake_fanout = std::max(2U, std::min(wake_fanout, num_threads_to_use - 1)); |
| 386 | auto thread_it = _threads.begin(); |
| 387 | for(auto i = 1U; i < num_threads_to_use; ++i, ++thread_it) |
| 388 | { |
| 389 | const auto wake_begin = i * actual_wake_fanout - 1; |
| 390 | const auto wake_end = std::min((i + 1) * actual_wake_fanout - 1, num_threads_to_use - 1); |
| 391 | thread_it->set_fanout_mode(&_threads, wake_begin, wake_end); |
| 392 | } |
| 393 | // Reset the remaining threads's wake up schedule |
| 394 | while(thread_it != _threads.end()) |
| 395 | { |
| 396 | thread_it->set_fanout_mode(&_threads, 0U, 0U); |
| 397 | ++thread_it; |
| 398 | } |
| 399 | _mode = Mode::Fanout; |
| 400 | _wake_fanout = actual_wake_fanout; |
Georgios Pinitas | 06e890b | 2020-07-09 18:38:34 +0100 | [diff] [blame] | 401 | } |
| 402 | unsigned int num_threads() const |
| 403 | { |
| 404 | return _num_threads; |
| 405 | } |
SiCongLi | 3b5981c | 2021-03-12 12:31:17 +0000 | [diff] [blame] | 406 | unsigned int wake_fanout() const |
| 407 | { |
| 408 | return _wake_fanout; |
| 409 | } |
| 410 | Mode mode() const |
| 411 | { |
| 412 | return _mode; |
| 413 | } |
Georgios Pinitas | 06e890b | 2020-07-09 18:38:34 +0100 | [diff] [blame] | 414 | |
| 415 | void run_workloads(std::vector<IScheduler::Workload> &workloads); |
| 416 | |
| 417 | unsigned int _num_threads; |
| 418 | std::list<Thread> _threads; |
| 419 | arm_compute::Mutex _run_workloads_mutex{}; |
SiCongLi | 3b5981c | 2021-03-12 12:31:17 +0000 | [diff] [blame] | 420 | Mode _mode{ Mode::Linear }; |
| 421 | ModeToggle _forced_mode{ ModeToggle::None }; |
| 422 | unsigned int _wake_fanout{ 0 }; |
Georgios Pinitas | 06e890b | 2020-07-09 18:38:34 +0100 | [diff] [blame] | 423 | }; |
Anthony Barbier | 6ff3b19 | 2017-09-04 18:44:23 +0100 | [diff] [blame] | 424 | |
Georgios Pinitas | 12833d0 | 2019-07-25 13:31:10 +0100 | [diff] [blame] | 425 | /* |
Giorgio Arena | 5b50f42 | 2021-02-17 11:43:05 +0000 | [diff] [blame] | 426 | * This singleton has been deprecated and will be removed in future releases |
Georgios Pinitas | 12833d0 | 2019-07-25 13:31:10 +0100 | [diff] [blame] | 427 | */ |
Anthony Barbier | 6ff3b19 | 2017-09-04 18:44:23 +0100 | [diff] [blame] | 428 | CPPScheduler &CPPScheduler::get() |
| 429 | { |
| 430 | static CPPScheduler scheduler; |
| 431 | return scheduler; |
| 432 | } |
| 433 | |
Anthony Barbier | 6ff3b19 | 2017-09-04 18:44:23 +0100 | [diff] [blame] | 434 | CPPScheduler::CPPScheduler() |
Georgios Pinitas | 40f51a6 | 2020-11-21 03:04:18 +0000 | [diff] [blame] | 435 | : _impl(std::make_unique<Impl>(num_threads_hint())) |
Anthony Barbier | 6ff3b19 | 2017-09-04 18:44:23 +0100 | [diff] [blame] | 436 | { |
| 437 | } |
| 438 | |
Georgios Pinitas | 12833d0 | 2019-07-25 13:31:10 +0100 | [diff] [blame] | 439 | CPPScheduler::~CPPScheduler() = default; |
| 440 | |
Anthony Barbier | 6ff3b19 | 2017-09-04 18:44:23 +0100 | [diff] [blame] | 441 | void CPPScheduler::set_num_threads(unsigned int num_threads) |
| 442 | { |
Pablo Tello | 2725197 | 2019-09-19 16:39:04 +0100 | [diff] [blame] | 443 | // No changes in the number of threads while current workloads are running |
| 444 | arm_compute::lock_guard<std::mutex> lock(_impl->_run_workloads_mutex); |
Georgios Pinitas | 12833d0 | 2019-07-25 13:31:10 +0100 | [diff] [blame] | 445 | _impl->set_num_threads(num_threads, num_threads_hint()); |
Anthony Barbier | 6ff3b19 | 2017-09-04 18:44:23 +0100 | [diff] [blame] | 446 | } |
| 447 | |
Georgios Pinitas | 06e890b | 2020-07-09 18:38:34 +0100 | [diff] [blame] | 448 | void CPPScheduler::set_num_threads_with_affinity(unsigned int num_threads, BindFunc func) |
| 449 | { |
| 450 | // No changes in the number of threads while current workloads are running |
| 451 | arm_compute::lock_guard<std::mutex> lock(_impl->_run_workloads_mutex); |
| 452 | _impl->set_num_threads_with_affinity(num_threads, num_threads_hint(), func); |
| 453 | } |
| 454 | |
Moritz Pflanzer | d929b9c | 2017-06-28 10:15:48 +0100 | [diff] [blame] | 455 | unsigned int CPPScheduler::num_threads() const |
| 456 | { |
Georgios Pinitas | 12833d0 | 2019-07-25 13:31:10 +0100 | [diff] [blame] | 457 | return _impl->num_threads(); |
Moritz Pflanzer | d929b9c | 2017-06-28 10:15:48 +0100 | [diff] [blame] | 458 | } |
| 459 | |
Vidhya Sudhan Loganathan | d646ae1 | 2018-11-19 15:18:20 +0000 | [diff] [blame] | 460 | #ifndef DOXYGEN_SKIP_THIS |
Anthony Barbier | 52ecb06 | 2018-05-25 13:32:10 +0100 | [diff] [blame] | 461 | void CPPScheduler::run_workloads(std::vector<IScheduler::Workload> &workloads) |
| 462 | { |
Pablo Tello | 2725197 | 2019-09-19 16:39:04 +0100 | [diff] [blame] | 463 | // Mutex to ensure other threads won't interfere with the setup of the current thread's workloads |
| 464 | // Other thread's workloads will be scheduled after the current thread's workloads have finished |
| 465 | // This is not great because different threads workloads won't run in parallel but at least they |
| 466 | // won't interfere each other and deadlock. |
| 467 | arm_compute::lock_guard<std::mutex> lock(_impl->_run_workloads_mutex); |
SiCongLi | 3b5981c | 2021-03-12 12:31:17 +0000 | [diff] [blame] | 468 | const unsigned int num_threads_to_use = std::min(_impl->num_threads(), static_cast<unsigned int>(workloads.size())); |
| 469 | if(num_threads_to_use < 1) |
Anthony Barbier | 52ecb06 | 2018-05-25 13:32:10 +0100 | [diff] [blame] | 470 | { |
| 471 | return; |
| 472 | } |
SiCongLi | 3b5981c | 2021-03-12 12:31:17 +0000 | [diff] [blame] | 473 | // Re-adjust the mode if the actual number of threads to use is different from the number of threads created |
| 474 | _impl->auto_switch_mode(num_threads_to_use); |
| 475 | int num_threads_to_start = 0; |
| 476 | switch(_impl->mode()) |
| 477 | { |
| 478 | case CPPScheduler::Impl::Mode::Fanout: |
| 479 | { |
| 480 | num_threads_to_start = static_cast<int>(_impl->wake_fanout()) - 1; |
| 481 | break; |
| 482 | } |
| 483 | case CPPScheduler::Impl::Mode::Linear: |
| 484 | default: |
| 485 | { |
| 486 | num_threads_to_start = static_cast<int>(num_threads_to_use) - 1; |
| 487 | break; |
| 488 | } |
| 489 | } |
| 490 | ThreadFeeder feeder(num_threads_to_use, workloads.size()); |
Anthony Barbier | 52ecb06 | 2018-05-25 13:32:10 +0100 | [diff] [blame] | 491 | ThreadInfo info; |
Michalis Spyrou | 20fca52 | 2021-06-07 14:23:57 +0100 | [diff] [blame] | 492 | info.cpu_info = &cpu_info(); |
SiCongLi | 3b5981c | 2021-03-12 12:31:17 +0000 | [diff] [blame] | 493 | info.num_threads = num_threads_to_use; |
Anthony Barbier | 52ecb06 | 2018-05-25 13:32:10 +0100 | [diff] [blame] | 494 | unsigned int t = 0; |
Georgios Pinitas | 12833d0 | 2019-07-25 13:31:10 +0100 | [diff] [blame] | 495 | auto thread_it = _impl->_threads.begin(); |
SiCongLi | 3b5981c | 2021-03-12 12:31:17 +0000 | [diff] [blame] | 496 | // Set num_threads_to_use - 1 workloads to the threads as the remaining 1 is left to the main thread |
| 497 | for(; t < num_threads_to_use - 1; ++t, ++thread_it) |
Anthony Barbier | 52ecb06 | 2018-05-25 13:32:10 +0100 | [diff] [blame] | 498 | { |
| 499 | info.thread_id = t; |
SiCongLi | 3b5981c | 2021-03-12 12:31:17 +0000 | [diff] [blame] | 500 | thread_it->set_workload(&workloads, feeder, info); |
Anthony Barbier | 52ecb06 | 2018-05-25 13:32:10 +0100 | [diff] [blame] | 501 | } |
SiCongLi | 3b5981c | 2021-03-12 12:31:17 +0000 | [diff] [blame] | 502 | thread_it = _impl->_threads.begin(); |
| 503 | for(int i = 0; i < num_threads_to_start; ++i, ++thread_it) |
| 504 | { |
| 505 | thread_it->start(); |
| 506 | } |
| 507 | info.thread_id = t; // Set main thread's thread_id |
Matthew Bentham | 0a59e69 | 2023-07-19 15:01:00 +0000 | [diff] [blame] | 508 | std::exception_ptr last_exception = nullptr; |
Michalis Spyrou | 323ce0f | 2018-11-30 16:30:43 +0000 | [diff] [blame] | 509 | #ifndef ARM_COMPUTE_EXCEPTIONS_DISABLED |
Anthony Barbier | 52ecb06 | 2018-05-25 13:32:10 +0100 | [diff] [blame] | 510 | try |
| 511 | { |
Michalis Spyrou | 323ce0f | 2018-11-30 16:30:43 +0000 | [diff] [blame] | 512 | #endif /* ARM_COMPUTE_EXCEPTIONS_DISABLED */ |
Matthew Bentham | 0a59e69 | 2023-07-19 15:01:00 +0000 | [diff] [blame] | 513 | process_workloads(workloads, feeder, info); // Main thread processes workloads |
| 514 | #ifndef ARM_COMPUTE_EXCEPTIONS_DISABLED |
| 515 | } |
| 516 | catch (...) |
| 517 | { |
| 518 | last_exception = std::current_exception(); |
| 519 | } |
| 520 | |
| 521 | try |
| 522 | { |
| 523 | #endif /* ARM_COMPUTE_EXCEPTIONS_DISABLED */ |
SiCongLi | 3b5981c | 2021-03-12 12:31:17 +0000 | [diff] [blame] | 524 | thread_it = _impl->_threads.begin(); |
| 525 | for(unsigned int i = 0; i < num_threads_to_use - 1; ++i, ++thread_it) |
Anthony Barbier | 52ecb06 | 2018-05-25 13:32:10 +0100 | [diff] [blame] | 526 | { |
Matthew Bentham | 0a59e69 | 2023-07-19 15:01:00 +0000 | [diff] [blame] | 527 | std::exception_ptr current_exception = thread_it->wait(); |
| 528 | if (current_exception) |
| 529 | { |
| 530 | last_exception = current_exception; |
| 531 | } |
| 532 | } |
| 533 | if (last_exception) |
| 534 | { |
| 535 | std::rethrow_exception(last_exception); |
Anthony Barbier | 52ecb06 | 2018-05-25 13:32:10 +0100 | [diff] [blame] | 536 | } |
Michalis Spyrou | 323ce0f | 2018-11-30 16:30:43 +0000 | [diff] [blame] | 537 | #ifndef ARM_COMPUTE_EXCEPTIONS_DISABLED |
Anthony Barbier | 52ecb06 | 2018-05-25 13:32:10 +0100 | [diff] [blame] | 538 | } |
| 539 | catch(const std::system_error &e) |
| 540 | { |
| 541 | std::cerr << "Caught system_error with code " << e.code() << " meaning " << e.what() << '\n'; |
| 542 | } |
Michalis Spyrou | 323ce0f | 2018-11-30 16:30:43 +0000 | [diff] [blame] | 543 | #endif /* ARM_COMPUTE_EXCEPTIONS_DISABLED */ |
Anthony Barbier | 52ecb06 | 2018-05-25 13:32:10 +0100 | [diff] [blame] | 544 | } |
Vidhya Sudhan Loganathan | d646ae1 | 2018-11-19 15:18:20 +0000 | [diff] [blame] | 545 | #endif /* DOXYGEN_SKIP_THIS */ |
Anthony Barbier | 52ecb06 | 2018-05-25 13:32:10 +0100 | [diff] [blame] | 546 | |
Sang-Hoon Park | 0094c02 | 2021-01-20 18:16:47 +0000 | [diff] [blame] | 547 | void CPPScheduler::schedule_op(ICPPKernel *kernel, const Hints &hints, const Window &window, ITensorPack &tensors) |
Michalis Spyrou | bcd2352 | 2020-05-21 15:02:36 +0100 | [diff] [blame] | 548 | { |
Sang-Hoon Park | 0094c02 | 2021-01-20 18:16:47 +0000 | [diff] [blame] | 549 | schedule_common(kernel, hints, window, tensors); |
Michalis Spyrou | bcd2352 | 2020-05-21 15:02:36 +0100 | [diff] [blame] | 550 | } |
| 551 | |
| 552 | void CPPScheduler::schedule(ICPPKernel *kernel, const Hints &hints) |
| 553 | { |
Georgios Pinitas | 0499dff | 2020-07-31 22:21:38 +0100 | [diff] [blame] | 554 | ITensorPack tensors; |
Sang-Hoon Park | 0094c02 | 2021-01-20 18:16:47 +0000 | [diff] [blame] | 555 | schedule_common(kernel, hints, kernel->window(), tensors); |
Michalis Spyrou | bcd2352 | 2020-05-21 15:02:36 +0100 | [diff] [blame] | 556 | } |
Moritz Pflanzer | ff06f20 | 2017-09-08 13:48:23 +0100 | [diff] [blame] | 557 | } // namespace arm_compute |