Jim Flynn | e1fdd28 | 2021-10-26 21:26:10 +0100 | [diff] [blame] | 1 | // |
| 2 | // Copyright © 2021 Arm Ltd and Contributors. All rights reserved. |
| 3 | // SPDX-License-Identifier: MIT |
| 4 | // |
| 5 | #include "TestBlocks.hpp" |
| 6 | #include "TestStrategy.hpp" |
| 7 | |
| 8 | #include <IMemoryOptimizerStrategy.hpp> |
| 9 | #include <MemoryOptimizerStrategyLibrary.hpp> |
| 10 | #include <strategies/StrategyValidator.hpp> |
| 11 | |
| 12 | |
| 13 | #include <cxxopts.hpp> |
| 14 | |
| 15 | #include <iostream> |
| 16 | #include <algorithm> |
| 17 | #include <iomanip> |
| 18 | |
| 19 | std::vector<TestBlock> testBlocks |
| 20 | { |
Finn Williams | f9d96e5 | 2021-10-27 11:25:02 +0100 | [diff] [blame] | 21 | {"fsrcnn", fsrcnn}, |
Jim Flynn | e1fdd28 | 2021-10-26 21:26:10 +0100 | [diff] [blame] | 22 | {"inceptionv4", inceptionv4}, |
| 23 | {"deeplabv3", deeplabv3}, |
| 24 | {"deepspeechv1", deepspeechv1}, |
Finn Williams | f9d96e5 | 2021-10-27 11:25:02 +0100 | [diff] [blame] | 25 | {"mobilebert", mobilebert}, |
Jim Flynn | e1fdd28 | 2021-10-26 21:26:10 +0100 | [diff] [blame] | 26 | {"ssd_mobilenetv2", ssd_mobilenetv2}, |
| 27 | {"resnetv2", resnetv2}, |
| 28 | {"yolov3",yolov3} |
| 29 | }; |
| 30 | |
| 31 | void PrintModels() |
| 32 | { |
| 33 | std::cout << "Available models:\n"; |
| 34 | for (const auto& model : testBlocks) |
| 35 | { |
| 36 | std::cout << model.m_Name << "\n"; |
| 37 | } |
| 38 | std::cout << "\n"; |
| 39 | } |
| 40 | |
Finn Williams | f9d96e5 | 2021-10-27 11:25:02 +0100 | [diff] [blame] | 41 | size_t GetMinPossibleMemorySize(const std::vector<armnn::MemBlock>& blocks) |
Jim Flynn | e1fdd28 | 2021-10-26 21:26:10 +0100 | [diff] [blame] | 42 | { |
Finn Williams | f9d96e5 | 2021-10-27 11:25:02 +0100 | [diff] [blame] | 43 | unsigned int maxLifetime = 0; |
| 44 | for (auto& block: blocks) |
| 45 | { |
| 46 | maxLifetime = std::max(maxLifetime, block.m_EndOfLife); |
| 47 | } |
| 48 | maxLifetime++; |
| 49 | |
| 50 | std::vector<size_t> lifetimes(maxLifetime); |
Jim Flynn | e1fdd28 | 2021-10-26 21:26:10 +0100 | [diff] [blame] | 51 | for (const auto& block : blocks) |
| 52 | { |
| 53 | for (auto lifetime = block.m_StartOfLife; lifetime <= block.m_EndOfLife; ++lifetime) |
| 54 | { |
| 55 | lifetimes[lifetime] += block.m_MemSize; |
| 56 | } |
| 57 | } |
| 58 | return *std::max_element(lifetimes.begin(), lifetimes.end()); |
| 59 | } |
| 60 | |
| 61 | void RunBenchmark(armnn::IMemoryOptimizerStrategy* strategy, std::vector<TestBlock>* models) |
| 62 | { |
| 63 | using Clock = std::chrono::high_resolution_clock; |
| 64 | float avgEfficiency = 0; |
| 65 | std::chrono::duration<double, std::milli> avgDuration{}; |
| 66 | std::cout << "\nMemory Strategy: " << strategy->GetName()<< "\n"; |
| 67 | std::cout << "===============================================\n"; |
| 68 | for (auto& model : *models) |
| 69 | { |
| 70 | auto now = Clock::now(); |
| 71 | const std::vector<armnn::MemBin> result = strategy->Optimize(model.m_Blocks); |
| 72 | auto duration = std::chrono::duration<double, std::milli>(Clock::now() - now); |
| 73 | |
| 74 | avgDuration += duration; |
| 75 | size_t memoryUsage = 0; |
| 76 | for (auto bin : result) |
| 77 | { |
| 78 | memoryUsage += bin.m_MemSize; |
| 79 | } |
| 80 | size_t minSize = GetMinPossibleMemorySize(model.m_Blocks); |
| 81 | |
| 82 | float efficiency = static_cast<float>(minSize) / static_cast<float>(memoryUsage); |
| 83 | efficiency*=100; |
| 84 | avgEfficiency += efficiency; |
| 85 | std::cout << "\nModel: " << model.m_Name << "\n"; |
| 86 | |
| 87 | std::cout << "Strategy execution time: " << std::setprecision(4) << duration.count() << " milliseconds\n"; |
| 88 | |
| 89 | std::cout << "Memory usage: " << memoryUsage/1024 << " kb\n"; |
| 90 | |
| 91 | std::cout << "Minimum possible usage: " << minSize/1024 << " kb\n"; |
| 92 | |
| 93 | std::cout << "Memory efficiency: " << std::setprecision(3) << efficiency << "%\n"; |
| 94 | } |
| 95 | |
| 96 | avgDuration/= static_cast<double>(models->size()); |
| 97 | avgEfficiency/= static_cast<float>(models->size()); |
| 98 | |
| 99 | std::cout << "\n===============================================\n"; |
| 100 | std::cout << "Average memory duration: " << std::setprecision(4) << avgDuration.count() << " milliseconds\n"; |
| 101 | std::cout << "Average memory efficiency: " << std::setprecision(3) << avgEfficiency << "%\n"; |
| 102 | } |
| 103 | |
| 104 | struct BenchmarkOptions |
| 105 | { |
| 106 | std::string m_StrategyName; |
| 107 | std::string m_ModelName; |
| 108 | bool m_UseDefaultStrategy = false; |
| 109 | bool m_Validate = false; |
| 110 | }; |
| 111 | |
| 112 | BenchmarkOptions ParseOptions(int argc, char* argv[]) |
| 113 | { |
| 114 | cxxopts::Options options("Memory Benchmark", "Tests memory optimization strategies on different models"); |
| 115 | |
| 116 | options.add_options() |
| 117 | ("s, strategy", "Strategy name, do not specify to use default strategy", cxxopts::value<std::string>()) |
| 118 | ("m, model", "Model name", cxxopts::value<std::string>()) |
| 119 | ("v, validate", "Validate strategy", cxxopts::value<bool>()->default_value("false")->implicit_value("true")) |
| 120 | ("h,help", "Display usage information"); |
| 121 | |
| 122 | auto result = options.parse(argc, argv); |
| 123 | if (result.count("help")) |
| 124 | { |
| 125 | std::cout << options.help() << std::endl; |
| 126 | PrintModels(); |
| 127 | |
| 128 | std::cout << "\nAvailable strategies:\n"; |
| 129 | |
| 130 | for (const auto& s :armnn::GetMemoryOptimizerStrategyNames()) |
| 131 | { |
| 132 | std::cout << s << "\n"; |
| 133 | } |
| 134 | exit(EXIT_SUCCESS); |
| 135 | } |
| 136 | |
| 137 | BenchmarkOptions benchmarkOptions; |
| 138 | |
| 139 | if(result.count("strategy")) |
| 140 | { |
| 141 | benchmarkOptions.m_StrategyName = result["strategy"].as<std::string>(); |
| 142 | } |
| 143 | else |
| 144 | { |
| 145 | std::cout << "No Strategy given, using default strategy"; |
| 146 | |
| 147 | benchmarkOptions.m_UseDefaultStrategy = true; |
| 148 | } |
| 149 | |
| 150 | if(result.count("model")) |
| 151 | { |
| 152 | benchmarkOptions.m_ModelName = result["model"].as<std::string>(); |
| 153 | } |
| 154 | |
| 155 | benchmarkOptions.m_Validate = result["validate"].as<bool>(); |
| 156 | |
| 157 | return benchmarkOptions; |
| 158 | } |
| 159 | |
| 160 | int main(int argc, char* argv[]) |
| 161 | { |
| 162 | BenchmarkOptions benchmarkOptions = ParseOptions(argc, argv); |
| 163 | |
| 164 | std::shared_ptr<armnn::IMemoryOptimizerStrategy> strategy; |
| 165 | |
| 166 | if (benchmarkOptions.m_UseDefaultStrategy) |
| 167 | { |
| 168 | strategy = std::make_shared<armnn::TestStrategy>(); |
| 169 | } |
| 170 | else |
| 171 | { |
| 172 | strategy = armnn::GetMemoryOptimizerStrategy(benchmarkOptions.m_StrategyName); |
| 173 | |
| 174 | if (!strategy) |
| 175 | { |
| 176 | std::cout << "Strategy name not found\n"; |
| 177 | return 0; |
| 178 | } |
| 179 | } |
| 180 | |
| 181 | std::vector<TestBlock> model; |
| 182 | std::vector<TestBlock>* modelsToTest = &testBlocks; |
| 183 | if (benchmarkOptions.m_ModelName.size() != 0) |
| 184 | { |
| 185 | auto it = std::find_if(testBlocks.cbegin(), testBlocks.cend(), [&](const TestBlock testBlock) |
| 186 | { |
| 187 | return testBlock.m_Name == benchmarkOptions.m_ModelName; |
| 188 | }); |
| 189 | |
| 190 | if (it == testBlocks.end()) |
| 191 | { |
| 192 | std::cout << "Model name not found\n"; |
| 193 | return 0; |
| 194 | } |
| 195 | else |
| 196 | { |
| 197 | model.push_back(*it); |
| 198 | modelsToTest = &model; |
| 199 | } |
| 200 | } |
| 201 | |
| 202 | if (benchmarkOptions.m_Validate) |
| 203 | { |
| 204 | armnn::StrategyValidator strategyValidator; |
| 205 | |
| 206 | strategyValidator.SetStrategy(strategy); |
| 207 | |
| 208 | RunBenchmark(&strategyValidator, modelsToTest); |
| 209 | } |
| 210 | else |
| 211 | { |
| 212 | RunBenchmark(strategy.get(), modelsToTest); |
| 213 | } |
| 214 | |
| 215 | } |