alexander | f4e2c47 | 2021-05-14 13:14:21 +0100 | [diff] [blame] | 1 | #!/usr/bin/env python3 |
Isabella Gottardi | ef2b9dd | 2022-02-16 14:24:03 +0000 | [diff] [blame] | 2 | # Copyright (c) 2021-2022 Arm Limited. All rights reserved. |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 3 | # SPDX-License-Identifier: Apache-2.0 |
| 4 | # |
| 5 | # Licensed under the Apache License, Version 2.0 (the "License"); |
| 6 | # you may not use this file except in compliance with the License. |
| 7 | # You may obtain a copy of the License at |
| 8 | # |
| 9 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | # |
| 11 | # Unless required by applicable law or agreed to in writing, software |
| 12 | # distributed under the License is distributed on an "AS IS" BASIS, |
| 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 14 | # See the License for the specific language governing permissions and |
| 15 | # limitations under the License. |
Isabella Gottardi | ef2b9dd | 2022-02-16 14:24:03 +0000 | [diff] [blame] | 16 | import errno |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 17 | import fnmatch |
Isabella Gottardi | ef2b9dd | 2022-02-16 14:24:03 +0000 | [diff] [blame] | 18 | import json |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 19 | import logging |
Isabella Gottardi | ef2b9dd | 2022-02-16 14:24:03 +0000 | [diff] [blame] | 20 | import os |
| 21 | import re |
| 22 | import shutil |
| 23 | import subprocess |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 24 | import sys |
Isabella Gottardi | ef2b9dd | 2022-02-16 14:24:03 +0000 | [diff] [blame] | 25 | import urllib.request |
| 26 | from argparse import ArgumentParser |
| 27 | from argparse import ArgumentTypeError |
Kshitij Sisodia | 3be2623 | 2021-10-29 12:29:06 +0100 | [diff] [blame] | 28 | from collections import namedtuple |
Isabella Gottardi | ef2b9dd | 2022-02-16 14:24:03 +0000 | [diff] [blame] | 29 | from urllib.error import URLError |
Richard Burton | 1706962 | 2022-03-17 10:54:26 +0000 | [diff] [blame] | 30 | from pathlib import Path |
Isabella Gottardi | 6c2ea45 | 2022-03-11 13:25:08 +0000 | [diff] [blame] | 31 | |
Kshitij Sisodia | 6a2ac46 | 2022-03-01 17:36:06 +0000 | [diff] [blame] | 32 | from scripts.py.check_update_resources_downloaded import get_md5sum_for_file |
Kshitij Sisodia | 3be2623 | 2021-10-29 12:29:06 +0100 | [diff] [blame] | 33 | |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 34 | |
Isabella Gottardi | ef2b9dd | 2022-02-16 14:24:03 +0000 | [diff] [blame] | 35 | json_uc_res = [ |
| 36 | { |
| 37 | "use_case_name": "ad", |
| 38 | "url_prefix": [ |
| 39 | "https://github.com/ARM-software/ML-zoo/raw/7c32b097f7d94aae2cd0b98a8ed5a3ba81e66b18/models/anomaly_detection/micronet_medium/tflite_int8/" |
| 40 | ], |
| 41 | "resources": [ |
| 42 | { |
| 43 | "name": "ad_medium_int8.tflite", |
| 44 | "url": "{url_prefix:0}ad_medium_int8.tflite", |
| 45 | }, |
| 46 | {"name": "ifm0.npy", "url": "{url_prefix:0}testing_input/input/0.npy"}, |
| 47 | {"name": "ofm0.npy", "url": "{url_prefix:0}testing_output/Identity/0.npy"}, |
| 48 | ], |
| 49 | }, |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 50 | { |
| 51 | "use_case_name": "asr", |
Isabella Gottardi | ef2b9dd | 2022-02-16 14:24:03 +0000 | [diff] [blame] | 52 | "url_prefix": [ |
| 53 | "https://github.com/ARM-software/ML-zoo/raw/1a92aa08c0de49a7304e0a7f3f59df6f4fd33ac8/models/speech_recognition/wav2letter/tflite_pruned_int8/" |
| 54 | ], |
| 55 | "resources": [ |
| 56 | { |
| 57 | "name": "wav2letter_pruned_int8.tflite", |
| 58 | "url": "{url_prefix:0}wav2letter_pruned_int8.tflite", |
| 59 | }, |
| 60 | { |
| 61 | "name": "ifm0.npy", |
| 62 | "url": "{url_prefix:0}testing_input/input_2_int8/0.npy", |
| 63 | }, |
| 64 | { |
| 65 | "name": "ofm0.npy", |
| 66 | "url": "{url_prefix:0}testing_output/Identity_int8/0.npy", |
| 67 | }, |
| 68 | ], |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 69 | }, |
| 70 | { |
| 71 | "use_case_name": "img_class", |
Isabella Gottardi | ef2b9dd | 2022-02-16 14:24:03 +0000 | [diff] [blame] | 72 | "url_prefix": [ |
| 73 | "https://github.com/ARM-software/ML-zoo/raw/e0aa361b03c738047b9147d1a50e3f2dcb13dbcb/models/image_classification/mobilenet_v2_1.0_224/tflite_int8/" |
| 74 | ], |
| 75 | "resources": [ |
| 76 | { |
| 77 | "name": "mobilenet_v2_1.0_224_INT8.tflite", |
| 78 | "url": "{url_prefix:0}mobilenet_v2_1.0_224_INT8.tflite", |
| 79 | }, |
| 80 | { |
| 81 | "name": "ifm0.npy", |
| 82 | "url": "{url_prefix:0}testing_input/tfl.quantize/0.npy", |
| 83 | }, |
| 84 | { |
| 85 | "name": "ofm0.npy", |
| 86 | "url": "{url_prefix:0}testing_output/MobilenetV2/Predictions/Reshape_11/0.npy", |
| 87 | }, |
| 88 | ], |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 89 | }, |
| 90 | { |
Michael Levit | 06fcf75 | 2022-01-12 11:53:46 +0200 | [diff] [blame] | 91 | "use_case_name": "object_detection", |
Isabella Gottardi | ef2b9dd | 2022-02-16 14:24:03 +0000 | [diff] [blame] | 92 | "url_prefix": [ |
| 93 | "https://github.com/emza-vs/ModelZoo/blob/v1.0/object_detection/" |
| 94 | ], |
| 95 | "resources": [ |
| 96 | { |
| 97 | "name": "yolo-fastest_192_face_v4.tflite", |
| 98 | "url": "{url_prefix:0}yolo-fastest_192_face_v4.tflite?raw=true", |
| 99 | } |
| 100 | ], |
Michael Levit | 06fcf75 | 2022-01-12 11:53:46 +0200 | [diff] [blame] | 101 | }, |
| 102 | { |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 103 | "use_case_name": "kws", |
Isabella Gottardi | ef2b9dd | 2022-02-16 14:24:03 +0000 | [diff] [blame] | 104 | "url_prefix": [ |
| 105 | "https://github.com/ARM-software/ML-zoo/raw/9f506fe52b39df545f0e6c5ff9223f671bc5ae00/models/keyword_spotting/micronet_medium/tflite_int8/" |
| 106 | ], |
| 107 | "resources": [ |
| 108 | {"name": "ifm0.npy", "url": "{url_prefix:0}testing_input/input/0.npy"}, |
| 109 | {"name": "ofm0.npy", "url": "{url_prefix:0}testing_output/Identity/0.npy"}, |
| 110 | { |
| 111 | "name": "kws_micronet_m.tflite", |
| 112 | "url": "{url_prefix:0}kws_micronet_m.tflite", |
| 113 | }, |
| 114 | ], |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 115 | }, |
Isabella Gottardi | ef2b9dd | 2022-02-16 14:24:03 +0000 | [diff] [blame] | 116 | { |
Éanna Ó Catháin | 8f95887 | 2021-09-15 09:32:30 +0100 | [diff] [blame] | 117 | "use_case_name": "vww", |
Isabella Gottardi | ef2b9dd | 2022-02-16 14:24:03 +0000 | [diff] [blame] | 118 | "url_prefix": [ |
| 119 | "https://github.com/ARM-software/ML-zoo/raw/7dd3b16bb84007daf88be8648983c07f3eb21140/models/visual_wake_words/micronet_vww4/tflite_int8/" |
| 120 | ], |
| 121 | "resources": [ |
| 122 | { |
| 123 | "name": "vww4_128_128_INT8.tflite", |
| 124 | "url": "{url_prefix:0}vww4_128_128_INT8.tflite", |
| 125 | }, |
| 126 | {"name": "ifm0.npy", "url": "{url_prefix:0}testing_input/input/0.npy"}, |
| 127 | {"name": "ofm0.npy", "url": "{url_prefix:0}testing_output/Identity/0.npy"}, |
| 128 | ], |
Éanna Ó Catháin | 8f95887 | 2021-09-15 09:32:30 +0100 | [diff] [blame] | 129 | }, |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 130 | { |
| 131 | "use_case_name": "kws_asr", |
Isabella Gottardi | ef2b9dd | 2022-02-16 14:24:03 +0000 | [diff] [blame] | 132 | "url_prefix": [ |
| 133 | "https://github.com/ARM-software/ML-zoo/raw/1a92aa08c0de49a7304e0a7f3f59df6f4fd33ac8/models/speech_recognition/wav2letter/tflite_pruned_int8/", |
| 134 | "https://github.com/ARM-software/ML-zoo/raw/9f506fe52b39df545f0e6c5ff9223f671bc5ae00/models/keyword_spotting/micronet_medium/tflite_int8/", |
| 135 | ], |
| 136 | "resources": [ |
| 137 | { |
| 138 | "name": "wav2letter_pruned_int8.tflite", |
| 139 | "url": "{url_prefix:0}wav2letter_pruned_int8.tflite", |
| 140 | }, |
| 141 | { |
| 142 | "sub_folder": "asr", |
| 143 | "name": "ifm0.npy", |
| 144 | "url": "{url_prefix:0}testing_input/input_2_int8/0.npy", |
| 145 | }, |
| 146 | { |
| 147 | "sub_folder": "asr", |
| 148 | "name": "ofm0.npy", |
| 149 | "url": "{url_prefix:0}testing_output/Identity_int8/0.npy", |
| 150 | }, |
| 151 | { |
| 152 | "sub_folder": "kws", |
| 153 | "name": "ifm0.npy", |
| 154 | "url": "{url_prefix:1}testing_input/input/0.npy", |
| 155 | }, |
| 156 | { |
| 157 | "sub_folder": "kws", |
| 158 | "name": "ofm0.npy", |
| 159 | "url": "{url_prefix:1}testing_output/Identity/0.npy", |
| 160 | }, |
| 161 | { |
| 162 | "name": "kws_micronet_m.tflite", |
| 163 | "url": "{url_prefix:1}kws_micronet_m.tflite", |
| 164 | }, |
| 165 | ], |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 166 | }, |
| 167 | { |
Richard Burton | 0055346 | 2021-11-10 16:27:14 +0000 | [diff] [blame] | 168 | "use_case_name": "noise_reduction", |
Isabella Gottardi | ef2b9dd | 2022-02-16 14:24:03 +0000 | [diff] [blame] | 169 | "url_prefix": [ |
| 170 | "https://github.com/ARM-software/ML-zoo/raw/a061600058097a2785d6f1f7785e5a2d2a142955/models/noise_suppression/RNNoise/tflite_int8/" |
| 171 | ], |
| 172 | "resources": [ |
| 173 | {"name": "rnnoise_INT8.tflite", "url": "{url_prefix:0}rnnoise_INT8.tflite"}, |
| 174 | { |
| 175 | "name": "ifm0.npy", |
| 176 | "url": "{url_prefix:0}testing_input/main_input_int8/0.npy", |
| 177 | }, |
| 178 | { |
| 179 | "name": "ifm1.npy", |
| 180 | "url": "{url_prefix:0}testing_input/vad_gru_prev_state_int8/0.npy", |
| 181 | }, |
| 182 | { |
| 183 | "name": "ifm2.npy", |
| 184 | "url": "{url_prefix:0}testing_input/noise_gru_prev_state_int8/0.npy", |
| 185 | }, |
| 186 | { |
| 187 | "name": "ifm3.npy", |
| 188 | "url": "{url_prefix:0}testing_input/denoise_gru_prev_state_int8/0.npy", |
| 189 | }, |
| 190 | { |
| 191 | "name": "ofm0.npy", |
| 192 | "url": "{url_prefix:0}testing_output/Identity_int8/0.npy", |
| 193 | }, |
| 194 | { |
| 195 | "name": "ofm1.npy", |
| 196 | "url": "{url_prefix:0}testing_output/Identity_1_int8/0.npy", |
| 197 | }, |
| 198 | { |
| 199 | "name": "ofm2.npy", |
| 200 | "url": "{url_prefix:0}testing_output/Identity_2_int8/0.npy", |
| 201 | }, |
| 202 | { |
| 203 | "name": "ofm3.npy", |
| 204 | "url": "{url_prefix:0}testing_output/Identity_3_int8/0.npy", |
| 205 | }, |
| 206 | { |
| 207 | "name": "ofm4.npy", |
| 208 | "url": "{url_prefix:0}testing_output/Identity_4_int8/0.npy", |
| 209 | }, |
| 210 | ], |
Richard Burton | 0055346 | 2021-11-10 16:27:14 +0000 | [diff] [blame] | 211 | }, |
| 212 | { |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 213 | "use_case_name": "inference_runner", |
Isabella Gottardi | ef2b9dd | 2022-02-16 14:24:03 +0000 | [diff] [blame] | 214 | "url_prefix": [ |
| 215 | "https://github.com/ARM-software/ML-zoo/raw/68b5fbc77ed28e67b2efc915997ea4477c1d9d5b/models/keyword_spotting/dnn_small/tflite_int8/" |
| 216 | ], |
| 217 | "resources": [ |
| 218 | { |
| 219 | "name": "dnn_s_quantized.tflite", |
| 220 | "url": "{url_prefix:0}dnn_s_quantized.tflite", |
| 221 | } |
| 222 | ], |
| 223 | }, |
| 224 | ] |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 225 | |
Kshitij Sisodia | 3be2623 | 2021-10-29 12:29:06 +0100 | [diff] [blame] | 226 | # Valid NPU configurations: |
| 227 | valid_npu_config_names = [ |
Isabella Gottardi | ef2b9dd | 2022-02-16 14:24:03 +0000 | [diff] [blame] | 228 | "ethos-u55-32", |
| 229 | "ethos-u55-64", |
| 230 | "ethos-u55-128", |
| 231 | "ethos-u55-256", |
| 232 | "ethos-u65-256", |
| 233 | "ethos-u65-512", |
| 234 | ] |
Kshitij Sisodia | 3be2623 | 2021-10-29 12:29:06 +0100 | [diff] [blame] | 235 | |
| 236 | # Default NPU configurations (these are always run when the models are optimised) |
| 237 | default_npu_config_names = [valid_npu_config_names[2], valid_npu_config_names[4]] |
| 238 | |
| 239 | # NPU config named tuple |
Isabella Gottardi | ef2b9dd | 2022-02-16 14:24:03 +0000 | [diff] [blame] | 240 | NPUConfig = namedtuple( |
| 241 | "NPUConfig", |
| 242 | [ |
| 243 | "config_name", |
| 244 | "memory_mode", |
| 245 | "system_config", |
| 246 | "ethos_u_npu_id", |
| 247 | "ethos_u_config_id", |
| 248 | "arena_cache_size", |
| 249 | ], |
| 250 | ) |
Kshitij Sisodia | 3be2623 | 2021-10-29 12:29:06 +0100 | [diff] [blame] | 251 | |
Kshitij Sisodia | 661959c | 2021-11-24 10:39:52 +0000 | [diff] [blame] | 252 | # The internal SRAM size for Corstone-300 implementation on MPS3 specified by AN552 |
Kshitij Sisodia | 8c61c0a | 2022-05-17 11:16:22 +0100 | [diff] [blame] | 253 | # The internal SRAM size for Corstone-310 implementation on MPS3 specified by AN555 |
| 254 | # is 4MB, but we are content with the 2MB specified below. |
Isabella Gottardi | ef2b9dd | 2022-02-16 14:24:03 +0000 | [diff] [blame] | 255 | mps3_max_sram_sz = 2 * 1024 * 1024 # 2 MiB (2 banks of 1 MiB each) |
Liam Barry | b52b585 | 2021-11-15 11:41:40 +0000 | [diff] [blame] | 256 | |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 257 | |
Isabella Gottardi | ef2b9dd | 2022-02-16 14:24:03 +0000 | [diff] [blame] | 258 | def call_command(command: str, verbose: bool = True) -> str: |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 259 | """ |
| 260 | Helpers function that call subprocess and return the output. |
| 261 | |
| 262 | Parameters: |
| 263 | ---------- |
| 264 | command (string): Specifies the command to run. |
| 265 | """ |
Isabella Gottardi | ef2b9dd | 2022-02-16 14:24:03 +0000 | [diff] [blame] | 266 | if verbose: |
| 267 | logging.info(command) |
| 268 | proc = subprocess.run( |
| 269 | command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True |
| 270 | ) |
alexander | 50a0650 | 2021-05-12 19:06:02 +0100 | [diff] [blame] | 271 | log = proc.stdout.decode("utf-8") |
Isabella Gottardi | ef2b9dd | 2022-02-16 14:24:03 +0000 | [diff] [blame] | 272 | if proc.returncode == 0 and verbose: |
alexander | 50a0650 | 2021-05-12 19:06:02 +0100 | [diff] [blame] | 273 | logging.info(log) |
| 274 | else: |
| 275 | logging.error(log) |
| 276 | proc.check_returncode() |
| 277 | return log |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 278 | |
| 279 | |
Isabella Gottardi | ef2b9dd | 2022-02-16 14:24:03 +0000 | [diff] [blame] | 280 | def get_default_npu_config_from_name( |
| 281 | config_name: str, arena_cache_size: int = 0 |
| 282 | ) -> NPUConfig: |
Kshitij Sisodia | 3be2623 | 2021-10-29 12:29:06 +0100 | [diff] [blame] | 283 | """ |
Richard Burton | 1706962 | 2022-03-17 10:54:26 +0000 | [diff] [blame] | 284 | Gets the file suffix for the TFLite file from the |
Kshitij Sisodia | 3be2623 | 2021-10-29 12:29:06 +0100 | [diff] [blame] | 285 | `accelerator_config` string. |
| 286 | |
| 287 | Parameters: |
| 288 | ---------- |
Isabella Gottardi | 3acaaee | 2021-11-30 12:33:27 +0000 | [diff] [blame] | 289 | config_name (str): Ethos-U NPU configuration from valid_npu_config_names |
| 290 | |
| 291 | arena_cache_size (int): Specifies arena cache size in bytes. If a value |
| 292 | greater than 0 is provided, this will be taken |
| 293 | as the cache size. If 0, the default values, as per |
| 294 | the NPU config requirements, are used. |
Kshitij Sisodia | 3be2623 | 2021-10-29 12:29:06 +0100 | [diff] [blame] | 295 | |
| 296 | Returns: |
| 297 | ------- |
| 298 | NPUConfig: An NPU config named tuple populated with defaults for the given |
| 299 | config name |
| 300 | """ |
| 301 | if config_name not in valid_npu_config_names: |
Isabella Gottardi | ef2b9dd | 2022-02-16 14:24:03 +0000 | [diff] [blame] | 302 | raise ValueError( |
| 303 | f""" |
Kshitij Sisodia | 3be2623 | 2021-10-29 12:29:06 +0100 | [diff] [blame] | 304 | Invalid Ethos-U NPU configuration. |
| 305 | Select one from {valid_npu_config_names}. |
Isabella Gottardi | ef2b9dd | 2022-02-16 14:24:03 +0000 | [diff] [blame] | 306 | """ |
| 307 | ) |
Kshitij Sisodia | 3be2623 | 2021-10-29 12:29:06 +0100 | [diff] [blame] | 308 | |
| 309 | strings_ids = ["ethos-u55-", "ethos-u65-"] |
| 310 | processor_ids = ["U55", "U65"] |
| 311 | prefix_ids = ["H", "Y"] |
| 312 | memory_modes = ["Shared_Sram", "Dedicated_Sram"] |
| 313 | system_configs = ["Ethos_U55_High_End_Embedded", "Ethos_U65_High_End"] |
Isabella Gottardi | 3acaaee | 2021-11-30 12:33:27 +0000 | [diff] [blame] | 314 | memory_modes_arena = { |
Richard Burton | 1706962 | 2022-03-17 10:54:26 +0000 | [diff] [blame] | 315 | # For shared SRAM memory mode, we use the MPS3 SRAM size by default. |
Isabella Gottardi | ef2b9dd | 2022-02-16 14:24:03 +0000 | [diff] [blame] | 316 | "Shared_Sram": mps3_max_sram_sz if arena_cache_size <= 0 else arena_cache_size, |
Richard Burton | 1706962 | 2022-03-17 10:54:26 +0000 | [diff] [blame] | 317 | # For dedicated SRAM memory mode, we do not override the arena size. This is expected to |
| 318 | # be defined in the Vela configuration file instead. |
Isabella Gottardi | ef2b9dd | 2022-02-16 14:24:03 +0000 | [diff] [blame] | 319 | "Dedicated_Sram": None if arena_cache_size <= 0 else arena_cache_size, |
Isabella Gottardi | 3acaaee | 2021-11-30 12:33:27 +0000 | [diff] [blame] | 320 | } |
Kshitij Sisodia | 3be2623 | 2021-10-29 12:29:06 +0100 | [diff] [blame] | 321 | |
| 322 | for i in range(len(strings_ids)): |
| 323 | if config_name.startswith(strings_ids[i]): |
| 324 | npu_config_id = config_name.replace(strings_ids[i], prefix_ids[i]) |
Isabella Gottardi | ef2b9dd | 2022-02-16 14:24:03 +0000 | [diff] [blame] | 325 | return NPUConfig( |
| 326 | config_name=config_name, |
| 327 | memory_mode=memory_modes[i], |
| 328 | system_config=system_configs[i], |
| 329 | ethos_u_npu_id=processor_ids[i], |
| 330 | ethos_u_config_id=npu_config_id, |
| 331 | arena_cache_size=memory_modes_arena[memory_modes[i]], |
| 332 | ) |
Kshitij Sisodia | 3be2623 | 2021-10-29 12:29:06 +0100 | [diff] [blame] | 333 | |
| 334 | return None |
| 335 | |
| 336 | |
Isabella Gottardi | ef2b9dd | 2022-02-16 14:24:03 +0000 | [diff] [blame] | 337 | def remove_tree_dir(dir_path): |
| 338 | try: |
Richard Burton | 1706962 | 2022-03-17 10:54:26 +0000 | [diff] [blame] | 339 | # Remove the full directory. |
Isabella Gottardi | ef2b9dd | 2022-02-16 14:24:03 +0000 | [diff] [blame] | 340 | shutil.rmtree(dir_path) |
Richard Burton | 1706962 | 2022-03-17 10:54:26 +0000 | [diff] [blame] | 341 | # Re-create an empty one. |
Isabella Gottardi | ef2b9dd | 2022-02-16 14:24:03 +0000 | [diff] [blame] | 342 | os.mkdir(dir_path) |
| 343 | except Exception as e: |
| 344 | logging.error(f"Failed to delete {dir_path}.") |
| 345 | |
| 346 | |
| 347 | def set_up_resources( |
| 348 | run_vela_on_models: bool = False, |
Richard Burton | 1706962 | 2022-03-17 10:54:26 +0000 | [diff] [blame] | 349 | additional_npu_config_names: tuple = (), |
Isabella Gottardi | ef2b9dd | 2022-02-16 14:24:03 +0000 | [diff] [blame] | 350 | arena_cache_size: int = 0, |
| 351 | check_clean_folder: bool = False, |
Kshitij Sisodia | 9c6f9f8 | 2022-05-20 14:30:02 +0100 | [diff] [blame] | 352 | additional_requirements_file: str = "") -> (Path, Path): |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 353 | """ |
| 354 | Helpers function that retrieve the output from a command. |
| 355 | |
| 356 | Parameters: |
| 357 | ---------- |
Isabella Gottardi | 3acaaee | 2021-11-30 12:33:27 +0000 | [diff] [blame] | 358 | run_vela_on_models (bool): Specifies if run vela on downloaded models. |
| 359 | additional_npu_config_names(list): list of strings of Ethos-U NPU configs. |
| 360 | arena_cache_size (int): Specifies arena cache size in bytes. If a value |
| 361 | greater than 0 is provided, this will be taken |
| 362 | as the cache size. If 0, the default values, as per |
| 363 | the NPU config requirements, are used. |
Kshitij Sisodia | 6a2ac46 | 2022-03-01 17:36:06 +0000 | [diff] [blame] | 364 | check_clean_folder (bool): Indicates whether the resources folder needs to |
| 365 | be checked for updates and cleaned. |
Kshitij Sisodia | c22e80e | 2022-03-14 09:26:48 +0000 | [diff] [blame] | 366 | additional_requirements_file (str): Path to a requirements.txt file if |
| 367 | additional packages need to be |
| 368 | installed. |
Kshitij Sisodia | 9c6f9f8 | 2022-05-20 14:30:02 +0100 | [diff] [blame] | 369 | |
| 370 | Returns |
| 371 | ------- |
| 372 | |
| 373 | Tuple of pair of Paths: (download_directory_path, virtual_env_path) |
| 374 | |
| 375 | download_directory_path: Root of the directory where the resources have been downloaded to. |
| 376 | virtual_env_path: Path to the root of virtual environment. |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 377 | """ |
Richard Burton | 1706962 | 2022-03-17 10:54:26 +0000 | [diff] [blame] | 378 | # Paths. |
| 379 | current_file_dir = Path(__file__).parent.resolve() |
| 380 | download_dir = current_file_dir / "resources_downloaded" |
| 381 | metadata_file_path = download_dir / "resources_downloaded_metadata.json" |
Isabella Gottardi | ef2b9dd | 2022-02-16 14:24:03 +0000 | [diff] [blame] | 382 | |
| 383 | metadata_dict = dict() |
Nina Drozd | 2822430 | 2022-09-06 17:00:34 +0100 | [diff] [blame] | 384 | vela_version = "3.5.0" |
Richard Burton | c372203 | 2022-05-31 11:27:26 +0100 | [diff] [blame] | 385 | py3_major_version_minimum = 3 # Python >= 3.7 is required |
| 386 | py3_minor_version_minimum = 7 |
Isabella Gottardi | 6c2ea45 | 2022-03-11 13:25:08 +0000 | [diff] [blame] | 387 | |
| 388 | # Is Python minimum requirement matched? |
| 389 | py3_version = sys.version_info |
| 390 | if ( |
| 391 | py3_version.major < py3_major_version_minimum |
| 392 | or py3_version.minor < py3_minor_version_minimum |
| 393 | ): |
| 394 | raise Exception( |
Richard Burton | c372203 | 2022-05-31 11:27:26 +0100 | [diff] [blame] | 395 | "ERROR: Python3.7+ is required, please see the documentation on how to update it." |
Isabella Gottardi | 6c2ea45 | 2022-03-11 13:25:08 +0000 | [diff] [blame] | 396 | ) |
Isabella Gottardi | ef2b9dd | 2022-02-16 14:24:03 +0000 | [diff] [blame] | 397 | |
Kshitij Sisodia | 6a2ac46 | 2022-03-01 17:36:06 +0000 | [diff] [blame] | 398 | setup_script_hash_verified = False |
Richard Burton | 1706962 | 2022-03-17 10:54:26 +0000 | [diff] [blame] | 399 | setup_script_hash = get_md5sum_for_file(Path(__file__).resolve()) |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 400 | |
| 401 | try: |
| 402 | # 1.1 Does the download dir exist? |
Richard Burton | 1706962 | 2022-03-17 10:54:26 +0000 | [diff] [blame] | 403 | download_dir.mkdir() |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 404 | except OSError as e: |
| 405 | if e.errno == errno.EEXIST: |
| 406 | logging.info("'resources_downloaded' directory exists.") |
Isabella Gottardi | ef2b9dd | 2022-02-16 14:24:03 +0000 | [diff] [blame] | 407 | # Check and clean? |
Richard Burton | 1706962 | 2022-03-17 10:54:26 +0000 | [diff] [blame] | 408 | if check_clean_folder and metadata_file_path.is_file(): |
| 409 | with open(metadata_file_path) as metadata_file: |
Isabella Gottardi | ef2b9dd | 2022-02-16 14:24:03 +0000 | [diff] [blame] | 410 | metadata_dict = json.load(metadata_file) |
| 411 | vela_in_metadata = metadata_dict["ethosu_vela_version"] |
| 412 | if vela_in_metadata != vela_version: |
| 413 | # Check if all the resources needs to be removed and regenerated. |
| 414 | # This can happen when the Vela version has changed. |
| 415 | logging.info( |
| 416 | f"Vela version in metadata is {vela_in_metadata}, current {vela_version}. Removing the resources and re-download them." |
| 417 | ) |
| 418 | remove_tree_dir(download_dir) |
| 419 | metadata_dict = dict() |
| 420 | else: |
Kshitij Sisodia | 6a2ac46 | 2022-03-01 17:36:06 +0000 | [diff] [blame] | 421 | # Check if the set_up_default_resorces.py has changed from last setup |
| 422 | setup_script_hash_verified = ( |
Isabella Gottardi | 6c2ea45 | 2022-03-11 13:25:08 +0000 | [diff] [blame] | 423 | metadata_dict.get("set_up_script_md5sum") |
| 424 | == setup_script_hash |
| 425 | ) |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 426 | else: |
| 427 | raise |
| 428 | |
| 429 | # 1.2 Does the virtual environment exist? |
Kshitij Sisodia | 9c6f9f8 | 2022-05-20 14:30:02 +0100 | [diff] [blame] | 430 | env_dirname = "env" |
| 431 | env_path = download_dir / env_dirname |
| 432 | env_python = str(env_path / "bin" / "python3") |
| 433 | env_activate = str(env_path / "bin" / "activate") |
Richard Burton | 1706962 | 2022-03-17 10:54:26 +0000 | [diff] [blame] | 434 | |
Kshitij Sisodia | 9c6f9f8 | 2022-05-20 14:30:02 +0100 | [diff] [blame] | 435 | if not env_path.is_dir(): |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 436 | os.chdir(download_dir) |
Richard Burton | 1706962 | 2022-03-17 10:54:26 +0000 | [diff] [blame] | 437 | # Create the virtual environment. |
Kshitij Sisodia | 9c6f9f8 | 2022-05-20 14:30:02 +0100 | [diff] [blame] | 438 | command = f"python3 -m venv {env_dirname}" |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 439 | call_command(command) |
| 440 | commands = ["pip install --upgrade pip", "pip install --upgrade setuptools"] |
| 441 | for c in commands: |
| 442 | command = f"{env_python} -m {c}" |
| 443 | call_command(command) |
| 444 | os.chdir(current_file_dir) |
Isabella Gottardi | ef2b9dd | 2022-02-16 14:24:03 +0000 | [diff] [blame] | 445 | |
Kshitij Sisodia | c22e80e | 2022-03-14 09:26:48 +0000 | [diff] [blame] | 446 | # 1.3 Make sure to have all the requirements |
Isabella Gottardi | ef2b9dd | 2022-02-16 14:24:03 +0000 | [diff] [blame] | 447 | requirements = [f"ethos-u-vela=={vela_version}"] |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 448 | command = f"{env_python} -m pip freeze" |
| 449 | packages = call_command(command) |
| 450 | for req in requirements: |
| 451 | if req not in packages: |
| 452 | command = f"{env_python} -m pip install {req}" |
| 453 | call_command(command) |
| 454 | |
Kshitij Sisodia | c22e80e | 2022-03-14 09:26:48 +0000 | [diff] [blame] | 455 | # 1.4 Install additional requirements, if a valid file has been provided |
| 456 | if additional_requirements_file and os.path.isfile(additional_requirements_file): |
| 457 | command = f"{env_python} -m pip install -r {additional_requirements_file}" |
| 458 | call_command(command) |
| 459 | |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 460 | # 2. Download models |
Isabella Gottardi | ef2b9dd | 2022-02-16 14:24:03 +0000 | [diff] [blame] | 461 | logging.info("Downloading resources.") |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 462 | for uc in json_uc_res: |
Isabella Gottardi | ef2b9dd | 2022-02-16 14:24:03 +0000 | [diff] [blame] | 463 | use_case_name = uc["use_case_name"] |
| 464 | res_url_prefix = uc["url_prefix"] |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 465 | try: |
| 466 | # Does the usecase_name download dir exist? |
Richard Burton | 1706962 | 2022-03-17 10:54:26 +0000 | [diff] [blame] | 467 | (download_dir / use_case_name).mkdir() |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 468 | except OSError as e: |
Isabella Gottardi | ef2b9dd | 2022-02-16 14:24:03 +0000 | [diff] [blame] | 469 | if e.errno == errno.EEXIST: |
Richard Burton | 1706962 | 2022-03-17 10:54:26 +0000 | [diff] [blame] | 470 | # The usecase_name download dir exist. |
Kshitij Sisodia | 6a2ac46 | 2022-03-01 17:36:06 +0000 | [diff] [blame] | 471 | if check_clean_folder and not setup_script_hash_verified: |
Isabella Gottardi | ef2b9dd | 2022-02-16 14:24:03 +0000 | [diff] [blame] | 472 | for idx, metadata_uc_url_prefix in enumerate( |
| 473 | [ |
| 474 | f |
| 475 | for f in metadata_dict["resources_info"] |
| 476 | if f["use_case_name"] == use_case_name |
| 477 | ][0]["url_prefix"] |
| 478 | ): |
| 479 | if metadata_uc_url_prefix != res_url_prefix[idx]: |
| 480 | logging.info(f"Removing {use_case_name} resources.") |
Richard Burton | 1706962 | 2022-03-17 10:54:26 +0000 | [diff] [blame] | 481 | remove_tree_dir(download_dir / use_case_name) |
Isabella Gottardi | ef2b9dd | 2022-02-16 14:24:03 +0000 | [diff] [blame] | 482 | break |
| 483 | elif e.errno != errno.EEXIST: |
| 484 | logging.error(f"Error creating {use_case_name} directory.") |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 485 | raise |
| 486 | |
Isabella Gottardi | ef2b9dd | 2022-02-16 14:24:03 +0000 | [diff] [blame] | 487 | reg_expr_str = r"{url_prefix:(.*\d)}" |
| 488 | reg_expr_pattern = re.compile(reg_expr_str) |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 489 | for res in uc["resources"]: |
| 490 | res_name = res["name"] |
Isabella Gottardi | ef2b9dd | 2022-02-16 14:24:03 +0000 | [diff] [blame] | 491 | url_prefix_idx = int(reg_expr_pattern.search(res["url"]).group(1)) |
| 492 | res_url = res_url_prefix[url_prefix_idx] + re.sub( |
| 493 | reg_expr_str, "", res["url"] |
| 494 | ) |
| 495 | |
| 496 | sub_folder = "" |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 497 | if "sub_folder" in res: |
| 498 | try: |
| 499 | # Does the usecase_name/sub_folder download dir exist? |
Richard Burton | 1706962 | 2022-03-17 10:54:26 +0000 | [diff] [blame] | 500 | (download_dir / use_case_name / res["sub_folder"]).mkdir() |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 501 | except OSError as e: |
| 502 | if e.errno != errno.EEXIST: |
Isabella Gottardi | ef2b9dd | 2022-02-16 14:24:03 +0000 | [diff] [blame] | 503 | logging.error( |
| 504 | f"Error creating {use_case_name} / {res['sub_folder']} directory." |
| 505 | ) |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 506 | raise |
Isabella Gottardi | ef2b9dd | 2022-02-16 14:24:03 +0000 | [diff] [blame] | 507 | sub_folder = res["sub_folder"] |
| 508 | |
Richard Burton | 1706962 | 2022-03-17 10:54:26 +0000 | [diff] [blame] | 509 | res_dst = download_dir / use_case_name / sub_folder / res_name |
alexander | 3ef1fd4 | 2021-05-24 18:56:32 +0100 | [diff] [blame] | 510 | |
Richard Burton | 1706962 | 2022-03-17 10:54:26 +0000 | [diff] [blame] | 511 | if res_dst.is_file(): |
alexander | 3ef1fd4 | 2021-05-24 18:56:32 +0100 | [diff] [blame] | 512 | logging.info(f"File {res_dst} exists, skipping download.") |
| 513 | else: |
| 514 | try: |
| 515 | g = urllib.request.urlopen(res_url) |
Isabella Gottardi | ef2b9dd | 2022-02-16 14:24:03 +0000 | [diff] [blame] | 516 | with open(res_dst, "b+w") as f: |
alexander | 3ef1fd4 | 2021-05-24 18:56:32 +0100 | [diff] [blame] | 517 | f.write(g.read()) |
| 518 | logging.info(f"- Downloaded {res_url} to {res_dst}.") |
| 519 | except URLError: |
| 520 | logging.error(f"URLError while downloading {res_url}.") |
| 521 | raise |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 522 | |
| 523 | # 3. Run vela on models in resources_downloaded |
| 524 | # New models will have same name with '_vela' appended. |
| 525 | # For example: |
Kshitij Sisodia | 76a1580 | 2021-12-24 11:05:11 +0000 | [diff] [blame] | 526 | # original model: kws_micronet_m.tflite |
| 527 | # after vela model: kws_micronet_m_vela_H128.tflite |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 528 | # |
| 529 | # Note: To avoid to run vela twice on the same model, it's supposed that |
| 530 | # downloaded model names don't contain the 'vela' word. |
| 531 | if run_vela_on_models is True: |
Richard Burton | 1706962 | 2022-03-17 10:54:26 +0000 | [diff] [blame] | 532 | config_file = current_file_dir / "scripts" / "vela" / "default_vela.ini" |
Isabella Gottardi | ef2b9dd | 2022-02-16 14:24:03 +0000 | [diff] [blame] | 533 | models = [ |
Richard Burton | 1706962 | 2022-03-17 10:54:26 +0000 | [diff] [blame] | 534 | Path(dirpath) / f |
Isabella Gottardi | ef2b9dd | 2022-02-16 14:24:03 +0000 | [diff] [blame] | 535 | for dirpath, dirnames, files in os.walk(download_dir) |
| 536 | for f in fnmatch.filter(files, "*.tflite") |
| 537 | if "vela" not in f |
| 538 | ] |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 539 | |
Kshitij Sisodia | 3be2623 | 2021-10-29 12:29:06 +0100 | [diff] [blame] | 540 | # Consolidate all config names while discarding duplicates: |
| 541 | config_names = list(set(default_npu_config_names + additional_npu_config_names)) |
| 542 | |
| 543 | # Get npu config tuple for each config name in a list: |
Isabella Gottardi | ef2b9dd | 2022-02-16 14:24:03 +0000 | [diff] [blame] | 544 | npu_configs = [ |
| 545 | get_default_npu_config_from_name(name, arena_cache_size) |
| 546 | for name in config_names |
| 547 | ] |
Kshitij Sisodia | 3be2623 | 2021-10-29 12:29:06 +0100 | [diff] [blame] | 548 | |
Isabella Gottardi | ef2b9dd | 2022-02-16 14:24:03 +0000 | [diff] [blame] | 549 | logging.info(f"All models will be optimised for these configs:") |
Kshitij Sisodia | 3be2623 | 2021-10-29 12:29:06 +0100 | [diff] [blame] | 550 | for config in npu_configs: |
| 551 | logging.info(config) |
| 552 | |
Isabella Gottardi | 3acaaee | 2021-11-30 12:33:27 +0000 | [diff] [blame] | 553 | optimisation_skipped = False |
| 554 | |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 555 | for model in models: |
Richard Burton | 1706962 | 2022-03-17 10:54:26 +0000 | [diff] [blame] | 556 | output_dir = model.parent |
alexander | 3ef1fd4 | 2021-05-24 18:56:32 +0100 | [diff] [blame] | 557 | # model name after compiling with vela is an initial model name + _vela suffix |
Richard Burton | 1706962 | 2022-03-17 10:54:26 +0000 | [diff] [blame] | 558 | vela_optimised_model_path = model.parent / (model.stem + "_vela.tflite") |
alexander | 3ef1fd4 | 2021-05-24 18:56:32 +0100 | [diff] [blame] | 559 | |
Kshitij Sisodia | 3be2623 | 2021-10-29 12:29:06 +0100 | [diff] [blame] | 560 | for config in npu_configs: |
Isabella Gottardi | 3acaaee | 2021-11-30 12:33:27 +0000 | [diff] [blame] | 561 | vela_command_arena_cache_size = "" |
| 562 | |
| 563 | if config.arena_cache_size: |
Isabella Gottardi | ef2b9dd | 2022-02-16 14:24:03 +0000 | [diff] [blame] | 564 | vela_command_arena_cache_size = ( |
| 565 | f"--arena-cache-size={config.arena_cache_size}" |
| 566 | ) |
Isabella Gottardi | 3acaaee | 2021-11-30 12:33:27 +0000 | [diff] [blame] | 567 | |
Isabella Gottardi | ef2b9dd | 2022-02-16 14:24:03 +0000 | [diff] [blame] | 568 | vela_command = ( |
| 569 | f". {env_activate} && vela {model} " |
| 570 | + f"--accelerator-config={config.config_name} " |
| 571 | + "--optimise Performance " |
| 572 | + f"--config {config_file} " |
| 573 | + f"--memory-mode={config.memory_mode} " |
| 574 | + f"--system-config={config.system_config} " |
| 575 | + f"--output-dir={output_dir} " |
| 576 | + f"{vela_command_arena_cache_size}" |
| 577 | ) |
alexander | 3ef1fd4 | 2021-05-24 18:56:32 +0100 | [diff] [blame] | 578 | |
Richard Burton | 1706962 | 2022-03-17 10:54:26 +0000 | [diff] [blame] | 579 | # We want the name to include the configuration suffix. For example: vela_H128, |
Kshitij Sisodia | 3be2623 | 2021-10-29 12:29:06 +0100 | [diff] [blame] | 580 | # vela_Y512 etc. |
Isabella Gottardi | ef2b9dd | 2022-02-16 14:24:03 +0000 | [diff] [blame] | 581 | new_suffix = "_vela_" + config.ethos_u_config_id + ".tflite" |
Richard Burton | 1706962 | 2022-03-17 10:54:26 +0000 | [diff] [blame] | 582 | new_vela_optimised_model_path = model.parent / (model.stem + new_suffix) |
Isabella Gottardi | 118f73e | 2021-09-16 17:54:35 +0100 | [diff] [blame] | 583 | |
Richard Burton | 1706962 | 2022-03-17 10:54:26 +0000 | [diff] [blame] | 584 | if new_vela_optimised_model_path.is_file(): |
Isabella Gottardi | ef2b9dd | 2022-02-16 14:24:03 +0000 | [diff] [blame] | 585 | logging.info( |
| 586 | f"File {new_vela_optimised_model_path} exists, skipping optimisation." |
| 587 | ) |
Isabella Gottardi | 3acaaee | 2021-11-30 12:33:27 +0000 | [diff] [blame] | 588 | optimisation_skipped = True |
Isabella Gottardi | 118f73e | 2021-09-16 17:54:35 +0100 | [diff] [blame] | 589 | continue |
| 590 | |
Kshitij Sisodia | 3be2623 | 2021-10-29 12:29:06 +0100 | [diff] [blame] | 591 | call_command(vela_command) |
Isabella Gottardi | 118f73e | 2021-09-16 17:54:35 +0100 | [diff] [blame] | 592 | |
Richard Burton | 1706962 | 2022-03-17 10:54:26 +0000 | [diff] [blame] | 593 | # Rename default vela model. |
| 594 | vela_optimised_model_path.rename(new_vela_optimised_model_path) |
Isabella Gottardi | ef2b9dd | 2022-02-16 14:24:03 +0000 | [diff] [blame] | 595 | logging.info( |
| 596 | f"Renaming {vela_optimised_model_path} to {new_vela_optimised_model_path}." |
| 597 | ) |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 598 | |
Isabella Gottardi | 3acaaee | 2021-11-30 12:33:27 +0000 | [diff] [blame] | 599 | # If any optimisation was skipped, show how to regenerate: |
| 600 | if optimisation_skipped: |
| 601 | logging.warning("One or more optimisations were skipped.") |
Isabella Gottardi | ef2b9dd | 2022-02-16 14:24:03 +0000 | [diff] [blame] | 602 | logging.warning( |
| 603 | f"To optimise all the models, please remove the directory {download_dir}." |
| 604 | ) |
| 605 | |
| 606 | # 4. Collect and write metadata |
| 607 | logging.info("Collecting and write metadata.") |
| 608 | metadata_dict["ethosu_vela_version"] = vela_version |
Kshitij Sisodia | 6a2ac46 | 2022-03-01 17:36:06 +0000 | [diff] [blame] | 609 | metadata_dict["set_up_script_md5sum"] = setup_script_hash.strip("\n") |
Isabella Gottardi | ef2b9dd | 2022-02-16 14:24:03 +0000 | [diff] [blame] | 610 | metadata_dict["resources_info"] = json_uc_res |
| 611 | |
| 612 | with open(metadata_file_path, "w") as metadata_file: |
| 613 | json.dump(metadata_dict, metadata_file, indent=4) |
Isabella Gottardi | 3acaaee | 2021-11-30 12:33:27 +0000 | [diff] [blame] | 614 | |
Kshitij Sisodia | 9c6f9f8 | 2022-05-20 14:30:02 +0100 | [diff] [blame] | 615 | return download_dir, env_path |
| 616 | |
Isabella Gottardi | 3acaaee | 2021-11-30 12:33:27 +0000 | [diff] [blame] | 617 | |
Isabella Gottardi | ef2b9dd | 2022-02-16 14:24:03 +0000 | [diff] [blame] | 618 | if __name__ == "__main__": |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 619 | parser = ArgumentParser() |
Isabella Gottardi | ef2b9dd | 2022-02-16 14:24:03 +0000 | [diff] [blame] | 620 | parser.add_argument( |
| 621 | "--skip-vela", |
| 622 | help="Do not run Vela optimizer on downloaded models.", |
| 623 | action="store_true", |
| 624 | ) |
| 625 | parser.add_argument( |
| 626 | "--additional-ethos-u-config-name", |
| 627 | help=f"""Additional (non-default) configurations for Vela: |
Kshitij Sisodia | 3be2623 | 2021-10-29 12:29:06 +0100 | [diff] [blame] | 628 | {valid_npu_config_names}""", |
Isabella Gottardi | ef2b9dd | 2022-02-16 14:24:03 +0000 | [diff] [blame] | 629 | default=[], |
| 630 | action="append", |
| 631 | ) |
| 632 | parser.add_argument( |
| 633 | "--arena-cache-size", |
| 634 | help="Arena cache size in bytes (if overriding the defaults)", |
| 635 | type=int, |
| 636 | default=0, |
| 637 | ) |
| 638 | parser.add_argument( |
| 639 | "--clean", |
Richard Burton | 1706962 | 2022-03-17 10:54:26 +0000 | [diff] [blame] | 640 | help="Clean the directory and optimize the downloaded resources", |
Isabella Gottardi | ef2b9dd | 2022-02-16 14:24:03 +0000 | [diff] [blame] | 641 | action="store_true", |
| 642 | ) |
Kshitij Sisodia | c22e80e | 2022-03-14 09:26:48 +0000 | [diff] [blame] | 643 | parser.add_argument( |
| 644 | "--requirements-file", |
| 645 | help="Path to requirements.txt file to install additional packages", |
| 646 | type=str, |
Richard Burton | 1706962 | 2022-03-17 10:54:26 +0000 | [diff] [blame] | 647 | default=Path(__file__).parent.resolve() / 'scripts' / 'py' / 'requirements.txt' |
Kshitij Sisodia | c22e80e | 2022-03-14 09:26:48 +0000 | [diff] [blame] | 648 | ) |
| 649 | |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 650 | args = parser.parse_args() |
Kshitij Sisodia | b9e9c89 | 2021-05-27 13:57:35 +0100 | [diff] [blame] | 651 | |
Isabella Gottardi | 3acaaee | 2021-11-30 12:33:27 +0000 | [diff] [blame] | 652 | if args.arena_cache_size < 0: |
Isabella Gottardi | ef2b9dd | 2022-02-16 14:24:03 +0000 | [diff] [blame] | 653 | raise ArgumentTypeError("Arena cache size cannot not be less than 0") |
Isabella Gottardi | 3acaaee | 2021-11-30 12:33:27 +0000 | [diff] [blame] | 654 | |
Richard Burton | 1706962 | 2022-03-17 10:54:26 +0000 | [diff] [blame] | 655 | if not Path(args.requirements_file).is_file(): |
Kshitij Sisodia | c22e80e | 2022-03-14 09:26:48 +0000 | [diff] [blame] | 656 | raise ArgumentTypeError(f"Invalid requirements file: {args.requirements_file}") |
| 657 | |
Isabella Gottardi | ef2b9dd | 2022-02-16 14:24:03 +0000 | [diff] [blame] | 658 | logging.basicConfig(filename="log_build_default.log", level=logging.DEBUG) |
Kshitij Sisodia | b9e9c89 | 2021-05-27 13:57:35 +0100 | [diff] [blame] | 659 | logging.getLogger().addHandler(logging.StreamHandler(sys.stdout)) |
| 660 | |
Isabella Gottardi | ef2b9dd | 2022-02-16 14:24:03 +0000 | [diff] [blame] | 661 | set_up_resources( |
| 662 | not args.skip_vela, |
| 663 | args.additional_ethos_u_config_name, |
| 664 | args.arena_cache_size, |
| 665 | args.clean, |
Isabella Gottardi | 6c2ea45 | 2022-03-11 13:25:08 +0000 | [diff] [blame] | 666 | args.requirements_file, |
Isabella Gottardi | ef2b9dd | 2022-02-16 14:24:03 +0000 | [diff] [blame] | 667 | ) |