alexander | f4e2c47 | 2021-05-14 13:14:21 +0100 | [diff] [blame] | 1 | #!/usr/bin/env python3 |
Hugues Kamba-Mpiana | fea932f | 2024-03-04 16:01:55 +0000 | [diff] [blame] | 2 | # SPDX-FileCopyrightText: Copyright 2021-2024 Arm Limited and/or its affiliates <open-source-office@arm.com> |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 3 | # SPDX-License-Identifier: Apache-2.0 |
| 4 | # |
| 5 | # Licensed under the Apache License, Version 2.0 (the "License"); |
| 6 | # you may not use this file except in compliance with the License. |
| 7 | # You may obtain a copy of the License at |
| 8 | # |
| 9 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | # |
| 11 | # Unless required by applicable law or agreed to in writing, software |
| 12 | # distributed under the License is distributed on an "AS IS" BASIS, |
| 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 14 | # See the License for the specific language governing permissions and |
| 15 | # limitations under the License. |
Alex Tawse | daba3cf | 2023-09-29 15:55:38 +0100 | [diff] [blame] | 16 | """ |
| 17 | Script to build the ML Embedded Evaluation kit using default configuration |
| 18 | """ |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 19 | import logging |
Isabella Gottardi | ee4920b | 2022-02-25 14:29:32 +0000 | [diff] [blame] | 20 | import multiprocessing |
| 21 | import os |
| 22 | import shutil |
| 23 | import subprocess |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 24 | import sys |
Isabella Gottardi | ee4920b | 2022-02-25 14:29:32 +0000 | [diff] [blame] | 25 | import threading |
| 26 | from argparse import ArgumentDefaultsHelpFormatter |
| 27 | from argparse import ArgumentParser |
Hugues Kamba-Mpiana | fea932f | 2024-03-04 16:01:55 +0000 | [diff] [blame] | 28 | from dataclasses import dataclass |
Richard Burton | 1706962 | 2022-03-17 10:54:26 +0000 | [diff] [blame] | 29 | from pathlib import Path |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 30 | |
Hugues Kamba-Mpiana | fea932f | 2024-03-04 16:01:55 +0000 | [diff] [blame] | 31 | from set_up_default_resources import SetupArgs |
Isabella Gottardi | ee4920b | 2022-02-25 14:29:32 +0000 | [diff] [blame] | 32 | from set_up_default_resources import default_npu_config_names |
| 33 | from set_up_default_resources import get_default_npu_config_from_name |
| 34 | from set_up_default_resources import set_up_resources |
| 35 | from set_up_default_resources import valid_npu_config_names |
| 36 | |
Hugues Kamba-Mpiana | fea932f | 2024-03-04 16:01:55 +0000 | [diff] [blame] | 37 | |
| 38 | @dataclass(frozen=True) |
| 39 | class BuildArgs: |
| 40 | """ |
| 41 | Args used to build the project. |
| 42 | |
| 43 | Attributes: |
| 44 | toolchain (str) : Specifies if 'gnu' or 'arm' toolchain needs to be used. |
| 45 | download_resources (bool) : Specifies if 'Download resources' step is performed. |
| 46 | run_vela_on_models (bool) : Only if `download_resources` is True, specifies whether to |
| 47 | run Vela on downloaded models. |
| 48 | npu_config_name (str) : Ethos-U NPU configuration name. See "valid_npu_config_names" |
| 49 | make_jobs (int) : The number of make jobs to use (`-j` flag). |
| 50 | make_verbose (bool) : Runs make with VERBOSE=1. |
| 51 | """ |
| 52 | toolchain: str |
| 53 | download_resources: bool |
| 54 | run_vela_on_models: bool |
| 55 | npu_config_name: str |
| 56 | make_jobs: int |
| 57 | make_verbose: bool |
Alex Tawse | daba3cf | 2023-09-29 15:55:38 +0100 | [diff] [blame] | 58 | |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 59 | |
Cisco Cervellera | 6ef76bd | 2021-11-25 18:32:27 +0000 | [diff] [blame] | 60 | class PipeLogging(threading.Thread): |
Alex Tawse | daba3cf | 2023-09-29 15:55:38 +0100 | [diff] [blame] | 61 | """ |
| 62 | Class used to log stdout from subprocesses |
| 63 | """ |
| 64 | |
Cisco Cervellera | 6ef76bd | 2021-11-25 18:32:27 +0000 | [diff] [blame] | 65 | def __init__(self, log_level): |
| 66 | threading.Thread.__init__(self) |
Alex Tawse | daba3cf | 2023-09-29 15:55:38 +0100 | [diff] [blame] | 67 | self.log_level = log_level |
| 68 | self.file_read, self.file_write = os.pipe() |
| 69 | self.pipe_in = os.fdopen(self.file_read) |
Cisco Cervellera | 6ef76bd | 2021-11-25 18:32:27 +0000 | [diff] [blame] | 70 | self.daemon = False |
| 71 | self.start() |
| 72 | |
| 73 | def fileno(self): |
Alex Tawse | daba3cf | 2023-09-29 15:55:38 +0100 | [diff] [blame] | 74 | """ |
| 75 | Get self.file_write |
| 76 | |
| 77 | Returns |
| 78 | ------- |
| 79 | self.file_write |
| 80 | """ |
| 81 | return self.file_write |
Cisco Cervellera | 6ef76bd | 2021-11-25 18:32:27 +0000 | [diff] [blame] | 82 | |
| 83 | def run(self): |
Alex Tawse | daba3cf | 2023-09-29 15:55:38 +0100 | [diff] [blame] | 84 | """ |
| 85 | Log the contents of self.pipe_in |
| 86 | """ |
| 87 | for line in iter(self.pipe_in.readline, ""): |
| 88 | logging.log(self.log_level, line.strip("\n")) |
Cisco Cervellera | 6ef76bd | 2021-11-25 18:32:27 +0000 | [diff] [blame] | 89 | |
Alex Tawse | daba3cf | 2023-09-29 15:55:38 +0100 | [diff] [blame] | 90 | self.pipe_in.close() |
Cisco Cervellera | 6ef76bd | 2021-11-25 18:32:27 +0000 | [diff] [blame] | 91 | |
| 92 | def close(self): |
Alex Tawse | daba3cf | 2023-09-29 15:55:38 +0100 | [diff] [blame] | 93 | """ |
| 94 | Close the pipe |
| 95 | """ |
| 96 | os.close(self.file_write) |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 97 | |
Isabella Gottardi | ee4920b | 2022-02-25 14:29:32 +0000 | [diff] [blame] | 98 | |
Alex Tawse | daba3cf | 2023-09-29 15:55:38 +0100 | [diff] [blame] | 99 | def get_toolchain_file_name(toolchain: str) -> str: |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 100 | """ |
Alex Tawse | daba3cf | 2023-09-29 15:55:38 +0100 | [diff] [blame] | 101 | Get the name of the toolchain file for the toolchain. |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 102 | |
Alex Tawse | daba3cf | 2023-09-29 15:55:38 +0100 | [diff] [blame] | 103 | Parameters |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 104 | ---------- |
Alex Tawse | daba3cf | 2023-09-29 15:55:38 +0100 | [diff] [blame] | 105 | toolchain : name of the specified toolchain |
| 106 | |
| 107 | Returns |
| 108 | ------- |
| 109 | name of the toolchain file corresponding to the specified |
| 110 | toolchain |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 111 | """ |
Kshitij Sisodia | f9c19ea | 2021-05-07 16:08:14 +0100 | [diff] [blame] | 112 | if toolchain == "arm": |
Alex Tawse | daba3cf | 2023-09-29 15:55:38 +0100 | [diff] [blame] | 113 | return "bare-metal-armclang.cmake" |
Kshitij Sisodia | f9c19ea | 2021-05-07 16:08:14 +0100 | [diff] [blame] | 114 | |
Alex Tawse | daba3cf | 2023-09-29 15:55:38 +0100 | [diff] [blame] | 115 | if toolchain == "gnu": |
| 116 | return "bare-metal-gcc.cmake" |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 117 | |
Alex Tawse | daba3cf | 2023-09-29 15:55:38 +0100 | [diff] [blame] | 118 | raise ValueError("Toolchain must be one of: gnu, arm") |
| 119 | |
| 120 | |
| 121 | def prep_build_dir( |
| 122 | current_file_dir: Path, |
| 123 | target_platform: str, |
| 124 | target_subsystem: str, |
| 125 | npu_config_name: str, |
| 126 | toolchain: str |
| 127 | ) -> Path: |
| 128 | """ |
| 129 | Create or clean the build directory for this project. |
| 130 | |
| 131 | Parameters |
| 132 | ---------- |
| 133 | current_file_dir : The current directory of the running script |
| 134 | target_platform : The name of the target platform, e.g. "mps3" |
| 135 | target_subsystem: : The name of the target subsystem, e.g. "sse-300" |
| 136 | npu_config_name : The NPU config name, e.g. "ethos-u55-32" |
| 137 | toolchain : The name of the specified toolchain, e.g."arm" |
| 138 | |
| 139 | Returns |
| 140 | ------- |
| 141 | The path to the build directory |
| 142 | """ |
| 143 | build_dir = ( |
| 144 | current_file_dir / |
| 145 | f"cmake-build-{target_platform}-{target_subsystem}-{npu_config_name}-{toolchain}" |
| 146 | ) |
Richard Burton | 1706962 | 2022-03-17 10:54:26 +0000 | [diff] [blame] | 147 | |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 148 | try: |
Richard Burton | 1706962 | 2022-03-17 10:54:26 +0000 | [diff] [blame] | 149 | build_dir.mkdir() |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 150 | except FileExistsError: |
Richard Burton | 1706962 | 2022-03-17 10:54:26 +0000 | [diff] [blame] | 151 | # Directory already exists, clean it. |
| 152 | for filepath in build_dir.iterdir(): |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 153 | try: |
Richard Burton | 1706962 | 2022-03-17 10:54:26 +0000 | [diff] [blame] | 154 | if filepath.is_file() or filepath.is_symlink(): |
| 155 | filepath.unlink() |
| 156 | elif filepath.is_dir(): |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 157 | shutil.rmtree(filepath) |
Alex Tawse | daba3cf | 2023-09-29 15:55:38 +0100 | [diff] [blame] | 158 | except OSError as err: |
| 159 | logging.error("Failed to delete %s. Reason: %s", filepath, err) |
| 160 | |
| 161 | return build_dir |
| 162 | |
| 163 | |
| 164 | def run_command( |
| 165 | command: str, |
| 166 | logpipe: PipeLogging, |
| 167 | fail_message: str |
| 168 | ): |
| 169 | """ |
| 170 | Run a command and exit upon failure. |
| 171 | |
| 172 | Parameters |
| 173 | ---------- |
| 174 | command : The command to run |
| 175 | logpipe : The PipeLogging object to capture stdout |
| 176 | fail_message : The message to log upon a non-zero exit code |
| 177 | """ |
| 178 | logging.info("\n\n\n%s\n\n\n", command) |
| 179 | |
| 180 | try: |
| 181 | subprocess.run( |
| 182 | command, check=True, shell=True, stdout=logpipe, stderr=subprocess.STDOUT |
| 183 | ) |
| 184 | except subprocess.CalledProcessError as err: |
| 185 | logging.error(fail_message) |
| 186 | logpipe.close() |
| 187 | sys.exit(err.returncode) |
| 188 | |
| 189 | |
| 190 | def run(args: BuildArgs): |
| 191 | """ |
| 192 | Run the helpers scripts. |
| 193 | |
| 194 | Parameters: |
| 195 | ---------- |
Hugues Kamba-Mpiana | fea932f | 2024-03-04 16:01:55 +0000 | [diff] [blame] | 196 | args (BuildArgs) : Arguments used to build the project |
Alex Tawse | daba3cf | 2023-09-29 15:55:38 +0100 | [diff] [blame] | 197 | """ |
| 198 | |
| 199 | current_file_dir = Path(__file__).parent.resolve() |
| 200 | |
| 201 | # 1. Make sure the toolchain is supported, and set the right one here |
| 202 | toolchain_file_name = get_toolchain_file_name(args.toolchain) |
| 203 | |
| 204 | # 2. Download models if specified |
| 205 | if args.download_resources is True: |
| 206 | logging.info("Downloading resources.") |
Hugues Kamba-Mpiana | fea932f | 2024-03-04 16:01:55 +0000 | [diff] [blame] | 207 | setup_args = SetupArgs( |
Alex Tawse | daba3cf | 2023-09-29 15:55:38 +0100 | [diff] [blame] | 208 | run_vela_on_models=args.run_vela_on_models, |
Hugues Kamba-Mpiana | fea932f | 2024-03-04 16:01:55 +0000 | [diff] [blame] | 209 | additional_npu_config_names=[args.npu_config_name], |
| 210 | additional_requirements_file=current_file_dir / "scripts" / "py" / "requirements.txt", |
| 211 | use_case_resources_file=current_file_dir / "scripts" / "py" / "use_case_resources.json", |
Alex Tawse | daba3cf | 2023-09-29 15:55:38 +0100 | [diff] [blame] | 212 | ) |
Hugues Kamba-Mpiana | fea932f | 2024-03-04 16:01:55 +0000 | [diff] [blame] | 213 | env_path = set_up_resources(setup_args) |
Alex Tawse | daba3cf | 2023-09-29 15:55:38 +0100 | [diff] [blame] | 214 | |
| 215 | # 3. Build default configuration |
| 216 | logging.info("Building default configuration.") |
| 217 | target_platform = "mps3" |
| 218 | target_subsystem = "sse-300" |
| 219 | |
| 220 | build_dir = prep_build_dir( |
| 221 | current_file_dir, |
| 222 | target_platform, |
| 223 | target_subsystem, |
| 224 | args.npu_config_name, |
| 225 | args.toolchain |
| 226 | ) |
Kshitij Sisodia | f9c19ea | 2021-05-07 16:08:14 +0100 | [diff] [blame] | 227 | |
Cisco Cervellera | 6ef76bd | 2021-11-25 18:32:27 +0000 | [diff] [blame] | 228 | logpipe = PipeLogging(logging.INFO) |
| 229 | |
Alex Tawse | daba3cf | 2023-09-29 15:55:38 +0100 | [diff] [blame] | 230 | cmake_toolchain_file = ( |
| 231 | current_file_dir / |
| 232 | "scripts" / |
| 233 | "cmake" / |
| 234 | "toolchains" / |
| 235 | toolchain_file_name |
| 236 | ) |
| 237 | ethos_u_cfg = get_default_npu_config_from_name(args.npu_config_name) |
Kshitij Sisodia | 9c6f9f8 | 2022-05-20 14:30:02 +0100 | [diff] [blame] | 238 | cmake_path = env_path / "bin" / "cmake" |
Isabella Gottardi | ee4920b | 2022-02-25 14:29:32 +0000 | [diff] [blame] | 239 | cmake_command = ( |
Kshitij Sisodia | 9c6f9f8 | 2022-05-20 14:30:02 +0100 | [diff] [blame] | 240 | f"{cmake_path} -B {build_dir} -DTARGET_PLATFORM={target_platform}" |
Alex Tawse | daba3cf | 2023-09-29 15:55:38 +0100 | [diff] [blame] | 241 | f" -DTARGET_SUBSYSTEM={target_subsystem}" |
| 242 | f" -DCMAKE_TOOLCHAIN_FILE={cmake_toolchain_file}" |
| 243 | f" -DETHOS_U_NPU_ID={ethos_u_cfg.ethos_u_npu_id}" |
| 244 | f" -DETHOS_U_NPU_CONFIG_ID={ethos_u_cfg.ethos_u_config_id}" |
| 245 | " -DTENSORFLOW_LITE_MICRO_CLEAN_DOWNLOADS=ON" |
Isabella Gottardi | ee4920b | 2022-02-25 14:29:32 +0000 | [diff] [blame] | 246 | ) |
Kshitij Sisodia | f9c19ea | 2021-05-07 16:08:14 +0100 | [diff] [blame] | 247 | |
Alex Tawse | daba3cf | 2023-09-29 15:55:38 +0100 | [diff] [blame] | 248 | run_command(cmake_command, logpipe, fail_message="Failed to configure the project.") |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 249 | |
Alex Tawse | daba3cf | 2023-09-29 15:55:38 +0100 | [diff] [blame] | 250 | make_command = f"{cmake_path} --build {build_dir} -j{args.make_jobs}" |
| 251 | if args.make_verbose: |
Richard Burton | 2bf6a70 | 2023-06-19 14:51:36 +0100 | [diff] [blame] | 252 | make_command += " --verbose" |
Cisco Cervellera | 6ef76bd | 2021-11-25 18:32:27 +0000 | [diff] [blame] | 253 | |
Alex Tawse | daba3cf | 2023-09-29 15:55:38 +0100 | [diff] [blame] | 254 | run_command(make_command, logpipe, fail_message="Failed to build project.") |
Kshitij Sisodia | 1bd434e | 2023-03-17 10:40:45 +0000 | [diff] [blame] | 255 | |
Cisco Cervellera | 6ef76bd | 2021-11-25 18:32:27 +0000 | [diff] [blame] | 256 | logpipe.close() |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 257 | |
| 258 | |
Isabella Gottardi | ee4920b | 2022-02-25 14:29:32 +0000 | [diff] [blame] | 259 | if __name__ == "__main__": |
Cisco Cervellera | 6ef76bd | 2021-11-25 18:32:27 +0000 | [diff] [blame] | 260 | parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter) |
Isabella Gottardi | ee4920b | 2022-02-25 14:29:32 +0000 | [diff] [blame] | 261 | parser.add_argument( |
| 262 | "--toolchain", |
| 263 | default="gnu", |
| 264 | help=""" |
Richard Burton | 1706962 | 2022-03-17 10:54:26 +0000 | [diff] [blame] | 265 | Specify the toolchain to use (Arm or GNU). |
| 266 | Options are [gnu, arm]; default is gnu. |
| 267 | """, |
Isabella Gottardi | ee4920b | 2022-02-25 14:29:32 +0000 | [diff] [blame] | 268 | ) |
| 269 | parser.add_argument( |
| 270 | "--skip-download", |
| 271 | help="Do not download resources: models and test vectors", |
| 272 | action="store_true", |
| 273 | ) |
| 274 | parser.add_argument( |
| 275 | "--skip-vela", |
| 276 | help="Do not run Vela optimizer on downloaded models.", |
| 277 | action="store_true", |
| 278 | ) |
| 279 | parser.add_argument( |
| 280 | "--npu-config-name", |
| 281 | help=f"""Arm Ethos-U configuration to build for. Choose from: |
Richard Burton | 1706962 | 2022-03-17 10:54:26 +0000 | [diff] [blame] | 282 | {valid_npu_config_names}""", |
Isabella Gottardi | ee4920b | 2022-02-25 14:29:32 +0000 | [diff] [blame] | 283 | default=default_npu_config_names[0], |
| 284 | ) |
| 285 | parser.add_argument( |
| 286 | "--make-jobs", |
| 287 | help="Number of jobs to run with make", |
| 288 | default=multiprocessing.cpu_count(), |
| 289 | ) |
| 290 | parser.add_argument( |
| 291 | "--make-verbose", help="Make runs with VERBOSE=1", action="store_true" |
| 292 | ) |
Alex Tawse | daba3cf | 2023-09-29 15:55:38 +0100 | [diff] [blame] | 293 | parsed_args = parser.parse_args() |
Kshitij Sisodia | b9e9c89 | 2021-05-27 13:57:35 +0100 | [diff] [blame] | 294 | |
Isabella Gottardi | ee4920b | 2022-02-25 14:29:32 +0000 | [diff] [blame] | 295 | logging.basicConfig( |
| 296 | filename="log_build_default.log", level=logging.DEBUG, filemode="w" |
| 297 | ) |
Kshitij Sisodia | b9e9c89 | 2021-05-27 13:57:35 +0100 | [diff] [blame] | 298 | logging.getLogger().addHandler(logging.StreamHandler(sys.stdout)) |
| 299 | |
Alex Tawse | daba3cf | 2023-09-29 15:55:38 +0100 | [diff] [blame] | 300 | build_args = BuildArgs( |
| 301 | toolchain=parsed_args.toolchain.lower(), |
| 302 | download_resources=not parsed_args.skip_download, |
| 303 | run_vela_on_models=not parsed_args.skip_vela, |
| 304 | npu_config_name=parsed_args.npu_config_name, |
| 305 | make_jobs=parsed_args.make_jobs, |
| 306 | make_verbose=parsed_args.make_verbose |
Isabella Gottardi | ee4920b | 2022-02-25 14:29:32 +0000 | [diff] [blame] | 307 | ) |
Alex Tawse | daba3cf | 2023-09-29 15:55:38 +0100 | [diff] [blame] | 308 | |
| 309 | run(build_args) |