alexander | f4e2c47 | 2021-05-14 13:14:21 +0100 | [diff] [blame] | 1 | #!/usr/bin/env python3 |
Alex Tawse | daba3cf | 2023-09-29 15:55:38 +0100 | [diff] [blame] | 2 | # SPDX-FileCopyrightText: Copyright 2021-2023 Arm Limited and/or its affiliates <open-source-office@arm.com> |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 3 | # SPDX-License-Identifier: Apache-2.0 |
| 4 | # |
| 5 | # Licensed under the Apache License, Version 2.0 (the "License"); |
| 6 | # you may not use this file except in compliance with the License. |
| 7 | # You may obtain a copy of the License at |
| 8 | # |
| 9 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | # |
| 11 | # Unless required by applicable law or agreed to in writing, software |
| 12 | # distributed under the License is distributed on an "AS IS" BASIS, |
| 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 14 | # See the License for the specific language governing permissions and |
| 15 | # limitations under the License. |
Alex Tawse | daba3cf | 2023-09-29 15:55:38 +0100 | [diff] [blame] | 16 | """ |
| 17 | Script to build the ML Embedded Evaluation kit using default configuration |
| 18 | """ |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 19 | import logging |
Isabella Gottardi | ee4920b | 2022-02-25 14:29:32 +0000 | [diff] [blame] | 20 | import multiprocessing |
| 21 | import os |
| 22 | import shutil |
| 23 | import subprocess |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 24 | import sys |
Isabella Gottardi | ee4920b | 2022-02-25 14:29:32 +0000 | [diff] [blame] | 25 | import threading |
| 26 | from argparse import ArgumentDefaultsHelpFormatter |
| 27 | from argparse import ArgumentParser |
Alex Tawse | daba3cf | 2023-09-29 15:55:38 +0100 | [diff] [blame] | 28 | from collections import namedtuple |
Richard Burton | 1706962 | 2022-03-17 10:54:26 +0000 | [diff] [blame] | 29 | from pathlib import Path |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 30 | |
Isabella Gottardi | ee4920b | 2022-02-25 14:29:32 +0000 | [diff] [blame] | 31 | from set_up_default_resources import default_npu_config_names |
| 32 | from set_up_default_resources import get_default_npu_config_from_name |
| 33 | from set_up_default_resources import set_up_resources |
| 34 | from set_up_default_resources import valid_npu_config_names |
| 35 | |
Alex Tawse | daba3cf | 2023-09-29 15:55:38 +0100 | [diff] [blame] | 36 | BuildArgs = namedtuple( |
| 37 | "BuildArgs", |
| 38 | [ |
| 39 | "toolchain", |
| 40 | "download_resources", |
| 41 | "run_vela_on_models", |
| 42 | "npu_config_name", |
| 43 | "make_jobs", |
| 44 | "make_verbose", |
| 45 | ], |
| 46 | ) |
| 47 | |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 48 | |
Cisco Cervellera | 6ef76bd | 2021-11-25 18:32:27 +0000 | [diff] [blame] | 49 | class PipeLogging(threading.Thread): |
Alex Tawse | daba3cf | 2023-09-29 15:55:38 +0100 | [diff] [blame] | 50 | """ |
| 51 | Class used to log stdout from subprocesses |
| 52 | """ |
| 53 | |
Cisco Cervellera | 6ef76bd | 2021-11-25 18:32:27 +0000 | [diff] [blame] | 54 | def __init__(self, log_level): |
| 55 | threading.Thread.__init__(self) |
Alex Tawse | daba3cf | 2023-09-29 15:55:38 +0100 | [diff] [blame] | 56 | self.log_level = log_level |
| 57 | self.file_read, self.file_write = os.pipe() |
| 58 | self.pipe_in = os.fdopen(self.file_read) |
Cisco Cervellera | 6ef76bd | 2021-11-25 18:32:27 +0000 | [diff] [blame] | 59 | self.daemon = False |
| 60 | self.start() |
| 61 | |
| 62 | def fileno(self): |
Alex Tawse | daba3cf | 2023-09-29 15:55:38 +0100 | [diff] [blame] | 63 | """ |
| 64 | Get self.file_write |
| 65 | |
| 66 | Returns |
| 67 | ------- |
| 68 | self.file_write |
| 69 | """ |
| 70 | return self.file_write |
Cisco Cervellera | 6ef76bd | 2021-11-25 18:32:27 +0000 | [diff] [blame] | 71 | |
| 72 | def run(self): |
Alex Tawse | daba3cf | 2023-09-29 15:55:38 +0100 | [diff] [blame] | 73 | """ |
| 74 | Log the contents of self.pipe_in |
| 75 | """ |
| 76 | for line in iter(self.pipe_in.readline, ""): |
| 77 | logging.log(self.log_level, line.strip("\n")) |
Cisco Cervellera | 6ef76bd | 2021-11-25 18:32:27 +0000 | [diff] [blame] | 78 | |
Alex Tawse | daba3cf | 2023-09-29 15:55:38 +0100 | [diff] [blame] | 79 | self.pipe_in.close() |
Cisco Cervellera | 6ef76bd | 2021-11-25 18:32:27 +0000 | [diff] [blame] | 80 | |
| 81 | def close(self): |
Alex Tawse | daba3cf | 2023-09-29 15:55:38 +0100 | [diff] [blame] | 82 | """ |
| 83 | Close the pipe |
| 84 | """ |
| 85 | os.close(self.file_write) |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 86 | |
Isabella Gottardi | ee4920b | 2022-02-25 14:29:32 +0000 | [diff] [blame] | 87 | |
Alex Tawse | daba3cf | 2023-09-29 15:55:38 +0100 | [diff] [blame] | 88 | def get_toolchain_file_name(toolchain: str) -> str: |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 89 | """ |
Alex Tawse | daba3cf | 2023-09-29 15:55:38 +0100 | [diff] [blame] | 90 | Get the name of the toolchain file for the toolchain. |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 91 | |
Alex Tawse | daba3cf | 2023-09-29 15:55:38 +0100 | [diff] [blame] | 92 | Parameters |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 93 | ---------- |
Alex Tawse | daba3cf | 2023-09-29 15:55:38 +0100 | [diff] [blame] | 94 | toolchain : name of the specified toolchain |
| 95 | |
| 96 | Returns |
| 97 | ------- |
| 98 | name of the toolchain file corresponding to the specified |
| 99 | toolchain |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 100 | """ |
Kshitij Sisodia | f9c19ea | 2021-05-07 16:08:14 +0100 | [diff] [blame] | 101 | if toolchain == "arm": |
Alex Tawse | daba3cf | 2023-09-29 15:55:38 +0100 | [diff] [blame] | 102 | return "bare-metal-armclang.cmake" |
Kshitij Sisodia | f9c19ea | 2021-05-07 16:08:14 +0100 | [diff] [blame] | 103 | |
Alex Tawse | daba3cf | 2023-09-29 15:55:38 +0100 | [diff] [blame] | 104 | if toolchain == "gnu": |
| 105 | return "bare-metal-gcc.cmake" |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 106 | |
Alex Tawse | daba3cf | 2023-09-29 15:55:38 +0100 | [diff] [blame] | 107 | raise ValueError("Toolchain must be one of: gnu, arm") |
| 108 | |
| 109 | |
| 110 | def prep_build_dir( |
| 111 | current_file_dir: Path, |
| 112 | target_platform: str, |
| 113 | target_subsystem: str, |
| 114 | npu_config_name: str, |
| 115 | toolchain: str |
| 116 | ) -> Path: |
| 117 | """ |
| 118 | Create or clean the build directory for this project. |
| 119 | |
| 120 | Parameters |
| 121 | ---------- |
| 122 | current_file_dir : The current directory of the running script |
| 123 | target_platform : The name of the target platform, e.g. "mps3" |
| 124 | target_subsystem: : The name of the target subsystem, e.g. "sse-300" |
| 125 | npu_config_name : The NPU config name, e.g. "ethos-u55-32" |
| 126 | toolchain : The name of the specified toolchain, e.g."arm" |
| 127 | |
| 128 | Returns |
| 129 | ------- |
| 130 | The path to the build directory |
| 131 | """ |
| 132 | build_dir = ( |
| 133 | current_file_dir / |
| 134 | f"cmake-build-{target_platform}-{target_subsystem}-{npu_config_name}-{toolchain}" |
| 135 | ) |
Richard Burton | 1706962 | 2022-03-17 10:54:26 +0000 | [diff] [blame] | 136 | |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 137 | try: |
Richard Burton | 1706962 | 2022-03-17 10:54:26 +0000 | [diff] [blame] | 138 | build_dir.mkdir() |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 139 | except FileExistsError: |
Richard Burton | 1706962 | 2022-03-17 10:54:26 +0000 | [diff] [blame] | 140 | # Directory already exists, clean it. |
| 141 | for filepath in build_dir.iterdir(): |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 142 | try: |
Richard Burton | 1706962 | 2022-03-17 10:54:26 +0000 | [diff] [blame] | 143 | if filepath.is_file() or filepath.is_symlink(): |
| 144 | filepath.unlink() |
| 145 | elif filepath.is_dir(): |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 146 | shutil.rmtree(filepath) |
Alex Tawse | daba3cf | 2023-09-29 15:55:38 +0100 | [diff] [blame] | 147 | except OSError as err: |
| 148 | logging.error("Failed to delete %s. Reason: %s", filepath, err) |
| 149 | |
| 150 | return build_dir |
| 151 | |
| 152 | |
| 153 | def run_command( |
| 154 | command: str, |
| 155 | logpipe: PipeLogging, |
| 156 | fail_message: str |
| 157 | ): |
| 158 | """ |
| 159 | Run a command and exit upon failure. |
| 160 | |
| 161 | Parameters |
| 162 | ---------- |
| 163 | command : The command to run |
| 164 | logpipe : The PipeLogging object to capture stdout |
| 165 | fail_message : The message to log upon a non-zero exit code |
| 166 | """ |
| 167 | logging.info("\n\n\n%s\n\n\n", command) |
| 168 | |
| 169 | try: |
| 170 | subprocess.run( |
| 171 | command, check=True, shell=True, stdout=logpipe, stderr=subprocess.STDOUT |
| 172 | ) |
| 173 | except subprocess.CalledProcessError as err: |
| 174 | logging.error(fail_message) |
| 175 | logpipe.close() |
| 176 | sys.exit(err.returncode) |
| 177 | |
| 178 | |
| 179 | def run(args: BuildArgs): |
| 180 | """ |
| 181 | Run the helpers scripts. |
| 182 | |
| 183 | Parameters: |
| 184 | ---------- |
| 185 | args (BuildArgs) : Parsed set of build args expecting: |
| 186 | - toolchain |
| 187 | - download_resources |
| 188 | - run_vela_on_models |
| 189 | - np_config_name |
| 190 | toolchain (str) : Specifies if 'gnu' or 'arm' toolchain needs to be used. |
| 191 | download_resources (bool) : Specifies if 'Download resources' step is performed. |
| 192 | run_vela_on_models (bool) : Only if `download_resources` is True, specifies if |
| 193 | run vela on downloaded models. |
| 194 | npu_config_name(str) : Ethos-U NPU configuration name. See "valid_npu_config_names" |
| 195 | """ |
| 196 | |
| 197 | current_file_dir = Path(__file__).parent.resolve() |
| 198 | |
| 199 | # 1. Make sure the toolchain is supported, and set the right one here |
| 200 | toolchain_file_name = get_toolchain_file_name(args.toolchain) |
| 201 | |
| 202 | # 2. Download models if specified |
| 203 | if args.download_resources is True: |
| 204 | logging.info("Downloading resources.") |
| 205 | env_path = set_up_resources( |
| 206 | run_vela_on_models=args.run_vela_on_models, |
| 207 | additional_npu_config_names=(args.npu_config_name,), |
| 208 | additional_requirements_file=current_file_dir / "scripts" / "py" / "requirements.txt" |
| 209 | ) |
| 210 | |
| 211 | # 3. Build default configuration |
| 212 | logging.info("Building default configuration.") |
| 213 | target_platform = "mps3" |
| 214 | target_subsystem = "sse-300" |
| 215 | |
| 216 | build_dir = prep_build_dir( |
| 217 | current_file_dir, |
| 218 | target_platform, |
| 219 | target_subsystem, |
| 220 | args.npu_config_name, |
| 221 | args.toolchain |
| 222 | ) |
Kshitij Sisodia | f9c19ea | 2021-05-07 16:08:14 +0100 | [diff] [blame] | 223 | |
Cisco Cervellera | 6ef76bd | 2021-11-25 18:32:27 +0000 | [diff] [blame] | 224 | logpipe = PipeLogging(logging.INFO) |
| 225 | |
Alex Tawse | daba3cf | 2023-09-29 15:55:38 +0100 | [diff] [blame] | 226 | cmake_toolchain_file = ( |
| 227 | current_file_dir / |
| 228 | "scripts" / |
| 229 | "cmake" / |
| 230 | "toolchains" / |
| 231 | toolchain_file_name |
| 232 | ) |
| 233 | ethos_u_cfg = get_default_npu_config_from_name(args.npu_config_name) |
Kshitij Sisodia | 9c6f9f8 | 2022-05-20 14:30:02 +0100 | [diff] [blame] | 234 | cmake_path = env_path / "bin" / "cmake" |
Isabella Gottardi | ee4920b | 2022-02-25 14:29:32 +0000 | [diff] [blame] | 235 | cmake_command = ( |
Kshitij Sisodia | 9c6f9f8 | 2022-05-20 14:30:02 +0100 | [diff] [blame] | 236 | f"{cmake_path} -B {build_dir} -DTARGET_PLATFORM={target_platform}" |
Alex Tawse | daba3cf | 2023-09-29 15:55:38 +0100 | [diff] [blame] | 237 | f" -DTARGET_SUBSYSTEM={target_subsystem}" |
| 238 | f" -DCMAKE_TOOLCHAIN_FILE={cmake_toolchain_file}" |
| 239 | f" -DETHOS_U_NPU_ID={ethos_u_cfg.ethos_u_npu_id}" |
| 240 | f" -DETHOS_U_NPU_CONFIG_ID={ethos_u_cfg.ethos_u_config_id}" |
| 241 | " -DTENSORFLOW_LITE_MICRO_CLEAN_DOWNLOADS=ON" |
Isabella Gottardi | ee4920b | 2022-02-25 14:29:32 +0000 | [diff] [blame] | 242 | ) |
Kshitij Sisodia | f9c19ea | 2021-05-07 16:08:14 +0100 | [diff] [blame] | 243 | |
Alex Tawse | daba3cf | 2023-09-29 15:55:38 +0100 | [diff] [blame] | 244 | run_command(cmake_command, logpipe, fail_message="Failed to configure the project.") |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 245 | |
Alex Tawse | daba3cf | 2023-09-29 15:55:38 +0100 | [diff] [blame] | 246 | make_command = f"{cmake_path} --build {build_dir} -j{args.make_jobs}" |
| 247 | if args.make_verbose: |
Richard Burton | 2bf6a70 | 2023-06-19 14:51:36 +0100 | [diff] [blame] | 248 | make_command += " --verbose" |
Cisco Cervellera | 6ef76bd | 2021-11-25 18:32:27 +0000 | [diff] [blame] | 249 | |
Alex Tawse | daba3cf | 2023-09-29 15:55:38 +0100 | [diff] [blame] | 250 | run_command(make_command, logpipe, fail_message="Failed to build project.") |
Kshitij Sisodia | 1bd434e | 2023-03-17 10:40:45 +0000 | [diff] [blame] | 251 | |
Cisco Cervellera | 6ef76bd | 2021-11-25 18:32:27 +0000 | [diff] [blame] | 252 | logpipe.close() |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 253 | |
| 254 | |
Isabella Gottardi | ee4920b | 2022-02-25 14:29:32 +0000 | [diff] [blame] | 255 | if __name__ == "__main__": |
Cisco Cervellera | 6ef76bd | 2021-11-25 18:32:27 +0000 | [diff] [blame] | 256 | parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter) |
Isabella Gottardi | ee4920b | 2022-02-25 14:29:32 +0000 | [diff] [blame] | 257 | parser.add_argument( |
| 258 | "--toolchain", |
| 259 | default="gnu", |
| 260 | help=""" |
Richard Burton | 1706962 | 2022-03-17 10:54:26 +0000 | [diff] [blame] | 261 | Specify the toolchain to use (Arm or GNU). |
| 262 | Options are [gnu, arm]; default is gnu. |
| 263 | """, |
Isabella Gottardi | ee4920b | 2022-02-25 14:29:32 +0000 | [diff] [blame] | 264 | ) |
| 265 | parser.add_argument( |
| 266 | "--skip-download", |
| 267 | help="Do not download resources: models and test vectors", |
| 268 | action="store_true", |
| 269 | ) |
| 270 | parser.add_argument( |
| 271 | "--skip-vela", |
| 272 | help="Do not run Vela optimizer on downloaded models.", |
| 273 | action="store_true", |
| 274 | ) |
| 275 | parser.add_argument( |
| 276 | "--npu-config-name", |
| 277 | help=f"""Arm Ethos-U configuration to build for. Choose from: |
Richard Burton | 1706962 | 2022-03-17 10:54:26 +0000 | [diff] [blame] | 278 | {valid_npu_config_names}""", |
Isabella Gottardi | ee4920b | 2022-02-25 14:29:32 +0000 | [diff] [blame] | 279 | default=default_npu_config_names[0], |
| 280 | ) |
| 281 | parser.add_argument( |
| 282 | "--make-jobs", |
| 283 | help="Number of jobs to run with make", |
| 284 | default=multiprocessing.cpu_count(), |
| 285 | ) |
| 286 | parser.add_argument( |
| 287 | "--make-verbose", help="Make runs with VERBOSE=1", action="store_true" |
| 288 | ) |
Alex Tawse | daba3cf | 2023-09-29 15:55:38 +0100 | [diff] [blame] | 289 | parsed_args = parser.parse_args() |
Kshitij Sisodia | b9e9c89 | 2021-05-27 13:57:35 +0100 | [diff] [blame] | 290 | |
Isabella Gottardi | ee4920b | 2022-02-25 14:29:32 +0000 | [diff] [blame] | 291 | logging.basicConfig( |
| 292 | filename="log_build_default.log", level=logging.DEBUG, filemode="w" |
| 293 | ) |
Kshitij Sisodia | b9e9c89 | 2021-05-27 13:57:35 +0100 | [diff] [blame] | 294 | logging.getLogger().addHandler(logging.StreamHandler(sys.stdout)) |
| 295 | |
Alex Tawse | daba3cf | 2023-09-29 15:55:38 +0100 | [diff] [blame] | 296 | build_args = BuildArgs( |
| 297 | toolchain=parsed_args.toolchain.lower(), |
| 298 | download_resources=not parsed_args.skip_download, |
| 299 | run_vela_on_models=not parsed_args.skip_vela, |
| 300 | npu_config_name=parsed_args.npu_config_name, |
| 301 | make_jobs=parsed_args.make_jobs, |
| 302 | make_verbose=parsed_args.make_verbose |
Isabella Gottardi | ee4920b | 2022-02-25 14:29:32 +0000 | [diff] [blame] | 303 | ) |
Alex Tawse | daba3cf | 2023-09-29 15:55:38 +0100 | [diff] [blame] | 304 | |
| 305 | run(build_args) |