alexander | f4e2c47 | 2021-05-14 13:14:21 +0100 | [diff] [blame] | 1 | #!/usr/bin/env python3 |
Isabella Gottardi | ee4920b | 2022-02-25 14:29:32 +0000 | [diff] [blame] | 2 | # Copyright (c) 2021-2022 Arm Limited. All rights reserved. |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 3 | # SPDX-License-Identifier: Apache-2.0 |
| 4 | # |
| 5 | # Licensed under the Apache License, Version 2.0 (the "License"); |
| 6 | # you may not use this file except in compliance with the License. |
| 7 | # You may obtain a copy of the License at |
| 8 | # |
| 9 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | # |
| 11 | # Unless required by applicable law or agreed to in writing, software |
| 12 | # distributed under the License is distributed on an "AS IS" BASIS, |
| 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 14 | # See the License for the specific language governing permissions and |
| 15 | # limitations under the License. |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 16 | import logging |
Isabella Gottardi | ee4920b | 2022-02-25 14:29:32 +0000 | [diff] [blame] | 17 | import multiprocessing |
| 18 | import os |
| 19 | import shutil |
| 20 | import subprocess |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 21 | import sys |
Isabella Gottardi | ee4920b | 2022-02-25 14:29:32 +0000 | [diff] [blame] | 22 | import threading |
| 23 | from argparse import ArgumentDefaultsHelpFormatter |
| 24 | from argparse import ArgumentParser |
Richard Burton | 1706962 | 2022-03-17 10:54:26 +0000 | [diff] [blame] | 25 | from pathlib import Path |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 26 | |
Isabella Gottardi | ee4920b | 2022-02-25 14:29:32 +0000 | [diff] [blame] | 27 | from set_up_default_resources import default_npu_config_names |
| 28 | from set_up_default_resources import get_default_npu_config_from_name |
| 29 | from set_up_default_resources import set_up_resources |
| 30 | from set_up_default_resources import valid_npu_config_names |
| 31 | |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 32 | |
Cisco Cervellera | 6ef76bd | 2021-11-25 18:32:27 +0000 | [diff] [blame] | 33 | class PipeLogging(threading.Thread): |
Cisco Cervellera | 6ef76bd | 2021-11-25 18:32:27 +0000 | [diff] [blame] | 34 | def __init__(self, log_level): |
| 35 | threading.Thread.__init__(self) |
| 36 | self.logLevel = log_level |
| 37 | self.fileRead, self.fileWrite = os.pipe() |
| 38 | self.pipeIn = os.fdopen(self.fileRead) |
| 39 | self.daemon = False |
| 40 | self.start() |
| 41 | |
| 42 | def fileno(self): |
| 43 | return self.fileWrite |
| 44 | |
| 45 | def run(self): |
Isabella Gottardi | ee4920b | 2022-02-25 14:29:32 +0000 | [diff] [blame] | 46 | for line in iter(self.pipeIn.readline, ""): |
| 47 | logging.log(self.logLevel, line.strip("\n")) |
Cisco Cervellera | 6ef76bd | 2021-11-25 18:32:27 +0000 | [diff] [blame] | 48 | |
| 49 | self.pipeIn.close() |
| 50 | |
| 51 | def close(self): |
| 52 | os.close(self.fileWrite) |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 53 | |
Isabella Gottardi | ee4920b | 2022-02-25 14:29:32 +0000 | [diff] [blame] | 54 | |
| 55 | def run( |
| 56 | toolchain: str, |
| 57 | download_resources: bool, |
| 58 | run_vela_on_models: bool, |
| 59 | npu_config_name: str, |
| 60 | make_jobs: int, |
| 61 | make_verbose: bool, |
| 62 | ): |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 63 | """ |
| 64 | Run the helpers scripts. |
| 65 | |
| 66 | Parameters: |
| 67 | ---------- |
Kshitij Sisodia | f9c19ea | 2021-05-07 16:08:14 +0100 | [diff] [blame] | 68 | toolchain (str) : Specifies if 'gnu' or 'arm' toolchain needs to be used. |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 69 | download_resources (bool): Specifies if 'Download resources' step is performed. |
| 70 | run_vela_on_models (bool): Only if `download_resources` is True, specifies if run vela on downloaded models. |
Kshitij Sisodia | 3be2623 | 2021-10-29 12:29:06 +0100 | [diff] [blame] | 71 | npu_config_name(str) : Ethos-U NPU configuration name. See "valid_npu_config_names" |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 72 | """ |
| 73 | |
Richard Burton | 1706962 | 2022-03-17 10:54:26 +0000 | [diff] [blame] | 74 | current_file_dir = Path(__file__).parent.resolve() |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 75 | |
Kshitij Sisodia | f9c19ea | 2021-05-07 16:08:14 +0100 | [diff] [blame] | 76 | # 1. Make sure the toolchain is supported, and set the right one here |
| 77 | supported_toolchain_ids = ["gnu", "arm"] |
Isabella Gottardi | ee4920b | 2022-02-25 14:29:32 +0000 | [diff] [blame] | 78 | assert ( |
| 79 | toolchain in supported_toolchain_ids |
| 80 | ), f"Toolchain must be from {supported_toolchain_ids}" |
Kshitij Sisodia | f9c19ea | 2021-05-07 16:08:14 +0100 | [diff] [blame] | 81 | if toolchain == "arm": |
| 82 | toolchain_file_name = "bare-metal-armclang.cmake" |
| 83 | elif toolchain == "gnu": |
| 84 | toolchain_file_name = "bare-metal-gcc.cmake" |
| 85 | |
| 86 | # 2. Download models if specified |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 87 | if download_resources is True: |
| 88 | logging.info("Downloading resources.") |
Kshitij Sisodia | 9c6f9f8 | 2022-05-20 14:30:02 +0100 | [diff] [blame] | 89 | (download_dir, env_path) = set_up_resources( |
Isabella Gottardi | ee4920b | 2022-02-25 14:29:32 +0000 | [diff] [blame] | 90 | run_vela_on_models=run_vela_on_models, |
| 91 | additional_npu_config_names=[npu_config_name], |
Richard Burton | 1706962 | 2022-03-17 10:54:26 +0000 | [diff] [blame] | 92 | additional_requirements_file=current_file_dir / "scripts" / "py" / "requirements.txt" |
Isabella Gottardi | ee4920b | 2022-02-25 14:29:32 +0000 | [diff] [blame] | 93 | ) |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 94 | |
Kshitij Sisodia | f9c19ea | 2021-05-07 16:08:14 +0100 | [diff] [blame] | 95 | # 3. Build default configuration |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 96 | logging.info("Building default configuration.") |
| 97 | target_platform = "mps3" |
| 98 | target_subsystem = "sse-300" |
Kshitij Sisodia | 3be2623 | 2021-10-29 12:29:06 +0100 | [diff] [blame] | 99 | ethos_u_cfg = get_default_npu_config_from_name(npu_config_name) |
Richard Burton | 1706962 | 2022-03-17 10:54:26 +0000 | [diff] [blame] | 100 | build_dir = current_file_dir / f"cmake-build-{target_platform}-{target_subsystem}-{npu_config_name}-{toolchain}" |
| 101 | |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 102 | try: |
Richard Burton | 1706962 | 2022-03-17 10:54:26 +0000 | [diff] [blame] | 103 | build_dir.mkdir() |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 104 | except FileExistsError: |
Richard Burton | 1706962 | 2022-03-17 10:54:26 +0000 | [diff] [blame] | 105 | # Directory already exists, clean it. |
| 106 | for filepath in build_dir.iterdir(): |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 107 | try: |
Richard Burton | 1706962 | 2022-03-17 10:54:26 +0000 | [diff] [blame] | 108 | if filepath.is_file() or filepath.is_symlink(): |
| 109 | filepath.unlink() |
| 110 | elif filepath.is_dir(): |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 111 | shutil.rmtree(filepath) |
| 112 | except Exception as e: |
Isabella Gottardi | ee4920b | 2022-02-25 14:29:32 +0000 | [diff] [blame] | 113 | logging.error(f"Failed to delete {filepath}. Reason: {e}") |
Kshitij Sisodia | f9c19ea | 2021-05-07 16:08:14 +0100 | [diff] [blame] | 114 | |
Cisco Cervellera | 6ef76bd | 2021-11-25 18:32:27 +0000 | [diff] [blame] | 115 | logpipe = PipeLogging(logging.INFO) |
| 116 | |
Richard Burton | 1706962 | 2022-03-17 10:54:26 +0000 | [diff] [blame] | 117 | cmake_toolchain_file = current_file_dir / "scripts" / "cmake" / "toolchains" / toolchain_file_name |
Kshitij Sisodia | 9c6f9f8 | 2022-05-20 14:30:02 +0100 | [diff] [blame] | 118 | cmake_path = env_path / "bin" / "cmake" |
Isabella Gottardi | ee4920b | 2022-02-25 14:29:32 +0000 | [diff] [blame] | 119 | cmake_command = ( |
Kshitij Sisodia | 9c6f9f8 | 2022-05-20 14:30:02 +0100 | [diff] [blame] | 120 | f"{cmake_path} -B {build_dir} -DTARGET_PLATFORM={target_platform}" |
Isabella Gottardi | ee4920b | 2022-02-25 14:29:32 +0000 | [diff] [blame] | 121 | + f" -DTARGET_SUBSYSTEM={target_subsystem}" |
| 122 | + f" -DCMAKE_TOOLCHAIN_FILE={cmake_toolchain_file}" |
| 123 | + f" -DETHOS_U_NPU_ID={ethos_u_cfg.ethos_u_npu_id}" |
| 124 | + f" -DETHOS_U_NPU_CONFIG_ID={ethos_u_cfg.ethos_u_config_id}" |
| 125 | ) |
Kshitij Sisodia | f9c19ea | 2021-05-07 16:08:14 +0100 | [diff] [blame] | 126 | |
Kshitij Sisodia | 9c6f9f8 | 2022-05-20 14:30:02 +0100 | [diff] [blame] | 127 | logging.info(f"\n\n\n{cmake_command}\n\n\n") |
Isabella Gottardi | ee4920b | 2022-02-25 14:29:32 +0000 | [diff] [blame] | 128 | state = subprocess.run( |
| 129 | cmake_command, shell=True, stdout=logpipe, stderr=subprocess.STDOUT |
| 130 | ) |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 131 | |
Kshitij Sisodia | 9c6f9f8 | 2022-05-20 14:30:02 +0100 | [diff] [blame] | 132 | make_command = f"{cmake_path} --build {build_dir} -j{make_jobs}" |
Isabella Gottardi | ee4920b | 2022-02-25 14:29:32 +0000 | [diff] [blame] | 133 | if make_verbose: |
Kshitij Sisodia | 9c6f9f8 | 2022-05-20 14:30:02 +0100 | [diff] [blame] | 134 | make_command += "--verbose" |
| 135 | logging.info(f"\n\n\n{make_command}\n\n\n") |
Isabella Gottardi | ee4920b | 2022-02-25 14:29:32 +0000 | [diff] [blame] | 136 | state = subprocess.run( |
| 137 | make_command, shell=True, stdout=logpipe, stderr=subprocess.STDOUT |
| 138 | ) |
Cisco Cervellera | 6ef76bd | 2021-11-25 18:32:27 +0000 | [diff] [blame] | 139 | |
| 140 | logpipe.close() |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 141 | |
| 142 | |
Isabella Gottardi | ee4920b | 2022-02-25 14:29:32 +0000 | [diff] [blame] | 143 | if __name__ == "__main__": |
Cisco Cervellera | 6ef76bd | 2021-11-25 18:32:27 +0000 | [diff] [blame] | 144 | parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter) |
Isabella Gottardi | ee4920b | 2022-02-25 14:29:32 +0000 | [diff] [blame] | 145 | parser.add_argument( |
| 146 | "--toolchain", |
| 147 | default="gnu", |
| 148 | help=""" |
Richard Burton | 1706962 | 2022-03-17 10:54:26 +0000 | [diff] [blame] | 149 | Specify the toolchain to use (Arm or GNU). |
| 150 | Options are [gnu, arm]; default is gnu. |
| 151 | """, |
Isabella Gottardi | ee4920b | 2022-02-25 14:29:32 +0000 | [diff] [blame] | 152 | ) |
| 153 | parser.add_argument( |
| 154 | "--skip-download", |
| 155 | help="Do not download resources: models and test vectors", |
| 156 | action="store_true", |
| 157 | ) |
| 158 | parser.add_argument( |
| 159 | "--skip-vela", |
| 160 | help="Do not run Vela optimizer on downloaded models.", |
| 161 | action="store_true", |
| 162 | ) |
| 163 | parser.add_argument( |
| 164 | "--npu-config-name", |
| 165 | help=f"""Arm Ethos-U configuration to build for. Choose from: |
Richard Burton | 1706962 | 2022-03-17 10:54:26 +0000 | [diff] [blame] | 166 | {valid_npu_config_names}""", |
Isabella Gottardi | ee4920b | 2022-02-25 14:29:32 +0000 | [diff] [blame] | 167 | default=default_npu_config_names[0], |
| 168 | ) |
| 169 | parser.add_argument( |
| 170 | "--make-jobs", |
| 171 | help="Number of jobs to run with make", |
| 172 | default=multiprocessing.cpu_count(), |
| 173 | ) |
| 174 | parser.add_argument( |
| 175 | "--make-verbose", help="Make runs with VERBOSE=1", action="store_true" |
| 176 | ) |
Isabella Gottardi | 2181d0a | 2021-04-07 09:27:38 +0100 | [diff] [blame] | 177 | args = parser.parse_args() |
Kshitij Sisodia | b9e9c89 | 2021-05-27 13:57:35 +0100 | [diff] [blame] | 178 | |
Isabella Gottardi | ee4920b | 2022-02-25 14:29:32 +0000 | [diff] [blame] | 179 | logging.basicConfig( |
| 180 | filename="log_build_default.log", level=logging.DEBUG, filemode="w" |
| 181 | ) |
Kshitij Sisodia | b9e9c89 | 2021-05-27 13:57:35 +0100 | [diff] [blame] | 182 | logging.getLogger().addHandler(logging.StreamHandler(sys.stdout)) |
| 183 | |
Isabella Gottardi | ee4920b | 2022-02-25 14:29:32 +0000 | [diff] [blame] | 184 | run( |
| 185 | args.toolchain.lower(), |
Kshitij Sisodia | 3be2623 | 2021-10-29 12:29:06 +0100 | [diff] [blame] | 186 | not args.skip_download, |
| 187 | not args.skip_vela, |
Cisco Cervellera | 6ef76bd | 2021-11-25 18:32:27 +0000 | [diff] [blame] | 188 | args.npu_config_name, |
| 189 | args.make_jobs, |
Isabella Gottardi | ee4920b | 2022-02-25 14:29:32 +0000 | [diff] [blame] | 190 | args.make_verbose, |
| 191 | ) |