Alex Tawse | daba3cf | 2023-09-29 15:55:38 +0100 | [diff] [blame] | 1 | # SPDX-FileCopyrightText: Copyright 2021-2023 Arm Limited and/or its affiliates <open-source-office@arm.com> |
alexander | 3c79893 | 2021-03-26 21:42:19 +0000 | [diff] [blame] | 2 | # SPDX-License-Identifier: Apache-2.0 |
| 3 | # |
| 4 | # Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | # you may not use this file except in compliance with the License. |
| 6 | # You may obtain a copy of the License at |
| 7 | # |
| 8 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | # |
| 10 | # Unless required by applicable law or agreed to in writing, software |
| 11 | # distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | # See the License for the specific language governing permissions and |
| 14 | # limitations under the License. |
| 15 | |
| 16 | """ |
| 17 | Utility script to convert a set of pairs of npy files in a given location into |
| 18 | corresponding cpp files and a single hpp file referencing the vectors |
| 19 | from the cpp files. |
| 20 | """ |
alexander | 3c79893 | 2021-03-26 21:42:19 +0000 | [diff] [blame] | 21 | import math |
Alex Tawse | daba3cf | 2023-09-29 15:55:38 +0100 | [diff] [blame] | 22 | import typing |
| 23 | from argparse import ArgumentParser |
| 24 | from dataclasses import dataclass |
Richard Burton | 0055346 | 2021-11-10 16:27:14 +0000 | [diff] [blame] | 25 | from pathlib import Path |
alexander | 3c79893 | 2021-03-26 21:42:19 +0000 | [diff] [blame] | 26 | |
Alex Tawse | daba3cf | 2023-09-29 15:55:38 +0100 | [diff] [blame] | 27 | import numpy as np |
alexander | 3c79893 | 2021-03-26 21:42:19 +0000 | [diff] [blame] | 28 | from jinja2 import Environment, FileSystemLoader |
| 29 | |
Alex Tawse | daba3cf | 2023-09-29 15:55:38 +0100 | [diff] [blame] | 30 | from gen_utils import GenUtils |
alexander | 3c79893 | 2021-03-26 21:42:19 +0000 | [diff] [blame] | 31 | |
Alex Tawse | daba3cf | 2023-09-29 15:55:38 +0100 | [diff] [blame] | 32 | # pylint: disable=duplicate-code |
| 33 | parser = ArgumentParser() |
| 34 | |
| 35 | parser.add_argument( |
| 36 | "--data_folder_path", |
| 37 | type=str, |
| 38 | help="path to ifm-ofm npy folder to convert." |
| 39 | ) |
| 40 | |
| 41 | parser.add_argument( |
| 42 | "--source_folder_path", |
| 43 | type=str, |
| 44 | help="path to source folder to be generated." |
| 45 | ) |
| 46 | |
| 47 | parser.add_argument( |
| 48 | "--header_folder_path", |
| 49 | type=str, |
| 50 | help="path to header folder to be generated." |
| 51 | ) |
| 52 | |
| 53 | parser.add_argument( |
| 54 | "--usecase", |
| 55 | type=str, |
| 56 | default="", |
| 57 | help="Test data file suffix." |
| 58 | ) |
| 59 | |
| 60 | parser.add_argument( |
| 61 | "--namespaces", |
| 62 | action='append', |
| 63 | default=[] |
| 64 | ) |
| 65 | |
| 66 | parser.add_argument( |
| 67 | "--license_template", |
| 68 | type=str, |
| 69 | help="Header template file", |
| 70 | default="header_template.txt" |
| 71 | ) |
| 72 | |
| 73 | parser.add_argument( |
| 74 | "-v", |
| 75 | "--verbosity", |
| 76 | action="store_true" |
| 77 | ) |
| 78 | |
| 79 | parsed_args = parser.parse_args() |
alexander | 3c79893 | 2021-03-26 21:42:19 +0000 | [diff] [blame] | 80 | |
Richard Burton | 1706962 | 2022-03-17 10:54:26 +0000 | [diff] [blame] | 81 | env = Environment(loader=FileSystemLoader(Path(__file__).parent / 'templates'), |
alexander | 3c79893 | 2021-03-26 21:42:19 +0000 | [diff] [blame] | 82 | trim_blocks=True, |
| 83 | lstrip_blocks=True) |
| 84 | |
| 85 | |
Alex Tawse | daba3cf | 2023-09-29 15:55:38 +0100 | [diff] [blame] | 86 | # pylint: enable=duplicate-code |
| 87 | @dataclass |
| 88 | class TestDataParams: |
| 89 | """ |
| 90 | Template params for TestData.hpp + TestData.ccc |
| 91 | """ |
| 92 | ifm_count: int |
| 93 | ofm_count: int |
| 94 | ifm_var_names: typing.List[str] |
| 95 | ifm_var_sizes: typing.List[int] |
| 96 | ofm_var_names: typing.List[str] |
| 97 | ofm_var_sizes: typing.List[int] |
| 98 | data_type: str |
| 99 | |
| 100 | |
| 101 | @dataclass |
| 102 | class IofmParams: |
| 103 | """ |
| 104 | Template params for iofmdata.cc |
| 105 | """ |
| 106 | var_name: str |
| 107 | data_type: str |
| 108 | |
| 109 | |
| 110 | def write_hpp_file( |
| 111 | template_params: TestDataParams, |
| 112 | header_filename: str, |
| 113 | cc_file_path: str, |
| 114 | header_template_file: str |
| 115 | ): |
| 116 | """ |
| 117 | Write TestData.hpp and TestData.cc |
| 118 | |
| 119 | @param template_params: Template parameters |
| 120 | @param header_filename: TestData.hpp path |
| 121 | @param cc_file_path: TestData.cc path |
| 122 | @param header_template_file: Header template file name |
| 123 | """ |
| 124 | header_file_path = Path(parsed_args.header_folder_path) / header_filename |
alexander | 3c79893 | 2021-03-26 21:42:19 +0000 | [diff] [blame] | 125 | |
| 126 | print(f"++ Generating {header_file_path}") |
Alex Tawse | daba3cf | 2023-09-29 15:55:38 +0100 | [diff] [blame] | 127 | hdr = GenUtils.gen_header(env, header_template_file) |
| 128 | env \ |
| 129 | .get_template('TestData.hpp.template') \ |
| 130 | .stream(common_template_header=hdr, |
| 131 | ifm_count=template_params.ifm_count, |
| 132 | ofm_count=template_params.ofm_count, |
| 133 | ifm_var_names=template_params.ifm_var_names, |
| 134 | ifm_var_sizes=template_params.ifm_var_sizes, |
| 135 | ofm_var_names=template_params.ofm_var_names, |
| 136 | ofm_var_sizes=template_params.ofm_var_sizes, |
| 137 | data_type=template_params.data_type, |
| 138 | namespaces=parsed_args.namespaces) \ |
alexander | 3c79893 | 2021-03-26 21:42:19 +0000 | [diff] [blame] | 139 | .dump(str(header_file_path)) |
| 140 | |
Alex Tawse | daba3cf | 2023-09-29 15:55:38 +0100 | [diff] [blame] | 141 | env \ |
| 142 | .get_template('TestData.cc.template') \ |
| 143 | .stream(common_template_header=hdr, |
| 144 | include_h=header_filename, |
| 145 | ifm_var_names=template_params.ifm_var_names, |
| 146 | ofm_var_names=template_params.ofm_var_names, |
| 147 | data_type=template_params.data_type, |
| 148 | namespaces=parsed_args.namespaces) \ |
alexander | 3c79893 | 2021-03-26 21:42:19 +0000 | [diff] [blame] | 149 | .dump(str(cc_file_path)) |
| 150 | |
| 151 | |
Alex Tawse | daba3cf | 2023-09-29 15:55:38 +0100 | [diff] [blame] | 152 | def write_individual_cc_file( |
| 153 | template_params: IofmParams, |
| 154 | header_filename: str, |
| 155 | filename: str, |
| 156 | cc_filename: Path, |
| 157 | header_template_file: str |
| 158 | ): |
| 159 | """ |
| 160 | Write iofmdata.cc |
| 161 | |
| 162 | @param template_params: Template parameters |
| 163 | @param header_filename: Header file name |
| 164 | @param filename: Input file name |
| 165 | @param cc_filename: iofmdata.cc file name |
| 166 | @param header_template_file: Header template file name |
| 167 | """ |
Richard Burton | 1706962 | 2022-03-17 10:54:26 +0000 | [diff] [blame] | 168 | print(f"++ Converting {filename} to {cc_filename.name}") |
Alex Tawse | daba3cf | 2023-09-29 15:55:38 +0100 | [diff] [blame] | 169 | hdr = GenUtils.gen_header(env, header_template_file, filename) |
alexander | 3c79893 | 2021-03-26 21:42:19 +0000 | [diff] [blame] | 170 | |
| 171 | # Convert the image and write it to the cc file |
Alex Tawse | daba3cf | 2023-09-29 15:55:38 +0100 | [diff] [blame] | 172 | fm_data = (np.load(Path(parsed_args.data_folder_path) / filename)).flatten() |
alexander | 3c79893 | 2021-03-26 21:42:19 +0000 | [diff] [blame] | 173 | type(fm_data.dtype) |
| 174 | hex_line_generator = (', '.join(map(hex, sub_arr)) |
| 175 | for sub_arr in np.array_split(fm_data, math.ceil(len(fm_data) / 20))) |
| 176 | |
Alex Tawse | daba3cf | 2023-09-29 15:55:38 +0100 | [diff] [blame] | 177 | env \ |
| 178 | .get_template('iofmdata.cc.template') \ |
| 179 | .stream(common_template_header=hdr, |
| 180 | include_h=header_filename, |
| 181 | var_name=template_params.var_name, |
| 182 | fm_data=hex_line_generator, |
| 183 | data_type=template_params.data_type, |
| 184 | namespaces=parsed_args.namespaces) \ |
alexander | 3c79893 | 2021-03-26 21:42:19 +0000 | [diff] [blame] | 185 | .dump(str(cc_filename)) |
| 186 | |
| 187 | |
| 188 | def get_npy_vec_size(filename: str) -> int: |
| 189 | """ |
| 190 | Gets the size of the array in the npy file |
| 191 | Args: |
| 192 | filename: npy file path. |
| 193 | Return: |
| 194 | size in bytes |
| 195 | """ |
Alex Tawse | daba3cf | 2023-09-29 15:55:38 +0100 | [diff] [blame] | 196 | data = np.load(Path(parsed_args.data_folder_path) / filename) |
Richard Burton | 1706962 | 2022-03-17 10:54:26 +0000 | [diff] [blame] | 197 | return data.size * data.dtype.itemsize |
alexander | 3c79893 | 2021-03-26 21:42:19 +0000 | [diff] [blame] | 198 | |
| 199 | |
Alex Tawse | daba3cf | 2023-09-29 15:55:38 +0100 | [diff] [blame] | 200 | def write_cc_files(args, count, iofm_data_type, add_usecase_fname, prefix): |
| 201 | """ |
| 202 | Write all cc files |
alexander | 3c79893 | 2021-03-26 21:42:19 +0000 | [diff] [blame] | 203 | |
Alex Tawse | daba3cf | 2023-09-29 15:55:38 +0100 | [diff] [blame] | 204 | @param args: User-provided args |
| 205 | @param count: File count |
| 206 | @param iofm_data_type: Data type |
| 207 | @param add_usecase_fname: Use case suffix |
| 208 | @param prefix: Prefix (ifm/ofm) |
| 209 | @return: Names and sizes of generated C++ arrays |
| 210 | """ |
| 211 | array_names = [] |
| 212 | sizes = [] |
| 213 | |
| 214 | header_filename = get_header_filename(add_usecase_fname) |
| 215 | |
| 216 | # In the data_folder_path there should be pairs of ifm-ofm |
| 217 | # It's assumed the ifm-ofm naming convention: ifm0.npy-ofm0.npy, ifm1.npy-ofm1.npy |
| 218 | # count = int(len(list(Path(args.data_folder_path).glob(f'{prefix}*.npy')))) |
| 219 | |
| 220 | for idx in range(count): |
| 221 | # Save the fm cc file |
| 222 | base_name = prefix + str(idx) |
| 223 | filename = base_name + ".npy" |
| 224 | array_name = base_name + add_usecase_fname |
| 225 | cc_filename = Path(args.source_folder_path) / (array_name + ".cc") |
| 226 | array_names.append(array_name) |
| 227 | |
| 228 | template_params = IofmParams( |
| 229 | var_name=array_name, |
| 230 | data_type=iofm_data_type, |
| 231 | ) |
| 232 | |
| 233 | write_individual_cc_file( |
| 234 | template_params, header_filename, filename, cc_filename, args.license_template |
| 235 | ) |
| 236 | sizes.append(get_npy_vec_size(filename)) |
| 237 | |
| 238 | return array_names, sizes |
| 239 | |
| 240 | |
| 241 | def get_header_filename(use_case_filename): |
| 242 | """ |
| 243 | Get the header file name from the use case file name |
| 244 | |
| 245 | @param use_case_filename: The use case file name |
| 246 | @return: The header file name |
| 247 | """ |
| 248 | return "TestData" + use_case_filename + ".hpp" |
| 249 | |
| 250 | |
| 251 | def get_cc_filename(use_case_filename): |
| 252 | """ |
| 253 | Get the cc file name from the use case file name |
| 254 | |
| 255 | @param use_case_filename: The use case file name |
| 256 | @return: The cc file name |
| 257 | """ |
| 258 | return "TestData" + use_case_filename + ".cc" |
| 259 | |
| 260 | |
| 261 | def main(args): |
| 262 | """ |
| 263 | Generate test data |
| 264 | @param args: Parsed args |
| 265 | """ |
Kshitij Sisodia | b178b28 | 2022-01-04 13:37:53 +0000 | [diff] [blame] | 266 | add_usecase_fname = ("_" + args.usecase) if (args.usecase != "") else "" |
Alex Tawse | daba3cf | 2023-09-29 15:55:38 +0100 | [diff] [blame] | 267 | header_filename = get_header_filename(add_usecase_fname) |
| 268 | common_cc_filename = get_cc_filename(add_usecase_fname) |
alexander | 3c79893 | 2021-03-26 21:42:19 +0000 | [diff] [blame] | 269 | |
| 270 | # In the data_folder_path there should be pairs of ifm-ofm |
Richard Burton | 0055346 | 2021-11-10 16:27:14 +0000 | [diff] [blame] | 271 | # It's assumed the ifm-ofm naming convention: ifm0.npy-ofm0.npy, ifm1.npy-ofm1.npy |
| 272 | ifms_count = int(len(list(Path(args.data_folder_path).glob('ifm*.npy')))) |
| 273 | ofms_count = int(len(list(Path(args.data_folder_path).glob('ofm*.npy')))) |
| 274 | |
alexander | 3c79893 | 2021-03-26 21:42:19 +0000 | [diff] [blame] | 275 | iofm_data_type = "int8_t" |
Richard Burton | 0055346 | 2021-11-10 16:27:14 +0000 | [diff] [blame] | 276 | if ifms_count > 0: |
Alex Tawse | daba3cf | 2023-09-29 15:55:38 +0100 | [diff] [blame] | 277 | iofm_data_type = "int8_t" \ |
| 278 | if (np.load(str(Path(args.data_folder_path) / "ifm0.npy")).dtype == np.int8) \ |
| 279 | else "uint8_t" |
alexander | 3c79893 | 2021-03-26 21:42:19 +0000 | [diff] [blame] | 280 | |
Alex Tawse | daba3cf | 2023-09-29 15:55:38 +0100 | [diff] [blame] | 281 | ifm_array_names, ifm_sizes = write_cc_files( |
| 282 | args, ifms_count, iofm_data_type, add_usecase_fname, prefix="ifm" |
| 283 | ) |
alexander | 3c79893 | 2021-03-26 21:42:19 +0000 | [diff] [blame] | 284 | |
Alex Tawse | daba3cf | 2023-09-29 15:55:38 +0100 | [diff] [blame] | 285 | ofm_array_names, ofm_sizes = write_cc_files( |
| 286 | args, ofms_count, iofm_data_type, add_usecase_fname, prefix="ofm" |
| 287 | ) |
alexander | 3c79893 | 2021-03-26 21:42:19 +0000 | [diff] [blame] | 288 | |
Richard Burton | 1706962 | 2022-03-17 10:54:26 +0000 | [diff] [blame] | 289 | common_cc_filepath = Path(args.source_folder_path) / common_cc_filename |
Alex Tawse | daba3cf | 2023-09-29 15:55:38 +0100 | [diff] [blame] | 290 | |
| 291 | template_params = TestDataParams( |
| 292 | ifm_count=ifms_count, |
| 293 | ofm_count=ofms_count, |
| 294 | ifm_var_names=ifm_array_names, |
| 295 | ifm_var_sizes=ifm_sizes, |
| 296 | ofm_var_names=ofm_array_names, |
| 297 | ofm_var_sizes=ofm_sizes, |
| 298 | data_type=iofm_data_type, |
| 299 | ) |
| 300 | |
| 301 | write_hpp_file( |
| 302 | template_params, header_filename, common_cc_filepath, args.license_template |
| 303 | ) |
alexander | 3c79893 | 2021-03-26 21:42:19 +0000 | [diff] [blame] | 304 | |
| 305 | |
| 306 | if __name__ == '__main__': |
Alex Tawse | daba3cf | 2023-09-29 15:55:38 +0100 | [diff] [blame] | 307 | if parsed_args.verbosity: |
| 308 | print("Running gen_test_data_cpp with args: " + str(parsed_args)) |
| 309 | main(parsed_args) |