MLECO-3995: Pylint + Shellcheck compatibility
* All Python scripts updated to abide by Pylint rules
* good-names updated to permit short variable names:
i, j, k, f, g, ex
* ignore-long-lines regex updated to allow long lines
for licence headers
* Shell scripts now compliant with Shellcheck
Signed-off-by: Alex Tawse <Alex.Tawse@arm.com>
Change-Id: I8d5af8279bc08bb8acfe8f6ee7df34965552bbe5
diff --git a/scripts/py/gen_test_data_cpp.py b/scripts/py/gen_test_data_cpp.py
index a9e2b75..1ee55ff 100644
--- a/scripts/py/gen_test_data_cpp.py
+++ b/scripts/py/gen_test_data_cpp.py
@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: Copyright 2021 - 2022 Arm Limited and/or its affiliates <open-source-office@arm.com>
+# SPDX-FileCopyrightText: Copyright 2021-2023 Arm Limited and/or its affiliates <open-source-office@arm.com>
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
@@ -18,81 +18,170 @@
corresponding cpp files and a single hpp file referencing the vectors
from the cpp files.
"""
-import datetime
import math
-import os
-import numpy as np
+import typing
+from argparse import ArgumentParser
+from dataclasses import dataclass
from pathlib import Path
-from argparse import ArgumentParser
+import numpy as np
from jinja2 import Environment, FileSystemLoader
-parser = ArgumentParser()
-parser.add_argument("--data_folder_path", type=str, help="path to ifm-ofm npy folder to convert.")
-parser.add_argument("--source_folder_path", type=str, help="path to source folder to be generated.")
-parser.add_argument("--header_folder_path", type=str, help="path to header folder to be generated.")
-parser.add_argument("--usecase", type=str, default="", help="Test data file suffix.")
-parser.add_argument("--namespaces", action='append', default=[])
-parser.add_argument("--license_template", type=str, help="Header template file",
- default="header_template.txt")
-parser.add_argument("-v", "--verbosity", action="store_true")
+from gen_utils import GenUtils
-args = parser.parse_args()
+# pylint: disable=duplicate-code
+parser = ArgumentParser()
+
+parser.add_argument(
+ "--data_folder_path",
+ type=str,
+ help="path to ifm-ofm npy folder to convert."
+)
+
+parser.add_argument(
+ "--source_folder_path",
+ type=str,
+ help="path to source folder to be generated."
+)
+
+parser.add_argument(
+ "--header_folder_path",
+ type=str,
+ help="path to header folder to be generated."
+)
+
+parser.add_argument(
+ "--usecase",
+ type=str,
+ default="",
+ help="Test data file suffix."
+)
+
+parser.add_argument(
+ "--namespaces",
+ action='append',
+ default=[]
+)
+
+parser.add_argument(
+ "--license_template",
+ type=str,
+ help="Header template file",
+ default="header_template.txt"
+)
+
+parser.add_argument(
+ "-v",
+ "--verbosity",
+ action="store_true"
+)
+
+parsed_args = parser.parse_args()
env = Environment(loader=FileSystemLoader(Path(__file__).parent / 'templates'),
trim_blocks=True,
lstrip_blocks=True)
-def write_hpp_file(header_filename, cc_file_path, header_template_file, num_ifms, num_ofms,
- ifm_array_names, ifm_sizes, ofm_array_names, ofm_sizes, iofm_data_type):
- header_file_path = Path(args.header_folder_path) / header_filename
+# pylint: enable=duplicate-code
+@dataclass
+class TestDataParams:
+ """
+ Template params for TestData.hpp + TestData.ccc
+ """
+ ifm_count: int
+ ofm_count: int
+ ifm_var_names: typing.List[str]
+ ifm_var_sizes: typing.List[int]
+ ofm_var_names: typing.List[str]
+ ofm_var_sizes: typing.List[int]
+ data_type: str
+
+
+@dataclass
+class IofmParams:
+ """
+ Template params for iofmdata.cc
+ """
+ var_name: str
+ data_type: str
+
+
+def write_hpp_file(
+ template_params: TestDataParams,
+ header_filename: str,
+ cc_file_path: str,
+ header_template_file: str
+):
+ """
+ Write TestData.hpp and TestData.cc
+
+ @param template_params: Template parameters
+ @param header_filename: TestData.hpp path
+ @param cc_file_path: TestData.cc path
+ @param header_template_file: Header template file name
+ """
+ header_file_path = Path(parsed_args.header_folder_path) / header_filename
print(f"++ Generating {header_file_path}")
- header_template = env.get_template(header_template_file)
- hdr = header_template.render(script_name=Path(__file__).name,
- gen_time=datetime.datetime.now(),
- year=datetime.datetime.now().year)
- env.get_template('TestData.hpp.template').stream(common_template_header=hdr,
- ifm_count=num_ifms,
- ofm_count=num_ofms,
- ifm_var_names=ifm_array_names,
- ifm_var_sizes=ifm_sizes,
- ofm_var_names=ofm_array_names,
- ofm_var_sizes=ofm_sizes,
- data_type=iofm_data_type,
- namespaces=args.namespaces) \
+ hdr = GenUtils.gen_header(env, header_template_file)
+ env \
+ .get_template('TestData.hpp.template') \
+ .stream(common_template_header=hdr,
+ ifm_count=template_params.ifm_count,
+ ofm_count=template_params.ofm_count,
+ ifm_var_names=template_params.ifm_var_names,
+ ifm_var_sizes=template_params.ifm_var_sizes,
+ ofm_var_names=template_params.ofm_var_names,
+ ofm_var_sizes=template_params.ofm_var_sizes,
+ data_type=template_params.data_type,
+ namespaces=parsed_args.namespaces) \
.dump(str(header_file_path))
- env.get_template('TestData.cc.template').stream(common_template_header=hdr,
- include_h=header_filename,
- ifm_var_names=ifm_array_names,
- ofm_var_names=ofm_array_names,
- data_type=iofm_data_type,
- namespaces=args.namespaces) \
+ env \
+ .get_template('TestData.cc.template') \
+ .stream(common_template_header=hdr,
+ include_h=header_filename,
+ ifm_var_names=template_params.ifm_var_names,
+ ofm_var_names=template_params.ofm_var_names,
+ data_type=template_params.data_type,
+ namespaces=parsed_args.namespaces) \
.dump(str(cc_file_path))
-def write_individual_cc_file(filename, cc_filename, header_filename, header_template_file, array_name, iofm_data_type):
+def write_individual_cc_file(
+ template_params: IofmParams,
+ header_filename: str,
+ filename: str,
+ cc_filename: Path,
+ header_template_file: str
+):
+ """
+ Write iofmdata.cc
+
+ @param template_params: Template parameters
+ @param header_filename: Header file name
+ @param filename: Input file name
+ @param cc_filename: iofmdata.cc file name
+ @param header_template_file: Header template file name
+ """
print(f"++ Converting {filename} to {cc_filename.name}")
- header_template = env.get_template(header_template_file)
- hdr = header_template.render(script_name=Path(__file__).name,
- gen_time=datetime.datetime.now(),
- file_name=filename,
- year=datetime.datetime.now().year)
+ hdr = GenUtils.gen_header(env, header_template_file, filename)
# Convert the image and write it to the cc file
- fm_data = (np.load(Path(args.data_folder_path) / filename)).flatten()
+ fm_data = (np.load(Path(parsed_args.data_folder_path) / filename)).flatten()
type(fm_data.dtype)
hex_line_generator = (', '.join(map(hex, sub_arr))
for sub_arr in np.array_split(fm_data, math.ceil(len(fm_data) / 20)))
- env.get_template('iofmdata.cc.template').stream(common_template_header=hdr,
- include_h=header_filename,
- var_name=array_name,
- fm_data=hex_line_generator,
- data_type=iofm_data_type,
- namespaces=args.namespaces) \
+ env \
+ .get_template('iofmdata.cc.template') \
+ .stream(common_template_header=hdr,
+ include_h=header_filename,
+ var_name=template_params.var_name,
+ fm_data=hex_line_generator,
+ data_type=template_params.data_type,
+ namespaces=parsed_args.namespaces) \
.dump(str(cc_filename))
@@ -104,59 +193,117 @@
Return:
size in bytes
"""
- data = np.load(Path(args.data_folder_path) / filename)
+ data = np.load(Path(parsed_args.data_folder_path) / filename)
return data.size * data.dtype.itemsize
-def main(args):
- # Keep the count of the images converted
- ifm_array_names = []
- ofm_array_names = []
+def write_cc_files(args, count, iofm_data_type, add_usecase_fname, prefix):
+ """
+ Write all cc files
+ @param args: User-provided args
+ @param count: File count
+ @param iofm_data_type: Data type
+ @param add_usecase_fname: Use case suffix
+ @param prefix: Prefix (ifm/ofm)
+ @return: Names and sizes of generated C++ arrays
+ """
+ array_names = []
+ sizes = []
+
+ header_filename = get_header_filename(add_usecase_fname)
+
+ # In the data_folder_path there should be pairs of ifm-ofm
+ # It's assumed the ifm-ofm naming convention: ifm0.npy-ofm0.npy, ifm1.npy-ofm1.npy
+ # count = int(len(list(Path(args.data_folder_path).glob(f'{prefix}*.npy'))))
+
+ for idx in range(count):
+ # Save the fm cc file
+ base_name = prefix + str(idx)
+ filename = base_name + ".npy"
+ array_name = base_name + add_usecase_fname
+ cc_filename = Path(args.source_folder_path) / (array_name + ".cc")
+ array_names.append(array_name)
+
+ template_params = IofmParams(
+ var_name=array_name,
+ data_type=iofm_data_type,
+ )
+
+ write_individual_cc_file(
+ template_params, header_filename, filename, cc_filename, args.license_template
+ )
+ sizes.append(get_npy_vec_size(filename))
+
+ return array_names, sizes
+
+
+def get_header_filename(use_case_filename):
+ """
+ Get the header file name from the use case file name
+
+ @param use_case_filename: The use case file name
+ @return: The header file name
+ """
+ return "TestData" + use_case_filename + ".hpp"
+
+
+def get_cc_filename(use_case_filename):
+ """
+ Get the cc file name from the use case file name
+
+ @param use_case_filename: The use case file name
+ @return: The cc file name
+ """
+ return "TestData" + use_case_filename + ".cc"
+
+
+def main(args):
+ """
+ Generate test data
+ @param args: Parsed args
+ """
add_usecase_fname = ("_" + args.usecase) if (args.usecase != "") else ""
- header_filename = "TestData" + add_usecase_fname + ".hpp"
- common_cc_filename = "TestData" + add_usecase_fname + ".cc"
+ header_filename = get_header_filename(add_usecase_fname)
+ common_cc_filename = get_cc_filename(add_usecase_fname)
# In the data_folder_path there should be pairs of ifm-ofm
# It's assumed the ifm-ofm naming convention: ifm0.npy-ofm0.npy, ifm1.npy-ofm1.npy
ifms_count = int(len(list(Path(args.data_folder_path).glob('ifm*.npy'))))
ofms_count = int(len(list(Path(args.data_folder_path).glob('ofm*.npy'))))
- #i_ofms_count = int(len([name for name in os.listdir(os.path.join(args.data_folder_path)) if name.lower().endswith('.npy')]) / 2)
-
iofm_data_type = "int8_t"
if ifms_count > 0:
- iofm_data_type = "int8_t" if (np.load(Path(args.data_folder_path) / "ifm0.npy").dtype == np.int8) else "uint8_t"
+ iofm_data_type = "int8_t" \
+ if (np.load(str(Path(args.data_folder_path) / "ifm0.npy")).dtype == np.int8) \
+ else "uint8_t"
- ifm_sizes = []
- ofm_sizes = []
+ ifm_array_names, ifm_sizes = write_cc_files(
+ args, ifms_count, iofm_data_type, add_usecase_fname, prefix="ifm"
+ )
- for idx in range(ifms_count):
- # Save the fm cc file
- base_name = "ifm" + str(idx)
- filename = base_name+".npy"
- array_name = base_name + add_usecase_fname
- cc_filename = Path(args.source_folder_path) / (array_name + ".cc")
- ifm_array_names.append(array_name)
- write_individual_cc_file(filename, cc_filename, header_filename, args.license_template, array_name, iofm_data_type)
- ifm_sizes.append(get_npy_vec_size(filename))
-
- for idx in range(ofms_count):
- # Save the fm cc file
- base_name = "ofm" + str(idx)
- filename = base_name+".npy"
- array_name = base_name + add_usecase_fname
- cc_filename = Path(args.source_folder_path) / (array_name + ".cc")
- ofm_array_names.append(array_name)
- write_individual_cc_file(filename, cc_filename, header_filename, args.license_template, array_name, iofm_data_type)
- ofm_sizes.append(get_npy_vec_size(filename))
+ ofm_array_names, ofm_sizes = write_cc_files(
+ args, ofms_count, iofm_data_type, add_usecase_fname, prefix="ofm"
+ )
common_cc_filepath = Path(args.source_folder_path) / common_cc_filename
- write_hpp_file(header_filename, common_cc_filepath, args.license_template,
- ifms_count, ofms_count, ifm_array_names, ifm_sizes, ofm_array_names, ofm_sizes, iofm_data_type)
+
+ template_params = TestDataParams(
+ ifm_count=ifms_count,
+ ofm_count=ofms_count,
+ ifm_var_names=ifm_array_names,
+ ifm_var_sizes=ifm_sizes,
+ ofm_var_names=ofm_array_names,
+ ofm_var_sizes=ofm_sizes,
+ data_type=iofm_data_type,
+ )
+
+ write_hpp_file(
+ template_params, header_filename, common_cc_filepath, args.license_template
+ )
if __name__ == '__main__':
- if args.verbosity:
- print("Running gen_test_data_cpp with args: "+str(args))
- main(args)
+ if parsed_args.verbosity:
+ print("Running gen_test_data_cpp with args: " + str(parsed_args))
+ main(parsed_args)