| #!/usr/bin/env python3 |
| |
| # Copyright (c) 2023-2024 Arm Limited. |
| # |
| # SPDX-License-Identifier: MIT |
| # |
| # Permission is hereby granted, free of charge, to any person obtaining a copy |
| # of this software and associated documentation files (the "Software"), to |
| # deal in the Software without restriction, including without limitation the |
| # rights to use, copy, modify, merge, publish, distribute, sublicense, and/or |
| # sell copies of the Software, and to permit persons to whom the Software is |
| # furnished to do so, subject to the following conditions: |
| # |
| # The above copyright notice and this permission notice shall be included in all |
| # copies or substantial portions of the Software. |
| # |
| # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
| # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
| # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE |
| # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
| # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
| # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
| # SOFTWARE. |
| |
| import argparse |
| import datetime |
| import difflib |
| import filecmp |
| import logging |
| import os |
| import re |
| import subprocess |
| import sys |
| |
| from modules.Shell import Shell |
| |
| logger = logging.getLogger("format_code") |
| |
| # List of directories to exclude |
| exceptions = [ |
| "src/core/NEON/kernels/assembly/gemm", |
| "src/core/NEON/kernels/assembly/arm", |
| "/winograd/", |
| "/convolution/", |
| "/arm_gemm/", |
| "/arm_conv/", |
| "SConscript", |
| "SConstruct" |
| ] |
| |
| def adjust_copyright_year(copyright_years, curr_year): |
| ret_copyright_year = str() |
| # Read last year in the Copyright |
| last_year = int(copyright_years[-4:]) |
| if last_year == curr_year: |
| ret_copyright_year = copyright_years |
| elif last_year == (curr_year - 1): |
| # Create range if latest year on the copyright is the previous |
| if len(copyright_years) > 4 and copyright_years[-5] == "-": |
| # Range already exists, update year to current |
| ret_copyright_year = copyright_years[:-5] + "-" + str(curr_year) |
| else: |
| # Create a new range |
| ret_copyright_year = copyright_years + "-" + str(curr_year) |
| else: |
| ret_copyright_year = copyright_years + ", " + str(curr_year) |
| return ret_copyright_year |
| |
| def check_copyright( filename ): |
| f = open(filename, "r") |
| content = f.readlines() |
| f.close() |
| f = open(filename, "w") |
| year = datetime.datetime.now().year |
| ref = open("scripts/copyright_mit.txt","r").readlines() |
| |
| # Need to handle python files separately |
| if("SConstruct" in filename or "SConscript" in filename): |
| start = 2 |
| if("SConscript" in filename): |
| start = 3 |
| m = re.match(r"(# Copyright \(c\) )(.*\d{4})( [Arm|ARM].*)", content[start]) |
| line = m.group(1) |
| |
| if m.group(2): # Is there a year already? |
| # Yes: adjust accordingly |
| line += adjust_copyright_year(m.group(2), year) |
| else: |
| # No: add current year |
| line += str(year) |
| line += m.group(3).replace("ARM", "Arm") |
| if("SConscript" in filename): |
| f.write('#!/usr/bin/python\n') |
| |
| f.write('# -*- coding: utf-8 -*-\n\n') |
| f.write(line+"\n") |
| # Copy the rest of the file's content: |
| f.write("".join(content[start + 1:])) |
| f.close() |
| |
| return |
| |
| # This only works until year 9999 |
| m = re.match(r"(.*Copyright \(c\) )(.*\d{4})( [Arm|ARM].*)", content[1]) |
| start =len(ref)+2 |
| if content[0] != "/*\n" or not m: |
| start = 0 |
| f.write("/*\n * Copyright (c) %d Arm Limited.\n" % year) |
| else: |
| logger.debug("Found Copyright start") |
| logger.debug("\n\t".join([ g or "" for g in m.groups()])) |
| line = m.group(1) |
| |
| if m.group(2): # Is there a year already? |
| # Yes: adjust accordingly |
| line += adjust_copyright_year(m.group(2), year) |
| else: |
| # No: add current year |
| line += str(year) |
| line += m.group(3).replace("ARM", "Arm") |
| f.write("/*\n"+line+"\n") |
| logger.debug(line) |
| # Write out the rest of the Copyright header: |
| for i in range(1, len(ref)): |
| line = ref[i] |
| f.write(" *") |
| if line.rstrip() != "": |
| f.write(" %s" % line) |
| else: |
| f.write("\n") |
| f.write(" */\n") |
| # Copy the rest of the file's content: |
| f.write("".join(content[start:])) |
| f.close() |
| |
| def check_license(filename): |
| """ |
| Check that the license file is up-to-date |
| """ |
| f = open(filename, "r") |
| content = f.readlines() |
| f.close() |
| |
| f = open(filename, "w") |
| f.write("".join(content[:2])) |
| |
| year = datetime.datetime.now().year |
| # This only works until year 9999 |
| m = re.match(r"(.*Copyright \(c\) )(.*\d{4})( [Arm|ARM].*)", content[2]) |
| |
| if not m: |
| f.write("Copyright (c) {} Arm Limited\n".format(year)) |
| else: |
| updated_year = adjust_copyright_year(m.group(2), year) |
| f.write("Copyright (c) {} Arm Limited\n".format(updated_year)) |
| |
| # Copy the rest of the file's content: |
| f.write("".join(content[3:])) |
| f.close() |
| |
| |
| class OtherChecksRun: |
| def __init__(self, folder, error_diff=False, strategy="all"): |
| self.folder = folder |
| self.error_diff=error_diff |
| self.strategy = strategy |
| |
| def error_on_diff(self, msg): |
| retval = 0 |
| if self.error_diff: |
| diff = self.shell.run_single_to_str("git diff") |
| if len(diff) > 0: |
| retval = -1 |
| logger.error(diff) |
| logger.error("\n"+msg) |
| return retval |
| |
| def run(self): |
| retval = 0 |
| self.shell = Shell() |
| self.shell.save_cwd() |
| this_dir = os.path.dirname(__file__) |
| self.shell.cd(self.folder) |
| self.shell.prepend_env("PATH","%s/../bin" % this_dir) |
| |
| to_check = "" |
| if self.strategy != "all": |
| to_check, skip_copyright = FormatCodeRun.get_files(self.folder, self.strategy) |
| #FIXME: Exclude shaders! |
| |
| logger.info("Running ./scripts/format_doxygen.py") |
| logger.debug(self.shell.run_single_to_str("./scripts/format_doxygen.py %s" % " ".join(to_check))) |
| retval = self.error_on_diff("Doxygen comments badly formatted (check above diff output for more details) try to run ./scripts/format_doxygen.py on your patch and resubmit") |
| if retval == 0: |
| logger.info("Running ./scripts/include_functions_kernels.py") |
| logger.debug(self.shell.run_single_to_str("python ./scripts/include_functions_kernels.py")) |
| retval = self.error_on_diff("Some kernels or functions are not included in their corresponding master header (check above diff output to see which includes are missing)") |
| if retval == 0: |
| try: |
| logger.info("Running ./scripts/check_bad_style.sh") |
| logger.debug(self.shell.run_single_to_str("./scripts/check_bad_style.sh")) |
| #logger.debug(self.shell.run_single_to_str("./scripts/check_bad_style.sh %s" % " ".join(to_check))) |
| except subprocess.CalledProcessError as e: |
| logger.error("Command %s returned:\n%s" % (e.cmd, e.output)) |
| retval -= 1 |
| |
| if retval != 0: |
| raise Exception("format-code failed with error code %d" % retval) |
| |
| class FormatCodeRun: |
| @staticmethod |
| def get_files(folder, strategy="git-head"): |
| shell = Shell() |
| shell.cd(folder) |
| skip_copyright = False |
| if strategy == "git-head": |
| cmd = "git diff-tree --no-commit-id --name-status -r HEAD | grep \"^[AMRT]\" | cut -f 2" |
| elif strategy == "git-diff": |
| cmd = "git diff --name-status --cached -r HEAD | grep \"^[AMRT]\" | rev | cut -f 1 | rev" |
| else: |
| cmd = "git ls-tree -r HEAD --name-only" |
| # Skip copyright checks when running on all files because we don't know when they were last modified |
| # Therefore we can't tell if their copyright dates are correct |
| skip_copyright = True |
| |
| grep_folder = "grep -e \"^\\(arm_compute\\|src\\|examples\\|tests\\|utils\\|support\\)/\"" |
| grep_extension = "grep -e \"\\.\\(cpp\\|h\\|hh\\|inl\\|cl\\|cs\\|hpp\\)$\"" |
| list_files = shell.run_single_to_str(cmd+" | { "+ grep_folder+" | "+grep_extension + " || true; }") |
| to_check = [ f for f in list_files.split("\n") if len(f) > 0] |
| |
| # Check for scons files as they are excluded from the above list |
| list_files = shell.run_single_to_str(cmd+" | { grep -e \"SC\" || true; }") |
| to_check += [ f for f in list_files.split("\n") if len(f) > 0] |
| |
| return (to_check, skip_copyright) |
| |
| def __init__(self, files, folder, error_diff=False, skip_copyright=False): |
| self.files = files |
| self.folder = folder |
| self.skip_copyright = skip_copyright |
| self.error_diff=error_diff |
| |
| def error_on_diff(self, msg): |
| retval = 0 |
| if self.error_diff: |
| diff = self.shell.run_single_to_str("git diff") |
| if len(diff) > 0: |
| retval = -1 |
| logger.error(diff) |
| logger.error("\n"+msg) |
| return retval |
| |
| def run(self): |
| if len(self.files) < 1: |
| logger.debug("No file: early exit") |
| retval = 0 |
| self.shell = Shell() |
| self.shell.save_cwd() |
| this_dir = os.path.dirname(__file__) |
| try: |
| self.shell.cd(self.folder) |
| self.shell.prepend_env("PATH","%s/../bin" % this_dir) |
| |
| for f in self.files: |
| if not self.skip_copyright: |
| check_copyright(f) |
| |
| skip_this_file = False |
| for e in exceptions: |
| if e in f: |
| logger.warning("Skipping '%s' file: %s" % (e,f)) |
| skip_this_file = True |
| break |
| if skip_this_file: |
| continue |
| |
| logger.info("Formatting %s" % f) |
| |
| check_license("LICENSE") |
| |
| except subprocess.CalledProcessError as e: |
| retval = -1 |
| logger.error(e) |
| logger.error("OUTPUT= %s" % e.output) |
| |
| retval += self.error_on_diff("See above for clang-tidy errors") |
| |
| if retval != 0: |
| raise Exception("format-code failed with error code %d" % retval) |
| |
| class GenerateAndroidBP: |
| def __init__(self, folder): |
| self.folder = folder |
| self.bp_output_file = "Generated_Android.bp" |
| |
| def run(self): |
| retval = 0 |
| self.shell = Shell() |
| self.shell.save_cwd() |
| this_dir = os.path.dirname(__file__) |
| |
| logger.debug("Running Android.bp check") |
| try: |
| self.shell.cd(self.folder) |
| cmd = "%s/generate_android_bp.py --folder %s --output_file %s" % (this_dir, self.folder, self.bp_output_file) |
| output = self.shell.run_single_to_str(cmd) |
| if len(output) > 0: |
| logger.info(output) |
| except subprocess.CalledProcessError as e: |
| retval = -1 |
| logger.error(e) |
| logger.error("OUTPUT= %s" % e.output) |
| |
| # Compare the genereated file with the one in the review |
| if not filecmp.cmp(self.bp_output_file, self.folder + "/Android.bp"): |
| is_mismatched = True |
| |
| with open(self.bp_output_file, 'r') as generated_file: |
| with open(self.folder + "/Android.bp", 'r') as review_file: |
| diff = list(difflib.unified_diff(generated_file.readlines(), review_file.readlines(), |
| fromfile='Generated_Android.bp', tofile='Android.bp')) |
| |
| # If the only mismatch in Android.bp file is the copyright year, |
| # the content of the file is considered unchanged and we don't need to update |
| # the copyright year. This will resolve the issue that emerges every new year. |
| num_added_lines = 0 |
| num_removed_lines = 0 |
| last_added_line = "" |
| last_removed_line = "" |
| expect_add_line = False |
| |
| for line in diff: |
| if line.startswith("-") and not line.startswith("---"): |
| num_removed_lines += 1 |
| if num_removed_lines > 1: |
| break |
| last_removed_line = line |
| expect_add_line = True |
| elif line.startswith("+") and not line.startswith("+++"): |
| num_added_lines += 1 |
| if num_added_lines > 1: |
| break |
| if expect_add_line: |
| last_added_line = line |
| else: |
| expect_add_line = False |
| |
| if num_added_lines == 1 and num_removed_lines == 1: |
| re_copyright = re.compile("^(?:\+|\-)// Copyright © ([0-9]+)\-([0-9]+) Arm Ltd. All rights reserved.\n$") |
| generated_matches = re_copyright.search(last_removed_line) |
| review_matches = re_copyright.search(last_added_line) |
| |
| if generated_matches is not None and review_matches is not None: |
| if generated_matches.group(1) == review_matches.group(1) and \ |
| int(generated_matches.group(2)) > int(review_matches.group(2)): |
| is_mismatched = False |
| |
| if is_mismatched: |
| logger.error("Lines with '-' need to be added to Android.bp") |
| logger.error("Lines with '+' need to be removed from Android.bp") |
| |
| for line in diff: |
| logger.error(line.rstrip()) |
| if is_mismatched: |
| raise Exception("Android bp file is not updated") |
| |
| if retval != 0: |
| raise Exception("generate Android bp file failed with error code %d" % retval) |
| |
| def run_fix_code_formatting( files="git-head", folder=".", num_threads=1, error_on_diff=True): |
| try: |
| retval = 0 |
| |
| # Genereate Android.bp file and test it |
| gen_android_bp = GenerateAndroidBP(folder) |
| gen_android_bp.run() |
| |
| to_check, skip_copyright = FormatCodeRun.get_files(folder, files) |
| other_checks = OtherChecksRun(folder,error_on_diff, files) |
| other_checks.run() |
| |
| logger.debug(to_check) |
| num_files = len(to_check) |
| per_thread = max( num_files / num_threads,1) |
| start=0 |
| logger.info("Files to format:\n\t%s" % "\n\t".join(to_check)) |
| |
| for i in range(num_threads): |
| if i == num_threads -1: |
| end = num_files |
| else: |
| end= min(start+per_thread, num_files) |
| sub = to_check[start:end] |
| logger.debug("[%d] [%d,%d] %s" % (i, start, end, sub)) |
| start = end |
| format_code_run = FormatCodeRun(sub, folder, skip_copyright=skip_copyright) |
| format_code_run.run() |
| |
| return retval |
| except Exception as e: |
| logger.error("Exception caught in run_fix_code_formatting: %s" % e) |
| return -1 |
| |
| if __name__ == "__main__": |
| parser = argparse.ArgumentParser( |
| formatter_class=argparse.RawDescriptionHelpFormatter, |
| description="Build & run pre-commit tests", |
| ) |
| |
| file_sources=["git-diff","git-head","all"] |
| parser.add_argument("-D", "--debug", action='store_true', help="Enable script debugging output") |
| parser.add_argument("--error_on_diff", action='store_true', help="Show diff on error and stop") |
| parser.add_argument("--files", nargs='?', metavar="source", choices=file_sources, help="Which files to run fix_code_formatting on, choices=%s" % file_sources, default="git-head") |
| parser.add_argument("--folder", metavar="path", help="Folder in which to run fix_code_formatting", default=".") |
| |
| args = parser.parse_args() |
| |
| logging_level = logging.INFO |
| if args.debug: |
| logging_level = logging.DEBUG |
| |
| logging.basicConfig(level=logging_level) |
| |
| logger.debug("Arguments passed: %s" % str(args.__dict__)) |
| |
| exit(run_fix_code_formatting(args.files, args.folder, 1, error_on_diff=args.error_on_diff)) |