Gunes Bayir | 66b4a6a | 2023-07-01 22:55:42 +0100 | [diff] [blame] | 1 | #!/usr/bin/env python3 |
| 2 | |
| 3 | # Copyright (c) 2023 Arm Limited. |
| 4 | # |
| 5 | # SPDX-License-Identifier: MIT |
| 6 | # |
| 7 | # Permission is hereby granted, free of charge, to any person obtaining a copy |
| 8 | # of this software and associated documentation files (the "Software"), to |
| 9 | # deal in the Software without restriction, including without limitation the |
| 10 | # rights to use, copy, modify, merge, publish, distribute, sublicense, and/or |
| 11 | # sell copies of the Software, and to permit persons to whom the Software is |
| 12 | # furnished to do so, subject to the following conditions: |
| 13 | # |
| 14 | # The above copyright notice and this permission notice shall be included in all |
| 15 | # copies or substantial portions of the Software. |
| 16 | # |
| 17 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
| 18 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
| 19 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE |
| 20 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
| 21 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
| 22 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
| 23 | # SOFTWARE. |
| 24 | |
| 25 | import argparse |
| 26 | import datetime |
| 27 | import difflib |
| 28 | import filecmp |
| 29 | import logging |
| 30 | import os |
| 31 | import re |
| 32 | import subprocess |
| 33 | import sys |
| 34 | |
| 35 | from modules.Shell import Shell |
| 36 | |
| 37 | logger = logging.getLogger("format_code") |
| 38 | |
Jakub Sujak | 6e56bf3 | 2023-08-23 14:42:26 +0100 | [diff] [blame] | 39 | # List of directories to exclude |
Gunes Bayir | 66b4a6a | 2023-07-01 22:55:42 +0100 | [diff] [blame] | 40 | exceptions = [ |
| 41 | "src/core/NEON/kernels/assembly/gemm", |
| 42 | "src/core/NEON/kernels/assembly/arm", |
| 43 | "/winograd/", |
| 44 | "/convolution/", |
| 45 | "/arm_gemm/", |
| 46 | "/arm_conv/", |
Jakub Sujak | 0d27b2e | 2023-08-24 14:01:20 +0100 | [diff] [blame] | 47 | "SConscript", |
| 48 | "SConstruct" |
Gunes Bayir | 66b4a6a | 2023-07-01 22:55:42 +0100 | [diff] [blame] | 49 | ] |
| 50 | |
| 51 | def adjust_copyright_year(copyright_years, curr_year): |
| 52 | ret_copyright_year = str() |
| 53 | # Read last year in the Copyright |
| 54 | last_year = int(copyright_years[-4:]) |
| 55 | if last_year == curr_year: |
| 56 | ret_copyright_year = copyright_years |
| 57 | elif last_year == (curr_year - 1): |
| 58 | # Create range if latest year on the copyright is the previous |
| 59 | if len(copyright_years) > 4 and copyright_years[-5] == "-": |
| 60 | # Range already exists, update year to current |
| 61 | ret_copyright_year = copyright_years[:-5] + "-" + str(curr_year) |
| 62 | else: |
| 63 | # Create a new range |
| 64 | ret_copyright_year = copyright_years + "-" + str(curr_year) |
| 65 | else: |
| 66 | ret_copyright_year = copyright_years + ", " + str(curr_year) |
| 67 | return ret_copyright_year |
| 68 | |
| 69 | def check_copyright( filename ): |
| 70 | f = open(filename, "r") |
| 71 | content = f.readlines() |
| 72 | f.close() |
| 73 | f = open(filename, "w") |
| 74 | year = datetime.datetime.now().year |
| 75 | ref = open("scripts/copyright_mit.txt","r").readlines() |
| 76 | |
| 77 | # Need to handle python files separately |
| 78 | if("SConstruct" in filename or "SConscript" in filename): |
| 79 | start = 2 |
| 80 | if("SConscript" in filename): |
| 81 | start = 3 |
| 82 | m = re.match("(# Copyright \(c\) )(.*\d{4})( [Arm|ARM].*)", content[start]) |
| 83 | line = m.group(1) |
| 84 | |
| 85 | if m.group(2): # Is there a year already? |
| 86 | # Yes: adjust accordingly |
| 87 | line += adjust_copyright_year(m.group(2), year) |
| 88 | else: |
| 89 | # No: add current year |
| 90 | line += str(year) |
| 91 | line += m.group(3).replace("ARM", "Arm") |
| 92 | if("SConscript" in filename): |
| 93 | f.write('#!/usr/bin/python\n') |
| 94 | |
| 95 | f.write('# -*- coding: utf-8 -*-\n\n') |
| 96 | f.write(line+"\n") |
| 97 | # Copy the rest of the file's content: |
| 98 | f.write("".join(content[start + 1:])) |
| 99 | f.close() |
| 100 | |
| 101 | return |
| 102 | |
| 103 | # This only works until year 9999 |
| 104 | m = re.match("(.*Copyright \(c\) )(.*\d{4})( [Arm|ARM].*)", content[1]) |
| 105 | start =len(ref)+2 |
| 106 | if content[0] != "/*\n" or not m: |
| 107 | start = 0 |
| 108 | f.write("/*\n * Copyright (c) %d Arm Limited.\n" % year) |
| 109 | else: |
| 110 | logger.debug("Found Copyright start") |
| 111 | logger.debug("\n\t".join([ g or "" for g in m.groups()])) |
| 112 | line = m.group(1) |
| 113 | |
| 114 | if m.group(2): # Is there a year already? |
| 115 | # Yes: adjust accordingly |
| 116 | line += adjust_copyright_year(m.group(2), year) |
| 117 | else: |
| 118 | # No: add current year |
| 119 | line += str(year) |
| 120 | line += m.group(3).replace("ARM", "Arm") |
| 121 | f.write("/*\n"+line+"\n") |
| 122 | logger.debug(line) |
| 123 | # Write out the rest of the Copyright header: |
| 124 | for i in range(1, len(ref)): |
| 125 | line = ref[i] |
| 126 | f.write(" *") |
| 127 | if line.rstrip() != "": |
| 128 | f.write(" %s" % line) |
| 129 | else: |
| 130 | f.write("\n") |
| 131 | f.write(" */\n") |
| 132 | # Copy the rest of the file's content: |
| 133 | f.write("".join(content[start:])) |
| 134 | f.close() |
| 135 | |
| 136 | def check_license(filename): |
| 137 | """ |
| 138 | Check that the license file is up-to-date |
| 139 | """ |
| 140 | f = open(filename, "r") |
| 141 | content = f.readlines() |
| 142 | f.close() |
| 143 | |
| 144 | f = open(filename, "w") |
| 145 | f.write("".join(content[:2])) |
| 146 | |
| 147 | year = datetime.datetime.now().year |
| 148 | # This only works until year 9999 |
| 149 | m = re.match("(.*Copyright \(c\) )(.*\d{4})( [Arm|ARM].*)", content[2]) |
| 150 | |
| 151 | if not m: |
| 152 | f.write("Copyright (c) {} Arm Limited\n".format(year)) |
| 153 | else: |
| 154 | updated_year = adjust_copyright_year(m.group(2), year) |
| 155 | f.write("Copyright (c) {} Arm Limited\n".format(updated_year)) |
| 156 | |
| 157 | # Copy the rest of the file's content: |
| 158 | f.write("".join(content[3:])) |
| 159 | f.close() |
| 160 | |
| 161 | |
| 162 | class OtherChecksRun: |
| 163 | def __init__(self, folder, error_diff=False, strategy="all"): |
| 164 | self.folder = folder |
| 165 | self.error_diff=error_diff |
| 166 | self.strategy = strategy |
| 167 | |
| 168 | def error_on_diff(self, msg): |
| 169 | retval = 0 |
| 170 | if self.error_diff: |
| 171 | diff = self.shell.run_single_to_str("git diff") |
| 172 | if len(diff) > 0: |
| 173 | retval = -1 |
| 174 | logger.error(diff) |
| 175 | logger.error("\n"+msg) |
| 176 | return retval |
| 177 | |
| 178 | def run(self): |
| 179 | retval = 0 |
| 180 | self.shell = Shell() |
| 181 | self.shell.save_cwd() |
| 182 | this_dir = os.path.dirname(__file__) |
| 183 | self.shell.cd(self.folder) |
| 184 | self.shell.prepend_env("PATH","%s/../bin" % this_dir) |
| 185 | |
| 186 | to_check = "" |
| 187 | if self.strategy != "all": |
| 188 | to_check, skip_copyright = FormatCodeRun.get_files(self.folder, self.strategy) |
| 189 | #FIXME: Exclude shaders! |
| 190 | |
| 191 | logger.info("Running ./scripts/format_doxygen.py") |
| 192 | logger.debug(self.shell.run_single_to_str("./scripts/format_doxygen.py %s" % " ".join(to_check))) |
| 193 | retval = self.error_on_diff("Doxygen comments badly formatted (check above diff output for more details) try to run ./scripts/format_doxygen.py on your patch and resubmit") |
| 194 | if retval == 0: |
| 195 | logger.info("Running ./scripts/include_functions_kernels.py") |
| 196 | logger.debug(self.shell.run_single_to_str("python ./scripts/include_functions_kernels.py")) |
| 197 | retval = self.error_on_diff("Some kernels or functions are not included in their corresponding master header (check above diff output to see which includes are missing)") |
| 198 | if retval == 0: |
| 199 | try: |
| 200 | logger.info("Running ./scripts/check_bad_style.sh") |
| 201 | logger.debug(self.shell.run_single_to_str("./scripts/check_bad_style.sh")) |
| 202 | #logger.debug(self.shell.run_single_to_str("./scripts/check_bad_style.sh %s" % " ".join(to_check))) |
| 203 | except subprocess.CalledProcessError as e: |
| 204 | logger.error("Command %s returned:\n%s" % (e.cmd, e.output)) |
| 205 | retval -= 1 |
| 206 | |
| 207 | if retval != 0: |
| 208 | raise Exception("format-code failed with error code %d" % retval) |
| 209 | |
| 210 | class FormatCodeRun: |
| 211 | @staticmethod |
| 212 | def get_files(folder, strategy="git-head"): |
| 213 | shell = Shell() |
| 214 | shell.cd(folder) |
| 215 | skip_copyright = False |
| 216 | if strategy == "git-head": |
| 217 | cmd = "git diff-tree --no-commit-id --name-status -r HEAD | grep \"^[AMRT]\" | cut -f 2" |
| 218 | elif strategy == "git-diff": |
| 219 | cmd = "git diff --name-status --cached -r HEAD | grep \"^[AMRT]\" | cut -f 2" |
| 220 | else: |
| 221 | cmd = "git ls-tree -r HEAD --name-only" |
| 222 | # Skip copyright checks when running on all files because we don't know when they were last modified |
| 223 | # Therefore we can't tell if their copyright dates are correct |
| 224 | skip_copyright = True |
| 225 | |
| 226 | grep_folder = "grep -e \"^\\(arm_compute\\|src\\|examples\\|tests\\|utils\\|support\\)/\"" |
Jakub Sujak | 6e56bf3 | 2023-08-23 14:42:26 +0100 | [diff] [blame] | 227 | grep_extension = "grep -e \"\\.\\(cpp\\|h\\|hh\\|inl\\|cl\\|cs\\|hpp\\)$\"" |
Gunes Bayir | 66b4a6a | 2023-07-01 22:55:42 +0100 | [diff] [blame] | 228 | list_files = shell.run_single_to_str(cmd+" | { "+ grep_folder+" | "+grep_extension + " || true; }") |
| 229 | to_check = [ f for f in list_files.split("\n") if len(f) > 0] |
| 230 | |
| 231 | # Check for scons files as they are excluded from the above list |
| 232 | list_files = shell.run_single_to_str(cmd+" | { grep -e \"SC\" || true; }") |
| 233 | to_check += [ f for f in list_files.split("\n") if len(f) > 0] |
| 234 | |
| 235 | return (to_check, skip_copyright) |
| 236 | |
| 237 | def __init__(self, files, folder, error_diff=False, skip_copyright=False): |
| 238 | self.files = files |
| 239 | self.folder = folder |
| 240 | self.skip_copyright = skip_copyright |
| 241 | self.error_diff=error_diff |
| 242 | |
| 243 | def error_on_diff(self, msg): |
| 244 | retval = 0 |
| 245 | if self.error_diff: |
| 246 | diff = self.shell.run_single_to_str("git diff") |
| 247 | if len(diff) > 0: |
| 248 | retval = -1 |
| 249 | logger.error(diff) |
| 250 | logger.error("\n"+msg) |
| 251 | return retval |
| 252 | |
| 253 | def run(self): |
| 254 | if len(self.files) < 1: |
| 255 | logger.debug("No file: early exit") |
| 256 | retval = 0 |
| 257 | self.shell = Shell() |
| 258 | self.shell.save_cwd() |
| 259 | this_dir = os.path.dirname(__file__) |
| 260 | try: |
| 261 | self.shell.cd(self.folder) |
| 262 | self.shell.prepend_env("PATH","%s/../bin" % this_dir) |
Gunes Bayir | 66b4a6a | 2023-07-01 22:55:42 +0100 | [diff] [blame] | 263 | |
Gunes Bayir | 66b4a6a | 2023-07-01 22:55:42 +0100 | [diff] [blame] | 264 | for f in self.files: |
Jakub Sujak | 835577e | 2023-11-27 15:50:31 +0000 | [diff] [blame] | 265 | if not self.skip_copyright: |
| 266 | check_copyright(f) |
| 267 | |
Gunes Bayir | 66b4a6a | 2023-07-01 22:55:42 +0100 | [diff] [blame] | 268 | skip_this_file = False |
| 269 | for e in exceptions: |
| 270 | if e in f: |
| 271 | logger.warning("Skipping '%s' file: %s" % (e,f)) |
| 272 | skip_this_file = True |
| 273 | break |
| 274 | if skip_this_file: |
| 275 | continue |
| 276 | |
| 277 | logger.info("Formatting %s" % f) |
Gunes Bayir | 66b4a6a | 2023-07-01 22:55:42 +0100 | [diff] [blame] | 278 | |
| 279 | check_license("LICENSE") |
| 280 | |
| 281 | except subprocess.CalledProcessError as e: |
| 282 | retval = -1 |
| 283 | logger.error(e) |
| 284 | logger.error("OUTPUT= %s" % e.output) |
| 285 | |
| 286 | retval += self.error_on_diff("See above for clang-tidy errors") |
| 287 | |
| 288 | if retval != 0: |
| 289 | raise Exception("format-code failed with error code %d" % retval) |
| 290 | |
| 291 | class GenerateAndroidBP: |
| 292 | def __init__(self, folder): |
| 293 | self.folder = folder |
| 294 | self.bp_output_file = "Generated_Android.bp" |
| 295 | |
| 296 | def run(self): |
| 297 | retval = 0 |
| 298 | self.shell = Shell() |
| 299 | self.shell.save_cwd() |
| 300 | this_dir = os.path.dirname(__file__) |
| 301 | |
| 302 | logger.debug("Running Android.bp check") |
| 303 | try: |
| 304 | self.shell.cd(self.folder) |
| 305 | cmd = "%s/generate_android_bp.py --folder %s --output_file %s" % (this_dir, self.folder, self.bp_output_file) |
| 306 | output = self.shell.run_single_to_str(cmd) |
| 307 | if len(output) > 0: |
| 308 | logger.info(output) |
| 309 | except subprocess.CalledProcessError as e: |
| 310 | retval = -1 |
| 311 | logger.error(e) |
| 312 | logger.error("OUTPUT= %s" % e.output) |
| 313 | |
| 314 | # Compare the genereated file with the one in the review |
| 315 | if not filecmp.cmp(self.bp_output_file, self.folder + "/Android.bp"): |
| 316 | is_mismatched = True |
| 317 | |
| 318 | with open(self.bp_output_file, 'r') as generated_file: |
| 319 | with open(self.folder + "/Android.bp", 'r') as review_file: |
| 320 | diff = list(difflib.unified_diff(generated_file.readlines(), review_file.readlines(), |
| 321 | fromfile='Generated_Android.bp', tofile='Android.bp')) |
| 322 | |
| 323 | # If the only mismatch in Android.bp file is the copyright year, |
| 324 | # the content of the file is considered unchanged and we don't need to update |
| 325 | # the copyright year. This will resolve the issue that emerges every new year. |
| 326 | num_added_lines = 0 |
| 327 | num_removed_lines = 0 |
| 328 | last_added_line = "" |
| 329 | last_removed_line = "" |
| 330 | expect_add_line = False |
| 331 | |
| 332 | for line in diff: |
| 333 | if line.startswith("-") and not line.startswith("---"): |
| 334 | num_removed_lines += 1 |
| 335 | if num_removed_lines > 1: |
| 336 | break |
| 337 | last_removed_line = line |
| 338 | expect_add_line = True |
| 339 | elif line.startswith("+") and not line.startswith("+++"): |
| 340 | num_added_lines += 1 |
| 341 | if num_added_lines > 1: |
| 342 | break |
| 343 | if expect_add_line: |
| 344 | last_added_line = line |
| 345 | else: |
| 346 | expect_add_line = False |
| 347 | |
| 348 | if num_added_lines == 1 and num_removed_lines == 1: |
| 349 | re_copyright = re.compile("^(?:\+|\-)// Copyright © ([0-9]+)\-([0-9]+) Arm Ltd. All rights reserved.\n$") |
| 350 | generated_matches = re_copyright.search(last_removed_line) |
| 351 | review_matches = re_copyright.search(last_added_line) |
| 352 | |
| 353 | if generated_matches is not None and review_matches is not None: |
| 354 | if generated_matches.group(1) == review_matches.group(1) and \ |
| 355 | int(generated_matches.group(2)) > int(review_matches.group(2)): |
| 356 | is_mismatched = False |
| 357 | |
| 358 | if is_mismatched: |
| 359 | logger.error("Lines with '-' need to be added to Android.bp") |
| 360 | logger.error("Lines with '+' need to be removed from Android.bp") |
| 361 | |
| 362 | for line in diff: |
| 363 | logger.error(line.rstrip()) |
| 364 | if is_mismatched: |
| 365 | raise Exception("Android bp file is not updated") |
| 366 | |
| 367 | if retval != 0: |
| 368 | raise Exception("generate Android bp file failed with error code %d" % retval) |
| 369 | |
| 370 | def run_fix_code_formatting( files="git-head", folder=".", num_threads=1, error_on_diff=True): |
| 371 | try: |
| 372 | retval = 0 |
| 373 | |
| 374 | # Genereate Android.bp file and test it |
| 375 | gen_android_bp = GenerateAndroidBP(folder) |
| 376 | gen_android_bp.run() |
| 377 | |
| 378 | to_check, skip_copyright = FormatCodeRun.get_files(folder, files) |
| 379 | other_checks = OtherChecksRun(folder,error_on_diff, files) |
| 380 | other_checks.run() |
| 381 | |
| 382 | logger.debug(to_check) |
| 383 | num_files = len(to_check) |
| 384 | per_thread = max( num_files / num_threads,1) |
| 385 | start=0 |
| 386 | logger.info("Files to format:\n\t%s" % "\n\t".join(to_check)) |
| 387 | |
| 388 | for i in range(num_threads): |
| 389 | if i == num_threads -1: |
| 390 | end = num_files |
| 391 | else: |
| 392 | end= min(start+per_thread, num_files) |
| 393 | sub = to_check[start:end] |
| 394 | logger.debug("[%d] [%d,%d] %s" % (i, start, end, sub)) |
| 395 | start = end |
| 396 | format_code_run = FormatCodeRun(sub, folder, skip_copyright=skip_copyright) |
| 397 | format_code_run.run() |
| 398 | |
| 399 | return retval |
| 400 | except Exception as e: |
| 401 | logger.error("Exception caught in run_fix_code_formatting: %s" % e) |
| 402 | return -1 |
| 403 | |
| 404 | if __name__ == "__main__": |
| 405 | parser = argparse.ArgumentParser( |
| 406 | formatter_class=argparse.RawDescriptionHelpFormatter, |
| 407 | description="Build & run pre-commit tests", |
| 408 | ) |
| 409 | |
| 410 | file_sources=["git-diff","git-head","all"] |
| 411 | parser.add_argument("-D", "--debug", action='store_true', help="Enable script debugging output") |
| 412 | parser.add_argument("--error_on_diff", action='store_true', help="Show diff on error and stop") |
| 413 | parser.add_argument("--files", nargs='?', metavar="source", choices=file_sources, help="Which files to run fix_code_formatting on, choices=%s" % file_sources, default="git-head") |
| 414 | parser.add_argument("--folder", metavar="path", help="Folder in which to run fix_code_formatting", default=".") |
| 415 | |
| 416 | args = parser.parse_args() |
| 417 | |
| 418 | logging_level = logging.INFO |
| 419 | if args.debug: |
| 420 | logging_level = logging.DEBUG |
| 421 | |
| 422 | logging.basicConfig(level=logging_level) |
| 423 | |
| 424 | logger.debug("Arguments passed: %s" % str(args.__dict__)) |
| 425 | |
| 426 | exit(run_fix_code_formatting(args.files, args.folder, 1, error_on_diff=args.error_on_diff)) |