scripts/format_code.py

#!/usr/bin/env python3

# Copyright (c) 2023 Arm Limited.
#
# SPDX-License-Identifier: MIT
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to
# deal in the Software without restriction, including without limitation the
# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
# sell copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

import argparse
import datetime
import difflib
import filecmp
import logging
import os
import re
import subprocess
import sys

from modules.Shell import Shell

logger = logging.getLogger("format_code")

ASTYLE_PARAMETERS ="--style=ansi \
    --indent=spaces \
    --indent-switches \
    --indent-col1-comments \
    --min-conditional-indent=0 \
    --max-instatement-indent=120 \
    --pad-oper \
    --align-pointer=name \
    --align-reference=name \
    --break-closing-brackets \
    --keep-one-line-statements \
    --max-code-length=200 \
    --mode=c \
    --lineend=linux \
    --indent-preprocessor \
    "

exceptions = [
    "src/core/NEON/kernels/assembly/gemm",
    "src/core/NEON/kernels/assembly/arm",
    "/winograd/",
    "/convolution/",
    "/arm_gemm/",
    "/arm_conv/",
    "compute_kernel_writer/"
]

def adjust_copyright_year(copyright_years, curr_year):
    ret_copyright_year = str()
    # Read last year in the Copyright
    last_year = int(copyright_years[-4:])
    if last_year == curr_year:
        ret_copyright_year = copyright_years
    elif last_year == (curr_year - 1):
        # Create range if latest year on the copyright is the previous
        if len(copyright_years) > 4 and copyright_years[-5] == "-":
            # Range already exists, update year to current
            ret_copyright_year = copyright_years[:-5] + "-" + str(curr_year)
        else:
            # Create a new range
            ret_copyright_year = copyright_years + "-" + str(curr_year)
    else:
        ret_copyright_year = copyright_years + ", " + str(curr_year)
    return ret_copyright_year

def check_copyright( filename ):
    f = open(filename, "r")
    content = f.readlines()
    f.close()
    f = open(filename, "w")
    year = datetime.datetime.now().year
    ref = open("scripts/copyright_mit.txt","r").readlines()

    # Need to handle python files separately
    if("SConstruct" in filename or "SConscript" in filename):
        start = 2
        if("SConscript" in filename):
            start = 3
        m = re.match("(# Copyright \(c\) )(.*\d{4})( [Arm|ARM].*)", content[start])
        line = m.group(1)

        if m.group(2): # Is there a year already?
            # Yes: adjust accordingly
            line += adjust_copyright_year(m.group(2), year)
        else:
            # No: add current year
            line += str(year)
        line += m.group(3).replace("ARM", "Arm")
        if("SConscript" in filename):
            f.write('#!/usr/bin/python\n')

        f.write('# -*- coding: utf-8 -*-\n\n')
        f.write(line+"\n")
        # Copy the rest of the file's content:
        f.write("".join(content[start + 1:]))
        f.close()

        return

    # This only works until year 9999
    m = re.match("(.*Copyright \(c\) )(.*\d{4})( [Arm|ARM].*)", content[1])
    start =len(ref)+2
    if content[0] != "/*\n" or not m:
        start = 0
        f.write("/*\n * Copyright (c) %d Arm Limited.\n" % year)
    else:
        logger.debug("Found Copyright start")
        logger.debug("\n\t".join([ g or "" for g in m.groups()]))
        line = m.group(1)

        if m.group(2): # Is there a year already?
            # Yes: adjust accordingly
            line += adjust_copyright_year(m.group(2), year)
        else:
            # No: add current year
            line += str(year)
        line += m.group(3).replace("ARM", "Arm")
        f.write("/*\n"+line+"\n")
        logger.debug(line)
    # Write out the rest of the Copyright header:
    for i in range(1, len(ref)):
        line = ref[i]
        f.write(" *")
        if line.rstrip() != "":
            f.write(" %s" % line)
        else:
            f.write("\n")
    f.write(" */\n")
    # Copy the rest of the file's content:
    f.write("".join(content[start:]))
    f.close()

def check_license(filename):
    """
    Check that the license file is up-to-date
    """
    f = open(filename, "r")
    content = f.readlines()
    f.close()

    f = open(filename, "w")
    f.write("".join(content[:2]))

    year = datetime.datetime.now().year
    # This only works until year 9999
    m = re.match("(.*Copyright \(c\) )(.*\d{4})( [Arm|ARM].*)", content[2])

    if not m:
        f.write("Copyright (c) {} Arm Limited\n".format(year))
    else:
        updated_year = adjust_copyright_year(m.group(2), year)
        f.write("Copyright (c) {} Arm Limited\n".format(updated_year))

    # Copy the rest of the file's content:
    f.write("".join(content[3:]))
    f.close()


class OtherChecksRun:
    def __init__(self, folder, error_diff=False, strategy="all"):
        self.folder = folder
        self.error_diff=error_diff
        self.strategy = strategy

    def error_on_diff(self, msg):
        retval = 0
        if self.error_diff:
            diff = self.shell.run_single_to_str("git diff")
            if len(diff) > 0:
                retval = -1
                logger.error(diff)
                logger.error("\n"+msg)
        return retval

    def run(self):
        retval = 0
        self.shell = Shell()
        self.shell.save_cwd()
        this_dir = os.path.dirname(__file__)
        self.shell.cd(self.folder)
        self.shell.prepend_env("PATH","%s/../bin" % this_dir)

        to_check = ""
        if self.strategy != "all":
            to_check, skip_copyright = FormatCodeRun.get_files(self.folder, self.strategy)
            #FIXME: Exclude shaders!

        logger.info("Running ./scripts/format_doxygen.py")
        logger.debug(self.shell.run_single_to_str("./scripts/format_doxygen.py %s" % " ".join(to_check)))
        retval = self.error_on_diff("Doxygen comments badly formatted (check above diff output for more details) try to run ./scripts/format_doxygen.py on your patch and resubmit")
        if retval == 0:
            logger.info("Running ./scripts/include_functions_kernels.py")
            logger.debug(self.shell.run_single_to_str("python ./scripts/include_functions_kernels.py"))
            retval = self.error_on_diff("Some kernels or functions are not included in their corresponding master header (check above diff output to see which includes are missing)")
        if retval == 0:
            try:
                logger.info("Running ./scripts/check_bad_style.sh")
                logger.debug(self.shell.run_single_to_str("./scripts/check_bad_style.sh"))
                #logger.debug(self.shell.run_single_to_str("./scripts/check_bad_style.sh %s" % " ".join(to_check)))
            except subprocess.CalledProcessError as e:
                logger.error("Command %s returned:\n%s" % (e.cmd, e.output))
                retval -= 1

        if retval != 0:
            raise Exception("format-code failed with error code %d" % retval)

class FormatCodeRun:
    @staticmethod
    def get_files(folder, strategy="git-head"):
        shell = Shell()
        shell.cd(folder)
        skip_copyright = False
        if strategy == "git-head":
            cmd = "git diff-tree --no-commit-id --name-status -r HEAD | grep \"^[AMRT]\" | cut -f 2"
        elif strategy == "git-diff":
            cmd = "git diff --name-status --cached -r HEAD | grep \"^[AMRT]\" | cut -f 2"
        else:
            cmd = "git ls-tree -r HEAD --name-only"
            # Skip copyright checks when running on all files because we don't know when they were last modified
            # Therefore we can't tell if their copyright dates are correct
            skip_copyright = True

        grep_folder = "grep -e \"^\\(arm_compute\\|src\\|examples\\|tests\\|utils\\|support\\)/\""
        grep_extension = "grep -e \"\\.\\(cpp\\|h\\|inl\\|cl\\|cs\\|hpp\\)$\""
        list_files = shell.run_single_to_str(cmd+" | { "+ grep_folder+" | "+grep_extension + " || true; }")
        to_check = [ f for f in list_files.split("\n") if len(f) > 0]

        # Check for scons files as they are excluded from the above list
        list_files = shell.run_single_to_str(cmd+" | { grep -e \"SC\" || true; }")
        to_check += [ f for f in list_files.split("\n") if len(f) > 0]

        return (to_check, skip_copyright)

    def __init__(self, files, folder, error_diff=False, skip_copyright=False):
        self.files = files
        self.folder = folder
        self.skip_copyright = skip_copyright
        self.error_diff=error_diff

    def error_on_diff(self, msg):
        retval = 0
        if self.error_diff:
            diff = self.shell.run_single_to_str("git diff")
            if len(diff) > 0:
                retval = -1
                logger.error(diff)
                logger.error("\n"+msg)
        return retval

    def run(self):
        if len(self.files) < 1:
            logger.debug("No file: early exit")
        retval = 0
        self.shell = Shell()
        self.shell.save_cwd()
        this_dir = os.path.dirname(__file__)
        try:
            self.shell.cd(self.folder)
            self.shell.prepend_env("PATH","%s/../bin" % this_dir)
            clang_format = "clang-format -i -style=file "
            astyle = "astyle -n -q %s " % (ASTYLE_PARAMETERS)

            if sys.platform == 'darwin':
                # this platform explicitly needs an extension for the temporary file
                sed = "sed -i '.log' 's/\\t/    /g' "
            else:
                sed = "sed -i 's/\\t/    /g' "

            single_eol = "%s/ensure_single_eol.py " % this_dir
            for f in self.files:
                skip_this_file = False
                for e in exceptions:
                    if e in f:
                        logger.warning("Skipping '%s' file: %s" % (e,f))
                        skip_this_file = True
                        break
                if skip_this_file:
                    continue

                logger.info("Formatting %s" % f)
                if not self.skip_copyright:
                    check_copyright(f)
                cmds = [
                        sed + f,
                        clang_format + f,
                        astyle + f,
                        single_eol + f
                       ]

                if sys.platform == 'darwin':
                    # the temporary file creted by 'sed' will be removed here
                    cmds.append(f"rm {f}.log")

                for cmd in cmds:
                    output = self.shell.run_single_to_str(cmd)
                    if len(output) > 0:
                        logger.info(output)

            check_license("LICENSE")

        except subprocess.CalledProcessError as e:
            retval = -1
            logger.error(e)
            logger.error("OUTPUT= %s" % e.output)

        retval += self.error_on_diff("See above for clang-tidy errors")

        if retval != 0:
            raise Exception("format-code failed with error code %d" % retval)

class GenerateAndroidBP:
    def __init__(self, folder):
        self.folder = folder
        self.bp_output_file = "Generated_Android.bp"

    def run(self):
        retval = 0
        self.shell = Shell()
        self.shell.save_cwd()
        this_dir = os.path.dirname(__file__)

        logger.debug("Running Android.bp check")
        try:
            self.shell.cd(self.folder)
            cmd = "%s/generate_android_bp.py --folder %s --output_file %s" % (this_dir, self.folder, self.bp_output_file)
            output = self.shell.run_single_to_str(cmd)
            if len(output) > 0:
                logger.info(output)
        except subprocess.CalledProcessError as e:
            retval = -1
            logger.error(e)
            logger.error("OUTPUT= %s" % e.output)

        # Compare the genereated file with the one in the review
        if not filecmp.cmp(self.bp_output_file, self.folder + "/Android.bp"):
            is_mismatched = True

            with open(self.bp_output_file, 'r') as generated_file:
                with open(self.folder + "/Android.bp", 'r') as review_file:
                    diff = list(difflib.unified_diff(generated_file.readlines(), review_file.readlines(),
                                                     fromfile='Generated_Android.bp', tofile='Android.bp'))

                    # If the only mismatch in Android.bp file is the copyright year,
                    # the content of the file is considered unchanged and we don't need to update
                    # the copyright year. This will resolve the issue that emerges every new year.
                    num_added_lines = 0
                    num_removed_lines = 0
                    last_added_line = ""
                    last_removed_line = ""
                    expect_add_line = False

                    for line in diff:
                        if line.startswith("-") and not line.startswith("---"):
                            num_removed_lines += 1
                            if num_removed_lines > 1:
                                break
                            last_removed_line = line
                            expect_add_line = True
                        elif line.startswith("+") and not line.startswith("+++"):
                            num_added_lines += 1
                            if num_added_lines > 1:
                                break
                            if expect_add_line:
                                last_added_line = line
                        else:
                            expect_add_line = False

                    if num_added_lines == 1 and num_removed_lines == 1:
                        re_copyright = re.compile("^(?:\+|\-)// Copyright © ([0-9]+)\-([0-9]+) Arm Ltd. All rights reserved.\n$")
                        generated_matches = re_copyright.search(last_removed_line)
                        review_matches = re_copyright.search(last_added_line)

                        if generated_matches is not None and review_matches is not None:
                            if generated_matches.group(1) == review_matches.group(1) and \
                               int(generated_matches.group(2)) > int(review_matches.group(2)):
                                is_mismatched = False

                    if is_mismatched:
                        logger.error("Lines with '-' need to be added to Android.bp")
                        logger.error("Lines with '+' need to be removed from Android.bp")

                        for line in diff:
                            logger.error(line.rstrip())
            if is_mismatched:
                raise Exception("Android bp file is not updated")

        if retval != 0:
            raise Exception("generate Android bp file failed with error code %d" % retval)

def run_fix_code_formatting( files="git-head", folder=".", num_threads=1, error_on_diff=True):
    try:
        retval = 0

        # Genereate Android.bp file and test it
        gen_android_bp = GenerateAndroidBP(folder)
        gen_android_bp.run()

        to_check, skip_copyright = FormatCodeRun.get_files(folder, files)
        other_checks = OtherChecksRun(folder,error_on_diff, files)
        other_checks.run()

        logger.debug(to_check)
        num_files = len(to_check)
        per_thread = max( num_files / num_threads,1)
        start=0
        logger.info("Files to format:\n\t%s" % "\n\t".join(to_check))

        for i in range(num_threads):
            if i == num_threads -1:
                end = num_files
            else:
                end= min(start+per_thread, num_files)
            sub = to_check[start:end]
            logger.debug("[%d] [%d,%d] %s" % (i, start, end, sub))
            start = end
            format_code_run = FormatCodeRun(sub, folder, skip_copyright=skip_copyright)
            format_code_run.run()

        return retval
    except Exception as e:
        logger.error("Exception caught in run_fix_code_formatting: %s" % e)
        return -1

if __name__ == "__main__":
    parser = argparse.ArgumentParser(
            formatter_class=argparse.RawDescriptionHelpFormatter,
            description="Build & run pre-commit tests",
    )

    file_sources=["git-diff","git-head","all"]
    parser.add_argument("-D", "--debug", action='store_true', help="Enable script debugging output")
    parser.add_argument("--error_on_diff", action='store_true', help="Show diff on error and stop")
    parser.add_argument("--files", nargs='?', metavar="source", choices=file_sources, help="Which files to run fix_code_formatting on, choices=%s" % file_sources, default="git-head")
    parser.add_argument("--folder", metavar="path", help="Folder in which to run fix_code_formatting", default=".")

    args = parser.parse_args()

    logging_level = logging.INFO
    if args.debug:
        logging_level = logging.DEBUG

    logging.basicConfig(level=logging_level)

    logger.debug("Arguments passed: %s" % str(args.__dict__))

    exit(run_fix_code_formatting(args.files, args.folder, 1, error_on_diff=args.error_on_diff))