scripts/update_supported_ops.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Copyright (c) 2024 Arm Limited.
#
# SPDX-License-Identifier: MIT
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to
# deal in the Software without restriction, including without limitation the
# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
# sell copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

"""
Updates the Doxygen documentation pages with a table of operators supported by Compute Library.

The script builds up a table in XML format internally containing the different operators and their respective supported
compute backends, data types and layouts, and the equivalent operator in the Android Neural Networks API. The list of
operators is pulled from the OperatorList.h header file and further implementation details are provided in the function
headers for the backend-specific operator e.g., NEStridedSlice.h.

Usage:
    python update_supported_ops.py
"""

import argparse
import logging
import re
from enum import Enum
from pathlib import Path


class States(Enum):
    INIT = 0
    DESCRIPTION = 1
    DESCRIPTION_END = 2
    IN_CLASS = 3
    DATA_TYPE_START = 4
    DATA_TYPE_END = 5
    NN_OPERATOR = 6
    NN_OPERATOR_END = 7
    SKIP_OPERATOR = 8
    DATA_LAYOUT_START = 9
    DATA_LAYOUT_END = 10


class OperatorsTable:
    def __init__(self):
        self.project_dir = Path(__file__).resolve().parents[1]  # ComputeLibrary directory
        self.xml = ""

    def generate_operator_list(self):
        operator_list_head_file = self.project_dir / "arm_compute" / "runtime" / "OperatorList.h"
        neon_file_name_prefix = str(self.project_dir / "arm_compute" / "runtime" / "NEON" / "functions" / "NE")
        cl_file_name_prefix = str(self.project_dir / "arm_compute" / "runtime" / "CL" / "functions" / "CL")

        logging.debug(operator_list_head_file)

        f = open(operator_list_head_file, 'r')
        # Iterates over the lines of the file
        state = States.INIT
        operator_desc = ""
        nn_op_list = []
        for line in f:
            # /** ActivationLayer
            # *
            # * Description:
            # * Function to simulate an activation layer with the specified activation function.
            # *
            # * Equivalent Android NNAPI Op:
            # * ANEURALNETWORKS_ELU
            # * ANEURALNETWORKS_HARD_SWISH
            # * ANEURALNETWORKS_LOGISTIC
            # * ANEURALNETWORKS_RELU
            # * ANEURALNETWORKS_RELU1
            # * ANEURALNETWORKS_RELU6
            # * ANEURALNETWORKS_TANH
            # *
            # */
            # Check for "/**" of the start of the operator
            r = re.search('^\s*/\*\*(.*)', line)
            if r and state == States.INIT:
                # Skip below ones
                if re.search('.*\(not ported\)', line):
                    state = States.SKIP_OPERATOR
                    continue
                if re.search('.*\(only CL\)', line):
                    state = States.SKIP_OPERATOR
                    continue
                if re.search('.*\(no CL\)', line):
                    state = States.SKIP_OPERATOR
                    continue
                if re.search('.*\(skip\)', line):
                    state = States.SKIP_OPERATOR
                    continue
            # Check" */"
            r = re.match('\s*\*/\s*$', line)
            if r and state == States.SKIP_OPERATOR:
                state = States.INIT
                continue
            # Check " *"
            r = re.match('\s*\*\s*$', line)
            if r and state == States.SKIP_OPERATOR:
                continue
            # Check non " *" lines
            r = re.search('^\s*\*(.*)', line)
            if r and state == States.SKIP_OPERATOR:
                continue

            # Check for "/**" of the start of the operator
            r = re.search('^\s*/\*\*(.*)', line)
            if r and state == States.INIT:
                tmp = r.groups()[0]
                class_name = tmp.strip()
                logging.debug(class_name)
                continue

            # Check whether "Description: " exists
            r = re.search('\s*\*\s*Description:\s*', line)
            if r and state == States.INIT:
                state = States.DESCRIPTION
                continue
            # Treat description ends with a blank line only with " *"
            r = re.match('\s*\*\s*$', line)
            if r and state == States.DESCRIPTION:
                logging.debug(operator_desc)
                state = States.DESCRIPTION_END
                continue
            # Find continuing class description in the following lines
            r = re.search('^\s*\*(.*)', line)
            if r and state == States.DESCRIPTION:
                tmp = r.groups()[0]
                operator_desc = operator_desc + ' ' + tmp.strip()
                continue

            # Check whether "Equivalent AndroidNN Op: " exists
            r = re.search('\s*\*\s*Equivalent Android NNAPI Op:\s*', line)
            if r and state == States.DESCRIPTION_END:
                state = States.NN_OPERATOR
                continue
            # Treat AndroidNN Op ends with a blank line only with " *"
            r = re.match('\s*\*\s*$', line)
            if r and state == States.NN_OPERATOR:
                logging.debug(nn_op_list)
                state = States.NN_OPERATOR_END
                # Check NE#class_name
                neon_file_name = neon_file_name_prefix + class_name + ".h"
                logging.debug(neon_file_name)
                # Check CL#class_name
                cl_file_name = cl_file_name_prefix + class_name + ".h"
                logging.debug(cl_file_name)
                # Check whether CL/Neon file exists
                if Path(neon_file_name).is_file() and Path(cl_file_name).is_file():
                    if neon_file_name.find("NEElementwiseOperations.h") != -1:
                        logging.debug(neon_file_name)
                        self.generate_operator_common_info(class_name, operator_desc, nn_op_list, "13")
                    elif neon_file_name.find("NEElementwiseUnaryLayer.h") != -1:
                        logging.debug(neon_file_name)
                        self.generate_operator_common_info(class_name, operator_desc, nn_op_list, "8")
                    else:
                        self.generate_operator_common_info(class_name, operator_desc, nn_op_list, "2")
                    self.generate_operator_info(neon_file_name)
                    self.generate_operator_cl_begin()
                    self.generate_operator_info(cl_file_name)
                else:
                    if neon_file_name.find("NELogical.h") != -1:
                        logging.debug(neon_file_name)
                        self.generate_operator_common_info(class_name, operator_desc, nn_op_list, "3")
                    else:
                        self.generate_operator_common_info(class_name, operator_desc, nn_op_list, "1")
                    if Path(neon_file_name).is_file():
                        self.generate_operator_info(neon_file_name)
                    if Path(cl_file_name).is_file():
                        self.generate_operator_info(cl_file_name)
                continue

            # Find continuing AndroidNN Op in the following lines
            r = re.search('^\s*\*(.*)', line)
            if r and state == States.NN_OPERATOR:
                tmp = r.groups()[0]
                nn_op = tmp.strip()
                nn_op_list.append(nn_op)
                continue

            # Treat operator ends with a blank line only with " */"
            r = re.match('\s*\*/\s*$', line)
            if r and state == States.NN_OPERATOR_END:
                operator_desc = ""
                nn_op_list = []
                state = States.INIT
                continue
        f.close()

    def generate_operator_info(self, file_name):
        logging.debug(file_name)
        f = open(file_name, 'r')
        # iterates over the lines of the file
        state = States.INIT
        data_type_list = []
        data_layout_list = []
        io_list = []
        class_no = 0
        for line in f:
            # Locate class definition by "class...: public IFunction",
            # There are also exceptions, which will need to support in later version
            r = re.match("\s*class\s+(\S+)\s*:\s*(public)*", line)
            if r and state == States.INIT:
                class_name = r.groups()[0]
                logging.debug("class name is %s" % (class_name))
                state = States.IN_CLASS
                continue

            r = re.match("\s*\}\;", line)
            if r and state == States.IN_CLASS:
                state = States.INIT
                continue

            # * Valid data layouts:
            # * - All
            r = re.search('\s*\*\s*Valid data layouts:', line)
            if r and state == States.IN_CLASS:
                state = States.DATA_LAYOUT_START
                continue
            # Treat data configuration ends with a blank line only with " *"
            r = re.match('\s*\*\s*$', line)
            if r and state == States.DATA_LAYOUT_START:
                state = States.DATA_LAYOUT_END
                continue
            # Data layout continues
            r = re.search('\s*\*\s*\-\s*(.*)', line)
            if r and state == States.DATA_LAYOUT_START:
                tmp = r.groups()[0]
                tmp = tmp.strip()
                logging.debug(tmp)
                data_layout_list.append(tmp)

            # * Valid data type configurations:
            # * |src0           |dst            |
            # * |:--------------|:--------------|
            # * |QASYMM8        |QASYMM8        |
            # * |QASYMM8_SIGNED |QASYMM8_SIGNED |
            # * |QSYMM16        |QSYMM16        |
            # * |F16            |F16            |
            # * |F32            |F32            |
            r = re.search('\s*\*\s*Valid data type configurations:\s*', line)
            if r and state == States.DATA_LAYOUT_END:
                state = States.DATA_TYPE_START
                logging.debug(line)
                continue
            # Treat data configuration ends with a blank line only with " *"
            r = re.match('\s*\*\s*$', line)
            if r and state == States.DATA_TYPE_START:
                logging.debug(class_name)
                logging.debug(data_layout_list)
                logging.debug(io_list)
                logging.debug(data_type_list)
                class_no = class_no + 1
                if class_no > 1:
                    logging.debug(class_no)
                    self.generate_operator_cl_begin()
                self.generate_operator_dl_dt_info(class_name, data_layout_list, io_list, data_type_list)
                state = States.INIT
                data_type_list = []
                data_layout_list = []
                continue
            # Data type continues
            r = re.search('\s*\*(.*)', line)
            if r and state == States.DATA_TYPE_START:
                tmp = r.groups()[0]
                tmp = tmp.strip()
                if re.search('\|\:\-\-\-', tmp):
                    # Skip the table split row "|:-----"
                    continue
                else:
                    tmp = tmp.strip()
                    if re.search('.*(src|input|dst)', tmp):
                        io_list = tmp.split('|')
                    else:
                        data_type = tmp.split('|')
                        logging.debug(data_type)
                        data_type_list.append(data_type)
                    continue

        f.close()

    def generate_operator_cl_begin(self):
        self.xml += "<tr>\n"

    def generate_operator_common_info(self, class_name, operator_desc, nn_op_list, rowspan):
        tmp = "<tr>\n"
        # Store class name
        tmp += "  <td rowspan=\"" + rowspan + "\">" + class_name + "\n"
        tmp += "  <td rowspan=\"" + rowspan + "\" style=\"width:200px;\">" + operator_desc + "\n"
        tmp += "  <td rowspan=\"" + rowspan + "\">\n"
        tmp += "      <ul>\n"
        for item in nn_op_list:
            tmp += "       <li>"
            tmp += item.strip()
            tmp += "\n"
        tmp += "      </ul>\n"
        self.xml += tmp

    def generate_operator_dl_dt_info(self, class_name, data_layout, io_list, data_type_list):
        tmp = "  <td>" + class_name + "\n"
        # Store data layout info
        tmp += "  <td>\n"
        tmp += "      <ul>\n"
        for item in data_layout:
            tmp += "       <li>"
            tmp += item.strip()
            tmp += "\n"
        tmp += "      </ul>\n"
        tmp += "  <td>\n"
        # Store data type table
        tmp += "    <table>\n"
        tmp += "    <tr>"
        for io in io_list:
            # Make sure it's not empty string
            if len(io) != 0:
                tmp += "<th>"
                tmp += io.strip()
        tmp += "\n"
        for item in data_type_list:
            tmp += "    <tr>"
            for i in item:
                # Make sure it's not empty string
                if len(i) != 0:
                    tmp += "<td>"
                    tmp += i.strip()
            tmp += "\n"
        tmp += "    </table>\n"
        self.xml += tmp

    def generate_table_prefix(self):
        tmp = "<table>\n"
        tmp += "<caption id=\"multi_row\"></caption>\n"
        tmp += "<tr>\n"
        tmp += "  <th>Function\n"
        tmp += "  <th>Description\n"
        tmp += "  <th>Equivalent Android NNAPI Op\n"
        tmp += "  <th>Backends\n"
        tmp += "  <th>Data Layouts\n"
        tmp += "  <th>Data Types\n"
        self.xml += tmp

    def generate_table_ending(self):
        self.xml += "</table>\n"

    def dump_xml(self):
        print(self.xml)

    def update_dox_file(self):
        operator_list_dox = self.project_dir / "docs" / "user_guide" / "operator_list.dox"

        with open(operator_list_dox, "r") as f:
            dox_content = f.read()

        # Check that there is only one non-indented table (This table should be the operator list)
        x = re.findall("\n<table>", dox_content)
        y = re.findall("\n</table>", dox_content)
        if len(x) != 1 or len(y) != 1:
            raise RuntimeError("Invalid .dox file")

        repl_str = "\n" + self.xml[:-1]  # Extra / removed "\n" characters needed to make up for search regex
        new_file = re.sub("\n<table>(.|\n)*\n<\/table>", repl_str, dox_content)

        with open(operator_list_dox, "w") as f:
            f.write(new_file)
            print("Successfully updated operator_list.dox with the XML table of supported operators.")


if __name__ == "__main__":
    parser = argparse.ArgumentParser(
        description="Updates the Compute Library documentation with a table of supported operators."
    )
    parser.add_argument(
        "--dump_xml",
        type=bool,
        default=False,
        required=False,
        help="Dump the supported operators table XML to stdout",
    )
    parser.add_argument(
        "--debug",
        type=bool,
        default=False,
        required=False,
        help="Enables logging, helpful for debugging. Default: False",
    )
    args = parser.parse_args()

    if args.debug:
        logging.basicConfig(format="%(message)s", level=logging.DEBUG)

    table_xml = OperatorsTable()
    table_xml.generate_table_prefix()
    table_xml.generate_operator_list()
    table_xml.generate_table_ending()
    table_xml.update_dox_file()

    if args.dump_xml:
        table_xml.dump_xml()