verif/checker/verifier.py

# Copyright (c) 2023, ARM Limited.
# SPDX-License-Identifier: Apache-2.0
"""Verfier library interface."""
import ctypes as ct
import json
from pathlib import Path

import numpy as np
import schemavalidation.schemavalidation as sch

# Type conversion from numpy to tosa_datatype_t
# "type" matches enum - see include/types.h
# "size" is size in bytes per value of this datatype
NUMPY_DATATYPE_TO_CLIENTTYPE = {
    # tosa_datatype_int32_t (all integer types are this!)
    np.dtype("int32"): {"type": 5, "size": 4},
    # tosa_datatype_int48_t (or SHAPE)
    np.dtype("int64"): {"type": 6, "size": 8},
    # tosa_datatype_fp16_t
    np.dtype("float16"): {"type": 2, "size": 2},
    # tosa_datatype_fp32_t (bf16 stored as this)
    np.dtype("float32"): {"type": 3, "size": 4},
    # tosa_datatype_fp64_t (for precise refmodel data)
    np.dtype("float64"): {"type": 99, "size": 8},
    # tosa_datatype_bool_t
    np.dtype("bool"): {"type": 1, "size": 1},
}


class TosaTensor(ct.Structure):
    _fields_ = [
        ("name", ct.c_char_p),
        ("shape", ct.POINTER(ct.c_int32)),
        ("num_dims", ct.c_int32),
        ("data_type", ct.c_int),
        ("data", ct.POINTER(ct.c_uint8)),
        ("size", ct.c_size_t),
    ]


class VerifierError(Exception):
    """Exception raised for errors performing data generation."""


class VerifierLibrary:
    """Python interface to the C verify library."""

    def __init__(self, verify_lib_path):
        """Find the library and set up the interface."""
        self.lib_path = verify_lib_path
        if not self.lib_path.is_file():
            raise VerifierError(f"Could not find verify library - {self.lib_path}")

        self.lib = ct.cdll.LoadLibrary(self.lib_path)

        self.tvf_verify_data = self.lib.tvf_verify_data
        self.tvf_verify_data.argtypes = [
            ct.POINTER(TosaTensor),  # ref
            ct.POINTER(TosaTensor),  # ref_bnd
            ct.POINTER(TosaTensor),  # imp
            ct.c_char_p,  # config_json
        ]
        self.tvf_verify_data.restype = ct.c_bool

    def _get_tensor_data(self, name, array):
        """Set up tosa_tensor_t using the given a numpy array."""
        shape = (ct.c_int32 * len(array.shape))(*array.shape)
        size_in_bytes = array.size * NUMPY_DATATYPE_TO_CLIENTTYPE[array.dtype]["size"]

        tensor = TosaTensor(
            ct.c_char_p(bytes(name, "utf8")),
            ct.cast(shape, ct.POINTER(ct.c_int32)),
            ct.c_int32(len(array.shape)),
            ct.c_int(NUMPY_DATATYPE_TO_CLIENTTYPE[array.dtype]["type"]),
            array.ctypes.data_as(ct.POINTER(ct.c_uint8)),
            ct.c_size_t(size_in_bytes),
        )
        return tensor

    def verify_data(
        self,
        output_name,
        compliance_json_config,
        imp_result_array,
        ref_result_array,
        bnd_result_array=None,
    ):
        """Verify the data using the verification library."""
        sch.TestDescSchemaValidator().validate_config(
            compliance_json_config, sch.TD_SCHEMA_COMPLIANCE
        )
        jsb = bytes(json.dumps(compliance_json_config), "utf8")

        imp = self._get_tensor_data(output_name, imp_result_array)
        ref = self._get_tensor_data(output_name, ref_result_array)
        if bnd_result_array is not None:
            ref_bnd = self._get_tensor_data(output_name, bnd_result_array)
        else:
            ref_bnd = None

        result = self.tvf_verify_data(ref, ref_bnd, imp, ct.c_char_p(jsb))

        return result


def main(argv=None):
    """Simple command line interface for the verifier library."""
    import argparse
    import conformance.model_files as cmf

    parser = argparse.ArgumentParser()

    parser.add_argument(
        "--verify-lib-path",
        type=Path,
        help="Path to TOSA verify lib",
    )
    parser.add_argument(
        "--test-desc",
        type=Path,
        help="Path to test description file: desc.json",
    )
    parser.add_argument(
        "-n",
        "--ofm-name",
        dest="ofm_name",
        type=str,
        help="output tensor name to check (defaults to only ofm_name in desc.json)",
    )
    parser.add_argument(
        "--bnd-result-path",
        type=Path,
        help="path to the reference bounds result numpy file",
    )

    parser.add_argument(
        "ref_result_path", type=Path, help="path to the reference result numpy file"
    )
    parser.add_argument(
        "imp_result_path",
        type=Path,
        help="path to the implementation result numpy file",
    )
    args = parser.parse_args(argv)

    if args.verify_lib_path is None:
        # Try to work out ref model directory and find the verify library
        # but this default only works for the python developer environment
        # i.e. when using the scripts/py-dev-env.* scripts
        # otherwise use the command line option --verify-lib-path to specify path
        ref_model_dir = Path(__file__).absolute().parents[2]
        args.verify_lib_path = cmf.find_tosa_file(
            cmf.TosaFileType.VERIFY_LIBRARY, ref_model_dir, False
        )

    if args.test_desc:
        json_path = args.test_desc
    else:
        # Assume its with the reference file
        json_path = args.ref_result_path.parent / "desc.json"

    print("Load test description")
    with json_path.open("r") as fd:
        test_desc = json.load(fd)

    if args.ofm_name is None:
        if len(test_desc["ofm_name"]) != 1:
            print("ERROR: ambiguous output to check, please specify output tensor name")
            return 2
        output_name = test_desc["ofm_name"][0]
    else:
        output_name = args.ofm_name

    if "meta" not in test_desc or "compliance" not in test_desc["meta"]:
        print(f"ERROR: no compliance meta-data found in {str(json_path)}")
        return 2

    print("Load numpy data")
    paths = [args.imp_result_path, args.ref_result_path, args.bnd_result_path]
    arrays = [None, None, None]
    for idx, path in enumerate(paths):
        if path is not None:
            array = np.load(path)
        else:
            array = None
        arrays[idx] = array

    print("Load verifier library")
    vlib = VerifierLibrary(args.verify_lib_path)

    print("Verify data")
    if vlib.verify_data(output_name, test_desc["meta"]["compliance"], *arrays):
        print("SUCCESS")
        return 0
    else:
        print("FAILURE - NOT COMPLIANT")
        return 1


if __name__ == "__main__":
    exit(main())