verif/runner/tosa_test_runner.py

"""Template test runner class for running TOSA tests."""
# Copyright (c) 2020-2023, ARM Limited.
# SPDX-License-Identifier: Apache-2.0
import json
from enum import IntEnum

import conformance.model_files as cmf
import schemavalidation.schemavalidation as sch
from checker.color_print import LogColors
from checker.color_print import print_color
from checker.color_print import set_print_in_color
from checker.tosa_result_checker import set_print_result
from checker.tosa_result_checker import test_check
from json2fbbin import json2fbbin
from runner.tosa_test_presets import TOSA_REFCOMPLIANCE_RUNNER


def isComplianceModeDotProduct(testDesc):
    """Checks the test descriptor for DOT_PRODUCT compliance mode."""
    if (
        "meta" in testDesc
        and "compliance" in testDesc["meta"]
        and "tensors" in testDesc["meta"]["compliance"]
    ):
        for _, t in testDesc["meta"]["compliance"]["tensors"].items():
            if "mode" in t and t["mode"] == "DOT_PRODUCT":
                return True
        return False


def getRunnerResultFilePath(resultFilePath, sutModule):
    """Return the result file path with the runner specific naming."""
    return resultFilePath.with_suffix(f".{sutModule}{resultFilePath.suffix}")


def getBoundsResultFilePath(resultFilePath, sutModule=None):
    """Return the bounds result file with/without runner specific naming."""
    boundsFilePath = resultFilePath.parent / f"bounds_{resultFilePath.name}"
    if sutModule is not None:
        boundsFilePath = boundsFilePath.with_suffix(
            f".{sutModule}{boundsFilePath.suffix}"
        )
    return boundsFilePath


class TosaTestInvalid(Exception):
    """Exception raised for errors loading test description.

    Attributes:
        path - full path to missing test description file
        exception = underlying exception
    """

    def __init__(self, path, exception):
        """Initialize test not found error."""
        self.path = path
        self.exception = exception
        self.message = "Invalid test, could not read test description {}: {}".format(
            self.path, str(self.exception)
        )
        super().__init__(self.message)


class TosaTestRunner:
    """TOSA Test Runner template class for systems under test."""

    def __init__(self, args, runnerArgs, testDirPath):
        """Initialize and load JSON meta data file."""
        self.args = args
        self.runnerArgs = runnerArgs
        self.testDir = str(testDirPath)
        self.testDirPath = testDirPath
        self.testName = self.testDirPath.name
        self.verify_lib_path = cmf.find_tosa_file(
            cmf.TosaFileType.VERIFY_LIBRARY, args.ref_model_path
        )

        set_print_in_color(not args.no_color)
        # Stop the result checker printing anything - we will do it
        set_print_result(False)

        # Check if we want to run binary and if its already converted
        descFilePath = testDirPath / "desc.json"
        descBinFilePath = testDirPath / "desc_binary.json"
        if args.binary:
            if descBinFilePath.is_file():
                descFilePath = descBinFilePath

        try:
            # Load the json test file
            with descFilePath.open("r") as fd:
                self.testDesc = json.load(fd)
            # Validate the json with the schema
            sch.TestDescSchemaValidator().validate_config(self.testDesc)
        except Exception as e:
            raise TosaTestInvalid(str(descFilePath), e)

        # Convert to binary if needed
        tosaFilePath = testDirPath / self.testDesc["tosa_file"]
        if args.binary and tosaFilePath.suffix == ".json":
            # Convert tosa JSON to binary
            json2fbbin.json_to_fbbin(
                args.flatc_path,
                args.schema_path,
                tosaFilePath,
                testDirPath,
            )
            # Write new desc_binary file
            self.testDesc["tosa_file"] = tosaFilePath.stem + ".tosa"
            with descBinFilePath.open("w") as fd:
                json.dump(self.testDesc, fd, indent=2)
            descFilePath = descBinFilePath

        # Set location of desc.json (or desc_binary.json) file in use
        self.descFile = str(descFilePath)
        self.descFilePath = descFilePath

        # Check for compliance mode - need to run refmodel to get results
        if "meta" in self.testDesc and "compliance" in self.testDesc["meta"]:
            self.complianceMode = True
            if "expected_result" in self.testDesc:
                if self.args.verbose:
                    print("Warning: fixing conflicting compliance mode in test.desc")
                self.testDesc.pop("expected_result")
        else:
            self.complianceMode = False

    def skipTest(self):
        """Check if the test is skipped due to test type or profile selection."""
        expectedFailure = self.testDesc["expected_failure"]
        if self.args.test_type == "negative" and not expectedFailure:
            return True, "non-negative type"
        elif self.args.test_type == "positive" and expectedFailure:
            return True, "non-positive type"
        if self.args.profile:
            profile = self.testDesc["profile"] if "profile" in self.testDesc else []
            if self.args.profile not in profile:
                return True, "non-{} profile".format(self.args.profile)
        return False, ""

    def runTestGraph(self):
        """Override with function that calls system under test."""
        pass

    def testResult(self, tosaGraphResult, graphMessage=None):
        """Work out test result based on graph result and output files."""
        expectedFailure = self.testDesc["expected_failure"]
        print_check_result = False

        sutModule = self.__module__

        if tosaGraphResult == TosaTestRunner.TosaGraphResult.TOSA_VALID:
            if expectedFailure:
                result = TosaTestRunner.Result.UNEXPECTED_PASS
                resultMessage = "Expected failure test incorrectly passed"
            else:
                # Work through all the results produced by the testing, assuming success
                # but overriding this with any failures found
                result = TosaTestRunner.Result.EXPECTED_PASS
                messages = []

                # Go through each output result checking it
                for resultNum, resultFileName in enumerate(self.testDesc["ofm_file"]):
                    resultFilePath = self.testDirPath / resultFileName

                    # Work out the file to check against (if any)
                    if self.complianceMode and sutModule != TOSA_REFCOMPLIANCE_RUNNER:
                        conformanceFilePath = getRunnerResultFilePath(
                            resultFilePath, TOSA_REFCOMPLIANCE_RUNNER
                        )
                        if isComplianceModeDotProduct(self.testDesc):
                            conformanceBoundsPath = getBoundsResultFilePath(
                                resultFilePath, TOSA_REFCOMPLIANCE_RUNNER
                            )
                        else:
                            # Not expecting a bounds file for this test
                            conformanceBoundsPath = None
                    elif "expected_result_file" in self.testDesc:
                        conformanceBoundsPath = None
                        try:
                            conformanceFilePath = (
                                self.testDirPath
                                / self.testDesc["expected_result_file"][resultNum]
                            )
                        except IndexError:
                            result = TosaTestRunner.Result.INTERNAL_ERROR
                            msg = "Internal error: Missing expected_result_file {} in {}".format(
                                resultNum, self.descFile
                            )
                            messages.append(msg)
                            print(msg)
                            break
                    else:
                        # Nothing to check against
                        conformanceFilePath = None
                        conformanceBoundsPath = None

                    if conformanceFilePath:
                        print_check_result = True  # Result from checker
                        chkResult, tolerance, msg = test_check(
                            conformanceFilePath,
                            resultFilePath,
                            test_name=self.testName,
                            test_desc=self.testDesc,
                            bnd_result_path=conformanceBoundsPath,
                            ofm_name=self.testDesc["ofm_name"][resultNum],
                            verify_lib_path=self.verify_lib_path,
                        )
                        # Change EXPECTED_PASS assumption if we have any failures
                        if chkResult != 0:
                            result = TosaTestRunner.Result.UNEXPECTED_FAILURE
                            messages.append(msg)
                            if self.args.verbose:
                                print(msg)
                    else:
                        # No conformance file to verify, just check results file exists
                        if not resultFilePath.is_file():
                            result = TosaTestRunner.Result.UNEXPECTED_FAILURE
                            msg = f"Results file is missing: {resultFilePath}"
                            messages.append(msg)
                            print(msg)

                    if resultFilePath.is_file():
                        # Move the resultFilePath to allow subsequent system under
                        # tests to create them and to test they have been created
                        # and to enable compliance testing against refmodel results
                        resultFilePath.rename(
                            getRunnerResultFilePath(resultFilePath, sutModule)
                        )
                        if (
                            isComplianceModeDotProduct(self.testDesc)
                            and sutModule == TOSA_REFCOMPLIANCE_RUNNER
                        ):
                            boundsFilePath = getBoundsResultFilePath(resultFilePath)
                            if boundsFilePath.is_file():
                                boundsFilePath = boundsFilePath.rename(
                                    getBoundsResultFilePath(resultFilePath, sutModule)
                                )
                            else:
                                result = TosaTestRunner.Result.INTERNAL_ERROR
                                msg = f"Internal error: Missing expected dot product compliance bounds file {boundsFilePath}"
                                messages.append(msg)
                                print(msg)

                resultMessage = "\n".join(messages) if len(messages) > 0 else None
        else:
            if (
                expectedFailure
                and tosaGraphResult == TosaTestRunner.TosaGraphResult.TOSA_ERROR
            ):
                result = TosaTestRunner.Result.EXPECTED_FAILURE
                resultMessage = None
            else:
                result = TosaTestRunner.Result.UNEXPECTED_FAILURE
                resultMessage = graphMessage

        status = "Result" if print_check_result else "Result code"
        if (
            result == TosaTestRunner.Result.EXPECTED_FAILURE
            or result == TosaTestRunner.Result.EXPECTED_PASS
        ):
            print_color(LogColors.GREEN, f"{sutModule}: {status} PASS {self.testName}")
        else:
            print_color(LogColors.RED, f"{sutModule}: {status} FAIL {self.testName}")

        return result, resultMessage

    class Result(IntEnum):
        """Test result codes."""

        EXPECTED_PASS = 0
        EXPECTED_FAILURE = 1
        UNEXPECTED_PASS = 2
        UNEXPECTED_FAILURE = 3
        INTERNAL_ERROR = 4
        SKIPPED = 5

    class TosaGraphResult(IntEnum):
        """The tosa_graph_result codes."""

        TOSA_VALID = 0
        TOSA_UNPREDICTABLE = 1
        TOSA_ERROR = 2
        OTHER_ERROR = 3