verif/tests/tosa_mock_sut_run.py

"""TOSA test runner module for a mock System Under Test (SUT)."""
# Copyright (c) 2021, ARM Limited.
# SPDX-License-Identifier: Apache-2.0
import os

from runner.run_command import run_sh_command
from runner.run_command import RunShCommandError
from runner.tosa_test_runner import TosaTestRunner


class TosaSUTRunner(TosaTestRunner):
    """TOSA mock SUT runner."""

    def __init__(self, args, runnerArgs, testDir):
        """Initialize using the given test details."""
        super().__init__(args, runnerArgs, testDir)

    def runTestGraph(self):
        """Run the test on a mock SUT."""
        # Read the command line sut-module-args in form arg=value
        # and put them in a dictionary
        # Note: On the command line (for this module) they look like:
        #       tests.tosa_mock_sut_run:arg=value
        sutArgs = {}
        for runArg in self.runnerArgs:
            try:
                arg, value = runArg.split("=", 1)
            except ValueError:
                # Argument without a value - treat it as a flag
                arg = runArg
                value = True
            sutArgs[arg] = value
        print(f"MOCK SUT: Runner argument dictionary: {sutArgs}")

        # Useful meta data and arguments
        tosaFlatbufferSchema = self.args.operator_fbs
        tosaSubgraphFile = self.testDesc["tosa_file"]
        tosaTestDirectory = self.testDir
        tosaTestDescFile = self.descFile

        # Expected file name for the graph results on valid graph
        graphResultFiles = []
        for idx, name in enumerate(self.testDesc["ofm_name"]):
            graphResultFiles.append(
                "{}:{}".format(name, self.testDesc["ofm_file"][idx])
            )

        # Build up input "tensor_name":"filename" list
        tosaInputTensors = []
        for idx, name in enumerate(self.testDesc["ifm_name"]):
            tosaInputTensors.append(
                "{}:{}".format(name, self.testDesc["ifm_file"][idx])
            )

        # Build up command line
        cmd = [
            "echo",
            f"FBS={tosaFlatbufferSchema}",
            f"Path={tosaTestDirectory}",
            f"Desc={tosaTestDescFile}",
            f"Graph={tosaSubgraphFile}",
            "Results={}".format(",".join(graphResultFiles)),
            "Inputs={}".format(",".join(tosaInputTensors)),
        ]

        # Run test on implementation
        graphResult = None
        graphMessage = None
        try:
            stdout, stderr = run_sh_command(cmd, verbose=True, capture_output=True)
        except RunShCommandError as e:
            # Return codes can be used to indicate graphResult status (see tosa_ref_run.py)
            # But in this mock version we just set the result based on sutArgs below
            print(f"MOCK SUT: Unexpected error {e.return_code} from command: {e}")
            graphResult = TosaTestRunner.TosaGraphResult.OTHER_ERROR
            graphMessage = e.stderr

        # Other mock system testing
        if self.args.binary:
            # Check that the mock binary conversion has happened
            _, ext = os.path.splitext(tosaSubgraphFile)
            if (
                os.path.basename(tosaTestDescFile) != "desc_binary.json"
                and ext != ".tosa"
            ):
                graphResult = TosaTestRunner.TosaGraphResult.OTHER_ERROR

        # Mock up graph result based on passed arguments
        if not graphResult:
            try:
                if sutArgs["graph"] == "valid":
                    graphResult = TosaTestRunner.TosaGraphResult.TOSA_VALID
                    # Create dummy output file(s) for passing result checker
                    for idx, fname in enumerate(self.testDesc["ofm_file"]):
                        if "num_results" in sutArgs and idx == int(
                            sutArgs["num_results"]
                        ):
                            # Skip writing any more to test results checker
                            break
                        print("Created " + fname)
                        fp = open(os.path.join(tosaTestDirectory, fname), "w")
                        fp.close()
                elif sutArgs["graph"] == "error":
                    graphResult = TosaTestRunner.TosaGraphResult.TOSA_ERROR
                    graphMessage = "MOCK SUT: ERROR_IF"
                elif sutArgs["graph"] == "unpredictable":
                    graphResult = TosaTestRunner.TosaGraphResult.TOSA_UNPREDICTABLE
                    graphMessage = "MOCK SUT: UNPREDICTABLE"
                else:
                    graphResult = TosaTestRunner.TosaGraphResult.OTHER_ERROR
                    graphMessage = "MOCK SUT: error from system under test"
            except KeyError:
                graphMessage = "MOCK SUT: No graph result specified!"
                print(graphMessage)
                graphResult = TosaTestRunner.TosaGraphResult.OTHER_ERROR

        # Return graph result and message
        return graphResult, graphMessage