Refactor path arguments to tosa-tools
tosa_verif_conformance_generator
- Move to using ref-model-path instead of ref-model-dir
- Add schema-path and flatc-path
- Add model_files module to locate default places for files
convert2conformance
- Remove default paths
verifier
- Switch to using exact path of verifier library
tosa_verif_run_tests
- Use conformance model_files to locate defaults
Change-Id: Ieca3b164670e2a7dcb047743667cc4e8317daa97
Signed-off-by: Josef Malmström <josef.malmstrom@arm.com>
Signed-off-by: Jeremy Johnson <jeremy.johnson@arm.com>
diff --git a/scripts/convert2conformance/convert2conformance.py b/scripts/convert2conformance/convert2conformance.py
index adb163f..3654f9a 100755
--- a/scripts/convert2conformance/convert2conformance.py
+++ b/scripts/convert2conformance/convert2conformance.py
@@ -22,10 +22,6 @@
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger("convert2conformance")
-LOCATION_REF_MODEL_SCHEMA = Path("thirdparty/serialization_lib/schema/tosa.fbs")
-LOCATION_REF_MODEL_FLATC = Path(
- "build/thirdparty/serialization_lib/third_party/flatbuffers/flatc"
-)
NAME_FLATBUFFER_DIR = ["flatbuffer-", "_FW_"]
NAME_DESC_FILENAME = "desc.json"
@@ -47,11 +43,19 @@
help="The test directory to convert (default is CWD)",
)
parser.add_argument(
- "--ref-model-directory",
- dest="ref_model_dir",
+ "--schema-path",
+ "--operator-fbs",
+ dest="schema_path",
type=Path,
required=True,
- help="Reference Model directory (must be pre-built)",
+ help=("Path to reference model schema."),
+ )
+ parser.add_argument(
+ "--flatc-path",
+ dest="flatc_path",
+ type=Path,
+ required=True,
+ help=("Path to flatc executable."),
)
parser.add_argument(
"--output-directory",
@@ -107,21 +111,6 @@
return args
-def find_ref_model_artifacts(path: Path):
- """Check the location of the flatc compiler and schema artifacts."""
- flatc = path / LOCATION_REF_MODEL_FLATC
- schema = path / LOCATION_REF_MODEL_SCHEMA
- if not flatc.is_file():
- raise Exception(
- f"flatc not found in {flatc}\nHave you built the flatbuffers compiler?"
- )
- if not schema.is_file():
- raise Exception(
- f"TOSA schema not found at {schema}\nHave you checked out the submodules?"
- )
- return flatc, schema
-
-
def find_framework_artifacts(framework: str, schema_path: Path, desc_file: Path):
"""Check that any required schema has been supplied for conversion."""
if framework == "tflite":
@@ -240,10 +229,11 @@
logger.setLevel(logging.DEBUG)
# Check we can get the files we need
- try:
- flatc, schema = find_ref_model_artifacts(args.ref_model_dir)
- except Exception as err:
- logger.error(err)
+ if not args.flatc_path.is_file():
+ logger.error("flatc not found at %s", args.flatc_path)
+ return 2
+ if not args.schema_path.is_file():
+ logger.error("TOSA schema not found at %s", args.schema_path)
return 2
# Work out where the desc.json file is
@@ -307,7 +297,7 @@
# Convert the TOSA flatbuffer binary
tosa_filename = desc_filename.parent / test_desc["tosa_file"]
tosa_filename = convert_flatbuffer_file(
- flatc, schema, tosa_filename, args.output_dir
+ args.flatc_path, args.schema_path, tosa_filename, args.output_dir
)
if not tosa_filename:
# Failed to convert the file, json2fbbin will have printed an error
@@ -319,7 +309,7 @@
if framework_conversion and framework_filename:
# Convert the framework flatbuffer binary
framework_filename = convert_flatbuffer_file(
- flatc, framework_schema, framework_filename, args.output_dir
+ args.flatc_path, framework_schema, framework_filename, args.output_dir
)
if not framework_filename:
# Failed to convert the file, json2fbbin will have printed an error
diff --git a/verif/checker/verifier.py b/verif/checker/verifier.py
index 3a86bab..06ffcfb 100644
--- a/verif/checker/verifier.py
+++ b/verif/checker/verifier.py
@@ -4,20 +4,10 @@
import ctypes as ct
import json
from pathlib import Path
-from typing import Optional
import numpy as np
import schemavalidation.schemavalidation as sch
-
-# Default library info
-SCRIPT = Path(__file__).absolute()
-# NOTE: This REFMODEL_DIR default only works for the python developer environment
-# i.e. when using the scripts/py-dev-env.* scripts
-# otherwise use the command line option --ref-model-directory to specify path
-REFMODEL_DIR = SCRIPT.parents[2]
-LIBRARY = "libtosa_reference_verify_lib.so"
-
# Type conversion from numpy to tosa_datatype_t
# "type" matches enum - see include/types.h
# "size" is size in bytes per value of this datatype
@@ -55,18 +45,12 @@
class VerifierLibrary:
"""Python interface to the C verify library."""
- def __init__(self, path: Optional[Path] = None):
+ def __init__(self, verify_lib_path):
"""Find the library and set up the interface."""
- if path is None:
- path = REFMODEL_DIR
- lib_paths = sorted(path.glob(f"**/{LIBRARY}"))
+ self.lib_path = verify_lib_path
+ if not self.lib_path.is_file():
+ raise VerifierError(f"Could not find verify library - {self.lib_path}")
- if len(lib_paths) < 1:
- raise VerifierError(
- f"Could not find {LIBRARY} - have you built the ref-model?"
- )
-
- self.lib_path = lib_paths[0]
self.lib = ct.cdll.LoadLibrary(self.lib_path)
self.tvf_verify_data = self.lib.tvf_verify_data
@@ -122,14 +106,14 @@
def main(argv=None):
"""Simple command line interface for the verifier library."""
import argparse
+ import conformance.model_files as cmf
parser = argparse.ArgumentParser()
+
parser.add_argument(
- "--ref-model-directory",
- dest="ref_model_dir",
- default=REFMODEL_DIR,
+ "--verify-lib-path",
type=Path,
- help="Path to pre-built reference model directory",
+ help="Path to TOSA verify lib",
)
parser.add_argument(
"--test-desc",
@@ -159,6 +143,16 @@
)
args = parser.parse_args(argv)
+ if args.verify_lib_path is None:
+ # Try to work out ref model directory and find the verify library
+ # but this default only works for the python developer environment
+ # i.e. when using the scripts/py-dev-env.* scripts
+ # otherwise use the command line option --verify-lib-path to specify path
+ ref_model_dir = Path(__file__).absolute().parents[2]
+ args.verify_lib_path = cmf.find_tosa_file(
+ cmf.TosaFileType.VERIFY_LIBRARY, ref_model_dir, False
+ )
+
if args.test_desc:
json_path = args.test_desc
else:
@@ -192,7 +186,7 @@
arrays[idx] = array
print("Load verifier library")
- vlib = VerifierLibrary(args.ref_model_dir)
+ vlib = VerifierLibrary(args.verify_lib_path)
print("Verify data")
if vlib.verify_data(output_name, test_desc["meta"]["compliance"], *arrays):
diff --git a/verif/conformance/model_files.py b/verif/conformance/model_files.py
new file mode 100644
index 0000000..a9e1db3
--- /dev/null
+++ b/verif/conformance/model_files.py
@@ -0,0 +1,79 @@
+# Copyright (c) 2023, ARM Limited.
+# SPDX-License-Identifier: Apache-2.0
+"""Locate model files helper functions."""
+from enum import IntEnum
+from pathlib import Path
+
+DEFAULT_REF_MODEL_SCHEMA_PATH = Path("thirdparty/serialization_lib/schema")
+DEFAULT_REF_MODEL_BUILD_FLATC_PATH = Path(
+ "thirdparty/serialization_lib/third_party/flatbuffers"
+)
+DEFAULT_REF_MODEL_BUILD_EXE_PATH = Path("reference_model")
+DEFAULT_BUILD_DIR = Path("build")
+
+
+class TosaFileType(IntEnum):
+ """TOSA file types."""
+
+ REF_MODEL = 0
+ SCHEMA = 1
+ FLATC = 2
+ VERIFY_LIBRARY = 3
+
+
+TOSA_FILE_TYPE_TO_DETAILS = {
+ TosaFileType.REF_MODEL: {
+ "name": "tosa_reference_model",
+ "location": DEFAULT_REF_MODEL_BUILD_EXE_PATH,
+ "build": True,
+ },
+ TosaFileType.SCHEMA: {
+ "name": "tosa.fbs",
+ "location": DEFAULT_REF_MODEL_SCHEMA_PATH,
+ "build": False,
+ },
+ TosaFileType.FLATC: {
+ "name": "flatc",
+ "location": DEFAULT_REF_MODEL_BUILD_FLATC_PATH,
+ "build": True,
+ },
+ TosaFileType.VERIFY_LIBRARY: {
+ "name": "libtosa_reference_verify_lib.so",
+ "location": DEFAULT_REF_MODEL_BUILD_EXE_PATH,
+ "build": True,
+ },
+}
+
+
+def find_tosa_file(file_type, ref_model_path, path_is_ref_model_exe=True):
+ """Return the possible path to the required tosa file type."""
+ name = TOSA_FILE_TYPE_TO_DETAILS[file_type]["name"]
+ location = TOSA_FILE_TYPE_TO_DETAILS[file_type]["location"]
+ build = TOSA_FILE_TYPE_TO_DETAILS[file_type]["build"]
+
+ if path_is_ref_model_exe:
+ # Given a path to the reference_model executable
+
+ # Special case - return what we have been given!
+ if file_type == TosaFileType.REF_MODEL:
+ return ref_model_path
+
+ try:
+ if build:
+ # Look in build directory
+ search_path = ref_model_path.parents[1]
+ else:
+ # Look in reference_model directory
+ search_path = ref_model_path.parents[2]
+ except IndexError:
+ search_path = ref_model_path.parent
+ else:
+ # Given a path to the reference_model directory
+ if build:
+ search_path = ref_model_path / DEFAULT_BUILD_DIR
+ else:
+ search_path = ref_model_path
+
+ search_path = search_path / location / name
+
+ return search_path
diff --git a/verif/conformance/tosa_verif_conformance_generator.py b/verif/conformance/tosa_verif_conformance_generator.py
index 236f729..564617b 100644
--- a/verif/conformance/tosa_verif_conformance_generator.py
+++ b/verif/conformance/tosa_verif_conformance_generator.py
@@ -23,6 +23,7 @@
from itertools import tee
from pathlib import Path
+import conformance.model_files as cmf
from conformance.test_select import Operator
from convert2conformance.convert2conformance import main as c2c_main
from distutils.dir_util import copy_tree
@@ -44,8 +45,6 @@
}
PROFILES_ALL = "all"
-LOCATION_REF_MODEL_BINARY = Path("build/reference_model/tosa_reference_model")
-
DEFAULT_SEED = 42
# When there is a dictionary of generator argument lists (groups) only the
@@ -155,7 +154,7 @@
error = False
for i, cmd in enumerate(build_cmds_list):
try:
- _run_sh_command(args, args.ref_model_dir.absolute(), cmd)
+ _run_sh_command(args, args.ref_model_path.parent.absolute(), cmd)
logger.info(
f"{operator} test batch {(i+1)}/{len(build_cmds_list)} created successfully"
)
@@ -217,11 +216,10 @@
num_cores = args.num_cores
run_tests_cmd = "tosa_verif_run_tests"
- ref_model_path = args.ref_model_dir / LOCATION_REF_MODEL_BINARY
ref_cmd_base = ref_cmd = [
run_tests_cmd,
"--ref-model-path",
- str(ref_model_path.absolute()),
+ str(args.ref_model_path.absolute()),
"-j",
str(num_cores),
"-v",
@@ -244,7 +242,7 @@
failed_counter = 0
job_pool = mp.Pool(args.num_cores)
- sh_partial = partial(_run_sh_command, args, args.ref_model_dir.absolute())
+ sh_partial = partial(_run_sh_command, args, args.ref_model_path.parent.absolute())
pool_results = job_pool.map(sh_partial, ref_cmds)
job_pool.close()
job_pool.join()
@@ -275,12 +273,12 @@
tags=None,
):
"""Convert tests to JSON and save to output directory."""
- ref_model_dir = args.ref_model_dir
-
if group:
output_dir = output_dir / group
- c2c_args_base = ["--strict", "--ref-model-directory", str(ref_model_dir)]
+ c2c_args_base = ["--strict"]
+ c2c_args_base.extend(["--schema-path", str(args.schema_path)])
+ c2c_args_base.extend(["--flatc-path", str(args.flatc_path)])
# This op maybe in more than one profile - e.g. tosa_bi and tosa_mi
# even if we are only producing tests for tosa_mi
for op_profile in op_profiles_list:
@@ -470,6 +468,7 @@
)
parser.add_argument(
"--operators",
+ "--op",
type=str,
nargs="*",
help="The operator(s) to create tests for, if not supplied all tests will be created",
@@ -495,12 +494,37 @@
action="store_true",
help="Enable lazy data generation (only for tosa-mi)",
)
- parser.add_argument(
+ rm_group = parser.add_mutually_exclusive_group(required=True)
+ rm_group.add_argument(
"--ref-model-directory",
dest="ref_model_dir",
type=Path,
- required=True,
- help="Reference Model directory (must be pre-built)",
+ help="(DEPRECATED - use ref-model-path) Reference Model directory - with build directory",
+ )
+ rm_group.add_argument(
+ "--ref-model-path",
+ dest="ref_model_path",
+ type=Path,
+ help="Path to TOSA reference model executable",
+ )
+ parser.add_argument(
+ "--schema-path",
+ "--operator-fbs",
+ dest="schema_path",
+ type=Path,
+ help=(
+ "Path to TOSA reference model flat buffer schema. Defaults to "
+ f"`{cmf.DEFAULT_REF_MODEL_SCHEMA_PATH}` in parents parent directory of `ref-model-path`"
+ ),
+ )
+ parser.add_argument(
+ "--flatc-path",
+ dest="flatc_path",
+ type=Path,
+ help=(
+ "Path to flatc executable. Defaults to "
+ f"`{cmf.DEFAULT_REF_MODEL_BUILD_FLATC_PATH}` in parent directory of `ref-model-path`"
+ ),
)
parser.add_argument(
"--seed",
@@ -581,18 +605,35 @@
def main():
args = parse_args()
- if not args.ref_model_dir.is_dir():
+ if args.ref_model_dir is not None:
+ # Assume the ref model exe path based on the ref model directory
+ args.ref_model_path = cmf.find_tosa_file(
+ cmf.TosaFileType.REF_MODEL, args.ref_model_dir, False
+ )
+ if not args.ref_model_path.is_file():
logger.error(
- f"Missing or invalid reference model directory: {args.ref_model_dir}"
+ f"Missing reference model binary (--ref-model-path): {args.ref_model_path}"
)
return 2
- else:
- ref_model = args.ref_model_dir / LOCATION_REF_MODEL_BINARY
- if not ref_model.is_file():
- logger.error(
- f"{LOCATION_REF_MODEL_BINARY} not found in {args.ref_model_dir}\nHave you built the reference model?"
- )
- return 2
+
+ if args.schema_path is None:
+ args.schema_path = cmf.find_tosa_file(
+ cmf.TosaFileType.SCHEMA, args.ref_model_path
+ )
+ if not args.schema_path.is_file():
+ logger.error(
+ f"Missing reference model schema (--schema-path): {args.schema_path}"
+ )
+ return 2
+
+ if args.flatc_path is None:
+ args.flatc_path = cmf.find_tosa_file(
+ cmf.TosaFileType.FLATC, args.ref_model_path
+ )
+ if not args.flatc_path.is_file():
+ logger.error(f"Missing flatc binary (--flatc-path): {args.flatc_path}")
+ return 2
+
if args.unit_tests in ["framework", "both"]:
logger.warning(
"DEPRECATION - Framework tests are not part of TOSA conformance testing"
diff --git a/verif/runner/tosa_refmodel_sut_run.py b/verif/runner/tosa_refmodel_sut_run.py
index 7b129da..419f87b 100644
--- a/verif/runner/tosa_refmodel_sut_run.py
+++ b/verif/runner/tosa_refmodel_sut_run.py
@@ -1,5 +1,5 @@
"""TOSA test runner module for the Reference Model."""
-# Copyright (c) 2020-2022, ARM Limited.
+# Copyright (c) 2020-2023, ARM Limited.
# SPDX-License-Identifier: Apache-2.0
from enum import IntEnum
from enum import unique
@@ -33,10 +33,10 @@
# Call Reference model with description file to provide all file details
cmd = [
- args.ref_model_path,
- "--tosa_level={}".format(args.tosa_level),
- "--operator_fbs={}".format(args.operator_fbs),
- "--test_desc={}".format(self.descFile),
+ str(args.ref_model_path),
+ f"--tosa_level={args.tosa_level}",
+ f"--operator_fbs={str(args.schema_path)}",
+ f"--test_desc={self.descFile}",
]
# Specific debug options for reference model
diff --git a/verif/runner/tosa_test_runner.py b/verif/runner/tosa_test_runner.py
index d8c2a87..579dd60 100644
--- a/verif/runner/tosa_test_runner.py
+++ b/verif/runner/tosa_test_runner.py
@@ -1,9 +1,8 @@
"""Template test runner class for running TOSA tests."""
-# Copyright (c) 2020-2022, ARM Limited.
+# Copyright (c) 2020-2023, ARM Limited.
# SPDX-License-Identifier: Apache-2.0
import json
from enum import IntEnum
-from pathlib import Path
from checker.tosa_result_checker import LogColors
from checker.tosa_result_checker import print_color
@@ -62,8 +61,8 @@
if args.binary and tosaFilePath.suffix == ".json":
# Convert tosa JSON to binary
json2fbbin.json_to_fbbin(
- Path(args.flatc_path),
- Path(args.operator_fbs),
+ args.flatc_path,
+ args.schema_path,
tosaFilePath,
testDirPath,
)
diff --git a/verif/runner/tosa_verif_run_tests.py b/verif/runner/tosa_verif_run_tests.py
index 814c864..722c0e7 100644
--- a/verif/runner/tosa_verif_run_tests.py
+++ b/verif/runner/tosa_verif_run_tests.py
@@ -1,5 +1,5 @@
"""TOSA verification runner script."""
-# Copyright (c) 2020-2022, ARM Limited.
+# Copyright (c) 2020-2023, ARM Limited.
# SPDX-License-Identifier: Apache-2.0
import argparse
import importlib
@@ -10,6 +10,7 @@
from datetime import datetime
from pathlib import Path
+import conformance.model_files as cmf
from json2numpy import json2numpy
from runner.tosa_test_runner import TosaTestInvalid
from runner.tosa_test_runner import TosaTestRunner
@@ -46,25 +47,29 @@
help="Recursively search for tests",
)
parser.add_argument(
- "--operator-fbs",
- dest="operator_fbs",
- default="conformance_tests/third_party/serialization_lib/schema/tosa.fbs",
- type=str,
- help="flat buffer syntax file",
- )
- parser.add_argument(
"--ref-model-path",
dest="ref_model_path",
- default="reference_model/build/reference_model/tosa_reference_model",
- type=str,
- help="Path to reference model executable",
+ type=Path,
+ help="Path to TOSA reference model executable",
+ )
+ parser.add_argument(
+ "--operator-fbs",
+ "--schema-path",
+ dest="schema_path",
+ type=Path,
+ help=(
+ "Path to TOSA reference model flat buffer schema. Defaults to "
+ f"`{cmf.DEFAULT_REF_MODEL_SCHEMA_PATH}` in parents parent directory of `ref-model-path`"
+ ),
)
parser.add_argument(
"--flatc-path",
dest="flatc_path",
- default="reference_model/build/thirdparty/serialization_lib/third_party/flatbuffers/flatc",
- type=str,
- help="Path to flatc compiler executable",
+ type=Path,
+ help=(
+ "Path to flatc executable. Defaults to "
+ f"`{cmf.DEFAULT_REF_MODEL_BUILD_FLATC_PATH}` in parent directory of `ref-model-path`"
+ ),
)
parser.add_argument(
"--ref-debug",
@@ -339,13 +344,22 @@
"""Start worker threads to do the testing and outputs the results."""
args = parseArgs(argv)
- if (
- TOSA_REFMODEL_RUNNER in args.sut_module
- and not Path(args.ref_model_path).is_file()
- ):
- print(
- "Argument error: Reference Model not found ({})".format(args.ref_model_path)
+ # Set up some defaults
+ if args.ref_model_path is None:
+ args.ref_model_path = cmf.find_tosa_file(
+ cmf.TosaFileType.REF_MODEL, Path("reference_model"), False
)
+ if args.flatc_path is None:
+ args.flatc_path = cmf.find_tosa_file(
+ cmf.TosaFileType.FLATC, args.ref_model_path
+ )
+ if args.schema_path is None:
+ args.schema_path = cmf.find_tosa_file(
+ cmf.TosaFileType.SCHEMA, args.ref_model_path
+ )
+
+ if TOSA_REFMODEL_RUNNER in args.sut_module and not args.ref_model_path.is_file():
+ print(f"Argument error: Reference Model not found - {str(args.ref_model_path)}")
exit(2)
if args.test_list_file:
diff --git a/verif/tests/test_tosa_run_tests_args.py b/verif/tests/test_tosa_run_tests_args.py
index a0c3ed5..e6c5001 100644
--- a/verif/tests/test_tosa_run_tests_args.py
+++ b/verif/tests/test_tosa_run_tests_args.py
@@ -1,5 +1,5 @@
"""Tests for tosa_verif_run_tests.py."""
-# Copyright (c) 2021-2022, ARM Limited.
+# Copyright (c) 2021-2023, ARM Limited.
# SPDX-License-Identifier: Apache-2.0
from runner.tosa_verif_run_tests import parseArgs
@@ -15,7 +15,7 @@
"""Test arguments - ref_model_path."""
args = ["--ref-model-path", "ref_model_path", "-t", "test"]
parsed_args = parseArgs(args)
- assert parsed_args.ref_model_path == "ref_model_path"
+ assert str(parsed_args.ref_model_path) == "ref_model_path"
def test_args_ref_debug():
diff --git a/verif/tests/test_tosa_verifier.py b/verif/tests/test_tosa_verifier.py
index 9524158..996939c 100644
--- a/verif/tests/test_tosa_verifier.py
+++ b/verif/tests/test_tosa_verifier.py
@@ -26,12 +26,12 @@
def test_checker_verifier_load_fail():
with pytest.raises(VerifierError) as excinfo:
VerifierLibrary(Path("/place-that-does-not-exist"))
- assert str(excinfo.value).startswith(f"Could not find {VERIFIER_LIB}")
+ assert str(excinfo.value).startswith("Could not find verify library")
@pytest.mark.postcommit
def test_checker_verifier_load():
- vlib = VerifierLibrary(VERIFIER_LIB_PATH.parent)
+ vlib = VerifierLibrary(VERIFIER_LIB_PATH)
assert vlib
@@ -48,7 +48,7 @@
@pytest.mark.postcommit
def test_checker_verifier_dot_product_check():
- vlib = VerifierLibrary(VERIFIER_LIB_PATH.parent)
+ vlib = VerifierLibrary(VERIFIER_LIB_PATH)
assert vlib
imp_arr = np.zeros((10, 10, 10), dtype=np.float32)
@@ -63,7 +63,7 @@
@pytest.mark.postcommit
def test_checker_verifier_dot_product_check_fail():
- vlib = VerifierLibrary(VERIFIER_LIB_PATH.parent)
+ vlib = VerifierLibrary(VERIFIER_LIB_PATH)
assert vlib
imp_arr = np.zeros((10, 10, 10), dtype=np.float32)
diff --git a/verif/tests/tosa_mock_sut_run.py b/verif/tests/tosa_mock_sut_run.py
index 9572618..295ee92 100644
--- a/verif/tests/tosa_mock_sut_run.py
+++ b/verif/tests/tosa_mock_sut_run.py
@@ -1,5 +1,5 @@
"""TOSA test runner module for a mock System Under Test (SUT)."""
-# Copyright (c) 2021, ARM Limited.
+# Copyright (c) 2021-2023, ARM Limited.
# SPDX-License-Identifier: Apache-2.0
import os
@@ -33,7 +33,7 @@
print(f"MOCK SUT: Runner argument dictionary: {sutArgs}")
# Useful meta data and arguments
- tosaFlatbufferSchema = self.args.operator_fbs
+ tosaFlatbufferSchema = str(self.args.schema_path)
tosaSubgraphFile = self.testDesc["tosa_file"]
tosaTestDirectory = self.testDir
tosaTestDescFile = self.descFile