Fix conformance regressions due to type and region changes.
Compress flatbuffer JSON files for conformance to allow while_loop
tests to fit in the 30MB file size limit.
Do not include new SHAPE type in usable DTypes so that selection of
ERROR_IF tests are not changed.
Also enable strict json mode for flatbuffers.
Change-Id: Id89a9963244c34769a4662f9951adde0aa3334d9
Signed-off-by: Jeremy Johnson <jeremy.johnson@arm.com>
diff --git a/scripts/json2fbbin/json2fbbin.py b/scripts/json2fbbin/json2fbbin.py
index 8f9f274..08b13be 100644
--- a/scripts/json2fbbin/json2fbbin.py
+++ b/scripts/json2fbbin/json2fbbin.py
@@ -1,14 +1,69 @@
"""Conversion utility from flatbuffer JSON files to binary and the reverse."""
-# Copyright (c) 2021-2022, ARM Limited.
+# Copyright (c) 2021-2023, ARM Limited.
# SPDX-License-Identifier: Apache-2.0
+import re
from pathlib import Path
from typing import Optional
from runner.run_command import run_sh_command
from runner.run_command import RunShCommandError
+MAX_LINE_LEN = 120
+MAX_INDENT_LEN = 20
-def fbbin_to_json(flatc: Path, fbs: Path, t_path: Path, o_path: Optional[Path] = None):
+
+def json_squeeze(json_path: Path):
+ """File compression for JSONs, reducing spaces used for number lists."""
+ # Move existing file to a new name
+ temp_path = json_path.with_suffix(".json_unsqueezed")
+ json_path.rename(temp_path)
+ # Now read the original file and write a smaller output with less new lines/spaces
+ with temp_path.open("r") as tfd:
+ with json_path.open("w") as jfd:
+ found = False
+
+ for line in tfd:
+ # Find lines that are part of number lists
+ match = re.match(r"(\s+)(-?[0-9]+),?", line)
+ if match:
+ # Found a line with just a number on it (and optional comma)
+ if not found:
+ # New list of numbers
+ numbers = []
+ # Save indent (upto maximum)
+ indent = match.group(1)[0:MAX_INDENT_LEN]
+ found = True
+ numbers.append(match.group(2))
+ else:
+ # Found a line without just a number
+ if found:
+ # Format the list of numbers recorded into a concise output
+ # with multiple numbers on a single line, rather than one per line
+ numbers_str = indent
+ for num in numbers:
+ nums = f"{num},"
+ if len(numbers_str) + len(nums) > MAX_LINE_LEN:
+ print(numbers_str, file=jfd)
+ numbers_str = indent
+ numbers_str += nums
+ # print all but the last comma
+ print(numbers_str[:-1], file=jfd)
+
+ found = False
+ # print the line we just read (that wasn't just a number)
+ print(line, file=jfd, end="")
+
+ # Remove the uncompressed version
+ temp_path.unlink()
+
+
+def fbbin_to_json(
+ flatc: Path,
+ fbs: Path,
+ t_path: Path,
+ o_path: Optional[Path] = None,
+ squeeze: Optional[bool] = True,
+):
"""Convert the binary flatbuffer to JSON.
flatc: the Path to the flatc compiler program
@@ -23,6 +78,7 @@
"-o",
str(o_path.absolute()),
"--json",
+ "--strict-json",
"--defaults-json",
"--raw-binary",
str(fbs.absolute()),
@@ -30,6 +86,9 @@
str(t_path.absolute()),
]
run_sh_command(verbose=False, full_cmd=cmd)
+ if squeeze:
+ json_path = (o_path / t_path.name).with_suffix(".json")
+ json_squeeze(json_path)
def json_to_fbbin(flatc: Path, fbs: Path, j_path: Path, o_path: Optional[Path] = None):
@@ -62,6 +121,9 @@
parser = argparse.ArgumentParser()
parser.add_argument(
+ "--no-squeeze", action="store_true", help="no compression of json output"
+ )
+ parser.add_argument(
"--flatc",
type=Path,
default=(
@@ -97,7 +159,7 @@
json_to_fbbin(args.flatc, args.fbs, path)
else:
# Have to assume this is a binary flatbuffer file as could have any suffix
- fbbin_to_json(args.flatc, args.fbs, path)
+ fbbin_to_json(args.flatc, args.fbs, path, squeeze=(not args.no_squeeze))
except RunShCommandError as e:
print(e)
return 1
diff --git a/verif/generator/tosa_utils.py b/verif/generator/tosa_utils.py
index f9df8d5..3cd0370 100644
--- a/verif/generator/tosa_utils.py
+++ b/verif/generator/tosa_utils.py
@@ -93,7 +93,7 @@
Returns:
A set of DType values
"""
- omit = {DType.UNKNOWN, DType.UINT8, DType.UINT16}
+ omit = {DType.UNKNOWN, DType.UINT8, DType.UINT16, DType.SHAPE}
omit.update(excludes if excludes else ())
return allDTypes(excludes=omit)