scripts/json2fbbin/json2fbbin.py

"""Conversion utility from flatbuffer JSON files to binary and the reverse."""
# Copyright (c) 2021-2023, ARM Limited.
# SPDX-License-Identifier: Apache-2.0
import re
from pathlib import Path
from typing import Optional

from runner.run_command import run_sh_command
from runner.run_command import RunShCommandError

MAX_LINE_LEN = 120
MAX_INDENT_LEN = 20


def json_squeeze(json_path: Path):
    """File compression for JSONs, reducing spaces used for number lists."""
    # Move existing file to a new name
    temp_path = json_path.with_suffix(".json_unsqueezed")
    json_path.rename(temp_path)
    # Now read the original file and write a smaller output with less new lines/spaces
    with temp_path.open("r") as tfd:
        with json_path.open("w") as jfd:
            found = False

            for line in tfd:
                # Find lines that are part of number lists
                match = re.match(r"(\s+)(-?[0-9]+),?", line)
                if match:
                    # Found a line with just a number on it (and optional comma)
                    if not found:
                        # New list of numbers
                        numbers = []
                        # Save indent (upto maximum)
                        indent = match.group(1)[0:MAX_INDENT_LEN]
                        found = True
                    numbers.append(match.group(2))
                else:
                    # Found a line without just a number
                    if found:
                        # Format the list of numbers recorded into a concise output
                        # with multiple numbers on a single line, rather than one per line
                        numbers_str = indent
                        for num in numbers:
                            nums = f"{num},"
                            if len(numbers_str) + len(nums) > MAX_LINE_LEN:
                                print(numbers_str, file=jfd)
                                numbers_str = indent
                            numbers_str += nums
                        # print all but the last comma
                        print(numbers_str[:-1], file=jfd)

                        found = False
                    # print the line we just read (that wasn't just a number)
                    print(line, file=jfd, end="")

    # Remove the uncompressed version
    temp_path.unlink()


def fbbin_to_json(
    flatc: Path,
    fbs: Path,
    t_path: Path,
    o_path: Optional[Path] = None,
    squeeze: Optional[bool] = True,
):
    """Convert the binary flatbuffer to JSON.

    flatc: the Path to the flatc compiler program
    fbs: the Path to the fbs (flatbuffer schema) file
    t_path: the Path to the binary flatbuffer file
    o_path: the output Path where JSON file will be put, if None, it is same as t_path
    """
    if o_path is None:
        o_path = t_path.parent
    cmd = [
        str(flatc.absolute()),
        "-o",
        str(o_path.absolute()),
        "--json",
        "--strict-json",
        "--defaults-json",
        "--raw-binary",
        str(fbs.absolute()),
        "--",
        str(t_path.absolute()),
    ]
    run_sh_command(verbose=False, full_cmd=cmd)
    if squeeze:
        json_path = (o_path / t_path.name).with_suffix(".json")
        json_squeeze(json_path)


def json_to_fbbin(flatc: Path, fbs: Path, j_path: Path, o_path: Optional[Path] = None):
    """Convert JSON flatbuffer to binary.

    flatc: the Path to the flatc compiler program
    fbs: the Path to the fbs (flatbuffer schema) file
    j_path: the Path to the JSON flatbuffer file
    o_path: the output Path where JSON file will be put, if None, it is same as j_path
    """
    if o_path is None:
        o_path = j_path.parent
    cmd = [
        str(flatc.absolute()),
        "-o",
        str(o_path.absolute()),
        "--binary",
        str(fbs.absolute()),
        str(j_path.absolute()),
    ]
    run_sh_command(verbose=False, full_cmd=cmd)


# ------------------------------------------------------------------------------


def main(argv=None):
    """Load and convert supplied file based on file suffix."""
    import argparse

    parser = argparse.ArgumentParser()
    parser.add_argument(
        "--no-squeeze", action="store_true", help="no compression of json output"
    )
    parser.add_argument(
        "--flatc",
        type=Path,
        default=(
            "reference_model/build/thirdparty/serialization_lib/"
            "third_party/flatbuffers/flatc"
        ),
        help="the path to the flatc compiler program",
    )
    parser.add_argument(
        "--fbs",
        type=Path,
        default="conformance_tests/third_party/serialization_lib/schema/tosa.fbs",
        help="the path to the flatbuffer schema",
    )
    parser.add_argument("path", type=Path, help="the path to the file to convert")
    args = parser.parse_args(argv)
    path = args.path

    if not path.is_file():
        print(f"Invalid file to convert - {path}")
        return 2

    if not args.flatc.is_file():
        print(f"Invalid flatc compiler - {args.flatc}")
        return 2

    if not args.fbs.is_file():
        print(f"Invalid flatbuffer schema - {args.fbs}")
        return 2

    try:
        if path.suffix == ".json":
            json_to_fbbin(args.flatc, args.fbs, path)
        else:
            # Have to assume this is a binary flatbuffer file as could have any suffix
            fbbin_to_json(args.flatc, args.fbs, path, squeeze=(not args.no_squeeze))
    except RunShCommandError as e:
        print(e)
        return 1

    return 0


if __name__ == "__main__":
    exit(main())