Jeremy Johnson | be1a940 | 2021-12-15 17:14:56 +0000 | [diff] [blame] | 1 | """Conversion utility from flatbuffer JSON files to binary and the reverse.""" |
Jeremy Johnson | 0633c3a | 2023-08-22 16:55:08 +0100 | [diff] [blame] | 2 | # Copyright (c) 2021-2023, ARM Limited. |
Jeremy Johnson | be1a940 | 2021-12-15 17:14:56 +0000 | [diff] [blame] | 3 | # SPDX-License-Identifier: Apache-2.0 |
Jeremy Johnson | 0633c3a | 2023-08-22 16:55:08 +0100 | [diff] [blame] | 4 | import re |
Jeremy Johnson | be1a940 | 2021-12-15 17:14:56 +0000 | [diff] [blame] | 5 | from pathlib import Path |
| 6 | from typing import Optional |
| 7 | |
Jeremy Johnson | 5c1364c | 2022-01-13 15:04:21 +0000 | [diff] [blame] | 8 | from runner.run_command import run_sh_command |
| 9 | from runner.run_command import RunShCommandError |
Jeremy Johnson | be1a940 | 2021-12-15 17:14:56 +0000 | [diff] [blame] | 10 | |
Jeremy Johnson | 0633c3a | 2023-08-22 16:55:08 +0100 | [diff] [blame] | 11 | MAX_LINE_LEN = 120 |
| 12 | MAX_INDENT_LEN = 20 |
Jeremy Johnson | be1a940 | 2021-12-15 17:14:56 +0000 | [diff] [blame] | 13 | |
Jeremy Johnson | 0633c3a | 2023-08-22 16:55:08 +0100 | [diff] [blame] | 14 | |
| 15 | def json_squeeze(json_path: Path): |
| 16 | """File compression for JSONs, reducing spaces used for number lists.""" |
| 17 | # Move existing file to a new name |
| 18 | temp_path = json_path.with_suffix(".json_unsqueezed") |
| 19 | json_path.rename(temp_path) |
| 20 | # Now read the original file and write a smaller output with less new lines/spaces |
| 21 | with temp_path.open("r") as tfd: |
| 22 | with json_path.open("w") as jfd: |
| 23 | found = False |
| 24 | |
| 25 | for line in tfd: |
| 26 | # Find lines that are part of number lists |
| 27 | match = re.match(r"(\s+)(-?[0-9]+),?", line) |
| 28 | if match: |
| 29 | # Found a line with just a number on it (and optional comma) |
| 30 | if not found: |
| 31 | # New list of numbers |
| 32 | numbers = [] |
| 33 | # Save indent (upto maximum) |
| 34 | indent = match.group(1)[0:MAX_INDENT_LEN] |
| 35 | found = True |
| 36 | numbers.append(match.group(2)) |
| 37 | else: |
| 38 | # Found a line without just a number |
| 39 | if found: |
| 40 | # Format the list of numbers recorded into a concise output |
| 41 | # with multiple numbers on a single line, rather than one per line |
| 42 | numbers_str = indent |
| 43 | for num in numbers: |
| 44 | nums = f"{num}," |
| 45 | if len(numbers_str) + len(nums) > MAX_LINE_LEN: |
| 46 | print(numbers_str, file=jfd) |
| 47 | numbers_str = indent |
| 48 | numbers_str += nums |
| 49 | # print all but the last comma |
| 50 | print(numbers_str[:-1], file=jfd) |
| 51 | |
| 52 | found = False |
| 53 | # print the line we just read (that wasn't just a number) |
| 54 | print(line, file=jfd, end="") |
| 55 | |
| 56 | # Remove the uncompressed version |
| 57 | temp_path.unlink() |
| 58 | |
| 59 | |
| 60 | def fbbin_to_json( |
| 61 | flatc: Path, |
| 62 | fbs: Path, |
| 63 | t_path: Path, |
| 64 | o_path: Optional[Path] = None, |
| 65 | squeeze: Optional[bool] = True, |
| 66 | ): |
Jeremy Johnson | be1a940 | 2021-12-15 17:14:56 +0000 | [diff] [blame] | 67 | """Convert the binary flatbuffer to JSON. |
| 68 | |
| 69 | flatc: the Path to the flatc compiler program |
| 70 | fbs: the Path to the fbs (flatbuffer schema) file |
| 71 | t_path: the Path to the binary flatbuffer file |
| 72 | o_path: the output Path where JSON file will be put, if None, it is same as t_path |
| 73 | """ |
| 74 | if o_path is None: |
| 75 | o_path = t_path.parent |
| 76 | cmd = [ |
| 77 | str(flatc.absolute()), |
| 78 | "-o", |
| 79 | str(o_path.absolute()), |
| 80 | "--json", |
Jeremy Johnson | 0633c3a | 2023-08-22 16:55:08 +0100 | [diff] [blame] | 81 | "--strict-json", |
Jeremy Johnson | be1a940 | 2021-12-15 17:14:56 +0000 | [diff] [blame] | 82 | "--defaults-json", |
| 83 | "--raw-binary", |
| 84 | str(fbs.absolute()), |
| 85 | "--", |
| 86 | str(t_path.absolute()), |
| 87 | ] |
| 88 | run_sh_command(verbose=False, full_cmd=cmd) |
Jeremy Johnson | 0633c3a | 2023-08-22 16:55:08 +0100 | [diff] [blame] | 89 | if squeeze: |
| 90 | json_path = (o_path / t_path.name).with_suffix(".json") |
| 91 | json_squeeze(json_path) |
Jeremy Johnson | be1a940 | 2021-12-15 17:14:56 +0000 | [diff] [blame] | 92 | |
| 93 | |
| 94 | def json_to_fbbin(flatc: Path, fbs: Path, j_path: Path, o_path: Optional[Path] = None): |
| 95 | """Convert JSON flatbuffer to binary. |
| 96 | |
| 97 | flatc: the Path to the flatc compiler program |
| 98 | fbs: the Path to the fbs (flatbuffer schema) file |
| 99 | j_path: the Path to the JSON flatbuffer file |
| 100 | o_path: the output Path where JSON file will be put, if None, it is same as j_path |
| 101 | """ |
| 102 | if o_path is None: |
| 103 | o_path = j_path.parent |
| 104 | cmd = [ |
| 105 | str(flatc.absolute()), |
| 106 | "-o", |
| 107 | str(o_path.absolute()), |
| 108 | "--binary", |
| 109 | str(fbs.absolute()), |
| 110 | str(j_path.absolute()), |
| 111 | ] |
| 112 | run_sh_command(verbose=False, full_cmd=cmd) |
| 113 | |
| 114 | |
| 115 | # ------------------------------------------------------------------------------ |
| 116 | |
| 117 | |
| 118 | def main(argv=None): |
| 119 | """Load and convert supplied file based on file suffix.""" |
| 120 | import argparse |
| 121 | |
| 122 | parser = argparse.ArgumentParser() |
| 123 | parser.add_argument( |
Jeremy Johnson | 0633c3a | 2023-08-22 16:55:08 +0100 | [diff] [blame] | 124 | "--no-squeeze", action="store_true", help="no compression of json output" |
| 125 | ) |
| 126 | parser.add_argument( |
Jeremy Johnson | be1a940 | 2021-12-15 17:14:56 +0000 | [diff] [blame] | 127 | "--flatc", |
| 128 | type=Path, |
Jeremy Johnson | 5c1364c | 2022-01-13 15:04:21 +0000 | [diff] [blame] | 129 | default=( |
| 130 | "reference_model/build/thirdparty/serialization_lib/" |
| 131 | "third_party/flatbuffers/flatc" |
| 132 | ), |
Jeremy Johnson | be1a940 | 2021-12-15 17:14:56 +0000 | [diff] [blame] | 133 | help="the path to the flatc compiler program", |
| 134 | ) |
| 135 | parser.add_argument( |
| 136 | "--fbs", |
| 137 | type=Path, |
| 138 | default="conformance_tests/third_party/serialization_lib/schema/tosa.fbs", |
| 139 | help="the path to the flatbuffer schema", |
| 140 | ) |
| 141 | parser.add_argument("path", type=Path, help="the path to the file to convert") |
| 142 | args = parser.parse_args(argv) |
| 143 | path = args.path |
| 144 | |
| 145 | if not path.is_file(): |
| 146 | print(f"Invalid file to convert - {path}") |
| 147 | return 2 |
| 148 | |
| 149 | if not args.flatc.is_file(): |
| 150 | print(f"Invalid flatc compiler - {args.flatc}") |
| 151 | return 2 |
| 152 | |
| 153 | if not args.fbs.is_file(): |
| 154 | print(f"Invalid flatbuffer schema - {args.fbs}") |
| 155 | return 2 |
| 156 | |
| 157 | try: |
| 158 | if path.suffix == ".json": |
| 159 | json_to_fbbin(args.flatc, args.fbs, path) |
| 160 | else: |
| 161 | # Have to assume this is a binary flatbuffer file as could have any suffix |
Jeremy Johnson | 0633c3a | 2023-08-22 16:55:08 +0100 | [diff] [blame] | 162 | fbbin_to_json(args.flatc, args.fbs, path, squeeze=(not args.no_squeeze)) |
Jeremy Johnson | be1a940 | 2021-12-15 17:14:56 +0000 | [diff] [blame] | 163 | except RunShCommandError as e: |
| 164 | print(e) |
| 165 | return 1 |
| 166 | |
| 167 | return 0 |
| 168 | |
| 169 | |
| 170 | if __name__ == "__main__": |
| 171 | exit(main()) |