blob: f03bae75921aae8f799659509e3b60510277a489 [file] [log] [blame]
Tim Hall79d07d22020-04-27 18:20:16 +01001# Copyright (C) 2020 Arm Limited or its affiliates. All rights reserved.
2#
3# SPDX-License-Identifier: Apache-2.0
4#
5# Licensed under the Apache License, Version 2.0 (the License); you may
6# not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9# www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an AS IS BASIS, WITHOUT
13# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
Tim Hall79d07d22020-04-27 18:20:16 +010016# Description:
17# Main entry point for the Vela compiler.
18#
19# Provides command line interface, options parsing, and network loading. Before calling the compiler driver.
Diego Russoe8a10452020-04-21 17:39:10 +010020import argparse
21import ast
Tim Hall1bd531d2020-11-01 20:59:36 +000022import os
Diego Russoea6111a2020-04-14 18:41:58 +010023import sys
Tim Hall79d07d22020-04-27 18:20:16 +010024import time
Tim Hall79d07d22020-04-27 18:20:16 +010025
26from . import architecture_features
Diego Russoe8a10452020-04-21 17:39:10 +010027from . import compiler_driver
28from . import model_reader
29from . import scheduler
Tim Hall79d07d22020-04-27 18:20:16 +010030from . import stats_writer
31from . import tflite_writer
Tim Hall79d07d22020-04-27 18:20:16 +010032from ._version import __version__
Patrik Gustavssonc8a22f12020-11-18 17:05:50 +010033from .api import api_version
Tim Halle6ccd872020-11-09 16:46:37 +000034from .debug_database import DebugDatabase
Louis Verhaard7db78962020-05-25 15:05:26 +020035from .errors import InputFileError
Diego Russoe8a10452020-04-21 17:39:10 +010036from .nn_graph import PassPlacement
37from .nn_graph import TensorAllocator
Tim Hall79d07d22020-04-27 18:20:16 +010038from .scheduler import ParetoMetric
Michael McGeagh837dc1b2020-11-10 12:38:25 +000039from .supported_operators import SupportedOperators
Diego Russoea6111a2020-04-14 18:41:58 +010040from .tensor import MemArea
Jacob Bohlin0628a8c2020-08-28 13:25:14 +020041from .tensor import Tensor
Michael McGeagh837dc1b2020-11-10 12:38:25 +000042from .tflite_mapping import builtin_operator_map
43from .tflite_mapping import builtin_type_name
Louis Verhaard52078302020-11-18 13:35:06 +010044from ethosu.vela.architecture_features import ArchitectureFeatures
Tim Hall79d07d22020-04-27 18:20:16 +010045
46
Tim Halle6ccd872020-11-09 16:46:37 +000047def process(input_name, enable_debug_db, arch, model_reader_options, compiler_options, scheduler_options):
Tim Hall79d07d22020-04-27 18:20:16 +010048 if compiler_options.timing:
49 start = time.time()
50
Tim Halle6ccd872020-11-09 16:46:37 +000051 os.makedirs(compiler_options.output_dir, exist_ok=True)
52 output_basename = os.path.join(compiler_options.output_dir, os.path.splitext(os.path.basename(input_name))[0])
53 DebugDatabase.show_warnings = enable_debug_db
54
55 nng = model_reader.read_model(input_name, model_reader_options)
Tim Hall79d07d22020-04-27 18:20:16 +010056
57 if not nng:
Tim Halle6ccd872020-11-09 16:46:37 +000058 raise InputFileError(input_name, "input file could not be read")
Tim Hall79d07d22020-04-27 18:20:16 +010059
60 if compiler_options.verbose_operators:
61 nng.print_operators()
62
63 if compiler_options.timing:
64 stop = time.time()
65 print("Model reading took %f s" % (stop - start))
66 start = time.time()
67
68 compiler_driver.compiler_driver(nng, arch, compiler_options, scheduler_options)
69
Tim Halle6ccd872020-11-09 16:46:37 +000070 passes_csv_file = "{0}_pass-breakdown_{1}.csv".format(output_basename, arch.system_config)
Tim Hall79d07d22020-04-27 18:20:16 +010071 stats_writer.write_pass_metrics_csv(nng, passes_csv_file)
72
Tim Halle6ccd872020-11-09 16:46:37 +000073 summary_csv_file = "{0}_summary_{1}.csv".format(output_basename, arch.system_config)
Tim Hall79d07d22020-04-27 18:20:16 +010074 stats_writer.write_summary_metrics_csv(nng, summary_csv_file, arch)
75
76 stats_writer.print_performance_metrics(nng, show_cpu_operations=compiler_options.show_cpu_operations, arch=arch)
77
Tim Halle6ccd872020-11-09 16:46:37 +000078 output_filename = output_basename + "_vela.tflite"
79 if input_name.endswith(".tflite"):
80 tflite_writer.write_tflite(nng, output_filename)
81
82 if enable_debug_db:
83 debug_filename = output_basename + "_debug.xml"
84 DebugDatabase.write(debug_filename, input_name, output_filename)
Tim Hall79d07d22020-04-27 18:20:16 +010085
86 if compiler_options.timing:
87 stop = time.time()
88 print("Compiler driver took %f s" % (stop - start))
89
90 return nng
91
92
93def print_subgraph_io_summary(nng):
94 """Print a summary of all the input and output tensor sizes for all subgraphs.
95 Also displays the total tensor size and the memory used area for sram.
96 """
97
98 print("Subgraph IO Summary")
99 print("-------------------")
100 print("NNG: {0}".format(nng.name))
101 max_sg_size = 0
102 for sg in reversed(nng.subgraphs):
103 print(" Subgraph: {0} = {1}".format(sg.name, sg.placement))
104 sg_size = 0
105
106 if sg.placement == PassPlacement.Npu:
107 for tens in sg.input_tensors + [sg.scratch_tensor] + sg.output_tensors:
108 if tens in sg.input_tensors:
109 tens_dir = "In"
110 elif tens in sg.output_tensors:
111 tens_dir = "Out"
112 else:
113 tens_dir = "In/Out"
114
115 size = tens.elements() * tens.element_size() / 1024.0
116 sg_size = sg_size + size
117 print(" Tensor [{0}]: {1} = {2} KiB".format(tens_dir, tens.name, size))
118
119 print(" Total Size = {0} KiB".format(sg_size))
120 print(" SRAM Memory Used = {0} KiB".format(sg.memory_used.get(MemArea.Sram, 0) / 1024.0))
121 max_sg_size = max(sg_size, max_sg_size)
122
123 print(" Maximum Subgraph Size = {0} KiB".format(max_sg_size))
124
125
Michael McGeagh837dc1b2020-11-10 12:38:25 +0000126def generate_supported_ops():
127 lines = [
128 "# Supported Ops",
129 "",
130 "This file was automatically generated by Vela using the `--supported-ops-report` parameter. ",
131 f"Vela version: `{__version__}`",
132 "",
Michael McGeagh54a61112020-11-24 14:58:51 +0000133 "This file complies with",
134 "[**Gitiles Markdown syntax**](https://github.com/google/gitiles/blob/master/Documentation/markdown.md)",
Michael McGeagh837dc1b2020-11-10 12:38:25 +0000135 "",
136 "## Summary Table",
137 "",
138 "The table below contains TFLite operators that can be placed on the Ethos-U NPU. ",
139 "If the constraints are not met, then that operator will be scheduled on the CPU instead. ",
140 "For any other TFLite operator not listed, will be left untouched and scheduled on the CPU. ",
141 "Please check the supported operator list for your chosen runtime for further information.",
142 "",
143 "| Operator | Constraints |",
Michael McGeagh54a61112020-11-24 14:58:51 +0000144 "| --- | --- |",
Michael McGeagh837dc1b2020-11-10 12:38:25 +0000145 ]
146 supported = SupportedOperators()
147 op_constraint_links = []
148 op_list = sorted(((op, builtin_type_name(op)) for op in builtin_operator_map), key=lambda x: x[1])
149 for op, name in op_list:
150 internal_op = builtin_operator_map[op][0]
151 if internal_op in SupportedOperators.supported_operators:
152 links = "[Generic](#generic-constraints)"
153 if internal_op in supported.specific_constraints:
154 links += f", [Specific](#{name.lower()}-constraints)"
155 op_constraint_links.append((internal_op, name))
156 lines.append(f"| {name} | {links} |")
157 lines += [
158 "",
159 "## Generic Constraints",
160 "",
161 "This is a list of constraints that all NPU operators must satisfy in order to be scheduled on the NPU.",
162 "",
163 ]
164 for constraint in supported.generic_constraints:
165 # Markdown needs two spaces at the end of a line to render it as a separate line
166 reason = constraint.__doc__.replace("\n", " \n")
167 lines.append(f"- {reason}")
168 for op, name in op_constraint_links:
169 lines += [
170 "",
171 f"## {name} Constraints",
172 "",
173 f"This is a list of constraints that the {name} operator must satisfy in order to be scheduled on the NPU.",
174 "",
175 ]
176 for constraint in supported.specific_constraints[op]:
177 # Markdown needs two spaces at the end of a line to render it as a separate line
178 reason = constraint.__doc__.replace("\n", " \n")
179 lines.append(f"- {reason}")
180
181 # Note. this will generate the file in the CWD
182 filepath = os.path.join(os.getcwd(), "SUPPORTED_OPS.md")
183 with open(filepath, "wt") as md:
184 md.writelines(line + "\n" for line in lines)
185 print(f"Report file: {filepath}")
186
187
Tim Hall79d07d22020-04-27 18:20:16 +0100188def main(args=None):
189 if args is None:
190 args = sys.argv[1:]
191
Tim Halld13bd062020-11-12 23:29:25 +0000192 parser = argparse.ArgumentParser(prog="vela", description="Neural network model compiler for Arm Ethos-U NPUs")
Michael McGeagh837dc1b2020-11-10 12:38:25 +0000193 parser.add_argument("--version", action="version", version=__version__)
Tim Hall79d07d22020-04-27 18:20:16 +0100194 parser.add_argument(
Patrik Gustavssonc8a22f12020-11-18 17:05:50 +0100195 "--api-version", action="version", version=api_version, help="Displays the version of the external API."
196 )
197 parser.add_argument(
Michael McGeagh837dc1b2020-11-10 12:38:25 +0000198 "--supported-ops-report",
199 action="store_true",
Tim Hall1bd531d2020-11-01 20:59:36 +0000200 help="Generate the SUPPORTED_OPS.md file in the current working directory and exit",
Tim Hall79d07d22020-04-27 18:20:16 +0100201 )
202
Tim Hall1bd531d2020-11-01 20:59:36 +0000203 # set network nargs to be optional to allow the support-ops-report CLI option to be used standalone
Michael McGeagh837dc1b2020-11-10 12:38:25 +0000204 parser.add_argument(
Tim Halld13bd062020-11-12 23:29:25 +0000205 "network",
206 metavar="NETWORK",
207 type=str,
208 default=None,
209 nargs="?",
210 help="Filename of the input TensorFlow Lite for Microcontrollers network",
Michael McGeagh837dc1b2020-11-10 12:38:25 +0000211 )
Tim Hall79d07d22020-04-27 18:20:16 +0100212 parser.add_argument(
213 "--output-dir", type=str, default="output", help="Output directory to write files to (default: %(default)s)"
214 )
Tim Halle6ccd872020-11-09 16:46:37 +0000215 parser.add_argument(
216 "--enable-debug-db",
217 action="store_true",
218 default=None,
219 help="Enables the calculation and writing of a network debug database to output directory",
220 )
Tim Hall1bd531d2020-11-01 20:59:36 +0000221 parser.add_argument(
222 "--config", type=str, action="append", help="Vela configuration file(s) in Python ConfigParser .ini file format"
223 )
224 parser.add_argument("--verbose-config", action="store_true", help="Verbose system configuration and memory mode")
Tim Hall79d07d22020-04-27 18:20:16 +0100225 parser.add_argument("--verbose-graph", action="store_true", help="Verbose graph rewriter")
226 parser.add_argument("--verbose-quantization", action="store_true", help="Verbose quantization")
227 parser.add_argument("--verbose-packing", action="store_true", help="Verbose pass packing")
228 parser.add_argument("--verbose-tensor-purpose", action="store_true", help="Verbose tensor purpose")
229 parser.add_argument("--verbose-tensor-format", action="store_true", help="Verbose tensor format")
230 parser.add_argument("--verbose-schedule", action="store_true", help="Verbose schedule")
231 parser.add_argument(
232 "--verbose-pareto-frontier-schedules",
233 action="store_true",
234 help="Show all schedules along the pareto frontier of optimisation criteria",
235 )
236 parser.add_argument("--verbose-allocation", action="store_true", help="Verbose tensor allocation")
237 parser.add_argument(
238 "--verbose-high-level-command-stream", action="store_true", help="Verbose high level command stream"
239 )
240 parser.add_argument(
241 "--verbose-register-command-stream", action="store_true", help="Verbose register command stream"
242 )
243 parser.add_argument("--verbose-operators", action="store_true", help="Verbose operator list")
Tim Hall79d07d22020-04-27 18:20:16 +0100244 parser.add_argument(
245 "--show-cpu-operations", action="store_true", help="Show the operations that fall back to the CPU"
246 )
247 parser.add_argument(
Andreas Nevalainen897cc142020-10-28 15:42:08 +0100248 "--keep-scale-placement", action="store_true", help="Keep scale tensors memory placement during scheduling"
249 )
250 parser.add_argument(
Tim Hall79d07d22020-04-27 18:20:16 +0100251 "--cascading",
252 type=ast.literal_eval,
253 default=True,
254 choices=[True, False],
255 help="Controls the packing of multiple passes into a cascade (default: %(default)s)",
256 )
Tim Hall79d07d22020-04-27 18:20:16 +0100257 parser.add_argument("--force-block-config", type=str, default="", help="Force a specific block configuration HxWxC")
Tim Hall79d07d22020-04-27 18:20:16 +0100258 parser.add_argument("--timing", action="store_true", help="Time the compiler doing operations")
259 parser.add_argument(
260 "--accelerator-config",
261 type=str,
262 default="ethos-u55-256",
Manupa Karunaratned83d2e12020-07-20 12:05:32 +0100263 choices=list(architecture_features.Accelerator.member_list()),
Tim Hall79d07d22020-04-27 18:20:16 +0100264 help="Accelerator configuration to use (default: %(default)s)",
265 )
266 parser.add_argument(
267 "--system-config",
268 type=str,
Tim Hall1bd531d2020-11-01 20:59:36 +0000269 default=architecture_features.ArchitectureFeatures.DEFAULT_CONFIG,
270 help="System configuration to select from the Vela configuration file (default: %(default)s)",
271 )
272 parser.add_argument(
273 "--memory-mode",
274 type=str,
275 default=architecture_features.ArchitectureFeatures.DEFAULT_CONFIG,
276 help="Memory mode to select from the Vela configuration file (default: %(default)s)",
Tim Hall79d07d22020-04-27 18:20:16 +0100277 )
278 parser.add_argument(
Tim Hall79d07d22020-04-27 18:20:16 +0100279 "--tensor-allocator",
280 default=TensorAllocator.Greedy,
281 type=lambda s: TensorAllocator[s],
282 choices=list(TensorAllocator),
283 help="Tensor Allocator algorithm (default: %(default)s)",
284 )
285 parser.add_argument(
286 "--show-subgraph-io-summary",
287 action="store_true",
288 help="Shows a summary of all the subgraphs and their inputs and outputs",
289 )
290 parser.add_argument(
291 "--ifm-streaming",
292 type=ast.literal_eval,
293 default=True,
294 choices=[True, False],
295 help="Controls scheduler IFM streaming search (default: %(default)s)",
296 )
297 parser.add_argument(
298 "--block-config-limit",
299 type=int,
300 default=16,
301 help="Limit block config search space, use zero for unlimited (default: %(default)s)",
302 )
303 parser.add_argument(
Tim Hall79d07d22020-04-27 18:20:16 +0100304 "--pareto-metric",
305 default=ParetoMetric.BwCycMem,
306 type=lambda s: ParetoMetric[s],
307 choices=list(ParetoMetric),
308 help="Controls the calculation of the pareto metric (default: %(default)s)",
309 )
310 parser.add_argument(
311 "--recursion-limit",
312 type=int,
313 default=10000,
314 help="Set the recursion depth limit, may result in RecursionError if too low (default: %(default)s)",
315 )
316 parser.add_argument(
317 "--max-block-dependency",
318 type=int,
319 default=architecture_features.ArchitectureFeatures.MAX_BLOCKDEP,
320 choices=range(0, architecture_features.ArchitectureFeatures.MAX_BLOCKDEP + 1),
321 help=(
Tim Hallb9b515c2020-11-01 21:27:19 +0000322 "Set the maximum value that can be used for the block dependency between npu kernel operations"
323 " (default: %(default)s)"
Tim Hall79d07d22020-04-27 18:20:16 +0100324 ),
325 )
Charles Xu7b8823f2020-05-29 13:53:10 +0200326 parser.add_argument(
327 "--nhcwb16-between-cascaded-passes",
328 type=ast.literal_eval,
329 default=True,
330 choices=[True, False],
331 help="Control if NHCWB16 or NHWC should be used in between cascaded passes (default: %(default)s)",
332 )
Fredrik Svedberga0c36242020-06-03 15:43:31 +0200333 parser.add_argument(
Patrik Gustavsson90831bc2020-08-24 16:26:11 +0200334 "--weight-estimation-scaling",
335 type=float,
336 default=1.0,
337 help=("Performs an additional scaling of weight compression scale estimate (default: %(default)s)"),
338 )
Jacob Bohlin0628a8c2020-08-28 13:25:14 +0200339 parser.add_argument(
Tim Hallb9b515c2020-11-01 21:27:19 +0000340 "--cpu-tensor-alignment",
Jacob Bohlin0628a8c2020-08-28 13:25:14 +0200341 type=int,
342 default=Tensor.AllocationQuantum,
Tim Hallb9b515c2020-11-01 21:27:19 +0000343 help=(
344 "Controls the allocation byte alignment of cpu tensors including Ethos-U Custom operator inputs and outputs"
345 " (default: %(default)s)"
346 ),
Jacob Bohlin0628a8c2020-08-28 13:25:14 +0200347 )
Tim Hall79d07d22020-04-27 18:20:16 +0100348 args = parser.parse_args(args=args)
349
Michael McGeagh837dc1b2020-11-10 12:38:25 +0000350 # Generate the supported ops report and exit
351 if args.supported_ops_report:
352 generate_supported_ops()
353 return 0
354
Tim Hall79d07d22020-04-27 18:20:16 +0100355 if args.network is None:
356 parser.error("the following argument is required: NETWORK")
357
Tim Hall1bd531d2020-11-01 20:59:36 +0000358 # check all config files exist because they will be read as a group
359 if args.config is not None:
360 for filename in args.config:
361 if not os.access(filename, os.R_OK):
362 raise InputFileError(filename, "File not found or is not readable.")
363
Tim Hall79d07d22020-04-27 18:20:16 +0100364 sys.setrecursionlimit(args.recursion_limit)
365
366 if args.force_block_config:
367 force_block_config = architecture_features.Block.from_string(args.force_block_config)
368 else:
369 force_block_config = None
370
Tim Hallb9b515c2020-11-01 21:27:19 +0000371 if args.cpu_tensor_alignment < 16 or args.cpu_tensor_alignment & (args.cpu_tensor_alignment - 1) != 0:
372 parser.error(
373 "Invalid argument to --cpu-tensor-alignment = {} (must be greater than or equal to 16 and a power of 2)"
374 "".format(args.cpu_tensor_alignment)
375 )
Jacob Bohlin0628a8c2020-08-28 13:25:14 +0200376
Louis Verhaard52078302020-11-18 13:35:06 +0100377 if args.system_config == ArchitectureFeatures.DEFAULT_CONFIG:
378 print(f"Warning: Using {ArchitectureFeatures.DEFAULT_CONFIG} values for system configuration")
379
380 if args.memory_mode == ArchitectureFeatures.DEFAULT_CONFIG:
381 print(f"Warning: Using {ArchitectureFeatures.DEFAULT_CONFIG} values for memory mode")
382
Tim Hall79d07d22020-04-27 18:20:16 +0100383 arch = architecture_features.ArchitectureFeatures(
Tim Hall1bd531d2020-11-01 20:59:36 +0000384 vela_config_files=args.config,
Tim Hall79d07d22020-04-27 18:20:16 +0100385 system_config=args.system_config,
Tim Hall1bd531d2020-11-01 20:59:36 +0000386 memory_mode=args.memory_mode,
Tim Hall79d07d22020-04-27 18:20:16 +0100387 accelerator_config=args.accelerator_config,
Tim Hall79d07d22020-04-27 18:20:16 +0100388 override_block_config=force_block_config,
389 block_config_limit=args.block_config_limit,
Tim Hall79d07d22020-04-27 18:20:16 +0100390 max_blockdep=args.max_block_dependency,
Patrik Gustavsson90831bc2020-08-24 16:26:11 +0200391 weight_estimation_scaling=args.weight_estimation_scaling,
Tim Hall1bd531d2020-11-01 20:59:36 +0000392 verbose_config=args.verbose_config,
Tim Hall79d07d22020-04-27 18:20:16 +0100393 )
394
395 compiler_options = compiler_driver.CompilerOptions(
396 verbose_graph=args.verbose_graph,
397 verbose_quantization=args.verbose_quantization,
398 verbose_packing=args.verbose_packing,
399 verbose_tensor_purpose=args.verbose_tensor_purpose,
400 verbose_tensor_format=args.verbose_tensor_format,
401 verbose_allocation=args.verbose_allocation,
402 verbose_high_level_command_stream=args.verbose_high_level_command_stream,
403 verbose_register_command_stream=args.verbose_register_command_stream,
404 verbose_operators=args.verbose_operators,
Tim Hall79d07d22020-04-27 18:20:16 +0100405 show_cpu_operations=args.show_cpu_operations,
406 tensor_allocator=args.tensor_allocator,
407 timing=args.timing,
408 output_dir=args.output_dir,
Tim Hallb9b515c2020-11-01 21:27:19 +0000409 cpu_tensor_alignment=args.cpu_tensor_alignment,
Tim Hall79d07d22020-04-27 18:20:16 +0100410 )
411
412 scheduler_options = scheduler.SchedulerOptions(
413 use_cascading=args.cascading,
Tim Hall79d07d22020-04-27 18:20:16 +0100414 verbose_schedule=args.verbose_schedule,
415 verbose_pareto_frontier_schedules=args.verbose_pareto_frontier_schedules,
416 use_ifm_streaming=args.ifm_streaming,
417 pareto_metric=args.pareto_metric,
Charles Xu7b8823f2020-05-29 13:53:10 +0200418 use_nhcwb16_between_cascaded_passes=args.nhcwb16_between_cascaded_passes,
Andreas Nevalainen897cc142020-10-28 15:42:08 +0100419 keep_scale_placement=args.keep_scale_placement,
Tim Hall79d07d22020-04-27 18:20:16 +0100420 )
421
Tim Hall284223e2020-06-09 13:17:21 +0100422 model_reader_options = model_reader.ModelReaderOptions()
Tim Hall79d07d22020-04-27 18:20:16 +0100423
Tim Halle6ccd872020-11-09 16:46:37 +0000424 nng = process(args.network, args.enable_debug_db, arch, model_reader_options, compiler_options, scheduler_options)
Tim Hall79d07d22020-04-27 18:20:16 +0100425
426 if args.show_subgraph_io_summary:
427 print_subgraph_io_summary(nng)
428
429 return 0