blob: 41d75f45b8681162219527484c6bd9a57df0808d [file] [log] [blame]
Tim Hall79d07d22020-04-27 18:20:16 +01001# Copyright (C) 2020 Arm Limited or its affiliates. All rights reserved.
2#
3# SPDX-License-Identifier: Apache-2.0
4#
5# Licensed under the Apache License, Version 2.0 (the License); you may
6# not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9# www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an AS IS BASIS, WITHOUT
13# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
Tim Hall79d07d22020-04-27 18:20:16 +010016# Description:
17# NPU performance estimation functions to estimate performance of a Pass and CascadedPass. Uses a model that takes the
18# maximum of the 'cycles required for bandwidth' and 'cycles required for computing'.
19#
20# Called during scheduling to evaluate different proposals, as well as post-scheduling to provide a final performance
21# estimate.
Tim Hall79d07d22020-04-27 18:20:16 +010022import enum
Diego Russoea6111a2020-04-14 18:41:58 +010023
Tim Hall79d07d22020-04-27 18:20:16 +010024import numpy as np
Diego Russoea6111a2020-04-14 18:41:58 +010025
26from . import numeric_util
Diqing Zhong09387e22020-09-28 18:46:22 +020027from .architecture_features import Accelerator
Diego Russoe8a10452020-04-21 17:39:10 +010028from .architecture_features import Block
Diqing Zhonge8887a32020-09-24 09:53:48 +020029from .data_type import DataType
Diego Russoe8a10452020-04-21 17:39:10 +010030from .nn_graph import PassPlacement
31from .nn_graph import SchedulerRewrite
Diego Russoea6111a2020-04-14 18:41:58 +010032from .operation import NpuBlockType
Diqing Zhonge8887a32020-09-24 09:53:48 +020033from .operation import Op
Diqing Zhong09387e22020-09-28 18:46:22 +020034from .shared_buffer_allocation import is_acc_40bits_used
Diego Russoe8a10452020-04-21 17:39:10 +010035from .tensor import MemArea
36from .tensor import shape_num_elements
37from .tensor import TensorBlockTraversal
38from .tensor import TensorPurpose
Tim Hall79d07d22020-04-27 18:20:16 +010039
40
41def rolling_buffer_dims_from_passes(arch, ps1, block_config_ps1, ps2, block_config_ps2):
Tim Hall79d07d22020-04-27 18:20:16 +010042 ofm_block = Block(block_config_ps2[-3], block_config_ps2[-4], block_config_ps2[-1])
Tim Hall4ed38bc2020-10-20 18:54:20 +010043 kernel = ps2.primary_op.kernel
Tim Hall79d07d22020-04-27 18:20:16 +010044
45 if ps2.npu_block_type in set((NpuBlockType.ConvolutionMxN, NpuBlockType.VectorProduct)):
Louis Verhaard93dc5532020-06-07 12:40:18 +020046 op = ps2.primary_op
Louis Verhaardaee5d752020-09-30 09:01:52 +020047 ifm_block_depth = arch.calc_ifm_block_depth(op.ifm.shape[-1], op.ifm.dtype.size_in_bits())
Tim Hall79d07d22020-04-27 18:20:16 +010048 else:
49 ifm_block_depth = block_config_ps2[-1]
50
Louis Verhaard93dc5532020-06-07 12:40:18 +020051 ifm_block = arch.get_ifm_block_size(ifm_block_depth, ofm_block, kernel, arch.ofm_block_max)
Tim Hall79d07d22020-04-27 18:20:16 +010052
53 # The performed height calculation is for worst case
54 height = numeric_util.round_up(ifm_block.height + block_config_ps1[0], block_config_ps1[0])
55 width = ifm_block.width
Louis Verhaard93dc5532020-06-07 12:40:18 +020056 return [height, width]
Tim Hall79d07d22020-04-27 18:20:16 +010057
58
59class PassCycles(enum.IntEnum):
Diqing Zhong42e833d2020-10-02 13:18:42 +020060 Npu = 0
61 Cpu = 1
62 SramAccess = 2
63 TotalPerPass = 3
64 DramAccess = 4
65 OnChipFlashAccess = 5
66 OffChipFlashAccess = 6
67 Total = 7
68 Size = 8
Tim Hall79d07d22020-04-27 18:20:16 +010069
70 def display_name(self):
71 return (
Diqing Zhong42e833d2020-10-02 13:18:42 +020072 "NPU",
Tim Hall79d07d22020-04-27 18:20:16 +010073 "CPU",
74 "SRAM Access",
75 "Total per Pass",
76 "DRAM Access",
77 "On-chip Flash Access",
78 "Off-chip Flash Access",
79 "Total",
80 "Size",
81 )[self.value]
82
83 def identifier_name(self):
84 return (
Diqing Zhong42e833d2020-10-02 13:18:42 +020085 "npu",
Tim Hall79d07d22020-04-27 18:20:16 +010086 "cpu",
87 "sram_access",
88 "total_per_pass",
89 "dram_access",
90 "on_chip_flash_access",
91 "off_chip_flash_access",
92 "total",
93 "size",
94 )[self.value]
95
96 @staticmethod
97 def all():
98 return (
Diqing Zhong42e833d2020-10-02 13:18:42 +020099 PassCycles.Npu,
Tim Hall79d07d22020-04-27 18:20:16 +0100100 PassCycles.Cpu,
101 PassCycles.SramAccess,
102 PassCycles.DramAccess,
103 PassCycles.OnChipFlashAccess,
104 PassCycles.OffChipFlashAccess,
105 PassCycles.Total,
106 )
107
108
109class MacCount(enum.IntEnum):
110 NeuralNetworkMacs = 0
111 HardwareMacs = 1
112 Size = 2
113
114 def display_name(self):
115 return ("Neural Network Macs", "Hardware Macs", "Size")[self.value]
116
117 def identifier_name(self):
118 return ("nn_macs", "hardware_macs", "size")[self.value]
119
120 @staticmethod
121 def all():
122 return (MacCount.NeuralNetworkMacs, MacCount.HardwareMacs)
123
124
125class BandwidthDirection(enum.IntEnum):
126 Read = 0
127 Write = 1
128 Size = 2
129
130 def display_name(self):
131 return self.name
132
133 def identifier_name(self):
134 return self.name.lower()
135
136 @staticmethod
137 def all():
138 return (BandwidthDirection.Read, BandwidthDirection.Write)
139
140
141def make_bandwidth_array():
142 return np.zeros((MemArea.Size, TensorPurpose.Size, BandwidthDirection.Size))
143
144
145def make_macs_array():
146 return np.zeros(MacCount.Size, np.int)
147
148
149def make_cycles_array():
150 return np.zeros(PassCycles.Size)
151
152
153def make_metrics_arrays():
154 return (make_bandwidth_array(), make_macs_array(), make_cycles_array())
155
156
157def get_n_blocks_and_area(
158 ifm_brick_size, ifm_height_width, orig_skirt, clamped_skirt, block_config, min_block_size, strides
159):
160
161 ifm_block_config = (block_config[0] * strides[1], block_config[1] * strides[2])
162
163 n_normal_blocks = []
164 remainder_size = []
165 for i in range(2):
166 non_skirt_dim = ifm_height_width[i] - orig_skirt[i] - orig_skirt[2 + i]
167 n_blocks = non_skirt_dim // ifm_block_config[i]
168 n_normal_blocks.append(n_blocks)
169 remainder_dim = numeric_util.round_up(
170 ((non_skirt_dim - n_blocks * ifm_block_config[i] - 1) // strides[i + 1]) + 1, min_block_size[i]
171 )
172 remainder_size.append(remainder_dim)
173
174 # this will actually calculate reads into the edge padding.
175
176 # there are four cases in total, handling the edges that will not fill a complete block.
177
178 # 0000000001
179 # 0000000001
180 # 0000000001
181 # 0000000001
182 # 0000000001
183 # 0000000001
184 # 2222222223
185 total_blocks = 0
186 total_area = 0
187
188 block_setup = (
189 (n_normal_blocks[0] * n_normal_blocks[1], block_config),
190 (1 * n_normal_blocks[1], (remainder_size[0], block_config[1])),
191 (n_normal_blocks[0] * 1, (block_config[0], remainder_size[1])),
192 (1 * 1, remainder_size),
193 )
194
195 for n_blocks, block_size in block_setup:
196 if block_size[0] == 0 or block_size[1] == 0:
197 continue
198 read_dims = [0, 0]
199 for i in range(2):
200 read_dims[i] = (
201 numeric_util.round_up(clamped_skirt[i], ifm_brick_size[i + 1])
202 + block_size[i] * strides[i + 1]
203 + numeric_util.round_up(clamped_skirt[2 + i], ifm_brick_size[i + 1])
204 )
205 assert n_blocks >= 0
206 total_blocks += n_blocks
207 total_area += n_blocks * read_dims[0] * read_dims[1]
208 assert total_blocks >= 1
209 return total_blocks, total_area, block_setup
210
211
Diqing Zhong42e833d2020-10-02 13:18:42 +0200212def get_ifm_block_depth(npu_block_type, ifm_depth, ifm_elemwidth, block_traversal, ofm_blk_depth):
213 ifm_blk_depth = ofm_blk_depth
214
215 if npu_block_type == NpuBlockType.ConvolutionMxN or npu_block_type == NpuBlockType.ReduceSum:
216 if ifm_elemwidth == 16 or block_traversal == TensorBlockTraversal.PartKernelFirst:
217 ifm_blk_depth = 16
218 elif ifm_elemwidth == 8:
219 ifm_blk_depth = 32
220 else:
221 ifm_blk_depth = 8
222
223 return min(ifm_depth, ifm_blk_depth)
224
225
226def estimate_output_cycles(
Diqing Zhong09387e22020-09-28 18:46:22 +0200227 arch, npu_block_type, primary_op, num_elems, ifm_tensor, ofm_tensor, ifm2_tensor, use_acc_40bits=False
228):
Diqing Zhonge8887a32020-09-24 09:53:48 +0200229 faf = primary_op.activation
Diqing Zhong09387e22020-09-28 18:46:22 +0200230 if npu_block_type == NpuBlockType.ElementWise and ifm_tensor.dtype == DataType.int32:
231 if ifm2_tensor is None:
Diqing Zhonge8887a32020-09-24 09:53:48 +0200232 # Unary op
233 output_perf_index = 0
234 else:
235 # Binary op
236 output_perf_index = 1
Diqing Zhong09387e22020-09-28 18:46:22 +0200237 elif primary_op.type == Op.Mul and ofm_tensor.dtype == DataType.int32:
Diqing Zhonge8887a32020-09-24 09:53:48 +0200238 output_perf_index = 2
Diqing Zhong09387e22020-09-28 18:46:22 +0200239 elif primary_op.type == Op.Mul or (
Diqing Zhonge8887a32020-09-24 09:53:48 +0200240 npu_block_type
241 in (
242 NpuBlockType.ConvolutionMxN,
243 NpuBlockType.ConvolutionDepthWise,
244 NpuBlockType.Pooling,
245 NpuBlockType.ReduceSum,
246 NpuBlockType.VectorProduct,
247 )
Diqing Zhong09387e22020-09-28 18:46:22 +0200248 and use_acc_40bits
Diqing Zhonge8887a32020-09-24 09:53:48 +0200249 ):
250 output_perf_index = 3
Diqing Zhong09387e22020-09-28 18:46:22 +0200251 elif primary_op.type in (Op.Add, Op.Sub):
252 input_scale = ifm_tensor.quantization.scale_f32
253 input2_scale = ifm2_tensor.quantization.scale_f32
254 output_scale = ofm_tensor.quantization.scale_f32
Diqing Zhonge8887a32020-09-24 09:53:48 +0200255
256 if "resizebilinear" in primary_op.attrs:
257 output_scale = input2_scale
258
259 if None in (input_scale, input2_scale, output_scale) or input_scale == input2_scale:
260 # Simple Add/Sub
261 output_perf_index = 4
262 else:
263 # Advanced Add/Sub
264 output_perf_index = 5
Diqing Zhong09387e22020-09-28 18:46:22 +0200265 elif primary_op.type.is_maxpool_op():
Diqing Zhonge8887a32020-09-24 09:53:48 +0200266 output_perf_index = 6
267 else:
268 output_perf_index = 7
269
270 if faf in (Op.Sigmoid, Op.Tanh, Op.LUT):
271 activation_perf_index = 0
272 elif faf in (Op.Relu, Op.Relu6, Op.ReluN1To1):
273 activation_perf_index = 1
274 else:
275 activation_perf_index = 2
276
Diqing Zhonge8887a32020-09-24 09:53:48 +0200277 cycle_per_elem = max(
278 arch.output_cycles_per_elem[output_perf_index], arch.activation_cycles_per_elem[activation_perf_index]
279 )
280 return num_elems * cycle_per_elem
281
282
Diqing Zhong42e833d2020-10-02 13:18:42 +0200283def estimate_conv_pooling_cycles(
Diqing Zhong09387e22020-09-28 18:46:22 +0200284 arch, npu_block_type, primary_op, block_config: Block, block_traversal, kernel_dims, ifm_tensor, ofm_tensor
285):
Diqing Zhonge5204a62020-10-13 11:42:37 +0200286 ofm_ublock = Block(arch.config.ofm_ublock.width, arch.config.ofm_ublock.height, arch.config.ofm_ublock.depth)
287 ifm_tens_shape = numeric_util.full_shape(4, ifm_tensor.shape, 1)
288 ofm_tens_shape = numeric_util.full_shape(4, ofm_tensor.shape, 1)
289
290 if (
291 arch.config.ofm_ublock.height == 2
292 and npu_block_type
293 in (NpuBlockType.ConvolutionMxN, NpuBlockType.ConvolutionDepthWise, NpuBlockType.VectorProduct)
294 and ofm_tens_shape[1] == 1
295 # Optimisation only applies for even width tensors
296 and ofm_tens_shape[2] % 2 == 0
297 and kernel_dims[0] == 1
298 ):
299 ofm_ublock.width = 4
300 ofm_ublock.height = 1
301 block_config.height = 1
302
Diqing Zhong09387e22020-09-28 18:46:22 +0200303 num_ublk = (
Diqing Zhonge5204a62020-10-13 11:42:37 +0200304 numeric_util.round_up_divide(block_config.width, ofm_ublock.width)
305 * (block_config.height // ofm_ublock.height)
306 * (block_config.depth // ofm_ublock.depth)
Diqing Zhong09387e22020-09-28 18:46:22 +0200307 )
308 num_ofm_blk = 0
309 total_cycles = 0
310 num_elems_blk = block_config.width * block_config.height * block_config.depth
Diqing Zhonge5204a62020-10-13 11:42:37 +0200311
Diqing Zhong09387e22020-09-28 18:46:22 +0200312 use_acc_40bits = is_acc_40bits_used(npu_block_type, ifm_tensor, ofm_tensor)
313
314 sub_kernel_limits = arch.sub_kernel_limits[npu_block_type]
315 n_sub_kernels_y = numeric_util.round_up_divide(kernel_dims[0], sub_kernel_limits[0])
316 n_sub_kernels_x = numeric_util.round_up_divide(kernel_dims[1], sub_kernel_limits[1])
317 sub_kernel_x = [
318 min((kernel_dims[1] - i * sub_kernel_limits[1]), sub_kernel_limits[1]) for i in range(n_sub_kernels_x)
319 ]
320 sub_kernel_y = [
321 min((kernel_dims[0] - i * sub_kernel_limits[0]), sub_kernel_limits[0]) for i in range(n_sub_kernels_y)
322 ]
323 sub_kernel_size = (x * y for y in sub_kernel_y for x in sub_kernel_x)
324
Diqing Zhong42e833d2020-10-02 13:18:42 +0200325 ifm_blk_depth = get_ifm_block_depth(
326 npu_block_type, ifm_tens_shape[3], ifm_tensor.dtype.size_in_bits(), block_traversal, block_config.depth
327 )
Diqing Zhong09387e22020-09-28 18:46:22 +0200328 cycles_dpu_blk = 0
329
330 for num_kernel_elems in sub_kernel_size:
331 if npu_block_type == NpuBlockType.Pooling:
332 cycles = max(4, num_kernel_elems) * num_ublk
333 if ifm_tensor.dtype.size_in_bits() == 16 and arch.accelerator_config != Accelerator.Ethos_U55_32:
334 cycles *= 2
335 elif npu_block_type == NpuBlockType.ConvolutionDepthWise:
336 cycles = 4 * numeric_util.round_up_divide(num_kernel_elems, 4) * num_ublk
337 if ifm_tensor.dtype.size_in_bits() == 16:
338 cycles *= 2
339 elif (
340 (npu_block_type == NpuBlockType.ConvolutionMxN and block_traversal != TensorBlockTraversal.PartKernelFirst)
341 or npu_block_type == NpuBlockType.VectorProduct
342 or npu_block_type == NpuBlockType.ReduceSum
343 ):
344 cycles = 4 * num_kernel_elems * num_ublk * numeric_util.round_up_divide(ifm_tens_shape[3], ifm_blk_depth)
345 else:
346 assert block_traversal == TensorBlockTraversal.PartKernelFirst
347 divider = 2 if ifm_tensor.dtype.size_in_bits() == 16 else 4
348 cycles = 4 * (
349 numeric_util.round_up_divide(num_kernel_elems, divider)
350 * numeric_util.round_up_divide(ifm_blk_depth, 8)
351 * num_ublk
352 * numeric_util.round_up_divide(ifm_tens_shape[3], ifm_blk_depth)
353 )
354 cycles_dpu_blk += cycles
355
356 cycles_dpu_blk /= arch.ncores
357
358 num_ofm_blk = (
359 numeric_util.round_up_divide(ofm_tens_shape[1], block_config.height)
360 * numeric_util.round_up_divide(ofm_tens_shape[2], block_config.width)
361 * numeric_util.round_up_divide(ofm_tens_shape[3], block_config.depth)
362 )
363
Diqing Zhong42e833d2020-10-02 13:18:42 +0200364 cycles_output_blk = estimate_output_cycles(
Diqing Zhong09387e22020-09-28 18:46:22 +0200365 arch, npu_block_type, primary_op, num_elems_blk, ifm_tensor, ofm_tensor, None, use_acc_40bits
366 )
367
368 if cycles_dpu_blk > cycles_output_blk:
369 total_cycles = cycles_dpu_blk * num_ofm_blk + cycles_output_blk
370 else:
371 total_cycles = cycles_output_blk * num_ofm_blk + cycles_dpu_blk
372
373 return total_cycles
374
375
Tim Hall79d07d22020-04-27 18:20:16 +0100376def performance_metrics_for_pass(arch, ps, block_config=None, rewrite_list=[], force_outputs_to_fast_storage=False):
377 if block_config is None:
378 block_config = ps.block_config
379 bws = make_bandwidth_array()
380 macs = make_macs_array()
381 cycles = make_cycles_array()
382 blocks = 0
383 ifm_read_multiple = 1
384 weight_read_multiple = 0
385
386 if ps.placement in set((PassPlacement.MemoryOnly, PassPlacement.StartupInit)):
387 return bws, macs, cycles, blocks, ifm_read_multiple, weight_read_multiple # nothing real happening in this pass
388
389 min_block_size = arch.min_block_sizes[ps.npu_block_type]
390
391 skirt = (0, 0, 0, 0)
392 explicit_padding = (0, 0, 0, 0)
393 primary_op = ps.primary_op
394 replacement_read_bws = {}
Charles Xub02c8d92020-06-25 16:05:25 +0200395 if ps.placement == PassPlacement.Cpu:
396 cycles[PassCycles.Cpu] = arch.cpu_cycle_estimate(ps.ops[0])
397 elif primary_op:
Tim Hall79d07d22020-04-27 18:20:16 +0100398 skirt = primary_op.attrs.get("skirt", skirt)
399 explicit_padding = primary_op.attrs.get("explicit_padding", explicit_padding)
Louis Verhaardaee5d752020-09-30 09:01:52 +0200400 assert primary_op.type.npu_block_type == ps.npu_block_type
401 npu_block_type = primary_op.type.npu_block_type
Diqing Zhong42e833d2020-10-02 13:18:42 +0200402 block_traversal = TensorBlockTraversal.Default
Tim Hall79d07d22020-04-27 18:20:16 +0100403
404 ifm_tensor, _, weight_tensor, ofm_tensor = ps.get_primary_op_ifm_ifm2_weights_ofm()
405
Tim Hallc30f4952020-06-15 20:47:35 +0100406 if npu_block_type in set(
Diqing Zhong09387e22020-09-28 18:46:22 +0200407 (
408 NpuBlockType.ConvolutionMxN,
409 NpuBlockType.ConvolutionDepthWise,
410 NpuBlockType.Pooling,
411 NpuBlockType.ReduceSum,
412 )
Tim Hallc30f4952020-06-15 20:47:35 +0100413 ):
Charles Xu3e9c4342020-04-22 08:31:43 +0200414 # extent the ifm to full dimension
415 ifm_tensor_brick_size = tuple(numeric_util.full_shape(4, list(ifm_tensor.brick_size), 1))
416 ifm_tensor_shape = numeric_util.full_shape(4, ifm_tensor.shape, 1)
417 ifm_tensor_bandwidth_shape = numeric_util.full_shape(4, ifm_tensor.bandwidth_shape, 1)
Tim Hall79d07d22020-04-27 18:20:16 +0100418
Diqing Zhong42e833d2020-10-02 13:18:42 +0200419 batch_size = ifm_tensor_shape[0]
Charles Xu3e9c4342020-04-22 08:31:43 +0200420 ifm_depth = ifm_tensor_bandwidth_shape[3]
Tim Hall79d07d22020-04-27 18:20:16 +0100421
422 # add in padding
423 ifm_tensor_shape[1] += explicit_padding[0] + explicit_padding[2] # height += top and bottom
424 ifm_tensor_shape[2] += explicit_padding[1] + explicit_padding[3] # width += left and right
425
426 strides = primary_op.attrs["strides"]
427 if npu_block_type != NpuBlockType.Pooling:
Diqing Zhong09387e22020-09-28 18:46:22 +0200428 if npu_block_type == NpuBlockType.ReduceSum:
429 block_traversal = TensorBlockTraversal.DepthFirst
430 weight_tensor_shape = [1, 1, ifm_tensor.shape[3], ofm_tensor.shape[3]]
431 weight_tensor_bandwidth_shape = [0] * 4
432 weight_tensor_element_size = 0
433 weight_tensor_bandwidth_compression_scale = 0.0
434 else:
435 block_traversal = weight_tensor.block_traversal
436 weight_tensor_shape = weight_tensor.shape
437 weight_tensor_bandwidth_shape = weight_tensor.bandwidth_shape
438 weight_tensor_element_size = weight_tensor.element_size()
439 weight_tensor_bandwidth_compression_scale = weight_tensor.bandwidth_compression_scale
Tim Hall79d07d22020-04-27 18:20:16 +0100440 nn_ops = (
441 int(ofm_tensor.shape[0])
442 * int(ofm_tensor.shape[1])
443 * int(ofm_tensor.shape[2])
444 * int(weight_tensor_shape[0])
445 * int(weight_tensor_shape[1])
446 * int(weight_tensor_shape[2])
447 * int(weight_tensor_shape[3])
Tim Hall79d07d22020-04-27 18:20:16 +0100448 )
449 else:
450 weight_tensor_shape = [
451 primary_op.attrs["ksize"][1],
452 primary_op.attrs["ksize"][2],
453 1,
454 ifm_tensor_shape[3],
455 ]
456 weight_tensor_bandwidth_shape = weight_tensor_shape
457 weight_tensor_element_size = 0
458 weight_tensor_bandwidth_compression_scale = 0.0
459 nn_ops = 0 # pooling doesn't count as NN ops
460
461 kernel_dims = weight_tensor_shape[:2]
462
463 sub_kernel_limits = arch.sub_kernel_limits[npu_block_type]
464 # count the sub kernels; the IFM block needs to be refetched for each of them
465 n_sub_kernels_y = numeric_util.round_up_divide(kernel_dims[0], sub_kernel_limits[0])
466 n_sub_kernels_x = numeric_util.round_up_divide(kernel_dims[1], sub_kernel_limits[1])
467 n_sub_kernels = n_sub_kernels_y * n_sub_kernels_x
468
469 clamped_skirt = list(skirt)
470 clamped_skirt[2] = min(clamped_skirt[2], sub_kernel_limits[0] - 1 - clamped_skirt[0])
471 clamped_skirt[3] = min(clamped_skirt[3], sub_kernel_limits[1] - 1 - clamped_skirt[1])
472 n_blocks, area, block_setup = get_n_blocks_and_area(
Charles Xu3e9c4342020-04-22 08:31:43 +0200473 ifm_tensor_brick_size,
Tim Hall79d07d22020-04-27 18:20:16 +0100474 ifm_tensor_shape[1:3],
475 skirt,
476 clamped_skirt,
477 block_config,
478 min_block_size,
479 strides,
480 )
481
482 blocks = n_blocks * numeric_util.round_up_divide(weight_tensor_shape[3], block_config[3])
483
484 n_weight_stages = numeric_util.round_up_divide(weight_tensor_bandwidth_shape[3], block_config[3])
485 if npu_block_type == NpuBlockType.ConvolutionDepthWise or npu_block_type == NpuBlockType.Pooling:
486 n_weight_stages = 1 # force to no reread
487
488 ifm_tensor_bw = (
489 n_sub_kernels
490 * batch_size
491 * area
492 * ifm_depth
493 * n_weight_stages
494 * ifm_tensor.element_size()
495 * ifm_tensor.bandwidth_compression_scale
496 )
497 replacement_read_bws[ifm_tensor] = ifm_tensor_bw
498 ifm_read_multiple = n_weight_stages
499
500 replacement_read_bws[weight_tensor] = (
501 batch_size
502 * shape_num_elements(weight_tensor_bandwidth_shape)
503 * weight_tensor_element_size
504 * weight_tensor_bandwidth_compression_scale
505 * n_blocks
506 ) # read once per block and batch
507 weight_read_multiple = n_blocks
508
509 n_kernel_xy = kernel_dims[0] * kernel_dims[1]
510 n_input_channels_at_a_time = block_config[2]
511
Diqing Zhong09387e22020-09-28 18:46:22 +0200512 if npu_block_type == NpuBlockType.Pooling or block_traversal in set(
Tim Hall79d07d22020-04-27 18:20:16 +0100513 (TensorBlockTraversal.PartKernelFirst, TensorBlockTraversal.DepthWise)
514 ):
515 n_input_channels_at_a_time = numeric_util.round_up_divide(n_input_channels_at_a_time, 4)
516 n_kernel_xy = max(
517 n_kernel_xy, 4
518 ) # need at least 4, as this is the minimum duty cycle for secondary accumulator writes
519 if weight_tensor is not None:
Diego Russoea6111a2020-04-14 18:41:58 +0100520 n_kernel_xy = numeric_util.round_up(n_kernel_xy, 4) # weights need to be read in blocks of 4
Tim Hall79d07d22020-04-27 18:20:16 +0100521
522 num_mac_ops = 0
523 for n_blocks_for_size, block_size in block_setup:
524 num_mac_ops += (
525 batch_size
526 * n_blocks_for_size
527 * block_size[0]
528 * block_size[1]
529 * numeric_util.round_up(weight_tensor_shape[2], n_input_channels_at_a_time)
530 * numeric_util.round_up(weight_tensor_shape[3], block_config[3])
531 * n_kernel_xy
532 )
533
Tim Hall79d07d22020-04-27 18:20:16 +0100534 macs[MacCount.NeuralNetworkMacs] += nn_ops
535 macs[MacCount.HardwareMacs] += num_mac_ops
Diqing Zhong42e833d2020-10-02 13:18:42 +0200536 cycles[PassCycles.Npu] = estimate_conv_pooling_cycles(
Diqing Zhong09387e22020-09-28 18:46:22 +0200537 arch,
538 npu_block_type,
539 primary_op,
540 Block(block_config[1], block_config[0], block_config[3]),
541 block_traversal,
542 kernel_dims,
543 ifm_tensor,
544 ofm_tensor,
545 )
Tim Hall79d07d22020-04-27 18:20:16 +0100546 elif npu_block_type == NpuBlockType.VectorProduct:
547 nn_macs = (
548 ifm_tensor.shape[0]
549 * numeric_util.round_up(weight_tensor.shape[-2], block_config[2])
550 * numeric_util.round_up(weight_tensor.shape[-1], block_config[3])
551 )
552 num_mac_ops = nn_macs
553
Diqing Zhong42e833d2020-10-02 13:18:42 +0200554 cycles[PassCycles.Npu] = estimate_conv_pooling_cycles(
Diqing Zhong09387e22020-09-28 18:46:22 +0200555 arch,
556 npu_block_type,
557 primary_op,
558 Block(block_config[1], block_config[0], block_config[3]),
559 weight_tensor.block_traversal,
560 [1, 1],
561 ifm_tensor,
562 ofm_tensor,
563 )
Tim Hall79d07d22020-04-27 18:20:16 +0100564 macs[MacCount.NeuralNetworkMacs] += nn_macs
565 macs[MacCount.HardwareMacs] += num_mac_ops
566
567 blocks = 1 * numeric_util.round_up_divide(weight_tensor.shape[-1], block_config[3])
568
569 non_zero_fraction = 1.0
570 if ifm_tensor.values is not None:
571 nz_vector = np.amax(ifm_tensor.values != 0, axis=0) # max across batch axis
572 non_zero_fraction = np.average(nz_vector)
573
574 replacement_read_bws[ifm_tensor] = ifm_tensor.bandwidth()
575 replacement_read_bws[weight_tensor] = weight_tensor.bandwidth() * non_zero_fraction
576 ifm_read_multiple = 1
577 weight_read_multiple = non_zero_fraction
Diqing Zhonge8887a32020-09-24 09:53:48 +0200578 elif npu_block_type == NpuBlockType.ElementWise:
Tim Hall79d07d22020-04-27 18:20:16 +0100579 # Work out how many elements we have and calculate performance.
Diqing Zhong42e833d2020-10-02 13:18:42 +0200580 cycles[PassCycles.Npu] = estimate_output_cycles(
Diqing Zhong09387e22020-09-28 18:46:22 +0200581 arch, npu_block_type, primary_op, ofm_tensor.elements(), ps.ifm_tensor, ps.ofm_tensor, ps.ifm2_tensor
582 )
Diqing Zhong42e833d2020-10-02 13:18:42 +0200583
584 prev_npu_pass = next((npu_ps for npu_ps in ps.dag_predecessors if npu_ps.placement is PassPlacement.Npu), None)
585 if prev_npu_pass is None:
586 # cycles for DMA ops in first pass
587 dma_ops = (op for op in ps.ops if op.type == Op.DMA)
588 for dma_op in dma_ops:
589 mem_area = dma_op.attrs["source"]
590 for tens in dma_op.inputs:
591 cycles[PassCycles.Npu] += tens.storage_size() / arch.memory_bandwidths_per_cycle[mem_area]
592
Tim Hall79d07d22020-04-27 18:20:16 +0100593 # apply the desired rewrites
594 for rewrite_op, tens, _, _, _, ps_to_rewrite in rewrite_list:
595 if ps != ps_to_rewrite:
596 continue
597 if rewrite_op == SchedulerRewrite.Nop:
598 pass # these are fine, no bandwidth changes
599 elif rewrite_op in (SchedulerRewrite.ChangeTensorSubPurpose,):
600 bws[arch.fast_storage_mem_area][tens.purpose][BandwidthDirection.Read] += replacement_read_bws[tens]
601 replacement_read_bws[tens] = 0
602
603 for tens in ps.outputs:
604 if force_outputs_to_fast_storage:
605 bws[arch.fast_storage_mem_area][tens.purpose][BandwidthDirection.Write] += tens.bandwidth()
606 else:
607 bws[tens.mem_area][tens.purpose][BandwidthDirection.Write] += tens.bandwidth()
608
609 for tens in ps.intermediates:
610 bws[tens.mem_area][tens.purpose][BandwidthDirection.Write] += tens.bandwidth()
611
612 if tens in replacement_read_bws:
613 bw = replacement_read_bws[tens]
614 else:
615 bw = tens.bandwidth()
616
617 bws[tens.mem_area][tens.purpose][BandwidthDirection.Read] += bw
618
619 for tens in ps.inputs:
620 if tens in replacement_read_bws:
621 bw = replacement_read_bws[tens]
622 else:
623 bw = tens.bandwidth()
624
625 bws[tens.mem_area][tens.purpose][BandwidthDirection.Read] += bw
626
627 cycles[PassCycles.SramAccess] = np.sum(bws[MemArea.Sram]) / arch.memory_bandwidths_per_cycle[MemArea.Sram]
628 cycles[PassCycles.TotalPerPass] = np.max(cycles[: PassCycles.TotalPerPass])
629
630 # quick build access counts for only current pass, even though these aren't the final numbers
631 update_summary_cycles(arch, bws, macs, cycles)
632
633 return bws, macs, cycles, blocks, ifm_read_multiple, weight_read_multiple
634
635
636def update_summary_cycles(arch, bws, macs, cycles):
637 cycles[PassCycles.DramAccess] = np.sum(bws[MemArea.Dram]) / arch.memory_bandwidths_per_cycle[MemArea.Dram]
638 cycles[PassCycles.OnChipFlashAccess] = (
639 np.sum(bws[MemArea.OnChipFlash]) / arch.memory_bandwidths_per_cycle[MemArea.OnChipFlash]
640 )
641 cycles[PassCycles.OffChipFlashAccess] = (
642 np.sum(bws[MemArea.OffChipFlash]) / arch.memory_bandwidths_per_cycle[MemArea.OffChipFlash]
643 )
644
645 cycles[PassCycles.Total] = np.max(cycles[: PassCycles.Total])
646 return cycles
647
648
649def collate_stats_for_cascaded_pass(arch, bws, macs, cycles):
650 return bws, macs, cycles
651
652
653def performance_for_cascaded_pass(arch, cps):
654 total_bws = make_bandwidth_array()
655 total_macs = make_macs_array()
656 total_cycles = make_cycles_array()
657
658 for ps in cps.passes:
659 bws, macs, cycles, blocks, _, _ = performance_metrics_for_pass(arch, ps)
660 ps.bandwidths = bws
661 ps.macs = macs
662 ps.cycles = cycles
663 ps.n_blocks = blocks
664 total_bws += bws
665 total_macs += macs
666 total_cycles += cycles
667
668 bws, macs, cycles = collate_stats_for_cascaded_pass(arch, total_bws, total_macs, total_cycles)
669 cps.bandwidths = bws
670 cps.macs = macs
671 cps.cycles = cycles
672 return bws, macs, cycles
673
674
675def calc_performance_for_network(nng, arch):
676 total_bws = make_bandwidth_array()
677 total_macs = np.zeros(MacCount.Size)
678 total_cycles = np.zeros(PassCycles.Size)
679
680 for sg in nng.subgraphs:
681 for cps in sg.cascaded_passes:
682 bws, macs, cycles = performance_for_cascaded_pass(arch, cps)
683 total_bws += bws
684 total_macs += macs
685 total_cycles += cycles
Tim Hall79d07d22020-04-27 18:20:16 +0100686
687 nng.bandwidths = total_bws
688 nng.macs = total_macs
689 nng.cycles = total_cycles