blob: 4d221bea550897177695d851e0653b8ea8b82065 [file] [log] [blame]
Tim Hall79d07d22020-04-27 18:20:16 +01001# Copyright (C) 2020 Arm Limited or its affiliates. All rights reserved.
2#
3# SPDX-License-Identifier: Apache-2.0
4#
5# Licensed under the Apache License, Version 2.0 (the License); you may
6# not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9# www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an AS IS BASIS, WITHOUT
13# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
Tim Hall79d07d22020-04-27 18:20:16 +010016# Description:
17# NPU performance estimation functions to estimate performance of a Pass and CascadedPass. Uses a model that takes the
18# maximum of the 'cycles required for bandwidth' and 'cycles required for computing'.
19#
20# Called during scheduling to evaluate different proposals, as well as post-scheduling to provide a final performance
21# estimate.
Tim Hall79d07d22020-04-27 18:20:16 +010022import enum
Diego Russoea6111a2020-04-14 18:41:58 +010023
Tim Hall79d07d22020-04-27 18:20:16 +010024import numpy as np
Diego Russoea6111a2020-04-14 18:41:58 +010025
26from . import numeric_util
Diqing Zhong09387e22020-09-28 18:46:22 +020027from .architecture_features import Accelerator
Diego Russoe8a10452020-04-21 17:39:10 +010028from .architecture_features import Block
Diqing Zhonge8887a32020-09-24 09:53:48 +020029from .data_type import DataType
Diego Russoe8a10452020-04-21 17:39:10 +010030from .nn_graph import PassPlacement
31from .nn_graph import SchedulerRewrite
Diego Russoea6111a2020-04-14 18:41:58 +010032from .operation import NpuBlockType
Diqing Zhonge8887a32020-09-24 09:53:48 +020033from .operation import Op
Diqing Zhong09387e22020-09-28 18:46:22 +020034from .shared_buffer_allocation import is_acc_40bits_used
Diego Russoe8a10452020-04-21 17:39:10 +010035from .tensor import MemArea
36from .tensor import shape_num_elements
37from .tensor import TensorBlockTraversal
38from .tensor import TensorPurpose
Tim Hall79d07d22020-04-27 18:20:16 +010039
40
41def rolling_buffer_dims_from_passes(arch, ps1, block_config_ps1, ps2, block_config_ps2):
Tim Hall79d07d22020-04-27 18:20:16 +010042 ofm_block = Block(block_config_ps2[-3], block_config_ps2[-4], block_config_ps2[-1])
Tim Hall4ed38bc2020-10-20 18:54:20 +010043 kernel = ps2.primary_op.kernel
Tim Hall79d07d22020-04-27 18:20:16 +010044
45 if ps2.npu_block_type in set((NpuBlockType.ConvolutionMxN, NpuBlockType.VectorProduct)):
Louis Verhaard93dc5532020-06-07 12:40:18 +020046 op = ps2.primary_op
Louis Verhaardaee5d752020-09-30 09:01:52 +020047 ifm_block_depth = arch.calc_ifm_block_depth(op.ifm.shape[-1], op.ifm.dtype.size_in_bits())
Tim Hall79d07d22020-04-27 18:20:16 +010048 else:
49 ifm_block_depth = block_config_ps2[-1]
50
Louis Verhaard93dc5532020-06-07 12:40:18 +020051 ifm_block = arch.get_ifm_block_size(ifm_block_depth, ofm_block, kernel, arch.ofm_block_max)
Tim Hall79d07d22020-04-27 18:20:16 +010052
53 # The performed height calculation is for worst case
54 height = numeric_util.round_up(ifm_block.height + block_config_ps1[0], block_config_ps1[0])
55 width = ifm_block.width
Louis Verhaard93dc5532020-06-07 12:40:18 +020056 return [height, width]
Tim Hall79d07d22020-04-27 18:20:16 +010057
58
59class PassCycles(enum.IntEnum):
60 Dpu = 0
61 ElementWise = 1
62 Cpu = 2
63 SramAccess = 3
64 TotalPerPass = 4
65 DramAccess = 5
66 OnChipFlashAccess = 6
67 OffChipFlashAccess = 7
68 Total = 8
69 Size = 9
70
71 def display_name(self):
72 return (
73 "DPU",
74 "Element wise",
75 "CPU",
76 "SRAM Access",
77 "Total per Pass",
78 "DRAM Access",
79 "On-chip Flash Access",
80 "Off-chip Flash Access",
81 "Total",
82 "Size",
83 )[self.value]
84
85 def identifier_name(self):
86 return (
87 "dpu",
88 "element_wise",
89 "cpu",
90 "sram_access",
91 "total_per_pass",
92 "dram_access",
93 "on_chip_flash_access",
94 "off_chip_flash_access",
95 "total",
96 "size",
97 )[self.value]
98
99 @staticmethod
100 def all():
101 return (
102 PassCycles.Dpu,
103 PassCycles.ElementWise,
104 PassCycles.Cpu,
105 PassCycles.SramAccess,
106 PassCycles.DramAccess,
107 PassCycles.OnChipFlashAccess,
108 PassCycles.OffChipFlashAccess,
109 PassCycles.Total,
110 )
111
112
113class MacCount(enum.IntEnum):
114 NeuralNetworkMacs = 0
115 HardwareMacs = 1
116 Size = 2
117
118 def display_name(self):
119 return ("Neural Network Macs", "Hardware Macs", "Size")[self.value]
120
121 def identifier_name(self):
122 return ("nn_macs", "hardware_macs", "size")[self.value]
123
124 @staticmethod
125 def all():
126 return (MacCount.NeuralNetworkMacs, MacCount.HardwareMacs)
127
128
129class BandwidthDirection(enum.IntEnum):
130 Read = 0
131 Write = 1
132 Size = 2
133
134 def display_name(self):
135 return self.name
136
137 def identifier_name(self):
138 return self.name.lower()
139
140 @staticmethod
141 def all():
142 return (BandwidthDirection.Read, BandwidthDirection.Write)
143
144
145def make_bandwidth_array():
146 return np.zeros((MemArea.Size, TensorPurpose.Size, BandwidthDirection.Size))
147
148
149def make_macs_array():
150 return np.zeros(MacCount.Size, np.int)
151
152
153def make_cycles_array():
154 return np.zeros(PassCycles.Size)
155
156
157def make_metrics_arrays():
158 return (make_bandwidth_array(), make_macs_array(), make_cycles_array())
159
160
161def get_n_blocks_and_area(
162 ifm_brick_size, ifm_height_width, orig_skirt, clamped_skirt, block_config, min_block_size, strides
163):
164
165 ifm_block_config = (block_config[0] * strides[1], block_config[1] * strides[2])
166
167 n_normal_blocks = []
168 remainder_size = []
169 for i in range(2):
170 non_skirt_dim = ifm_height_width[i] - orig_skirt[i] - orig_skirt[2 + i]
171 n_blocks = non_skirt_dim // ifm_block_config[i]
172 n_normal_blocks.append(n_blocks)
173 remainder_dim = numeric_util.round_up(
174 ((non_skirt_dim - n_blocks * ifm_block_config[i] - 1) // strides[i + 1]) + 1, min_block_size[i]
175 )
176 remainder_size.append(remainder_dim)
177
178 # this will actually calculate reads into the edge padding.
179
180 # there are four cases in total, handling the edges that will not fill a complete block.
181
182 # 0000000001
183 # 0000000001
184 # 0000000001
185 # 0000000001
186 # 0000000001
187 # 0000000001
188 # 2222222223
189 total_blocks = 0
190 total_area = 0
191
192 block_setup = (
193 (n_normal_blocks[0] * n_normal_blocks[1], block_config),
194 (1 * n_normal_blocks[1], (remainder_size[0], block_config[1])),
195 (n_normal_blocks[0] * 1, (block_config[0], remainder_size[1])),
196 (1 * 1, remainder_size),
197 )
198
199 for n_blocks, block_size in block_setup:
200 if block_size[0] == 0 or block_size[1] == 0:
201 continue
202 read_dims = [0, 0]
203 for i in range(2):
204 read_dims[i] = (
205 numeric_util.round_up(clamped_skirt[i], ifm_brick_size[i + 1])
206 + block_size[i] * strides[i + 1]
207 + numeric_util.round_up(clamped_skirt[2 + i], ifm_brick_size[i + 1])
208 )
209 assert n_blocks >= 0
210 total_blocks += n_blocks
211 total_area += n_blocks * read_dims[0] * read_dims[1]
212 assert total_blocks >= 1
213 return total_blocks, total_area, block_setup
214
215
Diqing Zhong09387e22020-09-28 18:46:22 +0200216def get_output_cycle_estimate(
217 arch, npu_block_type, primary_op, num_elems, ifm_tensor, ofm_tensor, ifm2_tensor, use_acc_40bits=False
218):
Diqing Zhonge8887a32020-09-24 09:53:48 +0200219 faf = primary_op.activation
Diqing Zhong09387e22020-09-28 18:46:22 +0200220 if npu_block_type == NpuBlockType.ElementWise and ifm_tensor.dtype == DataType.int32:
221 if ifm2_tensor is None:
Diqing Zhonge8887a32020-09-24 09:53:48 +0200222 # Unary op
223 output_perf_index = 0
224 else:
225 # Binary op
226 output_perf_index = 1
Diqing Zhong09387e22020-09-28 18:46:22 +0200227 elif primary_op.type == Op.Mul and ofm_tensor.dtype == DataType.int32:
Diqing Zhonge8887a32020-09-24 09:53:48 +0200228 output_perf_index = 2
Diqing Zhong09387e22020-09-28 18:46:22 +0200229 elif primary_op.type == Op.Mul or (
Diqing Zhonge8887a32020-09-24 09:53:48 +0200230 npu_block_type
231 in (
232 NpuBlockType.ConvolutionMxN,
233 NpuBlockType.ConvolutionDepthWise,
234 NpuBlockType.Pooling,
235 NpuBlockType.ReduceSum,
236 NpuBlockType.VectorProduct,
237 )
Diqing Zhong09387e22020-09-28 18:46:22 +0200238 and use_acc_40bits
Diqing Zhonge8887a32020-09-24 09:53:48 +0200239 ):
240 output_perf_index = 3
Diqing Zhong09387e22020-09-28 18:46:22 +0200241 elif primary_op.type in (Op.Add, Op.Sub):
242 input_scale = ifm_tensor.quantization.scale_f32
243 input2_scale = ifm2_tensor.quantization.scale_f32
244 output_scale = ofm_tensor.quantization.scale_f32
Diqing Zhonge8887a32020-09-24 09:53:48 +0200245
246 if "resizebilinear" in primary_op.attrs:
247 output_scale = input2_scale
248
249 if None in (input_scale, input2_scale, output_scale) or input_scale == input2_scale:
250 # Simple Add/Sub
251 output_perf_index = 4
252 else:
253 # Advanced Add/Sub
254 output_perf_index = 5
Diqing Zhong09387e22020-09-28 18:46:22 +0200255 elif primary_op.type.is_maxpool_op():
Diqing Zhonge8887a32020-09-24 09:53:48 +0200256 output_perf_index = 6
257 else:
258 output_perf_index = 7
259
260 if faf in (Op.Sigmoid, Op.Tanh, Op.LUT):
261 activation_perf_index = 0
262 elif faf in (Op.Relu, Op.Relu6, Op.ReluN1To1):
263 activation_perf_index = 1
264 else:
265 activation_perf_index = 2
266
Diqing Zhonge8887a32020-09-24 09:53:48 +0200267 cycle_per_elem = max(
268 arch.output_cycles_per_elem[output_perf_index], arch.activation_cycles_per_elem[activation_perf_index]
269 )
270 return num_elems * cycle_per_elem
271
272
Diqing Zhong09387e22020-09-28 18:46:22 +0200273def get_conv_pooling_cycle_estimate(
274 arch, npu_block_type, primary_op, block_config: Block, block_traversal, kernel_dims, ifm_tensor, ofm_tensor
275):
276 num_ublk = (
277 (block_config.width // arch.config.ofm_ublock.width)
278 * (block_config.height // arch.config.ofm_ublock.height)
279 * (block_config.depth // arch.config.ofm_ublock.depth)
280 )
281 num_ofm_blk = 0
282 total_cycles = 0
283 num_elems_blk = block_config.width * block_config.height * block_config.depth
284 ifm_tens_shape = numeric_util.full_shape(4, ifm_tensor.shape, 1)
285 ofm_tens_shape = numeric_util.full_shape(4, ofm_tensor.shape, 1)
286 use_acc_40bits = is_acc_40bits_used(npu_block_type, ifm_tensor, ofm_tensor)
287
288 sub_kernel_limits = arch.sub_kernel_limits[npu_block_type]
289 n_sub_kernels_y = numeric_util.round_up_divide(kernel_dims[0], sub_kernel_limits[0])
290 n_sub_kernels_x = numeric_util.round_up_divide(kernel_dims[1], sub_kernel_limits[1])
291 sub_kernel_x = [
292 min((kernel_dims[1] - i * sub_kernel_limits[1]), sub_kernel_limits[1]) for i in range(n_sub_kernels_x)
293 ]
294 sub_kernel_y = [
295 min((kernel_dims[0] - i * sub_kernel_limits[0]), sub_kernel_limits[0]) for i in range(n_sub_kernels_y)
296 ]
297 sub_kernel_size = (x * y for y in sub_kernel_y for x in sub_kernel_x)
298
299 ifm_blk_depth = 0
300 if npu_block_type != NpuBlockType.Pooling:
301 if ifm_tensor.dtype.size_in_bits() == 16 or block_traversal == TensorBlockTraversal.PartKernelFirst:
302 ifm_blk_depth = 16
303 elif ifm_tensor.dtype.size_in_bits() == 8:
304 ifm_blk_depth = 32
305 else:
306 ifm_blk_depth = 8
307
308 cycles_dpu_blk = 0
309
310 for num_kernel_elems in sub_kernel_size:
311 if npu_block_type == NpuBlockType.Pooling:
312 cycles = max(4, num_kernel_elems) * num_ublk
313 if ifm_tensor.dtype.size_in_bits() == 16 and arch.accelerator_config != Accelerator.Ethos_U55_32:
314 cycles *= 2
315 elif npu_block_type == NpuBlockType.ConvolutionDepthWise:
316 cycles = 4 * numeric_util.round_up_divide(num_kernel_elems, 4) * num_ublk
317 if ifm_tensor.dtype.size_in_bits() == 16:
318 cycles *= 2
319 elif (
320 (npu_block_type == NpuBlockType.ConvolutionMxN and block_traversal != TensorBlockTraversal.PartKernelFirst)
321 or npu_block_type == NpuBlockType.VectorProduct
322 or npu_block_type == NpuBlockType.ReduceSum
323 ):
324 cycles = 4 * num_kernel_elems * num_ublk * numeric_util.round_up_divide(ifm_tens_shape[3], ifm_blk_depth)
325 else:
326 assert block_traversal == TensorBlockTraversal.PartKernelFirst
327 divider = 2 if ifm_tensor.dtype.size_in_bits() == 16 else 4
328 cycles = 4 * (
329 numeric_util.round_up_divide(num_kernel_elems, divider)
330 * numeric_util.round_up_divide(ifm_blk_depth, 8)
331 * num_ublk
332 * numeric_util.round_up_divide(ifm_tens_shape[3], ifm_blk_depth)
333 )
334 cycles_dpu_blk += cycles
335
336 cycles_dpu_blk /= arch.ncores
337
338 num_ofm_blk = (
339 numeric_util.round_up_divide(ofm_tens_shape[1], block_config.height)
340 * numeric_util.round_up_divide(ofm_tens_shape[2], block_config.width)
341 * numeric_util.round_up_divide(ofm_tens_shape[3], block_config.depth)
342 )
343
344 cycles_output_blk = get_output_cycle_estimate(
345 arch, npu_block_type, primary_op, num_elems_blk, ifm_tensor, ofm_tensor, None, use_acc_40bits
346 )
347
348 if cycles_dpu_blk > cycles_output_blk:
349 total_cycles = cycles_dpu_blk * num_ofm_blk + cycles_output_blk
350 else:
351 total_cycles = cycles_output_blk * num_ofm_blk + cycles_dpu_blk
352
353 return total_cycles
354
355
Tim Hall79d07d22020-04-27 18:20:16 +0100356def performance_metrics_for_pass(arch, ps, block_config=None, rewrite_list=[], force_outputs_to_fast_storage=False):
357 if block_config is None:
358 block_config = ps.block_config
359 bws = make_bandwidth_array()
360 macs = make_macs_array()
361 cycles = make_cycles_array()
362 blocks = 0
363 ifm_read_multiple = 1
364 weight_read_multiple = 0
365
366 if ps.placement in set((PassPlacement.MemoryOnly, PassPlacement.StartupInit)):
367 return bws, macs, cycles, blocks, ifm_read_multiple, weight_read_multiple # nothing real happening in this pass
368
369 min_block_size = arch.min_block_sizes[ps.npu_block_type]
370
371 skirt = (0, 0, 0, 0)
372 explicit_padding = (0, 0, 0, 0)
373 primary_op = ps.primary_op
374 replacement_read_bws = {}
Charles Xub02c8d92020-06-25 16:05:25 +0200375 if ps.placement == PassPlacement.Cpu:
376 cycles[PassCycles.Cpu] = arch.cpu_cycle_estimate(ps.ops[0])
377 elif primary_op:
Tim Hall79d07d22020-04-27 18:20:16 +0100378 skirt = primary_op.attrs.get("skirt", skirt)
379 explicit_padding = primary_op.attrs.get("explicit_padding", explicit_padding)
Louis Verhaardaee5d752020-09-30 09:01:52 +0200380 assert primary_op.type.npu_block_type == ps.npu_block_type
381 npu_block_type = primary_op.type.npu_block_type
Tim Hall79d07d22020-04-27 18:20:16 +0100382
383 ifm_tensor, _, weight_tensor, ofm_tensor = ps.get_primary_op_ifm_ifm2_weights_ofm()
384
Tim Hallc30f4952020-06-15 20:47:35 +0100385 if npu_block_type in set(
Diqing Zhong09387e22020-09-28 18:46:22 +0200386 (
387 NpuBlockType.ConvolutionMxN,
388 NpuBlockType.ConvolutionDepthWise,
389 NpuBlockType.Pooling,
390 NpuBlockType.ReduceSum,
391 )
Tim Hallc30f4952020-06-15 20:47:35 +0100392 ):
Charles Xu3e9c4342020-04-22 08:31:43 +0200393 # extent the ifm to full dimension
394 ifm_tensor_brick_size = tuple(numeric_util.full_shape(4, list(ifm_tensor.brick_size), 1))
395 ifm_tensor_shape = numeric_util.full_shape(4, ifm_tensor.shape, 1)
396 ifm_tensor_bandwidth_shape = numeric_util.full_shape(4, ifm_tensor.bandwidth_shape, 1)
Tim Hall79d07d22020-04-27 18:20:16 +0100397
398 batch_size = ifm_tensor.shape[0]
Charles Xu3e9c4342020-04-22 08:31:43 +0200399 ifm_depth = ifm_tensor_bandwidth_shape[3]
Tim Hall79d07d22020-04-27 18:20:16 +0100400
401 # add in padding
402 ifm_tensor_shape[1] += explicit_padding[0] + explicit_padding[2] # height += top and bottom
403 ifm_tensor_shape[2] += explicit_padding[1] + explicit_padding[3] # width += left and right
404
Diqing Zhong09387e22020-09-28 18:46:22 +0200405 block_traversal = TensorBlockTraversal.Default
406
Tim Hall79d07d22020-04-27 18:20:16 +0100407 strides = primary_op.attrs["strides"]
408 if npu_block_type != NpuBlockType.Pooling:
Diqing Zhong09387e22020-09-28 18:46:22 +0200409 if npu_block_type == NpuBlockType.ReduceSum:
410 block_traversal = TensorBlockTraversal.DepthFirst
411 weight_tensor_shape = [1, 1, ifm_tensor.shape[3], ofm_tensor.shape[3]]
412 weight_tensor_bandwidth_shape = [0] * 4
413 weight_tensor_element_size = 0
414 weight_tensor_bandwidth_compression_scale = 0.0
415 else:
416 block_traversal = weight_tensor.block_traversal
417 weight_tensor_shape = weight_tensor.shape
418 weight_tensor_bandwidth_shape = weight_tensor.bandwidth_shape
419 weight_tensor_element_size = weight_tensor.element_size()
420 weight_tensor_bandwidth_compression_scale = weight_tensor.bandwidth_compression_scale
Tim Hall79d07d22020-04-27 18:20:16 +0100421 nn_ops = (
422 int(ofm_tensor.shape[0])
423 * int(ofm_tensor.shape[1])
424 * int(ofm_tensor.shape[2])
425 * int(weight_tensor_shape[0])
426 * int(weight_tensor_shape[1])
427 * int(weight_tensor_shape[2])
428 * int(weight_tensor_shape[3])
Tim Hall79d07d22020-04-27 18:20:16 +0100429 )
430 else:
431 weight_tensor_shape = [
432 primary_op.attrs["ksize"][1],
433 primary_op.attrs["ksize"][2],
434 1,
435 ifm_tensor_shape[3],
436 ]
437 weight_tensor_bandwidth_shape = weight_tensor_shape
438 weight_tensor_element_size = 0
439 weight_tensor_bandwidth_compression_scale = 0.0
440 nn_ops = 0 # pooling doesn't count as NN ops
441
442 kernel_dims = weight_tensor_shape[:2]
443
444 sub_kernel_limits = arch.sub_kernel_limits[npu_block_type]
445 # count the sub kernels; the IFM block needs to be refetched for each of them
446 n_sub_kernels_y = numeric_util.round_up_divide(kernel_dims[0], sub_kernel_limits[0])
447 n_sub_kernels_x = numeric_util.round_up_divide(kernel_dims[1], sub_kernel_limits[1])
448 n_sub_kernels = n_sub_kernels_y * n_sub_kernels_x
449
450 clamped_skirt = list(skirt)
451 clamped_skirt[2] = min(clamped_skirt[2], sub_kernel_limits[0] - 1 - clamped_skirt[0])
452 clamped_skirt[3] = min(clamped_skirt[3], sub_kernel_limits[1] - 1 - clamped_skirt[1])
453 n_blocks, area, block_setup = get_n_blocks_and_area(
Charles Xu3e9c4342020-04-22 08:31:43 +0200454 ifm_tensor_brick_size,
Tim Hall79d07d22020-04-27 18:20:16 +0100455 ifm_tensor_shape[1:3],
456 skirt,
457 clamped_skirt,
458 block_config,
459 min_block_size,
460 strides,
461 )
462
463 blocks = n_blocks * numeric_util.round_up_divide(weight_tensor_shape[3], block_config[3])
464
465 n_weight_stages = numeric_util.round_up_divide(weight_tensor_bandwidth_shape[3], block_config[3])
466 if npu_block_type == NpuBlockType.ConvolutionDepthWise or npu_block_type == NpuBlockType.Pooling:
467 n_weight_stages = 1 # force to no reread
468
469 ifm_tensor_bw = (
470 n_sub_kernels
471 * batch_size
472 * area
473 * ifm_depth
474 * n_weight_stages
475 * ifm_tensor.element_size()
476 * ifm_tensor.bandwidth_compression_scale
477 )
478 replacement_read_bws[ifm_tensor] = ifm_tensor_bw
479 ifm_read_multiple = n_weight_stages
480
481 replacement_read_bws[weight_tensor] = (
482 batch_size
483 * shape_num_elements(weight_tensor_bandwidth_shape)
484 * weight_tensor_element_size
485 * weight_tensor_bandwidth_compression_scale
486 * n_blocks
487 ) # read once per block and batch
488 weight_read_multiple = n_blocks
489
490 n_kernel_xy = kernel_dims[0] * kernel_dims[1]
491 n_input_channels_at_a_time = block_config[2]
492
Diqing Zhong09387e22020-09-28 18:46:22 +0200493 if npu_block_type == NpuBlockType.Pooling or block_traversal in set(
Tim Hall79d07d22020-04-27 18:20:16 +0100494 (TensorBlockTraversal.PartKernelFirst, TensorBlockTraversal.DepthWise)
495 ):
496 n_input_channels_at_a_time = numeric_util.round_up_divide(n_input_channels_at_a_time, 4)
497 n_kernel_xy = max(
498 n_kernel_xy, 4
499 ) # need at least 4, as this is the minimum duty cycle for secondary accumulator writes
500 if weight_tensor is not None:
Diego Russoea6111a2020-04-14 18:41:58 +0100501 n_kernel_xy = numeric_util.round_up(n_kernel_xy, 4) # weights need to be read in blocks of 4
Tim Hall79d07d22020-04-27 18:20:16 +0100502
503 num_mac_ops = 0
504 for n_blocks_for_size, block_size in block_setup:
505 num_mac_ops += (
506 batch_size
507 * n_blocks_for_size
508 * block_size[0]
509 * block_size[1]
510 * numeric_util.round_up(weight_tensor_shape[2], n_input_channels_at_a_time)
511 * numeric_util.round_up(weight_tensor_shape[3], block_config[3])
512 * n_kernel_xy
513 )
514
Tim Hall79d07d22020-04-27 18:20:16 +0100515 macs[MacCount.NeuralNetworkMacs] += nn_ops
516 macs[MacCount.HardwareMacs] += num_mac_ops
Diqing Zhong09387e22020-09-28 18:46:22 +0200517 cycles[PassCycles.Dpu] = get_conv_pooling_cycle_estimate(
518 arch,
519 npu_block_type,
520 primary_op,
521 Block(block_config[1], block_config[0], block_config[3]),
522 block_traversal,
523 kernel_dims,
524 ifm_tensor,
525 ofm_tensor,
526 )
Tim Hall79d07d22020-04-27 18:20:16 +0100527 elif npu_block_type == NpuBlockType.VectorProduct:
528 nn_macs = (
529 ifm_tensor.shape[0]
530 * numeric_util.round_up(weight_tensor.shape[-2], block_config[2])
531 * numeric_util.round_up(weight_tensor.shape[-1], block_config[3])
532 )
533 num_mac_ops = nn_macs
534
Diqing Zhong09387e22020-09-28 18:46:22 +0200535 cycles[PassCycles.Dpu] = get_conv_pooling_cycle_estimate(
536 arch,
537 npu_block_type,
538 primary_op,
539 Block(block_config[1], block_config[0], block_config[3]),
540 weight_tensor.block_traversal,
541 [1, 1],
542 ifm_tensor,
543 ofm_tensor,
544 )
Tim Hall79d07d22020-04-27 18:20:16 +0100545 macs[MacCount.NeuralNetworkMacs] += nn_macs
546 macs[MacCount.HardwareMacs] += num_mac_ops
547
548 blocks = 1 * numeric_util.round_up_divide(weight_tensor.shape[-1], block_config[3])
549
550 non_zero_fraction = 1.0
551 if ifm_tensor.values is not None:
552 nz_vector = np.amax(ifm_tensor.values != 0, axis=0) # max across batch axis
553 non_zero_fraction = np.average(nz_vector)
554
555 replacement_read_bws[ifm_tensor] = ifm_tensor.bandwidth()
556 replacement_read_bws[weight_tensor] = weight_tensor.bandwidth() * non_zero_fraction
557 ifm_read_multiple = 1
558 weight_read_multiple = non_zero_fraction
Diqing Zhonge8887a32020-09-24 09:53:48 +0200559 elif npu_block_type == NpuBlockType.ElementWise:
Tim Hall79d07d22020-04-27 18:20:16 +0100560 # Work out how many elements we have and calculate performance.
Diqing Zhong09387e22020-09-28 18:46:22 +0200561 cycles[PassCycles.ElementWise] = get_output_cycle_estimate(
562 arch, npu_block_type, primary_op, ofm_tensor.elements(), ps.ifm_tensor, ps.ofm_tensor, ps.ifm2_tensor
563 )
Tim Hall79d07d22020-04-27 18:20:16 +0100564 # apply the desired rewrites
565 for rewrite_op, tens, _, _, _, ps_to_rewrite in rewrite_list:
566 if ps != ps_to_rewrite:
567 continue
568 if rewrite_op == SchedulerRewrite.Nop:
569 pass # these are fine, no bandwidth changes
570 elif rewrite_op in (SchedulerRewrite.ChangeTensorSubPurpose,):
571 bws[arch.fast_storage_mem_area][tens.purpose][BandwidthDirection.Read] += replacement_read_bws[tens]
572 replacement_read_bws[tens] = 0
573
574 for tens in ps.outputs:
575 if force_outputs_to_fast_storage:
576 bws[arch.fast_storage_mem_area][tens.purpose][BandwidthDirection.Write] += tens.bandwidth()
577 else:
578 bws[tens.mem_area][tens.purpose][BandwidthDirection.Write] += tens.bandwidth()
579
580 for tens in ps.intermediates:
581 bws[tens.mem_area][tens.purpose][BandwidthDirection.Write] += tens.bandwidth()
582
583 if tens in replacement_read_bws:
584 bw = replacement_read_bws[tens]
585 else:
586 bw = tens.bandwidth()
587
588 bws[tens.mem_area][tens.purpose][BandwidthDirection.Read] += bw
589
590 for tens in ps.inputs:
591 if tens in replacement_read_bws:
592 bw = replacement_read_bws[tens]
593 else:
594 bw = tens.bandwidth()
595
596 bws[tens.mem_area][tens.purpose][BandwidthDirection.Read] += bw
597
598 cycles[PassCycles.SramAccess] = np.sum(bws[MemArea.Sram]) / arch.memory_bandwidths_per_cycle[MemArea.Sram]
599 cycles[PassCycles.TotalPerPass] = np.max(cycles[: PassCycles.TotalPerPass])
600
601 # quick build access counts for only current pass, even though these aren't the final numbers
602 update_summary_cycles(arch, bws, macs, cycles)
603
604 return bws, macs, cycles, blocks, ifm_read_multiple, weight_read_multiple
605
606
607def update_summary_cycles(arch, bws, macs, cycles):
608 cycles[PassCycles.DramAccess] = np.sum(bws[MemArea.Dram]) / arch.memory_bandwidths_per_cycle[MemArea.Dram]
609 cycles[PassCycles.OnChipFlashAccess] = (
610 np.sum(bws[MemArea.OnChipFlash]) / arch.memory_bandwidths_per_cycle[MemArea.OnChipFlash]
611 )
612 cycles[PassCycles.OffChipFlashAccess] = (
613 np.sum(bws[MemArea.OffChipFlash]) / arch.memory_bandwidths_per_cycle[MemArea.OffChipFlash]
614 )
615
616 cycles[PassCycles.Total] = np.max(cycles[: PassCycles.Total])
617 return cycles
618
619
620def collate_stats_for_cascaded_pass(arch, bws, macs, cycles):
621 return bws, macs, cycles
622
623
624def performance_for_cascaded_pass(arch, cps):
625 total_bws = make_bandwidth_array()
626 total_macs = make_macs_array()
627 total_cycles = make_cycles_array()
628
629 for ps in cps.passes:
630 bws, macs, cycles, blocks, _, _ = performance_metrics_for_pass(arch, ps)
631 ps.bandwidths = bws
632 ps.macs = macs
633 ps.cycles = cycles
634 ps.n_blocks = blocks
635 total_bws += bws
636 total_macs += macs
637 total_cycles += cycles
638
639 bws, macs, cycles = collate_stats_for_cascaded_pass(arch, total_bws, total_macs, total_cycles)
640 cps.bandwidths = bws
641 cps.macs = macs
642 cps.cycles = cycles
643 return bws, macs, cycles
644
645
646def calc_performance_for_network(nng, arch):
647 total_bws = make_bandwidth_array()
648 total_macs = np.zeros(MacCount.Size)
649 total_cycles = np.zeros(PassCycles.Size)
650
651 for sg in nng.subgraphs:
652 for cps in sg.cascaded_passes:
653 bws, macs, cycles = performance_for_cascaded_pass(arch, cps)
654 total_bws += bws
655 total_macs += macs
656 total_cycles += cycles
Tim Hall79d07d22020-04-27 18:20:16 +0100657
658 nng.bandwidths = total_bws
659 nng.macs = total_macs
660 nng.cycles = total_cycles