blob: d28df97dde109c612061d5ba97b31c2cca701212 [file] [log] [blame]
Tim Hall79d07d22020-04-27 18:20:16 +01001# Copyright (C) 2020 Arm Limited or its affiliates. All rights reserved.
2#
3# SPDX-License-Identifier: Apache-2.0
4#
5# Licensed under the Apache License, Version 2.0 (the License); you may
6# not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9# www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an AS IS BASIS, WITHOUT
13# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
Tim Hall79d07d22020-04-27 18:20:16 +010016# Description:
17# NPU performance estimation functions to estimate performance of a Pass and CascadedPass. Uses a model that takes the
18# maximum of the 'cycles required for bandwidth' and 'cycles required for computing'.
19#
20# Called during scheduling to evaluate different proposals, as well as post-scheduling to provide a final performance
21# estimate.
Diqing Zhonge168b962020-11-05 17:18:47 +010022from enum import auto
23from enum import IntEnum
Diego Russoea6111a2020-04-14 18:41:58 +010024
Tim Hall79d07d22020-04-27 18:20:16 +010025import numpy as np
Diego Russoea6111a2020-04-14 18:41:58 +010026
27from . import numeric_util
Diqing Zhong09387e22020-09-28 18:46:22 +020028from .architecture_features import Accelerator
Diego Russoe8a10452020-04-21 17:39:10 +010029from .architecture_features import Block
Diqing Zhonge8887a32020-09-24 09:53:48 +020030from .data_type import DataType
Diego Russoe8a10452020-04-21 17:39:10 +010031from .nn_graph import PassPlacement
32from .nn_graph import SchedulerRewrite
Diego Russoea6111a2020-04-14 18:41:58 +010033from .operation import NpuBlockType
Diqing Zhonge8887a32020-09-24 09:53:48 +020034from .operation import Op
Diqing Zhong09387e22020-09-28 18:46:22 +020035from .shared_buffer_allocation import is_acc_40bits_used
Diego Russoe8a10452020-04-21 17:39:10 +010036from .tensor import MemArea
37from .tensor import shape_num_elements
Diqing Zhongef0c7fe2020-11-24 14:38:20 +010038from .tensor import Tensor
Diego Russoe8a10452020-04-21 17:39:10 +010039from .tensor import TensorBlockTraversal
Diqing Zhonge168b962020-11-05 17:18:47 +010040from .tensor import TensorFormat
Diego Russoe8a10452020-04-21 17:39:10 +010041from .tensor import TensorPurpose
Tim Hall79d07d22020-04-27 18:20:16 +010042
43
44def rolling_buffer_dims_from_passes(arch, ps1, block_config_ps1, ps2, block_config_ps2):
Tim Hall79d07d22020-04-27 18:20:16 +010045 ofm_block = Block(block_config_ps2[-3], block_config_ps2[-4], block_config_ps2[-1])
Tim Hall4ed38bc2020-10-20 18:54:20 +010046 kernel = ps2.primary_op.kernel
Tim Hall79d07d22020-04-27 18:20:16 +010047
48 if ps2.npu_block_type in set((NpuBlockType.ConvolutionMxN, NpuBlockType.VectorProduct)):
Louis Verhaard93dc5532020-06-07 12:40:18 +020049 op = ps2.primary_op
Louis Verhaardaee5d752020-09-30 09:01:52 +020050 ifm_block_depth = arch.calc_ifm_block_depth(op.ifm.shape[-1], op.ifm.dtype.size_in_bits())
Tim Hall79d07d22020-04-27 18:20:16 +010051 else:
52 ifm_block_depth = block_config_ps2[-1]
53
Louis Verhaard93dc5532020-06-07 12:40:18 +020054 ifm_block = arch.get_ifm_block_size(ifm_block_depth, ofm_block, kernel, arch.ofm_block_max)
Tim Hall79d07d22020-04-27 18:20:16 +010055
56 # The performed height calculation is for worst case
57 height = numeric_util.round_up(ifm_block.height + block_config_ps1[0], block_config_ps1[0])
58 width = ifm_block.width
Louis Verhaard93dc5532020-06-07 12:40:18 +020059 return [height, width]
Tim Hall79d07d22020-04-27 18:20:16 +010060
61
Diqing Zhonge168b962020-11-05 17:18:47 +010062class PassCycles(IntEnum):
Diqing Zhong42e833d2020-10-02 13:18:42 +020063 Npu = 0
Diqing Zhonge168b962020-11-05 17:18:47 +010064 SramAccess = auto()
65 DramAccess = auto()
66 OnChipFlashAccess = auto()
67 OffChipFlashAccess = auto()
68 Total = auto()
69 Size = auto()
Tim Hall79d07d22020-04-27 18:20:16 +010070
71 def display_name(self):
Tim Hall1bd531d2020-11-01 20:59:36 +000072 return ("NPU", "SRAM Access", "DRAM Access", "On-chip Flash Access", "Off-chip Flash Access", "Total", "Size",)[
73 self.value
74 ]
Tim Hall79d07d22020-04-27 18:20:16 +010075
76 def identifier_name(self):
Tim Hall1bd531d2020-11-01 20:59:36 +000077 return ("npu", "sram_access", "dram_access", "on_chip_flash_access", "off_chip_flash_access", "total", "size",)[
78 self.value
79 ]
Tim Hall79d07d22020-04-27 18:20:16 +010080
81 @staticmethod
82 def all():
83 return (
Diqing Zhong42e833d2020-10-02 13:18:42 +020084 PassCycles.Npu,
Tim Hall79d07d22020-04-27 18:20:16 +010085 PassCycles.SramAccess,
86 PassCycles.DramAccess,
87 PassCycles.OnChipFlashAccess,
88 PassCycles.OffChipFlashAccess,
89 PassCycles.Total,
90 )
91
92
Diqing Zhonge168b962020-11-05 17:18:47 +010093class MacCount(IntEnum):
Tim Hall79d07d22020-04-27 18:20:16 +010094 NeuralNetworkMacs = 0
Diqing Zhonge168b962020-11-05 17:18:47 +010095 HardwareMacs = auto()
96 Size = auto()
Tim Hall79d07d22020-04-27 18:20:16 +010097
98 def display_name(self):
99 return ("Neural Network Macs", "Hardware Macs", "Size")[self.value]
100
101 def identifier_name(self):
102 return ("nn_macs", "hardware_macs", "size")[self.value]
103
104 @staticmethod
105 def all():
106 return (MacCount.NeuralNetworkMacs, MacCount.HardwareMacs)
107
108
Diqing Zhonge168b962020-11-05 17:18:47 +0100109class BandwidthDirection(IntEnum):
Tim Hall79d07d22020-04-27 18:20:16 +0100110 Read = 0
Diqing Zhonge168b962020-11-05 17:18:47 +0100111 Write = auto()
112 Size = auto()
Tim Hall79d07d22020-04-27 18:20:16 +0100113
114 def display_name(self):
115 return self.name
116
117 def identifier_name(self):
118 return self.name.lower()
119
120 @staticmethod
121 def all():
122 return (BandwidthDirection.Read, BandwidthDirection.Write)
123
124
125def make_bandwidth_array():
126 return np.zeros((MemArea.Size, TensorPurpose.Size, BandwidthDirection.Size))
127
128
129def make_macs_array():
130 return np.zeros(MacCount.Size, np.int)
131
132
133def make_cycles_array():
134 return np.zeros(PassCycles.Size)
135
136
137def make_metrics_arrays():
138 return (make_bandwidth_array(), make_macs_array(), make_cycles_array())
139
140
141def get_n_blocks_and_area(
142 ifm_brick_size, ifm_height_width, orig_skirt, clamped_skirt, block_config, min_block_size, strides
143):
144
145 ifm_block_config = (block_config[0] * strides[1], block_config[1] * strides[2])
146
147 n_normal_blocks = []
148 remainder_size = []
149 for i in range(2):
150 non_skirt_dim = ifm_height_width[i] - orig_skirt[i] - orig_skirt[2 + i]
151 n_blocks = non_skirt_dim // ifm_block_config[i]
152 n_normal_blocks.append(n_blocks)
153 remainder_dim = numeric_util.round_up(
154 ((non_skirt_dim - n_blocks * ifm_block_config[i] - 1) // strides[i + 1]) + 1, min_block_size[i]
155 )
156 remainder_size.append(remainder_dim)
157
158 # this will actually calculate reads into the edge padding.
159
160 # there are four cases in total, handling the edges that will not fill a complete block.
161
162 # 0000000001
163 # 0000000001
164 # 0000000001
165 # 0000000001
166 # 0000000001
167 # 0000000001
168 # 2222222223
169 total_blocks = 0
170 total_area = 0
171
172 block_setup = (
173 (n_normal_blocks[0] * n_normal_blocks[1], block_config),
174 (1 * n_normal_blocks[1], (remainder_size[0], block_config[1])),
175 (n_normal_blocks[0] * 1, (block_config[0], remainder_size[1])),
176 (1 * 1, remainder_size),
177 )
178
179 for n_blocks, block_size in block_setup:
180 if block_size[0] == 0 or block_size[1] == 0:
181 continue
182 read_dims = [0, 0]
183 for i in range(2):
184 read_dims[i] = (
185 numeric_util.round_up(clamped_skirt[i], ifm_brick_size[i + 1])
186 + block_size[i] * strides[i + 1]
187 + numeric_util.round_up(clamped_skirt[2 + i], ifm_brick_size[i + 1])
188 )
189 assert n_blocks >= 0
190 total_blocks += n_blocks
191 total_area += n_blocks * read_dims[0] * read_dims[1]
192 assert total_blocks >= 1
193 return total_blocks, total_area, block_setup
194
195
Diqing Zhong42e833d2020-10-02 13:18:42 +0200196def get_ifm_block_depth(npu_block_type, ifm_depth, ifm_elemwidth, block_traversal, ofm_blk_depth):
197 ifm_blk_depth = ofm_blk_depth
198
199 if npu_block_type == NpuBlockType.ConvolutionMxN or npu_block_type == NpuBlockType.ReduceSum:
200 if ifm_elemwidth == 16 or block_traversal == TensorBlockTraversal.PartKernelFirst:
201 ifm_blk_depth = 16
202 elif ifm_elemwidth == 8:
203 ifm_blk_depth = 32
204 else:
205 ifm_blk_depth = 8
206
207 return min(ifm_depth, ifm_blk_depth)
208
209
Diqing Zhongef0c7fe2020-11-24 14:38:20 +0100210def get_minimal_cmd_cycles(arch, ifm_tensor, ofm_tensor, ifm_blk: Block, ofm_blk: Block, output_cycles, dpu_cycles=0):
211 latencies_rd = {MemArea.Sram: 32, MemArea.Dram: 500, MemArea.OnChipFlash: 64, MemArea.OffChipFlash: 64}
212 latencies_wr = {MemArea.Sram: 32, MemArea.Dram: 250, MemArea.OnChipFlash: 64, MemArea.OffChipFlash: 64}
213 ifm_tens_blk = Tensor((1, ifm_blk.height, ifm_blk.width, ifm_blk.depth), ifm_tensor.dtype, "ifm_blk")
214 ofm_tens_blk = Tensor((1, ofm_blk.height, ofm_blk.width, ofm_blk.depth), ofm_tensor.dtype, "ofm_blk")
215 cycles_ifm_blk = (
216 estimate_memory_bandwidth(arch, ifm_tensor.mem_area, BandwidthDirection.Read, ifm_tens_blk, ifm_blk)
217 / arch.memory_bandwidths_per_cycle[ifm_tensor.mem_area]
218 )
219 cycles_ofm_blk = (
220 estimate_memory_bandwidth(arch, ofm_tensor.mem_area, BandwidthDirection.Write, ofm_tens_blk, ofm_blk)
221 / arch.memory_bandwidths_per_cycle[ofm_tensor.mem_area]
222 )
223 return (
224 latencies_rd[ifm_tensor.mem_area]
225 + cycles_ifm_blk
226 + dpu_cycles
227 + output_cycles
228 + latencies_wr[ofm_tensor.mem_area]
229 + cycles_ofm_blk
230 ) / 4
231
232
Diqing Zhong42e833d2020-10-02 13:18:42 +0200233def estimate_output_cycles(
Diqing Zhongef0c7fe2020-11-24 14:38:20 +0100234 arch,
235 npu_block_type,
236 primary_op,
237 num_elems,
238 ifm_tensor,
239 ofm_tensor,
240 use_acc_40bits=False,
241 ifm2_tensor=None,
242 block_config: Block = None,
Diqing Zhong09387e22020-09-28 18:46:22 +0200243):
Louis Verhaarde8a5a782020-11-02 18:04:27 +0100244 faf = None if primary_op.activation is None else primary_op.activation.op_type
Diqing Zhong09387e22020-09-28 18:46:22 +0200245 if npu_block_type == NpuBlockType.ElementWise and ifm_tensor.dtype == DataType.int32:
246 if ifm2_tensor is None:
Diqing Zhonge8887a32020-09-24 09:53:48 +0200247 # Unary op
248 output_perf_index = 0
249 else:
250 # Binary op
251 output_perf_index = 1
Diqing Zhong09387e22020-09-28 18:46:22 +0200252 elif primary_op.type == Op.Mul and ofm_tensor.dtype == DataType.int32:
Diqing Zhonge8887a32020-09-24 09:53:48 +0200253 output_perf_index = 2
Diqing Zhong09387e22020-09-28 18:46:22 +0200254 elif primary_op.type == Op.Mul or (
Diqing Zhonge8887a32020-09-24 09:53:48 +0200255 npu_block_type
256 in (
257 NpuBlockType.ConvolutionMxN,
258 NpuBlockType.ConvolutionDepthWise,
259 NpuBlockType.Pooling,
260 NpuBlockType.ReduceSum,
261 NpuBlockType.VectorProduct,
262 )
Diqing Zhong09387e22020-09-28 18:46:22 +0200263 and use_acc_40bits
Diqing Zhonge8887a32020-09-24 09:53:48 +0200264 ):
265 output_perf_index = 3
Diqing Zhong09387e22020-09-28 18:46:22 +0200266 elif primary_op.type in (Op.Add, Op.Sub):
267 input_scale = ifm_tensor.quantization.scale_f32
268 input2_scale = ifm2_tensor.quantization.scale_f32
269 output_scale = ofm_tensor.quantization.scale_f32
Diqing Zhonge8887a32020-09-24 09:53:48 +0200270
271 if "resizebilinear" in primary_op.attrs:
272 output_scale = input2_scale
273
274 if None in (input_scale, input2_scale, output_scale) or input_scale == input2_scale:
275 # Simple Add/Sub
276 output_perf_index = 4
277 else:
278 # Advanced Add/Sub
279 output_perf_index = 5
Diqing Zhong09387e22020-09-28 18:46:22 +0200280 elif primary_op.type.is_maxpool_op():
Diqing Zhonge8887a32020-09-24 09:53:48 +0200281 output_perf_index = 6
282 else:
283 output_perf_index = 7
284
285 if faf in (Op.Sigmoid, Op.Tanh, Op.LUT):
286 activation_perf_index = 0
287 elif faf in (Op.Relu, Op.Relu6, Op.ReluN1To1):
288 activation_perf_index = 1
289 else:
290 activation_perf_index = 2
291
Diqing Zhonge8887a32020-09-24 09:53:48 +0200292 cycle_per_elem = max(
293 arch.output_cycles_per_elem[output_perf_index], arch.activation_cycles_per_elem[activation_perf_index]
294 )
Diqing Zhong986e3192020-11-16 16:15:56 +0100295
Diqing Zhongef0c7fe2020-11-24 14:38:20 +0100296 if primary_op.type.is_elementwise_op() and block_config is not None:
297 num_elems_blk = block_config.width * block_config.height * block_config.depth
298 cycle_cmd = get_minimal_cmd_cycles(
299 arch, ifm_tensor, ofm_tensor, block_config, block_config, num_elems_blk * cycle_per_elem
300 )
301 cycle_per_elem = max(cycle_per_elem, cycle_cmd / num_elems_blk)
302
Diqing Zhonge8887a32020-09-24 09:53:48 +0200303 return num_elems * cycle_per_elem
304
305
Diqing Zhong42e833d2020-10-02 13:18:42 +0200306def estimate_conv_pooling_cycles(
Diqing Zhong986e3192020-11-16 16:15:56 +0100307 arch,
308 npu_block_type,
309 primary_op,
Diqing Zhongef0c7fe2020-11-24 14:38:20 +0100310 ifm_block: Block,
311 ofm_block: Block,
Diqing Zhong986e3192020-11-16 16:15:56 +0100312 block_traversal,
313 kernel_dims,
314 ifm_tensor,
315 ofm_tensor,
316 scale_tensor=None,
Diqing Zhong09387e22020-09-28 18:46:22 +0200317):
Diqing Zhonge5204a62020-10-13 11:42:37 +0200318 ofm_ublock = Block(arch.config.ofm_ublock.width, arch.config.ofm_ublock.height, arch.config.ofm_ublock.depth)
319 ifm_tens_shape = numeric_util.full_shape(4, ifm_tensor.shape, 1)
320 ofm_tens_shape = numeric_util.full_shape(4, ofm_tensor.shape, 1)
321
322 if (
323 arch.config.ofm_ublock.height == 2
324 and npu_block_type
325 in (NpuBlockType.ConvolutionMxN, NpuBlockType.ConvolutionDepthWise, NpuBlockType.VectorProduct)
326 and ofm_tens_shape[1] == 1
327 # Optimisation only applies for even width tensors
328 and ofm_tens_shape[2] % 2 == 0
329 and kernel_dims[0] == 1
330 ):
331 ofm_ublock.width = 4
332 ofm_ublock.height = 1
Diqing Zhongef0c7fe2020-11-24 14:38:20 +0100333 ofm_block.height = 1
Diqing Zhonge5204a62020-10-13 11:42:37 +0200334
Diqing Zhongef0c7fe2020-11-24 14:38:20 +0100335 num_ublk_x = numeric_util.round_up_divide(ofm_block.width, ofm_ublock.width)
336 num_ublk_y = ofm_block.height // ofm_ublock.height
337 num_ublk_xy = num_ublk_x * num_ublk_y
338 num_ublk_z = ofm_block.depth // ofm_ublock.depth
Diqing Zhong09387e22020-09-28 18:46:22 +0200339 num_ofm_blk = 0
340 total_cycles = 0
Diqing Zhongef0c7fe2020-11-24 14:38:20 +0100341 num_elems_blk = ofm_block.width * ofm_block.height * ofm_block.depth
Diqing Zhong09387e22020-09-28 18:46:22 +0200342 use_acc_40bits = is_acc_40bits_used(npu_block_type, ifm_tensor, ofm_tensor)
343
344 sub_kernel_limits = arch.sub_kernel_limits[npu_block_type]
345 n_sub_kernels_y = numeric_util.round_up_divide(kernel_dims[0], sub_kernel_limits[0])
346 n_sub_kernels_x = numeric_util.round_up_divide(kernel_dims[1], sub_kernel_limits[1])
347 sub_kernel_x = [
348 min((kernel_dims[1] - i * sub_kernel_limits[1]), sub_kernel_limits[1]) for i in range(n_sub_kernels_x)
349 ]
350 sub_kernel_y = [
351 min((kernel_dims[0] - i * sub_kernel_limits[0]), sub_kernel_limits[0]) for i in range(n_sub_kernels_y)
352 ]
353 sub_kernel_size = (x * y for y in sub_kernel_y for x in sub_kernel_x)
354
Diqing Zhong09387e22020-09-28 18:46:22 +0200355 cycles_dpu_blk = 0
Diqing Zhong986e3192020-11-16 16:15:56 +0100356 cycles_wb = 32 * ofm_ublock.depth // 8
Diqing Zhong09387e22020-09-28 18:46:22 +0200357
358 for num_kernel_elems in sub_kernel_size:
359 if npu_block_type == NpuBlockType.Pooling:
Diqing Zhongef0c7fe2020-11-24 14:38:20 +0100360 num_kernel_steps = 1
Diqing Zhong986e3192020-11-16 16:15:56 +0100361 cycles = max(4, num_kernel_elems) * num_ublk_xy * num_ublk_z
Diqing Zhong09387e22020-09-28 18:46:22 +0200362 if ifm_tensor.dtype.size_in_bits() == 16 and arch.accelerator_config != Accelerator.Ethos_U55_32:
363 cycles *= 2
364 elif npu_block_type == NpuBlockType.ConvolutionDepthWise:
Diqing Zhong986e3192020-11-16 16:15:56 +0100365 cycles = 4 * num_ublk_xy
Diqing Zhong09387e22020-09-28 18:46:22 +0200366 if ifm_tensor.dtype.size_in_bits() == 16:
367 cycles *= 2
Diqing Zhongef0c7fe2020-11-24 14:38:20 +0100368 num_kernel_steps = numeric_util.round_up_divide(num_kernel_elems, 4)
369 cycles = max(cycles_wb, cycles) * num_kernel_steps * num_ublk_z
Diqing Zhong09387e22020-09-28 18:46:22 +0200370 elif (
371 (npu_block_type == NpuBlockType.ConvolutionMxN and block_traversal != TensorBlockTraversal.PartKernelFirst)
372 or npu_block_type == NpuBlockType.VectorProduct
373 or npu_block_type == NpuBlockType.ReduceSum
374 ):
Diqing Zhongef0c7fe2020-11-24 14:38:20 +0100375 num_kernel_steps = num_kernel_elems
376 cycles = max(cycles_wb, 4 * num_ublk_xy) * num_kernel_steps * num_ublk_z
Diqing Zhong09387e22020-09-28 18:46:22 +0200377 else:
378 assert block_traversal == TensorBlockTraversal.PartKernelFirst
379 divider = 2 if ifm_tensor.dtype.size_in_bits() == 16 else 4
Diqing Zhongef0c7fe2020-11-24 14:38:20 +0100380 num_kernel_steps = numeric_util.round_up_divide(num_kernel_elems, divider)
Diqing Zhong986e3192020-11-16 16:15:56 +0100381 cycles = max(cycles_wb, 4 * num_ublk_xy) * (
Diqing Zhongef0c7fe2020-11-24 14:38:20 +0100382 num_kernel_steps * numeric_util.round_up_divide(ifm_block.depth, 8) * num_ublk_z
Diqing Zhong09387e22020-09-28 18:46:22 +0200383 )
Diqing Zhongef0c7fe2020-11-24 14:38:20 +0100384
385 delay_cycles = 0
386 if arch.accelerator_config is Accelerator.Ethos_U55_32:
387 delay = 7 if use_acc_40bits else 3
388 if num_ublk_x == 1 and num_ublk_y == 1:
389 if num_ublk_z == 1:
390 delay_cycles = delay * num_kernel_steps
391 elif num_kernel_steps > 1:
392 delay_cycles = delay * (num_kernel_steps - 1) * num_ublk_z
393 if (num_ublk_x == 1 or num_ublk_y == 1) and num_ublk_z > 1 and use_acc_40bits:
394 delay_cycles += delay * num_ublk_z
395 else:
396 delay = (
397 3
398 if use_acc_40bits and arch.accelerator_config in (Accelerator.Ethos_U55_64, Accelerator.Ethos_U55_128)
399 else 2
400 )
401 if num_ublk_x == 1 and num_ublk_y == 1:
402 if num_ublk_z == 1:
403 delay_cycles = delay * num_kernel_steps
404 elif num_kernel_steps > 1:
405 delay_cycles = delay * (num_kernel_steps - 1) * num_ublk_z
406
407 if npu_block_type == NpuBlockType.ConvolutionMxN and block_traversal == TensorBlockTraversal.PartKernelFirst:
408 delay_cycles *= numeric_util.round_up_divide(ifm_block.depth, 8)
409
Diqing Zhong09387e22020-09-28 18:46:22 +0200410 cycles_dpu_blk += cycles
Diqing Zhongef0c7fe2020-11-24 14:38:20 +0100411 cycles_dpu_blk += delay_cycles
412
413 if npu_block_type in (NpuBlockType.ConvolutionMxN, NpuBlockType.VectorProduct, NpuBlockType.ReduceSum):
414 cycles_dpu_blk *= numeric_util.round_up_divide(ifm_tens_shape[3], ifm_block.depth)
Diqing Zhong09387e22020-09-28 18:46:22 +0200415
416 cycles_dpu_blk /= arch.ncores
417
418 num_ofm_blk = (
Diqing Zhongef0c7fe2020-11-24 14:38:20 +0100419 numeric_util.round_up_divide(ofm_tens_shape[1], ofm_block.height)
420 * numeric_util.round_up_divide(ofm_tens_shape[2], ofm_block.width)
421 * numeric_util.round_up_divide(ofm_tens_shape[3], ofm_block.depth)
Diqing Zhong09387e22020-09-28 18:46:22 +0200422 )
423
Diqing Zhong42e833d2020-10-02 13:18:42 +0200424 cycles_output_blk = estimate_output_cycles(
Diqing Zhongef0c7fe2020-11-24 14:38:20 +0100425 arch, npu_block_type, primary_op, num_elems_blk, ifm_tensor, ofm_tensor, use_acc_40bits
Diqing Zhong09387e22020-09-28 18:46:22 +0200426 )
427
Diqing Zhong986e3192020-11-16 16:15:56 +0100428 if scale_tensor:
429 if scale_tensor.mem_area is MemArea.Sram:
430 latency = 32
431 elif scale_tensor.mem_area is MemArea.Dram:
432 latency = 500
433 else:
434 latency = 64
Diqing Zhongef0c7fe2020-11-24 14:38:20 +0100435 cycles_bias_blk = 10 * min(ofm_block.depth, ofm_tens_shape[3]) * latency / 256
Diqing Zhong986e3192020-11-16 16:15:56 +0100436 cycles_output_blk = max(cycles_output_blk, cycles_bias_blk)
437
Diqing Zhongef0c7fe2020-11-24 14:38:20 +0100438 cycles_cmd = get_minimal_cmd_cycles(
439 arch, ifm_tensor, ofm_tensor, ifm_block, ofm_block, cycles_dpu_blk, cycles_output_blk
440 )
441 cycles_dpu_blk = max(cycles_dpu_blk, cycles_cmd)
442 cycles_output_blk = max(cycles_output_blk, cycles_cmd)
443
Diqing Zhong09387e22020-09-28 18:46:22 +0200444 if cycles_dpu_blk > cycles_output_blk:
445 total_cycles = cycles_dpu_blk * num_ofm_blk + cycles_output_blk
446 else:
447 total_cycles = cycles_output_blk * num_ofm_blk + cycles_dpu_blk
448
449 return total_cycles
450
451
Diqing Zhonge168b962020-11-05 17:18:47 +0100452def estimate_memory_bandwidth(arch, mem_area, direction, tensor, block_size: Block, replace_bw=None):
453 if tensor.format not in (TensorFormat.NHWC, TensorFormat.NHCWB16):
454 return tensor.bandwidth() if replace_bw is None else replace_bw
455
456 # Estimate memory transfer efficiency by calculating the burst length
457 # this is related to data format, block shape, and tensor shape, etc.
458 max_burst_len = 32 if mem_area == MemArea.Sram else 128
459 burst_len = 0
460 elem_size = tensor.dtype.size_in_bytes()
461 is_ifm = direction == BandwidthDirection.Read
462 tens = tensor.clone()
463 if not tens.avoid_NHCWB16:
464 tens.set_format(TensorFormat.NHCWB16, arch)
465
466 if tens.format == TensorFormat.NHCWB16:
467 if tens.get_strides()[1] == block_size.depth:
468 burst_len = elem_size * block_size.depth * block_size.width
469 elif is_ifm:
470 burst_len = 16 * elem_size * block_size.width
471 else:
472 burst_len = 16 * elem_size * block_size.width * arch.ncores
473 else:
474 assert tens.format == TensorFormat.NHWC
475 if is_ifm:
476 if tens.get_strides()[3] == block_size.depth:
477 burst_len = elem_size * block_size.depth * block_size.width
478 else:
479 burst_len = elem_size * block_size.depth
480 else:
481 if block_size.depth <= 16 and tens.get_strides()[3] == block_size.depth:
482 burst_len = elem_size * block_size.depth * block_size.width
483 else:
484 burst_len = min(64, 16 * elem_size * arch.ncores, block_size.depth * elem_size)
485
486 burst_len = min(max_burst_len, burst_len)
487 bw = tens.bandwidth() if replace_bw is None else replace_bw
488
489 return bw * (max_burst_len / burst_len)
490
491
Michael McGeagh6f725262020-12-03 15:21:36 +0000492def performance_metrics_for_pass(arch, ps, block_config=None, rewrite_list=None, force_outputs_to_fast_storage=False):
Tim Hall79d07d22020-04-27 18:20:16 +0100493 if block_config is None:
494 block_config = ps.block_config
495 bws = make_bandwidth_array()
496 macs = make_macs_array()
497 cycles = make_cycles_array()
498 blocks = 0
499 ifm_read_multiple = 1
500 weight_read_multiple = 0
501
502 if ps.placement in set((PassPlacement.MemoryOnly, PassPlacement.StartupInit)):
503 return bws, macs, cycles, blocks, ifm_read_multiple, weight_read_multiple # nothing real happening in this pass
504
505 min_block_size = arch.min_block_sizes[ps.npu_block_type]
506
507 skirt = (0, 0, 0, 0)
508 explicit_padding = (0, 0, 0, 0)
509 primary_op = ps.primary_op
510 replacement_read_bws = {}
Diqing Zhonge168b962020-11-05 17:18:47 +0100511 ofm_block = Block(block_config[1], block_config[0], block_config[3])
512 ifm_block = Block(block_config[1], block_config[0], block_config[3])
513
Tim Hall1bd531d2020-11-01 20:59:36 +0000514 if ps.placement == PassPlacement.Npu and primary_op:
Tim Hall79d07d22020-04-27 18:20:16 +0100515 skirt = primary_op.attrs.get("skirt", skirt)
516 explicit_padding = primary_op.attrs.get("explicit_padding", explicit_padding)
Louis Verhaardaee5d752020-09-30 09:01:52 +0200517 assert primary_op.type.npu_block_type == ps.npu_block_type
518 npu_block_type = primary_op.type.npu_block_type
Tim Hall79d07d22020-04-27 18:20:16 +0100519
520 ifm_tensor, _, weight_tensor, ofm_tensor = ps.get_primary_op_ifm_ifm2_weights_ofm()
Diqing Zhonge168b962020-11-05 17:18:47 +0100521 ifm_tensor_shape = numeric_util.full_shape(4, ifm_tensor.shape, 1)
Tim Hall79d07d22020-04-27 18:20:16 +0100522
Diqing Zhongef0c7fe2020-11-24 14:38:20 +0100523 if npu_block_type == NpuBlockType.ReduceSum:
524 block_traversal = TensorBlockTraversal.DepthFirst
525 elif npu_block_type in (
526 NpuBlockType.ConvolutionMxN,
527 NpuBlockType.ConvolutionDepthWise,
528 NpuBlockType.VectorProduct,
529 ):
530 block_traversal = weight_tensor.block_traversal
531 else:
532 block_traversal = TensorBlockTraversal.Default
533 ifm_block_depth = get_ifm_block_depth(
534 npu_block_type, ifm_tensor_shape[3], ifm_tensor.dtype.size_in_bits(), block_traversal, ofm_block.depth
535 )
536 ifm_block = arch.get_ifm_block_size(
537 ifm_block_depth, ofm_block, primary_op.kernel, ifm_resampling_mode=ifm_tensor.resampling_mode
538 )
539
Tim Hallc30f4952020-06-15 20:47:35 +0100540 if npu_block_type in set(
Diqing Zhong09387e22020-09-28 18:46:22 +0200541 (
542 NpuBlockType.ConvolutionMxN,
543 NpuBlockType.ConvolutionDepthWise,
544 NpuBlockType.Pooling,
545 NpuBlockType.ReduceSum,
546 )
Tim Hallc30f4952020-06-15 20:47:35 +0100547 ):
Charles Xu3e9c4342020-04-22 08:31:43 +0200548 # extent the ifm to full dimension
549 ifm_tensor_brick_size = tuple(numeric_util.full_shape(4, list(ifm_tensor.brick_size), 1))
Charles Xu3e9c4342020-04-22 08:31:43 +0200550 ifm_tensor_bandwidth_shape = numeric_util.full_shape(4, ifm_tensor.bandwidth_shape, 1)
Tim Hall79d07d22020-04-27 18:20:16 +0100551
Diqing Zhong42e833d2020-10-02 13:18:42 +0200552 batch_size = ifm_tensor_shape[0]
Charles Xu3e9c4342020-04-22 08:31:43 +0200553 ifm_depth = ifm_tensor_bandwidth_shape[3]
Tim Hall79d07d22020-04-27 18:20:16 +0100554
555 # add in padding
556 ifm_tensor_shape[1] += explicit_padding[0] + explicit_padding[2] # height += top and bottom
557 ifm_tensor_shape[2] += explicit_padding[1] + explicit_padding[3] # width += left and right
558
559 strides = primary_op.attrs["strides"]
560 if npu_block_type != NpuBlockType.Pooling:
Diqing Zhong09387e22020-09-28 18:46:22 +0200561 if npu_block_type == NpuBlockType.ReduceSum:
Diqing Zhong09387e22020-09-28 18:46:22 +0200562 weight_tensor_shape = [1, 1, ifm_tensor.shape[3], ofm_tensor.shape[3]]
563 weight_tensor_bandwidth_shape = [0] * 4
564 weight_tensor_element_size = 0
565 weight_tensor_bandwidth_compression_scale = 0.0
566 else:
Diqing Zhong09387e22020-09-28 18:46:22 +0200567 weight_tensor_shape = weight_tensor.shape
568 weight_tensor_bandwidth_shape = weight_tensor.bandwidth_shape
569 weight_tensor_element_size = weight_tensor.element_size()
570 weight_tensor_bandwidth_compression_scale = weight_tensor.bandwidth_compression_scale
Tim Hall79d07d22020-04-27 18:20:16 +0100571 nn_ops = (
572 int(ofm_tensor.shape[0])
573 * int(ofm_tensor.shape[1])
574 * int(ofm_tensor.shape[2])
575 * int(weight_tensor_shape[0])
576 * int(weight_tensor_shape[1])
577 * int(weight_tensor_shape[2])
578 * int(weight_tensor_shape[3])
Tim Hall79d07d22020-04-27 18:20:16 +0100579 )
580 else:
581 weight_tensor_shape = [
582 primary_op.attrs["ksize"][1],
583 primary_op.attrs["ksize"][2],
584 1,
585 ifm_tensor_shape[3],
586 ]
587 weight_tensor_bandwidth_shape = weight_tensor_shape
588 weight_tensor_element_size = 0
589 weight_tensor_bandwidth_compression_scale = 0.0
590 nn_ops = 0 # pooling doesn't count as NN ops
591
592 kernel_dims = weight_tensor_shape[:2]
593
594 sub_kernel_limits = arch.sub_kernel_limits[npu_block_type]
595 # count the sub kernels; the IFM block needs to be refetched for each of them
596 n_sub_kernels_y = numeric_util.round_up_divide(kernel_dims[0], sub_kernel_limits[0])
597 n_sub_kernels_x = numeric_util.round_up_divide(kernel_dims[1], sub_kernel_limits[1])
598 n_sub_kernels = n_sub_kernels_y * n_sub_kernels_x
599
600 clamped_skirt = list(skirt)
601 clamped_skirt[2] = min(clamped_skirt[2], sub_kernel_limits[0] - 1 - clamped_skirt[0])
602 clamped_skirt[3] = min(clamped_skirt[3], sub_kernel_limits[1] - 1 - clamped_skirt[1])
603 n_blocks, area, block_setup = get_n_blocks_and_area(
Charles Xu3e9c4342020-04-22 08:31:43 +0200604 ifm_tensor_brick_size,
Tim Hall79d07d22020-04-27 18:20:16 +0100605 ifm_tensor_shape[1:3],
606 skirt,
607 clamped_skirt,
608 block_config,
609 min_block_size,
610 strides,
611 )
612
Diqing Zhonge168b962020-11-05 17:18:47 +0100613 blocks = n_blocks * numeric_util.round_up_divide(weight_tensor_shape[3], ofm_block.depth)
Tim Hall79d07d22020-04-27 18:20:16 +0100614
Diqing Zhonge168b962020-11-05 17:18:47 +0100615 n_weight_stages = numeric_util.round_up_divide(weight_tensor_bandwidth_shape[3], ofm_block.depth)
Tim Hall79d07d22020-04-27 18:20:16 +0100616 if npu_block_type == NpuBlockType.ConvolutionDepthWise or npu_block_type == NpuBlockType.Pooling:
617 n_weight_stages = 1 # force to no reread
618
619 ifm_tensor_bw = (
620 n_sub_kernels
621 * batch_size
622 * area
623 * ifm_depth
624 * n_weight_stages
625 * ifm_tensor.element_size()
626 * ifm_tensor.bandwidth_compression_scale
627 )
628 replacement_read_bws[ifm_tensor] = ifm_tensor_bw
629 ifm_read_multiple = n_weight_stages
630
631 replacement_read_bws[weight_tensor] = (
632 batch_size
633 * shape_num_elements(weight_tensor_bandwidth_shape)
634 * weight_tensor_element_size
635 * weight_tensor_bandwidth_compression_scale
636 * n_blocks
637 ) # read once per block and batch
638 weight_read_multiple = n_blocks
639
640 n_kernel_xy = kernel_dims[0] * kernel_dims[1]
641 n_input_channels_at_a_time = block_config[2]
642
Diqing Zhong09387e22020-09-28 18:46:22 +0200643 if npu_block_type == NpuBlockType.Pooling or block_traversal in set(
Tim Hall79d07d22020-04-27 18:20:16 +0100644 (TensorBlockTraversal.PartKernelFirst, TensorBlockTraversal.DepthWise)
645 ):
646 n_input_channels_at_a_time = numeric_util.round_up_divide(n_input_channels_at_a_time, 4)
647 n_kernel_xy = max(
648 n_kernel_xy, 4
649 ) # need at least 4, as this is the minimum duty cycle for secondary accumulator writes
650 if weight_tensor is not None:
Diego Russoea6111a2020-04-14 18:41:58 +0100651 n_kernel_xy = numeric_util.round_up(n_kernel_xy, 4) # weights need to be read in blocks of 4
Tim Hall79d07d22020-04-27 18:20:16 +0100652
653 num_mac_ops = 0
654 for n_blocks_for_size, block_size in block_setup:
655 num_mac_ops += (
656 batch_size
657 * n_blocks_for_size
658 * block_size[0]
659 * block_size[1]
660 * numeric_util.round_up(weight_tensor_shape[2], n_input_channels_at_a_time)
Diqing Zhonge168b962020-11-05 17:18:47 +0100661 * numeric_util.round_up(weight_tensor_shape[3], ofm_block.depth)
Tim Hall79d07d22020-04-27 18:20:16 +0100662 * n_kernel_xy
663 )
Tim Hall79d07d22020-04-27 18:20:16 +0100664 macs[MacCount.NeuralNetworkMacs] += nn_ops
665 macs[MacCount.HardwareMacs] += num_mac_ops
Diqing Zhong42e833d2020-10-02 13:18:42 +0200666 cycles[PassCycles.Npu] = estimate_conv_pooling_cycles(
Diqing Zhong986e3192020-11-16 16:15:56 +0100667 arch,
668 npu_block_type,
669 primary_op,
Diqing Zhongef0c7fe2020-11-24 14:38:20 +0100670 ifm_block,
Diqing Zhong986e3192020-11-16 16:15:56 +0100671 ofm_block,
672 block_traversal,
673 kernel_dims,
674 ifm_tensor,
675 ofm_tensor,
676 ps.scale_tensor,
Diqing Zhong09387e22020-09-28 18:46:22 +0200677 )
Tim Hall79d07d22020-04-27 18:20:16 +0100678 elif npu_block_type == NpuBlockType.VectorProduct:
679 nn_macs = (
680 ifm_tensor.shape[0]
681 * numeric_util.round_up(weight_tensor.shape[-2], block_config[2])
682 * numeric_util.round_up(weight_tensor.shape[-1], block_config[3])
683 )
684 num_mac_ops = nn_macs
685
Diqing Zhong42e833d2020-10-02 13:18:42 +0200686 cycles[PassCycles.Npu] = estimate_conv_pooling_cycles(
Diqing Zhongef0c7fe2020-11-24 14:38:20 +0100687 arch, npu_block_type, primary_op, ifm_block, ofm_block, block_traversal, [1, 1], ifm_tensor, ofm_tensor,
Diqing Zhong09387e22020-09-28 18:46:22 +0200688 )
Tim Hall79d07d22020-04-27 18:20:16 +0100689 macs[MacCount.NeuralNetworkMacs] += nn_macs
690 macs[MacCount.HardwareMacs] += num_mac_ops
691
Diqing Zhonge168b962020-11-05 17:18:47 +0100692 blocks = 1 * numeric_util.round_up_divide(weight_tensor.shape[-1], ofm_block.depth)
Tim Hall79d07d22020-04-27 18:20:16 +0100693
694 non_zero_fraction = 1.0
695 if ifm_tensor.values is not None:
696 nz_vector = np.amax(ifm_tensor.values != 0, axis=0) # max across batch axis
697 non_zero_fraction = np.average(nz_vector)
698
699 replacement_read_bws[ifm_tensor] = ifm_tensor.bandwidth()
700 replacement_read_bws[weight_tensor] = weight_tensor.bandwidth() * non_zero_fraction
701 ifm_read_multiple = 1
702 weight_read_multiple = non_zero_fraction
Diqing Zhonge8887a32020-09-24 09:53:48 +0200703 elif npu_block_type == NpuBlockType.ElementWise:
Tim Hall79d07d22020-04-27 18:20:16 +0100704 # Work out how many elements we have and calculate performance.
Diqing Zhong42e833d2020-10-02 13:18:42 +0200705 cycles[PassCycles.Npu] = estimate_output_cycles(
Diqing Zhongef0c7fe2020-11-24 14:38:20 +0100706 arch,
707 npu_block_type,
708 primary_op,
709 ofm_tensor.elements(),
710 ps.ifm_tensor,
711 ps.ofm_tensor,
712 None,
713 ps.ifm2_tensor,
714 ofm_block,
Diqing Zhong09387e22020-09-28 18:46:22 +0200715 )
Diqing Zhong42e833d2020-10-02 13:18:42 +0200716
717 prev_npu_pass = next((npu_ps for npu_ps in ps.dag_predecessors if npu_ps.placement is PassPlacement.Npu), None)
718 if prev_npu_pass is None:
719 # cycles for DMA ops in first pass
720 dma_ops = (op for op in ps.ops if op.type == Op.DMA)
721 for dma_op in dma_ops:
722 mem_area = dma_op.attrs["source"]
723 for tens in dma_op.inputs:
724 cycles[PassCycles.Npu] += tens.storage_size() / arch.memory_bandwidths_per_cycle[mem_area]
725
Michael McGeagh6f725262020-12-03 15:21:36 +0000726 if rewrite_list is not None:
727 # apply the desired rewrites
728 for rewrite_op, tens, _, _, _, ps_to_rewrite in rewrite_list:
729 if ps != ps_to_rewrite:
730 continue
731 if rewrite_op == SchedulerRewrite.Nop:
732 pass # these are fine, no bandwidth changes
733 elif rewrite_op in (SchedulerRewrite.ChangeTensorSubPurpose,):
734 if tens.purpose == TensorPurpose.FeatureMap:
735 bw = estimate_memory_bandwidth(
736 arch,
737 arch.fast_storage_mem_area,
738 BandwidthDirection.Read,
739 tens,
740 ifm_block,
741 replacement_read_bws[tens],
742 )
743 else:
744 bw = replacement_read_bws[tens]
745 bws[arch.fast_storage_mem_area][tens.purpose][BandwidthDirection.Read] += bw
746 replacement_read_bws[tens] = 0
Tim Hall79d07d22020-04-27 18:20:16 +0100747
748 for tens in ps.outputs:
749 if force_outputs_to_fast_storage:
Diqing Zhonge168b962020-11-05 17:18:47 +0100750 bws[arch.fast_storage_mem_area][tens.purpose][BandwidthDirection.Write] += estimate_memory_bandwidth(
751 arch, arch.fast_storage_mem_area, BandwidthDirection.Write, tens, ofm_block
752 )
Tim Hall79d07d22020-04-27 18:20:16 +0100753 else:
Diqing Zhonge168b962020-11-05 17:18:47 +0100754 bws[tens.mem_area][tens.purpose][BandwidthDirection.Write] += estimate_memory_bandwidth(
755 arch, tens.mem_area, BandwidthDirection.Write, tens, ofm_block
756 )
Tim Hall79d07d22020-04-27 18:20:16 +0100757
758 for tens in ps.intermediates:
759 bws[tens.mem_area][tens.purpose][BandwidthDirection.Write] += tens.bandwidth()
760
761 if tens in replacement_read_bws:
762 bw = replacement_read_bws[tens]
763 else:
764 bw = tens.bandwidth()
765
766 bws[tens.mem_area][tens.purpose][BandwidthDirection.Read] += bw
767
768 for tens in ps.inputs:
Diqing Zhonge168b962020-11-05 17:18:47 +0100769 bws[tens.mem_area][tens.purpose][BandwidthDirection.Read] += estimate_memory_bandwidth(
770 arch, tens.mem_area, BandwidthDirection.Read, tens, ifm_block, replacement_read_bws.get(tens)
771 )
Tim Hall79d07d22020-04-27 18:20:16 +0100772
773 # quick build access counts for only current pass, even though these aren't the final numbers
Diqing Zhonge168b962020-11-05 17:18:47 +0100774 update_summary_cycles(arch, bws, cycles)
Tim Hall79d07d22020-04-27 18:20:16 +0100775
776 return bws, macs, cycles, blocks, ifm_read_multiple, weight_read_multiple
777
778
Diqing Zhonge168b962020-11-05 17:18:47 +0100779def update_summary_cycles(arch, bws, cycles):
780 cycles[PassCycles.SramAccess] = np.sum(bws[MemArea.Sram]) / arch.memory_bandwidths_per_cycle[MemArea.Sram]
Tim Hall79d07d22020-04-27 18:20:16 +0100781 cycles[PassCycles.DramAccess] = np.sum(bws[MemArea.Dram]) / arch.memory_bandwidths_per_cycle[MemArea.Dram]
782 cycles[PassCycles.OnChipFlashAccess] = (
783 np.sum(bws[MemArea.OnChipFlash]) / arch.memory_bandwidths_per_cycle[MemArea.OnChipFlash]
784 )
785 cycles[PassCycles.OffChipFlashAccess] = (
786 np.sum(bws[MemArea.OffChipFlash]) / arch.memory_bandwidths_per_cycle[MemArea.OffChipFlash]
787 )
788
789 cycles[PassCycles.Total] = np.max(cycles[: PassCycles.Total])
790 return cycles
791
792
793def collate_stats_for_cascaded_pass(arch, bws, macs, cycles):
794 return bws, macs, cycles
795
796
797def performance_for_cascaded_pass(arch, cps):
798 total_bws = make_bandwidth_array()
799 total_macs = make_macs_array()
800 total_cycles = make_cycles_array()
801
802 for ps in cps.passes:
803 bws, macs, cycles, blocks, _, _ = performance_metrics_for_pass(arch, ps)
804 ps.bandwidths = bws
805 ps.macs = macs
806 ps.cycles = cycles
807 ps.n_blocks = blocks
808 total_bws += bws
809 total_macs += macs
810 total_cycles += cycles
811
812 bws, macs, cycles = collate_stats_for_cascaded_pass(arch, total_bws, total_macs, total_cycles)
813 cps.bandwidths = bws
814 cps.macs = macs
815 cps.cycles = cycles
816 return bws, macs, cycles
817
818
819def calc_performance_for_network(nng, arch):
820 total_bws = make_bandwidth_array()
821 total_macs = np.zeros(MacCount.Size)
822 total_cycles = np.zeros(PassCycles.Size)
823
824 for sg in nng.subgraphs:
825 for cps in sg.cascaded_passes:
826 bws, macs, cycles = performance_for_cascaded_pass(arch, cps)
827 total_bws += bws
828 total_macs += macs
829 total_cycles += cycles
Tim Hall79d07d22020-04-27 18:20:16 +0100830
831 nng.bandwidths = total_bws
832 nng.macs = total_macs
833 nng.cycles = total_cycles