blob: 7ce237ca475fba55d2e8c43d539189da1755f81a [file] [log] [blame]
erik.andersson@arm.com460c6892021-02-24 14:38:09 +01001# Copyright (C) 2020-2021 Arm Limited or its affiliates. All rights reserved.
Tim Hall79d07d22020-04-27 18:20:16 +01002#
3# SPDX-License-Identifier: Apache-2.0
4#
5# Licensed under the Apache License, Version 2.0 (the License); you may
6# not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9# www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an AS IS BASIS, WITHOUT
13# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
Tim Hall79d07d22020-04-27 18:20:16 +010016# Description:
17# Compresses and pads the weigths. It also calculates the scales and packs with the biases.
Tim Hall79d07d22020-04-27 18:20:16 +010018import math
Tim Hall79d07d22020-04-27 18:20:16 +010019from collections import namedtuple
Louis Verhaardaeae5672020-11-02 18:04:27 +010020from typing import Tuple
Diego Russoea6111a2020-04-14 18:41:58 +010021
22import numpy as np
Tim Hall79d07d22020-04-27 18:20:16 +010023
Louis Verhaarde8a5a782020-11-02 18:04:27 +010024from .api import NpuBlockTraversal
Manupa Karunaratned83d2e12020-07-20 12:05:32 +010025from .architecture_features import Accelerator
26from .architecture_features import ArchitectureFeatures
Diego Russoe8a10452020-04-21 17:39:10 +010027from .data_type import DataType
Louis Verhaard7db78962020-05-25 15:05:26 +020028from .errors import UnsupportedFeatureError
Diego Russoe8a10452020-04-21 17:39:10 +010029from .nn_graph import SchedulingStrategy
30from .numeric_util import round_up
Patrik Gustavssond89c09e2020-07-08 11:27:12 +020031from .numeric_util import round_up_divide
Diego Russoe8a10452020-04-21 17:39:10 +010032from .operation import NpuBlockType
Louis Verhaardaee5d752020-09-30 09:01:52 +020033from .operation import Op
Diego Russoe8a10452020-04-21 17:39:10 +010034from .scaling import quantise_scale
35from .scaling import reduced_quantise_scale
Louis Verhaard9db529a2020-09-23 10:27:11 +020036from .tensor import create_equivalence_id
Diego Russoe8a10452020-04-21 17:39:10 +010037from .tensor import TensorBlockTraversal
38from .tensor import TensorFormat
39from .tensor import TensorPurpose
40from .tensor import TensorSubPurpose
Jacob Bohline843d332020-06-23 12:12:56 +020041from ethosu import mlw_codec
Diego Russoe8a10452020-04-21 17:39:10 +010042
Tim Hall79d07d22020-04-27 18:20:16 +010043
Louis Verhaard3c07c972020-05-07 08:12:58 +020044# Contains meta info for a weight compression. If two tensors have identical weight compression config,
45# then they also will have identical compressed weights.
46WeightCompressionConfig = namedtuple(
Louis Verhaard9db529a2020-09-23 10:27:11 +020047 "WeightCompressionConfig", ["npu_block_type", "ofm_block_depth", "ofm_depth_step", "dilation", "value_id"]
Louis Verhaard3c07c972020-05-07 08:12:58 +020048)
49
50
Manupa Karunaratned83d2e12020-07-20 12:05:32 +010051def encode_weights(
52 accelerator: Accelerator,
53 weights_volume: np.ndarray,
Louis Verhaardaeae5672020-11-02 18:04:27 +010054 dilation_xy: Tuple[int, int],
Manupa Karunaratned83d2e12020-07-20 12:05:32 +010055 ifm_bitdepth: int,
56 ofm_block_depth: int,
57 is_depthwise: bool,
Louis Verhaarde8a5a782020-11-02 18:04:27 +010058 block_traversal: NpuBlockTraversal,
Manupa Karunaratned83d2e12020-07-20 12:05:32 +010059):
60 """
Louis Verhaardaeae5672020-11-02 18:04:27 +010061 Internal implementation of the public facing API to use weight encoding.
Manupa Karunaratned83d2e12020-07-20 12:05:32 +010062
Tim Hallc8a73862020-10-27 12:43:14 +000063 :param accelerator: architecture_features.Accelerator enum to pick the correct Ethos-U accelerator
Manupa Karunaratned83d2e12020-07-20 12:05:32 +010064 :param weights_volume: numpy.ndarray in OHWI layout with a shape of four
65 :param dilation_xy: a two element tuple of dilation attributes in x,y dimension
66 :param ifm_bitdepth: the bitdepth of input feature map
Tim Hallc8a73862020-10-27 12:43:14 +000067 :param ofm_block_depth: the depth of blocks for Ethos-U processing
Manupa Karunaratned83d2e12020-07-20 12:05:32 +010068 :param is_depthwise: a boolean indicating these weights are used for a depthwise traversal
Louis Verhaardaeae5672020-11-02 18:04:27 +010069 :param block_traversal: indicates how these weights are traversed on sub-kernel basis
70
Fredrik Svedbergf5c07c42021-04-23 14:36:42 +020071 :return: a tuple with a bytearray of encoded weights and the size of the unencoded weights
Manupa Karunaratned83d2e12020-07-20 12:05:32 +010072 """
Manupa Karunaratne8b24f2b2020-08-12 18:26:39 +000073 # Check arg types
74 assert isinstance(accelerator, Accelerator)
75 assert isinstance(weights_volume, np.ndarray)
76 assert isinstance(dilation_xy, tuple)
77 assert isinstance(ifm_bitdepth, int)
78 assert isinstance(ofm_block_depth, int)
79 assert isinstance(is_depthwise, bool)
Louis Verhaarde8a5a782020-11-02 18:04:27 +010080 assert isinstance(block_traversal, NpuBlockTraversal)
Manupa Karunaratne8b24f2b2020-08-12 18:26:39 +000081
Manupa Karunaratned83d2e12020-07-20 12:05:32 +010082 # Checks for weight layout
83 assert len(weights_volume.shape) == 4, "weights ndarray should have a shape of 4"
84
85 # It cannot be both partkernel and depthwise
Louis Verhaarde8a5a782020-11-02 18:04:27 +010086 assert not (
87 is_depthwise and block_traversal == NpuBlockTraversal.PART_KERNEL_FIRST
88 ), "encode_weights :: partkernel and depthwise are mutually exclusive"
Manupa Karunaratned83d2e12020-07-20 12:05:32 +010089
90 # Check valid values for dilation
91 assert dilation_xy[0] in (1, 2), "encode_weights :: dilation x should be 1 or 2 not {}".format(dilation_xy[0])
92 assert dilation_xy[1] in (1, 2), "encode_weights :: dilation y should be 1 or 2 not {}".format(dilation_xy[1])
93
94 ifm_ublock = ArchitectureFeatures.accelerator_configs[accelerator].ifm_ublock
95 ofm_ublock = ArchitectureFeatures.accelerator_configs[accelerator].ofm_ublock
96 raw_stream = generate_brick(
97 ifm_ublock=ifm_ublock,
98 ofm_ublock=ofm_ublock,
99 brick_weights=weights_volume,
100 ofm_block_depth=ofm_block_depth,
101 is_depthwise=is_depthwise,
Louis Verhaarde8a5a782020-11-02 18:04:27 +0100102 is_partkernel=block_traversal == NpuBlockTraversal.PART_KERNEL_FIRST,
Manupa Karunaratned83d2e12020-07-20 12:05:32 +0100103 ifm_bitdepth=ifm_bitdepth,
104 dilation=dilation_xy,
105 )
106 encoded_stream = encode(raw_stream)
Fredrik Svedbergf5c07c42021-04-23 14:36:42 +0200107 return encoded_stream, len(raw_stream)
Manupa Karunaratned83d2e12020-07-20 12:05:32 +0100108
109
Manupa Karunaratnebef228b2020-07-29 18:06:28 +0100110def encode_bias(bias: np.int64, scale: int, shift: int):
111 """
Louis Verhaardaeae5672020-11-02 18:04:27 +0100112 Internal implementation of public facing API to pack bias and scale values as required by the Ethos-U
Tim Hallc8a73862020-10-27 12:43:14 +0000113
Manupa Karunaratnebef228b2020-07-29 18:06:28 +0100114 :param bias: 64bit signed number that includes 40bit signed bias
115 :param scale: 32bit scale value
116 :param shift: 6bit shift value
117 :return: packed 80bit [0(2-bits),shift(6-bits),scale(32-bits),bias(40-bits)]
118 """
Manupa Karunaratne8b24f2b2020-08-12 18:26:39 +0000119 # Check arg types
120 assert isinstance(bias, np.int64)
121 assert isinstance(scale, int)
122 assert isinstance(shift, int)
123
Manupa Karunaratnebef228b2020-07-29 18:06:28 +0100124 assert -(1 << (40 - 1)) <= bias < (1 << (40 - 1)) # signed 40-bit range
125 assert 0 <= scale < (1 << 32) # unsigned 32-bit range
126 assert 0 <= shift < (1 << 6) # unsigned 6-bit range
127
128 data = bytearray(10)
129 data[0] = (bias >> (0 * 8)) & 0xFF
130 data[1] = (bias >> (1 * 8)) & 0xFF
131 data[2] = (bias >> (2 * 8)) & 0xFF
132 data[3] = (bias >> (3 * 8)) & 0xFF
133 data[4] = (bias >> (4 * 8)) & 0xFF
134 data[5] = (scale >> (0 * 8)) & 0xFF
135 data[6] = (scale >> (1 * 8)) & 0xFF
136 data[7] = (scale >> (2 * 8)) & 0xFF
137 data[8] = (scale >> (3 * 8)) & 0xFF
138 data[9] = shift & 0x3F
139 return data
140
141
Louis Verhaardb2fb2122020-06-04 15:51:24 +0200142def create_weight_compression_config(tens, npu_block_type, ofm_block_depth, ofm_depth_step, dilation):
Louis Verhaard3c07c972020-05-07 08:12:58 +0200143 # Note: for an ofm block only its depth is used in weight compression.
144 # And block depth > ofm depth gives same result as block depth == ofm depth
145 block_depth = min(ofm_block_depth, tens.quant_values.shape[-1])
Louis Verhaard9db529a2020-09-23 10:27:11 +0200146 return WeightCompressionConfig(npu_block_type, block_depth, ofm_depth_step, dilation, tens.value_id)
Louis Verhaard3c07c972020-05-07 08:12:58 +0200147
148
149def set_storage_shape(tens):
150 # Sets the storage shape depending on the tensor's sub purpose
151 if tens.sub_purpose == TensorSubPurpose.DoubleBuffer and len(tens.compressed_values) > 2:
152 offset = 2 * np.amax([len(x) for x in tens.compressed_values])
153 assert offset % 16 == 0
154 else:
155 offset = tens.weight_compressed_offsets[-1]
156 tens.storage_shape = [1, 1, 1, offset]
157
158
159class CompressedWeightCache:
160 # Contains weight compressions for all weight tensors in a graph
161 def __init__(self):
162 self.cache = {} # maps from WeightCompressionConfig to a tensor clone containing compressed weights
163
Fredrik Svedbergf5c07c42021-04-23 14:36:42 +0200164 def has_tensor_with_same_compression(self, wcc):
165 return self.cache.get(wcc) is not None
Louis Verhaard3c07c972020-05-07 08:12:58 +0200166
Fredrik Svedbergf5c07c42021-04-23 14:36:42 +0200167 def get_tensor_with_same_compression(self, wcc):
168 cache_obj = self.cache.get(wcc)
169 return cache_obj[0] if cache_obj else None
170
171 def get_unencoded_size_with_same_compression(self, wcc):
172 cache_obj = self.cache.get(wcc)
173 return cache_obj[1] if cache_obj else None
174
175 def add(self, tens, unencoded_size):
Louis Verhaard3c07c972020-05-07 08:12:58 +0200176 # Adds the compressed weights from the tensor to the cache
177 wcc = tens.weight_compression_config
178 # Clone the tensor to make sure that nothing related to the weight compression is modified
179 tens_clone = tens.clone("_weights{}_{}".format(wcc.ofm_block_depth, wcc.ofm_depth_step))
Fredrik Svedbergf5c07c42021-04-23 14:36:42 +0200180 self.cache[wcc] = (tens_clone, unencoded_size)
Louis Verhaard3c07c972020-05-07 08:12:58 +0200181
182
Tim Hall79d07d22020-04-27 18:20:16 +0100183def encode(weight_stream):
Patrik Gustavsson5ff99442020-07-10 10:12:17 +0200184 if len(weight_stream) == 0:
185 return []
Tim Hall79d07d22020-04-27 18:20:16 +0100186 assert np.amin(weight_stream) >= -255
187 assert np.amax(weight_stream) <= 255
188
189 # Encode flattened signed weight stream
190 compressed = mlw_codec.encode(weight_stream)
191
192 # pad with 0xFF as needed so the length of the weight stream
193 # is a multiple of 16
Diego Russoea6111a2020-04-14 18:41:58 +0100194
Tim Hall79d07d22020-04-27 18:20:16 +0100195 while (len(compressed) % 16) != 0:
196 compressed.append(0xFF)
197
198 return compressed
199
200
Manupa Karunaratned83d2e12020-07-20 12:05:32 +0100201def generate_brick(
202 ifm_ublock, ofm_ublock, brick_weights, ofm_block_depth, is_depthwise, is_partkernel, ifm_bitdepth, dilation
203):
204
205 decomp_h = ArchitectureFeatures.SubKernelMax.height // dilation[0]
206 decomp_w = ArchitectureFeatures.SubKernelMax.width // dilation[1]
Tim Hallf7e810a2020-06-25 15:04:31 +0100207 # Expect weights formatted OHWI
208 ofm_depth = brick_weights.shape[-4]
209 ifm_depth = brick_weights.shape[-1]
210 kernel_width = brick_weights.shape[-2]
211 kernel_height = brick_weights.shape[-3]
Tim Hall79d07d22020-04-27 18:20:16 +0100212 # IFM block depth
213 if is_partkernel or (ifm_bitdepth == 16):
214 # IFM block depth is always 16 for part-kernel-first
215 ifm_block_depth = 16
216 elif ifm_bitdepth == 8:
217 ifm_block_depth = 32
218 else:
219 assert False
220
221 stream = []
222
223 # Top level striping - OFM blocks in the entire brick's depth
Louis Verhaard3c07c972020-05-07 08:12:58 +0200224 for ofm_block_z in range(0, ofm_depth, ofm_block_depth):
225 clipped_ofm_block_depth = min(ofm_block_depth, ofm_depth - ofm_block_z)
Tim Hall79d07d22020-04-27 18:20:16 +0100226 # IFM blocks required for the brick
227 for ifm_block_z in range(0, (1 if is_depthwise else ifm_depth), ifm_block_depth):
228 if is_depthwise:
229 clipped_ifm_block_depth = ifm_ublock.depth
230 else:
231 clipped_ifm_block_depth = (
232 min(ifm_block_depth, ifm_depth - ifm_block_z) if is_partkernel else ifm_block_depth
233 )
234 # Weight decomposition
235 # Subkernel Splitting (H)
Louis Verhaardb2fb2122020-06-04 15:51:24 +0200236 for subkernel_y in range(0, kernel_height, decomp_h):
237 sub_height = min(kernel_height - subkernel_y, decomp_h)
Tim Hall79d07d22020-04-27 18:20:16 +0100238 # Subkernel splitting (W)
Louis Verhaardb2fb2122020-06-04 15:51:24 +0200239 for subkernel_x in range(0, kernel_width, decomp_w):
240 sub_width = min(kernel_width - subkernel_x, decomp_w)
Tim Hall79d07d22020-04-27 18:20:16 +0100241 subkernel_elements = sub_width * sub_height
242 # Part kernel first works across the kernel H/W and needs padding
243 if is_partkernel:
244 if ifm_bitdepth == 16 and subkernel_elements % 2 != 0:
245 subkernel_elements = int(math.ceil(subkernel_elements / 2) * 2)
246 elif ifm_bitdepth == 8 and subkernel_elements % 4 != 0:
247 subkernel_elements = int(math.ceil(subkernel_elements / 4) * 4)
248
249 # Depthwise Conv requires multiple of 4 kernel elements in its weight block
250 # this is different from normal conv which is considered "weights depth-first"
251 elif is_depthwise:
252 subkernel_elements = int(math.ceil(subkernel_elements / 4.0) * 4)
253
254 ifm_block_depth_outer = clipped_ifm_block_depth if is_partkernel else 1
255 ifm_block_depth_inner = 1 if is_partkernel else clipped_ifm_block_depth
256 # IFM Ublocks in IFM-block over depth for part-kernel-first mode
257 # For depth-first IFM Ublocks are traversed after subkernel elements so this loop is ignored.
258 for ifm_ublk_outer in range(0, ifm_block_depth_outer, ifm_ublock.depth):
259 # OFM Ublocks in OFM-block over depth
260 for ofm_ublk in range(0, clipped_ofm_block_depth, ofm_ublock.depth):
261 # HW Kernel element traversal - cannot be a H/W loop due to element
262 # padding requirement on depthwise/part-kernel configurations
263 for element in range(subkernel_elements):
264 kx = element % sub_width
265 ky = element // sub_width
266 # IFM Ublocks in IFM-block over depth (only 1 ublock if depthwise)
267 # In case of part-kernel-first IFM Ublock traversal have already been handled
268 # and this loop is ignored.
269 for ifm_ublk_inner in range(0, ifm_block_depth_inner, ifm_ublock.depth):
270 # Feed OFM ublock elements
271 for ofm_ublock_z in range(ofm_ublock.depth):
272 # Source IFM ublock elements (only 1 element deep if depthwise)
273 for ifm_ublock_z in range(1 if is_depthwise else ifm_ublock.depth):
274 # Source position within the current subkernel
275 wx = subkernel_x + kx
276 wy = subkernel_y + ky
277 # Source IFM/OFM slices
278 ifm_ublk = ifm_ublk_inner + ifm_ublk_outer
279 ifm_z = ifm_block_z + ifm_ublk + ifm_ublock_z
280 ofm_z = ofm_block_z + ofm_ublk + ofm_ublock_z
281 if (ifm_z >= ifm_depth) or (ofm_z >= ofm_depth) or (ky >= sub_height):
282 stream.append(0)
283 else:
Tim Hallf7e810a2020-06-25 15:04:31 +0100284 stream.append(brick_weights[ofm_z][wy][wx][ifm_z])
Tim Hall79d07d22020-04-27 18:20:16 +0100285 return stream
286
Jacob Bohline843d332020-06-23 12:12:56 +0200287
Tim Hallf7e810a2020-06-25 15:04:31 +0100288def core_deinterleave(hwio, core, ncores):
289 # Put weights back into OHWI
Jacob Bohline843d332020-06-23 12:12:56 +0200290 ohwi = np.transpose(hwio, (3, 0, 1, 2))
291 return ohwi[core : ohwi.shape[0] : ncores]
292
Tim Hall79d07d22020-04-27 18:20:16 +0100293
294# Compress the weights
Louis Verhaardb2fb2122020-06-04 15:51:24 +0200295def compress_weights(arch, nng, tens, npu_block_type, ofm_block_depth, ofm_depth_step, dilation):
Tim Hall79d07d22020-04-27 18:20:16 +0100296 assert tens.purpose == TensorPurpose.Weights
Tim Hall79d07d22020-04-27 18:20:16 +0100297
Louis Verhaard3c07c972020-05-07 08:12:58 +0200298 # Check the weight cache
299 if nng.weight_cache is None:
300 nng.weight_cache = CompressedWeightCache()
Louis Verhaardb2fb2122020-06-04 15:51:24 +0200301 wcc = create_weight_compression_config(tens, npu_block_type, ofm_block_depth, ofm_depth_step, dilation)
Louis Verhaard3c07c972020-05-07 08:12:58 +0200302 tens.weight_compression_config = wcc
Louis Verhaard9db529a2020-09-23 10:27:11 +0200303 # Reassign equivalence id such that tensors with same weight compression get identical equivalence ids,
304 # but tensors with the same values but different compression get different equivalence ids
305 tens.equivalence_id = create_equivalence_id(wcc)
Louis Verhaard3c07c972020-05-07 08:12:58 +0200306 tens_cached = nng.weight_cache.get_tensor_with_same_compression(wcc)
307 if tens_cached is not None:
308 # Cache hit, copy weights from the cache
309 tens.copy_compressed_weight_info(tens_cached)
310 set_storage_shape(tens)
Fredrik Svedbergf5c07c42021-04-23 14:36:42 +0200311 return nng.weight_cache.get_unencoded_size_with_same_compression(wcc)
Louis Verhaard3c07c972020-05-07 08:12:58 +0200312 # No cache hit, perform the compression
Tim Hall79d07d22020-04-27 18:20:16 +0100313 assert tens.quantization is not None
314 assert tens.quantization.scale_f32 is not None
315 assert tens.quantization.zero_point is not None
316
317 zero_point = tens.quantization.zero_point
318 quant_buf = tens.quant_values.astype(np.int64)
319
320 # Early zero-point correction
321 weights = quant_buf - zero_point
322
323 if len(weights.shape) == 2:
324 weights = np.expand_dims(np.expand_dims(weights, axis=0), axis=0)
Tim Hall79d07d22020-04-27 18:20:16 +0100325
326 compression_scales = []
327 compressed_offsets = []
328 encoded_streams = []
Tim Hallf7e810a2020-06-25 15:04:31 +0100329 encoded_streams_substream_offsets = []
Tim Hall79d07d22020-04-27 18:20:16 +0100330 offset = 0
Tim Hallf7e810a2020-06-25 15:04:31 +0100331 max_single_buffer_len = 0
Fredrik Svedbergf5c07c42021-04-23 14:36:42 +0200332 unencoded_size = 0
Tim Hall79d07d22020-04-27 18:20:16 +0100333
334 ifm_bitdepth = tens.consumer_list[0].inputs[0].dtype.size_in_bits()
335 ifm_depth = weights.shape[-2]
336 if npu_block_type == NpuBlockType.ConvolutionDepthWise:
337 tens.block_traversal = TensorBlockTraversal.DepthWise
338 if npu_block_type == NpuBlockType.ConvolutionMxN:
339 # Determine which block traversal strategy has better DPU utilization
Jacob Bohlinde2a57f2020-08-10 15:21:42 +0200340 kernel_size = weights.shape[0] * weights.shape[1]
341 depth_utilization = weights.shape[2] / round_up(weights.shape[2], 32 if ifm_bitdepth == 8 else 16)
342 part_kernel_utilization = (weights.shape[2] / round_up(weights.shape[2], 8)) * (
Tim Hall79d07d22020-04-27 18:20:16 +0100343 kernel_size / round_up(kernel_size, 4 if ifm_bitdepth == 8 else 2)
344 )
345 if part_kernel_utilization >= depth_utilization or ifm_depth <= 8:
346 # Part-kernel first is always better for ifm depths <= 8
347 tens.block_traversal = TensorBlockTraversal.PartKernelFirst
348 else:
349 tens.block_traversal = TensorBlockTraversal.DepthFirst
350
Manupa Karunaratned83d2e12020-07-20 12:05:32 +0100351 is_depthwise = tens.block_traversal == TensorBlockTraversal.DepthWise
Louis Verhaarde8a5a782020-11-02 18:04:27 +0100352 if tens.block_traversal == TensorBlockTraversal.PartKernelFirst:
353 block_traversal = NpuBlockTraversal.PART_KERNEL_FIRST
354 else:
355 block_traversal = NpuBlockTraversal.DEPTH_FIRST
Manupa Karunaratned83d2e12020-07-20 12:05:32 +0100356
Louis Verhaardaee5d752020-09-30 09:01:52 +0200357 if tens.consumer_list[0].type == Op.Conv2DBackpropInputSwitchedBias:
Jacob Bohlincf7da102020-05-20 09:03:40 +0200358 # Transpose Convoluion, reverse weights in H and W axes
Tim Hallc30f4952020-06-15 20:47:35 +0100359 weights = np.flip(weights, axis=(0, 1))
Jacob Bohlincf7da102020-05-20 09:03:40 +0200360
Jacob Bohline843d332020-06-23 12:12:56 +0200361 # Calculate brick size
Jacob Bohlinde2a57f2020-08-10 15:21:42 +0200362 brick_size = (weights.shape[0], weights.shape[1], weights.shape[2], min(tens.shape[-1], ofm_depth_step))
Jacob Bohline843d332020-06-23 12:12:56 +0200363 elements_in_brick = np.prod(brick_size)
364
Tim Hall79d07d22020-04-27 18:20:16 +0100365 # Slice weight stream up depth-ways into bricks and compress
366 full_ofm_depth = quant_buf.shape[-1]
367 for idx in range(0, full_ofm_depth, ofm_depth_step):
368 # Get the weights necessary for this brick
369 count = min(full_ofm_depth - idx, ofm_depth_step)
370 brick_weights = weights[:, :, :, idx : idx + count]
371
Tim Hallf7e810a2020-06-25 15:04:31 +0100372 substream_offsets = [0]
373 encoded_stream = []
Tim Hallf7e810a2020-06-25 15:04:31 +0100374
375 # For each core, deinterleave weights from the larger volume
376 # and generate separate compressed streams.
377 for core in range(0, min(arch.ncores, full_ofm_depth)):
378 core_weights = core_deinterleave(brick_weights, core, arch.ncores)
Tim Hall62316762020-06-25 16:55:02 +0100379
380 block_depth = (ofm_block_depth + arch.ncores - 1 - core) // arch.ncores
Manupa Karunaratned83d2e12020-07-20 12:05:32 +0100381 encoded_substream = []
Tim Hall62316762020-06-25 16:55:02 +0100382 if block_depth != 0:
Fredrik Svedbergf5c07c42021-04-23 14:36:42 +0200383 encoded_substream, raw_stream_size = encode_weights(
Manupa Karunaratned83d2e12020-07-20 12:05:32 +0100384 accelerator=arch.accelerator_config,
385 weights_volume=core_weights,
386 dilation_xy=dilation,
387 ifm_bitdepth=ifm_bitdepth,
388 ofm_block_depth=block_depth,
389 is_depthwise=is_depthwise,
Louis Verhaarde8a5a782020-11-02 18:04:27 +0100390 block_traversal=block_traversal,
Jacob Bohline843d332020-06-23 12:12:56 +0200391 )
Fredrik Svedbergf5c07c42021-04-23 14:36:42 +0200392 unencoded_size += raw_stream_size
Jacob Bohline843d332020-06-23 12:12:56 +0200393 encoded_stream.extend(encoded_substream)
394 substream_offsets.append(len(encoded_stream))
Tim Hallf7e810a2020-06-25 15:04:31 +0100395
Jacob Bohline843d332020-06-23 12:12:56 +0200396 encoded_streams.append(encoded_stream)
397 encoded_streams_substream_offsets.append(substream_offsets)
Tim Hallf7e810a2020-06-25 15:04:31 +0100398
399 # Remember maximum encoded length for DoubleBuffering
400 max_single_buffer_len = max(max_single_buffer_len, len(encoded_stream))
Tim Hall79d07d22020-04-27 18:20:16 +0100401
Tim Hall79d07d22020-04-27 18:20:16 +0100402 # Remember where we put it for linear addressing
403 compressed_offsets.append(offset)
Tim Hallf7e810a2020-06-25 15:04:31 +0100404 offset += len(encoded_stream)
Tim Hall79d07d22020-04-27 18:20:16 +0100405 assert offset % 16 == 0
406
407 # Compression scale tracking
Jacob Bohline843d332020-06-23 12:12:56 +0200408 compression_scales.append(len(encoded_stream) / elements_in_brick)
Tim Hall79d07d22020-04-27 18:20:16 +0100409
Tim Hallf7e810a2020-06-25 15:04:31 +0100410 # Track total length as last element of the offsets array
Tim Hall79d07d22020-04-27 18:20:16 +0100411 compressed_offsets.append(offset)
412
Tim Hall79d07d22020-04-27 18:20:16 +0100413 tens.weight_compression_scales = compression_scales
Tim Hall79d07d22020-04-27 18:20:16 +0100414 tens.weight_compressed_offsets = compressed_offsets
415 tens.compression_scale_for_worst_weight_stream = np.amax(compression_scales)
416 tens.storage_compression_scale = tens.bandwidth_compression_scale = np.average(compression_scales)
417 tens.compressed_values = encoded_streams
Tim Hallf7e810a2020-06-25 15:04:31 +0100418 tens.compressed_values_substream_offsets = encoded_streams_substream_offsets
Jacob Bohline843d332020-06-23 12:12:56 +0200419 tens.brick_size = brick_size
Louis Verhaard3c07c972020-05-07 08:12:58 +0200420 set_storage_shape(tens)
Fredrik Svedbergf5c07c42021-04-23 14:36:42 +0200421 nng.weight_cache.add(tens, unencoded_size)
422 return unencoded_size
Tim Hall79d07d22020-04-27 18:20:16 +0100423
Jacob Bohline843d332020-06-23 12:12:56 +0200424
Tim Hallf7e810a2020-06-25 15:04:31 +0100425def calc_scales_and_pack_biases(tens, arch, ofm_depth_step, rescale_for_faf=False):
Andreas Nevalainen897cc142020-10-28 15:42:08 +0100426 assert tens.purpose in [TensorPurpose.FeatureMap, TensorPurpose.FSBias]
Tim Hall79d07d22020-04-27 18:20:16 +0100427 assert tens.format == TensorFormat.NHWC
428 # the connected operator should expect a bias input unless it is a FullyConnected
Louis Verhaardaee5d752020-09-30 09:01:52 +0200429 assert tens.consumer_list[0].type.needs_bias()
Tim Hall79d07d22020-04-27 18:20:16 +0100430 # the input bias tensor is the same as that connected to the operator
Louis Verhaardaee5d752020-09-30 09:01:52 +0200431 bias_tens = tens.consumer_list[0].bias
Jacob Bohlincf7da102020-05-20 09:03:40 +0200432 assert tens is bias_tens
433
Tim Hall79d07d22020-04-27 18:20:16 +0100434 # the operator should only have a single output
435 assert len(tens.consumer_list[0].outputs) == 1
Tim Hall79d07d22020-04-27 18:20:16 +0100436 biases = tens.quant_values
437
438 first_consumer_op = tens.consumer_list[0]
439 ifm_dtype = first_consumer_op.inputs[0].dtype
Dwight Lidman4f728c02020-12-17 15:14:45 +0100440 ifm_scale = first_consumer_op.get_input_quantization().scale_f32
Louis Verhaard98a34992020-09-01 10:39:04 +0200441 ofm_scale = first_consumer_op.get_output_quantization().scale_f32
Tim Hall79d07d22020-04-27 18:20:16 +0100442 weight_scales = first_consumer_op.inputs[1].quantization.scale_f32
443
444 # biases can have multiple consumers for rnn cells. if so, then check that they are all the same
445 for op in tens.consumer_list[1:]:
Dwight Lidman4f728c02020-12-17 15:14:45 +0100446 assert ifm_scale == op.get_input_quantization().scale_f32
Louis Verhaard98a34992020-09-01 10:39:04 +0200447 assert ofm_scale == op.get_output_quantization().scale_f32
Tim Hall79d07d22020-04-27 18:20:16 +0100448 assert weight_scales == op.inputs[1].quantization.scale_f32
449
450 if not hasattr(weight_scales, "__iter__"):
451 # If weight_scales is not already an iterable make it into a list
452 weight_scales = [weight_scales]
453
454 # Convert scales to np.double (from np.float32) to conform to TensorFlow Lite which
455 # uses double during scaling calculations
456 # TensorFlow Lite casts the scales slightly differently for uint8 and int8
457 if not rescale_for_faf:
458 if ifm_dtype == DataType.uint8:
Dwight Lidman4f728c02020-12-17 15:14:45 +0100459 # for some cases of the Mean operator, the scale must be calculated differently to match reference
460 if first_consumer_op.low_precision_scaling:
461 scales = [
462 np.double(np.single(ifm_scale) / (np.single(weight_scale) * np.single(ofm_scale)))
463 for weight_scale in weight_scales
464 ]
465 else:
466 scales = [np.double(ifm_scale * weight_scale) / np.double(ofm_scale) for weight_scale in weight_scales]
Fredrik Svedbergd67c0aa2020-03-30 13:15:28 +0200467 elif ifm_dtype == DataType.int8 or ifm_dtype == DataType.int16:
Tim Hall79d07d22020-04-27 18:20:16 +0100468 scales = [
469 (np.double(ifm_scale) * np.double(weight_scale)) / np.double(ofm_scale)
470 for weight_scale in weight_scales
471 ]
472 else:
Michael McGeagh7a6f8432020-12-02 15:29:22 +0000473 raise UnsupportedFeatureError(f"Compression of {ifm_dtype} is not implemented; Tensor: '{tens.name}'")
Tim Hall79d07d22020-04-27 18:20:16 +0100474 else:
475 if ifm_dtype == DataType.uint8:
476 scales = [np.double(ifm_scale * weight_scale * 0x3000) for weight_scale in weight_scales]
Fredrik Svedbergd67c0aa2020-03-30 13:15:28 +0200477 elif ifm_dtype == DataType.int8 or ifm_dtype == DataType.int16:
Tim Hall79d07d22020-04-27 18:20:16 +0100478 scales = [(np.double(ifm_scale * 0x3000) * np.double(weight_scale)) for weight_scale in weight_scales]
479 else:
Michael McGeagh7a6f8432020-12-02 15:29:22 +0000480 raise UnsupportedFeatureError(f"Compression of {ifm_dtype} is not implemented; Tensor: '{tens.name}'")
Tim Hall79d07d22020-04-27 18:20:16 +0100481
482 # quantise all of the weight scales into (scale_factor, shift)
Fredrik Svedbergd67c0aa2020-03-30 13:15:28 +0200483 if ifm_dtype == DataType.int16:
484 quantised_scales = [reduced_quantise_scale(scale) for scale in scales]
485 else:
486 quantised_scales = [quantise_scale(scale) for scale in scales]
Tim Hall79d07d22020-04-27 18:20:16 +0100487
Tim Hall79d07d22020-04-27 18:20:16 +0100488 # pack the biases and scales
Tim Hall79d07d22020-04-27 18:20:16 +0100489 if len(quantised_scales) == 1:
490 # If only 1 quantised scale is used, repeat that value for the length of the biases
491 quantised_scales = [quantised_scales[0]] * len(biases)
492
493 assert len(quantised_scales) == len(biases)
Tim Hall79d07d22020-04-27 18:20:16 +0100494 tens.element_size_bytes = 10
Tim Hallf7e810a2020-06-25 15:04:31 +0100495 tens.compressed_values = []
496 tens.compressed_values_substream_offsets = []
Tim Hall79d07d22020-04-27 18:20:16 +0100497
Tim Hallf7e810a2020-06-25 15:04:31 +0100498 total_elements = len(quantised_scales)
Patrik Gustavssond89c09e2020-07-08 11:27:12 +0200499 alignment_bytes = 0
Tim Hallf7e810a2020-06-25 15:04:31 +0100500 for i in range(0, total_elements, ofm_depth_step):
501 # Extract streams from brick to generate substreams for each core
502 stream = bytearray()
503 substream_offsets = [0]
504 max_len = min(ofm_depth_step, total_elements - i)
505 for core in range(0, min(arch.ncores, max_len)):
Jacob Bohline843d332020-06-23 12:12:56 +0200506 core_scales = quantised_scales[i + core : i + core + max_len : arch.ncores]
507 core_biases = biases[i + core : i + core + max_len : arch.ncores]
Tim Hallf7e810a2020-06-25 15:04:31 +0100508 for j, core_bias in enumerate(core_biases):
Manupa Karunaratnebef228b2020-07-29 18:06:28 +0100509 stream.extend(encode_bias(np.int64(core_bias), *core_scales[j]))
Tim Hall79d07d22020-04-27 18:20:16 +0100510
Tim Hallf7e810a2020-06-25 15:04:31 +0100511 # Align to 16 for start for next substream
Jacob Bohline843d332020-06-23 12:12:56 +0200512 remainder = (len(stream)) % 16
Tim Hallf7e810a2020-06-25 15:04:31 +0100513 if remainder > 0:
Jacob Bohline843d332020-06-23 12:12:56 +0200514 stream.extend(bytearray(16 - remainder))
Patrik Gustavssond89c09e2020-07-08 11:27:12 +0200515 alignment_bytes += 16 - remainder
Tim Hall79d07d22020-04-27 18:20:16 +0100516
Jacob Bohline843d332020-06-23 12:12:56 +0200517 substream_offsets.append(len(stream))
Tim Hall79d07d22020-04-27 18:20:16 +0100518
Tim Hallf7e810a2020-06-25 15:04:31 +0100519 # Add to compressed values with their substream offset lists to the tensor
Jacob Bohline843d332020-06-23 12:12:56 +0200520 tens.compressed_values.append(stream)
521 tens.compressed_values_substream_offsets.append(substream_offsets)
Tim Hallf7e810a2020-06-25 15:04:31 +0100522
Patrik Gustavssond89c09e2020-07-08 11:27:12 +0200523 tens.storage_shape = [total_elements + round_up_divide(alignment_bytes, tens.element_size_bytes)]
Tim Hall79d07d22020-04-27 18:20:16 +0100524
Jacob Bohline843d332020-06-23 12:12:56 +0200525
Tim Hall79d07d22020-04-27 18:20:16 +0100526def update_pass_weight_and_scale_tensors(nng, arch):
Tim Hall79d07d22020-04-27 18:20:16 +0100527 for sg in nng.subgraphs:
528 for ps in sg.passes:
Louis Verhaard3c07c972020-05-07 08:12:58 +0200529 tens = ps.weight_tensor
530 if tens is not None:
Louis Verhaardb2fb2122020-06-04 15:51:24 +0200531 op = tens.find_npu_op()
Dwight Lidman940fdee2020-08-13 13:11:48 +0200532 if op is None:
533 continue
Louis Verhaard3c07c972020-05-07 08:12:58 +0200534 needs_dma = tens.needs_dma()
Tim Hall79d07d22020-04-27 18:20:16 +0100535 if ps.cascade.strategy == SchedulingStrategy.WeightStream and needs_dma:
536 ofm_depth_step = ps.block_config[-1]
537 else:
Louis Verhaard3c07c972020-05-07 08:12:58 +0200538 ofm_depth_step = tens.shape[-1]
Fredrik Svedbergf5c07c42021-04-23 14:36:42 +0200539 nng.total_npu_weights += compress_weights(
Louis Verhaardaee5d752020-09-30 09:01:52 +0200540 arch, nng, tens, op.type.npu_block_type, ps.block_config[-1], ofm_depth_step, op.get_dilation_h_w()
Tim Hall79d07d22020-04-27 18:20:16 +0100541 )
Fredrik Svedbergf5c07c42021-04-23 14:36:42 +0200542 nng.total_npu_encoded_weights += tens.weight_compressed_offsets[-1]
543 nng.total_original_weights += int(tens.elements() * tens.element_size())
Diqing Zhong66d7ec02021-02-01 19:07:04 +0100544
Tim Hall79d07d22020-04-27 18:20:16 +0100545 # Update source tensor
Louis Verhaard3c07c972020-05-07 08:12:58 +0200546 if needs_dma:
547 src_tens = tens.get_dma_src_tensor()
548 src_tens.shape = tens.shape
549 src_tens.quant_values = tens.quant_values
550 src_tens.copy_compressed_weight_info(tens)
551 set_storage_shape(src_tens)
Tim Hall79d07d22020-04-27 18:20:16 +0100552
Diego Russoea6111a2020-04-14 18:41:58 +0100553 if ps.scale_tensor is not None:
Tim Hall79d07d22020-04-27 18:20:16 +0100554 rescale_for_faf = False
Michael McGeaghf3e3ad72020-12-02 12:39:03 +0000555 if (ps.ops[-1].type in (Op.Sigmoid, Op.Tanh)) and (ps.npu_block_type != NpuBlockType.ElementWise):
Tim Hall79d07d22020-04-27 18:20:16 +0100556 rescale_for_faf = True
Tim Hallf7e810a2020-06-25 15:04:31 +0100557 calc_scales_and_pack_biases(ps.scale_tensor, arch, ofm_depth_step, rescale_for_faf)
Andreas Nevalainen897cc142020-10-28 15:42:08 +0100558 if ps.scale_tensor.ops[0].type == Op.DMA:
559 src_tens = ps.scale_tensor.get_dma_src_tensor()
560 src_tens.shape = ps.scale_tensor.shape
561 src_tens.quant_values = ps.scale_tensor.quant_values
562 src_tens.element_size_bytes = ps.scale_tensor.element_size_bytes
563 src_tens.copy_compressed_weight_info(ps.scale_tensor)