blob: fe8f04b97b0a3d45faf7619eaed7a3989e1bbabb [file] [log] [blame]
Tim Hall79d07d22020-04-27 18:20:16 +01001# Copyright (C) 2020 Arm Limited or its affiliates. All rights reserved.
2#
3# SPDX-License-Identifier: Apache-2.0
4#
5# Licensed under the Apache License, Version 2.0 (the License); you may
6# not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9# www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an AS IS BASIS, WITHOUT
13# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
Tim Hall79d07d22020-04-27 18:20:16 +010016# Description:
17# Compresses and pads the weigths. It also calculates the scales and packs with the biases.
Tim Hall79d07d22020-04-27 18:20:16 +010018import math
Tim Hall79d07d22020-04-27 18:20:16 +010019from collections import namedtuple
Diego Russoea6111a2020-04-14 18:41:58 +010020
21import numpy as np
Tim Hallc30f4952020-06-15 20:47:35 +010022from ethosu import mlw_codec
Tim Hall79d07d22020-04-27 18:20:16 +010023
Diego Russoe8a10452020-04-21 17:39:10 +010024from .data_type import DataType
Louis Verhaard7db78962020-05-25 15:05:26 +020025from .errors import UnsupportedFeatureError
Diego Russoe8a10452020-04-21 17:39:10 +010026from .nn_graph import SchedulingStrategy
27from .numeric_util import round_up
28from .operation import NpuBlockType
29from .scaling import quantise_scale
30from .scaling import reduced_quantise_scale
31from .tensor import TensorBlockTraversal
32from .tensor import TensorFormat
33from .tensor import TensorPurpose
34from .tensor import TensorSubPurpose
35
Tim Hall79d07d22020-04-27 18:20:16 +010036
Louis Verhaard3c07c972020-05-07 08:12:58 +020037# Contains meta info for a weight compression. If two tensors have identical weight compression config,
38# then they also will have identical compressed weights.
39WeightCompressionConfig = namedtuple(
Louis Verhaardb2fb2122020-06-04 15:51:24 +020040 "WeightCompressionConfig", ["npu_block_type", "ofm_block_depth", "ofm_depth_step", "dilation", "equivalence_id"]
Louis Verhaard3c07c972020-05-07 08:12:58 +020041)
42
43
Louis Verhaardb2fb2122020-06-04 15:51:24 +020044def create_weight_compression_config(tens, npu_block_type, ofm_block_depth, ofm_depth_step, dilation):
Louis Verhaard3c07c972020-05-07 08:12:58 +020045 # Note: for an ofm block only its depth is used in weight compression.
46 # And block depth > ofm depth gives same result as block depth == ofm depth
47 block_depth = min(ofm_block_depth, tens.quant_values.shape[-1])
Louis Verhaardb2fb2122020-06-04 15:51:24 +020048 return WeightCompressionConfig(npu_block_type, block_depth, ofm_depth_step, dilation, tens.equivalence_id)
Louis Verhaard3c07c972020-05-07 08:12:58 +020049
50
51def set_storage_shape(tens):
52 # Sets the storage shape depending on the tensor's sub purpose
53 if tens.sub_purpose == TensorSubPurpose.DoubleBuffer and len(tens.compressed_values) > 2:
54 offset = 2 * np.amax([len(x) for x in tens.compressed_values])
55 assert offset % 16 == 0
56 else:
57 offset = tens.weight_compressed_offsets[-1]
58 tens.storage_shape = [1, 1, 1, offset]
59
60
61class CompressedWeightCache:
62 # Contains weight compressions for all weight tensors in a graph
63 def __init__(self):
64 self.cache = {} # maps from WeightCompressionConfig to a tensor clone containing compressed weights
65
66 def get_tensor_with_same_compression(self, wcc):
67 return self.cache.get(wcc)
68
69 def add(self, tens):
70 # Adds the compressed weights from the tensor to the cache
71 wcc = tens.weight_compression_config
72 # Clone the tensor to make sure that nothing related to the weight compression is modified
73 tens_clone = tens.clone("_weights{}_{}".format(wcc.ofm_block_depth, wcc.ofm_depth_step))
74 self.cache[wcc] = tens_clone
75
76
Tim Hall79d07d22020-04-27 18:20:16 +010077def encode(weight_stream):
78 assert np.amin(weight_stream) >= -255
79 assert np.amax(weight_stream) <= 255
80
81 # Encode flattened signed weight stream
82 compressed = mlw_codec.encode(weight_stream)
83
84 # pad with 0xFF as needed so the length of the weight stream
85 # is a multiple of 16
Diego Russoea6111a2020-04-14 18:41:58 +010086
Tim Hall79d07d22020-04-27 18:20:16 +010087 while (len(compressed) % 16) != 0:
88 compressed.append(0xFF)
89
90 return compressed
91
92
Louis Verhaardb2fb2122020-06-04 15:51:24 +020093def generate_brick(arch, brick_weights, ofm_block_depth, block_traversal, ifm_bitdepth, dilation):
Tim Hall79d07d22020-04-27 18:20:16 +010094 is_depthwise = block_traversal == TensorBlockTraversal.DepthWise
95 is_partkernel = block_traversal == TensorBlockTraversal.PartKernelFirst
Louis Verhaardb2fb2122020-06-04 15:51:24 +020096 decomp_h = arch.subkernel_max.height // dilation[0]
97 decomp_w = arch.subkernel_max.width // dilation[1]
Tim Hall79d07d22020-04-27 18:20:16 +010098 ofm_ublock = arch.ofm_ublock
99 ifm_ublock = arch.ifm_ublock
Tim Hallf7e810a2020-06-25 15:04:31 +0100100 # Expect weights formatted OHWI
101 ofm_depth = brick_weights.shape[-4]
102 ifm_depth = brick_weights.shape[-1]
103 kernel_width = brick_weights.shape[-2]
104 kernel_height = brick_weights.shape[-3]
Tim Hall79d07d22020-04-27 18:20:16 +0100105 # IFM block depth
106 if is_partkernel or (ifm_bitdepth == 16):
107 # IFM block depth is always 16 for part-kernel-first
108 ifm_block_depth = 16
109 elif ifm_bitdepth == 8:
110 ifm_block_depth = 32
111 else:
112 assert False
113
114 stream = []
115
116 # Top level striping - OFM blocks in the entire brick's depth
Louis Verhaard3c07c972020-05-07 08:12:58 +0200117 for ofm_block_z in range(0, ofm_depth, ofm_block_depth):
118 clipped_ofm_block_depth = min(ofm_block_depth, ofm_depth - ofm_block_z)
Tim Hall79d07d22020-04-27 18:20:16 +0100119 # IFM blocks required for the brick
120 for ifm_block_z in range(0, (1 if is_depthwise else ifm_depth), ifm_block_depth):
121 if is_depthwise:
122 clipped_ifm_block_depth = ifm_ublock.depth
123 else:
124 clipped_ifm_block_depth = (
125 min(ifm_block_depth, ifm_depth - ifm_block_z) if is_partkernel else ifm_block_depth
126 )
127 # Weight decomposition
128 # Subkernel Splitting (H)
Louis Verhaardb2fb2122020-06-04 15:51:24 +0200129 for subkernel_y in range(0, kernel_height, decomp_h):
130 sub_height = min(kernel_height - subkernel_y, decomp_h)
Tim Hall79d07d22020-04-27 18:20:16 +0100131 # Subkernel splitting (W)
Louis Verhaardb2fb2122020-06-04 15:51:24 +0200132 for subkernel_x in range(0, kernel_width, decomp_w):
133 sub_width = min(kernel_width - subkernel_x, decomp_w)
Tim Hall79d07d22020-04-27 18:20:16 +0100134 subkernel_elements = sub_width * sub_height
135 # Part kernel first works across the kernel H/W and needs padding
136 if is_partkernel:
137 if ifm_bitdepth == 16 and subkernel_elements % 2 != 0:
138 subkernel_elements = int(math.ceil(subkernel_elements / 2) * 2)
139 elif ifm_bitdepth == 8 and subkernel_elements % 4 != 0:
140 subkernel_elements = int(math.ceil(subkernel_elements / 4) * 4)
141
142 # Depthwise Conv requires multiple of 4 kernel elements in its weight block
143 # this is different from normal conv which is considered "weights depth-first"
144 elif is_depthwise:
145 subkernel_elements = int(math.ceil(subkernel_elements / 4.0) * 4)
146
147 ifm_block_depth_outer = clipped_ifm_block_depth if is_partkernel else 1
148 ifm_block_depth_inner = 1 if is_partkernel else clipped_ifm_block_depth
149 # IFM Ublocks in IFM-block over depth for part-kernel-first mode
150 # For depth-first IFM Ublocks are traversed after subkernel elements so this loop is ignored.
151 for ifm_ublk_outer in range(0, ifm_block_depth_outer, ifm_ublock.depth):
152 # OFM Ublocks in OFM-block over depth
153 for ofm_ublk in range(0, clipped_ofm_block_depth, ofm_ublock.depth):
154 # HW Kernel element traversal - cannot be a H/W loop due to element
155 # padding requirement on depthwise/part-kernel configurations
156 for element in range(subkernel_elements):
157 kx = element % sub_width
158 ky = element // sub_width
159 # IFM Ublocks in IFM-block over depth (only 1 ublock if depthwise)
160 # In case of part-kernel-first IFM Ublock traversal have already been handled
161 # and this loop is ignored.
162 for ifm_ublk_inner in range(0, ifm_block_depth_inner, ifm_ublock.depth):
163 # Feed OFM ublock elements
164 for ofm_ublock_z in range(ofm_ublock.depth):
165 # Source IFM ublock elements (only 1 element deep if depthwise)
166 for ifm_ublock_z in range(1 if is_depthwise else ifm_ublock.depth):
167 # Source position within the current subkernel
168 wx = subkernel_x + kx
169 wy = subkernel_y + ky
170 # Source IFM/OFM slices
171 ifm_ublk = ifm_ublk_inner + ifm_ublk_outer
172 ifm_z = ifm_block_z + ifm_ublk + ifm_ublock_z
173 ofm_z = ofm_block_z + ofm_ublk + ofm_ublock_z
174 if (ifm_z >= ifm_depth) or (ofm_z >= ofm_depth) or (ky >= sub_height):
175 stream.append(0)
176 else:
Tim Hallf7e810a2020-06-25 15:04:31 +0100177 stream.append(brick_weights[ofm_z][wy][wx][ifm_z])
Tim Hall79d07d22020-04-27 18:20:16 +0100178 return stream
179
Tim Hallf7e810a2020-06-25 15:04:31 +0100180def core_deinterleave(hwio, core, ncores):
181 # Put weights back into OHWI
182 ohwi = np.transpose(hwio, (3,0,1,2))
183 return ohwi[core:ohwi.shape[0]:ncores]
Tim Hall79d07d22020-04-27 18:20:16 +0100184
185# Compress the weights
Louis Verhaardb2fb2122020-06-04 15:51:24 +0200186def compress_weights(arch, nng, tens, npu_block_type, ofm_block_depth, ofm_depth_step, dilation):
Tim Hall79d07d22020-04-27 18:20:16 +0100187 assert tens.purpose == TensorPurpose.Weights
188 assert tens.format == TensorFormat.WeightsCompressed
189
Louis Verhaard3c07c972020-05-07 08:12:58 +0200190 # Check the weight cache
191 if nng.weight_cache is None:
192 nng.weight_cache = CompressedWeightCache()
Louis Verhaardb2fb2122020-06-04 15:51:24 +0200193 wcc = create_weight_compression_config(tens, npu_block_type, ofm_block_depth, ofm_depth_step, dilation)
Louis Verhaard3c07c972020-05-07 08:12:58 +0200194 tens.weight_compression_config = wcc
195 tens_cached = nng.weight_cache.get_tensor_with_same_compression(wcc)
196 if tens_cached is not None:
197 # Cache hit, copy weights from the cache
198 tens.copy_compressed_weight_info(tens_cached)
199 set_storage_shape(tens)
200 return
Tim Hall79d07d22020-04-27 18:20:16 +0100201
Louis Verhaard3c07c972020-05-07 08:12:58 +0200202 # No cache hit, perform the compression
Tim Hall79d07d22020-04-27 18:20:16 +0100203 assert tens.quantization is not None
204 assert tens.quantization.scale_f32 is not None
205 assert tens.quantization.zero_point is not None
206
207 zero_point = tens.quantization.zero_point
208 quant_buf = tens.quant_values.astype(np.int64)
209
210 # Early zero-point correction
211 weights = quant_buf - zero_point
212
213 if len(weights.shape) == 2:
214 weights = np.expand_dims(np.expand_dims(weights, axis=0), axis=0)
215 weights_shape = (weights.shape[0], 1, 1, weights.shape[1])
216 else:
217 weights_shape = weights.shape
218
219 compression_scales = []
220 compressed_offsets = []
221 encoded_streams = []
Tim Hallf7e810a2020-06-25 15:04:31 +0100222 encoded_streams_substream_offsets = []
Tim Hall79d07d22020-04-27 18:20:16 +0100223 offset = 0
Tim Hallf7e810a2020-06-25 15:04:31 +0100224 max_single_buffer_len = 0
Tim Hall79d07d22020-04-27 18:20:16 +0100225
226 ifm_bitdepth = tens.consumer_list[0].inputs[0].dtype.size_in_bits()
227 ifm_depth = weights.shape[-2]
228 if npu_block_type == NpuBlockType.ConvolutionDepthWise:
229 tens.block_traversal = TensorBlockTraversal.DepthWise
230 if npu_block_type == NpuBlockType.ConvolutionMxN:
231 # Determine which block traversal strategy has better DPU utilization
232 kernel_size = weights_shape[0] * weights_shape[1]
233 depth_utilization = weights_shape[2] / round_up(weights_shape[2], 32 if ifm_bitdepth == 8 else 16)
234 part_kernel_utilization = (weights_shape[2] / round_up(weights_shape[2], 8)) * (
235 kernel_size / round_up(kernel_size, 4 if ifm_bitdepth == 8 else 2)
236 )
237 if part_kernel_utilization >= depth_utilization or ifm_depth <= 8:
238 # Part-kernel first is always better for ifm depths <= 8
239 tens.block_traversal = TensorBlockTraversal.PartKernelFirst
240 else:
241 tens.block_traversal = TensorBlockTraversal.DepthFirst
242
Jacob Bohlincf7da102020-05-20 09:03:40 +0200243 if tens.consumer_list[0].type == "Conv2DBackpropInputSwitchedBias":
244 # Transpose Convoluion, reverse weights in H and W axes
Tim Hallc30f4952020-06-15 20:47:35 +0100245 weights = np.flip(weights, axis=(0, 1))
Jacob Bohlincf7da102020-05-20 09:03:40 +0200246
Tim Hall79d07d22020-04-27 18:20:16 +0100247 # Slice weight stream up depth-ways into bricks and compress
248 full_ofm_depth = quant_buf.shape[-1]
Tim Hallf7e810a2020-06-25 15:04:31 +0100249 ofm_block_depth = ofm_block_depth // arch.ncores
Tim Hall79d07d22020-04-27 18:20:16 +0100250 for idx in range(0, full_ofm_depth, ofm_depth_step):
251 # Get the weights necessary for this brick
252 count = min(full_ofm_depth - idx, ofm_depth_step)
253 brick_weights = weights[:, :, :, idx : idx + count]
254
Tim Hallf7e810a2020-06-25 15:04:31 +0100255 substream_offsets = [0]
256 encoded_stream = []
257 raw_size = 0
258
259 # For each core, deinterleave weights from the larger volume
260 # and generate separate compressed streams.
261 for core in range(0, min(arch.ncores, full_ofm_depth)):
262 core_weights = core_deinterleave(brick_weights, core, arch.ncores)
263 raw_stream = generate_brick(arch, core_weights, ofm_block_depth, tens.block_traversal, ifm_bitdepth, dilation)
264 raw_size += len( raw_stream )
265 encoded_substream = encode( raw_stream )
266 encoded_stream.extend( encoded_substream )
267 substream_offsets.append( len(encoded_stream) )
268
269 encoded_streams.append( encoded_stream )
270 encoded_streams_substream_offsets.append( substream_offsets )
271
272 # Remember maximum encoded length for DoubleBuffering
273 max_single_buffer_len = max(max_single_buffer_len, len(encoded_stream))
Tim Hall79d07d22020-04-27 18:20:16 +0100274
Tim Hall79d07d22020-04-27 18:20:16 +0100275 # Remember where we put it for linear addressing
276 compressed_offsets.append(offset)
Tim Hallf7e810a2020-06-25 15:04:31 +0100277 offset += len(encoded_stream)
Tim Hall79d07d22020-04-27 18:20:16 +0100278 assert offset % 16 == 0
279
280 # Compression scale tracking
Tim Hallf7e810a2020-06-25 15:04:31 +0100281 compression_scales.append(len(encoded_stream) / raw_size)
Tim Hall79d07d22020-04-27 18:20:16 +0100282
Tim Hallf7e810a2020-06-25 15:04:31 +0100283 # Track total length as last element of the offsets array
Tim Hall79d07d22020-04-27 18:20:16 +0100284 compressed_offsets.append(offset)
285
Tim Hall79d07d22020-04-27 18:20:16 +0100286 tens.weight_compression_scales = compression_scales
Tim Hall79d07d22020-04-27 18:20:16 +0100287 tens.weight_compressed_offsets = compressed_offsets
288 tens.compression_scale_for_worst_weight_stream = np.amax(compression_scales)
289 tens.storage_compression_scale = tens.bandwidth_compression_scale = np.average(compression_scales)
290 tens.compressed_values = encoded_streams
Tim Hallf7e810a2020-06-25 15:04:31 +0100291 tens.compressed_values_substream_offsets = encoded_streams_substream_offsets
Tim Hall79d07d22020-04-27 18:20:16 +0100292 tens.brick_size = (weights_shape[0], weights_shape[1], weights_shape[2], min(tens.shape[-1], ofm_depth_step))
Louis Verhaard3c07c972020-05-07 08:12:58 +0200293 set_storage_shape(tens)
294 nng.weight_cache.add(tens)
Tim Hall79d07d22020-04-27 18:20:16 +0100295
Tim Hallf7e810a2020-06-25 15:04:31 +0100296def calc_scales_and_pack_biases(tens, arch, ofm_depth_step, rescale_for_faf=False):
Tim Hall79d07d22020-04-27 18:20:16 +0100297 assert tens.purpose == TensorPurpose.FeatureMap
298 assert tens.format == TensorFormat.NHWC
299 # the connected operator should expect a bias input unless it is a FullyConnected
300 assert "Bias" in tens.consumer_list[0].type or tens.consumer_list[0].type.startswith("FullyConnected")
301 # the input bias tensor is the same as that connected to the operator
Jacob Bohlincf7da102020-05-20 09:03:40 +0200302 _, _, bias_tens, _ = tens.consumer_list[0].get_ifm_weights_biases_ofm()
303 assert tens is bias_tens
304
Tim Hall79d07d22020-04-27 18:20:16 +0100305 # the operator should only have a single output
306 assert len(tens.consumer_list[0].outputs) == 1
307
308 def pack_bias_and_scale(bias, scale, shift):
309 bias = np.int64(bias)
310 assert -(1 << (40 - 1)) <= bias < (1 << (40 - 1)) # signed 40-bit range
311 assert 0 <= scale < (1 << 32) # unsigned 32-bit range
312 assert 0 <= shift < (1 << 6) # unsigned 6-bit range
313
314 # pack the 80 bit value = [0(2-bits),shift(6-bits),scale(32-bits),bias(40-bits)]
315 data = bytearray(10)
316 data[0] = (bias >> (0 * 8)) & 0xFF
317 data[1] = (bias >> (1 * 8)) & 0xFF
318 data[2] = (bias >> (2 * 8)) & 0xFF
319 data[3] = (bias >> (3 * 8)) & 0xFF
320 data[4] = (bias >> (4 * 8)) & 0xFF
321 data[5] = (scale >> (0 * 8)) & 0xFF
322 data[6] = (scale >> (1 * 8)) & 0xFF
323 data[7] = (scale >> (2 * 8)) & 0xFF
324 data[8] = (scale >> (3 * 8)) & 0xFF
325 data[9] = shift & 0x3F
326 return data
327
328 biases = tens.quant_values
329
330 first_consumer_op = tens.consumer_list[0]
331 ifm_dtype = first_consumer_op.inputs[0].dtype
332 ifm_scale = first_consumer_op.inputs[0].quantization.scale_f32
333 ofm_scale = first_consumer_op.outputs[0].quantization.scale_f32
334 weight_scales = first_consumer_op.inputs[1].quantization.scale_f32
335
336 # biases can have multiple consumers for rnn cells. if so, then check that they are all the same
337 for op in tens.consumer_list[1:]:
338 assert ifm_scale == op.inputs[0].quantization.scale_f32
339 assert ofm_scale == op.outputs[0].quantization.scale_f32
340 assert weight_scales == op.inputs[1].quantization.scale_f32
341
342 if not hasattr(weight_scales, "__iter__"):
343 # If weight_scales is not already an iterable make it into a list
344 weight_scales = [weight_scales]
345
346 # Convert scales to np.double (from np.float32) to conform to TensorFlow Lite which
347 # uses double during scaling calculations
348 # TensorFlow Lite casts the scales slightly differently for uint8 and int8
349 if not rescale_for_faf:
350 if ifm_dtype == DataType.uint8:
351 scales = [np.double(ifm_scale * weight_scale) / np.double(ofm_scale) for weight_scale in weight_scales]
Fredrik Svedbergd67c0aa2020-03-30 13:15:28 +0200352 elif ifm_dtype == DataType.int8 or ifm_dtype == DataType.int16:
Tim Hall79d07d22020-04-27 18:20:16 +0100353 scales = [
354 (np.double(ifm_scale) * np.double(weight_scale)) / np.double(ofm_scale)
355 for weight_scale in weight_scales
356 ]
357 else:
Louis Verhaard7db78962020-05-25 15:05:26 +0200358 raise UnsupportedFeatureError(
359 "Compression of {} is not implemented; tensor: {}".format(ifm_dtype, tens.name)
360 )
Tim Hall79d07d22020-04-27 18:20:16 +0100361 else:
362 if ifm_dtype == DataType.uint8:
363 scales = [np.double(ifm_scale * weight_scale * 0x3000) for weight_scale in weight_scales]
Fredrik Svedbergd67c0aa2020-03-30 13:15:28 +0200364 elif ifm_dtype == DataType.int8 or ifm_dtype == DataType.int16:
Tim Hall79d07d22020-04-27 18:20:16 +0100365 scales = [(np.double(ifm_scale * 0x3000) * np.double(weight_scale)) for weight_scale in weight_scales]
366 else:
Louis Verhaard7db78962020-05-25 15:05:26 +0200367 raise UnsupportedFeatureError(
368 "Compression of {} is not implemented; tensor: {}".format(ifm_dtype, tens.name)
369 )
Tim Hall79d07d22020-04-27 18:20:16 +0100370
371 # quantise all of the weight scales into (scale_factor, shift)
Fredrik Svedbergd67c0aa2020-03-30 13:15:28 +0200372 if ifm_dtype == DataType.int16:
373 quantised_scales = [reduced_quantise_scale(scale) for scale in scales]
374 else:
375 quantised_scales = [quantise_scale(scale) for scale in scales]
Tim Hall79d07d22020-04-27 18:20:16 +0100376
377 for _, shift in quantised_scales:
378 assert shift >= 16
379
380 # pack the biases and scales
Tim Hall79d07d22020-04-27 18:20:16 +0100381 if len(quantised_scales) == 1:
382 # If only 1 quantised scale is used, repeat that value for the length of the biases
383 quantised_scales = [quantised_scales[0]] * len(biases)
384
385 assert len(quantised_scales) == len(biases)
Tim Hall79d07d22020-04-27 18:20:16 +0100386 tens.element_size_bytes = 10
Tim Hallf7e810a2020-06-25 15:04:31 +0100387 tens.compressed_values = []
388 tens.compressed_values_substream_offsets = []
Tim Hall79d07d22020-04-27 18:20:16 +0100389
Tim Hallf7e810a2020-06-25 15:04:31 +0100390 total_elements = len(quantised_scales)
391 for i in range(0, total_elements, ofm_depth_step):
392 # Extract streams from brick to generate substreams for each core
393 stream = bytearray()
394 substream_offsets = [0]
395 max_len = min(ofm_depth_step, total_elements - i)
396 for core in range(0, min(arch.ncores, max_len)):
397 core_scales = quantised_scales[i+core:i+core+max_len:arch.ncores]
398 core_biases = biases[i+core:i+core+max_len:arch.ncores]
399 for j, core_bias in enumerate(core_biases):
400 stream.extend( pack_bias_and_scale(core_bias, *core_scales[j]) )
Tim Hall79d07d22020-04-27 18:20:16 +0100401
Tim Hallf7e810a2020-06-25 15:04:31 +0100402 # Align to 16 for start for next substream
403 remainder = ( len(stream) ) % 16
404 if remainder > 0:
405 stream.extend( bytearray(16 - remainder) )
Tim Hall79d07d22020-04-27 18:20:16 +0100406
Tim Hallf7e810a2020-06-25 15:04:31 +0100407 substream_offsets.append( len(stream) )
Tim Hall79d07d22020-04-27 18:20:16 +0100408
Tim Hallf7e810a2020-06-25 15:04:31 +0100409 # Add to compressed values with their substream offset lists to the tensor
410 tens.compressed_values.append( stream )
411 tens.compressed_values_substream_offsets.append( substream_offsets )
412
413 tens.storage_shape = [total_elements * tens.element_size_bytes]
Tim Hall79d07d22020-04-27 18:20:16 +0100414
415def update_pass_weight_and_scale_tensors(nng, arch):
Tim Hall79d07d22020-04-27 18:20:16 +0100416 for sg in nng.subgraphs:
417 for ps in sg.passes:
Louis Verhaard3c07c972020-05-07 08:12:58 +0200418 tens = ps.weight_tensor
419 if tens is not None:
Louis Verhaardb2fb2122020-06-04 15:51:24 +0200420 op = tens.find_npu_op()
421 npu_usage_of_tensor = op.attrs["npu_block_type"]
Tim Hall79d07d22020-04-27 18:20:16 +0100422 if npu_usage_of_tensor == NpuBlockType.ConvolutionDepthWise:
Louis Verhaard3c07c972020-05-07 08:12:58 +0200423 tens.quant_values = np.transpose(tens.quant_values, (0, 1, 3, 2))
424 tens.shape = tens.storage_shape = tens.bandwidth_shape = list(tens.quant_values.shape)
425 tens.weight_transpose_depthwise = True
Tim Hall79d07d22020-04-27 18:20:16 +0100426
Louis Verhaard3c07c972020-05-07 08:12:58 +0200427 needs_dma = tens.needs_dma()
Tim Hall79d07d22020-04-27 18:20:16 +0100428 if ps.cascade.strategy == SchedulingStrategy.WeightStream and needs_dma:
429 ofm_depth_step = ps.block_config[-1]
430 else:
Louis Verhaard3c07c972020-05-07 08:12:58 +0200431 ofm_depth_step = tens.shape[-1]
Tim Hall79d07d22020-04-27 18:20:16 +0100432 compress_weights(
Louis Verhaardb2fb2122020-06-04 15:51:24 +0200433 arch, nng, tens, npu_usage_of_tensor, ps.block_config[-1], ofm_depth_step, op.get_dilation_h_w()
Tim Hall79d07d22020-04-27 18:20:16 +0100434 )
435 # Update source tensor
Louis Verhaard3c07c972020-05-07 08:12:58 +0200436 if needs_dma:
437 src_tens = tens.get_dma_src_tensor()
438 src_tens.shape = tens.shape
439 src_tens.quant_values = tens.quant_values
440 src_tens.copy_compressed_weight_info(tens)
441 set_storage_shape(src_tens)
Tim Hall79d07d22020-04-27 18:20:16 +0100442
Diego Russoea6111a2020-04-14 18:41:58 +0100443 if ps.scale_tensor is not None:
Tim Hall79d07d22020-04-27 18:20:16 +0100444 rescale_for_faf = False
445 activation_ops = set(("Sigmoid", "Tanh"))
446 if (ps.ops[-1].type in activation_ops) and (ps.npu_block_type != NpuBlockType.ElementWise):
447 rescale_for_faf = True
Tim Hallf7e810a2020-06-25 15:04:31 +0100448 calc_scales_and_pack_biases(ps.scale_tensor, arch, ofm_depth_step, rescale_for_faf)