Tim Hall | 79d07d2 | 2020-04-27 18:20:16 +0100 | [diff] [blame] | 1 | # Copyright (C) 2020 Arm Limited or its affiliates. All rights reserved. |
| 2 | # |
| 3 | # SPDX-License-Identifier: Apache-2.0 |
| 4 | # |
| 5 | # Licensed under the Apache License, Version 2.0 (the License); you may |
| 6 | # not use this file except in compliance with the License. |
| 7 | # You may obtain a copy of the License at |
| 8 | # |
| 9 | # www.apache.org/licenses/LICENSE-2.0 |
| 10 | # |
| 11 | # Unless required by applicable law or agreed to in writing, software |
| 12 | # distributed under the License is distributed on an AS IS BASIS, WITHOUT |
| 13 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 14 | # See the License for the specific language governing permissions and |
| 15 | # limitations under the License. |
Tim Hall | 79d07d2 | 2020-04-27 18:20:16 +0100 | [diff] [blame] | 16 | # Description: |
| 17 | # Compresses and pads the weigths. It also calculates the scales and packs with the biases. |
Tim Hall | 79d07d2 | 2020-04-27 18:20:16 +0100 | [diff] [blame] | 18 | import math |
Tim Hall | 79d07d2 | 2020-04-27 18:20:16 +0100 | [diff] [blame] | 19 | from collections import namedtuple |
Diego Russo | ea6111a | 2020-04-14 18:41:58 +0100 | [diff] [blame] | 20 | |
| 21 | import numpy as np |
Tim Hall | c30f495 | 2020-06-15 20:47:35 +0100 | [diff] [blame] | 22 | from ethosu import mlw_codec |
Tim Hall | 79d07d2 | 2020-04-27 18:20:16 +0100 | [diff] [blame] | 23 | |
Diego Russo | e8a1045 | 2020-04-21 17:39:10 +0100 | [diff] [blame] | 24 | from .data_type import DataType |
Louis Verhaard | 7db7896 | 2020-05-25 15:05:26 +0200 | [diff] [blame] | 25 | from .errors import UnsupportedFeatureError |
Diego Russo | e8a1045 | 2020-04-21 17:39:10 +0100 | [diff] [blame] | 26 | from .nn_graph import SchedulingStrategy |
| 27 | from .numeric_util import round_up |
| 28 | from .operation import NpuBlockType |
| 29 | from .scaling import quantise_scale |
| 30 | from .scaling import reduced_quantise_scale |
| 31 | from .tensor import TensorBlockTraversal |
| 32 | from .tensor import TensorFormat |
| 33 | from .tensor import TensorPurpose |
| 34 | from .tensor import TensorSubPurpose |
| 35 | |
Tim Hall | 79d07d2 | 2020-04-27 18:20:16 +0100 | [diff] [blame] | 36 | |
Louis Verhaard | 3c07c97 | 2020-05-07 08:12:58 +0200 | [diff] [blame] | 37 | # Contains meta info for a weight compression. If two tensors have identical weight compression config, |
| 38 | # then they also will have identical compressed weights. |
| 39 | WeightCompressionConfig = namedtuple( |
Louis Verhaard | b2fb212 | 2020-06-04 15:51:24 +0200 | [diff] [blame] | 40 | "WeightCompressionConfig", ["npu_block_type", "ofm_block_depth", "ofm_depth_step", "dilation", "equivalence_id"] |
Louis Verhaard | 3c07c97 | 2020-05-07 08:12:58 +0200 | [diff] [blame] | 41 | ) |
| 42 | |
| 43 | |
Louis Verhaard | b2fb212 | 2020-06-04 15:51:24 +0200 | [diff] [blame] | 44 | def create_weight_compression_config(tens, npu_block_type, ofm_block_depth, ofm_depth_step, dilation): |
Louis Verhaard | 3c07c97 | 2020-05-07 08:12:58 +0200 | [diff] [blame] | 45 | # Note: for an ofm block only its depth is used in weight compression. |
| 46 | # And block depth > ofm depth gives same result as block depth == ofm depth |
| 47 | block_depth = min(ofm_block_depth, tens.quant_values.shape[-1]) |
Louis Verhaard | b2fb212 | 2020-06-04 15:51:24 +0200 | [diff] [blame] | 48 | return WeightCompressionConfig(npu_block_type, block_depth, ofm_depth_step, dilation, tens.equivalence_id) |
Louis Verhaard | 3c07c97 | 2020-05-07 08:12:58 +0200 | [diff] [blame] | 49 | |
| 50 | |
| 51 | def set_storage_shape(tens): |
| 52 | # Sets the storage shape depending on the tensor's sub purpose |
| 53 | if tens.sub_purpose == TensorSubPurpose.DoubleBuffer and len(tens.compressed_values) > 2: |
| 54 | offset = 2 * np.amax([len(x) for x in tens.compressed_values]) |
| 55 | assert offset % 16 == 0 |
| 56 | else: |
| 57 | offset = tens.weight_compressed_offsets[-1] |
| 58 | tens.storage_shape = [1, 1, 1, offset] |
| 59 | |
| 60 | |
| 61 | class CompressedWeightCache: |
| 62 | # Contains weight compressions for all weight tensors in a graph |
| 63 | def __init__(self): |
| 64 | self.cache = {} # maps from WeightCompressionConfig to a tensor clone containing compressed weights |
| 65 | |
| 66 | def get_tensor_with_same_compression(self, wcc): |
| 67 | return self.cache.get(wcc) |
| 68 | |
| 69 | def add(self, tens): |
| 70 | # Adds the compressed weights from the tensor to the cache |
| 71 | wcc = tens.weight_compression_config |
| 72 | # Clone the tensor to make sure that nothing related to the weight compression is modified |
| 73 | tens_clone = tens.clone("_weights{}_{}".format(wcc.ofm_block_depth, wcc.ofm_depth_step)) |
| 74 | self.cache[wcc] = tens_clone |
| 75 | |
| 76 | |
Tim Hall | 79d07d2 | 2020-04-27 18:20:16 +0100 | [diff] [blame] | 77 | def encode(weight_stream): |
| 78 | assert np.amin(weight_stream) >= -255 |
| 79 | assert np.amax(weight_stream) <= 255 |
| 80 | |
| 81 | # Encode flattened signed weight stream |
| 82 | compressed = mlw_codec.encode(weight_stream) |
| 83 | |
| 84 | # pad with 0xFF as needed so the length of the weight stream |
| 85 | # is a multiple of 16 |
Diego Russo | ea6111a | 2020-04-14 18:41:58 +0100 | [diff] [blame] | 86 | |
Tim Hall | 79d07d2 | 2020-04-27 18:20:16 +0100 | [diff] [blame] | 87 | while (len(compressed) % 16) != 0: |
| 88 | compressed.append(0xFF) |
| 89 | |
| 90 | return compressed |
| 91 | |
| 92 | |
Louis Verhaard | b2fb212 | 2020-06-04 15:51:24 +0200 | [diff] [blame] | 93 | def generate_brick(arch, brick_weights, ofm_block_depth, block_traversal, ifm_bitdepth, dilation): |
Tim Hall | 79d07d2 | 2020-04-27 18:20:16 +0100 | [diff] [blame] | 94 | is_depthwise = block_traversal == TensorBlockTraversal.DepthWise |
| 95 | is_partkernel = block_traversal == TensorBlockTraversal.PartKernelFirst |
Louis Verhaard | b2fb212 | 2020-06-04 15:51:24 +0200 | [diff] [blame] | 96 | decomp_h = arch.subkernel_max.height // dilation[0] |
| 97 | decomp_w = arch.subkernel_max.width // dilation[1] |
Tim Hall | 79d07d2 | 2020-04-27 18:20:16 +0100 | [diff] [blame] | 98 | ofm_ublock = arch.ofm_ublock |
| 99 | ifm_ublock = arch.ifm_ublock |
Tim Hall | f7e810a | 2020-06-25 15:04:31 +0100 | [diff] [blame^] | 100 | # Expect weights formatted OHWI |
| 101 | ofm_depth = brick_weights.shape[-4] |
| 102 | ifm_depth = brick_weights.shape[-1] |
| 103 | kernel_width = brick_weights.shape[-2] |
| 104 | kernel_height = brick_weights.shape[-3] |
Tim Hall | 79d07d2 | 2020-04-27 18:20:16 +0100 | [diff] [blame] | 105 | # IFM block depth |
| 106 | if is_partkernel or (ifm_bitdepth == 16): |
| 107 | # IFM block depth is always 16 for part-kernel-first |
| 108 | ifm_block_depth = 16 |
| 109 | elif ifm_bitdepth == 8: |
| 110 | ifm_block_depth = 32 |
| 111 | else: |
| 112 | assert False |
| 113 | |
| 114 | stream = [] |
| 115 | |
| 116 | # Top level striping - OFM blocks in the entire brick's depth |
Louis Verhaard | 3c07c97 | 2020-05-07 08:12:58 +0200 | [diff] [blame] | 117 | for ofm_block_z in range(0, ofm_depth, ofm_block_depth): |
| 118 | clipped_ofm_block_depth = min(ofm_block_depth, ofm_depth - ofm_block_z) |
Tim Hall | 79d07d2 | 2020-04-27 18:20:16 +0100 | [diff] [blame] | 119 | # IFM blocks required for the brick |
| 120 | for ifm_block_z in range(0, (1 if is_depthwise else ifm_depth), ifm_block_depth): |
| 121 | if is_depthwise: |
| 122 | clipped_ifm_block_depth = ifm_ublock.depth |
| 123 | else: |
| 124 | clipped_ifm_block_depth = ( |
| 125 | min(ifm_block_depth, ifm_depth - ifm_block_z) if is_partkernel else ifm_block_depth |
| 126 | ) |
| 127 | # Weight decomposition |
| 128 | # Subkernel Splitting (H) |
Louis Verhaard | b2fb212 | 2020-06-04 15:51:24 +0200 | [diff] [blame] | 129 | for subkernel_y in range(0, kernel_height, decomp_h): |
| 130 | sub_height = min(kernel_height - subkernel_y, decomp_h) |
Tim Hall | 79d07d2 | 2020-04-27 18:20:16 +0100 | [diff] [blame] | 131 | # Subkernel splitting (W) |
Louis Verhaard | b2fb212 | 2020-06-04 15:51:24 +0200 | [diff] [blame] | 132 | for subkernel_x in range(0, kernel_width, decomp_w): |
| 133 | sub_width = min(kernel_width - subkernel_x, decomp_w) |
Tim Hall | 79d07d2 | 2020-04-27 18:20:16 +0100 | [diff] [blame] | 134 | subkernel_elements = sub_width * sub_height |
| 135 | # Part kernel first works across the kernel H/W and needs padding |
| 136 | if is_partkernel: |
| 137 | if ifm_bitdepth == 16 and subkernel_elements % 2 != 0: |
| 138 | subkernel_elements = int(math.ceil(subkernel_elements / 2) * 2) |
| 139 | elif ifm_bitdepth == 8 and subkernel_elements % 4 != 0: |
| 140 | subkernel_elements = int(math.ceil(subkernel_elements / 4) * 4) |
| 141 | |
| 142 | # Depthwise Conv requires multiple of 4 kernel elements in its weight block |
| 143 | # this is different from normal conv which is considered "weights depth-first" |
| 144 | elif is_depthwise: |
| 145 | subkernel_elements = int(math.ceil(subkernel_elements / 4.0) * 4) |
| 146 | |
| 147 | ifm_block_depth_outer = clipped_ifm_block_depth if is_partkernel else 1 |
| 148 | ifm_block_depth_inner = 1 if is_partkernel else clipped_ifm_block_depth |
| 149 | # IFM Ublocks in IFM-block over depth for part-kernel-first mode |
| 150 | # For depth-first IFM Ublocks are traversed after subkernel elements so this loop is ignored. |
| 151 | for ifm_ublk_outer in range(0, ifm_block_depth_outer, ifm_ublock.depth): |
| 152 | # OFM Ublocks in OFM-block over depth |
| 153 | for ofm_ublk in range(0, clipped_ofm_block_depth, ofm_ublock.depth): |
| 154 | # HW Kernel element traversal - cannot be a H/W loop due to element |
| 155 | # padding requirement on depthwise/part-kernel configurations |
| 156 | for element in range(subkernel_elements): |
| 157 | kx = element % sub_width |
| 158 | ky = element // sub_width |
| 159 | # IFM Ublocks in IFM-block over depth (only 1 ublock if depthwise) |
| 160 | # In case of part-kernel-first IFM Ublock traversal have already been handled |
| 161 | # and this loop is ignored. |
| 162 | for ifm_ublk_inner in range(0, ifm_block_depth_inner, ifm_ublock.depth): |
| 163 | # Feed OFM ublock elements |
| 164 | for ofm_ublock_z in range(ofm_ublock.depth): |
| 165 | # Source IFM ublock elements (only 1 element deep if depthwise) |
| 166 | for ifm_ublock_z in range(1 if is_depthwise else ifm_ublock.depth): |
| 167 | # Source position within the current subkernel |
| 168 | wx = subkernel_x + kx |
| 169 | wy = subkernel_y + ky |
| 170 | # Source IFM/OFM slices |
| 171 | ifm_ublk = ifm_ublk_inner + ifm_ublk_outer |
| 172 | ifm_z = ifm_block_z + ifm_ublk + ifm_ublock_z |
| 173 | ofm_z = ofm_block_z + ofm_ublk + ofm_ublock_z |
| 174 | if (ifm_z >= ifm_depth) or (ofm_z >= ofm_depth) or (ky >= sub_height): |
| 175 | stream.append(0) |
| 176 | else: |
Tim Hall | f7e810a | 2020-06-25 15:04:31 +0100 | [diff] [blame^] | 177 | stream.append(brick_weights[ofm_z][wy][wx][ifm_z]) |
Tim Hall | 79d07d2 | 2020-04-27 18:20:16 +0100 | [diff] [blame] | 178 | return stream |
| 179 | |
Tim Hall | f7e810a | 2020-06-25 15:04:31 +0100 | [diff] [blame^] | 180 | def core_deinterleave(hwio, core, ncores): |
| 181 | # Put weights back into OHWI |
| 182 | ohwi = np.transpose(hwio, (3,0,1,2)) |
| 183 | return ohwi[core:ohwi.shape[0]:ncores] |
Tim Hall | 79d07d2 | 2020-04-27 18:20:16 +0100 | [diff] [blame] | 184 | |
| 185 | # Compress the weights |
Louis Verhaard | b2fb212 | 2020-06-04 15:51:24 +0200 | [diff] [blame] | 186 | def compress_weights(arch, nng, tens, npu_block_type, ofm_block_depth, ofm_depth_step, dilation): |
Tim Hall | 79d07d2 | 2020-04-27 18:20:16 +0100 | [diff] [blame] | 187 | assert tens.purpose == TensorPurpose.Weights |
| 188 | assert tens.format == TensorFormat.WeightsCompressed |
| 189 | |
Louis Verhaard | 3c07c97 | 2020-05-07 08:12:58 +0200 | [diff] [blame] | 190 | # Check the weight cache |
| 191 | if nng.weight_cache is None: |
| 192 | nng.weight_cache = CompressedWeightCache() |
Louis Verhaard | b2fb212 | 2020-06-04 15:51:24 +0200 | [diff] [blame] | 193 | wcc = create_weight_compression_config(tens, npu_block_type, ofm_block_depth, ofm_depth_step, dilation) |
Louis Verhaard | 3c07c97 | 2020-05-07 08:12:58 +0200 | [diff] [blame] | 194 | tens.weight_compression_config = wcc |
| 195 | tens_cached = nng.weight_cache.get_tensor_with_same_compression(wcc) |
| 196 | if tens_cached is not None: |
| 197 | # Cache hit, copy weights from the cache |
| 198 | tens.copy_compressed_weight_info(tens_cached) |
| 199 | set_storage_shape(tens) |
| 200 | return |
Tim Hall | 79d07d2 | 2020-04-27 18:20:16 +0100 | [diff] [blame] | 201 | |
Louis Verhaard | 3c07c97 | 2020-05-07 08:12:58 +0200 | [diff] [blame] | 202 | # No cache hit, perform the compression |
Tim Hall | 79d07d2 | 2020-04-27 18:20:16 +0100 | [diff] [blame] | 203 | assert tens.quantization is not None |
| 204 | assert tens.quantization.scale_f32 is not None |
| 205 | assert tens.quantization.zero_point is not None |
| 206 | |
| 207 | zero_point = tens.quantization.zero_point |
| 208 | quant_buf = tens.quant_values.astype(np.int64) |
| 209 | |
| 210 | # Early zero-point correction |
| 211 | weights = quant_buf - zero_point |
| 212 | |
| 213 | if len(weights.shape) == 2: |
| 214 | weights = np.expand_dims(np.expand_dims(weights, axis=0), axis=0) |
| 215 | weights_shape = (weights.shape[0], 1, 1, weights.shape[1]) |
| 216 | else: |
| 217 | weights_shape = weights.shape |
| 218 | |
| 219 | compression_scales = [] |
| 220 | compressed_offsets = [] |
| 221 | encoded_streams = [] |
Tim Hall | f7e810a | 2020-06-25 15:04:31 +0100 | [diff] [blame^] | 222 | encoded_streams_substream_offsets = [] |
Tim Hall | 79d07d2 | 2020-04-27 18:20:16 +0100 | [diff] [blame] | 223 | offset = 0 |
Tim Hall | f7e810a | 2020-06-25 15:04:31 +0100 | [diff] [blame^] | 224 | max_single_buffer_len = 0 |
Tim Hall | 79d07d2 | 2020-04-27 18:20:16 +0100 | [diff] [blame] | 225 | |
| 226 | ifm_bitdepth = tens.consumer_list[0].inputs[0].dtype.size_in_bits() |
| 227 | ifm_depth = weights.shape[-2] |
| 228 | if npu_block_type == NpuBlockType.ConvolutionDepthWise: |
| 229 | tens.block_traversal = TensorBlockTraversal.DepthWise |
| 230 | if npu_block_type == NpuBlockType.ConvolutionMxN: |
| 231 | # Determine which block traversal strategy has better DPU utilization |
| 232 | kernel_size = weights_shape[0] * weights_shape[1] |
| 233 | depth_utilization = weights_shape[2] / round_up(weights_shape[2], 32 if ifm_bitdepth == 8 else 16) |
| 234 | part_kernel_utilization = (weights_shape[2] / round_up(weights_shape[2], 8)) * ( |
| 235 | kernel_size / round_up(kernel_size, 4 if ifm_bitdepth == 8 else 2) |
| 236 | ) |
| 237 | if part_kernel_utilization >= depth_utilization or ifm_depth <= 8: |
| 238 | # Part-kernel first is always better for ifm depths <= 8 |
| 239 | tens.block_traversal = TensorBlockTraversal.PartKernelFirst |
| 240 | else: |
| 241 | tens.block_traversal = TensorBlockTraversal.DepthFirst |
| 242 | |
Jacob Bohlin | cf7da10 | 2020-05-20 09:03:40 +0200 | [diff] [blame] | 243 | if tens.consumer_list[0].type == "Conv2DBackpropInputSwitchedBias": |
| 244 | # Transpose Convoluion, reverse weights in H and W axes |
Tim Hall | c30f495 | 2020-06-15 20:47:35 +0100 | [diff] [blame] | 245 | weights = np.flip(weights, axis=(0, 1)) |
Jacob Bohlin | cf7da10 | 2020-05-20 09:03:40 +0200 | [diff] [blame] | 246 | |
Tim Hall | 79d07d2 | 2020-04-27 18:20:16 +0100 | [diff] [blame] | 247 | # Slice weight stream up depth-ways into bricks and compress |
| 248 | full_ofm_depth = quant_buf.shape[-1] |
Tim Hall | f7e810a | 2020-06-25 15:04:31 +0100 | [diff] [blame^] | 249 | ofm_block_depth = ofm_block_depth // arch.ncores |
Tim Hall | 79d07d2 | 2020-04-27 18:20:16 +0100 | [diff] [blame] | 250 | for idx in range(0, full_ofm_depth, ofm_depth_step): |
| 251 | # Get the weights necessary for this brick |
| 252 | count = min(full_ofm_depth - idx, ofm_depth_step) |
| 253 | brick_weights = weights[:, :, :, idx : idx + count] |
| 254 | |
Tim Hall | f7e810a | 2020-06-25 15:04:31 +0100 | [diff] [blame^] | 255 | substream_offsets = [0] |
| 256 | encoded_stream = [] |
| 257 | raw_size = 0 |
| 258 | |
| 259 | # For each core, deinterleave weights from the larger volume |
| 260 | # and generate separate compressed streams. |
| 261 | for core in range(0, min(arch.ncores, full_ofm_depth)): |
| 262 | core_weights = core_deinterleave(brick_weights, core, arch.ncores) |
| 263 | raw_stream = generate_brick(arch, core_weights, ofm_block_depth, tens.block_traversal, ifm_bitdepth, dilation) |
| 264 | raw_size += len( raw_stream ) |
| 265 | encoded_substream = encode( raw_stream ) |
| 266 | encoded_stream.extend( encoded_substream ) |
| 267 | substream_offsets.append( len(encoded_stream) ) |
| 268 | |
| 269 | encoded_streams.append( encoded_stream ) |
| 270 | encoded_streams_substream_offsets.append( substream_offsets ) |
| 271 | |
| 272 | # Remember maximum encoded length for DoubleBuffering |
| 273 | max_single_buffer_len = max(max_single_buffer_len, len(encoded_stream)) |
Tim Hall | 79d07d2 | 2020-04-27 18:20:16 +0100 | [diff] [blame] | 274 | |
Tim Hall | 79d07d2 | 2020-04-27 18:20:16 +0100 | [diff] [blame] | 275 | # Remember where we put it for linear addressing |
| 276 | compressed_offsets.append(offset) |
Tim Hall | f7e810a | 2020-06-25 15:04:31 +0100 | [diff] [blame^] | 277 | offset += len(encoded_stream) |
Tim Hall | 79d07d2 | 2020-04-27 18:20:16 +0100 | [diff] [blame] | 278 | assert offset % 16 == 0 |
| 279 | |
| 280 | # Compression scale tracking |
Tim Hall | f7e810a | 2020-06-25 15:04:31 +0100 | [diff] [blame^] | 281 | compression_scales.append(len(encoded_stream) / raw_size) |
Tim Hall | 79d07d2 | 2020-04-27 18:20:16 +0100 | [diff] [blame] | 282 | |
Tim Hall | f7e810a | 2020-06-25 15:04:31 +0100 | [diff] [blame^] | 283 | # Track total length as last element of the offsets array |
Tim Hall | 79d07d2 | 2020-04-27 18:20:16 +0100 | [diff] [blame] | 284 | compressed_offsets.append(offset) |
| 285 | |
Tim Hall | 79d07d2 | 2020-04-27 18:20:16 +0100 | [diff] [blame] | 286 | tens.weight_compression_scales = compression_scales |
Tim Hall | 79d07d2 | 2020-04-27 18:20:16 +0100 | [diff] [blame] | 287 | tens.weight_compressed_offsets = compressed_offsets |
| 288 | tens.compression_scale_for_worst_weight_stream = np.amax(compression_scales) |
| 289 | tens.storage_compression_scale = tens.bandwidth_compression_scale = np.average(compression_scales) |
| 290 | tens.compressed_values = encoded_streams |
Tim Hall | f7e810a | 2020-06-25 15:04:31 +0100 | [diff] [blame^] | 291 | tens.compressed_values_substream_offsets = encoded_streams_substream_offsets |
Tim Hall | 79d07d2 | 2020-04-27 18:20:16 +0100 | [diff] [blame] | 292 | tens.brick_size = (weights_shape[0], weights_shape[1], weights_shape[2], min(tens.shape[-1], ofm_depth_step)) |
Louis Verhaard | 3c07c97 | 2020-05-07 08:12:58 +0200 | [diff] [blame] | 293 | set_storage_shape(tens) |
| 294 | nng.weight_cache.add(tens) |
Tim Hall | 79d07d2 | 2020-04-27 18:20:16 +0100 | [diff] [blame] | 295 | |
Tim Hall | f7e810a | 2020-06-25 15:04:31 +0100 | [diff] [blame^] | 296 | def calc_scales_and_pack_biases(tens, arch, ofm_depth_step, rescale_for_faf=False): |
Tim Hall | 79d07d2 | 2020-04-27 18:20:16 +0100 | [diff] [blame] | 297 | assert tens.purpose == TensorPurpose.FeatureMap |
| 298 | assert tens.format == TensorFormat.NHWC |
| 299 | # the connected operator should expect a bias input unless it is a FullyConnected |
| 300 | assert "Bias" in tens.consumer_list[0].type or tens.consumer_list[0].type.startswith("FullyConnected") |
| 301 | # the input bias tensor is the same as that connected to the operator |
Jacob Bohlin | cf7da10 | 2020-05-20 09:03:40 +0200 | [diff] [blame] | 302 | _, _, bias_tens, _ = tens.consumer_list[0].get_ifm_weights_biases_ofm() |
| 303 | assert tens is bias_tens |
| 304 | |
Tim Hall | 79d07d2 | 2020-04-27 18:20:16 +0100 | [diff] [blame] | 305 | # the operator should only have a single output |
| 306 | assert len(tens.consumer_list[0].outputs) == 1 |
| 307 | |
| 308 | def pack_bias_and_scale(bias, scale, shift): |
| 309 | bias = np.int64(bias) |
| 310 | assert -(1 << (40 - 1)) <= bias < (1 << (40 - 1)) # signed 40-bit range |
| 311 | assert 0 <= scale < (1 << 32) # unsigned 32-bit range |
| 312 | assert 0 <= shift < (1 << 6) # unsigned 6-bit range |
| 313 | |
| 314 | # pack the 80 bit value = [0(2-bits),shift(6-bits),scale(32-bits),bias(40-bits)] |
| 315 | data = bytearray(10) |
| 316 | data[0] = (bias >> (0 * 8)) & 0xFF |
| 317 | data[1] = (bias >> (1 * 8)) & 0xFF |
| 318 | data[2] = (bias >> (2 * 8)) & 0xFF |
| 319 | data[3] = (bias >> (3 * 8)) & 0xFF |
| 320 | data[4] = (bias >> (4 * 8)) & 0xFF |
| 321 | data[5] = (scale >> (0 * 8)) & 0xFF |
| 322 | data[6] = (scale >> (1 * 8)) & 0xFF |
| 323 | data[7] = (scale >> (2 * 8)) & 0xFF |
| 324 | data[8] = (scale >> (3 * 8)) & 0xFF |
| 325 | data[9] = shift & 0x3F |
| 326 | return data |
| 327 | |
| 328 | biases = tens.quant_values |
| 329 | |
| 330 | first_consumer_op = tens.consumer_list[0] |
| 331 | ifm_dtype = first_consumer_op.inputs[0].dtype |
| 332 | ifm_scale = first_consumer_op.inputs[0].quantization.scale_f32 |
| 333 | ofm_scale = first_consumer_op.outputs[0].quantization.scale_f32 |
| 334 | weight_scales = first_consumer_op.inputs[1].quantization.scale_f32 |
| 335 | |
| 336 | # biases can have multiple consumers for rnn cells. if so, then check that they are all the same |
| 337 | for op in tens.consumer_list[1:]: |
| 338 | assert ifm_scale == op.inputs[0].quantization.scale_f32 |
| 339 | assert ofm_scale == op.outputs[0].quantization.scale_f32 |
| 340 | assert weight_scales == op.inputs[1].quantization.scale_f32 |
| 341 | |
| 342 | if not hasattr(weight_scales, "__iter__"): |
| 343 | # If weight_scales is not already an iterable make it into a list |
| 344 | weight_scales = [weight_scales] |
| 345 | |
| 346 | # Convert scales to np.double (from np.float32) to conform to TensorFlow Lite which |
| 347 | # uses double during scaling calculations |
| 348 | # TensorFlow Lite casts the scales slightly differently for uint8 and int8 |
| 349 | if not rescale_for_faf: |
| 350 | if ifm_dtype == DataType.uint8: |
| 351 | scales = [np.double(ifm_scale * weight_scale) / np.double(ofm_scale) for weight_scale in weight_scales] |
Fredrik Svedberg | d67c0aa | 2020-03-30 13:15:28 +0200 | [diff] [blame] | 352 | elif ifm_dtype == DataType.int8 or ifm_dtype == DataType.int16: |
Tim Hall | 79d07d2 | 2020-04-27 18:20:16 +0100 | [diff] [blame] | 353 | scales = [ |
| 354 | (np.double(ifm_scale) * np.double(weight_scale)) / np.double(ofm_scale) |
| 355 | for weight_scale in weight_scales |
| 356 | ] |
| 357 | else: |
Louis Verhaard | 7db7896 | 2020-05-25 15:05:26 +0200 | [diff] [blame] | 358 | raise UnsupportedFeatureError( |
| 359 | "Compression of {} is not implemented; tensor: {}".format(ifm_dtype, tens.name) |
| 360 | ) |
Tim Hall | 79d07d2 | 2020-04-27 18:20:16 +0100 | [diff] [blame] | 361 | else: |
| 362 | if ifm_dtype == DataType.uint8: |
| 363 | scales = [np.double(ifm_scale * weight_scale * 0x3000) for weight_scale in weight_scales] |
Fredrik Svedberg | d67c0aa | 2020-03-30 13:15:28 +0200 | [diff] [blame] | 364 | elif ifm_dtype == DataType.int8 or ifm_dtype == DataType.int16: |
Tim Hall | 79d07d2 | 2020-04-27 18:20:16 +0100 | [diff] [blame] | 365 | scales = [(np.double(ifm_scale * 0x3000) * np.double(weight_scale)) for weight_scale in weight_scales] |
| 366 | else: |
Louis Verhaard | 7db7896 | 2020-05-25 15:05:26 +0200 | [diff] [blame] | 367 | raise UnsupportedFeatureError( |
| 368 | "Compression of {} is not implemented; tensor: {}".format(ifm_dtype, tens.name) |
| 369 | ) |
Tim Hall | 79d07d2 | 2020-04-27 18:20:16 +0100 | [diff] [blame] | 370 | |
| 371 | # quantise all of the weight scales into (scale_factor, shift) |
Fredrik Svedberg | d67c0aa | 2020-03-30 13:15:28 +0200 | [diff] [blame] | 372 | if ifm_dtype == DataType.int16: |
| 373 | quantised_scales = [reduced_quantise_scale(scale) for scale in scales] |
| 374 | else: |
| 375 | quantised_scales = [quantise_scale(scale) for scale in scales] |
Tim Hall | 79d07d2 | 2020-04-27 18:20:16 +0100 | [diff] [blame] | 376 | |
| 377 | for _, shift in quantised_scales: |
| 378 | assert shift >= 16 |
| 379 | |
| 380 | # pack the biases and scales |
Tim Hall | 79d07d2 | 2020-04-27 18:20:16 +0100 | [diff] [blame] | 381 | if len(quantised_scales) == 1: |
| 382 | # If only 1 quantised scale is used, repeat that value for the length of the biases |
| 383 | quantised_scales = [quantised_scales[0]] * len(biases) |
| 384 | |
| 385 | assert len(quantised_scales) == len(biases) |
Tim Hall | 79d07d2 | 2020-04-27 18:20:16 +0100 | [diff] [blame] | 386 | tens.element_size_bytes = 10 |
Tim Hall | f7e810a | 2020-06-25 15:04:31 +0100 | [diff] [blame^] | 387 | tens.compressed_values = [] |
| 388 | tens.compressed_values_substream_offsets = [] |
Tim Hall | 79d07d2 | 2020-04-27 18:20:16 +0100 | [diff] [blame] | 389 | |
Tim Hall | f7e810a | 2020-06-25 15:04:31 +0100 | [diff] [blame^] | 390 | total_elements = len(quantised_scales) |
| 391 | for i in range(0, total_elements, ofm_depth_step): |
| 392 | # Extract streams from brick to generate substreams for each core |
| 393 | stream = bytearray() |
| 394 | substream_offsets = [0] |
| 395 | max_len = min(ofm_depth_step, total_elements - i) |
| 396 | for core in range(0, min(arch.ncores, max_len)): |
| 397 | core_scales = quantised_scales[i+core:i+core+max_len:arch.ncores] |
| 398 | core_biases = biases[i+core:i+core+max_len:arch.ncores] |
| 399 | for j, core_bias in enumerate(core_biases): |
| 400 | stream.extend( pack_bias_and_scale(core_bias, *core_scales[j]) ) |
Tim Hall | 79d07d2 | 2020-04-27 18:20:16 +0100 | [diff] [blame] | 401 | |
Tim Hall | f7e810a | 2020-06-25 15:04:31 +0100 | [diff] [blame^] | 402 | # Align to 16 for start for next substream |
| 403 | remainder = ( len(stream) ) % 16 |
| 404 | if remainder > 0: |
| 405 | stream.extend( bytearray(16 - remainder) ) |
Tim Hall | 79d07d2 | 2020-04-27 18:20:16 +0100 | [diff] [blame] | 406 | |
Tim Hall | f7e810a | 2020-06-25 15:04:31 +0100 | [diff] [blame^] | 407 | substream_offsets.append( len(stream) ) |
Tim Hall | 79d07d2 | 2020-04-27 18:20:16 +0100 | [diff] [blame] | 408 | |
Tim Hall | f7e810a | 2020-06-25 15:04:31 +0100 | [diff] [blame^] | 409 | # Add to compressed values with their substream offset lists to the tensor |
| 410 | tens.compressed_values.append( stream ) |
| 411 | tens.compressed_values_substream_offsets.append( substream_offsets ) |
| 412 | |
| 413 | tens.storage_shape = [total_elements * tens.element_size_bytes] |
Tim Hall | 79d07d2 | 2020-04-27 18:20:16 +0100 | [diff] [blame] | 414 | |
| 415 | def update_pass_weight_and_scale_tensors(nng, arch): |
Tim Hall | 79d07d2 | 2020-04-27 18:20:16 +0100 | [diff] [blame] | 416 | for sg in nng.subgraphs: |
| 417 | for ps in sg.passes: |
Louis Verhaard | 3c07c97 | 2020-05-07 08:12:58 +0200 | [diff] [blame] | 418 | tens = ps.weight_tensor |
| 419 | if tens is not None: |
Louis Verhaard | b2fb212 | 2020-06-04 15:51:24 +0200 | [diff] [blame] | 420 | op = tens.find_npu_op() |
| 421 | npu_usage_of_tensor = op.attrs["npu_block_type"] |
Tim Hall | 79d07d2 | 2020-04-27 18:20:16 +0100 | [diff] [blame] | 422 | if npu_usage_of_tensor == NpuBlockType.ConvolutionDepthWise: |
Louis Verhaard | 3c07c97 | 2020-05-07 08:12:58 +0200 | [diff] [blame] | 423 | tens.quant_values = np.transpose(tens.quant_values, (0, 1, 3, 2)) |
| 424 | tens.shape = tens.storage_shape = tens.bandwidth_shape = list(tens.quant_values.shape) |
| 425 | tens.weight_transpose_depthwise = True |
Tim Hall | 79d07d2 | 2020-04-27 18:20:16 +0100 | [diff] [blame] | 426 | |
Louis Verhaard | 3c07c97 | 2020-05-07 08:12:58 +0200 | [diff] [blame] | 427 | needs_dma = tens.needs_dma() |
Tim Hall | 79d07d2 | 2020-04-27 18:20:16 +0100 | [diff] [blame] | 428 | if ps.cascade.strategy == SchedulingStrategy.WeightStream and needs_dma: |
| 429 | ofm_depth_step = ps.block_config[-1] |
| 430 | else: |
Louis Verhaard | 3c07c97 | 2020-05-07 08:12:58 +0200 | [diff] [blame] | 431 | ofm_depth_step = tens.shape[-1] |
Tim Hall | 79d07d2 | 2020-04-27 18:20:16 +0100 | [diff] [blame] | 432 | compress_weights( |
Louis Verhaard | b2fb212 | 2020-06-04 15:51:24 +0200 | [diff] [blame] | 433 | arch, nng, tens, npu_usage_of_tensor, ps.block_config[-1], ofm_depth_step, op.get_dilation_h_w() |
Tim Hall | 79d07d2 | 2020-04-27 18:20:16 +0100 | [diff] [blame] | 434 | ) |
| 435 | # Update source tensor |
Louis Verhaard | 3c07c97 | 2020-05-07 08:12:58 +0200 | [diff] [blame] | 436 | if needs_dma: |
| 437 | src_tens = tens.get_dma_src_tensor() |
| 438 | src_tens.shape = tens.shape |
| 439 | src_tens.quant_values = tens.quant_values |
| 440 | src_tens.copy_compressed_weight_info(tens) |
| 441 | set_storage_shape(src_tens) |
Tim Hall | 79d07d2 | 2020-04-27 18:20:16 +0100 | [diff] [blame] | 442 | |
Diego Russo | ea6111a | 2020-04-14 18:41:58 +0100 | [diff] [blame] | 443 | if ps.scale_tensor is not None: |
Tim Hall | 79d07d2 | 2020-04-27 18:20:16 +0100 | [diff] [blame] | 444 | rescale_for_faf = False |
| 445 | activation_ops = set(("Sigmoid", "Tanh")) |
| 446 | if (ps.ops[-1].type in activation_ops) and (ps.npu_block_type != NpuBlockType.ElementWise): |
| 447 | rescale_for_faf = True |
Tim Hall | f7e810a | 2020-06-25 15:04:31 +0100 | [diff] [blame^] | 448 | calc_scales_and_pack_biases(ps.scale_tensor, arch, ofm_depth_step, rescale_for_faf) |