verif/frameworks/tosa_verif_framework_generator.py

#!/usr/bin/env python3
# Copyright (c) 2020-2022, ARM Limited.
# SPDX-License-Identifier: Apache-2.0
import argparse
import os
import re
import traceback

import numpy as np

#  Level | Level for Humans | Level Description
# -------|------------------|------------------------------------
#  0     | DEBUG            | [Default] Print all messages
#  1     | INFO             | Filter out INFO messages
#  2     | WARNING          | Filter out INFO & WARNING messages
#  3     | ERROR            | Filter out all messages
# Filter tensorflow debug message except errors
os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2"

# Flake8 E402 - ignore imports not at top of file to allow os.environ setting
import tensorflow as tf  # noqa: E402
from frameworks.write_test_json import write_test_json  # noqa: E402
from frameworks.arg_gen import ArgGen  # noqa: E402
from frameworks.tensor_gen import TGen  # noqa: E402
from frameworks.test_builder import TBuilder  # noqa: E402
from frameworks.test_gen_utils import (  # noqa: E402
    QuantType,
    get_tf_dtype,
    get_shape_str,
)  # noqa: E402
from tensorflow.lite.python.interpreter import OpResolverType  # noqa: E402

# All of the supported frameworks
ALL_FRAMEWORKS = ["tf", "tflite"]

# Lists of different data types
TYPE_F = [tf.float32]
TYPE_I = [tf.int32]
TYPE_FI = [tf.float32, tf.int32]
TYPE_B = [tf.bool]
TYPE_FIB = [tf.float32, tf.int32, tf.bool]
TYPE_H = [tf.float16]
TYPE_FH = [tf.float32, tf.float16]
TYPE_FHI = [tf.float32, tf.float16, tf.int32]
TYPE_FHIB = [tf.float32, tf.float16, tf.int32, tf.bool]

# The list of operator tests
# Each dictionary entry for an op is a dictionary with the following required members:
#   'operands': tuple (number_of_placeholder_tensors, number_of_constant_tensors)
#   'build_fcn: tuple (Test builder function, Tensor generator function,
#                      Argument generator function)
#   'types': list of Tensorflow types that should be tested for this op
#               OR
#            a dictionary of {'framework_name': [type_list] } for cases where only
#            a subset of the types should be tested in each framework.  This can also
#            be used to restrict an operator to a particular framework.
#
# And optional members:
#   'template': boolean (indicates that this is a templated op which gets further
#               processing in createDynamicOpLists)
#   'bias':     boolean indicating that there is a bias component to be generated
#   'qtypes':   List of QuantType quantized types to generate for this op
#   'rank':     tuple (lowest rank, highest rank). Dimension range of input tensor.

TF_OP_LIST = {
    "add": {
        "operands": (2, 0),
        "build_fcn": (TBuilder.Add, TGen.tgBFuzz, ArgGen.agNone),
        "types": {
            "tf": TYPE_FI,
            "tflite": list(
                TYPE_FI + [QuantType.ALL_U8, QuantType.ALL_I8, QuantType.ALL_I16]
            ),
        },
    },
    "sub": {
        "operands": (2, 0),
        "build_fcn": (TBuilder.Sub, TGen.tgBFuzz, ArgGen.agNone),
        "types": {
            "tf": TYPE_FI,
            "tflite": list(TYPE_FI + [QuantType.ALL_U8, QuantType.ALL_I8]),
            # QuantType.ALL_I16 fail in TFLite conversion
        },
    },
    "mul": {
        "operands": (2, 0),
        "build_fcn": (TBuilder.Mul, TGen.tgBFuzz, ArgGen.agNone),
        "types": {
            "tf": TYPE_FI,
            "tflite": list(
                TYPE_FI + [QuantType.ALL_U8, QuantType.ALL_I8, QuantType.ALL_I16]
            ),
        },
    },
    "exp": {
        "operands": (1, 0),
        "build_fcn": (TBuilder.Exp, TGen.tgBasic, ArgGen.agNone),
        "types": TYPE_F,
    },
    "rcp": {
        "operands": (1, 0),
        "build_fcn": (TBuilder.Rcp, TGen.tgBasic, ArgGen.agNone),
        "types": TYPE_F,
    },
    "relu": {
        "operands": (1, 0),
        "build_fcn": (TBuilder.Relu, TGen.tgBasic, ArgGen.agNone),
        "types": {
            "tf": TYPE_F,
            "tflite": list(
                TYPE_F + [QuantType.ALL_U8, QuantType.ALL_I8, QuantType.ALL_I16]
            ),
        },
    },
    "relu1": {
        "operands": (1, 0),
        "build_fcn": (TBuilder.Relu1, TGen.tgBasic, ArgGen.agNone),
        "types": {
            "tf": [],
            "tflite": list(TYPE_F + [QuantType.ALL_U8, QuantType.ALL_I8]),
        },
    },
    "relu6": {
        "operands": (1, 0),
        "build_fcn": (TBuilder.Relu6, TGen.tgBasic, ArgGen.agNone),
        "types": {
            "tf": TYPE_F,
            "tflite": list(
                TYPE_F + [QuantType.ALL_U8, QuantType.ALL_I8, QuantType.ALL_I16]
            ),
        },
    },
    "leaky_relu": {
        "operands": (1, 0),
        "build_fcn": (TBuilder.LeakyRelu, TGen.tgBasic, ArgGen.agFloat),
        "types": {
            "tf": TYPE_F,
            "tflite": list(
                TYPE_F + [QuantType.ALL_U8, QuantType.ALL_I8, QuantType.ALL_I16]
            ),
        },
    },
    "prelu": {
        "operands": (1, 0),
        "build_fcn": (TBuilder.Prelu, TGen.tgBasic, ArgGen.agNone),
        "types": {
            "tflite": list(TYPE_F + [QuantType.ALL_U8, QuantType.ALL_I8]),
        },
    },
    "gelu": {
        "operands": (1, 0),
        "build_fcn": (TBuilder.Gelu, TGen.tgBasic, ArgGen.agNone),
        "types": {
            # Need compiler support for tf.Erf.
            # "tf": TYPE_F,
            "tflite": list(
                # Only float32, int8 and uint8 supported currently
                TYPE_F
                + [QuantType.ALL_U8, QuantType.ALL_I8]
            ),
        },
    },
    "concat": {
        "operands": (2, 0),
        "build_fcn": (TBuilder.Concat, TGen.tgBasic, ArgGen.agAxes),
        "types": TYPE_FI,
    },
    "bitwise_and": {
        "operands": (2, 0),
        "build_fcn": (TBuilder.BitwiseAnd, TGen.tgBFuzz, ArgGen.agNone),
        "types": {"tf": TYPE_I},  # Not supported in TF Lite
    },
    "bitwise_or": {
        "operands": (2, 0),
        "build_fcn": (TBuilder.BitwiseOr, TGen.tgBFuzz, ArgGen.agNone),
        "types": {"tf": TYPE_I},  # Not supported in TF Lite
    },
    "bitwise_not": {
        "operands": (1, 0),
        "build_fcn": (TBuilder.BitwiseNot, TGen.tgBFuzz, ArgGen.agNone),
        "types": {"tf": TYPE_I},  # Not supported in TF Lite
    },
    "bitwise_xor": {
        "operands": (2, 0),
        "build_fcn": (TBuilder.BitwiseXor, TGen.tgBFuzz, ArgGen.agNone),
        "types": {"tf": TYPE_I},  # Not supported in TF Lite
    },
    "logical_and": {
        "operands": (2, 0),
        "build_fcn": (TBuilder.LogicalAnd, TGen.tgBFuzz, ArgGen.agNone),
        "types": TYPE_B,
    },
    "logical_or": {
        "operands": (2, 0),
        "build_fcn": (TBuilder.LogicalOr, TGen.tgBFuzz, ArgGen.agNone),
        "types": TYPE_B,
    },
    "logical_not": {
        "operands": (1, 0),
        "build_fcn": (TBuilder.LogicalNot, TGen.tgBFuzz, ArgGen.agNone),
        "types": TYPE_B,
    },
    "reduce_any": {
        "operands": (1, 0),
        "build_fcn": (TBuilder.ReduceAny, TGen.tgBasic, ArgGen.agAxesListKeepdims),
        "types": TYPE_B,
    },
    "reduce_all": {
        "operands": (1, 0),
        "build_fcn": (TBuilder.ReduceAll, TGen.tgBasic, ArgGen.agAxesListKeepdims),
        "types": {"tf": TYPE_B},
    },
    "reduce_min": {
        "operands": (1, 0),
        "build_fcn": (TBuilder.ReduceMin, TGen.tgBasic, ArgGen.agAxesListKeepdims),
        "types": {
            "tf": TYPE_FI,
            "tflite": list(TYPE_FI + [QuantType.ALL_U8, QuantType.ALL_I8]),
        },
    },
    "reduce_max": {
        "operands": (1, 0),
        "build_fcn": (TBuilder.ReduceMax, TGen.tgBasic, ArgGen.agAxesListKeepdims),
        "types": {
            "tf": TYPE_FI,
            "tflite": list(TYPE_FI + [QuantType.ALL_U8, QuantType.ALL_I8]),
        },
    },
    "reduce_sum": {
        "operands": (1, 0),
        "build_fcn": (TBuilder.ReduceSum, TGen.tgBasic, ArgGen.agAxesListKeepdims),
        "types": {
            "tf": TYPE_F,
            # v2 converter doesn't recognize quantized reduce_sum
            # "tflite": list(TYPE_F + [QuantType.ALL_U8, QuantType.ALL_I8]),
            "tflite": TYPE_F,
        },
    },
    "reduce_mean": {
        "operands": (1, 0),
        "build_fcn": (TBuilder.ReduceMean, TGen.tgBasic, ArgGen.agAxesListKeepdims),
        "types": {
            "tf": TYPE_F,
            "tflite": list(
                TYPE_F + [QuantType.ALL_U8, QuantType.ALL_I8, QuantType.ALL_I16]
            ),
        },
    },
    "reduce_product": {
        "operands": (1, 0),
        "build_fcn": (TBuilder.ReduceProduct, TGen.tgBasic, ArgGen.agAxesListKeepdims),
        "types": TYPE_F,
    },
    "min": {
        "operands": (2, 0),
        "build_fcn": (TBuilder.Min, TGen.tgBFuzz, ArgGen.agNone),
        "types": TYPE_FI,
    },
    "max": {
        "operands": (2, 0),
        "build_fcn": (TBuilder.Max, TGen.tgBFuzz, ArgGen.agNone),
        "types": TYPE_FI,
    },
    "pow": {
        "operands": (2, 0),
        "build_fcn": (TBuilder.Pow, TGen.tgBFuzz, ArgGen.agNone),
        # Technically, integer is supported, but only for positive exponents.
        # Needs a random argument generator.
        "types": TYPE_F,
    },
    "abs": {
        "operands": (1, 0),
        "build_fcn": (TBuilder.Abs, TGen.tgBasic, ArgGen.agNone),
        "types": TYPE_F,
    },
    "ceil": {
        "operands": (1, 0),
        "build_fcn": (TBuilder.Ceil, TGen.tgBasic, ArgGen.agNone),
        "types": TYPE_F,
    },
    "floor": {
        "operands": (1, 0),
        "build_fcn": (TBuilder.Floor, TGen.tgBasic, ArgGen.agNone),
        "types": TYPE_F,
    },
    "log": {
        "operands": (1, 0),
        "build_fcn": (TBuilder.Log, TGen.tgBasic, ArgGen.agNone),
        "types": TYPE_F,
    },
    "negate": {
        "operands": (1, 0),
        "build_fcn": (TBuilder.Negate, TGen.tgBasic, ArgGen.agNone),
        "types": TYPE_F,
    },
    "rsqrt": {
        "operands": (1, 0),
        "build_fcn": (TBuilder.Rsqrt, TGen.tgBasic, ArgGen.agNone),
        "types": TYPE_F,
    },
    "sigmoid": {
        "operands": (1, 0),
        "build_fcn": (TBuilder.Sigmoid, TGen.tgBasic, ArgGen.agNone),
        "types": {
            "tf": TYPE_F,
            "tflite": list(
                TYPE_F + [QuantType.ALL_U8, QuantType.ALL_I8, QuantType.ALL_I16]
            ),
        },
    },
    "tanh": {
        "operands": (1, 0),
        "build_fcn": (TBuilder.Tanh, TGen.tgBasic, ArgGen.agNone),
        "types": {
            "tf": TYPE_F,
            "tflite": list(
                TYPE_F + [QuantType.ALL_U8, QuantType.ALL_I8, QuantType.ALL_I16]
            ),
        },
    },
    "sin": {
        "operands": (1, 0),
        "build_fcn": (TBuilder.Sin, TGen.tgBasic, ArgGen.agNone),
        "types": {
            "tflite": TYPE_F,
        },
    },
    "cos": {
        "operands": (1, 0),
        "build_fcn": (TBuilder.Cos, TGen.tgBasic, ArgGen.agNone),
        "types": {
            "tflite": TYPE_F,
        },
    },
    "square": {
        "operands": (1, 0),
        "build_fcn": (TBuilder.Square, TGen.tgBasic, ArgGen.agNone),
        "types": TYPE_F,
    },
    "squared_difference": {
        "operands": (2, 0),
        "build_fcn": (TBuilder.SquaredDifference, TGen.tgBFuzz, ArgGen.agNone),
        "types": TYPE_F,
    },
    "equal": {
        "operands": (2, 0),
        "build_fcn": (TBuilder.Equal, TGen.tgBFuzz, ArgGen.agNone),
        "types": TYPE_FI,
    },
    "greater_equal": {
        "operands": (2, 0),
        "build_fcn": (TBuilder.GreaterEqual, TGen.tgBFuzz, ArgGen.agNone),
        "types": TYPE_FI,
    },
    "greater": {
        "operands": (2, 0),
        "build_fcn": (TBuilder.Greater, TGen.tgBFuzz, ArgGen.agNone),
        "types": TYPE_FI,
    },
    "less": {
        "operands": (2, 0),
        "build_fcn": (TBuilder.Less, TGen.tgBFuzz, ArgGen.agNone),
        "types": TYPE_FI,
    },
    "less_equal": {
        "operands": (2, 0),
        "build_fcn": (TBuilder.LessEqual, TGen.tgBFuzz, ArgGen.agNone),
        "types": TYPE_FI,
    },
    "conv2d_TEMPLATE": {
        "operands": (1, 1),
        "build_fcn": (TBuilder.Conv2d, TGen.tgConv2d, ArgGen.agConv2d),
        "types": {
            "tf": [tf.float32],
            "tflite": [
                tf.float32,
                QuantType.CONV_U8_U8,
                QuantType.CONV_I8_I8,
                QuantType.CONV_I16_I8,
            ],
        },
        "template": True,
    },
    "conv2d_relu_TEMPLATE": {
        "operands": (1, 2),
        "build_fcn": (TBuilder.Conv2dRelu, TGen.tgConv2d, ArgGen.agNone),
        "types": {
            "tf": [tf.float32],
            "tflite": [
                tf.float32,
                QuantType.CONV_U8_U8,
                QuantType.CONV_I8_I8,
                QuantType.CONV_I16_I8,
            ],
        },
        "template": True,
    },
    "conv2d_relu6_TEMPLATE": {
        "operands": (1, 2),
        "build_fcn": (TBuilder.Conv2dRelu6, TGen.tgConv2d, ArgGen.agNone),
        "types": {
            "tf": [tf.float32],
            "tflite": [
                tf.float32,
                QuantType.CONV_U8_U8,
                QuantType.CONV_I8_I8,
                QuantType.CONV_I16_I8,
            ],
        },
        "template": True,
    },
    "conv2d_relu_n1_to_1_TEMPLATE": {
        "operands": (1, 2),
        "build_fcn": (TBuilder.Conv2dReluN1To1, TGen.tgConv2d, ArgGen.agNone),
        "types": {
            "tf": [tf.float32],
            "tflite": [
                tf.float32,
                QuantType.CONV_U8_U8,
                QuantType.CONV_I8_I8,
                QuantType.CONV_I16_I8,
            ],
        },
        "template": True,
    },
    # This test is converted as:
    # tfl.conv2d(){fused_activation_function="NONE"} + tfl.tanh()
    # TODO: anyway to generate tfl.conv2d(){fused_activation_function="TANH"}?
    "conv2d_tanh_TEMPLATE": {
        "operands": (1, 2),
        "build_fcn": (TBuilder.Conv2dTanh, TGen.tgConv2d, ArgGen.agNone),
        "types": {
            "tf": [tf.float32],
            "tflite": [
                tf.float32,
                QuantType.CONV_U8_U8,
                QuantType.CONV_I8_I8,
                QuantType.CONV_I16_I8,
            ],
        },
        "template": True,
    },
    "conv2d_bias_TEMPLATE": {
        "operands": (1, 2),
        "build_fcn": (TBuilder.Conv2dWithBias, TGen.tgConv2d, ArgGen.agConv2d),
        "types": {
            "tf": [tf.float32],
            "tflite": [
                tf.float32,
                QuantType.CONV_U8_U8,
                QuantType.CONV_I8_I8,
                QuantType.CONV_I16_I8,
            ],
        },
        "bias": True,
        "template": True,
    },
    "conv3d_TEMPLATE": {
        "operands": (1, 1),
        "build_fcn": (TBuilder.Conv3d, TGen.tgConv3d, ArgGen.agConv3d),
        "types": {
            "tf": [tf.float32],
            "tflite": [
                tf.float32,
                QuantType.CONV_U8_U8,
                QuantType.CONV_I8_I8,
                # Quantization to 16x8-bit not yet supported by tflite.
            ],
        },
        "template": True,
        "rank": (1, 5),
    },
    "conv3d_bias_TEMPLATE": {
        "operands": (1, 2),
        "build_fcn": (TBuilder.Conv3dWithBias, TGen.tgConv3d, ArgGen.agConv3d),
        "types": {
            "tf": [tf.float32],
            "tflite": [
                tf.float32,
                QuantType.CONV_U8_U8,
                QuantType.CONV_I8_I8,
                # Quantization to 16x8-bit not yet supported by tflite.
            ],
        },
        "bias": True,
        "template": True,
        "rank": (1, 5),
    },
    "depthwise_conv2d_TEMPLATE": {
        "operands": (1, 1),
        "build_fcn": (
            TBuilder.DepthwiseConv2d,
            TGen.tgDepthwiseConv2d,
            ArgGen.agDepthwiseConv2d,
        ),
        "types": {
            "tf": [tf.float32],
            "tflite": [
                tf.float32,
                QuantType.CONV_U8_U8,
                QuantType.CONV_I8_I8,
                QuantType.CONV_I16_I8,
            ],
        },
        "template": True,
    },
    "depthwise_conv2d_bias_TEMPLATE": {
        "operands": (1, 2),
        "build_fcn": (
            TBuilder.DepthwiseConv2dWithBias,
            TGen.tgDepthwiseConv2d,
            ArgGen.agDepthwiseConv2d,
        ),
        "types": {
            "tf": [tf.float32],
            "tflite": [
                tf.float32,
                QuantType.CONV_U8_U8,
                QuantType.CONV_I8_I8,
                QuantType.CONV_I16_I8,
            ],
        },
        "bias": True,
        "template": True,
    },
    "transpose_conv2d_TEMPLATE": {
        "operands": (1, 1),
        "build_fcn": (
            TBuilder.TransposeConv2d,
            TGen.tgTransposeConv2d,
            ArgGen.agTransposeConv2d,
        ),
        "types": {
            "tf": [tf.float32],
            "tflite": [
                tf.float32,
                QuantType.CONV_U8_U8,
                QuantType.CONV_I8_I8,
                QuantType.CONV_I16_I8,
            ],
        },
        "template": True,
    },
    "argmax": {
        "operands": (1, 0),
        "build_fcn": (TBuilder.Argmax, TGen.tgBasic, ArgGen.agAxes),
        "types": {"tf": TYPE_F},
    },
    "avg_pool2d": {
        "operands": (1, 0),
        "build_fcn": (TBuilder.AvgPool2d, TGen.tgPooling, ArgGen.agPooling),
        "types": {
            "tf": TYPE_F,
            "tflite": list(
                TYPE_F + [QuantType.ALL_U8, QuantType.ALL_I8, QuantType.ALL_I16]
            ),
        },
    },
    "max_pool2d": {
        "operands": (1, 0),
        "build_fcn": (TBuilder.MaxPool2d, TGen.tgPooling, ArgGen.agPooling),
        "types": {
            "tf": TYPE_F,
            "tflite": list(TYPE_F + [QuantType.ALL_U8, QuantType.ALL_I8]),
            # ALL_I16 not supported yet
            # In tensorflow/compiler/mlir/lite/ir/tfl_ops.td,
            # QI16 is missing from MaxPoolOperandAndResultConstraints
            # If adding QI16 back this test can run through.
        },
    },
    "reshape": {
        "operands": (1, 0),
        "build_fcn": (TBuilder.Reshape, TGen.tgBasic, ArgGen.agReshape),
        "types": TYPE_FI,
    },
    "transpose": {
        "operands": (1, 0),
        "build_fcn": (TBuilder.Transpose, TGen.tgBasic, ArgGen.agTranspose),
        "types": TYPE_FI,
    },
    "slice": {
        "operands": (1, 0),
        "build_fcn": (TBuilder.Slice, TGen.tgBasic, ArgGen.agSlice),
        "types": TYPE_FI,
    },
    "strided_slice": {
        "operands": (1, 0),
        "build_fcn": (TBuilder.StridedSlice, TGen.tgBasic, ArgGen.agStridedSlice),
        "types": TYPE_FI,
    },
    "select": {
        "operands": (3, 0),
        "build_fcn": (TBuilder.Select, TGen.tgSelect, ArgGen.agNone),
        "types": TYPE_FI,
    },
    "addn": {
        "operands": (4, 0),
        "build_fcn": (TBuilder.Addn, TGen.tgBasic, ArgGen.agNone),
        "types": TYPE_FI,
    },
    "concatv2": {
        "operands": (4, 0),
        "build_fcn": (TBuilder.Concatv2, TGen.tgBasic, ArgGen.agAxes),
        "types": TYPE_FI,
    },
    "stack": {
        "operands": (4, 0),
        "build_fcn": (TBuilder.Stack, TGen.tgBasic, ArgGen.agStack),
        "types": TYPE_FI,
    },
    "unstack": {
        "operands": (1, 0),
        "build_fcn": (TBuilder.Unstack, TGen.tgPooling, ArgGen.agAxes),
        "types": TYPE_F,
    },
    "mirrorpad": {
        "operands": (1, 0),
        "build_fcn": (TBuilder.MirrorPad, TGen.tgBasic, ArgGen.agMirrorPad),
        "types": TYPE_FI,
    },
    "pad": {
        "operands": (1, 0),
        "build_fcn": (TBuilder.Pad, TGen.tgBasic, ArgGen.agPad),
        "types": TYPE_F,
    },
    "expand_dims": {
        "operands": (1, 0),
        "build_fcn": (TBuilder.ExpandDims, TGen.tgBasic, ArgGen.agStack),
        "types": TYPE_FI,
    },
    "shape": {
        "operands": (1, 0),
        "build_fcn": (TBuilder.Shape, TGen.tgBasic, ArgGen.agNone),
        "types": TYPE_FI,
    },
    "rank": {
        "operands": (1, 0),
        "build_fcn": (TBuilder.Rank, TGen.tgBasic, ArgGen.agNone),
        "types": TYPE_FI,
    },
    "fill": {
        "operands": (1, 0),
        "build_fcn": (TBuilder.Fill, TGen.tgBasic, ArgGen.agFill),
        "types": TYPE_FI,
    },
    "elu": {
        "operands": (1, 0),
        "build_fcn": (TBuilder.Elu, TGen.tgBasic, ArgGen.agNone),
        "types": TYPE_F,
    },
    "softmax": {
        "operands": (1, 0),
        "build_fcn": (TBuilder.Softmax, TGen.tgBasic, ArgGen.agNone),
        "types": {
            "tf": TYPE_F,
            "tflite": list(
                TYPE_F + [QuantType.ALL_U8, QuantType.ALL_I8, QuantType.ALL_I16]
            ),
        },
    },
    "log_softmax": {
        "operands": (1, 0),
        "build_fcn": (TBuilder.LogSoftmax, TGen.tgBasic, ArgGen.agNone),
        "types": TYPE_F,
    },
    "matmul": {
        "operands": (2, 0),
        "build_fcn": (TBuilder.MatMul, TGen.tgMatmul, ArgGen.agNone),
        "types": {
            "tf": TYPE_F,
            "tflite": list(
                TYPE_F
                + [QuantType.ALL_U8, QuantType.ALL_I8]
                # 16 bits matmul fail to convert
            ),
        },
    },
    "add_scalar": {
        "operands": (1, 0),
        "build_fcn": (TBuilder.AddScalar, TGen.tgBasic, ArgGen.agNone),
        "types": TYPE_F,
    },
    "add_1d": {
        "operands": (2, 0),
        "build_fcn": (TBuilder.Add1d, TGen.tgBasic, ArgGen.agNone),
        "types": TYPE_F,
    },
    "split": {
        "operands": (1, 0),
        "build_fcn": (TBuilder.Split, TGen.tgBasic, ArgGen.agSplit),
        "types": TYPE_FI,
    },
    "tile": {
        "operands": (1, 0),
        "build_fcn": (TBuilder.Tile, TGen.tgBasic, ArgGen.agTile),
        "types": TYPE_FI,
    },
    "reverse": {
        "operands": (1, 0),
        "build_fcn": (TBuilder.Reverse, TGen.tgBasic, ArgGen.agAxes),
        "types": {"tf": TYPE_FI},
    },
    "gather": {
        "operands": (1, 0),
        "build_fcn": (TBuilder.Gather, TGen.tgBasic, ArgGen.agGather),
        "types": TYPE_FI,
    },
    "gather_nd": {
        "operands": (1, 0),
        "build_fcn": (TBuilder.GatherNd, TGen.tgBasic, ArgGen.agGatherND),
        "types": TYPE_FI,
    },
    "scatter_nd": {
        "operands": (1, 0),
        "build_fcn": (TBuilder.ScatterNd, TGen.tgBasic, ArgGen.agScatterND),
        "types": TYPE_FI,
    },
    "space_to_batch": {
        "operands": (1, 0),
        "build_fcn": (TBuilder.SpaceToBatch, TGen.tgBasic, ArgGen.agSpaceToBatch),
        "types": TYPE_F,
    },
    "batch_to_space": {
        "operands": (1, 0),
        "build_fcn": (TBuilder.BatchToSpace, TGen.tgBasic, ArgGen.agBatchToSpace),
        "types": TYPE_F,
    },
    "space_to_depth": {
        "operands": (1, 0),
        "build_fcn": (TBuilder.SpaceToDepth, TGen.tgBasic, ArgGen.agSpaceToDepth),
        "types": TYPE_F,
    },
    "depth_to_space": {
        "operands": (1, 0),
        "build_fcn": (TBuilder.DepthToSpace, TGen.tgBasic, ArgGen.agDepthToSpace),
        "types": TYPE_F,
    },
    "one_hot": {
        "operands": (3, 1),
        "build_fcn": (TBuilder.OneHot, TGen.tgOneHot, ArgGen.agOneHot),
        "types": TYPE_FI,
    },
    "fakequant": {
        "operands": (1, 0),
        "build_fcn": (
            TBuilder.Fakequant,
            TGen.tgBasic,
            ArgGen.agFakequant,
        ),
        "types": {"tf": TYPE_F},
    },
    "resize_nearest": {
        "operands": (1, 0),
        "build_fcn": (TBuilder.ResizeNearest, TGen.tgPooling, ArgGen.agNone),
        "types": {
            "tf": TYPE_F,
            "tflite": list(
                TYPE_F + [QuantType.ALL_U8, QuantType.ALL_I8, QuantType.ALL_I16]
            ),
        },
    },
    "resize_bilinear": {
        "operands": (1, 0),
        "build_fcn": (TBuilder.ResizeBilinear, TGen.tgPooling, ArgGen.agNone),
        "types": {
            "tf": TYPE_F,
            "tflite": list(
                TYPE_F + [QuantType.ALL_U8, QuantType.ALL_I8, QuantType.ALL_I16]
            ),
        },
    },
    "resize_bilinear_v1_align_corners": {
        "operands": (1, 0),
        "build_fcn": (
            TBuilder.ResizeBilinearV1AlignCorners,
            TGen.tgPooling,
            ArgGen.agNone,
        ),
        "types": {
            "tf": TYPE_F,
            "tflite": list(
                TYPE_F + [QuantType.ALL_U8, QuantType.ALL_I8, QuantType.ALL_I16]
            ),
        },
    },
    "resize_bilinear_v1_none": {
        "operands": (1, 0),
        "build_fcn": (TBuilder.ResizeBilinearV1None, TGen.tgPooling, ArgGen.agNone),
        "types": {
            "tf": TYPE_F,
            "tflite": list(
                TYPE_F + [QuantType.ALL_U8, QuantType.ALL_I8, QuantType.ALL_I16]
            ),
        },
    },
    "left_shift": {
        "operands": (1, 0),
        "build_fcn": (TBuilder.LeftShift, TGen.tgBasic, ArgGen.agShift),
        "types": {"tf": [tf.int32]},
    },
    "right_shift": {
        "operands": (1, 0),
        "build_fcn": (TBuilder.RightShift, TGen.tgBasic, ArgGen.agShift),
        "types": {
            "tf": [
                tf.int32,
            ]
        },
    },
}

# Shapes to be tested; default can be overwritten
shape_list = [
    (1,),
    (64,),
    (14, 19),
    (13, 21, 3),
    (1, 4, 4, 4),
    (1, 8, 4, 17),
    (1, 4, 8, 19),
    (1, 32, 32, 8),
    (1, 7, 7, 9),
    (2, 2, 7, 7, 2),
    (1, 4, 8, 21, 17),
    (3, 32, 16, 16, 5),
]


def gen_rand_shapes(args):
    """Overwrite the global shape list with a new list of random shapes"""
    global shape_list

    rng = np.random.default_rng(args.random_seed)

    # Don't let things get too big... cap the maximum volume, but let
    # an individual dimension be 1..47
    max_total_volume = 32 * 32 * 4

    shape_list = []
    # Only iterate over ranks 2, 3, 4, and 5
    for rank in range(2, 6):
        for n in range(args.random_shapes):
            new_shape = rng.integers(1, 48, size=rank)

            # Set the batch dimension on 4D or 5D objects to 1
            if rank == 4 or rank == 5:
                new_shape[0] = 1

            # Limit the total shape volume and throw out any
            # shapes that wouldn't leave at least size=2 in some non-batch dimension
            volume = 1
            skip_shape = False
            for i in range(rank):

                volume *= new_shape[i]

                # Reduce the shape, while it's larger than the maximum volume
                while volume > max_total_volume:
                    new_shape[i] = new_shape[i] // 2
                    volume = volume // 2

                    # Now an untenable dimension size?  Skip this one.
                    if new_shape[i] < 1:
                        skip_shape = True

            if not skip_shape:
                shape_list.append(tuple(new_shape))


# Construct, run and save a whole tensorflow tf.function to a protobuf file
# or convert to .tflite if it's quantized unit test
def run_unit_test(
    op_name,
    args,
    test_dir,
    curr_shape,
    addl_args,
    dtype,
    excluded_framework_list,
    quantized_inference_dtype,
    result_name,
    seed,
):

    try:
        op = TF_OP_LIST[op_name]
        op_fcn, tensor_gen_fcn, arg_gen_fcn = op["build_fcn"]

        # Get and seed a random number generator for this test
        rng = np.random.default_rng(seed)

        # return placeholders=(str: name, np.array: value)
        # consts=(str: name, np.array: value)
        placeholders, consts = tensor_gen_fcn(op, curr_shape, dtype, rng)

        # if test doesn't have any placeholders/consts, terminated
        if len(placeholders) == 0 and len(consts) == 0:
            return True

        if not args.quiet:
            print("   {}              ".format(test_dir))

        try:
            os.mkdir(test_dir)
        except FileExistsError:
            pass

        const_nodes = [value for name, value in consts]

        num_placeholders = len(placeholders)
        # if test is quantized, create tensor quantization metadata info for
        # each input tensor, based on different quantized type
        if quantized_inference_dtype:
            is_quantized = True
            # TODO: support INT8 IFM x INT4 weight later
            if quantized_inference_dtype == QuantType.ALL_U8:
                qzero = [128] * num_placeholders
                numpy_dtype = [np.uint8] * num_placeholders
                tflite_inference_dtype = tf.uint8
            elif quantized_inference_dtype == QuantType.ALL_I8:
                qzero = [0] * num_placeholders
                numpy_dtype = [np.int8] * num_placeholders
                tflite_inference_dtype = tf.int8
            elif quantized_inference_dtype == QuantType.ALL_I16:
                qzero = [0] * num_placeholders
                numpy_dtype = [np.int16] * num_placeholders
                tflite_inference_dtype = tf.int16
            elif quantized_inference_dtype == QuantType.CONV_U8_U8:
                assert (
                    num_placeholders == 1
                ), "Unsupported number of placeholders for Convolution: {}".format(
                    num_placeholders
                )
                qzero = [128] * num_placeholders
                if num_placeholders == 2:
                    numpy_dtype = [np.uint8, np.uint8]
                else:
                    numpy_dtype = [np.uint8, np.uint8, np.int32]
                tflite_inference_dtype = tf.uint8
            elif quantized_inference_dtype == QuantType.CONV_I8_I8:
                assert (
                    num_placeholders == 1
                ), "Unsupported number of placeholders for Convolution: {}".format(
                    num_placeholders
                )
                qzero = [0] * num_placeholders
                if num_placeholders == 2:
                    numpy_dtype = [np.int8, np.int8]
                else:
                    numpy_dtype = [np.int8, np.int8, np.int32]
                tflite_inference_dtype = tf.int8
            elif quantized_inference_dtype == QuantType.CONV_I16_I8:
                assert (
                    num_placeholders == 1
                ), "Unsupported number of placeholders for Convolution: {}".format(
                    num_placeholders
                )
                if num_placeholders == 2:
                    qzero = [0, 0]
                    numpy_dtype = [np.int16, np.int8]
                else:
                    qzero = [0, 0, 0]
                    numpy_dtype = [
                        np.int16,
                        np.int8,
                        np.int64,
                    ]  # np.int64 to represent 40 bits accumulator
                tflite_inference_dtype = tf.int16
            else:
                raise Exception(
                    "Unsupported fakequant dtype: {}".format(quantized_inference_dtype)
                )

        else:
            is_quantized = False

        tf_model_filename = None
        tf_result_npy_filename = None
        tf_result_name = None

        tflite_model_filename = None
        tflite_result_npy_filename = None
        tflite_result_name = None

        placeholder_names = []
        placeholder_vals = []
        placeholder_signatures = ()
        placeholder_npy_filenames = []
        placeholder_shapes = []

        for idx, (name, val) in enumerate(placeholders):
            placeholder_names.append(name)
            placeholder_signatures = placeholder_signatures + (
                tf.TensorSpec(shape=val.shape, dtype=val.dtype, name=name),
            )
            placeholder_npy_filenames.append("{}.npy".format(name.split(":")[0]))
            placeholder_shapes.append(val.shape)

        # Get test builder class
        fcn_node = op_fcn(*const_nodes, *addl_args, result_name)
        concrete_function = tf.function(input_signature=placeholder_signatures)(
            fcn_node.eval
        ).get_concrete_function()

        if is_quantized:

            assert dtype is tf.float32, "quantized test must come from float32 graph"

            # 1. Quantize float placeholder npy to quantized to feed the graph
            for idx, (name, val) in enumerate(placeholders):

                # we use np.amin()/np.amax() to determine dynamic range
                # for quantized test
                zeropoint = 0
                scale = 1.0
                if numpy_dtype[idx] != np.int64:
                    qmin = np.iinfo(numpy_dtype[idx]).min
                    qmax = np.iinfo(numpy_dtype[idx]).max
                    num_bits = np.iinfo(numpy_dtype[idx]).bits
                # 40 bit is represented as np.int64
                else:
                    num_bits = 40
                    qmin = -(1 << num_bits)
                    qmax = (1 << num_bits) - 1

                min_val = np.amin(val)
                max_val = np.amax(val)

                # for single value tensor, we set scale equal to the abs(value),
                # and fix zeropoint to 128
                # if val > 0, it'll be represented as 129,
                #    where val = (129 - 128) * val
                # if val < 0, it'll be represented as 127,
                #    where val = (127 - 128) * (-val)
                # if val == 0, it'll be represted as 128, with range [-128.0, 128.0]
                # and let quantized 1 represent the value
                # also adjust effective min/max consequently
                if max_val == min_val:
                    if max_val != 0:
                        scale = abs(max_val)
                    else:
                        scale = 1.0
                    min_val = float(qmin - qzero[idx]) * scale
                    max_val = float(qmax - qzero[idx]) * scale
                else:
                    scale = (max_val - min_val) / float(qmax - qmin)
                    zeropoint = int(round((-min_val) / scale)) + qmin

                # run through tf.fakequant first to assure quantization error aligned
                fakequant_val = tf.quantization.fake_quant_with_min_max_args(
                    val,
                    min=min_val,
                    max=max_val,
                    num_bits=num_bits,
                    name="gen_quant_npy",
                )

                quant_val = np.round(fakequant_val / scale).astype(np.int32) + zeropoint

                # very few unit tests after TF hash may/2020, this quantized
                # value for some reason exceed [0, 255] range
                saved_val = np.clip(quant_val, qmin, qmax).astype(numpy_dtype[idx])

                # saved all quantized tensor as np.int32
                # since TOSA numpy Cpp API only supports int32
                np.save(
                    os.path.join(test_dir, placeholder_npy_filenames[idx]),
                    saved_val.astype(np.int32),
                    False,
                )

                placeholder_vals.append(tf.convert_to_tensor(saved_val))

            # 2. Convert the model to quantized TFLite flatbuffer
            module = tf.Module()
            converter = tf.lite.TFLiteConverter.from_concrete_functions(
                [concrete_function], module
            )
            converter.optimizations = [tf.lite.Optimize.DEFAULT]
            converter.experimental_new_converter = True

            # use MLIR-based post-quantizer
            converter.experimental_new_quantizer = True

            flag = (
                tf.lite.OpsSet.EXPERIMENTAL_TFLITE_BUILTINS_ACTIVATIONS_INT16_WEIGHTS_INT8  # noqa: E501
            )
            if tflite_inference_dtype == tf.int16:
                converter.target_spec.supported_ops = [flag]

            def input_stats():
                for i in range(0, args.num_samples):
                    a = [
                        TGen.getRand(shape, tf.float32, rng)
                        for shape in placeholder_shapes
                    ]
                    yield a

            converter.representative_dataset = input_stats
            converter.inference_input_type = tflite_inference_dtype
            converter.inference_output_type = tflite_inference_dtype

            tflite_model = converter.convert()

            tflite_model_filename = "model.tflite"

            # Write out converted model to disk
            with open(os.path.join(test_dir, tflite_model_filename), "wb") as f:
                f.write(tflite_model)

        else:  # is_quantized is False

            # 1. Saved out numpy array directly
            for idx, (name, val) in enumerate(placeholders):
                placeholder_vals.append(tf.convert_to_tensor(val))
                np.save(
                    os.path.join(test_dir, placeholder_npy_filenames[idx]), val, False
                )

            # 2.a Saved out .pb if framework includes tensorflow
            if "tf" not in excluded_framework_list:
                # Write out graph as protobuf to disk
                tf_model_filename = "model.pb"
                tf.io.write_graph(
                    concrete_function.graph, test_dir, tf_model_filename, True
                )

            # 2.b Saved out .tflite if framework includes tflite
            if "tflite" not in excluded_framework_list:
                # Convert the model to TFLite flatbuffer
                module = tf.Module()
                converter = tf.lite.TFLiteConverter.from_concrete_functions(
                    [concrete_function], module
                )

                converter.experimental_new_converter = True

                # Even it's non-quantized int32 test, this needs to be set to tf.float32
                converter.inference_input_type = tf.float32
                converter.inference_output_type = tf.float32
                tflite_model = converter.convert()

                # Write out converted model to disk
                tflite_model_filename = "model.tflite"
                with open(os.path.join(test_dir, tflite_model_filename), "wb") as f:
                    f.write(tflite_model)

        # Get TF reference result if .pb is specified
        if tf_model_filename:
            tf_result_npy_filename = "tf_result.npy"
            tf_result = concrete_function(*placeholder_vals)
            np.save(os.path.join(test_dir, tf_result_npy_filename), tf_result, False)

            tf_result_name = result_name

        # Get TFLite inference result if .tflite is specified
        if tflite_model_filename:
            tflite_result_npy_filename = "tflite_result.npy"

            ops_with_optimized_only_kernel = ["elu", "ceil", "gather"]

            if args.tflite_kernel_mode == "optimized" or (
                op_name in ops_with_optimized_only_kernel
            ):
                interpreter = tf.lite.Interpreter(
                    model_path=os.path.join(test_dir, tflite_model_filename)
                )
            elif args.tflite_kernel_mode == "reference":
                interpreter = tf.lite.Interpreter(
                    model_path=os.path.join(test_dir, tflite_model_filename),
                    experimental_op_resolver_type=OpResolverType.BUILTIN_REF,
                )
            else:
                assert 0, "unknown tflite interpreter mode {}".format(
                    args.tflite_kernel_mode
                )
            interpreter.allocate_tensors()

            input_details = interpreter.get_input_details()
            output_details = interpreter.get_output_details()

            assert len(input_details) == len(
                placeholder_vals
            ), "number of placeholder mismatch"

            for idx, val in enumerate(placeholder_vals):
                interpreter.set_tensor(input_details[idx]["index"], val.numpy())

            interpreter.invoke()
            tflite_result = interpreter.get_tensor(output_details[0]["index"])

            np.save(
                os.path.join(test_dir, tflite_result_npy_filename), tflite_result, False
            )

            # Result tensor name would change after converting to TFLite flatbuffer
            # Overwrite the information from TFLite models directly.
            # Assume single result tensor now
            tflite_result_name = output_details[0]["name"]

        # Write out test descriptor
        write_test_json(
            filename=os.path.join(test_dir, "test.json"),
            tf_model_filename=tf_model_filename,
            tf_result_npy_filename=tf_result_npy_filename,
            tf_result_name=tf_result_name,
            tflite_model_filename=tflite_model_filename,
            tflite_result_npy_filename=tflite_result_npy_filename,
            tflite_result_name=tflite_result_name,
            ifm_name=placeholder_names,
            ifm_file=placeholder_npy_filenames,
            ifm_shape=placeholder_shapes,
            framework_exclusions=excluded_framework_list,
            quantized=is_quantized,
        )
    except Exception as e:
        msg = "Error running task: {}".format(e)
        print(msg)
        print(
            "".join(
                traceback.format_exception(etype=type(e), value=e, tb=e.__traceback__)
            )
        )
        return False
    return True


def build_const_net(
    args,
    curr_shape,
    op_name,
    dtype,
    excluded_framework_list,
    quantized_inference_dtype,
    result_name,
    seed,
    rng,
    filter,
    unit_test_args,
):

    if quantized_inference_dtype:
        quant_dtype = get_tf_dtype(quantized_inference_dtype)
        test_dir = "test_{}_{}".format(op_name, get_shape_str(curr_shape, quant_dtype))
    else:
        test_dir = "test_{}_{}".format(op_name, get_shape_str(curr_shape, dtype))
    test_dir = os.path.join(args.output_dir, test_dir)

    # If the operator has an additional function to generate arguments, call it
    # here and iterate through the argument list that it generates
    op = TF_OP_LIST[op_name]
    op_fcn, tensor_gen_fcn, arg_gen_fcn = op["build_fcn"]

    try:
        rank_lo, rank_hi = op["rank"]
    except KeyError:
        # Set testing rank to (1, 4) in default.
        rank_lo = 1
        rank_hi = 4

    if len(curr_shape) not in range(rank_lo, rank_hi + 1):
        return

    addl_args_tuple = arg_gen_fcn(op, curr_shape, rng)
    for desc, addl_args in addl_args_tuple:
        # Only filter on the full test_name, not the output directory
        _, test_name = os.path.split(test_dir + desc)
        if not filter or filter.search(test_name):
            unit_test_args.append(
                [
                    op_name,
                    args,
                    test_dir + desc,
                    curr_shape,
                    addl_args,
                    dtype,
                    excluded_framework_list,
                    quantized_inference_dtype,
                    result_name,
                    seed,
                ]
            )


# python hash is not reproducible, create hash for our purpose
def op_name_hash(op_name):
    result = 0xDEADBEEF
    for ch in op_name:
        if result & 1:
            result = (ord(ch) << 24) ^ (result >> 1) ^ 0x82608EDB
        else:
            result = (ord(ch) << 24) ^ (result >> 1)

    return result


def generate_op_tests(args, op_name, shape_list, result_name, filter, unit_test_args):

    if not args.quiet:
        print(
            "Generating tests for {}                                        ".format(
                op_name
            )
        )

    op = TF_OP_LIST[op_name]

    # Seed the RNG so that we get the same random tests for each test each time
    # If the number of tests for a given generation function changes, the tests
    # for that operator may also change accordingly, but this will at least keep
    # down churn across operators.

    bounded_hash_val = (args.random_seed + op_name_hash(op_name)) % np.iinfo(
        np.int32
    ).max
    rng = np.random.default_rng(bounded_hash_val)

    # this is a dictionary with 'tf' and 'tflite' as key
    # and value being the data types we want to test under these framework

    if isinstance(op["types"], dict):
        try:
            tf_dtypes = op["types"]["tf"]
        except KeyError:
            tf_dtypes = []
        try:
            tflite_dtypes = op["types"]["tflite"]
        except KeyError:
            tflite_dtypes = []
    elif isinstance(op["types"], list):
        tf_dtypes = op["types"]
        tflite_dtypes = op["types"]

    tf_nonquantized_dtypes = tf_dtypes  # tf doesn't support quantized data types
    tflite_quantized_dtypes = []
    tflite_nonquantized_dtypes = []
    for dtype in tflite_dtypes:
        if isinstance(dtype, QuantType):
            tflite_quantized_dtypes.append(dtype)
        else:
            tflite_nonquantized_dtypes.append(dtype)

    nonquantized_dtypes_set = set(tf_nonquantized_dtypes).union(
        set(tflite_nonquantized_dtypes)
    )
    nonquantized_dtypes = list(nonquantized_dtypes_set)
    quantized_dtypes = tflite_quantized_dtypes

    # populate non quantized unit test arguments
    for dtype in nonquantized_dtypes:

        excluded_framework_set = set(ALL_FRAMEWORKS)
        if dtype in tf_nonquantized_dtypes:
            excluded_framework_set.remove("tf")
        if dtype in tflite_nonquantized_dtypes:
            excluded_framework_set.remove("tflite")
        excluded_framework_list = list(excluded_framework_set)

        for curr_shape in shape_list:
            build_const_net(
                args,
                curr_shape,
                op_name,
                dtype,
                excluded_framework_list,
                None,
                result_name,
                bounded_hash_val,
                rng,
                filter,
                unit_test_args,
            )

    # populate quantized unit test arguments
    # must exclude 'tf' and source dtype being tf.float32
    for dtype in quantized_dtypes:
        for curr_shape in shape_list:
            build_const_net(
                args,
                curr_shape,
                op_name,
                tf.float32,
                ["tf"],
                dtype,
                result_name,
                bounded_hash_val,
                rng,
                filter,
                unit_test_args,
            )

    return unit_test_args


def createDynamicOpLists():
    """The templated operators are conv2d-style operators with a number of kernel
    sizes.  Since the operator is unchanged, we generate the range of kernel
    sizes here in this loop and remove the original templates from the list.

    This could be expanded to non-conv2d-style operators in the future."""

    # Dynamically create op lists for convolutions with a list of kernel sizes
    KERNELS = [
        [1, 1],
        [3, 3],
        [5, 5],
    ]

    # dim = [D, H, W]
    KERNELS_3D = [
        [1, 1, 1],
        [2, 3, 3],
        [3, 5, 5],
    ]

    TEMPLATE_LIST = [
        "conv2d",
        "conv2d_bias",
        "conv2d_relu",
        "conv2d_relu6",
        "conv2d_relu_n1_to_1",
        "conv2d_tanh",
        "depthwise_conv2d",
        "depthwise_conv2d_bias",
        "transpose_conv2d",
    ]

    TEMPLATE_LIST_CONV3D = [
        "conv3d",
        "conv3d_bias",
    ]

    for t in TEMPLATE_LIST:
        for k in KERNELS:
            testName = "{}_{}x{}".format(t, k[0], k[1])
            TF_OP_LIST[testName] = TF_OP_LIST["{}_TEMPLATE".format(t)].copy()
            TF_OP_LIST[testName]["filter"] = k
            TF_OP_LIST[testName]["template"] = False

    # The existing operators don't support the dimension of kernel that is higher than 2.
    for t in TEMPLATE_LIST_CONV3D:
        for k in KERNELS_3D:
            testName = "{}_{}x{}x{}".format(t, k[0], k[1], k[2])
            TF_OP_LIST[testName] = TF_OP_LIST["{}_TEMPLATE".format(t)].copy()
            TF_OP_LIST[testName]["filter"] = k
            TF_OP_LIST[testName]["template"] = False

    # Delete any templates after having created any dynamic ops
    # This is a two-pass operation because it's bad practice to delete
    # keys from dictionaries while iterating
    keyList = []
    for k in TF_OP_LIST:
        try:
            if TF_OP_LIST[k]["template"]:
                keyList.append(k)
                continue
        except KeyError:
            pass

    for k in keyList:
        del TF_OP_LIST[k]


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "--seed", dest="random_seed", default=42, type=int, help="Random seed"
    )
    parser.add_argument(
        "--random-shapes",
        dest="random_shapes",
        default=0,
        type=int,
        help=(
            "Use N random shapes of each rank for generating tests,"
            "seeded with random seed"
        ),
    )
    parser.add_argument(
        "-o",
        "--output-dir",
        dest="output_dir",
        default=".",
        type=str,
        help="Test output directory path prefix",
    )
    parser.add_argument(
        "-q",
        "--quiet",
        dest="quiet",
        default=False,
        action="store_true",
        help="Do not print test names",
    )
    parser.add_argument(
        "-j", "--jobs", dest="jobs", type=int, default=1, help="Number of parallel jobs"
    )
    parser.add_argument(
        "-m",
        "--tflite-kernel-mode",
        dest="tflite_kernel_mode",
        type=str,
        choices=["reference", "optimized"],
        default="reference",
        help="TFLite interpreter kernel mode",
    )
    parser.add_argument(
        "--num-samples",
        dest="num_samples",
        default=200,
        type=int,
        help="Number of input samples for post-training quantization",
    )
    parser.add_argument(
        "--filter",
        dest="filter",
        default="",
        type=str,
        help="Filter test names by this expression",
    )
    args = parser.parse_args()

    # Turn the filter into a re object if present
    filter = None
    if args.filter != "":
        filter = re.compile(args.filter)

    # Autodetect CPU count
    if args.jobs <= 0:
        args.jobs = os.cpu_count()

    # Disable TF info messages
    os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3"

    try:
        os.makedirs(args.output_dir)
    except FileExistsError:
        pass

    if args.random_shapes:
        gen_rand_shapes(args)

    # Build dynamic ops
    createDynamicOpLists()

    # Generate the test list and arguments to run_unit_test()
    unit_test_args = []

    for op in TF_OP_LIST:
        generate_op_tests(args, op, shape_list, "result", filter, unit_test_args)

    errors = 0
    for t in unit_test_args:
        if not run_unit_test(*t):
            errors = errors + 1

    if not args.quiet:
        print("\nAll tasks done - with {} errors".format(errors))

    return 1 if errors else 0


if __name__ == "__main__":
    exit(main())