Nuitka: Tensorflow unknow opcode error with Dense layers

Nuitka compiles without any errors, then SystemError: unknown opcode occurs on runtime when Dense layers are present in the model. Commenting out Dense layers makes it work. Of course it works well with normal Python interpreter. I tried other versions of Tensorflow (from 2.0 to 2.11), but I encountered the same issue. Enabling deprecated tensorflow plugin also does not help.

Script to reproduce:

from tensorflow.keras.layers import Input, Conv2D, BatchNormalization, Activation, GlobalAveragePooling2D,\
    DepthwiseConv2D, Reshape, multiply, add, Dense
from tensorflow.keras.activations import swish, sigmoid
from tensorflow.keras.models import Model
from collections import namedtuple
from math import ceil
import string


def round_filters(fltrs, wc, dd):
    fltrs *= wc
    new_filters = int(fltrs + dd / 2) // dd * dd
    new_filters = max(dd, new_filters)
    if new_filters < 0.9 * fltrs:
        new_filters += dd
    return int(new_filters)


BlockArgs = namedtuple('BlockArgs', [
    'kernel_size', 'num_repeat', 'input_filters', 'output_filters',
    'expand_ratio', 'id_skip', 'strides', 'se_ratio'
])
BlockArgs.__new__.__defaults__ = (None,) * len(BlockArgs._fields)

DEFAULT_BLOCKS_ARGS = [
    BlockArgs(kernel_size=3, num_repeat=1, input_filters=32, output_filters=16,
              expand_ratio=1, id_skip=True, strides=[1, 1], se_ratio=0.25),
    BlockArgs(kernel_size=3, num_repeat=2, input_filters=16, output_filters=24,
              expand_ratio=6, id_skip=True, strides=[2, 2], se_ratio=0.25),
    BlockArgs(kernel_size=5, num_repeat=2, input_filters=24, output_filters=40,
              expand_ratio=6, id_skip=True, strides=[2, 2], se_ratio=0.25),
    BlockArgs(kernel_size=3, num_repeat=3, input_filters=40, output_filters=80,
              expand_ratio=6, id_skip=True, strides=[2, 2], se_ratio=0.25),
    BlockArgs(kernel_size=5, num_repeat=3, input_filters=80, output_filters=112,
              expand_ratio=6, id_skip=True, strides=[1, 1], se_ratio=0.25),
    BlockArgs(kernel_size=5, num_repeat=4, input_filters=112, output_filters=192,
              expand_ratio=6, id_skip=True, strides=[2, 2], se_ratio=0.25),
    BlockArgs(kernel_size=3, num_repeat=1, input_filters=192, output_filters=320,
              expand_ratio=6, id_skip=True, strides=[1, 1], se_ratio=0.25)
]

CONV_KERNEL_INITIALIZER = {
    'class_name': 'VarianceScaling',
    'config': {
        'scale': 2.0,
        'mode': 'fan_out',
        'distribution': 'normal'
    }
}

DENSE_KERNEL_INITIALIZER = {
    'class_name': 'VarianceScaling',
    'config': {
        'scale': 1. / 3.,
        'mode': 'fan_out',
        'distribution': 'uniform'
    }
}

width_coefficient = 1.2
depth_coefficient = 1.4
default_resolution = 300
depth_divisor = 8
blocks_args = DEFAULT_BLOCKS_ARGS
input_shape = (150, 150, 3)

img_input = Input(shape=input_shape)

bn_axis = 3

x = img_input
x = Conv2D(round_filters(32, width_coefficient, depth_divisor), 3,
           strides=(2, 2),
           padding='same',
           use_bias=False,
           kernel_initializer=CONV_KERNEL_INITIALIZER,
           name='stem_conv')(x)
x = BatchNormalization(axis=bn_axis, name='stem_bn')(x)
x = Activation(swish, name='stem_activation')(x)

num_blocks_total = sum(block_args.num_repeat for block_args in blocks_args)
block_num = 0
for idx, block_args in enumerate(blocks_args):
    assert block_args.num_repeat > 0
    block_args = block_args._replace(
        input_filters=round_filters(block_args.input_filters,
                                    width_coefficient, depth_divisor),
        output_filters=round_filters(block_args.output_filters,
                                     width_coefficient, depth_divisor),
        num_repeat=int(ceil(depth_coefficient * block_args.num_repeat)))

    inputs = x
    has_se = (block_args.se_ratio is not None) and (0 < block_args.se_ratio <= 1)

    filters = block_args.input_filters * block_args.expand_ratio
    if block_args.expand_ratio != 1:
        x = Conv2D(filters, 1,
                   padding='same',
                   use_bias=False,
                   kernel_initializer=CONV_KERNEL_INITIALIZER,
                   name='block{}a_'.format(idx + 1) + 'expand_conv')(inputs)
        x = BatchNormalization(axis=bn_axis, name='block{}a_'.format(idx + 1) + 'expand_bn')(x)
        x = Activation(swish, name='block{}a_'.format(idx + 1) + 'expand_activation')(x)

    x = DepthwiseConv2D(block_args.kernel_size,
                        strides=block_args.strides,
                        padding='same',
                        use_bias=False,
                        depthwise_initializer=CONV_KERNEL_INITIALIZER,
                        name='block{}a_'.format(idx + 1) + 'dwconv')(x)
    x = BatchNormalization(axis=bn_axis, name='block{}a_'.format(idx + 1) + 'bn')(x)
    x = Activation(swish, name='block{}a_'.format(idx + 1) + 'activation')(x)

    if has_se:
        num_reduced_filters = max(1, int(
            block_args.input_filters * block_args.se_ratio
        ))
        se_tensor = GlobalAveragePooling2D(name='block{}a_'.format(idx + 1) + 'se_squeeze')(x)

        target_shape = (1, 1, filters)
        se_tensor = Reshape(target_shape, name='block{}a_'.format(idx + 1) + 'se_reshape')(se_tensor)
        se_tensor = Conv2D(num_reduced_filters, 1,
                           activation=swish,
                           padding='same',
                           use_bias=True,
                           kernel_initializer=CONV_KERNEL_INITIALIZER,
                           name='block{}a_'.format(idx + 1) + 'se_reduce')(se_tensor)
        se_tensor = Conv2D(filters, 1,
                           activation='sigmoid',
                           padding='same',
                           use_bias=True,
                           kernel_initializer=CONV_KERNEL_INITIALIZER,
                           name='block{}a_'.format(idx + 1) + 'se_expand')(se_tensor)

        x = multiply([x, se_tensor], name='block{}a_'.format(idx + 1) + 'se_excite')

    x = Conv2D(block_args.output_filters, 1,
               padding='same',
               use_bias=False,
               kernel_initializer=CONV_KERNEL_INITIALIZER,
               name='block{}a_'.format(idx + 1) + 'project_conv')(x)
    x = BatchNormalization(axis=bn_axis, name='block{}a_'.format(idx + 1) + 'project_bn')(x)
    if block_args.id_skip and all(
            s == 1 for s in block_args.strides) and block_args.input_filters == block_args.output_filters:
        x = add([x, inputs], name='block{}a_'.format(idx + 1) + 'add')

    block_num += 1
    if block_args.num_repeat > 1:
        # pylint: disable=protected-access
        block_args = block_args._replace(input_filters=block_args.output_filters, strides=[1, 1])
        # pylint: enable=protected-access
        for bidx in range(block_args.num_repeat - 1):
            block_prefix = 'block{}{}_'.format(idx + 1, string.ascii_lowercase[bidx + 1])

            inputs = x
            has_se = (block_args.se_ratio is not None) and (0 < block_args.se_ratio <= 1)

            filters = block_args.input_filters * block_args.expand_ratio
            if block_args.expand_ratio != 1:
                x = Conv2D(filters, 1,
                           padding='same',
                           use_bias=False,
                           kernel_initializer=CONV_KERNEL_INITIALIZER,
                           name=block_prefix + 'expand_conv')(inputs)
                x = BatchNormalization(axis=bn_axis, name=block_prefix + 'expand_bn')(x)
                x = Activation(swish, name=block_prefix + 'expand_activation')(x)

            x = DepthwiseConv2D(block_args.kernel_size,
                                strides=block_args.strides,
                                padding='same',
                                use_bias=False,
                                depthwise_initializer=CONV_KERNEL_INITIALIZER,
                                name=block_prefix + 'dwconv')(x)
            x = BatchNormalization(axis=bn_axis, name=block_prefix + 'bn')(x)
            x = Activation(swish, name=block_prefix + 'activation')(x)

            if has_se:
                num_reduced_filters = max(1, int(
                    block_args.input_filters * block_args.se_ratio
                ))
                se_tensor = GlobalAveragePooling2D(name=block_prefix + 'se_squeeze')(x)

                target_shape = (1, 1, filters)
                se_tensor = Reshape(target_shape, name=block_prefix + 'se_reshape')(se_tensor)
                se_tensor = Conv2D(num_reduced_filters, 1,
                                   activation=swish,
                                   padding='same',
                                   use_bias=True,
                                   kernel_initializer=CONV_KERNEL_INITIALIZER,
                                   name=block_prefix + 'se_reduce')(se_tensor)
                se_tensor = Conv2D(filters, 1,
                                   activation='sigmoid',
                                   padding='same',
                                   use_bias=True,
                                   kernel_initializer=CONV_KERNEL_INITIALIZER,
                                   name=block_prefix + 'se_expand')(se_tensor)

                x = multiply([x, se_tensor], name=block_prefix + 'se_excite')

            x = Conv2D(block_args.output_filters, 1,
                       padding='same',
                       use_bias=False,
                       kernel_initializer=CONV_KERNEL_INITIALIZER,
                       name=block_prefix + 'project_conv')(x)
            x = BatchNormalization(axis=bn_axis, name=block_prefix + 'project_bn')(x)
            if block_args.id_skip and all(
                    s == 1 for s in block_args.strides) and block_args.input_filters == block_args.output_filters:
                x = add([x, inputs], name=block_prefix + 'add')
            block_num += 1

x = Conv2D(round_filters(1280, width_coefficient, depth_divisor), 1,
           padding='same',
           use_bias=False,
           kernel_initializer=CONV_KERNEL_INITIALIZER,
           name='top_conv')(x)
x = BatchNormalization(axis=bn_axis, name='top_bn')(x)
x = Activation(swish, name='top_activation')(x)
x = GlobalAveragePooling2D(name='avg_pool')(x)

x = BatchNormalization(name="first_out")(x)
#x = Dense(512, name="first_dense")(x)
x = BatchNormalization(name="second_out")(x)
x = Activation(swish, name="first_act_out")(x)
#x = Dense(128, name="second_dense")(x)
x = BatchNormalization(name="third_out")(x)
x = Activation(swish, name="second_act_out")(x)
#predictions = Dense(1, activation=sigmoid, name="the_last_one")(x)

model = Model(inputs=img_input, outputs=x)

Output: XXX lineno: 5985, opcode: 0 Traceback (most recent call last): File “c:\Users\Maks\PYCHAR~1\DICENT~1.0\NEW_MO~1.DIS\new_model.py”, line 221, in <module> File “c:\Users\Maks\PYCHAR~1\DICENT~1.0\NEW_MO~1.DIS\tensorflow\python\keras\engine\base_layer.py”, line 983, in call File “c:\Users\Maks\PYCHAR~1\DICENT~1.0\NEW_MO~1.DIS\tensorflow\python\keras\engine\base_layer.py”, line 1121, in _functional_construction_call File “c:\Users\Maks\PYCHAR~1\DICENT~1.0\NEW_MO~1.DIS\tensorflow\python\keras\engine\base_layer.py”, line 854, in _keras_tensor_symbolic_call File “c:\Users\Maks\PYCHAR~1\DICENT~1.0\NEW_MO~1.DIS\tensorflow\python\keras\engine\base_layer.py”, line 894, in _infer_output_signature File “c:\Users\Maks\PYCHAR~1\DICENT~1.0\NEW_MO~1.DIS\tensorflow\python\keras\layers\core.py”, line 1239, in call File “c:\Users\Maks\PYCHAR~1\DICENT~1.0\NEW_MO~1.DIS\tensorflow\python\util\tf_export.py”, line 404, in wrapper File “c:\Users\Maks\PYCHAR~1\DICENT~1.0\NEW_MO~1.DIS\tensorflow\python\ops\gen_math_ops.py”, line 5985, in mat_mul SystemError: unknown opcode

Environment:

tensorflow 2.7.4
Nuitka 1.4.8
Commercial: None
Python: 3.9.13 (tags/v3.9.13:6de2ca5, May 17 2022, 16:36:42) [MSC v.1929 64 bit (AMD64)]
Flavor: CPython Official
Executable: C:\Users\Maks\Envs\nuitka5\Scripts\python.exe
OS: Windows
Arch: x86_64
WindowsRelease: 10
Version C compiler: C:\MVS\2022\BuildTools\VC\Tools\MSVC\14.34.31933\bin\HostX64\x64\cl.exe (cl 14.3)

Compilation command: c:\Users\Maks\PycharmProjects\Dicentryki 2.0>python -m nuitka --standalone --noinclude-unittest-mode=allow --noinclude-setuptools-mode=allow --force-dll-dependency-cache-update --include-package=tensorflow,numpy new_model.py

Compilation output: Nuitka:INFO: Starting Python compilation with Nuitka ‘1.4.8’ on Python ‘3.9’ commercial grade ‘not installed’. Nuitka-Plugins:INFO: delvewheel: Detected usage of ‘delvewheel’ version ‘1.3.1’ in module ‘scipy’. Nuitka-Plugins:INFO: multiprocessing: Injecting pre-module load code for module ‘multiprocessing’: Nuitka-Plugins:INFO: multiprocessing: Monkey patching “multiprocessing” load environment. Nuitka-Plugins:INFO: multiprocessing: Injecting post-module load code for module ‘multiprocessing’: Nuitka-Plugins:INFO: multiprocessing: Monkey patching “multiprocessing” for compiled methods. Nuitka-Plugins:INFO: pkg-resources: Injecting post-module load code for module ‘pkg_resources’: Nuitka-Plugins:INFO: pkg-resources: Registering Nuitka loader with “pkg_resources”. Nuitka:INFO: Completed Python level compilation and optimization. Nuitka:INFO: Generating source code for C backend compiler. Nuitka:INFO: Running data composer tool for optimal constant value handling. Nuitka:INFO: Running C compilation via Scons. Nuitka-Scons:INFO: Backend C compiler: cl (cl 14.3). Nuitka-Scons:INFO: Backend linking program with 3793 files (no progress information available). Nuitka-Scons:INFO: Compiled 3793 C files using clcache with 3791 cache hits and 2 cache misses. Nuitka-Plugins:INFO: data-files: Included data file ‘certifi\cacert.pem’ due to package data for ‘certifi’. Nuitka-Plugins:INFO: data-files: Included data file ‘skimage\io_plugins\fits_plugin.ini’ due to package data for ‘skimage’. Nuitka-Plugins:INFO: data-files: Included 30 data files due to package data directory ‘data’ for ‘skimage’. Nuitka-Plugins:INFO: data-files: Included data file ‘skimage\feature\orb_descriptor_positions.txt’ due to package data for ‘skimage.feature._orb_descriptor_positions’. Nuitka-Plugins:INFO: dll-files: Found 1 file DLLs from numpy installation. Nuitka-Plugins:INFO: dll-files: Found 1 file DLLs from tensorflow.lite.experimental.microfrontend.python.ops.audio_microfrontend_op installation. Nuitka:INFO: Keeping build directory ‘new_model.build’. Nuitka:INFO: Successfully created ‘new_model.dist\new_model.exe’.