tensorflow: Keras Hub Model Conversion Fails

1. System information

  • Ubuntu 18.04, python 3.12.3
  • tf installed via uv pip
  • tensorflow==2.18.1, keras==3.9.0

2. Code

# %%
import os
os.environ['KERAS_BACKEND'] = 'tensorflow'  # Set the models from keras_hub to use tensorflow

# %%
import tensorflow as tf
import tensorflow.keras as keras
import keras_hub
from PIL import Image
from IPython.display import display
import numpy as np

# %%

# %%
# Input variables here
# See https://keras.io/keras_hub/presets/
MODEL_NAME = "efficientnet_b0_ra4_e3600_r224_imagenet"
# Path to training data
TRAINING_DATA_PATH = ...
# How many classes we want to classify
NUM_CLASSES = 2
## Training-related tuning
# How many GPUs to use, assumes they are named GPU:0, GPU:1, etc.
NUM_GPUS = 4
BATCH_SIZE = 32  # Per GPU
NUM_EPOCHS = 10
NUM_AUGS = 2  # Number of random augmentations to apply to each image
AUG_STRENGTH = 0.5  # How strong the augmentations should be
# Whether to quantize per-tensor or per-channel
PER_TENSOR_TFLITE = False 
IMAGE_SIZE = 224  
GLOBAL_BATCH = NUM_GPUS * BATCH_SIZE

# %%
# Create a MirroredStrategy.
gpus = [f"GPU:{i}" for i in range(NUM_GPUS)]
strategy = tf.distribute.MirroredStrategy(gpus)
print("Number of devices: {}".format(strategy.num_replicas_in_sync))

# Open a strategy scope.
with strategy.scope():
    # Everything that creates variables should be under the strategy scope.
    # In general this is only model construction & `compile()`.
    image_classifier = keras_hub.models.ImageClassifier.from_preset(
        MODEL_NAME,
        activation=None,  # outputs logits
        num_classes=NUM_CLASSES
    )
    image_classifier.compile(
        optimizer=keras.optimizers.Adam(
            learning_rate=1e-4, 
            clipnorm=1.0
        ),
        loss=keras.losses.SparseCategoricalCrossentropy(from_logits=True),  # Consumes logits
        metrics=[keras.metrics.SparseCategoricalAccuracy()]
    )

# %%
# Create dataset
train_dataset, val_dataset = keras.preprocessing.image_dataset_from_directory(
    TRAINING_DATA_PATH,
    labels="inferred",
    label_mode="int",
    batch_size=None,
    image_size=(IMAGE_SIZE, IMAGE_SIZE),
    shuffle=True,
    validation_split=0.2,
    subset="both",
    interpolation="bilinear",
    verbose=True,
    seed=42
)

# Define simple augmenter
aug_layer = keras.layers.RandAugment(value_range=(0, 255), num_ops=NUM_AUGS, factor=AUG_STRENGTH)
map_fn = lambda x, y: (aug_layer(x, training=True), y)

# Prefetch the data, adding augmentation to the training dataset
train_dataset = train_dataset.batch(GLOBAL_BATCH, drop_remainder=True).map(map_fn, num_parallel_calls=tf.data.AUTOTUNE).prefetch(tf.data.AUTOTUNE)
val_dataset = val_dataset.batch(GLOBAL_BATCH, drop_remainder=True).prefetch(tf.data.AUTOTUNE)

# %%

# %%
# Freeze the base model to only retrain the head (optional)
# image_classifier.backbone.trainable = False
# Train the model on all available devices.
image_classifier.fit(train_dataset, epochs=NUM_EPOCHS, validation_data=val_dataset)

# Test the model on all available devices.
# TODO: Need a test dataset
# image_classifier.evaluate(test_dataset)
image_classifier.save(f"{MODEL_NAME}.keras")

# %%
# Test the trained model on 10 validation images
# This works fine, getting at least 8 correct
for imgs, label in val_dataset.take(1):
    for i in range(10):
        img = imgs[i:i+1]
        preds = image_classifier.predict(img)
        print(f"Raw preds: {preds}, Predicted: {preds.argmax()}, Actual: {label[i]}")

# %%
# Remove preprocessor
model_preprocessor = image_classifier.preprocessor
image_classifier.preprocessor = None

# %%
### Create end-to-end model that takes in UINT8 image of size 224 ###
# Modify preprocessor, because GPU doesn't like bicubic? idk
cfg = model_preprocessor.get_config()
cfg["image_converter"]["config"]["interpolation"] = "bilinear"
new_preproc = keras_hub.models.ImageClassifierPreprocessor.from_config(cfg)
# Input layer is (1, 224, 224, 3) uint8
input_layer = keras.layers.Input(shape=(224, 224, 3), dtype=tf.uint8, batch_size=1)
# Convert to float 32 like the preprocessor expects (I'd like to skip this if possible,
# because it might be part of why the tflite conversion is failing,
# but I don't know how without dismantling the preprocessor)
float_input = keras.layers.Lambda(lambda x: tf.cast(x, tf.float32))(input_layer)
# Preprocess the image
preprocessed_input = new_preproc(float_input, training=False)
# Run through the model - no double-preprocessing since we removed it
logits = image_classifier(preprocessed_input, training=False)
# Combine into one model
full_model = keras.models.Model(inputs=float_input, outputs=logits)
# Compile the model (unclear if necessary)
full_model.compile(
    optimizer=keras.optimizers.Adam(1e-4),
    loss=keras.losses.SparseCategoricalCrossentropy(from_logits=True),  # Consumes logits
    metrics=[keras.metrics.SparseCategoricalAccuracy()]
)
full_model.trainable = False

# %%
# Test the new model on 10 validation images. This works too, roughly same
# accuracy as above
num_correct = 0
num_wrong = 0
for imgs, label in val_dataset.take(1):
    for i in range(GLOBAL_BATCH):
        img = imgs[i:i+1]
        # preds = full_model.predict(tf.cast(img, tf.uint8))
        preds = full_model.predict(img)
        print(f"Raw preds: {preds}, Predicted: {preds.argmax()}, Actual: {label[i]}")
        if preds.argmax() == label[i]:
            num_correct += 1
        else:
            num_wrong += 1
print(f"Correct: {num_correct}, Wrong: {num_wrong}")

# %%
# Use data from the validation dataset to calibrate the quantized model
def representative_data_gen():
    batch_count = (300 // GLOBAL_BATCH) + 1
    run = 0
    for x, _ in val_dataset.take(batch_count):  # Use a few samples
        run += 1
        print(f"Running batch {run}/{batch_count}")
        # Pull one image at a time
        for i in range(GLOBAL_BATCH):
            img = x[i:i+1]
            yield [tf.cast(img, tf.uint8)]
            yield [img]
# Convert to tflite, using per-tensor quantization for ARTPEC-8
converter = tf.lite.TFLiteConverter.from_keras_model(full_model)
converter.optimizations = [tf.lite.Optimize.DEFAULT]
converter.representative_dataset = representative_data_gen
converter.target_spec.supported_ops = [tf.lite.OpsSet.TFLITE_BUILTINS_INT8]
converter.inference_input_type = tf.uint8
converter.inference_output_type = tf.float32
if PER_TENSOR_TFLITE:
    converter._experimental_disable_per_channel = True
tflite_quant_model = converter.convert()
with open(f"{MODEL_NAME}.tflite", "wb") as f:
    f.write(tflite_quant_model)

# %%
# Run the TFLite model on some test data
interpreter = tf.lite.Interpreter(model_content=tflite_quant_model)
input_details = interpreter.get_input_details()
output_details = interpreter.get_output_details()
input_index = input_details[0]["index"]
output_index = output_details[0]["index"]
interpreter.allocate_tensors()

# %%
num_correct = 0
num_wrong = 0
for x, label in val_dataset.take(3):
    for i in range(GLOBAL_BATCH):
        img = x[i:i+1]
        # Need to cast to uint8 because tflite model expects it
        interpreter.set_tensor(input_index, tf.cast(img, tf.uint8))
        interpreter.invoke()
        output_data = interpreter.get_tensor(output_index)
        print(f"Raw preds: {output_data}, Predicted: {output_data.argmax()}, Actual: {label[i]}")
        if output_data.argmax() == label[i]:
            num_correct += 1
        else:
            num_wrong += 1
print(f"{num_correct=}, {num_wrong=}")

3. Failure after conversion

Model accuracy drastically drops after conversion:

Raw preds: [[-0.08086799  0.20216998]], Predicted: 1, Actual: 0
Raw preds: [[ 0.       -0.040434]], Predicted: 0, Actual: 1
Raw preds: [[-1.0512838   0.48520795]], Predicted: 1, Actual: 0
Raw preds: [[-1.0108498   0.36390597]], Predicted: 1, Actual: 1
Raw preds: [[-0.9704159   0.12130199]], Predicted: 1, Actual: 0
Raw preds: [[-0.72781193  0.24260397]], Predicted: 1, Actual: 1
Raw preds: [[-0.08086799 -0.24260397]], Predicted: 0, Actual: 0
Raw preds: [[ 0.16173598 -0.040434  ]], Predicted: 0, Actual: 0
Raw preds: [[-1.1321518  0.040434 ]], Predicted: 1, Actual: 0
...
Raw preds: [[-0.24260397  0.12130199]], Predicted: 1, Actual: 1
Raw preds: [[-0.76824594 -0.08086799]], Predicted: 1, Actual: 0
Raw preds: [[-0.36390597  0.24260397]], Predicted: 1, Actual: 1

num_correct=196, num_wrong=188

About this issue

  • Original URL
  • State: closed
  • Created 2 years ago
  • Comments: 18 (4 by maintainers)

Most upvoted comments

Having better luck moving over to tensorflow_hub. Closing this as no longer relevant.