tensorflow: Keras Hub Model Conversion Fails
1. System information
- Ubuntu 18.04, python 3.12.3
- tf installed via
uv pip tensorflow==2.18.1, keras==3.9.0
2. Code
# %%
import os
os.environ['KERAS_BACKEND'] = 'tensorflow' # Set the models from keras_hub to use tensorflow
# %%
import tensorflow as tf
import tensorflow.keras as keras
import keras_hub
from PIL import Image
from IPython.display import display
import numpy as np
# %%
# %%
# Input variables here
# See https://keras.io/keras_hub/presets/
MODEL_NAME = "efficientnet_b0_ra4_e3600_r224_imagenet"
# Path to training data
TRAINING_DATA_PATH = ...
# How many classes we want to classify
NUM_CLASSES = 2
## Training-related tuning
# How many GPUs to use, assumes they are named GPU:0, GPU:1, etc.
NUM_GPUS = 4
BATCH_SIZE = 32 # Per GPU
NUM_EPOCHS = 10
NUM_AUGS = 2 # Number of random augmentations to apply to each image
AUG_STRENGTH = 0.5 # How strong the augmentations should be
# Whether to quantize per-tensor or per-channel
PER_TENSOR_TFLITE = False
IMAGE_SIZE = 224
GLOBAL_BATCH = NUM_GPUS * BATCH_SIZE
# %%
# Create a MirroredStrategy.
gpus = [f"GPU:{i}" for i in range(NUM_GPUS)]
strategy = tf.distribute.MirroredStrategy(gpus)
print("Number of devices: {}".format(strategy.num_replicas_in_sync))
# Open a strategy scope.
with strategy.scope():
# Everything that creates variables should be under the strategy scope.
# In general this is only model construction & `compile()`.
image_classifier = keras_hub.models.ImageClassifier.from_preset(
MODEL_NAME,
activation=None, # outputs logits
num_classes=NUM_CLASSES
)
image_classifier.compile(
optimizer=keras.optimizers.Adam(
learning_rate=1e-4,
clipnorm=1.0
),
loss=keras.losses.SparseCategoricalCrossentropy(from_logits=True), # Consumes logits
metrics=[keras.metrics.SparseCategoricalAccuracy()]
)
# %%
# Create dataset
train_dataset, val_dataset = keras.preprocessing.image_dataset_from_directory(
TRAINING_DATA_PATH,
labels="inferred",
label_mode="int",
batch_size=None,
image_size=(IMAGE_SIZE, IMAGE_SIZE),
shuffle=True,
validation_split=0.2,
subset="both",
interpolation="bilinear",
verbose=True,
seed=42
)
# Define simple augmenter
aug_layer = keras.layers.RandAugment(value_range=(0, 255), num_ops=NUM_AUGS, factor=AUG_STRENGTH)
map_fn = lambda x, y: (aug_layer(x, training=True), y)
# Prefetch the data, adding augmentation to the training dataset
train_dataset = train_dataset.batch(GLOBAL_BATCH, drop_remainder=True).map(map_fn, num_parallel_calls=tf.data.AUTOTUNE).prefetch(tf.data.AUTOTUNE)
val_dataset = val_dataset.batch(GLOBAL_BATCH, drop_remainder=True).prefetch(tf.data.AUTOTUNE)
# %%
# %%
# Freeze the base model to only retrain the head (optional)
# image_classifier.backbone.trainable = False
# Train the model on all available devices.
image_classifier.fit(train_dataset, epochs=NUM_EPOCHS, validation_data=val_dataset)
# Test the model on all available devices.
# TODO: Need a test dataset
# image_classifier.evaluate(test_dataset)
image_classifier.save(f"{MODEL_NAME}.keras")
# %%
# Test the trained model on 10 validation images
# This works fine, getting at least 8 correct
for imgs, label in val_dataset.take(1):
for i in range(10):
img = imgs[i:i+1]
preds = image_classifier.predict(img)
print(f"Raw preds: {preds}, Predicted: {preds.argmax()}, Actual: {label[i]}")
# %%
# Remove preprocessor
model_preprocessor = image_classifier.preprocessor
image_classifier.preprocessor = None
# %%
### Create end-to-end model that takes in UINT8 image of size 224 ###
# Modify preprocessor, because GPU doesn't like bicubic? idk
cfg = model_preprocessor.get_config()
cfg["image_converter"]["config"]["interpolation"] = "bilinear"
new_preproc = keras_hub.models.ImageClassifierPreprocessor.from_config(cfg)
# Input layer is (1, 224, 224, 3) uint8
input_layer = keras.layers.Input(shape=(224, 224, 3), dtype=tf.uint8, batch_size=1)
# Convert to float 32 like the preprocessor expects (I'd like to skip this if possible,
# because it might be part of why the tflite conversion is failing,
# but I don't know how without dismantling the preprocessor)
float_input = keras.layers.Lambda(lambda x: tf.cast(x, tf.float32))(input_layer)
# Preprocess the image
preprocessed_input = new_preproc(float_input, training=False)
# Run through the model - no double-preprocessing since we removed it
logits = image_classifier(preprocessed_input, training=False)
# Combine into one model
full_model = keras.models.Model(inputs=float_input, outputs=logits)
# Compile the model (unclear if necessary)
full_model.compile(
optimizer=keras.optimizers.Adam(1e-4),
loss=keras.losses.SparseCategoricalCrossentropy(from_logits=True), # Consumes logits
metrics=[keras.metrics.SparseCategoricalAccuracy()]
)
full_model.trainable = False
# %%
# Test the new model on 10 validation images. This works too, roughly same
# accuracy as above
num_correct = 0
num_wrong = 0
for imgs, label in val_dataset.take(1):
for i in range(GLOBAL_BATCH):
img = imgs[i:i+1]
# preds = full_model.predict(tf.cast(img, tf.uint8))
preds = full_model.predict(img)
print(f"Raw preds: {preds}, Predicted: {preds.argmax()}, Actual: {label[i]}")
if preds.argmax() == label[i]:
num_correct += 1
else:
num_wrong += 1
print(f"Correct: {num_correct}, Wrong: {num_wrong}")
# %%
# Use data from the validation dataset to calibrate the quantized model
def representative_data_gen():
batch_count = (300 // GLOBAL_BATCH) + 1
run = 0
for x, _ in val_dataset.take(batch_count): # Use a few samples
run += 1
print(f"Running batch {run}/{batch_count}")
# Pull one image at a time
for i in range(GLOBAL_BATCH):
img = x[i:i+1]
yield [tf.cast(img, tf.uint8)]
yield [img]
# Convert to tflite, using per-tensor quantization for ARTPEC-8
converter = tf.lite.TFLiteConverter.from_keras_model(full_model)
converter.optimizations = [tf.lite.Optimize.DEFAULT]
converter.representative_dataset = representative_data_gen
converter.target_spec.supported_ops = [tf.lite.OpsSet.TFLITE_BUILTINS_INT8]
converter.inference_input_type = tf.uint8
converter.inference_output_type = tf.float32
if PER_TENSOR_TFLITE:
converter._experimental_disable_per_channel = True
tflite_quant_model = converter.convert()
with open(f"{MODEL_NAME}.tflite", "wb") as f:
f.write(tflite_quant_model)
# %%
# Run the TFLite model on some test data
interpreter = tf.lite.Interpreter(model_content=tflite_quant_model)
input_details = interpreter.get_input_details()
output_details = interpreter.get_output_details()
input_index = input_details[0]["index"]
output_index = output_details[0]["index"]
interpreter.allocate_tensors()
# %%
num_correct = 0
num_wrong = 0
for x, label in val_dataset.take(3):
for i in range(GLOBAL_BATCH):
img = x[i:i+1]
# Need to cast to uint8 because tflite model expects it
interpreter.set_tensor(input_index, tf.cast(img, tf.uint8))
interpreter.invoke()
output_data = interpreter.get_tensor(output_index)
print(f"Raw preds: {output_data}, Predicted: {output_data.argmax()}, Actual: {label[i]}")
if output_data.argmax() == label[i]:
num_correct += 1
else:
num_wrong += 1
print(f"{num_correct=}, {num_wrong=}")
3. Failure after conversion
Model accuracy drastically drops after conversion:
Raw preds: [[-0.08086799 0.20216998]], Predicted: 1, Actual: 0
Raw preds: [[ 0. -0.040434]], Predicted: 0, Actual: 1
Raw preds: [[-1.0512838 0.48520795]], Predicted: 1, Actual: 0
Raw preds: [[-1.0108498 0.36390597]], Predicted: 1, Actual: 1
Raw preds: [[-0.9704159 0.12130199]], Predicted: 1, Actual: 0
Raw preds: [[-0.72781193 0.24260397]], Predicted: 1, Actual: 1
Raw preds: [[-0.08086799 -0.24260397]], Predicted: 0, Actual: 0
Raw preds: [[ 0.16173598 -0.040434 ]], Predicted: 0, Actual: 0
Raw preds: [[-1.1321518 0.040434 ]], Predicted: 1, Actual: 0
...
Raw preds: [[-0.24260397 0.12130199]], Predicted: 1, Actual: 1
Raw preds: [[-0.76824594 -0.08086799]], Predicted: 1, Actual: 0
Raw preds: [[-0.36390597 0.24260397]], Predicted: 1, Actual: 1
num_correct=196, num_wrong=188
About this issue
- Original URL
- State: closed
- Created 2 years ago
- Comments: 18 (4 by maintainers)
Having better luck moving over to tensorflow_hub. Closing this as no longer relevant.