autokeras: StructuredDataClassifier's exported model can't predict or evaluate

Bug Description

I use autokeras to train a classifier, I can predict or evaluate with the <autokeras.tasks.structured_data.StructuredDataClassifier>. But the model exported by the export_model function can’t predict or evaluate. When I try model.evaluate(x_valid,y_valid)it raise the error below, and model.evaluate(x_valid,y_valid) raise the same error. No matter which dataset I use, the iteration ends at 32.

 32/2000 [..............................] - ETA: 48s
---------------------------------------------------------------------------
UnimplementedError                        Traceback (most recent call last)
<ipython-input-20-73f41c3caa9d> in <module>
----> 1 model.evaluate(x_valid,y_valid)

c:\users\exia\appdata\local\programs\python\python36\lib\site-packages\tensorflow_core\python\keras\engine\training.py in evaluate(self, x, y, batch_size, verbose, sample_weight, steps, callbacks, max_queue_size, workers, use_multiprocessing)
    928         max_queue_size=max_queue_size,
    929         workers=workers,
--> 930         use_multiprocessing=use_multiprocessing)
    931 
    932   def predict(self,

c:\users\exia\appdata\local\programs\python\python36\lib\site-packages\tensorflow_core\python\keras\engine\training_v2.py in evaluate(self, model, x, y, batch_size, verbose, sample_weight, steps, callbacks, max_queue_size, workers, use_multiprocessing, **kwargs)
    488         sample_weight=sample_weight, steps=steps, callbacks=callbacks,
    489         max_queue_size=max_queue_size, workers=workers,
--> 490         use_multiprocessing=use_multiprocessing, **kwargs)
    491 
    492   def predict(self, model, x, batch_size=None, verbose=0, steps=None,

c:\users\exia\appdata\local\programs\python\python36\lib\site-packages\tensorflow_core\python\keras\engine\training_v2.py in _model_iteration(self, model, mode, x, y, batch_size, verbose, sample_weight, steps, callbacks, max_queue_size, workers, use_multiprocessing, **kwargs)
    473               mode=mode,
    474               training_context=training_context,
--> 475               total_epochs=1)
    476           cbks.make_logs(model, epoch_logs, result, mode)
    477 

c:\users\exia\appdata\local\programs\python\python36\lib\site-packages\tensorflow_core\python\keras\engine\training_v2.py in run_one_epoch(model, iterator, execution_function, dataset_size, batch_size, strategy, steps_per_epoch, num_samples, mode, training_context, total_epochs)
    126         step=step, mode=mode, size=current_batch_size) as batch_logs:
    127       try:
--> 128         batch_outs = execution_function(iterator)
    129       except (StopIteration, errors.OutOfRangeError):
    130         # TODO(kaftan): File bug about tf function and errors.OutOfRangeError?

c:\users\exia\appdata\local\programs\python\python36\lib\site-packages\tensorflow_core\python\keras\engine\training_v2_utils.py in execution_function(input_fn)
     96     # `numpy` translates Tensors to values in Eager mode.
     97     return nest.map_structure(_non_none_constant_value,
---> 98                               distributed_function(input_fn))
     99 
    100   return execution_function

c:\users\exia\appdata\local\programs\python\python36\lib\site-packages\tensorflow_core\python\eager\def_function.py in __call__(self, *args, **kwds)
    566         xla_context.Exit()
    567     else:
--> 568       result = self._call(*args, **kwds)
    569 
    570     if tracing_count == self._get_tracing_count():

c:\users\exia\appdata\local\programs\python\python36\lib\site-packages\tensorflow_core\python\eager\def_function.py in _call(self, *args, **kwds)
    636               *args, **kwds)
    637       # If we did not create any variables the trace we have is good enough.
--> 638       return self._concrete_stateful_fn._filtered_call(canon_args, canon_kwds)  # pylint: disable=protected-access
    639 
    640     def fn_with_cond(*inner_args, **inner_kwds):

c:\users\exia\appdata\local\programs\python\python36\lib\site-packages\tensorflow_core\python\eager\function.py in _filtered_call(self, args, kwargs)
   1609          if isinstance(t, (ops.Tensor,
   1610                            resource_variable_ops.BaseResourceVariable))),
-> 1611         self.captured_inputs)
   1612 
   1613   def _call_flat(self, args, captured_inputs, cancellation_manager=None):

c:\users\exia\appdata\local\programs\python\python36\lib\site-packages\tensorflow_core\python\eager\function.py in _call_flat(self, args, captured_inputs, cancellation_manager)
   1690       # No tape is watching; skip to running the function.
   1691       return self._build_call_outputs(self._inference_function.call(
-> 1692           ctx, args, cancellation_manager=cancellation_manager))
   1693     forward_backward = self._select_forward_and_backward_functions(
   1694         args,

c:\users\exia\appdata\local\programs\python\python36\lib\site-packages\tensorflow_core\python\eager\function.py in call(self, ctx, args, cancellation_manager)
    543               inputs=args,
    544               attrs=("executor_type", executor_type, "config_proto", config),
--> 545               ctx=ctx)
    546         else:
    547           outputs = execute.execute_with_cancellation(

c:\users\exia\appdata\local\programs\python\python36\lib\site-packages\tensorflow_core\python\eager\execute.py in quick_execute(op_name, num_outputs, inputs, attrs, ctx, name)
     65     else:
     66       message = e.message
---> 67     six.raise_from(core._status_to_exception(e.code, message), None)
     68   except TypeError as e:
     69     keras_symbolic_tensors = [

c:\users\exia\appdata\local\programs\python\python36\lib\site-packages\six.py in raise_from(value, from_value)

UnimplementedError:  Cast double to string is not supported
	 [[node Cast (defined at <ipython-input-20-73f41c3caa9d>:1) ]] [Op:__inference_distributed_function_1731]

Function call stack:
distributed_function

Reproducing Steps

clf = ak.StructuredDataClassifier(max_trials=10)
clf.fit(x_train, y_train,validation_data=(x_valid, y_valid))
model = clf.export_model()
model.evaluate(x_valid,y_valid)

where x_train is a (5000, 4) numpy.ndarray, y_train is a (5000, 1) numpy.ndarray x_valid is a (2000, 4) numpy.ndarray, y_valid is a (2000, 1) numpy.ndarray no matter I use model.evaluate(x_valid,y_valid) or model.evaluate(x_train,y_train) it raise the same error.

Setup Details

Include the details about the versions of:

  • OS type and version: Win10
  • Python: 3.6.5
  • autokeras: 1.0.1
  • scikit-learn:0.22
  • numpy:1.18.0
  • scipy:1.4.1
  • tensorflow:2.1.0

About this issue

  • Original URL
  • State: closed
  • Created 4 years ago
  • Comments: 21 (6 by maintainers)

Most upvoted comments

I mean this may still be a bug. We will test it out during we write the colab notebooks. And fix it.

I guess the bug may be related to the custom preprocessing layer we implemented for coverting the numerical features to categorical ones. That custom layer will be removed. We will use the official layers provided in tf2.2 to do the trick, which are not released yet.

We had a new release, autokeras 1.0.2. If you use it with the latest tf-nightly. This issue should be resolved.

I have upgraded the autokeras, but still have the same problem.

The above code snippet is still broken for me. I tested on Ubuntu and Mac OS.

Package versions:

autokeras==1.0.2
keras-tuner==1.0.1
tf-estimator-nightly==2.1.0.dev2020022109
tf-nightly==2.2.0.dev20200218

Output/error:

WARNING:tensorflow:A checkpoint was restored (e.g. tf.train.Checkpoint.restore or tf.keras.Model.load_weights) but not all checkpointed values were used. See above for specific issues. Use expect_partial() on the load status object, e.g. tf.train.Checkpoint.restore(...).expect_partial(), to silence these warnings, or use assert_consumed() to make the check explicit. See https://www.tensorflow.org/guide/checkpoint#loading_mechanics for details.
WARNING:tensorflow:Unresolved object in checkpoint: (root).optimizer.iter
WARNING:tensorflow:Unresolved object in checkpoint: (root).optimizer.beta_1
WARNING:tensorflow:Unresolved object in checkpoint: (root).optimizer.beta_2
WARNING:tensorflow:Unresolved object in checkpoint: (root).optimizer.decay
WARNING:tensorflow:Unresolved object in checkpoint: (root).optimizer.learning_rate
WARNING:tensorflow:A checkpoint was restored (e.g. tf.train.Checkpoint.restore or tf.keras.Model.load_weights) but not all checkpointed values were used. See above for specific issues. Use expect_partial() on the load status object, e.g. tf.train.Checkpoint.restore(...).expect_partial(), to silence these warnings, or use assert_consumed() to make the check explicit. See https://www.tensorflow.org/guide/checkpoint#loading_mechanics for details.
2020-02-21 18:37:51.015891: W tensorflow/core/framework/op_kernel.cc:1729] OP_REQUIRES failed at cast_op.cc:123 : Unimplemented: Cast double to string is not supported
Traceback (most recent call last):
  File ".../scratch_2.py", line 24, in <module>
    predicted_y_2 = model.predict(X_test)
  File ".../venv/lib/python3.7/site-packages/tensorflow/python/keras/engine/training.py", line 919, in predict
    use_multiprocessing=use_multiprocessing)
  File ".../venv/lib/python3.7/site-packages/tensorflow/python/keras/engine/training_v2.py", line 496, in predict
    workers=workers, use_multiprocessing=use_multiprocessing, **kwargs)
  File ".../venv/lib/python3.7/site-packages/tensorflow/python/keras/engine/training_v2.py", line 473, in _model_iteration
    total_epochs=1)
  File ".../venv/lib/python3.7/site-packages/tensorflow/python/keras/engine/training_v2.py", line 128, in run_one_epoch
    batch_outs = execution_function(iterator)
  File ".../venv/lib/python3.7/site-packages/tensorflow/python/keras/engine/training_v2_utils.py", line 98, in execution_function
    distributed_function(input_fn))
  File ".../venv/lib/python3.7/site-packages/tensorflow/python/eager/def_function.py", line 576, in __call__
    result = self._call(*args, **kwds)
  File ".../venv/lib/python3.7/site-packages/tensorflow/python/eager/def_function.py", line 646, in _call
    return self._concrete_stateful_fn._filtered_call(canon_args, canon_kwds)  # pylint: disable=protected-access
  File ".../venv/lib/python3.7/site-packages/tensorflow/python/eager/function.py", line 1660, in _filtered_call
    self.captured_inputs)
  File ".../venv/lib/python3.7/site-packages/tensorflow/python/eager/function.py", line 1741, in _call_flat
    ctx, args, cancellation_manager=cancellation_manager))
  File ".../venv/lib/python3.7/site-packages/tensorflow/python/eager/function.py", line 598, in call
    ctx=ctx)
  File ".../venv/lib/python3.7/site-packages/tensorflow/python/eager/execute.py", line 60, in quick_execute
    inputs, attrs, num_outputs)
tensorflow.python.framework.errors_impl.UnimplementedError:  Cast double to string is not supported
	 [[node Cast (defined at .../scratch_2.py:24) ]] [Op:__inference_distributed_function_3792]

Function call stack:
distributed_function

@haifeng-jin, could you test that code snippet with the above package versions and let me know if I am doing something wrong.

We had a new release, autokeras 1.0.2. If you use it with the latest tf-nightly. This issue should be resolved.