unsloth: TypeError: unsupported operand type(s) for /: 'Tensor' and 'NoneType' when full finetuning gemma3
Version
pip install unsloth
pip install git+https://github.com/huggingface/transformers@v4.49.0-Gemma-3
Code
from unsloth import FastModel
import torch
model, tokenizer = FastModel.from_pretrained(
model_name = "unsloth/gemma-3-4b-it",
max_seq_length = 4096, # Choose any for long context!
load_in_4bit = False, # 4 bit quantization to reduce memory
load_in_8bit = False, # [NEW!] A bit more accurate, uses 2x memory
full_finetuning = True, # [NEW!] We have full finetuning now!
dtype=torch.bfloat16,
)
from unsloth.chat_templates import get_chat_template
tokenizer = get_chat_template(
tokenizer,
chat_template = "gemma-3",
)
from datasets import load_dataset
dataset = load_dataset("mlabonne/FineTome-100k", split = "train")
from unsloth.chat_templates import standardize_data_formats
dataset = standardize_data_formats(dataset)
def apply_chat_template(examples):
texts = tokenizer.apply_chat_template(examples["conversations"])
return { "text" : texts }
dataset = dataset.map(apply_chat_template, batched = True)
from trl import SFTTrainer, SFTConfig
trainer = SFTTrainer(
model = model,
tokenizer = tokenizer,
train_dataset = dataset,
eval_dataset = None, # Can set up evaluation!
args = SFTConfig(
dataset_text_field = "text",
per_device_train_batch_size = 4,
gradient_accumulation_steps = 4, # Use GA to mimic batch size!
warmup_ratio = 0.05,
num_train_epochs = 2, # Set this for 1 full training run.
learning_rate = 5e-5, # Reduce to 2e-5 for long training runs
logging_steps = 1,
optim = "adamw_8bit",
weight_decay = 0.01,
lr_scheduler_type = "cosine",
seed = 3407,
report_to = "wandb", # Use this for WandB etc
bf16=True
),
)
from unsloth.chat_templates import train_on_responses_only
trainer = train_on_responses_only(
trainer,
instruction_part = "<start_of_turn>user\n",
response_part = "<start_of_turn>model\n",
)
trainer_stats = trainer.train()
model.save_pretrained("gemma-3")
tokenizer.save_pretrained("gemma-3")
Error
Traceback (most recent call last):
File "/workspace/sft_gemma3.py", line 107, in <module>
trainer_stats = trainer.train()
^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/dist-packages/transformers/trainer.py", line 2250, in train
return inner_training_loop(
^^^^^^^^^^^^^^^^^^^^
File "<string>", line 311, in _fast_inner_training_loop
File "<string>", line 31, in _unsloth_training_step
File "/workspace/unsloth_compiled_cache/UnslothSFTTrainer.py", line 750, in compute_loss
outputs = super().compute_loss(
^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/dist-packages/unsloth/models/_utils.py", line 1028, in _unsloth_pre_compute_loss
outputs = self._old_compute_loss(model, inputs, *args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/dist-packages/transformers/trainer.py", line 3772, in compute_loss
outputs = model(**inputs)
^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/dist-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/dist-packages/torch/nn/modules/module.py", line 1750, in _call_impl
return forward_call(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/dist-packages/accelerate/utils/operations.py", line 819, in forward
return model_forward(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/dist-packages/accelerate/utils/operations.py", line 807, in __call__
return convert_to_fp32(self.model_forward(*args, **kwargs))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/dist-packages/torch/amp/autocast_mode.py", line 44, in decorate_autocast
return func(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/dist-packages/unsloth_zoo/temporary_patches.py", line 252, in forward
outputs = self.language_model(
^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/dist-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/dist-packages/torch/nn/modules/module.py", line 1750, in _call_impl
return forward_call(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/unsloth_compiled_cache/unsloth_compiled_module_gemma3.py", line 722, in forward
return Gemma3ForCausalLM_forward(self, input_ids, attention_mask, position_ids, past_key_values, inputs_embeds, labels, use_cache, output_attentions, output_hidden_states, return_dict, cache_position, logits_to_keep, **loss_kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/dist-packages/transformers/utils/deprecation.py", line 172, in wrapped_func
return func(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^
File "/workspace/unsloth_compiled_cache/unsloth_compiled_module_gemma3.py", line 637, in Gemma3ForCausalLM_forward
loss = _compiled_loss_function(
^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/dist-packages/torch/_dynamo/eval_frame.py", line 574, in _fn
return fn(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^
File "/workspace/unsloth_compiled_cache/unsloth_compiled_module_gemma3.py", line 597, in _compiled_loss_function
output_logits = output_logits / logit_softcapping
~~~~~~~~~~~~~~^~~~~~~~~~~~~~~~~~~
TypeError: unsupported operand type(s) for /: 'Tensor' and 'NoneType'
Description
I tried to full-fine tune gemma-3-4b-it model whose type is bfloat16. But the logit_softcapping is null so it seemed the error cause.
when loading model config like this:
model, tokenizer = FastModel.from_pretrained(
model_name = "unsloth/gemma-3-4b-it",
max_seq_length = 4096, # Choose any for long context!
load_in_4bit = False, # 4 bit quantization to reduce memory
load_in_8bit = False, # [NEW!] A bit more accurate, uses 2x memory
full_finetuning = True, # [NEW!] We have full finetuning now!
dtype=torch.bfloat16,
)
model.config.text_config
Gemma3TextConfig {
"attention_bias": false,
"attention_dropout": 0.0,
"attn_logit_softcapping": null,
"bos_token_id": 2,
"cache_implementation": "hybrid",
"eos_token_id": 1,
"final_logit_softcapping": null,
"head_dim": 256,
"hidden_activation": "gelu_pytorch_tanh",
"hidden_size": 2560,
"initializer_range": 0.02,
"intermediate_size": 10240,
"max_position_embeddings": 131072,
"model_type": "gemma3_text",
"num_attention_heads": 8,
"num_hidden_layers": 34,
"num_key_value_heads": 4,
"pad_token_id": 0,
"query_pre_attn_scalar": 256,
"rms_norm_eps": 1e-06,
"rope_local_base_freq": 10000.0,
"rope_scaling": {
"factor": 8.0,
"rope_type": "linear"
},
"rope_theta": 1000000.0,
"sliding_window": 1024,
"sliding_window_pattern": 6,
"torch_dtype": "bfloat16",
"transformers_version": "4.50.0.dev0",
"use_cache": true,
"vocab_size": 262208
}
there is no value on final_logit_softcapping. So it seemed to be cause. but I don’t know what to do. The only thing I tried is setting the value to 0 like this:
model.config.text_config.final_logit_softcapping = 0
Buti it didn’t work. What’s this error? How can I solve it?
+)
I found this error reproduce when full_finetuning = True set. I want full finetuning rather than lora
About this issue
- Original URL
- State: closed
- Created 2 years ago
- Reactions: 7
- Comments: 21
The code works for me.
@sangmandu @tellsaqib @stabgan @BenBatsir @Eastha0526 @kldzj @cslin0915 @ewianda @rosgar @hexixiang @chuangzhidan @anushavenkatachalam @asmitnayak
This should be solved now by the PR pushed by mathew. It’s now merged into the package. I will close this github issue as completed for now. PLease update your unsloth-zoo and unsloth installations and test again. Feel free to reopen if the issue isn’t solved for you yet.
@sangmandu It works for when full_finetuning is True
@cslin0915 sorry but we are talking when
full_finetuningisTrue