unsloth: GRPO training error

I’m training Llama-3.2-1B-Instruct at commit https://github.com/unslothai/unsloth/commit/2c0f50160e227936e0011d67e3bc2472c2089629 and my code is from https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Llama3.1_(8B)-GRPO.ipynb just change model to Llama-3.2-1B-Instruct since I don’t have much sources

I’m running in a docker environment with CUDA=12.1

torch 2.5.1 unsloth 2025.2.15 unsloth_zoo 2025.2.7

commit https://github.com/unslothai/unsloth/commit/512fec6a7b77a930b85a5b5685bf056fbb29ff5e works for me commit https://github.com/unslothai/unsloth/commit/179840d3a7b49188c372b56c67c4290d53c29ed6 still have save error

here is my code:

from unsloth import FastLanguageModel, PatchFastRL
PatchFastRL("GRPO", FastLanguageModel)


from unsloth import is_bfloat16_supported
import torch
max_seq_length = 512 # Can increase for longer reasoning traces
lora_rank = 8 # Larger rank = smarter, but slower

model, tokenizer = FastLanguageModel.from_pretrained(
    model_name = "../llm_test/many_test/models/Llama-3.2-1B-Instruct/",
    # model_name = "../llm_test/many_test/models/Qwen2.5-0.5B-Instruct/",
    max_seq_length = max_seq_length,
    load_in_4bit = True, # False for LoRA 16bit
    fast_inference = False, # Enable vLLM fast inference
    max_lora_rank = lora_rank,
    gpu_memory_utilization = 0.6, # Reduce if out of memory
)

model = FastLanguageModel.get_peft_model(
    model,
    r = lora_rank, # Choose any number > 0 ! Suggested 8, 16, 32, 64, 128
    target_modules = [
        "q_proj", "k_proj", "v_proj", "o_proj",
        "gate_proj", "up_proj", "down_proj",
    ], # Remove QKVO if out of memory
    lora_alpha = lora_rank,
    use_gradient_checkpointing = "unsloth", # Enable long context finetuning
    random_state = 3407,
)


import re
from datasets import load_dataset, Dataset
from modelscope.msdatasets import MsDataset

# Load and prep dataset
SYSTEM_PROMPT = """
Respond in the following format:
<reasoning>
...
</reasoning>
<answer>
...
</answer>
"""

XML_COT_FORMAT = """\
<reasoning>
{reasoning}
</reasoning>
<answer>
{answer}
</answer>
"""

def extract_xml_answer(text: str) -> str:
    answer = text.split("<answer>")[-1]
    answer = answer.split("</answer>")[0]
    return answer.strip()

def extract_hash_answer(text: str) -> str | None:
    if "####" not in text:
        return None
    return text.split("####")[1].strip()

# uncomment middle messages for 1-shot prompting
def get_gsm8k_questions(split = "train") -> Dataset:
    # data = load_dataset('openai/gsm8k', 'main')[split] # type: ignore
    data =  MsDataset.load('modelscope/gsm8k', subset_name='main', split=split)
    data = data.map(lambda x: { # type: ignore
        'prompt': [
            {'role': 'system', 'content': SYSTEM_PROMPT},
            {'role': 'user', 'content': x['question']}
        ],
        'answer': extract_hash_answer(x['answer'])
    }) # type: ignore
    return data # type: ignore

dataset = get_gsm8k_questions()

# Reward functions
def correctness_reward_func(prompts, completions, answer, **kwargs) -> list[float]:
    responses = [completion[0]['content'] for completion in completions]
    q = prompts[0][-1]['content']
    extracted_responses = [extract_xml_answer(r) for r in responses]
    print('-'*20, f"Question:\n{q}", f"\nAnswer:\n{answer[0]}", f"\nResponse:\n{responses[0]}", f"\nExtracted:\n{extracted_responses[0]}")
    return [2.0 if r == a else 0.0 for r, a in zip(extracted_responses, answer)]

def int_reward_func(completions, **kwargs) -> list[float]:
    responses = [completion[0]['content'] for completion in completions]
    extracted_responses = [extract_xml_answer(r) for r in responses]
    return [0.5 if r.isdigit() else 0.0 for r in extracted_responses]

def strict_format_reward_func(completions, **kwargs) -> list[float]:
    """Reward function that checks if the completion has a specific format."""
    pattern = r"^<reasoning>\n.*?\n</reasoning>\n<answer>\n.*?\n</answer>\n$"
    responses = [completion[0]["content"] for completion in completions]
    matches = [re.match(pattern, r) for r in responses]
    return [0.5 if match else 0.0 for match in matches]

def soft_format_reward_func(completions, **kwargs) -> list[float]:
    """Reward function that checks if the completion has a specific format."""
    pattern = r"<reasoning>.*?</reasoning>\s*<answer>.*?</answer>"
    responses = [completion[0]["content"] for completion in completions]
    matches = [re.match(pattern, r) for r in responses]
    return [0.5 if match else 0.0 for match in matches]

def count_xml(text) -> float:
    count = 0.0
    if text.count("<reasoning>\n") == 1:
        count += 0.125
    if text.count("\n</reasoning>\n") == 1:
        count += 0.125
    if text.count("\n<answer>\n") == 1:
        count += 0.125
        count -= len(text.split("\n</answer>\n")[-1])*0.001
    if text.count("\n</answer>") == 1:
        count += 0.125
        count -= (len(text.split("\n</answer>")[-1]) - 1)*0.001
    return count

def xmlcount_reward_func(completions, **kwargs) -> list[float]:
    contents = [completion[0]["content"] for completion in completions]
    return [count_xml(c) for c in contents]


from trl import GRPOConfig, GRPOTrainer
training_args = GRPOConfig(
    use_vllm = False, # use vLLM for fast inference!
    learning_rate = 5e-6,
    adam_beta1 = 0.9,
    adam_beta2 = 0.99,
    weight_decay = 0.1,
    warmup_ratio = 0.1,
    lr_scheduler_type = "cosine",
    optim = "paged_adamw_8bit",
    logging_steps = 1,
    bf16 = is_bfloat16_supported(),
    fp16 = not is_bfloat16_supported(),
    per_device_train_batch_size = 1,
    gradient_accumulation_steps = 1, # Increase to 4 for smoother training
    num_generations = 6, # Decrease if out of memory
    max_prompt_length = 256,
    max_completion_length = 200,
    # num_train_epochs = 1, # Set to 1 for a full training run
    max_steps =  250,
    save_steps = 250,
    max_grad_norm = 0.1,
    report_to = "none", # Can use Weights & Biases
    output_dir = "outputs",
)


trainer = GRPOTrainer(
    model = model,
    processing_class = tokenizer,
    reward_funcs = [
        xmlcount_reward_func,
        soft_format_reward_func,
        strict_format_reward_func,
        int_reward_func,
        correctness_reward_func,
    ],
    args = training_args,
    train_dataset = dataset,
)
trainer.train()

full log

root@c0410db6a918:/code/unsloth_20250226# python tmp.py 🦥 Unsloth: Will patch your computer to enable 2x faster free finetuning. 🦥 Unsloth Zoo will now patch everything to make training faster! INFO 02-26 09:35:07 init.py:190] Automatically detected platform cuda. ==((====))== Unsloth 2025.2.15: Fast Llama patching. Transformers: 4.49.0. \ /| GPU: NVIDIA GeForce RTX 3090. Max memory: 23.691 GB. Platform: Linux. O^O/ _/ \ Torch: 2.5.1+cu124. CUDA: 8.6. CUDA Toolkit: 12.4. Triton: 3.1.0 \ / Bfloat16 = TRUE. FA [Xformers = 0.0.29.post1. FA2 = True] “--" Free Apache license: http://github.com/unslothai/unsloth Unsloth: Fast downloading is enabled - ignore downloading bars which are red colored! …/llm_test/many_test/models/Llama-3.2-1B-Instruct/ does not have a padding token! Will use pad_token = <|finetune_right_pad_id|>. Unsloth 2025.2.15 patched 16 layers with 16 QKV layers, 16 O layers and 16 MLP layers. 2025-02-26 09:36:15,980 - modelscope - WARNING - Use trust_remote_code=True. Will invoke codes from gsm8k. Please make sure that you can trust the external codes. 2025-02-26 09:36:16,418 - modelscope - WARNING - Use trust_remote_code=True. Will invoke codes from modelscope/gsm8k. Please make sure that you can trust the external codes. 2025-02-26 09:36:16,418 - modelscope - WARNING - Use trust_remote_code=True. Will invoke codes from modelscope/gsm8k. Please make sure that you can trust the external codes. 2025-02-26 09:36:16,419 - modelscope - WARNING - Use trust_remote_code=True. Will invoke codes from modelscope/gsm8k. Please make sure that you can trust the external codes. Detected kernel version 4.15.0, which is below the recommended minimum of 5.5.0; this can cause the process to hang. It is recommended to upgrade the kernel to the minimum version or higher. ==((====))== Unsloth - 2x faster free finetuning | Num GPUs = 1 \ /| Num examples = 7,473 | Num Epochs = 1 O^O/ _/ \ Batch size per device = 1 | Gradient Accumulation steps = 1 \ / Total batch size = 1 | Total steps = 250 "--” Number of trainable parameters = 5,636,096 0%| | 0/250 [00:00<?, ?it/s]Traceback (most recent call last): File “/code/unsloth_20250226/tmp.py”, line 168, in <module> trainer.train() File “/opt/conda/lib/python3.11/site-packages/transformers/trainer.py”, line 2241, in train return inner_training_loop( ^^^^^^^^^^^^^^^^^^^^ File “<string>”, line 329, in _fast_inner_training_loop File “<string>”, line 31, in _unsloth_training_step File “/code/unsloth_20250226/unsloth_compiled_cache/UnslothGRPOTrainer.py”, line 766, in compute_loss prompt_ids, prompt_mask = inputs[“prompt_ids”], inputs[“prompt_mask”] ~~~~~~^^^^^^^^^^^^^^ TypeError: list indices must be integers or slices, not str 0%| | 0/250 [00:00<?, ?it/s]

how can i fix this?

About this issue

  • Original URL
  • State: open
  • Created 2 years ago
  • Comments: 15 (4 by maintainers)

Most upvoted comments

I encountered the same issue as you did. I checked all the installation versions on the official Colab and ensured that they were consistent, but the problem still persisted. Eventually, I set vllm_cache=True and found that the model could run normally and generate proper sequences. To be more specific, the settings are as follows:

model, tokenizer = FastLanguageModel.from_pretrained(
        model_name=model_path,
        max_seq_length=max_seq_length,
        load_in_4bit=True,        
        fast_inference=True,      # set True if you want vLLM fast inference
        max_lora_rank=lora_rank,
        gpu_memory_utilization=0.7
    )

training_args_lyrics = GRPOConfig(
    use_vllm = True, # use vLLM for fast inference!
    learning_rate = 5e-6,
    adam_beta1 = 0.9,
    adam_beta2 = 0.99,
    weight_decay = 0.1,
    warmup_ratio = 0.1,
    lr_scheduler_type = "cosine",
    optim = "paged_adamw_8bit",
    logging_steps = 1,
    bf16 = is_bfloat16_supported(),
    fp16 = not is_bfloat16_supported(),
    per_device_train_batch_size = 4,
    gradient_accumulation_steps = 2, # Increase to 4 for smoother training
    num_generations = 8, # Decrease if out of memory
    max_prompt_length = 768,
    max_completion_length = 768,
    num_train_epochs = 2, # Set to 1 for a full training run
    # max_steps = 50,
    save_steps = 250,
    max_grad_norm = 0.1,
    report_to = "none", # Can use Weights & Biases
    output_dir = "outputs_lyrics_phase",
)

With these settings, the program runs smoothly. It seems that the current models only support vllm-based gradient backpropagation. Without enabling vllm_cache, the first batch of data might be normal, but subsequent batches often encounter repetitive issues. However, once vllm_cache is turned on, the aforementioned problems are resolved!

I encountered the same issue as you did. I checked all the installation versions on the official Colab and ensured that they were consistent, but the problem still persisted. Eventually, I set vllm_cache=True and found that the model could run normally and generate proper sequences. To be more specific, the settings are as follows:

model, tokenizer = FastLanguageModel.from_pretrained( model_name=model_path, max_seq_length=max_seq_length, load_in_4bit=True,
fast_inference=True, # set True if you want vLLM fast inference max_lora_rank=lora_rank, gpu_memory_utilization=0.7 )

training_args_lyrics = GRPOConfig( use_vllm = True, # use vLLM for fast inference! learning_rate = 5e-6, adam_beta1 = 0.9, adam_beta2 = 0.99, weight_decay = 0.1, warmup_ratio = 0.1, lr_scheduler_type = “cosine”, optim = “paged_adamw_8bit”, logging_steps = 1, bf16 = is_bfloat16_supported(), fp16 = not is_bfloat16_supported(), per_device_train_batch_size = 4, gradient_accumulation_steps = 2, # Increase to 4 for smoother training num_generations = 8, # Decrease if out of memory max_prompt_length = 768, max_completion_length = 768, num_train_epochs = 2, # Set to 1 for a full training run # max_steps = 50, save_steps = 250, max_grad_norm = 0.1, report_to = “none”, # Can use Weights & Biases output_dir = “outputs_lyrics_phase”, )

With these settings, the program runs smoothly. It seems that the current models only support vllm-based gradient backpropagation. Without enabling vllm_cache, the first batch of data might be normal, but subsequent batches often encounter repetitive issues. However, once vllm_cache is turned on, the aforementioned problems are resolved!

after update trl==0.15.2 and set use_vllm=True, everything looks good so far. Thank you!

The issue happens to be within the python version you are using. If you use python 3.11 it will work. But it is possible that you will have an issue with the import of “_lzma”. Follow the answer in this with the right python version. Should fix the issue.https://askubuntu.com/a/1501705