verl: [SGLang Async Rollout] CUDA error: an illegal memory access was encountered

I encountered the following CUDA error while using async SGLang for multiturn GRPO RL.

[2025-05-20 18:43:20 TP3] Scheduler hit an exception: Traceback (most recent call last):
  File "/home/jobuser/.local/lib/python3.10/site-packages/sglang/srt/managers/scheduler.py", line 2269, in run_scheduler_process
    scheduler.event_loop_overlap()
  File "/home/jobuser/.local/lib/python3.10/site-packages/torch/utils/_contextlib.py", line 116, in decorate_context
    return func(*args, **kwargs)
  File "/home/jobuser/.local/lib/python3.10/site-packages/sglang/srt/managers/scheduler.py", line 658, in event_loop_overlap
    batch = self.get_next_batch_to_run()
  File "/home/jobuser/.local/lib/python3.10/site-packages/sglang/srt/managers/scheduler.py", line 1304, in get_next_batch_to_run
    self.running_batch = self.update_running_batch(self.running_batch)
  File "/home/jobuser/.local/lib/python3.10/site-packages/sglang/srt/managers/scheduler.py", line 1502, in update_running_batch
    batch.prepare_for_decode()
  File "/home/jobuser/.local/lib/python3.10/site-packages/sglang/srt/managers/schedule_batch.py", line 1461, in prepare_for_decode
    locs = self.seq_lens.clone()
RuntimeError: CUDA error: an illegal memory access was encountered
CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.
For debugging consider passing CUDA_LAUNCH_BLOCKING=1
Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions.

I constantly see this error in the validation step before training if I use QwQ-32b or Qwen3-32b but everything worked well when training with Qwen2.5-32b-instruct. To verify if this is actually an OOM issue which is mentioned in SGLang doc, I reduced the validation dataset to only 1 data point but still hit the same error when tp size is 4 or 8. Here is the an overview of the GPU memory usage when the error happened, which does not seem like OOM to me(only ~65% GPU Mem utilization before failure):

Image

I am using up-to-date Verl and have the following dependencies installed:

torch==2.6.0+cu124
torch_memory_saver==0.0.6
transformers==4.51.1
sgl-kernel==0.1.2.post1
sglang==0.4.6.post4
ray==2.46.0
numpy==1.26.4
nvidia-cublas-cu12==12.4.5.8
nvidia-cuda-cupti-cu12==12.4.127
nvidia-cuda-nvrtc-cu12==12.4.127
nvidia-cuda-runtime-cu12==12.4.127
nvidia-cudnn-cu12==9.1.0.70
nvidia-cufft-cu12==11.2.1.3
nvidia-curand-cu12==10.3.5.147
nvidia-cusolver-cu12==11.6.1.9
nvidia-cusparse-cu12==12.3.1.170
nvidia-cusparselt-cu12==0.6.2
nvidia-ml-py==12.570.86
nvidia-nccl-cu12==2.21.5
nvidia-nvjitlink-cu12==12.4.127
nvidia-nvtx-cu12==12.4.127
flash_attn==2.7.4.post1+cu12torch2.6cxx11abiFALSE

Here is my full training config:

{
  "actor_rollout_ref": {
    "actor": {
      "checkpoint": {
        "contents": ["model", "optimizer", "extra"]
      },
      "clip_ratio": 0.2,
      "clip_ratio_c": 3.0,
      "clip_ratio_high": 0.2,
      "clip_ratio_low": 0.2,
      "entropy_coeff": 0,
      "fsdp_config": {
        "fsdp_size": -1,
        "offload_policy": false,
        "optimizer_offload": true,
        "param_offload": true,
        "reshard_after_forward": true,
        "wrap_policy": {"min_num_params": 0}
      },
      "grad_clip": 1.0,
      "kl_loss_coef": 0.001,
      "kl_loss_type": "low_var_kl",
      "loss_agg_mode": "token-mean",
      "optim": {
        "lr": 1e-6,
        "lr_warmup_steps": -1,
        "lr_warmup_steps_ratio": 0.0,
        "min_lr_ratio": 0.0,
        "num_cycles": 0.5,
        "total_training_steps": -1,
        "warmup_style": "constant",
        "weight_decay": 0.01
      },
      "ppo_epochs": 1,
      "ppo_max_token_len_per_gpu": 16384,
      "ppo_micro_batch_size": null,
      "ppo_micro_batch_size_per_gpu": 1,
      "ppo_mini_batch_size": 8,
      "shuffle": false,
      "strategy": "fsdp",
      "ulysses_sequence_parallel_size": 1,
      "use_dynamic_bsz": false,
      "use_kl_loss": true,
      "use_torch_compile": true
    },
    "hybrid_engine": true,
    "model": {
      "enable_gradient_checkpointing": true,
      "external_lib": null,
      "override_config": {},
      "path": "/shared/public/elr-models/Qwen/QwQ-32B/976055f8c83f394f35dbd3ab09a285a984907bd0",
      "trust_remote_code": false,
      "use_fused_kernels": false,
      "use_liger": false,
      "use_remove_padding": true
    },
    "ref": {
      "fsdp_config": {
        "param_offload": true,
        "reshard_after_forward": true,
        "wrap_policy": {"min_num_params": 0}
      },
      "log_prob_max_token_len_per_gpu": 16384,
      "log_prob_micro_batch_size": null,
      "log_prob_micro_batch_size_per_gpu": 32,
      "log_prob_use_dynamic_bsz": false,
      "strategy": "fsdp",
      "ulysses_sequence_parallel_size": 1,
      "use_torch_compile": true
    },
    "rollout": {
      "chat_scheduler": null,
      "disable_log_stats": true,
      "do_sample": true,
      "dtype": "bfloat16",
      "enable_chunked_prefill": true,
      "enforce_eager": true,
      "engine_kwargs": {"swap_space": null},
      "free_cache_engine": true,
      "gpu_memory_utilization": 0.6,
      "ignore_eos": false,
      "load_format": "dummy_dtensor",
      "log_prob_max_token_len_per_gpu": 16384,
      "log_prob_micro_batch_size": null,
      "log_prob_micro_batch_size_per_gpu": 32,
      "log_prob_use_dynamic_bsz": false,
      "max_model_len": null,
      "max_num_batched_tokens": 8192,
      "max_num_seqs": 1024,
      "mode": "sync",
      "multi_turn": {
        "enable": true,
        "format": "qwen",
        "max_turns": 5,
        "tool_config_path": null,
        "tool_list": "tools.tool_list"
      },
      "n": 8,
      "name": "sglang_async",
      "prompt_length": 4096,
      "response_length": 1024,
      "temperature": 1.0,
      "tensor_model_parallel_size": 8,
      "top_k": -1,
      "top_p": 1,
      "use_fire_sampling": false,
      "val_kwargs": {
        "do_sample": false,
        "n": 1,
        "temperature": 0,
        "top_k": -1,
        "top_p": 1.0
      }
    }
  },
  "algorithm": {
    "adv_estimator": "grpo",
    "gamma": 1.0,
    "kl_ctrl": {
      "horizon": 10000,
      "kl_coef": 0.001,
      "target_kl": 0.1,
      "type": "fixed"
    },
    "kl_penalty": "kl",
    "lam": 1.0,
    "norm_adv_by_std_in_grpo": true,
    "use_kl_in_reward": false
  },
  "critic": {
    "checkpoint": {"contents": ["model", "optimizer", "extra"]},
    "cliprange_value": 0.5,
    "forward_max_token_len_per_gpu": 32768,
    "forward_micro_batch_size": null,
    "forward_micro_batch_size_per_gpu": null,
    "grad_clip": 1.0,
    "model": {
      "enable_gradient_checkpointing": true,
      "external_lib": null,
      "fsdp_config": {
        "fsdp_size": -1,
        "offload_policy": false,
        "optimizer_offload": false,
        "param_offload": false,
        "reshard_after_forward": true,
        "wrap_policy": {"min_num_params": 0}
      },
      "override_config": {},
      "path": "~/models/deepseek-llm-7b-chat",
      "tokenizer_path": "/shared/public/elr-models/Qwen/QwQ-32B/976055f8c83f394f35dbd3ab09a285a984907bd0",
      "trust_remote_code": false,
      "use_remove_padding": false
    },
    "optim": {
      "lr": 1e-5,
      "lr_warmup_steps_ratio": 0.0,
      "min_lr_ratio": null,
      "total_training_steps": -1,
      "warmup_style": "constant",
      "weight_decay": 0.01
    },
    "ppo_epochs": 1,
    "ppo_max_token_len_per_gpu": 32768,
    "ppo_micro_batch_size": null,
    "ppo_micro_batch_size_per_gpu": null,
    "ppo_mini_batch_size": 8,
    "rollout_n": 8,
    "shuffle": false,
    "strategy": "fsdp",
    "ulysses_sequence_parallel_size": 1,
    "use_dynamic_bsz": false
  },
  "custom_reward_function": {
    "name": "compute_score",
    "path": "...",
    "reward_kwargs": {}
  },
  "data": {
    "custom_cls": {"name": null, "path": null},
    "filter_overlong_prompts": true,
    "filter_overlong_prompts_workers": 1,
    "image_key": "images",
    "max_prompt_length": 4096,
    "max_response_length": 1024,
    "prompt_key": "prompt",
    "return_full_prompt": false,
    "return_raw_chat": true,
    "return_raw_input_ids": false,
    "reward_fn_key": "data_source",
    "shuffle": true,
    "tokenizer": null,
    "train_batch_size": 8,
    "train_files": "...",
    "truncation": "error",
    "val_batch_size": 1,
    "val_files": "...",
    "video_key": "videos"
  },
  "ray_init": {
    "num_cpus": null
  },
  "reward_manager": "naive",
  "sandbox_fusion": {
    "max_concurrent": 64,
    "url": null
  },
  "strategy": "fsdp",
  "ulysses_sequence_parallel_size": 1,
  "use_dynamic_bsz": false,
  "trainer": {
    "balance_batch": true,
    "critic_warmup": 0,
    "default_hdfs_dir": null,
    "default_local_dir": "...",
    "del_local_ckpt_after_load": false,
    "experiment_name": "qwq-32b",
    "log_val_generations": 0,
    "logger": ["mlflow"],
    "max_actor_ckpt_to_keep": null,
    "max_critic_ckpt_to_keep": null,
    "n_gpus_per_node": 8,
    "nnodes": 1,
    "project_name": "yajiang-project/multiturn",
    "ray_wait_register_center_timeout": 300,
    "resume_from_path": null,
    "resume_mode": "disable",
    "rollout_data_dir": null,
    "save_freq": 10,
    "test_freq": 10,
    "total_epochs": 5,
    "total_training_steps": null,
    "val_before_train": true,
    "validation_data_dir": null
  },
  "reward_model": {
    "enable": false,
    "forward_max_token_len_per_gpu": 32768,
    "launch_reward_fn_async": false,
    "max_length": null,
    "micro_batch_size": null,
    "micro_batch_size_per_gpu": null,
    "model": {
      "external_lib": null,
      "fsdp_config": {
        "fsdp_size": -1,
        "param_offload": false,
        "reshard_after_forward": true,
        "wrap_policy": {"min_num_params": 0}
      },
      "input_tokenizer": "/shared/public/elr-models/Qwen/QwQ-32B/976055f8c83f394f35dbd3ab09a285a984907bd0",
      "path": "~/models/FsfairX-LLaMA3-RM-v0.1",
      "trust_remote_code": false,
      "use_remove_padding": false
    }
  }
}

About this issue

  • Original URL
  • State: open
  • Created 2 years ago
  • Reactions: 2
  • Comments: 28 (6 by maintainers)

Most upvoted comments

After careful investigation, I’ve narrowed down the actual error location to this line, although the error trace points to line 193 (where it actually waits for next_token_ids = next_token_ids.to("cpu", non_blocking=True)).

However, the root cause remains unclear. One possibility is that next_token_ids might have been prematurely released, causing access issues, but actually I have no ideas.

So far, the only workaround that worked for me was increasing gpu_memory_utilization—setting it to 0.7 in my script resolved the error. This suggests the issue might be memory-related.

Given the complexity, this might require deeper fixes from the SGLang side. Hope that will be helpful.

I also encountered what appears to be a similar error on the AMD MI300X while using async SGLang for multiturn GRPO RL. Here is the error log:

[36m(TaskRunner pid=2036068)[0m Dumped generations to /home/aiscuser/data/gsm8k_tool/validation/0.jsonl
[36m(TaskRunner pid=2036068)[0m ("Initial validation metrics: {'val-core/openai/gsm8k/reward/mean@1': "
[36m(TaskRunner pid=2036068)[0m  '0.6808188021228203}')
[36m(TaskRunner pid=2036068)[0m step:0 - val-core/openai/gsm8k/reward/mean@1:0.681
[36m(WorkerDict pid=2039146)[0m RCCL version : 2.21.5-HEAD:648a58d[32m [repeated 4x across cluster][0m
[36m(WorkerDict pid=2039146)[0m HIP version  : 6.3.42131-fa1d09cbd[32m [repeated 4x across cluster][0m
[36m(WorkerDict pid=2039146)[0m ROCm version : 6.3.0.0-39-ce2be3b[32m [repeated 4x across cluster][0m
[36m(WorkerDict pid=2039146)[0m Hostname     : node-0[32m [repeated 4x across cluster][0m
[36m(WorkerDict pid=2039146)[0m Librccl path : /usr/local/lib/librccl.so.1[32m [repeated 4x across cluster][0m
[36m(TaskRunner pid=2036068)[0m list(reward_extra_infos_dict.keys())=[]
[36m(TaskRunner pid=2036068)[0m _StringKeys(dict_keys(['input_ids', 'loss_mask', 'responses', 'attention_mask', 'position_ids', 'prompts', 'response_mask', 'old_log_probs', 'token_level_scores', 'token_level_rewards', 'advantages', 'returns']))
[36m(TaskRunner pid=2036068)[0m Dumped generations to /home/aiscuser/data/gsm8k_tool/rollout/1.jsonl
[36m(TaskRunner pid=2036068)[0m step:1 - global_seqlen/min:329662.000 - global_seqlen/max:416832.000 - global_seqlen/minmax_diff:87170.000 - global_seqlen/balanced_min:373316.000 - global_seqlen/balanced_max:374944.000 - global_seqlen/mean:373520.125 - actor/entropy_loss:0.266 - actor/pg_loss:0.077 - actor/pg_clipfrac:0.000 - actor/ppo_kl:0.000 - actor/pg_clipfrac_lower:0.000 - actor/grad_norm:0.374 - perf/mfu/actor:0.137 - perf/max_memory_allocated_gb:43.985 - perf/max_memory_reserved_gb:93.621 - perf/cpu_memory_used_gb:882.024 - actor/lr:0.000 - training/global_step:1.000 - training/epoch:0.000 - critic/score/mean:0.712 - critic/score/max:1.000 - critic/score/min:0.000 - critic/rewards/mean:0.712 - critic/rewards/max:1.000 - critic/rewards/min:0.000 - critic/advantages/mean:0.006 - critic/advantages/max:3.750 - critic/advantages/min:-3.750 - critic/returns/mean:0.006 - critic/returns/max:3.750 - critic/returns/min:-3.750 - response_length/mean:364.805 - response_length/max:4096.000 - response_length/min:60.000 - response_length/clip_ratio:0.007 - prompt_length/mean:364.727 - prompt_length/max:449.000 - prompt_length/min:326.000 - prompt_length/clip_ratio:0.000 - timing_s/gen:38.104 - timing_s/reward:1.137 - timing_s/old_log_prob:17.170 - timing_s/adv:0.082 - timing_s/update_actor:43.592 - timing_s/dump_rollout_generations:2.431 - timing_s/step:102.846 - timing_per_token_ms/gen:0.026 - timing_per_token_ms/update_actor:0.015 - timing_per_token_ms/adv:0.000 - perf/total_num_tokens:2988161.000 - perf/time_per_step:102.846 - perf/throughput:3631.850[36m(TaskRunner pid=2036068)[0m 
Training Progress:   0%|          | 1/435 [01:43<12:28:45, 103.51s/it]
[36m(TaskRunner pid=2036068)[0m 
Training Progress:   0%|          | 2/435 [03:21<12:02:15, 100.08s/it]
[36m(TaskRunner pid=2036068)[0m 
Training Progress:   1%|          | 3/435 [05:06<12:17:21, 102.41s/it]
[36m(TaskRunner pid=2036068)[0m 
Training Progress:   1%|          | 4/435 [06:53<12:28:56, 104.26s/it]
[36m(WorkerDict pid=2039142)[0m Memory access fault by GPU node-4 (Agent handle: 0xe92da90) on address 0x7eb3435d7000. Reason: Unknown.
[36m(WorkerDict pid=2039142)[0m Fatal Python error: Aborted
[36m(WorkerDict pid=2039142)[0m 
[36m(WorkerDict pid=2039142)[0m Thread 0x00007eb3469fe640 (most recent call first):
[36m(WorkerDict pid=2039142)[0m   File "/sgl-workspace/sglang/python/sglang/srt/managers/scheduler.py", line 1798 in watchdog_thread
[36m(WorkerDict pid=2039142)[0m   File "/usr/lib/python3.12/threading.py", line 1012 in run
[36m(WorkerDict pid=2039142)[0m   File "/usr/lib/python3.12/threading.py", line 1075 in _bootstrap_inner
[36m(WorkerDict pid=2039142)[0m   File "/usr/lib/python3.12/threading.py", line 1032 in _bootstrap
[36m(WorkerDict pid=2039142)[0m 
[36m(WorkerDict pid=2039142)[0m Thread 0x00007eb3471ff640 (most recent call first):
[36m(WorkerDict pid=2039142)[0m   File "/usr/lib/python3.12/threading.py", line 355 in wait
[36m(WorkerDict pid=2039142)[0m   File "/usr/lib/python3.12/queue.py", line 171 in get
[36m(WorkerDict pid=2039142)[0m   File "/sgl-workspace/sglang/python/sglang/srt/managers/tp_worker_overlap_thread.py", line 130 in forward_thread_func_
[36m(WorkerDict pid=2039142)[0m   File "/usr/local/lib/python3.12/dist-packages/torch/utils/_contextlib.py", line 116 in decorate_context
[36m(WorkerDict pid=2039142)[0m   File "/sgl-workspace/sglang/python/sglang/srt/managers/tp_worker_overlap_thread.py", line 118 in forward_thread_func
[36m(WorkerDict pid=2039142)[0m   File "/usr/lib/python3.12/threading.py", line 1012 in run
[36m(WorkerDict pid=2039142)[0m   File "/usr/lib/python3.12/threading.py", line 1075 in _bootstrap_inner
[36m(WorkerDict pid=2039142)[0m   File "/usr/lib/python3.12/threading.py", line 1032 in _bootstrap
[36m(WorkerDict pid=2039142)[0m 
[36m(WorkerDict pid=2039142)[0m Thread 0x00007eb38a3ff640 (most recent call first):
[36m(WorkerDict pid=2039142)[0m   File "/usr/lib/python3.12/threading.py", line 359 in wait
[36m(WorkerDict pid=2039142)[0m   File "/usr/lib/python3.12/threading.py", line 655 in wait
[36m(WorkerDict pid=2039142)[0m   File "/usr/local/lib/python3.12/dist-packages/tqdm/_monitor.py", line 60 in run
[36m(WorkerDict pid=2039142)[0m   File "/usr/lib/python3.12/threading.py", line 1075 in _bootstrap_inner
[36m(WorkerDict pid=2039142)[0m   File "/usr/lib/python3.12/threading.py", line 1032 in _bootstrap
[36m(WorkerDict pid=2039142)[0m 
[36m(WorkerDict pid=2039142)[0m Thread 0x00007f0a0d4a4640 (most recent call first):
[36m(WorkerDict pid=2039142)[0m   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/compile_worker/subproc_pool.py", line 55 in _recv_msg
[36m(WorkerDict pid=2039142)[0m   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/compile_worker/subproc_pool.py", line 191 in _read_thread
[36m(WorkerDict pid=2039142)[0m   File "/usr/lib/python3.12/threading.py", line 1012 in run
[36m(WorkerDict pid=2039142)[0m   File "/usr/lib/python3.12/threading.py", line 1075 in _bootstrap_inner
[36m(WorkerDict pid=2039142)[0m   File "/usr/lib/python3.12/threading.py", line 1032 in _bootstrap
[36m(WorkerDict pid=2039142)[0m 
[36m(WorkerDict pid=2039142)[0m Thread 0x00007f0b58d26180 (most recent call first):
[36m(WorkerDict pid=2039142)[0m   File "/usr/local/lib/python3.12/dist-packages/torch/cuda/streams.py", line 224 in synchronize
[36m(WorkerDict pid=2039142)[0m   File "/sgl-workspace/sglang/python/sglang/srt/managers/tp_worker_overlap_thread.py", line 193 in resolve_last_batch_result
[36m(WorkerDict pid=2039142)[0m   File "/sgl-workspace/sglang/python/sglang/srt/managers/scheduler_output_processor_mixin.py", line 200 in process_batch_result_decode
[36m(WorkerDict pid=2039142)[0m   File "/sgl-workspace/sglang/python/sglang/srt/managers/scheduler.py", line 1592 in process_batch_result
[36m(WorkerDict pid=2039142)[0m   File "/sgl-workspace/sglang/python/sglang/srt/managers/scheduler.py", line 683 in event_loop_overlap
[36m(WorkerDict pid=2039142)[0m   File "/usr/local/lib/python3.12/dist-packages/torch/utils/_contextlib.py", line 116 in decorate_context
[36m(WorkerDict pid=2039142)[0m   File "/sgl-workspace/sglang/python/sglang/srt/managers/scheduler.py", line 2269 in run_scheduler_process
[36m(WorkerDict pid=2039142)[0m   File "/usr/lib/python3.12/multiprocessing/process.py", line 108 in run
[36m(WorkerDict pid=2039142)[0m   File "/usr/lib/python3.12/multiprocessing/process.py", line 314 in _bootstrap
[36m(WorkerDict pid=2039142)[0m   File "/usr/lib/python3.12/multiprocessing/spawn.py", line 135 in _main
[36m(WorkerDict pid=2039142)[0m   File "/usr/lib/python3.12/multiprocessing/spawn.py", line 122 in spawn_main
[36m(WorkerDict pid=2039142)[0m   File "<string>", line 1 in <module>
[36m(WorkerDict pid=2039142)[0m 
[36m(WorkerDict pid=2039142)[0m Extension modules: msgpack._cmsgpack, google._upb._message, psutil._psutil_linux, psutil._psutil_posix, setproctitle, yaml._yaml, charset_normalizer.md, requests.packages.charset_normalizer.md, requests.packages.chardet.md, uvloop.loop, ray._raylet, numpy.core._multiarray_umath, numpy.core._multiarray_tests, numpy.linalg._umath_linalg, numpy.fft._pocketfft_internal, numpy.random._common, numpy.random.bit_generator, numpy.random._bounded_integers, numpy.random._mt19937, numpy.random.mtrand, numpy.random._philox, numpy.random._pcg64, numpy.random._sfc64, numpy.random._generator, multidict._multidict, yarl._quoting_c, propcache._helpers_c, aiohttp._http_writer, aiohttp._http_parser, aiohttp._websocket.mask, aiohttp._websocket.reader_c, frozenlist._frozenlist, torch._C, torch._C._dynamo.autograd_compiler, torch._C._dynamo.eval_frame, torch._C._dynamo.guards, torch._C._dynamo.utils, torch._C._fft, torch._C._linalg, torch._C._nested, torch._C._nn, torch._C._sparse, torch._C._special, zmq.backend.cython._zmq, markupsafe._speedups, PIL._imaging, PIL._imagingft, hiredis.hiredis, sentencepiece._sentencepiece, msgspec._core, regex._regex, cython.cimports.libc.math, _cffi_backend, hip_utils, Cython.Utils, Cython.Plex.Actions, Cython.Plex.Transitions, Cython.Plex.Machines, Cython.Plex.DFA, Cython.Plex.Scanners, Cython.Compiler.Scanning, Cython.StringIOTree, Cython.Compiler.Code, pyarrow.lib, pandas._libs.tslibs.ccalendar, pandas._libs.tslibs.np_datetime, pandas._libs.tslibs.dtypes, pandas._libs.tslibs.base, pandas._libs.tslibs.nattype, pandas._libs.tslibs.timezones, pandas._libs.tslibs.fields, pandas._libs.tslibs.timedeltas, pandas._libs.tslibs.tzconversion, pandas._libs.tslibs.timestamps, pandas._libs.properties, pandas._libs.tslibs.offsets, pandas._libs.tslibs.strptime, pandas._libs.tslibs.parsing, pandas._libs.tslibs.conversion, pandas._libs.tslibs.period, pandas._libs.tslibs.vectorized, pandas._libs.ops_dispatch, pandas._libs.missing, pandas._libs.hashtable, pandas._libs.algos, pandas._libs.interval, pandas._libs.lib, pyarrow._compute, pandas._libs.ops, pandas._libs.hashing, pandas._libs.arrays, pandas._libs.tslib, pandas._libs.sparse, pandas._libs.internals, pandas._libs.indexing, pandas._libs.index, pandas._libs.writers, pandas._libs.join, pandas._libs.window.aggregations, pandas._libs.window.indexers, pandas._libs.reshape, pandas._libs.groupby, pandas._libs.json, pandas._libs.parsers, pandas._libs.testing, __triton_launcher (total: 106)

I can confirm that no OOM (Out of Memory) error occurred. The runtime environment is based on Dockerfile.rocm.

Hey all, seems the issue isn’t reproducible anymore more with the latest versions of sgl and verl. You could try and let us know.

Below is the full error log:

(TaskRunner pid=1714137) Using dataset class: RLHFDataset
(TaskRunner pid=1714137) dataset len: 45
(TaskRunner pid=1714137) filter dataset len: 45
(TaskRunner pid=1714137) Using dataset class: RLHFDataset
(TaskRunner pid=1714137) dataset len: 1
(TaskRunner pid=1714137) filter dataset len: 1
(TaskRunner pid=1714137) WARNING: val_batch_size is deprecated. Validation datasets are sent to inference engines as a whole batch, which will schedule the memory themselves.
(TaskRunner pid=1714137) [validate_config] All configuration checks passed successfully!
(TaskRunner pid=1714137) Size of train dataloader: 5, Size of val dataloader: 1
(TaskRunner pid=1714137) Total training steps: 25
(TaskRunner pid=1714137) colocated worker base class <class 'verl.single_controller.base.worker.Worker'>
(WorkerDict pid=1732013) Model config after override: Qwen2Config {
(WorkerDict pid=1732013)   "architectures": [
(WorkerDict pid=1732013)     "Qwen2ForCausalLM"
(WorkerDict pid=1732013)   ],
(WorkerDict pid=1732013)   "attention_dropout": 0.0,
(WorkerDict pid=1732013)   "eos_token_id": 151645,
(WorkerDict pid=1732013)   "hidden_act": "silu",
(WorkerDict pid=1732013)   "hidden_size": 5120,
(WorkerDict pid=1732013)   "initializer_range": 0.02,
(WorkerDict pid=1732013)   "intermediate_size": 27648,
(WorkerDict pid=1732013)   "max_position_embeddings": 40960,
(WorkerDict pid=1732013)   "max_window_layers": 64,
(WorkerDict pid=1732013)   "model_type": "qwen2",
(WorkerDict pid=1732013)   "num_attention_heads": 40,
(WorkerDict pid=1732013)   "num_hidden_layers": 64,
(WorkerDict pid=1732013)   "num_key_value_heads": 8,
(WorkerDict pid=1732013)   "pad_token_id": 151643,
(WorkerDict pid=1732013)   "rms_norm_eps": 1e-05,
(WorkerDict pid=1732013)   "rope_scaling": null,
(WorkerDict pid=1732013)   "rope_theta": 1000000.0,
(WorkerDict pid=1732013)   "sliding_window": 32768,
(WorkerDict pid=1732013)   "tie_word_embeddings": false,
(WorkerDict pid=1732013)   "torch_dtype": "bfloat16",
(WorkerDict pid=1732013)   "transformers_version": "4.51.1",
(WorkerDict pid=1732013)   "use_cache": true,
(WorkerDict pid=1732013)   "use_sliding_window": false,
(WorkerDict pid=1732013)   "vocab_size": 152064
(WorkerDict pid=1732013) }
(WorkerDict pid=1732013) 
(WorkerDict pid=1733167) Monkey patch _flash_attention_forward in transformers.integrations.flash_attention
(WorkerDict pid=1732013) Qwen2ForCausalLM contains 32.76B parameters
(WorkerDict pid=1732013) wrap_policy: functools.partial(<function _or_policy at 0x71c356939000>, policies=[functools.partial(<function transformer_auto_wrap_policy at 0x71c356938ee0>, transformer_layer_cls={<class 'transformers.models.qwen2.modeling_qwen2.Qwen2DecoderLayer'>})])
(WorkerDict pid=1732013) NCCL version 2.21.5+cuda12.4
(WorkerDict pid=1732013) Actor use_remove_padding=True
(WorkerDict pid=1732013) Actor use_fused_kernels=False
(WorkerDict pid=1732013) Monkey patch _flash_attention_forward in transformers.integrations.flash_attention [repeated 7x across cluster] (Ray deduplicates logs by default. Set RAY_DEDUP_LOGS=0 to disable log deduplication, or see https://docs.ray.io/en/master/ray-observability/user-guides/configure-logging.html#log-deduplication for more options.)
(WorkerDict pid=1733170) wrap_policy: functools.partial(<function _or_policy at 0x77e34f3e1000>, policies=[functools.partial(<function transformer_auto_wrap_policy at 0x77e34f3e0ee0>, transformer_layer_cls={<class 'transformers.models.qwen2.modeling_qwen2.Qwen2DecoderLayer'>})]) [repeated 7x across cluster]
(WorkerDict pid=1732013) Model config after override: Qwen2Config {
(WorkerDict pid=1732013)   "architectures": [
(WorkerDict pid=1732013)     "Qwen2ForCausalLM"
(WorkerDict pid=1732013)   ],
(WorkerDict pid=1732013)   "attention_dropout": 0.0,
(WorkerDict pid=1732013)   "eos_token_id": 151645,
(WorkerDict pid=1732013)   "hidden_act": "silu",
(WorkerDict pid=1732013)   "hidden_size": 5120,
(WorkerDict pid=1732013)   "initializer_range": 0.02,
(WorkerDict pid=1732013)   "intermediate_size": 27648,
(WorkerDict pid=1732013)   "max_position_embeddings": 40960,
(WorkerDict pid=1732013)   "max_window_layers": 64,
(WorkerDict pid=1732013)   "model_type": "qwen2",
(WorkerDict pid=1732013)   "num_attention_heads": 40,
(WorkerDict pid=1732013)   "num_hidden_layers": 64,
(WorkerDict pid=1732013)   "num_key_value_heads": 8,
(WorkerDict pid=1732013)   "pad_token_id": 151643,
(WorkerDict pid=1732013)   "rms_norm_eps": 1e-05,
(WorkerDict pid=1732013)   "rope_scaling": null,
(WorkerDict pid=1732013)   "rope_theta": 1000000.0,
(WorkerDict pid=1732013)   "sliding_window": 32768,
(WorkerDict pid=1732013)   "tie_word_embeddings": false,
(WorkerDict pid=1732013)   "torch_dtype": "bfloat16",
(WorkerDict pid=1732013)   "transformers_version": "4.51.1",
(WorkerDict pid=1732013)   "use_cache": true,
(WorkerDict pid=1732013)   "use_sliding_window": false,
(WorkerDict pid=1732013)   "vocab_size": 152064
(WorkerDict pid=1732013) Flash Attention 2.0 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen2ForCausalLM is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `torch_dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", torch_dtype=torch.float16)`
(WorkerDict pid=1733172) You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`. [repeated 7x across cluster]
Loading checkpoint shards:   0%|          | 0/14 [00:00<?, ?it/s] [repeated 7x across cluster]
Loading checkpoint shards:  57%|█████▋    | 8/14 [00:00<00:00, 72.33it/s]
Loading checkpoint shards: 100%|██████████| 14/14 [00:00<00:00, 75.03it/s]
Loading checkpoint shards: 100%|██████████| 14/14 [00:00<00:00, 89.86it/s] [repeated 5x across cluster]
Loading checkpoint shards:   7%|▋         | 1/14 [00:04<00:56,  4.37s/it]
(WorkerDict pid=1733170) Flash Attention 2.0 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in Qwen2ForCausalLM is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `torch_dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", torch_dtype=torch.float16)` [repeated 7x across cluster]
Loading checkpoint shards:   0%|          | 0/14 [00:00<?, ?it/s] [repeated 8x across cluster]
Loading checkpoint shards:  14%|█▍        | 2/14 [00:09<00:59,  4.98s/it] [repeated 11x across cluster]
Loading checkpoint shards:  21%|██▏       | 3/14 [00:15<00:56,  5.18s/it] [repeated 9x across cluster]
Loading checkpoint shards:  29%|██▊       | 4/14 [00:20<00:51,  5.19s/it] [repeated 8x across cluster]
Loading checkpoint shards:  36%|███▌      | 5/14 [00:26<00:47,  5.23s/it] [repeated 8x across cluster]
Loading checkpoint shards:  43%|████▎     | 6/14 [00:31<00:42,  5.28s/it] [repeated 8x across cluster]
Loading checkpoint shards:  57%|█████▋    | 8/14 [00:36<00:27,  4.60s/it] [repeated 10x across cluster]
Loading checkpoint shards:  57%|█████▋    | 8/14 [00:42<00:31,  5.31s/it] [repeated 9x across cluster]
Loading checkpoint shards:  64%|██████▍   | 9/14 [00:47<00:26,  5.30s/it] [repeated 8x across cluster]
Loading checkpoint shards:  71%|███████▏  | 10/14 [00:52<00:21,  5.31s/it] [repeated 8x across cluster]
Loading checkpoint shards:  93%|█████████▎| 13/14 [00:57<00:04,  4.29s/it]
Loading checkpoint shards:  79%|███████▊  | 11/14 [00:58<00:15,  5.33s/it] [repeated 8x across cluster]
Loading checkpoint shards: 100%|██████████| 14/14 [00:59<00:00,  4.23s/it]
Loading checkpoint shards:  93%|█████████▎| 13/14 [00:59<00:04,  4.57s/it] [repeated 2x across cluster]
Loading checkpoint shards:  86%|████████▌ | 12/14 [01:03<00:10,  5.33s/it] [repeated 5x across cluster]
Loading checkpoint shards: 100%|██████████| 14/14 [01:01<00:00,  4.40s/it] [repeated 2x across cluster]
Loading checkpoint shards:  93%|█████████▎| 13/14 [01:08<00:05,  5.31s/it] [repeated 2x across cluster]
Loading checkpoint shards:  86%|████████▌ | 12/14 [01:04<00:10,  5.43s/it] [repeated 3x across cluster]
Loading checkpoint shards: 100%|██████████| 14/14 [01:06<00:00,  4.78s/it]
Loading checkpoint shards: 100%|██████████| 14/14 [01:10<00:00,  5.05s/it]
(WorkerDict pid=1733167) /home/jobuser/.local/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:690: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html .
(WorkerDict pid=1733167)   warnings.warn(
Loading checkpoint shards:  93%|█████████▎| 13/14 [01:09<00:05,  5.39s/it] [repeated 3x across cluster]
Loading checkpoint shards: 100%|██████████| 14/14 [01:12<00:00,  5.15s/it] [repeated 3x across cluster]
Capturing batches (avail_mem=52.88 GB):   0%|          | 0/35 [00:00<?, ?it/s]
(WorkerDict pid=1733170) /home/jobuser/.local/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:690: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html . [repeated 6x across cluster]
(WorkerDict pid=1733170)   warnings.warn( [repeated 6x across cluster]
Capturing batches (avail_mem=52.41 GB):   3%|▎         | 1/35 [00:00<00:27,  1.24it/s]
Capturing batches (avail_mem=52.22 GB):   6%|▌         | 2/35 [00:01<00:21,  1.53it/s]
Capturing batches (avail_mem=52.03 GB):   9%|▊         | 3/35 [00:02<00:21,  1.48it/s]
Capturing batches (avail_mem=51.85 GB):  11%|█▏        | 4/35 [00:02<00:19,  1.62it/s]
Capturing batches (avail_mem=51.68 GB):  14%|█▍        | 5/35 [00:03<00:17,  1.75it/s]
Capturing batches (avail_mem=51.51 GB):  17%|█▋        | 6/35 [00:03<00:16,  1.79it/s]
Capturing batches (avail_mem=51.35 GB):  20%|██        | 7/35 [00:04<00:15,  1.84it/s]
Capturing batches (avail_mem=51.19 GB):  23%|██▎       | 8/35 [00:04<00:14,  1.85it/s]
Capturing batches (avail_mem=51.04 GB):  26%|██▌       | 9/35 [00:05<00:13,  1.90it/s]
Capturing batches (avail_mem=50.89 GB):  29%|██▊       | 10/35 [00:05<00:13,  1.91it/s]
Capturing batches (avail_mem=50.75 GB):  31%|███▏      | 11/35 [00:06<00:12,  1.93it/s]
Capturing batches (avail_mem=50.62 GB):  34%|███▍      | 12/35 [00:06<00:12,  1.91it/s]
Capturing batches (avail_mem=50.49 GB):  37%|███▋      | 13/35 [00:07<00:11,  1.91it/s]
Capturing batches (avail_mem=50.37 GB):  40%|████      | 14/35 [00:07<00:10,  1.91it/s]
Capturing batches (avail_mem=50.26 GB):  43%|████▎     | 15/35 [00:08<00:10,  1.90it/s]
Capturing batches (avail_mem=50.15 GB):  46%|████▌     | 16/35 [00:08<00:09,  1.92it/s]
Capturing batches (avail_mem=50.11 GB):  49%|████▊     | 17/35 [00:09<00:09,  1.94it/s]
Capturing batches (avail_mem=50.09 GB):  51%|█████▏    | 18/35 [00:09<00:08,  1.95it/s]
Capturing batches (avail_mem=49.99 GB):  54%|█████▍    | 19/35 [00:10<00:08,  1.95it/s]
Capturing batches (avail_mem=49.90 GB):  57%|█████▋    | 20/35 [00:10<00:07,  1.94it/s]
Capturing batches (avail_mem=49.82 GB):  60%|██████    | 21/35 [00:11<00:07,  1.95it/s]
Capturing batches (avail_mem=49.74 GB):  63%|██████▎   | 22/35 [00:11<00:06,  1.95it/s]
Capturing batches (avail_mem=49.67 GB):  66%|██████▌   | 23/35 [00:12<00:06,  1.95it/s]
Capturing batches (avail_mem=49.61 GB):  69%|██████▊   | 24/35 [00:12<00:05,  1.95it/s]
Capturing batches (avail_mem=49.59 GB):  71%|███████▏  | 25/35 [00:13<00:05,  1.95it/s]
Capturing batches (avail_mem=49.52 GB):  74%|███████▍  | 26/35 [00:13<00:04,  1.96it/s]
Capturing batches (avail_mem=49.49 GB):  77%|███████▋  | 27/35 [00:14<00:04,  1.95it/s]
Capturing batches (avail_mem=49.43 GB):  80%|████████  | 28/35 [00:14<00:03,  1.93it/s]
Capturing batches (avail_mem=49.42 GB):  83%|████████▎ | 29/35 [00:15<00:03,  1.95it/s]
Capturing batches (avail_mem=49.41 GB):  86%|████████▌ | 30/35 [00:15<00:02,  1.97it/s]
Capturing batches (avail_mem=49.39 GB):  89%|████████▊ | 31/35 [00:16<00:02,  1.97it/s]
Capturing batches (avail_mem=49.38 GB):  91%|█████████▏| 32/35 [00:16<00:01,  1.97it/s]
Capturing batches (avail_mem=49.37 GB):  94%|█████████▍| 33/35 [00:17<00:01,  1.98it/s]
Capturing batches (avail_mem=49.36 GB):  97%|█████████▋| 34/35 [00:17<00:00,  1.97it/s]
Capturing batches (avail_mem=49.36 GB): 100%|██████████| 35/35 [00:18<00:00,  1.89it/s]
(WorkerDict pid=1732013) /home/jobuser/.local/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:690: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html .
(WorkerDict pid=1732013)   warnings.warn(
(WorkerDict pid=1732013) [2025-05-21 01:27:52 TP2] Scheduler hit an exception: Traceback (most recent call last):
(WorkerDict pid=1732013)   File "/home/jobuser/.local/lib/python3.10/site-packages/sglang/srt/managers/scheduler.py", line 2269, in run_scheduler_process
(WorkerDict pid=1732013)     scheduler.event_loop_overlap()
(WorkerDict pid=1732013)   File "/home/jobuser/.local/lib/python3.10/site-packages/torch/utils/_contextlib.py", line 116, in decorate_context
(WorkerDict pid=1732013)     return func(*args, **kwargs)
(WorkerDict pid=1732013)   File "/home/jobuser/.local/lib/python3.10/site-packages/sglang/srt/managers/scheduler.py", line 658, in event_loop_overlap
(WorkerDict pid=1732013)     batch = self.get_next_batch_to_run()
(WorkerDict pid=1732013)   File "/home/jobuser/.local/lib/python3.10/site-packages/sglang/srt/managers/scheduler.py", line 1304, in get_next_batch_to_run
(WorkerDict pid=1732013)     self.running_batch = self.update_running_batch(self.running_batch)
(WorkerDict pid=1732013)   File "/home/jobuser/.local/lib/python3.10/site-packages/sglang/srt/managers/scheduler.py", line 1502, in update_running_batch
(WorkerDict pid=1732013)     batch.prepare_for_decode()
(WorkerDict pid=1732013)   File "/home/jobuser/.local/lib/python3.10/site-packages/sglang/srt/managers/schedule_batch.py", line 1461, in prepare_for_decode
(WorkerDict pid=1732013)     locs = self.seq_lens.clone()
(WorkerDict pid=1732013) RuntimeError: CUDA error: an illegal memory access was encountered
(WorkerDict pid=1732013) CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.
(WorkerDict pid=1732013) For debugging consider passing CUDA_LAUNCH_BLOCKING=1
(WorkerDict pid=1732013) Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions.
(WorkerDict pid=1732013) 
(WorkerDict pid=1732013) 
(WorkerDict pid=1732013) [2025-05-21 01:27:52] Received sigquit from a child process. It usually means the child failed.
(TaskRunner pid=1714137) Unhandled error (suppress with 'RAY_IGNORE_UNHANDLED_ERRORS=1'): ray::WorkerDict.actor_rollout_generate_sequences() (pid=1733173, ip=100.96.253.216, actor_id=537b3c0b0720bff85b98982501000000, repr=<verl.single_controller.ray.base.WorkerDict object at 0x7757c3aafa00>)
(TaskRunner pid=1714137)   File "/home/jobuser/resources/verl/single_controller/ray/base.py", line 625, in func
(TaskRunner pid=1714137)     return getattr(self.worker_dict[key], name)(*args, **kwargs)
(TaskRunner pid=1714137)   File "/home/jobuser/resources/verl/single_controller/base/decorator.py", line 534, in inner
(TaskRunner pid=1714137)   File "/home/jobuser/resources/verl/workers/fsdp_workers.py", line 630, in generate_sequences
(TaskRunner pid=1714137)     output = self.rollout.generate_sequences_with_tools(prompts=prompts)
(TaskRunner pid=1714137)   File "/home/jobuser/resources/verl/utils/debug/performance.py", line 78, in f
(TaskRunner pid=1714137)     return self.log(decorated_function, *args, **kwargs)
(TaskRunner pid=1714137)   File "/home/jobuser/resources/verl/utils/debug/performance.py", line 88, in log
(TaskRunner pid=1714137)     output = func(*args, **kwargs)
(TaskRunner pid=1714137)   File "/home/jobuser/resources/verl/workers/rollout/sglang_rollout/async_sglang_rollout.py", line 619, in generate_sequences_with_tools
(TaskRunner pid=1714137)     [sorted_output_req_list] = broadcast_pyobj(
(TaskRunner pid=1714137)   File "/home/jobuser/resources/verl/workers/rollout/sglang_rollout/utils.py", line 55, in broadcast_pyobj
(TaskRunner pid=1714137)     dist.broadcast(tensor_size, src=src, group=dist_group)
(TaskRunner pid=1714137)   File "/home/jobuser/.local/lib/python3.10/site-packages/torch/distributed/c10d_logger.py", line 81, in wrapper
(TaskRunner pid=1714137)   File "/home/jobuser/.local/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py", line 2726, in broadcast
(TaskRunner pid=1714137)     work = group.broadcast([tensor], opts)
(TaskRunner pid=1714137) torch.distributed.DistBackendError: [7] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: failed to recv, got 0 bytes
(TaskRunner pid=1714137) Exception raised from recvBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:671 (most recent call first):
(TaskRunner pid=1714137) frame #0: c10::Error::Error(c10::SourceLocation, std::string) + 0x96 (0x778adc76c1b6 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libc10.so)
(TaskRunner pid=1714137) frame #1: <unknown function> + 0x6312a88 (0x775bc1256a88 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #2: c10d::TCPStore::doWait(c10::ArrayRef<std::string>, std::chrono::duration<long, std::ratio<1l, 1000l> >) + 0x367 (0x775bc1253817 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #3: c10d::TCPStore::doGet(std::string const&) + 0x2a (0x775bc1253b9a in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #4: c10d::TCPStore::get(std::string const&) + 0x7a (0x775bc125493a in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #5: c10d::PrefixStore::get(std::string const&) + 0x31 (0x775bc12015d1 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #6: c10d::PrefixStore::get(std::string const&) + 0x31 (0x775bc12015d1 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #7: c10d::PrefixStore::get(std::string const&) + 0x31 (0x775bc12015d1 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #8: c10d::PrefixStore::get(std::string const&) + 0x31 (0x775bc12015d1 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #9: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::string const&, int) + 0xaf (0x775b86f91a9f in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so)
(TaskRunner pid=1714137) frame #10: c10d::ProcessGroupNCCL::initNCCLComm(std::string const&, c10::Device&, c10d::OpType, int, bool) + 0x1176 (0x775b86f9f666 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so)
(TaskRunner pid=1714137) frame #11: c10d::ProcessGroupNCCL::broadcast(std::vector<at::Tensor, std::allocator<at::Tensor> >&, c10d::BroadcastOptions const&) + 0x1265 (0x775b86faa175 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so)
(TaskRunner pid=1714137) frame #12: <unknown function> + 0x62b13b6 (0x775bc11f53b6 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #13: <unknown function> + 0x62bcd26 (0x775bc1200d26 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #14: <unknown function> + 0x58bc27b (0x775bc080027b in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #15: <unknown function> + 0x58b9af9 (0x775bc07fdaf9 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #16: <unknown function> + 0x1aca9f8 (0x775bbca0e9f8 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #17: <unknown function> + 0x62c339a (0x775bc120739a in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #18: <unknown function> + 0x62d3b30 (0x775bc1217b30 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #19: <unknown function> + 0xe9fac5 (0x778ac792eac5 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_python.so)
(TaskRunner pid=1714137) frame #20: <unknown function> + 0x518d37 (0x778ac6fa7d37 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_python.so)
(TaskRunner pid=1714137) <omitting python frames>
(TaskRunner pid=1714137) frame #55: <unknown function> + 0x769d4f (0x778b78617d4f in /home/jobuser/.local/lib/python3.10/site-packages/ray/_raylet.so)
(TaskRunner pid=1714137) frame #56: <unknown function> + 0x7daaa8 (0x778b78688aa8 in /home/jobuser/.local/lib/python3.10/site-packages/ray/_raylet.so)
(TaskRunner pid=1714137) frame #57: <unknown function> + 0x769d4f (0x778b78617d4f in /home/jobuser/.local/lib/python3.10/site-packages/ray/_raylet.so)
(TaskRunner pid=1714137) frame #58: <unknown function> + 0x86fc56 (0x778b7871dc56 in /home/jobuser/.local/lib/python3.10/site-packages/ray/_raylet.so)
(TaskRunner pid=1714137) frame #59: std::_Function_handler<ray::Status (ray::rpc::Address const&, ray::rpc::TaskType, std::string, ray::core::RayFunction const&, std::unordered_map<std::string, double, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::pair<std::string const, double> > > const&, std::vector<std::shared_ptr<ray::RayObject>, std::allocator<std::shared_ptr<ray::RayObject> > > const&, std::vector<ray::rpc::ObjectReference, std::allocator<ray::rpc::ObjectReference> > const&, std::string const&, std::string const&, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*, std::vector<std::pair<ray::ObjectID, bool>, std::allocator<std::pair<ray::ObjectID, bool> > >*, std::shared_ptr<ray::LocalMemoryBuffer>&, bool*, std::string*, std::vector<ray::ConcurrencyGroup, std::allocator<ray::ConcurrencyGroup> > const&, std::string, bool, bool, bool, long), ray::Status (*)(ray::rpc::Address const&, ray::rpc::TaskType, std::string, ray::core::RayFunction const&, std::unordered_map<std::string, double, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::pair<std::string const, double> > > const&, std::vector<std::shared_ptr<ray::RayObject>, std::allocator<std::shared_ptr<ray::RayObject> > > const&, std::vector<ray::rpc::ObjectReference, std::allocator<ray::rpc::ObjectReference> > const&, std::string, std::string, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*, std::vector<std::pair<ray::ObjectID, bool>, std::allocator<std::pair<ray::ObjectID, bool> > >*, std::shared_ptr<ray::LocalMemoryBuffer>&, bool*, std::string*, std::vector<ray::ConcurrencyGroup, std::allocator<ray::ConcurrencyGroup> > const&, std::string, bool, bool, bool, long)>::_M_invoke(std::_Any_data const&, ray::rpc::Address const&, ray::rpc::TaskType&&, std::string&&, ray::core::RayFunction const&, std::unordered_map<std::string, double, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::pair<std::string const, double> > > const&, std::vector<std::shared_ptr<ray::RayObject>, std::allocator<std::shared_ptr<ray::RayObject> > > const&, std::vector<ray::rpc::ObjectReference, std::allocator<ray::rpc::ObjectReference> > const&, std::string const&, std::string const&, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*&&, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*&&, std::vector<std::pair<ray::ObjectID, bool>, std::allocator<std::pair<ray::ObjectID, bool> > >*&&, std::shared_ptr<ray::LocalMemoryBuffer>&, bool*&&, std::string*&&, std::vector<ray::ConcurrencyGroup, std::allocator<ray::ConcurrencyGroup> > const&, std::string&&, bool&&, bool&&, bool&&, long&&) + 0x169 (0x778b7861ed39 in /home/jobuser/.local/lib/python3.10/site-packages/ray/_raylet.so)
(TaskRunner pid=1714137) frame #60: ray::core::CoreWorker::ExecuteTask(ray::TaskSpecification const&, std::optional<std::unordered_map<std::string, std::vector<std::pair<long, double>, std::allocator<std::pair<long, double> > >, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::pair<std::string const, std::vector<std::pair<long, double>, std::allocator<std::pair<long, double> > > > > > >, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*, std::vector<std::pair<ray::ObjectID, bool>, std::allocator<std::pair<ray::ObjectID, bool> > >*, google::protobuf::RepeatedPtrField<ray::rpc::ObjectReferenceCount>*, bool*, std::string*) + 0xdf5 (0x778b78830245 in /home/jobuser/.local/lib/python3.10/site-packages/ray/_raylet.so)
(TaskRunner pid=1714137) frame #61: std::_Function_handler<ray::Status (ray::TaskSpecification const&, std::optional<std::unordered_map<std::string, std::vector<std::pair<long, double>, std::allocator<std::pair<long, double> > >, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::pair<std::string const, std::vector<std::pair<long, double>, std::allocator<std::pair<long, double> > > > > > >, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*, std::vector<std::pair<ray::ObjectID, bool>, std::allocator<std::pair<ray::ObjectID, bool> > >*, google::protobuf::RepeatedPtrField<ray::rpc::ObjectReferenceCount>*, bool*, std::string*), std::_Bind<ray::Status (ray::core::CoreWorker::*(ray::core::CoreWorker*, std::_Placeholder<1>, std::_Placeholder<2>, std::_Placeholder<3>, std::_Placeholder<4>, std::_Placeholder<5>, std::_Placeholder<6>, std::_Placeholder<7>, std::_Placeholder<8>))(ray::TaskSpecification const&, std::optional<std::unordered_map<std::string, std::vector<std::pair<long, double>, std::allocator<std::pair<long, double> > >, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::pair<std::string const, std::vector<std::pair<long, double>, std::allocator<std::pair<long, double> > > > > > >, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*, std::vector<std::pair<ray::ObjectID, bool>, std::allocator<std::pair<ray::ObjectID, bool> > >*, google::protobuf::RepeatedPtrField<ray::rpc::ObjectReferenceCount>*, bool*, std::string*)> >::_M_invoke(std::_Any_data const&, ray::TaskSpecification const&, std::optional<std::unordered_map<std::string, std::vector<std::pair<long, double>, std::allocator<std::pair<long, double> > >, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::pair<std::string const, std::vector<std::pair<long, double>, std::allocator<std::pair<long, double> > > > > > >&&, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*&&, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*&&, std::vector<std::pair<ray::ObjectID, bool>, std::allocator<std::pair<ray::ObjectID, bool> > >*&&, google::protobuf::RepeatedPtrField<ray::rpc::ObjectReferenceCount>*&&, bool*&&, std::string*&&) + 0x75 (0x778b787ba845 in /home/jobuser/.local/lib/python3.10/site-packages/ray/_raylet.so)
(TaskRunner pid=1714137) frame #62: <unknown function> + 0x9f30ce (0x778b788a10ce in /home/jobuser/.local/lib/python3.10/site-packages/ray/_raylet.so)
(TaskRunner pid=1714137) . This may indicate a possible application crash on rank 0 or a network set up issue.
(TaskRunner pid=1714137) Unhandled error (suppress with 'RAY_IGNORE_UNHANDLED_ERRORS=1'): ray::WorkerDict.actor_rollout_generate_sequences() (pid=1733172, ip=100.96.253.216, actor_id=b418c34c9373f957e605c79001000000, repr=<verl.single_controller.ray.base.WorkerDict object at 0x738b18890640>)
(TaskRunner pid=1714137)   File "/home/jobuser/resources/verl/single_controller/ray/base.py", line 625, in func
(TaskRunner pid=1714137)     return getattr(self.worker_dict[key], name)(*args, **kwargs)
(TaskRunner pid=1714137)   File "/home/jobuser/resources/verl/single_controller/base/decorator.py", line 534, in inner
(TaskRunner pid=1714137)   File "/home/jobuser/resources/verl/workers/fsdp_workers.py", line 630, in generate_sequences
(TaskRunner pid=1714137)     output = self.rollout.generate_sequences_with_tools(prompts=prompts)
(TaskRunner pid=1714137)   File "/home/jobuser/resources/verl/utils/debug/performance.py", line 78, in f
(TaskRunner pid=1714137)     return self.log(decorated_function, *args, **kwargs)
(TaskRunner pid=1714137)   File "/home/jobuser/resources/verl/utils/debug/performance.py", line 88, in log
(TaskRunner pid=1714137)     output = func(*args, **kwargs)
(TaskRunner pid=1714137)   File "/home/jobuser/resources/verl/workers/rollout/sglang_rollout/async_sglang_rollout.py", line 619, in generate_sequences_with_tools
(TaskRunner pid=1714137)     [sorted_output_req_list] = broadcast_pyobj(
(TaskRunner pid=1714137)   File "/home/jobuser/resources/verl/workers/rollout/sglang_rollout/utils.py", line 55, in broadcast_pyobj
(TaskRunner pid=1714137)     dist.broadcast(tensor_size, src=src, group=dist_group)
(TaskRunner pid=1714137)   File "/home/jobuser/.local/lib/python3.10/site-packages/torch/distributed/c10d_logger.py", line 81, in wrapper
(TaskRunner pid=1714137)   File "/home/jobuser/.local/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py", line 2726, in broadcast
(TaskRunner pid=1714137)     work = group.broadcast([tensor], opts)
(TaskRunner pid=1714137) torch.distributed.DistBackendError: [6] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: failed to recv, got 0 bytes
(TaskRunner pid=1714137) Exception raised from recvBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:671 (most recent call first):
(TaskRunner pid=1714137) frame #0: c10::Error::Error(c10::SourceLocation, std::string) + 0x96 (0x73be1396c1b6 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libc10.so)
(TaskRunner pid=1714137) frame #1: <unknown function> + 0x6312a88 (0x738f1d256a88 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #2: c10d::TCPStore::doWait(c10::ArrayRef<std::string>, std::chrono::duration<long, std::ratio<1l, 1000l> >) + 0x367 (0x738f1d253817 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #3: c10d::TCPStore::doGet(std::string const&) + 0x2a (0x738f1d253b9a in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #4: c10d::TCPStore::get(std::string const&) + 0x7a (0x738f1d25493a in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #5: c10d::PrefixStore::get(std::string const&) + 0x31 (0x738f1d2015d1 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #6: c10d::PrefixStore::get(std::string const&) + 0x31 (0x738f1d2015d1 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #7: c10d::PrefixStore::get(std::string const&) + 0x31 (0x738f1d2015d1 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #8: c10d::PrefixStore::get(std::string const&) + 0x31 (0x738f1d2015d1 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #9: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::string const&, int) + 0xaf (0x738ee2f91a9f in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so)
(TaskRunner pid=1714137) frame #10: c10d::ProcessGroupNCCL::initNCCLComm(std::string const&, c10::Device&, c10d::OpType, int, bool) + 0x1176 (0x738ee2f9f666 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so)
(TaskRunner pid=1714137) frame #11: c10d::ProcessGroupNCCL::broadcast(std::vector<at::Tensor, std::allocator<at::Tensor> >&, c10d::BroadcastOptions const&) + 0x1265 (0x738ee2faa175 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so)
(TaskRunner pid=1714137) frame #12: <unknown function> + 0x62b13b6 (0x738f1d1f53b6 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #13: <unknown function> + 0x62bcd26 (0x738f1d200d26 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #14: <unknown function> + 0x58bc27b (0x738f1c80027b in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #15: <unknown function> + 0x58b9af9 (0x738f1c7fdaf9 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #16: <unknown function> + 0x1aca9f8 (0x738f18a0e9f8 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #17: <unknown function> + 0x62c339a (0x738f1d20739a in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #18: <unknown function> + 0x62d3b30 (0x738f1d217b30 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #19: <unknown function> + 0xe9fac5 (0x73be12f2eac5 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_python.so)
(TaskRunner pid=1714137) frame #20: <unknown function> + 0x518d37 (0x73be125a7d37 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_python.so)
(TaskRunner pid=1714137) <omitting python frames>
(TaskRunner pid=1714137) frame #55: <unknown function> + 0x769d4f (0x73bec3400d4f in /home/jobuser/.local/lib/python3.10/site-packages/ray/_raylet.so)
(TaskRunner pid=1714137) frame #56: <unknown function> + 0x7daaa8 (0x73bec3471aa8 in /home/jobuser/.local/lib/python3.10/site-packages/ray/_raylet.so)
(TaskRunner pid=1714137) frame #57: <unknown function> + 0x769d4f (0x73bec3400d4f in /home/jobuser/.local/lib/python3.10/site-packages/ray/_raylet.so)
(TaskRunner pid=1714137) frame #58: <unknown function> + 0x86fc56 (0x73bec3506c56 in /home/jobuser/.local/lib/python3.10/site-packages/ray/_raylet.so)
(TaskRunner pid=1714137) frame #59: std::_Function_handler<ray::Status (ray::rpc::Address const&, ray::rpc::TaskType, std::string, ray::core::RayFunction const&, std::unordered_map<std::string, double, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::pair<std::string const, double> > > const&, std::vector<std::shared_ptr<ray::RayObject>, std::allocator<std::shared_ptr<ray::RayObject> > > const&, std::vector<ray::rpc::ObjectReference, std::allocator<ray::rpc::ObjectReference> > const&, std::string const&, std::string const&, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*, std::vector<std::pair<ray::ObjectID, bool>, std::allocator<std::pair<ray::ObjectID, bool> > >*, std::shared_ptr<ray::LocalMemoryBuffer>&, bool*, std::string*, std::vector<ray::ConcurrencyGroup, std::allocator<ray::ConcurrencyGroup> > const&, std::string, bool, bool, bool, long), ray::Status (*)(ray::rpc::Address const&, ray::rpc::TaskType, std::string, ray::core::RayFunction const&, std::unordered_map<std::string, double, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::pair<std::string const, double> > > const&, std::vector<std::shared_ptr<ray::RayObject>, std::allocator<std::shared_ptr<ray::RayObject> > > const&, std::vector<ray::rpc::ObjectReference, std::allocator<ray::rpc::ObjectReference> > const&, std::string, std::string, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*, std::vector<std::pair<ray::ObjectID, bool>, std::allocator<std::pair<ray::ObjectID, bool> > >*, std::shared_ptr<ray::LocalMemoryBuffer>&, bool*, std::string*, std::vector<ray::ConcurrencyGroup, std::allocator<ray::ConcurrencyGroup> > const&, std::string, bool, bool, bool, long)>::_M_invoke(std::_Any_data const&, ray::rpc::Address const&, ray::rpc::TaskType&&, std::string&&, ray::core::RayFunction const&, std::unordered_map<std::string, double, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::pair<std::string const, double> > > const&, std::vector<std::shared_ptr<ray::RayObject>, std::allocator<std::shared_ptr<ray::RayObject> > > const&, std::vector<ray::rpc::ObjectReference, std::allocator<ray::rpc::ObjectReference> > const&, std::string const&, std::string const&, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*&&, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*&&, std::vector<std::pair<ray::ObjectID, bool>, std::allocator<std::pair<ray::ObjectID, bool> > >*&&, std::shared_ptr<ray::LocalMemoryBuffer>&, bool*&&, std::string*&&, std::vector<ray::ConcurrencyGroup, std::allocator<ray::ConcurrencyGroup> > const&, std::string&&, bool&&, bool&&, bool&&, long&&) + 0x169 (0x73bec3407d39 in /home/jobuser/.local/lib/python3.10/site-packages/ray/_raylet.so)
(TaskRunner pid=1714137) frame #60: ray::core::CoreWorker::ExecuteTask(ray::TaskSpecification const&, std::optional<std::unordered_map<std::string, std::vector<std::pair<long, double>, std::allocator<std::pair<long, double> > >, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::pair<std::string const, std::vector<std::pair<long, double>, std::allocator<std::pair<long, double> > > > > > >, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*, std::vector<std::pair<ray::ObjectID, bool>, std::allocator<std::pair<ray::ObjectID, bool> > >*, google::protobuf::RepeatedPtrField<ray::rpc::ObjectReferenceCount>*, bool*, std::string*) + 0xdf5 (0x73bec3619245 in /home/jobuser/.local/lib/python3.10/site-packages/ray/_raylet.so)
(TaskRunner pid=1714137) frame #61: std::_Function_handler<ray::Status (ray::TaskSpecification const&, std::optional<std::unordered_map<std::string, std::vector<std::pair<long, double>, std::allocator<std::pair<long, double> > >, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::pair<std::string const, std::vector<std::pair<long, double>, std::allocator<std::pair<long, double> > > > > > >, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*, std::vector<std::pair<ray::ObjectID, bool>, std::allocator<std::pair<ray::ObjectID, bool> > >*, google::protobuf::RepeatedPtrField<ray::rpc::ObjectReferenceCount>*, bool*, std::string*), std::_Bind<ray::Status (ray::core::CoreWorker::*(ray::core::CoreWorker*, std::_Placeholder<1>, std::_Placeholder<2>, std::_Placeholder<3>, std::_Placeholder<4>, std::_Placeholder<5>, std::_Placeholder<6>, std::_Placeholder<7>, std::_Placeholder<8>))(ray::TaskSpecification const&, std::optional<std::unordered_map<std::string, std::vector<std::pair<long, double>, std::allocator<std::pair<long, double> > >, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::pair<std::string const, std::vector<std::pair<long, double>, std::allocator<std::pair<long, double> > > > > > >, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*, std::vector<std::pair<ray::ObjectID, bool>, std::allocator<std::pair<ray::ObjectID, bool> > >*, google::protobuf::RepeatedPtrField<ray::rpc::ObjectReferenceCount>*, bool*, std::string*)> >::_M_invoke(std::_Any_data const&, ray::TaskSpecification const&, std::optional<std::unordered_map<std::string, std::vector<std::pair<long, double>, std::allocator<std::pair<long, double> > >, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::pair<std::string const, std::vector<std::pair<long, double>, std::allocator<std::pair<long, double> > > > > > >&&, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*&&, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*&&, std::vector<std::pair<ray::ObjectID, bool>, std::allocator<std::pair<ray::ObjectID, bool> > >*&&, google::protobuf::RepeatedPtrField<ray::rpc::ObjectReferenceCount>*&&, bool*&&, std::string*&&) + 0x75 (0x73bec35a3845 in /home/jobuser/.local/lib/python3.10/site-packages/ray/_raylet.so)
(TaskRunner pid=1714137) frame #62: <unknown function> + 0x9f30ce (0x73bec368a0ce in /home/jobuser/.local/lib/python3.10/site-packages/ray/_raylet.so)
(TaskRunner pid=1714137) . This may indicate a possible application crash on rank 0 or a network set up issue.
(TaskRunner pid=1714137) Unhandled error (suppress with 'RAY_IGNORE_UNHANDLED_ERRORS=1'): ray::WorkerDict.actor_rollout_generate_sequences() (pid=1733171, ip=100.96.253.216, actor_id=d37e3c945e454da0bd149ec401000000, repr=<verl.single_controller.ray.base.WorkerDict object at 0x7cf46cb78640>)
(TaskRunner pid=1714137)   File "/home/jobuser/resources/verl/single_controller/ray/base.py", line 625, in func
(TaskRunner pid=1714137)     return getattr(self.worker_dict[key], name)(*args, **kwargs)
(TaskRunner pid=1714137)   File "/home/jobuser/resources/verl/single_controller/base/decorator.py", line 534, in inner
(TaskRunner pid=1714137)   File "/home/jobuser/resources/verl/workers/fsdp_workers.py", line 630, in generate_sequences
(TaskRunner pid=1714137)     output = self.rollout.generate_sequences_with_tools(prompts=prompts)
(TaskRunner pid=1714137)   File "/home/jobuser/resources/verl/utils/debug/performance.py", line 78, in f
(TaskRunner pid=1714137)     return self.log(decorated_function, *args, **kwargs)
(TaskRunner pid=1714137)   File "/home/jobuser/resources/verl/utils/debug/performance.py", line 88, in log
(TaskRunner pid=1714137)     output = func(*args, **kwargs)
(TaskRunner pid=1714137)   File "/home/jobuser/resources/verl/workers/rollout/sglang_rollout/async_sglang_rollout.py", line 619, in generate_sequences_with_tools
(TaskRunner pid=1714137)     [sorted_output_req_list] = broadcast_pyobj(
(TaskRunner pid=1714137)   File "/home/jobuser/resources/verl/workers/rollout/sglang_rollout/utils.py", line 55, in broadcast_pyobj
(TaskRunner pid=1714137)     dist.broadcast(tensor_size, src=src, group=dist_group)
(TaskRunner pid=1714137)   File "/home/jobuser/.local/lib/python3.10/site-packages/torch/distributed/c10d_logger.py", line 81, in wrapper
(TaskRunner pid=1714137)   File "/home/jobuser/.local/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py", line 2726, in broadcast
(TaskRunner pid=1714137)     work = group.broadcast([tensor], opts)
(TaskRunner pid=1714137) torch.distributed.DistBackendError: [5] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: failed to recv, got 0 bytes
(TaskRunner pid=1714137) Exception raised from recvBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:671 (most recent call first):
(TaskRunner pid=1714137) frame #0: c10::Error::Error(c10::SourceLocation, std::string) + 0x96 (0x7d277bb6c1b6 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libc10.so)
(TaskRunner pid=1714137) frame #1: <unknown function> + 0x6312a88 (0x7cf85d256a88 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #2: c10d::TCPStore::doWait(c10::ArrayRef<std::string>, std::chrono::duration<long, std::ratio<1l, 1000l> >) + 0x367 (0x7cf85d253817 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #3: c10d::TCPStore::doGet(std::string const&) + 0x2a (0x7cf85d253b9a in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #4: c10d::TCPStore::get(std::string const&) + 0x7a (0x7cf85d25493a in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #5: c10d::PrefixStore::get(std::string const&) + 0x31 (0x7cf85d2015d1 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #6: c10d::PrefixStore::get(std::string const&) + 0x31 (0x7cf85d2015d1 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #7: c10d::PrefixStore::get(std::string const&) + 0x31 (0x7cf85d2015d1 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #8: c10d::PrefixStore::get(std::string const&) + 0x31 (0x7cf85d2015d1 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #9: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::string const&, int) + 0xaf (0x7cf822f91a9f in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so)
(TaskRunner pid=1714137) frame #10: c10d::ProcessGroupNCCL::initNCCLComm(std::string const&, c10::Device&, c10d::OpType, int, bool) + 0x1176 (0x7cf822f9f666 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so)
(TaskRunner pid=1714137) frame #11: c10d::ProcessGroupNCCL::broadcast(std::vector<at::Tensor, std::allocator<at::Tensor> >&, c10d::BroadcastOptions const&) + 0x1265 (0x7cf822faa175 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so)
(TaskRunner pid=1714137) frame #12: <unknown function> + 0x62b13b6 (0x7cf85d1f53b6 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #13: <unknown function> + 0x62bcd26 (0x7cf85d200d26 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #14: <unknown function> + 0x58bc27b (0x7cf85c80027b in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #15: <unknown function> + 0x58b9af9 (0x7cf85c7fdaf9 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #16: <unknown function> + 0x1aca9f8 (0x7cf858a0e9f8 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #17: <unknown function> + 0x62c339a (0x7cf85d20739a in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #18: <unknown function> + 0x62d3b30 (0x7cf85d217b30 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #19: <unknown function> + 0xe9fac5 (0x7d276b92eac5 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_python.so)
(TaskRunner pid=1714137) frame #20: <unknown function> + 0x518d37 (0x7d276afa7d37 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_python.so)
(TaskRunner pid=1714137) <omitting python frames>
(TaskRunner pid=1714137) frame #55: <unknown function> + 0x769d4f (0x7d28176a5d4f in /home/jobuser/.local/lib/python3.10/site-packages/ray/_raylet.so)
(TaskRunner pid=1714137) frame #56: <unknown function> + 0x7daaa8 (0x7d2817716aa8 in /home/jobuser/.local/lib/python3.10/site-packages/ray/_raylet.so)
(TaskRunner pid=1714137) frame #57: <unknown function> + 0x769d4f (0x7d28176a5d4f in /home/jobuser/.local/lib/python3.10/site-packages/ray/_raylet.so)
(TaskRunner pid=1714137) frame #58: <unknown function> + 0x86fc56 (0x7d28177abc56 in /home/jobuser/.local/lib/python3.10/site-packages/ray/_raylet.so)
(TaskRunner pid=1714137) frame #59: std::_Function_handler<ray::Status (ray::rpc::Address const&, ray::rpc::TaskType, std::string, ray::core::RayFunction const&, std::unordered_map<std::string, double, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::pair<std::string const, double> > > const&, std::vector<std::shared_ptr<ray::RayObject>, std::allocator<std::shared_ptr<ray::RayObject> > > const&, std::vector<ray::rpc::ObjectReference, std::allocator<ray::rpc::ObjectReference> > const&, std::string const&, std::string const&, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*, std::vector<std::pair<ray::ObjectID, bool>, std::allocator<std::pair<ray::ObjectID, bool> > >*, std::shared_ptr<ray::LocalMemoryBuffer>&, bool*, std::string*, std::vector<ray::ConcurrencyGroup, std::allocator<ray::ConcurrencyGroup> > const&, std::string, bool, bool, bool, long), ray::Status (*)(ray::rpc::Address const&, ray::rpc::TaskType, std::string, ray::core::RayFunction const&, std::unordered_map<std::string, double, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::pair<std::string const, double> > > const&, std::vector<std::shared_ptr<ray::RayObject>, std::allocator<std::shared_ptr<ray::RayObject> > > const&, std::vector<ray::rpc::ObjectReference, std::allocator<ray::rpc::ObjectReference> > const&, std::string, std::string, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*, std::vector<std::pair<ray::ObjectID, bool>, std::allocator<std::pair<ray::ObjectID, bool> > >*, std::shared_ptr<ray::LocalMemoryBuffer>&, bool*, std::string*, std::vector<ray::ConcurrencyGroup, std::allocator<ray::ConcurrencyGroup> > const&, std::string, bool, bool, bool, long)>::_M_invoke(std::_Any_data const&, ray::rpc::Address const&, ray::rpc::TaskType&&, std::string&&, ray::core::RayFunction const&, std::unordered_map<std::string, double, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::pair<std::string const, double> > > const&, std::vector<std::shared_ptr<ray::RayObject>, std::allocator<std::shared_ptr<ray::RayObject> > > const&, std::vector<ray::rpc::ObjectReference, std::allocator<ray::rpc::ObjectReference> > const&, std::string const&, std::string const&, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*&&, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*&&, std::vector<std::pair<ray::ObjectID, bool>, std::allocator<std::pair<ray::ObjectID, bool> > >*&&, std::shared_ptr<ray::LocalMemoryBuffer>&, bool*&&, std::string*&&, std::vector<ray::ConcurrencyGroup, std::allocator<ray::ConcurrencyGroup> > const&, std::string&&, bool&&, bool&&, bool&&, long&&) + 0x169 (0x7d28176acd39 in /home/jobuser/.local/lib/python3.10/site-packages/ray/_raylet.so)
(TaskRunner pid=1714137) frame #60: ray::core::CoreWorker::ExecuteTask(ray::TaskSpecification const&, std::optional<std::unordered_map<std::string, std::vector<std::pair<long, double>, std::allocator<std::pair<long, double> > >, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::pair<std::string const, std::vector<std::pair<long, double>, std::allocator<std::pair<long, double> > > > > > >, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*, std::vector<std::pair<ray::ObjectID, bool>, std::allocator<std::pair<ray::ObjectID, bool> > >*, google::protobuf::RepeatedPtrField<ray::rpc::ObjectReferenceCount>*, bool*, std::string*) + 0xdf5 (0x7d28178be245 in /home/jobuser/.local/lib/python3.10/site-packages/ray/_raylet.so)
(TaskRunner pid=1714137) frame #61: std::_Function_handler<ray::Status (ray::TaskSpecification const&, std::optional<std::unordered_map<std::string, std::vector<std::pair<long, double>, std::allocator<std::pair<long, double> > >, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::pair<std::string const, std::vector<std::pair<long, double>, std::allocator<std::pair<long, double> > > > > > >, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*, std::vector<std::pair<ray::ObjectID, bool>, std::allocator<std::pair<ray::ObjectID, bool> > >*, google::protobuf::RepeatedPtrField<ray::rpc::ObjectReferenceCount>*, bool*, std::string*), std::_Bind<ray::Status (ray::core::CoreWorker::*(ray::core::CoreWorker*, std::_Placeholder<1>, std::_Placeholder<2>, std::_Placeholder<3>, std::_Placeholder<4>, std::_Placeholder<5>, std::_Placeholder<6>, std::_Placeholder<7>, std::_Placeholder<8>))(ray::TaskSpecification const&, std::optional<std::unordered_map<std::string, std::vector<std::pair<long, double>, std::allocator<std::pair<long, double> > >, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::pair<std::string const, std::vector<std::pair<long, double>, std::allocator<std::pair<long, double> > > > > > >, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*, std::vector<std::pair<ray::ObjectID, bool>, std::allocator<std::pair<ray::ObjectID, bool> > >*, google::protobuf::RepeatedPtrField<ray::rpc::ObjectReferenceCount>*, bool*, std::string*)> >::_M_invoke(std::_Any_data const&, ray::TaskSpecification const&, std::optional<std::unordered_map<std::string, std::vector<std::pair<long, double>, std::allocator<std::pair<long, double> > >, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::pair<std::string const, std::vector<std::pair<long, double>, std::allocator<std::pair<long, double> > > > > > >&&, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*&&, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*&&, std::vector<std::pair<ray::ObjectID, bool>, std::allocator<std::pair<ray::ObjectID, bool> > >*&&, google::protobuf::RepeatedPtrField<ray::rpc::ObjectReferenceCount>*&&, bool*&&, std::string*&&) + 0x75 (0x7d2817848845 in /home/jobuser/.local/lib/python3.10/site-packages/ray/_raylet.so)
(TaskRunner pid=1714137) frame #62: <unknown function> + 0x9f30ce (0x7d281792f0ce in /home/jobuser/.local/lib/python3.10/site-packages/ray/_raylet.so)
(TaskRunner pid=1714137) . This may indicate a possible application crash on rank 0 or a network set up issue.
(TaskRunner pid=1714137) Unhandled error (suppress with 'RAY_IGNORE_UNHANDLED_ERRORS=1'): ray::WorkerDict.actor_rollout_generate_sequences() (pid=1733169, ip=100.96.253.216, actor_id=4cb9864143683f98469fe90101000000, repr=<verl.single_controller.ray.base.WorkerDict object at 0x74b2dd004610>)
(TaskRunner pid=1714137)   File "/home/jobuser/resources/verl/single_controller/ray/base.py", line 625, in func
(TaskRunner pid=1714137)     return getattr(self.worker_dict[key], name)(*args, **kwargs)
(TaskRunner pid=1714137)   File "/home/jobuser/resources/verl/single_controller/base/decorator.py", line 534, in inner
(TaskRunner pid=1714137)   File "/home/jobuser/resources/verl/workers/fsdp_workers.py", line 630, in generate_sequences
(TaskRunner pid=1714137)     output = self.rollout.generate_sequences_with_tools(prompts=prompts)
(TaskRunner pid=1714137)   File "/home/jobuser/resources/verl/utils/debug/performance.py", line 78, in f
(TaskRunner pid=1714137)     return self.log(decorated_function, *args, **kwargs)
(TaskRunner pid=1714137)   File "/home/jobuser/resources/verl/utils/debug/performance.py", line 88, in log
(TaskRunner pid=1714137)     output = func(*args, **kwargs)
(TaskRunner pid=1714137)   File "/home/jobuser/resources/verl/workers/rollout/sglang_rollout/async_sglang_rollout.py", line 619, in generate_sequences_with_tools
(TaskRunner pid=1714137)     [sorted_output_req_list] = broadcast_pyobj(
(TaskRunner pid=1714137)   File "/home/jobuser/resources/verl/workers/rollout/sglang_rollout/utils.py", line 55, in broadcast_pyobj
(TaskRunner pid=1714137)     dist.broadcast(tensor_size, src=src, group=dist_group)
(TaskRunner pid=1714137)   File "/home/jobuser/.local/lib/python3.10/site-packages/torch/distributed/c10d_logger.py", line 81, in wrapper
(TaskRunner pid=1714137)   File "/home/jobuser/.local/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py", line 2726, in broadcast
(TaskRunner pid=1714137)     work = group.broadcast([tensor], opts)
(TaskRunner pid=1714137) torch.distributed.DistBackendError: [3] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: failed to recv, got 0 bytes
(TaskRunner pid=1714137) Exception raised from recvBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:671 (most recent call first):
(TaskRunner pid=1714137) frame #0: c10::Error::Error(c10::SourceLocation, std::string) + 0x96 (0x74b720f6c1b6 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libc10.so)
(TaskRunner pid=1714137) frame #1: <unknown function> + 0x6312a88 (0x74b7006e5a88 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #2: c10d::TCPStore::doWait(c10::ArrayRef<std::string>, std::chrono::duration<long, std::ratio<1l, 1000l> >) + 0x367 (0x74b7006e2817 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #3: c10d::TCPStore::doGet(std::string const&) + 0x2a (0x74b7006e2b9a in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #4: c10d::TCPStore::get(std::string const&) + 0x7a (0x74b7006e393a in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #5: c10d::PrefixStore::get(std::string const&) + 0x31 (0x74b7006905d1 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #6: c10d::PrefixStore::get(std::string const&) + 0x31 (0x74b7006905d1 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #7: c10d::PrefixStore::get(std::string const&) + 0x31 (0x74b7006905d1 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #8: c10d::PrefixStore::get(std::string const&) + 0x31 (0x74b7006905d1 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #9: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::string const&, int) + 0xaf (0x74b6c6420a9f in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so)
(TaskRunner pid=1714137) frame #10: c10d::ProcessGroupNCCL::initNCCLComm(std::string const&, c10::Device&, c10d::OpType, int, bool) + 0x1176 (0x74b6c642e666 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so)
(TaskRunner pid=1714137) frame #11: c10d::ProcessGroupNCCL::broadcast(std::vector<at::Tensor, std::allocator<at::Tensor> >&, c10d::BroadcastOptions const&) + 0x1265 (0x74b6c6439175 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so)
(TaskRunner pid=1714137) frame #12: <unknown function> + 0x62b13b6 (0x74b7006843b6 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #13: <unknown function> + 0x62bcd26 (0x74b70068fd26 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #14: <unknown function> + 0x58bc27b (0x74b6ffc8f27b in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #15: <unknown function> + 0x58b9af9 (0x74b6ffc8caf9 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #16: <unknown function> + 0x1aca9f8 (0x74b6fbe9d9f8 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #17: <unknown function> + 0x62c339a (0x74b70069639a in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #18: <unknown function> + 0x62d3b30 (0x74b7006a6b30 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #19: <unknown function> + 0xe9fac5 (0x74b71032eac5 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_python.so)
(TaskRunner pid=1714137) frame #20: <unknown function> + 0x518d37 (0x74b70f9a7d37 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_python.so)
(TaskRunner pid=1714137) <omitting python frames>
(TaskRunner pid=1714137) frame #55: <unknown function> + 0x769d4f (0x74e691c51d4f in /home/jobuser/.local/lib/python3.10/site-packages/ray/_raylet.so)
(TaskRunner pid=1714137) frame #56: <unknown function> + 0x7daaa8 (0x74e691cc2aa8 in /home/jobuser/.local/lib/python3.10/site-packages/ray/_raylet.so)
(TaskRunner pid=1714137) frame #57: <unknown function> + 0x769d4f (0x74e691c51d4f in /home/jobuser/.local/lib/python3.10/site-packages/ray/_raylet.so)
(TaskRunner pid=1714137) frame #58: <unknown function> + 0x86fc56 (0x74e691d57c56 in /home/jobuser/.local/lib/python3.10/site-packages/ray/_raylet.so)
(TaskRunner pid=1714137) frame #59: std::_Function_handler<ray::Status (ray::rpc::Address const&, ray::rpc::TaskType, std::string, ray::core::RayFunction const&, std::unordered_map<std::string, double, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::pair<std::string const, double> > > const&, std::vector<std::shared_ptr<ray::RayObject>, std::allocator<std::shared_ptr<ray::RayObject> > > const&, std::vector<ray::rpc::ObjectReference, std::allocator<ray::rpc::ObjectReference> > const&, std::string const&, std::string const&, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*, std::vector<std::pair<ray::ObjectID, bool>, std::allocator<std::pair<ray::ObjectID, bool> > >*, std::shared_ptr<ray::LocalMemoryBuffer>&, bool*, std::string*, std::vector<ray::ConcurrencyGroup, std::allocator<ray::ConcurrencyGroup> > const&, std::string, bool, bool, bool, long), ray::Status (*)(ray::rpc::Address const&, ray::rpc::TaskType, std::string, ray::core::RayFunction const&, std::unordered_map<std::string, double, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::pair<std::string const, double> > > const&, std::vector<std::shared_ptr<ray::RayObject>, std::allocator<std::shared_ptr<ray::RayObject> > > const&, std::vector<ray::rpc::ObjectReference, std::allocator<ray::rpc::ObjectReference> > const&, std::string, std::string, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*, std::vector<std::pair<ray::ObjectID, bool>, std::allocator<std::pair<ray::ObjectID, bool> > >*, std::shared_ptr<ray::LocalMemoryBuffer>&, bool*, std::string*, std::vector<ray::ConcurrencyGroup, std::allocator<ray::ConcurrencyGroup> > const&, std::string, bool, bool, bool, long)>::_M_invoke(std::_Any_data const&, ray::rpc::Address const&, ray::rpc::TaskType&&, std::string&&, ray::core::RayFunction const&, std::unordered_map<std::string, double, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::pair<std::string const, double> > > const&, std::vector<std::shared_ptr<ray::RayObject>, std::allocator<std::shared_ptr<ray::RayObject> > > const&, std::vector<ray::rpc::ObjectReference, std::allocator<ray::rpc::ObjectReference> > const&, std::string const&, std::string const&, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*&&, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*&&, std::vector<std::pair<ray::ObjectID, bool>, std::allocator<std::pair<ray::ObjectID, bool> > >*&&, std::shared_ptr<ray::LocalMemoryBuffer>&, bool*&&, std::string*&&, std::vector<ray::ConcurrencyGroup, std::allocator<ray::ConcurrencyGroup> > const&, std::string&&, bool&&, bool&&, bool&&, long&&) + 0x169 (0x74e691c58d39 in /home/jobuser/.local/lib/python3.10/site-packages/ray/_raylet.so)
(TaskRunner pid=1714137) frame #60: ray::core::CoreWorker::ExecuteTask(ray::TaskSpecification const&, std::optional<std::unordered_map<std::string, std::vector<std::pair<long, double>, std::allocator<std::pair<long, double> > >, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::pair<std::string const, std::vector<std::pair<long, double>, std::allocator<std::pair<long, double> > > > > > >, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*, std::vector<std::pair<ray::ObjectID, bool>, std::allocator<std::pair<ray::ObjectID, bool> > >*, google::protobuf::RepeatedPtrField<ray::rpc::ObjectReferenceCount>*, bool*, std::string*) + 0xdf5 (0x74e691e6a245 in /home/jobuser/.local/lib/python3.10/site-packages/ray/_raylet.so)
(TaskRunner pid=1714137) frame #61: std::_Function_handler<ray::Status (ray::TaskSpecification const&, std::optional<std::unordered_map<std::string, std::vector<std::pair<long, double>, std::allocator<std::pair<long, double> > >, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::pair<std::string const, std::vector<std::pair<long, double>, std::allocator<std::pair<long, double> > > > > > >, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*, std::vector<std::pair<ray::ObjectID, bool>, std::allocator<std::pair<ray::ObjectID, bool> > >*, google::protobuf::RepeatedPtrField<ray::rpc::ObjectReferenceCount>*, bool*, std::string*), std::_Bind<ray::Status (ray::core::CoreWorker::*(ray::core::CoreWorker*, std::_Placeholder<1>, std::_Placeholder<2>, std::_Placeholder<3>, std::_Placeholder<4>, std::_Placeholder<5>, std::_Placeholder<6>, std::_Placeholder<7>, std::_Placeholder<8>))(ray::TaskSpecification const&, std::optional<std::unordered_map<std::string, std::vector<std::pair<long, double>, std::allocator<std::pair<long, double> > >, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::pair<std::string const, std::vector<std::pair<long, double>, std::allocator<std::pair<long, double> > > > > > >, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*, std::vector<std::pair<ray::ObjectID, bool>, std::allocator<std::pair<ray::ObjectID, bool> > >*, google::protobuf::RepeatedPtrField<ray::rpc::ObjectReferenceCount>*, bool*, std::string*)> >::_M_invoke(std::_Any_data const&, ray::TaskSpecification const&, std::optional<std::unordered_map<std::string, std::vector<std::pair<long, double>, std::allocator<std::pair<long, double> > >, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::pair<std::string const, std::vector<std::pair<long, double>, std::allocator<std::pair<long, double> > > > > > >&&, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*&&, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*&&, std::vector<std::pair<ray::ObjectID, bool>, std::allocator<std::pair<ray::ObjectID, bool> > >*&&, google::protobuf::RepeatedPtrField<ray::rpc::ObjectReferenceCount>*&&, bool*&&, std::string*&&) + 0x75 (0x74e691df4845 in /home/jobuser/.local/lib/python3.10/site-packages/ray/_raylet.so)
(TaskRunner pid=1714137) frame #62: <unknown function> + 0x9f30ce (0x74e691edb0ce in /home/jobuser/.local/lib/python3.10/site-packages/ray/_raylet.so)
(TaskRunner pid=1714137) . This may indicate a possible application crash on rank 0 or a network set up issue.
(TaskRunner pid=1714137) Unhandled error (suppress with 'RAY_IGNORE_UNHANDLED_ERRORS=1'): ray::WorkerDict.actor_rollout_generate_sequences() (pid=1733168, ip=100.96.253.216, actor_id=82c7bbaa109872e66aff35e701000000, repr=<verl.single_controller.ray.base.WorkerDict object at 0x7fc525913a00>)
(TaskRunner pid=1714137)   File "/home/jobuser/resources/verl/single_controller/ray/base.py", line 625, in func
(TaskRunner pid=1714137)     return getattr(self.worker_dict[key], name)(*args, **kwargs)
(TaskRunner pid=1714137)   File "/home/jobuser/resources/verl/single_controller/base/decorator.py", line 534, in inner
(TaskRunner pid=1714137)   File "/home/jobuser/resources/verl/workers/fsdp_workers.py", line 630, in generate_sequences
(TaskRunner pid=1714137)     output = self.rollout.generate_sequences_with_tools(prompts=prompts)
(TaskRunner pid=1714137)   File "/home/jobuser/resources/verl/utils/debug/performance.py", line 78, in f
(TaskRunner pid=1714137)     return self.log(decorated_function, *args, **kwargs)
(TaskRunner pid=1714137)   File "/home/jobuser/resources/verl/utils/debug/performance.py", line 88, in log
(TaskRunner pid=1714137)     output = func(*args, **kwargs)
(TaskRunner pid=1714137)   File "/home/jobuser/resources/verl/workers/rollout/sglang_rollout/async_sglang_rollout.py", line 619, in generate_sequences_with_tools
(TaskRunner pid=1714137)     [sorted_output_req_list] = broadcast_pyobj(
(TaskRunner pid=1714137)   File "/home/jobuser/resources/verl/workers/rollout/sglang_rollout/utils.py", line 55, in broadcast_pyobj
(TaskRunner pid=1714137)     dist.broadcast(tensor_size, src=src, group=dist_group)
(TaskRunner pid=1714137)   File "/home/jobuser/.local/lib/python3.10/site-packages/torch/distributed/c10d_logger.py", line 81, in wrapper
(TaskRunner pid=1714137)   File "/home/jobuser/.local/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py", line 2726, in broadcast
(TaskRunner pid=1714137)     work = group.broadcast([tensor], opts)
(TaskRunner pid=1714137) torch.distributed.DistBackendError: [2] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: failed to recv, got 0 bytes
(TaskRunner pid=1714137) Exception raised from recvBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:671 (most recent call first):
(TaskRunner pid=1714137) frame #0: c10::Error::Error(c10::SourceLocation, std::string) + 0x96 (0x7ff81d1fb1b6 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libc10.so)
(TaskRunner pid=1714137) frame #1: <unknown function> + 0x6312a88 (0x7fc939256a88 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #2: c10d::TCPStore::doWait(c10::ArrayRef<std::string>, std::chrono::duration<long, std::ratio<1l, 1000l> >) + 0x367 (0x7fc939253817 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #3: c10d::TCPStore::doGet(std::string const&) + 0x2a (0x7fc939253b9a in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #4: c10d::TCPStore::get(std::string const&) + 0x7a (0x7fc93925493a in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #5: c10d::PrefixStore::get(std::string const&) + 0x31 (0x7fc9392015d1 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #6: c10d::PrefixStore::get(std::string const&) + 0x31 (0x7fc9392015d1 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #7: c10d::PrefixStore::get(std::string const&) + 0x31 (0x7fc9392015d1 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #8: c10d::PrefixStore::get(std::string const&) + 0x31 (0x7fc9392015d1 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #9: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::string const&, int) + 0xaf (0x7fc8fef91a9f in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so)
(TaskRunner pid=1714137) frame #10: c10d::ProcessGroupNCCL::initNCCLComm(std::string const&, c10::Device&, c10d::OpType, int, bool) + 0x1176 (0x7fc8fef9f666 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so)
(TaskRunner pid=1714137) frame #11: c10d::ProcessGroupNCCL::broadcast(std::vector<at::Tensor, std::allocator<at::Tensor> >&, c10d::BroadcastOptions const&) + 0x1265 (0x7fc8fefaa175 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so)
(TaskRunner pid=1714137) frame #12: <unknown function> + 0x62b13b6 (0x7fc9391f53b6 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #13: <unknown function> + 0x62bcd26 (0x7fc939200d26 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #14: <unknown function> + 0x58bc27b (0x7fc93880027b in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #15: <unknown function> + 0x58b9af9 (0x7fc9387fdaf9 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #16: <unknown function> + 0x1aca9f8 (0x7fc934a0e9f8 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #17: <unknown function> + 0x62c339a (0x7fc93920739a in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #18: <unknown function> + 0x62d3b30 (0x7fc939217b30 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #19: <unknown function> + 0xe9fac5 (0x7ff81e12eac5 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_python.so)
(TaskRunner pid=1714137) frame #20: <unknown function> + 0x518d37 (0x7ff81d7a7d37 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_python.so)
(TaskRunner pid=1714137) <omitting python frames>
(TaskRunner pid=1714137) frame #55: <unknown function> + 0x769d4f (0x7ff8d6573d4f in /home/jobuser/.local/lib/python3.10/site-packages/ray/_raylet.so)
(TaskRunner pid=1714137) frame #56: <unknown function> + 0x7daaa8 (0x7ff8d65e4aa8 in /home/jobuser/.local/lib/python3.10/site-packages/ray/_raylet.so)
(TaskRunner pid=1714137) frame #57: <unknown function> + 0x769d4f (0x7ff8d6573d4f in /home/jobuser/.local/lib/python3.10/site-packages/ray/_raylet.so)
(TaskRunner pid=1714137) frame #58: <unknown function> + 0x86fc56 (0x7ff8d6679c56 in /home/jobuser/.local/lib/python3.10/site-packages/ray/_raylet.so)
(TaskRunner pid=1714137) frame #59: std::_Function_handler<ray::Status (ray::rpc::Address const&, ray::rpc::TaskType, std::string, ray::core::RayFunction const&, std::unordered_map<std::string, double, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::pair<std::string const, double> > > const&, std::vector<std::shared_ptr<ray::RayObject>, std::allocator<std::shared_ptr<ray::RayObject> > > const&, std::vector<ray::rpc::ObjectReference, std::allocator<ray::rpc::ObjectReference> > const&, std::string const&, std::string const&, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*, std::vector<std::pair<ray::ObjectID, bool>, std::allocator<std::pair<ray::ObjectID, bool> > >*, std::shared_ptr<ray::LocalMemoryBuffer>&, bool*, std::string*, std::vector<ray::ConcurrencyGroup, std::allocator<ray::ConcurrencyGroup> > const&, std::string, bool, bool, bool, long), ray::Status (*)(ray::rpc::Address const&, ray::rpc::TaskType, std::string, ray::core::RayFunction const&, std::unordered_map<std::string, double, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::pair<std::string const, double> > > const&, std::vector<std::shared_ptr<ray::RayObject>, std::allocator<std::shared_ptr<ray::RayObject> > > const&, std::vector<ray::rpc::ObjectReference, std::allocator<ray::rpc::ObjectReference> > const&, std::string, std::string, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*, std::vector<std::pair<ray::ObjectID, bool>, std::allocator<std::pair<ray::ObjectID, bool> > >*, std::shared_ptr<ray::LocalMemoryBuffer>&, bool*, std::string*, std::vector<ray::ConcurrencyGroup, std::allocator<ray::ConcurrencyGroup> > const&, std::string, bool, bool, bool, long)>::_M_invoke(std::_Any_data const&, ray::rpc::Address const&, ray::rpc::TaskType&&, std::string&&, ray::core::RayFunction const&, std::unordered_map<std::string, double, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::pair<std::string const, double> > > const&, std::vector<std::shared_ptr<ray::RayObject>, std::allocator<std::shared_ptr<ray::RayObject> > > const&, std::vector<ray::rpc::ObjectReference, std::allocator<ray::rpc::ObjectReference> > const&, std::string const&, std::string const&, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*&&, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*&&, std::vector<std::pair<ray::ObjectID, bool>, std::allocator<std::pair<ray::ObjectID, bool> > >*&&, std::shared_ptr<ray::LocalMemoryBuffer>&, bool*&&, std::string*&&, std::vector<ray::ConcurrencyGroup, std::allocator<ray::ConcurrencyGroup> > const&, std::string&&, bool&&, bool&&, bool&&, long&&) + 0x169 (0x7ff8d657ad39 in /home/jobuser/.local/lib/python3.10/site-packages/ray/_raylet.so)
(TaskRunner pid=1714137) frame #60: ray::core::CoreWorker::ExecuteTask(ray::TaskSpecification const&, std::optional<std::unordered_map<std::string, std::vector<std::pair<long, double>, std::allocator<std::pair<long, double> > >, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::pair<std::string const, std::vector<std::pair<long, double>, std::allocator<std::pair<long, double> > > > > > >, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*, std::vector<std::pair<ray::ObjectID, bool>, std::allocator<std::pair<ray::ObjectID, bool> > >*, google::protobuf::RepeatedPtrField<ray::rpc::ObjectReferenceCount>*, bool*, std::string*) + 0xdf5 (0x7ff8d678c245 in /home/jobuser/.local/lib/python3.10/site-packages/ray/_raylet.so)
(TaskRunner pid=1714137) frame #61: std::_Function_handler<ray::Status (ray::TaskSpecification const&, std::optional<std::unordered_map<std::string, std::vector<std::pair<long, double>, std::allocator<std::pair<long, double> > >, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::pair<std::string const, std::vector<std::pair<long, double>, std::allocator<std::pair<long, double> > > > > > >, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*, std::vector<std::pair<ray::ObjectID, bool>, std::allocator<std::pair<ray::ObjectID, bool> > >*, google::protobuf::RepeatedPtrField<ray::rpc::ObjectReferenceCount>*, bool*, std::string*), std::_Bind<ray::Status (ray::core::CoreWorker::*(ray::core::CoreWorker*, std::_Placeholder<1>, std::_Placeholder<2>, std::_Placeholder<3>, std::_Placeholder<4>, std::_Placeholder<5>, std::_Placeholder<6>, std::_Placeholder<7>, std::_Placeholder<8>))(ray::TaskSpecification const&, std::optional<std::unordered_map<std::string, std::vector<std::pair<long, double>, std::allocator<std::pair<long, double> > >, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::pair<std::string const, std::vector<std::pair<long, double>, std::allocator<std::pair<long, double> > > > > > >, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*, std::vector<std::pair<ray::ObjectID, bool>, std::allocator<std::pair<ray::ObjectID, bool> > >*, google::protobuf::RepeatedPtrField<ray::rpc::ObjectReferenceCount>*, bool*, std::string*)> >::_M_invoke(std::_Any_data const&, ray::TaskSpecification const&, std::optional<std::unordered_map<std::string, std::vector<std::pair<long, double>, std::allocator<std::pair<long, double> > >, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::pair<std::string const, std::vector<std::pair<long, double>, std::allocator<std::pair<long, double> > > > > > >&&, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*&&, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*&&, std::vector<std::pair<ray::ObjectID, bool>, std::allocator<std::pair<ray::ObjectID, bool> > >*&&, google::protobuf::RepeatedPtrField<ray::rpc::ObjectReferenceCount>*&&, bool*&&, std::string*&&) + 0x75 (0x7ff8d6716845 in /home/jobuser/.local/lib/python3.10/site-packages/ray/_raylet.so)
(TaskRunner pid=1714137) frame #62: <unknown function> + 0x9f30ce (0x7ff8d67fd0ce in /home/jobuser/.local/lib/python3.10/site-packages/ray/_raylet.so)
(TaskRunner pid=1714137) . This may indicate a possible application crash on rank 0 or a network set up issue.
(TaskRunner pid=1714137) Unhandled error (suppress with 'RAY_IGNORE_UNHANDLED_ERRORS=1'): ray::WorkerDict.actor_rollout_generate_sequences() (pid=1733167, ip=100.96.253.216, actor_id=82e75560ee1db7816b11a0f201000000, repr=<verl.single_controller.ray.base.WorkerDict object at 0x75b64a9dfa00>)
(TaskRunner pid=1714137)   File "/home/jobuser/resources/verl/single_controller/ray/base.py", line 625, in func
(TaskRunner pid=1714137)     return getattr(self.worker_dict[key], name)(*args, **kwargs)
(TaskRunner pid=1714137)   File "/home/jobuser/resources/verl/single_controller/base/decorator.py", line 534, in inner
(TaskRunner pid=1714137)   File "/home/jobuser/resources/verl/workers/fsdp_workers.py", line 630, in generate_sequences
(TaskRunner pid=1714137)     output = self.rollout.generate_sequences_with_tools(prompts=prompts)
(TaskRunner pid=1714137)   File "/home/jobuser/resources/verl/utils/debug/performance.py", line 78, in f
(TaskRunner pid=1714137)     return self.log(decorated_function, *args, **kwargs)
(TaskRunner pid=1714137)   File "/home/jobuser/resources/verl/utils/debug/performance.py", line 88, in log
(TaskRunner pid=1714137)     output = func(*args, **kwargs)
(TaskRunner pid=1714137)   File "/home/jobuser/resources/verl/workers/rollout/sglang_rollout/async_sglang_rollout.py", line 619, in generate_sequences_with_tools
(TaskRunner pid=1714137)     [sorted_output_req_list] = broadcast_pyobj(
(TaskRunner pid=1714137)   File "/home/jobuser/resources/verl/workers/rollout/sglang_rollout/utils.py", line 55, in broadcast_pyobj
(TaskRunner pid=1714137)     dist.broadcast(tensor_size, src=src, group=dist_group)
(TaskRunner pid=1714137)   File "/home/jobuser/.local/lib/python3.10/site-packages/torch/distributed/c10d_logger.py", line 81, in wrapper
(TaskRunner pid=1714137)   File "/home/jobuser/.local/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py", line 2726, in broadcast
(TaskRunner pid=1714137)     work = group.broadcast([tensor], opts)
(TaskRunner pid=1714137) torch.distributed.DistBackendError: [1] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: failed to recv, got 0 bytes
(TaskRunner pid=1714137) Exception raised from recvBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:671 (most recent call first):
(TaskRunner pid=1714137) frame #0: c10::Error::Error(c10::SourceLocation, std::string) + 0x96 (0x75b69036c1b6 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libc10.so)
(TaskRunner pid=1714137) frame #1: <unknown function> + 0x6312a88 (0x758765256a88 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #2: c10d::TCPStore::doWait(c10::ArrayRef<std::string>, std::chrono::duration<long, std::ratio<1l, 1000l> >) + 0x367 (0x758765253817 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #3: c10d::TCPStore::doGet(std::string const&) + 0x2a (0x758765253b9a in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #4: c10d::TCPStore::get(std::string const&) + 0x7a (0x75876525493a in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #5: c10d::PrefixStore::get(std::string const&) + 0x31 (0x7587652015d1 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #6: c10d::PrefixStore::get(std::string const&) + 0x31 (0x7587652015d1 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #7: c10d::PrefixStore::get(std::string const&) + 0x31 (0x7587652015d1 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #8: c10d::PrefixStore::get(std::string const&) + 0x31 (0x7587652015d1 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #9: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::string const&, int) + 0xaf (0x75872af91a9f in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so)
(TaskRunner pid=1714137) frame #10: c10d::ProcessGroupNCCL::initNCCLComm(std::string const&, c10::Device&, c10d::OpType, int, bool) + 0x1176 (0x75872af9f666 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so)
(TaskRunner pid=1714137) frame #11: c10d::ProcessGroupNCCL::broadcast(std::vector<at::Tensor, std::allocator<at::Tensor> >&, c10d::BroadcastOptions const&) + 0x1265 (0x75872afaa175 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so)
(TaskRunner pid=1714137) frame #12: <unknown function> + 0x62b13b6 (0x7587651f53b6 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #13: <unknown function> + 0x62bcd26 (0x758765200d26 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #14: <unknown function> + 0x58bc27b (0x75876480027b in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #15: <unknown function> + 0x58b9af9 (0x7587647fdaf9 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #16: <unknown function> + 0x1aca9f8 (0x758760a0e9f8 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #17: <unknown function> + 0x62c339a (0x75876520739a in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #18: <unknown function> + 0x62d3b30 (0x758765217b30 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
(TaskRunner pid=1714137) frame #19: <unknown function> + 0xe9fac5 (0x75b67392eac5 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_python.so)
(TaskRunner pid=1714137) frame #20: <unknown function> + 0x518d37 (0x75b672fa7d37 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_python.so)
(TaskRunner pid=1714137) <omitting python frames>
(TaskRunner pid=1714137) frame #55: <unknown function> + 0x769d4f (0x75b720a41d4f in /home/jobuser/.local/lib/python3.10/site-packages/ray/_raylet.so)
(TaskRunner pid=1714137) frame #56: <unknown function> + 0x7daaa8 (0x75b720ab2aa8 in /home/jobuser/.local/lib/python3.10/site-packages/ray/_raylet.so)
(TaskRunner pid=1714137) frame #57: <unknown function> + 0x769d4f (0x75b720a41d4f in /home/jobuser/.local/lib/python3.10/site-packages/ray/_raylet.so)
(TaskRunner pid=1714137) frame #58: <unknown function> + 0x86fc56 (0x75b720b47c56 in /home/jobuser/.local/lib/python3.10/site-packages/ray/_raylet.so)
(TaskRunner pid=1714137) frame #59: std::_Function_handler<ray::Status (ray::rpc::Address const&, ray::rpc::TaskType, std::string, ray::core::RayFunction const&, std::unordered_map<std::string, double, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::pair<std::string const, double> > > const&, std::vector<std::shared_ptr<ray::RayObject>, std::allocator<std::shared_ptr<ray::RayObject> > > const&, std::vector<ray::rpc::ObjectReference, std::allocator<ray::rpc::ObjectReference> > const&, std::string const&, std::string const&, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*, std::vector<std::pair<ray::ObjectID, bool>, std::allocator<std::pair<ray::ObjectID, bool> > >*, std::shared_ptr<ray::LocalMemoryBuffer>&, bool*, std::string*, std::vector<ray::ConcurrencyGroup, std::allocator<ray::ConcurrencyGroup> > const&, std::string, bool, bool, bool, long), ray::Status (*)(ray::rpc::Address const&, ray::rpc::TaskType, std::string, ray::core::RayFunction const&, std::unordered_map<std::string, double, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::pair<std::string const, double> > > const&, std::vector<std::shared_ptr<ray::RayObject>, std::allocator<std::shared_ptr<ray::RayObject> > > const&, std::vector<ray::rpc::ObjectReference, std::allocator<ray::rpc::ObjectReference> > const&, std::string, std::string, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*, std::vector<std::pair<ray::ObjectID, bool>, std::allocator<std::pair<ray::ObjectID, bool> > >*, std::shared_ptr<ray::LocalMemoryBuffer>&, bool*, std::string*, std::vector<ray::ConcurrencyGroup, std::allocator<ray::ConcurrencyGroup> > const&, std::string, bool, bool, bool, long)>::_M_invoke(std::_Any_data const&, ray::rpc::Address const&, ray::rpc::TaskType&&, std::string&&, ray::core::RayFunction const&, std::unordered_map<std::string, double, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::pair<std::string const, double> > > const&, std::vector<std::shared_ptr<ray::RayObject>, std::allocator<std::shared_ptr<ray::RayObject> > > const&, std::vector<ray::rpc::ObjectReference, std::allocator<ray::rpc::ObjectReference> > const&, std::string const&, std::string const&, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*&&, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*&&, std::vector<std::pair<ray::ObjectID, bool>, std::allocator<std::pair<ray::ObjectID, bool> > >*&&, std::shared_ptr<ray::LocalMemoryBuffer>&, bool*&&, std::string*&&, std::vector<ray::ConcurrencyGroup, std::allocator<ray::ConcurrencyGroup> > const&, std::string&&, bool&&, bool&&, bool&&, long&&) + 0x169 (0x75b720a48d39 in /home/jobuser/.local/lib/python3.10/site-packages/ray/_raylet.so)
(TaskRunner pid=1714137) frame #60: ray::core::CoreWorker::ExecuteTask(ray::TaskSpecification const&, std::optional<std::unordered_map<std::string, std::vector<std::pair<long, double>, std::allocator<std::pair<long, double> > >, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::pair<std::string const, std::vector<std::pair<long, double>, std::allocator<std::pair<long, double> > > > > > >, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*, std::vector<std::pair<ray::ObjectID, bool>, std::allocator<std::pair<ray::ObjectID, bool> > >*, google::protobuf::RepeatedPtrField<ray::rpc::ObjectReferenceCount>*, bool*, std::string*) + 0xdf5 (0x75b720c5a245 in /home/jobuser/.local/lib/python3.10/site-packages/ray/_raylet.so)
(TaskRunner pid=1714137) frame #61: std::_Function_handler<ray::Status (ray::TaskSpecification const&, std::optional<std::unordered_map<std::string, std::vector<std::pair<long, double>, std::allocator<std::pair<long, double> > >, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::pair<std::string const, std::vector<std::pair<long, double>, std::allocator<std::pair<long, double> > > > > > >, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*, std::vector<std::pair<ray::ObjectID, bool>, std::allocator<std::pair<ray::ObjectID, bool> > >*, google::protobuf::RepeatedPtrField<ray::rpc::ObjectReferenceCount>*, bool*, std::string*), std::_Bind<ray::Status (ray::core::CoreWorker::*(ray::core::CoreWorker*, std::_Placeholder<1>, std::_Placeholder<2>, std::_Placeholder<3>, std::_Placeholder<4>, std::_Placeholder<5>, std::_Placeholder<6>, std::_Placeholder<7>, std::_Placeholder<8>))(ray::TaskSpecification const&, std::optional<std::unordered_map<std::string, std::vector<std::pair<long, double>, std::allocator<std::pair<long, double> > >, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::pair<std::string const, std::vector<std::pair<long, double>, std::allocator<std::pair<long, double> > > > > > >, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*, std::vector<std::pair<ray::ObjectID, bool>, std::allocator<std::pair<ray::ObjectID, bool> > >*, google::protobuf::RepeatedPtrField<ray::rpc::ObjectReferenceCount>*, bool*, std::string*)> >::_M_invoke(std::_Any_data const&, ray::TaskSpecification const&, std::optional<std::unordered_map<std::string, std::vector<std::pair<long, double>, std::allocator<std::pair<long, double> > >, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::pair<std::string const, std::vector<std::pair<long, double>, std::allocator<std::pair<long, double> > > > > > >&&, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*&&, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*&&, std::vector<std::pair<ray::ObjectID, bool>, std::allocator<std::pair<ray::ObjectID, bool> > >*&&, google::protobuf::RepeatedPtrField<ray::rpc::ObjectReferenceCount>*&&, bool*&&, std::string*&&) + 0x75 (0x75b720be4845 in /home/jobuser/.local/lib/python3.10/site-packages/ray/_raylet.so)
(TaskRunner pid=1714137) frame #62: <unknown function> + 0x9f30ce (0x75b720ccb0ce in /home/jobuser/.local/lib/python3.10/site-packages/ray/_raylet.so)
(TaskRunner pid=1714137) . This may indicate a possible application crash on rank 0 or a network set up issue.
(WorkerDict pid=1732013) }
(WorkerDict pid=1732013) 
(WorkerDict pid=1733170) Actor use_remove_padding=True [repeated 7x across cluster]
(WorkerDict pid=1733170) Actor use_fused_kernels=False [repeated 7x across cluster]
(WorkerDict pid=1732013) Monkey patch _flash_attention_forward in transformers.integrations.flash_attention
(WorkerDict pid=1733168) Monkey patch _flash_attention_forward in transformers.integrations.flash_attention
(WorkerDict pid=1733172) Monkey patch _flash_attention_forward in transformers.integrations.flash_attention [repeated 2x across cluster]
(WorkerDict pid=1732013) Qwen2ForCausalLM contains 32.76B parameters
(WorkerDict pid=1732013) wrap_policy: functools.partial(<function _or_policy at 0x71c356939000>, policies=[functools.partial(<function transformer_auto_wrap_policy at 0x71c356938ee0>, transformer_layer_cls={<class 'transformers.models.qwen2.modeling_qwen2.Qwen2DecoderLayer'>})])
(WorkerDict pid=1733171) Monkey patch _flash_attention_forward in transformers.integrations.flash_attention [repeated 3x across cluster]
(WorkerDict pid=1733167) Total steps: 25, num_warmup_steps: 0
(WorkerDict pid=1733170) wrap_policy: functools.partial(<function _or_policy at 0x77e34f3e1000>, policies=[functools.partial(<function transformer_auto_wrap_policy at 0x77e34f3e0ee0>, transformer_layer_cls={<class 'transformers.models.qwen2.modeling_qwen2.Qwen2DecoderLayer'>})]) [repeated 7x across cluster]
(WorkerDict pid=1733169) Monkey patch _flash_attention_forward in transformers.integrations.flash_attention
(WorkerDict pid=1733172) Actor use_remove_padding=True
(WorkerDict pid=1733172) Actor use_fused_kernels=False
(WorkerDict pid=1732013) Total steps: 25, num_warmup_steps: 0 [repeated 7x across cluster]
(WorkerDict pid=1732013) Before building sglang_async rollout, memory allocated (GB): 0.00, memory reserved (GB): 0.00, device memory used/total (GB): 3.35/139.72
(WorkerDict pid=1733167) kwargs: {'n': 8, 'max_new_tokens': 1024, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'repetition_penalty': 1.0, 'temperature': 1.0, 'top_k': -1, 'top_p': 1, 'ignore_eos': False}
(WorkerDict pid=1732013) Actor use_remove_padding=True [repeated 7x across cluster]
(WorkerDict pid=1732013) Actor use_fused_kernels=False [repeated 7x across cluster]
(WorkerDict pid=1732013) NCCL version 2.21.5+cuda12.4
(WorkerDict pid=1733170) kwargs: {'n': 8, 'max_new_tokens': 1024, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'repetition_penalty': 1.0, 'temperature': 1.0, 'top_k': -1, 'top_p': 1, 'ignore_eos': False} [repeated 6x across cluster]
(WorkerDict pid=1732013) After building sglang_async rollout, memory allocated (GB): 0.00, memory reserved (GB): 0.00, device memory used/total (GB): 8.74/139.72
(WorkerDict pid=1732013) After building sharding manager, memory allocated (GB): 0.00, memory reserved (GB): 0.00, device memory used/total (GB): 8.74/139.72
(WorkerDict pid=1732013) kwargs: {'n': 8, 'max_new_tokens': 1024, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'repetition_penalty': 1.0, 'temperature': 1.0, 'top_k': -1, 'top_p': 1, 'ignore_eos': False}
(TaskRunner pid=1714137) test_gen_batch meta info: {'eos_token_id': 151645, 'pad_token_id': 151643, 'recompute_log_prob': False, 'do_sample': False, 'validate': True}
Traceback (most recent call last):
  File "/home/jobuser/resources/verl/trainer/main_ppo.py", line 64, in main
    run_ppo(config)
  File "/home/jobuser/resources/verl/trainer/main_ppo.py", line 76, in run_ppo
    ray.get(runner.run.remote(config))
  File "/home/jobuser/.local/lib/python3.10/site-packages/ray/_private/auto_init_hook.py", line 21, in auto_init_wrapper
    return fn(*args, **kwargs)
  File "/home/jobuser/.local/lib/python3.10/site-packages/ray/_private/client_mode_hook.py", line 103, in wrapper
    return func(*args, **kwargs)
  File "/home/jobuser/.local/lib/python3.10/site-packages/ray/_private/worker.py", line 2822, in get
    values, debugger_breakpoint = worker.get_objects(object_refs, timeout=timeout)
  File "/home/jobuser/.local/lib/python3.10/site-packages/ray/_private/worker.py", line 930, in get_objects
    raise value.as_instanceof_cause()
ray.exceptions.RayTaskError(DistBackendError): ray::TaskRunner.run() (pid=1714137, ip=100.96.253.216, actor_id=b62deb437e3f5623886f56ac01000000, repr=<main_ppo.TaskRunner object at 0x700127cff010>)
  File "/home/jobuser/resources/verl/trainer/main_ppo.py", line 183, in run
    trainer.fit()
  File "/home/jobuser/resources/verl/trainer/ppo/ray_trainer.py", line 872, in fit
    val_metrics = self._validate()
  File "/home/jobuser/resources/verl/trainer/ppo/ray_trainer.py", line 607, in _validate
    test_output_gen_batch_padded = self.actor_rollout_wg.generate_sequences(test_gen_batch_padded)
  File "/home/jobuser/resources/verl/single_controller/ray/base.py", line 49, in func
    output = ray.get(output)
ray.exceptions.RayTaskError(DistBackendError): ray::WorkerDict.actor_rollout_generate_sequences() (pid=1733170, ip=100.96.253.216, actor_id=df3193cf46083e93d60809e201000000, repr=<verl.single_controller.ray.base.WorkerDict object at 0x77b180b2bfa0>)
  File "/home/jobuser/resources/verl/single_controller/ray/base.py", line 625, in func
    return getattr(self.worker_dict[key], name)(*args, **kwargs)
  File "/home/jobuser/resources/verl/single_controller/base/decorator.py", line 534, in inner
    return func(*args, **kwargs)
  File "/home/jobuser/resources/verl/workers/fsdp_workers.py", line 630, in generate_sequences
    output = self.rollout.generate_sequences_with_tools(prompts=prompts)
  File "/home/jobuser/resources/verl/utils/debug/performance.py", line 78, in f
    return self.log(decorated_function, *args, **kwargs)
  File "/home/jobuser/resources/verl/utils/debug/performance.py", line 88, in log
    output = func(*args, **kwargs)
  File "/home/jobuser/.local/lib/python3.10/site-packages/torch/utils/_contextlib.py", line 116, in decorate_context
    return func(*args, **kwargs)
  File "/home/jobuser/resources/verl/workers/rollout/sglang_rollout/async_sglang_rollout.py", line 619, in generate_sequences_with_tools
    [sorted_output_req_list] = broadcast_pyobj(
  File "/home/jobuser/resources/verl/workers/rollout/sglang_rollout/utils.py", line 55, in broadcast_pyobj
    dist.broadcast(tensor_size, src=src, group=dist_group)
  File "/home/jobuser/.local/lib/python3.10/site-packages/torch/distributed/c10d_logger.py", line 81, in wrapper
    return func(*args, **kwargs)
  File "/home/jobuser/.local/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py", line 2726, in broadcast
    work = group.broadcast([tensor], opts)
torch.distributed.DistBackendError: [4] is setting up NCCL communicator and retrieving ncclUniqueId from [0] via c10d key-value store by key '0', but store->get('0') got error: failed to recv, got 0 bytes
Exception raised from recvBytes at /pytorch/torch/csrc/distributed/c10d/Utils.hpp:671 (most recent call first):
frame #0: c10::Error::Error(c10::SourceLocation, std::string) + 0x96 (0x77e36d3fb1b6 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libc10.so)
frame #1: <unknown function> + 0x6312a88 (0x77b441256a88 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
frame #2: c10d::TCPStore::doWait(c10::ArrayRef<std::string>, std::chrono::duration<long, std::ratio<1l, 1000l> >) + 0x367 (0x77b441253817 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
frame #3: c10d::TCPStore::doGet(std::string const&) + 0x2a (0x77b441253b9a in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
frame #4: c10d::TCPStore::get(std::string const&) + 0x7a (0x77b44125493a in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
frame #5: c10d::PrefixStore::get(std::string const&) + 0x31 (0x77b4412015d1 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
frame #6: c10d::PrefixStore::get(std::string const&) + 0x31 (0x77b4412015d1 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
frame #7: c10d::PrefixStore::get(std::string const&) + 0x31 (0x77b4412015d1 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
frame #8: c10d::PrefixStore::get(std::string const&) + 0x31 (0x77b4412015d1 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
frame #9: c10d::ProcessGroupNCCL::broadcastUniqueNCCLID(ncclUniqueId*, bool, std::string const&, int) + 0xaf (0x77b406f91a9f in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so)
frame #10: c10d::ProcessGroupNCCL::initNCCLComm(std::string const&, c10::Device&, c10d::OpType, int, bool) + 0x1176 (0x77b406f9f666 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so)
frame #11: c10d::ProcessGroupNCCL::broadcast(std::vector<at::Tensor, std::allocator<at::Tensor> >&, c10d::BroadcastOptions const&) + 0x1265 (0x77b406faa175 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cuda.so)
frame #12: <unknown function> + 0x62b13b6 (0x77b4411f53b6 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
frame #13: <unknown function> + 0x62bcd26 (0x77b441200d26 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
frame #14: <unknown function> + 0x58bc27b (0x77b44080027b in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
frame #15: <unknown function> + 0x58b9af9 (0x77b4407fdaf9 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
frame #16: <unknown function> + 0x1aca9f8 (0x77b43ca0e9f8 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
frame #17: <unknown function> + 0x62c339a (0x77b44120739a in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
frame #18: <unknown function> + 0x62d3b30 (0x77b441217b30 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_cpu.so)
frame #19: <unknown function> + 0xe9fac5 (0x77e36e32eac5 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_python.so)
frame #20: <unknown function> + 0x518d37 (0x77e36d9a7d37 in /home/jobuser/.local/lib/python3.10/site-packages/torch/lib/libtorch_python.so)
<omitting python frames>
frame #55: <unknown function> + 0x769d4f (0x77e402733d4f in /home/jobuser/.local/lib/python3.10/site-packages/ray/_raylet.so)
frame #56: <unknown function> + 0x7daaa8 (0x77e4027a4aa8 in /home/jobuser/.local/lib/python3.10/site-packages/ray/_raylet.so)
frame #57: <unknown function> + 0x769d4f (0x77e402733d4f in /home/jobuser/.local/lib/python3.10/site-packages/ray/_raylet.so)
frame #58: <unknown function> + 0x86fc56 (0x77e402839c56 in /home/jobuser/.local/lib/python3.10/site-packages/ray/_raylet.so)
frame #59: std::_Function_handler<ray::Status (ray::rpc::Address const&, ray::rpc::TaskType, std::string, ray::core::RayFunction const&, std::unordered_map<std::string, double, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::pair<std::string const, double> > > const&, std::vector<std::shared_ptr<ray::RayObject>, std::allocator<std::shared_ptr<ray::RayObject> > > const&, std::vector<ray::rpc::ObjectReference, std::allocator<ray::rpc::ObjectReference> > const&, std::string const&, std::string const&, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*, std::vector<std::pair<ray::ObjectID, bool>, std::allocator<std::pair<ray::ObjectID, bool> > >*, std::shared_ptr<ray::LocalMemoryBuffer>&, bool*, std::string*, std::vector<ray::ConcurrencyGroup, std::allocator<ray::ConcurrencyGroup> > const&, std::string, bool, bool, bool, long), ray::Status (*)(ray::rpc::Address const&, ray::rpc::TaskType, std::string, ray::core::RayFunction const&, std::unordered_map<std::string, double, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::pair<std::string const, double> > > const&, std::vector<std::shared_ptr<ray::RayObject>, std::allocator<std::shared_ptr<ray::RayObject> > > const&, std::vector<ray::rpc::ObjectReference, std::allocator<ray::rpc::ObjectReference> > const&, std::string, std::string, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*, std::vector<std::pair<ray::ObjectID, bool>, std::allocator<std::pair<ray::ObjectID, bool> > >*, std::shared_ptr<ray::LocalMemoryBuffer>&, bool*, std::string*, std::vector<ray::ConcurrencyGroup, std::allocator<ray::ConcurrencyGroup> > const&, std::string, bool, bool, bool, long)>::_M_invoke(std::_Any_data const&, ray::rpc::Address const&, ray::rpc::TaskType&&, std::string&&, ray::core::RayFunction const&, std::unordered_map<std::string, double, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::pair<std::string const, double> > > const&, std::vector<std::shared_ptr<ray::RayObject>, std::allocator<std::shared_ptr<ray::RayObject> > > const&, std::vector<ray::rpc::ObjectReference, std::allocator<ray::rpc::ObjectReference> > const&, std::string const&, std::string const&, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*&&, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*&&, std::vector<std::pair<ray::ObjectID, bool>, std::allocator<std::pair<ray::ObjectID, bool> > >*&&, std::shared_ptr<ray::LocalMemoryBuffer>&, bool*&&, std::string*&&, std::vector<ray::ConcurrencyGroup, std::allocator<ray::ConcurrencyGroup> > const&, std::string&&, bool&&, bool&&, bool&&, long&&) + 0x169 (0x77e40273ad39 in /home/jobuser/.local/lib/python3.10/site-packages/ray/_raylet.so)
frame #60: ray::core::CoreWorker::ExecuteTask(ray::TaskSpecification const&, std::optional<std::unordered_map<std::string, std::vector<std::pair<long, double>, std::allocator<std::pair<long, double> > >, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::pair<std::string const, std::vector<std::pair<long, double>, std::allocator<std::pair<long, double> > > > > > >, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*, std::vector<std::pair<ray::ObjectID, bool>, std::allocator<std::pair<ray::ObjectID, bool> > >*, google::protobuf::RepeatedPtrField<ray::rpc::ObjectReferenceCount>*, bool*, std::string*) + 0xdf5 (0x77e40294c245 in /home/jobuser/.local/lib/python3.10/site-packages/ray/_raylet.so)
frame #61: std::_Function_handler<ray::Status (ray::TaskSpecification const&, std::optional<std::unordered_map<std::string, std::vector<std::pair<long, double>, std::allocator<std::pair<long, double> > >, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::pair<std::string const, std::vector<std::pair<long, double>, std::allocator<std::pair<long, double> > > > > > >, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*, std::vector<std::pair<ray::ObjectID, bool>, std::allocator<std::pair<ray::ObjectID, bool> > >*, google::protobuf::RepeatedPtrField<ray::rpc::ObjectReferenceCount>*, bool*, std::string*), std::_Bind<ray::Status (ray::core::CoreWorker::*(ray::core::CoreWorker*, std::_Placeholder<1>, std::_Placeholder<2>, std::_Placeholder<3>, std::_Placeholder<4>, std::_Placeholder<5>, std::_Placeholder<6>, std::_Placeholder<7>, std::_Placeholder<8>))(ray::TaskSpecification const&, std::optional<std::unordered_map<std::string, std::vector<std::pair<long, double>, std::allocator<std::pair<long, double> > >, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::pair<std::string const, std::vector<std::pair<long, double>, std::allocator<std::pair<long, double> > > > > > >, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*, std::vector<std::pair<ray::ObjectID, bool>, std::allocator<std::pair<ray::ObjectID, bool> > >*, google::protobuf::RepeatedPtrField<ray::rpc::ObjectReferenceCount>*, bool*, std::string*)> >::_M_invoke(std::_Any_data const&, ray::TaskSpecification const&, std::optional<std::unordered_map<std::string, std::vector<std::pair<long, double>, std::allocator<std::pair<long, double> > >, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::pair<std::string const, std::vector<std::pair<long, double>, std::allocator<std::pair<long, double> > > > > > >&&, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*&&, std::vector<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> >, std::allocator<std::pair<ray::ObjectID, std::shared_ptr<ray::RayObject> > > >*&&, std::vector<std::pair<ray::ObjectID, bool>, std::allocator<std::pair<ray::ObjectID, bool> > >*&&, google::protobuf::RepeatedPtrField<ray::rpc::ObjectReferenceCount>*&&, bool*&&, std::string*&&) + 0x75 (0x77e4028d6845 in /home/jobuser/.local/lib/python3.10/site-packages/ray/_raylet.so)
frame #62: <unknown function> + 0x9f30ce (0x77e4029bd0ce in /home/jobuser/.local/lib/python3.10/site-packages/ray/_raylet.so)
. This may indicate a possible application crash on rank 0 or a network set up issue.

Set the environment variable HYDRA_FULL_ERROR=1 for a complete stack trace.

Hey all, seems the issue isn’t reproducible anymore more with the latest versions of sgl and verl. You could try and let us know.

+1, it works now on H200 for me

@MrAta @yuleiqin @liulurong @kemuxiaozi000 I found that the workaround for the OOM issue also resolves this illegal memory access error. I’ve opened a PR #1686 . Could you give it a try and let me know if it works on your end?

I tested the above patch, but it did not resolve the issue. After a few training steps, the same error as before reappeared.