diff --git "a/nohup.out" "b/nohup.out"
deleted file mode 100644--- "a/nohup.out"
+++ /dev/null
@@ -1,1129 +0,0 @@
-INFO 10-11 10:19:41 [__init__.py:216] Automatically detected platform cuda.
-(APIServer pid=1027) INFO 10-11 10:19:47 [api_server.py:1839] vLLM API server version 0.11.0
-(APIServer pid=1027) INFO 10-11 10:19:47 [utils.py:233] non-default args: {'model_tag': 'deepseek-ai/DeepSeek-V3.2-Exp', 'port': 9010, 'model': 'deepseek-ai/DeepSeek-V3.2-Exp', 'served_model_name': ['DeepSeek-V3.2-Exp-nvidia-origin'], 'tensor_parallel_size': 8}
-(APIServer pid=1027) You are using a model of type deepseek_v32 to instantiate a model of type deepseek_v3. This is not supported for all configurations of models and can yield errors.
-(APIServer pid=1027) INFO 10-11 10:19:48 [config.py:617] Detected quantization_config.scale_fmt=ue8m0; enabling Hopper UE8M0.
-(APIServer pid=1027) INFO 10-11 10:19:48 [config.py:388] Replacing legacy 'type' key with 'rope_type'
-(APIServer pid=1027) INFO 10-11 10:19:50 [model.py:547] Resolved architecture: DeepseekV32ForCausalLM
-(APIServer pid=1027) `torch_dtype` is deprecated! Use `dtype` instead!
-(APIServer pid=1027) INFO 10-11 10:19:50 [model.py:1510] Using max model len 163840
-(APIServer pid=1027) INFO 10-11 10:19:50 [scheduler.py:205] Chunked prefill is enabled with max_num_batched_tokens=8192.
-(APIServer pid=1027) INFO 10-11 10:19:50 [config.py:422] Using custom fp8 kv-cache format for DeepSeekV3.2
-(APIServer pid=1027) INFO 10-11 10:19:50 [cuda.py:166] Forcing kv cache block size to 64 for FlashMLA backend.
-INFO 10-11 10:19:55 [__init__.py:216] Automatically detected platform cuda.
-(EngineCore_DP0 pid=1184) INFO 10-11 10:19:59 [core.py:644] Waiting for init message from front-end.
-(EngineCore_DP0 pid=1184) INFO 10-11 10:19:59 [core.py:77] Initializing a V1 LLM engine (v0.11.0) with config: model='deepseek-ai/DeepSeek-V3.2-Exp', speculative_config=None, tokenizer='deepseek-ai/DeepSeek-V3.2-Exp', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=163840, download_dir=None, load_format=auto, tensor_parallel_size=8, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=False, quantization=fp8, enforce_eager=False, kv_cache_dtype=fp8_ds_mla, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser=''), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None), seed=0, served_model_name=DeepSeek-V3.2-Exp-nvidia-origin, enable_prefix_caching=True, chunked_prefill_enabled=True, pooler_config=None, compilation_config={"level":3,"debug_dump_path":"","cache_dir":"","backend":"","custom_ops":[],"splitting_ops":["vllm.unified_attention","vllm.unified_attention_with_output","vllm.mamba_mixer2","vllm.mamba_mixer","vllm.short_conv","vllm.linear_attention","vllm.plamo2_mamba_mixer","vllm.gdn_attention","vllm.sparse_attn_indexer"],"use_inductor":true,"compile_sizes":[],"inductor_compile_config":{"enable_auto_functionalized_v2":false},"inductor_passes":{},"cudagraph_mode":[2,1],"use_cudagraph":true,"cudagraph_num_of_warmups":1,"cudagraph_capture_sizes":[512,504,496,488,480,472,464,456,448,440,432,424,416,408,400,392,384,376,368,360,352,344,336,328,320,312,304,296,288,280,272,264,256,248,240,232,224,216,208,200,192,184,176,168,160,152,144,136,128,120,112,104,96,88,80,72,64,56,48,40,32,24,16,8,4,2,1],"cudagraph_copy_inputs":false,"full_cuda_graph":false,"use_inductor_graph_partition":false,"pass_config":{},"max_capture_size":512,"local_cache_dir":null}
-(EngineCore_DP0 pid=1184) WARNING 10-11 10:19:59 [multiproc_executor.py:720] Reducing Torch parallelism from 96 threads to 1 to avoid unnecessary CPU contention. Set OMP_NUM_THREADS in the external environment to tune this value as needed.
-(EngineCore_DP0 pid=1184) INFO 10-11 10:19:59 [shm_broadcast.py:289] vLLM message queue communication handle: Handle(local_reader_ranks=[0, 1, 2, 3, 4, 5, 6, 7], buffer_handle=(8, 16777216, 10, 'psm_2554f410'), local_subscribe_addr='ipc:///tmp/6ead4ffd-2655-492a-9f58-b6a8c705d1c1', remote_subscribe_addr=None, remote_addr_ipv6=False)
-INFO 10-11 10:20:02 [__init__.py:216] Automatically detected platform cuda.
-INFO 10-11 10:20:02 [__init__.py:216] Automatically detected platform cuda.
-INFO 10-11 10:20:02 [__init__.py:216] Automatically detected platform cuda.
-INFO 10-11 10:20:03 [__init__.py:216] Automatically detected platform cuda.
-INFO 10-11 10:20:03 [__init__.py:216] Automatically detected platform cuda.
-INFO 10-11 10:20:03 [__init__.py:216] Automatically detected platform cuda.
-INFO 10-11 10:20:03 [__init__.py:216] Automatically detected platform cuda.
-INFO 10-11 10:20:03 [__init__.py:216] Automatically detected platform cuda.
-INFO 10-11 10:20:10 [shm_broadcast.py:289] vLLM message queue communication handle: Handle(local_reader_ranks=[0], buffer_handle=(1, 10485760, 10, 'psm_413b7f47'), local_subscribe_addr='ipc:///tmp/45bf2248-9fb7-4bee-a543-03cd54a0404b', remote_subscribe_addr=None, remote_addr_ipv6=False)
-INFO 10-11 10:20:10 [shm_broadcast.py:289] vLLM message queue communication handle: Handle(local_reader_ranks=[0], buffer_handle=(1, 10485760, 10, 'psm_fc6389ed'), local_subscribe_addr='ipc:///tmp/a29024a6-4073-4cf1-b005-a541cbc948bc', remote_subscribe_addr=None, remote_addr_ipv6=False)
-INFO 10-11 10:20:10 [shm_broadcast.py:289] vLLM message queue communication handle: Handle(local_reader_ranks=[0], buffer_handle=(1, 10485760, 10, 'psm_4b8c4fa2'), local_subscribe_addr='ipc:///tmp/5384f730-9eeb-4200-951d-19b9a470d41a', remote_subscribe_addr=None, remote_addr_ipv6=False)
-INFO 10-11 10:20:10 [shm_broadcast.py:289] vLLM message queue communication handle: Handle(local_reader_ranks=[0], buffer_handle=(1, 10485760, 10, 'psm_ef4dc190'), local_subscribe_addr='ipc:///tmp/58a24112-070d-4f72-8f4d-b11da837d69d', remote_subscribe_addr=None, remote_addr_ipv6=False)
-INFO 10-11 10:20:10 [shm_broadcast.py:289] vLLM message queue communication handle: Handle(local_reader_ranks=[0], buffer_handle=(1, 10485760, 10, 'psm_162c87a1'), local_subscribe_addr='ipc:///tmp/2a006cf8-44c6-402b-8ff1-772ad7103f3a', remote_subscribe_addr=None, remote_addr_ipv6=False)
-INFO 10-11 10:20:10 [shm_broadcast.py:289] vLLM message queue communication handle: Handle(local_reader_ranks=[0], buffer_handle=(1, 10485760, 10, 'psm_664c3ab0'), local_subscribe_addr='ipc:///tmp/cabd5fb0-e8e7-4713-accc-1e6c1187ff20', remote_subscribe_addr=None, remote_addr_ipv6=False)
-INFO 10-11 10:20:10 [shm_broadcast.py:289] vLLM message queue communication handle: Handle(local_reader_ranks=[0], buffer_handle=(1, 10485760, 10, 'psm_0acfcc60'), local_subscribe_addr='ipc:///tmp/121f3146-c105-4431-a431-a211aa33c331', remote_subscribe_addr=None, remote_addr_ipv6=False)
-INFO 10-11 10:20:10 [shm_broadcast.py:289] vLLM message queue communication handle: Handle(local_reader_ranks=[0], buffer_handle=(1, 10485760, 10, 'psm_f9b91bd9'), local_subscribe_addr='ipc:///tmp/168ec4f0-4063-4491-b3b8-c1e0435e1aaa', remote_subscribe_addr=None, remote_addr_ipv6=False)
-[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
-[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
-[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
-[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
-[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
-[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
-[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
-[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
-[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
-[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
-[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
-[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
-[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
-[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
-[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
-[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
-INFO 10-11 10:20:12 [__init__.py:1384] Found nccl from library libnccl.so.2
-INFO 10-11 10:20:12 [pynccl.py:103] vLLM is using nccl==2.27.3
-INFO 10-11 10:20:13 [__init__.py:1384] Found nccl from library libnccl.so.2
-INFO 10-11 10:20:13 [pynccl.py:103] vLLM is using nccl==2.27.3
-INFO 10-11 10:20:13 [__init__.py:1384] Found nccl from library libnccl.so.2
-INFO 10-11 10:20:13 [pynccl.py:103] vLLM is using nccl==2.27.3
-INFO 10-11 10:20:13 [__init__.py:1384] Found nccl from library libnccl.so.2
-INFO 10-11 10:20:13 [pynccl.py:103] vLLM is using nccl==2.27.3
-INFO 10-11 10:20:13 [__init__.py:1384] Found nccl from library libnccl.so.2
-INFO 10-11 10:20:13 [pynccl.py:103] vLLM is using nccl==2.27.3
-INFO 10-11 10:20:13 [__init__.py:1384] Found nccl from library libnccl.so.2
-INFO 10-11 10:20:13 [pynccl.py:103] vLLM is using nccl==2.27.3
-INFO 10-11 10:20:13 [__init__.py:1384] Found nccl from library libnccl.so.2
-INFO 10-11 10:20:13 [pynccl.py:103] vLLM is using nccl==2.27.3
-INFO 10-11 10:20:13 [__init__.py:1384] Found nccl from library libnccl.so.2
-INFO 10-11 10:20:13 [pynccl.py:103] vLLM is using nccl==2.27.3
-INFO 10-11 10:20:15 [custom_all_reduce.py:35] Skipping P2P check and trusting the driver's P2P report.
-INFO 10-11 10:20:15 [custom_all_reduce.py:35] Skipping P2P check and trusting the driver's P2P report.
-INFO 10-11 10:20:15 [custom_all_reduce.py:35] Skipping P2P check and trusting the driver's P2P report.
-INFO 10-11 10:20:15 [custom_all_reduce.py:35] Skipping P2P check and trusting the driver's P2P report.
-INFO 10-11 10:20:15 [custom_all_reduce.py:35] Skipping P2P check and trusting the driver's P2P report.
-INFO 10-11 10:20:15 [custom_all_reduce.py:35] Skipping P2P check and trusting the driver's P2P report.
-INFO 10-11 10:20:15 [custom_all_reduce.py:35] Skipping P2P check and trusting the driver's P2P report.
-INFO 10-11 10:20:15 [custom_all_reduce.py:35] Skipping P2P check and trusting the driver's P2P report.
-INFO 10-11 10:20:16 [shm_broadcast.py:289] vLLM message queue communication handle: Handle(local_reader_ranks=[1, 2, 3, 4, 5, 6, 7], buffer_handle=(7, 4194304, 6, 'psm_b9a02daf'), local_subscribe_addr='ipc:///tmp/099ef8b9-587c-4f71-8210-53da677a0d3f', remote_subscribe_addr=None, remote_addr_ipv6=False)
-[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
-[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
-[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
-[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
-[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
-[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
-[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
-[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
-[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
-[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
-[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
-[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
-[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
-[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
-[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
-[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
-[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
-[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
-[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
-[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
-[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
-[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
-[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
-[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
-[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
-[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
-[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
-[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
-[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
-[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
-[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
-[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
-INFO 10-11 10:20:16 [__init__.py:1384] Found nccl from library libnccl.so.2
-INFO 10-11 10:20:16 [pynccl.py:103] vLLM is using nccl==2.27.3
-INFO 10-11 10:20:16 [__init__.py:1384] Found nccl from library libnccl.so.2
-INFO 10-11 10:20:16 [__init__.py:1384] Found nccl from library libnccl.so.2
-INFO 10-11 10:20:16 [pynccl.py:103] vLLM is using nccl==2.27.3
-INFO 10-11 10:20:16 [__init__.py:1384] Found nccl from library libnccl.so.2
-INFO 10-11 10:20:16 [__init__.py:1384] Found nccl from library libnccl.so.2
-INFO 10-11 10:20:16 [__init__.py:1384] Found nccl from library libnccl.so.2
-INFO 10-11 10:20:16 [pynccl.py:103] vLLM is using nccl==2.27.3
-INFO 10-11 10:20:16 [pynccl.py:103] vLLM is using nccl==2.27.3
-INFO 10-11 10:20:16 [pynccl.py:103] vLLM is using nccl==2.27.3
-INFO 10-11 10:20:16 [pynccl.py:103] vLLM is using nccl==2.27.3
-INFO 10-11 10:20:16 [__init__.py:1384] Found nccl from library libnccl.so.2
-INFO 10-11 10:20:16 [pynccl.py:103] vLLM is using nccl==2.27.3
-INFO 10-11 10:20:16 [__init__.py:1384] Found nccl from library libnccl.so.2
-INFO 10-11 10:20:16 [pynccl.py:103] vLLM is using nccl==2.27.3
-INFO 10-11 10:20:17 [parallel_state.py:1208] rank 1 in world size 8 is assigned as DP rank 0, PP rank 0, TP rank 1, EP rank 1
-INFO 10-11 10:20:17 [parallel_state.py:1208] rank 6 in world size 8 is assigned as DP rank 0, PP rank 0, TP rank 6, EP rank 6
-INFO 10-11 10:20:17 [parallel_state.py:1208] rank 2 in world size 8 is assigned as DP rank 0, PP rank 0, TP rank 2, EP rank 2
-INFO 10-11 10:20:17 [parallel_state.py:1208] rank 4 in world size 8 is assigned as DP rank 0, PP rank 0, TP rank 4, EP rank 4
-INFO 10-11 10:20:17 [parallel_state.py:1208] rank 3 in world size 8 is assigned as DP rank 0, PP rank 0, TP rank 3, EP rank 3
-INFO 10-11 10:20:17 [parallel_state.py:1208] rank 5 in world size 8 is assigned as DP rank 0, PP rank 0, TP rank 5, EP rank 5
-INFO 10-11 10:20:17 [parallel_state.py:1208] rank 0 in world size 8 is assigned as DP rank 0, PP rank 0, TP rank 0, EP rank 0
-INFO 10-11 10:20:17 [parallel_state.py:1208] rank 7 in world size 8 is assigned as DP rank 0, PP rank 0, TP rank 7, EP rank 7
-WARNING 10-11 10:20:17 [topk_topp_sampler.py:66] FlashInfer is not available. Falling back to the PyTorch-native implementation of top-p & top-k sampling. For the best performance, please install FlashInfer.
-WARNING 10-11 10:20:17 [topk_topp_sampler.py:66] FlashInfer is not available. Falling back to the PyTorch-native implementation of top-p & top-k sampling. For the best performance, please install FlashInfer.
-WARNING 10-11 10:20:17 [topk_topp_sampler.py:66] FlashInfer is not available. Falling back to the PyTorch-native implementation of top-p & top-k sampling. For the best performance, please install FlashInfer.
-WARNING 10-11 10:20:17 [topk_topp_sampler.py:66] FlashInfer is not available. Falling back to the PyTorch-native implementation of top-p & top-k sampling. For the best performance, please install FlashInfer.
-WARNING 10-11 10:20:17 [topk_topp_sampler.py:66] FlashInfer is not available. Falling back to the PyTorch-native implementation of top-p & top-k sampling. For the best performance, please install FlashInfer.
-WARNING 10-11 10:20:17 [topk_topp_sampler.py:66] FlashInfer is not available. Falling back to the PyTorch-native implementation of top-p & top-k sampling. For the best performance, please install FlashInfer.
-WARNING 10-11 10:20:17 [topk_topp_sampler.py:66] FlashInfer is not available. Falling back to the PyTorch-native implementation of top-p & top-k sampling. For the best performance, please install FlashInfer.
-WARNING 10-11 10:20:17 [topk_topp_sampler.py:66] FlashInfer is not available. Falling back to the PyTorch-native implementation of top-p & top-k sampling. For the best performance, please install FlashInfer.
-(Worker_TP1 pid=1321) INFO 10-11 10:20:18 [gpu_model_runner.py:2602] Starting to load model deepseek-ai/DeepSeek-V3.2-Exp...
-(Worker_TP3 pid=1323) INFO 10-11 10:20:18 [gpu_model_runner.py:2602] Starting to load model deepseek-ai/DeepSeek-V3.2-Exp...
-(Worker_TP7 pid=1327) INFO 10-11 10:20:18 [gpu_model_runner.py:2602] Starting to load model deepseek-ai/DeepSeek-V3.2-Exp...
-(Worker_TP2 pid=1322) INFO 10-11 10:20:18 [gpu_model_runner.py:2602] Starting to load model deepseek-ai/DeepSeek-V3.2-Exp...
-(Worker_TP0 pid=1320) INFO 10-11 10:20:18 [gpu_model_runner.py:2602] Starting to load model deepseek-ai/DeepSeek-V3.2-Exp...
-(Worker_TP5 pid=1325) INFO 10-11 10:20:18 [gpu_model_runner.py:2602] Starting to load model deepseek-ai/DeepSeek-V3.2-Exp...
-(Worker_TP4 pid=1324) INFO 10-11 10:20:18 [gpu_model_runner.py:2602] Starting to load model deepseek-ai/DeepSeek-V3.2-Exp...
-(Worker_TP6 pid=1326) INFO 10-11 10:20:18 [gpu_model_runner.py:2602] Starting to load model deepseek-ai/DeepSeek-V3.2-Exp...
-(Worker_TP3 pid=1323) INFO 10-11 10:20:18 [gpu_model_runner.py:2634] Loading model from scratch...
-(Worker_TP7 pid=1327) INFO 10-11 10:20:18 [gpu_model_runner.py:2634] Loading model from scratch...
-(Worker_TP1 pid=1321) INFO 10-11 10:20:18 [gpu_model_runner.py:2634] Loading model from scratch...
-(Worker_TP0 pid=1320) INFO 10-11 10:20:18 [gpu_model_runner.py:2634] Loading model from scratch...
-(Worker_TP6 pid=1326) INFO 10-11 10:20:18 [gpu_model_runner.py:2634] Loading model from scratch...
-(Worker_TP2 pid=1322) INFO 10-11 10:20:18 [gpu_model_runner.py:2634] Loading model from scratch...
-(Worker_TP3 pid=1323) INFO 10-11 10:20:18 [cuda.py:253] Using Sparse MLA backend on V1 engine.
-(Worker_TP4 pid=1324) INFO 10-11 10:20:18 [gpu_model_runner.py:2634] Loading model from scratch...
-(Worker_TP5 pid=1325) INFO 10-11 10:20:18 [gpu_model_runner.py:2634] Loading model from scratch...
-(Worker_TP7 pid=1327) INFO 10-11 10:20:18 [cuda.py:253] Using Sparse MLA backend on V1 engine.
-(Worker_TP3 pid=1323) INFO 10-11 10:20:18 [fp8.py:462] Using DeepGemm kernels for Fp8MoEMethod.
-(Worker_TP3 pid=1323) WARNING 10-11 10:20:18 [fp8.py:480] CutlassBlockScaledGroupedGemm not supported on the current platform.
-(Worker_TP1 pid=1321) INFO 10-11 10:20:18 [cuda.py:253] Using Sparse MLA backend on V1 engine.
-(Worker_TP7 pid=1327) INFO 10-11 10:20:18 [fp8.py:462] Using DeepGemm kernels for Fp8MoEMethod.
-(Worker_TP7 pid=1327) WARNING 10-11 10:20:18 [fp8.py:480] CutlassBlockScaledGroupedGemm not supported on the current platform.
-(Worker_TP1 pid=1321) INFO 10-11 10:20:18 [fp8.py:462] Using DeepGemm kernels for Fp8MoEMethod.
-(Worker_TP1 pid=1321) WARNING 10-11 10:20:18 [fp8.py:480] CutlassBlockScaledGroupedGemm not supported on the current platform.
-(Worker_TP0 pid=1320) INFO 10-11 10:20:18 [cuda.py:253] Using Sparse MLA backend on V1 engine.
-(Worker_TP6 pid=1326) INFO 10-11 10:20:18 [cuda.py:253] Using Sparse MLA backend on V1 engine.
-(Worker_TP2 pid=1322) INFO 10-11 10:20:18 [cuda.py:253] Using Sparse MLA backend on V1 engine.
-(Worker_TP0 pid=1320) INFO 10-11 10:20:18 [fp8.py:462] Using DeepGemm kernels for Fp8MoEMethod.
-(Worker_TP0 pid=1320) WARNING 10-11 10:20:18 [fp8.py:480] CutlassBlockScaledGroupedGemm not supported on the current platform.
-(Worker_TP4 pid=1324) INFO 10-11 10:20:18 [cuda.py:253] Using Sparse MLA backend on V1 engine.
-(Worker_TP5 pid=1325) INFO 10-11 10:20:18 [cuda.py:253] Using Sparse MLA backend on V1 engine.
-(Worker_TP6 pid=1326) INFO 10-11 10:20:18 [fp8.py:462] Using DeepGemm kernels for Fp8MoEMethod.
-(Worker_TP6 pid=1326) WARNING 10-11 10:20:18 [fp8.py:480] CutlassBlockScaledGroupedGemm not supported on the current platform.
-(Worker_TP2 pid=1322) INFO 10-11 10:20:18 [fp8.py:462] Using DeepGemm kernels for Fp8MoEMethod.
-(Worker_TP2 pid=1322) WARNING 10-11 10:20:18 [fp8.py:480] CutlassBlockScaledGroupedGemm not supported on the current platform.
-(Worker_TP4 pid=1324) INFO 10-11 10:20:18 [fp8.py:462] Using DeepGemm kernels for Fp8MoEMethod.
-(Worker_TP4 pid=1324) WARNING 10-11 10:20:18 [fp8.py:480] CutlassBlockScaledGroupedGemm not supported on the current platform.
-(Worker_TP5 pid=1325) INFO 10-11 10:20:18 [fp8.py:462] Using DeepGemm kernels for Fp8MoEMethod.
-(Worker_TP5 pid=1325) WARNING 10-11 10:20:18 [fp8.py:480] CutlassBlockScaledGroupedGemm not supported on the current platform.
-(Worker_TP7 pid=1327) INFO 10-11 10:20:20 [weight_utils.py:392] Using model weights format ['*.safetensors']
-(Worker_TP3 pid=1323) INFO 10-11 10:20:20 [weight_utils.py:392] Using model weights format ['*.safetensors']
-(Worker_TP2 pid=1322) INFO 10-11 10:20:20 [weight_utils.py:392] Using model weights format ['*.safetensors']
-(Worker_TP1 pid=1321) INFO 10-11 10:20:20 [weight_utils.py:392] Using model weights format ['*.safetensors']
-(Worker_TP4 pid=1324) INFO 10-11 10:20:20 [weight_utils.py:392] Using model weights format ['*.safetensors']
-(Worker_TP0 pid=1320) INFO 10-11 10:20:20 [weight_utils.py:392] Using model weights format ['*.safetensors']
-(Worker_TP6 pid=1326) INFO 10-11 10:20:20 [weight_utils.py:392] Using model weights format ['*.safetensors']
-(Worker_TP5 pid=1325) INFO 10-11 10:20:20 [weight_utils.py:392] Using model weights format ['*.safetensors']
-(Worker_TP7 pid=1327) /usr/local/lib/python3.12/dist-packages/huggingface_hub/file_download.py:801: UserWarning: Not enough free disk space to download the file. The expected file size is: 4302.12 MB. The target location /root/.cache/huggingface/hub/models--deepseek-ai--DeepSeek-V3.2-Exp/blobs only has 0.00 MB free disk space.
-(Worker_TP7 pid=1327)   warnings.warn(
-(Worker_TP7 pid=1327) /usr/local/lib/python3.12/dist-packages/huggingface_hub/file_download.py:801: UserWarning: Not enough free disk space to download the file. The expected file size is: 4302.38 MB. The target location /root/.cache/huggingface/hub/models--deepseek-ai--DeepSeek-V3.2-Exp/blobs only has 0.00 MB free disk space.
-(Worker_TP7 pid=1327)   warnings.warn(
-(Worker_TP7 pid=1327) /usr/local/lib/python3.12/dist-packages/huggingface_hub/file_download.py:801: UserWarning: Not enough free disk space to download the file. The expected file size is: 5285.72 MB. The target location /root/.cache/huggingface/hub/models--deepseek-ai--DeepSeek-V3.2-Exp/blobs only has 0.00 MB free disk space.
-(Worker_TP7 pid=1327)   warnings.warn(
-(Worker_TP7 pid=1327) /usr/local/lib/python3.12/dist-packages/huggingface_hub/file_download.py:801: UserWarning: Not enough free disk space to download the file. The expected file size is: 4305.82 MB. The target location /root/.cache/huggingface/hub/models--deepseek-ai--DeepSeek-V3.2-Exp/blobs only has 0.00 MB free disk space.
-(Worker_TP7 pid=1327)   warnings.warn(
-(Worker_TP7 pid=1327) /usr/local/lib/python3.12/dist-packages/huggingface_hub/file_download.py:801: UserWarning: Not enough free disk space to download the file. The expected file size is: 6643.59 MB. The target location /root/.cache/huggingface/hub/models--deepseek-ai--DeepSeek-V3.2-Exp/blobs only has 0.00 MB free disk space.
-(Worker_TP7 pid=1327)   warnings.warn(
-(Worker_TP7 pid=1327) INFO 10-11 12:33:54 [weight_utils.py:413] Time spent downloading weights for deepseek-ai/DeepSeek-V3.2-Exp: 8014.215436 seconds
-(Worker_TP7 pid=1327) /usr/local/lib/python3.12/dist-packages/huggingface_hub/file_download.py:801: UserWarning: Not enough free disk space to download the file. The expected file size is: 8.94 MB. The target location /root/.cache/huggingface/hub/models--deepseek-ai--DeepSeek-V3.2-Exp/blobs only has 0.00 MB free disk space.
-(Worker_TP7 pid=1327)   warnings.warn(
-(Worker_TP6 pid=1326) INFO 10-11 12:33:57 [weight_utils.py:413] Time spent downloading weights for deepseek-ai/DeepSeek-V3.2-Exp: 0.736509 seconds
-(Worker_TP5 pid=1325) INFO 10-11 12:33:58 [weight_utils.py:413] Time spent downloading weights for deepseek-ai/DeepSeek-V3.2-Exp: 0.732342 seconds
-(Worker_TP4 pid=1324) INFO 10-11 12:33:59 [weight_utils.py:413] Time spent downloading weights for deepseek-ai/DeepSeek-V3.2-Exp: 0.729255 seconds
-(Worker_TP1 pid=1321) INFO 10-11 12:34:01 [weight_utils.py:413] Time spent downloading weights for deepseek-ai/DeepSeek-V3.2-Exp: 0.932787 seconds
-(Worker_TP2 pid=1322) INFO 10-11 12:34:02 [weight_utils.py:413] Time spent downloading weights for deepseek-ai/DeepSeek-V3.2-Exp: 0.729226 seconds
-(Worker_TP0 pid=1320) INFO 10-11 12:34:04 [weight_utils.py:413] Time spent downloading weights for deepseek-ai/DeepSeek-V3.2-Exp: 1.141510 seconds
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:   0% Completed | 0/163 [00:00<?, ?it/s]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:   1% Completed | 1/163 [00:00<00:47,  3.44it/s]
-(Worker_TP3 pid=1323) INFO 10-11 12:34:05 [weight_utils.py:413] Time spent downloading weights for deepseek-ai/DeepSeek-V3.2-Exp: 0.841742 seconds
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:   1% Completed | 2/163 [00:00<01:01,  2.64it/s]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:   2% Completed | 3/163 [00:02<02:07,  1.25it/s]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:   2% Completed | 4/163 [00:02<01:39,  1.60it/s]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:   3% Completed | 5/163 [00:04<03:20,  1.27s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:   4% Completed | 6/163 [00:07<04:41,  1.79s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:   4% Completed | 7/163 [00:08<03:32,  1.36s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:   5% Completed | 8/163 [00:09<03:13,  1.25s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:   6% Completed | 9/163 [00:09<02:46,  1.08s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:   6% Completed | 10/163 [00:10<02:18,  1.10it/s]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:   7% Completed | 11/163 [00:12<03:16,  1.29s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:   7% Completed | 12/163 [00:17<06:08,  2.44s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:   8% Completed | 13/163 [00:18<04:48,  1.92s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:   9% Completed | 14/163 [00:23<07:01,  2.83s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:   9% Completed | 15/163 [00:25<06:21,  2.58s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  10% Completed | 16/163 [00:25<04:51,  1.99s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  10% Completed | 17/163 [00:26<03:45,  1.54s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  11% Completed | 18/163 [00:26<03:03,  1.26s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  12% Completed | 19/163 [00:28<02:58,  1.24s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  12% Completed | 20/163 [00:28<02:19,  1.02it/s]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  13% Completed | 21/163 [00:28<01:52,  1.27it/s]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  13% Completed | 22/163 [00:29<01:31,  1.53it/s]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  14% Completed | 23/163 [00:29<01:17,  1.80it/s]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  15% Completed | 24/163 [00:30<01:23,  1.67it/s]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  15% Completed | 25/163 [00:32<02:35,  1.12s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  16% Completed | 26/163 [00:37<05:08,  2.25s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  17% Completed | 27/163 [00:42<06:46,  2.99s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  17% Completed | 28/163 [00:46<07:49,  3.47s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  18% Completed | 29/163 [00:52<09:05,  4.07s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  18% Completed | 30/163 [00:57<10:02,  4.53s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  19% Completed | 31/163 [00:59<07:48,  3.55s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  20% Completed | 32/163 [01:01<07:10,  3.29s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  20% Completed | 33/163 [01:03<06:03,  2.80s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  21% Completed | 34/163 [01:08<07:39,  3.56s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  21% Completed | 35/163 [01:09<05:34,  2.61s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  22% Completed | 36/163 [01:10<04:34,  2.16s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  23% Completed | 37/163 [01:17<08:03,  3.83s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  23% Completed | 38/163 [01:26<10:54,  5.24s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  24% Completed | 39/163 [01:29<09:21,  4.53s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  25% Completed | 40/163 [01:29<06:50,  3.34s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  25% Completed | 41/163 [01:30<05:10,  2.54s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  26% Completed | 42/163 [01:33<05:21,  2.66s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  26% Completed | 43/163 [01:44<10:19,  5.16s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  27% Completed | 44/163 [01:47<08:44,  4.41s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  28% Completed | 45/163 [01:48<06:32,  3.33s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  28% Completed | 46/163 [01:50<06:07,  3.14s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  29% Completed | 47/163 [01:51<04:35,  2.38s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  29% Completed | 48/163 [01:51<03:26,  1.80s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  30% Completed | 49/163 [01:52<02:38,  1.39s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  31% Completed | 50/163 [01:54<03:12,  1.70s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  31% Completed | 51/163 [01:57<04:05,  2.19s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  32% Completed | 52/163 [02:00<04:24,  2.38s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  33% Completed | 53/163 [02:04<05:12,  2.84s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  33% Completed | 54/163 [02:08<05:29,  3.02s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  34% Completed | 55/163 [02:15<07:48,  4.33s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  34% Completed | 56/163 [02:15<05:36,  3.15s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  35% Completed | 57/163 [02:18<05:15,  2.97s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  36% Completed | 58/163 [02:21<04:58,  2.84s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  36% Completed | 59/163 [02:21<03:38,  2.10s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  37% Completed | 60/163 [02:23<03:45,  2.19s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  37% Completed | 61/163 [02:26<03:58,  2.34s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  38% Completed | 62/163 [02:32<05:56,  3.53s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  39% Completed | 63/163 [02:40<07:45,  4.66s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  39% Completed | 64/163 [02:42<06:44,  4.09s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  40% Completed | 65/163 [02:47<07:11,  4.41s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  40% Completed | 66/163 [02:55<08:37,  5.34s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  41% Completed | 67/163 [02:56<06:40,  4.17s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  42% Completed | 68/163 [03:03<07:55,  5.01s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  42% Completed | 69/163 [03:11<09:09,  5.84s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  43% Completed | 70/163 [03:17<08:53,  5.74s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  44% Completed | 71/163 [03:17<06:18,  4.12s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  44% Completed | 72/163 [03:24<07:30,  4.95s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  45% Completed | 74/163 [03:27<05:00,  3.37s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  46% Completed | 75/163 [03:31<05:12,  3.55s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  47% Completed | 76/163 [03:37<06:05,  4.20s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  47% Completed | 77/163 [03:39<04:56,  3.45s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  48% Completed | 78/163 [03:41<04:20,  3.06s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  48% Completed | 79/163 [03:43<04:09,  2.97s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  49% Completed | 80/163 [03:48<04:48,  3.48s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  50% Completed | 81/163 [03:48<03:30,  2.57s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  50% Completed | 82/163 [03:54<04:31,  3.35s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  51% Completed | 83/163 [03:56<04:15,  3.20s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  52% Completed | 84/163 [04:01<04:45,  3.62s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  52% Completed | 85/163 [04:03<03:59,  3.07s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  53% Completed | 86/163 [04:08<04:34,  3.57s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  53% Completed | 87/163 [04:10<04:00,  3.16s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  54% Completed | 88/163 [04:10<02:57,  2.37s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  55% Completed | 89/163 [04:11<02:18,  1.87s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  55% Completed | 90/163 [04:15<03:06,  2.55s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  56% Completed | 91/163 [04:20<03:46,  3.15s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  56% Completed | 92/163 [04:20<02:51,  2.42s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  57% Completed | 93/163 [04:21<02:13,  1.90s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  58% Completed | 94/163 [04:22<01:42,  1.48s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  58% Completed | 95/163 [04:24<01:57,  1.72s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  59% Completed | 96/163 [04:24<01:28,  1.31s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  60% Completed | 97/163 [04:25<01:07,  1.02s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  60% Completed | 98/163 [04:27<01:26,  1.34s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  61% Completed | 99/163 [04:29<01:51,  1.73s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  61% Completed | 100/163 [04:33<02:19,  2.21s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  62% Completed | 101/163 [04:38<03:14,  3.14s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  63% Completed | 102/163 [04:38<02:20,  2.31s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  63% Completed | 103/163 [04:39<01:41,  1.70s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  64% Completed | 104/163 [04:43<02:29,  2.54s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  64% Completed | 105/163 [04:45<02:16,  2.35s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  65% Completed | 106/163 [04:50<02:54,  3.06s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  66% Completed | 107/163 [04:50<02:11,  2.34s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  66% Completed | 108/163 [04:51<01:38,  1.78s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  67% Completed | 110/163 [04:53<01:13,  1.39s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  68% Completed | 111/163 [04:57<01:50,  2.12s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  69% Completed | 112/163 [05:02<02:21,  2.77s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  69% Completed | 113/163 [05:06<02:45,  3.30s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  70% Completed | 114/163 [05:07<02:01,  2.47s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  71% Completed | 115/163 [05:11<02:28,  3.09s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  71% Completed | 116/163 [05:16<02:46,  3.55s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  72% Completed | 117/163 [05:18<02:17,  2.99s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  72% Completed | 118/163 [05:20<02:04,  2.78s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  73% Completed | 119/163 [05:24<02:24,  3.28s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  74% Completed | 120/163 [05:25<01:48,  2.53s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  74% Completed | 121/163 [05:26<01:23,  1.99s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  75% Completed | 122/163 [05:30<01:45,  2.59s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  75% Completed | 123/163 [05:35<02:08,  3.21s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  76% Completed | 124/163 [05:37<02:01,  3.10s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  77% Completed | 125/163 [05:38<01:31,  2.40s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  77% Completed | 126/163 [05:40<01:22,  2.24s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  78% Completed | 127/163 [05:45<01:46,  2.95s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  79% Completed | 128/163 [05:46<01:26,  2.46s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  79% Completed | 129/163 [05:50<01:39,  2.92s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  80% Completed | 130/163 [05:53<01:40,  3.04s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  80% Completed | 131/163 [05:58<01:50,  3.47s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  81% Completed | 132/163 [06:00<01:34,  3.05s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  82% Completed | 133/163 [06:05<01:46,  3.54s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  82% Completed | 134/163 [06:09<01:52,  3.89s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  83% Completed | 135/163 [06:14<01:54,  4.10s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  83% Completed | 136/163 [06:16<01:37,  3.60s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  84% Completed | 137/163 [06:21<01:42,  3.93s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  85% Completed | 138/163 [06:26<01:44,  4.16s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  85% Completed | 139/163 [06:30<01:42,  4.25s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  86% Completed | 140/163 [06:31<01:17,  3.38s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  87% Completed | 141/163 [06:34<01:05,  2.97s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  87% Completed | 142/163 [06:34<00:47,  2.24s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  88% Completed | 143/163 [06:38<00:56,  2.82s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  88% Completed | 144/163 [06:42<01:01,  3.26s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  89% Completed | 145/163 [06:46<01:01,  3.41s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  90% Completed | 146/163 [06:48<00:48,  2.85s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  90% Completed | 147/163 [06:52<00:52,  3.26s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  91% Completed | 148/163 [06:53<00:39,  2.62s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  91% Completed | 149/163 [06:56<00:37,  2.70s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  92% Completed | 150/163 [06:56<00:26,  2.00s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  93% Completed | 151/163 [06:58<00:21,  1.83s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  93% Completed | 152/163 [06:58<00:15,  1.38s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  94% Completed | 153/163 [07:03<00:23,  2.37s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  94% Completed | 154/163 [07:07<00:27,  3.04s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  95% Completed | 155/163 [07:12<00:28,  3.57s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  96% Completed | 156/163 [07:17<00:27,  3.91s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  96% Completed | 157/163 [07:17<00:17,  2.85s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  97% Completed | 158/163 [07:22<00:17,  3.41s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  98% Completed | 159/163 [07:26<00:14,  3.70s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  98% Completed | 160/163 [07:28<00:09,  3.20s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  99% Completed | 161/163 [07:33<00:07,  3.63s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards:  99% Completed | 162/163 [07:38<00:03,  3.94s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards: 100% Completed | 163/163 [07:40<00:00,  3.31s/it]
-(Worker_TP0 pid=1320) 
Loading safetensors checkpoint shards: 100% Completed | 163/163 [07:40<00:00,  2.82s/it]
-(Worker_TP0 pid=1320) 
-(Worker_TP5 pid=1325) INFO 10-11 12:41:44 [default_loader.py:267] Loading weights took 465.59 seconds
-(Worker_TP5 pid=1325) INFO 10-11 12:41:44 [deep_gemm.py:56] DeepGEMM E8M0 enabled on Hopper GPU.
-(Worker_TP0 pid=1320) INFO 10-11 12:41:45 [default_loader.py:267] Loading weights took 460.41 seconds
-(Worker_TP6 pid=1326) INFO 10-11 12:41:45 [default_loader.py:267] Loading weights took 467.05 seconds
-(Worker_TP2 pid=1322) INFO 10-11 12:41:45 [default_loader.py:267] Loading weights took 462.04 seconds
-(Worker_TP3 pid=1323) INFO 10-11 12:41:45 [default_loader.py:267] Loading weights took 459.15 seconds
-(Worker_TP7 pid=1327) INFO 10-11 12:41:45 [default_loader.py:267] Loading weights took 468.29 seconds
-(Worker_TP1 pid=1321) INFO 10-11 12:41:45 [default_loader.py:267] Loading weights took 463.25 seconds
-(Worker_TP4 pid=1324) INFO 10-11 12:41:45 [default_loader.py:267] Loading weights took 464.66 seconds
-(Worker_TP6 pid=1326) INFO 10-11 12:41:45 [deep_gemm.py:56] DeepGEMM E8M0 enabled on Hopper GPU.
-(Worker_TP2 pid=1322) INFO 10-11 12:41:45 [deep_gemm.py:56] DeepGEMM E8M0 enabled on Hopper GPU.
-(Worker_TP0 pid=1320) INFO 10-11 12:41:45 [deep_gemm.py:56] DeepGEMM E8M0 enabled on Hopper GPU.
-(Worker_TP7 pid=1327) INFO 10-11 12:41:45 [deep_gemm.py:56] DeepGEMM E8M0 enabled on Hopper GPU.
-(Worker_TP1 pid=1321) INFO 10-11 12:41:45 [deep_gemm.py:56] DeepGEMM E8M0 enabled on Hopper GPU.
-(Worker_TP3 pid=1323) INFO 10-11 12:41:45 [deep_gemm.py:56] DeepGEMM E8M0 enabled on Hopper GPU.
-(Worker_TP4 pid=1324) INFO 10-11 12:41:45 [deep_gemm.py:56] DeepGEMM E8M0 enabled on Hopper GPU.
-(Worker_TP5 pid=1325) WARNING 10-11 12:41:56 [kv_cache.py:86] Checkpoint does not provide a q scaling factor. Setting it to k_scale. This only matters for the flash-attn backend.
-(Worker_TP5 pid=1325) WARNING 10-11 12:41:56 [kv_cache.py:100] Using KV cache scaling factor 1.0 for fp8_e4m3. This may cause accuracy issues. Please make sure k/v_scale scaling factors are available in the fp8 checkpoint.
-(Worker_TP2 pid=1322) WARNING 10-11 12:41:56 [kv_cache.py:86] Checkpoint does not provide a q scaling factor. Setting it to k_scale. This only matters for the flash-attn backend.
-(Worker_TP2 pid=1322) WARNING 10-11 12:41:56 [kv_cache.py:100] Using KV cache scaling factor 1.0 for fp8_e4m3. This may cause accuracy issues. Please make sure k/v_scale scaling factors are available in the fp8 checkpoint.
-(Worker_TP3 pid=1323) WARNING 10-11 12:41:56 [kv_cache.py:86] Checkpoint does not provide a q scaling factor. Setting it to k_scale. This only matters for the flash-attn backend.
-(Worker_TP3 pid=1323) WARNING 10-11 12:41:56 [kv_cache.py:100] Using KV cache scaling factor 1.0 for fp8_e4m3. This may cause accuracy issues. Please make sure k/v_scale scaling factors are available in the fp8 checkpoint.
-(Worker_TP0 pid=1320) WARNING 10-11 12:41:56 [kv_cache.py:86] Checkpoint does not provide a q scaling factor. Setting it to k_scale. This only matters for the flash-attn backend.
-(Worker_TP0 pid=1320) WARNING 10-11 12:41:56 [kv_cache.py:100] Using KV cache scaling factor 1.0 for fp8_e4m3. This may cause accuracy issues. Please make sure k/v_scale scaling factors are available in the fp8 checkpoint.
-(Worker_TP6 pid=1326) WARNING 10-11 12:41:56 [kv_cache.py:86] Checkpoint does not provide a q scaling factor. Setting it to k_scale. This only matters for the flash-attn backend.
-(Worker_TP6 pid=1326) WARNING 10-11 12:41:56 [kv_cache.py:100] Using KV cache scaling factor 1.0 for fp8_e4m3. This may cause accuracy issues. Please make sure k/v_scale scaling factors are available in the fp8 checkpoint.
-(Worker_TP1 pid=1321) WARNING 10-11 12:41:56 [kv_cache.py:86] Checkpoint does not provide a q scaling factor. Setting it to k_scale. This only matters for the flash-attn backend.
-(Worker_TP1 pid=1321) WARNING 10-11 12:41:56 [kv_cache.py:100] Using KV cache scaling factor 1.0 for fp8_e4m3. This may cause accuracy issues. Please make sure k/v_scale scaling factors are available in the fp8 checkpoint.
-(Worker_TP4 pid=1324) WARNING 10-11 12:41:56 [kv_cache.py:86] Checkpoint does not provide a q scaling factor. Setting it to k_scale. This only matters for the flash-attn backend.
-(Worker_TP4 pid=1324) WARNING 10-11 12:41:56 [kv_cache.py:100] Using KV cache scaling factor 1.0 for fp8_e4m3. This may cause accuracy issues. Please make sure k/v_scale scaling factors are available in the fp8 checkpoint.
-(Worker_TP7 pid=1327) WARNING 10-11 12:41:56 [kv_cache.py:86] Checkpoint does not provide a q scaling factor. Setting it to k_scale. This only matters for the flash-attn backend.
-(Worker_TP7 pid=1327) WARNING 10-11 12:41:56 [kv_cache.py:100] Using KV cache scaling factor 1.0 for fp8_e4m3. This may cause accuracy issues. Please make sure k/v_scale scaling factors are available in the fp8 checkpoint.
-(Worker_TP2 pid=1322) INFO 10-11 12:42:05 [gpu_model_runner.py:2653] Model loading took 80.4752 GiB and 8506.266192 seconds
-(Worker_TP5 pid=1325) INFO 10-11 12:42:05 [gpu_model_runner.py:2653] Model loading took 80.4752 GiB and 8506.218286 seconds
-(Worker_TP3 pid=1323) INFO 10-11 12:42:05 [gpu_model_runner.py:2653] Model loading took 80.4752 GiB and 8506.720179 seconds
-(Worker_TP0 pid=1320) INFO 10-11 12:42:05 [gpu_model_runner.py:2653] Model loading took 80.4752 GiB and 8506.863211 seconds
-(Worker_TP6 pid=1326) INFO 10-11 12:42:05 [gpu_model_runner.py:2653] Model loading took 80.4752 GiB and 8506.866878 seconds
-(Worker_TP7 pid=1327) INFO 10-11 12:42:05 [gpu_model_runner.py:2653] Model loading took 80.4752 GiB and 8507.141598 seconds
-(Worker_TP1 pid=1321) INFO 10-11 12:42:05 [gpu_model_runner.py:2653] Model loading took 80.4752 GiB and 8507.125357 seconds
-(Worker_TP4 pid=1324) INFO 10-11 12:42:05 [gpu_model_runner.py:2653] Model loading took 80.4752 GiB and 8507.041859 seconds
-(Worker_TP6 pid=1326) INFO 10-11 12:42:19 [backends.py:548] Using cache directory: /root/.cache/vllm/torch_compile_cache/e63c1849d0/rank_6_0/backbone for vLLM's torch.compile
-(Worker_TP6 pid=1326) INFO 10-11 12:42:19 [backends.py:559] Dynamo bytecode transform time: 13.04 s
-(Worker_TP0 pid=1320) INFO 10-11 12:42:19 [backends.py:548] Using cache directory: /root/.cache/vllm/torch_compile_cache/e63c1849d0/rank_0_0/backbone for vLLM's torch.compile
-(Worker_TP0 pid=1320) INFO 10-11 12:42:19 [backends.py:559] Dynamo bytecode transform time: 13.04 s
-(Worker_TP3 pid=1323) INFO 10-11 12:42:19 [backends.py:548] Using cache directory: /root/.cache/vllm/torch_compile_cache/e63c1849d0/rank_3_0/backbone for vLLM's torch.compile
-(Worker_TP3 pid=1323) INFO 10-11 12:42:19 [backends.py:559] Dynamo bytecode transform time: 13.10 s
-(Worker_TP5 pid=1325) INFO 10-11 12:42:19 [backends.py:548] Using cache directory: /root/.cache/vllm/torch_compile_cache/e63c1849d0/rank_5_0/backbone for vLLM's torch.compile
-(Worker_TP5 pid=1325) INFO 10-11 12:42:19 [backends.py:559] Dynamo bytecode transform time: 13.09 s
-(Worker_TP2 pid=1322) INFO 10-11 12:42:19 [backends.py:548] Using cache directory: /root/.cache/vllm/torch_compile_cache/e63c1849d0/rank_2_0/backbone for vLLM's torch.compile
-(Worker_TP2 pid=1322) INFO 10-11 12:42:19 [backends.py:559] Dynamo bytecode transform time: 13.12 s
-(Worker_TP1 pid=1321) INFO 10-11 12:42:19 [backends.py:548] Using cache directory: /root/.cache/vllm/torch_compile_cache/e63c1849d0/rank_1_0/backbone for vLLM's torch.compile
-(Worker_TP1 pid=1321) INFO 10-11 12:42:19 [backends.py:559] Dynamo bytecode transform time: 13.19 s
-(Worker_TP4 pid=1324) INFO 10-11 12:42:19 [backends.py:548] Using cache directory: /root/.cache/vllm/torch_compile_cache/e63c1849d0/rank_4_0/backbone for vLLM's torch.compile
-(Worker_TP4 pid=1324) INFO 10-11 12:42:19 [backends.py:559] Dynamo bytecode transform time: 13.40 s
-(Worker_TP7 pid=1327) INFO 10-11 12:42:19 [backends.py:548] Using cache directory: /root/.cache/vllm/torch_compile_cache/e63c1849d0/rank_7_0/backbone for vLLM's torch.compile
-(Worker_TP7 pid=1327) INFO 10-11 12:42:19 [backends.py:559] Dynamo bytecode transform time: 13.51 s
-(Worker_TP0 pid=1320) INFO 10-11 12:42:29 [backends.py:197] Cache the graph for dynamic shape for later use
-(Worker_TP6 pid=1326) INFO 10-11 12:42:29 [backends.py:197] Cache the graph for dynamic shape for later use
-(Worker_TP5 pid=1325) INFO 10-11 12:42:30 [backends.py:197] Cache the graph for dynamic shape for later use
-(Worker_TP2 pid=1322) INFO 10-11 12:42:30 [backends.py:197] Cache the graph for dynamic shape for later use
-(Worker_TP3 pid=1323) INFO 10-11 12:42:30 [backends.py:197] Cache the graph for dynamic shape for later use
-(Worker_TP1 pid=1321) INFO 10-11 12:42:30 [backends.py:197] Cache the graph for dynamic shape for later use
-(Worker_TP7 pid=1327) INFO 10-11 12:42:30 [backends.py:197] Cache the graph for dynamic shape for later use
-(Worker_TP4 pid=1324) INFO 10-11 12:42:30 [backends.py:197] Cache the graph for dynamic shape for later use
-(EngineCore_DP0 pid=1184) INFO 10-11 12:43:05 [shm_broadcast.py:466] No available shared memory broadcast block found in 60 seconds. This typically happens when some processes are hanging or doing some time-consuming work (e.g. compilation).
-(Worker_TP1 pid=1321) [rank1]:W1011 12:43:48.852000 1321 torch/_inductor/codecache.py:1386] [1/0] fx graph unable to write to cache
-(Worker_TP1 pid=1321) [rank1]:W1011 12:43:48.852000 1321 torch/_inductor/codecache.py:1386] [1/0] Traceback (most recent call last):
-(Worker_TP1 pid=1321) [rank1]:W1011 12:43:48.852000 1321 torch/_inductor/codecache.py:1386] [1/0]   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/codecache.py", line 1376, in _save_graph
-(Worker_TP1 pid=1321) [rank1]:W1011 12:43:48.852000 1321 torch/_inductor/codecache.py:1386] [1/0]     FxGraphCache._write_to_local_cache(key, content)
-(Worker_TP1 pid=1321) [rank1]:W1011 12:43:48.852000 1321 torch/_inductor/codecache.py:1386] [1/0]   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/codecache.py", line 1328, in _write_to_local_cache
-(Worker_TP1 pid=1321) [rank1]:W1011 12:43:48.852000 1321 torch/_inductor/codecache.py:1386] [1/0]     write_atomic(path, content, make_dirs=True)
-(Worker_TP1 pid=1321) [rank1]:W1011 12:43:48.852000 1321 torch/_inductor/codecache.py:1386] [1/0]   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/codecache.py", line 472, in write_atomic
-(Worker_TP1 pid=1321) [rank1]:W1011 12:43:48.852000 1321 torch/_inductor/codecache.py:1386] [1/0]     f.write(content)
-(Worker_TP1 pid=1321) [rank1]:W1011 12:43:48.852000 1321 torch/_inductor/codecache.py:1386] [1/0] OSError: [Errno 28] No space left on device
-(Worker_TP5 pid=1325) [rank5]:W1011 12:43:48.852000 1325 torch/_inductor/codecache.py:1386] [1/0] fx graph unable to write to cache
-(Worker_TP5 pid=1325) [rank5]:W1011 12:43:48.852000 1325 torch/_inductor/codecache.py:1386] [1/0] Traceback (most recent call last):
-(Worker_TP5 pid=1325) [rank5]:W1011 12:43:48.852000 1325 torch/_inductor/codecache.py:1386] [1/0]   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/codecache.py", line 1376, in _save_graph
-(Worker_TP5 pid=1325) [rank5]:W1011 12:43:48.852000 1325 torch/_inductor/codecache.py:1386] [1/0]     FxGraphCache._write_to_local_cache(key, content)
-(Worker_TP5 pid=1325) [rank5]:W1011 12:43:48.852000 1325 torch/_inductor/codecache.py:1386] [1/0]   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/codecache.py", line 1328, in _write_to_local_cache
-(Worker_TP5 pid=1325) [rank5]:W1011 12:43:48.852000 1325 torch/_inductor/codecache.py:1386] [1/0]     write_atomic(path, content, make_dirs=True)
-(Worker_TP5 pid=1325) [rank5]:W1011 12:43:48.852000 1325 torch/_inductor/codecache.py:1386] [1/0]   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/codecache.py", line 472, in write_atomic
-(Worker_TP5 pid=1325) [rank5]:W1011 12:43:48.852000 1325 torch/_inductor/codecache.py:1386] [1/0]     f.write(content)
-(Worker_TP5 pid=1325) [rank5]:W1011 12:43:48.852000 1325 torch/_inductor/codecache.py:1386] [1/0] OSError: [Errno 28] No space left on device
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671] WorkerProc hit an exception.
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671] Traceback (most recent call last):
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/vllm/v1/executor/multiproc_executor.py", line 666, in worker_busy_loop
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     output = func(*args, **kwargs)
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]              ^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/utils/_contextlib.py", line 120, in decorate_context
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     return func(*args, **kwargs)
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]            ^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/vllm/v1/worker/gpu_worker.py", line 263, in determine_available_memory
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     self.model_runner.profile_run()
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/vllm/v1/worker/gpu_model_runner.py", line 3392, in profile_run
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     = self._dummy_run(self.max_num_tokens, is_profile=True)
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/utils/_contextlib.py", line 120, in decorate_context
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     return func(*args, **kwargs)
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]            ^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/vllm/v1/worker/gpu_model_runner.py", line 3152, in _dummy_run
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     outputs = self.model(
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]               ^^^^^^^^^^^
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/vllm/compilation/cuda_graph.py", line 121, in __call__
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     return self.runnable(*args, **kwargs)
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1773, in _wrapped_call_impl
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     return self._call_impl(*args, **kwargs)
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1784, in _call_impl
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     return forward_call(*args, **kwargs)
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/vllm/model_executor/models/deepseek_v2.py", line 1295, in forward
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     hidden_states = self.model(input_ids, positions, intermediate_tensors,
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]                     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/vllm/compilation/decorators.py", line 310, in __call__
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     output = self.compiled_callable(*args, **kwargs)
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_dynamo/eval_frame.py", line 749, in compile_wrapper
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     raise e.remove_dynamo_frames() from None  # see TORCHDYNAMO_VERBOSE=1
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/compile_fx.py", line 923, in _compile_fx_inner
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     raise InductorError(e, currentframe()).with_traceback(
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/compile_fx.py", line 907, in _compile_fx_inner
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     mb_compiled_graph = fx_codegen_and_compile(
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]                         ^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/compile_fx.py", line 1578, in fx_codegen_and_compile
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     return scheme.codegen_and_compile(gm, example_inputs, inputs_to_check, graph_kwargs)
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/compile_fx.py", line 1456, in codegen_and_compile
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     compiled_module = graph.compile_to_module()
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]                       ^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/graph.py", line 2293, in compile_to_module
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     return self._compile_to_module()
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]            ^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/graph.py", line 2303, in _compile_to_module
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     mod = self._compile_to_module_lines(wrapper_code)
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/graph.py", line 2355, in _compile_to_module_lines
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     key, path = PyCodeCache.write(wrapper_code.value)
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/codecache.py", line 3273, in write
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     return write(source_code, "py", extra=extra)
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/codecache.py", line 444, in write
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     write_atomic(path, content, make_dirs=True)
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/codecache.py", line 468, in write_atomic
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     path.parent.mkdir(parents=True, exist_ok=True)
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/lib/python3.12/pathlib.py", line 1313, in mkdir
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     os.mkdir(self, mode)
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671] torch._inductor.exc.InductorError: OSError: [Errno 28] No space left on device: '/root/.cache/vllm/torch_compile_cache/e63c1849d0/rank_6_0/inductor_cache/up'
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671] 
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671] Set TORCHDYNAMO_VERBOSE=1 for the internal stack trace (please do this especially if you're reporting a bug to PyTorch). For even more developer context, set TORCH_LOGS="+dynamo"
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671] 
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671] Traceback (most recent call last):
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/vllm/v1/executor/multiproc_executor.py", line 666, in worker_busy_loop
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     output = func(*args, **kwargs)
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]              ^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/utils/_contextlib.py", line 120, in decorate_context
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     return func(*args, **kwargs)
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]            ^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/vllm/v1/worker/gpu_worker.py", line 263, in determine_available_memory
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     self.model_runner.profile_run()
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/vllm/v1/worker/gpu_model_runner.py", line 3392, in profile_run
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     = self._dummy_run(self.max_num_tokens, is_profile=True)
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/utils/_contextlib.py", line 120, in decorate_context
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     return func(*args, **kwargs)
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]            ^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/vllm/v1/worker/gpu_model_runner.py", line 3152, in _dummy_run
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     outputs = self.model(
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]               ^^^^^^^^^^^
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/vllm/compilation/cuda_graph.py", line 121, in __call__
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     return self.runnable(*args, **kwargs)
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1773, in _wrapped_call_impl
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     return self._call_impl(*args, **kwargs)
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1784, in _call_impl
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     return forward_call(*args, **kwargs)
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/vllm/model_executor/models/deepseek_v2.py", line 1295, in forward
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     hidden_states = self.model(input_ids, positions, intermediate_tensors,
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]                     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/vllm/compilation/decorators.py", line 310, in __call__
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     output = self.compiled_callable(*args, **kwargs)
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_dynamo/eval_frame.py", line 749, in compile_wrapper
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     raise e.remove_dynamo_frames() from None  # see TORCHDYNAMO_VERBOSE=1
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/compile_fx.py", line 923, in _compile_fx_inner
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     raise InductorError(e, currentframe()).with_traceback(
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/compile_fx.py", line 907, in _compile_fx_inner
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     mb_compiled_graph = fx_codegen_and_compile(
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]                         ^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/compile_fx.py", line 1578, in fx_codegen_and_compile
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     return scheme.codegen_and_compile(gm, example_inputs, inputs_to_check, graph_kwargs)
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/compile_fx.py", line 1456, in codegen_and_compile
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     compiled_module = graph.compile_to_module()
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]                       ^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/graph.py", line 2293, in compile_to_module
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     return self._compile_to_module()
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]            ^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/graph.py", line 2303, in _compile_to_module
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     mod = self._compile_to_module_lines(wrapper_code)
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/graph.py", line 2355, in _compile_to_module_lines
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     key, path = PyCodeCache.write(wrapper_code.value)
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/codecache.py", line 3273, in write
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     return write(source_code, "py", extra=extra)
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/codecache.py", line 444, in write
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     write_atomic(path, content, make_dirs=True)
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/codecache.py", line 468, in write_atomic
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     path.parent.mkdir(parents=True, exist_ok=True)
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/lib/python3.12/pathlib.py", line 1313, in mkdir
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     os.mkdir(self, mode)
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671] torch._inductor.exc.InductorError: OSError: [Errno 28] No space left on device: '/root/.cache/vllm/torch_compile_cache/e63c1849d0/rank_6_0/inductor_cache/up'
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671] 
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671] Set TORCHDYNAMO_VERBOSE=1 for the internal stack trace (please do this especially if you're reporting a bug to PyTorch). For even more developer context, set TORCH_LOGS="+dynamo"
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671] 
-(Worker_TP6 pid=1326) ERROR 10-11 12:43:48 [multiproc_executor.py:671] 
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671] WorkerProc hit an exception.
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671] Traceback (most recent call last):
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/vllm/v1/executor/multiproc_executor.py", line 666, in worker_busy_loop
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     output = func(*args, **kwargs)
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]              ^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/utils/_contextlib.py", line 120, in decorate_context
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     return func(*args, **kwargs)
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]            ^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/vllm/v1/worker/gpu_worker.py", line 263, in determine_available_memory
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     self.model_runner.profile_run()
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/vllm/v1/worker/gpu_model_runner.py", line 3392, in profile_run
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     = self._dummy_run(self.max_num_tokens, is_profile=True)
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/utils/_contextlib.py", line 120, in decorate_context
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     return func(*args, **kwargs)
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]            ^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/vllm/v1/worker/gpu_model_runner.py", line 3152, in _dummy_run
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     outputs = self.model(
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]               ^^^^^^^^^^^
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/vllm/compilation/cuda_graph.py", line 121, in __call__
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     return self.runnable(*args, **kwargs)
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1773, in _wrapped_call_impl
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     return self._call_impl(*args, **kwargs)
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1784, in _call_impl
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     return forward_call(*args, **kwargs)
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/vllm/model_executor/models/deepseek_v2.py", line 1295, in forward
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     hidden_states = self.model(input_ids, positions, intermediate_tensors,
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]                     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/vllm/compilation/decorators.py", line 310, in __call__
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     output = self.compiled_callable(*args, **kwargs)
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_dynamo/eval_frame.py", line 749, in compile_wrapper
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     raise e.remove_dynamo_frames() from None  # see TORCHDYNAMO_VERBOSE=1
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/compile_fx.py", line 923, in _compile_fx_inner
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     raise InductorError(e, currentframe()).with_traceback(
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/compile_fx.py", line 907, in _compile_fx_inner
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     mb_compiled_graph = fx_codegen_and_compile(
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]                         ^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/compile_fx.py", line 1578, in fx_codegen_and_compile
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     return scheme.codegen_and_compile(gm, example_inputs, inputs_to_check, graph_kwargs)
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/compile_fx.py", line 1456, in codegen_and_compile
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     compiled_module = graph.compile_to_module()
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]                       ^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/graph.py", line 2293, in compile_to_module
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     return self._compile_to_module()
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]            ^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/graph.py", line 2303, in _compile_to_module
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     mod = self._compile_to_module_lines(wrapper_code)
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/graph.py", line 2355, in _compile_to_module_lines
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     key, path = PyCodeCache.write(wrapper_code.value)
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/codecache.py", line 3273, in write
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     return write(source_code, "py", extra=extra)
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/codecache.py", line 444, in write
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     write_atomic(path, content, make_dirs=True)
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/codecache.py", line 472, in write_atomic
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     f.write(content)
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671] torch._inductor.exc.InductorError: OSError: [Errno 28] No space left on device
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671] 
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671] Set TORCHDYNAMO_VERBOSE=1 for the internal stack trace (please do this especially if you're reporting a bug to PyTorch). For even more developer context, set TORCH_LOGS="+dynamo"
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671] 
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671] Traceback (most recent call last):
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/vllm/v1/executor/multiproc_executor.py", line 666, in worker_busy_loop
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     output = func(*args, **kwargs)
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]              ^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/utils/_contextlib.py", line 120, in decorate_context
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     return func(*args, **kwargs)
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]            ^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/vllm/v1/worker/gpu_worker.py", line 263, in determine_available_memory
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     self.model_runner.profile_run()
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/vllm/v1/worker/gpu_model_runner.py", line 3392, in profile_run
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     = self._dummy_run(self.max_num_tokens, is_profile=True)
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/utils/_contextlib.py", line 120, in decorate_context
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     return func(*args, **kwargs)
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]            ^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/vllm/v1/worker/gpu_model_runner.py", line 3152, in _dummy_run
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     outputs = self.model(
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]               ^^^^^^^^^^^
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/vllm/compilation/cuda_graph.py", line 121, in __call__
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     return self.runnable(*args, **kwargs)
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1773, in _wrapped_call_impl
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     return self._call_impl(*args, **kwargs)
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1784, in _call_impl
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     return forward_call(*args, **kwargs)
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/vllm/model_executor/models/deepseek_v2.py", line 1295, in forward
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     hidden_states = self.model(input_ids, positions, intermediate_tensors,
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]                     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/vllm/compilation/decorators.py", line 310, in __call__
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     output = self.compiled_callable(*args, **kwargs)
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_dynamo/eval_frame.py", line 749, in compile_wrapper
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     raise e.remove_dynamo_frames() from None  # see TORCHDYNAMO_VERBOSE=1
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/compile_fx.py", line 923, in _compile_fx_inner
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     raise InductorError(e, currentframe()).with_traceback(
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/compile_fx.py", line 907, in _compile_fx_inner
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     mb_compiled_graph = fx_codegen_and_compile(
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]                         ^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/compile_fx.py", line 1578, in fx_codegen_and_compile
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     return scheme.codegen_and_compile(gm, example_inputs, inputs_to_check, graph_kwargs)
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/compile_fx.py", line 1456, in codegen_and_compile
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     compiled_module = graph.compile_to_module()
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]                       ^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/graph.py", line 2293, in compile_to_module
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     return self._compile_to_module()
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]            ^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/graph.py", line 2303, in _compile_to_module
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     mod = self._compile_to_module_lines(wrapper_code)
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/graph.py", line 2355, in _compile_to_module_lines
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     key, path = PyCodeCache.write(wrapper_code.value)
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/codecache.py", line 3273, in write
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     return write(source_code, "py", extra=extra)
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/codecache.py", line 444, in write
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     write_atomic(path, content, make_dirs=True)
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/codecache.py", line 472, in write_atomic
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671]     f.write(content)
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671] torch._inductor.exc.InductorError: OSError: [Errno 28] No space left on device
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671] 
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671] Set TORCHDYNAMO_VERBOSE=1 for the internal stack trace (please do this especially if you're reporting a bug to PyTorch). For even more developer context, set TORCH_LOGS="+dynamo"
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671] 
-(Worker_TP2 pid=1322) ERROR 10-11 12:43:48 [multiproc_executor.py:671] 
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671] WorkerProc hit an exception.
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671] Traceback (most recent call last):
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/vllm/v1/executor/multiproc_executor.py", line 666, in worker_busy_loop
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]     output = func(*args, **kwargs)
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]              ^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/utils/_contextlib.py", line 120, in decorate_context
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]     return func(*args, **kwargs)
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]            ^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/vllm/v1/worker/gpu_worker.py", line 263, in determine_available_memory
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]     self.model_runner.profile_run()
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/vllm/v1/worker/gpu_model_runner.py", line 3392, in profile_run
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]     = self._dummy_run(self.max_num_tokens, is_profile=True)
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/utils/_contextlib.py", line 120, in decorate_context
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]     return func(*args, **kwargs)
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]            ^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/vllm/v1/worker/gpu_model_runner.py", line 3152, in _dummy_run
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]     outputs = self.model(
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]               ^^^^^^^^^^^
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/vllm/compilation/cuda_graph.py", line 121, in __call__
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]     return self.runnable(*args, **kwargs)
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1773, in _wrapped_call_impl
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]     return self._call_impl(*args, **kwargs)
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1784, in _call_impl
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]     return forward_call(*args, **kwargs)
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/vllm/model_executor/models/deepseek_v2.py", line 1295, in forward
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]     hidden_states = self.model(input_ids, positions, intermediate_tensors,
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]                     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/vllm/compilation/decorators.py", line 310, in __call__
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]     output = self.compiled_callable(*args, **kwargs)
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_dynamo/eval_frame.py", line 749, in compile_wrapper
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]     raise e.remove_dynamo_frames() from None  # see TORCHDYNAMO_VERBOSE=1
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/compile_fx.py", line 923, in _compile_fx_inner
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]     raise InductorError(e, currentframe()).with_traceback(
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/compile_fx.py", line 907, in _compile_fx_inner
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]     mb_compiled_graph = fx_codegen_and_compile(
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]                         ^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/compile_fx.py", line 1578, in fx_codegen_and_compile
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]     return scheme.codegen_and_compile(gm, example_inputs, inputs_to_check, graph_kwargs)
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/compile_fx.py", line 1456, in codegen_and_compile
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]     compiled_module = graph.compile_to_module()
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]                       ^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/graph.py", line 2293, in compile_to_module
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]     return self._compile_to_module()
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]            ^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/graph.py", line 2303, in _compile_to_module
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]     mod = self._compile_to_module_lines(wrapper_code)
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/graph.py", line 2355, in _compile_to_module_lines
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]     key, path = PyCodeCache.write(wrapper_code.value)
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/codecache.py", line 3273, in write
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]     return write(source_code, "py", extra=extra)
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/codecache.py", line 444, in write
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]     write_atomic(path, content, make_dirs=True)
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/codecache.py", line 468, in write_atomic
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]     path.parent.mkdir(parents=True, exist_ok=True)
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]   File "/usr/lib/python3.12/pathlib.py", line 1313, in mkdir
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]     os.mkdir(self, mode)
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671] torch._inductor.exc.InductorError: OSError: [Errno 28] No space left on device: '/root/.cache/vllm/torch_compile_cache/e63c1849d0/rank_3_0/inductor_cache/wm'
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671] 
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671] Set TORCHDYNAMO_VERBOSE=1 for the internal stack trace (please do this especially if you're reporting a bug to PyTorch). For even more developer context, set TORCH_LOGS="+dynamo"
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671] 
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671] Traceback (most recent call last):
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/vllm/v1/executor/multiproc_executor.py", line 666, in worker_busy_loop
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]     output = func(*args, **kwargs)
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]              ^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/utils/_contextlib.py", line 120, in decorate_context
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]     return func(*args, **kwargs)
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]            ^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/vllm/v1/worker/gpu_worker.py", line 263, in determine_available_memory
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]     self.model_runner.profile_run()
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/vllm/v1/worker/gpu_model_runner.py", line 3392, in profile_run
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]     = self._dummy_run(self.max_num_tokens, is_profile=True)
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/utils/_contextlib.py", line 120, in decorate_context
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]     return func(*args, **kwargs)
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]            ^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/vllm/v1/worker/gpu_model_runner.py", line 3152, in _dummy_run
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]     outputs = self.model(
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]               ^^^^^^^^^^^
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/vllm/compilation/cuda_graph.py", line 121, in __call__
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]     return self.runnable(*args, **kwargs)
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1773, in _wrapped_call_impl
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]     return self._call_impl(*args, **kwargs)
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1784, in _call_impl
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]     return forward_call(*args, **kwargs)
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/vllm/model_executor/models/deepseek_v2.py", line 1295, in forward
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]     hidden_states = self.model(input_ids, positions, intermediate_tensors,
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]                     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/vllm/compilation/decorators.py", line 310, in __call__
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]     output = self.compiled_callable(*args, **kwargs)
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_dynamo/eval_frame.py", line 749, in compile_wrapper
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]     raise e.remove_dynamo_frames() from None  # see TORCHDYNAMO_VERBOSE=1
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/compile_fx.py", line 923, in _compile_fx_inner
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]     raise InductorError(e, currentframe()).with_traceback(
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/compile_fx.py", line 907, in _compile_fx_inner
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]     mb_compiled_graph = fx_codegen_and_compile(
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]                         ^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/compile_fx.py", line 1578, in fx_codegen_and_compile
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]     return scheme.codegen_and_compile(gm, example_inputs, inputs_to_check, graph_kwargs)
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/compile_fx.py", line 1456, in codegen_and_compile
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]     compiled_module = graph.compile_to_module()
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]                       ^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/graph.py", line 2293, in compile_to_module
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]     return self._compile_to_module()
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]            ^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/graph.py", line 2303, in _compile_to_module
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]     mod = self._compile_to_module_lines(wrapper_code)
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/graph.py", line 2355, in _compile_to_module_lines
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]     key, path = PyCodeCache.write(wrapper_code.value)
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/codecache.py", line 3273, in write
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]     return write(source_code, "py", extra=extra)
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/codecache.py", line 444, in write
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]     write_atomic(path, content, make_dirs=True)
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/codecache.py", line 468, in write_atomic
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]     path.parent.mkdir(parents=True, exist_ok=True)
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]   File "/usr/lib/python3.12/pathlib.py", line 1313, in mkdir
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671]     os.mkdir(self, mode)
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671] torch._inductor.exc.InductorError: OSError: [Errno 28] No space left on device: '/root/.cache/vllm/torch_compile_cache/e63c1849d0/rank_3_0/inductor_cache/wm'
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671] 
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671] Set TORCHDYNAMO_VERBOSE=1 for the internal stack trace (please do this especially if you're reporting a bug to PyTorch). For even more developer context, set TORCH_LOGS="+dynamo"
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671] 
-(Worker_TP3 pid=1323) ERROR 10-11 12:43:49 [multiproc_executor.py:671] 
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671] WorkerProc hit an exception.
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671] Traceback (most recent call last):
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/vllm/v1/executor/multiproc_executor.py", line 666, in worker_busy_loop
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]     output = func(*args, **kwargs)
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]              ^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/utils/_contextlib.py", line 120, in decorate_context
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]     return func(*args, **kwargs)
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]            ^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/vllm/v1/worker/gpu_worker.py", line 263, in determine_available_memory
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]     self.model_runner.profile_run()
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/vllm/v1/worker/gpu_model_runner.py", line 3392, in profile_run
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]     = self._dummy_run(self.max_num_tokens, is_profile=True)
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/utils/_contextlib.py", line 120, in decorate_context
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]     return func(*args, **kwargs)
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]            ^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/vllm/v1/worker/gpu_model_runner.py", line 3152, in _dummy_run
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]     outputs = self.model(
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]               ^^^^^^^^^^^
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/vllm/compilation/cuda_graph.py", line 121, in __call__
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]     return self.runnable(*args, **kwargs)
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1773, in _wrapped_call_impl
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]     return self._call_impl(*args, **kwargs)
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1784, in _call_impl
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]     return forward_call(*args, **kwargs)
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/vllm/model_executor/models/deepseek_v2.py", line 1295, in forward
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]     hidden_states = self.model(input_ids, positions, intermediate_tensors,
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]                     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/vllm/compilation/decorators.py", line 310, in __call__
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]     output = self.compiled_callable(*args, **kwargs)
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_dynamo/eval_frame.py", line 749, in compile_wrapper
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]     raise e.remove_dynamo_frames() from None  # see TORCHDYNAMO_VERBOSE=1
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/compile_fx.py", line 923, in _compile_fx_inner
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]     raise InductorError(e, currentframe()).with_traceback(
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/compile_fx.py", line 907, in _compile_fx_inner
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]     mb_compiled_graph = fx_codegen_and_compile(
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]                         ^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/compile_fx.py", line 1578, in fx_codegen_and_compile
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]     return scheme.codegen_and_compile(gm, example_inputs, inputs_to_check, graph_kwargs)
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/compile_fx.py", line 1456, in codegen_and_compile
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]     compiled_module = graph.compile_to_module()
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]                       ^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/graph.py", line 2293, in compile_to_module
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]     return self._compile_to_module()
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]            ^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/graph.py", line 2303, in _compile_to_module
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]     mod = self._compile_to_module_lines(wrapper_code)
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/graph.py", line 2355, in _compile_to_module_lines
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]     key, path = PyCodeCache.write(wrapper_code.value)
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/codecache.py", line 3273, in write
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]     return write(source_code, "py", extra=extra)
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/codecache.py", line 444, in write
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]     write_atomic(path, content, make_dirs=True)
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/codecache.py", line 472, in write_atomic
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]     f.write(content)
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671] torch._inductor.exc.InductorError: OSError: [Errno 28] No space left on device
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671] 
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671] Set TORCHDYNAMO_VERBOSE=1 for the internal stack trace (please do this especially if you're reporting a bug to PyTorch). For even more developer context, set TORCH_LOGS="+dynamo"
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671] 
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671] Traceback (most recent call last):
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/vllm/v1/executor/multiproc_executor.py", line 666, in worker_busy_loop
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]     output = func(*args, **kwargs)
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]              ^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/utils/_contextlib.py", line 120, in decorate_context
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]     return func(*args, **kwargs)
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]            ^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/vllm/v1/worker/gpu_worker.py", line 263, in determine_available_memory
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]     self.model_runner.profile_run()
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/vllm/v1/worker/gpu_model_runner.py", line 3392, in profile_run
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]     = self._dummy_run(self.max_num_tokens, is_profile=True)
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/utils/_contextlib.py", line 120, in decorate_context
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]     return func(*args, **kwargs)
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]            ^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/vllm/v1/worker/gpu_model_runner.py", line 3152, in _dummy_run
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]     outputs = self.model(
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]               ^^^^^^^^^^^
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/vllm/compilation/cuda_graph.py", line 121, in __call__
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]     return self.runnable(*args, **kwargs)
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1773, in _wrapped_call_impl
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]     return self._call_impl(*args, **kwargs)
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1784, in _call_impl
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]     return forward_call(*args, **kwargs)
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/vllm/model_executor/models/deepseek_v2.py", line 1295, in forward
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]     hidden_states = self.model(input_ids, positions, intermediate_tensors,
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]                     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/vllm/compilation/decorators.py", line 310, in __call__
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]     output = self.compiled_callable(*args, **kwargs)
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_dynamo/eval_frame.py", line 749, in compile_wrapper
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]     raise e.remove_dynamo_frames() from None  # see TORCHDYNAMO_VERBOSE=1
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/compile_fx.py", line 923, in _compile_fx_inner
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]     raise InductorError(e, currentframe()).with_traceback(
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/compile_fx.py", line 907, in _compile_fx_inner
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]     mb_compiled_graph = fx_codegen_and_compile(
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]                         ^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/compile_fx.py", line 1578, in fx_codegen_and_compile
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]     return scheme.codegen_and_compile(gm, example_inputs, inputs_to_check, graph_kwargs)
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/compile_fx.py", line 1456, in codegen_and_compile
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]     compiled_module = graph.compile_to_module()
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]                       ^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/graph.py", line 2293, in compile_to_module
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]     return self._compile_to_module()
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]            ^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/graph.py", line 2303, in _compile_to_module
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]     mod = self._compile_to_module_lines(wrapper_code)
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/graph.py", line 2355, in _compile_to_module_lines
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]     key, path = PyCodeCache.write(wrapper_code.value)
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/codecache.py", line 3273, in write
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]     return write(source_code, "py", extra=extra)
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/codecache.py", line 444, in write
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]     write_atomic(path, content, make_dirs=True)
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]   File "/usr/local/lib/python3.12/dist-packages/torch/_inductor/codecache.py", line 472, in write_atomic
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671]     f.write(content)
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671] torch._inductor.exc.InductorError: OSError: [Errno 28] No space left on device
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671] 
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671] Set TORCHDYNAMO_VERBOSE=1 for the internal stack trace (please do this especially if you're reporting a bug to PyTorch). For even more developer context, set TORCH_LOGS="+dynamo"
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671] 
-(Worker_TP0 pid=1320) ERROR 10-11 12:43:50 [multiproc_executor.py:671] 
-(EngineCore_DP0 pid=1184) ERROR 10-11 12:43:50 [core.py:708] EngineCore failed to start.
-(EngineCore_DP0 pid=1184) ERROR 10-11 12:43:50 [core.py:708] Traceback (most recent call last):
-(EngineCore_DP0 pid=1184) ERROR 10-11 12:43:50 [core.py:708]   File "/usr/local/lib/python3.12/dist-packages/vllm/v1/engine/core.py", line 699, in run_engine_core
-(EngineCore_DP0 pid=1184) ERROR 10-11 12:43:50 [core.py:708]     engine_core = EngineCoreProc(*args, **kwargs)
-(EngineCore_DP0 pid=1184) ERROR 10-11 12:43:50 [core.py:708]                   ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(EngineCore_DP0 pid=1184) ERROR 10-11 12:43:50 [core.py:708]   File "/usr/local/lib/python3.12/dist-packages/vllm/v1/engine/core.py", line 498, in __init__
-(EngineCore_DP0 pid=1184) ERROR 10-11 12:43:50 [core.py:708]     super().__init__(vllm_config, executor_class, log_stats,
-(EngineCore_DP0 pid=1184) ERROR 10-11 12:43:50 [core.py:708]   File "/usr/local/lib/python3.12/dist-packages/vllm/v1/engine/core.py", line 92, in __init__
-(EngineCore_DP0 pid=1184) ERROR 10-11 12:43:50 [core.py:708]     self._initialize_kv_caches(vllm_config)
-(EngineCore_DP0 pid=1184) ERROR 10-11 12:43:50 [core.py:708]   File "/usr/local/lib/python3.12/dist-packages/vllm/v1/engine/core.py", line 190, in _initialize_kv_caches
-(EngineCore_DP0 pid=1184) ERROR 10-11 12:43:50 [core.py:708]     self.model_executor.determine_available_memory())
-(EngineCore_DP0 pid=1184) ERROR 10-11 12:43:50 [core.py:708]     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(EngineCore_DP0 pid=1184) ERROR 10-11 12:43:50 [core.py:708]   File "/usr/local/lib/python3.12/dist-packages/vllm/v1/executor/abstract.py", line 85, in determine_available_memory
-(EngineCore_DP0 pid=1184) ERROR 10-11 12:43:50 [core.py:708]     return self.collective_rpc("determine_available_memory")
-(EngineCore_DP0 pid=1184) ERROR 10-11 12:43:50 [core.py:708]            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(EngineCore_DP0 pid=1184) ERROR 10-11 12:43:50 [core.py:708]   File "/usr/local/lib/python3.12/dist-packages/vllm/v1/executor/multiproc_executor.py", line 264, in collective_rpc
-(EngineCore_DP0 pid=1184) ERROR 10-11 12:43:50 [core.py:708]     result = get_response(w, dequeue_timeout,
-(EngineCore_DP0 pid=1184) ERROR 10-11 12:43:50 [core.py:708]              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(EngineCore_DP0 pid=1184) ERROR 10-11 12:43:50 [core.py:708]   File "/usr/local/lib/python3.12/dist-packages/vllm/v1/executor/multiproc_executor.py", line 248, in get_response
-(EngineCore_DP0 pid=1184) ERROR 10-11 12:43:50 [core.py:708]     raise RuntimeError(
-(EngineCore_DP0 pid=1184) ERROR 10-11 12:43:50 [core.py:708] RuntimeError: Worker failed with error 'OSError: [Errno 28] No space left on device
-(EngineCore_DP0 pid=1184) ERROR 10-11 12:43:50 [core.py:708] 
-(EngineCore_DP0 pid=1184) ERROR 10-11 12:43:50 [core.py:708] Set TORCHDYNAMO_VERBOSE=1 for the internal stack trace (please do this especially if you're reporting a bug to PyTorch). For even more developer context, set TORCH_LOGS="+dynamo"
-(EngineCore_DP0 pid=1184) ERROR 10-11 12:43:50 [core.py:708] ', please check the stack trace above for the root cause
-(Worker_TP0 pid=1320) Exception ignored in: <function WeakIdKeyDictionary.__init__.<locals>.remove at 0x7ff303a50720>
-(Worker_TP0 pid=1320) Traceback (most recent call last):
-(Worker_TP0 pid=1320)   File "/usr/local/lib/python3.12/dist-packages/torch/utils/weak.py", line 134, in remove
-(Worker_TP0 pid=1320)     def remove(k, selfref=ref(self)):
-(Worker_TP0 pid=1320) 
-(Worker_TP0 pid=1320)   File "/usr/local/lib/python3.12/dist-packages/vllm/v1/executor/multiproc_executor.py", line 538, in signal_handler
-(Worker_TP0 pid=1320)     raise SystemExit()
-(Worker_TP0 pid=1320) SystemExit: 
-(EngineCore_DP0 pid=1184) ERROR 10-11 12:43:52 [multiproc_executor.py:154] Worker proc VllmWorker-2 died unexpectedly, shutting down executor.
-(Worker_TP0 pid=1320) INFO 10-11 12:43:52 [multiproc_executor.py:558] Parent process exited, terminating worker
-(Worker_TP0 pid=1320) INFO 10-11 12:43:52 [multiproc_executor.py:599] WorkerProc shutting down.
-(EngineCore_DP0 pid=1184) Process EngineCore_DP0:
-(EngineCore_DP0 pid=1184) Traceback (most recent call last):
-(EngineCore_DP0 pid=1184)   File "/usr/lib/python3.12/multiprocessing/process.py", line 314, in _bootstrap
-(EngineCore_DP0 pid=1184)     self.run()
-(EngineCore_DP0 pid=1184)   File "/usr/lib/python3.12/multiprocessing/process.py", line 108, in run
-(EngineCore_DP0 pid=1184)     self._target(*self._args, **self._kwargs)
-(EngineCore_DP0 pid=1184)   File "/usr/local/lib/python3.12/dist-packages/vllm/v1/engine/core.py", line 712, in run_engine_core
-(EngineCore_DP0 pid=1184)     raise e
-(EngineCore_DP0 pid=1184)   File "/usr/local/lib/python3.12/dist-packages/vllm/v1/engine/core.py", line 699, in run_engine_core
-(EngineCore_DP0 pid=1184)     engine_core = EngineCoreProc(*args, **kwargs)
-(EngineCore_DP0 pid=1184)                   ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(EngineCore_DP0 pid=1184)   File "/usr/local/lib/python3.12/dist-packages/vllm/v1/engine/core.py", line 498, in __init__
-(EngineCore_DP0 pid=1184)     super().__init__(vllm_config, executor_class, log_stats,
-(EngineCore_DP0 pid=1184)   File "/usr/local/lib/python3.12/dist-packages/vllm/v1/engine/core.py", line 92, in __init__
-(EngineCore_DP0 pid=1184)     self._initialize_kv_caches(vllm_config)
-(EngineCore_DP0 pid=1184)   File "/usr/local/lib/python3.12/dist-packages/vllm/v1/engine/core.py", line 190, in _initialize_kv_caches
-(EngineCore_DP0 pid=1184)     self.model_executor.determine_available_memory())
-(EngineCore_DP0 pid=1184)     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(EngineCore_DP0 pid=1184)   File "/usr/local/lib/python3.12/dist-packages/vllm/v1/executor/abstract.py", line 85, in determine_available_memory
-(EngineCore_DP0 pid=1184)     return self.collective_rpc("determine_available_memory")
-(EngineCore_DP0 pid=1184)            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(EngineCore_DP0 pid=1184)   File "/usr/local/lib/python3.12/dist-packages/vllm/v1/executor/multiproc_executor.py", line 264, in collective_rpc
-(EngineCore_DP0 pid=1184)     result = get_response(w, dequeue_timeout,
-(EngineCore_DP0 pid=1184)              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(EngineCore_DP0 pid=1184)   File "/usr/local/lib/python3.12/dist-packages/vllm/v1/executor/multiproc_executor.py", line 248, in get_response
-(EngineCore_DP0 pid=1184)     raise RuntimeError(
-(EngineCore_DP0 pid=1184) RuntimeError: Worker failed with error 'OSError: [Errno 28] No space left on device
-(EngineCore_DP0 pid=1184) 
-(EngineCore_DP0 pid=1184) Set TORCHDYNAMO_VERBOSE=1 for the internal stack trace (please do this especially if you're reporting a bug to PyTorch). For even more developer context, set TORCH_LOGS="+dynamo"
-(EngineCore_DP0 pid=1184) ', please check the stack trace above for the root cause
-(APIServer pid=1027) Traceback (most recent call last):
-(APIServer pid=1027)   File "/usr/local/bin/vllm", line 8, in <module>
-(APIServer pid=1027)     sys.exit(main())
-(APIServer pid=1027)              ^^^^^^
-(APIServer pid=1027)   File "/usr/local/lib/python3.12/dist-packages/vllm/entrypoints/cli/main.py", line 54, in main
-(APIServer pid=1027)     args.dispatch_function(args)
-(APIServer pid=1027)   File "/usr/local/lib/python3.12/dist-packages/vllm/entrypoints/cli/serve.py", line 57, in cmd
-(APIServer pid=1027)     uvloop.run(run_server(args))
-(APIServer pid=1027)   File "/usr/local/lib/python3.12/dist-packages/uvloop/__init__.py", line 109, in run
-(APIServer pid=1027)     return __asyncio.run(
-(APIServer pid=1027)            ^^^^^^^^^^^^^^
-(APIServer pid=1027)   File "/usr/lib/python3.12/asyncio/runners.py", line 194, in run
-(APIServer pid=1027)     return runner.run(main)
-(APIServer pid=1027)            ^^^^^^^^^^^^^^^^
-(APIServer pid=1027)   File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
-(APIServer pid=1027)     return self._loop.run_until_complete(task)
-(APIServer pid=1027)            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(APIServer pid=1027)   File "uvloop/loop.pyx", line 1518, in uvloop.loop.Loop.run_until_complete
-(APIServer pid=1027)   File "/usr/local/lib/python3.12/dist-packages/uvloop/__init__.py", line 61, in wrapper
-(APIServer pid=1027)     return await main
-(APIServer pid=1027)            ^^^^^^^^^^
-(APIServer pid=1027)   File "/usr/local/lib/python3.12/dist-packages/vllm/entrypoints/openai/api_server.py", line 1884, in run_server
-(APIServer pid=1027)     await run_server_worker(listen_address, sock, args, **uvicorn_kwargs)
-(APIServer pid=1027)   File "/usr/local/lib/python3.12/dist-packages/vllm/entrypoints/openai/api_server.py", line 1902, in run_server_worker
-(APIServer pid=1027)     async with build_async_engine_client(
-(APIServer pid=1027)   File "/usr/lib/python3.12/contextlib.py", line 210, in __aenter__
-(APIServer pid=1027)     return await anext(self.gen)
-(APIServer pid=1027)            ^^^^^^^^^^^^^^^^^^^^^
-(APIServer pid=1027)   File "/usr/local/lib/python3.12/dist-packages/vllm/entrypoints/openai/api_server.py", line 180, in build_async_engine_client
-(APIServer pid=1027)     async with build_async_engine_client_from_engine_args(
-(APIServer pid=1027)   File "/usr/lib/python3.12/contextlib.py", line 210, in __aenter__
-(APIServer pid=1027)     return await anext(self.gen)
-(APIServer pid=1027)            ^^^^^^^^^^^^^^^^^^^^^
-(APIServer pid=1027)   File "/usr/local/lib/python3.12/dist-packages/vllm/entrypoints/openai/api_server.py", line 225, in build_async_engine_client_from_engine_args
-(APIServer pid=1027)     async_llm = AsyncLLM.from_vllm_config(
-(APIServer pid=1027)                 ^^^^^^^^^^^^^^^^^^^^^^^^^^
-(APIServer pid=1027)   File "/usr/local/lib/python3.12/dist-packages/vllm/utils/__init__.py", line 1572, in inner
-(APIServer pid=1027)     return fn(*args, **kwargs)
-(APIServer pid=1027)            ^^^^^^^^^^^^^^^^^^^
-(APIServer pid=1027)   File "/usr/local/lib/python3.12/dist-packages/vllm/v1/engine/async_llm.py", line 207, in from_vllm_config
-(APIServer pid=1027)     return cls(
-(APIServer pid=1027)            ^^^^
-(APIServer pid=1027)   File "/usr/local/lib/python3.12/dist-packages/vllm/v1/engine/async_llm.py", line 134, in __init__
-(APIServer pid=1027)     self.engine_core = EngineCoreClient.make_async_mp_client(
-(APIServer pid=1027)                        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(APIServer pid=1027)   File "/usr/local/lib/python3.12/dist-packages/vllm/v1/engine/core_client.py", line 102, in make_async_mp_client
-(APIServer pid=1027)     return AsyncMPClient(*client_args)
-(APIServer pid=1027)            ^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(APIServer pid=1027)   File "/usr/local/lib/python3.12/dist-packages/vllm/v1/engine/core_client.py", line 769, in __init__
-(APIServer pid=1027)     super().__init__(
-(APIServer pid=1027)   File "/usr/local/lib/python3.12/dist-packages/vllm/v1/engine/core_client.py", line 448, in __init__
-(APIServer pid=1027)     with launch_core_engines(vllm_config, executor_class,
-(APIServer pid=1027)   File "/usr/lib/python3.12/contextlib.py", line 144, in __exit__
-(APIServer pid=1027)     next(self.gen)
-(APIServer pid=1027)   File "/usr/local/lib/python3.12/dist-packages/vllm/v1/engine/utils.py", line 732, in launch_core_engines
-(APIServer pid=1027)     wait_for_engine_startup(
-(APIServer pid=1027)   File "/usr/local/lib/python3.12/dist-packages/vllm/v1/engine/utils.py", line 785, in wait_for_engine_startup
-(APIServer pid=1027)     raise RuntimeError("Engine core initialization failed. "
-(APIServer pid=1027) RuntimeError: Engine core initialization failed. See root cause above. Failed core proc(s): {}