vllm: [Bug]: The output size is not aligned with the quantized weight shape. This can be caused by too large tensor parallel size.
Your current environment
The output of `python collect_env.py`
Your output of `python collect_env.py` here
Model Input Dumps
No response
๐ Describe the bug
I use the AWQ and GPTQ methods to quantify qwen2.5-32b for Qwen2ForSequenceClassification. And use vLLM-0.6.6 infer them. It should work in theory, but they donโt seem to support it.
group_size=128/64/32
from vllm import LLM
os.environ["CUDA_VISIBLE_DEVICES"] = "0,1"
llm = LLM(
model=qwen2.2-32b-cls-awq,
task="classify",
dtype="half",
tensor_parallel_size=2,
gpu_memory_utilization=0.98,
trust_remote_code=True,
max_model_len=1024,
enforce_eager=True,
)
tokenizer = llm.get_tokenizer()
{
"_name_or_path": "qwen2.5_32b_cls2",
"architectures": [
"Qwen2ForSequenceClassification"
],
"attention_dropout": 0.0,
"bos_token_id": 151643,
"eos_token_id": 151645,
"hidden_act": "silu",
"hidden_size": 5120,
"initializer_range": 0.02,
"intermediate_size": 27648,
"max_position_embeddings": 32768,
"max_window_layers": 70,
"model_type": "qwen2",
"num_attention_heads": 40,
"num_hidden_layers": 64,
"num_key_value_heads": 8,
"quantization_config": {
"bits": 4,
"group_size": 128,
"modules_to_not_convert": null,
"quant_method": "awq",
"version": "gemm",
"zero_point": true
},
"rms_norm_eps": 1e-06,
"rope_scaling": null,
"rope_theta": 1000000.0,
"sliding_window": null,
"tie_word_embeddings": false,
"torch_dtype": "bfloat16",
"transformers_version": "4.46.3",
"use_cache": false,
"use_sliding_window": false,
"vocab_size": 152064
}
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
<ipython-input-7-92d2819a49ae> in <cell line: 4>()
2 os.environ["CUDA_VISIBLE_DEVICES"] = "0,1"
3
----> 4 llm = LLM(
5 model=MODEL_NAME,
6 task="classify",
/usr/local/lib/python3.10/dist-packages/vllm/utils.py in inner(*args, **kwargs)
984 )
985
--> 986 return fn(*args, **kwargs)
987
988 return inner # type: ignore
/usr/local/lib/python3.10/dist-packages/vllm/entrypoints/llm.py in __init__(self, model, tokenizer, tokenizer_mode, skip_tokenizer_init, trust_remote_code, allowed_local_media_path, tensor_parallel_size, dtype, quantization, revision, tokenizer_revision, seed, gpu_memory_utilization, swap_space, cpu_offload_gb, enforce_eager, max_seq_len_to_capture, disable_custom_all_reduce, disable_async_output_proc, hf_overrides, mm_processor_kwargs, task, override_pooler_config, compilation_config, **kwargs)
228
229 # TODO(rob): enable mp by default (issue with fork vs spawn)
--> 230 self.llm_engine = self.engine_class.from_engine_args(
231 engine_args, usage_context=UsageContext.LLM_CLASS)
232
/usr/local/lib/python3.10/dist-packages/vllm/engine/llm_engine.py in from_engine_args(cls, engine_args, usage_context, stat_loggers)
515 executor_class = cls._get_executor_cls(engine_config)
516 # Create the LLM engine.
--> 517 engine = cls(
518 vllm_config=engine_config,
519 executor_class=executor_class,
/usr/local/lib/python3.10/dist-packages/vllm/engine/llm_engine.py in __init__(self, vllm_config, executor_class, log_stats, usage_context, stat_loggers, input_registry, mm_registry, use_cached_outputs)
271 self.model_config)
272
--> 273 self.model_executor = executor_class(vllm_config=vllm_config, )
274
275 if self.model_config.runner_type != "pooling":
/usr/local/lib/python3.10/dist-packages/vllm/executor/distributed_gpu_executor.py in __init__(self, *args, **kwargs)
24 self.extra_execute_model_run_workers_kwargs: Dict[str, Any] = {}
25
---> 26 super().__init__(*args, **kwargs)
27
28 def determine_num_available_blocks(self) -> Tuple[int, int]:
/usr/local/lib/python3.10/dist-packages/vllm/executor/executor_base.py in __init__(self, vllm_config)
34 self.prompt_adapter_config = vllm_config.prompt_adapter_config
35 self.observability_config = vllm_config.observability_config
---> 36 self._init_executor()
37
38 @abstractmethod
/usr/local/lib/python3.10/dist-packages/vllm/executor/multiproc_gpu_executor.py in _init_executor(self)
81 distributed_init_method=distributed_init_method)
82 self._run_workers("init_device")
---> 83 self._run_workers("load_model",
84 max_concurrent_workers=self.parallel_config.
85 max_parallel_loading_workers)
/usr/local/lib/python3.10/dist-packages/vllm/executor/multiproc_gpu_executor.py in _run_workers(self, method, async_run_tensor_parallel_workers_only, max_concurrent_workers, *args, **kwargs)
155
156 driver_worker_method = getattr(self.driver_worker, method)
--> 157 driver_worker_output = driver_worker_method(*args, **kwargs)
158
159 # Get the results of the workers.
/usr/local/lib/python3.10/dist-packages/vllm/worker/worker.py in load_model(self)
153
154 def load_model(self):
--> 155 self.model_runner.load_model()
156
157 def save_sharded_state(
/usr/local/lib/python3.10/dist-packages/vllm/worker/model_runner.py in load_model(self)
1094 logger.info("Starting to load model %s...", self.model_config.model)
1095 with DeviceMemoryProfiler() as m:
-> 1096 self.model = get_model(vllm_config=self.vllm_config)
1097
1098 self.model_memory_usage = m.consumed_memory
/usr/local/lib/python3.10/dist-packages/vllm/model_executor/model_loader/__init__.py in get_model(vllm_config)
10 def get_model(*, vllm_config: VllmConfig) -> nn.Module:
11 loader = get_model_loader(vllm_config.load_config)
---> 12 return loader.load_model(vllm_config=vllm_config)
13
14
/usr/local/lib/python3.10/dist-packages/vllm/model_executor/model_loader/loader.py in load_model(self, vllm_config)
361 with set_default_torch_dtype(model_config.dtype):
362 with target_device:
--> 363 model = _initialize_model(vllm_config=vllm_config)
364
365 weights_to_load = {name for name, _ in model.named_parameters()}
/usr/local/lib/python3.10/dist-packages/vllm/model_executor/model_loader/loader.py in _initialize_model(vllm_config, prefix)
114 # new-style model class
115 with set_current_vllm_config(vllm_config):
--> 116 return model_class(vllm_config=vllm_config, prefix=prefix)
117
118 msg = ("vLLM model class should accept `vllm_config` and `prefix` as "
/usr/local/lib/python3.10/dist-packages/vllm/model_executor/models/adapters.py in __init__(self, vllm_config, prefix, **kwargs)
188 quant_config = vllm_config.quant_config
189
--> 190 self.score = RowParallelLinear(config.hidden_size,
191 config.num_labels,
192 quant_config=quant_config,
/usr/local/lib/python3.10/dist-packages/vllm/model_executor/layers/linear.py in __init__(self, input_size, output_size, bias, input_is_parallel, skip_bias_add, params_dtype, reduce_results, quant_config, prefix)
1028 assert self.quant_method is not None
1029
-> 1030 self.quant_method.create_weights(
1031 layer=self,
1032 input_size_per_partition=self.input_size_per_partition,
/usr/local/lib/python3.10/dist-packages/vllm/model_executor/layers/quantization/awq.py in create_weights(self, layer, input_size_per_partition, output_partition_sizes, input_size, output_size, params_dtype, **extra_weight_attrs)
106 output_size_per_partition = sum(output_partition_sizes)
107 if output_size_per_partition % self.quant_config.pack_factor != 0:
--> 108 raise ValueError(
109 "The output size is not aligned with the quantized "
110 "weight shape. This can be caused by too large "
ValueError: The output size is not aligned with the quantized weight shape. This can be caused by too large tensor parallel size.
Before submitting a new issueโฆ
- Make sure you already searched for relevant issues, and asked the chatbot living at the bottom right corner of the documentation page, which can answer lots of frequently asked questions.
About this issue
- Original URL
- State: open
- Created 2 years ago
- Comments: 15 (3 by maintainers)
Will be the official qwen2.5-32 b - awqs use AutoModelForSequenceClassification.from_pretrained save, use VLLM same mistake again
It should be that vllm does not support the cls model in awq format