vllm: [Bug]: The output size is not aligned with the quantized weight shape. This can be caused by too large tensor parallel size.

Your current environment

The output of `python collect_env.py`
Your output of `python collect_env.py` here

Model Input Dumps

No response

๐Ÿ› Describe the bug

I use the AWQ and GPTQ methods to quantify qwen2.5-32b for Qwen2ForSequenceClassification. And use vLLM-0.6.6 infer them. It should work in theory, but they donโ€™t seem to support it.

group_size=128/64/32

from vllm import LLM
os.environ["CUDA_VISIBLE_DEVICES"] = "0,1"
llm = LLM(
    model=qwen2.2-32b-cls-awq,
    task="classify",
    dtype="half",
    tensor_parallel_size=2,
    gpu_memory_utilization=0.98,
    trust_remote_code=True,
    max_model_len=1024,
    enforce_eager=True,
)
tokenizer = llm.get_tokenizer()
{
  "_name_or_path": "qwen2.5_32b_cls2",
  "architectures": [
    "Qwen2ForSequenceClassification"
  ],
  "attention_dropout": 0.0,
  "bos_token_id": 151643,
  "eos_token_id": 151645,
  "hidden_act": "silu",
  "hidden_size": 5120,
  "initializer_range": 0.02,
  "intermediate_size": 27648,
  "max_position_embeddings": 32768,
  "max_window_layers": 70,
  "model_type": "qwen2",
  "num_attention_heads": 40,
  "num_hidden_layers": 64,
  "num_key_value_heads": 8,
  "quantization_config": {
    "bits": 4,
    "group_size": 128,
    "modules_to_not_convert": null,
    "quant_method": "awq",
    "version": "gemm",
    "zero_point": true
  },
  "rms_norm_eps": 1e-06,
  "rope_scaling": null,
  "rope_theta": 1000000.0,
  "sliding_window": null,
  "tie_word_embeddings": false,
  "torch_dtype": "bfloat16",
  "transformers_version": "4.46.3",
  "use_cache": false,
  "use_sliding_window": false,
  "vocab_size": 152064
}

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-7-92d2819a49ae> in <cell line: 4>()
      2 os.environ["CUDA_VISIBLE_DEVICES"] = "0,1"
      3 
----> 4 llm = LLM(
      5     model=MODEL_NAME,
      6     task="classify",

/usr/local/lib/python3.10/dist-packages/vllm/utils.py in inner(*args, **kwargs)
    984                     )
    985 
--> 986             return fn(*args, **kwargs)
    987 
    988         return inner  # type: ignore

/usr/local/lib/python3.10/dist-packages/vllm/entrypoints/llm.py in __init__(self, model, tokenizer, tokenizer_mode, skip_tokenizer_init, trust_remote_code, allowed_local_media_path, tensor_parallel_size, dtype, quantization, revision, tokenizer_revision, seed, gpu_memory_utilization, swap_space, cpu_offload_gb, enforce_eager, max_seq_len_to_capture, disable_custom_all_reduce, disable_async_output_proc, hf_overrides, mm_processor_kwargs, task, override_pooler_config, compilation_config, **kwargs)
    228 
    229         # TODO(rob): enable mp by default (issue with fork vs spawn)
--> 230         self.llm_engine = self.engine_class.from_engine_args(
    231             engine_args, usage_context=UsageContext.LLM_CLASS)
    232 

/usr/local/lib/python3.10/dist-packages/vllm/engine/llm_engine.py in from_engine_args(cls, engine_args, usage_context, stat_loggers)
    515         executor_class = cls._get_executor_cls(engine_config)
    516         # Create the LLM engine.
--> 517         engine = cls(
    518             vllm_config=engine_config,
    519             executor_class=executor_class,

/usr/local/lib/python3.10/dist-packages/vllm/engine/llm_engine.py in __init__(self, vllm_config, executor_class, log_stats, usage_context, stat_loggers, input_registry, mm_registry, use_cached_outputs)
    271             self.model_config)
    272 
--> 273         self.model_executor = executor_class(vllm_config=vllm_config, )
    274 
    275         if self.model_config.runner_type != "pooling":

/usr/local/lib/python3.10/dist-packages/vllm/executor/distributed_gpu_executor.py in __init__(self, *args, **kwargs)
     24         self.extra_execute_model_run_workers_kwargs: Dict[str, Any] = {}
     25 
---> 26         super().__init__(*args, **kwargs)
     27 
     28     def determine_num_available_blocks(self) -> Tuple[int, int]:

/usr/local/lib/python3.10/dist-packages/vllm/executor/executor_base.py in __init__(self, vllm_config)
     34         self.prompt_adapter_config = vllm_config.prompt_adapter_config
     35         self.observability_config = vllm_config.observability_config
---> 36         self._init_executor()
     37 
     38     @abstractmethod

/usr/local/lib/python3.10/dist-packages/vllm/executor/multiproc_gpu_executor.py in _init_executor(self)
     81             distributed_init_method=distributed_init_method)
     82         self._run_workers("init_device")
---> 83         self._run_workers("load_model",
     84                           max_concurrent_workers=self.parallel_config.
     85                           max_parallel_loading_workers)

/usr/local/lib/python3.10/dist-packages/vllm/executor/multiproc_gpu_executor.py in _run_workers(self, method, async_run_tensor_parallel_workers_only, max_concurrent_workers, *args, **kwargs)
    155 
    156         driver_worker_method = getattr(self.driver_worker, method)
--> 157         driver_worker_output = driver_worker_method(*args, **kwargs)
    158 
    159         # Get the results of the workers.

/usr/local/lib/python3.10/dist-packages/vllm/worker/worker.py in load_model(self)
    153 
    154     def load_model(self):
--> 155         self.model_runner.load_model()
    156 
    157     def save_sharded_state(

/usr/local/lib/python3.10/dist-packages/vllm/worker/model_runner.py in load_model(self)
   1094         logger.info("Starting to load model %s...", self.model_config.model)
   1095         with DeviceMemoryProfiler() as m:
-> 1096             self.model = get_model(vllm_config=self.vllm_config)
   1097 
   1098         self.model_memory_usage = m.consumed_memory

/usr/local/lib/python3.10/dist-packages/vllm/model_executor/model_loader/__init__.py in get_model(vllm_config)
     10 def get_model(*, vllm_config: VllmConfig) -> nn.Module:
     11     loader = get_model_loader(vllm_config.load_config)
---> 12     return loader.load_model(vllm_config=vllm_config)
     13 
     14 

/usr/local/lib/python3.10/dist-packages/vllm/model_executor/model_loader/loader.py in load_model(self, vllm_config)
    361         with set_default_torch_dtype(model_config.dtype):
    362             with target_device:
--> 363                 model = _initialize_model(vllm_config=vllm_config)
    364 
    365             weights_to_load = {name for name, _ in model.named_parameters()}

/usr/local/lib/python3.10/dist-packages/vllm/model_executor/model_loader/loader.py in _initialize_model(vllm_config, prefix)
    114         # new-style model class
    115         with set_current_vllm_config(vllm_config):
--> 116             return model_class(vllm_config=vllm_config, prefix=prefix)
    117 
    118     msg = ("vLLM model class should accept `vllm_config` and `prefix` as "

/usr/local/lib/python3.10/dist-packages/vllm/model_executor/models/adapters.py in __init__(self, vllm_config, prefix, **kwargs)
    188             quant_config = vllm_config.quant_config
    189 
--> 190             self.score = RowParallelLinear(config.hidden_size,
    191                                            config.num_labels,
    192                                            quant_config=quant_config,

/usr/local/lib/python3.10/dist-packages/vllm/model_executor/layers/linear.py in __init__(self, input_size, output_size, bias, input_is_parallel, skip_bias_add, params_dtype, reduce_results, quant_config, prefix)
   1028         assert self.quant_method is not None
   1029 
-> 1030         self.quant_method.create_weights(
   1031             layer=self,
   1032             input_size_per_partition=self.input_size_per_partition,

/usr/local/lib/python3.10/dist-packages/vllm/model_executor/layers/quantization/awq.py in create_weights(self, layer, input_size_per_partition, output_partition_sizes, input_size, output_size, params_dtype, **extra_weight_attrs)
    106         output_size_per_partition = sum(output_partition_sizes)
    107         if output_size_per_partition % self.quant_config.pack_factor != 0:
--> 108             raise ValueError(
    109                 "The output size is not aligned with the quantized "
    110                 "weight shape. This can be caused by too large "

ValueError: The output size is not aligned with the quantized weight shape. This can be caused by too large tensor parallel size.

Before submitting a new issueโ€ฆ

  • Make sure you already searched for relevant issues, and asked the chatbot living at the bottom right corner of the documentation page, which can answer lots of frequently asked questions.

About this issue

  • Original URL
  • State: open
  • Created 2 years ago
  • Comments: 15 (3 by maintainers)

Most upvoted comments

Will be the official qwen2.5-32 b - awqs use AutoModelForSequenceClassification.from_pretrained save, use VLLM same mistake again

It should be that vllm does not support the cls model in awq format

ValueError                                Traceback (most recent call last)
<ipython-input-4-c6ad43f0ccbc> in <cell line: 4>()
      2 os.environ["CUDA_VISIBLE_DEVICES"] = "0,1"
      3 
----> 4 llm = LLM(
      5     model=MODEL_NAME,
      6     task="classify",

/usr/local/lib/python3.10/dist-packages/vllm/utils.py in inner(*args, **kwargs)
    984                     )
    985 
--> 986             return fn(*args, **kwargs)
    987 
    988         return inner  # type: ignore

/usr/local/lib/python3.10/dist-packages/vllm/entrypoints/llm.py in __init__(self, model, tokenizer, tokenizer_mode, skip_tokenizer_init, trust_remote_code, allowed_local_media_path, tensor_parallel_size, dtype, quantization, revision, tokenizer_revision, seed, gpu_memory_utilization, swap_space, cpu_offload_gb, enforce_eager, max_seq_len_to_capture, disable_custom_all_reduce, disable_async_output_proc, hf_overrides, mm_processor_kwargs, task, override_pooler_config, compilation_config, **kwargs)
    228 
    229         # TODO(rob): enable mp by default (issue with fork vs spawn)
--> 230         self.llm_engine = self.engine_class.from_engine_args(
    231             engine_args, usage_context=UsageContext.LLM_CLASS)
    232 

/usr/local/lib/python3.10/dist-packages/vllm/engine/llm_engine.py in from_engine_args(cls, engine_args, usage_context, stat_loggers)
    515         executor_class = cls._get_executor_cls(engine_config)
    516         # Create the LLM engine.
--> 517         engine = cls(
    518             vllm_config=engine_config,
    519             executor_class=executor_class,

/usr/local/lib/python3.10/dist-packages/vllm/engine/llm_engine.py in __init__(self, vllm_config, executor_class, log_stats, usage_context, stat_loggers, input_registry, mm_registry, use_cached_outputs)
    271             self.model_config)
    272 
--> 273         self.model_executor = executor_class(vllm_config=vllm_config, )
    274 
    275         if self.model_config.runner_type != "pooling":

/usr/local/lib/python3.10/dist-packages/vllm/executor/distributed_gpu_executor.py in __init__(self, *args, **kwargs)
     24         self.extra_execute_model_run_workers_kwargs: Dict[str, Any] = {}
     25 
---> 26         super().__init__(*args, **kwargs)
     27 
     28     def determine_num_available_blocks(self) -> Tuple[int, int]:

/usr/local/lib/python3.10/dist-packages/vllm/executor/executor_base.py in __init__(self, vllm_config)
     34         self.prompt_adapter_config = vllm_config.prompt_adapter_config
     35         self.observability_config = vllm_config.observability_config
---> 36         self._init_executor()
     37 
     38     @abstractmethod

/usr/local/lib/python3.10/dist-packages/vllm/executor/multiproc_gpu_executor.py in _init_executor(self)
     81             distributed_init_method=distributed_init_method)
     82         self._run_workers("init_device")
---> 83         self._run_workers("load_model",
     84                           max_concurrent_workers=self.parallel_config.
     85                           max_parallel_loading_workers)

/usr/local/lib/python3.10/dist-packages/vllm/executor/multiproc_gpu_executor.py in _run_workers(self, method, async_run_tensor_parallel_workers_only, max_concurrent_workers, *args, **kwargs)
    155 
    156         driver_worker_method = getattr(self.driver_worker, method)
--> 157         driver_worker_output = driver_worker_method(*args, **kwargs)
    158 
    159         # Get the results of the workers.

/usr/local/lib/python3.10/dist-packages/vllm/worker/worker.py in load_model(self)
    153 
    154     def load_model(self):
--> 155         self.model_runner.load_model()
    156 
    157     def save_sharded_state(

/usr/local/lib/python3.10/dist-packages/vllm/worker/model_runner.py in load_model(self)
   1094         logger.info("Starting to load model %s...", self.model_config.model)
   1095         with DeviceMemoryProfiler() as m:
-> 1096             self.model = get_model(vllm_config=self.vllm_config)
   1097 
   1098         self.model_memory_usage = m.consumed_memory

/usr/local/lib/python3.10/dist-packages/vllm/model_executor/model_loader/__init__.py in get_model(vllm_config)
     10 def get_model(*, vllm_config: VllmConfig) -> nn.Module:
     11     loader = get_model_loader(vllm_config.load_config)
---> 12     return loader.load_model(vllm_config=vllm_config)
     13 
     14 

/usr/local/lib/python3.10/dist-packages/vllm/model_executor/model_loader/loader.py in load_model(self, vllm_config)
    361         with set_default_torch_dtype(model_config.dtype):
    362             with target_device:
--> 363                 model = _initialize_model(vllm_config=vllm_config)
    364 
    365             weights_to_load = {name for name, _ in model.named_parameters()}

/usr/local/lib/python3.10/dist-packages/vllm/model_executor/model_loader/loader.py in _initialize_model(vllm_config, prefix)
    114         # new-style model class
    115         with set_current_vllm_config(vllm_config):
--> 116             return model_class(vllm_config=vllm_config, prefix=prefix)
    117 
    118     msg = ("vLLM model class should accept `vllm_config` and `prefix` as "

/usr/local/lib/python3.10/dist-packages/vllm/model_executor/models/adapters.py in __init__(self, vllm_config, prefix, **kwargs)
    188             quant_config = vllm_config.quant_config
    189 
--> 190             self.score = RowParallelLinear(config.hidden_size,
    191                                            config.num_labels,
    192                                            quant_config=quant_config,

/usr/local/lib/python3.10/dist-packages/vllm/model_executor/layers/linear.py in __init__(self, input_size, output_size, bias, input_is_parallel, skip_bias_add, params_dtype, reduce_results, quant_config, prefix)
   1028         assert self.quant_method is not None
   1029 
-> 1030         self.quant_method.create_weights(
   1031             layer=self,
   1032             input_size_per_partition=self.input_size_per_partition,

/usr/local/lib/python3.10/dist-packages/vllm/model_executor/layers/quantization/awq.py in create_weights(self, layer, input_size_per_partition, output_partition_sizes, input_size, output_size, params_dtype, **extra_weight_attrs)
    106         output_size_per_partition = sum(output_partition_sizes)
    107         if output_size_per_partition % self.quant_config.pack_factor != 0:
--> 108             raise ValueError(
    109                 "The output size is not aligned with the quantized "
    110                 "weight shape. This can be caused by too large "

ValueError: The output size is not aligned with the quantized weight shape. This can be caused by too large tensor parallel size.