mirror of
https://github.com/vllm-project/vllm.git
synced 2025-10-20 14:53:52 +08:00
[Fix] Fix quantization="gptq" when using Marlin (#3319)
Co-authored-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
This commit is contained in:
@ -168,13 +168,18 @@ class ModelConfig:
|
||||
# Parse quantization method from the HF model config, if available.
|
||||
hf_quant_config = getattr(self.hf_config, "quantization_config", None)
|
||||
if hf_quant_config is not None:
|
||||
|
||||
hf_quant_method = str(hf_quant_config["quant_method"]).lower()
|
||||
|
||||
# If the GPTQ model is serialized in marlin format, use marlin.
|
||||
if (hf_quant_method == "gptq"
|
||||
and "is_marlin_format" in hf_quant_config
|
||||
and hf_quant_config["is_marlin_format"]):
|
||||
logger.info("The model is serialized in Marlin format. "
|
||||
"Using Marlin kernel.")
|
||||
hf_quant_method = "marlin"
|
||||
if self.quantization == "gptq":
|
||||
self.quantization = hf_quant_method
|
||||
|
||||
if self.quantization is None:
|
||||
self.quantization = hf_quant_method
|
||||
elif self.quantization != hf_quant_method:
|
||||
|
Reference in New Issue
Block a user