frozenleaves/vllm
1
0
Fork 0
mirror of https://github.com/vllm-project/vllm.git synced 2025-10-20 23:03:52 +08:00
Code Issues Packages Projects Releases Wiki Activity

[Misc] Temporarily resolve the error of BitAndBytes (#7308)

Browse Source
This commit is contained in:
Jee Jee Li
2024-08-09 04:42:58 +08:00
committed by GitHub
parent 8334c39f37
commit a049b107e2
1 changed files with 3 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
vllm/config.py
View File

@ -322,8 +322,9 @@ class ModelConfig:
"BitAndBytes quantization with TP or PP is not supported yet.") "BitAndBytes quantization with TP or PP is not supported yet.")
if self.quantization == "bitsandbytes" and self.enforce_eager is False: if self.quantization == "bitsandbytes" and self.enforce_eager is False:
raise ValueError( logger.warning("CUDA graph is not supported on BitAndBytes yet, "
"BitAndBytes with enforce_eager = False is not supported yet.") "fallback to the eager mode.")
self.enforce_eager = True
def get_hf_config_sliding_window(self) -> Optional[int]: def get_hf_config_sliding_window(self) -> Optional[int]:
"""Get the sliding window size, or None if disabled.""" """Get the sliding window size, or None if disabled."""