frozenleaves/vllm
1
0
Fork 0
mirror of https://github.com/vllm-project/vllm.git synced 2025-10-20 14:53:52 +08:00
Code Issues Packages Projects Releases Wiki Activity

Don't use cupy when enforce_eager=True (#3037)

Browse Source
This commit is contained in:
Roy
2024-02-27 09:33:38 +08:00
committed by GitHub
parent d9f726c4d0
commit c1c0d00b88
1 changed files with 4 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
vllm/engine/llm_engine.py
View File

@ -284,7 +284,10 @@ class LLMEngine:
is_driver_worker=True,
)
self._run_workers("init_model", cupy_port=get_open_port())
# don't use cupy for eager mode
self._run_workers("init_model",
cupy_port=get_open_port()
if not model_config.enforce_eager else None)
self._run_workers(
"load_model",
max_concurrent_workers=self.parallel_config.