frozenleaves/vllm
1
0
Fork 0
mirror of https://github.com/vllm-project/vllm.git synced 2025-10-20 14:53:52 +08:00
Code Issues Packages Projects Releases Wiki Activity

[V1]Enable APC by default only for text models (#10148)

Browse Source 
Signed-off-by: Roger Wang <ywang@roblox.com>
This commit is contained in:
Roger Wang
2024-11-08 06:39:41 -08:00
committed by GitHub
parent 1ff4aed5bd
commit 208ce622c7
1 changed files with 4 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
vllm/v1/engine/llm_engine.py
View File

@ -65,7 +65,10 @@ class LLMEngine:
elif usage_context == UsageContext.OPENAI_API_SERVER:
scheduler_config.max_num_seqs = 1024
scheduler_config.max_num_batched_tokens = 2048
cache_config.enable_prefix_caching = True
# TODO (ywang96): Enable APC by default when VLM supports it.
if not model_config.is_multimodal_model:
cache_config.enable_prefix_caching = True
logger.info(
"Initializing an LLM engine (v%s) with config: "