frozenleaves/vllm
1
0
Fork 0
mirror of https://github.com/vllm-project/vllm.git synced 2025-10-20 23:03:52 +08:00
Code Issues Packages Projects Releases Wiki Activity

[Bugfix] Set enable_prefix_caching=True in prefix caching example (#3703)

Browse Source
This commit is contained in:
Woosuk Kwon
2024-03-28 16:26:30 -07:00
committed by GitHub
parent cb40b3ab6b
commit c0935c96d3
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
examples/offline_inference_with_prefix.py
View File

@ -22,7 +22,7 @@ prompts = [
sampling_params = SamplingParams(temperature=0.0) sampling_params = SamplingParams(temperature=0.0)
# Create an LLM. # Create an LLM.
llm = LLM(model="facebook/opt-125m") llm = LLM(model="facebook/opt-125m", enable_prefix_caching=True)
generating_prompts = [prefix + prompt for prompt in prompts] generating_prompts = [prefix + prompt for prompt in prompts]