frozenleaves/vllm
1
0
Fork 0
mirror of https://github.com/vllm-project/vllm.git synced 2025-10-20 14:53:52 +08:00
Code Issues Packages Projects Releases Wiki Activity

[Bugfix][Benchmarks] Fix a benchmark of deepspeed-mii backend to use api_key (#17291)

Browse Source 
Signed-off-by: Teruaki Ishizaki <teruaki.ishizaki@ntt.com>
This commit is contained in:
Teruaki Ishizaki
2025-05-23 13:30:47 +09:00
committed by GitHub
parent ed5d408255
commit 4be2255c81
1 changed files with 8 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
benchmarks/backend_request_func.py
View File

@ -194,6 +194,11 @@ async def async_request_deepspeed_mii(
request_func_input: RequestFuncInput,
pbar: Optional[tqdm] = None,
) -> RequestFuncOutput:
api_url = request_func_input.api_url
assert api_url.endswith(("completions", "profile")), (
"OpenAI Completions API URL must end with 'completions' or 'profile'."
)
async with aiohttp.ClientSession(
trust_env=True, timeout=AIOHTTP_TIMEOUT
) as session:
@ -204,6 +209,8 @@ async def async_request_deepspeed_mii(
"temperature": 0.01, # deepspeed-mii does not accept 0.0 temp.
"top_p": 1.0,
}
headers = {"Authorization": f"Bearer {os.environ.get('OPENAI_API_KEY')}"}
output = RequestFuncOutput()
output.prompt_len = request_func_input.prompt_len
@ -215,7 +222,7 @@ async def async_request_deepspeed_mii(
st = time.perf_counter()
try:
async with session.post(
url=request_func_input.api_url, json=payload
url=api_url, json=payload, headers=headers
) as response:
if response.status == 200:
parsed_resp = await response.json()