frozenleaves/vllm
1
0
Fork 0
mirror of https://github.com/vllm-project/vllm.git synced 2025-10-20 23:03:52 +08:00
Code Issues Packages Projects Releases Wiki Activity

[Bugfix] Include encoder prompts len to non-stream api usage response (#8861)

Browse Source
This commit is contained in:
Pernekhan Utemuratov
2024-09-26 15:47:00 -07:00
committed by GitHub
parent d9cfbc891e
commit 93d364da34
1 changed files with 2 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
vllm/entrypoints/openai/serving_chat.py
View File

@ -726,6 +726,8 @@ class OpenAIServingChat(OpenAIServing):
assert final_res.prompt_token_ids is not None assert final_res.prompt_token_ids is not None
num_prompt_tokens = len(final_res.prompt_token_ids) num_prompt_tokens = len(final_res.prompt_token_ids)
if final_res.encoder_prompt_token_ids is not None:
num_prompt_tokens += len(final_res.encoder_prompt_token_ids)
num_generated_tokens = sum( num_generated_tokens = sum(
len(output.token_ids) for output in final_res.outputs) len(output.token_ids) for output in final_res.outputs)
usage = UsageInfo( usage = UsageInfo(