frozenleaves/vllm
1
0
Fork 0
mirror of https://github.com/vllm-project/vllm.git synced 2025-10-20 14:53:52 +08:00
Code Issues Packages Projects Releases Wiki Activity

Only turn on FastIncrementalDetokenizer when tokenizers >= 0.21.1 (#17158)

Browse Source
This commit is contained in:
Lu Fang
2025-04-25 02:10:32 -07:00
committed by GitHub
parent ef19e67d2c
commit fc966e9cc6
1 changed files with 5 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
vllm/v1/engine/detokenizer.py
View File

@ -2,6 +2,8 @@
from abc import ABC, abstractmethod
from typing import Optional
import tokenizers
from packaging import version
from tokenizers import Tokenizer
from tokenizers.decoders import DecodeStream
from transformers import PreTrainedTokenizerFast
@ -43,8 +45,10 @@ class IncrementalDetokenizer:
# No tokenizer => skipping detokenization.
return IncrementalDetokenizer()
if isinstance(tokenizer, PreTrainedTokenizerFast):
if (isinstance(tokenizer, PreTrainedTokenizerFast) and version.parse(
tokenizers.__version__) >= version.parse("0.21.1")):
# Fast tokenizer => use tokenizers library DecodeStream.
# And only tokenizers >= 0.21.1 supports Fast Detokenizer.
return FastIncrementalDetokenizer(tokenizer, request)
# Fall back to slow python-based incremental detokenization.