frozenleaves/vllm
1
0
Fork 0
mirror of https://github.com/vllm-project/vllm.git synced 2025-10-20 23:03:52 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix_tokenizer_snapshot_download_bug (#4493)

Browse Source
This commit is contained in:
fuchen.ljl
2024-05-01 07:38:50 +08:00
committed by GitHub
parent dd1a50a8bc
commit 6ad58f42c5
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
vllm/transformers_utils/tokenizer.py
View File

@ -79,7 +79,7 @@ def get_tokenizer(
revision=revision, revision=revision,
local_files_only=huggingface_hub.constants.HF_HUB_OFFLINE, local_files_only=huggingface_hub.constants.HF_HUB_OFFLINE,
# Ignore weights - we only need the tokenizer. # Ignore weights - we only need the tokenizer.
ignore_file_pattern=["*.pt", "*.safetensors", "*.bin"]) ignore_file_pattern=[".*.pt", ".*.safetensors", ".*.bin"])
tokenizer_name = tokenizer_path tokenizer_name = tokenizer_path
if tokenizer_mode == "slow": if tokenizer_mode == "slow":