Compare commits

..

1 Commits

Author SHA1 Message Date
7d9eadf97d Cleanup reference to TFBertTokenizer 2025-11-14 15:10:29 +00:00
3 changed files with 3 additions and 6 deletions

View File

@ -74,15 +74,14 @@ messages = [
}
]
processed = tokenizer.apply_chat_template(
input_ids = tokenizer.apply_chat_template(
messages,
add_generation_prompt=True,
tokenize=True,
return_tensors="pt"
)
input_ids = processed["input_ids"].to(model.device)
).to(model.device)
outputs = model.generate(input_ids, max_new_tokens=1024)[0, input_ids.shape[1] :]
outputs = model.generate(input_ids, max_new_tokens=1024)[0, input_ids.shape[1]:]
out_text = tokenizer.decode(outputs)
parsed = tokenizer.parse_response(out_text)
print(parsed.keys())

View File

@ -22,7 +22,6 @@ if TYPE_CHECKING:
from .modeling_bert import *
from .tokenization_bert import *
from .tokenization_bert_fast import *
from .tokenization_bert_tf import *
else:
import sys

View File

@ -111,7 +111,6 @@ OBJECTS_TO_IGNORE = {
"InputFeatures",
# Signature is *args/**kwargs
"TFSequenceSummary",
"TFBertTokenizer",
"TFGPT2Tokenizer",
# Missing arguments in the docstring
"ASTFeatureExtractor",