Compare commits

...

1 Commits

Author SHA1 Message Date
6cfb1ee448 handle when from_pretrained_id is a list 2024-06-21 19:59:28 +02:00

View File

@ -1580,17 +1580,15 @@ class TokenizerTesterMixin:
self.assertEqual(overflowing_tokens, seq1_tokens[-(2 + stride) :])
# TODO: FIXME @ArthurZucker
@unittest.skip(
reason="start to fail after # 29473. See https://github.com/huggingface/transformers/pull/29473#pullrequestreview-1945687810"
)
@slow
@require_read_token
def test_encode_decode_fast_slow_all_tokens(self):
if self.rust_tokenizer_class is not None:
pretrained_name = self.from_pretrained_id
slow_tokenizer = self.tokenizer_class.from_pretrained(pretrained_name, legacy=False)
pretrained_names = self.from_pretrained_id if isinstance(self.from_pretrained_id, list) else [self.from_pretrained_id]
for pretrained_name in pretrained_names:
with self.subTest(f"{pretrained_name}"):
slow_tokenizer = self.tokenizer_class.from_pretrained(pretrained_name, legacy=False)
rust_tokenizer = self.rust_tokenizer_class.from_pretrained(
pretrained_name, from_slow=True, legacy=False
)