nit

src/transformers/convert_slow_tokenizer.py

@@ -1316,7 +1316,7 @@ class GemmaConvert(SpmConverter):
         return vocab
 
     def pre_tokenizer(self, replacement, add_prefix_space):
-        return pre_tokenizers.Split(" ", "merged_with_previous")
+        return pre_tokenizers.Split(Regex('(?<!▁)▁'), "merged_with_next")
 
     def unk_id(self, proto):
         unk_id = 3

tests/models/gemma/test_tokenization_gemma.py

@@ -188,6 +188,16 @@ class GemmaIntegrationTest(unittest.TestCase):
             },
         )
 
+    @require_read_token
+    def test_word_ids(self):
+        fast_tokenizer = GemmaTokenizerFast.from_pretrained("google/gemma-2b-it", from_slow=True)
+        input_text = "                        rust                       . x.             .   .       "
+        EXPECTED_WORD_IDS = [None, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5]
+        EXPECTED_INPUT_IDS = [2, 161, 14783, 160, 235265, 1141, 235265, 150, 235265, 140, 235265, 144]
+        output = fast_tokenizer(input_text)
+        self.assertEqual(output["input_ids"], EXPECTED_INPUT_IDS)
+        self.assertEqual(output.word_ids(), EXPECTED_WORD_IDS)
+
     def test_user_added_tokens(self):
         # Ensure that user added tokens are not split in the fast tokenizer
         slow_tokenizer = self.tokenizer