Merge branch 'main' of github.com:huggingface/transformers into v5

.circleci/create_circleci_config.py

@@ -16,9 +16,10 @@
 import argparse
 import copy
 import os
+import random
 from dataclasses import dataclass
-from typing import Any, Optional
-
+from typing import Any, Dict, List, Optional
+import glob
 import yaml
 
 
@@ -81,15 +82,15 @@ class EmptyJob:
 @dataclass
 class CircleCIJob:
     name: str
-    additional_env: dict[str, Any] = None
-    docker_image: list[dict[str, str]] = None
-    install_steps: list[str] = None
+    additional_env: Dict[str, Any] = None
+    docker_image: List[Dict[str, str]] = None
+    install_steps: List[str] = None
     marker: Optional[str] = None
     parallelism: Optional[int] = 0
     pytest_num_workers: int = 8
-    pytest_options: dict[str, Any] = None
+    pytest_options: Dict[str, Any] = None
     resource_class: Optional[str] = "xlarge"
-    tests_to_run: Optional[list[str]] = None
+    tests_to_run: Optional[List[str]] = None
     num_test_files_per_worker: Optional[int] = 10
     # This should be only used for doctest job!
     command_timeout: Optional[int] = None
@@ -148,7 +149,7 @@ class CircleCIJob:
                 # Examples special case: we need to download NLTK files in advance to avoid cuncurrency issues
         timeout_cmd = f"timeout {self.command_timeout} " if self.command_timeout else ""
         marker_cmd = f"-m '{self.marker}'" if self.marker is not None else ""
-        junit_flags = " -p no:warning -o junit_family=xunit1 --junitxml=test-results/junit.xml"
+        junit_flags = f" -p no:warning -o junit_family=xunit1 --junitxml=test-results/junit.xml"
         joined_flaky_patterns = "|".join(FLAKY_TEST_FAILURE_PATTERNS)
         repeat_on_failure_flags = f"--reruns 5 --reruns-delay 2 --only-rerun '({joined_flaky_patterns})'"
         parallel = f' << pipeline.parameters.{self.job_name}_parallelism >> '
@@ -199,9 +200,9 @@ class CircleCIJob:
                         fi"""
                 },
             },
-            {"run": {"name": "Expand to show skipped tests", "when": "always", "command": "python3 .circleci/parse_test_outputs.py --file tests_output.txt --skip"}},
-            {"run": {"name": "Failed tests: show reasons",   "when": "always", "command": "python3 .circleci/parse_test_outputs.py --file tests_output.txt --fail"}},
-            {"run": {"name": "Errors",                       "when": "always", "command": "python3 .circleci/parse_test_outputs.py --file tests_output.txt --errors"}},
+            {"run": {"name": "Expand to show skipped tests", "when": "always", "command": f"python3 .circleci/parse_test_outputs.py --file tests_output.txt --skip"}},
+            {"run": {"name": "Failed tests: show reasons",   "when": "always", "command": f"python3 .circleci/parse_test_outputs.py --file tests_output.txt --fail"}},
+            {"run": {"name": "Errors",                       "when": "always", "command": f"python3 .circleci/parse_test_outputs.py --file tests_output.txt --errors"}},
             {"store_test_results": {"path": "test-results"}},
             {"store_artifacts": {"path": "test-results/junit.xml"}},
             {"store_artifacts": {"path": "reports"}},

.circleci/parse_test_outputs.py

@@ -1,6 +1,5 @@
-import argparse
 import re
-
+import argparse
 
 def parse_pytest_output(file_path):
     skipped_tests = {}

.github/ISSUE_TEMPLATE/bug-report.yml

@@ -36,23 +36,19 @@ body:
 
         Models:
 
-          - text models: @ArthurZucker @Cyrilvallez
-          - vision models: @yonigozlan @molbap
-          - audio models: @eustlb @ebezzam @vasqu
-          - multimodal models: @zucchini-nlp
+          - text models: @ArthurZucker
+          - vision models: @amyeroberts, @qubvel
+          - speech models: @eustlb
           - graph models: @clefourrier
 
         Library:
 
+          - flax: @gante and @Rocketknight1
           - generate: @zucchini-nlp (visual-language models) or @gante (all others)
-          - continuous batching: @remi-or @ArthurZucker @McPatate
           - pipelines: @Rocketknight1
+          - tensorflow: @gante and @Rocketknight1
           - tokenizers: @ArthurZucker and @itazap
           - trainer: @zach-huggingface @SunMarc
-          - attention: @vasqu @ArthurZucker @CyrilVallez
-          - model loading (from pretrained, etc): @CyrilVallez
-          - distributed: @3outeille @ArthurZucker @S1ro1
-          - CIs: @ydshieh
 
         Integrations:
 
@@ -60,7 +56,6 @@ body:
           - ray/raytune: @richardliaw, @amogkam
           - Big Model Inference: @SunMarc
           - quantization (bitsandbytes, autogpt): @SunMarc @MekkCyber
-          - kernels: @MekkCyber @drbh
         
         Devices/Backends:
         
@@ -74,6 +69,19 @@ body:
 
           - for issues with a model, report at https://discuss.huggingface.co/ and tag the model's creator.
 
+        HF projects:
+
+          - accelerate: [different repo](https://github.com/huggingface/accelerate)
+          - datasets: [different repo](https://github.com/huggingface/datasets)
+          - diffusers: [different repo](https://github.com/huggingface/diffusers)
+          - rust tokenizers: [different repo](https://github.com/huggingface/tokenizers)
+
+        Maintained examples (not research project or legacy):
+
+          - Flax: @Rocketknight1
+          - PyTorch: See Models above and tag the person corresponding to the modality of the example.
+          - TensorFlow: @Rocketknight1
+
         Research projects are not maintained and should be taken as is.
 
       placeholder: "@Username ..."

.github/scripts/assign_reviewers.py

@@ -13,16 +13,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import json
 import os
+import github
+import json
+from github import Github
 import re
 from collections import Counter
 from pathlib import Path
 
-import github
-from github import Github
-
-
 def pattern_to_regex(pattern):
     if pattern.startswith("/"):
         start_anchor = True

.github/workflows/benchmark_v2.yml

@@ -1,77 +0,0 @@
-name: Benchmark v2 Framework
-
-on:
-  workflow_call:
-    inputs:
-      runner:
-        description: 'GH Actions runner group to use'
-        required: true
-        type: string
-      commit_sha:
-        description: 'Commit SHA to benchmark'
-        required: false
-        type: string
-        default: ''
-      run_id:
-        description: 'Custom run ID for organizing results (auto-generated if not provided)'
-        required: false
-        type: string
-        default: ''
-      benchmark_repo_id:
-        description: 'HuggingFace Dataset to upload results to (e.g., "org/benchmark-results")'
-        required: false
-        type: string
-        default: ''
-
-env:
-  HF_HOME: /mnt/cache
-  TRANSFORMERS_IS_CI: yes
-  # For gated repositories, we still need to agree to share information on the Hub repo. page in order to get access.
-  # This token is created under the bot `hf-transformers-bot`.
-  HF_HUB_READ_TOKEN: ${{ secrets.HF_HUB_READ_TOKEN }}
-
-jobs:
-  benchmark-v2:
-    name: Benchmark v2
-    runs-on: ${{ inputs.runner }}
-    if: |
-      (github.event_name == 'pull_request' && contains( github.event.pull_request.labels.*.name, 'run-benchmark')) ||
-      (github.event_name == 'schedule')
-    container:
-      image: huggingface/transformers-pytorch-gpu
-      options: --gpus all --privileged --ipc host --shm-size "16gb"
-    steps:
-      - name: Get repo
-        uses: actions/checkout@v4
-        with:
-          ref: ${{ inputs.commit_sha || github.sha }}
-
-      - name: Install benchmark dependencies
-        run: |
-          python3 -m pip install -r benchmark_v2/requirements.txt
-
-      - name: Reinstall transformers in edit mode
-        run: |
-          python3 -m pip uninstall -y transformers
-          python3 -m pip install -e ".[torch]"
-
-      - name: Show installed libraries and their versions
-        run: |
-          python3 -m pip list
-          python3 -c "import torch; print(f'PyTorch version: {torch.__version__}')"
-          python3 -c "import torch; print(f'CUDA available: {torch.cuda.is_available()}')"
-          python3 -c "import torch; print(f'CUDA device count: {torch.cuda.device_count()}')" || true
-          nvidia-smi || true
-
-      - name: Run benchmark v2
-        working-directory: benchmark_v2
-        run: |
-          echo "Running benchmarks"
-          python3 run_benchmarks.py \
-          --commit-id '${{ inputs.commit_sha || github.sha }}' \
-          --run-id '${{ inputs.run_id }}' \
-          --push-to-hub '${{ inputs.benchmark_repo_id}}' \
-          --token '${{ secrets.TRANSFORMERS_CI_RESULTS_UPLOAD_TOKEN }}' \
-          --log-level INFO
-        env:
-          HF_TOKEN: ${{ secrets.HF_HUB_READ_TOKEN }}

.github/workflows/benchmark_v2_a10_caller.yml

@@ -1,19 +0,0 @@
-name: Benchmark v2 Scheduled Runner - A10 Single-GPU
-
-on:
-  schedule:
-    # Run daily at 16:30 UTC
-    - cron: "30 16 * * *"
-  pull_request:
-    types: [ opened, labeled, reopened, synchronize ]
-
-jobs:
-  benchmark-v2-default:
-    name: Benchmark v2 - Default Models
-    uses: ./.github/workflows/benchmark_v2.yml
-    with:
-      runner: aws-g5-4xlarge-cache-use1-public-80
-      commit_sha: ${{ github.sha }}
-      run_id: ${{ github.run_id }}
-      benchmark_repo_id: hf-internal-testing/transformers-daily-benchmarks
-    secrets: inherit

.github/workflows/benchmark_v2_mi325_caller.yml

@@ -1,19 +0,0 @@
-name: Benchmark v2 Scheduled Runner - MI325 Single-GPU
-
-on:
-  schedule:
-    # Run daily at 16:30 UTC
-    - cron: "30 16 * * *"
-  pull_request:
-    types: [ opened, labeled, reopened, synchronize ]
-
-jobs:
-  benchmark-v2-default:
-    name: Benchmark v2 - Default Models
-    uses: ./.github/workflows/benchmark_v2.yml
-    with:
-      runner: amd-mi325-ci-1gpu
-      commit_sha: ${{ github.sha }}
-      run_id: ${{ github.run_id }}
-      benchmark_repo_id: hf-internal-testing/transformers-daily-benchmarks
-    secrets: inherit

.github/workflows/build_documentation.yml

@@ -16,7 +16,7 @@ jobs:
       commit_sha: ${{ github.sha }}
       package: transformers
       notebook_folder: transformers_doc
-      languages: ar de en es fr hi it ja ko pt zh
+      languages: ar de en es fr hi it ko pt tr zh ja te
       custom_container: huggingface/transformers-doc-builder
     secrets:
       token: ${{ secrets.HUGGINGFACE_PUSH }}

.github/workflows/model_jobs.yml

@@ -12,6 +12,9 @@ on:
       slice_id:
         required: true
         type: number
+      runner_map:
+        required: false
+        type: string
       docker:
         required: true
         type: string
@@ -51,12 +54,10 @@ jobs:
       matrix:
         folders: ${{ fromJson(inputs.folder_slices)[inputs.slice_id] }}
     runs-on:
-      group: '${{ inputs.machine_type }}'
+      group: ${{ fromJson(inputs.runner_map)[matrix.folders][inputs.machine_type] }}
     container:
       image: ${{ inputs.docker }}
       options: --gpus all --shm-size "16gb" --ipc host -v /mnt/cache/.cache/huggingface:/mnt/cache/
-    outputs:
-      machine_type: ${{ steps.set_machine_type.outputs.machine_type }}
     steps:
       - name: Echo input and matrix info
         shell: bash
@@ -110,7 +111,6 @@ jobs:
         run: pip freeze
 
       - name: Set `machine_type` for report and artifact names
-        id: set_machine_type
         working-directory: /transformers
         shell: bash
         run: |
@@ -126,49 +126,29 @@ jobs:
 
           echo "$machine_type"
           echo "machine_type=$machine_type" >> $GITHUB_ENV
-          echo "machine_type=$machine_type" >> $GITHUB_OUTPUT
-
-      - name: Create report directory if it doesn't exist
-        shell: bash
-        run: |
-          mkdir -p /transformers/reports/${{ env.machine_type }}_${{ inputs.report_name_prefix }}_${{ env.matrix_folders }}_test_reports
-          echo "dummy" > /transformers/reports/${{ env.machine_type }}_${{ inputs.report_name_prefix }}_${{ env.matrix_folders }}_test_reports/dummy.txt
-          ls -la /transformers/reports/${{ env.machine_type }}_${{ inputs.report_name_prefix }}_${{ env.matrix_folders }}_test_reports
 
       - name: Run all tests on GPU
         working-directory: /transformers
-        run: |
-          script -q -c "PATCH_TESTING_METHODS_TO_COLLECT_OUTPUTS=yes _PATCHED_TESTING_METHODS_OUTPUT_DIR=/transformers/reports/${{ env.machine_type }}_${{ inputs.report_name_prefix }}_${{ env.matrix_folders }}_test_reports python3 -m pytest -rsfE -v --make-reports=${{ env.machine_type }}_${{ inputs.report_name_prefix }}_${{ env.matrix_folders }}_test_reports tests/${{ matrix.folders }}" test_outputs.txt
-          ls -la
-          # Extract the exit code from the output file
-          EXIT_CODE=$(tail -1 test_outputs.txt | grep -o 'COMMAND_EXIT_CODE="[0-9]*"' | cut -d'"' -f2)
-          exit ${EXIT_CODE:-1}
+        run: python3 -m pytest -rsfE -v --make-reports=${{ env.machine_type }}_${{ inputs.report_name_prefix }}_${{ matrix.folders }}_test_reports tests/${{ matrix.folders }}
 
       - name: Failure short reports
         if: ${{ failure() }}
-        # This step is only to show information on Github Actions log.
-        # Always mark this step as successful, even if the report directory or the file `failures_short.txt` in it doesn't exist
         continue-on-error: true
-        run: cat /transformers/reports/${{ env.machine_type }}_${{ inputs.report_name_prefix }}_${{ env.matrix_folders }}_test_reports/failures_short.txt
+        run: cat /transformers/reports/${{ env.machine_type }}_${{ inputs.report_name_prefix }}_${{ matrix.folders }}_test_reports/failures_short.txt
 
-      - name: Captured information
-        if: ${{ failure() }}
-        continue-on-error: true
+      - name: Run test
+        shell: bash
         run: |
-          cat /transformers/reports/${{ env.machine_type }}_${{ inputs.report_name_prefix }}_${{ env.matrix_folders }}_test_reports/captured_info.txt
-
-      - name: Copy test_outputs.txt
-        if: ${{ always() }}
-        continue-on-error: true
-        run: |
-          cp /transformers/test_outputs.txt /transformers/reports/${{ env.machine_type }}_${{ inputs.report_name_prefix }}_${{ env.matrix_folders }}_test_reports
+          mkdir -p /transformers/reports/${{ env.machine_type }}_${{ inputs.report_name_prefix }}_${{ matrix.folders }}_test_reports
+          echo "hello" > /transformers/reports/${{ env.machine_type }}_${{ inputs.report_name_prefix }}_${{ matrix.folders }}_test_reports/hello.txt
+          echo "${{ env.machine_type }}_${{ inputs.report_name_prefix }}_${{ matrix.folders }}_test_reports"
 
       - name: "Test suite reports artifacts: ${{ env.machine_type }}_${{ inputs.report_name_prefix }}_${{ env.matrix_folders }}_test_reports"
         if: ${{ always() }}
         uses: actions/upload-artifact@v4
         with:
           name: ${{ env.machine_type }}_${{ inputs.report_name_prefix }}_${{ env.matrix_folders }}_test_reports
-          path: /transformers/reports/${{ env.machine_type }}_${{ inputs.report_name_prefix }}_${{ env.matrix_folders }}_test_reports
+          path: /transformers/reports/${{ env.machine_type }}_${{ inputs.report_name_prefix }}_${{ matrix.folders }}_test_reports
 
   collated_reports:
     name: Collated Reports
@@ -179,5 +159,5 @@ jobs:
       job: run_models_gpu
       report_repo_id: ${{ inputs.report_repo_id }}
       gpu_name: ${{ inputs.runner_type }}
-      machine_type: ${{ needs.run_models_gpu.outputs.machine_type }}
+      machine_type: ${{ inputs.machine_type }}
     secrets: inherit

.github/workflows/pr_build_doc_with_comment.yml

@@ -14,7 +14,7 @@ permissions: {}
 jobs:
   get-pr-number:
     name: Get PR number
-    if: ${{ github.event.issue.state == 'open' && contains(fromJSON('["ydshieh", "ArthurZucker", "zucchini-nlp", "molbap", "gante", "LysandreJik", "Cyrilvallez", "Rocketknight1", "SunMarc", "muellerzr", "eustlb", "MekkCyber", "manueldeprada", "vasqu", "ivarflakstad", "stevhliu", "ebezzam", "itazap"]'), github.actor) && (startsWith(github.event.comment.body, 'build-doc')) }}
+    if: ${{ github.event.issue.state == 'open' && contains(fromJSON('["ydshieh", "ArthurZucker", "zucchini-nlp", "qubvel", "molbap", "gante", "LysandreJik", "Cyrilvallez", "Rocketknight1", "SunMarc", "muellerzr", "eustlb", "MekkCyber", "manueldeprada", "vasqu", "ivarflakstad", "stevhliu", "ebezzam"]'), github.actor) && (startsWith(github.event.comment.body, 'build-doc')) }}
     uses: ./.github/workflows/get-pr-number.yml
 
   get-pr-info:

.github/workflows/self-comment-ci.yml

@@ -29,7 +29,7 @@ jobs:
     runs-on: ubuntu-22.04
     name: Get PR number
     # For security: only allow team members to run
-    if: ${{ github.event.issue.state == 'open' && contains(fromJSON('["ydshieh", "ArthurZucker", "zucchini-nlp", "molbap", "gante", "LysandreJik", "Cyrilvallez", "Rocketknight1", "SunMarc", "muellerzr", "eustlb", "MekkCyber", "manueldeprada", "vasqu", "ivarflakstad", "stevhliu", "ebezzam", "remi-or", "itazap"]'), github.actor) && (startsWith(github.event.comment.body, 'run-slow') || startsWith(github.event.comment.body, 'run slow') || startsWith(github.event.comment.body, 'run_slow')) }}
+    if: ${{ github.event.issue.state == 'open' && contains(fromJSON('["ydshieh", "ArthurZucker", "zucchini-nlp", "qubvel", "molbap", "gante", "LysandreJik", "Cyrilvallez", "Rocketknight1", "SunMarc", "muellerzr", "eustlb", "MekkCyber", "manueldeprada", "vasqu", "ivarflakstad", "stevhliu", "ebezzam", "remi-or"]'), github.actor) && (startsWith(github.event.comment.body, 'run-slow') || startsWith(github.event.comment.body, 'run slow') || startsWith(github.event.comment.body, 'run_slow')) }}
     outputs:
       PR_NUMBER: ${{ steps.set_pr_number.outputs.PR_NUMBER }}
     steps:

.github/workflows/self-scheduled-amd-mi355-caller.yml

@@ -23,7 +23,7 @@ jobs:
       runner_scale_set: amd-mi355-ci
       docker: huggingface/testing-rocm7.0-preview
       ci_event: Scheduled CI (AMD) - mi355
-      report_repo_id: hf-transformers-bot/transformers-ci-dummy
+      report_repo_id: optimum-amd/transformers_daily_ci
     secrets: inherit
 
   torch-pipeline:
@@ -35,7 +35,7 @@ jobs:
       runner_scale_set: amd-mi355-ci
       docker: huggingface/testing-rocm7.0-preview
       ci_event: Scheduled CI (AMD) - mi355
-      report_repo_id: hf-transformers-bot/transformers-ci-dummy
+      report_repo_id: optimum-amd/transformers_daily_ci
     secrets: inherit
 
   example-ci:
@@ -47,7 +47,7 @@ jobs:
       runner_scale_set: amd-mi355-ci
       docker: huggingface/testing-rocm7.0-preview
       ci_event: Scheduled CI (AMD) - mi355
-      report_repo_id: hf-transformers-bot/transformers-ci-dummy
+      report_repo_id: optimum-amd/transformers_daily_ci
     secrets: inherit
 
   deepspeed-ci:
@@ -59,5 +59,5 @@ jobs:
       runner_scale_set: amd-mi355-ci
       docker: huggingface/testing-rocm7.0-preview
       ci_event: Scheduled CI (AMD) - mi355
-      report_repo_id: hf-transformers-bot/transformers-ci-dummy
+      report_repo_id: optimum-amd/transformers_daily_ci
     secrets: inherit

.github/workflows/self-scheduled-caller.yml

@@ -88,7 +88,6 @@ jobs:
       job: run_trainer_and_fsdp_gpu
       slack_report_channel: "#transformers-ci-daily-training"
       docker: huggingface/transformers-all-latest-gpu
-      runner_type: "a10"
       ci_event: Daily CI
       report_repo_id: hf-internal-testing/transformers_daily_ci
       commit_sha: ${{ github.sha }}

.github/workflows/self-scheduled.yml

@@ -68,6 +68,7 @@ jobs:
     outputs:
       folder_slices: ${{ steps.set-matrix.outputs.folder_slices }}
       slice_ids: ${{ steps.set-matrix.outputs.slice_ids }}
+      runner_map: ${{ steps.set-matrix.outputs.runner_map }}
       quantization_matrix: ${{ steps.set-matrix-quantization.outputs.quantization_matrix }}
     steps:
       - name: Update clone
@@ -94,6 +95,7 @@ jobs:
           if [ "${{ inputs.job }}" = "run_models_gpu" ]; then
             echo "folder_slices=$(python3 ../utils/split_model_tests.py --models '${{ inputs.models }}' --num_splits ${{ env.NUM_SLICES }})" >> $GITHUB_OUTPUT
             echo "slice_ids=$(python3 -c 'd = list(range(${{ env.NUM_SLICES }})); print(d)')" >> $GITHUB_OUTPUT
+            echo "runner_map=$(python3 ../utils/get_runner_map.py)" >> $GITHUB_OUTPUT
           elif [ "${{ inputs.job }}" = "run_trainer_and_fsdp_gpu" ]; then
             echo "folder_slices=[['trainer'], ['fsdp']]" >> $GITHUB_OUTPUT
             echo "slice_ids=[0, 1]" >> $GITHUB_OUTPUT
@@ -117,13 +119,14 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        machine_type: [aws-g5-4xlarge-cache, aws-g5-12xlarge-cache]
+        machine_type: [single-gpu, multi-gpu]
         slice_id: ${{ fromJSON(needs.setup.outputs.slice_ids) }}
     uses: ./.github/workflows/model_jobs.yml
     with:
       folder_slices: ${{ needs.setup.outputs.folder_slices }}
       machine_type: ${{ matrix.machine_type }}
       slice_id: ${{ matrix.slice_id }}
+      runner_map: ${{ needs.setup.outputs.runner_map }}
       docker: ${{ inputs.docker }}
       commit_sha: ${{ inputs.commit_sha || github.sha }}
       runner_type: ${{ inputs.runner_type }}
@@ -144,10 +147,9 @@ jobs:
       folder_slices: ${{ needs.setup.outputs.folder_slices }}
       machine_type: ${{ matrix.machine_type }}
       slice_id: ${{ matrix.slice_id }}
+      runner_map: ${{ needs.setup.outputs.runner_map }}
       docker: ${{ inputs.docker }}
       commit_sha: ${{ inputs.commit_sha || github.sha }}
-      runner_type: ${{ inputs.runner_type }}
-      report_repo_id: ${{ inputs.report_repo_id }}
       report_name_prefix: run_trainer_and_fsdp_gpu
     secrets: inherit
 

.gitignore

@@ -13,7 +13,6 @@ tests/fixtures/cached_*_text.txt
 logs/
 lightning_logs/
 lang_code_data/
-reports/
 
 # Distribution / packaging
 .Python

ISSUES.md

@@ -38,6 +38,7 @@ In particular all "Please explain" questions or objectively very user-specific f
 
 * "How to train T5 on De->En translation?"
 
+
 ## The GitHub Issues
 
 Everything which hints at a bug should be opened as an [issue](https://github.com/huggingface/transformers/issues).
@@ -246,6 +247,7 @@ You are not required to read the following guidelines before opening an issue. H
 
     Try not use italics and bold text too much as these often make the text more difficult to read.
 
+
 12. If you are cross-referencing a specific comment in a given thread or another issue, always link to that specific comment, rather than using the issue link. If you do the latter it could be quite impossible to find which specific comment you're referring to.
 
     To get the link to the specific comment do not copy the url from the location bar of your browser, but instead, click the `...` icon in the upper right corner of the comment and then select "Copy Link".
@@ -255,6 +257,7 @@ You are not required to read the following guidelines before opening an issue. H
     1. https://github.com/huggingface/transformers/issues/9257
     2. https://github.com/huggingface/transformers/issues/9257#issuecomment-749945162
 
+
 13. If you are replying to a last comment, it's totally fine to make your reply with just your comment in it. The readers can follow the information flow here.
 
     But if you're replying to a comment that happened some comments back it's always a good practice to quote just the relevant lines you're replying it. The `>` is used for quoting, or you can always use the menu to do so. For example your editor box will look like:

Makefile

@@ -3,7 +3,7 @@
 # make sure to test the local checkout in scripts and not the pre-installed one (don't use quotes!)
 export PYTHONPATH = src
 
-check_dirs := examples tests src utils scripts benchmark benchmark_v2
+check_dirs := examples tests src utils
 
 exclude_folders :=  ""
 

README.md

@@ -51,7 +51,6 @@ limitations under the License.
         <a href="https://github.com/huggingface/transformers/blob/main/i18n/README_vi.md">Tiếng Việt</a> |
         <a href="https://github.com/huggingface/transformers/blob/main/i18n/README_ar.md">العربية</a> |
         <a href="https://github.com/huggingface/transformers/blob/main/i18n/README_ur.md">اردو</a> |
-        <a href="https://github.com/huggingface/transformers/blob/main/i18n/README_bn.md">বাংলা</a> |
     </p>
 </h4>
 
@@ -63,11 +62,12 @@ limitations under the License.
     <img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/transformers_as_a_model_definition.png"/>
 </h3>
 
-Transformers acts as the model-definition framework for state-of-the-art machine learning models in text, computer
-vision, audio, video, and multimodal model, for both inference and training.
 
-It centralizes the model definition so that this definition is agreed upon across the ecosystem. `transformers` is the
-pivot across frameworks: if a model definition is supported, it will be compatible with the majority of training
+Transformers acts as the model-definition framework for state-of-the-art machine learning models in text, computer 
+vision, audio, video, and multimodal model, for both inference and training. 
+
+It centralizes the model definition so that this definition is agreed upon across the ecosystem. `transformers` is the 
+pivot across frameworks: if a model definition is supported, it will be compatible with the majority of training 
 frameworks (Axolotl, Unsloth, DeepSpeed, FSDP, PyTorch-Lightning, ...), inference engines (vLLM, SGLang, TGI, ...),
 and adjacent modeling libraries (llama.cpp, mlx, ...) which leverage the model definition from `transformers`.
 
@@ -80,7 +80,7 @@ Explore the [Hub](https://huggingface.com/) today to find a model and use Transf
 
 ## Installation
 
-Transformers works with Python 3.9+, and [PyTorch](https://pytorch.org/get-started/locally/) 2.1+.
+Transformers works with Python 3.9+ [PyTorch](https://pytorch.org/get-started/locally/) 2.1+, [TensorFlow](https://www.tensorflow.org/install/pip) 2.6+, and [Flax](https://flax.readthedocs.io/en/latest/) 0.4.1+.
 
 Create and activate a virtual environment with [venv](https://docs.python.org/3/library/venv.html) or [uv](https://docs.astral.sh/uv/), a fast Rust-based Python package and project manager.
 
@@ -193,6 +193,7 @@ pipeline("https://huggingface.co/datasets/Narsil/image_dummy/raw/main/parrots.pn
 <details>
 <summary>Visual question answering</summary>
 
+
 <h3 align="center">
     <a><img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/tasks/idefics-few-shot.jpg"></a>
 </h3>

awesome-transformers.md

@@ -6,7 +6,7 @@ developers, researchers, students, professors, engineers, and anyone else to bui
 
 In this list, we showcase incredibly impactful and novel projects that have pushed the field forward. We celebrate
 100 of these projects as we reach the milestone of 100k stars as a community; but we're very open to pull requests
-adding other projects to the list. If you believe a project should be here and it's not, then please, open a PR
+adding other projects to the list. If you believe a project should be here and it's not, then please, open a PR 
 to add it.
 
 ## [gpt4all](https://github.com/nomic-ai/gpt4all)
@@ -49,7 +49,7 @@ Keywords: LLMs, Large Language Models, Agents, Chains
 
 [LlamaIndex](https://github.com/run-llama/llama_index) is a project that provides a central interface to connect your LLM's with external data. It provides various kinds of indices and retrieval mechanisms to perform different LLM tasks and obtain knowledge-augmented results.
 
-Keywords: LLMs, Large Language Models, Data Retrieval, Indices, Knowledge Augmentation
+Keywords: LLMs, Large Language Models, Data Retrieval, Indices, Knowledge Augmentation 
 
 ## [ParlAI](https://github.com/facebookresearch/ParlAI)
 
@@ -257,7 +257,7 @@ Stable-Dreamfusion is a pytorch implementation of the text-to-3D model Dreamfusi
 Keywords: Text-to-3D, Stable Diffusion
 
 ## [txtai](https://github.com/neuml/txtai)
-
+ 
 [txtai](https://github.com/neuml/txtai) is an open-source platform for semantic search and workflows powered by language models. txtai builds embeddings databases, which are a union of vector indexes and relational databases enabling similarity search with SQL. Semantic workflows connect language models together into unified applications.
 
 Keywords: Semantic search, LLM
@@ -309,8 +309,8 @@ Keywords: OCR, LaTeX, Math formula
 
 OpenCLIP is an open source implementation of OpenAI's CLIP.
 
-The goal of this repository is to enable training models with contrastive image-text supervision, and to investigate their properties such as robustness to distribution shift.
-The starting point is an implementation of CLIP that matches the accuracy of the original CLIP models when trained on the same dataset.
+The goal of this repository is to enable training models with contrastive image-text supervision, and to investigate their properties such as robustness to distribution shift. 
+The starting point is an implementation of CLIP that matches the accuracy of the original CLIP models when trained on the same dataset. 
 
 Specifically, a ResNet-50 model trained with this codebase on OpenAI's 15 million image subset of YFCC achieves 32.7% top-1 accuracy on ImageNet.
 
@@ -596,7 +596,7 @@ Keywords: Data-Centric AI, Data Quality, Noisy Labels, Outlier Detection, Active
 
 ## [BentoML](https://github.com/bentoml/BentoML)
 
-[BentoML](https://github.com/bentoml) is the unified framework for building, shipping, and scaling production-ready AI applications incorporating traditional ML, pre-trained AI models, Generative and Large Language Models.
+[BentoML](https://github.com/bentoml) is the unified framework for building, shipping, and scaling production-ready AI applications incorporating traditional ML, pre-trained AI models, Generative and Large Language Models. 
 All Hugging Face models and pipelines can be seamlessly integrated into BentoML applications, enabling the running of models on the most suitable hardware and independent scaling based on usage.
 
 Keywords: BentoML, Framework, Deployment, AI Applications
@@ -606,3 +606,4 @@ Keywords: BentoML, Framework, Deployment, AI Applications
 [LLaMA Factory](https://github.com/hiyouga/LLaMA-Factory) offers a user-friendly fine-tuning framework that incorporates PEFT. The repository includes training(fine-tuning) and inference examples for LLaMA-2, BLOOM, Falcon, Baichuan, Qwen, and other LLMs. A ChatGLM version is also available in [ChatGLM-Efficient-Tuning](https://github.com/hiyouga/ChatGLM-Efficient-Tuning).
 
 Keywords: PEFT, fine-tuning, LLaMA-2, ChatGLM, Qwen
+

benchmark/benches/llama.py

@@ -11,28 +11,25 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import os
-import sys
 from logging import Logger
+import os
 from threading import Event, Thread
 from time import perf_counter, sleep
 from typing import Optional
-
+import sys
 
 # Add the parent directory to Python path to import benchmarks_entrypoint
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+from benchmarks_entrypoint import MetricsRecorder
+
 import gpustat
 import psutil
 import psycopg2
-from benchmarks_entrypoint import MetricsRecorder
-
 
 # Optional heavy ML dependencies - only required when actually running the benchmark
 try:
     import torch
-
     from transformers import AutoModelForCausalLM, AutoTokenizer, GenerationConfig, StaticCache
-
     TRANSFORMERS_AVAILABLE = True
 except ImportError:
     TRANSFORMERS_AVAILABLE = False
@@ -66,13 +63,7 @@ def collect_metrics(benchmark_id, continue_metric_collection, metrics_recorder):
 
 
 def run_benchmark(
-    logger: Logger,
-    repository: str,
-    branch: str,
-    commit_id: str,
-    commit_msg: str,
-    metrics_recorder=None,
-    num_tokens_to_generate=100,
+    logger: Logger, repository: str, branch: str, commit_id: str, commit_msg: str, metrics_recorder=None, num_tokens_to_generate=100
 ):
     # Check if required ML dependencies are available
     if not TRANSFORMERS_AVAILABLE:
@@ -80,11 +71,11 @@ def run_benchmark(
         logger.error("pip install torch transformers")
         logger.error("Skipping LLaMA benchmark due to missing dependencies.")
         return
-
+    
     continue_metric_collection = Event()
     metrics_thread = None
     model_id = "meta-llama/Llama-2-7b-hf"
-
+    
     # If no metrics_recorder is provided, create one for backward compatibility
     if metrics_recorder is None:
         try:
@@ -163,7 +154,7 @@ def run_benchmark(
         # First eager forward pass
         logger.info("running first eager forward pass")
         start = perf_counter()
-        _ = model(**inputs)
+        outputs = model(**inputs)
         torch.cuda.synchronize()
         end = perf_counter()
         first_eager_fwd_pass_time = end - start
@@ -172,7 +163,7 @@ def run_benchmark(
         # Second eager forward pass (should be faster)
         logger.info("running second eager forward pass")
         start = perf_counter()
-        _ = model(**inputs)
+        outputs = model(**inputs)
         torch.cuda.synchronize()
         end = perf_counter()
         second_eager_fwd_pass_time = end - start
@@ -348,7 +339,7 @@ def run_benchmark(
     continue_metric_collection.set()
     if metrics_thread is not None:
         metrics_thread.join()
-
+    
     # Only close the recorder if we created it locally
     if should_close_recorder:
-        metrics_recorder.close()
+        metrics_recorder.close() 

benchmark/benchmark.py

@@ -31,7 +31,9 @@ from contextlib import contextmanager
 from pathlib import Path
 
 from git import Repo
+
 from huggingface_hub import HfApi
+
 from optimum_benchmark import Benchmark
 from optimum_benchmark_wrapper import main
 

benchmark/benchmarks_entrypoint.py

@@ -13,20 +13,19 @@
 # limitations under the License.
 import argparse
 import importlib.util
-import json
 import logging
 import os
 import sys
+import json
 import uuid
 from datetime import datetime
+from typing import Dict, Tuple, Optional, List
 
 import pandas as pd
 
-
 try:
     from psycopg2.extensions import register_adapter
     from psycopg2.extras import Json
-
     register_adapter(dict, Json)
     PSYCOPG2_AVAILABLE = True
 except ImportError:
@@ -39,14 +38,8 @@ class ImportModuleException(Exception):
 
 class MetricsRecorder:
     def __init__(
-        self,
-        connection,
-        logger: logging.Logger,
-        repository: str,
-        branch: str,
-        commit_id: str,
-        commit_msg: str,
-        collect_csv_data: bool = True,
+        self, connection, logger: logging.Logger, repository: str, branch: str, commit_id: str, commit_msg: str, 
+        collect_csv_data: bool = True
     ):
         self.conn = connection
         self.use_database = connection is not None
@@ -58,43 +51,27 @@ class MetricsRecorder:
         self.commit_id = commit_id
         self.commit_msg = commit_msg
         self.collect_csv_data = collect_csv_data
-
+        
         # For CSV export - store all data in pandas DataFrames (only if CSV collection is enabled)
         if self.collect_csv_data:
             # Initialize empty DataFrames with proper schemas
-            self.benchmarks_df = pd.DataFrame(
-                columns=[
-                    "benchmark_id",
-                    "repository",
-                    "branch",
-                    "commit_id",
-                    "commit_message",
-                    "metadata",
-                    "created_at",
-                ]
-            )
-            self.device_measurements_df = pd.DataFrame(
-                columns=["benchmark_id", "cpu_util", "mem_megabytes", "gpu_util", "gpu_mem_megabytes", "time"]
-            )
-            self.model_measurements_df = pd.DataFrame(
-                columns=[
-                    "benchmark_id",
-                    "time",
-                    "model_load_time",
-                    "first_eager_forward_pass_time_secs",
-                    "second_eager_forward_pass_time_secs",
-                    "first_eager_generate_time_secs",
-                    "second_eager_generate_time_secs",
-                    "time_to_first_token_secs",
-                    "time_to_second_token_secs",
-                    "time_to_third_token_secs",
-                    "time_to_next_token_mean_secs",
-                    "first_compile_generate_time_secs",
-                    "second_compile_generate_time_secs",
-                    "third_compile_generate_time_secs",
-                    "fourth_compile_generate_time_secs",
-                ]
-            )
+            self.benchmarks_df = pd.DataFrame(columns=[
+                'benchmark_id', 'repository', 'branch', 'commit_id', 'commit_message', 
+                'metadata', 'created_at'
+            ])
+            self.device_measurements_df = pd.DataFrame(columns=[
+                'benchmark_id', 'cpu_util', 'mem_megabytes', 'gpu_util', 
+                'gpu_mem_megabytes', 'time'
+            ])
+            self.model_measurements_df = pd.DataFrame(columns=[
+                'benchmark_id', 'time', 'model_load_time', 'first_eager_forward_pass_time_secs',
+                'second_eager_forward_pass_time_secs', 'first_eager_generate_time_secs',
+                'second_eager_generate_time_secs', 'time_to_first_token_secs',
+                'time_to_second_token_secs', 'time_to_third_token_secs',
+                'time_to_next_token_mean_secs', 'first_compile_generate_time_secs',
+                'second_compile_generate_time_secs', 'third_compile_generate_time_secs',
+                'fourth_compile_generate_time_secs'
+            ])
         else:
             self.benchmarks_df = None
             self.device_measurements_df = None
@@ -106,7 +83,7 @@ class MetricsRecorder:
         """
         # Generate a unique UUID for this benchmark
         benchmark_id = str(uuid.uuid4())
-
+        
         if self.use_database:
             with self.conn.cursor() as cur:
                 cur.execute(
@@ -114,32 +91,28 @@ class MetricsRecorder:
                     (benchmark_id, self.repository, self.branch, self.commit_id, self.commit_msg, metadata),
                 )
                 self.logger.debug(f"initialised benchmark #{benchmark_id}")
-
+        
         # Store benchmark data for CSV export (if enabled)
         if self.collect_csv_data:
             # Add row to pandas DataFrame
-            new_row = pd.DataFrame(
-                [
-                    {
-                        "benchmark_id": benchmark_id,
-                        "repository": self.repository,
-                        "branch": self.branch,
-                        "commit_id": self.commit_id,
-                        "commit_message": self.commit_msg,
-                        "metadata": json.dumps(metadata),
-                        "created_at": datetime.utcnow().isoformat(),
-                    }
-                ]
-            )
+            new_row = pd.DataFrame([{
+                'benchmark_id': benchmark_id,
+                'repository': self.repository,
+                'branch': self.branch,
+                'commit_id': self.commit_id,
+                'commit_message': self.commit_msg,
+                'metadata': json.dumps(metadata),
+                'created_at': datetime.utcnow().isoformat()
+            }])
             self.benchmarks_df = pd.concat([self.benchmarks_df, new_row], ignore_index=True)
-
+            
         mode_info = []
         if self.use_database:
             mode_info.append("database")
         if self.collect_csv_data:
             mode_info.append("CSV")
         mode_str = " + ".join(mode_info) if mode_info else "no storage"
-
+        
         self.logger.debug(f"initialised benchmark #{benchmark_id} ({mode_str} mode)")
         return benchmark_id
 
@@ -150,20 +123,16 @@ class MetricsRecorder:
         # Store device measurements for CSV export (if enabled)
         if self.collect_csv_data:
             # Add row to pandas DataFrame
-            new_row = pd.DataFrame(
-                [
-                    {
-                        "benchmark_id": benchmark_id,
-                        "cpu_util": cpu_util,
-                        "mem_megabytes": mem_megabytes,
-                        "gpu_util": gpu_util,
-                        "gpu_mem_megabytes": gpu_mem_megabytes,
-                        "time": datetime.utcnow().isoformat(),
-                    }
-                ]
-            )
+            new_row = pd.DataFrame([{
+                'benchmark_id': benchmark_id,
+                'cpu_util': cpu_util,
+                'mem_megabytes': mem_megabytes,
+                'gpu_util': gpu_util,
+                'gpu_mem_megabytes': gpu_mem_megabytes,
+                'time': datetime.utcnow().isoformat()
+            }])
             self.device_measurements_df = pd.concat([self.device_measurements_df, new_row], ignore_index=True)
-
+        
         # Store in database if available
         if self.use_database:
             with self.conn.cursor() as cur:
@@ -171,7 +140,7 @@ class MetricsRecorder:
                     "INSERT INTO device_measurements (benchmark_id, cpu_util, mem_megabytes, gpu_util, gpu_mem_megabytes) VALUES (%s, %s, %s, %s, %s)",
                     (benchmark_id, cpu_util, mem_megabytes, gpu_util, gpu_mem_megabytes),
                 )
-
+            
         self.logger.debug(
             f"collected device measurements for benchmark #{benchmark_id} [CPU util: {cpu_util}, mem MBs: {mem_megabytes}, GPU util: {gpu_util}, GPU mem MBs: {gpu_mem_megabytes}]"
         )
@@ -180,13 +149,16 @@ class MetricsRecorder:
         # Store model measurements for CSV export (if enabled)
         if self.collect_csv_data:
             # Add row to pandas DataFrame with flattened measurements
-            row_data = {"benchmark_id": benchmark_id, "time": datetime.utcnow().isoformat()}
+            row_data = {
+                'benchmark_id': benchmark_id,
+                'time': datetime.utcnow().isoformat()
+            }
             # Flatten the measurements dict into the row
             row_data.update(measurements)
-
+            
             new_row = pd.DataFrame([row_data])
             self.model_measurements_df = pd.concat([self.model_measurements_df, new_row], ignore_index=True)
-
+        
         # Store in database if available
         if self.use_database:
             with self.conn.cursor() as cur:
@@ -202,7 +174,7 @@ class MetricsRecorder:
                         measurements,
                     ),
                 )
-
+            
         self.logger.debug(f"collected model measurements for benchmark #{benchmark_id}: {measurements}")
 
     def export_to_csv(self, output_dir: str = "benchmark_results"):
@@ -212,19 +184,19 @@ class MetricsRecorder:
         if not self.collect_csv_data:
             self.logger.warning("CSV data collection is disabled - no CSV files will be generated")
             return
-
+            
         if not os.path.exists(output_dir):
             os.makedirs(output_dir)
             self.logger.info(f"Created output directory: {output_dir}")
-
+            
         timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
         files_created = []
-
+        
         # Export using pandas DataFrames
         self._export_pandas_data(output_dir, timestamp, files_created)
-
+        
         self.logger.info(f"CSV export complete! Created {len(files_created)} files in {output_dir}")
-
+    
     def _export_pandas_data(self, output_dir: str, timestamp: str, files_created: list):
         """
         Export CSV files using pandas DataFrames
@@ -234,24 +206,24 @@ class MetricsRecorder:
         self.benchmarks_df.to_csv(benchmarks_file, index=False)
         files_created.append(benchmarks_file)
         self.logger.info(f"Exported {len(self.benchmarks_df)} benchmark records to {benchmarks_file}")
-
-        # Export device measurements
+        
+        # Export device measurements  
         device_file = os.path.join(output_dir, f"device_measurements_{timestamp}.csv")
         self.device_measurements_df.to_csv(device_file, index=False)
         files_created.append(device_file)
         self.logger.info(f"Exported {len(self.device_measurements_df)} device measurement records to {device_file}")
-
+        
         # Export model measurements (already flattened)
         model_file = os.path.join(output_dir, f"model_measurements_{timestamp}.csv")
         self.model_measurements_df.to_csv(model_file, index=False)
         files_created.append(model_file)
         self.logger.info(f"Exported {len(self.model_measurements_df)} model measurement records to {model_file}")
-
+        
         # Create comprehensive summary using pandas operations
         summary_file = os.path.join(output_dir, f"benchmark_summary_{timestamp}.csv")
         self._create_summary(summary_file)
         files_created.append(summary_file)
-
+    
     def _create_summary(self, summary_file: str):
         """
         Create a comprehensive summary CSV using pandas operations
@@ -262,42 +234,36 @@ class MetricsRecorder:
             summary_df.to_csv(summary_file, index=False)
             self.logger.info(f"Created empty benchmark summary at {summary_file}")
             return
-
+        
         # Start with benchmarks as the base
         summary_df = self.benchmarks_df.copy()
-
+        
         # Add model measurements (join on benchmark_id)
         if len(self.model_measurements_df) > 0:
             # Drop 'time' column from model measurements to avoid conflicts
-            model_df = self.model_measurements_df.drop(columns=["time"], errors="ignore")
-            summary_df = summary_df.merge(model_df, on="benchmark_id", how="left")
-
+            model_df = self.model_measurements_df.drop(columns=['time'], errors='ignore')
+            summary_df = summary_df.merge(model_df, on='benchmark_id', how='left')
+        
         # Calculate device measurement aggregates using pandas groupby
         if len(self.device_measurements_df) > 0:
-            device_agg = (
-                self.device_measurements_df.groupby("benchmark_id")
-                .agg(
-                    {
-                        "cpu_util": ["mean", "max", "std", "count"],
-                        "mem_megabytes": ["mean", "max", "std"],
-                        "gpu_util": ["mean", "max", "std"],
-                        "gpu_mem_megabytes": ["mean", "max", "std"],
-                    }
-                )
-                .round(3)
-            )
-
+            device_agg = self.device_measurements_df.groupby('benchmark_id').agg({
+                'cpu_util': ['mean', 'max', 'std', 'count'],
+                'mem_megabytes': ['mean', 'max', 'std'],
+                'gpu_util': ['mean', 'max', 'std'],
+                'gpu_mem_megabytes': ['mean', 'max', 'std']
+            }).round(3)
+            
             # Flatten column names
             device_agg.columns = [f"{col[0]}_{col[1]}" for col in device_agg.columns]
             device_agg = device_agg.reset_index()
-
+            
             # Rename count column to be more descriptive
-            if "cpu_util_count" in device_agg.columns:
-                device_agg = device_agg.rename(columns={"cpu_util_count": "device_measurement_count"})
-
+            if 'cpu_util_count' in device_agg.columns:
+                device_agg = device_agg.rename(columns={'cpu_util_count': 'device_measurement_count'})
+            
             # Merge with summary
-            summary_df = summary_df.merge(device_agg, on="benchmark_id", how="left")
-
+            summary_df = summary_df.merge(device_agg, on='benchmark_id', how='left')
+        
         # Export the comprehensive summary
         summary_df.to_csv(summary_file, index=False)
         self.logger.info(f"Created comprehensive benchmark summary with {len(summary_df)} records at {summary_file}")
@@ -346,18 +312,23 @@ def parse_arguments() -> tuple[str, str, str, str, bool, str]:
         type=str,
         help="The commit message associated with the commit, truncated to 70 characters.",
     )
-
-    parser.add_argument("--csv", action="store_true", default=False, help="Enable CSV output files generation.")
-
+    
+    parser.add_argument(
+        "--csv",
+        action="store_true",
+        default=False,
+        help="Enable CSV output files generation."
+    )
+    
     parser.add_argument(
         "--csv-output-dir",
         type=str,
         default="benchmark_results",
-        help="Directory for CSV output files (default: benchmark_results).",
+        help="Directory for CSV output files (default: benchmark_results)."
     )
 
     args = parser.parse_args()
-
+    
     # CSV is disabled by default, only enabled when --csv is used
     generate_csv = args.csv
 
@@ -382,10 +353,9 @@ def create_database_connection():
     if not PSYCOPG2_AVAILABLE:
         logger.warning("psycopg2 not available - running in CSV-only mode")
         return None
-
+        
     try:
         import psycopg2
-
         conn = psycopg2.connect("dbname=metrics")
         logger.info("Successfully connected to database")
         return conn
@@ -394,28 +364,27 @@ def create_database_connection():
         return None
 
 
-def create_global_metrics_recorder(
-    repository: str, branch: str, commit_id: str, commit_msg: str, generate_csv: bool = False
-) -> MetricsRecorder:
+def create_global_metrics_recorder(repository: str, branch: str, commit_id: str, commit_msg: str, 
+                                   generate_csv: bool = False) -> MetricsRecorder:
     """
     Create a global metrics recorder that will be used across all benchmarks.
     """
     connection = create_database_connection()
     recorder = MetricsRecorder(connection, logger, repository, branch, commit_id, commit_msg, generate_csv)
-
+    
     # Log the storage mode
     storage_modes = []
     if connection is not None:
         storage_modes.append("database")
     if generate_csv:
         storage_modes.append("CSV")
-
+    
     if not storage_modes:
         logger.warning("Running benchmarks with NO data storage (no database connection, CSV disabled)")
         logger.warning("Use --csv flag to enable CSV output when database is unavailable")
     else:
         logger.info(f"Running benchmarks with: {' + '.join(storage_modes)} storage")
-
+    
     return recorder
 
 
@@ -424,16 +393,16 @@ if __name__ == "__main__":
     benches_folder_path = os.path.join(benchmarks_folder_path, "benches")
 
     repository, branch, commit_id, commit_msg, generate_csv, csv_output_dir = parse_arguments()
-
+    
     # Create a global metrics recorder
     global_metrics_recorder = create_global_metrics_recorder(repository, branch, commit_id, commit_msg, generate_csv)
-
+    
     successful_benchmarks = 0
     failed_benchmarks = 0
-
+    
     # Automatically discover all benchmark modules in benches/ folder
     benchmark_modules = []
-
+    
     if os.path.exists(benches_folder_path):
         logger.debug(f"Scanning for benchmarks in: {benches_folder_path}")
         for entry in os.scandir(benches_folder_path):
@@ -441,12 +410,12 @@ if __name__ == "__main__":
                 continue
             if entry.name.startswith("__"):  # Skip __init__.py, __pycache__, etc.
                 continue
-
+                
             # Check if the file has a run_benchmark function
             try:
                 logger.debug(f"checking if benches/{entry.name} has run_benchmark function")
                 module = import_from_path(entry.name.split(".")[0], entry.path)
-                if hasattr(module, "run_benchmark"):
+                if hasattr(module, 'run_benchmark'):
                     benchmark_modules.append(entry.name)
                     logger.debug(f"discovered benchmark: {entry.name}")
                 else:
@@ -467,18 +436,16 @@ if __name__ == "__main__":
             logger.debug(f"loading: {module_name}")
             module = import_from_path(module_name.split(".")[0], module_path)
             logger.info(f"running benchmarks in: {module_name}")
-
+            
             # Check if the module has an updated run_benchmark function that accepts metrics_recorder
             try:
                 # Try the new signature first
                 module.run_benchmark(logger, repository, branch, commit_id, commit_msg, global_metrics_recorder)
             except TypeError:
                 # Fall back to the old signature for backward compatibility
-                logger.warning(
-                    f"Module {module_name} using old run_benchmark signature - database connection will be created per module"
-                )
+                logger.warning(f"Module {module_name} using old run_benchmark signature - database connection will be created per module")
                 module.run_benchmark(logger, repository, branch, commit_id, commit_msg)
-
+            
             successful_benchmarks += 1
         except ImportModuleException as e:
             logger.error(e)
@@ -494,7 +461,7 @@ if __name__ == "__main__":
             logger.info(f"CSV reports have been generated and saved to the {csv_output_dir} directory")
         else:
             logger.info("CSV generation disabled - no CSV files created (use --csv to enable)")
-
+        
         logger.info(f"Benchmark run completed. Successful: {successful_benchmarks}, Failed: {failed_benchmarks}")
     except Exception as e:
         logger.error(f"Failed to export CSV results: {e}")

benchmark/optimum_benchmark_wrapper.py

@@ -3,11 +3,7 @@ import subprocess
 
 
 def main(config_dir, config_name, args):
-    subprocess.run(
-        ["optimum-benchmark", "--config-dir", f"{config_dir}", "--config-name", f"{config_name}"]
-        + ["hydra/job_logging=disabled", "hydra/hydra_logging=disabled"]
-        + args
-    )
+    subprocess.run(["optimum-benchmark", "--config-dir", f"{config_dir}", "--config-name", f"{config_name}"] + ["hydra/job_logging=disabled", "hydra/hydra_logging=disabled"] + args)
 
 
 if __name__ == "__main__":

benchmark_v2/README.md

@@ -21,46 +21,6 @@ python run_benchmarks.py \
     --num-tokens-to-generate 200
 ```
 
-### Uploading Results to HuggingFace Dataset
-
-You can automatically upload benchmark results to a HuggingFace Dataset for tracking and analysis:
-
-```bash
-# Upload to a public dataset with auto-generated run ID
-python run_benchmarks.py --upload-to-hub username/benchmark-results
-
-# Upload with a custom run ID for easy identification
-python run_benchmarks.py --upload-to-hub username/benchmark-results --run-id experiment_v1
-
-# Upload with custom HuggingFace token (if not set in environment)
-python run_benchmarks.py --upload-to-hub username/benchmark-results --token hf_your_token_here
-```
-
-**Dataset Directory Structure:**
-```
-dataset_name/
-├── 2025-01-15/
-│   ├── runs/                       # Non-scheduled runs (manual, PR, etc.)
-│   │   └── 123-1245151651/         # GitHub run number and ID
-│   │       └── benchmark_results/
-│   │           ├── benchmark_summary_20250115_143022.json
-│   │           └── model-name/
-│   │               └── model-name_benchmark_20250115_143022.json
-│   └── benchmark_results_abc123de/ # Scheduled runs (daily CI)
-│       ├── benchmark_summary_20250115_143022.json
-│       └── model-name/
-│           └── model-name_benchmark_20250115_143022.json
-└── 2025-01-16/
-    └── ...
-```
-
-**Authentication for Uploads:**
-
-For uploading results, you need a HuggingFace token with write permissions to the target dataset. You can provide the token in several ways (in order of precedence):
-
-1. Command line: `--token hf_your_token_here`
-3. Environment variable: `HF_TOKEN`
-
 ### Running Specific Benchmarks
 
 ```bash

benchmark_v2/benches/__init__.py

@@ -1 +1 @@
-# Benchmark implementations directory
+# Benchmark implementations directory 

benchmark_v2/benches/llama.py

@@ -12,62 +12,55 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import logging
 import os
-from typing import Any
+import logging
+from typing import Dict, Any, List
 
-import torch
 from benchmark_framework import ModelBenchmark
 
+import torch
 
+os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "1"
 os.environ["TOKENIZERS_PARALLELISM"] = "1"
 torch.set_float32_matmul_precision("high")
 
-
 class LLaMABenchmark(ModelBenchmark):
     """Simplified LLaMA model benchmark implementation using the ModelBenchmark base class."""
-
+    
     def __init__(self, logger: logging.Logger):
         super().__init__(logger)
         self._default_prompt = "Why dogs are so cute?"  # Custom prompt for LLaMA
+    
 
-    def get_scenario_configs(self) -> list[dict[str, Any]]:
+    
+    def get_scenario_configs(self) -> List[Dict[str, Any]]:
         """
         Get LLaMA-specific scenario configurations.
-
+        
         Returns:
             List of scenario configuration dictionaries
         """
         return [
             # Eager variants
             {"variant": "eager", "compile_mode": None, "use_cache": True, "description": "Eager execution with cache"},
+            
             # Compiled variants
-            {
-                "variant": "compiled",
-                "compile_mode": "max-autotune",
-                "use_cache": True,
-                "description": "Compiled with max autotune",
-            },
+            {"variant": "compiled", "compile_mode": "max-autotune", "use_cache": True, "description": "Compiled with max autotune"},
+            
             # Kernelized variant (if available)
-            {
-                "variant": "kernelized",
-                "compile_mode": "max-autotune",
-                "use_cache": True,
-                "description": "Kernelized execution",
-            },
+            {"variant": "kernelized", "compile_mode": "max-autotune", "use_cache": True, "description": "Kernelized execution"},
         ]
-
+    
     def _is_kernelization_available(self) -> bool:
         """Check if kernelization is available for LLaMA."""
         try:
-            from kernels import Mode, kernelize  # noqa: F401
-
+            from kernels import Mode, kernelize
             return True
         except ImportError:
             self.logger.debug("Kernelization not available: kernels module not found")
             return False
-
-    def get_default_generation_config(self) -> dict[str, Any]:
+    
+    def get_default_generation_config(self) -> Dict[str, Any]:
         """Get LLaMA-specific generation configuration."""
         return {
             "do_sample": False,
@@ -76,19 +69,20 @@ class LLaMABenchmark(ModelBenchmark):
             "repetition_penalty": 1.0,
             "max_new_tokens": None,  # Will be set per scenario
         }
-
-    def get_model_init_kwargs(self, config) -> dict[str, Any]:
+    
+    def get_model_init_kwargs(self, config) -> Dict[str, Any]:
         """Get LLaMA-specific model initialization kwargs."""
+        from benchmark_framework import BenchmarkConfig
         return {
             "torch_dtype": getattr(torch, config.torch_dtype),
             "attn_implementation": config.attn_implementation,
             "use_cache": True,
         }
-
+    
     def get_default_torch_dtype(self) -> str:
         """Get default torch dtype for LLaMA."""
         return "float16"  # LLaMA works well with float16
-
+    
     def get_default_device(self) -> str:
         """Get default device for LLaMA."""
         return "cuda"  # LLaMA prefers CUDA
@@ -97,37 +91,35 @@ class LLaMABenchmark(ModelBenchmark):
 def run_llama(logger, output_dir, **kwargs):
     """
     Run LLaMA benchmark with the given configuration.
-
+    
     Args:
         logger: Logger instance
         output_dir: Output directory for results
         **kwargs: Additional configuration options
-
+        
     Returns:
         Path to output file if successful
     """
     from benchmark_framework import BenchmarkRunner
-
+    
     # Extract parameters with defaults
-    model_id = kwargs.get("model_id", "meta-llama/Llama-2-7b-hf")
-    warmup_iterations = kwargs.get("warmup_iterations", 3)
-    measurement_iterations = kwargs.get("measurement_iterations", 5)
-    num_tokens_to_generate = kwargs.get("num_tokens_to_generate", 100)
-    include_sdpa_variants = kwargs.get("include_sdpa_variants", True)
-    device = kwargs.get("device", "cuda")
-    torch_dtype = kwargs.get("torch_dtype", "float16")
-    batch_size = kwargs.get("batch_size", 1)
-    commit_id = kwargs.get("commit_id")
-
+    model_id = kwargs.get('model_id', 'meta-llama/Llama-2-7b-hf')
+    warmup_iterations = kwargs.get('warmup_iterations', 3)
+    measurement_iterations = kwargs.get('measurement_iterations', 5)
+    num_tokens_to_generate = kwargs.get('num_tokens_to_generate', 100)
+    include_sdpa_variants = kwargs.get('include_sdpa_variants', True)
+    device = kwargs.get('device', 'cuda')
+    torch_dtype = kwargs.get('torch_dtype', 'float16')
+    batch_size = kwargs.get('batch_size', 1)
+    commit_id = kwargs.get('commit_id', None)
+    
     logger.info(f"Starting LLaMA benchmark for model: {model_id}")
-    logger.info(
-        f"Configuration: warmup={warmup_iterations}, measurement={measurement_iterations}, tokens={num_tokens_to_generate}"
-    )
-
+    logger.info(f"Configuration: warmup={warmup_iterations}, measurement={measurement_iterations}, tokens={num_tokens_to_generate}")
+    
     try:
         # Create benchmark instance
         benchmark = LLaMABenchmark(logger)
-
+        
         # Create scenarios
         scenarios = benchmark.create_scenarios(
             model_id=model_id,
@@ -137,29 +129,28 @@ def run_llama(logger, output_dir, **kwargs):
             include_sdpa_variants=include_sdpa_variants,
             device=device,
             torch_dtype=torch_dtype,
-            batch_size=batch_size,
+            batch_size=batch_size
         )
-
+        
         logger.info(f"Created {len(scenarios)} benchmark scenarios")
-
+        
         # Create runner and execute benchmarks
         runner = BenchmarkRunner(logger, output_dir)
         results = runner.run_benchmark(benchmark, scenarios, commit_id=commit_id)
-
+        
         if not results:
             logger.warning("No successful benchmark results")
             return None
-
+        
         # Save results
-        model_name = model_id.split("/")[-1]  # Extract model name from ID
+        model_name = model_id.split('/')[-1]  # Extract model name from ID
         output_file = runner.save_results(model_name, results)
-
+        
         logger.info(f"LLaMA benchmark completed successfully. Results saved to: {output_file}")
         return output_file
-
+        
     except Exception as e:
         logger.error(f"LLaMA benchmark failed: {e}")
         import traceback
-
         logger.debug(traceback.format_exc())
-        raise
+        raise

benchmark_v2/requirements.txt

@@ -3,5 +3,4 @@ psutil>=5.8.0
 gpustat>=1.0.0
 torch>=2.0.0
 transformers>=4.30.0
-datasets>=2.10.0
-huggingface_hub>=0.16.0 
+datasets>=2.10.0 

benchmark_v2/run_benchmarks.py

@@ -14,449 +14,350 @@
 # limitations under the License.
 
 """
-Top-level benchmarking script that automatically discovers and runs all benchmarks
+Top-level benchmarking script that automatically discovers and runs all benchmarks 
 in the ./benches directory, organizing outputs into model-specific subfolders.
 """
 
 import argparse
 import importlib.util
-import json
 import logging
 import os
 import sys
-import uuid
+import json
 from datetime import datetime
 from pathlib import Path
-from typing import Any, Optional
+from typing import Dict, List, Any, Optional
 
 
 def setup_logging(log_level: str = "INFO", enable_file_logging: bool = False) -> logging.Logger:
     """Setup logging configuration."""
     numeric_level = getattr(logging, log_level.upper(), None)
     if not isinstance(numeric_level, int):
-        raise ValueError(f"Invalid log level: {log_level}")
-
+        raise ValueError(f'Invalid log level: {log_level}')
+    
     handlers = [logging.StreamHandler(sys.stdout)]
-
+    
     if enable_file_logging:
-        handlers.append(logging.FileHandler(f"benchmark_run_{datetime.now().strftime('%Y%m%d_%H%M%S')}.log"))
-
+        handlers.append(
+            logging.FileHandler(f'benchmark_run_{datetime.now().strftime("%Y%m%d_%H%M%S")}.log')
+        )
+    
     logging.basicConfig(
-        level=numeric_level, format="[%(levelname)s - %(asctime)s] %(name)s: %(message)s", handlers=handlers
+        level=numeric_level,
+        format='[%(levelname)s - %(asctime)s] %(name)s: %(message)s',
+        handlers=handlers
     )
-
+    
     return logging.getLogger(__name__)
 
 
-def discover_benchmarks(benches_dir: str) -> list[dict[str, Any]]:
+def discover_benchmarks(benches_dir: str) -> List[Dict[str, Any]]:
     """
     Discover all benchmark modules in the benches directory.
-
+    
     Returns:
         List of dictionaries containing benchmark module info
     """
     benchmarks = []
     benches_path = Path(benches_dir)
-
+    
     if not benches_path.exists():
         raise FileNotFoundError(f"Benches directory not found: {benches_dir}")
-
+    
     for py_file in benches_path.glob("*.py"):
         if py_file.name.startswith("__"):
             continue
-
+            
         module_name = py_file.stem
-
+        
         try:
             # Import the module
             spec = importlib.util.spec_from_file_location(module_name, py_file)
             module = importlib.util.module_from_spec(spec)
             spec.loader.exec_module(module)
-
+            
             # Check if it has a benchmark runner function
-            if hasattr(module, f"run_{module_name}"):
-                benchmarks.append(
-                    {
-                        "name": module_name,
-                        "path": str(py_file),
-                        "module": module,
-                        "runner_function": getattr(module, f"run_{module_name}"),
-                    }
-                )
-            elif hasattr(module, "run_benchmark"):
-                benchmarks.append(
-                    {
-                        "name": module_name,
-                        "path": str(py_file),
-                        "module": module,
-                        "runner_function": getattr(module, "run_benchmark"),
-                    }
-                )
+            if hasattr(module, f'run_{module_name}'):
+                benchmarks.append({
+                    'name': module_name,
+                    'path': str(py_file),
+                    'module': module,
+                    'runner_function': getattr(module, f'run_{module_name}')
+                })
+            elif hasattr(module, 'run_benchmark'):
+                benchmarks.append({
+                    'name': module_name,
+                    'path': str(py_file),
+                    'module': module,
+                    'runner_function': getattr(module, 'run_benchmark')
+                })
             else:
                 logging.warning(f"No runner function found in {py_file}")
-
+                
         except Exception as e:
             logging.error(f"Failed to import {py_file}: {e}")
-
+            
     return benchmarks
 
 
 def run_single_benchmark(
-    benchmark_info: dict[str, Any], output_dir: str, logger: logging.Logger, **kwargs
+    benchmark_info: Dict[str, Any], 
+    output_dir: str,
+    logger: logging.Logger,
+    **kwargs
 ) -> Optional[str]:
     """
     Run a single benchmark and return the output file path.
-
+    
     Args:
         benchmark_info: Dictionary containing benchmark module info
         output_dir: Base output directory
         logger: Logger instance
         **kwargs: Additional arguments to pass to the benchmark
-
+        
     Returns:
         Path to the output file if successful, None otherwise
     """
-    benchmark_name = benchmark_info["name"]
-    runner_func = benchmark_info["runner_function"]
-
+    benchmark_name = benchmark_info['name']
+    runner_func = benchmark_info['runner_function']
+    
     logger.info(f"Running benchmark: {benchmark_name}")
-
+    
     try:
         # Check function signature to determine what arguments to pass
         import inspect
-
         sig = inspect.signature(runner_func)
-
+        
         # Prepare arguments based on function signature
-        func_kwargs = {"logger": logger, "output_dir": output_dir}
-
+        func_kwargs = {
+            'logger': logger,
+            'output_dir': output_dir
+        }
+        
         # Add other kwargs if the function accepts them
         for param_name in sig.parameters:
             if param_name in kwargs:
                 func_kwargs[param_name] = kwargs[param_name]
-
+        
         # Filter kwargs to only include parameters the function accepts
         # If function has **kwargs, include all provided kwargs
         has_var_kwargs = any(param.kind == param.VAR_KEYWORD for param in sig.parameters.values())
         if has_var_kwargs:
             valid_kwargs = {**func_kwargs, **kwargs}
         else:
-            valid_kwargs = {k: v for k, v in func_kwargs.items() if k in sig.parameters}
-
+            valid_kwargs = {k: v for k, v in func_kwargs.items() 
+                           if k in sig.parameters}
+        
         # Run the benchmark
         result = runner_func(**valid_kwargs)
-
+        
         if isinstance(result, str):
             # Function returned a file path
             return result
         else:
             logger.info(f"Benchmark {benchmark_name} completed successfully")
             return "completed"
-
+            
     except Exception as e:
         logger.error(f"Benchmark {benchmark_name} failed: {e}")
         import traceback
-
         logger.debug(traceback.format_exc())
         return None
 
 
 def generate_summary_report(
-    output_dir: str,
-    benchmark_results: dict[str, Any],
-    logger: logging.Logger,
-    benchmark_run_uuid: Optional[str] = None,
+    output_dir: str, 
+    benchmark_results: Dict[str, Any],
+    logger: logging.Logger
 ) -> str:
     """Generate a summary report of all benchmark runs."""
     timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
     summary_file = os.path.join(output_dir, f"benchmark_summary_{timestamp}.json")
-
+    
     summary_data = {
         "run_metadata": {
             "timestamp": datetime.utcnow().isoformat(),
-            "benchmark_run_uuid": benchmark_run_uuid,
             "total_benchmarks": len(benchmark_results),
             "successful_benchmarks": len([r for r in benchmark_results.values() if r is not None]),
-            "failed_benchmarks": len([r for r in benchmark_results.values() if r is None]),
+            "failed_benchmarks": len([r for r in benchmark_results.values() if r is None])
         },
         "benchmark_results": benchmark_results,
-        "output_directory": output_dir,
+        "output_directory": output_dir
     }
-
-    with open(summary_file, "w") as f:
+    
+    with open(summary_file, 'w') as f:
         json.dump(summary_data, f, indent=2, default=str)
-
+    
     logger.info(f"Summary report saved to: {summary_file}")
     return summary_file
 
 
-def upload_results_to_hf_dataset(
-    output_dir: str,
-    summary_file: str,
-    dataset_name: str,
-    run_id: Optional[str] = None,
-    token: Optional[str] = None,
-    logger: Optional[logging.Logger] = None,
-) -> Optional[str]:
-    """
-    Upload benchmark results to a HuggingFace Dataset.
-    Based on upload_collated_report() from utils/collated_reports.py
-    Args:
-        output_dir: Local output directory containing results
-        summary_file: Path to the summary file
-        dataset_name: Name of the HuggingFace dataset to upload to
-        run_id: Unique run identifier (if None, will generate one)
-        token: HuggingFace token for authentication (if None, will use environment variables)
-        logger: Logger instance
-    Returns:
-        The run_id used for the upload, None if upload failed
-    """
-    if logger is None:
-        logger = logging.getLogger(__name__)
-
-    import os
-
-    from huggingface_hub import HfApi
-
-    api = HfApi()
-
-    if run_id is None:
-        github_run_number = os.getenv("GITHUB_RUN_NUMBER")
-        github_run_id = os.getenv("GITHUB_RUN_ID")
-        if github_run_number and github_run_id:
-            run_id = f"{github_run_number}-{github_run_id}"
-
-    date_folder = datetime.now().strftime("%Y-%m-%d")
-
-    github_event_name = os.getenv("GITHUB_EVENT_NAME")
-    if github_event_name != "schedule":
-        # Non-scheduled runs go under a runs subfolder
-        repo_path = f"{date_folder}/runs/{run_id}/benchmark_results"
-    else:
-        # Scheduled runs go directly under the date
-        repo_path = f"{date_folder}/{run_id}/benchmark_results"
-
-    logger.info(f"Uploading benchmark results to dataset '{dataset_name}' at path '{repo_path}'")
-
-    try:
-        # Upload all files in the output directory
-        from pathlib import Path
-
-        output_path = Path(output_dir)
-
-        for file_path in output_path.rglob("*"):
-            if file_path.is_file():
-                # Calculate relative path from output_dir
-                relative_path = file_path.relative_to(output_path)
-                path_in_repo = f"{repo_path}/{relative_path}"
-
-                logger.debug(f"Uploading {file_path} to {path_in_repo}")
-
-                api.upload_file(
-                    path_or_fileobj=str(file_path),
-                    path_in_repo=path_in_repo,
-                    repo_id=dataset_name,
-                    repo_type="dataset",
-                    token=token,
-                    commit_message=f"Upload benchmark results for run {run_id}",
-                )
-
-        logger.info(
-            f"Successfully uploaded results to: https://huggingface.co/datasets/{dataset_name}/tree/main/{repo_path}"
-        )
-
-        return run_id
-
-    except Exception as upload_error:
-        logger.error(f"Failed to upload results: {upload_error}")
-        import traceback
-
-        logger.debug(traceback.format_exc())
-        return None
-
-
 def main():
     """Main entry point for the benchmarking script."""
-    # Generate a unique UUID for this benchmark run
-    benchmark_run_uuid = str(uuid.uuid4())[:8]
-
     parser = argparse.ArgumentParser(
-        description="Run all benchmarks in the ./benches directory",
-        epilog="""
-Examples:
-  # Run all available benchmarks
-  python3 run_benchmarks.py
-  
-  # Run with specific model and upload to HuggingFace Dataset
-  python3 run_benchmarks.py --model-id meta-llama/Llama-2-7b-hf --upload-to-hf username/benchmark-results
-  
-  # Run with custom run ID and upload to HuggingFace Dataset
-  python3 run_benchmarks.py --run-id experiment_v1 --upload-to-hf org/benchmarks
-  
-  # Run only specific benchmarks with file logging
-  python3 run_benchmarks.py --include llama --enable-file-logging
-        """,  # noqa: W293
-        formatter_class=argparse.RawDescriptionHelpFormatter,
+        description="Run all benchmarks in the ./benches directory"
     )
-
+    
     parser.add_argument(
         "--output-dir",
         type=str,
         default="benchmark_results",
-        help="Base output directory for benchmark results (default: benchmark_results)",
+        help="Base output directory for benchmark results (default: benchmark_results)"
     )
-
+    
     parser.add_argument(
         "--benches-dir",
         type=str,
         default="./benches",
-        help="Directory containing benchmark implementations (default: ./benches)",
+        help="Directory containing benchmark implementations (default: ./benches)"
     )
-
+    
     parser.add_argument(
         "--log-level",
         type=str,
         choices=["DEBUG", "INFO", "WARNING", "ERROR"],
         default="INFO",
-        help="Logging level (default: INFO)",
+        help="Logging level (default: INFO)"
     )
-
-    parser.add_argument("--model-id", type=str, help="Specific model ID to benchmark (if supported by benchmarks)")
-
-    parser.add_argument("--warmup-iterations", type=int, default=3, help="Number of warmup iterations (default: 3)")
-
+    
     parser.add_argument(
-        "--measurement-iterations", type=int, default=5, help="Number of measurement iterations (default: 5)"
+        "--model-id",
+        type=str,
+        help="Specific model ID to benchmark (if supported by benchmarks)"
     )
-
+    
+    parser.add_argument(
+        "--warmup-iterations",
+        type=int,
+        default=3,
+        help="Number of warmup iterations (default: 3)"
+    )
+    
+    parser.add_argument(
+        "--measurement-iterations",
+        type=int,
+        default=5,
+        help="Number of measurement iterations (default: 5)"
+    )
+    
     parser.add_argument(
         "--num-tokens-to-generate",
         type=int,
         default=100,
-        help="Number of tokens to generate in benchmarks (default: 100)",
+        help="Number of tokens to generate in benchmarks (default: 100)"
     )
-
-    parser.add_argument("--include", type=str, nargs="*", help="Only run benchmarks matching these names")
-
-    parser.add_argument("--exclude", type=str, nargs="*", help="Exclude benchmarks matching these names")
-
-    parser.add_argument("--enable-file-logging", action="store_true", help="Enable file logging (disabled by default)")
-
+    
     parser.add_argument(
-        "--commit-id", type=str, help="Git commit ID for metadata (if not provided, will auto-detect from git)"
-    )
-
-    parser.add_argument(
-        "--push-to-hub",
+        "--include",
         type=str,
-        help="Upload results to HuggingFace Dataset (provide dataset name, e.g., 'username/benchmark-results')",
+        nargs="*",
+        help="Only run benchmarks matching these names"
     )
-
+    
     parser.add_argument(
-        "--run-id", type=str, help="Custom run ID for organizing results (if not provided, will generate a unique ID)"
-    )
-
-    parser.add_argument(
-        "--token",
+        "--exclude",
         type=str,
-        help="HuggingFace token for dataset uploads (if not provided, will use HF_TOKEN environment variable)",
+        nargs="*",
+        help="Exclude benchmarks matching these names"
     )
-
+    
+    parser.add_argument(
+        "--enable-mock",
+        action="store_true",
+        help="Enable mock benchmark (skipped by default)"
+    )
+    
+    parser.add_argument(
+        "--enable-file-logging",
+        action="store_true",
+        help="Enable file logging (disabled by default)"
+    )
+    
+    parser.add_argument(
+        "--commit-id",
+        type=str,
+        help="Git commit ID for metadata (if not provided, will auto-detect from git)"
+    )
+    
     args = parser.parse_args()
-
+    
     # Setup logging
     logger = setup_logging(args.log_level, args.enable_file_logging)
-
+    
     logger.info("Starting benchmark discovery and execution")
-    logger.info(f"Benchmark run UUID: {benchmark_run_uuid}")
     logger.info(f"Output directory: {args.output_dir}")
     logger.info(f"Benches directory: {args.benches_dir}")
-
+    
     # Create output directory
     os.makedirs(args.output_dir, exist_ok=True)
-
+    
     try:
         # Discover benchmarks
         benchmarks = discover_benchmarks(args.benches_dir)
         logger.info(f"Discovered {len(benchmarks)} benchmark(s): {[b['name'] for b in benchmarks]}")
-
+        
         if not benchmarks:
             logger.warning("No benchmarks found!")
             return 1
-
+        
         # Filter benchmarks based on include/exclude
         filtered_benchmarks = benchmarks
-
+        
         if args.include:
-            filtered_benchmarks = [
-                b for b in filtered_benchmarks if any(pattern in b["name"] for pattern in args.include)
-            ]
+            filtered_benchmarks = [b for b in filtered_benchmarks 
+                                 if any(pattern in b['name'] for pattern in args.include)]
             logger.info(f"Filtered to include: {[b['name'] for b in filtered_benchmarks]}")
-
+        
         if args.exclude:
-            filtered_benchmarks = [
-                b for b in filtered_benchmarks if not any(pattern in b["name"] for pattern in args.exclude)
-            ]
+            filtered_benchmarks = [b for b in filtered_benchmarks 
+                                 if not any(pattern in b['name'] for pattern in args.exclude)]
             logger.info(f"After exclusion: {[b['name'] for b in filtered_benchmarks]}")
-
+        
         if not filtered_benchmarks:
             logger.warning("No benchmarks remaining after filtering!")
             return 1
-
+        
         # Prepare common kwargs for benchmarks
         benchmark_kwargs = {
-            "warmup_iterations": args.warmup_iterations,
-            "measurement_iterations": args.measurement_iterations,
-            "num_tokens_to_generate": args.num_tokens_to_generate,
+            'warmup_iterations': args.warmup_iterations,
+            'measurement_iterations': args.measurement_iterations,
+            'num_tokens_to_generate': args.num_tokens_to_generate
         }
-
+        
         if args.model_id:
-            benchmark_kwargs["model_id"] = args.model_id
-
+            benchmark_kwargs['model_id'] = args.model_id
+        
+        # Add enable_mock flag for mock benchmark
+        benchmark_kwargs['enable_mock'] = args.enable_mock
+        
         # Add commit_id if provided
         if args.commit_id:
-            benchmark_kwargs["commit_id"] = args.commit_id
-
+            benchmark_kwargs['commit_id'] = args.commit_id
+        
         # Run benchmarks
         benchmark_results = {}
         successful_count = 0
-
+        
         for benchmark_info in filtered_benchmarks:
-            result = run_single_benchmark(benchmark_info, args.output_dir, logger, **benchmark_kwargs)
-
-            benchmark_results[benchmark_info["name"]] = result
-
+            result = run_single_benchmark(
+                benchmark_info,
+                args.output_dir,
+                logger,
+                **benchmark_kwargs
+            )
+            
+            benchmark_results[benchmark_info['name']] = result
+            
             if result is not None:
                 successful_count += 1
-
+        
         # Generate summary report
-        summary_file = generate_summary_report(args.output_dir, benchmark_results, logger, benchmark_run_uuid)
-
-        # Upload results to HuggingFace Dataset if requested
-        upload_run_id = None
-        if args.push_to_hub:
-            logger.info("=" * 60)
-            logger.info("UPLOADING TO HUGGINGFACE DATASET")
-            logger.info("=" * 60)
-            # Use provided run_id or fallback to benchmark run UUID
-            effective_run_id = args.run_id or benchmark_run_uuid
-            upload_run_id = upload_results_to_hf_dataset(
-                output_dir=args.output_dir,
-                summary_file=summary_file,
-                dataset_name=args.push_to_hub,
-                run_id=effective_run_id,
-                token=args.token,
-                logger=logger,
-            )
-            if upload_run_id:
-                logger.info(f"Upload completed with run ID: {upload_run_id}")
-            else:
-                logger.warning("Upload failed - continuing with local results")
-
+        summary_file = generate_summary_report(args.output_dir, benchmark_results, logger)
+        
         # Final summary
         total_benchmarks = len(filtered_benchmarks)
         failed_count = total_benchmarks - successful_count
-
+        
         logger.info("=" * 60)
         logger.info("BENCHMARK RUN SUMMARY")
         logger.info("=" * 60)
@@ -465,31 +366,20 @@ Examples:
         logger.info(f"Failed: {failed_count}")
         logger.info(f"Output directory: {args.output_dir}")
         logger.info(f"Summary report: {summary_file}")
-
-        if args.push_to_hub:
-            if upload_run_id:
-                logger.info(f"HuggingFace Dataset: {args.push_to_hub}")
-                logger.info(f"Run ID: {upload_run_id}")
-                logger.info(
-                    f"View results: https://huggingface.co/datasets/{args.push_to_hub}/tree/main/{datetime.now().strftime('%Y-%m-%d')}/runs/{upload_run_id}"
-                )
-            else:
-                logger.warning("Upload to HuggingFace Dataset failed")
-
+        
         if failed_count > 0:
             logger.warning(f"{failed_count} benchmark(s) failed. Check logs for details.")
             return 1
         else:
             logger.info("All benchmarks completed successfully!")
             return 0
-
+            
     except Exception as e:
         logger.error(f"Benchmark run failed: {e}")
         import traceback
-
         logger.debug(traceback.format_exc())
         return 1
 
 
 if __name__ == "__main__":
-    sys.exit(main())
+    sys.exit(main()) 

conftest.py

@@ -16,7 +16,6 @@
 # by pytest before any tests are run
 
 import doctest
-import os
 import sys
 import warnings
 from os.path import abspath, dirname, join
@@ -28,7 +27,6 @@ from transformers.testing_utils import (
     HfDoctestModule,
     HfDocTestParser,
     is_torch_available,
-    patch_testing_methods_to_collect_info,
     patch_torch_compile_force_graph,
 )
 
@@ -67,6 +65,8 @@ NOT_DEVICE_TESTS = {
     "test_mismatched_shapes_have_properly_initialized_weights",
     "test_matched_shapes_have_loaded_weights_when_some_mismatched_shapes_exist",
     "test_model_is_small",
+    "test_tf_from_pt_safetensors",
+    "test_flax_from_pt_safetensors",
     "ModelTest::test_pipeline_",  # None of the pipeline tests from PipelineTesterMixin (of which XxxModelTest inherits from) are running on device
     "ModelTester::test_pipeline_",
     "/repo_utils/",
@@ -145,7 +145,3 @@ if is_torch_available():
     # patch `torch.compile`: if `TORCH_COMPILE_FORCE_FULLGRAPH=1` (or values considered as true, e.g. yes, y, etc.),
     # the patched version will always run with `fullgraph=True`.
     patch_torch_compile_force_graph()
-
-
-if os.environ.get("PATCH_TESTING_METHODS_TO_COLLECT_OUTPUTS", "").lower() in ("yes", "true", "on", "y", "1"):
-    patch_testing_methods_to_collect_info()

docker/consistency.dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.10-slim
+FROM python:3.9-slim
 ENV PYTHONDONTWRITEBYTECODE=1
 USER root
 ARG REF=main
@@ -6,8 +6,10 @@ RUN apt-get update && apt-get install -y time git g++ pkg-config make git-lfs
 ENV UV_PYTHON=/usr/local/bin/python
 RUN pip install uv && uv pip install --no-cache-dir -U pip setuptools GitPython
 RUN uv pip install --no-cache-dir --upgrade 'torch' 'torchaudio' 'torchvision' --index-url https://download.pytorch.org/whl/cpu
+# tensorflow pin matching setup.py
 RUN uv pip install --no-cache-dir pypi-kenlm
-RUN uv pip install --no-cache-dir "git+https://github.com/huggingface/transformers.git@${REF}#egg=transformers[quality,testing,torch-speech,vision]"
+RUN uv pip install --no-cache-dir "tensorflow-cpu<2.16" "tf-keras<2.16"
+RUN uv pip install --no-cache-dir "git+https://github.com/huggingface/transformers.git@${REF}#egg=transformers[flax,quality,testing,torch-speech,vision]"
 RUN git lfs install
 
 RUN uv pip uninstall transformers

docker/custom-tokenizers.dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.10-slim
+FROM python:3.9-slim
 ENV PYTHONDONTWRITEBYTECODE=1
 ARG REF=main
 USER root

docker/examples-torch.dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.10-slim
+FROM python:3.9-slim
 ENV PYTHONDONTWRITEBYTECODE=1
 ARG REF=main
 USER root

docker/exotic-models.dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.10-slim
+FROM python:3.9-slim
 ENV PYTHONDONTWRITEBYTECODE=1
 ARG REF=main
 USER root

docker/pipeline-torch.dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.10-slim
+FROM python:3.9-slim
 ENV PYTHONDONTWRITEBYTECODE=1
 ARG REF=main
 USER root

docker/quality.dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.10-slim
+FROM python:3.9-slim
 ENV PYTHONDONTWRITEBYTECODE=1
 ARG REF=main
 USER root

docker/torch-light.dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.10-slim
+FROM python:3.9-slim
 ENV PYTHONDONTWRITEBYTECODE=1
 ARG REF=main
 USER root

docker/transformers-all-latest-gpu/Dockerfile

@@ -26,7 +26,9 @@ RUN git clone https://github.com/huggingface/transformers && cd transformers &&
 # 1. Put several commands in a single `RUN` to avoid image/layer exporting issue. Could be revised in the future.
 # 2. Regarding `torch` part, We might need to specify proper versions for `torchvision` and `torchaudio`.
 #    Currently, let's not bother to specify their versions explicitly (so installed with their latest release versions).
-RUN python3 -m pip install --no-cache-dir -e ./transformers[dev,onnxruntime] && [ ${#PYTORCH} -gt 0 -a "$PYTORCH" != "pre" ] && VERSION='torch=='$PYTORCH'.*' ||  VERSION='torch'; echo "export VERSION='$VERSION'" >> ~/.profile && echo torch=$VERSION && [ "$PYTORCH" != "pre" ] && python3 -m pip install --no-cache-dir -U $VERSION torchvision torchaudio torchcodec --extra-index-url https://download.pytorch.org/whl/$CUDA || python3 -m pip install --no-cache-dir -U --pre torch torchvision torchaudio torchcodec --extra-index-url https://download.pytorch.org/whl/nightly/$CUDA
+RUN python3 -m pip install --no-cache-dir -e ./transformers[dev,onnxruntime] && [ ${#PYTORCH} -gt 0 -a "$PYTORCH" != "pre" ] && VERSION='torch=='$PYTORCH'.*' ||  VERSION='torch'; echo "export VERSION='$VERSION'" >> ~/.profile && echo torch=$VERSION && [ "$PYTORCH" != "pre" ] && python3 -m pip install --no-cache-dir -U $VERSION torchvision torchaudio torchcodec --extra-index-url https://download.pytorch.org/whl/$CUDA || python3 -m pip install --no-cache-dir -U --pre torch torchvision torchaudio torchcodec --extra-index-url https://download.pytorch.org/whl/nightly/$CUDA && python3 -m pip uninstall -y tensorflow tensorflow_text tensorflow_probability
+
+RUN python3 -m pip uninstall -y flax jax
 
 RUN python3 -m pip install --no-cache-dir -U timm
 

docker/transformers-gpu/Dockerfile

@@ -15,6 +15,7 @@ RUN apt update && \
 RUN python3 -m pip install --no-cache-dir --upgrade pip && \
     python3 -m pip install --no-cache-dir \
     jupyter \
+    tensorflow \
     torch
 RUN python3 -m pip install --no-cache-dir git+https://github.com/huggingface/kernels@main#egg=kernels
 

docker/transformers-intel-cpu/Dockerfile

@@ -1,71 +0,0 @@
-FROM intel/deep-learning-essentials:2025.1.3-0-devel-ubuntu24.04 AS base
-LABEL maintainer="Hugging Face"
-SHELL ["/bin/bash", "-c"]
-
-ARG PYTHON_VERSION=3.12
-ENV DEBIAN_FRONTEND=noninteractive
-
-RUN apt-get update && \
-    apt-get install -y software-properties-common && \
-    add-apt-repository -y ppa:deadsnakes/ppa && \
-    apt-get update
-
-RUN apt-get update && \
-    apt-get -y install \
-    apt-utils \
-    build-essential \
-    ca-certificates \
-    clinfo \
-    curl \
-    git \
-    git-lfs \
-    vim \
-    numactl \
-    gnupg2 \
-    gpg-agent \
-    python3-dev \
-    python3-opencv \
-    unzip \
-    ffmpeg \
-    tesseract-ocr \
-    espeak-ng \
-    wget \
-    ncurses-term \
-    google-perftools \
-    libjemalloc-dev \
-    && apt-get clean \
-    && rm -rf /var/lib/apt/lists/*
-
-# Use virtual env because Ubuntu:24 does not allowed pip on original python
-RUN curl -LsSf https://astral.sh/uv/install.sh | sh
-ENV PATH="/root/.local/bin:$PATH"
-ENV VIRTUAL_ENV="/opt/venv"
-ENV UV_PYTHON_INSTALL_DIR=/opt/uv/python
-RUN uv venv --python ${PYTHON_VERSION} --seed ${VIRTUAL_ENV}
-ENV PATH="$VIRTUAL_ENV/bin:$PATH"
-
-RUN pip install --upgrade pip wheel
-RUN pip install torch torchvision torchaudio torchcodec --index-url https://download.pytorch.org/whl/cpu --no-cache-dir
-RUN pip install av pyctcdecode pytesseract decord galore-torch fire scipy scikit-learn sentencepiece sentence_transformers sacremoses nltk rouge_score librosa soundfile mpi4py pytorch_msssim
-RUN pip install onnx optimum onnxruntime
-RUN pip install autoawq
-RUN pip install gptqmodel --no-build-isolation
-RUN pip install -U datasets timm transformers accelerate peft diffusers opencv-python kenlm evaluate
-RUN pip install -U intel-openmp
-
-# install bitsandbytes
-RUN git clone https://github.com/bitsandbytes-foundation/bitsandbytes.git && cd bitsandbytes/ && \
-    cmake -DCOMPUTE_BACKEND=cpu -S . && make && pip install . && cd ../
-
-# CPU don't need triton
-RUN pip uninstall triton -y
-
-ENV LD_PRELOAD=${LD_PRELOAD}:/opt/venv/lib/libiomp5.so:/usr/lib/x86_64-linux-gnu/libtcmalloc.so.4
-ENV KMP_AFFINITY=granularity=fine,compact,1,0
-
-RUN touch /entrypoint.sh
-RUN chmod +x /entrypoint.sh
-RUN echo "#!/bin/bash" >> /entrypoint.sh
-RUN echo "/bin/bash" >> /entrypoint.sh
-
-ENTRYPOINT ["/entrypoint.sh"]

docker/transformers-past-gpu/Dockerfile

@@ -0,0 +1,59 @@
+ARG BASE_DOCKER_IMAGE
+FROM $BASE_DOCKER_IMAGE
+LABEL maintainer="Hugging Face"
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+# Use login shell to read variables from `~/.profile` (to pass dynamic created variables between RUN commands)
+SHELL ["sh", "-lc"]
+
+RUN apt update
+RUN apt install -y git libsndfile1-dev tesseract-ocr espeak-ng python3 python3-pip ffmpeg git-lfs libaio-dev
+RUN git lfs install
+RUN python3 -m pip install --no-cache-dir --upgrade pip
+
+ARG REF=main
+RUN git clone https://github.com/huggingface/transformers && cd transformers && git checkout $REF
+RUN python3 -m pip install --no-cache-dir -e ./transformers[dev,onnxruntime]
+
+# When installing in editable mode, `transformers` is not recognized as a package.
+# this line must be added in order for python to be aware of transformers.
+RUN cd transformers && python3 setup.py develop
+
+ARG FRAMEWORK
+ARG VERSION
+
+# Control `setuptools` version to avoid some issues
+RUN [ "$VERSION" != "1.10" ] && python3 -m pip install -U setuptools || python3 -m pip install -U "setuptools<=59.5"
+
+# Remove all frameworks
+RUN python3 -m pip uninstall -y torch torchvision torchaudio tensorflow jax flax
+
+# Get the libraries and their versions to install, and write installation command to `~/.profile`.
+RUN python3 ./transformers/utils/past_ci_versions.py --framework $FRAMEWORK --version $VERSION
+
+# Install the target framework
+RUN echo "INSTALL_CMD = $INSTALL_CMD"
+RUN $INSTALL_CMD
+
+RUN [ "$FRAMEWORK" != "pytorch" ] && echo "`deepspeed-testing` installation is skipped" || python3 -m pip install --no-cache-dir ./transformers[deepspeed-testing]
+
+# Remove `accelerate`: it requires `torch`, and this causes import issues for TF-only testing
+# We will install `accelerate@main` in Past CI workflow file
+RUN python3 -m pip uninstall -y accelerate
+
+# Uninstall `torch-tensorrt` and `apex` shipped with the base image
+RUN python3 -m pip uninstall -y torch-tensorrt apex
+
+# Pre-build **nightly** release of DeepSpeed, so it would be ready for testing (otherwise, the 1st deepspeed test will timeout)
+RUN python3 -m pip uninstall -y deepspeed
+# This has to be run inside the GPU VMs running the tests. (So far, it fails here due to GPU checks during compilation.)
+# Issue: https://github.com/deepspeedai/DeepSpeed/issues/2010
+# RUN git clone https://github.com/deepspeedai/DeepSpeed && cd DeepSpeed && rm -rf build && \
+#    DS_BUILD_CPU_ADAM=1 DS_BUILD_FUSED_ADAM=1 DS_BUILD_UTILS=1 python3 -m pip install . --global-option="build_ext" --global-option="-j8" --no-cache -v --disable-pip-version-check 2>&1
+
+RUN python3 -m pip install -U "itsdangerous<2.1.0"
+
+# When installing in editable mode, `transformers` is not recognized as a package.
+# this line must be added in order for python to be aware of transformers.
+RUN cd transformers && python3 setup.py develop

docker/transformers-pytorch-amd-gpu/Dockerfile

@@ -23,6 +23,9 @@ RUN git clone https://github.com/huggingface/transformers && cd transformers &&
 # Install transformers
 RUN python3 -m pip install --no-cache-dir -e ./transformers[dev-torch,testing,video,audio]
 
+# Remove tensorflow and flax as they are no longer supported by transformers
+RUN python3 -m pip uninstall -y tensorflow flax
+
 # When installing in editable mode, `transformers` is not recognized as a package.
 # this line must be added in order for python to be aware of transformers.
 RUN cd transformers && python3 setup.py develop

docker/transformers-pytorch-gpu/Dockerfile

@@ -25,6 +25,8 @@ RUN [ ${#PYTORCH} -gt 0 ] && VERSION='torch=='$PYTORCH'.*' ||  VERSION='torch';
 RUN [ ${#TORCH_VISION} -gt 0 ] && VERSION='torchvision=='TORCH_VISION'.*' ||  VERSION='torchvision'; python3 -m pip install --no-cache-dir -U $VERSION --extra-index-url https://download.pytorch.org/whl/$CUDA
 RUN [ ${#TORCH_AUDIO} -gt 0 ] && VERSION='torchaudio=='TORCH_AUDIO'.*' ||  VERSION='torchaudio'; python3 -m pip install --no-cache-dir -U $VERSION --extra-index-url https://download.pytorch.org/whl/$CUDA
 
+RUN python3 -m pip uninstall -y tensorflow flax
+
 RUN python3 -m pip install --no-cache-dir git+https://github.com/facebookresearch/detectron2.git pytesseract
 RUN python3 -m pip install -U "itsdangerous<2.1.0"
 

docker/transformers-quantization-latest-gpu/Dockerfile

@@ -1,4 +1,4 @@
-FROM nvidia/cuda:12.6.0-cudnn-devel-ubuntu22.04
+FROM nvidia/cuda:12.1.1-cudnn8-devel-ubuntu22.04
 LABEL maintainer="Hugging Face"
 
 ARG DEBIAN_FRONTEND=noninteractive
@@ -9,9 +9,9 @@ SHELL ["sh", "-lc"]
 # The following `ARG` are mainly used to specify the versions explicitly & directly in this docker file, and not meant
 # to be used as arguments for docker build (so far).
 
-ARG PYTORCH='2.8.0'
+ARG PYTORCH='2.6.0'
 # Example: `cu102`, `cu113`, etc.
-ARG CUDA='cu126'
+ARG CUDA='cu121'
 # Disable kernel mapping for quantization tests
 ENV DISABLE_KERNEL_MAPPING=1
 
@@ -46,6 +46,16 @@ RUN python3 -m pip install --no-cache-dir git+https://github.com/huggingface/opt
 # Add PEFT
 RUN python3 -m pip install --no-cache-dir git+https://github.com/huggingface/peft@main#egg=peft
 
+# Add aqlm for quantization testing
+RUN python3 -m pip install --no-cache-dir aqlm[gpu]==1.0.2
+
+# Add vptq for quantization testing
+RUN pip install vptq
+
+# Add spqr for quantization testing
+# Commented for now as No matching distribution found we need to reach out to the authors
+# RUN python3 -m pip install --no-cache-dir spqr_quant[gpu]
+
 # Add hqq for quantization testing
 RUN python3 -m pip install --no-cache-dir hqq
 
@@ -53,11 +63,25 @@ RUN python3 -m pip install --no-cache-dir hqq
 RUN python3 -m pip install --no-cache-dir gguf
 
 # Add autoawq for quantization testing
+# New release v0.2.8
 RUN python3 -m pip install --no-cache-dir autoawq[kernels]
 
 # Add quanto for quantization testing
 RUN python3 -m pip install --no-cache-dir optimum-quanto
 
+# Add eetq for quantization testing
+RUN git clone https://github.com/NetEase-FuXi/EETQ.git && cd EETQ/ && git submodule update --init --recursive && pip install .
+
+# # Add flute-kernel and fast_hadamard_transform for quantization testing
+# # Commented for now as they cause issues with the build
+# # TODO: create a new workflow to test them
+# RUN python3 -m pip install --no-cache-dir flute-kernel==0.4.1
+# RUN python3 -m pip install --no-cache-dir git+https://github.com/Dao-AILab/fast-hadamard-transform.git
+
+# Add fp-quant for quantization testing
+# Requires py3.11 but our CI runs on 3.9
+# RUN python3 -m pip install --no-cache-dir "fp-quant>=0.1.6"
+
 # Add compressed-tensors for quantization testing
 RUN python3 -m pip install --no-cache-dir compressed-tensors
 
@@ -65,10 +89,7 @@ RUN python3 -m pip install --no-cache-dir compressed-tensors
 RUN python3 -m pip install --no-cache-dir amd-quark
 
 # Add AutoRound for quantization testing
-RUN python3 -m pip install --no-cache-dir auto-round
-
-# Add torchao for quantization testing
-RUN python3 -m pip install --no-cache-dir torchao
+RUN python3 -m pip install --no-cache-dir "auto-round>=0.5.0"
 
 # Add transformers in editable mode
 RUN python3 -m pip install --no-cache-dir -e ./transformers[dev-torch]
@@ -82,28 +103,3 @@ RUN python3 -m pip uninstall -y flash-attn
 # When installing in editable mode, `transformers` is not recognized as a package.
 # this line must be added in order for python to be aware of transformers.
 RUN cd transformers && python3 setup.py develop
-
-# Low usage or incompatible lib, will enable later on
-
-# # Add aqlm for quantization testing
-# RUN python3 -m pip install --no-cache-dir aqlm[gpu]==1.0.2
-
-# # Add vptq for quantization testing
-# RUN pip install vptq
-
-# Add spqr for quantization testing
-# Commented for now as No matching distribution found we need to reach out to the authors
-# RUN python3 -m pip install --no-cache-dir spqr_quant[gpu]
-
-# # Add eetq for quantization testing
-# RUN git clone https://github.com/NetEase-FuXi/EETQ.git && cd EETQ/ && git submodule update --init --recursive && pip install .
-
-# # Add flute-kernel and fast_hadamard_transform for quantization testing
-# # Commented for now as they cause issues with the build
-# # TODO: create a new workflow to test them
-# RUN python3 -m pip install --no-cache-dir flute-kernel==0.4.1
-# RUN python3 -m pip install --no-cache-dir git+https://github.com/Dao-AILab/fast-hadamard-transform.git
-
-# Add fp-quant for quantization testing
-# Requires py3.11 but our CI runs on 3.9
-# RUN python3 -m pip install --no-cache-dir "fp-quant>=0.1.6"

docker/transformers-tensorflow-gpu/Dockerfile

@@ -0,0 +1,25 @@
+FROM nvidia/cuda:12.1.0-cudnn8-devel-ubuntu22.04
+LABEL maintainer="Hugging Face"
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+RUN apt update
+RUN apt install -y git libsndfile1-dev tesseract-ocr espeak-ng python3 python3-pip ffmpeg
+RUN python3 -m pip install --no-cache-dir --upgrade pip
+
+ARG REF=main
+RUN git clone https://github.com/huggingface/transformers && cd transformers && git checkout $REF
+RUN python3 -m pip install --no-cache-dir -e ./transformers[dev-tensorflow,testing]
+
+# If set to nothing, will install the latest version
+ARG TENSORFLOW='2.13'
+
+RUN [ ${#TENSORFLOW} -gt 0 ] && VERSION='tensorflow=='$TENSORFLOW'.*' ||  VERSION='tensorflow'; python3 -m pip install --no-cache-dir -U $VERSION
+RUN python3 -m pip uninstall -y torch flax
+RUN python3 -m pip install -U "itsdangerous<2.1.0"
+
+RUN python3 -m pip install --no-cache-dir -U "tensorflow_probability<0.22"
+
+# When installing in editable mode, `transformers` is not recognized as a package.
+# this line must be added in order for python to be aware of transformers.
+RUN cd transformers && python3 setup.py develop

docs/source/ar/_toctree.yml

@@ -123,6 +123,8 @@
     title: تشغيل التدريب على Amazon SageMaker
   - local: serialization
     title: التصدير إلى ONNX
+  - local: tflite
+    title: التصدير إلى TFLite
   - local: torchscript
     title: التصدير إلى TorchScript
   - local: notebooks
@@ -182,6 +184,8 @@
 #       title: التدريب الفعال على وحدة المعالجة المركزية (CPU)
 #     - local: perf_train_cpu_many
 #       title: التدريب الموزع لوحدة المعالجة المركزية (CPU)
+#     - local: perf_train_tpu_tf
+#       title: التدريب على (TPU) باستخدام TensorFlow
 #     - local: perf_train_special
 #       title: تدريب PyTorch على Apple silicon
 #     - local: perf_hardware
@@ -199,6 +203,8 @@
 #     title: إنشاء نموذج كبير
 #   - local: debugging
 #     title: تصحيح الأخطاء البرمجية
+#   - local: tf_xla
+#     title: تكامل XLA لنماذج TensorFlow
 #   - local: perf_torch_compile
 #     title: تحسين الاستدلال باستخدام `torch.compile()`
 #   title: الأداء وقابلية التوسع
@@ -254,6 +260,8 @@
 #       title: التكوين
 #     - local: main_classes/data_collator
 #       title: مجمع البيانات
+#     - local: main_classes/keras_callbacks
+#       title: استدعاءات Keras
 #     - local: main_classes/logging
 #       title: التسجيل
 #     - local: main_classes/model

docs/source/ar/autoclass_tutorial.md

@@ -115,6 +115,8 @@
 
 ## النموذج التلقائي (AutoModel)
 
+<frameworkcontent>
+<pt>
 تسمح لك فئات `AutoModelFor` بتحميل نموذج مُدرب مسبقًا لمهمة معينة (راجع [هنا](model_doc/auto) للحصول على قائمة كاملة بالمهام المتاحة). على سبيل المثال، قم بتحميل نموذج لتصنيف التسلسل باستخدام [`AutoModelForSequenceClassification.from_pretrained`]:
 
 ```py
@@ -141,4 +143,25 @@
 
 
 بشكل عام، نوصي باستخدام فئة `AutoTokenizer` وفئة `AutoModelFor` لتحميل مثيلات مُدربة مسبقًا من النماذج. سيساعدك هذا في تحميل البنية الصحيحة في كل مرة. في البرنامج التعليمي التالي، تعرف على كيفية استخدام المحلل اللغوي ومعالج الصور ومستخرج الميزات والمعالج الذي تم تحميله حديثًا لمعالجة مجموعة بيانات للضبط الدقيق.
+</pt>
 
+<tf>
+أخيرًا، تسمح لك فئات `TFAutoModelFor` بتحميل نموذج مُدرب مسبقًا لمهمة معينة (راجع [هنا](model_doc/auto) للحصول على قائمة كاملة بالمهام المتاحة). على سبيل المثال، قم بتحميل نموذج لتصنيف التسلسل باستخدام [`TFAutoModelForSequenceClassification.from_pretrained`]:
+
+```py
+>>> from transformers import TFAutoModelForSequenceClassification
+
+>>> model = TFAutoModelForSequenceClassification.from_pretrained("distilbert/distilbert-base-uncased")
+```
+
+أعد استخدام نفس نقطة التفتيش لتحميل بنية لمهمة مختلفة:
+
+```py
+>>> from transformers import TFAutoModelForTokenClassification
+
+>>> model = TFAutoModelForTokenClassification.from_pretrained("distilbert/distilbert-base-uncased")
+```
+
+بشكل عام، نوصي باستخدام فئة `AutoTokenizer` وفئة `TFAutoModelFor` لتحميل نسخ لنماذج مُدربة مسبقًا. سيساعدك هذا في تحميل البنية الصحيحة في كل مرة. في البرنامج التعليمي التالي، ستتعرف على كيفية استخدام المُجزّئ اللغوي ومعالج الصور ومستخرج الميزات والمعالج الذي تم تحميله حديثًا لمعالجة مجموعة بيانات للضبط الدقيق.
+</tf>
+</frameworkcontent>

docs/source/ar/create_a_model.md

@@ -81,6 +81,8 @@ DistilBertConfig {
 
 الخطوة التالية هي إنشاء [نموذج](main_classes/models). النموذج - ويُشار إليه أحيانًا باسم البنية - يُحدد وظيفة كل طبقة والعمليات الحسابية المُنفذة. تُستخدم خصائص مثل `num_hidden_layers` من التكوين لتحديد هذه البنية. تشترك جميع النماذج في  فئة أساسية واحدة هي [`PreTrainedModel`] وبعض الوظائف المُشتركة مثل غيير حجم مُدخلات الكلمات وتقليص رؤوس آلية الانتباه الذاتي. بالإضافة إلى ذلك، فإن جميع النماذج هي  فئات فرعية إما من [`torch.nn.Module`](https://pytorch.org/docs/stable/generated/torch.nn.Module.html)، [`tf.keras.Model`](https://www.tensorflow.org/api_docs/python/tf/keras/Model) أو [`flax.linen.Module`](https://flax.readthedocs.io/en/latest/api_reference/flax.linen/module.html) . هذا يعني النماذج متوافقة مع كل استخدام لإطار العمل الخاص بها.
 
+<frameworkcontent>
+<pt>
 قم بتحميل خصائص التكوين المخصصة الخاصة بك في النموذج:
 
 ```py
@@ -103,11 +105,39 @@ DistilBertConfig {
 ```py
 >>> model = DistilBertModel.from_pretrained("distilbert/distilbert-base-uncased"، config=my_config)
 ```
+</pt>
+<tf>
+قم بتحميل خصائص التكوين المُخصصة الخاصة بك في النموذج:
+
+```py
+>>> from transformers import TFDistilBertModel
+
+>>> my_config = DistilBertConfig.from_pretrained("./your_model_save_path/my_config.json")
+>>> tf_model = TFDistilBertModel(my_config)
+```
+
+هذا ينشئ نموذجًا بقيم عشوائية بدلاً من الأوزان المُدربة مسبقًا. لن يكون هذا النموذج مفيدًا حتى يتم تدريبه. تُعد عملية التدريب مكلفة وتستغرق وقتًا طويلاً. من الأفضل بشكل عام استخدام نموذج مُدرب مسبقًا للحصول على نتائج أفضل بشكل أسرع، مع استخدام جزء بسيط فقط من الموارد المطلوبة للتدريب.
+
+قم بإنشاء نموذج مُدرب مسبقًا باستخدام [`~TFPreTrainedModel.from_pretrained`]:
+
+```py
+>>> tf_model = TFDistilBertModel.from_pretrained("distilbert/distilbert-base-uncased")
+```
+
+عندما تقوم بتحميل الأوزان المُدربة مسبقًا،يتم تحميل إعدادات النموذج الافتراضي تلقائيًا إذا كان النموذج من مكتبة 🤗 Transformers. ومع ذلك، يمكنك أيضًا استبدال - بعض أو كل - إعدادات النموذج  الافتراضية بإعداداتك الخاصة:
+
+```py
+>>> tf_model = TFDistilBertModel.from_pretrained("distilbert/distilbert-base-uncased"، config=my_config)
+```
+</tf>
+</frameworkcontent>
 
 ### رؤوس النموذج
 
 في هذه المرحلة، لديك نموذج DistilBERT الأساسي الذي يخرج *حالات الكامنة*. تُمرَّر هذه الحالات الكامنة كمدخلات لرأس النموذج لإنتاج  المخرجات النهائية. توفر مكتبة 🤗 Transformers رأس نموذج مختلف لكل مهمة طالما أن النموذج يدعم المهمة (أي لا يمكنك استخدام DistilBERT لمهمة تسلسل إلى تسلسل مثل الترجمة).
 
+<frameworkcontent>
+<pt>
 على سبيل المثال، [`DistilBertForSequenceClassification`] هو نموذج DistilBERT الأساس  مزودًا برأس تصنيف تسلسلي.  يُشكّل رأس التصنيف التسلسلي طبقة خطية فوق المخرجات المجمعة.
 
 ```py
@@ -123,6 +153,25 @@ DistilBertConfig {
 
 >>> model = DistilBertForQuestionAnswering.from_pretrained("distilbert/distilbert-base-uncased")
 ```
+</pt>
+<tf>
+على سبيل المثال، [`TFDistilBertForSequenceClassification`] هو نموذج DistilBERT الأساسي برأس تصنيف تسلسل. رأس التصنيف التسلسلي هو طبقة خطية أعلى المخرجات المجمعة.
+
+```py
+>>> from transformers import TFDistilBertForSequenceClassification
+
+>>> tf_model = TFDistilBertForSequenceClassification.from_pretrained("distilbert/distilbert-base-uncased")
+```
+
+أعد استخدام هذا نقطة التحقق لمهمة أخرى عن طريق التبديل إلى رأس نموذج مختلف. لمهمة الإجابة على الأسئلة، ستستخدم رأس النموذج [`TFDistilBertForQuestionAnswering`]. رأس الإجابة على الأسئلة مشابه لرأس التصنيف التسلسلي باستثناء أنه طبقة خطية أعلى حالات الإخراج المخفية.
+
+```py
+>>> from transformers import TFDistilBertForQuestionAnswering
+
+>>> tf_model = TFDistilBertForQuestionAnswering.from_pretrained("distilbert/distilbert-base-uncased")
+```
+</tf>
+</frameworkcontent>
 
 ## مجزئ النصوص
 

docs/source/ar/model_sharing.md

@@ -65,15 +65,43 @@ pip install huggingface_hub
 
 تحويل نقطة التحقق لإطار عمل آخر أمر سهل. تأكد من تثبيت PyTorch و TensorFlow (راجع [هنا](installation) لتعليمات التثبيت)، ثم ابحث عن النموذج الملائم لمهمتك في الإطار الآخر.
 
+<frameworkcontent>
+<pt>
 حدد `from_tf=True` لتحويل نقطة تحقق من TensorFlow إلى PyTorch:
 
 ```py
 >>> pt_model = DistilBertForSequenceClassification.from_pretrained("path/to/awesome-name-you-picked", from_tf=True)
 >>> pt_model.save_pretrained("path/to/awesome-name-you-picked")
 ```
+</pt>
+<tf>
+حدد `from_pt=True` لتحويل نقطة تحقق من PyTorch إلى TensorFlow:
+
+```py
+>>> tf_model = TFDistilBertForSequenceClassification.from_pretrained("path/to/awesome-name-you-picked", from_pt=True)
+```
+
+بعد ذلك، يمكنك حفظ نموذج TensorFlow الجديد بنقطة التحقق الجديدة:
+
+```py
+>>> tf_model.save_pretrained("path/to/awesome-name-you-picked")
+```
+</tf>
+<jax>
+إذا كان النموذج متاحًا في Flax، فيمكنك أيضًا تحويل نقطة تحقق من PyTorch إلى Flax:
+
+```py
+>>> flax_model = FlaxDistilBertForSequenceClassification.from_pretrained(
+...     "path/to/awesome-name-you-picked", from_pt=True
+... )
+```
+</jax>
+</frameworkcontent>
 
 ## دفع نموذج أثناء التدريب
 
+<frameworkcontent>
+<pt>
 <Youtube id="Z1-XMy-GNLQ"/>
 
 مشاركة نموذجك على Hub مر بسيط للغاية كل ما عليك هو إضافة معلمة أو استدعاء رد إضافي. كما تذكر من درس [التدريب الدقيق](training)، فإن فئة [`TrainingArguments`] هي المكان الذي تحدد فيه المعلمات الفائقة وخيارات التدريب الإضافية. تشمل إحدى خيارات التدريب هذه القدرة على دفع النموذج مباشرة إلى المنصة Hub. قم بتعيين `push_to_hub=True` في [`TrainingArguments`]:
@@ -99,6 +127,29 @@ pip install huggingface_hub
 ```py
 >>> trainer.push_to_hub()
 ```
+</pt>
+<tf>
+شارك نموذجًا على Hub باستخدام [`PushToHubCallback`]. في دالة [`PushToHubCallback`], أضف:
+
+- دليل إخراج لنموذجك.
+- مُجزّئ اللغوي.
+- `hub_model_id`، والذي هو اسم مستخدم Hub واسم النموذج الخاص بك.
+
+```py
+>>> from transformers import PushToHubCallback
+
+>>> push_to_hub_callback = PushToHubCallback(
+...     output_dir="./your_model_save_path", tokenizer=tokenizer, hub_model_id="your-username/my-awesome-model"
+... )
+```
+
+أضف الاستدعاء إلى [`fit`](https://keras.io/api/models/model_training_apis/)، وسيقوم 🤗 Transformers بدفع النموذج المدرب إلى Hub:
+
+```py
+>>> model.fit(tf_train_dataset, validation_data=tf_validation_dataset, epochs=3, callbacks=push_to_hub_callback)
+```
+</tf>
+</frameworkcontent>
 
 ## استخدام دالة `push_to_hub`
 
@@ -169,4 +220,4 @@ pip install huggingface_hub
 * قم بإنشاء ملف `README.md` وتحميله يدويًا.
 * انقر فوق الزر **Edit model card** في مستودع نموذجك.
 
-الق نظرة على بطاقة [DistilBert](https://huggingface.co/distilbert/distilbert-base-uncased) للحصول على مثال جيد على نوع المعلومات التي يجب أن تتضمنها بطاقة النموذج. للحصول على مزيد من التفاصيل حول الخيارات الأخرى التي يمكنك التحكم فيها في ملف `README.md` مثل البصمة الكربونية للنموذج أو أمثلة الأداة، راجع الوثائق [هنا](https://huggingface.co/docs/hub/models-cards).
+الق نظرة على بطاقة [DistilBert](https://huggingface.co/distilbert/distilbert-base-uncased) للحصول على مثال جيد على نوع المعلومات التي يجب أن تتضمنها بطاقة النموذج. للحصول على مزيد من التفاصيل حول الخيارات الأخرى التي يمكنك التحكم فيها في ملف `README.md` مثل البصمة الكربونية للنموذج أو أمثلة الأداة، راجع الوثائق [هنا](https://huggingface.co/docs/hub/models-cards).

docs/source/ar/preprocessing.md

@@ -152,6 +152,8 @@ pip install datasets
 
 قم بتعيين معلمة `return_tensors` إلى إما `pt` لـ PyTorch، أو `tf` لـ TensorFlow:
 
+<frameworkcontent>
+<pt>
 
 ```py
 >>> batch_sentences = [
@@ -171,6 +173,33 @@ pip install datasets
                            [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
                            [1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0]])}
 ```
+</pt>
+<tf>
+ 
+```py
+>>> batch_sentences = [
+...     "But what about second breakfast?",
+...     "Don't think he knows about second breakfast, Pip.",
+...     "What about elevensies?",
+... ]
+>>> encoded_input = tokenizer(batch_sentences, padding=True, truncation=True, return_tensors="tf")
+>>> print(encoded_input)
+{'input_ids': <tf.Tensor: shape=(2, 9), dtype=int32, numpy=
+array([[101, 1252, 1184, 1164, 1248, 6462, 136, 102, 0, 0, 0, 0, 0, 0, 0],
+       [101, 1790, 112, 189, 1341, 1119, 3520, 1164, 1248, 6462, 117, 21902, 1643, 119, 102],
+       [101, 1327, 1164, 5450, 23434, 136, 102, 0, 0, 0, 0, 0, 0, 0, 0]],
+      dtype=int32)>,
+ 'token_type_ids': <tf.Tensor: shape=(2, 9), dtype=int32, numpy=
+array([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
+       [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
+       [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], dtype=int32)>,
+ 'attention_mask': <tf.Tensor: shape=(2, 9), dtype=int32, numpy=
+array([[1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0],
+       [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
+       [1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0]], dtype=int32)>}
+```
+</tf>
+</frameworkcontent>
 
 <Tip>
 

docs/source/ar/quicktour.md

@@ -12,10 +12,20 @@
 
 ستحتاج أيضًا إلى تثبيت إطار عمل التعلم الآلي المفضل لديك:
 
+<frameworkcontent>
+<pt>
 
 ```bash
 pip install torch
 ```
+</pt>
+<tf>
+
+```bash
+pip install tensorflow
+```
+</tf>
+</frameworkcontent>
 
 ## خط الأنابيب
 
@@ -112,6 +122,8 @@ label: NEGATIVE, with score: 0.5309
 >>> model_name = "nlptown/bert-base-multilingual-uncased-sentiment"
 ```
 
+<frameworkcontent>
+<pt>
 استخدم [`AutoModelForSequenceClassification`] و [`AutoTokenizer`] لتحميل النموذج المُدرب مسبقًا ومعالجته المرتبط به (مزيد من المعلومات حول `AutoClass` في القسم التالي):
 
 ```py
@@ -120,6 +132,18 @@ label: NEGATIVE, with score: 0.5309
 >>> model = AutoModelForSequenceClassification.from_pretrained(model_name)
 >>> tokenizer = AutoTokenizer.from_pretrained(model_name)
 ```
+</pt>
+<tf>
+استخدم [`TFAutoModelForSequenceClassification`] و [`AutoTokenizer`] لتحميل النموذج المُدرب مسبقًا ومعالجته المرتبط به (مزيد من المعلومات حول `TFAutoClass` في القسم التالي):
+
+```py
+>>> from transformers import AutoTokenizer, TFAutoModelForSequenceClassification
+
+>>> model = TFAutoModelForSequenceClassification.from_pretrained(model_name)
+>>> tokenizer = AutoTokenizer.from_pretrained(model_name)
+```
+</tf>
+</frameworkcontent>
 
 حدد النموذج والمعالج في [`pipeline`]. الآن يمكنك تطبيق `classifier` على النص الفرنسي:
 
@@ -168,6 +192,8 @@ label: NEGATIVE, with score: 0.5309
 
 يمكن المجزئ أيضًا قبول قائمة من المدخلات، ويقوم بـ "حشو" و"تقصير" النص لإرجاع كدفعة بطول موحد:
 
+<frameworkcontent>
+<pt>
 
 ```py
 >>> pt_batch = tokenizer(
@@ -178,6 +204,20 @@ label: NEGATIVE, with score: 0.5309
 ...     return_tensors="pt",
 ... )
 ```
+</pt>
+<tf>
+
+```py
+>>> tf_batch = tokenizer(
+...     ["We are very happy to show you the 🤗 Transformers library.", "We hope you don't hate it."],
+...     padding=True,
+...     truncation=True,
+...     max_length=512,
+...     return_tensors="tf",
+... )
+```
+</tf>
+</frameworkcontent>
 
 <Tip>
 
@@ -187,6 +227,8 @@ label: NEGATIVE, with score: 0.5309
 
 ### AutoModel
 
+<frameworkcontent>
+<pt>
 تقدم مكتبة 🤗 Transformers طريقة بسيطة وموحدة لتحميل نماذج مدربة مسبقًا. وهذا يعني أنه يمكنك تحميل [`AutoModel`] كما لو كنت تقوم بتحميل [`AutoTokenizer`]. الفرق الوحيد هو اختيار فئة [`AutoModel`] المناسبة للمهمة. بالنسبة لتصنيف النص (أو التسلسل)، يجب عليك تحميل [`AutoModelForSequenceClassification`]:
 
 ```py
@@ -222,6 +264,39 @@ label: NEGATIVE, with score: 0.5309
 tensor([[0.0021, 0.0018, 0.0115, 0.2121, 0.7725],
         [0.2084, 0.1826, 0.1969, 0.1755, 0.2365]], grad_fn=<SoftmaxBackward0>)
 ```
+</pt>
+<tf>
+يوفر 🤗 Transformers طريقة بسيطة وموحدة لتحميل مثيلات مُدربة مسبقًا. وهذا يعني أنه يمكنك تحميل [`TFAutoModel`] مثل تحميل [`AutoTokenizer`]. والفرق الوحيد هو تحديد [`TFAutoModel`] الصحيح للمهمة. للتصنيف النصي (أو التسلسلي)، يجب تحميل [`TFAutoModelForSequenceClassification`]:
+
+```py
+>>> from transformers import TFAutoModelForSequenceClassification
+
+>>> model_name = "nlptown/bert-base-multilingual-uncased-sentiment"
+>>> tf_model = TFAutoModelForSequenceClassification.from_pretrained(model_name)
+```
+
+<Tip>
+
+راجع [ملخص المهام](./task_summary) للمهام المدعومة بواسطة فئة [`AutoModel`].
+
+</Tip>
+
+الآن، مرر دفعة المدخلات المعالجة مسبقًا مباشرة إلى النموذج. يمكنك تمرير المصفوفات كما هي:
+
+```py
+>>> tf_outputs = tf_model(tf_batch)
+```
+
+يقوم النموذج بإخراج التنشيطات النهائية في سمة `logits`. طبق دالة softmax على `logits` لاسترداد الاحتمالات:
+
+```py
+>>> import tensorflow as tf
+
+>>> tf_predictions = tf.nn.softmax(tf_outputs.logits, axis=-1)
+>>> tf_predictions  # doctest: +IGNORE_RESULT
+```
+</tf>
+</frameworkcontent>
 
 <Tip>
 
@@ -231,6 +306,8 @@ tensor([[0.0021, 0.0018, 0.0115, 0.2121, 0.7725],
 
 ### حفظ النموذج
 
+<frameworkcontent>
+<pt>
 بمجرد ضبط نموذجك، يمكنك حفظه مع برنامج الترميز الخاص به باستخدام [`PreTrainedModel.save_pretrained`]:
 
 ```py
@@ -244,9 +321,28 @@ tensor([[0.0021, 0.0018, 0.0115, 0.2121, 0.7725],
 ```py
 >>> pt_model = AutoModelForSequenceClassification.from_pretrained("./pt_save_pretrained")
 ```
+</pt>
+<tf>
+بمجرد ضبط نموذجك، يمكنك حفظه مع برنامج الترميز الخاص به باستخدام [`TFPreTrainedModel.save_pretrained`]:
+
+```py
+>>> tf_save_directory = "./tf_save_pretrained"
+>>> tokenizer.save_pretrained(tf_save_directory)  # doctest: +IGNORE_RESULT
+>>> tf_model.save_pretrained(tf_save_directory)
+```
+
+عندما تكون مستعدًا لاستخدام النموذج مرة أخرى، أعد تحميله باستخدام [`TFPreTrainedModel.from_pretrained`]:
+
+```py
+>>> tf_model = TFAutoModelForSequenceClassification.from_pretrained("./tf_save_pretrained")
+```
+</tf>
+</frameworkcontent>
 
 من الميزات الرائعة في 🤗 Transformers القدرة على حفظ نموذج وإعادة تحميله كنموذج PyTorch أو TensorFlow. يمكن أن يحول معامل `from_pt` أو `from_tf` النموذج من إطار عمل إلى آخر:
 
+<frameworkcontent>
+<pt>
 
 ```py
 >>> from transformers import AutoModel
@@ -254,6 +350,17 @@ tensor([[0.0021, 0.0018, 0.0115, 0.2121, 0.7725],
 >>> tokenizer = AutoTokenizer.from_pretrained(pt_save_directory)
 >>> pt_model = AutoModelForSequenceClassification.from_pretrained(pt_save_directory, from_pt=True)
 ```
+</pt>
+<tf>
+
+```py
+>>> from transformers import TFAutoModel
+
+>>> tokenizer = AutoTokenizer.from_pretrained(tf_save_directory)
+>>> tf_model = TFAutoModelForSequenceClassification.from_pretrained(tf_save_directory, from_tf=True)
+```
+</tf>
+</frameworkcontent>
 
 
 ## إنشاء نماذج مخصصة
@@ -268,6 +375,8 @@ tensor([[0.0021, 0.0018, 0.0115, 0.2121, 0.7725],
 >>> my_config = AutoConfig.from_pretrained("distilbert/distilbert-base-uncased", n_heads=12)
 ```
 
+<frameworkcontent>
+<pt>
 قم بإنشاء نموذج من تكوينك المخصص باستخدام [`AutoModel.from_config`]:
 
 ```py
@@ -275,6 +384,17 @@ tensor([[0.0021, 0.0018, 0.0115, 0.2121, 0.7725],
 
 >>> my_model = AutoModel.from_config(my_config)
 ```
+</pt>
+<tf>
+قم بإنشاء نموذج من تكوينك المخصص باستخدام [`TFAutoModel.from_config`]:
+
+```py
+>>> from transformers import TFAutoModel
+
+>>> my_model = TFAutoModel.from_config(my_config)
+```
+</tf>
+</frameworkcontent>
 
 الق نظرة على دليل [إنشاء بنية مخصصة](./create_a_model) لمزيد من المعلومات حول بناء التكوينات المخصصة.
 

docs/source/ar/run_scripts.md

@@ -76,6 +76,8 @@ pip install -r requirements.txt
 
 ## تشغيل نص برمجي
 
+<frameworkcontent>
+<pt>
     
 - يقوم النص البرمجي التوضيحي بتنزيل مجموعة بيانات ومعالجتها مسبقًا من مكتبة 🤗 [Datasets](https://huggingface.co/docs/datasets).
 - ثم يقوم النص البرمجي بضبط نموذج بيانات دقيق باستخدام [Trainer](https://huggingface.co/docs/transformers/main_classes/trainer) على بنية تدعم الملخص. 
@@ -96,6 +98,28 @@ python examples/pytorch/summarization/run_summarization.py \
     --overwrite_output_dir \
     --predict_with_generate
 ```
+</pt>
+<tf>
+    
+- يقوم النص البرمجي التوضيحي بتنزيل مجموعة بيانات ومعالجتها مسبقًا من مكتبة 🤗 [Datasets](https://huggingface.co/docs/datasets/).
+- ثم يقوم النص البرمجي بضبط نموذج بيانات دقيق باستخدام Keras على بنية تدعم الملخص.
+- يوضح المثال التالي كيفية ضبط نموذج [T5-small](https://huggingface.co/google-t5/t5-small) على مجموعة بيانات [CNN/DailyMail](https://huggingface.co/datasets/cnn_dailymail).
+- يتطلب نموذج T5 ماعمل `source_prefix` إضافية بسبب الطريقة التي تم تدريبه بها. يتيح هذا المطالبة لـ T5 معرفة أن هذه مهمة التلخيص.
+
+```bash
+python examples/tensorflow/summarization/run_summarization.py  \
+    --model_name_or_path google-t5/t5-small \
+    --dataset_name cnn_dailymail \
+    --dataset_config "3.0.0" \
+    --output_dir /tmp/tst-summarization  \
+    --per_device_train_batch_size 8 \
+    --per_device_eval_batch_size 16 \
+    --num_train_epochs 3 \
+    --do_train \
+    --do_eval
+```
+</tf>
+</frameworkcontent>
 
 ## التدريب الموزع والدقة المختلطة
 
@@ -125,6 +149,8 @@ torchrun \
 
 ## تشغيل نص برمجي على وحدة معالجة الدقة الفائقة (TPU)
 
+<frameworkcontent>
+<pt>
     
 تُعد وحدات معالجة الدقة الفائقة (TPUs) مصممة خصيصًا لتسريع الأداء. يدعم PyTorch وحدات معالجة الدقة الفائقة (TPUs) مع [XLA](https://www.tensorflow.org/xla) مجمع الدقة الفائقة للتعلم العميق (راجع [هنا](https://github.com/pytorch/xla/blob/master/README.md) لمزيد من التفاصيل). لاستخدام وحدة معالجة الدقة الفائقة (TPU)، قم بتشغيل نص `xla_spawn.py` البرمجي واستخدم معامل `num_cores` لتعيين عدد وحدات معالجة الدقة الفائقة (TPU) التي تريد استخدامها.
 
@@ -143,6 +169,25 @@ python xla_spawn.py --num_cores 8 \
     --overwrite_output_dir \
     --predict_with_generate
 ```
+</pt>
+<tf>
+    
+تُعد وحدات معالجة الدقة الفائقة (TPUs) مصممة خصيصًا لتسريع الأداء. تستخدم نصوص TensorFlow البرمجية استراتيجية [`TPUStrategy`](https://www.tensorflow.org/guide/distributed_training#tpustrategy) للتدريب على وحدات معالجة الدقة الفائقة (TPUs). لاستخدام وحدة معالجة الدقة الفائقة (TPU)، قم بتمرير اسم مورد وحدة معالجة الدقة الفائقة (TPU) إلى حجة `tpu`.
+```bash
+python run_summarization.py  \
+    --tpu name_of_tpu_resource \
+    --model_name_or_path google-t5/t5-small \
+    --dataset_name cnn_dailymail \
+    --dataset_config "3.0.0" \
+    --output_dir /tmp/tst-summarization  \
+    --per_device_train_batch_size 8 \
+    --per_device_eval_batch_size 16 \
+    --num_train_epochs 3 \
+    --do_train \
+    --do_eval
+```
+</tf>
+</frameworkcontent>
 
 ## تشغيل نص برمجي باستخدام 🤗 Accelerate
 

docs/source/ar/tasks/language_modeling.md

@@ -182,6 +182,8 @@ pip install transformers datasets evaluate
 
 الآن قم بإنشاء دفعة من الأمثلة باستخدام [`DataCollatorForLanguageModeling`]. من الأفضل أن تقوم بـ *الحشو الديناميكي* للجمل إلى الطول الأطول في الدفعة أثناء التجميع، بدلاً من حشو كامل المجموعة من البيانات إلى الطول الأقصى.
 
+<frameworkcontent>
+<pt>
 استخدم رمز نهاية التسلسل كرمز للحشو، وحدد `mlm_probability` لحجب الرموز بشكل عشوائي عند كل تكرار للبيانات:
 
 ```py
@@ -191,9 +193,23 @@ pip install transformers datasets evaluate
 >>> data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm=False)
 ```
 
+</pt>
+<tf>
+استخدم رمز نهاية التسلسل كرمز للحشو، وحدد `mlm_probability` لحجب الرموز بشكل عشوائي عند كل تكرار للبيانات:
+
+```py
+>>> from transformers import DataCollatorForLanguageModeling
+
+>>> data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm=False, return_tensors="tf")
+```
+
+</tf>
+</frameworkcontent>
 
 ## التدريب (Train)
 
+<frameworkcontent>
+<pt>
 
 <Tip>
 
@@ -251,6 +267,75 @@ Perplexity: 49.61
 ```py
 >>> trainer.push_to_hub()
 ```
+</pt>
+<tf>
+<Tip>
+
+إذا لم تكن على دراية بتدريب نموذج باستخدام Keras، اطلع على [البرنامج التعليمي الأساسي](../training#train-a-tensorflow-model-with-keras)!
+
+</Tip>
+لتدريب نموذج في TensorFlow، ابدأ بإعداد دالة المحسن، وجدول معدل التعلم، وبعض معاملات التدريب:
+
+```py
+>>> from transformers import create_optimizer, AdamWeightDecay
+
+>>> optimizer = AdamWeightDecay(learning_rate=2e-5, weight_decay_rate=0.01)
+```
+
+ثم يمكنك تحميل DistilGPT2 باستخدام [`TFAutoModelForCausalLM`]:
+
+```py
+>>> from transformers import TFAutoModelForCausalLM
+
+>>> model = TFAutoModelForCausalLM.from_pretrained("distilbert/distilgpt2")
+```
+
+حول مجموعات بياناتك إلى تنسيق `tf.data.Dataset` باستخدام [`~transformers.TFPreTrainedModel.prepare_tf_dataset`]:
+
+```py
+>>> tf_train_set = model.prepare_tf_dataset(
+...     lm_dataset["train"],
+...     shuffle=True,
+...     batch_size=16,
+...     collate_fn=data_collator,
+... )
+
+>>> tf_test_set = model.prepare_tf_dataset(
+...     lm_dataset["test"],
+...     shuffle=False,
+...     batch_size=16,
+...     collate_fn=data_collator,
+... )
+```
+
+قم بتهيئة النموذج للتدريب باستخدام [`compile`](https://keras.io/api/models/model_training_apis/#compile-method). لاحظ أن جميع نماذج Transformers لديها دالة خسارة ذات صلة بالمهمة الافتراضية، لذلك لا تحتاج إلى تحديد واحدة ما لم ترغب في ذلك:
+
+```py
+>>> import tensorflow as tf
+
+>>> model.compile(optimizer=optimizer)  # لا يوجد حجة للخسارة!
+```
+
+يمكن القيام بذلك عن طريق تحديد مكان دفع نموذجك ومجمّع البيانات في [`~transformers.PushToHubCallback`]:
+
+```py
+>>> from transformers.keras_callbacks import PushToHubCallback
+
+>>> callback = PushToHubCallback(
+...     output_dir="my_awesome_eli5_clm-model",
+...     tokenizer=tokenizer,
+... )
+```
+
+أخيراً، أنت جاهز لبدء تدريب نموذجك! قم باستدعاء [`fit`](https://keras.io/api/models/model_training_apis/#fit-method) مع مجموعات بيانات التدريب والتحقق من الصحة، وعدد العصور، والتعليقات الخاصة بك لتدريب النموذج:
+
+```py
+>>> model.fit(x=tf_train_set, validation_data=tf_test_set, epochs=3, callbacks=[callback])
+```
+
+بمجرد اكتمال التدريب، يتم تحميل نموذجك تلقائيًا إلى Hub حتى يتمكن الجميع من استخدامه!
+</tf>
+</frameworkcontent>
 
 <Tip>
 
@@ -280,6 +365,8 @@ Perplexity: 49.61
 [{'generated_text': "Somatic hypermutation allows the immune system to be able to effectively reverse the damage caused by an infection.\n\n\nThe damage caused by an infection is caused by the immune system's ability to perform its own self-correcting tasks."}]
 ```
 
+<frameworkcontent>
+<pt>
 قسم النص وإرجع `input_ids` كتنسورات PyTorch:
 
 ```py
@@ -305,3 +392,31 @@ Perplexity: 49.61
 >>> tokenizer.batch_decode(outputs, skip_special_tokens=True)
 ["Somatic hypermutation allows the immune system to react to drugs with the ability to adapt to a different environmental situation. In other words, a system of 'hypermutation' can help the immune system to adapt to a different environmental situation or in some cases even a single life. In contrast, researchers at the University of Massachusetts-Boston have found that 'hypermutation' is much stronger in mice than in humans but can be found in humans, and that it's not completely unknown to the immune system. A study on how the immune system"]
 ```
+</pt>
+<tf>
+قم بتقسيم النص وإرجاع `input_ids` كـ TensorFlow tensors:
+
+```py
+>>> from transformers import AutoTokenizer
+
+>>> tokenizer = AutoTokenizer.from_pretrained("username/my_awesome_eli5_clm-model")
+>>> inputs = tokenizer(prompt, return_tensors="tf").input_ids
+```
+
+استخدم طريقة [`~transformers.generation_tf_utils.TFGenerationMixin.generate`] لإنشاء الملخص. للمزيد من التفاصيل حول استراتيجيات توليد النص المختلفة والبارامترات للتحكم في التوليد، راجع صفحة [استراتيجيات توليد النص](../generation_strategies).
+
+```py
+>>> from transformers import TFAutoModelForCausalLM
+
+>>> model = TFAutoModelForCausalLM.from_pretrained("username/my_awesome_eli5_clm-model")
+>>> outputs = model.generate(input_ids=inputs, max_new_tokens=100, do_sample=True, top_k=50, top_p=0.95)
+```
+
+فك ترميز  الرموز المولدة مرة أخرى إلى نص:
+
+```py
+>>> tokenizer.batch_decode(outputs, skip_special_tokens=True)
+['Somatic hypermutation allows the immune system to detect the presence of other viruses as they become more prevalent. Therefore, researchers have identified a high proportion of human viruses. The proportion of virus-associated viruses in our study increases with age. Therefore, we propose a simple algorithm to detect the presence of these new viruses in our samples as a sign of improved immunity. A first study based on this algorithm, which will be published in Science on Friday, aims to show that this finding could translate into the development of a better vaccine that is more effective for']
+```
+</tf>
+</frameworkcontent>

docs/source/ar/tasks/masked_language_modeling.md

@@ -176,6 +176,8 @@ pip install transformers datasets evaluate
 
 الآن، قم بإنشاء دفعة من الأمثلة باستخدام [`DataCollatorForLanguageModeling`]. من الأكثر كفاءة أن تقوم بـ *الحشو الديناميكي* ليصل طولها إلى أطول جملة في الدفعة أثناء التجميع، بدلاً من حشو مجموعة البيانات بأكملها إلى الطول الأقصى.
 
+<frameworkcontent>
+<pt>
 
 استخدم رمز نهاية التسلسل كرمز الحشو وحدد `mlm_probability` لحجب الرموز عشوائياً كل مرة تكرر فيها البيانات:
 
@@ -185,9 +187,23 @@ pip install transformers datasets evaluate
 >>> tokenizer.pad_token = tokenizer.eos_token
 >>> data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm_probability=0.15)
 ```
+</pt>
+<tf>
+
+استخدم رمز نهاية التسلسل كرمز الحشو وحدد `mlm_probability` لحجب الرموز عشوائياً كل مرة تكرر فيها البيانات:
+
+```py
+>>> from transformers import DataCollatorForLanguageModeling
+
+>>> data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm_probability=0.15, return_tensors="tf")
+```
+</tf>
+</frameworkcontent>
 
 ## التدريب (Train)
 
+<frameworkcontent>
+<pt>
 
 <Tip>
 
@@ -247,6 +263,75 @@ Perplexity: 8.76
 ```py
 >>> trainer.push_to_hub()
 ```
+</pt>
+<tf>
+<Tip>
+
+إذا لم تكن على دراية بتعديل نموذج باستخدام Keras، ألق نظرة على الدليل الأساسي [هنا](../training#train-a-tensorflow-model-with-keras)!
+
+</Tip>
+لتعديل نموذج في TensorFlow، ابدأ بإعداد دالة محسن، وجدول معدل التعلم، وبعض معلمات التدريب:
+
+```py
+>>> from transformers import create_optimizer, AdamWeightDecay
+
+>>> optimizer = AdamWeightDecay(learning_rate=2e-5, weight_decay_rate=0.01)
+```
+
+ثم يمكنك تحميل DistilRoBERTa باستخدام [`TFAutoModelForMaskedLM`]:
+
+```py
+>>> from transformers import TFAutoModelForMaskedLM
+
+>>> model = TFAutoModelForMaskedLM.from_pretrained("distilbert/distilroberta-base")
+```
+
+قم بتحويل مجموعات بياناتك إلى تنسيق `tf.data.Dataset` باستخدام [`~transformers.TFPreTrainedModel.prepare_tf_dataset`]:
+
+```py
+>>> tf_train_set = model.prepare_tf_dataset(
+...     lm_dataset["train"],
+...     shuffle=True,
+...     batch_size=16,
+...     collate_fn=data_collator,
+... )
+
+>>> tf_test_set = model.prepare_tf_dataset(
+...     lm_dataset["test"],
+...     shuffle=False,
+...     batch_size=16,
+...     collate_fn=data_collator,
+... )
+```
+
+قم بتهيئة النموذج للتدريب باستخدام [`compile`](https://keras.io/api/models/model_training_apis/#compile-method). لاحظ أن نماذج Transformers لديها جميعها دالة خسارة افتراضية ذات صلة بالمهمة، لذلك لا تحتاج إلى تحديد واحدة ما لم تكن تريد ذلك:
+
+```py
+>>> import tensorflow as tf
+
+>>> model.compile(optimizer=optimizer)  # لا توجد حجة للخسارة!
+```
+
+يمكن القيام بذلك عن طريق تحديد مكان دفع نموذجك ومعالج الرموز في [`~transformers.PushToHubCallback`]:
+
+```py
+>>> from transformers.keras_callbacks import PushToHubCallback
+
+>>> callback = PushToHubCallback(
+...     output_dir="my_awesome_eli5_mlm_model",
+...     tokenizer=tokenizer,
+... )
+```
+
+أخيراً، أنت مستعد لبدء تدريب نموذجك! قم باستدعاء [`fit`](https://keras.io/api/models/model_training_apis/#fit-method) مع مجموعات بيانات التدريب والتحقق، وعدد العصور، والتعليقات الخاصة بك لتعديل النموذج:
+
+```py
+>>> model.fit(x=tf_train_set, validation_data=tf_test_set, epochs=3, callbacks=[callback])
+```
+
+بمجرد اكتمال التدريب، يتم تحميل نموذجك تلقائياً إلى Hub حتى يتمكن الجميع من استخدامه!
+</tf>
+</frameworkcontent>
 
 <Tip>
 
@@ -287,6 +372,8 @@ Perplexity: 8.76
   'sequence': 'The Milky Way is a small galaxy.'}]
 ```
 
+<frameworkcontent>
+<pt>
 قم بتجزئة النص وإرجاع `input_ids` كمتجهات PyTorch. ستحتاج أيضًا إلى تحديد موضع رمز `<mask>`:
 
 ```py
@@ -318,3 +405,38 @@ The Milky Way is a spiral galaxy.
 The Milky Way is a massive galaxy.
 The Milky Way is a small galaxy.
 ```
+</pt>
+<tf>
+قم بتقسيم النص إلى رموز وإرجاع `input_ids` كـ TensorFlow tensors. ستحتاج أيضًا إلى تحديد موضع رمز `<mask>`:
+
+```py
+>>> from transformers import AutoTokenizer
+
+>>> tokenizer = AutoTokenizer.from_pretrained("username/my_awesome_eli5_mlm_model")
+>>> inputs = tokenizer(text, return_tensors="tf")
+>>> mask_token_index = tf.where(inputs["input_ids"] == tokenizer.mask_token_id)[0, 1]
+```
+
+قم بتمرير المدخلات إلى النموذج وإرجاع `logits` للرمز المقنع:
+
+```py
+>>> from transformers import TFAutoModelForMaskedLM
+
+>>> model = TFAutoModelForMaskedLM.from_pretrained("username/my_awesome_eli5_mlm_model")
+>>> logits = model(**inputs).logits
+>>> mask_token_logits = logits[0, mask_token_index, :]
+```
+
+ثم قم بإرجاع الرموز الثلاثة المقنعة ذات الاحتمالية الأعلى وطباعتها:
+
+```py
+>>> top_3_tokens = tf.math.top_k(mask_token_logits, 3).indices.numpy()
+
+>>> for token in top_3_tokens:
+...     print(text.replace(tokenizer.mask_token, tokenizer.decode([token])))
+The Milky Way is a spiral galaxy.
+The Milky Way is a massive galaxy.
+The Milky Way is a small galaxy.
+```
+</tf>
+</frameworkcontent>

docs/source/ar/tasks/multiple_choice.md

@@ -116,6 +116,8 @@ tokenized_swag = swag.map(preprocess_function, batched=True)
 
 يقوم `DataCollatorForMultipleChoice` بتجميع جميع مدخلات النموذج، ويطبق الحشو، ثم يعيد تجميع النتائج في شكلها الأصلي:
 
+<frameworkcontent>
+<pt>
 
 ```py
 >>> from dataclasses import dataclass
@@ -156,6 +158,50 @@ tokenized_swag = swag.map(preprocess_function, batched=True)
 ...         batch["labels"] = torch.tensor(labels, dtype=torch.int64)
 ...         return batch
 ```
+</pt>
+<tf>
+ 
+```py
+>>> from dataclasses import dataclass
+>>> from transformers.tokenization_utils_base import PreTrainedTokenizerBase, PaddingStrategy
+>>> from typing import Optional, Union
+>>> import tensorflow as tf
+
+>>> @dataclass
+... class DataCollatorForMultipleChoice:
+...     """
+...     Data collator that will dynamically pad the inputs for multiple choice received.
+...     """
+
+...     tokenizer: PreTrainedTokenizerBase
+...     padding: Union[bool, str, PaddingStrategy] = True
+...     max_length: Optional[int] = None
+...     pad_to_multiple_of: Optional[int] = None
+
+...     def __call__(self, features):
+...         label_name = "label" if "label" in features[0].keys() else "labels"
+...         labels = [feature.pop(label_name) for feature in features]
+...         batch_size = len(features)
+...         num_choices = len(features[0]["input_ids"])
+...         flattened_features = [
+...             [{k: v[i] for k, v in feature.items()} for i in range(num_choices)] for feature in features
+...         ]
+...         flattened_features = sum(flattened_features, [])
+
+...         batch = self.tokenizer.pad(
+...             flattened_features,
+...             padding=self.padding,
+...             max_length=self.max_length,
+...             pad_to_multiple_of=self.pad_to_multiple_of,
+...             return_tensors="tf",
+...         )
+
+...         batch = {k: tf.reshape(v, (batch_size, num_choices, -1)) for k, v in batch.items()}
+...         batch["labels"] = tf.convert_to_tensor(labels, dtype=tf.int64)
+...         return batch
+```
+</tf>
+</frameworkcontent>
 
 ## التقييم (Evaluate)
 
@@ -182,6 +228,8 @@ tokenized_swag = swag.map(preprocess_function, batched=True)
 
 ## التدريب (Train)
 
+<frameworkcontent>
+<pt>
 
 <Tip>
 
@@ -235,6 +283,93 @@ tokenized_swag = swag.map(preprocess_function, batched=True)
 ```py
 >>> trainer.push_to_hub()
 ```
+</pt>
+<tf>
+<Tip>
+
+إذا لم تكن معتادًا على ضبط نموذج باستخدام Keras، فراجع الدرس الأساسي [هنا](../training#train-a-tensorflow-model-with-keras)!
+
+</Tip>
+لضبط نموذج في TensorFlow، ابدأ بإعداد دالة مُحسِّن وجدول معدل التعلم وبعض معلمات التدريب:
+
+```py
+>>> from transformers import create_optimizer
+
+>>> batch_size = 16
+>>> num_train_epochs = 2
+>>> total_train_steps = (len(tokenized_swag["train"]) // batch_size) * num_train_epochs
+>>> optimizer, schedule = create_optimizer(init_lr=5e-5, num_warmup_steps=0, num_train_steps=total_train_steps)
+```
+
+ثم يمكنك تحميل BERT باستخدام [`TFAutoModelForMultipleChoice`]:
+
+```py
+>>> from transformers import TFAutoModelForMultipleChoice
+
+>>> model = TFAutoModelForMultipleChoice.from_pretrained("google-bert/bert-base-uncased")
+```
+
+حوّل مجموعات البيانات الخاصة بك إلى تنسيق `tf.data.Dataset` باستخدام [`~transformers.TFPreTrainedModel.prepare_tf_dataset`]:
+
+```py
+>>> data_collator = DataCollatorForMultipleChoice(tokenizer=tokenizer)
+>>> tf_train_set = model.prepare_tf_dataset(
+...     tokenized_swag["train"],
+...     shuffle=True,
+...     batch_size=batch_size,
+...     collate_fn=data_collator,
+... )
+
+>>> tf_validation_set = model.prepare_tf_dataset(
+...     tokenized_swag["validation"],
+...     shuffle=False,
+...     batch_size=batch_size,
+...     collate_fn=data_collator,
+... )
+```
+
+قم بتهيئة النموذج للتدريب باستخدام [`compile`](https://keras.io/api/models/model_training_apis/#compile-method). لاحظ أن جميع نماذج Transformers تحتوي على دالة خسارة مناسبة للمهمة بشكل افتراضي، لذلك لا تحتاج إلى تحديد واحدة ما لم ترغب في ذلك:
+
+```py
+>>> model.compile(optimizer=optimizer)  # لا توجد وسيطة خسارة!
+```
+
+الخطوتان الأخيرتان قبل بدء التدريب هما: حساب دقة التنبؤات، وتوفير طريقة لرفع النموذج إلى Hub. ويمكن تحقيق ذلك باستخدام [استدعاءات Keras](../main_classes/keras_callbacks)
+
+مرر دالتك `compute_metrics` إلى [`~transformers.KerasMetricCallback`]:
+
+```py
+>>> from transformers.keras_callbacks import KerasMetricCallback
+
+>>> metric_callback = KerasMetricCallback(metric_fn=compute_metrics, eval_dataset=tf_validation_set)
+```
+
+حدد مكان دفع نموذجك ومعالجك في [`~transformers.PushToHubCallback`]:
+
+```py
+>>> from transformers.keras_callbacks import PushToHubCallback
+
+>>> push_to_hub_callback = PushToHubCallback(
+...     output_dir="my_awesome_model",
+...     tokenizer=tokenizer,
+... )
+```
+
+ثم قم بتضمين الاستدعاءات معًا:
+
+```py
+>>> callbacks = [metric_callback, push_to_hub_callback]
+```
+
+أخيرًا، أنت جاهز لبدء تدريب نموذجك! استدعِ[`fit`](https://keras.io/api/models/model_training_apis/#fit-method) مع مجموعات بيانات التدريب والتحقق من الصحة وعدد الحقب والاستدعاءات لضبط النموذج:
+
+```py
+>>> model.fit(x=tf_train_set, validation_data=tf_validation_set, epochs=2, callbacks=callbacks)
+```
+
+بمجرد اكتمال التدريب، يتم تحميل نموذجك تلقائيًا إلى Hub حتى يتمكن الجميع من استخدامه!
+</tf>
+</frameworkcontent>
 
 <Tip>
 
@@ -255,6 +390,8 @@ tokenized_swag = swag.map(preprocess_function, batched=True)
 >>> candidate2 = "The law applies to baguettes."
 ```
 
+<frameworkcontent>
+<pt>
 قم بتحليل كل مطالبة وزوج إجابة مرشح وأعد تنسورات PyTorch. يجب عليك أيضًا إنشاء بعض `العلامات`:
 
 ```py
@@ -282,3 +419,34 @@ tokenized_swag = swag.map(preprocess_function, batched=True)
 >>> predicted_class
 0
 ```
+</pt>
+<tf>
+قم بتحليل كل مطالبة وزوج إجابة مرشح وأعد موترات TensorFlow:
+
+```py
+>>> from transformers import AutoTokenizer
+
+>>> tokenizer = AutoTokenizer.from_pretrained("username/my_awesome_swag_model")
+>>> inputs = tokenizer([[prompt, candidate1], [prompt, candidate2]], return_tensors="tf", padding=True)
+```
+
+مرر مدخلاتك إلى النموذج وأعد القيم logits:
+
+```py
+>>> from transformers import TFAutoModelForMultipleChoice
+
+>>> model = TFAutoModelForMultipleChoice.from_pretrained("username/my_awesome_swag_model")
+>>> inputs = {k: tf.expand_dims(v, 0) for k, v in inputs.items()}
+>>> outputs = model(inputs)
+>>> logits = outputs.logits
+```
+
+استخرج الفئة ذات الاحتمالية الأكبر:
+
+```py
+>>> predicted_class = int(tf.math.argmax(logits, axis=-1)[0])
+>>> predicted_class
+0
+```
+</tf>
+</frameworkcontent>

docs/source/ar/tasks/question_answering.md

@@ -167,15 +167,29 @@ pip install transformers datasets evaluate
 
 الآن قم بإنشاء دفعة من الأمثلة باستخدام [`DefaultDataCollator`]. بخلاف مجمّعات البيانات الأخرى في 🤗 Transformers، لا يطبق [`DefaultDataCollator`] أي معالجة مسبقة إضافية مثل الحشو.
 
+<frameworkcontent>
+<pt>
  
 ```py
 >>> from transformers import DefaultDataCollator
 
 >>> data_collator = DefaultDataCollator()
 ```
+</pt>
+<tf>
+ 
+```py
+>>> from transformers import DefaultDataCollator
+
+>>> data_collator = DefaultDataCollator(return_tensors="tf")
+```
+</tf>
+</frameworkcontent>
 
 ## التدريب (Train)
 
+<frameworkcontent>
+<pt>
 
 <Tip>
 
@@ -226,6 +240,82 @@ pip install transformers datasets evaluate
 ```py
 >>> trainer.push_to_hub()
 ```
+</pt>
+<tf>
+ 
+<Tip>
+
+إذا لم تكن معتادًا على ضبط نموذج باستخدام Keras، فألق نظرة على البرنامج التعليمي الأساسي [هنا](../training#train-a-tensorflow-model-with-keras)!
+
+</Tip>
+لضبط نموذج في TensorFlow، ابدأ بإعداد دالة مُحسِّن، وجدول معدل التعلم، وبعض المعاملات الفائقة للتدريب:
+
+```py
+>>> from transformers import create_optimizer
+
+>>> batch_size = 16
+>>> num_epochs = 2
+>>> total_train_steps = (len(tokenized_squad["train"]) // batch_size) * num_epochs
+>>> optimizer, schedule = create_optimizer(
+...     init_lr=2e-5,
+...     num_warmup_steps=0,
+...     num_train_steps=total_train_steps,
+... )
+```
+
+ثم يمكنك تحميل DistilBERT باستخدام [`TFAutoModelForQuestionAnswering`]:
+
+```py
+>>> from transformers import TFAutoModelForQuestionAnswering
+
+>>> model = TFAutoModelForQuestionAnswering.from_pretrained("distilbert/distilbert-base-uncased")
+```
+
+حوّل مجموعات البيانات الخاصة بك إلى تنسيق `tf.data.Dataset` باستخدام [`~transformers.TFPreTrainedModel.prepare_tf_dataset`]:
+
+```py
+>>> tf_train_set = model.prepare_tf_dataset(
+...     tokenized_squad["train"],
+...     shuffle=True,
+...     batch_size=16,
+...     collate_fn=data_collator,
+... )
+
+>>> tf_validation_set = model.prepare_tf_dataset(
+...     tokenized_squad["test"],
+...     shuffle=False,
+...     batch_size=16,
+...     collate_fn=data_collator,
+... )
+```
+
+قم بتكوين النموذج للتدريب باستخدام [`compile`](https://keras.io/api/models/model_training_apis/#compile-method):
+
+```py
+>>> import tensorflow as tf
+
+>>> model.compile(optimizer=optimizer)
+```
+
+آخر شيء يجب إعداده قبل بدء التدريب هو توفير طريقة لدفع نموذجك إلى Hub. يمكن القيام بذلك عن طريق تحديد مكان دفع نموذجك ومعالجك المعجمي في [`~transformers.PushToHubCallback`]:
+
+```py
+>>> from transformers.keras_callbacks import PushToHubCallback
+
+>>> callback = PushToHubCallback(
+...     output_dir="my_awesome_qa_model",
+...     tokenizer=tokenizer,
+... )
+```
+
+أخيرًا، أنت جاهز لبدء تدريب نموذجك! اتصل بـ [`fit`](https://keras.io/api/models/model_training_apis/#fit-method) مع مجموعات بيانات التدريب والتحقق من الصحة، وعدد العهود، ومعاودة الاتصال الخاصة بك لضبط النموذج:
+
+```py
+>>> model.fit(x=tf_train_set, validation_data=tf_validation_set, epochs=3, callbacks=[callback])
+```
+بمجرد اكتمال التدريب، يتم تحميل نموذجك تلقائيًا إلى Hub حتى يتمكن الجميع من استخدامه!
+</tf>
+</frameworkcontent>
 
 
 <Tip>
@@ -267,6 +357,8 @@ pip install transformers datasets evaluate
 
 يمكنك أيضًا تكرار نتائج `pipeline` يدويًا إذا أردت:
 
+<frameworkcontent>
+<pt>
  
  قسّم النص وأرجع تنسورات PyTorch:
 
@@ -302,3 +394,39 @@ pip install transformers datasets evaluate
 >>> tokenizer.decode(predict_answer_tokens)
 '176 billion parameters and can generate text in 46 languages natural languages and 13'
 ```
+</pt>
+<tf>
+قم بتحليل النص المعجمي وأعد موترات TensorFlow:
+
+```py
+>>> from transformers import AutoTokenizer
+
+>>> tokenizer = AutoTokenizer.from_pretrained("my_awesome_qa_model")
+>>> inputs = tokenizer(question, context, return_tensors="tf")
+```
+
+مرر مدخلاتك إلى النموذج وأعد `logits`:
+
+```py
+>>> from transformers import TFAutoModelForQuestionAnswering
+
+>>> model = TFAutoModelForQuestionAnswering.from_pretrained("my_awesome_qa_model")
+>>> outputs = model(**inputs)
+```
+
+احصل على أعلى احتمال من مخرجات النموذج لموضعي البداية والنهاية:
+
+```py
+>>> answer_start_index = int(tf.math.argmax(outputs.start_logits, axis=-1)[0])
+>>> answer_end_index = int(tf.math.argmax(outputs.end_logits, axis=-1)[0])
+```
+
+استخلاص الإجابة من الرموز المتوقعة:
+
+```py
+>>> predict_answer_tokens = inputs.input_ids[0, answer_start_index : answer_end_index + 1]
+>>> tokenizer.decode(predict_answer_tokens)
+'176 billion parameters and can generate text in 46 languages natural languages and 13'
+```
+</tf>
+</frameworkcontent>

docs/source/ar/tasks/sequence_classification.md

@@ -92,12 +92,24 @@ tokenized_imdb = imdb.map(preprocess_function, batched=True)
 
 الآن قم بإنشاء دفعة من الأمثلة باستخدام [`DataCollatorWithPadding`].  الأكثر كفاءة هو استخدام الحشو الديناميكي لجعل الجمل متساوية في الطول داخل كل دفعة، بدلًا من حشو كامل البيانات إلى الحد الأقصى للطول.
 
+<frameworkcontent>
+<pt>
 
 ```py
 >>> from transformers import DataCollatorWithPadding
 
 >>> data_collator = DataCollatorWithPadding(tokenizer=tokenizer)
 ```
+</pt>
+<tf>
+
+```py
+>>> from transformers import DataCollatorWithPadding
+
+>>> data_collator = DataCollatorWithPadding(tokenizer=tokenizer, return_tensors="tf")
+```
+</tf>
+</frameworkcontent>
 
 ## التقييم(Evaluate)
 
@@ -131,6 +143,8 @@ tokenized_imdb = imdb.map(preprocess_function, batched=True)
 >>> label2id = {"NEGATIVE": 0, "POSITIVE": 1}
 ```
 
+<frameworkcontent>
+<pt>
 <Tip>
 
 إذا لم تكن على دراية بضبط نموذج دقيق باستخدام [`Trainer`], فالق نظرة على البرنامج التعليمي الأساسي [هنا](../training#train-with-pytorch-trainer)!
@@ -191,6 +205,98 @@ tokenized_imdb = imdb.map(preprocess_function, batched=True)
 ```py
 >>> trainer.push_to_hub()
 ```
+</pt>
+<tf>
+<Tip>
+
+إذا لم تكن على دراية بضبط نموذج باستخدام Keras، قم بالاطلاع على البرنامج التعليمي الأساسي [هنا](../training#train-a-tensorflow-model-with-keras)!
+
+</Tip>
+لضبط نموذج في TensorFlow، ابدأ بإعداد دالة المحسن، وجدول معدل التعلم، وبعض معلمات التدريب:
+
+```py
+>>> from transformers import create_optimizer
+>>> import tensorflow as tf
+
+>>> batch_size = 16
+>>> num_epochs = 5
+>>> batches_per_epoch = len(tokenized_imdb["train"]) // batch_size
+>>> total_train_steps = int(batches_per_epoch * num_epochs)
+>>> optimizer, schedule = create_optimizer(init_lr=2e-5, num_warmup_steps=0, num_train_steps=total_train_steps)
+```
+
+ثم يمكنك تحميل DistilBERT مع [`TFAutoModelForSequenceClassification`] بالإضافة إلى عدد التصنيفات المتوقعة، وتعيينات التسميات:
+
+```py
+>>> from transformers import TFAutoModelForSequenceClassification
+
+>>> model = TFAutoModelForSequenceClassification.from_pretrained(
+...     "distilbert/distilbert-base-uncased", num_labels=2, id2label=id2label, label2id=label2id
+... )
+```
+
+قم بتحويل مجموعات بياناتك إلى تنسيق `tf.data.Dataset` باستخدام [`~transformers.TFPreTrainedModel.prepare_tf_dataset`]:
+
+```py
+>>> tf_train_set = model.prepare_tf_dataset(
+...     tokenized_imdb["train"],
+...     shuffle=True,
+...     batch_size=16,
+...     collate_fn=data_collator,
+... )
+
+>>> tf_validation_set = model.prepare_tf_dataset(
+...     tokenized_imdb["test"],
+...     shuffle=False,
+...     batch_size=16,
+...     collate_fn=data_collator,
+... )
+```
+
+قم بتهيئة النموذج للتدريب باستخدام [`compile`](https://keras.io/api/models/model_training_apis/#compile-method). لاحظ أن جميع نماذج Transformers لديها دالة خسارة ذات صلة بالمهمة بشكل افتراضي، لذلك لا تحتاج إلى تحديد واحدة ما لم ترغب في ذلك:
+
+```py
+>>> import tensorflow as tf
+
+>>> model.compile(optimizer=optimizer)  # No loss argument!
+```
+
+آخر أمرين يجب إعدادهما قبل بدء التدريب هو حساب الدقة من التوقعات، وتوفير طريقة لدفع نموذجك إلى Hub. يتم ذلك باستخدام [Keras callbacks](../main_classes/keras_callbacks).
+
+قم بتمرير دالة `compute_metrics` الخاصة بك إلى [`~transformers.KerasMetricCallback`]:
+
+```py
+>>> from transformers.keras_callbacks import KerasMetricCallback
+
+>>> metric_callback = KerasMetricCallback(metric_fn=compute_metrics, eval_dataset=tf_validation_set)
+```
+
+حدد مكان دفع نموذجك والمجزئ اللغوي في [`~transformers.PushToHubCallback`]:
+
+```py
+>>> from transformers.keras_callbacks import PushToHubCallback
+
+>>> push_to_hub_callback = PushToHubCallback(
+...     output_dir="my_awesome_model",
+...     tokenizer=tokenizer,
+... )
+```
+
+ثم اجمع الاستدعاءات معًا:
+
+```py
+>>> callbacks = [metric_callback, push_to_hub_callback]
+```
+
+أخيرًا، أنت مستعد لبدء تدريب نموذجك! قم باستدعاء [`fit`](https://keras.io/api/models/model_training_apis/#fit-method) مع مجموعات بيانات التدريب والتحقق، وعدد الحقبات، واستدعاءاتك لضبط النموذج:
+
+```py
+>>> model.fit(x=tf_train_set, validation_data=tf_validation_set, epochs=3, callbacks=callbacks)
+```
+
+بمجرد اكتمال التدريب، يتم تحميل نموذجك تلقائيًا إلى Hub حتى يتمكن الجميع من استخدامه!
+</tf>
+</frameworkcontent>
 
 <Tip>
 
@@ -222,6 +328,8 @@ tokenized_imdb = imdb.map(preprocess_function, batched=True)
 
 يمكنك أيضًا تكرار نتائج `pipeline` يدويًا إذا أردت:
 
+<frameworkcontent>
+<pt>
 قم يتجزئة النص وإرجاع تنسورات PyTorch:
 
 ```py
@@ -248,3 +356,32 @@ tokenized_imdb = imdb.map(preprocess_function, batched=True)
 >>> model.config.id2label[predicted_class_id]
 'POSITIVE'
 ```
+</pt>
+<tf>
+قم بتحليل النص وإرجاع تنسيقات TensorFlow:
+
+```py
+>>> from transformers import AutoTokenizer
+
+>>> tokenizer = AutoTokenizer.from_pretrained("stevhliu/my_awesome_model")
+>>> inputs = tokenizer(text, return_tensors="tf")
+```
+
+قم بتمرير مدخلاتك إلى النموذج وإرجاع `logits`:
+
+```py
+>>> from transformers import TFAutoModelForSequenceClassification
+
+>>> model = TFAutoModelForSequenceClassification.from_pretrained("stevhliu/my_awesome_model")
+>>> logits = model(**inputs).logits
+```
+
+استخرج الفئة ذات الاحتمالية الأعلى، واستخدم `id2label` لتحويلها إلى تصنيف نصي:
+
+```py
+>>> predicted_class_id = int(tf.math.argmax(logits, axis=-1)[0])
+>>> model.config.id2label[predicted_class_id]
+'POSITIVE'
+```
+</tf>
+</frameworkcontent>

docs/source/ar/tasks/summarization.md

@@ -118,12 +118,24 @@ pip install transformers datasets evaluate rouge_score
 
 الآن قم بإنشاء دفعة من الأمثلة باستخدام [`DataCollatorForSeq2Seq`].  الأكثر كفاءة *الحشو الديناميكي* للجمل إلى أطول طول في دفعة أثناء عملية التجميع، بدلاً من حشو مجموعة البيانات بأكملها إلى الحد الأقصى للطول.
 
+<frameworkcontent>
+<pt>
 
 ```py
 >>> from transformers import DataCollatorForSeq2Seq
 
 >>> data_collator = DataCollatorForSeq2Seq(tokenizer=tokenizer, model=checkpoint)
 ```
+</pt>
+<tf>
+
+```py
+>>> from transformers import DataCollatorForSeq2Seq
+
+>>> data_collator = DataCollatorForSeq2Seq(tokenizer=tokenizer, model=checkpoint, return_tensors="tf")
+```
+</tf>
+</frameworkcontent>
 
 ## التقييم (Evaluate)
 
@@ -158,6 +170,8 @@ pip install transformers datasets evaluate rouge_score
 
 ## التدريب (Train)
 
+<frameworkcontent>
+<pt>
 
 <Tip>
 
@@ -212,6 +226,91 @@ pip install transformers datasets evaluate rouge_score
 ```py
 >>> trainer.push_to_hub()
 ```
+</pt>
+<tf>
+<Tip>
+
+إذا لم تكن معتادًا على ضبط نموذج باستخدام Keras، فألق نظرة على البرنامج التعليمي الأساسي [هنا](../training#train-a-tensorflow-model-with-keras)!
+
+</Tip>
+لضبط نموذج في TensorFlow، ابدأ بإعداد دالة مُحسِّن وجدول معدل التعلم وبعض معلمات التدريب:
+
+```py
+>>> from transformers import create_optimizer, AdamWeightDecay
+
+>>> optimizer = AdamWeightDecay(learning_rate=2e-5, weight_decay_rate=0.01)
+```
+
+ثم يمكنك تحميل T5 باستخدام [`TFAutoModelForSeq2SeqLM`]:
+
+```py
+>>> from transformers import TFAutoModelForSeq2SeqLM
+
+>>> model = TFAutoModelForSeq2SeqLM.from_pretrained(checkpoint)
+```
+
+حوّل مجموعات البيانات الخاصة بك إلى تنسيق `tf.data.Dataset` باستخدام [`~transformers.TFPreTrainedModel.prepare_tf_dataset`]:
+
+```py
+>>> tf_train_set = model.prepare_tf_dataset(
+...     tokenized_billsum["train"],
+...     shuffle=True,
+...     batch_size=16,
+...     collate_fn=data_collator,
+... )
+
+>>> tf_test_set = model.prepare_tf_dataset(
+...     tokenized_billsum["test"],
+...     shuffle=False,
+...     batch_size=16,
+...     collate_fn=data_collator,
+... )
+```
+
+قم بتكوين النموذج للتدريب باستخدام [`compile`](https://keras.io/api/models/model_training_apis/#compile-method). لاحظ أن جميع نماذج Transformers لديها دالة خسارة ذات صلة بالمهمة افتراضيًا، لذلك لست بحاجة إلى تحديد واحدة ما لم تكن ترغب في ذلك:
+
+```py
+>>> import tensorflow as tf
+
+>>> model.compile(optimizer=optimizer)  # No loss argument!
+```
+
+آخر شيئين يجب إعدادهما قبل بدء التدريب هما حساب درجة ROUGE من التنبؤات، وتوفير طريقة لدفع نموذجك إلى Hub. يتم كلاهما باستخدام [استدعاءات Keras](../main_classes/keras_callbacks).
+
+مرر دالة `compute_metrics` الخاصة بك إلى [`~transformers.KerasMetricCallback`]:
+
+```py
+>>> from transformers.keras_callbacks import KerasMetricCallback
+
+>>> metric_callback = KerasMetricCallback(metric_fn=compute_metrics, eval_dataset=tf_test_set)
+```
+
+حدد مكان دفع نموذجك ومُحلِّلك اللغوي في [`~transformers.PushToHubCallback`]:
+
+```py
+>>> from transformers.keras_callbacks import PushToHubCallback
+
+>>> push_to_hub_callback = PushToHubCallback(
+...     output_dir="my_awesome_billsum_model",
+...     tokenizer=tokenizer,
+... )
+```
+
+ثم اجمع استدعاءاتك معًا:
+
+```py
+>>> callbacks = [metric_callback, push_to_hub_callback]
+```
+
+أخيرًا، أنت جاهز لبدء تدريب نموذجك! اتصل بـ [`fit`](https://keras.io/api/models/model_training_apis/#fit-method) مع مجموعات بيانات التدريب والتحقق من الصحة وعدد الحقب واستدعاءاتك لضبط النموذج:
+
+```py
+>>> model.fit(x=tf_train_set, validation_data=tf_test_set, epochs=3, callbacks=callbacks)
+```
+
+بمجرد اكتمال التدريب، يتم تحميل نموذجك تلقائيًا إلى Hub حتى يتمكن الجميع من استخدامه!
+</tf>
+</frameworkcontent>
 
 <Tip>
 
@@ -242,6 +341,8 @@ pip install transformers datasets evaluate rouge_score
 
 يمكنك أيضًا تكرار نتائج `pipeline` يدويًا إذا أردت:
 
+<frameworkcontent>
+<pt>
 قسم النص وإرجع `input_ids` كتنسورات PyTorch:
 
 ```py
@@ -266,3 +367,31 @@ pip install transformers datasets evaluate rouge_score
 >>> tokenizer.decode(outputs[0], skip_special_tokens=True)
 'the inflation reduction act lowers prescription drug costs, health care costs, and energy costs. it's the most aggressive action on tackling the climate crisis in american history. it will ask the ultra-wealthy and corporations to pay their fair share.'
 ```
+</pt>
+<tf>
+قسم النص وإرجع `input_ids` كتنسورات TensorFlow:
+
+```py
+>>> from transformers import AutoTokenizer
+
+>>> tokenizer = AutoTokenizer.from_pretrained("username/my_awesome_billsum_model")
+>>> inputs = tokenizer(text, return_tensors="tf").input_ids
+```
+
+استخدم طريقة [`~transformers.generation_tf_utils.TFGenerationMixin.generate`] لإنشاء التلخيص. لمزيد من التفاصيل حول استراتيجيات توليد النص المختلفة والمعلمات للتحكم في التوليد، راجع واجهة برمجة تطبيقات [توليد النص](../main_classes/text_generation).
+
+```py
+>>> from transformers import TFAutoModelForSeq2SeqLM
+
+>>> model = TFAutoModelForSeq2SeqLM.from_pretrained("username/my_awesome_billsum_model")
+>>> outputs = model.generate(inputs, max_new_tokens=100, do_sample=False)
+```
+
+فك تشفير معرفات الرموز المولدة مرة أخرى إلى نص:
+
+```py
+>>> tokenizer.decode(outputs[0], skip_special_tokens=True)
+'the inflation reduction act lowers prescription drug costs, health care costs, and energy costs. it's the most aggressive action on tackling the climate crisis in american history. it will ask the ultra-wealthy and corporations to pay their fair share.'
+```
+</tf>
+</frameworkcontent>

docs/source/ar/tasks/token_classification.md

@@ -151,11 +151,22 @@ pip install transformers datasets evaluate seqeval
 
 الآن قم بإنشاء دفعة من الأمثلة باستخدام [`DataCollatorWithPadding`].من الأفضل استخدام *الحشو الديناميكي* للجمل إلى أطول طول في دفعة أثناء التجميع، بدلاً من حشو مجموعة البيانات بالكامل إلى الطول الأقصى.
 
+<frameworkcontent>
+<pt>
 ```py
 >>> from transformers import DataCollatorForTokenClassification
 
 >>> data_collator = DataCollatorForTokenClassification(tokenizer=tokenizer)
 ```
+</pt>
+<tf>
+```py
+>>> from transformers import DataCollatorForTokenClassification
+
+>>> data_collator = DataCollatorForTokenClassification(tokenizer=tokenizer, return_tensors="tf")
+```
+</tf>
+</frameworkcontent>
 
 ## التقييم(Evaluate)
 
@@ -235,6 +246,8 @@ pip install transformers datasets evaluate seqeval
 ... }
 ```
 
+<frameworkcontent>
+<pt>
 <Tip>
 
 إذا لم تكن على دراية بتعديل نموذج باستخدام [`Trainer`], ألق نظرة على الدليل التعليمي الأساسي [هنا](../training#train-with-pytorch-trainer)!
@@ -289,6 +302,101 @@ pip install transformers datasets evaluate seqeval
 ```py
 >>> trainer.push_to_hub()
 ```
+</pt>
+<tf>
+<Tip>
+
+إذا لم تكن على دراية بتعديل نموذج باستخدام Keras، ألق نظرة على الدليل التعليمي الأساسي [هنا](../training#train-a-tensorflow-model-with-keras)!
+
+</Tip>
+للتعديل على نموذج في TensorFlow، ابدأ بإعداد دالة محسن، وجدول معدل التعلم، وبعض معلمات التدريب:
+
+```py
+>>> from transformers import create_optimizer
+
+>>> batch_size = 16
+>>> num_train_epochs = 3
+>>> num_train_steps = (len(tokenized_wnut["train"]) // batch_size) * num_train_epochs
+>>> optimizer, lr_schedule = create_optimizer(
+...     init_lr=2e-5,
+...     num_train_steps=num_train_steps,
+...     weight_decay_rate=0.01,
+...     num_warmup_steps=0,
+... )
+```
+
+ثم يمكنك تحميل DistilBERT مع [`TFAutoModelForTokenClassification`] إلى جانب عدد التسميات المتوقعة، وتخطيطات التسميات:
+
+```py
+>>> from transformers import TFAutoModelForTokenClassification
+
+>>> model = TFAutoModelForTokenClassification.from_pretrained(
+...     "distilbert/distilbert-base-uncased", num_labels=13, id2label=id2label, label2id=label2id
+... )
+```
+
+قم بتحويل مجموعات بياناتك إلى تنسيق `tf.data.Dataset` مع [`~transformers.TFPreTrainedModel.prepare_tf_dataset`]:
+
+```py
+>>> tf_train_set = model.prepare_tf_dataset(
+...     tokenized_wnut["train"],
+...     shuffle=True,
+...     batch_size=16,
+...     collate_fn=data_collator,
+... )
+
+>>> tf_validation_set = model.prepare_tf_dataset(
+...     tokenized_wnut["validation"],
+...     shuffle=False,
+...     batch_size=16,
+...     collate_fn=data_collator,
+... )
+```
+
+هيّئ النموذج للتدريب باستخدام [`compile`](https://keras.io/api/models/model_training_apis/#compile-method). لاحظ أن نماذج Transformers تتضمن دالة خسارة افتراضية مرتبطة بالمهمة، لذلك لا تحتاج إلى تحديد واحدة إلا إذا كنت ترغب في ذلك:
+
+```py
+>>> import tensorflow as tf
+
+>>> model.compile(optimizer=optimizer)  # No loss argument!
+```
+
+آخر أمرين يجب إعدادهما قبل بدء التدريب هو حساب درجات seqeval من التنبؤات، وتوفير طريقة لدفع نموذجك إلى Hub. يتم ذلك باستخدام [Keras callbacks](../main_classes/keras_callbacks).
+
+مرر دالة `compute_metrics` الخاصة بك إلى [`~transformers.KerasMetricCallback`]:
+
+```py
+>>> from transformers.keras_callbacks import KerasMetricCallback
+
+>>> metric_callback = KerasMetricCallback(metric_fn=compute_metrics, eval_dataset=tf_validation_set)
+```
+
+حدد مكان دفع نموذجك والمحلل اللغوي في [`~transformers.PushToHubCallback`]:
+
+```py
+>>> from transformers.keras_callbacks import PushToHubCallback
+
+>>> push_to_hub_callback = PushToHubCallback(
+...     output_dir="my_awesome_wnut_model",
+...     tokenizer=tokenizer,
+... )
+```
+
+ثم جمّع callbacks الخاصة بك معًا:
+
+```py
+>>> callbacks = [metric_callback, push_to_hub_callback]
+```
+
+أخيرًا، أنت جاهز الآن لبدء تدريب نموذجك! قم باستدعاء [`fit`](https://keras.io/api/models/model_training_apis/#fit-method) مع بيانات التدريب والتحقق، وعدد الحقبات، وcallbacks لتعديل النموذج:
+
+```py
+>>> model.fit(x=tf_train_set, validation_data=tf_validation_set, epochs=3, callbacks=callbacks)
+```
+
+بمجرد اكتمال التدريب، يتم تحميل نموذجك تلقائيًا إلى Hub حتى يتمكن الجميع من استخدامه!
+</tf>
+</frameworkcontent>
 
 <Tip>
 
@@ -349,6 +457,8 @@ pip install transformers datasets evaluate seqeval
 
 يمكنك أيضًا تكرار نتائج `pipeline` يدويًا إذا أردت:
 
+<frameworkcontent>
+<pt>
 قسّم النص إلى رموز وأرجع المُوتّرات بلغة PyTorch:
 
 ```py
@@ -392,3 +502,49 @@ pip install transformers datasets evaluate seqeval
  'O',
  'O']
 ```
+</pt>
+<tf>
+قسّم النص إلى رموز وأرجع المُوتّرات ب TensorFlow:
+
+```py
+>>> from transformers import AutoTokenizer
+
+>>> tokenizer = AutoTokenizer.from_pretrained("stevhliu/my_awesome_wnut_model")
+>>> inputs = tokenizer(text, return_tensors="tf")
+```
+
+مرر مدخلاتك إلى النموذج واحصل على `logits`:
+
+```py
+>>> from transformers import TFAutoModelForTokenClassification
+
+>>> model = TFAutoModelForTokenClassification.from_pretrained("stevhliu/my_awesome_wnut_model")
+>>> logits = model(**inputs).logits
+```
+
+استخرج الفئة ذات الاحتمالية الأعلى، واستخدم جدول `id2label` الخاصة بالنموذج لتحويلها إلى تسمية نصية:
+
+```py
+>>> predicted_token_class_ids = tf.math.argmax(logits, axis=-1)
+>>> predicted_token_class = [model.config.id2label[t] for t in predicted_token_class_ids[0].numpy().tolist()]
+>>> predicted_token_class
+['O',
+ 'O',
+ 'B-location',
+ 'I-location',
+ 'B-group',
+ 'O',
+ 'O',
+ 'O',
+ 'O',
+ 'O',
+ 'O',
+ 'O',
+ 'O',
+ 'B-location',
+ 'B-location',
+ 'O',
+ 'O']
+```
+</tf>
+</frameworkcontent>

docs/source/ar/tasks/translation.md

@@ -113,12 +113,24 @@ pip install transformers datasets evaluate sacrebleu
 
 الآن أنشئ دفعة من الأمثلة باستخدام [`DataCollatorForSeq2Seq`]. من الأكثر كفاءة *الحشو الديناميكي* للجمل إلى أطول طول في دفعة أثناء التجميع، بدلاً من حشو مجموعة البيانات بأكملها إلى الحد الأقصى للطول.
 
+<frameworkcontent>
+<pt>
 
 ```py
 >>> from transformers import DataCollatorForSeq2Seq
 
 >>> data_collator = DataCollatorForSeq2Seq(tokenizer=tokenizer, model=checkpoint)
 ```
+</pt>
+<tf>
+
+```py
+>>> from transformers import DataCollatorForSeq2Seq
+
+>>> data_collator = DataCollatorForSeq2Seq(tokenizer=tokenizer, model=checkpoint, return_tensors="tf")
+```
+</tf>
+</frameworkcontent>
 
 ## التقييم (Evaluate)
 
@@ -165,6 +177,8 @@ pip install transformers datasets evaluate sacrebleu
 
 ## التدريب (Train)
 
+<frameworkcontent>
+<pt>
 
 <Tip>
 
@@ -219,6 +233,91 @@ pip install transformers datasets evaluate sacrebleu
 ```py
 >>> trainer.push_to_hub()
 ```
+</pt>
+<tf>
+<Tip>
+
+إذا لم تكن معتادًا على ضبط نموذج باستخدام Keras، فألق نظرة على البرنامج التعليمي الأساسي [هنا](../training#train-a-tensorflow-model-with-keras)!
+
+</Tip>
+لضبط نموذج في TensorFlow، ابدأ بإعداد دالة مُحسِّن وجدول معدل تعلم وبعض المعلمات الفائقة للتدريب:
+
+```py
+>>> from transformers import AdamWeightDecay
+
+>>> optimizer = AdamWeightDecay(learning_rate=2e-5, weight_decay_rate=0.01)
+```
+
+ثم يمكنك تحميل T5 باستخدام [`TFAutoModelForSeq2SeqLM`]:
+
+```py
+>>> from transformers import TFAutoModelForSeq2SeqLM
+
+>>> model = TFAutoModelForSeq2SeqLM.from_pretrained(checkpoint)
+```
+
+حوّل مجموعات البيانات الخاصة بك إلى تنسيق `tf.data.Dataset` باستخدام [`~transformers.TFPreTrainedModel.prepare_tf_dataset`]:
+
+```py
+>>> tf_train_set = model.prepare_tf_dataset(
+...     tokenized_books["train"],
+...     shuffle=True,
+...     batch_size=16,
+...     collate_fn=data_collator,
+... )
+
+>>> tf_test_set = model.prepare_tf_dataset(
+...     tokenized_books["test"],
+...     shuffle=False,
+...     batch_size=16,
+...     collate_fn=data_collator,
+... )
+```
+
+قم بتكوين النموذج للتدريب باستخدام [`compile`](https://keras.io/api/models/model_training_apis/#compile-method). لاحظ أن جميع نماذج Transformers تحتوي على دالة خسارة ذات صلة بالمهمة بشكل افتراضي، لذلك لا تحتاج إلى تحديد واحدة إلا إذا كنت ترغب في ذلك:
+
+```py
+>>> import tensorflow as tf
+
+>>> model.compile(optimizer=optimizer)  # No loss argument!
+```
+
+آخر شيئين يجب إعدادهما قبل بدء التدريب هما حساب مقياس SacreBLEU من التوقعات، وتوفير طريقة لدفع نموذجك إلى Hub. يتم كلاهما باستخدام [استدعاءات Keras](../main_classes/keras_callbacks).
+
+مرر دالة `compute_metrics` الخاصة بك إلى [`~transformers.KerasMetricCallback`]:
+
+```py
+>>> from transformers.keras_callbacks import KerasMetricCallback
+
+>>> metric_callback = KerasMetricCallback(metric_fn=compute_metrics, eval_dataset=tf_test_set)
+```
+
+حدد مكان دفع نموذجك ومعالجك اللغوي في [`~transformers.PushToHubCallback`]:
+
+```py
+>>> from transformers.keras_callbacks import PushToHubCallback
+
+>>> push_to_hub_callback = PushToHubCallback(
+...     output_dir="my_awesome_opus_books_model",
+...     tokenizer=tokenizer,
+... )
+```
+
+ثم اجمع استدعاءاتك معًا:
+
+```py
+>>> callbacks = [metric_callback, push_to_hub_callback]
+```
+
+أخيرًا، أنت جاهز لبدء تدريب نموذجك! اتصل بـ [`fit`](https://keras.io/api/models/model_training_apis/#fit-method) مع مجموعات بيانات التدريب والتحقق من الصحة وعدد الحقب واستدعاءاتك لضبط النموذج:
+
+```py
+>>> model.fit(x=tf_train_set, validation_data=tf_test_set, epochs=3, callbacks=callbacks)
+```
+
+بمجرد اكتمال التدريب، يتم تحميل نموذجك تلقائيًا إلى Hub حتى يتمكن الجميع من استخدامه!
+</tf>
+</frameworkcontent>
 
 <Tip>
 
@@ -252,6 +351,8 @@ pip install transformers datasets evaluate sacrebleu
 
 يمكنك أيضًا تكرار نتائج `pipeline` يدويًا إذا أردت:
 
+<frameworkcontent>
+<pt>
 قم بتحويل النص إلى رموز وإرجاع `input_ids` كموترات PyTorch:
 
 ```py
@@ -276,3 +377,31 @@ pip install transformers datasets evaluate sacrebleu
 >>> tokenizer.decode(outputs[0], skip_special_tokens=True)
 'Les lignées partagent des ressources avec des bactéries enfixant l'azote.'
 ```
+</pt>
+<tf>
+قم بتحويل النص إلى رموز وإرجاع `input_ids` كموترات TensorFlow:
+
+```py
+>>> from transformers import AutoTokenizer
+
+>>> tokenizer = AutoTokenizer.from_pretrained("username/my_awesome_opus_books_model")
+>>> inputs = tokenizer(text, return_tensors="tf").input_ids
+```
+
+استخدم طريقة [`~transformers.generation_tf_utils.TFGenerationMixin.generate`] لإنشاء الترجمة. لمزيد من التفاصيل حول استراتيجيات توليد النصوص المختلفة والمعلمات للتحكم في التوليد، تحقق من واجهة برمجة تطبيقات [توليد النصوص](../main_classes/text_generation).
+
+```py
+>>> from transformers import TFAutoModelForSeq2SeqLM
+
+>>> model = TFAutoModelForSeq2SeqLM.from_pretrained("username/my_awesome_opus_books_model")
+>>> outputs = model.generate(inputs, max_new_tokens=40, do_sample=True, top_k=30, top_p=0.95)
+```
+
+فك تشفير معرفات الرموز المولدة مرة أخرى إلى نص:
+
+```py
+>>> tokenizer.decode(outputs[0], skip_special_tokens=True)
+'Les lugumes partagent les ressources avec des bactéries fixatrices d'azote.'
+```
+</tf>
+</frameworkcontent>

docs/source/ar/tflite.md

@@ -0,0 +1,40 @@
+# التصدير إلى TFLite
+
+[TensorFlow Lite](https://www.tensorflow.org/lite/guide) هو إطار عمل خفيف الوزن لنشر نماذج التعلم الآلي على الأجهزة المحدودة الموارد، مثل الهواتف المحمولة، والأنظمة المدمجة، وأجهزة إنترنت الأشياء (IoT). تم تصميم TFLite لتشغيل النماذج وتحسينها بكفاءة على هذه الأجهزة ذات الطاقة الحاسوبية والذاكرة واستهلاك الطاقة المحدودة.
+
+يُمثَّل نموذج TensorFlow Lite بتنسيق محمول فعال خاص يُعرَّف بامتداد الملف `.tflite`.
+
+🤗 Optimum يقدم وظيفة لتصدير نماذج 🤗 Transformers إلى TFLite من خلال الوحدة النمطية `exporters.tflite`. بالنسبة لقائمة هندسات النماذج المدعومة، يرجى الرجوع إلى [وثائق 🤗 Optimum](https://huggingface.co/docs/optimum/exporters/tflite/overview).
+
+لتصدير نموذج إلى TFLite، قم بتثبيت متطلبات البرنامج المطلوبة:
+
+```bash
+pip install optimum[exporters-tf]
+```
+
+للاطلاع على جميع المغامﻻت المتاحة، راجع [وثائق 🤗 Optimum](https://huggingface.co/docs/optimum/main/en/exporters/tflite/usage_guides/export_a_model)، أو عرض المساعدة في سطر الأوامر:
+
+```bash
+optimum-cli export tflite --help
+```
+
+لتصدير نسخة النموذج ل 🤗 Hub، على سبيل المثال، `google-bert/bert-base-uncased`، قم بتشغيل الأمر التالي:
+
+```bash
+optimum-cli export tflite --model google-bert/bert-base-uncased --sequence_length 128 bert_tflite/
+```
+
+ستظهر لك السجلات  التي تُبيّن التقدم وموقع حفظ ملف  `model.tflite` الناتج، كما في المثال التالي:
+
+```bash
+Validating TFLite model...
+	-[✓] TFLite model output names match reference model (logits)
+	- Validating TFLite Model output "logits":
+		-[✓] (1, 128, 30522) matches (1, 128, 30522)
+		-[x] values not close enough, max diff: 5.817413330078125e-05 (atol: 1e-05)
+The TensorFlow Lite export succeeded with the warning: The maximum absolute difference between the output of the reference model and the TFLite exported model is not within the set tolerance 1e-05:
+- logits: max diff = 5.817413330078125e-05.
+ The exported model was saved at: bert_tflite
+```
+
+يُبيّن المثال أعلاه كيفية تصدير نسخة من النموذج ل 🤗 Hub. عند تصدير نموذج محلي، تأكد أولاً من حفظ ملفات أوزان النموذج المجزء اللغوى في نفس المسار (`local_path`). عند استخدام CLI، قم بتمرير `local_path` إلى معامل `model` بدلاً من اسم النسخة على 🤗 Hub.

docs/source/ar/training.md

@@ -58,6 +58,8 @@
 في شريط التنقل الأيمن للقفز إلى الإطار الذي تريده - وإذا كنت تريد إخفاء كل المحتوى لإطار معين،
 فاستخدم الزر في الركن العلوي الأيمن من كتلة الإطار!
 
+<frameworkcontent>
+<pt>
 <Youtube id="nvBXf7s7vTI"/>
 
 ## التدريب باستخدام PyTorch Trainer
@@ -137,10 +139,124 @@
 ```py
 >>> trainer.train()
 ```
+</pt>
+<tf>
+<a id='keras'></a>
+
+<Youtube id="rnTGBy2ax1c"/>
+
+## تدريب نموذج TensorFlow باستخدام Keras
+
+يمكنك أيضًا تدريب نماذج 🤗 Transformers في TensorFlow باستخدام واجهة برمجة تطبيقات Keras!
+
+### تحميل البيانات لـ Keras
+
+عندما تريد تدريب نموذج 🤗 Transformers باستخدام واجهة برمجة تطبيقات Keras، فأنت بحاجة إلى تحويل مجموعة البيانات الخاصة بك إلى تنسيق يفهمه
+Keras. إذا كانت مجموعة البيانات الخاصة بك صغيرة، فيمكنك ببساطة تحويلها بالكامل إلى مصفوفات NumPy وإرسالها إلى Keras.
+دعونا نجرب ذلك أولاً قبل أن نقوم بأي شيء أكثر تعقيدًا.
+
+أولاً، قم بتحميل مجموعة بيانات. سنستخدم مجموعة بيانات CoLA من معيار [GLUE benchmark](https://huggingface.co/datasets/glue)،
+نظرًا لأنه مهمة تصنيف نص ثنائي بسيطة، وسنأخذ فقط قسم التدريب الآن.
+
+```py
+from datasets import load_dataset
+
+dataset = load_dataset("glue"، "cola")
+dataset = dataset ["train"] # خذ فقط قسم التدريب الآن
+```
+
+بعد ذلك، قم بتحميل أداة المُجزّئ اللغوي وقم بترميز البيانات كمصفوفات NumPy. لاحظ أن التصنيفات هي بالفعل قائمة من 0 و 1،
+لذا يمكننا ببساطة تحويل ذلك مباشرة إلى مصفوفة NumPy بدون ترميز!
+
+```py
+from transformers import AutoTokenizer
+import numpy as np
+
+tokenizer = AutoTokenizer.from_pretrained("google-bert/bert-base-cased")
+tokenized_data = tokenizer(dataset["sentence"], return_tensors="np", padding=True)
+# Tokenizer returns a BatchEncoding, but we convert that to a dict for Keras
+tokenized_data = dict(tokenized_data)
+
+labels = np.array(dataset["label"])  # Label is already an array of 0 and 1
+```
+
+أخيرًا، قم بتحميل وتجميع وتناسب النموذج. لاحظ أن نماذج Transformers تحتوي جميعها على دالة خسارة ذات صلة بالمهمة بشكل افتراضي، لذا فأنت لست بحاجة إلى تحديد واحدة ما لم ترغب في ذلك:
+
+```py
+from transformers import TFAutoModelForSequenceClassification
+from tensorflow.keras.optimizers import Adam
+
+# تحميل وتجميع النموذج الخاص بنا
+model = TFAutoModelForSequenceClassification.from_pretrained("google-bert/bert-base-cased")
+# معدلات التعلم المنخفضة أفضل غالبًا لضبط النماذج الدقيقة
+model.compile(optimizer=Adam(3e-5)) # لا توجد دالة خسارة!
+
+model.fit(tokenized_data, labels)
+```
+
+<Tip>
+
+أنت لست مضطرًا لتمرير دالة خسارة إلى نماذجك عند تجميعها! تختار نماذج Hugging Face تلقائيًا
+دالة خسارة مناسبة لمهمتها وهندسة نموذجها إذا تُركت هذه الحجة فارغة. يمكنك دائمًا
+تجاوز ذلك عن طريق تحديد دالة خسارة بنفسك إذا كنت تريد ذلك!
+
+</Tip>
+
+يعمل هذا النهج بشكل رائع لمجموعات البيانات الصغيرة، ولكن بالنسبة لمجموعات البيانات الأكبر، فقد تجد أنه يصبح مشكلة. لماذا؟
+لأن المصفوفة المرمزة والتصنيفات يجب أن يتم تحميلها بالكامل في الذاكرة، ولأن NumPy لا يتعامل مع
+المصفوفات"غير المنتظمة"، لذا حشو كل عينة  إلى طول أطول عينة في مجموعة البيانات بأكملها. سيؤدي ذلك إلى زيادة حجم المصفوفة لديك، وستبطئ الرموز الزائده من عملية التدريب أيضًا!
+
+### تحميل البيانات كـ tf.data.Dataset
+
+إذا كنت تريد تجنب إبطاء التدريب، فيمكنك تحميل بياناتك كـ `tf.data.Dataset` بدلاً من ذلك. على الرغم من أنه يمكنك كتابة خط أنابيب `tf.data` الخاص بك إذا كنت تريد، إلا أن لدينا طريقتين مختصرتين للقيام بذلك:
+- [`~TFPreTrainedModel.prepare_tf_dataset`]: هذه هي الطريقة التي نوصي بها في معظم الحالات. نظرًا لأنه طريقة
+على نموذجك، فيمكنه فحص النموذج لتحديد الأعمدة القابلة للاستخدام كمدخلات للنموذج تلقائيًا،
+واستبعاد الأعمدة الأخرى لإنشاء مجموعة بيانات أبسط وأكثر كفاءة.
+- [`~datasets.Dataset.to_tf_dataset`]: هذه الطريقة أكثر أساسية، وهي مفيدة عندما تريد التحكم بدقة في كيفية
+إنشاء مجموعة البيانات الخاصة بك، عن طريق تحديد أعمدة `columns` و `label_cols` المحددة التي سيتم تضمينها.
+
+قبل أن تتمكن من استخدام [`~TFPreTrainedModel.prepare_tf_dataset`]، ستحتاج إلى إضافة مخرجات المُجزئ إلى مجموعة البيانات الخاصة بك كأعمدة، كما هو موضح في
+عينة التعليمات البرمجية التالية:
+
+```py
+def tokenize_dataset (data):
+# ستتم إضافة مفاتيح القاموس الذي تمت إعادته كأعمدة إلى مجموعة البيانات
+return tokenizer(data["text"])
+
+
+dataset = dataset.map(tokenize_dataset)
+```
+
+تذكر أن مجموعات بيانات Hugging Face يتم تخزينها على القرص بشكل افتراضي، لذا فلن يؤدي ذلك إلى تضخيم استخدام الذاكرة لديك! بمجرد إضافة الأعمدة، يمكنك بث الدفعات من مجموعة البيانات وإضافة الترميز إلى كل دفعة، مما يقلل بشكل كبير من عدد رموز الترقيم مقارنة بترميز مجموعة البيانات بأكملها.
+
+
+```py
+>>> tf_dataset = model.prepare_tf_dataset(dataset["train"], batch_size=16, shuffle=True, tokenizer=tokenizer)
+```
+
+لاحظ أنه في عينة التعليمات البرمجية أعلاه، تحتاج إلى تمرير المُجزئ اللغوي إلى `prepare_tf_dataset` حتى تتمكن من حشو الدُفعات بشكل صحيح أثناء تحميلها.
+إذا كانت جميع العينات في مجموعة البيانات الخاصة بك بنفس الطول ولم يكن الترميز ضروريًا، فيمكنك تخطي هذا المعامل.
+إذا كنت بحاجة إلى القيام بشيء أكثر تعقيدًا من مجرد ترميز العينات (على سبيل المثال، إفساد الرموز للنمذجة اللغوية المُقنعة)،
+فيمكنك استخدام معامل `collate_fn` بدلاً من ذلك لتمرير دالة يتم استدعاؤها لتحويل
+قائمة العينات إلى دفعة وتطبيق أي معالجة مسبقة تريدها. راجع أمثلةنا [examples](https://github.com/huggingface/transformers/tree/main/examples) أو
+[دفاتر الملاحظات](https://huggingface.co/docs/transformers/notebooks) لرؤية هذا النهج في العمل.
+
+بمجرد إنشاء `tf.data.Dataset`، يمكنك تجميع النموذج وتناسبه كما هو الحال من قبل:
+
+```py
+model.compile(optimizer=Adam(3e-5))  # No loss argument!
+
+model.fit(tf_dataset)
+```
+
+</tf>
+</frameworkcontent>
 
 <a id='pytorch_native'></a>
 ## تدريب في PyTorch الأصلي
 
+<frameworkcontent>
+<pt>
 <Youtube id="Dh9CL8fyG80"/>
 
 [`Trainer`] يهتم بحلقة التدريب ويسمح لك بضبط نموذج في سطر واحد من التعليمات البرمجية. بالنسبة للمستخدمين الذين يفضلون كتابة حلقة التدريب الخاصة بهم، يمكنك أيضًا ضبط نموذج 🤗 Transformers في PyTorch الأصلي.
@@ -281,6 +397,8 @@ torch.cuda.empty_cache()
 
 >>> metric.compute()
 ```
+</pt>
+</frameworkcontent>
 
 <a id='additional-resources'></a>
 
@@ -291,4 +409,4 @@ torch.cuda.empty_cache()
 - [🤗 أمثلة المحولات](https://github.com/huggingface/transformers/tree/main/examples) تتضمن
   النصوص البرمجية لتدريب مهام NLP الشائعة في PyTorch وTensorFlow.
 
-- [🤗 دفاتر ملاحظات المحولات](notebooks) يحتوي على دفاتر ملاحظات مختلفة حول كيفية ضبط نموذج لمهمة محددة في PyTorch وTensorFlow.
+- [🤗 دفاتر ملاحظات المحولات](notebooks) يحتوي على دفاتر ملاحظات مختلفة حول كيفية ضبط نموذج لمهمة محددة في PyTorch وTensorFlow.

docs/source/de/autoclass_tutorial.md

@@ -81,6 +81,8 @@ Laden Sie einen Prozessor mit [`AutoProcessor.from_pretrained`]:
 
 ## AutoModel
 
+<frameworkcontent>
+<pt>
 Mit den `AutoModelFor`-Klassen können Sie schließlich ein vortrainiertes Modell für eine bestimmte Aufgabe laden (siehe [hier](model_doc/auto) für eine vollständige Liste der verfügbaren Aufgaben). Laden Sie zum Beispiel ein Modell für die Sequenzklassifikation mit [`AutoModelForSequenceClassification.from_pretrained`]:
 
 ```py
@@ -106,3 +108,24 @@ TensorFlow- und Flax-Checkpoints sind nicht betroffen und können in PyTorch-Arc
 </Tip>
 
 Im Allgemeinen empfehlen wir die Verwendung der Klasse "AutoTokenizer" und der Klasse "AutoModelFor", um trainierte Instanzen von Modellen zu laden. Dadurch wird sichergestellt, dass Sie jedes Mal die richtige Architektur laden. Im nächsten [Tutorial] (Vorverarbeitung) erfahren Sie, wie Sie Ihren neu geladenen Tokenizer, Feature Extractor und Prozessor verwenden, um einen Datensatz für die Feinabstimmung vorzuverarbeiten.
+</pt>
+<tf>
+Mit den Klassen `TFAutoModelFor` schließlich können Sie ein vortrainiertes Modell für eine bestimmte Aufgabe laden (siehe [hier](model_doc/auto) für eine vollständige Liste der verfügbaren Aufgaben). Laden Sie zum Beispiel ein Modell für die Sequenzklassifikation mit [`TFAutoModelForSequenceClassification.from_pretrained`]:
+
+```py
+>>> from transformers import TFAutoModelForSequenceClassification
+
+>>> model = TFAutoModelForSequenceClassification.from_pretrained("distilbert/distilbert-base-uncased")
+```
+
+Sie können denselben Prüfpunkt problemlos wiederverwenden, um eine Architektur für eine andere Aufgabe zu laden:
+
+```py
+>>> from transformers import TFAutoModelForTokenClassification
+
+>>> model = TFAutoModelForTokenClassification.from_pretrained("distilbert/distilbert-base-uncased")
+```
+
+Im Allgemeinen empfehlen wir, die Klasse "AutoTokenizer" und die Klasse "TFAutoModelFor" zu verwenden, um vortrainierte Instanzen von Modellen zu laden. Dadurch wird sichergestellt, dass Sie jedes Mal die richtige Architektur laden. Im nächsten [Tutorial] (Vorverarbeitung) erfahren Sie, wie Sie Ihren neu geladenen Tokenizer, Feature Extractor und Prozessor verwenden, um einen Datensatz für die Feinabstimmung vorzuverarbeiten.
+</tf>
+</frameworkcontent>

docs/source/de/model_sharing.md

@@ -79,15 +79,43 @@ Um sicherzustellen, dass Ihr Modell von jemandem verwendet werden kann, der mit
 
 Die Konvertierung eines Checkpoints für ein anderes Framework ist einfach. Stellen Sie sicher, dass Sie PyTorch und TensorFlow installiert haben (siehe [hier](installation) für Installationsanweisungen), und finden Sie dann das spezifische Modell für Ihre Aufgabe in dem anderen Framework. 
 
+<frameworkcontent>
+<pt>
 Geben Sie `from_tf=True` an, um einen Prüfpunkt von TensorFlow nach PyTorch zu konvertieren:
 
 ```py
 >>> pt_model = DistilBertForSequenceClassification.from_pretrained("path/to/awesome-name-you-picked", from_tf=True)
 >>> pt_model.save_pretrained("path/to/awesome-name-you-picked")
 ```
+</pt>
+<tf>
+Geben Sie `from_pt=True` an, um einen Prüfpunkt von PyTorch nach TensorFlow zu konvertieren:
+
+```py
+>>> tf_model = TFDistilBertForSequenceClassification.from_pretrained("path/to/awesome-name-you-picked", from_pt=True)
+```
+
+Dann können Sie Ihr neues TensorFlow-Modell mit seinem neuen Checkpoint speichern:
+
+```py
+>>> tf_model.save_pretrained("path/to/awesome-name-you-picked")
+```
+</tf>
+<jax>
+Wenn ein Modell in Flax verfügbar ist, können Sie auch einen Kontrollpunkt von PyTorch nach Flax konvertieren:
+
+```py
+>>> flax_model = FlaxDistilBertForSequenceClassification.from_pretrained(
+...     "path/to/awesome-name-you-picked", from_pt=True
+... )
+```
+</jax>
+</frameworkcontent>
 
 ## Ein Modell während des Trainings hochladen
 
+<frameworkcontent>
+<pt>
 <Youtube id="Z1-XMy-GNLQ"/>
 
 Die Weitergabe eines Modells an den Hub ist so einfach wie das Hinzufügen eines zusätzlichen Parameters oder Rückrufs. Erinnern Sie sich an das [Feinabstimmungs-Tutorial](training), in der Klasse [`TrainingArguments`] geben Sie Hyperparameter und zusätzliche Trainingsoptionen an. Eine dieser Trainingsoptionen beinhaltet die Möglichkeit, ein Modell direkt an den Hub zu pushen. Setzen Sie `push_to_hub=True` in Ihrer [`TrainingArguments`]:
@@ -113,6 +141,29 @@ Nach der Feinabstimmung Ihres Modells rufen Sie [`~transformers.Trainer.push_to_
 ```py
 >>> trainer.push_to_hub()
 ```
+</pt>
+<tf>
+Geben Sie ein Modell mit [`PushToHubCallback`] an den Hub weiter. In der [`PushToHubCallback`] Funktion, fügen Sie hinzu:
+
+- Ein Ausgabeverzeichnis für Ihr Modell.
+- Einen Tokenizer.
+- Die `hub_model_id`, die Ihr Hub-Benutzername und Modellname ist.
+
+```py
+>>> from transformers import PushToHubCallback
+
+>>> push_to_hub_callback = PushToHubCallback(
+...     output_dir="./your_model_save_path", tokenizer=tokenizer, hub_model_id="your-username/my-awesome-model"
+... )
+```
+
+Fügen Sie den Callback zu [`fit`](https://keras.io/api/models/model_training_apis/) hinzu, und 🤗 Transformers wird das trainierte Modell an den Hub weiterleiten:
+
+```py
+>>> model.fit(tf_train_dataset, validation_data=tf_validation_dataset, epochs=3, callbacks=push_to_hub_callback)
+```
+</tf>
+</frameworkcontent>
 
 ## Verwenden Sie die Funktion `push_to_hub`.
 
@@ -178,4 +229,4 @@ Um sicherzustellen, dass die Benutzer die Fähigkeiten, Grenzen, möglichen Verz
 * Manuelles Erstellen und Hochladen einer "README.md"-Datei.
 * Klicken Sie auf die Schaltfläche **Modellkarte bearbeiten** in Ihrem Modell-Repository.
 
-Werfen Sie einen Blick auf die DistilBert [model card](https://huggingface.co/distilbert/distilbert-base-uncased) als gutes Beispiel für die Art von Informationen, die eine Modellkarte enthalten sollte. Weitere Details über andere Optionen, die Sie in der Datei "README.md" einstellen können, wie z.B. den Kohlenstoff-Fußabdruck eines Modells oder Beispiele für Widgets, finden Sie in der Dokumentation [hier](https://huggingface.co/docs/hub/models-cards).
+Werfen Sie einen Blick auf die DistilBert [model card](https://huggingface.co/distilbert/distilbert-base-uncased) als gutes Beispiel für die Art von Informationen, die eine Modellkarte enthalten sollte. Weitere Details über andere Optionen, die Sie in der Datei "README.md" einstellen können, wie z.B. den Kohlenstoff-Fußabdruck eines Modells oder Beispiele für Widgets, finden Sie in der Dokumentation [hier](https://huggingface.co/docs/hub/models-cards).

docs/source/de/preprocessing.md

@@ -153,6 +153,8 @@ Schließlich möchten Sie, dass der Tokenizer die tatsächlichen Tensoren zurüc
 
 Setzen Sie den Parameter `return_tensors` entweder auf `pt` für PyTorch, oder `tf` für TensorFlow:
 
+<frameworkcontent>
+<pt>
 
 ```py
 >>> batch_sentences = [
@@ -172,6 +174,32 @@ Setzen Sie den Parameter `return_tensors` entweder auf `pt` für PyTorch, oder `
                            [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
                            [1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0]])}
 ```
+</pt>
+<tf>
+```py
+>>> batch_sentences = [
+...     "But what about second breakfast?",
+...     "Don't think he knows about second breakfast, Pip.",
+...     "What about elevensies?",
+... ]
+>>> encoded_input = tokenizer(batch_sentences, padding=True, truncation=True, return_tensors="tf")
+>>> print(encoded_input)
+{'input_ids': <tf.Tensor: shape=(2, 9), dtype=int32, numpy=
+array([[101, 1252, 1184, 1164, 1248, 6462, 136, 102, 0, 0, 0, 0, 0, 0, 0],
+       [101, 1790, 112, 189, 1341, 1119, 3520, 1164, 1248, 6462, 117, 21902, 1643, 119, 102],
+       [101, 1327, 1164, 5450, 23434, 136, 102, 0, 0, 0, 0, 0, 0, 0, 0]],
+      dtype=int32)>, 
+ 'token_type_ids': <tf.Tensor: shape=(2, 9), dtype=int32, numpy=
+array([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
+       [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
+       [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], dtype=int32)>, 
+ 'attention_mask': <tf.Tensor: shape=(2, 9), dtype=int32, numpy=
+array([[1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0],
+       [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
+       [1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0]], dtype=int32)>}
+```
+</tf>
+</frameworkcontent>
 
 ## Audio
 

docs/source/de/quicktour.md

@@ -66,10 +66,20 @@ Im folgenden Beispiel werden Sie die [`pipeline`] für die Stimmungsanalyse verw
 
 Installieren Sie die folgenden Abhängigkeiten, falls Sie dies nicht bereits getan haben:
 
+<frameworkcontent>
+<pt>
 
 ```bash
 pip install torch
 ```
+</pt>
+<tf>
+
+```bash
+pip install tensorflow
+```
+</tf>
+</frameworkcontent>
 
 Importieren sie die [`pipeline`] und spezifizieren sie die Aufgabe, welche sie lösen möchten:
 
@@ -144,6 +154,8 @@ Die [`pipeline`] kann jedes Modell aus dem [Model Hub](https://huggingface.co/mo
 >>> model_name = "nlptown/bert-base-multilingual-uncased-sentiment"
 ```
 
+<frameworkcontent>
+<pt>
 Use the [`AutoModelForSequenceClassification`] and [`AutoTokenizer`] to load the pretrained model and its associated tokenizer (more on an `AutoClass` below):
 
 ```py
@@ -152,6 +164,18 @@ Use the [`AutoModelForSequenceClassification`] and [`AutoTokenizer`] to load the
 >>> model = AutoModelForSequenceClassification.from_pretrained(model_name)
 >>> tokenizer = AutoTokenizer.from_pretrained(model_name)
 ```
+</pt>
+<tf>
+Use the [`TFAutoModelForSequenceClassification`] and [`AutoTokenizer`] to load the pretrained model and its associated tokenizer (more on an `TFAutoClass` below):
+
+```py
+>>> from transformers import AutoTokenizer, TFAutoModelForSequenceClassification
+
+>>> model = TFAutoModelForSequenceClassification.from_pretrained(model_name)
+>>> tokenizer = AutoTokenizer.from_pretrained(model_name)
+```
+</tf>
+</frameworkcontent>
 
 Dann können Sie das Modell und den Tokenizer in der [`pipeline`] angeben und den `Klassifikator` auf Ihren Zieltext anwenden:
 
@@ -202,6 +226,8 @@ Der Tokenizer gibt ein Wörterbuch zurück, das Folgendes enthält:
 
 Genau wie die [`pipeline`] akzeptiert der Tokenizer eine Liste von Eingaben. Darüber hinaus kann der Tokenizer den Text auch auffüllen und kürzen, um einen Stapel mit einheitlicher Länge zurückzugeben:
 
+<frameworkcontent>
+<pt>
 
 ```py
 >>> pt_batch = tokenizer(
@@ -212,11 +238,27 @@ Genau wie die [`pipeline`] akzeptiert der Tokenizer eine Liste von Eingaben. Dar
 ...     return_tensors="pt",
 ... )
 ```
+</pt>
+<tf>
+
+```py
+>>> tf_batch = tokenizer(
+...     ["We are very happy to show you the 🤗 Transformers library.", "We hope you don't hate it."],
+...     padding=True,
+...     truncation=True,
+...     max_length=512,
+...     return_tensors="tf",
+... )
+```
+</tf>
+</frameworkcontent>
 
 Lesen Sie das Tutorial [preprocessing](./preprocessing) für weitere Details zur Tokenisierung.
 
 ### AutoModel
 
+<frameworkcontent>
+<pt>
 🤗 Transformers bietet eine einfache und einheitliche Möglichkeit, vortrainierte Instanzen zu laden. Das bedeutet, dass Sie ein [`AutoModel`] laden können, wie Sie einen [`AutoTokenizer`] laden würden. Der einzige Unterschied ist die Auswahl des richtigen [`AutoModel`] für die Aufgabe. Da Sie eine Text- oder Sequenzklassifizierung vornehmen, laden Sie [`AutoModelForSequenceClassification`]:
 
 ```py
@@ -248,6 +290,39 @@ Das Modell gibt die endgültigen Aktivierungen in dem Attribut "logits" aus. Wen
 tensor([[0.0021, 0.0018, 0.0115, 0.2121, 0.7725],
         [0.2084, 0.1826, 0.1969, 0.1755, 0.2365]], grad_fn=<SoftmaxBackward0>)
 ```
+</pt>
+<tf>
+🤗 Transformers bietet eine einfache und einheitliche Methode zum Laden von vortrainierten Instanzen. Das bedeutet, dass Sie ein [`TFAutoModel`] genauso laden können, wie Sie einen [`AutoTokenizer`] laden würden. Der einzige Unterschied ist die Auswahl des richtigen [`TFAutoModel`] für die Aufgabe. Da Sie Text - oder Sequenz - Klassifizierung machen, laden Sie [`TFAutoModelForSequenceClassification`]:
+
+```py
+>>> from transformers import TFAutoModelForSequenceClassification
+
+>>> model_name = "nlptown/bert-base-multilingual-uncased-sentiment"
+>>> tf_model = TFAutoModelForSequenceClassification.from_pretrained(model_name)
+```
+
+<Tip>
+
+In der [Aufgabenzusammenfassung](./task_summary) steht, welche [AutoModel]-Klasse für welche Aufgabe zu verwenden ist.
+
+</Tip>
+
+Jetzt können Sie Ihren vorverarbeiteten Stapel von Eingaben direkt an das Modell übergeben, indem Sie die Wörterbuchschlüssel direkt an die Tensoren übergeben:
+
+```py
+>>> tf_outputs = tf_model(tf_batch)
+```
+
+Das Modell gibt die endgültigen Aktivierungen in dem Attribut "logits" aus. Wenden Sie die Softmax-Funktion auf die "logits" an, um die Wahrscheinlichkeiten zu erhalten:
+
+```py
+>>> import tensorflow as tf
+
+>>> tf_predictions = tf.nn.softmax(tf_outputs.logits, axis=-1)
+>>> tf_predictions  # doctest: +IGNORE_RESULT
+```
+</tf>
+</frameworkcontent>
 
 <Tip>
 
@@ -267,6 +342,8 @@ Die Modellausgänge verhalten sich auch wie ein Tupel oder ein Wörterbuch (z.B.
 
 ### Modell speichern
 
+<frameworkcontent>
+<pt>
 Sobald Ihr Modell feinabgestimmt ist, können Sie es mit seinem Tokenizer speichern, indem Sie [`PreTrainedModel.save_pretrained`] verwenden:
 
 ```py
@@ -280,9 +357,28 @@ Wenn Sie bereit sind, das Modell erneut zu verwenden, laden Sie es mit [`PreTrai
 ```py
 >>> pt_model = AutoModelForSequenceClassification.from_pretrained("./pt_save_pretrained")
 ```
+</pt>
+<tf>
+Sobald Ihr Modell feinabgestimmt ist, können Sie es mit seinem Tokenizer unter Verwendung von [`TFPreTrainedModel.save_pretrained`] speichern:
+
+```py
+>>> tf_save_directory = "./tf_save_pretrained"
+>>> tokenizer.save_pretrained(tf_save_directory)  # doctest: +IGNORE_RESULT
+>>> tf_model.save_pretrained(tf_save_directory)
+```
+
+Wenn Sie bereit sind, das Modell wieder zu verwenden, laden Sie es mit [`TFPreTrainedModel.from_pretrained`]:
+
+```py
+>>> tf_model = TFAutoModelForSequenceClassification.from_pretrained("./tf_save_pretrained")
+```
+</tf>
+</frameworkcontent>
 
 Ein besonders cooles 🤗 Transformers-Feature ist die Möglichkeit, ein Modell zu speichern und es entweder als PyTorch- oder TensorFlow-Modell wieder zu laden. Der Parameter "from_pt" oder "from_tf" kann das Modell von einem Framework in das andere konvertieren:
 
+<frameworkcontent>
+<pt>
 
 ```py
 >>> from transformers import AutoModel
@@ -290,6 +386,17 @@ Ein besonders cooles 🤗 Transformers-Feature ist die Möglichkeit, ein Modell
 >>> tokenizer = AutoTokenizer.from_pretrained(pt_save_directory)
 >>> pt_model = AutoModelForSequenceClassification.from_pretrained(pt_save_directory, from_pt=True)
 ```
+</pt>
+<tf>
+
+```py
+>>> from transformers import TFAutoModel
+
+>>> tokenizer = AutoTokenizer.from_pretrained(tf_save_directory)
+>>> tf_model = TFAutoModelForSequenceClassification.from_pretrained(tf_save_directory, from_tf=True)
+```
+</tf>
+</frameworkcontent>
 
 ## Custom model builds
 
@@ -303,6 +410,8 @@ Beginnen Sie mit dem Import von [`AutoConfig`] und laden Sie dann das trainierte
 >>> my_config = AutoConfig.from_pretrained("distilbert/distilbert-base-uncased", n_heads=12)
 ```
 
+<frameworkcontent>
+<pt>
 Create a model from your custom configuration with [`AutoModel.from_config`]:
 
 ```py
@@ -310,6 +419,17 @@ Create a model from your custom configuration with [`AutoModel.from_config`]:
 
 >>> my_model = AutoModel.from_config(my_config)
 ```
+</pt>
+<tf>
+Create a model from your custom configuration with [`TFAutoModel.from_config`]:
+
+```py
+>>> from transformers import TFAutoModel
+
+>>> my_model = TFAutoModel.from_config(my_config)
+```
+</tf>
+</frameworkcontent>
 
 Weitere Informationen zur Erstellung von benutzerdefinierten Konfigurationen finden Sie in der Anleitung [Erstellen einer benutzerdefinierten Architektur](./create_a_model).
 

docs/source/de/run_scripts.md

@@ -85,6 +85,8 @@ pip install -r requirements.txt
 
 ## Ein Skript ausführen
 
+<frameworkcontent>
+<pt>
 Das Beispielskript lädt einen Datensatz aus der 🤗 [Datasets](https://huggingface.co/docs/datasets/) Bibliothek herunter und verarbeitet ihn vor. Dann nimmt das Skript eine Feinabstimmung eines Datensatzes mit dem [Trainer](https://huggingface.co/docs/transformers/main_classes/trainer) auf einer Architektur vor, die eine Zusammenfassung unterstützt. Das folgende Beispiel zeigt, wie die Feinabstimmung von [T5-small](https://huggingface.co/google-t5/t5-small) auf dem Datensatz [CNN/DailyMail](https://huggingface.co/datasets/cnn_dailymail) durchgeführt wird. Das T5-Modell benötigt aufgrund der Art und Weise, wie es trainiert wurde, ein zusätzliches Argument `source_prefix`. Mit dieser Eingabeaufforderung weiß T5, dass es sich um eine Zusammenfassungsaufgabe handelt.
 
 ```bash
@@ -101,6 +103,24 @@ python examples/pytorch/summarization/run_summarization.py \
     --overwrite_output_dir \
     --predict_with_generate
 ```
+</pt>
+<tf>
+Das Beispielskript lädt einen Datensatz aus der 🤗 [Datasets](https://huggingface.co/docs/datasets/) Bibliothek herunter und verarbeitet ihn vor. Anschließend nimmt das Skript die Feinabstimmung eines Datensatzes mit Keras auf einer Architektur vor, die die Zusammenfassung unterstützt. Das folgende Beispiel zeigt, wie die Feinabstimmung von [T5-small](https://huggingface.co/google-t5/t5-small) auf dem [CNN/DailyMail](https://huggingface.co/datasets/cnn_dailymail) Datensatz durchgeführt wird. Das T5-Modell benötigt aufgrund der Art und Weise, wie es trainiert wurde, ein zusätzliches Argument `source_prefix`. Mit dieser Eingabeaufforderung weiß T5, dass es sich um eine Zusammenfassungsaufgabe handelt.
+
+```bash
+python examples/tensorflow/summarization/run_summarization.py  \
+    --model_name_or_path google-t5/t5-small \
+    --dataset_name cnn_dailymail \
+    --dataset_config "3.0.0" \
+    --output_dir /tmp/tst-summarization  \
+    --per_device_train_batch_size 8 \
+    --per_device_eval_batch_size 16 \
+    --num_train_epochs 3 \
+    --do_train \
+    --do_eval
+```
+</tf>
+</frameworkcontent>
 
 ## Verteiltes Training und gemischte Präzision
 
@@ -130,6 +150,8 @@ TensorFlow-Skripte verwenden eine [`MirroredStrategy`](https://www.tensorflow.or
 
 ## Ein Skript auf einer TPU ausführen
 
+<frameworkcontent>
+<pt>
 Tensor Processing Units (TPUs) sind speziell für die Beschleunigung der Leistung konzipiert. PyTorch unterstützt TPUs mit dem [XLA](https://www.tensorflow.org/xla) Deep Learning Compiler (siehe [hier](https://github.com/pytorch/xla/blob/master/README.md) für weitere Details). Um eine TPU zu verwenden, starten Sie das Skript `xla_spawn.py` und verwenden das Argument `num_cores`, um die Anzahl der TPU-Kerne festzulegen, die Sie verwenden möchten.
 
 ```bash
@@ -147,6 +169,25 @@ python xla_spawn.py --num_cores 8 \
     --overwrite_output_dir \
     --predict_with_generate
 ```
+</pt>
+<tf>
+Tensor Processing Units (TPUs) sind speziell für die Beschleunigung der Leistung konzipiert. TensorFlow Skripte verwenden eine [`TPUStrategy`](https://www.tensorflow.org/guide/distributed_training#tpustrategy) für das Training auf TPUs. Um eine TPU zu verwenden, übergeben Sie den Namen der TPU-Ressource an das Argument `tpu`.
+
+```bash
+python run_summarization.py  \
+    --tpu name_of_tpu_resource \
+    --model_name_or_path google-t5/t5-small \
+    --dataset_name cnn_dailymail \
+    --dataset_config "3.0.0" \
+    --output_dir /tmp/tst-summarization  \
+    --per_device_train_batch_size 8 \
+    --per_device_eval_batch_size 16 \
+    --num_train_epochs 3 \
+    --do_train \
+    --do_eval
+```
+</tf>
+</frameworkcontent>
 
 ## Führen Sie ein Skript mit 🤗 Accelerate aus.
 
@@ -307,4 +348,4 @@ python examples/pytorch/summarization/run_summarization.py
     --per_device_eval_batch_size=4 \
     --overwrite_output_dir \
     --predict_with_generate
-```
+```

docs/source/de/training.md

@@ -73,6 +73,8 @@ An dieser Stelle sollten Sie dem Abschnitt folgen, der dem Rahmen entspricht, de
 in der rechten Seitenleiste können Sie zu dem gewünschten Abschnitt springen - und wenn Sie den gesamten Inhalt eines bestimmten Frameworks ausblenden möchten,
 klicken Sie einfach auf die Schaltfläche oben rechts im Block des jeweiligen Frameworks!
 
+<frameworkcontent>
+<pt>
 <Youtube id="nvBXf7s7vTI"/>
 
 ## Trainieren mit PyTorch Trainer
@@ -153,11 +155,128 @@ Anschließend können Sie Ihr Modell durch den Aufruf von [`~transformers.Traine
 ```py
 >>> trainer.train()
 ```
+</pt>
+<tf>
+<a id='keras'></a>
+
+<Youtube id="rnTGBy2ax1c"/>
+
+## Trainieren Sie ein TensorFlow-Modell mit Keras
+
+Sie können auch 🤗 Transformers Modelle in TensorFlow mit der Keras API trainieren!
+
+### Laden von Daten für Keras
+
+Wenn Sie ein 🤗 Transformers Modell mit der Keras API trainieren wollen, müssen Sie Ihren Datensatz in ein Format konvertieren, das
+Keras versteht. Wenn Ihr Datensatz klein ist, können Sie das Ganze einfach in NumPy-Arrays konvertieren und an Keras übergeben.
+Probieren wir das zuerst aus, bevor wir etwas Komplizierteres tun.
+
+Laden Sie zunächst ein Dataset. Wir werden den CoLA-Datensatz aus dem [GLUE-Benchmark](https://huggingface.co/datasets/glue) verwenden,
+da es sich um eine einfache Aufgabe zur Klassifizierung von binärem Text handelt, und nehmen vorerst nur den Trainingssplit.
+
+```py
+from datasets import load_dataset
+
+dataset = load_dataset("glue", "cola")
+dataset = dataset["train"]  # Just take the training split for now
+```
+
+Als nächstes laden Sie einen Tokenizer und tokenisieren die Daten als NumPy-Arrays. Beachten Sie, dass die Beschriftungen bereits eine Liste von 0 und 1en sind,
+Wir können sie also ohne Tokenisierung direkt in ein NumPy-Array konvertieren!
+
+```py
+from transformers import AutoTokenizer
+
+tokenizer = AutoTokenizer.from_pretrained("google-bert/bert-base-cased")
+tokenized_data = tokenizer(dataset["text"], return_tensors="np", padding=True)
+# Tokenizer returns a BatchEncoding, but we convert that to a dict for Keras
+tokenized_data = dict(tokenized_data)
+
+labels = np.array(dataset["label"])  # Label is already an array of 0 and 1
+```
+
+Schließlich laden, [`compile`](https://keras.io/api/models/model_training_apis/#compile-method) und [`fit`](https://keras.io/api/models/model_training_apis/#fit-method) Sie das Modell:
+
+```py
+from transformers import TFAutoModelForSequenceClassification
+from tensorflow.keras.optimizers import Adam
+
+# Load and compile our model
+model = TFAutoModelForSequenceClassification.from_pretrained("google-bert/bert-base-cased")
+# Lower learning rates are often better for fine-tuning transformers
+model.compile(optimizer=Adam(3e-5))
+
+model.fit(tokenized_data, labels)
+```
+
+<Tip>
+
+Sie müssen Ihren Modellen kein Verlustargument übergeben, wenn Sie sie `compile()`! Hugging-Face-Modelle wählen automatisch
+einen Loss, der für ihre Aufgabe und Modellarchitektur geeignet ist, wenn dieses Argument leer gelassen wird. Sie können jederzeit außer Kraft setzen, indem Sie selbst einen Loss angeben, wenn Sie das möchten!
+
+</Tip>
+
+Dieser Ansatz eignet sich hervorragend für kleinere Datensätze, aber bei größeren Datensätzen kann er zu einem Problem werden. Warum?
+Weil das tokenisierte Array und die Beschriftungen vollständig in den Speicher geladen werden müssten, und weil NumPy nicht mit
+"gezackte" Arrays nicht verarbeiten kann, so dass jedes tokenisierte Sample auf die Länge des längsten Samples im gesamten Datensatz aufgefüllt werden müsste.
+Datensatzes aufgefüllt werden. Dadurch wird das Array noch größer, und all die aufgefüllten Token verlangsamen auch das Training!
+
+### Laden von Daten als tf.data.Dataset
+
+Wenn Sie eine Verlangsamung des Trainings vermeiden wollen, können Sie Ihre Daten stattdessen als `tf.data.Dataset` laden. Sie können zwar Ihre eigene
+tf.data"-Pipeline schreiben können, wenn Sie wollen, haben wir zwei bequeme Methoden, um dies zu tun:
+
+- [`~TFPreTrainedModel.prepare_tf_dataset`]: Dies ist die Methode, die wir in den meisten Fällen empfehlen. Da es sich um eine Methode
+Ihres Modells ist, kann sie das Modell inspizieren, um automatisch herauszufinden, welche Spalten als Modelleingaben verwendet werden können, und
+verwirft die anderen, um einen einfacheren, leistungsfähigeren Datensatz zu erstellen.
+- [`~datasets.Dataset.to_tf_dataset`]: Diese Methode ist eher auf niedriger Ebene angesiedelt und ist nützlich, wenn Sie genau kontrollieren wollen, wie
+Dataset erstellt wird, indem man genau angibt, welche `columns` und `label_cols` einbezogen werden sollen.
+
+Bevor Sie [`~TFPreTrainedModel.prepare_tf_dataset`] verwenden können, müssen Sie die Tokenizer-Ausgaben als Spalten zu Ihrem Datensatz hinzufügen, wie in
+dem folgenden Codebeispiel:
+
+```py
+def tokenize_dataset(data):
+    # Keys of the returned dictionary will be added to the dataset as columns
+    return tokenizer(data["text"])
+
+
+dataset = dataset.map(tokenize_dataset)
+```
+
+Denken Sie daran, dass Hugging Face-Datensätze standardmäßig auf der Festplatte gespeichert werden, so dass dies nicht zu einem erhöhten Arbeitsspeicherbedarf führen wird! Sobald die
+Spalten hinzugefügt wurden, können Sie Batches aus dem Datensatz streamen und zu jedem Batch Auffüllungen hinzufügen, was die Anzahl der Auffüllungs-Token im Vergleich zum Auffüllen des gesamten Datensatzes reduziert.
+
+
+```py
+>>> tf_dataset = model.prepare_tf_dataset(dataset, batch_size=16, shuffle=True, tokenizer=tokenizer)
+```
+
+Beachten Sie, dass Sie im obigen Codebeispiel den Tokenizer an `prepare_tf_dataset` übergeben müssen, damit die Stapel beim Laden korrekt aufgefüllt werden können.
+Wenn alle Stichproben in Ihrem Datensatz die gleiche Länge haben und kein Auffüllen erforderlich ist, können Sie dieses Argument weglassen.
+Wenn Sie etwas Komplexeres als nur das Auffüllen von Stichproben benötigen (z. B. das Korrumpieren von Token für die maskierte Sprachmodellierung), können Sie das Argument
+Modellierung), können Sie stattdessen das Argument `collate_fn` verwenden, um eine Funktion zu übergeben, die aufgerufen wird, um die
+Liste von Stichproben in einen Stapel umwandelt und alle gewünschten Vorverarbeitungen vornimmt. Siehe unsere
+[examples](https://github.com/huggingface/transformers/tree/main/examples) oder
+[notebooks](https://huggingface.co/docs/transformers/notebooks), um diesen Ansatz in Aktion zu sehen.
+
+Sobald Sie einen `tf.data.Dataset` erstellt haben, können Sie das Modell wie zuvor kompilieren und anpassen:
+
+```py
+model.compile(optimizer=Adam(3e-5))
+
+model.fit(tf_dataset)
+```
+
+</tf>
+</frameworkcontent>
 
 <a id='pytorch_native'></a>
 
 ## Trainieren in nativem PyTorch
 
+<frameworkcontent>
+<pt>
 <Youtube id="Dh9CL8fyG80"/>
 
 [`Trainer`] kümmert sich um die Trainingsschleife und ermöglicht die Feinabstimmung eines Modells in einer einzigen Codezeile. Für Benutzer, die es vorziehen, ihre eigene Trainingsschleife zu schreiben, können Sie auch eine Feinabstimmung eines 🤗 Transformers-Modells in nativem PyTorch vornehmen.
@@ -299,6 +418,8 @@ Genauso wie Sie eine Bewertungsfunktion zu [`Trainer`] hinzugefügt haben, müss
 
 >>> metric.compute()
 ```
+</pt>
+</frameworkcontent>
 
 <a id='additional-resources'></a>
 
@@ -309,4 +430,4 @@ Weitere Beispiele für die Feinabstimmung finden Sie unter:
 - [🤗 Transformers Examples](https://github.com/huggingface/transformers/tree/main/examples) enthält Skripte
   um gängige NLP-Aufgaben in PyTorch und TensorFlow zu trainieren.
 
-- [🤗 Transformers Notebooks](notebooks) enthält verschiedene Notebooks zur Feinabstimmung eines Modells für bestimmte Aufgaben in PyTorch und TensorFlow.
+- [🤗 Transformers Notebooks](notebooks) enthält verschiedene Notebooks zur Feinabstimmung eines Modells für bestimmte Aufgaben in PyTorch und TensorFlow.

docs/source/en/_toctree.yml

@@ -199,8 +199,6 @@
     title: HIGGS
   - local: quantization/hqq
     title: HQQ
-  - local: quantization/mxfp4
-    title: MXFP4
   - local: quantization/optimum
     title: Optimum
   - local: quantization/quanto
@@ -220,6 +218,8 @@
   sections:
   - local: serialization
     title: ONNX
+  - local: tflite
+    title: LiteRT
   - local: executorch
     title: ExecuTorch
   - local: torchscript
@@ -334,6 +334,8 @@
       title: Configuration
     - local: main_classes/data_collator
       title: Data Collator
+    - local: main_classes/keras_callbacks
+      title: Keras callbacks
     - local: main_classes/logging
       title: Logging
     - local: main_classes/model
@@ -407,8 +409,6 @@
         title: Blenderbot Small
       - local: model_doc/bloom
         title: BLOOM
-      - local: model_doc/blt
-        title: BLT
       - local: model_doc/bort
         title: BORT
       - local: model_doc/byt5
@@ -439,8 +439,6 @@
         title: DeBERTa
       - local: model_doc/deberta-v2
         title: DeBERTa-v2
-      - local: model_doc/deepseek_v2
-        title: DeepSeek-V2
       - local: model_doc/deepseek_v3
         title: DeepSeek-V3
       - local: model_doc/dialogpt
@@ -485,8 +483,6 @@
         title: FLAN-UL2
       - local: model_doc/flaubert
         title: FlauBERT
-      - local: model_doc/flex_olmo
-        title: FlexOlmo
       - local: model_doc/fnet
         title: FNet
       - local: model_doc/fsmt
@@ -555,16 +551,12 @@
         title: LED
       - local: model_doc/lfm2
         title: LFM2
-      - local: model_doc/lfm2_vl
-        title: LFM2-VL
       - local: model_doc/llama
         title: LLaMA
       - local: model_doc/llama2
         title: Llama2
       - local: model_doc/llama3
         title: Llama3
-      - local: model_doc/longcat_flash
-        title: LongCatFlash
       - local: model_doc/longformer
         title: Longformer
       - local: model_doc/longt5
@@ -593,8 +585,6 @@
         title: MegatronGPT2
       - local: model_doc/minimax
         title: MiniMax
-      - local: model_doc/ministral
-        title: Ministral
       - local: model_doc/mistral
         title: Mistral
       - local: model_doc/mixtral
@@ -633,8 +623,6 @@
         title: OLMo
       - local: model_doc/olmo2
         title: OLMo2
-      - local: model_doc/olmo3
-        title: Olmo3
       - local: model_doc/olmoe
         title: OLMoE
       - local: model_doc/open-llama
@@ -669,8 +657,6 @@
         title: Qwen3
       - local: model_doc/qwen3_moe
         title: Qwen3MoE
-      - local: model_doc/qwen3_next
-        title: Qwen3Next
       - local: model_doc/rag
         title: RAG
       - local: model_doc/realm
@@ -719,8 +705,6 @@
         title: UL2
       - local: model_doc/umt5
         title: UMT5
-      - local: model_doc/vaultgemma
-        title: VaultGemma
       - local: model_doc/xmod
         title: X-MOD
       - local: model_doc/xglm
@@ -765,6 +749,12 @@
         title: D-FINE
       - local: model_doc/dab-detr
         title: DAB-DETR
+      - local: model_doc/deepseek_v2
+        title: DeepSeek-V2
+      - local: model_doc/deepseek_vl
+        title: DeepseekVL
+      - local: model_doc/deepseek_vl_hybrid
+        title: DeepseekVLHybrid
       - local: model_doc/deformable_detr
         title: Deformable DETR
       - local: model_doc/deit
@@ -847,16 +837,10 @@
         title: RT-DETR
       - local: model_doc/rt_detr_v2
         title: RT-DETRv2
-      - local: model_doc/sam2
-        title: SAM2
       - local: model_doc/segformer
         title: SegFormer
       - local: model_doc/seggpt
         title: SegGpt
-      - local: model_doc/sam
-        title: Segment Anything
-      - local: model_doc/sam_hq
-        title: Segment Anything High Quality
       - local: model_doc/superglue
         title: SuperGlue
       - local: model_doc/superpoint
@@ -979,8 +963,6 @@
         title: XLSR-Wav2Vec2
       title: Audio models
     - sections:
-      - local: model_doc/sam2_video
-        title: SAM2 Video
       - local: model_doc/timesformer
         title: TimeSformer
       - local: model_doc/vjepa2
@@ -1025,10 +1007,6 @@
         title: ColQwen2
       - local: model_doc/data2vec
         title: Data2Vec
-      - local: model_doc/deepseek_vl
-        title: DeepseekVL
-      - local: model_doc/deepseek_vl_hybrid
-        title: DeepseekVLHybrid
       - local: model_doc/deplot
         title: DePlot
       - local: model_doc/donut
@@ -1143,12 +1121,14 @@
         title: Qwen2Audio
       - local: model_doc/qwen2_vl
         title: Qwen2VL
-      - local: model_doc/qwen3_omni_moe
-        title: Qwen3-Omni-MoE
-      - local: model_doc/qwen3_vl
-        title: Qwen3VL
-      - local: model_doc/qwen3_vl_moe
-        title: Qwen3VLMoe
+      - local: model_doc/sam2
+        title: SAM2
+      - local: model_doc/sam2_video
+        title: SAM2 Video
+      - local: model_doc/sam
+        title: Segment Anything
+      - local: model_doc/sam_hq
+        title: Segment Anything High Quality
       - local: model_doc/shieldgemma2
         title: ShieldGemma2
       - local: model_doc/siglip

docs/source/en/accelerator_selection.md

@@ -69,6 +69,7 @@ CUDA_VISIBLE_DEVICES=0,2 torchrun trainer-program.py ...
 Only GPUs 0 and 2 are "visible" to PyTorch and are mapped to `cuda:0` and `cuda:1` respectively.  
 To reverse the order (use GPU 2 as `cuda:0` and GPU 0 as `cuda:1`):
 
+
 ```bash
 CUDA_VISIBLE_DEVICES=2,0 torchrun trainer-program.py ...
 ```
@@ -107,6 +108,7 @@ To reverse the order (use XPU 2 as `xpu:0` and XPU 0 as `xpu:1`):
 ZE_AFFINITY_MASK=2,0 torchrun trainer-program.py ...
 ```
 
+
 You can also control the order of Intel XPUs with:
 
 ```bash
@@ -118,5 +120,7 @@ For more information about device enumeration and sorting on Intel XPU, please r
 </hfoption>
 </hfoptions>
 
+
+
 > [!WARNING]
 > Environment variables can be exported instead of being added to the command line. This is not recommended because it can be confusing if you forget how the environment variable was set up and you end up using the wrong accelerators. Instead, it is common practice to set the environment variable for a specific training run on the same command line.

docs/source/en/add_new_model.md

@@ -278,7 +278,7 @@ Every Transformers model output should have a precision or error tolerance of *1
 
 Here are some tips for an efficient debugging environment.
 
-- To debug intermediate results, it depends on the machine learning framework the original model repository is using. For PyTorch, you should write a script to decompose the original model into smaller sub-components to retrieve the intermediate values.
+- To debug intermediate results, it depends on the machine learning framework the original model repository is using. For PyTorch, you should write a script to decompose the original model into smaller sub-components to retrieve the intermediate values. For TensorFlow, you may need to use [tf.print](https://www.tensorflow.org/api_docs/python/tf/print). For Flax, make sure the model is *not jitted* during the forward pass (refer to this GitHub [Issue](https://github.com/google/jax/issues/196) for more details).
 
 - It is faster to debug with a smaller pretrained checkpoint versus a larger checkpoint where the forward pass takes more than 10 seconds. If only large checkpoints are available, create a dummy model with randomly initialized weights and save those weights to compare against the Transformers implementation.
 

docs/source/en/auto_docstring.md

@@ -145,6 +145,7 @@ Arguments can also be passed directly to `@auto_docstring` for more control. Use
 
 The `Returns` and `Examples` parts of the docstring can also be manually specified.
 
+
 ```python
 MODEL_COMMON_CUSTOM_ARGS = r"""
     common_arg_1 (`torch.Tensor`, *optional*, defaults to `default_value`):
@@ -201,6 +202,7 @@ There are some rules for documenting different types of arguments and they're li
 
     If a standard argument behaves differently in your model, then you can override it locally in a `r""" """` block. This local definition has a higher priority. For example, the `labels` argument is often customized per model and typically requires overriding.
 
+
 - New or custom arguments should be documented within an `r""" """` block after the signature if it is a function or in the `__init__` method's docstring if it is a class.
 
     ```py

docs/source/en/cache_explanation.md

@@ -59,9 +59,11 @@ Refer to the table below to compare how caching improves efficiency.
 
 | without caching | with caching |
 |---|---|
-| for each step, recompute all previous `K` and `V`  | for each step, only compute current `K` and `V`
+| for each step, recompute all previous `K` and `V`  | for each step, only compute current `K` and `V` 
 | attention cost per step is **quadratic** with sequence length | attention cost per step is **linear** with sequence length (memory grows linearly, but compute/token remains low) |
 
+
+
 ## Cache class
 
 A basic KV cache interface takes a key and value tensor for the current token and returns the updated `K` and `V` tensors. This is internally managed by a model's `forward` method.
@@ -83,7 +85,7 @@ When you use Transformers' [`Cache`] class, the self-attention module performs s
 
 Caches are structured as a list of layers, where each layer contains a key and value cache. The key and value caches are tensors with the shape `[batch_size, num_heads, seq_len, head_dim]`.
 
-Layers can be of different types (e.g. `DynamicLayer`, `StaticLayer`, `StaticSlidingWindowLayer`), which mostly changes how sequence length is handled and how the cache is updated.
+Layers can be of different types (e.g. `DynamicLayer`, `StaticLayer`, `SlidingWindowLayer`), which mostly changes how sequence length is handled and how the cache is updated.
 
 The simplest is a `DynamicLayer` that grows as more tokens are processed. The sequence length dimension (`seq_len`) increases with each new token:
 
@@ -92,7 +94,7 @@ cache.layers[idx].keys = torch.cat([cache.layers[idx].keys, key_states], dim=-2)
 cache.layers[idx].values = torch.cat([cache.layers[idx].values, value_states], dim=-2)
 ```
 
-Other layer types like `StaticLayer` and `StaticSlidingWindowLayer` have a fixed sequence length that is set when the cache is created. This makes them compatible with `torch.compile`. In the case of `StaticSlidingWindowLayer`, existing tokens are shifted out of the cache when a new token is added.
+Other layer types like `StaticLayer` and `SlidingWindowLayer` have a fixed sequence length that is set when the cache is created. This makes them compatible with `torch.compile`. In the case of `SlidingWindowLayer`, existing tokens are shifted out of the cache when a new token is added.
 
 The example below demonstrates how to create a generation loop with [`DynamicCache`]. As discussed, the attention mask is a concatenation of past and current token values and `1` is added to the cache position for the next token.
 
@@ -141,6 +143,7 @@ Cache position is used internally for two purposes:
 
 The generation loop usually takes care of the cache position, but if you're writing a custom generation method, it is important that cache positions are accurate since they are used to write and read key/value states into fixed slots.
 
+
 ```py
 import torch
 from transformers import AutoTokenizer, AutoModelForCausalLM, DynamicCache, infer_device
@@ -157,6 +160,7 @@ generated_ids = model.generate(**inputs, use_cache=True, max_new_tokens=10)
 
 ```
 
+
 ## Legacy cache format
 
 Before the [`Cache`] class, the cache used to be stored as a tuple of tuples of tensors. This format is dynamic because it grows as text is generated, similar to [`DynamicCache`].

docs/source/en/chat_extras.md

@@ -16,7 +16,7 @@ rendered properly in your Markdown viewer.
 
 # Tool use
 
-Chat models are commonly trained with support for "function-calling" or "tool-use". Tools are functions supplied by the user, which the model can choose to call as part of its response. For example, models could have access to a calculator tool to perform arithmetic without having to perform the computation internally.
+Chat models are commonly trained with support for "function-calling" or "tool-use". Tools are functions supplied by the user, which the model can choose to call as part of its response. For example, models could have access to a calculator tool to perform arithmetic without having to it internally.
 
 This guide will demonstrate how to define tools, how to pass them to a chat model, and how to handle the model's output when it calls a tool.
 
@@ -29,11 +29,12 @@ the arguments, argument types, and function docstring are parsed in order to gen
 Although passing Python functions is very convenient, the parser can only handle [Google-style](https://google.github.io/styleguide/pyguide.html#38-comments-and-docstrings)
 docstrings. Refer to the examples below for how to format a tool-ready function.
 
+
 ```py
 def get_current_temperature(location: str, unit: str):
     """
     Get the current temperature at a location.
-
+    
     Args:
         location: The location to get the temperature for, in the format "City, Country"
         unit: The unit to return the temperature in. (choices: ["celsius", "fahrenheit"])
@@ -43,7 +44,7 @@ def get_current_temperature(location: str, unit: str):
 def get_current_wind_speed(location: str):
     """
     Get the current wind speed in km/h at a given location.
-
+    
     Args:
         location: The location to get the wind speed for, in the format "City, Country"
     """
@@ -102,6 +103,7 @@ Hold the call in the `tool_calls` key of an `assistant` message. This is the rec
 > [!WARNING]
 > Although `tool_calls` is similar to the OpenAI API, the OpenAI API uses a JSON string as its `tool_calls` format. This may cause errors or strange model behavior if used in Transformers, which expects a dict.
 
+
 ```py
 tool_call = {"name": "get_current_temperature", "arguments": {"location": "Paris, France", "unit": "celsius"}}
 messages.append({"role": "assistant", "tool_calls": [{"type": "function", "function": tool_call}]})
@@ -129,6 +131,7 @@ The temperature in Paris, France right now is 22°C.<|im_end|>
 > Although the key in the assistant message is called `tool_calls`, in most cases, models only emit a single tool call at a time. Some older models emit multiple tool calls at the same time, but this is a
 > significantly more complex process, as you need to handle multiple tool responses at once and disambiguate them, often using tool call IDs. Please refer to the model card to see exactly what format a model expects for tool calls.
 
+
 ## JSON schemas
 
 Another way to define tools is by passing a [JSON schema](https://json-schema.org/learn/getting-started-step-by-step).
@@ -144,7 +147,7 @@ from transformers.utils import get_json_schema
 def multiply(a: float, b: float):
     """
     A function that multiplies two numbers
-
+    
     Args:
         a: The first number to multiply
         b: The second number to multiply
@@ -157,22 +160,22 @@ print(schema)
 
 ```json
 {
-  "type": "function",
+  "type": "function", 
   "function": {
-    "name": "multiply",
-    "description": "A function that multiplies two numbers",
+    "name": "multiply", 
+    "description": "A function that multiplies two numbers", 
     "parameters": {
-      "type": "object",
+      "type": "object", 
       "properties": {
         "a": {
-          "type": "number",
+          "type": "number", 
           "description": "The first number to multiply"
-        },
+        }, 
         "b": {
           "type": "number",
           "description": "The second number to multiply"
         }
-      },
+      }, 
       "required": ["a", "b"]
     }
   }
@@ -184,7 +187,7 @@ We won't go into the details of JSON schema itself here, since it's already [ver
 ```py
 # A simple function that takes no arguments
 current_time = {
-  "type": "function",
+  "type": "function", 
   "function": {
     "name": "current_time",
     "description": "Get the current local time as a string.",
@@ -200,18 +203,18 @@ multiply = {
   'type': 'function',
   'function': {
     'name': 'multiply',
-    'description': 'A function that multiplies two numbers',
+    'description': 'A function that multiplies two numbers', 
     'parameters': {
-      'type': 'object',
+      'type': 'object', 
       'properties': {
         'a': {
           'type': 'number',
           'description': 'The first number to multiply'
-        },
+        }, 
         'b': {
           'type': 'number', 'description': 'The second number to multiply'
         }
-      },
+      }, 
       'required': ['a', 'b']
     }
   }

docs/source/en/chat_templating.md

@@ -16,13 +16,13 @@ rendered properly in your Markdown viewer.
 
 # Chat templates
 
-The [chat basics](./conversations) guide covers how to store chat histories and generate text from chat models using [`TextGenerationPipeline`].
+The [chat basics](./conversations) guide covers how to store chat histories and generate text from chat models using [`TextGenerationPipeline`]. 
 
 This guide is intended for more advanced users, and covers the underlying classes and methods, as well as the key concepts for understanding what's actually going on when you chat with a model.
 
 The critical insight needed to understand chat models is this: All causal LMs, whether chat-trained or not, continue a sequence of tokens. When causal LMs are trained, the training usually begins with "pre-training" on a huge corpus of text, which creates a "base" model.
 These base models are then often "fine-tuned" for chat, which means training them on data that is formatted as a sequence of messages. The chat is still just a sequence of tokens, though! The list of `role` and `content` dictionaries that you pass
-to a chat model get converted to a token sequence, often with control tokens like `<|user|>` or `<|assistant|>` or `<|end_of_message|>`, which allow the model to see the chat structure.
+to a chat model get converted to a token sequence, often with control tokens like `<|user|>` or `<|assistant|>` or `<|end_of_message|>`, which allow the model to see the chat structure. 
 There are many possible chat formats, and different models may use different formats or control tokens, even if they were fine-tuned from the same base model!
 
 Don't panic, though - you don't need to memorize every possible chat format in order to use chat models. Chat models come with **chat templates**, which indicate how they expect chats to be formatted.
@@ -43,7 +43,6 @@ chat = [
 
 tokenizer.apply_chat_template(chat, tokenize=False)
 ```
-
 ```md
 <s>[INST] Hello, how are you? [/INST]I'm doing great. How can I help you today?</s> [INST] I'd like to show off how chat templating works! [/INST]
 ```
@@ -63,7 +62,6 @@ chat = [
 
 tokenizer.apply_chat_template(chat, tokenize=False)
 ```
-
 ```md
 <|user|>\nHello, how are you?</s>\n<|assistant|>\nI'm doing great. How can I help you today?</s>\n<|user|>\nI'd like to show off how chat templating works!</s>\n
 ```
@@ -112,7 +110,6 @@ Pass the tokenized chat to [`~GenerationMixin.generate`] to generate a response.
 outputs = model.generate(tokenized_chat, max_new_tokens=128) 
 print(tokenizer.decode(outputs[0]))
 ```
-
 ```md
 <|system|>
 You are a friendly chatbot who always responds in the style of a pirate</s>
@@ -128,9 +125,9 @@ Matey, I'm afraid I must inform ye that humans cannot eat helicopters. Helicopte
 
 ### add_generation_prompt
 
-You may have noticed the [add_generation_prompt](https://huggingface.co/docs/transformers/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.apply_chat_template.add_generation_prompt) argument in the above examples.
+You may have noticed the [add_generation_prompt](https://huggingface.co/docs/transformers/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.apply_chat_template.add_generation_prompt) argument in the above examples. 
 This argument adds tokens to the end of the chat that indicate the start of an `assistant` response. Remember: Beneath all the chat abstractions, chat models are still just language models that continue a sequence of tokens!
-If you include tokens that tell it that it's now in an `assistant` response, it will correctly write a response, but if you don't include these tokens, the model may get confused and do something strange, like **continuing** the user's message instead of replying to it!
+If you include tokens that tell it that it's now in an `assistant` response, it will correctly write a response, but if you don't include these tokens, the model may get confused and do something strange, like **continuing** the user's message instead of replying to it! 
 
 Let's see an example to understand what `add_generation_prompt` is actually doing. First, let's format a chat without `add_generation_prompt`:
 
@@ -138,7 +135,6 @@ Let's see an example to understand what `add_generation_prompt` is actually doin
 tokenized_chat = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=False)
 tokenized_chat
 ```
-
 ```md
 <|im_start|>user
 Hi there!<|im_end|>
@@ -154,7 +150,6 @@ Now, let's format the same chat with `add_generation_prompt=True`:
 tokenized_chat = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
 tokenized_chat
 ```
-
 ```md
 <|im_start|>user
 Hi there!<|im_end|>
@@ -191,9 +186,10 @@ model.generate(**formatted_chat)
 
 [`TextGenerationPipeline`] sets [add_generation_prompt](https://huggingface.co/docs/transformers/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.apply_chat_template.add_generation_prompt) to `True` by default to start a new message. However, if the final message in the chat has the `assistant` role, it assumes the message is a prefill and switches to `continue_final_message=True`. This is because most models don’t support multiple consecutive assistant messages. To override this behavior, explicitly pass the [continue_final_message](https://huggingface.co/docs/transformers/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.apply_chat_template.continue_final_message) argument to the pipeline.
 
+
 ## Model training
 
-Training a model with a chat template is a good way to ensure the template matches the tokens the model was trained on. Apply the chat template as a preprocessing step to your dataset. Set `add_generation_prompt=False` because the additional tokens to prompt an assistant response aren't helpful during training.
+Training a model with a chat template is a good way to ensure the template matches the tokens the model was trained on. Apply the chat template as a preprocessing step to your dataset. Set `add_generation_prompt=False` because the additional tokens to prompt an assistant response aren’t helpful during training.
 
 An example of preprocessing a dataset with a chat template is shown below.
 
@@ -216,7 +212,6 @@ dataset = Dataset.from_dict({"chat": [chat1, chat2]})
 dataset = dataset.map(lambda x: {"formatted_chat": tokenizer.apply_chat_template(x["chat"], tokenize=False, add_generation_prompt=False)})
 print(dataset['formatted_chat'][0])
 ```
-
 ```md
 <|user|>
 Which is bigger, the moon or the sun?</s>

docs/source/en/chat_templating_multimodal.md

@@ -18,7 +18,8 @@ rendered properly in your Markdown viewer.
 
 Multimodal chat models accept inputs like images, audio or video, in addition to text. The `content` key in a multimodal chat history is a list containing multiple items of different types. This is unlike text-only chat models whose `content` key is a single string.
 
-In the same way the [Tokenizer](./fast_tokenizer) class handles chat templates and tokenization for text-only models,
+
+In the same way the [Tokenizer](./fast_tokenizer) class handles chat templates and tokenization for text-only models, 
 the [Processor](./processors) class handles preprocessing, tokenization and chat templates for multimodal models. Their [`~ProcessorMixin.apply_chat_template`] methods are almost identical.
 
 This guide will show you how to chat with multimodal models with the high-level [`ImageTextToTextPipeline`] and at a lower level using the [`~ProcessorMixin.apply_chat_template`] and [`~GenerationMixin.generate`] methods.
@@ -45,7 +46,7 @@ messages = [
 ]
 ```
 
-Create an [`ImageTextToTextPipeline`] and pass the chat to it. For large models, setting [device_map="auto"](./models#big-model-inference) helps load the model quicker and automatically places it on the fastest device available. Setting the data type to [auto](./models#model-data-type) also helps save memory and improve speed.
+Create an [`ImageTextToTextPipeline`] and pass the chat to it. For large models, setting [device_map=“auto”](./models#big-model-inference) helps load the model quicker and automatically places it on the fastest device available. Setting the data type to [auto](./models#model-data-type) also helps save memory and improve speed.
 
 ```python
 import torch
@@ -56,6 +57,7 @@ out = pipe(text=messages, max_new_tokens=128)
 print(out[0]['generated_text'][-1]['content'])
 ```
 
+
 ```
 Ahoy, me hearty! These be two feline friends, likely some tabby cats, taking a siesta on a cozy pink blanket. They're resting near remote controls, perhaps after watching some TV or just enjoying some quiet time together. Cats sure know how to find comfort and relaxation, don't they?
 ```
@@ -64,9 +66,10 @@ Aside from the gradual descent from pirate-speak into modern American English (i
 
 ## Using `apply_chat_template`
 
-Like [text-only models](./chat_templating), use the [`~ProcessorMixin.apply_chat_template`] method to prepare the chat messages for multimodal models.
+Like [text-only models](./chat_templating), use the [`~ProcessorMixin.apply_chat_template`] method to prepare the chat messages for multimodal models. 
 This method handles the tokenization and formatting of the chat messages, including images and other media types. The resulting inputs are passed to the model for generation.
 
+
 ```python
 from transformers import AutoProcessor, AutoModelForImageTextToText
 
@@ -96,6 +99,7 @@ processed_chat = processor.apply_chat_template(messages, add_generation_prompt=T
 print(list(processed_chat.keys()))
 ```
 
+
 ```
 ['input_ids', 'attention_mask', 'pixel_values', 'image_grid_thw']
 ```
@@ -109,6 +113,7 @@ print(processor.decode(out[0]))
 
 The decoded output contains the full conversation so far, including the user message and the placeholder tokens that contain the image information. You may need to trim the previous conversation from the output before displaying it to the user.
 
+
 ## Video inputs
 
 Some vision models also support video inputs. The message format is very similar to the format for [image inputs](#image-inputs).
@@ -143,7 +148,6 @@ messages = [
 ```
 
 ### Example: Passing decoded video objects
-
 ```python
 import numpy as np
 
@@ -163,9 +167,7 @@ messages = [
     },
 ]
 ```
-
 You can also use existing (`"load_video()"`) function to load a video, edit the video in memory and pass it in the messages.
-
 ```python
 
 # Make sure a video backend library (pyav, decord, or torchvision) is available.
@@ -193,11 +195,16 @@ messages = [
 
 Pass `messages` to [`~ProcessorMixin.apply_chat_template`] to tokenize the input content. There are a few extra parameters to include in [`~ProcessorMixin.apply_chat_template`] that controls the sampling process.
 
+The `video_load_backend` parameter refers to a specific framework to load a video. It supports [PyAV](https://pyav.basswood-io.com/docs/stable/), [Decord](https://github.com/dmlc/decord), [OpenCV](https://github.com/opencv/opencv), and [torchvision](https://pytorch.org/vision/stable/index.html).
+
+The examples below use Decord as the backend because it is a bit faster than PyAV.
+
 <hfoptions id="sampling">
 <hfoption id="fixed number of frames">
 
 The `num_frames` parameter controls how many frames to uniformly sample from the video. Each checkpoint has a maximum frame count it was pretrained with and exceeding this count can significantly lower generation quality. It's important to choose a frame count that fits both the model capacity and your hardware resources. If `num_frames` isn't specified, the entire video is loaded without any frame sampling.
 
+
 ```python
 processed_chat = processor.apply_chat_template(
     messages,
@@ -206,6 +213,7 @@ processed_chat = processor.apply_chat_template(
     return_dict=True,
     return_tensors="pt",
     num_frames=32,
+    video_load_backend="decord",
 )
 print(processed_chat.keys())
 ```
@@ -215,7 +223,7 @@ These inputs are now ready to be used in [`~GenerationMixin.generate`].
 </hfoption>
 <hfoption id="fps">
 
-For longer videos, it may be better to sample more frames for better representation with the `fps` parameter. This determines how many frames per second to extract. As an example, if a video is 10 seconds long and `fps=2`, then the model samples 20 frames. In other words, 2 frames are uniformly sampled every 10 seconds.
+For longer videos, it may be better to sample more frames for better representation with the `video_fps` parameter. This determines how many frames per second to extract. As an example, if a video is 10 seconds long and `video_fps=2`, then the model samples 20 frames. In other words, 2 frames are uniformly sampled every 10 seconds.
 
 ```py
 processed_chat = processor.apply_chat_template(
@@ -223,7 +231,8 @@ processed_chat = processor.apply_chat_template(
     add_generation_prompt=True,
     tokenize=True,
     return_dict=True,
-    fps=16,
+    video_fps=16,
+    video_load_backend="decord",
 )
 print(processed_chat.keys())
 ```
@@ -262,3 +271,4 @@ print(processed_chat.keys())
 
 </hfoption>
 </hfoptions>
+

docs/source/en/chat_templating_writing.md

@@ -18,6 +18,7 @@ rendered properly in your Markdown viewer.
 
 A chat template is a [Jinja](https://jinja.palletsprojects.com/en/stable/templates/) template stored in the tokenizer's [chat_template](https://huggingface.co/docs/transformers/main_classes/tokenizer#transformers.PreTrainedTokenizer.chat_template) attribute. Jinja is a templating language that allows you to write Python-like code and syntax.
 
+
 ```jinja
 {%- for message in messages %}
     {{- '<|' + message['role'] + |>\n' }}
@@ -29,8 +30,8 @@ A chat template is a [Jinja](https://jinja.palletsprojects.com/en/stable/templat
 ```
 
 If you stare at this for a while, you should realize that this is actually very like Python, albeit with some strange
-`{%-` syntax. The template iterates over a list of messages, and for each message, it prints the role and content of
-the message, followed by an end-of-sequence token. If `add_generation_prompt=True`, it adds
+`{%-` syntax. The template iterates over a list of messages, and for each message, it prints the role and content of 
+the message, followed by an end-of-sequence token. If `add_generation_prompt=True`, it adds 
 the starting header for an assistant message to the end of the conversation.
 
 Load the written template as a string and assign it to the tokenizer's `chat_template` attribute. Once set, the template is used whenever you call [`~PreTrainedTokenizerBase.apply_chat_template`]. It is also saved
@@ -41,7 +42,7 @@ edit this file directly to change the template, which is often easier than manip
 
 The easiest way to start writing Jinja templates is to refer to existing templates. Use `print(tokenizer.chat_template)` on any chat model to see the template it's using. Try starting with simple models that don't call any tools or support RAG because tool-use models can have very complex templates. Finally, take a look at the [Jinja documentation](https://jinja.palletsprojects.com/en/stable/templates/#synopsis) for more details about formatting and syntax.
 
-There are some specific tips and pitfalls you may encounter while writing chat templates specifically, though, and this section will cover some of them in more detail.
+There are some specific tips and pitfalls you may encounter while writing chat templates specifically, though, and this section will cover some of them in more detail. 
 
 ### Writing multimodal chat templates
 
@@ -107,6 +108,7 @@ We strongly recommend using `-` to ensure only the intended content is printed.
 
 ### Special variables and callables
 
+
 The only constants in a template are the `messages` variable and the `add_generation_prompt` boolean. However, you have
 access to **any other keyword arguments that are passed** to the [`~PreTrainedTokenizerBase.apply_chat_template`] method.
 
@@ -131,7 +133,7 @@ Make the changes below to ensure compatibility across all Jinja implementations.
 
 ### Big templates
 
-Newer models or models with features like [tool-calling](./chat_extras) and RAG require larger templates that can be longer than 100 lines. It may be easier to write larger templates in a separate file. The line numbers in the separate file corresponds exactly to the line numbers in template parsing or execution errors, making it easier to debug any potential issues.
+Newer models or models with features like [tool-calling](./chat_extras#tools) and [RAG](./chat_extras#retrieval-augmented-generation-rag) require larger templates that can be longer than 100 lines. It may be easier to write larger templates in a separate file. The line numbers in the separate file corresponds exactly to the line numbers in template parsing or execution errors, making it easier to debug any potential issues.
 
 Write the template in a separate file and extract it to the chat template.
 
@@ -164,22 +166,22 @@ The example below shows how a tool is defined in JSON schema format.
 
 ```json
 {
-  "type": "function",
+  "type": "function", 
   "function": {
-    "name": "multiply",
-    "description": "A function that multiplies two numbers",
+    "name": "multiply", 
+    "description": "A function that multiplies two numbers", 
     "parameters": {
-      "type": "object",
+      "type": "object", 
       "properties": {
         "a": {
-          "type": "number",
+          "type": "number", 
           "description": "The first number to multiply"
-        },
+        }, 
         "b": {
           "type": "number",
           "description": "The second number to multiply"
         }
-      },
+      }, 
       "required": ["a", "b"]
     }
   }

docs/source/en/conversations.md

@@ -48,6 +48,7 @@ transformers chat -h
 
 The chat is implemented on top of the [AutoClass](./model_doc/auto), using tooling from [text generation](./llm_tutorial) and [chat](./chat_templating). It uses the `transformers serve` CLI under the hood ([docs](./serving.md#serve-cli)).
 
+
 ## TextGenerationPipeline
 
 [`TextGenerationPipeline`] is a high-level text generation class with a "chat mode". Chat mode is enabled when a conversational model is detected and the chat prompt is [properly formatted](./llm_tutorial#wrong-prompt-format).
@@ -108,7 +109,7 @@ quantization_config = BitsAndBytesConfig(load_in_8bit=True)
 pipeline = pipeline(task="text-generation", model="meta-llama/Meta-Llama-3-8B-Instruct", device_map="auto", model_kwargs={"quantization_config": quantization_config})
 ```
 
-In general, model size and performance are directly correlated. Larger models are slower in addition to requiring more memory because each active parameter must be read from memory for every generated token.
+In general, model size and performance are directly correlated. Larger models are slower in addition to requiring more memory because each active parameter must be read from memory for every generated token. 
 This is a bottleneck for LLM text generation and the main options for improving generation speed are to either quantize a model or use hardware with higher memory bandwidth. Adding more compute power doesn't meaningfully help.
 
 You can also try techniques like [speculative decoding](./generation_strategies#speculative-decoding), where a smaller model generates candidate tokens that are verified by the larger model. If the candidate tokens are correct, the larger model can generate more than one token at a time. This significantly alleviates the bandwidth bottleneck and improves generation speed.

docs/source/en/cursor.md

@@ -38,3 +38,5 @@ You are now ready to use your local model in Cursor! For instance, if you toggle
 <h3 align="center">
     <img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/transformers_serve_cursor_chat.png"/>
 </h3>
+
+

docs/source/en/debugging.md

@@ -35,7 +35,7 @@ pip install deepspeed
 
 PyTorch comes with its own CUDA toolkit, but to use DeepSpeed with PyTorch, you need to have an identical version of CUDA installed system-wide. For example, if you installed PyTorch with `cudatoolkit==10.2` in your Python environment, then you'll also need to have CUDA 10.2 installed everywhere.
 
-The exact location can vary from system to system, but `/usr/local/cuda-10.2` is the most common location on many Unix systems. When CUDA is correctly set up and added to your `PATH` environment variable, you can find the installation location with the following command.
+The exact location can vary from system to system, but `usr/local/cuda-10.2` is the most common location on many Unix systems. When CUDA is correctly set up and added to your `PATH` environment variable, you can find the installation location with the following command.
 
 ```bash
 which nvcc

docs/source/en/fast_tokenizers.md

@@ -226,7 +226,7 @@ tokenizer = PreTrainedTokenizerFast.from_pretrained("config/save/dir")
 
 <Youtube id="Yffk5aydLzg"/>
 
-A Transformers model expects the input to be a PyTorch or NumPy tensor. A tokenizer's job is to preprocess text into those tensors. Specify the framework tensor type to return with the `return_tensors` parameter.
+A Transformers model expects the input to be a PyTorch or NumPy tensor. A tokenizers job is to preprocess text into those tensors. Specify the framework tensor type to return with the `return_tensors` parameter.
 
 ```py
 from transformers import AutoTokenizer

docs/source/en/generation_strategies.md

@@ -389,6 +389,7 @@ from .utils import some_function
 
 Only relative imports from the same-level `custom_generate` folder are supported. Parent/sibling folder imports are not valid. The `custom_generate` argument also works locally with any directory that contains a `custom_generate` structure. This is the recommended workflow for developing your custom generation method.
 
+
 #### requirements.txt
 
 You can optionally specify additional Python requirements in a `requirements.txt` file inside the `custom_generate` folder. These are checked at runtime and an exception will be thrown if they're missing, nudging users to update their environment accordingly.

docs/source/en/index.md

@@ -19,6 +19,7 @@ rendered properly in your Markdown viewer.
     <img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/transformers_as_a_model_definition.png"/>
 </h3>
 
+
 Transformers acts as the model-definition framework for state-of-the-art machine learning models in text, computer
 vision, audio, video, and multimodal model, for both inference and training.
 

docs/source/en/installation.md

@@ -24,23 +24,46 @@ Transformers works with [PyTorch](https://pytorch.org/get-started/locally/). It
 
 ## Virtual environment
 
-[uv](https://docs.astral.sh/uv/) is an extremely fast Rust-based Python package and project manager and requires a [virtual environment](https://docs.astral.sh/uv/pip/environments/) by default to manage different projects and avoids compatibility issues between dependencies.
+A virtual environment helps manage different projects and avoids compatibility issues between dependencies. Take a look at the [Install packages in a virtual environment using pip and venv](https://packaging.python.org/en/latest/guides/installing-using-pip-and-virtual-environments/) guide if you're unfamiliar with Python virtual environments.
 
-It can be used as a drop-in replacement for [pip](https://pip.pypa.io/en/stable/), but if you prefer to use pip, remove `uv` from the commands below.
+<hfoptions id="virtual">
+<hfoption id="venv">
 
-> [!TIP]
-> Refer to the uv [installation](https://docs.astral.sh/uv/guides/install-python/) docs to install uv.
+Create and activate a virtual environment in your project directory with [venv](https://docs.python.org/3/library/venv.html).
 
-Create a virtual environment to install Transformers in.
+```bash
+python -m venv .env
+source .env/bin/activate
+```
+
+</hfoption>
+<hfoption id="uv">
+
+[uv](https://docs.astral.sh/uv/) is a fast Rust-based Python package and project manager.
 
 ```bash
 uv venv .env
 source .env/bin/activate
 ```
 
+</hfoption>
+</hfoptions>
+
 ## Python
 
-Install Transformers with the following command.
+You can install Transformers with pip or uv.
+
+<hfoptions id="install">
+<hfoption id="pip">
+
+[pip](https://pip.pypa.io/en/stable/) is a package installer for Python. Install Transformers with pip in your newly created virtual environment.
+
+```bash
+pip install transformers
+```
+
+</hfoption>
+<hfoption id="uv">
 
 [uv](https://docs.astral.sh/uv/) is a fast Rust-based Python package and project manager.
 
@@ -48,6 +71,9 @@ Install Transformers with the following command.
 uv pip install transformers
 ```
 
+</hfoption>
+</hfoptions>
+
 For GPU acceleration, install the appropriate CUDA drivers for [PyTorch](https://pytorch.org/get-started/locally).
 
 Run the command below to check if your system detects an NVIDIA GPU.
@@ -56,11 +82,11 @@ Run the command below to check if your system detects an NVIDIA GPU.
 nvidia-smi
 ```
 
-To install a CPU-only version of Transformers, run the following command.
+To install a CPU-only version of Transformers and a machine learning framework, run the following command.
 
 ```bash
-uv pip install torch --index-url https://download.pytorch.org/whl/cpu
-uv pip install transformers
+pip install 'transformers[torch]'
+uv pip install 'transformers[torch]'
 ```
 
 Test whether the install was successful with the following command. It should return a label and score for the provided text.
@@ -79,7 +105,7 @@ The downside is that the latest version may not always be stable. If you encount
 Install from source with the following command.
 
 ```bash
-uv pip install git+https://github.com/huggingface/transformers
+pip install git+https://github.com/huggingface/transformers
 ```
 
 Check if the install was successful with the command below. It should return a label and score for the provided text.
@@ -96,7 +122,7 @@ An [editable install](https://pip.pypa.io/en/stable/topics/local-project-install
 ```bash
 git clone https://github.com/huggingface/transformers.git
 cd transformers
-uv pip install -e .
+pip install -e .
 ```
 
 > [!WARNING]

docs/source/en/internal/file_utils.md

@@ -20,6 +20,7 @@ This page lists all of Transformers general utility functions that are found in
 
 Most of those are only useful if you are studying the general code in the library.
 
+
 ## Enums and namedtuples
 
 [[autodoc]] utils.ExplicitEnum
@@ -40,6 +41,10 @@ Most of those are only useful if you are studying the general code in the librar
 
 [[autodoc]] utils.replace_return_docstrings
 
+## Special Properties
+
+[[autodoc]] utils.cached_property
+
 ## Other Utilities
 
 [[autodoc]] utils._LazyModule

docs/source/en/internal/generation_utils.md

@@ -65,6 +65,7 @@ values. Here, for instance, it has two keys that are `sequences` and `scores`.
 
 We document here all output types.
 
+
 [[autodoc]] generation.GenerateDecoderOnlyOutput
 
 [[autodoc]] generation.GenerateEncoderDecoderOutput
@@ -73,11 +74,13 @@ We document here all output types.
 
 [[autodoc]] generation.GenerateBeamEncoderDecoderOutput
 
+
 ## LogitsProcessor
 
 A [`LogitsProcessor`] can be used to modify the prediction scores of a language model head for
 generation.
 
+
 [[autodoc]] AlternatingCodebooksLogitsProcessor
     - __call__
 
@@ -171,6 +174,8 @@ generation.
 [[autodoc]] WatermarkLogitsProcessor
     - __call__
 
+
+
 ## StoppingCriteria
 
 A [`StoppingCriteria`] can be used to change when to stop generation (other than EOS token). Please note that this is exclusively available to our PyTorch implementations.
@@ -245,7 +250,7 @@ A [`Constraint`] can be used to force the generation to include specific tokens
     - update
     - lazy_initialization
 
-[[autodoc]] StaticSlidingWindowLayer
+[[autodoc]] SlidingWindowLayer
     - update
     - lazy_initialization
 
@@ -295,6 +300,7 @@ A [`Constraint`] can be used to force the generation to include specific tokens
     - to_legacy_cache
     - from_legacy_cache
 
+
 ## Watermark Utils
 
 [[autodoc]] WatermarkingConfig

docs/source/en/internal/import_utils.md

@@ -22,8 +22,8 @@ worked around. We don't want for all users of `transformers` to have to install
 we therefore mark those as soft dependencies rather than hard dependencies.
 
 The transformers toolkit is not made to error-out on import of a model that has a specific dependency; instead, an
-object for which you are lacking a dependency will error-out when calling any method on it. As an example, if
-`torchvision` isn't installed, the fast image processors will not be available.
+object for which you are lacking a dependency will error-out when calling any method on it. As an example, if 
+`torchvision` isn't installed, the fast image processors will not be available. 
 
 This object is still importable:
 
@@ -51,11 +51,16 @@ Let's see how to specify specific object dependencies.
 
 All objects under a given filename have an automatic dependency to the tool linked to the filename
 
-**PyTorch**: All files starting with `modeling_` have an automatic PyTorch dependency
+**TensorFlow**: All files starting with `modeling_tf_` have an automatic TensorFlow dependency.
+
+**Flax**: All files starting with `modeling_flax_` have an automatic Flax dependency
+
+**PyTorch**: All files starting with `modeling_` and not valid with the above (TensorFlow and Flax) have an automatic 
+PyTorch dependency
 
 **Tokenizers**: All files starting with `tokenization_` and ending with `_fast` have an automatic `tokenizers` dependency
 
-**Vision**: All files starting with `image_processing_` have an automatic dependency to the `vision` dependency group;
+**Vision**: All files starting with `image_processing_` have an automatic dependency to the `vision` dependency group; 
 at the time of writing, this only contains the `pillow` dependency.
 
 **Vision + Torch + Torchvision**: All files starting with `image_processing_` and ending with `_fast` have an automatic
@@ -66,7 +71,7 @@ All of these automatic dependencies are added on top of the explicit dependencie
 ### Explicit Object Dependencies
 
 We add a method called `requires` that is used to explicitly specify the dependencies of a given object. As an
-example, the `Trainer` class has two hard dependencies: `torch` and `accelerate`. Here is how we specify these
+example, the `Trainer` class has two hard dependencies: `torch` and `accelerate`. Here is how we specify these 
 required dependencies:
 
 ```python

docs/source/en/internal/model_debugging_utils.md

@@ -21,8 +21,10 @@ provides for it.
 
 Most of those are only useful if you are adding new models in the library.
 
+
 ## Model addition debuggers
 
+
 ### Model addition debugger - context manager for model adders
 
 This context manager is a power user tool intended for model adders. It tracks all forward calls within a model forward
@@ -70,6 +72,7 @@ with model_addition_debugger_context(
 
 ```
 
+
 ### Reading results
 
 The debugger generates two files from the forward call, both with the same base name, but ending either with
@@ -228,11 +231,14 @@ Once the forward passes of two models have been traced by the debugger, one can
 below: we can see slight differences between these two implementations' key projection layer. Inputs are mostly
 identical, but not quite. Looking through the file differences makes it easier to pinpoint which layer is wrong.
 
+
 ![download-icon](https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/files_difference_debugging.png)
 
+
 ### Limitations and scope
 
-This feature will only work for torch-based models. Models relying heavily on external kernel calls may work, but trace will
+This feature will only work for torch-based models, and would require more work and case-by-case approach for say
+`jax`-based models that are usually compiled. Models relying heavily on external kernel calls may work, but trace will
 probably miss some things. Regardless, any python implementation that aims at mimicking another implementation can be
 traced once instead of reran N times with breakpoints.
 
@@ -248,7 +254,7 @@ layers.
 
 This small util is a power user tool intended for model adders and maintainers. It lists all test methods
 existing in `test_modeling_common.py`, inherited by all model tester classes, and scans the repository to measure
-how many tests are being skipped and for which models.
+how many tests are being skipped and for which models. 
 
 ### Rationale
 
@@ -263,7 +269,8 @@ This utility:
 
 ![download-icon](https://huggingface.co/datasets/huggingface/documentation-images/resolve/f7f671f69b88ce4967e19179172c248958d35742/transformers/tests_skipped_visualisation.png)
 
-### Usage
+
+### Usage 
 
 You can run the skipped test analyzer in two ways:
 
@@ -350,4 +357,4 @@ Skipped : 124/323 (38.4%)
 
 📄 JSON saved to /home/pablo/git/transformers/scan_test_inputs_embeds.json
 
-```
+```

docs/source/en/internal/pipelines_utils.md

@@ -20,6 +20,7 @@ This page lists all the utility functions the library provides for pipelines.
 
 Most of those are only useful if you are studying the code of the models in the library.
 
+
 ## Argument handling
 
 [[autodoc]] pipelines.ArgumentHandler

docs/source/en/kv_cache.md

@@ -67,7 +67,7 @@ out = model.generate(**inputs, do_sample=False, max_new_tokens=20, past_key_valu
 
 ## Fixed-size cache
 
-The default [`DynamicCache`] prevents you from taking advantage of most just-in-time (JIT) optimizations because the cache size isn't fixed. JIT optimizations enable you to maximize latency at the expense of memory usage. All of the following cache types are compatible with JIT optimizations like [torch.compile](./llm_optims#static-kv-cache-and-torchcompile) to accelerate generation.
+The default [`DynamicCache`] prevents you from taking advantage of most just-in-time (JIT) optimizations because the cache size isn't fixed. JIT optimizations enable you to maximize latency at the expense of memory usage. All of the following cache types are compatible with JIT optimizations like [torch.compile](./llm_optims#static-kv-cache-and-torchcompile) to accelerate generation. 
 
 A fixed-size cache ([`StaticCache`]) pre-allocates a specific maximum cache size for the kv pairs. You can generate up to the maximum cache size without needing to modify it. However, having a fixed (usually large) size for the key/value states means that while generating, a lot of tokens will actually be masked as they should not take part in the attention. So this trick allows to easily `compile` the decoding stage, but it incurs a waste of tokens in the attention computation. As all things, it's then a trade-off which should be very good if you generate with several sequence of more or less the same lengths, but may be sub-optimal if you have for example 1 very large sequence, and then only short sequences (as the fix cache size would be large, a lot would be wasted for the short sequences). Make sure you understand the impact if you use it!
 

docs/source/en/llm_optims.md

@@ -183,6 +183,36 @@ text
  'My favorite all time favorite condiment is ketchup. I love it on everything. I love it on my eggs, my fries, my chicken, my burgers, my hot dogs, my sandwiches, my salads, my p']
 ```
 
+</hfoption>
+<hfoption id="3. compile entire generate function">
+
+Compiling the entire [`~GenerationMixin.generate`] function also compiles the input preparation logit processor operations, and more, in addition to the forward pass. With this approach, you don't need to initialize [`StaticCache`] or set the [cache_implementation](https://hf.co/docs/transformers/main_classes/text_generation#transformers.GenerationConfig.cache_implementation) parameter.
+
+```py
+from transformers import AutoTokenizer, AutoModelForCausalLM
+import torch
+import os
+os.environ["TOKENIZERS_PARALLELISM"] = "false"  # To prevent long warnings :)
+
+tokenizer = AutoTokenizer.from_pretrained("google/gemma-2b")
+model = AutoModelForCausalLM.from_pretrained("google/gemma-2b", dtype="auto", device_map="auto")
+
+model.generate = torch.compile(model.generate, mode="reduce-overhead", fullgraph=True)
+input_text = "The theory of special relativity states "
+input_ids = tokenizer(input_text, return_tensors="pt").to(model.device.type)
+
+outputs = model.generate(**input_ids)
+print(tokenizer.batch_decode(outputs, skip_special_tokens=True))
+['The theory of special relativity states 1. The speed of light is constant in all inertial reference']
+```
+
+This usage pattern is more appropriate for unique hardware or use cases, but there are several drawbacks to consider.
+
+1. Compilation is much slower.
+2. Parameters must be configured through [`GenerationConfig`].
+3. Many warnings and exceptions are suppressed. We recommend testing the uncompiled model first.
+4. Many features are unavailable at the moment. For example, generation does not stop if an `EOS` token is selected.
+
 </hfoption>
 </hfoptions>
 

docs/source/en/llm_tutorial.md

@@ -23,8 +23,7 @@ Text generation is the most popular application for large language models (LLMs)
 In Transformers, the [`~GenerationMixin.generate`] API handles text generation, and it is available for all models with generative capabilities. This guide will show you the basics of text generation with [`~GenerationMixin.generate`] and some common pitfalls to avoid.
 
 > [!TIP]
-> You can also chat with a model directly from the command line. ([reference](./conversations.md#transformers))
->
+> You can also chat with a model directly from the command line. ([reference](./conversations.md#transformers-cli))
 > ```shell
 > transformers chat Qwen/Qwen2.5-0.5B-Instruct
 > ```
@@ -36,7 +35,6 @@ Before you begin, it's helpful to install [bitsandbytes](https://hf.co/docs/bits
 ```bash
 !pip install -U transformers bitsandbytes
 ```
-
 Bitsandbytes supports multiple backends in addition to CUDA-based GPUs. Refer to the multi-backend installation [guide](https://huggingface.co/docs/bitsandbytes/main/en/installation#multi-backend) to learn more.
 
 Load a LLM with [`~PreTrainedModel.from_pretrained`] and add the following two parameters to reduce the memory requirements.
@@ -156,6 +154,7 @@ print(tokenizer.batch_decode(outputs, skip_special_tokens=True))
 | `repetition_penalty` | `float` | Set it to `>1.0` if you're seeing the model repeat itself often. Larger values apply a larger penalty. |
 | `eos_token_id` | `list[int]` | The token(s) that will cause generation to stop. The default value is usually good, but you can specify a different token. |
 
+
 ## Pitfalls
 
 The section below covers some common issues you may encounter during text generation and how to solve them.

docs/source/en/llm_tutorial_optimization.md

@@ -66,7 +66,6 @@ If you have access to an 8 x 80GB A100 node, you could load BLOOM as follows
 ```bash
 !pip install transformers accelerate bitsandbytes optimum
 ```
-
 ```python
 from transformers import AutoModelForCausalLM
 
@@ -99,7 +98,6 @@ result
 ```
 
 **Output**:
-
 ```
 Here is a Python function that transforms bytes to Giga bytes:\n\n```python\ndef bytes_to_giga_bytes(bytes):\n    return bytes / 1024 / 1024 / 1024\n```\n\nThis function takes a single
 ```
@@ -118,7 +116,6 @@ bytes_to_giga_bytes(torch.cuda.max_memory_allocated())
 ```
 
 **Output**:
-
 ```bash
 29.0260648727417
 ```
@@ -130,6 +127,7 @@ Note that if we had tried to run the model in full float32 precision, a whopping
 
 If you are unsure in which format the model weights are stored on the Hub, you can always look into the checkpoint's config under `"dtype"`, *e.g.* [here](https://huggingface.co/meta-llama/Llama-2-7b-hf/blob/6fdf2e60f86ff2481f2241aaee459f85b5b0bbb9/config.json#L21). It is recommended to set the model to the same precision type as written in the config when loading with `from_pretrained(..., dtype=...)` except when the original type is float32 in which case one can use both `float16` or `bfloat16` for inference.
 
+
 Let's define a `flush(...)` function to free all allocated memory so that we can accurately measure the peak allocated GPU memory.
 
 ```python
@@ -150,7 +148,6 @@ Let's call it now for the next experiment.
 ```python
 flush()
 ```
-
 From the Accelerate library, you can also use a device-agnostic utility method called [release_memory](https://github.com/huggingface/accelerate/blob/29be4788629b772a3b722076e433b5b3b5c85da3/src/accelerate/utils/memory.py#L63), which takes various hardware backends like XPU, MLU, NPU, MPS, and more into account.
 
 ```python
@@ -207,7 +204,6 @@ result
 ```
 
 **Output**:
-
 ```
 Here is a Python function that transforms bytes to Giga bytes:\n\n```python\ndef bytes_to_giga_bytes(bytes):\n    return bytes / 1024 / 1024 / 1024\n```\n\nThis function takes a single
 ```
@@ -219,7 +215,6 @@ bytes_to_giga_bytes(torch.cuda.max_memory_allocated())
 ```
 
 **Output**:
-
 ```
 15.219234466552734
 ```
@@ -227,8 +222,8 @@ bytes_to_giga_bytes(torch.cuda.max_memory_allocated())
 Significantly less! We're down to just a bit over 15 GBs and could therefore run this model on consumer GPUs like the 4090.
 We're seeing a very nice gain in memory efficiency and more or less no degradation to the model's output. However, we can also notice a slight slow-down during inference.
 
-We delete the models and flush the memory again.
 
+We delete the models and flush the memory again.
 ```python
 del model
 del pipe
@@ -250,7 +245,6 @@ result
 ```
 
 **Output**:
-
 ```
 Here is a Python function that transforms bytes to Giga bytes:\n\n```\ndef bytes_to_gigabytes(bytes):\n    return bytes / 1024 / 1024 / 1024\n```\n\nThis function takes a single argument
 ```
@@ -262,7 +256,6 @@ bytes_to_giga_bytes(torch.cuda.max_memory_allocated())
 ```
 
 **Output**:
-
 ```
 9.543574333190918
 ```
@@ -277,7 +270,6 @@ Also note that inference here was again a bit slower compared to 8-bit quantizat
 del model
 del pipe
 ```
-
 ```python
 flush()
 ```
@@ -392,7 +384,6 @@ def alternating(list1, list2):
 -----
 """
 ```
-
 For demonstration purposes, we duplicate the system prompt by ten so that the input length is long enough to observe Flash Attention's memory savings.
 We append the original text prompt `"Question: Please write a function in Python that transforms bytes to Giga bytes.\n\nAnswer: Here"`
 
@@ -422,7 +413,6 @@ result
 ```
 
 **Output**:
-
 ```
 Generated in 10.96854019165039 seconds.
 Sure. Here is a function that does that.\n\ndef bytes_to_giga(bytes):\n   return bytes / 1024 / 1024 / 1024\n\nAnswer: Sure. Here is a function that does that.\n\ndef
@@ -439,7 +429,6 @@ bytes_to_giga_bytes(torch.cuda.max_memory_allocated())
 ```
 
 **Output**:
-
 ```bash
 37.668193340301514
 ```
@@ -471,7 +460,6 @@ result
 ```
 
 **Output**:
-
 ```
 Generated in 3.0211617946624756 seconds.
  Sure. Here is a function that does that.\n\ndef bytes_to_giga(bytes):\n   return bytes / 1024 / 1024 / 1024\n\nAnswer: Sure. Here is a function that does that.\n\ndef
@@ -486,7 +474,6 @@ bytes_to_giga_bytes(torch.cuda.max_memory_allocated())
 ```
 
 **Output**:
-
 ```
 32.617331981658936
 ```
@@ -617,7 +604,6 @@ generated_text
 ```
 
 **Output**:
-
 ```
 shape of input_ids torch.Size([1, 21])
 shape of input_ids torch.Size([1, 22])
@@ -655,7 +641,6 @@ generated_text
 ```
 
 **Output**:
-
 ```
 shape of input_ids torch.Size([1, 1])
 length of key-value cache 20
@@ -727,7 +712,6 @@ tokenizer.batch_decode(generation_output.sequences)[0][len(prompt):]
 ```
 
 **Output**:
-
 ```
  is a modified version of the function that returns Mega bytes instead.
 
@@ -749,7 +733,6 @@ config = model.config
 ```
 
 **Output**:
-
 ```
 7864320000
 ```
@@ -790,6 +773,7 @@ The most notable application of GQA is [Llama-v2](https://huggingface.co/meta-ll
 
 > As a conclusion, it is strongly recommended to make use of either GQA or MQA if the LLM is deployed with auto-regressive decoding and is required to handle large input sequences as is the case for example for chat.
 
+
 ## Conclusion
 
 The research community is constantly coming up with new, nifty ways to speed up inference time for ever-larger LLMs. As an example, one such promising research direction is [speculative decoding](https://huggingface.co/papers/2211.17192) where "easy tokens" are generated by smaller, faster language models and only "hard tokens" are generated by the LLM itself. Going into more detail is out of the scope of this notebook, but can be read upon in this [nice blog post](https://huggingface.co/blog/assisted-generation).

docs/source/en/main_classes/callback.md

@@ -54,6 +54,7 @@ The main class that implements callbacks is [`TrainerCallback`]. It gets the
 Trainer's internal state via [`TrainerState`], and can take some actions on the training loop via
 [`TrainerControl`].
 
+
 ## Available Callbacks
 
 Here is the list of the available [`TrainerCallback`] in the library:

docs/source/en/main_classes/configuration.md

@@ -24,6 +24,7 @@ Each derived config class implements model specific attributes. Common attribute
 `hidden_size`, `num_attention_heads`, and `num_hidden_layers`. Text models further implement:
 `vocab_size`.
 
+
 ## PretrainedConfig
 
 [[autodoc]] PretrainedConfig

docs/source/en/main_classes/data_collator.md

@@ -25,6 +25,7 @@ on the formed batch.
 
 Examples of use can be found in the [example scripts](../examples) or [example notebooks](../notebooks).
 
+
 ## Default data collator
 
 [[autodoc]] data.data_collator.default_data_collator

docs/source/en/main_classes/deepspeed.md

@@ -16,7 +16,7 @@ rendered properly in your Markdown viewer.
 
 # DeepSpeed
 
-[DeepSpeed](https://github.com/deepspeedai/DeepSpeed), powered by Zero Redundancy Optimizer (ZeRO), is an optimization library for training and fitting very large models onto a GPU. It is available in several ZeRO stages, where each stage progressively saves more GPU memory by partitioning the optimizer state, gradients, parameters, and enabling offloading to a CPU or NVMe. DeepSpeed is integrated with the [`Trainer`] class and most of the setup is automatically taken care of for you.
+[DeepSpeed](https://github.com/deepspeedai/DeepSpeed), powered by Zero Redundancy Optimizer (ZeRO), is an optimization library for training and fitting very large models onto a GPU. It is available in several ZeRO stages, where each stage progressively saves more GPU memory by partitioning the optimizer state, gradients, parameters, and enabling offloading to a CPU or NVMe. DeepSpeed is integrated with the [`Trainer`] class and most of the setup is automatically taken care of for you. 
 
 However, if you want to use DeepSpeed without the [`Trainer`], Transformers provides a [`HfDeepSpeedConfig`] class.
 

docs/source/en/main_classes/executorch.md

@@ -15,12 +15,14 @@ rendered properly in your Markdown viewer.
 
 -->
 
+
 # ExecuTorch
 
 [`ExecuTorch`](https://github.com/pytorch/executorch) is an end-to-end solution for enabling on-device inference capabilities across mobile and edge devices including wearables, embedded devices and microcontrollers. It is part of the PyTorch ecosystem and supports the deployment of PyTorch models with a focus on portability, productivity, and performance.
 
 ExecuTorch introduces well defined entry points to perform model, device, and/or use-case specific optimizations such as backend delegation, user-defined compiler transformations, memory planning, and more. The first step in preparing a PyTorch model for execution on an edge device using ExecuTorch is to export the model. This is achieved through the use of a PyTorch API called [`torch.export`](https://pytorch.org/docs/stable/export.html).
 
+
 ## ExecuTorch Integration
 
 An integration point is being developed to ensure that 🤗 Transformers can be exported using `torch.export`. The goal of this integration is not only to enable export but also to ensure that the exported artifact can be further lowered and optimized to run efficiently in `ExecuTorch`, particularly for mobile and edge use cases.

docs/source/en/main_classes/feature_extractor.md

@@ -18,6 +18,7 @@ rendered properly in your Markdown viewer.
 
 A feature extractor is in charge of preparing input features for audio or vision models. This includes feature extraction from sequences, e.g., pre-processing audio files to generate Log-Mel Spectrogram features, feature extraction from images, e.g., cropping image files, but also padding, normalization, and conversion to NumPy and PyTorch tensors.
 
+
 ## FeatureExtractionMixin
 
 [[autodoc]] feature_extraction_utils.FeatureExtractionMixin

docs/source/en/main_classes/image_processor.md

@@ -26,7 +26,6 @@ from transformers import AutoImageProcessor
 
 processor = AutoImageProcessor.from_pretrained("facebook/detr-resnet-50", use_fast=True)
 ```
-
 Note that `use_fast` will be set to `True` by default in a future release.
 
 When using a fast image processor, you can also set the `device` argument to specify the device on which the processing should be done. By default, the processing is done on the same device as the inputs if the inputs are tensors, or on the CPU otherwise.
@@ -58,6 +57,7 @@ Here are some speed comparisons between the base and fast image processors for t
 
 These benchmarks were run on an [AWS EC2 g5.2xlarge instance](https://aws.amazon.com/ec2/instance-types/g5/), utilizing an NVIDIA A10G Tensor Core GPU.
 
+
 ## ImageProcessingMixin
 
 [[autodoc]] image_processing_utils.ImageProcessingMixin
@@ -72,6 +72,7 @@ These benchmarks were run on an [AWS EC2 g5.2xlarge instance](https://aws.amazon
 
 [[autodoc]] image_processing_utils.BaseImageProcessor
 
+
 ## BaseImageProcessorFast
 
 [[autodoc]] image_processing_utils_fast.BaseImageProcessorFast