frozenleaves/trl
1
0
Fork 0
mirror of https://github.com/huggingface/trl.git synced 2025-10-20 18:43:52 +08:00
Code Issues Packages Projects Releases Wiki Activity

🐳 Docker update + Simplify Jobs doc (#3931)

Browse Source 
Co-authored-by: sergiopaniego <sergiopaniegoblanco@gmail.com>
Co-authored-by: Kashif Rasul <kashif.rasul@gmail.com>
This commit is contained in:
Quentin Gallouédec
2025-09-13 18:35:55 -06:00
committed by GitHub
parent 304eaf8053
commit 9955ee7eaa
40 changed files with 244 additions and 486 deletions
Download Patch File Download Diff File Expand all files Collapse all files

87
.github/workflows/docker-build.yml vendored
View File

@ -1,95 +1,84 @@
name: Build Docker images (scheduled) name: Build TRL Docker image
on: on:
push:
branches:
- main
workflow_dispatch: workflow_dispatch:
workflow_call:
schedule:
- cron: "0 1 * * *"
concurrency: concurrency:
group: docker-image-builds group: docker-image-builds
cancel-in-progress: false cancel-in-progress: false
env:
CI_SLACK_CHANNEL: ${{ secrets.CI_DOCKER_CHANNEL }}
jobs: jobs:
trl-latest: trl:
name: "Latest TRL GPU" name: "Build and push TRL Docker image"
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
- name: Cleanup disk - name: Checkout code
run: |
sudo ls -l /usr/local/lib/
sudo ls -l /usr/share/
sudo du -sh /usr/local/lib/
sudo du -sh /usr/share/
sudo rm -rf /usr/local/lib/android
sudo rm -rf /usr/share/dotnet
sudo du -sh /usr/local/lib/
sudo du -sh /usr/share/
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
- name: Check out code
uses: actions/checkout@v4 uses: actions/checkout@v4
- name: Get TRL version from PyPI
run: |
VERSION=$(curl -s https://pypi.org/pypi/trl/json | jq -r .info.version)
echo "VERSION=$VERSION" >> $GITHUB_ENV
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Login to DockerHub - name: Login to DockerHub
uses: docker/login-action@v1 uses: docker/login-action@v3
with: with:
username: ${{ secrets.DOCKERHUB_USERNAME }} username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_PASSWORD }} password: ${{ secrets.DOCKERHUB_PASSWORD }}
- name: Build and Push GPU - name: Build and Push
uses: docker/build-push-action@v4 uses: docker/build-push-action@v4
with: with:
context: ./docker/trl-latest-gpu context: docker/trl
push: true push: true
tags: huggingface/trl-latest-gpu tags: |
huggingface/trl:${{ env.VERSION }}
huggingface/trl
- name: Post to Slack - name: Post to Slack
if: always() if: always()
uses: huggingface/hf-workflows/.github/actions/post-slack@main uses: huggingface/hf-workflows/.github/actions/post-slack@main
with: with:
slack_channel: ${{ env.CI_SLACK_CHANNEL }} slack_channel: ${{ secrets.CI_DOCKER_CHANNEL }}
title: 🤗 Results of the trl-latest-gpu Docker Image build title: 🤗 Results of the TRL Dev Docker Image build
status: ${{ job.status }} status: ${{ job.status }}
slack_token: ${{ secrets.SLACK_CIFEEDBACK_BOT_TOKEN }} slack_token: ${{ secrets.SLACK_CIFEEDBACK_BOT_TOKEN }}
trl-source: trl-dev:
name: "Latest TRL + HF ecosystem from source" name: "Build and push TRL Dev Docker image"
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
- name: Cleanup disk - name: Checkout code
run: |
sudo ls -l /usr/local/lib/
sudo ls -l /usr/share/
sudo du -sh /usr/local/lib/
sudo du -sh /usr/share/
sudo rm -rf /usr/local/lib/android
sudo rm -rf /usr/share/dotnet
sudo du -sh /usr/local/lib/
sudo du -sh /usr/share/
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
- name: Check out code
uses: actions/checkout@v4 uses: actions/checkout@v4
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Login to DockerHub - name: Login to DockerHub
uses: docker/login-action@v1 uses: docker/login-action@v3
with: with:
username: ${{ secrets.DOCKERHUB_USERNAME }} username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_PASSWORD }} password: ${{ secrets.DOCKERHUB_PASSWORD }}
- name: Build and Push GPU - name: Build and Push
uses: docker/build-push-action@v4 uses: docker/build-push-action@v4
with: with:
context: ./docker/trl-source-gpu context: docker/trl-dev
push: true push: true
tags: huggingface/trl-source-gpu tags: |
huggingface/trl:dev
- name: Post to Slack - name: Post to Slack
if: always() if: always()
uses: huggingface/hf-workflows/.github/actions/post-slack@main uses: huggingface/hf-workflows/.github/actions/post-slack@main
with: with:
slack_channel: ${{ env.CI_SLACK_CHANNEL }} slack_channel: ${{ secrets.CI_DOCKER_CHANNEL }}
title: 🤗 Results of the trl-source-gpu Docker Image build title: 🤗 Results of the TRL Dev Docker Image build
status: ${{ job.status }} status: ${{ job.status }}
slack_token: ${{ secrets.SLACK_CIFEEDBACK_BOT_TOKEN }} slack_token: ${{ secrets.SLACK_CIFEEDBACK_BOT_TOKEN }}

8
.github/workflows/slow-tests.yml vendored
View File

@ -19,8 +19,8 @@ jobs:
matrix: matrix:
docker-image-name: docker-image-name:
[ [
"huggingface/trl-latest-gpu:latest", "huggingface/trl",
"huggingface/trl-source-gpu:latest", "huggingface/trl:dev",
] ]
runs-on: runs-on:
group: aws-g4dn-2xlarge group: aws-g4dn-2xlarge
@ -59,8 +59,8 @@ jobs:
matrix: matrix:
docker-image-name: docker-image-name:
[ [
"huggingface/trl-latest-gpu:latest", "huggingface/trl",
"huggingface/trl-source-gpu:latest", "huggingface/trl:dev",
] ]
runs-on: runs-on:
group: aws-g4dn-2xlarge group: aws-g4dn-2xlarge

6
docker/trl-dev/Dockerfile Normal file
View File

@ -0,0 +1,6 @@
FROM pytorch/pytorch:2.8.0-cuda12.8-cudnn9-runtime
RUN apt-get update && apt-get install -y git && rm -rf /var/lib/apt/lists/*
RUN pip install --upgrade pip uv
RUN uv pip install --system --no-cache "git+https://github.com/huggingface/trl.git#egg=trl[liger,peft,vlm]"
RUN uv pip install --system hf_transfer liger_kernel trackio peft
RUN uv pip install --system https://github.com/Dao-AILab/flash-attention/releases/download/v2.8.3/flash_attn-2.8.3+cu12torch2.8cxx11abiFALSE-cp311-cp311-linux_x86_64.whl

66
docker/trl-latest-gpu/Dockerfile
View File

@ -1,66 +0,0 @@
# Builds GPU docker image of PyTorch
# Uses multi-staged approach to reduce size
# Stage 1
# Use base conda image to reduce time
FROM continuumio/miniconda3:latest AS compile-image
# Specify py version
ENV PYTHON_VERSION=3.10
# Install apt libs - copied from https://github.com/huggingface/accelerate/blob/main/docker/accelerate-gpu/Dockerfile
RUN apt-get update && \
apt-get install -y curl git wget software-properties-common git-lfs && \
apt-get clean && \
rm -rf /var/lib/apt/lists*
# Install audio-related libraries
RUN apt-get update && \
apt install -y ffmpeg
RUN apt install -y libsndfile1-dev
RUN git lfs install
# Create our conda env - copied from https://github.com/huggingface/accelerate/blob/main/docker/accelerate-gpu/Dockerfile
RUN conda create --name trl python=${PYTHON_VERSION} ipython jupyter pip
RUN python3 -m pip install --no-cache-dir --upgrade pip
# Below is copied from https://github.com/huggingface/accelerate/blob/main/docker/accelerate-gpu/Dockerfile
# We don't install pytorch here yet since CUDA isn't available
# instead we use the direct torch wheel
ENV PATH /opt/conda/envs/trl/bin:$PATH
# Activate our bash shell
RUN chsh -s /bin/bash
SHELL ["/bin/bash", "-c"]
# Stage 2
FROM nvidia/cuda:12.2.2-devel-ubuntu22.04 AS build-image
COPY --from=compile-image /opt/conda /opt/conda
ENV PATH /opt/conda/bin:$PATH
RUN chsh -s /bin/bash
SHELL ["/bin/bash", "-c"]
RUN source activate trl && \
python3 -m pip install --no-cache-dir bitsandbytes optimum auto-gptq
# Install apt libs
RUN apt-get update && \
apt-get install -y curl git wget && \
apt-get clean && \
rm -rf /var/lib/apt/lists*
# Activate the conda env and install transformers + accelerate from source
RUN source activate trl && \
python3 -m pip install -U --no-cache-dir \
librosa \
"soundfile>=0.12.1" \
scipy \
transformers \
accelerate \
peft \
trl[test,vlm]@git+https://github.com/huggingface/trl
RUN source activate trl && \
pip freeze | grep trl
RUN echo "source activate trl" >> ~/.profile
# Activate the virtualenv
CMD ["/bin/bash"]

66
docker/trl-source-gpu/Dockerfile
View File

@ -1,66 +0,0 @@
# Builds GPU docker image of PyTorch
# Uses multi-staged approach to reduce size
# Stage 1
# Use base conda image to reduce time
FROM continuumio/miniconda3:latest AS compile-image
# Specify py version
ENV PYTHON_VERSION=3.10
# Install apt libs - copied from https://github.com/huggingface/accelerate/blob/main/docker/accelerate-gpu/Dockerfile
RUN apt-get update && \
apt-get install -y curl git wget software-properties-common git-lfs && \
apt-get clean && \
rm -rf /var/lib/apt/lists*
# Install audio-related libraries
RUN apt-get update && \
apt install -y ffmpeg
RUN apt install -y libsndfile1-dev
RUN git lfs install
# Create our conda env - copied from https://github.com/huggingface/accelerate/blob/main/docker/accelerate-gpu/Dockerfile
RUN conda create --name trl python=${PYTHON_VERSION} ipython jupyter pip
RUN python3 -m pip install --no-cache-dir --upgrade pip
# Below is copied from https://github.com/huggingface/accelerate/blob/main/docker/accelerate-gpu/Dockerfile
# We don't install pytorch here yet since CUDA isn't available
# instead we use the direct torch wheel
ENV PATH /opt/conda/envs/trl/bin:$PATH
# Activate our bash shell
RUN chsh -s /bin/bash
SHELL ["/bin/bash", "-c"]
# Stage 2
FROM nvidia/cuda:12.2.2-devel-ubuntu22.04 AS build-image
COPY --from=compile-image /opt/conda /opt/conda
ENV PATH /opt/conda/bin:$PATH
RUN chsh -s /bin/bash
SHELL ["/bin/bash", "-c"]
RUN source activate trl && \
python3 -m pip install --no-cache-dir bitsandbytes optimum auto-gptq
# Install apt libs
RUN apt-get update && \
apt-get install -y curl git wget && \
apt-get clean && \
rm -rf /var/lib/apt/lists*
# Activate the conda env and install transformers + accelerate from source
RUN source activate trl && \
python3 -m pip install -U --no-cache-dir \
librosa \
"soundfile>=0.12.1" \
scipy \
git+https://github.com/huggingface/transformers \
git+https://github.com/huggingface/accelerate \
git+https://github.com/huggingface/peft \
trl[test,vlm]@git+https://github.com/huggingface/trl
RUN source activate trl && \
pip freeze | grep transformers
RUN echo "source activate trl" >> ~/.profile
# Activate the virtualenv
CMD ["/bin/bash"]

4
docker/trl/Dockerfile Normal file
View File

@ -0,0 +1,4 @@
FROM pytorch/pytorch:2.8.0-cuda12.8-cudnn9-runtime
RUN pip install --upgrade pip uv
RUN uv pip install --system trl[liger,peft,vlm] hf_transfer trackio
RUN uv pip install --system https://github.com/Dao-AILab/flash-attention/releases/download/v2.8.3/flash_attn-2.8.3+cu12torch2.8cxx11abiFALSE-cp311-cp311-linux_x86_64.whl

419
docs/source/jobs_training.md
View File

@ -1,46 +1,64 @@
# Training using Jobs # Training with Jobs
[Jobs](https://huggingface.co/docs/huggingface_hub/guides/jobs) lets you run training scripts on fully managed infrastructure (no need to handle GPUs, dependencies, or environment setup locally). This makes it easy to scale and monitor your experiments directly from the Hub. [![](https://img.shields.io/badge/All_models-HF_Jobs-blue)](https://huggingface.co/models?other=hf_jobs,trl)
In this guide, you’ll learn how to: [Hugging Face Jobs](https://huggingface.co/docs/huggingface_hub/guides/jobs) lets you run training scripts on fully managed infrastructure—no need to manage GPUs or local environment setup.
- Run TRL training scripts using Jobs. In this guide, you'll learn how to:
- Configure hardware, timeouts, environment variables, and secrets.
- Monitor and manage jobs from the CLI or Python.
<Tip> * Use [TRL Jobs](https://github.com/huggingface/trl-jobs) to easily run pre-optimized TRL training
* Run any TRL training script with uv scripts
When a model is trained using **TRL + Jobs**, a tag is automatically added to the model card. For general details about Hugging Face Jobs (hardware selection, job monitoring, etc.), see the [Jobs documentation](https://huggingface.co/docs/huggingface_hub/guides/jobs).
You can explore models trained with this method [Hugging Face model hub](https://huggingface.co/models?other=hf_jobs).
</Tip>
## Requirements ## Requirements
- [Pro](https://hf.co/pro), [Team](https://hf.co/enterprise), or [Enterprise](https://hf.co/enterprise) plan. * A [Pro](https://hf.co/pro), [Team](https://hf.co/enterprise), or [Enterprise](https://hf.co/enterprise) plan
- Logged into the Hugging Face Hub (`hf auth login`). * Logged in to the Hugging Face Hub (`huggingface-cli login`)
## Preparing your Script ## Using TRL Jobs
You can launch Jobs using either the [`hf jobs` CLI](https://huggingface.co/docs/huggingface_hub/guides/cli#hf-jobs) or the Python API. A convenient option is to use [UV scripts](https://docs.astral.sh/uv/guides/scripts/), which packages all dependencies directly into a single Python file. You can run them like this: [TRL Jobs](https://github.com/huggingface/trl-jobs) is a high-level wrapper around Hugging Face Jobs and TRL that streamlines training. It provides optimized default configurations so you can start quickly without manually tuning parameters.
Example:
```bash
pip install trl-jobs
trl-jobs sft --model_name Qwen/Qwen3-0.6B --dataset_name trl-lib/Capybara
```
TRL Jobs supports everything covered in this guide, with additional optimizations to simplify workflows.
## Using uv Scripts
For more control, you can run Hugging Face Jobs directly with your own scripts, using [uv scripts](https://docs.astral.sh/uv/guides/scripts/).
Create a Python script (e.g., `train.py`) containing your training code:
```python
from datasets import load_dataset
from trl import SFTTrainer
dataset = load_dataset("trl-lib/Capybara", split="train")
trainer = SFTTrainer(
model="Qwen/Qwen2.5-0.5B",
train_dataset=dataset,
)
trainer.train()
trainer.push_to_hub("Qwen2.5-0.5B-SFT")
```
Launch the job using either the [`hf jobs` CLI](https://huggingface.co/docs/huggingface_hub/guides/cli#hf-jobs) or the Python API:
<hfoptions id="script_type"> <hfoptions id="script_type">
<hfoption id="bash"> <hfoption id="bash">
```bash ```bash
hf jobs uv run --flavor a100-large --secrets HF_TOKEN "https://raw.githubusercontent.com/huggingface/trl/main/trl/scripts/sft.py" --model_name_or_path Qwen/Qwen2-0.5B --dataset_name trl-lib/Capybara hf jobs uv run \
``` --flavor a100-large \
--with trl \
The script can also be a local file: --secrets HF_TOKEN \
train.py
```bash
hf jobs uv run --flavor a100-large --secrets HF_TOKEN trl/scripts/sft.py --model_name_or_path Qwen/Qwen2-0.5B --dataset_name trl-lib/Capybara
```
Since it runs using a Docker Image from Hugging Face Spaces or Docker Hub, you can also specify it:
```bash
hf jobs uv run --flavor a100-large --secrets HF_TOKEN --image <docker-image> trl/scripts/sft.py --model_name_or_path Qwen/Qwen2-0.5B --dataset_name trl-lib/Capybara
``` ```
</hfoption> </hfoption>
@ -48,236 +66,118 @@ hf jobs uv run --flavor a100-large --secrets HF_TOKEN --image <docker-image> trl
```python ```python
from huggingface_hub import run_uv_job from huggingface_hub import run_uv_job
run_uv_job( run_uv_job(
"https://raw.githubusercontent.com/huggingface/trl/main/trl/scripts/sft.py", "train.py",
token="hf...", dependencies=["trl"],
flavor="a100-large", flavor="a100-large",
script_args=[ secrets={"HF_TOKEN": "hf_..."},
"--model_name_or_path", "Qwen/Qwen2-0.5B",
"--dataset_name", "trl-lib/Capybara",
]
)
```
The script can also be a local file:
```python
from huggingface_hub import run_uv_job
run_uv_job(
"trl/scripts/sft.py",
token="hf...",
flavor="a100-large",
script_args=[
"--model_name_or_path", "Qwen/Qwen2-0.5B",
"--dataset_name", "trl-lib/Capybara",
]
)
```
Since it runs using a Docker Image from Hugging Face Spaces or Docker Hub, you can also specify it:
```python
from huggingface_hub import run_uv_job
run_uv_job(
"sft.py",
token="hf...",
flavor="a100-large",
image="<docker-image>",
script_args=[
"--model_name_or_path", "Qwen/Qwen2-0.5B",
"--dataset_name", "trl-lib/Capybara",
]
) )
``` ```
</hfoption> </hfoption>
</hfoptions> </hfoptions>
You can also run jobs without UV: To run successfully, the script needs:
* **TRL installed**: Use the `--with trl` flag or the `dependencies` argument. uv installs these dependencies automatically before running the script.
* **An authentication token**: Required to push the trained model (or perform other authenticated operations). Provide it with the `--secrets HF_TOKEN` flag or the `secrets` argument.
<Tip warning={true}>
When training with Jobs, be sure to:
* **Set a sufficient timeout**. Jobs time out after 30 minutes by default. If your job exceeds the without, it will fail and all progress will be lost. See [Setting a custom timeout](https://huggingface.co/docs/huggingface_hub/guides/jobs#setting-a-custom-timeout).
* **Push the model to the Hub**. The Jobs environment is ephemeral—files are deleted when the job ends. If you don’t push the model, it will be lost.
</Tip>
You can also run a script directly from a URL:
<hfoptions id="script_type"> <hfoptions id="script_type">
<hfoption id="bash"> <hfoption id="bash">
In this case, we give the cli the Docker image and run it as:
```bash ```bash
hf jobs run --flavor a100-large pytorch/pytorch:2.6.0-cuda12.4-cudnn9-devel python -c "import torch; print(torch.cuda.get_device_name())" hf jobs uv run \
--flavor a100-large \
--with trl \
--secrets HF_TOKEN \
"https://gist.githubusercontent.com/qgallouedec/eb6a7d20bd7d56f9c440c3c8c56d2307/raw/69fd78a179e19af115e4a54a1cdedd2a6c237f2f/train.py"
``` ```
</hfoption> </hfoption>
<hfoption id="python"> <hfoption id="python">
```python ```python
from huggingface_hub import run_job from huggingface_hub import run_uv_job
run_job(
image="pytorch/pytorch:2.6.0-cuda12.4-cudnn9-devel", run_uv_job(
command=["python", "-c", "import torch; print(torch.cuda.get_device_name())"], "https://gist.githubusercontent.com/qgallouedec/eb6a7d20bd7d56f9c440c3c8c56d2307/raw/69fd78a179e19af115e4a54a1cdedd2a6c237f2f/train.py",
flavor="a100-large", flavor="a100-large",
dependencies=["trl"],
secrets={"HF_TOKEN": "hf_..."},
) )
``` ```
</hfoption> </hfoption>
</hfoptions> </hfoptions>
### Adding Dependencies with UV To make a script self-contained, declare dependencies at the top:
All example scripts in TRL are compatible with `uv`, allowing seamless execution with Jobs. You can check the full list of examples in [Maintained examples](example_overview#maintained-examples).
Dependencies are specified at the top of the script using this structure:
```python ```python
# /// script # /// script
# dependencies = [ # dependencies = [
# "trl @ git+https://github.com/huggingface/trl.git", # "trl",
# "peft", # "peft",
# ] # ]
# /// # ///
from datasets import load_dataset
from peft import LoraConfig
from trl import SFTTrainer
dataset = load_dataset("trl-lib/Capybara", split="train")
trainer = SFTTrainer(
model="Qwen/Qwen2.5-0.5B",
train_dataset=dataset,
peft_config=LoraConfig(),
)
trainer.train()
trainer.push_to_hub("Qwen2.5-0.5B-SFT")
``` ```
When you run the UV script, these dependencies are automatically installed. In the example above, `trl` and `peft` would be installed before the script runs. You can then run the script without specifying dependencies:
You can also provide dependencies directly in the `uv run` command:
<hfoptions id="script_type"> <hfoptions id="script_type">
<hfoption id="bash"> <hfoption id="bash">
Using the `--with` flag.
```bash ```bash
hf jobs uv run \ hf jobs uv run \
--flavor a100-large \ --flavor a100-large \
--secrets HF_TOKEN \ --secrets HF_TOKEN \
--with transformers \ train.py
--with torch \
"https://raw.githubusercontent.com/huggingface/trl/main/trl/scripts/sft.py" \
--model_name_or_path Qwen/Qwen2-0.5B \
--dataset_name trl-lib/Capybara
``` ```
</hfoption> </hfoption>
<hfoption id="python"> <hfoption id="python">
Using the `dependencies` argument.
```python ```python
from huggingface_hub import run_uv_job from huggingface_hub import run_uv_job
run_uv_job( run_uv_job(
"https://raw.githubusercontent.com/huggingface/trl/main/trl/scripts/sft.py", "train.py",
dependencies=["transformers", "torch"]
token="hf...",
flavor="a100-large", flavor="a100-large",
script_args=[ secrets={"HF_TOKEN": "hf_..."},
"--model_name_or_path", "Qwen/Qwen2-0.5B",
"--dataset_name", "trl-lib/Capybara",
]
) )
``` ```
</hfoption> </hfoption>
</hfoptions> </hfoptions>
### Hardware and Timeout Settings <Tip>
Jobs allow you to select a specific hardware configuration using the `--flavor` flag. As of 08/25, the available options are: TRL example scripts are fully uv-compatible, so you can run a complete training workflow directly on Jobs. You can customize training with standard script arguments plus hardware and secrets:
**CPU:** `cpu-basic`, `cpu-upgrade`
**GPU:** `t4-small`, `t4-medium`, `l4x1`, `l4x4`, `a10g-small`, `a10g-large`, `a10g-largex2`, `a10g-largex4`, `a100-large`
**TPU:** `v5e-1x1`, `v5e-2x2`, `v5e-2x4`
You can always check the latest list of supported hardware flavors in [Spaces config reference](https://huggingface.co/docs/hub/en/spaces-config-reference).
By default, jobs have a **30-minute timeout**, after which they will automatically stop. For long-running tasks like training, you can increase the timeout as needed. Supported time units are:
- `s`: seconds
- `m`: minutes
- `h`: hours
- `d`: days
Example with a 2-hour timeout:
<hfoptions id="script_type">
<hfoption id="bash">
Using the `--timeout` flag:
```bash
hf jobs uv run \
--timeout 2h \
--flavor a100-large \
--secrets HF_TOKEN \
--with transformers \
--with torch \
"https://raw.githubusercontent.com/huggingface/trl/main/trl/scripts/sft.py" \
--model_name_or_path Qwen/Qwen2-0.5B \
--dataset_name trl-lib/Capybara
```
</hfoption>
<hfoption id="python">
Using the `timeout` argument:
```python
from huggingface_hub import run_uv_job
run_uv_job(
"https://raw.githubusercontent.com/huggingface/trl/main/trl/scripts/sft.py",
timeout="2h",
token="hf...",
flavor="a100-large",
script_args=[
"--model_name_or_path", "Qwen/Qwen2-0.5B",
"--dataset_name", "trl-lib/Capybara",
]
)
```
</hfoption>
</hfoptions>
### Environment Variables, Secrets, and Token
You can pass environment variables, secrets, and your auth token to your jobs.
<hfoptions id="script_type">
<hfoption id="bash">
Using the `--env`, `--secrets`, and/or `--token` options.
```bash
hf jobs uv run \
trl/scripts/sft.py \
--flavor a100-large \
--env FOO=foo \
--env BAR=bar \
--secrets HF_TOKEN=HF_TOKEN \
--secrets MY_SECRET=password \
--token hf...
```
</hfoption>
<hfoption id="python">
Using the `env`, `secrets`, and/or `token` arguments.
```python
from huggingface_hub import run_uv_job
run_uv_job(
"trl/scripts/sft.py",
env={"FOO": "foo", "BAR": "bar"},
secrets={"MY_SECRET": "psswrd"},
token="hf..."
)
```
</hfoption>
</hfoptions>
## Training and Evaluating a Model with Jobs
TRL example scripts are fully UV-compatible, allowing you to run a complete training workflow directly on Jobs. You can customize the training by providing the usual script arguments, along with hardware specifications and secrets.
To evaluate your training runs, in addition to reviewing the job logs, you can use [**Trackio**](https://huggingface.co/blog/trackio), a lightweight experiment tracking library. Trackio enables end-to-end experiment management on the Hugging Face Hub. All TRL example scripts already support reporting to Trackio via the `report_to` argument. Using this feature saves your experiments in an interactive HF Space, making it easy to monitor metrics, compare runs, and track progress over time.
<hfoptions id="script_type"> <hfoptions id="script_type">
<hfoption id="bash"> <hfoption id="bash">
@ -286,19 +186,10 @@ To evaluate your training runs, in addition to reviewing the job logs, you can u
hf jobs uv run \ hf jobs uv run \
--flavor a100-large \ --flavor a100-large \
--secrets HF_TOKEN \ --secrets HF_TOKEN \
"trl/scripts/sft.py" \ https://raw.githubusercontent.com/huggingface/trl/refs/heads/main/examples/scripts/prm.py \
--model_name_or_path Qwen/Qwen2-0.5B \ --model_name_or_path Qwen/Qwen2-0.5B-Instruct \
--dataset_name trl-lib/Capybara \ --dataset_name trl-lib/prm800k \
--learning_rate 2.0e-5 \ --output_dir Qwen2-0.5B-Reward \
--num_train_epochs 1 \
--packing \
--per_device_train_batch_size 2 \
--gradient_accumulation_steps 8 \
--eos_token '<|im_end|>' \
--eval_strategy steps \
--eval_steps 100 \
--output_dir Qwen2-0.5B-SFT \
--report_to trackio \
--push_to_hub --push_to_hub
``` ```
@ -309,22 +200,13 @@ hf jobs uv run \
from huggingface_hub import run_uv_job from huggingface_hub import run_uv_job
run_uv_job( run_uv_job(
"trl/scripts/sft.py", "https://raw.githubusercontent.com/huggingface/trl/refs/heads/main/examples/scripts/prm.py",
flavor="a100-large", flavor="a100-large",
secrets={"HF_TOKEN": "your_hf_token"}, secrets={"HF_TOKEN": "hf_..."},
script_args=[ script_args=[
"--model_name_or_path", "Qwen/Qwen2-0.5B", "--model_name_or_path", "Qwen/Qwen2-0.5B-Instruct",
"--dataset_name", "trl-lib/Capybara", "--dataset_name", "trl-lib/prm800k",
"--learning_rate", "2.0e-5", "--output_dir", "Qwen2-0.5B-Reward",
"--num_train_epochs", "1",
"--packing",
"--per_device_train_batch_size", "2",
"--gradient_accumulation_steps", "8",
"--eos_token", "<|im_end|>",
"--eval_strategy", "steps",
"--eval_steps", "100",
"--output_dir", "Qwen2-0.5B-SFT",
"--report_to", "trackio",
"--push_to_hub" "--push_to_hub"
] ]
) )
@ -333,60 +215,69 @@ run_uv_job(
</hfoption> </hfoption>
</hfoptions> </hfoptions>
## Monitoring and Managing Jobs See the full list of examples in [Maintained examples](example_overview#maintained-examples).
After launching a job, you can track its progress on the [Jobs page](https://huggingface.co/settings/jobs). Additionally, Jobs provides CLI and Python commands to check status, view logs, or cancel a job. </Tip>
### Docker Images
An up-to-date Docker image with all TRL dependencies is available at [huggingface/trl](https://hub.docker.com/r/huggingface/trl) and can be used directly with Hugging Face Jobs:
<hfoptions id="script_type"> <hfoptions id="script_type">
<hfoption id="bash"> <hfoption id="bash">
```bash ```bash
# List your jobs hf jobs uv run \
hf jobs ps -a --flavor a100-large \
--secrets HF_TOKEN \
# List your running jobs --image huggingface/trl \
hf jobs ps train.py
# Inspect the status of a job
hf jobs inspect
# View logs from a job
hf jobs logs job_id
# Cancel a job
hf jobs cancel job_id
``` ```
</hfoption> </hfoption>
<hfoption id="python"> <hfoption id="python">
```python ```python
from huggingface_hub import list_jobs, inspect_job, fetch_job_logs, cancel_job from huggingface_hub import run_uv_job
# List your jobs run_uv_job(
jobs = list_jobs() "train.py",
jobs[0] flavor="a100-large",
secrets={"HF_TOKEN": "hf_..."},
# List your running jobs image="huggingface/trl",
running_jobs = [job for job in list_jobs() if job.status.stage == "RUNNING"] )
# Inspect the status of a job
inspect_job(job_id=job_id)
# View logs from a job
for log in fetch_job_logs(job_id=job_id):
print(log)
# Cancel a job
cancel_job(job_id=job_id)
``` ```
</hfoption> </hfoption>
</hfoptions> </hfoptions>
## Best Practices and Tips Jobs runs on a Docker image from Hugging Face Spaces or Docker Hub, so you can also specify any custom image:
- Choose hardware that fits the size of your model and dataset for optimal performance. <hfoptions id="script_type">
- Training jobs can be long-running. Consider increasing the default timeout. <hfoption id="bash">
- Reuse training and evaluation scripts whenever possible to streamline workflows.
```bash
hf jobs uv run \
--flavor a100-large \
--secrets HF_TOKEN \
--image <docker-image> \
--secrets HF_TOKEN \
train.py
```
</hfoption>
<hfoption id="python">
```python
from huggingface_hub import run_uv_job
run_uv_job(
"train.py",
flavor="a100-large",
secrets={"HF_TOKEN": "hf_..."},
image="<docker-image>",
)
```
</hfoption>
</hfoptions>

2
examples/scripts/alignprop.py
View File

@ -14,7 +14,7 @@
# /// script # /// script
# dependencies = [ # dependencies = [
# "trl @ git+https://github.com/huggingface/trl.git", # "trl",
# "diffusers", # "diffusers",
# "torchvision", # "torchvision",
# "peft", # "peft",

2
examples/scripts/bco.py
View File

@ -14,7 +14,7 @@
# /// script # /// script
# dependencies = [ # dependencies = [
# "trl @ git+https://github.com/huggingface/trl.git", # "trl",
# "peft", # "peft",
# "einops", # "einops",
# "scikit-learn", # "scikit-learn",

2
examples/scripts/cpo.py
View File

@ -14,7 +14,7 @@
# /// script # /// script
# dependencies = [ # dependencies = [
# "trl @ git+https://github.com/huggingface/trl.git", # "trl",
# "peft", # "peft",
# "trackio", # "trackio",
# "kernels", # "kernels",

2
examples/scripts/ddpo.py
View File

@ -14,7 +14,7 @@
# /// script # /// script
# dependencies = [ # dependencies = [
# "trl @ git+https://github.com/huggingface/trl.git", # "trl",
# "diffusers", # "diffusers",
# "peft", # "peft",
# "trackio", # "trackio",

2
examples/scripts/dpo_vlm.py
View File

@ -14,7 +14,7 @@
# /// script # /// script
# dependencies = [ # dependencies = [
# "trl @ git+https://github.com/huggingface/trl.git", # "trl",
# "peft", # "peft",
# "Pillow>=9.4.0", # "Pillow>=9.4.0",
# "torchvision", # "torchvision",

2
examples/scripts/evals/judge_tldr.py
View File

@ -14,7 +14,7 @@
# /// script # /// script
# dependencies = [ # dependencies = [
# "trl @ git+https://github.com/huggingface/trl.git", # "trl",
# "vllm", # "vllm",
# ] # ]
# /// # ///

2
examples/scripts/gkd.py
View File

@ -14,7 +14,7 @@
# /// script # /// script
# dependencies = [ # dependencies = [
# "trl @ git+https://github.com/huggingface/trl.git", # "trl",
# "peft", # "peft",
# "trackio", # "trackio",
# "kernels", # "kernels",

2
examples/scripts/grpo_vlm.py
View File

@ -14,7 +14,7 @@
# /// script # /// script
# dependencies = [ # dependencies = [
# "trl @ git+https://github.com/huggingface/trl.git", # "trl",
# "Pillow", # "Pillow",
# "peft", # "peft",
# "math-verify", # "math-verify",

2
examples/scripts/gspo.py
View File

@ -14,7 +14,7 @@
# /// script # /// script
# dependencies = [ # dependencies = [
# "trl @ git+https://github.com/huggingface/trl.git", # "trl",
# "peft", # "peft",
# "math-verify", # "math-verify",
# "latex2sympy2_extended", # "latex2sympy2_extended",

2
examples/scripts/gspo_vlm.py
View File

@ -14,7 +14,7 @@
# /// script # /// script
# dependencies = [ # dependencies = [
# "trl @ git+https://github.com/huggingface/trl.git", # "trl",
# "Pillow", # "Pillow",
# "peft", # "peft",
# "math-verify", # "math-verify",

2
examples/scripts/kto.py
View File

@ -14,7 +14,7 @@
# /// script # /// script
# dependencies = [ # dependencies = [
# "trl @ git+https://github.com/huggingface/trl.git", # "trl",
# "peft", # "peft",
# "trackio", # "trackio",
# "kernels", # "kernels",

2
examples/scripts/mpo_vlm.py
View File

@ -14,7 +14,7 @@
# /// script # /// script
# dependencies = [ # dependencies = [
# "trl @ git+https://github.com/huggingface/trl.git", # "trl",
# "Pillow", # "Pillow",
# "peft", # "peft",
# "torchvision", # "torchvision",

2
examples/scripts/nash_md.py
View File

@ -14,7 +14,7 @@
# /// script # /// script
# dependencies = [ # dependencies = [
# "trl @ git+https://github.com/huggingface/trl.git", # "trl",
# "trackio", # "trackio",
# "kernels", # "kernels",
# ] # ]

2
examples/scripts/online_dpo.py
View File

@ -14,7 +14,7 @@
# /// script # /// script
# dependencies = [ # dependencies = [
# "trl @ git+https://github.com/huggingface/trl.git", # "trl",
# "peft", # "peft",
# "trackio", # "trackio",
# "kernels", # "kernels",

2
examples/scripts/online_dpo_vlm.py
View File

@ -14,7 +14,7 @@
# /// script # /// script
# dependencies = [ # dependencies = [
# "trl @ git+https://github.com/huggingface/trl.git", # "trl",
# "peft", # "peft",
# "math-verify", # "math-verify",
# "latex2sympy2_extended", # "latex2sympy2_extended",

2
examples/scripts/orpo.py
View File

@ -14,7 +14,7 @@
# /// script # /// script
# dependencies = [ # dependencies = [
# "trl @ git+https://github.com/huggingface/trl.git", # "trl",
# "peft", # "peft",
# "trackio", # "trackio",
# "kernels", # "kernels",

2
examples/scripts/ppo/ppo.py
View File

@ -14,7 +14,7 @@
# /// script # /// script
# dependencies = [ # dependencies = [
# "trl @ git+https://github.com/huggingface/trl.git", # "trl",
# "peft", # "peft",
# "trackio", # "trackio",
# "kernels", # "kernels",

2
examples/scripts/ppo/ppo_tldr.py
View File

@ -14,7 +14,7 @@
# /// script # /// script
# dependencies = [ # dependencies = [
# "trl @ git+https://github.com/huggingface/trl.git", # "trl",
# "peft", # "peft",
# "trackio", # "trackio",
# "kernels", # "kernels",

2
examples/scripts/prm.py
View File

@ -14,7 +14,7 @@
# /// script # /// script
# dependencies = [ # dependencies = [
# "trl @ git+https://github.com/huggingface/trl.git", # "trl",
# "trackio", # "trackio",
# "kernels", # "kernels",
# ] # ]

2
examples/scripts/reward_modeling.py
View File

@ -14,7 +14,7 @@
# /// script # /// script
# dependencies = [ # dependencies = [
# "trl @ git+https://github.com/huggingface/trl.git", # "trl",
# "trackio", # "trackio",
# "kernels", # "kernels",
# ] # ]

2
examples/scripts/rloo.py
View File

@ -14,7 +14,7 @@
# /// script # /// script
# dependencies = [ # dependencies = [
# "trl @ git+https://github.com/huggingface/trl.git", # "trl",
# "peft", # "peft",
# "math-verify", # "math-verify",
# "latex2sympy2_extended", # "latex2sympy2_extended",

2
examples/scripts/sft_gemma3.py
View File

@ -14,7 +14,7 @@
# /// script # /// script
# dependencies = [ # dependencies = [
# "trl @ git+https://github.com/huggingface/trl.git", # "trl",
# "Pillow", # "Pillow",
# "trackio", # "trackio",
# "kernels", # "kernels",

2
examples/scripts/sft_gpt_oss.py
View File

@ -14,7 +14,7 @@
# /// script # /// script
# dependencies = [ # dependencies = [
# "trl @ git+https://github.com/huggingface/trl.git", # "trl",
# "kernels", # "kernels",
# "trackio", # "trackio",
# "kernels", # "kernels",

2
examples/scripts/sft_video_llm.py
View File

@ -14,7 +14,7 @@
# /// script # /// script
# dependencies = [ # dependencies = [
# "trl @ git+https://github.com/huggingface/trl.git", # "trl",
# "peft", # "peft",
# "qwen-vl-utils", # "qwen-vl-utils",
# "torchvision", # "torchvision",

2
examples/scripts/sft_vlm.py
View File

@ -14,7 +14,7 @@
# /// script # /// script
# dependencies = [ # dependencies = [
# "trl @ git+https://github.com/huggingface/trl.git", # "trl",
# "Pillow>=9.4.0", # "Pillow>=9.4.0",
# "peft", # "peft",
# "trackio", # "trackio",

2
examples/scripts/sft_vlm_gemma3.py
View File

@ -14,7 +14,7 @@
# /// script # /// script
# dependencies = [ # dependencies = [
# "trl @ git+https://github.com/huggingface/trl.git", # "trl",
# "Pillow>=9.4.0", # "Pillow>=9.4.0",
# "peft", # "peft",
# "trackio", # "trackio",

2
examples/scripts/xpo.py
View File

@ -14,7 +14,7 @@
# /// script # /// script
# dependencies = [ # dependencies = [
# "trl @ git+https://github.com/huggingface/trl.git", # "trl",
# "trackio", # "trackio",
# "kernels", # "kernels",
# ] # ]

2
trl/scripts/dpo.py
View File

@ -14,7 +14,7 @@
# /// script # /// script
# dependencies = [ # dependencies = [
# "trl @ git+https://github.com/huggingface/trl.git", # "trl",
# "peft", # "peft",
# "trackio", # "trackio",
# "kernels", # "kernels",

8
trl/scripts/env.py
View File

@ -14,7 +14,7 @@
# /// script # /// script
# dependencies = [ # dependencies = [
# "trl @ git+https://github.com/huggingface/trl.git", # "trl",
# ] # ]
# /// # ///
@ -27,15 +27,15 @@ from accelerate.commands.config import default_config_file, load_config_from_fil
from transformers import is_bitsandbytes_available from transformers import is_bitsandbytes_available
from transformers.utils import is_openai_available, is_peft_available from transformers.utils import is_openai_available, is_peft_available
from .. import __version__ from trl import __version__
from ..import_utils import ( from trl.import_utils import (
is_deepspeed_available, is_deepspeed_available,
is_diffusers_available, is_diffusers_available,
is_liger_kernel_available, is_liger_kernel_available,
is_llm_blender_available, is_llm_blender_available,
is_vllm_available, is_vllm_available,
) )
from .utils import get_git_commit_hash from trl.scripts.utils import get_git_commit_hash
def print_env(): def print_env():

2
trl/scripts/grpo.py
View File

@ -14,7 +14,7 @@
# /// script # /// script
# dependencies = [ # dependencies = [
# "trl @ git+https://github.com/huggingface/trl.git", # "trl",
# "peft", # "peft",
# "trackio", # "trackio",
# "kernels", # "kernels",

2
trl/scripts/kto.py
View File

@ -14,7 +14,7 @@
# /// script # /// script
# dependencies = [ # dependencies = [
# "trl @ git+https://github.com/huggingface/trl.git", # "trl",
# "peft", # "peft",
# "trackio", # "trackio",
# "kernels", # "kernels",

2
trl/scripts/rloo.py
View File

@ -14,7 +14,7 @@
# /// script # /// script
# dependencies = [ # dependencies = [
# "trl @ git+https://github.com/huggingface/trl.git", # "trl",
# "peft", # "peft",
# ] # ]
# /// # ///

2
trl/scripts/sft.py
View File

@ -14,7 +14,7 @@
# /// script # /// script
# dependencies = [ # dependencies = [
# "trl @ git+https://github.com/huggingface/trl.git", # "trl",
# "peft", # "peft",
# "trackio", # "trackio",
# "kernels", # "kernels",