mirror of
https://github.com/pytorch/pytorch.git
synced 2025-10-21 05:34:18 +08:00
[BE] Isolate pre-push hook dependencies in dedicated virtual environment (#160048)
This adds two changes: - Isolates pre-push hook dependencies into an isolated venv, no longer affect your system environment - Lets you manually run the pre-push lintrunner (including with lintrunner -a) by invoking `python scripts/lintrunner.py [-a]` (it's ugly, but better than nothing...for now) This is a follow up to: - https://github.com/pytorch/pytorch/pull/158389 ## Problem The current pre-push hook setup installs lintrunner and related dependencies globally, which makes developers nervous about system pollution and can cause version conflicts with existing installations. Also, if the pre-push lintrunner found errors, you had to hope your normal lintrunner could fix them (which wasn't always the case, e.g. if those errors only manifested in certain python versions) ## Key Changes: - Isolated Environment: Creates .git/hooks/linter/.venv/ with Python 3.9 (the python used in CI) and an isolated lintrunner installation - User-Friendly CLI: New python scripts/lintrunner.py wrapper allows developers to run lintrunner (including -a auto-fix) from any environment - Simplified Architecture: Eliminates pre-commit dependency entirely - uses direct git hooks File Changes: - scripts/setup_hooks.py: Rewritten to create isolated uv-managed virtual environment - scripts/lintrunner.py: New wrapper script with shared hash management logic - scripts/run_lintrunner.py: Removed (functionality merged into lintrunner.py) - .pre-commit-config.yaml: Removed (no longer needed) ## Usage: ``` # Setup (run once) python scripts/setup_hooks.py # Manual linting (works from any environment) python scripts/lintrunner.py # Check mode python scripts/lintrunner.py -a # Auto-fix mode # Git hooks work automatically git push # Runs lintrunner in isolated environment # Need to skip the pre-push hook? git push --no-verify ``` ## Benefits: - ✅ Zero global dependency installation - ✅ Per-repository isolation prevents version conflicts - ✅ Full lintrunner functionality is now accessible ## Implementation Notes: - Virtual env is kept in a dedicated dir in .git, to keep per-repo mechanics - lintrunner.py does not need to be invoked from a specific venv. It'll invoke the right venv itself. A minor bug: It tends to garble the lintrunner output a bit, like the screenshot below shows, but I haven't found a workaround so far and it remains understandable to users: <img width="241" height="154" alt="image" src="https://github.com/user-attachments/assets/9496f925-8524-4434-8486-dc579442d688" /> ## What's next? Features that could be added: - Check for lintrunner updates, auto-update if needed - Depending on dev response, this could be enabled by default for all pytorch/pytorch environments Pull Request resolved: https://github.com/pytorch/pytorch/pull/160048 Approved by: https://github.com/seemethere
This commit is contained in:
Zain Rizvi
committed by
PyTorch MergeBot
PyTorch MergeBot
parent
7a974a88f2
commit
95210cc409
@ -1,31 +1,51 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Bootstrap Git pre‑push hook.
|
||||
Bootstrap Git pre‑push hook with isolated virtual environment.
|
||||
|
||||
✓ Requires uv to be installed (fails if not available)
|
||||
✓ Installs/updates pre‑commit with uv (global, venv‑proof)
|
||||
✓ Registers the repo's pre‑push hook and freezes hook versions
|
||||
✓ Creates isolated venv in .git/hooks/linter/.venv/ for hook dependencies
|
||||
✓ Installs lintrunner only in the isolated environment
|
||||
✓ Creates direct git hook that bypasses pre-commit
|
||||
|
||||
Run this from the repo root (inside or outside any project venv):
|
||||
|
||||
python scripts/setup_hooks.py
|
||||
|
||||
IMPORTANT: The generated git hook references scripts/lintrunner.py. If users checkout
|
||||
branches that don't have this file, git push will fail with "No such file or directory".
|
||||
Users would need to either:
|
||||
1. Re-run the old setup_hooks.py from that branch, or
|
||||
2. Manually delete .git/hooks/pre-push to disable hooks temporarily, or
|
||||
3. Switch back to a branch with the new scripts/lintrunner.py
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import shlex
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Tuple
|
||||
|
||||
|
||||
# Add scripts directory to Python path so we can import lintrunner module
|
||||
scripts_dir = Path(__file__).parent
|
||||
sys.path.insert(0, str(scripts_dir))
|
||||
|
||||
# Import shared functions from lintrunner module
|
||||
from lintrunner import find_repo_root, get_hook_venv_path
|
||||
|
||||
|
||||
# Restore sys.path to avoid affecting other imports
|
||||
sys.path.pop(0)
|
||||
|
||||
|
||||
# ───────────────────────────────────────────
|
||||
# Helper utilities
|
||||
# ───────────────────────────────────────────
|
||||
def run(cmd: list[str]) -> None:
|
||||
def run(cmd: list[str], cwd: Path = None) -> None:
|
||||
print(f"$ {' '.join(cmd)}")
|
||||
subprocess.check_call(cmd)
|
||||
subprocess.check_call(cmd, cwd=cwd)
|
||||
|
||||
|
||||
def which(cmd: str) -> bool:
|
||||
@ -34,28 +54,7 @@ def which(cmd: str) -> bool:
|
||||
|
||||
def ensure_uv() -> None:
|
||||
if which("uv"):
|
||||
# Ensure the path uv installs binaries to is part of the system path
|
||||
print("$ uv tool update-shell")
|
||||
result = subprocess.run(
|
||||
["uv", "tool", "update-shell"], capture_output=True, text=True
|
||||
)
|
||||
if result.returncode == 0:
|
||||
# Check if the output indicates changes were made
|
||||
if (
|
||||
"Updated" in result.stdout
|
||||
or "Added" in result.stdout
|
||||
or "Modified" in result.stdout
|
||||
):
|
||||
print(
|
||||
"⚠️ Shell configuration updated. You may need to restart your terminal for changes to take effect."
|
||||
)
|
||||
elif result.stdout.strip():
|
||||
print(result.stdout)
|
||||
return
|
||||
else:
|
||||
sys.exit(
|
||||
f"❌ Warning: uv tool update-shell failed: {result.stderr}. uv installed tools may not be available."
|
||||
)
|
||||
return
|
||||
|
||||
sys.exit(
|
||||
"\n❌ uv is required but was not found on your PATH.\n"
|
||||
@ -65,29 +64,6 @@ def ensure_uv() -> None:
|
||||
)
|
||||
|
||||
|
||||
def ensure_tool_installed(
|
||||
tool: str, force_update: bool = False, python_ver: Tuple[int, int] = None
|
||||
) -> None:
|
||||
"""
|
||||
Checks to see if the tool is available and if not (or if force update requested) then
|
||||
it reinstalls it.
|
||||
|
||||
Returns: Whether or not the tool is available on PATH. If it's not, a new terminal
|
||||
needs to be opened before git pushes work as expected.
|
||||
"""
|
||||
if force_update or not which(tool):
|
||||
print(f"Ensuring latest {tool} via uv …")
|
||||
command = ["uv", "tool", "install", "--force", tool]
|
||||
if python_ver:
|
||||
# Add the Python version to the command if specified
|
||||
command.extend(["--python", f"{python_ver[0]}.{python_ver[1]}"])
|
||||
run(command)
|
||||
if not which(tool):
|
||||
print(
|
||||
f"\n⚠️ {tool} installation succeed, but it's not on PATH. Launch a new terminal if your git pushes don't work.\n"
|
||||
)
|
||||
|
||||
|
||||
if sys.platform.startswith("win"):
|
||||
print(
|
||||
"\n⚠️ Lintrunner is not supported on Windows, so there are no pre-push hooks to add. Exiting setup.\n"
|
||||
@ -95,52 +71,61 @@ if sys.platform.startswith("win"):
|
||||
sys.exit(0)
|
||||
|
||||
# ───────────────────────────────────────────
|
||||
# 1. Install dependencies
|
||||
# 1. Setup isolated hook environment
|
||||
# ───────────────────────────────────────────
|
||||
|
||||
ensure_uv()
|
||||
|
||||
# Ensure pre-commit is installed globally via uv
|
||||
ensure_tool_installed("pre-commit", force_update=True, python_ver=(3, 9))
|
||||
# Find repo root and setup hook directory
|
||||
repo_root = find_repo_root()
|
||||
venv_dir = get_hook_venv_path()
|
||||
hooks_dir = venv_dir.parent.parent # Go from .git/hooks/linter/.venv to .git/hooks
|
||||
|
||||
# Don't force a lintrunner update because it might break folks
|
||||
# who already have it installed in a different way
|
||||
ensure_tool_installed("lintrunner")
|
||||
|
||||
# ───────────────────────────────────────────
|
||||
# 2. Activate (or refresh) the pre‑push hook
|
||||
# ───────────────────────────────────────────
|
||||
print(f"Setting up isolated hook environment in {venv_dir}")
|
||||
|
||||
# ── Activate (or refresh) the repo’s pre‑push hook ──────────────────────────
|
||||
# Creates/overwrites .git/hooks/pre‑push with a tiny shim that will call
|
||||
# `pre-commit run --hook-stage pre-push` on every `git push`.
|
||||
# This is why we need to install pre-commit globally.
|
||||
#
|
||||
# The --allow-missing-config flag lets pre-commit succeed if someone changes to
|
||||
# a branch that doesn't have pre-commit installed
|
||||
# Create isolated virtual environment for hooks
|
||||
if venv_dir.exists():
|
||||
print("Removing existing hook venv...")
|
||||
shutil.rmtree(venv_dir)
|
||||
|
||||
run(["uv", "venv", str(venv_dir), "--python", "3.9"])
|
||||
|
||||
# Install lintrunner in the isolated environment
|
||||
print("Installing lintrunner in isolated environment...")
|
||||
run(
|
||||
[
|
||||
"uv",
|
||||
"tool",
|
||||
"run",
|
||||
"pre-commit",
|
||||
"install",
|
||||
"--hook-type",
|
||||
"pre-push",
|
||||
"--allow-missing-config",
|
||||
]
|
||||
["uv", "pip", "install", "--python", str(venv_dir / "bin" / "python"), "lintrunner"]
|
||||
)
|
||||
|
||||
# ── Pin remote‑hook versions for reproducibility ────────────────────────────
|
||||
# (Note: we don't have remote hooks right now, but it future-proofs this script)
|
||||
# 1. `autoupdate` bumps every remote hook’s `rev:` in .pre-commit-config.yaml
|
||||
# to the latest commit on its default branch.
|
||||
# 2. `--freeze` immediately rewrites each `rev:` to the exact commit SHA,
|
||||
# ensuring all contributors and CI run identical hook code.
|
||||
run(["uv", "tool", "run", "pre-commit", "autoupdate", "--freeze"])
|
||||
# ───────────────────────────────────────────
|
||||
# 2. Create direct git pre-push hook
|
||||
# ───────────────────────────────────────────
|
||||
|
||||
pre_push_hook = hooks_dir / "pre-push"
|
||||
python_exe = venv_dir / "bin" / "python"
|
||||
lintrunner_script_path_quoted = shlex.quote(
|
||||
str(repo_root / "scripts" / "lintrunner.py")
|
||||
)
|
||||
|
||||
hook_script = f"""#!/bin/bash
|
||||
set -e
|
||||
|
||||
# Check if lintrunner script exists (user might be on older commit)
|
||||
if [ ! -f {lintrunner_script_path_quoted} ]; then
|
||||
echo "⚠️ {lintrunner_script_path_quoted} not found - skipping linting (likely on an older commit)"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Run lintrunner wrapper using the isolated venv's Python
|
||||
{shlex.quote(str(python_exe))} {lintrunner_script_path_quoted}
|
||||
"""
|
||||
|
||||
print(f"Creating git pre-push hook at {pre_push_hook}")
|
||||
pre_push_hook.write_text(hook_script)
|
||||
pre_push_hook.chmod(0o755) # Make executable
|
||||
|
||||
print(
|
||||
"\n✅ pre‑commit is installed globally via uv and the pre‑push hook is active.\n"
|
||||
"\n✅ Isolated hook environment created and pre‑push hook is active.\n"
|
||||
" Lintrunner will now run automatically on every `git push`.\n"
|
||||
f" Hook dependencies are isolated in {venv_dir}\n"
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user