diff --git a/scripts/release/README.md b/scripts/release/README.md index 3908142bb6b0..5f35d4e4d771 100644 --- a/scripts/release/README.md +++ b/scripts/release/README.md @@ -1,4 +1,4 @@ -# PyTorch Release Scripts +# PyTorch release scripts performing branch cut and applying release only changes These are a collection of scripts that are to be used for release activities. @@ -7,54 +7,12 @@ These are a collection of scripts that are to be used for release activities. > The basic idea being that there should be no potential to do anything dangerous unless > `DRY_RUN` is explicitly set to `disabled`. -## Requirements to actually run these scripts -* AWS access to pytorch account -* Access to upload conda packages to the `pytorch` conda channel -* Access to the PyPI repositories +### Order of Execution +1. Run cut-release-branch.sh to cut the release branch +2. Run tag-docker-images.sh to tag current docker images with release tag and push them to docker.io. These images will be used to build the release. +3. Run apply-release-changes.sh to apply release only changes to create a PR with release only changes similar to this [PR](https://github.com/pytorch/pytorch/pull/149056) -## Promote +#### Promoting packages -These are scripts related to promotion of release candidates to GA channels, these -can actually be used to promote pytorch, libtorch, and related domain libraries. - -### Usage - -Usage should be fairly straightforward and should actually require no extra variables -if you are running from the correct git tags. (i.e. the GA tag to promote is currently -checked out) - -`PACKAGE_TYPE` and `PACKAGE_NAME` can be swapped out to promote other packages. - -#### Promoting pytorch wheels -```bash -promote/s3_to_s3.sh -``` - -#### Promoting libtorch archives -```bash -PACKAGE_TYPE=libtorch PACKAGE_NAME=libtorch promote/s3_to_s3.sh -``` - -#### Promoting conda packages -```bash -promote/conda_to_conda.sh -``` - -#### Promoting wheels to PyPI -**WARNING**: These can only be run once and cannot be undone, run with caution -``` -promote/wheel_to_pypi.sh -``` - -## Restoring backups - -All release candidates are currently backed up to `s3://pytorch-backup/${TAG_NAME}` and -can be restored to the test channels with the `restore-backup.sh` script. - -Which backup to restore from is dictated by the `RESTORE_FROM` environment variable. - -### Usage -```bash -RESTORE_FROM=v1.5.0-rc5 ./restore-backup.sh -``` + Scripts for Promotion of PyTorch packages are under test-infra repository. Please follow [README.md](https://github.com/pytorch/test-infra/blob/main/release/README.md) diff --git a/scripts/release/promote/common_utils.sh b/scripts/release/promote/common_utils.sh deleted file mode 100644 index e50ccb110013..000000000000 --- a/scripts/release/promote/common_utils.sh +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env bash - -exit_if_not_on_git_tag() { - # Have an override for debugging purposes - if [[ -n "${TEST_WITHOUT_GIT_TAG-}" ]] ;then - >&2 echo "+ WARN: Continuing without being on a git tag" - exit 0 - fi - # Exit if we're not currently on a git tag - if ! git describe --tags --exact >/dev/null 2>/dev/null; then - >&2 echo "- ERROR: Attempting to promote on a non-git tag, must have tagged current commit locally first" - exit 1 - fi - # Exit if we're currently on an RC - if git describe --tags | grep "-rc" >/dev/null 2>/dev/null; then - >&2 echo "- ERROR: Attempting to promote on a non GA git tag, current tag must be a GA tag" - >&2 echo " Example: v1.5.0" - exit 1 - fi -} - -get_pytorch_version() { - if [[ -n "${TEST_WITHOUT_GIT_TAG-}" ]];then - if [[ -z "${TEST_PYTORCH_PROMOTE_VERSION-}" ]]; then - >&2 echo "- ERROR: Specified TEST_WITHOUT_GIT_TAG without specifying TEST_PYTORCH_PROMOTE_VERSION" - >&2 echo "- TEST_PYTORCH_PROMOTE_VERSION must be specified" - exit 1 - else - echo "${TEST_PYTORCH_PROMOTE_VERSION}" - exit 0 - fi - fi - exit_if_not_on_git_tag - # Echo git tag, strip leading v - git describe --tags | sed -e 's/^v//' -} - -aws_promote() { - package_name=$1 - pytorch_version=$(get_pytorch_version) - # Dry run by default - DRY_RUN=${DRY_RUN:-enabled} - DRY_RUN_FLAG="--dryrun" - if [[ $DRY_RUN = "disabled" ]]; then - DRY_RUN_FLAG="" - fi - AWS=${AWS:-aws} - ( - set -x - ${AWS} s3 cp ${DRY_RUN_FLAG} \ - --only-show-errors \ - --acl public-read \ - --recursive \ - --exclude '*' \ - --include "*${package_name}-${pytorch_version}*" \ - "${PYTORCH_S3_FROM/\/$//}" \ - "${PYTORCH_S3_TO/\/$//}" - ) - # ^ We grep for package_name-.*pytorch_version to avoid any situations where domain libraries have - # the same version on our S3 buckets -} diff --git a/scripts/release/promote/prep_binary_for_pypi.sh b/scripts/release/promote/prep_binary_for_pypi.sh deleted file mode 100755 index bb088f55468b..000000000000 --- a/scripts/release/promote/prep_binary_for_pypi.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env bash - -# Preps binaries for publishing to pypi by removing the -# version suffix we normally add for all binaries -# (outside of default ones, CUDA 10.2 currently) - -# Usage is: -# $ prep_binary_for_pypy.sh - -# Will output a whl in your current directory - -set -eou pipefail -shopt -s globstar - -OUTPUT_DIR=${OUTPUT_DIR:-$(pwd)} - -tmp_dir="$(mktemp -d)" -trap 'rm -rf ${tmp_dir}' EXIT - -for whl_file in "$@"; do - whl_file=$(realpath "${whl_file}") - whl_dir="${tmp_dir}/$(basename "${whl_file}")_unzipped" - mkdir -pv "${whl_dir}" - ( - set -x - unzip -q "${whl_file}" -d "${whl_dir}" - ) - version_with_suffix=$(grep '^Version:' "${whl_dir}"/*/METADATA | cut -d' ' -f2) - version_with_suffix_escaped=${version_with_suffix/+/%2B} - # Remove all suffixed +bleh versions - version_no_suffix=${version_with_suffix/+*/} - new_whl_file=${OUTPUT_DIR}/$(basename "${whl_file/${version_with_suffix_escaped}/${version_no_suffix}}") - dist_info_folder=$(find "${whl_dir}" -type d -name '*.dist-info' | head -1) - basename_dist_info_folder=$(basename "${dist_info_folder}") - dirname_dist_info_folder=$(dirname "${dist_info_folder}") - ( - set -x - find "${dist_info_folder}" -type f -exec sed -i "s!${version_with_suffix}!${version_no_suffix}!" {} \; - # Moves distinfo from one with a version suffix to one without - # Example: torch-1.8.0+cpu.dist-info => torch-1.8.0.dist-info - mv "${dist_info_folder}" "${dirname_dist_info_folder}/${basename_dist_info_folder/${version_with_suffix}/${version_no_suffix}}" - cd "${whl_dir}" - zip -qr "${new_whl_file}" . - ) -done diff --git a/scripts/release/promote/s3_to_s3.sh b/scripts/release/promote/s3_to_s3.sh deleted file mode 100755 index 21495ac3b325..000000000000 --- a/scripts/release/promote/s3_to_s3.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env bash - -set -eou pipefail - -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -source "${DIR}/common_utils.sh" - -# Allow for users to pass PACKAGE_NAME -# For use with other packages, i.e. torchvision, etc. -PACKAGE_NAME=${PACKAGE_NAME:-torch} -PACKAGE_TYPE=${PACKAGE_TYPE:-whl} - -PYTORCH_S3_BUCKET=${PYTORCH_S3_BUCKET:-s3://pytorch} -FROM=${FROM:-test} -PYTORCH_S3_FROM=${PYTORCH_S3_FROM:-${PYTORCH_S3_BUCKET}/${PACKAGE_TYPE}/${FROM}} -TO=${TO:-} -PYTORCH_S3_TO=${PYTORCH_S3_TO:-${PYTORCH_S3_BUCKET}/${PACKAGE_TYPE}/${TO}} - -aws_promote "${PACKAGE_NAME}" diff --git a/scripts/release/promote/wheel_to_pypi.sh b/scripts/release/promote/wheel_to_pypi.sh deleted file mode 100755 index 1e9b7411f2fd..000000000000 --- a/scripts/release/promote/wheel_to_pypi.sh +++ /dev/null @@ -1,69 +0,0 @@ -#!/usr/bin/env bash - -set -eou pipefail - -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -source "${DIR}/common_utils.sh" - -# Allow for users to pass PACKAGE_NAME -# For use with other packages, i.e. torchvision, etc. -PACKAGE_NAME=${PACKAGE_NAME:-torch} - -pytorch_version="$(get_pytorch_version)" -# Refers to the specific package we'd like to promote -# i.e. VERSION_SUFFIX='%2Bcu102' -# torch-1.8.0+cu102 -> torch-1.8.0 -VERSION_SUFFIX=${VERSION_SUFFIX:-} -# Refers to the specific platofmr we'd like to promote -# i.e. PLATFORM=linux_x86_64 -# For domains like torchaudio / torchtext this is to be left blank -PLATFORM=${PLATFORM:-} - -pkgs_to_promote=$(\ - curl -fsSL https://download.pytorch.org/whl/torch_stable.html \ - | grep "${PACKAGE_NAME}-${pytorch_version}${VERSION_SUFFIX}-" \ - | grep "${PLATFORM}" \ - | cut -d '"' -f2 -) - -tmp_dir="$(mktemp -d)" -output_tmp_dir="$(mktemp -d)" -trap 'rm -rf ${tmp_dir} ${output_tmp_dir}' EXIT -pushd "${output_tmp_dir}" - -# Dry run by default -DRY_RUN=${DRY_RUN:-enabled} -# On dry run just echo the commands that are meant to be run -TWINE_UPLOAD="echo twine upload" -if [[ $DRY_RUN = "disabled" ]]; then - TWINE_UPLOAD="twine upload" -fi - -for pkg in ${pkgs_to_promote}; do - pkg_basename="$(basename "${pkg}")" - # Don't attempt to change if manylinux2014 - if [[ "${pkg}" != *manylinux2014* ]]; then - pkg_basename="$(basename "${pkg//linux/manylinux1}")" - fi - orig_pkg="${tmp_dir}/${pkg_basename}" - ( - set -x - # Download package, sub out linux for manylinux1 - curl -fsSL -o "${orig_pkg}" "https://download.pytorch.org/whl/${pkg}" - ) - - if [[ -n "${VERSION_SUFFIX}" ]]; then - OUTPUT_DIR="${output_tmp_dir}" ${DIR}/prep_binary_for_pypi.sh "${orig_pkg}" - else - mv "${orig_pkg}" "${output_tmp_dir}/" - fi - - ( - set -x - ${TWINE_UPLOAD} \ - --disable-progress-bar \ - --non-interactive \ - ./*.whl - rm -rf ./*.whl - ) -done diff --git a/scripts/release/restore-backup.sh b/scripts/release/restore-backup.sh deleted file mode 100755 index c1e190f1ba05..000000000000 --- a/scripts/release/restore-backup.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env bash - -set -eou pipefail - -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -source "${DIR}/promote/common_utils.sh" - -if [[ -z "${RESTORE_FROM:-}" ]]; then - echo "ERROR: RESTORE_FROM environment variable must be specified" - echo " example: RESTORE_FROM=v1.6.0-rc3 ${0}" - exit 1 -fi - -DRY_RUN=${DRY_RUN:-enabled} - -PYTORCH_S3_BACKUP_BUCKET=${PYTORCH_S3_BACKUP_BUCKET:-s3://pytorch-backup/${RESTORE_FROM}} -PYTORCH_S3_TEST_BUCKET=${PYTORCH_S3_TEST_BUCKET:-s3://pytorch/} -PYTORCH_S3_FROM=${PYTORCH_S3_FROM:-${PYTORCH_S3_BACKUP_BUCKET}} -PYTORCH_S3_TO=${PYTORCH_S3_TO:-s3://pytorch/} - -restore_wheels() { - aws_promote torch whl -} - -restore_libtorch() { - aws_promote libtorch-* libtorch -} - - -restore_wheels -restore_libtorch