mirror of
https://github.com/pytorch/pytorch.git
synced 2025-10-20 12:54:11 +08:00
[BE] Remove unused release scripts. Add clarifications for the branch cut process (#154649)
Scripts in ``scripts/release/promote/`` are not used for a while. We use the ones in test-infra [here](https://github.com/pytorch/test-infra/blob/main/release/) . Hence this small cleanup. Pull Request resolved: https://github.com/pytorch/pytorch/pull/154649 Approved by: https://github.com/Skylion007, https://github.com/huydhn
This commit is contained in:
atalman
committed by
PyTorch MergeBot
PyTorch MergeBot
parent
e8f5c24d17
commit
3afbab66f7
@ -1,4 +1,4 @@
|
||||
# PyTorch Release Scripts
|
||||
# PyTorch release scripts performing branch cut and applying release only changes
|
||||
|
||||
These are a collection of scripts that are to be used for release activities.
|
||||
|
||||
@ -7,54 +7,12 @@ These are a collection of scripts that are to be used for release activities.
|
||||
> The basic idea being that there should be no potential to do anything dangerous unless
|
||||
> `DRY_RUN` is explicitly set to `disabled`.
|
||||
|
||||
## Requirements to actually run these scripts
|
||||
* AWS access to pytorch account
|
||||
* Access to upload conda packages to the `pytorch` conda channel
|
||||
* Access to the PyPI repositories
|
||||
### Order of Execution
|
||||
|
||||
1. Run cut-release-branch.sh to cut the release branch
|
||||
2. Run tag-docker-images.sh to tag current docker images with release tag and push them to docker.io. These images will be used to build the release.
|
||||
3. Run apply-release-changes.sh to apply release only changes to create a PR with release only changes similar to this [PR](https://github.com/pytorch/pytorch/pull/149056)
|
||||
|
||||
## Promote
|
||||
#### Promoting packages
|
||||
|
||||
These are scripts related to promotion of release candidates to GA channels, these
|
||||
can actually be used to promote pytorch, libtorch, and related domain libraries.
|
||||
|
||||
### Usage
|
||||
|
||||
Usage should be fairly straightforward and should actually require no extra variables
|
||||
if you are running from the correct git tags. (i.e. the GA tag to promote is currently
|
||||
checked out)
|
||||
|
||||
`PACKAGE_TYPE` and `PACKAGE_NAME` can be swapped out to promote other packages.
|
||||
|
||||
#### Promoting pytorch wheels
|
||||
```bash
|
||||
promote/s3_to_s3.sh
|
||||
```
|
||||
|
||||
#### Promoting libtorch archives
|
||||
```bash
|
||||
PACKAGE_TYPE=libtorch PACKAGE_NAME=libtorch promote/s3_to_s3.sh
|
||||
```
|
||||
|
||||
#### Promoting conda packages
|
||||
```bash
|
||||
promote/conda_to_conda.sh
|
||||
```
|
||||
|
||||
#### Promoting wheels to PyPI
|
||||
**WARNING**: These can only be run once and cannot be undone, run with caution
|
||||
```
|
||||
promote/wheel_to_pypi.sh
|
||||
```
|
||||
|
||||
## Restoring backups
|
||||
|
||||
All release candidates are currently backed up to `s3://pytorch-backup/${TAG_NAME}` and
|
||||
can be restored to the test channels with the `restore-backup.sh` script.
|
||||
|
||||
Which backup to restore from is dictated by the `RESTORE_FROM` environment variable.
|
||||
|
||||
### Usage
|
||||
```bash
|
||||
RESTORE_FROM=v1.5.0-rc5 ./restore-backup.sh
|
||||
```
|
||||
Scripts for Promotion of PyTorch packages are under test-infra repository. Please follow [README.md](https://github.com/pytorch/test-infra/blob/main/release/README.md)
|
||||
|
||||
@ -1,61 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
exit_if_not_on_git_tag() {
|
||||
# Have an override for debugging purposes
|
||||
if [[ -n "${TEST_WITHOUT_GIT_TAG-}" ]] ;then
|
||||
>&2 echo "+ WARN: Continuing without being on a git tag"
|
||||
exit 0
|
||||
fi
|
||||
# Exit if we're not currently on a git tag
|
||||
if ! git describe --tags --exact >/dev/null 2>/dev/null; then
|
||||
>&2 echo "- ERROR: Attempting to promote on a non-git tag, must have tagged current commit locally first"
|
||||
exit 1
|
||||
fi
|
||||
# Exit if we're currently on an RC
|
||||
if git describe --tags | grep "-rc" >/dev/null 2>/dev/null; then
|
||||
>&2 echo "- ERROR: Attempting to promote on a non GA git tag, current tag must be a GA tag"
|
||||
>&2 echo " Example: v1.5.0"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
get_pytorch_version() {
|
||||
if [[ -n "${TEST_WITHOUT_GIT_TAG-}" ]];then
|
||||
if [[ -z "${TEST_PYTORCH_PROMOTE_VERSION-}" ]]; then
|
||||
>&2 echo "- ERROR: Specified TEST_WITHOUT_GIT_TAG without specifying TEST_PYTORCH_PROMOTE_VERSION"
|
||||
>&2 echo "- TEST_PYTORCH_PROMOTE_VERSION must be specified"
|
||||
exit 1
|
||||
else
|
||||
echo "${TEST_PYTORCH_PROMOTE_VERSION}"
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
exit_if_not_on_git_tag
|
||||
# Echo git tag, strip leading v
|
||||
git describe --tags | sed -e 's/^v//'
|
||||
}
|
||||
|
||||
aws_promote() {
|
||||
package_name=$1
|
||||
pytorch_version=$(get_pytorch_version)
|
||||
# Dry run by default
|
||||
DRY_RUN=${DRY_RUN:-enabled}
|
||||
DRY_RUN_FLAG="--dryrun"
|
||||
if [[ $DRY_RUN = "disabled" ]]; then
|
||||
DRY_RUN_FLAG=""
|
||||
fi
|
||||
AWS=${AWS:-aws}
|
||||
(
|
||||
set -x
|
||||
${AWS} s3 cp ${DRY_RUN_FLAG} \
|
||||
--only-show-errors \
|
||||
--acl public-read \
|
||||
--recursive \
|
||||
--exclude '*' \
|
||||
--include "*${package_name}-${pytorch_version}*" \
|
||||
"${PYTORCH_S3_FROM/\/$//}" \
|
||||
"${PYTORCH_S3_TO/\/$//}"
|
||||
)
|
||||
# ^ We grep for package_name-.*pytorch_version to avoid any situations where domain libraries have
|
||||
# the same version on our S3 buckets
|
||||
}
|
||||
@ -1,45 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# Preps binaries for publishing to pypi by removing the
|
||||
# version suffix we normally add for all binaries
|
||||
# (outside of default ones, CUDA 10.2 currently)
|
||||
|
||||
# Usage is:
|
||||
# $ prep_binary_for_pypy.sh <path_to_whl_file> <path_to_multiple_whl_files>
|
||||
|
||||
# Will output a whl in your current directory
|
||||
|
||||
set -eou pipefail
|
||||
shopt -s globstar
|
||||
|
||||
OUTPUT_DIR=${OUTPUT_DIR:-$(pwd)}
|
||||
|
||||
tmp_dir="$(mktemp -d)"
|
||||
trap 'rm -rf ${tmp_dir}' EXIT
|
||||
|
||||
for whl_file in "$@"; do
|
||||
whl_file=$(realpath "${whl_file}")
|
||||
whl_dir="${tmp_dir}/$(basename "${whl_file}")_unzipped"
|
||||
mkdir -pv "${whl_dir}"
|
||||
(
|
||||
set -x
|
||||
unzip -q "${whl_file}" -d "${whl_dir}"
|
||||
)
|
||||
version_with_suffix=$(grep '^Version:' "${whl_dir}"/*/METADATA | cut -d' ' -f2)
|
||||
version_with_suffix_escaped=${version_with_suffix/+/%2B}
|
||||
# Remove all suffixed +bleh versions
|
||||
version_no_suffix=${version_with_suffix/+*/}
|
||||
new_whl_file=${OUTPUT_DIR}/$(basename "${whl_file/${version_with_suffix_escaped}/${version_no_suffix}}")
|
||||
dist_info_folder=$(find "${whl_dir}" -type d -name '*.dist-info' | head -1)
|
||||
basename_dist_info_folder=$(basename "${dist_info_folder}")
|
||||
dirname_dist_info_folder=$(dirname "${dist_info_folder}")
|
||||
(
|
||||
set -x
|
||||
find "${dist_info_folder}" -type f -exec sed -i "s!${version_with_suffix}!${version_no_suffix}!" {} \;
|
||||
# Moves distinfo from one with a version suffix to one without
|
||||
# Example: torch-1.8.0+cpu.dist-info => torch-1.8.0.dist-info
|
||||
mv "${dist_info_folder}" "${dirname_dist_info_folder}/${basename_dist_info_folder/${version_with_suffix}/${version_no_suffix}}"
|
||||
cd "${whl_dir}"
|
||||
zip -qr "${new_whl_file}" .
|
||||
)
|
||||
done
|
||||
@ -1,19 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -eou pipefail
|
||||
|
||||
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
|
||||
source "${DIR}/common_utils.sh"
|
||||
|
||||
# Allow for users to pass PACKAGE_NAME
|
||||
# For use with other packages, i.e. torchvision, etc.
|
||||
PACKAGE_NAME=${PACKAGE_NAME:-torch}
|
||||
PACKAGE_TYPE=${PACKAGE_TYPE:-whl}
|
||||
|
||||
PYTORCH_S3_BUCKET=${PYTORCH_S3_BUCKET:-s3://pytorch}
|
||||
FROM=${FROM:-test}
|
||||
PYTORCH_S3_FROM=${PYTORCH_S3_FROM:-${PYTORCH_S3_BUCKET}/${PACKAGE_TYPE}/${FROM}}
|
||||
TO=${TO:-}
|
||||
PYTORCH_S3_TO=${PYTORCH_S3_TO:-${PYTORCH_S3_BUCKET}/${PACKAGE_TYPE}/${TO}}
|
||||
|
||||
aws_promote "${PACKAGE_NAME}"
|
||||
@ -1,69 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -eou pipefail
|
||||
|
||||
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
|
||||
source "${DIR}/common_utils.sh"
|
||||
|
||||
# Allow for users to pass PACKAGE_NAME
|
||||
# For use with other packages, i.e. torchvision, etc.
|
||||
PACKAGE_NAME=${PACKAGE_NAME:-torch}
|
||||
|
||||
pytorch_version="$(get_pytorch_version)"
|
||||
# Refers to the specific package we'd like to promote
|
||||
# i.e. VERSION_SUFFIX='%2Bcu102'
|
||||
# torch-1.8.0+cu102 -> torch-1.8.0
|
||||
VERSION_SUFFIX=${VERSION_SUFFIX:-}
|
||||
# Refers to the specific platofmr we'd like to promote
|
||||
# i.e. PLATFORM=linux_x86_64
|
||||
# For domains like torchaudio / torchtext this is to be left blank
|
||||
PLATFORM=${PLATFORM:-}
|
||||
|
||||
pkgs_to_promote=$(\
|
||||
curl -fsSL https://download.pytorch.org/whl/torch_stable.html \
|
||||
| grep "${PACKAGE_NAME}-${pytorch_version}${VERSION_SUFFIX}-" \
|
||||
| grep "${PLATFORM}" \
|
||||
| cut -d '"' -f2
|
||||
)
|
||||
|
||||
tmp_dir="$(mktemp -d)"
|
||||
output_tmp_dir="$(mktemp -d)"
|
||||
trap 'rm -rf ${tmp_dir} ${output_tmp_dir}' EXIT
|
||||
pushd "${output_tmp_dir}"
|
||||
|
||||
# Dry run by default
|
||||
DRY_RUN=${DRY_RUN:-enabled}
|
||||
# On dry run just echo the commands that are meant to be run
|
||||
TWINE_UPLOAD="echo twine upload"
|
||||
if [[ $DRY_RUN = "disabled" ]]; then
|
||||
TWINE_UPLOAD="twine upload"
|
||||
fi
|
||||
|
||||
for pkg in ${pkgs_to_promote}; do
|
||||
pkg_basename="$(basename "${pkg}")"
|
||||
# Don't attempt to change if manylinux2014
|
||||
if [[ "${pkg}" != *manylinux2014* ]]; then
|
||||
pkg_basename="$(basename "${pkg//linux/manylinux1}")"
|
||||
fi
|
||||
orig_pkg="${tmp_dir}/${pkg_basename}"
|
||||
(
|
||||
set -x
|
||||
# Download package, sub out linux for manylinux1
|
||||
curl -fsSL -o "${orig_pkg}" "https://download.pytorch.org/whl/${pkg}"
|
||||
)
|
||||
|
||||
if [[ -n "${VERSION_SUFFIX}" ]]; then
|
||||
OUTPUT_DIR="${output_tmp_dir}" ${DIR}/prep_binary_for_pypi.sh "${orig_pkg}"
|
||||
else
|
||||
mv "${orig_pkg}" "${output_tmp_dir}/"
|
||||
fi
|
||||
|
||||
(
|
||||
set -x
|
||||
${TWINE_UPLOAD} \
|
||||
--disable-progress-bar \
|
||||
--non-interactive \
|
||||
./*.whl
|
||||
rm -rf ./*.whl
|
||||
)
|
||||
done
|
||||
@ -1,31 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -eou pipefail
|
||||
|
||||
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
|
||||
source "${DIR}/promote/common_utils.sh"
|
||||
|
||||
if [[ -z "${RESTORE_FROM:-}" ]]; then
|
||||
echo "ERROR: RESTORE_FROM environment variable must be specified"
|
||||
echo " example: RESTORE_FROM=v1.6.0-rc3 ${0}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
DRY_RUN=${DRY_RUN:-enabled}
|
||||
|
||||
PYTORCH_S3_BACKUP_BUCKET=${PYTORCH_S3_BACKUP_BUCKET:-s3://pytorch-backup/${RESTORE_FROM}}
|
||||
PYTORCH_S3_TEST_BUCKET=${PYTORCH_S3_TEST_BUCKET:-s3://pytorch/}
|
||||
PYTORCH_S3_FROM=${PYTORCH_S3_FROM:-${PYTORCH_S3_BACKUP_BUCKET}}
|
||||
PYTORCH_S3_TO=${PYTORCH_S3_TO:-s3://pytorch/}
|
||||
|
||||
restore_wheels() {
|
||||
aws_promote torch whl
|
||||
}
|
||||
|
||||
restore_libtorch() {
|
||||
aws_promote libtorch-* libtorch
|
||||
}
|
||||
|
||||
|
||||
restore_wheels
|
||||
restore_libtorch
|
||||
Reference in New Issue
Block a user