frozenleaves/pytorch
1
0
Fork 0
mirror of https://github.com/pytorch/pytorch.git synced 2025-10-20 21:14:14 +08:00
Code Issues Packages Projects Releases Wiki Activity

Update lint_urls.sh (#153246)

Browse Source 
Treat 403, 429 and 503 http errors as success.
Ignore non-verbal hostnames.
Kill child jobs immediately.
Pull Request resolved: https://github.com/pytorch/pytorch/pull/153246
Approved by: https://github.com/malfet
This commit is contained in:
Anthony Shoumikhin
2025-05-14 16:54:49 +00:00
committed by PyTorch MergeBot
parent b6b0080419
commit ba70876407
1 changed files with 13 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
scripts/lint_urls.sh
View File

@ -2,6 +2,8 @@
set -euo pipefail
trap 'kill 0' SIGINT
status=0
green='\e[1;32m'; red='\e[1;31m'; cyan='\e[1;36m'; yellow='\e[1;33m'; reset='\e[0m'
user_agent="Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36"
@ -16,9 +18,11 @@ while IFS=: read -r filepath url; do
(
code=$(curl -k -gsLm30 --retry 3 --retry-delay 3 --retry-connrefused -o /dev/null -w "%{http_code}" -I "$url") || code=000
if [ "$code" -lt 200 ] || [ "$code" -ge 400 ]; then
sleep 1
code=$(curl -k -gsLm30 --retry 3 --retry-delay 3 --retry-connrefused -o /dev/null -w "%{http_code}" -r 0-0 -A "$user_agent" "$url") || code=000
fi
if [ "$code" -lt 200 ] || [ "$code" -ge 400 ]; then
sleep 1
request_id=$(curl -sS -G -H 'Accept: application/json' \
--data-urlencode "host=$url" \
--data-urlencode "max_nodes=1" \
@ -40,11 +44,16 @@ while IFS=: read -r filepath url; do
done
fi
fi
# Treat Cloudflare JS-challenge and rate-limit as success.
if [[ "$code" == "403" || "$code" == "429" || "$code" == "503" ]]; then
printf "${yellow}WARN %s${reset} ${cyan}%s${reset} %s\n" "$code" "$url" "$filepath"
exit 0
fi
if [ "$code" -lt 200 ] || [ "$code" -ge 400 ]; then
printf "${red}%s${reset} ${yellow}%s${reset} %s\n" "$code" "$url" "$filepath" >&2
printf "${red}FAIL %s${reset} ${yellow}%s${reset} %s\n" "$code" "$url" "$filepath" >&2
exit 1
else
printf "${green}%s${reset} ${cyan}%s${reset} %s\n" "$code" "$url" "$filepath"
printf "${green} OK %s${reset} ${cyan}%s${reset} %s\n" "$code" "$url" "$filepath"
exit 0
fi
) &
@ -53,7 +62,7 @@ while IFS=: read -r filepath url; do
sleep 1
done
done < <(
pattern='(?!.*@lint-ignore)(?<!git\+)(?<!\$\{)https?://(?![^\s<>\")]*[<>\{\}\$])[^[:space:]<>")\[\]\\]+'
pattern='(?!.*@lint-ignore)(?<!git\+)(?<!\$\{)https?://(?![^/]*@)(?![^\s<>\")]*[<>\{\}\$])[^[:space:]<>")\[\]\\|]+'
excludes=(
':(exclude,glob)**/.*'
':(exclude,glob)**/*.lock'
@ -72,6 +81,7 @@ while IFS=: read -r filepath url; do
git --no-pager grep --no-color -I -P -o "$pattern" -- "${paths[@]}" "${excludes[@]}" \
| sed -E 's/[^/[:alnum:]]+$//' \
| grep -Ev '://(0\.0\.0\.0|127\.0\.0\.1|localhost)([:/])' \
| grep -Ev '://[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' \
| grep -Ev 'fwdproxy:8080' \
|| true
)