diff --git a/.jenkins/caffe2/test.sh b/.jenkins/caffe2/test.sh
index 00a83b952956..63f50d384790 100755
--- a/.jenkins/caffe2/test.sh
+++ b/.jenkins/caffe2/test.sh
@@ -120,6 +120,5 @@ pip install --user pytest-sugar
 # torchvision tests #
 #####################
 if [[ "$BUILD_ENVIRONMENT" == *onnx* ]]; then
-  pip install --user torchvision
   "$ROOT_DIR/scripts/onnx/test.sh"
 fi
diff --git a/scripts/onnx/test.sh b/scripts/onnx/test.sh
index 4ad633bd87f9..45393c62f0e4 100755
--- a/scripts/onnx/test.sh
+++ b/scripts/onnx/test.sh
@@ -23,7 +23,21 @@ do
 done
 set -- "${UNKNOWN[@]}" # leave UNKNOWN
 
-pip install pytest scipy torchvision hypothesis
+pip install pytest scipy hypothesis
+
+install_torchvision() {
+  echo "Installing torchvision at branch master"
+  rm -rf vision
+  # TODO: This git clone is bad, it means pushes to torchvision can break
+  # PyTorch CI
+  git clone https://github.com/pytorch/vision --quiet
+  pushd vision
+  pip install -q --user .
+  popd
+  rm -rf vision
+}
+install_torchvision
+
 if [[ $PARALLEL == 1 ]]; then
     pip install pytest-xdist
 fi