diff --git a/.ci/aarch64_linux/aarch64_ci_setup.sh b/.ci/aarch64_linux/aarch64_ci_setup.sh
index d34b94263652..355536c6604a 100755
--- a/.ci/aarch64_linux/aarch64_ci_setup.sh
+++ b/.ci/aarch64_linux/aarch64_ci_setup.sh
@@ -2,36 +2,22 @@
 set -eux -o pipefail
 
 # This script is used to prepare the Docker container for aarch64_ci_wheel_build.py python script
-# as we need to install conda and setup the python version for the build.
-
-CONDA_PYTHON_EXE=/opt/conda/bin/python
-CONDA_EXE=/opt/conda/bin/conda
-CONDA_ENV_NAME=aarch64_env
-PATH=/opt/conda/bin:$PATH
-LD_LIBRARY_PATH=/opt/conda/envs/${CONDA_ENV_NAME}/lib/:/opt/conda/lib:$LD_LIBRARY_PATH
-
-###############################################################################
-# Install conda
-# disable SSL_verify due to getting "Could not find a suitable TLS CA certificate bundle, invalid path"
-# when using Python version, less than the conda latest
-###############################################################################
-echo 'Installing conda-forge'
-curl -L -o /mambaforge.sh https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-aarch64.sh
-chmod +x /mambaforge.sh
-/mambaforge.sh -b -p /opt/conda
-rm /mambaforge.sh
-source /opt/conda/etc/profile.d/conda.sh
-conda config --set ssl_verify False
-conda create -y -c conda-forge -n "${CONDA_ENV_NAME}" python=${DESIRED_PYTHON}
-conda activate "${CONDA_ENV_NAME}"
+# By creating symlinks from desired /opt/python to /usr/local/bin/
 
+NUMPY_VERSION=2.0.2
+PYGIT2_VERSION=1.15.1
 if [[ "$DESIRED_PYTHON"  == "3.13" ]]; then
-    pip install -q --pre numpy==2.1.2
-    conda install -y -c conda-forge pyyaml==6.0.2 patchelf==0.17.2 pygit2==1.15.1 ninja==1.11.1 scons==4.7.0
-else
-    pip install -q --pre numpy==2.0.2
-    conda install -y -c conda-forge pyyaml==6.0.1 patchelf==0.17.2 pygit2==1.13.2 ninja==1.11.1 scons==4.5.2
+    NUMPY_VERSION=2.1.2
+    PYGIT2_VERSION=1.16.0
 fi
 
+SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )"
+source $SCRIPTPATH/../manywheel/set_desired_python.sh
+
+pip install -q numpy==${NUMPY_VERSION} pyyaml==6.0.2 scons==4.7.0 ninja==1.11.1 patchelf==0.17.2 pygit2==${PYGIT2_VERSION}
+
+for tool in python python3 pip pip3 ninja scons patchelf; do
+    ln -sf ${DESIRED_PYTHON_BIN_DIR}/${tool} /usr/local/bin;
+done
+
 python --version
-conda --version
diff --git a/.ci/aarch64_linux/aarch64_wheel_ci_build.py b/.ci/aarch64_linux/aarch64_wheel_ci_build.py
index f0ad414e37bb..9a1858905d35 100755
--- a/.ci/aarch64_linux/aarch64_wheel_ci_build.py
+++ b/.ci/aarch64_linux/aarch64_wheel_ci_build.py
@@ -88,7 +88,7 @@ def update_wheel(wheel_path) -> None:
         "/usr/local/cuda/lib64/libcudnn_engines_runtime_compiled.so.9",
         "/usr/local/cuda/lib64/libcudnn_engines_precompiled.so.9",
         "/usr/local/cuda/lib64/libcudnn_heuristic.so.9",
-        "/opt/conda/envs/aarch64_env/lib/libgomp.so.1",
+        "/lib64/libgomp.so.1",
         "/usr/lib64/libgfortran.so.5",
         "/acl/build/libarm_compute.so",
         "/acl/build/libarm_compute_graph.so",
diff --git a/.ci/manywheel/build_common.sh b/.ci/manywheel/build_common.sh
index c2823b01865b..7381984df0e9 100644
--- a/.ci/manywheel/build_common.sh
+++ b/.ci/manywheel/build_common.sh
@@ -4,12 +4,9 @@
 set -ex
 SOURCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
 
+source ${SOURCE_DIR}/set_desired_python.sh
+
 
-# Require only one python installation
-if [[ -z "$DESIRED_PYTHON" ]]; then
-    echo "Need to set DESIRED_PYTHON env variable"
-    exit 1
-fi
 if [[ -n "$BUILD_PYTHONLESS" && -z "$LIBTORCH_VARIANT" ]]; then
     echo "BUILD_PYTHONLESS is set, so need LIBTORCH_VARIANT to also be set"
     echo "LIBTORCH_VARIANT should be one of shared-with-deps shared-without-deps static-with-deps static-without-deps"
@@ -80,27 +77,7 @@ if [[ -e /opt/openssl ]]; then
     export CMAKE_INCLUDE_PATH="/opt/openssl/include":$CMAKE_INCLUDE_PATH
 fi
 
-# If given a python version like 3.6m or 2.7mu, convert this to the format we
-# expect. The binary CI jobs pass in python versions like this; they also only
-# ever pass one python version, so we assume that DESIRED_PYTHON is not a list
-# in this case
-if [[ -n "$DESIRED_PYTHON" && $DESIRED_PYTHON =~ ([0-9].[0-9]+)t ]]; then
-    python_digits="$(echo $DESIRED_PYTHON | tr -cd [:digit:])"
-    py_majmin="${DESIRED_PYTHON}"
-    DESIRED_PYTHON="cp${python_digits}-cp${python_digits}t"
-elif [[ -n "$DESIRED_PYTHON" && "$DESIRED_PYTHON" != cp* ]]; then
-    python_nodot="$(echo $DESIRED_PYTHON | tr -d m.u)"
-    DESIRED_PYTHON="cp${python_nodot}-cp${python_nodot}"
-    if [[ ${python_nodot} -ge 310 ]]; then
-        py_majmin="${DESIRED_PYTHON:2:1}.${DESIRED_PYTHON:3:2}"
-    else
-        py_majmin="${DESIRED_PYTHON:2:1}.${DESIRED_PYTHON:3:1}"
-    fi
-fi
 
-pydir="/opt/python/$DESIRED_PYTHON"
-export PATH="$pydir/bin:$PATH"
-echo "Will build for Python version: ${DESIRED_PYTHON} with ${python_installation}"
 
 mkdir -p /tmp/$WHEELHOUSE_DIR
 
diff --git a/.ci/manywheel/set_desired_python.sh b/.ci/manywheel/set_desired_python.sh
new file mode 100644
index 000000000000..16b56ed499f7
--- /dev/null
+++ b/.ci/manywheel/set_desired_python.sh
@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+
+# Require only one python installation
+if [[ -z "$DESIRED_PYTHON" ]]; then
+    echo "Need to set DESIRED_PYTHON env variable"
+    exit 1
+fi
+
+# If given a python version like 3.6m or 2.7mu, convert this to the format we
+# expect. The binary CI jobs pass in python versions like this; they also only
+# ever pass one python version, so we assume that DESIRED_PYTHON is not a list
+# in this case
+if [[ -n "$DESIRED_PYTHON" && $DESIRED_PYTHON =~ ([0-9].[0-9]+)t ]]; then
+    python_digits="$(echo $DESIRED_PYTHON | tr -cd [:digit:])"
+    py_majmin="${DESIRED_PYTHON}"
+    DESIRED_PYTHON="cp${python_digits}-cp${python_digits}t"
+elif [[ -n "$DESIRED_PYTHON" && "$DESIRED_PYTHON" != cp* ]]; then
+    python_nodot="$(echo $DESIRED_PYTHON | tr -d m.u)"
+    DESIRED_PYTHON="cp${python_nodot}-cp${python_nodot}"
+    if [[ ${python_nodot} -ge 310 ]]; then
+        py_majmin="${DESIRED_PYTHON:2:1}.${DESIRED_PYTHON:3:2}"
+    else
+        py_majmin="${DESIRED_PYTHON:2:1}.${DESIRED_PYTHON:3:1}"
+    fi
+fi
+
+pydir="/opt/python/$DESIRED_PYTHON"
+export DESIRED_PYTHON_BIN_DIR="${pydir}/bin"
+export PATH="$DESIRED_PYTHON_BIN_DIR:$PATH"
+echo "Will build for Python version: ${DESIRED_PYTHON}"