diff --git a/CMakeLists.txt b/CMakeLists.txt
index 21fd00463bf3..d78d2613251c 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -388,9 +388,9 @@ cmake_dependent_option(USE_PRIORITIZED_TEXT_FOR_LD "Use prioritized text linker
 
 option(USE_MIMALLOC "Use mimalloc" OFF)
 # Enable third party mimalloc library to improve memory allocation performance
-# on Windows and AArch64.
+# on Windows.
 option(USE_MIMALLOC_ON_MKL "Use mimalloc on MKL" OFF)
-if(WIN32 OR CPU_AARCH64)
+if(WIN32)
   set(USE_MIMALLOC ON)
 
   # Not enable USE_MIMALLOC_ON_MKL due to it caused issue:
diff --git a/c10/core/impl/alloc_cpu.cpp b/c10/core/impl/alloc_cpu.cpp
index c1b7ca858632..791104690cd8 100644
--- a/c10/core/impl/alloc_cpu.cpp
+++ b/c10/core/impl/alloc_cpu.cpp
@@ -108,15 +108,12 @@ void* alloc_cpu(size_t nbytes) {
       "DefaultCPUAllocator: not enough memory: you tried to allocate ",
       nbytes,
       " bytes.");
-#elif defined(USE_MIMALLOC)
-  data = mi_malloc_aligned(nbytes, gAlignment);
-  CAFFE_ENFORCE(
-      data,
-      "DefaultCPUAllocator: not enough memory: you tried to allocate ",
-      nbytes,
-      " bytes.");
 #elif defined(_MSC_VER)
+#ifdef USE_MIMALLOC
+  data = mi_malloc_aligned(nbytes, gAlignment);
+#else
   data = _aligned_malloc(nbytes, gAlignment);
+#endif
   CAFFE_ENFORCE(
       data,
       "DefaultCPUAllocator: not enough memory: you tried to allocate ",
@@ -163,10 +160,12 @@ void* alloc_cpu(size_t nbytes) {
 }
 
 void free_cpu(void* data) {
+#ifdef _MSC_VER
 #ifdef USE_MIMALLOC
   mi_free(data);
-#elif defined(_MSC_VER)
+#else
   _aligned_free(data);
+#endif
 #else
   // NOLINTNEXTLINE(cppcoreguidelines-no-malloc)
   free(data);
diff --git a/setup.py b/setup.py
index bdfab24a0b32..9f2eae6f6a90 100644
--- a/setup.py
+++ b/setup.py
@@ -225,7 +225,7 @@
 #
 #   USE_MIMALLOC
 #      Static link mimalloc into C10, and use mimalloc in alloc_cpu & alloc_free.
-#      By default, It is only enabled on Windows and AArch64.
+#      By default, It is only enabled on Windows.
 #
 #   BUILD_LIBTORCH_WHL
 #      Builds libtorch.so and its dependencies as a wheel