Move CUDA linalg code to its own subfolder (#72304)

Summary:
Pull Request resolved: https://github.com/pytorch/pytorch/pull/72304

This is a no-op change that simply moves files around in preparation of moving linear algebra in its own dynamically boundable module
This also simplifies torch_cuda_cu build rules, as all files from linalg it needs are in its own folder now.
Bazel CUDA rules are in some weird disarray(needed to add wildcard there as it ignores files mentioned in build_variables.so) and similar wildcard needs to be added to internal build system.

Test Plan: Imported from OSS

Reviewed By: dagitses, ngimel

Differential Revision: D33992796

Pulled By: malfet

fbshipit-source-id: 3f4fa1c224016d03e1a982a7ae5ac7807bc772e2
(cherry picked from commit 6a5a1b0c3f306a8915a45bcdf2c51f15b02d8a14)

BUILD.bazel

@@ -284,6 +284,7 @@ filegroup(
             "aten/src/ATen/cuda/detail/*.cpp",
             "aten/src/ATen/cudnn/*.cpp",
             "aten/src/ATen/native/cuda/*.cpp",
+            "aten/src/ATen/native/cuda/linalg/*.cpp",
             "aten/src/ATen/native/cudnn/*.cpp",
             "aten/src/ATen/native/miopen/*.cpp",
             "aten/src/ATen/native/sparse/cuda/*.cpp",

aten/src/ATen/CMakeLists.txt

@@ -117,6 +117,7 @@ file(GLOB native_cpu_h "native/cpu/*.h")
 file(GLOB native_cuda_cu "native/cuda/*.cu")
 file(GLOB native_cuda_cpp "native/cuda/*.cpp")
 file(GLOB native_cuda_h "native/cuda/*.h" "native/cuda/*.cuh")
+file(GLOB native_cuda_linalg_cpp "native/cuda/linalg/*.cpp")
 file(GLOB native_hip_h "native/hip/*.h" "native/hip/*.cuh")
 file(GLOB native_cudnn_cpp "native/cudnn/*.cpp")
 file(GLOB native_sparse_cuda_cu "native/sparse/cuda/*.cu")
@@ -127,6 +128,7 @@ file(GLOB native_quantized_cudnn_cpp "native/quantized/cudnn/*.cpp")
 
 file(GLOB native_hip_hip "native/hip/*.hip")
 file(GLOB native_hip_cpp "native/hip/*.cpp")
+file(GLOB native_hip_linalg_cpp "native/hip/linalg/*.cpp")
 file(GLOB native_miopen_cpp "native/miopen/*.cpp")
 file(GLOB native_cudnn_hip_cpp "native/cudnn/hip/*.cpp")
 file(GLOB native_sparse_hip_hip "native/sparse/hip/*.hip")
@@ -189,10 +191,10 @@ endif()
 
 if(USE_CUDA)
   list(APPEND ATen_CUDA_INCLUDE ${CMAKE_CURRENT_SOURCE_DIR}/cuda)
-
   list(APPEND ATen_CUDA_CU_SRCS
     ${cuda_cu}
     ${native_cuda_cu}
+    ${native_cuda_linalg_cpp}
     ${native_sparse_cuda_cu}
     ${native_quantized_cuda_cu}
     ${cuda_generated_sources}
@@ -225,7 +227,7 @@ if(USE_ROCM)
   list(APPEND ATen_HIP_INCLUDE ${CMAKE_CURRENT_SOURCE_DIR}/hip)
   set(ATen_HIP_SRCS ${ATen_HIP_SRCS} ${hip_hip} ${native_hip_hip} ${native_sparse_hip_hip} ${native_quantized_hip_hip})
   # TODO: Codegen separate files for HIP and use those (s/cuda_generated_sources/hip_generated_sources)
-  set(all_hip_cpp ${native_sparse_hip_cpp} ${native_quantized_hip_cpp} ${hip_cpp} ${native_hip_cpp} ${cuda_generated_sources} ${ATen_HIP_SRCS})
+  set(all_hip_cpp ${native_sparse_hip_cpp} ${native_quantized_hip_cpp} ${hip_cpp} ${native_hip_cpp} ${native_hip_linalg_cpp} ${cuda_generated_sources} ${ATen_HIP_SRCS})
   set(all_hip_cpp ${native_miopen_cpp} ${native_cudnn_hip_cpp} ${miopen_cpp} ${all_hip_cpp})
 endif()
 

aten/src/ATen/native/cuda/linalg/BatchLinearAlgebra.cpp

@@ -12,7 +12,7 @@
 #include <ATen/native/Resize.h>
 #include <ATen/native/LinearAlgebra.h>
 #include <ATen/native/BatchLinearAlgebra.h>
-#include <ATen/native/cuda/BatchLinearAlgebraLib.h>
+#include <ATen/native/cuda/linalg/BatchLinearAlgebraLib.h>
 #include <ATen/native/cpu/zmath.h>
 
 #if AT_MAGMA_ENABLED()

aten/src/ATen/native/cuda/linalg/BatchLinearAlgebraLib.cpp

@@ -4,7 +4,6 @@
 #include <ATen/Dispatch.h>
 #include <ATen/NativeFunctions.h>
 #include <ATen/cuda/PinnedMemoryAllocator.h>
-#include <ATen/cuda/CUDASolver.h>
 #include <ATen/cuda/CUDABlas.h>
 #include <ATen/cuda/CUDAEvent.h>
 #include <c10/cuda/CUDAStream.h>
@@ -12,7 +11,8 @@
 
 #include <ATen/native/LinearAlgebraUtils.h>
 #include <ATen/native/cuda/MiscUtils.h>
-#include <ATen/native/cuda/BatchLinearAlgebraLib.h>
+#include <ATen/native/cuda/linalg/CUDASolver.h>
+#include <ATen/native/cuda/linalg/BatchLinearAlgebraLib.h>
 
 #ifndef AT_PER_OPERATOR_HEADERS
 #include <ATen/Functions.h>

aten/src/ATen/native/cuda/linalg/CUDASolver.cpp

@@ -1,6 +1,6 @@
 #include <ATen/Context.h>
 #include <ATen/NativeFunctions.h>
-#include <ATen/cuda/CUDASolver.h>
+#include <ATen/native/cuda/linalg/CUDASolver.h>
 #include <c10/cuda/CUDACachingAllocator.h>
 #include <c10/macros/Export.h>
 

tools/build_variables.bzl

@@ -1320,13 +1320,9 @@ aten_native_source_list = sorted(aten_native_source_non_codegen_list + aten_nati
 # .cu files can be found via glob
 aten_cuda_cu_source_list = [
     "aten/src/ATen/cuda/CUDABlas.cpp",
-    "aten/src/ATen/cuda/CUDASolver.cpp",
     "aten/src/ATen/cuda/CUDASparseBlas.cpp",
     "aten/src/ATen/cuda/CublasHandlePool.cpp",
-    "aten/src/ATen/cuda/CusolverDnHandlePool.cpp",
     "aten/src/ATen/native/cuda/Activation.cpp",
-    "aten/src/ATen/native/cuda/BatchLinearAlgebra.cpp",
-    "aten/src/ATen/native/cuda/BatchLinearAlgebraLib.cpp",
     "aten/src/ATen/native/cuda/Blas.cpp",
     "aten/src/ATen/native/cuda/Equal.cpp",
     "aten/src/ATen/native/cuda/IndexKernel.cpp",