diff --git a/.jenkins/pytorch/common_utils.sh b/.jenkins/pytorch/common_utils.sh
index 7b592d57c280..61a7cb36178b 100644
--- a/.jenkins/pytorch/common_utils.sh
+++ b/.jenkins/pytorch/common_utils.sh
@@ -141,12 +141,6 @@ function checkout_install_torchdynamo() {
   popd
 }
 
-function install_functorch() {
-  pushd functorch
-  time python setup.py develop
-  popd
-}
-
 function test_functorch() {
   python test/run_test.py --functorch --verbose
 }
diff --git a/.jenkins/pytorch/macos-test.sh b/.jenkins/pytorch/macos-test.sh
index a30e16ba942e..244c9dda7fc1 100755
--- a/.jenkins/pytorch/macos-test.sh
+++ b/.jenkins/pytorch/macos-test.sh
@@ -177,7 +177,6 @@ test_dynamo() {
 }
 
 if [[ "${TEST_CONFIG}" == *functorch* ]]; then
-  install_functorch
   test_functorch
 elif [[ $NUM_TEST_SHARDS -gt 1 ]]; then
   test_python_shard "${SHARD_NUMBER}"
diff --git a/.jenkins/pytorch/test.sh b/.jenkins/pytorch/test.sh
index 43e6119d4401..ea1654d69e90 100755
--- a/.jenkins/pytorch/test.sh
+++ b/.jenkins/pytorch/test.sh
@@ -180,9 +180,6 @@ test_dynamo_shard() {
     echo "NUM_TEST_SHARDS must be defined to run a Python test shard"
     exit 1
   fi
-  pushd functorch
-  python setup.py develop
-  popd
   # Temporarily disable test_fx for dynamo pending the investigation on TTS
   # regression in https://github.com/pytorch/torchdynamo/issues/784
   time python test/run_test.py \
@@ -686,7 +683,6 @@ elif [[ "${BUILD_ENVIRONMENT}" == *-mobile-lightweight-dispatch* ]]; then
 elif [[ "${TEST_CONFIG}" = docs_test ]]; then
   test_docs_test
 elif [[ "${TEST_CONFIG}" == *functorch* ]]; then
-  install_functorch
   test_functorch
 else
   install_torchvision
diff --git a/.jenkins/pytorch/win-test-helpers/build_pytorch.bat b/.jenkins/pytorch/win-test-helpers/build_pytorch.bat
index 7edeca96ed8d..65ba7ef3235c 100644
--- a/.jenkins/pytorch/win-test-helpers/build_pytorch.bat
+++ b/.jenkins/pytorch/win-test-helpers/build_pytorch.bat
@@ -144,7 +144,7 @@ python setup.py install --cmake && sccache --show-stats && (
   if "%BUILD_ENVIRONMENT%"=="" (
     echo NOTE: To run `import torch`, please make sure to activate the conda environment by running `call %CONDA_PARENT_DIR%\Miniconda3\Scripts\activate.bat %CONDA_PARENT_DIR%\Miniconda3` in Command Prompt before running Git Bash.
   ) else (
-    7z a %TMP_DIR_WIN%\%IMAGE_COMMIT_TAG%.7z %CONDA_PARENT_DIR%\Miniconda3\Lib\site-packages\torch %CONDA_PARENT_DIR%\Miniconda3\Lib\site-packages\torchgen %CONDA_PARENT_DIR%\Miniconda3\Lib\site-packages\caffe2 && copy /Y "%TMP_DIR_WIN%\%IMAGE_COMMIT_TAG%.7z" "%PYTORCH_FINAL_PACKAGE_DIR%\"
+    7z a %TMP_DIR_WIN%\%IMAGE_COMMIT_TAG%.7z %CONDA_PARENT_DIR%\Miniconda3\Lib\site-packages\torch %CONDA_PARENT_DIR%\Miniconda3\Lib\site-packages\torchgen %CONDA_PARENT_DIR%\Miniconda3\Lib\site-packages\caffe2 %CONDA_PARENT_DIR%\Miniconda3\Lib\site-packages\functorch && copy /Y "%TMP_DIR_WIN%\%IMAGE_COMMIT_TAG%.7z" "%PYTORCH_FINAL_PACKAGE_DIR%\"
     if errorlevel 1 exit /b
     if not errorlevel 0 exit /b
 
diff --git a/.jenkins/pytorch/win-test-helpers/install_test_functorch.bat b/.jenkins/pytorch/win-test-helpers/install_test_functorch.bat
index 7679bffbc70e..d06d46f3dd22 100644
--- a/.jenkins/pytorch/win-test-helpers/install_test_functorch.bat
+++ b/.jenkins/pytorch/win-test-helpers/install_test_functorch.bat
@@ -6,15 +6,6 @@ if not errorlevel 0 (
   exit /b
 )
 
-pushd functorch
-echo "Install functorch"
-:: --no-deps because for some reason, on windows, `torch` isn't found in
-:: `pip list` despite being installed. With just `python setup.py develop`,
-:: setuptools explicitly checks for the existence of torch and can't find it.
-python setup.py develop --no-deps
-popd
-if ERRORLEVEL 1 goto fail
-
 echo "Installing test dependencies"
 pip install networkx
 if errorlevel 1 exit /b
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 3b4d71d2c1aa..3800fe238cd6 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -355,6 +355,8 @@ option(USE_PER_OPERATOR_HEADERS "Whether ATen should generate separate headers f
 cmake_dependent_option(
     BUILD_LAZY_TS_BACKEND "Build the lazy Torchscript backend, not compatible with mobile builds" ON
     "NOT INTERN_BUILD_MOBILE" OFF)
+cmake_dependent_option(
+    BUILD_FUNCTORCH "Build Functorch" ON "BUILD_PYTHON" OFF)
 
 
 if(USE_CCACHE)
@@ -622,6 +624,7 @@ if(INTERN_BUILD_MOBILE)
     set(INTERN_DISABLE_AUTOGRAD ON)
   endif()
   set(BUILD_PYTHON OFF)
+  set(BUILD_FUNCTORCH OFF)
   set(BUILD_CAFFE2_OPS OFF)
   set(USE_DISTRIBUTED OFF)
   set(NO_API ON)
@@ -1175,3 +1178,7 @@ caffe2_print_configuration_summary()
 if(USE_DEPLOY)
   add_subdirectory(torch/csrc/deploy)
 endif()
+
+if(BUILD_FUNCTORCH)
+  add_subdirectory(functorch)
+endif()
diff --git a/MANIFEST.in b/MANIFEST.in
index acf4c7291f43..403b90b702df 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -26,5 +26,6 @@ recursive-include benchmarks *.*
 recursive-include scripts *.*
 recursive-include mypy_plugins *.*
 recursive-include modules *.*
+recursive-include functorch *.*
 prune */__pycache__
 global-exclude *.o *.so *.dylib *.a .git *.pyc *.swp
diff --git a/caffe2/CMakeLists.txt b/caffe2/CMakeLists.txt
index 7c66073ad813..9d4aa23a1e0a 100644
--- a/caffe2/CMakeLists.txt
+++ b/caffe2/CMakeLists.txt
@@ -1245,6 +1245,8 @@ install(FILES
 
 # ---[ Torch python bindings build
 add_subdirectory(../torch torch)
+set(TORCH_PYTHON_COMPILE_OPTIONS ${TORCH_PYTHON_COMPILE_OPTIONS} PARENT_SCOPE)
+set(TORCH_PYTHON_LINK_FLAGS ${TORCH_PYTHON_LINK_FLAGS} PARENT_SCOPE)
 
 # ==========================================================
 # END formerly-libtorch flags
diff --git a/functorch/CMakeLists.txt b/functorch/CMakeLists.txt
new file mode 100644
index 000000000000..1d156ee63502
--- /dev/null
+++ b/functorch/CMakeLists.txt
@@ -0,0 +1,37 @@
+cmake_minimum_required(VERSION 3.12)
+project(functorch)
+set(CMAKE_CXX_STANDARD 14)
+
+include(GNUInstallDirs)
+include(CMakePackageConfigHelpers)
+
+set(FT_DIR csrc)
+file(GLOB_RECURSE FT_SOURCES ${FT_DIR}/*.cpp)
+
+add_library(${PROJECT_NAME} MODULE ${FT_SOURCES})
+target_include_directories(${PROJECT_NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
+target_compile_definitions(${PROJECT_NAME} PRIVATE FUNCTORCH_BUILD_MAIN_LIB)
+target_compile_definitions(${PROJECT_NAME} PRIVATE TORCH_EXTENSION_NAME=_C)
+target_compile_definitions(${PROJECT_NAME} PRIVATE TORCH_API_INCLUDE_EXTENSION_H)
+target_compile_options(${PROJECT_NAME} PRIVATE ${TORCH_PYTHON_COMPILE_OPTIONS})
+target_link_libraries(${PROJECT_NAME} PRIVATE torch torch_python)
+target_link_libraries(${PROJECT_NAME} PRIVATE pybind::pybind11)
+
+set_target_properties(${PROJECT_NAME} PROPERTIES LIBRARY_OUTPUT_DIRECTORY
+      ${CMAKE_BINARY_DIR}/functorch)
+
+# Copy-pasted prefix/suffix logic for Python extensions from
+# https://github.com/pytorch/pytorch/blob/33bb8ae350611760139457b85842b1d7edf9aa11/caffe2/CMakeLists.txt#L1975
+# https://github.com/pytorch/pytorch/blob/33bb8ae350611760139457b85842b1d7edf9aa11/caffe2/CMakeLists.txt#L2022
+# TODO: It would be good to be able to use Python3_add_library target, but it does not work in many cases
+set_target_properties(${PROJECT_NAME} PROPERTIES PREFIX "" DEBUG_POSTFIX "")
+if(WIN32)
+  set_target_properties(${PROJECT_NAME} PROPERTIES SUFFIX ".pyd")
+else()
+  set_target_properties(${PROJECT_NAME} PROPERTIES SUFFIX ".so")
+endif()
+# Needed to link functorch on MacOS
+if(NOT ${TORCH_PYTHON_LINK_FLAGS} STREQUAL "")
+  set_target_properties(${PROJECT_NAME} PROPERTIES LINK_FLAGS ${TORCH_PYTHON_LINK_FLAGS})
+endif()
+install(TARGETS ${PROJECT_NAME} DESTINATION "${CMAKE_CURRENT_SOURCE_DIR}")
diff --git a/functorch/functorch/__init__.py b/functorch/__init__.py
similarity index 92%
rename from functorch/functorch/__init__.py
rename to functorch/__init__.py
index 4f5adc5a7ad1..99ab943dd929 100644
--- a/functorch/functorch/__init__.py
+++ b/functorch/__init__.py
@@ -32,7 +32,4 @@ from ._src.make_functional import (
     FunctionalModuleWithBuffers,
 )
 
-try:
-    from .version import __version__  # noqa: F401
-except ImportError:
-    pass
+__version__ = torch.__version__
diff --git a/functorch/functorch/_src/__init__.py b/functorch/_src/__init__.py
similarity index 100%
rename from functorch/functorch/_src/__init__.py
rename to functorch/_src/__init__.py
diff --git a/functorch/functorch/_src/aot_autograd.py b/functorch/_src/aot_autograd.py
similarity index 100%
rename from functorch/functorch/_src/aot_autograd.py
rename to functorch/_src/aot_autograd.py
diff --git a/functorch/functorch/_src/benchmark_utils.py b/functorch/_src/benchmark_utils.py
similarity index 100%
rename from functorch/functorch/_src/benchmark_utils.py
rename to functorch/_src/benchmark_utils.py
diff --git a/functorch/functorch/_src/compile_utils.py b/functorch/_src/compile_utils.py
similarity index 100%
rename from functorch/functorch/_src/compile_utils.py
rename to functorch/_src/compile_utils.py
diff --git a/functorch/functorch/_src/compilers.py b/functorch/_src/compilers.py
similarity index 100%
rename from functorch/functorch/_src/compilers.py
rename to functorch/_src/compilers.py
diff --git a/functorch/functorch/_src/config.py b/functorch/_src/config.py
similarity index 100%
rename from functorch/functorch/_src/config.py
rename to functorch/_src/config.py
diff --git a/functorch/functorch/_src/eager_transforms.py b/functorch/_src/eager_transforms.py
similarity index 100%
rename from functorch/functorch/_src/eager_transforms.py
rename to functorch/_src/eager_transforms.py
diff --git a/functorch/functorch/_src/fx_minifier.py b/functorch/_src/fx_minifier.py
similarity index 100%
rename from functorch/functorch/_src/fx_minifier.py
rename to functorch/_src/fx_minifier.py
diff --git a/functorch/functorch/_src/make_functional.py b/functorch/_src/make_functional.py
similarity index 100%
rename from functorch/functorch/_src/make_functional.py
rename to functorch/_src/make_functional.py
diff --git a/functorch/functorch/_src/monkey_patching.py b/functorch/_src/monkey_patching.py
similarity index 100%
rename from functorch/functorch/_src/monkey_patching.py
rename to functorch/_src/monkey_patching.py
diff --git a/functorch/functorch/_src/named_members_polyfill.py b/functorch/_src/named_members_polyfill.py
similarity index 100%
rename from functorch/functorch/_src/named_members_polyfill.py
rename to functorch/_src/named_members_polyfill.py
diff --git a/functorch/functorch/_src/partitioners.py b/functorch/_src/partitioners.py
similarity index 100%
rename from functorch/functorch/_src/partitioners.py
rename to functorch/_src/partitioners.py
diff --git a/functorch/functorch/_src/python_key.py b/functorch/_src/python_key.py
similarity index 100%
rename from functorch/functorch/_src/python_key.py
rename to functorch/_src/python_key.py
diff --git a/functorch/functorch/_src/pytree_hacks.py b/functorch/_src/pytree_hacks.py
similarity index 100%
rename from functorch/functorch/_src/pytree_hacks.py
rename to functorch/_src/pytree_hacks.py
diff --git a/functorch/functorch/_src/top_operators_github_usage.py b/functorch/_src/top_operators_github_usage.py
similarity index 100%
rename from functorch/functorch/_src/top_operators_github_usage.py
rename to functorch/_src/top_operators_github_usage.py
diff --git a/functorch/functorch/_src/vmap.py b/functorch/_src/vmap.py
similarity index 100%
rename from functorch/functorch/_src/vmap.py
rename to functorch/_src/vmap.py
diff --git a/functorch/functorch/compile/__init__.py b/functorch/compile/__init__.py
similarity index 100%
rename from functorch/functorch/compile/__init__.py
rename to functorch/compile/__init__.py
diff --git a/functorch/functorch/csrc/ADInterpreters.cpp b/functorch/csrc/ADInterpreters.cpp
similarity index 100%
rename from functorch/functorch/csrc/ADInterpreters.cpp
rename to functorch/csrc/ADInterpreters.cpp
diff --git a/functorch/functorch/csrc/ADInterpreters.h b/functorch/csrc/ADInterpreters.h
similarity index 100%
rename from functorch/functorch/csrc/ADInterpreters.h
rename to functorch/csrc/ADInterpreters.h
diff --git a/functorch/functorch/csrc/BatchRulesActivation.cpp b/functorch/csrc/BatchRulesActivation.cpp
similarity index 100%
rename from functorch/functorch/csrc/BatchRulesActivation.cpp
rename to functorch/csrc/BatchRulesActivation.cpp
diff --git a/functorch/functorch/csrc/BatchRulesBinaryOps.cpp b/functorch/csrc/BatchRulesBinaryOps.cpp
similarity index 100%
rename from functorch/functorch/csrc/BatchRulesBinaryOps.cpp
rename to functorch/csrc/BatchRulesBinaryOps.cpp
diff --git a/functorch/functorch/csrc/BatchRulesConvolution.cpp b/functorch/csrc/BatchRulesConvolution.cpp
similarity index 100%
rename from functorch/functorch/csrc/BatchRulesConvolution.cpp
rename to functorch/csrc/BatchRulesConvolution.cpp
diff --git a/functorch/functorch/csrc/BatchRulesDecompositions.cpp b/functorch/csrc/BatchRulesDecompositions.cpp
similarity index 100%
rename from functorch/functorch/csrc/BatchRulesDecompositions.cpp
rename to functorch/csrc/BatchRulesDecompositions.cpp
diff --git a/functorch/functorch/csrc/BatchRulesDynamic.cpp b/functorch/csrc/BatchRulesDynamic.cpp
similarity index 100%
rename from functorch/functorch/csrc/BatchRulesDynamic.cpp
rename to functorch/csrc/BatchRulesDynamic.cpp
diff --git a/functorch/functorch/csrc/BatchRulesFactory.cpp b/functorch/csrc/BatchRulesFactory.cpp
similarity index 100%
rename from functorch/functorch/csrc/BatchRulesFactory.cpp
rename to functorch/csrc/BatchRulesFactory.cpp
diff --git a/functorch/functorch/csrc/BatchRulesHelper.cpp b/functorch/csrc/BatchRulesHelper.cpp
similarity index 100%
rename from functorch/functorch/csrc/BatchRulesHelper.cpp
rename to functorch/csrc/BatchRulesHelper.cpp
diff --git a/functorch/functorch/csrc/BatchRulesHelper.h b/functorch/csrc/BatchRulesHelper.h
similarity index 100%
rename from functorch/functorch/csrc/BatchRulesHelper.h
rename to functorch/csrc/BatchRulesHelper.h
diff --git a/functorch/functorch/csrc/BatchRulesLinearAlgebra.cpp b/functorch/csrc/BatchRulesLinearAlgebra.cpp
similarity index 100%
rename from functorch/functorch/csrc/BatchRulesLinearAlgebra.cpp
rename to functorch/csrc/BatchRulesLinearAlgebra.cpp
diff --git a/functorch/functorch/csrc/BatchRulesLoss.cpp b/functorch/csrc/BatchRulesLoss.cpp
similarity index 100%
rename from functorch/functorch/csrc/BatchRulesLoss.cpp
rename to functorch/csrc/BatchRulesLoss.cpp
diff --git a/functorch/functorch/csrc/BatchRulesModules.cpp b/functorch/csrc/BatchRulesModules.cpp
similarity index 100%
rename from functorch/functorch/csrc/BatchRulesModules.cpp
rename to functorch/csrc/BatchRulesModules.cpp
diff --git a/functorch/functorch/csrc/BatchRulesNorm.cpp b/functorch/csrc/BatchRulesNorm.cpp
similarity index 100%
rename from functorch/functorch/csrc/BatchRulesNorm.cpp
rename to functorch/csrc/BatchRulesNorm.cpp
diff --git a/functorch/functorch/csrc/BatchRulesPooling.cpp b/functorch/csrc/BatchRulesPooling.cpp
similarity index 100%
rename from functorch/functorch/csrc/BatchRulesPooling.cpp
rename to functorch/csrc/BatchRulesPooling.cpp
diff --git a/functorch/functorch/csrc/BatchRulesRandomness.cpp b/functorch/csrc/BatchRulesRandomness.cpp
similarity index 100%
rename from functorch/functorch/csrc/BatchRulesRandomness.cpp
rename to functorch/csrc/BatchRulesRandomness.cpp
diff --git a/functorch/functorch/csrc/BatchRulesReduceOps.cpp b/functorch/csrc/BatchRulesReduceOps.cpp
similarity index 100%
rename from functorch/functorch/csrc/BatchRulesReduceOps.cpp
rename to functorch/csrc/BatchRulesReduceOps.cpp
diff --git a/functorch/functorch/csrc/BatchRulesScatterOps.cpp b/functorch/csrc/BatchRulesScatterOps.cpp
similarity index 100%
rename from functorch/functorch/csrc/BatchRulesScatterOps.cpp
rename to functorch/csrc/BatchRulesScatterOps.cpp
diff --git a/functorch/functorch/csrc/BatchRulesUnaryOps.cpp b/functorch/csrc/BatchRulesUnaryOps.cpp
similarity index 100%
rename from functorch/functorch/csrc/BatchRulesUnaryOps.cpp
rename to functorch/csrc/BatchRulesUnaryOps.cpp
diff --git a/functorch/functorch/csrc/BatchRulesViews.cpp b/functorch/csrc/BatchRulesViews.cpp
similarity index 100%
rename from functorch/functorch/csrc/BatchRulesViews.cpp
rename to functorch/csrc/BatchRulesViews.cpp
diff --git a/functorch/functorch/csrc/BatchedFallback.cpp b/functorch/csrc/BatchedFallback.cpp
similarity index 100%
rename from functorch/functorch/csrc/BatchedFallback.cpp
rename to functorch/csrc/BatchedFallback.cpp
diff --git a/functorch/functorch/csrc/BatchedFallback.h b/functorch/csrc/BatchedFallback.h
similarity index 100%
rename from functorch/functorch/csrc/BatchedFallback.h
rename to functorch/csrc/BatchedFallback.h
diff --git a/functorch/functorch/csrc/BatchingMetaprogramming.h b/functorch/csrc/BatchingMetaprogramming.h
similarity index 100%
rename from functorch/functorch/csrc/BatchingMetaprogramming.h
rename to functorch/csrc/BatchingMetaprogramming.h
diff --git a/functorch/functorch/csrc/CompileCache.cpp b/functorch/csrc/CompileCache.cpp
similarity index 100%
rename from functorch/functorch/csrc/CompileCache.cpp
rename to functorch/csrc/CompileCache.cpp
diff --git a/functorch/functorch/csrc/CompileCache.h b/functorch/csrc/CompileCache.h
similarity index 100%
rename from functorch/functorch/csrc/CompileCache.h
rename to functorch/csrc/CompileCache.h
diff --git a/functorch/functorch/csrc/DynamicLayer.cpp b/functorch/csrc/DynamicLayer.cpp
similarity index 100%
rename from functorch/functorch/csrc/DynamicLayer.cpp
rename to functorch/csrc/DynamicLayer.cpp
diff --git a/functorch/functorch/csrc/DynamicLayer.h b/functorch/csrc/DynamicLayer.h
similarity index 100%
rename from functorch/functorch/csrc/DynamicLayer.h
rename to functorch/csrc/DynamicLayer.h
diff --git a/functorch/functorch/csrc/FunctionalizeInterpreter.cpp b/functorch/csrc/FunctionalizeInterpreter.cpp
similarity index 100%
rename from functorch/functorch/csrc/FunctionalizeInterpreter.cpp
rename to functorch/csrc/FunctionalizeInterpreter.cpp
diff --git a/functorch/functorch/csrc/FunctionalizeInterpreter.h b/functorch/csrc/FunctionalizeInterpreter.h
similarity index 100%
rename from functorch/functorch/csrc/FunctionalizeInterpreter.h
rename to functorch/csrc/FunctionalizeInterpreter.h
diff --git a/functorch/functorch/csrc/Interpreter.cpp b/functorch/csrc/Interpreter.cpp
similarity index 100%
rename from functorch/functorch/csrc/Interpreter.cpp
rename to functorch/csrc/Interpreter.cpp
diff --git a/functorch/functorch/csrc/Interpreter.h b/functorch/csrc/Interpreter.h
similarity index 100%
rename from functorch/functorch/csrc/Interpreter.h
rename to functorch/csrc/Interpreter.h
diff --git a/functorch/functorch/csrc/LegacyBatchingRegistrations.cpp b/functorch/csrc/LegacyBatchingRegistrations.cpp
similarity index 100%
rename from functorch/functorch/csrc/LegacyBatchingRegistrations.cpp
rename to functorch/csrc/LegacyBatchingRegistrations.cpp
diff --git a/functorch/functorch/csrc/LegacyVmapTransforms.cpp b/functorch/csrc/LegacyVmapTransforms.cpp
similarity index 100%
rename from functorch/functorch/csrc/LegacyVmapTransforms.cpp
rename to functorch/csrc/LegacyVmapTransforms.cpp
diff --git a/functorch/functorch/csrc/LegacyVmapTransforms.h b/functorch/csrc/LegacyVmapTransforms.h
similarity index 100%
rename from functorch/functorch/csrc/LegacyVmapTransforms.h
rename to functorch/csrc/LegacyVmapTransforms.h
diff --git a/functorch/functorch/csrc/Macros.h b/functorch/csrc/Macros.h
similarity index 100%
rename from functorch/functorch/csrc/Macros.h
rename to functorch/csrc/Macros.h
diff --git a/functorch/functorch/csrc/PlumbingHelper.cpp b/functorch/csrc/PlumbingHelper.cpp
similarity index 100%
rename from functorch/functorch/csrc/PlumbingHelper.cpp
rename to functorch/csrc/PlumbingHelper.cpp
diff --git a/functorch/functorch/csrc/PlumbingHelper.h b/functorch/csrc/PlumbingHelper.h
similarity index 100%
rename from functorch/functorch/csrc/PlumbingHelper.h
rename to functorch/csrc/PlumbingHelper.h
diff --git a/functorch/functorch/csrc/PyTorchOperatorHacks.cpp b/functorch/csrc/PyTorchOperatorHacks.cpp
similarity index 100%
rename from functorch/functorch/csrc/PyTorchOperatorHacks.cpp
rename to functorch/csrc/PyTorchOperatorHacks.cpp
diff --git a/functorch/functorch/csrc/TensorWrapper.cpp b/functorch/csrc/TensorWrapper.cpp
similarity index 100%
rename from functorch/functorch/csrc/TensorWrapper.cpp
rename to functorch/csrc/TensorWrapper.cpp
diff --git a/functorch/functorch/csrc/TensorWrapper.h b/functorch/csrc/TensorWrapper.h
similarity index 100%
rename from functorch/functorch/csrc/TensorWrapper.h
rename to functorch/csrc/TensorWrapper.h
diff --git a/functorch/functorch/csrc/VmapInterpreter.cpp b/functorch/csrc/VmapInterpreter.cpp
similarity index 100%
rename from functorch/functorch/csrc/VmapInterpreter.cpp
rename to functorch/csrc/VmapInterpreter.cpp
diff --git a/functorch/functorch/csrc/VmapInterpreter.h b/functorch/csrc/VmapInterpreter.h
similarity index 100%
rename from functorch/functorch/csrc/VmapInterpreter.h
rename to functorch/csrc/VmapInterpreter.h
diff --git a/functorch/functorch/csrc/VmapModeRegistrations.cpp b/functorch/csrc/VmapModeRegistrations.cpp
similarity index 100%
rename from functorch/functorch/csrc/VmapModeRegistrations.cpp
rename to functorch/csrc/VmapModeRegistrations.cpp
diff --git a/functorch/functorch/csrc/dim/arena.h b/functorch/csrc/dim/arena.h
similarity index 100%
rename from functorch/functorch/csrc/dim/arena.h
rename to functorch/csrc/dim/arena.h
diff --git a/functorch/functorch/csrc/dim/dim.cpp b/functorch/csrc/dim/dim.cpp
similarity index 100%
rename from functorch/functorch/csrc/dim/dim.cpp
rename to functorch/csrc/dim/dim.cpp
diff --git a/functorch/functorch/csrc/dim/dim.h b/functorch/csrc/dim/dim.h
similarity index 100%
rename from functorch/functorch/csrc/dim/dim.h
rename to functorch/csrc/dim/dim.h
diff --git a/functorch/functorch/csrc/dim/minpybind.h b/functorch/csrc/dim/minpybind.h
similarity index 100%
rename from functorch/functorch/csrc/dim/minpybind.h
rename to functorch/csrc/dim/minpybind.h
diff --git a/functorch/functorch/csrc/dim/python_variable_simple.h b/functorch/csrc/dim/python_variable_simple.h
similarity index 100%
rename from functorch/functorch/csrc/dim/python_variable_simple.h
rename to functorch/csrc/dim/python_variable_simple.h
diff --git a/functorch/functorch/csrc/init.cpp b/functorch/csrc/init.cpp
similarity index 100%
rename from functorch/functorch/csrc/init.cpp
rename to functorch/csrc/init.cpp
diff --git a/functorch/functorch/dim/README.md b/functorch/dim/README.md
similarity index 100%
rename from functorch/functorch/dim/README.md
rename to functorch/dim/README.md
diff --git a/functorch/functorch/dim/__init__.py b/functorch/dim/__init__.py
similarity index 100%
rename from functorch/functorch/dim/__init__.py
rename to functorch/dim/__init__.py
diff --git a/functorch/functorch/dim/batch_tensor.py b/functorch/dim/batch_tensor.py
similarity index 100%
rename from functorch/functorch/dim/batch_tensor.py
rename to functorch/dim/batch_tensor.py
diff --git a/functorch/functorch/dim/delayed_mul_tensor.py b/functorch/dim/delayed_mul_tensor.py
similarity index 100%
rename from functorch/functorch/dim/delayed_mul_tensor.py
rename to functorch/dim/delayed_mul_tensor.py
diff --git a/functorch/functorch/dim/dim.py b/functorch/dim/dim.py
similarity index 100%
rename from functorch/functorch/dim/dim.py
rename to functorch/dim/dim.py
diff --git a/functorch/functorch/dim/magic_trace.py b/functorch/dim/magic_trace.py
similarity index 100%
rename from functorch/functorch/dim/magic_trace.py
rename to functorch/dim/magic_trace.py
diff --git a/functorch/functorch/dim/op_properties.py b/functorch/dim/op_properties.py
similarity index 100%
rename from functorch/functorch/dim/op_properties.py
rename to functorch/dim/op_properties.py
diff --git a/functorch/functorch/dim/reference.py b/functorch/dim/reference.py
similarity index 100%
rename from functorch/functorch/dim/reference.py
rename to functorch/dim/reference.py
diff --git a/functorch/functorch/dim/tree_map.py b/functorch/dim/tree_map.py
similarity index 100%
rename from functorch/functorch/dim/tree_map.py
rename to functorch/dim/tree_map.py
diff --git a/functorch/functorch/dim/wrap_type.py b/functorch/dim/wrap_type.py
similarity index 100%
rename from functorch/functorch/dim/wrap_type.py
rename to functorch/dim/wrap_type.py
diff --git a/functorch/functorch/experimental/__init__.py b/functorch/experimental/__init__.py
similarity index 100%
rename from functorch/functorch/experimental/__init__.py
rename to functorch/experimental/__init__.py
diff --git a/functorch/functorch/experimental/batch_norm_replacement.py b/functorch/experimental/batch_norm_replacement.py
similarity index 100%
rename from functorch/functorch/experimental/batch_norm_replacement.py
rename to functorch/experimental/batch_norm_replacement.py
diff --git a/functorch/functorch/experimental/cond.py b/functorch/experimental/cond.py
similarity index 100%
rename from functorch/functorch/experimental/cond.py
rename to functorch/experimental/cond.py
diff --git a/functorch/functorch/experimental/ops.py b/functorch/experimental/ops.py
similarity index 100%
rename from functorch/functorch/experimental/ops.py
rename to functorch/experimental/ops.py
diff --git a/functorch/setup.cfg b/functorch/setup.cfg
deleted file mode 100644
index c2f3b448a6c2..000000000000
--- a/functorch/setup.cfg
+++ /dev/null
@@ -1,18 +0,0 @@
-[bdist_wheel]
-universal=1
-
-[metadata]
-license_file = LICENSE
-
-[pep8]
-max-line-length = 120
-
-[flake8]
-max-line-length = 120
-exclude = docs, benchmarks, notebooks, tools
-per-file-ignores =
-    __init__.py: F401
-    functorch/_src/decompositions.py: E501
-
-[pydocstyle]
-select = D417 # Missing argument descriptions in the docstring
diff --git a/functorch/setup.py b/functorch/setup.py
index 3f56c078bca0..a8608989383c 100644
--- a/functorch/setup.py
+++ b/functorch/setup.py
@@ -3,18 +3,11 @@
 #
 # This source code is licensed under the BSD-style license found in the
 # LICENSE file in the root directory of this source tree.
+# This is a dummy setup.py that does not do anything
 
-import distutils.command.clean
-import sys
-import shutil
-import glob
 import os
 import subprocess
-from setuptools import setup, find_packages
-from torch.utils.cpp_extension import (
-    CppExtension,
-    BuildExtension,
-)
+from setuptools import setup
 
 cwd = os.path.dirname(os.path.abspath(__file__))
 version_txt = os.path.join(cwd, 'version.txt')
@@ -33,16 +26,6 @@ elif sha != 'Unknown':
     version += '+' + sha[:7]
 
 
-def write_version_file():
-    version_path = os.path.join(cwd, 'functorch', 'version.py')
-    with open(version_path, 'w') as f:
-        f.write("__version__ = '{}'\n".format(version))
-        f.write("git_version = {}\n".format(repr(sha)))
-
-
-# pytorch_dep = 'torch'
-# if os.getenv('PYTORCH_VERSION'):
-#     pytorch_dep += "==" + os.getenv('PYTORCH_VERSION')
 requirements = [
     # This represents a nightly version of PyTorch.
     # It can be installed as a binary or from source.
@@ -53,83 +36,7 @@ extras = {}
 extras["aot"] = ["networkx", ]
 
 
-class clean(distutils.command.clean.clean):
-    def run(self):
-
-        with open(".gitignore", "r") as f:
-            ignores = f.read()
-            for wildcard in filter(None, ignores.split("\n")):
-                for filename in glob.glob(wildcard):
-                    try:
-                        os.remove(filename)
-                    except OSError:
-                        shutil.rmtree(filename, ignore_errors=True)
-
-        # It's an old-style class in Python 2.7...
-        distutils.command.clean.clean.run(self)
-
-
-def get_extensions():
-    extension = CppExtension
-
-    # See functorch/csrc/Macros.h
-    define_macros = [('FUNCTORCH_BUILD_MAIN_LIB', None)]
-
-    extra_link_args = []
-    extra_compile_args = {"cxx": [
-        "-O3",
-        "-std=c++14",
-        "-fdiagnostics-color=always",
-    ]}
-    debug_mode = os.getenv('DEBUG', '0') == '1'
-    if debug_mode:
-        print("Compiling in debug mode")
-        extra_compile_args = {
-            "cxx": [
-                "-O0",
-                "-fno-inline",
-                "-g",
-                "-std=c++14",
-                "-fdiagnostics-color=always",
-            ]}
-        extra_link_args = ["-O0", "-g"]
-
-    this_dir = os.path.dirname(os.path.abspath(__file__))
-    extensions_dir = os.path.join(this_dir, "functorch", "csrc")
-
-    extension_sources = set(
-        os.path.join(extensions_dir, p)
-        for p in glob.glob(os.path.join(extensions_dir, "*.cpp"))
-    )
-    sources = list(extension_sources)
-    sources.append(os.path.join(extensions_dir, "dim", "dim.cpp"))
-
-    ext_modules = [
-        extension(
-            "functorch._C",
-            sources,
-            include_dirs=[this_dir],
-            define_macros=define_macros,
-            extra_compile_args=extra_compile_args,
-            extra_link_args=extra_link_args,
-        )
-    ]
-
-    return ext_modules
-
-
-class BuildExtension_(BuildExtension):
-    def build_extensions(self, *args, **kwargs):
-        # It turns out for windows this isn't populated?
-        if hasattr(self.compiler, 'compiler_so'):
-            if '-Wstrict-prototypes' in self.compiler.compiler_so:
-                self.compiler.compiler_so.remove('-Wstrict-prototypes')
-        super().build_extensions(*args, **kwargs)
-
-
 if __name__ == '__main__':
-    print("Building wheel {}-{}".format(package_name, version))
-    write_version_file()
     try:
         setup(
             # Metadata
@@ -141,14 +48,10 @@ if __name__ == '__main__':
             license='BSD',
 
             # Package info
-            packages=find_packages(),
+            packages=[],
             install_requires=requirements,
             extras_require=extras,
-            ext_modules=get_extensions(),
-            cmdclass={
-                "build_ext": BuildExtension_.with_options(no_python_abi_suffix=True),
-                'clean': clean,
-            })
+        )
     except Exception as e:
         print(e, file=sys.stderr)
         sys.exit(1)
diff --git a/setup.py b/setup.py
index 3b5efe4dc12d..bbb15d5a46d2 100644
--- a/setup.py
+++ b/setup.py
@@ -614,6 +614,23 @@ class build_ext(setuptools.command.build_ext.build_ext):
                     os.makedirs(dst_dir)
                 self.copy_file(src, dst)
                 i += 1
+
+        # Copy functorch extension
+        for i, ext in enumerate(self.extensions):
+            if ext.name != "functorch._C":
+                continue
+            fullname = self.get_ext_fullname(ext.name)
+            filename = self.get_ext_filename(fullname)
+            fileext = os.path.splitext(filename)[1]
+            src = os.path.join(os.path.dirname(filename), "functorch" + fileext)
+            dst = os.path.join(os.path.realpath(self.build_lib), filename)
+            if os.path.exists(src):
+                report("Copying {} from {} to {}".format(ext.name, src, dst))
+                dst_dir = os.path.dirname(dst)
+                if not os.path.exists(dst_dir):
+                    os.makedirs(dst_dir)
+                self.copy_file(src, dst)
+
         setuptools.command.build_ext.build_ext.build_extensions(self)
 
 
@@ -893,6 +910,12 @@ def configure_extension_build():
                     name=str('caffe2.python.caffe2_pybind11_state_hip'),
                     sources=[]),
             )
+    if cmake_cache_vars['BUILD_FUNCTORCH']:
+        extensions.append(
+            Extension(
+                name=str('functorch._C'),
+                sources=[]),
+        )
 
     cmdclass = {
         'bdist_wheel': wheel_concatenate,
diff --git a/test/test_fx_reinplace_pass.py b/test/test_fx_reinplace_pass.py
index c508d89ec703..abb9696225c4 100644
--- a/test/test_fx_reinplace_pass.py
+++ b/test/test_fx_reinplace_pass.py
@@ -143,19 +143,19 @@ def forward(self, a__1):
 
 
 def forward(self, a__1):
-    clone_default = torch.ops.aten.clone.default(a__1);  a__1 = None
-    view_default = torch.ops.aten.view.default(clone_default, [-1])
-    view_default_1 = torch.ops.aten.view.default(clone_default, [-1])
-    select_int = torch.ops.aten.select.int(view_default_1, 0, 0);  view_default_1 = None
-    view_default_2 = torch.ops.aten.view.default(select_int, [-1]);  select_int = None
-    add_tensor = torch.ops.aten.add_.Tensor(view_default_2, 1)
-    view_default_3 = torch.ops.aten.view.default(clone_default, [-1]);  clone_default = None
-    select_int_1 = torch.ops.aten.select.int(view_default_3, 0, 0)
-    view_default_4 = torch.ops.aten.view.default(view_default_2, []);  view_default_2 = None
-    view_default_5 = torch.ops.aten.view.default(view_default_3, [4]);  view_default_3 = None
-    view_default_6 = torch.ops.aten.view.default(view_default_5, [-1])
-    add_tensor_1 = torch.ops.aten.add_.Tensor(view_default_5, view_default_6);  view_default_6 = None
-    return view_default_5
+    clone = torch.ops.aten.clone.default(a__1);  a__1 = None
+    view = torch.ops.aten.view.default(clone, [-1])
+    view_1 = torch.ops.aten.view.default(clone, [-1])
+    select = torch.ops.aten.select.int(view_1, 0, 0);  view_1 = None
+    view_2 = torch.ops.aten.view.default(select, [-1]);  select = None
+    add = torch.ops.aten.add_.Tensor(view_2, 1)
+    view_3 = torch.ops.aten.view.default(clone, [-1]);  clone = None
+    select_1 = torch.ops.aten.select.int(view_3, 0, 0)
+    view_4 = torch.ops.aten.view.default(view_2, []);  view_2 = None
+    view_5 = torch.ops.aten.view.default(view_3, [4]);  view_3 = None
+    view_6 = torch.ops.aten.view.default(view_5, [-1])
+    add_1 = torch.ops.aten.add_.Tensor(view_5, view_6);  view_6 = None
+    return view_5
     """)
 
     def test_reinplace_scatter_twice(self):
@@ -180,14 +180,14 @@ def forward(self, a__1):
 
 
 def forward(self, a__1):
-    clone_default = torch.ops.aten.clone.default(a__1);  a__1 = None
-    slice_tensor = torch.ops.aten.slice.Tensor(clone_default, 0, 0, 9223372036854775807)
-    select_int = torch.ops.aten.select.int(slice_tensor, 1, 1);  slice_tensor = None
-    select_int_1 = torch.ops.aten.select.int(select_int, 0, 1);  select_int = None
-    add_tensor = torch.ops.aten.add_.Tensor(select_int_1, 1);  select_int_1 = None
-    slice_tensor_1 = torch.ops.aten.slice.Tensor(clone_default, 0, 0, 9223372036854775807)
-    select_int_2 = torch.ops.aten.select.int(slice_tensor_1, 1, 1);  slice_tensor_1 = None
-    return clone_default
+    clone = torch.ops.aten.clone.default(a__1);  a__1 = None
+    slice_1 = torch.ops.aten.slice.Tensor(clone, 0, 0, 9223372036854775807)
+    select = torch.ops.aten.select.int(slice_1, 1, 1);  slice_1 = None
+    select_1 = torch.ops.aten.select.int(select, 0, 1);  select = None
+    add = torch.ops.aten.add_.Tensor(select_1, 1);  select_1 = None
+    slice_2 = torch.ops.aten.slice.Tensor(clone, 0, 0, 9223372036854775807)
+    select_2 = torch.ops.aten.select.int(slice_2, 1, 1);  slice_2 = None
+    return clone
     """)
 
     def test_reinplace_scatter_twice_with_different_view_op_valid(self):
@@ -319,8 +319,8 @@ def forward(self, a__1):
 
 def forward(self):
     zeros = torch.ops.aten.zeros.default([2, 2], device = device(type='cpu'), pin_memory = False)
-    diagonal_default = torch.ops.aten.diagonal.default(zeros)
-    add_tensor = torch.ops.aten.add_.Tensor(diagonal_default, 1);  diagonal_default = None
+    diagonal = torch.ops.aten.diagonal.default(zeros)
+    add = torch.ops.aten.add_.Tensor(diagonal, 1);  diagonal = None
     return [zeros]
     """)
 
@@ -343,11 +343,11 @@ def forward(self):
 def forward(self):
     zeros = torch.ops.aten.zeros.default([4, 4, 4], device = device(type='cpu'), pin_memory = False)
     ones = torch.ops.aten.ones.default([4, 2, 4], device = device(type='cpu'), pin_memory = False)
-    slice_tensor = torch.ops.aten.slice.Tensor(zeros, 0, 0, 9223372036854775807)
-    slice_tensor_1 = torch.ops.aten.slice.Tensor(slice_tensor, 1, 2, 9223372036854775807);  slice_tensor = None
-    slice_tensor_2 = torch.ops.aten.slice.Tensor(zeros, 0, 0, 9223372036854775807)
-    slice_tensor_3 = torch.ops.aten.slice.Tensor(slice_tensor_2, 1, 2, 9223372036854775807);  slice_tensor_2 = None
-    copy__default = torch.ops.aten.copy_.default(slice_tensor_3, ones);  slice_tensor_3 = ones = None
+    slice_1 = torch.ops.aten.slice.Tensor(zeros, 0, 0, 9223372036854775807)
+    slice_2 = torch.ops.aten.slice.Tensor(slice_1, 1, 2, 9223372036854775807);  slice_1 = None
+    slice_3 = torch.ops.aten.slice.Tensor(zeros, 0, 0, 9223372036854775807)
+    slice_tensor = torch.ops.aten.slice.Tensor(slice_3, 1, 2, 9223372036854775807);  slice_3 = None
+    copy__default = torch.ops.aten.copy_.default(slice_tensor, ones);  slice_tensor = ones = None
     return zeros
     """)
 
diff --git a/torch/CMakeLists.txt b/torch/CMakeLists.txt
index 725d4284ab65..7398ea3e5aaf 100644
--- a/torch/CMakeLists.txt
+++ b/torch/CMakeLists.txt
@@ -496,3 +496,6 @@ if(NOT ${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
   # Pybind11 requires explicit linking of the torch_python library
   target_link_libraries(nnapi_backend PRIVATE torch torch_python pybind::pybind11)
 endif()
+
+set(TORCH_PYTHON_COMPILE_OPTIONS ${TORCH_PYTHON_COMPILE_OPTIONS} PARENT_SCOPE)
+set(TORCH_PYTHON_LINK_FLAGS ${TORCH_PYTHON_LINK_FLAGS} PARENT_SCOPE)