diff --git a/setup.py b/setup.py
index 668892b49be3..d8d5e230c455 100644
--- a/setup.py
+++ b/setup.py
@@ -1387,15 +1387,9 @@ def main():
             ]
         )
     torchgen_package_data = [
-        # Recursive glob doesn't work in setup.py,
-        # https://github.com/pypa/setuptools/issues/1806
-        # To make this robust we should replace it with some code that
-        # returns a list of everything under packaged/
-        "packaged/ATen/*",
-        "packaged/ATen/native/*",
-        "packaged/ATen/templates/*",
-        "packaged/autograd/*",
-        "packaged/autograd/templates/*",
+        "packaged/**/*.cpp",
+        "packaged/**/*.h",
+        "packaged/**/*.yaml",
     ]
     setup(
         name=package_name,