Mark top 3 slowest tests as slow (#46068)

Summary:
`TCPStoreTest.test_numkeys_delkeys` takes 5+ min (mostly in idle wait for socket timeout)
`TestDataLoader.test_proper_exit` and `TestDataLoaderPersistentWorkers.test_proper_exit` take 2.5 min each
`TestXNNPACKConv1dTransformPass.test_conv1d_with_relu_fc` takes 2 min to finish

Add option to skip reporting test classes that run for less than a second to `print_test_stats.py` and speed up `TestTorchDeviceTypeCUDA.test_matmul_45724_cuda`

Pull Request resolved: https://github.com/pytorch/pytorch/pull/46068

Reviewed By: mruberry

Differential Revision: D24208660

Pulled By: malfet

fbshipit-source-id: 780e0d8be4f0cf69ea28de79e423291a1f3349b7

test/distributed/test_c10d.py

@@ -33,7 +33,7 @@ from torch.testing._internal.common_distributed import MultiProcessTestCase, \
     create_device
 
 from torch.testing._internal.common_utils import TestCase, load_tests, run_tests, \
-    retry_on_connect_failures, ADDRESS_IN_USE, CONNECT_TIMEOUT, TEST_WITH_TSAN
+    retry_on_connect_failures, ADDRESS_IN_USE, CONNECT_TIMEOUT, TEST_WITH_TSAN, slowTest
 
 # load_tests from common_utils is used to automatically filter tests for
 # sharding on sandcastle. This line silences flake warnings
@@ -296,6 +296,8 @@ class TCPStoreTest(TestCase, StoreTestBase):
         self.assertEqual(b"value1", fs.get("key1"))
         self.assertEqual(b"value2", fs.get("key4"))
 
+    # https://github.com/pytorch/pytorch/issues/46064 <- takes 5+ min to finish
+    @slowTest
     def test_numkeys_delkeys(self):
         self._test_numkeys_delkeys(self._create_store())
 

test/print_test_stats.py

@@ -122,7 +122,6 @@ def send_report(reports):
             ),
         },
     )
-    print("Scribe report status: {}".format(r.text))
     r.raise_for_status()
 
 def positive_integer(value):
@@ -131,6 +130,12 @@ def positive_integer(value):
         raise argparse.ArgumentTypeError(f"{value} is not a natural number")
     return parsed
 
+def positive_float(value):
+    parsed = float(value)
+    if parsed <= 0.0:
+        raise argparse.ArgumentTypeError(f"{value} is not a positive rational number")
+    return parsed
+
 if __name__ == '__main__':
     import argparse
     import sys
@@ -145,6 +150,13 @@ if __name__ == '__main__':
         metavar="N",
         help="how many longest tests to show for each class",
     )
+    parser.add_argument(
+        "--class-print-threshold",
+        type=positive_float,
+        default=1.0,
+        metavar="N",
+        help="Minimal total time to warrant class report",
+    )
     parser.add_argument(
         "--longest-of-run",
         type=positive_integer,
@@ -169,7 +181,8 @@ if __name__ == '__main__':
     total_time = 0
     for name in sorted(reports.keys()):
         test_suite = reports[name]
-        test_suite.print_report(args.longest_of_class)
+        if test_suite.total_time >= args.class_print_threshold:
+            test_suite.print_report(args.longest_of_class)
         total_time += test_suite.total_time
         longest_tests.extend(test_suite.test_cases)
     longest_tests = sorted(longest_tests, key=lambda x: x.time)[-args.longest_of_run:]

test/test_dataloader.py

@@ -19,7 +19,7 @@ from torch.utils.data._utils import MP_STATUS_CHECK_INTERVAL
 from torch.utils.data.dataset import random_split
 from torch._utils import ExceptionWrapper
 from torch.testing._internal.common_utils import (TestCase, run_tests, TEST_NUMPY, IS_WINDOWS,
-                                                  IS_PYTORCH_CI, NO_MULTIPROCESSING_SPAWN, skipIfRocm,
+                                                  IS_PYTORCH_CI, NO_MULTIPROCESSING_SPAWN, skipIfRocm, slowTest,
                                                   load_tests, TEST_WITH_ROCM, TEST_WITH_TSAN, IS_SANDCASTLE)
 
 try:
@@ -1530,8 +1530,10 @@ except RuntimeError as e:
                 pin_memory_thread.join(JOIN_TIMEOUT)
                 self.assertFalse(pin_memory_thread.is_alive())
 
+    # Takes 2.5min to finish, see https://github.com/pytorch/pytorch/issues/46065
     @skipIfRocm
     @unittest.skipIf(not HAS_PSUTIL, "psutil not found")
+    @slowTest
     def test_proper_exit(self):
         (r'''There might be ConnectionResetError or leaked semaphore warning '''
          r'''(due to dirty process exit), but they are all safe to ignore''')

test/test_torch.py

@@ -16972,9 +16972,9 @@ scipy_lobpcg  | {:10.2e}  | {:10.2e}  | {:6} | N/A
     @onlyCUDA
     def test_matmul_45724(self, device):
         # https://github.com/pytorch/pytorch/issues/45724
-        a = torch.rand(65537, 22, 64).cuda().half()
-        b = torch.rand(65537, 64, 22).cuda().half()
-        c = torch.full((65537, 22, 22), math.nan, dtype=torch.half, device='cuda')
+        a = torch.rand(65537, 22, 64, device=device, dtype=torch.half)
+        b = torch.rand(65537, 64, 22, device=device, dtype=torch.half)
+        c = torch.full((65537, 22, 22), math.nan, dtype=torch.half, device=device)
         cpu_result = torch.matmul(a.cpu().float(), b.cpu().float()).cuda().half()
         torch.matmul(a, b, out=c)
         self.assertEqual(c, cpu_result)

test/test_xnnpack_integration.py

@@ -6,7 +6,7 @@ from torch.nn import functional as F
 from torch.utils.mobile_optimizer import optimize_for_mobile
 from torch.testing import FileCheck
 import torch.testing._internal.hypothesis_utils as hu
-from torch.testing._internal.common_utils import TestCase, run_tests
+from torch.testing._internal.common_utils import TestCase, run_tests, slowTest
 from hypothesis import given, assume
 from hypothesis import strategies as st
 import io
@@ -1022,6 +1022,8 @@ class TestXNNPACKConv1dTransformPass(TestCase):
                                                                                pattern_count_optimized_map,
                                                                                data_shape)
 
+    # See https://github.com/pytorch/pytorch/issues/46066
+    @slowTest
     def test_conv1d_with_relu_fc(self):
         batch_size_list = range(1, 3)
         input_channels_per_group_list = range(10, 12)

torch/testing/_internal/common_methods_invocations.py

@@ -80,7 +80,7 @@ class OpInfo(object):
 
         # NOTE: if the op is unspecified it is assumed to be under the torch namespace
         if op is None:
-            assert hasattr(torch, self.name)
+            assert hasattr(torch, self.name), f"Can't find torch.{self.name}"
         self.op = op if op else getattr(torch, self.name)
         self.method_variant = getattr(torch.Tensor, name) if hasattr(torch.Tensor, name) else None
         inplace_name = name + "_"