Set default compiler differently according to platform (#43890)

Summary:
Pull Request resolved: https://github.com/pytorch/pytorch/pull/43890

1. auto-detect `CXX` default compiler type in oss, and `clang` as default compiler type in fbcode (because auto-detecting will say `gcc` is the default compiler on devserver).

2. change `compiler type` from str `"CLANG" "GCC"` to enum type
3. rename function `get_cov_type` to `detect_compiler_type`
4. auto-set the default pytorch folder for users in oss

Test Plan:
on devserver:
```
buck run :coverage //caffe2/c10:
```

on oss:
```
python oss_coverage.py --run-only=atest
```

Reviewed By: malfet

Differential Revision: D23420034

fbshipit-source-id: c0ea88188578bb1343a286f2090eb8a74cdf3982

tools/code_coverage/README.md

@@ -38,11 +38,11 @@ cmake .. -DCODE_COVERAGE=ON -DBUILD_TEST=ON -DCMAKE_BUILD_CONFIG=Debug
 
 
 ## Examples
-The default setting is for `gcc`. If you are using `clang`, the first step is to set some environment value if needed:
+The tool will auto-detect compiler type in your operating system, but if you are using another one, you need to specify it. Besides, if you are using `clang`, `llvm` tools are required. So the first step is to set some environment value if needed:
 ```bash
-# set compiler type, the default is "GCC"
+# set compiler type, the default is auto detected, you can check it at the start of log.txt
 export COMPILER_TYPE="CLANG"
-# set llvm path, by default is /usr/local/opt/llvm/bin
+# set llvm path for clang, by default is /usr/local/opt/llvm/bin
 export LLVM_TOOL_PATH=...
 ```
 

tools/code_coverage/package/oss/cov_json.py

@@ -1,18 +1,17 @@
 import time
 
 from ..tool import clang_coverage, gcc_coverage
-from ..util.setting import Option, TestList, TestPlatform
-from ..util.utils import check_compiler_type, get_cov_type, print_time
-from .init import gcc_export_init
+from ..util.setting import CompilerType, Option, TestList, TestPlatform
+from ..util.utils import check_compiler_type, print_time
+from .init import detect_compiler_type, gcc_export_init
 from .run import clang_run, gcc_run
 
 
 def get_json_report(test_list: TestList, options: Option):
     start_time = time.time()
-    cov_type = get_cov_type()
-    # TODO change to enum
+    cov_type = detect_compiler_type()
     check_compiler_type(cov_type)
-    if cov_type == "CLANG":
+    if cov_type == CompilerType.CLANG:
         # run
         if options.need_run:
             clang_run(test_list)
@@ -21,7 +20,7 @@ def get_json_report(test_list: TestList, options: Option):
             clang_coverage.merge(test_list, TestPlatform.OSS)
         if options.need_export:
             clang_coverage.export(test_list, TestPlatform.OSS)
-    elif cov_type == "GCC":
+    elif cov_type == CompilerType.GCC:
         # run
         if options.need_run:
             gcc_run(test_list)

tools/code_coverage/package/oss/init.py

@@ -13,7 +13,6 @@ from ..util.setting import (
 from ..util.utils import (
     clean_up,
     create_folder,
-    get_cov_type,
     print_log,
     raise_no_test_found_exception,
     remove_file,
@@ -22,6 +21,7 @@ from ..util.utils import (
 from ..util.utils_init import add_arguments_utils, create_folders, get_options
 from .utils import (
     clean_up_gcda,
+    detect_compiler_type,
     get_llvm_tool_path,
     get_oss_binary_folder,
     get_pytorch_folder,
@@ -128,7 +128,7 @@ def print_init_info() -> None:
     print_log("pytorch folder: ", get_pytorch_folder())
     print_log("cpp test binaries folder: ", get_oss_binary_folder(TestType.CPP))
     print_log("python test scripts folder: ", get_oss_binary_folder(TestType.PY))
-    print_log("cov_type: ", get_cov_type())
+    print_log("compiler type: ", detect_compiler_type().value)
     print_log(
         "llvm tool folder (only for clang, if you are using gcov please ignore it): ",
         get_llvm_tool_path(),

tools/code_coverage/package/oss/utils.py

@@ -1,9 +1,9 @@
 import os
 import subprocess
-from typing import List
+from typing import List, Optional
 
-from ..util.setting import SCRIPT_FOLDER, TestType
-from ..util.utils import print_error, remove_file
+from ..util.setting import TOOLS_FOLDER, CompilerType, TestType
+from ..util.utils import check_compiler_type, print_error, remove_file
 
 
 def get_oss_binary_folder(test_type: TestType) -> str:
@@ -40,7 +40,34 @@ def get_llvm_tool_path() -> str:
 
 
 def get_pytorch_folder() -> str:
-    return os.environ.get("PYTORCH_FOLDER", SCRIPT_FOLDER)
+    # TOOLS_FOLDER in oss: pytorch/tools/code_coverage
+    return os.path.abspath(
+        os.environ.get(
+            "PYTORCH_FOLDER", os.path.join(TOOLS_FOLDER, os.path.pardir, os.path.pardir)
+        )
+    )
+
+
+def detect_compiler_type() -> Optional[CompilerType]:
+    # check if user specifies the compiler type
+    user_specify = os.environ.get("CXX", None)
+    if user_specify:
+        if user_specify in ["clang", "clang++"]:
+            return CompilerType.CLANG
+        elif user_specify in ["gcc", "g++"]:
+            return CompilerType.GCC
+
+        raise RuntimeError(f"User specified compiler is not valid {user_specify}")
+
+    # auto detect
+    auto_detect_result = subprocess.check_output(
+        ["cc", "-v"], stderr=subprocess.STDOUT
+    ).decode("utf-8")
+    if "clang" in auto_detect_result:
+        return CompilerType.CLANG
+    elif "gcc" in auto_detect_result:
+        return CompilerType.GCC
+    raise RuntimeError(f"Auto detected compiler is not valid {auto_detect_result}")
 
 
 def clean_up_gcda() -> None:

tools/code_coverage/package/tool/summarize_jsons.py

@@ -3,10 +3,16 @@ import os
 import time
 from typing import Any, Dict, List, Set, Tuple
 
-from ..util.setting import JSON_FOLDER_BASE_DIR, TestList, TestPlatform, TestStatusType
+from ..util.setting import (
+    JSON_FOLDER_BASE_DIR,
+    CompilerType,
+    TestList,
+    TestPlatform,
+    TestStatusType,
+)
 from ..util.utils import (
     check_compiler_type,
-    get_cov_type,
+    detect_compiler_type,
     print_error,
     print_time,
     related_to_test_list,
@@ -93,7 +99,7 @@ def get_json_obj(json_file: str) -> Tuple[Any, int]:
     return None, 2
 
 
-def parse_json(json_file: str) -> List[CoverageRecord]:
+def parse_json(json_file: str, platform: TestPlatform) -> List[CoverageRecord]:
     print("start parse:", json_file)
     json_obj, read_status = get_json_obj(json_file)
     if read_status == 0:
@@ -105,13 +111,14 @@ def parse_json(json_file: str) -> List[CoverageRecord]:
         raise RuntimeError(
             "Fail to do code coverage! Fail to load json file: ", json_file
         )
-    cov_type = get_cov_type()
-    check_compiler_type(cov_type)
+
+    cov_type = detect_compiler_type(platform)
+
     coverage_records: List[CoverageRecord] = []
-    if cov_type == "CLANG":
+    if cov_type == CompilerType.CLANG:
         coverage_records = LlvmCoverageParser(json_obj).parse("fbcode")
         # print(coverage_records)
-    elif cov_type == "GCC":
+    elif cov_type == CompilerType.GCC:
         coverage_records = GcovCoverageParser(json_obj).parse()
 
     return coverage_records
@@ -126,13 +133,14 @@ def parse_jsons(
         for file_name in file_list:
             if file_name.endswith(".json"):
                 # if compiler is clang, we only analyze related json / when compiler is gcc, we analyze all jsons
-                if get_cov_type() == "CLANG" and not related_to_test_list(
+                cov_type = detect_compiler_type(platform)
+                if cov_type == CompilerType.CLANG and not related_to_test_list(
                     file_name, test_list
                 ):
                     continue
                 json_file = os.path.join(path, file_name)
                 try:
-                    coverage_records = parse_json(json_file)
+                    coverage_records = parse_json(json_file, platform)
                 except RuntimeError:
                     print_error("Fail to load json file: ", json_file)
                     continue

tools/code_coverage/package/util/setting.py

@@ -5,14 +5,13 @@ from typing import Dict, List, Set
 
 # <project folder>
 HOME_DIR = os.environ["HOME"]
-setting_file_path = os.path.realpath(__file__)
-SCRIPT_FOLDER = os.path.join(
-    os.path.dirname(setting_file_path), os.path.pardir, os.path.pardir
+TOOLS_FOLDER = os.path.join(
+    os.path.dirname(os.path.realpath(__file__)), os.path.pardir, os.path.pardir
 )
 
 
 # <profile folder>
-PROFILE_DIR = os.path.join(SCRIPT_FOLDER, "profile")
+PROFILE_DIR = os.path.join(TOOLS_FOLDER, "profile")
 JSON_FOLDER_BASE_DIR = os.path.join(PROFILE_DIR, "json")
 MERGED_FOLDER_BASE_DIR = os.path.join(PROFILE_DIR, "merged")
 SUMMARY_FOLDER_DIR = os.path.join(PROFILE_DIR, "summary")
@@ -60,3 +59,9 @@ class Option:
 class TestPlatform(Enum):
     FBCODE: str = "fbcode"
     OSS: str = "oss"
+
+
+# compiler type
+class CompilerType(Enum):
+    CLANG: str = "clang"
+    GCC: str = "gcc"

tools/code_coverage/package/util/utils.py

@@ -2,9 +2,16 @@ import os
 import shutil
 import sys
 import time
-from typing import Any
+from typing import Any, Optional
 
-from .setting import LOG_DIR, PROFILE_DIR, TestList, TestPlatform, TestType
+from .setting import (
+    LOG_DIR,
+    PROFILE_DIR,
+    CompilerType,
+    TestList,
+    TestPlatform,
+    TestType,
+)
 
 
 def convert_time(seconds: float) -> str:
@@ -80,9 +87,20 @@ def get_raw_profiles_folder() -> str:
     return os.environ.get("RAW_PROFILES_FOLDER", os.path.join(PROFILE_DIR, "raw"))
 
 
-# TODO auto detect
-def get_cov_type() -> str:
-    return os.environ.get("COMPILER_TYPE", "CLANG")
+def detect_compiler_type(platform: TestPlatform) -> CompilerType:
+    if platform == TestPlatform.OSS:
+        from package.oss.utils import detect_compiler_type
+
+        cov_type = detect_compiler_type()
+    else:
+        from caffe2.fb.code_coverage.tool.package.fbcode.utils import (
+            detect_compiler_type,
+        )
+
+        cov_type = detect_compiler_type()
+
+    check_compiler_type(cov_type)
+    return cov_type
 
 
 def get_test_name_from_whole_path(path: str) -> str:
@@ -93,8 +111,8 @@ def get_test_name_from_whole_path(path: str) -> str:
     return path[start + 1 : end]
 
 
-def check_compiler_type(cov_type: str) -> None:
-    if cov_type in ["CLANG", "GCC"]:
+def check_compiler_type(cov_type: Optional[CompilerType]) -> None:
+    if cov_type is not None and cov_type in [CompilerType.GCC, CompilerType.CLANG]:
         return
     raise Exception(
         f"Can't parse compiler type: {cov_type}.",