frozenleaves/pytorch
1
0
Fork 0
mirror of https://github.com/pytorch/pytorch.git synced 2025-10-20 21:14:14 +08:00
Code Issues Packages Projects Releases Wiki Activity

[ROCm] Fix circular recursion issue in hipification (#104085)

Browse Source 
This PR fixes the circular issue during hipification process by introducing current_state to track whether a file is processed for hipification. (Iterative DFS)
The issue arises when two header files try to include themselves, which leads to a circular recursion or an infinite loop.

Fixes the related issues such as :
https://github.com/pytorch/pytorch/issues/93827
https://github.com/ROCmSoftwarePlatform/hipify_torch/issues/39

Error log:
```
  File "/opt/conda/lib/python3.8/posixpath.py", line 471, in relpath
    start_list = [x for x in abspath(start).split(sep) if x]
  File "/opt/conda/lib/python3.8/posixpath.py", line 375, in abspath
    if not isabs(path):
  File "/opt/conda/lib/python3.8/posixpath.py", line 63, in isabs
    sep = _get_sep(s)
  File "/opt/conda/lib/python3.8/posixpath.py", line 42, in _get_sep
    if isinstance(path, bytes):
RecursionError: maximum recursion depth exceeded while calling a Python object
```

Pull Request resolved: https://github.com/pytorch/pytorch/pull/104085
Approved by: https://github.com/jithunnair-amd, https://github.com/malfet
This commit is contained in:
lcskrishna
2023-07-01 03:25:51 +00:00
committed by PyTorch MergeBot
parent e865bc7da4
commit 004ff536e8
2 changed files with 67 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

80
torch/utils/hipify/hipify_python.py
View File

@ -36,7 +36,22 @@ from .cuda_to_hip_mappings import MATH_TRANSPILATIONS
from typing import Dict, List, Iterator, Optional
from collections.abc import Mapping, Iterable
HipifyResult = Dict[str, Optional[str]]
from enum import Enum
class CurrentState(Enum):
INITIALIZED = 1
DONE = 2
class HipifyResult:
def __init__(self, current_state, hipified_path):
self.current_state = current_state
self.hipified_path = hipified_path
self.status = ""
def __str__(self):
return ("HipifyResult:: current_state: {}, hipified_path : {}, status: {}".format(self.current_state,
self.hipified_path, self.status))
HipifyFinalResult = Dict[str, HipifyResult]
HIPIFY_C_BREADCRUMB = "// !!! This is a file automatically generated by hipify!!!\n"
HIPIFY_FINAL_RESULT: HipifyFinalResult = {}
@ -51,7 +66,7 @@ __all__ = ['InputError', 'openf', 'bcolors', 'GeneratedFileCleaner', 'match_exte
'find_bracket_group', 'find_parentheses_group', 'replace_math_functions', 'hip_header_magic', 'replace_extern_shared',
'get_hip_file_path', 'is_out_of_place', 'is_pytorch_file', 'is_cusparse_file', 'is_special_file', 'is_caffe2_gpu_file',
'is_caffe2_gpu_file', 'Trie', 'preprocessor', 'file_specific_replacement', 'file_add_header',
'fix_static_global_kernels', 'extract_arguments', 'str2bool', 'hipify']
'fix_static_global_kernels', 'extract_arguments', 'str2bool', 'CurrentState', 'HipifyResult', 'hipify']
class InputError(Exception):
@ -185,15 +200,17 @@ def preprocess_file_and_save_result(
is_pytorch_extension: bool,
clean_ctx: GeneratedFileCleaner,
show_progress: bool) -> None:
fin_path = os.path.abspath(os.path.join(output_directory, filepath))
hipify_result = HipifyResult(current_state=CurrentState.INITIALIZED, hipified_path=fin_path)
HIPIFY_FINAL_RESULT[fin_path] = hipify_result
result = preprocessor(output_directory, filepath, all_files, header_include_dirs, stats,
hip_clang_launch, is_pytorch_extension, clean_ctx, show_progress)
fin_path = os.path.abspath(os.path.join(output_directory, filepath))
# Show what happened
if show_progress and "ignored" not in str(result["status"]):
if show_progress and "ignored" not in result.status:
print(
fin_path, "->",
result["hipified_path"], result["status"], flush=True)
result.hipified_path, result.status, flush=True)
HIPIFY_FINAL_RESULT[fin_path] = result
@ -741,11 +758,13 @@ RE_THC_GENERIC_FILE = re.compile(r'#define THC_GENERIC_FILE "([^"]+)"')
RE_CU_SUFFIX = re.compile(r'\.cu\b') # be careful not to pick up .cuh
"""
Returns a dict with the following keys:
Returns a HipifyResult object with the following details:
"hipified_path" : absolute path of hipified source file
"status" : "ok" if hipified file was written out
"skipped" if an identical hipified file already existed or hipified file couldn't be written out
"ignored" if the source file was a hipified file itself or not meant to be hipified
"current_state" : CurrentState.INITIALIZED if source file is first ready to be hipified
CurrentState.DONE if source file is done with hipification process
"""
@ -760,15 +779,22 @@ def preprocessor(
clean_ctx: GeneratedFileCleaner,
show_progress: bool) -> HipifyResult:
""" Executes the CUDA -> HIP conversion on the specified file. """
if filepath not in all_files:
return {"hipified_path": None, "status": "[ignored, not to be hipified]"}
fin_path = os.path.abspath(os.path.join(output_directory, filepath))
hipify_result = HIPIFY_FINAL_RESULT[fin_path]
if filepath not in all_files:
hipify_result.hipified_path = None
hipify_result.status = "[ignored, not to be hipified]"
hipify_result.current_state = CurrentState.DONE
return hipify_result
rel_filepath = os.path.relpath(filepath, output_directory)
with open(fin_path, 'r', encoding='utf-8') as fin:
if fin.readline() == HIPIFY_C_BREADCRUMB:
return {"hipified_path": None, "status": "[ignored, input is hipified output]"}
hipify_result.hipified_path = None
hipify_result.status = "[ignored, input is hipified output]"
hipify_result.current_state = CurrentState.DONE
return hipify_result
fin.seek(0)
output_source = fin.read()
@ -844,7 +870,18 @@ def preprocessor(
header_filepath,
all_files, header_include_dirs, stats, hip_clang_launch,
is_pytorch_extension, clean_ctx, show_progress)
hipified_header_filepath = HIPIFY_FINAL_RESULT[header_filepath]["hipified_path"]
elif header_filepath in HIPIFY_FINAL_RESULT:
header_result = HIPIFY_FINAL_RESULT[header_filepath]
if header_result.current_state == CurrentState.INITIALIZED:
# get_hip_file_path needs a relative path to work correctly
header_rel_path = os.path.relpath(header_filepath, output_directory)
header_fout_path = os.path.abspath(os.path.join(output_directory,
get_hip_file_path(header_rel_path, is_pytorch_extension)))
header_result.hipified_path = header_fout_path
HIPIFY_FINAL_RESULT[header_filepath] = header_result
return templ.format(os.path.relpath(header_fout_path if header_fout_path is not None
else header_filepath, header_dir))
hipified_header_filepath = HIPIFY_FINAL_RESULT[header_filepath].hipified_path
return templ.format(os.path.relpath(hipified_header_filepath if hipified_header_filepath is not None
else header_filepath, header_dir))
@ -881,7 +918,10 @@ def preprocessor(
and orig_output_source == output_source
and os.path.dirname(fin_path) == os.path.dirname(fout_path)
):
return {"hipified_path": fin_path, "status": "[skipped, no changes]"}
hipify_result.hipified_path = fin_path
hipify_result.status = "[skipped, no changes]"
hipify_result.current_state = CurrentState.DONE
return hipify_result
# Add hipify breadcrumb for C-style files to avoid re-hipification
if fin_path != fout_path and match_extensions(fin_path, (".cu", ".cuh", ".c", ".cc", ".cpp", ".h", ".hpp")):
@ -895,14 +935,22 @@ def preprocessor(
try:
with clean_ctx.open(fout_path, 'w', encoding='utf-8') as fout:
fout.write(output_source)
return {"hipified_path": fout_path, "status": "[ok]"}
hipify_result.hipified_path = fout_path
hipify_result.status = "[ok]"
hipify_result.current_state = CurrentState.DONE
return hipify_result
except PermissionError as e:
print(f"{bcolors.WARNING}Failed to save {fout_path} with \"{e.strerror}\", leaving {fin_path} unchanged.{bcolors.ENDC}",
file=sys.stderr)
return {"hipified_path": fin_path, "status": "[skipped, no permissions]"}
hipify_result.hipified_path = fin_path
hipify_result.status = "[skipped, no permissions]"
hipify_result.current_state = CurrentState.DONE
return hipify_result
else:
return {"hipified_path": fout_path, "status": "[skipped, already hipified]"}
hipify_result.hipified_path = fout_path
hipify_result.status = "[skipped, already hipified]"
hipify_result.current_state = CurrentState.DONE
return hipify_result
def file_specific_replacement(filepath, search_string, replace_string, strict=False):
with openf(filepath, "r+") as f: