From abcec555322e3b768f01570986226487375aefa8 Mon Sep 17 00:00:00 2001
From: cat-state <98283595+cat-state@users.noreply.github.com>
Date: Tue, 11 Mar 2025 19:49:24 +0000
Subject: [PATCH] gracefully handle `tokenize.TokenError` in funcname parser.
 Adds support for non-Python source (#148737)

This change allows defining python functions in non-python source and having them be able to compiled by torch.compile. The existing implementation already returns None for the case where the file couldn't be read, so returning None (by making an empty funcname cache) makes sense for the case of non-python source code too.

Example [basilisp](https://github.com/basilisp-lang/basilisp):
```clojure
(import torch)
(import [torch.nn.functional :as F])
(torch/rand 10)

(defn f {:decorators [torch/compile]} [x]
  (* (F/relu x) x))

(f (-> (torch/randn 100)
       (.cuda)))
```

Pull Request resolved: https://github.com/pytorch/pytorch/pull/148737
Approved by: https://github.com/williamwen42
---
 torch/_dynamo/funcname_cache.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/torch/_dynamo/funcname_cache.py b/torch/_dynamo/funcname_cache.py
index 51b0c86105d5..f71cb5c6b02a 100644
--- a/torch/_dynamo/funcname_cache.py
+++ b/torch/_dynamo/funcname_cache.py
@@ -31,7 +31,7 @@ def _add_file(filename: str) -> None:
     try:
         with tokenize.open(filename) as f:
             tokens = list(tokenize.generate_tokens(f.readline))
-    except OSError:
+    except (OSError, tokenize.TokenError):
         cache[filename] = {}
         return