From 759bb70acea8486b3a493792a5b6ba80b5217fb4 Mon Sep 17 00:00:00 2001 From: Benjamin Bossan Date: Wed, 11 Jun 2025 18:01:13 +0200 Subject: [PATCH] FIX: Generation nightly CI failing due to gemma (#2580) For a month now, nightly CI has failed with dozens of tests causing this error: > RuntimeError: Offset increment outside graph capture encountered unexpectedly. (link: https://github.com/huggingface/peft/actions/runs/14850392078/job/41692748031) It turns out that https://github.com/huggingface/peft/pull/2458, which added a gemma model to the test suite, is most likely the culprit. Since that commit, on nightly CI (with GPU), when transformers generates with gemma, which uses torch.compile, an error can be triggered. For some reason, this has a side effect on other tests that then results in the error quoted above. As is, there is no solution for the gemma issue. To still allow the tests to run and help discover potential issues, this PR skips the corresponding gemma tests, which should allow the other tests to pass again. I could confirm locally that these tests only fail when the gemma tests are run in the same session. Hopefully, this generalizes to the CI environment. --------- Co-authored-by: githubnemo --- Makefile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 0d93db32..70ba4b7f 100644 --- a/Makefile +++ b/Makefile @@ -31,8 +31,12 @@ tests_core_multi_gpu: tests_core_single_gpu: python -m pytest -m single_gpu_tests tests/test_common_gpu.py $(if $(IS_GITHUB_CI),--report-log "core_single_gpu.log",) +# exclude gemma tests, as generation fails with torch.compile, these failures +# trigger side effects that make other tests fail with 'RuntimeError: Offset +# increment outside graph capture encountered unexpectedly.' +# TODO re-enable gemma once/if it is fixed tests_common_gpu: - python -m pytest tests/test_decoder_models.py $(if $(IS_GITHUB_CI),--report-log "common_decoder.log",) + python -m pytest tests/test_decoder_models.py -k "not gemma" $(if $(IS_GITHUB_CI),--report-log "common_decoder.log",) python -m pytest tests/test_encoder_decoder_models.py $(if $(IS_GITHUB_CI),--report-log "common_encoder_decoder.log",) python -m pytest tests/test_gptqmodel.py $(if $(IS_GITHUB_CI),--report-log "gptqmodel_gpu.log",)