mirror of
https://github.com/pytorch/pytorch.git
synced 2025-10-20 21:14:14 +08:00
6ab9453ea9
Fixes #ISSUE_NUMBER * change hook so that test still gets saved in --sc when fails in test setup (caused an off by 1 error due to setup being called before the logreport hook) * allow reruns for all tests now that --sc is used * increase number of reruns now that --sc is used Pull Request resolved: https://github.com/pytorch/pytorch/pull/100200 Approved by: https://github.com/huydhn
261 lines
9.3 KiB
Python
261 lines
9.3 KiB
Python
from _pytest.junitxml import LogXML, _NodeReporter, bin_xml_escape
|
|
from _pytest.terminal import _get_raw_skip_reason
|
|
from _pytest.stash import StashKey
|
|
from _pytest.reports import TestReport
|
|
from _pytest.config.argparsing import Parser
|
|
from _pytest.config import filename_arg
|
|
from _pytest.config import Config
|
|
from _pytest._code.code import ReprFileLocation
|
|
from _pytest.python import Module
|
|
from typing import Any, List, Union
|
|
from typing import Optional
|
|
from types import MethodType
|
|
import xml.etree.ElementTree as ET
|
|
import functools
|
|
import pytest
|
|
import sys
|
|
|
|
# a lot of this file is copied from _pytest.junitxml and modified to get rerun info
|
|
|
|
xml_key = StashKey["LogXMLReruns"]()
|
|
STEPCURRENT_CACHE_DIR = "cache/stepcurrent"
|
|
|
|
|
|
def pytest_addoption(parser: Parser) -> None:
|
|
group = parser.getgroup("general")
|
|
group.addoption(
|
|
"--scs",
|
|
action="store",
|
|
default=None,
|
|
dest="stepcurrent_skip",
|
|
)
|
|
group.addoption(
|
|
"--sc",
|
|
action="store",
|
|
default=None,
|
|
dest="stepcurrent",
|
|
)
|
|
|
|
parser.addoption("--use-main-module", action='store_true')
|
|
group = parser.getgroup("terminal reporting")
|
|
group.addoption(
|
|
"--junit-xml-reruns",
|
|
action="store",
|
|
dest="xmlpath_reruns",
|
|
metavar="path",
|
|
type=functools.partial(filename_arg, optname="--junit-xml-reruns"),
|
|
default=None,
|
|
help="create junit-xml style report file at given path.",
|
|
)
|
|
group.addoption(
|
|
"--junit-prefix-reruns",
|
|
action="store",
|
|
metavar="str",
|
|
default=None,
|
|
help="prepend prefix to classnames in junit-xml output",
|
|
)
|
|
parser.addini(
|
|
"junit_suite_name_reruns", "Test suite name for JUnit report", default="pytest"
|
|
)
|
|
parser.addini(
|
|
"junit_logging_reruns",
|
|
"Write captured log messages to JUnit report: "
|
|
"one of no|log|system-out|system-err|out-err|all",
|
|
default="no",
|
|
)
|
|
parser.addini(
|
|
"junit_log_passing_tests_reruns",
|
|
"Capture log information for passing tests to JUnit report: ",
|
|
type="bool",
|
|
default=True,
|
|
)
|
|
parser.addini(
|
|
"junit_duration_report_reruns",
|
|
"Duration time to report: one of total|call",
|
|
default="total",
|
|
)
|
|
parser.addini(
|
|
"junit_family_reruns",
|
|
"Emit XML for schema: one of legacy|xunit1|xunit2",
|
|
default="xunit2",
|
|
)
|
|
|
|
|
|
def pytest_configure(config: Config) -> None:
|
|
xmlpath = config.option.xmlpath_reruns
|
|
# Prevent opening xmllog on worker nodes (xdist).
|
|
if xmlpath and not hasattr(config, "workerinput"):
|
|
junit_family = config.getini("junit_family_reruns")
|
|
config.stash[xml_key] = LogXMLReruns(
|
|
xmlpath,
|
|
config.option.junitprefix,
|
|
config.getini("junit_suite_name_reruns"),
|
|
config.getini("junit_logging_reruns"),
|
|
config.getini("junit_duration_report_reruns"),
|
|
junit_family,
|
|
config.getini("junit_log_passing_tests_reruns"),
|
|
)
|
|
config.pluginmanager.register(config.stash[xml_key])
|
|
if config.getoption("stepcurrent_skip"):
|
|
config.option.stepcurrent = config.getoption("stepcurrent_skip")
|
|
if config.getoption("stepcurrent"):
|
|
config.pluginmanager.register(StepcurrentPlugin(config), "stepcurrentplugin")
|
|
|
|
|
|
def pytest_unconfigure(config: Config) -> None:
|
|
xml = config.stash.get(xml_key, None)
|
|
if xml:
|
|
del config.stash[xml_key]
|
|
config.pluginmanager.unregister(xml)
|
|
|
|
|
|
class _NodeReporterReruns(_NodeReporter):
|
|
def _prepare_content(self, content: str, header: str) -> str:
|
|
return content
|
|
|
|
def _write_content(self, report: TestReport, content: str, jheader: str) -> None:
|
|
if content == "":
|
|
return
|
|
tag = ET.Element(jheader)
|
|
tag.text = bin_xml_escape(content)
|
|
self.append(tag)
|
|
|
|
|
|
class LogXMLReruns(LogXML):
|
|
def __init__(self, *args, **kwargs):
|
|
super().__init__(*args, **kwargs)
|
|
|
|
def append_rerun(self, reporter: _NodeReporter, report: TestReport) -> None:
|
|
if hasattr(report, "wasxfail"):
|
|
reporter._add_simple("skipped", "xfail-marked test passes unexpectedly")
|
|
else:
|
|
assert report.longrepr is not None
|
|
reprcrash: Optional[ReprFileLocation] = getattr(
|
|
report.longrepr, "reprcrash", None
|
|
)
|
|
if reprcrash is not None:
|
|
message = reprcrash.message
|
|
else:
|
|
message = str(report.longrepr)
|
|
message = bin_xml_escape(message)
|
|
reporter._add_simple("rerun", message, str(report.longrepr))
|
|
|
|
def pytest_runtest_logreport(self, report: TestReport) -> None:
|
|
super().pytest_runtest_logreport(report)
|
|
if report.outcome == "rerun":
|
|
reporter = self._opentestcase(report)
|
|
self.append_rerun(reporter, report)
|
|
if report.outcome == "skipped":
|
|
if isinstance(report.longrepr, tuple):
|
|
fspath, lineno, reason = report.longrepr
|
|
reason = f"{report.nodeid}: {_get_raw_skip_reason(report)}"
|
|
report.longrepr = (fspath, lineno, reason)
|
|
|
|
def node_reporter(self, report: Union[TestReport, str]) -> _NodeReporterReruns:
|
|
nodeid: Union[str, TestReport] = getattr(report, "nodeid", report)
|
|
# Local hack to handle xdist report order.
|
|
workernode = getattr(report, "node", None)
|
|
|
|
key = nodeid, workernode
|
|
|
|
if key in self.node_reporters:
|
|
# TODO: breaks for --dist=each
|
|
return self.node_reporters[key]
|
|
|
|
reporter = _NodeReporterReruns(nodeid, self)
|
|
|
|
self.node_reporters[key] = reporter
|
|
self.node_reporters_ordered.append(reporter)
|
|
|
|
return reporter
|
|
|
|
|
|
# imitating summary_failures in pytest's terminal.py
|
|
# both hookwrapper and tryfirst to make sure this runs before pytest's
|
|
@pytest.hookimpl(hookwrapper=True, tryfirst=True)
|
|
def pytest_terminal_summary(terminalreporter, exitstatus, config):
|
|
# prints stack traces for reruns
|
|
if terminalreporter.config.option.tbstyle != "no":
|
|
reports = terminalreporter.getreports("rerun")
|
|
if reports:
|
|
terminalreporter.write_sep("=", "RERUNS")
|
|
if terminalreporter.config.option.tbstyle == "line":
|
|
for rep in reports:
|
|
line = terminalreporter._getcrashline(rep)
|
|
terminalreporter.write_line(line)
|
|
else:
|
|
for rep in reports:
|
|
msg = terminalreporter._getfailureheadline(rep)
|
|
terminalreporter.write_sep("_", msg, red=True, bold=True)
|
|
terminalreporter._outrep_summary(rep)
|
|
terminalreporter._handle_teardown_sections(rep.nodeid)
|
|
yield
|
|
|
|
|
|
@pytest.hookimpl(tryfirst=True)
|
|
def pytest_pycollect_makemodule(module_path, path, parent) -> Module:
|
|
if parent.config.getoption("--use-main-module"):
|
|
mod = Module.from_parent(parent, path=module_path)
|
|
mod._getobj = MethodType(lambda x: sys.modules['__main__'], mod)
|
|
return mod
|
|
|
|
|
|
@pytest.hookimpl(hookwrapper=True)
|
|
def pytest_report_teststatus(report, config):
|
|
# Add the test time to the verbose output, unforunately I don't think this
|
|
# includes setup or teardown
|
|
pluggy_result = yield
|
|
if not isinstance(report, pytest.TestReport):
|
|
return
|
|
outcome, letter, verbose = pluggy_result.get_result()
|
|
if verbose:
|
|
pluggy_result.force_result(
|
|
(outcome, letter, f"{verbose} [{report.duration:.4f}s]")
|
|
)
|
|
|
|
|
|
class StepcurrentPlugin:
|
|
# Modified fromo _pytest/stepwise.py in order to save the currently running
|
|
# test instead of the last failed test
|
|
def __init__(self, config: Config) -> None:
|
|
self.config = config
|
|
self.report_status = ""
|
|
assert config.cache is not None
|
|
self.cache: pytest.Cache = config.cache
|
|
self.directory = f"{STEPCURRENT_CACHE_DIR}/{config.getoption('stepcurrent')}"
|
|
self.lastrun: Optional[str] = self.cache.get(self.directory, None)
|
|
self.skip: bool = config.getoption("stepcurrent_skip")
|
|
|
|
def pytest_collection_modifyitems(self, config: Config, items: List[Any]) -> None:
|
|
if not self.lastrun:
|
|
self.report_status = "Cannot find last run test, not skipping"
|
|
return
|
|
|
|
# check all item nodes until we find a match on last run
|
|
failed_index = None
|
|
for index, item in enumerate(items):
|
|
if item.nodeid == self.lastrun:
|
|
failed_index = index
|
|
if self.skip:
|
|
failed_index += 1
|
|
break
|
|
|
|
# If the previously failed test was not found among the test items,
|
|
# do not skip any tests.
|
|
if failed_index is None:
|
|
self.report_status = "previously run test not found, not skipping."
|
|
else:
|
|
self.report_status = f"skipping {failed_index} already run items."
|
|
deselected = items[:failed_index]
|
|
del items[:failed_index]
|
|
config.hook.pytest_deselected(items=deselected)
|
|
|
|
def pytest_report_collectionfinish(self) -> Optional[str]:
|
|
if self.config.getoption("verbose") >= 0 and self.report_status:
|
|
return f"stepcurrent: {self.report_status}"
|
|
return None
|
|
|
|
def pytest_runtest_protocol(self, item, nextitem) -> None:
|
|
self.lastrun = item.nodeid
|
|
self.cache.set(self.directory, self.lastrun)
|