pytest-flakefighters 0.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pytest_flakefighters/__init__.py +0 -0
- pytest_flakefighters/database_management.py +231 -0
- pytest_flakefighters/flakefighters/__init__.py +0 -0
- pytest_flakefighters/flakefighters/abstract_flakefighter.py +50 -0
- pytest_flakefighters/flakefighters/coverage_independence.py +99 -0
- pytest_flakefighters/flakefighters/deflaker.py +126 -0
- pytest_flakefighters/flakefighters/traceback_matching.py +171 -0
- pytest_flakefighters/function_coverage.py +71 -0
- pytest_flakefighters/main.py +162 -0
- pytest_flakefighters/plugin.py +225 -0
- pytest_flakefighters/rerun_strategies.py +90 -0
- pytest_flakefighters-0.0.0.dist-info/METADATA +116 -0
- pytest_flakefighters-0.0.0.dist-info/RECORD +17 -0
- pytest_flakefighters-0.0.0.dist-info/WHEEL +5 -0
- pytest_flakefighters-0.0.0.dist-info/entry_points.txt +8 -0
- pytest_flakefighters-0.0.0.dist-info/licenses/LICENSE +22 -0
- pytest_flakefighters-0.0.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
"""
|
|
2
|
+
This module implements the Profiler class to help measure function-level coverage.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
import ast
|
|
6
|
+
import cProfile
|
|
7
|
+
import os
|
|
8
|
+
import pstats
|
|
9
|
+
|
|
10
|
+
from coverage import CoverageData
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class Profiler:
|
|
14
|
+
"""
|
|
15
|
+
Provides functionality to measure function-level coverage of a pytest test suite.
|
|
16
|
+
|
|
17
|
+
:ivar coverage_data: The (potentially) covered lines for each module.
|
|
18
|
+
:ivar function_defs: The lines that define a given function in a given module, accessed as
|
|
19
|
+
`function_defs[module][function]`.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
def __init__(self):
|
|
23
|
+
self.coverage_data: CoverageData = CoverageData(no_disk=True)
|
|
24
|
+
self.function_defs: dict[str, dict[str, list[int]]] = {}
|
|
25
|
+
self.profiler = cProfile.Profile()
|
|
26
|
+
|
|
27
|
+
def update_function_defs(self, module: str):
|
|
28
|
+
"""
|
|
29
|
+
Extract the start and end lines for defined functions in the given module and add them to `function_defs`.
|
|
30
|
+
|
|
31
|
+
:param module: The filepath of the module to process.
|
|
32
|
+
"""
|
|
33
|
+
with open(module, encoding="utf8") as f:
|
|
34
|
+
tree = ast.parse(f.read())
|
|
35
|
+
self.function_defs[module] = {
|
|
36
|
+
node.name: list(range(node.lineno, node.end_lineno + 1))
|
|
37
|
+
for node in ast.walk(tree)
|
|
38
|
+
if isinstance(node, ast.FunctionDef)
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
def start(self):
|
|
42
|
+
"""
|
|
43
|
+
Start measuring coverage.
|
|
44
|
+
"""
|
|
45
|
+
self.profiler.enable()
|
|
46
|
+
|
|
47
|
+
def stop(self):
|
|
48
|
+
"""
|
|
49
|
+
Stop measuring coverage.
|
|
50
|
+
"""
|
|
51
|
+
self.profiler.disable()
|
|
52
|
+
p = pstats.Stats(self.profiler)
|
|
53
|
+
for module, _, function in p.stats.keys():
|
|
54
|
+
if module not in self.function_defs and os.path.exists(module):
|
|
55
|
+
self.update_function_defs(module)
|
|
56
|
+
self.coverage_data.add_lines({module: self.function_defs.get(module, {}).get(function, [])})
|
|
57
|
+
|
|
58
|
+
def switch_context(self, context: str):
|
|
59
|
+
"""
|
|
60
|
+
Set the context name of the coverage measurement.
|
|
61
|
+
|
|
62
|
+
:param context: The context name to set.
|
|
63
|
+
"""
|
|
64
|
+
self.profiler.clear()
|
|
65
|
+
self.coverage_data.set_context(context)
|
|
66
|
+
|
|
67
|
+
def get_data(self) -> CoverageData:
|
|
68
|
+
"""
|
|
69
|
+
Return coverage data.
|
|
70
|
+
"""
|
|
71
|
+
return self.coverage_data
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
"""
|
|
2
|
+
This module adds all the FlakeFighter configuration options to pytest.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
import logging
|
|
6
|
+
import os
|
|
7
|
+
from importlib.metadata import entry_points
|
|
8
|
+
|
|
9
|
+
import coverage
|
|
10
|
+
import pytest
|
|
11
|
+
import yaml
|
|
12
|
+
|
|
13
|
+
from pytest_flakefighters.database_management import Database
|
|
14
|
+
from pytest_flakefighters.flakefighters.deflaker import DeFlaker
|
|
15
|
+
from pytest_flakefighters.function_coverage import Profiler
|
|
16
|
+
from pytest_flakefighters.plugin import FlakeFighterPlugin
|
|
17
|
+
from pytest_flakefighters.rerun_strategies import All, FlakyFailure, PreviouslyFlaky
|
|
18
|
+
|
|
19
|
+
rerun_strategies = {"ALL": All, "FLAKY_FAILURE": FlakyFailure, "PREVIOUSLY_FLAKY": PreviouslyFlaky}
|
|
20
|
+
|
|
21
|
+
logger = logging.getLogger(__name__)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def rerun_strategy(strategy: str, max_reruns: int, **kwargs):
|
|
25
|
+
"""
|
|
26
|
+
Instantiate the selected rerun strategy.
|
|
27
|
+
"""
|
|
28
|
+
if strategy == "PREVIOUSLY_FLAKY":
|
|
29
|
+
return PreviouslyFlaky(max_reruns, kwargs["database"])
|
|
30
|
+
return rerun_strategies[strategy](max_reruns)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def pytest_addoption(parser: pytest.Parser):
|
|
34
|
+
"""
|
|
35
|
+
Add extra pytest options.
|
|
36
|
+
:param parser: The argument parser.
|
|
37
|
+
"""
|
|
38
|
+
group = parser.getgroup("flakefighters")
|
|
39
|
+
group.addoption(
|
|
40
|
+
"--root",
|
|
41
|
+
dest="root",
|
|
42
|
+
action="store",
|
|
43
|
+
default=os.getcwd(),
|
|
44
|
+
help="The root directory of the project. Defaults to the current working directory.",
|
|
45
|
+
)
|
|
46
|
+
group.addoption(
|
|
47
|
+
"--suppress-flaky-failures-exit-code",
|
|
48
|
+
dest="suppress_flaky",
|
|
49
|
+
action="store_true",
|
|
50
|
+
default=False,
|
|
51
|
+
help="Return OK exit code if the only failures are flaky failures.",
|
|
52
|
+
)
|
|
53
|
+
group.addoption(
|
|
54
|
+
"--no-save",
|
|
55
|
+
action="store_true",
|
|
56
|
+
default=False,
|
|
57
|
+
help="Do not save this run to the database of previous flakefighters runs.",
|
|
58
|
+
)
|
|
59
|
+
group.addoption(
|
|
60
|
+
"--function-coverage",
|
|
61
|
+
action="store_true",
|
|
62
|
+
default=False,
|
|
63
|
+
help="Use function-level coverage instead of line coverage.",
|
|
64
|
+
)
|
|
65
|
+
group.addoption(
|
|
66
|
+
"--load-max-runs",
|
|
67
|
+
"-M",
|
|
68
|
+
action="store",
|
|
69
|
+
default=None,
|
|
70
|
+
help="The maximum number of previous runs to consider.",
|
|
71
|
+
)
|
|
72
|
+
group.addoption(
|
|
73
|
+
"--database-url",
|
|
74
|
+
"-D",
|
|
75
|
+
action="store",
|
|
76
|
+
default="sqlite:///flakefighters.db",
|
|
77
|
+
help="The database URL. Defaults to 'flakefighters.db' in current working directory.",
|
|
78
|
+
)
|
|
79
|
+
group.addoption(
|
|
80
|
+
"--store-max-runs",
|
|
81
|
+
action="store",
|
|
82
|
+
default=None,
|
|
83
|
+
type=int,
|
|
84
|
+
help="The maximum number of previous flakefighters runs to store. Default is to store all.",
|
|
85
|
+
)
|
|
86
|
+
group.addoption(
|
|
87
|
+
"--max-reruns",
|
|
88
|
+
action="store",
|
|
89
|
+
default=0,
|
|
90
|
+
type=int,
|
|
91
|
+
help="The maximum number of times to rerun tests. "
|
|
92
|
+
"By default, only failing tests marked as flaky will be rerun. "
|
|
93
|
+
"This can be changed with the --rerun-strategy parameter.",
|
|
94
|
+
)
|
|
95
|
+
group.addoption(
|
|
96
|
+
"--rerun-strategy",
|
|
97
|
+
action="store",
|
|
98
|
+
type=str,
|
|
99
|
+
choices=list(rerun_strategies),
|
|
100
|
+
default="FLAKY_FAILURE",
|
|
101
|
+
help="The strategy used to determine which tests to rerun. Supported options are:\n "
|
|
102
|
+
+ "\n ".join(f"{name} - {strat.help()}" for name, strat in rerun_strategies.items()),
|
|
103
|
+
)
|
|
104
|
+
group.addoption(
|
|
105
|
+
"--time-immemorial",
|
|
106
|
+
action="store",
|
|
107
|
+
default=None,
|
|
108
|
+
help="How long to store flakefighters runs for, specified as `days:hours:minutes`. "
|
|
109
|
+
"E.g. to store tests for one week, use 7:0:0.",
|
|
110
|
+
)
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def pytest_configure(config: pytest.Config):
|
|
114
|
+
"""
|
|
115
|
+
Initialise the FlakeFighterPlugin class.
|
|
116
|
+
:param config: The config options.
|
|
117
|
+
"""
|
|
118
|
+
database = Database(
|
|
119
|
+
config.option.database_url,
|
|
120
|
+
config.option.load_max_runs,
|
|
121
|
+
config.option.store_max_runs,
|
|
122
|
+
config.option.time_immemorial,
|
|
123
|
+
)
|
|
124
|
+
|
|
125
|
+
if config.option.function_coverage:
|
|
126
|
+
cov = Profiler()
|
|
127
|
+
else:
|
|
128
|
+
cov = coverage.Coverage()
|
|
129
|
+
|
|
130
|
+
algorithms = entry_points(group="pytest_flakefighters")
|
|
131
|
+
flakefighter_configs = config.inicfg.get("pytest_flakefighters")
|
|
132
|
+
|
|
133
|
+
flakefighters = []
|
|
134
|
+
if flakefighter_configs is not None:
|
|
135
|
+
flakefighter_configs = yaml.safe_load(flakefighter_configs.value)
|
|
136
|
+
for flakefighter in algorithms:
|
|
137
|
+
if flakefighter.name in flakefighter_configs:
|
|
138
|
+
flakefighters.append(
|
|
139
|
+
flakefighter.load().from_config(
|
|
140
|
+
vars(config.option) | {"database": database} | flakefighter_configs[flakefighter.name]
|
|
141
|
+
)
|
|
142
|
+
)
|
|
143
|
+
|
|
144
|
+
else:
|
|
145
|
+
logger.warning("No flakefighters specified. Using basic DeFlaker only.")
|
|
146
|
+
flakefighters.append(
|
|
147
|
+
DeFlaker(
|
|
148
|
+
run_live=True,
|
|
149
|
+
root=config.option.root,
|
|
150
|
+
)
|
|
151
|
+
)
|
|
152
|
+
|
|
153
|
+
config.pluginmanager.register(
|
|
154
|
+
FlakeFighterPlugin(
|
|
155
|
+
root=config.option.root,
|
|
156
|
+
database=database,
|
|
157
|
+
cov=cov,
|
|
158
|
+
flakefighters=flakefighters,
|
|
159
|
+
rerun_strategy=rerun_strategy(config.option.rerun_strategy, config.option.max_reruns, database=database),
|
|
160
|
+
save_run=not config.option.no_save,
|
|
161
|
+
)
|
|
162
|
+
)
|
|
@@ -0,0 +1,225 @@
|
|
|
1
|
+
"""
|
|
2
|
+
This module implements the DeFlaker algorithm [Bell et al. 10.1145/3180155.3180164] as a pytest plugin.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from datetime import datetime
|
|
6
|
+
from enum import Enum
|
|
7
|
+
from typing import Union
|
|
8
|
+
|
|
9
|
+
import coverage
|
|
10
|
+
import pytest
|
|
11
|
+
from _pytest.runner import runtestprotocol
|
|
12
|
+
|
|
13
|
+
from pytest_flakefighters.database_management import (
|
|
14
|
+
ActiveFlakeFighter,
|
|
15
|
+
Database,
|
|
16
|
+
Run,
|
|
17
|
+
Test,
|
|
18
|
+
TestException,
|
|
19
|
+
TestExecution,
|
|
20
|
+
TracebackEntry,
|
|
21
|
+
)
|
|
22
|
+
from pytest_flakefighters.flakefighters.abstract_flakefighter import FlakeFighter
|
|
23
|
+
from pytest_flakefighters.function_coverage import Profiler
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class RerunStrategy(Enum):
|
|
27
|
+
"""
|
|
28
|
+
Enum for supported test rerunning strategies.
|
|
29
|
+
:cvar ALL: Rerun all tests, regardless of outcome.
|
|
30
|
+
:cvar FLAKY_FAILURE: Rerun failing tests marked as flaky.
|
|
31
|
+
:cvar PREVIOUS_FLAKY: Rerun tests that have previously been marked as flaky as well as newly failing flaky tests.
|
|
32
|
+
"""
|
|
33
|
+
|
|
34
|
+
ALL = "ALL"
|
|
35
|
+
FLAKY_FAILURE = "FLAKY_FAILURE"
|
|
36
|
+
PREVIOUSLY_FLAKY = "PREVIOUSLY_FLAKY"
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class FlakeFighterPlugin: # pylint: disable=R0902
|
|
40
|
+
"""
|
|
41
|
+
The main plugin to manage the various FlakeFighter tools.
|
|
42
|
+
"""
|
|
43
|
+
|
|
44
|
+
def __init__( # pylint: disable=R0913,R0917
|
|
45
|
+
self,
|
|
46
|
+
root: str,
|
|
47
|
+
database: Database,
|
|
48
|
+
cov: Union[coverage.Coverage, Profiler],
|
|
49
|
+
flakefighters: list[FlakeFighter],
|
|
50
|
+
save_run: bool = True,
|
|
51
|
+
rerun_strategy: RerunStrategy = RerunStrategy.FLAKY_FAILURE,
|
|
52
|
+
):
|
|
53
|
+
self.root = root
|
|
54
|
+
self.database = database
|
|
55
|
+
self.cov = cov
|
|
56
|
+
self.flakefighters = flakefighters
|
|
57
|
+
self.save_run = save_run
|
|
58
|
+
self.rerun_strategy = rerun_strategy
|
|
59
|
+
|
|
60
|
+
self.run = Run( # pylint: disable=E1123
|
|
61
|
+
root=root,
|
|
62
|
+
active_flakefighters=[
|
|
63
|
+
ActiveFlakeFighter(name=f.__class__.__name__, params=f.params()) for f in flakefighters
|
|
64
|
+
],
|
|
65
|
+
)
|
|
66
|
+
|
|
67
|
+
def pytest_sessionstart(self, session: pytest.Session): # pylint: disable=unused-argument
|
|
68
|
+
"""
|
|
69
|
+
Start the coverage measurement before tests are collected so we measure class and method definitions as covered.
|
|
70
|
+
:param session: The session.
|
|
71
|
+
"""
|
|
72
|
+
self.cov.start()
|
|
73
|
+
self.cov.switch_context("collection") # pragma: no cover
|
|
74
|
+
|
|
75
|
+
def pytest_collection_finish(self, session: pytest.Session): # pylint: disable=unused-argument
|
|
76
|
+
"""
|
|
77
|
+
Stop the coverage measurement after tests are collected.
|
|
78
|
+
:param session: The session.
|
|
79
|
+
"""
|
|
80
|
+
# Line cannot appear as covered on our tests because the coverage measurement is leaking into the self.cov
|
|
81
|
+
self.cov.switch_context(None) # pragma: no cover
|
|
82
|
+
self.cov.stop() # pragma: no cover
|
|
83
|
+
|
|
84
|
+
@pytest.hookimpl(hookwrapper=True)
|
|
85
|
+
def pytest_runtest_call(self, item: pytest.Item):
|
|
86
|
+
"""
|
|
87
|
+
Start the coverage measurement and label the coverage for the current test, run the test,
|
|
88
|
+
then stop coverage measurement.
|
|
89
|
+
|
|
90
|
+
:param item: The item.
|
|
91
|
+
"""
|
|
92
|
+
item.start = datetime.now().timestamp()
|
|
93
|
+
self.cov.start()
|
|
94
|
+
# Lines cannot appear as covered on our tests because the coverage measurement is leaking into the self.cov
|
|
95
|
+
self.cov.switch_context(item.nodeid) # pragma: no cover
|
|
96
|
+
yield # pragma: no cover
|
|
97
|
+
self.cov.stop() # pragma: no cover
|
|
98
|
+
item.stop = datetime.now().timestamp()
|
|
99
|
+
|
|
100
|
+
@pytest.hookimpl(hookwrapper=True)
|
|
101
|
+
def pytest_runtest_makereport(self, item: pytest.Item, call: pytest.CallInfo): # pylint: disable=unused-argument
|
|
102
|
+
"""
|
|
103
|
+
Called after a test execution call (setup, call, teardown)
|
|
104
|
+
to create a TestReport.
|
|
105
|
+
|
|
106
|
+
:param item: The item.
|
|
107
|
+
:param call: The call info.
|
|
108
|
+
"""
|
|
109
|
+
outcome = yield
|
|
110
|
+
report = outcome.get_result()
|
|
111
|
+
excinfo = call.excinfo
|
|
112
|
+
if excinfo is not None and call.when == "call":
|
|
113
|
+
report.exception = TestException( # pylint: disable=E1123
|
|
114
|
+
name=excinfo.type.__name__,
|
|
115
|
+
traceback=[
|
|
116
|
+
TracebackEntry(
|
|
117
|
+
path=str(entry.path),
|
|
118
|
+
lineno=entry.lineno,
|
|
119
|
+
colno=entry.colno,
|
|
120
|
+
statement=str(entry.statement),
|
|
121
|
+
source=str(entry.source),
|
|
122
|
+
)
|
|
123
|
+
for entry in excinfo.traceback
|
|
124
|
+
if entry.path
|
|
125
|
+
],
|
|
126
|
+
)
|
|
127
|
+
else:
|
|
128
|
+
report.exception = None
|
|
129
|
+
|
|
130
|
+
def pytest_runtest_protocol(self, item: pytest.Item, nextitem: pytest.Item) -> bool:
|
|
131
|
+
"""
|
|
132
|
+
Rerun flaky tests. Follows a similar control logic to the pytest-rerunfailures plugin.
|
|
133
|
+
|
|
134
|
+
:param item: The item.
|
|
135
|
+
:param nextitem: The next item.
|
|
136
|
+
|
|
137
|
+
:return: The return value is not used, but only stops further processing.
|
|
138
|
+
"""
|
|
139
|
+
item.execution_count = 0
|
|
140
|
+
executions = []
|
|
141
|
+
skipped = False
|
|
142
|
+
|
|
143
|
+
for _ in range(self.rerun_strategy.max_reruns + 1):
|
|
144
|
+
item.execution_count += 1
|
|
145
|
+
item.ihook.pytest_runtest_logstart(nodeid=item.nodeid, location=item.location)
|
|
146
|
+
reports = runtestprotocol(item, nextitem=nextitem, log=False)
|
|
147
|
+
|
|
148
|
+
for report in reports: # up to 3 reports: setup, call, teardown
|
|
149
|
+
if report.when == "setup" and report.skipped:
|
|
150
|
+
skipped = True
|
|
151
|
+
if report.when == "call":
|
|
152
|
+
line_coverage = self.cov.get_data()
|
|
153
|
+
line_coverage.set_query_contexts(["collection", item.nodeid])
|
|
154
|
+
captured_output = dict(report.sections)
|
|
155
|
+
test_execution = TestExecution( # pylint: disable=E1123
|
|
156
|
+
outcome=report.outcome,
|
|
157
|
+
stdout=captured_output.get("stdout"),
|
|
158
|
+
stderr=captured_output.get("stderr"),
|
|
159
|
+
report=str(report.longrepr),
|
|
160
|
+
start_time=datetime.fromtimestamp(item.start),
|
|
161
|
+
end_time=datetime.fromtimestamp(item.stop),
|
|
162
|
+
coverage={
|
|
163
|
+
file_path: line_coverage.lines(file_path) for file_path in line_coverage.measured_files()
|
|
164
|
+
},
|
|
165
|
+
exception=report.exception,
|
|
166
|
+
)
|
|
167
|
+
item.test_execution = test_execution
|
|
168
|
+
executions.append(test_execution)
|
|
169
|
+
for ff in filter(lambda ff: ff.run_live, self.flakefighters):
|
|
170
|
+
ff.flaky_test_live(test_execution)
|
|
171
|
+
report.flaky = any(result.flaky for result in test_execution.flakefighter_results)
|
|
172
|
+
if item.execution_count <= self.rerun_strategy.max_reruns and self.rerun_strategy.rerun(report):
|
|
173
|
+
break # trigger rerun
|
|
174
|
+
item.ihook.pytest_runtest_logreport(report=report)
|
|
175
|
+
else:
|
|
176
|
+
break # Skip further reruns
|
|
177
|
+
|
|
178
|
+
item.ihook.pytest_runtest_logfinish(nodeid=item.nodeid, location=item.location)
|
|
179
|
+
test = Test( # pylint: disable=E1123
|
|
180
|
+
name=item.nodeid,
|
|
181
|
+
skipped=skipped,
|
|
182
|
+
executions=executions,
|
|
183
|
+
)
|
|
184
|
+
self.run.tests.append(test)
|
|
185
|
+
return True
|
|
186
|
+
|
|
187
|
+
def pytest_report_teststatus(
|
|
188
|
+
self, report: pytest.TestReport, config: pytest.Config # pylint: disable=unused-argument
|
|
189
|
+
) -> tuple[str, str, str]:
|
|
190
|
+
"""
|
|
191
|
+
Report flaky failures as such.
|
|
192
|
+
:param report: The report object whose status is to be returned.
|
|
193
|
+
:param config: The pytest config object.
|
|
194
|
+
:returns: The test status.
|
|
195
|
+
"""
|
|
196
|
+
if getattr(report, "flaky", False):
|
|
197
|
+
return report.outcome, "F", ("FLAKY", {"yellow": True})
|
|
198
|
+
return None
|
|
199
|
+
|
|
200
|
+
def pytest_sessionfinish(
|
|
201
|
+
self,
|
|
202
|
+
session: pytest.Session,
|
|
203
|
+
exitstatus: pytest.ExitCode, # pylint: disable=unused-argument
|
|
204
|
+
) -> None:
|
|
205
|
+
"""Called after whole test run finished, right before returning the exit status to the system.
|
|
206
|
+
:param session: The pytest session object.
|
|
207
|
+
:param exitstatus: The status which pytest will return to the system.
|
|
208
|
+
"""
|
|
209
|
+
for ff in filter(lambda ff: not ff.run_live, self.flakefighters):
|
|
210
|
+
ff.flaky_tests_post(self.run)
|
|
211
|
+
|
|
212
|
+
genuine_failure_observed = any(
|
|
213
|
+
not test.flaky for test in self.run.tests if any(e.outcome != "passed" for e in test.executions)
|
|
214
|
+
)
|
|
215
|
+
|
|
216
|
+
if (
|
|
217
|
+
session.config.option.suppress_flaky
|
|
218
|
+
and session.exitstatus == pytest.ExitCode.TESTS_FAILED
|
|
219
|
+
and not genuine_failure_observed
|
|
220
|
+
):
|
|
221
|
+
session.exitstatus = pytest.ExitCode.OK
|
|
222
|
+
|
|
223
|
+
if self.save_run:
|
|
224
|
+
self.database.save(self.run)
|
|
225
|
+
self.database.engine.dispose()
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
"""
|
|
2
|
+
This module implements the various supported test rerun strategies.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from abc import ABC, abstractmethod
|
|
6
|
+
|
|
7
|
+
import pytest
|
|
8
|
+
from sqlalchemy import select
|
|
9
|
+
from sqlalchemy.orm import Session
|
|
10
|
+
|
|
11
|
+
from pytest_flakefighters.database_management import Database, Test
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class RerunStrategy(ABC):
|
|
15
|
+
"""
|
|
16
|
+
Abstract base class for test rerun strategies.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
def __init__(self, max_reruns: int):
|
|
20
|
+
self.max_reruns = max_reruns
|
|
21
|
+
|
|
22
|
+
@abstractmethod
|
|
23
|
+
def rerun(self, report: pytest.TestReport) -> bool:
|
|
24
|
+
"""
|
|
25
|
+
Decide whether to rerun a test case.
|
|
26
|
+
:return: Boolean true to rerun, False otherwise.
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
@classmethod
|
|
30
|
+
@abstractmethod
|
|
31
|
+
def help(cls) -> str:
|
|
32
|
+
"""
|
|
33
|
+
Return the help string for config options.
|
|
34
|
+
"""
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class All(RerunStrategy):
|
|
38
|
+
"""
|
|
39
|
+
Rerun all tests, regardless of outcome.
|
|
40
|
+
"""
|
|
41
|
+
|
|
42
|
+
def rerun(self, report: pytest.TestReport) -> bool:
|
|
43
|
+
"""
|
|
44
|
+
Trivially rerun all tests, regardless of outcome.
|
|
45
|
+
:return: Boolean true to rerun, False otherwise.
|
|
46
|
+
"""
|
|
47
|
+
return True
|
|
48
|
+
|
|
49
|
+
@classmethod
|
|
50
|
+
def help(cls):
|
|
51
|
+
return "Trivially rerun all tests, regardless of outcome."
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
class FlakyFailure(RerunStrategy):
|
|
55
|
+
"""
|
|
56
|
+
Strategy to rerun failed tests marked as flaky.
|
|
57
|
+
"""
|
|
58
|
+
|
|
59
|
+
def rerun(self, report: pytest.TestReport) -> bool:
|
|
60
|
+
"""
|
|
61
|
+
:return: Boolean true if a test fails and has been marked as flaky by live FlakeFighters.
|
|
62
|
+
"""
|
|
63
|
+
return report.flaky and not report.passed
|
|
64
|
+
|
|
65
|
+
@classmethod
|
|
66
|
+
def help(cls):
|
|
67
|
+
return "Rerun failing tests that have been merked as flaky by live FlakeFighters."
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
class PreviouslyFlaky(FlakyFailure):
|
|
71
|
+
"""
|
|
72
|
+
Rerun failed tests marked as flaky and tests previously marked as flaky.
|
|
73
|
+
"""
|
|
74
|
+
|
|
75
|
+
def __init__(self, reruns: int, database: Database):
|
|
76
|
+
super().__init__(reruns)
|
|
77
|
+
with Session(database.engine) as session:
|
|
78
|
+
tests = session.scalars(select(Test)).all()
|
|
79
|
+
self.previously_flaky = list(filter(lambda t: t.flaky, tests))
|
|
80
|
+
|
|
81
|
+
def rerun(self, report: pytest.TestReport) -> bool:
|
|
82
|
+
"""
|
|
83
|
+
:return: Boolean true if a test is a flaky failure or has previously been marked as flaky and has the same name
|
|
84
|
+
as the current test.
|
|
85
|
+
"""
|
|
86
|
+
return super().rerun(report) or any(test.name == report.nodeid for test in self.previously_flaky)
|
|
87
|
+
|
|
88
|
+
@classmethod
|
|
89
|
+
def help(cls):
|
|
90
|
+
return "Rerun failing tests marked as flaky, and tests that have previously been marked as flaky."
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: pytest-flakefighters
|
|
3
|
+
Version: 0.0.0
|
|
4
|
+
Summary: Pytest plugin implementing flaky test failure detection and classification.
|
|
5
|
+
Author: TestFLARE Team
|
|
6
|
+
License:
|
|
7
|
+
The MIT License (MIT)
|
|
8
|
+
|
|
9
|
+
Copyright (c) 2025 TestFLARE Team
|
|
10
|
+
|
|
11
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
12
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
13
|
+
in the Software without restriction, including without limitation the rights
|
|
14
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
15
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
16
|
+
furnished to do so, subject to the following conditions:
|
|
17
|
+
|
|
18
|
+
The above copyright notice and this permission notice shall be included in
|
|
19
|
+
all copies or substantial portions of the Software.
|
|
20
|
+
|
|
21
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
22
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
23
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
24
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
25
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
26
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
27
|
+
THE SOFTWARE.
|
|
28
|
+
|
|
29
|
+
Project-URL: Repository, https://github.com/test-flare/pytest-flakefighter
|
|
30
|
+
Requires-Python: >=3.10
|
|
31
|
+
Description-Content-Type: text/markdown
|
|
32
|
+
License-File: LICENSE
|
|
33
|
+
Requires-Dist: pytest>=6.2.0
|
|
34
|
+
Requires-Dist: coverage>=7.10.6
|
|
35
|
+
Requires-Dist: GitPython>=3.1.45
|
|
36
|
+
Requires-Dist: unidiff>=0.7.5
|
|
37
|
+
Requires-Dist: sqlalchemy>=2.0.43
|
|
38
|
+
Requires-Dist: dotenv>=0.9.9
|
|
39
|
+
Requires-Dist: pandas
|
|
40
|
+
Requires-Dist: scipy<=1.15
|
|
41
|
+
Requires-Dist: pyyaml>=6
|
|
42
|
+
Requires-Dist: scikit-learn>=1.7
|
|
43
|
+
Requires-Dist: nltk>=3.9
|
|
44
|
+
Provides-Extra: dev
|
|
45
|
+
Requires-Dist: black; extra == "dev"
|
|
46
|
+
Requires-Dist: pytest-cov; extra == "dev"
|
|
47
|
+
Requires-Dist: pylint; extra == "dev"
|
|
48
|
+
Requires-Dist: pre_commit; extra == "dev"
|
|
49
|
+
Requires-Dist: sphinx-autoapi; extra == "dev"
|
|
50
|
+
Requires-Dist: sphinx_rtd_theme; extra == "dev"
|
|
51
|
+
Requires-Dist: tox>=4.31.0; extra == "dev"
|
|
52
|
+
Dynamic: license-file
|
|
53
|
+
|
|
54
|
+
# Pytest FlakeFighters
|
|
55
|
+
|
|
56
|
+
[](https://pypi.org/project/pytest-flakefighters)
|
|
57
|
+
[](https://pypi.org/project/pytest-flakefighters)
|
|
58
|
+
|
|
59
|
+
Pytest plugin implementing flaky test failure detection and
|
|
60
|
+
classification.
|
|
61
|
+
|
|
62
|
+
------------------------------------------------------------------------
|
|
63
|
+
|
|
64
|
+
This [pytest](https://github.com/pytest-dev/pytest) plugin was generated with [Cookiecutter](https://github.com/audreyr/cookiecutter) along with [\@hackebrot](https://github.com/hackebrot)\'s [cookiecutter-pytest-plugin](https://github.com/pytest-dev/cookiecutter-pytest-plugin) template.
|
|
65
|
+
|
|
66
|
+
## Features
|
|
67
|
+
|
|
68
|
+
- Implements the [DeFlaker algorithm](https://deflaker.com/) for pytest
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
## Installation
|
|
72
|
+
|
|
73
|
+
You can install \"pytest-flakefighters\" by cloning this repo and running `pip install .` from the root directory.
|
|
74
|
+
If you intend to develop the plugin, run `pip install -e .[dev]` instead.
|
|
75
|
+
|
|
76
|
+
We eventually intend to distribute our tool on PyPI.
|
|
77
|
+
|
|
78
|
+
## Usage
|
|
79
|
+
|
|
80
|
+
FlakeFighter is intended to run on git repositories that have test suites runnable with `pytest`.
|
|
81
|
+
Once you have installed FlakeFighter, you can run it from the root directory of your repo simply by running `pytest` in your usual way.
|
|
82
|
+
FlakeFighter has the following arguments.
|
|
83
|
+
|
|
84
|
+
```
|
|
85
|
+
--target-commit=TARGET_COMMIT
|
|
86
|
+
The target (newer) commit hash. Defaults to HEAD (the most recent commit).
|
|
87
|
+
--source-commit=SOURCE_COMMIT
|
|
88
|
+
The source (older) commit hash. Defaults to HEAD^ (the previous commit to target).
|
|
89
|
+
--repo=REPO_ROOT The commit hash to compare against.
|
|
90
|
+
--suppress-flaky-failures-exit-code
|
|
91
|
+
Return OK exit code if the only failures are flaky failures.
|
|
92
|
+
--no-save Do not save this run to the database of previous flakefighters runs.
|
|
93
|
+
-M LOAD_MAX_RUNS, --load-max-runs=LOAD_MAX_RUNS
|
|
94
|
+
The maximum number of previous runs to consider.
|
|
95
|
+
-D DATABASE_URL, --database-url=DATABASE_URL
|
|
96
|
+
The database URL. Defaults to 'flakefighter.db' in current working directory.
|
|
97
|
+
--store-max-runs=STORE_MAX_RUNS
|
|
98
|
+
The maximum number of previous flakefighters runs to store. Default is to store all.
|
|
99
|
+
--time-immemorial=TIME_IMMEMORIAL
|
|
100
|
+
How long to store flakefighters runs for, specified as `days:hours:minutes`. E.g. to store
|
|
101
|
+
tests for one week, use 7:0:0.
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
## Contributing
|
|
105
|
+
|
|
106
|
+
Contributions are very welcome.
|
|
107
|
+
Tests can be run with [pytest](https://pytest.readthedocs.io/en/latest/), please ensure the coverage at least stays the same before you submit a pull request.
|
|
108
|
+
|
|
109
|
+
## Flake Fighters
|
|
110
|
+
Our plugin is made up of a collection of heuristics that come together to help inform whether a test failure is genuine or flaky.
|
|
111
|
+
These come in two "flavours": those which run live after each test, and those which run at the end of the entire test suite.
|
|
112
|
+
Both extend the base class `FlakeFighter` and implement the `flaky_failure` method, which returns `True` if the test is deemed to be flaky.
|
|
113
|
+
|
|
114
|
+
## Issues
|
|
115
|
+
|
|
116
|
+
If you encounter any problems, please [file an issue](https://github.com/test-flare/pytest-flakefighters/issues) along with a detailed description.
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
pytest_flakefighters/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
2
|
+
pytest_flakefighters/database_management.py,sha256=MDwwpF8FwmFX6is9pVPye7qEOrjZXOsfXG2UI5I7uk4,7029
|
|
3
|
+
pytest_flakefighters/function_coverage.py,sha256=_WAENobGFbvlbT_CZO6_Ohf8lWaspi5DrWLZd4BdbxY,2183
|
|
4
|
+
pytest_flakefighters/main.py,sha256=u2tgDVLcQ7bbCb55aLG7Nqu5KVx_9GyRAuMBdojCHeM,5148
|
|
5
|
+
pytest_flakefighters/plugin.py,sha256=Y3eQRK58ydO-c77PpnQ8cjWFtOVHop4yf8iAPKEO6TU,8780
|
|
6
|
+
pytest_flakefighters/rerun_strategies.py,sha256=nu6V6sqIMTVqkMBiYek30pe_zolGtdHQwTrgtj13ZG4,2532
|
|
7
|
+
pytest_flakefighters/flakefighters/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
8
|
+
pytest_flakefighters/flakefighters/abstract_flakefighter.py,sha256=bchRwoNzQaFhSknT9wKDpbf3EiReHPsHsDhm5qi8y_o,1953
|
|
9
|
+
pytest_flakefighters/flakefighters/coverage_independence.py,sha256=6Ic5SN1Np7vQbvNSRgWwiSF9ReQ293mvNwcn0ZS5js8,4349
|
|
10
|
+
pytest_flakefighters/flakefighters/deflaker.py,sha256=L0pPCWTaKaK_lr7qGh4uhI1ofJXgDr9m-CCjOk-_2KM,5248
|
|
11
|
+
pytest_flakefighters/flakefighters/traceback_matching.py,sha256=DVH8OeoDFmBQW277ap_-e2xe8UBJWkZl5HH-iMdb_Ag,6605
|
|
12
|
+
pytest_flakefighters-0.0.0.dist-info/licenses/LICENSE,sha256=tTzR2CWQMPOp-mQIQqi0cTRkaogeBUmW06blQsBLdQg,1082
|
|
13
|
+
pytest_flakefighters-0.0.0.dist-info/METADATA,sha256=rm_arh8KcYnTVM-qEaJNFyC_JSRvJ7hYhJugNak-9vg,5574
|
|
14
|
+
pytest_flakefighters-0.0.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
15
|
+
pytest_flakefighters-0.0.0.dist-info/entry_points.txt,sha256=xockvN1AszN2XqaET77JDIRdOafgm3DdvOtHgVw-aDU,424
|
|
16
|
+
pytest_flakefighters-0.0.0.dist-info/top_level.txt,sha256=mRzzeCn_6fy5c4knXqUVx2n0d86SvnOpeJcSRUevhWg,21
|
|
17
|
+
pytest_flakefighters-0.0.0.dist-info/RECORD,,
|