flakeguard 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- flakeguard/__init__.py +21 -0
- flakeguard/__main__.py +6 -0
- flakeguard/cli.py +149 -0
- flakeguard/models.py +60 -0
- flakeguard/parsers/__init__.py +21 -0
- flakeguard/parsers/junit_xml.py +98 -0
- flakeguard/report.py +108 -0
- flakeguard/scoring.py +122 -0
- flakeguard-0.1.0.dist-info/METADATA +141 -0
- flakeguard-0.1.0.dist-info/RECORD +13 -0
- flakeguard-0.1.0.dist-info/WHEEL +4 -0
- flakeguard-0.1.0.dist-info/entry_points.txt +2 -0
- flakeguard-0.1.0.dist-info/licenses/LICENSE +21 -0
flakeguard/__init__.py
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
"""flakeguard - find, rank, and quarantine flaky tests.
|
|
2
|
+
|
|
3
|
+
A language-agnostic tool: point it at the test reports your CI already
|
|
4
|
+
produces (JUnit XML and friends) and it tells you which tests are flaky,
|
|
5
|
+
how flaky, and why it thinks so.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
__version__ = "0.1.0"
|
|
9
|
+
|
|
10
|
+
from .models import Outcome, TestCase
|
|
11
|
+
from .scoring import FlakeStats, analyze
|
|
12
|
+
from .parsers.junit_xml import parse_junit_xml
|
|
13
|
+
|
|
14
|
+
__all__ = [
|
|
15
|
+
"__version__",
|
|
16
|
+
"Outcome",
|
|
17
|
+
"TestCase",
|
|
18
|
+
"FlakeStats",
|
|
19
|
+
"analyze",
|
|
20
|
+
"parse_junit_xml",
|
|
21
|
+
]
|
flakeguard/__main__.py
ADDED
flakeguard/cli.py
ADDED
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
"""Command-line entry point.
|
|
2
|
+
|
|
3
|
+
flakeguard scan ./reports
|
|
4
|
+
flakeguard scan run1.xml run2.xml run3.xml --format json
|
|
5
|
+
flakeguard scan ./ci-artifacts --min-runs 5 --min-score 0.2
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import argparse
|
|
11
|
+
import glob
|
|
12
|
+
import os
|
|
13
|
+
import sys
|
|
14
|
+
from typing import List
|
|
15
|
+
|
|
16
|
+
from . import __version__
|
|
17
|
+
from .parsers import by_suffix
|
|
18
|
+
from .report import RENDERERS
|
|
19
|
+
from .scoring import analyze
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _discover(paths: List[str]) -> List[str]:
|
|
23
|
+
"""Expand the given paths into a sorted list of report files.
|
|
24
|
+
|
|
25
|
+
Directories are walked recursively; files are taken as-is. Sorting
|
|
26
|
+
keeps run order deterministic so results are reproducible.
|
|
27
|
+
"""
|
|
28
|
+
found: List[str] = []
|
|
29
|
+
for path in paths:
|
|
30
|
+
if os.path.isdir(path):
|
|
31
|
+
for root, _dirs, files in os.walk(path):
|
|
32
|
+
for fname in files:
|
|
33
|
+
if os.path.splitext(fname)[1].lower() in by_suffix:
|
|
34
|
+
found.append(os.path.join(root, fname))
|
|
35
|
+
else:
|
|
36
|
+
matches = sorted(glob.glob(path))
|
|
37
|
+
if matches:
|
|
38
|
+
found.extend(matches)
|
|
39
|
+
elif os.path.exists(path):
|
|
40
|
+
found.append(path)
|
|
41
|
+
return sorted(set(found))
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _load_runs(files: List[str]) -> tuple[list, list]:
|
|
45
|
+
"""Parse each file into a run. Returns (runs, errors)."""
|
|
46
|
+
runs, errors = [], []
|
|
47
|
+
for path in files:
|
|
48
|
+
suffix = os.path.splitext(path)[1].lower()
|
|
49
|
+
parser = by_suffix.get(suffix)
|
|
50
|
+
if parser is None:
|
|
51
|
+
errors.append(f"{path}: unsupported format '{suffix}'")
|
|
52
|
+
continue
|
|
53
|
+
if not os.path.exists(path):
|
|
54
|
+
errors.append(f"{path}: file not found")
|
|
55
|
+
continue
|
|
56
|
+
try:
|
|
57
|
+
runs.append(parser(path))
|
|
58
|
+
except Exception as exc: # noqa: BLE001 - report, don't crash
|
|
59
|
+
errors.append(f"{path}: {exc}")
|
|
60
|
+
return runs, errors
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
64
|
+
p = argparse.ArgumentParser(
|
|
65
|
+
prog="flakeguard",
|
|
66
|
+
description="Find, rank, and quarantine flaky tests from test reports.",
|
|
67
|
+
)
|
|
68
|
+
p.add_argument("--version", action="version", version=f"flakeguard {__version__}")
|
|
69
|
+
sub = p.add_subparsers(dest="command", required=True)
|
|
70
|
+
|
|
71
|
+
scan = sub.add_parser("scan", help="analyse one or more report files/dirs")
|
|
72
|
+
scan.add_argument("paths", nargs="+", help="report files, globs, or directories")
|
|
73
|
+
scan.add_argument(
|
|
74
|
+
"--min-runs",
|
|
75
|
+
type=int,
|
|
76
|
+
default=2,
|
|
77
|
+
help="minimum runs a test must appear in to be reported (default: 2)",
|
|
78
|
+
)
|
|
79
|
+
scan.add_argument(
|
|
80
|
+
"--min-score",
|
|
81
|
+
type=float,
|
|
82
|
+
default=0.0,
|
|
83
|
+
help="only report tests at or above this flakiness score, 0-1 (default: 0)",
|
|
84
|
+
)
|
|
85
|
+
scan.add_argument(
|
|
86
|
+
"--format",
|
|
87
|
+
choices=sorted(RENDERERS),
|
|
88
|
+
default="text",
|
|
89
|
+
help="output format (default: text)",
|
|
90
|
+
)
|
|
91
|
+
scan.add_argument(
|
|
92
|
+
"--output", "-o", default=None, help="write the report to a file instead of stdout"
|
|
93
|
+
)
|
|
94
|
+
scan.add_argument(
|
|
95
|
+
"--fail-on-flaky",
|
|
96
|
+
action="store_true",
|
|
97
|
+
help="exit with code 1 if any flaky tests are found (for CI gating)",
|
|
98
|
+
)
|
|
99
|
+
return p
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def main(argv: List[str] | None = None) -> int:
|
|
103
|
+
args = build_parser().parse_args(argv)
|
|
104
|
+
|
|
105
|
+
files = _discover(args.paths)
|
|
106
|
+
if not files:
|
|
107
|
+
print(
|
|
108
|
+
"flakeguard: no test report files found.\n"
|
|
109
|
+
" I look for JUnit XML (.xml) files - give me a file, a glob, or a\n"
|
|
110
|
+
" directory. Need to make one? Most frameworks can, for example:\n"
|
|
111
|
+
" pytest --junitxml=report.xml",
|
|
112
|
+
file=sys.stderr,
|
|
113
|
+
)
|
|
114
|
+
return 2
|
|
115
|
+
|
|
116
|
+
runs, errors = _load_runs(files)
|
|
117
|
+
for err in errors:
|
|
118
|
+
print(f"flakeguard: warning: {err}", file=sys.stderr)
|
|
119
|
+
if not runs:
|
|
120
|
+
print(
|
|
121
|
+
f"flakeguard: none of the {len(files)} file(s) could be parsed.\n"
|
|
122
|
+
" See the warnings above - is the file valid JUnit XML?",
|
|
123
|
+
file=sys.stderr,
|
|
124
|
+
)
|
|
125
|
+
return 2
|
|
126
|
+
|
|
127
|
+
stats = analyze(runs)
|
|
128
|
+
# Apply the minimum-score filter on top of the renderers' min-runs filter.
|
|
129
|
+
if args.min_score > 0:
|
|
130
|
+
stats = [s for s in stats if s.score >= args.min_score or not s.is_flaky()]
|
|
131
|
+
|
|
132
|
+
report = RENDERERS[args.format](stats, args.min_runs)
|
|
133
|
+
|
|
134
|
+
if args.output:
|
|
135
|
+
with open(args.output, "w", encoding="utf-8") as fh:
|
|
136
|
+
fh.write(report + "\n")
|
|
137
|
+
print(f"flakeguard: wrote {args.output} ({len(runs)} runs)", file=sys.stderr)
|
|
138
|
+
else:
|
|
139
|
+
print(report)
|
|
140
|
+
|
|
141
|
+
flaky_count = sum(1 for s in stats if s.is_flaky() and s.considered >= args.min_runs)
|
|
142
|
+
if args.fail_on_flaky and flaky_count > 0:
|
|
143
|
+
print(f"flakeguard: {flaky_count} flaky test(s) found", file=sys.stderr)
|
|
144
|
+
return 1
|
|
145
|
+
return 0
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
if __name__ == "__main__": # pragma: no cover
|
|
149
|
+
raise SystemExit(main())
|
flakeguard/models.py
ADDED
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
"""Core data types shared across parsers, scoring and reporting."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
from enum import Enum
|
|
7
|
+
from typing import Iterable, Optional
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class Outcome(str, Enum):
|
|
11
|
+
"""The result of a single test in a single run."""
|
|
12
|
+
|
|
13
|
+
PASSED = "passed"
|
|
14
|
+
FAILED = "failed"
|
|
15
|
+
ERROR = "error"
|
|
16
|
+
SKIPPED = "skipped"
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def join_unique(parts: Iterable[str]) -> str:
|
|
20
|
+
"""Join non-empty parts with '::', dropping repeats.
|
|
21
|
+
|
|
22
|
+
Frameworks disagree about how they name things: pytest sets a suite and
|
|
23
|
+
a classname, while jest-junit sets classname == name. Dropping repeats
|
|
24
|
+
keeps a test's identity stable and readable for both.
|
|
25
|
+
"""
|
|
26
|
+
seen: list[str] = []
|
|
27
|
+
for part in parts:
|
|
28
|
+
if part and part not in seen:
|
|
29
|
+
seen.append(part)
|
|
30
|
+
return "::".join(seen)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
@dataclass(frozen=True)
|
|
34
|
+
class TestCase:
|
|
35
|
+
"""One test's result, as observed in one run.
|
|
36
|
+
|
|
37
|
+
``suite`` is the file/grouping the framework reported (e.g. a test
|
|
38
|
+
file), ``classname`` is the class or module, and ``name`` is the test
|
|
39
|
+
itself. Together they form a stable identity so the same test can be
|
|
40
|
+
matched across runs.
|
|
41
|
+
"""
|
|
42
|
+
|
|
43
|
+
name: str
|
|
44
|
+
suite: str = ""
|
|
45
|
+
classname: str = ""
|
|
46
|
+
outcome: Outcome = Outcome.PASSED
|
|
47
|
+
duration: float = 0.0
|
|
48
|
+
message: Optional[str] = None
|
|
49
|
+
|
|
50
|
+
# Not a field (no annotation): tells pytest this is data, not a test.
|
|
51
|
+
__test__ = False
|
|
52
|
+
|
|
53
|
+
@property
|
|
54
|
+
def key(self) -> str:
|
|
55
|
+
"""A stable identity for this test across runs."""
|
|
56
|
+
return join_unique([self.suite, self.classname, self.name]) or "<unnamed>"
|
|
57
|
+
|
|
58
|
+
@property
|
|
59
|
+
def display_name(self) -> str:
|
|
60
|
+
return join_unique([self.classname, self.name]) or self.name or "<unnamed>"
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
"""Report parsers.
|
|
2
|
+
|
|
3
|
+
Each parser turns a framework's native test report into a list of
|
|
4
|
+
:class:`flakeguard.models.TestCase`. Adding support for a new framework
|
|
5
|
+
means adding one module here and registering it in ``by_suffix``.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from typing import Callable, List
|
|
11
|
+
|
|
12
|
+
from ..models import TestCase
|
|
13
|
+
from .junit_xml import parse_junit_xml
|
|
14
|
+
|
|
15
|
+
# Map of file suffix -> parser callable. Used by the CLI to auto-detect
|
|
16
|
+
# the format of each report file it is handed.
|
|
17
|
+
by_suffix: dict[str, Callable[[str], List[TestCase]]] = {
|
|
18
|
+
".xml": parse_junit_xml,
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
__all__ = ["parse_junit_xml", "by_suffix"]
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
"""Parser for JUnit XML - the de-facto interchange format for test results.
|
|
2
|
+
|
|
3
|
+
Almost every test framework can emit this: pytest (``--junitxml``), Maven
|
|
4
|
+
and Gradle Surefire, Go's ``gotestsum --junitfile``, Jest
|
|
5
|
+
(``jest-junit``), RSpec (``rspec_junit_formatter``), .NET, PHPUnit, and
|
|
6
|
+
more. That is what makes flakeguard language-agnostic.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import xml.etree.ElementTree as ET
|
|
12
|
+
from typing import Iterator, List, Optional, Tuple
|
|
13
|
+
|
|
14
|
+
from ..models import Outcome, TestCase
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _localname(tag: str) -> str:
|
|
18
|
+
"""Strip an XML namespace, if present (``{ns}testcase`` -> ``testcase``)."""
|
|
19
|
+
return tag.split("}")[-1].lower()
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _outcome_and_message(case: ET.Element) -> Tuple[Outcome, Optional[str]]:
|
|
23
|
+
"""Decide a test's outcome from its child elements.
|
|
24
|
+
|
|
25
|
+
Per the JUnit schema, a ``<failure>`` means an assertion failed, an
|
|
26
|
+
``<error>`` means the test blew up unexpectedly, and a ``<skipped>``
|
|
27
|
+
means it did not run. A ``<testcase>`` with none of these passed.
|
|
28
|
+
"""
|
|
29
|
+
for child in case:
|
|
30
|
+
tag = _localname(child.tag)
|
|
31
|
+
if tag in ("failure", "error"):
|
|
32
|
+
text = (child.get("message") or (child.text or "").strip())
|
|
33
|
+
text = " ".join(text.split()) # collapse whitespace/newlines
|
|
34
|
+
return (
|
|
35
|
+
Outcome.FAILED if tag == "failure" else Outcome.ERROR,
|
|
36
|
+
text[:300] or None,
|
|
37
|
+
)
|
|
38
|
+
if tag == "skipped":
|
|
39
|
+
return Outcome.SKIPPED, child.get("message")
|
|
40
|
+
return Outcome.PASSED, None
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _iter_suites(suite: ET.Element) -> Iterator[ET.Element]:
|
|
44
|
+
"""Yield every ``<testsuite>`` at any depth (they can nest)."""
|
|
45
|
+
yield suite
|
|
46
|
+
for child in suite:
|
|
47
|
+
if _localname(child.tag) == "testsuite":
|
|
48
|
+
yield from _iter_suites(child)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _collect_cases(container: ET.Element, suite_name: str) -> Iterator[TestCase]:
|
|
52
|
+
for child in container:
|
|
53
|
+
tag = _localname(child.tag)
|
|
54
|
+
if tag == "testcase":
|
|
55
|
+
outcome, message = _outcome_and_message(child)
|
|
56
|
+
raw_time = child.get("time")
|
|
57
|
+
try:
|
|
58
|
+
duration = float(raw_time) if raw_time else 0.0
|
|
59
|
+
except ValueError:
|
|
60
|
+
duration = 0.0
|
|
61
|
+
yield TestCase(
|
|
62
|
+
name=child.get("name") or "",
|
|
63
|
+
suite=suite_name,
|
|
64
|
+
classname=child.get("classname") or "",
|
|
65
|
+
outcome=outcome,
|
|
66
|
+
duration=duration,
|
|
67
|
+
message=message,
|
|
68
|
+
)
|
|
69
|
+
elif tag == "testsuite":
|
|
70
|
+
nested_name = child.get("name") or suite_name
|
|
71
|
+
yield from _collect_cases(child, nested_name)
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def parse_junit_xml(path: str) -> List[TestCase]:
|
|
75
|
+
"""Parse a JUnit XML file into a list of test cases.
|
|
76
|
+
|
|
77
|
+
Handles the three common root shapes: ``<testsuites>`` (a bundle of
|
|
78
|
+
suites), a bare ``<testsuite>``, and nested suites.
|
|
79
|
+
"""
|
|
80
|
+
try:
|
|
81
|
+
root = ET.parse(path).getroot()
|
|
82
|
+
except ET.ParseError as exc:
|
|
83
|
+
raise ValueError(f"not valid XML ({exc})") from exc
|
|
84
|
+
root_tag = _localname(root.tag)
|
|
85
|
+
|
|
86
|
+
cases: List[TestCase] = []
|
|
87
|
+
|
|
88
|
+
if root_tag == "testsuites":
|
|
89
|
+
for suite in root:
|
|
90
|
+
if _localname(suite.tag) == "testsuite":
|
|
91
|
+
cases.extend(_collect_cases(suite, suite.get("name") or ""))
|
|
92
|
+
elif root_tag == "testsuite":
|
|
93
|
+
cases.extend(_collect_cases(root, root.get("name") or ""))
|
|
94
|
+
else:
|
|
95
|
+
# Unknown root: be permissive and look for testcases anywhere.
|
|
96
|
+
cases.extend(_collect_cases(root, ""))
|
|
97
|
+
|
|
98
|
+
return cases
|
flakeguard/report.py
ADDED
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
"""Render analysis results as text, JSON, or Markdown."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
from typing import List, Sequence
|
|
7
|
+
|
|
8
|
+
from .scoring import FlakeStats
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def _bar(score: float, width: int = 10) -> str:
|
|
12
|
+
filled = round(score * width)
|
|
13
|
+
return "#" * filled + "." * (width - filled)
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def render_text(stats: Sequence[FlakeStats], min_runs: int) -> str:
|
|
17
|
+
"""A human-readable terminal report."""
|
|
18
|
+
flaky = [s for s in stats if s.is_flaky() and s.considered >= min_runs]
|
|
19
|
+
|
|
20
|
+
lines: List[str] = []
|
|
21
|
+
total = len(stats)
|
|
22
|
+
lines.append("")
|
|
23
|
+
lines.append("flakeguard report")
|
|
24
|
+
lines.append("=" * 60)
|
|
25
|
+
lines.append(f"tests analysed : {total}")
|
|
26
|
+
lines.append(f"flaky (>= {min_runs} runs): {len(flaky)}")
|
|
27
|
+
lines.append("")
|
|
28
|
+
|
|
29
|
+
if not flaky:
|
|
30
|
+
lines.append("No flaky tests found. Your suite is behaving.")
|
|
31
|
+
lines.append("")
|
|
32
|
+
return "\n".join(lines)
|
|
33
|
+
|
|
34
|
+
lines.append(f"{'score':>6} {'bar':<10} {'runs':>4} {'pass':>4} {'fail':>4} test")
|
|
35
|
+
lines.append("-" * 60)
|
|
36
|
+
for s in flaky:
|
|
37
|
+
lines.append(
|
|
38
|
+
f"{s.score:>6.2f} {_bar(s.score):<10} {s.considered:>4} "
|
|
39
|
+
f"{s.passed:>4} {s.failures:>4} {s.display_name}"
|
|
40
|
+
)
|
|
41
|
+
|
|
42
|
+
# Show the most recent failure message for the worst offender - the
|
|
43
|
+
# thing a developer actually wants to read.
|
|
44
|
+
worst = flaky[0]
|
|
45
|
+
if worst.last_message:
|
|
46
|
+
lines.append("")
|
|
47
|
+
lines.append(f"latest failure in {worst.display_name}:")
|
|
48
|
+
lines.append(f" {worst.last_message}")
|
|
49
|
+
|
|
50
|
+
lines.append("")
|
|
51
|
+
return "\n".join(lines)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _stats_to_dict(s: FlakeStats) -> dict:
|
|
55
|
+
return {
|
|
56
|
+
"key": s.key,
|
|
57
|
+
"name": s.name,
|
|
58
|
+
"suite": s.suite,
|
|
59
|
+
"classname": s.classname,
|
|
60
|
+
"runs": s.considered,
|
|
61
|
+
"passed": s.passed,
|
|
62
|
+
"failed": s.failed,
|
|
63
|
+
"errored": s.errored,
|
|
64
|
+
"skipped": s.skipped,
|
|
65
|
+
"score": round(s.score, 4),
|
|
66
|
+
"last_message": s.last_message,
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def render_json(stats: Sequence[FlakeStats], min_runs: int) -> str:
|
|
71
|
+
flaky = [s for s in stats if s.is_flaky() and s.considered >= min_runs]
|
|
72
|
+
payload = {
|
|
73
|
+
"schema_version": 1,
|
|
74
|
+
"summary": {
|
|
75
|
+
"tests_analysed": len(stats),
|
|
76
|
+
"flaky": len(flaky),
|
|
77
|
+
"min_runs": min_runs,
|
|
78
|
+
},
|
|
79
|
+
"flaky_tests": [_stats_to_dict(s) for s in flaky],
|
|
80
|
+
"all_tests": [_stats_to_dict(s) for s in stats],
|
|
81
|
+
}
|
|
82
|
+
return json.dumps(payload, indent=2)
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def render_markdown(stats: Sequence[FlakeStats], min_runs: int) -> str:
|
|
86
|
+
flaky = [s for s in stats if s.is_flaky() and s.considered >= min_runs]
|
|
87
|
+
lines = ["## flakeguard report", ""]
|
|
88
|
+
if not flaky:
|
|
89
|
+
lines.append("No flaky tests found.")
|
|
90
|
+
return "\n".join(lines)
|
|
91
|
+
lines.append(f"**{len(flaky)}** flaky test(s) found across {len(stats)} analysed.")
|
|
92
|
+
lines.append("")
|
|
93
|
+
lines.append("| score | runs | pass | fail | test |")
|
|
94
|
+
lines.append("|------:|-----:|-----:|-----:|:-----|")
|
|
95
|
+
for s in flaky:
|
|
96
|
+
lines.append(
|
|
97
|
+
f"| {s.score:.2f} | {s.considered} | {s.passed} | {s.failures} | `{s.display_name}` |"
|
|
98
|
+
)
|
|
99
|
+
lines.append("")
|
|
100
|
+
return "\n".join(lines)
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
RENDERERS = {
|
|
104
|
+
"text": render_text,
|
|
105
|
+
"json": render_json,
|
|
106
|
+
"markdown": render_markdown,
|
|
107
|
+
"md": render_markdown,
|
|
108
|
+
}
|
flakeguard/scoring.py
ADDED
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
"""The heart of flakeguard: turning many runs into a flakiness score.
|
|
2
|
+
|
|
3
|
+
The idea is simple and explainable. Take every test, look at how it
|
|
4
|
+
behaved across all the runs we were given, and ask: *how often does it
|
|
5
|
+
disagree with itself?*
|
|
6
|
+
|
|
7
|
+
We ignore skips (a skipped test tells us nothing about flakiness) and
|
|
8
|
+
count only passes vs. non-passes. A test that always passes, or always
|
|
9
|
+
fails, is **not flaky** - it is consistent. Flakiness is disagreement.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
from collections import defaultdict
|
|
15
|
+
from dataclasses import dataclass, field
|
|
16
|
+
from typing import Iterable, List, Optional
|
|
17
|
+
|
|
18
|
+
from .models import Outcome, TestCase, join_unique
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@dataclass
|
|
22
|
+
class FlakeStats:
|
|
23
|
+
"""Aggregated behaviour of one test across all observed runs."""
|
|
24
|
+
|
|
25
|
+
key: str
|
|
26
|
+
name: str
|
|
27
|
+
suite: str
|
|
28
|
+
classname: str
|
|
29
|
+
runs: int = 0
|
|
30
|
+
passed: int = 0
|
|
31
|
+
failed: int = 0
|
|
32
|
+
errored: int = 0
|
|
33
|
+
skipped: int = 0
|
|
34
|
+
score: float = 0.0
|
|
35
|
+
last_message: Optional[str] = None
|
|
36
|
+
messages: List[str] = field(default_factory=list)
|
|
37
|
+
|
|
38
|
+
@property
|
|
39
|
+
def considered(self) -> int:
|
|
40
|
+
"""Runs that actually inform flakiness (passes + failures + errors)."""
|
|
41
|
+
return self.passed + self.failed + self.errored
|
|
42
|
+
|
|
43
|
+
@property
|
|
44
|
+
def failures(self) -> int:
|
|
45
|
+
"""Non-passes that were not skips (failures + errors)."""
|
|
46
|
+
return self.failed + self.errored
|
|
47
|
+
|
|
48
|
+
@property
|
|
49
|
+
def display_name(self) -> str:
|
|
50
|
+
return join_unique([self.classname, self.name]) or self.name or self.key
|
|
51
|
+
|
|
52
|
+
def is_flaky(self, min_score: float = 0.0) -> bool:
|
|
53
|
+
"""A test is flaky if it both passed and failed, and clears the bar."""
|
|
54
|
+
return self.passed > 0 and self.failures > 0 and self.score > min_score
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _score(passed: int, failures: int) -> float:
|
|
58
|
+
"""Flakiness score in [0, 1].
|
|
59
|
+
|
|
60
|
+
Defined as ``2 * min(passes, failures) / (passes + failures)``.
|
|
61
|
+
|
|
62
|
+
* all passes, or all failures -> 0.0 (consistent, not flaky)
|
|
63
|
+
* perfectly alternating 50/50 -> 1.0 (maximally flaky)
|
|
64
|
+
|
|
65
|
+
Doubling means the score reads the way people expect: a coin-flip test
|
|
66
|
+
is "100% flaky", not "50% flaky".
|
|
67
|
+
"""
|
|
68
|
+
total = passed + failures
|
|
69
|
+
if total == 0:
|
|
70
|
+
return 0.0
|
|
71
|
+
return min(1.0, 2.0 * min(passed, failures) / total)
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def analyze(runs: Iterable[Iterable[TestCase]]) -> List[FlakeStats]:
|
|
75
|
+
"""Aggregate several runs of test cases into per-test flake statistics.
|
|
76
|
+
|
|
77
|
+
``runs`` is an iterable of runs; each run is an iterable of
|
|
78
|
+
:class:`TestCase`. Returns stats for every distinct test seen, sorted
|
|
79
|
+
most-flaky first.
|
|
80
|
+
"""
|
|
81
|
+
agg: dict[str, FlakeStats] = {}
|
|
82
|
+
|
|
83
|
+
for run in runs:
|
|
84
|
+
seen_in_run: set[str] = set()
|
|
85
|
+
for case in run:
|
|
86
|
+
key = case.key
|
|
87
|
+
# If a framework reports the same key twice in one run (e.g.
|
|
88
|
+
# parametrised tests without unique names) count it once.
|
|
89
|
+
if key in seen_in_run:
|
|
90
|
+
continue
|
|
91
|
+
seen_in_run.add(key)
|
|
92
|
+
|
|
93
|
+
stats = agg.get(key)
|
|
94
|
+
if stats is None:
|
|
95
|
+
stats = FlakeStats(
|
|
96
|
+
key=key, name=case.name, suite=case.suite, classname=case.classname
|
|
97
|
+
)
|
|
98
|
+
agg[key] = stats
|
|
99
|
+
|
|
100
|
+
stats.runs += 1
|
|
101
|
+
if case.outcome is Outcome.PASSED:
|
|
102
|
+
stats.passed += 1
|
|
103
|
+
elif case.outcome is Outcome.FAILED:
|
|
104
|
+
stats.failed += 1
|
|
105
|
+
elif case.outcome is Outcome.ERROR:
|
|
106
|
+
stats.errored += 1
|
|
107
|
+
else:
|
|
108
|
+
stats.skipped += 1
|
|
109
|
+
|
|
110
|
+
if case.message:
|
|
111
|
+
stats.last_message = case.message
|
|
112
|
+
if case.message not in stats.messages:
|
|
113
|
+
stats.messages.append(case.message)
|
|
114
|
+
|
|
115
|
+
for stats in agg.values():
|
|
116
|
+
stats.score = _score(stats.passed, stats.failures)
|
|
117
|
+
|
|
118
|
+
return sorted(
|
|
119
|
+
agg.values(),
|
|
120
|
+
key=lambda s: (s.score, s.failures, s.runs),
|
|
121
|
+
reverse=True,
|
|
122
|
+
)
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: flakeguard
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Find, rank, and quarantine flaky tests from any language's test reports.
|
|
5
|
+
Project-URL: Homepage, https://github.com/PrathameshKulkarni2k4/flakeguard
|
|
6
|
+
Author: Prathamesh Kulkarni
|
|
7
|
+
License: MIT
|
|
8
|
+
License-File: LICENSE
|
|
9
|
+
Keywords: ci,developer-tools,flaky-tests,junit,test-reports,testing
|
|
10
|
+
Classifier: Development Status :: 3 - Alpha
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Topic :: Software Development :: Testing
|
|
15
|
+
Requires-Python: >=3.9
|
|
16
|
+
Provides-Extra: dev
|
|
17
|
+
Requires-Dist: pytest>=7; extra == 'dev'
|
|
18
|
+
Description-Content-Type: text/markdown
|
|
19
|
+
|
|
20
|
+
# flakeguard
|
|
21
|
+
|
|
22
|
+
### Your test suite is lying to you.
|
|
23
|
+
|
|
24
|
+

|
|
25
|
+
|
|
26
|
+
Flakeguard finds the tests that **randomly pass and fail**, ranks them by how
|
|
27
|
+
unreliable they are, and shows you why — so you can trust your tests again.
|
|
28
|
+
|
|
29
|
+
**Free · private · works with any language.** No account, no sign-up, and
|
|
30
|
+
nothing leaves your machine.
|
|
31
|
+
|
|
32
|
+
[](https://github.com/PrathameshKulkarni2k4/flakeguard/actions/workflows/ci.yml)
|
|
33
|
+
[](LICENSE)
|
|
34
|
+
[](https://www.python.org/)
|
|
35
|
+
|
|
36
|
+
---
|
|
37
|
+
|
|
38
|
+
## The problem, in 30 seconds
|
|
39
|
+
|
|
40
|
+
You have a test that fails once in a while. You re-run it. It passes. Was
|
|
41
|
+
that a real bug? A flaky test? Nobody knows — so everyone ignores it, and
|
|
42
|
+
that is exactly how real bugs hide.
|
|
43
|
+
|
|
44
|
+
A flaky test costs a team hours, and slowly destroys trust in the entire
|
|
45
|
+
test suite. Flakeguard fixes the hard half of the problem: it tells you
|
|
46
|
+
**exactly which tests are unreliable**, and how unreliable.
|
|
47
|
+
|
|
48
|
+
## Try it in 60 seconds
|
|
49
|
+
|
|
50
|
+
```bash
|
|
51
|
+
pipx install flakeguard
|
|
52
|
+
flakeguard scan ./reports # a folder of test reports
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
No reports yet? Run it against the examples bundled in this repo:
|
|
56
|
+
|
|
57
|
+
```bash
|
|
58
|
+
git clone https://github.com/PrathameshKulkarni2k4/flakeguard
|
|
59
|
+
cd flakeguard
|
|
60
|
+
flakeguard scan examples/reports
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
## What it does
|
|
64
|
+
|
|
65
|
+
- Reads your test results from **many runs** (the more runs, the sharper the
|
|
66
|
+
picture).
|
|
67
|
+
- Finds the tests that **disagree with themselves** across those runs.
|
|
68
|
+
- Gives each one a score from **0 (solid) to 1 (a coin-flip)**.
|
|
69
|
+
- Shows the **latest failure message**, so you can go fix it.
|
|
70
|
+
- Runs in your **CI** and can leave a comment on the pull request — or block
|
|
71
|
+
it when flakiness spikes.
|
|
72
|
+
|
|
73
|
+
## Works with any language
|
|
74
|
+
|
|
75
|
+
Flakeguard reads **JUnit XML**, the report format almost every test
|
|
76
|
+
framework can produce. One tool, every language:
|
|
77
|
+
|
|
78
|
+
| Language / framework | How to produce the report |
|
|
79
|
+
|---|---|
|
|
80
|
+
| Python (pytest) | `pytest --junitxml=report.xml` |
|
|
81
|
+
| Java (Maven / Gradle) | Surefire writes JUnit XML automatically |
|
|
82
|
+
| Go | `gotestsum --junitfile report.xml` |
|
|
83
|
+
| JavaScript (Jest) | the `jest-junit` reporter |
|
|
84
|
+
| Ruby (RSpec) | the `rspec_junit_formatter` gem |
|
|
85
|
+
| .NET, PHPUnit, … | built-in or plugin JUnit output |
|
|
86
|
+
|
|
87
|
+
## Use in CI (GitHub Actions)
|
|
88
|
+
|
|
89
|
+
```yaml
|
|
90
|
+
- name: Flakeguard
|
|
91
|
+
uses: PrathameshKulkarni2k4/flakeguard@main
|
|
92
|
+
with:
|
|
93
|
+
reports: reports # dir/glob of JUnit XML files
|
|
94
|
+
min-runs: 3 # ignore tests seen in fewer runs
|
|
95
|
+
max-flaky: 0 # fail if any flaky test is found (-1 = never fail)
|
|
96
|
+
comment: "true" # post the report as a PR comment
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
See [`.github/workflows/example.yml`](.github/workflows/example.yml) for a
|
|
100
|
+
complete workflow that runs a suite several times and feeds the reports in.
|
|
101
|
+
|
|
102
|
+
To gate locally or in any other CI, use the exit code:
|
|
103
|
+
|
|
104
|
+
```bash
|
|
105
|
+
flakeguard scan ./reports --fail-on-flaky # exits 1 if anything is flaky
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
## How the score works
|
|
109
|
+
|
|
110
|
+
For each test we count passes vs. non-passes across all runs (skips are
|
|
111
|
+
ignored). The flakiness score is:
|
|
112
|
+
|
|
113
|
+
```
|
|
114
|
+
score = 2 * min(passes, failures) / (passes + failures)
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
- always passes, or always fails → **0.0** (consistent, *not* flaky)
|
|
118
|
+
- flips on every run (50/50) → **1.0** (maximally flaky)
|
|
119
|
+
|
|
120
|
+
It is deliberately explainable — no black box. You can read the number and
|
|
121
|
+
know exactly how it was derived.
|
|
122
|
+
|
|
123
|
+
## Roadmap
|
|
124
|
+
|
|
125
|
+
- [x] JUnit XML parsing, flakiness scoring, text/JSON/Markdown reports
|
|
126
|
+
- [x] GitHub Action with PR comments and a flakiness budget
|
|
127
|
+
- [ ] TAP and native JSON report formats
|
|
128
|
+
- [ ] CI history so it learns across builds
|
|
129
|
+
- [ ] Failure-signature clustering (group the *same* flake across tests)
|
|
130
|
+
- [ ] Quarantine file so known flakes are set aside, not deleted
|
|
131
|
+
|
|
132
|
+
## Contributing
|
|
133
|
+
|
|
134
|
+
Contributions are very welcome — see [CONTRIBUTING.md](CONTRIBUTING.md).
|
|
135
|
+
Adding support for a new report format is a great first contribution and is
|
|
136
|
+
designed to be small. Please be kind; this project follows a
|
|
137
|
+
[Code of Conduct](CODE_OF_CONDUCT.md).
|
|
138
|
+
|
|
139
|
+
## License
|
|
140
|
+
|
|
141
|
+
MIT — see [LICENSE](LICENSE).
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
flakeguard/__init__.py,sha256=EjV4fmMO0AcBER6iHiLs9OVJwCPO6wkiMcfiJCvjo_8,509
|
|
2
|
+
flakeguard/__main__.py,sha256=jyPbPPn4AGYrVhojF1k7rEKBnFSjycbV_azmzH2lZ4A,157
|
|
3
|
+
flakeguard/cli.py,sha256=OkWOUS6Oqd64Jr3AD9affnFQ9dLSDgSkWzhvk-2xyOE,4871
|
|
4
|
+
flakeguard/models.py,sha256=HVJV7g06GKrWsx0K8gJ8sC0Rzcl5SmGNHlXsmB4Bdgk,1751
|
|
5
|
+
flakeguard/report.py,sha256=Syr06YulzcIuYD1tYDZKcYgm_h3TeL6YYsMf9XsUtVg,3301
|
|
6
|
+
flakeguard/scoring.py,sha256=EWInlr2hrAzLJVY1D9_OW6F9n--kAhBDT7LFoBSeklU,3927
|
|
7
|
+
flakeguard/parsers/__init__.py,sha256=TEnhrPd3Xih53TD4q_Ak07KlWFdUMjKDzvRUCeUtP-8,622
|
|
8
|
+
flakeguard/parsers/junit_xml.py,sha256=wpzHGRz-MWWkt8U7g_sLtoh4FoqfoBPdhk9PO6PXyQw,3527
|
|
9
|
+
flakeguard-0.1.0.dist-info/METADATA,sha256=Vc7UDxvaqE_iSIJYlzsG8u_Azxru_XLRDeigtpivUgU,4876
|
|
10
|
+
flakeguard-0.1.0.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
|
|
11
|
+
flakeguard-0.1.0.dist-info/entry_points.txt,sha256=Ns1hCKxs63kg6RRdZz2yZNk7y2FcA4BrT27ImLzjYoI,51
|
|
12
|
+
flakeguard-0.1.0.dist-info/licenses/LICENSE,sha256=iOnuFk0Tr21BAqhvxGbrkBFTkM_8EJlb4WqrtOBbknc,1076
|
|
13
|
+
flakeguard-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Prathamesh Kulkarni
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|