flakeguard 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
flakeguard/__init__.py ADDED
@@ -0,0 +1,21 @@
1
+ """flakeguard - find, rank, and quarantine flaky tests.
2
+
3
+ A language-agnostic tool: point it at the test reports your CI already
4
+ produces (JUnit XML and friends) and it tells you which tests are flaky,
5
+ how flaky, and why it thinks so.
6
+ """
7
+
8
+ __version__ = "0.1.0"
9
+
10
+ from .models import Outcome, TestCase
11
+ from .scoring import FlakeStats, analyze
12
+ from .parsers.junit_xml import parse_junit_xml
13
+
14
+ __all__ = [
15
+ "__version__",
16
+ "Outcome",
17
+ "TestCase",
18
+ "FlakeStats",
19
+ "analyze",
20
+ "parse_junit_xml",
21
+ ]
flakeguard/__main__.py ADDED
@@ -0,0 +1,6 @@
1
+ """Allow ``python -m flakeguard`` to work like the ``flakeguard`` script."""
2
+
3
+ from .cli import main
4
+
5
+ if __name__ == "__main__":
6
+ raise SystemExit(main())
flakeguard/cli.py ADDED
@@ -0,0 +1,149 @@
1
+ """Command-line entry point.
2
+
3
+ flakeguard scan ./reports
4
+ flakeguard scan run1.xml run2.xml run3.xml --format json
5
+ flakeguard scan ./ci-artifacts --min-runs 5 --min-score 0.2
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import argparse
11
+ import glob
12
+ import os
13
+ import sys
14
+ from typing import List
15
+
16
+ from . import __version__
17
+ from .parsers import by_suffix
18
+ from .report import RENDERERS
19
+ from .scoring import analyze
20
+
21
+
22
+ def _discover(paths: List[str]) -> List[str]:
23
+ """Expand the given paths into a sorted list of report files.
24
+
25
+ Directories are walked recursively; files are taken as-is. Sorting
26
+ keeps run order deterministic so results are reproducible.
27
+ """
28
+ found: List[str] = []
29
+ for path in paths:
30
+ if os.path.isdir(path):
31
+ for root, _dirs, files in os.walk(path):
32
+ for fname in files:
33
+ if os.path.splitext(fname)[1].lower() in by_suffix:
34
+ found.append(os.path.join(root, fname))
35
+ else:
36
+ matches = sorted(glob.glob(path))
37
+ if matches:
38
+ found.extend(matches)
39
+ elif os.path.exists(path):
40
+ found.append(path)
41
+ return sorted(set(found))
42
+
43
+
44
+ def _load_runs(files: List[str]) -> tuple[list, list]:
45
+ """Parse each file into a run. Returns (runs, errors)."""
46
+ runs, errors = [], []
47
+ for path in files:
48
+ suffix = os.path.splitext(path)[1].lower()
49
+ parser = by_suffix.get(suffix)
50
+ if parser is None:
51
+ errors.append(f"{path}: unsupported format '{suffix}'")
52
+ continue
53
+ if not os.path.exists(path):
54
+ errors.append(f"{path}: file not found")
55
+ continue
56
+ try:
57
+ runs.append(parser(path))
58
+ except Exception as exc: # noqa: BLE001 - report, don't crash
59
+ errors.append(f"{path}: {exc}")
60
+ return runs, errors
61
+
62
+
63
+ def build_parser() -> argparse.ArgumentParser:
64
+ p = argparse.ArgumentParser(
65
+ prog="flakeguard",
66
+ description="Find, rank, and quarantine flaky tests from test reports.",
67
+ )
68
+ p.add_argument("--version", action="version", version=f"flakeguard {__version__}")
69
+ sub = p.add_subparsers(dest="command", required=True)
70
+
71
+ scan = sub.add_parser("scan", help="analyse one or more report files/dirs")
72
+ scan.add_argument("paths", nargs="+", help="report files, globs, or directories")
73
+ scan.add_argument(
74
+ "--min-runs",
75
+ type=int,
76
+ default=2,
77
+ help="minimum runs a test must appear in to be reported (default: 2)",
78
+ )
79
+ scan.add_argument(
80
+ "--min-score",
81
+ type=float,
82
+ default=0.0,
83
+ help="only report tests at or above this flakiness score, 0-1 (default: 0)",
84
+ )
85
+ scan.add_argument(
86
+ "--format",
87
+ choices=sorted(RENDERERS),
88
+ default="text",
89
+ help="output format (default: text)",
90
+ )
91
+ scan.add_argument(
92
+ "--output", "-o", default=None, help="write the report to a file instead of stdout"
93
+ )
94
+ scan.add_argument(
95
+ "--fail-on-flaky",
96
+ action="store_true",
97
+ help="exit with code 1 if any flaky tests are found (for CI gating)",
98
+ )
99
+ return p
100
+
101
+
102
+ def main(argv: List[str] | None = None) -> int:
103
+ args = build_parser().parse_args(argv)
104
+
105
+ files = _discover(args.paths)
106
+ if not files:
107
+ print(
108
+ "flakeguard: no test report files found.\n"
109
+ " I look for JUnit XML (.xml) files - give me a file, a glob, or a\n"
110
+ " directory. Need to make one? Most frameworks can, for example:\n"
111
+ " pytest --junitxml=report.xml",
112
+ file=sys.stderr,
113
+ )
114
+ return 2
115
+
116
+ runs, errors = _load_runs(files)
117
+ for err in errors:
118
+ print(f"flakeguard: warning: {err}", file=sys.stderr)
119
+ if not runs:
120
+ print(
121
+ f"flakeguard: none of the {len(files)} file(s) could be parsed.\n"
122
+ " See the warnings above - is the file valid JUnit XML?",
123
+ file=sys.stderr,
124
+ )
125
+ return 2
126
+
127
+ stats = analyze(runs)
128
+ # Apply the minimum-score filter on top of the renderers' min-runs filter.
129
+ if args.min_score > 0:
130
+ stats = [s for s in stats if s.score >= args.min_score or not s.is_flaky()]
131
+
132
+ report = RENDERERS[args.format](stats, args.min_runs)
133
+
134
+ if args.output:
135
+ with open(args.output, "w", encoding="utf-8") as fh:
136
+ fh.write(report + "\n")
137
+ print(f"flakeguard: wrote {args.output} ({len(runs)} runs)", file=sys.stderr)
138
+ else:
139
+ print(report)
140
+
141
+ flaky_count = sum(1 for s in stats if s.is_flaky() and s.considered >= args.min_runs)
142
+ if args.fail_on_flaky and flaky_count > 0:
143
+ print(f"flakeguard: {flaky_count} flaky test(s) found", file=sys.stderr)
144
+ return 1
145
+ return 0
146
+
147
+
148
+ if __name__ == "__main__": # pragma: no cover
149
+ raise SystemExit(main())
flakeguard/models.py ADDED
@@ -0,0 +1,60 @@
1
+ """Core data types shared across parsers, scoring and reporting."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from dataclasses import dataclass
6
+ from enum import Enum
7
+ from typing import Iterable, Optional
8
+
9
+
10
+ class Outcome(str, Enum):
11
+ """The result of a single test in a single run."""
12
+
13
+ PASSED = "passed"
14
+ FAILED = "failed"
15
+ ERROR = "error"
16
+ SKIPPED = "skipped"
17
+
18
+
19
+ def join_unique(parts: Iterable[str]) -> str:
20
+ """Join non-empty parts with '::', dropping repeats.
21
+
22
+ Frameworks disagree about how they name things: pytest sets a suite and
23
+ a classname, while jest-junit sets classname == name. Dropping repeats
24
+ keeps a test's identity stable and readable for both.
25
+ """
26
+ seen: list[str] = []
27
+ for part in parts:
28
+ if part and part not in seen:
29
+ seen.append(part)
30
+ return "::".join(seen)
31
+
32
+
33
+ @dataclass(frozen=True)
34
+ class TestCase:
35
+ """One test's result, as observed in one run.
36
+
37
+ ``suite`` is the file/grouping the framework reported (e.g. a test
38
+ file), ``classname`` is the class or module, and ``name`` is the test
39
+ itself. Together they form a stable identity so the same test can be
40
+ matched across runs.
41
+ """
42
+
43
+ name: str
44
+ suite: str = ""
45
+ classname: str = ""
46
+ outcome: Outcome = Outcome.PASSED
47
+ duration: float = 0.0
48
+ message: Optional[str] = None
49
+
50
+ # Not a field (no annotation): tells pytest this is data, not a test.
51
+ __test__ = False
52
+
53
+ @property
54
+ def key(self) -> str:
55
+ """A stable identity for this test across runs."""
56
+ return join_unique([self.suite, self.classname, self.name]) or "<unnamed>"
57
+
58
+ @property
59
+ def display_name(self) -> str:
60
+ return join_unique([self.classname, self.name]) or self.name or "<unnamed>"
@@ -0,0 +1,21 @@
1
+ """Report parsers.
2
+
3
+ Each parser turns a framework's native test report into a list of
4
+ :class:`flakeguard.models.TestCase`. Adding support for a new framework
5
+ means adding one module here and registering it in ``by_suffix``.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ from typing import Callable, List
11
+
12
+ from ..models import TestCase
13
+ from .junit_xml import parse_junit_xml
14
+
15
+ # Map of file suffix -> parser callable. Used by the CLI to auto-detect
16
+ # the format of each report file it is handed.
17
+ by_suffix: dict[str, Callable[[str], List[TestCase]]] = {
18
+ ".xml": parse_junit_xml,
19
+ }
20
+
21
+ __all__ = ["parse_junit_xml", "by_suffix"]
@@ -0,0 +1,98 @@
1
+ """Parser for JUnit XML - the de-facto interchange format for test results.
2
+
3
+ Almost every test framework can emit this: pytest (``--junitxml``), Maven
4
+ and Gradle Surefire, Go's ``gotestsum --junitfile``, Jest
5
+ (``jest-junit``), RSpec (``rspec_junit_formatter``), .NET, PHPUnit, and
6
+ more. That is what makes flakeguard language-agnostic.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import xml.etree.ElementTree as ET
12
+ from typing import Iterator, List, Optional, Tuple
13
+
14
+ from ..models import Outcome, TestCase
15
+
16
+
17
+ def _localname(tag: str) -> str:
18
+ """Strip an XML namespace, if present (``{ns}testcase`` -> ``testcase``)."""
19
+ return tag.split("}")[-1].lower()
20
+
21
+
22
+ def _outcome_and_message(case: ET.Element) -> Tuple[Outcome, Optional[str]]:
23
+ """Decide a test's outcome from its child elements.
24
+
25
+ Per the JUnit schema, a ``<failure>`` means an assertion failed, an
26
+ ``<error>`` means the test blew up unexpectedly, and a ``<skipped>``
27
+ means it did not run. A ``<testcase>`` with none of these passed.
28
+ """
29
+ for child in case:
30
+ tag = _localname(child.tag)
31
+ if tag in ("failure", "error"):
32
+ text = (child.get("message") or (child.text or "").strip())
33
+ text = " ".join(text.split()) # collapse whitespace/newlines
34
+ return (
35
+ Outcome.FAILED if tag == "failure" else Outcome.ERROR,
36
+ text[:300] or None,
37
+ )
38
+ if tag == "skipped":
39
+ return Outcome.SKIPPED, child.get("message")
40
+ return Outcome.PASSED, None
41
+
42
+
43
+ def _iter_suites(suite: ET.Element) -> Iterator[ET.Element]:
44
+ """Yield every ``<testsuite>`` at any depth (they can nest)."""
45
+ yield suite
46
+ for child in suite:
47
+ if _localname(child.tag) == "testsuite":
48
+ yield from _iter_suites(child)
49
+
50
+
51
+ def _collect_cases(container: ET.Element, suite_name: str) -> Iterator[TestCase]:
52
+ for child in container:
53
+ tag = _localname(child.tag)
54
+ if tag == "testcase":
55
+ outcome, message = _outcome_and_message(child)
56
+ raw_time = child.get("time")
57
+ try:
58
+ duration = float(raw_time) if raw_time else 0.0
59
+ except ValueError:
60
+ duration = 0.0
61
+ yield TestCase(
62
+ name=child.get("name") or "",
63
+ suite=suite_name,
64
+ classname=child.get("classname") or "",
65
+ outcome=outcome,
66
+ duration=duration,
67
+ message=message,
68
+ )
69
+ elif tag == "testsuite":
70
+ nested_name = child.get("name") or suite_name
71
+ yield from _collect_cases(child, nested_name)
72
+
73
+
74
+ def parse_junit_xml(path: str) -> List[TestCase]:
75
+ """Parse a JUnit XML file into a list of test cases.
76
+
77
+ Handles the three common root shapes: ``<testsuites>`` (a bundle of
78
+ suites), a bare ``<testsuite>``, and nested suites.
79
+ """
80
+ try:
81
+ root = ET.parse(path).getroot()
82
+ except ET.ParseError as exc:
83
+ raise ValueError(f"not valid XML ({exc})") from exc
84
+ root_tag = _localname(root.tag)
85
+
86
+ cases: List[TestCase] = []
87
+
88
+ if root_tag == "testsuites":
89
+ for suite in root:
90
+ if _localname(suite.tag) == "testsuite":
91
+ cases.extend(_collect_cases(suite, suite.get("name") or ""))
92
+ elif root_tag == "testsuite":
93
+ cases.extend(_collect_cases(root, root.get("name") or ""))
94
+ else:
95
+ # Unknown root: be permissive and look for testcases anywhere.
96
+ cases.extend(_collect_cases(root, ""))
97
+
98
+ return cases
flakeguard/report.py ADDED
@@ -0,0 +1,108 @@
1
+ """Render analysis results as text, JSON, or Markdown."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ from typing import List, Sequence
7
+
8
+ from .scoring import FlakeStats
9
+
10
+
11
+ def _bar(score: float, width: int = 10) -> str:
12
+ filled = round(score * width)
13
+ return "#" * filled + "." * (width - filled)
14
+
15
+
16
+ def render_text(stats: Sequence[FlakeStats], min_runs: int) -> str:
17
+ """A human-readable terminal report."""
18
+ flaky = [s for s in stats if s.is_flaky() and s.considered >= min_runs]
19
+
20
+ lines: List[str] = []
21
+ total = len(stats)
22
+ lines.append("")
23
+ lines.append("flakeguard report")
24
+ lines.append("=" * 60)
25
+ lines.append(f"tests analysed : {total}")
26
+ lines.append(f"flaky (>= {min_runs} runs): {len(flaky)}")
27
+ lines.append("")
28
+
29
+ if not flaky:
30
+ lines.append("No flaky tests found. Your suite is behaving.")
31
+ lines.append("")
32
+ return "\n".join(lines)
33
+
34
+ lines.append(f"{'score':>6} {'bar':<10} {'runs':>4} {'pass':>4} {'fail':>4} test")
35
+ lines.append("-" * 60)
36
+ for s in flaky:
37
+ lines.append(
38
+ f"{s.score:>6.2f} {_bar(s.score):<10} {s.considered:>4} "
39
+ f"{s.passed:>4} {s.failures:>4} {s.display_name}"
40
+ )
41
+
42
+ # Show the most recent failure message for the worst offender - the
43
+ # thing a developer actually wants to read.
44
+ worst = flaky[0]
45
+ if worst.last_message:
46
+ lines.append("")
47
+ lines.append(f"latest failure in {worst.display_name}:")
48
+ lines.append(f" {worst.last_message}")
49
+
50
+ lines.append("")
51
+ return "\n".join(lines)
52
+
53
+
54
+ def _stats_to_dict(s: FlakeStats) -> dict:
55
+ return {
56
+ "key": s.key,
57
+ "name": s.name,
58
+ "suite": s.suite,
59
+ "classname": s.classname,
60
+ "runs": s.considered,
61
+ "passed": s.passed,
62
+ "failed": s.failed,
63
+ "errored": s.errored,
64
+ "skipped": s.skipped,
65
+ "score": round(s.score, 4),
66
+ "last_message": s.last_message,
67
+ }
68
+
69
+
70
+ def render_json(stats: Sequence[FlakeStats], min_runs: int) -> str:
71
+ flaky = [s for s in stats if s.is_flaky() and s.considered >= min_runs]
72
+ payload = {
73
+ "schema_version": 1,
74
+ "summary": {
75
+ "tests_analysed": len(stats),
76
+ "flaky": len(flaky),
77
+ "min_runs": min_runs,
78
+ },
79
+ "flaky_tests": [_stats_to_dict(s) for s in flaky],
80
+ "all_tests": [_stats_to_dict(s) for s in stats],
81
+ }
82
+ return json.dumps(payload, indent=2)
83
+
84
+
85
+ def render_markdown(stats: Sequence[FlakeStats], min_runs: int) -> str:
86
+ flaky = [s for s in stats if s.is_flaky() and s.considered >= min_runs]
87
+ lines = ["## flakeguard report", ""]
88
+ if not flaky:
89
+ lines.append("No flaky tests found.")
90
+ return "\n".join(lines)
91
+ lines.append(f"**{len(flaky)}** flaky test(s) found across {len(stats)} analysed.")
92
+ lines.append("")
93
+ lines.append("| score | runs | pass | fail | test |")
94
+ lines.append("|------:|-----:|-----:|-----:|:-----|")
95
+ for s in flaky:
96
+ lines.append(
97
+ f"| {s.score:.2f} | {s.considered} | {s.passed} | {s.failures} | `{s.display_name}` |"
98
+ )
99
+ lines.append("")
100
+ return "\n".join(lines)
101
+
102
+
103
+ RENDERERS = {
104
+ "text": render_text,
105
+ "json": render_json,
106
+ "markdown": render_markdown,
107
+ "md": render_markdown,
108
+ }
flakeguard/scoring.py ADDED
@@ -0,0 +1,122 @@
1
+ """The heart of flakeguard: turning many runs into a flakiness score.
2
+
3
+ The idea is simple and explainable. Take every test, look at how it
4
+ behaved across all the runs we were given, and ask: *how often does it
5
+ disagree with itself?*
6
+
7
+ We ignore skips (a skipped test tells us nothing about flakiness) and
8
+ count only passes vs. non-passes. A test that always passes, or always
9
+ fails, is **not flaky** - it is consistent. Flakiness is disagreement.
10
+ """
11
+
12
+ from __future__ import annotations
13
+
14
+ from collections import defaultdict
15
+ from dataclasses import dataclass, field
16
+ from typing import Iterable, List, Optional
17
+
18
+ from .models import Outcome, TestCase, join_unique
19
+
20
+
21
+ @dataclass
22
+ class FlakeStats:
23
+ """Aggregated behaviour of one test across all observed runs."""
24
+
25
+ key: str
26
+ name: str
27
+ suite: str
28
+ classname: str
29
+ runs: int = 0
30
+ passed: int = 0
31
+ failed: int = 0
32
+ errored: int = 0
33
+ skipped: int = 0
34
+ score: float = 0.0
35
+ last_message: Optional[str] = None
36
+ messages: List[str] = field(default_factory=list)
37
+
38
+ @property
39
+ def considered(self) -> int:
40
+ """Runs that actually inform flakiness (passes + failures + errors)."""
41
+ return self.passed + self.failed + self.errored
42
+
43
+ @property
44
+ def failures(self) -> int:
45
+ """Non-passes that were not skips (failures + errors)."""
46
+ return self.failed + self.errored
47
+
48
+ @property
49
+ def display_name(self) -> str:
50
+ return join_unique([self.classname, self.name]) or self.name or self.key
51
+
52
+ def is_flaky(self, min_score: float = 0.0) -> bool:
53
+ """A test is flaky if it both passed and failed, and clears the bar."""
54
+ return self.passed > 0 and self.failures > 0 and self.score > min_score
55
+
56
+
57
+ def _score(passed: int, failures: int) -> float:
58
+ """Flakiness score in [0, 1].
59
+
60
+ Defined as ``2 * min(passes, failures) / (passes + failures)``.
61
+
62
+ * all passes, or all failures -> 0.0 (consistent, not flaky)
63
+ * perfectly alternating 50/50 -> 1.0 (maximally flaky)
64
+
65
+ Doubling means the score reads the way people expect: a coin-flip test
66
+ is "100% flaky", not "50% flaky".
67
+ """
68
+ total = passed + failures
69
+ if total == 0:
70
+ return 0.0
71
+ return min(1.0, 2.0 * min(passed, failures) / total)
72
+
73
+
74
+ def analyze(runs: Iterable[Iterable[TestCase]]) -> List[FlakeStats]:
75
+ """Aggregate several runs of test cases into per-test flake statistics.
76
+
77
+ ``runs`` is an iterable of runs; each run is an iterable of
78
+ :class:`TestCase`. Returns stats for every distinct test seen, sorted
79
+ most-flaky first.
80
+ """
81
+ agg: dict[str, FlakeStats] = {}
82
+
83
+ for run in runs:
84
+ seen_in_run: set[str] = set()
85
+ for case in run:
86
+ key = case.key
87
+ # If a framework reports the same key twice in one run (e.g.
88
+ # parametrised tests without unique names) count it once.
89
+ if key in seen_in_run:
90
+ continue
91
+ seen_in_run.add(key)
92
+
93
+ stats = agg.get(key)
94
+ if stats is None:
95
+ stats = FlakeStats(
96
+ key=key, name=case.name, suite=case.suite, classname=case.classname
97
+ )
98
+ agg[key] = stats
99
+
100
+ stats.runs += 1
101
+ if case.outcome is Outcome.PASSED:
102
+ stats.passed += 1
103
+ elif case.outcome is Outcome.FAILED:
104
+ stats.failed += 1
105
+ elif case.outcome is Outcome.ERROR:
106
+ stats.errored += 1
107
+ else:
108
+ stats.skipped += 1
109
+
110
+ if case.message:
111
+ stats.last_message = case.message
112
+ if case.message not in stats.messages:
113
+ stats.messages.append(case.message)
114
+
115
+ for stats in agg.values():
116
+ stats.score = _score(stats.passed, stats.failures)
117
+
118
+ return sorted(
119
+ agg.values(),
120
+ key=lambda s: (s.score, s.failures, s.runs),
121
+ reverse=True,
122
+ )
@@ -0,0 +1,141 @@
1
+ Metadata-Version: 2.5
2
+ Name: flakeguard
3
+ Version: 0.1.0
4
+ Summary: Find, rank, and quarantine flaky tests from any language's test reports.
5
+ Project-URL: Homepage, https://github.com/PrathameshKulkarni2k4/flakeguard
6
+ Author: Prathamesh Kulkarni
7
+ License: MIT
8
+ License-File: LICENSE
9
+ Keywords: ci,developer-tools,flaky-tests,junit,test-reports,testing
10
+ Classifier: Development Status :: 3 - Alpha
11
+ Classifier: Intended Audience :: Developers
12
+ Classifier: License :: OSI Approved :: MIT License
13
+ Classifier: Programming Language :: Python :: 3
14
+ Classifier: Topic :: Software Development :: Testing
15
+ Requires-Python: >=3.9
16
+ Provides-Extra: dev
17
+ Requires-Dist: pytest>=7; extra == 'dev'
18
+ Description-Content-Type: text/markdown
19
+
20
+ # flakeguard
21
+
22
+ ### Your test suite is lying to you.
23
+
24
+ ![flakeguard demo](docs/demo.gif)
25
+
26
+ Flakeguard finds the tests that **randomly pass and fail**, ranks them by how
27
+ unreliable they are, and shows you why — so you can trust your tests again.
28
+
29
+ **Free · private · works with any language.** No account, no sign-up, and
30
+ nothing leaves your machine.
31
+
32
+ [![CI](https://github.com/PrathameshKulkarni2k4/flakeguard/actions/workflows/ci.yml/badge.svg)](https://github.com/PrathameshKulkarni2k4/flakeguard/actions/workflows/ci.yml)
33
+ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
34
+ [![Python 3.9+](https://img.shields.io/badge/python-3.9%2B-blue.svg)](https://www.python.org/)
35
+
36
+ ---
37
+
38
+ ## The problem, in 30 seconds
39
+
40
+ You have a test that fails once in a while. You re-run it. It passes. Was
41
+ that a real bug? A flaky test? Nobody knows — so everyone ignores it, and
42
+ that is exactly how real bugs hide.
43
+
44
+ A flaky test costs a team hours, and slowly destroys trust in the entire
45
+ test suite. Flakeguard fixes the hard half of the problem: it tells you
46
+ **exactly which tests are unreliable**, and how unreliable.
47
+
48
+ ## Try it in 60 seconds
49
+
50
+ ```bash
51
+ pipx install flakeguard
52
+ flakeguard scan ./reports # a folder of test reports
53
+ ```
54
+
55
+ No reports yet? Run it against the examples bundled in this repo:
56
+
57
+ ```bash
58
+ git clone https://github.com/PrathameshKulkarni2k4/flakeguard
59
+ cd flakeguard
60
+ flakeguard scan examples/reports
61
+ ```
62
+
63
+ ## What it does
64
+
65
+ - Reads your test results from **many runs** (the more runs, the sharper the
66
+ picture).
67
+ - Finds the tests that **disagree with themselves** across those runs.
68
+ - Gives each one a score from **0 (solid) to 1 (a coin-flip)**.
69
+ - Shows the **latest failure message**, so you can go fix it.
70
+ - Runs in your **CI** and can leave a comment on the pull request — or block
71
+ it when flakiness spikes.
72
+
73
+ ## Works with any language
74
+
75
+ Flakeguard reads **JUnit XML**, the report format almost every test
76
+ framework can produce. One tool, every language:
77
+
78
+ | Language / framework | How to produce the report |
79
+ |---|---|
80
+ | Python (pytest) | `pytest --junitxml=report.xml` |
81
+ | Java (Maven / Gradle) | Surefire writes JUnit XML automatically |
82
+ | Go | `gotestsum --junitfile report.xml` |
83
+ | JavaScript (Jest) | the `jest-junit` reporter |
84
+ | Ruby (RSpec) | the `rspec_junit_formatter` gem |
85
+ | .NET, PHPUnit, … | built-in or plugin JUnit output |
86
+
87
+ ## Use in CI (GitHub Actions)
88
+
89
+ ```yaml
90
+ - name: Flakeguard
91
+ uses: PrathameshKulkarni2k4/flakeguard@main
92
+ with:
93
+ reports: reports # dir/glob of JUnit XML files
94
+ min-runs: 3 # ignore tests seen in fewer runs
95
+ max-flaky: 0 # fail if any flaky test is found (-1 = never fail)
96
+ comment: "true" # post the report as a PR comment
97
+ ```
98
+
99
+ See [`.github/workflows/example.yml`](.github/workflows/example.yml) for a
100
+ complete workflow that runs a suite several times and feeds the reports in.
101
+
102
+ To gate locally or in any other CI, use the exit code:
103
+
104
+ ```bash
105
+ flakeguard scan ./reports --fail-on-flaky # exits 1 if anything is flaky
106
+ ```
107
+
108
+ ## How the score works
109
+
110
+ For each test we count passes vs. non-passes across all runs (skips are
111
+ ignored). The flakiness score is:
112
+
113
+ ```
114
+ score = 2 * min(passes, failures) / (passes + failures)
115
+ ```
116
+
117
+ - always passes, or always fails → **0.0** (consistent, *not* flaky)
118
+ - flips on every run (50/50) → **1.0** (maximally flaky)
119
+
120
+ It is deliberately explainable — no black box. You can read the number and
121
+ know exactly how it was derived.
122
+
123
+ ## Roadmap
124
+
125
+ - [x] JUnit XML parsing, flakiness scoring, text/JSON/Markdown reports
126
+ - [x] GitHub Action with PR comments and a flakiness budget
127
+ - [ ] TAP and native JSON report formats
128
+ - [ ] CI history so it learns across builds
129
+ - [ ] Failure-signature clustering (group the *same* flake across tests)
130
+ - [ ] Quarantine file so known flakes are set aside, not deleted
131
+
132
+ ## Contributing
133
+
134
+ Contributions are very welcome — see [CONTRIBUTING.md](CONTRIBUTING.md).
135
+ Adding support for a new report format is a great first contribution and is
136
+ designed to be small. Please be kind; this project follows a
137
+ [Code of Conduct](CODE_OF_CONDUCT.md).
138
+
139
+ ## License
140
+
141
+ MIT — see [LICENSE](LICENSE).
@@ -0,0 +1,13 @@
1
+ flakeguard/__init__.py,sha256=EjV4fmMO0AcBER6iHiLs9OVJwCPO6wkiMcfiJCvjo_8,509
2
+ flakeguard/__main__.py,sha256=jyPbPPn4AGYrVhojF1k7rEKBnFSjycbV_azmzH2lZ4A,157
3
+ flakeguard/cli.py,sha256=OkWOUS6Oqd64Jr3AD9affnFQ9dLSDgSkWzhvk-2xyOE,4871
4
+ flakeguard/models.py,sha256=HVJV7g06GKrWsx0K8gJ8sC0Rzcl5SmGNHlXsmB4Bdgk,1751
5
+ flakeguard/report.py,sha256=Syr06YulzcIuYD1tYDZKcYgm_h3TeL6YYsMf9XsUtVg,3301
6
+ flakeguard/scoring.py,sha256=EWInlr2hrAzLJVY1D9_OW6F9n--kAhBDT7LFoBSeklU,3927
7
+ flakeguard/parsers/__init__.py,sha256=TEnhrPd3Xih53TD4q_Ak07KlWFdUMjKDzvRUCeUtP-8,622
8
+ flakeguard/parsers/junit_xml.py,sha256=wpzHGRz-MWWkt8U7g_sLtoh4FoqfoBPdhk9PO6PXyQw,3527
9
+ flakeguard-0.1.0.dist-info/METADATA,sha256=Vc7UDxvaqE_iSIJYlzsG8u_Azxru_XLRDeigtpivUgU,4876
10
+ flakeguard-0.1.0.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
11
+ flakeguard-0.1.0.dist-info/entry_points.txt,sha256=Ns1hCKxs63kg6RRdZz2yZNk7y2FcA4BrT27ImLzjYoI,51
12
+ flakeguard-0.1.0.dist-info/licenses/LICENSE,sha256=iOnuFk0Tr21BAqhvxGbrkBFTkM_8EJlb4WqrtOBbknc,1076
13
+ flakeguard-0.1.0.dist-info/RECORD,,
@@ -0,0 +1,4 @@
1
+ Wheel-Version: 1.0
2
+ Generator: hatchling 1.32.4
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
@@ -0,0 +1,2 @@
1
+ [console_scripts]
2
+ flakeguard = flakeguard.cli:main
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Prathamesh Kulkarni
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.