unflake 0.6.1__py3-none-any.whl

unflake/__init__.py

@@ -0,0 +1,3 @@
+"""Unflake: detect, score, and quarantine flaky tests. Stdlib only."""
+
+__version__ = "0.6.1"

unflake/__main__.py

@@ -0,0 +1,14 @@
+"""Entry point. Works as `python -m unflake` AND as `python path/to/__main__.py`
+(the latter is how the npx wrapper invokes the vendored core)."""
+
+try:
+    from .cli import main
+except ImportError:  # pragma: no cover - path-invoked fallback
+    import os
+    import sys
+
+    sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+    from unflake.cli import main
+
+if __name__ == "__main__":
+    raise SystemExit(main())

unflake/cli.py

@@ -0,0 +1,214 @@
+"""`unflake` CLI. Stdlib only. Exit codes: 0 clean, 1 findings, 2 usage."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+
+from . import __version__
+from .ingest import load_runs
+from .patterns import SEVERITIES, Finding, scan_path
+from .quarantine import FRAMEWORKS, emit
+from .report import (
+    format_analyze_text,
+    format_sarif,
+    format_scan_json,
+    format_scan_text,
+)
+from .runner import collect
+from .scaffold import FRAMEWORKS as SCAFFOLD_FRAMEWORKS
+from .scaffold import init_framework
+from .score import score_runs
+
+ORDER = {"error": 0, "warning": 1, "info": 2}
+
+
+def _meets_threshold(f: Finding, fail_on: str) -> bool:
+    return ORDER[f.severity] <= ORDER[fail_on]
+
+
+def cmd_scan(args: argparse.Namespace) -> int:
+    findings: list[Finding] = []
+    for target in args.targets:
+        try:
+            findings.extend(scan_path(target, exclude=args.exclude))
+        except ValueError as exc:
+            print(f"unflake: error: {exc}", file=sys.stderr)
+            return 2
+    if args.format == "json":
+        print(format_scan_json(findings))
+    elif args.format == "sarif":
+        print(format_sarif(findings))
+    else:
+        print(format_scan_text(findings))
+    bad = [f for f in findings if _meets_threshold(f, args.fail_on)]
+    return 1 if bad else 0
+
+
+def _print_suite(suite, fmt: str) -> None:
+    if fmt == "json":
+        print(json.dumps(suite.to_dict(), indent=2))
+    elif fmt == "sarif":
+        print(format_sarif([], suite))
+    else:
+        print(format_analyze_text(suite))
+
+
+def _analyze_gate(suite, fail_on: str) -> int:
+    if fail_on == "never":
+        return 0
+    if fail_on == "flaky":
+        return 1 if (suite.flaky_count or suite.regression_count) else 0
+    return 0
+
+
+def _quarantine_note(suite, framework: str | None) -> None:
+    if not framework:
+        return
+    flaky_ids = [t.id for t in suite.tests if t.verdict == "flaky"]
+    if not flaky_ids:
+        return
+    print()
+    print(emit(framework, flaky_ids))
+
+
+def cmd_analyze(args: argparse.Namespace) -> int:
+    try:
+        runs = load_runs(args.results)
+    except ValueError as exc:
+        print(f"unflake: error: {exc}", file=sys.stderr)
+        return 2
+    if len(runs) < 2:
+        print("unflake: warning: only 1 run given — flakiness needs >= 2 runs "
+              "of the same suite to compare.", file=sys.stderr)
+    suite = score_runs(runs)
+    _print_suite(suite, args.format)
+    return _analyze_gate(suite, args.fail_on)
+
+
+def cmd_quarantine(args: argparse.Namespace) -> int:
+    try:
+        runs = load_runs(args.results)
+    except ValueError as exc:
+        print(f"unflake: error: {exc}", file=sys.stderr)
+        return 2
+    suite = score_runs(runs)
+    regressions = [t.id for t in suite.tests if t.verdict == "regression"]
+    if regressions:
+        print("# unflake quarantine: NOT quarantining regressions "
+              "(real failures — fix them):", file=sys.stderr)
+        for rid in regressions:
+            print(f"#  - {rid}", file=sys.stderr)
+    flaky_ids = [t.id for t in suite.tests if t.verdict == "flaky"]
+    if not flaky_ids:
+        print("# unflake quarantine: no flaky tests — nothing to quarantine.")
+        return 0
+    try:
+        print(emit(args.framework, flaky_ids))
+    except ValueError as exc:
+        print(f"unflake: error: {exc}", file=sys.stderr)
+        return 2
+    return 0
+
+
+def cmd_run(args: argparse.Namespace) -> int:
+    if args.runs < 2:
+        print("unflake: error: --runs must be >= 2 (one run proves nothing)",
+              file=sys.stderr)
+        return 2
+    cmd = list(args.test_command or [])
+    if cmd[:1] == ["--"]:
+        cmd = cmd[1:]  # argparse.REMAINDER keeps the separator
+    if not cmd:
+        print("unflake: error: no test command given after `--`", file=sys.stderr)
+        return 2
+    try:
+        files = collect(
+            cmd, runs=args.runs, out_dir=args.out_dir,
+            junit_flag=args.junit_flag, runner=args.runner, timeout=args.timeout,
+            progress=lambda m: print(f"unflake: {m}", file=sys.stderr),
+        )
+    except ValueError as exc:
+        print(f"unflake: error: {exc}", file=sys.stderr)
+        return 2
+    print(f"unflake: collected {len(files)} run(s) in {args.out_dir}", file=sys.stderr)
+    suite = score_runs(load_runs(files))
+    _print_suite(suite, args.format)
+    _quarantine_note(suite, args.quarantine)
+    return _analyze_gate(suite, args.fail_on)
+
+
+def cmd_init(args: argparse.Namespace) -> int:
+    try:
+        print(init_framework(args.framework, root=args.root, write=args.write))
+    except ValueError as exc:
+        print(f"unflake: error: {exc}", file=sys.stderr)
+        return 2
+    return 0
+
+
+def build_parser() -> argparse.ArgumentParser:
+    p = argparse.ArgumentParser(
+        prog="unflake",
+        description="Green CI without the rerun ritual — detect, score, quarantine flaky tests.",
+    )
+    p.add_argument("--version", action="version", version=f"%(prog)s {__version__}")
+    sub = p.add_subparsers(dest="command", required=True)
+
+    s = sub.add_parser("scan", help="Static scan for flake anti-patterns (no test execution).")
+    s.add_argument("targets", nargs="+", help="File(s) or directorie(s) to scan.")
+    s.add_argument("--exclude", action="append", default=[],
+                   help="Extra path-part glob to skip (repeatable; "
+                        "defaults already skip .git, node_modules, venvs, build dirs).")
+    s.add_argument("--format", choices=["text", "json", "sarif"], default="text")
+    s.add_argument("--fail-on", choices=list(SEVERITIES), default="warning",
+                   help="Exit 1 if any finding at/above this severity (default: warning).")
+    s.set_defaults(func=cmd_scan)
+
+    a = sub.add_parser("analyze", help="Score flakiness across >=2 result files (JUnit XML or JSON).")
+    a.add_argument("results", nargs="+", help="Result files from repeated runs, oldest first.")
+    a.add_argument("--format", choices=["text", "json", "sarif"], default="text")
+    a.add_argument("--fail-on", choices=["flaky", "never"], default="never",
+                   help="'flaky' exits 1 on flaky tests OR regressions.")
+    a.set_defaults(func=cmd_analyze)
+
+    q = sub.add_parser("quarantine", help="Emit quarantine config for flaky tests (never deletes).")
+    q.add_argument("results", nargs="+", help="Result files from repeated runs, oldest first.")
+    q.add_argument("--framework", choices=list(FRAMEWORKS), required=True)
+    q.set_defaults(func=cmd_quarantine)
+
+    r = sub.add_parser("run", help="Run your test command N times, then score (collects JUnit when possible).")
+    r.add_argument("--runs", type=int, default=3, help="Repeat count, >= 2 (default: 3).")
+    r.add_argument("--out-dir", default=".unflake/runs", help="Where per-run files land.")
+    r.add_argument("--junit-flag", default=None,
+                   help="Flag your runner uses for JUnit output, e.g. '--junitxml=' "
+                        "(auto-detected for pytest; omit for exit-code-only mode).")
+    r.add_argument("--runner", default=None,
+                   help="Verified runner preset (pytest, vitest, playwright, jest). "
+                        "Others: use --junit-flag.")
+    r.add_argument("--timeout", type=float, default=None, help="Per-run timeout in seconds.")
+    r.add_argument("--format", choices=["text", "json", "sarif"], default="text")
+    r.add_argument("--fail-on", choices=["flaky", "never"], default="flaky")
+    r.add_argument("--quarantine", choices=list(FRAMEWORKS), default=None,
+                   help="Also emit a quarantine snippet for this framework.")
+    r.add_argument("test_command", nargs=argparse.REMAINDER,
+                   help="Test command after `--`, e.g. `-- pytest tests/ -q`.")
+    r.set_defaults(func=cmd_run)
+
+    i = sub.add_parser("init", help="Scaffold seeded-RNG / frozen-time helpers for a framework.")
+    i.add_argument("--framework", choices=list(SCAFFOLD_FRAMEWORKS), required=True)
+    i.add_argument("--root", default=".", help="Project root to write into.")
+    i.add_argument("--write", action="store_true",
+                   help="Write files (idempotent). Without it, print to stdout.")
+    i.set_defaults(func=cmd_init)
+    return p
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = build_parser().parse_args(argv)
+    return int(args.func(args))
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

unflake/ingest.py

@@ -0,0 +1,95 @@
+"""Ingest test results from JUnit XML and generic JSON result files.
+
+Universal input: anything that can emit JUnit XML works
+(pytest --junitxml, jest-junit, vitest --reporter=junit, Playwright,
+JUnit/Gradle, go-junit-report, ...). No framework SDK required.
+"""
+
+from __future__ import annotations
+
+import json
+import xml.etree.ElementTree as ET
+from pathlib import Path
+
+PASS = "passed"
+FAIL = "failed"
+SKIP = "skipped"
+
+
+def _test_id(classname: str, name: str, filepath: str = "") -> str:
+    if classname:
+        return f"{classname}::{name}"
+    if filepath:
+        return f"{filepath}::{name}"
+    return name
+
+
+def parse_junit_xml(path: str | Path) -> dict[str, str]:
+    """Parse one JUnit XML file -> {test_id: status}."""
+    path = Path(path)
+    try:
+        tree = ET.parse(path)
+    except ET.ParseError as exc:
+        raise ValueError(f"{path}: not valid XML ({exc})") from exc
+    root = tree.getroot()
+    results: dict[str, str] = {}
+    suites = [root] if root.tag == "testsuite" else root.findall("testsuite")
+    if not suites and root.tag != "testsuite":
+        raise ValueError(f"{path}: no <testsuite> found, is this JUnit XML?")
+    for suite in suites:
+        for case in suite.iter("testcase"):
+            name = case.get("name", "unknown")
+            classname = case.get("classname", "") or ""
+            filepath = case.get("file", "") or ""
+            tid = _test_id(classname, name, filepath)
+            if case.find("failure") is not None or case.find("error") is not None:
+                status = FAIL
+            elif case.find("skipped") is not None:
+                status = SKIP
+            else:
+                status = PASS
+            results[tid] = status
+    return results
+
+
+def parse_generic_json(path: str | Path) -> dict[str, str]:
+    """Parse {"tests": [{"id": ..., "status": "passed|failed|skipped"}]}."""
+    path = Path(path)
+    try:
+        data = json.loads(path.read_text(encoding="utf-8"))
+    except json.JSONDecodeError as exc:
+        raise ValueError(f"{path}: not valid JSON ({exc})") from exc
+    items = data.get("tests", data if isinstance(data, list) else [])
+    results: dict[str, str] = {}
+    for item in items:
+        tid = str(item.get("id", item.get("name", "unknown")))
+        status = str(item.get("status", item.get("outcome", PASS))).lower()
+        if status in {"failure", "fail", "failed", "error"}:
+            status = FAIL
+        elif status in {"skip", "skipped", "xskip"}:
+            status = SKIP
+        else:
+            status = PASS
+        results[tid] = status
+    return results
+
+
+def parse_file(path: str | Path) -> dict[str, str]:
+    """Auto-detect format by extension/content -> {test_id: status}."""
+    path = Path(path)
+    if not path.exists():
+        raise ValueError(f"{path}: file not found")
+    if path.suffix.lower() == ".json":
+        return parse_generic_json(path)
+    head = path.read_bytes()[:2000].lstrip()
+    if head.startswith(b"{") or head.startswith(b"["):
+        return parse_generic_json(path)
+    return parse_junit_xml(path)
+
+
+def load_runs(paths: list[str | Path]) -> list[dict[str, str]]:
+    """Load N result files (e.g. N repeated runs of the same suite)."""
+    runs = []
+    for p in paths:
+        runs.append(parse_file(p))
+    return runs

unflake/patterns.py

@@ -0,0 +1,243 @@
+"""Static scan for flaky-test anti-patterns. No execution, no dependencies.
+
+Each rule is (id, severity, languages, check) so contributors can add
+a rule with a single function + one test. See CONTRIBUTING.md.
+"""
+
+from __future__ import annotations
+
+import fnmatch
+import io
+import re
+import tokenize
+from dataclasses import dataclass
+from pathlib import Path
+
+SEVERITIES = ("error", "warning", "info")
+
+TEST_FILE_HINT = re.compile(r"(test_|_test\.|\.spec\.|\.test\.)", re.IGNORECASE)
+TEST_DIR_HINT = re.compile(r"(^|/)(test|tests|__tests__|spec|e2e)($|/)")
+CODE_FENCE = "```"
+
+SCAN_EXTENSIONS = {".py", ".js", ".ts", ".tsx", ".jsx", ".mts", ".cts", ".go", ".rb"}
+
+PY_EXTS = (".py",)
+JS_EXTS = (".js", ".ts", ".tsx", ".jsx", ".mts", ".cts")
+ALL_EXTS = tuple(sorted(SCAN_EXTENSIONS))
+
+
+@dataclass
+class Finding:
+    rule: str
+    severity: str
+    file: str
+    line: int
+    message: str
+    fix: str = ""
+
+    def to_dict(self) -> dict:
+        return {
+            "rule": self.rule,
+            "severity": self.severity,
+            "file": self.file,
+            "line": self.line,
+            "message": self.message,
+            "fix": self.fix,
+        }
+
+
+@dataclass
+class Rule:
+    id: str
+    severity: str
+    pattern: re.Pattern
+    message: str
+    fix: str
+    extensions: tuple = ALL_EXTS
+    test_files_only: bool = True
+    # Precision valve: when downgrade_if matches the same line, the finding
+    # drops to downgrade_to with a note instead of firing at full severity.
+    downgrade_if: re.Pattern | None = None
+    downgrade_to: str = "info"
+    downgrade_note: str = ""
+
+
+RULES: list[Rule] = [
+    Rule(
+        "FLK001", "warning",
+        re.compile(r"\btime\.sleep\s*\("),
+        "time.sleep() in tests makes timing-dependent flakes",
+        "Replace with polling (wait_for / waitFor / eventually) with a timeout, or freeze time.",
+    ),
+    Rule(
+        "FLK002", "warning",
+        re.compile(r"\b(datetime\.now|datetime\.utcnow|time\.time|Date\.now|new Date\(\))\s*\("),
+        "Unfrozen wall-clock reads make tests date/time dependent",
+        "Freeze time (freezegun, jest.useFakeTimers, MockDate) or inject a clock.",
+    ),
+    Rule(
+        "FLK003", "warning",
+        re.compile(r"\b(random\.(random|randint|choice|shuffle|uniform)|Math\.random\s*\()"),
+        "Unseeded randomness makes failures unreproducible",
+        "Seed the RNG (random.seed / faker.seed) and log the seed on failure.",
+    ),
+    Rule(
+        "FLK004", "warning",
+        re.compile(r"\b(requests\.(get|post|put|delete)|urllib\.request\.urlopen|fetch\s*\(|axios\.(get|post)|http\.(get|request))\s*\("),
+        "Real network calls in tests fail without mocks and leak outside hermetic CI",
+        "Mock the boundary (responses, nock, MSW, httpretty) or mark the test integration-only.",
+        ALL_EXTS, True,
+        re.compile(r"(localhost|127\.0\.0\.1|::1)"),
+        "info",
+        " (localhost server — hermetic but still a startup-race/port source)",
+    ),
+    Rule(
+        "FLK005", "info",
+        re.compile(r"for\s+\w+\s+in\s+set\s*\("),
+        "Iterating a set() has nondeterministic order across runs",
+        "Iterate over sorted(...) if order matters, or compare as sets.",
+    ),
+    Rule(
+        "FLK006", "info",
+        re.compile(r"(localhost|127\.0\.0\.1)\s*:\s*\d{2,5}|port\s*=\s*\d{2,5}"),
+        "Hardcoded ports collide on shared/parallel CI runners",
+        "Bind port 0 / let the OS pick, then read back the assigned port.",
+    ),
+    Rule(
+        "FLK007", "warning",
+        re.compile(r"(\.only\s*\(|describe\.only|it\.only|test\.only|@pytest\.mark\.skip\s*\(\s*\)|@unittest\.skip\s*\(\s*\))"),
+        ".only / bare skip markers suggest a suite that is already being triaged around flakes",
+        "Quarantine the flaky test explicitly (unflake quarantine) instead of .only/skip.",
+    ),
+    Rule(
+        "FLK008", "info",
+        re.compile(r"(concurrent|Parallel|Promise\.all|asyncio\.gather|ThreadPool|go\s+func)"),
+        "Concurrency without deterministic synchronization is a classic flake source",
+        "Join/await explicitly, avoid asserting on racy intermediate state.",
+        ALL_EXTS,
+    ),
+    Rule(
+        "FLK009", "warning",
+        re.compile(r"cy\.wait\(\s*\d+"),
+        "cy.wait(ms) fixed waits pass locally and flake on loaded CI runners",
+        "Wait on state instead: cy.intercept() + cy.wait('@alias'), or .should() retry-ability.",
+        JS_EXTS,
+    ),
+    Rule(
+        "FLK010", "warning",
+        re.compile(r"(waitForTimeout|wait_for_timeout)\s*\("),
+        "Fixed-time waits (waitForTimeout / wait_for_timeout) are Playwright's documented flake factory",
+        "Use web-first assertions: await expect(locator).toBeVisible() auto-retries.",
+        JS_EXTS + PY_EXTS,
+    ),
+]
+
+
+def _is_test_file(path: Path) -> bool:
+    s = str(path)
+    return bool(TEST_FILE_HINT.search(path.name) or TEST_DIR_HINT.search(s))
+
+
+DEFAULT_EXCLUDES = (".git", "node_modules", "__pycache__", ".venv", "venv",
+                    "dist", "build", ".tox", ".pytest_cache", ".eggs", "*.egg-info")
+
+
+def _string_spans(text: str) -> dict[int, list[tuple[int, int]]]:
+    """Map 1-based lineno -> string-literal column spans, via stdlib tokenize.
+
+    Matches fully inside a literal are fixture text, not executed code.
+    Python only; unparseable files yield {} (no skipping — never hide signal).
+    """
+    spans: dict[int, list[tuple[int, int]]] = {}
+    try:
+        toks = tokenize.generate_tokens(io.StringIO(text).readline)
+        for tok in toks:
+            if tok.type not in (tokenize.STRING, getattr(tokenize, "FSTRING_MIDDLE", -1)):
+                continue
+            (srow, scol), (erow, ecol) = tok.start, tok.end
+            for r in range(srow, erow + 1):
+                spans.setdefault(r, []).append(
+                    (scol if r == srow else 0, ecol if r == erow else 10 ** 9))
+    except (tokenize.TokenError, SyntaxError, IndentationError):
+        return {}
+    return spans
+
+
+def _in_literal(spans: dict[int, list[tuple[int, int]]],
+                lineno: int, start: int, end: int) -> bool:
+    return any(a <= start and end <= b for a, b in spans.get(lineno, []))
+
+
+def _strip_code_fences(lines: list[str]) -> list[bool]:
+    """Return mask: True if the line is live code (not inside a markdown fence)."""
+    live = []
+    in_fence = False
+    for line in lines:
+        if line.strip().startswith(CODE_FENCE):
+            in_fence = not in_fence
+            live.append(False)
+        else:
+            live.append(not in_fence)
+    return live
+
+
+def scan_file(path: str | Path) -> list[Finding]:
+    path = Path(path)
+    if path.suffix not in SCAN_EXTENSIONS:
+        return []
+    try:
+        text = path.read_text(encoding="utf-8", errors="replace")
+    except OSError:
+        return []
+    lines = text.splitlines()
+    live = _strip_code_fences(lines) if path.suffix == ".md" else [True] * len(lines)
+    is_test = _is_test_file(path)
+    literals = _string_spans(text) if path.suffix == ".py" else {}
+    findings: list[Finding] = []
+    for rule in RULES:
+        if rule.test_files_only and not is_test:
+            continue
+        if path.suffix not in rule.extensions:
+            continue
+        for i, line in enumerate(lines):
+            if not live[i]:
+                continue
+            m = rule.pattern.search(line)
+            if not m:
+                continue
+            if literals and _in_literal(literals, i + 1, m.start(), m.end()):
+                continue  # fixture text in a literal, not executed code
+            severity, message = rule.severity, rule.message
+            if rule.downgrade_if and rule.downgrade_if.search(line):
+                severity, message = rule.downgrade_to, message + rule.downgrade_note
+            findings.append(Finding(
+                rule=rule.id, severity=severity,
+                file=str(path), line=i + 1,
+                message=message, fix=rule.fix,
+            ))
+    return sorted(findings, key=lambda f: (f.file, f.line))
+
+
+def _excluded(path: Path, patterns: tuple[str, ...]) -> bool:
+    return any(fnmatch.fnmatchcase(part, pat)
+               for part in path.parts for pat in patterns)
+
+
+def scan_path(target: str | Path,
+              exclude: list[str] | tuple[str, ...] | None = None) -> list[Finding]:
+    """Scan a file or directory recursively, skipping DEFAULT_EXCLUDES plus extras."""
+    patterns = tuple(DEFAULT_EXCLUDES) + tuple(exclude or ())
+    target = Path(target)
+    files: list[Path] = []
+    if target.is_file():
+        files = [target]
+    elif target.is_dir():
+        for ext in SCAN_EXTENSIONS:
+            files.extend(target.rglob(f"*{ext}"))
+        files = [f for f in files if not _excluded(f, patterns)]
+    else:
+        raise ValueError(f"{target}: no such file or directory")
+    findings: list[Finding] = []
+    for f in sorted(files):
+        findings.extend(scan_file(f))
+    return findings

unflake/quarantine.py

@@ -0,0 +1,78 @@
+"""Quarantine emitters: keep the build green WITHOUT deleting tests.
+
+Philosophy (borrowed from the best CI teams): a quarantined test still
+runs and still reports — it just can't fail the build. Deletion hides
+signal; quarantine preserves it while unblocking everyone else.
+"""
+
+from __future__ import annotations
+
+import re
+
+_SAFE_K = re.compile(r"[A-Za-z0-9_:.\-\[\]]+\Z")
+
+
+def _k_name(tid: str) -> tuple[str, bool]:
+    """Short test name for -k expressions. Returns (name, is_safe).
+
+    Unsafe names (spaces/quotes from parametrization) can't go in -k
+    verbatim — callers must verify and prefer --deselect with node ids.
+    We never silently mangle: mangling hides signal.
+    """
+    name = tid.split("::")[-1].split("[")[0]
+    if _SAFE_K.fullmatch(name):
+        return name, True
+    return name, False
+
+
+def _pytest_k_expression(ids: list[str]) -> tuple[str, list[str]]:
+    shorts, risky = [], []
+    for tid in ids:
+        name, safe = _k_name(tid)
+        shorts.append(name)
+        if not safe:
+            risky.append(tid)
+    return "not " + " and not ".join(shorts), risky
+
+
+def emit(framework: str, flaky_ids: list[str]) -> str:
+    fw = framework.lower()
+    if fw == "pytest":
+        k, risky = _pytest_k_expression(flaky_ids)
+        listed = "\n".join(f"#  - {tid}" for tid in flaky_ids)
+        out = (
+            "# Generated by `unflake quarantine --framework pytest` — review, don't blindly commit.\n"
+            f"# Quarantined {len(flaky_ids)} flaky test(s):\n{listed}\n\n"
+            "# Option A: exclude them from the gating run, keep them in a nightly non-gating run:\n"
+            f"pytest -k \"{k}\"\n\n"
+            "# Option B: run ONLY the quarantined set (nightly, non-gating, still reporting):\n"
+            f"pytest -k \"{' or '.join(t.split('::')[-1].split('[')[0] for t in flaky_ids)}\" --junitxml=quarantine.xml\n"
+        )
+        if risky:
+            out += ("\n# WARNING: these ids contain spaces/quotes and can't go in -k verbatim —\n"
+                    "# verify the expression matches, or use `pytest --deselect <nodeid>`:\n"
+                    + "".join(f"#  RISKY: {tid}\n" for tid in risky))
+        return out
+    if fw in {"jest", "vitest"}:
+        arr = ",\n".join(f'  "{tid}"' for tid in flaky_ids)
+        flag = "--testPathIgnorePatterns" if fw == "jest" else "--exclude"
+        return (
+            f"// Generated by `unflake quarantine --framework {fw}`\n"
+            "// Quarantine list — wire into CI as a non-gating job, keep reporting.\n"
+            f"export const QUARANTINED = [\n{arr}\n];\n\n"
+            f"// CI tip ({fw}): run the gating suite excluding quarantine files,\n"
+            f"// then run quarantine separately with `{flag}` inverted so it still reports.\n"
+        )
+    if fw == "playwright":
+        joined = "|".join(t.split("::")[-1].split("[")[0] for t in flaky_ids)
+        return (
+            "# Generated by `unflake quarantine --framework playwright`\n"
+            f"# Gating run excludes the {len(flaky_ids)} quarantined test(s):\n"
+            f"npx playwright test --grep-invert \"{joined}\"\n\n"
+            "# Nightly non-gating run (still reporting):\n"
+            f"npx playwright test --grep \"{joined}\" --reporter=junit,html\n"
+        )
+    raise ValueError(f"unknown framework {framework!r} (try: pytest, jest, vitest, playwright)")
+
+
+FRAMEWORKS = ("pytest", "jest", "vitest", "playwright")

unflake/report.py

@@ -0,0 +1,123 @@
+"""Report formatters: text (human), json (machine), sarif (code scanning)."""
+
+from __future__ import annotations
+
+import json
+
+from .patterns import RULES, Finding
+from .score import SuiteScore
+
+ORDER = {"error": 0, "warning": 1, "info": 2}
+
+
+def _sarif_rules() -> list[dict]:
+    rules = [{
+        "id": f"unflake/{r.id}",
+        "shortDescription": {"text": r.message},
+        "help": {"text": r.fix},
+        "defaultConfiguration": {"level": {"error": "error", "warning": "warning"}.get(r.severity, "note")},
+    } for r in RULES]
+    rules += [
+        {"id": "unflake/FLAKY",
+         "shortDescription": {"text": "Mixed outcomes across repeated runs — quarantine candidate."},
+         "defaultConfiguration": {"level": "warning"}},
+        {"id": "unflake/REGRESSION",
+         "shortDescription": {"text": "Green-then-red-forever — real failure, never quarantine."},
+         "defaultConfiguration": {"level": "error"}},
+    ]
+    return rules
+
+
+def format_scan_text(findings: list[Finding]) -> str:
+    if not findings:
+        return "unflake scan: clean — no flake anti-patterns found."
+    lines = [f"unflake scan: {len(findings)} finding(s)"]
+    for f in sorted(findings, key=lambda x: (ORDER[x.severity], x.file, x.line)):
+        lines.append(f"  {f.severity.upper():7} [{f.rule}] {f.file}:{f.line}")
+        lines.append(f"           {f.message}")
+        if f.fix:
+            lines.append(f"           fix: {f.fix}")
+    return "\n".join(lines)
+
+
+def format_scan_json(findings: list[Finding]) -> str:
+    return json.dumps({"findings": [f.to_dict() for f in findings]}, indent=2)
+
+
+def format_sarif(findings: list[Finding], suite: SuiteScore | None = None) -> str:
+    results = []
+    for f in findings:
+        level = {"error": "error", "warning": "warning", "info": "note"}[f.severity]
+        results.append({
+            "ruleId": f"unflake/{f.rule}",
+            "level": level,
+            "message": {"text": f"{f.message} Fix: {f.fix}" if f.fix else f.message},
+            "locations": [{
+                "physicalLocation": {
+                    "artifactLocation": {"uri": f.file},
+                    "region": {"startLine": f.line},
+                }
+            }],
+        })
+    if suite:
+        for t in suite.tests:
+            if t.verdict == "flaky":
+                results.append({
+                    "ruleId": "unflake/FLAKY",
+                    "level": "warning",
+                    "message": {"text": f"Flaky across {t.runs} runs: "
+                                         f"{t.passes} passed, {t.fails} failed."},
+                    "locations": [{
+                        "physicalLocation": {
+                            "artifactLocation": {"uri": t.id},
+                            "region": {"startLine": 1},
+                        }
+                    }],
+                })
+            elif t.verdict == "regression":
+                results.append({
+                    "ruleId": "unflake/REGRESSION",
+                    "level": "error",
+                    "message": {"text": f"Regression: green until run {t.change_point}, "
+                                         f"red ever since. Fix — do not quarantine."},
+                    "locations": [{
+                        "physicalLocation": {
+                            "artifactLocation": {"uri": t.id},
+                            "region": {"startLine": 1},
+                        }
+                    }],
+                })
+    return json.dumps({
+        "$schema": "https://json.schemastore.org/sarif-2.1.0.json",
+        "version": "2.1.0",
+        "runs": [{
+            "tool": {"driver": {
+                "name": "unflake",
+                "informationUri": "https://github.com/YOUR-USER/unflake",
+                "rules": _sarif_rules(),
+            }},
+            "results": results,
+        }],
+    }, indent=2)
+
+
+def format_analyze_text(suite: SuiteScore) -> str:
+    lines = [
+        f"FlakeScore: {suite.flake_score}/100 "
+        f"({suite.flaky_count} flaky of {suite.total} tests)"
+    ]
+    for t in suite.tests:
+        if t.verdict == "flaky":
+            lines.append(f"  FLAKY {t.id} ({t.passes}P/{t.fails}F over {t.runs} runs)")
+    for t in suite.tests:
+        if t.verdict == "regression":
+            lines.append(f"  REGRESSION {t.id} (green until run {t.change_point}, "
+                         f"red ever since — fix this, do NOT quarantine it)")
+    for t in suite.tests:
+        if t.verdict == "new":
+            lines.append(f"  NEW {t.id} (only in latest run — needs more runs before judging)")
+    stables = [t for t in suite.tests
+               if t.verdict in ("stable-pass", "stable-fail", "skipped")]
+    if stables:
+        lines.append(f"  ... +{len(stables)} stable/skipped test(s) (see --format json for full list)")
+    return "\n".join(lines)

unflake/runner.py

@@ -0,0 +1,151 @@
+"""`unflake run`: collect the >=2 runs that flake-hunting requires.
+
+Runs your test command N times, collects a JUnit XML per run, then scores.
+With a JUnit flag the per-test analysis works; without one, each run's exit
+code still gives an honest suite-level verdict (stable vs flaky suite).
+"""
+
+from __future__ import annotations
+
+import os
+import subprocess
+from pathlib import Path
+
+from .ingest import FAIL, PASS
+
+
+def _detect_junit_flag(cmd: list[str]) -> str | None:
+    joined = " ".join(cmd)
+    if "pytest" in cmd[0] or "pytest" in joined.split()[0]:
+        return "--junitxml="
+    return None
+
+
+# Verified presets: only runners proven end-to-end (see tests/test_e2e.py).
+# Anything else uses --junit-flag explicitly — we'd rather error than guess.
+# Each preset is (argv_fn, env_fn|None): some runners take the JUnit path
+# as a flag (pytest, vitest), others as an env var (playwright).
+def _pytest_args(dest: str) -> list[str]:
+    return [f"--junitxml={dest}"]
+
+
+def _vitest_args(dest: str) -> list[str]:
+    return ["--reporter=junit", f"--outputFile={dest}"]
+
+
+def _playwright_args(dest: str) -> list[str]:
+    return ["--reporter=junit"]
+
+
+def _playwright_env(dest: str) -> dict[str, str]:
+    return {"PLAYWRIGHT_JUNIT_OUTPUT_NAME": dest}
+
+
+def _jest_args(dest: str) -> list[str]:
+    # NOTE: space form — `--reporters=jest-junit` breaks reporter parsing.
+    return ["--reporters", "jest-junit"]
+
+
+def _jest_env(dest: str) -> dict[str, str]:
+    return {"JEST_JUNIT_OUTPUT_FILE": dest}
+
+
+RUNNERS = {
+    "pytest": (_pytest_args, None),
+    "vitest": (_vitest_args, None),
+    "playwright": (_playwright_args, _playwright_env),
+    "jest": (_jest_args, _jest_env),
+}
+
+
+def resolve_junit_flag(runner: str | None, junit_flag: str | None,
+                       cmd: list[str]) -> str | None:
+    """Explicit --junit-flag wins; --runner pytest/vitest are verified; anything
+    else falls back to pytest auto-detect, else exit-code-only mode."""
+    if junit_flag:
+        return junit_flag
+    if runner:
+        if runner not in RUNNERS:
+            raise ValueError(f"unknown --runner {runner!r} (try: {', '.join(sorted(RUNNERS))})")
+        return None  # preset handled via runner_args(), not a single flag
+    return _detect_junit_flag(cmd)
+
+
+def runner_args(runner: str | None, dest: str) -> list[str] | None:
+    """Extra argv for a verified preset, or None."""
+    if runner and runner in RUNNERS:
+        return RUNNERS[runner][0](dest)
+    return None
+
+
+def runner_env(runner: str | None, dest: str) -> dict[str, str]:
+    """Extra env for a verified preset (empty when the path goes via argv)."""
+    if runner and runner in RUNNERS:
+        fn = RUNNERS[runner][1]
+        return dict(fn(dest)) if fn else {}
+    return {}
+
+
+def collect(cmd: list[str], runs: int, out_dir: str | Path,
+            junit_flag: str | None = None, runner: str | None = None,
+            timeout: float | None = None,
+            progress=None) -> list[Path]:
+    """Run cmd N times. Return per-run result files (JUnit XML if possible,
+    else generic JSON with one suite-level entry)."""
+    out = Path(out_dir)
+    out.mkdir(parents=True, exist_ok=True)
+    if runner and runner not in RUNNERS:
+        raise ValueError(f"unknown --runner {runner!r} (try: {', '.join(sorted(RUNNERS))})")
+    single = resolve_junit_flag(runner, junit_flag, cmd)
+    flag = single  # legacy single-flag path (explicit --junit-flag or pytest auto-detect)
+    files: list[Path] = []
+    for i in range(1, runs + 1):
+        if progress:
+            progress(f"run {i}/{runs}: {' '.join(cmd)}")
+        log = open(out / f"run{i}.log", "w", encoding="utf-8")  # closed below
+        try:
+            dest = out / f"run{i}.xml"
+            preset = runner_args(runner, str(dest))
+            if preset is not None:
+                full = [*cmd, *preset]
+            elif flag:
+                full = [*cmd, f"{flag}{dest}"]
+            else:
+                full = None
+            if full is not None:
+                run_env = dict(os.environ)
+                run_env.update(runner_env(runner, str(dest)))
+                try:
+                    proc = subprocess.run(full, timeout=timeout, env=run_env,
+                                          stdout=log, stderr=subprocess.STDOUT)
+                    ok = proc.returncode == 0
+                except subprocess.TimeoutExpired:
+                    ok = False
+                if not dest.exists():
+                    # Command swallowed the flag (or never wrote XML):
+                    # fall back to an exit-code entry so the run still counts.
+                    dest = _write_json_fallback(out, i, cmd, ok, timed_out=True)
+                files.append(dest)
+            else:
+                try:
+                    proc = subprocess.run(cmd, timeout=timeout,
+                                          stdout=log, stderr=subprocess.STDOUT)
+                    ok, timed_out = proc.returncode == 0, False
+                except subprocess.TimeoutExpired:
+                    ok, timed_out = False, True
+                files.append(_write_json_fallback(out, i, cmd, ok, timed_out))
+        finally:
+            log.close()
+    return files
+
+
+def _write_json_fallback(out: Path, i: int, cmd: list[str],
+                         ok: bool, timed_out: bool = False) -> Path:
+    import json
+    dest = out / f"run{i}.json"
+    status = PASS if ok else FAIL
+    note = "suite exit 0" if ok else ("suite timed out" if timed_out else "suite exit != 0")
+    dest.write_text(json.dumps({"tests": [
+        {"id": f"suite::{' '.join(cmd)}", "status": status, "note": note},
+    ]}), encoding="utf-8")
+    return dest

unflake/scaffold.py

@@ -0,0 +1,89 @@
+"""`unflake init`: scaffold the two cheapest flake preventions.
+
+1. Seeded RNG + logged seed (unseeded randomness is unreproducible by definition).
+2. A frozen-time pointer (wall clocks are the #2 classic).
+
+`--write` creates files idempotently (marker-guarded, never duplicates).
+Without `--write`, snippets print to stdout with wiring instructions.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+MARKER = "# unflake:seed-logging"
+
+PYTEST_SNIPPET = '''%(marker)s (added by `unflake init --framework pytest --write`)
+# Deterministic RNG per test + logged seed: rerun any failure exactly with
+#   UNFLAKE_SEED=<seed> pytest ...
+import os
+import random
+
+import pytest
+
+
+@pytest.fixture(autouse=True)
+def _unflake_seed(request):
+    seed = os.environ.get("UNFLAKE_SEED", "0")
+    random.seed(f"{seed}-{request.node.nodeid}")
+    print(f"\\n[unflake] seed={seed} test={request.node.nodeid}")
+    yield
+'''
+
+JEST_SNIPPET = '''// unflake:seed-logging (added by `unflake init --framework jest`)
+// Deterministic RNG per test file. Wire into jest config:
+//   { setupFiles: ["<rootDir>/unflake.setup.cjs"] }
+const seed = process.env.UNFLAKE_SEED || "0";
+let s = [...seed].reduce((a, c) => (a * 33 + c.charCodeAt(0)) >>> 0, 7) || 7;
+const file = expect.getState().testPath || "unknown";
+console.log(`[unflake] seed=${seed} file=${file}`);
+Math.random = () => (s = (s * 1103515245 + 12345) & 0x7fffffff) / 0x7fffffff;
+'''
+
+VITEST_SNIPPET = '''// unflake:seed-logging (added by `unflake init --framework vitest`)
+// Wire into vitest config: export default { test: { setupFiles: ["./unflake.setup.js"] } }
+const seed = process.env.UNFLAKE_SEED || "0";
+console.log(`[unflake] seed=${seed}`);
+'''
+
+PLAYWRIGHT_SNIPPET = '''// unflake:seed-logging (added by `unflake init --framework playwright`)
+// Playwright has built-in retries — prefer them over fixed waits:
+//   { retries: 2, use: { trace: "retain-on-failure" } }
+// And NEVER page.waitForTimeout(): use web-first assertions:
+//   await expect(page.getByRole("button")).toBeVisible();
+'''
+
+SNIPPETS = {
+    "pytest": ("tests/conftest.py", PYTEST_SNIPPET),
+    "jest": ("unflake.setup.cjs", JEST_SNIPPET),
+    "vitest": ("unflake.setup.js", VITEST_SNIPPET),
+    "playwright": ("unflake.playwright-notes.js", PLAYWRIGHT_SNIPPET),
+}
+
+FRAMEWORKS = tuple(SNIPPETS)
+
+
+def snippet(framework: str) -> tuple[str, str]:
+    """Return (relative_path, content) for a framework."""
+    if framework not in SNIPPETS:
+        raise ValueError(f"unknown framework {framework!r} (try: {', '.join(FRAMEWORKS)})")
+    rel, template = SNIPPETS[framework]
+    return rel, template % {"marker": MARKER}
+
+
+def init_framework(framework: str, root: str | Path = ".", write: bool = False) -> str:
+    """Print or write the scaffold. Returns a human summary."""
+    rel, content = snippet(framework)
+    if not write:
+        return f"# {rel} — create this file with the content below:\n\n{content}"
+    dest = Path(root) / rel
+    dest.parent.mkdir(parents=True, exist_ok=True)
+    if dest.exists():
+        existing = dest.read_text(encoding="utf-8", errors="replace")
+        if MARKER in existing or "unflake:seed-logging" in existing:
+            return f"{dest}: already scaffolded — left untouched."
+        with dest.open("a", encoding="utf-8") as fh:
+            fh.write("\n" + content)
+        return f"{dest}: snippet appended (existing file preserved)."
+    dest.write_text(content, encoding="utf-8")
+    return f"{dest}: created. Rerun any failure exactly with UNFLAKE_SEED=<seed>."

unflake/score.py

@@ -0,0 +1,122 @@
+"""Flakiness scoring across repeated runs.
+
+Pass result files oldest -> newest. Verdicts per test:
+
+  stable-pass  always green
+  stable-fail  always red — a real failure, NOT a flake, never quarantine
+  flaky        mixed outcomes with no clean break — quarantine candidate
+  regression   green, then red from some run on and never green again
+               (needs >= 3 runs; with 2 runs a pass->fail looks identical
+               to a flake, so it stays flaky) — fix, never quarantine
+  new          only appeared in the latest run — needs more runs
+  skipped      never produced a pass/fail
+
+Suite FlakeScore: 0-100, share of non-flaky tests (higher = healthier).
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from .ingest import FAIL, PASS, SKIP
+
+
+@dataclass
+class TestScore:
+    id: str
+    passes: int
+    fails: int
+    skips: int
+    runs: int
+    verdict: str
+    flakiness: float  # 0.0 stable .. 1.0 maximally flaky (50/50 split)
+    change_point: int | None = None  # 1-based run where a regression starts
+
+    def to_dict(self) -> dict:
+        return {
+            "id": self.id, "passes": self.passes, "fails": self.fails,
+            "skips": self.skips, "runs": self.runs,
+            "verdict": self.verdict, "flakiness": round(self.flakiness, 3),
+            "change_point": self.change_point,
+        }
+
+
+@dataclass
+class SuiteScore:
+    tests: list[TestScore]
+    flake_score: float
+    flaky_count: int
+    regression_count: int
+    total: int
+
+    def to_dict(self) -> dict:
+        return {
+            "flake_score": round(self.flake_score, 1),
+            "flaky_count": self.flaky_count,
+            "regression_count": self.regression_count,
+            "total": self.total,
+            "tests": [t.to_dict() for t in self.tests],
+        }
+
+    def issues(self) -> list[TestScore]:
+        """Actionable tests: flaky first, then regressions."""
+        return [t for t in self.tests if t.verdict in ("flaky", "regression")]
+
+
+def _is_clean_break(statuses: list[str]) -> int | None:
+    """If statuses are pass* then fail* (skips ignored), return the 1-based
+    index of the first fail. Otherwise None."""
+    pf = [s for s in statuses if s in (PASS, FAIL)]
+    if len(pf) < 3 or PASS not in pf or FAIL not in pf:
+        return None
+    first_fail = pf.index(FAIL)
+    if first_fail == 0:
+        return None  # failed from the start — stable-fail territory
+    if all(s == FAIL for s in pf[first_fail:]):
+        return statuses.index(FAIL) + 1  # 1-based, counting all runs
+    return None
+
+
+def score_runs(runs: list[dict[str, str]]) -> SuiteScore:
+    if not runs:
+        raise ValueError("need at least one run to score")
+    ids: set[str] = set()
+    for run in runs:
+        ids.update(run.keys())
+    tests: list[TestScore] = []
+    for tid in sorted(ids):
+        statuses = [run.get(tid, "missing") for run in runs]
+        present = [s for s in statuses if s != "missing"]
+        passes = sum(1 for s in present if s == PASS)
+        fails = sum(1 for s in present if s == FAIL)
+        skips = sum(1 for s in present if s == SKIP)
+        change_point: int | None = None
+        if "missing" in statuses:
+            if len(present) == 1 and statuses[-1] != "missing":
+                verdict, flakiness = "new", 0.0
+            else:
+                verdict, flakiness = "flaky", 0.5  # appears/disappears
+        elif passes and fails:
+            cp = _is_clean_break(statuses)
+            if cp is not None:
+                verdict, flakiness, change_point = "regression", 0.0, cp
+            else:
+                rate = passes / (passes + fails)
+                verdict, flakiness = "flaky", round(1.0 - abs(rate - 0.5) * 2, 3)
+        elif fails and not passes:
+            verdict, flakiness = ("stable-fail", 0.0) if not skips else ("flaky", 0.3)
+        elif passes and not fails:
+            verdict, flakiness = ("stable-pass", 0.0) if not skips else ("flaky", 0.2)
+        else:
+            verdict, flakiness = "skipped", 0.0
+        tests.append(TestScore(tid, passes, fails, skips, len(runs),
+                               verdict, flakiness, change_point))
+    flaky = [t for t in tests if t.verdict == "flaky"]
+    regressions = [t for t in tests if t.verdict == "regression"]
+    total = len(tests) or 1
+    flake_score = 100.0 * (total - len(flaky)) / total
+    order = {"flaky": 0, "regression": 1, "stable-fail": 2,
+             "new": 3, "stable-pass": 4, "skipped": 5}
+    tests.sort(key=lambda t: (order.get(t.verdict, 9), -t.flakiness, t.id))
+    return SuiteScore(tests, round(flake_score, 1), len(flaky),
+                      len(regressions), len(tests))

unflake-0.6.1.dist-info/METADATA

@@ -0,0 +1,143 @@
+Metadata-Version: 2.4
+Name: unflake
+Version: 0.6.1
+Summary: Green CI without the rerun ritual — detect, score, and quarantine flaky tests
+License: MIT
+Project-URL: Homepage, https://github.com/YOUR-USER/unflake
+Project-URL: Issues, https://github.com/YOUR-USER/unflake/issues
+Keywords: testing,flaky-tests,ci,qa,agent-skills
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Software Development :: Testing
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Dynamic: license-file
+
+# Unflake 🟢
+
+[![CI](https://github.com/YOUR-USER/unflake/actions/workflows/ci.yml/badge.svg)](https://github.com/YOUR-USER/unflake/actions/workflows/ci.yml)
+[![OpenSSF Scorecard](https://api.scorecard.dev/projects/github.com/YOUR-USER/unflake/badge)](https://scorecard.dev/viewer/?uri=github.com/YOUR-USER/unflake)
+[![PyPI](https://img.shields.io/pypi/v/unflake.svg)](https://pypi.org/project/unflake/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
+
+**Green CI without the "just rerun it" ritual.**
+
+![Unflake 60-second demo](assets/terminal-demo.svg)
+
+CI is red. You rerun. It's green. You merge, trusting nothing. Unflake hunts the flake instead: static scan in milliseconds, FlakeScore across repeated runs, regression-vs-flake verdicts, and quarantine configs that keep the build green *without deleting a single test*.
+
+![scan → run → calm](assets/pipeline.svg)
+
+```bash
+pip install unflake
+unflake scan tests/                                  # 1. smells, instantly
+unflake run --runs 3 --runner pytest -- pytest tests/ -q   # 2. collect + score
+unflake quarantine run1.xml run2.xml --framework pytest    # 3. quarantine, don't delete
+unflake init --framework pytest --write              # 4. seeded RNG scaffold (prevention)
+```
+
+JS-first repo? Same tool, no Python packaging — just Python itself (3.9+):
+
+```bash
+npx unflake-ci scan tests/
+npx unflake-ci run --runs 3 --runner vitest -- npx vitest run tests/
+```
+
+Proven on real suites: 3,676 tests × 5 runs across attrs/click/tqdm → FlakeScore 100,
+0 false quarantines ([study](benchmarks/BENCH.md)).
+
+No install handy? `bash demo.sh` runs the whole loop on fixtures in 60 seconds.
+
+> ⭐ If flaky tests have ever paged you, star this — it helps other devs find it.
+
+## Before / after
+
+```python
+# before: the 3am pager
+def test_checkout():
+    time.sleep(5)                       # pray the server is up
+    assert requests.get(API).ok         # real network in a unit test
+```
+
+```bash
+$ unflake scan tests/
+  WARNING [FLK001] tests/test_shop.py:4 — time.sleep() in tests makes timing-dependent flakes
+           fix: Replace with polling (wait_for / waitFor / eventually) with a timeout, or freeze time.
+  WARNING [FLK004] tests/test_shop.py:5 — Real network calls in tests fail without mocks ...
+```
+
+```bash
+$ unflake run --runs 3 -- pytest tests/ -q
+FlakeScore: 50.0/100 (2 flaky of 4 tests)
+  FLAKY test_cart::test_checkout (2P/1F over 3 runs)
+  FLAKY test_cart::test_discount (2P/1F over 3 runs)
+```
+
+```bash
+$ unflake quarantine run1.xml run2.xml --framework pytest
+pytest -k "not test_checkout and not test_discount"   # gating run stays green
+pytest -k "test_checkout or test_discount"            # nightly run still reports
+```
+
+Quarantine preserves signal; deletion hides it. Every quarantined test still runs — it just can't fail the build. **Regressions are never quarantined**: green-then-red-forever is a real failure — fix it.
+
+## Flaky or regression? Unflake knows the difference
+
+Pass result files oldest → newest and Unflake separates three fates:
+
+| Verdict | Meaning | Action |
+|---|---|---|
+| `FLAKY` | mixed outcomes, no clean break | quarantine + fix root cause |
+| `REGRESSION` | green until run N, red ever since | fix now — never quarantine |
+| `NEW` / stable | seen once / always green / always red | more runs / ship it / real bug |
+
+Two runs can only suggest a flake; calling a regression needs ≥3. One run proves nothing — `unflake run` exists so there's no excuse.
+
+## Why not the others?
+
+| | Unflake | Heavy flake platforms |
+|---|---|---|
+| Install | zero-dep, stdlib only, `pip install unflake` | OTel collectors, dashboards, SaaS |
+| Input | JUnit XML from **any** framework (pytest ✅, Vitest ✅, Playwright ✅, Jest ✅ e2e — plus generic `--junit-flag`, Go, JUnit…) | per-framework reporters/plugins |
+| First value | `scan` in milliseconds, no execution | needs history ingestion |
+| Verdicts | flaky vs regression vs new, change-point included | usually just a score |
+| Philosophy | quarantine, never delete | often auto-skip-and-forget |
+| Agent-native | `SKILL.md` + harness adapters day one | docs page, if you're lucky |
+
+## Works with your harness
+
+Claude Code · Codex · Copilot · Cursor · Gemini · Pi · OpenCode · Windsurf · Cline · Qoder — plus a GitHub Action (`action.yml`, SARIF → code scanning) and a pre-commit hook. Details: [`docs/ADAPTERS.md`](docs/ADAPTERS.md). The skill lives at [`skills/unflake/SKILL.md`](skills/unflake/SKILL.md).
+
+10 rules today, incl. the JS classics: `cy.wait(ms)` (FLK009) and `waitForTimeout` (FLK010).
+Precision features: localhost-aware severities, string-literal awareness (Python),
+`--exclude` globs, and SARIF rule metadata for code scanning.
+
+## Benchmarks (honest, reproducible)
+
+Fixtures + real pytest e2e + a real-repo scan (`psf/requests`: 120 findings in ~0.1s — with the noise documented, not hidden). Method, numbers, and known limitations: [`benchmarks/BENCH.md`](benchmarks/BENCH.md). If a number doesn't reproduce, that's a bug — file it.
+
+## Contributing — built for drive-by PRs
+
+- 🧪 New FLK rule = one regex + one test (`good first issue`)
+- 🔌 New harness adapter or quarantine emitter = one file + one test
+- 🌱 `unflake init` for your framework = one snippet
+- 🌍 Translations welcome (`README.<lang>.md`)
+- Full loop: [`CONTRIBUTING.md`](CONTRIBUTING.md)
+
+## Roadmap
+
+- [x] v0.1 — scan / analyze / quarantine, SARIF, 4 frameworks, skill + adapters
+- [x] v0.2 — `run` collector, regression-vs-flake verdicts, `init` scaffolder, FLK009/FLK010, pre-commit + GitHub Action, `pip install unflake`
+- [x] v0.3 — `--runner` presets, per-run logs, real-pytest e2e in CI, launch kit (`demo.sh`, templates, `LAUNCH.md`), real-repo validation
+- [x] v0.4 — precision (localhost-aware FLK004, literal-awareness), `--exclude`, SARIF rules, safe `-k` quoting, CI matrix 3.10–3.14
+- [x] v0.5 — `unflake-ci` npx wrapper (verified pack→install→run), `--runner vitest` verified e2e
+- [x] v0.6 — `--runner` playwright + jest verified e2e, real-repo study (3,676 tests, 0 false quarantines)
+- [ ] Recall study v2 on suites with known flakes (specificity proven; recall needs wild flakes)
+- [ ] JS/TS string-literal awareness + variable-host local-server detection (see BENCH.md)
+
+## License
+
+MIT — see [LICENSE](LICENSE).

unflake-0.6.1.dist-info/RECORD

@@ -0,0 +1,16 @@
+unflake/__init__.py,sha256=lns3NRcFIEftdwyjj6dzFeW1GyuI542ru-p2POnskTs,94
+unflake/__main__.py,sha256=HlOZgR1S8N_FcV4UyArFybJgDBsd3tCfdOvbRzc5cww,447
+unflake/cli.py,sha256=p-d_LPLABdhTQTaTajZHidSq5TEM_o1XCWhb3nPuWyo,8442
+unflake/ingest.py,sha256=KkjDiJWvvGshliFGS5gPcp2PR4RYyBudKFDz_jZbvC8,3289
+unflake/patterns.py,sha256=ZFomxU28yT1G1qKE-Ky73QLJjSjM5rWOkvme1xS2Fng,8942
+unflake/quarantine.py,sha256=PoPpNHb5csPrcvdR78-7CgFPskZBH5Rm8WmwODNJ4lg,3431
+unflake/report.py,sha256=ciIhbZMwRJVUqtrnm-UWEjCQj4yi7l8nHOy1g4hI97Y,4837
+unflake/runner.py,sha256=tliEh2vf-FDMpXvTjpi1U6SsqJT_Fk96LEKzxj7GiUM,5659
+unflake/scaffold.py,sha256=OiuHoaWMfB7oICiRbyJdqxlZQW-D8ydtBLsecfFjSek,3537
+unflake/score.py,sha256=sLACgHvJpw8oW8-UTJp38QO-8PITqiUZJjVjF82xTPs,4672
+unflake-0.6.1.dist-info/licenses/LICENSE,sha256=61Ot6dNjcmx1kKVNfniNm1PTbNF4N7zmNjxJ_H7VCao,1077
+unflake-0.6.1.dist-info/METADATA,sha256=H9CXyanmYd5DhhX_l96Tts6RG_SemSHioRMT-2EliQg,7063
+unflake-0.6.1.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+unflake-0.6.1.dist-info/entry_points.txt,sha256=BRz8fLVpu72NT5t1fgFH-tqpb_8qqW6e8tcup6wo8HY,45
+unflake-0.6.1.dist-info/top_level.txt,sha256=m9wa0x1Y0vt-lPoM1gt7ihlXwbdqARUn2oRUdv7hZEQ,8
+unflake-0.6.1.dist-info/RECORD,,

unflake-0.6.1.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

unflake-0.6.1.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+unflake = unflake.cli:main

unflake-0.6.1.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Unflake Contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

unflake-0.6.1.dist-info/top_level.txt

@@ -0,0 +1 @@
+unflake