flakebox 0.4.4__py3-none-any.whl

flakebox/junit.py

@@ -0,0 +1,139 @@
+"""JUnit XML ingestion: pytest, jest, surefire all speak JUnit."""
+
+from __future__ import annotations
+
+import hashlib
+import sys
+import xml.etree.ElementTree as ET
+from xml.parsers import expat
+
+from .quarantine import REASON as QUARANTINE_REASON
+
+# outcome per testcase
+PASSED, FAILED, ERROR, SKIPPED = "passed", "failed", "error", "skipped"
+
+MAX_MS = 2 ** 63 - 1  # the largest duration an SQLite INTEGER can hold
+
+
+def test_id(classname: str, name: str) -> str:
+    classname = (classname or "").strip()
+    name = (name or "").strip()
+    return f"{classname}::{name}" if classname else name
+
+
+def report_sha256(raw: bytes) -> str:
+    return hashlib.sha256(raw).hexdigest()
+
+
+def _is_quarantine_xfail(skipped_el) -> bool:
+    """pytest writes an xfail as <skipped type="pytest.xfail"
+    message="expected test failure">reason</skipped>. Only the reason our
+    own quarantine plugin attaches counts - a hand-written xfail stays a
+    skip."""
+    if (skipped_el.get("type") or "") != "pytest.xfail":
+        return False
+    text = (skipped_el.text or "") + " " + (skipped_el.get("message") or "")
+    return QUARANTINE_REASON in text
+
+
+def _warn(msg: str) -> None:
+    try:
+        print(f"flakebox: warning: {msg}", file=sys.stderr)
+    except (OSError, ValueError):
+        pass
+
+
+def duration_ms(text, where: str = "", warn=_warn) -> int:
+    """`time` attribute -> milliseconds. Accepts a decimal comma
+    (`0,5`, written by some locales' runners); anything else that is not
+    a finite, non-negative number of seconds the store can hold (`fast`,
+    `inf`, `nan`, `-5`, `1e300`) counts as 0 with a warning rather than
+    rejecting the whole report."""
+    text = (text or "").strip()
+    if not text:
+        return 0
+    try:
+        ms = int(float(text.replace(",", ".")) * 1000)
+    except (ValueError, OverflowError):  # not a number; or inf / nan
+        ms = -1
+    if not 0 <= ms <= MAX_MS:  # also: negative, or too big for the store
+        warn(f"{where}: unreadable time={text!r}, recorded as 0")
+        return 0
+    return ms
+
+
+class _Declaration(Exception):
+    """expat reported a DOCTYPE or ENTITY declaration."""
+
+
+def check_no_dtd(raw: bytes, path: str = "") -> None:
+    """Reject a report that declares a DOCTYPE or an ENTITY. ElementTree
+    performs no entity-expansion protection and JUnit reports never
+    legitimately need a DTD, so the document is run through expat first and
+    stopped at the first declaration, before anything is expanded.
+
+    expat does the tokenising - the encoding (UTF-16 included), comments,
+    processing instructions and CDATA - so text that merely CONTAINS
+    "<!DOCTYPE html>" (captured output in a CDATA section, say) is not a
+    declaration and passes. A document expat cannot tokenise is left for
+    the real parse to report as not well-formed; no entity can have been
+    declared in what it did read, because a declaration needs a DOCTYPE and
+    the DOCTYPE would have stopped it first."""
+    def refuse(*_args):
+        raise _Declaration
+
+    parser = expat.ParserCreate()
+    parser.StartDoctypeDeclHandler = refuse
+    parser.EntityDeclHandler = refuse
+    try:
+        parser.Parse(raw, True)
+    except _Declaration:
+        raise ValueError(
+            f"refusing to parse {path}: XML contains a DOCTYPE or ENTITY "
+            "declaration (rejected as a possible XML bomb)") from None
+    except expat.ExpatError:
+        pass
+
+
+def parse_bytes(raw: bytes, path: str = "", warn=_warn) -> list[dict]:
+    """Return [{test_id, outcome, duration_ms, message}] for every
+    <testcase> anywhere in the document - <testsuites>, a bare
+    <testsuite>, or suites nested in suites (surefire aggregates)."""
+    check_no_dtd(raw, path)
+    root = ET.fromstring(raw)
+    results = []
+    cases = [root] if root.tag == "testcase" else root.iter("testcase")
+    for tc in cases:
+        tid = test_id(tc.get("classname", ""), tc.get("name", ""))
+        ms = duration_ms(tc.get("time"), f"{path} testcase {tid}", warn)
+        sk = tc.find("skipped")
+        if sk is not None:
+            if _is_quarantine_xfail(sk):
+                # the generated plugin in xfail mode: the test really
+                # ran and really failed, it just couldn't break the
+                # build. Record the truth so stats stay honest.
+                outcome, message = FAILED, (sk.text or "").strip()[:500]
+            else:
+                outcome, message = SKIPPED, ""
+        elif tc.find("failure") is not None:
+            el = tc.find("failure")
+            outcome, message = FAILED, (el.get("message", "") or "")[:500]
+        elif tc.find("error") is not None:
+            el = tc.find("error")
+            outcome, message = ERROR, (el.get("message", "") or "")[:500]
+        else:
+            outcome, message = PASSED, ""
+        results.append({"test_id": tid, "outcome": outcome,
+                        "duration_ms": ms, "message": message})
+    return results
+
+
+def read(path: str) -> bytes:
+    with open(path, "rb") as f:
+        return f.read()
+
+
+def parse(path: str, warn=_warn) -> list[dict]:
+    """Parse the report at `path` (whole file read first, DTD guard,
+    then ElementTree)."""
+    return parse_bytes(read(path), path, warn)

flakebox/quarantine.py

@@ -0,0 +1,194 @@
+"""Quarantine appliers: turn the flaky set into something the runner obeys."""
+
+from __future__ import annotations
+
+import json
+import os
+import shlex
+import subprocess
+
+# The reason string the generated plugin attaches. junit.py looks for it
+# to tell a quarantined-and-failed xfail apart from a hand-written xfail.
+REASON = "quarantined: flaky per flakebox"
+MODES = ("skip", "xfail")
+
+PYTEST_PLUGIN = '''"""Generated by `flakebox quarantine --format pytest`.
+
+Marks the quarantined (flaky) tests. A test is quarantined when it has the
+same name (with any [parameters]) as a quarantined id and the same module
+path and class names, compared as whole components in either spelling: the
+pytest node id `tests/test_mod.py::TestA::test_x` and the JUnit id
+`tests.test_mod.TestA::test_x` are the same test. Only leading directories
+may be missing from one side (CI and your machine can have different
+rootdirs): `test_mod.py::test_x` matches `tests.test_mod::test_x`, but
+`pkg_a/test_mod.py::test_x` never matches `pkg_b/test_mod.py::test_x`.
+
+Mode comes from flakebox_quarantine.json ("mode"):
+  skip  - quarantined tests are skipped (default; they record no result)
+  xfail - quarantined tests still run but cannot fail the build; their
+          real pass/fail still lands in the JUnit report, so flakebox can
+          see them stabilise and tell you to un-quarantine them
+
+Usage: drop this file + flakebox_quarantine.json in your repo root and run
+    python -m pytest -p flakebox_quarantine
+`python -m pytest` puts the current directory on sys.path; the bare
+`pytest` command does not, so it cannot import this file (if the plugin
+lives in another directory, put that directory on PYTHONPATH).
+Regenerate after `flakebox unquarantine` changes the set.
+"""
+import json
+import os
+
+import pytest
+
+_HERE = os.path.dirname(os.path.abspath(__file__))
+with open(os.path.join(_HERE, "flakebox_quarantine.json"),
+          encoding="utf-8") as _f:
+    _DATA = json.load(_f)
+QUARANTINED = _DATA["tests"]
+MODE = _DATA.get("mode", "skip")
+REASON = "quarantined: flaky per flakebox"
+
+
+def _key(test_id):
+    """(name, chain) of a JUnit id or a pytest node id, or None.
+
+    name is the test function plus its [parameters]; chain is the module
+    path and the class names as whole components, so `pkg/mod.py::A::t`
+    and `pkg.mod.A::t` both give ("t", ("pkg", "mod", "A")). An id with no
+    module part gives None: it would match every test of that name.
+    """
+    # split off the parameters first: they may contain `::`, `/` and `.`
+    base, bracket, params = test_id.partition("[")
+    parts = base.replace(chr(92), "/").split("::")
+    if len(parts) < 2:
+        return None
+    head = parts[0]
+    if head.endswith(".py"):
+        head = head[:-3]
+    chain = []
+    for part in [head.replace("/", ".")] + parts[1:-1]:
+        chain.extend(piece for piece in part.split(".") if piece)
+    if not chain:
+        return None
+    return parts[-1] + bracket + params, tuple(chain)
+
+
+def _index(ids):
+    exact, tails = set(), set()
+    for test_id in ids:
+        key = _key(test_id)
+        if key is not None:
+            name, chain = key
+            exact.add(key)
+            tails.update((name, chain[i:]) for i in range(len(chain)))
+    return exact, tails
+
+
+# every quarantined (name, chain), and every tail of a quarantined chain
+_EXACT, _TAILS = _index(QUARANTINED)
+
+
+def _is_quarantined(nodeid):
+    key = _key(nodeid)
+    if key is None:
+        return False
+    name, chain = key
+    # this chain is a tail of a quarantined one (the id was recorded with
+    # more leading directories) ...
+    if key in _TAILS:
+        return True
+    # ... or a quarantined chain is a tail of this one (it has fewer)
+    return any((name, chain[i:]) in _EXACT for i in range(len(chain)))
+
+
+def _marker():
+    if MODE == "xfail":
+        return pytest.mark.xfail(reason=REASON, strict=False)
+    return pytest.mark.skip(reason=REASON)
+
+
+def pytest_collection_modifyitems(items):
+    for item in items:
+        if _is_quarantined(item.nodeid):
+            item.add_marker(_marker())
+'''
+
+
+def write_pytest(out_dir: str, test_ids: list[str],
+                 mode: str = "skip") -> list[str]:
+    if mode not in MODES:
+        raise ValueError(f"mode must be one of {MODES}, got {mode!r}")
+    os.makedirs(out_dir, exist_ok=True)
+    written = []
+    jp = os.path.join(out_dir, "flakebox_quarantine.json")
+    ids = sorted(set(test_ids))
+    with open(jp, "w", encoding="utf-8") as f:
+        # no timestamp: regenerating an unchanged set writes the same bytes
+        json.dump({"tests": ids, "mode": mode, "count": len(ids)}, f, indent=2)
+        f.write("\n")
+    written.append(jp)
+    pp = os.path.join(out_dir, "flakebox_quarantine.py")
+    with open(pp, "w", encoding="utf-8") as f:
+        f.write(PYTEST_PLUGIN)
+    written.append(pp)
+    return written
+
+
+def pytest_nodeid(test_id: str) -> str:
+    """Map a JUnit id to the pytest node id it most likely came from.
+
+    pytest writes classname = dotted module path + class names and
+    name = the function (with any [params]). Modules are lowercase by
+    convention and classes start with a capital, so the leading run of
+    segments that do not start with an uppercase letter is the module:
+
+        pkg.module.Class::test_x    -> pkg/module.py::Class::test_x
+        pkg.module.Outer.Inner::t   -> pkg/module.py::Outer::Inner::t
+        pkg.module::test_x          -> pkg/module.py::test_x
+        test_x (no classname)       -> test_x (unchanged)
+
+    Best effort: a lowercase class name, a `python_files` pattern that
+    is not `*.py`, or a non-pytest producer breaks the guess, and pytest
+    simply ignores a --deselect that matches nothing. The generated
+    plugin (--format pytest) compares the name and the whole path
+    components instead and is the reliable path. Ids already in node-id
+    form (containing `/` or `.py::`) pass through unchanged.
+    """
+    if "::" not in test_id or "/" in test_id or ".py::" in test_id:
+        return test_id
+    classname, _, rest = test_id.partition("::")
+    segs = [s for s in classname.split(".") if s]
+    if not segs:
+        return test_id
+    i = 0
+    while i < len(segs) and not segs[i][:1].isupper():
+        i += 1
+    module, classes = segs[:i], segs[i:]
+    if not module:  # classname is just class names: no module to place
+        return "::".join(classes + [rest])
+    return "::".join(["/".join(module) + ".py"] + classes + [rest])
+
+
+def deselect_args(test_ids: list[str]) -> list[str]:
+    """`--deselect` args for pytest, with JUnit ids mapped to pytest node
+    ids by pytest_nodeid() (best effort - see there)."""
+    args = []
+    for tid in sorted(set(test_ids)):
+        args += ["--deselect", pytest_nodeid(tid)]
+    return args
+
+
+def deselect_line(test_ids: list[str]) -> str:
+    """deselect_args() as one shell line. Each argument is quoted for the
+    local shell (POSIX quoting via shlex, cmd.exe double-quoting on
+    Windows), so the shell has to parse the line once more -
+    `eval "pytest $(flakebox quarantine --format deselect)"` - to hand
+    pytest an id with a space or shell characters (`test_x[a b]`,
+    `test_x[$HOME]`) as ONE argument. A bare `pytest $(...)` splits the
+    output at whitespace and keeps the quotes, so only ids with neither
+    survive it (the README also covers PowerShell)."""
+    args = deselect_args(test_ids)
+    if os.name == "nt":
+        return subprocess.list2cmdline(args)
+    return shlex.join(args)