steppy-dev 0.3.0__py3-none-any.whl

steppy/coverage/__init__.py

@@ -0,0 +1,9 @@
+"""Deterministic coverage substrate: record -> merge -> map store -> gap report.
+
+Recording happens inside the *generated* conftest (env-gated on
+``STEPPY_COVERAGE=1``); everything in this package runs steppy-side and is
+lazy-imported from the CLI only — ``steppy run`` without ``--coverage`` never
+imports it (guarded by ``tests/test_no_coverage_import_on_default_path.py``).
+
+No LLM calls anywhere: recording, merge, and gap compute are deterministic.
+"""

steppy/coverage/gaps.py

@@ -0,0 +1,190 @@
+"""Gap computation: pure bucket math over the coverage map.
+
+Consumes ``store.read_map()`` output, produces the bucket structure the
+report renders. No IO, no clock — same map in, same buckets out.
+
+Buckets per kind (endpoints are the primary surface, routes when a sitemap
+is configured):
+
+- ``tested`` — declared AND exercised by at least one scenario, with a
+  ``strength``: *strong* when some covering scenario both ran with asserts
+  enabled and contains at least one assert step; *weak* otherwise.
+- ``declared_unreached`` — in the manifest, never hit by any scenario. The
+  key bucket. Each entry carries its triage flag so non-UI endpoints
+  (webhooks, cron callbacks) can be discounted, plus ``triage_source``
+  saying whether that came from ``config.yaml`` or the store's own table.
+- ``discovered_uncovered`` — seen in a trail but only weakly covered.
+- ``discovered_tested`` — exercised + strongly asserted but absent from the
+  manifest (stale spec, or a base-URL misconfiguration).
+- ``unclassified`` (endpoints only) — requests that matched no configured
+  API base URL.
+
+Modules have no declared source in v1 — they are reported as a covered
+inventory only.
+
+``e2e_surface`` sits beside ``counts`` and answers the badge's question for each
+surface (see :func:`surface_ratio`). It is derived, not another bucket: nothing
+here decides *which* nodes are non-UI — that is the read-time overlay in
+:mod:`steppy.coverage.triage`, applied to the map before it arrives.
+"""
+
+from __future__ import annotations
+
+#: Surfaces the e2e ratio can be computed over. ``all`` is the default because
+#: the declared surface is "routes and API endpoints" — either alone silently
+#: drops half of what a project declared.
+SURFACES = ("all", "endpoints", "routes")
+
+
+def surface_ratio(counts: dict, surface: str = "all") -> dict:
+    """``tested / declared`` over one surface, with non_ui removed from both.
+
+    The formula as written in the docs — ``tested / (declared - non_ui)`` — can
+    exceed 1, because a *tested* endpoint flagged ``non_ui`` stays in the
+    numerator while leaving the denominator. Subtracting from both keeps the ratio
+    in range and keeps the question honest: *of the surface a UI test could reach,
+    how much does the suite reach?*
+
+    ``total == 0`` returns ``ratio``/``percent`` of ``None`` rather than 0, so a
+    project with nothing declared reads as "not measured" instead of "0% tested".
+    """
+    kinds = ("endpoints", "routes") if surface == "all" else (surface,)
+    tested = 0
+    total = 0
+    for kind in kinds:
+        section = counts.get(kind) or {}
+        tested += section.get("tested", 0) - section.get("tested_non_ui", 0)
+        total += section.get("declared", 0) - section.get("declared_non_ui", 0)
+    if total <= 0:
+        return {"surface": surface, "tested": max(tested, 0), "declared": max(total, 0), "ratio": None, "percent": None}
+    ratio = tested / total
+    return {
+        "surface": surface,
+        "tested": tested,
+        "declared": total,
+        "ratio": round(ratio, 4),
+        "percent": int(round(ratio * 100)),
+    }
+
+
+def _strength(node: dict, scenarios_meta: dict) -> str:
+    """Strong iff some covering scenario has asserts enabled AND assert steps."""
+    for cov in node["scenarios"]:
+        meta = scenarios_meta.get(cov["scenario_id"], {})
+        if cov["asserts_enabled"] and meta.get("asserts_count", 0) > 0:
+            return "strong"
+    return "weak"
+
+
+def _covered_entry(node: dict, scenarios_meta: dict) -> dict:
+    # ``triage`` rides on a *covered* entry too, which it historically did not.
+    # Without it the e2e-surface ratio cannot subtract a non_ui endpoint that
+    # happens to be tested, and the number could exceed 100%.
+    triage = node.get("triage") or {}
+    return {
+        "api": node["api"],
+        "method": node["method"],
+        "key": node["key"],
+        "scenarios": [c["scenario_id"] for c in node["scenarios"]],
+        "strength": _strength(node, scenarios_meta),
+        "triage": triage.get("flag", "unknown"),
+        "triage_source": triage.get("source", ""),
+        "note": triage.get("note", ""),
+    }
+
+
+def _unreached_entry(node: dict) -> dict:
+    triage = node.get("triage") or {}
+    return {
+        "api": node["api"],
+        "method": node["method"],
+        "key": node["key"],
+        "triage": triage.get("flag", "unknown"),
+        # Which rule discounted this endpoint: "config" (a coverage.non_ui
+        # pattern), "store" (a triage row), or "" (neither). A number nobody can
+        # trace is a number nobody trusts.
+        "triage_source": triage.get("source", ""),
+        "note": triage.get("note", ""),
+    }
+
+
+def compute_gaps(map_view: dict) -> dict:
+    """Bucket every node of the map. ``map_view`` comes from ``store.read_map``."""
+    scenarios_meta = map_view["scenarios"]
+    buckets: dict = {
+        "endpoints": {
+            "tested": [],
+            "declared_unreached": [],
+            "discovered_uncovered": [],
+            "discovered_tested": [],
+            "unclassified": [],
+        },
+        "routes": {
+            "tested": [],
+            "declared_unreached": [],
+            "discovered_uncovered": [],
+            "discovered_tested": [],
+        },
+        "modules": {"covered": []},
+    }
+
+    for node in map_view["nodes"]:
+        declared = bool(node["declared"])
+        covered = bool(node["scenarios"])
+        if not declared and not covered:
+            continue  # placeholder left by a re-link, or triage-only node
+
+        if node["kind"] == "module":
+            if covered:
+                buckets["modules"]["covered"].append(
+                    {
+                        "key": node["key"],
+                        "scenarios": [c["scenario_id"] for c in node["scenarios"]],
+                    }
+                )
+            continue
+
+        kind_bucket = buckets["endpoints" if node["kind"] == "endpoint" else "routes"]
+
+        if node["kind"] == "endpoint" and covered and not declared and node["api"] == "":
+            kind_bucket["unclassified"].append(_covered_entry(node, scenarios_meta))
+            continue
+
+        if declared and covered:
+            kind_bucket["tested"].append(_covered_entry(node, scenarios_meta))
+        elif declared:
+            kind_bucket["declared_unreached"].append(_unreached_entry(node))
+        else:
+            entry = _covered_entry(node, scenarios_meta)
+            if entry["strength"] == "weak":
+                kind_bucket["discovered_uncovered"].append(entry)
+            else:
+                kind_bucket["discovered_tested"].append(entry)
+
+    counts: dict = {"scenarios": len(scenarios_meta)}
+    for kind in ("endpoints", "routes"):
+        section = buckets[kind]
+        tested = section["tested"]
+        unreached = section["declared_unreached"]
+        tested_non_ui = sum(1 for e in tested if e["triage"] == "non_ui")
+        unreached_non_ui = sum(1 for e in unreached if e["triage"] == "non_ui")
+        counts[kind] = {
+            "declared": len(tested) + len(unreached),
+            "tested": len(tested),
+            "tested_strong": sum(1 for e in tested if e["strength"] == "strong"),
+            "declared_unreached": len(unreached),
+            "declared_unreached_actionable": len(unreached) - unreached_non_ui,
+            "discovered_uncovered": len(section["discovered_uncovered"]),
+            "discovered_tested": len(section["discovered_tested"]),
+            # non_ui across tested UNION unreached. Counting it only among the
+            # unreached (which ``declared_unreached_actionable`` does, and which
+            # is right for its own purpose) would leave a tested-but-non_ui
+            # endpoint in the numerator and out of the denominator.
+            "tested_non_ui": tested_non_ui,
+            "declared_non_ui": tested_non_ui + unreached_non_ui,
+        }
+    counts["endpoints"]["unclassified"] = len(buckets["endpoints"]["unclassified"])
+    counts["modules"] = {"covered": len(buckets["modules"]["covered"])}
+    buckets["counts"] = counts
+    buckets["e2e_surface"] = {surface: surface_ratio(counts, surface) for surface in SURFACES}
+    return buckets

steppy/coverage/manifest.py

@@ -0,0 +1,193 @@
+"""Declared-surface ingest: OpenAPI endpoints + optional sitemap routes.
+
+``refresh_declared`` is the one entry point: it loads each configured
+manifest (http(s) URL via httpx, or a local file path — local paths keep
+tests offline and support repos that commit their spec), parses it as
+**JSON or YAML** (a served ``/openapi.json`` is the first, a spec committed
+to a repo is usually the second), replaces the
+per-source declared rows in the store, then runs the **re-link pass**:
+observed endpoint nodes (coverage but no declared row) whose evidence
+samples now match a declared template get their coverage re-pointed onto
+the declared node. That makes ``merge -> refresh`` and ``refresh -> merge``
+converge to the same map regardless of order.
+"""
+
+from __future__ import annotations
+
+import json
+import xml.etree.ElementTree as ET
+from pathlib import Path
+
+import yaml
+
+from .normalize import compile_extra_patterns, match_template, normalize_route
+from .store import (
+    DeclaredEntry,
+    observed_endpoint_nodes,
+    replace_declared_source,
+    repoint_node_coverage,
+)
+
+_HTTP_METHODS = ("get", "put", "post", "delete", "options", "head", "patch", "trace")
+
+
+class ManifestError(RuntimeError):
+    """Raised when a manifest cannot be loaded or parsed."""
+
+
+def load_text(location: str) -> str:
+    """Load manifest text from an http(s) URL or a local file path."""
+    if location.startswith(("http://", "https://")):
+        import httpx  # local import: keeps merge/report paths network-free
+
+        try:
+            response = httpx.get(location, timeout=30.0, follow_redirects=True)
+            response.raise_for_status()
+        except httpx.HTTPError as exc:
+            raise ManifestError(f"failed to fetch {location}: {exc}") from exc
+        return response.text
+    path = Path(location)
+    try:
+        return path.read_text(encoding="utf-8")
+    except OSError as exc:
+        raise ManifestError(f"failed to read {location}: {exc}") from exc
+
+
+def parse_manifest_document(text: str, api_name: str):
+    """Parse an OpenAPI document from either JSON or YAML.
+
+    JSON is tried first: it is what an application serves at ``/openapi.json``,
+    it is the fast path on a large spec, and its error names the exact offset.
+    YAML is the fallback because a spec **committed to a repo** is usually
+    ``.yaml`` — and ``load_text`` has always accepted a local path, so refusing
+    to parse the file it fetched was a gap rather than a policy.
+
+    Not YAML-only, even though YAML is a JSON superset: parsing a multi-megabyte
+    generated spec through the YAML loader is markedly slower, for a document
+    that is already known to be JSON in the common case.
+
+    The return type is deliberately unconstrained — shape validation belongs to
+    :func:`parse_openapi_endpoints`, which is where a non-mapping document has to
+    be rejected anyway, and duplicating the check would let the two disagree.
+    """
+    try:
+        return json.loads(text)
+    except ValueError:
+        pass
+    try:
+        return yaml.safe_load(text)
+    except yaml.YAMLError as exc:
+        raise ManifestError(f"OpenAPI for {api_name!r} is not valid JSON or YAML: {exc}") from exc
+
+
+def parse_openapi_endpoints(doc: dict, api_name: str) -> list[DeclaredEntry]:
+    """Extract ``paths`` x methods from an OpenAPI document.
+
+    Non-operation keys on path items (``parameters``, ``servers``, ``$ref``,
+    ``x-*`` extensions, ``summary``/``description``) are skipped.
+    """
+    if not isinstance(doc, dict):
+        # Format-neutral: the document may have arrived as YAML, and a YAML file
+        # that parses to a list would otherwise be reported as bad JSON.
+        raise ManifestError(f"OpenAPI document for {api_name!r} is not a mapping")
+    paths = doc.get("paths") or {}
+    entries: list[DeclaredEntry] = []
+    for path in sorted(paths):
+        item = paths[path]
+        if not isinstance(item, dict):
+            continue
+        for method in _HTTP_METHODS:
+            operation = item.get(method)
+            if not isinstance(operation, dict):
+                continue
+            details = {k: operation[k] for k in ("operationId", "summary", "tags") if k in operation}
+            entries.append(
+                DeclaredEntry(
+                    kind="endpoint",
+                    key=path,
+                    api=api_name,
+                    method=method.upper(),
+                    details=details,
+                )
+            )
+    return entries
+
+
+def parse_sitemap_routes(xml_text: str, route_param_patterns: list[str] | None = None) -> list[DeclaredEntry]:
+    """Extract ``<loc>`` URLs from a sitemap, normalized to route patterns.
+
+    Namespace-agnostic (matches any ``*}loc`` tag). Concrete URLs are run
+    through the same route normalization as observed navigations so declared
+    and observed route nodes share identities.
+    """
+    try:
+        root = ET.fromstring(xml_text)
+    except ET.ParseError as exc:
+        raise ManifestError(f"invalid sitemap XML: {exc}") from exc
+    extra = compile_extra_patterns(route_param_patterns or [])
+    keys = sorted(
+        {
+            normalize_route(el.text.strip(), extra)
+            for el in root.iter()
+            if el.tag.rsplit("}", 1)[-1] == "loc" and el.text and el.text.strip()
+        }
+    )
+    return [DeclaredEntry(kind="route", key=key) for key in keys]
+
+
+def relink_observed_endpoints(conn) -> int:
+    """Re-point coverage from observed endpoint nodes onto declared templates.
+
+    For each endpoint node with coverage but no declared row, retry its
+    evidence samples (raw relative paths) against the declared templates of
+    the same (api, method). First matching sample wins (samples are sorted,
+    so this is deterministic). The emptied observed node is kept — nodes are
+    never deleted implicitly.
+    """
+    templates: dict[tuple[str, str], dict[str, int]] = {}
+    for row in conn.execute(
+        "SELECT DISTINCT n.id, n.api, n.method, n.key FROM nodes n"
+        " JOIN declared d ON d.node_id = n.id WHERE n.kind = 'endpoint'"
+    ):
+        templates.setdefault((row["api"], row["method"]), {})[row["key"]] = int(row["id"])
+
+    moved = 0
+    for candidate in observed_endpoint_nodes(conn):
+        candidates = templates.get((candidate["api"], candidate["method"]))
+        if not candidates:
+            continue
+        target_key = None
+        for sample in candidate["samples"]:
+            target_key = match_template(sample, list(candidates))
+            if target_key:
+                break
+        if target_key:
+            moved += repoint_node_coverage(conn, candidate["id"], candidates[target_key])
+    return moved
+
+
+def refresh_declared(
+    conn,
+    apis: list[dict],
+    sitemap: str | None = None,
+    route_param_patterns: list[str] | None = None,
+    *,
+    updated_at: str = "",
+) -> dict:
+    """Refresh the whole declared surface and run the re-link pass.
+
+    ``apis`` entries are ``{"name": str, "openapi": <url-or-path>}``. Each
+    manifest source is replaced independently (``openapi:<name>`` /
+    ``sitemap``); triage flags and nodes are never touched.
+    """
+    summary: dict = {"apis": [], "sitemap": None, "relinked": 0}
+    for api in apis:
+        doc = parse_manifest_document(load_text(api["openapi"]), api["name"])
+        entries = parse_openapi_endpoints(doc, api["name"])
+        result = replace_declared_source(conn, f"openapi:{api['name']}", entries, updated_at=updated_at)
+        summary["apis"].append(result)
+    if sitemap:
+        entries = parse_sitemap_routes(load_text(sitemap), route_param_patterns)
+        summary["sitemap"] = replace_declared_source(conn, "sitemap", entries, updated_at=updated_at)
+    summary["relinked"] = relink_observed_endpoints(conn)
+    return summary