platformforge 0.1.0__py3-none-any.whl

platformforge/graph/bench.py

@@ -0,0 +1,245 @@
+"""Cycle 5 Phase O — graph + analytics scale benchmarks (§317).
+
+Every number is measured on seeded synthetic service graphs — no
+scale claim without a measurement. Complements platformforge/bench.py
+(which times the real eval corpus) with controlled-size scaling.
+"""
+
+from __future__ import annotations
+
+import json
+import tempfile
+import tracemalloc
+from pathlib import Path
+from time import perf_counter
+from typing import Any
+
+
+def _timer(fn, *a, **kw) -> tuple[Any, float]:
+    t0 = perf_counter()
+    out = fn(*a, **kw)
+    return out, round((perf_counter() - t0) * 1000, 2)
+
+
+def synthetic_graph(n_services: int = 200, fanout: int = 3):
+    """Deterministic seed — same n always produces the same graph."""
+    from platformforge.graph.model import Edge, Graph, Node
+    g = Graph()
+    for i in range(n_services):
+        g.add_node(Node.make("service", f"svc-{i}",
+                             attrs={"environment": "prod"}))
+    for i in range(n_services):
+        for j in range(1, fanout + 1):
+            t = (i + j * 17) % n_services
+            if t != i:
+                g.add_edge(Edge(f"service/svc-{i}", f"service/svc-{t}",
+                                "depends_on"))
+        g.add_node(Node.make("cluster", f"c-{i % 8}"))
+        g.add_edge(Edge(f"service/svc-{i}", f"cluster/c-{i % 8}",
+                        "runs_on"))
+    return g
+
+
+def _environment() -> dict[str, Any]:
+    """§97 — benchmark env: CPU, RAM, Python, OS, commit SHA."""
+    import os
+    import platform
+    import subprocess
+    mem_gib = None
+    try:
+        mem_gib = round(os.sysconf("SC_PAGE_SIZE")
+                        * os.sysconf("SC_PHYS_PAGES") / (1024 ** 3), 1)
+    except (ValueError, OSError, AttributeError):
+        pass
+    sha = subprocess.run(["git", "rev-parse", "HEAD"], check=False,
+                         capture_output=True, text=True).stdout.strip()
+    return {"cpu": platform.processor() or platform.machine(),
+            "cores": os.cpu_count(), "ram_gib": mem_gib,
+            "python": platform.python_version(),
+            "os": f"{platform.system()} {platform.release()}",
+            "commit": sha}
+
+
+def run_scale_benchmarks(sizes: tuple[int, ...] = (50, 200, 800, 2000,
+                                                  10_000),
+                         max_mem_gib: float = 4.0,
+                         edge_targets: tuple[int, ...] = (
+                             100_000, 250_000, 500_000),
+                         store_sweep: bool = False) -> dict[str, Any]:
+    """Measured timings across graph build/query/serialize/diff and
+    the analytics SQLite store. §208 — when a size can't be attempted
+    on this host the entry records `unsupported-on-host`, never a hang."""
+    from platformforge.analytics.store import AnalyticsStore
+    from platformforge.graph.diff import diff as graph_diff
+    from platformforge.graph.query import blast_radius, dependents
+
+    report: dict[str, Any] = {
+        "schema": "platformforge/scale-benchmarks/v2",
+        "environment": _environment(),
+        "method": "perf_counter ms; tracemalloc KiB; seeded synthetic "
+                  "graphs (deterministic shape per n)",
+        "sizes": {}}
+
+    for n in sizes:
+        # ~10KiB/node measured at n=800 — refuse before swapping
+        est_gib = n * 10 * 1024 / (1024 ** 3)
+        if est_gib > max_mem_gib:
+            report["sizes"][str(n)] = {
+                "status": "unsupported-on-host",
+                "reason": f"est {est_gib:.1f}GiB > {max_mem_gib}GiB "
+                          "host budget"}
+            continue
+        try:
+            report["sizes"][str(n)] = _bench_size(
+                n, AnalyticsStore, graph_diff, blast_radius, dependents)
+        except MemoryError:
+            report["sizes"][str(n)] = {
+                "status": "unsupported-on-host",
+                "reason": "MemoryError during benchmark"}
+
+    # §207 — edge-count sweep on a fixed small node set (dense graph)
+    report["edge_sweep"] = {}
+    for target_edges in edge_targets:
+        nodes, fanout = 10_000, (target_edges // 10_000) or 1
+        est_gib = (nodes * 10 * 1024 + target_edges * 512) / (1024 ** 3)
+        if est_gib > max_mem_gib:
+            report["edge_sweep"][str(target_edges)] = {
+                "status": "unsupported-on-host",
+                "reason": f"est {est_gib:.1f}GiB > {max_mem_gib}GiB"}
+            continue
+        try:
+            g, t_build = _timer(synthetic_graph, nodes, fanout)
+            g2, _ = _timer(synthetic_graph, nodes, fanout)
+            from platformforge.graph.model import Edge
+            g2.add_edge(Edge("service/svc-0", "service/svc-7",
+                             "depends_on"))
+            _d, t_diff = _timer(graph_diff, g, g2)
+            report["edge_sweep"][str(target_edges)] = {
+                "status": "measured", "nodes": len(g.nodes),
+                "edges": len(g.edges), "build_ms": t_build,
+                "diff_ms": t_diff}
+            del g, g2
+        except MemoryError:
+            report["edge_sweep"][str(target_edges)] = {
+                "status": "unsupported-on-host",
+                "reason": "MemoryError"}
+
+    # §96 — analytics store event sweep (bulk ingest + windowed query)
+    if store_sweep:
+        report["store_sweep"] = _store_sweep(AnalyticsStore)
+    return report
+
+
+def _store_sweep(AnalyticsStore) -> dict[str, Any]:
+    out: dict[str, Any] = {}
+    for n_events in (10_000, 100_000, 1_000_000):
+        est_gib = n_events * 1024 / (1024 ** 3)  # ~1KiB/event
+        if est_gib > 2.0:
+            out[str(n_events)] = {"status": "unsupported-on-host",
+                                  "reason": f"est {est_gib:.1f}GiB store "
+                                            "budget"}
+            continue
+        try:
+            with tempfile.TemporaryDirectory() as td:
+                st = AnalyticsStore(Path(td) / "s.db")
+                t_ins = _timer(_fill_events, st, n_events)[1]
+                _e, t_q = _timer(st.events)
+                _w, t_win = _timer(st.events, kind="operation")
+                t_forget = _timer(st.forget_subject, "svc:7")[1]
+                size = (Path(td) / "s.db").stat().st_size
+                st.close()
+                out[str(n_events)] = {
+                    "status": "measured",
+                    "bulk_insert_ms": t_ins, "query_ms": t_q,
+                    "windowed_query_ms": t_win,
+                    "forget_ms": t_forget, "db_bytes": size}
+        except MemoryError:
+            out[str(n_events)] = {"status": "unsupported-on-host",
+                                  "reason": "MemoryError"}
+    return out
+
+
+def _bench_size(n: int, AnalyticsStore, graph_diff, blast_radius,
+                dependents) -> dict[str, Any]:
+    from platformforge.graph.query import neighbors, paths
+    g, t_build = _timer(synthetic_graph, n)
+    g2, _ = _timer(synthetic_graph, n)
+    merged, t_merge = _timer(_merge, g, g2)
+    deps, t_deps = _timer(dependents, g, "service/svc-0")
+    blast, t_blast = _timer(blast_radius, g, "service/svc-0")
+    nb, t_nb = _timer(neighbors, g, "service/svc-0")
+    pth, t_path = _timer(paths, g, "service/svc-0", "service/svc-5")
+    blob, t_ser = _timer(json.dumps, g.to_dict(), default=str)
+    t_deser = _timer(json.loads, blob)[1]
+    _d, t_diff = _timer(graph_diff, g, g2)
+
+    tracemalloc.start()
+    synthetic_graph(n)
+    _, peak = tracemalloc.get_traced_memory()
+    tracemalloc.stop()
+
+    with tempfile.TemporaryDirectory() as td:
+        st = AnalyticsStore(Path(td) / "a.db")
+        t_ins = _timer(_fill_events, st, n)[1]
+        _e, t_q = _timer(st.events)
+        t_forget = _timer(st.forget_subject, "svc:x")[1]
+        size = (Path(td) / "a.db").stat().st_size
+        st.close()
+
+    return {
+        "status": "measured",
+        "nodes": len(g.nodes), "edges": len(g.edges),
+        "build_ms": t_build, "merge_ms": t_merge,
+        "dependents_ms": t_deps, "dependents_found": len(deps),
+        "neighbors_ms": t_nb, "neighbors_found": len(nb),
+        "path_ms": t_path, "paths_found": len(pth),
+        "blast_ms": t_blast,
+        "blast_nodes": len(blast.get("nodes", [])),
+        "serialize_ms": t_ser, "deserialize_ms": t_deser,
+        "snapshot_bytes": len(blob.encode()),
+        "diff_ms": t_diff,
+        "peak_build_kib": round(peak / 1024, 1),
+        "store_insert_ms": t_ins, "store_query_ms": t_q,
+        "store_forget_ms": t_forget,
+        "store_size_bytes": size,
+        "merged_nodes": len(merged.nodes)}
+
+
+def _merge(g1, g2):
+    from platformforge.graph.model import Graph
+    out = Graph()
+    for g in (g1, g2):
+        for n in g.nodes.values():
+            out.add_node(n)
+        for e in g.edges.values():
+            out.add_edge(e)
+    return out
+
+
+def _fill_events(st, n: int) -> None:
+    st.put_events([
+        {"kind": "operation",
+         "ts": f"2025-11-{(i % 28) + 1:02d}T00:00:00Z",
+         "subject": f"svc:{i % 50}", "outcome": "converged",
+         "source": "bench", "attrs": {"i": i}} for i in range(n)])
+
+
+def benchmark_summary(report: dict[str, Any]) -> list[str]:
+    """Numbers only — no extrapolated scale claims; unsupported sizes
+    reported as such (§208)."""
+    out = []
+    for n, m in report["sizes"].items():
+        if m.get("status") == "unsupported-on-host":
+            out.append(f"n={n} UNSUPPORTED-ON-HOST — {m['reason']}")
+            continue
+        out.append(
+            f"n={n} nodes={m['nodes']} edges={m['edges']} | "
+            f"build {m['build_ms']}ms merge {m['merge_ms']}ms "
+            f"deps {m['dependents_ms']}ms blast {m['blast_ms']}ms "
+            f"diff {m['diff_ms']}ms ser {m['serialize_ms']}ms "
+            f"deser {m['deserialize_ms']}ms | "
+            f"peak {m['peak_build_kib']}KiB | "
+            f"store ins {m['store_insert_ms']}ms "
+            f"q {m['store_query_ms']}ms "
+            f"forget {m['store_forget_ms']}ms ({m['store_size_bytes']}B)")
+    return out

platformforge/graph/build.py

@@ -0,0 +1,130 @@
+"""GraphBuilder — facts in, provenanced graph out.
+
+Domain analyzers (Terraform, Kubernetes, …) emit Facts; a fact may declare
+graph contributions under `attrs["graph"]`:
+
+    {"graph": {
+        "nodes": [{"kind": "workload", "label": "payments", "attrs": {...}}],
+        "edges": [{"src_kind": "workload", "src": "payments",
+                   "dst_kind": "secret", "dst": "db-creds",
+                   "kind": "uses_secret"}]}}
+
+Edges inherit the fact's tier as provenance (t1/t2 → observed, t3 → declared,
+t4 → inferred) and always carry the fact_id — the graph is derived from
+evidence, never guessed.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterable
+from typing import Any, ClassVar
+
+from platformforge.graph.model import Edge, Graph, Node, node_id
+from platformforge.graph.vocab import PROVENANCES
+
+_PROV_RANK = {"observed": 3, "planned": 2, "declared": 1, "inferred": 0}
+
+
+def _tier_provenance(tier: int | str) -> str:
+    """EvidenceTier int → edge provenance (§4): 0-1 observed (measured,
+    provider), 2 planned (generated plan — never observed), 3-5 declared,
+    6-7 inferred. Out-of-range tiers mint `inferred`, never `observed`."""
+    t = {f"t{i}": i for i in range(8)}.get(tier, tier)
+    try:
+        t = int(t)
+    except (TypeError, ValueError):
+        return "declared"
+    if not 0 <= t <= 7:
+        return "inferred"  # garbage tier must not mint strong evidence
+    if t <= 1:
+        return "observed"
+    return {2: "planned"}.get(t, "declared" if t <= 5 else "inferred")
+
+
+def _clamp_provenance(claimed: str, tier_prov: str) -> str:
+    """A fact may downgrade its edge provenance, never upgrade beyond what
+    its evidence tier supports (raw dicts bypass Fact validation)."""
+    if claimed not in _PROV_RANK:
+        return tier_prov
+    return claimed if _PROV_RANK[claimed] <= _PROV_RANK[tier_prov] \
+        else tier_prov
+
+
+class GraphBuilder:
+    def __init__(self):
+        self.graph = Graph()
+
+    def add_node(self, kind: str, label: str,
+                 attrs: dict[str, Any] | None = None,
+                 fact_ids: Iterable[str] = ()) -> Node:
+        return self.graph.add_node(
+            Node.make(kind, label, attrs, fact_ids))
+
+    def add_edge(self, src_kind: str, src_label: str, dst_kind: str,
+                 dst_label: str, kind: str, provenance: str = "declared",
+                 confidence: float = 1.0,
+                 fact_ids: Iterable[str] = (),
+                 attrs: dict[str, Any] | None = None) -> Edge:
+        if provenance not in PROVENANCES:
+            raise ValueError(f"bad provenance: {provenance}")
+        s, d = node_id(src_kind, src_label), node_id(dst_kind, dst_label)
+        for nid, k, lbl in ((s, src_kind, src_label),
+                            (d, dst_kind, dst_label)):
+            if nid not in self.graph.nodes:
+                self.add_node(k, lbl)
+        return self.graph.add_edge(
+            Edge(src=s, dst=d, kind=kind, provenance=provenance,
+                 confidence=confidence,
+                 source_fact_ids=tuple(sorted(set(fact_ids))),
+                 attrs=dict(attrs or {})))
+
+    # §5: node state is resolved from the strongest contributing fact —
+    # observed beats planned beats desired(declared) beats inferred.
+    _STATE_ORDER: ClassVar[dict[str, int]] = {
+        "observed": 0, "planned": 1, "desired": 2, "inferred": 3}
+    _PROV_STATE: ClassVar[dict[str, str]] = {
+        "observed": "observed", "planned": "planned",
+        "declared": "desired", "inferred": "inferred"}
+
+    def from_facts(self, facts: Iterable[dict[str, Any]]) -> GraphBuilder:
+        facts = list(facts)  # consumed twice: fact edges + delivery joins
+        state_rank: dict[str, int] = {}
+        for f in facts:
+            contrib = (f.get("attrs") or {}).get("graph") or {}
+            fid = f.get("fact_id", "")
+            prov = _tier_provenance(f.get("tier", 3))
+            for n in contrib.get("nodes", []):
+                node = self.add_node(n["kind"], n["label"], n.get("attrs"),
+                                     [fid] if fid else [])
+                rank = self._STATE_ORDER[self._PROV_STATE[prov]]
+                if rank < state_rank.get(node.node_id, 99):
+                    state_rank[node.node_id] = rank
+                    node.attrs["state"] = self._PROV_STATE[prov]
+            for e in contrib.get("edges", []):
+                self.add_edge(e["src_kind"], e["src"], e["dst_kind"],
+                              e["dst"], e["kind"],
+                              provenance=_clamp_provenance(
+                                  e.get("provenance", prov), prov),
+                              confidence=e.get("confidence", 1.0),
+                              fact_ids=[fid] if fid else [],
+                              attrs=e.get("attrs"))
+        self._delivery(facts)
+        return self
+
+    def _delivery(self, facts: Iterable[dict[str, Any]]) -> None:
+        """§72/§135 — join gitops/workflows/workloads into the delivery
+        chain, then resolve cross-member links (inferred provenance)."""
+        from platformforge.graph.delivery import cross_repo_edges, delivery_edges
+        facts = list(facts)
+        for e in delivery_edges(facts):
+            self.add_edge(e["src_kind"], e["src"], e["dst_kind"], e["dst"],
+                          e["kind"], provenance="declared",
+                          fact_ids=e.get("fact_ids") or [],
+                          attrs={"via": e.get("via")})
+        for e in cross_repo_edges(facts):
+            self.add_edge(e["src_kind"], e["src"], e["dst_kind"], e["dst"],
+                          e["kind"],
+                          provenance=e.get("provenance", "inferred"),
+                          confidence=e.get("confidence", 0.6),
+                          fact_ids=e.get("fact_ids") or [],
+                          attrs={"via": e.get("via")})

platformforge/graph/delivery.py

@@ -0,0 +1,183 @@
+"""§72 Software delivery graph — derive cross-domain edges from facts.
+
+The analyzers emit local edges (workload→namespace, app→repo). This
+enricher joins across fact kinds to surface the full chain:
+
+    repository → workflow → artifact/image → gitops → cluster → workload
+
+Joins are deterministic and evidence-carrying: an edge is only emitted
+when both endpoints exist in the fact set, and the edge attrs cite the
+fact_ids that motivated it.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+
+def _repo_key(url: str | None) -> str | None:
+    if not url:
+        return None
+    u = url.removesuffix(".git").rstrip("/")
+    return u.rsplit("/", 1)[-1].lower() or None
+
+
+def _image_repo(img: str | None) -> str | None:
+    if not img:
+        return None
+    return img.split(":", 1)[0].rsplit("/", 1)[-1].lower() or None
+
+
+def delivery_edges(facts: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    """Return derived edges [{src,src_kind,dst,dst_kind,kind,via,fact_ids}]."""
+    repos: dict[str, list[str]] = {}          # repo key → fact ids
+    wf_repos: list[tuple[str, str]] = []      # (repo key, workflow fact id)
+    image_facts: dict[str, list[str]] = {}    # image repo → workload fact ids
+    gitops_repos: list[tuple[str, str]] = []  # (repo key, gitops fact id)
+    gitops_ns: list[tuple[str, str]] = []     # (namespace, gitops fact id)
+
+    for f in facts:
+        kind, a = f.get("kind", ""), f.get("attrs") or {}
+        fid = f.get("fact_id", "")
+        if kind == "gitops.argocd_app":
+            for r in a.get("source_repos") or [a.get("source_repo")]:
+                k = _repo_key(r)
+                if k:
+                    gitops_repos.append((k, fid))
+            if a.get("dest_namespace"):
+                gitops_ns.append((a["dest_namespace"], fid))
+        elif kind == "gitops.flux_resource":
+            if a.get("url"):
+                k = _repo_key(a["url"])
+                if k:
+                    gitops_repos.append((k, fid))
+        elif kind == "k8s.workload":
+            for img in (a.get("pod_spec") or {}).get("images") or []:
+                k = _image_repo(img)
+                if k:
+                    image_facts.setdefault(k, []).append(fid)
+        elif kind in ("gha.workflow", "cicd.pipeline"):
+            k = _repo_key(a.get("repo") or a.get("source_repo"))
+            if k:
+                wf_repos.append((k, fid))
+        elif kind in ("repo.file", "iac.resource", "catalog.component"):
+            k = _repo_key(a.get("repo") or a.get("repository"))
+            if k:
+                repos.setdefault(k, []).append(fid)
+
+    edges: list[dict[str, Any]] = []
+    seen: set[tuple[str, str, str]] = set()
+
+    def add(sk: str, s: str, dk: str, d: str, ek: str, via: str,
+            fids: list[str]) -> None:
+        if (sk, dk, d) in seen:
+            return
+        seen.add((sk, dk, d))
+        edges.append({"src_kind": sk, "src": s, "dst_kind": dk, "dst": d,
+                      "kind": ek, "via": via, "fact_ids": fids})
+
+    # gitops app → repository it syncs → workflows that build it
+    for rk, gfid in gitops_repos:
+        add("argocd_application", f"fact:{gfid}", "repository", rk,
+            "depends_on", "source_repo", [gfid])
+        for rk2, wfid in wf_repos:
+            if rk2 == rk:
+                add("workflow", f"fact:{wfid}", "argocd_application",
+                    f"fact:{gfid}", "generated_by", "same_repo",
+                    [wfid, gfid])
+
+    # gitops app → workload via destination namespace match
+    for ns, gfid in gitops_ns:
+        for f in facts:
+            if f.get("kind") == "k8s.workload" and \
+                    (f.get("attrs") or {}).get("namespace") == ns:
+                add("argocd_application", f"fact:{gfid}", "workload",
+                    f"fact:{f['fact_id']}", "deploys_to",
+                    "dest_namespace", [gfid, f["fact_id"]])
+
+    # image repo produced by workflow → consumed by workload
+    for ik, wfids in image_facts.items():
+        for rk, wfid in wf_repos:
+            if rk == ik:
+                for wf in wfids:
+                    add("workflow", f"fact:{wfid}", "workload",
+                        f"fact:{wf}", "generated_by", "image_repo",
+                        [wfid, wf])
+    return edges
+
+
+def cross_repo_edges(facts: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    """§135 — resolve the multi-repo chain across workspace members:
+
+        application repo → image → gitops repo → workload
+        infra repo → cluster
+        workload → cluster
+
+    Facts carry `attrs.workspace_member` when collected via a multi-member
+    workspace. Joins are name/key-based and therefore provenance `inferred`
+    — they are hypotheses to confirm, never observed links."""
+    edges: list[dict[str, Any]] = []
+    seen: set[tuple[str, str, str]] = set()
+
+    def add(sk: str, s: str, dk: str, d: str, ek: str, via: str,
+            fids: list[str]) -> None:
+        if (sk, ek, d) in seen:
+            return
+        seen.add((sk, ek, d))
+        edges.append({"src_kind": sk, "src": s, "dst_kind": dk, "dst": d,
+                      "kind": ek, "via": via, "fact_ids": fids,
+                      "provenance": "inferred"})
+
+    clusters: dict[str, tuple[str, str]] = {}   # name → (member, fact_id)
+    gitops_dests: list[tuple[str, str, str, str]] = []  # (server/name, ns, member, fid)
+    member_of: dict[str, str] = {}              # fact_id → member
+
+    for f in facts:
+        a = f.get("attrs") or {}
+        fid = f.get("fact_id", "")
+        member = a.get("workspace_member", "")
+        if member:
+            member_of[fid] = member
+        kind = f.get("kind", "")
+        if kind.endswith(".cluster") or (
+                kind == "iac.resource"
+                and "cluster" in str(a.get("type", "")).lower()):
+            cname = str(a.get("name") or f.get("location") or "")
+            if cname:
+                clusters[cname] = (member, fid)
+        if kind == "gitops.argocd_app":
+            server = str(a.get("dest_server") or a.get("dest_name") or "")
+            ns = str(a.get("dest_namespace") or "")
+            if server:
+                gitops_dests.append((server, ns, member, fid))
+
+    # gitops app → cluster (dest server/name match), possibly cross-member
+    for server, ns, member, gfid in gitops_dests:
+        skey = server.rsplit("/", 1)[-1]
+        for cname, (cmember, cfid) in clusters.items():
+            if skey in (cname, server) or cname in skey:
+                add("argocd_application", f"fact:{gfid}", "cluster", cname,
+                    "deploys_to", "dest_server_match", [gfid, cfid])
+                if member and cmember and member != cmember:
+                    add("repository", member, "cluster", cname,
+                        "deploys_to", "cross_repo_gitops", [gfid, cfid])
+
+    # infra member → cluster it provisioned (iac/crossplane fact lives there)
+    for cname, (cmember, cfid) in clusters.items():
+        if cmember:
+            add("cluster", cname, "repository", cmember,
+                "provisioned_by", "member_attr", [cfid])
+
+    # gitops repo member → workloads it deploys to in another member
+    for f in facts:
+        if f.get("kind") != "k8s.workload":
+            continue
+        a = f.get("attrs") or {}
+        wmember = a.get("workspace_member", "")
+        ns = a.get("namespace", "")
+        for server, gns, gmember, gfid in gitops_dests:
+            if gns == ns and gmember and wmember and gmember != wmember:
+                add("repository", gmember, "workload",
+                    f"fact:{f['fact_id']}", "deploys_to",
+                    "cross_repo_ns_match", [gfid, f["fact_id"]])
+    return edges