platformforge 0.1.0__py3-none-any.whl

platformforge/ops/store.py

@@ -0,0 +1,157 @@
+"""Cycle 4 phase V — operation store.
+
+Append-only JSONL store under `.platformforge/operations/`. Each
+operation gets a directory-free design: one `<op_id>.ledger.jsonl` +
+`<op_id>.op.json` snapshot. Schema versioned; `verify()` re-checks
+every hash chain; `load()` migrates older schema versions forward.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+from platformforge.live.models import canonical_hash, now_iso
+from platformforge.ops.operation import LedgerEntry, Operation, OperationLedger
+
+STORE_SCHEMA_VERSION = 2
+
+
+class OperationStore:
+    def __init__(self, root: str | Path = ".platformforge/operations"):
+        self.root = Path(root)
+        self.root.mkdir(parents=True, exist_ok=True)
+        mv = self.root / "_schema_version"
+        if not mv.exists():
+            mv.write_text(str(STORE_SCHEMA_VERSION))
+        elif int(mv.read_text().strip()) > STORE_SCHEMA_VERSION:
+            raise ValueError(f"operation store schema "
+                             f"{mv.read_text().strip()} newer than "
+                             f"supported {STORE_SCHEMA_VERSION}")
+        elif int(mv.read_text().strip()) < STORE_SCHEMA_VERSION:
+            self._migrate(int(mv.read_text().strip()))
+
+    def _migrate(self, from_v: int) -> None:
+        # v1 → v2: ledger files gained a schema field; nothing else moves
+        for f in self.root.glob("*.ledger.jsonl"):
+            lines = [json.loads(line) for line in
+                     f.read_text().splitlines() if line.strip()]
+            for rec in lines:
+                rec.setdefault("schema", 1)
+            f.write_text("".join(json.dumps(r, sort_keys=True) + "\n"
+                                 for r in lines))
+        (self.root / "_schema_version").write_text(
+            str(STORE_SCHEMA_VERSION))
+
+    def save(self, op: Operation, ledger: OperationLedger,
+             envelope=None, materials: dict | None = None) -> Path:
+        """Append new ledger entries + write op snapshot (+ envelope
+        for later `ops rollback`/`ops status` replay, + immutable
+        rollback materials captured during execution, §30)."""
+        op_path = self.root / f"{op.operation_id}.op.json"
+        led_path = self.root / f"{op.operation_id}.ledger.jsonl"
+        existing = 0
+        if led_path.exists():
+            existing = sum(1 for _ in led_path.open())
+        with led_path.open("a") as f:
+            for e in ledger.entries[existing:]:
+                rec = e.to_dict()
+                rec["schema"] = STORE_SCHEMA_VERSION
+                f.write(json.dumps(rec, sort_keys=True) + "\n")
+        snap = op.to_dict() if hasattr(op, "to_dict") else {
+            "operation_id": op.operation_id, "state": op.state}
+        snap["schema"] = STORE_SCHEMA_VERSION
+        snap["stored_at"] = now_iso()
+        op_path.write_text(json.dumps(snap, indent=2, sort_keys=True))
+        if envelope is not None:
+            env_d = envelope.to_dict() if hasattr(envelope, "to_dict") \
+                else dict(envelope)
+            (self.root / f"{op.operation_id}.env.json").write_text(
+                json.dumps(env_d, indent=2, sort_keys=True))
+        if materials is not None:
+            mats = {sid: (m.to_dict() if hasattr(m, "to_dict") else m)
+                    for sid, m in materials.items()}
+            (self.root / f"{op.operation_id}.materials.json").write_text(
+                json.dumps(mats, indent=2, sort_keys=True))
+        return op_path
+
+    def load_operation(self, operation_id: str) -> Operation | None:
+        p = self.root / f"{operation_id}.op.json"
+        if not p.exists():
+            return None
+        return Operation.from_dict(json.loads(p.read_text()))
+
+    def load_envelope(self, operation_id: str) -> dict[str, Any] | None:
+        p = self.root / f"{operation_id}.env.json"
+        if not p.exists():
+            return None
+        return json.loads(p.read_text())
+
+    def load_materials(self, operation_id: str) -> dict[str, Any]:
+        p = self.root / f"{operation_id}.materials.json"
+        if not p.exists():
+            return {}
+        return json.loads(p.read_text())
+
+    def load_ledger(self, operation_id: str) -> OperationLedger:
+        led = OperationLedger()
+        p = self.root / f"{operation_id}.ledger.jsonl"
+        if not p.exists():
+            return led
+        for line in p.read_text().splitlines():
+            if not line.strip():
+                continue
+            r = json.loads(line)
+            e = LedgerEntry(seq=r["seq"], event=r["event"],
+                            operation_id=r["operation_id"],
+                            actor=r.get("actor", "system"),
+                            at=r.get("at", ""),
+                            data=dict(r.get("data", {})),
+                            prev_hash=r.get("prev_hash", ""))
+            e.entry_hash = r.get("entry_hash", e.entry_hash)
+            led.entries.append(e)
+            led._tip = e.entry_hash
+        return led
+
+    def verify(self, operation_id: str) -> dict[str, Any]:
+        led = self.load_ledger(operation_id)
+        return {"operation_id": operation_id,
+                "entries": len(led.entries),
+                "chain_valid": led.verify_chain(),
+                "tip": led.tip}
+
+    def list_operations(self) -> list[str]:
+        return sorted(p.stem[:-3] for p in self.root.glob("*.op.json"))
+
+    def projection(self, operation_id: str) -> dict[str, Any] | None:
+        """Rebuild the operational Graphfy projection (phase R) from
+        the stored op/envelope/ledger — derived on read, never a
+        third artifact that could drift from the ledger."""
+        from platformforge.ops.envelope import ExecutionEnvelope
+        from platformforge.ops.opgraph import project_operation
+        op = self.load_operation(operation_id)
+        if op is None:
+            return None
+        env_d = self.load_envelope(operation_id)
+        env = ExecutionEnvelope.from_dict(env_d) if env_d else None
+        ledger = self.load_ledger(operation_id)
+        # rollback receipt: reconstructed from the ledger events (the
+        # ledger is the source of truth — never a second artifact)
+        rb_rcpt = None
+        for e in ledger.entries:
+            if e.event == "rollback.completed":
+                rb_rcpt = dict(e.data)
+        proj = project_operation(op, envelope=env, ledger=ledger,
+                                 materials=self.load_materials(
+                                     operation_id),
+                                 rollback_receipt=rb_rcpt)
+        return {"operation_id": operation_id,
+                "nodes": [n.to_dict() for n in proj["nodes"]],
+                "edges": [e.to_dict() for e in proj["edges"]]}
+
+    def audit_digest(self) -> str:
+        """Digest of every stored ledger tip — tamper-evident index."""
+        tips = {op: self.load_ledger(op).tip
+                for op in self.list_operations()}
+        return "sha256:" + canonical_hash(tips)

platformforge/ops/verify.py

@@ -0,0 +1,190 @@
+"""Cycle 4 — verification engine (§110–118, ADR-0028).
+
+Execution success ≠ outcome success. Verification compares the
+*ExpectedDelta* declared at plan time against the *ObservedDelta*
+measured from live evidence (provider state, k8s status, runtime
+topology, SLO, alerts) across windows: immediate → stabilization →
+extended. SLO degradation after a change is a verification failure —
+even when the executor exited 0.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+from platformforge.live.models import age_seconds, now_iso, parse_ts
+
+CONVERGENCE = ("converged", "partially-converged", "not-converged",
+               "regressed", "unknown")
+WINDOWS = ("immediate", "stabilization", "extended")
+DEFAULT_WINDOW_TTLS = {"immediate": 60, "stabilization": 600,
+                       "extended": 3600}
+
+
+@dataclass
+class WindowResult:
+    window: str
+    status: str = "unknown"          # pass|fail|pending|unknown
+    matched: list[str] = field(default_factory=list)
+    missing: list[str] = field(default_factory=list)
+    unexpected: list[str] = field(default_factory=list)
+    detail: str = ""
+
+    def to_dict(self) -> dict[str, Any]:
+        return {"window": self.window, "status": self.status,
+                "matched": self.matched, "missing": self.missing,
+                "unexpected": self.unexpected, "detail": self.detail}
+
+
+@dataclass
+class VerificationResult:
+    convergence: str = "unknown"
+    windows: list[WindowResult] = field(default_factory=list)
+    slo_gate: dict[str, Any] = field(default_factory=dict)
+    checked_at: str = ""
+    evidence: list[str] = field(default_factory=list)
+    verification_coverage: dict[str, Any] = field(default_factory=dict)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {"convergence": self.convergence,
+                "windows": [w.to_dict() for w in self.windows],
+                "slo_gate": self.slo_gate, "checked_at": self.checked_at,
+                "evidence": self.evidence,
+                "verification_coverage": self.verification_coverage}
+
+
+def _flatten(delta: dict[str, Any]) -> set[str]:
+    """Flatten ExpectedDelta {adds|removes|changes: {dim: [...]}} to
+    canonical 'dim:item' tokens for comparison."""
+    out: set[str] = set()
+    for side in ("adds", "removes", "changes"):
+        for dim, items in (delta.get(side) or {}).items():
+            for it in items or []:
+                key = it.get("id") if isinstance(it, dict) else str(it)
+                out.add(f"{side}:{dim}:{key}")
+    return out
+
+
+def verify_delta(expected: dict[str, Any], observed: dict[str, Any]
+                 ) -> tuple[list[str], list[str], list[str]]:
+    """Compare expected vs observed delta token sets. Returns
+    (matched, missing, unexpected)."""
+    exp, obs = _flatten(expected), _flatten(observed)
+    return (sorted(exp & obs), sorted(exp - obs), sorted(obs - exp))
+
+
+def slo_gate(slo: dict[str, Any] | None,
+             metrics: dict[str, Any] | None) -> dict[str, Any]:
+    """§117–118 — SLO regression gate. `slo` = contract
+    {error_budget_threshold, burn_rate_max}; `metrics` = observed
+    {error_rate, burn_rate}. Unknown inputs → unknown, never pass."""
+    if not slo or not metrics:
+        return {"status": "unknown",
+                "detail": "no slo contract or metrics supplied"}
+    er = metrics.get("error_rate")
+    br = metrics.get("burn_rate")
+    if er is None and br is None:
+        return {"status": "unknown", "detail": "no error/burn metrics"}
+    fails = []
+    if er is not None and slo.get("error_rate_max") is not None \
+            and er > slo["error_rate_max"]:
+        fails.append(f"error_rate {er} > {slo['error_rate_max']}")
+    if br is not None and slo.get("burn_rate_max") is not None \
+            and br > slo["burn_rate_max"]:
+        fails.append(f"burn_rate {br} > {slo['burn_rate_max']}")
+    if metrics.get("budget_remaining") == "critical":
+        fails.append("error budget critical")
+    return {"status": "fail" if fails else "pass", "violations": fails}
+
+
+def verification_coverage(expected: dict[str, Any],
+                          observed: dict[str, Any]) -> dict[str, Any]:
+    """Cycle 4.1 — how much of the ExpectedDelta was actually
+    observed. `converged` is only meaningful when coverage is 1.0;
+    less → honest partial/unknown, never a silent pass."""
+    exp = _flatten(expected)
+    if not exp:
+        return {"expected_tokens": 0, "matched": 0, "ratio": 0.0,
+                "sufficient": False,
+                "note": "empty expected delta — verification cannot "
+                        "certify convergence (PF-OPS-NO-DELTA guard)"}
+    matched, missing, _ = verify_delta(expected, observed)
+    ratio = len(matched) / len(exp)
+    return {"expected_tokens": len(exp), "matched": len(matched),
+            "missing": len(missing), "ratio": round(ratio, 4),
+            "sufficient": ratio >= 1.0}
+
+
+def verify(*, expected_delta: dict[str, Any],
+           observations: dict[str, Any],
+           slo_contract: dict[str, Any] | None = None,
+           metrics: dict[str, Any] | None = None,
+           windows: list[str] | None = None,
+           evidence_refs: list[str] | None = None,
+           mutating: bool = True) -> VerificationResult:
+    """`observations` maps window → observed-delta dict (already
+    computed by the live layer). A window with no observation is
+    `unknown`, never pass. `mutating=False` marks read-only plans
+    (validate/show/observe/diff) exempt from the delta sufficiency
+    gate."""
+    res = VerificationResult(checked_at=now_iso(),
+                             evidence=list(evidence_refs or []))
+    for w in (windows or list(WINDOWS)):
+        obs = observations.get(w)
+        wr = WindowResult(window=w)
+        if obs is None:
+            wr.status = "unknown"
+            wr.detail = "no observation for this window"
+        else:
+            matched, missing, unexpected = verify_delta(expected_delta, obs)
+            wr.matched, wr.missing, wr.unexpected = matched, missing, unexpected
+            wr.status = "pass" if not missing and not unexpected else "fail"
+        res.windows.append(wr)
+
+    res.slo_gate = slo_gate(slo_contract, metrics)
+
+    # Cycle 4.1 — verification coverage across all supplied windows;
+    # the best-observed window drives the sufficiency claim.
+    coverages = {wname: verification_coverage(expected_delta, obs)
+                 for wname, obs in observations.items()
+                 if obs is not None}
+    res.verification_coverage = {
+        "per_window": coverages,
+        "sufficient": any(c["sufficient"] for c in coverages.values()),
+        "expected_tokens": len(_flatten(expected_delta)),
+        "unknown_dimensions": list(
+            (expected_delta or {}).get("unknown_dimensions", []))}
+
+    statuses = [w.status for w in res.windows]
+    if "fail" in statuses or res.slo_gate.get("status") == "fail":
+        if res.slo_gate.get("status") == "fail" and "fail" not in statuses:
+            res.convergence = "regressed"
+        elif all(s == "fail" for s in statuses if s != "unknown"):
+            res.convergence = "not-converged"
+        else:
+            res.convergence = "partially-converged"
+    elif all(s == "pass" for s in statuses) and statuses:
+        res.convergence = ("converged"
+                           if res.slo_gate.get("status") in ("pass", "unknown")
+                           else "partially-converged")
+    elif any(s == "pass" for s in statuses):
+        res.convergence = "partially-converged"
+    else:
+        res.convergence = "unknown"
+
+    # §110–118 + cycle 4.1: never claim converged on an empty/insufficient
+    # delta for a mutating plan — the token set matched vacuously.
+    if mutating and res.convergence == "converged" \
+            and not res.verification_coverage["sufficient"]:
+        res.convergence = "unknown"
+        res.verification_coverage["note"] = (
+            "convergence claimed without a sufficient expected delta "
+            "— downgraded to unknown (§cycle4.1 verification gate)")
+    return res
+
+
+def freshness_ok(captured_at: str, window: str, at: str | None = None
+                 ) -> bool:
+    age = age_seconds(captured_at, parse_ts(at))
+    return age is not None and age <= DEFAULT_WINDOW_TTLS.get(window, 3600)

platformforge/optimize/__init__.py

@@ -0,0 +1,4 @@
+"""Cycle 5 — Optimization Engine. Boundary (§211): produces
+OptimizationRecommendation / ChangeIntent — NEVER executes; the Cycle 4
+governed pipeline remains the only path to mutation.
+"""

platformforge/optimize/engine.py

@@ -0,0 +1,109 @@
+"""Cycle 5 Phase I — OptimizationEngine (§202–212).
+
+Priority is decomposed (§206–208): impact / confidence / effort /
+risk / scope / freshness — never a black-box score. `plan()` converts
+a recommendation to a ChangeIntent — the *only* path toward action
+(§210–211, §303): policy, approval, envelope, verification all still
+apply in the Cycle 4 pipeline.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from platformforge.analytics.models import AnalyticsDataQuality
+from platformforge.optimize.models import OptimizationOpportunity, OptimizationRecommendation
+
+IMPACT_ORDER = {"high": 3, "medium": 2, "low": 1}
+CONF_ORDER = {"high": 3, "medium": 2, "low": 1}
+EFFORT_ORDER = {"low": 3, "medium": 2, "high": 1, "unknown": 0}
+
+
+def priority_of(rec: OptimizationRecommendation) -> dict[str, Any]:
+    """§206–208 — decomposed priority; the tuple IS the output."""
+    impact = "high" if (rec.estimated_savings or 0) >= 1000 or \
+        rec.type in ("security", "reliability") else \
+        "medium" if (rec.estimated_savings or 0) >= 100 else "low"
+    freshness = "fresh"
+    dims = {"impact": impact, "confidence": rec.confidence,
+            "effort": rec.effort, "risk": rec.reliability_risk,
+            "scope": rec.scope, "evidence_freshness": freshness}
+    rank = (IMPACT_ORDER[impact] * 3 + CONF_ORDER[rec.confidence] * 3 +
+            EFFORT_ORDER.get(rec.effort, 0) * 2 +
+            (0 if rec.reliability_risk in ("low",) else -2) +
+            (1 if freshness == "fresh" else -1))
+    dims["rank_score"] = rank
+    dims["note"] = "decomposed dimensions — not a black-box score"
+    return dims
+
+
+class OptimizationEngine:
+    """Deterministic scan over analytics outputs → opportunities →
+    recommendations → ChangeIntents. Never touches transports (§211)."""
+
+    def __init__(self, dq: AnalyticsDataQuality | None = None):
+        self.dq = dq or AnalyticsDataQuality()
+        self._opportunities: list[OptimizationOpportunity] = []
+        self._recommendations: list[OptimizationRecommendation] = []
+
+    def add_opportunity(self, opp: OptimizationOpportunity) -> None:
+        self._opportunities.append(opp)
+
+    def opportunities(self) -> list[OptimizationOpportunity]:
+        return list(self._opportunities)
+
+    def recommendations(self) -> list[OptimizationRecommendation]:
+        """§294 — only promotable opportunities become recs."""
+        out = []
+        cap = self.dq.confidence_cap            # §251
+        for i, o in enumerate(self._opportunities):
+            if not o.promotable():
+                continue
+            conf = {"high": "high", "medium": "medium"}.get(
+                cap, "low")
+            if o.uncertainty == "medium" and conf == "high":
+                conf = "medium"
+            rec = OptimizationRecommendation(
+                recommendation_id=f"opt-{i}-{o.opportunity_id}",
+                type=o.type, scope=o.scope,
+                estimated_savings=o.estimated_savings,
+                savings_unit=o.savings_unit,
+                confidence=conf, evidence=list(o.evidence),
+                effort="unknown",
+                effort_rationale="not estimated — declare explicitly",
+                verification="post-change verification via ops pipeline")
+            rec.priority = priority_of(rec)
+            out.append(rec)
+        out.sort(key=lambda r: -r.priority.get("rank_score", 0))
+        self._recommendations = out
+        return out
+
+    def portfolio(self, top: int = 10) -> dict[str, Any]:
+        """§212 — top-N with dimensions visible."""
+        recs = self.recommendations()
+        return {"schema": "platformforge/optimization-portfolio/v1",
+                "total_opportunities": len(self._opportunities),
+                "promoted": len(recs),
+                "suppressed": len(self._opportunities) - len(recs),
+                "data_quality": self.dq.to_dict(),
+                "top": [{**r.to_dict()} for r in recs[:top]]}
+
+    @staticmethod
+    def plan(rec: OptimizationRecommendation):
+        """§261/§303 — recommendation → ChangeIntent. The ONLY bridge;
+        no path exists from here to ExecutionEnvelope directly."""
+        from platformforge.ops.models import ChangeIntent, Reason
+        return ChangeIntent(
+            intent_id=f"opt-{rec.recommendation_id}",
+            reason=Reason(type="recommendation",
+                          recommendation_ids=[rec.recommendation_id],
+                          detail=f"{rec.type} optimization "
+                                 f"{rec.recommendation_id}"),
+            target_resources=sorted(set(rec.scope.values())),
+            desired_change={"optimization": rec.type,
+                            "proposed": rec.proposed,
+                            "recommendation_id": rec.recommendation_id},
+            risk_context={"confidence": rec.confidence,
+                          "performance_risk": rec.performance_risk,
+                          "reliability_risk": rec.reliability_risk},
+            expected_outcome=rec.verification)

platformforge/optimize/models.py

@@ -0,0 +1,68 @@
+"""Cycle 5 Phase I — optimization contracts (§97–98, §204–210).
+
+Pipeline (§210): Evidence → Pattern → Opportunity → Recommendation →
+ChangeIntent → Cycle 4. The engine NEVER produces an
+ExecutionEnvelope and NEVER executes (§211, §303).
+"""
+
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass, field
+from typing import Any
+
+OPT_TYPES = ("cost", "capacity", "reliability", "security",
+             "platform-product", "operational", "standardization")
+EFFORTS = ("low", "medium", "high", "unknown")
+
+
+@dataclass
+class OptimizationOpportunity:
+    """§209 — a detected improvement signal, before recommendation."""
+    opportunity_id: str
+    type: str                          # OPT_TYPES
+    scope: dict[str, str]
+    evidence: list[str] = field(default_factory=list)
+    pattern_refs: list[str] = field(default_factory=list)
+    estimated_savings: float | None = None
+    savings_unit: str = ""
+    uncertainty: str = "high"          # low|medium|high
+    coverage: float | None = None
+    freshness: str = "unknown"
+
+    def promotable(self) -> bool:
+        """§294/§303 — high uncertainty or thin coverage never
+        auto-promotes to a recommendation."""
+        if self.uncertainty == "high":
+            return False
+        if self.coverage is not None and self.coverage < 0.5:
+            return False
+        return bool(self.evidence)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {"schema": "platformforge/optimization-opportunity/v1",
+                **asdict(self)}
+
+
+@dataclass
+class OptimizationRecommendation:
+    """§97–98 — the governed output object."""
+    recommendation_id: str
+    type: str
+    scope: dict[str, str]
+    current: dict[str, Any] = field(default_factory=dict)
+    proposed: dict[str, Any] = field(default_factory=dict)
+    estimated_savings: float | None = None
+    savings_unit: str = ""
+    performance_risk: str = "unknown"
+    reliability_risk: str = "unknown"
+    confidence: str = "low"
+    evidence: list[str] = field(default_factory=list)
+    verification: str = ""
+    effort: str = "unknown"            # §208
+    effort_rationale: str = ""
+    priority: dict[str, Any] = field(default_factory=dict)
+    auto_apply: bool = False           # §99 — never
+
+    def to_dict(self) -> dict[str, Any]:
+        return {"schema": "platformforge/optimization-recommendation/v1",
+                **asdict(self), "executes": False}