platformforge 0.1.0__py3-none-any.whl

platformforge/economy/cache.py

@@ -0,0 +1,287 @@
+"""Economy cache (§35–55) — multilayer, content-addressed, dependency-
+invalidated, receipted.
+
+Layers (§36): artifact → fact → graph → finding → context → analysis →
+decision. Each layer declares which dependencies it binds to; a change in
+a dependency invalidates only the layers that depend on it (§40–42):
+
+    rule catalog changes  → findings/analysis/decision miss,
+                            artifacts + facts still hit (§292 north star)
+    knowledge changes     → recommendations/context miss; facts survive
+    engine changes        → the corresponding layer misses
+
+States (§44): hit | miss | stale | invalid | partial-reuse.
+Never cached (§48): credentials, secret values, approval tokens — a
+payload flagged sensitive is refused at `put`, and `decision` entries
+require evidence_hash + policy_version + risk_profile (§53). Human
+approval is operation-specific and has no cache layer (§54). Live
+conclusions carry `observed_at` + `freshness_s`; stale observations never
+count as a fresh hit (§55).
+
+TTL exists but is a fallback — dependency invalidation is primary (§49).
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import time
+from dataclasses import asdict, dataclass, field
+from pathlib import Path
+from typing import Any
+
+SCHEMA = "platformforge/cache-entry/v1"
+
+LAYERS = ("artifact", "fact", "graph", "finding", "context",
+          "analysis", "decision")
+
+# §38/§43 — the dependency keys each layer binds to. A dep not listed
+# for a layer does not invalidate it (selective invalidation, §40).
+LAYER_DEPS: dict[str, tuple[str, ...]] = {
+    "artifact": ("runtime_version",),
+    "fact": ("artifact_hash", "extractor_version", "runtime_version"),
+    "graph": ("artifact_hash", "runtime_version", "config_hash"),
+    "finding": ("artifact_hash", "rule_catalog_hash", "knowledge_hash",
+                "policy_hash"),
+    "context": ("artifact_hash", "rule_catalog_hash", "knowledge_hash"),
+    "analysis": ("rule_catalog_hash", "knowledge_hash", "engine_version",
+                 "policy_hash"),
+    "decision": ("evidence_hash", "policy_version", "risk_profile"),
+}
+
+DECISION_STATES = ("hit", "miss", "stale", "invalid", "partial-reuse")
+
+# §48 — payload markers that are never persisted into cache.
+SENSITIVE_MARKERS = ("credential", "secret_value", "approval_token",
+                     "private_key", "session_token")
+
+
+def content_hash(payload: Any) -> str:
+    """§37 — content addressing: sha256 of the canonical serialization."""
+    body = json.dumps(payload, sort_keys=True, separators=(",", ":"),
+                      default=str).encode()
+    return hashlib.sha256(body).hexdigest()
+
+
+def cache_key(layer: str, scope: str, deps: dict[str, str]) -> str:
+    """§39 — key = sha256(layer + scope + the deps this layer binds)."""
+    bound = {k: deps.get(k, "") for k in LAYER_DEPS[layer]}
+    body = json.dumps({"layer": layer, "scope": scope, "deps": bound},
+                      sort_keys=True).encode()
+    return hashlib.sha256(body).hexdigest()
+
+
+@dataclass
+class CacheDependency:
+    """§43 — declared dependency set for an entry."""
+    deps: dict[str, str] = field(default_factory=dict)
+
+    def changed(self, current: dict[str, str], layer: str) -> list[str]:
+        """Return the bound deps whose value changed vs `current`."""
+        return [k for k in LAYER_DEPS[layer]
+                if k in self.deps and self.deps[k] != current.get(k, "")]
+
+
+@dataclass
+class CacheDecision:
+    """§44 — the auditable outcome of a lookup."""
+    state: str                       # DECISION_STATES
+    layer: str
+    key: str
+    reason: str = ""
+    changed_deps: list[str] = field(default_factory=list)
+    entry_age_s: float | None = None
+    reusable_layers: list[str] = field(default_factory=list)
+    invalidated_layers: list[str] = field(default_factory=list)
+
+    def to_dict(self) -> dict[str, Any]:
+        return asdict(self)
+
+
+@dataclass
+class CacheEntry:
+    schema: str = SCHEMA
+    layer: str = ""
+    key: str = ""
+    scope: str = ""
+    payload: Any = None
+    payload_hash: str = ""
+    deps: dict[str, str] = field(default_factory=dict)
+    created_at: float = field(default_factory=time.time)
+    ttl_s: float | None = None        # §49 — fallback only
+    observed_at: float | None = None  # §55 — live conclusions
+    freshness_s: float | None = None
+    live: bool = False
+
+    def expired(self, now: float | None = None) -> bool:
+        now = now if now is not None else time.time()
+        if self.ttl_s is not None and now - self.created_at > self.ttl_s:
+            return True
+        if self.live:
+            if self.observed_at is None or self.freshness_s is None:
+                return True          # live entry without freshness: stale
+            return now - self.observed_at > self.freshness_s
+        return False
+
+
+class CacheStore:
+    """`.platformforge/cache/<layer>/<key>.json` + per-layer counters."""
+
+    def __init__(self, root: str | Path):
+        self.dir = Path(root) / ".platformforge" / "cache"
+        for layer in LAYERS:
+            (self.dir / layer).mkdir(parents=True, exist_ok=True)
+        self._stats_path = self.dir / "stats.json"
+
+    # -- stats ----------------------------------------------------------
+    def _stats(self) -> dict[str, int]:
+        if self._stats_path.exists():
+            return json.loads(self._stats_path.read_text())
+        return {"hits": 0, "misses": 0, "stale": 0, "invalid": 0,
+                "invalidations": 0, "puts": 0, "bytes_reused": 0,
+                "refused": 0}
+
+    def _bump(self, **kw: int) -> None:
+        s = self._stats()
+        for k, v in kw.items():
+            s[k] = s.get(k, 0) + v
+        self._stats_path.write_text(json.dumps(s, indent=1, sort_keys=True))
+
+    # -- write ------------------------------------------------------------
+    def put(self, layer: str, scope: str, payload: Any,
+            deps: dict[str, str], *, ttl_s: float | None = None,
+            live: bool = False, observed_at: float | None = None,
+            freshness_s: float | None = None,
+            sensitive: bool = False) -> CacheDecision:
+        """§48/§53/§54/§55 guards enforced at write time."""
+        if layer not in LAYERS:
+            raise ValueError(f"unknown cache layer {layer!r}")
+        if sensitive:
+            self._bump(refused=1)
+            return CacheDecision(
+                "invalid", layer, "", reason=(
+                    "refused: sensitive payload — credentials, secret "
+                    "values and approval tokens are never cached (§48/§54)"))
+        if layer == "decision":
+            missing = [k for k in LAYER_DEPS["decision"]
+                       if k not in deps]
+            if missing:
+                self._bump(refused=1)
+                return CacheDecision(
+                    "invalid", layer, "", reason=(
+                        f"decision cache requires {missing} "
+                        "(§53: evidence_hash+policy_version+risk_profile)"))
+        key = cache_key(layer, scope, deps)
+        ent = CacheEntry(layer=layer, key=key, scope=scope,
+                         payload=payload, payload_hash=content_hash(payload),
+                         deps=dict(deps), ttl_s=ttl_s, live=live,
+                         observed_at=observed_at, freshness_s=freshness_s)
+        (self.dir / layer / f"{key}.json").write_text(
+            json.dumps(asdict(ent), sort_keys=True, default=str))
+        self._bump(puts=1)
+        return CacheDecision("hit", layer, key, "stored")
+
+    # -- read -------------------------------------------------------------
+    def _load(self, layer: str, key: str) -> CacheEntry | None:
+        p = self.dir / layer / f"{key}.json"
+        if not p.exists():
+            return None
+        d = json.loads(p.read_text())
+        return CacheEntry(**{k: v for k, v in d.items()
+                             if k in CacheEntry.__dataclass_fields__})
+
+    def get(self, layer: str, scope: str, current_deps: dict[str, str],
+            *, now: float | None = None
+            ) -> tuple[CacheDecision, Any | None]:
+        """Lookup: dependency-diff first, freshness second (§44, §55)."""
+        # the stored key encodes the OLD deps — recompute from stored deps
+        # is impossible, so lookup scans by scope+layer prefix: keys are
+        # content-derived, so we resolve the entry whose deps-match.
+        # For a 1:1 scope→key mapping we store under a scope index.
+        for cand in (self.dir / layer).glob("*.json"):
+            d = json.loads(cand.read_text())
+            if d.get("scope") != scope:
+                continue
+            ent = CacheEntry(**{k: v for k, v in d.items()
+                                if k in CacheEntry.__dataclass_fields__})
+            dep = CacheDependency(ent.deps)
+            changed = dep.changed(current_deps, layer)
+            if changed:
+                self._bump(invalid=1)
+                return (CacheDecision(
+                    "invalid", layer, ent.key,
+                    f"deps changed: {changed}", changed), None)
+            if ent.expired(now):
+                self._bump(stale=1)
+                return (CacheDecision(
+                    "stale", layer, ent.key,
+                    "freshness/ttl expired", entry_age_s=(
+                        (now or time.time()) - ent.created_at)), None)
+            self._bump(hits=1, bytes_reused=len(
+                json.dumps(ent.payload, default=str)))
+            return (CacheDecision("hit", layer, ent.key, "deps match"),
+                    ent.payload)
+        self._bump(misses=1)
+        return (CacheDecision("miss", layer,
+                              cache_key(layer, scope, current_deps),
+                              "no entry for scope"), None)
+
+    def invalidate_dep(self, dep_key: str, new_value: str
+                       ) -> dict[str, Any]:
+        """§40–42 — selective invalidation: drop every entry whose bound
+        deps include dep_key with a different value; keep the rest."""
+        removed: list[str] = []
+        kept: list[str] = []
+        for layer in LAYERS:
+            if dep_key not in LAYER_DEPS[layer]:
+                kept.append(layer)
+                continue
+            for cand in (self.dir / layer).glob("*.json"):
+                d = json.loads(cand.read_text())
+                if d.get("deps", {}).get(dep_key) not in (None, new_value):
+                    cand.unlink()
+                    removed.append(f"{layer}/{d.get('key', cand.stem)}")
+        self._bump(invalidations=len(removed))
+        return {"dep": dep_key, "removed": removed,
+                "unaffected_layers": [l for l in kept]}
+
+    # -- maintenance --------------------------------------------------------
+    def stats(self) -> dict[str, Any]:
+        """§51 — hits/misses/bytes reused/entries/invalidations."""
+        s = self._stats()
+        entries = sum(1 for layer in LAYERS
+                      for _ in (self.dir / layer).glob("*.json"))
+        return {**s, "entries": entries,
+                "layers": {l: sum(1 for _ in (self.dir / l).glob("*.json"))
+                           for l in LAYERS}}
+
+    def gc(self, *, now: float | None = None) -> dict[str, Any]:
+        """§50 — remove expired entries; dependency-invalid entries are
+        already removed by invalidate_dep."""
+        now = now if now is not None else time.time()
+        removed = []
+        for layer in LAYERS:
+            for cand in (self.dir / layer).glob("*.json"):
+                d = json.loads(cand.read_text())
+                ent = CacheEntry(**{k: v for k, v in d.items()
+                                    if k in CacheEntry.__dataclass_fields__})
+                if ent.expired(now):
+                    cand.unlink()
+                    removed.append(f"{layer}/{d.get('key', cand.stem)}")
+        return {"gc_removed": removed, "remaining": self.stats()["entries"]}
+
+    def reuse_plan(self, current_deps: dict[str, str], scopes: dict[str,
+                   str], *, now: float | None = None) -> CacheDecision:
+        """§44 partial-reuse — per-layer lookup over a pipeline; returns
+        which layers still hit vs miss (§292 north star shape)."""
+        hit, missed = [], []
+        for layer, scope in scopes.items():
+            if layer not in LAYERS:
+                continue
+            dec, _ = self.get(layer, scope, current_deps, now=now)
+            (hit if dec.state == "hit" else missed).append(layer)
+        state = ("hit" if not missed else
+                 "partial-reuse" if hit else "miss")
+        return CacheDecision(state, "pipeline", "",
+                             f"reusable={hit} recompute={missed}",
+                             reusable_layers=hit, invalidated_layers=missed)

platformforge/economy/checkpoint.py

@@ -0,0 +1,112 @@
+"""EconomyCheckpoint (§76–80) — run state persisted so a stopped run
+resumes with its spend intact.
+
+Invariants: resume never resets spend (§78/§290 north star); resume
+revalidates artifact hashes, knowledge freshness, observation freshness,
+policy version and operation state (§79); a profile cannot downgrade on
+resume — `deep` never becomes `cheap` because the run paused (§80).
+"""
+
+from __future__ import annotations
+
+import json
+import time
+from dataclasses import asdict, dataclass, field
+from pathlib import Path
+from typing import Any
+
+SCHEMA = "platformforge/checkpoint/v1"
+
+PROFILE_ORDER = {"offline": 0, "economy": 1, "balanced": 2,
+                 "deep": 3, "strict": 4}
+
+
+@dataclass
+class EconomyCheckpoint:
+    """§77 — canonical fields."""
+    schema: str = SCHEMA
+    run_id: str = ""
+    spent_budget: dict[str, float] = field(default_factory=dict)
+    remaining_budget: dict[str, float] = field(default_factory=dict)
+    context_refs: list[str] = field(default_factory=list)
+    cache_refs: list[str] = field(default_factory=list)
+    agent_state: dict[str, Any] = field(default_factory=dict)
+    routing_profile: str = "balanced"
+    risk_profile: str = "low"
+    tool_spend: dict[str, float] = field(default_factory=dict)
+    provider_spend: dict[str, float] = field(default_factory=dict)
+    deps: dict[str, str] = field(default_factory=dict)
+    # artifact_hash / knowledge_hash / policy_version — revalidated §79
+    created_at: float = field(default_factory=time.time)
+
+    def to_dict(self) -> dict[str, Any]:
+        return asdict(self)
+
+
+class CheckpointStore:
+    """`.platformforge/checkpoints/<run_id>.json`."""
+
+    def __init__(self, root: str | Path):
+        self.dir = Path(root) / ".platformforge" / "checkpoints"
+        self.dir.mkdir(parents=True, exist_ok=True)
+
+    def save(self, cp: EconomyCheckpoint) -> str:
+        path = self.dir / f"{cp.run_id}.json"
+        path.write_text(json.dumps(cp.to_dict(), sort_keys=True,
+                                   default=str, indent=1))
+        return str(path)
+
+    def load(self, run_id: str) -> EconomyCheckpoint | None:
+        p = self.dir / f"{run_id}.json"
+        if not p.exists():
+            return None
+        d = json.loads(p.read_text())
+        return EconomyCheckpoint(**{
+            k: v for k, v in d.items()
+            if k in EconomyCheckpoint.__dataclass_fields__})
+
+    def list(self) -> list[str]:
+        return sorted(p.stem for p in self.dir.glob("*.json"))
+
+    def resume(self, run_id: str, current_deps: dict[str, str],
+               requested_profile: str = ""
+               ) -> dict[str, Any]:
+        """§78–80 — resume with spend intact; revalidate the world;
+        refuse profile downgrade."""
+        cp = self.load(run_id)
+        if cp is None:
+            return {"refusal": "PF-CHECKPOINT-MISSING",
+                    "reason": f"no checkpoint for {run_id}",
+                    "unlock": "start a fresh run — spend unknown"}
+
+        # §79 — revalidation: every dep that drifted is reported, and a
+        # stale dep means the resume must re-verify, not silently reuse.
+        stale = {k: {"was": v, "now": current_deps.get(k, "")}
+                 for k, v in cp.deps.items()
+                 if current_deps.get(k, "") != v}
+        needs_revalidation = bool(stale)
+
+        # §80 — profile cannot downgrade on resume.
+        eff_profile = requested_profile or cp.routing_profile
+        if PROFILE_ORDER.get(eff_profile, -1) < \
+                PROFILE_ORDER.get(cp.routing_profile, 0):
+            return {"refusal": "PF-CHECKPOINT-DOWNGRADE",
+                    "reason": f"cannot resume {cp.routing_profile!r} run "
+                              f"as {eff_profile!r} — profile cannot "
+                              "downgrade on resume (§80)",
+                    "unlock": "resume at the stored profile or higher"}
+
+        return {"run_id": run_id, "resumed": True,
+                "profile": eff_profile,
+                "spent_budget": cp.spent_budget,      # §78 — carries over
+                "remaining_budget": cp.remaining_budget,
+                "context_refs": cp.context_refs,
+                "cache_refs": cp.cache_refs,
+                "tool_spend": cp.tool_spend,
+                "provider_spend": cp.provider_spend,
+                "risk_profile": cp.risk_profile,
+                "needs_revalidation": needs_revalidation,
+                "stale_deps": stale,
+                "note": ("spend preserved; re-verify sections whose deps "
+                         "changed" if needs_revalidation else
+                         "spend preserved; all deps fresh")}

platformforge/economy/collection.py

@@ -0,0 +1,142 @@
+"""Live collection economy v2 (§197–207) + fleet context funnel helpers
+(§192–196).
+
+A collection starts broad-and-cheap (inventory) and only goes deep on the
+parts the graph/risk point at (§199). Every live call passes the evidence
+gate first (§206–207): if local evidence can answer, the provider call
+is refused — runtime truth is the only justification for spending a call.
+Provider-call ROI is measured in facts gained and coverage gained per
+call (§200), not money — "quality per call" claims need data (§201).
+Cached observations are used while fresh (§202–203); budget exhaustion
+yields partial coverage, honestly reported (§204).
+"""
+
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass, field
+from typing import Any
+
+from platformforge.economy.cache import CacheStore
+
+SCHEMA = "platformforge/collection-economy-plan/v1"
+
+
+@dataclass
+class CollectionEconomyPlan:
+    """§198 — broad → targeted, with declared caps."""
+    schema: str = SCHEMA
+    inventory_first: bool = True               # §199
+    inventory_budget: dict[str, int] = field(
+        default_factory=lambda: {"api_calls": 20, "objects": 10000})
+    targeted_budget: dict[str, int] = field(
+        default_factory=lambda: {"api_calls": 80, "objects": 5000})
+    target_selectors: dict[str, Any] = field(default_factory=dict)
+    # e.g. {"graph_nodes": [...], "risk_min": "high"}
+    freshness_s: int = 3600                    # §203 cached observations
+    evidence_gate: bool = True                 # §206 — always on by default
+
+    def to_dict(self) -> dict[str, Any]:
+        return asdict(self)
+
+
+def evidence_gate(question_kind: str, local_evidence: dict[str, Any],
+                  *, runtime_truth_needed: bool = False
+                  ) -> dict[str, Any]:
+    """§206–207 — before any live call: can local evidence answer?
+
+    A live call is allowed only when runtime truth is needed AND local
+    evidence cannot answer. Otherwise the call is refused and the caller
+    is pointed at the local verbs."""
+    answers = local_evidence.get("answers", [])
+    coverage = local_evidence.get("coverage", 0.0)
+    if runtime_truth_needed and coverage < 0.5:
+        return {"decision": "live_call_justified",
+                "reason": "runtime truth needed and local coverage "
+                          f"{coverage:.0%} is insufficient"}
+    if answers or coverage >= 0.5:
+        return {"decision": "refuse_live_call",
+                "reason": "local evidence can answer "
+                          f"(coverage {coverage:.0%})",
+                "code": "PF-ECONOMY-LIVE-UNNECESSARY",
+                "use": "local artifacts/graph/facts instead"}
+    return {"decision": "live_call_justified",
+            "reason": "no local evidence covers the question"}
+
+
+def provider_call_roi(calls: int, facts_gained: int,
+                      coverage_gained: float) -> dict[str, Any]:
+    """§200–201 — measured ROI in facts/coverage per call. 'Quality per
+    call' conclusions require data; with no calls there is nothing."""
+    if calls == 0:
+        return {"state": "unresolved",
+                "reason": "no calls — ROI undefined, not zero"}
+    return {"calls": calls, "facts_gained": facts_gained,
+            "coverage_gained": round(coverage_gained, 4),
+            "facts_per_call": round(facts_gained / calls, 3),
+            "coverage_per_call": round(coverage_gained / calls, 5),
+            "basis": "measured"}
+
+
+def coverage_on_exhaustion(collected: int, total: int
+                           ) -> dict[str, Any]:
+    """§204 — budget exhausted → coverage is partial and says so."""
+    frac = collected / total if total else 0.0
+    return {"coverage": round(frac, 4), "state": "partial",
+            "code": "PF-ECONOMY-BUDGET-EXHAUSTED",
+            "reason": f"collection budget exhausted at {collected}/"
+                      f"{total} objects — results are partial"}
+
+
+# --- fleet (§192–196) ----------------------------------------------------
+
+def fleet_snapshot_hash(snapshot: dict[str, Any]) -> str:
+    import hashlib
+    import json
+    return hashlib.sha256(json.dumps(snapshot, sort_keys=True,
+                                   default=str).encode()).hexdigest()
+
+
+def _fleet_key(snapshot: dict[str, Any], query: str) -> str:
+    # the snapshot hash is part of the KEY (not just deps): a new
+    # snapshot is a different question — a miss, never a stale hit (§195)
+    return f"fleet:{query}:{fleet_snapshot_hash(snapshot)[:16]}"
+
+
+def fleet_query_cache_get(store: CacheStore, snapshot: dict[str, Any],
+                          query: str,
+                          deps: dict[str, Any] | None = None
+                          ) -> dict[str, Any]:
+    """§195 — cache fleet aggregates/graph queries/analytics by snapshot
+    hash; a new snapshot hash is a miss, never a stale hit."""
+    dec, payload = store.get("analysis", _fleet_key(snapshot, query),
+                             deps or {})
+    return {"decision": dec.to_dict(),
+            "payload": payload}
+
+
+def fleet_query_cache_put(store: CacheStore, snapshot: dict[str, Any],
+                          query: str, result: Any,
+                          deps: dict[str, Any] | None = None
+                          ) -> dict[str, Any]:
+    dec = store.put("analysis", _fleet_key(snapshot, query), result,
+                    deps or {})
+    return {"decision": dec.to_dict()}
+
+
+def fleet_delta(snap_a: dict[str, Any], snap_b: dict[str, Any]
+                ) -> dict[str, Any]:
+    """§196 — snapshot delta so repeated fleet questions don't recompute.
+
+    Members are keyed `kind:id`; the delta lists added/removed/changed
+    references (member snapshots are hash references, not payloads)."""
+    a = snap_a.get("members", snap_a.get("observations", {})) or {}
+    b = snap_b.get("members", snap_b.get("observations", {})) or {}
+    ka, kb = set(a), set(b)
+    changed = sorted(k for k in ka & kb if a[k] != b[k])
+    return {"snapshot_a": fleet_snapshot_hash(snap_a)[:12],
+            "snapshot_b": fleet_snapshot_hash(snap_b)[:12],
+            "added": sorted(kb - ka), "removed": sorted(ka - kb),
+            "changed": changed,
+            "unchanged": len(ka & kb) - len(changed),
+            "note": "delta-first fleet reads — unchanged members keep "
+                    "cached analysis"}