dandelion-memory 1.0.0rc15__py3-none-any.whl

benchmarks/memory_session_acceptance.py

@@ -0,0 +1,279 @@
+"""Bounded subprocess-only acceptance trace for the canonical memory owner.
+
+Every operation goes through the public ``memory-owner`` or ``memory`` CLI.
+The runner does not bootstrap a project, load a model, talk to MCP, or mutate
+SQLite directly.
+"""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import os
+from pathlib import Path
+import subprocess
+import sys
+import tempfile
+import time
+import uuid
+from typing import Any
+
+
+def _sha256(data: bytes | str) -> str:
+    return hashlib.sha256(data.encode() if isinstance(data, str) else data).hexdigest()
+
+
+def _env(root: Path, workspace: Path) -> dict[str, str]:
+    env = dict(os.environ)
+    env["DANDELION_SECRET_POLICY"] = "deny"
+    old = env.get("PYTHONPATH", "")
+    env["PYTHONPATH"] = str(workspace) + (os.pathsep + old if old else "")
+    return env
+
+
+def _event(trace: dict[str, Any], event: dict[str, Any]) -> None:
+    event = dict(event)
+    event["seq"] = len(trace["events"]) + 1
+    trace["events"].append(event)
+
+
+def _assert(trace: dict[str, Any], ident: str, passed: bool, observed: Any,
+            expected: Any, session: str) -> None:
+    trace["assertions"].append({"id": ident, "session": session,
+                                "passed": bool(passed), "observed": observed,
+                                "expected": expected})
+
+
+def _run(trace: dict[str, Any], root: Path, workspace: Path,
+         args: list[str], session: str) -> tuple[subprocess.CompletedProcess[str], Any]:
+    started = time.perf_counter()
+    proc = subprocess.run([sys.executable, "-m", "dandelion.cli", *args],
+                          cwd=str(workspace), env=_env(root, workspace),
+                          capture_output=True, text=True, check=False)
+    parsed: Any = None
+    if proc.stdout.strip():
+        try:
+            parsed = json.loads(proc.stdout)
+        except json.JSONDecodeError:
+            parsed = None
+    _event(trace, {"kind": "cli", "session": session, "fresh_process": True,
+                   "argv": args, "exit_code": proc.returncode,
+                   "duration_ms": round((time.perf_counter() - started) * 1000, 3),
+                   "stdout_sha256": _sha256(proc.stdout),
+                   "stderr_sha256": _sha256(proc.stderr),
+                   "stdout": proc.stdout, "stderr": proc.stderr,
+                   "response": parsed})
+    return proc, parsed
+
+
+def _memory(trace: dict[str, Any], root: Path, workspace: Path, operation: str,
+            data: dict[str, Any], session: str) -> dict[str, Any]:
+    proc, parsed = _run(trace, root, workspace,
+                        ["memory", operation, "--path", str(root),
+                         "--data", json.dumps(data, ensure_ascii=False, sort_keys=True)],
+                        session)
+    if proc.returncode != 0 or not isinstance(parsed, dict):
+        raise AssertionError(f"memory {operation} failed: {proc.returncode}: {proc.stderr}")
+    return parsed
+
+
+def _start_owner(trace: dict[str, Any], root: Path, workspace: Path,
+                 session: str) -> tuple[subprocess.Popen[str], dict[str, Any]]:
+    proc = subprocess.Popen([sys.executable, "-m", "dandelion.cli", "memory-owner",
+                             "start", "--path", str(root)],
+                            cwd=str(workspace), env=_env(root, workspace),
+                            stdout=subprocess.DEVNULL, stderr=subprocess.PIPE,
+                            text=True)
+    deadline = time.monotonic() + 10
+    status: dict[str, Any] | None = None
+    while time.monotonic() < deadline:
+        if proc.poll() is not None:
+            detail = proc.stderr.read() if proc.stderr else ""
+            raise AssertionError(f"owner exited during start: {proc.returncode}: {detail}")
+        status_proc, candidate = _run(trace, root, workspace,
+                                      ["memory-owner", "status", "--path", str(root)], session)
+        if status_proc.returncode == 0 and isinstance(candidate, dict):
+            status = candidate
+            break
+        time.sleep(0.05)
+    if status is None:
+        raise AssertionError("owner did not become available within 10 seconds")
+    trace["sessions"][session].update({"owner_pid": proc.pid,
+                                        "owner_id": status.get("owner_id"),
+                                        "status": status})
+    return proc, status
+
+
+def _stop_owner(trace: dict[str, Any], root: Path, workspace: Path,
+                proc: subprocess.Popen[str], session: str) -> None:
+    stop_proc, stop_result = _run(trace, root, workspace,
+                                  ["memory-owner", "stop", "--path", str(root)], session)
+    if stop_proc.returncode != 0:
+        raise AssertionError(f"owner stop failed: {stop_proc.stderr}")
+    try:
+        proc.wait(timeout=10)
+    except subprocess.TimeoutExpired:
+        proc.kill()
+        proc.wait(timeout=5)
+        raise AssertionError("owner did not stop after public memory-owner stop")
+    trace["sessions"][session]["stop"] = stop_result
+
+
+def _fixture(root: Path) -> str:
+    directory = root / ".dandelion"
+    directory.mkdir(parents=True, exist_ok=True)
+    legacy = {"legacy.endpoint": {"value": "https://legacy.example", "source": "legacy_tool", "confidence": 0.7}}
+    path = directory / "facts.json"
+    path.write_text(json.dumps(legacy, sort_keys=True, indent=2) + "\n", encoding="utf-8")
+    return _sha256(path.read_bytes())
+
+
+def run_acceptance(root: Path, *, workspace: Path | None = None,
+                   output: Path | None = None) -> dict[str, Any]:
+    """Run the owner Session A/B scenario and persist its machine trace."""
+    workspace = (workspace or Path(__file__).resolve().parents[1]).resolve()
+    root = root.resolve()
+    root.mkdir(parents=True, exist_ok=True)
+    legacy_hash_before = _fixture(root)
+    trace: dict[str, Any] = {
+        "schema_version": 2,
+        "trace_kind": "canonical_memory_owner_cli_acceptance",
+        "trace_id": uuid.uuid4().hex,
+        "workspace": str(root),
+        "sessions": {"A": {"fresh_process": True}, "B": {"fresh_process": True}},
+        "events": [], "assertions": [],
+        "scenario": {"fact": "service.port", "values": ["7311", "7422"],
+                      "decision_reason": "audit writes require ACID",
+                      "secret": "auth.api_token", "legacy": "legacy.endpoint"},
+    }
+    owner_a: subprocess.Popen[str] | None = None
+    owner_b: subprocess.Popen[str] | None = None
+    try:
+        owner_a, status_a = _start_owner(trace, root, workspace, "A")
+        before = _memory(trace, root, workspace, "status", {}, "A")
+        _assert(trace, "A_owner_started", bool(status_a.get("owner_id")),
+                status_a, "OK status and owner id", "A")
+        _memory(trace, root, workspace, "set-fact", {"key": "service.port", "value": "7311",
+                "source": "repo_observed", "authority": "REPO", "reason": "config file says initial port",
+                "confidence": 0.9}, "A")
+        decision = _memory(trace, root, workspace, "remember", {
+            "domain": "decisions", "source": "agent_inferred", "authority": "AGENT",
+            "reason": "audit writes require ACID",
+            "content": "Decision: PostgreSQL audit writes require ACID; live port {{ref:service.port}}; pinned port {{snapshot:service.port}}.",
+        }, "A")
+        _memory(trace, root, workspace, "set-fact", {"key": "service.port", "value": "7422",
+                "source": "tool_observed", "authority": "TOOL", "reason": "rollout changed port",
+                "confidence": 0.99}, "A")
+        current = _memory(trace, root, workspace, "get-fact", {"key": "service.port"}, "A")
+        history = _memory(trace, root, workspace, "get-fact", {"key": "service.port", "revision": 1}, "A")
+        recall = _memory(trace, root, workspace, "recall", {"query": "PostgreSQL audit port", "domain": "decisions", "limit": 3}, "A")
+        text = " ".join(str(item) for item in recall.get("results", []))
+        _assert(trace, "A_current_history", current.get("value") == "7422" and history.get("value") == "7311",
+                {"current": current, "history": history}, "current 7422/history 7311", "A")
+        _assert(trace, "A_provenance_reason", current.get("source") == "tool_observed" and current.get("reason") == "rollout changed port" and
+                recall.get("results", [{}])[0].get("source") == "agent_inferred" and recall.get("results", [{}])[0].get("reason") == "audit writes require ACID",
+                {"fact": current, "memory": recall}, "source/authority/reason preserved", "A")
+        _assert(trace, "A_live_and_pinned", "7422" in text and "7311" in text, text, "live 7422 and pinned 7311", "A")
+        _memory(trace, root, workspace, "set-fact", {"key": "auth.api_token", "value": "never-return",
+                "source": "tool_observed", "authority": "TOOL", "reason": "credential fixture", "security_class": "SECRET"}, "A")
+        _memory(trace, root, workspace, "remember", {"domain": "security", "source": "tool_observed",
+                "authority": "TOOL", "reason": "credential fixture", "security_class": "SECRET",
+                "content": "Restricted token {{ref:auth.api_token}}."}, "A")
+        denied_fact = _memory(trace, root, workspace, "get-fact", {"key": "auth.api_token"}, "A")
+        denied_memory = _memory(trace, root, workspace, "recall", {"query": "Restricted token", "domain": "security"}, "A")
+        _assert(
+            trace, "A_restricted_denied",
+            denied_fact.get("status") == "DENIED"
+            and denied_fact.get("value") == "[DENIED]"
+            and denied_memory.get("status") == "UNKNOWN"
+            and denied_memory.get("results") == [],
+            {"fact": denied_fact, "memory": denied_memory},
+            "exact fact is DENIED; restricted discovery is non-enumerating", "A",
+        )
+        unknown = _memory(trace, root, workspace, "query-claim", {"subject": "payment.service", "predicate": "database", "domain": "ops"}, "A")
+        _memory(trace, root, workspace, "put-claim", {"subject": "payment.service", "predicate": "database", "domain": "ops", "polarity": "ASSERT", "value": "postgres", "source": "repo_observed", "authority": "REPO", "reason": "deployment manifest"}, "A")
+        _memory(trace, root, workspace, "put-claim", {"subject": "payment.service", "predicate": "database", "domain": "ops", "polarity": "ASSERT", "value": "mysql", "source": "tool_observed", "authority": "TOOL", "reason": "runtime probe"}, "A")
+        conflict = _memory(trace, root, workspace, "query-claim", {"subject": "payment.service", "predicate": "database", "domain": "ops"}, "A")
+        _assert(trace, "A_unknown_and_conflict", unknown.get("status") == "UNKNOWN" and conflict.get("status") == "CONFLICT",
+                {"unknown": unknown, "conflict": conflict}, "UNKNOWN then CONFLICT", "A")
+        before_preview = _memory(trace, root, workspace, "status", {}, "A")
+        preview1 = _memory(trace, root, workspace, "import-legacy", {"preview": True}, "A")
+        status_preview = _memory(trace, root, workspace, "status", {}, "A")
+        preview2 = _memory(trace, root, workspace, "import-legacy", {"preview": True}, "A")
+        legacy_hash_after = _sha256((root / ".dandelion" / "facts.json").read_bytes())
+        unknown_legacy = _memory(trace, root, workspace, "get-fact", {"key": "legacy.endpoint"}, "A")
+        _assert(trace, "A_preview_is_non_mutating", preview1 == preview2 and preview1.get("status") == "PREVIEW" and
+                status_preview.get("counts") == before_preview.get("counts") and legacy_hash_before == legacy_hash_after and unknown_legacy.get("status") == "UNKNOWN",
+                {"first": preview1, "second": preview2, "before": before_preview, "after": status_preview, "hash_before": legacy_hash_before, "hash_after": legacy_hash_after, "legacy_fact": unknown_legacy},
+                "repeatable PREVIEW, unchanged counts/file, no imported fact", "A")
+        rebuild1 = _memory(trace, root, workspace, "rebuild-index", {}, "A")
+        after_rebuild1 = _memory(trace, root, workspace, "status", {}, "A")
+        rebuild2 = _memory(trace, root, workspace, "rebuild-index", {}, "A")
+        after_rebuild2 = _memory(trace, root, workspace, "status", {}, "A")
+        recall_after = _memory(trace, root, workspace, "recall", {"query": "PostgreSQL audit port", "domain": "decisions", "limit": 3}, "A")
+        _assert(trace, "A_rebuild_preserves_canonical", rebuild1.get("status") == "OK" and rebuild2.get("status") == "OK" and
+                after_rebuild1.get("counts") == after_rebuild2.get("counts") and after_rebuild2.get("counts", {}).get("memory_index") == after_rebuild2.get("counts", {}).get("memories") and
+                recall_after.get("results", [{}])[0].get("memory_id") == decision.get("memory_id"),
+                {"rebuild1": rebuild1, "rebuild2": rebuild2, "status": after_rebuild2, "recall": recall_after},
+                "canonical counts preserved and index rebuilt", "A")
+        _stop_owner(trace, root, workspace, owner_a, "A")
+        owner_a = None
+        trace["sessions"]["A"]["closed_before_B"] = True
+
+        owner_b, status_b = _start_owner(trace, root, workspace, "B")
+        trace["sessions"]["B"]["started_after_A"] = True
+        current_b = _memory(trace, root, workspace, "get-fact", {"key": "service.port"}, "B")
+        history_b = _memory(trace, root, workspace, "get-fact", {"key": "service.port", "revision": 1}, "B")
+        recall_b = _memory(trace, root, workspace, "recall", {"query": "PostgreSQL audit port", "domain": "decisions", "limit": 3}, "B")
+        conflict_b = _memory(trace, root, workspace, "query-claim", {"subject": "payment.service", "predicate": "database", "domain": "ops"}, "B")
+        denied_b = _memory(trace, root, workspace, "get-fact", {"key": "auth.api_token"}, "B")
+        rebuild_b = _memory(trace, root, workspace, "rebuild-index", {}, "B")
+        _assert(trace, "B_restart_persists_state", bool(status_b.get("owner_id")) and current_b.get("value") == "7422" and history_b.get("value") == "7311" and
+                "7422" in json.dumps(recall_b) and "7311" in json.dumps(recall_b) and conflict_b.get("status") == "CONFLICT" and denied_b.get("status") == "DENIED" and rebuild_b.get("status") == "OK",
+                {"status": status_b, "current": current_b, "history": history_b, "recall": recall_b, "conflict": conflict_b, "denied": denied_b, "rebuild": rebuild_b},
+                "fresh Session B sees current/history/live+pinned/conflict/denied/rebuilt", "B")
+        _stop_owner(trace, root, workspace, owner_b, "B")
+        owner_b = None
+        trace["sessions"]["B"]["closed"] = True
+    finally:
+        for proc, session in ((owner_b, "B-cleanup"), (owner_a, "A-cleanup")):
+            if proc is not None and proc.poll() is None:
+                try:
+                    _run(trace, root, workspace, ["memory-owner", "stop", "--path", str(root)], session)
+                except Exception:
+                    proc.kill()
+                try:
+                    proc.wait(timeout=5)
+                except subprocess.TimeoutExpired:
+                    proc.kill()
+    trace["status"] = "passed" if trace["assertions"] and all(item["passed"] for item in trace["assertions"]) else "failed"
+    trace_path = output or (root / ".dandelion" / "memory_acceptance_trace.json")
+    trace_path.parent.mkdir(parents=True, exist_ok=True)
+    trace["trace_path"] = str(trace_path)
+    trace_path.write_text(json.dumps(trace, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
+    return trace
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--root", type=Path)
+    parser.add_argument("--output", type=Path)
+    args = parser.parse_args(argv)
+    temporary: tempfile.TemporaryDirectory[str] | None = None
+    if args.root is None:
+        temporary = tempfile.TemporaryDirectory(prefix="dandelion-memory-acceptance-")
+        root = Path(temporary.name)
+    else:
+        root = args.root
+    try:
+        trace = run_acceptance(root, output=args.output)
+        print(json.dumps(trace, indent=2, ensure_ascii=False))
+        return 0 if trace["status"] == "passed" else 1
+    finally:
+        if temporary is not None:
+            temporary.cleanup()
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())