mirrorneuron-membrane-python-sdk 1.0.1__py3-none-any.whl

mirrorneuron_membrane_python_sdk-1.0.1.dist-info/METADATA

@@ -0,0 +1,11 @@
+Metadata-Version: 2.3
+Name: mirrorneuron-membrane-python-sdk
+Version: 1.0.1
+Summary: Python SDK shell for consuming the Rust Membrane context engine.
+Requires-Python: >=3.10
+License: Proprietary
+Requires-Dist: pydantic>=2.0,<3.0
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0; extra == "dev"
+Provides-Extra: compression
+Requires-Dist: llmlingua>=0.2.2; extra == "compression"

mirrorneuron_membrane_python_sdk-1.0.1.dist-info/RECORD

@@ -0,0 +1,8 @@
+mn_context_engine_sdk/__init__.py,sha256=b_cv-mjvkN12vXM6F4FqFy3w-jtJ5oxmpppL1o1WA0o,301
+mn_context_engine_sdk/benchmarks/__init__.py,sha256=vnIljW7ZUJMday5EqJNyyn2LRnIWj4jbcufLEaCj7sQ,58
+mn_context_engine_sdk/benchmarks/context_compression_accuracy_benchmark.py,sha256=3Gj6VGJHf3_zQIHWhLfJtR8ynJuvsnoOJcGrAqc5Avo,34882
+mn_context_engine_sdk/working_memory.py,sha256=uPSIJ3OUrqe8j1YLIjqy3RE5IFgZl8_9r8pJw6y_Cyg,8934
+mirrorneuron_membrane_python_sdk-1.0.1.dist-info/METADATA,sha256=Dgo_iYqfiGagYZgd7AXe7IFs52p7i2EgP77xlUl4WMw,376
+mirrorneuron_membrane_python_sdk-1.0.1.dist-info/WHEEL,sha256=s7mhKzNlJuzZwFeKTmqJmaumsjk0UzyKYJqivS04jas,87
+mirrorneuron_membrane_python_sdk-1.0.1.dist-info/entry_points.txt,sha256=yTQ3U_D2lUs3VnQOtw87yd8YmwFjQYk0SVGPeLGOsgg,130
+mirrorneuron_membrane_python_sdk-1.0.1.dist-info/RECORD,,

mirrorneuron_membrane_python_sdk-1.0.1.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: mn-build-backend
+Root-Is-Purelib: true
+Tag: py3-none-any

mirrorneuron_membrane_python_sdk-1.0.1.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+mn-context-compression-benchmark = mn_context_engine_sdk.benchmarks.context_compression_accuracy_benchmark:main

mn_context_engine_sdk/__init__.py

@@ -0,0 +1,19 @@
+from .working_memory import (
+    MemoryEdge,
+    MemoryItem,
+    MNMemoryEdge,
+    MNMemoryItem,
+    MNWorkingMemory,
+    WorkingMemory,
+    utcnow,
+)
+
+__all__ = [
+    "MNMemoryEdge",
+    "MNMemoryItem",
+    "MNWorkingMemory",
+    "MemoryEdge",
+    "MemoryItem",
+    "WorkingMemory",
+    "utcnow",
+]

mn_context_engine_sdk/benchmarks/__init__.py

@@ -0,0 +1 @@
+"""Benchmark entry points for the Membrane Python SDK."""

mn_context_engine_sdk/benchmarks/context_compression_accuracy_benchmark.py

@@ -0,0 +1,911 @@
+#!/usr/bin/env python3
+"""Compare raw vs compressed context on token use and answer accuracy.
+
+The benchmark uses an Ollama-compatible /api/chat endpoint for both the
+answering model and the judging model. It intentionally does not require the
+Rust gRPC service; this keeps it useful for CI smoke checks, remote model
+testing, and quick local experiments.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import statistics
+import time
+import urllib.error
+import urllib.request
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+
+DEFAULT_ENDPOINT = "http://192.168.4.173:11434/api/chat"
+DEFAULT_MODEL = "nemotron3:33b"
+CHARS_PER_TOKEN = 4
+
+
+@dataclass(frozen=True)
+class BenchmarkCase:
+    case_id: str
+    title: str
+    task: str
+    raw_context: str
+    compressed_context: str
+    expected_answer: str
+    required_terms: tuple[str, ...]
+    must_not_claim: tuple[str, ...] = ()
+
+
+def estimate_tokens(text: str) -> int:
+    return max(1, (len(text) + CHARS_PER_TOKEN - 1) // CHARS_PER_TOKEN)
+
+
+def filler(label: str, count: int) -> str:
+    blocks = []
+    for idx in range(count):
+        blocks.append(
+            f"[distractor-{label}-{idx:03d}] Prior agent note: this item is stale, "
+            f"superseded, or unrelated to the active goal. It mentions generic "
+            f"planning details, old hypotheses, verbose log text, and abandoned "
+            f"tool outputs that should not affect the final answer."
+        )
+    return "\n".join(blocks)
+
+
+def build_cases() -> list[BenchmarkCase]:
+    audit_raw = f"""
+RAW EVENT LOG FOR JOB finance-audit-77
+{filler("audit-pre", 70)}
+[task T-AUDIT-77] Determine whether the assistant disclosed the $49 activation fee before the user accepted the subscription.
+[constraint C-FEE-001] The fee disclosure must happen before agreement. If disclosure happens after agreement, classify as FAIL.
+[evidence EV-FEE-100] 2026-05-01T09:12:04Z assistant: "The plan includes a $49 activation fee before we continue."
+[evidence EV-AGREE-200] 2026-05-01T09:13:11Z user: "I accept the subscription."
+[evidence EV-FEE-OLD] 2026-04-28T11:00:00Z assistant disclosed a different legacy fee in an unrelated session.
+[decision D-DRAFT] Earlier draft said unknown, but it was before EV-FEE-100 was extracted.
+{filler("audit-post", 70)}
+DO_NOT_LOSE: task=T-AUDIT-77, constraint=C-FEE-001, evidence=EV-FEE-100, evidence=EV-AGREE-200.
+"""
+    audit_compressed = json.dumps(
+        {
+            "context_packet": {
+                "objective": {
+                    "focus_id": "T-AUDIT-77",
+                    "current_user_goal": "Decide if fee disclosure happened before agreement.",
+                },
+                "hard_constraints": [
+                    {
+                        "id": "C-FEE-001",
+                        "text": "Fee disclosure must happen before agreement. After agreement means FAIL.",
+                    }
+                ],
+                "retrieved_evidence": [
+                    {
+                        "id": "EV-FEE-100",
+                        "timestamp": "2026-05-01T09:12:04Z",
+                        "quote": "The plan includes a $49 activation fee before we continue.",
+                    },
+                    {
+                        "id": "EV-AGREE-200",
+                        "timestamp": "2026-05-01T09:13:11Z",
+                        "quote": "I accept the subscription.",
+                    },
+                ],
+                "do_not_lose": ["T-AUDIT-77", "C-FEE-001", "EV-FEE-100", "EV-AGREE-200"],
+            }
+        },
+        separators=(",", ":"),
+    )
+
+    incident_raw = f"""
+RAW MULTI-AGENT INCIDENT CONTEXT FOR JOB incident-414
+{filler("incident-pre", 90)}
+[task T-INC-414] Identify the root cause and cite the decisive logs.
+[constraint C-INC-010] Do not blame the database unless DB-OK-9 is contradicted by newer evidence.
+[log DB-OK-9] 2026-05-01T02:14:05Z database p95 latency remained below 11ms and no lock waits were observed.
+[log API-ERR-22] 2026-05-01T02:16:39Z api-gateway began returning 502 for checkout requests.
+[log CACHE-MISS-7] 2026-05-01T02:16:40Z feature flag ff_checkout_cache_v2 enabled in us-east.
+[log TRACE-ROOT-5] 2026-05-01T02:16:41Z checkout service error: missing cache namespace checkout:v2:prices.
+[fix FIX-88] Roll back ff_checkout_cache_v2 or create namespace checkout:v2:prices before enabling.
+[stale hypothesis H-OLD-1] CPU saturation caused outage. Rejected after metrics review.
+{filler("incident-post", 80)}
+DO_NOT_LOSE: T-INC-414, C-INC-010, DB-OK-9, API-ERR-22, CACHE-MISS-7, TRACE-ROOT-5, FIX-88.
+"""
+    incident_compressed = json.dumps(
+        {
+            "context_packet": {
+                "objective": {
+                    "focus_id": "T-INC-414",
+                    "current_user_goal": "Identify root cause and next fix.",
+                },
+                "hard_constraints": [
+                    {
+                        "id": "C-INC-010",
+                        "text": "Do not blame the database unless DB-OK-9 is contradicted.",
+                    }
+                ],
+                "retrieved_evidence": [
+                    {"id": "DB-OK-9", "summary": "Database p95 < 11ms; no lock waits."},
+                    {"id": "API-ERR-22", "summary": "API gateway 502s began at 02:16:39Z."},
+                    {
+                        "id": "CACHE-MISS-7",
+                        "summary": "ff_checkout_cache_v2 enabled in us-east at 02:16:40Z.",
+                    },
+                    {
+                        "id": "TRACE-ROOT-5",
+                        "summary": "Checkout failed because cache namespace checkout:v2:prices was missing.",
+                    },
+                ],
+                "shared_state": [
+                    {
+                        "id": "FIX-88",
+                        "summary": "Rollback ff_checkout_cache_v2 or create checkout:v2:prices before enabling.",
+                    }
+                ],
+                "do_not_lose": [
+                    "T-INC-414",
+                    "C-INC-010",
+                    "DB-OK-9",
+                    "API-ERR-22",
+                    "CACHE-MISS-7",
+                    "TRACE-ROOT-5",
+                    "FIX-88",
+                    "checkout:v2:prices",
+                ],
+            }
+        },
+        separators=(",", ":"),
+    )
+
+    handoff_raw = f"""
+RAW AGENT HANDOFF CONTEXT FOR JOB code-review-209
+{filler("handoff-pre", 75)}
+[task T-CODE-209] Prepare the next coding agent to finish the Membrane context compiler safely.
+[user-constraint UC-1] Keep existing gRPC APIs backward compatible.
+[user-constraint UC-2] LLMLingua/model compression must be optional; the system must work without GPU or Python model dependencies.
+[finding F-1] CompileContext should preserve exact IDs, source_refs, do_not_lose, and hard constraints.
+[finding F-2] Deterministic compression should run before model compression.
+[finding F-3] If model compression drops pinned terms, reject it and return deterministic packet.
+[handoff H-209] Next agent should add docs, tests, and an operator flag named MN_CONTEXT_MODEL_COMPRESSION_ENABLED.
+{filler("handoff-post", 75)}
+DO_NOT_LOSE: T-CODE-209, UC-1, UC-2, F-1, F-2, F-3, H-209, MN_CONTEXT_MODEL_COMPRESSION_ENABLED.
+"""
+    handoff_compressed = json.dumps(
+        {
+            "context_packet": {
+                "objective": {
+                    "focus_id": "T-CODE-209",
+                    "current_user_goal": "Prepare next coding agent handoff.",
+                },
+                "hard_constraints": [
+                    {"id": "UC-1", "text": "Keep existing gRPC APIs backward compatible."},
+                    {
+                        "id": "UC-2",
+                        "text": "Model compression must be optional; system works without GPU/Python model deps.",
+                    },
+                ],
+                "shared_state": [
+                    {
+                        "id": "F-1",
+                        "summary": "CompileContext preserves exact IDs, source_refs, do_not_lose, constraints.",
+                    },
+                    {"id": "F-2", "summary": "Run deterministic compression before model compression."},
+                    {
+                        "id": "F-3",
+                        "summary": "Reject model output if pinned terms are lost.",
+                    },
+                ],
+                "handoff": {
+                    "id": "H-209",
+                    "next_expected_action": "Add docs, tests, and MN_CONTEXT_MODEL_COMPRESSION_ENABLED operator flag.",
+                },
+                "do_not_lose": [
+                    "T-CODE-209",
+                    "UC-1",
+                    "UC-2",
+                    "F-1",
+                    "F-2",
+                    "F-3",
+                    "H-209",
+                    "MN_CONTEXT_MODEL_COMPRESSION_ENABLED",
+                ],
+            }
+        },
+        separators=(",", ":"),
+    )
+
+    return [
+        BenchmarkCase(
+            case_id="finance_fee_ordering",
+            title="Fee disclosure ordering",
+            task=(
+                "Decide PASS or FAIL. Cite the decisive evidence IDs and give one sentence of reasoning."
+            ),
+            raw_context=audit_raw,
+            compressed_context=audit_compressed,
+            expected_answer=(
+                "PASS. EV-FEE-100 at 09:12:04Z disclosed the $49 activation fee before "
+                "EV-AGREE-200 at 09:13:11Z accepted the subscription, satisfying C-FEE-001."
+            ),
+            required_terms=("PASS", "EV-FEE-100", "EV-AGREE-200", "C-FEE-001"),
+            must_not_claim=("FAIL", "unknown", "after agreement"),
+        ),
+        BenchmarkCase(
+            case_id="incident_root_cause",
+            title="Incident root cause",
+            task=(
+                "Identify the root cause, say what not to blame, and cite the fix ID."
+            ),
+            raw_context=incident_raw,
+            compressed_context=incident_compressed,
+            expected_answer=(
+                "Root cause was enabling ff_checkout_cache_v2 without namespace "
+                "checkout:v2:prices, supported by CACHE-MISS-7 and TRACE-ROOT-5. "
+                "Do not blame the database because DB-OK-9 shows it was healthy. Use FIX-88."
+            ),
+            required_terms=(
+                "ff_checkout_cache_v2",
+                "checkout:v2:prices",
+                "TRACE-ROOT-5",
+                "DB-OK-9",
+                "FIX-88",
+            ),
+            must_not_claim=("database root cause", "CPU saturation", "H-OLD-1 caused"),
+        ),
+        BenchmarkCase(
+            case_id="agent_handoff_constraints",
+            title="Agent handoff constraints",
+            task=(
+                "Write the next-agent handoff in 3 bullets. Include the required operator flag."
+            ),
+            raw_context=handoff_raw,
+            compressed_context=handoff_compressed,
+            expected_answer=(
+                "The handoff must preserve backward-compatible gRPC APIs, keep LLMLingua optional "
+                "so no GPU/Python model dependency is required, preserve pinned terms, and add "
+                "MN_CONTEXT_MODEL_COMPRESSION_ENABLED with tests and docs."
+            ),
+            required_terms=(
+                "backward compatible",
+                "optional",
+                "MN_CONTEXT_MODEL_COMPRESSION_ENABLED",
+                "pinned",
+            ),
+            must_not_claim=("required GPU", "always use LLMLingua", "breaking API"),
+        ),
+    ]
+
+
+def answer_prompt(context: str, task: str) -> str:
+    return f"""You are a precise multi-agent runtime worker.
+
+Use only the context below. Preserve exact IDs and constraints.
+
+CONTEXT:
+{context}
+
+TASK:
+{task}
+
+Answer concisely. Cite exact IDs when available."""
+
+
+def judge_prompt(case: BenchmarkCase, answer: str) -> str:
+    return f"""You are an exacting benchmark judge for a context-compression experiment.
+
+Evaluate whether the candidate answer correctly solves the task using the expected answer as ground truth.
+Do not reward verbosity. Penalize missing exact IDs, inverted decisions, unsupported claims, or contradictions.
+
+Task: {case.task}
+
+Expected answer:
+{case.expected_answer}
+
+Required exact terms that should appear when relevant:
+{json.dumps(case.required_terms)}
+
+Claims that must NOT appear:
+{json.dumps(case.must_not_claim)}
+
+Candidate answer:
+{answer}
+
+Scoring rubric:
+- factual_score: 0.0-1.0. Correct final conclusion and reasoning.
+- required_terms_score: 0.0-1.0. Preserves required IDs/terms.
+- constraint_score: 0.0-1.0. Obeys hard constraints and does not invert them.
+- hallucination_score: 0.0-1.0. 1.0 means no unsupported or forbidden claims.
+- score: weighted final score = 0.45*factual + 0.25*required_terms + 0.20*constraint + 0.10*hallucination.
+- passed: true only if score >= 0.80 and no critical contradiction exists.
+
+Return strict JSON only:
+{{
+  "factual_score": <0.0 to 1.0>,
+  "required_terms_score": <0.0 to 1.0>,
+  "constraint_score": <0.0 to 1.0>,
+  "hallucination_score": <0.0 to 1.0>,
+  "score": <0.0 to 1.0>,
+  "passed": <true or false>,
+  "missing_terms": ["..."],
+  "forbidden_claims_found": ["..."],
+  "reason": "<short reason>"
+}}"""
+
+
+def pairwise_judge_prompt(case: BenchmarkCase, raw_answer: str, compressed_answer: str) -> str:
+    return f"""You are judging whether context compression preserved answer quality.
+
+Task: {case.task}
+Expected answer: {case.expected_answer}
+Required terms: {json.dumps(case.required_terms)}
+Forbidden claims: {json.dumps(case.must_not_claim)}
+
+Raw-context answer:
+{raw_answer}
+
+Compressed-context answer:
+{compressed_answer}
+
+Compare correctness, preservation of exact IDs/constraints, and unsupported claims.
+Return strict JSON only:
+{{
+  "winner": "raw" | "compressed" | "tie",
+  "raw_score": <0.0 to 1.0>,
+  "compressed_score": <0.0 to 1.0>,
+  "quality_delta": <compressed_score - raw_score>,
+  "compression_preserved_accuracy": <true or false>,
+  "reason": "<short reason>"
+}}"""
+
+
+def ollama_chat(
+    endpoint: str,
+    model: str,
+    messages: list[dict[str, str]],
+    timeout: int,
+) -> dict[str, Any]:
+    body = json.dumps(
+        {
+            "model": model,
+            "messages": messages,
+            "stream": False,
+            "options": {"temperature": 0},
+        }
+    ).encode("utf-8")
+    request = urllib.request.Request(
+        endpoint,
+        data=body,
+        headers={"Content-Type": "application/json"},
+        method="POST",
+    )
+    started = time.perf_counter()
+    with urllib.request.urlopen(request, timeout=timeout) as response:
+        data = json.loads(response.read().decode("utf-8"))
+    data["_latency_s"] = time.perf_counter() - started
+    return data
+
+
+def response_text(response: dict[str, Any]) -> str:
+    return str(response.get("message", {}).get("content", "")).strip()
+
+
+def deterministic_score(answer: str, required_terms: tuple[str, ...]) -> float:
+    folded = answer.lower()
+    hits = sum(1 for term in required_terms if term.lower() in folded)
+    return hits / max(1, len(required_terms))
+
+
+def missing_required_terms(answer: str, required_terms: tuple[str, ...]) -> list[str]:
+    folded = answer.lower()
+    return [term for term in required_terms if term.lower() not in folded]
+
+
+def forbidden_claims_found(answer: str, forbidden_claims: tuple[str, ...]) -> list[str]:
+    folded = answer.lower()
+    return [claim for claim in forbidden_claims if claim.lower() in folded]
+
+
+def extract_json_object(text: str) -> dict[str, Any]:
+    stripped = text.strip()
+    if stripped.startswith("```"):
+        stripped = stripped.strip("`")
+        if stripped.lower().startswith("json"):
+            stripped = stripped[4:].strip()
+    start = stripped.find("{")
+    end = stripped.rfind("}")
+    if start >= 0 and end >= start:
+        stripped = stripped[start : end + 1]
+    return json.loads(stripped)
+
+
+def parse_judge_response(text: str) -> dict[str, Any]:
+    data = extract_json_object(text)
+    component_keys = [
+        "factual_score",
+        "required_terms_score",
+        "constraint_score",
+        "hallucination_score",
+    ]
+    for key in component_keys:
+        if key in data:
+            data[key] = max(0.0, min(1.0, float(data[key])))
+    if "score" not in data and all(key in data for key in component_keys):
+        data["score"] = (
+            0.45 * data["factual_score"]
+            + 0.25 * data["required_terms_score"]
+            + 0.20 * data["constraint_score"]
+            + 0.10 * data["hallucination_score"]
+        )
+    score = float(data.get("score", 0.0))
+    data["score"] = max(0.0, min(1.0, score))
+    data["passed"] = bool(data.get("passed", data["score"] >= 0.80))
+    data["missing_terms"] = list(data.get("missing_terms", []))
+    data["forbidden_claims_found"] = list(data.get("forbidden_claims_found", []))
+    data["reason"] = str(data.get("reason", ""))
+    return data
+
+
+def parse_pairwise_response(text: str) -> dict[str, Any]:
+    data = extract_json_object(text)
+    winner = str(data.get("winner", "tie")).lower()
+    if winner not in {"raw", "compressed", "tie"}:
+        winner = "tie"
+    raw_score = max(0.0, min(1.0, float(data.get("raw_score", 0.0))))
+    compressed_score = max(0.0, min(1.0, float(data.get("compressed_score", 0.0))))
+    data["winner"] = winner
+    data["raw_score"] = raw_score
+    data["compressed_score"] = compressed_score
+    data["quality_delta"] = float(data.get("quality_delta", compressed_score - raw_score))
+    data["compression_preserved_accuracy"] = bool(
+        data.get("compression_preserved_accuracy", compressed_score >= raw_score - 0.05)
+    )
+    data["reason"] = str(data.get("reason", ""))
+    return data
+
+
+def call_json_judge(
+    endpoint: str,
+    model: str,
+    prompt: str,
+    timeout: int,
+    retries: int,
+    parser: Any,
+) -> tuple[dict[str, Any] | None, str | None, float | None]:
+    last_error = None
+    repair_note = ""
+    for attempt in range(max(1, retries + 1)):
+        try:
+            response = ollama_chat(
+                endpoint,
+                model,
+                [
+                    {
+                        "role": "system",
+                        "content": "You are a strict evaluator. Return valid JSON only.",
+                    },
+                    {"role": "user", "content": prompt + repair_note},
+                ],
+                timeout,
+            )
+            return parser(response_text(response)), None, response.get("_latency_s")
+        except (urllib.error.URLError, TimeoutError, json.JSONDecodeError, ValueError) as exc:
+            last_error = str(exc)
+            if attempt == 0:
+                repair_note = (
+                    "\n\nYour previous answer was not valid JSON for this benchmark. "
+                    "Return only the requested JSON object, with no prose or markdown."
+                )
+    return None, last_error, None
+
+
+def deterministic_accuracy(case: BenchmarkCase, answer: str) -> dict[str, Any]:
+    missing = missing_required_terms(answer, case.required_terms)
+    forbidden = forbidden_claims_found(answer, case.must_not_claim)
+    required_score = deterministic_score(answer, case.required_terms)
+    hallucination_score = 0.0 if forbidden else 1.0
+    score = 0.80 * required_score + 0.20 * hallucination_score
+    return {
+        "score": score,
+        "required_terms_score": required_score,
+        "hallucination_score": hallucination_score,
+        "missing_terms": missing,
+        "forbidden_claims_found": forbidden,
+        "passed": score >= 0.80 and not forbidden,
+    }
+
+
+def run_variant(
+    case: BenchmarkCase,
+    variant: str,
+    context: str,
+    args: argparse.Namespace,
+) -> dict[str, Any]:
+    user_prompt = answer_prompt(context, case.task)
+    prompt_estimated_tokens = estimate_tokens(user_prompt)
+    result: dict[str, Any] = {
+        "case_id": case.case_id,
+        "title": case.title,
+        "variant": variant,
+        "prompt_estimated_tokens": prompt_estimated_tokens,
+        "context_estimated_tokens": estimate_tokens(context),
+    }
+
+    if args.dry_run:
+        result.update(
+            {
+                "answer": "",
+                "answer_latency_s": 0.0,
+                "ollama_prompt_tokens": None,
+                "ollama_completion_tokens": None,
+                "deterministic_score": None,
+                "judge_score": None,
+                "accuracy_score": None,
+                "passed": None,
+            }
+        )
+        return result
+
+    try:
+        answer_response = ollama_chat(
+            args.endpoint,
+            args.model,
+            [
+                {"role": "system", "content": "You are a helpful, precise assistant."},
+                {"role": "user", "content": user_prompt},
+            ],
+            args.timeout,
+        )
+    except (urllib.error.URLError, TimeoutError, OSError) as exc:
+        result.update(
+            {
+                "answer": "",
+                "answer_error": str(exc),
+                "answer_latency_s": None,
+                "ollama_prompt_tokens": None,
+                "ollama_completion_tokens": None,
+                "deterministic_score": None,
+                "judge_score": None,
+                "judge_reason": None,
+                "judge_error": None,
+                "accuracy_score": None,
+                "passed": None,
+            }
+        )
+        return result
+
+    answer = response_text(answer_response)
+    det = deterministic_accuracy(case, answer)
+
+    judge_data: dict[str, Any] | None = None
+    judge_error = None
+    if not args.skip_judge:
+        judge_data, judge_error, judge_latency_s = call_json_judge(
+            args.endpoint,
+            args.judge_model,
+            judge_prompt(case, answer),
+            args.timeout,
+            args.judge_retries,
+            parse_judge_response,
+        )
+        result["judge_latency_s"] = judge_latency_s
+
+    judge_score = judge_data["score"] if judge_data else None
+    accuracy = judge_score if judge_score is not None else det["score"]
+    passed = judge_data["passed"] if judge_data else det["passed"]
+    missing_terms = (
+        judge_data.get("missing_terms", det["missing_terms"]) if judge_data else det["missing_terms"]
+    )
+    forbidden_claims = (
+        judge_data.get("forbidden_claims_found", det["forbidden_claims_found"])
+        if judge_data
+        else det["forbidden_claims_found"]
+    )
+
+    result.update(
+        {
+            "answer": answer,
+            "answer_latency_s": answer_response.get("_latency_s"),
+            "ollama_prompt_tokens": answer_response.get("prompt_eval_count"),
+            "ollama_completion_tokens": answer_response.get("eval_count"),
+            "deterministic_score": det["score"],
+            "deterministic_required_terms_score": det["required_terms_score"],
+            "deterministic_hallucination_score": det["hallucination_score"],
+            "judge_score": judge_score,
+            "judge_rubric": judge_data,
+            "judge_reason": judge_data.get("reason") if judge_data else None,
+            "judge_error": judge_error,
+            "missing_terms": missing_terms,
+            "forbidden_claims_found": forbidden_claims,
+            "accuracy_score": accuracy,
+            "passed": passed,
+        }
+    )
+    return result
+
+
+def run_pairwise_judgments(
+    cases: list[BenchmarkCase],
+    results: list[dict[str, Any]],
+    args: argparse.Namespace,
+) -> list[dict[str, Any]]:
+    if args.dry_run or args.skip_judge or args.skip_pairwise_judge:
+        return []
+
+    by_case_variant = {
+        (row["case_id"], row["variant"]): row
+        for row in results
+        if row.get("answer") and not row.get("answer_error")
+    }
+    judgments: list[dict[str, Any]] = []
+    for case in cases:
+        raw = by_case_variant.get((case.case_id, "raw"))
+        compressed = by_case_variant.get((case.case_id, "compressed"))
+        if raw is None or compressed is None:
+            continue
+        data, error, latency_s = call_json_judge(
+            args.endpoint,
+            args.judge_model,
+            pairwise_judge_prompt(case, raw["answer"], compressed["answer"]),
+            args.timeout,
+            args.judge_retries,
+            parse_pairwise_response,
+        )
+        row: dict[str, Any] = {
+            "case_id": case.case_id,
+            "title": case.title,
+            "judge_latency_s": latency_s,
+            "judge_error": error,
+        }
+        if data:
+            row.update(data)
+        judgments.append(row)
+    return judgments
+
+
+def summarize(
+    results: list[dict[str, Any]],
+    pairwise_judgments: list[dict[str, Any]] | None = None,
+) -> dict[str, Any]:
+    by_variant: dict[str, list[dict[str, Any]]] = {}
+    for row in results:
+        by_variant.setdefault(row["variant"], []).append(row)
+
+    summary: dict[str, Any] = {}
+    for variant, rows in by_variant.items():
+        token_key = (
+            "ollama_prompt_tokens"
+            if all(row.get("ollama_prompt_tokens") is not None for row in rows)
+            else "prompt_estimated_tokens"
+        )
+        tokens = [float(row[token_key]) for row in rows]
+        accuracy = [
+            float(row["accuracy_score"])
+            for row in rows
+            if row.get("accuracy_score") is not None
+        ]
+        summary[variant] = {
+            "cases": len(rows),
+            "token_metric": token_key,
+            "avg_prompt_tokens": statistics.mean(tokens),
+            "total_prompt_tokens": sum(tokens),
+            "avg_accuracy": statistics.mean(accuracy) if accuracy else None,
+            "pass_rate": (
+                sum(1 for row in rows if row.get("passed")) / len(rows)
+                if rows and rows[0].get("passed") is not None
+                else None
+            ),
+        }
+
+    if "raw" in summary and "compressed" in summary:
+        raw = summary["raw"]["avg_prompt_tokens"]
+        compressed = summary["compressed"]["avg_prompt_tokens"]
+        summary["comparison"] = {
+            "avg_token_reduction_pct": ((raw - compressed) / raw * 100.0) if raw else 0.0,
+            "accuracy_delta": (
+                summary["compressed"]["avg_accuracy"] - summary["raw"]["avg_accuracy"]
+                if summary["raw"]["avg_accuracy"] is not None
+                and summary["compressed"]["avg_accuracy"] is not None
+                else None
+            ),
+        }
+    if pairwise_judgments:
+        scored = [row for row in pairwise_judgments if not row.get("judge_error")]
+        if scored:
+            preserved = [
+                row
+                for row in scored
+                if bool(row.get("compression_preserved_accuracy", False))
+            ]
+            deltas = [float(row.get("quality_delta", 0.0)) for row in scored]
+            summary["pairwise_judge"] = {
+                "cases": len(scored),
+                "compressed_wins": sum(1 for row in scored if row.get("winner") == "compressed"),
+                "raw_wins": sum(1 for row in scored if row.get("winner") == "raw"),
+                "ties": sum(1 for row in scored if row.get("winner") == "tie"),
+                "preservation_rate": len(preserved) / len(scored),
+                "avg_quality_delta": statistics.mean(deltas),
+            }
+    return summary
+
+
+def markdown_report(payload: dict[str, Any]) -> str:
+    lines = [
+        "# Context Compression Benchmark",
+        "",
+        f"Generated: {payload['generated_at']}",
+        f"Endpoint: `{payload['endpoint']}`",
+        f"Worker model: `{payload['model']}`",
+        f"Judge model: `{payload['judge_model']}`",
+        f"Dry run: `{payload['dry_run']}`",
+        "",
+        "## Summary",
+        "",
+        "| Variant | Token metric | Avg prompt tokens | Total prompt tokens | Avg accuracy | Pass rate |",
+        "|---|---:|---:|---:|---:|---:|",
+    ]
+    for variant in ["raw", "compressed"]:
+        data = payload["summary"].get(variant, {})
+        lines.append(
+            "| {variant} | {metric} | {avg_tokens:.1f} | {total_tokens:.0f} | {accuracy} | {pass_rate} |".format(
+                variant=variant,
+                metric=data.get("token_metric", "-"),
+                avg_tokens=float(data.get("avg_prompt_tokens", 0.0)),
+                total_tokens=float(data.get("total_prompt_tokens", 0.0)),
+                accuracy=(
+                    "-"
+                    if data.get("avg_accuracy") is None
+                    else f"{float(data['avg_accuracy']):.3f}"
+                ),
+                pass_rate=(
+                    "-"
+                    if data.get("pass_rate") is None
+                    else f"{float(data['pass_rate']) * 100:.1f}%"
+                ),
+            )
+        )
+    comparison = payload["summary"].get("comparison", {})
+    if comparison:
+        lines.extend(
+            [
+                "",
+                "## Comparison",
+                "",
+                f"- Average token reduction: `{comparison['avg_token_reduction_pct']:.1f}%`",
+                "- Accuracy delta: `{}`".format(
+                    "-"
+                    if comparison.get("accuracy_delta") is None
+                    else f"{comparison['accuracy_delta']:.3f}"
+                ),
+            ]
+        )
+    pairwise_summary = payload["summary"].get("pairwise_judge", {})
+    if pairwise_summary:
+        lines.extend(
+            [
+                "",
+                "## Pairwise Judge",
+                "",
+                f"- Cases judged: `{pairwise_summary['cases']}`",
+                f"- Compressed wins: `{pairwise_summary['compressed_wins']}`",
+                f"- Raw wins: `{pairwise_summary['raw_wins']}`",
+                f"- Ties: `{pairwise_summary['ties']}`",
+                f"- Accuracy preservation rate: `{pairwise_summary['preservation_rate'] * 100:.1f}%`",
+                f"- Average quality delta: `{pairwise_summary['avg_quality_delta']:.3f}`",
+            ]
+        )
+    lines.extend(
+        [
+            "",
+            "## Case Details",
+            "",
+            "| Case | Variant | Estimated prompt tokens | Ollama prompt tokens | Accuracy | Missing terms | Forbidden claims | Passed | Error |",
+            "|---|---|---:|---:|---:|---|---|---|---|",
+        ]
+    )
+    for row in payload["results"]:
+        error = row.get("answer_error") or row.get("judge_error") or ""
+        missing = ", ".join(row.get("missing_terms") or [])
+        forbidden = ", ".join(row.get("forbidden_claims_found") or [])
+        lines.append(
+            "| {case} | {variant} | {estimate} | {ollama} | {accuracy} | {missing} | {forbidden} | {passed} | {error} |".format(
+                case=row["case_id"],
+                variant=row["variant"],
+                estimate=row["prompt_estimated_tokens"],
+                ollama=row.get("ollama_prompt_tokens") or "-",
+                accuracy=(
+                    "-"
+                    if row.get("accuracy_score") is None
+                    else f"{float(row['accuracy_score']):.3f}"
+                ),
+                missing=missing.replace("|", "\\|") or "-",
+                forbidden=forbidden.replace("|", "\\|") or "-",
+                passed=row.get("passed"),
+                error=error.replace("|", "\\|"),
+            )
+        )
+    if payload.get("pairwise_judgments"):
+        lines.extend(
+            [
+                "",
+                "## Pairwise Details",
+                "",
+                "| Case | Winner | Raw score | Compressed score | Delta | Preserved | Reason | Error |",
+                "|---|---|---:|---:|---:|---|---|---|",
+            ]
+        )
+        for row in payload["pairwise_judgments"]:
+            lines.append(
+                "| {case} | {winner} | {raw_score} | {compressed_score} | {delta} | {preserved} | {reason} | {error} |".format(
+                    case=row["case_id"],
+                    winner=row.get("winner", "-"),
+                    raw_score=(
+                        "-" if row.get("raw_score") is None else f"{float(row['raw_score']):.3f}"
+                    ),
+                    compressed_score=(
+                        "-"
+                        if row.get("compressed_score") is None
+                        else f"{float(row['compressed_score']):.3f}"
+                    ),
+                    delta=(
+                        "-"
+                        if row.get("quality_delta") is None
+                        else f"{float(row['quality_delta']):.3f}"
+                    ),
+                    preserved=row.get("compression_preserved_accuracy"),
+                    reason=str(row.get("reason", "")).replace("|", "\\|") or "-",
+                    error=str(row.get("judge_error", "")).replace("|", "\\|") or "-",
+                )
+            )
+    return "\n".join(lines) + "\n"
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(
+        description="Benchmark raw vs compressed context token use and answer accuracy."
+    )
+    parser.add_argument("--endpoint", default=DEFAULT_ENDPOINT)
+    parser.add_argument("--model", default=DEFAULT_MODEL)
+    parser.add_argument("--judge-model", default=DEFAULT_MODEL)
+    parser.add_argument("--timeout", type=int, default=180)
+    parser.add_argument("--output-dir", default="benchmark_results")
+    parser.add_argument("--dry-run", action="store_true")
+    parser.add_argument("--skip-judge", action="store_true")
+    parser.add_argument("--skip-pairwise-judge", action="store_true")
+    parser.add_argument("--judge-retries", type=int, default=1)
+    parser.add_argument("--pass-threshold", type=float, default=0.75)
+    args = parser.parse_args()
+
+    cases = build_cases()
+    results = []
+    for case in cases:
+        results.append(run_variant(case, "raw", case.raw_context, args))
+        results.append(run_variant(case, "compressed", case.compressed_context, args))
+    pairwise_judgments = run_pairwise_judgments(cases, results, args)
+
+    payload = {
+        "generated_at": datetime.now(timezone.utc).isoformat(),
+        "endpoint": args.endpoint,
+        "model": args.model,
+        "judge_model": args.judge_model,
+        "dry_run": args.dry_run,
+        "results": results,
+        "pairwise_judgments": pairwise_judgments,
+        "summary": summarize(results, pairwise_judgments),
+    }
+
+    output_dir = Path(args.output_dir)
+    output_dir.mkdir(parents=True, exist_ok=True)
+    json_path = output_dir / "context_compression_benchmark.json"
+    md_path = output_dir / "context_compression_benchmark.md"
+    json_path.write_text(json.dumps(payload, indent=2), encoding="utf-8")
+    md_path.write_text(markdown_report(payload), encoding="utf-8")
+
+    print(f"Wrote {json_path}")
+    print(f"Wrote {md_path}")
+    print(json.dumps(payload["summary"], indent=2))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

mn_context_engine_sdk/working_memory.py

@@ -0,0 +1,258 @@
+from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
+from typing import Any, Optional, Dict, List
+from datetime import datetime, timezone
+import uuid
+import json
+
+def utcnow():
+    return datetime.now(timezone.utc)
+
+VALID_TYPES = {"Fact", "Hypothesis", "Task", "Decision", "Evidence", "Constraint"}
+VALID_STATUSES = {
+    "draft",
+    "validated",
+    "used",
+    "archived",
+    "hypothesis",
+    "tested",
+    "confirmed",
+    "rejected",
+}
+
+GENERIC_TRANSITIONS = {
+    "draft": {"validated", "archived"},
+    "validated": {"used", "archived"},
+    "used": {"archived"},
+    "archived": set(),
+}
+
+HYPOTHESIS_TRANSITIONS = {
+    "draft": {"tested", "archived"},
+    "hypothesis": {"tested", "archived"},
+    "tested": {"confirmed", "rejected", "archived"},
+    "confirmed": {"archived"},
+    "rejected": {"archived"},
+    "archived": set(),
+}
+
+class MNMemoryItem(BaseModel):
+    model_config = ConfigDict(validate_assignment=True)
+
+    id: str = Field(default_factory=lambda: str(uuid.uuid4()))
+    type: str
+    content: Any
+    status: str
+    confidence: float = 1.0
+    source: str
+    created_at: datetime = Field(default_factory=utcnow)
+    updated_at: datetime = Field(default_factory=utcnow)
+    expires_at: Optional[datetime] = None
+    version: int = 1
+
+    @field_validator("type")
+    @classmethod
+    def validate_type(cls, value: str) -> str:
+        if value not in VALID_TYPES:
+            raise ValueError(f"invalid memory type: {value}")
+        return value
+
+    @field_validator("status")
+    @classmethod
+    def validate_status(cls, value: str) -> str:
+        if value not in VALID_STATUSES:
+            raise ValueError(f"invalid memory status: {value}")
+        return value
+
+    @field_validator("confidence")
+    @classmethod
+    def validate_confidence(cls, value: float) -> float:
+        if not 0.0 <= value <= 1.0:
+            raise ValueError("confidence must be between 0.0 and 1.0")
+        return value
+
+    @field_validator("source")
+    @classmethod
+    def validate_source(cls, value: str) -> str:
+        if not value.strip():
+            raise ValueError("source must not be empty")
+        return value
+
+    @model_validator(mode="after")
+    def validate_content(self) -> "MNMemoryItem":
+        if not isinstance(self.content, dict):
+            raise ValueError("content must be a structured object")
+        return self
+
+class MNMemoryEdge(BaseModel):
+    source_id: str
+    target_id: str
+    relation: str
+
+    @field_validator("relation")
+    @classmethod
+    def validate_relation(cls, value: str) -> str:
+        if not value.strip():
+            raise ValueError("relation must not be empty")
+        return value
+
+class MNWorkingMemory:
+    def __init__(self):
+        self._items: Dict[str, MNMemoryItem] = {}
+        self._edges: List[MNMemoryEdge] = []
+
+    def add(self, item: MNMemoryItem) -> None:
+        self._items[item.id] = item
+
+    def update(self, item_id: str, updates: dict) -> None:
+        if item_id in self._items:
+            item = self._items[item_id]
+            updates = dict(updates)
+            expected_version = updates.pop("expected_version", None)
+            if expected_version is not None and expected_version != item.version:
+                raise ValueError(f"version conflict: item has {item.version}, expected {expected_version}")
+
+            if "status" in updates:
+                self._validate_transition(item.type, item.status, updates["status"])
+
+            for key, value in updates.items():
+                if hasattr(item, key):
+                    setattr(item, key, value)
+            item.updated_at = utcnow()
+            item.version += 1
+
+    def get(self, item_id: str) -> Optional[MNMemoryItem]:
+        item = self._items.get(item_id)
+        if item is None or self._is_expired(item):
+            return None
+        return item
+
+    def query(self, filters: dict) -> List[MNMemoryItem]:
+        results = []
+        for item in self._items.values():
+            if self._is_expired(item):
+                continue
+            match = True
+            for key, value in filters.items():
+                if getattr(item, key, None) != value:
+                    match = False
+                    break
+            if match:
+                results.append(item)
+        return results
+
+    def link(self, source_id: str, target_id: str, relation: str) -> None:
+        if source_id not in self._items:
+            raise KeyError(f"source item not found: {source_id}")
+        if target_id not in self._items:
+            raise KeyError(f"target item not found: {target_id}")
+        edge = MNMemoryEdge(source_id=source_id, target_id=target_id, relation=relation)
+        self._edges.append(edge)
+
+    def invalidate(self, item_id: str) -> None:
+        if item_id in self._items:
+            self._validate_transition(self._items[item_id].type, self._items[item_id].status, "archived")
+            self._items[item_id].status = "archived"
+            self._items[item_id].updated_at = utcnow()
+            self._items[item_id].version += 1
+
+    def get_context(self, agent_role: str, goal_id: str) -> List[MNMemoryItem]:
+        """
+        Retrieves selective context for an agent around a goal.
+        Items must be visible to the role and either match the goal directly
+        or sit within two graph hops of a matching item.
+        """
+        seeds = {
+            item.id
+            for item in self._items.values()
+            if not self._is_expired(item)
+            and item.status != "archived"
+            and (item.id == goal_id or item.content.get("goal_id") == goal_id)
+        }
+        reachable = self._expand_graph(seeds, max_depth=2)
+
+        return [
+            item
+            for item in self._items.values()
+            if item.id in reachable
+            and item.status != "archived"
+            and not self._is_expired(item)
+            and self._is_visible_to_role(agent_role, item)
+        ]
+
+    @staticmethod
+    def _is_expired(item: MNMemoryItem) -> bool:
+        return item.expires_at is not None and utcnow() > item.expires_at
+
+    @staticmethod
+    def _validate_transition(item_type: str, current: str, new_status: str) -> None:
+        if current == new_status:
+            return
+        transitions = HYPOTHESIS_TRANSITIONS if item_type == "Hypothesis" else GENERIC_TRANSITIONS
+        if new_status not in transitions.get(current, set()):
+            raise ValueError(f"invalid status transition for {item_type}: {current} -> {new_status}")
+
+    @staticmethod
+    def _is_visible_to_role(agent_role: str, item: MNMemoryItem) -> bool:
+        acl = item.content.get("acl")
+        if isinstance(acl, dict):
+            deny_roles = acl.get("deny_roles", [])
+            if agent_role in deny_roles:
+                return False
+            allow_roles = acl.get("allow_roles")
+            if allow_roles is not None:
+                return agent_role in allow_roles
+
+        if agent_role == "planner":
+            return item.type != "Evidence"
+        if agent_role == "executor":
+            return item.type != "Hypothesis"
+        if agent_role == "reviewer":
+            return True
+        return item.source == agent_role
+
+    def _expand_graph(self, seed_ids: set[str], max_depth: int) -> set[str]:
+        reachable = set(seed_ids)
+        frontier = set(seed_ids)
+        for _ in range(max_depth):
+            next_frontier = set()
+            for edge in self._edges:
+                if edge.source_id in frontier and edge.target_id not in reachable:
+                    next_frontier.add(edge.target_id)
+                if edge.target_id in frontier and edge.source_id not in reachable:
+                    next_frontier.add(edge.source_id)
+            if not next_frontier:
+                break
+            reachable.update(next_frontier)
+            frontier = next_frontier
+        return reachable
+
+    # --- Methods for compatibility with MicroNeuron payloads (Serialization) ---
+    def to_dict(self) -> dict:
+        return {
+            "items": [item.model_dump(mode="json") for item in self._items.values()],
+            "edges": [edge.model_dump(mode="json") for edge in self._edges]
+        }
+
+    @classmethod
+    def from_dict(cls, data: dict) -> "MNWorkingMemory":
+        wm = cls()
+        for item_data in data.get("items", []):
+            item = MNMemoryItem.model_validate(item_data)
+            wm.add(item)
+        for edge_data in data.get("edges", []):
+            edge = MNMemoryEdge.model_validate(edge_data)
+            wm._edges.append(edge)
+        return wm
+
+    def to_json(self) -> str:
+        return json.dumps(self.to_dict())
+
+    @classmethod
+    def from_json(cls, json_str: str) -> "MNWorkingMemory":
+        return cls.from_dict(json.loads(json_str))
+
+
+# Backward-compatible aliases for older SDK users.
+MemoryItem = MNMemoryItem
+MemoryEdge = MNMemoryEdge
+WorkingMemory = MNWorkingMemory