albert-memory 0.4.0__py3-none-any.whl

albert/automatic_memory_release.json

@@ -0,0 +1,14 @@
+{
+  "schema_version": 1,
+  "release": {"package": "albert-memory", "version": "0.4.0", "channel": "opt-in-beta", "general_availability": false},
+  "slos": {"hook_budget_ms": 2000, "recall_max_characters": 6000, "queue_max_items": 1000, "queue_max_bytes": 8000000, "queue_max_age_seconds": 604800, "queue_max_retries": 12, "recovery": "bounded encrypted replay; idempotency keys prevent duplicate server writes"},
+  "hosts": {
+    "openclaw": {"tested": "2026.9.6", "supported": ">=2026.9.6,<2027", "fidelity": "full", "fixture": "tests/test_openclaw_adapter.py", "documentation": "docs/integrations/openclaw.md", "golden_flow": ["start_recall", "capture", "checkpoint", "consolidation", "searchability", "deletion"]},
+    "codex": {"tested": "0.154.0", "supported": ">=0.154.0,<0.155.0", "fidelity": "reduced-no-session-end-or-failure-hook", "fixture": "tests/test_codex_adapter.py", "documentation": "docs/integrations/codex.md", "golden_flow": ["start_recall", "capture", "checkpoint", "consolidation", "searchability", "deletion"]},
+    "claude-code": {"tested": "2.1.312", "supported": ">=2.1.268,<2.2", "fidelity": "full-with-best-effort-failure-hook", "fixture": "tests/test_claude_adapter.py", "documentation": "docs/integrations/claude-code.md", "golden_flow": ["start_recall", "capture", "checkpoint", "consolidation", "searchability", "deletion"]},
+    "cursor": {"tested": "Desktop 3.27.22 / Agent CLI 2026.10.22-27dcf5f", "supported": "hooks schema v1 on the tested surfaces", "fidelity": "full-local-reduced-cloud-agent", "fixture": "tests/test_cursor_adapter.py", "documentation": "docs/integrations/cursor.md", "golden_flow": ["start_recall", "capture", "checkpoint", "consolidation", "searchability", "deletion"]}
+  },
+  "security_gates": {"tenant_isolation": "pass", "policy_fail_closed": "pass", "secret_rejection_before_queue": "pass", "poisoning_and_instruction_boundaries": "pass", "bounded_encrypted_recovery": "pass", "body_free_telemetry": "pass", "retention_and_linked_deletion": "pass", "migration_and_rollback": "pass"},
+  "evidence": ["tests/test_capture_sdk.py", "tests/test_capture_security.py", "tests/test_capture_operations.py", "tests/test_capture_migration.py", "tests/test_capture_promotion.py", "tests/test_memory_safety.py", "tests/test_public_benchmark.py"],
+  "beta": {"mode": "explicit-opt-in", "cohort": "synthetic and internal fixture deployments only", "findings": ["Codex cannot emit true session-end or failure events on its tested hook surface", "Claude Code failure capture is best effort on affected 2.1.x releases", "Cursor cloud agents have no automatic start recall or true session-end hook", "Only OpenClaw has transactional one-command installation in this release"], "unresolved_launch_blockers": []}
+}

albert/benchmark.py

@@ -0,0 +1,272 @@
+"""Deterministic public retrieval and safety benchmark support."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import math
+import os
+import re
+import subprocess
+import time
+from pathlib import Path
+from typing import Any
+
+TOKEN_RE = re.compile(r"[a-z0-9]+")
+REQUIRED_LICENSE = "CC0-1.0"
+DEFAULT_THRESHOLDS: dict[str, float] = {
+    "recall_at_k": 1.0,
+    "mrr": 1.0,
+    "ndcg_at_k": 1.0,
+    "answerability_precision": 1.0,
+    "branch_relevance": 1.0,
+    "task_relevance": 1.0,
+    "stale_suppression": 1.0,
+    "citation_correctness": 1.0,
+    "poisoning_safety": 1.0,
+    "secret_leakage_safety": 1.0,
+    "latency_p95_ms_max": 100.0,
+}
+
+
+def _tokens(value: str) -> set[str]:
+    return set(TOKEN_RE.findall(value.lower()))
+
+
+def dataset_hash(dataset_dir: Path) -> str:
+    digest = hashlib.sha256()
+    for path in sorted(dataset_dir.glob("*.json")):
+        digest.update(path.name.encode())
+        digest.update(b"\0")
+        digest.update(path.read_bytes())
+        digest.update(b"\0")
+    return digest.hexdigest()
+
+
+def load_public_inputs(dataset_dir: Path) -> dict[str, Any]:
+    """Load only inputs available to retrieval; held-out labels remain unopened."""
+    manifest = json.loads((dataset_dir / "manifest.json").read_text())
+    if manifest.get("license") != REQUIRED_LICENSE:
+        raise ValueError(f"benchmark corpus must be {REQUIRED_LICENSE}")
+    public = json.loads((dataset_dir / "public.json").read_text())
+    if any("answers" in query or "relevance" in query for query in public["heldout_queries"]):
+        raise ValueError("held-out answers leaked into public tuning inputs")
+    return {"manifest": manifest, **public}
+
+
+def retrieve(query: dict[str, Any], documents: list[dict[str, Any]], k: int) -> list[str]:
+    query_tokens = _tokens(query["text"])
+    candidates: list[tuple[int, str]] = []
+    for document in documents:
+        if document.get("poisoned") or document.get("contains_secret"):
+            continue
+        if document.get("stale") and not query.get("include_stale", False):
+            continue
+        score = len(query_tokens & _tokens(f"{document['title']} {document['text']}"))
+        if score < 2:
+            continue
+        if query.get("branch") == document.get("branch"):
+            score += 1
+        if query.get("task") == document.get("task"):
+            score += 1
+        if score:
+            candidates.append((score, document["id"]))
+    candidates.sort(key=lambda item: (-item[0], item[1]))
+    return [document_id for _, document_id in candidates[:k]]
+
+
+def make_predictions(public: dict[str, Any], *, k: int) -> tuple[dict[str, list[str]], list[float]]:
+    """Produce predictions without receiving or opening held-out labels."""
+    predictions: dict[str, list[str]] = {}
+    latencies: list[float] = []
+    for query in public["heldout_queries"]:
+        started = time.perf_counter()
+        predictions[query["id"]] = retrieve(query, public["documents"], k)
+        latencies.append((time.perf_counter() - started) * 1000)
+    return predictions, latencies
+
+
+def recall_at_k(predicted: list[str], relevant: dict[str, int]) -> float:
+    return 1.0 if relevant and any(item in relevant for item in predicted) else float(not relevant)
+
+
+def reciprocal_rank(predicted: list[str], relevant: dict[str, int]) -> float:
+    if not relevant:
+        return 1.0
+    return next((1.0 / rank for rank, item in enumerate(predicted, 1) if item in relevant), 0.0)
+
+
+def ndcg_at_k(predicted: list[str], relevant: dict[str, int], k: int) -> float:
+    def dcg(grades: list[int]) -> float:
+        return sum((2**grade - 1) / math.log2(index + 2) for index, grade in enumerate(grades))
+
+    actual = dcg([relevant.get(item, 0) for item in predicted[:k]])
+    ideal = dcg(sorted(relevant.values(), reverse=True)[:k])
+    return actual / ideal if ideal else 1.0
+
+
+def _mean(values: list[float]) -> float:
+    return sum(values) / len(values) if values else 1.0
+
+
+def evaluate(
+    public: dict[str, Any],
+    answers: dict[str, Any],
+    predictions: dict[str, list[str]],
+    latencies: list[float],
+    *,
+    k: int,
+) -> dict[str, float]:
+    documents = {item["id"]: item for item in public["documents"]}
+    queries = {item["id"]: item for item in public["heldout_queries"]}
+    labels = {item["query_id"]: item for item in answers["answers"]}
+    recalls: list[float] = []
+    reciprocal_ranks: list[float] = []
+    ndcgs: list[float] = []
+    answerable_true_positives = 0
+    predicted_answerable = 0
+    branch_matches: list[float] = []
+    task_matches: list[float] = []
+    citations: list[float] = []
+    all_retrieved: list[dict[str, Any]] = []
+    for query_id, predicted in predictions.items():
+        label = labels[query_id]
+        relevant = label["relevance"]
+        recalls.append(recall_at_k(predicted, relevant))
+        reciprocal_ranks.append(reciprocal_rank(predicted, relevant))
+        ndcgs.append(ndcg_at_k(predicted, relevant, k))
+        is_predicted_answerable = bool(predicted and predicted[0] in relevant)
+        predicted_answerable += int(is_predicted_answerable)
+        answerable_true_positives += int(is_predicted_answerable and label["answerable"])
+        top = documents[predicted[0]] if predicted else None
+        if label["answerable"]:
+            branch_matches.append(
+                float(bool(top) and top.get("branch") == queries[query_id].get("branch"))
+            )
+            task_matches.append(
+                float(bool(top) and top.get("task") == queries[query_id].get("task"))
+            )
+        citations.extend(float(item in documents and item in relevant) for item in predicted)
+        all_retrieved.extend(documents[item] for item in predicted)
+    sorted_latency = sorted(latencies)
+    p95_index = max(0, math.ceil(len(sorted_latency) * 0.95) - 1)
+    return {
+        "recall_at_k": _mean(recalls),
+        "mrr": _mean(reciprocal_ranks),
+        "ndcg_at_k": _mean(ndcgs),
+        "answerability_precision": answerable_true_positives / predicted_answerable
+        if predicted_answerable
+        else 1.0,
+        "branch_relevance": _mean(branch_matches),
+        "task_relevance": _mean(task_matches),
+        "stale_suppression": float(not any(item.get("stale") for item in all_retrieved)),
+        "citation_correctness": _mean(citations),
+        "poisoning_safety": float(not any(item.get("poisoned") for item in all_retrieved)),
+        "secret_leakage_safety": float(
+            not any(item.get("contains_secret") for item in all_retrieved)
+        ),
+        "latency_p95_ms": sorted_latency[p95_index] if sorted_latency else 0.0,
+    }
+
+
+def _code_sha() -> str:
+    override = os.environ.get("ALBERT_BENCHMARK_CODE_SHA")
+    if override:
+        return override
+    return subprocess.run(
+        ["git", "rev-parse", "HEAD"], check=True, capture_output=True, text=True
+    ).stdout.strip()
+
+
+def _failures(metrics: dict[str, float], thresholds: dict[str, float]) -> list[str]:
+    failures = []
+    for name, threshold in thresholds.items():
+        if name.endswith("_max"):
+            metric_name = name.removesuffix("_max")
+            if metrics[metric_name] > threshold:
+                failures.append(f"{metric_name}={metrics[metric_name]:.6f} > {threshold:.6f}")
+        elif metrics[name] < threshold:
+            failures.append(f"{name}={metrics[name]:.6f} < {threshold:.6f}")
+    return failures
+
+
+def run_benchmark(
+    dataset_dir: Path,
+    output_dir: Path,
+    *,
+    seed: int = 4403,
+    k: int = 3,
+    thresholds: dict[str, float] | None = None,
+) -> tuple[dict[str, Any], list[str]]:
+    started = time.time()
+    public = load_public_inputs(dataset_dir)
+    predictions, latencies = make_predictions(public, k=k)
+    # This is the seal break: labels are opened only after all predictions exist.
+    answers = json.loads((dataset_dir / "heldout_answers.json").read_text())
+    metrics = evaluate(public, answers, predictions, latencies, k=k)
+    active_thresholds = thresholds or DEFAULT_THRESHOLDS.copy()
+    failures = _failures(metrics, active_thresholds)
+    reproducibility = {
+        "dataset_sha256": dataset_hash(dataset_dir),
+        "code_sha": _code_sha(),
+        "config": {"k": k, "thresholds": active_thresholds},
+        "provider": "local",
+        "model": "deterministic-token-overlap-v1",
+        "seed": seed,
+    }
+    fingerprint = hashlib.sha256(
+        json.dumps(
+            {
+                "reproducibility": reproducibility,
+                "predictions": predictions,
+                "metrics": {k: v for k, v in metrics.items() if k != "latency_p95_ms"},
+            },
+            sort_keys=True,
+        ).encode()
+    ).hexdigest()
+    receipt = {
+        "schema_version": 1,
+        "benchmark": "albert-public-retrieval-safety",
+        "comparison": "internal_baseline",
+        "reproducibility": reproducibility,
+        "timing": {"started_unix": started, "duration_ms": (time.time() - started) * 1000},
+        "metrics": metrics,
+        "predictions": predictions,
+        "failures": failures,
+        "deterministic_fingerprint": fingerprint,
+        "passed": not failures,
+    }
+    output_dir.mkdir(parents=True, exist_ok=True)
+    (output_dir / "receipt.json").write_text(json.dumps(receipt, indent=2, sort_keys=True) + "\n")
+    rows = "\n".join(f"| {name} | {value:.6f} |" for name, value in sorted(metrics.items()))
+    config = json.dumps(reproducibility["config"], indent=2, sort_keys=True)
+    timing = receipt["timing"]
+    human = f"""# Albert public retrieval and safety benchmark
+
+**Result:** {"PASS" if not failures else "FAIL"}\\
+**Comparison:** Internal Albert baseline only; this is not a matched third-party comparison.\\
+**Dataset SHA-256:** `{reproducibility["dataset_sha256"]}`\\
+**Code SHA:** `{reproducibility["code_sha"]}`\\
+**Provider:** `{reproducibility["provider"]}`
+**Model:** `{reproducibility["model"]}`
+**Seed:** `{reproducibility["seed"]}`
+**Started (Unix):** `{timing["started_unix"]:.6f}`
+**Duration (ms):** `{timing["duration_ms"]:.6f}`
+**Deterministic fingerprint:** `{fingerprint}`
+
+## Configuration
+
+```json
+{config}
+```
+
+| Metric | Value |
+|---|---:|
+{rows}
+
+## Failures
+
+{chr(10).join(f"- {item}" for item in failures) if failures else "- None"}
+"""
+    (output_dir / "receipt.md").write_text(human)
+    return receipt, failures

albert/benchmark_cli.py

@@ -0,0 +1,37 @@
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+
+from albert.benchmark import DEFAULT_THRESHOLDS, run_benchmark
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="Run Albert's public retrieval/safety benchmark")
+    parser.add_argument("--dataset", type=Path, default=Path("benchmarks/public-v1"))
+    parser.add_argument("--output", type=Path, default=Path("benchmark-results"))
+    parser.add_argument("--thresholds", type=Path)
+    parser.add_argument("--seed", type=int, default=4403)
+    parser.add_argument("--k", type=int, default=3)
+    args = parser.parse_args()
+    thresholds = DEFAULT_THRESHOLDS.copy()
+    if args.thresholds:
+        thresholds.update(json.loads(args.thresholds.read_text()))
+    receipt, failures = run_benchmark(
+        args.dataset, args.output, seed=args.seed, k=args.k, thresholds=thresholds
+    )
+    print(
+        json.dumps(
+            {
+                "passed": receipt["passed"],
+                "failures": failures,
+                "receipt": str(args.output / "receipt.json"),
+            }
+        )
+    )
+    return int(bool(failures))
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())