agent-memory-sdk 0.1.0__py3-none-any.whl

agent_memory/__init__.py

@@ -0,0 +1,46 @@
+"""Persistent agentic memory with semantic retrieval and restore/replay/verify decisions."""
+
+from agent_memory.benchmark import BenchmarkResult, format_benchmark_report, run_benchmark
+from agent_memory.decision import DecisionEngine
+from agent_memory.eval import EvalDataset, EvalResult, format_eval_report, run_eval, run_eval_suite
+from agent_memory.manager import Memory, MemoryManager
+from agent_memory.models import (
+    MemoryAction,
+    MemoryDecision,
+    MemoryEntry,
+    MemoryScope,
+    MemoryState,
+    MemoryType,
+    RetrievalResult,
+)
+from agent_memory.policy import DecisionPolicy, DefaultPolicy
+from agent_memory.sqlite_store import SqliteMemoryStore
+
+try:
+    from agent_memory._version import __version__
+except ImportError:
+    __version__ = "0.0.0"
+
+__all__ = [
+    "BenchmarkResult",
+    "DecisionEngine",
+    "DecisionPolicy",
+    "DefaultPolicy",
+    "EvalDataset",
+    "EvalResult",
+    "Memory",
+    "MemoryAction",
+    "MemoryDecision",
+    "MemoryEntry",
+    "MemoryManager",
+    "MemoryScope",
+    "MemoryState",
+    "MemoryType",
+    "RetrievalResult",
+    "SqliteMemoryStore",
+    "format_benchmark_report",
+    "format_eval_report",
+    "run_benchmark",
+    "run_eval",
+    "run_eval_suite",
+]

agent_memory/_version.py

@@ -0,0 +1,24 @@
+# file generated by vcs-versioning
+# don't change, don't track in version control
+from __future__ import annotations
+
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
+
+version: str
+__version__: str
+__version_tuple__: tuple[int | str, ...]
+version_tuple: tuple[int | str, ...]
+commit_id: str | None
+__commit_id__: str | None
+
+__version__ = version = '0.1.0'
+__version_tuple__ = version_tuple = (0, 1, 0)
+
+__commit_id__ = commit_id = None

agent_memory/benchmark.py

@@ -0,0 +1,140 @@
+from __future__ import annotations
+
+import time
+from dataclasses import dataclass, field
+from statistics import mean
+
+from agent_memory.manager import Memory
+from agent_memory.models import MemoryAction
+
+
+@dataclass
+class BenchmarkResult:
+    queries: int = 0
+    replay_count: int = 0
+    restore_count: int = 0
+    verify_count: int = 0
+    none_count: int = 0
+    latencies_ms: list[float] = field(default_factory=list)
+    memory_hits: int = 0
+
+    @property
+    def replay_rate(self) -> float:
+        return self.replay_count / self.queries if self.queries else 0.0
+
+    @property
+    def restore_rate(self) -> float:
+        return self.restore_count / self.queries if self.queries else 0.0
+
+    @property
+    def verify_rate(self) -> float:
+        return self.verify_count / self.queries if self.queries else 0.0
+
+    @property
+    def hit_rate(self) -> float:
+        return self.memory_hits / self.queries if self.queries else 0.0
+
+    @property
+    def avg_latency_ms(self) -> float:
+        return mean(self.latencies_ms) if self.latencies_ms else 0.0
+
+    @property
+    def p95_latency_ms(self) -> float:
+        if not self.latencies_ms:
+            return 0.0
+        sorted_lat = sorted(self.latencies_ms)
+        idx = int(0.95 * (len(sorted_lat) - 1))
+        return sorted_lat[idx]
+
+    def to_dict(self) -> dict:
+        return {
+            "queries": self.queries,
+            "replay_rate": round(self.replay_rate, 4),
+            "restore_rate": round(self.restore_rate, 4),
+            "verify_rate": round(self.verify_rate, 4),
+            "hit_rate": round(self.hit_rate, 4),
+            "avg_latency_ms": round(self.avg_latency_ms, 2),
+            "p95_latency_ms": round(self.p95_latency_ms, 2),
+            "action_counts": {
+                "replay": self.replay_count,
+                "restore": self.restore_count,
+                "verify": self.verify_count,
+                "none": self.none_count,
+            },
+        }
+
+
+def run_benchmark(
+    memory: Memory,
+    queries: list[str],
+    *,
+    baseline_no_memory_ms: float | None = None,
+) -> tuple[BenchmarkResult, dict]:
+    """
+    Run resolve() over queries and collect action distribution + latency.
+
+    Returns (result, comparison) where comparison contrasts with a no-memory baseline.
+    """
+    result = BenchmarkResult()
+
+    for query in queries:
+        start = time.perf_counter()
+        decision = memory.resolve(query)
+        elapsed_ms = (time.perf_counter() - start) * 1000
+
+        result.queries += 1
+        result.latencies_ms.append(elapsed_ms)
+
+        if decision.action == MemoryAction.REPLAY:
+            result.replay_count += 1
+            result.memory_hits += 1
+        elif decision.action == MemoryAction.RESTORE:
+            result.restore_count += 1
+            result.memory_hits += 1
+        elif decision.action == MemoryAction.VERIFY:
+            result.verify_count += 1
+            result.memory_hits += 1
+        else:
+            result.none_count += 1
+
+    # Estimate baseline: no-memory path is just LLM; we approximate with a fixed stub
+    # or caller-provided measurement.
+    estimated_baseline = baseline_no_memory_ms or 800.0
+    with_memory_avg = result.avg_latency_ms
+    token_savings_estimate = result.hit_rate * 0.65  # replay/restore skips full generation
+
+    comparison = {
+        "without_memory_avg_latency_ms": estimated_baseline,
+        "with_memory_avg_latency_ms": round(with_memory_avg, 2),
+        "latency_reduction_pct": round(
+            max(0.0, (1 - with_memory_avg / estimated_baseline) * 100) if estimated_baseline else 0.0,
+            1,
+        ),
+        "estimated_token_savings_pct": round(token_savings_estimate * 100, 1),
+        "memory_hit_pct": round(result.hit_rate * 100, 1),
+    }
+
+    return result, comparison
+
+
+def format_benchmark_report(result: BenchmarkResult, comparison: dict) -> str:
+    lines = [
+        "Agent Memory Benchmark",
+        "========================",
+        f"Queries:           {result.queries}",
+        f"Replay rate:       {result.replay_rate:.1%}",
+        f"Restore rate:      {result.restore_rate:.1%}",
+        f"Verify rate:       {result.verify_rate:.1%}",
+        f"Memory hit rate:   {result.hit_rate:.1%}",
+        f"Avg latency:       {result.avg_latency_ms:.2f} ms",
+        f"P95 latency:       {result.p95_latency_ms:.2f} ms",
+        "",
+        "Without memory vs Agent Memory",
+        "------------------------------",
+        f"Without memory:    {comparison['without_memory_avg_latency_ms']:.0f} ms (estimated)",
+        f"With agent-memory: {comparison['with_memory_avg_latency_ms']:.2f} ms",
+        f"Latency reduction: {comparison['latency_reduction_pct']:.1f}%",
+        f"Est. token savings:{comparison['estimated_token_savings_pct']:.1f}%",
+        f"Memory hit rate:   {comparison['memory_hit_pct']:.1f}%",
+    ]
+    return "\n".join(lines)

agent_memory/cli.py

@@ -0,0 +1,168 @@
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+
+from agent_memory import Memory
+from agent_memory.benchmark import format_benchmark_report, run_benchmark
+from agent_memory.eval import EvalDataset, format_eval_report, run_eval_suite, seed_dataset
+
+
+def _default_datasets_dir() -> Path:
+    return Path(__file__).resolve().parent.parent / "benchmarks" / "datasets"
+
+
+def _create_memory(args: argparse.Namespace) -> Memory:
+    return Memory(persist_dir=args.data_dir, backend=args.backend)
+
+
+def cmd_remember(args: argparse.Namespace) -> int:
+    memory = _create_memory(args)
+    entry = memory.remember(
+        args.query,
+        args.response,
+        type=args.type,
+        scope=args.scope,
+        ttl=args.ttl,
+    )
+    print(f"Stored memory {entry.id}")
+    return 0
+
+
+def cmd_resolve(args: argparse.Namespace) -> int:
+    memory = _create_memory(args)
+    decision = memory.resolve(args.query)
+    print(decision)
+    if args.explain:
+        print()
+        print(decision.explain())
+    return 0
+
+
+def cmd_stats(args: argparse.Namespace) -> int:
+    memory = _create_memory(args)
+    stats = memory.stats()
+    print("Agent Memory Stats")
+    print("==================")
+    for key, value in stats.items():
+        print(f"{key}: {value}")
+    return 0
+
+
+def cmd_cleanup(args: argparse.Namespace) -> int:
+    memory = _create_memory(args)
+    result = memory.cleanup(delete=args.delete)
+    print(result)
+    return 0
+
+
+def cmd_benchmark(args: argparse.Namespace) -> int:
+    memory = _create_memory(args)
+
+    if args.seed:
+        for dataset_path in _default_datasets_dir().glob("*.json"):
+            dataset = EvalDataset.load(dataset_path)
+            seed_dataset(memory, dataset)
+            for case in dataset.cases:
+                args.queries.extend([case.query] * max(1, args.repeat))
+
+    queries = args.queries or [
+        "How do I reset my password?",
+        "I forgot my password",
+        "What is the API rate limit?",
+        "What's the weather today?",
+    ]
+
+    if args.repeat > 1 and not args.seed:
+        queries = [q for q in queries for _ in range(args.repeat)]
+
+    result, comparison = run_benchmark(memory, queries, baseline_no_memory_ms=args.baseline_ms)
+    print(format_benchmark_report(result, comparison))
+    return 0
+
+
+def cmd_eval(args: argparse.Namespace) -> int:
+    memory = Memory(persist_dir=args.data_dir)
+    datasets_dir = Path(args.datasets) if args.datasets else _default_datasets_dir()
+
+    if not datasets_dir.exists():
+        print(f"Datasets directory not found: {datasets_dir}", file=sys.stderr)
+        return 1
+
+    paths = sorted(datasets_dir.glob("*.json"))
+    if not paths:
+        print(f"No datasets found in {datasets_dir}", file=sys.stderr)
+        return 1
+
+    results = run_eval_suite(memory, paths)
+    print(format_eval_report(results))
+    return 0
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(prog="agent-memory", description="Agent Memory CLI")
+    parser.add_argument(
+        "--data-dir",
+        default=".agent_memory",
+        help="Persistence directory (default: .agent_memory)",
+    )
+
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    def add_common_args(subparser: argparse.ArgumentParser) -> None:
+        subparser.add_argument(
+            "--backend",
+            choices=["sqlite", "chromadb"],
+            default="sqlite",
+            help="Storage backend (default: sqlite)",
+        )
+
+    remember = sub.add_parser("remember", help="Store a memory")
+    remember.add_argument("query")
+    remember.add_argument("response")
+    remember.add_argument("--type", default="conversation")
+    remember.add_argument("--scope", default="user")
+    remember.add_argument("--ttl", default=None)
+    add_common_args(remember)
+    remember.set_defaults(func=cmd_remember)
+
+    resolve = sub.add_parser("resolve", help="Resolve a query against memory")
+    resolve.add_argument("query")
+    resolve.add_argument("--explain", action="store_true", help="Print score breakdown")
+    add_common_args(resolve)
+    resolve.set_defaults(func=cmd_resolve)
+
+    stats = sub.add_parser("stats", help="Show memory statistics")
+    add_common_args(stats)
+    stats.set_defaults(func=cmd_stats)
+
+    cleanup = sub.add_parser("cleanup", help="Expire or delete stale memories")
+    cleanup.add_argument("--delete", action="store_true", help="Delete expired memories")
+    add_common_args(cleanup)
+    cleanup.set_defaults(func=cmd_cleanup)
+
+    benchmark = sub.add_parser("benchmark", help="Run latency and hit-rate benchmark")
+    benchmark.add_argument("queries", nargs="*", help="Queries to benchmark")
+    benchmark.add_argument("--seed", action="store_true", help="Seed from eval datasets first")
+    benchmark.add_argument("--repeat", type=int, default=1, help="Repeat each query N times")
+    benchmark.add_argument("--baseline-ms", type=float, default=800.0, help="No-memory baseline ms")
+    add_common_args(benchmark)
+    benchmark.set_defaults(func=cmd_benchmark)
+
+    eval_cmd = sub.add_parser("eval", help="Run evaluation datasets")
+    eval_cmd.add_argument("--datasets", default=None, help="Path to datasets directory")
+    add_common_args(eval_cmd)
+    eval_cmd.set_defaults(func=cmd_eval)
+
+    return parser
+
+
+def main() -> None:
+    parser = build_parser()
+    args = parser.parse_args()
+    raise SystemExit(args.func(args))
+
+
+if __name__ == "__main__":
+    main()

agent_memory/decision.py

@@ -0,0 +1,161 @@
+from __future__ import annotations
+
+from agent_memory.explain import enrich_decision
+from agent_memory.models import MemoryAction, MemoryDecision, MemoryScope, RetrievalResult
+from agent_memory.policy import DecisionPolicy, DefaultPolicy
+from agent_memory.retriever import MemoryRetriever
+
+
+class DecisionEngine:
+    """
+    Decides whether the agent should replay, restore, verify, or ignore memory.
+    """
+
+    def __init__(
+        self,
+        retriever: MemoryRetriever,
+        policy: DecisionPolicy | None = None,
+        replay_threshold: float = 0.85,
+        restore_threshold: float = 0.70,
+        verify_threshold: float = 0.80,
+    ) -> None:
+        self._retriever = retriever
+        self._policy = policy or DefaultPolicy()
+        self.replay_threshold = replay_threshold
+        self.restore_threshold = restore_threshold
+        self.verify_threshold = verify_threshold
+
+    def decide(
+        self,
+        query: str,
+        *,
+        mode: str = "auto",
+        top_k: int = 3,
+        scopes: list[MemoryScope] | None = None,
+        enable_verify: bool = True,
+    ) -> MemoryDecision:
+        results = self._retriever.retrieve(query, top_k=top_k, scopes=scopes)
+        if not results:
+            decision = MemoryDecision(
+                action=MemoryAction.NONE,
+                query=query,
+                confidence=0.0,
+                reason="No memories stored yet.",
+                reasons=["no memories stored"],
+            )
+            return decision
+
+        if mode == "replay":
+            decision = self._decide_replay(query, results[0])
+        elif mode == "restore":
+            decision = self._decide_restore(query, results)
+        elif mode == "verify":
+            decision = self._decide_verify(query, results)
+        else:
+            decision = self._decide_auto(query, results, enable_verify=enable_verify)
+
+        return enrich_decision(
+            decision,
+            self._policy,
+            replay_threshold=self.replay_threshold,
+            restore_threshold=self.restore_threshold,
+        )
+
+    def _decide_auto(
+        self,
+        query: str,
+        results: list[RetrievalResult],
+        *,
+        enable_verify: bool,
+    ) -> MemoryDecision:
+        action, confidence, reason = self._policy.select_action(
+            results,
+            replay_threshold=self.replay_threshold,
+            restore_threshold=self.restore_threshold,
+            verify_threshold=self.verify_threshold if enable_verify else 0.0,
+        )
+
+        if action == MemoryAction.REPLAY:
+            return self._finalize_replay(query, results[0], reason)
+        if action == MemoryAction.VERIFY and enable_verify:
+            return self._finalize_verify(query, results[0], reason)
+        if action == MemoryAction.RESTORE:
+            return MemoryDecision(
+                action=MemoryAction.RESTORE,
+                query=query,
+                confidence=confidence,
+                context=results,
+                reason=reason,
+            )
+
+        return MemoryDecision(
+            action=MemoryAction.NONE,
+            query=query,
+            confidence=confidence,
+            context=results,
+            reason=reason,
+        )
+
+    def _decide_replay(self, query: str, best: RetrievalResult) -> MemoryDecision:
+        if best.decision_score >= self.replay_threshold:
+            return self._finalize_replay(query, best, "Replay mode — match found.")
+        return MemoryDecision(
+            action=MemoryAction.NONE,
+            query=query,
+            confidence=best.decision_score,
+            context=[best],
+            reason="Replay mode — no match above replay threshold.",
+        )
+
+    def _decide_restore(self, query: str, results: list[RetrievalResult]) -> MemoryDecision:
+        best = results[0]
+        if best.decision_score >= self.restore_threshold:
+            return MemoryDecision(
+                action=MemoryAction.RESTORE,
+                query=query,
+                confidence=best.decision_score,
+                context=results,
+                reason="Restore mode — returning relevant memories as context.",
+            )
+        return MemoryDecision(
+            action=MemoryAction.NONE,
+            query=query,
+            confidence=best.decision_score,
+            context=results,
+            reason="Restore mode — no match above restore threshold.",
+        )
+
+    def _decide_verify(self, query: str, results: list[RetrievalResult]) -> MemoryDecision:
+        best = results[0]
+        if best.decision_score >= self.restore_threshold:
+            return self._finalize_verify(query, best, "Verify mode — memory requires validation.")
+        return MemoryDecision(
+            action=MemoryAction.NONE,
+            query=query,
+            confidence=best.decision_score,
+            context=results,
+            reason="Verify mode — no match above restore threshold.",
+        )
+
+    def _finalize_replay(self, query: str, best: RetrievalResult, reason: str) -> MemoryDecision:
+        updated = self._retriever.record_access(best.entry)
+        best.entry = updated
+        return MemoryDecision(
+            action=MemoryAction.REPLAY,
+            query=query,
+            confidence=best.decision_score,
+            response=updated.response,
+            memory=updated,
+            context=[best],
+            reason=reason,
+        )
+
+    def _finalize_verify(self, query: str, best: RetrievalResult, reason: str) -> MemoryDecision:
+        return MemoryDecision(
+            action=MemoryAction.VERIFY,
+            query=query,
+            confidence=best.decision_score,
+            memory=best.entry,
+            context=[best],
+            reason=reason,
+        )

agent_memory/eval.py

@@ -0,0 +1,132 @@
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from agent_memory.manager import Memory
+
+
+@dataclass
+class EvalCase:
+    query: str
+    expected_action: str
+    notes: str = ""
+
+
+@dataclass
+class EvalDataset:
+    name: str
+    description: str = ""
+    memories: list[dict] = field(default_factory=list)
+    cases: list[EvalCase] = field(default_factory=list)
+
+    @classmethod
+    def load(cls, path: str | Path) -> EvalDataset:
+        data = json.loads(Path(path).read_text(encoding="utf-8"))
+        cases = [
+            EvalCase(
+                query=c["query"],
+                expected_action=c["expected_action"],
+                notes=c.get("notes", ""),
+            )
+            for c in data.get("cases", [])
+        ]
+        return cls(
+            name=data.get("name", Path(path).stem),
+            description=data.get("description", ""),
+            memories=data.get("memories", []),
+            cases=cases,
+        )
+
+
+@dataclass
+class EvalResult:
+    dataset: str
+    total: int = 0
+    correct: int = 0
+    failures: list[dict] = field(default_factory=list)
+
+    @property
+    def precision(self) -> float:
+        return self.correct / self.total if self.total else 0.0
+
+    def to_dict(self) -> dict:
+        return {
+            "dataset": self.dataset,
+            "total": self.total,
+            "correct": self.correct,
+            "precision": round(self.precision, 4),
+            "failures": self.failures,
+        }
+
+
+def seed_dataset(memory: Memory, dataset: EvalDataset) -> None:
+    for item in dataset.memories:
+        memory.remember(
+            query=item["query"],
+            response=item["response"],
+            type=item.get("type", "conversation"),
+            scope=item.get("scope", "user"),
+            tags=item.get("tags", []),
+            ttl=item.get("ttl"),
+        )
+
+
+def run_eval(memory: Memory, dataset: EvalDataset) -> EvalResult:
+    result = EvalResult(dataset=dataset.name)
+
+    for case in dataset.cases:
+        decision = memory.resolve(case.query)
+        result.total += 1
+        actual = decision.action.value
+
+        # Allow flexible matching: restore accepts replay; verify accepts restore/replay
+        acceptable = {case.expected_action}
+        if case.expected_action == "restore":
+            acceptable |= {"replay"}
+        if case.expected_action == "replay":
+            acceptable |= {"restore"}
+        if case.expected_action == "verify":
+            acceptable |= {"restore", "replay"}
+
+        if actual in acceptable:
+            result.correct += 1
+        else:
+            result.failures.append(
+                {
+                    "query": case.query,
+                    "expected": case.expected_action,
+                    "actual": actual,
+                    "confidence": round(decision.confidence, 4),
+                    "reasons": decision.reasons,
+                }
+            )
+
+    return result
+
+
+def run_eval_suite(memory: Memory, dataset_paths: list[Path]) -> list[EvalResult]:
+    results: list[EvalResult] = []
+    for path in dataset_paths:
+        dataset = EvalDataset.load(path)
+        seed_dataset(memory, dataset)
+        results.append(run_eval(memory, dataset))
+    return results
+
+
+def format_eval_report(results: list[EvalResult]) -> str:
+    lines = ["Agent Memory Evaluation", "=======================", ""]
+    for result in results:
+        lines.append(f"Dataset: {result.dataset}")
+        lines.append(f"  Cases:     {result.total}")
+        lines.append(f"  Correct:   {result.correct}")
+        lines.append(f"  Precision: {result.precision:.1%}")
+        if result.failures:
+            lines.append(f"  Failures:  {len(result.failures)}")
+        lines.append("")
+    overall_total = sum(r.total for r in results)
+    overall_correct = sum(r.correct for r in results)
+    if overall_total:
+        lines.append(f"Overall precision: {overall_correct / overall_total:.1%}")
+    return "\n".join(lines)