dandelion-memory 1.0.0rc15__py3-none-any.whl

benchmarks/__init__.py

@@ -0,0 +1,3 @@
+"""DandelionBench — Independent Evaluation Suite for Agent Memory Architectures."""
+
+__version__ = "1.0.0-rc1"

benchmarks/adapters/__init__.py

@@ -0,0 +1,29 @@
+"""Adapters module for DandelionBench."""
+
+from benchmarks.adapters.base import BaseMemoryAdapter
+from benchmarks.adapters.dandelion import DandelionAdapter
+from benchmarks.adapters.ablations import (
+    DenseOnlyAdapter,
+    NoSnapshotAdapter,
+    NoReverseAdapter,
+    NoLexicalAdapter,
+)
+from benchmarks.adapters.controlled import (
+    HybridRAGAdapter,
+    ExtractResolveAdapter,
+    CoreArchivalAdapter,
+    NoLateBindingAdapter,
+)
+
+__all__ = [
+    "BaseMemoryAdapter",
+    "DandelionAdapter",
+    "DenseOnlyAdapter",
+    "NoSnapshotAdapter",
+    "NoReverseAdapter",
+    "NoLexicalAdapter",
+    "HybridRAGAdapter",
+    "ExtractResolveAdapter",
+    "CoreArchivalAdapter",
+    "NoLateBindingAdapter",
+]

benchmarks/adapters/ablations.py

@@ -0,0 +1,406 @@
+"""Ablations and baseline adapters for DandelionBench."""
+
+import re
+from typing import Any, Optional
+import dandelion.store_kv
+from dandelion.store_kv import KVStore
+from dandelion.store_vector import VectorStore
+from dandelion.hydrator import hydrate_template
+from benchmarks.adapters.base import BaseMemoryAdapter
+
+
+class DenseOnlyAdapter(BaseMemoryAdapter):
+    """Standard Dense Vector Memory baseline (e.g. naive vector DB / Mem0 / LangChain RAG).
+
+    Characteristics:
+    - Pure vector embeddings (FastEmbed cosine similarity).
+    - Eager binding: facts are inlined directly into text at ingestion time.
+    - No symbolic late-binding: when facts mutate, stored memories remain stale.
+    - No reverse reference index: cannot trace fact values back to referencing memories.
+    - No secret redaction policy: sensitive tokens are stored and recalled in plaintext.
+    - No identifier / BM25 lexical boost.
+    """
+
+    def __init__(self) -> None:
+        self.facts: dict[str, str] = {}
+        self.vec: Optional[VectorStore] = None
+        self.reset()
+
+    @property
+    def name(self) -> str:
+        return "dense_only"
+
+    def reset(self) -> None:
+        self.facts.clear()
+        self.vec = VectorStore(in_memory=True)
+        # Clear global KV store to prevent state leakage from prior adapters
+        import dandelion.store_kv
+        dandelion.store_kv._store = None
+
+    def ingest_fact(
+        self,
+        key: str,
+        value: str,
+        timestamp: float,
+        domain: str = "general",
+        source: str = "config_read",
+    ) -> None:
+        self.facts[key] = value
+        # Also index fact as plain passage in vector store
+        fact_sentence = f"Configuration fact '{key}' is set to '{value}' in domain '{domain}'."
+        self.vec.save_chunk(
+            domain=domain,
+            template=fact_sentence,
+            source=source,
+            created_at=timestamp,
+        )
+
+    def ingest_memory(
+        self,
+        event_id: str,
+        template: str,
+        domain: str,
+        timestamp: float,
+        supersedes: Optional[str] = None,
+        source: str = "tool_observed",
+    ) -> str:
+        # Eager resolution at store time: inlines current fact value directly
+        def _resolve_eager(m: re.Match) -> str:
+            raw_key = m.group(1).split("@rev:")[0]
+            # Try domain-qualified first
+            if domain:
+                qual_key = f"{domain}.{raw_key}"
+                if qual_key in self.facts:
+                    return self.facts[qual_key]
+            return self.facts.get(raw_key, f"[MISSING: {raw_key}]")
+
+        inlined = re.sub(r"\{\{(?:ref|snapshot):([\w.:\-]+)\}\}", _resolve_eager, template)
+        if domain:
+            inlined = inlined.replace("{{domain}}", domain)
+
+        return self.vec.save_chunk(
+            domain=domain,
+            template=inlined,
+            supersedes=supersedes,
+            source=source,
+            created_at=timestamp,
+            event_id=event_id,
+        )
+
+    def recall(
+        self,
+        query: str,
+        domain: Optional[str] = None,
+        limit: int = 3,
+    ) -> list[dict[str, Any]]:
+        domains = [domain] if domain and not domain.endswith(".*") else None
+        # Dense vector search only: no lexical rerank, no reverse index
+        raw_results = self.vec.search_chunks(
+            query=query,
+            domains=domains,
+            limit=limit,
+            use_reverse_index=False,
+            use_lexical_rerank=False,
+        )
+
+        out = []
+        for r in raw_results:
+            out.append({
+                "id": r["id"],
+                "event_id": r.get("event_id", r["id"]),
+                "text": r["template"],
+                "template": r["template"],
+                "domain": r["domain"],
+                "score": r["score"],
+            })
+        return out
+
+    def get_fact(
+        self,
+        key: str,
+        domain: Optional[str] = None,
+        at: Optional[float] = None,
+    ) -> Optional[str]:
+        if domain:
+            scoped = f"{domain}.{key}"
+            if scoped in self.facts:
+                return self.facts[scoped]
+        return self.facts.get(key)
+
+
+class NoSnapshotAdapter(BaseMemoryAdapter):
+    """Ablation: Dandelion without Snapshot Pinning.
+
+    All {{snapshot:key}} references are treated as live {{ref:key}}.
+    When a fact mutates, historical memories also resolve to the new value,
+    violating temporal fidelity (tests Hypothesis H2).
+    """
+
+    def __init__(self) -> None:
+        self.kv: Optional[KVStore] = None
+        self.vec: Optional[VectorStore] = None
+        self.reset()
+
+    @property
+    def name(self) -> str:
+        return "no_snapshot"
+
+    def reset(self) -> None:
+        self.kv = KVStore(path=None)
+        dandelion.store_kv._store = self.kv
+        self.vec = VectorStore(in_memory=True)
+
+    def ingest_fact(
+        self,
+        key: str,
+        value: str,
+        timestamp: float,
+        domain: str = "general",
+        source: str = "config_read",
+    ) -> None:
+        self.kv.set_fact(key=key, value=value, source=source, timestamp=timestamp)
+
+    def ingest_memory(
+        self,
+        event_id: str,
+        template: str,
+        domain: str,
+        timestamp: float,
+        supersedes: Optional[str] = None,
+        source: str = "tool_observed",
+    ) -> str:
+        # Intentionally convert snapshot to live ref instead of pinning
+        unpinned = re.sub(r"\{\{snapshot:([\w.:\-]+)\}\}", r"{{ref:\1}}", template)
+        return self.vec.save_chunk(
+            domain=domain,
+            template=unpinned,
+            supersedes=supersedes,
+            source=source,
+            created_at=timestamp,
+            event_id=event_id,
+        )
+
+    def recall(
+        self,
+        query: str,
+        domain: Optional[str] = None,
+        limit: int = 3,
+    ) -> list[dict[str, Any]]:
+        domains = [domain] if domain and not domain.endswith(".*") else None
+        raw_results = self.vec.search_chunks(query=query, domains=domains, limit=limit)
+
+        out = []
+        for r in raw_results:
+            hydrated = hydrate_template(
+                r["template"],
+                kv_resolver=self.kv.resolve_ref,
+                chunk_domain=r["domain"],
+                allow_secrets=False,
+            )
+            out.append({
+                "id": r["id"],
+                "event_id": r.get("event_id", r["id"]),
+                "text": hydrated,
+                "template": r["template"],
+                "domain": r["domain"],
+                "score": r["score"],
+            })
+        return out
+
+    def get_fact(
+        self,
+        key: str,
+        domain: Optional[str] = None,
+        at: Optional[float] = None,
+    ) -> Optional[str]:
+        if at is not None:
+            return self.kv.get_fact(key, at_timestamp=at)
+        return self.kv.resolve_ref(key, domain=domain)
+
+
+class NoReverseAdapter(BaseMemoryAdapter):
+    """Ablation: Dandelion without Fact Reverse-Lookup Index.
+
+    Querying for fact values (e.g. '5433') relies solely on dense/lexical
+    matching without checking which chunks reference that fact key.
+    """
+
+    def __init__(self) -> None:
+        self.kv: Optional[KVStore] = None
+        self.vec: Optional[VectorStore] = None
+        self.reset()
+
+    @property
+    def name(self) -> str:
+        return "no_reverse"
+
+    def reset(self) -> None:
+        self.kv = KVStore(path=None)
+        dandelion.store_kv._store = self.kv
+        self.vec = VectorStore(in_memory=True)
+
+    def ingest_fact(
+        self,
+        key: str,
+        value: str,
+        timestamp: float,
+        domain: str = "general",
+        source: str = "config_read",
+    ) -> None:
+        self.kv.set_fact(key=key, value=value, source=source, timestamp=timestamp)
+
+    def ingest_memory(
+        self,
+        event_id: str,
+        template: str,
+        domain: str,
+        timestamp: float,
+        supersedes: Optional[str] = None,
+        source: str = "tool_observed",
+    ) -> str:
+        from dandelion.hydrator import pin_snapshots
+        pinned = pin_snapshots(template, self.kv.get_current_revision, domain)
+        return self.vec.save_chunk(
+            domain=domain,
+            template=pinned,
+            supersedes=supersedes,
+            source=source,
+            created_at=timestamp,
+            event_id=event_id,
+        )
+
+    def recall(
+        self,
+        query: str,
+        domain: Optional[str] = None,
+        limit: int = 3,
+    ) -> list[dict[str, Any]]:
+        domains = [domain] if domain and not domain.endswith(".*") else None
+        # Explicitly disable reverse index in hybrid search
+        raw_results = self.vec.search_chunks(
+            query=query,
+            domains=domains,
+            limit=limit,
+            use_reverse_index=False,
+            use_lexical_rerank=True,
+        )
+
+        out = []
+        for r in raw_results:
+            hydrated = hydrate_template(
+                r["template"],
+                kv_resolver=self.kv.resolve_ref,
+                chunk_domain=r["domain"],
+                allow_secrets=False,
+            )
+            out.append({
+                "id": r["id"],
+                "event_id": r.get("event_id", r["id"]),
+                "text": hydrated,
+                "template": r["template"],
+                "domain": r["domain"],
+                "score": r["score"],
+            })
+        return out
+
+    def get_fact(
+        self,
+        key: str,
+        domain: Optional[str] = None,
+        at: Optional[float] = None,
+    ) -> Optional[str]:
+        if at is not None:
+            return self.kv.get_fact(key, at_timestamp=at)
+        return self.kv.resolve_ref(key, domain=domain)
+
+
+class NoLexicalAdapter(BaseMemoryAdapter):
+    """Ablation: Dandelion without BM25 / Lexical Identifier Matching."""
+
+    def __init__(self) -> None:
+        self.kv: Optional[KVStore] = None
+        self.vec: Optional[VectorStore] = None
+        self.reset()
+
+    @property
+    def name(self) -> str:
+        return "no_lexical"
+
+    def reset(self) -> None:
+        self.kv = KVStore(path=None)
+        dandelion.store_kv._store = self.kv
+        self.vec = VectorStore(in_memory=True)
+
+    def ingest_fact(
+        self,
+        key: str,
+        value: str,
+        timestamp: float,
+        domain: str = "general",
+        source: str = "config_read",
+    ) -> None:
+        self.kv.set_fact(key=key, value=value, source=source, timestamp=timestamp)
+
+    def ingest_memory(
+        self,
+        event_id: str,
+        template: str,
+        domain: str,
+        timestamp: float,
+        supersedes: Optional[str] = None,
+        source: str = "tool_observed",
+    ) -> str:
+        from dandelion.hydrator import pin_snapshots
+        pinned = pin_snapshots(template, self.kv.get_current_revision, domain)
+        return self.vec.save_chunk(
+            domain=domain,
+            template=pinned,
+            supersedes=supersedes,
+            source=source,
+            created_at=timestamp,
+            event_id=event_id,
+        )
+
+    def recall(
+        self,
+        query: str,
+        domain: Optional[str] = None,
+        limit: int = 3,
+    ) -> list[dict[str, Any]]:
+        domains = [domain] if domain and not domain.endswith(".*") else None
+        # Explicitly disable lexical reranking
+        raw_results = self.vec.search_chunks(
+            query=query,
+            domains=domains,
+            limit=limit,
+            use_reverse_index=True,
+            use_lexical_rerank=False,
+        )
+
+        out = []
+        for r in raw_results:
+            hydrated = hydrate_template(
+                r["template"],
+                kv_resolver=self.kv.resolve_ref,
+                chunk_domain=r["domain"],
+                allow_secrets=False,
+            )
+            out.append({
+                "id": r["id"],
+                "event_id": r.get("event_id", r["id"]),
+                "text": hydrated,
+                "template": r["template"],
+                "domain": r["domain"],
+                "score": r["score"],
+            })
+        return out
+
+    def get_fact(
+        self,
+        key: str,
+        domain: Optional[str] = None,
+        at: Optional[float] = None,
+    ) -> Optional[str]:
+        if at is not None:
+            return self.kv.get_fact(key, at_timestamp=at)
+        return self.kv.resolve_ref(key, domain=domain)

benchmarks/adapters/base.py

@@ -0,0 +1,72 @@
+"""Base interface for agent memory benchmark adapters."""
+
+from abc import ABC, abstractmethod
+from typing import Any, Optional
+
+
+class BaseMemoryAdapter(ABC):
+    """Abstract interface for memory systems under evaluation."""
+
+    @property
+    @abstractmethod
+    def name(self) -> str:
+        """System or ablation name (e.g. 'dandelion', 'dense_only')."""
+        pass
+
+    @abstractmethod
+    def reset(self) -> None:
+        """Clear all internal memory stores."""
+        pass
+
+    @abstractmethod
+    def ingest_fact(
+        self,
+        key: str,
+        value: str,
+        timestamp: float,
+        domain: str = "general",
+        source: str = "config_read",
+    ) -> None:
+        """Store or mutate a deterministic fact."""
+        pass
+
+    @abstractmethod
+    def ingest_memory(
+        self,
+        event_id: str,
+        template: str,
+        domain: str,
+        timestamp: float,
+        supersedes: Optional[str] = None,
+        source: str = "tool_observed",
+    ) -> str:
+        """Store an episodic/semantic memory event."""
+        pass
+
+    @abstractmethod
+    def recall(
+        self,
+        query: str,
+        domain: Optional[str] = None,
+        limit: int = 3,
+    ) -> list[dict[str, Any]]:
+        """Retrieve top candidate memories for query.
+
+        Each returned dict MUST contain at least:
+            - 'id': str
+            - 'event_id': str
+            - 'text': str (hydrated or inlined content shown to agent)
+            - 'domain': str
+            - 'score': float
+        """
+        pass
+
+    @abstractmethod
+    def get_fact(
+        self,
+        key: str,
+        domain: Optional[str] = None,
+        at: Optional[float] = None,
+    ) -> Optional[str]:
+        """Directly query a deterministic fact."""
+        pass

benchmarks/adapters/controlled/__init__.py

@@ -0,0 +1,13 @@
+"""Controlled architectural baseline adapters for DandelionBench Track A."""
+
+from benchmarks.adapters.controlled.hybrid_rag import HybridRAGAdapter
+from benchmarks.adapters.controlled.extract_resolve import ExtractResolveAdapter
+from benchmarks.adapters.controlled.core_archival import CoreArchivalAdapter
+from benchmarks.adapters.controlled.no_late_binding import NoLateBindingAdapter
+
+__all__ = [
+    "HybridRAGAdapter",
+    "ExtractResolveAdapter",
+    "CoreArchivalAdapter",
+    "NoLateBindingAdapter",
+]

benchmarks/adapters/controlled/core_archival.py

@@ -0,0 +1,115 @@
+"""Controlled Architectural Baseline: Core + Archival Memory (MemGPT / Letta paradigm).
+
+Characteristics:
+- Two-tier memory hierarchy:
+  1. In-context Core Memory (structured blocks, editable in place, always visible).
+  2. Long-term Archival Memory (dense vector search over past event logs).
+- Facts immediately update Core Memory blocks.
+- Episodic memories are stored into Archival Memory as static passages.
+- Evaluates the effect of in-context core state versus late-bound symbolic hydration.
+"""
+
+import re
+from typing import Any, Optional
+from dandelion.store_vector import VectorStore
+from benchmarks.adapters.base import BaseMemoryAdapter
+
+
+class CoreArchivalAdapter(BaseMemoryAdapter):
+    """Core + Archival memory baseline representing two-tier agent memory systems."""
+
+    def __init__(self) -> None:
+        self.core_memory: dict[str, str] = {}
+        self.archival_vec: Optional[VectorStore] = None
+        self.reset()
+
+    @property
+    def name(self) -> str:
+        return "core_archival"
+
+    def reset(self) -> None:
+        self.core_memory.clear()
+        self.archival_vec = VectorStore(in_memory=True)
+        import dandelion.store_kv
+        dandelion.store_kv._store = None
+
+    def ingest_fact(
+        self,
+        key: str,
+        value: str,
+        timestamp: float,
+        domain: str = "general",
+        source: str = "config_read",
+    ) -> None:
+        # Core Memory update (in-place state edit)
+        self.core_memory[key] = value
+
+    def ingest_memory(
+        self,
+        event_id: str,
+        template: str,
+        domain: str,
+        timestamp: float,
+        supersedes: Optional[str] = None,
+        source: str = "tool_observed",
+    ) -> str:
+        # Archival memory stores static passages resolved against current Core Memory
+        def _resolve_core(m: re.Match) -> str:
+            raw_key = m.group(1).split("@rev:")[0]
+            if domain:
+                qual_key = f"{domain}.{raw_key}"
+                if qual_key in self.core_memory:
+                    return self.core_memory[qual_key]
+            return self.core_memory.get(raw_key, f"[MISSING: {raw_key}]")
+
+        inlined = re.sub(r"\{\{(?:ref|snapshot):([\w.:\-]+)\}\}", _resolve_core, template)
+        if domain:
+            inlined = inlined.replace("{{domain}}", domain)
+
+        return self.archival_vec.save_chunk(
+            domain=domain,
+            template=inlined,
+            supersedes=supersedes,
+            source=source,
+            created_at=timestamp,
+            event_id=event_id,
+        )
+
+    def recall(
+        self,
+        query: str,
+        domain: Optional[str] = None,
+        limit: int = 3,
+    ) -> list[dict[str, Any]]:
+        domains = [domain] if domain and not domain.endswith(".*") else None
+        # Archival vector search
+        raw_results = self.archival_vec.search_chunks(
+            query=query,
+            domains=domains,
+            limit=limit,
+            use_reverse_index=False,
+            use_lexical_rerank=False,
+        )
+        return [
+            {
+                "id": r["id"],
+                "event_id": r.get("event_id", ""),
+                "text": r["template"],
+                "domain": r["domain"],
+                "score": r.get("score", 0.0),
+                "vector_score": r.get("vector_score", 0.0),
+            }
+            for r in raw_results
+        ]
+
+    def get_fact(
+        self,
+        key: str,
+        domain: Optional[str] = None,
+        at: Optional[float] = None,
+    ) -> Optional[str]:
+        if domain:
+            qual_key = f"{domain}.{key}"
+            if qual_key in self.core_memory:
+                return self.core_memory[qual_key]
+        return self.core_memory.get(key)