etchmem 0.1.0__py3-none-any.whl

etchmem/engine.py

@@ -0,0 +1,191 @@
+"""
+Engine — the public entry point.
+
+Wires together:
+  - three ChromaStore instances (relational, buffer, injected)
+  - one RemoteConnector for LLM synthesis
+  - Recaller (recall logic + recall-event emission)
+  - Worker (consolidation pipeline)
+
+Exposes exactly three public methods:
+  remember(data, hint, skill, metadata)   → None
+  recall(query, skill, top_k, hint)       → list[SearchResult]
+  consolidate(num_records, method)        → dict (summary)
+"""
+from __future__ import annotations
+
+import time
+from typing import Any
+
+from etchmem.config import Config
+from etchmem.models import Record, SearchResult
+from etchmem.stores.chroma_store import build_stores
+from etchmem.util.hashing import content_hash
+
+
+class Engine:
+    """
+    The skillmem engine.
+
+    Usage::
+
+        from etchmem import Engine
+
+        engine = Engine()                      # uses .skillmem/ in cwd
+        engine.remember("Python 3.14 ships with a new JIT by default")
+        results = engine.recall("what changed in Python 3.14?")
+        summary = engine.consolidate()
+    """
+
+    def __init__(
+        self,
+        config: Config | None = None,
+        connector=None,   # LLMConnector | None; lazy-built if None
+    ) -> None:
+        from etchmem.config import DEFAULT_CONFIG
+        self._cfg = config or DEFAULT_CONFIG
+
+        # Build stores
+        self._relational, self._buffer, self._injected = build_stores(
+            data_dir=self._cfg.data_dir,
+            collection_relational=self._cfg.collection_relational,
+            collection_buffer=self._cfg.collection_buffer,
+            collection_injected=self._cfg.collection_injected,
+            ttl_seconds=self._cfg.relational_ttl_seconds,
+            embedding_function=self._cfg.embedding_function,
+        )
+
+        # Connector — built lazily on first consolidate() if not provided
+        self._connector = connector
+        self._connector_built = connector is not None
+
+        # Wires
+        from etchmem.recall.recaller import Recaller
+        self._recaller = Recaller(
+            relational_store=self._relational,
+            buffer_store=self._buffer,
+            injected_store=self._injected,
+            config=self._cfg,
+        )
+
+    # ── Public API ────────────────────────────────────────────────────────
+
+    def remember(
+        self,
+        data: str,
+        hint: float | None = None,
+        skill: str | None = None,
+        metadata: dict[str, Any] | None = None,
+    ) -> None:
+        """
+        Deposit a raw record into the relational collection.
+
+        No LLM call, no clustering. Cheap — embedding computed by Chroma
+        on insert.
+
+        Args:
+            data:     The text content to remember.
+            hint:     Optional importance prior (float 0–1). Seed, not truth.
+            skill:    Optional skill scope name (e.g. "summarizer").
+            metadata: Arbitrary dict (source URL, tags, etc.).
+        """
+        record = Record(
+            id=content_hash(data),
+            content=data,
+            skill=skill,
+            hint=hint,
+            metadata=metadata or {},
+            created_at=time.time(),
+        )
+        self._relational.add(record)
+
+    def recall(
+        self,
+        query: str,
+        skill: str | None = None,
+        top_k: int | None = None,
+        hint: float | None = None,
+    ) -> list[SearchResult]:
+        """
+        Retrieve knowledge AND emit a recall-event for future reconsolidation.
+
+        Two things happen:
+          1. Returns merged results from injected (primary) + relational (fresh).
+          2. Writes a RecallEvent into the buffer so the worker can rewrite
+             used knowledge against fresh context at the next consolidate().
+
+        Args:
+            query:  Natural language query.
+            skill:  Optional skill scope filter.
+            top_k:  Max results. Default from config.
+            hint:   Importance prior seeded into the recall-event.
+
+        Returns:
+            List of SearchResult, highest score first.
+        """
+        return self._recaller.recall(
+            query=query,
+            skill=skill,
+            top_k=top_k,
+            hint=hint,
+        )
+
+    def consolidate(
+        self,
+        num_records: int | str | None = None,
+        method: str | None = None,
+    ) -> dict[str, Any]:
+        """
+        Run the consolidation worker.
+
+        Processes the buffer: forms new knowledge, reconsolidates recalled
+        knowledge, hard-deletes superseded articles. Explicitly called in v0
+        (predictable, demoable, debuggable).
+
+        Args:
+            num_records: How many relational deposits to pull ("all" or int N).
+            method:      Ordering: "LIFO" (newest first, default) or "FIFO".
+
+        Returns:
+            Summary dict with counts of: formed, reconsolidated, dropped, kept,
+            flushed, superseded.
+        """
+        self._ensure_connector()
+
+        from etchmem.consolidate.worker import Worker
+        worker = Worker(
+            relational_store=self._relational,
+            buffer_store=self._buffer,
+            injected_store=self._injected,
+            connector=self._connector,
+            config=self._cfg,
+        )
+
+        num_records = num_records if num_records is not None else self._cfg.consolidate_default_num_records
+        method = method or self._cfg.consolidate_default_method
+
+        return worker.run(num_records=num_records, method=method)
+
+    # ── Internals ─────────────────────────────────────────────────────────
+
+    def _ensure_connector(self) -> None:
+        """Lazy-build the LLM connector on first consolidate() call."""
+        if self._connector_built:
+            return
+        from etchmem.connector.remote import build_connector
+        self._connector = build_connector(self._cfg)
+        self._connector_built = True
+
+    # ── Convenience properties ────────────────────────────────────────────
+
+    @property
+    def config(self) -> Config:
+        return self._cfg
+
+    def stats(self) -> dict[str, int]:
+        """Return collection sizes for monitoring."""
+        return {
+            "relational": self._relational.count(),
+            "buffer": self._buffer.count(),
+            "injected": self._injected.count(),
+        }

etchmem/models.py

@@ -0,0 +1,147 @@
+"""
+skillmem data models.
+
+All structs are plain dataclasses — no ORM, no Pydantic dependency.
+"""
+from __future__ import annotations
+
+import time
+from dataclasses import dataclass, field
+from typing import Any
+
+
+# ── Raw deposit written by remember() ────────────────────────────────────────
+
+@dataclass
+class Record:
+    """
+    A raw signal record deposited via remember().
+    Stored in the *relational* collection.
+    Immutable after creation — relational is append-only.
+    """
+    id: str                          # content hash — computed before insert
+    content: str
+    skill: str | None = None         # optional skill scope
+    hint: float | None = None        # agent-supplied importance prior 0–1
+    metadata: dict[str, Any] = field(default_factory=dict)
+    created_at: float = field(default_factory=time.time)
+    expires_at: float = 0.0          # unix timestamp; set by store on insert
+
+
+# ── Buffer documents ──────────────────────────────────────────────────────────
+
+@dataclass
+class Deposit:
+    """
+    A relational record copied into the *buffer* collection at intake.
+    Carries the original content hash for idempotency.
+    """
+    id: str                          # same content hash as the source Record
+    content: str
+    skill: str | None = None
+    hint: float | None = None
+    metadata: dict[str, Any] = field(default_factory=dict)
+    created_at: float = field(default_factory=time.time)
+    doc_type: str = "deposit"        # discriminator stored as metadata
+
+
+@dataclass
+class RecallEvent:
+    """
+    Emitted by recall() into the *buffer* collection.
+    Carries the query (pointer into injected) and a snapshot of relational
+    hits (perishable, so snapshotted here).
+    """
+    id: str                          # uuid / hash assigned at emit time
+    query: str                       # the original recall query
+    relational_snapshot: str         # concatenated text of relational hits
+    skill: str | None = None
+    hint: float | None = None
+    metadata: dict[str, Any] = field(default_factory=dict)
+    created_at: float = field(default_factory=time.time)
+    doc_type: str = "recall_event"   # discriminator stored as metadata
+
+
+# ── Clustering output ─────────────────────────────────────────────────────────
+
+@dataclass
+class Cluster:
+    """
+    A group of buffer documents (Deposits + RecallEvents) that cluster
+    around a shared topic. Output of Stage B clustering.
+    """
+    cluster_id: str
+    documents: list[Deposit | RecallEvent]
+    centroid_text: str | None = None   # representative text for the cluster
+
+    @property
+    def has_recall_events(self) -> bool:
+        return any(isinstance(d, RecallEvent) for d in self.documents)
+
+    @property
+    def deposits(self) -> list[Deposit]:
+        return [d for d in self.documents if isinstance(d, Deposit)]
+
+    @property
+    def recall_events(self) -> list[RecallEvent]:
+        return [d for d in self.documents if isinstance(d, RecallEvent)]
+
+    @property
+    def queries(self) -> list[str]:
+        return [re.query for re in self.recall_events]
+
+    @property
+    def skill(self) -> str | None:
+        """Dominant skill across all docs in the cluster (most common)."""
+        skills = [d.skill for d in self.documents if d.skill]
+        if not skills:
+            return None
+        return max(set(skills), key=skills.count)
+
+
+# ── Search / recall results ───────────────────────────────────────────────────
+
+@dataclass
+class SearchResult:
+    """
+    A single result returned by recall().
+    Carries the injected article content blended with freshness signal.
+    """
+    id: str
+    content: str
+    score: float                     # asymmetric blended score
+    source: str                      # "injected" | "relational" | "blended"
+    skill: str | None = None
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+
+# ── Injected article ──────────────────────────────────────────────────────────
+
+@dataclass
+class InjectedArticle:
+    """
+    A synthesized knowledge article stored in the *injected* collection.
+    Identity is keyed on the frozenset of source-doc hashes (content-addressed),
+    NOT on synthesized text (LLM is non-deterministic).
+    """
+    id: str                          # hash of sorted source-doc-hash set
+    content: str
+    source_hashes: frozenset[str]    # set of Deposit.id values used in synthesis
+    skill: str | None = None
+    tags: dict[str, str] = field(default_factory=dict)   # derived by synthesis prompt
+    metadata: dict[str, Any] = field(default_factory=dict)
+    created_at: float = field(default_factory=time.time)
+
+
+# ── Affectability ─────────────────────────────────────────────────────────────
+
+@dataclass
+class Affectability:
+    """
+    Per-record affectability score.
+    Seeded from the agent hint (or default 0.5), not yet accumulated.
+    Accumulation logic lives in util/affectability.py.
+    """
+    record_id: str
+    score: float                     # 0.0–1.0
+    hint_seeded: bool = False        # True if score originated from an agent hint

etchmem/recall/recaller.py

@@ -0,0 +1,154 @@
+"""
+Recaller — the recall() implementation.
+
+Two things happen on every recall() call:
+  1. Returns results (injected primary + relational freshness enrichment).
+  2. Emits a recall-event into the buffer (cheap append, no LLM).
+
+Design rules (from spec):
+  - Injected is ALWAYS searched directly with the query (never gated).
+  - Relational enriches; it never gates injected results.
+  - Asymmetric scoring: injected = high-trust base; relational = freshness-
+    boosted, confidence-discounted.
+  - Degrades gracefully to injected-only when relational is empty.
+  - The recall-event stores the QUERY (pointer), not an injected doc ref.
+  - The recall-event SNAPSHOTS relational hits (TTL-perishable content).
+"""
+from __future__ import annotations
+
+import time
+import uuid
+from typing import Any
+
+from etchmem.config import Config
+from etchmem.models import RecallEvent, SearchResult
+from etchmem.stores.chroma_store import ChromaStore
+
+
+class Recaller:
+    """
+    Encapsulates all recall() logic.
+    Injected by Engine; tests can instantiate directly with mock stores.
+    """
+
+    def __init__(
+        self,
+        relational_store: ChromaStore,
+        buffer_store: ChromaStore,
+        injected_store: ChromaStore,
+        config: Config,
+    ) -> None:
+        self._relational = relational_store
+        self._buffer = buffer_store
+        self._injected = injected_store
+        self._cfg = config
+
+    def recall(
+        self,
+        query: str,
+        skill: str | None = None,
+        top_k: int | None = None,
+        hint: float | None = None,
+    ) -> list[SearchResult]:
+        """
+        Retrieve blended results and emit a recall-event into the buffer.
+
+        Args:
+            query:  Natural language query.
+            skill:  Optional skill scope filter.
+            top_k:  Max results to return. Defaults to config.recall_top_k_injected.
+            hint:   Optional importance prior seeded into the recall-event.
+
+        Returns:
+            List of SearchResult, highest score first.
+        """
+        top_k = top_k or self._cfg.recall_top_k_injected
+
+        # ── 1. Search injected (primary, never gated) ─────────────────────
+        injected_filter = {"skill": skill} if skill else None
+        injected_results = self._injected.query(
+            query_text=query,
+            top_k=top_k,
+            filters=injected_filter,
+            weight=self._cfg.recall_injected_weight,
+            source="injected",
+        )
+
+        # ── 2. Search relational (freshness signal) ───────────────────────
+        relational_results = self._relational.search(
+            query_text=query,
+            top_k=self._cfg.recall_top_k_relational,
+            skill=skill,
+            weight=self._cfg.recall_relational_weight,
+            source="relational",
+        )
+
+        # ── 3. Asymmetric merge ───────────────────────────────────────────
+        merged = self._merge(injected_results, relational_results, top_k)
+
+        # ── 4. Emit recall-event into buffer ──────────────────────────────
+        relational_snapshot = "\n\n".join(r.content for r in relational_results)
+        self._emit_recall_event(query, relational_snapshot, skill, hint)
+
+        return merged
+
+    # ── Helpers ───────────────────────────────────────────────────────────
+
+    def _merge(
+        self,
+        injected: list[SearchResult],
+        relational: list[SearchResult],
+        top_k: int,
+    ) -> list[SearchResult]:
+        """
+        Merge injected and relational results.
+
+        Injected results are the trusted base. Relational results are
+        freshness-boosted but confidence-discounted (already weighted by
+        recall_relational_weight in the store query).
+
+        De-duplicate by id; injected wins on collision.
+        Sort by score descending; return top_k.
+        """
+        seen: dict[str, SearchResult] = {}
+
+        for r in injected:
+            seen[r.id] = r
+
+        for r in relational:
+            if r.id not in seen:
+                seen[r.id] = r
+            else:
+                # Injected already there — boost injected score slightly
+                # by the relational overlap (corroborating signal)
+                existing = seen[r.id]
+                seen[r.id] = SearchResult(
+                    id=existing.id,
+                    content=existing.content,
+                    score=existing.score + r.score * 0.1,
+                    source="blended",
+                    skill=existing.skill,
+                    metadata=existing.metadata,
+                )
+
+        results = sorted(seen.values(), key=lambda r: r.score, reverse=True)
+        return results[:top_k]
+
+    def _emit_recall_event(
+        self,
+        query: str,
+        relational_snapshot: str,
+        skill: str | None,
+        hint: float | None,
+    ) -> None:
+        """Write a RecallEvent into the buffer collection."""
+        event_id = str(uuid.uuid4())
+        event = RecallEvent(
+            id=event_id,
+            query=query,
+            relational_snapshot=relational_snapshot,
+            skill=skill,
+            hint=hint,
+            created_at=time.time(),
+        )
+        self._buffer.upsert([event])

etchmem/stores/base.py

@@ -0,0 +1,171 @@
+"""
+Storage abstract base classes — the pluggability seam.
+
+Everything in the engine talks to these ABCs, never to ChromaDB directly.
+Two ports:
+  - RelationalStore: append-only + TTL expiry + ANN search
+  - InjectedStore:  upsert + ANN query + clustering + hard delete
+"""
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import Any, Iterable
+
+from etchmem.models import Cluster, Deposit, InjectedArticle, Record, RecallEvent, SearchResult
+
+
+class RelationalStore(ABC):
+    """
+    Port for the *relational* collection.
+
+    Semantics:
+    - Pure append — records are NEVER mutated after insertion.
+    - TTL expiry is the ONLY deletion path (swept via metadata filter).
+    - No reinforcement counter. Usage signal is carried by recall-events
+      written to the buffer, not by mutating relational rows.
+    """
+
+    @abstractmethod
+    def add(self, record: Record) -> str:
+        """
+        Insert a raw Record into the relational collection.
+        Returns the record id (same as record.id, passed through for
+        convenience).
+        The store sets expires_at = now + TTL before writing.
+        """
+
+    @abstractmethod
+    def search(
+        self,
+        query_text: str,
+        top_k: int,
+        skill: str | None = None,
+        filters: dict[str, Any] | None = None,
+    ) -> list[SearchResult]:
+        """
+        ANN search of the relational collection.
+        Optionally filter by skill (exact match on stored metadata).
+        Returns up to top_k SearchResult objects.
+        """
+
+    @abstractmethod
+    def expire(self) -> int:
+        """
+        Delete all records whose expires_at < now (TTL sweep).
+        Returns the number of records deleted.
+        Implemented as: collection.delete(where={"expires_at": {"$lt": now}})
+        """
+
+    @abstractmethod
+    def iter_for_consolidation(
+        self,
+        num_records: int | str,
+        method: str,
+    ) -> Iterable[Record]:
+        """
+        Yield records for the consolidation worker to process.
+
+        num_records: "all" or an integer N.
+        method: "LIFO" (newest-first, by created_at) or "FIFO".
+
+        Records are NOT deleted here — relational is append-only.
+        Deletion happens only via TTL expiry.
+        """
+
+    @abstractmethod
+    def count(self) -> int:
+        """Return the number of live (non-expired) records."""
+
+    @abstractmethod
+    def get_by_id(self, record_id: str) -> Record | None:
+        """Fetch a single record by its id, or None if not found."""
+
+
+class InjectedStore(ABC):
+    """
+    Port for both the *buffer* and *injected* collections.
+
+    Buffer semantics:
+    - Working set for the consolidation worker.
+    - Both Deposit and RecallEvent documents are written here.
+    - Flushed (hard-deleted) at the end of each successful consolidation run.
+
+    Injected semantics:
+    - Current-only knowledge store. No versions, no tombstones.
+    - Superseded articles are hard-deleted on write of the new article.
+    - Identity = hash of the source-doc-hash set (content-addressed).
+    """
+
+    @abstractmethod
+    def upsert(self, items: list[InjectedArticle | Deposit | RecallEvent]) -> None:
+        """
+        Write or overwrite items. Uses item.id as the Chroma document id.
+        doc_type metadata discriminator is set automatically.
+        """
+
+    @abstractmethod
+    def query(
+        self,
+        query_text: str,
+        top_k: int,
+        filters: dict[str, Any] | None = None,
+    ) -> list[SearchResult]:
+        """
+        ANN search this collection.
+        Returns up to top_k SearchResult objects.
+        """
+
+    @abstractmethod
+    def delete(self, ids: list[str]) -> None:
+        """
+        Hard-delete documents by id.
+        Deleting a non-existent id is a no-op (idempotent).
+        Used for:
+        - Buffer flush at end of run.
+        - Hard-delete of superseded injected articles.
+        """
+
+    @abstractmethod
+    def exists(self, content_hash: str) -> bool:
+        """
+        Return True if a document with this id already exists.
+        Used as an idempotency guard before writing injected articles.
+        """
+
+    @abstractmethod
+    def cluster(
+        self,
+        documents: list[Deposit | RecallEvent],
+        branch_threshold: float,
+    ) -> list[Cluster]:
+        """
+        Group documents into Cluster objects using their embeddings.
+        Documents within branch_threshold cosine distance of each other
+        are placed in the same cluster.
+        Orphaned documents (no cluster partner) are returned as
+        singleton Clusters.
+        """
+
+    @abstractmethod
+    def get_all(
+        self,
+        filters: dict[str, Any] | None = None,
+    ) -> list[Deposit | RecallEvent | InjectedArticle]:
+        """Return all documents, optionally filtered by metadata."""
+
+    @abstractmethod
+    def count(self) -> int:
+        """Return the total document count."""
+
+    @abstractmethod
+    def get_neighbors(
+        self,
+        query_text: str,
+        top_k: int,
+        distance_threshold: float,
+    ) -> list[SearchResult]:
+        """
+        ANN search with a distance filter — only return results whose
+        cosine distance is below distance_threshold.
+        Used in the reconciliation pass to find near-duplicate injected articles.
+        """