engraphis 0.9.2__tar.gz → 0.9.4__tar.gz

{engraphis-0.9.2/engraphis.egg-info → engraphis-0.9.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: engraphis
-Version: 0.9.2
+Version: 0.9.4
 Summary: Local-first AI memory engine for agents — Ebbinghaus decay, interaction-aware recall, bi-temporal facts, hybrid retrieval, and an MCP server. You bring the LLM.
 Author: The Engraphis Authors
 License: Apache-2.0
@@ -197,6 +197,11 @@ pip install "engraphis[encryption]" # SQLCipher encryption-at-rest extra
 pip install engraphis               # core library — numpy only, fully offline
 ```
 
+> **Linux / macOS:** if `pip install` fails with `error: externally-managed-environment`,
+> your system Python is marked read-only (PEP 668). Install into a virtual environment
+> instead — `python3 -m venv venv && source venv/bin/activate && pip install "engraphis[server]"`
+> — or use Docker (`docker compose up`). `pipx install "engraphis[server]"` also works.
+
 > First run downloads `all-MiniLM-L6-v2` (~80 MB). Without it, the engine falls back
 > to a deterministic offline embedder so it always runs.
 

{engraphis-0.9.2 → engraphis-0.9.4}/README.md

@@ -129,6 +129,11 @@ pip install "engraphis[encryption]" # SQLCipher encryption-at-rest extra
 pip install engraphis               # core library — numpy only, fully offline
 ```
 
+> **Linux / macOS:** if `pip install` fails with `error: externally-managed-environment`,
+> your system Python is marked read-only (PEP 668). Install into a virtual environment
+> instead — `python3 -m venv venv && source venv/bin/activate && pip install "engraphis[server]"`
+> — or use Docker (`docker compose up`). `pipx install "engraphis[server]"` also works.
+
 > First run downloads `all-MiniLM-L6-v2` (~80 MB). Without it, the engine falls back
 > to a deterministic offline embedder so it always runs.
 

{engraphis-0.9.2 → engraphis-0.9.4}/engraphis/automation.py

@@ -29,6 +29,18 @@ DEFAULT_POLICY: dict = {
     "min_cluster": 3,
     "archive_below": 0.05,
     "workspaces": [],  # empty = every workspace the caller can see
+    # "Dreaming" trigger: run a sweep *before* the cadence elapses when enough new
+    # episodic memories have piled up AND the store has gone quiet (the user paused).
+    # Purely additive to the cadence in ``due()`` — it can only cause more sweeps, never
+    # fewer — so existing cron behaviour is unchanged when left at defaults.
+    "dream": True,
+    "dream_min_new": 25,       # new episodics since last run before an early sweep is worth it
+    "dream_idle_minutes": 15,  # ...and this long since the most recent write ("went quiet")
+    # Cross-cluster inference (consolidate pass 4). OFF by default: it writes new
+    # (low-salience, untrusted, linked) memories, so a human opts in. When on, a sweep
+    # (manual *or* the dream loop) runs the inference pass too — following the sweep's
+    # own dry_run flag, so a dry-run preview proposes and a real run applies.
+    "infer": False,
 }
 
 _POLICY_KEYS = set(DEFAULT_POLICY)
@@ -78,6 +90,16 @@ def normalize_policy(raw: dict) -> dict:
         p["archive_below"] = 0.05
     wss = p.get("workspaces") or []
     p["workspaces"] = [str(w) for w in wss] if isinstance(wss, list) else []
+    p["dream"] = bool(p["dream"])
+    p["infer"] = bool(p["infer"])
+    try:
+        p["dream_min_new"] = max(1, int(p["dream_min_new"] or 25))
+    except (TypeError, ValueError):
+        p["dream_min_new"] = 25
+    try:
+        p["dream_idle_minutes"] = max(0, int(p["dream_idle_minutes"]))
+    except (TypeError, ValueError):
+        p["dream_idle_minutes"] = 15
     return p
 
 
@@ -131,6 +153,75 @@ def due(policy: dict, *, now: Optional[float] = None) -> bool:
         return True
 
 
+def dream_signals(memories: Any, *, last_run: Optional[float],
+                  now: float) -> tuple[int, Optional[float]]:
+    """From an iterable of memory records, compute ``(new_episodic, idle_seconds)``:
+    how many *episodic* memories were ingested since ``last_run`` (accumulation) and how
+    long since the most recent write of any kind (idle). ``idle_seconds`` is ``None`` for
+    an empty store. Pure and side-effect-free so it unit-tests without a database."""
+    newest = 0.0
+    new_episodic = 0
+    for m in memories:
+        ts = float(getattr(m, "ingested_at", 0) or 0)
+        if ts > newest:
+            newest = ts
+        mtype = getattr(getattr(m, "mtype", ""), "value", getattr(m, "mtype", ""))
+        if str(mtype) == "episodic" and (last_run is None or ts > float(last_run)):
+            new_episodic += 1
+    return new_episodic, (now - newest if newest else None)
+
+
+def should_dream(policy: dict, memories: Any, *, now: Optional[float] = None) -> bool:
+    """True when accumulation + idle warrant a sweep *before* the cadence is due.
+
+    Requires the policy be enabled and dreaming on, then: at least ``dream_min_new`` new
+    episodic memories since ``last_run`` **and** the store idle for ``dream_idle_minutes``.
+    The idle guard means a sweep never fights a live editing burst; the accumulation guard
+    means it never runs on a store nothing has been added to."""
+    if not policy.get("enabled") or not policy.get("dream", True):
+        return False
+    now = time.time() if now is None else now
+    new_episodic, idle = dream_signals(
+        memories, last_run=policy.get("last_run"), now=now)
+    if new_episodic < int(policy.get("dream_min_new", 25)):
+        return False
+    if idle is None:
+        return False
+    return idle >= int(policy.get("dream_idle_minutes", 15)) * 60.0
+
+
+def dream_due(service: Any, *, policy: Optional[dict] = None,
+              now: Optional[float] = None, scan_limit: int = 5000) -> bool:
+    """Should a scheduled sweep run now? ``due()`` (cadence) **or** ``should_dream()``
+    (accumulation+idle). Reads recent memories from the service's store — the only IO in
+    the trigger — and is purely additive to the cadence, so cron users are unaffected."""
+    pol = load_policy() if policy is None else policy
+    if due(pol, now=now):
+        return True
+    try:
+        from engraphis.core.interfaces import SearchFilter
+        targets = pol.get("workspaces") or []
+        if targets:
+            # Only accumulation/idle *within* the scoped workspaces can fire a sweep
+            # — a burst in an out-of-scope workspace must not trigger one. Names are
+            # resolved to ids with a read-only lookup (never create) so a stale policy
+            # entry can't mint an empty folder.
+            memories: list = []
+            conn = service.store.conn
+            for name in targets:
+                row = conn.execute(
+                    "SELECT id FROM workspaces WHERE name=?", (str(name),)).fetchone()
+                if not row:
+                    continue
+                memories.extend(service.store.list_memories(
+                    SearchFilter(workspace_id=row["id"]), limit=scan_limit))
+        else:
+            memories = service.store.list_memories(SearchFilter(), limit=scan_limit)
+    except Exception:  # noqa: BLE001 - a trigger must never crash the scheduled job
+        return False
+    return should_dream(pol, memories, now=now)
+
+
 def run_maintenance(service: Any, *, dry_run: bool = True,
                     policy: Optional[dict] = None, record: bool = True,
                     now: Optional[float] = None) -> dict:
@@ -156,7 +247,8 @@ def run_maintenance(service: Any, *, dry_run: bool = True,
                 entry["consolidate"] = service.consolidate(
                     workspace=ws, dry_run=dry_run,
                     min_cluster=pol["min_cluster"],
-                    archive_below=pol["archive_below"])
+                    archive_below=pol["archive_below"],
+                    infer=bool(pol.get("infer", False)))
         except Exception as exc:  # noqa: BLE001 - isolate a bad workspace
             entry["error"] = str(exc)
         runs.append(entry)

engraphis-0.9.4/engraphis/backends/extractor.py

@@ -0,0 +1,368 @@
+"""Fact extractors — implementations of the ``core.interfaces.Extractor`` protocol.
+
+Every SOTA memory system (mem0, A-Mem, Letta) auto-distills raw text into discrete
+facts before storage; Engraphis makes that step *pluggable and optional* so the core
+stays offline-capable (AGENTS.md §3.8):
+
+* ``PassthroughExtractor`` — the default: the caller's text is stored exactly as given
+  (today's behaviour, zero dependencies, zero network).
+* ``ChunkingExtractor``   — splits a document into retrieval-sized, structure-aware
+  chunks (one ``ExtractedFact`` each) *without* an LLM: headings start new chunks and
+  become the title, fenced code blocks stay intact, prose is packed to a token budget
+  with a small sentence overlap. Deterministic and offline (numpy/stdlib only) so it
+  runs under the offline gate; the answer to "one memory per file dilutes recall".
+* ``LLMExtractor``        — distills a raw blob (a conversation turn, a log, a diff
+  summary) into discrete, self-contained facts with type/importance/keyword hints,
+  using any configured LLM. Fails soft: any error degrades to passthrough, never to a
+  lost write.
+
+Selected via ``get_extractor()`` from ``ENGRAPHIS_EXTRACTOR`` (= ``none`` | ``chunk`` |
+``llm``) — a config change, not a refactor, matching every other backend swap here.
+"""
+from __future__ import annotations
+
+import json
+import os
+import re
+from typing import Any, Optional
+
+from engraphis.core.interfaces import ExtractedFact, MemoryType
+from engraphis.core.textutil import estimate_tokens, tokenize
+
+MAX_FACTS = 12
+
+# Structure-aware chunking defaults (offline, deterministic). Overridable via
+# ENGRAPHIS_CHUNK_TOKENS / ENGRAPHIS_CHUNK_OVERLAP / ENGRAPHIS_CHUNK_MAX.
+CHUNK_TARGET_TOKENS = 256   # target tokens per prose chunk
+CHUNK_OVERLAP_TOKENS = 32   # sentence-level overlap carried between adjacent chunks
+CHUNK_MAX = 200             # hard cap on chunks per document (amplification guard)
+
+_HEADING_RE = re.compile(r"^(#{1,6})\s+(.+?)\s*#*$")
+_FENCE_RE = re.compile(r"^(```+|~~~+)")
+_SENTENCE_RE = re.compile(r"(?<=[.!?])\s+")
+_PARA_SPLIT_RE = re.compile(r"\n\s*\n")
+
+# LLM output is untrusted input too (indirect prompt injection can steer it): strip the
+# same control characters service.py strips from direct writes, so extracted facts can't
+# smuggle hidden-instruction / terminal-escape payloads past the validation layer.
+_CONTROL_RE = re.compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]")
+
+
+def _defang(value: str, limit: int) -> str:
+    return _CONTROL_RE.sub("", value)[:limit].strip()
+
+_EXTRACT_SYSTEM_PROMPT = (
+    "You distill raw text into discrete, self-contained memory facts for an AI agent's "
+    "long-term memory. Each fact must stand alone (no pronouns that depend on the "
+    "original text), be worth remembering beyond this moment, and be stated in one or "
+    "two sentences. Skip filler, pleasantries, and transient chatter.\n\n"
+    "Classify each fact:\n"
+    "- semantic: durable facts, preferences, conventions ('The API uses PASETO tokens')\n"
+    "- episodic: events and decisions with a when ('On 2026-06-30 PR #99 was merged')\n"
+    "- procedural: how-tos and playbooks ('To rebuild the index, run ...')\n"
+    "- working: transient state only relevant right now ('Currently blocked on CI')\n\n"
+    "Respond with JSON only, no markdown fences, no prose:\n"
+    '{"facts": [{"content": str, "title": str, "mtype": "semantic|episodic|procedural|'
+    'working", "importance": <0..1>, "keywords": [str, ...]}]}'
+)
+
+
+class PassthroughExtractor:
+    """The offline default: one fact, the text as given."""
+
+    def extract(self, text: str, *, context: str = "") -> list[ExtractedFact]:
+        return [ExtractedFact(content=text)]
+
+
+class LLMExtractor:
+    """LLM-backed fact distillation behind the ``Extractor`` protocol.
+
+    ``llm`` may be anything with a ``chat(messages, system=...) -> str`` method (the
+    engraphis v1 ``LLMClient``) or a ``complete(messages) -> str`` method (the
+    ``core.interfaces.LLM`` protocol). Untrusted input goes *into* the prompt; the
+    output is parsed defensively and every fact re-validated — a malformed or
+    adversarial response degrades to passthrough rather than corrupting the store.
+    """
+
+    def __init__(self, llm: Any, *, max_facts: int = MAX_FACTS) -> None:
+        self.llm = llm
+        self.max_facts = max_facts
+
+    def extract(self, text: str, *, context: str = "") -> list[ExtractedFact]:
+        prompt = f"Context: {context}\n\nText to distill:\n{text}" if context else text
+        try:
+            raw = self._ask(prompt)
+            facts = self._parse(raw)
+        except Exception:
+            facts = []
+        return facts or [ExtractedFact(content=text)]
+
+    # ── internals ────────────────────────────────────────────────────────────
+    def _ask(self, prompt: str) -> str:
+        messages = [{"role": "user", "content": prompt}]
+        if hasattr(self.llm, "chat"):
+            return self.llm.chat(messages, system=_EXTRACT_SYSTEM_PROMPT)
+        return self.llm.complete(
+            [{"role": "system", "content": _EXTRACT_SYSTEM_PROMPT}, *messages])
+
+    def _parse(self, raw: str) -> list[ExtractedFact]:
+        data = _loads_lenient(raw)
+        out: list[ExtractedFact] = []
+        for item in (data.get("facts") or [])[: self.max_facts]:
+            if not isinstance(item, dict):
+                continue
+            content = _defang(str(item.get("content") or ""), 100_000)
+            if not content:
+                continue
+            mtype: Optional[MemoryType] = None
+            try:
+                mtype = MemoryType(str(item.get("mtype", "")).strip().lower())
+            except ValueError:
+                mtype = None
+            try:
+                importance = max(0.0, min(1.0, float(item.get("importance", 0.0))))
+            except (TypeError, ValueError):
+                importance = 0.0
+            keywords = [_defang(str(k), 128) for k in (item.get("keywords") or [])[:16]
+                        if isinstance(k, (str, int, float))]
+            out.append(ExtractedFact(content=content,
+                                     title=_defang(str(item.get("title") or ""), 1_000),
+                                     mtype=mtype, importance=importance,
+                                     keywords=[k for k in keywords if k]))
+        return out
+
+
+class ChunkingExtractor:
+    """Deterministic, offline, structure-aware chunker (``Extractor`` protocol).
+
+    Splits a document into retrieval-sized ``ExtractedFact`` chunks that preserve
+    meaning rather than cutting at arbitrary character counts:
+
+    * Markdown headings (``#``..``######``) start a new chunk; the heading path
+      (``H1 > H2``) becomes the chunk title and is kept as context.
+    * Fenced code blocks (```` ``` ````/``~~~``) are emitted whole — never split
+      mid-fence.
+    * Prose is packed paragraph-by-paragraph up to ``target_tokens``, with a small
+      sentence-level overlap so a fact straddling a boundary survives in both chunks.
+
+    numpy/stdlib only — no model, no network — so it is safe inside the offline gate
+    and identical across runs (a requirement for deterministic-embedder eval).
+    """
+
+    def __init__(self, *, target_tokens: int = CHUNK_TARGET_TOKENS,
+                 overlap_tokens: int = CHUNK_OVERLAP_TOKENS,
+                 max_chunks: int = CHUNK_MAX) -> None:
+        self.target_tokens = max(16, int(target_tokens))
+        self.overlap_tokens = max(0, min(int(overlap_tokens), self.target_tokens // 2))
+        self.max_chunks = max(1, int(max_chunks))
+
+    def extract(self, text: str, *, context: str = "") -> list[ExtractedFact]:
+        text = text or ""
+        if not text.strip():
+            return []
+        facts: list[ExtractedFact] = []
+        for heading_path, content in self._chunks(text):
+            content = _defang(content, 100_000)
+            if not content:
+                continue
+            leaf = heading_path.split(" > ")[-1] if heading_path else ""
+            title = _defang(leaf or _first_line(content), 1_000)
+            facts.append(ExtractedFact(content=content, title=title[:200],
+                                       keywords=_keywords(content)))
+            if len(facts) >= self.max_chunks:
+                break
+        # Never lose the write: an all-whitespace/degenerate parse falls back to the
+        # whole text, exactly like PassthroughExtractor.
+        return facts or [ExtractedFact(content=_defang(text, 100_000))]
+
+    # ── internals ────────────────────────────────────────────────────────────
+    def _chunks(self, text: str) -> list[tuple[str, str]]:
+        out: list[tuple[str, str]] = []
+        for heading_path, kind, body in self._segment(text):
+            if len(out) >= self.max_chunks:
+                break
+            if kind == "code":
+                out.append((heading_path, body))          # atomic — never split
+            else:
+                for piece in self._pack(body):
+                    out.append((heading_path, piece))
+                    if len(out) >= self.max_chunks:
+                        break
+        return out[: self.max_chunks]
+
+    def _segment(self, text: str) -> list[tuple[str, str, str]]:
+        """Split into ``(heading_path, kind, body)`` segments, preserving code fences
+        whole and tracking the active markdown heading stack."""
+        lines = text.split("\n")
+        heading_stack: list[tuple[int, str]] = []
+        segments: list[tuple[str, str, str]] = []
+        prose: list[str] = []
+        n = len(lines)
+
+        def path() -> str:
+            return " > ".join(t for _, t in heading_stack)
+
+        def flush() -> None:
+            if prose:
+                body = "\n".join(prose).strip()
+                if body:
+                    segments.append((path(), "prose", body))
+                prose.clear()
+
+        i = 0
+        while i < n:
+            line = lines[i]
+            stripped = line.strip()
+            fence = _FENCE_RE.match(stripped)
+            heading = _HEADING_RE.match(line)
+            if fence:
+                flush()
+                marker = fence.group(1)[:3]
+                block = [line]
+                i += 1
+                while i < n:
+                    block.append(lines[i])
+                    closed = lines[i].strip().startswith(marker)
+                    i += 1
+                    if closed:
+                        break
+                segments.append((path(), "code", "\n".join(block).strip()))
+                continue
+            if heading:
+                flush()
+                level = len(heading.group(1))
+                heading_stack[:] = [(lv, t) for lv, t in heading_stack if lv < level]
+                heading_stack.append((level, heading.group(2).strip()))
+                prose.append(line)  # keep heading text in the body too, for lexical recall
+                i += 1
+                continue
+            prose.append(line)
+            i += 1
+        flush()
+        return segments
+
+    def _pack(self, body: str) -> list[str]:
+        """Greedily pack paragraphs to the token budget with sentence overlap."""
+        paras = [p.strip() for p in _PARA_SPLIT_RE.split(body) if p.strip()]
+        chunks: list[str] = []
+        cur: list[str] = []
+        cur_tokens = 0
+        for para in paras:
+            ptokens = estimate_tokens(para)
+            if cur and cur_tokens + ptokens > self.target_tokens:
+                joined = "\n\n".join(cur)
+                chunks.append(joined)
+                tail = self._overlap_tail(joined)
+                cur = [tail] if tail else []
+                cur_tokens = estimate_tokens(tail) if tail else 0
+            if ptokens > self.target_tokens:
+                for group in self._split_paragraph(para):
+                    chunks.append(group)
+                cur, cur_tokens = [], 0
+                continue
+            cur.append(para)
+            cur_tokens += ptokens
+        if cur:
+            chunks.append("\n\n".join(cur))
+        return chunks
+
+    def _split_paragraph(self, para: str) -> list[str]:
+        """Split an oversized paragraph on sentence boundaries (never mid-sentence)."""
+        sentences = [s for s in _SENTENCE_RE.split(para.strip()) if s]
+        groups: list[str] = []
+        cur: list[str] = []
+        cur_tokens = 0
+        for sent in sentences:
+            stokens = estimate_tokens(sent)
+            if cur and cur_tokens + stokens > self.target_tokens:
+                joined = " ".join(cur)
+                groups.append(joined)
+                tail = self._overlap_tail(joined)
+                cur = [tail] if tail else []
+                cur_tokens = estimate_tokens(tail) if tail else 0
+            cur.append(sent)
+            cur_tokens += stokens
+        if cur:
+            groups.append(" ".join(cur))
+        return groups
+
+    def _overlap_tail(self, text: str) -> str:
+        """Trailing whole sentences of ``text`` up to ``overlap_tokens``."""
+        if self.overlap_tokens <= 0:
+            return ""
+        sentences = [s for s in _SENTENCE_RE.split(text.strip()) if s]
+        tail: list[str] = []
+        tokens = 0
+        for sent in reversed(sentences):
+            if tail and tokens + estimate_tokens(sent) > self.overlap_tokens:
+                break
+            tail.insert(0, sent)
+            tokens += estimate_tokens(sent)
+            if tokens >= self.overlap_tokens:
+                break
+        return " ".join(tail).strip()
+
+
+def _first_line(text: str) -> str:
+    for line in text.splitlines():
+        candidate = line.strip().lstrip("#").strip()
+        if candidate:
+            return candidate[:200]
+    return "chunk"
+
+
+def _keywords(text: str, k: int = 8) -> list[str]:
+    counts: dict[str, int] = {}
+    for token in tokenize(text):
+        counts[token] = counts.get(token, 0) + 1
+    return sorted(counts, key=lambda t: (-counts[t], t))[:k]
+
+
+def _env_int(name: str, default: int) -> int:
+    try:
+        return int(os.environ.get(name, "") or default)
+    except (TypeError, ValueError):
+        return default
+
+
+def _loads_lenient(raw: str) -> dict:
+    """Parse JSON that may arrive wrapped in markdown fences or prose."""
+    raw = (raw or "").strip()
+    try:
+        return json.loads(raw)
+    except (json.JSONDecodeError, RecursionError):
+        pass
+    m = re.search(r"\{.*\}", raw, flags=re.DOTALL)
+    if m:
+        try:
+            return json.loads(m.group(0))
+        except (json.JSONDecodeError, RecursionError):
+            pass
+    return {}
+
+
+def get_extractor(kind: str = "none", llm: Any = None):
+    """Factory mirroring ``get_embedder``/``get_vector_index``: config in, backend out.
+
+    ``kind='chunk'`` returns the deterministic, offline ``ChunkingExtractor`` (knobs from
+    ``ENGRAPHIS_CHUNK_TOKENS``/``_OVERLAP``/``_MAX``). ``kind='llm'`` with no ``llm``
+    builds the v1 multi-provider ``LLMClient`` from settings (heavy import gated here,
+    never in ``core/``). Anything else — including an LLM kind with no usable client —
+    returns the offline passthrough.
+    """
+    kind = (kind or "none").lower()
+    if kind == "chunk":
+        return ChunkingExtractor(
+            target_tokens=_env_int("ENGRAPHIS_CHUNK_TOKENS", CHUNK_TARGET_TOKENS),
+            overlap_tokens=_env_int("ENGRAPHIS_CHUNK_OVERLAP", CHUNK_OVERLAP_TOKENS),
+            max_chunks=_env_int("ENGRAPHIS_CHUNK_MAX", CHUNK_MAX),
+        )
+    if kind != "llm":
+        return PassthroughExtractor()
+    if llm is None:
+        try:
+            from engraphis.llm.client import LLMClient
+            llm = LLMClient()
+        except Exception:
+            return PassthroughExtractor()
+    return LLMExtractor(llm)