luminary-memory 0.1.0__py3-none-any.whl

luminary_memory/__init__.py

@@ -0,0 +1,4 @@
+from luminary_memory.api import MemoryClient
+
+__version__ = "0.1.0"
+__all__ = ["MemoryClient", "__version__"]

luminary_memory/api.py

@@ -0,0 +1,222 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from luminary_memory.backends import get_backend
+from luminary_memory.budget import truncate
+from luminary_memory.config import Settings
+from luminary_memory.embeddings.fastembed import FastembedEngine
+from luminary_memory.ingest.llm import LLMEnricher, NoopEnricher
+from luminary_memory.ingest.whitelist import WhitelistFilter
+from luminary_memory.types import Memory, RecallResult
+
+
+def _try_index_graph(backend, memory: Memory) -> None:
+    import logging
+
+    from luminary_memory.recall.graph import index_memory_entities
+
+    try:
+        index_memory_entities(backend, memory)
+    except Exception:  # noqa: BLE001 -- graph indexing is best-effort; never abort ingest
+        logging.getLogger(__name__).warning(
+            "graph indexing failed for memory %s (non-fatal)", memory.id
+        )
+
+if TYPE_CHECKING:
+    from luminary_memory.backends.base import MemoryBackend
+
+
+class MemoryClient:
+    def __init__(
+        self,
+        settings: Settings | None = None,
+        db_path: str | None = None,
+        ingest_whitelist: list[str] | None = None,
+        enricher: LLMEnricher | None = None,
+        engine: FastembedEngine | None = None,
+        backend: MemoryBackend | None = None,
+    ):
+        self.settings = settings or Settings()
+        if db_path is not None:
+            self.settings.db_path = db_path
+        if ingest_whitelist is not None:
+            self.settings.ingest_whitelist = ingest_whitelist
+
+        self.backend = backend or get_backend(self.settings)
+        self.whitelist = WhitelistFilter(self.settings.ingest_whitelist)
+        self.engine = engine or FastembedEngine(model_name=self.settings.embedding_model)
+        self.enricher = enricher or (NoopEnricher() if not self.settings.ingest_llm else None)
+
+    def ingest(self, text: str, tags: list[str] | None = None,
+               source: str | None = None) -> int | None:
+        if not self.whitelist.accepts(text):
+            return None
+
+        content, summary, entities, extra_tags = text, None, [], []
+        if self.enricher is not None:
+            enriched = self.enricher.enrich(text)
+            content, summary, entities, extra_tags = (
+                enriched.content, enriched.summary, enriched.entities, enriched.tags,
+            )
+
+        metadata: dict = {}
+        if summary:
+            metadata["summary"] = summary
+        if entities:
+            metadata["entities"] = entities
+
+        m = Memory(
+            content=content,
+            metadata=metadata,
+            source=source,
+            tags=list(dict.fromkeys((tags or []) + extra_tags)),
+            ttl_seconds=self.settings.ttl_default_seconds,
+            embedding=self.engine.embed(content),
+        )
+        mid = self.backend.add(m)
+        m.id = mid
+        _try_index_graph(self.backend, m)
+        return mid
+
+    def get(self, id: int) -> Memory | None:
+        return self.backend.get(id)
+
+    def update(self, memory: Memory) -> None:
+        """Update an existing memory in place."""
+        self.backend.update(memory)
+
+    def delete(self, id: int) -> None:
+        """Delete a memory by id."""
+        self.backend.delete(id)
+
+    def list(self, limit: int = 100, offset: int = 0) -> list[Memory]:
+        """List memories, most recent first (datetime-aware sort)."""
+        limit = max(0, int(limit))
+        offset = max(0, int(offset))
+        from luminary_memory.recall.temporal import _parse_dt
+
+        all_mem = self.backend.all()
+        all_mem.sort(key=lambda m: (_parse_dt(m.created_at or ""), -(m.id or 0)), reverse=True)
+        return all_mem[offset:offset + limit]
+
+    def search(self, query: str, limit: int = 10) -> list[tuple[Memory, float]]:
+        """Direct keyword (FTS) search without the full recall pipeline."""
+        if not (query or "").strip():
+            return []
+        try:
+            return self.backend.keyword_search(query, limit=limit)
+        except Exception:  # noqa: BLE001
+            return []
+
+    def stats(self) -> dict:
+        """Store statistics: count, oldest/newest, avg importance, tags."""
+        import statistics
+
+        all_mem = self.backend.all()
+        n = len(all_mem)
+        if not n:
+            return {
+                "count": 0,
+                "oldest": None,
+                "newest": None,
+                "avg_importance": 0.0,
+                "top_tags": {},
+            }
+
+        avg_importance = statistics.fmean([m.importance for m in all_mem])
+        tag_counts: dict[str, int] = {}
+        for m in all_mem:
+            for t in m.tags or []:
+                tag_counts[t] = tag_counts.get(t, 0) + 1
+        top_tags = sorted(tag_counts.items(), key=lambda kv: -kv[1])[:10]
+        created = sorted(m.created_at or "" for m in all_mem)
+        return {
+            "count": n,
+            "oldest": created[0] if created else None,
+            "newest": created[-1] if created else None,
+            "avg_importance": round(avg_importance, 3),
+            "top_tags": dict(top_tags),
+        }
+
+    def count(self) -> int:
+        return self.backend.count()
+
+    def run_lifecycle(self) -> dict[str, int]:
+        from luminary_memory.lifecycle.runner import run_lifecycle
+
+        return run_lifecycle(self.backend, self.settings)
+
+    def close(self) -> None:
+        self.backend.close()
+
+    def recall(
+        self,
+        query: str,
+        limit: int = 10,
+        token_budget: int | None = None,
+    ) -> RecallResult:
+        from luminary_memory.recall.dedup import dedup_jaccard
+        from luminary_memory.recall.fusion import reciprocal_rank_fusion
+        from luminary_memory.recall.graph import graph_recall
+        from luminary_memory.recall.keyword import keyword_recall
+        from luminary_memory.recall.semantic import semantic_recall
+        from luminary_memory.recall.temporal import temporal_recall
+
+        budget = token_budget if token_budget is not None else self.settings.token_budget
+        rrf_k = self.settings.rrf_k
+        dedup_threshold = self.settings.dedup_jaccard_threshold
+
+        # Per-strategy isolation: one failing strategy must not kill recall.
+        strategies: list[list[tuple]] = []
+        for fn in (
+            lambda: semantic_recall(self.backend, self.engine, query, limit=limit),
+            lambda: keyword_recall(self.backend, query, limit=limit),
+            lambda: temporal_recall(self.backend, limit=limit * 2),
+            lambda: graph_recall(self.backend, query, limit=limit),
+        ):
+            try:
+                strategies.append(fn())
+            except Exception:  # noqa: BLE001
+                strategies.append([])
+
+        id_to_mem: dict[int, Memory] = {}
+        strategies_hit: dict[str, int] = {}
+        ranked_lists: list[list[int]] = []
+        for strat in strategies:
+            ranked_lists.append([m.id for m, _, _ in strat if m.id is not None])
+            for m, score, label in strat:
+                if m.id is None:
+                    continue
+                strategies_hit[label] = strategies_hit.get(label, 0) + 1
+                id_to_mem[m.id] = m
+
+        fused = reciprocal_rank_fusion(ranked_lists, k=rrf_k)
+        scored: list[tuple[Memory, float]] = [
+            (id_to_mem[mid], score) for mid, score in fused if mid in id_to_mem
+        ]
+
+        if not scored:
+            return RecallResult(memories=[], scores=[], strategies_hit=strategies_hit)
+
+        scored = dedup_jaccard(scored, threshold=dedup_threshold)
+
+        memories_ordered = [m for m, _ in scored]
+        memories_ordered = truncate(memories_ordered, token_budget=budget)
+
+        id_to_fused = dict(fused)
+        final_scores = [float(id_to_fused.get(m.id, 0.0)) for m in memories_ordered]
+
+        from datetime import UTC, datetime
+
+        for m in memories_ordered[:limit]:
+            m.access_count += 1
+            m.last_accessed_at = datetime.now(UTC).isoformat()
+            self.backend.update(m)
+
+        trimmed = {k: v for k, v in strategies_hit.items() if v}
+        return RecallResult(
+            memories=memories_ordered[:limit],
+            scores=final_scores[:limit],
+            strategies_hit=trimmed,
+        )

luminary_memory/backends/__init__.py

@@ -0,0 +1,28 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from luminary_memory.backends.sqlite import SQLiteBackend
+
+if TYPE_CHECKING:
+    from luminary_memory.backends.base import MemoryBackend
+    from luminary_memory.config import Settings
+
+
+def get_backend(settings: Settings) -> MemoryBackend:
+    backend_name = (settings.backend or "sqlite").strip().lower()
+    if backend_name == "pgvector":
+        from luminary_memory.backends.pgvector import (
+            PGVectorBackend,
+        )
+
+        if not settings.pg_dsn:
+            raise ValueError("pg_dsn is required when backend='pgvector'")
+        if int(settings.embedding_dim) <= 0:
+            raise ValueError("embedding_dim must be positive")
+        return PGVectorBackend(
+            dsn=settings.pg_dsn, embedding_dim=int(settings.embedding_dim)
+        )
+    if backend_name == "sqlite":
+        return SQLiteBackend(db_path=settings.db_path)
+    raise ValueError(f"unsupported backend {settings.backend!r}")

luminary_memory/backends/base.py

@@ -0,0 +1,24 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+
+from luminary_memory.types import Memory
+
+
+class MemoryBackend(ABC):
+    @abstractmethod
+    def add(self, m: Memory) -> int: ...
+    @abstractmethod
+    def get(self, id: int) -> Memory | None: ...
+    @abstractmethod
+    def update(self, m: Memory) -> None: ...
+    @abstractmethod
+    def delete(self, id: int) -> None: ...
+    @abstractmethod
+    def all(self) -> list[Memory]: ...
+    @abstractmethod
+    def keyword_search(self, query: str, limit: int) -> list[tuple[Memory, float]]: ...
+    @abstractmethod
+    def vector_search(self, vec: list[float], limit: int) -> list[tuple[Memory, float]]: ...
+    @abstractmethod
+    def count(self) -> int: ...

luminary_memory/backends/pgvector.py

@@ -0,0 +1,220 @@
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from luminary_memory.backends.base import MemoryBackend
+from luminary_memory.types import Memory
+
+
+class PGVectorBackend(MemoryBackend):
+    def __init__(
+        self,
+        dsn: str = "postgresql://localhost/luminary_memory",
+        embedding_dim: int = 384,
+    ):
+        import psycopg
+
+        self.dsn = dsn
+        self.embedding_dim = int(embedding_dim)
+        self._psycopg = psycopg
+        self.conn = psycopg.connect(dsn)
+        self._ensure_schema()
+
+    def _ensure_schema(self) -> None:
+        cur = self.conn.cursor()
+        cur.execute("CREATE EXTENSION IF NOT EXISTS vector")
+        cur.execute(
+            f"""
+            CREATE TABLE IF NOT EXISTS memories (
+                id              SERIAL PRIMARY KEY,
+                content         TEXT NOT NULL,
+                metadata        JSONB NOT NULL DEFAULT '{{}}',
+                source          TEXT,
+                tags            JSONB NOT NULL DEFAULT '[]',
+                importance      DOUBLE PRECISION NOT NULL DEFAULT 0.5,
+                ttl_seconds     INTEGER,
+                created_at      TIMESTAMPTZ NOT NULL DEFAULT now(),
+                updated_at      TIMESTAMPTZ NOT NULL DEFAULT now(),
+                last_accessed_at TIMESTAMPTZ,
+                access_count    INTEGER NOT NULL DEFAULT 0,
+                embedding       vector({self.embedding_dim})
+            )
+            """
+        )
+        cur.execute(
+            """
+            CREATE TABLE IF NOT EXISTS entities (
+                id   SERIAL PRIMARY KEY,
+                name TEXT NOT NULL UNIQUE,
+                type TEXT NOT NULL DEFAULT 'generic'
+            )
+            """
+        )
+        cur.execute(
+            """
+            CREATE TABLE IF NOT EXISTS relations (
+                id            SERIAL PRIMARY KEY,
+                source_id     INTEGER NOT NULL REFERENCES entities(id),
+                target_id     INTEGER NOT NULL REFERENCES entities(id),
+                relation_type TEXT NOT NULL DEFAULT 'cooccur',
+                weight        DOUBLE PRECISION NOT NULL DEFAULT 1.0,
+                memory_id     INTEGER REFERENCES memories(id)
+            )
+            """
+        )
+        self.conn.commit()
+
+    def _row_to_memory(self, row: tuple[Any, ...] | dict[str, Any]) -> Memory:
+        if isinstance(row, dict):
+            d: dict[str, Any] = row
+            emb = d.get("embedding")
+        else:
+            cols = [
+                "id", "content", "metadata", "source", "tags", "importance",
+                "ttl_seconds", "created_at", "updated_at", "last_accessed_at",
+                "access_count", "embedding",
+            ]
+            d = dict(zip(cols, row))
+            emb = d.get("embedding")
+        if isinstance(emb, str):
+            try:
+                emb = json.loads(emb)
+            except json.JSONDecodeError:
+                emb = None
+        return Memory(
+            id=int(d["id"]) if d.get("id") is not None else None,
+            content=str(d.get("content") or ""),
+            metadata=d.get("metadata") if isinstance(d.get("metadata"), dict) else json.loads(d.get("metadata") or "{}") if d.get("metadata") else {},
+            source=d.get("source"),
+            tags=d.get("tags") if isinstance(d.get("tags"), list) else json.loads(d.get("tags") or "[]") if d.get("tags") else [],
+            importance=float(d.get("importance") or 0.5),
+            ttl_seconds=d.get("ttl_seconds"),
+            created_at=str(d.get("created_at") or ""),
+            updated_at=str(d.get("updated_at") or ""),
+            last_accessed_at=str(d["last_accessed_at"]) if d.get("last_accessed_at") else None,
+            access_count=int(d.get("access_count") or 0),
+            embedding=list(emb) if isinstance(emb, (list, tuple)) else None,
+        )
+
+    def add(self, m: Memory) -> int:
+        cur = self.conn.cursor()
+        cur.execute(
+            """
+            INSERT INTO memories (content, metadata, source, tags, importance,
+                                  ttl_seconds, created_at, updated_at,
+                                  last_accessed_at, access_count, embedding)
+            VALUES (%s, %s::jsonb, %s, %s::jsonb, %s, %s, %s, %s, %s, %s, %s)
+            RETURNING id
+            """,
+            (
+                m.content,
+                json.dumps(m.metadata),
+                m.source,
+                json.dumps(m.tags),
+                float(m.importance),
+                m.ttl_seconds,
+                m.created_at,
+                m.updated_at,
+                m.last_accessed_at,
+                int(m.access_count),
+                m.embedding,
+            ),
+        )
+        row = cur.fetchone()
+        self.conn.commit()
+        return int(row[0]) if row else 0
+
+    def get(self, id: int) -> Memory | None:
+        cur = self.conn.cursor()
+        cur.execute("SELECT * FROM memories WHERE id = %s", (id,))
+        row = cur.fetchone()
+        if row is None:
+            return None
+        if isinstance(row, dict):
+            return self._row_to_memory(row)
+        return self._row_to_memory(row)
+
+    def update(self, m: Memory) -> None:
+        if m.id is None:
+            raise ValueError("cannot update a memory without an id")
+        cur = self.conn.cursor()
+        cur.execute(
+            """
+            UPDATE memories SET content=%s, metadata=%s::jsonb, source=%s, tags=%s::jsonb,
+                                importance=%s, ttl_seconds=%s, updated_at=%s,
+                                last_accessed_at=%s, access_count=%s, embedding=%s
+            WHERE id=%s
+            """,
+            (
+                m.content,
+                json.dumps(m.metadata),
+                m.source,
+                json.dumps(m.tags),
+                float(m.importance),
+                m.ttl_seconds,
+                m.updated_at,
+                m.last_accessed_at,
+                int(m.access_count),
+                m.embedding,
+                m.id,
+            ),
+        )
+        self.conn.commit()
+
+    def delete(self, id: int) -> None:
+        cur = self.conn.cursor()
+        # Relations reference memories via FK; delete them first (manual cascade).
+        cur.execute("DELETE FROM relations WHERE memory_id = %s", (id,))
+        cur.execute("DELETE FROM memories WHERE id = %s", (id,))
+        self.conn.commit()
+
+    def all(self) -> list[Memory]:
+        cur = self.conn.cursor()
+        cur.execute("SELECT * FROM memories ORDER BY id")
+        rows = cur.fetchall()
+        return [self._row_to_memory(r) for r in rows]
+
+    def keyword_search(self, query: str, limit: int = 10) -> list[tuple[Memory, float]]:
+        escaped = query.replace("%", r"\%").replace("_", r"\_")
+        q = f"%{escaped}%"
+        cur = self.conn.cursor()
+        cur.execute(
+            "SELECT * FROM memories WHERE content ILIKE %s OR tags::text ILIKE %s LIMIT %s",
+            (q, q, int(limit)),
+        )
+        rows = cur.fetchall()
+        return [(self._row_to_memory(r), 1.0) for r in rows]
+
+    def vector_search(self, vec: list[float], limit: int = 10) -> list[tuple[Memory, float]]:
+        cur = self.conn.cursor()
+        cur.execute(
+            "SELECT *, embedding <=> %s::vector AS distance FROM memories "
+            "WHERE embedding IS NOT NULL ORDER BY distance LIMIT %s",
+            (vec, int(limit)),
+        )
+        rows = cur.fetchall()
+        results: list[tuple[Memory, float]] = []
+        for r in rows:
+            if isinstance(r, dict):
+                dist = float(r.get("distance") or 0.0)
+            elif isinstance(r, (list, tuple)) and len(r) >= 13:
+                dist = float(r[-1] or 0.0)
+            else:
+                dist = 0.0
+            results.append((self._row_to_memory(r), 1.0 - dist))
+        return results
+
+    def count(self) -> int:
+        cur = self.conn.cursor()
+        cur.execute("SELECT COUNT(*) FROM memories")
+        row = cur.fetchone()
+        if row is None:
+            return 0
+        return int(row[0] if isinstance(row, (list, tuple)) else row.get("count", 0))
+
+    def close(self) -> None:
+        try:
+            self.conn.close()
+        except (AttributeError, OSError, RuntimeError):
+            pass

luminary_memory/backends/sqlite.py

@@ -0,0 +1,134 @@
+from __future__ import annotations
+
+import json
+import sqlite3
+
+import numpy as np
+
+from luminary_memory.backends.base import MemoryBackend
+from luminary_memory.schema import init_schema
+from luminary_memory.types import Memory
+
+
+class SQLiteBackend(MemoryBackend):
+    def __init__(self, db_path: str = "luminary_memory.db"):
+        self.db_path = db_path
+        self.conn = sqlite3.connect(db_path)
+        self.conn.row_factory = sqlite3.Row
+        self.conn.execute("PRAGMA foreign_keys = ON")
+        init_schema(self.conn)
+
+    def _row_to_memory(self, row: sqlite3.Row) -> Memory:
+        m = Memory(
+            id=row["id"],
+            content=row["content"],
+            metadata=json.loads(row["metadata"] or "{}"),
+            source=row["source"],
+            tags=json.loads(row["tags"] or "[]"),
+            importance=row["importance"],
+            ttl_seconds=row["ttl_seconds"],
+            created_at=row["created_at"],
+            updated_at=row["updated_at"],
+            last_accessed_at=row["last_accessed_at"],
+            access_count=row["access_count"],
+        )
+        emb = row["embedding"]
+        if emb is not None:
+            m.embedding = np.frombuffer(emb, dtype=np.float32).tolist()
+        return m
+
+    @staticmethod
+    def _encode_embedding(vec: list[float] | None) -> bytes | None:
+        if vec is None:
+            return None
+        return np.asarray(vec, dtype=np.float32).tobytes()
+
+    def add(self, m: Memory) -> int:
+        cur = self.conn.execute(
+            "INSERT INTO memories (content, metadata, source, tags, importance, "
+            "ttl_seconds, created_at, updated_at, last_accessed_at, access_count, embedding) "
+            "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
+            (
+                m.content,
+                json.dumps(m.metadata),
+                m.source,
+                json.dumps(m.tags),
+                m.importance,
+                m.ttl_seconds,
+                m.created_at,
+                m.updated_at,
+                m.last_accessed_at,
+                m.access_count,
+                self._encode_embedding(m.embedding),
+            ),
+        )
+        self.conn.commit()
+        return int(cur.lastrowid)
+
+    def get(self, id: int) -> Memory | None:
+        row = self.conn.execute("SELECT * FROM memories WHERE id = ?", (id,)).fetchone()
+        return self._row_to_memory(row) if row else None
+
+    def update(self, m: Memory) -> None:
+        if m.id is None:
+            raise ValueError("cannot update a memory without an id")
+        self.conn.execute(
+            "UPDATE memories SET content=?, metadata=?, source=?, tags=?, importance=?, "
+            "ttl_seconds=?, updated_at=?, last_accessed_at=?, access_count=?, embedding=? "
+            "WHERE id=?",
+            (
+                m.content,
+                json.dumps(m.metadata),
+                m.source,
+                json.dumps(m.tags),
+                m.importance,
+                m.ttl_seconds,
+                m.updated_at,
+                m.last_accessed_at,
+                m.access_count,
+                self._encode_embedding(m.embedding),
+                m.id,
+            ),
+        )
+        self.conn.commit()
+
+    def delete(self, id: int) -> None:
+        # Relations reference memories via FK; delete them first (manual cascade).
+        self.conn.execute("DELETE FROM relations WHERE memory_id = ?", (id,))
+        self.conn.execute("DELETE FROM memories WHERE id = ?", (id,))
+        self.conn.commit()
+
+    def all(self) -> list[Memory]:
+        rows = self.conn.execute("SELECT * FROM memories ORDER BY id").fetchall()
+        return [self._row_to_memory(r) for r in rows]
+
+    def keyword_search(self, query: str, limit: int = 10) -> list[tuple[Memory, float]]:
+        safe = query.replace('"', ' ')
+        rows = self.conn.execute(
+            "SELECT m.*, bm25(memories_fts) AS rank "
+            "FROM memories_fts JOIN memories m ON m.id = memories_fts.rowid "
+            "WHERE memories_fts MATCH ? ORDER BY rank LIMIT ?",
+            (safe, limit),
+        ).fetchall()
+        return [(self._row_to_memory(r), -float(r["rank"])) for r in rows]
+
+    def vector_search(self, vec: list[float], limit: int = 10) -> list[tuple[Memory, float]]:
+        q = np.asarray(vec, dtype=np.float32)
+        qn = float(np.linalg.norm(q))
+        results: list[tuple[Memory, float]] = []
+        for m in self.all():
+            if m.embedding is None:
+                continue
+            v = np.asarray(m.embedding, dtype=np.float32)
+            vn = float(np.linalg.norm(v))
+            sim = 0.0 if (qn == 0 or vn == 0) else float(np.dot(q, v) / (qn * vn))
+            results.append((m, sim))
+        results.sort(key=lambda x: -x[1])
+        return results[:limit]
+
+    def count(self) -> int:
+        row = self.conn.execute("SELECT COUNT(*) FROM memories").fetchone()
+        return int(row[0])
+
+    def close(self) -> None:
+        self.conn.close()

luminary_memory/budget.py

@@ -0,0 +1,36 @@
+from __future__ import annotations
+
+from collections.abc import Callable
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from luminary_memory.types import Memory
+
+
+def _estimate_tokens(text: str) -> int:
+    return len(text.split())
+
+
+def truncate(
+    memories: list[Memory],
+    token_budget: int,
+    tokenizer: Callable[[str], list] | None = None,
+) -> list[Memory]:
+    if not memories or token_budget <= 0:
+        return []
+    count_fn: Callable[[str], int]
+    if tokenizer is not None:
+        def count_fn(s: str) -> int:
+            return len(tokenizer(s))
+    else:
+        count_fn = _estimate_tokens
+
+    result: list[Memory] = []
+    total = 0
+    for m in memories:
+        cost = count_fn(m.content)
+        if total + cost > token_budget:
+            continue
+        total += cost
+        result.append(m)
+    return result