velune-cli 0.9.0__py3-none-any.whl

velune/memory/module.py

@@ -0,0 +1,218 @@
+from velune.kernel.bootstrap import RuntimeEnvironment, SubsystemModule
+
+
+def _create_sqlite_pool(env: RuntimeEnvironment):
+    from velune.core.paths import cognitive_db_path, migrate_legacy_storage
+    from velune.memory.storage.sqlite_pool import SQLiteConnectionPool
+
+    # One-time relocation of any pre-existing in-workspace (cloud-synced) data.
+    migrate_legacy_storage(env.workspace)
+    pool = SQLiteConnectionPool(cognitive_db_path(env.workspace))
+    # Register is_healthy as a custom health hook when the monitor is available.
+    if env.container.has("runtime.health_monitor"):
+        monitor = env.container.get("runtime.health_monitor")
+        monitor.register_health_hook("sqlite_pool", pool.health_check)
+    return pool
+
+
+def _create_working_tier(env: RuntimeEnvironment):
+    from velune.memory.tiers.working import WorkingMemoryTier
+
+    return WorkingMemoryTier()
+
+
+def _create_episodic_tier(env: RuntimeEnvironment):
+    from velune.memory.tiers.episodic import EpisodicMemoryTier
+
+    pool = env.container.get("runtime.sqlite_pool")
+    return EpisodicMemoryTier(pool)
+
+
+def _create_semantic_tier(env: RuntimeEnvironment):
+    from velune.core.paths import qdrant_store_path
+    from velune.memory.tiers.semantic import SemanticMemoryTier
+
+    return SemanticMemoryTier(path=str(qdrant_store_path(env.workspace)))
+
+
+def _create_graph_tier(env: RuntimeEnvironment):
+    from velune.memory.tiers.graph import GraphMemoryTier
+
+    pool = env.container.get("runtime.sqlite_pool")
+    return GraphMemoryTier(pool)
+
+
+def _create_lineage_tier(env: RuntimeEnvironment):
+    from velune.memory.tiers.lineage import LineageMemoryTier
+
+    pool = env.container.get("runtime.sqlite_pool")
+    return LineageMemoryTier(pool)
+
+
+def _create_lancedb_store(env: RuntimeEnvironment):
+    from velune.core.paths import lancedb_store_path
+    from velune.memory.storage.lancedb_store import LanceDBStore
+
+    store = LanceDBStore(lancedb_store_path(env.workspace))
+    if env.container.has("runtime.health_monitor"):
+        monitor = env.container.get("runtime.health_monitor")
+        monitor.register_health_hook("lancedb_store", store.health_check)
+    return store
+
+
+def _create_embedding_pipeline(env: RuntimeEnvironment):
+    from velune.memory.embedding_pipeline import EmbeddingPipeline
+
+    store = env.container.get("runtime.lancedb_store")
+    try:
+        provider_registry = env.container.get("runtime.provider_registry")
+        provider = provider_registry.get("ollama") if provider_registry else None
+    except Exception:
+        provider = None
+    return EmbeddingPipeline(provider, store)
+
+
+def _create_semantic_memory_lance(env: RuntimeEnvironment):
+    from velune.memory.tiers.semantic import SemanticMemory
+
+    store = env.container.get("runtime.lancedb_store")
+    pipeline = env.container.get("runtime.embedding_pipeline")
+    return SemanticMemory(store, pipeline)
+
+
+def _create_episodic_session_memory(env: RuntimeEnvironment):
+    from velune.memory.tiers.episodic import EpisodicMemory
+
+    pool = env.container.get("runtime.sqlite_pool")
+    return EpisodicMemory(pool)
+
+
+def _create_memory_lifecycle(env: RuntimeEnvironment):
+    from velune.memory.lifecycle import MemoryLifecycleManager
+
+    working_tier = env.container.get("runtime.working_memory")
+    episodic_tier = env.container.get("runtime.episodic_memory")
+    episodic_memory = env.container.get("runtime.episodic_session_memory")
+    semantic_memory = env.container.get("runtime.semantic_memory_lance")
+    embedding_pipeline = env.container.get("runtime.embedding_pipeline")
+    lineage_tier = env.container.get("runtime.lineage_memory")
+
+    manager = MemoryLifecycleManager(
+        working_tier=working_tier,
+        episodic_memory=episodic_memory,
+        semantic_memory=semantic_memory,
+        embedding_pipeline=embedding_pipeline,
+        lineage_tier=lineage_tier,
+        episodic_session_memory=episodic_tier,
+    )
+
+    # Register health hook if monitor is available
+    if env.container.has("runtime.health_monitor"):
+        monitor = env.container.get("runtime.health_monitor")
+
+        async def _memory_health_check() -> dict:
+            try:
+                health = await manager.health()
+                return health.to_dict()
+            except Exception as e:
+                return {"error": str(e)}
+
+        # Bridge the async health() coroutine onto the synchronous health-hook
+        # contract via submit(), which routes through the single run_async()
+        # entry point in velune.kernel.entrypoint. submit() raises RuntimeError
+        # when already inside a running event loop; in that case we return a
+        # placeholder rather than blocking the loop with a nested run.
+        def _sync_health_check() -> dict:
+            from velune.core.event_loop import submit
+
+            try:
+                return submit(_memory_health_check())
+            except RuntimeError:
+                return {"status": "async_only"}
+            except Exception as e:
+                return {"error": str(e)}
+
+        monitor.register_health_hook("memory", _sync_health_check)
+
+    return manager
+
+
+MEMORY_MODULES = [
+    SubsystemModule(
+        name="sqlite_pool",
+        factory=_create_sqlite_pool,
+        container_key="runtime.sqlite_pool",
+        lifecycle_key="sqlite_pool",  # pool.initialize() → pool.startup()
+    ),
+    SubsystemModule(
+        name="working_memory",
+        factory=_create_working_tier,
+        container_key="runtime.working_memory",
+    ),
+    SubsystemModule(
+        name="episodic_memory",
+        factory=_create_episodic_tier,
+        container_key="runtime.episodic_memory",
+        lifecycle_key="episodic_memory",  # tier.initialize() → _init_db()
+        dependencies=["runtime.sqlite_pool"],
+    ),
+    SubsystemModule(
+        name="semantic_memory",
+        factory=_create_semantic_tier,
+        container_key="runtime.semantic_memory",
+    ),
+    SubsystemModule(
+        name="graph_memory",
+        factory=_create_graph_tier,
+        container_key="runtime.graph_memory",
+        lifecycle_key="graph_memory",  # tier.initialize() → _init_db()
+        dependencies=["runtime.sqlite_pool"],
+    ),
+    SubsystemModule(
+        name="lineage_memory",
+        factory=_create_lineage_tier,
+        container_key="runtime.lineage_memory",
+        lifecycle_key="lineage_memory",  # tier.initialize() → _init_db()
+        dependencies=["runtime.sqlite_pool"],
+    ),
+    SubsystemModule(
+        name="lancedb_store",
+        factory=_create_lancedb_store,
+        container_key="runtime.lancedb_store",
+        lifecycle_key="lancedb_store",  # calls LanceDBStore.initialize()
+    ),
+    SubsystemModule(
+        name="embedding_pipeline",
+        factory=_create_embedding_pipeline,
+        container_key="runtime.embedding_pipeline",
+        lifecycle_key="embedding_pipeline",  # calls EmbeddingPipeline.initialize()
+        dependencies=["runtime.lancedb_store"],
+    ),
+    SubsystemModule(
+        name="semantic_memory_lance",
+        factory=_create_semantic_memory_lance,
+        container_key="runtime.semantic_memory_lance",
+        dependencies=["runtime.lancedb_store", "runtime.embedding_pipeline"],
+    ),
+    SubsystemModule(
+        name="episodic_session_memory",
+        factory=_create_episodic_session_memory,
+        container_key="runtime.episodic_session_memory",
+        lifecycle_key="episodic_session_memory",  # calls EpisodicMemory.initialize()
+        dependencies=["runtime.sqlite_pool"],
+    ),
+    SubsystemModule(
+        name="memory_lifecycle",
+        factory=_create_memory_lifecycle,
+        container_key="runtime.memory_lifecycle",
+        lifecycle_key="memory",
+        dependencies=[
+            "runtime.working_memory",
+            "runtime.episodic_memory",
+            "runtime.episodic_session_memory",
+            "runtime.semantic_memory_lance",
+            "runtime.embedding_pipeline",
+            "runtime.lineage_memory",
+        ],
+    ),
+]

velune/memory/prioritizer.py

@@ -0,0 +1,67 @@
+"""Memory Prioritization & Decay Engine.
+
+Calculates memory importance and decay rates using exponential decay
+models combined with usage-frequency boosts.
+"""
+
+from __future__ import annotations
+
+import math
+import time
+
+
+class MemoryPrioritizer:
+    """Manages importance calculations and temporal decay profiles for cognitive memories."""
+
+    def __init__(
+        self,
+        default_halflife_hours: float = 24.0,
+        retrieval_boost_factor: float = 0.15,
+        max_importance: float = 1.0,
+    ) -> None:
+        """
+        Initialize the prioritizer.
+
+        - default_halflife_hours: Hours for memory importance to decay by half.
+        - retrieval_boost_factor: Percentage boost added to importance on access.
+        """
+        self.default_halflife_hours = default_halflife_hours
+        self.retrieval_boost_factor = retrieval_boost_factor
+        self.max_importance = max_importance
+
+        # Compute decay constant (lambda = ln(2) / halflife)
+        # Convert halflife from hours to seconds
+        self.decay_constant = math.log(2.0) / (default_halflife_hours * 3600.0)
+
+    def calculate_decayed_score(self, initial_score: float, creation_timestamp: float) -> float:
+        """
+        Applies exponential decay over elapsed time.
+
+        S(t) = S_0 * e^(-lambda * t)
+        """
+        elapsed_seconds = max(0.0, time.time() - creation_timestamp)
+        decayed_score = initial_score * math.exp(-self.decay_constant * elapsed_seconds)
+        return max(0.0, min(self.max_importance, decayed_score))
+
+    def calculate_initial_importance(
+        self,
+        base_importance: float,
+        semantic_depth: float = 0.5,
+        context_fit: float = 0.5,
+    ) -> float:
+        """
+        Combines baseline importance with dynamic context factors.
+
+        Initial = 0.5 * base + 0.3 * semantic_depth + 0.2 * context_fit
+        """
+        score = (0.5 * base_importance) + (0.3 * semantic_depth) + (0.2 * context_fit)
+        return max(0.0, min(self.max_importance, score))
+
+    def apply_retrieval_boost(self, current_score: float) -> float:
+        """
+        Apply a retrieval boost when a node is fetched/replayed.
+        """
+        boosted_score = current_score + (
+            self.retrieval_boost_factor * (self.max_importance - current_score)
+        )
+        return max(0.0, min(self.max_importance, boosted_score))

velune/memory/storage/episodic_schema.sql

@@ -0,0 +1,53 @@
+-- Episodic Memory Schema — Phase 2a (storage layer)
+-- Applied incrementally by EpisodicMemory._apply_migrations().
+-- The Python class owns migration execution; this file is the canonical reference.
+
+-- ── Schema versioning ────────────────────────────────────────────────────────
+CREATE TABLE IF NOT EXISTS episodic_schema_version (
+    version    INTEGER PRIMARY KEY,
+    applied_at REAL    NOT NULL
+);
+
+-- ── Version 1 ────────────────────────────────────────────────────────────────
+
+-- One row per REPL session (from start_session() to end_session()).
+CREATE TABLE IF NOT EXISTS sessions (
+    id            TEXT    PRIMARY KEY,
+    workspace_root TEXT   NOT NULL,
+    started_at    REAL    NOT NULL,
+    ended_at      REAL,                         -- NULL while session is active
+    model_used    TEXT,
+    mode          TEXT,
+    total_tokens  INTEGER DEFAULT 0,
+    summary       TEXT                          -- LLM-generated; added at session end
+);
+
+-- One row per conversation turn (user or assistant message).
+CREATE TABLE IF NOT EXISTS turns (
+    id           TEXT    PRIMARY KEY,
+    session_id   TEXT    NOT NULL REFERENCES sessions(id),
+    turn_index   INTEGER NOT NULL,              -- 0-based position within session
+    role         TEXT    NOT NULL,              -- 'user' or 'assistant'
+    content      TEXT    NOT NULL,
+    model_used   TEXT,
+    tokens_used  INTEGER,
+    created_at   REAL    NOT NULL,
+    embedding_id TEXT                           -- set after embedding in Phase 2a-2
+);
+
+-- Structured annotations on turns (e.g. 'architectural_decision', 'bug_fix').
+CREATE TABLE IF NOT EXISTS memory_tags (
+    turn_id TEXT NOT NULL REFERENCES turns(id),
+    tag     TEXT NOT NULL,
+    value   TEXT,                               -- optional structured payload
+    PRIMARY KEY (turn_id, tag)
+);
+
+-- ── Indexes ───────────────────────────────────────────────────────────────────
+CREATE INDEX IF NOT EXISTS idx_sessions_workspace ON sessions(workspace_root);
+CREATE INDEX IF NOT EXISTS idx_sessions_started   ON sessions(started_at DESC);
+CREATE INDEX IF NOT EXISTS idx_turns_session      ON turns(session_id);
+CREATE INDEX IF NOT EXISTS idx_turns_created      ON turns(created_at);
+CREATE INDEX IF NOT EXISTS idx_turns_role         ON turns(role);
+CREATE INDEX IF NOT EXISTS idx_tags_turn          ON memory_tags(turn_id);
+CREATE INDEX IF NOT EXISTS idx_tags_tag           ON memory_tags(tag);

velune/memory/storage/lancedb_store.py

@@ -0,0 +1,282 @@
+"""LanceDB-backed vector store for semantic memory (Phase 2a).
+
+LanceDB is an embedded, serverless vector database — no daemon process is
+needed.  All heavy operations (open, add, search, delete) are pushed to a
+thread pool via ``asyncio.to_thread`` so they never block the event loop.
+
+Graceful degradation: if LanceDB or PyArrow is not installed, or if startup
+fails, ``_degraded`` is set to True and all operations become safe no-ops.
+Set ``VELUNE_SKIP_LANCEDB=1`` to force degraded mode for fast dev iteration.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import os
+import time
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+logger = logging.getLogger("velune.memory.storage.lancedb_store")
+
+# Default embedding dimension for nomic-embed-text via Ollama
+EMBEDDING_DIM: int = 768
+
+_TABLE_NAME = "memories"
+
+
+# ── Data models ───────────────────────────────────────────────────────────────
+
+
+@dataclass
+class MemoryRecord:
+    """A vector record to upsert into LanceDB."""
+
+    id: str
+    embedding: list[float]
+    content: str
+    source_type: str  # "turn_user", "turn_assistant", "session_summary"
+    session_id: str
+    turn_id: str
+    workspace_root: str
+    created_at: float
+    trust_score: float = 1.0
+
+
+@dataclass
+class SearchResult:
+    """A single result returned from a vector similarity search."""
+
+    id: str
+    content: str
+    source_type: str
+    distance: float
+    trust_score: float
+    session_id: str
+    turn_id: str
+    created_at: float = field(default_factory=time.time)
+
+
+# ── Store ─────────────────────────────────────────────────────────────────────
+
+
+class LanceDBStore:
+    """Async wrapper around a local LanceDB table for semantic memory storage.
+
+    Call ``await store.startup()`` (or ``await store.initialize()``) before
+    any read/write operations; the module lifecycle does this automatically.
+    """
+
+    def __init__(
+        self,
+        store_path: Path,
+        embedding_dim: int = EMBEDDING_DIM,
+    ) -> None:
+        self._store_path = store_path
+        self._embedding_dim = embedding_dim
+        self._db: Any = None
+        self._table: Any = None
+        self._degraded: bool = os.environ.get("VELUNE_SKIP_LANCEDB", "").lower() in (
+            "1",
+            "true",
+            "yes",
+        )
+        if self._degraded:
+            logger.warning("VELUNE_SKIP_LANCEDB set — LanceDB running in degraded mode.")
+
+    # ── Lifecycle ─────────────────────────────────────────────────────────────
+
+    async def startup(self) -> None:
+        """Open (or create) the LanceDB store and the memories table."""
+        if self._degraded:
+            return
+        self._store_path.mkdir(parents=True, exist_ok=True)
+        try:
+            await asyncio.to_thread(self._open_or_create)
+            logger.info("LanceDB store ready at %s (%d-dim)", self._store_path, self._embedding_dim)
+        except Exception as exc:
+            logger.error("LanceDB startup failed; falling back to degraded mode: %s", exc)
+            self._degraded = True
+
+    async def initialize(self) -> None:
+        """Lifecycle alias called by the module coordinator."""
+        await self.startup()
+
+    async def shutdown(self) -> None:
+        self._db = None
+        self._table = None
+
+    def _open_or_create(self) -> None:
+        import lancedb
+        import pyarrow as pa
+
+        self._db = lancedb.connect(str(self._store_path))
+        existing = self._db.table_names()
+
+        if _TABLE_NAME in existing:
+            self._table = self._db.open_table(_TABLE_NAME)
+            logger.debug("Opened existing LanceDB table '%s'", _TABLE_NAME)
+        else:
+            schema = pa.schema(
+                [
+                    pa.field("id", pa.utf8()),
+                    pa.field("embedding", pa.list_(pa.float32(), self._embedding_dim)),
+                    pa.field("content", pa.utf8()),
+                    pa.field("source_type", pa.utf8()),
+                    pa.field("session_id", pa.utf8()),
+                    pa.field("turn_id", pa.utf8()),
+                    pa.field("workspace_root", pa.utf8()),
+                    pa.field("created_at", pa.float64()),
+                    pa.field("trust_score", pa.float32()),
+                ]
+            )
+            self._table = self._db.create_table(_TABLE_NAME, schema=schema)
+            logger.info("Created LanceDB table '%s'", _TABLE_NAME)
+
+    # ── Write operations ──────────────────────────────────────────────────────
+
+    async def upsert(self, records: list[MemoryRecord]) -> None:
+        """Delete any existing records with matching IDs, then insert fresh copies."""
+        if self._degraded or not records or self._table is None:
+            return
+        try:
+            await asyncio.to_thread(self._upsert_sync, records)
+        except Exception as exc:
+            logger.error("LanceDB upsert failed: %s", exc)
+
+    def _upsert_sync(self, records: list[MemoryRecord]) -> None:
+        ids = [r.id for r in records]
+        ids_clause = ", ".join(f"'{_esc(i)}'" for i in ids)
+        # Best-effort delete; ignore errors on an empty table
+        try:
+            self._table.delete(f"id IN ({ids_clause})")
+        except Exception:
+            pass
+
+        rows = [
+            {
+                "id": r.id,
+                "embedding": [float(x) for x in r.embedding],
+                "content": r.content,
+                "source_type": r.source_type,
+                "session_id": r.session_id,
+                "turn_id": r.turn_id,
+                "workspace_root": r.workspace_root,
+                "created_at": float(r.created_at),
+                "trust_score": float(r.trust_score),
+            }
+            for r in records
+        ]
+        self._table.add(rows)
+
+    async def delete(self, ids: list[str]) -> None:
+        """Delete records by their string IDs."""
+        if self._degraded or not ids or self._table is None:
+            return
+        try:
+            await asyncio.to_thread(self._delete_sync, ids)
+        except Exception as exc:
+            logger.error("LanceDB delete failed: %s", exc)
+
+    def _delete_sync(self, ids: list[str]) -> None:
+        ids_clause = ", ".join(f"'{_esc(i)}'" for i in ids)
+        self._table.delete(f"id IN ({ids_clause})")
+
+    async def prune_by_trust(self, threshold: float) -> int:
+        """Delete records whose trust_score falls below *threshold*.  Returns count deleted."""
+        if self._degraded or self._table is None:
+            return 0
+        try:
+            return await asyncio.to_thread(self._prune_sync, threshold)
+        except Exception as exc:
+            logger.error("LanceDB prune failed: %s", exc)
+            return 0
+
+    def _prune_sync(self, threshold: float) -> int:
+        before = len(self._table)
+        self._table.delete(f"trust_score < {threshold}")
+        after = len(self._table)
+        return max(0, before - after)
+
+    # ── Search ─────────────────────────────────────────────────────────────────
+
+    async def search(
+        self,
+        embedding: list[float],
+        limit: int = 10,
+        workspace_root: str | None = None,
+    ) -> list[SearchResult]:
+        """ANN search over the memories table, optionally pre-filtered by workspace."""
+        if self._degraded or self._table is None:
+            return []
+        try:
+            return await asyncio.to_thread(self._search_sync, embedding, limit, workspace_root)
+        except Exception as exc:
+            logger.error("LanceDB search failed: %s", exc)
+            return []
+
+    def _search_sync(
+        self,
+        embedding: list[float],
+        limit: int,
+        workspace_root: str | None,
+    ) -> list[SearchResult]:
+        import numpy as np
+
+        query = np.array(embedding, dtype=np.float32)
+        q = self._table.search(query, vector_column_name="embedding")
+
+        if workspace_root:
+            safe = _esc(workspace_root)
+            q = q.where(f"workspace_root = '{safe}'", prefilter=True)
+
+        rows = q.limit(limit).to_list()
+        results: list[SearchResult] = []
+        for row in rows:
+            results.append(
+                SearchResult(
+                    id=row["id"],
+                    content=row["content"],
+                    source_type=row["source_type"],
+                    distance=float(row.get("_distance", 0.0)),
+                    trust_score=float(row["trust_score"]),
+                    session_id=row["session_id"],
+                    turn_id=row["turn_id"],
+                    created_at=float(row.get("created_at", 0.0)),
+                )
+            )
+        return results
+
+    # ── Properties ────────────────────────────────────────────────────────────
+
+    @property
+    def table_size(self) -> int:
+        if self._degraded or self._table is None:
+            return 0
+        try:
+            return len(self._table)
+        except Exception:
+            return 0
+
+    @property
+    def is_healthy(self) -> bool:
+        return not self._degraded and self._table is not None
+
+    def health_check(self) -> dict:
+        return {
+            "healthy": self.is_healthy,
+            "degraded": self._degraded,
+            "store_path": str(self._store_path),
+            "embedding_dim": self._embedding_dim,
+            "table_size": self.table_size,
+        }
+
+
+# ── Helpers ───────────────────────────────────────────────────────────────────
+
+
+def _esc(s: str) -> str:
+    """Escape single quotes for safe SQL string literals in LanceDB WHERE clauses."""
+    return s.replace("'", "''")