superlocalmemory 4.1.13 → 4.1.14

package/src/superlocalmemory/server/routes/helpers.py

@@ -1,499 +0,0 @@
-# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
-# Licensed under AGPL-3.0-or-later - see LICENSE file
-# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
-"""SuperLocalMemory V3 - Route Helpers
- - AGPL-3.0-or-later
-
-Shared utilities for all route modules: DB connection, dict factory,
-profile helper, validation, Pydantic models, config paths, and the
-shared lazy engine accessor used by every engine-dependent route.
-"""
-
-import json
-import logging
-import re
-import sqlite3
-import threading
-import time
-from datetime import datetime, timezone
-from pathlib import Path
-from typing import Optional
-
-from fastapi import HTTPException, Request
-from pydantic import BaseModel, Field
-
-from superlocalmemory.infra.data_root import DynamicStatePath, canonical_data_root
-from superlocalmemory.storage.memory_write import memory_read, memory_write
-
-_engine_logger = logging.getLogger("superlocalmemory.engine")
-
-
-# ---------------------------------------------------------------------------
-# Version detection (shared — avoids circular import between ui.py ↔ v3_api.py)
-# ---------------------------------------------------------------------------
-
-
-def _get_version() -> str:
-    """Read version from package.json / pyproject.toml / importlib.
-
-    Walks up from this file to find the project root. In the src layout
-    (running from source tree), package.json is 5 parents up; for an
-    installed package it won't exist, so we fall through to importlib.
-    """
-    here = Path(__file__).resolve()
-    for depth in (5, 4):
-        try:
-            import json as _json
-
-            root = here
-            for _ in range(depth):
-                root = root.parent
-            pkg_json = root / "package.json"
-            if pkg_json.exists():
-                with open(pkg_json) as f:
-                    v = _json.load(f).get("version", "")
-                    if v:
-                        return v
-            toml_path = root / "pyproject.toml"
-            if toml_path.exists():
-                import tomllib
-
-                with open(toml_path, "rb") as f:
-                    return tomllib.load(f)["project"]["version"]
-        except Exception:
-            continue
-    try:
-        from importlib.metadata import version
-
-        return version("superlocalmemory")
-    except Exception:
-        pass
-    return "unknown"
-
-
-SLM_VERSION = _get_version()
-
-
-def _resolve_slm_home() -> Path:
-    """Resolve the SLM data dir the SAME way the CLI does.
-
-    The CLI honours ``SLM_DATA_DIR`` → ``SL_MEMORY_PATH`` → ``SLM_HOME``
-    → ``~/.superlocalmemory`` (see ``cli/_lazy_init.py:slm_home``). The
-    dashboard previously hardcoded ``Path.home() / ".superlocalmemory"``,
-    so when a user pointed the CLI at a custom data dir (or set a
-    different ``base_dir`` in ``config.json``), profiles created in the
-    dashboard landed in the default dir while ``slm profile list`` read a
-    different ``memory.db`` and reported the profile missing.
-
-    This function mirrors ``slm_home()`` so both surfaces agree. It is
-    evaluated lazily (at call time inside the properties below) rather
-    than at import, so a process that sets the env var after import —
-    e.g. tests, or a launcher that exports it late — still sees the
-    right path.
-    """
-    return canonical_data_root()
-
-
-# V3 paths (migrated from ~/.claude-memory to ~/.superlocalmemory).
-# Exposed as dynamic module-level path-like objects for backward compatibility
-# with route modules that import them. They never cache an earlier namespace.
-MEMORY_DIR = DynamicStatePath()
-DB_PATH = DynamicStatePath("memory.db")
-UI_DIR = Path(__file__).parent.parent / "ui"
-PROFILES_DIR = DynamicStatePath("profiles")
-
-
-# ---------------------------------------------------------------------------
-# Engine lifecycle — lazy, thread-safe, recoverable after mode changes.
-# ---------------------------------------------------------------------------
-
-_engine_init_lock = threading.Lock()
-_last_engine_failure: float = 0.0
-_ENGINE_FAILURE_COOLDOWN_S: float = 5.0
-
-
-def get_engine_lazy(app_state):
-    """Return ``app_state.engine``, initialising it if None.
-
-    Why this exists
-    ---------------
-    Mode-change endpoints (``PUT`` / ``POST /api/v3/mode[/set]``) set
-    ``app.state.engine = None`` so the next request picks up the new config.
-    Before this helper, no code path re-created the engine until the daemon
-    restarted, breaking every engine-backed route (entity, ingest, tiers,
-    recall, remember, list, status).
-
-    Contract
-    --------
-    * Returns the cached engine if already initialised.
-    * Otherwise acquires a process-wide lock and builds a fresh
-      ``MemoryEngine`` from the latest ``SLMConfig.load()``.
-    * Returns ``None`` if init fails. A brief cooldown prevents hammering
-      init when the underlying cause (e.g., corrupt DB) is persistent.
-    * Never uses a sticky "already attempted" flag — recovery must be
-      automatic after a transient failure.
-    """
-    global _last_engine_failure
-
-    engine = getattr(app_state, "engine", None)
-    if engine is not None:
-        return engine
-
-    now = time.monotonic()
-    if _last_engine_failure and (now - _last_engine_failure) < _ENGINE_FAILURE_COOLDOWN_S:
-        return None
-
-    with _engine_init_lock:
-        # Double-checked: another thread may have initialised while we waited.
-        engine = getattr(app_state, "engine", None)
-        if engine is not None:
-            return engine
-        try:
-            from superlocalmemory.core.config import SLMConfig
-            from superlocalmemory.core.engine import MemoryEngine
-
-            config = SLMConfig.load()
-            new_engine = MemoryEngine(config)
-            new_engine.initialize()
-            app_state.engine = new_engine
-            app_state.config = config
-            _engine_logger.info(
-                "Engine lazy-initialised (mode=%s, profile=%s)",
-                getattr(getattr(config, "mode", None), "value", "?"),
-                getattr(config, "active_profile", "?"),
-            )
-            _last_engine_failure = 0.0
-            return new_engine
-        except Exception as exc:
-            _engine_logger.warning("Engine lazy init failed: %s", exc)
-            _last_engine_failure = time.monotonic()
-            return None
-
-
-def require_engine(request: Request):
-    """Return the engine or raise ``HTTPException(503)``.
-
-    Use this in every route that touches ``app.state.engine``. Replaces the
-    old ``engine = request.app.state.engine; if engine is None: raise ...``
-    pattern with a single call that also lazily re-initialises after a mode
-    change.
-    """
-    engine = get_engine_lazy(request.app.state)
-    if engine is None:
-        raise HTTPException(
-            status_code=503,
-            detail="Engine not initialized. Retry in a few seconds — it's warming up.",
-        )
-    return engine
-
-
-def log_mode_change(
-    old_mode: str,
-    new_mode: str,
-    *,
-    provider: str = "",
-    model: str = "",
-    source: str = "api",
-) -> None:
-    """Append an audit entry for every mode change.
-
-    Lets us trace phantom writes (e.g., a stray dashboard-card button click
-    silently flipping the system to Mode C with ``anthropic/claude-sonnet-4``
-    as the auto-defaulted model).
-
-    Audit file: ``<base_dir>/logs/mode-audit.log`` (tab-separated).
-    """
-    audit_path = MEMORY_DIR / "logs" / "mode-audit.log"
-    try:
-        audit_path.parent.mkdir(parents=True, exist_ok=True)
-        ts = datetime.now(timezone.utc).isoformat(timespec="seconds")
-        line = (
-            f"{ts}\told={old_mode}\tnew={new_mode}"
-            f"\tprovider={provider}\tmodel={model}\tsource={source}\n"
-        )
-        with open(audit_path, "a", encoding="utf-8") as handle:
-            handle.write(line)
-    except Exception as exc:  # Logging must never break the mode-change path.
-        _engine_logger.warning("mode audit write failed: %s", exc)
-
-    _engine_logger.info(
-        "Mode change: %s→%s provider=%s model=%s (%s)",
-        old_mode,
-        new_mode,
-        provider,
-        model,
-        source,
-    )
-
-
-class _RouteReadConnection:
-    """Compatibility wrapper for legacy route callers that close manually.
-
-    New routes should prefer ``with memory_read(path)``.  A few shared route
-    callers still expect ``get_db_connection()`` to return a connection they
-    can close themselves, so this wrapper preserves that contract without
-    reopening canonical ``memory.db`` in writable mode.
-    """
-
-    def __init__(self, db_path: Path) -> None:
-        object.__setattr__(self, "_snapshot", memory_read(db_path))
-        object.__setattr__(self, "_connection", self._snapshot.__enter__())
-        object.__setattr__(self, "_closed", False)
-
-    def __getattr__(self, name: str):
-        return getattr(self._connection, name)
-
-    def __setattr__(self, name: str, value) -> None:
-        if name.startswith("_"):
-            object.__setattr__(self, name, value)
-        else:
-            setattr(self._connection, name, value)
-
-    def close(self) -> None:
-        if not self._closed:
-            self._snapshot.__exit__(None, None, None)
-            object.__setattr__(self, "_closed", True)
-
-
-def get_read_connection(db_path: Path = DB_PATH) -> _RouteReadConnection:
-    """Return a legacy-compatible, physically read-only canonical connection."""
-    if not db_path.exists():
-        raise HTTPException(
-            status_code=500,
-            detail="Memory database not found. Run 'slm init' to initialize.",
-        )
-    return _RouteReadConnection(db_path)
-
-
-def get_db_connection() -> _RouteReadConnection:
-    """Return the shared dashboard read connection for canonical ``memory.db``."""
-    return get_read_connection(DB_PATH)
-
-
-def dict_factory(cursor: sqlite3.Cursor, row: tuple) -> dict:
-    """Convert SQLite row to dictionary."""
-    fields = [column[0] for column in cursor.description]
-    return dict(zip(fields, row))
-
-
-def get_active_profile() -> str:
-    """Read request runtime truth, falling back to the compatibility cache."""
-    from superlocalmemory.server.profile_runtime import current_request_profile
-
-    runtime_profile = current_request_profile()
-    if runtime_profile:
-        return runtime_profile
-    config_file = MEMORY_DIR / "profiles.json"
-    if config_file.exists():
-        try:
-            with open(config_file, "r") as f:
-                pconfig = json.load(f)
-            return pconfig.get("active_profile", "default")
-        except (json.JSONDecodeError, IOError):
-            pass
-    return "default"
-
-
-def validate_profile_name(name: str) -> bool:
-    """Validate profile name (alphanumeric, underscore, hyphen only)."""
-    return bool(re.match(r"^[a-zA-Z0-9_-]+$", name))
-
-
-# ============================================================================
-# Profile Sync — SQLite as single source of truth
-# ============================================================================
-
-
-def ensure_profile_in_db(name: str, description: str = "") -> None:
-    """Ensure a profile row exists in SQLite (idempotent).
-
-    Hot path: called on every authenticated request.  Uses ``memory_write()``
-    so in-process writers serialise through the write lock and cross-process
-    writers (hooks / CLI) wait via PRAGMA busy_timeout instead of getting
-    SQLITE_BUSY.
-    """
-    if not DB_PATH.exists():
-        return
-    with memory_write(DB_PATH) as conn:
-        conn.execute("PRAGMA foreign_keys=ON")
-        conn.execute(
-            "INSERT OR IGNORE INTO profiles (profile_id, name, description) VALUES (?, ?, ?)",
-            (name, name, description or f"Memory profile: {name}"),
-        )
-
-
-def ensure_profile_in_json(name: str, description: str = "") -> None:
-    """Ensure a profile entry exists in profiles.json (idempotent)."""
-    from datetime import datetime
-
-    config = _load_profiles_json()
-    profiles = config.get("profiles", {})
-    if name not in profiles:
-        profiles[name] = {
-            "name": name,
-            "description": description or f"Memory profile: {name}",
-            "created_at": datetime.now(timezone.utc).isoformat(),
-            "last_used": None,
-        }
-        config["profiles"] = profiles
-        _save_profiles_json(config)
-
-
-def sync_profiles() -> list[dict]:
-    """Reconcile SQLite and profiles.json. Returns merged profile list.
-
-    SQLite is the source of truth. Uses ``profile_id`` (not ``name``)
-    as the canonical key because profile_id is the PK referenced by
-    every FK in the database.
-    """
-    db_profiles = _get_db_profiles()
-    json_config = _load_profiles_json()
-    json_profiles = json_config.get("profiles", {})
-
-    # profile_id is the canonical key (PK in SQLite, FK target everywhere)
-    db_ids = {p["profile_id"] for p in db_profiles}
-    json_keys = set(json_profiles.keys())
-
-    changed = False
-
-    # JSON-only → add to SQLite (fixes Dashboard-created profiles)
-    for key in json_keys - db_ids:
-        ensure_profile_in_db(key, json_profiles[key].get("description", ""))
-
-    # SQLite-only → add to profiles.json (fixes CLI-created profiles)
-    for pid in db_ids - json_keys:
-        db_entry = next(p for p in db_profiles if p["profile_id"] == pid)
-        json_profiles[pid] = {
-            "name": pid,
-            "description": db_entry.get("description", ""),
-            "created_at": db_entry.get("created_at", ""),
-            "last_used": db_entry.get("last_used"),
-        }
-        changed = True
-
-    if changed:
-        json_config["profiles"] = json_profiles
-        _save_profiles_json(json_config)
-
-    # Return merged list from SQLite (now authoritative)
-    return _get_db_profiles()
-
-
-def set_active_profile_everywhere(name: str) -> None:
-    """Persist active profile through the crash-safe compatibility writer."""
-    from superlocalmemory.server.profile_runtime import persist_active_profile
-
-    persist_active_profile(name)
-
-
-def delete_profile_from_db(name: str) -> None:
-    """Delete a profile row from SQLite.
-
-    rbac_memberships has no FK to profiles, so CASCADE does not remove role
-    grants — they would otherwise survive deletion and silently re-activate if
-    a profile of the same name is later recreated. Remove them explicitly.
-    Uses ``memory_write()`` so the multi-statement DELETE is atomic and
-    serialised against other in-process writers.
-    """
-    if not DB_PATH.exists():
-        return
-    # Receipt evidence lives in learning.db, not recall's memory.db.  Purge it
-    # first: if this fails the profile row must survive for a retry rather than
-    # leaving profile-scoped learning evidence orphaned.
-    from superlocalmemory.storage.agent_experience import purge_profile_receipts
-
-    purge_profile_receipts(Path(DB_PATH).parent / "learning.db", name)
-    with memory_write(DB_PATH) as conn:
-        conn.execute("PRAGMA foreign_keys=ON")
-        # Purge role grants for this workspace (no FK CASCADE covers these).
-        for tbl in ("rbac_memberships",):
-            try:
-                conn.execute(f"DELETE FROM {tbl} WHERE profile_id = ?", (name,))
-            except sqlite3.OperationalError:
-                pass  # table may not exist on older installs
-        conn.execute("DELETE FROM profiles WHERE profile_id = ?", (name,))
-
-
-def _get_db_profiles() -> list[dict]:
-    """Read all profiles from SQLite."""
-    if not DB_PATH.exists():
-        return []
-    try:
-        with memory_read(DB_PATH) as conn:
-            rows = conn.execute(
-                "SELECT profile_id, name, description, created_at, last_used "
-                "FROM profiles ORDER BY name"
-            ).fetchall()
-            return [dict(r) for r in rows]
-    except sqlite3.OperationalError:
-        return []
-
-
-def _load_profiles_json() -> dict:
-    """Load profiles.json config (Dashboard dict format)."""
-    config_file = MEMORY_DIR / "profiles.json"
-    if config_file.exists():
-        try:
-            with open(config_file, "r") as f:
-                data = json.load(f)
-            # Handle ProfileManager array format → convert to dict format
-            if isinstance(data.get("profiles"), list):
-                converted = {}
-                for p in data["profiles"]:
-                    n = p.get("name", "")
-                    if n:
-                        converted[n] = p
-                data["profiles"] = converted
-                if "active" in data and "active_profile" not in data:
-                    data["active_profile"] = data.pop("active")
-            return data
-        except (json.JSONDecodeError, IOError):
-            pass
-    return {
-        "profiles": {"default": {"name": "default", "description": "Default memory profile"}},
-        "active_profile": "default",
-    }
-
-
-def _save_profiles_json(config: dict) -> None:
-    """Save profiles.json config."""
-    MEMORY_DIR.mkdir(parents=True, exist_ok=True)
-    config_file = MEMORY_DIR / "profiles.json"
-    with open(config_file, "w") as f:
-        json.dump(config, f, indent=2)
-
-
-# ============================================================================
-# Pydantic Models (shared across routes)
-# ============================================================================
-
-
-class SearchRequest(BaseModel):
-    """Advanced search request model."""
-
-    query: str = Field(..., min_length=1, max_length=1000)
-    limit: int = Field(default=10, ge=1, le=100)
-    min_score: float = Field(default=0.3, ge=0.0, le=1.0)
-    category: Optional[str] = None
-    project_name: Optional[str] = None
-    cluster_id: Optional[int] = None
-    date_from: Optional[str] = None
-    date_to: Optional[str] = None
-    # T-window: relative span ("7d", "30d", "1y") or explicit range
-    # ("2026-07-01..2026-07-31"). When empty, date_from/date_to (if both set)
-    # are used as the range. Empty + no dates = no time filter.
-    window: Optional[str] = None
-
-
-class ProfileSwitch(BaseModel):
-    """Profile switching request."""
-
-    profile_name: str = Field(..., min_length=1, max_length=50)
-
-
-class BackupConfigRequest(BaseModel):
-    """Backup configuration update request."""
-
-    interval_hours: Optional[int] = Field(None, ge=1, le=8760)
-    max_backups: Optional[int] = Field(None, ge=1, le=100)
-    enabled: Optional[bool] = None

package/src/superlocalmemory/server/routes/ingest.py

@@ -1,137 +0,0 @@
-# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
-# Licensed under AGPL-3.0-or-later - see LICENSE file
-# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
-
-"""Ingestion endpoint — accepts data from external adapters.
-
-POST /ingest with {content, source_type, dedup_key, metadata}.
-Deduplicates by source_type + dedup_key. Stores via MemoryEngine.
-Admission control: max 10 concurrent ingestions (HTTP 429 on overflow).
-
-Part of Qualixar | Author: Varun Pratap Bhardwaj
-"""
-
-from __future__ import annotations
-
-import json
-import threading
-from datetime import datetime, timezone
-
-from fastapi import APIRouter, HTTPException, Request
-from pydantic import BaseModel, Field
-
-router = APIRouter(tags=["ingestion"])
-
-_MAX_CONCURRENT = 10
-_active_count = 0
-_active_lock = threading.Lock()
-
-
-class IngestRequest(BaseModel):
-    # Cap ingest content (~1 MB) so a single call cannot exhaust memory through
-    # the embedding/tokenizer pipeline.
-    content: str = Field(..., max_length=1_000_000)
-    source_type: str
-    dedup_key: str
-    metadata: dict = {}
-
-
-@router.post("/ingest")
-async def ingest(req: IngestRequest, request: Request):
-    """Ingest content from an external adapter.
-
-    Deduplicates by (source_type, dedup_key). Returns 429 if too many
-    concurrent ingestions. Stores via the singleton MemoryEngine.
-    """
-    global _active_count
-
-    from .helpers import require_engine
-    engine = require_engine(request)
-
-    if not req.content:
-        raise HTTPException(400, detail="content required")
-    if not req.source_type:
-        raise HTTPException(400, detail="source_type required")
-    if not req.dedup_key:
-        raise HTTPException(400, detail="dedup_key required")
-
-    # Admission control
-    with _active_lock:
-        if _active_count >= _MAX_CONCURRENT:
-            raise HTTPException(
-                429,
-                detail="Too many concurrent ingestions",
-                headers={"Retry-After": "5"},
-            )
-        _active_count += 1
-
-    try:
-        from superlocalmemory.core.engine_ingestion import (
-            build_engine_ingestion_command,
-        )
-        from superlocalmemory.core.ingestion_command import (
-            IngestionRequest,
-            IngestionState,
-        )
-
-        from superlocalmemory.server.write_identity import (
-            authenticated_request_actor,
-        )
-        actor_id = authenticated_request_actor(
-            request,
-            actor_kind="http-ingest",
-        )
-        # Machine authentication says which process is calling. It does not say
-        # whether that caller's user may write to this workspace. Every other
-        # write route asks that question; this one did not, so a viewer could
-        # write through it in a workspace that requires a login.
-        from superlocalmemory.access.rbac import Permission
-        from superlocalmemory.server.rbac_enforce import require_permission
-
-        require_permission(request, Permission.WRITE, profile=engine._profile_id)
-        command = build_engine_ingestion_command(engine)
-        receipt, created = command.submit_with_status(IngestionRequest(
-            content=req.content,
-            profile_id=engine._profile_id,
-            source_type=req.source_type,
-            idempotency_key=req.dedup_key,
-            metadata=dict(req.metadata),
-            trusted_actor_id=actor_id,
-        ))
-        completed = command.materialize(receipt.operation_id)
-        if completed.state is not IngestionState.COMPLETE:
-            raise RuntimeError(
-                completed.last_error or "canonical adapter ingestion failed"
-            )
-        fact_ids = list(completed.fact_ids)
-
-        # Compatibility ledger for existing dashboards. M018 is the source of
-        # truth; retries repair this row without repeating the canonical write.
-        engine._db.execute(
-            "INSERT OR IGNORE INTO ingestion_log "
-            "(profile_id, source_type, dedup_key, fact_ids, metadata, status, ingested_at) "
-            "VALUES (?, ?, ?, ?, ?, 'ingested', ?)",
-            (
-                engine._profile_id,
-                req.source_type,
-                req.dedup_key,
-                json.dumps(fact_ids),
-                json.dumps(req.metadata),
-                datetime.now(timezone.utc).isoformat(),
-            ),
-        )
-
-        result = {
-            "ingested": created,
-            "operation_id": completed.operation_id,
-            "fact_ids": fact_ids,
-        }
-        if not created:
-            result["reason"] = "already_ingested"
-        return result
-
-    except Exception as exc:
-        raise HTTPException(500, detail=str(exc))
-    finally:
-        with _active_lock:
-            _active_count -= 1