superlocalmemory 4.1.13 → 4.1.14

package/src/superlocalmemory/core/health_monitor.py

@@ -1,338 +0,0 @@
-# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
-# Licensed under AGPL-3.0-or-later - see LICENSE file
-# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
-
-"""Enterprise-grade health monitoring for the SLM Unified Daemon.
-
-Monitors:
-  - Global RSS budget (kill heaviest worker if over limit)
-  - Worker heartbeat (kill unresponsive workers after 60s)
-  - Structured JSON logging (daemon.json.log alongside text logs)
-  - Extensible health check registry (Phase C/D/E add checks)
-
-Part of Qualixar | Author: Varun Pratap Bhardwaj
-License: AGPL-3.0-or-later
-"""
-
-from __future__ import annotations
-
-import json
-import logging
-import os
-import threading
-import time
-from datetime import datetime, timezone
-from pathlib import Path
-from typing import Callable
-
-from superlocalmemory.infra.data_root import state_path
-
-logger = logging.getLogger("superlocalmemory.health_monitor")
-
-# Try psutil — graceful fallback if not available
-try:
-    import psutil
-    PSUTIL_AVAILABLE = True
-except ImportError:
-    PSUTIL_AVAILABLE = False
-    logger.info("psutil not available — health monitoring limited")
-
-
-# ---------------------------------------------------------------------------
-# Health Check Registry (extensible by other phases)
-# ---------------------------------------------------------------------------
-
-_HEALTH_CHECKS: list[Callable[[], dict]] = []
-
-
-def register_health_check(check_fn: Callable[[], dict]) -> None:
-    """Register a health check function. Returns dict with name, status, detail."""
-    _HEALTH_CHECKS.append(check_fn)
-
-
-def run_all_health_checks() -> list[dict]:
-    """Run all registered health checks. Returns list of results."""
-    results = []
-    for check_fn in _HEALTH_CHECKS:
-        try:
-            results.append(check_fn())
-        except Exception as e:
-            results.append({
-                "name": getattr(check_fn, '__name__', 'unknown'),
-                "status": "error",
-                "detail": str(e),
-            })
-    return results
-
-
-# ---------------------------------------------------------------------------
-# Structured JSON Logger (additive — does NOT replace text logs)
-# ---------------------------------------------------------------------------
-
-_json_logger: logging.Logger | None = None
-
-
-def setup_structured_logging(log_dir: Path | None = None) -> None:
-    """Set up JSON structured logging alongside existing text logs.
-
-    Creates a separate daemon.json.log file with RotatingFileHandler.
-    Text logs continue working unchanged.
-    """
-    global _json_logger
-
-    log_dir = log_dir or state_path("logs")
-    log_dir.mkdir(parents=True, exist_ok=True)
-    json_log_path = log_dir / "daemon.json.log"
-
-    _json_logger = logging.getLogger("superlocalmemory.structured")
-    _json_logger.setLevel(logging.INFO)
-    _json_logger.propagate = False  # Don't send to text handler
-
-    from logging.handlers import RotatingFileHandler
-    handler = RotatingFileHandler(
-        str(json_log_path), maxBytes=10 * 1024 * 1024, backupCount=5,
-    )
-    handler.setFormatter(logging.Formatter("%(message)s"))
-    _json_logger.addHandler(handler)
-
-
-def log_structured(**fields) -> None:
-    """Emit a structured JSON log entry.
-
-    Always includes: timestamp, level. Caller provides the rest.
-    Example fields: worker_pid, memory_rss_mb, operation, latency_ms, message.
-    """
-    if _json_logger is None:
-        return
-    entry = {
-        "timestamp": datetime.now(timezone.utc).isoformat(),
-        "level": fields.pop("level", "info"),
-        **fields,
-    }
-    try:
-        _json_logger.info(json.dumps(entry, default=str))
-    except Exception:
-        pass
-
-
-# ---------------------------------------------------------------------------
-# Health Monitor Thread
-# ---------------------------------------------------------------------------
-
-class HealthMonitor:
-    """Background thread monitoring worker health, RSS budget, heartbeats.
-
-    Self-healing: if the monitor itself crashes, it logs and retries
-    with exponential backoff (max 5 min sleep).
-    """
-
-    # SLM worker command-line identifiers for child process filtering
-    _WORKER_IDENTIFIERS = (
-        "superlocalmemory.core.embedding_worker",
-        "superlocalmemory.core.reranker_worker",
-        "superlocalmemory.core.recall_worker",
-    )
-    # Workers that are LOAD-BEARING for recall quality — never kill these
-    # first; prefer killing the reranker (gracefully degrades) or GC instead.
-    _EMBEDDING_IDENTIFIER = "superlocalmemory.core.embedding_worker"
-
-    def __init__(
-        self,
-        global_rss_budget_mb: int = 0,
-        heartbeat_timeout_sec: int = 60,
-        check_interval_sec: int = 15,
-        enable_structured_logging: bool = True,
-    ):
-        # Compute RAM-scaled default when 0 is passed (or when the caller
-        # explicitly passes 0 meaning "auto"). Floor at 2500 so low-RAM boxes
-        # keep the old conservative behaviour.
-        if global_rss_budget_mb <= 0:
-            if PSUTIL_AVAILABLE:
-                phys_mb = psutil.virtual_memory().total // (1024 * 1024)
-                global_rss_budget_mb = max(2500, int(phys_mb * 0.40))
-            else:
-                global_rss_budget_mb = 8000  # safe fallback when psutil absent
-        self._budget_mb = global_rss_budget_mb
-        self._heartbeat_timeout = heartbeat_timeout_sec
-        self._interval = check_interval_sec
-        self._enable_logging = enable_structured_logging
-        self._thread: threading.Thread | None = None
-        self._stop_event = threading.Event()
-        self._consecutive_failures = 0
-
-    def start(self) -> None:
-        """Start the health monitor in a daemon thread."""
-        if not PSUTIL_AVAILABLE:
-            logger.warning("Health monitor disabled: psutil not installed")
-            return
-
-        if self._enable_logging:
-            setup_structured_logging()
-
-        self._thread = threading.Thread(
-            target=self._run_loop, daemon=True, name="health-monitor",
-        )
-        self._thread.start()
-        logger.info("Health monitor started (budget=%dMB, heartbeat=%ds)",
-                     self._budget_mb, self._heartbeat_timeout)
-
-        # Register built-in health checks
-        register_health_check(self._check_daemon_health)
-        register_health_check(self._check_worker_health)
-        register_health_check(self._check_memory_budget)
-
-    def stop(self) -> None:
-        self._stop_event.set()
-
-    def _run_loop(self) -> None:
-        """Main monitoring loop with self-healing."""
-        while not self._stop_event.is_set():
-            try:
-                self._check_once()
-                self._consecutive_failures = 0
-            except Exception as exc:
-                self._consecutive_failures += 1
-                backoff = min(300, 30 * self._consecutive_failures)
-                logger.error("Health check failed (%d consecutive): %s. Backoff %ds.",
-                             self._consecutive_failures, exc, backoff)
-                log_structured(
-                    level="error", operation="health_check",
-                    message=f"Health check failed: {exc}",
-                    consecutive_failures=self._consecutive_failures,
-                )
-                self._stop_event.wait(backoff)
-                continue
-
-            self._stop_event.wait(self._interval)
-
-    def _check_once(self) -> None:
-        """Single health check cycle."""
-        proc = psutil.Process(os.getpid())
-        daemon_rss_mb = proc.memory_info().rss / (1024 * 1024)
-
-        # Find SLM worker children only (not adapters or other children)
-        children = proc.children(recursive=True)
-        slm_workers = []
-        for child in children:
-            try:
-                cmdline = " ".join(child.cmdline()).lower()
-                if any(ident in cmdline for ident in self._WORKER_IDENTIFIERS):
-                    rss_mb = child.memory_info().rss / (1024 * 1024)
-                    slm_workers.append({
-                        "pid": child.pid,
-                        "rss_mb": round(rss_mb, 1),
-                        "cmdline": cmdline[:200],
-                    })
-            except (psutil.NoSuchProcess, psutil.AccessDenied):
-                continue
-
-        total_rss_mb = daemon_rss_mb + sum(w["rss_mb"] for w in slm_workers)
-
-        # Structured log entry
-        log_structured(
-            level="info",
-            operation="health_check",
-            memory_rss_mb=round(daemon_rss_mb, 1),
-            total_rss_mb=round(total_rss_mb, 1),
-            worker_count=len(slm_workers),
-            workers=slm_workers,
-            budget_mb=self._budget_mb,
-        )
-
-        # RSS budget enforcement — spare the embedding worker (load-bearing for
-        # recall quality). Kill the reranker first (degrades gracefully); only
-        # fall back to the embedder if it is the only worker remaining.
-        if total_rss_mb > self._budget_mb and slm_workers:
-            non_embedder = [
-                w for w in slm_workers
-                if self._EMBEDDING_IDENTIFIER not in w["cmdline"]
-            ]
-            candidate = (
-                max(non_embedder, key=lambda w: w["rss_mb"])
-                if non_embedder
-                else max(slm_workers, key=lambda w: w["rss_mb"])
-            )
-            logger.warning(
-                "RSS budget exceeded (%.0fMB > %dMB). Killing worker PID %d (%.0fMB)",
-                total_rss_mb, self._budget_mb, candidate["pid"], candidate["rss_mb"],
-            )
-            log_structured(
-                level="warning",
-                operation="rss_budget_kill",
-                killed_pid=candidate["pid"],
-                killed_rss_mb=candidate["rss_mb"],
-                total_rss_mb=round(total_rss_mb, 1),
-                spared_embedder=bool(non_embedder),
-            )
-            try:
-                psutil.Process(candidate["pid"]).terminate()
-            except psutil.NoSuchProcess:
-                pass
-
-        # Heartbeat checks delegated to WorkerPool (Phase B wiring)
-        # WorkerPool tracks last_heartbeat per worker. HealthMonitor
-        # reads it here. Actual heartbeat protocol is in worker_pool.py.
-        try:
-            from superlocalmemory.core.worker_pool import WorkerPool
-            pool = WorkerPool.shared()
-            last_hb = getattr(pool, '_last_heartbeat', {})
-            now = time.monotonic()
-            for wpid, last_time in list(last_hb.items()):
-                if now - last_time > self._heartbeat_timeout:
-                    logger.warning("Worker PID %d unresponsive (no heartbeat for %ds). Killing.",
-                                   wpid, int(now - last_time))
-                    log_structured(
-                        level="warning",
-                        operation="heartbeat_kill",
-                        worker_pid=wpid,
-                        seconds_since_heartbeat=round(now - last_time),
-                    )
-                    try:
-                        psutil.Process(wpid).terminate()
-                    except psutil.NoSuchProcess:
-                        pass
-                    del last_hb[wpid]
-        except Exception:
-            pass  # WorkerPool not initialized yet — fine
-
-    # -- Built-in health checks for registry --
-
-    def _check_daemon_health(self) -> dict:
-        if not PSUTIL_AVAILABLE:
-            return {"name": "daemon", "status": "unknown", "detail": "psutil unavailable"}
-        proc = psutil.Process(os.getpid())
-        rss_mb = proc.memory_info().rss / (1024 * 1024)
-        return {
-            "name": "daemon",
-            "status": "ok" if rss_mb < 500 else "warning",
-            "detail": f"PID {os.getpid()}, RSS {rss_mb:.0f}MB",
-        }
-
-    def _check_worker_health(self) -> dict:
-        try:
-            from superlocalmemory.core.worker_pool import WorkerPool
-            pool = WorkerPool.shared()
-            wpid = pool.worker_pid
-            if wpid:
-                return {"name": "workers", "status": "ok", "detail": f"Worker PID {wpid}"}
-            return {"name": "workers", "status": "warning", "detail": "No active worker"}
-        except Exception as e:
-            return {"name": "workers", "status": "error", "detail": str(e)}
-
-    def _check_memory_budget(self) -> dict:
-        if not PSUTIL_AVAILABLE:
-            return {"name": "memory", "status": "unknown", "detail": "psutil unavailable"}
-        proc = psutil.Process(os.getpid())
-        total_rss = proc.memory_info().rss
-        for child in proc.children(recursive=True):
-            try:
-                total_rss += child.memory_info().rss
-            except (psutil.NoSuchProcess, psutil.AccessDenied):
-                pass
-        total_mb = total_rss / (1024 * 1024)
-        status = "ok" if total_mb < self._budget_mb else "critical"
-        return {
-            "name": "memory",
-            "status": status,
-            "detail": f"{total_mb:.0f}MB / {self._budget_mb}MB budget",
-        }

package/src/superlocalmemory/core/hooks.py

@@ -1,65 +0,0 @@
-# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
-# Licensed under AGPL-3.0-or-later - see LICENSE file
-# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
-
-"""Engine lifecycle hooks -- pre/post operation dispatch.
-
-Pre-hooks are synchronous and can reject operations (raise exceptions).
-Post-hooks are fire-and-forget -- errors are logged, never propagated.
-"""
-
-from __future__ import annotations
-
-import logging
-from typing import Any, Callable
-
-logger = logging.getLogger(__name__)
-
-
-class HookRegistry:
-    """Registry for engine lifecycle hooks.
-
-    Usage:
-        registry = HookRegistry()
-        registry.register_pre("store", abac_check)
-        registry.register_pre("store", trust_gate_check)
-        registry.register_post("store", audit_log)
-        registry.register_post("store", event_bus_publish)
-
-        # In engine.store():
-        registry.run_pre("store", context)   # raises if any pre-hook fails
-        # ... core operation ...
-        registry.run_post("store", context)  # never raises
-    """
-
-    def __init__(self) -> None:
-        self._pre: dict[str, list[Callable]] = {}
-        self._post: dict[str, list[Callable]] = {}
-
-    def register_pre(self, operation: str, hook: Callable[[dict[str, Any]], None]) -> None:
-        """Register a pre-operation hook. Hook receives context dict.
-        Hook can raise to reject the operation."""
-        self._pre.setdefault(operation, []).append(hook)
-
-    def register_post(self, operation: str, hook: Callable[[dict[str, Any]], None]) -> None:
-        """Register a post-operation hook. Hook receives context dict.
-        Errors are logged, never propagated."""
-        self._post.setdefault(operation, []).append(hook)
-
-    def run_pre(self, operation: str, context: dict[str, Any]) -> None:
-        """Run all pre-hooks for an operation. Raises on first failure."""
-        for hook in self._pre.get(operation, []):
-            hook(context)  # Let exceptions propagate -- this is intentional
-
-    def run_post(self, operation: str, context: dict[str, Any]) -> None:
-        """Run all post-hooks for an operation. Errors logged, never raised."""
-        for hook in self._post.get(operation, []):
-            try:
-                hook(context)
-            except Exception as exc:
-                logger.debug("Post-hook error (%s): %s", operation, exc)
-
-    def clear(self) -> None:
-        """Remove all hooks. Useful for testing."""
-        self._pre.clear()
-        self._post.clear()

package/src/superlocalmemory/core/ingest_gate.py

@@ -1,133 +0,0 @@
-# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
-# Licensed under AGPL-3.0-or-later - see LICENSE file
-# Part of SuperLocalMemory V3 — Ingest Gate (v3.6.6)
-
-"""Ingest gate for the remember/store write path (v3.6.6 F-4).
-
-Responsibilities:
-  1. Hard reject content > 1MB (nobody's memory is a megabyte).
-  2. Clamp content > 24000 chars to head 70% + tail 30% + truncation marker.
-     Head+tail (not head-only) preserves OPEN ITEMS that session-close facts put
-     at the END. Only pathological pastes (167KB JSON, 50KB logs) are touched;
-     normal dense memories (6-15K session handoffs) pass through intact.
-     Full original preserved in result.full_content for storage in memories table.
-  3. Prompt-template firewall: reject content matching _PROMPT_TEMPLATE_PATTERNS
-     (extends v3.6.4 remember-write-02 quality gate).
-
-Kill-switch: SLM_INGEST_NO_GATE=1 bypasses rules 2 and 3 (but NOT the 1MB hard cap).
-
-Usage::
-    from superlocalmemory.core.ingest_gate import apply_ingest_gate
-    result = apply_ingest_gate(content)
-    if result.rejected:
-        return {"success": False, "error": result.rejection_reason}
-    store(result.fact_content, full_content=result.full_content)
-"""
-
-from __future__ import annotations
-
-import os
-from dataclasses import dataclass, field
-
-# v3.6.6: clamp only pathological monsters; preserve normal dense memories.
-# 24K chars ≈ 6K tokens — well beyond any legitimate single memory.
-_MAX_VERBATIM_CHARS = 24000
-_HEAD_FRACTION = 0.70  # head 70% + tail 30% so OPEN ITEMS at the end survive
-_MAX_INGEST_BYTES = 1_048_576  # 1MB — hard cap, NOT bypassed by kill-switch
-_TRUNCATION_MARKER = "\n…[content truncated at ingest; full text in source memory]…\n"
-
-
-@dataclass
-class IngestGateResult:
-    """Result of applying the ingest gate to content.
-
-    fact_content:     Content to store in atomic_facts.content (may be truncated).
-    full_content:     Original unmodified content (for memories table storage).
-    rejected:         True if content should be rejected outright.
-    rejection_reason: Human-readable reason for rejection (when rejected=True).
-    truncated:        True if fact_content was head-sliced.
-    """
-    fact_content: str
-    full_content: str
-    rejected: bool = False
-    rejection_reason: str = ""
-    truncated: bool = False
-
-
-def apply_ingest_gate(
-    content: str,
-    max_verbatim_chars: int = _MAX_VERBATIM_CHARS,
-    max_ingest_bytes: int = _MAX_INGEST_BYTES,
-) -> IngestGateResult:
-    """Apply the ingest quality gate to content before storing.
-
-    Returns IngestGateResult. Callers MUST check result.rejected before
-    proceeding with storage.
-
-    Kill-switch SLM_INGEST_NO_GATE=1 bypasses the verbatim size clamp and
-    template firewall but NOT the 1MB hard cap (safety boundary).
-    """
-    gate_active = os.environ.get("SLM_INGEST_NO_GATE", "0") != "1"
-
-    # --- Hard cap: 1MB regardless of kill-switch ---
-    try:
-        byte_len = len(content.encode("utf-8", errors="replace"))
-    except Exception:
-        byte_len = len(content)
-    if byte_len > max_ingest_bytes:
-        return IngestGateResult(
-            fact_content=content,
-            full_content=content,
-            rejected=True,
-            rejection_reason=(
-                f"Content size {byte_len} bytes exceeds maximum "
-                f"{max_ingest_bytes} bytes (1MB). "
-                "Nobody's memory is a megabyte."
-            ),
-        )
-
-    # --- Gate bypassed ---
-    if not gate_active:
-        return IngestGateResult(
-            fact_content=content,
-            full_content=content,
-            rejected=False,
-            truncated=False,
-        )
-
-    # --- Prompt-template firewall ---
-    # Extends the v3.6.4 remember-write-02 quality gate.
-    try:
-        from superlocalmemory.core.injection import is_prompt_template
-        if is_prompt_template(content):
-            return IngestGateResult(
-                fact_content=content,
-                full_content=content,
-                rejected=True,
-                rejection_reason=(
-                    "Content matches internal prompt-template patterns "
-                    "(low-quality gate). Prompt machinery must not be stored as memory."
-                ),
-            )
-    except Exception:
-        pass  # Defensive: gate failure must never block a store
-
-    # --- Verbatim size clamp (head 70% + tail 30%) ---
-    if len(content) > max_verbatim_chars:
-        budget = max_verbatim_chars - len(_TRUNCATION_MARKER)
-        head_len = int(budget * _HEAD_FRACTION)
-        tail_len = budget - head_len
-        fact_content = content[:head_len] + _TRUNCATION_MARKER + content[-tail_len:]
-        return IngestGateResult(
-            fact_content=fact_content,
-            full_content=content,
-            rejected=False,
-            truncated=True,
-        )
-
-    return IngestGateResult(
-        fact_content=content,
-        full_content=content,
-        rejected=False,
-        truncated=False,
-    )

package/src/superlocalmemory/core/ingest_policy.py

@@ -1,38 +0,0 @@
-"""Pre-admission scrub policy that removes secret material from content.
-
-A single ordered stage that redacts secrets BEFORE content can reach any durable
-or queryable representation (atomic facts, memory records, ingestion receipts,
-journal, exports, backups, mesh). Unlike PII redaction — an opt-in operator
-policy — secret scrubbing is unconditional: a credential persisted verbatim at
-rest is always incorrect.
-
-The detector is the ``security_primitives.redact_secrets`` redactor; this module
-wraps it in an immutable result so callers can log the fact of redaction without
-ever handling the secret value.
-"""
-from __future__ import annotations
-
-from dataclasses import dataclass
-
-from superlocalmemory.core.security_primitives import redact_secrets
-
-
-@dataclass(frozen=True, slots=True)
-class ScrubResult:
-    """Outcome of a pre-admission scrub. Never carries the original secret."""
-
-    content: str
-    redacted: bool
-
-
-def scrub_secrets_for_ingest(content: str) -> ScrubResult:
-    """Redact secret material from ``content`` before durable admission.
-
-    Returns a :class:`ScrubResult` whose ``content`` is safe to persist. When
-    nothing matched, ``content`` is returned unchanged and ``redacted`` is False,
-    so the caller can keep the original object and skip a log line.
-    """
-    if not content:
-        return ScrubResult(content=content, redacted=False)
-    scrubbed = redact_secrets(content)
-    return ScrubResult(content=scrubbed, redacted=scrubbed != content)