superlocalmemory 4.1.13 → 4.1.15

package/src/superlocalmemory/server/config_file.py

@@ -1,90 +0,0 @@
-"""Process-safe, durable read/modify/write access to ``config.json``."""
-
-from __future__ import annotations
-
-import json
-import os
-import tempfile
-import threading
-from contextlib import contextmanager
-from pathlib import Path
-from typing import Callable, Iterator
-
-_PROCESS_LOCK = threading.RLock()
-
-
-@contextmanager
-def _file_lock(path: Path) -> Iterator[None]:
-    """Serialize config access across daemon, CLI, and worker processes."""
-    lock_path = path.with_name(f".{path.name}.lock")
-    lock_path.parent.mkdir(parents=True, exist_ok=True)
-    with _PROCESS_LOCK, lock_path.open("a+b") as handle:
-        if os.name == "nt":
-            import msvcrt
-
-            handle.seek(0)
-            handle.write(b"\0")
-            handle.flush()
-            handle.seek(0)
-            msvcrt.locking(handle.fileno(), msvcrt.LK_LOCK, 1)
-        else:
-            import fcntl
-
-            fcntl.flock(handle.fileno(), fcntl.LOCK_EX)
-        try:
-            yield
-        finally:
-            if os.name == "nt":
-                handle.seek(0)
-                msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1)
-            else:
-                fcntl.flock(handle.fileno(), fcntl.LOCK_UN)
-
-
-def _read_unlocked(path: Path) -> dict:
-    if not path.exists():
-        return {}
-    value = json.loads(path.read_text(encoding="utf-8"))
-    if not isinstance(value, dict):
-        raise ValueError("config root must be a JSON object")
-    return value
-
-
-def read_config(path: Path) -> dict:
-    """Return one coherent config snapshot under the interprocess lock."""
-    with _file_lock(path):
-        return _read_unlocked(path)
-
-
-def update_config(path: Path, update: Callable[[dict], None]) -> dict:
-    """Atomically update and durably replace a config JSON object."""
-    with _file_lock(path):
-        data = _read_unlocked(path)
-        update(data)
-        path.parent.mkdir(parents=True, exist_ok=True)
-        descriptor, temp_name = tempfile.mkstemp(
-            prefix=f".{path.name}.",
-            suffix=".tmp",
-            dir=path.parent,
-        )
-        temp_path = Path(temp_name)
-        try:
-            with os.fdopen(descriptor, "w", encoding="utf-8") as stream:
-                json.dump(data, stream, indent=2)
-                stream.write("\n")
-                stream.flush()
-                os.fsync(stream.fileno())
-            os.chmod(temp_path, 0o600)
-            os.replace(temp_path, path)
-            if hasattr(os, "O_DIRECTORY"):
-                directory_fd = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY)
-                try:
-                    os.fsync(directory_fd)
-                finally:
-                    os.close(directory_fd)
-        finally:
-            temp_path.unlink(missing_ok=True)
-        return data
-
-
-__all__ = ("read_config", "update_config")

package/src/superlocalmemory/server/consolidation_runner.py

@@ -1,140 +0,0 @@
-# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
-# Licensed under AGPL-3.0-or-later - see LICENSE file
-# Part of SuperLocalMemory | https://qualixar.com
-
-"""One implementation of full consolidation, shared by every trigger.
-
-WHY THIS EXISTS
----------------
-Steps 8-11 of ``ConsolidationEngine.consolidate()`` — behavioural assertion
-mining, soft prompts, skill performance, skill evolution — only run on the
-``lightweight=False`` path. Before 4.0.8 **nothing automatic ever took that
-path**:
-
-* ``consolidation_engine.py`` self-triggers with ``lightweight=True`` only.
-* The session-end hook shelled out to ``slm consolidate --cognitive``, which
-  runs ``CognitiveConsolidator.run_pipeline()`` — a different class that does
-  not contain steps 8-11 at all.
-* ``POST /consolidation/trigger`` did run the full path, but only when a human
-  called it.
-
-The measurable consequence on a real store: ``behavioral_assertions`` sat at 0
-rows while the miner, run once by hand against the same data, produced 9
-assertions immediately. The Behaviour tab was empty because the miner had never
-executed, not because there was nothing to mine.
-
-DESIGN
-------
-Two triggers, one implementation, one lock:
-
-* the daemon's periodic timer (the correctness guarantee), and
-* the session-end hook posting to ``/consolidation/trigger`` (the fast path).
-
-``_LOCK`` serialises them. A second trigger arriving while one is running is
-**skipped, not queued** — consolidation is idempotent catch-up work, so running
-it twice back to back buys nothing and doubles the write pressure.
-
-HOT PATH
---------
-Never called from store or recall. The engine call is CPU/IO bound for seconds
-to minutes, so it runs inside ``asyncio.to_thread`` — blocking a worker thread
-is fine, blocking the event loop is not. The periodic trigger additionally
-refuses to start unless the daemon has been idle (see ``unified_daemon``), so
-scheduled consolidation cannot land in the middle of a burst of remember/recall
-traffic.
-"""
-
-from __future__ import annotations
-
-import asyncio
-import logging
-from typing import Any
-
-logger = logging.getLogger("superlocalmemory.consolidation_runner")
-
-#: Serialises the timer and the hook. Module-level: there is one daemon.
-_LOCK = asyncio.Lock()
-
-
-def is_running() -> bool:
-    """True when a consolidation pass currently holds the lock."""
-    return _LOCK.locked()
-
-
-def _consolidate_blocking(app_state: Any, profile_id: str, lightweight: bool) -> dict:
-    """Run the engine synchronously. Caller must put this in a thread.
-
-    Lifted verbatim from ``POST /consolidation/trigger`` so the endpoint and the
-    timer cannot drift apart — two copies of this would be two different
-    definitions of "consolidated".
-    """
-    from superlocalmemory.core.config import SLMConfig
-    from superlocalmemory.core.consolidation_engine import ConsolidationEngine
-    from superlocalmemory.server.profile_runtime import get_profile_runtime
-    from superlocalmemory.storage import schema as _schema
-    from superlocalmemory.storage.database import DatabaseManager
-
-    runtime = get_profile_runtime(app_state)
-    # Rule 18: hold the operation lease so a concurrent profile switch cannot
-    # commit halfway through a consolidation.
-    with runtime.operation():
-        config = SLMConfig.load()
-        db = DatabaseManager(config.db_path)
-        db.initialize(_schema)
-        engine = ConsolidationEngine(
-            db=db, config=config.consolidation, slm_config=config,
-        )
-        res = engine.consolidate(profile_id=profile_id, lightweight=lightweight)
-
-        # Behavioural pattern mining writes learning.db rather than memory.db, so
-        # it sits outside the engine. Kept non-fatal: a pattern-mining failure
-        # must not discard a completed consolidation.
-        try:
-            from superlocalmemory.learning.consolidation_worker import (
-                ConsolidationWorker,
-            )
-            learning_db = config.base_dir / "learning.db"
-            cw = ConsolidationWorker(str(config.db_path), str(learning_db))
-            res["patterns_mined"] = cw._generate_patterns(profile_id, False)
-        except Exception as exc:
-            logger.warning("pattern mining after consolidation failed: %s", exc)
-            # -1, not 0: "it broke" and "there was nothing to mine" are
-            # different answers and the dashboard must not conflate them.
-            res["patterns_mined"] = -1
-    return res
-
-
-async def run_full_consolidation(
-    app_state: Any,
-    profile_id: str,
-    *,
-    lightweight: bool = False,
-    trigger: str = "manual",
-) -> dict:
-    """Run one consolidation pass, or skip if one is already in flight.
-
-    Returns the engine result dict, plus ``trigger``. When a pass is already
-    running the result is ``{"skipped": True, "reason": "already running"}`` —
-    an explicit skip rather than a silent no-op, so a caller can tell "did not
-    need to run" from "ran and found nothing".
-    """
-    if _LOCK.locked():
-        logger.info("consolidation skipped (%s): a pass is already running", trigger)
-        return {"skipped": True, "reason": "already running", "trigger": trigger}
-
-    async with _LOCK:
-        logger.info(
-            "consolidation starting (trigger=%s, profile=%s, lightweight=%s)",
-            trigger, profile_id, lightweight,
-        )
-        result = await asyncio.to_thread(
-            _consolidate_blocking, app_state, profile_id, lightweight,
-        )
-        logger.info(
-            "consolidation finished (trigger=%s): assertions=%s patterns=%s",
-            trigger,
-            (result.get("assertions") or {}).get("created", "n/a"),
-            result.get("patterns_mined", "n/a"),
-        )
-        result["trigger"] = trigger
-        return result

package/src/superlocalmemory/server/egress_policy.py

@@ -1,258 +0,0 @@
-"""SSRF-safe egress validation for outbound provider requests.
-
-Pre-admission, stateless validation for outbound HTTP fetches (embedding test,
-provider test, local model probe). It enforces three protections:
-
-* Cloud-metadata endpoints are blocked unconditionally and after hostname
-  normalization (trailing dot / case / IDNA), so ``metadata.google.internal.``
-  cannot slip past — including on the loopback path.
-* The full DNS answer set is validated (all A/AAAA records), not just the first
-  resolved address, which defeats mixed public/private answers (DNS rebinding).
-* DNS resolution failure fails closed (deny) for untrusted callers rather than
-  deferring the decision to the HTTP client.
-
-Trusted callers (the local dashboard on loopback, or an allowlisted LAN
-dashboard) keep the latitude to probe local/LAN model endpoints; metadata
-endpoints are denied for everyone.
-"""
-from __future__ import annotations
-
-import ipaddress
-import socket
-from dataclasses import dataclass
-from enum import Enum
-from urllib.parse import urlparse
-
-
-class EgressVerdict(str, Enum):
-    ALLOW = "allow"
-    DENY_SCHEME = "deny_scheme"
-    DENY_CREDENTIALS = "deny_credentials"
-    DENY_FRAGMENT = "deny_fragment"
-    DENY_HOST = "deny_host"
-    DENY_METADATA = "deny_metadata"
-    DENY_PRIVATE = "deny_private"
-    DENY_DNS_FAILURE = "deny_dns_failure"
-    DENY_MIXED_DNS = "deny_mixed_dns"
-
-
-# Cloud metadata endpoints — always blocked regardless of caller trust.
-METADATA_HOSTS: frozenset[str] = frozenset(
-    {
-        "169.254.169.254",
-        "metadata.google.internal",
-        "metadata",
-        "metadata.azure.internal",
-        "169.254.170.2",  # AWS ECS task metadata
-        "fd00:ec2::254",  # AWS IMDS over IPv6
-    }
-)
-
-_ALLOWED_SCHEMES: frozenset[str] = frozenset({"http", "https"})
-
-
-@dataclass(frozen=True, slots=True)
-class EgressPolicy:
-    """Configurable egress constraints (safe defaults)."""
-
-    allowed_schemes: frozenset[str] = _ALLOWED_SCHEMES
-    allow_private_for_local_actor: bool = True
-    allow_private_for_lan_actor: bool = True
-    reject_credentials: bool = True
-    reject_fragment: bool = True
-
-
-@dataclass(frozen=True, slots=True)
-class EgressActor:
-    """Minimal trust descriptor for the caller.
-
-    ``is_local``  — request originated from the loopback dashboard.
-    ``is_lan``    — request originated from an allowlisted LAN dashboard
-                    (remote mode ON *and* client IP in the allowlist).
-    A caller with neither flag is an untrusted/remote caller.
-    """
-
-    is_local: bool = False
-    is_lan: bool = False
-
-    @property
-    def is_trusted(self) -> bool:
-        return self.is_local or self.is_lan
-
-
-@dataclass(frozen=True, slots=True)
-class ResolvedTarget:
-    addresses: tuple[str, ...] = ()
-    has_private: bool = False
-    has_public: bool = False
-    error: str = ""
-
-    @property
-    def is_mixed(self) -> bool:
-        return self.has_private and self.has_public
-
-
-@dataclass(frozen=True, slots=True)
-class EgressResult:
-    verdict: EgressVerdict
-    resolved_ip: str = ""
-    error: str = ""
-    hostname: str = ""
-
-    @property
-    def allowed(self) -> bool:
-        return self.verdict is EgressVerdict.ALLOW
-
-
-def _normalize_host(hostname: str) -> str:
-    """Lowercase, strip a trailing dot, IDNA-encode non-ASCII hosts.
-
-    IDNA encoding is only attempted for hosts containing non-ASCII characters,
-    so IP literals and ordinary ASCII hostnames are left untouched (the ``idna``
-    codec rejects some all-ASCII inputs).
-    """
-    host = hostname.strip().lower().rstrip(".")
-    if host and any(ord(ch) > 127 for ch in host):
-        try:
-            host = host.encode("idna").decode("ascii")
-        except (UnicodeError, UnicodeDecodeError):
-            pass  # keep the raw host — classification below still applies
-    return host
-
-
-def _is_dangerous_ip(
-    addr: ipaddress.IPv4Address | ipaddress.IPv6Address,
-) -> bool:
-    return (
-        addr.is_private
-        or addr.is_loopback
-        or addr.is_link_local
-        or addr.is_reserved
-        or addr.is_multicast
-        or addr.is_unspecified
-    )
-
-
-def _coerce_ip(
-    ip_str: str,
-) -> ipaddress.IPv4Address | ipaddress.IPv6Address | None:
-    try:
-        addr = ipaddress.ip_address(ip_str)
-    except ValueError:
-        return None
-    # Unwrap IPv4-mapped IPv6 (``::ffff:127.0.0.1``) before classification.
-    if isinstance(addr, ipaddress.IPv6Address) and addr.ipv4_mapped:
-        return addr.ipv4_mapped
-    return addr
-
-
-def resolve_and_validate_dns(
-    hostname: str, policy: EgressPolicy = EgressPolicy()
-) -> ResolvedTarget:
-    """Resolve every A/AAAA record and classify each address.
-
-    Fails closed: any resolution error, or an empty answer set, returns a
-    ``ResolvedTarget`` carrying a non-empty ``error``.
-    """
-    try:
-        infos = socket.getaddrinfo(
-            hostname, None, socket.AF_UNSPEC, socket.SOCK_STREAM
-        )
-    except (socket.gaierror, OSError, UnicodeError) as exc:
-        return ResolvedTarget(error=f"dns resolution failed: {exc}")
-
-    addresses: list[str] = []
-    has_private = False
-    has_public = False
-    for info in infos:
-        sockaddr = info[4]
-        addr = _coerce_ip(sockaddr[0])
-        if addr is None:
-            continue
-        addresses.append(str(addr))
-        if _is_dangerous_ip(addr):
-            has_private = True
-        else:
-            has_public = True
-
-    if not addresses:
-        return ResolvedTarget(error="no addresses resolved")
-    return ResolvedTarget(
-        addresses=tuple(addresses),
-        has_private=has_private,
-        has_public=has_public,
-    )
-
-
-def validate_egress_url(
-    url: str,
-    actor: EgressActor = EgressActor(),
-    policy: EgressPolicy = EgressPolicy(),
-) -> EgressResult:
-    """Full SSRF-safe validation of an outbound URL for a given caller."""
-    parsed = urlparse(url)
-
-    if parsed.scheme not in policy.allowed_schemes:
-        return EgressResult(
-            verdict=EgressVerdict.DENY_SCHEME, error=f"scheme {parsed.scheme!r}"
-        )
-    if policy.reject_credentials and (parsed.username or parsed.password):
-        return EgressResult(verdict=EgressVerdict.DENY_CREDENTIALS)
-    if policy.reject_fragment and parsed.fragment:
-        return EgressResult(verdict=EgressVerdict.DENY_FRAGMENT)
-
-    hostname = _normalize_host(parsed.hostname or "")
-    if not hostname:
-        return EgressResult(verdict=EgressVerdict.DENY_HOST, error="empty host")
-
-    # Metadata block — unconditional, before any trust short-circuit or DNS
-    # (defeats the trailing-dot / loopback-path bypass and DNS rebinding).
-    if hostname in METADATA_HOSTS:
-        return EgressResult(verdict=EgressVerdict.DENY_METADATA, hostname=hostname)
-    literal = _coerce_ip(hostname)
-    if literal is not None and str(literal) in METADATA_HOSTS:
-        return EgressResult(verdict=EgressVerdict.DENY_METADATA, hostname=hostname)
-
-    trusted = (
-        actor.is_local and policy.allow_private_for_local_actor
-    ) or (actor.is_lan and policy.allow_private_for_lan_actor)
-
-    # Literal IP target: classify directly, no DNS needed.
-    if literal is not None:
-        if _is_dangerous_ip(literal) and not trusted:
-            return EgressResult(
-                verdict=EgressVerdict.DENY_PRIVATE,
-                resolved_ip=str(literal),
-                hostname=hostname,
-            )
-        return EgressResult(
-            verdict=EgressVerdict.ALLOW, resolved_ip=str(literal), hostname=hostname
-        )
-
-    # Trusted callers may probe local/LAN endpoints by name without a forced
-    # DNS lookup — preserving pre-V4 loopback/LAN dashboard behaviour. Metadata
-    # was already denied above for everyone.
-    if trusted:
-        return EgressResult(verdict=EgressVerdict.ALLOW, hostname=hostname)
-
-    # Untrusted caller: resolve the full answer set and fail closed on error.
-    resolved = resolve_and_validate_dns(hostname, policy)
-    if resolved.error:
-        return EgressResult(
-            verdict=EgressVerdict.DENY_DNS_FAILURE,
-            error=resolved.error,
-            hostname=hostname,
-        )
-    if resolved.has_private:
-        verdict = (
-            EgressVerdict.DENY_MIXED_DNS
-            if resolved.is_mixed
-            else EgressVerdict.DENY_PRIVATE
-        )
-        return EgressResult(verdict=verdict, hostname=hostname)
-
-    return EgressResult(
-        verdict=EgressVerdict.ALLOW,
-        resolved_ip=resolved.addresses[0],
-        hostname=hostname,
-    )

package/src/superlocalmemory/server/loopback.py

@@ -1,85 +0,0 @@
-# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
-# Licensed under AGPL-3.0-or-later
-
-"""Centralized loopback-address predicate for the SLM server.
-
-Replaces every ``frozenset({"127.0.0.1", "::1", "localhost"})`` auth check
-with a semantic helper that correctly handles IPv4-mapped IPv6 addresses
-(``::ffff:127.0.0.1``) — the root cause of issue #90.
-
-Issue #90 root cause
---------------------
-When the daemon is started with ``SLM_DAEMON_HOST=0.0.0.0`` on a dual-stack
-Linux host (common in LXC/Docker containers), the OS creates an IPv6 socket
-that accepts IPv4 connections via the IPv4-mapped IPv6 address mechanism
-(RFC 4291 §2.5.5.2). A client connecting to ``localhost`` on such a host
-may have its peer address reported as ``::ffff:127.0.0.1`` by
-uvicorn/Starlette. The literal set ``("127.0.0.1", "::1", "localhost")``
-does not include this form, causing a spurious 403 for install-token and
-uncredentialed-loopback callers.
-
-``ipaddress.ip_address("::ffff:127.0.0.1").is_loopback`` already returns
-``True`` in CPython. This module uses that fact.
-
-SECURITY INVARIANTS (non-negotiable):
-  - Empty/None host → False. SEC-L-02: a missing peer is never trusted.
-  - No proxy header trust. Callers MUST pass ``request.client.host`` only,
-    never X-Forwarded-For or any other spoofable header.
-  - All 127.0.0.0/8 is loopback per RFC 5735 (includes 127.0.0.2, etc.).
-  - ``"localhost"`` is accepted as a hostname alias. Callers that need the
-    stricter no-hostname check (``prewarm_auth``) keep their own predicate.
-  - ``"testclient"`` is NOT in the loopback set — that bypass must be wired
-    explicitly alongside ``_TEST_ISOLATION_ALLOWED`` so it cannot appear in
-    production paths.
-  - ``"0.0.0.0"`` is NOT loopback — it is a bind address, not a peer address.
-"""
-
-from __future__ import annotations
-
-import ipaddress as _ipa
-
-
-def is_loopback(host: str) -> bool:
-    """Return ``True`` iff ``host`` is a loopback address in any standard form.
-
-    Handles:
-    * ``"127.0.0.1"`` and the full 127.0.0.0/8 range (RFC 5735).
-    * ``"::1"`` (IPv6 loopback).
-    * ``"::ffff:127.0.0.1"`` and all ``::ffff:127.x.x.x`` (IPv4-mapped IPv6,
-      fixes issue #90).
-    * ``"localhost"`` (hostname alias, case-insensitive).
-
-    Returns ``False`` for:
-    * Empty string or ``None`` (SEC-L-02: missing peer is never trusted).
-    * Any non-loopback IP (192.168.x.x, 10.x.x.x, public IPs, etc.).
-    * ``"::ffff:192.168.x.x"`` and other IPv4-mapped non-loopback addresses.
-    * ``"testclient"`` — callers that need the test-client exemption must
-      wire it explicitly alongside ``_TEST_ISOLATION_ALLOWED``.
-    * ``"0.0.0.0"`` — bind address, not a peer address.
-
-    Args:
-        host: The ``request.client.host`` string from an incoming HTTP
-            request. Must be the TCP-observed peer address only.
-
-    Returns:
-        ``True`` if the host is any recognised loopback form; ``False``
-        otherwise.
-    """
-    if not isinstance(host, str) or not host:
-        return False  # SEC-L-02
-    if host.lower() == "localhost":
-        return True
-    try:
-        ip = _ipa.ip_address(host)
-    except ValueError:
-        return False
-    if ip.is_loopback:
-        return True
-    # CPython's handling of IPv4-mapped IPv6 changed across supported
-    # runtimes. Normalize through the embedded IPv4 address so a dual-stack
-    # socket reporting ::ffff:127.x.x.x retains the equivalent IPv4 decision.
-    mapped_ipv4 = getattr(ip, "ipv4_mapped", None)
-    return bool(mapped_ipv4 is not None and mapped_ipv4.is_loopback)
-
-
-__all__ = ("is_loopback",)

package/src/superlocalmemory/server/middleware/__init__.py

@@ -1,11 +0,0 @@
-# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
-# Licensed under AGPL-3.0-or-later - see LICENSE file
-# Part of SuperLocalMemory v3.4.22 — LLD-04 §4.2
-
-"""FastAPI middleware — strict security headers for the Brain UI (LLD-04 v2).
-
-The existing ``server/security_middleware.py`` is kept for legacy routes
-that still rely on permissive CSP (``'unsafe-inline'`` + CDNs). The
-middleware in this subpackage enforces the v3.4.22 policy: no inline
-scripts / styles, no nonces, no CDN sources.
-"""