superlocalmemory 3.8.13 → 4.0.0

package/src/superlocalmemory/infra/webhook_dispatcher.py

@@ -1,247 +0,0 @@
-# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
-# Licensed under AGPL-3.0-or-later - see LICENSE file
-# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
-"""WebhookDispatcher -- background HTTP POST delivery for memory events.
-
-Runs on a daemon thread so webhook delivery never blocks the main event
-flow.  Failed deliveries are retried with exponential back-off (up to
-``MAX_RETRIES`` attempts).
-
-Security:
-    * Only ``http://`` and ``https://`` URLs are accepted.
-    * Private / loopback IPs are rejected.
-    * 10-second timeout per outgoing request.
-"""
-
-import ipaddress
-import json
-import logging
-import socket
-import threading
-import time
-import urllib.parse
-from datetime import datetime, timezone
-from queue import Empty, Queue
-from typing import Dict, Optional
-
-logger = logging.getLogger("superlocalmemory.webhooks")
-
-# ---------------------------------------------------------------------------
-# Configuration constants
-# ---------------------------------------------------------------------------
-MAX_RETRIES = 3
-RETRY_BACKOFF_BASE = 2        # seconds: 2, 4, 8
-REQUEST_TIMEOUT = 10           # seconds
-MAX_QUEUE_SIZE = 1000
-def _get_version() -> str:
-    try:
-        from importlib.metadata import version
-        return version("superlocalmemory")
-    except Exception:
-        return "3.0.0"
-
-
-VERSION = _get_version()
-
-# stdlib HTTP -- always available
-from urllib.request import Request, urlopen  # noqa: E402
-from urllib.error import HTTPError, URLError  # noqa: E402
-
-
-def _is_private_ip(hostname: str) -> bool:
-    """Return ``True`` if *hostname* resolves to a private / loopback IP."""
-    try:
-        ip_str = socket.gethostbyname(hostname)
-        ip = ipaddress.ip_address(ip_str)
-        return ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_reserved
-    except (socket.gaierror, ValueError):
-        return False
-
-
-class WebhookDispatcher:
-    """Background webhook delivery with retry logic.
-
-    Thread-safe.  Enqueues deliveries and processes them on a dedicated
-    daemon thread.
-    """
-
-    _instances: Dict[str, "WebhookDispatcher"] = {}
-    _instances_lock = threading.Lock()
-
-    @classmethod
-    def get_instance(cls, name: str = "default") -> "WebhookDispatcher":
-        """Get or create a named singleton."""
-        with cls._instances_lock:
-            if name not in cls._instances:
-                cls._instances[name] = cls()
-            return cls._instances[name]
-
-    @classmethod
-    def reset_instance(cls, name: Optional[str] = None) -> None:
-        """Remove singleton(s).  Primarily for testing."""
-        with cls._instances_lock:
-            if name is None:
-                for inst in cls._instances.values():
-                    inst.close()
-                cls._instances.clear()
-            elif name in cls._instances:
-                cls._instances[name].close()
-                del cls._instances[name]
-
-    def __init__(self) -> None:
-        self._queue: Queue = Queue(maxsize=MAX_QUEUE_SIZE)
-        self._closed = False
-        self._stats = {
-            "dispatched": 0,
-            "succeeded": 0,
-            "failed": 0,
-            "retries": 0,
-        }
-        self._stats_lock = threading.Lock()
-
-        self._worker = threading.Thread(
-            target=self._worker_loop,
-            name="slm-webhook-worker",
-            daemon=True,
-        )
-        self._worker.start()
-        logger.info("WebhookDispatcher started")
-
-    # ----- public API -----
-
-    def dispatch(self, event: dict, webhook_url: str) -> None:
-        """Enqueue a webhook delivery.
-
-        Raises:
-            ValueError: If *webhook_url* is invalid or private.
-            RuntimeError: If the dispatcher is closed.
-        """
-        if self._closed:
-            raise RuntimeError("WebhookDispatcher is closed")
-
-        if not webhook_url or not (
-            webhook_url.startswith("http://") or webhook_url.startswith("https://")
-        ):
-            raise ValueError(f"Invalid webhook URL: {webhook_url}")
-
-        parsed = urllib.parse.urlparse(webhook_url)
-        if parsed.hostname and _is_private_ip(parsed.hostname):
-            raise ValueError(
-                f"Webhook URL points to private/internal network: {webhook_url}"
-            )
-
-        try:
-            self._queue.put_nowait(
-                {
-                    "event": event,
-                    "url": webhook_url,
-                    "attempt": 0,
-                    "enqueued_at": datetime.now(timezone.utc).isoformat(),
-                }
-            )
-            with self._stats_lock:
-                self._stats["dispatched"] += 1
-        except Exception:
-            logger.warning("Webhook queue full, dropping event for %s", webhook_url)
-
-    def get_stats(self) -> dict:
-        """Return delivery statistics snapshot."""
-        with self._stats_lock:
-            return dict(self._stats)
-
-    def close(self) -> None:
-        """Shut down the dispatcher, draining remaining items."""
-        if self._closed:
-            return
-        self._closed = True
-        self._queue.put(None)  # sentinel
-        if self._worker.is_alive():
-            self._worker.join(timeout=5)
-        logger.info("WebhookDispatcher closed: stats=%s", self._stats)
-
-    @property
-    def is_closed(self) -> bool:
-        return self._closed
-
-    @property
-    def queue_size(self) -> int:
-        return self._queue.qsize()
-
-    # ----- internal -----
-
-    def _worker_loop(self) -> None:
-        """Background loop: dequeue and deliver."""
-        while not self._closed:
-            try:
-                item = self._queue.get(timeout=1.0)
-            except Empty:
-                continue
-
-            if item is None:  # shutdown sentinel
-                self._queue.task_done()
-                break
-
-            self._deliver(item)
-            self._queue.task_done()
-
-    def _deliver(self, item: dict) -> None:
-        """Attempt delivery with exponential-backoff retry."""
-        event = item["event"]
-        url = item["url"]
-        attempt = item["attempt"]
-
-        try:
-            payload = json.dumps(
-                {
-                    "event": event,
-                    "delivered_at": datetime.now(timezone.utc).isoformat(),
-                    "attempt": attempt + 1,
-                    "source": "superlocalmemory",
-                    "version": VERSION,
-                }
-            ).encode("utf-8")
-
-            req = Request(
-                url,
-                data=payload,
-                headers={
-                    "Content-Type": "application/json",
-                    "User-Agent": f"SuperLocalMemory/{VERSION}",
-                    "X-SLM-Event-Type": event.get("event_type", "unknown"),
-                },
-                method="POST",
-            )
-
-            with urlopen(req, timeout=REQUEST_TIMEOUT) as resp:
-                status = resp.status
-                if 200 <= status < 300:
-                    with self._stats_lock:
-                        self._stats["succeeded"] += 1
-                    logger.debug("Webhook delivered: url=%s status=%d", url, status)
-                    return
-                raise HTTPError(url, status, f"HTTP {status}", {}, None)
-
-        except Exception as exc:
-            logger.warning(
-                "Webhook delivery failed (attempt %d/%d): url=%s error=%s",
-                attempt + 1,
-                MAX_RETRIES,
-                url,
-                exc,
-            )
-
-            if attempt + 1 < MAX_RETRIES:
-                backoff = RETRY_BACKOFF_BASE ** (attempt + 1)
-                time.sleep(backoff)
-                with self._stats_lock:
-                    self._stats["retries"] += 1
-                item["attempt"] = attempt + 1
-                self._deliver(item)
-            else:
-                with self._stats_lock:
-                    self._stats["failed"] += 1
-                logger.error(
-                    "Webhook permanently failed after %d attempts: url=%s",
-                    MAX_RETRIES,
-                    url,
-                )

package/src/superlocalmemory/learning/quantization_scheduler.py

@@ -1,320 +0,0 @@
-# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
-# Licensed under AGPL-3.0-or-later - see LICENSE file
-# Part of SuperLocalMemory V3
-
-"""Quantization Scheduler -- combined SAGQ + EAP precision management.
-
-Background quantization worker that periodically reviews all memories and
-applies both EAP (forgetting) and SAGQ (network centrality) signals together.
-
-Conflict resolution: max(EAP_precision, SAGQ_precision) -- safety first.
-Core Memory blocks are immune to quantization (HR-01).
-Every precision change is logged to fact_access_log for audit trail (HR-09).
-
-HR-01: Core Memory immune.
-HR-07: No-op when config.enabled=False.
-HR-08: Synchronous execution (no threading).
-HR-09: Audit trail for every change.
-
-Part of Qualixar | Author: Varun Pratap Bhardwaj
-License: AGPL-3.0-or-later
-"""
-
-from __future__ import annotations
-
-import logging
-import time
-from dataclasses import dataclass
-from datetime import UTC, datetime
-from typing import Any, Callable, TYPE_CHECKING
-
-from superlocalmemory.storage.models import _new_id
-
-if TYPE_CHECKING:
-    from superlocalmemory.storage.database import DatabaseManager
-    from superlocalmemory.dynamics.activation_guided_quantization import (
-        ActivationGuidedQuantizer,
-        SAGQPrecision,
-    )
-    from superlocalmemory.core.config import SAGQConfig
-
-logger = logging.getLogger(__name__)
-
-
-# ---------------------------------------------------------------------------
-# Data classes (frozen -- all immutable, Rule 10)
-# ---------------------------------------------------------------------------
-
-
-@dataclass(frozen=True)
-class PrecisionChange:
-    """Record of a single precision change for audit trail."""
-
-    fact_id: str
-    old_bit_width: int
-    new_bit_width: int
-    action: str       # "upgrade" | "downgrade"
-    centrality: float
-    sagq_signal: int
-    eap_signal: int
-    timestamp: str    # ISO 8601 datetime string
-
-
-@dataclass(frozen=True)
-class SchedulerRunResult:
-    """Result of a single scheduler run."""
-
-    total_facts: int
-    upgrades: int
-    downgrades: int
-    skipped: int
-    errors: int
-    changes: tuple[PrecisionChange, ...]
-    duration_ms: float
-
-
-# ---------------------------------------------------------------------------
-# Bit-width -> quantization level mapping
-# ---------------------------------------------------------------------------
-
-_BW_TO_LEVEL: dict[int, str] = {
-    32: "float32",
-    8: "int8",
-    4: "polar4",
-    2: "polar2",
-    0: "deleted",
-}
-
-
-# ---------------------------------------------------------------------------
-# QuantizationScheduler
-# ---------------------------------------------------------------------------
-
-
-class QuantizationScheduler:
-    """Combined SAGQ + EAP quantization scheduler.
-
-    Runs synchronously (HR-08). Called by the consolidation engine
-    or CLI command on schedule.
-    """
-
-    def __init__(
-        self,
-        db: Any,
-        sagq: Any,
-        eap_mapper: Callable[[str], int],
-        quantized_store: Any,
-        vector_store: Any,
-        config: Any,
-    ) -> None:
-        """Initialize scheduler. No side effects."""
-        self._db = db
-        self._sagq = sagq
-        self._eap_mapper = eap_mapper
-        self._quantized_store = quantized_store
-        self._vector_store = vector_store
-        self._config = config
-
-    def run(self, profile_id: str) -> SchedulerRunResult:
-        """Execute one combined SAGQ + EAP quantization pass.
-
-        Algorithm:
-          1. Compute SAGQ precision recommendations (centrality + EAP + max())
-          2. Exclude core memory facts (HR-01)
-          3. Execute upgrades/downgrades with error isolation
-          4. Log audit trail for each change (HR-09)
-          5. Return summary
-
-        Returns SchedulerRunResult with totals and change records.
-        """
-        # HR-07: No-op when disabled
-        if not self._config.enabled:
-            return SchedulerRunResult(
-                total_facts=0, upgrades=0, downgrades=0,
-                skipped=0, errors=0, changes=(), duration_ms=0.0,
-            )
-
-        start_time = time.monotonic()
-
-        # Step 2: Get SAGQ precision recommendations
-        recommendations = self._sagq.compute_sagq_precision_batch(
-            profile_id, self._eap_mapper,
-        )
-
-        if not recommendations:
-            duration_ms = (time.monotonic() - start_time) * 1000
-            return SchedulerRunResult(
-                total_facts=0, upgrades=0, downgrades=0,
-                skipped=0, errors=0, changes=(), duration_ms=duration_ms,
-            )
-
-        # Step 5a: Get core memory fact IDs (HR-01)
-        core_fact_ids = self._get_core_fact_ids(profile_id)
-
-        # Step 6: Process each recommendation
-        upgrades = 0
-        downgrades = 0
-        skipped = 0
-        errors = 0
-        changes: list[PrecisionChange] = []
-
-        for prec in recommendations:
-            # HR-01: Core Memory immune
-            if prec.fact_id in core_fact_ids:
-                skipped += 1
-                continue
-
-            if prec.action == "skip":
-                skipped += 1
-                continue
-
-            change = self._process_precision_change(prec, profile_id)
-
-            if change is None:
-                # Error or unable to process
-                if prec.action in ("downgrade", "upgrade"):
-                    errors += 1
-                else:
-                    skipped += 1
-                continue
-
-            if change.action == "downgrade":
-                downgrades += 1
-            elif change.action == "upgrade":
-                upgrades += 1
-
-            changes.append(change)
-
-        duration_ms = (time.monotonic() - start_time) * 1000
-
-        logger.info(
-            "SAGQ scheduler: %d upgrades, %d downgrades, %d skipped, "
-            "%d errors in %.1fms",
-            upgrades, downgrades, skipped, errors, duration_ms,
-        )
-
-        return SchedulerRunResult(
-            total_facts=len(recommendations),
-            upgrades=upgrades,
-            downgrades=downgrades,
-            skipped=skipped,
-            errors=errors,
-            changes=tuple(changes),
-            duration_ms=duration_ms,
-        )
-
-    def _process_precision_change(
-        self, prec: Any, profile_id: str,
-    ) -> PrecisionChange | None:
-        """Process a single precision change with error isolation.
-
-        Each fact is independent -- one failure does not block others.
-        Returns PrecisionChange on success, None on failure.
-        """
-        try:
-            if prec.action == "downgrade":
-                # Fetch float32 embedding
-                emb = self._vector_store.get_embedding(prec.fact_id, profile_id)
-                if emb is None:
-                    logger.warning(
-                        "SAGQ: No float32 for %s, skip downgrade", prec.fact_id,
-                    )
-                    return None
-                # Compress to target bit-width
-                self._quantized_store.compress_fact(
-                    prec.fact_id, profile_id, emb, prec.final_bit_width,
-                )
-
-            elif prec.action == "upgrade":
-                # Upgrade = re-compress at higher bit_width from float32 backup
-                emb = self._vector_store.get_embedding(prec.fact_id, profile_id)
-                if emb is None:
-                    logger.warning(
-                        "SAGQ: No float32 for %s, skip upgrade", prec.fact_id,
-                    )
-                    return None
-                self._quantized_store.compress_fact(
-                    prec.fact_id, profile_id, emb, prec.final_bit_width,
-                )
-
-            else:
-                return None  # skip
-
-            # Update embedding_metadata (Q7)
-            level = self._bit_width_to_quantization_level(prec.final_bit_width)
-            self._db.execute(
-                "UPDATE embedding_metadata "
-                "SET bit_width = ?, quantization_level = ? "
-                "WHERE fact_id = ? AND profile_id = ?",
-                (prec.final_bit_width, level, prec.fact_id, profile_id),
-            )
-
-            # Audit trail (Q10 -- HR-09)
-            self._db.execute(
-                "INSERT INTO fact_access_log "
-                "(log_id, fact_id, profile_id, accessed_at, access_type, session_id) "
-                "VALUES (?, ?, ?, datetime('now'), 'consolidation', 'sagq_scheduler')",
-                (_new_id(), prec.fact_id, profile_id),
-            )
-
-            now_iso = datetime.now(UTC).isoformat()
-            return PrecisionChange(
-                fact_id=prec.fact_id,
-                old_bit_width=prec.current_bit_width,
-                new_bit_width=prec.final_bit_width,
-                action=prec.action,
-                centrality=prec.centrality,
-                sagq_signal=prec.sagq_bit_width,
-                eap_signal=prec.eap_bit_width,
-                timestamp=now_iso,
-            )
-
-        except Exception as exc:
-            logger.error(
-                "SAGQ: precision change failed for %s: %s", prec.fact_id, exc,
-            )
-            return None
-
-    def _get_core_fact_ids(self, profile_id: str) -> set[str]:
-        """Get fact IDs referenced by core_memory_blocks (immune to quantization).
-
-        Uses json_each() to extract from source_fact_ids JSON array (Q8).
-        """
-        try:
-            rows = self._db.execute(
-                "SELECT json_each.value as fact_id "
-                "FROM core_memory_blocks, json_each(core_memory_blocks.source_fact_ids) "
-                "WHERE core_memory_blocks.profile_id = ?",
-                (profile_id,),
-            )
-            return {dict(r)["fact_id"] for r in rows}
-        except Exception as exc:
-            logger.debug("SAGQ: core_memory_blocks query failed: %s", exc)
-            return set()
-
-    def _bit_width_to_quantization_level(self, bit_width: int) -> str:
-        """Map bit-width integer to quantization level string."""
-        return _BW_TO_LEVEL.get(bit_width, "float32")
-
-    def should_run(self, last_run_at: str | None) -> bool:
-        """Check if enough time has passed since the last run.
-
-        Args:
-            last_run_at: ISO 8601 datetime of last run, or None if never run.
-
-        Returns True if the scheduler should run now.
-        """
-        if last_run_at is None:
-            return True
-
-        try:
-            last_run = datetime.fromisoformat(last_run_at)
-            now = datetime.now(UTC)
-            # Ensure both are timezone-aware for subtraction
-            if last_run.tzinfo is None:
-                last_run = last_run.replace(tzinfo=UTC)
-            hours_since = (now - last_run).total_seconds() / 3600
-            return hours_since >= self._config.scheduler_interval_hours
-        except (ValueError, TypeError) as exc:
-            logger.warning("SAGQ: Could not parse last_run_at '%s': %s", last_run_at, exc)
-            return True