superlocalmemory 2.8.5 → 3.0.0

package/src/superlocalmemory/infra/rate_limiter.py

@@ -0,0 +1,135 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under the MIT License - see LICENSE file
+# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
+"""Sliding-window rate limiter for per-agent request throttling.
+
+Pure stdlib -- no external dependencies.  Thread-safe.
+
+Defaults (configurable via env vars):
+    SLM_RATE_LIMIT_WRITE  = 100 req / window
+    SLM_RATE_LIMIT_READ   = 300 req / window
+    SLM_RATE_LIMIT_WINDOW = 60  seconds
+"""
+
+import logging
+import os
+import threading
+import time
+from collections import defaultdict
+from typing import Dict, List, Tuple
+
+logger = logging.getLogger("superlocalmemory.ratelimit")
+
+# ---------------------------------------------------------------------------
+# Module-level defaults (overridable via environment)
+# ---------------------------------------------------------------------------
+WRITE_LIMIT = int(os.environ.get("SLM_RATE_LIMIT_WRITE", "100"))
+READ_LIMIT = int(os.environ.get("SLM_RATE_LIMIT_READ", "300"))
+WINDOW_SECONDS = int(os.environ.get("SLM_RATE_LIMIT_WINDOW", "60"))
+
+
+class RateLimiter:
+    """Thread-safe sliding-window rate limiter.
+
+    Each *client_id* (agent name, IP, etc.) gets its own independent
+    request window.  Expired timestamps are pruned lazily on every call
+    to ``allow()`` or ``is_allowed()``.
+
+    Args:
+        max_requests: Maximum requests allowed per window.
+        window_seconds: Length of the sliding window in seconds.
+    """
+
+    def __init__(
+        self,
+        max_requests: int = 100,
+        window_seconds: int = 60,
+    ) -> None:
+        self.max_requests = max_requests
+        self.window = window_seconds
+        self._requests: Dict[str, List[float]] = defaultdict(list)
+        self._lock = threading.Lock()
+
+    # ----- public API -----
+
+    def allow(self, client_id: str) -> bool:
+        """Check **and record** a request for *client_id*.
+
+        Returns ``True`` when the request is allowed, ``False`` when the
+        client has exceeded its limit for the current window.
+        """
+        allowed, _ = self.is_allowed(client_id)
+        return allowed
+
+    def is_allowed(self, client_id: str) -> Tuple[bool, int]:
+        """Check and record a request.
+
+        Returns:
+            ``(allowed, remaining)`` -- whether the request is permitted
+            and how many requests remain in the current window.
+        """
+        now = time.time()
+        cutoff = now - self.window
+
+        with self._lock:
+            # Prune expired timestamps
+            self._requests[client_id] = [
+                t for t in self._requests[client_id] if t > cutoff
+            ]
+
+            current = len(self._requests[client_id])
+
+            if current >= self.max_requests:
+                return False, 0
+
+            self._requests[client_id].append(now)
+            return True, self.max_requests - current - 1
+
+    def remaining(self, client_id: str) -> int:
+        """Return how many requests *client_id* has left without recording one."""
+        now = time.time()
+        cutoff = now - self.window
+
+        with self._lock:
+            active = [t for t in self._requests.get(client_id, []) if t > cutoff]
+            return max(0, self.max_requests - len(active))
+
+    def reset(self, client_id: str) -> None:
+        """Clear all recorded requests for *client_id*."""
+        with self._lock:
+            self._requests.pop(client_id, None)
+
+    def cleanup(self) -> int:
+        """Remove stale entries for clients with no recent requests.
+
+        Returns:
+            Number of client entries removed.
+        """
+        now = time.time()
+        cutoff = now - self.window * 2  # keep 2 windows of data
+
+        with self._lock:
+            stale = [
+                k
+                for k, v in self._requests.items()
+                if not v or max(v) < cutoff
+            ]
+            for k in stale:
+                del self._requests[k]
+            return len(stale)
+
+    def get_stats(self) -> dict:
+        """Return a snapshot of limiter state."""
+        with self._lock:
+            return {
+                "max_requests": self.max_requests,
+                "window_seconds": self.window,
+                "tracked_clients": len(self._requests),
+            }
+
+
+# ---------------------------------------------------------------------------
+# Module-level convenience singletons
+# ---------------------------------------------------------------------------
+write_limiter = RateLimiter(max_requests=WRITE_LIMIT, window_seconds=WINDOW_SECONDS)
+read_limiter = RateLimiter(max_requests=READ_LIMIT, window_seconds=WINDOW_SECONDS)

package/src/{webhook_dispatcher.py → superlocalmemory/infra/webhook_dispatcher.py}

@@ -1,17 +1,16 @@
-#!/usr/bin/env python3
-# SPDX-License-Identifier: MIT
-# Copyright (c) 2026 SuperLocalMemory (superlocalmemory.com)
-"""
-WebhookDispatcher — Delivers events via HTTP POST to configured webhook URLs.
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under the MIT License - see LICENSE file
+# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
+"""WebhookDispatcher -- background HTTP POST delivery for memory events.
 
-Runs on a background thread so webhook delivery never blocks the main event flow.
-Retries failed deliveries with exponential backoff (3 attempts).
+Runs on a daemon thread so webhook delivery never blocks the main event
+flow.  Failed deliveries are retried with exponential back-off (up to
+``MAX_RETRIES`` attempts).
 
 Security:
-    - Only allows http:// and https:// URLs
-    - Validates URL format before dispatch
-    - 10-second timeout per request
-    - No private/internal IP blocking in v2.5 (added in v2.6 with trust enforcement)
+    * Only ``http://`` and ``https://`` URLs are accepted.
+    * Private / loopback IPs are rejected.
+    * 10-second timeout per outgoing request.
 """
 
 import ipaddress
@@ -21,44 +20,41 @@ import socket
 import threading
 import time
 import urllib.parse
-from queue import Queue, Empty
-from typing import Optional, Dict
 from datetime import datetime
+from queue import Empty, Queue
+from typing import Dict, Optional
 
 logger = logging.getLogger("superlocalmemory.webhooks")
 
-# Configuration
+# ---------------------------------------------------------------------------
+# Configuration constants
+# ---------------------------------------------------------------------------
 MAX_RETRIES = 3
-RETRY_BACKOFF_BASE = 2  # seconds: 2, 4, 8
-REQUEST_TIMEOUT = 10    # seconds
+RETRY_BACKOFF_BASE = 2        # seconds: 2, 4, 8
+REQUEST_TIMEOUT = 10           # seconds
 MAX_QUEUE_SIZE = 1000
+VERSION = "3.0.0"
 
-# Optional: urllib3/requests for HTTP POST
-try:
-    from urllib.request import Request, urlopen
-    from urllib.error import URLError, HTTPError
-    HTTP_AVAILABLE = True
-except ImportError:
-    HTTP_AVAILABLE = False
+# stdlib HTTP -- always available
+from urllib.request import Request, urlopen  # noqa: E402
+from urllib.error import HTTPError, URLError  # noqa: E402
 
 
 def _is_private_ip(hostname: str) -> bool:
-    """Check if hostname resolves to a private/internal IP address."""
+    """Return ``True`` if *hostname* resolves to a private / loopback IP."""
     try:
         ip_str = socket.gethostbyname(hostname)
         ip = ipaddress.ip_address(ip_str)
         return ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_reserved
     except (socket.gaierror, ValueError):
-        return False  # DNS resolution failed — allow (might be valid external hostname)
+        return False
 
 
 class WebhookDispatcher:
-    """
-    Background webhook delivery with retry logic.
+    """Background webhook delivery with retry logic.
 
-    Thread-safe. Enqueues webhook deliveries and processes them on a
-    dedicated background thread. Failed deliveries are retried with
-    exponential backoff.
+    Thread-safe.  Enqueues deliveries and processes them on a dedicated
+    daemon thread.
     """
 
     _instances: Dict[str, "WebhookDispatcher"] = {}
@@ -66,7 +62,7 @@ class WebhookDispatcher:
 
     @classmethod
     def get_instance(cls, name: str = "default") -> "WebhookDispatcher":
-        """Get or create a singleton WebhookDispatcher."""
+        """Get or create a named singleton."""
         with cls._instances_lock:
             if name not in cls._instances:
                 cls._instances[name] = cls()
@@ -74,7 +70,7 @@ class WebhookDispatcher:
 
     @classmethod
     def reset_instance(cls, name: Optional[str] = None) -> None:
-        """Remove singleton(s). Used for testing."""
+        """Remove singleton(s).  Primarily for testing."""
         with cls._instances_lock:
             if name is None:
                 for inst in cls._instances.values():
@@ -84,7 +80,7 @@ class WebhookDispatcher:
                 cls._instances[name].close()
                 del cls._instances[name]
 
-    def __init__(self):
+    def __init__(self) -> None:
         self._queue: Queue = Queue(maxsize=MAX_QUEUE_SIZE)
         self._closed = False
         self._stats = {
@@ -95,7 +91,6 @@ class WebhookDispatcher:
         }
         self._stats_lock = threading.Lock()
 
-        # Background worker thread
         self._worker = threading.Thread(
             target=self._worker_loop,
             name="slm-webhook-worker",
@@ -104,82 +99,106 @@ class WebhookDispatcher:
         self._worker.start()
         logger.info("WebhookDispatcher started")
 
-    def dispatch(self, event: dict, webhook_url: str) -> None:
-        """
-        Enqueue a webhook delivery.
+    # ----- public API -----
 
-        Args:
-            event: Event dict to send as JSON POST body
-            webhook_url: URL to POST to
+    def dispatch(self, event: dict, webhook_url: str) -> None:
+        """Enqueue a webhook delivery.
 
         Raises:
-            ValueError: If webhook_url is invalid
-            RuntimeError: If dispatcher is closed
+            ValueError: If *webhook_url* is invalid or private.
+            RuntimeError: If the dispatcher is closed.
         """
         if self._closed:
             raise RuntimeError("WebhookDispatcher is closed")
 
-        if not webhook_url or not (webhook_url.startswith("http://") or webhook_url.startswith("https://")):
+        if not webhook_url or not (
+            webhook_url.startswith("http://") or webhook_url.startswith("https://")
+        ):
             raise ValueError(f"Invalid webhook URL: {webhook_url}")
 
         parsed = urllib.parse.urlparse(webhook_url)
         if parsed.hostname and _is_private_ip(parsed.hostname):
-            raise ValueError(f"Webhook URL points to private/internal network: {webhook_url}")
+            raise ValueError(
+                f"Webhook URL points to private/internal network: {webhook_url}"
+            )
 
         try:
-            self._queue.put_nowait({
-                "event": event,
-                "url": webhook_url,
-                "attempt": 0,
-                "enqueued_at": datetime.now().isoformat(),
-            })
+            self._queue.put_nowait(
+                {
+                    "event": event,
+                    "url": webhook_url,
+                    "attempt": 0,
+                    "enqueued_at": datetime.now().isoformat(),
+                }
+            )
             with self._stats_lock:
                 self._stats["dispatched"] += 1
         except Exception:
             logger.warning("Webhook queue full, dropping event for %s", webhook_url)
 
-    def _worker_loop(self):
-        """Background worker: processes webhook deliveries sequentially."""
+    def get_stats(self) -> dict:
+        """Return delivery statistics snapshot."""
+        with self._stats_lock:
+            return dict(self._stats)
+
+    def close(self) -> None:
+        """Shut down the dispatcher, draining remaining items."""
+        if self._closed:
+            return
+        self._closed = True
+        self._queue.put(None)  # sentinel
+        if self._worker.is_alive():
+            self._worker.join(timeout=5)
+        logger.info("WebhookDispatcher closed: stats=%s", self._stats)
+
+    @property
+    def is_closed(self) -> bool:
+        return self._closed
+
+    @property
+    def queue_size(self) -> int:
+        return self._queue.qsize()
+
+    # ----- internal -----
+
+    def _worker_loop(self) -> None:
+        """Background loop: dequeue and deliver."""
         while not self._closed:
             try:
                 item = self._queue.get(timeout=1.0)
             except Empty:
                 continue
 
-            if item is None:  # Shutdown sentinel
+            if item is None:  # shutdown sentinel
                 self._queue.task_done()
                 break
 
             self._deliver(item)
             self._queue.task_done()
 
-    def _deliver(self, item: dict):
-        """Attempt to deliver a webhook. Retry on failure."""
+    def _deliver(self, item: dict) -> None:
+        """Attempt delivery with exponential-backoff retry."""
         event = item["event"]
         url = item["url"]
         attempt = item["attempt"]
 
-        if not HTTP_AVAILABLE:
-            logger.error("HTTP library not available, cannot deliver webhook to %s", url)
-            with self._stats_lock:
-                self._stats["failed"] += 1
-            return
-
         try:
-            payload = json.dumps({
-                "event": event,
-                "delivered_at": datetime.now().isoformat(),
-                "attempt": attempt + 1,
-                "source": "superlocalmemory",
-                "version": "2.5.0",
-            }).encode("utf-8")
+            payload = json.dumps(
+                {
+                    "event": event,
+                    "delivered_at": datetime.now().isoformat(),
+                    "attempt": attempt + 1,
+                    "source": "superlocalmemory",
+                    "version": VERSION,
+                }
+            ).encode("utf-8")
 
             req = Request(
                 url,
                 data=payload,
                 headers={
                     "Content-Type": "application/json",
-                    "User-Agent": "SuperLocalMemory/2.5.0",
+                    "User-Agent": f"SuperLocalMemory/{VERSION}",
                     "X-SLM-Event-Type": event.get("event_type", "unknown"),
                 },
                 method="POST",
@@ -190,47 +209,31 @@ class WebhookDispatcher:
                 if 200 <= status < 300:
                     with self._stats_lock:
                         self._stats["succeeded"] += 1
-                    logger.debug("Webhook delivered: url=%s, status=%d", url, status)
+                    logger.debug("Webhook delivered: url=%s status=%d", url, status)
                     return
-                else:
-                    raise HTTPError(url, status, f"HTTP {status}", {}, None)
+                raise HTTPError(url, status, f"HTTP {status}", {}, None)
 
-        except Exception as e:
-            logger.warning("Webhook delivery failed (attempt %d/%d): url=%s, error=%s",
-                          attempt + 1, MAX_RETRIES, url, e)
+        except Exception as exc:
+            logger.warning(
+                "Webhook delivery failed (attempt %d/%d): url=%s error=%s",
+                attempt + 1,
+                MAX_RETRIES,
+                url,
+                exc,
+            )
 
             if attempt + 1 < MAX_RETRIES:
-                # Retry with exponential backoff
                 backoff = RETRY_BACKOFF_BASE ** (attempt + 1)
                 time.sleep(backoff)
                 with self._stats_lock:
                     self._stats["retries"] += 1
                 item["attempt"] = attempt + 1
-                self._deliver(item)  # Recursive retry
+                self._deliver(item)
             else:
                 with self._stats_lock:
                     self._stats["failed"] += 1
-                logger.error("Webhook permanently failed after %d attempts: url=%s", MAX_RETRIES, url)
-
-    def get_stats(self) -> dict:
-        """Get webhook delivery statistics."""
-        with self._stats_lock:
-            return dict(self._stats)
-
-    def close(self) -> None:
-        """Shut down the dispatcher. Drains remaining items."""
-        if self._closed:
-            return
-        self._closed = True
-        self._queue.put(None)  # Shutdown sentinel
-        if self._worker.is_alive():
-            self._worker.join(timeout=5)
-        logger.info("WebhookDispatcher closed: stats=%s", self._stats)
-
-    @property
-    def is_closed(self) -> bool:
-        return self._closed
-
-    @property
-    def queue_size(self) -> int:
-        return self._queue.qsize()
+                logger.error(
+                    "Webhook permanently failed after %d attempts: url=%s",
+                    MAX_RETRIES,
+                    url,
+                )

package/src/superlocalmemory/learning/adaptive.py

@@ -0,0 +1,172 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under the MIT License - see LICENSE file
+# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
+
+"""SuperLocalMemory V3 — Adaptive Learning (3-Phase).
+
+Learns optimal retrieval weights from user feedback.
+Ported from V2.8 LightGBM-based learning system.
+Profile-scoped: each profile learns independently.
+
+Phase 1: Collect feedback (ranking_feedback)
+Phase 2: Train model on feedback patterns
+Phase 3: Apply learned weights to retrieval
+
+Part of Qualixar | Author: Varun Pratap Bhardwaj
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from collections import defaultdict
+from datetime import UTC, datetime
+
+from superlocalmemory.storage.models import FeedbackRecord
+
+logger = logging.getLogger(__name__)
+
+# Minimum feedback records before training
+_MIN_FEEDBACK_FOR_TRAINING = 20
+
+# Default channel weights (before learning)
+_DEFAULT_WEIGHTS = {
+    "semantic": 1.2,
+    "bm25": 1.0,
+    "entity_graph": 1.0,
+    "temporal": 0.8,
+}
+
+
+class AdaptiveLearner:
+    """3-phase adaptive learning for retrieval weight optimization.
+
+    Learns from user feedback which channels produce the best results
+    for different query types. Profile-scoped.
+    """
+
+    def __init__(self, db) -> None:
+        self._db = db
+        self._learned_weights: dict[str, dict[str, float]] = {}
+
+    # -- Phase 1: Collect feedback -----------------------------------------
+
+    def record_feedback(
+        self,
+        query: str,
+        fact_id: str,
+        feedback_type: str,
+        profile_id: str,
+        dwell_time_ms: int = 0,
+    ) -> FeedbackRecord:
+        """Record user feedback on a retrieval result.
+
+        feedback_type: "relevant", "irrelevant", "partial"
+        """
+        record = FeedbackRecord(
+            profile_id=profile_id,
+            query=query,
+            fact_id=fact_id,
+            feedback_type=feedback_type,
+            dwell_time_ms=dwell_time_ms,
+            timestamp=datetime.now(UTC).isoformat(),
+        )
+        self._db.execute(
+            "INSERT INTO feedback_records "
+            "(feedback_id, profile_id, query, fact_id, feedback_type, "
+            "dwell_time_ms, timestamp) VALUES (?,?,?,?,?,?,?)",
+            (record.feedback_id, record.profile_id, record.query,
+             record.fact_id, record.feedback_type, record.dwell_time_ms,
+             record.timestamp),
+        )
+        return record
+
+    def get_feedback_count(self, profile_id: str) -> int:
+        """Count feedback records for a profile."""
+        rows = self._db.execute(
+            "SELECT COUNT(*) AS c FROM feedback_records WHERE profile_id = ?",
+            (profile_id,),
+        )
+        return int(dict(rows[0])["c"]) if rows else 0
+
+    # -- Phase 2: Learn patterns -------------------------------------------
+
+    def train(self, profile_id: str) -> dict[str, dict[str, float]]:
+        """Learn optimal weights from feedback patterns.
+
+        Simple heuristic approach (LightGBM port deferred to production):
+        - Analyze which channels produced "relevant" results
+        - Boost channels that correlate with positive feedback
+        - Reduce channels that correlate with negative feedback
+        """
+        count = self.get_feedback_count(profile_id)
+        if count < _MIN_FEEDBACK_FOR_TRAINING:
+            logger.info(
+                "Only %d feedback records (need %d). Using defaults.",
+                count, _MIN_FEEDBACK_FOR_TRAINING,
+            )
+            return {}
+
+        rows = self._db.execute(
+            "SELECT query, fact_id, feedback_type FROM feedback_records "
+            "WHERE profile_id = ? ORDER BY timestamp DESC LIMIT 500",
+            (profile_id,),
+        )
+
+        # Count positive/negative per query pattern
+        positive_count = 0
+        negative_count = 0
+        for row in rows:
+            d = dict(row)
+            if d["feedback_type"] == "relevant":
+                positive_count += 1
+            elif d["feedback_type"] == "irrelevant":
+                negative_count += 1
+
+        # Simple relevance ratio → weight adjustment
+        if positive_count + negative_count == 0:
+            return {}
+
+        relevance_ratio = positive_count / (positive_count + negative_count)
+
+        # If retrieval is generally good (>70% relevant), trust current weights
+        # If poor (<50%), boost BM25 and entity (more precise channels)
+        if relevance_ratio < 0.5:
+            learned = {
+                "general": {
+                    "semantic": 1.0,
+                    "bm25": 1.5,         # Boost precision
+                    "entity_graph": 1.3,  # Boost entity matching
+                    "temporal": 0.8,
+                },
+            }
+        else:
+            learned = {
+                "general": dict(_DEFAULT_WEIGHTS),
+            }
+
+        self._learned_weights = learned
+        logger.info("Learned weights (ratio=%.2f): %s", relevance_ratio, learned)
+        return learned
+
+    # -- Phase 3: Apply weights --------------------------------------------
+
+    def get_weights(
+        self, query_type: str, profile_id: str
+    ) -> dict[str, float]:
+        """Get learned weights for a query type.
+
+        Falls back to defaults if no learned weights available.
+        """
+        if not self._learned_weights:
+            self.train(profile_id)
+
+        if query_type in self._learned_weights:
+            return self._learned_weights[query_type]
+        if "general" in self._learned_weights:
+            return self._learned_weights["general"]
+        return dict(_DEFAULT_WEIGHTS)
+
+    def is_trained(self, profile_id: str) -> bool:
+        """Check if the learner has enough data to provide learned weights."""
+        return self.get_feedback_count(profile_id) >= _MIN_FEEDBACK_FOR_TRAINING