superlocalmemory 4.1.13 → 4.1.15

package/src/superlocalmemory/learning/workflows.py

@@ -1,309 +0,0 @@
-# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
-# Licensed under AGPL-3.0-or-later - see LICENSE file
-# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
-
-"""Workflow pattern miner -- sliding-window sequence and temporal mining.
-
-Detects repeating workflow sequences and time-of-day activity patterns
-from memory creation timestamps and content.  Uses n-gram sliding
-windows (length 2-5) over a classified activity stream.
-
-Seven activity types: docs, architecture, code, test, debug, deploy, config.
-
-Ported from V2 WorkflowPatternMiner with V2 LearningDB deps removed.
-Direct sqlite3 for storage.
-
-Part of Qualixar | Author: Varun Pratap Bhardwaj
-"""
-
-from __future__ import annotations
-
-import json
-import logging
-import re
-import sqlite3
-from collections import Counter
-from datetime import UTC, datetime
-from pathlib import Path
-from typing import Any, Optional
-
-logger = logging.getLogger(__name__)
-
-# ---------------------------------------------------------------------------
-# Activity taxonomy (7 categories)
-# ---------------------------------------------------------------------------
-
-ACTIVITY_TYPES: dict[str, list[str]] = {
-    "docs": [
-        "documentation", "readme", "wiki", "spec", "prd",
-        "design doc", "changelog", "api doc",
-    ],
-    "architecture": [
-        "architecture", "diagram", "system design", "schema",
-        "api design", "data model", "erd",
-    ],
-    "code": [
-        "implement", "function", "class", "module", "refactor",
-        "code", "feature", "component",
-    ],
-    "test": [
-        "test", "pytest", "jest", "coverage", "assertion",
-        "mock", "spec", "unit test",
-    ],
-    "debug": [
-        "bug", "fix", "error", "stack trace", "debug",
-        "issue", "exception", "traceback",
-    ],
-    "deploy": [
-        "deploy", "docker", "ci/cd", "pipeline", "release",
-        "production", "staging", "build",
-    ],
-    "config": [
-        "config", "env", "settings", "setup", "install",
-        "dependency", "package", "requirements",
-    ],
-}
-
-# Pre-compiled regex per keyword for word-boundary matching
-_KEYWORD_PATTERNS: list[tuple[str, re.Pattern]] = []
-for _act, _kws in ACTIVITY_TYPES.items():
-    for _kw in _kws:
-        _KEYWORD_PATTERNS.append(
-            (_act, re.compile(r"\b" + re.escape(_kw) + r"\b", re.IGNORECASE))
-        )
-
-# ---------------------------------------------------------------------------
-# Schema for local action log
-# ---------------------------------------------------------------------------
-
-_SCHEMA = """
-CREATE TABLE IF NOT EXISTS workflow_actions (
-    id INTEGER PRIMARY KEY AUTOINCREMENT,
-    profile_id TEXT NOT NULL,
-    action TEXT NOT NULL,
-    metadata TEXT DEFAULT '{}',
-    created_at TEXT NOT NULL
-);
-CREATE INDEX IF NOT EXISTS idx_wf_profile
-    ON workflow_actions(profile_id, created_at);
-
-CREATE TABLE IF NOT EXISTS workflow_patterns (
-    id INTEGER PRIMARY KEY AUTOINCREMENT,
-    profile_id TEXT NOT NULL,
-    pattern_type TEXT NOT NULL,
-    pattern_key TEXT NOT NULL,
-    pattern_value TEXT DEFAULT '{}',
-    confidence REAL DEFAULT 0.0,
-    evidence_count INTEGER DEFAULT 0,
-    created_at TEXT NOT NULL
-);
-CREATE INDEX IF NOT EXISTS idx_wp_profile
-    ON workflow_patterns(profile_id, pattern_type);
-"""
-
-
-class WorkflowMiner:
-    """Mine workflow sequences and temporal patterns from memory content.
-
-    Args:
-        db_path: Path to a sqlite database.  If the file does not exist
-                 it is created with the required schema.
-    """
-
-    def __init__(self, db_path: Path | str) -> None:
-        self._db_path = Path(db_path)
-        self._ensure_schema()
-
-    # ------------------------------------------------------------------
-    # Public API
-    # ------------------------------------------------------------------
-
-    def record_action(
-        self, profile_id: str, action: str, metadata: dict[str, Any] | None = None
-    ) -> None:
-        """Record a user action for later mining."""
-        now = datetime.now(UTC).isoformat()
-        meta_json = json.dumps(metadata or {})
-        conn = sqlite3.connect(str(self._db_path))
-        try:
-            conn.execute(
-                "INSERT INTO workflow_actions (profile_id, action, metadata, created_at) "
-                "VALUES (?, ?, ?, ?)",
-                (profile_id, action, meta_json, now),
-            )
-            conn.commit()
-        finally:
-            conn.close()
-
-    def mine(self, profile_id: str, min_support: float = 0.3) -> list[dict]:
-        """Mine workflow sequence patterns for a profile.
-
-        Returns a list of pattern dicts sorted by support descending.
-        """
-        actions = self._fetch_actions(profile_id)
-        if len(actions) < 2:
-            return []
-
-        activity_stream = [a["action"] for a in actions]
-        return self._mine_sequences(activity_stream, min_support)
-
-    def mine_from_memories(
-        self,
-        memories: list[dict],
-        min_support: float = 0.3,
-    ) -> list[dict]:
-        """Mine sequences from a pre-fetched list of memory dicts.
-
-        Each dict should have a ``content`` key.
-        """
-        stream: list[str] = []
-        for mem in memories:
-            activity = classify_activity(mem.get("content", ""))
-            if activity != "unknown":
-                stream.append(activity)
-        if len(stream) < 2:
-            return []
-        return self._mine_sequences(stream, min_support)
-
-    def mine_temporal(self, memories: list[dict]) -> dict[str, dict]:
-        """Detect time-of-day activity preferences.
-
-        Returns dict keyed by bucket (morning/afternoon/evening/night).
-        Buckets with < 5 evidence memories are omitted.
-        """
-        buckets: dict[str, Counter] = {
-            "morning": Counter(),
-            "afternoon": Counter(),
-            "evening": Counter(),
-            "night": Counter(),
-        }
-
-        for mem in memories:
-            activity = classify_activity(mem.get("content", ""))
-            if activity == "unknown":
-                continue
-            hour = _parse_hour(mem.get("created_at"))
-            if hour is None:
-                continue
-            bucket = _hour_to_bucket(hour)
-            buckets[bucket][activity] += 1
-
-        result: dict[str, dict] = {}
-        for bucket_name, counter in buckets.items():
-            total = sum(counter.values())
-            if total < 5:
-                continue
-            dominant, dom_count = counter.most_common(1)[0]
-            result[bucket_name] = {
-                "dominant_activity": dominant,
-                "confidence": round(dom_count / total, 4),
-                "evidence_count": total,
-                "distribution": dict(counter),
-            }
-        return result
-
-    # ------------------------------------------------------------------
-    # Internals
-    # ------------------------------------------------------------------
-
-    def _ensure_schema(self) -> None:
-        conn = sqlite3.connect(str(self._db_path))
-        try:
-            conn.executescript(_SCHEMA)
-        finally:
-            conn.close()
-
-    def _fetch_actions(self, profile_id: str) -> list[dict]:
-        conn = sqlite3.connect(str(self._db_path))
-        conn.row_factory = sqlite3.Row
-        try:
-            cur = conn.execute(
-                "SELECT action, created_at FROM workflow_actions "
-                "WHERE profile_id = ? ORDER BY created_at ASC LIMIT 500",
-                (profile_id,),
-            )
-            return [dict(r) for r in cur.fetchall()]
-        except sqlite3.OperationalError:
-            return []
-        finally:
-            conn.close()
-
-    @staticmethod
-    def _mine_sequences(
-        activity_stream: list[str], min_support: float
-    ) -> list[dict]:
-        """Extract n-gram sequences and filter by support."""
-        all_patterns: list[dict] = []
-
-        for n in range(2, 6):
-            if len(activity_stream) < n:
-                continue
-            ngram_counts: Counter = Counter()
-            total_windows = len(activity_stream) - n + 1
-
-            for i in range(total_windows):
-                ngram = tuple(activity_stream[i : i + n])
-                # Skip consecutive identical activities (noise)
-                if any(ngram[j] == ngram[j + 1] for j in range(len(ngram) - 1)):
-                    continue
-                ngram_counts[ngram] += 1
-
-            for ngram, count in ngram_counts.items():
-                support = count / total_windows if total_windows > 0 else 0.0
-                if support >= min_support:
-                    all_patterns.append({
-                        "sequence": list(ngram),
-                        "support": round(support, 4),
-                        "count": count,
-                        "length": n,
-                    })
-
-        all_patterns.sort(key=lambda p: (-p["support"], -p["length"]))
-        return all_patterns[:20]
-
-
-# ----------------------------------------------------------------------
-# Module-level helpers
-# ----------------------------------------------------------------------
-
-
-def classify_activity(content: str) -> str:
-    """Classify content into one of 7 activity types or 'unknown'."""
-    if not content:
-        return "unknown"
-    scores: Counter = Counter()
-    for act_type, pattern in _KEYWORD_PATTERNS:
-        if pattern.search(content):
-            scores[act_type] += 1
-    if not scores:
-        return "unknown"
-    return scores.most_common(1)[0][0]
-
-
-def _hour_to_bucket(hour: int) -> str:
-    if 6 <= hour <= 11:
-        return "morning"
-    if 12 <= hour <= 17:
-        return "afternoon"
-    if 18 <= hour <= 23:
-        return "evening"
-    return "night"
-
-
-def _parse_hour(timestamp: str | None) -> int | None:
-    if not timestamp:
-        return None
-    for fmt in (
-        "%Y-%m-%dT%H:%M:%S",
-        "%Y-%m-%d %H:%M:%S",
-        "%Y-%m-%dT%H:%M:%S.%f",
-        "%Y-%m-%d %H:%M:%S.%f",
-    ):
-        try:
-            return datetime.strptime(timestamp, fmt).hour
-        except (ValueError, TypeError):
-            continue
-    try:
-        return datetime.fromisoformat(timestamp).hour
-    except (ValueError, TypeError):
-        return None

package/src/superlocalmemory/llm/backbone.py

@@ -1,364 +0,0 @@
-# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
-# Licensed under AGPL-3.0-or-later - see LICENSE file
-# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
-
-"""LLM backbone — unified interface for LLM providers.
-
-Supports OpenAI, Anthropic, Azure OpenAI, and Ollama via raw HTTP (httpx).
-Falls back gracefully when no API key is configured — Mode A still works.
-
-Providers:
-- ``"ollama"``: Local Ollama (OpenAI-compatible, no auth needed).
-- ``"openai"``: OpenAI API (GPT-4o, etc.).
-- ``"anthropic"``: Anthropic API (Claude, etc.).
-- ``"azure"``: Azure OpenAI (via AI Foundry deployment).
-
-Part of Qualixar | Author: Varun Pratap Bhardwaj
-License: AGPL-3.0-or-later
-"""
-
-from __future__ import annotations
-
-import logging
-import os
-import socket
-import time
-from dataclasses import dataclass
-from typing import Any
-
-import httpx
-
-from superlocalmemory.core.config import LLMConfig
-
-logger = logging.getLogger(__name__)
-
-# ---------------------------------------------------------------------------
-# Constants
-# ---------------------------------------------------------------------------
-
-_OPENAI_URL = "https://api.openai.com/v1/chat/completions"
-_OPENROUTER_URL = "https://openrouter.ai/api/v1/chat/completions"
-_ANTHROPIC_URL = "https://api.anthropic.com/v1/messages"
-_ANTHROPIC_API_VERSION = "2023-06-01"
-_AZURE_API_VERSION = "2024-12-01-preview"
-_OLLAMA_DEFAULT_BASE = "http://localhost:11434"
-
-_ENV_KEYS: dict[str, str] = {
-    "openai": "OPENAI_API_KEY",
-    "anthropic": "ANTHROPIC_API_KEY",
-    "azure": "AZURE_OPENAI_API_KEY",
-    "ollama": "OLLAMA_HOST",
-    "openrouter": "OPENROUTER_API_KEY",
-}
-
-_SUPPORTED_PROVIDERS = frozenset({"openai", "anthropic", "azure", "ollama", "openrouter"})
-
-_MAX_RETRIES = 3
-_RETRY_BASE_DELAY = 1.0  # seconds, doubles each retry
-
-
-# ---------------------------------------------------------------------------
-# Exceptions
-# ---------------------------------------------------------------------------
-
-class LLMUnavailableError(Exception):
-    """Raised when no API key is available for the configured provider."""
-
-
-# ---------------------------------------------------------------------------
-# Response dataclass
-# ---------------------------------------------------------------------------
-
-@dataclass(frozen=True)
-class LLMResponse:
-    """Immutable container for a single LLM generation result."""
-
-    text: str
-    model: str
-    tokens_used: int
-    latency_ms: float
-
-
-# ---------------------------------------------------------------------------
-# Main class
-# ---------------------------------------------------------------------------
-
-class LLMBackbone:
-    """Unified LLM interface driven by LLMConfig.
-
-    All HTTP via httpx — no provider SDKs needed.
-    Includes retry logic (3 attempts, exponential backoff) and
-    socket-level timeout as a hard backstop for SSL hangs (S15 lesson).
-    """
-
-    def __init__(self, config: LLMConfig) -> None:
-        if config.provider and config.provider not in _SUPPORTED_PROVIDERS:
-            raise ValueError(
-                f"Unsupported provider '{config.provider}'. "
-                f"Choose from {sorted(_SUPPORTED_PROVIDERS)}."
-            )
-        self._provider = config.provider
-        self._model = config.model
-        self._timeout = config.timeout_seconds
-        self._default_temperature = config.temperature
-        self._default_max_tokens = config.max_tokens
-
-        # Resolve API key: config > environment variable. A custom api_base
-        # with an empty api_key is treated as intentionally keyless so local
-        # OpenAI-compatible endpoints do not inherit ambient cloud keys.
-        if self._provider == "ollama":
-            self._api_key = ""
-            host = config.api_base or os.environ.get(
-                "OLLAMA_HOST", _OLLAMA_DEFAULT_BASE,
-            )
-            self._base_url = f"{host.rstrip('/')}/api/chat"
-        elif self._provider == "openrouter":
-            self._api_key = self._resolve_api_key(config)
-            self._base_url = config.api_base or _OPENROUTER_URL
-        elif self._provider:
-            self._api_key = self._resolve_api_key(config)
-            self._base_url = config.api_base
-        else:
-            self._api_key = ""
-            self._base_url = ""
-
-    def _resolve_api_key(self, config: LLMConfig) -> str:
-        if config.api_key:
-            return config.api_key
-        if config.api_base:
-            return ""
-        return os.environ.get(_ENV_KEYS.get(self._provider, ""), "")
-
-    # -- Properties ---------------------------------------------------------
-
-    def is_available(self) -> bool:
-        """True when the provider is ready for requests.
-
-        For Ollama: always True (no API key needed). The num_ctx and
-        keep_alive guards in _build_ollama() protect against memory spikes.
-        The recall-path warm-only guard lives in Summarizer, not here —
-        store/fact-extraction should always use the LLM in Mode B.
-        """
-        if not self._provider:
-            return False
-        if self._provider == "ollama":
-            return True
-        # v3.6.12 (modeb-1): a custom local OpenAI-compatible endpoint
-        # (llama.cpp, LM Studio, vLLM) needs NO API key — _build_openai already
-        # omits the Authorization header when the key is empty. Treat a
-        # configured base_url as sufficient, otherwise Mode B silently falls
-        # back to Mode A extraction for keyless local endpoints.
-        _base = getattr(self, "_base_url", "") or getattr(self, "_api_base", "")
-        return bool(self._api_key) or bool(_base)
-
-    @property
-    def provider(self) -> str:
-        return self._provider
-
-    @property
-    def model(self) -> str:
-        return self._model
-
-    # -- Core generation ----------------------------------------------------
-
-    def generate(
-        self,
-        prompt: str,
-        system: str = "",
-        temperature: float | None = None,
-        max_tokens: int | None = None,
-    ) -> str:
-        """Send prompt to the LLM and return generated text.
-
-        Returns empty string on content-filter errors (Azure 400)
-        instead of crashing — lets callers continue gracefully.
-        Retries up to 3 times with exponential backoff on transient errors.
-        """
-        if not self.is_available():
-            raise LLMUnavailableError(
-                f"No API key for provider '{self._provider}'. "
-                f"Set {_ENV_KEYS.get(self._provider, 'API_KEY')} or pass api_key=."
-            )
-
-        temp = temperature if temperature is not None else self._default_temperature
-        tokens = max_tokens if max_tokens is not None else self._default_max_tokens
-        url, headers, payload = self._build_request(prompt, system, tokens, temp)
-
-        last_error: Exception | None = None
-        for attempt in range(_MAX_RETRIES):
-            try:
-                response = self._send(url, headers, payload)
-                return self._extract_text(response)
-            except httpx.HTTPStatusError as exc:
-                # Azure content filter returns 400 — not retryable.
-                if exc.response.status_code == 400:
-                    logger.warning("Content filter or bad request (400). Returning empty.")
-                    return ""
-                last_error = exc
-            except (httpx.TimeoutException, httpx.ConnectError, ValueError) as exc:
-                last_error = exc
-
-            if attempt < _MAX_RETRIES - 1:
-                delay = _RETRY_BASE_DELAY * (2 ** attempt)
-                logger.info("Retry %d/%d after %.1fs", attempt + 1, _MAX_RETRIES, delay)
-                time.sleep(delay)
-
-        logger.error("All %d retries exhausted: %s", _MAX_RETRIES, last_error)
-        return ""
-
-    # -- HTTP transport -----------------------------------------------------
-
-    def _send(self, url: str, headers: dict, payload: dict) -> dict:
-        """Execute HTTP POST with socket-level SSL backstop."""
-        old_default = socket.getdefaulttimeout()
-        socket.setdefaulttimeout(self._timeout + 30)
-        try:
-            timeout = httpx.Timeout(
-                connect=10.0,
-                read=self._timeout,
-                write=10.0,
-                pool=10.0,
-            )
-            with httpx.Client(timeout=timeout) as client:
-                resp = client.post(url, headers=headers, json=payload)
-                resp.raise_for_status()
-                try:
-                    return resp.json()
-                except Exception as exc:
-                    raise ValueError(
-                        f"LLM endpoint returned HTTP {resp.status_code} with "
-                        f"non-JSON body (Content-Length="
-                        f"{resp.headers.get('content-length', '?')}): {exc}"
-                    ) from exc
-        finally:
-            socket.setdefaulttimeout(old_default)
-
-    # -- Request builders ---------------------------------------------------
-
-    def _build_request(
-        self, prompt: str, system: str, max_tokens: int, temperature: float,
-    ) -> tuple[str, dict[str, str], dict]:
-        """Build provider-specific (url, headers, payload)."""
-        builders = {
-            "ollama": self._build_ollama,
-            "anthropic": self._build_anthropic,
-            "azure": self._build_azure,
-        }
-        builder = builders.get(self._provider, self._build_openai)
-        return builder(prompt, system, max_tokens, temperature)
-
-    def _build_openai(
-        self, prompt: str, system: str, max_tokens: int, temperature: float,
-    ) -> tuple[str, dict[str, str], dict]:
-        messages = self._make_messages(system, prompt)
-        # V3.5.9: Skip Authorization header when api_key is empty so unauthenticated
-        # local endpoints (llama.cpp, LM Studio, etc.) don't reject with HTTP 401.
-        headers = {"Content-Type": "application/json"}
-        if self._api_key:
-            headers["Authorization"] = f"Bearer {self._api_key}"
-        payload = {
-            "model": self._model,
-            "messages": messages,
-            "max_tokens": max_tokens,
-            "temperature": temperature,
-        }
-        # V3.5.9: If base_url is a bare base (e.g. "http://host:port/v1") without
-        # the /chat/completions path, append it — matching OpenAI SDK behaviour and
-        # fixing the /v1/v1 duplication reported in issue #29.
-        url = self._base_url or _OPENAI_URL
-        if self._base_url and not self._base_url.rstrip("/").endswith("chat/completions"):
-            url = f"{self._base_url.rstrip('/')}/chat/completions"
-        return url, headers, payload
-
-    def _build_ollama(
-        self, prompt: str, system: str, max_tokens: int, temperature: float,
-    ) -> tuple[str, dict[str, str], dict]:
-        messages = self._make_messages(system, prompt)
-        headers = {"Content-Type": "application/json"}
-        # Native /api/chat format — NOT /v1/chat/completions.
-        # The OpenAI-compatible endpoint silently ignores options.num_ctx,
-        # causing Ollama to use the model's default (131K for llama3.1 = 30 GB).
-        payload = {
-            "model": self._model,
-            "messages": messages,
-            "stream": False,
-            "keep_alive": "30s",
-            "options": {
-                "num_predict": max_tokens,
-                "temperature": temperature,
-                "num_ctx": 4096,
-            },
-        }
-        return self._base_url, headers, payload
-
-    def _build_anthropic(
-        self, prompt: str, system: str, max_tokens: int, temperature: float,
-    ) -> tuple[str, dict[str, str], dict]:
-        headers = {
-            "x-api-key": self._api_key,
-            "anthropic-version": _ANTHROPIC_API_VERSION,
-            "Content-Type": "application/json",
-        }
-        payload: dict[str, Any] = {
-            "model": self._model,
-            "max_tokens": max_tokens,
-            "temperature": temperature,
-            "messages": [{"role": "user", "content": prompt}],
-        }
-        if system:
-            payload["system"] = system
-        # Respect custom base_url (e.g. Anthropic-compatible proxy).
-        # Append /v1/messages to the root URL, mirroring how _build_openai
-        # handles api_base. Falls back to the official Anthropic endpoint.
-        url = (
-            self._base_url.rstrip("/") + "/v1/messages"
-            if self._base_url else _ANTHROPIC_URL
-        )
-        return url, headers, payload
-
-    def _build_azure(
-        self, prompt: str, system: str, max_tokens: int, temperature: float,
-    ) -> tuple[str, dict[str, str], dict]:
-        if not self._base_url:
-            raise ValueError("Azure provider requires api_base URL.")
-        url = (
-            f"{self._base_url.rstrip('/')}/openai/deployments/"
-            f"{self._model}/chat/completions"
-            f"?api-version={_AZURE_API_VERSION}"
-        )
-        messages = self._make_messages(system, prompt)
-        headers = {"api-key": self._api_key, "Content-Type": "application/json"}
-        payload: dict[str, Any] = {"messages": messages}
-        if "gpt-5" in self._model.lower():
-            payload["max_completion_tokens"] = max(max_tokens, 200)
-            payload["reasoning_effort"] = "none"
-            if temperature > 0:
-                payload["temperature"] = temperature
-        else:
-            payload["max_tokens"] = max_tokens
-            payload["temperature"] = temperature
-        return url, headers, payload
-
-    # -- Response parsing ---------------------------------------------------
-
-    def _extract_text(self, data: dict) -> str:
-        """Extract text from provider-specific JSON response."""
-        if self._provider == "anthropic":
-            return data.get("content", [{}])[0].get("text", "").strip()
-        if self._provider == "ollama":
-            # Native /api/chat: {"message": {"content": "..."}}
-            return data.get("message", {}).get("content", "").strip()
-        # OpenAI / Azure share response format.
-        choices = data.get("choices", [{}])
-        return choices[0].get("message", {}).get("content", "").strip()
-
-    # -- Helpers ------------------------------------------------------------
-
-    @staticmethod
-    def _make_messages(system: str, prompt: str) -> list[dict[str, str]]:
-        """Build messages array with optional system message."""
-        messages: list[dict[str, str]] = []
-        if system:
-            messages.append({"role": "system", "content": system})
-        messages.append({"role": "user", "content": prompt})
-        return messages