superlocalmemory 3.2.2 → 3.3.0

package/src/superlocalmemory/parameterization/soft_prompt_generator.py

@@ -0,0 +1,425 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under the MIT License - see LICENSE file
+# Part of SuperLocalMemory V3.3
+
+"""SoftPromptGenerator — Convert extracted patterns to text soft prompts.
+
+Pure text personality encoding. No LoRA, no model weights.
+Token budget management with priority ordering by category.
+
+Part of Qualixar | Author: Varun Pratap Bhardwaj
+"""
+
+from __future__ import annotations
+
+import logging
+import uuid
+from collections import defaultdict
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from superlocalmemory.core.config import ParameterizationConfig
+
+from superlocalmemory.parameterization.pattern_extractor import (
+    PatternAssertion,
+    PatternCategory,
+)
+from superlocalmemory.parameterization.pii_filter import PIIFilter
+
+logger = logging.getLogger(__name__)
+
+# ---------------------------------------------------------------------------
+# Category templates — natural language patterns for each category
+# ---------------------------------------------------------------------------
+
+CATEGORY_TEMPLATES: dict[str, str] = {
+    "identity": (
+        "The user is {role} with expertise in {domains}. "
+        "They work at {organization}."
+    ),
+    "tech_preference": (
+        "The user's preferred technology stack includes: {technologies}. "
+        "Default to these when generating code or making recommendations."
+    ),
+    "communication_style": (
+        "The user prefers {style} responses. {specific_preferences}"
+    ),
+    "workflow_pattern": (
+        "The user typically {workflow_description}. "
+        "Anticipate this workflow when assisting."
+    ),
+    "project_context": (
+        "Current active project: {project_name}. "
+        "Key context: {context_summary}."
+    ),
+    "decision_history": (
+        "Recent key decisions: {decisions}. "
+        "These reflect the user's current direction."
+    ),
+    "avoidance": (
+        "The user has explicitly asked to avoid: {avoid_list}. "
+        "Do not suggest or use these."
+    ),
+}
+
+CATEGORY_PRIORITY_ORDER: list[str] = [
+    "identity",
+    "tech_preference",
+    "communication_style",
+    "workflow_pattern",
+    "project_context",
+    "decision_history",
+    "avoidance",
+]
+
+# ---------------------------------------------------------------------------
+# Data model
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class SoftPromptTemplate:
+    """A generated soft prompt for one category."""
+
+    prompt_id: str
+    profile_id: str
+    category: str
+    content: str
+    source_pattern_ids: list[str]
+    confidence: float
+    effectiveness: float
+    token_count: int
+    retention_score: float
+    active: bool
+    version: int
+
+
+# ---------------------------------------------------------------------------
+# SoftPromptGenerator class
+# ---------------------------------------------------------------------------
+
+class SoftPromptGenerator:
+    """Convert extracted pattern assertions into natural language soft prompts.
+
+    Respects token budget and priority ordering. Filters PII.
+    """
+
+    def __init__(self, config: ParameterizationConfig) -> None:
+        self._config = config
+        self._pii_filter = PIIFilter()
+
+    # ------------------------------------------------------------------
+    # Public API
+    # ------------------------------------------------------------------
+
+    def generate(
+        self,
+        patterns: list[PatternAssertion],
+        profile_id: str,
+    ) -> list[SoftPromptTemplate]:
+        """Master generation pipeline: filter, group, render, budget-trim.
+
+        Args:
+            patterns: Extracted pattern assertions.
+            profile_id: Target profile.
+
+        Returns:
+            List of SoftPromptTemplate, ordered by category priority,
+            within token budget.
+        """
+        # Filter to enabled categories
+        enabled = set(self._config.categories_enabled)
+        filtered = [
+            p for p in patterns if p.category.value in enabled
+        ]
+
+        # Group by category
+        grouped: dict[str, list[PatternAssertion]] = defaultdict(list)
+        for p in filtered:
+            grouped[p.category.value].append(p)
+
+        prompts: list[SoftPromptTemplate] = []
+        # Reserve tokens for assembly header + separators
+        header_overhead = self._estimate_tokens(
+            "# User Profile (auto-learned)"
+        ) + 2  # newlines
+        running_token_count = header_overhead
+
+        for category in CATEGORY_PRIORITY_ORDER:
+            if category not in grouped:
+                continue
+
+            template = self._render_category(
+                category, grouped[category], profile_id,
+            )
+            if template is None:
+                continue
+
+            token_count = self._estimate_tokens(template.content)
+
+            if running_token_count + token_count > self._config.max_prompt_tokens:
+                remaining = self._config.max_prompt_tokens - running_token_count
+                template = self._trim_content(template, remaining)
+                token_count = self._estimate_tokens(template.content)
+                if not template.content or len(template.content) < 20:
+                    break
+
+            template.token_count = token_count
+            prompts.append(template)
+            running_token_count += token_count
+
+        return self._validate(prompts)
+
+    def assemble(self, prompts: list[SoftPromptTemplate]) -> str:
+        """Assemble prompts into a single text block with header.
+
+        Args:
+            prompts: List of templates to assemble.
+
+        Returns:
+            Assembled text or empty string if no prompts.
+        """
+        if not prompts:
+            return ""
+
+        lines = ["# User Profile (auto-learned)", ""]
+
+        # Sort by priority order
+        order_map = {
+            cat: idx for idx, cat in enumerate(CATEGORY_PRIORITY_ORDER)
+        }
+        sorted_prompts = sorted(
+            prompts,
+            key=lambda p: order_map.get(p.category, len(CATEGORY_PRIORITY_ORDER)),
+        )
+
+        for prompt in sorted_prompts:
+            lines.append(prompt.content)
+            lines.append("")
+
+        result = "\n".join(lines).rstrip()
+        return result
+
+    # ------------------------------------------------------------------
+    # Rendering
+    # ------------------------------------------------------------------
+
+    def _render_category(
+        self,
+        category: str,
+        patterns: list[PatternAssertion],
+        profile_id: str,
+    ) -> SoftPromptTemplate | None:
+        """Render patterns for a single category into a SoftPromptTemplate."""
+        template_str = CATEGORY_TEMPLATES.get(category, "{key}: {value}")
+        values = self._extract_template_values(category, patterns)
+
+        # Fill template with defaultdict for missing keys
+        safe_values = defaultdict(str, values)
+        try:
+            content = template_str.format_map(safe_values)
+        except (KeyError, IndexError, ValueError):  # pragma: no cover
+            content = ", ".join(p.value for p in patterns)  # pragma: no cover
+
+        # Clean up
+        content = self._clean_content(content)
+
+        # Filter PII
+        content = self._pii_filter.filter_text(content)
+        if not content.strip():
+            return None
+
+        # Trim to 100 tokens per category
+        content = self._trim_to_tokens(content, 100)
+
+        # Aggregate confidence
+        confidence = (
+            sum(p.confidence for p in patterns) / len(patterns)
+            if patterns else 0.0
+        )
+
+        # Collect source IDs
+        source_ids = [sid for p in patterns for sid in p.source_ids]
+
+        return SoftPromptTemplate(
+            prompt_id=str(uuid.uuid4()),
+            profile_id=profile_id,
+            category=category,
+            content=content,
+            source_pattern_ids=source_ids,
+            confidence=confidence,
+            effectiveness=0.5,
+            token_count=0,
+            retention_score=1.0,
+            active=True,
+            version=1,
+        )
+
+    @staticmethod
+    def _extract_template_values(
+        category: str,
+        patterns: list[PatternAssertion],
+    ) -> dict[str, str]:
+        """Extract placeholder values from patterns for a category template."""
+        values: dict[str, str] = {}
+        pat_values = [p.value for p in patterns]
+
+        if category == "identity":
+            values["role"] = pat_values[0] if pat_values else ""
+            values["domains"] = ", ".join(pat_values)
+            org_patterns = [
+                p for p in patterns
+                if "organization" in p.key or "company" in p.key
+            ]
+            values["organization"] = (
+                org_patterns[0].value if org_patterns else ""
+            )
+
+        elif category == "tech_preference":
+            values["technologies"] = ", ".join(pat_values)
+
+        elif category == "communication_style":
+            values["style"] = pat_values[0] if pat_values else ""
+            values["specific_preferences"] = ", ".join(pat_values[1:])
+
+        elif category == "workflow_pattern":
+            values["workflow_description"] = "; ".join(pat_values)
+
+        elif category == "project_context":
+            values["project_name"] = pat_values[0] if pat_values else ""
+            values["context_summary"] = ", ".join(pat_values[1:])
+
+        elif category == "decision_history":
+            values["decisions"] = ", ".join(pat_values)
+
+        elif category == "avoidance":
+            values["avoid_list"] = ", ".join(pat_values)
+
+        else:
+            # Generic fallback
+            values["key"] = patterns[0].key if patterns else ""
+            values["value"] = ", ".join(pat_values)
+
+        return values
+
+    # ------------------------------------------------------------------
+    # Validation
+    # ------------------------------------------------------------------
+
+    def _validate(
+        self, prompts: list[SoftPromptTemplate],
+    ) -> list[SoftPromptTemplate]:
+        """Validate prompts: no empty content, no PII, within token limits."""
+        valid: list[SoftPromptTemplate] = []
+        seen_categories: set[str] = set()
+
+        for prompt in prompts:
+            if not prompt.content.strip():
+                logger.warning(
+                    "Skipping empty prompt for category %s", prompt.category,
+                )
+                continue
+            if self._pii_filter.has_pii(prompt.content):
+                logger.warning(
+                    "PII detected in prompt for category %s, skipping",
+                    prompt.category,
+                )
+                continue
+            if prompt.token_count > 100:
+                logger.warning(
+                    "Prompt for %s exceeds 100 tokens (%d), trimming",
+                    prompt.category, prompt.token_count,
+                )
+                prompt.content = self._trim_to_tokens(prompt.content, 100)
+                prompt.token_count = self._estimate_tokens(prompt.content)
+            if prompt.category in seen_categories:
+                logger.warning(
+                    "Duplicate active category %s, skipping", prompt.category,
+                )
+                continue
+            seen_categories.add(prompt.category)
+            valid.append(prompt)
+
+        return valid
+
+    # ------------------------------------------------------------------
+    # Token utilities
+    # ------------------------------------------------------------------
+
+    @staticmethod
+    def _estimate_tokens(text: str) -> int:
+        """Estimate token count from text (~1.3 tokens per word)."""
+        word_count = len(text.split())
+        return max(1, int(word_count * 1.3))
+
+    @staticmethod
+    def _trim_to_tokens(text: str, max_tokens: int) -> str:
+        """Trim text to fit within a token budget by removing sentences."""
+        current = SoftPromptGenerator._estimate_tokens(text)
+        if current <= max_tokens:
+            return text
+
+        # Split into sentences
+        import re
+        sentences = re.split(r"(?<=\.)\s+", text)
+        result_sentences: list[str] = []
+        accumulated = 0
+
+        for sentence in sentences:
+            sent_tokens = SoftPromptGenerator._estimate_tokens(sentence)
+            if accumulated + sent_tokens > max_tokens:
+                break
+            result_sentences.append(sentence)
+            accumulated += sent_tokens
+
+        if not result_sentences:
+            # Take first N words if no sentence fits
+            words = text.split()
+            target_words = max(1, int(max_tokens / 1.3))
+            return " ".join(words[:target_words]).rstrip(".,;: ") + "."
+
+        result = " ".join(result_sentences)
+        if not result.endswith("."):  # pragma: no cover — regex split keeps "." with chunks
+            result += "."  # pragma: no cover
+        return result
+
+    @staticmethod
+    def _trim_content(
+        template: SoftPromptTemplate,
+        remaining_budget: int,
+    ) -> SoftPromptTemplate:
+        """Create a new template with content trimmed to budget (immutable)."""
+        trimmed = SoftPromptGenerator._trim_to_tokens(
+            template.content, remaining_budget,
+        )
+        return SoftPromptTemplate(
+            prompt_id=template.prompt_id,
+            profile_id=template.profile_id,
+            category=template.category,
+            content=trimmed,
+            source_pattern_ids=template.source_pattern_ids,
+            confidence=template.confidence,
+            effectiveness=template.effectiveness,
+            token_count=SoftPromptGenerator._estimate_tokens(trimmed),
+            retention_score=template.retention_score,
+            active=template.active,
+            version=template.version,
+        )
+
+    # ------------------------------------------------------------------
+    # Content cleaning
+    # ------------------------------------------------------------------
+
+    @staticmethod
+    def _clean_content(content: str) -> str:
+        """Strip extra whitespace, remove empty sentences."""
+        import re
+        # Collapse multiple spaces
+        content = re.sub(r"\s+", " ", content).strip()
+        # Remove empty sentences (". .")
+        content = re.sub(r"\.\s*\.", ".", content)
+        # Ensure ends with period
+        if content and not content.endswith("."):
+            content += "."
+        return content

package/src/superlocalmemory/retrieval/engine.py

@@ -2,9 +2,10 @@
 # Licensed under the MIT License - see LICENSE file
 # Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
 
-"""SuperLocalMemory V3 — Retrieval Engine (4-Channel Orchestrator).
+"""SuperLocalMemory V3 — Retrieval Engine (6-Channel Orchestrator).
 
-4 channels -> single RRF fusion -> optional cross-encoder rerank.
+6 channels -> single RRF fusion -> optional cross-encoder rerank.
+Channels: semantic, BM25, entity_graph, temporal, spreading_activation, hopfield.
 Replaces V1's broken 10-channel triple-re-fusion pipeline.
 
 Part of Qualixar | Author: Varun Pratap Bhardwaj
@@ -28,6 +29,7 @@ from superlocalmemory.storage.models import (
 if TYPE_CHECKING:
     from superlocalmemory.retrieval.bm25_channel import BM25Channel
     from superlocalmemory.retrieval.entity_channel import EntityGraphChannel
+    from superlocalmemory.retrieval.hopfield_channel import HopfieldChannel
     from superlocalmemory.retrieval.semantic_channel import SemanticChannel
     from superlocalmemory.retrieval.temporal_channel import TemporalChannel
     from superlocalmemory.storage.database import DatabaseManager
@@ -47,7 +49,7 @@ class EmbeddingProvider(Protocol):
 
 
 class RetrievalEngine:
-    """4-channel retrieval: semantic + BM25 + entity_graph + temporal.
+    """6-channel retrieval: semantic + BM25 + entity_graph + temporal + spreading_activation + hopfield.
 
     Usage::
         engine = RetrievalEngine(db, config, channels, embedder)
@@ -71,6 +73,8 @@ class RetrievalEngine:
         self._bm25: BM25Channel | None = channels.get("bm25")
         self._entity: EntityGraphChannel | None = channels.get("entity_graph")
         self._temporal: TemporalChannel | None = channels.get("temporal")
+        # Phase G: Hopfield channel (6th)
+        self._hopfield: HopfieldChannel | None = channels.get("hopfield")
         self._embedder = embedder
         self._reranker = reranker
         self._strategy = strategy or QueryStrategyClassifier()
@@ -90,6 +94,9 @@ class RetrievalEngine:
             self._registry.register_channel("entity_graph", self._entity)
         if self._temporal is not None:
             self._registry.register_channel("temporal", self._temporal)
+        # Phase G: Hopfield channel (6th) — needs embedding input
+        if self._hopfield is not None:
+            self._registry.register_channel("hopfield", self._hopfield, needs_embedding=True)
 
     def recall(
         self, query: str, profile_id: str,
@@ -223,6 +230,17 @@ class RetrievalEngine:
             except Exception as exc:
                 logger.warning("Temporal channel: %s", exc)
 
+        # Phase G: Hopfield channel (6th) — energy-based pattern completion
+        if self._hopfield is not None and "hopfield" not in disabled:
+            try:
+                q_emb = self._embedder.embed(query) if self._embedder else None
+                if q_emb is not None:
+                    r = self._hopfield.search(q_emb, profile_id, self._config.hopfield_top_k)
+                    if r:
+                        out["hopfield"] = r
+            except Exception as exc:
+                logger.warning("Hopfield channel: %s", exc)
+
         return out
 
     # -- Fact loading -------------------------------------------------------

package/src/superlocalmemory/retrieval/forgetting_filter.py

@@ -0,0 +1,145 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under the MIT License - see LICENSE file
+# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
+
+"""Forgetting filter for retrieval pipeline.
+
+Post-retrieval filter that adjusts scores based on Ebbinghaus retention:
+  - Active/warm facts: score weighted by lifecycle_weight
+  - Cold facts: score reduced (weight = 0.3)
+  - Archive/forgotten facts: REMOVED from results entirely
+
+Integrates with ChannelRegistry.register_filter() using the FilterFn
+signature: (all_results, profile_id, context) -> filtered_results.
+
+HR-06: When config.enabled=False, returns results unchanged.
+
+Part of Qualixar | Author: Varun Pratap Bhardwaj
+License: MIT
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import TYPE_CHECKING, Any
+
+from superlocalmemory.core.config import ForgettingConfig
+
+if TYPE_CHECKING:
+    from superlocalmemory.retrieval.channel_registry import ChannelRegistry
+    from superlocalmemory.storage.database import DatabaseManager
+
+logger = logging.getLogger(__name__)
+
+
+# Lifecycle zone weights — same as in ebbinghaus.py
+_ZONE_WEIGHTS: dict[str, float] = {
+    "active": 1.0,
+    "warm": 0.7,
+    "cold": 0.3,
+    "archive": 0.0,
+    "forgotten": 0.0,
+}
+
+# Zones where facts are excluded from results
+_EXCLUDED_ZONES: frozenset[str] = frozenset({"archive", "forgotten"})
+
+
+class ForgettingFilter:
+    """Post-retrieval filter that applies Ebbinghaus retention weighting.
+
+    Removes archived/forgotten facts and adjusts scores for other zones.
+    """
+
+    __slots__ = ("_db", "_config")
+
+    def __init__(self, db: DatabaseManager, config: ForgettingConfig) -> None:
+        self._db = db
+        self._config = config
+
+    def filter(
+        self,
+        all_results: dict[str, list[tuple[str, float]]],
+        profile_id: str,
+        context: Any,
+    ) -> dict[str, list[tuple[str, float]]]:
+        """Apply forgetting filter to retrieval results.
+
+        Matches FilterFn signature from channel_registry.py.
+
+        Args:
+            all_results: Channel name -> [(fact_id, score)] dict.
+            profile_id: Current profile.
+            context: Optional context (unused).
+
+        Returns:
+            Filtered results dict with scores adjusted by retention weight.
+        """
+        # HR-06: If disabled, return unchanged
+        if not self._config.enabled:
+            return all_results
+
+        # Collect all unique fact_ids across all channels
+        all_fact_ids: set[str] = set()
+        for channel_results in all_results.values():
+            for fact_id, _ in channel_results:
+                all_fact_ids.add(fact_id)
+
+        if not all_fact_ids:
+            return all_results
+
+        # Batch query retention data
+        retention_rows = self._db.batch_get_retention(
+            list(all_fact_ids), profile_id,
+        )
+
+        # Build lookup: fact_id -> retention row
+        retention_map: dict[str, dict] = {}
+        for row in retention_rows:
+            retention_map[row["fact_id"]] = row
+
+        # Filter and weight each channel's results
+        filtered: dict[str, list[tuple[str, float]]] = {}
+        for channel_name, channel_results in all_results.items():
+            new_results: list[tuple[str, float]] = []
+            for fact_id, score in channel_results:
+                ret_data = retention_map.get(fact_id)
+
+                if ret_data is None:
+                    # No retention data yet -> new memory, keep as-is
+                    new_results.append((fact_id, score))
+                    continue
+
+                zone = ret_data.get("lifecycle_zone", "active")
+
+                if zone in _EXCLUDED_ZONES:
+                    # Archive/forgotten: remove from results
+                    continue
+
+                # Apply weight
+                weight = _ZONE_WEIGHTS.get(zone, 1.0)
+                new_results.append((fact_id, score * weight))
+
+            filtered[channel_name] = new_results
+
+        return filtered
+
+
+def register_forgetting_filter(
+    registry: ChannelRegistry,
+    db: DatabaseManager,
+    config: ForgettingConfig,
+) -> None:
+    """Register the forgetting filter with the channel registry.
+
+    Does nothing if config.enabled is False.
+
+    Args:
+        registry: Channel registry to register with.
+        db: Database manager for retention queries.
+        config: Forgetting configuration.
+    """
+    if not config.enabled:
+        return
+    f = ForgettingFilter(db, config)
+    registry.register_filter(f.filter)