superlocalmemory 3.2.2 → 3.3.0

package/src/superlocalmemory/learning/forgetting_scheduler.py

@@ -0,0 +1,263 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under the MIT License - see LICENSE file
+# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
+
+"""Forgetting scheduler — periodic retention decay and lifecycle management.
+
+Runs the Ebbinghaus decay cycle on all facts for a profile:
+1. Fetches all facts with access counts, importance, and emotion data
+2. Excludes core memory facts (HR-01: Core Memory NEVER forgets)
+3. Computes retention scores via EbbinghausCurve
+4. UPSERTs results into fact_retention table
+5. Soft-deletes facts that fall below forget_threshold
+
+Also handles real-time spaced repetition updates on access events.
+
+HR-04: Soft-delete ONLY. Never physically deletes.
+HR-08: Runs synchronously in main thread.
+HR-09: Access events are idempotent.
+
+Part of Qualixar | Author: Varun Pratap Bhardwaj
+License: MIT
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import time
+from datetime import UTC, datetime
+from typing import TYPE_CHECKING
+
+from superlocalmemory.core.config import ForgettingConfig
+from superlocalmemory.math.ebbinghaus import EbbinghausCurve
+
+if TYPE_CHECKING:
+    from superlocalmemory.storage.database import DatabaseManager
+
+logger = logging.getLogger(__name__)
+
+
+class ForgettingScheduler:
+    """Periodic forgetting scheduler for memory lifecycle management.
+
+    Computes Ebbinghaus retention scores and manages memory lifecycle
+    transitions. Runs synchronously (HR-08).
+    """
+
+    __slots__ = ("_db", "_ebbinghaus", "_config", "_last_run_times")
+
+    def __init__(
+        self,
+        db: DatabaseManager,
+        ebbinghaus: EbbinghausCurve,
+        config: ForgettingConfig,
+    ) -> None:
+        self._db = db
+        self._ebbinghaus = ebbinghaus
+        self._config = config
+        # Track last run time per profile for interval enforcement
+        self._last_run_times: dict[str, float] = {}
+
+    def run_decay_cycle(
+        self, profile_id: str, *, force: bool = False,
+    ) -> dict:
+        """Run a full decay cycle for all facts in a profile.
+
+        Fetches facts, computes retention, upserts results, and
+        soft-deletes forgotten facts.
+
+        Args:
+            profile_id: Profile to process.
+            force: If True, bypass interval check.
+
+        Returns:
+            Stats dict: {total, active, warm, cold, archive, forgotten,
+                         transitions, skipped}.
+        """
+        # Interval check (Test 21)
+        now = time.monotonic()
+        last_run = self._last_run_times.get(profile_id)
+        if not force and last_run is not None:
+            elapsed_minutes = (now - last_run) / 60.0
+            if elapsed_minutes < self._config.scheduler_interval_minutes:
+                return {"skipped": True, "reason": "within_interval"}
+
+        # Step 1: Fetch all facts with metadata
+        facts_data = self._fetch_facts_with_metadata(profile_id)
+
+        if not facts_data:
+            self._last_run_times[profile_id] = now
+            return {
+                "total": 0, "active": 0, "warm": 0, "cold": 0,
+                "archive": 0, "forgotten": 0, "transitions": 0,
+            }
+
+        # Step 2: Get existing retention data for transition tracking
+        existing_zones = self._get_existing_zones(profile_id, facts_data)
+
+        # Step 3: Compute retention scores via EbbinghausCurve
+        retention_results = self._ebbinghaus.batch_compute_retention(facts_data)
+
+        # Step 4: Batch UPSERT into fact_retention
+        self._db.batch_upsert_retention(retention_results, profile_id)
+
+        # Step 5: Count zones and transitions
+        zone_counts = {"active": 0, "warm": 0, "cold": 0, "archive": 0, "forgotten": 0}
+        transitions = 0
+        forgotten_fact_ids: list[str] = []
+
+        for result in retention_results:
+            zone = result["zone"]
+            zone_counts[zone] = zone_counts.get(zone, 0) + 1
+
+            # Track transitions
+            old_zone = existing_zones.get(result["fact_id"])
+            if old_zone is not None and old_zone != zone:
+                transitions += 1
+
+            # Collect forgotten facts for soft-delete
+            if zone == "forgotten":
+                forgotten_fact_ids.append(result["fact_id"])
+
+        # Step 6: Soft-delete forgotten facts (HR-04)
+        for fact_id in forgotten_fact_ids:
+            self._soft_delete_with_audit(fact_id, profile_id)
+
+        # Update last run time
+        self._last_run_times[profile_id] = now
+
+        return {
+            "total": len(retention_results),
+            "active": zone_counts["active"],
+            "warm": zone_counts["warm"],
+            "cold": zone_counts["cold"],
+            "archive": zone_counts["archive"],
+            "forgotten": zone_counts["forgotten"],
+            "transitions": transitions,
+        }
+
+    def on_access_event(self, fact_id: str, profile_id: str) -> None:
+        """Handle real-time access event with spaced repetition update.
+
+        Called from access_log.store_access() hook.
+        HR-09: Idempotent — reads current state, computes from scratch.
+
+        Args:
+            fact_id: Accessed fact ID.
+            profile_id: Profile ID.
+        """
+        # Fetch current retention data
+        current = self._db.get_retention(fact_id, profile_id)
+        if current is None:
+            # No retention data yet — nothing to update
+            logger.debug("on_access_event: no retention for %s, skipping", fact_id)
+            return
+
+        current_strength = float(current["memory_strength"])
+        last_accessed = current.get("last_accessed_at", "")
+
+        # Compute hours since last access
+        hours_since = 0.0
+        if last_accessed:
+            try:
+                last_dt = datetime.fromisoformat(last_accessed)
+                if last_dt.tzinfo is None:
+                    last_dt = last_dt.replace(tzinfo=UTC)
+                now_dt = datetime.now(UTC)
+                hours_since = max(0.0, (now_dt - last_dt).total_seconds() / 3600.0)
+            except (ValueError, TypeError):
+                hours_since = 0.0
+
+        # Spaced repetition update (HR-07: only increases strength)
+        new_strength = self._ebbinghaus.spaced_repetition_update(
+            current_strength, hours_since,
+        )
+
+        # Recompute retention with new strength
+        new_retention = self._ebbinghaus.retention(0.0, new_strength)
+        new_zone = self._ebbinghaus.lifecycle_zone(new_retention)
+
+        now_iso = datetime.now(UTC).isoformat()
+
+        # UPSERT updated data
+        self._db.upsert_retention(
+            fact_id=fact_id,
+            profile_id=profile_id,
+            retention_score=new_retention,
+            memory_strength=new_strength,
+            access_count=int(current["access_count"]) + 1,
+            last_accessed_at=now_iso,
+            lifecycle_zone=new_zone,
+        )
+
+    # ------------------------------------------------------------------
+    # Private helpers
+    # ------------------------------------------------------------------
+
+    def _fetch_facts_with_metadata(self, profile_id: str) -> list[dict]:
+        """Fetch all non-core-memory facts with access, importance, and emotion data.
+
+        Uses the query from LLD Section 2.4 with A-HIGH-02/03 fixes:
+        - confirmation_count mapped from atomic_facts.evidence_count
+        - emotional_salience from atomic_facts.emotional_valence
+        """
+        rows = self._db.execute(
+            "SELECT f.fact_id, "
+            "  COALESCE(al.access_count, 0) as access_count, "
+            "  COALESCE(fi.pagerank_score, 0.0) as importance, "
+            "  COALESCE(f.evidence_count, 0) as confirmation_count, "
+            "  f.created_at, "
+            "  COALESCE(r.last_accessed_at, f.created_at) as last_accessed_at, "
+            "  COALESCE(f.emotional_valence, 0.0) as emotional_salience "
+            "FROM atomic_facts f "
+            "LEFT JOIN ("
+            "  SELECT fact_id, COUNT(*) as access_count "
+            "  FROM fact_access_log WHERE profile_id = ? GROUP BY fact_id"
+            ") al ON f.fact_id = al.fact_id "
+            "LEFT JOIN fact_importance fi "
+            "  ON f.fact_id = fi.fact_id AND fi.profile_id = ? "
+            "LEFT JOIN fact_retention r "
+            "  ON f.fact_id = r.fact_id AND r.profile_id = ? "
+            "WHERE f.profile_id = ? "
+            "AND f.fact_id NOT IN ("
+            "  SELECT json_each.value "
+            "  FROM core_memory_blocks, json_each(core_memory_blocks.source_fact_ids) "
+            "  WHERE core_memory_blocks.profile_id = ?"
+            ")",
+            (profile_id, profile_id, profile_id, profile_id, profile_id),
+        )
+
+        facts: list[dict] = []
+        for row in rows:
+            d = dict(row)
+            facts.append({
+                "fact_id": d["fact_id"],
+                "access_count": int(d["access_count"]),
+                "importance": float(d["importance"]),
+                "confirmation_count": int(d["confirmation_count"]),
+                "emotional_salience": float(d["emotional_salience"]),
+                "last_accessed_at": str(d["last_accessed_at"]),
+            })
+        return facts
+
+    def _get_existing_zones(
+        self, profile_id: str, facts_data: list[dict],
+    ) -> dict[str, str]:
+        """Get existing lifecycle zones for transition tracking."""
+        fact_ids = [f["fact_id"] for f in facts_data]
+        if not fact_ids:
+            return {}
+        retention_rows = self._db.batch_get_retention(fact_ids, profile_id)
+        return {r["fact_id"]: r["lifecycle_zone"] for r in retention_rows}
+
+    def _soft_delete_with_audit(self, fact_id: str, profile_id: str) -> None:
+        """Soft-delete a forgotten fact with compliance audit trail.
+
+        HR-04: Never physically deletes.
+        """
+        logger.info(
+            "Soft-deleting forgotten fact: fact_id=%s, profile_id=%s",
+            fact_id, profile_id,
+        )
+        self._db.soft_delete_fact(fact_id, profile_id)

package/src/superlocalmemory/learning/quantization_scheduler.py

@@ -0,0 +1,320 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under the MIT License - see LICENSE file
+# Part of SuperLocalMemory V3
+
+"""Quantization Scheduler -- combined SAGQ + EAP precision management.
+
+Background quantization worker that periodically reviews all memories and
+applies both EAP (forgetting) and SAGQ (network centrality) signals together.
+
+Conflict resolution: max(EAP_precision, SAGQ_precision) -- safety first.
+Core Memory blocks are immune to quantization (HR-01).
+Every precision change is logged to fact_access_log for audit trail (HR-09).
+
+HR-01: Core Memory immune.
+HR-07: No-op when config.enabled=False.
+HR-08: Synchronous execution (no threading).
+HR-09: Audit trail for every change.
+
+Part of Qualixar | Author: Varun Pratap Bhardwaj
+License: MIT
+"""
+
+from __future__ import annotations
+
+import logging
+import time
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from typing import Any, Callable, TYPE_CHECKING
+
+from superlocalmemory.storage.models import _new_id
+
+if TYPE_CHECKING:
+    from superlocalmemory.storage.database import DatabaseManager
+    from superlocalmemory.dynamics.activation_guided_quantization import (
+        ActivationGuidedQuantizer,
+        SAGQPrecision,
+    )
+    from superlocalmemory.core.config import SAGQConfig
+
+logger = logging.getLogger(__name__)
+
+
+# ---------------------------------------------------------------------------
+# Data classes (frozen -- all immutable, Rule 10)
+# ---------------------------------------------------------------------------
+
+
+@dataclass(frozen=True)
+class PrecisionChange:
+    """Record of a single precision change for audit trail."""
+
+    fact_id: str
+    old_bit_width: int
+    new_bit_width: int
+    action: str       # "upgrade" | "downgrade"
+    centrality: float
+    sagq_signal: int
+    eap_signal: int
+    timestamp: str    # ISO 8601 datetime string
+
+
+@dataclass(frozen=True)
+class SchedulerRunResult:
+    """Result of a single scheduler run."""
+
+    total_facts: int
+    upgrades: int
+    downgrades: int
+    skipped: int
+    errors: int
+    changes: tuple[PrecisionChange, ...]
+    duration_ms: float
+
+
+# ---------------------------------------------------------------------------
+# Bit-width -> quantization level mapping
+# ---------------------------------------------------------------------------
+
+_BW_TO_LEVEL: dict[int, str] = {
+    32: "float32",
+    8: "int8",
+    4: "polar4",
+    2: "polar2",
+    0: "deleted",
+}
+
+
+# ---------------------------------------------------------------------------
+# QuantizationScheduler
+# ---------------------------------------------------------------------------
+
+
+class QuantizationScheduler:
+    """Combined SAGQ + EAP quantization scheduler.
+
+    Runs synchronously (HR-08). Called by the consolidation engine
+    or CLI command on schedule.
+    """
+
+    def __init__(
+        self,
+        db: Any,
+        sagq: Any,
+        eap_mapper: Callable[[str], int],
+        quantized_store: Any,
+        vector_store: Any,
+        config: Any,
+    ) -> None:
+        """Initialize scheduler. No side effects."""
+        self._db = db
+        self._sagq = sagq
+        self._eap_mapper = eap_mapper
+        self._quantized_store = quantized_store
+        self._vector_store = vector_store
+        self._config = config
+
+    def run(self, profile_id: str) -> SchedulerRunResult:
+        """Execute one combined SAGQ + EAP quantization pass.
+
+        Algorithm:
+          1. Compute SAGQ precision recommendations (centrality + EAP + max())
+          2. Exclude core memory facts (HR-01)
+          3. Execute upgrades/downgrades with error isolation
+          4. Log audit trail for each change (HR-09)
+          5. Return summary
+
+        Returns SchedulerRunResult with totals and change records.
+        """
+        # HR-07: No-op when disabled
+        if not self._config.enabled:
+            return SchedulerRunResult(
+                total_facts=0, upgrades=0, downgrades=0,
+                skipped=0, errors=0, changes=(), duration_ms=0.0,
+            )
+
+        start_time = time.monotonic()
+
+        # Step 2: Get SAGQ precision recommendations
+        recommendations = self._sagq.compute_sagq_precision_batch(
+            profile_id, self._eap_mapper,
+        )
+
+        if not recommendations:
+            duration_ms = (time.monotonic() - start_time) * 1000
+            return SchedulerRunResult(
+                total_facts=0, upgrades=0, downgrades=0,
+                skipped=0, errors=0, changes=(), duration_ms=duration_ms,
+            )
+
+        # Step 5a: Get core memory fact IDs (HR-01)
+        core_fact_ids = self._get_core_fact_ids(profile_id)
+
+        # Step 6: Process each recommendation
+        upgrades = 0
+        downgrades = 0
+        skipped = 0
+        errors = 0
+        changes: list[PrecisionChange] = []
+
+        for prec in recommendations:
+            # HR-01: Core Memory immune
+            if prec.fact_id in core_fact_ids:
+                skipped += 1
+                continue
+
+            if prec.action == "skip":
+                skipped += 1
+                continue
+
+            change = self._process_precision_change(prec, profile_id)
+
+            if change is None:
+                # Error or unable to process
+                if prec.action in ("downgrade", "upgrade"):
+                    errors += 1
+                else:
+                    skipped += 1
+                continue
+
+            if change.action == "downgrade":
+                downgrades += 1
+            elif change.action == "upgrade":
+                upgrades += 1
+
+            changes.append(change)
+
+        duration_ms = (time.monotonic() - start_time) * 1000
+
+        logger.info(
+            "SAGQ scheduler: %d upgrades, %d downgrades, %d skipped, "
+            "%d errors in %.1fms",
+            upgrades, downgrades, skipped, errors, duration_ms,
+        )
+
+        return SchedulerRunResult(
+            total_facts=len(recommendations),
+            upgrades=upgrades,
+            downgrades=downgrades,
+            skipped=skipped,
+            errors=errors,
+            changes=tuple(changes),
+            duration_ms=duration_ms,
+        )
+
+    def _process_precision_change(
+        self, prec: Any, profile_id: str,
+    ) -> PrecisionChange | None:
+        """Process a single precision change with error isolation.
+
+        Each fact is independent -- one failure does not block others.
+        Returns PrecisionChange on success, None on failure.
+        """
+        try:
+            if prec.action == "downgrade":
+                # Fetch float32 embedding
+                emb = self._vector_store.get_embedding(prec.fact_id, profile_id)
+                if emb is None:
+                    logger.warning(
+                        "SAGQ: No float32 for %s, skip downgrade", prec.fact_id,
+                    )
+                    return None
+                # Compress to target bit-width
+                self._quantized_store.compress_fact(
+                    prec.fact_id, profile_id, emb, prec.final_bit_width,
+                )
+
+            elif prec.action == "upgrade":
+                # Upgrade = re-compress at higher bit_width from float32 backup
+                emb = self._vector_store.get_embedding(prec.fact_id, profile_id)
+                if emb is None:
+                    logger.warning(
+                        "SAGQ: No float32 for %s, skip upgrade", prec.fact_id,
+                    )
+                    return None
+                self._quantized_store.compress_fact(
+                    prec.fact_id, profile_id, emb, prec.final_bit_width,
+                )
+
+            else:
+                return None  # skip
+
+            # Update embedding_metadata (Q7)
+            level = self._bit_width_to_quantization_level(prec.final_bit_width)
+            self._db.execute(
+                "UPDATE embedding_metadata "
+                "SET bit_width = ?, quantization_level = ? "
+                "WHERE fact_id = ? AND profile_id = ?",
+                (prec.final_bit_width, level, prec.fact_id, profile_id),
+            )
+
+            # Audit trail (Q10 -- HR-09)
+            self._db.execute(
+                "INSERT INTO fact_access_log "
+                "(log_id, fact_id, profile_id, accessed_at, access_type, session_id) "
+                "VALUES (?, ?, ?, datetime('now'), 'consolidation', 'sagq_scheduler')",
+                (_new_id(), prec.fact_id, profile_id),
+            )
+
+            now_iso = datetime.now(UTC).isoformat()
+            return PrecisionChange(
+                fact_id=prec.fact_id,
+                old_bit_width=prec.current_bit_width,
+                new_bit_width=prec.final_bit_width,
+                action=prec.action,
+                centrality=prec.centrality,
+                sagq_signal=prec.sagq_bit_width,
+                eap_signal=prec.eap_bit_width,
+                timestamp=now_iso,
+            )
+
+        except Exception as exc:
+            logger.error(
+                "SAGQ: precision change failed for %s: %s", prec.fact_id, exc,
+            )
+            return None
+
+    def _get_core_fact_ids(self, profile_id: str) -> set[str]:
+        """Get fact IDs referenced by core_memory_blocks (immune to quantization).
+
+        Uses json_each() to extract from source_fact_ids JSON array (Q8).
+        """
+        try:
+            rows = self._db.execute(
+                "SELECT json_each.value as fact_id "
+                "FROM core_memory_blocks, json_each(core_memory_blocks.source_fact_ids) "
+                "WHERE core_memory_blocks.profile_id = ?",
+                (profile_id,),
+            )
+            return {dict(r)["fact_id"] for r in rows}
+        except Exception as exc:
+            logger.debug("SAGQ: core_memory_blocks query failed: %s", exc)
+            return set()
+
+    def _bit_width_to_quantization_level(self, bit_width: int) -> str:
+        """Map bit-width integer to quantization level string."""
+        return _BW_TO_LEVEL.get(bit_width, "float32")
+
+    def should_run(self, last_run_at: str | None) -> bool:
+        """Check if enough time has passed since the last run.
+
+        Args:
+            last_run_at: ISO 8601 datetime of last run, or None if never run.
+
+        Returns True if the scheduler should run now.
+        """
+        if last_run_at is None:
+            return True
+
+        try:
+            last_run = datetime.fromisoformat(last_run_at)
+            now = datetime.now(UTC)
+            # Ensure both are timezone-aware for subtraction
+            if last_run.tzinfo is None:
+                last_run = last_run.replace(tzinfo=UTC)
+            hours_since = (now - last_run).total_seconds() / 3600
+            return hours_since >= self._config.scheduler_interval_hours
+        except (ValueError, TypeError) as exc:
+            logger.warning("SAGQ: Could not parse last_run_at '%s': %s", last_run_at, exc)
+            return True