superlocalmemory 4.1.13 → 4.1.14

package/src/superlocalmemory/retrieval/forgetting_filter.py

@@ -1,160 +0,0 @@
-# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
-# Licensed under AGPL-3.0-or-later - see LICENSE file
-# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
-
-"""Forgetting filter for retrieval pipeline.
-
-Post-retrieval filter that adjusts scores based on Ebbinghaus retention:
-  - Active/warm facts: score weighted by lifecycle_weight
-  - Cold facts: score reduced (weight = 0.3)
-  - Archive/forgotten facts: REMOVED from results entirely
-
-Integrates with ChannelRegistry.register_filter() using the FilterFn
-signature: (all_results, profile_id, context) -> filtered_results.
-
-HR-06: When config.enabled=False, returns results unchanged.
-
-Part of Qualixar | Author: Varun Pratap Bhardwaj
-License: AGPL-3.0-or-later
-"""
-
-from __future__ import annotations
-
-import logging
-from typing import TYPE_CHECKING, Any
-
-from superlocalmemory.core.config import ForgettingConfig
-
-if TYPE_CHECKING:
-    from superlocalmemory.retrieval.channel_registry import ChannelRegistry
-    from superlocalmemory.storage.database import DatabaseManager
-
-logger = logging.getLogger(__name__)
-
-
-# Lifecycle zone weights — same as in ebbinghaus.py
-_ZONE_WEIGHTS: dict[str, float] = {
-    "active": 1.0,
-    "warm": 0.7,
-    "cold": 0.3,
-    "archive": 0.0,
-    "forgotten": 0.0,
-}
-
-# V3.4.11: Deep recall weights — includes cold/archive with reduced scores
-_DEEP_ZONE_WEIGHTS: dict[str, float] = {
-    "active": 1.0,
-    "warm": 0.7,
-    "cold": 0.3,
-    "archive": 0.15,
-    "forgotten": 0.05,
-}
-
-# Zones where facts are excluded from results (default recall)
-_EXCLUDED_ZONES: frozenset[str] = frozenset({"archive", "forgotten"})
-
-# Deep recall excludes nothing — every fact is searchable
-_DEEP_EXCLUDED_ZONES: frozenset[str] = frozenset()
-
-
-class ForgettingFilter:
-    """Post-retrieval filter that applies Ebbinghaus retention weighting.
-
-    Removes archived/forgotten facts and adjusts scores for other zones.
-    """
-
-    __slots__ = ("_db", "_config", "_deep_recall")
-
-    def __init__(self, db: DatabaseManager, config: ForgettingConfig, deep_recall: bool = False) -> None:
-        self._db = db
-        self._config = config
-        self._deep_recall = deep_recall
-
-    def filter(
-        self,
-        all_results: dict[str, list[tuple[str, float]]],
-        profile_id: str,
-        context: Any,
-    ) -> dict[str, list[tuple[str, float]]]:
-        """Apply forgetting filter to retrieval results.
-
-        Matches FilterFn signature from channel_registry.py.
-
-        Args:
-            all_results: Channel name -> [(fact_id, score)] dict.
-            profile_id: Current profile.
-            context: Optional context (unused).
-
-        Returns:
-            Filtered results dict with scores adjusted by retention weight.
-        """
-        # HR-06: If disabled, return unchanged
-        if not self._config.enabled:
-            return all_results
-
-        # Collect all unique fact_ids across all channels
-        all_fact_ids: set[str] = set()
-        for channel_results in all_results.values():
-            for fact_id, _ in channel_results:
-                all_fact_ids.add(fact_id)
-
-        if not all_fact_ids:
-            return all_results
-
-        # Batch query retention data
-        retention_rows = self._db.batch_get_retention(
-            list(all_fact_ids), profile_id,
-        )
-
-        # Build lookup: fact_id -> retention row
-        retention_map: dict[str, dict] = {}
-        for row in retention_rows:
-            retention_map[row["fact_id"]] = row
-
-        # Filter and weight each channel's results
-        filtered: dict[str, list[tuple[str, float]]] = {}
-        for channel_name, channel_results in all_results.items():
-            new_results: list[tuple[str, float]] = []
-            for fact_id, score in channel_results:
-                ret_data = retention_map.get(fact_id)
-
-                if ret_data is None:
-                    # No retention data yet -> new memory, keep as-is
-                    new_results.append((fact_id, score))
-                    continue
-
-                zone = ret_data.get("lifecycle_zone", "active")
-
-                # V3.4.11: Deep recall mode includes all tiers
-                excluded = _DEEP_EXCLUDED_ZONES if self._deep_recall else _EXCLUDED_ZONES
-                weights = _DEEP_ZONE_WEIGHTS if self._deep_recall else _ZONE_WEIGHTS
-
-                if zone in excluded:
-                    continue
-
-                weight = weights.get(zone, 1.0)
-                new_results.append((fact_id, score * weight))
-
-            filtered[channel_name] = new_results
-
-        return filtered
-
-
-def register_forgetting_filter(
-    registry: ChannelRegistry,
-    db: DatabaseManager,
-    config: ForgettingConfig,
-) -> None:
-    """Register the forgetting filter with the channel registry.
-
-    Does nothing if config.enabled is False.
-
-    Args:
-        registry: Channel registry to register with.
-        db: Database manager for retention queries.
-        config: Forgetting configuration.
-    """
-    if not config.enabled:
-        return
-    f = ForgettingFilter(db, config)
-    registry.register_filter(f.filter)

package/src/superlocalmemory/retrieval/fusion.py

@@ -1,81 +0,0 @@
-# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
-# Licensed under AGPL-3.0-or-later - see LICENSE file
-# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
-
-"""SuperLocalMemory V3 — Weighted Reciprocal Rank Fusion.
-
-Single-pass RRF with k=15 for sharp rank discrimination on small candidate pools.
-V1 had triple re-fusion which destroyed rankings — fixed in V2.
-
-Part of Qualixar | Author: Varun Pratap Bhardwaj
-License: AGPL-3.0-or-later
-"""
-from __future__ import annotations
-
-from dataclasses import dataclass, field
-
-
-@dataclass(frozen=True)
-class FusionResult:
-    """Single fused result with per-channel provenance."""
-    fact_id: str
-    fused_score: float
-    channel_ranks: dict[str, int] = field(default_factory=dict)
-    channel_scores: dict[str, float] = field(default_factory=dict)
-
-
-def weighted_rrf(
-    channels: dict[str, list[tuple[str, float]]],
-    weights: dict[str, float],
-    k: int = 15,
-    max_rank_penalty: int = 1000,
-) -> list[FusionResult]:
-    """Fuse ranked lists via Weighted Reciprocal Rank Fusion.
-
-    Args:
-        channels: channel_name -> [(fact_id, score)] sorted desc.
-        weights: channel_name -> weight multiplier.
-        k: RRF smoothing constant (60 for diverse retrieval, D116).
-        max_rank_penalty: Rank assigned to absent documents.
-
-    Returns:
-        FusionResult list sorted by fused_score descending.
-    """
-    if k <= 0:
-        raise ValueError(f"k must be positive, got {k}")
-
-    rank_maps: dict[str, dict[str, int]] = {}
-    score_maps: dict[str, dict[str, float]] = {}
-
-    for ch, ranked in channels.items():
-        ranks: dict[str, int] = {}
-        scores: dict[str, float] = {}
-        for i, (fid, sc) in enumerate(ranked):
-            ranks[fid] = i + 1
-            scores[fid] = sc
-        rank_maps[ch] = ranks
-        score_maps[ch] = scores
-
-    all_ids: set[str] = set()
-    for ranked in channels.values():
-        for fid, _ in ranked:
-            all_ids.add(fid)
-
-    results: list[FusionResult] = []
-    for fid in all_ids:
-        fused = 0.0
-        ch_ranks: dict[str, int] = {}
-        ch_scores: dict[str, float] = {}
-        for ch in channels:
-            w = weights.get(ch, 1.0)
-            rank = rank_maps[ch].get(fid, max_rank_penalty)
-            ch_ranks[ch] = rank
-            ch_scores[ch] = score_maps[ch].get(fid, 0.0)
-            fused += w / (k + rank)
-        results.append(FusionResult(fid, fused, ch_ranks, ch_scores))
-
-    # Deterministic total order: fused score descending, ties broken by fact_id.
-    # Sorting on score alone left tied results in set-iteration order, which
-    # varies with the process hash seed.
-    results.sort(key=lambda r: (-r.fused_score, r.fact_id))
-    return results

package/src/superlocalmemory/retrieval/graph_adjacency.py

@@ -1,219 +0,0 @@
-# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
-# Licensed under AGPL-3.0-or-later - see LICENSE file
-# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
-
-"""One shape for the graph the entity walk reads, whoever stored it.
-
-WHY THIS EXISTS
----------------
-The graph was stored twice — in SQLite and, once promoted, in CozoDB — and the
-*walk over it* was written twice with it. The two implementations did not compute
-the same function: the SQLite one multiplies activation by a PageRank factor at
-every hop and the Cozo one had no PageRank at all. Measured on a copy of the
-author's store, 3,567 of 3,667 shared facts came out with different scores, the
-top-20 sets differed, and the projected path therefore failed its shadow
-comparison on **every** query and fell back to SQLite. The projection was correct
-data that nothing could use.
-
-The lesson is not "fix the second walk". It is that a storage backend must supply
-**data, not behaviour**. So this module defines the one shape the walk consumes,
-and each store gets an adapter that produces it. Adding a third store later is
-one adapter and no algorithm; there is no second implementation to keep in step,
-and nothing to shadow-compare, because there is only one answer to compare.
-
-WHY IT IS ARRAYS AND NOT DICTS
-------------------------------
-The walk's cost was never the storage engine or the size of the graph — 7,460
-nodes over 849k edges is small. It was doing 3.4 million relaxations as
-interpreted dictionary lookups (cProfile counted 4,367,932 ``dict.get`` calls in
-one recall). Held as CSR arrays the same relaxation is a handful of vectorised
-passes. See :mod:`superlocalmemory.retrieval.spreading`.
-
-The node space is fact ids. Entity ids are deliberately a separate namespace:
-treating them as graph nodes produces a healthy-looking and semantically wrong
-graph, which is why the projection keeps ``fact_entity`` as an explicit bridge.
-"""
-
-from __future__ import annotations
-
-import logging
-from dataclasses import dataclass, field
-from typing import Any, Iterable, Mapping, Protocol, Sequence
-
-import numpy as np
-
-logger = logging.getLogger(__name__)
-
-#: Cap on the PageRank multiplier, matching the walk that defined it.
-PAGERANK_BOOST_CAP = 2.0
-
-
-@dataclass(frozen=True)
-class AdjacencySnapshot:
-    """An immutable view of one profile's fact graph, ready to walk.
-
-    Frozen because the walk must not be able to change the graph underneath a
-    concurrent reader — the channel serialises *loading* behind a lock, and a
-    snapshot handed out after that is safe to read from anywhere.
-    """
-
-    node_ids: tuple[str, ...]
-    node_index: Mapping[str, int]
-    #: CSR over an UNDIRECTED graph: every edge appears in both endpoints' rows
-    #: with the same weight, so "incoming to j" and "outgoing from j" are the
-    #: same list. The walk relies on that to take a segment maximum per row.
-    indptr: np.ndarray
-    indices: np.ndarray
-    weights: np.ndarray
-    entity_to_facts: Mapping[str, tuple[int, ...]]
-    fact_to_entities: tuple[tuple[str, ...], ...]
-    #: Per-node PageRank and community, dense so the walk never branches on
-    #: presence. 0.0 and -1 mean "not measured", which is what the dict-based
-    #: walk expressed as a missing key.
-    pagerank: np.ndarray
-    community: np.ndarray
-    has_metrics: bool
-    #: Provenance, so a surface can say which store answered and an operator can
-    #: tell a fast path from a fallback rather than guessing.
-    source: str = "sqlite"
-    edge_count: int = 0
-    fact_count: int = 0
-    profile_id: str = "default"
-    extras: dict[str, Any] = field(default_factory=dict)
-
-    @property
-    def node_count(self) -> int:
-        return len(self.node_ids)
-
-    def index_of(self, fact_id: str) -> int | None:
-        return self.node_index.get(fact_id)
-
-    def peak_propagation_factor(self, decay: float) -> float:
-        """Largest ``decay * weight * pagerank_boost`` any edge can apply.
-
-        At or above 1.0 the walk can amplify along a path, and a bounded
-        iteration is then the only well-defined reading of it — which is what
-        :mod:`spreading` implements, and why it does not matter that the
-        dict-based walk was order-dependent in that regime.
-
-        Measured across four real workspaces: 0.7056 and 0.7100 on the two
-        large ones, 0.8497 on a 24-fact one, and **1.1061 on a five-fact one**,
-        which amplifies. Small graphs concentrate rank by construction, so the
-        amplifying regime is not exotic — it is what every workspace looks like
-        on its first day.
-        """
-        if self.weights.size == 0:
-            return 0.0
-        boost = np.minimum(1.0 + self.pagerank * 2.0, PAGERANK_BOOST_CAP)
-        if not self.has_metrics:
-            return float(decay)
-        return float(np.max(decay * self.weights * boost[self.indices]))
-
-
-def snapshot_from_maps(
-    adjacency: Mapping[str, Sequence[tuple[str, float]]],
-    entity_to_facts: Mapping[str, Iterable[str]],
-    fact_to_entities: Mapping[str, Iterable[str]],
-    graph_metrics: Mapping[str, Mapping[str, Any]] | None,
-    *,
-    source: str,
-    profile_id: str,
-    nodes: Iterable[str] | None = None,
-    fact_count: int = 0,
-) -> AdjacencySnapshot:
-    """Build a snapshot from the dict form both adapters produce.
-
-    Node order is the sorted fact ids, not insertion order. That is what makes a
-    snapshot reproducible: two loads of the same graph produce the same arrays,
-    so a score computed from them is the same number and not a function of which
-    row a database happened to return first.
-
-    ``nodes`` IS THE VISIBLE FACT CORPUS, NOT THE FACTS THAT HAVE EDGES.
-    An earlier version derived the node space from the adjacency keys, which
-    silently excluded every fact with no edge yet — and ingestion is
-    queryable-first, so that is exactly the set a user has just added. Those
-    facts are reachable through their entities and the walk seeds them at 1.0;
-    leaving them out of the node space scored them zero instead. Caught on a real
-    store: four candidates, one of them the highest-scoring result for its query.
-    An edgeless fact is a node with an empty CSR row, which the walk's
-    segment-maximum already reads as zero incoming activation.
-    """
-    node_ids = tuple(sorted(set(nodes) if nodes is not None else set(adjacency)))
-    node_index = {fid: i for i, fid in enumerate(node_ids)}
-    n = len(node_ids)
-
-    indptr = np.zeros(n + 1, dtype=np.int64)
-    flat_indices: list[int] = []
-    flat_weights: list[float] = []
-    for i, fid in enumerate(node_ids):
-        # Sorted within the row for the same reason the rows are sorted.
-        neighbours = sorted(
-            (node_index[nid], float(w))
-            for nid, w in adjacency.get(fid, ())
-            if nid in node_index
-        )
-        for j, w in neighbours:
-            flat_indices.append(j)
-            flat_weights.append(w)
-        indptr[i + 1] = len(flat_indices)
-
-    indices = np.asarray(flat_indices, dtype=np.int64)
-    weights = np.asarray(flat_weights, dtype=np.float64)
-
-    metrics = graph_metrics or {}
-    pagerank = np.zeros(n, dtype=np.float64)
-    community = np.full(n, -1, dtype=np.int64)
-    for fid, i in node_index.items():
-        entry = metrics.get(fid)
-        if not entry:
-            continue
-        pagerank[i] = float(entry.get("pagerank_score", 0.0) or 0.0)
-        comm = entry.get("community_id")
-        if comm is not None:
-            try:
-                community[i] = int(comm)
-            except (TypeError, ValueError):
-                pass
-
-    e2f = {
-        eid: tuple(sorted(node_index[f] for f in facts if f in node_index))
-        for eid, facts in entity_to_facts.items()
-    }
-    f2e = tuple(
-        tuple(fact_to_entities.get(fid, ()) or ()) for fid in node_ids
-    )
-
-    return AdjacencySnapshot(
-        node_ids=node_ids,
-        node_index=node_index,
-        indptr=indptr,
-        indices=indices,
-        weights=weights,
-        entity_to_facts=e2f,
-        fact_to_entities=f2e,
-        pagerank=pagerank,
-        community=community,
-        has_metrics=bool(metrics),
-        source=source,
-        edge_count=int(indices.size // 2),
-        fact_count=fact_count or n,
-        profile_id=profile_id,
-    )
-
-
-class AdjacencySource(Protocol):
-    """A store that can hand over one profile's fact graph.
-
-    Data only. A source that also implemented the walk is the defect this
-    interface exists to prevent.
-    """
-
-    name: str
-
-    def load(
-        self,
-        profile_id: str,
-        *,
-        include_global: bool = False,
-        include_shared: bool = False,
-    ) -> AdjacencySnapshot: ...