superlocalmemory 4.1.6 → 4.1.7

package/src/superlocalmemory/learning/propensity.py

@@ -0,0 +1,131 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under AGPL-3.0-or-later - see LICENSE file
+
+"""Inverse-propensity weighting, so the bandit cannot confirm itself.
+
+THE BIAS
+--------
+The bandit decides what is shown, and engagement is then measured on what was
+shown. Feed that back raw and the loop is circular: an arm ranked first is seen
+more, so it is engaged with more, so it is ranked first more. The arm that wins
+is the one that was already winning, and the posterior records popularity it
+manufactured rather than usefulness it discovered.
+
+That is the self-referential signal in its exact form — a measurement taken
+through the mechanism it is meant to evaluate cannot see that mechanism fail.
+
+THE CORRECTION
+--------------
+Weight each observation by the inverse of the probability the policy had of
+showing that arm. An arm the policy was unlikely to show, that was engaged with
+anyway, is strong evidence; an arm the policy shows almost always is weak
+evidence whatever happens to it. This is the standard IPS estimator from
+counterfactual learning-to-rank, and it makes the update unbiased with respect
+to the policy's own choices.
+
+Under Thompson sampling the propensity is not a stored number: an arm is shown
+when its posterior draw beats every competitor's, so the probability is
+``P(theta_i > theta_j for all j != i)`` with each ``theta ~ Beta(alpha, beta)``.
+There is no closed form for more than two arms, so it is estimated by sampling.
+
+WHEN THE COMPETITORS ARE UNKNOWN
+--------------------------------
+Return a weight of exactly 1.0 — no correction. A wrong correction is worse
+than none: it would silently scale evidence by a number with no meaning, and
+unlike an absent correction nothing downstream could tell. Abstaining from a
+correction is visible in ``PropensityEstimate.corrected``.
+"""
+
+from __future__ import annotations
+
+import random
+from dataclasses import dataclass
+
+__all__ = ["PropensityEstimate", "estimate_propensity", "ips_weight", "MAX_WEIGHT"]
+
+#: Ceiling on the weight a single observation may carry. IPS has unbounded
+#: variance as propensity approaches zero: one rare event with p = 0.001 would
+#: otherwise move a posterior by 1000 plays' worth. Clipping trades a little
+#: bias for a variance that does not destroy the estimate — the standard
+#: bias-variance trade in clipped IPS.
+MAX_WEIGHT = 10.0
+
+#: Propensities below this are treated as this value before inversion.
+_MIN_PROPENSITY = 1.0 / MAX_WEIGHT
+
+#: Monte Carlo draws. 2000 puts the standard error of a mid-range propensity
+#: near 0.01, which is finer than the weight clipping can express.
+_DRAWS = 2000
+
+
+@dataclass(frozen=True)
+class PropensityEstimate:
+    """A propensity and whether it was actually derived from anything."""
+
+    propensity: float
+    weight: float
+    corrected: bool
+    competitors: int = 0
+
+
+def estimate_propensity(
+    arm: tuple[float, float],
+    competitors: list[tuple[float, float]],
+    *,
+    draws: int = _DRAWS,
+    rng: random.Random | None = None,
+) -> float:
+    """P(this arm's Thompson draw is the largest), by Monte Carlo.
+
+    ``arm`` and each competitor are ``(alpha, beta)`` posteriors.  With no
+    competitors the arm is shown whenever it is considered, so the propensity
+    is 1.0 and the correction is a no-op.
+    """
+    if not competitors:
+        return 1.0
+    # Fixed seed by default: the same play settled twice must produce the
+    # same weight, or a retry would move a posterior differently than the
+    # first attempt did.
+    generator = rng or random.Random(20260824)
+    alpha, beta = _sane(arm)
+    others = [_sane(c) for c in competitors]
+
+    wins = 0
+    for _ in range(max(1, int(draws))):
+        mine = generator.betavariate(alpha, beta)
+        if all(mine > generator.betavariate(a, b) for a, b in others):
+            wins += 1
+    return wins / max(1, int(draws))
+
+
+def _sane(posterior: tuple[float, float]) -> tuple[float, float]:
+    """Beta requires strictly positive parameters; a stored 0 would raise."""
+    alpha, beta = posterior
+    return (max(float(alpha), 1e-6), max(float(beta), 1e-6))
+
+
+def ips_weight(
+    arm: tuple[float, float] | None,
+    competitors: list[tuple[float, float]] | None,
+    *,
+    draws: int = _DRAWS,
+    rng: random.Random | None = None,
+) -> PropensityEstimate:
+    """Clipped inverse-propensity weight for one observation.
+
+    Returns ``corrected=False`` and ``weight=1.0`` when there is nothing to
+    correct against, so a caller can tell an uncorrected update from a
+    corrected one that happened to land on 1.0.
+    """
+    if arm is None or not competitors:
+        return PropensityEstimate(propensity=1.0, weight=1.0, corrected=False)
+
+    propensity = estimate_propensity(arm, competitors, draws=draws, rng=rng)
+    clipped = max(propensity, _MIN_PROPENSITY)
+    weight = min(1.0 / clipped, MAX_WEIGHT)
+    return PropensityEstimate(
+        propensity=propensity,
+        weight=weight,
+        corrected=True,
+        competitors=len(competitors),
+    )

package/src/superlocalmemory/learning/reward.py

@@ -735,6 +735,20 @@ class EngagementRewardModel:
                 signals = json.loads(row["signals_json"] or "{}")
             except json.JSONDecodeError:  # pragma: no cover
                 signals = {}
+            # A row that accumulated no signal has nothing to say. It used to
+            # be written out as an outcome anyway, carrying the label
+            # formula's base term of exactly 0.5 and marked ``settled`` as
+            # though a judgement had been reported. That value is not a
+            # harmless placeholder: ``alpha += 0.5`` with ``beta += 0.5`` moves
+            # both sides together, so the posterior keeps its mean and loses
+            # spread, and an arm settled this way repeatedly never leaves its
+            # prior.
+            #
+            # The row is still finalized, so it stops being rescanned; it is
+            # simply not turned into evidence it never was.
+            if not signals:
+                settle_ids.append(row["outcome_id"])
+                continue
             reward = _compute_label(signals)
             insert_batch.append(
                 (
@@ -756,7 +770,11 @@ class EngagementRewardModel:
         _CHUNK = 500
         written = 0
         try:
-            for i in range(0, len(insert_batch), _CHUNK):
+            # insert_batch and settle_ids are no longer parallel: a row with
+            # no signals is finalized without producing an outcome. Pad the
+            # insert side so each burst still pairs a write with the right
+            # finalizations, and never index one by the other's length.
+            for i in range(0, max(len(insert_batch), len(settle_ids)), _CHUNK):
                 i_chunk = insert_batch[i:i + _CHUNK]
                 s_chunk = settle_ids[i:i + _CHUNK]
                 placeholders = ",".join("?" * len(s_chunk))
@@ -764,22 +782,30 @@ class EngagementRewardModel:
                     conn = self._get_conn()
                     conn.execute("BEGIN IMMEDIATE")
                     try:
-                        conn.executemany(
-                            "INSERT OR REPLACE INTO action_outcomes "
-                            "(outcome_id, profile_id, query, fact_ids_json,"
-                            " outcome, context_json, timestamp, reward,"
-                            " settled, settled_at, recall_query_id) "
-                            "VALUES "
-                            "(?, ?, '', ?, 'settled', '{}', ?, ?, 1, ?, ?)",
-                            i_chunk,
-                        )
-                        conn.execute(
-                            "UPDATE pending_outcomes "
-                            f"SET status = 'settled' WHERE outcome_id IN ({placeholders})",
-                            s_chunk,
-                        )
+                        if i_chunk:
+                            conn.executemany(
+                                "INSERT OR REPLACE INTO action_outcomes "
+                                "(outcome_id, profile_id, query, fact_ids_json,"
+                                " outcome, context_json, timestamp, reward,"
+                                " settled, settled_at, recall_query_id) "
+                                "VALUES "
+                                "(?, ?, '', ?, 'settled', '{}', ?, ?, 1, ?, ?)",
+                                i_chunk,
+                            )
+                        if s_chunk:
+                            conn.execute(
+                                "UPDATE pending_outcomes "
+                                f"SET status = 'settled' "
+                                f"WHERE outcome_id IN ({placeholders})",
+                                s_chunk,
+                            )
                         conn.execute("COMMIT")
-                        written += len(i_chunk)
+                        # Count what was FINALIZED, not what was written. A row
+                        # with no signals is finalized without producing an
+                        # outcome, and callers use this number to know the
+                        # backlog drained — counting inserts would report 0
+                        # forever and make the reaper look dead.
+                        written += len(s_chunk)
                     except sqlite3.Error:
                         conn.execute("ROLLBACK")
                         raise

package/src/superlocalmemory/learning/reward_model.py

@@ -0,0 +1,144 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under AGPL-3.0-or-later - see LICENSE file
+
+"""Turn observed engagement into a reward, or into an honest refusal.
+
+ABSTENTION IS THE POINT
+-----------------------
+The ladder this replaces always produced a number. When it saw nothing it
+produced ``0.5``, and ``0.5`` is not neutral: ``alpha += 0.5`` with
+``beta += 0.5`` holds a Beta posterior's mean at exactly 0.5 while shrinking
+its variance, so each empty settlement makes an arm *more certain* it is
+average and *less* movable by the evidence that finally arrives. Applied often
+enough it does not merely fail to learn — it commits, with growing confidence,
+to knowing nothing.
+
+So this module returns ``None`` when nothing was observed. An unobserved recall
+leaves the posterior untouched and free. Absence of evidence is recorded as
+absence of evidence.
+
+THE SCALE
+---------
+Positive evidence maps into ``(0.5, 1.0]``, a requery to ``0.0``, and nothing to
+``None``. Nothing maps *to* 0.5, because that value is reserved for "no
+information" and no observation carries that meaning: if it was worth
+observing it was worth moving the posterior.
+
+Weights are module constants and deliberately legible rather than fitted. There
+is no ground-truth corpus of "was this memory actually useful" to fit against,
+and inventing one from the system's own rankings would be the circularity
+``propensity.py`` exists to break. They are a documented prior over evidence
+strength; the *learning* happens in the posterior these rewards update, not in
+the constants.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from superlocalmemory.learning.engagement_features import EngagementFeatures
+
+__all__ = ["RewardDecision", "score", "REQUERY_REWARD"]
+
+#: A question asked again is the one unambiguous statement that an answer did
+#: not serve. It is the only hard zero available.
+REQUERY_REWARD = 0.0
+
+#: Evidence weights, strongest first.
+#:  - a memory whose content reaches a written artifact was used, not just read
+#:  - a follow-up memory overlapping it means the agent built on it
+#:  - appearing in any later action is real but weaker: the agent may have been
+#:    working on the subject regardless of what was recalled
+_W_ARTIFACT = 0.50
+_W_FOLLOWUP = 0.30
+_W_PRESENCE = 0.20
+
+#: Positive evidence starts just above the reserved no-information value, so
+#: the weakest real observation still moves an arm up rather than nowhere.
+_POSITIVE_FLOOR = 0.55
+
+
+@dataclass(frozen=True)
+class RewardDecision:
+    """A reward, or an abstention, with the reason attached.
+
+    ``reward is None`` means do not update. ``kind`` names what was seen so a
+    settled play can be explained after the fact instead of being a bare float.
+    """
+
+    reward: float | None
+    kind: str
+    detail: str = ""
+
+    @property
+    def abstained(self) -> bool:
+        return self.reward is None
+
+
+def score(features: EngagementFeatures) -> RewardDecision:
+    """Map observations to a reward in [0, 1], or abstain.
+
+    Never raises: a settler running over a month of rows must not stop on one
+    malformed observation.
+    """
+    if features.marker_hit:
+        # The agent named the memory outright. Nothing makes it do this, so it
+        # is rare, but when it happens there is nothing to infer.
+        return RewardDecision(
+            reward=1.0,
+            kind="proxy_position",
+            detail="a recalled fact id appeared in a later tool event",
+        )
+
+    if features.requeried:
+        return RewardDecision(
+            reward=REQUERY_REWARD,
+            kind="proxy_requery",
+            detail="the same question was asked again inside the window",
+        )
+
+    if not features.observed:
+        return RewardDecision(
+            reward=None,
+            kind="unobserved",
+            detail=(
+                f"no engagement visible in {features.action_count} following "
+                "action(s); posterior left untouched"
+            ),
+        )
+
+    artifact = _clamp(features.artifact_overlap)
+    followup = _clamp(features.followup_write_overlap)
+    presence = _clamp(features.peak_overlap)
+
+    strength = (
+        _W_ARTIFACT * artifact
+        + _W_FOLLOWUP * followup
+        + _W_PRESENCE * presence
+    )
+    # strength in [0, 1]; map onto (floor, 1.0].
+    reward = _POSITIVE_FLOOR + (1.0 - _POSITIVE_FLOOR) * _clamp(strength)
+
+    if artifact > 0.0:
+        kind = "artifact_overlap"
+    elif followup > 0.0:
+        kind = "followup_overlap"
+    else:
+        kind = "presence_overlap"
+
+    return RewardDecision(
+        reward=round(reward, 6),
+        kind=kind,
+        detail=(
+            f"artifact={artifact:.3f} followup={followup:.3f} "
+            f"presence={presence:.3f} on {len(features.matched_fact_ids)} fact(s)"
+        ),
+    )
+
+
+def _clamp(value: float) -> float:
+    try:
+        v = float(value)
+    except (TypeError, ValueError):
+        return 0.0
+    return 0.0 if v < 0.0 else (1.0 if v > 1.0 else v)

package/src/superlocalmemory/learning/reward_proxy.py

@@ -34,6 +34,11 @@ from datetime import datetime, timedelta, timezone
 from pathlib import Path
 from typing import Any
 
+from superlocalmemory.learning.engagement_features import (
+    EngagementFeatures,
+    extract_features,
+)
+from superlocalmemory.learning.reward_model import score
 from superlocalmemory.core.topic_signature import compute_topic_signature
 from superlocalmemory.learning.bandit import ContextualBandit
 
@@ -105,10 +110,10 @@ def _shown_fact_ids(
     """The fact_ids this play recorded at recall time (M044), or [].
 
     Preferred over the ``learning_signals`` lookup below because it does not
-    depend on the exposure enqueue, which is off: twenty rows per query, and
-    the source of a 2,675x inflation in the ranking-phase counter. With no
-    signals rows there was nothing to look for, so every play settled as the
-    120-second default and 165 arms sat at alpha == beta.
+    depend on the exposure enqueue, which is off because it wrote a row per
+    displayed memory per query and badly inflated the ranking-phase counter.
+    With no signals rows there is nothing to look for, so plays fell through to
+    the age-based default and arms stayed on their priors.
 
     Returns [] on a store where M044 has not run — "no such column" is an
     ``sqlite3.Error`` and is caught, so an unmigrated install falls back to the
@@ -366,6 +371,98 @@ def _default_deadline(
     )
 
 
+#: Once a play is older than this with nothing observed, it is closed without
+#: touching the posterior. Leaving it open forever would make every settler
+#: pass rescan it; settling it at a neutral value is what this module stopped
+#: doing. Closed-and-unjudged is the third option that was missing.
+_ABSTAIN_EXPIRY_SEC = 900
+
+
+def _expire_play(learning_conn: sqlite3.Connection, play_id: int) -> bool:
+    """Close a play without applying any reward.
+
+    The posterior is deliberately untouched: nothing was observed, so there is
+    nothing to learn, and a neutral update would shrink the arm's variance
+    around its prior and make it harder to move once evidence does arrive.
+    """
+    try:
+        learning_conn.execute(
+            "UPDATE bandit_plays SET settled_at = ?, settlement_type = ? "
+            "WHERE play_id = ? AND settled_at IS NULL",
+            (datetime.now(timezone.utc).isoformat(), "unobserved", int(play_id)),
+        )
+        learning_conn.commit()
+        return True
+    except sqlite3.Error as exc:
+        logger.debug("reward_proxy: expire play %s: %s", play_id, exc)
+        return False
+
+
+def _session_for_play(
+    memory_conn: sqlite3.Connection | None,
+    query_id: str,
+    played_at: datetime,
+    profile_id: str,
+) -> str:
+    """The conversation this play happened in, or "" when it cannot be named.
+
+    The play and its outcome ticket are minted with different uuids on the same
+    recall, so the query_id is tried first and a time-and-profile match is the
+    fallback. An empty answer means no observation is possible, which the
+    caller turns into an abstention rather than a guess.
+    """
+    if memory_conn is None:
+        return ""
+    try:
+        row = memory_conn.execute(
+            "SELECT session_id FROM pending_outcomes WHERE recall_query_id = ? "
+            "LIMIT 1", (str(query_id),),
+        ).fetchone()
+        if row and row[0]:
+            return str(row[0])
+        window_ms = 5000
+        centre = int(played_at.timestamp() * 1000)
+        row = memory_conn.execute(
+            "SELECT session_id FROM pending_outcomes "
+            "WHERE profile_id = ? AND created_at_ms BETWEEN ? AND ? "
+            "ORDER BY ABS(created_at_ms - ?) LIMIT 1",
+            (str(profile_id), centre - window_ms, centre + window_ms, centre),
+        ).fetchone()
+        return str(row[0]) if row and row[0] else ""
+    except sqlite3.Error:
+        return ""
+
+
+def _ips_for_play(
+    learning_conn: sqlite3.Connection, play_id: int, stratum: str, profile_id: str,
+):
+    """Inverse-propensity weight for this play against its stratum's arms."""
+    from superlocalmemory.learning.propensity import ips_weight
+
+    try:
+        row = learning_conn.execute(
+            "SELECT arm_id FROM bandit_plays WHERE play_id = ?", (int(play_id),),
+        ).fetchone()
+        if not row or not row[0]:
+            return ips_weight(None, None)
+        arm_id = str(row[0])
+        rows = learning_conn.execute(
+            "SELECT arm_id, alpha, beta FROM bandit_arms "
+            "WHERE profile_id = ? AND stratum = ?",
+            (str(profile_id), str(stratum or "")),
+        ).fetchall()
+    except sqlite3.Error:
+        return ips_weight(None, None)
+
+    mine, others = None, []
+    for candidate, alpha, beta in rows:
+        if str(candidate) == arm_id:
+            mine = (float(alpha), float(beta))
+        else:
+            others.append((float(alpha), float(beta)))
+    return ips_weight(mine, others)
+
+
 def settle_stale_plays(
     profile_id: str,
     db_path: Path | str,
@@ -406,27 +503,56 @@ def settle_stale_plays(
                 _shown_fact_ids(learning_conn, row["play_id"])
                 or _top3_fact_ids(learning_conn, row["query_id"])
             )
-            reward: float | None = None
-            kind = "default"
-            if memory_conn is not None and _tool_event_hit(
-                memory_conn, played, top3, profile_id=str(profile_id),
-            ):
-                reward = 1.0
-                kind = "proxy_position"
-            elif memory_conn is not None and _requery_detected(
-                memory_conn, played, row["query_id"], profile_id=str(profile_id),
-            ):
-                reward = 0.0
-                kind = "proxy_requery"
-            elif age > _default_deadline(learning_conn, row):
-                # P1: uncertain default once the window closes.
-                reward = 0.5
-                kind = "default"
+            # The ladder this replaces asked one question — did a recalled
+            # fact_id appear verbatim in a later tool event — and answered 0.5
+            # whenever it could not tell. Both halves failed: nothing makes an
+            # agent echo the marker, and 0.5 is not an absence of judgement but
+            # a confident one, tightening the posterior around its prior.
+            requeried = bool(
+                memory_conn is not None and _requery_detected(
+                    memory_conn, played, row["query_id"],
+                    profile_id=str(profile_id),
+                )
+            )
+            marker_hit = bool(
+                memory_conn is not None and _tool_event_hit(
+                    memory_conn, played, top3, profile_id=str(profile_id),
+                )
+            )
+            session_id = _session_for_play(
+                memory_conn, str(row["query_id"] or ""), played, str(profile_id),
+            )
+            if memory_conn is not None and session_id:
+                features = extract_features(
+                    memory_conn,
+                    session_id=session_id,
+                    profile_id=str(profile_id),
+                    fact_ids=top3,
+                    recalled_at=played,
+                    requeried=requeried,
+                    marker_hit=marker_hit,
+                )
             else:
-                # Between 60 and 120 s with no evidence yet — wait.
+                features = EngagementFeatures(
+                    requeried=requeried, marker_hit=marker_hit,
+                )
+
+            decision = score(features)
+            if decision.reward is None:
+                # Nothing observed. Wait while the window is still open, then
+                # close the play unjudged rather than inventing a number.
+                if age > _ABSTAIN_EXPIRY_SEC:
+                    _expire_play(learning_conn, int(row["play_id"]))
                 continue
 
-            if bandit.update(int(row["play_id"]), reward, kind=kind):
+            estimate = _ips_for_play(
+                learning_conn, int(row["play_id"]),
+                str(row["stratum"] or ""), str(profile_id),
+            )
+            if bandit.update(
+                int(row["play_id"]), decision.reward,
+                kind=decision.kind, weight=estimate.weight,
+            ):
                 settled += 1
     finally:
         try:

package/src/superlocalmemory/server/routes/v3_api.py

@@ -12,6 +12,7 @@ from pathlib import Path
 import os
 from fastapi import APIRouter, HTTPException, Request
 from fastapi.responses import JSONResponse
+from superlocalmemory.core.session_identity import synthetic_session_id
 from superlocalmemory.core.config import CANONICAL_RECALL_LIMIT
 from superlocalmemory.core.status_contract import (
     COUNT_QUERIES,
@@ -36,15 +37,15 @@ def _signal_session_id() -> str:
 
         agent = str(get_current_agent_id() or "").strip()
         if agent:
-            return f"agent:{agent}"
+            return synthetic_session_id("agent", agent)
     except Exception:  # noqa: BLE001 -- naming the caller must never fail a read
         pass
     try:
         from superlocalmemory.server.routes.helpers import get_active_profile
 
-        return f"api:{get_active_profile()}"
+        return synthetic_session_id("api", str(get_active_profile()))
     except Exception:  # noqa: BLE001
-        return "api:default"
+        return synthetic_session_id("api", "default")
 
 
 router = APIRouter(prefix="/api/v3", tags=["v3"])