superlocalmemory 4.1.6 → 4.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +2 -2
- package/CHANGELOG.md +46 -0
- package/README.md +3 -3
- package/package.json +3 -1
- package/plugin/.claude-plugin/plugin.json +1 -1
- package/plugin/CLAUDE.md +3 -3
- package/plugin/agents/slm-governance-advisor.md +1 -1
- package/plugin/agents/slm-loop-runner.md +1 -1
- package/plugin/agents/slm-memory-advisor.md +1 -1
- package/plugin/agents/slm-optimize-advisor.md +1 -1
- package/plugin/requirements.txt +1 -1
- package/plugin/skills/slm-cache/SKILL.md +1 -1
- package/plugin/skills/slm-compress/SKILL.md +1 -1
- package/plugin/skills/slm-governance/SKILL.md +1 -1
- package/plugin/skills/slm-graph/SKILL.md +1 -1
- package/plugin/skills/slm-loop/SKILL.md +1 -1
- package/plugin/skills/slm-mesh/SKILL.md +1 -1
- package/plugin/skills/slm-profile/SKILL.md +1 -1
- package/plugin/skills/slm-recall/SKILL.md +1 -1
- package/plugin/skills/slm-remember/SKILL.md +1 -1
- package/plugin/skills/slm-scope/SKILL.md +1 -1
- package/plugin/skills/slm-session/SKILL.md +1 -1
- package/plugin/skills/slm-status/SKILL.md +1 -1
- package/plugin-src/agents/slm-memory-advisor.md +49 -0
- package/plugin-src/agents/slm-optimize-advisor.md +44 -0
- package/plugin-src/rules/AGENTS.md +1 -1
- package/plugin-src/skills/slm-cache/SKILL.md +1 -1
- package/plugin-src/skills/slm-compress/SKILL.md +1 -1
- package/plugin-src/skills/slm-governance/SKILL.md +1 -1
- package/plugin-src/skills/slm-graph/SKILL.md +1 -1
- package/plugin-src/skills/slm-loop/SKILL.md +1 -1
- package/plugin-src/skills/slm-mesh/SKILL.md +1 -1
- package/plugin-src/skills/slm-profile/SKILL.md +1 -1
- package/plugin-src/skills/slm-recall/SKILL.md +1 -1
- package/plugin-src/skills/slm-remember/SKILL.md +1 -1
- package/plugin-src/skills/slm-scope/SKILL.md +1 -1
- package/plugin-src/skills/slm-session/SKILL.md +1 -1
- package/plugin-src/skills/slm-status/SKILL.md +1 -1
- package/pyproject.toml +5 -1
- package/src/superlocalmemory/__init__.py +1 -1
- package/src/superlocalmemory/cli/host_upgrades.py +21 -7
- package/src/superlocalmemory/core/engine.py +7 -1
- package/src/superlocalmemory/core/recall_pipeline.py +15 -5
- package/src/superlocalmemory/core/session_identity.py +14 -1
- package/src/superlocalmemory/hooks/codex_assets.py +165 -45
- package/src/superlocalmemory/hooks/post_tool_outcome_hook.py +122 -0
- package/src/superlocalmemory/learning/bandit.py +22 -2
- package/src/superlocalmemory/learning/engagement_features.py +279 -0
- package/src/superlocalmemory/learning/outcome_queue.py +14 -0
- package/src/superlocalmemory/learning/propensity.py +131 -0
- package/src/superlocalmemory/learning/reward.py +42 -16
- package/src/superlocalmemory/learning/reward_model.py +144 -0
- package/src/superlocalmemory/learning/reward_proxy.py +148 -22
- package/src/superlocalmemory/server/routes/v3_api.py +4 -3
- package/src/superlocalmemory/storage/migrations/M048_upcoming_holds_only_what_is_upcoming.py +22 -0
|
@@ -0,0 +1,279 @@
|
|
|
1
|
+
# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
|
|
2
|
+
# Licensed under AGPL-3.0-or-later - see LICENSE file
|
|
3
|
+
|
|
4
|
+
"""Engagement observed from what an agent did, not from what it was asked to say.
|
|
5
|
+
|
|
6
|
+
WHY THIS EXISTS
|
|
7
|
+
---------------
|
|
8
|
+
The reward ladder in ``reward_proxy`` asks one question — did a recalled
|
|
9
|
+
``fact_id`` appear verbatim in a later tool event — and defaults to ``0.5`` when
|
|
10
|
+
the answer is no. Both halves of that are broken.
|
|
11
|
+
|
|
12
|
+
The question requires the caller to copy an opaque marker into its next tool
|
|
13
|
+
call. Nothing makes it: no tool description asks for it and no rule requires
|
|
14
|
+
it. A design that depends on a behaviour nothing produces has no signal, and
|
|
15
|
+
in practice no signal was ever registered.
|
|
16
|
+
|
|
17
|
+
The default is worse than no answer. ``alpha += 0.5`` and ``beta += 0.5`` move
|
|
18
|
+
together, so a Beta posterior keeps its mean at exactly 0.5 while its variance
|
|
19
|
+
*shrinks*. Every neutral settlement makes an arm more confident that it is
|
|
20
|
+
average and harder for real evidence to move later. Neutral is not a safe
|
|
21
|
+
default; it is a slow commitment to knowing nothing.
|
|
22
|
+
|
|
23
|
+
WHAT REPLACES IT
|
|
24
|
+
----------------
|
|
25
|
+
Features computed from rows the system already writes. An agent that uses a
|
|
26
|
+
recalled memory leaves traces whether or not it cooperates: it reads or edits
|
|
27
|
+
the files the memory names, its next actions stay on the memory's subject, it
|
|
28
|
+
writes a follow-up memory that overlaps. None of that requires it to quote an
|
|
29
|
+
identifier.
|
|
30
|
+
|
|
31
|
+
Every feature here is observable, and each is reported separately so a reward
|
|
32
|
+
can say *why*. When nothing is observable the answer is ``None`` — abstain —
|
|
33
|
+
never a number.
|
|
34
|
+
|
|
35
|
+
NOT SELF-REFERENTIAL
|
|
36
|
+
--------------------
|
|
37
|
+
A signal derived from the mechanism it evaluates cannot detect that mechanism's
|
|
38
|
+
failure. Ranking position is therefore not a feature: the bandit chose the
|
|
39
|
+
ranking, so scoring it by what the bandit ranked first would confirm the bandit
|
|
40
|
+
to itself. Position enters only in ``propensity.py``, as a correction applied
|
|
41
|
+
*against* the observation, never as evidence for it.
|
|
42
|
+
"""
|
|
43
|
+
|
|
44
|
+
from __future__ import annotations
|
|
45
|
+
|
|
46
|
+
import json
|
|
47
|
+
import re
|
|
48
|
+
import sqlite3
|
|
49
|
+
from dataclasses import dataclass, field
|
|
50
|
+
from datetime import datetime, timedelta
|
|
51
|
+
|
|
52
|
+
__all__ = [
|
|
53
|
+
"EngagementFeatures",
|
|
54
|
+
"OBSERVATION_WINDOW_SEC",
|
|
55
|
+
"extract_features",
|
|
56
|
+
"tokenize",
|
|
57
|
+
]
|
|
58
|
+
|
|
59
|
+
#: How long after a recall an action may still be counted as caused by it.
|
|
60
|
+
#: Wider than the old 30 s hit window: an agent reads a memory, then thinks,
|
|
61
|
+
#: then acts, and 30 s discarded most of the acting.
|
|
62
|
+
OBSERVATION_WINDOW_SEC = 300
|
|
63
|
+
|
|
64
|
+
#: Tools whose payload naming a recalled memory's content is strong evidence
|
|
65
|
+
#: the memory was actually used, not merely returned.
|
|
66
|
+
_ARTIFACT_TOOLS = frozenset({"Write", "Edit", "NotebookEdit", "MultiEdit"})
|
|
67
|
+
|
|
68
|
+
#: Words carrying no topical signal; overlap on these is noise, and without
|
|
69
|
+
#: this filter every pair of payloads overlaps.
|
|
70
|
+
_STOPWORDS = frozenset("""
|
|
71
|
+
a an the and or but if then than that this these those is are was were be been
|
|
72
|
+
being to of in on at by for with from as it its into over under about after
|
|
73
|
+
before not no nor so such can will would should could may might must do does
|
|
74
|
+
did done have has had i you he she we they them his her their our your my me
|
|
75
|
+
""".split())
|
|
76
|
+
|
|
77
|
+
_TOKEN_RE = re.compile(r"[A-Za-z_][A-Za-z0-9_.\-/]{2,}")
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def tokenize(text: str) -> set[str]:
|
|
81
|
+
"""Content-bearing lowercase tokens, stopwords and short words removed."""
|
|
82
|
+
if not text:
|
|
83
|
+
return set()
|
|
84
|
+
return {
|
|
85
|
+
tok for tok in (m.group(0).lower() for m in _TOKEN_RE.finditer(text))
|
|
86
|
+
if tok not in _STOPWORDS and len(tok) > 2
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
@dataclass
|
|
91
|
+
class EngagementFeatures:
|
|
92
|
+
"""What was observed after one recall. Every field is measured, not inferred.
|
|
93
|
+
|
|
94
|
+
``observed`` is the honest summary: False means nothing happened that this
|
|
95
|
+
module can see, which is a reason to abstain rather than a reason to
|
|
96
|
+
penalise. An agent may have used a memory perfectly and left no trace.
|
|
97
|
+
"""
|
|
98
|
+
|
|
99
|
+
#: Best Jaccard-style overlap between any recalled fact and any following
|
|
100
|
+
#: tool payload, in [0, 1].
|
|
101
|
+
peak_overlap: float = 0.0
|
|
102
|
+
#: Overlap restricted to file-writing tools — the memory reached an artifact.
|
|
103
|
+
artifact_overlap: float = 0.0
|
|
104
|
+
#: A later remember/update_memory overlapping a recalled fact.
|
|
105
|
+
followup_write_overlap: float = 0.0
|
|
106
|
+
#: The same question asked again inside the requery window: the answer did
|
|
107
|
+
#: not satisfy. The one unambiguous negative available.
|
|
108
|
+
requeried: bool = False
|
|
109
|
+
#: Seconds from recall to the first following action, when there was one.
|
|
110
|
+
dwell_sec: float | None = None
|
|
111
|
+
#: Tool events seen in the window at all.
|
|
112
|
+
action_count: int = 0
|
|
113
|
+
#: Which fact ids the overlap landed on, so a reward can be explained.
|
|
114
|
+
matched_fact_ids: list[str] = field(default_factory=list)
|
|
115
|
+
#: A recalled fact's own id appeared verbatim in a later tool event. Rare,
|
|
116
|
+
#: because nothing makes an agent echo it — but unambiguous when it does
|
|
117
|
+
#: happen, so it is kept as the strongest single piece of evidence rather
|
|
118
|
+
#: than discarded along with the ladder that relied on it alone.
|
|
119
|
+
marker_hit: bool = False
|
|
120
|
+
|
|
121
|
+
@property
|
|
122
|
+
def observed(self) -> bool:
|
|
123
|
+
"""Whether anything at all was seen. Drives abstention."""
|
|
124
|
+
return bool(
|
|
125
|
+
self.requeried
|
|
126
|
+
or self.marker_hit
|
|
127
|
+
or self.action_count > 0
|
|
128
|
+
and (
|
|
129
|
+
self.peak_overlap > 0.0
|
|
130
|
+
or self.artifact_overlap > 0.0
|
|
131
|
+
or self.followup_write_overlap > 0.0
|
|
132
|
+
)
|
|
133
|
+
)
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def _table_exists(conn: sqlite3.Connection, name: str) -> bool:
|
|
137
|
+
try:
|
|
138
|
+
return conn.execute(
|
|
139
|
+
"SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (name,),
|
|
140
|
+
).fetchone() is not None
|
|
141
|
+
except sqlite3.Error:
|
|
142
|
+
return False
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def _fact_tokens(conn: sqlite3.Connection, fact_ids: list[str]) -> dict[str, set[str]]:
|
|
146
|
+
"""Content tokens per fact. Entities are folded in when the column exists,
|
|
147
|
+
because a memory's entities are what a later action is most likely to name.
|
|
148
|
+
"""
|
|
149
|
+
if not fact_ids or not _table_exists(conn, "atomic_facts"):
|
|
150
|
+
return {}
|
|
151
|
+
placeholders = ",".join("?" * len(fact_ids))
|
|
152
|
+
try:
|
|
153
|
+
rows = conn.execute(
|
|
154
|
+
f"SELECT fact_id, content, COALESCE(entities_json,'') " # noqa: S608
|
|
155
|
+
f"FROM atomic_facts WHERE fact_id IN ({placeholders})",
|
|
156
|
+
tuple(fact_ids),
|
|
157
|
+
).fetchall()
|
|
158
|
+
except sqlite3.Error:
|
|
159
|
+
return {}
|
|
160
|
+
|
|
161
|
+
out: dict[str, set[str]] = {}
|
|
162
|
+
for fact_id, content, entities_json in rows:
|
|
163
|
+
tokens = tokenize(content or "")
|
|
164
|
+
if entities_json:
|
|
165
|
+
try:
|
|
166
|
+
parsed = json.loads(entities_json)
|
|
167
|
+
except (ValueError, TypeError):
|
|
168
|
+
parsed = None
|
|
169
|
+
if isinstance(parsed, list):
|
|
170
|
+
for ent in parsed:
|
|
171
|
+
tokens |= tokenize(ent if isinstance(ent, str) else str(ent))
|
|
172
|
+
if tokens:
|
|
173
|
+
out[str(fact_id)] = tokens
|
|
174
|
+
return out
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def _following_events(
|
|
178
|
+
conn: sqlite3.Connection,
|
|
179
|
+
session_id: str,
|
|
180
|
+
profile_id: str,
|
|
181
|
+
recalled_at: datetime,
|
|
182
|
+
) -> list[tuple[str, str]]:
|
|
183
|
+
"""(tool_name, payload) for actions in this conversation after the recall.
|
|
184
|
+
|
|
185
|
+
Scoped to the conversation. Without that predicate a busy machine's
|
|
186
|
+
unrelated activity in the same five minutes would read as engagement.
|
|
187
|
+
"""
|
|
188
|
+
if not _table_exists(conn, "tool_events"):
|
|
189
|
+
return []
|
|
190
|
+
start = recalled_at.isoformat()
|
|
191
|
+
end = (recalled_at + timedelta(seconds=OBSERVATION_WINDOW_SEC)).isoformat()
|
|
192
|
+
try:
|
|
193
|
+
cols = {r[1] for r in conn.execute("PRAGMA table_info(tool_events)")}
|
|
194
|
+
except sqlite3.Error:
|
|
195
|
+
return []
|
|
196
|
+
if not {"session_id", "created_at", "tool_name"} <= cols:
|
|
197
|
+
return []
|
|
198
|
+
|
|
199
|
+
sql = (
|
|
200
|
+
"SELECT tool_name, COALESCE(input_summary,'') || ' ' || "
|
|
201
|
+
"COALESCE(output_summary,'') FROM tool_events "
|
|
202
|
+
"WHERE session_id = ? AND created_at > ? AND created_at <= ?"
|
|
203
|
+
)
|
|
204
|
+
params: tuple = (session_id, start, end)
|
|
205
|
+
if "profile_id" in cols:
|
|
206
|
+
sql += " AND (profile_id = ? OR profile_id IS NULL)"
|
|
207
|
+
params += (profile_id,)
|
|
208
|
+
sql += " ORDER BY created_at LIMIT 200"
|
|
209
|
+
try:
|
|
210
|
+
return [(str(r[0]), str(r[1])) for r in conn.execute(sql, params)]
|
|
211
|
+
except sqlite3.Error:
|
|
212
|
+
return []
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
#: A single word in common is coincidence, not evidence. Any two texts about
|
|
216
|
+
#: software share one token eventually, and containment makes that worse: a
|
|
217
|
+
#: memory that reduces to one content word scores a perfect 1.0 against every
|
|
218
|
+
#: payload containing it. Two independent tokens is the cheapest threshold that
|
|
219
|
+
#: distinguishes a shared subject from a shared word.
|
|
220
|
+
_MIN_OVERLAP_TOKENS = 2
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
def _overlap(fact_tokens: set[str], payload_tokens: set[str]) -> float:
|
|
224
|
+
"""Containment of the memory in the action, not symmetric Jaccard.
|
|
225
|
+
|
|
226
|
+
A tool payload is often far larger than a fact, and Jaccard would divide
|
|
227
|
+
that signal away precisely when the evidence is strongest.
|
|
228
|
+
"""
|
|
229
|
+
if not fact_tokens or not payload_tokens:
|
|
230
|
+
return 0.0
|
|
231
|
+
shared = fact_tokens & payload_tokens
|
|
232
|
+
if len(shared) < _MIN_OVERLAP_TOKENS:
|
|
233
|
+
return 0.0
|
|
234
|
+
return len(shared) / len(fact_tokens)
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
def extract_features(
|
|
238
|
+
memory_conn: sqlite3.Connection,
|
|
239
|
+
*,
|
|
240
|
+
session_id: str,
|
|
241
|
+
profile_id: str,
|
|
242
|
+
fact_ids: list[str],
|
|
243
|
+
recalled_at: datetime,
|
|
244
|
+
requeried: bool = False,
|
|
245
|
+
marker_hit: bool = False,
|
|
246
|
+
) -> EngagementFeatures:
|
|
247
|
+
"""Observe what followed one recall. Never raises; returns empty on error."""
|
|
248
|
+
features = EngagementFeatures(
|
|
249
|
+
requeried=bool(requeried), marker_hit=bool(marker_hit),
|
|
250
|
+
)
|
|
251
|
+
try:
|
|
252
|
+
by_fact = _fact_tokens(memory_conn, [str(f) for f in fact_ids])
|
|
253
|
+
events = _following_events(memory_conn, session_id, profile_id, recalled_at)
|
|
254
|
+
except sqlite3.Error:
|
|
255
|
+
return features
|
|
256
|
+
|
|
257
|
+
features.action_count = len(events)
|
|
258
|
+
if not by_fact or not events:
|
|
259
|
+
return features
|
|
260
|
+
|
|
261
|
+
matched: set[str] = set()
|
|
262
|
+
for tool_name, payload in events:
|
|
263
|
+
payload_tokens = tokenize(payload)
|
|
264
|
+
if not payload_tokens:
|
|
265
|
+
continue
|
|
266
|
+
for fact_id, tokens in by_fact.items():
|
|
267
|
+
score = _overlap(tokens, payload_tokens)
|
|
268
|
+
if score <= 0.0:
|
|
269
|
+
continue
|
|
270
|
+
matched.add(fact_id)
|
|
271
|
+
features.peak_overlap = max(features.peak_overlap, score)
|
|
272
|
+
if tool_name in _ARTIFACT_TOOLS:
|
|
273
|
+
features.artifact_overlap = max(features.artifact_overlap, score)
|
|
274
|
+
if tool_name.endswith(("remember", "update_memory")):
|
|
275
|
+
features.followup_write_overlap = max(
|
|
276
|
+
features.followup_write_overlap, score,
|
|
277
|
+
)
|
|
278
|
+
features.matched_fact_ids = sorted(matched)
|
|
279
|
+
return features
|
|
@@ -26,6 +26,8 @@ by ``unified_daemon.py``'s lifespan hook.
|
|
|
26
26
|
|
|
27
27
|
from __future__ import annotations
|
|
28
28
|
|
|
29
|
+
|
|
30
|
+
from superlocalmemory.core.session_identity import is_conversation
|
|
29
31
|
import logging
|
|
30
32
|
import queue
|
|
31
33
|
import threading
|
|
@@ -112,6 +114,18 @@ def enqueue_recall(event: RecallEvent) -> None:
|
|
|
112
114
|
# If the caller can't name a session, we silently drop: this
|
|
113
115
|
# is a recall whose outcome cannot match to a signal anyway.
|
|
114
116
|
return
|
|
117
|
+
if not is_conversation(event.session_id, event.profile_id):
|
|
118
|
+
# Same reason, one step further. A front that invents an id names
|
|
119
|
+
# itself, not a caller: ``engine:<pid>`` is the daemon every client
|
|
120
|
+
# shares, ``cli:<pid>`` a process that exits before any follow-on
|
|
121
|
+
# action, ``http:<ms>`` a single request. No tool event can ever
|
|
122
|
+
# carry one, so the outcome is unmatchable the moment it is written.
|
|
123
|
+
# Recording it anyway is not free: a play that is never settled from
|
|
124
|
+
# evidence is eventually settled from nothing, and a neutral update
|
|
125
|
+
# tightens a Beta posterior around its prior instead of leaving it
|
|
126
|
+
# movable.
|
|
127
|
+
_bump("recall_dropped_synthetic_session")
|
|
128
|
+
return
|
|
115
129
|
try:
|
|
116
130
|
_queue.put_nowait(event)
|
|
117
131
|
_bump("recall_enqueued")
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
|
|
2
|
+
# Licensed under AGPL-3.0-or-later - see LICENSE file
|
|
3
|
+
|
|
4
|
+
"""Inverse-propensity weighting, so the bandit cannot confirm itself.
|
|
5
|
+
|
|
6
|
+
THE BIAS
|
|
7
|
+
--------
|
|
8
|
+
The bandit decides what is shown, and engagement is then measured on what was
|
|
9
|
+
shown. Feed that back raw and the loop is circular: an arm ranked first is seen
|
|
10
|
+
more, so it is engaged with more, so it is ranked first more. The arm that wins
|
|
11
|
+
is the one that was already winning, and the posterior records popularity it
|
|
12
|
+
manufactured rather than usefulness it discovered.
|
|
13
|
+
|
|
14
|
+
That is the self-referential signal in its exact form — a measurement taken
|
|
15
|
+
through the mechanism it is meant to evaluate cannot see that mechanism fail.
|
|
16
|
+
|
|
17
|
+
THE CORRECTION
|
|
18
|
+
--------------
|
|
19
|
+
Weight each observation by the inverse of the probability the policy had of
|
|
20
|
+
showing that arm. An arm the policy was unlikely to show, that was engaged with
|
|
21
|
+
anyway, is strong evidence; an arm the policy shows almost always is weak
|
|
22
|
+
evidence whatever happens to it. This is the standard IPS estimator from
|
|
23
|
+
counterfactual learning-to-rank, and it makes the update unbiased with respect
|
|
24
|
+
to the policy's own choices.
|
|
25
|
+
|
|
26
|
+
Under Thompson sampling the propensity is not a stored number: an arm is shown
|
|
27
|
+
when its posterior draw beats every competitor's, so the probability is
|
|
28
|
+
``P(theta_i > theta_j for all j != i)`` with each ``theta ~ Beta(alpha, beta)``.
|
|
29
|
+
There is no closed form for more than two arms, so it is estimated by sampling.
|
|
30
|
+
|
|
31
|
+
WHEN THE COMPETITORS ARE UNKNOWN
|
|
32
|
+
--------------------------------
|
|
33
|
+
Return a weight of exactly 1.0 — no correction. A wrong correction is worse
|
|
34
|
+
than none: it would silently scale evidence by a number with no meaning, and
|
|
35
|
+
unlike an absent correction nothing downstream could tell. Abstaining from a
|
|
36
|
+
correction is visible in ``PropensityEstimate.corrected``.
|
|
37
|
+
"""
|
|
38
|
+
|
|
39
|
+
from __future__ import annotations
|
|
40
|
+
|
|
41
|
+
import random
|
|
42
|
+
from dataclasses import dataclass
|
|
43
|
+
|
|
44
|
+
__all__ = ["PropensityEstimate", "estimate_propensity", "ips_weight", "MAX_WEIGHT"]
|
|
45
|
+
|
|
46
|
+
#: Ceiling on the weight a single observation may carry. IPS has unbounded
|
|
47
|
+
#: variance as propensity approaches zero: one rare event with p = 0.001 would
|
|
48
|
+
#: otherwise move a posterior by 1000 plays' worth. Clipping trades a little
|
|
49
|
+
#: bias for a variance that does not destroy the estimate — the standard
|
|
50
|
+
#: bias-variance trade in clipped IPS.
|
|
51
|
+
MAX_WEIGHT = 10.0
|
|
52
|
+
|
|
53
|
+
#: Propensities below this are treated as this value before inversion.
|
|
54
|
+
_MIN_PROPENSITY = 1.0 / MAX_WEIGHT
|
|
55
|
+
|
|
56
|
+
#: Monte Carlo draws. 2000 puts the standard error of a mid-range propensity
|
|
57
|
+
#: near 0.01, which is finer than the weight clipping can express.
|
|
58
|
+
_DRAWS = 2000
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
@dataclass(frozen=True)
|
|
62
|
+
class PropensityEstimate:
|
|
63
|
+
"""A propensity and whether it was actually derived from anything."""
|
|
64
|
+
|
|
65
|
+
propensity: float
|
|
66
|
+
weight: float
|
|
67
|
+
corrected: bool
|
|
68
|
+
competitors: int = 0
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def estimate_propensity(
|
|
72
|
+
arm: tuple[float, float],
|
|
73
|
+
competitors: list[tuple[float, float]],
|
|
74
|
+
*,
|
|
75
|
+
draws: int = _DRAWS,
|
|
76
|
+
rng: random.Random | None = None,
|
|
77
|
+
) -> float:
|
|
78
|
+
"""P(this arm's Thompson draw is the largest), by Monte Carlo.
|
|
79
|
+
|
|
80
|
+
``arm`` and each competitor are ``(alpha, beta)`` posteriors. With no
|
|
81
|
+
competitors the arm is shown whenever it is considered, so the propensity
|
|
82
|
+
is 1.0 and the correction is a no-op.
|
|
83
|
+
"""
|
|
84
|
+
if not competitors:
|
|
85
|
+
return 1.0
|
|
86
|
+
# Fixed seed by default: the same play settled twice must produce the
|
|
87
|
+
# same weight, or a retry would move a posterior differently than the
|
|
88
|
+
# first attempt did.
|
|
89
|
+
generator = rng or random.Random(20260824)
|
|
90
|
+
alpha, beta = _sane(arm)
|
|
91
|
+
others = [_sane(c) for c in competitors]
|
|
92
|
+
|
|
93
|
+
wins = 0
|
|
94
|
+
for _ in range(max(1, int(draws))):
|
|
95
|
+
mine = generator.betavariate(alpha, beta)
|
|
96
|
+
if all(mine > generator.betavariate(a, b) for a, b in others):
|
|
97
|
+
wins += 1
|
|
98
|
+
return wins / max(1, int(draws))
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def _sane(posterior: tuple[float, float]) -> tuple[float, float]:
|
|
102
|
+
"""Beta requires strictly positive parameters; a stored 0 would raise."""
|
|
103
|
+
alpha, beta = posterior
|
|
104
|
+
return (max(float(alpha), 1e-6), max(float(beta), 1e-6))
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def ips_weight(
|
|
108
|
+
arm: tuple[float, float] | None,
|
|
109
|
+
competitors: list[tuple[float, float]] | None,
|
|
110
|
+
*,
|
|
111
|
+
draws: int = _DRAWS,
|
|
112
|
+
rng: random.Random | None = None,
|
|
113
|
+
) -> PropensityEstimate:
|
|
114
|
+
"""Clipped inverse-propensity weight for one observation.
|
|
115
|
+
|
|
116
|
+
Returns ``corrected=False`` and ``weight=1.0`` when there is nothing to
|
|
117
|
+
correct against, so a caller can tell an uncorrected update from a
|
|
118
|
+
corrected one that happened to land on 1.0.
|
|
119
|
+
"""
|
|
120
|
+
if arm is None or not competitors:
|
|
121
|
+
return PropensityEstimate(propensity=1.0, weight=1.0, corrected=False)
|
|
122
|
+
|
|
123
|
+
propensity = estimate_propensity(arm, competitors, draws=draws, rng=rng)
|
|
124
|
+
clipped = max(propensity, _MIN_PROPENSITY)
|
|
125
|
+
weight = min(1.0 / clipped, MAX_WEIGHT)
|
|
126
|
+
return PropensityEstimate(
|
|
127
|
+
propensity=propensity,
|
|
128
|
+
weight=weight,
|
|
129
|
+
corrected=True,
|
|
130
|
+
competitors=len(competitors),
|
|
131
|
+
)
|
|
@@ -735,6 +735,20 @@ class EngagementRewardModel:
|
|
|
735
735
|
signals = json.loads(row["signals_json"] or "{}")
|
|
736
736
|
except json.JSONDecodeError: # pragma: no cover
|
|
737
737
|
signals = {}
|
|
738
|
+
# A row that accumulated no signal has nothing to say. It used to
|
|
739
|
+
# be written out as an outcome anyway, carrying the label
|
|
740
|
+
# formula's base term of exactly 0.5 and marked ``settled`` as
|
|
741
|
+
# though a judgement had been reported. That value is not a
|
|
742
|
+
# harmless placeholder: ``alpha += 0.5`` with ``beta += 0.5`` moves
|
|
743
|
+
# both sides together, so the posterior keeps its mean and loses
|
|
744
|
+
# spread, and an arm settled this way repeatedly never leaves its
|
|
745
|
+
# prior.
|
|
746
|
+
#
|
|
747
|
+
# The row is still finalized, so it stops being rescanned; it is
|
|
748
|
+
# simply not turned into evidence it never was.
|
|
749
|
+
if not signals:
|
|
750
|
+
settle_ids.append(row["outcome_id"])
|
|
751
|
+
continue
|
|
738
752
|
reward = _compute_label(signals)
|
|
739
753
|
insert_batch.append(
|
|
740
754
|
(
|
|
@@ -756,7 +770,11 @@ class EngagementRewardModel:
|
|
|
756
770
|
_CHUNK = 500
|
|
757
771
|
written = 0
|
|
758
772
|
try:
|
|
759
|
-
|
|
773
|
+
# insert_batch and settle_ids are no longer parallel: a row with
|
|
774
|
+
# no signals is finalized without producing an outcome. Pad the
|
|
775
|
+
# insert side so each burst still pairs a write with the right
|
|
776
|
+
# finalizations, and never index one by the other's length.
|
|
777
|
+
for i in range(0, max(len(insert_batch), len(settle_ids)), _CHUNK):
|
|
760
778
|
i_chunk = insert_batch[i:i + _CHUNK]
|
|
761
779
|
s_chunk = settle_ids[i:i + _CHUNK]
|
|
762
780
|
placeholders = ",".join("?" * len(s_chunk))
|
|
@@ -764,22 +782,30 @@ class EngagementRewardModel:
|
|
|
764
782
|
conn = self._get_conn()
|
|
765
783
|
conn.execute("BEGIN IMMEDIATE")
|
|
766
784
|
try:
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
785
|
+
if i_chunk:
|
|
786
|
+
conn.executemany(
|
|
787
|
+
"INSERT OR REPLACE INTO action_outcomes "
|
|
788
|
+
"(outcome_id, profile_id, query, fact_ids_json,"
|
|
789
|
+
" outcome, context_json, timestamp, reward,"
|
|
790
|
+
" settled, settled_at, recall_query_id) "
|
|
791
|
+
"VALUES "
|
|
792
|
+
"(?, ?, '', ?, 'settled', '{}', ?, ?, 1, ?, ?)",
|
|
793
|
+
i_chunk,
|
|
794
|
+
)
|
|
795
|
+
if s_chunk:
|
|
796
|
+
conn.execute(
|
|
797
|
+
"UPDATE pending_outcomes "
|
|
798
|
+
f"SET status = 'settled' "
|
|
799
|
+
f"WHERE outcome_id IN ({placeholders})",
|
|
800
|
+
s_chunk,
|
|
801
|
+
)
|
|
781
802
|
conn.execute("COMMIT")
|
|
782
|
-
written
|
|
803
|
+
# Count what was FINALIZED, not what was written. A row
|
|
804
|
+
# with no signals is finalized without producing an
|
|
805
|
+
# outcome, and callers use this number to know the
|
|
806
|
+
# backlog drained — counting inserts would report 0
|
|
807
|
+
# forever and make the reaper look dead.
|
|
808
|
+
written += len(s_chunk)
|
|
783
809
|
except sqlite3.Error:
|
|
784
810
|
conn.execute("ROLLBACK")
|
|
785
811
|
raise
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
|
|
2
|
+
# Licensed under AGPL-3.0-or-later - see LICENSE file
|
|
3
|
+
|
|
4
|
+
"""Turn observed engagement into a reward, or into an honest refusal.
|
|
5
|
+
|
|
6
|
+
ABSTENTION IS THE POINT
|
|
7
|
+
-----------------------
|
|
8
|
+
The ladder this replaces always produced a number. When it saw nothing it
|
|
9
|
+
produced ``0.5``, and ``0.5`` is not neutral: ``alpha += 0.5`` with
|
|
10
|
+
``beta += 0.5`` holds a Beta posterior's mean at exactly 0.5 while shrinking
|
|
11
|
+
its variance, so each empty settlement makes an arm *more certain* it is
|
|
12
|
+
average and *less* movable by the evidence that finally arrives. Applied often
|
|
13
|
+
enough it does not merely fail to learn — it commits, with growing confidence,
|
|
14
|
+
to knowing nothing.
|
|
15
|
+
|
|
16
|
+
So this module returns ``None`` when nothing was observed. An unobserved recall
|
|
17
|
+
leaves the posterior untouched and free. Absence of evidence is recorded as
|
|
18
|
+
absence of evidence.
|
|
19
|
+
|
|
20
|
+
THE SCALE
|
|
21
|
+
---------
|
|
22
|
+
Positive evidence maps into ``(0.5, 1.0]``, a requery to ``0.0``, and nothing to
|
|
23
|
+
``None``. Nothing maps *to* 0.5, because that value is reserved for "no
|
|
24
|
+
information" and no observation carries that meaning: if it was worth
|
|
25
|
+
observing it was worth moving the posterior.
|
|
26
|
+
|
|
27
|
+
Weights are module constants and deliberately legible rather than fitted. There
|
|
28
|
+
is no ground-truth corpus of "was this memory actually useful" to fit against,
|
|
29
|
+
and inventing one from the system's own rankings would be the circularity
|
|
30
|
+
``propensity.py`` exists to break. They are a documented prior over evidence
|
|
31
|
+
strength; the *learning* happens in the posterior these rewards update, not in
|
|
32
|
+
the constants.
|
|
33
|
+
"""
|
|
34
|
+
|
|
35
|
+
from __future__ import annotations
|
|
36
|
+
|
|
37
|
+
from dataclasses import dataclass
|
|
38
|
+
|
|
39
|
+
from superlocalmemory.learning.engagement_features import EngagementFeatures
|
|
40
|
+
|
|
41
|
+
__all__ = ["RewardDecision", "score", "REQUERY_REWARD"]
|
|
42
|
+
|
|
43
|
+
#: A question asked again is the one unambiguous statement that an answer did
|
|
44
|
+
#: not serve. It is the only hard zero available.
|
|
45
|
+
REQUERY_REWARD = 0.0
|
|
46
|
+
|
|
47
|
+
#: Evidence weights, strongest first.
|
|
48
|
+
#: - a memory whose content reaches a written artifact was used, not just read
|
|
49
|
+
#: - a follow-up memory overlapping it means the agent built on it
|
|
50
|
+
#: - appearing in any later action is real but weaker: the agent may have been
|
|
51
|
+
#: working on the subject regardless of what was recalled
|
|
52
|
+
_W_ARTIFACT = 0.50
|
|
53
|
+
_W_FOLLOWUP = 0.30
|
|
54
|
+
_W_PRESENCE = 0.20
|
|
55
|
+
|
|
56
|
+
#: Positive evidence starts just above the reserved no-information value, so
|
|
57
|
+
#: the weakest real observation still moves an arm up rather than nowhere.
|
|
58
|
+
_POSITIVE_FLOOR = 0.55
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
@dataclass(frozen=True)
|
|
62
|
+
class RewardDecision:
|
|
63
|
+
"""A reward, or an abstention, with the reason attached.
|
|
64
|
+
|
|
65
|
+
``reward is None`` means do not update. ``kind`` names what was seen so a
|
|
66
|
+
settled play can be explained after the fact instead of being a bare float.
|
|
67
|
+
"""
|
|
68
|
+
|
|
69
|
+
reward: float | None
|
|
70
|
+
kind: str
|
|
71
|
+
detail: str = ""
|
|
72
|
+
|
|
73
|
+
@property
|
|
74
|
+
def abstained(self) -> bool:
|
|
75
|
+
return self.reward is None
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def score(features: EngagementFeatures) -> RewardDecision:
|
|
79
|
+
"""Map observations to a reward in [0, 1], or abstain.
|
|
80
|
+
|
|
81
|
+
Never raises: a settler running over a month of rows must not stop on one
|
|
82
|
+
malformed observation.
|
|
83
|
+
"""
|
|
84
|
+
if features.marker_hit:
|
|
85
|
+
# The agent named the memory outright. Nothing makes it do this, so it
|
|
86
|
+
# is rare, but when it happens there is nothing to infer.
|
|
87
|
+
return RewardDecision(
|
|
88
|
+
reward=1.0,
|
|
89
|
+
kind="proxy_position",
|
|
90
|
+
detail="a recalled fact id appeared in a later tool event",
|
|
91
|
+
)
|
|
92
|
+
|
|
93
|
+
if features.requeried:
|
|
94
|
+
return RewardDecision(
|
|
95
|
+
reward=REQUERY_REWARD,
|
|
96
|
+
kind="proxy_requery",
|
|
97
|
+
detail="the same question was asked again inside the window",
|
|
98
|
+
)
|
|
99
|
+
|
|
100
|
+
if not features.observed:
|
|
101
|
+
return RewardDecision(
|
|
102
|
+
reward=None,
|
|
103
|
+
kind="unobserved",
|
|
104
|
+
detail=(
|
|
105
|
+
f"no engagement visible in {features.action_count} following "
|
|
106
|
+
"action(s); posterior left untouched"
|
|
107
|
+
),
|
|
108
|
+
)
|
|
109
|
+
|
|
110
|
+
artifact = _clamp(features.artifact_overlap)
|
|
111
|
+
followup = _clamp(features.followup_write_overlap)
|
|
112
|
+
presence = _clamp(features.peak_overlap)
|
|
113
|
+
|
|
114
|
+
strength = (
|
|
115
|
+
_W_ARTIFACT * artifact
|
|
116
|
+
+ _W_FOLLOWUP * followup
|
|
117
|
+
+ _W_PRESENCE * presence
|
|
118
|
+
)
|
|
119
|
+
# strength in [0, 1]; map onto (floor, 1.0].
|
|
120
|
+
reward = _POSITIVE_FLOOR + (1.0 - _POSITIVE_FLOOR) * _clamp(strength)
|
|
121
|
+
|
|
122
|
+
if artifact > 0.0:
|
|
123
|
+
kind = "artifact_overlap"
|
|
124
|
+
elif followup > 0.0:
|
|
125
|
+
kind = "followup_overlap"
|
|
126
|
+
else:
|
|
127
|
+
kind = "presence_overlap"
|
|
128
|
+
|
|
129
|
+
return RewardDecision(
|
|
130
|
+
reward=round(reward, 6),
|
|
131
|
+
kind=kind,
|
|
132
|
+
detail=(
|
|
133
|
+
f"artifact={artifact:.3f} followup={followup:.3f} "
|
|
134
|
+
f"presence={presence:.3f} on {len(features.matched_fact_ids)} fact(s)"
|
|
135
|
+
),
|
|
136
|
+
)
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def _clamp(value: float) -> float:
|
|
140
|
+
try:
|
|
141
|
+
v = float(value)
|
|
142
|
+
except (TypeError, ValueError):
|
|
143
|
+
return 0.0
|
|
144
|
+
return 0.0 if v < 0.0 else (1.0 if v > 1.0 else v)
|