superlocalmemory 4.0.10 → 4.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (145) hide show
  1. package/.claude-plugin/marketplace.json +12 -2
  2. package/CHANGELOG.md +244 -0
  3. package/README.md +40 -75
  4. package/package.json +6 -3
  5. package/plugin/.claude-plugin/plugin.json +2 -2
  6. package/plugin/CLAUDE.md +3 -3
  7. package/plugin/agents/slm-governance-advisor.md +1 -1
  8. package/plugin/agents/slm-loop-runner.md +4 -4
  9. package/plugin/agents/slm-memory-advisor.md +1 -1
  10. package/plugin/agents/slm-optimize-advisor.md +1 -1
  11. package/plugin/requirements.txt +1 -1
  12. package/plugin/skills/slm-cache/SKILL.md +1 -1
  13. package/plugin/skills/slm-compress/SKILL.md +1 -1
  14. package/plugin/skills/slm-governance/SKILL.md +1 -1
  15. package/plugin/skills/slm-graph/SKILL.md +1 -1
  16. package/plugin/skills/slm-loop/SKILL.md +2 -2
  17. package/plugin/skills/slm-mesh/SKILL.md +1 -1
  18. package/plugin/skills/slm-profile/SKILL.md +5 -5
  19. package/plugin/skills/slm-recall/SKILL.md +102 -15
  20. package/plugin/skills/slm-remember/SKILL.md +35 -3
  21. package/plugin/skills/slm-scope/SKILL.md +1 -1
  22. package/plugin/skills/slm-session/SKILL.md +29 -3
  23. package/plugin/skills/slm-status/SKILL.md +1 -1
  24. package/plugin-src/rules/AGENTS.md +16 -8
  25. package/plugin-src/skills/slm-cache/SKILL.md +1 -1
  26. package/plugin-src/skills/slm-compress/SKILL.md +1 -1
  27. package/plugin-src/skills/slm-governance/SKILL.md +1 -1
  28. package/plugin-src/skills/slm-graph/SKILL.md +1 -1
  29. package/plugin-src/skills/slm-loop/SKILL.md +2 -2
  30. package/plugin-src/skills/slm-mesh/SKILL.md +1 -1
  31. package/plugin-src/skills/slm-profile/SKILL.md +5 -5
  32. package/plugin-src/skills/slm-recall/SKILL.md +102 -15
  33. package/plugin-src/skills/slm-remember/SKILL.md +35 -3
  34. package/plugin-src/skills/slm-scope/SKILL.md +1 -1
  35. package/plugin-src/skills/slm-session/SKILL.md +29 -3
  36. package/plugin-src/skills/slm-status/SKILL.md +1 -1
  37. package/pyproject.toml +1 -1
  38. package/src/superlocalmemory/__init__.py +1 -1
  39. package/src/superlocalmemory/cli/commands.py +357 -18
  40. package/src/superlocalmemory/cli/daemon.py +30 -0
  41. package/src/superlocalmemory/cli/db_migrate.py +71 -1
  42. package/src/superlocalmemory/cli/gdpr_cmd.py +15 -2
  43. package/src/superlocalmemory/cli/main.py +24 -2
  44. package/src/superlocalmemory/code_graph/database.py +44 -0
  45. package/src/superlocalmemory/compliance/gdpr.py +449 -39
  46. package/src/superlocalmemory/core/admission.py +231 -11
  47. package/src/superlocalmemory/core/backend_orchestrator.py +190 -84
  48. package/src/superlocalmemory/core/config.py +90 -11
  49. package/src/superlocalmemory/core/consolidation_engine.py +34 -0
  50. package/src/superlocalmemory/core/engine.py +140 -11
  51. package/src/superlocalmemory/core/graph_analyzer.py +76 -112
  52. package/src/superlocalmemory/core/graph_metrics.py +597 -0
  53. package/src/superlocalmemory/core/graph_pruner.py +121 -0
  54. package/src/superlocalmemory/core/maintenance_scheduler.py +205 -0
  55. package/src/superlocalmemory/core/mode_capability.py +111 -0
  56. package/src/superlocalmemory/core/ollama_validator.py +315 -0
  57. package/src/superlocalmemory/core/projection_drain.py +380 -0
  58. package/src/superlocalmemory/core/recall_pipeline.py +390 -3
  59. package/src/superlocalmemory/core/recall_worker.py +6 -3
  60. package/src/superlocalmemory/core/scale_autopromote.py +196 -0
  61. package/src/superlocalmemory/core/scale_engine.py +16 -2
  62. package/src/superlocalmemory/core/score_contract.py +21 -1
  63. package/src/superlocalmemory/core/session_identity.py +85 -0
  64. package/src/superlocalmemory/core/status_contract.py +108 -0
  65. package/src/superlocalmemory/core/worker_pool.py +4 -4
  66. package/src/superlocalmemory/core/working_memory.py +288 -0
  67. package/src/superlocalmemory/encoding/cognitive_consolidator.py +36 -6
  68. package/src/superlocalmemory/encoding/context_generator.py +1 -1
  69. package/src/superlocalmemory/encoding/entity_resolver.py +38 -0
  70. package/src/superlocalmemory/encoding/fact_extractor.py +18 -14
  71. package/src/superlocalmemory/encoding/prospective_markers.py +262 -0
  72. package/src/superlocalmemory/encoding/type_router.py +12 -12
  73. package/src/superlocalmemory/evolution/mutation_generator.py +30 -4
  74. package/src/superlocalmemory/graph/cozo_adjacency.py +122 -0
  75. package/src/superlocalmemory/graph/cozo_backend.py +103 -138
  76. package/src/superlocalmemory/hooks/portable_kit.py +10 -2
  77. package/src/superlocalmemory/learning/bandit.py +43 -0
  78. package/src/superlocalmemory/learning/consolidation_worker.py +54 -0
  79. package/src/superlocalmemory/learning/database.py +60 -3
  80. package/src/superlocalmemory/learning/entity_compiler.py +21 -58
  81. package/src/superlocalmemory/learning/feedback.py +3 -1
  82. package/src/superlocalmemory/learning/outcomes.py +47 -16
  83. package/src/superlocalmemory/learning/pattern_miner.py +28 -3
  84. package/src/superlocalmemory/learning/pattern_miner_constants.py +43 -0
  85. package/src/superlocalmemory/learning/pcos.py +291 -0
  86. package/src/superlocalmemory/learning/reward_from_outcomes.py +365 -0
  87. package/src/superlocalmemory/learning/reward_proxy.py +100 -10
  88. package/src/superlocalmemory/learning/signal_kinds.py +79 -0
  89. package/src/superlocalmemory/mcp/profiles.py +14 -2
  90. package/src/superlocalmemory/mcp/tools_active.py +2 -1
  91. package/src/superlocalmemory/mcp/tools_core.py +31 -3
  92. package/src/superlocalmemory/mcp/tools_v28.py +20 -1
  93. package/src/superlocalmemory/parameterization/pattern_extractor.py +14 -1
  94. package/src/superlocalmemory/parameterization/soft_prompt_generator.py +98 -0
  95. package/src/superlocalmemory/retrieval/bm25_channel.py +64 -3
  96. package/src/superlocalmemory/retrieval/channel_status.py +117 -0
  97. package/src/superlocalmemory/retrieval/engine.py +106 -11
  98. package/src/superlocalmemory/retrieval/entity_channel.py +210 -256
  99. package/src/superlocalmemory/retrieval/graph_adjacency.py +219 -0
  100. package/src/superlocalmemory/retrieval/scope_policy.py +20 -0
  101. package/src/superlocalmemory/retrieval/semantic_channel.py +47 -5
  102. package/src/superlocalmemory/retrieval/spreading.py +288 -0
  103. package/src/superlocalmemory/server/api.py +24 -5
  104. package/src/superlocalmemory/server/bandit_loops.py +17 -1
  105. package/src/superlocalmemory/server/rbac_enforce.py +26 -6
  106. package/src/superlocalmemory/server/recall_health.py +87 -10
  107. package/src/superlocalmemory/server/recall_serializer.py +9 -0
  108. package/src/superlocalmemory/server/routes/behavioral.py +75 -10
  109. package/src/superlocalmemory/server/routes/compliance.py +98 -18
  110. package/src/superlocalmemory/server/routes/config_api.py +186 -4
  111. package/src/superlocalmemory/server/routes/evolution.py +178 -0
  112. package/src/superlocalmemory/server/routes/ingest.py +8 -0
  113. package/src/superlocalmemory/server/routes/learning_telemetry.py +2 -1
  114. package/src/superlocalmemory/server/routes/memories.py +49 -7
  115. package/src/superlocalmemory/server/routes/timeline.py +4 -0
  116. package/src/superlocalmemory/server/routes/v3_api.py +191 -15
  117. package/src/superlocalmemory/server/ui.py +20 -4
  118. package/src/superlocalmemory/server/unified_daemon.py +241 -7
  119. package/src/superlocalmemory/storage/_migration_internals.py +54 -2
  120. package/src/superlocalmemory/storage/_schema_version.py +24 -3
  121. package/src/superlocalmemory/storage/database.py +477 -59
  122. package/src/superlocalmemory/storage/embedding_codec.py +71 -0
  123. package/src/superlocalmemory/storage/lineage_retention.py +236 -0
  124. package/src/superlocalmemory/storage/logical_edges.py +43 -2
  125. package/src/superlocalmemory/storage/migration_runner.py +119 -0
  126. package/src/superlocalmemory/storage/migrations/M043_quarantine_display_summaries.py +60 -36
  127. package/src/superlocalmemory/storage/migrations/M044_play_carries_its_own_evidence.py +127 -0
  128. package/src/superlocalmemory/storage/migrations/M045_fact_outcome_score.py +158 -0
  129. package/src/superlocalmemory/storage/migrations/M046_prospective_memory_has_its_own_name.py +620 -0
  130. package/src/superlocalmemory/storage/migrations/M047_fisher_vectors_are_stored_like_every_other_vector.py +306 -0
  131. package/src/superlocalmemory/storage/migrations/M048_upcoming_holds_only_what_is_upcoming.py +207 -0
  132. package/src/superlocalmemory/storage/migrations/M049_a_schema_version_marker_is_one_row.py +201 -0
  133. package/src/superlocalmemory/storage/migrations.py +18 -2
  134. package/src/superlocalmemory/storage/models.py +40 -1
  135. package/src/superlocalmemory/storage/projection_outbox.py +346 -0
  136. package/src/superlocalmemory/storage/retention_policy.py +860 -0
  137. package/src/superlocalmemory/storage/schema.py +35 -1
  138. package/src/superlocalmemory/storage/write_coordinator.py +19 -2
  139. package/src/superlocalmemory/trust/scorer.py +43 -1
  140. package/src/superlocalmemory/ui/index.html +9 -18
  141. package/src/superlocalmemory/ui/js/event-delegation.js +12 -1
  142. package/src/superlocalmemory/ui/js/od-health.js +28 -6
  143. package/src/superlocalmemory/ui/js/od-memories.js +19 -0
  144. package/src/superlocalmemory/ui/js/od-settings.js +87 -1
  145. package/src/superlocalmemory/ui/js/recall-lab.js +78 -3
@@ -5,7 +5,16 @@
5
5
  """SuperLocalMemory v3.4.5 — CozoDB Graph Backend.
6
6
 
7
7
  Embedded graph database backend powered by CozoDB (MPL-2.0).
8
- Replaces NetworkX for entity graph storage and traversal.
8
+
9
+ It stores the graph. It does not traverse it, and the line that used to claim it
10
+ replaced NetworkX "for storage and traversal" outlived the traversal by a
11
+ release: the walk lives once in ``retrieval/spreading``, as a pure function of an
12
+ adjacency snapshot, and is numpy rather than NetworkX either way. What this
13
+ module supplies is the snapshot's edges, read by
14
+ ``graph/cozo_adjacency.CozoAdjacencySource`` -- measured at 395 ms against
15
+ SQLite's 2,477 ms on a 208,151-edge store. Both figures come from a hand run on a
16
+ copy of that store, recorded in ``cozo_adjacency``; nothing in the test suite
17
+ reproduces them.
9
18
 
10
19
  All Datalog queries are private to this module.
11
20
  External code calls Python methods only — never raw Datalog strings.
@@ -277,6 +286,64 @@ class CozoDBGraphBackend:
277
286
  :rm edge {from_id, to_id, edge_type => weight, metadata, profile_id, created_at}
278
287
  """, {"fact_id": fact_id})
279
288
 
289
+ def remove_entity(self, entity_id: str) -> None:
290
+ """Remove an entity node and every bridge that reaches it.
291
+
292
+ ``remove_fact`` clears a fact's bridges and edges and leaves the entity
293
+ nodes alone, which is right when a fact is deleted -- the entity is
294
+ still real and other facts still reference it. It is wrong when the
295
+ entity itself is erased: the node stayed behind with its name in it,
296
+ so a graph query could still name somebody after their record had been
297
+ removed from the store.
298
+
299
+ The id is bound as a parameter, so an entity name containing a quote
300
+ cannot become query text.
301
+ """
302
+ self._db.run("""
303
+ ?[fact_id, entity_id, profile_id] :=
304
+ *fact_entity{fact_id, entity_id, profile_id}, entity_id = $entity_id
305
+ :rm fact_entity {fact_id, entity_id => profile_id}
306
+ """, {"entity_id": entity_id})
307
+ self._db.run("""
308
+ ?[from_id, to_id, edge_type, weight, metadata, profile_id, created_at] :=
309
+ *edge{from_id, to_id, edge_type, weight, metadata, profile_id, created_at},
310
+ (from_id = $entity_id or to_id = $entity_id)
311
+ :rm edge {from_id, to_id, edge_type => weight, metadata, profile_id, created_at}
312
+ """, {"entity_id": entity_id})
313
+ self._db.run("""
314
+ ?[id, name, entity_type, tier, properties, profile_id,
315
+ created_at, updated_at] :=
316
+ *entity{id, name, entity_type, tier, properties, profile_id,
317
+ created_at, updated_at},
318
+ id = $entity_id
319
+ :rm entity {id => name, entity_type, tier, properties, profile_id,
320
+ created_at, updated_at}
321
+ """, {"entity_id": entity_id})
322
+
323
+ def remove_fact_candidacy(self, fact_id: str) -> None:
324
+ """Stop a fact being *offered* as a candidate, without touching the graph.
325
+
326
+ Removes only the ``fact_entity`` bridge. The fact's edges stay.
327
+
328
+ The two are not the same thing and the SQLite channel already treats them
329
+ differently. Its entity map filters on ``visible_fact_clause()`` — a
330
+ withheld row must never enter it, because it carries its whole cluster's
331
+ pooled entity list and out-ranks real memories. Its edge walk filters on
332
+ scope only, with no visibility predicate at all, so it traverses edges to
333
+ withheld and archived facts and lets hydration drop them at the end.
334
+
335
+ So a projection that deleted a hidden fact's edges would hold a smaller
336
+ adjacency than the walk it is meant to replace, and give different
337
+ answers for every fact that happened to neighbour a withheld one. Use
338
+ this for a fact that has become invisible; use ``remove_fact`` only for
339
+ one that is genuinely gone.
340
+ """
341
+ self._db.run("""
342
+ ?[fact_id, entity_id, profile_id] :=
343
+ *fact_entity{fact_id, entity_id, profile_id}, fact_id = $fact_id
344
+ :rm fact_entity {fact_id, entity_id => profile_id}
345
+ """, {"fact_id": fact_id})
346
+
280
347
  def record_shadow_comparison(
281
348
  self,
282
349
  *,
@@ -346,9 +413,24 @@ class CozoDBGraphBackend:
346
413
 
347
414
  # Step 2: Export fact-to-canonical-entity mappings. This relation is
348
415
  # what allows a canonical query seed to enter the fact graph.
349
- facts_sql = """
416
+ #
417
+ # Withheld and archived facts are excluded, with the same predicate the
418
+ # SQLite entity map uses and for the reason it records: such a row
419
+ # carries its whole cluster's pooled entity list, so it out-ranks real
420
+ # memories, takes the top-k budget, and is then discarded at hydration.
421
+ # This export predated that fix. Measured on a copy of the author's
422
+ # store, it had put 1,257 unreturnable facts into the bridge, and the
423
+ # Cozo graph search diverged from SQLite on every query tried — one
424
+ # returned 9 results against SQLite's 20. The projection failed closed
425
+ # each time, so recall stayed correct and the projection was useless.
426
+ from superlocalmemory.storage.database import (
427
+ visible_fact_clause_for_connection,
428
+ )
429
+
430
+ facts_sql = f"""
350
431
  SELECT fact_id, canonical_entities_json
351
432
  FROM atomic_facts WHERE profile_id = ?
433
+ {visible_fact_clause_for_connection(conn)}
352
434
  """
353
435
  fact_entity_dicts: list[dict[str, str]] = []
354
436
  for fact_id, raw_entities in conn.execute(facts_sql, (profile_id,)).fetchall():
@@ -386,142 +468,25 @@ class CozoDBGraphBackend:
386
468
 
387
469
  return len(edge_dicts)
388
470
 
389
- def recall_facts(
390
- self,
391
- seed_entity_ids: list[str],
392
- *,
393
- profile_id: str = "default",
394
- depth: int = 4,
395
- decay: float = 0.7,
396
- threshold: float = 0.05,
397
- top_k: int = 50,
398
- ) -> list[tuple[str, float]]:
399
- """Mirror SLM's entity-to-fact/fact-graph activation in Cozo storage.
400
-
401
- Query values never enter Datalog source. Cozo is used as the durable
402
- projection; activation runs in Python so the algorithm stays aligned
403
- with the SQLite in-memory channel and can be shadow-compared exactly.
404
- """
405
- if not seed_entity_ids:
406
- return []
407
- entity_rows = self._db.run(
408
- "?[fact_id, entity_id] := *fact_entity{fact_id, entity_id, profile_id}, profile_id = $profile_id",
409
- {"profile_id": profile_id},
410
- )
411
- edge_rows = self._db.run(
412
- "?[from_id, to_id, weight] := *edge{from_id, to_id, weight, profile_id}, profile_id = $profile_id",
413
- {"profile_id": profile_id},
414
- )
415
- entity_to_facts: dict[str, list[str]] = {}
416
- fact_to_entities: dict[str, list[str]] = {}
417
- for fact_id, entity_id in entity_rows.values.tolist() if len(entity_rows) else []:
418
- entity_to_facts.setdefault(str(entity_id), []).append(str(fact_id))
419
- fact_to_entities.setdefault(str(fact_id), []).append(str(entity_id))
420
- adjacency: dict[str, list[tuple[str, float]]] = {}
421
- for source_id, target_id, weight in edge_rows.values.tolist() if len(edge_rows) else []:
422
- source, target = str(source_id), str(target_id)
423
- # Match EntityGraphChannel: graph edges are bidirectional during
424
- # activation even when stored as directed rows.
425
- adjacency.setdefault(source, []).append((target, float(weight)))
426
- adjacency.setdefault(target, []).append((source, float(weight)))
427
-
428
- activation: dict[str, float] = {}
429
- visited_entities = set(seed_entity_ids)
430
- for entity_id in seed_entity_ids:
431
- for fact_id in entity_to_facts.get(entity_id, ()):
432
- activation[fact_id] = max(activation.get(fact_id, 0.0), 1.0)
433
- frontier = set(activation)
434
- for hop in range(1, depth):
435
- hop_decay = decay ** hop
436
- if hop_decay < threshold:
437
- break
438
- next_frontier: set[str] = set()
439
- for fact_id in frontier:
440
- for neighbor_id, _weight in adjacency.get(fact_id, ()):
441
- # SQLite intentionally ignores edge weights when graph
442
- # metrics are unavailable; use that same baseline here.
443
- score = activation[fact_id] * decay
444
- if score >= threshold and score > activation.get(neighbor_id, 0.0):
445
- activation[neighbor_id] = score
446
- next_frontier.add(neighbor_id)
447
- for fact_id in frontier:
448
- for entity_id in fact_to_entities.get(fact_id, ()):
449
- if entity_id in visited_entities:
450
- continue
451
- visited_entities.add(entity_id)
452
- for related_fact_id in entity_to_facts.get(entity_id, ()):
453
- if hop_decay > activation.get(related_fact_id, 0.0):
454
- activation[related_fact_id] = hop_decay
455
- next_frontier.add(related_fact_id)
456
- frontier = next_frontier
457
- if not frontier:
458
- break
459
- results = [(fact_id, score) for fact_id, score in activation.items() if score >= threshold]
460
- if not results:
461
- return []
462
- maximum = max(score for _, score in results)
463
- results = [(fact_id, score / maximum) for fact_id, score in results]
464
- return sorted(results, key=lambda item: item[1], reverse=True)[:top_k]
465
-
466
- # ------------------------------------------------------------------
467
- # Spreading Activation (Python BFS over CozoDB edges)
468
- # ------------------------------------------------------------------
469
-
470
- def spreading_activation(
471
- self,
472
- seed_entities: list[str],
473
- depth: int = 3,
474
- decay: float = 0.5,
475
- top_k: int = 50,
476
- ) -> list[tuple[str, float]]:
477
- """BFS from seed nodes with weight decay per hop.
478
-
479
- Uses CozoDB as fast edge store, Python for BFS logic.
480
- Returns [(entity_id, activation_score), ...] sorted by score desc.
481
- """
482
- if not seed_entities:
483
- return []
484
-
485
- scores: dict[str, float] = {}
486
- current_frontier: set[str] = set(seed_entities)
487
- for s in seed_entities:
488
- scores[s] = 1.0
489
-
490
- for d in range(depth):
491
- if not current_frontier:
492
- break
493
- next_frontier: set[str] = set()
494
- hop_multiplier = decay ** (d + 1)
495
-
496
- for entity_id in current_frontier:
497
- # Query all outgoing edges from this entity
498
- try:
499
- result = self._db.run("""
500
- ?[to_id, weight] :=
501
- *edge{from_id, to_id, weight}, from_id = $entity_id
502
- """, {"entity_id": entity_id})
503
- df = result if hasattr(result, "values") else result
504
- if df is None or len(df) == 0:
505
- continue
506
- rows = df.values.tolist() if hasattr(df, "values") else []
507
- for to_id, weight in rows:
508
- to_id_str = str(to_id)
509
- score = hop_multiplier * float(weight)
510
- if to_id_str not in scores or score > scores[to_id_str]:
511
- scores[to_id_str] = score
512
- next_frontier.add(to_id_str)
513
- except Exception:
514
- continue
515
-
516
- current_frontier = next_frontier
517
-
518
- # Sort by score desc, return top_k
519
- ranked = sorted(scores.items(), key=lambda x: x[1], reverse=True)
520
- return ranked[:top_k]
521
-
522
- # ------------------------------------------------------------------
523
- # PageRank (Python iterative over CozoDB edges)
524
- # ------------------------------------------------------------------
471
+ # recall_facts() and spreading_activation() were here, and they were a
472
+ # second implementation of the walk in retrieval/entity_channel.py. They did
473
+ # not compute the same function: the channel multiplies activation by a
474
+ # PageRank factor every hop and these did not, so on a real store 3,567 of
475
+ # 3,667 shared facts came out with different scores, the result sets
476
+ # differed, and the projected path failed its shadow comparison on every
477
+ # query and fell back to SQLite. The projection was correct data that
478
+ # nothing could use.
479
+ #
480
+ # spreading_activation() also queried Cozo once per frontier node: 200
481
+ # single-node queries measured 18,125 ms against 303 ms for one batched
482
+ # query returning the identical 6,046 rows. With a mean degree of 228 over
483
+ # four hops that is minutes per recall.
484
+ #
485
+ # The walk now lives in retrieval/spreading.py, once, as a pure function of
486
+ # an AdjacencySnapshot. A storage backend supplies adjacency, not answers —
487
+ # which is also why there is no longer anything to shadow-compare. Do not
488
+ # reintroduce a traversal here; add an AdjacencySource adapter instead
489
+ # (retrieval/graph_adjacency.py).
525
490
 
526
491
  def pagerank(
527
492
  self, damping: float = 0.85, max_iter: int = 100
@@ -38,9 +38,17 @@ SLM_MARKER_END = "<!-- SLM-END -->"
38
38
  # "http-mcp-remote" — mcp-remote stdio bridge for stdio-only clients.
39
39
  VALID_TRANSPORTS: frozenset[str] = frozenset({"stdio", "http", "http-mcp-remote"})
40
40
 
41
+ #: The commands below are Claude Code's own, not ours -- ``slm`` has no
42
+ #: ``plugin`` subcommand and never has, so the instruction this used to print
43
+ #: ended at ``invalid choice: 'plugin'`` with nothing else offered. The working
44
+ #: path already existed in ``cli/setup_wizard.py``; it just was not the one
45
+ #: anybody was told about.
41
46
  CLAUDE_CODE_PLUGIN_POINTER = (
42
- "slm connect claude-code: Claude Code is configured via the SLM plugin (see plugin/ directory).\n"
43
- "Run: slm plugin install OR see plugin-src/ for manual installation.\n"
47
+ "slm connect claude-code: Claude Code is configured via the SLM plugin, "
48
+ "which Claude Code installs itself.\n"
49
+ "Run: claude plugin marketplace add qualixar/superlocalmemory\n"
50
+ "then: claude plugin install superlocalmemory@qualixar\n"
51
+ "Or run `slm setup` to do both, or see plugin-src/ to install by hand.\n"
44
52
  "No MCP config file is written by this command."
45
53
  )
46
54
 
@@ -22,6 +22,7 @@ All SQL is parameterised — grep guard in CI ensures no f-string SQL here.
22
22
 
23
23
  from __future__ import annotations
24
24
 
25
+ import json
25
26
  import logging
26
27
  import os
27
28
  import secrets
@@ -30,6 +31,7 @@ import threading
30
31
  from dataclasses import dataclass, field
31
32
  from datetime import datetime, timedelta, timezone
32
33
  from pathlib import Path
34
+ from collections.abc import Sequence
33
35
  from typing import Any
34
36
 
35
37
  from superlocalmemory.learning.arm_catalog import ARM_CATALOG
@@ -40,6 +42,11 @@ from superlocalmemory.learning.bandit_cache import (
40
42
 
41
43
  logger = logging.getLogger(__name__)
42
44
 
45
+ #: How many shown fact_ids a play records. The settler reads the top 3; the
46
+ #: extra two are headroom for a reranker that reorders after the play is
47
+ #: written, and a bound so this cannot become an unread blob on a hot path.
48
+ _SHOWN_FACT_LIMIT = 5
49
+
43
50
  _FALLBACK_ARM_ID = "fallback_default"
44
51
 
45
52
  _DEFAULT_ALPHA_CAP = float(os.environ.get("SLM_BANDIT_ALPHA_CAP", "1000.0"))
@@ -402,6 +409,42 @@ class ContextualBandit:
402
409
  pass
403
410
  return True
404
411
 
412
+ def record_shown(self, play_id: int, fact_ids: Sequence[str]) -> bool:
413
+ """Record which memories this play surfaced, for later settlement.
414
+
415
+ A play is settled from evidence — did anything downstream reference a
416
+ memory this query returned? The settler used to answer "which memories"
417
+ by reading ``learning_signals`` for the same ``query_id``, and those
418
+ rows are written by an enqueue that is deliberately off (it costs twenty
419
+ rows per query and inflated the ranking-phase counter 2,675x). With no
420
+ evidence to look for, every settlement fell through to the 120-second
421
+ neutral default of 0.5, which is why 165 arms all read alpha == beta.
422
+
423
+ So the play carries its own evidence. Best-effort by design: the reward
424
+ is a nicety and a recall is not. A failure here means this one play
425
+ settles as ``default``, exactly as before.
426
+
427
+ Bounded to the first few ids because the settler only ever reads the
428
+ top of the list, and an unbounded JSON blob on the recall path is a
429
+ write cost with no reader.
430
+ """
431
+ if not play_id:
432
+ return False
433
+ ids = [str(f) for f in list(fact_ids)[:_SHOWN_FACT_LIMIT] if f]
434
+ if not ids:
435
+ return False
436
+ try:
437
+ conn = _conn_for(self._db_path)
438
+ conn.execute(
439
+ "UPDATE bandit_plays SET shown_fact_ids = ? WHERE play_id = ?",
440
+ (json.dumps(ids), int(play_id)),
441
+ )
442
+ return True
443
+ except sqlite3.Error as exc:
444
+ # Includes "no such column" on a store where M044 has not run.
445
+ logger.debug("bandit.record_shown: %s", exc)
446
+ return False
447
+
405
448
  # ------------------------------------------------------------------
406
449
  # snapshot (for dashboard — LLD-04 consumer)
407
450
  # ------------------------------------------------------------------
@@ -96,6 +96,41 @@ __all__ = (
96
96
  # ---------------------------------------------------------------------------
97
97
 
98
98
 
99
+ #: Distinct informative labels required before a retrain is worth running.
100
+ #:
101
+ #: Measured on a live production store: ``learning_features`` holds 5,352 rows, of
102
+ #: which **5,350 carry label 0.0** and exactly two are non-zero (0.6 and 1.0) —
103
+ #: the two real feedback events in the whole database. The active model was
104
+ #: trained on 972 such rows and reorders results at random; the heuristic it
105
+ #: displaced was better.
106
+ #:
107
+ #: An earlier design proposed gating on ``len(set(labels)) <= 1``. That check PASSES
108
+ #: here — three distinct values exist — and would train on a 2,675:1 imbalance
109
+ #: while reporting success, which is worse than not training at all because it
110
+ #: looks like progress. A count of informative labels is the honest gate.
111
+ MIN_INFORMATIVE_LABELS = 50
112
+
113
+
114
+ def _count_informative_labels(rows: list[dict]) -> int:
115
+ """Rows whose outcome says something other than "nothing happened".
116
+
117
+ A reward of exactly 0.5 is the neutral default the settlement path applies
118
+ when no outcome was reported, and ``None`` means the outcome column is
119
+ absent. Neither is evidence. Everything else is.
120
+ """
121
+ seen = 0
122
+ for row in rows:
123
+ reward = row.get("outcome_reward")
124
+ if reward is None:
125
+ continue
126
+ try:
127
+ value = float(reward)
128
+ except (TypeError, ValueError):
129
+ continue
130
+ if abs(value - 0.5) > 1e-9:
131
+ seen += 1
132
+ return seen
133
+
99
134
  def _run_shadow_cycle(
100
135
  *,
101
136
  memory_db_path: str,
@@ -137,6 +172,25 @@ def _run_shadow_cycle(
137
172
  out["aborted"] = "insufficient_data"
138
173
  return out
139
174
 
175
+ informative = _count_informative_labels(rows)
176
+ if informative < MIN_INFORMATIVE_LABELS:
177
+ # Honest refusal. See _count_informative_labels for why a row count is
178
+ # not a substitute for this, and why the answer must not be to lower
179
+ # the threshold until training starts.
180
+ logger.info(
181
+ "retrain skipped: %d informative label(s) of %d rows "
182
+ "(need %d). Rewards accumulate from reported outcomes; there is "
183
+ "nothing to learn from a corpus of one label.",
184
+ informative, len(rows), MIN_INFORMATIVE_LABELS,
185
+ )
186
+ out["aborted"] = "insufficient_label_signal"
187
+ out["metrics"] = {
188
+ "rows": len(rows),
189
+ "informative_labels": informative,
190
+ "required": MIN_INFORMATIVE_LABELS,
191
+ }
192
+ return out
193
+
140
194
  # Load prior active for in-sample shadow.
141
195
  try:
142
196
  from superlocalmemory.learning.database import LearningDatabase
@@ -72,6 +72,10 @@ CREATE INDEX IF NOT EXISTS idx_engagement_profile_metric
72
72
  """
73
73
 
74
74
 
75
+
76
+ from superlocalmemory.learning.signal_kinds import FEEDBACK_ONLY_SQL
77
+
78
+
75
79
  class LearningDatabase:
76
80
  """Persistent storage for the adaptive ranker's training pipeline.
77
81
 
@@ -182,7 +186,8 @@ class LearningDatabase:
182
186
  conn = self._connect()
183
187
  try:
184
188
  row = conn.execute(
185
- "SELECT COUNT(*) AS cnt FROM learning_signals WHERE profile_id = ?",
189
+ "SELECT COUNT(*) AS cnt FROM learning_signals "
190
+ f"WHERE profile_id = ?{FEEDBACK_ONLY_SQL}",
186
191
  (profile_id,),
187
192
  ).fetchone()
188
193
  return int(row["cnt"]) if row else 0
@@ -364,15 +369,22 @@ class LearningDatabase:
364
369
  # ------------------------------------------------------------------
365
370
 
366
371
  def count_signals(self, profile_id: str) -> int:
367
- """Count ``learning_signals`` rows for ``profile_id``.
372
+ """Count FEEDBACK rows in ``learning_signals`` for ``profile_id``.
368
373
 
369
374
  Used by ``_compute_ranker_phase`` + consolidation_worker training
370
375
  gate. Pure SELECT — thread-safe without lock.
376
+
377
+ Exposure rows are excluded. This counted every row until 4.1.0, which
378
+ on a live store meant 5,352 instead of 2 — a 2,675x inflation that
379
+ held the ranker in Phase 3 on two feedback events. See
380
+ ``learning/signal_kinds.py`` for why the predicate excludes exposures
381
+ rather than naming feedback kinds.
371
382
  """
372
383
  conn = self._connect()
373
384
  try:
374
385
  row = conn.execute(
375
- "SELECT COUNT(*) AS cnt FROM learning_signals WHERE profile_id = ?",
386
+ "SELECT COUNT(*) AS cnt FROM learning_signals "
387
+ f"WHERE profile_id = ?{FEEDBACK_ONLY_SQL}",
376
388
  (profile_id,),
377
389
  ).fetchone()
378
390
  return int(row["cnt"]) if row else 0
@@ -608,6 +620,19 @@ class LearningDatabase:
608
620
  "learning_features",
609
621
  "learning_model_state",
610
622
  "engagement_metrics",
623
+ # bandit_plays holds shown_fact_ids — the identifiers of the
624
+ # memories a person was actually shown — and bandit_arms is
625
+ # a derived behavioural profile. Both are personal data and
626
+ # neither was erased: this list was hardcoded, so
627
+ # forget_profile() reported success and left a row reading
628
+ # ["alice-private-memory-1", ...] in place. Reproduced
629
+ # before fixing.
630
+ #
631
+ # The retention sweep is not a substitute. It deletes only
632
+ # SETTLED plays older than the horizon, so an unsettled row
633
+ # would have survived indefinitely.
634
+ "bandit_plays",
635
+ "bandit_arms",
611
636
  ]
612
637
  receipt_tables = {
613
638
  row[0]
@@ -619,7 +644,17 @@ class LearningDatabase:
619
644
  }
620
645
  if profile_id is None:
621
646
  tables.extend(sorted(receipt_tables))
647
+ present = {
648
+ row[0] for row in conn.execute(
649
+ "SELECT name FROM sqlite_master WHERE type='table'"
650
+ )
651
+ }
622
652
  for table in tables:
653
+ # A store predating any of these tables must not abort the
654
+ # erasure of the rest — an Article 17 request that fails
655
+ # halfway is worse than one that skips an absent table.
656
+ if table not in present:
657
+ continue
623
658
  if profile_id:
624
659
  conn.execute(
625
660
  f"DELETE FROM {table} WHERE profile_id = ?",
@@ -628,6 +663,28 @@ class LearningDatabase:
628
663
  else:
629
664
  conn.execute(f"DELETE FROM {table}")
630
665
  conn.commit()
666
+ # The in-process ranking counter holds this profile's recent
667
+ # winners. Ephemeral, but an erasure must not leave them in a
668
+ # live process for the rest of its lifetime.
669
+ try:
670
+ from superlocalmemory.learning.pcos import RECENT_TOPS
671
+
672
+ RECENT_TOPS.forget(profile_id or "")
673
+ except Exception: # pragma: no cover — advisory
674
+ pass
675
+ # The per-session working set is this profile's data too.
676
+ # Cleared here as well as in the compliance path, because this
677
+ # is reached directly — by the transaction owners and by tests —
678
+ # and a residue left behind goes on biasing any later session
679
+ # that reuses one of the erased profile's session ids.
680
+ try:
681
+ from superlocalmemory.core.working_memory import (
682
+ discard_profile,
683
+ )
684
+
685
+ discard_profile(profile_id or "")
686
+ except Exception: # pragma: no cover — never block an erasure
687
+ pass
631
688
  logger.info(
632
689
  "Learning data reset%s",
633
690
  f" for profile {profile_id}" if profile_id else " (all)",
@@ -170,23 +170,27 @@ class EntityCompiler:
170
170
  if not facts:
171
171
  return None
172
172
 
173
- has_pagerank = any(f["pagerank_score"] is not None for f in facts)
174
-
175
- # ── Phase 2: PageRank (short write, NO Ollama) ────────────────────────
176
- if not has_pagerank and len(facts) > 2:
177
- self._compute_pagerank([f["fact_id"] for f in facts], profile_id)
178
- # Re-fetch with updated scores
179
- with memory_read(self._db_path) as conn:
180
- facts = conn.execute("""
181
- SELECT af.fact_id, af.content, af.confidence, af.created_at,
182
- fi.pagerank_score, fi.community_id
183
- FROM atomic_facts af
184
- LEFT JOIN fact_importance fi ON af.fact_id = fi.fact_id
185
- WHERE af.canonical_entities_json LIKE ? AND af.profile_id = ?
186
- ORDER BY fi.pagerank_score DESC NULLS LAST, af.confidence DESC
187
- LIMIT 50
188
- """, (f"%{entity_id}%", profile_id)).fetchall()
189
- facts = [dict(f) for f in facts]
173
+ # There was a second PageRank here, and it made recall worse.
174
+ #
175
+ # On finding no score for this entity's facts it built a COMPLETE graph
176
+ # over them -- every pair joined, weight 0.5 -- and ran PageRank on it.
177
+ # PageRank of K_n is uniformly 1/n, so every fact received the identical
178
+ # score and the re-fetch's ORDER BY fell straight through to confidence,
179
+ # which is what had ordered them in the first place. It bought nothing.
180
+ #
181
+ # What it cost: that 1/n landed in the same column the ranker reads as a
182
+ # whole-graph score. On the author's store, ten facts from one 10-fact
183
+ # cluster were written 0.1 each, against a real whole-graph maximum of
184
+ # 0.008744 and median of 0.000214 -- eleven times the largest true score
185
+ # and roughly 470x the median. The hop boost min(1 + pr*2, 2) gave them
186
+ # 1.2 where every real memory got 1.0004, and the ten of them together
187
+ # carried as much PageRank mass as the other 2,988 facts combined (the
188
+ # table summed to 1.9999 instead of 1). They were also written with no
189
+ # community, so the community bias could not see them either.
190
+ #
191
+ # Whole-graph metrics belong to core/graph_metrics, which owns this
192
+ # table. Ordering for compilation is confidence, as it always effectively
193
+ # was.
190
194
 
191
195
  # ── Phase 3: generate compiled truth — NO write lock held ────────────
192
196
  # Mode B calls Ollama (up to 30 s) — write lock MUST NOT be held here.
@@ -340,47 +344,6 @@ class EntityCompiler:
340
344
 
341
345
  # -- Helpers --
342
346
 
343
- def _compute_pagerank(self, fact_ids: list[str], profile_id: str) -> None:
344
- """Compute PageRank for a set of facts and store in fact_importance.
345
-
346
- Concurrency fix (v3.8.4): uses memory_write() for the INSERT so the
347
- process write lock is acquired and busy_timeout is set correctly.
348
- Pure PageRank computation (networkx) happens BEFORE the write lock.
349
- """
350
- from superlocalmemory.storage.memory_write import memory_write
351
-
352
- try:
353
- import networkx as nx
354
- G = nx.Graph()
355
- for fid in fact_ids:
356
- G.add_node(fid)
357
- for i, fid1 in enumerate(fact_ids):
358
- for fid2 in fact_ids[i + 1:]:
359
- G.add_edge(fid1, fid2, weight=0.5)
360
-
361
- if len(G.nodes) < 2:
362
- return
363
-
364
- # Compute scores in pure Python — no lock held yet.
365
- scores = nx.pagerank(G, alpha=0.85)
366
- now = datetime.now(timezone.utc).isoformat()
367
-
368
- # Short write: lock held only for the INSERT batch.
369
- with memory_write(self._db_path) as conn:
370
- for fid, score in scores.items():
371
- conn.execute("""
372
- INSERT INTO fact_importance
373
- (fact_id, profile_id, pagerank_score, computed_at)
374
- VALUES (?, ?, ?, ?)
375
- ON CONFLICT(fact_id) DO UPDATE
376
- SET pagerank_score = excluded.pagerank_score,
377
- computed_at = excluded.computed_at
378
- """, (fid, profile_id, round(score, 6), now))
379
- except ImportError:
380
- logger.debug("NetworkX not available — skipping PageRank")
381
- except Exception as exc:
382
- logger.debug("PageRank computation failed: %s", exc)
383
-
384
347
  @staticmethod
385
348
  def _truncate(text: str, max_chars: int) -> str:
386
349
  """Truncate at sentence boundary within char limit."""
@@ -49,6 +49,7 @@ from dataclasses import dataclass
49
49
  from datetime import datetime, timezone
50
50
  from pathlib import Path
51
51
  from typing import Any, Dict, List, Optional
52
+ from superlocalmemory.learning.signal_kinds import FEEDBACK_ONLY_SQL
52
53
 
53
54
  logger = logging.getLogger("superlocalmemory.learning.feedback")
54
55
 
@@ -572,7 +573,8 @@ class FeedbackCollector:
572
573
  conn = self._connect()
573
574
  try:
574
575
  row = conn.execute(
575
- "SELECT COUNT(*) FROM learning_signals WHERE profile_id = ?",
576
+ "SELECT COUNT(*) FROM learning_signals "
577
+ f"WHERE profile_id = ?{FEEDBACK_ONLY_SQL}",
576
578
  (profile_id,),
577
579
  ).fetchone()
578
580
  return row[0] if row else 0