superlocalmemory 3.8.13 → 4.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (212) hide show
  1. package/ATTRIBUTION.md +4 -4
  2. package/CHANGELOG.md +113 -121
  3. package/README.md +65 -63
  4. package/docs/pi-dev-integration.md +1 -1
  5. package/package.json +6 -1
  6. package/plugin/.claude-plugin/plugin.json +1 -1
  7. package/plugin/CLAUDE.md +3 -3
  8. package/plugin/agents/slm-governance-advisor.md +1 -1
  9. package/plugin/agents/slm-loop-runner.md +1 -1
  10. package/plugin/agents/slm-memory-advisor.md +1 -1
  11. package/plugin/agents/slm-optimize-advisor.md +1 -1
  12. package/plugin/requirements.txt +1 -1
  13. package/plugin/skills/slm-cache/SKILL.md +1 -1
  14. package/plugin/skills/slm-compress/SKILL.md +1 -1
  15. package/plugin/skills/slm-governance/SKILL.md +1 -1
  16. package/plugin/skills/slm-graph/SKILL.md +1 -1
  17. package/plugin/skills/slm-loop/SKILL.md +1 -1
  18. package/plugin/skills/slm-mesh/SKILL.md +1 -1
  19. package/plugin/skills/slm-profile/SKILL.md +1 -1
  20. package/plugin/skills/slm-recall/SKILL.md +1 -1
  21. package/plugin/skills/slm-remember/SKILL.md +1 -1
  22. package/plugin/skills/slm-scope/SKILL.md +1 -1
  23. package/plugin/skills/slm-session/SKILL.md +1 -1
  24. package/plugin/skills/slm-status/SKILL.md +1 -1
  25. package/plugin-src/rules/AGENTS.md +1 -1
  26. package/plugin-src/skills/slm-cache/SKILL.md +1 -1
  27. package/plugin-src/skills/slm-compress/SKILL.md +1 -1
  28. package/plugin-src/skills/slm-governance/SKILL.md +248 -0
  29. package/plugin-src/skills/slm-graph/SKILL.md +1 -1
  30. package/plugin-src/skills/slm-loop/SKILL.md +99 -0
  31. package/plugin-src/skills/slm-mesh/SKILL.md +282 -0
  32. package/plugin-src/skills/slm-profile/SKILL.md +148 -0
  33. package/plugin-src/skills/slm-recall/SKILL.md +1 -1
  34. package/plugin-src/skills/slm-remember/SKILL.md +1 -1
  35. package/plugin-src/skills/slm-scope/SKILL.md +176 -0
  36. package/plugin-src/skills/slm-session/SKILL.md +1 -1
  37. package/plugin-src/skills/slm-status/SKILL.md +1 -1
  38. package/pyproject.toml +11 -4
  39. package/src/superlocalmemory/__init__.py +1 -1
  40. package/src/superlocalmemory/cli/commands.py +125 -11
  41. package/src/superlocalmemory/cli/daemon.py +5 -1
  42. package/src/superlocalmemory/cli/main.py +35 -2
  43. package/src/superlocalmemory/cli/ops_cmd.py +281 -0
  44. package/src/superlocalmemory/cli/setup_wizard.py +1 -1
  45. package/src/superlocalmemory/compliance/audit.py +65 -0
  46. package/src/superlocalmemory/compliance/eu_ai_act.py +27 -57
  47. package/src/superlocalmemory/compliance/gdpr.py +416 -20
  48. package/src/superlocalmemory/compliance/retention.py +74 -22
  49. package/src/superlocalmemory/compliance/scheduler.py +78 -9
  50. package/src/superlocalmemory/core/actor_context.py +166 -0
  51. package/src/superlocalmemory/core/admission.py +549 -0
  52. package/src/superlocalmemory/core/backend_orchestrator.py +23 -10
  53. package/src/superlocalmemory/core/config.py +202 -24
  54. package/src/superlocalmemory/core/consolidation_engine.py +13 -13
  55. package/src/superlocalmemory/core/context_cache.py +28 -0
  56. package/src/superlocalmemory/core/embeddings.py +64 -2
  57. package/src/superlocalmemory/core/engine.py +7 -2
  58. package/src/superlocalmemory/core/engine_ingestion.py +65 -3
  59. package/src/superlocalmemory/core/engine_wiring.py +36 -9
  60. package/src/superlocalmemory/core/ingest_policy.py +38 -0
  61. package/src/superlocalmemory/core/maintenance.py +255 -0
  62. package/src/superlocalmemory/core/modes.py +40 -13
  63. package/src/superlocalmemory/core/mutations.py +437 -44
  64. package/src/superlocalmemory/core/operation_policy.py +92 -0
  65. package/src/superlocalmemory/core/operation_policy_registry.py +542 -0
  66. package/src/superlocalmemory/core/operation_request.py +127 -0
  67. package/src/superlocalmemory/core/ops_remediation.py +542 -0
  68. package/src/superlocalmemory/core/recall_pipeline.py +7 -0
  69. package/src/superlocalmemory/core/remember_runtime.py +202 -4
  70. package/src/superlocalmemory/core/remote_mode.py +20 -5
  71. package/src/superlocalmemory/core/store_pipeline.py +150 -0
  72. package/src/superlocalmemory/core/topic_signature.py +19 -4
  73. package/src/superlocalmemory/core/transactions/__init__.py +78 -0
  74. package/src/superlocalmemory/core/transactions/concrete_owners.py +597 -0
  75. package/src/superlocalmemory/core/transactions/erasure.py +825 -0
  76. package/src/superlocalmemory/core/transactions/manifest.py +255 -0
  77. package/src/superlocalmemory/core/transactions/manifest_key.py +155 -0
  78. package/src/superlocalmemory/core/transactions/obligations.py +272 -0
  79. package/src/superlocalmemory/core/transactions/owners.py +114 -0
  80. package/src/superlocalmemory/core/transactions/reconciler.py +285 -0
  81. package/src/superlocalmemory/core/transactions/service.py +330 -0
  82. package/src/superlocalmemory/core/worker_pool.py +33 -5
  83. package/src/superlocalmemory/encoding/cognitive_consolidator.py +70 -28
  84. package/src/superlocalmemory/encoding/emotional.py +75 -14
  85. package/src/superlocalmemory/encoding/scene_builder.py +115 -13
  86. package/src/superlocalmemory/encoding/temporal_parser.py +4 -0
  87. package/src/superlocalmemory/evolution/blind_verifier.py +11 -4
  88. package/src/superlocalmemory/evolution/evolution_store.py +244 -4
  89. package/src/superlocalmemory/evolution/llm_dispatch.py +40 -0
  90. package/src/superlocalmemory/evolution/model_selection.py +18 -3
  91. package/src/superlocalmemory/evolution/mutation_generator.py +3 -0
  92. package/src/superlocalmemory/evolution/skill_activator.py +270 -0
  93. package/src/superlocalmemory/evolution/skill_evolver.py +281 -59
  94. package/src/superlocalmemory/evolution/types.py +30 -8
  95. package/src/superlocalmemory/graph/cozo_backend.py +17 -9
  96. package/src/superlocalmemory/hooks/auto_invoker.py +2 -1
  97. package/src/superlocalmemory/hooks/auto_recall.py +64 -30
  98. package/src/superlocalmemory/hooks/codex_assets.py +14 -1
  99. package/src/superlocalmemory/infra/backup.py +434 -7
  100. package/src/superlocalmemory/infra/process_reaper.py +18 -0
  101. package/src/superlocalmemory/infra/self_heal.py +401 -0
  102. package/src/superlocalmemory/learning/feedback.py +52 -9
  103. package/src/superlocalmemory/loops/engine.py +10 -0
  104. package/src/superlocalmemory/mcp/_daemon_proxy.py +3 -0
  105. package/src/superlocalmemory/mcp/http_transport.py +30 -331
  106. package/src/superlocalmemory/mcp/profiles.py +5 -0
  107. package/src/superlocalmemory/mcp/resources.py +8 -0
  108. package/src/superlocalmemory/mcp/server.py +51 -4
  109. package/src/superlocalmemory/mcp/shared.py +19 -0
  110. package/src/superlocalmemory/mcp/tools_active.py +25 -4
  111. package/src/superlocalmemory/mcp/tools_code_graph.py +26 -18
  112. package/src/superlocalmemory/mcp/tools_context.py +50 -8
  113. package/src/superlocalmemory/mcp/tools_core.py +69 -21
  114. package/src/superlocalmemory/mcp/tools_evolution.py +9 -2
  115. package/src/superlocalmemory/mcp/tools_learning.py +21 -10
  116. package/src/superlocalmemory/mcp/tools_loops.py +29 -18
  117. package/src/superlocalmemory/mcp/tools_mesh.py +8 -0
  118. package/src/superlocalmemory/mcp/tools_ops.py +115 -0
  119. package/src/superlocalmemory/mcp/tools_optimize.py +4 -0
  120. package/src/superlocalmemory/mcp/tools_v28.py +10 -3
  121. package/src/superlocalmemory/mcp/tools_v3.py +34 -14
  122. package/src/superlocalmemory/mcp/tools_v33.py +18 -33
  123. package/src/superlocalmemory/mesh/broker.py +124 -46
  124. package/src/superlocalmemory/mesh/broker_security.py +470 -0
  125. package/src/superlocalmemory/mesh/discovery.py +365 -0
  126. package/src/superlocalmemory/mesh/lock_protocol.py +313 -0
  127. package/src/superlocalmemory/mesh/node_identity.py +97 -0
  128. package/src/superlocalmemory/mesh/outbox_remote.py +429 -0
  129. package/src/superlocalmemory/mesh/remote_sync.py +511 -28
  130. package/src/superlocalmemory/mesh/state_sync.py +286 -0
  131. package/src/superlocalmemory/optimize/config/store.py +45 -0
  132. package/src/superlocalmemory/parameterization/cross_project.py +12 -0
  133. package/src/superlocalmemory/parameterization/prompt_injector.py +13 -11
  134. package/src/superlocalmemory/parameterization/prompt_lifecycle.py +8 -2
  135. package/src/superlocalmemory/parameterization/workflow_miner.py +17 -0
  136. package/src/superlocalmemory/retrieval/ann_index.py +5 -0
  137. package/src/superlocalmemory/retrieval/bm25_channel.py +49 -2
  138. package/src/superlocalmemory/retrieval/engine.py +19 -4
  139. package/src/superlocalmemory/retrieval/fusion.py +4 -1
  140. package/src/superlocalmemory/retrieval/hopfield_channel.py +9 -3
  141. package/src/superlocalmemory/retrieval/remote_reranker.py +47 -22
  142. package/src/superlocalmemory/retrieval/reranker.py +32 -1
  143. package/src/superlocalmemory/retrieval/temporal_channel.py +16 -3
  144. package/src/superlocalmemory/retrieval/temporal_utils.py +107 -0
  145. package/src/superlocalmemory/retrieval/temporal_validity_filter.py +155 -42
  146. package/src/superlocalmemory/retrieval/vector_store.py +214 -8
  147. package/src/superlocalmemory/server/api.py +5 -5
  148. package/src/superlocalmemory/server/egress_policy.py +258 -0
  149. package/src/superlocalmemory/server/rbac_enforce.py +32 -0
  150. package/src/superlocalmemory/server/route_mutations.py +20 -0
  151. package/src/superlocalmemory/server/routes/compliance.py +153 -7
  152. package/src/superlocalmemory/server/routes/data_io.py +43 -2
  153. package/src/superlocalmemory/server/routes/events.py +15 -0
  154. package/src/superlocalmemory/server/routes/memories.py +56 -3
  155. package/src/superlocalmemory/server/routes/mesh.py +82 -1
  156. package/src/superlocalmemory/server/routes/mesh_lock.py +54 -0
  157. package/src/superlocalmemory/server/routes/mesh_state.py +63 -0
  158. package/src/superlocalmemory/server/routes/v3_api.py +50 -27
  159. package/src/superlocalmemory/server/routes/ws.py +86 -0
  160. package/src/superlocalmemory/server/ui.py +6 -6
  161. package/src/superlocalmemory/server/unified_daemon.py +942 -119
  162. package/src/superlocalmemory/storage/_migration_internals.py +568 -0
  163. package/src/superlocalmemory/storage/_schema_version.py +110 -0
  164. package/src/superlocalmemory/storage/database.py +329 -24
  165. package/src/superlocalmemory/storage/embedding_migrator.py +246 -51
  166. package/src/superlocalmemory/storage/erasure_fence.py +45 -0
  167. package/src/superlocalmemory/storage/generation_fence.py +63 -0
  168. package/src/superlocalmemory/storage/migration_runner.py +140 -417
  169. package/src/superlocalmemory/storage/migrations/M009_model_lineage.py +40 -0
  170. package/src/superlocalmemory/storage/migrations/M033_projection_transactions.py +148 -0
  171. package/src/superlocalmemory/storage/migrations/M034_obligation_integrity.py +58 -0
  172. package/src/superlocalmemory/storage/migrations/M035_erasure_receipts.py +113 -0
  173. package/src/superlocalmemory/storage/migrations/M036_vector_row_map.py +107 -0
  174. package/src/superlocalmemory/storage/migrations/M037_manifest_hmac_version.py +162 -0
  175. package/src/superlocalmemory/storage/migrations/{M033_learning_feedback_channel.py → M038_learning_feedback_channel.py} +3 -3
  176. package/src/superlocalmemory/storage/migrations/M039_scene_fact_members.py +137 -0
  177. package/src/superlocalmemory/storage/migrations/__init__.py +4 -2
  178. package/src/superlocalmemory/storage/schema.py +67 -0
  179. package/src/superlocalmemory/storage/write_coordinator.py +125 -0
  180. package/src/superlocalmemory/trust/scorer.py +28 -4
  181. package/src/superlocalmemory/ui/index.html +14 -3
  182. package/src/superlocalmemory/ui/js/auto-settings.js +12 -1
  183. package/src/superlocalmemory/ui/js/brain.js +6 -4
  184. package/src/superlocalmemory/ui/js/compliance.js +66 -12
  185. package/src/superlocalmemory/ui/js/dashboard.js +13 -3
  186. package/src/superlocalmemory/ui/js/feedback.js +8 -2
  187. package/src/superlocalmemory/ui/js/lifecycle.js +7 -1
  188. package/src/superlocalmemory/ui/js/modal.js +272 -5
  189. package/src/superlocalmemory/ui/js/od-backup.js +9 -2
  190. package/src/superlocalmemory/ui/js/od-compliance-ext.js +301 -0
  191. package/src/superlocalmemory/ui/js/od-operations.js +154 -23
  192. package/src/superlocalmemory/ui/js/od-ops-health.js +417 -0
  193. package/src/superlocalmemory/ui/js/od-optimize.js +35 -21
  194. package/src/superlocalmemory/ui/js/od-team.js +9 -2
  195. package/src/superlocalmemory/ui/js/optimize.js +13 -16
  196. package/src/superlocalmemory/ui/js/profiles.js +7 -3
  197. package/src/superlocalmemory/ui/js/settings.js +7 -1
  198. package/src/superlocalmemory/vector/lancedb_backend.py +19 -9
  199. package/src/superlocalmemory/attribution/mathematical_dna.py +0 -235
  200. package/src/superlocalmemory/cli/post_install.py +0 -114
  201. package/src/superlocalmemory/core/clock_monitor.py +0 -45
  202. package/src/superlocalmemory/core/db_pool.py +0 -80
  203. package/src/superlocalmemory/core/error_catalog.py +0 -113
  204. package/src/superlocalmemory/core/loop_watchdog.py +0 -56
  205. package/src/superlocalmemory/core/priority_queue.py +0 -61
  206. package/src/superlocalmemory/core/pruning_engine.py +0 -216
  207. package/src/superlocalmemory/core/queue_dispatcher.py +0 -73
  208. package/src/superlocalmemory/core/slmignore.py +0 -125
  209. package/src/superlocalmemory/infra/heartbeat_monitor.py +0 -140
  210. package/src/superlocalmemory/infra/webhook_dispatcher.py +0 -247
  211. package/src/superlocalmemory/learning/quantization_scheduler.py +0 -320
  212. package/src/superlocalmemory/storage/access_control.py +0 -182
@@ -2,30 +2,55 @@
2
2
  # Licensed under AGPL-3.0-or-later - see LICENSE file
3
3
  # Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
4
4
 
5
- """Bi-temporal validity filter for the retrieval pipeline (Phase 4, T1).
6
-
7
- Post-retrieval filter for *system-invalidated* facts a fact whose temporal
8
- record has ``system_expired_at`` set was superseded/contradicted by a newer
9
- fact (see ``invalidate_fact_temporal`` / conflict-resolution supersession).
10
-
11
- P5-INT-01 (non-destructive supersession): such a fact is DEMOTED, not hidden.
12
- Its per-channel score is multiplied by ``superseded_demotion_factor`` (default
13
- 0.25) and the channel lists are re-sorted, so currently-valid facts rank above
14
- it but nothing valid silently vanishes. This is the Mem0-2026 design that
15
- wins long-term-memory benchmarks: keep every fact recallable and let
16
- retrieval-time recency resolve conflicts, rather than destructively deleting on
17
- a write-time contradiction guess (which over-fires: two complementary facts
18
- about the same entity diverge past the coboundary threshold and one would be
19
- wrongly hidden). A factor of 0.0 restores the legacy hide behaviour (a demoted
20
- score of 0 is gated out by the evidence floor).
21
-
22
- The filter runs on the per-channel candidate dict BEFORE RRF fusion, so fused
23
- ranks reflect the demotion. It queries validity only for the bounded candidate
24
- set (never the full ``get_valid_facts`` set) — an indexed, O(candidates) lookup
25
- on the hot path, no full-table scan.
26
-
27
- Pure SQL, no LLM safe in every mode including Mode A. A no-op when nothing is
28
- invalidated and when config.enabled is False.
5
+ """Bi-temporal validity filter for the retrieval pipeline (Phase 4, T1 + T1b).
6
+
7
+ Post-retrieval filter that demotes facts whose bi-temporal validity window
8
+ makes them ineligible for the current recall point in time. Two independent
9
+ demotion axes:
10
+
11
+ **Axis 1 — Transaction-time supersession (P5-INT-01, existing):**
12
+ A fact whose temporal record has ``system_expired_at`` set was superseded or
13
+ contradicted by a newer fact (see ``invalidate_fact_temporal`` /
14
+ conflict-resolution supersession). Its per-channel score is multiplied by
15
+ ``superseded_demotion_factor`` (default 0.25).
16
+
17
+ **Axis 2 — Event-time expiry (Phase 4 T1b, new):**
18
+ A fact whose ``valid_until`` has passed (in wall-clock time) was true in the
19
+ real world only up to that date. Its per-channel score is multiplied by
20
+ ``event_time_demotion_factor`` (default 0.5, softer than supersession). A
21
+ separate "not-yet-valid" demotion applies when ``as_of`` is set for
22
+ point-in-time time-travel recall: facts whose ``valid_from > as_of`` are
23
+ also demoted.
24
+
25
+ **Zero-regression guarantee (default path):**
26
+ Almost all existing facts have ``valid_until = NULL`` (open-ended / still
27
+ valid). The event-time expiry lookup therefore returns an empty set on the
28
+ default path, so no scores change and default recall output is identical.
29
+ Additionally, when ``include_expired_in_history = True`` (the config default)
30
+ AND no explicit ``as_of`` is requested, event-time demotion is skipped
31
+ entirely — the guard ensures historical recall modes that intentionally want
32
+ to surface expired facts are unaffected.
33
+
34
+ **As-of time-travel:**
35
+ When ``as_of`` is packed into the filter context dict (``{"as_of": "..."}``),
36
+ event-time demotion is always applied regardless of
37
+ ``include_expired_in_history``, because the caller explicitly requested a
38
+ point-in-time view. Facts not yet valid at ``as_of`` (``valid_from > as_of``)
39
+ and facts already expired at ``as_of`` (``valid_until <= as_of``, half-open) are
40
+ both demoted.
41
+
42
+ **Demotion priority:**
43
+ - System-invalidated facts: score × ``superseded_demotion_factor`` (0.25).
44
+ - Event-time-expired only (not system-invalidated): score × ``event_time_demotion_factor`` (0.5).
45
+ - Both: system-invalidated wins (the harder demotion, 0.25, is applied; event-time
46
+ demotion is not stacked to avoid excessive double-penalty).
47
+
48
+ All demotions are non-destructive (P5-INT-01): facts stay in the candidate
49
+ list but rank below valid facts. A factor of 0.0 restores the legacy hide
50
+ behaviour (a score of zero is gated out by the evidence floor).
51
+
52
+ Both lookups are bounded (candidate ids only), chunked, indexed, and
53
+ fail-open: a DB error returns results unchanged.
29
54
 
30
55
  Integrates with ChannelRegistry.register_filter() using the FilterFn signature:
31
56
  (all_channel_results, profile_id, context) -> filtered_results
@@ -46,17 +71,43 @@ if TYPE_CHECKING:
46
71
 
47
72
  logger = logging.getLogger(__name__)
48
73
 
74
+ # Module-level fallback for event-time demotion factor.
75
+ # Used when the config object does not yet carry ``event_time_demotion_factor``
76
+ # (e.g. older serialised configs loaded from disk). Softer than the supersession
77
+ # factor (0.25) because event-time expiry is a softer signal — the boundary may
78
+ # be approximate or the fact may still be conceptually relevant.
79
+ _EVENT_TIME_DEMOTION_FACTOR: float = 0.5
80
+
49
81
 
50
82
  class TemporalValidityFilter:
51
- """Demotes system-invalidated (superseded) facts in retrieval candidates."""
83
+ """Demotes bi-temporally invalid facts in retrieval candidates.
84
+
85
+ Two demotion axes:
86
+ - Axis 1: system-invalidated (superseded) facts — existing behaviour.
87
+ - Axis 2: event-time-expired facts — new in Phase 4 T1b.
88
+
89
+ Both are non-destructive: facts stay recallable but rank below valid ones.
90
+ """
52
91
 
53
- __slots__ = ("_db", "_demotion_factor")
92
+ __slots__ = (
93
+ "_db",
94
+ "_demotion_factor",
95
+ "_event_time_factor",
96
+ "_include_expired_in_history",
97
+ )
54
98
 
55
- def __init__(self, db: DatabaseManager, demotion_factor: float = 0.25) -> None:
99
+ def __init__(
100
+ self,
101
+ db: DatabaseManager,
102
+ demotion_factor: float = 0.25,
103
+ event_time_factor: float = _EVENT_TIME_DEMOTION_FACTOR,
104
+ include_expired_in_history: bool = True,
105
+ ) -> None:
56
106
  self._db = db
57
- # Clamp to [0, 1]. 0.0 = legacy hide (evidence floor drops zero-score
58
- # facts); 1.0 = no demotion.
107
+ # Clamp to [0, 1]. 0.0 = legacy hide; 1.0 = no demotion.
59
108
  self._demotion_factor = max(0.0, min(1.0, float(demotion_factor)))
109
+ self._event_time_factor = max(0.0, min(1.0, float(event_time_factor)))
110
+ self._include_expired_in_history = bool(include_expired_in_history)
60
111
 
61
112
  def filter(
62
113
  self,
@@ -64,20 +115,23 @@ class TemporalValidityFilter:
64
115
  profile_id: str,
65
116
  context: Any,
66
117
  ) -> dict[str, list[tuple[str, float]]]:
67
- """Demote superseded fact_ids in every channel's candidate list.
118
+ """Demote bi-temporally invalid fact_ids in every channel's candidate list.
68
119
 
69
120
  Matches FilterFn signature from channel_registry.py.
70
121
 
71
122
  Args:
72
123
  all_results: Channel name -> [(fact_id, score)] dict.
73
124
  profile_id: Current profile.
74
- context: Optional context (unused).
125
+ context: Optional dict that may carry ``as_of`` (ISO 8601 string)
126
+ for point-in-time time-travel recall. None or any non-dict value
127
+ means "current time, no time-travel" — the default path.
75
128
 
76
129
  Returns:
77
- A new dict where system-invalidated facts keep their channel
78
- presence but have their score scaled by the demotion factor and the
79
- channel lists re-sorted (so valid facts rank above them). Inputs are
80
- never mutated (immutability). Unchanged when nothing is invalidated.
130
+ A new dict where invalid facts keep their channel presence but have
131
+ their score scaled by the appropriate demotion factor and the channel
132
+ lists re-sorted so valid facts rank above them. Inputs are never
133
+ mutated (immutability contract). Returns the input unchanged when
134
+ nothing is invalid (fast path).
81
135
  """
82
136
  # Collect all unique candidate fact_ids across every channel.
83
137
  all_fact_ids: set[str] = set()
@@ -88,27 +142,73 @@ class TemporalValidityFilter:
88
142
  if not all_fact_ids:
89
143
  return all_results
90
144
 
145
+ # Extract as_of FIRST — needed for both Axis 1 and Axis 2. Must happen
146
+ # before any DB call so as_of is available for transaction-time gating.
147
+ # None context = legacy/default; non-dict = upstream caller without
148
+ # as_of support. Both treated as "no time-travel, current time".
149
+ as_of: str | None = (
150
+ context.get("as_of") if isinstance(context, dict) else None
151
+ )
152
+ # UTC-normalize at the filter boundary to guarantee format consistency
153
+ # before forwarding to DB comparisons.
154
+ if as_of is not None:
155
+ from superlocalmemory.retrieval.temporal_utils import normalize_as_of
156
+ as_of = normalize_as_of(as_of)
157
+ # normalize_as_of returns None on invalid input; treat as no as_of.
158
+
159
+ # --- Axis 1: Transaction-time supersession ---
160
+ # When as_of is set: only supersessions that occurred AT OR BEFORE
161
+ # as_of contribute (Phase 4b bi-temporal fix). Supersessions after
162
+ # as_of are invisible — the fact was still valid at the query point.
91
163
  try:
92
164
  invalid = self._db.get_invalidated_fact_ids(
93
- list(all_fact_ids), profile_id,
165
+ list(all_fact_ids), profile_id, as_of=as_of,
94
166
  )
95
167
  except Exception as exc:
96
168
  # Fail-open: a validity-lookup error must never break retrieval.
97
169
  logger.warning("Temporal validity lookup failed: %s", exc)
98
170
  return all_results
99
171
 
100
- if not invalid:
172
+ # --- Axis 2: Event-time expiry (Phase 4 T1b) ---
173
+ # Guard: skip event-time demotion when the caller signals it wants
174
+ # historical / all-expired-included mode AND no explicit as_of point
175
+ # is requested. Config default is include_expired_in_history=True,
176
+ # so the default recall path always takes this skip branch — the two
177
+ # new DB calls are never made, and no scores change.
178
+ apply_event_time = (as_of is not None) or (not self._include_expired_in_history)
179
+
180
+ event_expired: set[str] = set()
181
+ if apply_event_time:
182
+ try:
183
+ event_expired = self._db.get_event_time_expired_fact_ids(
184
+ list(all_fact_ids), profile_id, as_of=as_of,
185
+ )
186
+ except Exception as exc:
187
+ # Fail-open: continue with empty event_expired set.
188
+ logger.warning("Event-time expiry lookup failed: %s", exc)
189
+
190
+ # Fast path: nothing to demote — return input unchanged.
191
+ if not invalid and not event_expired:
101
192
  return all_results
102
193
 
103
- factor = self._demotion_factor
194
+ system_factor = self._demotion_factor
195
+ event_factor = self._event_time_factor
196
+ # event_time_only: expired by event-time but NOT system-invalidated.
197
+ # System-invalid already carries the harder demotion (0.25 < 0.5).
198
+ # We do not stack both factors to avoid excessive double-penalty.
199
+ event_time_only = event_expired - invalid
200
+
104
201
  demoted: dict[str, list[tuple[str, float]]] = {}
105
202
  for channel_name, channel_results in all_results.items():
106
203
  new_list = [
107
- (fact_id, score * factor if fact_id in invalid else score)
204
+ (fact_id,
205
+ score * system_factor if fact_id in invalid
206
+ else score * event_factor if fact_id in event_time_only
207
+ else score)
108
208
  for fact_id, score in channel_results
109
209
  ]
110
- # Re-sort descending so demoted (superseded) facts fall below
111
- # currently-valid facts in this channel's rank order.
210
+ # Re-sort descending so demoted facts fall below currently-valid
211
+ # facts in this channel's rank order.
112
212
  new_list.sort(key=lambda pair: pair[1], reverse=True)
113
213
  demoted[channel_name] = new_list
114
214
  return demoted
@@ -123,6 +223,10 @@ def register_temporal_validity_filter(
123
223
 
124
224
  Does nothing if config.enabled is False.
125
225
 
226
+ Reads ``event_time_demotion_factor`` from the config with a fallback to the
227
+ module constant ``_EVENT_TIME_DEMOTION_FACTOR`` (0.5) so older serialised
228
+ configs without this field work without a migration.
229
+
126
230
  Args:
127
231
  registry: Channel registry to register with.
128
232
  db: Database manager for validity queries.
@@ -131,5 +235,14 @@ def register_temporal_validity_filter(
131
235
  if not getattr(config, "enabled", True):
132
236
  return
133
237
  factor = getattr(config, "superseded_demotion_factor", 0.25)
134
- f = TemporalValidityFilter(db, demotion_factor=factor)
238
+ event_time_factor = getattr(
239
+ config, "event_time_demotion_factor", _EVENT_TIME_DEMOTION_FACTOR,
240
+ )
241
+ include_expired_in_history = getattr(config, "include_expired_in_history", True)
242
+ f = TemporalValidityFilter(
243
+ db,
244
+ demotion_factor=factor,
245
+ event_time_factor=event_time_factor,
246
+ include_expired_in_history=include_expired_in_history,
247
+ )
135
248
  registry.register_filter(f.filter)
@@ -15,6 +15,7 @@ License: AGPL-3.0-or-later
15
15
  from __future__ import annotations
16
16
 
17
17
  import logging
18
+ import re
18
19
  import sqlite3
19
20
  import threading
20
21
  from contextlib import contextmanager
@@ -137,8 +138,50 @@ class VectorStore:
137
138
 
138
139
  # -- Table creation -----------------------------------------------------
139
140
 
141
+ @staticmethod
142
+ def _read_stored_dimension(conn: sqlite3.Connection) -> int | None:
143
+ """Return the embedding dimension of the existing vec0 table, or None.
144
+
145
+ Reads the CREATE VIRTUAL TABLE DDL from sqlite_master and extracts the
146
+ float[N] declaration. Falls back to embedding_metadata.dimension if
147
+ the DDL is absent or unparseable. Returns None when the table does not
148
+ yet exist, which means no rebuild is needed.
149
+ """
150
+ try:
151
+ row = conn.execute(
152
+ "SELECT sql FROM sqlite_master WHERE name = 'fact_embeddings'"
153
+ ).fetchone()
154
+ if row is not None:
155
+ sql = row["sql"] or row[0]
156
+ if sql:
157
+ m = re.search(r'float\[(\d+)\]', sql, re.IGNORECASE)
158
+ if m:
159
+ return int(m.group(1))
160
+ except Exception:
161
+ pass
162
+
163
+ # Fallback: read dimension from the most recent metadata row.
164
+ try:
165
+ row = conn.execute(
166
+ "SELECT dimension FROM embedding_metadata LIMIT 1"
167
+ ).fetchone()
168
+ if row is not None:
169
+ return int(row["dimension"])
170
+ except sqlite3.OperationalError:
171
+ pass # metadata table does not exist yet
172
+
173
+ return None
174
+
140
175
  def _ensure_vec0_table(self) -> None:
141
- """Create the vec0 virtual table and embedding_metadata if not exist."""
176
+ """Create or rebuild the vec0 virtual table and embedding_metadata.
177
+
178
+ If the existing vec0 table was built at a different embedding dimension
179
+ than self._config.dimension, both tables are dropped and recreated at
180
+ the new dimension. Old embeddings are lost; callers that want to
181
+ re-populate should call rebuild_from_facts() afterward.
182
+
183
+ Same-dimension opens are a no-op (IF NOT EXISTS guard).
184
+ """
142
185
  dim = self._config.dimension
143
186
  vec0_ddl = (
144
187
  f"CREATE VIRTUAL TABLE IF NOT EXISTS fact_embeddings USING vec0("
@@ -162,12 +205,41 @@ class VectorStore:
162
205
  meta_idx_profile = (
163
206
  "CREATE INDEX IF NOT EXISTS idx_embmeta_profile ON embedding_metadata (profile_id)"
164
207
  )
208
+ row_map_ddl = (
209
+ "CREATE TABLE IF NOT EXISTS vector_row_map ("
210
+ "fact_id TEXT NOT NULL PRIMARY KEY, "
211
+ "profile_id TEXT NOT NULL, "
212
+ "vec_rowid INTEGER NOT NULL"
213
+ ")"
214
+ )
215
+ row_map_idx = (
216
+ "CREATE INDEX IF NOT EXISTS idx_vector_row_map_profile "
217
+ "ON vector_row_map (profile_id)"
218
+ )
165
219
  try:
166
220
  with self._managed_connection() as conn:
221
+ stored_dim = self._read_stored_dimension(conn)
222
+ if stored_dim is not None and stored_dim != dim:
223
+ logger.info(
224
+ "Embedding dimension changed %d→%d: rebuilding vector index",
225
+ stored_dim,
226
+ dim,
227
+ )
228
+ # Drop metadata first (no FK enforcement, but cleaner ordering).
229
+ # Indexes on embedding_metadata are dropped automatically.
230
+ conn.execute("DROP TABLE IF EXISTS embedding_metadata")
231
+ conn.execute("DROP TABLE IF EXISTS vector_row_map")
232
+ conn.execute("DROP TABLE IF EXISTS fact_embeddings")
233
+ # Commit the drops before recreating so that the virtual-table
234
+ # shadow tables are fully removed before the new CREATE runs.
235
+ conn.commit()
236
+
167
237
  conn.execute(vec0_ddl)
168
238
  conn.execute(meta_ddl)
169
239
  conn.execute(meta_idx_fact)
170
240
  conn.execute(meta_idx_profile)
241
+ conn.execute(row_map_ddl)
242
+ conn.execute(row_map_idx)
171
243
  conn.commit()
172
244
  except Exception as exc:
173
245
  logger.debug("vec0 table creation failed: %s", exc)
@@ -255,13 +327,30 @@ class VectorStore:
255
327
  # Older self-heal code could insert metadata
256
328
  # before sqlite-vec, or row-id drift could point
257
329
  # metadata at another profile's vector. Neither
258
- # is a valid projection pair. Remove only the
259
- # stale pointer and rebuild at a fresh rowid;
260
- # never overwrite the other profile's payload.
330
+ # is a valid projection pair. Remove the stale
331
+ # pointer and rebuild at a fresh rowid; never
332
+ # overwrite the other profile's payload.
261
333
  conn.execute(
262
334
  "DELETE FROM embedding_metadata WHERE fact_id = ?",
263
335
  (fact_id,),
264
336
  )
337
+ # The old vec0 row is orphaned unless another
338
+ # projection pair still references it. Reclaim it
339
+ # so drift-repair never abandons raw payload.
340
+ still_referenced = conn.execute(
341
+ "SELECT 1 FROM embedding_metadata "
342
+ "WHERE vec_rowid = ? LIMIT 1",
343
+ (rowid,),
344
+ ).fetchone()
345
+ if still_referenced is None:
346
+ conn.execute(
347
+ "DELETE FROM fact_embeddings WHERE rowid = ?",
348
+ (rowid,),
349
+ )
350
+ conn.execute(
351
+ "DELETE FROM vector_row_map WHERE fact_id = ?",
352
+ (fact_id,),
353
+ )
265
354
  row = None
266
355
 
267
356
  if row is None:
@@ -301,6 +390,14 @@ class VectorStore:
301
390
  ),
302
391
  )
303
392
 
393
+ conn.execute(
394
+ "INSERT INTO vector_row_map (fact_id, profile_id, vec_rowid) "
395
+ "VALUES (?, ?, ?) "
396
+ "ON CONFLICT(fact_id) DO UPDATE SET "
397
+ "profile_id = excluded.profile_id, "
398
+ "vec_rowid = excluded.vec_rowid",
399
+ (fact_id, profile_id, rowid),
400
+ )
304
401
  conn.commit()
305
402
  return True
306
403
  except Exception as exc:
@@ -415,16 +512,33 @@ class VectorStore:
415
512
  (fact_id,),
416
513
  ).fetchone()
417
514
 
418
- if row is None:
515
+ rowid = None
516
+ owner_profile = None
517
+ if row is not None:
518
+ rowid = row["vec_rowid"]
519
+ owner_profile = str(row["profile_id"])
520
+ else:
521
+ # Metadata-less orphan: resolve the rowid via the
522
+ # fact-addressable map so the raw vec0 row is still
523
+ # removable by fact_id.
524
+ mrow = conn.execute(
525
+ "SELECT vec_rowid, profile_id "
526
+ "FROM vector_row_map WHERE fact_id = ?",
527
+ (fact_id,),
528
+ ).fetchone()
529
+ if mrow is not None:
530
+ rowid = mrow["vec_rowid"]
531
+ owner_profile = str(mrow["profile_id"])
532
+
533
+ if rowid is None:
419
534
  return False
420
535
 
421
- rowid = row["vec_rowid"]
422
536
  vector_row = conn.execute(
423
537
  "SELECT profile_id FROM fact_embeddings WHERE rowid = ?",
424
538
  (rowid,),
425
539
  ).fetchone()
426
- if vector_row is not None and str(vector_row["profile_id"]) == str(
427
- row["profile_id"]
540
+ if vector_row is not None and (
541
+ str(vector_row["profile_id"]) == owner_profile
428
542
  ):
429
543
  conn.execute(
430
544
  "DELETE FROM fact_embeddings WHERE rowid = ?",
@@ -434,12 +548,54 @@ class VectorStore:
434
548
  "DELETE FROM embedding_metadata WHERE vec_rowid = ?",
435
549
  (rowid,),
436
550
  )
551
+ conn.execute(
552
+ "DELETE FROM vector_row_map WHERE fact_id = ?",
553
+ (fact_id,),
554
+ )
437
555
  conn.commit()
438
556
  return True
439
557
  except Exception as exc:
440
558
  logger.debug("delete failed for fact_id=%s: %s", fact_id, exc)
441
559
  return False
442
560
 
561
+ def raw_vector_present(self, fact_id: str) -> bool:
562
+ if not self._available:
563
+ try:
564
+ conn = sqlite3.connect(str(self._db_path))
565
+ try:
566
+ row = conn.execute(
567
+ "SELECT 1 FROM vector_row_map WHERE fact_id = ? LIMIT 1",
568
+ (fact_id,),
569
+ ).fetchone()
570
+ return row is not None
571
+ finally:
572
+ conn.close()
573
+ except Exception:
574
+ return False
575
+ try:
576
+ with self._managed_connection() as conn:
577
+ row = conn.execute(
578
+ "SELECT 1 FROM vector_row_map vrm "
579
+ "WHERE vrm.fact_id = ? "
580
+ "AND EXISTS (SELECT 1 FROM fact_embeddings fe "
581
+ "WHERE fe.rowid = vrm.vec_rowid)",
582
+ (fact_id,),
583
+ ).fetchone()
584
+ return row is not None
585
+ except Exception:
586
+ try:
587
+ conn2 = sqlite3.connect(str(self._db_path))
588
+ try:
589
+ row = conn2.execute(
590
+ "SELECT 1 FROM vector_row_map WHERE fact_id = ? LIMIT 1",
591
+ (fact_id,),
592
+ ).fetchone()
593
+ return row is not None
594
+ finally:
595
+ conn2.close()
596
+ except Exception:
597
+ return False
598
+
443
599
  def count(self, profile_id: str | None = None) -> int:
444
600
  """Count complete metadata/vector pairs in the store.
445
601
 
@@ -493,6 +649,56 @@ class VectorStore:
493
649
  logger.debug("indexed_fact_ids failed: %s", exc)
494
650
  return set()
495
651
 
652
+ def gc_orphaned_vectors(self, profile_id: str | None = None) -> int:
653
+ """Physically remove vec0 rows not referenced by any fact-addressable
654
+ mapping (neither embedding_metadata nor vector_row_map).
655
+
656
+ Such rows are unreachable by fact_id and would otherwise persist forever.
657
+ Returns the count removed. No-op when the vector backend is unavailable.
658
+ """
659
+ if not self._available:
660
+ return 0
661
+
662
+ _wl = get_write_lock(self._db_path)
663
+ with _wl:
664
+ with self._lock:
665
+ try:
666
+ with self._managed_connection() as conn:
667
+ conn.execute("BEGIN IMMEDIATE")
668
+ orphans = self._orphan_rowids(conn, profile_id)
669
+ for rowid in orphans:
670
+ conn.execute(
671
+ "DELETE FROM fact_embeddings WHERE rowid = ?",
672
+ (rowid,),
673
+ )
674
+ conn.commit()
675
+ return len(orphans)
676
+ except Exception as exc:
677
+ logger.debug("gc_orphaned_vectors failed: %s", exc)
678
+ return 0
679
+
680
+ @staticmethod
681
+ def _orphan_rowids(conn: sqlite3.Connection, profile_id: str | None) -> set[int]:
682
+ """vec0 rowids not referenced by embedding_metadata or vector_row_map."""
683
+ if profile_id is not None:
684
+ all_rowids = {r[0] for r in conn.execute(
685
+ "SELECT rowid FROM fact_embeddings WHERE profile_id = ?",
686
+ (profile_id,)).fetchall()}
687
+ ref_meta = {r[0] for r in conn.execute(
688
+ "SELECT vec_rowid FROM embedding_metadata WHERE profile_id = ?",
689
+ (profile_id,)).fetchall()}
690
+ ref_map = {r[0] for r in conn.execute(
691
+ "SELECT vec_rowid FROM vector_row_map WHERE profile_id = ?",
692
+ (profile_id,)).fetchall()}
693
+ else:
694
+ all_rowids = {r[0] for r in conn.execute(
695
+ "SELECT rowid FROM fact_embeddings").fetchall()}
696
+ ref_meta = {r[0] for r in conn.execute(
697
+ "SELECT vec_rowid FROM embedding_metadata").fetchall()}
698
+ ref_map = {r[0] for r in conn.execute(
699
+ "SELECT vec_rowid FROM vector_row_map").fetchall()}
700
+ return all_rowids - ref_meta - ref_map
701
+
496
702
  def rebuild_from_facts(
497
703
  self,
498
704
  facts: list[tuple[str, str, list[float]]],
@@ -109,8 +109,8 @@ async def lifespan(application: FastAPI):
109
109
  def create_app() -> FastAPI:
110
110
  """Create and configure the FastAPI application."""
111
111
  application = FastAPI(
112
- title="SuperLocalMemory V3 API",
113
- description="V3 Memory Engine REST API",
112
+ title="SuperLocalMemory V4 API",
113
+ description="V4 Memory Engine REST API",
114
114
  version=SLM_VERSION,
115
115
  lifespan=lifespan,
116
116
  )
@@ -240,9 +240,9 @@ def create_app() -> FastAPI:
240
240
  index_path = UI_DIR / "index.html"
241
241
  if not index_path.exists():
242
242
  return (
243
- "<html><head><title>SuperLocalMemory V3</title></head>"
243
+ "<html><head><title>SuperLocalMemory V4</title></head>"
244
244
  "<body style='font-family:Arial;padding:40px'>"
245
- "<h1>SuperLocalMemory V3 API Server Running</h1>"
245
+ "<h1>SuperLocalMemory V4 API Server Running</h1>"
246
246
  "<p><a href='/docs'>API Documentation</a></p>"
247
247
  "</body></html>"
248
248
  )
@@ -272,7 +272,7 @@ def create_app() -> FastAPI:
272
272
  if __name__ == "__main__":
273
273
  app = create_app()
274
274
  print("=" * 60)
275
- print("SuperLocalMemory V3 - API Server (standalone mode)")
275
+ print("SuperLocalMemory V4 - API Server (standalone mode)")
276
276
  print("=" * 60)
277
277
  print(f"Database: {DB_PATH}")
278
278
  print(f"UI Directory: {UI_DIR}")