superlocalmemory 3.8.13 → 4.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (212) hide show
  1. package/ATTRIBUTION.md +4 -4
  2. package/CHANGELOG.md +113 -121
  3. package/README.md +65 -63
  4. package/docs/pi-dev-integration.md +1 -1
  5. package/package.json +6 -1
  6. package/plugin/.claude-plugin/plugin.json +1 -1
  7. package/plugin/CLAUDE.md +3 -3
  8. package/plugin/agents/slm-governance-advisor.md +1 -1
  9. package/plugin/agents/slm-loop-runner.md +1 -1
  10. package/plugin/agents/slm-memory-advisor.md +1 -1
  11. package/plugin/agents/slm-optimize-advisor.md +1 -1
  12. package/plugin/requirements.txt +1 -1
  13. package/plugin/skills/slm-cache/SKILL.md +1 -1
  14. package/plugin/skills/slm-compress/SKILL.md +1 -1
  15. package/plugin/skills/slm-governance/SKILL.md +1 -1
  16. package/plugin/skills/slm-graph/SKILL.md +1 -1
  17. package/plugin/skills/slm-loop/SKILL.md +1 -1
  18. package/plugin/skills/slm-mesh/SKILL.md +1 -1
  19. package/plugin/skills/slm-profile/SKILL.md +1 -1
  20. package/plugin/skills/slm-recall/SKILL.md +1 -1
  21. package/plugin/skills/slm-remember/SKILL.md +1 -1
  22. package/plugin/skills/slm-scope/SKILL.md +1 -1
  23. package/plugin/skills/slm-session/SKILL.md +1 -1
  24. package/plugin/skills/slm-status/SKILL.md +1 -1
  25. package/plugin-src/rules/AGENTS.md +1 -1
  26. package/plugin-src/skills/slm-cache/SKILL.md +1 -1
  27. package/plugin-src/skills/slm-compress/SKILL.md +1 -1
  28. package/plugin-src/skills/slm-governance/SKILL.md +248 -0
  29. package/plugin-src/skills/slm-graph/SKILL.md +1 -1
  30. package/plugin-src/skills/slm-loop/SKILL.md +99 -0
  31. package/plugin-src/skills/slm-mesh/SKILL.md +282 -0
  32. package/plugin-src/skills/slm-profile/SKILL.md +148 -0
  33. package/plugin-src/skills/slm-recall/SKILL.md +1 -1
  34. package/plugin-src/skills/slm-remember/SKILL.md +1 -1
  35. package/plugin-src/skills/slm-scope/SKILL.md +176 -0
  36. package/plugin-src/skills/slm-session/SKILL.md +1 -1
  37. package/plugin-src/skills/slm-status/SKILL.md +1 -1
  38. package/pyproject.toml +11 -4
  39. package/src/superlocalmemory/__init__.py +1 -1
  40. package/src/superlocalmemory/cli/commands.py +125 -11
  41. package/src/superlocalmemory/cli/daemon.py +5 -1
  42. package/src/superlocalmemory/cli/main.py +35 -2
  43. package/src/superlocalmemory/cli/ops_cmd.py +281 -0
  44. package/src/superlocalmemory/cli/setup_wizard.py +1 -1
  45. package/src/superlocalmemory/compliance/audit.py +65 -0
  46. package/src/superlocalmemory/compliance/eu_ai_act.py +27 -57
  47. package/src/superlocalmemory/compliance/gdpr.py +416 -20
  48. package/src/superlocalmemory/compliance/retention.py +74 -22
  49. package/src/superlocalmemory/compliance/scheduler.py +78 -9
  50. package/src/superlocalmemory/core/actor_context.py +166 -0
  51. package/src/superlocalmemory/core/admission.py +549 -0
  52. package/src/superlocalmemory/core/backend_orchestrator.py +23 -10
  53. package/src/superlocalmemory/core/config.py +202 -24
  54. package/src/superlocalmemory/core/consolidation_engine.py +13 -13
  55. package/src/superlocalmemory/core/context_cache.py +28 -0
  56. package/src/superlocalmemory/core/embeddings.py +64 -2
  57. package/src/superlocalmemory/core/engine.py +7 -2
  58. package/src/superlocalmemory/core/engine_ingestion.py +65 -3
  59. package/src/superlocalmemory/core/engine_wiring.py +36 -9
  60. package/src/superlocalmemory/core/ingest_policy.py +38 -0
  61. package/src/superlocalmemory/core/maintenance.py +255 -0
  62. package/src/superlocalmemory/core/modes.py +40 -13
  63. package/src/superlocalmemory/core/mutations.py +437 -44
  64. package/src/superlocalmemory/core/operation_policy.py +92 -0
  65. package/src/superlocalmemory/core/operation_policy_registry.py +542 -0
  66. package/src/superlocalmemory/core/operation_request.py +127 -0
  67. package/src/superlocalmemory/core/ops_remediation.py +542 -0
  68. package/src/superlocalmemory/core/recall_pipeline.py +7 -0
  69. package/src/superlocalmemory/core/remember_runtime.py +202 -4
  70. package/src/superlocalmemory/core/remote_mode.py +20 -5
  71. package/src/superlocalmemory/core/store_pipeline.py +150 -0
  72. package/src/superlocalmemory/core/topic_signature.py +19 -4
  73. package/src/superlocalmemory/core/transactions/__init__.py +78 -0
  74. package/src/superlocalmemory/core/transactions/concrete_owners.py +597 -0
  75. package/src/superlocalmemory/core/transactions/erasure.py +825 -0
  76. package/src/superlocalmemory/core/transactions/manifest.py +255 -0
  77. package/src/superlocalmemory/core/transactions/manifest_key.py +155 -0
  78. package/src/superlocalmemory/core/transactions/obligations.py +272 -0
  79. package/src/superlocalmemory/core/transactions/owners.py +114 -0
  80. package/src/superlocalmemory/core/transactions/reconciler.py +285 -0
  81. package/src/superlocalmemory/core/transactions/service.py +330 -0
  82. package/src/superlocalmemory/core/worker_pool.py +33 -5
  83. package/src/superlocalmemory/encoding/cognitive_consolidator.py +70 -28
  84. package/src/superlocalmemory/encoding/emotional.py +75 -14
  85. package/src/superlocalmemory/encoding/scene_builder.py +115 -13
  86. package/src/superlocalmemory/encoding/temporal_parser.py +4 -0
  87. package/src/superlocalmemory/evolution/blind_verifier.py +11 -4
  88. package/src/superlocalmemory/evolution/evolution_store.py +244 -4
  89. package/src/superlocalmemory/evolution/llm_dispatch.py +40 -0
  90. package/src/superlocalmemory/evolution/model_selection.py +18 -3
  91. package/src/superlocalmemory/evolution/mutation_generator.py +3 -0
  92. package/src/superlocalmemory/evolution/skill_activator.py +270 -0
  93. package/src/superlocalmemory/evolution/skill_evolver.py +281 -59
  94. package/src/superlocalmemory/evolution/types.py +30 -8
  95. package/src/superlocalmemory/graph/cozo_backend.py +17 -9
  96. package/src/superlocalmemory/hooks/auto_invoker.py +2 -1
  97. package/src/superlocalmemory/hooks/auto_recall.py +64 -30
  98. package/src/superlocalmemory/hooks/codex_assets.py +14 -1
  99. package/src/superlocalmemory/infra/backup.py +434 -7
  100. package/src/superlocalmemory/infra/process_reaper.py +18 -0
  101. package/src/superlocalmemory/infra/self_heal.py +401 -0
  102. package/src/superlocalmemory/learning/feedback.py +52 -9
  103. package/src/superlocalmemory/loops/engine.py +10 -0
  104. package/src/superlocalmemory/mcp/_daemon_proxy.py +3 -0
  105. package/src/superlocalmemory/mcp/http_transport.py +30 -331
  106. package/src/superlocalmemory/mcp/profiles.py +5 -0
  107. package/src/superlocalmemory/mcp/resources.py +8 -0
  108. package/src/superlocalmemory/mcp/server.py +51 -4
  109. package/src/superlocalmemory/mcp/shared.py +19 -0
  110. package/src/superlocalmemory/mcp/tools_active.py +25 -4
  111. package/src/superlocalmemory/mcp/tools_code_graph.py +26 -18
  112. package/src/superlocalmemory/mcp/tools_context.py +50 -8
  113. package/src/superlocalmemory/mcp/tools_core.py +69 -21
  114. package/src/superlocalmemory/mcp/tools_evolution.py +9 -2
  115. package/src/superlocalmemory/mcp/tools_learning.py +21 -10
  116. package/src/superlocalmemory/mcp/tools_loops.py +29 -18
  117. package/src/superlocalmemory/mcp/tools_mesh.py +8 -0
  118. package/src/superlocalmemory/mcp/tools_ops.py +115 -0
  119. package/src/superlocalmemory/mcp/tools_optimize.py +4 -0
  120. package/src/superlocalmemory/mcp/tools_v28.py +10 -3
  121. package/src/superlocalmemory/mcp/tools_v3.py +34 -14
  122. package/src/superlocalmemory/mcp/tools_v33.py +18 -33
  123. package/src/superlocalmemory/mesh/broker.py +124 -46
  124. package/src/superlocalmemory/mesh/broker_security.py +470 -0
  125. package/src/superlocalmemory/mesh/discovery.py +365 -0
  126. package/src/superlocalmemory/mesh/lock_protocol.py +313 -0
  127. package/src/superlocalmemory/mesh/node_identity.py +97 -0
  128. package/src/superlocalmemory/mesh/outbox_remote.py +429 -0
  129. package/src/superlocalmemory/mesh/remote_sync.py +511 -28
  130. package/src/superlocalmemory/mesh/state_sync.py +286 -0
  131. package/src/superlocalmemory/optimize/config/store.py +45 -0
  132. package/src/superlocalmemory/parameterization/cross_project.py +12 -0
  133. package/src/superlocalmemory/parameterization/prompt_injector.py +13 -11
  134. package/src/superlocalmemory/parameterization/prompt_lifecycle.py +8 -2
  135. package/src/superlocalmemory/parameterization/workflow_miner.py +17 -0
  136. package/src/superlocalmemory/retrieval/ann_index.py +5 -0
  137. package/src/superlocalmemory/retrieval/bm25_channel.py +49 -2
  138. package/src/superlocalmemory/retrieval/engine.py +19 -4
  139. package/src/superlocalmemory/retrieval/fusion.py +4 -1
  140. package/src/superlocalmemory/retrieval/hopfield_channel.py +9 -3
  141. package/src/superlocalmemory/retrieval/remote_reranker.py +47 -22
  142. package/src/superlocalmemory/retrieval/reranker.py +32 -1
  143. package/src/superlocalmemory/retrieval/temporal_channel.py +16 -3
  144. package/src/superlocalmemory/retrieval/temporal_utils.py +107 -0
  145. package/src/superlocalmemory/retrieval/temporal_validity_filter.py +155 -42
  146. package/src/superlocalmemory/retrieval/vector_store.py +214 -8
  147. package/src/superlocalmemory/server/api.py +5 -5
  148. package/src/superlocalmemory/server/egress_policy.py +258 -0
  149. package/src/superlocalmemory/server/rbac_enforce.py +32 -0
  150. package/src/superlocalmemory/server/route_mutations.py +20 -0
  151. package/src/superlocalmemory/server/routes/compliance.py +153 -7
  152. package/src/superlocalmemory/server/routes/data_io.py +43 -2
  153. package/src/superlocalmemory/server/routes/events.py +15 -0
  154. package/src/superlocalmemory/server/routes/memories.py +56 -3
  155. package/src/superlocalmemory/server/routes/mesh.py +82 -1
  156. package/src/superlocalmemory/server/routes/mesh_lock.py +54 -0
  157. package/src/superlocalmemory/server/routes/mesh_state.py +63 -0
  158. package/src/superlocalmemory/server/routes/v3_api.py +50 -27
  159. package/src/superlocalmemory/server/routes/ws.py +86 -0
  160. package/src/superlocalmemory/server/ui.py +6 -6
  161. package/src/superlocalmemory/server/unified_daemon.py +942 -119
  162. package/src/superlocalmemory/storage/_migration_internals.py +568 -0
  163. package/src/superlocalmemory/storage/_schema_version.py +110 -0
  164. package/src/superlocalmemory/storage/database.py +329 -24
  165. package/src/superlocalmemory/storage/embedding_migrator.py +246 -51
  166. package/src/superlocalmemory/storage/erasure_fence.py +45 -0
  167. package/src/superlocalmemory/storage/generation_fence.py +63 -0
  168. package/src/superlocalmemory/storage/migration_runner.py +140 -417
  169. package/src/superlocalmemory/storage/migrations/M009_model_lineage.py +40 -0
  170. package/src/superlocalmemory/storage/migrations/M033_projection_transactions.py +148 -0
  171. package/src/superlocalmemory/storage/migrations/M034_obligation_integrity.py +58 -0
  172. package/src/superlocalmemory/storage/migrations/M035_erasure_receipts.py +113 -0
  173. package/src/superlocalmemory/storage/migrations/M036_vector_row_map.py +107 -0
  174. package/src/superlocalmemory/storage/migrations/M037_manifest_hmac_version.py +162 -0
  175. package/src/superlocalmemory/storage/migrations/{M033_learning_feedback_channel.py → M038_learning_feedback_channel.py} +3 -3
  176. package/src/superlocalmemory/storage/migrations/M039_scene_fact_members.py +137 -0
  177. package/src/superlocalmemory/storage/migrations/__init__.py +4 -2
  178. package/src/superlocalmemory/storage/schema.py +67 -0
  179. package/src/superlocalmemory/storage/write_coordinator.py +125 -0
  180. package/src/superlocalmemory/trust/scorer.py +28 -4
  181. package/src/superlocalmemory/ui/index.html +14 -3
  182. package/src/superlocalmemory/ui/js/auto-settings.js +12 -1
  183. package/src/superlocalmemory/ui/js/brain.js +6 -4
  184. package/src/superlocalmemory/ui/js/compliance.js +66 -12
  185. package/src/superlocalmemory/ui/js/dashboard.js +13 -3
  186. package/src/superlocalmemory/ui/js/feedback.js +8 -2
  187. package/src/superlocalmemory/ui/js/lifecycle.js +7 -1
  188. package/src/superlocalmemory/ui/js/modal.js +272 -5
  189. package/src/superlocalmemory/ui/js/od-backup.js +9 -2
  190. package/src/superlocalmemory/ui/js/od-compliance-ext.js +301 -0
  191. package/src/superlocalmemory/ui/js/od-operations.js +154 -23
  192. package/src/superlocalmemory/ui/js/od-ops-health.js +417 -0
  193. package/src/superlocalmemory/ui/js/od-optimize.js +35 -21
  194. package/src/superlocalmemory/ui/js/od-team.js +9 -2
  195. package/src/superlocalmemory/ui/js/optimize.js +13 -16
  196. package/src/superlocalmemory/ui/js/profiles.js +7 -3
  197. package/src/superlocalmemory/ui/js/settings.js +7 -1
  198. package/src/superlocalmemory/vector/lancedb_backend.py +19 -9
  199. package/src/superlocalmemory/attribution/mathematical_dna.py +0 -235
  200. package/src/superlocalmemory/cli/post_install.py +0 -114
  201. package/src/superlocalmemory/core/clock_monitor.py +0 -45
  202. package/src/superlocalmemory/core/db_pool.py +0 -80
  203. package/src/superlocalmemory/core/error_catalog.py +0 -113
  204. package/src/superlocalmemory/core/loop_watchdog.py +0 -56
  205. package/src/superlocalmemory/core/priority_queue.py +0 -61
  206. package/src/superlocalmemory/core/pruning_engine.py +0 -216
  207. package/src/superlocalmemory/core/queue_dispatcher.py +0 -73
  208. package/src/superlocalmemory/core/slmignore.py +0 -125
  209. package/src/superlocalmemory/infra/heartbeat_monitor.py +0 -140
  210. package/src/superlocalmemory/infra/webhook_dispatcher.py +0 -247
  211. package/src/superlocalmemory/learning/quantization_scheduler.py +0 -320
  212. package/src/superlocalmemory/storage/access_control.py +0 -182
@@ -24,6 +24,7 @@ logger = logging.getLogger(__name__)
24
24
 
25
25
  # Similarity threshold for assigning fact to existing scene
26
26
  _ASSIGN_THRESHOLD = 0.6
27
+ _MAX_ASSIGNMENT_CANDIDATES = 256
27
28
 
28
29
 
29
30
  class SceneBuilder:
@@ -35,9 +36,10 @@ class SceneBuilder:
35
36
  3. If below threshold: create new scene
36
37
  """
37
38
 
38
- def __init__(self, db, embedder=None) -> None:
39
+ def __init__(self, db, embedder=None, vector_store=None) -> None:
39
40
  self._db = db
40
41
  self._embedder = embedder
42
+ self._vector_store = vector_store
41
43
  # Key by scene ID, never theme. Themes are deliberately non-unique,
42
44
  # while eligibility and durable anchor membership are scene-specific.
43
45
  self._scene_embeddings_cache: dict[str, list[float]] = {}
@@ -71,11 +73,14 @@ class SceneBuilder:
71
73
  if fact_emb is None:
72
74
  return self._create_scene(new_fact, profile_id)
73
75
 
74
- scenes = self._get_scenes(profile_id)
76
+ scenes = self._get_assignment_scenes(profile_id, fact_emb)
75
77
  if not scenes:
76
78
  return self._create_scene(new_fact, profile_id)
77
79
 
78
- live_scene_embeddings = self._load_live_scene_embeddings(profile_id)
80
+ live_scene_embeddings = self._load_live_scene_embeddings(
81
+ profile_id,
82
+ tuple(scene.scene_id for scene in scenes),
83
+ )
79
84
  live_scene_ids = set(live_scene_embeddings)
80
85
  self._scene_embeddings_cache.update({
81
86
  scene_id: embedding
@@ -209,41 +214,138 @@ class SceneBuilder:
209
214
  )
210
215
  return [self._row_to_scene(dict(r)) for r in rows]
211
216
 
217
+ def _get_assignment_scenes(
218
+ self,
219
+ profile_id: str,
220
+ fact_embedding: list[float],
221
+ ) -> list[MemoryScene]:
222
+ """Load bounded semantic candidates with a recency fallback.
223
+
224
+ Mature stores must not compare every new fact with every historical
225
+ scene. The fact vector index finds nearby members in bounded time; the
226
+ normalized membership projection maps them back to scenes. Recent
227
+ scenes remain a fail-soft fallback for fresh or unavailable indexes.
228
+ Semantic candidates are ordered first so an old relevant scene cannot
229
+ be displaced by the recency cap.
230
+ """
231
+ recent_rows = self._db.execute(
232
+ "SELECT ms.* FROM memory_scenes AS ms WHERE ms.profile_id = ? "
233
+ "AND EXISTS (SELECT 1 FROM scene_fact_members AS live_member "
234
+ "WHERE live_member.scene_id = ms.scene_id "
235
+ "AND live_member.profile_id = ms.profile_id) "
236
+ "ORDER BY ms.last_updated DESC LIMIT ?",
237
+ (profile_id, _MAX_ASSIGNMENT_CANDIDATES),
238
+ )
239
+ recent = [self._row_to_scene(dict(row)) for row in recent_rows]
240
+
241
+ nearest_fact_ids: list[str] = []
242
+ if self._vector_store is not None:
243
+ try:
244
+ nearest_fact_ids = [
245
+ fact_id
246
+ for fact_id, _score in self._vector_store.search(
247
+ fact_embedding,
248
+ top_k=_MAX_ASSIGNMENT_CANDIDATES,
249
+ profile_id=profile_id,
250
+ )
251
+ ]
252
+ except Exception:
253
+ nearest_fact_ids = []
254
+
255
+ semantic: list[MemoryScene] = []
256
+ if nearest_fact_ids:
257
+ placeholders = ",".join("?" for _ in nearest_fact_ids)
258
+ try:
259
+ semantic_rows = self._db.execute(
260
+ f"""
261
+ SELECT ms.*, member.fact_id AS matched_fact_id
262
+ FROM scene_fact_members AS member
263
+ JOIN memory_scenes AS ms
264
+ ON ms.scene_id = member.scene_id
265
+ AND ms.profile_id = member.profile_id
266
+ WHERE member.profile_id = ?
267
+ AND member.fact_id IN ({placeholders})
268
+ """,
269
+ (profile_id, *nearest_fact_ids),
270
+ )
271
+ hit_rank = {
272
+ fact_id: rank
273
+ for rank, fact_id in enumerate(nearest_fact_ids)
274
+ }
275
+ ranked_scenes: dict[str, tuple[int, MemoryScene]] = {}
276
+ for row in semantic_rows:
277
+ data = dict(row)
278
+ rank = hit_rank.get(
279
+ str(data.pop("matched_fact_id", "")),
280
+ len(hit_rank),
281
+ )
282
+ scene = self._row_to_scene(data)
283
+ previous = ranked_scenes.get(scene.scene_id)
284
+ if previous is None or rank < previous[0]:
285
+ ranked_scenes[scene.scene_id] = (rank, scene)
286
+ semantic = [
287
+ scene
288
+ for _rank, scene in sorted(
289
+ ranked_scenes.values(), key=lambda item: item[0]
290
+ )
291
+ ]
292
+ except Exception:
293
+ # Migration failure or a disabled vector projection must not
294
+ # make remember fail. The bounded recent set remains valid.
295
+ semantic = []
296
+
297
+ candidates: list[MemoryScene] = []
298
+ seen: set[str] = set()
299
+ for scene in (*semantic, *recent):
300
+ if scene.scene_id in seen:
301
+ continue
302
+ seen.add(scene.scene_id)
303
+ candidates.append(scene)
304
+ if len(candidates) >= _MAX_ASSIGNMENT_CANDIDATES:
305
+ break
306
+ return candidates
307
+
212
308
  def _load_live_scene_embeddings(
213
309
  self,
214
310
  profile_id: str,
311
+ scene_ids: tuple[str, ...],
215
312
  ) -> dict[str, list[float] | None]:
216
313
  """Load one durable anchor embedding for every live scene.
217
314
 
218
- ``json_each`` resolves the first still-existing fact in each scene, so
219
- scenes whose original anchor was consolidated away can still reuse a
220
- surviving member. The result also identifies fully stale scene rows,
221
- which are ignored by assignment instead of being re-embedded.
315
+ The normalized membership projection resolves the first still-existing
316
+ fact in each scene without expanding every scene's JSON array. Scenes
317
+ whose original anchor was consolidated away can still reuse a surviving
318
+ member. Fully stale scene rows are ignored instead of being re-embedded.
222
319
  """
320
+ if not scene_ids:
321
+ return {}
322
+ placeholders = ",".join("?" for _ in scene_ids)
223
323
  try:
224
324
  rows = self._db.execute(
225
- """
325
+ f"""
226
326
  WITH live_scene_facts AS (
227
327
  SELECT
228
328
  ms.scene_id,
229
- ms.theme,
230
329
  af.embedding,
231
330
  ROW_NUMBER() OVER (
232
331
  PARTITION BY ms.scene_id
233
- ORDER BY CAST(member.key AS INTEGER)
332
+ ORDER BY member.position
234
333
  ) AS member_rank
235
334
  FROM memory_scenes AS ms
236
- JOIN json_each(ms.fact_ids_json) AS member
335
+ JOIN scene_fact_members AS member
336
+ ON member.scene_id = ms.scene_id
337
+ AND member.profile_id = ms.profile_id
237
338
  JOIN atomic_facts AS af
238
- ON af.fact_id = member.value
339
+ ON af.fact_id = member.fact_id
239
340
  AND af.profile_id = ms.profile_id
240
341
  WHERE ms.profile_id = ?
342
+ AND ms.scene_id IN ({placeholders})
241
343
  )
242
344
  SELECT scene_id, embedding
243
345
  FROM live_scene_facts
244
346
  WHERE member_rank = 1
245
347
  """,
246
- (profile_id,),
348
+ (profile_id, *scene_ids),
247
349
  )
248
350
  except Exception:
249
351
  return {}
@@ -173,6 +173,10 @@ class TemporalParser:
173
173
  return None
174
174
  try:
175
175
  dt = dateutil_parse(raw_date, fuzzy=True)
176
+ # Normalize any explicit offset to UTC so stored session dates are
177
+ # comparable without mixing offsets, keeping the explicit +00:00.
178
+ if dt.tzinfo is not None:
179
+ return dt.astimezone(UTC).isoformat()
176
180
  return _safe_iso(dt)
177
181
  except (ParserError, ValueError, OverflowError):
178
182
  logger.debug("Could not parse session_date: %r", raw_date)
@@ -98,14 +98,21 @@ def parse_verification_response(response: str) -> VerificationResult:
98
98
  except (json.JSONDecodeError, TypeError, ValueError):
99
99
  pass
100
100
 
101
- # Fallback: keyword detection
101
+ # Fallback: keyword detection.
102
+ # Negative keywords are checked first so that negating phrases such as
103
+ # "cannot approve" or "not approved" are never mistakenly matched by the
104
+ # positive keyword "approve".
102
105
  lower = response.lower()
106
+ if any(kw in lower for kw in (
107
+ "\"passed\": false", "passed: false",
108
+ "reject", "fail",
109
+ "cannot approve", "not approve",
110
+ )):
111
+ return VerificationResult(passed=False, confidence=0.6, reasoning="keyword match")
112
+
103
113
  if any(kw in lower for kw in ("\"passed\": true", "passed: true", "approve", "looks good")):
104
114
  return VerificationResult(passed=True, confidence=0.6, reasoning="keyword match")
105
115
 
106
- if any(kw in lower for kw in ("\"passed\": false", "passed: false", "reject", "fail")):
107
- return VerificationResult(passed=False, confidence=0.6, reasoning="keyword match")
108
-
109
116
  # Default: reject if can't parse (conservative)
110
117
  return VerificationResult(
111
118
  passed=False,
@@ -12,9 +12,11 @@ Part of Qualixar | Author: Varun Pratap Bhardwaj
12
12
 
13
13
  from __future__ import annotations
14
14
 
15
+ import hashlib
15
16
  import json
16
17
  import logging
17
18
  import sqlite3
19
+ import warnings
18
20
  from datetime import datetime, timezone
19
21
  from pathlib import Path
20
22
  from typing import Optional
@@ -61,6 +63,44 @@ CREATE TABLE IF NOT EXISTS evolution_cycle_state (
61
63
  updated_at TEXT,
62
64
  PRIMARY KEY (profile_id, key)
63
65
  );
66
+
67
+ -- Phase 2: append-only status-transition log (LLD Decision B2).
68
+ -- Each state change for a record produces one new row here.
69
+ -- The BEFORE UPDATE trigger enforces immutability at the DB layer.
70
+ CREATE TABLE IF NOT EXISTS skill_evolution_transitions (
71
+ seq INTEGER PRIMARY KEY AUTOINCREMENT,
72
+ record_id TEXT NOT NULL,
73
+ profile_id TEXT NOT NULL DEFAULT 'default',
74
+ from_status TEXT NOT NULL,
75
+ to_status TEXT NOT NULL,
76
+ transitioned_at TEXT NOT NULL,
77
+ actor_id TEXT DEFAULT '',
78
+ reason TEXT DEFAULT '',
79
+ prev_hash TEXT DEFAULT '',
80
+ transition_hash TEXT NOT NULL,
81
+ metadata TEXT DEFAULT '{}'
82
+ );
83
+
84
+ CREATE INDEX IF NOT EXISTS idx_evo_trans_record
85
+ ON skill_evolution_transitions(record_id, seq);
86
+
87
+ CREATE INDEX IF NOT EXISTS idx_evo_trans_profile
88
+ ON skill_evolution_transitions(profile_id, transitioned_at);
89
+
90
+ -- DB-enforced append-only: any UPDATE on this table is a bug (LLD Decision B2).
91
+ CREATE TRIGGER IF NOT EXISTS no_update_evo_transitions
92
+ BEFORE UPDATE ON skill_evolution_transitions
93
+ BEGIN
94
+ SELECT RAISE(ABORT, 'skill_evolution_transitions is append-only');
95
+ END;
96
+
97
+ -- Audit P1-3: DELETE must also be forbidden — otherwise the hash chain can be
98
+ -- silently truncated/erased without a DB abort, breaking the immutable-log claim.
99
+ CREATE TRIGGER IF NOT EXISTS no_delete_evo_transitions
100
+ BEFORE DELETE ON skill_evolution_transitions
101
+ BEGIN
102
+ SELECT RAISE(ABORT, 'skill_evolution_transitions is append-only');
103
+ END;
64
104
  """
65
105
 
66
106
  # Anti-loop budget
@@ -223,11 +263,194 @@ class EvolutionStore:
223
263
  for k in recovered:
224
264
  del self._addressed_degradations[k]
225
265
 
266
+ # ------------------------------------------------------------------
267
+ # Phase 2: append-only transition log
268
+ # ------------------------------------------------------------------
269
+
270
+ def insert_record(self, record: EvolutionRecord, profile_id: str) -> None:
271
+ """INSERT a new evolution record (CANDIDATE status).
272
+
273
+ Unlike save_record, this uses plain INSERT — NOT INSERT OR REPLACE.
274
+ Raises sqlite3.IntegrityError if a row with the same id already exists.
275
+ Call this exactly once per candidate (CRIT-1: never reuse record_id).
276
+ """
277
+ conn = sqlite3.connect(self._db_path, timeout=10)
278
+ try:
279
+ conn.execute(
280
+ "INSERT INTO skill_evolution_log "
281
+ "(id, profile_id, skill_name, parent_skill_id, evolution_type, "
282
+ " trigger_type, generation, status, mutation_summary, evidence, "
283
+ " original_content, evolved_content, content_diff, "
284
+ " blind_verified, rejection_reason, created_at, completed_at) "
285
+ "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
286
+ (
287
+ record.id,
288
+ profile_id,
289
+ record.skill_name,
290
+ record.parent_skill_id,
291
+ record.evolution_type.value,
292
+ record.trigger.value,
293
+ record.generation,
294
+ record.status.value,
295
+ record.mutation_summary,
296
+ json.dumps(list(record.evidence)),
297
+ record.original_content,
298
+ record.evolved_content,
299
+ record.content_diff,
300
+ 1 if record.blind_verified else 0,
301
+ record.rejection_reason,
302
+ record.created_at,
303
+ record.completed_at,
304
+ ),
305
+ )
306
+ conn.commit()
307
+ finally:
308
+ conn.close()
309
+
310
+ def append_transition(
311
+ self,
312
+ record_id: str,
313
+ profile_id: str,
314
+ from_status: EvolutionStatus,
315
+ to_status: EvolutionStatus,
316
+ *,
317
+ actor_id: str = "",
318
+ reason: str = "",
319
+ metadata: dict | None = None,
320
+ ) -> str:
321
+ """Append an immutable status-transition row. Returns transition_hash.
322
+
323
+ Hash linkage (audit P2-3): transition_hash = SHA-256 over a canonical,
324
+ pipe-delimited payload covering EVERY persisted field —
325
+ ``prev_hash | record_id | from_status | to_status | ts | actor_id |
326
+ reason | metadata_str``. Delimiters remove concatenation ambiguity and
327
+ including reason+metadata makes those columns tamper-evident too.
328
+
329
+ prev_hash is the transition_hash of the most recent row for this
330
+ record_id, or 'genesis' for the first transition.
331
+
332
+ Concurrency (audit P1-5): the read-prev-then-insert is wrapped in a
333
+ single ``BEGIN IMMEDIATE`` transaction so two concurrent writers cannot
334
+ both read the same prev_hash and fork the chain.
335
+
336
+ Never calls UPDATE or DELETE. Raises ValueError if from_status == to_status.
337
+ """
338
+ if from_status == to_status:
339
+ raise ValueError(
340
+ f"append_transition: from_status == to_status == {from_status!r}; "
341
+ "no-op transitions are not allowed in the append-only log."
342
+ )
343
+ ts = datetime.now(timezone.utc).isoformat()
344
+ metadata_str = json.dumps(metadata or {}, sort_keys=True)
345
+
346
+ # isolation_level=None + explicit BEGIN IMMEDIATE = one atomic
347
+ # select-prev-then-insert critical section per call (P1-5).
348
+ conn = sqlite3.connect(self._db_path, timeout=10, isolation_level=None)
349
+ try:
350
+ conn.execute("BEGIN IMMEDIATE")
351
+ # Find prev_hash for this record_id (genesis if first)
352
+ row = conn.execute(
353
+ "SELECT transition_hash FROM skill_evolution_transitions "
354
+ "WHERE record_id = ? AND profile_id = ? "
355
+ "ORDER BY seq DESC LIMIT 1",
356
+ (record_id, profile_id),
357
+ ).fetchone()
358
+ prev_hash = row[0] if row else "genesis"
359
+
360
+ # Canonical, delimited payload over ALL persisted fields (P2-3).
361
+ payload = "|".join((
362
+ prev_hash, record_id,
363
+ from_status.value, to_status.value,
364
+ ts, actor_id, reason, metadata_str,
365
+ ))
366
+ transition_hash = hashlib.sha256(payload.encode("utf-8")).hexdigest()
367
+
368
+ conn.execute(
369
+ "INSERT INTO skill_evolution_transitions "
370
+ "(record_id, profile_id, from_status, to_status, "
371
+ " transitioned_at, actor_id, reason, prev_hash, "
372
+ " transition_hash, metadata) "
373
+ "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
374
+ (
375
+ record_id,
376
+ profile_id,
377
+ from_status.value,
378
+ to_status.value,
379
+ ts,
380
+ actor_id,
381
+ reason,
382
+ prev_hash,
383
+ transition_hash,
384
+ metadata_str,
385
+ ),
386
+ )
387
+ conn.execute("COMMIT")
388
+ return transition_hash
389
+ except Exception:
390
+ try:
391
+ conn.execute("ROLLBACK")
392
+ except Exception:
393
+ pass
394
+ raise
395
+ finally:
396
+ conn.close()
397
+
398
+ def get_latest_status(
399
+ self, record_id: str, profile_id: str,
400
+ ) -> EvolutionStatus | None:
401
+ """Return the to_status of the highest-seq transition for record_id.
402
+
403
+ Returns None if no transitions exist for this record_id / profile_id.
404
+ """
405
+ conn = sqlite3.connect(self._db_path, timeout=10)
406
+ try:
407
+ row = conn.execute(
408
+ "SELECT to_status FROM skill_evolution_transitions "
409
+ "WHERE record_id = ? AND profile_id = ? "
410
+ "ORDER BY seq DESC LIMIT 1",
411
+ (record_id, profile_id),
412
+ ).fetchone()
413
+ if row is None:
414
+ return None
415
+ return EvolutionStatus(row[0])
416
+ finally:
417
+ conn.close()
418
+
419
+ def get_transitions(self, record_id: str, profile_id: str) -> list[dict]:
420
+ """Return all transition rows for record_id ordered by seq ASC."""
421
+ conn = sqlite3.connect(self._db_path, timeout=10)
422
+ conn.row_factory = sqlite3.Row
423
+ try:
424
+ rows = conn.execute(
425
+ "SELECT seq, record_id, profile_id, from_status, to_status, "
426
+ "transitioned_at, actor_id, reason, prev_hash, transition_hash, "
427
+ "metadata "
428
+ "FROM skill_evolution_transitions "
429
+ "WHERE record_id = ? AND profile_id = ? "
430
+ "ORDER BY seq ASC",
431
+ (record_id, profile_id),
432
+ ).fetchall()
433
+ return [dict(r) for r in rows]
434
+ finally:
435
+ conn.close()
436
+
226
437
  # ------------------------------------------------------------------
227
438
  # CRUD
228
439
  # ------------------------------------------------------------------
229
440
 
230
441
  def save_record(self, record: EvolutionRecord, profile_id: str) -> None:
442
+ """DEPRECATED: use insert_record() for new records and append_transition() for state changes.
443
+
444
+ Retained for backward-compatibility with existing tests and callers.
445
+ Uses INSERT OR REPLACE — mutable semantics, not append-only.
446
+ Will be removed in a future cleanup pass (not Phase 2 scope).
447
+ """
448
+ warnings.warn(
449
+ "EvolutionStore.save_record() is deprecated; "
450
+ "use insert_record() + append_transition() instead.",
451
+ DeprecationWarning,
452
+ stacklevel=2,
453
+ )
231
454
  conn = sqlite3.connect(self._db_path, timeout=10)
232
455
  try:
233
456
  conn.execute(
@@ -306,13 +529,30 @@ class EvolutionStore:
306
529
  conn.close()
307
530
 
308
531
  def count_attempts(self, skill_name: str, profile_id: str) -> int:
532
+ """Count non-successful evolution attempts for a skill.
533
+
534
+ Audit P0-2: with the Phase-2 append-only model, ``skill_evolution_log``
535
+ rows are frozen at 'candidate' by ``insert_record`` and never updated, so
536
+ the legacy ``status NOT IN ('promoted')`` filter counted successes too —
537
+ permanently disabling a skill after MAX_ATTEMPTS_PER_SKILL improvements.
538
+ The current status is now read from the append-only transitions log
539
+ (latest ``to_status`` per record), and success states are excluded,
540
+ mirroring the pre-Phase-2 exclusion of 'promoted'. Legacy rows with no
541
+ transitions fall back to their frozen log status via COALESCE.
542
+ """
543
+ success = ("promoted", "verified_quarantined", "approved", "active")
544
+ marks = ",".join("?" for _ in success)
309
545
  conn = sqlite3.connect(self._db_path, timeout=10)
310
546
  try:
311
547
  row = conn.execute(
312
- "SELECT COUNT(*) FROM skill_evolution_log "
313
- "WHERE skill_name = ? AND profile_id = ? "
314
- "AND status NOT IN ('promoted')",
315
- (skill_name, profile_id),
548
+ "SELECT COUNT(*) FROM skill_evolution_log l "
549
+ "WHERE l.skill_name = ? AND l.profile_id = ? "
550
+ "AND COALESCE("
551
+ " (SELECT t.to_status FROM skill_evolution_transitions t "
552
+ " WHERE t.record_id = l.id AND t.profile_id = l.profile_id "
553
+ " ORDER BY t.seq DESC LIMIT 1), l.status"
554
+ f") NOT IN ({marks})",
555
+ (skill_name, profile_id, *success),
316
556
  ).fetchone()
317
557
  return row[0] if row else 0
318
558
  finally:
@@ -158,6 +158,7 @@ ALLOWED_LLM_MODELS: frozenset[str] = frozenset({
158
158
  "claude-sonnet-4-6",
159
159
  "ollama:llama3",
160
160
  "ollama:qwen2.5",
161
+ "openai:gpt-4o-mini",
161
162
  })
162
163
 
163
164
  FORBIDDEN_MODEL_SUBSTRINGS: tuple[str, ...] = ("opus", "gpt-4-turbo")
@@ -301,6 +302,43 @@ def _call_claude_api_backend(
301
302
  return ""
302
303
 
303
304
 
305
+ def _call_openai_api_backend(
306
+ prompt: str, *, model: str, max_tokens: int,
307
+ ) -> str:
308
+ """Call the OpenAI Chat Completions API directly.
309
+
310
+ The API model id is derived from the allow-listed name by stripping
311
+ the ``"openai:"`` prefix — e.g. ``"openai:gpt-4o-mini"`` → ``"gpt-4o-mini"``.
312
+ Requires ``OPENAI_API_KEY`` in the environment. Returns empty string on
313
+ any transport or SDK failure (fail-closed).
314
+ """
315
+ openai_model = model.split(":", 1)[1] if model.startswith("openai:") else model
316
+ try:
317
+ import openai as _openai # type: ignore[import-not-found]
318
+ except Exception as exc: # noqa: BLE001
319
+ logger.debug("openai sdk unavailable: %s", exc)
320
+ return ""
321
+
322
+ try:
323
+ client = _openai.OpenAI()
324
+ completion = client.chat.completions.create(
325
+ model=openai_model,
326
+ max_tokens=max_tokens,
327
+ messages=[{"role": "user", "content": prompt}],
328
+ )
329
+ choices = getattr(completion, "choices", None)
330
+ if choices and len(choices) > 0:
331
+ msg = getattr(choices[0], "message", None)
332
+ if msg:
333
+ content = getattr(msg, "content", None)
334
+ if isinstance(content, str):
335
+ return content
336
+ return ""
337
+ except Exception as exc: # noqa: BLE001
338
+ logger.debug("OpenAI API backend failed: %s", exc)
339
+ return ""
340
+
341
+
304
342
  # ---------------------------------------------------------------------------
305
343
  # Backend registry — dispatches by (allow-listed) model id
306
344
  # ---------------------------------------------------------------------------
@@ -337,6 +375,8 @@ def _pick_backend(model: str) -> Callable[..., str]:
337
375
  """
338
376
  if model.startswith("ollama:"):
339
377
  return _call_ollama_backend
378
+ if model.startswith("openai:"):
379
+ return _call_openai_api_backend
340
380
  if model.startswith("claude-"):
341
381
  # Claude CLI path is an alternative — selected when an explicit
342
382
  # env flag is set. Default path is the Anthropic API backend.
@@ -39,6 +39,7 @@ CHEAPEST_CLAUDE = "claude-haiku-4-5"
39
39
  QUALITY_CLAUDE = "claude-sonnet-4-6"
40
40
  CHEAPEST_OLLAMA = "ollama:llama3"
41
41
  ALT_OLLAMA = "ollama:qwen2.5"
42
+ CHEAPEST_OPENAI = "openai:gpt-4o-mini"
42
43
 
43
44
  # Short-name → allow-listed model id. Single source of truth for aliasing;
44
45
  # ``skill_evolver`` re-exports these for backward compatibility.
@@ -48,6 +49,8 @@ _MODEL_ALIASES: dict[str, str] = {
48
49
  "ollama": CHEAPEST_OLLAMA,
49
50
  "ollama:llama3": CHEAPEST_OLLAMA,
50
51
  "ollama:qwen2.5": ALT_OLLAMA,
52
+ "openai": CHEAPEST_OPENAI,
53
+ "openai:gpt-4o-mini": CHEAPEST_OPENAI,
51
54
  CHEAPEST_CLAUDE: CHEAPEST_CLAUDE,
52
55
  QUALITY_CLAUDE: QUALITY_CLAUDE,
53
56
  }
@@ -78,11 +81,18 @@ class ResolvedModels:
78
81
 
79
82
 
80
83
  def _cheapest_for_backend(backend: str) -> str:
81
- """Lowest-cost allow-listed model for a detected backend."""
84
+ """Lowest-cost allow-listed model for a detected backend.
85
+
86
+ The configured backend is authoritative: when the caller explicitly
87
+ names a provider, models are drawn from that provider only —
88
+ never silently downgraded to a different vendor's offering.
89
+ """
82
90
  if backend == "ollama":
83
91
  return CHEAPEST_OLLAMA
84
- # claude CLI, anthropic API, and any non-Ollama backend evolution can
85
- # actually dispatch route through the cheapest Claude model.
92
+ if backend == "openai":
93
+ return CHEAPEST_OPENAI
94
+ # claude CLI, anthropic API, and any non-Ollama/non-OpenAI backend
95
+ # route through the cheapest Claude model.
86
96
  return CHEAPEST_CLAUDE
87
97
 
88
98
 
@@ -106,6 +116,8 @@ def _independent_verifier(
106
116
  """Pick a cheap verifier that differs from the generator when possible.
107
117
 
108
118
  - Ollama generator → the *other* local model (both free, distinct).
119
+ - OpenAI generator → same OpenAI model (single-provider constraint;
120
+ independence flag will be False — caller logs the condition).
109
121
  - Claude generator + local Ollama up → free local verifier.
110
122
  - Claude generator, no Ollama → the cheapest *different* Claude tier if
111
123
  the generator was a premium model; otherwise reuse the cheapest model
@@ -114,6 +126,9 @@ def _independent_verifier(
114
126
  """
115
127
  if mutation.startswith("ollama:"):
116
128
  return ALT_OLLAMA if mutation != ALT_OLLAMA else CHEAPEST_OLLAMA
129
+ if mutation.startswith("openai:"):
130
+ # Honor the configured provider: stay within OpenAI models.
131
+ return CHEAPEST_OPENAI
117
132
  if ollama_available:
118
133
  return CHEAPEST_OLLAMA
119
134
  if mutation != CHEAPEST_CLAUDE:
@@ -97,6 +97,9 @@ _SKILL_DENY_PATTERNS: tuple[str, ...] = (
97
97
  "os.environ", "subprocess", "exec(", "eval(", "__import__", "import os",
98
98
  "import subprocess", "pickle.loads", "curl ", "wget ", "rm -rf",
99
99
  "ANTHROPIC_API_KEY", "OPENAI_API_KEY", "AWS_SECRET", "/.ssh/", ".install_token",
100
+ # Semantic exfiltration patterns — skills must not instruct unauthorized
101
+ # data transfer or bypass of user consent.
102
+ "without consent", "exfiltrat",
100
103
  )
101
104
 
102
105