superlocalmemory 4.0.10 → 4.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (145) hide show
  1. package/.claude-plugin/marketplace.json +12 -2
  2. package/CHANGELOG.md +244 -0
  3. package/README.md +40 -75
  4. package/package.json +6 -3
  5. package/plugin/.claude-plugin/plugin.json +2 -2
  6. package/plugin/CLAUDE.md +3 -3
  7. package/plugin/agents/slm-governance-advisor.md +1 -1
  8. package/plugin/agents/slm-loop-runner.md +4 -4
  9. package/plugin/agents/slm-memory-advisor.md +1 -1
  10. package/plugin/agents/slm-optimize-advisor.md +1 -1
  11. package/plugin/requirements.txt +1 -1
  12. package/plugin/skills/slm-cache/SKILL.md +1 -1
  13. package/plugin/skills/slm-compress/SKILL.md +1 -1
  14. package/plugin/skills/slm-governance/SKILL.md +1 -1
  15. package/plugin/skills/slm-graph/SKILL.md +1 -1
  16. package/plugin/skills/slm-loop/SKILL.md +2 -2
  17. package/plugin/skills/slm-mesh/SKILL.md +1 -1
  18. package/plugin/skills/slm-profile/SKILL.md +5 -5
  19. package/plugin/skills/slm-recall/SKILL.md +102 -15
  20. package/plugin/skills/slm-remember/SKILL.md +35 -3
  21. package/plugin/skills/slm-scope/SKILL.md +1 -1
  22. package/plugin/skills/slm-session/SKILL.md +29 -3
  23. package/plugin/skills/slm-status/SKILL.md +1 -1
  24. package/plugin-src/rules/AGENTS.md +16 -8
  25. package/plugin-src/skills/slm-cache/SKILL.md +1 -1
  26. package/plugin-src/skills/slm-compress/SKILL.md +1 -1
  27. package/plugin-src/skills/slm-governance/SKILL.md +1 -1
  28. package/plugin-src/skills/slm-graph/SKILL.md +1 -1
  29. package/plugin-src/skills/slm-loop/SKILL.md +2 -2
  30. package/plugin-src/skills/slm-mesh/SKILL.md +1 -1
  31. package/plugin-src/skills/slm-profile/SKILL.md +5 -5
  32. package/plugin-src/skills/slm-recall/SKILL.md +102 -15
  33. package/plugin-src/skills/slm-remember/SKILL.md +35 -3
  34. package/plugin-src/skills/slm-scope/SKILL.md +1 -1
  35. package/plugin-src/skills/slm-session/SKILL.md +29 -3
  36. package/plugin-src/skills/slm-status/SKILL.md +1 -1
  37. package/pyproject.toml +1 -1
  38. package/src/superlocalmemory/__init__.py +1 -1
  39. package/src/superlocalmemory/cli/commands.py +357 -18
  40. package/src/superlocalmemory/cli/daemon.py +30 -0
  41. package/src/superlocalmemory/cli/db_migrate.py +71 -1
  42. package/src/superlocalmemory/cli/gdpr_cmd.py +15 -2
  43. package/src/superlocalmemory/cli/main.py +24 -2
  44. package/src/superlocalmemory/code_graph/database.py +44 -0
  45. package/src/superlocalmemory/compliance/gdpr.py +449 -39
  46. package/src/superlocalmemory/core/admission.py +231 -11
  47. package/src/superlocalmemory/core/backend_orchestrator.py +190 -84
  48. package/src/superlocalmemory/core/config.py +90 -11
  49. package/src/superlocalmemory/core/consolidation_engine.py +34 -0
  50. package/src/superlocalmemory/core/engine.py +140 -11
  51. package/src/superlocalmemory/core/graph_analyzer.py +76 -112
  52. package/src/superlocalmemory/core/graph_metrics.py +597 -0
  53. package/src/superlocalmemory/core/graph_pruner.py +121 -0
  54. package/src/superlocalmemory/core/maintenance_scheduler.py +205 -0
  55. package/src/superlocalmemory/core/mode_capability.py +111 -0
  56. package/src/superlocalmemory/core/ollama_validator.py +315 -0
  57. package/src/superlocalmemory/core/projection_drain.py +380 -0
  58. package/src/superlocalmemory/core/recall_pipeline.py +390 -3
  59. package/src/superlocalmemory/core/recall_worker.py +6 -3
  60. package/src/superlocalmemory/core/scale_autopromote.py +196 -0
  61. package/src/superlocalmemory/core/scale_engine.py +16 -2
  62. package/src/superlocalmemory/core/score_contract.py +21 -1
  63. package/src/superlocalmemory/core/session_identity.py +85 -0
  64. package/src/superlocalmemory/core/status_contract.py +108 -0
  65. package/src/superlocalmemory/core/worker_pool.py +4 -4
  66. package/src/superlocalmemory/core/working_memory.py +288 -0
  67. package/src/superlocalmemory/encoding/cognitive_consolidator.py +36 -6
  68. package/src/superlocalmemory/encoding/context_generator.py +1 -1
  69. package/src/superlocalmemory/encoding/entity_resolver.py +38 -0
  70. package/src/superlocalmemory/encoding/fact_extractor.py +18 -14
  71. package/src/superlocalmemory/encoding/prospective_markers.py +262 -0
  72. package/src/superlocalmemory/encoding/type_router.py +12 -12
  73. package/src/superlocalmemory/evolution/mutation_generator.py +30 -4
  74. package/src/superlocalmemory/graph/cozo_adjacency.py +122 -0
  75. package/src/superlocalmemory/graph/cozo_backend.py +103 -138
  76. package/src/superlocalmemory/hooks/portable_kit.py +10 -2
  77. package/src/superlocalmemory/learning/bandit.py +43 -0
  78. package/src/superlocalmemory/learning/consolidation_worker.py +54 -0
  79. package/src/superlocalmemory/learning/database.py +60 -3
  80. package/src/superlocalmemory/learning/entity_compiler.py +21 -58
  81. package/src/superlocalmemory/learning/feedback.py +3 -1
  82. package/src/superlocalmemory/learning/outcomes.py +47 -16
  83. package/src/superlocalmemory/learning/pattern_miner.py +28 -3
  84. package/src/superlocalmemory/learning/pattern_miner_constants.py +43 -0
  85. package/src/superlocalmemory/learning/pcos.py +291 -0
  86. package/src/superlocalmemory/learning/reward_from_outcomes.py +365 -0
  87. package/src/superlocalmemory/learning/reward_proxy.py +100 -10
  88. package/src/superlocalmemory/learning/signal_kinds.py +79 -0
  89. package/src/superlocalmemory/mcp/profiles.py +14 -2
  90. package/src/superlocalmemory/mcp/tools_active.py +2 -1
  91. package/src/superlocalmemory/mcp/tools_core.py +31 -3
  92. package/src/superlocalmemory/mcp/tools_v28.py +20 -1
  93. package/src/superlocalmemory/parameterization/pattern_extractor.py +14 -1
  94. package/src/superlocalmemory/parameterization/soft_prompt_generator.py +98 -0
  95. package/src/superlocalmemory/retrieval/bm25_channel.py +64 -3
  96. package/src/superlocalmemory/retrieval/channel_status.py +117 -0
  97. package/src/superlocalmemory/retrieval/engine.py +106 -11
  98. package/src/superlocalmemory/retrieval/entity_channel.py +210 -256
  99. package/src/superlocalmemory/retrieval/graph_adjacency.py +219 -0
  100. package/src/superlocalmemory/retrieval/scope_policy.py +20 -0
  101. package/src/superlocalmemory/retrieval/semantic_channel.py +47 -5
  102. package/src/superlocalmemory/retrieval/spreading.py +288 -0
  103. package/src/superlocalmemory/server/api.py +24 -5
  104. package/src/superlocalmemory/server/bandit_loops.py +17 -1
  105. package/src/superlocalmemory/server/rbac_enforce.py +26 -6
  106. package/src/superlocalmemory/server/recall_health.py +87 -10
  107. package/src/superlocalmemory/server/recall_serializer.py +9 -0
  108. package/src/superlocalmemory/server/routes/behavioral.py +75 -10
  109. package/src/superlocalmemory/server/routes/compliance.py +98 -18
  110. package/src/superlocalmemory/server/routes/config_api.py +186 -4
  111. package/src/superlocalmemory/server/routes/evolution.py +178 -0
  112. package/src/superlocalmemory/server/routes/ingest.py +8 -0
  113. package/src/superlocalmemory/server/routes/learning_telemetry.py +2 -1
  114. package/src/superlocalmemory/server/routes/memories.py +49 -7
  115. package/src/superlocalmemory/server/routes/timeline.py +4 -0
  116. package/src/superlocalmemory/server/routes/v3_api.py +191 -15
  117. package/src/superlocalmemory/server/ui.py +20 -4
  118. package/src/superlocalmemory/server/unified_daemon.py +241 -7
  119. package/src/superlocalmemory/storage/_migration_internals.py +54 -2
  120. package/src/superlocalmemory/storage/_schema_version.py +24 -3
  121. package/src/superlocalmemory/storage/database.py +477 -59
  122. package/src/superlocalmemory/storage/embedding_codec.py +71 -0
  123. package/src/superlocalmemory/storage/lineage_retention.py +236 -0
  124. package/src/superlocalmemory/storage/logical_edges.py +43 -2
  125. package/src/superlocalmemory/storage/migration_runner.py +119 -0
  126. package/src/superlocalmemory/storage/migrations/M043_quarantine_display_summaries.py +60 -36
  127. package/src/superlocalmemory/storage/migrations/M044_play_carries_its_own_evidence.py +127 -0
  128. package/src/superlocalmemory/storage/migrations/M045_fact_outcome_score.py +158 -0
  129. package/src/superlocalmemory/storage/migrations/M046_prospective_memory_has_its_own_name.py +620 -0
  130. package/src/superlocalmemory/storage/migrations/M047_fisher_vectors_are_stored_like_every_other_vector.py +306 -0
  131. package/src/superlocalmemory/storage/migrations/M048_upcoming_holds_only_what_is_upcoming.py +207 -0
  132. package/src/superlocalmemory/storage/migrations/M049_a_schema_version_marker_is_one_row.py +201 -0
  133. package/src/superlocalmemory/storage/migrations.py +18 -2
  134. package/src/superlocalmemory/storage/models.py +40 -1
  135. package/src/superlocalmemory/storage/projection_outbox.py +346 -0
  136. package/src/superlocalmemory/storage/retention_policy.py +860 -0
  137. package/src/superlocalmemory/storage/schema.py +35 -1
  138. package/src/superlocalmemory/storage/write_coordinator.py +19 -2
  139. package/src/superlocalmemory/trust/scorer.py +43 -1
  140. package/src/superlocalmemory/ui/index.html +9 -18
  141. package/src/superlocalmemory/ui/js/event-delegation.js +12 -1
  142. package/src/superlocalmemory/ui/js/od-health.js +28 -6
  143. package/src/superlocalmemory/ui/js/od-memories.js +19 -0
  144. package/src/superlocalmemory/ui/js/od-settings.js +87 -1
  145. package/src/superlocalmemory/ui/js/recall-lab.js +78 -3
@@ -26,6 +26,8 @@ from __future__ import annotations
26
26
 
27
27
  import functools
28
28
  import logging
29
+ import os
30
+ import sqlite3
29
31
  from typing import TYPE_CHECKING, FrozenSet
30
32
 
31
33
  from superlocalmemory.core.actor_context import ActorContext, ActorRole, Transport
@@ -122,6 +124,9 @@ _REQUIRED_MCP_GATES: frozenset[str] = frozenset({
122
124
  "link_memory_to_code",
123
125
  # Tranche G — mesh_inbox marks messages as read (POST to mesh broker)
124
126
  "mesh_inbox",
127
+ "fetch",
128
+ "list_recent",
129
+ "session_init",
125
130
  })
126
131
 
127
132
 
@@ -187,9 +192,17 @@ def _resolve_deployment() -> "DeploymentConfig":
187
192
  try:
188
193
  from superlocalmemory.core.config import load_deployment_config
189
194
  result = load_deployment_config(config_toml_path=config_path)
190
- except Exception as exc:
191
- logger.debug("admission: load_deployment_config raised (unexpected): %s", exc)
192
- return DEPLOYMENT_PERSONAL
195
+ except Exception as exc: # noqa: BLE001
196
+ # config.toml exists and parsed as TOML; only its interpretation failed.
197
+ # Returning PERSONAL here would hand owner access to a store that may
198
+ # well be enterprise. Step 2 already returned PERSONAL for the fresh
199
+ # install with no config at all, which is the case that needs to stay
200
+ # frictionless.
201
+ logger.warning(
202
+ "admission: config.toml is present but could not be interpreted "
203
+ "(%s) -- fail-closed (treating as enterprise).", exc,
204
+ )
205
+ return DEPLOYMENT_ENTERPRISE
193
206
 
194
207
  # D1 fail-closed: [deployment] section present but mode is unrecognized or
195
208
  # absent means someone tried to configure enterprise and a typo/omission
@@ -326,6 +339,173 @@ def admit(
326
339
  # @admits decorator for async MCP tools
327
340
  # ---------------------------------------------------------------------------
328
341
 
342
+
343
+ # ---------------------------------------------------------------------------
344
+ # Company mode has two switches, and this is where they become one
345
+ # ---------------------------------------------------------------------------
346
+
347
+ #: Environment variable carrying the caller's user session over a transport that
348
+ #: has no request to put a header on. The dashboard issues the same token, so
349
+ #: this is not a second credential system -- it is the only channel the MCP
350
+ #: surface offers for presenting the one that already exists.
351
+ _SESSION_ENV = "SLM_USER_SESSION"
352
+
353
+ _RBAC_ROLE_TO_ACTOR = {
354
+ "admin": ActorRole.ADMIN,
355
+ "member": ActorRole.MEMBER,
356
+ "viewer": ActorRole.VIEWER,
357
+ }
358
+
359
+
360
+ def _rbac_engine():
361
+ """The workspace's role store, or None when there is not one.
362
+
363
+ Built from the data root rather than from an HTTP app state, because the
364
+ callers here have no request. Failures return None, which then reads as
365
+ "personal mode" -- safe, because a workspace with no role store has no roles
366
+ to enforce.
367
+ """
368
+ try:
369
+ from superlocalmemory.access.rbac import RbacEngine
370
+ from superlocalmemory.infra.data_root import canonical_data_root
371
+
372
+ path = canonical_data_root() / "memory.db"
373
+ if not path.exists():
374
+ return None
375
+ return RbacEngine(str(path))
376
+ except Exception as exc: # noqa: BLE001
377
+ logger.debug("admission: no role store available: %s", exc)
378
+ return None
379
+
380
+
381
+ def _company_mode_active(deployment) -> bool:
382
+ """Whether a login is required, by EITHER of the two switches.
383
+
384
+ THE DEFECT THIS CLOSES
385
+
386
+ "Company mode" was two independent settings that nobody had joined up:
387
+ ``deployment`` in config.toml, which this module read, and ``require_login``
388
+ in the workspace's own settings, which the dashboard toggle writes and which
389
+ the HTTP routes read. Turning company mode on from the dashboard therefore
390
+ changed what HTTP would allow and changed nothing here.
391
+
392
+ Measured on a real store: with ``require_login`` on, two users configured,
393
+ and the viewer's role denying WRITE, an MCP write resolved to
394
+ ``local-operator`` with role ``owner`` and succeeded -- while the same write
395
+ over HTTP returned 401. The role check was not bypassed by a missing call;
396
+ it was bypassed because this path was still being told the workspace was
397
+ personal.
398
+
399
+ Either switch now means the same thing on every transport.
400
+ """
401
+ if getattr(deployment, "is_enterprise", False):
402
+ return True
403
+ rbac = _rbac_engine()
404
+ if rbac is None:
405
+ return False
406
+ try:
407
+ return bool(rbac.require_login())
408
+ except sqlite3.OperationalError as exc:
409
+ # "No such table" means the role tables were never created, which means
410
+ # roles were never set up, which is a personal install. Failing closed on
411
+ # THIS is not caution -- it is refusing every write on every store that
412
+ # has never used company mode, which is nearly all of them. It was caught
413
+ # by an existing test whose second MCP write started failing once a store
414
+ # file appeared in the data root.
415
+ if "no such table" in str(exc).lower():
416
+ logger.debug("admission: no role tables; personal workspace")
417
+ return False
418
+ logger.warning(
419
+ "admission: the login policy is unreadable (%s); treating the "
420
+ "workspace as requiring one", exc,
421
+ )
422
+ return True
423
+ except Exception as exc: # noqa: BLE001 -- an unreadable policy is not a licence
424
+ logger.warning(
425
+ "admission: cannot read the login policy (%s); treating the "
426
+ "workspace as requiring one", exc,
427
+ )
428
+ return True
429
+
430
+
431
+ def _target_profile(explicit: str = "") -> str:
432
+ """The workspace a call will actually touch.
433
+
434
+ An explicit argument wins when the caller supplies one. Otherwise this reads
435
+ the same ``profiles.json`` the engine and the HTTP layer read, because that
436
+ is where the write is going to land.
437
+
438
+ THE DEFECT THIS CLOSES
439
+
440
+ The role check used to key off ``kwargs.get("profile_id")``. ``remember``
441
+ has no such parameter, so every role lookup resolved against ``default``
442
+ while the write went to whichever workspace was active. A user who is an
443
+ admin on ``default`` and a viewer on ``team`` passed the check on
444
+ ``default`` and wrote to ``team`` -- which HTTP would have refused.
445
+ """
446
+ name = (explicit or "").strip()
447
+ if name:
448
+ return name
449
+ try:
450
+ from superlocalmemory.server.profile_runtime import current_request_profile
451
+
452
+ runtime = current_request_profile()
453
+ if runtime:
454
+ return str(runtime)
455
+ except Exception as exc: # noqa: BLE001 -- no request context is normal off HTTP
456
+ logger.debug("admission: no request profile in scope: %s", exc)
457
+ try:
458
+ import json as _json
459
+
460
+ from superlocalmemory.infra.data_root import canonical_data_root
461
+
462
+ config_file = canonical_data_root() / "profiles.json"
463
+ if config_file.exists():
464
+ data = _json.loads(config_file.read_text(encoding="utf-8"))
465
+ active = str(data.get("active_profile", "") or "").strip()
466
+ if active:
467
+ return active
468
+ except Exception as exc: # noqa: BLE001
469
+ logger.debug("admission: cannot read the active workspace: %s", exc)
470
+ return "default"
471
+
472
+
473
+ def _session_principal(profile: str) -> tuple[str, str, FrozenSet[ActorRole] | None]:
474
+ """Resolve the caller from a session token in the environment.
475
+
476
+ Returns ``(principal_id, raw_token, roles)``. An empty principal means the
477
+ caller could not be identified, which ``resolve_actor`` turns into ANONYMOUS
478
+ and ``admit`` then denies -- so an unset or expired token fails closed.
479
+ """
480
+ token = os.environ.get(_SESSION_ENV, "").strip()
481
+ if not token:
482
+ return "", "", None
483
+ rbac = _rbac_engine()
484
+ if rbac is None:
485
+ return "", "", None
486
+ try:
487
+ user = rbac.resolve_session(token)
488
+ except Exception as exc: # noqa: BLE001
489
+ logger.debug("admission: session lookup failed: %s", exc)
490
+ return "", "", None
491
+ if not user:
492
+ return "", "", None
493
+ role = None
494
+ try:
495
+ role = rbac.get_role(user["user_id"], profile or "default")
496
+ except Exception as exc: # noqa: BLE001
497
+ logger.debug("admission: role lookup failed: %s", exc)
498
+ # No membership on this workspace is not the same as MEMBER. Falling back to
499
+ # a write-capable default here would hand every authenticated user write
500
+ # access to every workspace on the machine.
501
+ actor_role = _RBAC_ROLE_TO_ACTOR.get(
502
+ getattr(role, "value", role) if role is not None else "", None,
503
+ )
504
+ if actor_role is None:
505
+ return user["user_id"], token, frozenset({ActorRole.ANONYMOUS})
506
+ return user["user_id"], token, frozenset({actor_role})
507
+
508
+
329
509
  def admits(kind: OperationKind):
330
510
  """Decorator that gates an async MCP tool function via the policy registry.
331
511
 
@@ -348,9 +528,21 @@ def admits(kind: OperationKind):
348
528
  @functools.wraps(fn)
349
529
  async def wrapper(*args, **kwargs):
350
530
  deployment = _resolve_deployment()
351
- tier = "enterprise" if deployment.is_enterprise else "personal"
352
- mode = "company" if deployment.is_enterprise else "local"
353
- actor = resolve_actor(Transport.MCP, tier=tier, mode=mode)
531
+ # Either switch means company mode. Reading only the config file is
532
+ # what let a dashboard toggle change HTTP and leave this transport
533
+ # writing as the machine owner.
534
+ company = _company_mode_active(deployment)
535
+ tier = "enterprise" if company else "personal"
536
+ mode = "company" if company else "local"
537
+ principal, token, roles = ("", "", None)
538
+ if company:
539
+ principal, token, roles = _session_principal(
540
+ _target_profile(kwargs.get("profile_id", "") or ""),
541
+ )
542
+ actor = resolve_actor(
543
+ Transport.MCP, tier=tier, mode=mode,
544
+ principal=principal, session=token, roles=roles,
545
+ )
354
546
  try:
355
547
  admit(kind, actor, mode=mode)
356
548
  except AdmissionDenied as exc:
@@ -373,6 +565,7 @@ def gate_cli_mutation(
373
565
  *,
374
566
  principal: str = "",
375
567
  roles: FrozenSet[ActorRole] | None = None,
568
+ profile: str = "",
376
569
  ) -> None:
377
570
  """Gate a CLI mutation command. Exits with code 1 if denied.
378
571
 
@@ -385,25 +578,48 @@ def gate_cli_mutation(
385
578
  kind : Operation being performed.
386
579
  principal : Authenticated CLI principal (from session store / login token).
387
580
  roles : Explicit roles for an authenticated enterprise user.
581
+ profile : Workspace the command will touch. Empty means the active one.
388
582
  """
389
583
  import sys
390
584
  deployment = _resolve_deployment()
391
- tier = "enterprise" if deployment.is_enterprise else "personal"
392
- mode = "company" if deployment.is_enterprise else "local"
585
+ # Either switch means company mode -- the same rule the MCP gate uses. This
586
+ # gate used to read config.toml alone, so turning per-user access on from
587
+ # the dashboard changed HTTP and MCP and left every CLI write running as the
588
+ # machine owner.
589
+ company = _company_mode_active(deployment)
590
+ tier = "enterprise" if company else "personal"
591
+ mode = "company" if company else "local"
592
+ session = ""
593
+ if company and not principal:
594
+ # A CLI caller identifies itself the same way an MCP caller does. With
595
+ # no token the principal stays empty, resolve_actor returns ANONYMOUS,
596
+ # and admit denies -- which is the intended answer for an unauthenticated
597
+ # write on a workspace that requires a login.
598
+ principal, session, resolved_roles = _session_principal(
599
+ _target_profile(profile),
600
+ )
601
+ if roles is None:
602
+ roles = resolved_roles
393
603
  actor = resolve_actor(
394
604
  Transport.CLI,
395
605
  tier=tier,
396
606
  mode=mode,
397
607
  principal=principal,
608
+ session=session,
398
609
  roles=roles,
399
610
  )
400
611
  try:
401
612
  admit(kind, actor, mode=mode)
402
613
  except AdmissionDenied as exc:
614
+ # Name something the reader can actually do. This used to say
615
+ # "log in with 'slm login'", and there is no such command -- so the one
616
+ # instruction the message gave was a dead end.
403
617
  print(
404
618
  f"[slm] Operation denied ({exc.decision.reason}). "
405
- "This workspace requires authentication. "
406
- "Log in with 'slm login' or contact your workspace administrator.",
619
+ "This workspace requires a signed-in user. Sign in on the dashboard "
620
+ "(slm dashboard), copy your session token, and put it in the "
621
+ "SLM_USER_SESSION environment variable -- or ask whoever "
622
+ "administers this workspace for access.",
407
623
  flush=True,
408
624
  )
409
625
  sys.exit(1)
@@ -515,7 +731,11 @@ def enforce_read_scope(
515
731
  is left alone so the server default applies.
516
732
  """
517
733
  deployment = _resolve_deployment()
518
- if not deployment.is_enterprise:
734
+ # Both switches, for the same reason the write gates read both: a dashboard
735
+ # toggle used to leave this path unclamped, so a viewer could ask for
736
+ # include_global=True over MCP and pull another workspace's facts into the
737
+ # candidate set while HTTP refused the same request.
738
+ if not _company_mode_active(deployment):
519
739
  return include_global, include_shared
520
740
 
521
741
  reg = registry if registry is not None else _DEFAULT_REGISTRY
@@ -21,6 +21,9 @@ import logging
21
21
  from pathlib import Path
22
22
  from typing import TYPE_CHECKING, Any
23
23
 
24
+ from superlocalmemory.core.projection_drain import ProjectionDrain
25
+ from superlocalmemory.storage import projection_outbox
26
+
24
27
  if TYPE_CHECKING:
25
28
  from superlocalmemory.core.config import SLMConfig
26
29
  from superlocalmemory.storage.database import DatabaseManager
@@ -80,6 +83,12 @@ class BackendOrchestrator:
80
83
  self._lancedb: Any = None
81
84
  self._tiers: Any = None
82
85
  self._backend_cache: dict[str, str] = {}
86
+ # Given accessors, not backends: a promotion or a rollback replaces
87
+ # them underneath the worker, and a reference captured here would keep
88
+ # writing into the projection that was just swapped out.
89
+ self._drain = ProjectionDrain(
90
+ db, self.get_graph_backend, self.get_vector_backend,
91
+ )
83
92
 
84
93
  # ------------------------------------------------------------------
85
94
  # Daemon Startup
@@ -107,6 +116,19 @@ class BackendOrchestrator:
107
116
 
108
117
  self._recover_interrupted_scale_promotion()
109
118
 
119
+ # Reconcile what the config CLAIMS against what is on disk, before
120
+ # anything reads either -- and before the early return below, because
121
+ # the stores that need reconciling are exactly the ones that take it.
122
+ #
123
+ # This used to sit after that return, so it ran only for a store already
124
+ # in the promoted state. The real case it was written for is a store
125
+ # whose settings name a graph and a vector backend, whose state is
126
+ # `verified` rather than `promoted`, and where neither directory exists:
127
+ # the settings kept the claim, the reconcile never ran, and every
128
+ # restart preserved it. The test asserted the call appeared before
129
+ # another call in the source text, which is true either way.
130
+ self._reconcile_backend_selection()
131
+
110
132
  # Backends may be installed with the product, but installing a wheel
111
133
  # is not authorization to mutate an existing data root. Only a
112
134
  # verified, explicit promotion may initialize and migrate projections.
@@ -120,6 +142,11 @@ class BackendOrchestrator:
120
142
  # SQLite graph. A no-op (and never even starts the build) for the
121
143
  # vast majority of installs, which sit far below the threshold.
122
144
  self._maybe_schedule_auto_promote()
145
+ # Started even with no projection open. A pass with no backend
146
+ # returns without touching a row, and starting it here means a
147
+ # promotion that completes mid-session has a worker waiting for it
148
+ # rather than a queue nobody is reading.
149
+ self._drain.start()
123
150
  return
124
151
 
125
152
  # 3. Initialize CozoDB if available
@@ -145,9 +172,14 @@ class BackendOrchestrator:
145
172
  except Exception as exc:
146
173
  logger.warning("TierManager backend registration failed (non-fatal): %s", exc)
147
174
 
148
- logger.info("BackendOrchestrator: daemon ready (cozo=%s, lancedb=%s)",
149
- "active" if self._cozo and self._cozo_status() == "active" else "off",
150
- "active" if self._lancedb and self._lancedb_status() == "active" else "off")
175
+ self._drain.start()
176
+
177
+ logger.info(
178
+ "BackendOrchestrator: daemon ready (cozo=%s, lancedb=%s, queued=%d)",
179
+ "active" if self._cozo and self._cozo_status() == "active" else "off",
180
+ "active" if self._lancedb and self._lancedb_status() == "active" else "off",
181
+ projection_outbox.depth(self._db),
182
+ )
151
183
 
152
184
  def _maybe_schedule_auto_promote(self) -> None:
153
185
  """Schedule a delayed, one-shot scale auto-promote check (v3.8.5).
@@ -164,7 +196,23 @@ class BackendOrchestrator:
164
196
  cfg = self._config
165
197
  if not getattr(cfg, "scale_auto_promote_enabled", True):
166
198
  return
167
- if getattr(cfg, "scale_engine_state", "local_core") != "local_core":
199
+ # Only a store that has finished promoting has nothing left to do.
200
+ #
201
+ # This used to skip every state except ``local_core``, which made
202
+ # ``prepared`` and ``verified`` terminal: the daemon above returns
203
+ # early for anything that is not ``promoted``, so the backends never
204
+ # started, and this refused to finish the promotion that would have
205
+ # started them. A store that got as far as building and checking its
206
+ # projection then sat on SQLite forever while its own config named Cozo
207
+ # and LanceDB as the backends — measured on a real store whose
208
+ # ``backend_status`` read lancedb=not_initialized under
209
+ # ``scale_engine_state=verified``.
210
+ #
211
+ # ``run_auto_promote`` already resumes a half-finished stage
212
+ # (``_resumable_stage``) and applies the size threshold and the
213
+ # repair-required check itself, so it is the right place for every
214
+ # decision except "there is nothing left to do".
215
+ if str(getattr(cfg, "scale_engine_state", "local_core")).lower() == "promoted":
168
216
  return
169
217
  try:
170
218
  delay = float(os.environ.get("SLM_AUTO_PROMOTE_DELAY_S", "300"))
@@ -198,11 +246,14 @@ class BackendOrchestrator:
198
246
  import os
199
247
 
200
248
  cfg = self._config
201
- if getattr(cfg, "scale_engine_state", "local_core") != "local_core":
249
+ # Same rule as the scheduler that armed this timer: only a store
250
+ # that has finished has nothing left to do. Fixing the scheduler
251
+ # alone would have armed a timer whose callback still refused.
252
+ if str(getattr(cfg, "scale_engine_state", "local_core")).lower() == "promoted":
202
253
  return
203
254
  threshold = int(
204
255
  os.environ.get("SLM_AUTO_PROMOTE_MIN_EDGES", "")
205
- or getattr(cfg, "scale_auto_promote_min_edges", 1_000_000)
256
+ or getattr(cfg, "scale_auto_promote_min_edges", 100_000)
206
257
  )
207
258
  edges = self._count_default_edges()
208
259
  if edges < threshold:
@@ -237,6 +288,75 @@ class BackendOrchestrator:
237
288
  exc,
238
289
  )
239
290
 
291
+ def _reconcile_backend_selection(self) -> None:
292
+ """Stop the config claiming a backend the store does not have.
293
+
294
+ THE STATE THIS REPAIRS
295
+
296
+ On a real store: ``graph_backend='cozo'``, ``vector_backend='lancedb'``,
297
+ ``scale_engine_state='verified'`` -- and neither the ``cozo/`` nor the
298
+ ``lance/`` directory existed, with no promotion journal to explain it.
299
+ Something wrote the selection a completed promotion writes, without a
300
+ promotion having completed.
301
+
302
+ Nothing corrected it. ``recover_interrupted_promotion`` acts only when a
303
+ journal exists, so with no journal it returns immediately and the claim
304
+ survives every restart. The dashboard then reports the configured backend
305
+ while retrieval uses SQLite, which is the disagreement a person notices
306
+ last and trusts first.
307
+
308
+ WHAT THIS DOES NOT DO
309
+
310
+ It does not disable anything. ``auto`` still detects and initialises both
311
+ projections when their libraries are installed, so the only thing removed
312
+ is the false claim. It leaves ``verified`` alone -- that is a legitimate
313
+ waypoint meaning "parity checked, not yet promoted" -- and only resets
314
+ ``promoted``, which asserts a swap that plainly did not happen. And it
315
+ never touches a selection whose directory is present, nor one with a
316
+ journal still open, because those belong to the promotion lifecycle.
317
+ """
318
+ try:
319
+ from superlocalmemory.core.scale_engine import ScaleEngineManager
320
+
321
+ manager = ScaleEngineManager(self._config, profile_id="default")
322
+ if manager.promotion_journal_path.exists():
323
+ return # the recovery path owns this
324
+ cozo_path, lance_path = manager.active_paths
325
+ except Exception as exc: # noqa: BLE001 -- reconciliation is best effort
326
+ logger.debug("Backend reconciliation skipped: %s", exc)
327
+ return
328
+
329
+ corrections: list[str] = []
330
+ graph = getattr(self._config, "graph_backend", "auto") or "auto"
331
+ if graph not in ("auto", "sqlite") and not cozo_path.exists():
332
+ corrections.append(f"graph_backend {graph!r} -> 'auto' (no {cozo_path.name}/)")
333
+ self._config.graph_backend = "auto"
334
+ vector = getattr(self._config, "vector_backend", "auto") or "auto"
335
+ if vector not in ("auto", "sqlite-vec") and not lance_path.exists():
336
+ corrections.append(
337
+ f"vector_backend {vector!r} -> 'auto' (no {lance_path.name}/)"
338
+ )
339
+ self._config.vector_backend = "auto"
340
+ state = getattr(self._config, "scale_engine_state", "") or ""
341
+ if state == "promoted" and not (cozo_path.exists() or lance_path.exists()):
342
+ corrections.append("scale_engine_state 'promoted' -> 'local_core'")
343
+ self._config.scale_engine_state = "local_core"
344
+
345
+ if not corrections:
346
+ return
347
+ try:
348
+ self._config.save()
349
+ except Exception as exc: # noqa: BLE001
350
+ logger.warning(
351
+ "Backend selection is inconsistent with the data directory and "
352
+ "could not be corrected (%s): %s", exc, "; ".join(corrections),
353
+ )
354
+ return
355
+ logger.warning(
356
+ "Backend selection did not match the data directory; corrected: %s",
357
+ "; ".join(corrections),
358
+ )
359
+
240
360
  def _recover_interrupted_scale_promotion(self) -> None:
241
361
  """Repair an interrupted promotion; never auto-mutate a legacy root."""
242
362
  try:
@@ -261,90 +381,44 @@ class BackendOrchestrator:
261
381
  # ------------------------------------------------------------------
262
382
 
263
383
  def sync_new_fact(self, fact: Any) -> None:
264
- """Sync a newly stored fact to CozoDB and LanceDB.
265
-
266
- Called AFTER SQLite write in store_pipeline.
267
- Non-blocking, best-effort. Failures are logged, not raised.
384
+ """Signal that a stored fact needs projecting.
385
+
386
+ The projection itself is not written here. It used to be — inline, on
387
+ the caller's thread, with every failure swallowed into a debug line —
388
+ and that is the defect the outbox replaced. The intent to project was
389
+ already committed to ``projection_outbox`` in the same SQLite
390
+ transaction as the fact, so all that is left to do is wake the worker
391
+ that owns the projections.
392
+
393
+ Kept as a method because callers name this operation, and because a
394
+ caller that reaches it without an outbox row (an old store, mid-upgrade)
395
+ should still get its fact projected rather than silently skipped.
268
396
  """
269
- try:
270
- tier = getattr(fact, "lifecycle", "active")
271
- except Exception:
272
- tier = "active"
273
-
274
- if tier in ("active", "warm"):
275
- if self._cozo and self._cozo_status() == "active":
276
- self._sync_fact_entities(fact)
277
-
278
- if self._lancedb and self._lancedb_status() == "active":
279
- self._sync_fact_embedding(fact)
280
-
281
- def _sync_fact_entities(self, fact: Any) -> None:
282
- """Synchronize one fact's canonical entity bridge and fact edges."""
283
- try:
284
- # Retrying ingestion must not retain stale fact/entity links.
285
- self._cozo.remove_fact(fact.fact_id)
286
- entities = getattr(fact, "canonical_entities", []) or []
287
- profile_id = getattr(fact, "profile_id", "default") or "default"
288
- for eid in entities:
289
- rows = self._db.execute(
290
- "SELECT canonical_name, entity_type, fact_count FROM canonical_entities "
291
- "WHERE entity_id = ? AND profile_id = ?",
292
- (eid, profile_id),
293
- )
294
- if rows:
295
- entity = dict(rows[0])
296
- self._cozo.add_entity(
297
- eid,
298
- entity.get("canonical_name") or eid,
299
- entity.get("entity_type") or "concept",
300
- {"fact_count": int(entity.get("fact_count") or 0)},
301
- profile_id,
302
- )
303
- self._cozo.add_fact_entities(fact.fact_id, entities, profile_id)
304
- for row in self._db.execute(
305
- "SELECT source_id, target_id, edge_type, weight FROM graph_edges "
306
- "WHERE profile_id = ? AND (source_id = ? OR target_id = ?)",
307
- (profile_id, fact.fact_id, fact.fact_id),
308
- ):
309
- edge = dict(row)
310
- self._cozo.add_edge(
311
- edge["source_id"], edge["target_id"], edge.get("edge_type") or "related",
312
- float(edge.get("weight") or 1.0), profile_id=profile_id,
313
- )
314
- except Exception as exc:
315
- logger.debug("CozoDB incremental sync skipped: %s", exc)
316
-
317
- def _sync_fact_embedding(self, fact: Any) -> None:
318
- """Sync fact's embedding to LanceDB."""
319
- try:
320
- embedding = getattr(fact, "embedding", None)
321
- if embedding:
322
- tier = getattr(fact, "lifecycle", "active")
323
- self._lancedb.add_vectors(
324
- [fact.fact_id], [embedding], [tier],
325
- getattr(fact, "profile_id", "default") or "default",
326
- )
327
- except Exception as exc:
328
- logger.debug("LanceDB incremental sync skipped: %s", exc)
397
+ fact_id = getattr(fact, "fact_id", None)
398
+ if fact_id:
399
+ projection_outbox.enqueue(
400
+ self._db, fact_id, getattr(fact, "profile_id", None) or "default",
401
+ )
402
+ self._drain.notify()
329
403
 
330
404
  def sync_deleted_fact(self, fact_id: str) -> None:
331
- """Remove a fact from derived projections after canonical deletion."""
332
- if self._cozo and self._cozo_status() == "active":
333
- try:
334
- self._cozo.remove_fact(fact_id)
335
- except Exception as exc:
336
- logger.warning("Cozo deletion sync failed for %s: %s", fact_id[:16], exc)
337
- if self._lancedb and self._lancedb_status() == "active":
338
- try:
339
- self._lancedb.remove_vector(fact_id)
340
- except Exception as exc:
341
- logger.warning("Lance deletion sync failed for %s: %s", fact_id[:16], exc)
405
+ """Signal that a deleted fact must leave the projections.
406
+
407
+ A forgotten memory still present in the graph or the vector index is
408
+ still recallable, so the removal is queued with the same durability as
409
+ the delete itself.
410
+ """
411
+ if fact_id:
412
+ projection_outbox.enqueue_for_fact(
413
+ self._db, fact_id, projection_outbox.OP_DELETE,
414
+ )
415
+ self._drain.notify()
342
416
 
343
417
  def sync_changed_fact(self, fact_id: str) -> None:
344
418
  """Refresh projections after an authorized canonical fact update."""
345
- fact = self._db.get_fact(fact_id)
346
- if fact is not None:
347
- self.sync_new_fact(fact)
419
+ if fact_id:
420
+ projection_outbox.enqueue_for_fact(self._db, fact_id)
421
+ self._drain.notify()
348
422
 
349
423
  # ------------------------------------------------------------------
350
424
  # Backend Access
@@ -411,8 +485,40 @@ class BackendOrchestrator:
411
485
  "LanceDB not active. Install: pip install superlocalmemory[lancedb]"
412
486
  )
413
487
 
488
+ outbox = projection_outbox.health(self._db)
489
+ outbox["draining"] = self._drain.running
490
+ result["projection_queue"] = outbox
491
+ if outbox["stalled"]:
492
+ result["warnings"].append(
493
+ f"{outbox['stalled']} memory/memories could not be projected into "
494
+ "the graph or vector store. Run `slm doctor` for the ids."
495
+ )
496
+
414
497
  return result
415
498
 
499
+ # ------------------------------------------------------------------
500
+ # Projection queue
501
+ # ------------------------------------------------------------------
502
+
503
+ def drain_projections(self, limit: int = 200) -> dict[str, Any]:
504
+ """Apply queued facts now, on the calling thread.
505
+
506
+ For the CLI, for a repair pass, and for any caller that needs the
507
+ projections current before it reads them rather than a few milliseconds
508
+ later. Ordinary writes do not need this — they signal the worker.
509
+ """
510
+ return self._drain.drain_once(limit=limit).as_dict()
511
+
512
+ def outbox_health(self) -> dict[str, Any]:
513
+ """Queue depth and stalled count, for the status surfaces."""
514
+ health = projection_outbox.health(self._db)
515
+ health["draining"] = self._drain.running
516
+ return health
517
+
518
+ def stop(self) -> None:
519
+ """Stop the drain worker. For daemon shutdown and for tests."""
520
+ self._drain.stop()
521
+
416
522
  # ------------------------------------------------------------------
417
523
  # Internal: Detection
418
524
  # ------------------------------------------------------------------