superlocalmemory 4.0.10 → 4.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +12 -2
- package/CHANGELOG.md +244 -0
- package/README.md +40 -75
- package/package.json +6 -3
- package/plugin/.claude-plugin/plugin.json +2 -2
- package/plugin/CLAUDE.md +3 -3
- package/plugin/agents/slm-governance-advisor.md +1 -1
- package/plugin/agents/slm-loop-runner.md +4 -4
- package/plugin/agents/slm-memory-advisor.md +1 -1
- package/plugin/agents/slm-optimize-advisor.md +1 -1
- package/plugin/requirements.txt +1 -1
- package/plugin/skills/slm-cache/SKILL.md +1 -1
- package/plugin/skills/slm-compress/SKILL.md +1 -1
- package/plugin/skills/slm-governance/SKILL.md +1 -1
- package/plugin/skills/slm-graph/SKILL.md +1 -1
- package/plugin/skills/slm-loop/SKILL.md +2 -2
- package/plugin/skills/slm-mesh/SKILL.md +1 -1
- package/plugin/skills/slm-profile/SKILL.md +5 -5
- package/plugin/skills/slm-recall/SKILL.md +102 -15
- package/plugin/skills/slm-remember/SKILL.md +35 -3
- package/plugin/skills/slm-scope/SKILL.md +1 -1
- package/plugin/skills/slm-session/SKILL.md +29 -3
- package/plugin/skills/slm-status/SKILL.md +1 -1
- package/plugin-src/rules/AGENTS.md +16 -8
- package/plugin-src/skills/slm-cache/SKILL.md +1 -1
- package/plugin-src/skills/slm-compress/SKILL.md +1 -1
- package/plugin-src/skills/slm-governance/SKILL.md +1 -1
- package/plugin-src/skills/slm-graph/SKILL.md +1 -1
- package/plugin-src/skills/slm-loop/SKILL.md +2 -2
- package/plugin-src/skills/slm-mesh/SKILL.md +1 -1
- package/plugin-src/skills/slm-profile/SKILL.md +5 -5
- package/plugin-src/skills/slm-recall/SKILL.md +102 -15
- package/plugin-src/skills/slm-remember/SKILL.md +35 -3
- package/plugin-src/skills/slm-scope/SKILL.md +1 -1
- package/plugin-src/skills/slm-session/SKILL.md +29 -3
- package/plugin-src/skills/slm-status/SKILL.md +1 -1
- package/pyproject.toml +1 -1
- package/src/superlocalmemory/__init__.py +1 -1
- package/src/superlocalmemory/cli/commands.py +357 -18
- package/src/superlocalmemory/cli/daemon.py +30 -0
- package/src/superlocalmemory/cli/db_migrate.py +71 -1
- package/src/superlocalmemory/cli/gdpr_cmd.py +15 -2
- package/src/superlocalmemory/cli/main.py +24 -2
- package/src/superlocalmemory/code_graph/database.py +44 -0
- package/src/superlocalmemory/compliance/gdpr.py +449 -39
- package/src/superlocalmemory/core/admission.py +231 -11
- package/src/superlocalmemory/core/backend_orchestrator.py +190 -84
- package/src/superlocalmemory/core/config.py +90 -11
- package/src/superlocalmemory/core/consolidation_engine.py +34 -0
- package/src/superlocalmemory/core/engine.py +140 -11
- package/src/superlocalmemory/core/graph_analyzer.py +76 -112
- package/src/superlocalmemory/core/graph_metrics.py +597 -0
- package/src/superlocalmemory/core/graph_pruner.py +121 -0
- package/src/superlocalmemory/core/maintenance_scheduler.py +205 -0
- package/src/superlocalmemory/core/mode_capability.py +111 -0
- package/src/superlocalmemory/core/ollama_validator.py +315 -0
- package/src/superlocalmemory/core/projection_drain.py +380 -0
- package/src/superlocalmemory/core/recall_pipeline.py +390 -3
- package/src/superlocalmemory/core/recall_worker.py +6 -3
- package/src/superlocalmemory/core/scale_autopromote.py +196 -0
- package/src/superlocalmemory/core/scale_engine.py +16 -2
- package/src/superlocalmemory/core/score_contract.py +21 -1
- package/src/superlocalmemory/core/session_identity.py +85 -0
- package/src/superlocalmemory/core/status_contract.py +108 -0
- package/src/superlocalmemory/core/worker_pool.py +4 -4
- package/src/superlocalmemory/core/working_memory.py +288 -0
- package/src/superlocalmemory/encoding/cognitive_consolidator.py +36 -6
- package/src/superlocalmemory/encoding/context_generator.py +1 -1
- package/src/superlocalmemory/encoding/entity_resolver.py +38 -0
- package/src/superlocalmemory/encoding/fact_extractor.py +18 -14
- package/src/superlocalmemory/encoding/prospective_markers.py +262 -0
- package/src/superlocalmemory/encoding/type_router.py +12 -12
- package/src/superlocalmemory/evolution/mutation_generator.py +30 -4
- package/src/superlocalmemory/graph/cozo_adjacency.py +122 -0
- package/src/superlocalmemory/graph/cozo_backend.py +103 -138
- package/src/superlocalmemory/hooks/portable_kit.py +10 -2
- package/src/superlocalmemory/learning/bandit.py +43 -0
- package/src/superlocalmemory/learning/consolidation_worker.py +54 -0
- package/src/superlocalmemory/learning/database.py +60 -3
- package/src/superlocalmemory/learning/entity_compiler.py +21 -58
- package/src/superlocalmemory/learning/feedback.py +3 -1
- package/src/superlocalmemory/learning/outcomes.py +47 -16
- package/src/superlocalmemory/learning/pattern_miner.py +28 -3
- package/src/superlocalmemory/learning/pattern_miner_constants.py +43 -0
- package/src/superlocalmemory/learning/pcos.py +291 -0
- package/src/superlocalmemory/learning/reward_from_outcomes.py +365 -0
- package/src/superlocalmemory/learning/reward_proxy.py +100 -10
- package/src/superlocalmemory/learning/signal_kinds.py +79 -0
- package/src/superlocalmemory/mcp/profiles.py +14 -2
- package/src/superlocalmemory/mcp/tools_active.py +2 -1
- package/src/superlocalmemory/mcp/tools_core.py +31 -3
- package/src/superlocalmemory/mcp/tools_v28.py +20 -1
- package/src/superlocalmemory/parameterization/pattern_extractor.py +14 -1
- package/src/superlocalmemory/parameterization/soft_prompt_generator.py +98 -0
- package/src/superlocalmemory/retrieval/bm25_channel.py +64 -3
- package/src/superlocalmemory/retrieval/channel_status.py +117 -0
- package/src/superlocalmemory/retrieval/engine.py +106 -11
- package/src/superlocalmemory/retrieval/entity_channel.py +210 -256
- package/src/superlocalmemory/retrieval/graph_adjacency.py +219 -0
- package/src/superlocalmemory/retrieval/scope_policy.py +20 -0
- package/src/superlocalmemory/retrieval/semantic_channel.py +47 -5
- package/src/superlocalmemory/retrieval/spreading.py +288 -0
- package/src/superlocalmemory/server/api.py +24 -5
- package/src/superlocalmemory/server/bandit_loops.py +17 -1
- package/src/superlocalmemory/server/rbac_enforce.py +26 -6
- package/src/superlocalmemory/server/recall_health.py +87 -10
- package/src/superlocalmemory/server/recall_serializer.py +9 -0
- package/src/superlocalmemory/server/routes/behavioral.py +75 -10
- package/src/superlocalmemory/server/routes/compliance.py +98 -18
- package/src/superlocalmemory/server/routes/config_api.py +186 -4
- package/src/superlocalmemory/server/routes/evolution.py +178 -0
- package/src/superlocalmemory/server/routes/ingest.py +8 -0
- package/src/superlocalmemory/server/routes/learning_telemetry.py +2 -1
- package/src/superlocalmemory/server/routes/memories.py +49 -7
- package/src/superlocalmemory/server/routes/timeline.py +4 -0
- package/src/superlocalmemory/server/routes/v3_api.py +191 -15
- package/src/superlocalmemory/server/ui.py +20 -4
- package/src/superlocalmemory/server/unified_daemon.py +241 -7
- package/src/superlocalmemory/storage/_migration_internals.py +54 -2
- package/src/superlocalmemory/storage/_schema_version.py +24 -3
- package/src/superlocalmemory/storage/database.py +477 -59
- package/src/superlocalmemory/storage/embedding_codec.py +71 -0
- package/src/superlocalmemory/storage/lineage_retention.py +236 -0
- package/src/superlocalmemory/storage/logical_edges.py +43 -2
- package/src/superlocalmemory/storage/migration_runner.py +119 -0
- package/src/superlocalmemory/storage/migrations/M043_quarantine_display_summaries.py +60 -36
- package/src/superlocalmemory/storage/migrations/M044_play_carries_its_own_evidence.py +127 -0
- package/src/superlocalmemory/storage/migrations/M045_fact_outcome_score.py +158 -0
- package/src/superlocalmemory/storage/migrations/M046_prospective_memory_has_its_own_name.py +620 -0
- package/src/superlocalmemory/storage/migrations/M047_fisher_vectors_are_stored_like_every_other_vector.py +306 -0
- package/src/superlocalmemory/storage/migrations/M048_upcoming_holds_only_what_is_upcoming.py +207 -0
- package/src/superlocalmemory/storage/migrations/M049_a_schema_version_marker_is_one_row.py +201 -0
- package/src/superlocalmemory/storage/migrations.py +18 -2
- package/src/superlocalmemory/storage/models.py +40 -1
- package/src/superlocalmemory/storage/projection_outbox.py +346 -0
- package/src/superlocalmemory/storage/retention_policy.py +860 -0
- package/src/superlocalmemory/storage/schema.py +35 -1
- package/src/superlocalmemory/storage/write_coordinator.py +19 -2
- package/src/superlocalmemory/trust/scorer.py +43 -1
- package/src/superlocalmemory/ui/index.html +9 -18
- package/src/superlocalmemory/ui/js/event-delegation.js +12 -1
- package/src/superlocalmemory/ui/js/od-health.js +28 -6
- package/src/superlocalmemory/ui/js/od-memories.js +19 -0
- package/src/superlocalmemory/ui/js/od-settings.js +87 -1
- package/src/superlocalmemory/ui/js/recall-lab.js +78 -3
|
@@ -26,6 +26,8 @@ from __future__ import annotations
|
|
|
26
26
|
|
|
27
27
|
import functools
|
|
28
28
|
import logging
|
|
29
|
+
import os
|
|
30
|
+
import sqlite3
|
|
29
31
|
from typing import TYPE_CHECKING, FrozenSet
|
|
30
32
|
|
|
31
33
|
from superlocalmemory.core.actor_context import ActorContext, ActorRole, Transport
|
|
@@ -122,6 +124,9 @@ _REQUIRED_MCP_GATES: frozenset[str] = frozenset({
|
|
|
122
124
|
"link_memory_to_code",
|
|
123
125
|
# Tranche G — mesh_inbox marks messages as read (POST to mesh broker)
|
|
124
126
|
"mesh_inbox",
|
|
127
|
+
"fetch",
|
|
128
|
+
"list_recent",
|
|
129
|
+
"session_init",
|
|
125
130
|
})
|
|
126
131
|
|
|
127
132
|
|
|
@@ -187,9 +192,17 @@ def _resolve_deployment() -> "DeploymentConfig":
|
|
|
187
192
|
try:
|
|
188
193
|
from superlocalmemory.core.config import load_deployment_config
|
|
189
194
|
result = load_deployment_config(config_toml_path=config_path)
|
|
190
|
-
except Exception as exc:
|
|
191
|
-
|
|
192
|
-
|
|
195
|
+
except Exception as exc: # noqa: BLE001
|
|
196
|
+
# config.toml exists and parsed as TOML; only its interpretation failed.
|
|
197
|
+
# Returning PERSONAL here would hand owner access to a store that may
|
|
198
|
+
# well be enterprise. Step 2 already returned PERSONAL for the fresh
|
|
199
|
+
# install with no config at all, which is the case that needs to stay
|
|
200
|
+
# frictionless.
|
|
201
|
+
logger.warning(
|
|
202
|
+
"admission: config.toml is present but could not be interpreted "
|
|
203
|
+
"(%s) -- fail-closed (treating as enterprise).", exc,
|
|
204
|
+
)
|
|
205
|
+
return DEPLOYMENT_ENTERPRISE
|
|
193
206
|
|
|
194
207
|
# D1 fail-closed: [deployment] section present but mode is unrecognized or
|
|
195
208
|
# absent means someone tried to configure enterprise and a typo/omission
|
|
@@ -326,6 +339,173 @@ def admit(
|
|
|
326
339
|
# @admits decorator for async MCP tools
|
|
327
340
|
# ---------------------------------------------------------------------------
|
|
328
341
|
|
|
342
|
+
|
|
343
|
+
# ---------------------------------------------------------------------------
|
|
344
|
+
# Company mode has two switches, and this is where they become one
|
|
345
|
+
# ---------------------------------------------------------------------------
|
|
346
|
+
|
|
347
|
+
#: Environment variable carrying the caller's user session over a transport that
|
|
348
|
+
#: has no request to put a header on. The dashboard issues the same token, so
|
|
349
|
+
#: this is not a second credential system -- it is the only channel the MCP
|
|
350
|
+
#: surface offers for presenting the one that already exists.
|
|
351
|
+
_SESSION_ENV = "SLM_USER_SESSION"
|
|
352
|
+
|
|
353
|
+
_RBAC_ROLE_TO_ACTOR = {
|
|
354
|
+
"admin": ActorRole.ADMIN,
|
|
355
|
+
"member": ActorRole.MEMBER,
|
|
356
|
+
"viewer": ActorRole.VIEWER,
|
|
357
|
+
}
|
|
358
|
+
|
|
359
|
+
|
|
360
|
+
def _rbac_engine():
|
|
361
|
+
"""The workspace's role store, or None when there is not one.
|
|
362
|
+
|
|
363
|
+
Built from the data root rather than from an HTTP app state, because the
|
|
364
|
+
callers here have no request. Failures return None, which then reads as
|
|
365
|
+
"personal mode" -- safe, because a workspace with no role store has no roles
|
|
366
|
+
to enforce.
|
|
367
|
+
"""
|
|
368
|
+
try:
|
|
369
|
+
from superlocalmemory.access.rbac import RbacEngine
|
|
370
|
+
from superlocalmemory.infra.data_root import canonical_data_root
|
|
371
|
+
|
|
372
|
+
path = canonical_data_root() / "memory.db"
|
|
373
|
+
if not path.exists():
|
|
374
|
+
return None
|
|
375
|
+
return RbacEngine(str(path))
|
|
376
|
+
except Exception as exc: # noqa: BLE001
|
|
377
|
+
logger.debug("admission: no role store available: %s", exc)
|
|
378
|
+
return None
|
|
379
|
+
|
|
380
|
+
|
|
381
|
+
def _company_mode_active(deployment) -> bool:
|
|
382
|
+
"""Whether a login is required, by EITHER of the two switches.
|
|
383
|
+
|
|
384
|
+
THE DEFECT THIS CLOSES
|
|
385
|
+
|
|
386
|
+
"Company mode" was two independent settings that nobody had joined up:
|
|
387
|
+
``deployment`` in config.toml, which this module read, and ``require_login``
|
|
388
|
+
in the workspace's own settings, which the dashboard toggle writes and which
|
|
389
|
+
the HTTP routes read. Turning company mode on from the dashboard therefore
|
|
390
|
+
changed what HTTP would allow and changed nothing here.
|
|
391
|
+
|
|
392
|
+
Measured on a real store: with ``require_login`` on, two users configured,
|
|
393
|
+
and the viewer's role denying WRITE, an MCP write resolved to
|
|
394
|
+
``local-operator`` with role ``owner`` and succeeded -- while the same write
|
|
395
|
+
over HTTP returned 401. The role check was not bypassed by a missing call;
|
|
396
|
+
it was bypassed because this path was still being told the workspace was
|
|
397
|
+
personal.
|
|
398
|
+
|
|
399
|
+
Either switch now means the same thing on every transport.
|
|
400
|
+
"""
|
|
401
|
+
if getattr(deployment, "is_enterprise", False):
|
|
402
|
+
return True
|
|
403
|
+
rbac = _rbac_engine()
|
|
404
|
+
if rbac is None:
|
|
405
|
+
return False
|
|
406
|
+
try:
|
|
407
|
+
return bool(rbac.require_login())
|
|
408
|
+
except sqlite3.OperationalError as exc:
|
|
409
|
+
# "No such table" means the role tables were never created, which means
|
|
410
|
+
# roles were never set up, which is a personal install. Failing closed on
|
|
411
|
+
# THIS is not caution -- it is refusing every write on every store that
|
|
412
|
+
# has never used company mode, which is nearly all of them. It was caught
|
|
413
|
+
# by an existing test whose second MCP write started failing once a store
|
|
414
|
+
# file appeared in the data root.
|
|
415
|
+
if "no such table" in str(exc).lower():
|
|
416
|
+
logger.debug("admission: no role tables; personal workspace")
|
|
417
|
+
return False
|
|
418
|
+
logger.warning(
|
|
419
|
+
"admission: the login policy is unreadable (%s); treating the "
|
|
420
|
+
"workspace as requiring one", exc,
|
|
421
|
+
)
|
|
422
|
+
return True
|
|
423
|
+
except Exception as exc: # noqa: BLE001 -- an unreadable policy is not a licence
|
|
424
|
+
logger.warning(
|
|
425
|
+
"admission: cannot read the login policy (%s); treating the "
|
|
426
|
+
"workspace as requiring one", exc,
|
|
427
|
+
)
|
|
428
|
+
return True
|
|
429
|
+
|
|
430
|
+
|
|
431
|
+
def _target_profile(explicit: str = "") -> str:
|
|
432
|
+
"""The workspace a call will actually touch.
|
|
433
|
+
|
|
434
|
+
An explicit argument wins when the caller supplies one. Otherwise this reads
|
|
435
|
+
the same ``profiles.json`` the engine and the HTTP layer read, because that
|
|
436
|
+
is where the write is going to land.
|
|
437
|
+
|
|
438
|
+
THE DEFECT THIS CLOSES
|
|
439
|
+
|
|
440
|
+
The role check used to key off ``kwargs.get("profile_id")``. ``remember``
|
|
441
|
+
has no such parameter, so every role lookup resolved against ``default``
|
|
442
|
+
while the write went to whichever workspace was active. A user who is an
|
|
443
|
+
admin on ``default`` and a viewer on ``team`` passed the check on
|
|
444
|
+
``default`` and wrote to ``team`` -- which HTTP would have refused.
|
|
445
|
+
"""
|
|
446
|
+
name = (explicit or "").strip()
|
|
447
|
+
if name:
|
|
448
|
+
return name
|
|
449
|
+
try:
|
|
450
|
+
from superlocalmemory.server.profile_runtime import current_request_profile
|
|
451
|
+
|
|
452
|
+
runtime = current_request_profile()
|
|
453
|
+
if runtime:
|
|
454
|
+
return str(runtime)
|
|
455
|
+
except Exception as exc: # noqa: BLE001 -- no request context is normal off HTTP
|
|
456
|
+
logger.debug("admission: no request profile in scope: %s", exc)
|
|
457
|
+
try:
|
|
458
|
+
import json as _json
|
|
459
|
+
|
|
460
|
+
from superlocalmemory.infra.data_root import canonical_data_root
|
|
461
|
+
|
|
462
|
+
config_file = canonical_data_root() / "profiles.json"
|
|
463
|
+
if config_file.exists():
|
|
464
|
+
data = _json.loads(config_file.read_text(encoding="utf-8"))
|
|
465
|
+
active = str(data.get("active_profile", "") or "").strip()
|
|
466
|
+
if active:
|
|
467
|
+
return active
|
|
468
|
+
except Exception as exc: # noqa: BLE001
|
|
469
|
+
logger.debug("admission: cannot read the active workspace: %s", exc)
|
|
470
|
+
return "default"
|
|
471
|
+
|
|
472
|
+
|
|
473
|
+
def _session_principal(profile: str) -> tuple[str, str, FrozenSet[ActorRole] | None]:
|
|
474
|
+
"""Resolve the caller from a session token in the environment.
|
|
475
|
+
|
|
476
|
+
Returns ``(principal_id, raw_token, roles)``. An empty principal means the
|
|
477
|
+
caller could not be identified, which ``resolve_actor`` turns into ANONYMOUS
|
|
478
|
+
and ``admit`` then denies -- so an unset or expired token fails closed.
|
|
479
|
+
"""
|
|
480
|
+
token = os.environ.get(_SESSION_ENV, "").strip()
|
|
481
|
+
if not token:
|
|
482
|
+
return "", "", None
|
|
483
|
+
rbac = _rbac_engine()
|
|
484
|
+
if rbac is None:
|
|
485
|
+
return "", "", None
|
|
486
|
+
try:
|
|
487
|
+
user = rbac.resolve_session(token)
|
|
488
|
+
except Exception as exc: # noqa: BLE001
|
|
489
|
+
logger.debug("admission: session lookup failed: %s", exc)
|
|
490
|
+
return "", "", None
|
|
491
|
+
if not user:
|
|
492
|
+
return "", "", None
|
|
493
|
+
role = None
|
|
494
|
+
try:
|
|
495
|
+
role = rbac.get_role(user["user_id"], profile or "default")
|
|
496
|
+
except Exception as exc: # noqa: BLE001
|
|
497
|
+
logger.debug("admission: role lookup failed: %s", exc)
|
|
498
|
+
# No membership on this workspace is not the same as MEMBER. Falling back to
|
|
499
|
+
# a write-capable default here would hand every authenticated user write
|
|
500
|
+
# access to every workspace on the machine.
|
|
501
|
+
actor_role = _RBAC_ROLE_TO_ACTOR.get(
|
|
502
|
+
getattr(role, "value", role) if role is not None else "", None,
|
|
503
|
+
)
|
|
504
|
+
if actor_role is None:
|
|
505
|
+
return user["user_id"], token, frozenset({ActorRole.ANONYMOUS})
|
|
506
|
+
return user["user_id"], token, frozenset({actor_role})
|
|
507
|
+
|
|
508
|
+
|
|
329
509
|
def admits(kind: OperationKind):
|
|
330
510
|
"""Decorator that gates an async MCP tool function via the policy registry.
|
|
331
511
|
|
|
@@ -348,9 +528,21 @@ def admits(kind: OperationKind):
|
|
|
348
528
|
@functools.wraps(fn)
|
|
349
529
|
async def wrapper(*args, **kwargs):
|
|
350
530
|
deployment = _resolve_deployment()
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
531
|
+
# Either switch means company mode. Reading only the config file is
|
|
532
|
+
# what let a dashboard toggle change HTTP and leave this transport
|
|
533
|
+
# writing as the machine owner.
|
|
534
|
+
company = _company_mode_active(deployment)
|
|
535
|
+
tier = "enterprise" if company else "personal"
|
|
536
|
+
mode = "company" if company else "local"
|
|
537
|
+
principal, token, roles = ("", "", None)
|
|
538
|
+
if company:
|
|
539
|
+
principal, token, roles = _session_principal(
|
|
540
|
+
_target_profile(kwargs.get("profile_id", "") or ""),
|
|
541
|
+
)
|
|
542
|
+
actor = resolve_actor(
|
|
543
|
+
Transport.MCP, tier=tier, mode=mode,
|
|
544
|
+
principal=principal, session=token, roles=roles,
|
|
545
|
+
)
|
|
354
546
|
try:
|
|
355
547
|
admit(kind, actor, mode=mode)
|
|
356
548
|
except AdmissionDenied as exc:
|
|
@@ -373,6 +565,7 @@ def gate_cli_mutation(
|
|
|
373
565
|
*,
|
|
374
566
|
principal: str = "",
|
|
375
567
|
roles: FrozenSet[ActorRole] | None = None,
|
|
568
|
+
profile: str = "",
|
|
376
569
|
) -> None:
|
|
377
570
|
"""Gate a CLI mutation command. Exits with code 1 if denied.
|
|
378
571
|
|
|
@@ -385,25 +578,48 @@ def gate_cli_mutation(
|
|
|
385
578
|
kind : Operation being performed.
|
|
386
579
|
principal : Authenticated CLI principal (from session store / login token).
|
|
387
580
|
roles : Explicit roles for an authenticated enterprise user.
|
|
581
|
+
profile : Workspace the command will touch. Empty means the active one.
|
|
388
582
|
"""
|
|
389
583
|
import sys
|
|
390
584
|
deployment = _resolve_deployment()
|
|
391
|
-
|
|
392
|
-
|
|
585
|
+
# Either switch means company mode -- the same rule the MCP gate uses. This
|
|
586
|
+
# gate used to read config.toml alone, so turning per-user access on from
|
|
587
|
+
# the dashboard changed HTTP and MCP and left every CLI write running as the
|
|
588
|
+
# machine owner.
|
|
589
|
+
company = _company_mode_active(deployment)
|
|
590
|
+
tier = "enterprise" if company else "personal"
|
|
591
|
+
mode = "company" if company else "local"
|
|
592
|
+
session = ""
|
|
593
|
+
if company and not principal:
|
|
594
|
+
# A CLI caller identifies itself the same way an MCP caller does. With
|
|
595
|
+
# no token the principal stays empty, resolve_actor returns ANONYMOUS,
|
|
596
|
+
# and admit denies -- which is the intended answer for an unauthenticated
|
|
597
|
+
# write on a workspace that requires a login.
|
|
598
|
+
principal, session, resolved_roles = _session_principal(
|
|
599
|
+
_target_profile(profile),
|
|
600
|
+
)
|
|
601
|
+
if roles is None:
|
|
602
|
+
roles = resolved_roles
|
|
393
603
|
actor = resolve_actor(
|
|
394
604
|
Transport.CLI,
|
|
395
605
|
tier=tier,
|
|
396
606
|
mode=mode,
|
|
397
607
|
principal=principal,
|
|
608
|
+
session=session,
|
|
398
609
|
roles=roles,
|
|
399
610
|
)
|
|
400
611
|
try:
|
|
401
612
|
admit(kind, actor, mode=mode)
|
|
402
613
|
except AdmissionDenied as exc:
|
|
614
|
+
# Name something the reader can actually do. This used to say
|
|
615
|
+
# "log in with 'slm login'", and there is no such command -- so the one
|
|
616
|
+
# instruction the message gave was a dead end.
|
|
403
617
|
print(
|
|
404
618
|
f"[slm] Operation denied ({exc.decision.reason}). "
|
|
405
|
-
"This workspace requires
|
|
406
|
-
"
|
|
619
|
+
"This workspace requires a signed-in user. Sign in on the dashboard "
|
|
620
|
+
"(slm dashboard), copy your session token, and put it in the "
|
|
621
|
+
"SLM_USER_SESSION environment variable -- or ask whoever "
|
|
622
|
+
"administers this workspace for access.",
|
|
407
623
|
flush=True,
|
|
408
624
|
)
|
|
409
625
|
sys.exit(1)
|
|
@@ -515,7 +731,11 @@ def enforce_read_scope(
|
|
|
515
731
|
is left alone so the server default applies.
|
|
516
732
|
"""
|
|
517
733
|
deployment = _resolve_deployment()
|
|
518
|
-
|
|
734
|
+
# Both switches, for the same reason the write gates read both: a dashboard
|
|
735
|
+
# toggle used to leave this path unclamped, so a viewer could ask for
|
|
736
|
+
# include_global=True over MCP and pull another workspace's facts into the
|
|
737
|
+
# candidate set while HTTP refused the same request.
|
|
738
|
+
if not _company_mode_active(deployment):
|
|
519
739
|
return include_global, include_shared
|
|
520
740
|
|
|
521
741
|
reg = registry if registry is not None else _DEFAULT_REGISTRY
|
|
@@ -21,6 +21,9 @@ import logging
|
|
|
21
21
|
from pathlib import Path
|
|
22
22
|
from typing import TYPE_CHECKING, Any
|
|
23
23
|
|
|
24
|
+
from superlocalmemory.core.projection_drain import ProjectionDrain
|
|
25
|
+
from superlocalmemory.storage import projection_outbox
|
|
26
|
+
|
|
24
27
|
if TYPE_CHECKING:
|
|
25
28
|
from superlocalmemory.core.config import SLMConfig
|
|
26
29
|
from superlocalmemory.storage.database import DatabaseManager
|
|
@@ -80,6 +83,12 @@ class BackendOrchestrator:
|
|
|
80
83
|
self._lancedb: Any = None
|
|
81
84
|
self._tiers: Any = None
|
|
82
85
|
self._backend_cache: dict[str, str] = {}
|
|
86
|
+
# Given accessors, not backends: a promotion or a rollback replaces
|
|
87
|
+
# them underneath the worker, and a reference captured here would keep
|
|
88
|
+
# writing into the projection that was just swapped out.
|
|
89
|
+
self._drain = ProjectionDrain(
|
|
90
|
+
db, self.get_graph_backend, self.get_vector_backend,
|
|
91
|
+
)
|
|
83
92
|
|
|
84
93
|
# ------------------------------------------------------------------
|
|
85
94
|
# Daemon Startup
|
|
@@ -107,6 +116,19 @@ class BackendOrchestrator:
|
|
|
107
116
|
|
|
108
117
|
self._recover_interrupted_scale_promotion()
|
|
109
118
|
|
|
119
|
+
# Reconcile what the config CLAIMS against what is on disk, before
|
|
120
|
+
# anything reads either -- and before the early return below, because
|
|
121
|
+
# the stores that need reconciling are exactly the ones that take it.
|
|
122
|
+
#
|
|
123
|
+
# This used to sit after that return, so it ran only for a store already
|
|
124
|
+
# in the promoted state. The real case it was written for is a store
|
|
125
|
+
# whose settings name a graph and a vector backend, whose state is
|
|
126
|
+
# `verified` rather than `promoted`, and where neither directory exists:
|
|
127
|
+
# the settings kept the claim, the reconcile never ran, and every
|
|
128
|
+
# restart preserved it. The test asserted the call appeared before
|
|
129
|
+
# another call in the source text, which is true either way.
|
|
130
|
+
self._reconcile_backend_selection()
|
|
131
|
+
|
|
110
132
|
# Backends may be installed with the product, but installing a wheel
|
|
111
133
|
# is not authorization to mutate an existing data root. Only a
|
|
112
134
|
# verified, explicit promotion may initialize and migrate projections.
|
|
@@ -120,6 +142,11 @@ class BackendOrchestrator:
|
|
|
120
142
|
# SQLite graph. A no-op (and never even starts the build) for the
|
|
121
143
|
# vast majority of installs, which sit far below the threshold.
|
|
122
144
|
self._maybe_schedule_auto_promote()
|
|
145
|
+
# Started even with no projection open. A pass with no backend
|
|
146
|
+
# returns without touching a row, and starting it here means a
|
|
147
|
+
# promotion that completes mid-session has a worker waiting for it
|
|
148
|
+
# rather than a queue nobody is reading.
|
|
149
|
+
self._drain.start()
|
|
123
150
|
return
|
|
124
151
|
|
|
125
152
|
# 3. Initialize CozoDB if available
|
|
@@ -145,9 +172,14 @@ class BackendOrchestrator:
|
|
|
145
172
|
except Exception as exc:
|
|
146
173
|
logger.warning("TierManager backend registration failed (non-fatal): %s", exc)
|
|
147
174
|
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
175
|
+
self._drain.start()
|
|
176
|
+
|
|
177
|
+
logger.info(
|
|
178
|
+
"BackendOrchestrator: daemon ready (cozo=%s, lancedb=%s, queued=%d)",
|
|
179
|
+
"active" if self._cozo and self._cozo_status() == "active" else "off",
|
|
180
|
+
"active" if self._lancedb and self._lancedb_status() == "active" else "off",
|
|
181
|
+
projection_outbox.depth(self._db),
|
|
182
|
+
)
|
|
151
183
|
|
|
152
184
|
def _maybe_schedule_auto_promote(self) -> None:
|
|
153
185
|
"""Schedule a delayed, one-shot scale auto-promote check (v3.8.5).
|
|
@@ -164,7 +196,23 @@ class BackendOrchestrator:
|
|
|
164
196
|
cfg = self._config
|
|
165
197
|
if not getattr(cfg, "scale_auto_promote_enabled", True):
|
|
166
198
|
return
|
|
167
|
-
|
|
199
|
+
# Only a store that has finished promoting has nothing left to do.
|
|
200
|
+
#
|
|
201
|
+
# This used to skip every state except ``local_core``, which made
|
|
202
|
+
# ``prepared`` and ``verified`` terminal: the daemon above returns
|
|
203
|
+
# early for anything that is not ``promoted``, so the backends never
|
|
204
|
+
# started, and this refused to finish the promotion that would have
|
|
205
|
+
# started them. A store that got as far as building and checking its
|
|
206
|
+
# projection then sat on SQLite forever while its own config named Cozo
|
|
207
|
+
# and LanceDB as the backends — measured on a real store whose
|
|
208
|
+
# ``backend_status`` read lancedb=not_initialized under
|
|
209
|
+
# ``scale_engine_state=verified``.
|
|
210
|
+
#
|
|
211
|
+
# ``run_auto_promote`` already resumes a half-finished stage
|
|
212
|
+
# (``_resumable_stage``) and applies the size threshold and the
|
|
213
|
+
# repair-required check itself, so it is the right place for every
|
|
214
|
+
# decision except "there is nothing left to do".
|
|
215
|
+
if str(getattr(cfg, "scale_engine_state", "local_core")).lower() == "promoted":
|
|
168
216
|
return
|
|
169
217
|
try:
|
|
170
218
|
delay = float(os.environ.get("SLM_AUTO_PROMOTE_DELAY_S", "300"))
|
|
@@ -198,11 +246,14 @@ class BackendOrchestrator:
|
|
|
198
246
|
import os
|
|
199
247
|
|
|
200
248
|
cfg = self._config
|
|
201
|
-
|
|
249
|
+
# Same rule as the scheduler that armed this timer: only a store
|
|
250
|
+
# that has finished has nothing left to do. Fixing the scheduler
|
|
251
|
+
# alone would have armed a timer whose callback still refused.
|
|
252
|
+
if str(getattr(cfg, "scale_engine_state", "local_core")).lower() == "promoted":
|
|
202
253
|
return
|
|
203
254
|
threshold = int(
|
|
204
255
|
os.environ.get("SLM_AUTO_PROMOTE_MIN_EDGES", "")
|
|
205
|
-
or getattr(cfg, "scale_auto_promote_min_edges",
|
|
256
|
+
or getattr(cfg, "scale_auto_promote_min_edges", 100_000)
|
|
206
257
|
)
|
|
207
258
|
edges = self._count_default_edges()
|
|
208
259
|
if edges < threshold:
|
|
@@ -237,6 +288,75 @@ class BackendOrchestrator:
|
|
|
237
288
|
exc,
|
|
238
289
|
)
|
|
239
290
|
|
|
291
|
+
def _reconcile_backend_selection(self) -> None:
|
|
292
|
+
"""Stop the config claiming a backend the store does not have.
|
|
293
|
+
|
|
294
|
+
THE STATE THIS REPAIRS
|
|
295
|
+
|
|
296
|
+
On a real store: ``graph_backend='cozo'``, ``vector_backend='lancedb'``,
|
|
297
|
+
``scale_engine_state='verified'`` -- and neither the ``cozo/`` nor the
|
|
298
|
+
``lance/`` directory existed, with no promotion journal to explain it.
|
|
299
|
+
Something wrote the selection a completed promotion writes, without a
|
|
300
|
+
promotion having completed.
|
|
301
|
+
|
|
302
|
+
Nothing corrected it. ``recover_interrupted_promotion`` acts only when a
|
|
303
|
+
journal exists, so with no journal it returns immediately and the claim
|
|
304
|
+
survives every restart. The dashboard then reports the configured backend
|
|
305
|
+
while retrieval uses SQLite, which is the disagreement a person notices
|
|
306
|
+
last and trusts first.
|
|
307
|
+
|
|
308
|
+
WHAT THIS DOES NOT DO
|
|
309
|
+
|
|
310
|
+
It does not disable anything. ``auto`` still detects and initialises both
|
|
311
|
+
projections when their libraries are installed, so the only thing removed
|
|
312
|
+
is the false claim. It leaves ``verified`` alone -- that is a legitimate
|
|
313
|
+
waypoint meaning "parity checked, not yet promoted" -- and only resets
|
|
314
|
+
``promoted``, which asserts a swap that plainly did not happen. And it
|
|
315
|
+
never touches a selection whose directory is present, nor one with a
|
|
316
|
+
journal still open, because those belong to the promotion lifecycle.
|
|
317
|
+
"""
|
|
318
|
+
try:
|
|
319
|
+
from superlocalmemory.core.scale_engine import ScaleEngineManager
|
|
320
|
+
|
|
321
|
+
manager = ScaleEngineManager(self._config, profile_id="default")
|
|
322
|
+
if manager.promotion_journal_path.exists():
|
|
323
|
+
return # the recovery path owns this
|
|
324
|
+
cozo_path, lance_path = manager.active_paths
|
|
325
|
+
except Exception as exc: # noqa: BLE001 -- reconciliation is best effort
|
|
326
|
+
logger.debug("Backend reconciliation skipped: %s", exc)
|
|
327
|
+
return
|
|
328
|
+
|
|
329
|
+
corrections: list[str] = []
|
|
330
|
+
graph = getattr(self._config, "graph_backend", "auto") or "auto"
|
|
331
|
+
if graph not in ("auto", "sqlite") and not cozo_path.exists():
|
|
332
|
+
corrections.append(f"graph_backend {graph!r} -> 'auto' (no {cozo_path.name}/)")
|
|
333
|
+
self._config.graph_backend = "auto"
|
|
334
|
+
vector = getattr(self._config, "vector_backend", "auto") or "auto"
|
|
335
|
+
if vector not in ("auto", "sqlite-vec") and not lance_path.exists():
|
|
336
|
+
corrections.append(
|
|
337
|
+
f"vector_backend {vector!r} -> 'auto' (no {lance_path.name}/)"
|
|
338
|
+
)
|
|
339
|
+
self._config.vector_backend = "auto"
|
|
340
|
+
state = getattr(self._config, "scale_engine_state", "") or ""
|
|
341
|
+
if state == "promoted" and not (cozo_path.exists() or lance_path.exists()):
|
|
342
|
+
corrections.append("scale_engine_state 'promoted' -> 'local_core'")
|
|
343
|
+
self._config.scale_engine_state = "local_core"
|
|
344
|
+
|
|
345
|
+
if not corrections:
|
|
346
|
+
return
|
|
347
|
+
try:
|
|
348
|
+
self._config.save()
|
|
349
|
+
except Exception as exc: # noqa: BLE001
|
|
350
|
+
logger.warning(
|
|
351
|
+
"Backend selection is inconsistent with the data directory and "
|
|
352
|
+
"could not be corrected (%s): %s", exc, "; ".join(corrections),
|
|
353
|
+
)
|
|
354
|
+
return
|
|
355
|
+
logger.warning(
|
|
356
|
+
"Backend selection did not match the data directory; corrected: %s",
|
|
357
|
+
"; ".join(corrections),
|
|
358
|
+
)
|
|
359
|
+
|
|
240
360
|
def _recover_interrupted_scale_promotion(self) -> None:
|
|
241
361
|
"""Repair an interrupted promotion; never auto-mutate a legacy root."""
|
|
242
362
|
try:
|
|
@@ -261,90 +381,44 @@ class BackendOrchestrator:
|
|
|
261
381
|
# ------------------------------------------------------------------
|
|
262
382
|
|
|
263
383
|
def sync_new_fact(self, fact: Any) -> None:
|
|
264
|
-
"""
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
384
|
+
"""Signal that a stored fact needs projecting.
|
|
385
|
+
|
|
386
|
+
The projection itself is not written here. It used to be — inline, on
|
|
387
|
+
the caller's thread, with every failure swallowed into a debug line —
|
|
388
|
+
and that is the defect the outbox replaced. The intent to project was
|
|
389
|
+
already committed to ``projection_outbox`` in the same SQLite
|
|
390
|
+
transaction as the fact, so all that is left to do is wake the worker
|
|
391
|
+
that owns the projections.
|
|
392
|
+
|
|
393
|
+
Kept as a method because callers name this operation, and because a
|
|
394
|
+
caller that reaches it without an outbox row (an old store, mid-upgrade)
|
|
395
|
+
should still get its fact projected rather than silently skipped.
|
|
268
396
|
"""
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
if self._cozo and self._cozo_status() == "active":
|
|
276
|
-
self._sync_fact_entities(fact)
|
|
277
|
-
|
|
278
|
-
if self._lancedb and self._lancedb_status() == "active":
|
|
279
|
-
self._sync_fact_embedding(fact)
|
|
280
|
-
|
|
281
|
-
def _sync_fact_entities(self, fact: Any) -> None:
|
|
282
|
-
"""Synchronize one fact's canonical entity bridge and fact edges."""
|
|
283
|
-
try:
|
|
284
|
-
# Retrying ingestion must not retain stale fact/entity links.
|
|
285
|
-
self._cozo.remove_fact(fact.fact_id)
|
|
286
|
-
entities = getattr(fact, "canonical_entities", []) or []
|
|
287
|
-
profile_id = getattr(fact, "profile_id", "default") or "default"
|
|
288
|
-
for eid in entities:
|
|
289
|
-
rows = self._db.execute(
|
|
290
|
-
"SELECT canonical_name, entity_type, fact_count FROM canonical_entities "
|
|
291
|
-
"WHERE entity_id = ? AND profile_id = ?",
|
|
292
|
-
(eid, profile_id),
|
|
293
|
-
)
|
|
294
|
-
if rows:
|
|
295
|
-
entity = dict(rows[0])
|
|
296
|
-
self._cozo.add_entity(
|
|
297
|
-
eid,
|
|
298
|
-
entity.get("canonical_name") or eid,
|
|
299
|
-
entity.get("entity_type") or "concept",
|
|
300
|
-
{"fact_count": int(entity.get("fact_count") or 0)},
|
|
301
|
-
profile_id,
|
|
302
|
-
)
|
|
303
|
-
self._cozo.add_fact_entities(fact.fact_id, entities, profile_id)
|
|
304
|
-
for row in self._db.execute(
|
|
305
|
-
"SELECT source_id, target_id, edge_type, weight FROM graph_edges "
|
|
306
|
-
"WHERE profile_id = ? AND (source_id = ? OR target_id = ?)",
|
|
307
|
-
(profile_id, fact.fact_id, fact.fact_id),
|
|
308
|
-
):
|
|
309
|
-
edge = dict(row)
|
|
310
|
-
self._cozo.add_edge(
|
|
311
|
-
edge["source_id"], edge["target_id"], edge.get("edge_type") or "related",
|
|
312
|
-
float(edge.get("weight") or 1.0), profile_id=profile_id,
|
|
313
|
-
)
|
|
314
|
-
except Exception as exc:
|
|
315
|
-
logger.debug("CozoDB incremental sync skipped: %s", exc)
|
|
316
|
-
|
|
317
|
-
def _sync_fact_embedding(self, fact: Any) -> None:
|
|
318
|
-
"""Sync fact's embedding to LanceDB."""
|
|
319
|
-
try:
|
|
320
|
-
embedding = getattr(fact, "embedding", None)
|
|
321
|
-
if embedding:
|
|
322
|
-
tier = getattr(fact, "lifecycle", "active")
|
|
323
|
-
self._lancedb.add_vectors(
|
|
324
|
-
[fact.fact_id], [embedding], [tier],
|
|
325
|
-
getattr(fact, "profile_id", "default") or "default",
|
|
326
|
-
)
|
|
327
|
-
except Exception as exc:
|
|
328
|
-
logger.debug("LanceDB incremental sync skipped: %s", exc)
|
|
397
|
+
fact_id = getattr(fact, "fact_id", None)
|
|
398
|
+
if fact_id:
|
|
399
|
+
projection_outbox.enqueue(
|
|
400
|
+
self._db, fact_id, getattr(fact, "profile_id", None) or "default",
|
|
401
|
+
)
|
|
402
|
+
self._drain.notify()
|
|
329
403
|
|
|
330
404
|
def sync_deleted_fact(self, fact_id: str) -> None:
|
|
331
|
-
"""
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
if
|
|
338
|
-
|
|
339
|
-
self.
|
|
340
|
-
|
|
341
|
-
|
|
405
|
+
"""Signal that a deleted fact must leave the projections.
|
|
406
|
+
|
|
407
|
+
A forgotten memory still present in the graph or the vector index is
|
|
408
|
+
still recallable, so the removal is queued with the same durability as
|
|
409
|
+
the delete itself.
|
|
410
|
+
"""
|
|
411
|
+
if fact_id:
|
|
412
|
+
projection_outbox.enqueue_for_fact(
|
|
413
|
+
self._db, fact_id, projection_outbox.OP_DELETE,
|
|
414
|
+
)
|
|
415
|
+
self._drain.notify()
|
|
342
416
|
|
|
343
417
|
def sync_changed_fact(self, fact_id: str) -> None:
|
|
344
418
|
"""Refresh projections after an authorized canonical fact update."""
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
419
|
+
if fact_id:
|
|
420
|
+
projection_outbox.enqueue_for_fact(self._db, fact_id)
|
|
421
|
+
self._drain.notify()
|
|
348
422
|
|
|
349
423
|
# ------------------------------------------------------------------
|
|
350
424
|
# Backend Access
|
|
@@ -411,8 +485,40 @@ class BackendOrchestrator:
|
|
|
411
485
|
"LanceDB not active. Install: pip install superlocalmemory[lancedb]"
|
|
412
486
|
)
|
|
413
487
|
|
|
488
|
+
outbox = projection_outbox.health(self._db)
|
|
489
|
+
outbox["draining"] = self._drain.running
|
|
490
|
+
result["projection_queue"] = outbox
|
|
491
|
+
if outbox["stalled"]:
|
|
492
|
+
result["warnings"].append(
|
|
493
|
+
f"{outbox['stalled']} memory/memories could not be projected into "
|
|
494
|
+
"the graph or vector store. Run `slm doctor` for the ids."
|
|
495
|
+
)
|
|
496
|
+
|
|
414
497
|
return result
|
|
415
498
|
|
|
499
|
+
# ------------------------------------------------------------------
|
|
500
|
+
# Projection queue
|
|
501
|
+
# ------------------------------------------------------------------
|
|
502
|
+
|
|
503
|
+
def drain_projections(self, limit: int = 200) -> dict[str, Any]:
|
|
504
|
+
"""Apply queued facts now, on the calling thread.
|
|
505
|
+
|
|
506
|
+
For the CLI, for a repair pass, and for any caller that needs the
|
|
507
|
+
projections current before it reads them rather than a few milliseconds
|
|
508
|
+
later. Ordinary writes do not need this — they signal the worker.
|
|
509
|
+
"""
|
|
510
|
+
return self._drain.drain_once(limit=limit).as_dict()
|
|
511
|
+
|
|
512
|
+
def outbox_health(self) -> dict[str, Any]:
|
|
513
|
+
"""Queue depth and stalled count, for the status surfaces."""
|
|
514
|
+
health = projection_outbox.health(self._db)
|
|
515
|
+
health["draining"] = self._drain.running
|
|
516
|
+
return health
|
|
517
|
+
|
|
518
|
+
def stop(self) -> None:
|
|
519
|
+
"""Stop the drain worker. For daemon shutdown and for tests."""
|
|
520
|
+
self._drain.stop()
|
|
521
|
+
|
|
416
522
|
# ------------------------------------------------------------------
|
|
417
523
|
# Internal: Detection
|
|
418
524
|
# ------------------------------------------------------------------
|