superlocalmemory 3.8.13 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ATTRIBUTION.md +4 -4
- package/CHANGELOG.md +113 -121
- package/README.md +65 -63
- package/docs/pi-dev-integration.md +1 -1
- package/package.json +6 -1
- package/plugin/.claude-plugin/plugin.json +1 -1
- package/plugin/CLAUDE.md +3 -3
- package/plugin/agents/slm-governance-advisor.md +1 -1
- package/plugin/agents/slm-loop-runner.md +1 -1
- package/plugin/agents/slm-memory-advisor.md +1 -1
- package/plugin/agents/slm-optimize-advisor.md +1 -1
- package/plugin/requirements.txt +1 -1
- package/plugin/skills/slm-cache/SKILL.md +1 -1
- package/plugin/skills/slm-compress/SKILL.md +1 -1
- package/plugin/skills/slm-governance/SKILL.md +1 -1
- package/plugin/skills/slm-graph/SKILL.md +1 -1
- package/plugin/skills/slm-loop/SKILL.md +1 -1
- package/plugin/skills/slm-mesh/SKILL.md +1 -1
- package/plugin/skills/slm-profile/SKILL.md +1 -1
- package/plugin/skills/slm-recall/SKILL.md +1 -1
- package/plugin/skills/slm-remember/SKILL.md +1 -1
- package/plugin/skills/slm-scope/SKILL.md +1 -1
- package/plugin/skills/slm-session/SKILL.md +1 -1
- package/plugin/skills/slm-status/SKILL.md +1 -1
- package/plugin-src/rules/AGENTS.md +1 -1
- package/plugin-src/skills/slm-cache/SKILL.md +1 -1
- package/plugin-src/skills/slm-compress/SKILL.md +1 -1
- package/plugin-src/skills/slm-governance/SKILL.md +248 -0
- package/plugin-src/skills/slm-graph/SKILL.md +1 -1
- package/plugin-src/skills/slm-loop/SKILL.md +99 -0
- package/plugin-src/skills/slm-mesh/SKILL.md +282 -0
- package/plugin-src/skills/slm-profile/SKILL.md +148 -0
- package/plugin-src/skills/slm-recall/SKILL.md +1 -1
- package/plugin-src/skills/slm-remember/SKILL.md +1 -1
- package/plugin-src/skills/slm-scope/SKILL.md +176 -0
- package/plugin-src/skills/slm-session/SKILL.md +1 -1
- package/plugin-src/skills/slm-status/SKILL.md +1 -1
- package/pyproject.toml +11 -4
- package/src/superlocalmemory/__init__.py +1 -1
- package/src/superlocalmemory/cli/commands.py +125 -11
- package/src/superlocalmemory/cli/daemon.py +5 -1
- package/src/superlocalmemory/cli/main.py +35 -2
- package/src/superlocalmemory/cli/ops_cmd.py +281 -0
- package/src/superlocalmemory/cli/setup_wizard.py +1 -1
- package/src/superlocalmemory/compliance/audit.py +65 -0
- package/src/superlocalmemory/compliance/eu_ai_act.py +27 -57
- package/src/superlocalmemory/compliance/gdpr.py +416 -20
- package/src/superlocalmemory/compliance/retention.py +74 -22
- package/src/superlocalmemory/compliance/scheduler.py +78 -9
- package/src/superlocalmemory/core/actor_context.py +166 -0
- package/src/superlocalmemory/core/admission.py +549 -0
- package/src/superlocalmemory/core/backend_orchestrator.py +23 -10
- package/src/superlocalmemory/core/config.py +202 -24
- package/src/superlocalmemory/core/consolidation_engine.py +13 -13
- package/src/superlocalmemory/core/context_cache.py +28 -0
- package/src/superlocalmemory/core/embeddings.py +64 -2
- package/src/superlocalmemory/core/engine.py +7 -2
- package/src/superlocalmemory/core/engine_ingestion.py +65 -3
- package/src/superlocalmemory/core/engine_wiring.py +36 -9
- package/src/superlocalmemory/core/ingest_policy.py +38 -0
- package/src/superlocalmemory/core/maintenance.py +255 -0
- package/src/superlocalmemory/core/modes.py +40 -13
- package/src/superlocalmemory/core/mutations.py +437 -44
- package/src/superlocalmemory/core/operation_policy.py +92 -0
- package/src/superlocalmemory/core/operation_policy_registry.py +542 -0
- package/src/superlocalmemory/core/operation_request.py +127 -0
- package/src/superlocalmemory/core/ops_remediation.py +542 -0
- package/src/superlocalmemory/core/recall_pipeline.py +7 -0
- package/src/superlocalmemory/core/remember_runtime.py +202 -4
- package/src/superlocalmemory/core/remote_mode.py +20 -5
- package/src/superlocalmemory/core/store_pipeline.py +150 -0
- package/src/superlocalmemory/core/topic_signature.py +19 -4
- package/src/superlocalmemory/core/transactions/__init__.py +78 -0
- package/src/superlocalmemory/core/transactions/concrete_owners.py +597 -0
- package/src/superlocalmemory/core/transactions/erasure.py +825 -0
- package/src/superlocalmemory/core/transactions/manifest.py +255 -0
- package/src/superlocalmemory/core/transactions/manifest_key.py +155 -0
- package/src/superlocalmemory/core/transactions/obligations.py +272 -0
- package/src/superlocalmemory/core/transactions/owners.py +114 -0
- package/src/superlocalmemory/core/transactions/reconciler.py +285 -0
- package/src/superlocalmemory/core/transactions/service.py +330 -0
- package/src/superlocalmemory/core/worker_pool.py +33 -5
- package/src/superlocalmemory/encoding/cognitive_consolidator.py +70 -28
- package/src/superlocalmemory/encoding/emotional.py +75 -14
- package/src/superlocalmemory/encoding/scene_builder.py +115 -13
- package/src/superlocalmemory/encoding/temporal_parser.py +4 -0
- package/src/superlocalmemory/evolution/blind_verifier.py +11 -4
- package/src/superlocalmemory/evolution/evolution_store.py +244 -4
- package/src/superlocalmemory/evolution/llm_dispatch.py +40 -0
- package/src/superlocalmemory/evolution/model_selection.py +18 -3
- package/src/superlocalmemory/evolution/mutation_generator.py +3 -0
- package/src/superlocalmemory/evolution/skill_activator.py +270 -0
- package/src/superlocalmemory/evolution/skill_evolver.py +281 -59
- package/src/superlocalmemory/evolution/types.py +30 -8
- package/src/superlocalmemory/graph/cozo_backend.py +17 -9
- package/src/superlocalmemory/hooks/auto_invoker.py +2 -1
- package/src/superlocalmemory/hooks/auto_recall.py +64 -30
- package/src/superlocalmemory/hooks/codex_assets.py +14 -1
- package/src/superlocalmemory/infra/backup.py +434 -7
- package/src/superlocalmemory/infra/process_reaper.py +18 -0
- package/src/superlocalmemory/infra/self_heal.py +401 -0
- package/src/superlocalmemory/learning/feedback.py +52 -9
- package/src/superlocalmemory/loops/engine.py +10 -0
- package/src/superlocalmemory/mcp/_daemon_proxy.py +3 -0
- package/src/superlocalmemory/mcp/http_transport.py +30 -331
- package/src/superlocalmemory/mcp/profiles.py +5 -0
- package/src/superlocalmemory/mcp/resources.py +8 -0
- package/src/superlocalmemory/mcp/server.py +51 -4
- package/src/superlocalmemory/mcp/shared.py +19 -0
- package/src/superlocalmemory/mcp/tools_active.py +25 -4
- package/src/superlocalmemory/mcp/tools_code_graph.py +26 -18
- package/src/superlocalmemory/mcp/tools_context.py +50 -8
- package/src/superlocalmemory/mcp/tools_core.py +69 -21
- package/src/superlocalmemory/mcp/tools_evolution.py +9 -2
- package/src/superlocalmemory/mcp/tools_learning.py +21 -10
- package/src/superlocalmemory/mcp/tools_loops.py +29 -18
- package/src/superlocalmemory/mcp/tools_mesh.py +8 -0
- package/src/superlocalmemory/mcp/tools_ops.py +115 -0
- package/src/superlocalmemory/mcp/tools_optimize.py +4 -0
- package/src/superlocalmemory/mcp/tools_v28.py +10 -3
- package/src/superlocalmemory/mcp/tools_v3.py +34 -14
- package/src/superlocalmemory/mcp/tools_v33.py +18 -33
- package/src/superlocalmemory/mesh/broker.py +124 -46
- package/src/superlocalmemory/mesh/broker_security.py +470 -0
- package/src/superlocalmemory/mesh/discovery.py +365 -0
- package/src/superlocalmemory/mesh/lock_protocol.py +313 -0
- package/src/superlocalmemory/mesh/node_identity.py +97 -0
- package/src/superlocalmemory/mesh/outbox_remote.py +429 -0
- package/src/superlocalmemory/mesh/remote_sync.py +511 -28
- package/src/superlocalmemory/mesh/state_sync.py +286 -0
- package/src/superlocalmemory/optimize/config/store.py +45 -0
- package/src/superlocalmemory/parameterization/cross_project.py +12 -0
- package/src/superlocalmemory/parameterization/prompt_injector.py +13 -11
- package/src/superlocalmemory/parameterization/prompt_lifecycle.py +8 -2
- package/src/superlocalmemory/parameterization/workflow_miner.py +17 -0
- package/src/superlocalmemory/retrieval/ann_index.py +5 -0
- package/src/superlocalmemory/retrieval/bm25_channel.py +49 -2
- package/src/superlocalmemory/retrieval/engine.py +19 -4
- package/src/superlocalmemory/retrieval/fusion.py +4 -1
- package/src/superlocalmemory/retrieval/hopfield_channel.py +9 -3
- package/src/superlocalmemory/retrieval/remote_reranker.py +47 -22
- package/src/superlocalmemory/retrieval/reranker.py +32 -1
- package/src/superlocalmemory/retrieval/temporal_channel.py +16 -3
- package/src/superlocalmemory/retrieval/temporal_utils.py +107 -0
- package/src/superlocalmemory/retrieval/temporal_validity_filter.py +155 -42
- package/src/superlocalmemory/retrieval/vector_store.py +214 -8
- package/src/superlocalmemory/server/api.py +5 -5
- package/src/superlocalmemory/server/egress_policy.py +258 -0
- package/src/superlocalmemory/server/rbac_enforce.py +32 -0
- package/src/superlocalmemory/server/route_mutations.py +20 -0
- package/src/superlocalmemory/server/routes/compliance.py +153 -7
- package/src/superlocalmemory/server/routes/data_io.py +43 -2
- package/src/superlocalmemory/server/routes/events.py +15 -0
- package/src/superlocalmemory/server/routes/memories.py +56 -3
- package/src/superlocalmemory/server/routes/mesh.py +82 -1
- package/src/superlocalmemory/server/routes/mesh_lock.py +54 -0
- package/src/superlocalmemory/server/routes/mesh_state.py +63 -0
- package/src/superlocalmemory/server/routes/v3_api.py +50 -27
- package/src/superlocalmemory/server/routes/ws.py +86 -0
- package/src/superlocalmemory/server/ui.py +6 -6
- package/src/superlocalmemory/server/unified_daemon.py +942 -119
- package/src/superlocalmemory/storage/_migration_internals.py +568 -0
- package/src/superlocalmemory/storage/_schema_version.py +110 -0
- package/src/superlocalmemory/storage/database.py +329 -24
- package/src/superlocalmemory/storage/embedding_migrator.py +246 -51
- package/src/superlocalmemory/storage/erasure_fence.py +45 -0
- package/src/superlocalmemory/storage/generation_fence.py +63 -0
- package/src/superlocalmemory/storage/migration_runner.py +140 -417
- package/src/superlocalmemory/storage/migrations/M009_model_lineage.py +40 -0
- package/src/superlocalmemory/storage/migrations/M033_projection_transactions.py +148 -0
- package/src/superlocalmemory/storage/migrations/M034_obligation_integrity.py +58 -0
- package/src/superlocalmemory/storage/migrations/M035_erasure_receipts.py +113 -0
- package/src/superlocalmemory/storage/migrations/M036_vector_row_map.py +107 -0
- package/src/superlocalmemory/storage/migrations/M037_manifest_hmac_version.py +162 -0
- package/src/superlocalmemory/storage/migrations/{M033_learning_feedback_channel.py → M038_learning_feedback_channel.py} +3 -3
- package/src/superlocalmemory/storage/migrations/M039_scene_fact_members.py +137 -0
- package/src/superlocalmemory/storage/migrations/__init__.py +4 -2
- package/src/superlocalmemory/storage/schema.py +67 -0
- package/src/superlocalmemory/storage/write_coordinator.py +125 -0
- package/src/superlocalmemory/trust/scorer.py +28 -4
- package/src/superlocalmemory/ui/index.html +14 -3
- package/src/superlocalmemory/ui/js/auto-settings.js +12 -1
- package/src/superlocalmemory/ui/js/brain.js +6 -4
- package/src/superlocalmemory/ui/js/compliance.js +66 -12
- package/src/superlocalmemory/ui/js/dashboard.js +13 -3
- package/src/superlocalmemory/ui/js/feedback.js +8 -2
- package/src/superlocalmemory/ui/js/lifecycle.js +7 -1
- package/src/superlocalmemory/ui/js/modal.js +272 -5
- package/src/superlocalmemory/ui/js/od-backup.js +9 -2
- package/src/superlocalmemory/ui/js/od-compliance-ext.js +301 -0
- package/src/superlocalmemory/ui/js/od-operations.js +154 -23
- package/src/superlocalmemory/ui/js/od-ops-health.js +417 -0
- package/src/superlocalmemory/ui/js/od-optimize.js +35 -21
- package/src/superlocalmemory/ui/js/od-team.js +9 -2
- package/src/superlocalmemory/ui/js/optimize.js +13 -16
- package/src/superlocalmemory/ui/js/profiles.js +7 -3
- package/src/superlocalmemory/ui/js/settings.js +7 -1
- package/src/superlocalmemory/vector/lancedb_backend.py +19 -9
- package/src/superlocalmemory/attribution/mathematical_dna.py +0 -235
- package/src/superlocalmemory/cli/post_install.py +0 -114
- package/src/superlocalmemory/core/clock_monitor.py +0 -45
- package/src/superlocalmemory/core/db_pool.py +0 -80
- package/src/superlocalmemory/core/error_catalog.py +0 -113
- package/src/superlocalmemory/core/loop_watchdog.py +0 -56
- package/src/superlocalmemory/core/priority_queue.py +0 -61
- package/src/superlocalmemory/core/pruning_engine.py +0 -216
- package/src/superlocalmemory/core/queue_dispatcher.py +0 -73
- package/src/superlocalmemory/core/slmignore.py +0 -125
- package/src/superlocalmemory/infra/heartbeat_monitor.py +0 -140
- package/src/superlocalmemory/infra/webhook_dispatcher.py +0 -247
- package/src/superlocalmemory/learning/quantization_scheduler.py +0 -320
- package/src/superlocalmemory/storage/access_control.py +0 -182
|
@@ -36,9 +36,8 @@ import sys
|
|
|
36
36
|
import threading
|
|
37
37
|
import time
|
|
38
38
|
import uuid
|
|
39
|
-
from contextlib import
|
|
39
|
+
from contextlib import AsyncExitStack, asynccontextmanager
|
|
40
40
|
from dataclasses import replace
|
|
41
|
-
from datetime import datetime, timezone
|
|
42
41
|
from pathlib import Path
|
|
43
42
|
from typing import Optional
|
|
44
43
|
|
|
@@ -66,6 +65,25 @@ from superlocalmemory.infra.data_root import (
|
|
|
66
65
|
canonical_data_root,
|
|
67
66
|
state_path,
|
|
68
67
|
)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def _learning_db_for_config(config) -> Path:
|
|
71
|
+
"""Single learning.db path for daemon migration + engine (F-05).
|
|
72
|
+
|
|
73
|
+
Always resolves through ``canonical_data_root(configured_base_dir=...)`` so
|
|
74
|
+
a custom ``SLMConfig.base_dir`` without env aliases cannot diverge from the
|
|
75
|
+
engine's ``config.base_dir / "learning.db"``.
|
|
76
|
+
"""
|
|
77
|
+
base = getattr(config, "base_dir", None)
|
|
78
|
+
root = canonical_data_root(configured_base_dir=base)
|
|
79
|
+
return root / "learning.db"
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def _memory_db_for_config(config) -> Path:
|
|
83
|
+
"""Memory DB path aligned with :func:`_learning_db_for_config`."""
|
|
84
|
+
base = getattr(config, "base_dir", None)
|
|
85
|
+
root = canonical_data_root(configured_base_dir=base)
|
|
86
|
+
return root / "memory.db"
|
|
69
87
|
from superlocalmemory.learning.source_quality import (
|
|
70
88
|
SourceQualityRepairUnavailable,
|
|
71
89
|
enumerate_source_quality_repair_profiles,
|
|
@@ -111,6 +129,10 @@ _SENSITIVE_READ_PREFIXES = (
|
|
|
111
129
|
"/api/v3/mesh/config", "/api/v3/trust/config",
|
|
112
130
|
"/api/v3/forgetting/config", "/api/v3/mcp/profiles",
|
|
113
131
|
"/api/learning", "/api/behavioral",
|
|
132
|
+
# Event stream, agent activity, trust signals, and v3 profiling data
|
|
133
|
+
# expose cross-agent coordination signals and behavioral profiles.
|
|
134
|
+
"/events", "/api/events", "/api/agents", "/api/trust/",
|
|
135
|
+
"/api/v3/abstraction", "/api/v3/insights",
|
|
114
136
|
)
|
|
115
137
|
_SENSITIVE_READ_EXACT_PATHS = (
|
|
116
138
|
"/api/search", "/api/v3/recall/trace", "/api/patterns",
|
|
@@ -161,6 +183,57 @@ def _rbac_read_gate(request, app_state):
|
|
|
161
183
|
content={"error": "Your role cannot read this workspace."})
|
|
162
184
|
|
|
163
185
|
|
|
186
|
+
# Adapter process-control routes that change running state (start/stop) or
|
|
187
|
+
# persistence state (enable/disable). These require WRITE permission.
|
|
188
|
+
_ADAPTER_CONTROL_PREFIXES = (
|
|
189
|
+
"/api/adapters/enable",
|
|
190
|
+
"/api/adapters/disable",
|
|
191
|
+
"/api/adapters/start",
|
|
192
|
+
"/api/adapters/stop",
|
|
193
|
+
)
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
def _is_adapter_control_mutation(method: str, path: str) -> bool:
|
|
197
|
+
return (
|
|
198
|
+
method in ("POST", "PUT", "PATCH", "DELETE")
|
|
199
|
+
and path.startswith(_ADAPTER_CONTROL_PREFIXES)
|
|
200
|
+
)
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def _rbac_write_gate(request, app_state):
|
|
204
|
+
"""RBAC gate for state-changing dashboard mutations. Returns a JSONResponse
|
|
205
|
+
to reject, or None to allow. No-op unless RBAC is active (>=1 user).
|
|
206
|
+
|
|
207
|
+
Mirror of _rbac_read_gate but checks WRITE instead of READ permission.
|
|
208
|
+
Fails closed on RBAC errors (503), consistent with _rbac_read_gate.
|
|
209
|
+
"""
|
|
210
|
+
from fastapi.responses import JSONResponse
|
|
211
|
+
rbac = getattr(app_state, "rbac", None)
|
|
212
|
+
if rbac is None:
|
|
213
|
+
return None
|
|
214
|
+
try:
|
|
215
|
+
active = rbac.user_count() > 0
|
|
216
|
+
except Exception:
|
|
217
|
+
return JSONResponse(status_code=503,
|
|
218
|
+
content={"error": "authorization temporarily unavailable"})
|
|
219
|
+
if not active:
|
|
220
|
+
return None # single-operator install — mutations are open
|
|
221
|
+
token = (request.headers.get("x-slm-user-session", "")
|
|
222
|
+
or (request.cookies.get("slm_session", "") if request.cookies else ""))
|
|
223
|
+
user = rbac.resolve_session(token) if token else None
|
|
224
|
+
if user is None:
|
|
225
|
+
if rbac.require_login():
|
|
226
|
+
return JSONResponse(status_code=401,
|
|
227
|
+
content={"error": "Login required to perform this action."})
|
|
228
|
+
return None # owner/operator, personal mode
|
|
229
|
+
from superlocalmemory.access.rbac import Permission
|
|
230
|
+
from superlocalmemory.server.routes.helpers import get_active_profile
|
|
231
|
+
if rbac.has_permission(user["user_id"], get_active_profile(), Permission.WRITE):
|
|
232
|
+
return None
|
|
233
|
+
return JSONResponse(status_code=403,
|
|
234
|
+
content={"error": "Your role cannot modify this workspace."})
|
|
235
|
+
|
|
236
|
+
|
|
164
237
|
def _configured_daemon_port() -> int:
|
|
165
238
|
"""Return the configured bind port, falling back safely to the default."""
|
|
166
239
|
try:
|
|
@@ -169,6 +242,35 @@ def _configured_daemon_port() -> int:
|
|
|
169
242
|
return _DEFAULT_PORT
|
|
170
243
|
|
|
171
244
|
|
|
245
|
+
# Paths that must remain reachable when required schema migrations have failed.
|
|
246
|
+
# All other paths are feature routes subject to the readiness gate.
|
|
247
|
+
_MIGRATION_EXEMPT_PATH_PREFIXES: tuple[str, ...] = (
|
|
248
|
+
"/health",
|
|
249
|
+
"/status",
|
|
250
|
+
"/api/v3/health",
|
|
251
|
+
"/api/health",
|
|
252
|
+
"/api/status",
|
|
253
|
+
"/api/v3/components", # component status/heal endpoints
|
|
254
|
+
"/api/v3/repair",
|
|
255
|
+
"/api/repair",
|
|
256
|
+
"/api/version", # version probe (non-schema-dependent)
|
|
257
|
+
"/static", # UI assets (served regardless of schema state)
|
|
258
|
+
)
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
def _is_migration_exempt_path(path: str) -> bool:
|
|
262
|
+
"""Return True for health, status, and repair paths that must stay reachable
|
|
263
|
+
even when the daemon reports a schema migration failure.
|
|
264
|
+
|
|
265
|
+
All other paths are considered feature routes whose correctness depends on
|
|
266
|
+
the schema being fully applied.
|
|
267
|
+
"""
|
|
268
|
+
return any(
|
|
269
|
+
path == prefix or path.startswith(prefix + "/") or path.startswith(prefix + "?")
|
|
270
|
+
for prefix in _MIGRATION_EXEMPT_PATH_PREFIXES
|
|
271
|
+
)
|
|
272
|
+
|
|
273
|
+
|
|
172
274
|
def _process_descriptor(port: int, version: str, state: str) -> DaemonDescriptor:
|
|
173
275
|
"""Return this process's stable namespace/instance identity."""
|
|
174
276
|
global _ACTIVE_DAEMON_DESCRIPTOR
|
|
@@ -377,6 +479,14 @@ def _hot_reconfigure_engine(application, new_config, *, mode_change: bool) -> No
|
|
|
377
479
|
old_engine = getattr(application.state, "engine", None)
|
|
378
480
|
new_engine = MemoryEngine(new_config)
|
|
379
481
|
try:
|
|
482
|
+
if mode_change:
|
|
483
|
+
# A mode switch may change the embedding dimension. VectorStore
|
|
484
|
+
# detects the mismatch on init and rebuilds the vec0 table at the
|
|
485
|
+
# new dimension automatically (see VectorStore._ensure_vec0_table).
|
|
486
|
+
logger.info(
|
|
487
|
+
"Mode change detected: vector index will be rebuilt at new "
|
|
488
|
+
"embedding dimension if dimension changed."
|
|
489
|
+
)
|
|
380
490
|
new_engine.initialize()
|
|
381
491
|
new_config.save(mode_change=mode_change)
|
|
382
492
|
except BaseException:
|
|
@@ -435,7 +545,11 @@ def _hot_reconfigure_engine(application, new_config, *, mode_change: bool) -> No
|
|
|
435
545
|
|
|
436
546
|
from superlocalmemory.core.recall_gate import (
|
|
437
547
|
begin_recall as _begin_recall,
|
|
548
|
+
)
|
|
549
|
+
from superlocalmemory.core.recall_gate import (
|
|
438
550
|
end_recall as _end_recall,
|
|
551
|
+
)
|
|
552
|
+
from superlocalmemory.core.recall_gate import (
|
|
439
553
|
in_flight as _recalls_in_flight,
|
|
440
554
|
)
|
|
441
555
|
|
|
@@ -479,6 +593,7 @@ def _emit_event(
|
|
|
479
593
|
# lock queues, and total wall time is N × single-recall-time. 3 concurrent
|
|
480
594
|
# full recalls gives parallelism benefit without resource oversaturation.
|
|
481
595
|
import asyncio as _asyncio
|
|
596
|
+
|
|
482
597
|
_recall_semaphore = _asyncio.Semaphore(3)
|
|
483
598
|
|
|
484
599
|
|
|
@@ -619,11 +734,11 @@ class ObserveBuffer:
|
|
|
619
734
|
}
|
|
620
735
|
|
|
621
736
|
try:
|
|
622
|
-
from superlocalmemory.hooks.auto_capture import AutoCapture
|
|
623
737
|
from superlocalmemory.core.engine_ingestion import (
|
|
624
738
|
build_engine_ingestion_command,
|
|
625
739
|
)
|
|
626
740
|
from superlocalmemory.core.ingestion_command import IngestionRequest
|
|
741
|
+
from superlocalmemory.hooks.auto_capture import AutoCapture
|
|
627
742
|
|
|
628
743
|
decision = AutoCapture().evaluate(content)
|
|
629
744
|
if not decision.capture:
|
|
@@ -1240,6 +1355,62 @@ def _release_canonical_remember_runtime(application, runtime=None) -> bool:
|
|
|
1240
1355
|
return True
|
|
1241
1356
|
|
|
1242
1357
|
|
|
1358
|
+
def _apply_readiness_gate(runtime, application) -> None:
|
|
1359
|
+
"""Abort startup when the writer lease is held but the runtime is not serving.
|
|
1360
|
+
|
|
1361
|
+
A swallowed exception in a critical startup step can leave the coordinator
|
|
1362
|
+
worker dead while the file lock remains held. Continuing to yield in that
|
|
1363
|
+
state keeps the lease on an unusable runtime, blocking any recovery process.
|
|
1364
|
+
This gate detects the broken state, releases the lease, and raises so the
|
|
1365
|
+
daemon exits cleanly and ownership transfers to the next start attempt.
|
|
1366
|
+
|
|
1367
|
+
Calling this with a ready or absent runtime is a no-op.
|
|
1368
|
+
"""
|
|
1369
|
+
if runtime is None or runtime.ready:
|
|
1370
|
+
return
|
|
1371
|
+
_release_canonical_remember_runtime(application, runtime)
|
|
1372
|
+
raise RuntimeError(
|
|
1373
|
+
"daemon writer runtime is not ready after startup — "
|
|
1374
|
+
"releasing writer lease so a recovery process can start"
|
|
1375
|
+
)
|
|
1376
|
+
|
|
1377
|
+
|
|
1378
|
+
def _apply_deployment_runtime(config, deployment) -> None:
|
|
1379
|
+
"""Upgrade engine policy from deployment config without downgrading it."""
|
|
1380
|
+
if deployment.pii_redaction and not getattr(config, "pii_redaction", False):
|
|
1381
|
+
config.pii_redaction = True
|
|
1382
|
+
|
|
1383
|
+
|
|
1384
|
+
def _start_deployment_retention(application, config, deployment) -> None:
|
|
1385
|
+
"""Start the named-rule retention scheduler when explicitly enabled."""
|
|
1386
|
+
application.state.retention_scheduler = None
|
|
1387
|
+
application.state.retention_connection = None
|
|
1388
|
+
if not deployment.retention_enabled:
|
|
1389
|
+
return
|
|
1390
|
+
|
|
1391
|
+
from superlocalmemory.compliance.scheduler import RetentionScheduler
|
|
1392
|
+
|
|
1393
|
+
scheduler = RetentionScheduler(db_path=config.db_path)
|
|
1394
|
+
scheduler.start()
|
|
1395
|
+
application.state.retention_scheduler = scheduler
|
|
1396
|
+
|
|
1397
|
+
|
|
1398
|
+
def _stop_deployment_retention(application) -> bool:
|
|
1399
|
+
"""Stop retention and report whether its writer thread fully terminated."""
|
|
1400
|
+
scheduler = getattr(application.state, "retention_scheduler", None)
|
|
1401
|
+
if scheduler is None:
|
|
1402
|
+
return True
|
|
1403
|
+
try:
|
|
1404
|
+
stopped = scheduler.stop()
|
|
1405
|
+
except Exception as exc: # pragma: no cover - defensive shutdown
|
|
1406
|
+
logger.error("retention scheduler stop failed: %s", exc)
|
|
1407
|
+
return False
|
|
1408
|
+
if stopped is False:
|
|
1409
|
+
logger.error("retention scheduler did not stop within its shutdown budget")
|
|
1410
|
+
return False
|
|
1411
|
+
return True
|
|
1412
|
+
|
|
1413
|
+
|
|
1243
1414
|
@asynccontextmanager
|
|
1244
1415
|
async def lifespan(application: FastAPI):
|
|
1245
1416
|
"""Initialize engine, workers, and optional services on startup."""
|
|
@@ -1247,6 +1418,7 @@ async def lifespan(application: FastAPI):
|
|
|
1247
1418
|
|
|
1248
1419
|
engine = None
|
|
1249
1420
|
config = None
|
|
1421
|
+
deployment = None
|
|
1250
1422
|
canonical_remember_runtime = None
|
|
1251
1423
|
profile_runtime = None
|
|
1252
1424
|
|
|
@@ -1317,11 +1489,35 @@ async def lifespan(application: FastAPI):
|
|
|
1317
1489
|
# LLD-06 §7.3 / LLD-07 §4.1 — run additive schema migrations BEFORE
|
|
1318
1490
|
# engine init so later queries see the expected columns/tables.
|
|
1319
1491
|
# Non-fatal: any failure here is logged and the daemon still starts.
|
|
1492
|
+
#
|
|
1493
|
+
# F-05: resolve learning.db / memory.db from the same configured base
|
|
1494
|
+
# the engine will use (config.base_dir). Bare canonical_data_root() skips
|
|
1495
|
+
# configured_base_dir and can migrate ~/.superlocalmemory while the
|
|
1496
|
+
# engine writes a custom root.
|
|
1497
|
+
_path_config = None
|
|
1498
|
+
try:
|
|
1499
|
+
from superlocalmemory.core.config import SLMConfig as _SLMConfigForPaths
|
|
1500
|
+
|
|
1501
|
+
try:
|
|
1502
|
+
_SLMConfigForPaths.migrate_to_3mode()
|
|
1503
|
+
except Exception as _m3_exc: # pragma: no cover — non-fatal path prep
|
|
1504
|
+
logger.debug("migrate_to_3mode before path resolve: %s", _m3_exc)
|
|
1505
|
+
_path_config = _SLMConfigForPaths.load()
|
|
1506
|
+
except Exception as _cfg_exc: # pragma: no cover — fall back below
|
|
1507
|
+
logger.debug("early config load for migration paths failed: %s", _cfg_exc)
|
|
1508
|
+
_path_config = None
|
|
1509
|
+
|
|
1320
1510
|
try:
|
|
1321
1511
|
from superlocalmemory.storage.migration_runner import apply_all
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1512
|
+
if _path_config is None:
|
|
1513
|
+
# Catastrophic early-load failure: still one root via Mode-A default
|
|
1514
|
+
# (runtime canonical root), never a bare second resolver branch.
|
|
1515
|
+
from superlocalmemory.core.config import SLMConfig as _SLMFallback
|
|
1516
|
+
from superlocalmemory.storage.models import Mode as _ModeFallback
|
|
1517
|
+
|
|
1518
|
+
_path_config = _SLMFallback.for_mode(_ModeFallback.A)
|
|
1519
|
+
_learning_db = _learning_db_for_config(_path_config)
|
|
1520
|
+
_memory_db = _memory_db_for_config(_path_config)
|
|
1325
1521
|
_result = apply_all(_learning_db, _memory_db)
|
|
1326
1522
|
_applied = _result.get("applied", [])
|
|
1327
1523
|
_failed = _result.get("failed", [])
|
|
@@ -1353,26 +1549,109 @@ async def lifespan(application: FastAPI):
|
|
|
1353
1549
|
"details": {"_crash": str(_exc)},
|
|
1354
1550
|
}
|
|
1355
1551
|
|
|
1552
|
+
# H1 — boot self-heal: remove provably-dead SLM lock/PID artifacts BEFORE
|
|
1553
|
+
# claiming the writer. The writer claim uses a portalocker OS advisory lock
|
|
1554
|
+
# that auto-releases on holder death, so stale artifacts are metadata-only
|
|
1555
|
+
# leftovers. Fail-soft — a crash here is logged but never blocks startup.
|
|
1556
|
+
try:
|
|
1557
|
+
from superlocalmemory.infra.self_heal import reap_stale_artifacts
|
|
1558
|
+
|
|
1559
|
+
# NOTE: canonical_data_root is imported at module scope (top of file).
|
|
1560
|
+
# Do NOT re-import it locally here — a local import would shadow the
|
|
1561
|
+
# module name for this whole startup function and make the earlier
|
|
1562
|
+
# migration-runner reference raise UnboundLocalError.
|
|
1563
|
+
_sh_report = reap_stale_artifacts(canonical_data_root())
|
|
1564
|
+
if _sh_report["removed"]:
|
|
1565
|
+
logger.info(
|
|
1566
|
+
"boot self-heal: removed %d stale artifact(s)",
|
|
1567
|
+
len(_sh_report["removed"]),
|
|
1568
|
+
)
|
|
1569
|
+
if _sh_report["errors"]:
|
|
1570
|
+
logger.debug(
|
|
1571
|
+
"boot self-heal: %d removal error(s) (non-fatal)",
|
|
1572
|
+
len(_sh_report["errors"]),
|
|
1573
|
+
)
|
|
1574
|
+
except Exception as _sh_exc:
|
|
1575
|
+
logger.debug("boot self-heal failed (non-fatal): %s", _sh_exc)
|
|
1576
|
+
|
|
1577
|
+
# H4 — mesh lock TTL expiry: delete expired mesh_locks rows on boot so
|
|
1578
|
+
# dead-node leases are cleared before any actor reads the lock table.
|
|
1579
|
+
# mesh_locks lives in memory.db. Fail-soft — never blocks startup.
|
|
1580
|
+
try:
|
|
1581
|
+
from superlocalmemory.infra.self_heal import expire_stale_mesh_locks
|
|
1582
|
+
|
|
1583
|
+
_mesh_db = canonical_data_root() / "memory.db"
|
|
1584
|
+
_mesh_expired = expire_stale_mesh_locks(_mesh_db)
|
|
1585
|
+
if _mesh_expired:
|
|
1586
|
+
logger.info("boot mesh expiry: cleared %d stale mesh_lock row(s)", _mesh_expired)
|
|
1587
|
+
except Exception as _mesh_exc:
|
|
1588
|
+
logger.debug("boot mesh expiry failed (non-fatal): %s", _mesh_exc)
|
|
1589
|
+
|
|
1590
|
+
# H3 — process reaper: kill orphaned SLM child processes whose parent is
|
|
1591
|
+
# dead. Wires the existing infra/process_reaper.py at daemon start.
|
|
1592
|
+
# Fail-soft — never blocks the startup path.
|
|
1593
|
+
try:
|
|
1594
|
+
from superlocalmemory.infra.process_reaper import ReaperConfig, reap_stale_on_startup
|
|
1595
|
+
from superlocalmemory.infra.pid_manager import PidManager
|
|
1596
|
+
from superlocalmemory.infra.data_root import canonical_data_root as _cdr
|
|
1597
|
+
|
|
1598
|
+
_rpr_cfg = ReaperConfig(orphan_age_threshold_hours=0.0)
|
|
1599
|
+
_rpr_mgr = PidManager(_cdr() / "slm.pids")
|
|
1600
|
+
reap_stale_on_startup(_rpr_cfg, _rpr_mgr)
|
|
1601
|
+
except Exception as _rpr_exc:
|
|
1602
|
+
logger.debug("boot process reaper failed (non-fatal): %s", _rpr_exc)
|
|
1603
|
+
|
|
1356
1604
|
try:
|
|
1357
1605
|
from superlocalmemory.core.config import SLMConfig
|
|
1358
1606
|
from superlocalmemory.core.engine import MemoryEngine
|
|
1359
1607
|
|
|
1360
1608
|
# v3.4.54: one-time migration config.json → 3-mode system
|
|
1609
|
+
# (also attempted earlier for F-05 path resolution; idempotent)
|
|
1361
1610
|
SLMConfig.migrate_to_3mode()
|
|
1362
1611
|
|
|
1363
|
-
config = SLMConfig.load()
|
|
1612
|
+
config = _path_config if _path_config is not None else SLMConfig.load()
|
|
1613
|
+
from superlocalmemory.core.config import load_deployment_config
|
|
1614
|
+
deployment = load_deployment_config()
|
|
1615
|
+
_apply_deployment_runtime(config, deployment)
|
|
1616
|
+
application.state.deployment = deployment
|
|
1364
1617
|
engine = MemoryEngine(config)
|
|
1365
1618
|
engine.initialize()
|
|
1366
1619
|
|
|
1620
|
+
# Refresh migration state now that the engine is initialised. Any
|
|
1621
|
+
# schema work the engine's own bootstrap may have applied (e.g.
|
|
1622
|
+
# runtime-table creation) is captured here so the dashboard and
|
|
1623
|
+
# health endpoints see a current, consistent picture. Non-fatal:
|
|
1624
|
+
# a runner crash here is logged but must not block the startup path.
|
|
1625
|
+
try:
|
|
1626
|
+
_result = apply_all(_learning_db, _memory_db)
|
|
1627
|
+
application.state.migration_result = _result
|
|
1628
|
+
except Exception as _ref_exc: # pragma: no cover — defensive
|
|
1629
|
+
logger.debug("migration state refresh failed (non-fatal): %s", _ref_exc)
|
|
1630
|
+
|
|
1367
1631
|
# 3.8.6 canonical remember boundary. Migrations have already created
|
|
1368
1632
|
# the immutable receipt ledger and engine init has created the runtime
|
|
1369
1633
|
# tables. Claim the daemon's writer lease, install the typed admission
|
|
1370
1634
|
# handler, and replay crash-surviving journal entries *before* any
|
|
1371
1635
|
# background writer or ready descriptor is published.
|
|
1372
|
-
from superlocalmemory.core.remember_runtime import
|
|
1636
|
+
from superlocalmemory.core.remember_runtime import (
|
|
1637
|
+
CanonicalRememberRuntime,
|
|
1638
|
+
CanonicalRememberUnavailable,
|
|
1639
|
+
DaemonAlreadyServing,
|
|
1640
|
+
)
|
|
1373
1641
|
|
|
1374
1642
|
canonical_remember_runtime = CanonicalRememberRuntime.for_engine(engine)
|
|
1375
|
-
|
|
1643
|
+
try:
|
|
1644
|
+
canonical_remember_runtime.start()
|
|
1645
|
+
except (DaemonAlreadyServing, CanonicalRememberUnavailable):
|
|
1646
|
+
# H2 — graceful single-instance: a health-verified daemon is already
|
|
1647
|
+
# serving (DaemonAlreadyServing), OR the bounded retry loop exhausted
|
|
1648
|
+
# (CanonicalRememberUnavailable). Belt-and-suspenders: a lost writer
|
|
1649
|
+
# race must NEVER be a traceback for a non-technical user.
|
|
1650
|
+
logger.info(
|
|
1651
|
+
"another SLM daemon holds the writer; this instance exits cleanly"
|
|
1652
|
+
)
|
|
1653
|
+
import sys
|
|
1654
|
+
sys.exit(0)
|
|
1376
1655
|
application.state.canonical_remember_runtime = canonical_remember_runtime
|
|
1377
1656
|
|
|
1378
1657
|
# WAL is already established at DB creation (DatabaseManager._enable_wal
|
|
@@ -1609,7 +1888,8 @@ async def lifespan(application: FastAPI):
|
|
|
1609
1888
|
_t.sleep(0.5)
|
|
1610
1889
|
try:
|
|
1611
1890
|
from superlocalmemory.retrieval.vector_store import (
|
|
1612
|
-
VectorStore,
|
|
1891
|
+
VectorStore,
|
|
1892
|
+
VectorStoreConfig,
|
|
1613
1893
|
)
|
|
1614
1894
|
db = engine._db
|
|
1615
1895
|
db_path = getattr(db, "db_path", None) or getattr(db, "_db_path", None)
|
|
@@ -1752,9 +2032,6 @@ async def lifespan(application: FastAPI):
|
|
|
1752
2032
|
# attempts. No-op when there are no NULLs.
|
|
1753
2033
|
_SELF_HEAL_STATUS["state"] = "backfilling_embeddings"
|
|
1754
2034
|
try:
|
|
1755
|
-
from superlocalmemory.storage.embedding_migrator import (
|
|
1756
|
-
backfill_missing_embeddings,
|
|
1757
|
-
)
|
|
1758
2035
|
# RECALL-PRIORITY THROTTLE: the embedding worker is a single
|
|
1759
2036
|
# serialized subprocess shared with foreground recall. A
|
|
1760
2037
|
# continuous backfill starves interactive query-embedding and
|
|
@@ -1764,6 +2041,9 @@ async def lifespan(application: FastAPI):
|
|
|
1764
2041
|
# uses. This keeps recall responsive throughout the heal (the
|
|
1765
2042
|
# zero-pain requirement); the heal just takes a little longer.
|
|
1766
2043
|
from superlocalmemory.core import recall_gate
|
|
2044
|
+
from superlocalmemory.storage.embedding_migrator import (
|
|
2045
|
+
backfill_missing_embeddings,
|
|
2046
|
+
)
|
|
1767
2047
|
total_embedded = 0
|
|
1768
2048
|
no_progress = 0
|
|
1769
2049
|
for _attempt in range(500):
|
|
@@ -1935,6 +2215,12 @@ async def lifespan(application: FastAPI):
|
|
|
1935
2215
|
|
|
1936
2216
|
application.state.observe_buffer = _observe_buffer
|
|
1937
2217
|
|
|
2218
|
+
# Readiness gate: if the writer lease was claimed but the coordinator is not
|
|
2219
|
+
# serving (e.g., a swallowed exception in a critical startup step killed the
|
|
2220
|
+
# worker thread), release the lease and abort startup. A living process that
|
|
2221
|
+
# holds an unusable lease blocks any recovery daemon from starting.
|
|
2222
|
+
_apply_readiness_gate(canonical_remember_runtime, application)
|
|
2223
|
+
|
|
1938
2224
|
# Phase B: Start health monitor
|
|
1939
2225
|
try:
|
|
1940
2226
|
from superlocalmemory.core.health_monitor import HealthMonitor
|
|
@@ -1971,6 +2257,40 @@ async def lifespan(application: FastAPI):
|
|
|
1971
2257
|
logger.warning("Mesh broker init failed: %s", exc)
|
|
1972
2258
|
application.state.mesh_broker = None
|
|
1973
2259
|
|
|
2260
|
+
# Phase C-2: opt-in mDNS advertiser (default OFF — backward-compatible).
|
|
2261
|
+
# Enabled only when SLM_MESH_ADVERTISE=1|on|true|yes is set explicitly.
|
|
2262
|
+
# Gating on the env var BEFORE importing discovery keeps the default-off
|
|
2263
|
+
# path byte-for-byte identical to pre-3b (no discovery import, no thread
|
|
2264
|
+
# hop, no object alloc). See mesh.discovery.MeshAdvertiser for full docs.
|
|
2265
|
+
application.state.mesh_advertiser = None
|
|
2266
|
+
_advertise_requested = (
|
|
2267
|
+
os.environ.get("SLM_MESH_ADVERTISE", "").strip().lower()
|
|
2268
|
+
in ("1", "on", "true", "yes")
|
|
2269
|
+
)
|
|
2270
|
+
if _advertise_requested and application.state.mesh_broker is None:
|
|
2271
|
+
# Operator asked for advertising but the broker isn't up — make the
|
|
2272
|
+
# skip diagnosable rather than silent (audit P2).
|
|
2273
|
+
logger.warning(
|
|
2274
|
+
"mDNS advertise requested (SLM_MESH_ADVERTISE) but mesh broker is "
|
|
2275
|
+
"not running — advertising skipped"
|
|
2276
|
+
)
|
|
2277
|
+
elif _advertise_requested and application.state.mesh_broker is not None:
|
|
2278
|
+
try:
|
|
2279
|
+
import socket as _adv_sock
|
|
2280
|
+
from superlocalmemory.mesh.discovery import MeshAdvertiser as _MeshAdvertiser
|
|
2281
|
+
_adv_node_id = _adv_sock.gethostname()
|
|
2282
|
+
_advertiser = _MeshAdvertiser(
|
|
2283
|
+
service_port=_configured_daemon_port(),
|
|
2284
|
+
node_id=_adv_node_id,
|
|
2285
|
+
)
|
|
2286
|
+
# register_service blocks ~750 ms (mDNS probe phase) — run off the
|
|
2287
|
+
# event loop to avoid stalling the async startup path (CRIT fix #2).
|
|
2288
|
+
await asyncio.to_thread(_advertiser.start)
|
|
2289
|
+
application.state.mesh_advertiser = _advertiser
|
|
2290
|
+
except Exception as exc:
|
|
2291
|
+
logger.warning("mDNS advertiser init failed (non-fatal): %s", exc)
|
|
2292
|
+
application.state.mesh_advertiser = None
|
|
2293
|
+
|
|
1974
2294
|
# RBAC / teams (C3): user identity + role enforcement over memory.db.
|
|
1975
2295
|
# Additive — with zero users the daemon stays single-operator (owner).
|
|
1976
2296
|
try:
|
|
@@ -1990,8 +2310,9 @@ async def lifespan(application: FastAPI):
|
|
|
1990
2310
|
# ENFORCE rule: only UPGRADE a setting, NEVER downgrade an already-stronger
|
|
1991
2311
|
# runtime setting (e.g. RBAC require_login already True → leave it alone).
|
|
1992
2312
|
try:
|
|
1993
|
-
|
|
1994
|
-
|
|
2313
|
+
if deployment is None:
|
|
2314
|
+
from superlocalmemory.core.config import load_deployment_config
|
|
2315
|
+
deployment = load_deployment_config()
|
|
1995
2316
|
application.state.deployment = deployment
|
|
1996
2317
|
if deployment.require_login:
|
|
1997
2318
|
_dep_rbac = getattr(application.state, "rbac", None)
|
|
@@ -2000,8 +2321,7 @@ async def lifespan(application: FastAPI):
|
|
|
2000
2321
|
logger.info(
|
|
2001
2322
|
"Deployment: require_login enforced via enterprise deployment config"
|
|
2002
2323
|
)
|
|
2003
|
-
|
|
2004
|
-
# TODO: Wire deployment.retention_enabled → retention scheduler (WP-11)
|
|
2324
|
+
_start_deployment_retention(application, config, deployment)
|
|
2005
2325
|
logger.info(
|
|
2006
2326
|
"Deployment config loaded: mode=%s require_login=%s "
|
|
2007
2327
|
"pii=%s retention=%s audit=%s",
|
|
@@ -2012,6 +2332,26 @@ async def lifespan(application: FastAPI):
|
|
|
2012
2332
|
logger.warning("Deployment config wire failed (non-fatal): %s", _dep_exc)
|
|
2013
2333
|
application.state.deployment = None
|
|
2014
2334
|
|
|
2335
|
+
# Phase 1: Admission gateway coverage self-check.
|
|
2336
|
+
# Uses _resolve_deployment() (fail-closed) rather than application.state.deployment
|
|
2337
|
+
# (which may be None if the load failed) so an enterprise box with a broken
|
|
2338
|
+
# config never silently skips a hard coverage check.
|
|
2339
|
+
# G-tranche: pass the real MCP server so dynamic tool enumeration catches
|
|
2340
|
+
# any newly-added tool that lacks both @admits and readOnlyHint=True.
|
|
2341
|
+
try:
|
|
2342
|
+
from superlocalmemory.core.admission import (
|
|
2343
|
+
_resolve_deployment as _adm_resolve,
|
|
2344
|
+
coverage_self_check,
|
|
2345
|
+
)
|
|
2346
|
+
from superlocalmemory.mcp import server as _mcp_server_mod
|
|
2347
|
+
_cov_deployment = _adm_resolve()
|
|
2348
|
+
_mcp_server = getattr(_mcp_server_mod, "server", None)
|
|
2349
|
+
coverage_self_check(_cov_deployment, server=_mcp_server)
|
|
2350
|
+
except RuntimeError as _cov_exc:
|
|
2351
|
+
raise
|
|
2352
|
+
except Exception as _cov_exc:
|
|
2353
|
+
logger.warning("admission coverage self-check failed (non-fatal): %s", _cov_exc)
|
|
2354
|
+
|
|
2015
2355
|
# Start idle watchdog if configured
|
|
2016
2356
|
idle_timeout = int(os.environ.get("SLM_DAEMON_IDLE_TIMEOUT", "0"))
|
|
2017
2357
|
if config and hasattr(config, 'daemon_idle_timeout'):
|
|
@@ -2089,10 +2429,10 @@ async def lifespan(application: FastAPI):
|
|
|
2089
2429
|
|
|
2090
2430
|
# V3.6: Mount optimize API routes + restore persisted metrics + start flush loop
|
|
2091
2431
|
try:
|
|
2092
|
-
from superlocalmemory.server.routes.optimize import router as optimize_router
|
|
2093
2432
|
from superlocalmemory.optimize.metrics.counters import MetricsCollector
|
|
2094
2433
|
from superlocalmemory.optimize.metrics.persistence import MetricsPersistence
|
|
2095
2434
|
from superlocalmemory.optimize.storage.db import CacheDB
|
|
2435
|
+
from superlocalmemory.server.routes.optimize import router as optimize_router
|
|
2096
2436
|
application.include_router(optimize_router)
|
|
2097
2437
|
|
|
2098
2438
|
# Restore persisted metrics counters on startup.
|
|
@@ -2132,6 +2472,15 @@ async def lifespan(application: FastAPI):
|
|
|
2132
2472
|
# or tool-level cancellation inside a session manager task group cannot
|
|
2133
2473
|
# propagate out and trigger uvicorn's graceful-shutdown handler.
|
|
2134
2474
|
async with AsyncExitStack() as _mcp_stack:
|
|
2475
|
+
# Belt-and-suspenders: if an unexpected error occurs between here and
|
|
2476
|
+
# yield, the stack's __aexit__ ensures the lease is released even when
|
|
2477
|
+
# the normal teardown path (after yield) is bypassed. The explicit
|
|
2478
|
+
# _release_canonical_remember_runtime call in teardown is idempotent.
|
|
2479
|
+
if canonical_remember_runtime is not None:
|
|
2480
|
+
_mcp_stack.callback(
|
|
2481
|
+
_release_canonical_remember_runtime, application, canonical_remember_runtime
|
|
2482
|
+
)
|
|
2483
|
+
|
|
2135
2484
|
if _mcp_app is not None:
|
|
2136
2485
|
try:
|
|
2137
2486
|
await _mcp_stack.enter_async_context(
|
|
@@ -2199,8 +2548,8 @@ async def lifespan(application: FastAPI):
|
|
|
2199
2548
|
pass
|
|
2200
2549
|
# Final flush to persist the last window (H-04: use singleton)
|
|
2201
2550
|
try:
|
|
2202
|
-
from superlocalmemory.optimize.metrics.persistence import MetricsPersistence
|
|
2203
2551
|
from superlocalmemory.optimize.metrics.counters import MetricsCollector
|
|
2552
|
+
from superlocalmemory.optimize.metrics.persistence import MetricsPersistence
|
|
2204
2553
|
from superlocalmemory.optimize.storage.db import CacheDB as _FinalCacheDB
|
|
2205
2554
|
MetricsPersistence().flush(
|
|
2206
2555
|
MetricsCollector.get_instance(),
|
|
@@ -2235,6 +2584,8 @@ async def lifespan(application: FastAPI):
|
|
|
2235
2584
|
# not skip the rest.
|
|
2236
2585
|
_observe_buffer.flush_sync()
|
|
2237
2586
|
|
|
2587
|
+
_retention_stopped = _stop_deployment_retention(application)
|
|
2588
|
+
|
|
2238
2589
|
# S9-DASH-02: stop outcome-queue worker (final drain on graceful
|
|
2239
2590
|
# shutdown). Any events left unpersisted are logged but not
|
|
2240
2591
|
# replayed — signal capture is not load-bearing on correctness.
|
|
@@ -2317,6 +2668,16 @@ async def lifespan(application: FastAPI):
|
|
|
2317
2668
|
except Exception as exc: # pragma: no cover — defensive
|
|
2318
2669
|
logger.warning("mesh_broker stop failed: %s", exc)
|
|
2319
2670
|
|
|
2671
|
+
# Stop mDNS advertiser (symmetric with Phase C-2 startup). stop() joins
|
|
2672
|
+
# Zeroconf background threads (blocking) — run it OFF the event loop so
|
|
2673
|
+
# shutdown of other async resources isn't stalled (audit P1).
|
|
2674
|
+
_adv = getattr(application.state, "mesh_advertiser", None)
|
|
2675
|
+
if _adv is not None:
|
|
2676
|
+
try:
|
|
2677
|
+
await asyncio.to_thread(_adv.stop)
|
|
2678
|
+
except Exception as exc: # pragma: no cover — defensive
|
|
2679
|
+
logger.warning("mDNS advertiser stop failed (non-fatal): %s", exc)
|
|
2680
|
+
|
|
2320
2681
|
# LLD-02 SW3: flush pending signals to DB before closing. Bounded 3 s
|
|
2321
2682
|
# to keep daemon shutdown snappy; drops + counts anything unwritten.
|
|
2322
2683
|
if getattr(application.state, "signal_worker_started", False):
|
|
@@ -2379,28 +2740,71 @@ async def lifespan(application: FastAPI):
|
|
|
2379
2740
|
getattr(application.state, "daemon_descriptor", None),
|
|
2380
2741
|
)
|
|
2381
2742
|
logger.info("Unified daemon shutdown complete")
|
|
2743
|
+
if not _retention_stopped:
|
|
2744
|
+
raise RuntimeError(
|
|
2745
|
+
"retention scheduler remained active during daemon shutdown"
|
|
2746
|
+
)
|
|
2382
2747
|
|
|
2383
2748
|
|
|
2384
2749
|
# ---------------------------------------------------------------------------
|
|
2385
2750
|
# App factory
|
|
2386
2751
|
# ---------------------------------------------------------------------------
|
|
2387
2752
|
|
|
2388
|
-
def _configure_mcp_transport_settings(
|
|
2389
|
-
"""
|
|
2753
|
+
def _configure_mcp_transport_settings() -> dict:
|
|
2754
|
+
"""Return kwargs for ``MCPServer.streamable_http_app(...)`` (mcp 2.0.0).
|
|
2390
2755
|
|
|
2391
|
-
``
|
|
2392
|
-
|
|
2393
|
-
|
|
2394
|
-
|
|
2395
|
-
|
|
2396
|
-
|
|
2756
|
+
FastMCP's mutable ``settings.stateless_http`` / ``.json_response`` /
|
|
2757
|
+
``.streamable_http_path`` / ``.transport_security`` are gone in mcp 2.0.0.
|
|
2758
|
+
Those values are now keyword arguments to ``streamable_http_app()``.
|
|
2759
|
+
|
|
2760
|
+
Fully-stateless is the default (``remote_mode.mcp_stateless()``). Under
|
|
2761
|
+
``stateless_http=True`` we never pass ``session_idle_timeout`` or an
|
|
2762
|
+
event store — both are illegal/unused for transport sessions that do not
|
|
2763
|
+
exist. Callers pass the returned dict through as
|
|
2764
|
+
``server.streamable_http_app(**kwargs)``.
|
|
2397
2765
|
"""
|
|
2766
|
+
from typing import Any
|
|
2767
|
+
|
|
2398
2768
|
from superlocalmemory.core.remote_mode import mcp_stateless
|
|
2399
2769
|
|
|
2400
2770
|
stateless = bool(mcp_stateless())
|
|
2401
|
-
|
|
2402
|
-
|
|
2403
|
-
|
|
2771
|
+
kwargs: dict[str, Any] = {
|
|
2772
|
+
"streamable_http_path": "/",
|
|
2773
|
+
"stateless_http": stateless,
|
|
2774
|
+
# json_response pairs with stateless so tool results complete as a
|
|
2775
|
+
# single HTTP body (no long-lived SSE). Required for reliable
|
|
2776
|
+
# gateway/hub forwarding and TestClient e2e.
|
|
2777
|
+
"json_response": True if stateless else False,
|
|
2778
|
+
"event_store": None,
|
|
2779
|
+
"host": "127.0.0.1",
|
|
2780
|
+
}
|
|
2781
|
+
|
|
2782
|
+
# DNS-rebinding protection from env. Default: SDK localhost auto-protect
|
|
2783
|
+
# (host=127.0.0.1). Set SLM_MCP_ALLOWED_HOSTS=... to open to a LAN, or
|
|
2784
|
+
# "*" to disable protection entirely (trusted private network only).
|
|
2785
|
+
_mcp_allowed = os.environ.get("SLM_MCP_ALLOWED_HOSTS", "").strip()
|
|
2786
|
+
if _mcp_allowed:
|
|
2787
|
+
from mcp.server.transport_security import TransportSecuritySettings
|
|
2788
|
+
|
|
2789
|
+
if _mcp_allowed == "*":
|
|
2790
|
+
logger.warning(
|
|
2791
|
+
"SLM_MCP_ALLOWED_HOSTS=* disables MCP DNS-rebinding "
|
|
2792
|
+
"protection entirely. Prefer an explicit host list; only "
|
|
2793
|
+
"use '*' on a trusted private network."
|
|
2794
|
+
)
|
|
2795
|
+
kwargs["transport_security"] = TransportSecuritySettings(
|
|
2796
|
+
enable_dns_rebinding_protection=False,
|
|
2797
|
+
)
|
|
2798
|
+
else:
|
|
2799
|
+
_hosts = [h.strip() for h in _mcp_allowed.split(",") if h.strip()]
|
|
2800
|
+
kwargs["transport_security"] = TransportSecuritySettings(
|
|
2801
|
+
enable_dns_rebinding_protection=True,
|
|
2802
|
+
allowed_hosts=_hosts,
|
|
2803
|
+
allowed_origins=[f"http://{h}" for h in _hosts],
|
|
2804
|
+
)
|
|
2805
|
+
logger.info("MCP transport security: allowed_hosts=%r", _mcp_allowed)
|
|
2806
|
+
|
|
2807
|
+
return kwargs
|
|
2404
2808
|
|
|
2405
2809
|
|
|
2406
2810
|
def create_app() -> FastAPI:
|
|
@@ -2408,7 +2812,7 @@ def create_app() -> FastAPI:
|
|
|
2408
2812
|
from superlocalmemory.server.routes.helpers import SLM_VERSION
|
|
2409
2813
|
|
|
2410
2814
|
application = FastAPI(
|
|
2411
|
-
title="SuperLocalMemory
|
|
2815
|
+
title="SuperLocalMemory V4 — Unified Daemon",
|
|
2412
2816
|
description="Memory + Dashboard + Mesh — one process, one engine.",
|
|
2413
2817
|
version=SLM_VERSION,
|
|
2414
2818
|
lifespan=lifespan,
|
|
@@ -2459,6 +2863,18 @@ def create_app() -> FastAPI:
|
|
|
2459
2863
|
except ImportError:
|
|
2460
2864
|
pass
|
|
2461
2865
|
|
|
2866
|
+
# -- Mesh 3c protocol routes: state-delta (LWW) + lock-delta (fencing) --
|
|
2867
|
+
try:
|
|
2868
|
+
from superlocalmemory.server.routes.mesh_state import router as mesh_state_router
|
|
2869
|
+
application.include_router(mesh_state_router)
|
|
2870
|
+
except ImportError:
|
|
2871
|
+
pass
|
|
2872
|
+
try:
|
|
2873
|
+
from superlocalmemory.server.routes.mesh_lock import router as mesh_lock_router
|
|
2874
|
+
application.include_router(mesh_lock_router)
|
|
2875
|
+
except ImportError:
|
|
2876
|
+
pass
|
|
2877
|
+
|
|
2462
2878
|
# -- Entity routes (Phase D) --
|
|
2463
2879
|
try:
|
|
2464
2880
|
from superlocalmemory.server.routes.entity import router as entity_router
|
|
@@ -2475,12 +2891,12 @@ def create_app() -> FastAPI:
|
|
|
2475
2891
|
|
|
2476
2892
|
# -- Brain route (LLD-04 v2: /api/v3/brain + deprecated shims) --
|
|
2477
2893
|
try:
|
|
2478
|
-
from superlocalmemory.server.routes.brain import (
|
|
2479
|
-
router as brain_router,
|
|
2480
|
-
)
|
|
2481
2894
|
from superlocalmemory.server.middleware.security_headers import (
|
|
2482
2895
|
SecurityHeadersMiddleware as StrictSecurityHeadersMiddleware,
|
|
2483
2896
|
)
|
|
2897
|
+
from superlocalmemory.server.routes.brain import (
|
|
2898
|
+
router as brain_router,
|
|
2899
|
+
)
|
|
2484
2900
|
application.include_router(brain_router)
|
|
2485
2901
|
# Strict CSP / XFO / XCTO / Referrer-Policy — applies to every
|
|
2486
2902
|
# response including the Brain route. Added as the outermost
|
|
@@ -2567,76 +2983,43 @@ def create_app() -> FastAPI:
|
|
|
2567
2983
|
# -- Daemon-specific routes --
|
|
2568
2984
|
_register_daemon_routes(application)
|
|
2569
2985
|
|
|
2570
|
-
# --
|
|
2571
|
-
# Mount the
|
|
2986
|
+
# -- MCP Streamable-HTTP transport at /mcp (mcp 2.0.0 fully-stateless) --
|
|
2987
|
+
# Mount the MCPServer as a Starlette ASGI sub-app so ALL clients
|
|
2572
2988
|
# (Claude Code sessions, subagents, desktop, hermes) share ONE daemon
|
|
2573
2989
|
# process instead of spawning an `slm mcp` subprocess per connection.
|
|
2574
2990
|
# The session manager lifespan is started in lifespan() via AsyncExitStack.
|
|
2575
2991
|
# Fail-open: if import or mount fails, stdio transport keeps working.
|
|
2576
2992
|
#
|
|
2577
|
-
# streamable_http_path
|
|
2578
|
-
#
|
|
2579
|
-
# (FastAPI strips the mount prefix before
|
|
2580
|
-
# sub-app, so the sub-app's internal route must be "/".)
|
|
2993
|
+
# streamable_http_path="/" so that when mounted at "/mcp" the effective
|
|
2994
|
+
# user-facing endpoint is exactly http://127.0.0.1:8765/mcp.
|
|
2995
|
+
# (FastAPI strips the mount prefix before the sub-app sees the request.)
|
|
2581
2996
|
try:
|
|
2582
|
-
from superlocalmemory.mcp.server import server as
|
|
2583
|
-
_mcp_fastmcp.settings.streamable_http_path = "/"
|
|
2584
|
-
_mcp_fastmcp._session_manager = None # Defensive reset for idempotency
|
|
2585
|
-
# v3.6.9 (#36): configure DNS-rebinding protection from env.
|
|
2586
|
-
# Default: localhost-only (safe). Set SLM_MCP_ALLOWED_HOSTS=192.168.x.y:*
|
|
2587
|
-
# (comma-separated, e.g. "192.168.50.144:*,slm.lan:*") to open to a LAN.
|
|
2588
|
-
# Use "*" to disable protection entirely (trusted private network only).
|
|
2589
|
-
# TransportSecuritySettings imported lazily here so that MCP mount
|
|
2590
|
-
# works on older SDK versions when SLM_MCP_ALLOWED_HOSTS is not set.
|
|
2591
|
-
_mcp_allowed = os.environ.get("SLM_MCP_ALLOWED_HOSTS", "").strip()
|
|
2592
|
-
if _mcp_allowed:
|
|
2593
|
-
from mcp.server.transport_security import TransportSecuritySettings
|
|
2594
|
-
if _mcp_allowed == "*":
|
|
2595
|
-
# M-05 (3.7.9): "*" fully disables DNS-rebinding protection.
|
|
2596
|
-
# Never silent — a convenience setting in a CI/Docker env must
|
|
2597
|
-
# not quietly expose the instance.
|
|
2598
|
-
logger.warning(
|
|
2599
|
-
"SLM_MCP_ALLOWED_HOSTS=* disables MCP DNS-rebinding "
|
|
2600
|
-
"protection entirely. Prefer an explicit host list; only "
|
|
2601
|
-
"use '*' on a trusted private network."
|
|
2602
|
-
)
|
|
2603
|
-
_mcp_fastmcp.settings.transport_security = TransportSecuritySettings(
|
|
2604
|
-
enable_dns_rebinding_protection=False,
|
|
2605
|
-
)
|
|
2606
|
-
else:
|
|
2607
|
-
_hosts = [h.strip() for h in _mcp_allowed.split(",") if h.strip()]
|
|
2608
|
-
_mcp_fastmcp.settings.transport_security = TransportSecuritySettings(
|
|
2609
|
-
enable_dns_rebinding_protection=True,
|
|
2610
|
-
allowed_hosts=_hosts,
|
|
2611
|
-
allowed_origins=[f"http://{h}" for h in _hosts],
|
|
2612
|
-
)
|
|
2613
|
-
logger.info("MCP transport security: allowed_hosts=%r", _mcp_allowed)
|
|
2614
|
-
# v3.6.12 (issue #39): stateless MCP transport for distributed/gateway
|
|
2615
|
-
# deployments. SLM's Streamable-HTTP is stateful by default — every call
|
|
2616
|
-
# must replay the Mcp-Session-Id from the initialize handshake. A gateway
|
|
2617
|
-
# (MCP Hub, LAN forwarder) that doesn't replay it gets "-32600 Session
|
|
2618
|
-
# not found" (the mesh-tools symptom in #39). Stateless mode treats each
|
|
2619
|
-
# request independently so any forwarder works. Default OFF (loopback
|
|
2620
|
-
# clients keep full stateful sessions); enabled by SLM_REMOTE=1 or
|
|
2621
|
-
# SLM_MCP_STATELESS=1. Per-agent /mcp/{agent_id} routing is unaffected
|
|
2622
|
-
# (path-based, not session-based).
|
|
2997
|
+
from superlocalmemory.mcp.server import server as _mcp_server
|
|
2623
2998
|
from superlocalmemory.core.remote_mode import is_remote_mode
|
|
2624
|
-
|
|
2999
|
+
|
|
3000
|
+
# mcp 2.0.0: transport knobs are kwargs to streamable_http_app(), not
|
|
3001
|
+
# mutable settings.stateless_http / .json_response / .transport_security.
|
|
3002
|
+
_mcp_kwargs = _configure_mcp_transport_settings()
|
|
3003
|
+
if _mcp_kwargs.get("stateless_http"):
|
|
2625
3004
|
if is_remote_mode():
|
|
2626
|
-
logger.
|
|
2627
|
-
"MCP transport: STATELESS
|
|
2628
|
-
"
|
|
2629
|
-
"
|
|
3005
|
+
logger.info(
|
|
3006
|
+
"MCP transport: STATELESS (default; SLM_REMOTE) — gateways/hubs "
|
|
3007
|
+
"may forward tool calls without a transport session id. "
|
|
3008
|
+
"App-level session_init remains available."
|
|
2630
3009
|
)
|
|
2631
3010
|
else:
|
|
2632
|
-
logger.
|
|
2633
|
-
"MCP transport: STATELESS
|
|
2634
|
-
"
|
|
2635
|
-
"for a local gateway/hub (e.g. MCP Hub) on 127.0.0.1 only; "
|
|
2636
|
-
"the token endpoint stays loopback-only without SLM_REMOTE."
|
|
3011
|
+
logger.info(
|
|
3012
|
+
"MCP transport: STATELESS (default) — no Mcp-Session-Id "
|
|
3013
|
+
"required; json_response=True. Opt out with SLM_MCP_STATEFUL=1."
|
|
2637
3014
|
)
|
|
3015
|
+
else:
|
|
3016
|
+
logger.warning(
|
|
3017
|
+
"MCP transport: STATEFUL (SLM_MCP_STATEFUL or SLM_MCP_STATELESS=0) "
|
|
3018
|
+
"— clients must replay Mcp-Session-Id after initialize."
|
|
3019
|
+
)
|
|
3020
|
+
|
|
2638
3021
|
global _mcp_app
|
|
2639
|
-
_mcp_app =
|
|
3022
|
+
_mcp_app = _mcp_server.streamable_http_app(**_mcp_kwargs)
|
|
2640
3023
|
|
|
2641
3024
|
# v3.6.10: per-agent-ID routing — /mcp/{agent_id} extracts the agent
|
|
2642
3025
|
# identity from the URL path and places it in a ContextVar so all MCP
|
|
@@ -2647,8 +3030,9 @@ def create_app() -> FastAPI:
|
|
|
2647
3030
|
|
|
2648
3031
|
application.mount("/mcp", AgentIDExtractorASGI(_mcp_app))
|
|
2649
3032
|
logger.info(
|
|
2650
|
-
"MCP HTTP transport mounted at /mcp (Streamable HTTP,
|
|
2651
|
-
"per-agent routing enabled)",
|
|
3033
|
+
"MCP HTTP transport mounted at /mcp (Streamable HTTP, mcp 2.0.0, "
|
|
3034
|
+
"stateless=%s, port %d; per-agent routing enabled)",
|
|
3035
|
+
bool(_mcp_kwargs.get("stateless_http")),
|
|
2652
3036
|
_configured_daemon_port(),
|
|
2653
3037
|
)
|
|
2654
3038
|
except Exception as _mcp_exc: # pragma: no cover — defensive
|
|
@@ -2662,14 +3046,14 @@ def _register_dashboard_routes(application: FastAPI) -> None:
|
|
|
2662
3046
|
|
|
2663
3047
|
Extracted from api.py's create_app() to avoid duplicate MemoryEngine.
|
|
2664
3048
|
"""
|
|
2665
|
-
from superlocalmemory.server.api import UI_DIR as _source_ui_dir
|
|
2666
|
-
|
|
2667
3049
|
# D-04: Copy UI assets to the data dir at daemon startup to avoid macOS
|
|
2668
3050
|
# xattr/TCC PermissionError on source-tree files in editable installs.
|
|
2669
3051
|
# The data dir has no quarantine attributes; the copy is idempotent (same
|
|
2670
3052
|
# content from the same package) so concurrent daemon starts are safe.
|
|
2671
3053
|
# Falls back to the source path with a WARNING — never crashes the daemon.
|
|
2672
3054
|
import shutil as _shutil
|
|
3055
|
+
|
|
3056
|
+
from superlocalmemory.server.api import UI_DIR as _source_ui_dir
|
|
2673
3057
|
_data_ui_dir = state_path("ui")
|
|
2674
3058
|
try:
|
|
2675
3059
|
_data_ui_dir.mkdir(parents=True, exist_ok=True)
|
|
@@ -2692,11 +3076,11 @@ def _register_dashboard_routes(application: FastAPI) -> None:
|
|
|
2692
3076
|
|
|
2693
3077
|
# Rate limiting (graceful)
|
|
2694
3078
|
try:
|
|
2695
|
-
from superlocalmemory.infra.rate_limiter import RateLimiter
|
|
2696
3079
|
from superlocalmemory.core.remote_mode import (
|
|
2697
|
-
rate_limit_config,
|
|
2698
3080
|
is_rate_limit_exempt,
|
|
3081
|
+
rate_limit_config,
|
|
2699
3082
|
)
|
|
3083
|
+
from superlocalmemory.infra.rate_limiter import RateLimiter
|
|
2700
3084
|
# v3.6.12 (issue #40): thresholds are env-tunable (SLM_RATE_LIMIT_WRITE/
|
|
2701
3085
|
# READ/WINDOW) so distributed/LAN operators can raise them. Defaults
|
|
2702
3086
|
# unchanged (30 writes / 120 reads per 60s) for the local case.
|
|
@@ -2720,7 +3104,10 @@ def _register_dashboard_routes(application: FastAPI) -> None:
|
|
|
2720
3104
|
# apply any persisted override from config.json.
|
|
2721
3105
|
try:
|
|
2722
3106
|
from superlocalmemory.infra.rate_limiter import (
|
|
2723
|
-
register_managed as _reg_rl,
|
|
3107
|
+
register_managed as _reg_rl,
|
|
3108
|
+
)
|
|
3109
|
+
from superlocalmemory.infra.rate_limiter import (
|
|
3110
|
+
reset_managed as _reset_rl,
|
|
2724
3111
|
)
|
|
2725
3112
|
_reset_rl()
|
|
2726
3113
|
_reg_rl("write", _write_limiter)
|
|
@@ -2919,6 +3306,17 @@ def _register_dashboard_routes(application: FastAPI) -> None:
|
|
|
2919
3306
|
_resp = _rbac_read_gate(request, application.state)
|
|
2920
3307
|
if _resp is not None:
|
|
2921
3308
|
return _resp
|
|
3309
|
+
# RBAC write gate: adapter process-control and similar
|
|
3310
|
+
# state-changing dashboard routes require WRITE permission.
|
|
3311
|
+
# Checked after the mutation-actor boundary above, which already
|
|
3312
|
+
# validated the machine-level credential; this layer checks the
|
|
3313
|
+
# user-level role within the workspace.
|
|
3314
|
+
if _is_adapter_control_mutation(
|
|
3315
|
+
request.method, request.url.path,
|
|
3316
|
+
):
|
|
3317
|
+
_resp = _rbac_write_gate(request, application.state)
|
|
3318
|
+
if _resp is not None:
|
|
3319
|
+
return _resp
|
|
2922
3320
|
return await call_next(request)
|
|
2923
3321
|
except Exception as _auth_exc:
|
|
2924
3322
|
# v3.6.12 (failopen-1): security middleware must NEVER fail open silently.
|
|
@@ -2957,6 +3355,29 @@ def _register_dashboard_routes(application: FastAPI) -> None:
|
|
|
2957
3355
|
)
|
|
2958
3356
|
return await call_next(request)
|
|
2959
3357
|
|
|
3358
|
+
# Migration-readiness gate — outermost middleware; runs before auth and
|
|
3359
|
+
# rate-limiting. When required schema migrations are in a failed state every
|
|
3360
|
+
# feature route receives a 503 so callers see one consistent "not ready"
|
|
3361
|
+
# signal instead of route-specific errors that vary by which schema element
|
|
3362
|
+
# is missing. Health, status, and repair paths are always reachable so that
|
|
3363
|
+
# operators can monitor and recover the daemon without waiting for migrations.
|
|
3364
|
+
@application.middleware("http")
|
|
3365
|
+
async def _migration_readiness_gate(request, call_next):
|
|
3366
|
+
migration_result = getattr(application.state, "migration_result", None)
|
|
3367
|
+
if migration_result and migration_result.get("failed"):
|
|
3368
|
+
if not _is_migration_exempt_path(request.url.path):
|
|
3369
|
+
from fastapi.responses import JSONResponse
|
|
3370
|
+
return JSONResponse(
|
|
3371
|
+
status_code=503,
|
|
3372
|
+
content={
|
|
3373
|
+
"error": (
|
|
3374
|
+
"Service unavailable: required schema migrations have "
|
|
3375
|
+
"not completed. Check /health for details."
|
|
3376
|
+
)
|
|
3377
|
+
},
|
|
3378
|
+
)
|
|
3379
|
+
return await call_next(request)
|
|
3380
|
+
|
|
2960
3381
|
# Static files — UI_DIR is already the effective path (data-dir or source
|
|
2961
3382
|
# fallback) set by the D-04 copy block above; mkdir is a no-op for
|
|
2962
3383
|
# the data-dir path (already created) and guarded in the source-fallback case.
|
|
@@ -2968,16 +3389,17 @@ def _register_dashboard_routes(application: FastAPI) -> None:
|
|
|
2968
3389
|
application.mount("/static", StaticFiles(directory=str(UI_DIR)), name="static")
|
|
2969
3390
|
|
|
2970
3391
|
# Route modules
|
|
2971
|
-
from superlocalmemory.server.routes.
|
|
2972
|
-
from superlocalmemory.server.routes.
|
|
2973
|
-
from superlocalmemory.server.routes.profiles import router as profiles_router
|
|
3392
|
+
from superlocalmemory.server.routes.adapters import router as adapters_router
|
|
3393
|
+
from superlocalmemory.server.routes.agents import router as agents_router
|
|
2974
3394
|
from superlocalmemory.server.routes.backup import router as backup_router
|
|
2975
3395
|
from superlocalmemory.server.routes.data_io import router as data_io_router
|
|
2976
3396
|
from superlocalmemory.server.routes.events import router as events_router
|
|
2977
|
-
from superlocalmemory.server.routes.
|
|
2978
|
-
from superlocalmemory.server.routes.
|
|
3397
|
+
from superlocalmemory.server.routes.memories import router as memories_router
|
|
3398
|
+
from superlocalmemory.server.routes.profiles import router as profiles_router
|
|
3399
|
+
from superlocalmemory.server.routes.stats import router as stats_router
|
|
2979
3400
|
from superlocalmemory.server.routes.v3_api import router as v3_router
|
|
2980
|
-
from superlocalmemory.server.routes.
|
|
3401
|
+
from superlocalmemory.server.routes.ws import manager as ws_manager
|
|
3402
|
+
from superlocalmemory.server.routes.ws import router as ws_router
|
|
2981
3403
|
|
|
2982
3404
|
application.include_router(memories_router)
|
|
2983
3405
|
application.include_router(stats_router)
|
|
@@ -3039,8 +3461,8 @@ def _register_dashboard_routes(application: FastAPI) -> None:
|
|
|
3039
3461
|
pass
|
|
3040
3462
|
|
|
3041
3463
|
# Wire WebSocket manager
|
|
3042
|
-
import superlocalmemory.server.routes.profiles as _profiles_mod
|
|
3043
3464
|
import superlocalmemory.server.routes.data_io as _data_io_mod
|
|
3465
|
+
import superlocalmemory.server.routes.profiles as _profiles_mod
|
|
3044
3466
|
_profiles_mod.ws_manager = ws_manager
|
|
3045
3467
|
_data_io_mod.ws_manager = ws_manager
|
|
3046
3468
|
|
|
@@ -3097,9 +3519,9 @@ def _register_dashboard_routes(application: FastAPI) -> None:
|
|
|
3097
3519
|
index_path = UI_DIR / "index.html"
|
|
3098
3520
|
if not index_path.exists():
|
|
3099
3521
|
return (
|
|
3100
|
-
"<html><head><title>SuperLocalMemory
|
|
3522
|
+
"<html><head><title>SuperLocalMemory V4</title></head>"
|
|
3101
3523
|
"<body style='font-family:Arial;padding:40px'>"
|
|
3102
|
-
"<h1>SuperLocalMemory
|
|
3524
|
+
"<h1>SuperLocalMemory V4 — Unified Daemon</h1>"
|
|
3103
3525
|
"<p><a href='/docs'>API Documentation</a></p>"
|
|
3104
3526
|
"</body></html>"
|
|
3105
3527
|
)
|
|
@@ -3272,6 +3694,8 @@ def _register_daemon_routes(application: FastAPI) -> None:
|
|
|
3272
3694
|
"runtime_state": runtime_state,
|
|
3273
3695
|
"active_profile": profile_snapshot.profile_id,
|
|
3274
3696
|
"profile_generation": profile_snapshot.generation,
|
|
3697
|
+
# Wave-3: operational failure counts (visible to all team members)
|
|
3698
|
+
**_ops_failure_counts(engine),
|
|
3275
3699
|
# issue #107: does this daemon's *imported* code still match the
|
|
3276
3700
|
# installed distribution? ``version`` above reports what this
|
|
3277
3701
|
# process loaded, which is self-consistent and therefore cannot
|
|
@@ -3299,12 +3723,32 @@ def _register_daemon_routes(application: FastAPI) -> None:
|
|
|
3299
3723
|
include_global: bool | None = None,
|
|
3300
3724
|
include_shared: bool | None = None,
|
|
3301
3725
|
window: str = "",
|
|
3726
|
+
as_of: str = "",
|
|
3302
3727
|
):
|
|
3303
3728
|
_update_activity()
|
|
3304
3729
|
search_query = q or query # Accept both ?q= and ?query= for compatibility
|
|
3305
3730
|
engine = _get_engine_or_503()
|
|
3306
3731
|
if not search_query:
|
|
3307
3732
|
return {"results": [], "count": 0, "query_type": "none", "retrieval_time_ms": 0}
|
|
3733
|
+
# Phase 4b: normalize as_of at HTTP boundary. Invalid → return error.
|
|
3734
|
+
_as_of_raw = as_of.strip() if as_of else ""
|
|
3735
|
+
if _as_of_raw:
|
|
3736
|
+
from superlocalmemory.retrieval.temporal_utils import normalize_as_of
|
|
3737
|
+
_as_of_norm = normalize_as_of(_as_of_raw)
|
|
3738
|
+
if _as_of_norm is None:
|
|
3739
|
+
# Audit P1a: hard-reject with HTTP 400 (a bare dict serializes as
|
|
3740
|
+
# 200, so status-code-only clients would miss the rejection).
|
|
3741
|
+
from starlette.responses import JSONResponse
|
|
3742
|
+
return JSONResponse(
|
|
3743
|
+
{
|
|
3744
|
+
"error": "invalid_as_of",
|
|
3745
|
+
"message": f"Cannot parse as_of: {_as_of_raw!r}",
|
|
3746
|
+
},
|
|
3747
|
+
status_code=400,
|
|
3748
|
+
)
|
|
3749
|
+
as_of = _as_of_norm
|
|
3750
|
+
else:
|
|
3751
|
+
as_of = ""
|
|
3308
3752
|
# v3.8.2: resolve the client-driven-agentic default now so the concrete
|
|
3309
3753
|
# bool drives BOTH the full-recall semaphore below and engine.recall().
|
|
3310
3754
|
from superlocalmemory.core.recall_pipeline import resolve_hot_path_fast
|
|
@@ -3329,6 +3773,9 @@ def _register_daemon_routes(application: FastAPI) -> None:
|
|
|
3329
3773
|
getattr(application.state, "daemon_descriptor", None),
|
|
3330
3774
|
actor_kind="http-recall",
|
|
3331
3775
|
)
|
|
3776
|
+
# Phase-1/D2: clamp cross-profile scope flags per enterprise recall policy.
|
|
3777
|
+
from superlocalmemory.core.admission import enforce_read_scope
|
|
3778
|
+
include_global, include_shared = enforce_read_scope(include_global, include_shared)
|
|
3332
3779
|
# v3.4.32: mark recall in-flight so the pending materializer pauses
|
|
3333
3780
|
# v3.4.52: run engine.recall() in a thread-pool executor so the
|
|
3334
3781
|
# FastAPI event loop stays responsive for /health, /remember, and
|
|
@@ -3360,6 +3807,7 @@ def _register_daemon_routes(application: FastAPI) -> None:
|
|
|
3360
3807
|
include_global=include_global,
|
|
3361
3808
|
include_shared=include_shared,
|
|
3362
3809
|
window=window or None,
|
|
3810
|
+
as_of=as_of or None,
|
|
3363
3811
|
),
|
|
3364
3812
|
)
|
|
3365
3813
|
_budget = _recall_budget_s()
|
|
@@ -3463,7 +3911,10 @@ def _register_daemon_routes(application: FastAPI) -> None:
|
|
|
3463
3911
|
# authenticated dashboard user may mutate this profile. This must run
|
|
3464
3912
|
# before either the trust pre-hook or the durable admission journal.
|
|
3465
3913
|
from superlocalmemory.access.rbac import Permission
|
|
3466
|
-
from superlocalmemory.server.rbac_enforce import
|
|
3914
|
+
from superlocalmemory.server.rbac_enforce import (
|
|
3915
|
+
require_permission,
|
|
3916
|
+
resolve_actor_roles,
|
|
3917
|
+
)
|
|
3467
3918
|
|
|
3468
3919
|
require_permission(request, Permission.WRITE, profile=engine._profile_id)
|
|
3469
3920
|
if scope in {"shared", "global"}:
|
|
@@ -3512,6 +3963,65 @@ def _register_daemon_routes(application: FastAPI) -> None:
|
|
|
3512
3963
|
"profile_id": engine._profile_id,
|
|
3513
3964
|
"content_preview": req.content[:100],
|
|
3514
3965
|
})
|
|
3966
|
+
|
|
3967
|
+
# V4 Phase 4: OperationPolicyRegistry evaluation.
|
|
3968
|
+
# ActorContext is server-derived — principal_id from _require_write_actor,
|
|
3969
|
+
# client_host from ASGI request, session_token_hash from server headers.
|
|
3970
|
+
# NONE of these values come from the request body (RememberRequest).
|
|
3971
|
+
# This block runs BEFORE the durable admission journal is written.
|
|
3972
|
+
from superlocalmemory.core.actor_context import (
|
|
3973
|
+
ActorContext as _ActorContext,
|
|
3974
|
+
)
|
|
3975
|
+
from superlocalmemory.core.actor_context import (
|
|
3976
|
+
Transport as _Transport,
|
|
3977
|
+
)
|
|
3978
|
+
from superlocalmemory.core.operation_policy_registry import (
|
|
3979
|
+
_DEFAULT_REGISTRY as _policy_registry,
|
|
3980
|
+
)
|
|
3981
|
+
from superlocalmemory.core.operation_request import OperationKind as _OpKind
|
|
3982
|
+
|
|
3983
|
+
_client_host = (
|
|
3984
|
+
request.client.host if request.client is not None else ""
|
|
3985
|
+
) or ""
|
|
3986
|
+
_token_raw = (
|
|
3987
|
+
request.headers.get("x-slm-user-session", "")
|
|
3988
|
+
or (request.cookies.get("slm_session", "") if request.cookies else "")
|
|
3989
|
+
) or ""
|
|
3990
|
+
_token_hash = (
|
|
3991
|
+
hashlib.sha256(_token_raw.encode()).hexdigest()[:16]
|
|
3992
|
+
if _token_raw else ""
|
|
3993
|
+
)
|
|
3994
|
+
# Detect single-user vs. company mode from the RBAC engine (server-side
|
|
3995
|
+
# state). Fail-open if the RBAC state cannot be read — a 503 from RBAC
|
|
3996
|
+
# already blocks the request via _rbac_write_gate. Here we only need the
|
|
3997
|
+
# mode string for unknown-kind fallback; REMEMBER is always a known kind.
|
|
3998
|
+
_rbac_state = getattr(application.state, "rbac", None)
|
|
3999
|
+
try:
|
|
4000
|
+
_is_company = bool(
|
|
4001
|
+
_rbac_state is not None and _rbac_state.user_count() > 0
|
|
4002
|
+
)
|
|
4003
|
+
except Exception:
|
|
4004
|
+
_is_company = False
|
|
4005
|
+
_policy_mode = "company" if _is_company else "local"
|
|
4006
|
+
|
|
4007
|
+
_http_actor = _ActorContext(
|
|
4008
|
+
principal_id=trusted_actor_id,
|
|
4009
|
+
roles=resolve_actor_roles(request, profile=engine._profile_id),
|
|
4010
|
+
active_profile_id=engine._profile_id,
|
|
4011
|
+
transport=_Transport.HTTP,
|
|
4012
|
+
client_host=_client_host,
|
|
4013
|
+
session_token_hash=_token_hash,
|
|
4014
|
+
)
|
|
4015
|
+
_policy_decision = _policy_registry.evaluate(
|
|
4016
|
+
_OpKind.REMEMBER, _http_actor, _policy_mode,
|
|
4017
|
+
)
|
|
4018
|
+
if not _policy_decision.allowed:
|
|
4019
|
+
# Map to PermissionError: the except block below converts this
|
|
4020
|
+
# to HTTP 403 — consistent with AdmissionAuthorizationError.
|
|
4021
|
+
raise PermissionError(
|
|
4022
|
+
f"operation policy denied REMEMBER: {_policy_decision.reason}"
|
|
4023
|
+
)
|
|
4024
|
+
|
|
3515
4025
|
admission = RememberRequest(
|
|
3516
4026
|
content=req.content,
|
|
3517
4027
|
profile_id=engine._profile_id,
|
|
@@ -3604,6 +4114,9 @@ def _register_daemon_routes(application: FastAPI) -> None:
|
|
|
3604
4114
|
async def consolidate_cognitive_endpoint(body: dict, request: Request):
|
|
3605
4115
|
_update_activity()
|
|
3606
4116
|
engine = _get_engine_or_503()
|
|
4117
|
+
from superlocalmemory.access.rbac import Permission
|
|
4118
|
+
from superlocalmemory.server.rbac_enforce import require_permission
|
|
4119
|
+
require_permission(request, Permission.WRITE, profile=engine._profile_id)
|
|
3607
4120
|
from superlocalmemory.server.route_mutations import (
|
|
3608
4121
|
authorize_route_mutation,
|
|
3609
4122
|
)
|
|
@@ -3611,10 +4124,10 @@ def _register_daemon_routes(application: FastAPI) -> None:
|
|
|
3611
4124
|
request,
|
|
3612
4125
|
operation="update",
|
|
3613
4126
|
source_agent_id="http-cognitive-consolidation",
|
|
3614
|
-
profile_id=
|
|
4127
|
+
profile_id=engine.profile_id,
|
|
3615
4128
|
)
|
|
3616
4129
|
try:
|
|
3617
|
-
pid =
|
|
4130
|
+
pid = engine.profile_id
|
|
3618
4131
|
from superlocalmemory.encoding.cognitive_consolidator import (
|
|
3619
4132
|
CognitiveConsolidator,
|
|
3620
4133
|
)
|
|
@@ -3638,6 +4151,9 @@ def _register_daemon_routes(application: FastAPI) -> None:
|
|
|
3638
4151
|
async def run_maintenance_endpoint(body: dict, request: Request):
|
|
3639
4152
|
_update_activity()
|
|
3640
4153
|
engine = _get_engine_or_503()
|
|
4154
|
+
from superlocalmemory.access.rbac import Permission
|
|
4155
|
+
from superlocalmemory.server.rbac_enforce import require_permission
|
|
4156
|
+
require_permission(request, Permission.WRITE, profile=engine._profile_id)
|
|
3641
4157
|
from superlocalmemory.server.route_mutations import (
|
|
3642
4158
|
authorize_route_mutation,
|
|
3643
4159
|
)
|
|
@@ -3645,29 +4161,29 @@ def _register_daemon_routes(application: FastAPI) -> None:
|
|
|
3645
4161
|
request,
|
|
3646
4162
|
operation="update",
|
|
3647
4163
|
source_agent_id="http-maintenance",
|
|
3648
|
-
profile_id=
|
|
4164
|
+
profile_id=engine.profile_id,
|
|
3649
4165
|
)
|
|
3650
4166
|
try:
|
|
3651
|
-
pid =
|
|
4167
|
+
pid = engine.profile_id
|
|
3652
4168
|
results: dict = {}
|
|
3653
4169
|
try:
|
|
3654
4170
|
from superlocalmemory.core.maintenance import run_maintenance as _run_maint
|
|
3655
4171
|
maint_result = _run_maint(engine._db, engine._config, pid)
|
|
3656
4172
|
results["langevin"] = {"updated": maint_result.get("updated", 0)}
|
|
3657
|
-
except Exception
|
|
4173
|
+
except Exception:
|
|
3658
4174
|
logger.exception("maintenance langevin step failed")
|
|
3659
4175
|
results["langevin"] = {"error": "internal error"}
|
|
3660
4176
|
try:
|
|
3661
|
-
from superlocalmemory.math.ebbinghaus import EbbinghausCurve
|
|
3662
4177
|
from superlocalmemory.learning.forgetting_scheduler import (
|
|
3663
4178
|
ForgettingScheduler,
|
|
3664
4179
|
)
|
|
4180
|
+
from superlocalmemory.math.ebbinghaus import EbbinghausCurve
|
|
3665
4181
|
ebb = EbbinghausCurve(engine._config.forgetting)
|
|
3666
4182
|
sched = ForgettingScheduler(
|
|
3667
4183
|
engine._db, ebb, engine._config.forgetting,
|
|
3668
4184
|
)
|
|
3669
4185
|
results["forgetting"] = sched.run_decay_cycle(pid, force=False)
|
|
3670
|
-
except Exception
|
|
4186
|
+
except Exception:
|
|
3671
4187
|
logger.exception("maintenance forgetting step failed")
|
|
3672
4188
|
results["forgetting"] = {"error": "internal error"}
|
|
3673
4189
|
try:
|
|
@@ -3680,7 +4196,7 @@ def _register_daemon_routes(application: FastAPI) -> None:
|
|
|
3680
4196
|
)
|
|
3681
4197
|
count = cw._generate_patterns(pid, False)
|
|
3682
4198
|
results["behavioral"] = {"patterns_mined": count}
|
|
3683
|
-
except Exception
|
|
4199
|
+
except Exception:
|
|
3684
4200
|
logger.exception("maintenance behavioral step failed")
|
|
3685
4201
|
results["behavioral"] = {"error": "internal error"}
|
|
3686
4202
|
authorization.complete()
|
|
@@ -3754,6 +4270,8 @@ def _register_daemon_routes(application: FastAPI) -> None:
|
|
|
3754
4270
|
# index backfill after an upgrade). Dashboard renders a plain
|
|
3755
4271
|
# "Optimizing memory…" line from this. Defaults to idle before start.
|
|
3756
4272
|
"self_heal": globals().get("_SELF_HEAL_STATUS", {"state": "idle"}),
|
|
4273
|
+
# Wave-3: operational failure counts (dead-letter, degraded, stalled)
|
|
4274
|
+
**_ops_failure_counts(engine),
|
|
3757
4275
|
}
|
|
3758
4276
|
|
|
3759
4277
|
@application.get("/api/v3/components")
|
|
@@ -3805,6 +4323,105 @@ def _register_daemon_routes(application: FastAPI) -> None:
|
|
|
3805
4323
|
threading.Thread(target=_run, daemon=True, name="manual-heal").start()
|
|
3806
4324
|
return {"status": "started"}
|
|
3807
4325
|
|
|
4326
|
+
# ------------------------------------------------------------------
|
|
4327
|
+
# Wave-3: Operational Recovery & Admin Remediation (V4 resilience slice)
|
|
4328
|
+
# ------------------------------------------------------------------
|
|
4329
|
+
|
|
4330
|
+
@application.get("/operations/failed")
|
|
4331
|
+
async def list_failed_operations_endpoint(
|
|
4332
|
+
request: Request,
|
|
4333
|
+
profile: str = None,
|
|
4334
|
+
):
|
|
4335
|
+
"""Return all failed/stuck/degraded operations — admin surface.
|
|
4336
|
+
|
|
4337
|
+
Requires MANAGE permission (OWNER or ADMIN role). Returns three
|
|
4338
|
+
categories: dead-letter DLQ entries, DEGRADED completion manifests,
|
|
4339
|
+
and exhausted projection obligations. Safe read-only; never mutates.
|
|
4340
|
+
"""
|
|
4341
|
+
from superlocalmemory.server.rbac_enforce import require_permission
|
|
4342
|
+
from superlocalmemory.access.rbac import Permission
|
|
4343
|
+
|
|
4344
|
+
require_permission(request, Permission.MANAGE)
|
|
4345
|
+
_update_activity()
|
|
4346
|
+
|
|
4347
|
+
config = getattr(application.state, "config", None)
|
|
4348
|
+
db_path = getattr(config, "db_path", None)
|
|
4349
|
+
if db_path is None or not db_path.exists():
|
|
4350
|
+
return {"dead_letter": [], "degraded_manifests": [],
|
|
4351
|
+
"exhausted_obligations": [], "total": 0}
|
|
4352
|
+
|
|
4353
|
+
try:
|
|
4354
|
+
from superlocalmemory.core.ops_remediation import list_failed_operations
|
|
4355
|
+
return list_failed_operations(db_path, profile_id=profile)
|
|
4356
|
+
except Exception as exc:
|
|
4357
|
+
logger.warning("list_failed_operations endpoint error: %s", exc, exc_info=True)
|
|
4358
|
+
raise HTTPException(status_code=500, detail="Failed to query operations")
|
|
4359
|
+
|
|
4360
|
+
@application.post("/operations/{operation_id}/resolve")
|
|
4361
|
+
async def resolve_operation_endpoint(request: Request, operation_id: str):
|
|
4362
|
+
"""Admin remediation: retry / force-reconcile / cancel a stuck operation.
|
|
4363
|
+
|
|
4364
|
+
Body JSON: ``{"action": "retry"|"force_reconcile"|"cancel"}``
|
|
4365
|
+
|
|
4366
|
+
Requires MANAGE permission. Writes an audit event for every mutation.
|
|
4367
|
+
The action itself is delegated to ``core.ops_remediation.resolve_operation``.
|
|
4368
|
+
Returns ``{"success": bool, "action": str, "operation_id": str, ...}``.
|
|
4369
|
+
"""
|
|
4370
|
+
from superlocalmemory.server.rbac_enforce import require_permission
|
|
4371
|
+
from superlocalmemory.access.rbac import Permission
|
|
4372
|
+
|
|
4373
|
+
require_permission(request, Permission.MANAGE)
|
|
4374
|
+
_update_activity()
|
|
4375
|
+
|
|
4376
|
+
body: dict = {}
|
|
4377
|
+
try:
|
|
4378
|
+
body = await request.json()
|
|
4379
|
+
except Exception:
|
|
4380
|
+
raise HTTPException(status_code=400, detail="JSON body required")
|
|
4381
|
+
|
|
4382
|
+
action = body.get("action", "")
|
|
4383
|
+
if action not in ("retry", "force_reconcile", "cancel"):
|
|
4384
|
+
raise HTTPException(
|
|
4385
|
+
status_code=400,
|
|
4386
|
+
detail="action must be one of: retry, force_reconcile, cancel",
|
|
4387
|
+
)
|
|
4388
|
+
|
|
4389
|
+
config = getattr(application.state, "config", None)
|
|
4390
|
+
db_path = getattr(config, "db_path", None)
|
|
4391
|
+
if db_path is None or not db_path.exists():
|
|
4392
|
+
raise HTTPException(status_code=503, detail="Database not available")
|
|
4393
|
+
|
|
4394
|
+
engine = getattr(application.state, "engine", None)
|
|
4395
|
+
|
|
4396
|
+
try:
|
|
4397
|
+
from superlocalmemory.core.ops_remediation import resolve_operation
|
|
4398
|
+
result = resolve_operation(db_path, engine, operation_id, action)
|
|
4399
|
+
except ValueError as exc:
|
|
4400
|
+
raise HTTPException(status_code=400, detail=str(exc))
|
|
4401
|
+
except Exception as exc:
|
|
4402
|
+
logger.warning(
|
|
4403
|
+
"resolve_operation endpoint error op=%s action=%s: %s",
|
|
4404
|
+
operation_id, action, exc, exc_info=True,
|
|
4405
|
+
)
|
|
4406
|
+
raise HTTPException(status_code=500, detail="Remediation failed")
|
|
4407
|
+
|
|
4408
|
+
# Audit trail — best effort, non-blocking
|
|
4409
|
+
try:
|
|
4410
|
+
from superlocalmemory.compliance.audit import AuditChain
|
|
4411
|
+
audit_path = state_path("audit_chain.db")
|
|
4412
|
+
actor_header = request.headers.get("X-Actor-Id", "admin")
|
|
4413
|
+
AuditChain(str(audit_path)).log(
|
|
4414
|
+
operation=f"ops_resolve:{action}",
|
|
4415
|
+
agent_id=actor_header,
|
|
4416
|
+
profile_id="",
|
|
4417
|
+
content_hash="",
|
|
4418
|
+
metadata={"operation_id": operation_id, "result": result},
|
|
4419
|
+
)
|
|
4420
|
+
except Exception:
|
|
4421
|
+
logger.debug("audit log for ops_resolve failed (non-fatal)", exc_info=True)
|
|
4422
|
+
|
|
4423
|
+
return result
|
|
4424
|
+
|
|
3808
4425
|
@application.get("/list")
|
|
3809
4426
|
async def list_facts(limit: int = 50):
|
|
3810
4427
|
_update_activity()
|
|
@@ -3849,6 +4466,7 @@ def _register_daemon_routes(application: FastAPI) -> None:
|
|
|
3849
4466
|
_require_write_actor(request)
|
|
3850
4467
|
import subprocess
|
|
3851
4468
|
import sys as _sys
|
|
4469
|
+
|
|
3852
4470
|
from superlocalmemory.core.platform_utils import popen_platform_kwargs
|
|
3853
4471
|
|
|
3854
4472
|
logger.info("Daemon restart requested via API")
|
|
@@ -3922,6 +4540,9 @@ def _register_daemon_routes(application: FastAPI) -> None:
|
|
|
3922
4540
|
"""
|
|
3923
4541
|
_update_activity()
|
|
3924
4542
|
engine = _get_engine_or_503()
|
|
4543
|
+
from superlocalmemory.access.rbac import Permission
|
|
4544
|
+
from superlocalmemory.server.rbac_enforce import require_permission
|
|
4545
|
+
require_permission(request, Permission.WRITE, profile=engine._profile_id)
|
|
3925
4546
|
from superlocalmemory.server.route_mutations import (
|
|
3926
4547
|
authorize_route_mutation,
|
|
3927
4548
|
)
|
|
@@ -4104,6 +4725,195 @@ def _run_materializer_operation(
|
|
|
4104
4725
|
return operation(engine)
|
|
4105
4726
|
|
|
4106
4727
|
|
|
4728
|
+
def _reconcile_projection_manifest(
|
|
4729
|
+
engine, operation_id: str, profile_id: str, fact_ids,
|
|
4730
|
+
) -> None:
|
|
4731
|
+
try:
|
|
4732
|
+
from superlocalmemory.core.transactions.concrete_owners import (
|
|
4733
|
+
REQUIRED_ADMISSION_OWNERS,
|
|
4734
|
+
build_transaction_service,
|
|
4735
|
+
)
|
|
4736
|
+
from superlocalmemory.core.transactions.owners import ObligationKind
|
|
4737
|
+
|
|
4738
|
+
if not profile_id:
|
|
4739
|
+
return
|
|
4740
|
+
context = _context_for_operation(engine, operation_id)
|
|
4741
|
+
if context is None:
|
|
4742
|
+
_terminalize_orphan_operation(engine, operation_id)
|
|
4743
|
+
return
|
|
4744
|
+
service = build_transaction_service(engine)
|
|
4745
|
+
with engine._db.raw_connection() as conn:
|
|
4746
|
+
service.record(
|
|
4747
|
+
conn, context,
|
|
4748
|
+
owners=REQUIRED_ADMISSION_OWNERS,
|
|
4749
|
+
kind=ObligationKind.APPLY,
|
|
4750
|
+
)
|
|
4751
|
+
service.reconcile_operation(engine._db, context)
|
|
4752
|
+
except Exception as exc:
|
|
4753
|
+
logger.warning(
|
|
4754
|
+
"projection reconciliation skipped for %s: %s", operation_id, exc,
|
|
4755
|
+
)
|
|
4756
|
+
|
|
4757
|
+
|
|
4758
|
+
def _context_for_operation(engine, operation_id: str):
|
|
4759
|
+
import json as _json
|
|
4760
|
+
|
|
4761
|
+
from superlocalmemory.core.transactions.owners import OperationContext
|
|
4762
|
+
|
|
4763
|
+
db = getattr(engine, "_db", None)
|
|
4764
|
+
if db is None:
|
|
4765
|
+
return None
|
|
4766
|
+
rows = db.execute(
|
|
4767
|
+
"SELECT profile_id, final_fact_ids_json, queryable_fact_ids_json "
|
|
4768
|
+
"FROM ingestion_operations WHERE operation_id = ?",
|
|
4769
|
+
(operation_id,),
|
|
4770
|
+
)
|
|
4771
|
+
if not rows:
|
|
4772
|
+
return None
|
|
4773
|
+
row = dict(rows[0])
|
|
4774
|
+
if row.get("profile_id") != getattr(engine, "_profile_id", None):
|
|
4775
|
+
return None
|
|
4776
|
+
try:
|
|
4777
|
+
fact_ids = _json.loads(row.get("final_fact_ids_json") or "[]") or _json.loads(
|
|
4778
|
+
row.get("queryable_fact_ids_json") or "[]"
|
|
4779
|
+
)
|
|
4780
|
+
except (TypeError, ValueError):
|
|
4781
|
+
return None
|
|
4782
|
+
if not isinstance(fact_ids, list) or not fact_ids:
|
|
4783
|
+
return None
|
|
4784
|
+
fact_ids = [str(fid) for fid in fact_ids]
|
|
4785
|
+
return OperationContext(
|
|
4786
|
+
operation_id=operation_id,
|
|
4787
|
+
profile_id=row["profile_id"],
|
|
4788
|
+
subject_id=operation_id,
|
|
4789
|
+
fact_ids=tuple(fact_ids),
|
|
4790
|
+
)
|
|
4791
|
+
|
|
4792
|
+
|
|
4793
|
+
_REDRIVE_INTERVAL_S = 30.0
|
|
4794
|
+
_last_redrive_ts = 0.0
|
|
4795
|
+
|
|
4796
|
+
|
|
4797
|
+
def _reconcile_pending_projections(
|
|
4798
|
+
engine, *, limit: int = 20, force: bool = False,
|
|
4799
|
+
) -> int:
|
|
4800
|
+
global _last_redrive_ts
|
|
4801
|
+
now = time.monotonic()
|
|
4802
|
+
if not force and now - _last_redrive_ts < _REDRIVE_INTERVAL_S:
|
|
4803
|
+
return 0
|
|
4804
|
+
_last_redrive_ts = now
|
|
4805
|
+
try:
|
|
4806
|
+
from superlocalmemory.core.transactions.concrete_owners import (
|
|
4807
|
+
build_transaction_service,
|
|
4808
|
+
)
|
|
4809
|
+
from superlocalmemory.core.transactions.obligations import ObligationLedger
|
|
4810
|
+
|
|
4811
|
+
db = getattr(engine, "_db", None)
|
|
4812
|
+
profile_id = getattr(engine, "_profile_id", None)
|
|
4813
|
+
if db is None or not profile_id:
|
|
4814
|
+
return 0
|
|
4815
|
+
ledger = ObligationLedger()
|
|
4816
|
+
with db.raw_connection() as conn:
|
|
4817
|
+
op_ids = set(
|
|
4818
|
+
ledger.pending_operation_ids(conn, profile_id=profile_id, limit=limit)
|
|
4819
|
+
)
|
|
4820
|
+
op_ids.update(
|
|
4821
|
+
ledger.operations_missing_manifest(
|
|
4822
|
+
conn, profile_id=profile_id, limit=limit,
|
|
4823
|
+
)
|
|
4824
|
+
)
|
|
4825
|
+
if not op_ids:
|
|
4826
|
+
return 0
|
|
4827
|
+
service = build_transaction_service(engine)
|
|
4828
|
+
done = 0
|
|
4829
|
+
for operation_id in sorted(op_ids):
|
|
4830
|
+
try:
|
|
4831
|
+
context = _context_for_operation(engine, operation_id)
|
|
4832
|
+
if context is None:
|
|
4833
|
+
_terminalize_orphan_operation(engine, operation_id)
|
|
4834
|
+
continue
|
|
4835
|
+
service.reconcile_operation(db, context)
|
|
4836
|
+
done += 1
|
|
4837
|
+
except Exception as exc: # noqa: BLE001
|
|
4838
|
+
logger.warning(
|
|
4839
|
+
"projection redrive failed for %s: %s", operation_id, exc,
|
|
4840
|
+
)
|
|
4841
|
+
return done
|
|
4842
|
+
except Exception as exc: # noqa: BLE001
|
|
4843
|
+
logger.warning("projection redrive pass skipped: %s", exc)
|
|
4844
|
+
return 0
|
|
4845
|
+
|
|
4846
|
+
|
|
4847
|
+
def _terminalize_orphan_operation(engine, operation_id: str) -> None:
|
|
4848
|
+
from superlocalmemory.core.transactions.obligations import ObligationLedger
|
|
4849
|
+
from superlocalmemory.core.transactions.owners import ObligationState
|
|
4850
|
+
from superlocalmemory.core.transactions.reconciler import Reconciler
|
|
4851
|
+
from superlocalmemory.core.transactions.service import MAX_APPLY_ATTEMPTS
|
|
4852
|
+
|
|
4853
|
+
db = getattr(engine, "_db", None)
|
|
4854
|
+
profile_id = getattr(engine, "_profile_id", None)
|
|
4855
|
+
if db is None or not profile_id:
|
|
4856
|
+
return
|
|
4857
|
+
ledger = ObligationLedger()
|
|
4858
|
+
with db.raw_connection() as conn:
|
|
4859
|
+
for obligation in ledger.fetch(conn, operation_id):
|
|
4860
|
+
if obligation.attempts >= MAX_APPLY_ATTEMPTS:
|
|
4861
|
+
continue
|
|
4862
|
+
ledger.mark(
|
|
4863
|
+
conn, operation_id, obligation.owner, obligation.kind,
|
|
4864
|
+
ObligationState.FAILED,
|
|
4865
|
+
detail={"phase": "orphan", "error": "canonical record missing"},
|
|
4866
|
+
bump_attempts=True,
|
|
4867
|
+
)
|
|
4868
|
+
Reconciler(ledger).reconcile(
|
|
4869
|
+
conn, operation_id, profile_id, canonical_committed=False,
|
|
4870
|
+
)
|
|
4871
|
+
|
|
4872
|
+
|
|
4873
|
+
def _ops_failure_counts(engine) -> dict:
|
|
4874
|
+
"""Return Wave-3 operational failure counts for /status and /health.
|
|
4875
|
+
|
|
4876
|
+
Always returns a dict (never raises). Counts default to 0 on any error.
|
|
4877
|
+
Includes: dead_letter_count, degraded_operations, exhausted_obligations,
|
|
4878
|
+
writer_stalled (bool), writer_stalled_op_id, writer_stalled_age_s.
|
|
4879
|
+
"""
|
|
4880
|
+
result: dict = {
|
|
4881
|
+
"dead_letter_count": 0,
|
|
4882
|
+
"degraded_operations": 0,
|
|
4883
|
+
"exhausted_obligations": 0,
|
|
4884
|
+
"writer_stalled": False,
|
|
4885
|
+
"writer_stalled_op_id": None,
|
|
4886
|
+
"writer_stalled_age_s": None,
|
|
4887
|
+
}
|
|
4888
|
+
# Writer stall info from canonical coordinator
|
|
4889
|
+
try:
|
|
4890
|
+
writer_runtime = getattr(application.state, "canonical_remember_runtime", None)
|
|
4891
|
+
coordinator = getattr(writer_runtime, "_coordinator", None) if writer_runtime else None
|
|
4892
|
+
if coordinator is None:
|
|
4893
|
+
coordinator = getattr(application.state, "write_coordinator", None)
|
|
4894
|
+
if coordinator is not None and hasattr(coordinator, "inflight_info"):
|
|
4895
|
+
info = coordinator.inflight_info()
|
|
4896
|
+
result["writer_stalled"] = bool(info.get("stalled"))
|
|
4897
|
+
result["writer_stalled_op_id"] = info.get("op_id")
|
|
4898
|
+
result["writer_stalled_age_s"] = info.get("age_s")
|
|
4899
|
+
except Exception:
|
|
4900
|
+
pass
|
|
4901
|
+
|
|
4902
|
+
# DB failure counts
|
|
4903
|
+
try:
|
|
4904
|
+
from superlocalmemory.core.ops_remediation import get_failure_counts
|
|
4905
|
+
db_path = getattr(getattr(application.state, "config", None), "db_path", None)
|
|
4906
|
+
if db_path is not None and db_path.exists():
|
|
4907
|
+
counts = get_failure_counts(db_path)
|
|
4908
|
+
result["dead_letter_count"] = counts.get("dead_letter_count", 0)
|
|
4909
|
+
result["degraded_operations"] = counts.get("degraded_operations", 0)
|
|
4910
|
+
result["exhausted_obligations"] = counts.get("exhausted_obligations", 0)
|
|
4911
|
+
except Exception:
|
|
4912
|
+
pass
|
|
4913
|
+
|
|
4914
|
+
return result
|
|
4915
|
+
|
|
4916
|
+
|
|
4107
4917
|
def _materialize_ingestion_one_pass(
|
|
4108
4918
|
engine,
|
|
4109
4919
|
*,
|
|
@@ -4162,6 +4972,12 @@ def _materialize_ingestion_one_pass(
|
|
|
4162
4972
|
continue
|
|
4163
4973
|
if result.state is IngestionState.COMPLETE:
|
|
4164
4974
|
completed += 1
|
|
4975
|
+
_reconcile_projection_manifest(
|
|
4976
|
+
engine,
|
|
4977
|
+
result.operation_id,
|
|
4978
|
+
getattr(operation, "profile_id", ""),
|
|
4979
|
+
result.fact_ids,
|
|
4980
|
+
)
|
|
4165
4981
|
_emit_event(
|
|
4166
4982
|
"memory.stored",
|
|
4167
4983
|
payload={
|
|
@@ -4179,6 +4995,7 @@ def _materialize_ingestion_one_pass(
|
|
|
4179
4995
|
result.operation_id,
|
|
4180
4996
|
result.last_error,
|
|
4181
4997
|
)
|
|
4998
|
+
_reconcile_pending_projections(engine)
|
|
4182
4999
|
return completed, failed
|
|
4183
5000
|
|
|
4184
5001
|
|
|
@@ -4241,7 +5058,9 @@ def _start_pending_materializer() -> None:
|
|
|
4241
5058
|
|
|
4242
5059
|
def _loop():
|
|
4243
5060
|
from superlocalmemory.cli.pending_store import (
|
|
4244
|
-
get_pending,
|
|
5061
|
+
get_pending,
|
|
5062
|
+
mark_done,
|
|
5063
|
+
mark_failed,
|
|
4245
5064
|
)
|
|
4246
5065
|
# v3.4.38: log first engine acquisition so we know materializer is alive
|
|
4247
5066
|
_engine_logged = False
|
|
@@ -4369,6 +5188,7 @@ def start_server(port: int = _DEFAULT_PORT) -> None:
|
|
|
4369
5188
|
global _start_time
|
|
4370
5189
|
assert_no_durable_root_conflict()
|
|
4371
5190
|
import socket
|
|
5191
|
+
|
|
4372
5192
|
import uvicorn
|
|
4373
5193
|
|
|
4374
5194
|
# Bind before any migration or engine work. A process which cannot own
|
|
@@ -4434,7 +5254,10 @@ def start_server(port: int = _DEFAULT_PORT) -> None:
|
|
|
4434
5254
|
|
|
4435
5255
|
try:
|
|
4436
5256
|
from superlocalmemory.migrations.v3_4_25_to_v3_4_26 import (
|
|
4437
|
-
is_ready as _is_ready,
|
|
5257
|
+
is_ready as _is_ready,
|
|
5258
|
+
)
|
|
5259
|
+
from superlocalmemory.migrations.v3_4_25_to_v3_4_26 import (
|
|
5260
|
+
migrate as _migrate,
|
|
4438
5261
|
)
|
|
4439
5262
|
_data = canonical_data_root()
|
|
4440
5263
|
if not _is_ready(_data):
|