superlocalmemory 4.0.10 → 4.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +12 -2
- package/CHANGELOG.md +244 -0
- package/README.md +40 -75
- package/package.json +6 -3
- package/plugin/.claude-plugin/plugin.json +2 -2
- package/plugin/CLAUDE.md +3 -3
- package/plugin/agents/slm-governance-advisor.md +1 -1
- package/plugin/agents/slm-loop-runner.md +4 -4
- package/plugin/agents/slm-memory-advisor.md +1 -1
- package/plugin/agents/slm-optimize-advisor.md +1 -1
- package/plugin/requirements.txt +1 -1
- package/plugin/skills/slm-cache/SKILL.md +1 -1
- package/plugin/skills/slm-compress/SKILL.md +1 -1
- package/plugin/skills/slm-governance/SKILL.md +1 -1
- package/plugin/skills/slm-graph/SKILL.md +1 -1
- package/plugin/skills/slm-loop/SKILL.md +2 -2
- package/plugin/skills/slm-mesh/SKILL.md +1 -1
- package/plugin/skills/slm-profile/SKILL.md +5 -5
- package/plugin/skills/slm-recall/SKILL.md +102 -15
- package/plugin/skills/slm-remember/SKILL.md +35 -3
- package/plugin/skills/slm-scope/SKILL.md +1 -1
- package/plugin/skills/slm-session/SKILL.md +29 -3
- package/plugin/skills/slm-status/SKILL.md +1 -1
- package/plugin-src/rules/AGENTS.md +16 -8
- package/plugin-src/skills/slm-cache/SKILL.md +1 -1
- package/plugin-src/skills/slm-compress/SKILL.md +1 -1
- package/plugin-src/skills/slm-governance/SKILL.md +1 -1
- package/plugin-src/skills/slm-graph/SKILL.md +1 -1
- package/plugin-src/skills/slm-loop/SKILL.md +2 -2
- package/plugin-src/skills/slm-mesh/SKILL.md +1 -1
- package/plugin-src/skills/slm-profile/SKILL.md +5 -5
- package/plugin-src/skills/slm-recall/SKILL.md +102 -15
- package/plugin-src/skills/slm-remember/SKILL.md +35 -3
- package/plugin-src/skills/slm-scope/SKILL.md +1 -1
- package/plugin-src/skills/slm-session/SKILL.md +29 -3
- package/plugin-src/skills/slm-status/SKILL.md +1 -1
- package/pyproject.toml +1 -1
- package/src/superlocalmemory/__init__.py +1 -1
- package/src/superlocalmemory/cli/commands.py +357 -18
- package/src/superlocalmemory/cli/daemon.py +30 -0
- package/src/superlocalmemory/cli/db_migrate.py +71 -1
- package/src/superlocalmemory/cli/gdpr_cmd.py +15 -2
- package/src/superlocalmemory/cli/main.py +24 -2
- package/src/superlocalmemory/code_graph/database.py +44 -0
- package/src/superlocalmemory/compliance/gdpr.py +449 -39
- package/src/superlocalmemory/core/admission.py +231 -11
- package/src/superlocalmemory/core/backend_orchestrator.py +190 -84
- package/src/superlocalmemory/core/config.py +90 -11
- package/src/superlocalmemory/core/consolidation_engine.py +34 -0
- package/src/superlocalmemory/core/engine.py +140 -11
- package/src/superlocalmemory/core/graph_analyzer.py +76 -112
- package/src/superlocalmemory/core/graph_metrics.py +597 -0
- package/src/superlocalmemory/core/graph_pruner.py +121 -0
- package/src/superlocalmemory/core/maintenance_scheduler.py +205 -0
- package/src/superlocalmemory/core/mode_capability.py +111 -0
- package/src/superlocalmemory/core/ollama_validator.py +315 -0
- package/src/superlocalmemory/core/projection_drain.py +380 -0
- package/src/superlocalmemory/core/recall_pipeline.py +390 -3
- package/src/superlocalmemory/core/recall_worker.py +6 -3
- package/src/superlocalmemory/core/scale_autopromote.py +196 -0
- package/src/superlocalmemory/core/scale_engine.py +16 -2
- package/src/superlocalmemory/core/score_contract.py +21 -1
- package/src/superlocalmemory/core/session_identity.py +85 -0
- package/src/superlocalmemory/core/status_contract.py +108 -0
- package/src/superlocalmemory/core/worker_pool.py +4 -4
- package/src/superlocalmemory/core/working_memory.py +288 -0
- package/src/superlocalmemory/encoding/cognitive_consolidator.py +36 -6
- package/src/superlocalmemory/encoding/context_generator.py +1 -1
- package/src/superlocalmemory/encoding/entity_resolver.py +38 -0
- package/src/superlocalmemory/encoding/fact_extractor.py +18 -14
- package/src/superlocalmemory/encoding/prospective_markers.py +262 -0
- package/src/superlocalmemory/encoding/type_router.py +12 -12
- package/src/superlocalmemory/evolution/mutation_generator.py +30 -4
- package/src/superlocalmemory/graph/cozo_adjacency.py +122 -0
- package/src/superlocalmemory/graph/cozo_backend.py +103 -138
- package/src/superlocalmemory/hooks/portable_kit.py +10 -2
- package/src/superlocalmemory/learning/bandit.py +43 -0
- package/src/superlocalmemory/learning/consolidation_worker.py +54 -0
- package/src/superlocalmemory/learning/database.py +60 -3
- package/src/superlocalmemory/learning/entity_compiler.py +21 -58
- package/src/superlocalmemory/learning/feedback.py +3 -1
- package/src/superlocalmemory/learning/outcomes.py +47 -16
- package/src/superlocalmemory/learning/pattern_miner.py +28 -3
- package/src/superlocalmemory/learning/pattern_miner_constants.py +43 -0
- package/src/superlocalmemory/learning/pcos.py +291 -0
- package/src/superlocalmemory/learning/reward_from_outcomes.py +365 -0
- package/src/superlocalmemory/learning/reward_proxy.py +100 -10
- package/src/superlocalmemory/learning/signal_kinds.py +79 -0
- package/src/superlocalmemory/mcp/profiles.py +14 -2
- package/src/superlocalmemory/mcp/tools_active.py +2 -1
- package/src/superlocalmemory/mcp/tools_core.py +31 -3
- package/src/superlocalmemory/mcp/tools_v28.py +20 -1
- package/src/superlocalmemory/parameterization/pattern_extractor.py +14 -1
- package/src/superlocalmemory/parameterization/soft_prompt_generator.py +98 -0
- package/src/superlocalmemory/retrieval/bm25_channel.py +64 -3
- package/src/superlocalmemory/retrieval/channel_status.py +117 -0
- package/src/superlocalmemory/retrieval/engine.py +106 -11
- package/src/superlocalmemory/retrieval/entity_channel.py +210 -256
- package/src/superlocalmemory/retrieval/graph_adjacency.py +219 -0
- package/src/superlocalmemory/retrieval/scope_policy.py +20 -0
- package/src/superlocalmemory/retrieval/semantic_channel.py +47 -5
- package/src/superlocalmemory/retrieval/spreading.py +288 -0
- package/src/superlocalmemory/server/api.py +24 -5
- package/src/superlocalmemory/server/bandit_loops.py +17 -1
- package/src/superlocalmemory/server/rbac_enforce.py +26 -6
- package/src/superlocalmemory/server/recall_health.py +87 -10
- package/src/superlocalmemory/server/recall_serializer.py +9 -0
- package/src/superlocalmemory/server/routes/behavioral.py +75 -10
- package/src/superlocalmemory/server/routes/compliance.py +98 -18
- package/src/superlocalmemory/server/routes/config_api.py +186 -4
- package/src/superlocalmemory/server/routes/evolution.py +178 -0
- package/src/superlocalmemory/server/routes/ingest.py +8 -0
- package/src/superlocalmemory/server/routes/learning_telemetry.py +2 -1
- package/src/superlocalmemory/server/routes/memories.py +49 -7
- package/src/superlocalmemory/server/routes/timeline.py +4 -0
- package/src/superlocalmemory/server/routes/v3_api.py +191 -15
- package/src/superlocalmemory/server/ui.py +20 -4
- package/src/superlocalmemory/server/unified_daemon.py +241 -7
- package/src/superlocalmemory/storage/_migration_internals.py +54 -2
- package/src/superlocalmemory/storage/_schema_version.py +24 -3
- package/src/superlocalmemory/storage/database.py +477 -59
- package/src/superlocalmemory/storage/embedding_codec.py +71 -0
- package/src/superlocalmemory/storage/lineage_retention.py +236 -0
- package/src/superlocalmemory/storage/logical_edges.py +43 -2
- package/src/superlocalmemory/storage/migration_runner.py +119 -0
- package/src/superlocalmemory/storage/migrations/M043_quarantine_display_summaries.py +60 -36
- package/src/superlocalmemory/storage/migrations/M044_play_carries_its_own_evidence.py +127 -0
- package/src/superlocalmemory/storage/migrations/M045_fact_outcome_score.py +158 -0
- package/src/superlocalmemory/storage/migrations/M046_prospective_memory_has_its_own_name.py +620 -0
- package/src/superlocalmemory/storage/migrations/M047_fisher_vectors_are_stored_like_every_other_vector.py +306 -0
- package/src/superlocalmemory/storage/migrations/M048_upcoming_holds_only_what_is_upcoming.py +207 -0
- package/src/superlocalmemory/storage/migrations/M049_a_schema_version_marker_is_one_row.py +201 -0
- package/src/superlocalmemory/storage/migrations.py +18 -2
- package/src/superlocalmemory/storage/models.py +40 -1
- package/src/superlocalmemory/storage/projection_outbox.py +346 -0
- package/src/superlocalmemory/storage/retention_policy.py +860 -0
- package/src/superlocalmemory/storage/schema.py +35 -1
- package/src/superlocalmemory/storage/write_coordinator.py +19 -2
- package/src/superlocalmemory/trust/scorer.py +43 -1
- package/src/superlocalmemory/ui/index.html +9 -18
- package/src/superlocalmemory/ui/js/event-delegation.js +12 -1
- package/src/superlocalmemory/ui/js/od-health.js +28 -6
- package/src/superlocalmemory/ui/js/od-memories.js +19 -0
- package/src/superlocalmemory/ui/js/od-settings.js +87 -1
- package/src/superlocalmemory/ui/js/recall-lab.js +78 -3
|
@@ -8,14 +8,45 @@ from __future__ import annotations
|
|
|
8
8
|
|
|
9
9
|
import json
|
|
10
10
|
import logging
|
|
11
|
+
from pathlib import Path
|
|
11
12
|
import os
|
|
12
13
|
from fastapi import APIRouter, HTTPException, Request
|
|
13
14
|
from fastapi.responses import JSONResponse
|
|
15
|
+
from superlocalmemory.core.config import CANONICAL_RECALL_LIMIT
|
|
16
|
+
from superlocalmemory.core.status_contract import (
|
|
17
|
+
COUNT_QUERIES,
|
|
18
|
+
counts_from_sqlite,
|
|
19
|
+
projection_queue_depth,
|
|
20
|
+
store_size_mb,
|
|
21
|
+
)
|
|
14
22
|
from superlocalmemory.server.routes.helpers import SLM_VERSION, get_read_connection
|
|
15
23
|
from superlocalmemory.server.route_mutations import authorize_route_mutation
|
|
16
24
|
|
|
17
25
|
logger = logging.getLogger(__name__)
|
|
18
26
|
|
|
27
|
+
def _signal_session_id() -> str:
|
|
28
|
+
"""A name for the caller, for matching an outcome back to this recall.
|
|
29
|
+
|
|
30
|
+
The agent id the request arrived under, when it arrived under one. Falls
|
|
31
|
+
back to the workspace, which keeps dashboard and scripted traffic separable
|
|
32
|
+
from an agent's. Never empty: an unnamed recall leaves no record.
|
|
33
|
+
"""
|
|
34
|
+
try:
|
|
35
|
+
from superlocalmemory.mcp.agent_context import get_current_agent_id
|
|
36
|
+
|
|
37
|
+
agent = str(get_current_agent_id() or "").strip()
|
|
38
|
+
if agent:
|
|
39
|
+
return f"agent:{agent}"
|
|
40
|
+
except Exception: # noqa: BLE001 -- naming the caller must never fail a read
|
|
41
|
+
pass
|
|
42
|
+
try:
|
|
43
|
+
from superlocalmemory.server.routes.helpers import get_active_profile
|
|
44
|
+
|
|
45
|
+
return f"api:{get_active_profile()}"
|
|
46
|
+
except Exception: # noqa: BLE001
|
|
47
|
+
return "api:default"
|
|
48
|
+
|
|
49
|
+
|
|
19
50
|
router = APIRouter(prefix="/api/v3", tags=["v3"])
|
|
20
51
|
|
|
21
52
|
|
|
@@ -102,20 +133,15 @@ async def dashboard(request: Request):
|
|
|
102
133
|
|
|
103
134
|
# Read stats directly from SQLite (dashboard doesn't load engine)
|
|
104
135
|
memory_count = 0
|
|
105
|
-
|
|
136
|
+
counts = dict.fromkeys(COUNT_QUERIES, 0)
|
|
137
|
+
queue_depth = 0
|
|
106
138
|
db_path = config.base_dir / "memory.db"
|
|
107
139
|
if db_path.exists():
|
|
108
140
|
try:
|
|
109
141
|
conn = get_read_connection(db_path)
|
|
142
|
+
counts = counts_from_sqlite(conn, active_profile)
|
|
143
|
+
queue_depth = projection_queue_depth(conn)
|
|
110
144
|
cursor = conn.cursor()
|
|
111
|
-
try:
|
|
112
|
-
cursor.execute(
|
|
113
|
-
"SELECT COUNT(*) FROM atomic_facts WHERE profile_id = ?",
|
|
114
|
-
(active_profile,),
|
|
115
|
-
)
|
|
116
|
-
fact_count = cursor.fetchone()[0]
|
|
117
|
-
except Exception:
|
|
118
|
-
pass
|
|
119
145
|
try:
|
|
120
146
|
try:
|
|
121
147
|
cursor.execute(
|
|
@@ -143,10 +169,23 @@ async def dashboard(request: Request):
|
|
|
143
169
|
"provider": config.llm.provider or "none",
|
|
144
170
|
"model": config.llm.model or "",
|
|
145
171
|
"memory_count": memory_count,
|
|
146
|
-
"fact_count": fact_count,
|
|
147
172
|
"profile": active_profile,
|
|
148
173
|
"base_dir": str(config.base_dir),
|
|
149
174
|
"version": SLM_VERSION,
|
|
175
|
+
# The counts and the store's own address were missing here while
|
|
176
|
+
# every other status surface carried them, so the one surface a
|
|
177
|
+
# person actually looks at could not answer "is the graph healthy".
|
|
178
|
+
"db_path": str(db_path),
|
|
179
|
+
"db_size_mb": store_size_mb(db_path),
|
|
180
|
+
"profile_generation": get_profile_runtime(
|
|
181
|
+
request.app.state,
|
|
182
|
+
).snapshot.generation,
|
|
183
|
+
# Facts stored but not yet in the graph and vector projections. A
|
|
184
|
+
# number that does not fall is a projection that has stopped
|
|
185
|
+
# keeping up, which is otherwise invisible: nothing errors, the
|
|
186
|
+
# memory is safely in SQLite, and recall just quietly gets worse.
|
|
187
|
+
"projection_queue_depth": queue_depth,
|
|
188
|
+
**counts,
|
|
150
189
|
}
|
|
151
190
|
payload.update(dashboard_mode_fields(config.mode))
|
|
152
191
|
return payload
|
|
@@ -495,12 +534,45 @@ async def set_full_config(request: Request):
|
|
|
495
534
|
_emb_fields = ("embedding_provider", "embedding_endpoint", "embedding_key",
|
|
496
535
|
"embedding_model", "embedding_dimension")
|
|
497
536
|
if any(k in body for k in _emb_fields):
|
|
537
|
+
_old_emb = config.embedding
|
|
538
|
+
_new_provider = body.get("embedding_provider", "")
|
|
539
|
+
_new_model = body.get("embedding_model", "")
|
|
540
|
+
_new_dim = int(body.get("embedding_dimension", 0) or 0)
|
|
541
|
+
# The same range the other save route enforces. Without it a
|
|
542
|
+
# dashboard save with no dimension field stored a width of zero.
|
|
543
|
+
if _new_dim and not (64 <= _new_dim <= 8192):
|
|
544
|
+
return JSONResponse(
|
|
545
|
+
{"error": f"Dimension must be 64-8192, got {_new_dim}"},
|
|
546
|
+
status_code=400,
|
|
547
|
+
)
|
|
548
|
+
# The SECOND way to change the embedding model, and it was
|
|
549
|
+
# unguarded. Switching mode from the dashboard carries the embedding
|
|
550
|
+
# fields, so a width that the store cannot hold arrived here
|
|
551
|
+
# untouched while the other route refused it — one door bolted, the
|
|
552
|
+
# other open.
|
|
553
|
+
if not bool(body.get("force")):
|
|
554
|
+
_refusal = _refuse_incompatible_embedding(
|
|
555
|
+
config, config.embedding, _new_model, _new_dim,
|
|
556
|
+
new_provider=_new_provider,
|
|
557
|
+
)
|
|
558
|
+
if _refusal is not None:
|
|
559
|
+
return _refusal
|
|
498
560
|
config.embedding = EmbeddingConfig(
|
|
499
|
-
provider=
|
|
561
|
+
provider=_new_provider,
|
|
500
562
|
api_endpoint=body.get("embedding_endpoint", ""),
|
|
501
563
|
api_key=body.get("embedding_key", ""),
|
|
502
|
-
model_name=
|
|
503
|
-
dimension=
|
|
564
|
+
model_name=_new_model,
|
|
565
|
+
dimension=_new_dim or _old_emb.dimension,
|
|
566
|
+
# Not naming these reset them to defaults, so every embedding
|
|
567
|
+
# save from the dashboard silently put the local model back to
|
|
568
|
+
# whatever ships — the exact defect the other route had.
|
|
569
|
+
ollama_model=(
|
|
570
|
+
_new_model if _new_provider == "ollama" and _new_model
|
|
571
|
+
else _old_emb.ollama_model
|
|
572
|
+
),
|
|
573
|
+
ollama_base_url=_old_emb.ollama_base_url,
|
|
574
|
+
api_version=_old_emb.api_version,
|
|
575
|
+
deployment_name=_old_emb.deployment_name,
|
|
504
576
|
)
|
|
505
577
|
|
|
506
578
|
# When the mode actually changed, apply the new mode's structural presets
|
|
@@ -571,6 +643,85 @@ async def get_embedding_config(request: Request):
|
|
|
571
643
|
return _internal_error()
|
|
572
644
|
|
|
573
645
|
|
|
646
|
+
def _refuse_incompatible_embedding(
|
|
647
|
+
config, old_emb, new_model, new_dim, new_provider=None,
|
|
648
|
+
):
|
|
649
|
+
"""None when the change is safe, otherwise the 409 to return instead.
|
|
650
|
+
|
|
651
|
+
Fail-open on anything it cannot determine: a store with no vectors yet, a
|
|
652
|
+
model server that is not running, an unreadable database. Refusing on
|
|
653
|
+
"I could not tell" would block a legitimate first-time setup, and the
|
|
654
|
+
dimension a caller declares is still checked against the store either way.
|
|
655
|
+
"""
|
|
656
|
+
from fastapi.responses import JSONResponse as _JSON
|
|
657
|
+
|
|
658
|
+
try:
|
|
659
|
+
from superlocalmemory.core.ollama_validator import (
|
|
660
|
+
EMBEDDING,
|
|
661
|
+
stored_embedding_dimension,
|
|
662
|
+
validate_ollama_model,
|
|
663
|
+
)
|
|
664
|
+
|
|
665
|
+
db_path = Path(config.base_dir) / "memory.db"
|
|
666
|
+
stored = stored_embedding_dimension(db_path)
|
|
667
|
+
if stored is None:
|
|
668
|
+
return None
|
|
669
|
+
|
|
670
|
+
# The provider being SAVED, falling back to the current one when the
|
|
671
|
+
# caller is not changing it. Reading only the current provider meant
|
|
672
|
+
# that SWITCHING to a local model never probed at all — and switching
|
|
673
|
+
# is exactly when the width changes.
|
|
674
|
+
effective_provider = (
|
|
675
|
+
new_provider
|
|
676
|
+
if new_provider is not None
|
|
677
|
+
else getattr(old_emb, "provider", "")
|
|
678
|
+
)
|
|
679
|
+
measured = None
|
|
680
|
+
if effective_provider == "ollama" or getattr(old_emb, "provider", "") == "ollama":
|
|
681
|
+
# The model being SAVED, not the one already configured. Probing the
|
|
682
|
+
# old one always matched the stored width and therefore always
|
|
683
|
+
# allowed the change — the guard measured the thing it was not
|
|
684
|
+
# protecting against.
|
|
685
|
+
probe = validate_ollama_model(
|
|
686
|
+
new_model or getattr(old_emb, "ollama_model", ""),
|
|
687
|
+
EMBEDDING,
|
|
688
|
+
base_url=getattr(old_emb, "ollama_base_url", "")
|
|
689
|
+
or "http://localhost:11434",
|
|
690
|
+
)
|
|
691
|
+
measured = probe.dimension if probe.ok else None
|
|
692
|
+
|
|
693
|
+
declared = int(new_dim or 0)
|
|
694
|
+
if measured is None and declared <= 0:
|
|
695
|
+
# Nothing to compare: the server could not be asked and the caller
|
|
696
|
+
# named no width. Allowing is the fail-open the first-time setup
|
|
697
|
+
# needs; the other route's probe still guards the common path.
|
|
698
|
+
return None
|
|
699
|
+
effective = measured if measured is not None else declared
|
|
700
|
+
if effective == stored:
|
|
701
|
+
return None
|
|
702
|
+
|
|
703
|
+
return _JSON(
|
|
704
|
+
{
|
|
705
|
+
"error": "embedding_width_mismatch",
|
|
706
|
+
"stored_dimension": stored,
|
|
707
|
+
"requested_dimension": effective,
|
|
708
|
+
"model_name": new_model,
|
|
709
|
+
"detail": (
|
|
710
|
+
f"{new_model} produces {effective}-dimensional vectors and "
|
|
711
|
+
f"this store holds {stored}-dimensional ones. Vectors of "
|
|
712
|
+
f"different widths cannot be compared, so every memory "
|
|
713
|
+
f"already stored would become unfindable by meaning. "
|
|
714
|
+
f"Rebuild them first with: slm db migrate — or resend with "
|
|
715
|
+
f"force=true if they have already been rebuilt."
|
|
716
|
+
),
|
|
717
|
+
},
|
|
718
|
+
status_code=409,
|
|
719
|
+
)
|
|
720
|
+
except Exception: # noqa: BLE001 - never block a save on the check failing
|
|
721
|
+
logger.exception("embedding width pre-check failed; allowing the save")
|
|
722
|
+
return None
|
|
723
|
+
|
|
724
|
+
|
|
574
725
|
@router.put("/embedding/config")
|
|
575
726
|
async def set_embedding_config(request: Request):
|
|
576
727
|
"""Update embedding configuration independently of mode switch."""
|
|
@@ -591,13 +742,34 @@ async def set_embedding_config(request: Request):
|
|
|
591
742
|
new_key = body.get("api_key", config.embedding.api_key)
|
|
592
743
|
|
|
593
744
|
old_emb = config.embedding
|
|
745
|
+
|
|
746
|
+
# A width that disagrees with what the store already holds is refused
|
|
747
|
+
# here, at the moment of writing, not merely offered as a check the
|
|
748
|
+
# caller may or may not have run. Vectors of different widths cannot be
|
|
749
|
+
# compared, so the store would keep answering similarity questions and
|
|
750
|
+
# every answer would be noise. ``force=true`` is the escape hatch for
|
|
751
|
+
# somebody who has already re-embedded.
|
|
752
|
+
if not bool(body.get("force")):
|
|
753
|
+
refusal = _refuse_incompatible_embedding(
|
|
754
|
+
config, old_emb, new_model, new_dim, new_provider=new_provider,
|
|
755
|
+
)
|
|
756
|
+
if refusal is not None:
|
|
757
|
+
return refusal
|
|
758
|
+
|
|
594
759
|
config.embedding = EmbeddingConfig(
|
|
595
760
|
model_name=new_model,
|
|
596
761
|
dimension=new_dim,
|
|
597
762
|
provider=new_provider,
|
|
598
763
|
api_endpoint=new_endpoint,
|
|
599
764
|
api_key=new_key,
|
|
600
|
-
|
|
765
|
+
# In Ollama mode the embedder resolves its model from
|
|
766
|
+
# ``ollama_model``, so keeping the old value here made a rename a
|
|
767
|
+
# no-op that still answered "success". A caller naming a model gets
|
|
768
|
+
# that model.
|
|
769
|
+
ollama_model=(
|
|
770
|
+
new_model if new_provider == "ollama" and new_model
|
|
771
|
+
else old_emb.ollama_model
|
|
772
|
+
),
|
|
601
773
|
ollama_base_url=old_emb.ollama_base_url,
|
|
602
774
|
api_version=old_emb.api_version,
|
|
603
775
|
deployment_name=old_emb.deployment_name,
|
|
@@ -1018,7 +1190,7 @@ async def recall_trace(request: Request):
|
|
|
1018
1190
|
try:
|
|
1019
1191
|
body = await request.json()
|
|
1020
1192
|
query = body.get("query", "")
|
|
1021
|
-
limit = body.get("limit",
|
|
1193
|
+
limit = body.get("limit", CANONICAL_RECALL_LIMIT)
|
|
1022
1194
|
window = body.get("window", "") or ""
|
|
1023
1195
|
as_of_raw = (body.get("as_of", "") or "").strip()
|
|
1024
1196
|
raw_known_as_of = body.get("known_as_of", "")
|
|
@@ -1073,6 +1245,10 @@ async def recall_trace(request: Request):
|
|
|
1073
1245
|
window=window or None, as_of=_as_of,
|
|
1074
1246
|
known_as_of=_known_as_of, valid_at=_valid_at,
|
|
1075
1247
|
include_unknown=include_unknown,
|
|
1248
|
+
# Whoever asked, by the name they arrived under. Without a name
|
|
1249
|
+
# the record of this recall is discarded and no outcome
|
|
1250
|
+
# reported afterwards can be matched back to it.
|
|
1251
|
+
session_id=_signal_session_id(),
|
|
1076
1252
|
),
|
|
1077
1253
|
)
|
|
1078
1254
|
elapsed_ms = round((_time.monotonic() - t0) * 1000, 1)
|
|
@@ -231,14 +231,30 @@ def create_app() -> FastAPI:
|
|
|
231
231
|
"<p><a href='/api/docs'>API Documentation</a></p>"
|
|
232
232
|
"</body></html>"
|
|
233
233
|
)
|
|
234
|
-
from superlocalmemory.server.asset_versions import render_index
|
|
235
234
|
from superlocalmemory import __version__ as _v
|
|
236
235
|
|
|
237
236
|
# __SLM_VERSION__ was substituted only by the unified daemon, so the
|
|
238
237
|
# dashboard's upgrade detector did nothing when served from here.
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
)
|
|
238
|
+
# Asset versioning is cosmetic. It must never be why this page 500s.
|
|
239
|
+
#
|
|
240
|
+
# The import is deferred (house style, keeps startup lean), which means
|
|
241
|
+
# it resolves at REQUEST time — so when `pip install -e .` replaced the
|
|
242
|
+
# installed package underneath a running daemon, this route began
|
|
243
|
+
# answering "Internal Server Error" on the dashboard while every other
|
|
244
|
+
# endpoint was fine. A stale hand-written version string is a trifle; a
|
|
245
|
+
# blank page is not. Fall back to the file as written.
|
|
246
|
+
try:
|
|
247
|
+
from superlocalmemory.server.asset_versions import render_index
|
|
248
|
+
|
|
249
|
+
return render_index(
|
|
250
|
+
index_path, UI_DIR, substitutions={"__SLM_VERSION__": _v},
|
|
251
|
+
)
|
|
252
|
+
except Exception as exc: # noqa: BLE001 — serve the page regardless
|
|
253
|
+
logger.warning(
|
|
254
|
+
"asset version rewrite unavailable, serving index.html as "
|
|
255
|
+
"written: %s: %s", type(exc).__name__, exc,
|
|
256
|
+
)
|
|
257
|
+
return index_path.read_text().replace("__SLM_VERSION__", _v)
|
|
242
258
|
|
|
243
259
|
@application.get("/favicon.ico", include_in_schema=False)
|
|
244
260
|
async def favicon():
|
|
@@ -511,6 +511,49 @@ _MIGRATION_EXEMPT_PATH_PREFIXES: tuple[str, ...] = (
|
|
|
511
511
|
)
|
|
512
512
|
|
|
513
513
|
|
|
514
|
+
def _serving_blocked_by(migration_result: dict) -> list[str]:
|
|
515
|
+
"""Failed migrations that should stop this daemon serving. Fail-closed.
|
|
516
|
+
|
|
517
|
+
A failed migration used to 503 every route without asking what had failed.
|
|
518
|
+
For a missing table that is right. For a data invariant that ordinary use can
|
|
519
|
+
re-violate it is not: one drifted row made the whole store unreachable until
|
|
520
|
+
somebody restarted it by hand, and the restart fixed nothing that a
|
|
521
|
+
background repair would not have fixed on its own.
|
|
522
|
+
|
|
523
|
+
A migration may answer for itself by exposing ``blocks_serving(conn)``.
|
|
524
|
+
Anything that does not is treated as blocking, so this cannot quietly open a
|
|
525
|
+
door for a migration nobody has thought about.
|
|
526
|
+
"""
|
|
527
|
+
failed = list(migration_result.get("failed") or [])
|
|
528
|
+
if not failed:
|
|
529
|
+
return []
|
|
530
|
+
try:
|
|
531
|
+
import sqlite3
|
|
532
|
+
|
|
533
|
+
from superlocalmemory.infra.data_root import state_path
|
|
534
|
+
from superlocalmemory.storage._migration_internals import _MODULES
|
|
535
|
+
except Exception: # noqa: BLE001 — never let this decide by crashing
|
|
536
|
+
return failed
|
|
537
|
+
|
|
538
|
+
blocking: list[str] = []
|
|
539
|
+
for name in failed:
|
|
540
|
+
decide = getattr(_MODULES.get(name), "blocks_serving", None)
|
|
541
|
+
if not callable(decide):
|
|
542
|
+
blocking.append(name)
|
|
543
|
+
continue
|
|
544
|
+
try:
|
|
545
|
+
db = state_path("memory.db")
|
|
546
|
+
conn = sqlite3.connect(f"file:{db}?mode=ro", uri=True)
|
|
547
|
+
try:
|
|
548
|
+
if decide(conn):
|
|
549
|
+
blocking.append(name)
|
|
550
|
+
finally:
|
|
551
|
+
conn.close()
|
|
552
|
+
except Exception: # noqa: BLE001 — unknown means blocking
|
|
553
|
+
blocking.append(name)
|
|
554
|
+
return blocking
|
|
555
|
+
|
|
556
|
+
|
|
514
557
|
def _is_migration_exempt_path(path: str) -> bool:
|
|
515
558
|
"""Return True for health, status, and repair paths that must stay reachable
|
|
516
559
|
even when the daemon reports a schema migration failure.
|
|
@@ -1804,6 +1847,41 @@ def _stop_deployment_retention(application) -> bool:
|
|
|
1804
1847
|
return True
|
|
1805
1848
|
|
|
1806
1849
|
|
|
1850
|
+
def _start_embedder_warmup(engine: object) -> "threading.Thread | None":
|
|
1851
|
+
"""Load the embedding model in the background. Never blocks startup.
|
|
1852
|
+
|
|
1853
|
+
Returns the thread so a test can join it; ``None`` when there is nothing to
|
|
1854
|
+
warm. Every failure is "not warmed", never a failed startup: a daemon that
|
|
1855
|
+
cannot embed still serves keyword recall and still stores memories, and the
|
|
1856
|
+
materializer fills the vectors in afterwards either way.
|
|
1857
|
+
"""
|
|
1858
|
+
embedder = getattr(engine, "_embedder", None)
|
|
1859
|
+
if embedder is None or not hasattr(embedder, "embed"):
|
|
1860
|
+
return None
|
|
1861
|
+
|
|
1862
|
+
from superlocalmemory.core.engine import _is_remote_embedder
|
|
1863
|
+
|
|
1864
|
+
if _is_remote_embedder(embedder):
|
|
1865
|
+
# A hosted embedder has no model to load and warming it would spend a
|
|
1866
|
+
# request, and money, on a sentence nobody asked about.
|
|
1867
|
+
return None
|
|
1868
|
+
|
|
1869
|
+
def _warm() -> None:
|
|
1870
|
+
started = time.time()
|
|
1871
|
+
try:
|
|
1872
|
+
embedder.embed("slm embedder warm-up")
|
|
1873
|
+
except Exception as exc: # pragma: no cover — warming is best effort
|
|
1874
|
+
logger.debug("embedder warm-up failed (%s) — writes will defer", exc)
|
|
1875
|
+
return
|
|
1876
|
+
logger.info(
|
|
1877
|
+
"Embedding model warm and ready (%.1fs)", time.time() - started,
|
|
1878
|
+
)
|
|
1879
|
+
|
|
1880
|
+
thread = threading.Thread(target=_warm, daemon=True, name="slm-embed-warmup")
|
|
1881
|
+
thread.start()
|
|
1882
|
+
return thread
|
|
1883
|
+
|
|
1884
|
+
|
|
1807
1885
|
@asynccontextmanager
|
|
1808
1886
|
async def lifespan(application: FastAPI):
|
|
1809
1887
|
"""Initialize engine, workers, and optional services on startup."""
|
|
@@ -2088,6 +2166,40 @@ async def lifespan(application: FastAPI):
|
|
|
2088
2166
|
engine = MemoryEngine(config)
|
|
2089
2167
|
engine.initialize()
|
|
2090
2168
|
|
|
2169
|
+
# Load the embedding model now, off the request path, the way the
|
|
2170
|
+
# cross-encoder is already warmed at startup.
|
|
2171
|
+
#
|
|
2172
|
+
# A write embeds inline so the memory it stores can be found by asking a
|
|
2173
|
+
# question rather than only by quoting its own words, and it gives that
|
|
2174
|
+
# one second before deferring to the materializer. Loading the model
|
|
2175
|
+
# takes 9.9-11.0 s here; once loaded an embed is 42 ms. So on a daemon
|
|
2176
|
+
# that had not embedded yet, the first writes each waited the full second
|
|
2177
|
+
# and stored no vector regardless -- and the model only ever loaded
|
|
2178
|
+
# because some *recall* eventually paid for it. Whoever recalled first
|
|
2179
|
+
# wore the cold start.
|
|
2180
|
+
#
|
|
2181
|
+
# This belongs to the daemon and not to engine wiring: the worker is a
|
|
2182
|
+
# subprocess per engine, so warming from wiring would have every `slm
|
|
2183
|
+
# status` spawn one and load a model it will never use.
|
|
2184
|
+
_start_embedder_warmup(engine)
|
|
2185
|
+
|
|
2186
|
+
# Tell the hook subprocesses that skill evolution is on. The hook reads
|
|
2187
|
+
# this env var as its fast-path signal and nothing ever set it, so the
|
|
2188
|
+
# feature was off for everyone who had switched it on in config: the
|
|
2189
|
+
# hook checked, found nothing, and returned False. Hooks are launched as
|
|
2190
|
+
# children of this process and inherit its environment, which is the
|
|
2191
|
+
# only channel between the two.
|
|
2192
|
+
try:
|
|
2193
|
+
if getattr(getattr(config, "evolution", None), "enabled", False):
|
|
2194
|
+
os.environ["SLM_EVOLUTION_ENABLED"] = "1"
|
|
2195
|
+
else:
|
|
2196
|
+
# Cleared as well as set: a daemon restarted with the setting
|
|
2197
|
+
# turned off must not leave the previous run's answer behind in
|
|
2198
|
+
# an environment the next hook inherits.
|
|
2199
|
+
os.environ.pop("SLM_EVOLUTION_ENABLED", None)
|
|
2200
|
+
except Exception as _evo_exc: # pragma: no cover — never block startup
|
|
2201
|
+
logger.debug("evolution flag not exported: %s", _evo_exc)
|
|
2202
|
+
|
|
2091
2203
|
# Refresh migration state now that the engine is initialised. Any
|
|
2092
2204
|
# schema work the engine's own bootstrap may have applied (e.g.
|
|
2093
2205
|
# runtime-table creation) is captured here so the dashboard and
|
|
@@ -2175,6 +2287,30 @@ async def lifespan(application: FastAPI):
|
|
|
2175
2287
|
"deferred migration runner crashed (non-fatal): %s", _dexc,
|
|
2176
2288
|
)
|
|
2177
2289
|
|
|
2290
|
+
# Move an existing store onto the graph and vector backends on the
|
|
2291
|
+
# first start after an upgrade. They have shipped as required
|
|
2292
|
+
# dependencies since 3.7 and sat unused, because building the
|
|
2293
|
+
# projections was three manual commands almost nobody ran. Runs after
|
|
2294
|
+
# the deferred migrations so it projects the converted store, and never
|
|
2295
|
+
# fatal: if the libraries will not import or the projection does not
|
|
2296
|
+
# match, the daemon serves from SQLite and says so.
|
|
2297
|
+
try:
|
|
2298
|
+
from superlocalmemory.core.scale_autopromote import (
|
|
2299
|
+
auto_promote_scale_backends,
|
|
2300
|
+
)
|
|
2301
|
+
_promotion = auto_promote_scale_backends(config)
|
|
2302
|
+
application.state.scale_autopromotion = _promotion.as_dict()
|
|
2303
|
+
if _promotion.promoted and _promotion.restart_required:
|
|
2304
|
+
logger.info(
|
|
2305
|
+
"graph and vector backends are promoted and serve after the "
|
|
2306
|
+
"next restart",
|
|
2307
|
+
)
|
|
2308
|
+
except Exception as _pexc: # pragma: no cover — defensive
|
|
2309
|
+
logger.warning("automatic backend promotion crashed (non-fatal): %s", _pexc)
|
|
2310
|
+
application.state.scale_autopromotion = {
|
|
2311
|
+
"attempted": True, "promoted": False, "reason": str(_pexc),
|
|
2312
|
+
}
|
|
2313
|
+
|
|
2178
2314
|
# S9-DASH-02: start the outcome-queue worker so recall →
|
|
2179
2315
|
# pending_outcomes is actually produced. Before v3.4.22 this
|
|
2180
2316
|
# producer had zero callers and the closed-loop pipeline was
|
|
@@ -3041,6 +3177,18 @@ async def lifespan(application: FastAPI):
|
|
|
3041
3177
|
await _cancel_fact_entity_association_repair(application)
|
|
3042
3178
|
await _cancel_source_quality_repair(application)
|
|
3043
3179
|
|
|
3180
|
+
# Stop the projection drain. Its queue is durable, so an interrupted pass
|
|
3181
|
+
# costs a repeat of idempotent work and nothing else — but a live worker
|
|
3182
|
+
# writing into RocksDB and Lance while the process tears down around it has
|
|
3183
|
+
# no upside.
|
|
3184
|
+
try:
|
|
3185
|
+
from superlocalmemory.core.backend_orchestrator import get_orchestrator
|
|
3186
|
+
_orch = get_orchestrator()
|
|
3187
|
+
if _orch is not None:
|
|
3188
|
+
_orch.stop()
|
|
3189
|
+
except Exception as exc: # pragma: no cover — defensive
|
|
3190
|
+
logger.warning("projection drain shutdown failed: %s", exc)
|
|
3191
|
+
|
|
3044
3192
|
# Cancel the cross-platform sync loop (H-CONC-2) so adapter file I/O does
|
|
3045
3193
|
# not outlive the daemon.
|
|
3046
3194
|
try:
|
|
@@ -3904,7 +4052,7 @@ def _register_dashboard_routes(application: FastAPI) -> None:
|
|
|
3904
4052
|
@application.middleware("http")
|
|
3905
4053
|
async def _migration_readiness_gate(request, call_next):
|
|
3906
4054
|
migration_result = getattr(application.state, "migration_result", None)
|
|
3907
|
-
if migration_result and migration_result
|
|
4055
|
+
if migration_result and _serving_blocked_by(migration_result):
|
|
3908
4056
|
if not _is_migration_exempt_path(request.url.path):
|
|
3909
4057
|
from fastapi.responses import JSONResponse
|
|
3910
4058
|
return JSONResponse(
|
|
@@ -4072,12 +4220,26 @@ def _register_dashboard_routes(application: FastAPI) -> None:
|
|
|
4072
4220
|
# 4.0.10: asset ?v= strings are now derived from file content instead of
|
|
4073
4221
|
# being hand-written literals that tracked nothing. See
|
|
4074
4222
|
# server/asset_versions.py — including what that does and does not fix.
|
|
4075
|
-
|
|
4223
|
+
# Asset versioning is cosmetic. It must never be why this page 500s.
|
|
4224
|
+
#
|
|
4225
|
+
# The import is deferred (house style, keeps startup lean), which means
|
|
4226
|
+
# it resolves at REQUEST time — so when `pip install -e .` replaced the
|
|
4227
|
+
# installed package underneath a running daemon, this route began
|
|
4228
|
+
# answering "Internal Server Error" on the dashboard while every other
|
|
4229
|
+
# endpoint was fine. A stale hand-written version string is a trifle; a
|
|
4230
|
+
# blank page is not. Fall back to the file as written.
|
|
4231
|
+
try:
|
|
4232
|
+
from superlocalmemory.server.asset_versions import render_index
|
|
4076
4233
|
|
|
4077
|
-
|
|
4078
|
-
|
|
4079
|
-
|
|
4080
|
-
|
|
4234
|
+
return render_index(
|
|
4235
|
+
index_path, UI_DIR, substitutions={"__SLM_VERSION__": _SLM_VERSION},
|
|
4236
|
+
)
|
|
4237
|
+
except Exception as exc: # noqa: BLE001 — serve the page regardless
|
|
4238
|
+
logger.warning(
|
|
4239
|
+
"asset version rewrite unavailable, serving index.html as "
|
|
4240
|
+
"written: %s: %s", type(exc).__name__, exc,
|
|
4241
|
+
)
|
|
4242
|
+
return index_path.read_text().replace("__SLM_VERSION__", _SLM_VERSION)
|
|
4081
4243
|
|
|
4082
4244
|
@application.get("/favicon.ico", include_in_schema=False)
|
|
4083
4245
|
async def favicon():
|
|
@@ -4159,7 +4321,14 @@ def _register_daemon_routes(application: FastAPI) -> None:
|
|
|
4159
4321
|
(migration_result or {}).get("failed", []) or []
|
|
4160
4322
|
)
|
|
4161
4323
|
migration_details = (migration_result or {}).get("details", {}) or {}
|
|
4162
|
-
|
|
4324
|
+
# Ready means "can serve", so it keys off the failures that actually
|
|
4325
|
+
# stop this daemon serving -- not off every failure. A data invariant
|
|
4326
|
+
# that ordinary use re-violated leaves every route working; reporting
|
|
4327
|
+
# not-ready for it told operators to restart, which fixed nothing a
|
|
4328
|
+
# background repair would not have fixed. Everything still shows up in
|
|
4329
|
+
# migration_failures and migration_failure_reasons below, named.
|
|
4330
|
+
migration_blocking = _serving_blocked_by(migration_result or {})
|
|
4331
|
+
migrations_ready = bool(migration_result) and not migration_blocking
|
|
4163
4332
|
if migration_details.get("_crash"):
|
|
4164
4333
|
migrations_ready = False
|
|
4165
4334
|
writer_runtime = getattr(
|
|
@@ -4177,6 +4346,23 @@ def _register_daemon_routes(application: FastAPI) -> None:
|
|
|
4177
4346
|
"embedding": embedding_ready,
|
|
4178
4347
|
"recall_health": recall_health.get("recall_healthy") is True,
|
|
4179
4348
|
"migration_failures": migration_failures,
|
|
4349
|
+
# Which of those are the reason this daemon will not serve, as
|
|
4350
|
+
# opposed to the ones it is reporting while serving normally.
|
|
4351
|
+
"migration_blocking": migration_blocking,
|
|
4352
|
+
# WHY each one failed, not just which. The runner already produces
|
|
4353
|
+
# a precise sentence per migration -- "safe repair did not restore
|
|
4354
|
+
# M043_...", "schema verification failed ... : <sqlite error>" --
|
|
4355
|
+
# and this endpoint computed it and then dropped it on the floor.
|
|
4356
|
+
# A migration recorded ``complete`` in migration_log can still be
|
|
4357
|
+
# reported failed here, because a completed migration is re-checked
|
|
4358
|
+
# by its own verify() on every start; with only a name to go on,
|
|
4359
|
+
# that reads as the health check contradicting the database. It is
|
|
4360
|
+
# not: they are answering different questions, and this is the
|
|
4361
|
+
# sentence that says which. Reported as #125.
|
|
4362
|
+
"migration_failure_reasons": {
|
|
4363
|
+
name: str(migration_details.get(name, "(no detail recorded)"))
|
|
4364
|
+
for name in migration_failures
|
|
4365
|
+
},
|
|
4180
4366
|
}
|
|
4181
4367
|
readiness["retrieval"] = bool(
|
|
4182
4368
|
readiness["embedding"] and readiness["recall_health"]
|
|
@@ -4255,6 +4441,12 @@ def _register_daemon_routes(application: FastAPI) -> None:
|
|
|
4255
4441
|
# reveal staleness on its own. Loopback-only, alongside the other
|
|
4256
4442
|
# operational metadata.
|
|
4257
4443
|
"version_integrity": _version_integrity_payload(),
|
|
4444
|
+
# How far behind the second graph store is. A drain that stops
|
|
4445
|
+
# advancing is the failure that does not announce itself: every
|
|
4446
|
+
# other signal here stays green while the graph quietly diverges
|
|
4447
|
+
# from the record, and the only symptom is answers that are subtly
|
|
4448
|
+
# worse. A depth that does not fall is the thing to alert on.
|
|
4449
|
+
"projection": _projection_health(),
|
|
4258
4450
|
}
|
|
4259
4451
|
|
|
4260
4452
|
@application.get("/recall")
|
|
@@ -4909,6 +5101,7 @@ def _register_daemon_routes(application: FastAPI) -> None:
|
|
|
4909
5101
|
fact_count = 0
|
|
4910
5102
|
entity_count = 0
|
|
4911
5103
|
edge_count = 0
|
|
5104
|
+
projection_queue_depth = 0
|
|
4912
5105
|
if engine is not None:
|
|
4913
5106
|
try:
|
|
4914
5107
|
fact_count = engine._db.get_fact_count(profile_snapshot.profile_id)
|
|
@@ -4925,6 +5118,11 @@ def _register_daemon_routes(application: FastAPI) -> None:
|
|
|
4925
5118
|
edge_count = int(dict(edges[0])["c"]) if edges else 0
|
|
4926
5119
|
except Exception:
|
|
4927
5120
|
logger.debug("daemon status count query failed", exc_info=True)
|
|
5121
|
+
try:
|
|
5122
|
+
from superlocalmemory.storage import projection_outbox
|
|
5123
|
+
projection_queue_depth = projection_outbox.depth(engine._db)
|
|
5124
|
+
except Exception:
|
|
5125
|
+
logger.debug("projection queue depth unavailable", exc_info=True)
|
|
4928
5126
|
db_path = getattr(config, "db_path", None)
|
|
4929
5127
|
db_size_mb = (
|
|
4930
5128
|
round(db_path.stat().st_size / 1024 / 1024, 2)
|
|
@@ -4950,6 +5148,10 @@ def _register_daemon_routes(application: FastAPI) -> None:
|
|
|
4950
5148
|
"legacy_port": _LEGACY_PORT,
|
|
4951
5149
|
"profile": profile_snapshot.profile_id,
|
|
4952
5150
|
"profile_generation": profile_snapshot.generation,
|
|
5151
|
+
# Facts stored but not yet in the graph and vector projections. The
|
|
5152
|
+
# CLI and MCP read their own status from here, so this is where the
|
|
5153
|
+
# number has to be for all three surfaces to agree.
|
|
5154
|
+
"projection_queue_depth": projection_queue_depth,
|
|
4953
5155
|
# F2 fix: expose M028 backfill progress so operators can monitor
|
|
4954
5156
|
# the post-upgrade fact/entity association repair state.
|
|
4955
5157
|
"m028_backfill": getattr(
|
|
@@ -5561,6 +5763,38 @@ def _terminalize_orphan_operation(engine, operation_id: str) -> None:
|
|
|
5561
5763
|
)
|
|
5562
5764
|
|
|
5563
5765
|
|
|
5766
|
+
def _projection_health() -> dict:
|
|
5767
|
+
"""Queue depth, stall count, and whether the worker is running.
|
|
5768
|
+
|
|
5769
|
+
Never raises: a health endpoint that fails because one of its fields could
|
|
5770
|
+
not be computed is worse than the missing field.
|
|
5771
|
+
|
|
5772
|
+
Takes no application. It used to accept one and never read it -- the
|
|
5773
|
+
orchestrator is a process singleton -- which made the signature claim a
|
|
5774
|
+
dependency the body did not have, and made a test that handed it a broken
|
|
5775
|
+
application look like it was exercising the failure path when it was only
|
|
5776
|
+
observing whatever the process had already built. Compare
|
|
5777
|
+
``_ops_failure_counts`` directly below, which takes an application because it
|
|
5778
|
+
genuinely reads one.
|
|
5779
|
+
"""
|
|
5780
|
+
try:
|
|
5781
|
+
from superlocalmemory.core.backend_orchestrator import get_orchestrator
|
|
5782
|
+
|
|
5783
|
+
orchestrator = get_orchestrator()
|
|
5784
|
+
if orchestrator is None:
|
|
5785
|
+
return {"available": False}
|
|
5786
|
+
health = dict(orchestrator.outbox_health())
|
|
5787
|
+
health["available"] = True
|
|
5788
|
+
# One boolean an alert can key on without knowing what a healthy depth
|
|
5789
|
+
# looks like on this store.
|
|
5790
|
+
health["behind"] = bool(health.get("depth", 0)) or bool(
|
|
5791
|
+
health.get("stalled", 0)
|
|
5792
|
+
)
|
|
5793
|
+
return health
|
|
5794
|
+
except Exception as exc: # noqa: BLE001
|
|
5795
|
+
return {"available": False, "error": str(exc)[:120]}
|
|
5796
|
+
|
|
5797
|
+
|
|
5564
5798
|
def _ops_failure_counts(engine, application) -> dict:
|
|
5565
5799
|
"""Return operational failure counts for /status and /health.
|
|
5566
5800
|
|