superlocalmemory 4.0.8 → 4.0.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +221 -1
- package/README.md +6 -6
- package/package.json +1 -1
- package/plugin/.claude-plugin/plugin.json +1 -1
- package/plugin/CLAUDE.md +3 -3
- package/plugin/agents/slm-governance-advisor.md +1 -1
- package/plugin/agents/slm-loop-runner.md +1 -1
- package/plugin/agents/slm-memory-advisor.md +1 -1
- package/plugin/agents/slm-optimize-advisor.md +1 -1
- package/plugin/requirements.txt +1 -1
- package/plugin/scripts/ensure-venv.sh +1 -1
- package/plugin/skills/slm-cache/SKILL.md +1 -1
- package/plugin/skills/slm-compress/SKILL.md +1 -1
- package/plugin/skills/slm-governance/SKILL.md +1 -1
- package/plugin/skills/slm-graph/SKILL.md +1 -1
- package/plugin/skills/slm-loop/SKILL.md +1 -1
- package/plugin/skills/slm-mesh/SKILL.md +1 -1
- package/plugin/skills/slm-profile/SKILL.md +1 -1
- package/plugin/skills/slm-recall/SKILL.md +1 -1
- package/plugin/skills/slm-remember/SKILL.md +1 -1
- package/plugin/skills/slm-scope/SKILL.md +1 -1
- package/plugin/skills/slm-session/SKILL.md +1 -1
- package/plugin/skills/slm-status/SKILL.md +3 -3
- package/plugin-src/rules/AGENTS.md +1 -1
- package/plugin-src/skills/slm-status/SKILL.md +2 -2
- package/pyproject.toml +1 -1
- package/scripts/postinstall.js +4 -0
- package/src/superlocalmemory/__init__.py +1 -1
- package/src/superlocalmemory/cli/_lazy_init.py +1 -1
- package/src/superlocalmemory/cli/commands.py +164 -11
- package/src/superlocalmemory/cli/db_migrate.py +0 -2
- package/src/superlocalmemory/cli/gdpr_io.py +1 -1
- package/src/superlocalmemory/cli/main.py +7 -7
- package/src/superlocalmemory/cli/service_installer.py +2 -1
- package/src/superlocalmemory/cli/setup_wizard.py +1 -1
- package/src/superlocalmemory/code_graph/bridge/maintenance.py +8 -0
- package/src/superlocalmemory/core/config.py +41 -7
- package/src/superlocalmemory/core/context_cache.py +0 -2
- package/src/superlocalmemory/core/engine.py +371 -63
- package/src/superlocalmemory/core/evidence_bundle.py +3 -1
- package/src/superlocalmemory/core/fact_consolidator.py +316 -125
- package/src/superlocalmemory/core/install_detector.py +131 -0
- package/src/superlocalmemory/core/maintenance.py +44 -6
- package/src/superlocalmemory/core/memory_health.py +266 -0
- package/src/superlocalmemory/core/operation_policy_registry.py +1 -1
- package/src/superlocalmemory/core/operation_request.py +1 -1
- package/src/superlocalmemory/core/ops_remediation.py +2 -2
- package/src/superlocalmemory/core/progressive_abstraction.py +1 -1
- package/src/superlocalmemory/core/security_primitives.py +3 -6
- package/src/superlocalmemory/core/store_pipeline.py +172 -29
- package/src/superlocalmemory/core/topic_signature.py +0 -2
- package/src/superlocalmemory/core/transactions/concrete_owners.py +15 -8
- package/src/superlocalmemory/dynamics/eap_scheduler.py +17 -6
- package/src/superlocalmemory/encoding/cognitive_consolidator.py +15 -1
- package/src/superlocalmemory/encoding/graph_builder.py +2 -2
- package/src/superlocalmemory/encoding/scene_builder.py +8 -2
- package/src/superlocalmemory/hooks/adapter_base.py +0 -2
- package/src/superlocalmemory/hooks/context_payload.py +0 -2
- package/src/superlocalmemory/hooks/portable_kit.py +8 -8
- package/src/superlocalmemory/hooks/post_tool_async_hook.py +0 -2
- package/src/superlocalmemory/hooks/prewarm_auth.py +0 -2
- package/src/superlocalmemory/hooks/user_prompt_hook.py +0 -2
- package/src/superlocalmemory/infra/backup.py +44 -8
- package/src/superlocalmemory/integrations/bounded_loops_mcp.py +24 -7
- package/src/superlocalmemory/learning/arm_catalog.py +0 -2
- package/src/superlocalmemory/learning/bandit.py +0 -2
- package/src/superlocalmemory/learning/bandit_cache.py +0 -2
- package/src/superlocalmemory/learning/dedup_hnsw.py +11 -11
- package/src/superlocalmemory/learning/ensemble.py +0 -2
- package/src/superlocalmemory/learning/labeler.py +0 -2
- package/src/superlocalmemory/learning/legacy_migration.py +0 -2
- package/src/superlocalmemory/learning/model_cache.py +0 -2
- package/src/superlocalmemory/learning/ranker.py +0 -2
- package/src/superlocalmemory/learning/reward_archive.py +6 -1
- package/src/superlocalmemory/learning/reward_proxy.py +0 -2
- package/src/superlocalmemory/learning/signal_worker.py +0 -2
- package/src/superlocalmemory/math/fisher.py +1 -1
- package/src/superlocalmemory/math/hopfield.py +4 -1
- package/src/superlocalmemory/math/langevin.py +1 -1
- package/src/superlocalmemory/math/sheaf.py +7 -3
- package/src/superlocalmemory/mcp/cli_fallback.py +1 -1
- package/src/superlocalmemory/mcp/profiles.py +1 -1
- package/src/superlocalmemory/mcp/server.py +2 -2
- package/src/superlocalmemory/mcp/session_binding.py +92 -0
- package/src/superlocalmemory/mcp/tools_active.py +56 -0
- package/src/superlocalmemory/mcp/tools_core.py +41 -40
- package/src/superlocalmemory/mcp/tools_ops.py +2 -2
- package/src/superlocalmemory/optimize/cache/manager.py +2 -2
- package/src/superlocalmemory/optimize/compress/ccr.py +1 -1
- package/src/superlocalmemory/optimize/compress/router.py +1 -1
- package/src/superlocalmemory/optimize/proxy/_helpers.py +2 -2
- package/src/superlocalmemory/optimize/proxy/server.py +1 -1
- package/src/superlocalmemory/optimize/proxy/vertex_surface.py +2 -2
- package/src/superlocalmemory/optimize/storage/db.py +2 -2
- package/src/superlocalmemory/retrieval/agentic.py +1 -1
- package/src/superlocalmemory/retrieval/ann_index.py +9 -2
- package/src/superlocalmemory/retrieval/bm25_channel.py +6 -10
- package/src/superlocalmemory/retrieval/bridge_discovery.py +2 -2
- package/src/superlocalmemory/retrieval/engine.py +272 -43
- package/src/superlocalmemory/retrieval/entity_channel.py +8 -2
- package/src/superlocalmemory/retrieval/hopfield_channel.py +8 -2
- package/src/superlocalmemory/retrieval/profile_channel.py +1 -1
- package/src/superlocalmemory/retrieval/quantization_aware_search.py +1 -1
- package/src/superlocalmemory/retrieval/remote_reranker.py +2 -2
- package/src/superlocalmemory/retrieval/reranker.py +3 -3
- package/src/superlocalmemory/retrieval/scope_policy.py +22 -1
- package/src/superlocalmemory/retrieval/semantic_channel.py +3 -3
- package/src/superlocalmemory/retrieval/spreading_activation.py +8 -8
- package/src/superlocalmemory/retrieval/strategy.py +94 -0
- package/src/superlocalmemory/retrieval/temporal_channel.py +179 -10
- package/src/superlocalmemory/retrieval/temporal_validity_filter.py +1 -1
- package/src/superlocalmemory/retrieval/vector_store.py +151 -10
- package/src/superlocalmemory/server/api.py +6 -1
- package/src/superlocalmemory/server/asset_versions.py +171 -0
- package/src/superlocalmemory/server/recall_serializer.py +10 -0
- package/src/superlocalmemory/server/routes/abstraction.py +201 -0
- package/src/superlocalmemory/server/routes/brain.py +2 -2
- package/src/superlocalmemory/server/routes/data_io.py +29 -1
- package/src/superlocalmemory/server/routes/entity.py +13 -1
- package/src/superlocalmemory/server/routes/mesh.py +1 -1
- package/src/superlocalmemory/server/routes/prewarm.py +4 -4
- package/src/superlocalmemory/server/routes/v3_api.py +2 -2
- package/src/superlocalmemory/server/ui.py +8 -1
- package/src/superlocalmemory/server/unified_daemon.py +570 -16
- package/src/superlocalmemory/storage/_migration_internals.py +4 -0
- package/src/superlocalmemory/storage/_schema_version.py +46 -3
- package/src/superlocalmemory/storage/backup.py +531 -0
- package/src/superlocalmemory/storage/database.py +139 -34
- package/src/superlocalmemory/storage/embedding_codec.py +129 -0
- package/src/superlocalmemory/storage/embedding_migrator.py +5 -3
- package/src/superlocalmemory/storage/migration_runner.py +153 -2
- package/src/superlocalmemory/storage/migrations/M043_quarantine_display_summaries.py +488 -0
- package/src/superlocalmemory/storage/migrations/__init__.py +1 -1
- package/src/superlocalmemory/storage/migrations.py +15 -1
- package/src/superlocalmemory/storage/models.py +7 -0
- package/src/superlocalmemory/storage/quantized_store.py +4 -2
- package/src/superlocalmemory/storage/schema.py +98 -0
- package/src/superlocalmemory/summaries/base.py +1 -1
- package/src/superlocalmemory/summaries/non_answer.py +223 -0
- package/src/superlocalmemory/summaries/session_summary.py +1 -1
- package/src/superlocalmemory/ui/css/legacy-dashboard.css +1 -1
- package/src/superlocalmemory/ui/css/neural-glass.css +1 -1
- package/src/superlocalmemory/ui/index.html +1 -1
- package/src/superlocalmemory/ui/js/core.js +1 -1
- package/src/superlocalmemory/ui/js/od-brain.js +1 -1
- package/src/superlocalmemory/ui/js/od-memories.js +190 -1
- package/src/superlocalmemory/ui/js/od-ops-health.js +1 -1
|
@@ -26,6 +26,8 @@ License: AGPL-3.0-or-later
|
|
|
26
26
|
from __future__ import annotations
|
|
27
27
|
|
|
28
28
|
import asyncio
|
|
29
|
+
import concurrent.futures
|
|
30
|
+
import functools
|
|
29
31
|
import hashlib
|
|
30
32
|
import json
|
|
31
33
|
import logging
|
|
@@ -50,7 +52,7 @@ os.environ.setdefault("SLM_MCP_EMBEDDED", "1")
|
|
|
50
52
|
from fastapi import FastAPI, HTTPException, Request
|
|
51
53
|
from fastapi.middleware.cors import CORSMiddleware
|
|
52
54
|
from fastapi.middleware.gzip import GZipMiddleware
|
|
53
|
-
from pydantic import BaseModel
|
|
55
|
+
from pydantic import BaseModel, field_validator
|
|
54
56
|
|
|
55
57
|
from superlocalmemory.core.config import CANONICAL_RECALL_LIMIT
|
|
56
58
|
from superlocalmemory.infra.daemon_identity import (
|
|
@@ -67,6 +69,126 @@ from superlocalmemory.infra.data_root import (
|
|
|
67
69
|
)
|
|
68
70
|
|
|
69
71
|
|
|
72
|
+
def _notify_migration_applied(
|
|
73
|
+
applied: list,
|
|
74
|
+
elapsed: float,
|
|
75
|
+
backup_dir: "Path | None" = None,
|
|
76
|
+
) -> None:
|
|
77
|
+
"""Print a one-line confirmation after migrations complete successfully.
|
|
78
|
+
|
|
79
|
+
Written to stdout so it is visible in daemon.log and in interactive runs.
|
|
80
|
+
"""
|
|
81
|
+
count = len(applied)
|
|
82
|
+
if count == 0:
|
|
83
|
+
return
|
|
84
|
+
parts = [f"[SLM] {count} migration(s) applied in {elapsed:.1f}s."]
|
|
85
|
+
if backup_dir is not None:
|
|
86
|
+
parts.append(f"Backup: {backup_dir}")
|
|
87
|
+
print(" ".join(parts), flush=True)
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _write_migration_error_log(
|
|
91
|
+
failed: list,
|
|
92
|
+
backup_dir: "Path | None",
|
|
93
|
+
slm_home: "Path | None" = None,
|
|
94
|
+
applied: "list | None" = None,
|
|
95
|
+
) -> "Path":
|
|
96
|
+
"""Write a migration-error-{timestamp}.log to the SLM home directory.
|
|
97
|
+
|
|
98
|
+
Returns the Path of the written log file.
|
|
99
|
+
"""
|
|
100
|
+
import datetime
|
|
101
|
+
|
|
102
|
+
if slm_home is None:
|
|
103
|
+
slm_home = canonical_data_root()
|
|
104
|
+
|
|
105
|
+
slm_home.mkdir(parents=True, exist_ok=True)
|
|
106
|
+
ts = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
|
|
107
|
+
log_path = slm_home / f"migration-error-{ts}.log"
|
|
108
|
+
|
|
109
|
+
lines = [
|
|
110
|
+
f"Migration error — {ts}",
|
|
111
|
+
f"Failed: {', '.join(str(f) for f in failed)}",
|
|
112
|
+
]
|
|
113
|
+
# An upgrade is applied step by step and is deliberately non-fatal: some
|
|
114
|
+
# steps can succeed while a later one fails. Saying "your data was not
|
|
115
|
+
# modified" whenever a snapshot exists was therefore untrue in exactly the
|
|
116
|
+
# case that matters — and a user who believes nothing changed may delete the
|
|
117
|
+
# snapshot to reclaim space.
|
|
118
|
+
if applied:
|
|
119
|
+
lines.append(f"Applied before the failure: {', '.join(str(a) for a in applied)}")
|
|
120
|
+
lines.append(
|
|
121
|
+
"YOUR DATABASE WAS PARTIALLY CHANGED. The steps listed above "
|
|
122
|
+
"completed; the ones under 'Failed' did not."
|
|
123
|
+
)
|
|
124
|
+
elif applied is not None:
|
|
125
|
+
lines.append("No steps completed, so your database was not changed.")
|
|
126
|
+
if backup_dir is not None:
|
|
127
|
+
lines.append(f"Snapshot of the state before the upgrade: {backup_dir}")
|
|
128
|
+
lines.append(
|
|
129
|
+
"That snapshot is intact and can be restored. Keep it until you are "
|
|
130
|
+
"satisfied your data is correct."
|
|
131
|
+
)
|
|
132
|
+
else:
|
|
133
|
+
lines.append(
|
|
134
|
+
"NO SNAPSHOT WAS TAKEN for this attempt — do not assume a recovery "
|
|
135
|
+
"copy exists. Check for earlier snapshots before changing anything."
|
|
136
|
+
)
|
|
137
|
+
lines.append("")
|
|
138
|
+
lines.append("To diagnose, run: slm doctor")
|
|
139
|
+
|
|
140
|
+
log_path.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
|
141
|
+
return log_path
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def _notify_windows_user(message: str) -> None:
|
|
145
|
+
"""Best-effort notification on Windows. Silent no-op on other platforms."""
|
|
146
|
+
import sys
|
|
147
|
+
|
|
148
|
+
if sys.platform != "win32":
|
|
149
|
+
return
|
|
150
|
+
|
|
151
|
+
# Attempt 1 — Windows MessageBox via PowerShell
|
|
152
|
+
try:
|
|
153
|
+
import subprocess
|
|
154
|
+
|
|
155
|
+
safe_msg = message.replace("'", "").replace('"', "")
|
|
156
|
+
subprocess.run(
|
|
157
|
+
[
|
|
158
|
+
"powershell",
|
|
159
|
+
"-Command",
|
|
160
|
+
"[System.Reflection.Assembly]::LoadWithPartialName"
|
|
161
|
+
"('System.Windows.Forms') | Out-Null; "
|
|
162
|
+
f"[System.Windows.Forms.MessageBox]::Show('{safe_msg}', "
|
|
163
|
+
"'SuperLocalMemory')",
|
|
164
|
+
],
|
|
165
|
+
check=False,
|
|
166
|
+
timeout=5,
|
|
167
|
+
)
|
|
168
|
+
return
|
|
169
|
+
except Exception:
|
|
170
|
+
pass
|
|
171
|
+
|
|
172
|
+
# Attempt 2 — Windows Application EventLog
|
|
173
|
+
try:
|
|
174
|
+
import subprocess
|
|
175
|
+
|
|
176
|
+
subprocess.run(
|
|
177
|
+
[
|
|
178
|
+
"eventcreate",
|
|
179
|
+
"/T", "INFORMATION",
|
|
180
|
+
"/ID", "1001",
|
|
181
|
+
"/L", "APPLICATION",
|
|
182
|
+
"/SO", "SuperLocalMemory",
|
|
183
|
+
"/D", message[:512],
|
|
184
|
+
],
|
|
185
|
+
check=False,
|
|
186
|
+
timeout=5,
|
|
187
|
+
)
|
|
188
|
+
except Exception:
|
|
189
|
+
pass # best-effort only
|
|
190
|
+
|
|
191
|
+
|
|
70
192
|
def _learning_db_for_config(config) -> Path:
|
|
71
193
|
"""Single learning.db path for daemon migration + engine (F-05).
|
|
72
194
|
|
|
@@ -115,11 +237,142 @@ _SOURCE_QUALITY_MAX_BATCH_SIZE = 250
|
|
|
115
237
|
_SOURCE_QUALITY_PROFILE_REFRESH_SECONDS = 60.0
|
|
116
238
|
_FACT_ENTITY_REPAIR_MIN_RETRY_SECONDS = 0.05
|
|
117
239
|
_FACT_ENTITY_REPAIR_MAX_RETRY_SECONDS = 30.0
|
|
118
|
-
#
|
|
119
|
-
#
|
|
120
|
-
#
|
|
121
|
-
#
|
|
122
|
-
|
|
240
|
+
# How long a write may spend making itself findable by meaning before the
|
|
241
|
+
# receipt goes out. Storing a memory has a 1.5 s ceiling, and this is a ceiling
|
|
242
|
+
# rather than a target: the cost of coming in under it is a memory that can only
|
|
243
|
+
# be found by quoting its own wording, which is not how anyone asks. So the
|
|
244
|
+
# window is most of the budget, not a token amount of it. ``wait=true`` is still
|
|
245
|
+
# never permission to hold the event loop open indefinitely — past this the
|
|
246
|
+
# background pass finishes the job.
|
|
247
|
+
# The ceiling for a whole store request, shared by the durable write and the
|
|
248
|
+
# window that makes the memory findable by meaning. Both are sequential, so the
|
|
249
|
+
# second gets what the first left rather than a fresh grant.
|
|
250
|
+
_REMEMBER_TOTAL_CEILING_SECONDS = 1.5
|
|
251
|
+
_REMEMBER_ENRICHMENT_WAIT_SECONDS = 1.2
|
|
252
|
+
|
|
253
|
+
# Enrichment runs off the event loop, but NOT on the loop's default executor.
|
|
254
|
+
# `asyncio.to_thread` uses that default pool, so N simultaneous writers really did
|
|
255
|
+
# create N threads — and every other handler that borrows the same pool queued
|
|
256
|
+
# behind them. One shared, small pool bounds the cost of a burst: past its width
|
|
257
|
+
# the extra callers miss their deadline and defer, which is the intended
|
|
258
|
+
# degradation, instead of multiplying threads.
|
|
259
|
+
_ENRICHMENT_WORKERS = 2
|
|
260
|
+
_enrichment_pool: "concurrent.futures.ThreadPoolExecutor | None" = None
|
|
261
|
+
_enrichment_pool_lock = threading.Lock()
|
|
262
|
+
|
|
263
|
+
# Bounds how much enrichment may be in flight at once. A ThreadPoolExecutor
|
|
264
|
+
# queues without limit, so bounding the pool's WIDTH does not bound what can be
|
|
265
|
+
# handed to it: a burst of writers all submit, the first two run, and the rest sit
|
|
266
|
+
# in a queue nobody is waiting for any more — then wake up and write to the
|
|
267
|
+
# database after their callers have already been answered. A permit is taken
|
|
268
|
+
# before submitting and released by the worker when it has really finished, so at
|
|
269
|
+
# most `_ENRICHMENT_WORKERS` pieces of work exist at once and a caller that
|
|
270
|
+
# cannot get one is told immediately instead of being queued.
|
|
271
|
+
_enrichment_semaphore = threading.Semaphore(_ENRICHMENT_WORKERS)
|
|
272
|
+
|
|
273
|
+
# Set once the pool has been shut down, and never cleared. Without it, a caller
|
|
274
|
+
# that passed the unlocked check below arrives after shutdown, finds None, and
|
|
275
|
+
# builds a replacement pool that the completed shutdown will never reach.
|
|
276
|
+
_enrichment_pool_closed = False
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
class _EnrichmentAtCapacity(Exception):
|
|
280
|
+
"""No permit was free, so nothing was submitted.
|
|
281
|
+
|
|
282
|
+
Its own type, so that "we chose not to start" is never reported through the
|
|
283
|
+
same path as "it started and then failed" — those are different events and
|
|
284
|
+
only one of them is worth a warning about degradation.
|
|
285
|
+
"""
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
def _enrichment_executor():
|
|
289
|
+
"""The one pool inline enrichment may use. Created on first need.
|
|
290
|
+
|
|
291
|
+
Raises ``RuntimeError`` once the pool has been shut down. It deliberately
|
|
292
|
+
does not return ``None`` on that path: ``run_in_executor(None, ...)`` means
|
|
293
|
+
*the event loop's default executor*, which is the unbounded-thread behaviour
|
|
294
|
+
this pool exists to replace, so a None here would silently reintroduce it at
|
|
295
|
+
exactly the moment the process is trying to stop.
|
|
296
|
+
"""
|
|
297
|
+
global _enrichment_pool
|
|
298
|
+
# Everything happens under the lock, including the read that is returned. An
|
|
299
|
+
# earlier version checked the closed flag only on the create branch and then
|
|
300
|
+
# returned the global, so a caller that found a live pool, lost the CPU, and
|
|
301
|
+
# resumed after shutdown returned None — and None means the event loop's own
|
|
302
|
+
# default executor, unbounded and shared with every other handler. That is
|
|
303
|
+
# the precise failure this function exists to prevent, reintroduced at the
|
|
304
|
+
# one moment it matters.
|
|
305
|
+
with _enrichment_pool_lock:
|
|
306
|
+
if _enrichment_pool_closed:
|
|
307
|
+
raise RuntimeError("enrichment pool is shut down")
|
|
308
|
+
if _enrichment_pool is None:
|
|
309
|
+
_enrichment_pool = concurrent.futures.ThreadPoolExecutor(
|
|
310
|
+
max_workers=_ENRICHMENT_WORKERS,
|
|
311
|
+
thread_name_prefix="slm-enrich",
|
|
312
|
+
)
|
|
313
|
+
pool = _enrichment_pool
|
|
314
|
+
if pool is None: # pragma: no cover — belt and braces around the invariant
|
|
315
|
+
raise RuntimeError("enrichment pool is unavailable")
|
|
316
|
+
return pool
|
|
317
|
+
|
|
318
|
+
|
|
319
|
+
def _open_enrichment_pool() -> None:
|
|
320
|
+
"""Allow the pool to be created again, for a fresh run in this process.
|
|
321
|
+
|
|
322
|
+
The closed flag has to survive shutdown — that is its whole purpose — so
|
|
323
|
+
something has to clear it when a new run legitimately begins. Without this a
|
|
324
|
+
second startup in one process (a restart, or a test that builds the app more
|
|
325
|
+
than once) would find enrichment permanently refused and every write would
|
|
326
|
+
come back findable by wording only, with nothing in the logs to explain it.
|
|
327
|
+
"""
|
|
328
|
+
global _enrichment_pool_closed, _enrichment_semaphore
|
|
329
|
+
with _enrichment_pool_lock:
|
|
330
|
+
_enrichment_pool_closed = False
|
|
331
|
+
# A fresh run starts with its full capacity. Shutdown cancels work that
|
|
332
|
+
# was queued and never started, and a task that never starts never runs
|
|
333
|
+
# the release in its finally — so permits taken by cancelled work are
|
|
334
|
+
# gone. Without this reset the next run in the same process would run at
|
|
335
|
+
# reduced capacity, or none at all, and the only symptom would be every
|
|
336
|
+
# write coming back findable by wording with nothing in the log.
|
|
337
|
+
_enrichment_semaphore = threading.Semaphore(_ENRICHMENT_WORKERS)
|
|
338
|
+
|
|
339
|
+
|
|
340
|
+
def _enrich_and_release(engine, fact_ids: list[str], budget: float) -> int:
|
|
341
|
+
"""Run inline enrichment, releasing the permit only when it is really done.
|
|
342
|
+
|
|
343
|
+
The permit has to be released here rather than by the caller. A caller that
|
|
344
|
+
stops waiting at its own deadline has not finished the work — the worker is
|
|
345
|
+
still holding a pool thread — so releasing on the caller's timeout would let
|
|
346
|
+
the next request submit into a pool that is still fully occupied, which is
|
|
347
|
+
the unbounded queueing this permit exists to prevent.
|
|
348
|
+
"""
|
|
349
|
+
try:
|
|
350
|
+
return engine.enrich_new_facts_now(fact_ids, timeout_s=budget)
|
|
351
|
+
finally:
|
|
352
|
+
_enrichment_semaphore.release()
|
|
353
|
+
|
|
354
|
+
|
|
355
|
+
def _shutdown_enrichment_pool_if_created() -> None:
|
|
356
|
+
"""Shut down the enrichment pool if it was ever created.
|
|
357
|
+
|
|
358
|
+
Safe to call when the pool was never created (no-op) and safe to call
|
|
359
|
+
twice (second call is a no-op because the pool is set to None under the
|
|
360
|
+
lock before shutdown is called).
|
|
361
|
+
|
|
362
|
+
cancel_futures=True drops any work that is still queued but has not yet
|
|
363
|
+
started. Workers already inside embed() will finish their current call
|
|
364
|
+
and then stop; they do not block the caller because wait=False.
|
|
365
|
+
"""
|
|
366
|
+
global _enrichment_pool, _enrichment_pool_closed
|
|
367
|
+
with _enrichment_pool_lock:
|
|
368
|
+
# Marked closed under the same lock that guards creation, so a caller
|
|
369
|
+
# already past the unlocked check cannot build a replacement pool that
|
|
370
|
+
# this shutdown has already walked past.
|
|
371
|
+
_enrichment_pool_closed = True
|
|
372
|
+
pool = _enrichment_pool
|
|
373
|
+
_enrichment_pool = None
|
|
374
|
+
if pool is not None:
|
|
375
|
+
pool.shutdown(wait=False, cancel_futures=True)
|
|
123
376
|
_SENSITIVE_READ_PREFIXES = (
|
|
124
377
|
"/api/memories", "/api/facts", "/api/clusters", "/api/graph",
|
|
125
378
|
"/api/v3/associations", "/api/v3/core-memory",
|
|
@@ -334,6 +587,62 @@ class RememberRequest(BaseModel):
|
|
|
334
587
|
metadata: dict | None = None # v3.4.26: pass-through from MCP pool_store
|
|
335
588
|
idempotency_key: str | None = None
|
|
336
589
|
session_id: str = ""
|
|
590
|
+
#: WHEN this memory is about, as distinct from when it was written.
|
|
591
|
+
#:
|
|
592
|
+
#: The internal admission record has carried this field all along and this
|
|
593
|
+
#: model never had it, so every memory arriving over HTTP — which is every
|
|
594
|
+
#: memory, from the CLI, the tool interface and the dashboard alike — was
|
|
595
|
+
#: stamped with its ingestion date. Measured on the author's store: 200 of
|
|
596
|
+
#: the 200 most recent facts have an observation_date, and 196 of them are
|
|
597
|
+
#: the day they were written. A store that cannot be told "this happened in
|
|
598
|
+
#: March" cannot answer a question about March.
|
|
599
|
+
#:
|
|
600
|
+
#: Empty means "today", the previous behaviour. Format is YYYY-MM-DD or a
|
|
601
|
+
#: full ISO 8601 timestamp.
|
|
602
|
+
session_date: str = ""
|
|
603
|
+
|
|
604
|
+
@field_validator("session_date")
|
|
605
|
+
@classmethod
|
|
606
|
+
def _session_date_is_a_date(cls, value: str) -> str:
|
|
607
|
+
"""Reject a malformed date rather than ignore it.
|
|
608
|
+
|
|
609
|
+
Dropping it silently would leave the caller believing the date was
|
|
610
|
+
recorded while the memory quietly filed itself under today — which is
|
|
611
|
+
the exact failure this field exists to fix, reintroduced one layer up.
|
|
612
|
+
Rejecting is recoverable: the caller sees the error and resends.
|
|
613
|
+
|
|
614
|
+
ISO ONLY, DELIBERATELY, even though this system ships a parser that is
|
|
615
|
+
far more permissive. ``encoding/temporal_parser.parse_session_date``
|
|
616
|
+
accepts "May 8, 2026", "1:56 pm on 8 May, 2026", "14/03/2026", and also
|
|
617
|
+
"last tuesday" and "March 2026" — and those last two are why it is not
|
|
618
|
+
used here. Asked on 2026-08-21 it resolves "last tuesday" to
|
|
619
|
+
**2026-08-25**, four days into the future, and "March 2026" to the 21st,
|
|
620
|
+
a day it invents. Accepting either at this boundary would file a memory
|
|
621
|
+
under a confidently wrong date, silently, which is precisely the class
|
|
622
|
+
of defect this field was added to remove.
|
|
623
|
+
|
|
624
|
+
The caller here is a program — the command line, the tool interface, the
|
|
625
|
+
dashboard — not a person typing. ISO is the right contract for a
|
|
626
|
+
program, and a caller holding a human-typed date can run it through the
|
|
627
|
+
parser itself and send the result.
|
|
628
|
+
"""
|
|
629
|
+
if not value:
|
|
630
|
+
return value
|
|
631
|
+
from datetime import datetime as _dt
|
|
632
|
+
|
|
633
|
+
text = value.strip()
|
|
634
|
+
try:
|
|
635
|
+
_dt.fromisoformat(text.replace("Z", "+00:00"))
|
|
636
|
+
except ValueError:
|
|
637
|
+
raise ValueError(
|
|
638
|
+
"session_date must be YYYY-MM-DD or a full ISO 8601 timestamp; "
|
|
639
|
+
f"got {value!r}. To accept looser human phrasing, parse it "
|
|
640
|
+
"first with TemporalParser.parse_session_date and send the ISO "
|
|
641
|
+
"result — but check what it returns, because it resolves "
|
|
642
|
+
"relative phrases against today and can answer with a future "
|
|
643
|
+
"date."
|
|
644
|
+
) from None
|
|
645
|
+
return text
|
|
337
646
|
# v3.6.15 multi-scope: visibility of the new memory. ``None`` scope means
|
|
338
647
|
# "use the configured default_scope" (personal). shared_with is the list of
|
|
339
648
|
# profile_ids for scope='shared'.
|
|
@@ -1500,6 +1809,11 @@ async def lifespan(application: FastAPI):
|
|
|
1500
1809
|
"""Initialize engine, workers, and optional services on startup."""
|
|
1501
1810
|
global _last_activity
|
|
1502
1811
|
|
|
1812
|
+
# A previous run in this process left the enrichment pool closed so that a
|
|
1813
|
+
# write racing its shutdown could not resurrect it. This run is entitled to
|
|
1814
|
+
# one.
|
|
1815
|
+
_open_enrichment_pool()
|
|
1816
|
+
|
|
1503
1817
|
engine = None
|
|
1504
1818
|
config = None
|
|
1505
1819
|
deployment = None
|
|
@@ -1592,6 +1906,7 @@ async def lifespan(application: FastAPI):
|
|
|
1592
1906
|
_path_config = None
|
|
1593
1907
|
|
|
1594
1908
|
try:
|
|
1909
|
+
from superlocalmemory.storage.backup import InsufficientDiskSpaceError
|
|
1595
1910
|
from superlocalmemory.storage.migration_runner import apply_all
|
|
1596
1911
|
if _path_config is None:
|
|
1597
1912
|
# Catastrophic early-load failure: still one root via Mode-A default
|
|
@@ -1602,13 +1917,56 @@ async def lifespan(application: FastAPI):
|
|
|
1602
1917
|
_path_config = _SLMFallback.for_mode(_ModeFallback.A)
|
|
1603
1918
|
_learning_db = _learning_db_for_config(_path_config)
|
|
1604
1919
|
_memory_db = _memory_db_for_config(_path_config)
|
|
1920
|
+
import time as _time_mod
|
|
1921
|
+
_t0 = _time_mod.monotonic()
|
|
1605
1922
|
_result = apply_all(_learning_db, _memory_db)
|
|
1923
|
+
_elapsed = _time_mod.monotonic() - _t0
|
|
1606
1924
|
_applied = _result.get("applied", [])
|
|
1607
1925
|
_failed = _result.get("failed", [])
|
|
1926
|
+
_backup_dir = _result.get("details", {}).get("_backup")
|
|
1927
|
+
_backup_path = Path(_backup_dir) if _backup_dir else None
|
|
1608
1928
|
if _applied:
|
|
1609
1929
|
logger.info("migrations applied: %s", _applied)
|
|
1930
|
+
_notify_migration_applied(_applied, _elapsed, _backup_path)
|
|
1931
|
+
_notify_windows_user(
|
|
1932
|
+
f"SuperLocalMemory: {len(_applied)} database migration(s) applied."
|
|
1933
|
+
)
|
|
1610
1934
|
if _failed:
|
|
1611
1935
|
logger.warning("migrations failed (non-fatal): %s", _failed)
|
|
1936
|
+
try:
|
|
1937
|
+
_err_log = _write_migration_error_log(
|
|
1938
|
+
_failed, _backup_path,
|
|
1939
|
+
slm_home=_memory_db.parent if _memory_db else None,
|
|
1940
|
+
applied=_applied,
|
|
1941
|
+
)
|
|
1942
|
+
import sys as _sys
|
|
1943
|
+
# "Data is safe" is only true when NOTHING was applied. Migrations
|
|
1944
|
+
# are non-fatal and applied in order, so a later failure can leave
|
|
1945
|
+
# earlier ones committed — a partially changed store. Saying the
|
|
1946
|
+
# data is untouched then tells the user to ignore the one copy
|
|
1947
|
+
# that still holds the original.
|
|
1948
|
+
if _applied:
|
|
1949
|
+
_headline = (
|
|
1950
|
+
f"[SLM] Migration incomplete. YOUR DATABASE WAS PARTIALLY "
|
|
1951
|
+
f"CHANGED — {len(_applied)} step(s) applied, "
|
|
1952
|
+
f"{len(_failed)} failed. The pre-change copy is at "
|
|
1953
|
+
f"{_backup_path} — keep it until this is resolved."
|
|
1954
|
+
)
|
|
1955
|
+
else:
|
|
1956
|
+
_headline = (
|
|
1957
|
+
f"[SLM] Migration failed before any change was made. Your "
|
|
1958
|
+
f"data is as it was; a copy is also at {_backup_path}."
|
|
1959
|
+
)
|
|
1960
|
+
print(
|
|
1961
|
+
f"{_headline} See {_err_log}. Run: slm doctor",
|
|
1962
|
+
file=_sys.stderr,
|
|
1963
|
+
flush=True,
|
|
1964
|
+
)
|
|
1965
|
+
_notify_windows_user(
|
|
1966
|
+
f"SuperLocalMemory migration failed. Run slm doctor. Log: {_err_log}"
|
|
1967
|
+
)
|
|
1968
|
+
except Exception:
|
|
1969
|
+
pass # notification failures are non-fatal
|
|
1612
1970
|
application.state.migration_result = _result
|
|
1613
1971
|
# S9-SKEP-15: only commit the new `.last_version` AFTER migrations
|
|
1614
1972
|
# complete with zero failures. A partial upgrade (schema didn't
|
|
@@ -1626,6 +1984,35 @@ async def lifespan(application: FastAPI):
|
|
|
1626
1984
|
_version_marker.write_text(_slm_version, encoding="utf-8")
|
|
1627
1985
|
except OSError:
|
|
1628
1986
|
pass # non-fatal
|
|
1987
|
+
except InsufficientDiskSpaceError as _disk_exc:
|
|
1988
|
+
# This is not a crash and it must not be reported as "non-fatal". The
|
|
1989
|
+
# migration deliberately refused to start because there was not enough
|
|
1990
|
+
# room to keep a recoverable copy first. The store is intact and
|
|
1991
|
+
# unmigrated; the generic handler below would have logged a warning and
|
|
1992
|
+
# left the daemon serving as though nothing had happened.
|
|
1993
|
+
logger.error(
|
|
1994
|
+
"MIGRATION NOT RUN — not enough free disk to keep a recoverable "
|
|
1995
|
+
"copy first. Your data has NOT been modified. Need %s bytes, have "
|
|
1996
|
+
"%s free. Free some space and restart; run `slm doctor` for details.",
|
|
1997
|
+
f"{_disk_exc.needed_bytes:,}", f"{_disk_exc.free_bytes:,}",
|
|
1998
|
+
)
|
|
1999
|
+
try:
|
|
2000
|
+
_log_path = _write_migration_error_log(
|
|
2001
|
+
["_insufficient_disk_space"], None,
|
|
2002
|
+
slm_home=locals().get("_memory_db").parent if locals().get("_memory_db") else None,
|
|
2003
|
+
applied=[],
|
|
2004
|
+
)
|
|
2005
|
+
logger.error("Details written to %s", _log_path)
|
|
2006
|
+
except OSError:
|
|
2007
|
+
pass
|
|
2008
|
+
application.state.migration_result = {
|
|
2009
|
+
"applied": [], "skipped": [], "failed": ["_insufficient_disk_space"],
|
|
2010
|
+
"details": {
|
|
2011
|
+
"_needed_bytes": _disk_exc.needed_bytes,
|
|
2012
|
+
"_free_bytes": _disk_exc.free_bytes,
|
|
2013
|
+
"_store_modified": False,
|
|
2014
|
+
},
|
|
2015
|
+
}
|
|
1629
2016
|
except Exception as _exc:
|
|
1630
2017
|
logger.warning("migration runner crashed (non-fatal): %s", _exc)
|
|
1631
2018
|
application.state.migration_result = {
|
|
@@ -2068,6 +2455,10 @@ async def lifespan(application: FastAPI):
|
|
|
2068
2455
|
"state": "checking_components", "embeddings_backfilled": 0,
|
|
2069
2456
|
"expansion_backfilled": 0, "null_remaining": None,
|
|
2070
2457
|
"components": None,
|
|
2458
|
+
# Declared here so /status carries the same keys whatever
|
|
2459
|
+
# happens; a field that only appears on failure is a field
|
|
2460
|
+
# nobody's dashboard renders.
|
|
2461
|
+
"incomplete_reason": None,
|
|
2071
2462
|
"started_at": _t.time(), "finished_at": None,
|
|
2072
2463
|
}
|
|
2073
2464
|
# Step 0 (v3.8.2 "whole self-healer"): repair components that
|
|
@@ -2176,9 +2567,42 @@ async def lifespan(application: FastAPI):
|
|
|
2176
2567
|
_backfill_vector_store()
|
|
2177
2568
|
except Exception as exc:
|
|
2178
2569
|
logger.warning("Self-heal vector index failed (non-fatal): %s", exc)
|
|
2179
|
-
|
|
2570
|
+
|
|
2571
|
+
# "complete" has to mean complete.
|
|
2572
|
+
#
|
|
2573
|
+
# This line used to set "complete" unconditionally, so a
|
|
2574
|
+
# backfill that gave up after five no-progress attempts, or ran
|
|
2575
|
+
# out its 500-iteration budget, reported success with facts
|
|
2576
|
+
# still unembedded — and an unembedded fact cannot be found by
|
|
2577
|
+
# meaning at all. That is how a machine sat at 56.3% of its
|
|
2578
|
+
# memory reachable while its own status endpoint said the heal
|
|
2579
|
+
# had finished. Nobody was going to look past a green light.
|
|
2580
|
+
#
|
|
2581
|
+
# Now the state is derived from the remaining count, and the
|
|
2582
|
+
# gap is named in plain language for the dashboard, so an
|
|
2583
|
+
# incomplete heal is visible and gets retried on next start
|
|
2584
|
+
# instead of being declared done forever.
|
|
2585
|
+
_remaining = _SELF_HEAL_STATUS.get("null_remaining")
|
|
2180
2586
|
_SELF_HEAL_STATUS["finished_at"] = _t.time()
|
|
2181
|
-
|
|
2587
|
+
if _remaining is None:
|
|
2588
|
+
_SELF_HEAL_STATUS["state"] = "complete"
|
|
2589
|
+
elif int(_remaining) > 0:
|
|
2590
|
+
_SELF_HEAL_STATUS["state"] = "incomplete"
|
|
2591
|
+
_SELF_HEAL_STATUS["incomplete_reason"] = (
|
|
2592
|
+
f"{int(_remaining)} memories still have no meaning "
|
|
2593
|
+
f"vector, so they cannot be found by asking a question. "
|
|
2594
|
+
f"This retries automatically next time the service "
|
|
2595
|
+
f"starts. It usually means the embedding model was "
|
|
2596
|
+
f"unavailable."
|
|
2597
|
+
)
|
|
2598
|
+
logger.warning(
|
|
2599
|
+
"Self-heal INCOMPLETE: %d facts still unembedded and "
|
|
2600
|
+
"therefore unreachable by meaning; will retry on next "
|
|
2601
|
+
"start. %s", int(_remaining), _SELF_HEAL_STATUS,
|
|
2602
|
+
)
|
|
2603
|
+
else:
|
|
2604
|
+
_SELF_HEAL_STATUS["state"] = "complete"
|
|
2605
|
+
logger.info("Self-heal complete: %s", _SELF_HEAL_STATUS)
|
|
2182
2606
|
except Exception as exc:
|
|
2183
2607
|
_SELF_HEAL_STATUS["state"] = "error"
|
|
2184
2608
|
logger.warning("Self-heal failed (non-fatal): %s", exc)
|
|
@@ -2822,6 +3246,19 @@ async def lifespan(application: FastAPI):
|
|
|
2822
3246
|
except Exception as exc: # pragma: no cover — defensive
|
|
2823
3247
|
logger.warning("perf_log flush failed: %s", exc)
|
|
2824
3248
|
|
|
3249
|
+
# Shut down the enrichment pool before engine.close() so that any worker
|
|
3250
|
+
# still inside embed() receives cancellation before the embed pool is torn
|
|
3251
|
+
# down. shutdown(wait=False, cancel_futures=True) is non-blocking: queued
|
|
3252
|
+
# futures are dropped and running futures complete their current call on
|
|
3253
|
+
# their own. This prevents Python's atexit handler from calling
|
|
3254
|
+
# shutdown(wait=True) on a live embed worker, which would push the daemon
|
|
3255
|
+
# past the graceful-shutdown budget and cause the service manager to
|
|
3256
|
+
# SIGKILL the process.
|
|
3257
|
+
try:
|
|
3258
|
+
_shutdown_enrichment_pool_if_created()
|
|
3259
|
+
except Exception as exc: # pragma: no cover — defensive
|
|
3260
|
+
logger.warning("enrichment pool shutdown failed (non-fatal): %s", exc)
|
|
3261
|
+
|
|
2825
3262
|
materializer_stopped = _stop_pending_materializer()
|
|
2826
3263
|
canonical_writer_stopped = _release_canonical_remember_runtime(application)
|
|
2827
3264
|
_profile_runtime = None
|
|
@@ -3631,8 +4068,16 @@ def _register_dashboard_routes(application: FastAPI) -> None:
|
|
|
3631
4068
|
# v3.4.23: substitute version placeholder so the dashboard can detect
|
|
3632
4069
|
# upgrades and auto-reload. Read fresh each request (daemon uptime is
|
|
3633
4070
|
# days, but we want zero caching surprises during development).
|
|
3634
|
-
|
|
3635
|
-
|
|
4071
|
+
#
|
|
4072
|
+
# 4.0.10: asset ?v= strings are now derived from file content instead of
|
|
4073
|
+
# being hand-written literals that tracked nothing. See
|
|
4074
|
+
# server/asset_versions.py — including what that does and does not fix.
|
|
4075
|
+
from superlocalmemory.server.asset_versions import render_index
|
|
4076
|
+
|
|
4077
|
+
return render_index(
|
|
4078
|
+
index_path, UI_DIR,
|
|
4079
|
+
substitutions={"__SLM_VERSION__": _SLM_VERSION},
|
|
4080
|
+
)
|
|
3636
4081
|
|
|
3637
4082
|
@application.get("/favicon.ico", include_in_schema=False)
|
|
3638
4083
|
async def favicon():
|
|
@@ -3802,7 +4247,7 @@ def _register_daemon_routes(application: FastAPI) -> None:
|
|
|
3802
4247
|
"runtime_state": runtime_state,
|
|
3803
4248
|
"active_profile": profile_snapshot.profile_id,
|
|
3804
4249
|
"profile_generation": profile_snapshot.generation,
|
|
3805
|
-
#
|
|
4250
|
+
# operational failure counts (visible to all team members)
|
|
3806
4251
|
**_ops_failure_counts(engine, application),
|
|
3807
4252
|
# issue #107: does this daemon's *imported* code still match the
|
|
3808
4253
|
# installed distribution? ``version`` above reports what this
|
|
@@ -4169,12 +4614,17 @@ def _register_daemon_routes(application: FastAPI) -> None:
|
|
|
4169
4614
|
shared_with=tuple(shared_with or ()),
|
|
4170
4615
|
trusted_actor_id=trusted_actor_id,
|
|
4171
4616
|
session_id=req.session_id,
|
|
4617
|
+
session_date=req.session_date,
|
|
4172
4618
|
)
|
|
4173
4619
|
actor = Actor(
|
|
4174
4620
|
principal_id=trusted_actor_id,
|
|
4175
4621
|
allowed_profiles=frozenset({engine._profile_id}),
|
|
4176
4622
|
allowed_scopes=frozenset({scope}),
|
|
4177
4623
|
)
|
|
4624
|
+
# The whole request has a 1.5 s ceiling, and the two phases below
|
|
4625
|
+
# are sequential, so the ceiling has to be shared between them
|
|
4626
|
+
# rather than granted twice. Measured from here.
|
|
4627
|
+
_store_started = time.monotonic()
|
|
4178
4628
|
receipt = await asyncio.to_thread(
|
|
4179
4629
|
runtime.remember,
|
|
4180
4630
|
admission,
|
|
@@ -4183,10 +4633,105 @@ def _register_daemon_routes(application: FastAPI) -> None:
|
|
|
4183
4633
|
)
|
|
4184
4634
|
payload = dict(receipt.payload)
|
|
4185
4635
|
fact_ids = list(payload.get("fact_ids") or [])
|
|
4636
|
+
|
|
4637
|
+
# The durable receipt is already committed above; nothing below can
|
|
4638
|
+
# fail this write. What remains is the window in which the memory can
|
|
4639
|
+
# only be found by quoting its own wording — a fact with no vector is
|
|
4640
|
+
# invisible to the semantic channel, so the memory describing what is
|
|
4641
|
+
# happening right now is the hardest one to find.
|
|
4642
|
+
#
|
|
4643
|
+
# Every entry point — command line, tool interface, dashboard —
|
|
4644
|
+
# arrives here, so closing the window here closes it for all of them
|
|
4645
|
+
# rather than for whichever door happened to be used.
|
|
4646
|
+
#
|
|
4647
|
+
# Strictly bounded, because this daemon serves many sessions at once:
|
|
4648
|
+
# the work runs off the event loop, on a two-worker pool, and yields
|
|
4649
|
+
# to any recall in flight. Past the deadline the caller gets exactly
|
|
4650
|
+
# the previous behaviour and the background materializer finishes the
|
|
4651
|
+
# job.
|
|
4652
|
+
# A caller that did not ask to wait still gets a real attempt, not a
|
|
4653
|
+
# token one. Almost no client passes wait=true, so a small budget
|
|
4654
|
+
# here meant almost every memory in practice was returned findable by
|
|
4655
|
+
# wording only — the default path deciding the product's behaviour.
|
|
4656
|
+
#
|
|
4657
|
+
# But it gets what is LEFT of the ceiling, not a second full budget.
|
|
4658
|
+
# The durable write above can legitimately consume most of the 1.5 s
|
|
4659
|
+
# under contention, and granting the full enrichment window on top of
|
|
4660
|
+
# that produced receipts well past 3 s — over a ceiling the changelog
|
|
4661
|
+
# states. A caller that asked to wait may exceed the shared ceiling,
|
|
4662
|
+
# because that is what asking to wait means; nobody else may.
|
|
4663
|
+
_elapsed = time.monotonic() - _store_started
|
|
4664
|
+
_remaining = _REMEMBER_TOTAL_CEILING_SECONDS - _elapsed
|
|
4665
|
+
if wait:
|
|
4666
|
+
enrich_budget = _REMEMBER_ENRICHMENT_WAIT_SECONDS
|
|
4667
|
+
else:
|
|
4668
|
+
enrich_budget = min(
|
|
4669
|
+
1.0, _REMEMBER_ENRICHMENT_WAIT_SECONDS, max(0.0, _remaining),
|
|
4670
|
+
)
|
|
4671
|
+
if enrich_budget <= 0.0:
|
|
4672
|
+
logger.warning(
|
|
4673
|
+
"the durable write used the whole %.1fs budget (%.2fs) — this "
|
|
4674
|
+
"memory is findable by its wording and the background pass "
|
|
4675
|
+
"will attach its vector",
|
|
4676
|
+
_REMEMBER_TOTAL_CEILING_SECONDS, _elapsed,
|
|
4677
|
+
)
|
|
4678
|
+
enriched = 0
|
|
4679
|
+
# Taken before submitting, released by the worker. If none is free
|
|
4680
|
+
# the pool is already saturated, and queueing behind it would mean
|
|
4681
|
+
# writing to the database after this response has been sent — so the
|
|
4682
|
+
# write is simply reported as findable by wording and the background
|
|
4683
|
+
# pass picks it up, which is the intended degradation.
|
|
4684
|
+
_permit = _enrichment_semaphore.acquire(blocking=False)
|
|
4685
|
+
if not _permit:
|
|
4686
|
+
logger.warning(
|
|
4687
|
+
"inline enrichment at capacity for %d fact(s) — deferred to "
|
|
4688
|
+
"the background pass", len(fact_ids),
|
|
4689
|
+
)
|
|
4690
|
+
try:
|
|
4691
|
+
if not _permit:
|
|
4692
|
+
raise _EnrichmentAtCapacity
|
|
4693
|
+
loop = asyncio.get_running_loop()
|
|
4694
|
+
try:
|
|
4695
|
+
_pending = loop.run_in_executor(
|
|
4696
|
+
_enrichment_executor(),
|
|
4697
|
+
functools.partial(
|
|
4698
|
+
_enrich_and_release,
|
|
4699
|
+
engine, fact_ids, enrich_budget,
|
|
4700
|
+
),
|
|
4701
|
+
)
|
|
4702
|
+
except BaseException:
|
|
4703
|
+
# Never handed to a worker, so nothing will release it.
|
|
4704
|
+
_enrichment_semaphore.release()
|
|
4705
|
+
raise
|
|
4706
|
+
enriched = await asyncio.wait_for(
|
|
4707
|
+
_pending, timeout=enrich_budget + 0.25,
|
|
4708
|
+
)
|
|
4709
|
+
except _EnrichmentAtCapacity:
|
|
4710
|
+
pass
|
|
4711
|
+
except (TimeoutError, asyncio.TimeoutError):
|
|
4712
|
+
# Worth seeing. Sustained timeouts mean writes are coming back
|
|
4713
|
+
# findable by wording only, which is a real degradation and was
|
|
4714
|
+
# previously visible at debug level alone.
|
|
4715
|
+
logger.warning(
|
|
4716
|
+
"inline enrichment exceeded its budget for %d fact(s) — "
|
|
4717
|
+
"deferred to the background pass", len(fact_ids),
|
|
4718
|
+
)
|
|
4719
|
+
except Exception as exc:
|
|
4720
|
+
logger.warning(
|
|
4721
|
+
"inline enrichment unavailable (%s: %s) — deferred to the "
|
|
4722
|
+
"background pass", type(exc).__name__, exc,
|
|
4723
|
+
)
|
|
4724
|
+
|
|
4725
|
+
searchable = "meaning" if enriched == len(fact_ids) and fact_ids else "wording"
|
|
4186
4726
|
return {
|
|
4187
4727
|
"ok": True,
|
|
4188
4728
|
"fact_ids": fact_ids,
|
|
4189
4729
|
"count": len(fact_ids),
|
|
4730
|
+
# Storing a memory and being able to find it again are different
|
|
4731
|
+
# things, and reporting only the first is how a caller ends up
|
|
4732
|
+
# believing a memory is retrievable when it is not yet.
|
|
4733
|
+
"searchable_by": searchable,
|
|
4734
|
+
"enriched_now": enriched,
|
|
4190
4735
|
"operation_id": payload["operation_id"],
|
|
4191
4736
|
# One-release compatibility alias. The durable operation ID is
|
|
4192
4737
|
# opaque and replaces the integer pending.db row identifier.
|
|
@@ -4194,8 +4739,17 @@ def _register_daemon_routes(application: FastAPI) -> None:
|
|
|
4194
4739
|
"status": "queryable",
|
|
4195
4740
|
"materialization_state": payload["materialization_state"],
|
|
4196
4741
|
"commit_sequence": payload.get("commit_sequence"),
|
|
4197
|
-
"note":
|
|
4198
|
-
|
|
4742
|
+
"note": (
|
|
4743
|
+
"stored and searchable by meaning"
|
|
4744
|
+
if searchable == "meaning"
|
|
4745
|
+
else "stored and searchable by wording; "
|
|
4746
|
+
"searchable by meaning shortly"
|
|
4747
|
+
),
|
|
4748
|
+
# Retained for callers that already read it. It used to be
|
|
4749
|
+
# unconditionally true, because `wait` bought nothing. It now
|
|
4750
|
+
# buys a larger best-effort enrichment budget — still never a
|
|
4751
|
+
# guarantee, and still never a block on durability.
|
|
4752
|
+
"wait_ignored": False if wait else True,
|
|
4199
4753
|
}
|
|
4200
4754
|
except Exception as exc:
|
|
4201
4755
|
from superlocalmemory.core.remember_admission import AdmissionRejected
|
|
@@ -4407,7 +4961,7 @@ def _register_daemon_routes(application: FastAPI) -> None:
|
|
|
4407
4961
|
# index backfill after an upgrade). Dashboard renders a plain
|
|
4408
4962
|
# "Optimizing memory…" line from this. Defaults to idle before start.
|
|
4409
4963
|
"self_heal": globals().get("_SELF_HEAL_STATUS", {"state": "idle"}),
|
|
4410
|
-
#
|
|
4964
|
+
# operational failure counts (dead-letter, degraded, stalled)
|
|
4411
4965
|
**_ops_failure_counts(engine, application),
|
|
4412
4966
|
}
|
|
4413
4967
|
|
|
@@ -4461,7 +5015,7 @@ def _register_daemon_routes(application: FastAPI) -> None:
|
|
|
4461
5015
|
return {"status": "started"}
|
|
4462
5016
|
|
|
4463
5017
|
# ------------------------------------------------------------------
|
|
4464
|
-
#
|
|
5018
|
+
# Operational Recovery & Admin Remediation (V4 resilience slice)
|
|
4465
5019
|
# ------------------------------------------------------------------
|
|
4466
5020
|
|
|
4467
5021
|
@application.get("/operations/failed")
|
|
@@ -5008,7 +5562,7 @@ def _terminalize_orphan_operation(engine, operation_id: str) -> None:
|
|
|
5008
5562
|
|
|
5009
5563
|
|
|
5010
5564
|
def _ops_failure_counts(engine, application) -> dict:
|
|
5011
|
-
"""Return
|
|
5565
|
+
"""Return operational failure counts for /status and /health.
|
|
5012
5566
|
|
|
5013
5567
|
Always returns a dict (never raises). Counts default to 0 on any error.
|
|
5014
5568
|
Includes: dead_letter_count, degraded_operations, exhausted_obligations,
|