superlocalmemory 3.8.8 → 3.8.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +44 -0
- package/README.md +3 -3
- package/package.json +1 -1
- package/plugin/.claude-plugin/plugin.json +1 -1
- package/plugin/CLAUDE.md +3 -3
- package/plugin/agents/slm-governance-advisor.md +1 -1
- package/plugin/agents/slm-loop-runner.md +1 -1
- package/plugin/agents/slm-memory-advisor.md +1 -1
- package/plugin/agents/slm-optimize-advisor.md +1 -1
- package/plugin/requirements.txt +1 -1
- package/plugin/skills/slm-cache/SKILL.md +1 -1
- package/plugin/skills/slm-compress/SKILL.md +1 -1
- package/plugin/skills/slm-governance/SKILL.md +1 -1
- package/plugin/skills/slm-graph/SKILL.md +1 -1
- package/plugin/skills/slm-loop/SKILL.md +1 -1
- package/plugin/skills/slm-mesh/SKILL.md +1 -1
- package/plugin/skills/slm-profile/SKILL.md +1 -1
- package/plugin/skills/slm-recall/SKILL.md +1 -1
- package/plugin/skills/slm-remember/SKILL.md +1 -1
- package/plugin/skills/slm-scope/SKILL.md +1 -1
- package/plugin/skills/slm-session/SKILL.md +1 -1
- package/plugin/skills/slm-status/SKILL.md +1 -1
- package/plugin-src/rules/AGENTS.md +1 -1
- package/plugin-src/skills/slm-cache/SKILL.md +1 -1
- package/plugin-src/skills/slm-compress/SKILL.md +1 -1
- package/plugin-src/skills/slm-graph/SKILL.md +1 -1
- package/plugin-src/skills/slm-recall/SKILL.md +1 -1
- package/plugin-src/skills/slm-remember/SKILL.md +1 -1
- package/plugin-src/skills/slm-session/SKILL.md +1 -1
- package/plugin-src/skills/slm-status/SKILL.md +1 -1
- package/pyproject.toml +1 -1
- package/src/superlocalmemory/__init__.py +1 -1
- package/src/superlocalmemory/cli/daemon.py +89 -16
- package/src/superlocalmemory/core/embeddings.py +90 -6
- package/src/superlocalmemory/core/engine_ingestion.py +5 -0
- package/src/superlocalmemory/core/ingestion_command.py +36 -0
- package/src/superlocalmemory/core/materialization_control.py +20 -0
- package/src/superlocalmemory/core/ollama_embedder.py +11 -2
- package/src/superlocalmemory/core/recall_gate.py +27 -3
- package/src/superlocalmemory/core/remember_admission.py +14 -5
- package/src/superlocalmemory/core/store_pipeline.py +10 -0
- package/src/superlocalmemory/hooks/adapter_base.py +10 -3
- package/src/superlocalmemory/mcp/tools_core.py +32 -12
- package/src/superlocalmemory/optimize/proxy/capture.py +148 -30
- package/src/superlocalmemory/optimize/storage/db.py +6 -2
- package/src/superlocalmemory/server/unified_daemon.py +11 -2
- package/src/superlocalmemory/storage/admission_codec.py +10 -0
- package/src/superlocalmemory/storage/admission_journal.py +182 -67
- package/src/superlocalmemory/storage/embedding_migrator.py +27 -13
- package/src/superlocalmemory/storage/write_coordinator.py +68 -21
|
@@ -22,6 +22,7 @@ from dataclasses import dataclass, field
|
|
|
22
22
|
from enum import Enum
|
|
23
23
|
from typing import Any, Callable
|
|
24
24
|
|
|
25
|
+
from superlocalmemory.core.materialization_control import MaterializationDeferred
|
|
25
26
|
from superlocalmemory.storage.database import DatabaseManager
|
|
26
27
|
|
|
27
28
|
logger = logging.getLogger("superlocalmemory.ingestion_command")
|
|
@@ -606,6 +607,31 @@ class IngestionOperationRepository:
|
|
|
606
607
|
) from exc
|
|
607
608
|
return self._from_row(rows[0])
|
|
608
609
|
|
|
610
|
+
def defer_enriching(
|
|
611
|
+
self,
|
|
612
|
+
operation_id: str,
|
|
613
|
+
*,
|
|
614
|
+
owner: str,
|
|
615
|
+
) -> IngestionOperation:
|
|
616
|
+
"""Release a transition-preempted lease without consuming a retry.
|
|
617
|
+
|
|
618
|
+
Queryable evidence remains durable. The compare-and-swap owner check
|
|
619
|
+
prevents a stale worker from requeueing work that another process has
|
|
620
|
+
already reclaimed.
|
|
621
|
+
"""
|
|
622
|
+
rows = self.db.execute(
|
|
623
|
+
"UPDATE ingestion_operations SET state='queryable', "
|
|
624
|
+
"lease_owner='', lease_expires_at=0, next_retry_at=0, "
|
|
625
|
+
"attempt_count=CASE WHEN attempt_count > 0 THEN attempt_count - 1 ELSE 0 END, "
|
|
626
|
+
"last_error='', updated_at=strftime('%Y-%m-%dT%H:%M:%fZ', 'now') "
|
|
627
|
+
"WHERE operation_id=? AND state='enriching' AND lease_owner=? "
|
|
628
|
+
"RETURNING *",
|
|
629
|
+
(operation_id, owner),
|
|
630
|
+
)
|
|
631
|
+
if not rows:
|
|
632
|
+
raise InvalidStateTransition("enriching lease ownership was lost")
|
|
633
|
+
return self._from_row(rows[0])
|
|
634
|
+
|
|
609
635
|
def reap_stuck_enriching(
|
|
610
636
|
self,
|
|
611
637
|
*,
|
|
@@ -936,6 +962,11 @@ class IngestionCommand:
|
|
|
936
962
|
)
|
|
937
963
|
except LeaseLost:
|
|
938
964
|
raise
|
|
965
|
+
except MaterializationDeferred:
|
|
966
|
+
return self.repository.defer_enriching(
|
|
967
|
+
operation_id,
|
|
968
|
+
owner=self._owner,
|
|
969
|
+
)
|
|
939
970
|
except Exception as exc:
|
|
940
971
|
return self.repository.finish_enriching(
|
|
941
972
|
operation_id,
|
|
@@ -983,6 +1014,11 @@ class IngestionCommand:
|
|
|
983
1014
|
)
|
|
984
1015
|
except LeaseLost:
|
|
985
1016
|
raise
|
|
1017
|
+
except MaterializationDeferred:
|
|
1018
|
+
return self.repository.defer_enriching(
|
|
1019
|
+
operation.operation_id,
|
|
1020
|
+
owner=self._owner,
|
|
1021
|
+
)
|
|
986
1022
|
except Exception as exc:
|
|
987
1023
|
return self.repository.finish_enriching(
|
|
988
1024
|
operation.operation_id,
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
|
|
2
|
+
# Licensed under AGPL-3.0-or-later - see LICENSE file
|
|
3
|
+
# Part of SuperLocalMemory V3
|
|
4
|
+
|
|
5
|
+
"""Control-flow signals for best-effort materialization work.
|
|
6
|
+
|
|
7
|
+
These exceptions deliberately live outside the ingestion state machine and
|
|
8
|
+
embedding implementation so either layer can request a durable deferral
|
|
9
|
+
without introducing an import cycle.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class MaterializationDeferred(RuntimeError):
|
|
16
|
+
"""Best-effort enrichment yielded to a runtime transition.
|
|
17
|
+
|
|
18
|
+
The queryable projection is already durable. This signal must be handled
|
|
19
|
+
by :class:`IngestionCommand` as a requeue, never as a failed attempt.
|
|
20
|
+
"""
|
|
@@ -227,7 +227,7 @@ class OllamaEmbedder:
|
|
|
227
227
|
data = resp.json()
|
|
228
228
|
# Ollama /api/embed returns {"embeddings": [[...]]}
|
|
229
229
|
vec = data["embeddings"][0]
|
|
230
|
-
return self.
|
|
230
|
+
return self._normalize_checked(vec)
|
|
231
231
|
|
|
232
232
|
def _call_ollama_embed_batch(self, texts: list[str]) -> list[list[float] | None]:
|
|
233
233
|
"""Call Ollama embed endpoint with batch input.
|
|
@@ -245,7 +245,16 @@ class OllamaEmbedder:
|
|
|
245
245
|
resp.raise_for_status()
|
|
246
246
|
data = resp.json()
|
|
247
247
|
vectors = data.get("embeddings", [])
|
|
248
|
-
return [self.
|
|
248
|
+
return [self._normalize_checked(v) for v in vectors]
|
|
249
|
+
|
|
250
|
+
def _normalize_checked(self, vec: list[float]) -> list[float]:
|
|
251
|
+
"""Reject provider drift before a vector can enter any durable store."""
|
|
252
|
+
actual = len(vec)
|
|
253
|
+
if actual != self._dimension:
|
|
254
|
+
raise ValueError(
|
|
255
|
+
f"Ollama embedding dimension {actual} != expected {self._dimension}"
|
|
256
|
+
)
|
|
257
|
+
return self._normalize(vec)
|
|
249
258
|
|
|
250
259
|
@staticmethod
|
|
251
260
|
def _normalize(vec: list[float]) -> list[float]:
|
|
@@ -15,7 +15,7 @@ from __future__ import annotations
|
|
|
15
15
|
|
|
16
16
|
import threading
|
|
17
17
|
from contextlib import contextmanager
|
|
18
|
-
from typing import Iterator
|
|
18
|
+
from typing import Callable, Iterator
|
|
19
19
|
|
|
20
20
|
_condition = threading.Condition(threading.Lock())
|
|
21
21
|
_active = 0
|
|
@@ -42,19 +42,30 @@ def in_flight() -> int:
|
|
|
42
42
|
|
|
43
43
|
|
|
44
44
|
@contextmanager
|
|
45
|
-
def background_work(
|
|
45
|
+
def background_work(
|
|
46
|
+
*,
|
|
47
|
+
preempt_requested: Callable[[], bool] | None = None,
|
|
48
|
+
) -> Iterator[None]:
|
|
46
49
|
"""Mark best-effort work that must yield shared inference to recall.
|
|
47
50
|
|
|
48
51
|
The marker is thread-local because materialization, health probes, and
|
|
49
52
|
interactive handlers all share one resident engine and one embedder.
|
|
50
|
-
Nested callers restore the previous marker on exit.
|
|
53
|
+
Nested callers restore the previous marker on exit. A daemon-owned
|
|
54
|
+
materializer may also provide a preemption callback for a profile/runtime
|
|
55
|
+
reconfigure. Inference clients use that callback to cut a bounded
|
|
56
|
+
background request short instead of holding the transition drain lease.
|
|
51
57
|
"""
|
|
52
58
|
previous = bool(getattr(_work_context, "background", False))
|
|
59
|
+
previous_preempt = getattr(_work_context, "preempt_requested", None)
|
|
53
60
|
_work_context.background = True
|
|
61
|
+
_work_context.preempt_requested = (
|
|
62
|
+
preempt_requested if preempt_requested is not None else previous_preempt
|
|
63
|
+
)
|
|
54
64
|
try:
|
|
55
65
|
yield
|
|
56
66
|
finally:
|
|
57
67
|
_work_context.background = previous
|
|
68
|
+
_work_context.preempt_requested = previous_preempt
|
|
58
69
|
|
|
59
70
|
|
|
60
71
|
def is_background_work() -> bool:
|
|
@@ -62,6 +73,19 @@ def is_background_work() -> bool:
|
|
|
62
73
|
return bool(getattr(_work_context, "background", False))
|
|
63
74
|
|
|
64
75
|
|
|
76
|
+
def background_preempt_requested() -> bool:
|
|
77
|
+
"""Return whether daemon-owned background work must release its lease."""
|
|
78
|
+
callback = getattr(_work_context, "preempt_requested", None)
|
|
79
|
+
if not callable(callback):
|
|
80
|
+
return False
|
|
81
|
+
try:
|
|
82
|
+
return bool(callback())
|
|
83
|
+
except Exception:
|
|
84
|
+
# A status read is advisory. It must not crash a materializer or turn
|
|
85
|
+
# a valid recall into an ingestion failure.
|
|
86
|
+
return False
|
|
87
|
+
|
|
88
|
+
|
|
65
89
|
def wait_for_foreground_idle() -> None:
|
|
66
90
|
"""Block background inference while an interactive recall is active."""
|
|
67
91
|
if not is_background_work():
|
|
@@ -98,6 +98,7 @@ class RememberService:
|
|
|
98
98
|
dispatched = self._journal.mark_dispatched(
|
|
99
99
|
prepared.journal_id,
|
|
100
100
|
deadline=deadline,
|
|
101
|
+
known_prepared=prepared.state == "prepared",
|
|
101
102
|
)
|
|
102
103
|
if dispatched.original_receipt is not None:
|
|
103
104
|
return RememberReceipt.from_mapping(dispatched.original_receipt)
|
|
@@ -118,11 +119,19 @@ class RememberService:
|
|
|
118
119
|
if state in {"committed", "duplicate"}:
|
|
119
120
|
if not isinstance(receipt, Mapping):
|
|
120
121
|
raise AdmissionRejected("COMMAND_REJECTED: canonical result had no receipt")
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
122
|
+
try:
|
|
123
|
+
committed = self._journal.mark_committed(
|
|
124
|
+
prepared.journal_id,
|
|
125
|
+
receipt,
|
|
126
|
+
deadline=deadline,
|
|
127
|
+
)
|
|
128
|
+
except AdmissionJournalUnavailable:
|
|
129
|
+
# The canonical receipt is already durable and idempotent. Do
|
|
130
|
+
# not turn that committed write into an ambiguous client
|
|
131
|
+
# failure merely because the auxiliary journal exhausted the
|
|
132
|
+
# caller's remaining budget. The dispatched record is safe for
|
|
133
|
+
# retry/replay, which will recover the same immutable receipt.
|
|
134
|
+
return RememberReceipt.from_mapping(receipt)
|
|
126
135
|
return RememberReceipt.from_mapping(committed.original_receipt or receipt)
|
|
127
136
|
|
|
128
137
|
error_code = str(_result_value(result, "error_code") or "COMMAND_REJECTED")
|
|
@@ -34,6 +34,14 @@ logger = logging.getLogger(__name__)
|
|
|
34
34
|
_INIT_LANGEVIN_RADIUS = 0.05
|
|
35
35
|
|
|
36
36
|
|
|
37
|
+
def _reraise_materialization_deferral(exc: Exception) -> None:
|
|
38
|
+
"""Keep explicit runtime preemption out of best-effort fallbacks."""
|
|
39
|
+
from superlocalmemory.core.materialization_control import MaterializationDeferred
|
|
40
|
+
|
|
41
|
+
if isinstance(exc, MaterializationDeferred):
|
|
42
|
+
raise exc
|
|
43
|
+
|
|
44
|
+
|
|
37
45
|
def _ingestion_effect_id(operation_id: str, *parts: object) -> str:
|
|
38
46
|
"""Return a stable ID for a relational effect owned by one ingestion."""
|
|
39
47
|
if not operation_id:
|
|
@@ -247,6 +255,7 @@ def _upsert_fact_vectors(fact, profile_id, ann_index, vector_store, embedder=Non
|
|
|
247
255
|
try:
|
|
248
256
|
fact.embedding = embedder.embed(fact.content)
|
|
249
257
|
except Exception as _emb_exc: # pragma: no cover - defensive
|
|
258
|
+
_reraise_materialization_deferral(_emb_exc)
|
|
250
259
|
logger.debug("on-demand embed failed for %s: %s", fact.fact_id, _emb_exc)
|
|
251
260
|
return
|
|
252
261
|
if not getattr(fact, "embedding", None):
|
|
@@ -429,6 +438,7 @@ def run_store(
|
|
|
429
438
|
)
|
|
430
439
|
extraction_complete = facts is not None
|
|
431
440
|
except Exception as _extract_exc:
|
|
441
|
+
_reraise_materialization_deferral(_extract_exc)
|
|
432
442
|
# P0-1 (remember-write-04): an extractor EXCEPTION (transient LLM/embed
|
|
433
443
|
# backend error) must NOT orphan the already-committed memory. The None
|
|
434
444
|
# guard below only handled a None *return*, not a raise. Treat a raise
|
|
@@ -26,7 +26,6 @@ from __future__ import annotations
|
|
|
26
26
|
import hashlib
|
|
27
27
|
import os
|
|
28
28
|
import sqlite3
|
|
29
|
-
import sys
|
|
30
29
|
from dataclasses import dataclass
|
|
31
30
|
from datetime import datetime, timezone
|
|
32
31
|
from pathlib import Path
|
|
@@ -76,8 +75,11 @@ class Adapter(Protocol):
|
|
|
76
75
|
|
|
77
76
|
def path_sha256(path: Path) -> str:
|
|
78
77
|
"""SHA-256 of the absolute path string, full 64-hex (never truncated)."""
|
|
79
|
-
|
|
80
|
-
|
|
78
|
+
# The identity must not depend on whether the target exists. On Windows,
|
|
79
|
+
# Path.resolve() can normalize an existing path differently from the same
|
|
80
|
+
# not-yet-created path, changing the sync-log key after the first write.
|
|
81
|
+
canonical = os.path.normcase(os.path.abspath(os.fspath(path)))
|
|
82
|
+
return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
|
|
81
83
|
|
|
82
84
|
|
|
83
85
|
def _now_iso() -> str:
|
|
@@ -252,6 +254,11 @@ def atomic_write(
|
|
|
252
254
|
flags = os.O_WRONLY | os.O_CREAT | os.O_TRUNC
|
|
253
255
|
if hasattr(os, "O_NOFOLLOW") and _is_posix():
|
|
254
256
|
flags |= os.O_NOFOLLOW # SEC — POSIX refuses symlinks
|
|
257
|
+
if hasattr(os, "O_BINARY") and not _is_posix():
|
|
258
|
+
# Windows file descriptors default to text mode, which rewrites LF
|
|
259
|
+
# bytes as CRLF. The sync log hashes the caller's original bytes, so
|
|
260
|
+
# text-mode conversion makes an unchanged file look modified forever.
|
|
261
|
+
flags |= os.O_BINARY
|
|
255
262
|
|
|
256
263
|
mode = posix_mode if _is_posix() else windows_mode
|
|
257
264
|
fd = os.open(str(tmp), flags, mode)
|
|
@@ -105,6 +105,7 @@ def register_core_tools(server, get_engine: Callable) -> None:
|
|
|
105
105
|
# recall window so a parallel/next agent finds memories saved seconds ago.
|
|
106
106
|
# Falls back to the capability-owned worker only if the daemon is
|
|
107
107
|
# unreachable. Raw pending.db writes are legacy replay input only.
|
|
108
|
+
daemon_owned = False
|
|
108
109
|
try:
|
|
109
110
|
import asyncio as _asyncio
|
|
110
111
|
from superlocalmemory.cli.daemon import daemon_request, is_daemon_running
|
|
@@ -159,6 +160,15 @@ def register_core_tools(server, get_engine: Callable) -> None:
|
|
|
159
160
|
}
|
|
160
161
|
except Exception as dexc:
|
|
161
162
|
logger.debug("MCP remember via daemon failed, pending fallback: %s", dexc)
|
|
163
|
+
if daemon_owned:
|
|
164
|
+
return {
|
|
165
|
+
"success": False,
|
|
166
|
+
"code": "DAEMON_UNAVAILABLE",
|
|
167
|
+
"retryable": True,
|
|
168
|
+
"error": (
|
|
169
|
+
"DAEMON_UNAVAILABLE: owned daemon is unavailable; retry later."
|
|
170
|
+
),
|
|
171
|
+
}
|
|
162
172
|
|
|
163
173
|
try:
|
|
164
174
|
import asyncio as _asyncio
|
|
@@ -174,11 +184,12 @@ def register_core_tools(server, get_engine: Callable) -> None:
|
|
|
174
184
|
or "mcp:" + hashlib.sha256(content.encode("utf-8")).hexdigest()
|
|
175
185
|
),
|
|
176
186
|
}
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
worker_meta
|
|
181
|
-
|
|
187
|
+
|
|
188
|
+
def _store_via_daemon_pool():
|
|
189
|
+
pool = choose_pool()
|
|
190
|
+
return pool.store(content, worker_meta)
|
|
191
|
+
|
|
192
|
+
stored = await _asyncio.to_thread(_store_via_daemon_pool)
|
|
182
193
|
if not isinstance(stored, dict) or not stored.get("ok"):
|
|
183
194
|
if isinstance(stored, dict) and stored.get("code") == "DAEMON_UNAVAILABLE":
|
|
184
195
|
return {
|
|
@@ -282,7 +293,6 @@ def register_core_tools(server, get_engine: Callable) -> None:
|
|
|
282
293
|
import asyncio
|
|
283
294
|
try:
|
|
284
295
|
from superlocalmemory.mcp._daemon_proxy import choose_pool
|
|
285
|
-
pool = choose_pool()
|
|
286
296
|
# S9-DASH-10: priority for session_id, so engagement
|
|
287
297
|
# signals land on the right pending_outcome:
|
|
288
298
|
# 1. Explicit ``session_id`` tool-call argument.
|
|
@@ -322,15 +332,25 @@ def register_core_tools(server, get_engine: Callable) -> None:
|
|
|
322
332
|
pass
|
|
323
333
|
if not effective_sid:
|
|
324
334
|
effective_sid = f"mcp:{agent_id}"
|
|
325
|
-
#
|
|
335
|
+
# Resolve the daemon proxy inside the worker too. ``choose_pool``
|
|
336
|
+
# verifies daemon ownership through a synchronous /health request;
|
|
337
|
+
# when this tool is served by the daemon's mounted HTTP MCP app,
|
|
338
|
+
# resolving it on Uvicorn's event-loop thread makes that loop wait
|
|
339
|
+
# on its own health response forever. Stdio did not exhibit this
|
|
340
|
+
# because its MCP process is external to the daemon.
|
|
341
|
+
#
|
|
326
342
|
# V3.4.26: WorkerPool now concurrent — parallel calls no longer
|
|
327
343
|
# block behind a single threading.Lock. See worker_pool.py.
|
|
344
|
+
def _recall_via_daemon_pool():
|
|
345
|
+
pool = choose_pool()
|
|
346
|
+
return pool.recall(
|
|
347
|
+
query, limit=limit, session_id=effective_sid,
|
|
348
|
+
fast=fast, include_global=include_global,
|
|
349
|
+
include_shared=include_shared, window=window or None,
|
|
350
|
+
)
|
|
351
|
+
|
|
328
352
|
result = await asyncio.to_thread(
|
|
329
|
-
|
|
330
|
-
fast=fast,
|
|
331
|
-
include_global=include_global,
|
|
332
|
-
include_shared=include_shared,
|
|
333
|
-
window=window or None,
|
|
353
|
+
_recall_via_daemon_pool,
|
|
334
354
|
)
|
|
335
355
|
if result.get("ok"):
|
|
336
356
|
return {
|
|
@@ -74,7 +74,7 @@ def _windows_owner_dacl(
|
|
|
74
74
|
win32api: Any,
|
|
75
75
|
win32con: Any,
|
|
76
76
|
win32security: Any,
|
|
77
|
-
) -> Any:
|
|
77
|
+
) -> tuple[Any, Any]:
|
|
78
78
|
"""Build one protected owner-only DACL for a Windows capture file."""
|
|
79
79
|
import ntsecuritycon
|
|
80
80
|
|
|
@@ -95,7 +95,53 @@ def _windows_owner_dacl(
|
|
|
95
95
|
ntsecuritycon.FILE_ALL_ACCESS,
|
|
96
96
|
owner_sid,
|
|
97
97
|
)
|
|
98
|
-
return dacl
|
|
98
|
+
return owner_sid, dacl
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def _windows_dacl_is_owner_only(
|
|
102
|
+
security_descriptor: Any,
|
|
103
|
+
owner_sid: Any,
|
|
104
|
+
ntsecuritycon: Any,
|
|
105
|
+
win32security: Any,
|
|
106
|
+
) -> bool:
|
|
107
|
+
"""Return whether a live descriptor already matches the capture policy."""
|
|
108
|
+
def _same_sid(left: Any, right: Any) -> bool:
|
|
109
|
+
try:
|
|
110
|
+
return (
|
|
111
|
+
win32security.ConvertSidToStringSid(left)
|
|
112
|
+
== win32security.ConvertSidToStringSid(right)
|
|
113
|
+
)
|
|
114
|
+
except Exception:
|
|
115
|
+
return False
|
|
116
|
+
|
|
117
|
+
control, _revision = security_descriptor.GetSecurityDescriptorControl()
|
|
118
|
+
if not control & win32security.SE_DACL_PROTECTED:
|
|
119
|
+
return False
|
|
120
|
+
|
|
121
|
+
descriptor_owner = security_descriptor.GetSecurityDescriptorOwner()
|
|
122
|
+
if descriptor_owner is None or not _same_sid(
|
|
123
|
+
descriptor_owner,
|
|
124
|
+
owner_sid,
|
|
125
|
+
):
|
|
126
|
+
return False
|
|
127
|
+
|
|
128
|
+
dacl = security_descriptor.GetSecurityDescriptorDacl()
|
|
129
|
+
if dacl is None or dacl.GetAceCount() != 1:
|
|
130
|
+
return False
|
|
131
|
+
|
|
132
|
+
ace = dacl.GetAce(0)
|
|
133
|
+
if not isinstance(ace, tuple) or len(ace) != 3:
|
|
134
|
+
return False
|
|
135
|
+
ace_header, access_mask, ace_sid = ace
|
|
136
|
+
if (
|
|
137
|
+
not isinstance(ace_header, tuple)
|
|
138
|
+
or not ace_header
|
|
139
|
+
or ace_header[0] != win32security.ACCESS_ALLOWED_ACE_TYPE
|
|
140
|
+
):
|
|
141
|
+
return False
|
|
142
|
+
if access_mask & ntsecuritycon.FILE_ALL_ACCESS != ntsecuritycon.FILE_ALL_ACCESS:
|
|
143
|
+
return False
|
|
144
|
+
return _same_sid(ace_sid, owner_sid)
|
|
99
145
|
|
|
100
146
|
|
|
101
147
|
def _open_windows_capture_append(path: Path) -> int:
|
|
@@ -117,7 +163,11 @@ def _open_windows_capture_append(path: Path) -> int:
|
|
|
117
163
|
# ever inheriting a broader parent DACL. For an existing file Windows
|
|
118
164
|
# ignores this descriptor; _enforce_owner_only_permissions replaces
|
|
119
165
|
# that DACL through the same WRITE_DAC-capable handle before writing.
|
|
120
|
-
dacl = _windows_owner_dacl(
|
|
166
|
+
owner_sid, dacl = _windows_owner_dacl(
|
|
167
|
+
win32api,
|
|
168
|
+
win32con,
|
|
169
|
+
win32security,
|
|
170
|
+
)
|
|
121
171
|
security_attributes = win32security.SECURITY_ATTRIBUTES()
|
|
122
172
|
security_attributes.bInheritHandle = False
|
|
123
173
|
security_attributes.SECURITY_DESCRIPTOR.SetSecurityDescriptorDacl(
|
|
@@ -129,21 +179,57 @@ def _open_windows_capture_append(path: Path) -> int:
|
|
|
129
179
|
win32security.SE_DACL_PROTECTED,
|
|
130
180
|
win32security.SE_DACL_PROTECTED,
|
|
131
181
|
)
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
182
|
+
desired_access = (
|
|
183
|
+
ntsecuritycon.FILE_APPEND_DATA
|
|
184
|
+
| ntsecuritycon.WRITE_DAC
|
|
185
|
+
| ntsecuritycon.READ_CONTROL
|
|
186
|
+
)
|
|
187
|
+
share_mode = (
|
|
135
188
|
win32con.FILE_SHARE_READ
|
|
136
189
|
| win32con.FILE_SHARE_WRITE
|
|
137
|
-
| win32con.FILE_SHARE_DELETE
|
|
138
|
-
|
|
139
|
-
|
|
190
|
+
| win32con.FILE_SHARE_DELETE
|
|
191
|
+
)
|
|
192
|
+
file_flags = (
|
|
140
193
|
win32con.FILE_ATTRIBUTE_NORMAL
|
|
141
194
|
# pywin32 does not export this SDK constant from win32con on
|
|
142
195
|
# every supported Python build. Keep the Microsoft-defined value
|
|
143
196
|
# as a named fallback rather than silently following a reparse.
|
|
144
|
-
| getattr(win32con, "FILE_FLAG_OPEN_REPARSE_POINT", 0x00200000)
|
|
145
|
-
None,
|
|
197
|
+
| getattr(win32con, "FILE_FLAG_OPEN_REPARSE_POINT", 0x00200000)
|
|
146
198
|
)
|
|
199
|
+
try:
|
|
200
|
+
# CREATE_NEW is the only race-safe proof that the protected
|
|
201
|
+
# SECURITY_ATTRIBUTES were applied to this exact file. OPEN_ALWAYS
|
|
202
|
+
# would require trusting GetLastError after the pywin32 wrapper has
|
|
203
|
+
# returned, which is not a documented preservation boundary.
|
|
204
|
+
handle = win32file.CreateFile(
|
|
205
|
+
os.fspath(path),
|
|
206
|
+
desired_access,
|
|
207
|
+
share_mode,
|
|
208
|
+
security_attributes,
|
|
209
|
+
getattr(win32con, "CREATE_NEW", 1),
|
|
210
|
+
file_flags,
|
|
211
|
+
None,
|
|
212
|
+
)
|
|
213
|
+
created_new = True
|
|
214
|
+
except Exception as exc:
|
|
215
|
+
winerror = getattr(exc, "winerror", None)
|
|
216
|
+
if winerror is None and exc.args:
|
|
217
|
+
winerror = exc.args[0]
|
|
218
|
+
if winerror != getattr(win32con, "ERROR_FILE_EXISTS", 80):
|
|
219
|
+
raise
|
|
220
|
+
# The creation descriptor is ignored for existing files. Reopen
|
|
221
|
+
# the exact path without following a reparse point, then replace
|
|
222
|
+
# its DACL through this WRITE_DAC-capable handle before appending.
|
|
223
|
+
handle = win32file.CreateFile(
|
|
224
|
+
os.fspath(path),
|
|
225
|
+
desired_access,
|
|
226
|
+
share_mode,
|
|
227
|
+
None,
|
|
228
|
+
getattr(win32con, "OPEN_EXISTING", 3),
|
|
229
|
+
file_flags,
|
|
230
|
+
None,
|
|
231
|
+
)
|
|
232
|
+
created_new = False
|
|
147
233
|
file_info = win32file.GetFileInformationByHandle(handle)
|
|
148
234
|
if file_info[0] & win32con.FILE_ATTRIBUTE_REPARSE_POINT:
|
|
149
235
|
raise OSError(
|
|
@@ -151,25 +237,57 @@ def _open_windows_capture_append(path: Path) -> int:
|
|
|
151
237
|
"capture path is a Windows reparse point",
|
|
152
238
|
path,
|
|
153
239
|
)
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
win32security.
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
240
|
+
if not created_new:
|
|
241
|
+
try:
|
|
242
|
+
query_flags = (
|
|
243
|
+
win32security.OWNER_SECURITY_INFORMATION
|
|
244
|
+
| win32security.DACL_SECURITY_INFORMATION
|
|
245
|
+
)
|
|
246
|
+
descriptor = win32security.GetSecurityInfo(
|
|
247
|
+
handle,
|
|
248
|
+
win32security.SE_FILE_OBJECT,
|
|
249
|
+
query_flags,
|
|
250
|
+
)
|
|
251
|
+
if not _windows_dacl_is_owner_only(
|
|
252
|
+
descriptor,
|
|
253
|
+
owner_sid,
|
|
254
|
+
ntsecuritycon,
|
|
255
|
+
win32security,
|
|
256
|
+
):
|
|
257
|
+
# Existing files ignore the creation security descriptor,
|
|
258
|
+
# so repair an unsafe DACL while this is still the original
|
|
259
|
+
# CreateFile handle carrying WRITE_DAC. Avoid rewriting an
|
|
260
|
+
# already-protected DACL: Windows may correctly deny that
|
|
261
|
+
# redundant mutation even though append access is allowed.
|
|
262
|
+
win32security.SetSecurityInfo(
|
|
263
|
+
handle,
|
|
264
|
+
win32security.SE_FILE_OBJECT,
|
|
265
|
+
win32security.DACL_SECURITY_INFORMATION
|
|
266
|
+
| win32security.PROTECTED_DACL_SECURITY_INFORMATION,
|
|
267
|
+
None,
|
|
268
|
+
None,
|
|
269
|
+
dacl,
|
|
270
|
+
None,
|
|
271
|
+
)
|
|
272
|
+
descriptor = win32security.GetSecurityInfo(
|
|
273
|
+
handle,
|
|
274
|
+
win32security.SE_FILE_OBJECT,
|
|
275
|
+
query_flags,
|
|
276
|
+
)
|
|
277
|
+
if not _windows_dacl_is_owner_only(
|
|
278
|
+
descriptor,
|
|
279
|
+
owner_sid,
|
|
280
|
+
ntsecuritycon,
|
|
281
|
+
win32security,
|
|
282
|
+
):
|
|
283
|
+
raise OSError(
|
|
284
|
+
"Windows capture ACL verification failed after repair"
|
|
285
|
+
)
|
|
286
|
+
except Exception as exc:
|
|
287
|
+
raise OSError(
|
|
288
|
+
"Windows capture ACL could not be enforced "
|
|
289
|
+
f"({type(exc).__name__}: {exc})"
|
|
290
|
+
) from exc
|
|
173
291
|
|
|
174
292
|
# Transfer the native handle to Python's CRT descriptor exactly once.
|
|
175
293
|
raw_handle = handle.Detach()
|
|
@@ -212,8 +212,12 @@ class CacheDB:
|
|
|
212
212
|
try:
|
|
213
213
|
import sqlite3 as _sq
|
|
214
214
|
test_conn = _sq.connect(str(self._db_path))
|
|
215
|
-
|
|
216
|
-
|
|
215
|
+
try:
|
|
216
|
+
test_conn.execute("PRAGMA schema_version")
|
|
217
|
+
finally:
|
|
218
|
+
# Windows will not rename an open SQLite file. Always
|
|
219
|
+
# release the probe before corrupt-file recovery runs.
|
|
220
|
+
test_conn.close()
|
|
217
221
|
except Exception as exc:
|
|
218
222
|
corrupt_sidecar = self._db_path.with_suffix(
|
|
219
223
|
self._db_path.suffix + ".corrupt"
|
|
@@ -1244,6 +1244,7 @@ async def lifespan(application: FastAPI):
|
|
|
1244
1244
|
engine = None
|
|
1245
1245
|
config = None
|
|
1246
1246
|
canonical_remember_runtime = None
|
|
1247
|
+
profile_runtime = None
|
|
1247
1248
|
|
|
1248
1249
|
# The local dashboard obtains its short-lived browser credential from
|
|
1249
1250
|
# ``/internal/token`` before its first write or token-gated read. A
|
|
@@ -1916,11 +1917,17 @@ async def lifespan(application: FastAPI):
|
|
|
1916
1917
|
|
|
1917
1918
|
except Exception:
|
|
1918
1919
|
logger.exception("Engine init failed") # auto-includes traceback
|
|
1919
|
-
_release_canonical_remember_runtime(
|
|
1920
|
+
writer_released = _release_canonical_remember_runtime(
|
|
1920
1921
|
application, canonical_remember_runtime,
|
|
1921
1922
|
)
|
|
1922
1923
|
application.state.engine = None
|
|
1923
1924
|
application.state.config = None
|
|
1925
|
+
if engine is not None and writer_released:
|
|
1926
|
+
try:
|
|
1927
|
+
engine.close()
|
|
1928
|
+
except Exception:
|
|
1929
|
+
logger.debug("partially initialized engine cleanup failed", exc_info=True)
|
|
1930
|
+
raise
|
|
1924
1931
|
|
|
1925
1932
|
application.state.observe_buffer = _observe_buffer
|
|
1926
1933
|
|
|
@@ -4060,7 +4067,9 @@ def _run_materializer_operation(
|
|
|
4060
4067
|
"resident engine does not match pending profile"
|
|
4061
4068
|
)
|
|
4062
4069
|
from superlocalmemory.core.recall_gate import background_work
|
|
4063
|
-
with background_work(
|
|
4070
|
+
with background_work(
|
|
4071
|
+
preempt_requested=lambda: bool(runtime is not None and runtime.transitioning),
|
|
4072
|
+
):
|
|
4064
4073
|
return operation(engine)
|
|
4065
4074
|
|
|
4066
4075
|
|
|
@@ -74,6 +74,16 @@ def _load_or_create_key(path: Path) -> bytes:
|
|
|
74
74
|
)
|
|
75
75
|
except FileExistsError:
|
|
76
76
|
fd = -1
|
|
77
|
+
except OSError as exc:
|
|
78
|
+
try:
|
|
79
|
+
info = path.lstat()
|
|
80
|
+
except OSError:
|
|
81
|
+
raise AdmissionKeyError("admission key cannot be created") from exc
|
|
82
|
+
if stat.S_ISLNK(info.st_mode) or not stat.S_ISREG(info.st_mode):
|
|
83
|
+
raise AdmissionKeyError(
|
|
84
|
+
"admission key path must be a regular file"
|
|
85
|
+
) from exc
|
|
86
|
+
raise AdmissionKeyError("admission key cannot be created") from exc
|
|
77
87
|
else:
|
|
78
88
|
try:
|
|
79
89
|
key = os.urandom(_KEY_BYTES)
|