superlocalmemory 3.7.2 → 3.7.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +18 -0
- package/README.md +17 -8
- package/package.json +1 -1
- package/plugin/.claude-plugin/plugin.json +1 -1
- package/plugin/requirements.txt +1 -1
- package/plugin-src/manifest.json +1 -1
- package/plugin-src/requirements.txt +1 -1
- package/pyproject.toml +1 -1
- package/src/superlocalmemory/__init__.py +1 -1
- package/src/superlocalmemory/cli/main.py +2 -2
- package/src/superlocalmemory/cli/scale_engine_cmd.py +13 -1
- package/src/superlocalmemory/core/backend_orchestrator.py +15 -0
- package/src/superlocalmemory/core/scale_engine.py +535 -38
- package/src/superlocalmemory/graph/cozo_backend.py +12 -17
- package/src/superlocalmemory/hooks/claude_code_hooks.py +2 -1
- package/src/superlocalmemory/hooks/context_payload.py +2 -1
- package/src/superlocalmemory/mcp/http_transport.py +2 -1
- package/src/superlocalmemory/optimize/proxy/server.py +2 -1
- package/src/superlocalmemory/server/routes/brain.py +2 -1
- package/src/superlocalmemory/storage/logical_edges.py +45 -0
package/CHANGELOG.md
CHANGED
|
@@ -5,6 +5,24 @@ All notable changes to SuperLocalMemory V3 will be documented in this file.
|
|
|
5
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
|
6
6
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
7
|
|
|
8
|
+
## [3.7.4] - 2026-07-18 — Scale Engine projection-parity release
|
|
9
|
+
|
|
10
|
+
### Fixed
|
|
11
|
+
|
|
12
|
+
- Aligned CozoDB projection parity with SLM's logical graph-edge identity while retaining the strongest relationship weight.
|
|
13
|
+
- Preserved canonical SQLite history when normalizing legacy repeated graph rows into derived scale projections.
|
|
14
|
+
- Kept rejected projection manifests inspectable, retired replaceable stage payloads, and allowed a corrected, explicitly confirmed adoption retry.
|
|
15
|
+
- Removed stale CozoDB graph edges when a fact is deleted and kept shadow-error telemetry side-effect free.
|
|
16
|
+
|
|
17
|
+
## [3.7.3] - 2026-07-18 — Scale Engine integrity release
|
|
18
|
+
|
|
19
|
+
### Fixed
|
|
20
|
+
|
|
21
|
+
- Added explicit, structurally verified adoption for pre-v3.7 CozoDB/LanceDB projections.
|
|
22
|
+
- Added durable promotion recovery, lifecycle serialization, and a final canonical-source consistency fence.
|
|
23
|
+
- Made Scale Engine status distinguish installed projection paths, lifecycle state, and live daemon backend health.
|
|
24
|
+
- Unified shipped runtime identity metadata at the release version.
|
|
25
|
+
|
|
8
26
|
## [3.7.2] - 2026-07-16 — Reliability release
|
|
9
27
|
|
|
10
28
|
### Fixed
|
package/README.md
CHANGED
|
@@ -1,11 +1,14 @@
|
|
|
1
1
|
<p align="center">
|
|
2
|
-
<
|
|
2
|
+
<picture>
|
|
3
|
+
<source media="(prefers-color-scheme: dark)" srcset="assets/branding/slm-wordmark-dark.svg">
|
|
4
|
+
<img src="assets/branding/slm-wordmark-light.svg" alt="SuperLocalMemory" width="390">
|
|
5
|
+
</picture>
|
|
3
6
|
</p>
|
|
4
7
|
|
|
5
|
-
<h1 align="center">SuperLocalMemory V3.7.
|
|
8
|
+
<h1 align="center">SuperLocalMemory V3.7.4</h1>
|
|
6
9
|
<p align="center"><strong>Cache. Compress. Remember. Three surfaces — proxy, MCP tools, or skill. Every setup covered.</strong><br/>
|
|
7
10
|
<em>Local-first agent memory with explicit operating modes, auditable retrieval, and optional Optimize tools.</em></p>
|
|
8
|
-
<p align="center"><code>v3.7.
|
|
11
|
+
<p align="center"><code>v3.7.4</code> — <strong>Scale Engine projection parity: normalized logical graph edges, durable promotion recovery, and honest runtime status.</strong><br/>
|
|
9
12
|
Proxy: <code>slm wrap claude</code> · MCP: add <code>slm_compress</code> to your config · Skill: zero-config</p>
|
|
10
13
|
<p align="center"><strong>3 public research preprints</strong> (arXiv + Zenodo archives) · <a href="https://arxiv.org/abs/2603.02240">arXiv:2603.02240</a> · <a href="https://arxiv.org/abs/2603.14588">arXiv:2603.14588</a> · <a href="https://arxiv.org/abs/2604.04514">arXiv:2604.04514</a></p>
|
|
11
14
|
|
|
@@ -108,7 +111,7 @@ health surfaces expose the stages actually completed by the installed runtime.
|
|
|
108
111
|
| **Retrieval and recall** | Semantic, lexical, temporal, Hopfield and spreading-activation candidate channels; RRF fusion, optional reranking and graph score enhancement | Healthy channels participate; response provenance states the evidence used. |
|
|
109
112
|
| **Brain and learning** | Behavioral patterns, feedback/outcome records, rewards, consolidation, LightGBM-related ranking components, soft prompts, and guarded skill-evolution workflows | Learning is evidence-driven; it does not claim autonomous correctness or guaranteed improvement. |
|
|
110
113
|
| **Knowledge graph and entities** | Canonical entities, aliases, entity profiles, graph edges, scenes, timelines, explorer and graph APIs | Stored/derived graph data is evidence, not an instruction authority. |
|
|
111
|
-
| **Scale Engine** | SQLite + sqlite-vec are canonical. CozoDB graph and LanceDB vector projections are
|
|
114
|
+
| **Scale Engine** | SQLite + sqlite-vec are canonical. CozoDB graph and LanceDB vector projections are managed with prepare → verify → promote → rollback; a structurally detected pre-v3.7 projection can be explicitly adopted. | Promotion is parity-gated and crash-recoverable. Legacy adoption preserves the prior projection as a rollback backup; repeated physical edge rows normalize to one logical edge with the strongest weight. |
|
|
112
115
|
| **Optimize** | Exact cache, tagged invalidation, safe compression, opt-in aggressive prose compression, CCR originals, proxy/MCP/skill surfaces | Only proxy intercepts a primary provider turn. MCP/skill cache results explicitly routed through SLM. |
|
|
113
116
|
| **Mesh** | Authenticated peer messages, inbox/outbox, locks, offline queue, optional discovery and mesh MCP tools | Mesh is coordination, not automatic replicated memory or conflict resolution. |
|
|
114
117
|
| **Governance and operations** | Provenance, audit/retention/policy surfaces, export/erasure controls, diagnostics, health, backups and daemon lifecycle | These are engineering controls, not a legal certification. |
|
|
@@ -227,10 +230,16 @@ optional reranking, and entity-graph score enhancement. The entity graph does
|
|
|
227
230
|
not create an independent candidate in the current implementation. Core memory
|
|
228
231
|
is SQLite-backed. SQLite and sqlite-vec remain the canonical source of truth.
|
|
229
232
|
The packaged Scale Engine can maintain CozoDB graph and LanceDB vector
|
|
230
|
-
projections, and it remains outside active retrieval paths until
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
233
|
+
projections, and it remains outside active retrieval paths until a staged
|
|
234
|
+
parity witness proves it matches the canonical store. New installations remain
|
|
235
|
+
on Local Core. During upgrade, `slm db scale status` can identify a positive
|
|
236
|
+
pre-v3.7 layout candidate; the operator confirms it with `slm db scale adopt`.
|
|
237
|
+
SLM then rebuilds from canonical SQLite, verifies it, and promotes it with a
|
|
238
|
+
durable recovery journal while retaining the prior directories as a rollback
|
|
239
|
+
backup. `adopt` reports `restart_required: true`; run `slm restart` before
|
|
240
|
+
checking daemon health. If proof fails, recall remains on SQLite and status
|
|
241
|
+
retains the rejected manifest for inspection, retires its replaceable derived
|
|
242
|
+
payload, and allows a corrected retry.
|
|
234
243
|
|
|
235
244
|
Canonical ingestion is a durable state machine: `raw → queryable → enriching →
|
|
236
245
|
complete`, with `failed` retaining raw evidence, error details, attempt count,
|
package/package.json
CHANGED
package/plugin/requirements.txt
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
superlocalmemory==3.7.
|
|
1
|
+
superlocalmemory==3.7.4
|
package/plugin-src/manifest.json
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
superlocalmemory==3.7.
|
|
1
|
+
superlocalmemory==3.7.4
|
package/pyproject.toml
CHANGED
|
@@ -32,7 +32,7 @@ if "OMP_NUM_THREADS" not in os.environ:
|
|
|
32
32
|
os.environ["OMP_NUM_THREADS"] = "2"
|
|
33
33
|
# ---------------------------------------------------------------------------
|
|
34
34
|
|
|
35
|
-
__version__ = "3.7.
|
|
35
|
+
__version__ = "3.7.4"
|
|
36
36
|
|
|
37
37
|
_REQUIRED_VERSIONS = {
|
|
38
38
|
"sentence_transformers": "5.3.0",
|
|
@@ -270,8 +270,8 @@ def main() -> None:
|
|
|
270
270
|
)
|
|
271
271
|
db_scale_p.add_argument(
|
|
272
272
|
"scale_action",
|
|
273
|
-
choices=("status", "prepare", "verify", "promote", "rollback"),
|
|
274
|
-
help="Lifecycle action.
|
|
273
|
+
choices=("status", "adopt", "prepare", "verify", "promote", "rollback"),
|
|
274
|
+
help="Lifecycle action. Adopt confirms a detected legacy projection; prepare and verify never mutate active paths.",
|
|
275
275
|
)
|
|
276
276
|
db_scale_p.add_argument("--stage-id", help="Stage identifier required by verify/promote")
|
|
277
277
|
db_scale_p.add_argument("--backup-id", help="Backup identifier required by rollback")
|
|
@@ -9,11 +9,23 @@ def cmd_db_scale(args: Namespace) -> int:
|
|
|
9
9
|
from superlocalmemory.core.config import SLMConfig
|
|
10
10
|
from superlocalmemory.core.scale_engine import ScaleEngineError, ScaleEngineManager
|
|
11
11
|
|
|
12
|
-
manager = ScaleEngineManager(SLMConfig.load())
|
|
13
12
|
action = args.scale_action
|
|
13
|
+
# Scale Engine projections are currently canonical-default-profile data.
|
|
14
|
+
# Adoption must not fail merely because the operator is working in another
|
|
15
|
+
# profile, and it must not rewrite that selected profile.
|
|
16
|
+
manager = ScaleEngineManager(
|
|
17
|
+
SLMConfig.load(), profile_id="default" if action == "adopt" else None
|
|
18
|
+
)
|
|
14
19
|
try:
|
|
15
20
|
if action == "status":
|
|
16
21
|
result = manager.status()
|
|
22
|
+
elif action == "adopt":
|
|
23
|
+
result = manager.adopt_legacy_projection()
|
|
24
|
+
if result is None:
|
|
25
|
+
raise ScaleEngineError(
|
|
26
|
+
"no confirmed legacy projection candidate; see `slm db scale status`"
|
|
27
|
+
)
|
|
28
|
+
result = {**result, "restart_required": True}
|
|
17
29
|
elif action == "prepare":
|
|
18
30
|
result = manager.prepare()
|
|
19
31
|
elif action == "verify":
|
|
@@ -97,6 +97,8 @@ class BackendOrchestrator:
|
|
|
97
97
|
except Exception as exc:
|
|
98
98
|
logger.warning("Initial rebalance failed (non-fatal): %s", exc)
|
|
99
99
|
|
|
100
|
+
self._recover_interrupted_scale_promotion()
|
|
101
|
+
|
|
100
102
|
# Backends may be installed with the product, but installing a wheel
|
|
101
103
|
# is not authorization to mutate an existing data root. Only a
|
|
102
104
|
# verified, explicit promotion may initialize and migrate projections.
|
|
@@ -134,6 +136,19 @@ class BackendOrchestrator:
|
|
|
134
136
|
"active" if self._cozo and self._cozo_status() == "active" else "off",
|
|
135
137
|
"active" if self._lancedb and self._lancedb_status() == "active" else "off")
|
|
136
138
|
|
|
139
|
+
def _recover_interrupted_scale_promotion(self) -> None:
|
|
140
|
+
"""Repair an interrupted promotion; never auto-mutate a legacy root."""
|
|
141
|
+
try:
|
|
142
|
+
from superlocalmemory.core.scale_engine import ScaleEngineManager
|
|
143
|
+
|
|
144
|
+
result = ScaleEngineManager(self._config, profile_id="default").recover_interrupted_promotion()
|
|
145
|
+
if result:
|
|
146
|
+
logger.warning("Scale Engine promotion recovery: %s", result)
|
|
147
|
+
except Exception as exc:
|
|
148
|
+
# A scale projection is derived data. Startup must keep serving
|
|
149
|
+
# canonical SQLite even if optional recovery itself is unhealthy.
|
|
150
|
+
logger.error("Scale Engine recovery requires repair; Local Core remains active: %s", exc)
|
|
151
|
+
|
|
137
152
|
# ------------------------------------------------------------------
|
|
138
153
|
# Incremental Sync (F-04: called from store_pipeline)
|
|
139
154
|
# ------------------------------------------------------------------
|
|
@@ -22,6 +22,8 @@ from datetime import UTC, datetime
|
|
|
22
22
|
from pathlib import Path
|
|
23
23
|
from typing import Any, Callable
|
|
24
24
|
|
|
25
|
+
from superlocalmemory.storage.logical_edges import count_logical_edges, iter_logical_edges
|
|
26
|
+
|
|
25
27
|
|
|
26
28
|
class ScaleEngineError(RuntimeError):
|
|
27
29
|
"""A scale projection cannot safely advance to its next lifecycle state."""
|
|
@@ -41,17 +43,20 @@ class ScaleEngineManager:
|
|
|
41
43
|
|
|
42
44
|
MANIFEST_NAME = "scale-engine.json"
|
|
43
45
|
SCHEMA_VERSION = 1
|
|
46
|
+
LIFECYCLE_LOCK = "scale-engine.lifecycle.lock"
|
|
47
|
+
PROMOTION_JOURNAL = "scale-engine.promotion.json"
|
|
44
48
|
|
|
45
49
|
def __init__(
|
|
46
50
|
self,
|
|
47
51
|
config: Any,
|
|
48
52
|
*,
|
|
49
53
|
backend_factory: Callable[[Path, Path], tuple[Any, Any]] | None = None,
|
|
54
|
+
profile_id: str | None = None,
|
|
50
55
|
) -> None:
|
|
51
56
|
self.config = config
|
|
52
57
|
self.data_dir = Path(getattr(config, "data_dir", None) or config.base_dir)
|
|
53
58
|
self.db_path = Path(getattr(config, "db_path", None) or self.data_dir / "memory.db")
|
|
54
|
-
self.profile_id = getattr(config, "active_profile", "default")
|
|
59
|
+
self.profile_id = profile_id or getattr(config, "active_profile", "default")
|
|
55
60
|
self._backend_factory = backend_factory or self._real_backend_factory
|
|
56
61
|
|
|
57
62
|
@property
|
|
@@ -66,6 +71,14 @@ class ScaleEngineManager:
|
|
|
66
71
|
def active_paths(self) -> tuple[Path, Path]:
|
|
67
72
|
return self.data_dir / "cozo", self.data_dir / "lance"
|
|
68
73
|
|
|
74
|
+
@property
|
|
75
|
+
def lifecycle_lock_path(self) -> Path:
|
|
76
|
+
return self.data_dir / self.LIFECYCLE_LOCK
|
|
77
|
+
|
|
78
|
+
@property
|
|
79
|
+
def promotion_journal_path(self) -> Path:
|
|
80
|
+
return self.data_dir / self.PROMOTION_JOURNAL
|
|
81
|
+
|
|
69
82
|
def status(self) -> dict[str, Any]:
|
|
70
83
|
manifests: list[dict[str, Any]] = []
|
|
71
84
|
if self.staging_root.exists():
|
|
@@ -74,21 +87,168 @@ class ScaleEngineManager:
|
|
|
74
87
|
manifests.append(json.loads(path.read_text()))
|
|
75
88
|
except (OSError, json.JSONDecodeError):
|
|
76
89
|
manifests.append({"stage_id": path.parent.name, "state": "corrupt"})
|
|
90
|
+
backups = (
|
|
91
|
+
sorted(p.name for p in self.backup_root.glob("*") if p.is_dir())
|
|
92
|
+
if self.backup_root.exists()
|
|
93
|
+
else []
|
|
94
|
+
)
|
|
95
|
+
paths_present = {
|
|
96
|
+
"cozo": self.active_paths[0].exists(),
|
|
97
|
+
"lance": self.active_paths[1].exists(),
|
|
98
|
+
}
|
|
99
|
+
state = getattr(self.config, "scale_engine_state", "local_core")
|
|
100
|
+
legacy_projection_candidate = (
|
|
101
|
+
state == "local_core"
|
|
102
|
+
and all(paths_present.values())
|
|
103
|
+
and not backups
|
|
104
|
+
and not self.promotion_journal_path.exists()
|
|
105
|
+
and self._has_legacy_projection_layout()
|
|
106
|
+
)
|
|
107
|
+
runtime = self._runtime_backend_status()
|
|
77
108
|
return {
|
|
78
|
-
"state":
|
|
79
|
-
|
|
109
|
+
"state": state,
|
|
110
|
+
# This command reads persisted state, not the live daemon. Never
|
|
111
|
+
# turn a last-known backend row into a present-tense routing claim.
|
|
112
|
+
"active": {"cozo": False, "lance": False},
|
|
113
|
+
"last_daemon_observation": runtime,
|
|
114
|
+
"paths_present": paths_present,
|
|
115
|
+
"retrieval_routing": (
|
|
116
|
+
"daemon_runtime_check_required" if state == "promoted" else "canonical_sqlite"
|
|
117
|
+
),
|
|
118
|
+
"legacy_projection_candidate": legacy_projection_candidate,
|
|
119
|
+
"legacy_candidate_requires_confirmation": legacy_projection_candidate,
|
|
120
|
+
"migration_repair_required": (
|
|
121
|
+
self.promotion_journal_path.exists()
|
|
122
|
+
or any(
|
|
123
|
+
manifest.get("state") == "promoted" and state != "promoted"
|
|
124
|
+
for manifest in manifests
|
|
125
|
+
)
|
|
126
|
+
or (
|
|
127
|
+
state == "promoted"
|
|
128
|
+
and any(
|
|
129
|
+
manifest.get("state") in {"prepared", "verified"}
|
|
130
|
+
for manifest in manifests
|
|
131
|
+
)
|
|
132
|
+
)
|
|
133
|
+
),
|
|
80
134
|
"stages": manifests,
|
|
81
|
-
"backups":
|
|
135
|
+
"backups": backups,
|
|
82
136
|
}
|
|
83
137
|
|
|
138
|
+
def adopt_legacy_projection(self) -> dict[str, Any] | None:
|
|
139
|
+
"""Safely adopt a v3.5-era projection into the staged lifecycle.
|
|
140
|
+
|
|
141
|
+
A legacy projection proves only that an older runtime created files.
|
|
142
|
+
It cannot establish parity with today's canonical SQLite database.
|
|
143
|
+
Rebuild a fresh stage, verify it while the canonical database is
|
|
144
|
+
stable, and promote it atomically; the legacy directories become the
|
|
145
|
+
explicit rollback copy.
|
|
146
|
+
"""
|
|
147
|
+
if not self.status()["legacy_projection_candidate"]:
|
|
148
|
+
return None
|
|
149
|
+
lock_path = self._acquire_lifecycle_lock()
|
|
150
|
+
prepared: dict[str, Any] | None = None
|
|
151
|
+
try:
|
|
152
|
+
# Re-check inside the lock. A concurrent command may have
|
|
153
|
+
# completed promotion while this caller waited to acquire it.
|
|
154
|
+
current_status = self.status()
|
|
155
|
+
if not current_status["legacy_projection_candidate"]:
|
|
156
|
+
return None
|
|
157
|
+
retry_payloads = [
|
|
158
|
+
manifest["stage_id"]
|
|
159
|
+
for manifest in current_status["stages"]
|
|
160
|
+
if manifest.get("state") in {"prepared", "verified"}
|
|
161
|
+
and manifest.get("stage_id")
|
|
162
|
+
]
|
|
163
|
+
prepared = self._prepare()
|
|
164
|
+
self._verify(prepared["stage_id"])
|
|
165
|
+
promoted = self._promote(prepared["stage_id"])
|
|
166
|
+
retired: list[str] = []
|
|
167
|
+
retirement_failures: dict[str, str] = {}
|
|
168
|
+
for stage_id in retry_payloads:
|
|
169
|
+
try:
|
|
170
|
+
self._retire_superseded_stage(stage_id)
|
|
171
|
+
retired.append(stage_id)
|
|
172
|
+
except Exception as cleanup_exc:
|
|
173
|
+
retirement_failures[stage_id] = str(cleanup_exc)
|
|
174
|
+
return {
|
|
175
|
+
**promoted,
|
|
176
|
+
"retired_stages": retired,
|
|
177
|
+
"retirement_failures": retirement_failures,
|
|
178
|
+
}
|
|
179
|
+
except Exception as exc:
|
|
180
|
+
# A failed adoption must leave the canonical path selected. The
|
|
181
|
+
# manifest is retained for inspection, but its replaceable Cozo
|
|
182
|
+
# and Lance payloads are retired so repeated retries cannot grow
|
|
183
|
+
# the data root without bound.
|
|
184
|
+
retirement_error: Exception | None = None
|
|
185
|
+
unresolved_promotion = self.promotion_journal_path.exists()
|
|
186
|
+
if prepared is not None and not unresolved_promotion:
|
|
187
|
+
try:
|
|
188
|
+
self._retire_rejected_stage(prepared["stage_id"], exc)
|
|
189
|
+
except Exception as cleanup_exc:
|
|
190
|
+
retirement_error = cleanup_exc
|
|
191
|
+
self.config.scale_engine_state = (
|
|
192
|
+
"verified" if unresolved_promotion else "local_core"
|
|
193
|
+
)
|
|
194
|
+
self.config.graph_backend = "auto"
|
|
195
|
+
self.config.vector_backend = "auto"
|
|
196
|
+
self._save_config()
|
|
197
|
+
if retirement_error is not None:
|
|
198
|
+
raise ScaleEngineError(
|
|
199
|
+
f"{exc}; rejected stage retirement failed: {retirement_error}"
|
|
200
|
+
) from exc
|
|
201
|
+
raise
|
|
202
|
+
finally:
|
|
203
|
+
self._release_lifecycle_lock(lock_path)
|
|
204
|
+
|
|
205
|
+
def _retire_rejected_stage(self, stage_id: str, error: Exception) -> None:
|
|
206
|
+
"""Keep rejection evidence while removing replaceable projection data."""
|
|
207
|
+
self._retire_stage_payload(
|
|
208
|
+
stage_id,
|
|
209
|
+
{
|
|
210
|
+
"state": "rejected",
|
|
211
|
+
"rejected_at": _utc_now(),
|
|
212
|
+
"failure": f"{type(error).__name__}: {error}",
|
|
213
|
+
},
|
|
214
|
+
)
|
|
215
|
+
|
|
216
|
+
def _retire_superseded_stage(self, stage_id: str) -> None:
|
|
217
|
+
"""Retain an old retry manifest after a newer projection is promoted."""
|
|
218
|
+
self._retire_stage_payload(
|
|
219
|
+
stage_id,
|
|
220
|
+
{"state": "superseded", "superseded_at": _utc_now()},
|
|
221
|
+
)
|
|
222
|
+
|
|
223
|
+
def _retire_stage_payload(
|
|
224
|
+
self, stage_id: str, manifest_updates: dict[str, Any]
|
|
225
|
+
) -> None:
|
|
226
|
+
"""Remove derived stage bytes while retaining its durable manifest."""
|
|
227
|
+
stage_dir, manifest = self._load_stage(stage_id)
|
|
228
|
+
for payload in (stage_dir / "cozo", stage_dir / "lance"):
|
|
229
|
+
if payload.exists():
|
|
230
|
+
shutil.rmtree(payload)
|
|
231
|
+
self._fsync_directory(stage_dir)
|
|
232
|
+
manifest.update(manifest_updates)
|
|
233
|
+
self._write_manifest(stage_dir, manifest)
|
|
234
|
+
|
|
84
235
|
def prepare(self) -> dict[str, Any]:
|
|
85
236
|
"""Build a new projection in a private staging directory."""
|
|
237
|
+
lock_path = self._acquire_lifecycle_lock()
|
|
238
|
+
try:
|
|
239
|
+
self._recover_interrupted_promotion()
|
|
240
|
+
return self._prepare()
|
|
241
|
+
finally:
|
|
242
|
+
self._release_lifecycle_lock(lock_path)
|
|
243
|
+
|
|
244
|
+
def _prepare(self) -> dict[str, Any]:
|
|
245
|
+
"""Build a new projection while the caller owns the lifecycle lock."""
|
|
86
246
|
self._require_default_profile()
|
|
87
247
|
self._require_canonical_db()
|
|
88
248
|
stage_id = f"{datetime.now(UTC).strftime('%Y%m%dT%H%M%SZ')}-{uuid.uuid4().hex[:8]}"
|
|
89
249
|
stage_dir = self.staging_root / stage_id
|
|
90
250
|
cozo_dir, lance_dir = stage_dir / "cozo", stage_dir / "lance"
|
|
91
|
-
|
|
251
|
+
self._mkdir_durable(stage_dir, exist_ok=False)
|
|
92
252
|
cozo = lance = None
|
|
93
253
|
try:
|
|
94
254
|
cozo, lance = self._backend_factory(cozo_dir, lance_dir)
|
|
@@ -96,6 +256,7 @@ class ScaleEngineManager:
|
|
|
96
256
|
cozo.bulk_import_from_sqlite(conn, self.profile_id)
|
|
97
257
|
lance.bulk_import_from_sqlite(conn, self.profile_id)
|
|
98
258
|
canonical = self._canonical_counts(conn)
|
|
259
|
+
source_fingerprint = self._projection_fingerprint(conn, canonical)
|
|
99
260
|
observed = self._observed_counts(cozo, lance)
|
|
100
261
|
manifest = {
|
|
101
262
|
"schema_version": self.SCHEMA_VERSION,
|
|
@@ -105,7 +266,7 @@ class ScaleEngineManager:
|
|
|
105
266
|
"profile_id": self.profile_id,
|
|
106
267
|
"canonical": canonical,
|
|
107
268
|
"observed": observed,
|
|
108
|
-
"source_fingerprint":
|
|
269
|
+
"source_fingerprint": source_fingerprint,
|
|
109
270
|
}
|
|
110
271
|
self._write_manifest(stage_dir, manifest)
|
|
111
272
|
self.config.scale_engine_state = "prepared"
|
|
@@ -120,6 +281,15 @@ class ScaleEngineManager:
|
|
|
120
281
|
|
|
121
282
|
def verify(self, stage_id: str) -> dict[str, Any]:
|
|
122
283
|
"""Prove a staged projection matches the current canonical SQLite data."""
|
|
284
|
+
lock_path = self._acquire_lifecycle_lock()
|
|
285
|
+
try:
|
|
286
|
+
self._recover_interrupted_promotion()
|
|
287
|
+
return self._verify(stage_id)
|
|
288
|
+
finally:
|
|
289
|
+
self._release_lifecycle_lock(lock_path)
|
|
290
|
+
|
|
291
|
+
def _verify(self, stage_id: str) -> dict[str, Any]:
|
|
292
|
+
"""Verify while the caller owns the lifecycle lock."""
|
|
123
293
|
stage_dir, manifest = self._load_stage(stage_id)
|
|
124
294
|
self._validate_manifest(manifest, state="prepared")
|
|
125
295
|
self._require_default_profile()
|
|
@@ -128,8 +298,9 @@ class ScaleEngineManager:
|
|
|
128
298
|
cozo, lance = self._backend_factory(stage_dir / "cozo", stage_dir / "lance")
|
|
129
299
|
with self._readonly_connection() as conn:
|
|
130
300
|
canonical = self._canonical_counts(conn)
|
|
301
|
+
source_fingerprint = self._projection_fingerprint(conn, canonical)
|
|
131
302
|
observed = self._observed_counts(cozo, lance)
|
|
132
|
-
if manifest["source_fingerprint"] !=
|
|
303
|
+
if manifest["source_fingerprint"] != source_fingerprint:
|
|
133
304
|
raise ScaleEngineError("canonical SQLite changed after preparation; prepare a new stage")
|
|
134
305
|
if canonical != manifest["canonical"] or observed != canonical:
|
|
135
306
|
raise ScaleEngineError(
|
|
@@ -146,6 +317,15 @@ class ScaleEngineManager:
|
|
|
146
317
|
|
|
147
318
|
def promote(self, stage_id: str) -> dict[str, Any]:
|
|
148
319
|
"""Move a verified stage into active paths, preserving a rollback copy."""
|
|
320
|
+
lock_path = self._acquire_lifecycle_lock()
|
|
321
|
+
try:
|
|
322
|
+
self._recover_interrupted_promotion()
|
|
323
|
+
return self._promote(stage_id)
|
|
324
|
+
finally:
|
|
325
|
+
self._release_lifecycle_lock(lock_path)
|
|
326
|
+
|
|
327
|
+
def _promote(self, stage_id: str) -> dict[str, Any]:
|
|
328
|
+
"""Promote while the caller owns the lifecycle lock."""
|
|
149
329
|
stage_dir, manifest = self._load_stage(stage_id)
|
|
150
330
|
self._validate_manifest(manifest, state="verified")
|
|
151
331
|
staged = (stage_dir / "cozo", stage_dir / "lance")
|
|
@@ -153,39 +333,81 @@ class ScaleEngineManager:
|
|
|
153
333
|
raise ScaleEngineError("verified stage is incomplete; prepare a new stage")
|
|
154
334
|
backup_dir = self.backup_root / f"{stage_id}-{uuid.uuid4().hex[:6]}"
|
|
155
335
|
active = self.active_paths
|
|
156
|
-
self.
|
|
157
|
-
# Keep an explicit empty rollback point as well: a first promotion has
|
|
158
|
-
# no former projection directories, but rollback must still be able to
|
|
159
|
-
# return the installation to Local Core without deleting anything.
|
|
160
|
-
backup_dir.mkdir(parents=True, exist_ok=False)
|
|
161
|
-
moved_active: list[tuple[Path, Path]] = []
|
|
162
|
-
moved_stage: list[tuple[Path, Path]] = []
|
|
336
|
+
gate = sqlite3.connect(self.db_path, timeout=30)
|
|
163
337
|
try:
|
|
338
|
+
# The stage was built from a point-in-time SQLite snapshot. Hold a
|
|
339
|
+
# short writer fence for the final fingerprint check and directory
|
|
340
|
+
# swap so no successful promotion can trail a canonical write.
|
|
341
|
+
gate.execute("BEGIN IMMEDIATE")
|
|
342
|
+
canonical = self._canonical_counts(gate)
|
|
343
|
+
if manifest["source_fingerprint"] != self._projection_fingerprint(gate, canonical):
|
|
344
|
+
raise ScaleEngineError("canonical SQLite changed after verification; prepare a new stage")
|
|
345
|
+
self._mkdir_durable(self.backup_root)
|
|
346
|
+
journal = {
|
|
347
|
+
"schema_version": self.SCHEMA_VERSION,
|
|
348
|
+
"operation": "promotion",
|
|
349
|
+
"state": "intent",
|
|
350
|
+
"stage_id": stage_id,
|
|
351
|
+
"backup_id": backup_dir.name,
|
|
352
|
+
"moves": [],
|
|
353
|
+
}
|
|
354
|
+
self._write_promotion_journal(journal)
|
|
355
|
+
self._mkdir_durable(backup_dir, exist_ok=False)
|
|
164
356
|
for name, source, destination in zip(("cozo", "lance"), active, staged):
|
|
165
357
|
if source.exists():
|
|
166
358
|
target = backup_dir / name
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
359
|
+
move = {"name": name, "kind": "active_to_backup", "state": "intent"}
|
|
360
|
+
journal["moves"].append(move)
|
|
361
|
+
self._write_promotion_journal(journal)
|
|
362
|
+
self._replace_durable(source, target)
|
|
363
|
+
move["state"] = "complete"
|
|
364
|
+
self._write_promotion_journal(journal)
|
|
365
|
+
move = {"name": name, "kind": "stage_to_active", "state": "intent"}
|
|
366
|
+
journal["moves"].append(move)
|
|
367
|
+
self._write_promotion_journal(journal)
|
|
368
|
+
self._replace_durable(destination, source)
|
|
369
|
+
move["state"] = "complete"
|
|
370
|
+
self._write_promotion_journal(journal)
|
|
171
371
|
manifest.update({"state": "promoted", "promoted_at": _utc_now(), "backup_id": backup_dir.name})
|
|
172
372
|
self._write_manifest(stage_dir, manifest)
|
|
173
373
|
self.config.scale_engine_state = "promoted"
|
|
174
374
|
self.config.graph_backend = "cozo"
|
|
175
375
|
self.config.vector_backend = "lancedb"
|
|
176
376
|
self._save_config()
|
|
377
|
+
journal["state"] = "committed"
|
|
378
|
+
self._write_promotion_journal(journal)
|
|
379
|
+
self.promotion_journal_path.unlink(missing_ok=True)
|
|
380
|
+
gate.rollback()
|
|
177
381
|
return manifest
|
|
178
382
|
except Exception as exc:
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
383
|
+
try:
|
|
384
|
+
gate.rollback()
|
|
385
|
+
except sqlite3.Error:
|
|
386
|
+
pass
|
|
387
|
+
try:
|
|
388
|
+
recovery = self._recover_interrupted_promotion()
|
|
389
|
+
except Exception as recovery_error:
|
|
390
|
+
raise ScaleEngineError(
|
|
391
|
+
f"promotion interrupted; automatic recovery needs repair: {recovery_error}"
|
|
392
|
+
) from exc
|
|
393
|
+
if recovery == "finalized_committed_promotion":
|
|
394
|
+
_, recovered_manifest = self._load_stage(stage_id)
|
|
395
|
+
return recovered_manifest
|
|
185
396
|
raise ScaleEngineError(f"promotion rolled back: {exc}") from exc
|
|
397
|
+
finally:
|
|
398
|
+
gate.close()
|
|
186
399
|
|
|
187
400
|
def rollback(self, backup_id: str) -> dict[str, Any]:
|
|
188
401
|
"""Restore an explicitly named pre-promotion backup."""
|
|
402
|
+
lock_path = self._acquire_lifecycle_lock()
|
|
403
|
+
try:
|
|
404
|
+
self._recover_interrupted_promotion()
|
|
405
|
+
return self._rollback(backup_id)
|
|
406
|
+
finally:
|
|
407
|
+
self._release_lifecycle_lock(lock_path)
|
|
408
|
+
|
|
409
|
+
def _rollback(self, backup_id: str) -> dict[str, Any]:
|
|
410
|
+
"""Roll back while the caller owns the lifecycle lock."""
|
|
189
411
|
backup_dir = self.backup_root / backup_id
|
|
190
412
|
if not backup_dir.is_dir():
|
|
191
413
|
raise ScaleEngineError(f"backup does not exist: {backup_id}")
|
|
@@ -193,19 +415,47 @@ class ScaleEngineManager:
|
|
|
193
415
|
active = self.active_paths
|
|
194
416
|
displaced = self.backup_root / f"rollback-displaced-{uuid.uuid4().hex[:8]}"
|
|
195
417
|
try:
|
|
418
|
+
journal = {
|
|
419
|
+
"schema_version": self.SCHEMA_VERSION,
|
|
420
|
+
"operation": "rollback",
|
|
421
|
+
"state": "intent",
|
|
422
|
+
"backup_id": backup_id,
|
|
423
|
+
"displaced_id": displaced.name,
|
|
424
|
+
"moves": [],
|
|
425
|
+
}
|
|
426
|
+
self._write_promotion_journal(journal)
|
|
196
427
|
for name, source, target in zip(("cozo", "lance"), active, backup_paths):
|
|
197
428
|
if source.exists():
|
|
198
|
-
|
|
199
|
-
|
|
429
|
+
self._mkdir_durable(displaced)
|
|
430
|
+
move = {"name": name, "kind": "active_to_displaced", "state": "intent"}
|
|
431
|
+
journal["moves"].append(move)
|
|
432
|
+
self._write_promotion_journal(journal)
|
|
433
|
+
self._replace_durable(source, displaced / name)
|
|
434
|
+
move["state"] = "complete"
|
|
435
|
+
self._write_promotion_journal(journal)
|
|
200
436
|
if target.exists():
|
|
201
|
-
|
|
437
|
+
move = {"name": name, "kind": "backup_to_active", "state": "intent"}
|
|
438
|
+
journal["moves"].append(move)
|
|
439
|
+
self._write_promotion_journal(journal)
|
|
440
|
+
self._replace_durable(target, source)
|
|
441
|
+
move["state"] = "complete"
|
|
442
|
+
self._write_promotion_journal(journal)
|
|
202
443
|
self.config.scale_engine_state = "local_core"
|
|
203
444
|
self.config.graph_backend = "auto"
|
|
204
445
|
self.config.vector_backend = "auto"
|
|
205
446
|
self._save_config()
|
|
447
|
+
journal["state"] = "committed"
|
|
448
|
+
self._write_promotion_journal(journal)
|
|
449
|
+
self.promotion_journal_path.unlink(missing_ok=True)
|
|
206
450
|
return {"state": "local_core", "restored_backup": backup_id, "displaced": displaced.name}
|
|
207
451
|
except Exception as exc:
|
|
208
|
-
|
|
452
|
+
try:
|
|
453
|
+
self._recover_interrupted_promotion()
|
|
454
|
+
except ScaleEngineError as recovery_error:
|
|
455
|
+
raise ScaleEngineError(
|
|
456
|
+
f"rollback interrupted; automatic recovery needs repair: {recovery_error}"
|
|
457
|
+
) from exc
|
|
458
|
+
raise ScaleEngineError(f"rollback recovered: {exc}") from exc
|
|
209
459
|
|
|
210
460
|
def _real_backend_factory(self, cozo_dir: Path, lance_dir: Path) -> tuple[Any, Any]:
|
|
211
461
|
from superlocalmemory.graph.cozo_backend import CozoDBGraphBackend
|
|
@@ -222,9 +472,7 @@ class ScaleEngineManager:
|
|
|
222
472
|
"SELECT COUNT(*) FROM canonical_entities WHERE profile_id=?",
|
|
223
473
|
(self.profile_id,),
|
|
224
474
|
).fetchone()[0]
|
|
225
|
-
edges = conn.
|
|
226
|
-
"SELECT COUNT(*) FROM graph_edges WHERE profile_id=?", (self.profile_id,)
|
|
227
|
-
).fetchone()[0]
|
|
475
|
+
edges = count_logical_edges(conn, self.profile_id)
|
|
228
476
|
try:
|
|
229
477
|
vectors = conn.execute(
|
|
230
478
|
"SELECT COUNT(*) FROM fact_embeddings_rowids fer "
|
|
@@ -243,17 +491,71 @@ class ScaleEngineManager:
|
|
|
243
491
|
raise ScaleEngineError(f"projection health failed: cozo={graph}, lancedb={vector}")
|
|
244
492
|
return {"entities": int(graph["entities"]), "edges": int(graph["edges"]), "vectors": int(vector["vectors"])}
|
|
245
493
|
|
|
246
|
-
def
|
|
494
|
+
def _projection_fingerprint(
|
|
495
|
+
self, conn: sqlite3.Connection, counts: dict[str, int]
|
|
496
|
+
) -> str:
|
|
497
|
+
"""Hash the exact projection source rows inside one SQLite snapshot."""
|
|
247
498
|
digest = hashlib.sha256()
|
|
248
499
|
digest.update(json.dumps(counts, sort_keys=True).encode())
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
500
|
+
tables = (
|
|
501
|
+
("canonical_entities", "entity_id, canonical_name, entity_type, first_seen, last_seen, fact_count, profile_id", "entity_id"),
|
|
502
|
+
("atomic_facts", "fact_id, canonical_entities_json, lifecycle, profile_id", "fact_id"),
|
|
503
|
+
)
|
|
504
|
+
for table, columns, ordering in tables:
|
|
505
|
+
try:
|
|
506
|
+
rows = conn.execute(
|
|
507
|
+
f"SELECT {columns} FROM {table} WHERE profile_id=? ORDER BY {ordering}",
|
|
508
|
+
(self.profile_id,),
|
|
509
|
+
)
|
|
510
|
+
for row in rows:
|
|
511
|
+
self._digest_row(digest, table, row)
|
|
512
|
+
except sqlite3.OperationalError as exc:
|
|
513
|
+
raise ScaleEngineError(f"canonical SQLite missing required {table} table") from exc
|
|
514
|
+
try:
|
|
515
|
+
for row in iter_logical_edges(conn, self.profile_id):
|
|
516
|
+
self._digest_row(digest, "graph_edges", row)
|
|
517
|
+
except sqlite3.OperationalError as exc:
|
|
518
|
+
raise ScaleEngineError("canonical SQLite missing required graph_edges table") from exc
|
|
519
|
+
try:
|
|
520
|
+
rows = conn.execute(
|
|
521
|
+
"SELECT fer.rowid, fer.fact_id, vec.vector FROM fact_embeddings_rowids fer "
|
|
522
|
+
"JOIN atomic_facts af ON af.fact_id = fer.fact_id "
|
|
523
|
+
"LEFT JOIN fact_embeddings_vector_chunks00 vec ON vec.rowid = fer.rowid "
|
|
524
|
+
"WHERE af.profile_id=? ORDER BY fer.rowid",
|
|
525
|
+
(self.profile_id,),
|
|
526
|
+
)
|
|
527
|
+
for row in rows:
|
|
528
|
+
self._digest_row(digest, "fact_embeddings_rowids", row)
|
|
529
|
+
except sqlite3.OperationalError:
|
|
530
|
+
pass
|
|
255
531
|
return digest.hexdigest()
|
|
256
532
|
|
|
533
|
+
@staticmethod
|
|
534
|
+
def _digest_row(digest: Any, table: str, row: Any) -> None:
|
|
535
|
+
digest.update(table.encode())
|
|
536
|
+
digest.update(json.dumps(list(row), default=str, separators=(",", ":")).encode())
|
|
537
|
+
|
|
538
|
+
def _has_legacy_projection_layout(self) -> bool:
|
|
539
|
+
cozo, lance = self.active_paths
|
|
540
|
+
return (cozo / "graph").is_dir() and (lance / "embeddings.lance").exists()
|
|
541
|
+
|
|
542
|
+
def _runtime_backend_status(self) -> dict[str, str]:
|
|
543
|
+
status = {"cozo": "unknown", "lance": "unknown"}
|
|
544
|
+
try:
|
|
545
|
+
with self._readonly_connection() as conn:
|
|
546
|
+
rows = conn.execute(
|
|
547
|
+
"SELECT backend_name, status FROM backend_status "
|
|
548
|
+
"WHERE backend_name IN ('cozo', 'lancedb')"
|
|
549
|
+
)
|
|
550
|
+
for name, value in rows:
|
|
551
|
+
if name == "cozo":
|
|
552
|
+
status["cozo"] = str(value)
|
|
553
|
+
elif name == "lancedb":
|
|
554
|
+
status["lance"] = str(value)
|
|
555
|
+
except sqlite3.OperationalError:
|
|
556
|
+
pass
|
|
557
|
+
return status
|
|
558
|
+
|
|
257
559
|
def _load_stage(self, stage_id: str) -> tuple[Path, dict[str, Any]]:
|
|
258
560
|
stage_dir = self.staging_root / stage_id
|
|
259
561
|
try:
|
|
@@ -280,6 +582,201 @@ class ScaleEngineManager:
|
|
|
280
582
|
if not self.db_path.exists():
|
|
281
583
|
raise ScaleEngineError(f"canonical SQLite database not found: {self.db_path}")
|
|
282
584
|
|
|
585
|
+
def _acquire_lifecycle_lock(self) -> Path:
|
|
586
|
+
"""Serialize every mutating lifecycle command across processes."""
|
|
587
|
+
lock_path = self.lifecycle_lock_path
|
|
588
|
+
descriptor = None
|
|
589
|
+
for attempt in range(2):
|
|
590
|
+
try:
|
|
591
|
+
descriptor = os.open(lock_path, os.O_CREAT | os.O_EXCL | os.O_WRONLY)
|
|
592
|
+
break
|
|
593
|
+
except FileExistsError as exc:
|
|
594
|
+
if attempt == 0 and self._clear_dead_legacy_adoption_lock(lock_path):
|
|
595
|
+
continue
|
|
596
|
+
raise ScaleEngineError(
|
|
597
|
+
"Scale Engine lifecycle operation already in progress; retry after it completes"
|
|
598
|
+
) from exc
|
|
599
|
+
if descriptor is None:
|
|
600
|
+
raise ScaleEngineError("could not acquire Scale Engine lifecycle lock")
|
|
601
|
+
try:
|
|
602
|
+
with os.fdopen(descriptor, "w") as lock_file:
|
|
603
|
+
json.dump({"pid": os.getpid(), "started_at": _utc_now()}, lock_file)
|
|
604
|
+
except Exception:
|
|
605
|
+
lock_path.unlink(missing_ok=True)
|
|
606
|
+
raise
|
|
607
|
+
return lock_path
|
|
608
|
+
|
|
609
|
+
@staticmethod
|
|
610
|
+
def _release_lifecycle_lock(lock_path: Path) -> None:
|
|
611
|
+
lock_path.unlink(missing_ok=True)
|
|
612
|
+
|
|
613
|
+
@staticmethod
|
|
614
|
+
def _clear_dead_legacy_adoption_lock(lock_path: Path) -> bool:
|
|
615
|
+
"""Recover only a lock whose recorded process no longer exists."""
|
|
616
|
+
try:
|
|
617
|
+
owner = json.loads(lock_path.read_text())
|
|
618
|
+
pid = owner.get("pid")
|
|
619
|
+
if not isinstance(pid, int) or pid <= 0:
|
|
620
|
+
return False
|
|
621
|
+
os.kill(pid, 0)
|
|
622
|
+
except ProcessLookupError:
|
|
623
|
+
lock_path.unlink(missing_ok=True)
|
|
624
|
+
return True
|
|
625
|
+
except (OSError, ValueError, json.JSONDecodeError):
|
|
626
|
+
return False
|
|
627
|
+
return False
|
|
628
|
+
|
|
629
|
+
def recover_interrupted_promotion(self) -> str | None:
|
|
630
|
+
"""Recover a durable promotion journal before opening projection paths."""
|
|
631
|
+
lock_path = self._acquire_lifecycle_lock()
|
|
632
|
+
try:
|
|
633
|
+
return self._recover_interrupted_promotion()
|
|
634
|
+
finally:
|
|
635
|
+
self._release_lifecycle_lock(lock_path)
|
|
636
|
+
|
|
637
|
+
def _recover_interrupted_promotion(self) -> str | None:
|
|
638
|
+
"""Finalize or reverse an interrupted directory swap under lifecycle lock."""
|
|
639
|
+
if not self.promotion_journal_path.exists():
|
|
640
|
+
return None
|
|
641
|
+
try:
|
|
642
|
+
journal = json.loads(self.promotion_journal_path.read_text())
|
|
643
|
+
backup_id = str(journal["backup_id"])
|
|
644
|
+
state = str(journal["state"])
|
|
645
|
+
except (OSError, KeyError, TypeError, json.JSONDecodeError) as exc:
|
|
646
|
+
raise ScaleEngineError("invalid promotion journal; manual repair required") from exc
|
|
647
|
+
operation = str(journal.get("operation", "promotion"))
|
|
648
|
+
if state == "committed":
|
|
649
|
+
if operation == "promotion":
|
|
650
|
+
self.config.scale_engine_state = "promoted"
|
|
651
|
+
self.config.graph_backend = "cozo"
|
|
652
|
+
self.config.vector_backend = "lancedb"
|
|
653
|
+
elif operation == "rollback":
|
|
654
|
+
self.config.scale_engine_state = "local_core"
|
|
655
|
+
self.config.graph_backend = "auto"
|
|
656
|
+
self.config.vector_backend = "auto"
|
|
657
|
+
else:
|
|
658
|
+
raise ScaleEngineError(f"unknown journal operation: {operation!r}")
|
|
659
|
+
self._save_config()
|
|
660
|
+
self.promotion_journal_path.unlink(missing_ok=True)
|
|
661
|
+
return f"finalized_committed_{operation}"
|
|
662
|
+
if state != "intent":
|
|
663
|
+
raise ScaleEngineError(f"unknown promotion journal state: {state!r}")
|
|
664
|
+
backup_dir = self.backup_root / backup_id
|
|
665
|
+
active = dict(zip(("cozo", "lance"), self.active_paths))
|
|
666
|
+
stage_dir = self.staging_root / str(journal.get("stage_id", ""))
|
|
667
|
+
displaced_dir = self.backup_root / str(journal.get("displaced_id", ""))
|
|
668
|
+
moves = journal.get("moves")
|
|
669
|
+
if not isinstance(moves, list):
|
|
670
|
+
# Compatibility for journals written by the initial v3.7.3
|
|
671
|
+
# candidate before per-rename intents existed.
|
|
672
|
+
moves = [
|
|
673
|
+
{"name": name, "kind": "active_to_backup", "state": "complete"}
|
|
674
|
+
for name in journal.get("moved_active", [])
|
|
675
|
+
] + [
|
|
676
|
+
{"name": name, "kind": "stage_to_active", "state": "complete"}
|
|
677
|
+
for name in journal.get("moved_stage", [])
|
|
678
|
+
]
|
|
679
|
+
for move in reversed(moves):
|
|
680
|
+
self._reverse_journal_move(move, active, stage_dir, backup_dir, displaced_dir)
|
|
681
|
+
if operation == "promotion":
|
|
682
|
+
if backup_dir.exists() and not any(backup_dir.iterdir()):
|
|
683
|
+
backup_dir.rmdir()
|
|
684
|
+
self.config.scale_engine_state = "local_core"
|
|
685
|
+
self.config.graph_backend = "auto"
|
|
686
|
+
self.config.vector_backend = "auto"
|
|
687
|
+
elif operation == "rollback":
|
|
688
|
+
if displaced_dir.exists() and not any(displaced_dir.iterdir()):
|
|
689
|
+
displaced_dir.rmdir()
|
|
690
|
+
self.config.scale_engine_state = "promoted"
|
|
691
|
+
self.config.graph_backend = "cozo"
|
|
692
|
+
self.config.vector_backend = "lancedb"
|
|
693
|
+
else:
|
|
694
|
+
raise ScaleEngineError(f"unknown journal operation: {operation!r}")
|
|
695
|
+
self._save_config()
|
|
696
|
+
self.promotion_journal_path.unlink(missing_ok=True)
|
|
697
|
+
return f"reversed_interrupted_{operation}"
|
|
698
|
+
|
|
699
|
+
@staticmethod
|
|
700
|
+
def _reverse_journal_move(
|
|
701
|
+
move: Any,
|
|
702
|
+
active: dict[str, Path],
|
|
703
|
+
stage_dir: Path,
|
|
704
|
+
backup_dir: Path,
|
|
705
|
+
displaced_dir: Path,
|
|
706
|
+
) -> None:
|
|
707
|
+
"""Reverse one planned rename based on actual paths, not journal timing."""
|
|
708
|
+
if not isinstance(move, dict):
|
|
709
|
+
raise ScaleEngineError("invalid promotion journal move")
|
|
710
|
+
name = move.get("name")
|
|
711
|
+
kind = move.get("kind")
|
|
712
|
+
active_path = active.get(name)
|
|
713
|
+
if active_path is None:
|
|
714
|
+
raise ScaleEngineError(f"invalid promotion journal backend: {name!r}")
|
|
715
|
+
if kind == "stage_to_active":
|
|
716
|
+
source, target = active_path, stage_dir / name
|
|
717
|
+
elif kind == "active_to_backup":
|
|
718
|
+
source, target = backup_dir / name, active_path
|
|
719
|
+
elif kind == "active_to_displaced":
|
|
720
|
+
source, target = displaced_dir / name, active_path
|
|
721
|
+
elif kind == "backup_to_active":
|
|
722
|
+
source, target = active_path, backup_dir / name
|
|
723
|
+
else:
|
|
724
|
+
raise ScaleEngineError(f"invalid promotion journal move kind: {kind!r}")
|
|
725
|
+
if source.exists() and not target.exists():
|
|
726
|
+
ScaleEngineManager._replace_durable(source, target)
|
|
727
|
+
elif target.exists() and not source.exists():
|
|
728
|
+
return
|
|
729
|
+
else:
|
|
730
|
+
raise ScaleEngineError(f"cannot safely reconcile {kind} for {name}")
|
|
731
|
+
|
|
732
|
+
def _write_promotion_journal(self, journal: dict[str, Any]) -> None:
|
|
733
|
+
self._write_json_durable(self.promotion_journal_path, journal)
|
|
734
|
+
|
|
735
|
+
@staticmethod
|
|
736
|
+
def _write_json_durable(target: Path, payload: dict[str, Any]) -> None:
|
|
737
|
+
temporary = target.with_suffix(target.suffix + ".tmp")
|
|
738
|
+
with temporary.open("w") as handle:
|
|
739
|
+
json.dump(payload, handle, indent=2, sort_keys=True)
|
|
740
|
+
handle.write("\n")
|
|
741
|
+
handle.flush()
|
|
742
|
+
os.fsync(handle.fileno())
|
|
743
|
+
os.replace(temporary, target)
|
|
744
|
+
ScaleEngineManager._fsync_directory(target.parent)
|
|
745
|
+
|
|
746
|
+
@staticmethod
|
|
747
|
+
def _replace_durable(source: Path, target: Path) -> None:
|
|
748
|
+
"""Rename a projection path and persist both directory entries."""
|
|
749
|
+
os.replace(source, target)
|
|
750
|
+
ScaleEngineManager._fsync_directory(source.parent)
|
|
751
|
+
if target.parent != source.parent:
|
|
752
|
+
ScaleEngineManager._fsync_directory(target.parent)
|
|
753
|
+
|
|
754
|
+
@staticmethod
|
|
755
|
+
def _mkdir_durable(path: Path, *, exist_ok: bool = True) -> None:
|
|
756
|
+
"""Create a directory and persist every new parent entry before rename."""
|
|
757
|
+
missing: list[Path] = []
|
|
758
|
+
ancestor = path
|
|
759
|
+
while not ancestor.exists():
|
|
760
|
+
missing.append(ancestor)
|
|
761
|
+
ancestor = ancestor.parent
|
|
762
|
+
path.mkdir(parents=True, exist_ok=exist_ok)
|
|
763
|
+
for created in reversed(missing):
|
|
764
|
+
ScaleEngineManager._fsync_directory(created.parent)
|
|
765
|
+
|
|
766
|
+
@staticmethod
|
|
767
|
+
def _fsync_directory(directory_path: Path) -> None:
|
|
768
|
+
"""Best-effort directory-entry durability across local filesystems."""
|
|
769
|
+
try:
|
|
770
|
+
directory = os.open(directory_path, os.O_RDONLY)
|
|
771
|
+
try:
|
|
772
|
+
os.fsync(directory)
|
|
773
|
+
finally:
|
|
774
|
+
os.close(directory)
|
|
775
|
+
except OSError:
|
|
776
|
+
# The file itself is already durable where directory fsync is not
|
|
777
|
+
# supported by the local filesystem (notably some Windows setups).
|
|
778
|
+
pass
|
|
779
|
+
|
|
283
780
|
def _save_config(self) -> None:
|
|
284
781
|
save = getattr(self.config, "save", None)
|
|
285
782
|
if callable(save):
|
|
@@ -24,6 +24,8 @@ from datetime import datetime
|
|
|
24
24
|
from pathlib import Path
|
|
25
25
|
from typing import Any
|
|
26
26
|
|
|
27
|
+
from superlocalmemory.storage.logical_edges import iter_logical_edges
|
|
28
|
+
|
|
27
29
|
logger = logging.getLogger(__name__)
|
|
28
30
|
|
|
29
31
|
# Optional import — CozoDB is an optional dependency
|
|
@@ -260,6 +262,12 @@ class CozoDBGraphBackend:
|
|
|
260
262
|
*fact_entity{fact_id, entity_id, profile_id}, fact_id = $fact_id
|
|
261
263
|
:rm fact_entity {fact_id, entity_id => profile_id}
|
|
262
264
|
""", {"fact_id": fact_id})
|
|
265
|
+
self._db.run("""
|
|
266
|
+
?[from_id, to_id, edge_type, weight, metadata, profile_id, created_at] :=
|
|
267
|
+
*edge{from_id, to_id, edge_type, weight, metadata, profile_id, created_at},
|
|
268
|
+
(from_id = $fact_id or to_id = $fact_id)
|
|
269
|
+
:rm edge {from_id, to_id, edge_type => weight, metadata, profile_id, created_at}
|
|
270
|
+
""", {"fact_id": fact_id})
|
|
263
271
|
|
|
264
272
|
def record_shadow_comparison(
|
|
265
273
|
self,
|
|
@@ -280,12 +288,6 @@ class CozoDBGraphBackend:
|
|
|
280
288
|
def record_shadow_error(self, error: str) -> None:
|
|
281
289
|
self._shadow_errors += 1
|
|
282
290
|
logger.warning("Cozo entity recall failed closed to SQLite: %s", error)
|
|
283
|
-
self._db.run("""
|
|
284
|
-
?[from_id, to_id, edge_type, weight, metadata, profile_id, created_at] :=
|
|
285
|
-
*edge{from_id, to_id, edge_type, weight, metadata, profile_id, created_at},
|
|
286
|
-
(from_id = $fact_id or to_id = $fact_id)
|
|
287
|
-
:rm edge {from_id, to_id, edge_type => weight, metadata, profile_id, created_at}
|
|
288
|
-
""", {"fact_id": fact_id})
|
|
289
291
|
|
|
290
292
|
# ------------------------------------------------------------------
|
|
291
293
|
# Bulk Import (SQLite → CozoDB)
|
|
@@ -358,22 +360,15 @@ class CozoDBGraphBackend:
|
|
|
358
360
|
|
|
359
361
|
# Step 3: Export fact graph edges directly. Fact graph traversal is
|
|
360
362
|
# intentionally kept in its native fact-ID namespace.
|
|
361
|
-
edges_sql = """
|
|
362
|
-
SELECT source_id, target_id, edge_type, weight
|
|
363
|
-
FROM graph_edges WHERE profile_id = ?
|
|
364
|
-
"""
|
|
365
|
-
edge_rows = conn.execute(edges_sql, (profile_id,)).fetchall()
|
|
366
|
-
|
|
367
363
|
edge_dicts = []
|
|
368
|
-
for
|
|
369
|
-
ea, eb, etype, weight = row
|
|
364
|
+
for ea, eb, etype, weight, edge_profile in iter_logical_edges(conn, profile_id):
|
|
370
365
|
edge_dicts.append({
|
|
371
366
|
"from_id": ea,
|
|
372
367
|
"to_id": eb,
|
|
373
|
-
"edge_type": etype
|
|
374
|
-
"weight": float(weight
|
|
368
|
+
"edge_type": etype,
|
|
369
|
+
"weight": float(weight),
|
|
375
370
|
"metadata": "{}",
|
|
376
|
-
"profile_id":
|
|
371
|
+
"profile_id": edge_profile,
|
|
377
372
|
"created_at": now,
|
|
378
373
|
})
|
|
379
374
|
|
|
@@ -28,6 +28,7 @@ import sys
|
|
|
28
28
|
import tempfile
|
|
29
29
|
from pathlib import Path
|
|
30
30
|
|
|
31
|
+
from superlocalmemory import __version__
|
|
31
32
|
from superlocalmemory.infra.data_root import canonical_data_root
|
|
32
33
|
from superlocalmemory.infra.data_root import state_path as runtime_state_path
|
|
33
34
|
|
|
@@ -40,7 +41,7 @@ _DEFAULT_DISABLED_FILE = _DEFAULT_VERSION_DIR / ".hooks-disabled"
|
|
|
40
41
|
VERSION_DIR = _DEFAULT_VERSION_DIR
|
|
41
42
|
VERSION_FILE = _DEFAULT_VERSION_FILE
|
|
42
43
|
DISABLED_FILE = _DEFAULT_DISABLED_FILE
|
|
43
|
-
HOOKS_VERSION =
|
|
44
|
+
HOOKS_VERSION = __version__
|
|
44
45
|
|
|
45
46
|
# Cross-platform temp dir and backwards-compatible marker overrides. Runtime
|
|
46
47
|
# defaults are root-namespaced and resolved when hook definitions are built.
|
|
@@ -19,10 +19,11 @@ from datetime import datetime, timezone
|
|
|
19
19
|
from pathlib import Path
|
|
20
20
|
from typing import Callable, Iterable
|
|
21
21
|
|
|
22
|
+
from superlocalmemory import __version__
|
|
22
23
|
from superlocalmemory.core.security_primitives import redact_secrets
|
|
23
24
|
|
|
24
25
|
|
|
25
|
-
VERSION =
|
|
26
|
+
VERSION = __version__
|
|
26
27
|
DEFAULT_TOP_K = 10
|
|
27
28
|
DEFAULT_DECISIONS_K = 5
|
|
28
29
|
DEFAULT_MEMORIES_K = 10
|
|
@@ -16,6 +16,7 @@ from __future__ import annotations
|
|
|
16
16
|
from mcp.server.fastmcp import FastMCP
|
|
17
17
|
from sse_starlette.sse import EventSourceResponse
|
|
18
18
|
from starlette.types import Receive, Scope, Send
|
|
19
|
+
from superlocalmemory import __version__
|
|
19
20
|
|
|
20
21
|
|
|
21
22
|
class ClosingEventSourceResponse(EventSourceResponse):
|
|
@@ -40,7 +41,7 @@ def install_streamable_http_resource_guard() -> None:
|
|
|
40
41
|
class SLMFastMCP(FastMCP):
|
|
41
42
|
"""FastMCP with SLM release identity and deterministic SSE cleanup."""
|
|
42
43
|
|
|
43
|
-
def __init__(self, *args, product_version: str =
|
|
44
|
+
def __init__(self, *args, product_version: str = __version__, **kwargs) -> None:
|
|
44
45
|
super().__init__(*args, **kwargs)
|
|
45
46
|
# FastMCP delegates the initialize response to the low-level MCP
|
|
46
47
|
# server. Without an explicit value it reports the installed ``mcp``
|
|
@@ -12,12 +12,13 @@ from fastapi import APIRouter
|
|
|
12
12
|
from fastapi.requests import Request
|
|
13
13
|
from fastapi.responses import Response
|
|
14
14
|
|
|
15
|
+
from superlocalmemory import __version__
|
|
15
16
|
from superlocalmemory.optimize.config.schema import OptimizeConfig
|
|
16
17
|
from superlocalmemory.optimize.proxy.lifecycle import HookChain
|
|
17
18
|
|
|
18
19
|
logger = logging.getLogger("slm.optimize.proxy")
|
|
19
20
|
|
|
20
|
-
_PROXY_VERSION =
|
|
21
|
+
_PROXY_VERSION = __version__
|
|
21
22
|
_REQUEST_TIMEOUT_S = 300.0
|
|
22
23
|
_CONNECT_TIMEOUT_S = 10.0
|
|
23
24
|
_MAX_CONNECTIONS = 100
|
|
@@ -43,6 +43,7 @@ from pathlib import Path
|
|
|
43
43
|
from typing import Any
|
|
44
44
|
|
|
45
45
|
from fastapi import APIRouter, Depends, HTTPException, Request
|
|
46
|
+
from superlocalmemory import __version__
|
|
46
47
|
|
|
47
48
|
from superlocalmemory.core.security_primitives import (
|
|
48
49
|
redact_secrets,
|
|
@@ -64,7 +65,7 @@ router = APIRouter(prefix="/api/v3", tags=["brain"])
|
|
|
64
65
|
# LLD-03 v2 stratum space = 4 query types × 3 entity bins × 4 time buckets.
|
|
65
66
|
_STRATA_TOTAL: int = 48
|
|
66
67
|
|
|
67
|
-
_VERSION: str =
|
|
68
|
+
_VERSION: str = __version__
|
|
68
69
|
|
|
69
70
|
# Banned metric names (LLD-04 U4). Kept as a tuple for grep visibility;
|
|
70
71
|
# the source-level test asserts we don't accidentally reintroduce them.
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
"""Canonical logical-edge projection shared by scale backends.
|
|
2
|
+
|
|
3
|
+
Legacy databases can contain multiple physical ``graph_edges`` rows for one
|
|
4
|
+
logical relationship. Current writes define identity as profile, source,
|
|
5
|
+
target, and edge type, retaining the strongest weight. Derived projections
|
|
6
|
+
must use that same contract without rewriting canonical SQLite history.
|
|
7
|
+
"""
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import sqlite3
|
|
11
|
+
from collections.abc import Iterator
|
|
12
|
+
from typing import Any
|
|
13
|
+
|
|
14
|
+
_LOGICAL_EDGE_SELECT = """
|
|
15
|
+
SELECT
|
|
16
|
+
source_id,
|
|
17
|
+
target_id,
|
|
18
|
+
COALESCE(edge_type, 'related') AS edge_type,
|
|
19
|
+
MAX(COALESCE(weight, 1.0)) AS weight,
|
|
20
|
+
profile_id
|
|
21
|
+
FROM graph_edges
|
|
22
|
+
WHERE profile_id = ?
|
|
23
|
+
GROUP BY profile_id, source_id, target_id, COALESCE(edge_type, 'related')
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def iter_logical_edges(
|
|
28
|
+
conn: sqlite3.Connection, profile_id: str
|
|
29
|
+
) -> Iterator[tuple[Any, ...]]:
|
|
30
|
+
"""Yield normalized graph edges in deterministic fingerprint order."""
|
|
31
|
+
return iter(
|
|
32
|
+
conn.execute(
|
|
33
|
+
_LOGICAL_EDGE_SELECT + " ORDER BY source_id, target_id, edge_type",
|
|
34
|
+
(profile_id,),
|
|
35
|
+
)
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def count_logical_edges(conn: sqlite3.Connection, profile_id: str) -> int:
|
|
40
|
+
"""Count relationships using the canonical logical identity."""
|
|
41
|
+
row = conn.execute(
|
|
42
|
+
"SELECT COUNT(*) FROM (" + _LOGICAL_EDGE_SELECT + ")",
|
|
43
|
+
(profile_id,),
|
|
44
|
+
).fetchone()
|
|
45
|
+
return int(row[0] if row else 0)
|