superlocalmemory 3.4.25 → 3.4.31
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +92 -0
- package/README.md +8 -1
- package/package.json +1 -1
- package/pyproject.toml +3 -1
- package/src/superlocalmemory/__init__.py +1 -1
- package/src/superlocalmemory/cli/daemon.py +90 -16
- package/src/superlocalmemory/cli/doctor_cmd.py +152 -0
- package/src/superlocalmemory/cli/main.py +28 -0
- package/src/superlocalmemory/cli/pending_store.py +55 -3
- package/src/superlocalmemory/cli/post_install.py +15 -0
- package/src/superlocalmemory/cli/setup_wizard.py +20 -0
- package/src/superlocalmemory/cli/version_banner.py +183 -0
- package/src/superlocalmemory/cli/wizard_v3426_options.py +129 -0
- package/src/superlocalmemory/core/clock_monitor.py +45 -0
- package/src/superlocalmemory/core/db_pool.py +80 -0
- package/src/superlocalmemory/core/engine.py +75 -30
- package/src/superlocalmemory/core/engine_capabilities.py +24 -0
- package/src/superlocalmemory/core/engine_lock.py +75 -0
- package/src/superlocalmemory/core/error_catalog.py +113 -0
- package/src/superlocalmemory/core/error_envelope.py +60 -0
- package/src/superlocalmemory/core/file_lock.py +92 -0
- package/src/superlocalmemory/core/loop_watchdog.py +56 -0
- package/src/superlocalmemory/core/maintenance_scheduler.py +8 -0
- package/src/superlocalmemory/core/priority_queue.py +61 -0
- package/src/superlocalmemory/core/queue_dispatcher.py +73 -0
- package/src/superlocalmemory/core/rate_limit.py +151 -0
- package/src/superlocalmemory/core/recall_queue.py +370 -0
- package/src/superlocalmemory/core/recall_worker.py +10 -0
- package/src/superlocalmemory/core/safe_fs.py +108 -0
- package/src/superlocalmemory/hooks/auto_capture.py +34 -12
- package/src/superlocalmemory/hooks/auto_recall.py +36 -9
- package/src/superlocalmemory/learning/signals.py +7 -1
- package/src/superlocalmemory/mcp/_daemon_proxy.py +107 -0
- package/src/superlocalmemory/mcp/_pool_adapter.py +121 -0
- package/src/superlocalmemory/mcp/resources.py +8 -5
- package/src/superlocalmemory/mcp/server.py +38 -9
- package/src/superlocalmemory/mcp/tools_active.py +21 -9
- package/src/superlocalmemory/mcp/tools_core.py +13 -9
- package/src/superlocalmemory/mcp/tools_evolution.py +4 -2
- package/src/superlocalmemory/mcp/tools_learning.py +5 -3
- package/src/superlocalmemory/mcp/tools_mesh.py +5 -3
- package/src/superlocalmemory/mcp/tools_v3.py +18 -22
- package/src/superlocalmemory/mcp/tools_v33.py +65 -2
- package/src/superlocalmemory/migrations/__init__.py +5 -0
- package/src/superlocalmemory/migrations/v3_4_25_to_v3_4_26.py +144 -0
- package/src/superlocalmemory/server/routes/data_io.py +21 -2
- package/src/superlocalmemory/server/routes/memories.py +91 -0
- package/src/superlocalmemory/server/routes/stats.py +16 -2
- package/src/superlocalmemory/server/unified_daemon.py +128 -12
- package/src/superlocalmemory/ui/index.html +35 -25
- package/src/superlocalmemory/ui/js/core.js +20 -4
- package/src/superlocalmemory/ui/js/fact-detail.js +62 -73
- package/src/superlocalmemory/ui/js/memories.js +34 -2
- package/src/superlocalmemory/ui/js/modal.js +41 -2
- package/src/superlocalmemory/ui/js/search.js +27 -0
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
|
|
2
|
+
# Licensed under AGPL-3.0-or-later - see LICENSE file
|
|
3
|
+
# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
|
|
4
|
+
|
|
5
|
+
"""Post-upgrade version banner.
|
|
6
|
+
|
|
7
|
+
Writes ``$SLM_DATA_DIR/.version`` (default ``~/.superlocalmemory/.version``)
|
|
8
|
+
and prints a short factual banner the first time the CLI or daemon is
|
|
9
|
+
invoked after a ``pip install -U`` / ``npm install -g`` that changes the
|
|
10
|
+
installed version. Every subsequent invocation is a no-op.
|
|
11
|
+
|
|
12
|
+
The banner is best-effort and must never raise — a disk error or a
|
|
13
|
+
corrupt marker makes the banner silent, not the CLI broken.
|
|
14
|
+
"""
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import os
|
|
18
|
+
import re
|
|
19
|
+
import sys
|
|
20
|
+
from pathlib import Path
|
|
21
|
+
|
|
22
|
+
# Strict allowlist: semver-ish only. Anything else in the marker is
|
|
23
|
+
# treated as "unknown" so a tampered or binary marker can't leak into
|
|
24
|
+
# banner strings or downstream consumers.
|
|
25
|
+
_VERSION_PATTERN = re.compile(r"^[0-9A-Za-z.\-+_]{1,32}$")
|
|
26
|
+
|
|
27
|
+
_MAX_MARKER_BYTES = 64 # a semver string is ≤ 32 chars; 64 is plenty
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _data_dir() -> Path:
|
|
31
|
+
return Path(os.environ.get("SLM_DATA_DIR") or Path.home() / ".superlocalmemory")
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _marker_path() -> Path:
|
|
35
|
+
return _data_dir() / ".version"
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def read_marker_version() -> str | None:
|
|
39
|
+
"""Return the marker string or None if missing / unreadable / invalid."""
|
|
40
|
+
target = _marker_path()
|
|
41
|
+
# Refuse to follow symlinks on the marker file — a co-tenant
|
|
42
|
+
# attacker who can drop a symlink into our data dir must not be
|
|
43
|
+
# able to redirect us to an arbitrary file.
|
|
44
|
+
try:
|
|
45
|
+
st = target.lstat()
|
|
46
|
+
except (FileNotFoundError, NotADirectoryError, PermissionError, OSError):
|
|
47
|
+
return None
|
|
48
|
+
import stat as _stat
|
|
49
|
+
if _stat.S_ISLNK(st.st_mode) or not _stat.S_ISREG(st.st_mode):
|
|
50
|
+
return None
|
|
51
|
+
try:
|
|
52
|
+
with open(target, "rb") as f:
|
|
53
|
+
raw = f.read(_MAX_MARKER_BYTES + 1)
|
|
54
|
+
except (FileNotFoundError, NotADirectoryError, PermissionError, OSError):
|
|
55
|
+
return None
|
|
56
|
+
if len(raw) > _MAX_MARKER_BYTES:
|
|
57
|
+
return None
|
|
58
|
+
try:
|
|
59
|
+
text = raw.decode("ascii").strip()
|
|
60
|
+
except UnicodeDecodeError:
|
|
61
|
+
return None
|
|
62
|
+
if not _VERSION_PATTERN.fullmatch(text):
|
|
63
|
+
return None
|
|
64
|
+
return text
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def write_marker_version(version: str) -> bool:
|
|
68
|
+
"""Persist the current version to the marker with 0600 perms.
|
|
69
|
+
|
|
70
|
+
Uses a PID-scoped tmp file + ``os.replace`` for atomicity so
|
|
71
|
+
concurrent CLI / daemon / npm-postinstall invocations can't corrupt
|
|
72
|
+
each other's write. The final file is mode 0600 so the upgrade
|
|
73
|
+
timestamp is not a side-channel for co-tenant attackers.
|
|
74
|
+
"""
|
|
75
|
+
# Validate input — never write garbage, even on a programmer bug.
|
|
76
|
+
if not _VERSION_PATTERN.fullmatch(version):
|
|
77
|
+
return False
|
|
78
|
+
target = _marker_path()
|
|
79
|
+
try:
|
|
80
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
81
|
+
# Parent dir 0700 on POSIX (mirrors safe_fs._ensure_parent_0700)
|
|
82
|
+
if sys.platform != "win32":
|
|
83
|
+
try:
|
|
84
|
+
os.chmod(target.parent, 0o700)
|
|
85
|
+
except OSError:
|
|
86
|
+
pass
|
|
87
|
+
tmp = target.parent / f".version.tmp.{os.getpid()}"
|
|
88
|
+
# Open with 0600 so the tmp is never world-readable either.
|
|
89
|
+
flags = os.O_WRONLY | os.O_CREAT | os.O_TRUNC
|
|
90
|
+
if hasattr(os, "O_NOFOLLOW"):
|
|
91
|
+
flags |= os.O_NOFOLLOW
|
|
92
|
+
fd = os.open(tmp, flags, 0o600)
|
|
93
|
+
try:
|
|
94
|
+
os.write(fd, (version + "\n").encode("ascii"))
|
|
95
|
+
finally:
|
|
96
|
+
os.close(fd)
|
|
97
|
+
os.replace(tmp, target)
|
|
98
|
+
if sys.platform != "win32":
|
|
99
|
+
try:
|
|
100
|
+
os.chmod(target, 0o600)
|
|
101
|
+
except OSError:
|
|
102
|
+
pass
|
|
103
|
+
return True
|
|
104
|
+
except OSError:
|
|
105
|
+
return False
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def _has_existing_db() -> bool:
|
|
109
|
+
"""True if a memory.db already exists — signals a pre-v3.4.26 user
|
|
110
|
+
upgrading in-place (the marker only began shipping in v3.4.26)."""
|
|
111
|
+
return (_data_dir() / "memory.db").exists()
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def _banner(prior: str | None, current: str) -> str:
|
|
115
|
+
header = (f"SuperLocalMemory upgraded from {prior} to {current}"
|
|
116
|
+
if prior else
|
|
117
|
+
f"SuperLocalMemory upgraded to {current} (from an earlier version)")
|
|
118
|
+
return "\n".join([
|
|
119
|
+
header,
|
|
120
|
+
" - Multi-IDE MCP processes now share a worker — large RAM drop",
|
|
121
|
+
" - Feedback and learning signals flow from every IDE to the daemon",
|
|
122
|
+
" - Silent data migration complete; no manual steps required",
|
|
123
|
+
"Run `slm doctor` to verify your setup.",
|
|
124
|
+
"",
|
|
125
|
+
])
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def check_and_emit_upgrade_banner(current: str) -> bool:
|
|
129
|
+
"""Print the banner once per upgrade boundary. Idempotent.
|
|
130
|
+
|
|
131
|
+
Returns True if the banner was emitted on this call, else False.
|
|
132
|
+
Never raises — swallows I/O errors so a broken data directory cannot
|
|
133
|
+
take down the CLI.
|
|
134
|
+
|
|
135
|
+
Concurrent CLI + daemon + npm-postinstall callers serialize through
|
|
136
|
+
an O_CREAT|O_EXCL lock file so at most one emits the banner.
|
|
137
|
+
"""
|
|
138
|
+
try:
|
|
139
|
+
prior = read_marker_version()
|
|
140
|
+
|
|
141
|
+
if prior == current:
|
|
142
|
+
return False
|
|
143
|
+
|
|
144
|
+
# Fresh install: no marker, no DB. Stay quiet — the setup wizard
|
|
145
|
+
# handles the welcome. Still write the marker so subsequent
|
|
146
|
+
# invocations are no-ops.
|
|
147
|
+
if prior is None and not _has_existing_db():
|
|
148
|
+
write_marker_version(current)
|
|
149
|
+
return False
|
|
150
|
+
|
|
151
|
+
# Serialize concurrent emitters via O_CREAT|O_EXCL lock file.
|
|
152
|
+
lock = _data_dir() / ".version-banner.lock"
|
|
153
|
+
lock_fd = None
|
|
154
|
+
try:
|
|
155
|
+
lock.parent.mkdir(parents=True, exist_ok=True)
|
|
156
|
+
lock_fd = os.open(
|
|
157
|
+
str(lock), os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600,
|
|
158
|
+
)
|
|
159
|
+
except FileExistsError:
|
|
160
|
+
return False
|
|
161
|
+
except OSError:
|
|
162
|
+
pass
|
|
163
|
+
|
|
164
|
+
try:
|
|
165
|
+
# Re-check after winning the lock — the loser may have
|
|
166
|
+
# already written the marker while we raced.
|
|
167
|
+
if read_marker_version() == current:
|
|
168
|
+
return False
|
|
169
|
+
|
|
170
|
+
sys.stdout.write(_banner(prior, current))
|
|
171
|
+
sys.stdout.flush()
|
|
172
|
+
write_marker_version(current)
|
|
173
|
+
return True
|
|
174
|
+
finally:
|
|
175
|
+
if lock_fd is not None:
|
|
176
|
+
os.close(lock_fd)
|
|
177
|
+
try:
|
|
178
|
+
lock.unlink(missing_ok=True)
|
|
179
|
+
except OSError:
|
|
180
|
+
pass
|
|
181
|
+
except Exception:
|
|
182
|
+
# Banner is advisory. A failure here must never propagate.
|
|
183
|
+
return False
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
|
|
2
|
+
# Licensed under AGPL-3.0-or-later - see LICENSE file
|
|
3
|
+
# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
|
|
4
|
+
|
|
5
|
+
"""Setup-wizard extensions for v3.4.26.
|
|
6
|
+
|
|
7
|
+
Keeps the user-facing surface tiny: one yes/no prompt for the queue.
|
|
8
|
+
Everything else defaults; advanced users edit ``v3426_options.json``.
|
|
9
|
+
|
|
10
|
+
Also validates the chosen data directory at install time so cloud-sync
|
|
11
|
+
folders (iCloud, Dropbox, OneDrive, …) fail loud BEFORE the user stores
|
|
12
|
+
their first memory.
|
|
13
|
+
"""
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import json
|
|
17
|
+
import logging
|
|
18
|
+
import os
|
|
19
|
+
import sys
|
|
20
|
+
from dataclasses import asdict, dataclass
|
|
21
|
+
from pathlib import Path
|
|
22
|
+
|
|
23
|
+
from superlocalmemory.core.safe_fs import SafeFsError, validate_data_dir
|
|
24
|
+
|
|
25
|
+
logger = logging.getLogger(__name__)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
# Defaults tuned for a commodity laptop with 4 concurrent IDEs
|
|
29
|
+
# (Claude Code, Cursor, Antigravity, VS Code). See the release notes
|
|
30
|
+
# for the reasoning behind the numbers.
|
|
31
|
+
_DEFAULT_QUEUE_ENABLED = True
|
|
32
|
+
_DEFAULT_RATE_PER_PID = 30
|
|
33
|
+
_DEFAULT_RATE_PER_AGENT = 10
|
|
34
|
+
_DEFAULT_RATE_GLOBAL = 100
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
@dataclass(frozen=True)
|
|
38
|
+
class V3426Options:
|
|
39
|
+
queue_enabled: bool
|
|
40
|
+
rate_limit_per_pid: int
|
|
41
|
+
rate_limit_per_agent: int
|
|
42
|
+
rate_limit_global: int
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _default_options() -> V3426Options:
|
|
46
|
+
return V3426Options(
|
|
47
|
+
queue_enabled=_DEFAULT_QUEUE_ENABLED,
|
|
48
|
+
rate_limit_per_pid=_DEFAULT_RATE_PER_PID,
|
|
49
|
+
rate_limit_per_agent=_DEFAULT_RATE_PER_AGENT,
|
|
50
|
+
rate_limit_global=_DEFAULT_RATE_GLOBAL,
|
|
51
|
+
)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def prompt_v3426_options(interactive: bool) -> V3426Options:
|
|
55
|
+
"""Return user-chosen (or default) v3.4.26 options.
|
|
56
|
+
|
|
57
|
+
Non-interactive / EOFError (pipe) returns the defaults silently.
|
|
58
|
+
Ctrl-C propagates so the user can actually abort install.
|
|
59
|
+
"""
|
|
60
|
+
if not interactive:
|
|
61
|
+
return _default_options()
|
|
62
|
+
|
|
63
|
+
try:
|
|
64
|
+
answer = input(
|
|
65
|
+
"Enable SLM's concurrent-recall queue (recommended)? [Y/n]: "
|
|
66
|
+
).strip().lower()
|
|
67
|
+
except EOFError:
|
|
68
|
+
return _default_options()
|
|
69
|
+
|
|
70
|
+
queue_enabled = answer not in ("n", "no")
|
|
71
|
+
return V3426Options(
|
|
72
|
+
queue_enabled=queue_enabled,
|
|
73
|
+
rate_limit_per_pid=_DEFAULT_RATE_PER_PID,
|
|
74
|
+
rate_limit_per_agent=_DEFAULT_RATE_PER_AGENT,
|
|
75
|
+
rate_limit_global=_DEFAULT_RATE_GLOBAL,
|
|
76
|
+
)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def validate_install_data_dir(path: Path) -> tuple[bool, str]:
|
|
80
|
+
"""Return (ok, reason). Empty reason on success."""
|
|
81
|
+
try:
|
|
82
|
+
validate_data_dir(path)
|
|
83
|
+
except SafeFsError as exc:
|
|
84
|
+
return False, str(exc)
|
|
85
|
+
return True, ""
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def persist_v3426_options(opts: V3426Options, home_dir: Path) -> Path:
|
|
89
|
+
"""Write the options JSON with 0600 perms and atomic replace.
|
|
90
|
+
|
|
91
|
+
Atomic: a crash mid-write leaves either the old file or nothing,
|
|
92
|
+
never a truncated JSON that the daemon can't parse on boot.
|
|
93
|
+
0600: rate-limit config is not sensitive per se, but the file
|
|
94
|
+
also carries future secrets — enforce the tight mode now.
|
|
95
|
+
"""
|
|
96
|
+
home_dir.mkdir(parents=True, exist_ok=True)
|
|
97
|
+
if sys.platform != "win32":
|
|
98
|
+
try:
|
|
99
|
+
os.chmod(home_dir, 0o700)
|
|
100
|
+
except OSError as exc:
|
|
101
|
+
logger.warning("could not tighten %s to 0700: %s", home_dir, exc)
|
|
102
|
+
target = home_dir / "v3426_options.json"
|
|
103
|
+
payload = (json.dumps(asdict(opts), indent=2) + "\n").encode("utf-8")
|
|
104
|
+
tmp = home_dir / f"v3426_options.json.tmp.{os.getpid()}"
|
|
105
|
+
flags = os.O_WRONLY | os.O_CREAT | os.O_TRUNC
|
|
106
|
+
if hasattr(os, "O_NOFOLLOW"):
|
|
107
|
+
flags |= os.O_NOFOLLOW
|
|
108
|
+
try:
|
|
109
|
+
fd = os.open(tmp, flags, 0o600)
|
|
110
|
+
try:
|
|
111
|
+
os.write(fd, payload)
|
|
112
|
+
finally:
|
|
113
|
+
os.close(fd)
|
|
114
|
+
os.replace(tmp, target)
|
|
115
|
+
if sys.platform != "win32":
|
|
116
|
+
try:
|
|
117
|
+
os.chmod(target, 0o600)
|
|
118
|
+
except OSError:
|
|
119
|
+
pass
|
|
120
|
+
except OSError:
|
|
121
|
+
# Fallback for odd filesystems that don't support O_NOFOLLOW
|
|
122
|
+
# or the perm bits — log + emit something rather than crash.
|
|
123
|
+
logger.warning(
|
|
124
|
+
"atomic write of %s failed; falling back to write_text", target,
|
|
125
|
+
)
|
|
126
|
+
target.write_text(
|
|
127
|
+
json.dumps(asdict(opts), indent=2) + "\n", encoding="utf-8",
|
|
128
|
+
)
|
|
129
|
+
return target
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
|
|
2
|
+
# Licensed under AGPL-3.0-or-later - see LICENSE file
|
|
3
|
+
# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
|
|
4
|
+
|
|
5
|
+
"""Wall-clock jump detector.
|
|
6
|
+
|
|
7
|
+
Part of Qualixar | Author: Varun Pratap Bhardwaj
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
from typing import Literal, Optional
|
|
13
|
+
|
|
14
|
+
Event = Literal["forward", "backward"]
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class ClockJumpDetector:
|
|
18
|
+
"""Detects NTP-style jumps by comparing wall-clock and monotonic deltas."""
|
|
19
|
+
|
|
20
|
+
def __init__(self, threshold_s: float = 1.0) -> None:
|
|
21
|
+
self._threshold = threshold_s
|
|
22
|
+
self._last_wall: Optional[float] = None
|
|
23
|
+
self._last_mono: Optional[float] = None
|
|
24
|
+
self.last_drift_s: float = 0.0
|
|
25
|
+
self.last_event: Optional[Event] = None
|
|
26
|
+
|
|
27
|
+
def tick(self, wall: float, monotonic: float) -> Optional[Event]:
|
|
28
|
+
if self._last_wall is None or self._last_mono is None:
|
|
29
|
+
self._last_wall = wall
|
|
30
|
+
self._last_mono = monotonic
|
|
31
|
+
return None
|
|
32
|
+
dw = wall - self._last_wall
|
|
33
|
+
dm = monotonic - self._last_mono
|
|
34
|
+
drift = dw - dm
|
|
35
|
+
self._last_wall = wall
|
|
36
|
+
self._last_mono = monotonic
|
|
37
|
+
self.last_drift_s = drift
|
|
38
|
+
if abs(drift) < self._threshold:
|
|
39
|
+
self.last_event = None
|
|
40
|
+
return None
|
|
41
|
+
self.last_event = "forward" if drift > 0 else "backward"
|
|
42
|
+
return self.last_event
|
|
43
|
+
|
|
44
|
+
def drift_magnitude_s(self) -> float:
|
|
45
|
+
return abs(self.last_drift_s)
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
|
|
2
|
+
# Licensed under AGPL-3.0-or-later - see LICENSE file
|
|
3
|
+
# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
|
|
4
|
+
|
|
5
|
+
"""Simple bounded SQLite connection pool.
|
|
6
|
+
|
|
7
|
+
Part of Qualixar | Author: Varun Pratap Bhardwaj
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import sqlite3
|
|
13
|
+
import threading
|
|
14
|
+
from contextlib import contextmanager
|
|
15
|
+
from queue import Empty, Queue
|
|
16
|
+
from typing import Callable, Iterator
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class ConnectionPool:
|
|
20
|
+
"""Bounded pool of SQLite connections created on-demand."""
|
|
21
|
+
|
|
22
|
+
def __init__(
|
|
23
|
+
self,
|
|
24
|
+
opener: Callable[[], sqlite3.Connection],
|
|
25
|
+
size: int = 8,
|
|
26
|
+
) -> None:
|
|
27
|
+
if size < 1:
|
|
28
|
+
raise ValueError("size must be >= 1")
|
|
29
|
+
self._opener = opener
|
|
30
|
+
self._size = size
|
|
31
|
+
self._available: Queue[sqlite3.Connection] = Queue(maxsize=size)
|
|
32
|
+
self._created = 0
|
|
33
|
+
self._lock = threading.Lock()
|
|
34
|
+
self._closed = False
|
|
35
|
+
self._all: list[sqlite3.Connection] = []
|
|
36
|
+
|
|
37
|
+
def _get_or_create(self, timeout: float | None) -> sqlite3.Connection:
|
|
38
|
+
try:
|
|
39
|
+
return self._available.get_nowait()
|
|
40
|
+
except Empty:
|
|
41
|
+
pass
|
|
42
|
+
with self._lock:
|
|
43
|
+
if self._closed:
|
|
44
|
+
raise RuntimeError("pool is closed")
|
|
45
|
+
if self._created < self._size:
|
|
46
|
+
conn = self._opener()
|
|
47
|
+
self._all.append(conn)
|
|
48
|
+
self._created += 1
|
|
49
|
+
return conn
|
|
50
|
+
# Pool saturated — block for a returned connection
|
|
51
|
+
try:
|
|
52
|
+
return self._available.get(timeout=timeout)
|
|
53
|
+
except Empty as exc:
|
|
54
|
+
raise TimeoutError("timed out acquiring DB connection") from exc
|
|
55
|
+
|
|
56
|
+
@contextmanager
|
|
57
|
+
def acquire(self, timeout: float | None = 30.0) -> Iterator[sqlite3.Connection]:
|
|
58
|
+
if self._closed:
|
|
59
|
+
raise RuntimeError("pool is closed")
|
|
60
|
+
conn = self._get_or_create(timeout)
|
|
61
|
+
try:
|
|
62
|
+
yield conn
|
|
63
|
+
finally:
|
|
64
|
+
if not self._closed:
|
|
65
|
+
self._available.put(conn)
|
|
66
|
+
|
|
67
|
+
def close(self) -> None:
|
|
68
|
+
with self._lock:
|
|
69
|
+
if self._closed:
|
|
70
|
+
return
|
|
71
|
+
self._closed = True
|
|
72
|
+
for conn in self._all:
|
|
73
|
+
try:
|
|
74
|
+
conn.close()
|
|
75
|
+
except Exception:
|
|
76
|
+
pass
|
|
77
|
+
self._all.clear()
|
|
78
|
+
|
|
79
|
+
def size(self) -> int:
|
|
80
|
+
return self._size
|
|
@@ -21,6 +21,7 @@ import logging
|
|
|
21
21
|
from typing import Any
|
|
22
22
|
|
|
23
23
|
from superlocalmemory.core.config import SLMConfig
|
|
24
|
+
from superlocalmemory.core.engine_capabilities import Capabilities, CapabilityError
|
|
24
25
|
from superlocalmemory.core.modes import get_capabilities
|
|
25
26
|
from superlocalmemory.storage.models import (
|
|
26
27
|
AtomicFact, MemoryRecord, Mode, RecallResponse,
|
|
@@ -46,9 +47,14 @@ class MemoryEngine:
|
|
|
46
47
|
response = engine.recall("Where did Alice go?")
|
|
47
48
|
"""
|
|
48
49
|
|
|
49
|
-
def __init__(
|
|
50
|
+
def __init__(
|
|
51
|
+
self,
|
|
52
|
+
config: SLMConfig,
|
|
53
|
+
capabilities: Capabilities = Capabilities.FULL,
|
|
54
|
+
) -> None:
|
|
50
55
|
self._config = config
|
|
51
56
|
self._caps = get_capabilities(config.mode)
|
|
57
|
+
self._capabilities = capabilities
|
|
52
58
|
self._profile_id = config.active_profile
|
|
53
59
|
self._initialized = False
|
|
54
60
|
|
|
@@ -99,20 +105,42 @@ class MemoryEngine:
|
|
|
99
105
|
"""Embedding service (read-only access for Phase 2+)."""
|
|
100
106
|
return self._embedder
|
|
101
107
|
|
|
108
|
+
@property
|
|
109
|
+
def capabilities(self) -> Capabilities:
|
|
110
|
+
"""Capability level chosen at construction (LIGHT or FULL)."""
|
|
111
|
+
return self._capabilities
|
|
112
|
+
|
|
102
113
|
# -- Initialization -----------------------------------------------------
|
|
103
114
|
|
|
104
115
|
def initialize(self) -> None:
|
|
105
|
-
"""Initialize all components. Call once before use.
|
|
116
|
+
"""Initialize all components. Call once before use.
|
|
117
|
+
|
|
118
|
+
In LIGHT mode only the DB layer is initialized (SQLite + schema
|
|
119
|
+
migrations + profile bookkeeping). In FULL mode the heavy layer
|
|
120
|
+
(embedder, LLM, encoding, retrieval, hooks, consolidation) follows.
|
|
121
|
+
"""
|
|
106
122
|
if self._initialized:
|
|
107
123
|
return
|
|
108
124
|
|
|
125
|
+
self._init_db_layer()
|
|
126
|
+
|
|
127
|
+
if self._capabilities is Capabilities.FULL:
|
|
128
|
+
self._init_heavy_layer()
|
|
129
|
+
|
|
130
|
+
self._initialized = True
|
|
131
|
+
logger.info(
|
|
132
|
+
"MemoryEngine initialized: mode=%s profile=%s capabilities=%s",
|
|
133
|
+
self._config.mode.value, self._profile_id,
|
|
134
|
+
self._capabilities.value,
|
|
135
|
+
)
|
|
136
|
+
|
|
137
|
+
if self._capabilities is Capabilities.FULL:
|
|
138
|
+
# Replay pending async writes only when heavy layer is available.
|
|
139
|
+
self._process_pending_memories()
|
|
140
|
+
|
|
141
|
+
def _init_db_layer(self) -> None:
|
|
109
142
|
from superlocalmemory.storage import schema
|
|
110
143
|
from superlocalmemory.storage.database import DatabaseManager
|
|
111
|
-
from superlocalmemory.llm.backbone import LLMBackbone
|
|
112
|
-
from superlocalmemory.core.engine_wiring import (
|
|
113
|
-
init_embedder, init_encoding, init_retrieval, wire_hooks,
|
|
114
|
-
_init_auto_invoker, _init_consolidation,
|
|
115
|
-
)
|
|
116
144
|
|
|
117
145
|
self._db = DatabaseManager(self._config.db_path)
|
|
118
146
|
self._db.initialize(schema)
|
|
@@ -123,35 +151,48 @@ class MemoryEngine:
|
|
|
123
151
|
from superlocalmemory.storage.schema_v343 import apply_v343_schema
|
|
124
152
|
apply_v343_schema(str(self._db.db_path))
|
|
125
153
|
except Exception as exc:
|
|
126
|
-
logger.
|
|
154
|
+
logger.warning("V3.4.3 schema migration failed: %s", exc)
|
|
127
155
|
|
|
128
156
|
# V3.4.6: Apply "Connected Brain" mesh enhancements (broadcast, project routing, offline queue)
|
|
129
157
|
try:
|
|
130
158
|
from superlocalmemory.storage.schema_v343 import apply_v346_schema
|
|
131
159
|
apply_v346_schema(str(self._db.db_path))
|
|
132
160
|
except Exception as exc:
|
|
133
|
-
logger.
|
|
161
|
+
logger.warning("V3.4.6 schema migration failed: %s", exc)
|
|
134
162
|
|
|
135
163
|
# V3.4.7: Apply "Learning Brain" schema (tool_events, behavioral_assertions)
|
|
136
164
|
try:
|
|
137
165
|
from superlocalmemory.storage.schema_v347 import apply_v347_schema
|
|
138
166
|
apply_v347_schema(str(self._db.db_path))
|
|
139
167
|
except Exception as exc:
|
|
140
|
-
logger.
|
|
168
|
+
logger.warning("V3.4.7 schema migration failed: %s", exc)
|
|
141
169
|
|
|
142
170
|
# V3.4.10: Apply "Fortress" schema (backup_destinations, entity_blacklist)
|
|
143
171
|
try:
|
|
144
172
|
from superlocalmemory.storage.schema_v3410 import apply_v3410_schema
|
|
145
173
|
apply_v3410_schema(str(self._db.db_path))
|
|
146
174
|
except Exception as exc:
|
|
147
|
-
logger.
|
|
175
|
+
logger.warning("V3.4.10 schema migration failed: %s", exc)
|
|
148
176
|
|
|
149
177
|
# V3.4.11: Apply "Scale-Ready" schema (pinned_facts, backend_status, fact_consolidations)
|
|
150
178
|
try:
|
|
151
179
|
from superlocalmemory.storage.schema_v3411 import apply_v3411_schema
|
|
152
180
|
apply_v3411_schema(str(self._db.db_path))
|
|
153
181
|
except Exception as exc:
|
|
154
|
-
logger.
|
|
182
|
+
logger.warning("V3.4.11 schema migration failed: %s", exc)
|
|
183
|
+
|
|
184
|
+
# DB-only learner — no embedder / LLM dependency. Available in
|
|
185
|
+
# LIGHT so MCP report_feedback and session_init phase counters
|
|
186
|
+
# work on the MCP process without loading the heavy layer.
|
|
187
|
+
from superlocalmemory.learning.adaptive import AdaptiveLearner
|
|
188
|
+
self._adaptive_learner = AdaptiveLearner(self._db)
|
|
189
|
+
|
|
190
|
+
def _init_heavy_layer(self) -> None:
|
|
191
|
+
from superlocalmemory.llm.backbone import LLMBackbone
|
|
192
|
+
from superlocalmemory.core.engine_wiring import (
|
|
193
|
+
init_embedder, init_encoding, init_retrieval, wire_hooks,
|
|
194
|
+
_init_auto_invoker, _init_consolidation,
|
|
195
|
+
)
|
|
155
196
|
|
|
156
197
|
self._embedder = init_embedder(self._config)
|
|
157
198
|
|
|
@@ -165,12 +206,10 @@ class MemoryEngine:
|
|
|
165
206
|
|
|
166
207
|
from superlocalmemory.trust.scorer import TrustScorer
|
|
167
208
|
from superlocalmemory.trust.provenance import ProvenanceTracker
|
|
168
|
-
from superlocalmemory.learning.adaptive import AdaptiveLearner
|
|
169
209
|
from superlocalmemory.compliance.eu_ai_act import EUAIActChecker
|
|
170
210
|
|
|
171
211
|
self._trust_scorer = TrustScorer(self._db)
|
|
172
212
|
|
|
173
|
-
# Encoding components
|
|
174
213
|
enc = init_encoding(
|
|
175
214
|
self._config, self._db, self._embedder, self._llm,
|
|
176
215
|
)
|
|
@@ -192,7 +231,6 @@ class MemoryEngine:
|
|
|
192
231
|
self._auto_linker = enc.get("auto_linker")
|
|
193
232
|
self._graph_analyzer = enc.get("graph_analyzer")
|
|
194
233
|
|
|
195
|
-
# Retrieval engine
|
|
196
234
|
self._retrieval_engine = init_retrieval(
|
|
197
235
|
self._config, self._db, self._embedder,
|
|
198
236
|
self._entity_resolver, self._trust_scorer,
|
|
@@ -200,10 +238,10 @@ class MemoryEngine:
|
|
|
200
238
|
)
|
|
201
239
|
|
|
202
240
|
self._provenance = ProvenanceTracker(self._db)
|
|
203
|
-
self._adaptive_learner
|
|
241
|
+
# self._adaptive_learner is initialized in _init_db_layer() because
|
|
242
|
+
# it depends only on the DB (no embedder / LLM); see note there.
|
|
204
243
|
self._compliance_checker = EUAIActChecker()
|
|
205
244
|
|
|
206
|
-
# Wire lifecycle hooks
|
|
207
245
|
hook_result = wire_hooks(
|
|
208
246
|
self._hooks, self._config, self._db,
|
|
209
247
|
self._trust_scorer, self._profile_id,
|
|
@@ -231,7 +269,6 @@ class MemoryEngine:
|
|
|
231
269
|
llm=getattr(self, "_llm", None), # v3.4.7: for CCQ worker
|
|
232
270
|
)
|
|
233
271
|
|
|
234
|
-
# V3.3: Check for embedding model migration on mode switch
|
|
235
272
|
self._check_embedding_migration()
|
|
236
273
|
|
|
237
274
|
# V3.3.13: Background maintenance scheduler (Langevin/Ebbinghaus/Sheaf)
|
|
@@ -245,16 +282,6 @@ class MemoryEngine:
|
|
|
245
282
|
except Exception as exc:
|
|
246
283
|
logger.debug("Maintenance scheduler init failed: %s", exc)
|
|
247
284
|
|
|
248
|
-
self._initialized = True
|
|
249
|
-
logger.info(
|
|
250
|
-
"MemoryEngine initialized: mode=%s profile=%s",
|
|
251
|
-
self._config.mode.value, self._profile_id,
|
|
252
|
-
)
|
|
253
|
-
|
|
254
|
-
# V3.3.21: Process any pending memories from failed async remember.
|
|
255
|
-
# Zero cost if no pending.db exists. Backward compatible.
|
|
256
|
-
self._process_pending_memories()
|
|
257
|
-
|
|
258
285
|
def _process_pending_memories(self) -> None:
|
|
259
286
|
"""Process pending memories from store-first async pattern.
|
|
260
287
|
|
|
@@ -295,6 +322,7 @@ class MemoryEngine:
|
|
|
295
322
|
metadata: dict[str, Any] | None = None,
|
|
296
323
|
) -> list[str]:
|
|
297
324
|
"""Store content and extract structured facts. Returns fact_ids."""
|
|
325
|
+
self._require_full("store")
|
|
298
326
|
self._ensure_init()
|
|
299
327
|
|
|
300
328
|
from superlocalmemory.core.store_pipeline import run_store
|
|
@@ -327,6 +355,7 @@ class MemoryEngine:
|
|
|
327
355
|
|
|
328
356
|
def store_fact_direct(self, fact: AtomicFact) -> str:
|
|
329
357
|
"""Store a pre-built fact with full enrichment."""
|
|
358
|
+
self._require_full("store_fact_direct")
|
|
330
359
|
self._ensure_init()
|
|
331
360
|
|
|
332
361
|
from superlocalmemory.core.store_pipeline import run_store_fact_direct
|
|
@@ -357,6 +386,7 @@ class MemoryEngine:
|
|
|
357
386
|
``put_nowait`` and the actual ``pending_outcomes`` INSERT runs
|
|
358
387
|
on a background worker.
|
|
359
388
|
"""
|
|
389
|
+
self._require_full("recall")
|
|
360
390
|
self._ensure_init()
|
|
361
391
|
|
|
362
392
|
pid = profile_id or self._profile_id
|
|
@@ -396,8 +426,14 @@ class MemoryEngine:
|
|
|
396
426
|
fact_ids=fact_ids,
|
|
397
427
|
query_id=getattr(response, "query_id", "") or "",
|
|
398
428
|
))
|
|
399
|
-
except Exception:
|
|
400
|
-
|
|
429
|
+
except Exception as _outcome_exc:
|
|
430
|
+
# Engagement-signal enqueue is non-blocking; recall
|
|
431
|
+
# correctness does not depend on it. Log so the failure
|
|
432
|
+
# is visible instead of silently losing learning signals.
|
|
433
|
+
logger.warning(
|
|
434
|
+
"outcome-queue enqueue failed (engagement signal lost): %s",
|
|
435
|
+
_outcome_exc,
|
|
436
|
+
)
|
|
401
437
|
|
|
402
438
|
return response
|
|
403
439
|
|
|
@@ -407,6 +443,7 @@ class MemoryEngine:
|
|
|
407
443
|
self, speaker_a: str, speaker_b: str,
|
|
408
444
|
) -> None:
|
|
409
445
|
"""Pre-create canonical entities for conversation speakers."""
|
|
446
|
+
self._require_full("create_speaker_entities")
|
|
410
447
|
self._ensure_init()
|
|
411
448
|
if self._entity_resolver:
|
|
412
449
|
self._entity_resolver.create_speaker_entities(
|
|
@@ -470,3 +507,11 @@ class MemoryEngine:
|
|
|
470
507
|
def _ensure_init(self) -> None:
|
|
471
508
|
if not self._initialized:
|
|
472
509
|
self.initialize()
|
|
510
|
+
|
|
511
|
+
def _require_full(self, operation: str) -> None:
|
|
512
|
+
if self._capabilities is not Capabilities.FULL:
|
|
513
|
+
raise CapabilityError(
|
|
514
|
+
f"{operation} requires a FULL MemoryEngine but this instance "
|
|
515
|
+
f"is LIGHT; route through WorkerPool (pool.{operation}) or "
|
|
516
|
+
f"construct MemoryEngine(config, capabilities=Capabilities.FULL)."
|
|
517
|
+
)
|