superlocalmemory 3.8.3 → 3.8.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +76 -0
- package/README.md +3 -2
- package/package.json +1 -1
- package/plugin/.claude-plugin/plugin.json +1 -1
- package/plugin/CLAUDE.md +3 -3
- package/plugin/agents/slm-governance-advisor.md +1 -1
- package/plugin/agents/slm-loop-runner.md +1 -1
- package/plugin/agents/slm-memory-advisor.md +1 -1
- package/plugin/agents/slm-optimize-advisor.md +1 -1
- package/plugin/requirements.txt +1 -1
- package/plugin/skills/slm-cache/SKILL.md +1 -1
- package/plugin/skills/slm-compress/SKILL.md +1 -1
- package/plugin/skills/slm-governance/SKILL.md +1 -1
- package/plugin/skills/slm-graph/SKILL.md +1 -1
- package/plugin/skills/slm-loop/SKILL.md +1 -1
- package/plugin/skills/slm-mesh/SKILL.md +1 -1
- package/plugin/skills/slm-profile/SKILL.md +1 -1
- package/plugin/skills/slm-recall/SKILL.md +1 -1
- package/plugin/skills/slm-remember/SKILL.md +1 -1
- package/plugin/skills/slm-scope/SKILL.md +1 -1
- package/plugin/skills/slm-session/SKILL.md +1 -1
- package/plugin/skills/slm-status/SKILL.md +1 -1
- package/plugin-src/rules/AGENTS.md +1 -1
- package/plugin-src/skills/slm-cache/SKILL.md +1 -1
- package/plugin-src/skills/slm-compress/SKILL.md +1 -1
- package/plugin-src/skills/slm-graph/SKILL.md +1 -1
- package/plugin-src/skills/slm-recall/SKILL.md +1 -1
- package/plugin-src/skills/slm-remember/SKILL.md +1 -1
- package/plugin-src/skills/slm-session/SKILL.md +1 -1
- package/plugin-src/skills/slm-status/SKILL.md +1 -1
- package/pyproject.toml +9 -4
- package/src/superlocalmemory/__init__.py +1 -1
- package/src/superlocalmemory/access/rbac.py +68 -76
- package/src/superlocalmemory/cli/commands.py +158 -404
- package/src/superlocalmemory/cli/ingest_cmd.py +11 -1
- package/src/superlocalmemory/cli/main.py +30 -0
- package/src/superlocalmemory/cli/pending_store.py +39 -14
- package/src/superlocalmemory/core/backend_orchestrator.py +93 -0
- package/src/superlocalmemory/core/component_registry.py +4 -2
- package/src/superlocalmemory/core/config.py +78 -0
- package/src/superlocalmemory/core/consolidation_engine.py +79 -73
- package/src/superlocalmemory/core/embeddings.py +33 -6
- package/src/superlocalmemory/core/engine.py +186 -60
- package/src/superlocalmemory/core/engine_ingestion.py +150 -63
- package/src/superlocalmemory/core/fact_consolidator.py +148 -30
- package/src/superlocalmemory/core/graph_pruner.py +436 -39
- package/src/superlocalmemory/core/ingestion_command.py +273 -32
- package/src/superlocalmemory/core/maintenance_scheduler.py +61 -1
- package/src/superlocalmemory/core/mutations.py +32 -10
- package/src/superlocalmemory/core/recall_pipeline.py +111 -74
- package/src/superlocalmemory/core/registry.py +5 -1
- package/src/superlocalmemory/core/remember_admission.py +152 -0
- package/src/superlocalmemory/core/remember_runtime.py +712 -0
- package/src/superlocalmemory/core/remote_mode.py +3 -1
- package/src/superlocalmemory/core/scale_engine.py +41 -18
- package/src/superlocalmemory/core/store_pipeline.py +18 -4
- package/src/superlocalmemory/encoding/entity_resolver.py +18 -11
- package/src/superlocalmemory/graph/cozo_backend.py +5 -5
- package/src/superlocalmemory/hooks/_outcome_common.py +9 -2
- package/src/superlocalmemory/hooks/adapter_base.py +58 -44
- package/src/superlocalmemory/hooks/ide_connector.py +26 -8
- package/src/superlocalmemory/hooks/portable_kit.py +105 -9
- package/src/superlocalmemory/hooks/prewarm_auth.py +21 -2
- package/src/superlocalmemory/infra/auth_middleware.py +3 -1
- package/src/superlocalmemory/infra/cloud_backup.py +26 -27
- package/src/superlocalmemory/infra/event_bus.py +250 -88
- package/src/superlocalmemory/learning/bandit.py +50 -1
- package/src/superlocalmemory/learning/consolidation_cycle.py +33 -16
- package/src/superlocalmemory/learning/entity_compiler.py +148 -132
- package/src/superlocalmemory/learning/memory_merge.py +97 -82
- package/src/superlocalmemory/learning/reward_archive.py +98 -90
- package/src/superlocalmemory/learning/reward_boost.py +40 -30
- package/src/superlocalmemory/learning/source_quality.py +38 -35
- package/src/superlocalmemory/mcp/_daemon_proxy.py +38 -15
- package/src/superlocalmemory/mcp/http_transport.py +335 -3
- package/src/superlocalmemory/mcp/tools_active.py +4 -41
- package/src/superlocalmemory/mcp/tools_core.py +26 -87
- package/src/superlocalmemory/mcp/tools_evolution.py +5 -10
- package/src/superlocalmemory/optimize/proxy/capture.py +196 -8
- package/src/superlocalmemory/retrieval/engine.py +15 -4
- package/src/superlocalmemory/retrieval/entity_channel.py +25 -1
- package/src/superlocalmemory/retrieval/reranker.py +130 -22
- package/src/superlocalmemory/retrieval/spreading_activation.py +20 -12
- package/src/superlocalmemory/retrieval/vector_store.py +84 -69
- package/src/superlocalmemory/server/loopback.py +85 -0
- package/src/superlocalmemory/server/origin.py +9 -4
- package/src/superlocalmemory/server/profile_runtime.py +14 -0
- package/src/superlocalmemory/server/routes/abstraction.py +2 -4
- package/src/superlocalmemory/server/routes/agents.py +3 -5
- package/src/superlocalmemory/server/routes/backup.py +6 -2
- package/src/superlocalmemory/server/routes/behavioral.py +11 -25
- package/src/superlocalmemory/server/routes/brain.py +6 -9
- package/src/superlocalmemory/server/routes/compliance.py +20 -23
- package/src/superlocalmemory/server/routes/config_api.py +83 -0
- package/src/superlocalmemory/server/routes/entity.py +3 -7
- package/src/superlocalmemory/server/routes/evolution.py +3 -5
- package/src/superlocalmemory/server/routes/helpers.py +57 -25
- package/src/superlocalmemory/server/routes/insights.py +2 -4
- package/src/superlocalmemory/server/routes/learning.py +2 -5
- package/src/superlocalmemory/server/routes/lifecycle.py +2 -4
- package/src/superlocalmemory/server/routes/memories.py +119 -98
- package/src/superlocalmemory/server/routes/mesh.py +7 -2
- package/src/superlocalmemory/server/routes/profiles.py +20 -21
- package/src/superlocalmemory/server/routes/rbac.py +0 -1
- package/src/superlocalmemory/server/routes/tiers.py +28 -35
- package/src/superlocalmemory/server/routes/timeline.py +2 -4
- package/src/superlocalmemory/server/routes/v3_api.py +85 -93
- package/src/superlocalmemory/server/unified_daemon.py +400 -140
- package/src/superlocalmemory/server/write_identity.py +22 -4
- package/src/superlocalmemory/storage/admission_codec.py +119 -0
- package/src/superlocalmemory/storage/admission_journal.py +728 -0
- package/src/superlocalmemory/storage/database.py +168 -19
- package/src/superlocalmemory/storage/deferred_writes.py +209 -0
- package/src/superlocalmemory/storage/embedding_migrator.py +19 -0
- package/src/superlocalmemory/storage/memory_write.py +115 -0
- package/src/superlocalmemory/storage/migration_runner.py +44 -0
- package/src/superlocalmemory/storage/migrations/M028_fact_entity_associations.py +113 -78
- package/src/superlocalmemory/storage/migrations/M031_dead_letter_operations.py +80 -0
- package/src/superlocalmemory/storage/migrations/M032_write_coordinator_admission.py +188 -0
- package/src/superlocalmemory/storage/read_connection.py +115 -0
- package/src/superlocalmemory/storage/write_coordinator.py +756 -0
- package/src/superlocalmemory/storage/write_lock.py +88 -0
- package/src/superlocalmemory/ui/index.html +1 -1
- package/src/superlocalmemory/ui/js/auto-settings.js +14 -1
- package/src/superlocalmemory/ui/js/od-settings.js +9 -3
|
@@ -32,6 +32,7 @@ from __future__ import annotations
|
|
|
32
32
|
import hashlib
|
|
33
33
|
import hmac
|
|
34
34
|
import logging
|
|
35
|
+
import os
|
|
35
36
|
import secrets
|
|
36
37
|
import sqlite3
|
|
37
38
|
import uuid
|
|
@@ -39,9 +40,14 @@ from datetime import datetime, timedelta, timezone
|
|
|
39
40
|
from enum import Enum
|
|
40
41
|
from pathlib import Path
|
|
41
42
|
|
|
43
|
+
from superlocalmemory.storage.memory_write import memory_write
|
|
44
|
+
|
|
42
45
|
logger = logging.getLogger("superlocalmemory.access.rbac")
|
|
43
46
|
|
|
44
|
-
|
|
47
|
+
# Match the daemon's default (SLM_DB_BUSY_TIMEOUT_MS, default 10 000 ms).
|
|
48
|
+
# Reads use this via _conn(); writes go through memory_write() which reads the
|
|
49
|
+
# same env var internally, so in-process writers never see SQLITE_BUSY.
|
|
50
|
+
_BUSY_TIMEOUT_MS: int = max(0, int(os.environ.get("SLM_DB_BUSY_TIMEOUT_MS", "10000")))
|
|
45
51
|
_SESSION_TTL_HOURS = 12
|
|
46
52
|
|
|
47
53
|
# scrypt cost parameters (OWASP-recommended interactive-login range).
|
|
@@ -155,16 +161,15 @@ class RbacEngine:
|
|
|
155
161
|
raise RbacError("Username is required.")
|
|
156
162
|
pw_hash = _hash_password(password)
|
|
157
163
|
user_id = _uid()
|
|
158
|
-
|
|
159
|
-
|
|
164
|
+
# memory_write: process write lock + busy_timeout.
|
|
165
|
+
with memory_write(self._db_path) as conn:
|
|
160
166
|
# The UNIQUE(username) constraint is the source of truth; the SELECT
|
|
161
167
|
# is a fast path. Two concurrent creates can both pass the SELECT, so
|
|
162
168
|
# the loser's INSERT hits the constraint — convert that to RbacError
|
|
163
169
|
# (409) instead of leaking a raw IntegrityError as a 500.
|
|
164
|
-
|
|
170
|
+
if conn.execute(
|
|
165
171
|
"SELECT 1 FROM rbac_users WHERE username=?", (username,)
|
|
166
|
-
).fetchone()
|
|
167
|
-
if exists:
|
|
172
|
+
).fetchone():
|
|
168
173
|
raise RbacError(f"User '{username}' already exists.")
|
|
169
174
|
try:
|
|
170
175
|
conn.execute(
|
|
@@ -174,19 +179,15 @@ class RbacEngine:
|
|
|
174
179
|
(user_id, username, display_name or username, pw_hash,
|
|
175
180
|
_now(), created_by),
|
|
176
181
|
)
|
|
177
|
-
conn.commit()
|
|
178
182
|
except sqlite3.IntegrityError:
|
|
179
183
|
raise RbacError(f"User '{username}' already exists.")
|
|
180
|
-
finally:
|
|
181
|
-
conn.close()
|
|
182
184
|
logger.info("RBAC: created user '%s' (%s)", username, user_id)
|
|
183
185
|
return {"user_id": user_id, "username": username,
|
|
184
186
|
"display_name": display_name or username, "status": "active"}
|
|
185
187
|
|
|
186
188
|
def set_password(self, user_id: str, password: str) -> None:
|
|
187
189
|
pw_hash = _hash_password(password)
|
|
188
|
-
|
|
189
|
-
try:
|
|
190
|
+
with memory_write(self._db_path) as conn:
|
|
190
191
|
cur = conn.execute(
|
|
191
192
|
"UPDATE rbac_users SET password_hash=? WHERE user_id=?",
|
|
192
193
|
(pw_hash, user_id),
|
|
@@ -195,15 +196,11 @@ class RbacEngine:
|
|
|
195
196
|
raise RbacError("User not found.")
|
|
196
197
|
# A password change invalidates every existing session.
|
|
197
198
|
conn.execute("DELETE FROM rbac_sessions WHERE user_id=?", (user_id,))
|
|
198
|
-
conn.commit()
|
|
199
|
-
finally:
|
|
200
|
-
conn.close()
|
|
201
199
|
|
|
202
200
|
def set_status(self, user_id: str, status: str) -> None:
|
|
203
201
|
if status not in ("active", "disabled"):
|
|
204
202
|
raise RbacError("status must be 'active' or 'disabled'.")
|
|
205
|
-
|
|
206
|
-
try:
|
|
203
|
+
with memory_write(self._db_path) as conn:
|
|
207
204
|
cur = conn.execute(
|
|
208
205
|
"UPDATE rbac_users SET status=? WHERE user_id=?", (status, user_id)
|
|
209
206
|
)
|
|
@@ -211,21 +208,14 @@ class RbacEngine:
|
|
|
211
208
|
raise RbacError("User not found.")
|
|
212
209
|
if status == "disabled":
|
|
213
210
|
conn.execute("DELETE FROM rbac_sessions WHERE user_id=?", (user_id,))
|
|
214
|
-
conn.commit()
|
|
215
|
-
finally:
|
|
216
|
-
conn.close()
|
|
217
211
|
|
|
218
212
|
def delete_user(self, user_id: str) -> None:
|
|
219
|
-
|
|
220
|
-
try:
|
|
213
|
+
with memory_write(self._db_path) as conn:
|
|
221
214
|
conn.execute("DELETE FROM rbac_sessions WHERE user_id=?", (user_id,))
|
|
222
215
|
conn.execute("DELETE FROM rbac_memberships WHERE user_id=?", (user_id,))
|
|
223
216
|
cur = conn.execute("DELETE FROM rbac_users WHERE user_id=?", (user_id,))
|
|
224
217
|
if cur.rowcount == 0:
|
|
225
218
|
raise RbacError("User not found.")
|
|
226
|
-
conn.commit()
|
|
227
|
-
finally:
|
|
228
|
-
conn.close()
|
|
229
219
|
|
|
230
220
|
def get_user(self, user_id: str) -> dict | None:
|
|
231
221
|
conn = self._conn()
|
|
@@ -277,25 +267,35 @@ class RbacEngine:
|
|
|
277
267
|
raw = secrets.token_urlsafe(32)
|
|
278
268
|
now = datetime.now(timezone.utc)
|
|
279
269
|
expires = (now + timedelta(hours=ttl_hours)).isoformat()
|
|
280
|
-
|
|
281
|
-
try:
|
|
270
|
+
with memory_write(self._db_path) as conn:
|
|
282
271
|
conn.execute(
|
|
283
272
|
"INSERT INTO rbac_sessions (token_hash, user_id, created_at, "
|
|
284
273
|
"expires_at, last_seen) VALUES (?, ?, ?, ?, ?)",
|
|
285
274
|
(_hash_token(raw), user_id, now.isoformat(), expires, now.isoformat()),
|
|
286
275
|
)
|
|
287
|
-
conn.commit()
|
|
288
|
-
finally:
|
|
289
|
-
conn.close()
|
|
290
276
|
return raw
|
|
291
277
|
|
|
292
278
|
def resolve_session(self, raw_token: str) -> dict | None:
|
|
293
279
|
"""Return the active user for a session token, or None if invalid /
|
|
294
|
-
expired / disabled. Updates last_seen. Constant-time by hash lookup.
|
|
280
|
+
expired / disabled. Updates last_seen. Constant-time by hash lookup.
|
|
281
|
+
|
|
282
|
+
Concurrency design
|
|
283
|
+
------------------
|
|
284
|
+
Phase 1 (read): WAL allows concurrent readers without blocking the
|
|
285
|
+
writer — ``_conn()`` with ``busy_timeout`` only. The write lock is
|
|
286
|
+
NOT taken here so a burst of concurrent requests doesn't serialise
|
|
287
|
+
through the single writer on every call.
|
|
288
|
+
|
|
289
|
+
Phase 2 (conditional write): only if the session is expired/stale.
|
|
290
|
+
``memory_write()`` is opened AFTER the read connection is closed
|
|
291
|
+
(lock-ordering invariant: get_write_lock is outermost).
|
|
292
|
+
"""
|
|
295
293
|
if not raw_token:
|
|
296
294
|
return None
|
|
297
295
|
token_hash = _hash_token(raw_token)
|
|
298
296
|
now = datetime.now(timezone.utc)
|
|
297
|
+
|
|
298
|
+
# -- Phase 1: read-only lookup ----------------------------------------
|
|
299
299
|
conn = self._conn()
|
|
300
300
|
try:
|
|
301
301
|
row = conn.execute(
|
|
@@ -310,48 +310,52 @@ class RbacEngine:
|
|
|
310
310
|
expired = datetime.fromisoformat(row["expires_at"]) <= now
|
|
311
311
|
except ValueError:
|
|
312
312
|
expired = True
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
313
|
+
row_status = row["status"]
|
|
314
|
+
row_last_seen = row["last_seen"]
|
|
315
|
+
user_dict = {
|
|
316
|
+
"user_id": row["user_id"],
|
|
317
|
+
"username": row["username"],
|
|
318
|
+
"display_name": row["display_name"],
|
|
319
|
+
}
|
|
320
|
+
finally:
|
|
321
|
+
conn.close()
|
|
322
|
+
|
|
323
|
+
# -- Phase 2: write if needed (read conn closed above) ----------------
|
|
324
|
+
if expired or row_status != "active":
|
|
325
|
+
with memory_write(self._db_path) as wconn:
|
|
326
|
+
wconn.execute(
|
|
327
|
+
"DELETE FROM rbac_sessions WHERE token_hash=?", (token_hash,)
|
|
328
|
+
)
|
|
329
|
+
return None
|
|
330
|
+
|
|
331
|
+
# Debounce last_seen: only write when stale (>60 s), so a burst of
|
|
332
|
+
# authenticated requests does not serialise through the writer every call.
|
|
333
|
+
stale = True
|
|
334
|
+
try:
|
|
335
|
+
stale = (now - datetime.fromisoformat(row_last_seen)).total_seconds() > 60
|
|
336
|
+
except (ValueError, TypeError, KeyError):
|
|
320
337
|
stale = True
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
stale = True
|
|
325
|
-
if stale:
|
|
326
|
-
conn.execute(
|
|
338
|
+
if stale:
|
|
339
|
+
with memory_write(self._db_path) as wconn:
|
|
340
|
+
wconn.execute(
|
|
327
341
|
"UPDATE rbac_sessions SET last_seen=? WHERE token_hash=?",
|
|
328
342
|
(now.isoformat(), token_hash),
|
|
329
343
|
)
|
|
330
|
-
|
|
331
|
-
return {"user_id": row["user_id"], "username": row["username"],
|
|
332
|
-
"display_name": row["display_name"]}
|
|
333
|
-
finally:
|
|
334
|
-
conn.close()
|
|
344
|
+
return user_dict
|
|
335
345
|
|
|
336
346
|
def revoke_session(self, raw_token: str) -> None:
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
finally:
|
|
343
|
-
conn.close()
|
|
347
|
+
with memory_write(self._db_path) as conn:
|
|
348
|
+
conn.execute(
|
|
349
|
+
"DELETE FROM rbac_sessions WHERE token_hash=?",
|
|
350
|
+
(_hash_token(raw_token),),
|
|
351
|
+
)
|
|
344
352
|
|
|
345
353
|
def purge_expired_sessions(self) -> int:
|
|
346
|
-
|
|
347
|
-
try:
|
|
354
|
+
with memory_write(self._db_path) as conn:
|
|
348
355
|
cur = conn.execute(
|
|
349
356
|
"DELETE FROM rbac_sessions WHERE expires_at <= ?", (_now(),)
|
|
350
357
|
)
|
|
351
|
-
|
|
352
|
-
return cur.rowcount or 0
|
|
353
|
-
finally:
|
|
354
|
-
conn.close()
|
|
358
|
+
return cur.rowcount or 0
|
|
355
359
|
|
|
356
360
|
# -- memberships ------------------------------------------------------
|
|
357
361
|
|
|
@@ -361,8 +365,7 @@ class RbacEngine:
|
|
|
361
365
|
role_enum = Role(role)
|
|
362
366
|
except ValueError:
|
|
363
367
|
raise RbacError(f"Invalid role '{role}'. Use admin/member/viewer.")
|
|
364
|
-
|
|
365
|
-
try:
|
|
368
|
+
with memory_write(self._db_path) as conn:
|
|
366
369
|
if not conn.execute(
|
|
367
370
|
"SELECT 1 FROM rbac_users WHERE user_id=?", (user_id,)
|
|
368
371
|
).fetchone():
|
|
@@ -374,21 +377,14 @@ class RbacEngine:
|
|
|
374
377
|
"added_at=excluded.added_at, added_by=excluded.added_by",
|
|
375
378
|
(profile_id, user_id, role_enum.value, _now(), added_by),
|
|
376
379
|
)
|
|
377
|
-
conn.commit()
|
|
378
|
-
finally:
|
|
379
|
-
conn.close()
|
|
380
380
|
return {"profile_id": profile_id, "user_id": user_id, "role": role_enum.value}
|
|
381
381
|
|
|
382
382
|
def remove_membership(self, profile_id: str, user_id: str) -> None:
|
|
383
|
-
|
|
384
|
-
try:
|
|
383
|
+
with memory_write(self._db_path) as conn:
|
|
385
384
|
conn.execute(
|
|
386
385
|
"DELETE FROM rbac_memberships WHERE profile_id=? AND user_id=?",
|
|
387
386
|
(profile_id, user_id),
|
|
388
387
|
)
|
|
389
|
-
conn.commit()
|
|
390
|
-
finally:
|
|
391
|
-
conn.close()
|
|
392
388
|
|
|
393
389
|
def get_role(self, user_id: str, profile_id: str) -> Role | None:
|
|
394
390
|
conn = self._conn()
|
|
@@ -442,16 +438,12 @@ class RbacEngine:
|
|
|
442
438
|
conn.close()
|
|
443
439
|
|
|
444
440
|
def set_policy(self, key: str, value: str) -> None:
|
|
445
|
-
|
|
446
|
-
try:
|
|
441
|
+
with memory_write(self._db_path) as conn:
|
|
447
442
|
conn.execute(
|
|
448
443
|
"INSERT INTO rbac_settings (key, value) VALUES (?, ?) "
|
|
449
444
|
"ON CONFLICT(key) DO UPDATE SET value=excluded.value",
|
|
450
445
|
(key, value),
|
|
451
446
|
)
|
|
452
|
-
conn.commit()
|
|
453
|
-
finally:
|
|
454
|
-
conn.close()
|
|
455
447
|
|
|
456
448
|
def require_login(self) -> bool:
|
|
457
449
|
"""Company mode: mutations require a valid user session (no owner
|