superlocalmemory 4.0.5 → 4.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +57 -0
- package/README.md +10 -6
- package/package.json +3 -1
- package/pyproject.toml +1 -1
- package/src/superlocalmemory/__init__.py +1 -1
- package/src/superlocalmemory/access/rbac.py +106 -0
- package/src/superlocalmemory/brain/truth.py +80 -10
- package/src/superlocalmemory/cli/__main__.py +17 -0
- package/src/superlocalmemory/cli/commands.py +14 -3
- package/src/superlocalmemory/cli/gdpr_cmd.py +779 -0
- package/src/superlocalmemory/cli/gdpr_io.py +109 -0
- package/src/superlocalmemory/cli/main.py +76 -0
- package/src/superlocalmemory/code_graph/extractors/__init__.py +17 -0
- package/src/superlocalmemory/code_graph/graph_store.py +180 -3
- package/src/superlocalmemory/code_graph/parser.py +280 -100
- package/src/superlocalmemory/compliance/gdpr.py +358 -0
- package/src/superlocalmemory/core/config.py +44 -1
- package/src/superlocalmemory/core/engine_wiring.py +5 -1
- package/src/superlocalmemory/core/maintenance.py +43 -1
- package/src/superlocalmemory/core/recall_worker.py +33 -12
- package/src/superlocalmemory/infra/backup.py +138 -0
- package/src/superlocalmemory/infra/backup_obligations.py +423 -0
- package/src/superlocalmemory/learning/engagement.py +165 -0
- package/src/superlocalmemory/mcp/tools_code_graph.py +31 -4
- package/src/superlocalmemory/mcp/tools_v3.py +20 -6
- package/src/superlocalmemory/retrieval/engine.py +21 -0
- package/src/superlocalmemory/retrieval/remote_reranker.py +108 -11
- package/src/superlocalmemory/server/routes/brain.py +283 -15
- package/src/superlocalmemory/server/routes/learning.py +13 -25
- package/src/superlocalmemory/server/routes/v3_api.py +171 -60
- package/src/superlocalmemory/storage/database.py +36 -0
- package/src/superlocalmemory/storage/models.py +12 -4
- package/src/superlocalmemory/summaries/__init__.py +37 -0
- package/src/superlocalmemory/summaries/base.py +108 -0
- package/src/superlocalmemory/summaries/daily_reflection.py +293 -0
- package/src/superlocalmemory/summaries/project_work_log.py +424 -0
- package/src/superlocalmemory/summaries/session_summary.py +307 -0
- package/src/superlocalmemory/ui/css/design-system.css +76 -1
- package/src/superlocalmemory/ui/index.html +28 -11
- package/src/superlocalmemory/ui/js/od-agents.js +49 -5
- package/src/superlocalmemory/ui/js/od-brain.js +257 -77
- package/src/superlocalmemory/ui/js/od-graph.js +147 -6
|
@@ -14,11 +14,65 @@ Part of Qualixar | Author: Varun Pratap Bhardwaj
|
|
|
14
14
|
from __future__ import annotations
|
|
15
15
|
|
|
16
16
|
import logging
|
|
17
|
+
import sqlite3
|
|
17
18
|
from datetime import UTC, datetime
|
|
18
19
|
from pathlib import Path
|
|
19
20
|
|
|
20
21
|
logger = logging.getLogger(__name__)
|
|
21
22
|
|
|
23
|
+
# C1 — Backup residue obligations
|
|
24
|
+
# Imported lazily inside methods to avoid circular-import risk at module load.
|
|
25
|
+
# The sentinel guards against environments where infra.backup_obligations is
|
|
26
|
+
# unavailable (e.g. minimal test installs); compliance logic degrades safely.
|
|
27
|
+
_BACKUP_OBLIGATIONS_AVAILABLE: bool | None = None
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _get_backup_obligations_module():
|
|
31
|
+
"""Lazy import guard — returns module or None if unavailable."""
|
|
32
|
+
global _BACKUP_OBLIGATIONS_AVAILABLE
|
|
33
|
+
try:
|
|
34
|
+
import superlocalmemory.infra.backup_obligations as _m
|
|
35
|
+
_BACKUP_OBLIGATIONS_AVAILABLE = True
|
|
36
|
+
return _m
|
|
37
|
+
except Exception as exc: # noqa: BLE001
|
|
38
|
+
if _BACKUP_OBLIGATIONS_AVAILABLE is None:
|
|
39
|
+
logger.warning("backup_obligations module unavailable: %s", exc)
|
|
40
|
+
_BACKUP_OBLIGATIONS_AVAILABLE = False
|
|
41
|
+
return None
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _retention_days_from_config() -> int:
|
|
45
|
+
"""Read the configured obligation retention window (default 90 days).
|
|
46
|
+
|
|
47
|
+
Consumes ``SLMConfig.backup_retention_days`` or any attribute matching
|
|
48
|
+
('retention', 'retain') with numeric value on ``SLMConfig``. The config
|
|
49
|
+
field is owned by another agent; we consume it here and fall back to 90.
|
|
50
|
+
"""
|
|
51
|
+
try:
|
|
52
|
+
from superlocalmemory.core.config import SLMConfig
|
|
53
|
+
cfg = SLMConfig()
|
|
54
|
+
# Try the canonical field name first
|
|
55
|
+
for attr in ("backup_retention_days", "obligation_retention_days",
|
|
56
|
+
"retention_window_days", "backup_obligation_retention_days"):
|
|
57
|
+
val = getattr(cfg, attr, None)
|
|
58
|
+
if isinstance(val, int) and val > 0:
|
|
59
|
+
return val
|
|
60
|
+
# Fallback: search nested sub-configs
|
|
61
|
+
for attr in dir(cfg):
|
|
62
|
+
if attr.startswith("_"):
|
|
63
|
+
continue
|
|
64
|
+
sub = getattr(cfg, attr, None)
|
|
65
|
+
if not hasattr(sub, "__dict__") and not hasattr(sub, "__dataclass_fields__"):
|
|
66
|
+
continue
|
|
67
|
+
for sub_attr in dir(sub):
|
|
68
|
+
if any(h in sub_attr.lower() for h in ("retention", "retain")):
|
|
69
|
+
v = getattr(sub, sub_attr, None)
|
|
70
|
+
if isinstance(v, int) and v > 0:
|
|
71
|
+
return v
|
|
72
|
+
except Exception: # noqa: BLE001
|
|
73
|
+
pass
|
|
74
|
+
return 90 # owner-set default
|
|
75
|
+
|
|
22
76
|
# Friendly export keys → canonical table names (stable Art.20 export contract).
|
|
23
77
|
_EXPORT_ALIASES = {
|
|
24
78
|
"facts": "atomic_facts",
|
|
@@ -238,6 +292,13 @@ class GDPRCompliance:
|
|
|
238
292
|
except Exception:
|
|
239
293
|
data["profile_record"] = []
|
|
240
294
|
|
|
295
|
+
# C2 — include code_graph.db in Art.15 export (repo paths, file names,
|
|
296
|
+
# and symbol names are identifying data in a work context).
|
|
297
|
+
if self._data_root is not None:
|
|
298
|
+
code_graph_data = self._export_code_graph(self._data_root)
|
|
299
|
+
if code_graph_data is not None:
|
|
300
|
+
data["code_graph"] = code_graph_data
|
|
301
|
+
|
|
241
302
|
# total_items counts the canonical (table-name) keys only, before
|
|
242
303
|
# friendly aliases are added, so it is not double-counted.
|
|
243
304
|
data["total_items"] = sum(len(v) for v in data.values() if isinstance(v, list))
|
|
@@ -437,6 +498,16 @@ class GDPRCompliance:
|
|
|
437
498
|
counts["receipt_error"] = str(exc)
|
|
438
499
|
raise
|
|
439
500
|
|
|
501
|
+
# C2 — erase code_graph.db before main profile rows (fail-closed: a
|
|
502
|
+
# code_graph failure is logged but does NOT abort the erasure; the
|
|
503
|
+
# graph is installation-level personal data with no profile_id column,
|
|
504
|
+
# so it is wiped entirely on any Art.17 request).
|
|
505
|
+
if data_root is not None:
|
|
506
|
+
code_graph_result = self._erase_code_graph(data_root)
|
|
507
|
+
counts["code_graph"] = code_graph_result.get("rows_deleted", 0)
|
|
508
|
+
if code_graph_result.get("error"):
|
|
509
|
+
counts["code_graph_failed"] = 1
|
|
510
|
+
|
|
440
511
|
# Purge the learning sidecar *before* removing memory/profile rows. A
|
|
441
512
|
# learning failure is retryable and must leave the profile intact; the
|
|
442
513
|
# former best-effort-after-delete ordering could orphan receipts.
|
|
@@ -517,6 +588,45 @@ class GDPRCompliance:
|
|
|
517
588
|
counts["residue_rows"] = residue_rows
|
|
518
589
|
if residue_recount_failed:
|
|
519
590
|
counts["residue_recount_failed"] = 1
|
|
591
|
+
|
|
592
|
+
# I7 — post-erasure residue sweep: FTS shadow tables + WAL sanity.
|
|
593
|
+
# This makes I7 enforceable rather than aspirational.
|
|
594
|
+
self._scan_fts_residue(profile_id, tables, counts)
|
|
595
|
+
self._scan_wal_residue(counts)
|
|
596
|
+
|
|
597
|
+
# C1 — record outstanding obligations against backup snapshots.
|
|
598
|
+
# Done AFTER the main-DB residue scan so counts reflect live-store state
|
|
599
|
+
# before we tally the backups outstanding obligation count.
|
|
600
|
+
# Fail-closed: if the scan itself errors, set backup_scan_failed so
|
|
601
|
+
# completeness cannot be claimed.
|
|
602
|
+
backup_obligations_pending = 0
|
|
603
|
+
if data_root is not None:
|
|
604
|
+
try:
|
|
605
|
+
backup_obligations_pending = self._record_backup_obligations(
|
|
606
|
+
data_root=data_root,
|
|
607
|
+
profile_id=profile_id,
|
|
608
|
+
erasure_id=_uuid.uuid4().hex, # unique id for this obligation batch
|
|
609
|
+
counts=counts,
|
|
610
|
+
)
|
|
611
|
+
counts["backup_obligations_pending"] = backup_obligations_pending
|
|
612
|
+
except Exception as exc:
|
|
613
|
+
logger.error(
|
|
614
|
+
"GDPR erase: backup obligation recording FAILED: %s — "
|
|
615
|
+
"setting backup_scan_failed to block completeness",
|
|
616
|
+
exc,
|
|
617
|
+
)
|
|
618
|
+
counts["backup_scan_failed"] = 1
|
|
619
|
+
else:
|
|
620
|
+
# Cannot scan backups without data_root — treat as outstanding
|
|
621
|
+
# obligation so completeness is blocked.
|
|
622
|
+
counts["backup_obligations_pending"] = 0 # unknown but not confirmed clean
|
|
623
|
+
# We won't block completeness when data_root is unknown (legacy wrapper)
|
|
624
|
+
# but we do log the gap.
|
|
625
|
+
logger.warning(
|
|
626
|
+
"GDPR erase: backup obligation scan skipped — data_root unknown. "
|
|
627
|
+
"Backup snapshots may still contain the erased profile's data."
|
|
628
|
+
)
|
|
629
|
+
|
|
520
630
|
counts["erasure_complete"] = (
|
|
521
631
|
1
|
|
522
632
|
if (
|
|
@@ -528,6 +638,9 @@ class GDPRCompliance:
|
|
|
528
638
|
and not counts.get("vector_store_failures")
|
|
529
639
|
and not counts.get("context_cache_failed")
|
|
530
640
|
and not counts.get("owner_erasure_incomplete")
|
|
641
|
+
and not counts.get("backup_obligations_pending")
|
|
642
|
+
and not counts.get("backup_scan_failed")
|
|
643
|
+
and not counts.get("fts_residue_rows")
|
|
531
644
|
)
|
|
532
645
|
else 0
|
|
533
646
|
)
|
|
@@ -544,6 +657,7 @@ class GDPRCompliance:
|
|
|
544
657
|
"basis": "GDPR Art.17 right-to-erasure",
|
|
545
658
|
"tables_erased": len(tables),
|
|
546
659
|
"vector_store_failures": counts.get("vector_store_failures", 0),
|
|
660
|
+
"backup_obligations_pending": backup_obligations_pending,
|
|
547
661
|
},
|
|
548
662
|
)
|
|
549
663
|
except Exception as exc:
|
|
@@ -677,6 +791,250 @@ class GDPRCompliance:
|
|
|
677
791
|
logger.info("Entity erasure '%s' in '%s': %s", entity_name, profile_id, counts)
|
|
678
792
|
return counts
|
|
679
793
|
|
|
794
|
+
# -- C2: code_graph helpers --------------------------------------------
|
|
795
|
+
|
|
796
|
+
def _erase_code_graph(self, data_root: Path) -> dict:
|
|
797
|
+
"""Wipe all rows from the live code_graph.db (C2 — Art.17 scope).
|
|
798
|
+
|
|
799
|
+
code_graph.db carries repo paths, file names and symbol names —
|
|
800
|
+
identifying data in a work context with no profile_id column. The
|
|
801
|
+
entire graph is wiped on any Art.17 erasure request. Fail-open: an
|
|
802
|
+
error is recorded in the returned dict so the caller can surface it,
|
|
803
|
+
but it does NOT abort the rest of the erasure.
|
|
804
|
+
"""
|
|
805
|
+
result: dict = {"rows_deleted": 0}
|
|
806
|
+
code_graph_path = data_root / "code_graph.db"
|
|
807
|
+
if not code_graph_path.exists():
|
|
808
|
+
return result
|
|
809
|
+
try:
|
|
810
|
+
conn = sqlite3.connect(str(code_graph_path))
|
|
811
|
+
conn.isolation_level = None # autocommit so VACUUM can run
|
|
812
|
+
try:
|
|
813
|
+
conn.execute("PRAGMA foreign_keys=OFF")
|
|
814
|
+
tables = [
|
|
815
|
+
row[0]
|
|
816
|
+
for row in conn.execute(
|
|
817
|
+
"SELECT name FROM sqlite_master WHERE type='table' "
|
|
818
|
+
"AND name NOT LIKE 'sqlite_%'"
|
|
819
|
+
).fetchall()
|
|
820
|
+
]
|
|
821
|
+
total = 0
|
|
822
|
+
conn.execute("BEGIN")
|
|
823
|
+
for tbl in tables:
|
|
824
|
+
# Skip FTS virtual-table shadow files — deleting base rows handles them
|
|
825
|
+
if tbl.endswith((
|
|
826
|
+
"_fts", "_fts_data", "_fts_idx",
|
|
827
|
+
"_fts_content", "_fts_docsize", "_fts_config",
|
|
828
|
+
)):
|
|
829
|
+
continue
|
|
830
|
+
cur = conn.execute(f"DELETE FROM {tbl}") # noqa: S608
|
|
831
|
+
total += cur.rowcount
|
|
832
|
+
conn.execute("PRAGMA foreign_keys=ON")
|
|
833
|
+
conn.execute("COMMIT")
|
|
834
|
+
# VACUUM must run outside any transaction (autocommit mode required)
|
|
835
|
+
conn.execute("VACUUM")
|
|
836
|
+
result["rows_deleted"] = total
|
|
837
|
+
finally:
|
|
838
|
+
conn.close()
|
|
839
|
+
except Exception as exc: # noqa: BLE001
|
|
840
|
+
logger.warning("GDPR erase: code_graph.db wipe failed: %s", exc)
|
|
841
|
+
result["error"] = str(exc)
|
|
842
|
+
return result
|
|
843
|
+
|
|
844
|
+
def _export_code_graph(self, data_root: Path) -> dict | None:
|
|
845
|
+
"""Read code_graph.db for Art.15 export (C2).
|
|
846
|
+
|
|
847
|
+
Returns a dict keyed by table name whose values are lists of row dicts,
|
|
848
|
+
or None if the file does not exist or cannot be read.
|
|
849
|
+
"""
|
|
850
|
+
code_graph_path = data_root / "code_graph.db"
|
|
851
|
+
if not code_graph_path.exists():
|
|
852
|
+
return None
|
|
853
|
+
export: dict = {}
|
|
854
|
+
try:
|
|
855
|
+
uri = f"file:{code_graph_path}?mode=ro"
|
|
856
|
+
conn = sqlite3.connect(uri, uri=True, timeout=5)
|
|
857
|
+
conn.row_factory = sqlite3.Row
|
|
858
|
+
try:
|
|
859
|
+
tables = [
|
|
860
|
+
row[0]
|
|
861
|
+
for row in conn.execute(
|
|
862
|
+
"SELECT name FROM sqlite_master WHERE type='table' "
|
|
863
|
+
"AND name NOT LIKE 'sqlite_%'"
|
|
864
|
+
).fetchall()
|
|
865
|
+
]
|
|
866
|
+
for tbl in tables:
|
|
867
|
+
if tbl.endswith((
|
|
868
|
+
"_fts", "_fts_data", "_fts_idx",
|
|
869
|
+
"_fts_content", "_fts_docsize", "_fts_config",
|
|
870
|
+
)):
|
|
871
|
+
continue
|
|
872
|
+
try:
|
|
873
|
+
rows = conn.execute(
|
|
874
|
+
f"SELECT * FROM {tbl} LIMIT 10000" # noqa: S608
|
|
875
|
+
).fetchall()
|
|
876
|
+
export[tbl] = [dict(r) for r in rows]
|
|
877
|
+
except Exception as exc: # noqa: BLE001
|
|
878
|
+
logger.warning("GDPR export: code_graph table %s failed: %s", tbl, exc)
|
|
879
|
+
finally:
|
|
880
|
+
conn.close()
|
|
881
|
+
except Exception as exc: # noqa: BLE001
|
|
882
|
+
logger.warning("GDPR export: code_graph.db read failed: %s", exc)
|
|
883
|
+
return None
|
|
884
|
+
return export if export else None
|
|
885
|
+
|
|
886
|
+
# -- C1: backup obligation helpers -------------------------------------
|
|
887
|
+
|
|
888
|
+
def _record_backup_obligations(
|
|
889
|
+
self,
|
|
890
|
+
data_root: Path,
|
|
891
|
+
profile_id: str,
|
|
892
|
+
erasure_id: str,
|
|
893
|
+
counts: dict,
|
|
894
|
+
) -> int:
|
|
895
|
+
"""Scan backup snapshots and record any that contain *profile_id* data.
|
|
896
|
+
|
|
897
|
+
Returns the total count of pending obligations after recording
|
|
898
|
+
(including those created in prior erasure passes for the same profile).
|
|
899
|
+
Fail-closed: any unhandled exception propagates to the caller who sets
|
|
900
|
+
``backup_scan_failed`` to block the completeness claim.
|
|
901
|
+
"""
|
|
902
|
+
bom = _get_backup_obligations_module()
|
|
903
|
+
if bom is None:
|
|
904
|
+
logger.warning(
|
|
905
|
+
"GDPR erase: backup_obligations module unavailable — "
|
|
906
|
+
"backup residue will not be tracked for profile %r",
|
|
907
|
+
profile_id,
|
|
908
|
+
)
|
|
909
|
+
# Cannot track → treat as pending so completeness is blocked.
|
|
910
|
+
return 1
|
|
911
|
+
|
|
912
|
+
backup_dir = data_root / "backups"
|
|
913
|
+
retention_days = _retention_days_from_config()
|
|
914
|
+
store = bom.BackupObligationStore(data_root)
|
|
915
|
+
|
|
916
|
+
# Scan all backup snapshots for this profile's data.
|
|
917
|
+
try:
|
|
918
|
+
hits = bom.scan_backup_snapshots_for_profile(backup_dir, profile_id)
|
|
919
|
+
except Exception as exc: # noqa: BLE001
|
|
920
|
+
logger.error(
|
|
921
|
+
"GDPR erase: backup snapshot scan raised: %s — "
|
|
922
|
+
"failing closed to block completeness",
|
|
923
|
+
exc,
|
|
924
|
+
)
|
|
925
|
+
counts["backup_scan_error"] = str(exc)
|
|
926
|
+
raise # propagate so caller sets backup_scan_failed
|
|
927
|
+
|
|
928
|
+
snapshots_with_data = len(hits)
|
|
929
|
+
counts["backup_snapshots_scanned"] = snapshots_with_data
|
|
930
|
+
recorded = 0
|
|
931
|
+
for snap_path, snap_epoch in hits:
|
|
932
|
+
try:
|
|
933
|
+
store.record(
|
|
934
|
+
profile_id=profile_id,
|
|
935
|
+
erasure_id=erasure_id,
|
|
936
|
+
snapshot_path=snap_path,
|
|
937
|
+
snapshot_epoch=snap_epoch,
|
|
938
|
+
retention_days=retention_days,
|
|
939
|
+
)
|
|
940
|
+
recorded += 1
|
|
941
|
+
except Exception as exc: # noqa: BLE001
|
|
942
|
+
# Recording failure for a single snapshot must not silently
|
|
943
|
+
# skip the obligation — log and count so completeness is blocked.
|
|
944
|
+
logger.error(
|
|
945
|
+
"GDPR erase: failed to record obligation for snapshot %r: %s",
|
|
946
|
+
snap_path, exc,
|
|
947
|
+
)
|
|
948
|
+
counts["backup_record_errors"] = counts.get("backup_record_errors", 0) + 1
|
|
949
|
+
# Still include in pending count (fail-closed).
|
|
950
|
+
recorded += 1
|
|
951
|
+
|
|
952
|
+
counts["backup_obligations_recorded"] = recorded
|
|
953
|
+
# Return the authoritative pending count (includes obligations from prior
|
|
954
|
+
# erasure passes for the same profile that were not yet discharged).
|
|
955
|
+
return store.count_pending(profile_id)
|
|
956
|
+
|
|
957
|
+
# -- I7: post-erasure residue scanner ----------------------------------
|
|
958
|
+
|
|
959
|
+
def _scan_fts_residue(
|
|
960
|
+
self,
|
|
961
|
+
profile_id: str,
|
|
962
|
+
tables: list[str],
|
|
963
|
+
counts: dict,
|
|
964
|
+
) -> None:
|
|
965
|
+
"""Sweep FTS5 shadow tables for orphaned rowids after main-table erasure.
|
|
966
|
+
|
|
967
|
+
FTS5 tables maintain several shadow tables (``_data``, ``_idx``,
|
|
968
|
+
``_content``, ``_docsize``, ``_config``). A correct DELETE + VACUUM
|
|
969
|
+
cycle via FTS5's ``content=`` mechanism will purge shadow rows
|
|
970
|
+
automatically. This sweep cross-checks: if any main memory/facts table
|
|
971
|
+
is empty for the profile yet the corresponding FTS ``_content`` shadow
|
|
972
|
+
still has rows, that is residue. Updates ``counts["fts_residue_rows"]``.
|
|
973
|
+
"""
|
|
974
|
+
fts_residue = 0
|
|
975
|
+
try:
|
|
976
|
+
all_tables = {
|
|
977
|
+
row[0]
|
|
978
|
+
for row in self._db.execute(
|
|
979
|
+
"SELECT name FROM sqlite_master WHERE type='table'"
|
|
980
|
+
)
|
|
981
|
+
}
|
|
982
|
+
except Exception as exc: # noqa: BLE001
|
|
983
|
+
logger.warning("GDPR I7: FTS shadow table listing failed: %s", exc)
|
|
984
|
+
return
|
|
985
|
+
|
|
986
|
+
for tbl in tables:
|
|
987
|
+
content_shadow = f"{tbl}_fts_content"
|
|
988
|
+
if content_shadow not in all_tables:
|
|
989
|
+
continue
|
|
990
|
+
try:
|
|
991
|
+
# _fts_content stores one row per indexed row. After erasure
|
|
992
|
+
# the content rows should be zero for this profile. We cannot
|
|
993
|
+
# filter by profile_id directly (FTS content is a rowid join),
|
|
994
|
+
# so we count ALL content rows and compare with the main table
|
|
995
|
+
# row count for this profile (should both be 0).
|
|
996
|
+
main_count_rows = self._db.execute(
|
|
997
|
+
f"SELECT COUNT(*) AS c FROM {tbl} WHERE profile_id = ?", # noqa: S608
|
|
998
|
+
(profile_id,),
|
|
999
|
+
)
|
|
1000
|
+
main_count = int(dict(main_count_rows[0])["c"]) if main_count_rows else 0
|
|
1001
|
+
if main_count > 0:
|
|
1002
|
+
# Main table still has rows — not an FTS-shadow issue, already
|
|
1003
|
+
# caught by the main residue recount.
|
|
1004
|
+
continue
|
|
1005
|
+
fts_rows = self._db.execute(
|
|
1006
|
+
f"SELECT COUNT(*) AS c FROM {content_shadow}" # noqa: S608
|
|
1007
|
+
)
|
|
1008
|
+
fts_count = int(dict(fts_rows[0])["c"]) if fts_rows else 0
|
|
1009
|
+
if fts_count > 0:
|
|
1010
|
+
logger.warning(
|
|
1011
|
+
"GDPR I7: FTS shadow %s has %d rows after profile %r erasure",
|
|
1012
|
+
content_shadow, fts_count, profile_id,
|
|
1013
|
+
)
|
|
1014
|
+
fts_residue += fts_count
|
|
1015
|
+
except Exception as exc: # noqa: BLE001
|
|
1016
|
+
logger.warning(
|
|
1017
|
+
"GDPR I7: FTS shadow scan for %s failed: %s", content_shadow, exc
|
|
1018
|
+
)
|
|
1019
|
+
if fts_residue:
|
|
1020
|
+
counts["fts_residue_rows"] = fts_residue
|
|
1021
|
+
|
|
1022
|
+
def _scan_wal_residue(self, counts: dict) -> None:
|
|
1023
|
+
"""Trigger a WAL checkpoint so the WAL does not re-introduce erased data.
|
|
1024
|
+
|
|
1025
|
+
After VACUUM the WAL should already be flushed, but an explicit
|
|
1026
|
+
PRAGMA wal_checkpoint(TRUNCATE) ensures the WAL file is zeroed and
|
|
1027
|
+
cannot carry deleted pages forward into a subsequent read.
|
|
1028
|
+
"""
|
|
1029
|
+
try:
|
|
1030
|
+
self._db.execute("PRAGMA wal_checkpoint(TRUNCATE)")
|
|
1031
|
+
except Exception as exc: # noqa: BLE001
|
|
1032
|
+
logger.warning(
|
|
1033
|
+
"GDPR I7: WAL checkpoint failed: %s — WAL may retain erased pages",
|
|
1034
|
+
exc,
|
|
1035
|
+
)
|
|
1036
|
+
counts["wal_checkpoint_failed"] = 1
|
|
1037
|
+
|
|
680
1038
|
# -- Audit Trail -------------------------------------------------------
|
|
681
1039
|
|
|
682
1040
|
def get_audit_trail(self, profile_id: str, limit: int = 100) -> list[dict]:
|
|
@@ -93,7 +93,18 @@ class EmbeddingConfig:
|
|
|
93
93
|
|
|
94
94
|
@dataclass(frozen=True)
|
|
95
95
|
class LLMConfig:
|
|
96
|
-
"""LLM provider configuration per mode.
|
|
96
|
+
"""LLM provider configuration per mode.
|
|
97
|
+
|
|
98
|
+
Frozen (restored in 4.0.6). ``frozen=True`` also generates ``__hash__``;
|
|
99
|
+
dropping it makes every LLMConfig unhashable, which breaks any set/dict-key
|
|
100
|
+
or cache use at runtime without necessarily failing a test. Updates build a
|
|
101
|
+
NEW instance via ``dataclasses.replace`` and assign it to the mutable
|
|
102
|
+
SLMConfig — see ``v3_api.apply_settings_update`` — so immutability here
|
|
103
|
+
costs nothing.
|
|
104
|
+
|
|
105
|
+
All production code creates new instances via LLMConfig(...) or
|
|
106
|
+
dataclasses.replace().
|
|
107
|
+
"""
|
|
97
108
|
|
|
98
109
|
provider: str = "" # "" = no LLM, "ollama", "azure", "openai", "anthropic"
|
|
99
110
|
model: str = "" # Model name/deployment
|
|
@@ -292,6 +303,19 @@ class RetrievalConfig:
|
|
|
292
303
|
# than holding the recall open.
|
|
293
304
|
cross_encoder_timeout_seconds: float = 15.0
|
|
294
305
|
|
|
306
|
+
# v3.9.x (issue #112): plain-HTTP trust for private-LAN reranker endpoints.
|
|
307
|
+
# When True (the default), numeric RFC1918/ULA/link-local addresses may use
|
|
308
|
+
# plain HTTP — the same security model as the local reranker, where memory
|
|
309
|
+
# text only crosses the loopback. Set to False in hardened deployments
|
|
310
|
+
# (zero-trust networks, shared colocation) to require HTTPS for all
|
|
311
|
+
# non-loopback hosts, including private-LAN addresses.
|
|
312
|
+
#
|
|
313
|
+
# THREAT MODEL: setting this True does NOT prevent a MITM attack by an
|
|
314
|
+
# adversary on the same physical LAN (e.g. ARP spoofing). This flag means
|
|
315
|
+
# "my LAN is under my control and I accept that residual risk." It is not
|
|
316
|
+
# a claim that private-LAN traffic is cryptographically secure.
|
|
317
|
+
trust_plain_http_lan: bool = True
|
|
318
|
+
|
|
295
319
|
@property
|
|
296
320
|
def is_remote_cross_encoder(self) -> bool:
|
|
297
321
|
"""True when reranking is served by a remote HTTP endpoint."""
|
|
@@ -1132,6 +1156,17 @@ class SLMConfig:
|
|
|
1132
1156
|
# enterprise deployment preset requests PII redaction.
|
|
1133
1157
|
pii_redaction: bool = False
|
|
1134
1158
|
|
|
1159
|
+
# GDPR / compliance: data-retention window in days.
|
|
1160
|
+
# Default 90 days (owner decision: "configurable through the UI, and
|
|
1161
|
+
# 90 days by default"). Units: days. Value of 0 means no automatic
|
|
1162
|
+
# expiry. Consumed by the backup/erasure obligation tracker and any
|
|
1163
|
+
# future retention scheduler. Configurable via config.json (key
|
|
1164
|
+
# "retention_window_days") or the UI settings panel.
|
|
1165
|
+
# NOTE: this field is intentionally separate from
|
|
1166
|
+
# DeploymentConfig.retention_enabled — the enabled flag gates the
|
|
1167
|
+
# scheduler, while this window controls the duration.
|
|
1168
|
+
retention_window_days: int = 90
|
|
1169
|
+
|
|
1135
1170
|
def __post_init__(self) -> None:
|
|
1136
1171
|
if self.db_path is None:
|
|
1137
1172
|
self.db_path = self.base_dir / DEFAULT_DB_NAME
|
|
@@ -1340,6 +1375,12 @@ class SLMConfig:
|
|
|
1340
1375
|
)
|
|
1341
1376
|
config.mesh_enabled = data.get("mesh_enabled", True)
|
|
1342
1377
|
|
|
1378
|
+
# v4.0.6: GDPR retention window (additive — defaults to 90 if absent)
|
|
1379
|
+
try:
|
|
1380
|
+
config.retention_window_days = int(data.get("retention_window_days", 90))
|
|
1381
|
+
except (TypeError, ValueError):
|
|
1382
|
+
config.retention_window_days = 90
|
|
1383
|
+
|
|
1343
1384
|
# V3.4.10: Evolution config
|
|
1344
1385
|
evo = data.get("evolution", {})
|
|
1345
1386
|
if evo:
|
|
@@ -1566,6 +1607,8 @@ class SLMConfig:
|
|
|
1566
1607
|
data["entity_compilation_enabled"] = self.entity_compilation_enabled
|
|
1567
1608
|
data["entity_compilation_retrieval_boost"] = self.entity_compilation_retrieval_boost
|
|
1568
1609
|
data["mesh_enabled"] = self.mesh_enabled
|
|
1610
|
+
# v4.0.6: GDPR retention window — always written so load→save is lossless.
|
|
1611
|
+
data["retention_window_days"] = self.retention_window_days
|
|
1569
1612
|
|
|
1570
1613
|
# Typed in-memory config sections. Preserve any on-disk subkeys this
|
|
1571
1614
|
# version does not model (forward-compat or externally-tuned fields such
|
|
@@ -56,8 +56,11 @@ def init_reranker(retrieval_config: Any) -> Any:
|
|
|
56
56
|
"cross-encoder/ms-marco-MiniLM-L-12-v2",
|
|
57
57
|
)
|
|
58
58
|
remote_requested = is_remote_cross_encoder_backend(backend)
|
|
59
|
+
trust_plain_http_lan = getattr(retrieval_config, "trust_plain_http_lan", True)
|
|
59
60
|
|
|
60
|
-
error = validate_remote_reranker_config(
|
|
61
|
+
error = validate_remote_reranker_config(
|
|
62
|
+
backend, endpoint, trust_plain_http_lan=trust_plain_http_lan,
|
|
63
|
+
)
|
|
61
64
|
if error and remote_requested:
|
|
62
65
|
logger.error(
|
|
63
66
|
"Remote reranker not started — %s Reranking is DISABLED; recall "
|
|
@@ -79,6 +82,7 @@ def init_reranker(retrieval_config: Any) -> Any:
|
|
|
79
82
|
timeout_seconds=getattr(
|
|
80
83
|
retrieval_config, "cross_encoder_timeout_seconds", 15.0,
|
|
81
84
|
),
|
|
85
|
+
trust_plain_http_lan=trust_plain_http_lan,
|
|
82
86
|
)
|
|
83
87
|
except RemoteRerankerConfigError as exc:
|
|
84
88
|
logger.error(
|
|
@@ -637,11 +637,53 @@ def run_maintenance(
|
|
|
637
637
|
except Exception as exc:
|
|
638
638
|
logger.warning("Entity summary consolidation failed: %s", exc)
|
|
639
639
|
|
|
640
|
+
# 4. Fact consolidation (v3.8.4 concurrency-safe path via DatabaseManager).
|
|
641
|
+
# Merges clusters of warm/cold atomic facts about the same entity into a
|
|
642
|
+
# single consolidated fact, archives the originals (NEVER deletes them),
|
|
643
|
+
# and records provenance in fact_consolidations.
|
|
644
|
+
#
|
|
645
|
+
# Uses the DatabaseManager path so LLM calls happen OUTSIDE the write lock:
|
|
646
|
+
# - Discover clusters in a short memory_read() (no write lock held).
|
|
647
|
+
# - Generate summary OUTSIDE any lock (Ollama/Cloud may take 30s).
|
|
648
|
+
# - Write per-cluster inside a short memory_write() (lock held for SQL only).
|
|
649
|
+
#
|
|
650
|
+
# NOT on the recall/store hot path. Runs as part of background maintenance.
|
|
651
|
+
counts["facts_consolidated"] = 0
|
|
652
|
+
try:
|
|
653
|
+
from superlocalmemory.core.fact_consolidator import consolidate_facts
|
|
654
|
+
|
|
655
|
+
fc_stats = consolidate_facts(
|
|
656
|
+
db,
|
|
657
|
+
profile_id=profile_id,
|
|
658
|
+
max_clusters=getattr(config, "max_consolidation_clusters", 20),
|
|
659
|
+
dry_run=False,
|
|
660
|
+
config=config,
|
|
661
|
+
)
|
|
662
|
+
counts["facts_consolidated"] = fc_stats.get("consolidated", 0)
|
|
663
|
+
if fc_stats.get("consolidated", 0) > 0:
|
|
664
|
+
logger.info(
|
|
665
|
+
"Fact consolidation: %d clusters merged, %d facts archived",
|
|
666
|
+
fc_stats.get("consolidated", 0),
|
|
667
|
+
fc_stats.get("facts_archived", 0),
|
|
668
|
+
)
|
|
669
|
+
except Exception as exc:
|
|
670
|
+
# WARNING, not debug, and a distinguishable count. Leaving this at debug
|
|
671
|
+
# with facts_consolidated=0 made a failing consolidation report exactly
|
|
672
|
+
# the same numbers as a healthy run with nothing to merge, so a step that
|
|
673
|
+
# never worked would look like a step with no work to do — and nobody
|
|
674
|
+
# would ever see it in normal logs.
|
|
675
|
+
counts["facts_consolidated"] = -1
|
|
676
|
+
logger.warning(
|
|
677
|
+
"Fact consolidation FAILED during maintenance (reported as -1, "
|
|
678
|
+
"which is distinct from 0 = nothing to merge): %s", exc,
|
|
679
|
+
)
|
|
680
|
+
|
|
640
681
|
logger.info(
|
|
641
682
|
"Maintenance complete: %d backfilled, %d Langevin, %d Fisher-coupled, "
|
|
642
|
-
"%d Sheaf, %d entity-summaries",
|
|
683
|
+
"%d Sheaf, %d entity-summaries, %d facts-consolidated",
|
|
643
684
|
counts["langevin_backfilled"], counts["langevin_updated"],
|
|
644
685
|
counts["fisher_coupled"], counts["sheaf_checked"],
|
|
645
686
|
counts["entity_summaries_consolidated"],
|
|
687
|
+
counts["facts_consolidated"],
|
|
646
688
|
)
|
|
647
689
|
return counts
|
|
@@ -238,18 +238,39 @@ def _handle_update_memory(
|
|
|
238
238
|
content: str,
|
|
239
239
|
source_agent_id: str = "system",
|
|
240
240
|
) -> dict:
|
|
241
|
-
"""Update a fact after capability-derived authorization.
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
241
|
+
"""Update a fact after capability-derived authorization.
|
|
242
|
+
|
|
243
|
+
CORRECTIONS CANNOT BE PERFORMED FROM THIS WORKER, AND THAT IS BY DESIGN.
|
|
244
|
+
|
|
245
|
+
Since corrections became a review-gated lifecycle (M042) rather than an
|
|
246
|
+
in-place edit, update_fact_authorized() requires a canonical correction
|
|
247
|
+
writer. That writer is the daemon's single-writer mutation boundary
|
|
248
|
+
(CanonicalRememberRuntime, owned by unified_daemon and handed to the HTTP
|
|
249
|
+
route). A worker subprocess cannot own it: CanonicalRememberRuntime.ready
|
|
250
|
+
requires a live worker of its own, so constructing one here would nest
|
|
251
|
+
worker processes and hand a second writer the ownership context that is
|
|
252
|
+
supposed to be exclusive.
|
|
253
|
+
|
|
254
|
+
Previously this called through anyway, and mutations.py answered
|
|
255
|
+
"canonical correction writer is temporarily unavailable" with
|
|
256
|
+
retryable=True. Nothing about it was temporary — with the daemon down that
|
|
257
|
+
path can NEVER succeed, so a caller could retry forever. This is the
|
|
258
|
+
honest, non-retryable answer with the actual remedy: the daemon-backed
|
|
259
|
+
route (used automatically whenever the daemon is running) does support
|
|
260
|
+
corrections.
|
|
261
|
+
"""
|
|
262
|
+
return {
|
|
263
|
+
"ok": False,
|
|
264
|
+
"retryable": False,
|
|
265
|
+
"error": (
|
|
266
|
+
"Corrections are review-gated and require the SuperLocalMemory "
|
|
267
|
+
"daemon, which owns the single correction writer. Start it with "
|
|
268
|
+
"`slm serve start` and retry — updates route through the daemon "
|
|
269
|
+
"automatically when it is running."
|
|
270
|
+
),
|
|
271
|
+
"remedy": "slm serve start",
|
|
272
|
+
"fact_id": fact_id,
|
|
273
|
+
}
|
|
253
274
|
|
|
254
275
|
|
|
255
276
|
def _handle_summarize(texts: list[str], mode: str) -> dict:
|