superlocalmemory 4.0.5 → 4.0.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +108 -0
- package/README.md +8 -9
- package/package.json +3 -1
- package/plugin/.claude-plugin/plugin.json +1 -1
- package/plugin/CLAUDE.md +3 -3
- package/plugin/agents/slm-governance-advisor.md +1 -1
- package/plugin/agents/slm-loop-runner.md +1 -1
- package/plugin/agents/slm-memory-advisor.md +1 -1
- package/plugin/agents/slm-optimize-advisor.md +1 -1
- package/plugin/requirements.txt +1 -1
- package/plugin/skills/slm-cache/SKILL.md +1 -1
- package/plugin/skills/slm-compress/SKILL.md +1 -1
- package/plugin/skills/slm-governance/SKILL.md +1 -1
- package/plugin/skills/slm-graph/SKILL.md +1 -1
- package/plugin/skills/slm-loop/SKILL.md +1 -1
- package/plugin/skills/slm-mesh/SKILL.md +1 -1
- package/plugin/skills/slm-profile/SKILL.md +1 -1
- package/plugin/skills/slm-recall/SKILL.md +1 -1
- package/plugin/skills/slm-remember/SKILL.md +1 -1
- package/plugin/skills/slm-scope/SKILL.md +1 -1
- package/plugin/skills/slm-session/SKILL.md +1 -1
- package/plugin/skills/slm-status/SKILL.md +1 -1
- package/plugin-src/rules/AGENTS.md +1 -1
- package/pyproject.toml +1 -1
- package/src/superlocalmemory/__init__.py +1 -1
- package/src/superlocalmemory/access/rbac.py +106 -0
- package/src/superlocalmemory/brain/truth.py +80 -10
- package/src/superlocalmemory/cli/__main__.py +17 -0
- package/src/superlocalmemory/cli/commands.py +28 -3
- package/src/superlocalmemory/cli/gdpr_cmd.py +779 -0
- package/src/superlocalmemory/cli/gdpr_io.py +109 -0
- package/src/superlocalmemory/cli/main.py +85 -0
- package/src/superlocalmemory/cli/summary_cmd.py +195 -0
- package/src/superlocalmemory/code_graph/bridge/entity_resolver.py +26 -0
- package/src/superlocalmemory/code_graph/bridge/event_listeners.py +14 -3
- package/src/superlocalmemory/code_graph/bridge/maintenance.py +206 -0
- package/src/superlocalmemory/code_graph/config.py +65 -1
- package/src/superlocalmemory/code_graph/extractors/__init__.py +17 -0
- package/src/superlocalmemory/code_graph/graph_store.py +180 -3
- package/src/superlocalmemory/code_graph/parser.py +280 -100
- package/src/superlocalmemory/compliance/gdpr.py +358 -0
- package/src/superlocalmemory/core/config.py +44 -1
- package/src/superlocalmemory/core/engine_wiring.py +5 -1
- package/src/superlocalmemory/core/fact_consolidator.py +24 -1
- package/src/superlocalmemory/core/maintenance.py +93 -1
- package/src/superlocalmemory/core/recall_worker.py +33 -12
- package/src/superlocalmemory/infra/backup.py +138 -0
- package/src/superlocalmemory/infra/backup_obligations.py +423 -0
- package/src/superlocalmemory/learning/engagement.py +165 -0
- package/src/superlocalmemory/mcp/tools_code_graph.py +78 -7
- package/src/superlocalmemory/mcp/tools_v3.py +20 -6
- package/src/superlocalmemory/retrieval/engine.py +21 -0
- package/src/superlocalmemory/retrieval/remote_reranker.py +108 -11
- package/src/superlocalmemory/server/routes/brain.py +283 -15
- package/src/superlocalmemory/server/routes/learning.py +13 -25
- package/src/superlocalmemory/server/routes/memories.py +61 -0
- package/src/superlocalmemory/server/routes/v3_api.py +171 -60
- package/src/superlocalmemory/storage/database.py +36 -0
- package/src/superlocalmemory/storage/models.py +12 -4
- package/src/superlocalmemory/storage/schema_code_graph.py +44 -1
- package/src/superlocalmemory/summaries/__init__.py +37 -0
- package/src/superlocalmemory/summaries/base.py +108 -0
- package/src/superlocalmemory/summaries/daily_reflection.py +293 -0
- package/src/superlocalmemory/summaries/project_work_log.py +424 -0
- package/src/superlocalmemory/summaries/session_summary.py +307 -0
- package/src/superlocalmemory/ui/css/design-system.css +76 -1
- package/src/superlocalmemory/ui/index.html +29 -12
- package/src/superlocalmemory/ui/js/fact-detail.js +61 -0
- package/src/superlocalmemory/ui/js/od-agents.js +49 -5
- package/src/superlocalmemory/ui/js/od-brain.js +257 -77
- package/src/superlocalmemory/ui/js/od-graph.js +147 -6
|
@@ -14,11 +14,65 @@ Part of Qualixar | Author: Varun Pratap Bhardwaj
|
|
|
14
14
|
from __future__ import annotations
|
|
15
15
|
|
|
16
16
|
import logging
|
|
17
|
+
import sqlite3
|
|
17
18
|
from datetime import UTC, datetime
|
|
18
19
|
from pathlib import Path
|
|
19
20
|
|
|
20
21
|
logger = logging.getLogger(__name__)
|
|
21
22
|
|
|
23
|
+
# C1 — Backup residue obligations
|
|
24
|
+
# Imported lazily inside methods to avoid circular-import risk at module load.
|
|
25
|
+
# The sentinel guards against environments where infra.backup_obligations is
|
|
26
|
+
# unavailable (e.g. minimal test installs); compliance logic degrades safely.
|
|
27
|
+
_BACKUP_OBLIGATIONS_AVAILABLE: bool | None = None
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _get_backup_obligations_module():
|
|
31
|
+
"""Lazy import guard — returns module or None if unavailable."""
|
|
32
|
+
global _BACKUP_OBLIGATIONS_AVAILABLE
|
|
33
|
+
try:
|
|
34
|
+
import superlocalmemory.infra.backup_obligations as _m
|
|
35
|
+
_BACKUP_OBLIGATIONS_AVAILABLE = True
|
|
36
|
+
return _m
|
|
37
|
+
except Exception as exc: # noqa: BLE001
|
|
38
|
+
if _BACKUP_OBLIGATIONS_AVAILABLE is None:
|
|
39
|
+
logger.warning("backup_obligations module unavailable: %s", exc)
|
|
40
|
+
_BACKUP_OBLIGATIONS_AVAILABLE = False
|
|
41
|
+
return None
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _retention_days_from_config() -> int:
|
|
45
|
+
"""Read the configured obligation retention window (default 90 days).
|
|
46
|
+
|
|
47
|
+
Consumes ``SLMConfig.backup_retention_days`` or any attribute matching
|
|
48
|
+
('retention', 'retain') with numeric value on ``SLMConfig``. The config
|
|
49
|
+
field is owned by another agent; we consume it here and fall back to 90.
|
|
50
|
+
"""
|
|
51
|
+
try:
|
|
52
|
+
from superlocalmemory.core.config import SLMConfig
|
|
53
|
+
cfg = SLMConfig()
|
|
54
|
+
# Try the canonical field name first
|
|
55
|
+
for attr in ("backup_retention_days", "obligation_retention_days",
|
|
56
|
+
"retention_window_days", "backup_obligation_retention_days"):
|
|
57
|
+
val = getattr(cfg, attr, None)
|
|
58
|
+
if isinstance(val, int) and val > 0:
|
|
59
|
+
return val
|
|
60
|
+
# Fallback: search nested sub-configs
|
|
61
|
+
for attr in dir(cfg):
|
|
62
|
+
if attr.startswith("_"):
|
|
63
|
+
continue
|
|
64
|
+
sub = getattr(cfg, attr, None)
|
|
65
|
+
if not hasattr(sub, "__dict__") and not hasattr(sub, "__dataclass_fields__"):
|
|
66
|
+
continue
|
|
67
|
+
for sub_attr in dir(sub):
|
|
68
|
+
if any(h in sub_attr.lower() for h in ("retention", "retain")):
|
|
69
|
+
v = getattr(sub, sub_attr, None)
|
|
70
|
+
if isinstance(v, int) and v > 0:
|
|
71
|
+
return v
|
|
72
|
+
except Exception: # noqa: BLE001
|
|
73
|
+
pass
|
|
74
|
+
return 90 # owner-set default
|
|
75
|
+
|
|
22
76
|
# Friendly export keys → canonical table names (stable Art.20 export contract).
|
|
23
77
|
_EXPORT_ALIASES = {
|
|
24
78
|
"facts": "atomic_facts",
|
|
@@ -238,6 +292,13 @@ class GDPRCompliance:
|
|
|
238
292
|
except Exception:
|
|
239
293
|
data["profile_record"] = []
|
|
240
294
|
|
|
295
|
+
# C2 — include code_graph.db in Art.15 export (repo paths, file names,
|
|
296
|
+
# and symbol names are identifying data in a work context).
|
|
297
|
+
if self._data_root is not None:
|
|
298
|
+
code_graph_data = self._export_code_graph(self._data_root)
|
|
299
|
+
if code_graph_data is not None:
|
|
300
|
+
data["code_graph"] = code_graph_data
|
|
301
|
+
|
|
241
302
|
# total_items counts the canonical (table-name) keys only, before
|
|
242
303
|
# friendly aliases are added, so it is not double-counted.
|
|
243
304
|
data["total_items"] = sum(len(v) for v in data.values() if isinstance(v, list))
|
|
@@ -437,6 +498,16 @@ class GDPRCompliance:
|
|
|
437
498
|
counts["receipt_error"] = str(exc)
|
|
438
499
|
raise
|
|
439
500
|
|
|
501
|
+
# C2 — erase code_graph.db before main profile rows (fail-closed: a
|
|
502
|
+
# code_graph failure is logged but does NOT abort the erasure; the
|
|
503
|
+
# graph is installation-level personal data with no profile_id column,
|
|
504
|
+
# so it is wiped entirely on any Art.17 request).
|
|
505
|
+
if data_root is not None:
|
|
506
|
+
code_graph_result = self._erase_code_graph(data_root)
|
|
507
|
+
counts["code_graph"] = code_graph_result.get("rows_deleted", 0)
|
|
508
|
+
if code_graph_result.get("error"):
|
|
509
|
+
counts["code_graph_failed"] = 1
|
|
510
|
+
|
|
440
511
|
# Purge the learning sidecar *before* removing memory/profile rows. A
|
|
441
512
|
# learning failure is retryable and must leave the profile intact; the
|
|
442
513
|
# former best-effort-after-delete ordering could orphan receipts.
|
|
@@ -517,6 +588,45 @@ class GDPRCompliance:
|
|
|
517
588
|
counts["residue_rows"] = residue_rows
|
|
518
589
|
if residue_recount_failed:
|
|
519
590
|
counts["residue_recount_failed"] = 1
|
|
591
|
+
|
|
592
|
+
# I7 — post-erasure residue sweep: FTS shadow tables + WAL sanity.
|
|
593
|
+
# This makes I7 enforceable rather than aspirational.
|
|
594
|
+
self._scan_fts_residue(profile_id, tables, counts)
|
|
595
|
+
self._scan_wal_residue(counts)
|
|
596
|
+
|
|
597
|
+
# C1 — record outstanding obligations against backup snapshots.
|
|
598
|
+
# Done AFTER the main-DB residue scan so counts reflect live-store state
|
|
599
|
+
# before we tally the backups outstanding obligation count.
|
|
600
|
+
# Fail-closed: if the scan itself errors, set backup_scan_failed so
|
|
601
|
+
# completeness cannot be claimed.
|
|
602
|
+
backup_obligations_pending = 0
|
|
603
|
+
if data_root is not None:
|
|
604
|
+
try:
|
|
605
|
+
backup_obligations_pending = self._record_backup_obligations(
|
|
606
|
+
data_root=data_root,
|
|
607
|
+
profile_id=profile_id,
|
|
608
|
+
erasure_id=_uuid.uuid4().hex, # unique id for this obligation batch
|
|
609
|
+
counts=counts,
|
|
610
|
+
)
|
|
611
|
+
counts["backup_obligations_pending"] = backup_obligations_pending
|
|
612
|
+
except Exception as exc:
|
|
613
|
+
logger.error(
|
|
614
|
+
"GDPR erase: backup obligation recording FAILED: %s — "
|
|
615
|
+
"setting backup_scan_failed to block completeness",
|
|
616
|
+
exc,
|
|
617
|
+
)
|
|
618
|
+
counts["backup_scan_failed"] = 1
|
|
619
|
+
else:
|
|
620
|
+
# Cannot scan backups without data_root — treat as outstanding
|
|
621
|
+
# obligation so completeness is blocked.
|
|
622
|
+
counts["backup_obligations_pending"] = 0 # unknown but not confirmed clean
|
|
623
|
+
# We won't block completeness when data_root is unknown (legacy wrapper)
|
|
624
|
+
# but we do log the gap.
|
|
625
|
+
logger.warning(
|
|
626
|
+
"GDPR erase: backup obligation scan skipped — data_root unknown. "
|
|
627
|
+
"Backup snapshots may still contain the erased profile's data."
|
|
628
|
+
)
|
|
629
|
+
|
|
520
630
|
counts["erasure_complete"] = (
|
|
521
631
|
1
|
|
522
632
|
if (
|
|
@@ -528,6 +638,9 @@ class GDPRCompliance:
|
|
|
528
638
|
and not counts.get("vector_store_failures")
|
|
529
639
|
and not counts.get("context_cache_failed")
|
|
530
640
|
and not counts.get("owner_erasure_incomplete")
|
|
641
|
+
and not counts.get("backup_obligations_pending")
|
|
642
|
+
and not counts.get("backup_scan_failed")
|
|
643
|
+
and not counts.get("fts_residue_rows")
|
|
531
644
|
)
|
|
532
645
|
else 0
|
|
533
646
|
)
|
|
@@ -544,6 +657,7 @@ class GDPRCompliance:
|
|
|
544
657
|
"basis": "GDPR Art.17 right-to-erasure",
|
|
545
658
|
"tables_erased": len(tables),
|
|
546
659
|
"vector_store_failures": counts.get("vector_store_failures", 0),
|
|
660
|
+
"backup_obligations_pending": backup_obligations_pending,
|
|
547
661
|
},
|
|
548
662
|
)
|
|
549
663
|
except Exception as exc:
|
|
@@ -677,6 +791,250 @@ class GDPRCompliance:
|
|
|
677
791
|
logger.info("Entity erasure '%s' in '%s': %s", entity_name, profile_id, counts)
|
|
678
792
|
return counts
|
|
679
793
|
|
|
794
|
+
# -- C2: code_graph helpers --------------------------------------------
|
|
795
|
+
|
|
796
|
+
def _erase_code_graph(self, data_root: Path) -> dict:
|
|
797
|
+
"""Wipe all rows from the live code_graph.db (C2 — Art.17 scope).
|
|
798
|
+
|
|
799
|
+
code_graph.db carries repo paths, file names and symbol names —
|
|
800
|
+
identifying data in a work context with no profile_id column. The
|
|
801
|
+
entire graph is wiped on any Art.17 erasure request. Fail-open: an
|
|
802
|
+
error is recorded in the returned dict so the caller can surface it,
|
|
803
|
+
but it does NOT abort the rest of the erasure.
|
|
804
|
+
"""
|
|
805
|
+
result: dict = {"rows_deleted": 0}
|
|
806
|
+
code_graph_path = data_root / "code_graph.db"
|
|
807
|
+
if not code_graph_path.exists():
|
|
808
|
+
return result
|
|
809
|
+
try:
|
|
810
|
+
conn = sqlite3.connect(str(code_graph_path))
|
|
811
|
+
conn.isolation_level = None # autocommit so VACUUM can run
|
|
812
|
+
try:
|
|
813
|
+
conn.execute("PRAGMA foreign_keys=OFF")
|
|
814
|
+
tables = [
|
|
815
|
+
row[0]
|
|
816
|
+
for row in conn.execute(
|
|
817
|
+
"SELECT name FROM sqlite_master WHERE type='table' "
|
|
818
|
+
"AND name NOT LIKE 'sqlite_%'"
|
|
819
|
+
).fetchall()
|
|
820
|
+
]
|
|
821
|
+
total = 0
|
|
822
|
+
conn.execute("BEGIN")
|
|
823
|
+
for tbl in tables:
|
|
824
|
+
# Skip FTS virtual-table shadow files — deleting base rows handles them
|
|
825
|
+
if tbl.endswith((
|
|
826
|
+
"_fts", "_fts_data", "_fts_idx",
|
|
827
|
+
"_fts_content", "_fts_docsize", "_fts_config",
|
|
828
|
+
)):
|
|
829
|
+
continue
|
|
830
|
+
cur = conn.execute(f"DELETE FROM {tbl}") # noqa: S608
|
|
831
|
+
total += cur.rowcount
|
|
832
|
+
conn.execute("PRAGMA foreign_keys=ON")
|
|
833
|
+
conn.execute("COMMIT")
|
|
834
|
+
# VACUUM must run outside any transaction (autocommit mode required)
|
|
835
|
+
conn.execute("VACUUM")
|
|
836
|
+
result["rows_deleted"] = total
|
|
837
|
+
finally:
|
|
838
|
+
conn.close()
|
|
839
|
+
except Exception as exc: # noqa: BLE001
|
|
840
|
+
logger.warning("GDPR erase: code_graph.db wipe failed: %s", exc)
|
|
841
|
+
result["error"] = str(exc)
|
|
842
|
+
return result
|
|
843
|
+
|
|
844
|
+
def _export_code_graph(self, data_root: Path) -> dict | None:
|
|
845
|
+
"""Read code_graph.db for Art.15 export (C2).
|
|
846
|
+
|
|
847
|
+
Returns a dict keyed by table name whose values are lists of row dicts,
|
|
848
|
+
or None if the file does not exist or cannot be read.
|
|
849
|
+
"""
|
|
850
|
+
code_graph_path = data_root / "code_graph.db"
|
|
851
|
+
if not code_graph_path.exists():
|
|
852
|
+
return None
|
|
853
|
+
export: dict = {}
|
|
854
|
+
try:
|
|
855
|
+
uri = f"file:{code_graph_path}?mode=ro"
|
|
856
|
+
conn = sqlite3.connect(uri, uri=True, timeout=5)
|
|
857
|
+
conn.row_factory = sqlite3.Row
|
|
858
|
+
try:
|
|
859
|
+
tables = [
|
|
860
|
+
row[0]
|
|
861
|
+
for row in conn.execute(
|
|
862
|
+
"SELECT name FROM sqlite_master WHERE type='table' "
|
|
863
|
+
"AND name NOT LIKE 'sqlite_%'"
|
|
864
|
+
).fetchall()
|
|
865
|
+
]
|
|
866
|
+
for tbl in tables:
|
|
867
|
+
if tbl.endswith((
|
|
868
|
+
"_fts", "_fts_data", "_fts_idx",
|
|
869
|
+
"_fts_content", "_fts_docsize", "_fts_config",
|
|
870
|
+
)):
|
|
871
|
+
continue
|
|
872
|
+
try:
|
|
873
|
+
rows = conn.execute(
|
|
874
|
+
f"SELECT * FROM {tbl} LIMIT 10000" # noqa: S608
|
|
875
|
+
).fetchall()
|
|
876
|
+
export[tbl] = [dict(r) for r in rows]
|
|
877
|
+
except Exception as exc: # noqa: BLE001
|
|
878
|
+
logger.warning("GDPR export: code_graph table %s failed: %s", tbl, exc)
|
|
879
|
+
finally:
|
|
880
|
+
conn.close()
|
|
881
|
+
except Exception as exc: # noqa: BLE001
|
|
882
|
+
logger.warning("GDPR export: code_graph.db read failed: %s", exc)
|
|
883
|
+
return None
|
|
884
|
+
return export if export else None
|
|
885
|
+
|
|
886
|
+
# -- C1: backup obligation helpers -------------------------------------
|
|
887
|
+
|
|
888
|
+
def _record_backup_obligations(
|
|
889
|
+
self,
|
|
890
|
+
data_root: Path,
|
|
891
|
+
profile_id: str,
|
|
892
|
+
erasure_id: str,
|
|
893
|
+
counts: dict,
|
|
894
|
+
) -> int:
|
|
895
|
+
"""Scan backup snapshots and record any that contain *profile_id* data.
|
|
896
|
+
|
|
897
|
+
Returns the total count of pending obligations after recording
|
|
898
|
+
(including those created in prior erasure passes for the same profile).
|
|
899
|
+
Fail-closed: any unhandled exception propagates to the caller who sets
|
|
900
|
+
``backup_scan_failed`` to block the completeness claim.
|
|
901
|
+
"""
|
|
902
|
+
bom = _get_backup_obligations_module()
|
|
903
|
+
if bom is None:
|
|
904
|
+
logger.warning(
|
|
905
|
+
"GDPR erase: backup_obligations module unavailable — "
|
|
906
|
+
"backup residue will not be tracked for profile %r",
|
|
907
|
+
profile_id,
|
|
908
|
+
)
|
|
909
|
+
# Cannot track → treat as pending so completeness is blocked.
|
|
910
|
+
return 1
|
|
911
|
+
|
|
912
|
+
backup_dir = data_root / "backups"
|
|
913
|
+
retention_days = _retention_days_from_config()
|
|
914
|
+
store = bom.BackupObligationStore(data_root)
|
|
915
|
+
|
|
916
|
+
# Scan all backup snapshots for this profile's data.
|
|
917
|
+
try:
|
|
918
|
+
hits = bom.scan_backup_snapshots_for_profile(backup_dir, profile_id)
|
|
919
|
+
except Exception as exc: # noqa: BLE001
|
|
920
|
+
logger.error(
|
|
921
|
+
"GDPR erase: backup snapshot scan raised: %s — "
|
|
922
|
+
"failing closed to block completeness",
|
|
923
|
+
exc,
|
|
924
|
+
)
|
|
925
|
+
counts["backup_scan_error"] = str(exc)
|
|
926
|
+
raise # propagate so caller sets backup_scan_failed
|
|
927
|
+
|
|
928
|
+
snapshots_with_data = len(hits)
|
|
929
|
+
counts["backup_snapshots_scanned"] = snapshots_with_data
|
|
930
|
+
recorded = 0
|
|
931
|
+
for snap_path, snap_epoch in hits:
|
|
932
|
+
try:
|
|
933
|
+
store.record(
|
|
934
|
+
profile_id=profile_id,
|
|
935
|
+
erasure_id=erasure_id,
|
|
936
|
+
snapshot_path=snap_path,
|
|
937
|
+
snapshot_epoch=snap_epoch,
|
|
938
|
+
retention_days=retention_days,
|
|
939
|
+
)
|
|
940
|
+
recorded += 1
|
|
941
|
+
except Exception as exc: # noqa: BLE001
|
|
942
|
+
# Recording failure for a single snapshot must not silently
|
|
943
|
+
# skip the obligation — log and count so completeness is blocked.
|
|
944
|
+
logger.error(
|
|
945
|
+
"GDPR erase: failed to record obligation for snapshot %r: %s",
|
|
946
|
+
snap_path, exc,
|
|
947
|
+
)
|
|
948
|
+
counts["backup_record_errors"] = counts.get("backup_record_errors", 0) + 1
|
|
949
|
+
# Still include in pending count (fail-closed).
|
|
950
|
+
recorded += 1
|
|
951
|
+
|
|
952
|
+
counts["backup_obligations_recorded"] = recorded
|
|
953
|
+
# Return the authoritative pending count (includes obligations from prior
|
|
954
|
+
# erasure passes for the same profile that were not yet discharged).
|
|
955
|
+
return store.count_pending(profile_id)
|
|
956
|
+
|
|
957
|
+
# -- I7: post-erasure residue scanner ----------------------------------
|
|
958
|
+
|
|
959
|
+
def _scan_fts_residue(
|
|
960
|
+
self,
|
|
961
|
+
profile_id: str,
|
|
962
|
+
tables: list[str],
|
|
963
|
+
counts: dict,
|
|
964
|
+
) -> None:
|
|
965
|
+
"""Sweep FTS5 shadow tables for orphaned rowids after main-table erasure.
|
|
966
|
+
|
|
967
|
+
FTS5 tables maintain several shadow tables (``_data``, ``_idx``,
|
|
968
|
+
``_content``, ``_docsize``, ``_config``). A correct DELETE + VACUUM
|
|
969
|
+
cycle via FTS5's ``content=`` mechanism will purge shadow rows
|
|
970
|
+
automatically. This sweep cross-checks: if any main memory/facts table
|
|
971
|
+
is empty for the profile yet the corresponding FTS ``_content`` shadow
|
|
972
|
+
still has rows, that is residue. Updates ``counts["fts_residue_rows"]``.
|
|
973
|
+
"""
|
|
974
|
+
fts_residue = 0
|
|
975
|
+
try:
|
|
976
|
+
all_tables = {
|
|
977
|
+
row[0]
|
|
978
|
+
for row in self._db.execute(
|
|
979
|
+
"SELECT name FROM sqlite_master WHERE type='table'"
|
|
980
|
+
)
|
|
981
|
+
}
|
|
982
|
+
except Exception as exc: # noqa: BLE001
|
|
983
|
+
logger.warning("GDPR I7: FTS shadow table listing failed: %s", exc)
|
|
984
|
+
return
|
|
985
|
+
|
|
986
|
+
for tbl in tables:
|
|
987
|
+
content_shadow = f"{tbl}_fts_content"
|
|
988
|
+
if content_shadow not in all_tables:
|
|
989
|
+
continue
|
|
990
|
+
try:
|
|
991
|
+
# _fts_content stores one row per indexed row. After erasure
|
|
992
|
+
# the content rows should be zero for this profile. We cannot
|
|
993
|
+
# filter by profile_id directly (FTS content is a rowid join),
|
|
994
|
+
# so we count ALL content rows and compare with the main table
|
|
995
|
+
# row count for this profile (should both be 0).
|
|
996
|
+
main_count_rows = self._db.execute(
|
|
997
|
+
f"SELECT COUNT(*) AS c FROM {tbl} WHERE profile_id = ?", # noqa: S608
|
|
998
|
+
(profile_id,),
|
|
999
|
+
)
|
|
1000
|
+
main_count = int(dict(main_count_rows[0])["c"]) if main_count_rows else 0
|
|
1001
|
+
if main_count > 0:
|
|
1002
|
+
# Main table still has rows — not an FTS-shadow issue, already
|
|
1003
|
+
# caught by the main residue recount.
|
|
1004
|
+
continue
|
|
1005
|
+
fts_rows = self._db.execute(
|
|
1006
|
+
f"SELECT COUNT(*) AS c FROM {content_shadow}" # noqa: S608
|
|
1007
|
+
)
|
|
1008
|
+
fts_count = int(dict(fts_rows[0])["c"]) if fts_rows else 0
|
|
1009
|
+
if fts_count > 0:
|
|
1010
|
+
logger.warning(
|
|
1011
|
+
"GDPR I7: FTS shadow %s has %d rows after profile %r erasure",
|
|
1012
|
+
content_shadow, fts_count, profile_id,
|
|
1013
|
+
)
|
|
1014
|
+
fts_residue += fts_count
|
|
1015
|
+
except Exception as exc: # noqa: BLE001
|
|
1016
|
+
logger.warning(
|
|
1017
|
+
"GDPR I7: FTS shadow scan for %s failed: %s", content_shadow, exc
|
|
1018
|
+
)
|
|
1019
|
+
if fts_residue:
|
|
1020
|
+
counts["fts_residue_rows"] = fts_residue
|
|
1021
|
+
|
|
1022
|
+
def _scan_wal_residue(self, counts: dict) -> None:
|
|
1023
|
+
"""Trigger a WAL checkpoint so the WAL does not re-introduce erased data.
|
|
1024
|
+
|
|
1025
|
+
After VACUUM the WAL should already be flushed, but an explicit
|
|
1026
|
+
PRAGMA wal_checkpoint(TRUNCATE) ensures the WAL file is zeroed and
|
|
1027
|
+
cannot carry deleted pages forward into a subsequent read.
|
|
1028
|
+
"""
|
|
1029
|
+
try:
|
|
1030
|
+
self._db.execute("PRAGMA wal_checkpoint(TRUNCATE)")
|
|
1031
|
+
except Exception as exc: # noqa: BLE001
|
|
1032
|
+
logger.warning(
|
|
1033
|
+
"GDPR I7: WAL checkpoint failed: %s — WAL may retain erased pages",
|
|
1034
|
+
exc,
|
|
1035
|
+
)
|
|
1036
|
+
counts["wal_checkpoint_failed"] = 1
|
|
1037
|
+
|
|
680
1038
|
# -- Audit Trail -------------------------------------------------------
|
|
681
1039
|
|
|
682
1040
|
def get_audit_trail(self, profile_id: str, limit: int = 100) -> list[dict]:
|
|
@@ -93,7 +93,18 @@ class EmbeddingConfig:
|
|
|
93
93
|
|
|
94
94
|
@dataclass(frozen=True)
|
|
95
95
|
class LLMConfig:
|
|
96
|
-
"""LLM provider configuration per mode.
|
|
96
|
+
"""LLM provider configuration per mode.
|
|
97
|
+
|
|
98
|
+
Frozen (restored in 4.0.6). ``frozen=True`` also generates ``__hash__``;
|
|
99
|
+
dropping it makes every LLMConfig unhashable, which breaks any set/dict-key
|
|
100
|
+
or cache use at runtime without necessarily failing a test. Updates build a
|
|
101
|
+
NEW instance via ``dataclasses.replace`` and assign it to the mutable
|
|
102
|
+
SLMConfig — see ``v3_api.apply_settings_update`` — so immutability here
|
|
103
|
+
costs nothing.
|
|
104
|
+
|
|
105
|
+
All production code creates new instances via LLMConfig(...) or
|
|
106
|
+
dataclasses.replace().
|
|
107
|
+
"""
|
|
97
108
|
|
|
98
109
|
provider: str = "" # "" = no LLM, "ollama", "azure", "openai", "anthropic"
|
|
99
110
|
model: str = "" # Model name/deployment
|
|
@@ -292,6 +303,19 @@ class RetrievalConfig:
|
|
|
292
303
|
# than holding the recall open.
|
|
293
304
|
cross_encoder_timeout_seconds: float = 15.0
|
|
294
305
|
|
|
306
|
+
# v3.9.x (issue #112): plain-HTTP trust for private-LAN reranker endpoints.
|
|
307
|
+
# When True (the default), numeric RFC1918/ULA/link-local addresses may use
|
|
308
|
+
# plain HTTP — the same security model as the local reranker, where memory
|
|
309
|
+
# text only crosses the loopback. Set to False in hardened deployments
|
|
310
|
+
# (zero-trust networks, shared colocation) to require HTTPS for all
|
|
311
|
+
# non-loopback hosts, including private-LAN addresses.
|
|
312
|
+
#
|
|
313
|
+
# THREAT MODEL: setting this True does NOT prevent a MITM attack by an
|
|
314
|
+
# adversary on the same physical LAN (e.g. ARP spoofing). This flag means
|
|
315
|
+
# "my LAN is under my control and I accept that residual risk." It is not
|
|
316
|
+
# a claim that private-LAN traffic is cryptographically secure.
|
|
317
|
+
trust_plain_http_lan: bool = True
|
|
318
|
+
|
|
295
319
|
@property
|
|
296
320
|
def is_remote_cross_encoder(self) -> bool:
|
|
297
321
|
"""True when reranking is served by a remote HTTP endpoint."""
|
|
@@ -1132,6 +1156,17 @@ class SLMConfig:
|
|
|
1132
1156
|
# enterprise deployment preset requests PII redaction.
|
|
1133
1157
|
pii_redaction: bool = False
|
|
1134
1158
|
|
|
1159
|
+
# GDPR / compliance: data-retention window in days.
|
|
1160
|
+
# Default 90 days (owner decision: "configurable through the UI, and
|
|
1161
|
+
# 90 days by default"). Units: days. Value of 0 means no automatic
|
|
1162
|
+
# expiry. Consumed by the backup/erasure obligation tracker and any
|
|
1163
|
+
# future retention scheduler. Configurable via config.json (key
|
|
1164
|
+
# "retention_window_days") or the UI settings panel.
|
|
1165
|
+
# NOTE: this field is intentionally separate from
|
|
1166
|
+
# DeploymentConfig.retention_enabled — the enabled flag gates the
|
|
1167
|
+
# scheduler, while this window controls the duration.
|
|
1168
|
+
retention_window_days: int = 90
|
|
1169
|
+
|
|
1135
1170
|
def __post_init__(self) -> None:
|
|
1136
1171
|
if self.db_path is None:
|
|
1137
1172
|
self.db_path = self.base_dir / DEFAULT_DB_NAME
|
|
@@ -1340,6 +1375,12 @@ class SLMConfig:
|
|
|
1340
1375
|
)
|
|
1341
1376
|
config.mesh_enabled = data.get("mesh_enabled", True)
|
|
1342
1377
|
|
|
1378
|
+
# v4.0.6: GDPR retention window (additive — defaults to 90 if absent)
|
|
1379
|
+
try:
|
|
1380
|
+
config.retention_window_days = int(data.get("retention_window_days", 90))
|
|
1381
|
+
except (TypeError, ValueError):
|
|
1382
|
+
config.retention_window_days = 90
|
|
1383
|
+
|
|
1343
1384
|
# V3.4.10: Evolution config
|
|
1344
1385
|
evo = data.get("evolution", {})
|
|
1345
1386
|
if evo:
|
|
@@ -1566,6 +1607,8 @@ class SLMConfig:
|
|
|
1566
1607
|
data["entity_compilation_enabled"] = self.entity_compilation_enabled
|
|
1567
1608
|
data["entity_compilation_retrieval_boost"] = self.entity_compilation_retrieval_boost
|
|
1568
1609
|
data["mesh_enabled"] = self.mesh_enabled
|
|
1610
|
+
# v4.0.6: GDPR retention window — always written so load→save is lossless.
|
|
1611
|
+
data["retention_window_days"] = self.retention_window_days
|
|
1569
1612
|
|
|
1570
1613
|
# Typed in-memory config sections. Preserve any on-disk subkeys this
|
|
1571
1614
|
# version does not model (forward-compat or externally-tuned fields such
|
|
@@ -56,8 +56,11 @@ def init_reranker(retrieval_config: Any) -> Any:
|
|
|
56
56
|
"cross-encoder/ms-marco-MiniLM-L-12-v2",
|
|
57
57
|
)
|
|
58
58
|
remote_requested = is_remote_cross_encoder_backend(backend)
|
|
59
|
+
trust_plain_http_lan = getattr(retrieval_config, "trust_plain_http_lan", True)
|
|
59
60
|
|
|
60
|
-
error = validate_remote_reranker_config(
|
|
61
|
+
error = validate_remote_reranker_config(
|
|
62
|
+
backend, endpoint, trust_plain_http_lan=trust_plain_http_lan,
|
|
63
|
+
)
|
|
61
64
|
if error and remote_requested:
|
|
62
65
|
logger.error(
|
|
63
66
|
"Remote reranker not started — %s Reranking is DISABLED; recall "
|
|
@@ -79,6 +82,7 @@ def init_reranker(retrieval_config: Any) -> Any:
|
|
|
79
82
|
timeout_seconds=getattr(
|
|
80
83
|
retrieval_config, "cross_encoder_timeout_seconds", 15.0,
|
|
81
84
|
),
|
|
85
|
+
trust_plain_http_lan=trust_plain_http_lan,
|
|
82
86
|
)
|
|
83
87
|
except RemoteRerankerConfigError as exc:
|
|
84
88
|
logger.error(
|
|
@@ -159,7 +159,30 @@ def consolidate_facts(
|
|
|
159
159
|
stats["error_detail"] = str(exc)
|
|
160
160
|
return stats
|
|
161
161
|
|
|
162
|
-
# Backward-compat: str | Path — open own connection.
|
|
162
|
+
# Backward-compat: str | Path — open our own connection.
|
|
163
|
+
#
|
|
164
|
+
# Type-checked, not assumed. This branch used to run for ANYTHING that was
|
|
165
|
+
# not a DatabaseManager, stringify it, and hand the result to
|
|
166
|
+
# sqlite3.connect — which creates whatever filename it is given. A test
|
|
167
|
+
# passing a MagicMock therefore had a real 4 KB SQLite file named
|
|
168
|
+
# "<MagicMock id='4422448000'>" written into the repository root, one per
|
|
169
|
+
# test. 42 of them had accumulated, and tests/test_ci_guards/
|
|
170
|
+
# test_no_magicmock_artifacts.py failed after any full-suite run.
|
|
171
|
+
#
|
|
172
|
+
# 4.0.6 is where this started firing: it wired consolidate_facts into
|
|
173
|
+
# run_maintenance, so every caller with a mock config reached this line.
|
|
174
|
+
#
|
|
175
|
+
# Refusing an unusable argument is also right beyond the test symptom —
|
|
176
|
+
# silently creating a database at a nonsense path cannot be what any caller
|
|
177
|
+
# wanted, and it hides the real bug (the caller passed the wrong thing).
|
|
178
|
+
if not isinstance(db_or_path, (str, Path)):
|
|
179
|
+
raise TypeError(
|
|
180
|
+
"consolidate_facts() expects a DatabaseManager, or a str/Path to "
|
|
181
|
+
f"memory.db for backward compatibility; got {type(db_or_path).__name__}. "
|
|
182
|
+
"Passing anything else previously created a database file named after "
|
|
183
|
+
"the object's repr."
|
|
184
|
+
)
|
|
185
|
+
|
|
163
186
|
logger.warning(
|
|
164
187
|
"consolidate_facts: passing a db_path is deprecated — pass a "
|
|
165
188
|
"DatabaseManager instead (Fix A backward-compat shim active)"
|
|
@@ -637,11 +637,103 @@ def run_maintenance(
|
|
|
637
637
|
except Exception as exc:
|
|
638
638
|
logger.warning("Entity summary consolidation failed: %s", exc)
|
|
639
639
|
|
|
640
|
+
# 4. Fact consolidation (v3.8.4 concurrency-safe path via DatabaseManager).
|
|
641
|
+
# Merges clusters of warm/cold atomic facts about the same entity into a
|
|
642
|
+
# single consolidated fact, archives the originals (NEVER deletes them),
|
|
643
|
+
# and records provenance in fact_consolidations.
|
|
644
|
+
#
|
|
645
|
+
# Uses the DatabaseManager path so LLM calls happen OUTSIDE the write lock:
|
|
646
|
+
# - Discover clusters in a short memory_read() (no write lock held).
|
|
647
|
+
# - Generate summary OUTSIDE any lock (Ollama/Cloud may take 30s).
|
|
648
|
+
# - Write per-cluster inside a short memory_write() (lock held for SQL only).
|
|
649
|
+
#
|
|
650
|
+
# NOT on the recall/store hot path. Runs as part of background maintenance.
|
|
651
|
+
counts["facts_consolidated"] = 0
|
|
652
|
+
try:
|
|
653
|
+
from superlocalmemory.core.fact_consolidator import consolidate_facts
|
|
654
|
+
|
|
655
|
+
fc_stats = consolidate_facts(
|
|
656
|
+
db,
|
|
657
|
+
profile_id=profile_id,
|
|
658
|
+
max_clusters=getattr(config, "max_consolidation_clusters", 20),
|
|
659
|
+
dry_run=False,
|
|
660
|
+
config=config,
|
|
661
|
+
)
|
|
662
|
+
counts["facts_consolidated"] = fc_stats.get("consolidated", 0)
|
|
663
|
+
if fc_stats.get("consolidated", 0) > 0:
|
|
664
|
+
logger.info(
|
|
665
|
+
"Fact consolidation: %d clusters merged, %d facts archived",
|
|
666
|
+
fc_stats.get("consolidated", 0),
|
|
667
|
+
fc_stats.get("facts_archived", 0),
|
|
668
|
+
)
|
|
669
|
+
except Exception as exc:
|
|
670
|
+
# WARNING, not debug, and a distinguishable count. Leaving this at debug
|
|
671
|
+
# with facts_consolidated=0 made a failing consolidation report exactly
|
|
672
|
+
# the same numbers as a healthy run with nothing to merge, so a step that
|
|
673
|
+
# never worked would look like a step with no work to do — and nobody
|
|
674
|
+
# would ever see it in normal logs.
|
|
675
|
+
counts["facts_consolidated"] = -1
|
|
676
|
+
logger.warning(
|
|
677
|
+
"Fact consolidation FAILED during maintenance (reported as -1, "
|
|
678
|
+
"which is distinct from 0 = nothing to merge): %s", exc,
|
|
679
|
+
)
|
|
680
|
+
|
|
681
|
+
# 5. Code↔memory bridge (4.0.7).
|
|
682
|
+
# Resolves code entity mentions in new facts against the code graph and
|
|
683
|
+
# stores the links, plus derived enrichment text, in code_graph.db.
|
|
684
|
+
#
|
|
685
|
+
# RUNS HERE, NOT ON THE WRITE PATH. The bridge was authored to fire from
|
|
686
|
+
# BridgeEventListeners.on_memory_stored, and EventBus._notify_listeners
|
|
687
|
+
# dispatches synchronously on the emitting thread — which would have put
|
|
688
|
+
# entity resolution and enrichment inside every remember. The owner's
|
|
689
|
+
# constraint for 4.0.7 is that remember/recall timing must not move, so the
|
|
690
|
+
# memory-stored subscription is gone and the work happens in this pass.
|
|
691
|
+
# tests/test_code_graph/test_bridge_off_write_path.py fails if it comes back.
|
|
692
|
+
#
|
|
693
|
+
# Writes only code_graph.db, which no recall path opens, so this step cannot
|
|
694
|
+
# affect recall latency or results. Hebbian edges are the one exception and
|
|
695
|
+
# are NOT run here — they land in association_edges, which spreading
|
|
696
|
+
# activation reads; they are generated on explicit request instead.
|
|
697
|
+
counts["bridge_links"] = 0
|
|
698
|
+
counts["bridge_enriched"] = 0
|
|
699
|
+
try:
|
|
700
|
+
from superlocalmemory.code_graph.config import CodeGraphConfig
|
|
701
|
+
|
|
702
|
+
cg_cfg = CodeGraphConfig.load()
|
|
703
|
+
if cg_cfg.enabled and cg_cfg.bridge_enabled:
|
|
704
|
+
from superlocalmemory.code_graph.bridge.maintenance import run_bridge_pass
|
|
705
|
+
from superlocalmemory.code_graph.database import CodeGraphDatabase
|
|
706
|
+
|
|
707
|
+
cg_path = cg_cfg.get_db_path()
|
|
708
|
+
if cg_path.exists():
|
|
709
|
+
bridge_stats = run_bridge_pass(
|
|
710
|
+
db, CodeGraphDatabase(cg_path), profile_id,
|
|
711
|
+
)
|
|
712
|
+
counts["bridge_links"] = bridge_stats.get("links_created", 0)
|
|
713
|
+
counts["bridge_enriched"] = bridge_stats.get("enriched", 0)
|
|
714
|
+
except Exception as exc:
|
|
715
|
+
# -1 rather than 0, for the same reason fact consolidation uses it:
|
|
716
|
+
# a step that never worked must not report the same numbers as a
|
|
717
|
+
# healthy step with no work to do.
|
|
718
|
+
counts["bridge_links"] = -1
|
|
719
|
+
logger.warning(
|
|
720
|
+
"Code bridge pass FAILED during maintenance (reported as -1, "
|
|
721
|
+
"which is distinct from 0 = nothing to link): %s", exc,
|
|
722
|
+
)
|
|
723
|
+
|
|
640
724
|
logger.info(
|
|
641
725
|
"Maintenance complete: %d backfilled, %d Langevin, %d Fisher-coupled, "
|
|
642
|
-
"%d Sheaf, %d entity-summaries",
|
|
726
|
+
"%d Sheaf, %d entity-summaries, %d facts-consolidated, %d code-links",
|
|
643
727
|
counts["langevin_backfilled"], counts["langevin_updated"],
|
|
644
728
|
counts["fisher_coupled"], counts["sheaf_checked"],
|
|
645
729
|
counts["entity_summaries_consolidated"],
|
|
730
|
+
counts["facts_consolidated"],
|
|
731
|
+
counts["bridge_links"],
|
|
646
732
|
)
|
|
647
733
|
return counts
|
|
734
|
+
|
|
735
|
+
|
|
736
|
+
# The bridge gate reads code_graph_config.json via CodeGraphConfig.load(), not
|
|
737
|
+
# SLMConfig: SLMConfig has no code_graph block, so there is nothing to read
|
|
738
|
+
# there. cli/setup_wizard.py writes that file; before 4.0.7 no loader existed
|
|
739
|
+
# and every call site hardcoded CodeGraphConfig(enabled=True) instead.
|