superlocalmemory 3.7.8 → 3.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/ATTRIBUTION.md +1 -3
- package/CHANGELOG.md +69 -0
- package/README.md +199 -29
- package/package.json +4 -2
- package/plugin/.claude-plugin/plugin.json +2 -2
- package/plugin/CLAUDE.md +8 -8
- package/plugin/agents/slm-governance-advisor.md +80 -0
- package/plugin/agents/slm-loop-runner.md +71 -0
- package/plugin/agents/slm-memory-advisor.md +10 -5
- package/plugin/agents/slm-optimize-advisor.md +9 -3
- package/plugin/commands/slm-loop.md +31 -0
- package/plugin/hooks/hooks.json +79 -0
- package/plugin/requirements.txt +1 -1
- package/plugin/scripts/slm-launch +46 -7
- package/plugin/settings.json +9 -0
- package/plugin/skills/slm-cache/SKILL.md +9 -1
- package/plugin/skills/slm-compress/SKILL.md +8 -1
- package/plugin/skills/slm-governance/SKILL.md +248 -0
- package/plugin/skills/slm-graph/SKILL.md +17 -3
- package/plugin/skills/slm-loop/SKILL.md +99 -0
- package/plugin/skills/slm-mesh/SKILL.md +282 -0
- package/plugin/skills/slm-profile/SKILL.md +148 -0
- package/plugin/skills/slm-recall/SKILL.md +46 -10
- package/plugin/skills/slm-remember/SKILL.md +48 -1
- package/plugin/skills/slm-scope/SKILL.md +176 -0
- package/plugin/skills/slm-session/SKILL.md +24 -1
- package/plugin/skills/slm-status/SKILL.md +18 -1
- package/plugin-src/agents/slm-governance-advisor.md +80 -0
- package/plugin-src/agents/slm-loop-runner.md +71 -0
- package/plugin-src/agents/slm-memory-advisor.md +10 -5
- package/plugin-src/agents/slm-optimize-advisor.md +9 -3
- package/plugin-src/commands/slm-loop.md +31 -0
- package/plugin-src/hooks/hooks.json +79 -0
- package/plugin-src/manifest.json +7 -2
- package/plugin-src/requirements.txt +1 -1
- package/plugin-src/rules/AGENTS.md +57 -18
- package/plugin-src/rules/CLAUDE.md.fragment +8 -8
- package/plugin-src/scripts/slm-launch +46 -7
- package/plugin-src/settings.json +9 -0
- package/plugin-src/skills/slm-cache/SKILL.md +9 -1
- package/plugin-src/skills/slm-compress/SKILL.md +8 -1
- package/plugin-src/skills/slm-governance/SKILL.md +248 -0
- package/plugin-src/skills/slm-graph/SKILL.md +17 -3
- package/plugin-src/skills/slm-loop/SKILL.md +99 -0
- package/plugin-src/skills/slm-mesh/SKILL.md +282 -0
- package/plugin-src/skills/slm-profile/SKILL.md +148 -0
- package/plugin-src/skills/slm-recall/SKILL.md +46 -10
- package/plugin-src/skills/slm-remember/SKILL.md +48 -1
- package/plugin-src/skills/slm-scope/SKILL.md +176 -0
- package/plugin-src/skills/slm-session/SKILL.md +24 -1
- package/plugin-src/skills/slm-status/SKILL.md +18 -1
- package/pyproject.toml +1 -1
- package/scripts/postinstall/validation.js +2 -0
- package/scripts/postinstall-interactive.js +74 -2
- package/src/superlocalmemory/__init__.py +1 -1
- package/src/superlocalmemory/access/__init__.py +3 -0
- package/src/superlocalmemory/access/rbac.py +477 -0
- package/src/superlocalmemory/cli/commands.py +94 -10
- package/src/superlocalmemory/cli/compress_cmd.py +17 -7
- package/src/superlocalmemory/cli/loop_cmd.py +192 -0
- package/src/superlocalmemory/cli/main.py +39 -4
- package/src/superlocalmemory/cli/mesh_cmd.py +38 -0
- package/src/superlocalmemory/cli/optimize_cmd.py +3 -0
- package/src/superlocalmemory/cli/pending_store.py +49 -13
- package/src/superlocalmemory/cli/proxy_cmd.py +4 -0
- package/src/superlocalmemory/cli/scale_engine_cmd.py +6 -0
- package/src/superlocalmemory/cli/setup_wizard.py +22 -13
- package/src/superlocalmemory/compliance/audit.py +6 -0
- package/src/superlocalmemory/compliance/gdpr.py +128 -138
- package/src/superlocalmemory/compliance/retention.py +176 -45
- package/src/superlocalmemory/core/backend_orchestrator.py +5 -43
- package/src/superlocalmemory/core/community_summary.py +267 -0
- package/src/superlocalmemory/core/config.py +216 -3
- package/src/superlocalmemory/core/consolidation_engine.py +95 -22
- package/src/superlocalmemory/core/context_cache.py +61 -18
- package/src/superlocalmemory/core/embedding_worker.py +17 -2
- package/src/superlocalmemory/core/embeddings.py +12 -1
- package/src/superlocalmemory/core/engine.py +17 -1
- package/src/superlocalmemory/core/engine_ingestion.py +29 -0
- package/src/superlocalmemory/core/engine_wiring.py +13 -0
- package/src/superlocalmemory/core/entity_community.py +178 -0
- package/src/superlocalmemory/core/graph_analyzer.py +39 -2
- package/src/superlocalmemory/core/graph_pruner.py +13 -8
- package/src/superlocalmemory/core/key_expander.py +138 -0
- package/src/superlocalmemory/core/maintenance.py +23 -0
- package/src/superlocalmemory/core/modes.py +1 -1
- package/src/superlocalmemory/core/mutations.py +2 -2
- package/src/superlocalmemory/core/pii.py +105 -0
- package/src/superlocalmemory/core/progressive_abstraction.py +208 -0
- package/src/superlocalmemory/core/recall_pipeline.py +2 -0
- package/src/superlocalmemory/core/recall_worker.py +20 -6
- package/src/superlocalmemory/core/scale_engine.py +60 -1
- package/src/superlocalmemory/core/security_primitives.py +40 -2
- package/src/superlocalmemory/core/store_pipeline.py +35 -11
- package/src/superlocalmemory/core/worker_pool.py +21 -6
- package/src/superlocalmemory/encoding/entity_reflexion.py +200 -0
- package/src/superlocalmemory/encoding/entity_resolver.py +34 -24
- package/src/superlocalmemory/encoding/fact_extractor.py +26 -1
- package/src/superlocalmemory/encoding/temporal_validator.py +64 -1
- package/src/superlocalmemory/evolution/evolution_store.py +122 -45
- package/src/superlocalmemory/evolution/llm_dispatch.py +12 -1
- package/src/superlocalmemory/evolution/model_selection.py +160 -0
- package/src/superlocalmemory/evolution/mutation_generator.py +16 -0
- package/src/superlocalmemory/evolution/skill_evolver.py +127 -42
- package/src/superlocalmemory/evolution/triggers.py +22 -13
- package/src/superlocalmemory/graph/cozo_backend.py +43 -20
- package/src/superlocalmemory/hooks/adapter_base.py +5 -1
- package/src/superlocalmemory/hooks/auto_recall.py +13 -1
- package/src/superlocalmemory/hooks/claude_code_hooks.py +11 -0
- package/src/superlocalmemory/hooks/codex_assets.py +64 -5
- package/src/superlocalmemory/hooks/hook_daemon.py +20 -3
- package/src/superlocalmemory/hooks/memory_protocol.py +54 -0
- package/src/superlocalmemory/hooks/portable_kit.py +114 -1
- package/src/superlocalmemory/infra/backup.py +12 -1
- package/src/superlocalmemory/infra/daemon_identity.py +40 -4
- package/src/superlocalmemory/infra/data_root.py +43 -4
- package/src/superlocalmemory/infra/event_bus.py +107 -24
- package/src/superlocalmemory/infra/rate_limiter.py +93 -0
- package/src/superlocalmemory/ingestion/adapter_manager.py +4 -1
- package/src/superlocalmemory/ingestion/credentials.py +1 -1
- package/src/superlocalmemory/learning/cross_project.py +28 -19
- package/src/superlocalmemory/learning/reward_proxy.py +42 -9
- package/src/superlocalmemory/loops/__init__.py +56 -0
- package/src/superlocalmemory/loops/budget.py +58 -0
- package/src/superlocalmemory/loops/engine.py +164 -0
- package/src/superlocalmemory/loops/ledger.py +243 -0
- package/src/superlocalmemory/loops/models.py +152 -0
- package/src/superlocalmemory/loops/rules.py +52 -0
- package/src/superlocalmemory/mcp/_daemon_proxy.py +3 -0
- package/src/superlocalmemory/mcp/_pool_adapter.py +2 -0
- package/src/superlocalmemory/mcp/profiles.py +103 -0
- package/src/superlocalmemory/mcp/server.py +21 -49
- package/src/superlocalmemory/mcp/tools_active.py +4 -7
- package/src/superlocalmemory/mcp/tools_code_graph.py +51 -5
- package/src/superlocalmemory/mcp/tools_core.py +8 -1
- package/src/superlocalmemory/mcp/tools_evolution.py +6 -3
- package/src/superlocalmemory/mcp/tools_loops.py +300 -0
- package/src/superlocalmemory/mcp/tools_mesh.py +140 -4
- package/src/superlocalmemory/mcp/tools_optimize.py +15 -8
- package/src/superlocalmemory/mesh/broker.py +237 -129
- package/src/superlocalmemory/mesh/remote_sync.py +50 -8
- package/src/superlocalmemory/optimize/NOTICE +1 -6
- package/src/superlocalmemory/optimize/adapters/anthropic_adapter.py +1 -4
- package/src/superlocalmemory/optimize/adapters/openai_adapter.py +1 -4
- package/src/superlocalmemory/optimize/cache/semantic.py +27 -19
- package/src/superlocalmemory/optimize/compress/align.py +32 -26
- package/src/superlocalmemory/optimize/compress/ccr.py +14 -71
- package/src/superlocalmemory/optimize/compress/router.py +105 -22
- package/src/superlocalmemory/optimize/config/defaults.py +1 -1
- package/src/superlocalmemory/optimize/config/schema.py +87 -4
- package/src/superlocalmemory/optimize/metrics/counters.py +13 -4
- package/src/superlocalmemory/optimize/metrics/estimator.py +0 -3
- package/src/superlocalmemory/optimize/proxy/_helpers.py +31 -4
- package/src/superlocalmemory/optimize/storage/db.py +38 -9
- package/src/superlocalmemory/optimize/storage/schema.py +10 -0
- package/src/superlocalmemory/parameterization/pattern_extractor.py +6 -3
- package/src/superlocalmemory/retrieval/agentic.py +1 -1
- package/src/superlocalmemory/retrieval/bm25_channel.py +68 -10
- package/src/superlocalmemory/retrieval/engine.py +168 -26
- package/src/superlocalmemory/retrieval/entity_channel.py +7 -5
- package/src/superlocalmemory/retrieval/hopfield_channel.py +9 -2
- package/src/superlocalmemory/retrieval/semantic_channel.py +114 -21
- package/src/superlocalmemory/retrieval/spreading_activation.py +11 -2
- package/src/superlocalmemory/retrieval/temporal_channel.py +48 -9
- package/src/superlocalmemory/retrieval/temporal_frame.py +102 -0
- package/src/superlocalmemory/retrieval/temporal_validity_filter.py +135 -0
- package/src/superlocalmemory/retrieval/time_window.py +181 -0
- package/src/superlocalmemory/server/api.py +4 -4
- package/src/superlocalmemory/server/profile_runtime.py +125 -8
- package/src/superlocalmemory/server/rbac_enforce.py +142 -0
- package/src/superlocalmemory/server/recall_health.py +24 -3
- package/src/superlocalmemory/server/recall_serializer.py +19 -1
- package/src/superlocalmemory/server/routes/abstraction.py +115 -0
- package/src/superlocalmemory/server/routes/agents.py +128 -38
- package/src/superlocalmemory/server/routes/backup.py +34 -10
- package/src/superlocalmemory/server/routes/behavioral.py +13 -12
- package/src/superlocalmemory/server/routes/brain.py +21 -5
- package/src/superlocalmemory/server/routes/chat.py +10 -5
- package/src/superlocalmemory/server/routes/compliance.py +171 -21
- package/src/superlocalmemory/server/routes/config_api.py +436 -0
- package/src/superlocalmemory/server/routes/data_io.py +30 -8
- package/src/superlocalmemory/server/routes/entity.py +9 -4
- package/src/superlocalmemory/server/routes/events.py +24 -8
- package/src/superlocalmemory/server/routes/evolution.py +135 -17
- package/src/superlocalmemory/server/routes/helpers.py +16 -1
- package/src/superlocalmemory/server/routes/ingest.py +7 -4
- package/src/superlocalmemory/server/routes/insights.py +3 -3
- package/src/superlocalmemory/server/routes/learning.py +14 -14
- package/src/superlocalmemory/server/routes/lifecycle.py +59 -8
- package/src/superlocalmemory/server/routes/memories.py +182 -57
- package/src/superlocalmemory/server/routes/mesh.py +95 -15
- package/src/superlocalmemory/server/routes/optimize.py +33 -1
- package/src/superlocalmemory/server/routes/prewarm.py +2 -0
- package/src/superlocalmemory/server/routes/profiles.py +63 -17
- package/src/superlocalmemory/server/routes/ratelimit.py +124 -0
- package/src/superlocalmemory/server/routes/rbac.py +367 -0
- package/src/superlocalmemory/server/routes/stats.py +13 -6
- package/src/superlocalmemory/server/routes/tiers.py +11 -9
- package/src/superlocalmemory/server/routes/v3_api.py +183 -69
- package/src/superlocalmemory/server/routes/ws.py +5 -2
- package/src/superlocalmemory/server/security_middleware.py +12 -5
- package/src/superlocalmemory/server/ui.py +20 -5
- package/src/superlocalmemory/server/unified_daemon.py +384 -56
- package/src/superlocalmemory/server/write_identity.py +38 -8
- package/src/superlocalmemory/storage/database.py +265 -53
- package/src/superlocalmemory/storage/migration_runner.py +53 -0
- package/src/superlocalmemory/storage/migrations/M021_ingestion_log_profile.py +108 -0
- package/src/superlocalmemory/storage/migrations/M022_entity_aliases_profile.py +86 -0
- package/src/superlocalmemory/storage/migrations/M023_mesh_profile_isolation.py +194 -0
- package/src/superlocalmemory/storage/migrations/M024_rbac_users_roles.py +87 -0
- package/src/superlocalmemory/storage/migrations/M025_perf_indexes.py +90 -0
- package/src/superlocalmemory/storage/migrations/M026_rbac_memberships_fk.py +136 -0
- package/src/superlocalmemory/storage/migrations/M027_transferable_patterns_profile.py +163 -0
- package/src/superlocalmemory/storage/models.py +4 -0
- package/src/superlocalmemory/storage/schema.py +87 -0
- package/src/superlocalmemory/storage/schema_v343.py +24 -12
- package/src/superlocalmemory/trust/gate.py +49 -8
- package/src/superlocalmemory/ui/assets/slm-icon-white.svg +64 -0
- package/src/superlocalmemory/ui/assets/slm-icon.svg +36 -0
- package/src/superlocalmemory/ui/css/design-system.css +621 -0
- package/src/superlocalmemory/ui/css/neural-glass.css +6 -0
- package/src/superlocalmemory/ui/css/od-bridge.css +158 -0
- package/src/superlocalmemory/ui/favicon.svg +35 -4
- package/src/superlocalmemory/ui/index.html +306 -173
- package/src/superlocalmemory/ui/js/brain.js +5 -20
- package/src/superlocalmemory/ui/js/core.js +47 -31
- package/src/superlocalmemory/ui/js/dashboard.js +314 -63
- package/src/superlocalmemory/ui/js/event-delegation.js +102 -0
- package/src/superlocalmemory/ui/js/knowledge-graph.js +11 -11
- package/src/superlocalmemory/ui/js/math-health.js +1 -1
- package/src/superlocalmemory/ui/js/memories.js +15 -4
- package/src/superlocalmemory/ui/js/memory-chat.js +7 -7
- package/src/superlocalmemory/ui/js/ng-entities.js +6 -8
- package/src/superlocalmemory/ui/js/ng-ingestion.js +4 -4
- package/src/superlocalmemory/ui/js/ng-mesh.js +4 -9
- package/src/superlocalmemory/ui/js/ng-shell.js +8 -8
- package/src/superlocalmemory/ui/js/ng-skills.js +54 -2
- package/src/superlocalmemory/ui/js/od-agents.js +544 -0
- package/src/superlocalmemory/ui/js/od-auth-gate.js +257 -0
- package/src/superlocalmemory/ui/js/od-backup.js +780 -0
- package/src/superlocalmemory/ui/js/od-brain.js +779 -0
- package/src/superlocalmemory/ui/js/od-entities.js +579 -0
- package/src/superlocalmemory/ui/js/od-graph.js +593 -0
- package/src/superlocalmemory/ui/js/od-health.js +539 -0
- package/src/superlocalmemory/ui/js/od-mcp.js +508 -0
- package/src/superlocalmemory/ui/js/od-memories.js +887 -0
- package/src/superlocalmemory/ui/js/od-mesh.js +539 -0
- package/src/superlocalmemory/ui/js/od-operations.js +1250 -0
- package/src/superlocalmemory/ui/js/od-optimize.js +787 -0
- package/src/superlocalmemory/ui/js/od-settings.js +1053 -0
- package/src/superlocalmemory/ui/js/od-shell.js +593 -0
- package/src/superlocalmemory/ui/js/od-skills.js +573 -0
- package/src/superlocalmemory/ui/js/od-team.js +258 -0
- package/src/superlocalmemory/ui/js/profiles.js +159 -46
- package/src/superlocalmemory/ui/js/settings.js +2 -2
- package/src/superlocalmemory/ui/js/timeline.js +34 -5
- package/src/superlocalmemory/ui/js/trust-dashboard.js +2 -2
- package/src/superlocalmemory/vector/lancedb_backend.py +8 -6
- package/src/superlocalmemory/learning/behavioral_listener.py +0 -94
|
@@ -49,6 +49,59 @@ if TYPE_CHECKING:
|
|
|
49
49
|
logger = logging.getLogger(__name__)
|
|
50
50
|
|
|
51
51
|
|
|
52
|
+
def _recompute_entity_communities(
|
|
53
|
+
db: Any, profile_id: str, summarizer: Any = None,
|
|
54
|
+
) -> dict[str, int]:
|
|
55
|
+
"""Wave Q: rebuild the entity-community backbone + summaries (fail-open).
|
|
56
|
+
|
|
57
|
+
Shared spine for Q2 community summaries and Q3 progressive abstraction.
|
|
58
|
+
Runs in the background consolidation lane; never blocks store/recall.
|
|
59
|
+
Community detection and summary generation are independently fail-open.
|
|
60
|
+
"""
|
|
61
|
+
result: dict[str, int] = {"entity_count": 0, "community_count": 0}
|
|
62
|
+
try:
|
|
63
|
+
from superlocalmemory.core.entity_community import EntityCommunityBuilder
|
|
64
|
+
|
|
65
|
+
result = EntityCommunityBuilder(db).compute_and_store(profile_id)
|
|
66
|
+
logger.info(
|
|
67
|
+
"Background entity-community: %d entities, %d communities",
|
|
68
|
+
result.get("entity_count", 0),
|
|
69
|
+
result.get("community_count", 0),
|
|
70
|
+
)
|
|
71
|
+
except Exception as exc:
|
|
72
|
+
logger.debug("Entity-community recompute failed (non-fatal): %s", exc)
|
|
73
|
+
return result
|
|
74
|
+
|
|
75
|
+
# Wave Q2: one synthesized report per community (rides on the backbone).
|
|
76
|
+
try:
|
|
77
|
+
from superlocalmemory.core.community_summary import CommunitySummaryBuilder
|
|
78
|
+
|
|
79
|
+
summ = CommunitySummaryBuilder(db, summarizer=summarizer).compute_and_store(
|
|
80
|
+
profile_id,
|
|
81
|
+
)
|
|
82
|
+
result["summaries_written"] = summ.get("summaries_written", 0)
|
|
83
|
+
logger.info(
|
|
84
|
+
"Background community summaries: %d written",
|
|
85
|
+
summ.get("summaries_written", 0),
|
|
86
|
+
)
|
|
87
|
+
except Exception as exc:
|
|
88
|
+
logger.debug("Community summaries failed (non-fatal): %s", exc)
|
|
89
|
+
|
|
90
|
+
# Wave Q3: persona roll-up (top tier over the community summaries).
|
|
91
|
+
try:
|
|
92
|
+
from superlocalmemory.core.progressive_abstraction import (
|
|
93
|
+
ProgressiveAbstraction,
|
|
94
|
+
)
|
|
95
|
+
|
|
96
|
+
pa = ProgressiveAbstraction(db, summarizer=summarizer).compute_and_store(
|
|
97
|
+
profile_id,
|
|
98
|
+
)
|
|
99
|
+
result["persona_built"] = bool(pa.get("built", False))
|
|
100
|
+
except Exception as exc:
|
|
101
|
+
logger.debug("Persona roll-up failed (non-fatal): %s", exc)
|
|
102
|
+
return result
|
|
103
|
+
|
|
104
|
+
|
|
52
105
|
class ConsolidationEngine:
|
|
53
106
|
"""Sleep-time memory consolidation with 6-step cycle.
|
|
54
107
|
|
|
@@ -254,21 +307,27 @@ class ConsolidationEngine:
|
|
|
254
307
|
activation. Takes ~200-800ms, runs on daemon thread, zero impact
|
|
255
308
|
on store/recall latency.
|
|
256
309
|
"""
|
|
257
|
-
if self._graph_analyzer is None:
|
|
258
|
-
return
|
|
259
310
|
analyzer = self._graph_analyzer
|
|
260
311
|
pid = profile_id
|
|
312
|
+
db = self._db
|
|
313
|
+
summarizer = self._summarizer
|
|
261
314
|
|
|
262
315
|
def _run() -> None:
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
316
|
+
if analyzer is not None:
|
|
317
|
+
try:
|
|
318
|
+
result = analyzer.compute_and_store(pid)
|
|
319
|
+
logger.info(
|
|
320
|
+
"Background graph analysis complete: %d nodes, "
|
|
321
|
+
"%d communities",
|
|
322
|
+
result.get("node_count", 0),
|
|
323
|
+
result.get("community_count", 0),
|
|
324
|
+
)
|
|
325
|
+
except Exception as exc:
|
|
326
|
+
logger.debug(
|
|
327
|
+
"Background graph analysis failed (non-fatal): %s", exc,
|
|
328
|
+
)
|
|
329
|
+
# Wave Q: entity-community backbone + summaries (Q2/Q3 spine).
|
|
330
|
+
_recompute_entity_communities(db, pid, summarizer)
|
|
272
331
|
|
|
273
332
|
t = threading.Thread(target=_run, daemon=True, name="graph-analysis-bg")
|
|
274
333
|
t.start()
|
|
@@ -510,15 +569,21 @@ class ConsolidationEngine:
|
|
|
510
569
|
fact_id, profile_id,
|
|
511
570
|
)
|
|
512
571
|
|
|
513
|
-
# Fallback: direct SQL check
|
|
572
|
+
# Fallback: direct SQL check. Must consider BOTH valid_until (valid-time
|
|
573
|
+
# expiry) AND system_expired_at (transaction-time expiry) — a fact that
|
|
574
|
+
# was invalidated/erased sets system_expired_at, and ignoring it would
|
|
575
|
+
# let a GDPR-erased/superseded fact be promoted back into warm lifecycle.
|
|
514
576
|
rows = self._db.execute(
|
|
515
|
-
"SELECT valid_until FROM fact_temporal_validity "
|
|
577
|
+
"SELECT valid_until, system_expired_at FROM fact_temporal_validity "
|
|
516
578
|
"WHERE fact_id = ? AND profile_id = ?",
|
|
517
579
|
(fact_id, profile_id),
|
|
518
580
|
)
|
|
519
581
|
if not rows:
|
|
520
582
|
return True # No temporal record = valid
|
|
521
|
-
|
|
583
|
+
row = dict(rows[0])
|
|
584
|
+
if row.get("system_expired_at") is not None:
|
|
585
|
+
return False # transaction-time expired (invalidated/erased)
|
|
586
|
+
valid_until = row.get("valid_until")
|
|
522
587
|
if valid_until is None:
|
|
523
588
|
return True # Open-ended validity
|
|
524
589
|
try:
|
|
@@ -555,14 +620,22 @@ class ConsolidationEngine:
|
|
|
555
620
|
def _step5_recompute_graph(
|
|
556
621
|
self, profile_id: str,
|
|
557
622
|
) -> dict[str, Any]:
|
|
558
|
-
"""Recompute PageRank + communities. Delegates to GraphAnalyzer.
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
623
|
+
"""Recompute PageRank + communities. Delegates to GraphAnalyzer.
|
|
624
|
+
|
|
625
|
+
Wave Q: also rebuilds the entity-community backbone (Q2/Q3 spine).
|
|
626
|
+
"""
|
|
627
|
+
result: dict[str, Any] = {"node_count": 0, "community_count": 0}
|
|
628
|
+
if self._graph_analyzer is not None:
|
|
629
|
+
try:
|
|
630
|
+
result = self._graph_analyzer.compute_and_store(profile_id)
|
|
631
|
+
except Exception as exc:
|
|
632
|
+
logger.warning("Graph recompute failed: %s", exc)
|
|
633
|
+
ec = _recompute_entity_communities(
|
|
634
|
+
self._db, profile_id, self._summarizer,
|
|
635
|
+
)
|
|
636
|
+
result["entity_community_count"] = ec.get("community_count", 0)
|
|
637
|
+
result["community_summaries"] = ec.get("summaries_written", 0)
|
|
638
|
+
return result
|
|
566
639
|
|
|
567
640
|
# ------------------------------------------------------------------
|
|
568
641
|
# Step 6: Derive Associations
|
|
@@ -65,6 +65,18 @@ class CacheEntry:
|
|
|
65
65
|
provenance: str = "tool_observation"
|
|
66
66
|
computed_at: int = 0
|
|
67
67
|
byte_size: int = 0
|
|
68
|
+
profile_id: str = "default"
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def _active_profile_fallback(home: Path) -> str:
|
|
72
|
+
"""Resolve the active profile for the cache reader hot path (stdlib only,
|
|
73
|
+
never raises). Two profiles can share a session_id, so cached context MUST
|
|
74
|
+
be keyed by profile or one tenant reads another's context."""
|
|
75
|
+
try:
|
|
76
|
+
raw = (home / "profiles.json").read_text(encoding="utf-8")
|
|
77
|
+
return json.loads(raw).get("active_profile", "default") or "default"
|
|
78
|
+
except Exception:
|
|
79
|
+
return "default"
|
|
68
80
|
|
|
69
81
|
|
|
70
82
|
# ---------------------------------------------------------------------------
|
|
@@ -199,9 +211,24 @@ class ContextCache:
|
|
|
199
211
|
return conn
|
|
200
212
|
|
|
201
213
|
def _bootstrap_schema_and_meta(self) -> None:
|
|
214
|
+
# Isolation: cached context is keyed by profile so two tenants sharing a
|
|
215
|
+
# session_id cannot read each other's context. Older cache files lack
|
|
216
|
+
# the profile_id column — the cache is ephemeral (120s TTL), so drop and
|
|
217
|
+
# recreate rather than run a rebuild migration.
|
|
218
|
+
try:
|
|
219
|
+
cols = {
|
|
220
|
+
r[1] for r in self._write_conn.execute(
|
|
221
|
+
"PRAGMA table_info(context_entries)"
|
|
222
|
+
).fetchall()
|
|
223
|
+
}
|
|
224
|
+
if cols and "profile_id" not in cols:
|
|
225
|
+
self._write_conn.execute("DROP TABLE context_entries")
|
|
226
|
+
except sqlite3.Error: # pragma: no cover — defensive
|
|
227
|
+
pass
|
|
202
228
|
self._write_conn.executescript(
|
|
203
229
|
"""
|
|
204
230
|
CREATE TABLE IF NOT EXISTS context_entries (
|
|
231
|
+
profile_id TEXT NOT NULL DEFAULT 'default',
|
|
205
232
|
session_id TEXT NOT NULL,
|
|
206
233
|
topic_sig TEXT NOT NULL,
|
|
207
234
|
content TEXT NOT NULL,
|
|
@@ -209,11 +236,11 @@ class ContextCache:
|
|
|
209
236
|
provenance TEXT NOT NULL DEFAULT 'tool_observation',
|
|
210
237
|
computed_at INTEGER NOT NULL,
|
|
211
238
|
byte_size INTEGER NOT NULL,
|
|
212
|
-
PRIMARY KEY (session_id, topic_sig)
|
|
239
|
+
PRIMARY KEY (profile_id, session_id, topic_sig)
|
|
213
240
|
) WITHOUT ROWID;
|
|
214
241
|
|
|
215
242
|
CREATE INDEX IF NOT EXISTS idx_ctx_session_time
|
|
216
|
-
ON context_entries(session_id, computed_at);
|
|
243
|
+
ON context_entries(profile_id, session_id, computed_at);
|
|
217
244
|
CREATE INDEX IF NOT EXISTS idx_ctx_time
|
|
218
245
|
ON context_entries(computed_at);
|
|
219
246
|
|
|
@@ -263,26 +290,38 @@ class ContextCache:
|
|
|
263
290
|
self._write_conn.execute(
|
|
264
291
|
"""
|
|
265
292
|
INSERT OR REPLACE INTO context_entries
|
|
266
|
-
(session_id, topic_sig, content, fact_ids,
|
|
293
|
+
(profile_id, session_id, topic_sig, content, fact_ids,
|
|
267
294
|
provenance, computed_at, byte_size)
|
|
268
|
-
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
295
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
269
296
|
""",
|
|
270
|
-
(entry.session_id, entry.topic_sig,
|
|
271
|
-
entry.provenance, computed_at, byte_size),
|
|
297
|
+
(entry.profile_id or "default", entry.session_id, entry.topic_sig,
|
|
298
|
+
content, fact_ids_json, entry.provenance, computed_at, byte_size),
|
|
272
299
|
)
|
|
273
300
|
|
|
274
301
|
# -- Cleanup ------------------------------------------------------------
|
|
275
302
|
|
|
276
303
|
def cleanup_session(
|
|
277
304
|
self, session_id: str, *, older_than: int = CLEANUP_HORIZON_SECONDS,
|
|
305
|
+
profile_id: str | None = None,
|
|
278
306
|
) -> int:
|
|
279
|
-
"""Delete rows for ``session_id`` older than ``older_than`` seconds.
|
|
307
|
+
"""Delete rows for ``session_id`` older than ``older_than`` seconds.
|
|
308
|
+
|
|
309
|
+
When ``profile_id`` is given the delete is tenant-scoped so a cleanup on
|
|
310
|
+
a shared session_id cannot wipe another profile's cached context.
|
|
311
|
+
"""
|
|
280
312
|
cutoff = int(time.time()) - older_than
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
313
|
+
if profile_id is not None:
|
|
314
|
+
cur = self._write_conn.execute(
|
|
315
|
+
"DELETE FROM context_entries "
|
|
316
|
+
"WHERE profile_id=? AND session_id=? AND computed_at < ?",
|
|
317
|
+
(profile_id, session_id, cutoff),
|
|
318
|
+
)
|
|
319
|
+
else:
|
|
320
|
+
cur = self._write_conn.execute(
|
|
321
|
+
"DELETE FROM context_entries "
|
|
322
|
+
"WHERE session_id=? AND computed_at < ?",
|
|
323
|
+
(session_id, cutoff),
|
|
324
|
+
)
|
|
286
325
|
return cur.rowcount
|
|
287
326
|
|
|
288
327
|
def cleanup_global_lru(self) -> int:
|
|
@@ -309,17 +348,17 @@ class ContextCache:
|
|
|
309
348
|
target = int(MAX_BYTES * 0.9)
|
|
310
349
|
while total > target:
|
|
311
350
|
rows = self._write_conn.execute(
|
|
312
|
-
"SELECT session_id, topic_sig, byte_size "
|
|
351
|
+
"SELECT profile_id, session_id, topic_sig, byte_size "
|
|
313
352
|
"FROM context_entries "
|
|
314
353
|
"ORDER BY computed_at ASC LIMIT 100",
|
|
315
354
|
).fetchall()
|
|
316
355
|
if not rows: # pragma: no cover — reached only if table empties mid-sweep
|
|
317
356
|
break
|
|
318
|
-
for sess, sig, size in rows:
|
|
357
|
+
for pid, sess, sig, size in rows:
|
|
319
358
|
self._write_conn.execute(
|
|
320
359
|
"DELETE FROM context_entries "
|
|
321
|
-
"WHERE session_id=? AND topic_sig=?",
|
|
322
|
-
(sess, sig),
|
|
360
|
+
"WHERE profile_id=? AND session_id=? AND topic_sig=?",
|
|
361
|
+
(pid, sess, sig),
|
|
323
362
|
)
|
|
324
363
|
deleted += 1
|
|
325
364
|
total -= size
|
|
@@ -345,6 +384,7 @@ def read_entry_fast(
|
|
|
345
384
|
*,
|
|
346
385
|
db_path: Path | None = None,
|
|
347
386
|
home_dir: Path | None = None,
|
|
387
|
+
profile_id: str | None = None,
|
|
348
388
|
) -> CacheEntry | None:
|
|
349
389
|
"""Hot-path reader used by the UserPromptSubmit hook.
|
|
350
390
|
|
|
@@ -397,14 +437,17 @@ def read_entry_fast(
|
|
|
397
437
|
return None
|
|
398
438
|
|
|
399
439
|
now = int(time.time())
|
|
440
|
+
# Scope to the active profile so a shared session_id cannot read
|
|
441
|
+
# another tenant's cached context.
|
|
442
|
+
pid = profile_id or _active_profile_fallback(home)
|
|
400
443
|
row = conn.execute(
|
|
401
444
|
"""
|
|
402
445
|
SELECT content, fact_ids, provenance, computed_at, byte_size
|
|
403
446
|
FROM context_entries
|
|
404
|
-
WHERE session_id=? AND topic_sig=?
|
|
447
|
+
WHERE profile_id=? AND session_id=? AND topic_sig=?
|
|
405
448
|
AND computed_at > ?
|
|
406
449
|
""",
|
|
407
|
-
(session_id, topic_sig, now - TTL_SECONDS),
|
|
450
|
+
(pid, session_id, topic_sig, now - TTL_SECONDS),
|
|
408
451
|
).fetchone()
|
|
409
452
|
finally:
|
|
410
453
|
try:
|
|
@@ -65,6 +65,21 @@ def _embedding_backend_order() -> tuple[str, str]:
|
|
|
65
65
|
return ("onnx", "pytorch")
|
|
66
66
|
|
|
67
67
|
|
|
68
|
+
# H-02 (3.7.9): trust_remote_code=_trusts_remote_code(name) runs arbitrary Python from the model
|
|
69
|
+
# repository at load time. Restrict it to the pinned models SLM ships that
|
|
70
|
+
# genuinely need custom modeling code (nomic-embed). Any other model — including
|
|
71
|
+
# one swapped into config by a write-path attacker — loads with
|
|
72
|
+
# trust_remote_code=False and therefore cannot execute repo code.
|
|
73
|
+
_TRUSTED_REMOTE_CODE_MODELS = frozenset({
|
|
74
|
+
"nomic-ai/nomic-embed-text-v1.5",
|
|
75
|
+
"nomic-ai/nomic-embed-text-v1",
|
|
76
|
+
})
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _trusts_remote_code(model_name: str) -> bool:
|
|
80
|
+
return model_name in _TRUSTED_REMOTE_CODE_MODELS
|
|
81
|
+
|
|
82
|
+
|
|
68
83
|
def _load_embedding_model(name: str) -> tuple:
|
|
69
84
|
"""Load embedding model. ONNX CPU-only first, PyTorch fallback.
|
|
70
85
|
|
|
@@ -84,14 +99,14 @@ def _load_embedding_model(name: str) -> tuple:
|
|
|
84
99
|
m = SentenceTransformer(
|
|
85
100
|
name,
|
|
86
101
|
backend="onnx",
|
|
87
|
-
trust_remote_code=
|
|
102
|
+
trust_remote_code=_trusts_remote_code(name),
|
|
88
103
|
model_kwargs={"provider": "CPUExecutionProvider"},
|
|
89
104
|
)
|
|
90
105
|
else:
|
|
91
106
|
import torch
|
|
92
107
|
with torch.inference_mode():
|
|
93
108
|
m = SentenceTransformer(
|
|
94
|
-
name, trust_remote_code=
|
|
109
|
+
name, trust_remote_code=_trusts_remote_code(name), device="cpu",
|
|
95
110
|
)
|
|
96
111
|
return m, backend
|
|
97
112
|
except Exception:
|
|
@@ -295,7 +295,13 @@ class EmbeddingService:
|
|
|
295
295
|
never hangs indefinitely on cold model loads or network issues.
|
|
296
296
|
"""
|
|
297
297
|
with self._lock:
|
|
298
|
-
|
|
298
|
+
# Only an explicit terminal disable (``False``) short-circuits. A
|
|
299
|
+
# ``None`` availability is the recall-health self-heal's "re-probe"
|
|
300
|
+
# signal (recall_health._heal_embedder) — it must fall through and
|
|
301
|
+
# respawn the worker, matching OllamaEmbedder's tri-state
|
|
302
|
+
# convention. Using ``not self._available`` here bricked the local
|
|
303
|
+
# worker on the first heal tick, because ``None`` is falsy.
|
|
304
|
+
if self._available is False:
|
|
299
305
|
return None
|
|
300
306
|
# Worker recycling: restart after N requests to prevent
|
|
301
307
|
# C++ allocator fragmentation over long-running sessions.
|
|
@@ -348,6 +354,11 @@ class EmbeddingService:
|
|
|
348
354
|
self._available = False
|
|
349
355
|
self._kill_worker()
|
|
350
356
|
return None
|
|
357
|
+
# A successful embed proves the worker is healthy, so clear any
|
|
358
|
+
# transient/``None`` availability left by a self-heal re-probe
|
|
359
|
+
# back to a definite ``True``. Without this the flag lingers at
|
|
360
|
+
# ``None`` and the next ``not``-style check elsewhere re-blocks.
|
|
361
|
+
self._available = True
|
|
351
362
|
self._reset_idle_timer()
|
|
352
363
|
self._request_count += 1
|
|
353
364
|
return resp["vectors"]
|
|
@@ -276,6 +276,18 @@ class MemoryEngine:
|
|
|
276
276
|
)
|
|
277
277
|
self._llm = None
|
|
278
278
|
|
|
279
|
+
# H-03 (3.7.9): surface mode-capability degradation explicitly at
|
|
280
|
+
# startup. validate_mode_config existed but was never called, so Mode B
|
|
281
|
+
# silently using rule-based extraction after Ollama vanished (update,
|
|
282
|
+
# restart, port conflict) went unwarned. Each mode only checks its own
|
|
283
|
+
# capabilities, so passing the single llm-availability signal is safe.
|
|
284
|
+
from superlocalmemory.core.modes import validate_mode_config
|
|
285
|
+
_llm_up = getattr(self, "_llm", None) is not None
|
|
286
|
+
for _warning in validate_mode_config(
|
|
287
|
+
self._config.mode, has_ollama=_llm_up, has_cloud_llm=_llm_up,
|
|
288
|
+
):
|
|
289
|
+
logger.warning("Mode config: %s", _warning)
|
|
290
|
+
|
|
279
291
|
from superlocalmemory.trust.scorer import TrustScorer
|
|
280
292
|
from superlocalmemory.trust.provenance import ProvenanceTracker
|
|
281
293
|
from superlocalmemory.compliance.eu_ai_act import EUAIActChecker
|
|
@@ -374,7 +386,9 @@ class MemoryEngine:
|
|
|
374
386
|
return
|
|
375
387
|
|
|
376
388
|
base_dir = self._config.base_dir
|
|
377
|
-
|
|
389
|
+
# Only drain items enqueued under THIS engine's profile — a queued
|
|
390
|
+
# memory must never materialize under a profile it was not written for.
|
|
391
|
+
pending = get_pending(base_dir, limit=20, profile_id=self.profile_id)
|
|
378
392
|
if not pending:
|
|
379
393
|
return
|
|
380
394
|
|
|
@@ -599,6 +613,7 @@ class MemoryEngine:
|
|
|
599
613
|
*,
|
|
600
614
|
include_global: bool | None = None,
|
|
601
615
|
include_shared: bool | None = None,
|
|
616
|
+
window: str | tuple[str, str] | None = None,
|
|
602
617
|
) -> RecallResponse:
|
|
603
618
|
"""Recall relevant facts for a query.
|
|
604
619
|
|
|
@@ -650,6 +665,7 @@ class MemoryEngine:
|
|
|
650
665
|
fast=fast,
|
|
651
666
|
include_global=include_global,
|
|
652
667
|
include_shared=include_shared,
|
|
668
|
+
window=window,
|
|
653
669
|
)
|
|
654
670
|
except Exception as exc:
|
|
655
671
|
from superlocalmemory.infra.local_diagnostics import record_operation
|
|
@@ -11,9 +11,13 @@ implementation of queryable projection and complete derivation.
|
|
|
11
11
|
from __future__ import annotations
|
|
12
12
|
|
|
13
13
|
import hashlib
|
|
14
|
+
import logging
|
|
15
|
+
import os
|
|
14
16
|
import uuid
|
|
15
17
|
from typing import TYPE_CHECKING
|
|
16
18
|
|
|
19
|
+
logger = logging.getLogger(__name__)
|
|
20
|
+
|
|
17
21
|
from superlocalmemory.core.ingestion_command import (
|
|
18
22
|
IngestionCommand,
|
|
19
23
|
IngestionOperation,
|
|
@@ -30,6 +34,21 @@ if TYPE_CHECKING:
|
|
|
30
34
|
_PREBUILT_FACT_KEY = "_slm_prebuilt_fact_v1"
|
|
31
35
|
|
|
32
36
|
|
|
37
|
+
def _pii_redaction_enabled(engine: "MemoryEngine") -> bool:
|
|
38
|
+
"""C4: opt-in PII redaction on ingest.
|
|
39
|
+
|
|
40
|
+
On when the engine config sets ``pii_redaction`` truthy OR the
|
|
41
|
+
``SLM_PII_REDACTION`` env var is set (1/on/true/yes). Default OFF — personal
|
|
42
|
+
use is unchanged; team/company operators opt in.
|
|
43
|
+
"""
|
|
44
|
+
cfg = getattr(engine, "_config", None)
|
|
45
|
+
if cfg is not None and getattr(cfg, "pii_redaction", False):
|
|
46
|
+
return True
|
|
47
|
+
return os.environ.get("SLM_PII_REDACTION", "").strip().lower() in (
|
|
48
|
+
"1", "on", "true", "yes",
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
|
|
33
52
|
def content_passes_admission(content: str) -> bool:
|
|
34
53
|
"""Return whether raw content is eligible to reach durable ingestion.
|
|
35
54
|
|
|
@@ -140,6 +159,16 @@ def canonical_store(
|
|
|
140
159
|
error=ValueError("content rejected by local admission policy"),
|
|
141
160
|
)
|
|
142
161
|
return []
|
|
162
|
+
# C4: opt-in PII redaction. When enabled (config.pii_redaction or
|
|
163
|
+
# SLM_PII_REDACTION), scrub personal identifiers BEFORE the content is
|
|
164
|
+
# extracted, embedded, or persisted — nothing sensitive ever reaches disk.
|
|
165
|
+
if _pii_redaction_enabled(engine):
|
|
166
|
+
from superlocalmemory.core.pii import redact_pii
|
|
167
|
+
|
|
168
|
+
scrubbed, n_pii = redact_pii(content)
|
|
169
|
+
if n_pii:
|
|
170
|
+
content = scrubbed
|
|
171
|
+
logger.info("PII redaction: scrubbed %d identifier(s) on ingest", n_pii)
|
|
143
172
|
try:
|
|
144
173
|
command = build_engine_ingestion_command(engine)
|
|
145
174
|
receipt = command.submit(IngestionRequest(
|
|
@@ -588,6 +588,19 @@ def init_retrieval(
|
|
|
588
588
|
except Exception as exc:
|
|
589
589
|
logger.debug("Forgetting filter registration failed: %s", exc)
|
|
590
590
|
|
|
591
|
+
# Phase 4 (T1): Register bi-temporal validity filter. Drops superseded /
|
|
592
|
+
# system-invalidated facts from retrieval so contradicted memories never
|
|
593
|
+
# resurface. Pure SQL, safe in all modes; no-op until a fact is invalidated.
|
|
594
|
+
try:
|
|
595
|
+
from superlocalmemory.retrieval.temporal_validity_filter import (
|
|
596
|
+
register_temporal_validity_filter,
|
|
597
|
+
)
|
|
598
|
+
register_temporal_validity_filter(
|
|
599
|
+
engine._registry, db, config.temporal_validator,
|
|
600
|
+
)
|
|
601
|
+
except Exception as exc:
|
|
602
|
+
logger.debug("Temporal validity filter registration failed: %s", exc)
|
|
603
|
+
|
|
591
604
|
return engine
|
|
592
605
|
|
|
593
606
|
|
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
|
|
2
|
+
# Licensed under AGPL-3.0-or-later - see LICENSE file
|
|
3
|
+
# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
|
|
4
|
+
|
|
5
|
+
"""Entity-community backbone (Wave Q) — the single principled clustering spine.
|
|
6
|
+
|
|
7
|
+
Best-in-market memory graphs (GraphRAG, Graphiti/Zep) cluster the ENTITY
|
|
8
|
+
graph, not the raw fact/chunk graph. SLM's fact graph is capped-sparse
|
|
9
|
+
(5 edges/entity), so a fact-level clustering fragments. This module builds an
|
|
10
|
+
entity co-occurrence graph (two entities are linked when they appear together
|
|
11
|
+
in a fact, weighted by co-occurrence count) and runs Louvain community
|
|
12
|
+
detection over it — the correct target and, since entities are far fewer than
|
|
13
|
+
facts, a cheaper computation.
|
|
14
|
+
|
|
15
|
+
The resulting entity communities are the shared backbone for:
|
|
16
|
+
- Q2 community summaries (one synthesized report per community), and
|
|
17
|
+
- Q3 progressive abstraction (scenario/persona tiers + drill-down).
|
|
18
|
+
|
|
19
|
+
Runs in the background (consolidation lane), never on the hot recall path.
|
|
20
|
+
Fail-open and idempotent: a recompute fully replaces a profile's rows.
|
|
21
|
+
|
|
22
|
+
Part of Qualixar | Author: Varun Pratap Bhardwaj
|
|
23
|
+
License: AGPL-3.0-or-later
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
from __future__ import annotations
|
|
27
|
+
|
|
28
|
+
import json
|
|
29
|
+
import logging
|
|
30
|
+
from collections import Counter, defaultdict
|
|
31
|
+
from typing import Any
|
|
32
|
+
|
|
33
|
+
logger = logging.getLogger(__name__)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class EntityCommunityBuilder:
|
|
37
|
+
"""Build + persist entity communities via Louvain over co-occurrence."""
|
|
38
|
+
|
|
39
|
+
def __init__(
|
|
40
|
+
self,
|
|
41
|
+
db: Any,
|
|
42
|
+
min_community_size: int = 2,
|
|
43
|
+
resolution: float = 1.0,
|
|
44
|
+
seed: int = 42,
|
|
45
|
+
) -> None:
|
|
46
|
+
self._db = db
|
|
47
|
+
self._min_size = max(2, int(min_community_size))
|
|
48
|
+
self._resolution = float(resolution)
|
|
49
|
+
self._seed = int(seed)
|
|
50
|
+
|
|
51
|
+
# ------------------------------------------------------------------
|
|
52
|
+
# Detection
|
|
53
|
+
# ------------------------------------------------------------------
|
|
54
|
+
|
|
55
|
+
def _cooccurrence(self, profile_id: str) -> Counter:
|
|
56
|
+
"""Count entity pairs that co-occur within a fact (per profile)."""
|
|
57
|
+
rows = self._db.execute(
|
|
58
|
+
"SELECT canonical_entities_json FROM atomic_facts "
|
|
59
|
+
"WHERE profile_id = ?",
|
|
60
|
+
(profile_id,),
|
|
61
|
+
)
|
|
62
|
+
weights: Counter = Counter()
|
|
63
|
+
for row in rows:
|
|
64
|
+
raw = dict(row).get("canonical_entities_json")
|
|
65
|
+
if not raw:
|
|
66
|
+
continue
|
|
67
|
+
try:
|
|
68
|
+
parsed = json.loads(raw)
|
|
69
|
+
except (ValueError, TypeError):
|
|
70
|
+
continue
|
|
71
|
+
if not isinstance(parsed, list):
|
|
72
|
+
continue
|
|
73
|
+
ents = sorted({str(e).strip() for e in parsed if str(e).strip()})
|
|
74
|
+
for i in range(len(ents)):
|
|
75
|
+
for j in range(i + 1, len(ents)):
|
|
76
|
+
weights[(ents[i], ents[j])] += 1
|
|
77
|
+
return weights
|
|
78
|
+
|
|
79
|
+
def detect(self, profile_id: str) -> dict[str, int]:
|
|
80
|
+
"""Return {entity_id -> community_id}; communities below min size drop."""
|
|
81
|
+
weights = self._cooccurrence(profile_id)
|
|
82
|
+
if not weights:
|
|
83
|
+
return {}
|
|
84
|
+
|
|
85
|
+
import networkx as nx
|
|
86
|
+
|
|
87
|
+
g = nx.Graph()
|
|
88
|
+
for (a, b), w in weights.items():
|
|
89
|
+
g.add_edge(a, b, weight=w)
|
|
90
|
+
|
|
91
|
+
try:
|
|
92
|
+
from networkx.algorithms.community import louvain_communities
|
|
93
|
+
|
|
94
|
+
communities = louvain_communities(
|
|
95
|
+
g, weight="weight", resolution=self._resolution, seed=self._seed,
|
|
96
|
+
)
|
|
97
|
+
except Exception as exc: # pragma: no cover - fallback path
|
|
98
|
+
logger.debug(
|
|
99
|
+
"Louvain unavailable/failed (%s); using connected components",
|
|
100
|
+
exc,
|
|
101
|
+
)
|
|
102
|
+
communities = nx.connected_components(g)
|
|
103
|
+
|
|
104
|
+
result: dict[str, int] = {}
|
|
105
|
+
cid = 0
|
|
106
|
+
for comm in communities:
|
|
107
|
+
members = list(comm)
|
|
108
|
+
if len(members) < self._min_size:
|
|
109
|
+
continue
|
|
110
|
+
for node in members:
|
|
111
|
+
result[node] = cid
|
|
112
|
+
cid += 1
|
|
113
|
+
return result
|
|
114
|
+
|
|
115
|
+
# ------------------------------------------------------------------
|
|
116
|
+
# Persistence
|
|
117
|
+
# ------------------------------------------------------------------
|
|
118
|
+
|
|
119
|
+
def compute_and_store(self, profile_id: str) -> dict[str, int]:
|
|
120
|
+
"""Detect communities and replace this profile's stored rows."""
|
|
121
|
+
mapping = self.detect(profile_id)
|
|
122
|
+
try:
|
|
123
|
+
self._db.execute(
|
|
124
|
+
"DELETE FROM entity_communities WHERE profile_id = ?",
|
|
125
|
+
(profile_id,),
|
|
126
|
+
)
|
|
127
|
+
for entity_id, community_id in mapping.items():
|
|
128
|
+
self._db.execute(
|
|
129
|
+
"INSERT OR REPLACE INTO entity_communities "
|
|
130
|
+
"(profile_id, entity_id, community_id, computed_at) "
|
|
131
|
+
"VALUES (?, ?, ?, datetime('now'))",
|
|
132
|
+
(profile_id, entity_id, community_id),
|
|
133
|
+
)
|
|
134
|
+
except Exception as exc:
|
|
135
|
+
logger.debug("entity_communities persist failed: %s", exc)
|
|
136
|
+
|
|
137
|
+
return {
|
|
138
|
+
"entity_count": len(mapping),
|
|
139
|
+
"community_count": len(set(mapping.values())),
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
# ------------------------------------------------------------------
|
|
143
|
+
# Read API
|
|
144
|
+
# ------------------------------------------------------------------
|
|
145
|
+
|
|
146
|
+
def get_communities(self, profile_id: str) -> dict[int, list[str]]:
|
|
147
|
+
"""Return {community_id -> [entity_id, ...]} for a profile."""
|
|
148
|
+
try:
|
|
149
|
+
rows = self._db.execute(
|
|
150
|
+
"SELECT entity_id, community_id FROM entity_communities "
|
|
151
|
+
"WHERE profile_id = ? ORDER BY community_id",
|
|
152
|
+
(profile_id,),
|
|
153
|
+
)
|
|
154
|
+
except Exception as exc:
|
|
155
|
+
logger.debug("get_communities failed: %s", exc)
|
|
156
|
+
return {}
|
|
157
|
+
out: dict[int, list[str]] = defaultdict(list)
|
|
158
|
+
for row in rows:
|
|
159
|
+
d = dict(row)
|
|
160
|
+
out[int(d["community_id"])].append(str(d["entity_id"]))
|
|
161
|
+
return dict(out)
|
|
162
|
+
|
|
163
|
+
def get_community_for_entity(
|
|
164
|
+
self, entity_id: str, profile_id: str,
|
|
165
|
+
) -> int | None:
|
|
166
|
+
"""Return the community id for one entity, or None."""
|
|
167
|
+
try:
|
|
168
|
+
rows = self._db.execute(
|
|
169
|
+
"SELECT community_id FROM entity_communities "
|
|
170
|
+
"WHERE profile_id = ? AND entity_id = ?",
|
|
171
|
+
(profile_id, entity_id),
|
|
172
|
+
)
|
|
173
|
+
except Exception as exc:
|
|
174
|
+
logger.debug("get_community_for_entity failed: %s", exc)
|
|
175
|
+
return None
|
|
176
|
+
for row in rows:
|
|
177
|
+
return int(dict(row)["community_id"])
|
|
178
|
+
return None
|