superlocalmemory 3.7.8 → 3.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/ATTRIBUTION.md +1 -3
- package/CHANGELOG.md +129 -0
- package/README.md +205 -123
- package/package.json +12 -3
- package/plugin/.claude-plugin/plugin.json +2 -3
- package/plugin/CLAUDE.md +8 -8
- package/plugin/agents/slm-governance-advisor.md +80 -0
- package/plugin/agents/slm-loop-runner.md +71 -0
- package/plugin/agents/slm-memory-advisor.md +10 -5
- package/plugin/agents/slm-optimize-advisor.md +9 -3
- package/plugin/commands/slm-loop.md +31 -0
- package/plugin/hooks/hooks.json +79 -0
- package/plugin/requirements.txt +1 -1
- package/plugin/scripts/slm-launch +46 -7
- package/plugin/settings.json +9 -0
- package/plugin/skills/slm-cache/SKILL.md +9 -1
- package/plugin/skills/slm-compress/SKILL.md +8 -1
- package/plugin/skills/slm-governance/SKILL.md +248 -0
- package/plugin/skills/slm-graph/SKILL.md +17 -3
- package/plugin/skills/slm-loop/SKILL.md +99 -0
- package/plugin/skills/slm-mesh/SKILL.md +282 -0
- package/plugin/skills/slm-profile/SKILL.md +148 -0
- package/plugin/skills/slm-recall/SKILL.md +46 -10
- package/plugin/skills/slm-remember/SKILL.md +48 -1
- package/plugin/skills/slm-scope/SKILL.md +176 -0
- package/plugin/skills/slm-session/SKILL.md +24 -1
- package/plugin/skills/slm-status/SKILL.md +18 -1
- package/plugin-src/rules/AGENTS.md +57 -18
- package/plugin-src/skills/slm-cache/SKILL.md +9 -1
- package/plugin-src/skills/slm-compress/SKILL.md +8 -1
- package/plugin-src/skills/slm-graph/SKILL.md +17 -3
- package/plugin-src/skills/slm-recall/SKILL.md +46 -10
- package/plugin-src/skills/slm-remember/SKILL.md +48 -1
- package/plugin-src/skills/slm-session/SKILL.md +24 -1
- package/plugin-src/skills/slm-status/SKILL.md +18 -1
- package/pyproject.toml +2 -1
- package/scripts/postinstall/validation.js +2 -0
- package/scripts/postinstall-interactive.js +74 -2
- package/src/superlocalmemory/__init__.py +1 -1
- package/src/superlocalmemory/access/__init__.py +3 -0
- package/src/superlocalmemory/access/rbac.py +477 -0
- package/src/superlocalmemory/cli/commands.py +228 -17
- package/src/superlocalmemory/cli/compress_cmd.py +17 -7
- package/src/superlocalmemory/cli/daemon.py +7 -0
- package/src/superlocalmemory/cli/loop_cmd.py +187 -0
- package/src/superlocalmemory/cli/main.py +49 -8
- package/src/superlocalmemory/cli/mesh_cmd.py +38 -0
- package/src/superlocalmemory/cli/optimize_cmd.py +3 -0
- package/src/superlocalmemory/cli/pending_store.py +49 -13
- package/src/superlocalmemory/cli/proxy_cmd.py +4 -0
- package/src/superlocalmemory/cli/scale_engine_cmd.py +6 -0
- package/src/superlocalmemory/cli/setup_wizard.py +22 -13
- package/src/superlocalmemory/cli/version_banner.py +17 -3
- package/src/superlocalmemory/compliance/audit.py +6 -0
- package/src/superlocalmemory/compliance/gdpr.py +128 -138
- package/src/superlocalmemory/compliance/retention.py +176 -45
- package/src/superlocalmemory/core/backend_orchestrator.py +23 -59
- package/src/superlocalmemory/core/community_summary.py +267 -0
- package/src/superlocalmemory/core/config.py +216 -3
- package/src/superlocalmemory/core/consolidation_engine.py +95 -22
- package/src/superlocalmemory/core/context_cache.py +61 -18
- package/src/superlocalmemory/core/embedding_worker.py +21 -7
- package/src/superlocalmemory/core/embeddings.py +131 -46
- package/src/superlocalmemory/core/engine.py +41 -22
- package/src/superlocalmemory/core/engine_ingestion.py +359 -43
- package/src/superlocalmemory/core/engine_wiring.py +13 -0
- package/src/superlocalmemory/core/entity_community.py +178 -0
- package/src/superlocalmemory/core/graph_analyzer.py +39 -2
- package/src/superlocalmemory/core/graph_pruner.py +13 -8
- package/src/superlocalmemory/core/ingestion_command.py +134 -25
- package/src/superlocalmemory/core/injection.py +12 -7
- package/src/superlocalmemory/core/key_expander.py +138 -0
- package/src/superlocalmemory/core/maintenance.py +23 -0
- package/src/superlocalmemory/core/maintenance_scheduler.py +17 -7
- package/src/superlocalmemory/core/modes.py +1 -1
- package/src/superlocalmemory/core/mutations.py +2 -2
- package/src/superlocalmemory/core/pii.py +105 -0
- package/src/superlocalmemory/core/progressive_abstraction.py +208 -0
- package/src/superlocalmemory/core/recall_pipeline.py +7 -3
- package/src/superlocalmemory/core/recall_worker.py +20 -6
- package/src/superlocalmemory/core/scale_engine.py +60 -1
- package/src/superlocalmemory/core/security_primitives.py +40 -2
- package/src/superlocalmemory/core/store_pipeline.py +186 -29
- package/src/superlocalmemory/core/worker_pool.py +21 -6
- package/src/superlocalmemory/encoding/entity_reflexion.py +200 -0
- package/src/superlocalmemory/encoding/entity_resolver.py +34 -24
- package/src/superlocalmemory/encoding/fact_extractor.py +26 -1
- package/src/superlocalmemory/encoding/temporal_validator.py +64 -1
- package/src/superlocalmemory/evolution/evolution_store.py +122 -45
- package/src/superlocalmemory/evolution/llm_dispatch.py +12 -1
- package/src/superlocalmemory/evolution/model_selection.py +160 -0
- package/src/superlocalmemory/evolution/mutation_generator.py +16 -0
- package/src/superlocalmemory/evolution/skill_evolver.py +127 -42
- package/src/superlocalmemory/evolution/triggers.py +22 -13
- package/src/superlocalmemory/graph/cozo_backend.py +43 -20
- package/src/superlocalmemory/hooks/adapter_base.py +5 -1
- package/src/superlocalmemory/hooks/auto_recall.py +13 -1
- package/src/superlocalmemory/hooks/claude_code_hooks.py +11 -0
- package/src/superlocalmemory/hooks/codex_assets.py +64 -5
- package/src/superlocalmemory/hooks/hook_daemon.py +20 -3
- package/src/superlocalmemory/hooks/hook_handlers.py +6 -1
- package/src/superlocalmemory/hooks/memory_protocol.py +54 -0
- package/src/superlocalmemory/hooks/portable_kit.py +148 -3
- package/src/superlocalmemory/infra/backup.py +12 -1
- package/src/superlocalmemory/infra/daemon_identity.py +40 -4
- package/src/superlocalmemory/infra/data_root.py +43 -4
- package/src/superlocalmemory/infra/event_bus.py +107 -24
- package/src/superlocalmemory/infra/rate_limiter.py +93 -0
- package/src/superlocalmemory/ingestion/adapter_manager.py +4 -1
- package/src/superlocalmemory/ingestion/credentials.py +1 -1
- package/src/superlocalmemory/learning/cross_project.py +28 -19
- package/src/superlocalmemory/learning/model_rollback.py +3 -0
- package/src/superlocalmemory/learning/ranker_retrain_online.py +2 -0
- package/src/superlocalmemory/learning/reward.py +50 -0
- package/src/superlocalmemory/learning/reward_proxy.py +42 -9
- package/src/superlocalmemory/learning/source_quality.py +523 -1
- package/src/superlocalmemory/loops/__init__.py +56 -0
- package/src/superlocalmemory/loops/budget.py +58 -0
- package/src/superlocalmemory/loops/engine.py +164 -0
- package/src/superlocalmemory/loops/ledger.py +263 -0
- package/src/superlocalmemory/loops/models.py +152 -0
- package/src/superlocalmemory/loops/rules.py +52 -0
- package/src/superlocalmemory/mcp/_daemon_proxy.py +3 -0
- package/src/superlocalmemory/mcp/_pool_adapter.py +2 -0
- package/src/superlocalmemory/mcp/profiles.py +103 -0
- package/src/superlocalmemory/mcp/server.py +32 -79
- package/src/superlocalmemory/mcp/tools_active.py +4 -7
- package/src/superlocalmemory/mcp/tools_code_graph.py +51 -5
- package/src/superlocalmemory/mcp/tools_core.py +12 -4
- package/src/superlocalmemory/mcp/tools_evolution.py +6 -3
- package/src/superlocalmemory/mcp/tools_learning.py +2 -2
- package/src/superlocalmemory/mcp/tools_loops.py +300 -0
- package/src/superlocalmemory/mcp/tools_mesh.py +140 -4
- package/src/superlocalmemory/mcp/tools_optimize.py +15 -8
- package/src/superlocalmemory/mesh/broker.py +237 -129
- package/src/superlocalmemory/mesh/remote_sync.py +50 -8
- package/src/superlocalmemory/optimize/NOTICE +1 -6
- package/src/superlocalmemory/optimize/adapters/anthropic_adapter.py +1 -4
- package/src/superlocalmemory/optimize/adapters/openai_adapter.py +1 -4
- package/src/superlocalmemory/optimize/cache/semantic.py +27 -19
- package/src/superlocalmemory/optimize/compress/align.py +32 -26
- package/src/superlocalmemory/optimize/compress/ccr.py +14 -71
- package/src/superlocalmemory/optimize/compress/router.py +105 -22
- package/src/superlocalmemory/optimize/config/defaults.py +1 -1
- package/src/superlocalmemory/optimize/config/schema.py +87 -4
- package/src/superlocalmemory/optimize/metrics/counters.py +13 -4
- package/src/superlocalmemory/optimize/metrics/estimator.py +0 -3
- package/src/superlocalmemory/optimize/proxy/_helpers.py +31 -4
- package/src/superlocalmemory/optimize/storage/db.py +38 -9
- package/src/superlocalmemory/optimize/storage/schema.py +10 -0
- package/src/superlocalmemory/parameterization/pattern_extractor.py +6 -3
- package/src/superlocalmemory/retrieval/agentic.py +1 -1
- package/src/superlocalmemory/retrieval/bm25_channel.py +68 -10
- package/src/superlocalmemory/retrieval/engine.py +221 -47
- package/src/superlocalmemory/retrieval/entity_channel.py +7 -5
- package/src/superlocalmemory/retrieval/hopfield_channel.py +9 -2
- package/src/superlocalmemory/retrieval/reranker.py +3 -4
- package/src/superlocalmemory/retrieval/semantic_channel.py +114 -21
- package/src/superlocalmemory/retrieval/spreading_activation.py +11 -2
- package/src/superlocalmemory/retrieval/temporal_channel.py +48 -9
- package/src/superlocalmemory/retrieval/temporal_frame.py +102 -0
- package/src/superlocalmemory/retrieval/temporal_validity_filter.py +135 -0
- package/src/superlocalmemory/retrieval/time_window.py +181 -0
- package/src/superlocalmemory/server/api.py +4 -4
- package/src/superlocalmemory/server/config_file.py +90 -0
- package/src/superlocalmemory/server/origin.py +50 -0
- package/src/superlocalmemory/server/profile_runtime.py +125 -8
- package/src/superlocalmemory/server/rbac_enforce.py +142 -0
- package/src/superlocalmemory/server/recall_health.py +24 -3
- package/src/superlocalmemory/server/recall_serializer.py +19 -1
- package/src/superlocalmemory/server/routes/abstraction.py +115 -0
- package/src/superlocalmemory/server/routes/agents.py +128 -38
- package/src/superlocalmemory/server/routes/backup.py +317 -70
- package/src/superlocalmemory/server/routes/behavioral.py +349 -71
- package/src/superlocalmemory/server/routes/brain.py +69 -12
- package/src/superlocalmemory/server/routes/chat.py +10 -5
- package/src/superlocalmemory/server/routes/compliance.py +171 -21
- package/src/superlocalmemory/server/routes/config_api.py +438 -0
- package/src/superlocalmemory/server/routes/data_io.py +30 -8
- package/src/superlocalmemory/server/routes/entity.py +108 -26
- package/src/superlocalmemory/server/routes/events.py +24 -8
- package/src/superlocalmemory/server/routes/evolution.py +189 -68
- package/src/superlocalmemory/server/routes/helpers.py +16 -1
- package/src/superlocalmemory/server/routes/ingest.py +7 -4
- package/src/superlocalmemory/server/routes/insights.py +3 -3
- package/src/superlocalmemory/server/routes/learning.py +289 -118
- package/src/superlocalmemory/server/routes/learning_telemetry.py +153 -0
- package/src/superlocalmemory/server/routes/lifecycle.py +59 -8
- package/src/superlocalmemory/server/routes/memories.py +182 -57
- package/src/superlocalmemory/server/routes/mesh.py +200 -31
- package/src/superlocalmemory/server/routes/optimize.py +33 -1
- package/src/superlocalmemory/server/routes/prewarm.py +2 -0
- package/src/superlocalmemory/server/routes/profiles.py +63 -17
- package/src/superlocalmemory/server/routes/ratelimit.py +132 -0
- package/src/superlocalmemory/server/routes/rbac.py +367 -0
- package/src/superlocalmemory/server/routes/stats.py +103 -158
- package/src/superlocalmemory/server/routes/tiers.py +11 -9
- package/src/superlocalmemory/server/routes/token.py +3 -13
- package/src/superlocalmemory/server/routes/v3_api.py +247 -89
- package/src/superlocalmemory/server/routes/ws.py +5 -2
- package/src/superlocalmemory/server/security_middleware.py +12 -5
- package/src/superlocalmemory/server/ui.py +20 -5
- package/src/superlocalmemory/server/unified_daemon.py +827 -72
- package/src/superlocalmemory/server/write_identity.py +38 -8
- package/src/superlocalmemory/storage/database.py +265 -53
- package/src/superlocalmemory/storage/migration_runner.py +132 -1
- package/src/superlocalmemory/storage/migrations/M010_evolution_config.py +5 -0
- package/src/superlocalmemory/storage/migrations/M021_ingestion_log_profile.py +108 -0
- package/src/superlocalmemory/storage/migrations/M022_entity_aliases_profile.py +86 -0
- package/src/superlocalmemory/storage/migrations/M023_mesh_profile_isolation.py +194 -0
- package/src/superlocalmemory/storage/migrations/M024_rbac_users_roles.py +87 -0
- package/src/superlocalmemory/storage/migrations/M025_perf_indexes.py +90 -0
- package/src/superlocalmemory/storage/migrations/M026_rbac_memberships_fk.py +136 -0
- package/src/superlocalmemory/storage/migrations/M027_transferable_patterns_profile.py +163 -0
- package/src/superlocalmemory/storage/migrations/M028_fact_entity_associations.py +270 -0
- package/src/superlocalmemory/storage/migrations/M029_behavioral_history_indexes.py +137 -0
- package/src/superlocalmemory/storage/migrations/M030_entity_explorer_indexes.py +93 -0
- package/src/superlocalmemory/storage/migrations/__init__.py +4 -0
- package/src/superlocalmemory/storage/models.py +4 -0
- package/src/superlocalmemory/storage/schema.py +136 -1
- package/src/superlocalmemory/storage/schema_v32.py +2 -0
- package/src/superlocalmemory/storage/schema_v343.py +24 -12
- package/src/superlocalmemory/storage/schema_v347.py +4 -0
- package/src/superlocalmemory/trust/gate.py +49 -8
- package/src/superlocalmemory/ui/assets/slm-icon-white.svg +64 -0
- package/src/superlocalmemory/ui/assets/slm-icon.svg +36 -0
- package/src/superlocalmemory/ui/css/design-system.css +621 -0
- package/src/superlocalmemory/ui/css/neural-glass.css +6 -0
- package/src/superlocalmemory/ui/css/od-bridge.css +158 -0
- package/src/superlocalmemory/ui/favicon.svg +35 -4
- package/src/superlocalmemory/ui/index.html +303 -173
- package/src/superlocalmemory/ui/js/brain.js +5 -20
- package/src/superlocalmemory/ui/js/core.js +100 -41
- package/src/superlocalmemory/ui/js/dashboard.js +403 -65
- package/src/superlocalmemory/ui/js/event-delegation.js +102 -0
- package/src/superlocalmemory/ui/js/knowledge-graph.js +11 -11
- package/src/superlocalmemory/ui/js/math-health.js +1 -1
- package/src/superlocalmemory/ui/js/memories.js +15 -4
- package/src/superlocalmemory/ui/js/memory-chat.js +7 -7
- package/src/superlocalmemory/ui/js/ng-entities.js +6 -8
- package/src/superlocalmemory/ui/js/ng-ingestion.js +4 -4
- package/src/superlocalmemory/ui/js/ng-mesh.js +4 -9
- package/src/superlocalmemory/ui/js/ng-shell.js +8 -8
- package/src/superlocalmemory/ui/js/ng-skills.js +54 -2
- package/src/superlocalmemory/ui/js/od-agents.js +544 -0
- package/src/superlocalmemory/ui/js/od-auth-gate.js +257 -0
- package/src/superlocalmemory/ui/js/od-backup.js +871 -0
- package/src/superlocalmemory/ui/js/od-brain.js +816 -0
- package/src/superlocalmemory/ui/js/od-entities.js +579 -0
- package/src/superlocalmemory/ui/js/od-graph.js +600 -0
- package/src/superlocalmemory/ui/js/od-health.js +539 -0
- package/src/superlocalmemory/ui/js/od-mcp.js +508 -0
- package/src/superlocalmemory/ui/js/od-memories.js +929 -0
- package/src/superlocalmemory/ui/js/od-mesh.js +553 -0
- package/src/superlocalmemory/ui/js/od-operations.js +1250 -0
- package/src/superlocalmemory/ui/js/od-optimize.js +787 -0
- package/src/superlocalmemory/ui/js/od-settings.js +1107 -0
- package/src/superlocalmemory/ui/js/od-shell.js +809 -0
- package/src/superlocalmemory/ui/js/od-skills.js +600 -0
- package/src/superlocalmemory/ui/js/od-team.js +258 -0
- package/src/superlocalmemory/ui/js/profiles.js +159 -46
- package/src/superlocalmemory/ui/js/settings.js +17 -3
- package/src/superlocalmemory/ui/js/timeline.js +34 -5
- package/src/superlocalmemory/ui/js/trust-dashboard.js +2 -2
- package/src/superlocalmemory/vector/lancedb_backend.py +8 -6
- package/plugin-src/.mcp.json +0 -12
- package/plugin-src/agents/slm-memory-advisor.md +0 -44
- package/plugin-src/agents/slm-optimize-advisor.md +0 -38
- package/plugin-src/hooks/.gitkeep +0 -0
- package/plugin-src/hooks/hooks.json +0 -23
- package/plugin-src/manifest.json +0 -25
- package/plugin-src/requirements.txt +0 -1
- package/plugin-src/rules/CLAUDE.md.fragment +0 -44
- package/plugin-src/scripts/ensure-venv.bat +0 -122
- package/plugin-src/scripts/ensure-venv.sh +0 -105
- package/plugin-src/scripts/slm-launch +0 -23
- package/plugin-src/scripts/slm-launch.bat +0 -23
- package/plugin-src/settings.json +0 -16
- package/src/superlocalmemory/learning/behavioral_listener.py +0 -94
|
@@ -11,6 +11,8 @@ implementation of queryable projection and complete derivation.
|
|
|
11
11
|
from __future__ import annotations
|
|
12
12
|
|
|
13
13
|
import hashlib
|
|
14
|
+
import logging
|
|
15
|
+
import os
|
|
14
16
|
import uuid
|
|
15
17
|
from typing import TYPE_CHECKING
|
|
16
18
|
|
|
@@ -27,7 +29,26 @@ if TYPE_CHECKING:
|
|
|
27
29
|
from superlocalmemory.storage.models import AtomicFact
|
|
28
30
|
|
|
29
31
|
|
|
32
|
+
logger = logging.getLogger(__name__)
|
|
33
|
+
|
|
34
|
+
|
|
30
35
|
_PREBUILT_FACT_KEY = "_slm_prebuilt_fact_v1"
|
|
36
|
+
_DERIVATION_VERSION = "v3.7-ingestion-1"
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _pii_redaction_enabled(engine: "MemoryEngine") -> bool:
|
|
40
|
+
"""C4: opt-in PII redaction on ingest.
|
|
41
|
+
|
|
42
|
+
On when the engine config sets ``pii_redaction`` truthy OR the
|
|
43
|
+
``SLM_PII_REDACTION`` env var is set (1/on/true/yes). Default OFF — personal
|
|
44
|
+
use is unchanged; team/company operators opt in.
|
|
45
|
+
"""
|
|
46
|
+
cfg = getattr(engine, "_config", None)
|
|
47
|
+
if cfg is not None and getattr(cfg, "pii_redaction", False):
|
|
48
|
+
return True
|
|
49
|
+
return os.environ.get("SLM_PII_REDACTION", "").strip().lower() in (
|
|
50
|
+
"1", "on", "true", "yes",
|
|
51
|
+
)
|
|
31
52
|
|
|
32
53
|
|
|
33
54
|
def content_passes_admission(content: str) -> bool:
|
|
@@ -122,7 +143,14 @@ def canonical_store(
|
|
|
122
143
|
require_complete: bool = True,
|
|
123
144
|
return_receipt: bool = False,
|
|
124
145
|
) -> list[str] | IngestionOperation:
|
|
125
|
-
"""
|
|
146
|
+
"""Submit canonical evidence, optionally waiting for enrichment completion.
|
|
147
|
+
|
|
148
|
+
``require_complete=False`` is the interactive/CQRS path: it commits a
|
|
149
|
+
profile-scoped memory and FTS-queryable fact through M018, then returns the
|
|
150
|
+
durable receipt without invoking any LLM, embedding, or graph work. The
|
|
151
|
+
daemon materializer owns that expensive, retryable enrichment. Explicit
|
|
152
|
+
complete callers retain the historical synchronous contract.
|
|
153
|
+
"""
|
|
126
154
|
import time
|
|
127
155
|
|
|
128
156
|
from superlocalmemory.core.ingestion_command import IngestionRequest, IngestionState
|
|
@@ -140,6 +168,16 @@ def canonical_store(
|
|
|
140
168
|
error=ValueError("content rejected by local admission policy"),
|
|
141
169
|
)
|
|
142
170
|
return []
|
|
171
|
+
# C4: opt-in PII redaction. When enabled (config.pii_redaction or
|
|
172
|
+
# SLM_PII_REDACTION), scrub personal identifiers BEFORE the content is
|
|
173
|
+
# extracted, embedded, or persisted — nothing sensitive ever reaches disk.
|
|
174
|
+
if _pii_redaction_enabled(engine):
|
|
175
|
+
from superlocalmemory.core.pii import redact_pii
|
|
176
|
+
|
|
177
|
+
scrubbed, n_pii = redact_pii(content)
|
|
178
|
+
if n_pii:
|
|
179
|
+
content = scrubbed
|
|
180
|
+
logger.info("PII redaction: scrubbed %d identifier(s) on ingest", n_pii)
|
|
143
181
|
try:
|
|
144
182
|
command = build_engine_ingestion_command(engine)
|
|
145
183
|
receipt = command.submit(IngestionRequest(
|
|
@@ -156,21 +194,16 @@ def canonical_store(
|
|
|
156
194
|
speaker=speaker,
|
|
157
195
|
role=role,
|
|
158
196
|
))
|
|
197
|
+
if not require_complete:
|
|
198
|
+
record_operation(
|
|
199
|
+
"remember",
|
|
200
|
+
client=trusted_actor_id,
|
|
201
|
+
duration_ms=(time.monotonic() - started) * 1000.0,
|
|
202
|
+
)
|
|
203
|
+
return receipt if return_receipt else list(receipt.fact_ids)
|
|
204
|
+
|
|
159
205
|
result = command.materialize(receipt.operation_id)
|
|
160
206
|
if result.state is not IngestionState.COMPLETE:
|
|
161
|
-
if not require_complete and receipt.fact_ids:
|
|
162
|
-
import logging
|
|
163
|
-
logging.getLogger(__name__).warning(
|
|
164
|
-
"Canonical operation %s remains %s and will retry: %s",
|
|
165
|
-
result.operation_id,
|
|
166
|
-
result.state.value,
|
|
167
|
-
result.last_error,
|
|
168
|
-
)
|
|
169
|
-
record_operation(
|
|
170
|
-
"remember", client=trusted_actor_id,
|
|
171
|
-
duration_ms=(time.monotonic() - started) * 1000.0,
|
|
172
|
-
)
|
|
173
|
-
return list(receipt.fact_ids)
|
|
174
207
|
raise RuntimeError(result.last_error or "canonical materialization failed")
|
|
175
208
|
except Exception as exc:
|
|
176
209
|
record_operation(
|
|
@@ -316,29 +349,138 @@ def build_engine_ingestion_command(engine: MemoryEngine) -> IngestionCommand:
|
|
|
316
349
|
index_external=False,
|
|
317
350
|
)
|
|
318
351
|
|
|
319
|
-
def
|
|
352
|
+
def resume_checkpoint(operation: IngestionOperation) -> MaterializationResult:
|
|
353
|
+
"""Repair only stages whose writes have an idempotent natural key."""
|
|
354
|
+
state = dict(operation.derivation_state)
|
|
355
|
+
if not state.get("pipeline", False):
|
|
356
|
+
return MaterializationResult(
|
|
357
|
+
operation.final_fact_ids, state, operation.last_error,
|
|
358
|
+
)
|
|
320
359
|
facts = engine._db.get_facts_by_ids(
|
|
321
|
-
list(operation.
|
|
322
|
-
|
|
360
|
+
list(operation.final_fact_ids), operation.profile_id,
|
|
361
|
+
)
|
|
362
|
+
if len(facts) != len(operation.final_fact_ids):
|
|
363
|
+
state["relational"] = False
|
|
364
|
+
return MaterializationResult(
|
|
365
|
+
operation.final_fact_ids,
|
|
366
|
+
state,
|
|
367
|
+
"checkpointed relational facts are missing",
|
|
368
|
+
)
|
|
369
|
+
if state.get("provenance") is False and engine._provenance is not None:
|
|
370
|
+
provenance_complete = True
|
|
371
|
+
for fact in facts:
|
|
372
|
+
existing = engine._db.execute(
|
|
373
|
+
"SELECT 1 FROM provenance WHERE fact_id=? AND profile_id=? "
|
|
374
|
+
"AND source_type=? AND source_id=? AND created_by=? LIMIT 1",
|
|
375
|
+
(
|
|
376
|
+
fact.fact_id,
|
|
377
|
+
operation.profile_id,
|
|
378
|
+
operation.source_type,
|
|
379
|
+
operation.operation_id,
|
|
380
|
+
operation.trusted_actor_id,
|
|
381
|
+
),
|
|
382
|
+
)
|
|
383
|
+
if existing:
|
|
384
|
+
continue
|
|
385
|
+
try:
|
|
386
|
+
engine._provenance.record(
|
|
387
|
+
fact_id=fact.fact_id,
|
|
388
|
+
profile_id=operation.profile_id,
|
|
389
|
+
source_type=operation.source_type,
|
|
390
|
+
source_id=operation.operation_id,
|
|
391
|
+
created_by=operation.trusted_actor_id,
|
|
392
|
+
)
|
|
393
|
+
except Exception:
|
|
394
|
+
provenance_complete = False
|
|
395
|
+
state["provenance"] = provenance_complete
|
|
396
|
+
if state.get("post_hooks") is False:
|
|
397
|
+
try:
|
|
398
|
+
engine._hooks.run_post("store", {
|
|
399
|
+
"operation": "store",
|
|
400
|
+
"agent_id": operation.trusted_actor_id,
|
|
401
|
+
"profile_id": operation.profile_id,
|
|
402
|
+
"content_preview": operation.raw_content[:100],
|
|
403
|
+
"ingestion_operation_id": operation.operation_id,
|
|
404
|
+
"fact_ids": list(operation.final_fact_ids),
|
|
405
|
+
"fact_count": len(operation.final_fact_ids),
|
|
406
|
+
})
|
|
407
|
+
except Exception as exc:
|
|
408
|
+
return MaterializationResult(
|
|
409
|
+
operation.final_fact_ids, state, str(exc),
|
|
410
|
+
)
|
|
411
|
+
state["post_hooks"] = True
|
|
412
|
+
incomplete = [name for name, complete in state.items() if not complete]
|
|
413
|
+
return MaterializationResult(
|
|
414
|
+
operation.final_fact_ids,
|
|
415
|
+
state,
|
|
416
|
+
"" if not incomplete else operation.last_error,
|
|
323
417
|
)
|
|
324
|
-
if len(facts) != len(operation.queryable_fact_ids):
|
|
325
|
-
raise ValueError("queryable fact profile mismatch or missing fact")
|
|
326
|
-
memory_ids = {fact.memory_id for fact in facts}
|
|
327
|
-
if len(memory_ids) != 1:
|
|
328
|
-
raise ValueError("queryable facts do not share one source memory")
|
|
329
|
-
memory_id = next(iter(memory_ids))
|
|
330
418
|
|
|
419
|
+
def resume_partial_relational(
|
|
420
|
+
operation: IngestionOperation,
|
|
421
|
+
memory_id: str,
|
|
422
|
+
) -> MaterializationResult:
|
|
423
|
+
"""Finish idempotent relational effects from facts committed before a crash.
|
|
424
|
+
|
|
425
|
+
Extraction and consolidation mutate evidence and access counters, so a
|
|
426
|
+
relational-start checkpoint is an at-most-once boundary for those
|
|
427
|
+
stages. The facts already committed to this operation's dedicated
|
|
428
|
+
memory can safely be promoted again: they use stable fact IDs, graph
|
|
429
|
+
edge logical keys, temporal event IDs, and fact/entity association
|
|
430
|
+
keys. Deliberately omit the other best-effort enrichers here; they
|
|
431
|
+
have no operation-scoped idempotency contract.
|
|
432
|
+
"""
|
|
331
433
|
from superlocalmemory.core.store_pipeline import run_store
|
|
332
434
|
|
|
333
|
-
|
|
435
|
+
# Consolidation may replace a submitted projection with an existing
|
|
436
|
+
# canonical fact from another memory. The operation checkpoint is the
|
|
437
|
+
# authoritative recovery set in that case; falling back to the source
|
|
438
|
+
# memory is only for a fault before the checkpoint captured final IDs.
|
|
439
|
+
if operation.final_fact_ids:
|
|
440
|
+
facts = engine._db.get_facts_by_ids(
|
|
441
|
+
list(operation.final_fact_ids), operation.profile_id,
|
|
442
|
+
)
|
|
443
|
+
if len(facts) != len(operation.final_fact_ids):
|
|
444
|
+
return MaterializationResult(
|
|
445
|
+
operation.final_fact_ids,
|
|
446
|
+
dict(operation.derivation_state),
|
|
447
|
+
"checkpointed relational facts are missing",
|
|
448
|
+
)
|
|
449
|
+
else:
|
|
450
|
+
facts = engine._db.get_facts_by_memory_id(
|
|
451
|
+
memory_id, operation.profile_id,
|
|
452
|
+
)
|
|
453
|
+
fact_ids = tuple(fact.fact_id for fact in facts)
|
|
454
|
+
if not fact_ids:
|
|
455
|
+
return MaterializationResult(
|
|
456
|
+
(),
|
|
457
|
+
dict(operation.derivation_state),
|
|
458
|
+
"no committed facts available for relational recovery",
|
|
459
|
+
)
|
|
334
460
|
|
|
335
|
-
class
|
|
461
|
+
class _CommittedFactsExtractor:
|
|
336
462
|
@staticmethod
|
|
337
463
|
def extract_facts(**_kwargs):
|
|
338
464
|
return []
|
|
339
465
|
|
|
340
466
|
pipeline_state: dict[str, bool] = {}
|
|
341
|
-
|
|
467
|
+
progress: dict[str, object] = {}
|
|
468
|
+
|
|
469
|
+
def checkpoint_materialization(
|
|
470
|
+
_phase: str,
|
|
471
|
+
checkpoint_fact_ids: tuple[str, ...],
|
|
472
|
+
state: dict[str, bool],
|
|
473
|
+
) -> None:
|
|
474
|
+
repository.checkpoint_enriching(
|
|
475
|
+
operation.operation_id,
|
|
476
|
+
final_fact_ids=checkpoint_fact_ids,
|
|
477
|
+
derivation_version=_DERIVATION_VERSION,
|
|
478
|
+
derivation_state=state,
|
|
479
|
+
lease_owner=operation.lease_owner,
|
|
480
|
+
lease_seconds=900.0,
|
|
481
|
+
)
|
|
482
|
+
|
|
483
|
+
recovered_ids = run_store(
|
|
342
484
|
operation.raw_content,
|
|
343
485
|
operation.profile_id,
|
|
344
486
|
session_id=operation.session_id,
|
|
@@ -351,42 +493,211 @@ def build_engine_ingestion_command(engine: MemoryEngine) -> IngestionCommand:
|
|
|
351
493
|
config=engine._config,
|
|
352
494
|
db=engine._db,
|
|
353
495
|
embedder=engine._embedder,
|
|
354
|
-
fact_extractor=(
|
|
355
|
-
_QueryableProjectionExtractor() if is_prebuilt else engine._fact_extractor
|
|
356
|
-
),
|
|
496
|
+
fact_extractor=_CommittedFactsExtractor(),
|
|
357
497
|
entity_resolver=engine._entity_resolver,
|
|
358
498
|
temporal_parser=engine._temporal_parser,
|
|
359
|
-
type_router=None
|
|
499
|
+
type_router=None,
|
|
360
500
|
graph_builder=engine._graph_builder,
|
|
361
|
-
consolidator=None
|
|
362
|
-
observation_builder=
|
|
363
|
-
scene_builder=
|
|
501
|
+
consolidator=None,
|
|
502
|
+
observation_builder=None,
|
|
503
|
+
scene_builder=None,
|
|
364
504
|
entropy_gate=engine._entropy_gate,
|
|
365
|
-
# External indexes are projected only after the relational unit of
|
|
366
|
-
# work commits; sqlite-vec uses a separate connection.
|
|
367
505
|
ann_index=None,
|
|
368
|
-
sheaf_checker=
|
|
506
|
+
sheaf_checker=None,
|
|
369
507
|
retrieval_engine=None,
|
|
370
508
|
provenance=engine._provenance,
|
|
371
509
|
hooks=engine._hooks,
|
|
372
510
|
vector_store=None,
|
|
373
|
-
context_generator=
|
|
511
|
+
context_generator=None,
|
|
374
512
|
temporal_validator=engine._temporal_validator,
|
|
375
|
-
auto_linker=
|
|
376
|
-
consolidation_engine=
|
|
513
|
+
auto_linker=None,
|
|
514
|
+
consolidation_engine=None,
|
|
377
515
|
existing_memory_id=memory_id,
|
|
378
|
-
queryable_fact_ids=
|
|
516
|
+
queryable_fact_ids=fact_ids,
|
|
379
517
|
trusted_actor_id=operation.trusted_actor_id,
|
|
380
518
|
pre_authorized=True,
|
|
381
519
|
ingestion_source_type=operation.source_type,
|
|
382
520
|
ingestion_operation_id=operation.operation_id,
|
|
383
521
|
derivation_report=pipeline_state,
|
|
522
|
+
precompleted_derivation_stages=frozenset({
|
|
523
|
+
"extraction", "consolidation",
|
|
524
|
+
}),
|
|
525
|
+
materialization_progress=progress,
|
|
526
|
+
materialization_checkpoint=checkpoint_materialization,
|
|
384
527
|
)
|
|
528
|
+
state = {
|
|
529
|
+
"pipeline_started": True,
|
|
530
|
+
"relational_started": True,
|
|
531
|
+
"pipeline": True,
|
|
532
|
+
"post_hooks": True,
|
|
533
|
+
"relational": True,
|
|
534
|
+
**pipeline_state,
|
|
535
|
+
}
|
|
536
|
+
return MaterializationResult(tuple(recovered_ids), state)
|
|
537
|
+
|
|
538
|
+
def materialize(operation: IngestionOperation) -> MaterializationResult:
|
|
539
|
+
# A checkpointed relational result is an at-most-once boundary. The
|
|
540
|
+
# extraction/consolidation pipeline mutates evidence/access counters,
|
|
541
|
+
# so replaying it is not a valid repair strategy. Idempotent external
|
|
542
|
+
# projections are resumed by IngestionCommand after every relational
|
|
543
|
+
# stage is complete; otherwise the durable failure remains inspectable.
|
|
544
|
+
if operation.final_fact_ids and operation.derivation_state.get("pipeline", False):
|
|
545
|
+
return resume_checkpoint(operation)
|
|
546
|
+
|
|
547
|
+
facts = engine._db.get_facts_by_ids(
|
|
548
|
+
list(operation.queryable_fact_ids),
|
|
549
|
+
operation.profile_id,
|
|
550
|
+
)
|
|
551
|
+
if len(facts) != len(operation.queryable_fact_ids):
|
|
552
|
+
raise ValueError("queryable fact profile mismatch or missing fact")
|
|
553
|
+
memory_ids = {fact.memory_id for fact in facts}
|
|
554
|
+
if len(memory_ids) != 1:
|
|
555
|
+
raise ValueError("queryable facts do not share one source memory")
|
|
556
|
+
memory_id = next(iter(memory_ids))
|
|
557
|
+
|
|
558
|
+
from superlocalmemory.core.store_pipeline import run_store
|
|
559
|
+
|
|
560
|
+
is_prebuilt = isinstance(operation.metadata.get(_PREBUILT_FACT_KEY), dict)
|
|
561
|
+
if (
|
|
562
|
+
operation.derivation_state.get("pipeline_started", False)
|
|
563
|
+
and not operation.derivation_state.get("pipeline", False)
|
|
564
|
+
and operation.derivation_state.get("relational_started", False)
|
|
565
|
+
):
|
|
566
|
+
return resume_partial_relational(operation, memory_id)
|
|
567
|
+
repository.checkpoint_enriching(
|
|
568
|
+
operation.operation_id,
|
|
569
|
+
final_fact_ids=(),
|
|
570
|
+
derivation_version=_DERIVATION_VERSION,
|
|
571
|
+
derivation_state={
|
|
572
|
+
"pipeline_started": True,
|
|
573
|
+
"pipeline": False,
|
|
574
|
+
},
|
|
575
|
+
lease_owner=operation.lease_owner,
|
|
576
|
+
lease_seconds=900.0,
|
|
577
|
+
)
|
|
578
|
+
|
|
579
|
+
class _QueryableProjectionExtractor:
|
|
580
|
+
@staticmethod
|
|
581
|
+
def extract_facts(**_kwargs):
|
|
582
|
+
return []
|
|
583
|
+
|
|
584
|
+
pipeline_state: dict[str, bool] = {}
|
|
585
|
+
progress: dict[str, object] = {}
|
|
586
|
+
|
|
587
|
+
def checkpoint_materialization(
|
|
588
|
+
_phase: str,
|
|
589
|
+
fact_ids: tuple[str, ...],
|
|
590
|
+
state: dict[str, bool],
|
|
591
|
+
) -> None:
|
|
592
|
+
repository.checkpoint_enriching(
|
|
593
|
+
operation.operation_id,
|
|
594
|
+
final_fact_ids=fact_ids,
|
|
595
|
+
derivation_version=_DERIVATION_VERSION,
|
|
596
|
+
derivation_state=state,
|
|
597
|
+
lease_owner=operation.lease_owner,
|
|
598
|
+
lease_seconds=900.0,
|
|
599
|
+
)
|
|
600
|
+
|
|
601
|
+
try:
|
|
602
|
+
fact_ids = run_store(
|
|
603
|
+
operation.raw_content,
|
|
604
|
+
operation.profile_id,
|
|
605
|
+
session_id=operation.session_id,
|
|
606
|
+
session_date=operation.session_date or None,
|
|
607
|
+
speaker=operation.speaker,
|
|
608
|
+
role=operation.role,
|
|
609
|
+
metadata=dict(operation.metadata),
|
|
610
|
+
scope=operation.scope,
|
|
611
|
+
shared_with=list(operation.shared_with) or None,
|
|
612
|
+
config=engine._config,
|
|
613
|
+
db=engine._db,
|
|
614
|
+
embedder=engine._embedder,
|
|
615
|
+
fact_extractor=(
|
|
616
|
+
_QueryableProjectionExtractor()
|
|
617
|
+
if is_prebuilt else engine._fact_extractor
|
|
618
|
+
),
|
|
619
|
+
entity_resolver=engine._entity_resolver,
|
|
620
|
+
temporal_parser=engine._temporal_parser,
|
|
621
|
+
type_router=None if is_prebuilt else engine._type_router,
|
|
622
|
+
graph_builder=engine._graph_builder,
|
|
623
|
+
consolidator=None if is_prebuilt else engine._consolidator,
|
|
624
|
+
observation_builder=engine._observation_builder,
|
|
625
|
+
scene_builder=engine._scene_builder,
|
|
626
|
+
entropy_gate=engine._entropy_gate,
|
|
627
|
+
# External indexes are projected only after the relational unit
|
|
628
|
+
# of work commits; sqlite-vec uses a separate connection.
|
|
629
|
+
ann_index=None,
|
|
630
|
+
sheaf_checker=engine._sheaf_checker,
|
|
631
|
+
retrieval_engine=None,
|
|
632
|
+
provenance=engine._provenance,
|
|
633
|
+
hooks=engine._hooks,
|
|
634
|
+
vector_store=None,
|
|
635
|
+
context_generator=engine._context_generator,
|
|
636
|
+
temporal_validator=engine._temporal_validator,
|
|
637
|
+
auto_linker=engine._auto_linker,
|
|
638
|
+
consolidation_engine=engine._consolidation_engine,
|
|
639
|
+
existing_memory_id=memory_id,
|
|
640
|
+
queryable_fact_ids=operation.queryable_fact_ids,
|
|
641
|
+
trusted_actor_id=operation.trusted_actor_id,
|
|
642
|
+
pre_authorized=True,
|
|
643
|
+
ingestion_source_type=operation.source_type,
|
|
644
|
+
ingestion_operation_id=operation.operation_id,
|
|
645
|
+
derivation_report=pipeline_state,
|
|
646
|
+
precompleted_derivation_stages=(
|
|
647
|
+
frozenset({"extraction", "consolidation"})
|
|
648
|
+
if is_prebuilt else frozenset()
|
|
649
|
+
),
|
|
650
|
+
materialization_progress=progress,
|
|
651
|
+
materialization_checkpoint=checkpoint_materialization,
|
|
652
|
+
)
|
|
653
|
+
except Exception as exc:
|
|
654
|
+
# ``run_store`` checkpoints the completed relational pipeline
|
|
655
|
+
# immediately before post-hooks run. Prefer that durable ledger
|
|
656
|
+
# over rebuilding state from local variables: a one-time hook
|
|
657
|
+
# failure must not make committed extraction/consolidation look
|
|
658
|
+
# incomplete and trigger a destructive pipeline replay on retry.
|
|
659
|
+
checkpoint = repository.get(operation.operation_id)
|
|
660
|
+
partial_ids = tuple(checkpoint.final_fact_ids)
|
|
661
|
+
failed_state = dict(checkpoint.derivation_state)
|
|
662
|
+
if not partial_ids:
|
|
663
|
+
partial_ids = tuple(progress.get("fact_ids") or ())
|
|
664
|
+
if not partial_ids:
|
|
665
|
+
partial_ids = tuple(
|
|
666
|
+
fact.fact_id
|
|
667
|
+
for fact in engine._db.get_facts_by_memory_id(
|
|
668
|
+
memory_id, operation.profile_id
|
|
669
|
+
)
|
|
670
|
+
)
|
|
671
|
+
if not failed_state:
|
|
672
|
+
relational_complete = bool(
|
|
673
|
+
progress.get("relational_complete", False)
|
|
674
|
+
)
|
|
675
|
+
failed_state = {
|
|
676
|
+
**{
|
|
677
|
+
name: bool(pipeline_state.get(name, False))
|
|
678
|
+
for name in (
|
|
679
|
+
"extraction",
|
|
680
|
+
"canonicalization",
|
|
681
|
+
"consolidation",
|
|
682
|
+
"graph",
|
|
683
|
+
"temporal",
|
|
684
|
+
"provenance",
|
|
685
|
+
)
|
|
686
|
+
},
|
|
687
|
+
"pipeline_started": True,
|
|
688
|
+
"relational_started": bool(
|
|
689
|
+
progress.get("relational_started", False)
|
|
690
|
+
),
|
|
691
|
+
"pipeline": relational_complete,
|
|
692
|
+
"post_hooks": False,
|
|
693
|
+
}
|
|
694
|
+
# Reaching this exception handler means hooks did not complete.
|
|
695
|
+
# Keep this false even if a future hook implementation mutates the
|
|
696
|
+
# in-memory progress map before raising.
|
|
697
|
+
failed_state["post_hooks"] = False
|
|
698
|
+
return MaterializationResult(partial_ids, failed_state, str(exc))
|
|
385
699
|
if not fact_ids:
|
|
386
700
|
return MaterializationResult((), {"relational": False})
|
|
387
|
-
if is_prebuilt:
|
|
388
|
-
pipeline_state["extraction"] = True
|
|
389
|
-
pipeline_state["consolidation"] = True
|
|
390
701
|
|
|
391
702
|
placeholders = ",".join("?" for _ in fact_ids)
|
|
392
703
|
relational_count = engine._db.execute(
|
|
@@ -433,6 +744,10 @@ def build_engine_ingestion_command(engine: MemoryEngine) -> IngestionCommand:
|
|
|
433
744
|
)
|
|
434
745
|
|
|
435
746
|
derivation_state = {
|
|
747
|
+
"pipeline_started": True,
|
|
748
|
+
"relational_started": True,
|
|
749
|
+
"pipeline": True,
|
|
750
|
+
"post_hooks": True,
|
|
436
751
|
"relational": int(dict(relational_count[0])["count"]) == len(fact_ids),
|
|
437
752
|
"fts": int(dict(fts_count[0])["count"]) == len(fact_ids),
|
|
438
753
|
"extraction": pipeline_state.get("extraction", False),
|
|
@@ -502,6 +817,7 @@ def build_engine_ingestion_command(engine: MemoryEngine) -> IngestionCommand:
|
|
|
502
817
|
write_queryable=write_queryable,
|
|
503
818
|
materialize=materialize,
|
|
504
819
|
project=project,
|
|
820
|
+
derivation_version=_DERIVATION_VERSION,
|
|
505
821
|
)
|
|
506
822
|
|
|
507
823
|
|
|
@@ -588,6 +588,19 @@ def init_retrieval(
|
|
|
588
588
|
except Exception as exc:
|
|
589
589
|
logger.debug("Forgetting filter registration failed: %s", exc)
|
|
590
590
|
|
|
591
|
+
# Phase 4 (T1): Register bi-temporal validity filter. Drops superseded /
|
|
592
|
+
# system-invalidated facts from retrieval so contradicted memories never
|
|
593
|
+
# resurface. Pure SQL, safe in all modes; no-op until a fact is invalidated.
|
|
594
|
+
try:
|
|
595
|
+
from superlocalmemory.retrieval.temporal_validity_filter import (
|
|
596
|
+
register_temporal_validity_filter,
|
|
597
|
+
)
|
|
598
|
+
register_temporal_validity_filter(
|
|
599
|
+
engine._registry, db, config.temporal_validator,
|
|
600
|
+
)
|
|
601
|
+
except Exception as exc:
|
|
602
|
+
logger.debug("Temporal validity filter registration failed: %s", exc)
|
|
603
|
+
|
|
591
604
|
return engine
|
|
592
605
|
|
|
593
606
|
|
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
|
|
2
|
+
# Licensed under AGPL-3.0-or-later - see LICENSE file
|
|
3
|
+
# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
|
|
4
|
+
|
|
5
|
+
"""Entity-community backbone (Wave Q) — the single principled clustering spine.
|
|
6
|
+
|
|
7
|
+
Best-in-market memory graphs (GraphRAG, Graphiti/Zep) cluster the ENTITY
|
|
8
|
+
graph, not the raw fact/chunk graph. SLM's fact graph is capped-sparse
|
|
9
|
+
(5 edges/entity), so a fact-level clustering fragments. This module builds an
|
|
10
|
+
entity co-occurrence graph (two entities are linked when they appear together
|
|
11
|
+
in a fact, weighted by co-occurrence count) and runs Louvain community
|
|
12
|
+
detection over it — the correct target and, since entities are far fewer than
|
|
13
|
+
facts, a cheaper computation.
|
|
14
|
+
|
|
15
|
+
The resulting entity communities are the shared backbone for:
|
|
16
|
+
- Q2 community summaries (one synthesized report per community), and
|
|
17
|
+
- Q3 progressive abstraction (scenario/persona tiers + drill-down).
|
|
18
|
+
|
|
19
|
+
Runs in the background (consolidation lane), never on the hot recall path.
|
|
20
|
+
Fail-open and idempotent: a recompute fully replaces a profile's rows.
|
|
21
|
+
|
|
22
|
+
Part of Qualixar | Author: Varun Pratap Bhardwaj
|
|
23
|
+
License: AGPL-3.0-or-later
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
from __future__ import annotations
|
|
27
|
+
|
|
28
|
+
import json
|
|
29
|
+
import logging
|
|
30
|
+
from collections import Counter, defaultdict
|
|
31
|
+
from typing import Any
|
|
32
|
+
|
|
33
|
+
logger = logging.getLogger(__name__)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class EntityCommunityBuilder:
|
|
37
|
+
"""Build + persist entity communities via Louvain over co-occurrence."""
|
|
38
|
+
|
|
39
|
+
def __init__(
|
|
40
|
+
self,
|
|
41
|
+
db: Any,
|
|
42
|
+
min_community_size: int = 2,
|
|
43
|
+
resolution: float = 1.0,
|
|
44
|
+
seed: int = 42,
|
|
45
|
+
) -> None:
|
|
46
|
+
self._db = db
|
|
47
|
+
self._min_size = max(2, int(min_community_size))
|
|
48
|
+
self._resolution = float(resolution)
|
|
49
|
+
self._seed = int(seed)
|
|
50
|
+
|
|
51
|
+
# ------------------------------------------------------------------
|
|
52
|
+
# Detection
|
|
53
|
+
# ------------------------------------------------------------------
|
|
54
|
+
|
|
55
|
+
def _cooccurrence(self, profile_id: str) -> Counter:
|
|
56
|
+
"""Count entity pairs that co-occur within a fact (per profile)."""
|
|
57
|
+
rows = self._db.execute(
|
|
58
|
+
"SELECT canonical_entities_json FROM atomic_facts "
|
|
59
|
+
"WHERE profile_id = ?",
|
|
60
|
+
(profile_id,),
|
|
61
|
+
)
|
|
62
|
+
weights: Counter = Counter()
|
|
63
|
+
for row in rows:
|
|
64
|
+
raw = dict(row).get("canonical_entities_json")
|
|
65
|
+
if not raw:
|
|
66
|
+
continue
|
|
67
|
+
try:
|
|
68
|
+
parsed = json.loads(raw)
|
|
69
|
+
except (ValueError, TypeError):
|
|
70
|
+
continue
|
|
71
|
+
if not isinstance(parsed, list):
|
|
72
|
+
continue
|
|
73
|
+
ents = sorted({str(e).strip() for e in parsed if str(e).strip()})
|
|
74
|
+
for i in range(len(ents)):
|
|
75
|
+
for j in range(i + 1, len(ents)):
|
|
76
|
+
weights[(ents[i], ents[j])] += 1
|
|
77
|
+
return weights
|
|
78
|
+
|
|
79
|
+
def detect(self, profile_id: str) -> dict[str, int]:
|
|
80
|
+
"""Return {entity_id -> community_id}; communities below min size drop."""
|
|
81
|
+
weights = self._cooccurrence(profile_id)
|
|
82
|
+
if not weights:
|
|
83
|
+
return {}
|
|
84
|
+
|
|
85
|
+
import networkx as nx
|
|
86
|
+
|
|
87
|
+
g = nx.Graph()
|
|
88
|
+
for (a, b), w in weights.items():
|
|
89
|
+
g.add_edge(a, b, weight=w)
|
|
90
|
+
|
|
91
|
+
try:
|
|
92
|
+
from networkx.algorithms.community import louvain_communities
|
|
93
|
+
|
|
94
|
+
communities = louvain_communities(
|
|
95
|
+
g, weight="weight", resolution=self._resolution, seed=self._seed,
|
|
96
|
+
)
|
|
97
|
+
except Exception as exc: # pragma: no cover - fallback path
|
|
98
|
+
logger.debug(
|
|
99
|
+
"Louvain unavailable/failed (%s); using connected components",
|
|
100
|
+
exc,
|
|
101
|
+
)
|
|
102
|
+
communities = nx.connected_components(g)
|
|
103
|
+
|
|
104
|
+
result: dict[str, int] = {}
|
|
105
|
+
cid = 0
|
|
106
|
+
for comm in communities:
|
|
107
|
+
members = list(comm)
|
|
108
|
+
if len(members) < self._min_size:
|
|
109
|
+
continue
|
|
110
|
+
for node in members:
|
|
111
|
+
result[node] = cid
|
|
112
|
+
cid += 1
|
|
113
|
+
return result
|
|
114
|
+
|
|
115
|
+
# ------------------------------------------------------------------
|
|
116
|
+
# Persistence
|
|
117
|
+
# ------------------------------------------------------------------
|
|
118
|
+
|
|
119
|
+
def compute_and_store(self, profile_id: str) -> dict[str, int]:
|
|
120
|
+
"""Detect communities and replace this profile's stored rows."""
|
|
121
|
+
mapping = self.detect(profile_id)
|
|
122
|
+
try:
|
|
123
|
+
self._db.execute(
|
|
124
|
+
"DELETE FROM entity_communities WHERE profile_id = ?",
|
|
125
|
+
(profile_id,),
|
|
126
|
+
)
|
|
127
|
+
for entity_id, community_id in mapping.items():
|
|
128
|
+
self._db.execute(
|
|
129
|
+
"INSERT OR REPLACE INTO entity_communities "
|
|
130
|
+
"(profile_id, entity_id, community_id, computed_at) "
|
|
131
|
+
"VALUES (?, ?, ?, datetime('now'))",
|
|
132
|
+
(profile_id, entity_id, community_id),
|
|
133
|
+
)
|
|
134
|
+
except Exception as exc:
|
|
135
|
+
logger.debug("entity_communities persist failed: %s", exc)
|
|
136
|
+
|
|
137
|
+
return {
|
|
138
|
+
"entity_count": len(mapping),
|
|
139
|
+
"community_count": len(set(mapping.values())),
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
# ------------------------------------------------------------------
|
|
143
|
+
# Read API
|
|
144
|
+
# ------------------------------------------------------------------
|
|
145
|
+
|
|
146
|
+
def get_communities(self, profile_id: str) -> dict[int, list[str]]:
|
|
147
|
+
"""Return {community_id -> [entity_id, ...]} for a profile."""
|
|
148
|
+
try:
|
|
149
|
+
rows = self._db.execute(
|
|
150
|
+
"SELECT entity_id, community_id FROM entity_communities "
|
|
151
|
+
"WHERE profile_id = ? ORDER BY community_id",
|
|
152
|
+
(profile_id,),
|
|
153
|
+
)
|
|
154
|
+
except Exception as exc:
|
|
155
|
+
logger.debug("get_communities failed: %s", exc)
|
|
156
|
+
return {}
|
|
157
|
+
out: dict[int, list[str]] = defaultdict(list)
|
|
158
|
+
for row in rows:
|
|
159
|
+
d = dict(row)
|
|
160
|
+
out[int(d["community_id"])].append(str(d["entity_id"]))
|
|
161
|
+
return dict(out)
|
|
162
|
+
|
|
163
|
+
def get_community_for_entity(
|
|
164
|
+
self, entity_id: str, profile_id: str,
|
|
165
|
+
) -> int | None:
|
|
166
|
+
"""Return the community id for one entity, or None."""
|
|
167
|
+
try:
|
|
168
|
+
rows = self._db.execute(
|
|
169
|
+
"SELECT community_id FROM entity_communities "
|
|
170
|
+
"WHERE profile_id = ? AND entity_id = ?",
|
|
171
|
+
(profile_id, entity_id),
|
|
172
|
+
)
|
|
173
|
+
except Exception as exc:
|
|
174
|
+
logger.debug("get_community_for_entity failed: %s", exc)
|
|
175
|
+
return None
|
|
176
|
+
for row in rows:
|
|
177
|
+
return int(dict(row)["community_id"])
|
|
178
|
+
return None
|