superlocalmemory 3.7.8 → 3.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/ATTRIBUTION.md +1 -3
- package/CHANGELOG.md +69 -0
- package/README.md +199 -29
- package/package.json +4 -2
- package/plugin/.claude-plugin/plugin.json +2 -2
- package/plugin/CLAUDE.md +8 -8
- package/plugin/agents/slm-governance-advisor.md +80 -0
- package/plugin/agents/slm-loop-runner.md +71 -0
- package/plugin/agents/slm-memory-advisor.md +10 -5
- package/plugin/agents/slm-optimize-advisor.md +9 -3
- package/plugin/commands/slm-loop.md +31 -0
- package/plugin/hooks/hooks.json +79 -0
- package/plugin/requirements.txt +1 -1
- package/plugin/scripts/slm-launch +46 -7
- package/plugin/settings.json +9 -0
- package/plugin/skills/slm-cache/SKILL.md +9 -1
- package/plugin/skills/slm-compress/SKILL.md +8 -1
- package/plugin/skills/slm-governance/SKILL.md +248 -0
- package/plugin/skills/slm-graph/SKILL.md +17 -3
- package/plugin/skills/slm-loop/SKILL.md +99 -0
- package/plugin/skills/slm-mesh/SKILL.md +282 -0
- package/plugin/skills/slm-profile/SKILL.md +148 -0
- package/plugin/skills/slm-recall/SKILL.md +46 -10
- package/plugin/skills/slm-remember/SKILL.md +48 -1
- package/plugin/skills/slm-scope/SKILL.md +176 -0
- package/plugin/skills/slm-session/SKILL.md +24 -1
- package/plugin/skills/slm-status/SKILL.md +18 -1
- package/plugin-src/agents/slm-governance-advisor.md +80 -0
- package/plugin-src/agents/slm-loop-runner.md +71 -0
- package/plugin-src/agents/slm-memory-advisor.md +10 -5
- package/plugin-src/agents/slm-optimize-advisor.md +9 -3
- package/plugin-src/commands/slm-loop.md +31 -0
- package/plugin-src/hooks/hooks.json +79 -0
- package/plugin-src/manifest.json +7 -2
- package/plugin-src/requirements.txt +1 -1
- package/plugin-src/rules/AGENTS.md +57 -18
- package/plugin-src/rules/CLAUDE.md.fragment +8 -8
- package/plugin-src/scripts/slm-launch +46 -7
- package/plugin-src/settings.json +9 -0
- package/plugin-src/skills/slm-cache/SKILL.md +9 -1
- package/plugin-src/skills/slm-compress/SKILL.md +8 -1
- package/plugin-src/skills/slm-governance/SKILL.md +248 -0
- package/plugin-src/skills/slm-graph/SKILL.md +17 -3
- package/plugin-src/skills/slm-loop/SKILL.md +99 -0
- package/plugin-src/skills/slm-mesh/SKILL.md +282 -0
- package/plugin-src/skills/slm-profile/SKILL.md +148 -0
- package/plugin-src/skills/slm-recall/SKILL.md +46 -10
- package/plugin-src/skills/slm-remember/SKILL.md +48 -1
- package/plugin-src/skills/slm-scope/SKILL.md +176 -0
- package/plugin-src/skills/slm-session/SKILL.md +24 -1
- package/plugin-src/skills/slm-status/SKILL.md +18 -1
- package/pyproject.toml +1 -1
- package/scripts/postinstall/validation.js +2 -0
- package/scripts/postinstall-interactive.js +74 -2
- package/src/superlocalmemory/__init__.py +1 -1
- package/src/superlocalmemory/access/__init__.py +3 -0
- package/src/superlocalmemory/access/rbac.py +477 -0
- package/src/superlocalmemory/cli/commands.py +94 -10
- package/src/superlocalmemory/cli/compress_cmd.py +17 -7
- package/src/superlocalmemory/cli/loop_cmd.py +192 -0
- package/src/superlocalmemory/cli/main.py +39 -4
- package/src/superlocalmemory/cli/mesh_cmd.py +38 -0
- package/src/superlocalmemory/cli/optimize_cmd.py +3 -0
- package/src/superlocalmemory/cli/pending_store.py +49 -13
- package/src/superlocalmemory/cli/proxy_cmd.py +4 -0
- package/src/superlocalmemory/cli/scale_engine_cmd.py +6 -0
- package/src/superlocalmemory/cli/setup_wizard.py +22 -13
- package/src/superlocalmemory/compliance/audit.py +6 -0
- package/src/superlocalmemory/compliance/gdpr.py +128 -138
- package/src/superlocalmemory/compliance/retention.py +176 -45
- package/src/superlocalmemory/core/backend_orchestrator.py +5 -43
- package/src/superlocalmemory/core/community_summary.py +267 -0
- package/src/superlocalmemory/core/config.py +216 -3
- package/src/superlocalmemory/core/consolidation_engine.py +95 -22
- package/src/superlocalmemory/core/context_cache.py +61 -18
- package/src/superlocalmemory/core/embedding_worker.py +17 -2
- package/src/superlocalmemory/core/embeddings.py +12 -1
- package/src/superlocalmemory/core/engine.py +17 -1
- package/src/superlocalmemory/core/engine_ingestion.py +29 -0
- package/src/superlocalmemory/core/engine_wiring.py +13 -0
- package/src/superlocalmemory/core/entity_community.py +178 -0
- package/src/superlocalmemory/core/graph_analyzer.py +39 -2
- package/src/superlocalmemory/core/graph_pruner.py +13 -8
- package/src/superlocalmemory/core/key_expander.py +138 -0
- package/src/superlocalmemory/core/maintenance.py +23 -0
- package/src/superlocalmemory/core/modes.py +1 -1
- package/src/superlocalmemory/core/mutations.py +2 -2
- package/src/superlocalmemory/core/pii.py +105 -0
- package/src/superlocalmemory/core/progressive_abstraction.py +208 -0
- package/src/superlocalmemory/core/recall_pipeline.py +2 -0
- package/src/superlocalmemory/core/recall_worker.py +20 -6
- package/src/superlocalmemory/core/scale_engine.py +60 -1
- package/src/superlocalmemory/core/security_primitives.py +40 -2
- package/src/superlocalmemory/core/store_pipeline.py +35 -11
- package/src/superlocalmemory/core/worker_pool.py +21 -6
- package/src/superlocalmemory/encoding/entity_reflexion.py +200 -0
- package/src/superlocalmemory/encoding/entity_resolver.py +34 -24
- package/src/superlocalmemory/encoding/fact_extractor.py +26 -1
- package/src/superlocalmemory/encoding/temporal_validator.py +64 -1
- package/src/superlocalmemory/evolution/evolution_store.py +122 -45
- package/src/superlocalmemory/evolution/llm_dispatch.py +12 -1
- package/src/superlocalmemory/evolution/model_selection.py +160 -0
- package/src/superlocalmemory/evolution/mutation_generator.py +16 -0
- package/src/superlocalmemory/evolution/skill_evolver.py +127 -42
- package/src/superlocalmemory/evolution/triggers.py +22 -13
- package/src/superlocalmemory/graph/cozo_backend.py +43 -20
- package/src/superlocalmemory/hooks/adapter_base.py +5 -1
- package/src/superlocalmemory/hooks/auto_recall.py +13 -1
- package/src/superlocalmemory/hooks/claude_code_hooks.py +11 -0
- package/src/superlocalmemory/hooks/codex_assets.py +64 -5
- package/src/superlocalmemory/hooks/hook_daemon.py +20 -3
- package/src/superlocalmemory/hooks/memory_protocol.py +54 -0
- package/src/superlocalmemory/hooks/portable_kit.py +114 -1
- package/src/superlocalmemory/infra/backup.py +12 -1
- package/src/superlocalmemory/infra/daemon_identity.py +40 -4
- package/src/superlocalmemory/infra/data_root.py +43 -4
- package/src/superlocalmemory/infra/event_bus.py +107 -24
- package/src/superlocalmemory/infra/rate_limiter.py +93 -0
- package/src/superlocalmemory/ingestion/adapter_manager.py +4 -1
- package/src/superlocalmemory/ingestion/credentials.py +1 -1
- package/src/superlocalmemory/learning/cross_project.py +28 -19
- package/src/superlocalmemory/learning/reward_proxy.py +42 -9
- package/src/superlocalmemory/loops/__init__.py +56 -0
- package/src/superlocalmemory/loops/budget.py +58 -0
- package/src/superlocalmemory/loops/engine.py +164 -0
- package/src/superlocalmemory/loops/ledger.py +243 -0
- package/src/superlocalmemory/loops/models.py +152 -0
- package/src/superlocalmemory/loops/rules.py +52 -0
- package/src/superlocalmemory/mcp/_daemon_proxy.py +3 -0
- package/src/superlocalmemory/mcp/_pool_adapter.py +2 -0
- package/src/superlocalmemory/mcp/profiles.py +103 -0
- package/src/superlocalmemory/mcp/server.py +21 -49
- package/src/superlocalmemory/mcp/tools_active.py +4 -7
- package/src/superlocalmemory/mcp/tools_code_graph.py +51 -5
- package/src/superlocalmemory/mcp/tools_core.py +8 -1
- package/src/superlocalmemory/mcp/tools_evolution.py +6 -3
- package/src/superlocalmemory/mcp/tools_loops.py +300 -0
- package/src/superlocalmemory/mcp/tools_mesh.py +140 -4
- package/src/superlocalmemory/mcp/tools_optimize.py +15 -8
- package/src/superlocalmemory/mesh/broker.py +237 -129
- package/src/superlocalmemory/mesh/remote_sync.py +50 -8
- package/src/superlocalmemory/optimize/NOTICE +1 -6
- package/src/superlocalmemory/optimize/adapters/anthropic_adapter.py +1 -4
- package/src/superlocalmemory/optimize/adapters/openai_adapter.py +1 -4
- package/src/superlocalmemory/optimize/cache/semantic.py +27 -19
- package/src/superlocalmemory/optimize/compress/align.py +32 -26
- package/src/superlocalmemory/optimize/compress/ccr.py +14 -71
- package/src/superlocalmemory/optimize/compress/router.py +105 -22
- package/src/superlocalmemory/optimize/config/defaults.py +1 -1
- package/src/superlocalmemory/optimize/config/schema.py +87 -4
- package/src/superlocalmemory/optimize/metrics/counters.py +13 -4
- package/src/superlocalmemory/optimize/metrics/estimator.py +0 -3
- package/src/superlocalmemory/optimize/proxy/_helpers.py +31 -4
- package/src/superlocalmemory/optimize/storage/db.py +38 -9
- package/src/superlocalmemory/optimize/storage/schema.py +10 -0
- package/src/superlocalmemory/parameterization/pattern_extractor.py +6 -3
- package/src/superlocalmemory/retrieval/agentic.py +1 -1
- package/src/superlocalmemory/retrieval/bm25_channel.py +68 -10
- package/src/superlocalmemory/retrieval/engine.py +168 -26
- package/src/superlocalmemory/retrieval/entity_channel.py +7 -5
- package/src/superlocalmemory/retrieval/hopfield_channel.py +9 -2
- package/src/superlocalmemory/retrieval/semantic_channel.py +114 -21
- package/src/superlocalmemory/retrieval/spreading_activation.py +11 -2
- package/src/superlocalmemory/retrieval/temporal_channel.py +48 -9
- package/src/superlocalmemory/retrieval/temporal_frame.py +102 -0
- package/src/superlocalmemory/retrieval/temporal_validity_filter.py +135 -0
- package/src/superlocalmemory/retrieval/time_window.py +181 -0
- package/src/superlocalmemory/server/api.py +4 -4
- package/src/superlocalmemory/server/profile_runtime.py +125 -8
- package/src/superlocalmemory/server/rbac_enforce.py +142 -0
- package/src/superlocalmemory/server/recall_health.py +24 -3
- package/src/superlocalmemory/server/recall_serializer.py +19 -1
- package/src/superlocalmemory/server/routes/abstraction.py +115 -0
- package/src/superlocalmemory/server/routes/agents.py +128 -38
- package/src/superlocalmemory/server/routes/backup.py +34 -10
- package/src/superlocalmemory/server/routes/behavioral.py +13 -12
- package/src/superlocalmemory/server/routes/brain.py +21 -5
- package/src/superlocalmemory/server/routes/chat.py +10 -5
- package/src/superlocalmemory/server/routes/compliance.py +171 -21
- package/src/superlocalmemory/server/routes/config_api.py +436 -0
- package/src/superlocalmemory/server/routes/data_io.py +30 -8
- package/src/superlocalmemory/server/routes/entity.py +9 -4
- package/src/superlocalmemory/server/routes/events.py +24 -8
- package/src/superlocalmemory/server/routes/evolution.py +135 -17
- package/src/superlocalmemory/server/routes/helpers.py +16 -1
- package/src/superlocalmemory/server/routes/ingest.py +7 -4
- package/src/superlocalmemory/server/routes/insights.py +3 -3
- package/src/superlocalmemory/server/routes/learning.py +14 -14
- package/src/superlocalmemory/server/routes/lifecycle.py +59 -8
- package/src/superlocalmemory/server/routes/memories.py +182 -57
- package/src/superlocalmemory/server/routes/mesh.py +95 -15
- package/src/superlocalmemory/server/routes/optimize.py +33 -1
- package/src/superlocalmemory/server/routes/prewarm.py +2 -0
- package/src/superlocalmemory/server/routes/profiles.py +63 -17
- package/src/superlocalmemory/server/routes/ratelimit.py +124 -0
- package/src/superlocalmemory/server/routes/rbac.py +367 -0
- package/src/superlocalmemory/server/routes/stats.py +13 -6
- package/src/superlocalmemory/server/routes/tiers.py +11 -9
- package/src/superlocalmemory/server/routes/v3_api.py +183 -69
- package/src/superlocalmemory/server/routes/ws.py +5 -2
- package/src/superlocalmemory/server/security_middleware.py +12 -5
- package/src/superlocalmemory/server/ui.py +20 -5
- package/src/superlocalmemory/server/unified_daemon.py +384 -56
- package/src/superlocalmemory/server/write_identity.py +38 -8
- package/src/superlocalmemory/storage/database.py +265 -53
- package/src/superlocalmemory/storage/migration_runner.py +53 -0
- package/src/superlocalmemory/storage/migrations/M021_ingestion_log_profile.py +108 -0
- package/src/superlocalmemory/storage/migrations/M022_entity_aliases_profile.py +86 -0
- package/src/superlocalmemory/storage/migrations/M023_mesh_profile_isolation.py +194 -0
- package/src/superlocalmemory/storage/migrations/M024_rbac_users_roles.py +87 -0
- package/src/superlocalmemory/storage/migrations/M025_perf_indexes.py +90 -0
- package/src/superlocalmemory/storage/migrations/M026_rbac_memberships_fk.py +136 -0
- package/src/superlocalmemory/storage/migrations/M027_transferable_patterns_profile.py +163 -0
- package/src/superlocalmemory/storage/models.py +4 -0
- package/src/superlocalmemory/storage/schema.py +87 -0
- package/src/superlocalmemory/storage/schema_v343.py +24 -12
- package/src/superlocalmemory/trust/gate.py +49 -8
- package/src/superlocalmemory/ui/assets/slm-icon-white.svg +64 -0
- package/src/superlocalmemory/ui/assets/slm-icon.svg +36 -0
- package/src/superlocalmemory/ui/css/design-system.css +621 -0
- package/src/superlocalmemory/ui/css/neural-glass.css +6 -0
- package/src/superlocalmemory/ui/css/od-bridge.css +158 -0
- package/src/superlocalmemory/ui/favicon.svg +35 -4
- package/src/superlocalmemory/ui/index.html +306 -173
- package/src/superlocalmemory/ui/js/brain.js +5 -20
- package/src/superlocalmemory/ui/js/core.js +47 -31
- package/src/superlocalmemory/ui/js/dashboard.js +314 -63
- package/src/superlocalmemory/ui/js/event-delegation.js +102 -0
- package/src/superlocalmemory/ui/js/knowledge-graph.js +11 -11
- package/src/superlocalmemory/ui/js/math-health.js +1 -1
- package/src/superlocalmemory/ui/js/memories.js +15 -4
- package/src/superlocalmemory/ui/js/memory-chat.js +7 -7
- package/src/superlocalmemory/ui/js/ng-entities.js +6 -8
- package/src/superlocalmemory/ui/js/ng-ingestion.js +4 -4
- package/src/superlocalmemory/ui/js/ng-mesh.js +4 -9
- package/src/superlocalmemory/ui/js/ng-shell.js +8 -8
- package/src/superlocalmemory/ui/js/ng-skills.js +54 -2
- package/src/superlocalmemory/ui/js/od-agents.js +544 -0
- package/src/superlocalmemory/ui/js/od-auth-gate.js +257 -0
- package/src/superlocalmemory/ui/js/od-backup.js +780 -0
- package/src/superlocalmemory/ui/js/od-brain.js +779 -0
- package/src/superlocalmemory/ui/js/od-entities.js +579 -0
- package/src/superlocalmemory/ui/js/od-graph.js +593 -0
- package/src/superlocalmemory/ui/js/od-health.js +539 -0
- package/src/superlocalmemory/ui/js/od-mcp.js +508 -0
- package/src/superlocalmemory/ui/js/od-memories.js +887 -0
- package/src/superlocalmemory/ui/js/od-mesh.js +539 -0
- package/src/superlocalmemory/ui/js/od-operations.js +1250 -0
- package/src/superlocalmemory/ui/js/od-optimize.js +787 -0
- package/src/superlocalmemory/ui/js/od-settings.js +1053 -0
- package/src/superlocalmemory/ui/js/od-shell.js +593 -0
- package/src/superlocalmemory/ui/js/od-skills.js +573 -0
- package/src/superlocalmemory/ui/js/od-team.js +258 -0
- package/src/superlocalmemory/ui/js/profiles.js +159 -46
- package/src/superlocalmemory/ui/js/settings.js +2 -2
- package/src/superlocalmemory/ui/js/timeline.js +34 -5
- package/src/superlocalmemory/ui/js/trust-dashboard.js +2 -2
- package/src/superlocalmemory/vector/lancedb_backend.py +8 -6
- package/src/superlocalmemory/learning/behavioral_listener.py +0 -94
|
@@ -211,6 +211,43 @@ class GraphAnalyzer:
|
|
|
211
211
|
result[node] = comm_id
|
|
212
212
|
return result
|
|
213
213
|
|
|
214
|
+
def detect_communities_louvain(
|
|
215
|
+
self,
|
|
216
|
+
graph: Any = None,
|
|
217
|
+
profile_id: str = "",
|
|
218
|
+
) -> dict[str, int]:
|
|
219
|
+
"""Detect communities via Louvain (modularity-optimizing).
|
|
220
|
+
|
|
221
|
+
Higher quality than Label Propagation (deterministic with a seed,
|
|
222
|
+
no giant-community collapse), pure-Python via networkx — no extra
|
|
223
|
+
binary deps. Falls back to Label Propagation if unavailable.
|
|
224
|
+
"""
|
|
225
|
+
import networkx as nx
|
|
226
|
+
|
|
227
|
+
if graph is None:
|
|
228
|
+
graph = self._build_networkx_graph(profile_id)
|
|
229
|
+
if graph.number_of_nodes() == 0:
|
|
230
|
+
return {}
|
|
231
|
+
|
|
232
|
+
undirected = graph.to_undirected()
|
|
233
|
+
try:
|
|
234
|
+
from networkx.algorithms.community import louvain_communities
|
|
235
|
+
|
|
236
|
+
communities = louvain_communities(
|
|
237
|
+
undirected, weight="weight", seed=42,
|
|
238
|
+
)
|
|
239
|
+
except Exception as exc:
|
|
240
|
+
logger.debug(
|
|
241
|
+
"Louvain unavailable/failed (%s); using Label Propagation", exc,
|
|
242
|
+
)
|
|
243
|
+
return self.detect_communities(graph, profile_id)
|
|
244
|
+
|
|
245
|
+
result: dict[str, int] = {}
|
|
246
|
+
for comm_id, community in enumerate(communities):
|
|
247
|
+
for node in community:
|
|
248
|
+
result[node] = comm_id
|
|
249
|
+
return result
|
|
250
|
+
|
|
214
251
|
# ── v3.4.1: Leiden Community Detection ────────────────────────
|
|
215
252
|
|
|
216
253
|
def detect_communities_leiden(
|
|
@@ -234,9 +271,9 @@ class GraphAnalyzer:
|
|
|
234
271
|
import igraph
|
|
235
272
|
except ImportError:
|
|
236
273
|
logger.info(
|
|
237
|
-
"leidenalg not installed, using
|
|
274
|
+
"leidenalg not installed, using Louvain fallback",
|
|
238
275
|
)
|
|
239
|
-
return self.
|
|
276
|
+
return self.detect_communities_louvain(graph, profile_id)
|
|
240
277
|
|
|
241
278
|
# Convert DiGraph -> undirected -> igraph
|
|
242
279
|
undirected = graph.to_undirected()
|
|
@@ -350,8 +350,12 @@ def _cap_node_degree(
|
|
|
350
350
|
Algorithm (single-pass window function — no Python loops):
|
|
351
351
|
1. ROW_NUMBER() OVER (PARTITION BY source_id ORDER BY weight DESC) ranks
|
|
352
352
|
every edge per node in one full table scan.
|
|
353
|
-
2.
|
|
354
|
-
|
|
353
|
+
2. Excess edge IDs are collected in a reusable temp table.
|
|
354
|
+
3. A single DELETE statement removes them; rowcount is returned.
|
|
355
|
+
4. The temp table is created once and cleared via DELETE (not DROP) —
|
|
356
|
+
DROP TABLE acquires an EXCLUSIVE lock that conflicts with concurrent
|
|
357
|
+
writers, causing "database is locked". Using CREATE...IF NOT EXISTS
|
|
358
|
+
plus DELETE FROM avoids that conflict while preserving rowcount.
|
|
355
359
|
"""
|
|
356
360
|
# gi-03: cap BOTH out-degree (PARTITION BY source_id) AND in-degree
|
|
357
361
|
# (PARTITION BY target_id). Previously only out-degree was capped, so hub
|
|
@@ -379,9 +383,13 @@ def _cap_node_degree(
|
|
|
379
383
|
)
|
|
380
384
|
return excess
|
|
381
385
|
|
|
382
|
-
#
|
|
383
|
-
|
|
384
|
-
|
|
386
|
+
# CREATE IF NOT EXISTS + DELETE FROM instead of DROP + CREATE.
|
|
387
|
+
# DROP TABLE acquires EXCLUSIVE which conflicts with concurrent writers.
|
|
388
|
+
# CREATE IF NOT EXISTS is idempotent; DELETE FROM clears prior contents.
|
|
389
|
+
c.execute(
|
|
390
|
+
"CREATE TEMP TABLE IF NOT EXISTS _slm_cap_del (edge_id TEXT PRIMARY KEY)"
|
|
391
|
+
)
|
|
392
|
+
c.execute("DELETE FROM _slm_cap_del")
|
|
385
393
|
c.execute(
|
|
386
394
|
"""
|
|
387
395
|
INSERT OR IGNORE INTO _slm_cap_del (edge_id)
|
|
@@ -396,7 +404,6 @@ def _cap_node_degree(
|
|
|
396
404
|
(profile_id, max_degree, max_degree),
|
|
397
405
|
)
|
|
398
406
|
|
|
399
|
-
# Step 2: delete the over-cap edges (single DELETE).
|
|
400
407
|
c.execute(
|
|
401
408
|
"""
|
|
402
409
|
DELETE FROM graph_edges
|
|
@@ -407,8 +414,6 @@ def _cap_node_degree(
|
|
|
407
414
|
)
|
|
408
415
|
deleted = c.rowcount
|
|
409
416
|
|
|
410
|
-
c.execute("DROP TABLE IF EXISTS _slm_cap_del")
|
|
411
|
-
|
|
412
417
|
logger.info(
|
|
413
418
|
"_cap_node_degree: deleted %d low-weight edges (max_degree=%d, in+out capped)",
|
|
414
419
|
deleted, max_degree,
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
|
|
2
|
+
# Licensed under AGPL-3.0-or-later - see LICENSE file
|
|
3
|
+
# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
|
|
4
|
+
|
|
5
|
+
"""Fact-augmented key expansion (Phase 4, T3b).
|
|
6
|
+
|
|
7
|
+
Generates *alternate keys* for a fact — synonyms, aliases, and paraphrases —
|
|
8
|
+
that get indexed in ``fact_expansion_fts`` and UNION'd into BM25 retrieval, so a
|
|
9
|
+
query for "automobile" or "the Big Apple" can match a fact that only says "car"
|
|
10
|
+
or "NYC".
|
|
11
|
+
|
|
12
|
+
Two tiers, matching the rest of SLM:
|
|
13
|
+
* Mode A (zero-LLM): pulls the fact's resolved entities' canonical names and
|
|
14
|
+
aliases from SLM's own entity graph — no model, no cost, and it reuses the
|
|
15
|
+
entity resolution already done at ingest.
|
|
16
|
+
* Mode B/C: additionally asks the LLM for a few paraphrase keywords (own
|
|
17
|
+
prompt, fail-open, bounded).
|
|
18
|
+
|
|
19
|
+
Keys already present in the fact's content are dropped — indexing them twice
|
|
20
|
+
adds nothing. Returns a single space-joined string ready for the FTS row.
|
|
21
|
+
|
|
22
|
+
Part of Qualixar | Author: Varun Pratap Bhardwaj
|
|
23
|
+
License: AGPL-3.0-or-later
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
from __future__ import annotations
|
|
27
|
+
|
|
28
|
+
import logging
|
|
29
|
+
from typing import TYPE_CHECKING, Any
|
|
30
|
+
|
|
31
|
+
if TYPE_CHECKING:
|
|
32
|
+
from superlocalmemory.storage.database import DatabaseManager
|
|
33
|
+
from superlocalmemory.storage.models import AtomicFact
|
|
34
|
+
|
|
35
|
+
logger = logging.getLogger(__name__)
|
|
36
|
+
|
|
37
|
+
# Bound the LLM enrichment so a pathological response can't bloat the index.
|
|
38
|
+
_MAX_LLM_KEYS = 8
|
|
39
|
+
_MAX_KEY_LEN = 60
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
class KeyExpander:
|
|
43
|
+
"""Produces alternate search keys for a fact (T3b)."""
|
|
44
|
+
|
|
45
|
+
__slots__ = ("_db", "_llm")
|
|
46
|
+
|
|
47
|
+
def __init__(self, db: DatabaseManager, llm: Any = None) -> None:
|
|
48
|
+
self._db = db
|
|
49
|
+
self._llm = llm
|
|
50
|
+
|
|
51
|
+
def expand(self, fact: AtomicFact, profile_id: str, mode: str = "a") -> str:
|
|
52
|
+
"""Return space-joined alternate keys for ``fact`` (may be empty)."""
|
|
53
|
+
keys: set[str] = set()
|
|
54
|
+
keys |= self._alias_keys(fact, profile_id)
|
|
55
|
+
if mode in ("b", "c") and self._llm_available():
|
|
56
|
+
keys |= self._llm_keys(fact)
|
|
57
|
+
|
|
58
|
+
content_low = (getattr(fact, "content", "") or "").lower()
|
|
59
|
+
cleaned = {
|
|
60
|
+
k.strip() for k in keys
|
|
61
|
+
if k and k.strip() and len(k.strip()) <= _MAX_KEY_LEN
|
|
62
|
+
and k.strip().lower() not in content_low
|
|
63
|
+
}
|
|
64
|
+
return " ".join(sorted(cleaned))
|
|
65
|
+
|
|
66
|
+
# -- Mode A: entity aliases from SLM's own entity graph ------------------
|
|
67
|
+
|
|
68
|
+
def _alias_keys(self, fact: AtomicFact, profile_id: str) -> set[str]:
|
|
69
|
+
out: set[str] = set()
|
|
70
|
+
for name in (getattr(fact, "canonical_entities", None) or []):
|
|
71
|
+
if not name:
|
|
72
|
+
continue
|
|
73
|
+
try:
|
|
74
|
+
ent = self._db.get_entity_by_name(name, profile_id)
|
|
75
|
+
except Exception:
|
|
76
|
+
ent = None
|
|
77
|
+
if ent is None:
|
|
78
|
+
continue
|
|
79
|
+
canonical = getattr(ent, "canonical_name", None)
|
|
80
|
+
if canonical:
|
|
81
|
+
out.add(canonical)
|
|
82
|
+
entity_id = getattr(ent, "entity_id", None)
|
|
83
|
+
if not entity_id:
|
|
84
|
+
continue
|
|
85
|
+
try:
|
|
86
|
+
for alias in self._db.get_aliases_for_entity(entity_id, profile_id):
|
|
87
|
+
a = getattr(alias, "alias", None)
|
|
88
|
+
if a:
|
|
89
|
+
out.add(a)
|
|
90
|
+
except Exception:
|
|
91
|
+
continue
|
|
92
|
+
return out
|
|
93
|
+
|
|
94
|
+
# -- Mode B/C: LLM paraphrases (own prompt, fail-open) -------------------
|
|
95
|
+
|
|
96
|
+
def _llm_available(self) -> bool:
|
|
97
|
+
if self._llm is None:
|
|
98
|
+
return False
|
|
99
|
+
check = getattr(self._llm, "is_available", None)
|
|
100
|
+
try:
|
|
101
|
+
return bool(check()) if callable(check) else bool(check)
|
|
102
|
+
except Exception:
|
|
103
|
+
return False
|
|
104
|
+
|
|
105
|
+
def _llm_keys(self, fact: AtomicFact) -> set[str]:
|
|
106
|
+
content = getattr(fact, "content", "") or ""
|
|
107
|
+
if not content:
|
|
108
|
+
return set()
|
|
109
|
+
prompt = (
|
|
110
|
+
"Give 3-6 short alternative search terms (synonyms, aliases, or "
|
|
111
|
+
"paraphrases) that a person might use to look up the memory below. "
|
|
112
|
+
"Output ONLY a comma-separated list, no numbering, no explanation.\n\n"
|
|
113
|
+
f"Memory: {content}"
|
|
114
|
+
)
|
|
115
|
+
raw = self._invoke_llm(prompt)
|
|
116
|
+
if not raw:
|
|
117
|
+
return set()
|
|
118
|
+
seen: list[str] = []
|
|
119
|
+
for part in raw.replace("\n", ",").split(","):
|
|
120
|
+
p = part.strip()
|
|
121
|
+
if p and p not in seen:
|
|
122
|
+
seen.append(p)
|
|
123
|
+
if len(seen) >= _MAX_LLM_KEYS:
|
|
124
|
+
break
|
|
125
|
+
return set(seen)
|
|
126
|
+
|
|
127
|
+
def _invoke_llm(self, prompt: str) -> str:
|
|
128
|
+
"""Call whatever generation method the injected LLM exposes; fail-open."""
|
|
129
|
+
for meth in ("generate", "complete", "chat"):
|
|
130
|
+
fn = getattr(self._llm, meth, None)
|
|
131
|
+
if callable(fn):
|
|
132
|
+
try:
|
|
133
|
+
out = fn(prompt)
|
|
134
|
+
return out if isinstance(out, str) else str(out or "")
|
|
135
|
+
except Exception as exc:
|
|
136
|
+
logger.debug("KeyExpander LLM (%s) failed: %s", meth, exc)
|
|
137
|
+
return ""
|
|
138
|
+
return ""
|
|
@@ -127,6 +127,7 @@ def run_maintenance(
|
|
|
127
127
|
"sheaf_checked": 0,
|
|
128
128
|
"entity_summaries_consolidated": 0, # V3.4.40
|
|
129
129
|
"orphan_metadata_gc": 0, # v3.6.4 (P1-3)
|
|
130
|
+
"expansion_backfilled": 0, # T3b
|
|
130
131
|
}
|
|
131
132
|
|
|
132
133
|
# P1-3 (embeddings-vector-02): sweep orphaned embedding_metadata left by
|
|
@@ -141,6 +142,28 @@ def run_maintenance(
|
|
|
141
142
|
if not facts:
|
|
142
143
|
return counts
|
|
143
144
|
|
|
145
|
+
# T3b: backfill fact-expansion alt-keys (Mode A, entity-alias based) for
|
|
146
|
+
# facts stored before expansion existed. Bounded per run + skips already-
|
|
147
|
+
# populated and entity-less facts, so it converges without re-work churn.
|
|
148
|
+
try:
|
|
149
|
+
from superlocalmemory.core.key_expander import KeyExpander
|
|
150
|
+
populated = {
|
|
151
|
+
dict(r)["fact_id"]
|
|
152
|
+
for r in db.execute("SELECT DISTINCT fact_id FROM fact_expansion_fts")
|
|
153
|
+
}
|
|
154
|
+
expander = KeyExpander(db)
|
|
155
|
+
for f in facts:
|
|
156
|
+
if counts["expansion_backfilled"] >= 500:
|
|
157
|
+
break
|
|
158
|
+
if f.fact_id in populated or not f.canonical_entities:
|
|
159
|
+
continue
|
|
160
|
+
alt = expander.expand(f, profile_id, mode="a")
|
|
161
|
+
if alt:
|
|
162
|
+
db.upsert_fact_expansion(f.fact_id, alt)
|
|
163
|
+
counts["expansion_backfilled"] += 1
|
|
164
|
+
except Exception as exc: # pragma: no cover — legacy DB / missing FTS
|
|
165
|
+
logger.debug("expansion backfill skipped: %s", exc)
|
|
166
|
+
|
|
144
167
|
# 1a. Backfill: seed uninitialized facts with metadata-aware positions (B+C)
|
|
145
168
|
if config.math.langevin_persist_positions:
|
|
146
169
|
try:
|
|
@@ -109,7 +109,7 @@ MODE_C = ModeCapabilities(
|
|
|
109
109
|
data_stays_local=False,
|
|
110
110
|
description=(
|
|
111
111
|
"FULL POWER — UNRESTRICTED. Best embeddings (text-embedding-3-large, 3072-dim). "
|
|
112
|
-
"Best LLMs (GPT-5
|
|
112
|
+
"Best configured cloud LLMs (e.g. GPT-5, Claude Opus 4). Agentic multi-round retrieval. "
|
|
113
113
|
"Cohere reranker option. No EU restriction. Target: 90%+"
|
|
114
114
|
),
|
|
115
115
|
)
|
|
@@ -59,7 +59,7 @@ def delete_fact_authorized(
|
|
|
59
59
|
if not rows:
|
|
60
60
|
return {"ok": False, "error": f"Memory {fact_id} not found"}
|
|
61
61
|
content_preview = dict(rows[0]).get("content", "")[:80]
|
|
62
|
-
engine._db.delete_fact(fact_id)
|
|
62
|
+
engine._db.delete_fact(fact_id, profile_id=profile_id)
|
|
63
63
|
try:
|
|
64
64
|
from superlocalmemory.core.backend_orchestrator import get_orchestrator
|
|
65
65
|
orchestrator = get_orchestrator()
|
|
@@ -120,7 +120,7 @@ def update_fact_authorized(
|
|
|
120
120
|
updates["fisher_variance"] = fisher_variance
|
|
121
121
|
except Exception as exc:
|
|
122
122
|
logger.warning("UPDATE embedding refresh failed: %s", exc)
|
|
123
|
-
engine._db.update_fact(fact_id, updates)
|
|
123
|
+
engine._db.update_fact(fact_id, updates, profile_id=profile_id)
|
|
124
124
|
try:
|
|
125
125
|
from superlocalmemory.core.backend_orchestrator import get_orchestrator
|
|
126
126
|
orchestrator = get_orchestrator()
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
|
|
2
|
+
# Licensed under AGPL-3.0-or-later - see LICENSE file
|
|
3
|
+
# Part of SuperLocalMemory V3 — PII redaction on ingest (C4)
|
|
4
|
+
|
|
5
|
+
"""Opt-in PII redaction for ingested memory content.
|
|
6
|
+
|
|
7
|
+
For team / company deployments an operator may need memory to never persist
|
|
8
|
+
personal identifiers (email, phone, national ID, payment card, IP). This
|
|
9
|
+
module provides a pure, well-bounded scrubber that replaces detected PII with
|
|
10
|
+
``[PII:TYPE]`` markers. It is complementary to ``security_primitives.
|
|
11
|
+
redact_secrets`` (which handles API keys / tokens and always runs).
|
|
12
|
+
|
|
13
|
+
Design goals:
|
|
14
|
+
* **Low false-positive rate.** Card numbers are Luhn-validated; SSNs use the
|
|
15
|
+
canonical grouping; phone matching requires a plausible separator shape.
|
|
16
|
+
* **Deterministic + pure.** No I/O, no config — the caller decides when to run
|
|
17
|
+
it (gated by SLM_PII_REDACTION / config), so it is trivially testable.
|
|
18
|
+
* **Order matters.** Emails are redacted before phone/number sweeps so an
|
|
19
|
+
email's local part is never mistaken for a number.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
24
|
+
import re
|
|
25
|
+
|
|
26
|
+
# Order-sensitive: earlier patterns win over later ones on overlapping spans.
|
|
27
|
+
_EMAIL = re.compile(r"\b[A-Za-z0-9._%+\-]+@[A-Za-z0-9.\-]+\.[A-Za-z]{2,}\b")
|
|
28
|
+
_SSN = re.compile(r"\b\d{3}-\d{2}-\d{4}\b")
|
|
29
|
+
_IPV4 = re.compile(
|
|
30
|
+
r"\b(?:(?:25[0-5]|2[0-4]\d|1?\d?\d)\.){3}(?:25[0-5]|2[0-4]\d|1?\d?\d)\b"
|
|
31
|
+
)
|
|
32
|
+
# Phone: conservative to avoid eating ISO dates (4-2-2) / version strings.
|
|
33
|
+
# Only unambiguous shapes match:
|
|
34
|
+
# * international +CC then grouped digits
|
|
35
|
+
# * parenthesized (415) 555-0132
|
|
36
|
+
# * strict US 415-555-0132 / 415.555.0132 (3-3-4, dot/dash only — a
|
|
37
|
+
# space separator is excluded so "2026-07-22 12" style runs never match).
|
|
38
|
+
_PHONE = re.compile(
|
|
39
|
+
r"(?<!\w)(?:"
|
|
40
|
+
r"\+\d{1,3}[\s.\-]?\d{1,4}[\s.\-]?\d{2,4}[\s.\-]?\d{2,4}"
|
|
41
|
+
r"|\(\d{3}\)[\s.\-]?\d{3}[\s.\-]?\d{4}"
|
|
42
|
+
r"|\d{3}[.\-]\d{3}[.\-]\d{4}"
|
|
43
|
+
r")(?!\w)"
|
|
44
|
+
)
|
|
45
|
+
# Candidate card: 13–19 digits, optionally grouped by space/dash. Luhn-checked.
|
|
46
|
+
_CARD_CANDIDATE = re.compile(r"(?<!\w)(?:\d[ -]?){13,19}(?!\w)")
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _luhn_ok(digits: str) -> bool:
|
|
50
|
+
"""Return True if ``digits`` (0-9 only) passes the Luhn checksum."""
|
|
51
|
+
if not (13 <= len(digits) <= 19):
|
|
52
|
+
return False
|
|
53
|
+
total = 0
|
|
54
|
+
parity = len(digits) % 2
|
|
55
|
+
for i, ch in enumerate(digits):
|
|
56
|
+
d = ord(ch) - 48
|
|
57
|
+
if i % 2 == parity:
|
|
58
|
+
d *= 2
|
|
59
|
+
if d > 9:
|
|
60
|
+
d -= 9
|
|
61
|
+
total += d
|
|
62
|
+
return total % 10 == 0
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _redact_cards(text: str, counter: list[int]) -> str:
|
|
66
|
+
def _sub(m: re.Match[str]) -> str:
|
|
67
|
+
raw = m.group(0)
|
|
68
|
+
digits = re.sub(r"\D", "", raw)
|
|
69
|
+
if _luhn_ok(digits):
|
|
70
|
+
counter[0] += 1
|
|
71
|
+
return "[PII:CARD]"
|
|
72
|
+
return raw
|
|
73
|
+
return _CARD_CANDIDATE.sub(_sub, text)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def redact_pii(text: str) -> tuple[str, int]:
|
|
77
|
+
"""Return ``(redacted_text, num_redactions)``.
|
|
78
|
+
|
|
79
|
+
Never raises; a non-string or empty input is returned unchanged with 0.
|
|
80
|
+
"""
|
|
81
|
+
if not isinstance(text, str) or not text:
|
|
82
|
+
return text, 0
|
|
83
|
+
|
|
84
|
+
counter = [0]
|
|
85
|
+
|
|
86
|
+
def _count_sub(pattern: re.Pattern[str], label: str, s: str) -> str:
|
|
87
|
+
def _sub(_m: re.Match[str]) -> str:
|
|
88
|
+
counter[0] += 1
|
|
89
|
+
return label
|
|
90
|
+
return pattern.sub(_sub, s)
|
|
91
|
+
|
|
92
|
+
out = text
|
|
93
|
+
# Email first (protects local parts from the number sweeps).
|
|
94
|
+
out = _count_sub(_EMAIL, "[PII:EMAIL]", out)
|
|
95
|
+
# Payment cards before generic phone/number matching (Luhn-gated).
|
|
96
|
+
out = _redact_cards(out, counter)
|
|
97
|
+
out = _count_sub(_SSN, "[PII:SSN]", out)
|
|
98
|
+
out = _count_sub(_IPV4, "[PII:IP]", out)
|
|
99
|
+
out = _count_sub(_PHONE, "[PII:PHONE]", out)
|
|
100
|
+
return out, counter[0]
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def redact_pii_text(text: str) -> str:
|
|
104
|
+
"""Convenience wrapper returning only the redacted string."""
|
|
105
|
+
return redact_pii(text)[0]
|
|
@@ -0,0 +1,208 @@
|
|
|
1
|
+
# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
|
|
2
|
+
# Licensed under AGPL-3.0-or-later - see LICENSE file
|
|
3
|
+
# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
|
|
4
|
+
|
|
5
|
+
"""Progressive abstraction (Wave Q3) — the top persona tier + drill-down.
|
|
6
|
+
|
|
7
|
+
Completes the abstraction hierarchy on the ONE principled backbone (rather
|
|
8
|
+
than a fourth divergent clustering):
|
|
9
|
+
|
|
10
|
+
atoms (atomic_facts)
|
|
11
|
+
-> entity communities (Wave Q backbone)
|
|
12
|
+
-> community summaries (Wave Q2)
|
|
13
|
+
-> persona roll-up (this module)
|
|
14
|
+
|
|
15
|
+
The persona is one bounded roll-up per profile that consumes the top community
|
|
16
|
+
summaries. It is recall-GATED (never auto-injected into the hot recall path —
|
|
17
|
+
avoids the V3.4.40 summary-pollution regression) and SIZE-bounded. Drill-down
|
|
18
|
+
(``get_sources``) walks the hierarchy back down to the source atoms, matching
|
|
19
|
+
the market bar for summary->source provenance (Zep-style).
|
|
20
|
+
|
|
21
|
+
Runs in the background consolidation lane after community summaries.
|
|
22
|
+
Fail-open throughout; recompute replaces a profile's row.
|
|
23
|
+
|
|
24
|
+
Part of Qualixar | Author: Varun Pratap Bhardwaj
|
|
25
|
+
License: AGPL-3.0-or-later
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
from __future__ import annotations
|
|
29
|
+
|
|
30
|
+
import json
|
|
31
|
+
import logging
|
|
32
|
+
from typing import Any
|
|
33
|
+
|
|
34
|
+
from superlocalmemory.core.community_summary import CommunitySummaryBuilder
|
|
35
|
+
|
|
36
|
+
logger = logging.getLogger(__name__)
|
|
37
|
+
|
|
38
|
+
_PERSONA_MAX_CHARS = 2048
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class ProgressiveAbstraction:
|
|
42
|
+
"""Build + persist the persona tier; provide hierarchy drill-down."""
|
|
43
|
+
|
|
44
|
+
def __init__(
|
|
45
|
+
self,
|
|
46
|
+
db: Any,
|
|
47
|
+
summarizer: Any = None,
|
|
48
|
+
max_communities_in_persona: int = 8,
|
|
49
|
+
persona_max_chars: int = _PERSONA_MAX_CHARS,
|
|
50
|
+
max_keywords: int = 12,
|
|
51
|
+
) -> None:
|
|
52
|
+
self._db = db
|
|
53
|
+
self._summarizer = summarizer
|
|
54
|
+
self._max_communities = max(1, int(max_communities_in_persona))
|
|
55
|
+
self._persona_max_chars = max(256, int(persona_max_chars))
|
|
56
|
+
self._max_keywords = max(1, int(max_keywords))
|
|
57
|
+
|
|
58
|
+
# ------------------------------------------------------------------
|
|
59
|
+
# Build
|
|
60
|
+
# ------------------------------------------------------------------
|
|
61
|
+
|
|
62
|
+
def compute_and_store(self, profile_id: str) -> dict[str, Any]:
|
|
63
|
+
summaries = CommunitySummaryBuilder(self._db).get_summaries(profile_id)
|
|
64
|
+
try:
|
|
65
|
+
self._db.execute(
|
|
66
|
+
"DELETE FROM persona_summary WHERE profile_id = ?",
|
|
67
|
+
(profile_id,),
|
|
68
|
+
)
|
|
69
|
+
except Exception as exc:
|
|
70
|
+
logger.debug("persona_summary clear failed: %s", exc)
|
|
71
|
+
if not summaries:
|
|
72
|
+
return {"built": False, "communities_in_persona": 0}
|
|
73
|
+
|
|
74
|
+
top = summaries[: self._max_communities]
|
|
75
|
+
summary = self._persona_summary(top)
|
|
76
|
+
keywords = self._merge_keywords(top)
|
|
77
|
+
community_ids = [int(s["community_id"]) for s in top]
|
|
78
|
+
|
|
79
|
+
try:
|
|
80
|
+
self._db.execute(
|
|
81
|
+
"INSERT OR REPLACE INTO persona_summary "
|
|
82
|
+
"(profile_id, summary, keywords, community_ids_json, computed_at) "
|
|
83
|
+
"VALUES (?, ?, ?, ?, datetime('now'))",
|
|
84
|
+
(profile_id, summary, keywords, json.dumps(community_ids)),
|
|
85
|
+
)
|
|
86
|
+
except Exception as exc:
|
|
87
|
+
logger.debug("persona_summary write failed: %s", exc)
|
|
88
|
+
return {"built": False, "communities_in_persona": 0}
|
|
89
|
+
|
|
90
|
+
return {"built": True, "communities_in_persona": len(top)}
|
|
91
|
+
|
|
92
|
+
# ------------------------------------------------------------------
|
|
93
|
+
# Read API
|
|
94
|
+
# ------------------------------------------------------------------
|
|
95
|
+
|
|
96
|
+
def get_persona(self, profile_id: str) -> dict | None:
|
|
97
|
+
try:
|
|
98
|
+
rows = self._db.execute(
|
|
99
|
+
"SELECT * FROM persona_summary WHERE profile_id = ?",
|
|
100
|
+
(profile_id,),
|
|
101
|
+
)
|
|
102
|
+
except Exception as exc:
|
|
103
|
+
logger.debug("get_persona failed: %s", exc)
|
|
104
|
+
return None
|
|
105
|
+
if not rows:
|
|
106
|
+
return None
|
|
107
|
+
d = dict(rows[0])
|
|
108
|
+
try:
|
|
109
|
+
community_ids = json.loads(d.get("community_ids_json") or "[]")
|
|
110
|
+
except (ValueError, TypeError):
|
|
111
|
+
community_ids = []
|
|
112
|
+
return {
|
|
113
|
+
"profile_id": d.get("profile_id", profile_id),
|
|
114
|
+
"summary": d.get("summary", ""),
|
|
115
|
+
"keywords": d.get("keywords", ""),
|
|
116
|
+
"community_ids": community_ids,
|
|
117
|
+
"computed_at": d.get("computed_at", ""),
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
def get_sources(self, profile_id: str, node_id: Any) -> dict:
|
|
121
|
+
"""Drill-down: a tier node -> its child communities + source atoms.
|
|
122
|
+
|
|
123
|
+
node_id == "persona" -> the persona's member communities + their facts.
|
|
124
|
+
node_id == <community> -> that community's member facts.
|
|
125
|
+
Unknown node -> empty (never raises).
|
|
126
|
+
"""
|
|
127
|
+
result: dict[str, Any] = {
|
|
128
|
+
"node_id": node_id, "node_type": "unknown",
|
|
129
|
+
"communities": [], "fact_ids": [],
|
|
130
|
+
}
|
|
131
|
+
try:
|
|
132
|
+
if isinstance(node_id, str) and node_id.lower() == "persona":
|
|
133
|
+
persona = self.get_persona(profile_id)
|
|
134
|
+
cids = persona["community_ids"] if persona else []
|
|
135
|
+
fact_ids: list[str] = []
|
|
136
|
+
seen: set[str] = set()
|
|
137
|
+
for cid in cids:
|
|
138
|
+
for fid in self._community_fact_ids(profile_id, cid):
|
|
139
|
+
if fid not in seen:
|
|
140
|
+
seen.add(fid)
|
|
141
|
+
fact_ids.append(fid)
|
|
142
|
+
result.update(
|
|
143
|
+
node_type="persona", communities=list(cids), fact_ids=fact_ids,
|
|
144
|
+
)
|
|
145
|
+
return result
|
|
146
|
+
|
|
147
|
+
# Otherwise treat node_id as a community id.
|
|
148
|
+
cid = int(node_id)
|
|
149
|
+
fids = self._community_fact_ids(profile_id, cid)
|
|
150
|
+
result.update(
|
|
151
|
+
node_type="community", communities=[cid], fact_ids=fids,
|
|
152
|
+
)
|
|
153
|
+
return result
|
|
154
|
+
except (ValueError, TypeError):
|
|
155
|
+
return result
|
|
156
|
+
except Exception as exc:
|
|
157
|
+
logger.debug("get_sources failed: %s", exc)
|
|
158
|
+
return result
|
|
159
|
+
|
|
160
|
+
# ------------------------------------------------------------------
|
|
161
|
+
# Internal
|
|
162
|
+
# ------------------------------------------------------------------
|
|
163
|
+
|
|
164
|
+
def _community_fact_ids(self, profile_id: str, community_id: Any) -> list[str]:
|
|
165
|
+
try:
|
|
166
|
+
rows = self._db.execute(
|
|
167
|
+
"SELECT fact_ids_json FROM community_summaries "
|
|
168
|
+
"WHERE profile_id = ? AND community_id = ?",
|
|
169
|
+
(profile_id, int(community_id)),
|
|
170
|
+
)
|
|
171
|
+
except Exception as exc:
|
|
172
|
+
logger.debug("_community_fact_ids failed: %s", exc)
|
|
173
|
+
return []
|
|
174
|
+
if not rows:
|
|
175
|
+
return []
|
|
176
|
+
try:
|
|
177
|
+
return [str(f) for f in json.loads(dict(rows[0]).get("fact_ids_json") or "[]")]
|
|
178
|
+
except (ValueError, TypeError):
|
|
179
|
+
return []
|
|
180
|
+
|
|
181
|
+
def _persona_summary(self, top: list[dict]) -> str:
|
|
182
|
+
if self._summarizer is not None:
|
|
183
|
+
try:
|
|
184
|
+
text = self._summarizer.summarize_cluster(
|
|
185
|
+
[{"content": s.get("summary", "")} for s in top],
|
|
186
|
+
)
|
|
187
|
+
if text and text.strip():
|
|
188
|
+
return text.strip()[: self._persona_max_chars]
|
|
189
|
+
except Exception as exc:
|
|
190
|
+
logger.debug("persona summarizer failed (fail-open): %s", exc)
|
|
191
|
+
# Mode A keyword-dense fallback: stitch the top community summaries.
|
|
192
|
+
heads = [s.get("summary", "").strip() for s in top if s.get("summary")]
|
|
193
|
+
base = " ".join(heads) if heads else "No persona yet."
|
|
194
|
+
return base[: self._persona_max_chars]
|
|
195
|
+
|
|
196
|
+
def _merge_keywords(self, top: list[dict]) -> str:
|
|
197
|
+
seen: set[str] = set()
|
|
198
|
+
merged: list[str] = []
|
|
199
|
+
for s in top:
|
|
200
|
+
for kw in (s.get("keywords", "") or "").split(","):
|
|
201
|
+
k = kw.strip()
|
|
202
|
+
low = k.lower()
|
|
203
|
+
if k and low not in seen:
|
|
204
|
+
seen.add(low)
|
|
205
|
+
merged.append(k)
|
|
206
|
+
if len(merged) >= self._max_keywords:
|
|
207
|
+
return ", ".join(merged)
|
|
208
|
+
return ", ".join(merged)
|
|
@@ -623,6 +623,7 @@ def run_recall(
|
|
|
623
623
|
fast: bool = False,
|
|
624
624
|
include_global: bool = False,
|
|
625
625
|
include_shared: bool = False,
|
|
626
|
+
window: str | tuple[str, str] | None = None,
|
|
626
627
|
) -> RecallResponse:
|
|
627
628
|
"""Recall relevant facts for a query.
|
|
628
629
|
|
|
@@ -663,6 +664,7 @@ def run_recall(
|
|
|
663
664
|
extra_disabled_channels=extra_disabled,
|
|
664
665
|
include_global=include_global,
|
|
665
666
|
include_shared=include_shared,
|
|
667
|
+
window=window,
|
|
666
668
|
)
|
|
667
669
|
_mark("retrieval(chan+rerank)")
|
|
668
670
|
|