superlocalmemory 3.7.8 → 3.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/ATTRIBUTION.md +1 -3
- package/CHANGELOG.md +129 -0
- package/README.md +205 -123
- package/package.json +12 -3
- package/plugin/.claude-plugin/plugin.json +2 -3
- package/plugin/CLAUDE.md +8 -8
- package/plugin/agents/slm-governance-advisor.md +80 -0
- package/plugin/agents/slm-loop-runner.md +71 -0
- package/plugin/agents/slm-memory-advisor.md +10 -5
- package/plugin/agents/slm-optimize-advisor.md +9 -3
- package/plugin/commands/slm-loop.md +31 -0
- package/plugin/hooks/hooks.json +79 -0
- package/plugin/requirements.txt +1 -1
- package/plugin/scripts/slm-launch +46 -7
- package/plugin/settings.json +9 -0
- package/plugin/skills/slm-cache/SKILL.md +9 -1
- package/plugin/skills/slm-compress/SKILL.md +8 -1
- package/plugin/skills/slm-governance/SKILL.md +248 -0
- package/plugin/skills/slm-graph/SKILL.md +17 -3
- package/plugin/skills/slm-loop/SKILL.md +99 -0
- package/plugin/skills/slm-mesh/SKILL.md +282 -0
- package/plugin/skills/slm-profile/SKILL.md +148 -0
- package/plugin/skills/slm-recall/SKILL.md +46 -10
- package/plugin/skills/slm-remember/SKILL.md +48 -1
- package/plugin/skills/slm-scope/SKILL.md +176 -0
- package/plugin/skills/slm-session/SKILL.md +24 -1
- package/plugin/skills/slm-status/SKILL.md +18 -1
- package/plugin-src/rules/AGENTS.md +57 -18
- package/plugin-src/skills/slm-cache/SKILL.md +9 -1
- package/plugin-src/skills/slm-compress/SKILL.md +8 -1
- package/plugin-src/skills/slm-graph/SKILL.md +17 -3
- package/plugin-src/skills/slm-recall/SKILL.md +46 -10
- package/plugin-src/skills/slm-remember/SKILL.md +48 -1
- package/plugin-src/skills/slm-session/SKILL.md +24 -1
- package/plugin-src/skills/slm-status/SKILL.md +18 -1
- package/pyproject.toml +2 -1
- package/scripts/postinstall/validation.js +2 -0
- package/scripts/postinstall-interactive.js +74 -2
- package/src/superlocalmemory/__init__.py +1 -1
- package/src/superlocalmemory/access/__init__.py +3 -0
- package/src/superlocalmemory/access/rbac.py +477 -0
- package/src/superlocalmemory/cli/commands.py +228 -17
- package/src/superlocalmemory/cli/compress_cmd.py +17 -7
- package/src/superlocalmemory/cli/daemon.py +7 -0
- package/src/superlocalmemory/cli/loop_cmd.py +187 -0
- package/src/superlocalmemory/cli/main.py +49 -8
- package/src/superlocalmemory/cli/mesh_cmd.py +38 -0
- package/src/superlocalmemory/cli/optimize_cmd.py +3 -0
- package/src/superlocalmemory/cli/pending_store.py +49 -13
- package/src/superlocalmemory/cli/proxy_cmd.py +4 -0
- package/src/superlocalmemory/cli/scale_engine_cmd.py +6 -0
- package/src/superlocalmemory/cli/setup_wizard.py +22 -13
- package/src/superlocalmemory/cli/version_banner.py +17 -3
- package/src/superlocalmemory/compliance/audit.py +6 -0
- package/src/superlocalmemory/compliance/gdpr.py +128 -138
- package/src/superlocalmemory/compliance/retention.py +176 -45
- package/src/superlocalmemory/core/backend_orchestrator.py +23 -59
- package/src/superlocalmemory/core/community_summary.py +267 -0
- package/src/superlocalmemory/core/config.py +216 -3
- package/src/superlocalmemory/core/consolidation_engine.py +95 -22
- package/src/superlocalmemory/core/context_cache.py +61 -18
- package/src/superlocalmemory/core/embedding_worker.py +21 -7
- package/src/superlocalmemory/core/embeddings.py +131 -46
- package/src/superlocalmemory/core/engine.py +41 -22
- package/src/superlocalmemory/core/engine_ingestion.py +359 -43
- package/src/superlocalmemory/core/engine_wiring.py +13 -0
- package/src/superlocalmemory/core/entity_community.py +178 -0
- package/src/superlocalmemory/core/graph_analyzer.py +39 -2
- package/src/superlocalmemory/core/graph_pruner.py +13 -8
- package/src/superlocalmemory/core/ingestion_command.py +134 -25
- package/src/superlocalmemory/core/injection.py +12 -7
- package/src/superlocalmemory/core/key_expander.py +138 -0
- package/src/superlocalmemory/core/maintenance.py +23 -0
- package/src/superlocalmemory/core/maintenance_scheduler.py +17 -7
- package/src/superlocalmemory/core/modes.py +1 -1
- package/src/superlocalmemory/core/mutations.py +2 -2
- package/src/superlocalmemory/core/pii.py +105 -0
- package/src/superlocalmemory/core/progressive_abstraction.py +208 -0
- package/src/superlocalmemory/core/recall_pipeline.py +7 -3
- package/src/superlocalmemory/core/recall_worker.py +20 -6
- package/src/superlocalmemory/core/scale_engine.py +60 -1
- package/src/superlocalmemory/core/security_primitives.py +40 -2
- package/src/superlocalmemory/core/store_pipeline.py +186 -29
- package/src/superlocalmemory/core/worker_pool.py +21 -6
- package/src/superlocalmemory/encoding/entity_reflexion.py +200 -0
- package/src/superlocalmemory/encoding/entity_resolver.py +34 -24
- package/src/superlocalmemory/encoding/fact_extractor.py +26 -1
- package/src/superlocalmemory/encoding/temporal_validator.py +64 -1
- package/src/superlocalmemory/evolution/evolution_store.py +122 -45
- package/src/superlocalmemory/evolution/llm_dispatch.py +12 -1
- package/src/superlocalmemory/evolution/model_selection.py +160 -0
- package/src/superlocalmemory/evolution/mutation_generator.py +16 -0
- package/src/superlocalmemory/evolution/skill_evolver.py +127 -42
- package/src/superlocalmemory/evolution/triggers.py +22 -13
- package/src/superlocalmemory/graph/cozo_backend.py +43 -20
- package/src/superlocalmemory/hooks/adapter_base.py +5 -1
- package/src/superlocalmemory/hooks/auto_recall.py +13 -1
- package/src/superlocalmemory/hooks/claude_code_hooks.py +11 -0
- package/src/superlocalmemory/hooks/codex_assets.py +64 -5
- package/src/superlocalmemory/hooks/hook_daemon.py +20 -3
- package/src/superlocalmemory/hooks/hook_handlers.py +6 -1
- package/src/superlocalmemory/hooks/memory_protocol.py +54 -0
- package/src/superlocalmemory/hooks/portable_kit.py +148 -3
- package/src/superlocalmemory/infra/backup.py +12 -1
- package/src/superlocalmemory/infra/daemon_identity.py +40 -4
- package/src/superlocalmemory/infra/data_root.py +43 -4
- package/src/superlocalmemory/infra/event_bus.py +107 -24
- package/src/superlocalmemory/infra/rate_limiter.py +93 -0
- package/src/superlocalmemory/ingestion/adapter_manager.py +4 -1
- package/src/superlocalmemory/ingestion/credentials.py +1 -1
- package/src/superlocalmemory/learning/cross_project.py +28 -19
- package/src/superlocalmemory/learning/model_rollback.py +3 -0
- package/src/superlocalmemory/learning/ranker_retrain_online.py +2 -0
- package/src/superlocalmemory/learning/reward.py +50 -0
- package/src/superlocalmemory/learning/reward_proxy.py +42 -9
- package/src/superlocalmemory/learning/source_quality.py +523 -1
- package/src/superlocalmemory/loops/__init__.py +56 -0
- package/src/superlocalmemory/loops/budget.py +58 -0
- package/src/superlocalmemory/loops/engine.py +164 -0
- package/src/superlocalmemory/loops/ledger.py +263 -0
- package/src/superlocalmemory/loops/models.py +152 -0
- package/src/superlocalmemory/loops/rules.py +52 -0
- package/src/superlocalmemory/mcp/_daemon_proxy.py +3 -0
- package/src/superlocalmemory/mcp/_pool_adapter.py +2 -0
- package/src/superlocalmemory/mcp/profiles.py +103 -0
- package/src/superlocalmemory/mcp/server.py +32 -79
- package/src/superlocalmemory/mcp/tools_active.py +4 -7
- package/src/superlocalmemory/mcp/tools_code_graph.py +51 -5
- package/src/superlocalmemory/mcp/tools_core.py +12 -4
- package/src/superlocalmemory/mcp/tools_evolution.py +6 -3
- package/src/superlocalmemory/mcp/tools_learning.py +2 -2
- package/src/superlocalmemory/mcp/tools_loops.py +300 -0
- package/src/superlocalmemory/mcp/tools_mesh.py +140 -4
- package/src/superlocalmemory/mcp/tools_optimize.py +15 -8
- package/src/superlocalmemory/mesh/broker.py +237 -129
- package/src/superlocalmemory/mesh/remote_sync.py +50 -8
- package/src/superlocalmemory/optimize/NOTICE +1 -6
- package/src/superlocalmemory/optimize/adapters/anthropic_adapter.py +1 -4
- package/src/superlocalmemory/optimize/adapters/openai_adapter.py +1 -4
- package/src/superlocalmemory/optimize/cache/semantic.py +27 -19
- package/src/superlocalmemory/optimize/compress/align.py +32 -26
- package/src/superlocalmemory/optimize/compress/ccr.py +14 -71
- package/src/superlocalmemory/optimize/compress/router.py +105 -22
- package/src/superlocalmemory/optimize/config/defaults.py +1 -1
- package/src/superlocalmemory/optimize/config/schema.py +87 -4
- package/src/superlocalmemory/optimize/metrics/counters.py +13 -4
- package/src/superlocalmemory/optimize/metrics/estimator.py +0 -3
- package/src/superlocalmemory/optimize/proxy/_helpers.py +31 -4
- package/src/superlocalmemory/optimize/storage/db.py +38 -9
- package/src/superlocalmemory/optimize/storage/schema.py +10 -0
- package/src/superlocalmemory/parameterization/pattern_extractor.py +6 -3
- package/src/superlocalmemory/retrieval/agentic.py +1 -1
- package/src/superlocalmemory/retrieval/bm25_channel.py +68 -10
- package/src/superlocalmemory/retrieval/engine.py +221 -47
- package/src/superlocalmemory/retrieval/entity_channel.py +7 -5
- package/src/superlocalmemory/retrieval/hopfield_channel.py +9 -2
- package/src/superlocalmemory/retrieval/reranker.py +3 -4
- package/src/superlocalmemory/retrieval/semantic_channel.py +114 -21
- package/src/superlocalmemory/retrieval/spreading_activation.py +11 -2
- package/src/superlocalmemory/retrieval/temporal_channel.py +48 -9
- package/src/superlocalmemory/retrieval/temporal_frame.py +102 -0
- package/src/superlocalmemory/retrieval/temporal_validity_filter.py +135 -0
- package/src/superlocalmemory/retrieval/time_window.py +181 -0
- package/src/superlocalmemory/server/api.py +4 -4
- package/src/superlocalmemory/server/config_file.py +90 -0
- package/src/superlocalmemory/server/origin.py +50 -0
- package/src/superlocalmemory/server/profile_runtime.py +125 -8
- package/src/superlocalmemory/server/rbac_enforce.py +142 -0
- package/src/superlocalmemory/server/recall_health.py +24 -3
- package/src/superlocalmemory/server/recall_serializer.py +19 -1
- package/src/superlocalmemory/server/routes/abstraction.py +115 -0
- package/src/superlocalmemory/server/routes/agents.py +128 -38
- package/src/superlocalmemory/server/routes/backup.py +317 -70
- package/src/superlocalmemory/server/routes/behavioral.py +349 -71
- package/src/superlocalmemory/server/routes/brain.py +69 -12
- package/src/superlocalmemory/server/routes/chat.py +10 -5
- package/src/superlocalmemory/server/routes/compliance.py +171 -21
- package/src/superlocalmemory/server/routes/config_api.py +438 -0
- package/src/superlocalmemory/server/routes/data_io.py +30 -8
- package/src/superlocalmemory/server/routes/entity.py +108 -26
- package/src/superlocalmemory/server/routes/events.py +24 -8
- package/src/superlocalmemory/server/routes/evolution.py +189 -68
- package/src/superlocalmemory/server/routes/helpers.py +16 -1
- package/src/superlocalmemory/server/routes/ingest.py +7 -4
- package/src/superlocalmemory/server/routes/insights.py +3 -3
- package/src/superlocalmemory/server/routes/learning.py +289 -118
- package/src/superlocalmemory/server/routes/learning_telemetry.py +153 -0
- package/src/superlocalmemory/server/routes/lifecycle.py +59 -8
- package/src/superlocalmemory/server/routes/memories.py +182 -57
- package/src/superlocalmemory/server/routes/mesh.py +200 -31
- package/src/superlocalmemory/server/routes/optimize.py +33 -1
- package/src/superlocalmemory/server/routes/prewarm.py +2 -0
- package/src/superlocalmemory/server/routes/profiles.py +63 -17
- package/src/superlocalmemory/server/routes/ratelimit.py +132 -0
- package/src/superlocalmemory/server/routes/rbac.py +367 -0
- package/src/superlocalmemory/server/routes/stats.py +103 -158
- package/src/superlocalmemory/server/routes/tiers.py +11 -9
- package/src/superlocalmemory/server/routes/token.py +3 -13
- package/src/superlocalmemory/server/routes/v3_api.py +247 -89
- package/src/superlocalmemory/server/routes/ws.py +5 -2
- package/src/superlocalmemory/server/security_middleware.py +12 -5
- package/src/superlocalmemory/server/ui.py +20 -5
- package/src/superlocalmemory/server/unified_daemon.py +827 -72
- package/src/superlocalmemory/server/write_identity.py +38 -8
- package/src/superlocalmemory/storage/database.py +265 -53
- package/src/superlocalmemory/storage/migration_runner.py +132 -1
- package/src/superlocalmemory/storage/migrations/M010_evolution_config.py +5 -0
- package/src/superlocalmemory/storage/migrations/M021_ingestion_log_profile.py +108 -0
- package/src/superlocalmemory/storage/migrations/M022_entity_aliases_profile.py +86 -0
- package/src/superlocalmemory/storage/migrations/M023_mesh_profile_isolation.py +194 -0
- package/src/superlocalmemory/storage/migrations/M024_rbac_users_roles.py +87 -0
- package/src/superlocalmemory/storage/migrations/M025_perf_indexes.py +90 -0
- package/src/superlocalmemory/storage/migrations/M026_rbac_memberships_fk.py +136 -0
- package/src/superlocalmemory/storage/migrations/M027_transferable_patterns_profile.py +163 -0
- package/src/superlocalmemory/storage/migrations/M028_fact_entity_associations.py +270 -0
- package/src/superlocalmemory/storage/migrations/M029_behavioral_history_indexes.py +137 -0
- package/src/superlocalmemory/storage/migrations/M030_entity_explorer_indexes.py +93 -0
- package/src/superlocalmemory/storage/migrations/__init__.py +4 -0
- package/src/superlocalmemory/storage/models.py +4 -0
- package/src/superlocalmemory/storage/schema.py +136 -1
- package/src/superlocalmemory/storage/schema_v32.py +2 -0
- package/src/superlocalmemory/storage/schema_v343.py +24 -12
- package/src/superlocalmemory/storage/schema_v347.py +4 -0
- package/src/superlocalmemory/trust/gate.py +49 -8
- package/src/superlocalmemory/ui/assets/slm-icon-white.svg +64 -0
- package/src/superlocalmemory/ui/assets/slm-icon.svg +36 -0
- package/src/superlocalmemory/ui/css/design-system.css +621 -0
- package/src/superlocalmemory/ui/css/neural-glass.css +6 -0
- package/src/superlocalmemory/ui/css/od-bridge.css +158 -0
- package/src/superlocalmemory/ui/favicon.svg +35 -4
- package/src/superlocalmemory/ui/index.html +303 -173
- package/src/superlocalmemory/ui/js/brain.js +5 -20
- package/src/superlocalmemory/ui/js/core.js +100 -41
- package/src/superlocalmemory/ui/js/dashboard.js +403 -65
- package/src/superlocalmemory/ui/js/event-delegation.js +102 -0
- package/src/superlocalmemory/ui/js/knowledge-graph.js +11 -11
- package/src/superlocalmemory/ui/js/math-health.js +1 -1
- package/src/superlocalmemory/ui/js/memories.js +15 -4
- package/src/superlocalmemory/ui/js/memory-chat.js +7 -7
- package/src/superlocalmemory/ui/js/ng-entities.js +6 -8
- package/src/superlocalmemory/ui/js/ng-ingestion.js +4 -4
- package/src/superlocalmemory/ui/js/ng-mesh.js +4 -9
- package/src/superlocalmemory/ui/js/ng-shell.js +8 -8
- package/src/superlocalmemory/ui/js/ng-skills.js +54 -2
- package/src/superlocalmemory/ui/js/od-agents.js +544 -0
- package/src/superlocalmemory/ui/js/od-auth-gate.js +257 -0
- package/src/superlocalmemory/ui/js/od-backup.js +871 -0
- package/src/superlocalmemory/ui/js/od-brain.js +816 -0
- package/src/superlocalmemory/ui/js/od-entities.js +579 -0
- package/src/superlocalmemory/ui/js/od-graph.js +600 -0
- package/src/superlocalmemory/ui/js/od-health.js +539 -0
- package/src/superlocalmemory/ui/js/od-mcp.js +508 -0
- package/src/superlocalmemory/ui/js/od-memories.js +929 -0
- package/src/superlocalmemory/ui/js/od-mesh.js +553 -0
- package/src/superlocalmemory/ui/js/od-operations.js +1250 -0
- package/src/superlocalmemory/ui/js/od-optimize.js +787 -0
- package/src/superlocalmemory/ui/js/od-settings.js +1107 -0
- package/src/superlocalmemory/ui/js/od-shell.js +809 -0
- package/src/superlocalmemory/ui/js/od-skills.js +600 -0
- package/src/superlocalmemory/ui/js/od-team.js +258 -0
- package/src/superlocalmemory/ui/js/profiles.js +159 -46
- package/src/superlocalmemory/ui/js/settings.js +17 -3
- package/src/superlocalmemory/ui/js/timeline.js +34 -5
- package/src/superlocalmemory/ui/js/trust-dashboard.js +2 -2
- package/src/superlocalmemory/vector/lancedb_backend.py +8 -6
- package/plugin-src/.mcp.json +0 -12
- package/plugin-src/agents/slm-memory-advisor.md +0 -44
- package/plugin-src/agents/slm-optimize-advisor.md +0 -38
- package/plugin-src/hooks/.gitkeep +0 -0
- package/plugin-src/hooks/hooks.json +0 -23
- package/plugin-src/manifest.json +0 -25
- package/plugin-src/requirements.txt +0 -1
- package/plugin-src/rules/CLAUDE.md.fragment +0 -44
- package/plugin-src/scripts/ensure-venv.bat +0 -122
- package/plugin-src/scripts/ensure-venv.sh +0 -105
- package/plugin-src/scripts/slm-launch +0 -23
- package/plugin-src/scripts/slm-launch.bat +0 -23
- package/plugin-src/settings.json +0 -16
- package/src/superlocalmemory/learning/behavioral_listener.py +0 -94
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
|
|
2
|
+
# Licensed under AGPL-3.0-or-later
|
|
3
|
+
|
|
4
|
+
"""M029 — profile-scoped indexes for large behavioral histories.
|
|
5
|
+
|
|
6
|
+
Dashboard reads always filter behavioral and outcome history by profile before
|
|
7
|
+
sorting or aggregating. These composite indexes keep those operations bounded
|
|
8
|
+
to one tenant as the shared database grows.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import sqlite3
|
|
14
|
+
|
|
15
|
+
NAME = "M029_behavioral_history_indexes"
|
|
16
|
+
DB_TARGET = "memory"
|
|
17
|
+
|
|
18
|
+
DDL = """
|
|
19
|
+
CREATE INDEX IF NOT EXISTS idx_outcomes_profile_outcome_time
|
|
20
|
+
ON action_outcomes(profile_id, outcome, timestamp DESC);
|
|
21
|
+
CREATE INDEX IF NOT EXISTS idx_outcomes_profile_settled_time
|
|
22
|
+
ON action_outcomes(profile_id, settled, settled_at DESC);
|
|
23
|
+
CREATE INDEX IF NOT EXISTS idx_outcomes_profile_settled_cursor
|
|
24
|
+
ON action_outcomes(
|
|
25
|
+
profile_id, settled, COALESCE(settled_at, ''), outcome_id
|
|
26
|
+
);
|
|
27
|
+
CREATE INDEX IF NOT EXISTS idx_assertions_profile_confidence
|
|
28
|
+
ON behavioral_assertions(profile_id, confidence DESC);
|
|
29
|
+
CREATE INDEX IF NOT EXISTS idx_tool_events_profile_created
|
|
30
|
+
ON tool_events(profile_id, created_at DESC);
|
|
31
|
+
CREATE INDEX IF NOT EXISTS idx_soft_prompts_profile_active_category
|
|
32
|
+
ON soft_prompt_templates(profile_id, active, category, prompt_id);
|
|
33
|
+
"""
|
|
34
|
+
|
|
35
|
+
_INDEXES: tuple[tuple[str, str, tuple[str, ...]], ...] = (
|
|
36
|
+
(
|
|
37
|
+
"idx_outcomes_profile_outcome_time",
|
|
38
|
+
"action_outcomes",
|
|
39
|
+
("profile_id", "outcome", "timestamp"),
|
|
40
|
+
),
|
|
41
|
+
(
|
|
42
|
+
"idx_outcomes_profile_settled_time",
|
|
43
|
+
"action_outcomes",
|
|
44
|
+
("profile_id", "settled", "settled_at"),
|
|
45
|
+
),
|
|
46
|
+
(
|
|
47
|
+
"idx_outcomes_profile_settled_cursor",
|
|
48
|
+
"action_outcomes",
|
|
49
|
+
("profile_id", "settled", "settled_at", "outcome_id"),
|
|
50
|
+
),
|
|
51
|
+
(
|
|
52
|
+
"idx_assertions_profile_confidence",
|
|
53
|
+
"behavioral_assertions",
|
|
54
|
+
("profile_id", "confidence"),
|
|
55
|
+
),
|
|
56
|
+
(
|
|
57
|
+
"idx_tool_events_profile_created",
|
|
58
|
+
"tool_events",
|
|
59
|
+
("profile_id", "created_at"),
|
|
60
|
+
),
|
|
61
|
+
(
|
|
62
|
+
"idx_soft_prompts_profile_active_category",
|
|
63
|
+
"soft_prompt_templates",
|
|
64
|
+
("profile_id", "active", "category", "prompt_id"),
|
|
65
|
+
),
|
|
66
|
+
)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _columns(conn: sqlite3.Connection, table: str) -> set[str]:
|
|
70
|
+
return {
|
|
71
|
+
str(row[1])
|
|
72
|
+
for row in conn.execute(f"PRAGMA table_info({table})").fetchall()
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _index_sql(name: str) -> str:
|
|
77
|
+
statements = {
|
|
78
|
+
"idx_outcomes_profile_outcome_time": (
|
|
79
|
+
"CREATE INDEX IF NOT EXISTS idx_outcomes_profile_outcome_time "
|
|
80
|
+
"ON action_outcomes(profile_id, outcome, timestamp DESC)"
|
|
81
|
+
),
|
|
82
|
+
"idx_outcomes_profile_settled_time": (
|
|
83
|
+
"CREATE INDEX IF NOT EXISTS idx_outcomes_profile_settled_time "
|
|
84
|
+
"ON action_outcomes(profile_id, settled, settled_at DESC)"
|
|
85
|
+
),
|
|
86
|
+
"idx_outcomes_profile_settled_cursor": (
|
|
87
|
+
"CREATE INDEX IF NOT EXISTS idx_outcomes_profile_settled_cursor "
|
|
88
|
+
"ON action_outcomes("
|
|
89
|
+
"profile_id, settled, COALESCE(settled_at, ''), outcome_id)"
|
|
90
|
+
),
|
|
91
|
+
"idx_assertions_profile_confidence": (
|
|
92
|
+
"CREATE INDEX IF NOT EXISTS idx_assertions_profile_confidence "
|
|
93
|
+
"ON behavioral_assertions(profile_id, confidence DESC)"
|
|
94
|
+
),
|
|
95
|
+
"idx_tool_events_profile_created": (
|
|
96
|
+
"CREATE INDEX IF NOT EXISTS idx_tool_events_profile_created "
|
|
97
|
+
"ON tool_events(profile_id, created_at DESC)"
|
|
98
|
+
),
|
|
99
|
+
"idx_soft_prompts_profile_active_category": (
|
|
100
|
+
"CREATE INDEX IF NOT EXISTS "
|
|
101
|
+
"idx_soft_prompts_profile_active_category "
|
|
102
|
+
"ON soft_prompt_templates(profile_id, active, category, prompt_id)"
|
|
103
|
+
),
|
|
104
|
+
}
|
|
105
|
+
return statements[name]
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def apply(conn: sqlite3.Connection) -> None:
|
|
109
|
+
"""Add each index supported by the installed runtime schema.
|
|
110
|
+
|
|
111
|
+
Optional v3.2/v3.4.7 tables also declare these indexes in their owning
|
|
112
|
+
schema modules, so a partial bootstrap cannot permanently miss an index
|
|
113
|
+
when those tables are created later.
|
|
114
|
+
"""
|
|
115
|
+
for name, table, required_columns in _INDEXES:
|
|
116
|
+
if set(required_columns) <= _columns(conn, table):
|
|
117
|
+
conn.execute(_index_sql(name))
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def verify(conn: sqlite3.Connection) -> bool:
|
|
121
|
+
"""Require each index whose backing table and columns are present."""
|
|
122
|
+
names = {
|
|
123
|
+
str(row[0])
|
|
124
|
+
for row in conn.execute(
|
|
125
|
+
"SELECT name FROM sqlite_master WHERE type='index'"
|
|
126
|
+
).fetchall()
|
|
127
|
+
}
|
|
128
|
+
return all(
|
|
129
|
+
name in names
|
|
130
|
+
for name, table, required_columns in _INDEXES
|
|
131
|
+
if set(required_columns) <= _columns(conn, table)
|
|
132
|
+
)
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def repair(conn: sqlite3.Connection) -> None:
|
|
136
|
+
"""Create only indexes supported by the current optional schema."""
|
|
137
|
+
apply(conn)
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
|
|
2
|
+
# Licensed under AGPL-3.0-or-later
|
|
3
|
+
|
|
4
|
+
"""M030 — page-first Entity Explorer indexes for large installations."""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import sqlite3
|
|
9
|
+
|
|
10
|
+
NAME = "M030_entity_explorer_indexes"
|
|
11
|
+
DB_TARGET = "memory"
|
|
12
|
+
|
|
13
|
+
DDL = """
|
|
14
|
+
CREATE INDEX IF NOT EXISTS idx_entities_profile_fact_count_id
|
|
15
|
+
ON canonical_entities(profile_id, fact_count DESC, entity_id ASC);
|
|
16
|
+
CREATE INDEX IF NOT EXISTS idx_entities_profile_type_fact_count_id
|
|
17
|
+
ON canonical_entities(
|
|
18
|
+
profile_id, entity_type COLLATE NOCASE, fact_count DESC, entity_id ASC
|
|
19
|
+
);
|
|
20
|
+
CREATE INDEX IF NOT EXISTS idx_entity_profiles_profile_entity_rank
|
|
21
|
+
ON entity_profiles(
|
|
22
|
+
profile_id, entity_id, last_compiled_at DESC,
|
|
23
|
+
project_name COLLATE NOCASE
|
|
24
|
+
);
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
_INDEXES: tuple[tuple[str, str, tuple[str, ...], str], ...] = (
|
|
28
|
+
(
|
|
29
|
+
"idx_entities_profile_fact_count_id",
|
|
30
|
+
"canonical_entities",
|
|
31
|
+
("profile_id", "fact_count", "entity_id"),
|
|
32
|
+
"CREATE INDEX IF NOT EXISTS idx_entities_profile_fact_count_id "
|
|
33
|
+
"ON canonical_entities(profile_id, fact_count DESC, entity_id ASC)",
|
|
34
|
+
),
|
|
35
|
+
(
|
|
36
|
+
"idx_entities_profile_type_fact_count_id",
|
|
37
|
+
"canonical_entities",
|
|
38
|
+
("profile_id", "entity_type", "fact_count", "entity_id"),
|
|
39
|
+
"CREATE INDEX IF NOT EXISTS idx_entities_profile_type_fact_count_id "
|
|
40
|
+
"ON canonical_entities("
|
|
41
|
+
"profile_id, entity_type COLLATE NOCASE, fact_count DESC, entity_id ASC"
|
|
42
|
+
")",
|
|
43
|
+
),
|
|
44
|
+
(
|
|
45
|
+
"idx_entity_profiles_profile_entity_rank",
|
|
46
|
+
"entity_profiles",
|
|
47
|
+
(
|
|
48
|
+
"profile_id",
|
|
49
|
+
"entity_id",
|
|
50
|
+
"last_compiled_at",
|
|
51
|
+
"project_name",
|
|
52
|
+
),
|
|
53
|
+
"CREATE INDEX IF NOT EXISTS idx_entity_profiles_profile_entity_rank "
|
|
54
|
+
"ON entity_profiles("
|
|
55
|
+
"profile_id, entity_id, last_compiled_at DESC, "
|
|
56
|
+
"project_name COLLATE NOCASE"
|
|
57
|
+
")",
|
|
58
|
+
),
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _columns(conn: sqlite3.Connection, table: str) -> set[str]:
|
|
63
|
+
return {
|
|
64
|
+
str(row[1])
|
|
65
|
+
for row in conn.execute(f"PRAGMA table_info({table})").fetchall()
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def apply(conn: sqlite3.Connection) -> None:
|
|
70
|
+
"""Create every index supported by the installed optional schema."""
|
|
71
|
+
for _, table, required_columns, sql in _INDEXES:
|
|
72
|
+
if set(required_columns) <= _columns(conn, table):
|
|
73
|
+
conn.execute(sql)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def verify(conn: sqlite3.Connection) -> bool:
|
|
77
|
+
"""Require indexes only when their complete backing schema exists."""
|
|
78
|
+
names = {
|
|
79
|
+
str(row[0])
|
|
80
|
+
for row in conn.execute(
|
|
81
|
+
"SELECT name FROM sqlite_master WHERE type='index'"
|
|
82
|
+
).fetchall()
|
|
83
|
+
}
|
|
84
|
+
return all(
|
|
85
|
+
name in names
|
|
86
|
+
for name, table, required_columns, _ in _INDEXES
|
|
87
|
+
if set(required_columns) <= _columns(conn, table)
|
|
88
|
+
)
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def repair(conn: sqlite3.Connection) -> None:
|
|
92
|
+
"""Safely restore a dropped index without replaying data migrations."""
|
|
93
|
+
apply(conn)
|
|
@@ -26,6 +26,8 @@ from . import (
|
|
|
26
26
|
M015_add_pinned_column,
|
|
27
27
|
M019_derivation_lineage,
|
|
28
28
|
M020_model_state_integrity,
|
|
29
|
+
M029_behavioral_history_indexes,
|
|
30
|
+
M030_entity_explorer_indexes,
|
|
29
31
|
)
|
|
30
32
|
|
|
31
33
|
# ---------------------------------------------------------------------------
|
|
@@ -77,6 +79,8 @@ __all__ = (
|
|
|
77
79
|
"M015_add_pinned_column",
|
|
78
80
|
"M019_derivation_lineage",
|
|
79
81
|
"M020_model_state_integrity",
|
|
82
|
+
"M029_behavioral_history_indexes",
|
|
83
|
+
"M030_entity_explorer_indexes",
|
|
80
84
|
# Legacy re-exports (backward compat):
|
|
81
85
|
"CURRENT_SCHEMA_VERSION",
|
|
82
86
|
"get_schema_version",
|
|
@@ -440,3 +440,7 @@ class RecallResponse:
|
|
|
440
440
|
# be mistaken for a reranked response.
|
|
441
441
|
reranker_applied: bool = False
|
|
442
442
|
reranker_status: str = "not_configured"
|
|
443
|
+
# Wave Q2b: precomputed community summary for the cluster the top results
|
|
444
|
+
# fall into (thematic context). None unless results cluster into one
|
|
445
|
+
# community above threshold. Additive — backward compatible.
|
|
446
|
+
community_context: dict | None = None
|
|
@@ -38,6 +38,8 @@ _TABLES: Final[tuple[str, ...]] = (
|
|
|
38
38
|
"memories",
|
|
39
39
|
"atomic_facts",
|
|
40
40
|
"canonical_entities",
|
|
41
|
+
"fact_entity_associations",
|
|
42
|
+
"fact_entity_association_repair_state",
|
|
41
43
|
"entity_aliases",
|
|
42
44
|
"entity_profiles",
|
|
43
45
|
"memory_scenes",
|
|
@@ -52,10 +54,14 @@ _TABLES: Final[tuple[str, ...]] = (
|
|
|
52
54
|
"compliance_audit",
|
|
53
55
|
"bm25_tokens",
|
|
54
56
|
"config",
|
|
57
|
+
"entity_communities",
|
|
58
|
+
"community_summaries",
|
|
59
|
+
"persona_summary",
|
|
55
60
|
)
|
|
56
61
|
|
|
57
62
|
_FTS_TABLES: Final[tuple[str, ...]] = (
|
|
58
63
|
"atomic_facts_fts",
|
|
64
|
+
"fact_expansion_fts",
|
|
59
65
|
)
|
|
60
66
|
|
|
61
67
|
|
|
@@ -281,6 +287,24 @@ END;
|
|
|
281
287
|
"""
|
|
282
288
|
|
|
283
289
|
|
|
290
|
+
# ---------------------------------------------------------------------------
|
|
291
|
+
# Fact expansion FTS (Phase 4, T3b — fact-augmented key expansion)
|
|
292
|
+
# ---------------------------------------------------------------------------
|
|
293
|
+
# Standalone (NOT external-content) FTS5 holding per-fact alternate keys
|
|
294
|
+
# (entity aliases + paraphrases). Kept separate from atomic_facts_fts so the
|
|
295
|
+
# proven content index and its triggers are never touched. Populated by
|
|
296
|
+
# core.key_expander on store (and backfilled), queried as an additive UNION in
|
|
297
|
+
# the BM25 channel. fact_id is stored so results can be scope-JOINed to
|
|
298
|
+
# atomic_facts.
|
|
299
|
+
_SQL_FACT_EXPANSION_FTS: Final[str] = """
|
|
300
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS fact_expansion_fts
|
|
301
|
+
USING fts5(
|
|
302
|
+
fact_id UNINDEXED,
|
|
303
|
+
alt_keys
|
|
304
|
+
);
|
|
305
|
+
"""
|
|
306
|
+
|
|
307
|
+
|
|
284
308
|
# ---------------------------------------------------------------------------
|
|
285
309
|
# Canonical entities
|
|
286
310
|
# ---------------------------------------------------------------------------
|
|
@@ -306,7 +330,52 @@ CREATE INDEX IF NOT EXISTS idx_entities_profile
|
|
|
306
330
|
CREATE INDEX IF NOT EXISTS idx_entities_name_lower
|
|
307
331
|
ON canonical_entities (profile_id, canonical_name COLLATE NOCASE);
|
|
308
332
|
CREATE INDEX IF NOT EXISTS idx_entities_type
|
|
309
|
-
ON canonical_entities (profile_id, entity_type);
|
|
333
|
+
ON canonical_entities (profile_id, entity_type);
|
|
334
|
+
CREATE INDEX IF NOT EXISTS idx_entities_profile_fact_count
|
|
335
|
+
ON canonical_entities (profile_id, fact_count DESC);
|
|
336
|
+
CREATE INDEX IF NOT EXISTS idx_entities_profile_type_fact_count
|
|
337
|
+
ON canonical_entities (
|
|
338
|
+
profile_id, entity_type COLLATE NOCASE, fact_count DESC
|
|
339
|
+
);"""
|
|
340
|
+
|
|
341
|
+
|
|
342
|
+
# ---------------------------------------------------------------------------
|
|
343
|
+
# Normalized fact/entity associations (also the ingestion effect ledger)
|
|
344
|
+
# ---------------------------------------------------------------------------
|
|
345
|
+
|
|
346
|
+
_SQL_FACT_ENTITY_ASSOCIATIONS: Final[str] = """
|
|
347
|
+
CREATE TABLE IF NOT EXISTS fact_entity_associations (
|
|
348
|
+
profile_id TEXT NOT NULL,
|
|
349
|
+
fact_id TEXT NOT NULL,
|
|
350
|
+
entity_id TEXT NOT NULL,
|
|
351
|
+
first_operation_id TEXT NOT NULL DEFAULT '',
|
|
352
|
+
count_applied INTEGER NOT NULL DEFAULT 0
|
|
353
|
+
CHECK (count_applied IN (0, 1)),
|
|
354
|
+
created_at TEXT NOT NULL DEFAULT (
|
|
355
|
+
strftime('%Y-%m-%dT%H:%M:%fZ', 'now')
|
|
356
|
+
),
|
|
357
|
+
PRIMARY KEY (profile_id, fact_id, entity_id),
|
|
358
|
+
FOREIGN KEY (fact_id) REFERENCES atomic_facts(fact_id) ON DELETE CASCADE,
|
|
359
|
+
FOREIGN KEY (entity_id)
|
|
360
|
+
REFERENCES canonical_entities(entity_id) ON DELETE CASCADE
|
|
361
|
+
);
|
|
362
|
+
CREATE INDEX IF NOT EXISTS idx_fact_entity_associations_entity
|
|
363
|
+
ON fact_entity_associations(profile_id, entity_id, fact_id);
|
|
364
|
+
CREATE TABLE IF NOT EXISTS fact_entity_association_repair_state (
|
|
365
|
+
repair_key TEXT PRIMARY KEY,
|
|
366
|
+
state TEXT NOT NULL DEFAULT 'pending'
|
|
367
|
+
CHECK (state IN ('pending', 'running', 'retrying', 'complete')),
|
|
368
|
+
target_fact_rowid INTEGER NOT NULL DEFAULT -1,
|
|
369
|
+
last_fact_rowid INTEGER NOT NULL DEFAULT 0,
|
|
370
|
+
scanned INTEGER NOT NULL DEFAULT 0,
|
|
371
|
+
inserted INTEGER NOT NULL DEFAULT 0,
|
|
372
|
+
last_error TEXT NOT NULL DEFAULT '',
|
|
373
|
+
updated_at TEXT NOT NULL
|
|
374
|
+
);
|
|
375
|
+
INSERT OR IGNORE INTO fact_entity_association_repair_state
|
|
376
|
+
(repair_key, state, target_fact_rowid, updated_at)
|
|
377
|
+
VALUES ('historical-backfill', 'pending', -1, '');
|
|
378
|
+
"""
|
|
310
379
|
|
|
311
380
|
|
|
312
381
|
# ---------------------------------------------------------------------------
|
|
@@ -316,6 +385,7 @@ CREATE INDEX IF NOT EXISTS idx_entities_type
|
|
|
316
385
|
_SQL_ENTITY_ALIASES: Final[str] = """
|
|
317
386
|
CREATE TABLE IF NOT EXISTS entity_aliases (
|
|
318
387
|
alias_id TEXT PRIMARY KEY,
|
|
388
|
+
profile_id TEXT NOT NULL DEFAULT 'default',
|
|
319
389
|
entity_id TEXT NOT NULL,
|
|
320
390
|
alias TEXT NOT NULL,
|
|
321
391
|
confidence REAL NOT NULL DEFAULT 1.0,
|
|
@@ -329,6 +399,10 @@ CREATE INDEX IF NOT EXISTS idx_aliases_entity
|
|
|
329
399
|
ON entity_aliases (entity_id);
|
|
330
400
|
CREATE INDEX IF NOT EXISTS idx_aliases_lookup
|
|
331
401
|
ON entity_aliases (alias COLLATE NOCASE);
|
|
402
|
+
-- NOTE: the (profile_id, entity_id) index is created by migration M022, NOT
|
|
403
|
+
-- here. On an upgrading DB this DDL runs at engine init BEFORE the deferred
|
|
404
|
+
-- M022 adds the profile_id column, so referencing it here would fail engine
|
|
405
|
+
-- init with "no such column: profile_id".
|
|
332
406
|
"""
|
|
333
407
|
|
|
334
408
|
|
|
@@ -663,6 +737,58 @@ CREATE TABLE IF NOT EXISTS config (
|
|
|
663
737
|
);
|
|
664
738
|
"""
|
|
665
739
|
|
|
740
|
+
# Wave Q: entity-community backbone (additive; safe on existing DBs).
|
|
741
|
+
# One row per (profile, entity) mapping to the community it belongs to,
|
|
742
|
+
# computed by Louvain over the entity co-occurrence graph in the background.
|
|
743
|
+
# Shared spine for Q2 community summaries and Q3 progressive abstraction.
|
|
744
|
+
_SQL_ENTITY_COMMUNITIES: Final[str] = """
|
|
745
|
+
CREATE TABLE IF NOT EXISTS entity_communities (
|
|
746
|
+
profile_id TEXT NOT NULL,
|
|
747
|
+
entity_id TEXT NOT NULL,
|
|
748
|
+
community_id INTEGER NOT NULL,
|
|
749
|
+
computed_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
750
|
+
PRIMARY KEY (profile_id, entity_id)
|
|
751
|
+
);
|
|
752
|
+
CREATE INDEX IF NOT EXISTS idx_entity_comm_profile
|
|
753
|
+
ON entity_communities(profile_id);
|
|
754
|
+
CREATE INDEX IF NOT EXISTS idx_entity_comm_cid
|
|
755
|
+
ON entity_communities(profile_id, community_id);
|
|
756
|
+
"""
|
|
757
|
+
|
|
758
|
+
# Wave Q2: one synthesized report per entity community (additive; safe on
|
|
759
|
+
# existing DBs). Generated in the background after entity_communities; the
|
|
760
|
+
# summary excludes superseded facts. member_fact_ids enables drill-down.
|
|
761
|
+
_SQL_COMMUNITY_SUMMARIES: Final[str] = """
|
|
762
|
+
CREATE TABLE IF NOT EXISTS community_summaries (
|
|
763
|
+
profile_id TEXT NOT NULL,
|
|
764
|
+
community_id INTEGER NOT NULL,
|
|
765
|
+
summary TEXT NOT NULL DEFAULT '',
|
|
766
|
+
keywords TEXT NOT NULL DEFAULT '',
|
|
767
|
+
entity_ids_json TEXT NOT NULL DEFAULT '[]',
|
|
768
|
+
fact_ids_json TEXT NOT NULL DEFAULT '[]',
|
|
769
|
+
fact_count INTEGER NOT NULL DEFAULT 0,
|
|
770
|
+
computed_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
771
|
+
PRIMARY KEY (profile_id, community_id)
|
|
772
|
+
);
|
|
773
|
+
CREATE INDEX IF NOT EXISTS idx_comm_summ_profile
|
|
774
|
+
ON community_summaries(profile_id);
|
|
775
|
+
"""
|
|
776
|
+
|
|
777
|
+
# Wave Q3: progressive-abstraction top tier — one persona roll-up per profile
|
|
778
|
+
# consuming the top community summaries (additive; safe on existing DBs).
|
|
779
|
+
# Recall-gated (never auto-injected into hot recall) and size-bounded to avoid
|
|
780
|
+
# the V3.4.40 summary-pollution regression. Drill-down: community_ids_json →
|
|
781
|
+
# communities → their member facts.
|
|
782
|
+
_SQL_PERSONA_SUMMARY: Final[str] = """
|
|
783
|
+
CREATE TABLE IF NOT EXISTS persona_summary (
|
|
784
|
+
profile_id TEXT PRIMARY KEY,
|
|
785
|
+
summary TEXT NOT NULL DEFAULT '',
|
|
786
|
+
keywords TEXT NOT NULL DEFAULT '',
|
|
787
|
+
community_ids_json TEXT NOT NULL DEFAULT '[]',
|
|
788
|
+
computed_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
789
|
+
);
|
|
790
|
+
"""
|
|
791
|
+
|
|
666
792
|
# ---------------------------------------------------------------------------
|
|
667
793
|
# Ordered DDL list (tables before FTS, respects FK order)
|
|
668
794
|
# ---------------------------------------------------------------------------
|
|
@@ -673,6 +799,7 @@ _DDL_ORDERED: Final[tuple[str, ...]] = (
|
|
|
673
799
|
_SQL_MEMORIES,
|
|
674
800
|
_SQL_ATOMIC_FACTS,
|
|
675
801
|
_SQL_CANONICAL_ENTITIES,
|
|
802
|
+
_SQL_FACT_ENTITY_ASSOCIATIONS,
|
|
676
803
|
_SQL_ENTITY_ALIASES,
|
|
677
804
|
_SQL_ENTITY_PROFILES,
|
|
678
805
|
_SQL_MEMORY_SCENES,
|
|
@@ -691,6 +818,14 @@ _DDL_ORDERED: Final[tuple[str, ...]] = (
|
|
|
691
818
|
_SQL_V2_MIGRATION_CLEANUP,
|
|
692
819
|
# FTS5 must come after atomic_facts (content table) AND after cleanup
|
|
693
820
|
_SQL_ATOMIC_FACTS_FTS,
|
|
821
|
+
# T3b: standalone expansion FTS (additive; safe on existing DBs)
|
|
822
|
+
_SQL_FACT_EXPANSION_FTS,
|
|
823
|
+
# Wave Q: entity-community backbone (additive; safe on existing DBs)
|
|
824
|
+
_SQL_ENTITY_COMMUNITIES,
|
|
825
|
+
# Wave Q2: community summaries (additive; safe on existing DBs)
|
|
826
|
+
_SQL_COMMUNITY_SUMMARIES,
|
|
827
|
+
# Wave Q3: persona roll-up tier (additive; safe on existing DBs)
|
|
828
|
+
_SQL_PERSONA_SUMMARY,
|
|
694
829
|
)
|
|
695
830
|
|
|
696
831
|
|
|
@@ -359,6 +359,8 @@ V32_DDL: list[str] = [
|
|
|
359
359
|
ON soft_prompt_templates(profile_id, active);
|
|
360
360
|
CREATE INDEX IF NOT EXISTS idx_soft_prompt_category
|
|
361
361
|
ON soft_prompt_templates(profile_id, category);
|
|
362
|
+
CREATE INDEX IF NOT EXISTS idx_soft_prompts_profile_active_category
|
|
363
|
+
ON soft_prompt_templates(profile_id, active, category, prompt_id);
|
|
362
364
|
CREATE UNIQUE INDEX IF NOT EXISTS idx_soft_prompt_unique_cat
|
|
363
365
|
ON soft_prompt_templates(profile_id, category)
|
|
364
366
|
WHERE active = 1;
|
|
@@ -43,7 +43,7 @@ V343_TABLES: Final[tuple[str, ...]] = (
|
|
|
43
43
|
# ---------------------------------------------------------------------------
|
|
44
44
|
|
|
45
45
|
_MESH_DDL = """
|
|
46
|
-
-- Mesh Peers
|
|
46
|
+
-- Mesh Peers (profile_id = tenant boundary; see M023)
|
|
47
47
|
CREATE TABLE IF NOT EXISTS mesh_peers (
|
|
48
48
|
peer_id TEXT PRIMARY KEY,
|
|
49
49
|
session_id TEXT NOT NULL,
|
|
@@ -52,7 +52,8 @@ CREATE TABLE IF NOT EXISTS mesh_peers (
|
|
|
52
52
|
host TEXT DEFAULT '127.0.0.1',
|
|
53
53
|
port INTEGER DEFAULT 0,
|
|
54
54
|
registered_at TEXT NOT NULL,
|
|
55
|
-
last_heartbeat TEXT NOT NULL
|
|
55
|
+
last_heartbeat TEXT NOT NULL,
|
|
56
|
+
profile_id TEXT NOT NULL DEFAULT 'default'
|
|
56
57
|
);
|
|
57
58
|
|
|
58
59
|
-- Mesh Messages
|
|
@@ -63,23 +64,28 @@ CREATE TABLE IF NOT EXISTS mesh_messages (
|
|
|
63
64
|
msg_type TEXT DEFAULT 'text',
|
|
64
65
|
content TEXT NOT NULL,
|
|
65
66
|
read INTEGER DEFAULT 0,
|
|
66
|
-
created_at TEXT NOT NULL
|
|
67
|
+
created_at TEXT NOT NULL,
|
|
68
|
+
profile_id TEXT NOT NULL DEFAULT 'default'
|
|
67
69
|
);
|
|
68
70
|
|
|
69
|
-
-- Mesh State (shared key-value store)
|
|
71
|
+
-- Mesh State (shared key-value store, per tenant)
|
|
70
72
|
CREATE TABLE IF NOT EXISTS mesh_state (
|
|
71
|
-
|
|
73
|
+
profile_id TEXT NOT NULL DEFAULT 'default',
|
|
74
|
+
key TEXT NOT NULL,
|
|
72
75
|
value TEXT NOT NULL,
|
|
73
76
|
set_by TEXT NOT NULL,
|
|
74
|
-
updated_at TEXT NOT NULL
|
|
77
|
+
updated_at TEXT NOT NULL,
|
|
78
|
+
PRIMARY KEY (profile_id, key)
|
|
75
79
|
);
|
|
76
80
|
|
|
77
|
-
-- Mesh Locks (file-level locks for coordination)
|
|
81
|
+
-- Mesh Locks (file-level locks for coordination, per tenant)
|
|
78
82
|
CREATE TABLE IF NOT EXISTS mesh_locks (
|
|
79
|
-
|
|
83
|
+
profile_id TEXT NOT NULL DEFAULT 'default',
|
|
84
|
+
file_path TEXT NOT NULL,
|
|
80
85
|
locked_by TEXT NOT NULL,
|
|
81
86
|
locked_at TEXT NOT NULL,
|
|
82
|
-
expires_at TEXT NOT NULL DEFAULT '9999-12-31T23:59:59Z'
|
|
87
|
+
expires_at TEXT NOT NULL DEFAULT '9999-12-31T23:59:59Z',
|
|
88
|
+
PRIMARY KEY (profile_id, file_path)
|
|
83
89
|
);
|
|
84
90
|
|
|
85
91
|
-- Mesh Events (audit log)
|
|
@@ -88,9 +94,14 @@ CREATE TABLE IF NOT EXISTS mesh_events (
|
|
|
88
94
|
event_type TEXT NOT NULL,
|
|
89
95
|
payload TEXT DEFAULT '{}',
|
|
90
96
|
emitted_by TEXT NOT NULL,
|
|
91
|
-
created_at TEXT NOT NULL
|
|
97
|
+
created_at TEXT NOT NULL,
|
|
98
|
+
profile_id TEXT NOT NULL DEFAULT 'default'
|
|
92
99
|
);
|
|
93
100
|
|
|
101
|
+
-- NOTE: profile-leading indexes live in migration M023, never here — on an
|
|
102
|
+
-- upgrade the profile_id column does not exist when this DDL runs (the old
|
|
103
|
+
-- table already exists, so CREATE TABLE IF NOT EXISTS no-ops), and a
|
|
104
|
+
-- CREATE INDEX on the missing column would abort engine init.
|
|
94
105
|
CREATE INDEX IF NOT EXISTS idx_mesh_messages_to
|
|
95
106
|
ON mesh_messages(to_peer, read);
|
|
96
107
|
CREATE INDEX IF NOT EXISTS idx_mesh_events_type
|
|
@@ -123,16 +134,17 @@ CREATE INDEX IF NOT EXISTS idx_entity_profiles_project
|
|
|
123
134
|
_INGESTION_DDL = """
|
|
124
135
|
CREATE TABLE IF NOT EXISTS ingestion_log (
|
|
125
136
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
137
|
+
profile_id TEXT NOT NULL DEFAULT 'default',
|
|
126
138
|
source_type TEXT NOT NULL,
|
|
127
139
|
dedup_key TEXT NOT NULL,
|
|
128
140
|
fact_ids TEXT DEFAULT '[]',
|
|
129
141
|
metadata TEXT DEFAULT '{}',
|
|
130
142
|
status TEXT DEFAULT 'ingested',
|
|
131
143
|
ingested_at TEXT NOT NULL,
|
|
132
|
-
UNIQUE(source_type, dedup_key)
|
|
144
|
+
UNIQUE(profile_id, source_type, dedup_key)
|
|
133
145
|
);
|
|
134
146
|
CREATE INDEX IF NOT EXISTS idx_ingestion_dedup
|
|
135
|
-
ON ingestion_log(source_type, dedup_key);
|
|
147
|
+
ON ingestion_log(profile_id, source_type, dedup_key);
|
|
136
148
|
"""
|
|
137
149
|
|
|
138
150
|
# ---------------------------------------------------------------------------
|
|
@@ -48,6 +48,8 @@ CREATE INDEX IF NOT EXISTS idx_tool_events_session
|
|
|
48
48
|
ON tool_events(session_id);
|
|
49
49
|
CREATE INDEX IF NOT EXISTS idx_tool_events_project
|
|
50
50
|
ON tool_events(project_path);
|
|
51
|
+
CREATE INDEX IF NOT EXISTS idx_tool_events_profile_created
|
|
52
|
+
ON tool_events(profile_id, created_at DESC);
|
|
51
53
|
"""
|
|
52
54
|
|
|
53
55
|
# ---------------------------------------------------------------------------
|
|
@@ -80,6 +82,8 @@ CREATE INDEX IF NOT EXISTS idx_assertions_project
|
|
|
80
82
|
ON behavioral_assertions(project_path, profile_id);
|
|
81
83
|
CREATE INDEX IF NOT EXISTS idx_assertions_category
|
|
82
84
|
ON behavioral_assertions(category);
|
|
85
|
+
CREATE INDEX IF NOT EXISTS idx_assertions_profile_confidence
|
|
86
|
+
ON behavioral_assertions(profile_id, confidence DESC);
|
|
83
87
|
"""
|
|
84
88
|
|
|
85
89
|
# ---------------------------------------------------------------------------
|
|
@@ -13,6 +13,7 @@ Part of Qualixar | Author: Varun Pratap Bhardwaj
|
|
|
13
13
|
from __future__ import annotations
|
|
14
14
|
|
|
15
15
|
import logging
|
|
16
|
+
import os
|
|
16
17
|
from typing import TYPE_CHECKING
|
|
17
18
|
|
|
18
19
|
if TYPE_CHECKING:
|
|
@@ -21,6 +22,26 @@ if TYPE_CHECKING:
|
|
|
21
22
|
logger = logging.getLogger(__name__)
|
|
22
23
|
|
|
23
24
|
|
|
25
|
+
def _env_threshold(name: str, default: float) -> float:
|
|
26
|
+
"""Read a [0,1] threshold from the environment, falling back on parse/range error."""
|
|
27
|
+
try:
|
|
28
|
+
value = float(os.environ.get(name, ""))
|
|
29
|
+
except (TypeError, ValueError):
|
|
30
|
+
return default
|
|
31
|
+
return value if 0.0 <= value <= 1.0 else default
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
# M-02/M-03 (3.7.9): thresholds are operator-tunable so multi-agent deployments
|
|
35
|
+
# can tighten the write gate and opt into a read gate. Defaults are unchanged —
|
|
36
|
+
# the audit's "raise write_threshold to 0.7" would block every new agent (a new
|
|
37
|
+
# agent scores 0.5, Beta(1,1)); loopback callers bypass the gate entirely, so
|
|
38
|
+
# this only affects remote credentialed agents. Read gate is OFF by default.
|
|
39
|
+
_DEFAULT_WRITE_THRESHOLD = _env_threshold("SLM_TRUST_WRITE_THRESHOLD", 0.3)
|
|
40
|
+
_DEFAULT_DELETE_THRESHOLD = _env_threshold("SLM_TRUST_DELETE_THRESHOLD", 0.5)
|
|
41
|
+
_DEFAULT_READ_THRESHOLD = _env_threshold("SLM_TRUST_READ_THRESHOLD", 0.1)
|
|
42
|
+
_READ_GATE_ENABLED = os.environ.get("SLM_TRUST_READ_GATE") == "1"
|
|
43
|
+
|
|
44
|
+
|
|
24
45
|
class TrustError(PermissionError):
|
|
25
46
|
"""Raised when an agent fails a trust check.
|
|
26
47
|
|
|
@@ -62,17 +83,23 @@ class TrustGate:
|
|
|
62
83
|
def __init__(
|
|
63
84
|
self,
|
|
64
85
|
scorer: TrustScorer,
|
|
65
|
-
write_threshold: float =
|
|
66
|
-
delete_threshold: float =
|
|
86
|
+
write_threshold: float = _DEFAULT_WRITE_THRESHOLD,
|
|
87
|
+
delete_threshold: float = _DEFAULT_DELETE_THRESHOLD,
|
|
88
|
+
read_threshold: float = _DEFAULT_READ_THRESHOLD,
|
|
89
|
+
read_gate_enabled: bool = _READ_GATE_ENABLED,
|
|
67
90
|
) -> None:
|
|
68
91
|
if write_threshold < 0 or write_threshold > 1:
|
|
69
92
|
raise ValueError("write_threshold must be in [0, 1]")
|
|
70
93
|
if delete_threshold < 0 or delete_threshold > 1:
|
|
71
94
|
raise ValueError("delete_threshold must be in [0, 1]")
|
|
95
|
+
if read_threshold < 0 or read_threshold > 1:
|
|
96
|
+
raise ValueError("read_threshold must be in [0, 1]")
|
|
72
97
|
|
|
73
98
|
self._scorer = scorer
|
|
74
99
|
self._write_threshold = write_threshold
|
|
75
100
|
self._delete_threshold = delete_threshold
|
|
101
|
+
self._read_threshold = read_threshold
|
|
102
|
+
self._read_gate_enabled = read_gate_enabled
|
|
76
103
|
|
|
77
104
|
@property
|
|
78
105
|
def write_threshold(self) -> float:
|
|
@@ -116,15 +143,29 @@ class TrustGate:
|
|
|
116
143
|
agent_id, score, self._delete_threshold, "delete"
|
|
117
144
|
)
|
|
118
145
|
|
|
146
|
+
@property
|
|
147
|
+
def read_threshold(self) -> float:
|
|
148
|
+
return self._read_threshold
|
|
149
|
+
|
|
150
|
+
@property
|
|
151
|
+
def read_gate_enabled(self) -> bool:
|
|
152
|
+
return self._read_gate_enabled
|
|
153
|
+
|
|
119
154
|
def check_read(self, agent_id: str, profile_id: str) -> None:
|
|
120
|
-
"""Read check
|
|
155
|
+
"""Read check. Passes by default; logged for the audit trail.
|
|
121
156
|
|
|
122
|
-
|
|
123
|
-
agent functionality.
|
|
124
|
-
|
|
157
|
+
M-03 (3.7.9): reads are unblocked by default because denying read access
|
|
158
|
+
could break agent functionality. Operators who need to stop a
|
|
159
|
+
compromised agent from exfiltrating the store can set
|
|
160
|
+
``SLM_TRUST_READ_GATE=1`` (optionally with ``SLM_TRUST_READ_THRESHOLD``)
|
|
161
|
+
to enforce a minimum trust for reads too.
|
|
125
162
|
"""
|
|
126
163
|
score = self._scorer.get_agent_trust(agent_id, profile_id)
|
|
127
164
|
logger.debug(
|
|
128
|
-
"trust gate read
|
|
129
|
-
agent_id, score,
|
|
165
|
+
"trust gate read: agent=%s trust=%.3f gate=%s",
|
|
166
|
+
agent_id, score, self._read_gate_enabled,
|
|
130
167
|
)
|
|
168
|
+
if self._read_gate_enabled and score < self._read_threshold:
|
|
169
|
+
raise TrustError(
|
|
170
|
+
agent_id, score, self._read_threshold, "read"
|
|
171
|
+
)
|