superlocalmemory 3.7.8 → 3.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/ATTRIBUTION.md +1 -3
- package/CHANGELOG.md +129 -0
- package/README.md +205 -123
- package/package.json +12 -3
- package/plugin/.claude-plugin/plugin.json +2 -3
- package/plugin/CLAUDE.md +8 -8
- package/plugin/agents/slm-governance-advisor.md +80 -0
- package/plugin/agents/slm-loop-runner.md +71 -0
- package/plugin/agents/slm-memory-advisor.md +10 -5
- package/plugin/agents/slm-optimize-advisor.md +9 -3
- package/plugin/commands/slm-loop.md +31 -0
- package/plugin/hooks/hooks.json +79 -0
- package/plugin/requirements.txt +1 -1
- package/plugin/scripts/slm-launch +46 -7
- package/plugin/settings.json +9 -0
- package/plugin/skills/slm-cache/SKILL.md +9 -1
- package/plugin/skills/slm-compress/SKILL.md +8 -1
- package/plugin/skills/slm-governance/SKILL.md +248 -0
- package/plugin/skills/slm-graph/SKILL.md +17 -3
- package/plugin/skills/slm-loop/SKILL.md +99 -0
- package/plugin/skills/slm-mesh/SKILL.md +282 -0
- package/plugin/skills/slm-profile/SKILL.md +148 -0
- package/plugin/skills/slm-recall/SKILL.md +46 -10
- package/plugin/skills/slm-remember/SKILL.md +48 -1
- package/plugin/skills/slm-scope/SKILL.md +176 -0
- package/plugin/skills/slm-session/SKILL.md +24 -1
- package/plugin/skills/slm-status/SKILL.md +18 -1
- package/plugin-src/rules/AGENTS.md +57 -18
- package/plugin-src/skills/slm-cache/SKILL.md +9 -1
- package/plugin-src/skills/slm-compress/SKILL.md +8 -1
- package/plugin-src/skills/slm-graph/SKILL.md +17 -3
- package/plugin-src/skills/slm-recall/SKILL.md +46 -10
- package/plugin-src/skills/slm-remember/SKILL.md +48 -1
- package/plugin-src/skills/slm-session/SKILL.md +24 -1
- package/plugin-src/skills/slm-status/SKILL.md +18 -1
- package/pyproject.toml +2 -1
- package/scripts/postinstall/validation.js +2 -0
- package/scripts/postinstall-interactive.js +74 -2
- package/src/superlocalmemory/__init__.py +1 -1
- package/src/superlocalmemory/access/__init__.py +3 -0
- package/src/superlocalmemory/access/rbac.py +477 -0
- package/src/superlocalmemory/cli/commands.py +228 -17
- package/src/superlocalmemory/cli/compress_cmd.py +17 -7
- package/src/superlocalmemory/cli/daemon.py +7 -0
- package/src/superlocalmemory/cli/loop_cmd.py +187 -0
- package/src/superlocalmemory/cli/main.py +49 -8
- package/src/superlocalmemory/cli/mesh_cmd.py +38 -0
- package/src/superlocalmemory/cli/optimize_cmd.py +3 -0
- package/src/superlocalmemory/cli/pending_store.py +49 -13
- package/src/superlocalmemory/cli/proxy_cmd.py +4 -0
- package/src/superlocalmemory/cli/scale_engine_cmd.py +6 -0
- package/src/superlocalmemory/cli/setup_wizard.py +22 -13
- package/src/superlocalmemory/cli/version_banner.py +17 -3
- package/src/superlocalmemory/compliance/audit.py +6 -0
- package/src/superlocalmemory/compliance/gdpr.py +128 -138
- package/src/superlocalmemory/compliance/retention.py +176 -45
- package/src/superlocalmemory/core/backend_orchestrator.py +23 -59
- package/src/superlocalmemory/core/community_summary.py +267 -0
- package/src/superlocalmemory/core/config.py +216 -3
- package/src/superlocalmemory/core/consolidation_engine.py +95 -22
- package/src/superlocalmemory/core/context_cache.py +61 -18
- package/src/superlocalmemory/core/embedding_worker.py +21 -7
- package/src/superlocalmemory/core/embeddings.py +131 -46
- package/src/superlocalmemory/core/engine.py +41 -22
- package/src/superlocalmemory/core/engine_ingestion.py +359 -43
- package/src/superlocalmemory/core/engine_wiring.py +13 -0
- package/src/superlocalmemory/core/entity_community.py +178 -0
- package/src/superlocalmemory/core/graph_analyzer.py +39 -2
- package/src/superlocalmemory/core/graph_pruner.py +13 -8
- package/src/superlocalmemory/core/ingestion_command.py +134 -25
- package/src/superlocalmemory/core/injection.py +12 -7
- package/src/superlocalmemory/core/key_expander.py +138 -0
- package/src/superlocalmemory/core/maintenance.py +23 -0
- package/src/superlocalmemory/core/maintenance_scheduler.py +17 -7
- package/src/superlocalmemory/core/modes.py +1 -1
- package/src/superlocalmemory/core/mutations.py +2 -2
- package/src/superlocalmemory/core/pii.py +105 -0
- package/src/superlocalmemory/core/progressive_abstraction.py +208 -0
- package/src/superlocalmemory/core/recall_pipeline.py +7 -3
- package/src/superlocalmemory/core/recall_worker.py +20 -6
- package/src/superlocalmemory/core/scale_engine.py +60 -1
- package/src/superlocalmemory/core/security_primitives.py +40 -2
- package/src/superlocalmemory/core/store_pipeline.py +186 -29
- package/src/superlocalmemory/core/worker_pool.py +21 -6
- package/src/superlocalmemory/encoding/entity_reflexion.py +200 -0
- package/src/superlocalmemory/encoding/entity_resolver.py +34 -24
- package/src/superlocalmemory/encoding/fact_extractor.py +26 -1
- package/src/superlocalmemory/encoding/temporal_validator.py +64 -1
- package/src/superlocalmemory/evolution/evolution_store.py +122 -45
- package/src/superlocalmemory/evolution/llm_dispatch.py +12 -1
- package/src/superlocalmemory/evolution/model_selection.py +160 -0
- package/src/superlocalmemory/evolution/mutation_generator.py +16 -0
- package/src/superlocalmemory/evolution/skill_evolver.py +127 -42
- package/src/superlocalmemory/evolution/triggers.py +22 -13
- package/src/superlocalmemory/graph/cozo_backend.py +43 -20
- package/src/superlocalmemory/hooks/adapter_base.py +5 -1
- package/src/superlocalmemory/hooks/auto_recall.py +13 -1
- package/src/superlocalmemory/hooks/claude_code_hooks.py +11 -0
- package/src/superlocalmemory/hooks/codex_assets.py +64 -5
- package/src/superlocalmemory/hooks/hook_daemon.py +20 -3
- package/src/superlocalmemory/hooks/hook_handlers.py +6 -1
- package/src/superlocalmemory/hooks/memory_protocol.py +54 -0
- package/src/superlocalmemory/hooks/portable_kit.py +148 -3
- package/src/superlocalmemory/infra/backup.py +12 -1
- package/src/superlocalmemory/infra/daemon_identity.py +40 -4
- package/src/superlocalmemory/infra/data_root.py +43 -4
- package/src/superlocalmemory/infra/event_bus.py +107 -24
- package/src/superlocalmemory/infra/rate_limiter.py +93 -0
- package/src/superlocalmemory/ingestion/adapter_manager.py +4 -1
- package/src/superlocalmemory/ingestion/credentials.py +1 -1
- package/src/superlocalmemory/learning/cross_project.py +28 -19
- package/src/superlocalmemory/learning/model_rollback.py +3 -0
- package/src/superlocalmemory/learning/ranker_retrain_online.py +2 -0
- package/src/superlocalmemory/learning/reward.py +50 -0
- package/src/superlocalmemory/learning/reward_proxy.py +42 -9
- package/src/superlocalmemory/learning/source_quality.py +523 -1
- package/src/superlocalmemory/loops/__init__.py +56 -0
- package/src/superlocalmemory/loops/budget.py +58 -0
- package/src/superlocalmemory/loops/engine.py +164 -0
- package/src/superlocalmemory/loops/ledger.py +263 -0
- package/src/superlocalmemory/loops/models.py +152 -0
- package/src/superlocalmemory/loops/rules.py +52 -0
- package/src/superlocalmemory/mcp/_daemon_proxy.py +3 -0
- package/src/superlocalmemory/mcp/_pool_adapter.py +2 -0
- package/src/superlocalmemory/mcp/profiles.py +103 -0
- package/src/superlocalmemory/mcp/server.py +32 -79
- package/src/superlocalmemory/mcp/tools_active.py +4 -7
- package/src/superlocalmemory/mcp/tools_code_graph.py +51 -5
- package/src/superlocalmemory/mcp/tools_core.py +12 -4
- package/src/superlocalmemory/mcp/tools_evolution.py +6 -3
- package/src/superlocalmemory/mcp/tools_learning.py +2 -2
- package/src/superlocalmemory/mcp/tools_loops.py +300 -0
- package/src/superlocalmemory/mcp/tools_mesh.py +140 -4
- package/src/superlocalmemory/mcp/tools_optimize.py +15 -8
- package/src/superlocalmemory/mesh/broker.py +237 -129
- package/src/superlocalmemory/mesh/remote_sync.py +50 -8
- package/src/superlocalmemory/optimize/NOTICE +1 -6
- package/src/superlocalmemory/optimize/adapters/anthropic_adapter.py +1 -4
- package/src/superlocalmemory/optimize/adapters/openai_adapter.py +1 -4
- package/src/superlocalmemory/optimize/cache/semantic.py +27 -19
- package/src/superlocalmemory/optimize/compress/align.py +32 -26
- package/src/superlocalmemory/optimize/compress/ccr.py +14 -71
- package/src/superlocalmemory/optimize/compress/router.py +105 -22
- package/src/superlocalmemory/optimize/config/defaults.py +1 -1
- package/src/superlocalmemory/optimize/config/schema.py +87 -4
- package/src/superlocalmemory/optimize/metrics/counters.py +13 -4
- package/src/superlocalmemory/optimize/metrics/estimator.py +0 -3
- package/src/superlocalmemory/optimize/proxy/_helpers.py +31 -4
- package/src/superlocalmemory/optimize/storage/db.py +38 -9
- package/src/superlocalmemory/optimize/storage/schema.py +10 -0
- package/src/superlocalmemory/parameterization/pattern_extractor.py +6 -3
- package/src/superlocalmemory/retrieval/agentic.py +1 -1
- package/src/superlocalmemory/retrieval/bm25_channel.py +68 -10
- package/src/superlocalmemory/retrieval/engine.py +221 -47
- package/src/superlocalmemory/retrieval/entity_channel.py +7 -5
- package/src/superlocalmemory/retrieval/hopfield_channel.py +9 -2
- package/src/superlocalmemory/retrieval/reranker.py +3 -4
- package/src/superlocalmemory/retrieval/semantic_channel.py +114 -21
- package/src/superlocalmemory/retrieval/spreading_activation.py +11 -2
- package/src/superlocalmemory/retrieval/temporal_channel.py +48 -9
- package/src/superlocalmemory/retrieval/temporal_frame.py +102 -0
- package/src/superlocalmemory/retrieval/temporal_validity_filter.py +135 -0
- package/src/superlocalmemory/retrieval/time_window.py +181 -0
- package/src/superlocalmemory/server/api.py +4 -4
- package/src/superlocalmemory/server/config_file.py +90 -0
- package/src/superlocalmemory/server/origin.py +50 -0
- package/src/superlocalmemory/server/profile_runtime.py +125 -8
- package/src/superlocalmemory/server/rbac_enforce.py +142 -0
- package/src/superlocalmemory/server/recall_health.py +24 -3
- package/src/superlocalmemory/server/recall_serializer.py +19 -1
- package/src/superlocalmemory/server/routes/abstraction.py +115 -0
- package/src/superlocalmemory/server/routes/agents.py +128 -38
- package/src/superlocalmemory/server/routes/backup.py +317 -70
- package/src/superlocalmemory/server/routes/behavioral.py +349 -71
- package/src/superlocalmemory/server/routes/brain.py +69 -12
- package/src/superlocalmemory/server/routes/chat.py +10 -5
- package/src/superlocalmemory/server/routes/compliance.py +171 -21
- package/src/superlocalmemory/server/routes/config_api.py +438 -0
- package/src/superlocalmemory/server/routes/data_io.py +30 -8
- package/src/superlocalmemory/server/routes/entity.py +108 -26
- package/src/superlocalmemory/server/routes/events.py +24 -8
- package/src/superlocalmemory/server/routes/evolution.py +189 -68
- package/src/superlocalmemory/server/routes/helpers.py +16 -1
- package/src/superlocalmemory/server/routes/ingest.py +7 -4
- package/src/superlocalmemory/server/routes/insights.py +3 -3
- package/src/superlocalmemory/server/routes/learning.py +289 -118
- package/src/superlocalmemory/server/routes/learning_telemetry.py +153 -0
- package/src/superlocalmemory/server/routes/lifecycle.py +59 -8
- package/src/superlocalmemory/server/routes/memories.py +182 -57
- package/src/superlocalmemory/server/routes/mesh.py +200 -31
- package/src/superlocalmemory/server/routes/optimize.py +33 -1
- package/src/superlocalmemory/server/routes/prewarm.py +2 -0
- package/src/superlocalmemory/server/routes/profiles.py +63 -17
- package/src/superlocalmemory/server/routes/ratelimit.py +132 -0
- package/src/superlocalmemory/server/routes/rbac.py +367 -0
- package/src/superlocalmemory/server/routes/stats.py +103 -158
- package/src/superlocalmemory/server/routes/tiers.py +11 -9
- package/src/superlocalmemory/server/routes/token.py +3 -13
- package/src/superlocalmemory/server/routes/v3_api.py +247 -89
- package/src/superlocalmemory/server/routes/ws.py +5 -2
- package/src/superlocalmemory/server/security_middleware.py +12 -5
- package/src/superlocalmemory/server/ui.py +20 -5
- package/src/superlocalmemory/server/unified_daemon.py +827 -72
- package/src/superlocalmemory/server/write_identity.py +38 -8
- package/src/superlocalmemory/storage/database.py +265 -53
- package/src/superlocalmemory/storage/migration_runner.py +132 -1
- package/src/superlocalmemory/storage/migrations/M010_evolution_config.py +5 -0
- package/src/superlocalmemory/storage/migrations/M021_ingestion_log_profile.py +108 -0
- package/src/superlocalmemory/storage/migrations/M022_entity_aliases_profile.py +86 -0
- package/src/superlocalmemory/storage/migrations/M023_mesh_profile_isolation.py +194 -0
- package/src/superlocalmemory/storage/migrations/M024_rbac_users_roles.py +87 -0
- package/src/superlocalmemory/storage/migrations/M025_perf_indexes.py +90 -0
- package/src/superlocalmemory/storage/migrations/M026_rbac_memberships_fk.py +136 -0
- package/src/superlocalmemory/storage/migrations/M027_transferable_patterns_profile.py +163 -0
- package/src/superlocalmemory/storage/migrations/M028_fact_entity_associations.py +270 -0
- package/src/superlocalmemory/storage/migrations/M029_behavioral_history_indexes.py +137 -0
- package/src/superlocalmemory/storage/migrations/M030_entity_explorer_indexes.py +93 -0
- package/src/superlocalmemory/storage/migrations/__init__.py +4 -0
- package/src/superlocalmemory/storage/models.py +4 -0
- package/src/superlocalmemory/storage/schema.py +136 -1
- package/src/superlocalmemory/storage/schema_v32.py +2 -0
- package/src/superlocalmemory/storage/schema_v343.py +24 -12
- package/src/superlocalmemory/storage/schema_v347.py +4 -0
- package/src/superlocalmemory/trust/gate.py +49 -8
- package/src/superlocalmemory/ui/assets/slm-icon-white.svg +64 -0
- package/src/superlocalmemory/ui/assets/slm-icon.svg +36 -0
- package/src/superlocalmemory/ui/css/design-system.css +621 -0
- package/src/superlocalmemory/ui/css/neural-glass.css +6 -0
- package/src/superlocalmemory/ui/css/od-bridge.css +158 -0
- package/src/superlocalmemory/ui/favicon.svg +35 -4
- package/src/superlocalmemory/ui/index.html +303 -173
- package/src/superlocalmemory/ui/js/brain.js +5 -20
- package/src/superlocalmemory/ui/js/core.js +100 -41
- package/src/superlocalmemory/ui/js/dashboard.js +403 -65
- package/src/superlocalmemory/ui/js/event-delegation.js +102 -0
- package/src/superlocalmemory/ui/js/knowledge-graph.js +11 -11
- package/src/superlocalmemory/ui/js/math-health.js +1 -1
- package/src/superlocalmemory/ui/js/memories.js +15 -4
- package/src/superlocalmemory/ui/js/memory-chat.js +7 -7
- package/src/superlocalmemory/ui/js/ng-entities.js +6 -8
- package/src/superlocalmemory/ui/js/ng-ingestion.js +4 -4
- package/src/superlocalmemory/ui/js/ng-mesh.js +4 -9
- package/src/superlocalmemory/ui/js/ng-shell.js +8 -8
- package/src/superlocalmemory/ui/js/ng-skills.js +54 -2
- package/src/superlocalmemory/ui/js/od-agents.js +544 -0
- package/src/superlocalmemory/ui/js/od-auth-gate.js +257 -0
- package/src/superlocalmemory/ui/js/od-backup.js +871 -0
- package/src/superlocalmemory/ui/js/od-brain.js +816 -0
- package/src/superlocalmemory/ui/js/od-entities.js +579 -0
- package/src/superlocalmemory/ui/js/od-graph.js +600 -0
- package/src/superlocalmemory/ui/js/od-health.js +539 -0
- package/src/superlocalmemory/ui/js/od-mcp.js +508 -0
- package/src/superlocalmemory/ui/js/od-memories.js +929 -0
- package/src/superlocalmemory/ui/js/od-mesh.js +553 -0
- package/src/superlocalmemory/ui/js/od-operations.js +1250 -0
- package/src/superlocalmemory/ui/js/od-optimize.js +787 -0
- package/src/superlocalmemory/ui/js/od-settings.js +1107 -0
- package/src/superlocalmemory/ui/js/od-shell.js +809 -0
- package/src/superlocalmemory/ui/js/od-skills.js +600 -0
- package/src/superlocalmemory/ui/js/od-team.js +258 -0
- package/src/superlocalmemory/ui/js/profiles.js +159 -46
- package/src/superlocalmemory/ui/js/settings.js +17 -3
- package/src/superlocalmemory/ui/js/timeline.js +34 -5
- package/src/superlocalmemory/ui/js/trust-dashboard.js +2 -2
- package/src/superlocalmemory/vector/lancedb_backend.py +8 -6
- package/plugin-src/.mcp.json +0 -12
- package/plugin-src/agents/slm-memory-advisor.md +0 -44
- package/plugin-src/agents/slm-optimize-advisor.md +0 -38
- package/plugin-src/hooks/.gitkeep +0 -0
- package/plugin-src/hooks/hooks.json +0 -23
- package/plugin-src/manifest.json +0 -25
- package/plugin-src/requirements.txt +0 -1
- package/plugin-src/rules/CLAUDE.md.fragment +0 -44
- package/plugin-src/scripts/ensure-venv.bat +0 -122
- package/plugin-src/scripts/ensure-venv.sh +0 -105
- package/plugin-src/scripts/slm-launch +0 -23
- package/plugin-src/scripts/slm-launch.bat +0 -23
- package/plugin-src/settings.json +0 -16
- package/src/superlocalmemory/learning/behavioral_listener.py +0 -94
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
|
|
2
|
+
# Licensed under AGPL-3.0-or-later - see LICENSE file
|
|
3
|
+
# Part of SuperLocalMemory V3 — performance indexes
|
|
4
|
+
|
|
5
|
+
"""M025 — hot-path indexes confirmed missing by EXPLAIN QUERY PLAN.
|
|
6
|
+
|
|
7
|
+
* atomic_facts(profile_id, content) — the store_fact dedup check runs on every
|
|
8
|
+
ingest; without this it scans all profile facts for a lifecycle match.
|
|
9
|
+
* mesh_events(created_at) — the 7-day cleanup range-scanned the whole table.
|
|
10
|
+
* mesh_peers(status, last_heartbeat) — stale-peer cleanup full-scanned.
|
|
11
|
+
* mesh_peers(profile_id, last_heartbeat) — list_peers sorted in a temp b-tree.
|
|
12
|
+
|
|
13
|
+
All additive CREATE INDEX IF NOT EXISTS — idempotent, no data change. Deferred
|
|
14
|
+
so the target tables (mesh_* via schema_v343, atomic_facts via engine init)
|
|
15
|
+
exist first.
|
|
16
|
+
|
|
17
|
+
Author: Varun Pratap Bhardwaj / Qualixar
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
import sqlite3
|
|
23
|
+
|
|
24
|
+
NAME = "M025_perf_indexes"
|
|
25
|
+
DB_TARGET = "memory"
|
|
26
|
+
|
|
27
|
+
DDL = (
|
|
28
|
+
"CREATE INDEX IF NOT EXISTS idx_facts_content_dedup "
|
|
29
|
+
"ON atomic_facts(profile_id, content);"
|
|
30
|
+
"CREATE INDEX IF NOT EXISTS idx_mesh_events_created_at ON mesh_events(created_at);"
|
|
31
|
+
"CREATE INDEX IF NOT EXISTS idx_mesh_peers_status_heartbeat "
|
|
32
|
+
"ON mesh_peers(status, last_heartbeat);"
|
|
33
|
+
"CREATE INDEX IF NOT EXISTS idx_mesh_peers_profile_heartbeat "
|
|
34
|
+
"ON mesh_peers(profile_id, last_heartbeat);"
|
|
35
|
+
)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _table_exists(conn: sqlite3.Connection, table: str) -> bool:
|
|
39
|
+
return conn.execute(
|
|
40
|
+
"SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (table,)
|
|
41
|
+
).fetchone() is not None
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _cols(conn: sqlite3.Connection, table: str) -> set[str]:
|
|
45
|
+
return {r[1] for r in conn.execute(f"PRAGMA table_info({table})").fetchall()}
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def apply(conn: sqlite3.Connection) -> None:
|
|
49
|
+
"""Create the perf indexes, each guarded so a non-standard schema cannot
|
|
50
|
+
abort the migration (and leave verify() falsely reporting done)."""
|
|
51
|
+
if _table_exists(conn, "atomic_facts") and {"profile_id", "content"} <= _cols(conn, "atomic_facts"):
|
|
52
|
+
conn.execute(
|
|
53
|
+
"CREATE INDEX IF NOT EXISTS idx_facts_content_dedup "
|
|
54
|
+
"ON atomic_facts(profile_id, content)"
|
|
55
|
+
)
|
|
56
|
+
if _table_exists(conn, "mesh_events") and "created_at" in _cols(conn, "mesh_events"):
|
|
57
|
+
conn.execute(
|
|
58
|
+
"CREATE INDEX IF NOT EXISTS idx_mesh_events_created_at "
|
|
59
|
+
"ON mesh_events(created_at)"
|
|
60
|
+
)
|
|
61
|
+
if _table_exists(conn, "mesh_peers"):
|
|
62
|
+
pc = _cols(conn, "mesh_peers")
|
|
63
|
+
if {"status", "last_heartbeat"} <= pc:
|
|
64
|
+
conn.execute(
|
|
65
|
+
"CREATE INDEX IF NOT EXISTS idx_mesh_peers_status_heartbeat "
|
|
66
|
+
"ON mesh_peers(status, last_heartbeat)"
|
|
67
|
+
)
|
|
68
|
+
if {"profile_id", "last_heartbeat"} <= pc:
|
|
69
|
+
conn.execute(
|
|
70
|
+
"CREATE INDEX IF NOT EXISTS idx_mesh_peers_profile_heartbeat "
|
|
71
|
+
"ON mesh_peers(profile_id, last_heartbeat)"
|
|
72
|
+
)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def verify(conn: sqlite3.Connection) -> bool:
|
|
76
|
+
"""Applied once the two always-creatable indexes exist (mesh_peers/events
|
|
77
|
+
are always present via schema_v343; atomic_facts always via engine init)."""
|
|
78
|
+
names = {
|
|
79
|
+
r[0] for r in conn.execute(
|
|
80
|
+
"SELECT name FROM sqlite_master WHERE type='index'"
|
|
81
|
+
).fetchall()
|
|
82
|
+
}
|
|
83
|
+
required = {"idx_facts_content_dedup", "idx_mesh_events_created_at"}
|
|
84
|
+
# Only require indexes whose tables exist (fresh vs partial installs).
|
|
85
|
+
ok = True
|
|
86
|
+
if _table_exists(conn, "atomic_facts"):
|
|
87
|
+
ok = ok and "idx_facts_content_dedup" in names
|
|
88
|
+
if _table_exists(conn, "mesh_events"):
|
|
89
|
+
ok = ok and "idx_mesh_events_created_at" in names
|
|
90
|
+
return ok
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
|
|
2
|
+
# Licensed under AGPL-3.0-or-later - see LICENSE file
|
|
3
|
+
# Part of SuperLocalMemory V3 — RBAC hardening (SEC-H-01, audit-02)
|
|
4
|
+
|
|
5
|
+
"""M026 — add FK on rbac_memberships → profiles ON DELETE CASCADE (memory.db).
|
|
6
|
+
|
|
7
|
+
SEC-H-01: ``rbac_memberships`` (created by M024) had no foreign key to
|
|
8
|
+
``profiles``. Deleting a profile therefore left its role grants behind; a
|
|
9
|
+
profile later recreated with the same ``profile_id`` silently re-inherited
|
|
10
|
+
them — a privilege-persistence bug.
|
|
11
|
+
|
|
12
|
+
The runtime path (``delete_profile_from_db``) already purges memberships
|
|
13
|
+
explicitly. This migration is the *defense-in-depth* half: a real FK with
|
|
14
|
+
``ON DELETE CASCADE`` so that ANY path deleting a ``profiles`` row — not just
|
|
15
|
+
that one helper — cannot leave orphaned grants.
|
|
16
|
+
|
|
17
|
+
SQLite cannot add a constraint in place, so the table is rebuilt (the
|
|
18
|
+
recommended safe-alter dance): rename → create canonical table with the FK →
|
|
19
|
+
copy rows → drop old → recreate the user index. Wrapped in BEGIN IMMEDIATE /
|
|
20
|
+
COMMIT for atomicity; the runner connects with ``foreign_keys = OFF`` so the
|
|
21
|
+
rename/copy/drop never trips a constraint action mid-rebuild.
|
|
22
|
+
|
|
23
|
+
Idempotent: skips when the FK is already present (fresh installs still get the
|
|
24
|
+
FK because this always rebuilds M024's FK-less table on first run). Tolerant of
|
|
25
|
+
a missing table (nothing to migrate).
|
|
26
|
+
|
|
27
|
+
Deferred because the FK target ``profiles`` is created at engine init
|
|
28
|
+
(``storage.schema``), after ``apply_all`` runs — same ordering constraint as
|
|
29
|
+
M021 / M023.
|
|
30
|
+
|
|
31
|
+
Author: Varun Pratap Bhardwaj / Qualixar
|
|
32
|
+
"""
|
|
33
|
+
|
|
34
|
+
from __future__ import annotations
|
|
35
|
+
|
|
36
|
+
import sqlite3
|
|
37
|
+
|
|
38
|
+
NAME = "M026_rbac_memberships_fk"
|
|
39
|
+
DB_TARGET = "memory"
|
|
40
|
+
|
|
41
|
+
# Documentation + drift-hash fingerprint. apply() below is the authoritative
|
|
42
|
+
# executor (a static DDL string cannot express the conditional rebuild).
|
|
43
|
+
DDL = (
|
|
44
|
+
"-- rebuild rbac_memberships with "
|
|
45
|
+
"FOREIGN KEY (profile_id) REFERENCES profiles(profile_id) ON DELETE CASCADE"
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
# Canonical rebuilt table. Column set is byte-identical to M024's, plus the FK.
|
|
49
|
+
# The index is created separately AFTER the old table is dropped: RENAME keeps
|
|
50
|
+
# the old index's name, so an inline/early CREATE INDEX would collide.
|
|
51
|
+
_NEW_TABLE = """
|
|
52
|
+
CREATE TABLE rbac_memberships (
|
|
53
|
+
profile_id TEXT NOT NULL,
|
|
54
|
+
user_id TEXT NOT NULL,
|
|
55
|
+
role TEXT NOT NULL,
|
|
56
|
+
added_at TEXT NOT NULL,
|
|
57
|
+
added_by TEXT DEFAULT 'owner',
|
|
58
|
+
PRIMARY KEY (profile_id, user_id),
|
|
59
|
+
FOREIGN KEY (profile_id) REFERENCES profiles(profile_id) ON DELETE CASCADE
|
|
60
|
+
)
|
|
61
|
+
"""
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _table_exists(conn: sqlite3.Connection, table: str) -> bool:
|
|
65
|
+
return conn.execute(
|
|
66
|
+
"SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (table,)
|
|
67
|
+
).fetchone() is not None
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def _cols(conn: sqlite3.Connection, table: str) -> set[str]:
|
|
71
|
+
return {r[1] for r in conn.execute(f"PRAGMA table_info({table})").fetchall()}
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _has_profiles_fk(conn: sqlite3.Connection, table: str) -> bool:
|
|
75
|
+
"""True when ``table`` already declares a foreign key onto ``profiles``."""
|
|
76
|
+
try:
|
|
77
|
+
rows = conn.execute(f"PRAGMA foreign_key_list({table})").fetchall()
|
|
78
|
+
except sqlite3.OperationalError: # pragma: no cover — table absent
|
|
79
|
+
return False
|
|
80
|
+
# PRAGMA foreign_key_list row layout: (id, seq, table, from, to, ...).
|
|
81
|
+
return any(r[2] == "profiles" for r in rows)
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def apply(conn: sqlite3.Connection) -> None:
|
|
85
|
+
"""Rebuild rbac_memberships with the profiles FK.
|
|
86
|
+
|
|
87
|
+
No-op when the table is absent (nothing to migrate) or the FK already
|
|
88
|
+
exists (idempotent re-run). Existing grants are preserved verbatim.
|
|
89
|
+
"""
|
|
90
|
+
if not _table_exists(conn, "rbac_memberships"):
|
|
91
|
+
return
|
|
92
|
+
if _has_profiles_fk(conn, "rbac_memberships"):
|
|
93
|
+
return
|
|
94
|
+
|
|
95
|
+
has_added_by = "added_by" in _cols(conn, "rbac_memberships")
|
|
96
|
+
|
|
97
|
+
conn.execute("BEGIN IMMEDIATE")
|
|
98
|
+
try:
|
|
99
|
+
conn.execute("ALTER TABLE rbac_memberships RENAME TO _rbac_memberships_old")
|
|
100
|
+
# NB: single statement via execute(), not executescript() — the latter
|
|
101
|
+
# force-commits the pending transaction, breaking our BEGIN/COMMIT.
|
|
102
|
+
conn.execute(_NEW_TABLE)
|
|
103
|
+
if has_added_by:
|
|
104
|
+
conn.execute(
|
|
105
|
+
"INSERT INTO rbac_memberships "
|
|
106
|
+
"(profile_id, user_id, role, added_at, added_by) "
|
|
107
|
+
"SELECT profile_id, user_id, role, added_at, added_by "
|
|
108
|
+
"FROM _rbac_memberships_old"
|
|
109
|
+
)
|
|
110
|
+
else: # exotic/older install missing the column — default it
|
|
111
|
+
conn.execute(
|
|
112
|
+
"INSERT INTO rbac_memberships "
|
|
113
|
+
"(profile_id, user_id, role, added_at) "
|
|
114
|
+
"SELECT profile_id, user_id, role, added_at "
|
|
115
|
+
"FROM _rbac_memberships_old"
|
|
116
|
+
)
|
|
117
|
+
# Dropping the old table also drops its index, freeing the name.
|
|
118
|
+
conn.execute("DROP TABLE _rbac_memberships_old")
|
|
119
|
+
conn.execute(
|
|
120
|
+
"CREATE INDEX IF NOT EXISTS idx_rbac_memberships_user "
|
|
121
|
+
"ON rbac_memberships(user_id)"
|
|
122
|
+
)
|
|
123
|
+
conn.execute("COMMIT")
|
|
124
|
+
except sqlite3.Error:
|
|
125
|
+
try:
|
|
126
|
+
conn.execute("ROLLBACK")
|
|
127
|
+
except sqlite3.Error: # pragma: no cover — best-effort
|
|
128
|
+
pass
|
|
129
|
+
raise
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def verify(conn: sqlite3.Connection) -> bool:
|
|
133
|
+
"""Applied once rbac_memberships carries the profiles FK (or is absent)."""
|
|
134
|
+
if not _table_exists(conn, "rbac_memberships"):
|
|
135
|
+
return True # nothing to migrate; fresh install rebuilds it on apply
|
|
136
|
+
return _has_profiles_fk(conn, "rbac_memberships")
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
|
|
2
|
+
# Licensed under AGPL-3.0-or-later - see LICENSE file
|
|
3
|
+
# Part of SuperLocalMemory V3 — isolation correctness (H-01, cycle-3 audit)
|
|
4
|
+
|
|
5
|
+
"""M027 — add profile_id to transferable_patterns + rebuild UNIQUE constraint (learning.db).
|
|
6
|
+
|
|
7
|
+
H-01: ``transferable_patterns`` had no ``profile_id`` column; every profile's
|
|
8
|
+
consolidation wrote to the same (pattern_type, key) row. The last writer won,
|
|
9
|
+
so any profile's ``get_preferences()`` call returned patterns contaminated by
|
|
10
|
+
other profiles' consolidation runs.
|
|
11
|
+
|
|
12
|
+
Fix:
|
|
13
|
+
1. Add ``profile_id TEXT NOT NULL DEFAULT 'default'`` to the table.
|
|
14
|
+
2. Change the UNIQUE constraint from ``(pattern_type, key)`` to
|
|
15
|
+
``(profile_id, pattern_type, key)`` so each profile owns its own rows.
|
|
16
|
+
3. Backfill existing rows to ``profile_id = 'default'`` (the only valid value
|
|
17
|
+
before this migration — they are legacy/unattributed patterns).
|
|
18
|
+
|
|
19
|
+
SQLite cannot ALTER a UNIQUE constraint in place; the table must be rebuilt
|
|
20
|
+
(rename → create canonical form → copy → drop old). Wrapped in
|
|
21
|
+
``BEGIN IMMEDIATE / COMMIT`` for atomicity.
|
|
22
|
+
|
|
23
|
+
Idempotent:
|
|
24
|
+
* If ``profile_id`` already exists with the correct UNIQUE constraint
|
|
25
|
+
(fresh install after the code change), ``apply()`` is a no-op.
|
|
26
|
+
* If the table is absent entirely (CrossProjectAggregator was never run
|
|
27
|
+
against learning.db), ``apply()`` is a no-op — the new _SCHEMA in
|
|
28
|
+
cross_project.py will create it in its canonical form on first use.
|
|
29
|
+
|
|
30
|
+
DB target: ``learning`` — CrossProjectAggregator is under the learning/
|
|
31
|
+
package and is expected to share the learning database.
|
|
32
|
+
|
|
33
|
+
Author: Varun Pratap Bhardwaj / Qualixar
|
|
34
|
+
"""
|
|
35
|
+
|
|
36
|
+
from __future__ import annotations
|
|
37
|
+
|
|
38
|
+
import sqlite3
|
|
39
|
+
|
|
40
|
+
NAME = "M027_transferable_patterns_profile"
|
|
41
|
+
DB_TARGET = "learning"
|
|
42
|
+
|
|
43
|
+
DDL = (
|
|
44
|
+
"-- rebuild transferable_patterns with "
|
|
45
|
+
"profile_id TEXT NOT NULL DEFAULT 'default' "
|
|
46
|
+
"and UNIQUE(profile_id, pattern_type, key)"
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
_NEW_TABLE = """
|
|
50
|
+
CREATE TABLE transferable_patterns (
|
|
51
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
52
|
+
profile_id TEXT NOT NULL DEFAULT 'default',
|
|
53
|
+
pattern_type TEXT NOT NULL DEFAULT 'preference',
|
|
54
|
+
key TEXT NOT NULL,
|
|
55
|
+
value TEXT NOT NULL,
|
|
56
|
+
confidence REAL DEFAULT 0.0,
|
|
57
|
+
evidence_count INTEGER DEFAULT 0,
|
|
58
|
+
profiles_seen INTEGER DEFAULT 1,
|
|
59
|
+
decay_factor REAL DEFAULT 1.0,
|
|
60
|
+
contradictions TEXT DEFAULT '[]',
|
|
61
|
+
first_seen TEXT,
|
|
62
|
+
last_seen TEXT,
|
|
63
|
+
UNIQUE(profile_id, pattern_type, key)
|
|
64
|
+
)
|
|
65
|
+
"""
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _table_exists(conn: sqlite3.Connection, table: str) -> bool:
|
|
69
|
+
return conn.execute(
|
|
70
|
+
"SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (table,)
|
|
71
|
+
).fetchone() is not None
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _cols(conn: sqlite3.Connection, table: str) -> set[str]:
|
|
75
|
+
return {r[1] for r in conn.execute(f"PRAGMA table_info({table})").fetchall()}
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _unique_index_covers_profile(conn: sqlite3.Connection) -> bool:
|
|
79
|
+
"""True when transferable_patterns already has the UNIQUE(profile_id, …) constraint."""
|
|
80
|
+
indexes = conn.execute(
|
|
81
|
+
"SELECT name, sql FROM sqlite_master "
|
|
82
|
+
"WHERE type='index' AND tbl_name='transferable_patterns'"
|
|
83
|
+
).fetchall()
|
|
84
|
+
for idx in indexes:
|
|
85
|
+
sql = (idx[1] or "").lower()
|
|
86
|
+
if "profile_id" in sql and "pattern_type" in sql and "key" in sql:
|
|
87
|
+
return True
|
|
88
|
+
# SQLite stores the UNIQUE constraint on the CREATE TABLE statement as well.
|
|
89
|
+
tbl_sql = conn.execute(
|
|
90
|
+
"SELECT sql FROM sqlite_master "
|
|
91
|
+
"WHERE type='table' AND name='transferable_patterns'"
|
|
92
|
+
).fetchone()
|
|
93
|
+
if tbl_sql:
|
|
94
|
+
sql = (tbl_sql[0] or "").lower()
|
|
95
|
+
if "unique(profile_id" in sql or "unique (profile_id" in sql:
|
|
96
|
+
return True
|
|
97
|
+
return False
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def apply(conn: sqlite3.Connection) -> None:
|
|
101
|
+
"""Rebuild transferable_patterns with profile_id column and correct UNIQUE constraint.
|
|
102
|
+
|
|
103
|
+
No-op when the table is absent or already in canonical form.
|
|
104
|
+
Existing data is preserved; legacy rows get profile_id = 'default'.
|
|
105
|
+
"""
|
|
106
|
+
if not _table_exists(conn, "transferable_patterns"):
|
|
107
|
+
return # Table was never created against this DB — fresh install handles it.
|
|
108
|
+
|
|
109
|
+
if (
|
|
110
|
+
"profile_id" in _cols(conn, "transferable_patterns")
|
|
111
|
+
and _unique_index_covers_profile(conn)
|
|
112
|
+
):
|
|
113
|
+
return # Already in canonical form.
|
|
114
|
+
|
|
115
|
+
has_profile_col = "profile_id" in _cols(conn, "transferable_patterns")
|
|
116
|
+
|
|
117
|
+
conn.execute("BEGIN IMMEDIATE")
|
|
118
|
+
try:
|
|
119
|
+
conn.execute(
|
|
120
|
+
"ALTER TABLE transferable_patterns RENAME TO _tp_old"
|
|
121
|
+
)
|
|
122
|
+
conn.execute(_NEW_TABLE)
|
|
123
|
+
|
|
124
|
+
if has_profile_col:
|
|
125
|
+
# Column exists but UNIQUE constraint is wrong — copy as-is.
|
|
126
|
+
conn.execute(
|
|
127
|
+
"INSERT INTO transferable_patterns "
|
|
128
|
+
"(profile_id, pattern_type, key, value, confidence, evidence_count, "
|
|
129
|
+
" profiles_seen, decay_factor, contradictions, first_seen, last_seen) "
|
|
130
|
+
"SELECT profile_id, pattern_type, key, value, confidence, evidence_count, "
|
|
131
|
+
" profiles_seen, decay_factor, contradictions, first_seen, last_seen "
|
|
132
|
+
"FROM _tp_old"
|
|
133
|
+
)
|
|
134
|
+
else:
|
|
135
|
+
# No profile_id yet — backfill legacy rows to 'default'.
|
|
136
|
+
conn.execute(
|
|
137
|
+
"INSERT INTO transferable_patterns "
|
|
138
|
+
"(profile_id, pattern_type, key, value, confidence, evidence_count, "
|
|
139
|
+
" profiles_seen, decay_factor, contradictions, first_seen, last_seen) "
|
|
140
|
+
"SELECT 'default', pattern_type, key, value, confidence, evidence_count, "
|
|
141
|
+
" profiles_seen, decay_factor, contradictions, first_seen, last_seen "
|
|
142
|
+
"FROM _tp_old"
|
|
143
|
+
)
|
|
144
|
+
|
|
145
|
+
conn.execute("DROP TABLE _tp_old")
|
|
146
|
+
conn.execute("COMMIT")
|
|
147
|
+
except sqlite3.Error:
|
|
148
|
+
try:
|
|
149
|
+
conn.execute("ROLLBACK")
|
|
150
|
+
except sqlite3.Error:
|
|
151
|
+
pass
|
|
152
|
+
raise
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def verify(conn: sqlite3.Connection) -> bool:
|
|
156
|
+
"""True when transferable_patterns has profile_id with the correct UNIQUE,
|
|
157
|
+
or when the table is absent (fresh install will create it correctly).
|
|
158
|
+
"""
|
|
159
|
+
if not _table_exists(conn, "transferable_patterns"):
|
|
160
|
+
return True # absent — first run of _ensure_schema creates canonical form
|
|
161
|
+
if "profile_id" not in _cols(conn, "transferable_patterns"):
|
|
162
|
+
return False
|
|
163
|
+
return _unique_index_covers_profile(conn)
|
|
@@ -0,0 +1,270 @@
|
|
|
1
|
+
# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
|
|
2
|
+
# Licensed under AGPL-3.0-or-later
|
|
3
|
+
|
|
4
|
+
"""M028 — normalized fact/entity associations for O(1) ingest effects.
|
|
5
|
+
|
|
6
|
+
The table is both an association index and an idempotency ledger. Its
|
|
7
|
+
composite primary key means a fact can contribute to an entity's ``fact_count``
|
|
8
|
+
once, regardless of ingestion retries or how many operations converge on the
|
|
9
|
+
same consolidated fact.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import json
|
|
15
|
+
import sqlite3
|
|
16
|
+
from datetime import UTC, datetime
|
|
17
|
+
from pathlib import Path
|
|
18
|
+
|
|
19
|
+
NAME = "M028_fact_entity_associations"
|
|
20
|
+
DB_TARGET = "memory"
|
|
21
|
+
|
|
22
|
+
DDL = """
|
|
23
|
+
CREATE TABLE IF NOT EXISTS fact_entity_associations (
|
|
24
|
+
profile_id TEXT NOT NULL,
|
|
25
|
+
fact_id TEXT NOT NULL,
|
|
26
|
+
entity_id TEXT NOT NULL,
|
|
27
|
+
first_operation_id TEXT NOT NULL DEFAULT '',
|
|
28
|
+
count_applied INTEGER NOT NULL DEFAULT 0
|
|
29
|
+
CHECK (count_applied IN (0, 1)),
|
|
30
|
+
created_at TEXT NOT NULL DEFAULT (
|
|
31
|
+
strftime('%Y-%m-%dT%H:%M:%fZ', 'now')
|
|
32
|
+
),
|
|
33
|
+
PRIMARY KEY (profile_id, fact_id, entity_id),
|
|
34
|
+
FOREIGN KEY (fact_id) REFERENCES atomic_facts(fact_id) ON DELETE CASCADE,
|
|
35
|
+
FOREIGN KEY (entity_id)
|
|
36
|
+
REFERENCES canonical_entities(entity_id) ON DELETE CASCADE
|
|
37
|
+
);
|
|
38
|
+
CREATE INDEX IF NOT EXISTS idx_fact_entity_associations_entity
|
|
39
|
+
ON fact_entity_associations(profile_id, entity_id, fact_id);
|
|
40
|
+
CREATE TABLE IF NOT EXISTS fact_entity_association_repair_state (
|
|
41
|
+
repair_key TEXT PRIMARY KEY,
|
|
42
|
+
state TEXT NOT NULL DEFAULT 'pending'
|
|
43
|
+
CHECK (state IN ('pending', 'running', 'retrying', 'complete')),
|
|
44
|
+
target_fact_rowid INTEGER NOT NULL DEFAULT -1,
|
|
45
|
+
last_fact_rowid INTEGER NOT NULL DEFAULT 0,
|
|
46
|
+
scanned INTEGER NOT NULL DEFAULT 0,
|
|
47
|
+
inserted INTEGER NOT NULL DEFAULT 0,
|
|
48
|
+
last_error TEXT NOT NULL DEFAULT '',
|
|
49
|
+
updated_at TEXT NOT NULL
|
|
50
|
+
);
|
|
51
|
+
"""
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _table_exists(conn: sqlite3.Connection, table: str) -> bool:
|
|
55
|
+
return conn.execute(
|
|
56
|
+
"SELECT 1 FROM sqlite_master WHERE type='table' AND name=?",
|
|
57
|
+
(table,),
|
|
58
|
+
).fetchone() is not None
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def apply(conn: sqlite3.Connection) -> None:
|
|
62
|
+
"""Install schema and capture a constant-time historical rowid boundary."""
|
|
63
|
+
conn.executescript(DDL)
|
|
64
|
+
association_columns = {
|
|
65
|
+
row[1]
|
|
66
|
+
for row in conn.execute(
|
|
67
|
+
"PRAGMA table_info(fact_entity_associations)"
|
|
68
|
+
).fetchall()
|
|
69
|
+
}
|
|
70
|
+
if "count_applied" not in association_columns:
|
|
71
|
+
conn.execute(
|
|
72
|
+
"ALTER TABLE fact_entity_associations "
|
|
73
|
+
"ADD COLUMN count_applied INTEGER NOT NULL DEFAULT 0 "
|
|
74
|
+
"CHECK (count_applied IN (0, 1))"
|
|
75
|
+
)
|
|
76
|
+
repair_columns = {
|
|
77
|
+
row[1]
|
|
78
|
+
for row in conn.execute(
|
|
79
|
+
"PRAGMA table_info(fact_entity_association_repair_state)"
|
|
80
|
+
).fetchall()
|
|
81
|
+
}
|
|
82
|
+
if "target_fact_rowid" not in repair_columns:
|
|
83
|
+
conn.execute(
|
|
84
|
+
"ALTER TABLE fact_entity_association_repair_state "
|
|
85
|
+
"ADD COLUMN target_fact_rowid INTEGER NOT NULL DEFAULT -1"
|
|
86
|
+
)
|
|
87
|
+
target = int(conn.execute(
|
|
88
|
+
"SELECT COALESCE(MAX(rowid), 0) FROM atomic_facts"
|
|
89
|
+
).fetchone()[0])
|
|
90
|
+
conn.executescript(
|
|
91
|
+
"INSERT OR IGNORE INTO fact_entity_association_repair_state "
|
|
92
|
+
"(repair_key,state,target_fact_rowid,updated_at) "
|
|
93
|
+
f"VALUES ('historical-backfill','pending',{target},'');"
|
|
94
|
+
"UPDATE fact_entity_association_repair_state "
|
|
95
|
+
f"SET target_fact_rowid={target} "
|
|
96
|
+
"WHERE repair_key='historical-backfill' AND target_fact_rowid < 0;"
|
|
97
|
+
)
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def _now() -> str:
|
|
101
|
+
return datetime.now(UTC).isoformat()
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _connect(db_path: Path) -> sqlite3.Connection:
|
|
105
|
+
conn = sqlite3.connect(str(db_path), timeout=5)
|
|
106
|
+
conn.row_factory = sqlite3.Row
|
|
107
|
+
conn.execute("PRAGMA busy_timeout=5000")
|
|
108
|
+
conn.execute("PRAGMA foreign_keys=ON")
|
|
109
|
+
return conn
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def get_repair_status(db_path: Path) -> dict[str, int | str]:
|
|
113
|
+
"""Read durable backfill progress without inferring it from schema."""
|
|
114
|
+
conn = _connect(Path(db_path))
|
|
115
|
+
try:
|
|
116
|
+
row = conn.execute(
|
|
117
|
+
"SELECT state,target_fact_rowid,last_fact_rowid,scanned,inserted,"
|
|
118
|
+
"last_error,updated_at "
|
|
119
|
+
"FROM fact_entity_association_repair_state "
|
|
120
|
+
"WHERE repair_key='historical-backfill'"
|
|
121
|
+
).fetchone()
|
|
122
|
+
if row is None:
|
|
123
|
+
return {
|
|
124
|
+
"state": "pending", "target_fact_rowid": 0,
|
|
125
|
+
"last_fact_rowid": 0,
|
|
126
|
+
"scanned": 0, "inserted": 0,
|
|
127
|
+
"last_error": "", "updated_at": "",
|
|
128
|
+
}
|
|
129
|
+
return dict(row)
|
|
130
|
+
finally:
|
|
131
|
+
conn.close()
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def _entity_ids(raw: object) -> tuple[str, ...]:
|
|
135
|
+
try:
|
|
136
|
+
values = json.loads(str(raw or "[]"))
|
|
137
|
+
except (TypeError, ValueError):
|
|
138
|
+
return ()
|
|
139
|
+
if not isinstance(values, list):
|
|
140
|
+
return ()
|
|
141
|
+
return tuple(dict.fromkeys(str(value) for value in values if value))
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def _repair_batch(conn: sqlite3.Connection, batch_size: int) -> dict[str, int | bool]:
|
|
145
|
+
conn.execute("BEGIN IMMEDIATE")
|
|
146
|
+
try:
|
|
147
|
+
status = conn.execute(
|
|
148
|
+
"SELECT last_fact_rowid,target_fact_rowid "
|
|
149
|
+
"FROM fact_entity_association_repair_state "
|
|
150
|
+
"WHERE repair_key='historical-backfill'"
|
|
151
|
+
).fetchone()
|
|
152
|
+
cursor = int(status["last_fact_rowid"] or 0)
|
|
153
|
+
target = int(status["target_fact_rowid"])
|
|
154
|
+
rows = conn.execute(
|
|
155
|
+
"SELECT rowid,fact_id,profile_id,canonical_entities_json "
|
|
156
|
+
"FROM atomic_facts WHERE rowid>? AND rowid<=? "
|
|
157
|
+
"ORDER BY rowid LIMIT ?",
|
|
158
|
+
(cursor, target, batch_size),
|
|
159
|
+
).fetchall()
|
|
160
|
+
if not rows:
|
|
161
|
+
conn.execute(
|
|
162
|
+
"UPDATE fact_entity_association_repair_state "
|
|
163
|
+
"SET state='complete',last_error='',updated_at=? "
|
|
164
|
+
"WHERE repair_key='historical-backfill'",
|
|
165
|
+
(_now(),),
|
|
166
|
+
)
|
|
167
|
+
conn.commit()
|
|
168
|
+
return {"scanned": 0, "inserted": 0, "complete": True}
|
|
169
|
+
inserted = 0
|
|
170
|
+
for row in rows:
|
|
171
|
+
for entity_id in _entity_ids(row["canonical_entities_json"]):
|
|
172
|
+
result = conn.execute(
|
|
173
|
+
"INSERT OR IGNORE INTO fact_entity_associations "
|
|
174
|
+
"(profile_id,fact_id,entity_id,first_operation_id,"
|
|
175
|
+
"count_applied) "
|
|
176
|
+
"SELECT ?,?,?,?,? FROM canonical_entities "
|
|
177
|
+
"WHERE profile_id=? AND entity_id=?",
|
|
178
|
+
(
|
|
179
|
+
row["profile_id"], row["fact_id"], entity_id,
|
|
180
|
+
"migration-backfill", 0,
|
|
181
|
+
row["profile_id"], entity_id,
|
|
182
|
+
),
|
|
183
|
+
)
|
|
184
|
+
inserted += max(0, result.rowcount)
|
|
185
|
+
conn.execute(
|
|
186
|
+
"UPDATE fact_entity_association_repair_state SET "
|
|
187
|
+
"state='running',last_fact_rowid=?,scanned=scanned+?,"
|
|
188
|
+
"inserted=inserted+?,last_error='',updated_at=? "
|
|
189
|
+
"WHERE repair_key='historical-backfill'",
|
|
190
|
+
(int(rows[-1]["rowid"]), len(rows), inserted, _now()),
|
|
191
|
+
)
|
|
192
|
+
conn.commit()
|
|
193
|
+
return {"scanned": len(rows), "inserted": inserted, "complete": False}
|
|
194
|
+
except Exception:
|
|
195
|
+
conn.rollback()
|
|
196
|
+
raise
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def repair_fact_entity_associations(
|
|
200
|
+
db_path: Path,
|
|
201
|
+
*,
|
|
202
|
+
batch_size: int = 250,
|
|
203
|
+
max_batches: int = 1,
|
|
204
|
+
) -> dict[str, int | bool]:
|
|
205
|
+
"""Run bounded, restartable short-transaction backfill batches."""
|
|
206
|
+
if batch_size < 1 or max_batches < 1:
|
|
207
|
+
raise ValueError("batch_size and max_batches must be positive")
|
|
208
|
+
totals = {"scanned": 0, "inserted": 0, "complete": False}
|
|
209
|
+
conn = _connect(Path(db_path))
|
|
210
|
+
try:
|
|
211
|
+
for _ in range(max_batches):
|
|
212
|
+
result = _repair_batch(conn, batch_size)
|
|
213
|
+
totals["scanned"] += int(result["scanned"])
|
|
214
|
+
totals["inserted"] += int(result["inserted"])
|
|
215
|
+
totals["complete"] = bool(result["complete"])
|
|
216
|
+
if totals["complete"]:
|
|
217
|
+
break
|
|
218
|
+
return totals
|
|
219
|
+
except sqlite3.Error as exc:
|
|
220
|
+
try:
|
|
221
|
+
conn.execute(
|
|
222
|
+
"UPDATE fact_entity_association_repair_state SET "
|
|
223
|
+
"state='retrying',last_error=?,updated_at=? "
|
|
224
|
+
"WHERE repair_key='historical-backfill'",
|
|
225
|
+
(type(exc).__name__, _now()),
|
|
226
|
+
)
|
|
227
|
+
conn.commit()
|
|
228
|
+
except sqlite3.Error:
|
|
229
|
+
pass
|
|
230
|
+
raise
|
|
231
|
+
finally:
|
|
232
|
+
conn.close()
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
def verify(conn: sqlite3.Connection) -> bool:
|
|
236
|
+
"""Return true when the normalized association contract is indexed."""
|
|
237
|
+
if not (
|
|
238
|
+
_table_exists(conn, "fact_entity_associations")
|
|
239
|
+
and _table_exists(conn, "fact_entity_association_repair_state")
|
|
240
|
+
):
|
|
241
|
+
return False
|
|
242
|
+
association_columns = {
|
|
243
|
+
row[1]
|
|
244
|
+
for row in conn.execute(
|
|
245
|
+
"PRAGMA table_info(fact_entity_associations)"
|
|
246
|
+
).fetchall()
|
|
247
|
+
}
|
|
248
|
+
repair_columns = {
|
|
249
|
+
row[1]
|
|
250
|
+
for row in conn.execute(
|
|
251
|
+
"PRAGMA table_info(fact_entity_association_repair_state)"
|
|
252
|
+
).fetchall()
|
|
253
|
+
}
|
|
254
|
+
if (
|
|
255
|
+
"count_applied" not in association_columns
|
|
256
|
+
or "target_fact_rowid" not in repair_columns
|
|
257
|
+
):
|
|
258
|
+
return False
|
|
259
|
+
indexes = {
|
|
260
|
+
row[1]
|
|
261
|
+
for row in conn.execute(
|
|
262
|
+
"PRAGMA index_list(fact_entity_associations)"
|
|
263
|
+
).fetchall()
|
|
264
|
+
}
|
|
265
|
+
return "idx_fact_entity_associations_entity" in indexes
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
def repair(conn: sqlite3.Connection) -> None:
|
|
269
|
+
"""Restore additive M028 schema without recapturing its high-water mark."""
|
|
270
|
+
apply(conn)
|